From 83ee0af9bf74c56bfa32901f9c8bad7fda4bd3e6 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Wed, 12 Apr 2023 13:47:51 +0800 Subject: [PATCH 001/439] chore: fixed go dependencies moderate security issues (#2529) --- go.mod | 6 +- go.sum | 319 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 311 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index ce66694f2..1b2ec0e5f 100644 --- a/go.mod +++ b/go.mod @@ -132,7 +132,7 @@ require ( github.com/cloudflare/circl v1.1.0 // indirect github.com/cockroachdb/apd/v2 v2.0.1 // indirect github.com/containerd/cgroups v1.0.4 // indirect - github.com/containerd/containerd v1.6.15 // indirect + github.com/containerd/containerd v1.6.18 // indirect github.com/containers/image/v5 v5.24.0 // indirect github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect github.com/containers/ocicrypt v1.1.7 // indirect @@ -204,7 +204,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-getter v1.6.2 // indirect + github.com/hashicorp/go-getter v1.7.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect @@ -249,7 +249,7 @@ require ( github.com/mistifyio/go-zfs/v3 v3.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/go-testing-interface v1.14.0 // indirect + github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect diff --git a/go.sum b/go.sum index 1989c2b82..b1c5304f0 100644 --- a/go.sum +++ b/go.sum @@ -20,35 +20,173 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= contrib.go.opencensus.io/exporter/prometheus v0.4.1 h1:oObVeKo2NxpdF/fIfrPsNj6K0Prg0R0mHM+uANlYMiM= contrib.go.opencensus.io/exporter/prometheus v0.4.1/go.mod h1:t9wvfitlUjGXG2IXAZsuFq26mDGid/JwCEXp+gTG/9U= cuelang.org/go v0.4.3 h1:W3oBBjDTm7+IZfCKZAmC8uDG0eYfJL4Pp/xbbCMKaVo= @@ -200,7 +338,6 @@ github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:W github.com/authzed/controller-idioms v0.7.0 h1:HhNMUBb8hJzYqY3mhen3B2AC5nsIem3fBe0tC/AAOHo= github.com/authzed/controller-idioms v0.7.0/go.mod h1:0B/PmqCguKv8b3azSMF+HdyKpKr2o3UAZ5eo12Ze8Fo= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= -github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -346,8 +483,8 @@ github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09Zvgq github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.6.15 h1:4wWexxzLNHNE46aIETc6ge4TofO550v+BlLoANrbses= -github.com/containerd/containerd v1.6.15/go.mod h1:U2NnBPIhzJDm59xF7xB2MMHnKtggpZ+phKg8o2TKj2c= +github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns= +github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -739,6 +876,7 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= @@ -797,6 +935,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -808,6 +947,9 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -817,13 +959,24 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= @@ -874,8 +1027,8 @@ github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.6.2 h1:7jX7xcB+uVCliddZgeKyNxv0xoT7qL5KDtH7rU4IqIk= -github.com/hashicorp/go-getter v1.6.2/go.mod h1:IZCrswsZPeWv9IkVnLElzRU/gz/QPi6pZHn4tv6vbwA= +github.com/hashicorp/go-getter v1.7.0 h1:bzrYP+qu/gMrL1au7/aDvkoOVGUJpeKBgbqRHACAFDY= +github.com/hashicorp/go-getter v1.7.0/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v1.3.1 h1:vDwF1DFNZhntP4DAjuTpOw3uEgMUpXh1pB5fW9DqHpo= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -892,7 +1045,6 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -1011,12 +1163,12 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= @@ -1159,8 +1311,8 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.14.0 h1:/x0XQ6h+3U3nAyk1yx+bHPURrKa9sVVvYbuqZ7pIAtI= -github.com/mitchellh/go-testing-interface v1.14.0/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= @@ -1553,7 +1705,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= @@ -1861,6 +2013,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1869,9 +2022,18 @@ golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= @@ -1889,8 +2051,19 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1904,7 +2077,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2000,27 +2175,43 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2030,6 +2221,7 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= @@ -2139,6 +2331,9 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -2148,6 +2343,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= @@ -2175,6 +2373,33 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.107.0 h1:I2SlFjD8ZWabaIFOfeEDg3pf0BHJDh6iYQ1ic3Yu/UU= google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -2230,10 +2455,69 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20230104163317-caabf589fcbf h1:/JqRexUvugu6JURQ0O7RfV1EnvgrOxUV4tSjuAv0Sr0= google.golang.org/genproto v0.0.0-20230104163317-caabf589fcbf/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -2260,14 +2544,27 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From f126115c805b538ac010385f91271cf14f3a49fc Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Wed, 12 Apr 2023 17:17:41 +0800 Subject: [PATCH 002/439] feat: update pg14/15 parameters constraint (#2160) (#2322) --- .../apps/configuration/sync_upgrade_policy.go | 6 +- .../config/pg14-config-constraint.cue | 66 +++++++++++++++---- .../config/pg14-config-effect-scope.yaml | 42 +++++++++++- .../templates/configconstraint.yaml | 6 ++ 4 files changed, 103 insertions(+), 17 deletions(-) diff --git a/controllers/apps/configuration/sync_upgrade_policy.go b/controllers/apps/configuration/sync_upgrade_policy.go index 6cfed37b7..bbf8c98f9 100644 --- a/controllers/apps/configuration/sync_upgrade_policy.go +++ b/controllers/apps/configuration/sync_upgrade_policy.go @@ -17,6 +17,8 @@ limitations under the License. package configuration import ( + "fmt" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -104,13 +106,13 @@ func sync(params reconfigureParams, updatedParameters map[string]string, pods [] return makeReturnedStatus(ESAndRetryFailed), err } if len(pods) == 0 { - params.Ctx.Log.Info("no pods to update, and retry, selector: %v, current all pod: %v", params.ConfigConstraint.Selector) + params.Ctx.Log.Info(fmt.Sprintf("no pods to update, and retry, selector: %s", params.ConfigConstraint.Selector.String())) return makeReturnedStatus(ESRetry), nil } requireUpdatedCount := int32(len(pods)) for _, pod := range pods { - params.Ctx.Log.V(1).Info("sync pod: %s", pod.Name) + params.Ctx.Log.V(1).Info(fmt.Sprintf("sync pod: %s", pod.Name)) if podutil.IsMatchConfigVersion(&pod, configKey, versionHash) { progress++ continue diff --git a/deploy/postgresql/config/pg14-config-constraint.cue b/deploy/postgresql/config/pg14-config-constraint.cue index 31f7f924e..1d735c145 100644 --- a/deploy/postgresql/config/pg14-config-constraint.cue +++ b/deploy/postgresql/config/pg14-config-constraint.cue @@ -14,18 +14,26 @@ // PostgreSQL parameters: https://postgresqlco.nf/doc/en/param/ #PGParameter: { + // Allows tablespaces directly inside pg_tblspc, for testing, pg version: 15 + allow_in_place_tablespaces?: bool + // Allows modification of the structure of system tables as well as certain other risky actions on system tables. This is otherwise not allowed even for superusers. Ill-advised use of this setting can cause irretrievable data loss or seriously corrupt the database system. + allow_system_table_mods?: bool // Sets the application name to be reported in statistics and logs. application_name?: string // Sets the shell command that will be called to archive a WAL file. archive_command?: string + // The library to use for archiving completed WAL file segments. If set to an empty string (the default), archiving via shell is enabled, and archive_command is used. Otherwise, the specified shared library is used for archiving. The WAL archiver process is restarted by the postmaster when this parameter changes. For more information, see backup-archiving-wal and archive-modules. + archive_library?: string + // When archive_mode is enabled, completed WAL segments are sent to archive storage by setting archive_command or guc-archive-library. In addition to off, to disable, there are two modes: on, and always. During normal operation, there is no difference between the two modes, but when set to always the WAL archiver is enabled also during archive recovery or standby mode. In always mode, all files restored from the archive or streamed with streaming replication will be archived (again). See continuous-archiving-in-standby for details. + archive_mode: string & "always" | "on" | "off" // (s) Forces a switch to the next xlog file if a new file has not been started within N seconds. archive_timeout: int & >=0 & <=2147483647 | *300 @timeDurationResource(1s) // Enable input of NULL elements in arrays. - array_nulls?: bool & false | true + array_nulls?: bool // (s) Sets the maximum allowed time to complete client authentication. - authentication_timeout?: int & >=1 & <=600 @timeDurationResource() + authentication_timeout?: int & >=1 & <=600 @timeDurationResource(1s) // Use EXPLAIN ANALYZE for plan logging. - "auto_explain.log_analyze"?: bool & false | true + "auto_explain.log_analyze"?: bool // Log buffers usage. "auto_explain.log_buffers"?: bool & false | true // EXPLAIN format to be used for plan logging. @@ -50,7 +58,7 @@ "auto_explain.sample_rate"?: float & >=0 & <=1 // Starts the autovacuum subprocess. - autovacuum?: bool & false | true + autovacuum?: bool // Number of tuple inserts, updates or deletes prior to analyze as a fraction of reltuples. autovacuum_analyze_scale_factor: float & >=0 & <=100 | *0.05 @@ -59,7 +67,7 @@ autovacuum_analyze_threshold?: int & >=0 & <=2147483647 // Age at which to autovacuum a table to prevent transaction ID wraparound. - autovacuum_freeze_max_age?: int & >=100000000 & <=750000000 + autovacuum_freeze_max_age?: int & >=100000 & <=2000000000 // Sets the maximum number of simultaneously running autovacuum worker processes. autovacuum_max_workers?: int & >=1 & <=8388607 @@ -71,7 +79,7 @@ autovacuum_naptime: int & >=1 & <=2147483 | *15 @timeDurationResource(1s) // (ms) Vacuum cost delay in milliseconds, for autovacuum. - autovacuum_vacuum_cost_delay?: int & >=-1 & <=100 + autovacuum_vacuum_cost_delay?: int & >=-1 & <=100 @timeDurationResource() // Vacuum cost amount available before napping, for autovacuum. autovacuum_vacuum_cost_limit?: int & >=-1 & <=10000 @@ -92,9 +100,9 @@ autovacuum_work_mem?: int & >=-1 & <=2147483647 @storeResource(1KB) // (8Kb) Number of pages after which previously performed writes are flushed to disk. - backend_flush_after?: int & >=0 & <=256 + backend_flush_after?: int & >=0 & <=256 @storeResource(8KB) - // Sets whether \ is allowed in string literals. + // Sets whether "\" is allowed in string literals. backslash_quote?: string & "safe_encoding" | "on" | "off" // Log backtrace for errors in these functions. @@ -104,7 +112,7 @@ bgwriter_delay?: int & >=10 & <=10000 @timeDurationResource() // (8Kb) Number of pages after which previously performed writes are flushed to disk. - bgwriter_flush_after?: int & >=0 & <=256 + bgwriter_flush_after?: int & >=0 & <=256 @storeResource(8KB) // Background writer maximum number of LRU pages to flush per round. bgwriter_lru_maxpages?: int & >=0 & <=1000 @@ -352,6 +360,9 @@ // Use of huge pages on Linux. huge_pages?: string & "on" | "off" | "try" + // The size of huge page that should be requested. Controls the size of huge pages, when they are enabled with huge_pages. The default is zero (0). When set to 0, the default huge page size on the system will be used. This parameter can only be set at server start. + huge_page_size?: int & >=0 & <=2147483647 @storeResource(1KB) + // Sets the servers ident configuration file. ident_file?: string @@ -368,7 +379,7 @@ intervalstyle?: string & "postgres" | "postgres_verbose" | "sql_standard" | "iso_8601" // Allow JIT compilation. - jit: bool & false | true | *false + jit: bool // Perform JIT compilation if query is more expensive. jit_above_cost?: float & >=-1 & <=1.79769e+308 @@ -484,6 +495,9 @@ // (kB) Automatic log file rotation will occur after N kilobytes. log_rotation_size?: int & >=0 & <=2097151 @storeResource(1KB) + // Time between progress updates for long-running startup operations. Sets the amount of time after which the startup process will log a message about a long-running operation that is still in progress, as well as the interval between further progress messages for that operation. The default is 10 seconds. A setting of 0 disables the feature. If this value is specified without units, it is taken as milliseconds. This setting is applied separately to each operation. This parameter can only be set in the postgresql.conf file or on the server command line. + log_startup_progress_interval: int & >=0 & <=2147483647 @timeDurationResource() + // Sets the type of statements logged. log_statement?: string & "none" | "ddl" | "mod" | "all" @@ -491,7 +505,7 @@ log_statement_sample_rate?: float & >=0 & <=1 // Writes cumulative performance statistics to the server log. - log_statement_stats?: bool & false | true + log_statement_stats?: bool // (kB) Log the use of temporary files larger than this number of kilobytes. log_temp_files?: int & >=-1 & <=2147483647 @storeResource(1KB) @@ -577,6 +591,9 @@ // (8kB) Sets the minimum amount of index data for a parallel scan. min_parallel_index_scan_size?: int & >=0 & <=715827882 @storeResource(8KB) + // Sets the minimum size of relations to be considered for parallel scan. Sets the minimum size of relations to be considered for parallel scan. + min_parallel_relation_size?: int & >=0 & <=715827882 @storeResource(8KB) + // (8kB) Sets the minimum amount of table data for a parallel scan. min_parallel_table_scan_size?: int & >=0 & <=715827882 @storeResource(8KB) @@ -584,7 +601,7 @@ min_wal_size: int & >=128 & <=201326592 | *192 @storeResource(1MB) // (min) Time before a snapshot is too old to read pages changed after the snapshot was taken. - old_snapshot_threshold?: int & >=-1 & <=86400 + old_snapshot_threshold?: int & >=-1 & <=86400 @timeDurationResource(1min) // Emulate oracle's date output behaviour. "orafce.nls_date_format"?: string @@ -838,6 +855,12 @@ // Sets the TCP port the server listens on. port?: int & >=1 & <=65535 + // Sets the amount of time to wait after authentication on connection startup. The amount of time to delay when a new server process is started, after it conducts the authentication procedure. This is intended to give developers an opportunity to attach to the server process with a debugger. If this value is specified without units, it is taken as seconds. A value of zero (the default) disables the delay. This parameter cannot be changed after session start. + post_auth_delay?: int & >=0 & <=2147 @timeDurationResource(1s) + + // Sets the amount of time to wait before authentication on connection startup. The amount of time to delay just after a new server process is forked, before it conducts the authentication procedure. This is intended to give developers an opportunity to attach to the server process with a debugger to trace down misbehavior in authentication. If this value is specified without units, it is taken as seconds. A value of zero (the default) disables the delay. This parameter can only be set in the postgresql.conf file or on the server command line. + pre_auth_delay?: int & >=0 & <=60 @timeDurationResource(1s) + // Enable for disable GDAL drivers used with PostGIS in Postgres 9.3.5 and above. "postgis.gdal_enabled_drivers"?: string & "ENABLE_ALL" | "DISABLE_ALL" @@ -938,11 +961,14 @@ // (s) Time between TCP keepalive retransmits. tcp_keepalives_interval?: int & >=0 & <=2147483647 @timeDurationResource(1s) + // TCP user timeout. Specifies the amount of time that transmitted data may remain unacknowledged before the TCP connection is forcibly closed. If this value is specified without units, it is taken as milliseconds. A value of 0 (the default) selects the operating system's default. This parameter is supported only on systems that support TCP_USER_TIMEOUT; on other systems, it must be zero. In sessions connected via a Unix-domain socket, this parameter is ignored and always reads as zero. + tcp_user_timeout?: int & >=0 & <=2147483647 @timeDurationResource() + // (8kB) Sets the maximum number of temporary buffers used by each session. temp_buffers?: int & >=100 & <=1073741823 @storeResource(8KB) // (kB) Limits the total size of all temporary files used by each process. - temp_file_limit?: int & >=-1 & <=2147483647 @storeResource(8KB) + temp_file_limit?: int & >=-1 & <=2147483647 @storeResource(1KB) // Sets the tablespace(s) to use for temporary tables and sort files. temp_tablespaces?: string @@ -954,7 +980,7 @@ track_activities?: bool & false | true // Sets the size reserved for pg_stat_activity.current_query, in bytes. - track_activity_query_size: int & >=100 & <=1048576 | *4096 + track_activity_query_size: int & >=100 & <=1048576 | *4096 @storeResource() // Collects transaction commit time. track_commit_timestamp?: bool & false | true @@ -1028,6 +1054,12 @@ // Compresses full-page writes written in WAL file. wal_compression: bool & false | true | *true + // Sets the WAL resource managers for which WAL consistency checks are done. + wal_consistency_checking?: string + + // Buffer size for reading ahead in the WAL during recovery. + wal_decode_buffer_size: int & >=65536 & <=1073741823 | *524288 @storeResource() + // (MB) Sets the size of WAL files held for standby servers. wal_keep_size: int & >=0 & <=2147483647 | *2048 @storeResource(1MB) @@ -1040,6 +1072,12 @@ // (ms) Sets the maximum wait time to receive data from the primary. wal_receiver_timeout: int & >=0 & <=3600000 | *30000 @timeDurationResource() + // Recycles WAL files by renaming them. If set to on (the default), this option causes WAL files to be recycled by renaming them, avoiding the need to create new ones. On COW file systems, it may be faster to create new ones, so the option is given to disable this behavior. + wal_recycle?: bool + + // Sets the time to wait before retrying to retrieve WAL after a failed attempt. Specifies how long the standby server should wait when WAL data is not available from any sources (streaming replication, local pg_wal or WAL archive) before trying again to retrieve WAL data. If this value is specified without units, it is taken as milliseconds. The default value is 5 seconds. This parameter can only be set in the postgresql.conf file or on the server command line. + wal_retrieve_retry_interval: int & >=1 & <=2147483647 | *5000 @timeDurationResource() + // (ms) Sets the maximum time to wait for WAL replication. wal_sender_timeout: int & >=0 & <=3600000 | *30000 @timeDurationResource() diff --git a/deploy/postgresql/config/pg14-config-effect-scope.yaml b/deploy/postgresql/config/pg14-config-effect-scope.yaml index 6e4d69046..e103dfaa9 100644 --- a/deploy/postgresql/config/pg14-config-effect-scope.yaml +++ b/deploy/postgresql/config/pg14-config-effect-scope.yaml @@ -19,4 +19,44 @@ staticParameters: - bg_mon.history_buckets - pg_stat_statements.track_utility - extwlist.extensions - - extwlist.custom_path \ No newline at end of file + - extwlist.custom_path + +immutableParameters: + - archive_command + - archive_timeout + - backtrace_functions + - config_file + - cron.use_background_workers + - data_directory + - db_user_namespace + - exit_on_error + - fsync + - full_page_writes + - hba_file + - ident_file + - ignore_invalid_pages + - listen_addresses + - lo_compat_privileges + - log_directory + - log_file_mode + - logging_collector + - log_line_prefix + - log_timezone + - log_truncate_on_rotation + - port + - rds.max_tcp_buffers + - recovery_init_sync_method + - restart_after_crash + - ssl + - ssl_ca_file + - ssl_cert_file + - ssl_ciphers + - ssl_key_file + - stats_temp_directory + - superuser_reserved_connections + - unix_socket_directories + - unix_socket_group + - unix_socket_permissions + - update_process_title + - wal_receiver_create_temp_slot + - wal_sync_method diff --git a/deploy/postgresql/templates/configconstraint.yaml b/deploy/postgresql/templates/configconstraint.yaml index d5da741fa..97c1787e4 100644 --- a/deploy/postgresql/templates/configconstraint.yaml +++ b/deploy/postgresql/templates/configconstraint.yaml @@ -8,9 +8,15 @@ metadata: spec: reloadOptions: tplScriptTrigger: + sync: true scriptConfigMapRef: patroni-reload-script namespace: {{ .Release.Namespace }} + # update patroni master + selector: + matchLabels: + "apps.kubeblocks.postgres.patroni/role": "master" + # top level mysql configuration type cfgSchemaTopLevelName: PGParameter From 19cf83657940e57e41861c476063a20472c8a97d Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 12 Apr 2023 17:27:45 +0800 Subject: [PATCH 003/439] feat: improve backupPolicy api and add backupPolicyTemplate api in apps group. (#2460) --- .../v1alpha1/backuppolicytemplate_types.go | 240 + apis/apps/v1alpha1/clusterdefinition_types.go | 6 +- apis/apps/v1alpha1/type.go | 10 + apis/dataprotection/v1alpha1/backup_types.go | 27 +- .../v1alpha1/backup_types_test.go | 40 + .../v1alpha1/backuppolicy_types.go | 149 +- .../v1alpha1/backuppolicytemplate_types.go | 104 - apis/dataprotection/v1alpha1/types.go | 22 +- ...s.kubeblocks.io_backuppolicytemplates.yaml | 412 ++ ...apps.kubeblocks.io_clusterdefinitions.yaml | 18 +- ...otection.kubeblocks.io_backuppolicies.yaml | 5096 ++++++++++++----- ...n.kubeblocks.io_backuppolicytemplates.yaml | 144 - .../dataprotection.kubeblocks.io_backups.yaml | 4 - ...aprotection.kubeblocks.io_restorejobs.yaml | 18 +- config/crd/kustomization.yaml | 2 +- ...ection_in_apps_backuppolicytemplates.yaml} | 2 +- ...ebhook_in_apps_backuppolicytemplates.yaml} | 2 +- ...pps_backuppolicytemplate_editor_role.yaml} | 4 +- ...pps_backuppolicytemplate_viewer_role.yaml} | 4 +- config/rbac/role.yaml | 33 +- controllers/apps/cluster_controller.go | 8 +- controllers/apps/cluster_controller_test.go | 51 +- .../apps/opsrequest_controller_test.go | 6 +- controllers/apps/systemaccount_controller.go | 6 - controllers/apps/systemaccount_util.go | 4 +- .../dataprotection/backup_controller.go | 295 +- .../dataprotection/backup_controller_test.go | 24 +- .../dataprotection/backuppolicy_controller.go | 542 +- .../backuppolicy_controller_test.go | 259 +- .../dataprotection/cronjob_controller.go | 2 +- controllers/dataprotection/cue/cronjob.cue | 20 +- .../dataprotection/restorejob_controller.go | 21 +- .../restorejob_controller_test.go | 122 +- controllers/dataprotection/suite_test.go | 1 - controllers/dataprotection/type.go | 20 + .../templates/backuppolicytemplate.yaml | 39 +- .../templates/backuptool.yaml | 28 +- .../templates/clusterdefinition.yaml | 3 +- .../templates/backuppolicytemplate.yaml | 37 +- .../apecloud-mysql/templates/backuptool.yaml | 2 +- .../templates/clusterdefinition.yaml | 3 +- deploy/helm/config/rbac/role.yaml | 33 +- ...s.kubeblocks.io_backuppolicytemplates.yaml | 412 ++ ...apps.kubeblocks.io_clusterdefinitions.yaml | 18 +- ...otection.kubeblocks.io_backuppolicies.yaml | 5096 ++++++++++++----- ...n.kubeblocks.io_backuppolicytemplates.yaml | 144 - .../dataprotection.kubeblocks.io_backups.yaml | 4 - ...aprotection.kubeblocks.io_restorejobs.yaml | 18 +- deploy/helm/templates/deployment.yaml | 8 - ...pps_backuppolicytemplate_editor_role.yaml} | 4 +- ...pps_backuppolicytemplate_viewer_role.yaml} | 4 +- deploy/helm/values.yaml | 10 - .../templates/backuppolicytemplate.yaml | 25 +- .../templates/backuppolicytemplate.yaml | 37 +- .../templates/backuppolicytemplate.yaml | 25 +- .../redis/templates/backuppolicytemplate.yaml | 22 +- .../templates/backuppolicytemplate.yaml | 25 +- docs/release_notes/v0.5.0/v0.5.0.md | 8 + docs/user_docs/cli/cli.md | 4 +- docs/user_docs/cli/kbcli_backup-config.md | 6 - docs/user_docs/cli/kbcli_cluster.md | 4 +- docs/user_docs/cli/kbcli_cluster_backup.md | 9 +- .../cli/kbcli_cluster_edit-backup-policy.md | 72 + .../cli/kbcli_cluster_list-backup-policy.md | 60 + .../cli/kbcli_cluster_list-backup.md | 58 + .../cli/cmd/backupconfig/backup_config.go | 6 - internal/cli/cmd/cluster/cluster.go | 2 + internal/cli/cmd/cluster/dataprotection.go | 251 +- .../cli/cmd/cluster/dataprotection_test.go | 89 +- internal/cli/cmd/kubeblocks/status.go | 1 - .../cli/create/template/backup_template.cue | 2 - .../create/template/backuppolicy_template.cue | 57 - internal/cli/edit/edit.go | 72 + internal/cli/edit/edit_test.go | 75 + internal/cli/edit/suite_test.go | 29 + internal/cli/testing/fake.go | 16 +- internal/cli/types/types.go | 6 +- internal/cli/util/util_test.go | 2 +- internal/constant/const.go | 22 +- internal/controller/builder/builder.go | 16 - internal/controller/builder/builder_test.go | 23 - .../builder/cue/backup_policy_template.cue | 70 - .../builder/cue/conn_credential_template.cue | 2 +- internal/controller/component/component.go | 1 + .../component/restore_utils_test.go | 3 - internal/controller/component/type.go | 1 + .../lifecycle/cluster_plan_builder.go | 2 + .../controller/lifecycle/transform_utils.go | 30 +- .../transformer_backup_policy_tpl.go | 313 + .../lifecycle/transformer_object_action.go | 2 + .../transformer_sts_horizontal_scaling.go | 54 +- ...transformer_sts_horizontal_scaling_test.go | 2 +- internal/controllerutil/errors.go | 22 +- internal/generics/type.go | 2 +- internal/testutil/apps/backup_factory.go | 15 - .../testutil/apps/backuppolicy_factory.go | 159 +- .../apps/backuppolicytemplate_factory.go | 174 +- internal/testutil/apps/restorejob_factory.go | 3 +- test/integration/backup_mysql_test.go | 16 +- 99 files changed, 10302 insertions(+), 5174 deletions(-) create mode 100644 apis/apps/v1alpha1/backuppolicytemplate_types.go create mode 100644 apis/dataprotection/v1alpha1/backup_types_test.go delete mode 100644 apis/dataprotection/v1alpha1/backuppolicytemplate_types.go create mode 100644 config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml delete mode 100644 config/crd/bases/dataprotection.kubeblocks.io_backuppolicytemplates.yaml rename config/crd/patches/{cainjection_in_dataprotection_backuppolicytemplates.yaml => cainjection_in_apps_backuppolicytemplates.yaml} (81%) rename config/crd/patches/{webhook_in_dataprotection_backuppolicytemplates.yaml => webhook_in_apps_backuppolicytemplates.yaml} (85%) rename config/rbac/{dataprotection_backuppolicytemplate_editor_role.yaml => apps_backuppolicytemplate_editor_role.yaml} (85%) rename config/rbac/{dataprotection_backuppolicytemplate_viewer_role.yaml => apps_backuppolicytemplate_viewer_role.yaml} (83%) create mode 100644 deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml delete mode 100644 deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicytemplates.yaml rename deploy/helm/templates/rbac/{dataprotection_backuppolicytemplate_editor_role.yaml => apps_backuppolicytemplate_editor_role.yaml} (88%) rename deploy/helm/templates/rbac/{dataprotection_backuppolicytemplate_viewer_role.yaml => apps_backuppolicytemplate_viewer_role.yaml} (86%) create mode 100644 docs/user_docs/cli/kbcli_cluster_edit-backup-policy.md create mode 100644 docs/user_docs/cli/kbcli_cluster_list-backup-policy.md create mode 100644 docs/user_docs/cli/kbcli_cluster_list-backup.md delete mode 100644 internal/cli/create/template/backuppolicy_template.cue create mode 100644 internal/cli/edit/edit.go create mode 100644 internal/cli/edit/edit_test.go create mode 100644 internal/cli/edit/suite_test.go delete mode 100644 internal/controller/builder/cue/backup_policy_template.cue create mode 100644 internal/controller/lifecycle/transformer_backup_policy_tpl.go diff --git a/apis/apps/v1alpha1/backuppolicytemplate_types.go b/apis/apps/v1alpha1/backuppolicytemplate_types.go new file mode 100644 index 000000000..3ccf5fa9a --- /dev/null +++ b/apis/apps/v1alpha1/backuppolicytemplate_types.go @@ -0,0 +1,240 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// BackupPolicyTemplateSpec defines the desired state of BackupPolicyTemplate +type BackupPolicyTemplateSpec struct { + // clusterDefinitionRef references ClusterDefinition name, this is an immutable attribute. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + ClusterDefRef string `json:"clusterDefinitionRef"` + + // backupPolicies is a list of backup policy template for the specified componentDefinition. + // +patchMergeKey=componentDefRef + // +patchStrategy=merge,retainKeys + // +listType=map + // +listMapKey=componentDefRef + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinItems=1 + BackupPolicies []BackupPolicy `json:"backupPolicies"` +} + +type BackupPolicy struct { + // componentDefRef references componentDef defined in ClusterDefinition spec. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + ComponentDefRef string `json:"componentDefRef"` + + // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long + // the Backup should be retained. if not set, will be retained forever. + // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` + // +optional + TTL *string `json:"ttl,omitempty"` + + // schedule policy for backup. + // +optional + Schedule Schedule `json:"schedule,omitempty"` + + // the policy for snapshot backup. + // +optional + Snapshot *SnapshotPolicy `json:"snapshot,omitempty"` + + // the policy for full backup. + // +optional + Full *CommonBackupPolicy `json:"full,omitempty"` + + // the policy for incremental backup. + // +optional + Incremental *CommonBackupPolicy `json:"incremental,omitempty"` +} + +type Schedule struct { + // schedule policy for base backup. + // +optional + BaseBackup *BaseBackupSchedulePolicy `json:"baseBackup,omitempty"` + + // schedule policy for incremental backup. + // +optional + Incremental *SchedulePolicy `json:"incremental,omitempty"` +} + +type BaseBackupSchedulePolicy struct { + SchedulePolicy `json:",inline"` + // the type of base backup, only support full and snapshot. + // +kubebuilder:validation:Required + Type BaseBackupType `json:"type"` +} + +type SchedulePolicy struct { + // the cron expression for schedule, the timezone is in UTC. see https://en.wikipedia.org/wiki/Cron. + // +kubebuilder:validation:Required + CronExpression string `json:"cronExpression"` + + // enable or disable the schedule. + // +kubebuilder:validation:Required + Enable bool `json:"enable"` +} + +type SnapshotPolicy struct { + BasePolicy `json:",inline"` + + // execute hook commands for backup. + // +optional + Hooks *BackupPolicyHook `json:"hooks,omitempty"` +} + +type CommonBackupPolicy struct { + BasePolicy `json:",inline"` + + // which backup tool to perform database backup, only support one tool. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + BackupToolName string `json:"backupToolName,omitempty"` +} + +type BasePolicy struct { + // target instance for backup. + // +optional + Target TargetInstance `json:"target"` + + // the number of automatic backups to retain. Value must be non-negative integer. + // 0 means NO limit on the number of backups. + // +kubebuilder:default=7 + // +optional + BackupsHistoryLimit int32 `json:"backupsHistoryLimit,omitempty"` + + // count of backup stop retries on fail. + // +optional + OnFailAttempted int32 `json:"onFailAttempted,omitempty"` + + // define how to update metadata for backup status. + // +optional + BackupStatusUpdates []BackupStatusUpdate `json:"backupStatusUpdates,omitempty"` +} + +type TargetInstance struct { + // select instance of corresponding role for backup, role are: + // - the name of Leader/Follower/Leaner for Consensus component. + // - primary or secondary for Replication component. + // finally, invalid role of the component will be ignored. + // such as if workload type is Replication and component's replicas is 1, + // the secondary role is invalid. and it also will be ignored when component is Stateful/Stateless. + // the role will be transformed to a role LabelSelector for BackupPolicy's target attribute. + // +optional + Role string `json:"role"` + + // refer to spec.componentDef.systemAccounts.accounts[*].name in ClusterDefinition. + // the secret created by this account will be used to connect the database. + // if not set, the secret created by spec.ConnectionCredential of the ClusterDefinition will be used. + // it will be transformed to a secret for BackupPolicy's target secret. + // +optional + Account string `json:"account,omitempty"` + + // connectionCredentialKey defines connection credential key in secret + // which created by spec.ConnectionCredential of the ClusterDefinition. + // it will be ignored when "account" is set. + ConnectionCredentialKey ConnectionCredentialKey `json:"connectionCredentialKey,omitempty"` +} + +type ConnectionCredentialKey struct { + // the key of password in the ConnectionCredential secret. + // if not set, the default key is "password". + // +optional + PasswordKey *string `json:"passwordKey,omitempty"` + + // the key of username in the ConnectionCredential secret. + // if not set, the default key is "username". + // +optional + UsernameKey *string `json:"usernameKey,omitempty"` +} + +// BackupPolicyHook defines for the database execute commands before and after backup. +type BackupPolicyHook struct { + // pre backup to perform commands + // +optional + PreCommands []string `json:"preCommands,omitempty"` + + // post backup to perform commands + // +optional + PostCommands []string `json:"postCommands,omitempty"` + + // exec command with image + // +optional + Image string `json:"image,omitempty"` + + // which container can exec command + // +optional + ContainerName string `json:"containerName,omitempty"` +} + +type BackupStatusUpdate struct { + // specify the json path of backup object for patch. + // example: manifests.backupLog -- means patch the backup json path of status.manifests.backupLog. + // +optional + Path string `json:"path,omitempty"` + + // which container name that kubectl can execute. + // +optional + ContainerName string `json:"containerName,omitempty"` + + // the shell Script commands to collect backup status metadata. + // The script must exist in the container of ContainerName and the output format must be set to JSON. + // Note that outputting to stderr may cause the result format to not be in JSON. + // +optional + Script string `json:"script,omitempty"` + + // when to update the backup status, pre: before backup, post: after backup + // +optional + UpdateStage BackupStatusUpdateStage `json:"updateStage,omitempty"` +} + +// BackupPolicyTemplateStatus defines the observed state of BackupPolicyTemplate +type BackupPolicyTemplateStatus struct { +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:categories={kubeblocks},scope=Cluster,shortName=bpt +// +kubebuilder:printcolumn:name="CLUSTER-DEFINITION",type="string",JSONPath=".spec.clusterDefinitionRef",description="ClusterDefinition referenced by cluster." +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" + +// BackupPolicyTemplate is the Schema for the BackupPolicyTemplates API (defined by provider) +type BackupPolicyTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BackupPolicyTemplateSpec `json:"spec,omitempty"` + Status BackupPolicyTemplateStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// BackupPolicyTemplateList contains a list of BackupPolicyTemplate +type BackupPolicyTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BackupPolicyTemplate `json:"items"` +} + +func init() { + SchemeBuilder.Register(&BackupPolicyTemplate{}, &BackupPolicyTemplateList{}) +} diff --git a/apis/apps/v1alpha1/clusterdefinition_types.go b/apis/apps/v1alpha1/clusterdefinition_types.go index e9bd2ce22..88ed96c75 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types.go +++ b/apis/apps/v1alpha1/clusterdefinition_types.go @@ -481,7 +481,7 @@ type HorizontalScalePolicy struct { // Policy is in enum of {None, Snapshot}. The default policy is `None`. // None: Default policy, do nothing. // Snapshot: Do native volume snapshot before scaling and restore to newly scaled pods. - // Prefer backup job to create snapshot if `BackupTemplateSelector` can find a template. + // Prefer backup job to create snapshot if can find a backupPolicy from 'BackupPolicyTemplateName'. // Notice that 'Snapshot' policy will only take snapshot on one volumeMount, default is // the first volumeMount of first container (i.e. clusterdefinition.spec.components.podSpec.containers[0].volumeMounts[0]), // since take multiple snapshots at one time might cause consistency problem. @@ -489,9 +489,9 @@ type HorizontalScalePolicy struct { // +optional Type HScaleDataClonePolicyType `json:"type,omitempty"` - // backupTemplateSelector defines the label selector for finding associated BackupTemplate API object. + // BackupPolicyTemplateName reference the backup policy template. // +optional - BackupTemplateSelector map[string]string `json:"backupTemplateSelector,omitempty"` + BackupPolicyTemplateName string `json:"backupPolicyTemplateName,omitempty"` // volumeMountsName defines which volumeMount of the container to do backup, // only work if Type is not None diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index 8ff8b48c2..dce0c9c8d 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -442,6 +442,16 @@ const ( VolumeTypeLog VolumeType = "log" ) +// BaseBackupType the base backup type, keep synchronized with the BaseBackupType of the data protection API. +// +enum +// +kubebuilder:validation:Enum={full,snapshot} +type BaseBackupType string + +// BackupStatusUpdateStage defines the stage of backup status update. +// +enum +// +kubebuilder:validation:Enum={pre,post} +type BackupStatusUpdateStage string + func RegisterWebhookManager(mgr manager.Manager) { webhookMgr = &webhookManager{mgr.GetClient()} } diff --git a/apis/dataprotection/v1alpha1/backup_types.go b/apis/dataprotection/v1alpha1/backup_types.go index 7fe18d89d..a855ca849 100644 --- a/apis/dataprotection/v1alpha1/backup_types.go +++ b/apis/dataprotection/v1alpha1/backup_types.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha1 import ( + "fmt" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -35,11 +37,6 @@ type BackupSpec struct { // if backupType is incremental, parentBackupName is required. // +optional ParentBackupName string `json:"parentBackupName,omitempty"` - - // ttl is a time.Duration-parsable string describing how long - // the Backup should be retained for. - // +optional - TTL *metav1.Duration `json:"ttl,omitempty"` } // BackupStatus defines the observed state of Backup @@ -190,3 +187,23 @@ type BackupList struct { func init() { SchemeBuilder.Register(&Backup{}, &BackupList{}) } + +// Validate validates the BackupSpec and returns an error if invalid. +func (r *BackupSpec) Validate(backupPolicy *BackupPolicy) error { + notSupportedMessage := "backupPolicy: %s not supports %s backup in backupPolicy" + switch r.BackupType { + case BackupTypeSnapshot: + if backupPolicy.Spec.Snapshot == nil { + return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeSnapshot) + } + case BackupTypeFull: + if backupPolicy.Spec.Full == nil { + return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeFull) + } + case BackupTypeIncremental: + if backupPolicy.Spec.Incremental == nil { + return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeIncremental) + } + } + return nil +} diff --git a/apis/dataprotection/v1alpha1/backup_types_test.go b/apis/dataprotection/v1alpha1/backup_types_test.go new file mode 100644 index 000000000..f78ed10c4 --- /dev/null +++ b/apis/dataprotection/v1alpha1/backup_types_test.go @@ -0,0 +1,40 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + "time" +) + +func expectToDuration(t *testing.T, ttl string, baseNum, targetNum int) { + d := ToDuration(&ttl) + if d != time.Hour*time.Duration(baseNum)*time.Duration(targetNum) { + t.Errorf(`Expected duration is "%d*%d*time.Hour"", got %v`, targetNum, baseNum, d) + } +} + +func TestToDuration(t *testing.T) { + d := ToDuration(nil) + if d != time.Duration(0) { + t.Errorf("Expected duration is 0, got %v", d) + } + expectToDuration(t, "7d", 24, 7) + expectToDuration(t, "7D", 24, 7) + expectToDuration(t, "12h", 1, 12) + expectToDuration(t, "12H", 1, 12) +} diff --git a/apis/dataprotection/v1alpha1/backuppolicy_types.go b/apis/dataprotection/v1alpha1/backuppolicy_types.go index e3d2df55b..931cc0bbb 100644 --- a/apis/dataprotection/v1alpha1/backuppolicy_types.go +++ b/apis/dataprotection/v1alpha1/backuppolicy_types.go @@ -17,55 +17,98 @@ limitations under the License. package v1alpha1 import ( + "strconv" + "strings" + "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // BackupPolicySpec defines the desired state of BackupPolicy type BackupPolicySpec struct { - // policy can inherit from backup config and override some fields. - // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long + // the Backup should be retained. if not set, will be retained forever. + // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` // +optional - BackupPolicyTemplateName string `json:"backupPolicyTemplateName,omitempty"` + TTL *string `json:"ttl,omitempty"` - // The schedule in Cron format, the timezone is in UTC. see https://en.wikipedia.org/wiki/Cron. + // schedule policy for backup. // +optional - Schedule string `json:"schedule,omitempty"` + Schedule Schedule `json:"schedule,omitempty"` - // Backup ComponentDefRef. full or incremental or snapshot. if unset, default is snapshot. - // +kubebuilder:validation:Enum={full,incremental,snapshot} - // +kubebuilder:default=snapshot + // the policy for snapshot backup. // +optional - BackupType string `json:"backupType,omitempty"` + Snapshot *SnapshotPolicy `json:"snapshot,omitempty"` - // The number of automatic backups to retain. Value must be non-negative integer. - // 0 means NO limit on the number of backups. - // +kubebuilder:default=7 + // the policy for full backup. // +optional - BackupsHistoryLimit int32 `json:"backupsHistoryLimit,omitempty"` + Full *CommonBackupPolicy `json:"full,omitempty"` - // which backup tool to perform database backup, only support one tool. - // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + // the policy for incremental backup. // +optional - BackupToolName string `json:"backupToolName,omitempty"` + Incremental *CommonBackupPolicy `json:"incremental,omitempty"` +} - // TTL is a time.Duration-parseable string describing how long - // the Backup should be retained for. +type Schedule struct { + // schedule policy for base backup. // +optional - TTL *metav1.Duration `json:"ttl,omitempty"` + BaseBackup *BaseBackupSchedulePolicy `json:"baseBackup,omitempty"` + + // schedule policy for incremental backup. + // +optional + Incremental *SchedulePolicy `json:"incremental,omitempty"` +} - // database cluster service +type BaseBackupSchedulePolicy struct { + SchedulePolicy `json:",inline"` + // the type of base backup, only support full and snapshot. // +kubebuilder:validation:Required - Target TargetCluster `json:"target"` + Type BaseBackupType `json:"type"` +} + +type SchedulePolicy struct { + // the cron expression for schedule, the timezone is in UTC. see https://en.wikipedia.org/wiki/Cron. + // +kubebuilder:validation:Required + CronExpression string `json:"cronExpression"` + + // enable or disable the schedule. + // +kubebuilder:validation:Required + Enable bool `json:"enable"` +} + +type SnapshotPolicy struct { + BasePolicy `json:",inline"` // execute hook commands for backup. // +optional Hooks *BackupPolicyHook `json:"hooks,omitempty"` +} + +type CommonBackupPolicy struct { + BasePolicy `json:",inline"` // array of remote volumes from CSI driver definition. // +kubebuilder:validation:Required RemoteVolume corev1.Volume `json:"remoteVolume"` + // which backup tool to perform database backup, only support one tool. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + BackupToolName string `json:"backupToolName,omitempty"` +} + +type BasePolicy struct { + // target database cluster for backup. + // +kubebuilder:validation:Required + Target TargetCluster `json:"target"` + + // the number of automatic backups to retain. Value must be non-negative integer. + // 0 means NO limit on the number of backups. + // +kubebuilder:default=7 + // +optional + BackupsHistoryLimit int32 `json:"backupsHistoryLimit,omitempty"` + // count of backup stop retries on fail. // +optional OnFailAttempted int32 `json:"onFailAttempted,omitempty"` @@ -77,37 +120,39 @@ type BackupPolicySpec struct { // TargetCluster TODO (dsj): target cluster need redefined from Cluster API type TargetCluster struct { - // LabelSelector is used to find matching pods. + // labelsSelector is used to find matching pods. // Pods that match this label selector are counted to determine the number of pods // in their corresponding topology domain. // +kubebuilder:validation:Required // +kubebuilder:pruning:PreserveUnknownFields LabelsSelector *metav1.LabelSelector `json:"labelsSelector"` - // Secret is used to connect to the target database cluster. + // secret is used to connect to the target database cluster. // If not set, secret will be inherited from backup policy template. // if still not set, the controller will check if any system account for dataprotection has been created. // +optional Secret *BackupPolicySecret `json:"secret,omitempty"` } -// BackupPolicySecret defined for the target database secret that backup tool can connect. +// BackupPolicySecret defines for the target database secret that backup tool can connect. type BackupPolicySecret struct { // the secret name // +kubebuilder:validation:Required // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` Name string `json:"name"` - // UserKeyword the map keyword of the user in the connection credential secret - // +optional - UserKeyword string `json:"userKeyword,omitempty"` + // usernameKey the map key of the user in the connection credential secret + // +kubebuilder:validation:Required + // +kubebuilder:default=username + UsernameKey string `json:"usernameKey,omitempty"` - // PasswordKeyword the map keyword of the password in the connection credential secret - // +optional - PasswordKeyword string `json:"passwordKeyword,omitempty"` + // passwordKey the map key of the password in the connection credential secret + // +kubebuilder:validation:Required + // +kubebuilder:default=password + PasswordKey string `json:"passwordKey,omitempty"` } -// BackupPolicyHook defined for the database execute commands before and after backup. +// BackupPolicyHook defines for the database execute commands before and after backup. type BackupPolicyHook struct { // pre backup to perform commands // +optional @@ -159,28 +204,34 @@ type BackupStatusUpdate struct { // BackupPolicyStatus defines the observed state of BackupPolicy type BackupPolicyStatus struct { - // backup policy phase valid value: available, failed, new. + + // observedGeneration is the most recent generation observed for this + // BackupPolicy. It corresponds to the Cluster's generation, which is + // updated on mutation by the API Server. // +optional - Phase BackupPolicyTemplatePhase `json:"phase,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // backup policy phase valid value: Available, Failed. + // +optional + Phase BackupPolicyPhase `json:"phase,omitempty"` // the reason if backup policy check failed. // +optional FailureReason string `json:"failureReason,omitempty"` - // Information when was the last time the job was successfully scheduled. + // information when was the last time the job was successfully scheduled. // +optional LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"` - // Information when was the last time the job successfully completed. + // information when was the last time the job successfully completed. // +optional LastSuccessfulTime *metav1.Time `json:"lastSuccessfulTime,omitempty"` } // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:resource:categories={kubeblocks},scope=Namespaced +// +kubebuilder:resource:categories={kubeblocks},scope=Namespaced,shortName=bp // +kubebuilder:printcolumn:name="STATUS",type=string,JSONPath=`.status.phase` -// +kubebuilder:printcolumn:name="SCHEDULE",type=string,JSONPath=`.spec.schedule` // +kubebuilder:printcolumn:name="LAST SCHEDULE",type=string,JSONPath=`.status.lastScheduleTime` // +kubebuilder:printcolumn:name="AGE",type=date,JSONPath=`.metadata.creationTimestamp` @@ -205,3 +256,27 @@ type BackupPolicyList struct { func init() { SchemeBuilder.Register(&BackupPolicy{}, &BackupPolicyList{}) } + +func (r *BackupPolicySpec) GetCommonPolicy(backupType BackupType) *CommonBackupPolicy { + switch backupType { + case BackupTypeFull: + return r.Full + case BackupTypeIncremental: + return r.Incremental + } + return nil +} + +// ToDuration converts the ttl string to time.Duration. +func ToDuration(ttl *string) time.Duration { + if ttl == nil { + return time.Duration(0) + } + ttlLower := strings.ToLower(*ttl) + if strings.HasSuffix(ttlLower, "d") { + days, _ := strconv.Atoi(strings.ReplaceAll(ttlLower, "d", "")) + return time.Hour * 24 * time.Duration(days) + } + hours, _ := strconv.Atoi(strings.ReplaceAll(ttlLower, "h", "")) + return time.Hour * time.Duration(hours) +} diff --git a/apis/dataprotection/v1alpha1/backuppolicytemplate_types.go b/apis/dataprotection/v1alpha1/backuppolicytemplate_types.go deleted file mode 100644 index 988a34ac2..000000000 --- a/apis/dataprotection/v1alpha1/backuppolicytemplate_types.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright ApeCloud, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// BackupPolicyTemplateSpec defines the desired state of BackupPolicyTemplate -type BackupPolicyTemplateSpec struct { - // The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron. - // +optional - Schedule string `json:"schedule,omitempty"` - - // which backup tool to perform database backup, only support one tool. - // +kubebuilder:validation:Required - // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` - BackupToolName string `json:"backupToolName"` - - // TTL is a time.Duration-parseable string describing how long - // the Backup should be retained for. - // +optional - TTL *metav1.Duration `json:"ttl,omitempty"` - - // limit count of backup stop retries on fail. - // if unset, retry unlimit attempted. - // +optional - OnFailAttempted int32 `json:"onFailAttempted,omitempty"` - - // execute hook commands for backup. - // +optional - Hooks *BackupPolicyHook `json:"hooks,omitempty"` - - // CredentialKeyword determines backupTool connection credential keyword in secret. - // the backupTool gets the credentials according to the user and password keyword defined by secret - // +optional - CredentialKeyword *BackupPolicyCredentialKeyword `json:"credentialKeyword,omitempty"` - - // define how to update metadata for backup status. - // +optional - BackupStatusUpdates []BackupStatusUpdate `json:"backupStatusUpdates,omitempty"` -} - -// BackupPolicyCredentialKeyword defined for the target database secret that backup tool can connect. -type BackupPolicyCredentialKeyword struct { - // UserKeyword the map keyword of the user in the connection credential secret - // +kubebuilder:default=username - // +optional - UserKeyword string `json:"userKeyword,omitempty"` - - // PasswordKeyword the map keyword of the password in the connection credential secret - // +kubebuilder:default=password - // +optional - PasswordKeyword string `json:"passwordKeyword,omitempty"` -} - -// BackupPolicyTemplateStatus defines the observed state of BackupPolicyTemplate -type BackupPolicyTemplateStatus struct { - // +optional - Phase BackupPolicyTemplatePhase `json:"phase,omitempty"` - - // +optional - FailureReason string `json:"failureReason,omitempty"` -} - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:resource:categories={kubeblocks},scope=Cluster - -// BackupPolicyTemplate is the Schema for the BackupPolicyTemplates API (defined by provider) -type BackupPolicyTemplate struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec BackupPolicyTemplateSpec `json:"spec,omitempty"` - Status BackupPolicyTemplateStatus `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// BackupPolicyTemplateList contains a list of BackupPolicyTemplate -type BackupPolicyTemplateList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []BackupPolicyTemplate `json:"items"` -} - -func init() { - SchemeBuilder.Register(&BackupPolicyTemplate{}, &BackupPolicyTemplateList{}) -} diff --git a/apis/dataprotection/v1alpha1/types.go b/apis/dataprotection/v1alpha1/types.go index 70c784e95..4c459db01 100644 --- a/apis/dataprotection/v1alpha1/types.go +++ b/apis/dataprotection/v1alpha1/types.go @@ -39,16 +39,24 @@ const ( BackupTypeSnapshot BackupType = "snapshot" ) -// BackupPolicyTemplatePhase defines phases for BackupPolicyTemplate CR. +// BaseBackupType the base backup type. // +enum -// +kubebuilder:validation:Enum={New,Available,InProgress,Failed} -type BackupPolicyTemplatePhase string +// +kubebuilder:validation:Enum={full,snapshot} +type BaseBackupType string const ( - ConfigNew BackupPolicyTemplatePhase = "New" - ConfigAvailable BackupPolicyTemplatePhase = "Available" - ConfigInProgress BackupPolicyTemplatePhase = "InProgress" - ConfigFailed BackupPolicyTemplatePhase = "Failed" + BaseBackupTypeFull BaseBackupType = "full" + BaseBackupTypeSnapshot BaseBackupType = "snapshot" +) + +// BackupPolicyPhase defines phases for BackupPolicy CR. +// +enum +// +kubebuilder:validation:Enum={Available,Failed} +type BackupPolicyPhase string + +const ( + PolicyAvailable BackupPolicyPhase = "Available" + PolicyFailed BackupPolicyPhase = "Failed" ) // RestoreJobPhase The current phase. Valid values are New, InProgressPhy, InProgressLogic, Completed, Failed. diff --git a/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml b/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml new file mode 100644 index 000000000..e88cfd8e7 --- /dev/null +++ b/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml @@ -0,0 +1,412 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: backuppolicytemplates.apps.kubeblocks.io +spec: + group: apps.kubeblocks.io + names: + categories: + - kubeblocks + kind: BackupPolicyTemplate + listKind: BackupPolicyTemplateList + plural: backuppolicytemplates + shortNames: + - bpt + singular: backuppolicytemplate + scope: Cluster + versions: + - additionalPrinterColumns: + - description: ClusterDefinition referenced by cluster. + jsonPath: .spec.clusterDefinitionRef + name: CLUSTER-DEFINITION + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: BackupPolicyTemplate is the Schema for the BackupPolicyTemplates + API (defined by provider) + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupPolicyTemplateSpec defines the desired state of BackupPolicyTemplate + properties: + backupPolicies: + description: backupPolicies is a list of backup policy template for + the specified componentDefinition. + items: + properties: + componentDefRef: + description: componentDefRef references componentDef defined + in ClusterDefinition spec. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + full: + description: the policy for full backup. + properties: + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can + execute. + type: string + path: + description: 'specify the json path of backup object + for patch. example: manifests.backupLog -- means + patch the backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect + backup status metadata. The script must exist in + the container of ContainerName and the output format + must be set to JSON. Note that outputting to stderr + may cause the result format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: + before backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupToolName: + description: which backup tool to perform database backup, + only support one tool. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. + Value must be non-negative integer. 0 means NO limit on + the number of backups. + format: int32 + type: integer + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + target: + description: target instance for backup. + properties: + account: + description: refer to spec.componentDef.systemAccounts.accounts[*].name + in ClusterDefinition. the secret created by this account + will be used to connect the database. if not set, + the secret created by spec.ConnectionCredential of + the ClusterDefinition will be used. it will be transformed + to a secret for BackupPolicy's target secret. + type: string + connectionCredentialKey: + description: connectionCredentialKey defines connection + credential key in secret which created by spec.ConnectionCredential + of the ClusterDefinition. it will be ignored when + "account" is set. + properties: + passwordKey: + description: the key of password in the ConnectionCredential + secret. if not set, the default key is "password". + type: string + usernameKey: + description: the key of username in the ConnectionCredential + secret. if not set, the default key is "username". + type: string + type: object + role: + description: 'select instance of corresponding role + for backup, role are: - the name of Leader/Follower/Leaner + for Consensus component. - primary or secondary for + Replication component. finally, invalid role of the + component will be ignored. such as if workload type + is Replication and component''s replicas is 1, the + secondary role is invalid. and it also will be ignored + when component is Stateful/Stateless. the role will + be transformed to a role LabelSelector for BackupPolicy''s + target attribute.' + type: string + type: object + type: object + incremental: + description: the policy for incremental backup. + properties: + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can + execute. + type: string + path: + description: 'specify the json path of backup object + for patch. example: manifests.backupLog -- means + patch the backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect + backup status metadata. The script must exist in + the container of ContainerName and the output format + must be set to JSON. Note that outputting to stderr + may cause the result format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: + before backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupToolName: + description: which backup tool to perform database backup, + only support one tool. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. + Value must be non-negative integer. 0 means NO limit on + the number of backups. + format: int32 + type: integer + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + target: + description: target instance for backup. + properties: + account: + description: refer to spec.componentDef.systemAccounts.accounts[*].name + in ClusterDefinition. the secret created by this account + will be used to connect the database. if not set, + the secret created by spec.ConnectionCredential of + the ClusterDefinition will be used. it will be transformed + to a secret for BackupPolicy's target secret. + type: string + connectionCredentialKey: + description: connectionCredentialKey defines connection + credential key in secret which created by spec.ConnectionCredential + of the ClusterDefinition. it will be ignored when + "account" is set. + properties: + passwordKey: + description: the key of password in the ConnectionCredential + secret. if not set, the default key is "password". + type: string + usernameKey: + description: the key of username in the ConnectionCredential + secret. if not set, the default key is "username". + type: string + type: object + role: + description: 'select instance of corresponding role + for backup, role are: - the name of Leader/Follower/Leaner + for Consensus component. - primary or secondary for + Replication component. finally, invalid role of the + component will be ignored. such as if workload type + is Replication and component''s replicas is 1, the + secondary role is invalid. and it also will be ignored + when component is Stateful/Stateless. the role will + be transformed to a role LabelSelector for BackupPolicy''s + target attribute.' + type: string + type: object + type: object + schedule: + description: schedule policy for backup. + properties: + baseBackup: + description: schedule policy for base backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. + type: string + enable: + description: enable or disable the schedule. + type: boolean + type: + description: the type of base backup, only support full + and snapshot. + enum: + - full + - snapshot + type: string + required: + - cronExpression + - enable + - type + type: object + incremental: + description: schedule policy for incremental backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. + type: string + enable: + description: enable or disable the schedule. + type: boolean + required: + - cronExpression + - enable + type: object + type: object + snapshot: + description: the policy for snapshot backup. + properties: + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can + execute. + type: string + path: + description: 'specify the json path of backup object + for patch. example: manifests.backupLog -- means + patch the backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect + backup status metadata. The script must exist in + the container of ContainerName and the output format + must be set to JSON. Note that outputting to stderr + may cause the result format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: + before backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. + Value must be non-negative integer. 0 means NO limit on + the number of backups. + format: int32 + type: integer + hooks: + description: execute hook commands for backup. + properties: + containerName: + description: which container can exec command + type: string + image: + description: exec command with image + type: string + postCommands: + description: post backup to perform commands + items: + type: string + type: array + preCommands: + description: pre backup to perform commands + items: + type: string + type: array + type: object + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + target: + description: target instance for backup. + properties: + account: + description: refer to spec.componentDef.systemAccounts.accounts[*].name + in ClusterDefinition. the secret created by this account + will be used to connect the database. if not set, + the secret created by spec.ConnectionCredential of + the ClusterDefinition will be used. it will be transformed + to a secret for BackupPolicy's target secret. + type: string + connectionCredentialKey: + description: connectionCredentialKey defines connection + credential key in secret which created by spec.ConnectionCredential + of the ClusterDefinition. it will be ignored when + "account" is set. + properties: + passwordKey: + description: the key of password in the ConnectionCredential + secret. if not set, the default key is "password". + type: string + usernameKey: + description: the key of username in the ConnectionCredential + secret. if not set, the default key is "username". + type: string + type: object + role: + description: 'select instance of corresponding role + for backup, role are: - the name of Leader/Follower/Leaner + for Consensus component. - primary or secondary for + Replication component. finally, invalid role of the + component will be ignored. such as if workload type + is Replication and component''s replicas is 1, the + secondary role is invalid. and it also will be ignored + when component is Stateful/Stateless. the role will + be transformed to a role LabelSelector for BackupPolicy''s + target attribute.' + type: string + type: object + type: object + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + required: + - componentDefRef + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - componentDefRef + x-kubernetes-list-type: map + clusterDefinitionRef: + description: clusterDefinitionRef references ClusterDefinition name, + this is an immutable attribute. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + required: + - backupPolicies + - clusterDefinitionRef + type: object + status: + description: BackupPolicyTemplateStatus defines the observed state of + BackupPolicyTemplate + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml index 2916d11b2..d146df534 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml @@ -286,12 +286,10 @@ spec: description: horizontalScalePolicy controls the behavior of horizontal scale. properties: - backupTemplateSelector: - additionalProperties: - type: string - description: backupTemplateSelector defines the label selector - for finding associated BackupTemplate API object. - type: object + backupPolicyTemplateName: + description: BackupPolicyTemplateName reference the backup + policy template. + type: string type: default: None description: 'type controls what kind of data synchronization @@ -299,10 +297,10 @@ spec: Snapshot}. The default policy is `None`. None: Default policy, do nothing. Snapshot: Do native volume snapshot before scaling and restore to newly scaled pods. Prefer - backup job to create snapshot if `BackupTemplateSelector` - can find a template. Notice that ''Snapshot'' policy will - only take snapshot on one volumeMount, default is the - first volumeMount of first container (i.e. clusterdefinition.spec.components.podSpec.containers[0].volumeMounts[0]), + backup job to create snapshot if can find a backupPolicy + from ''BackupPolicyTemplateName''. Notice that ''Snapshot'' + policy will only take snapshot on one volumeMount, default + is the first volumeMount of first container (i.e. clusterdefinition.spec.components.podSpec.containers[0].volumeMounts[0]), since take multiple snapshots at one time might cause consistency problem.' enum: diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml index 6e359243d..870a98b38 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -14,6 +14,8 @@ spec: kind: BackupPolicy listKind: BackupPolicyList plural: backuppolicies + shortNames: + - bp singular: backuppolicy scope: Namespaced versions: @@ -21,9 +23,6 @@ spec: - jsonPath: .status.phase name: STATUS type: string - - jsonPath: .spec.schedule - name: SCHEDULE - type: string - jsonPath: .status.lastScheduleTime name: LAST SCHEDULE type: string @@ -51,1696 +50,3614 @@ spec: spec: description: BackupPolicySpec defines the desired state of BackupPolicy properties: - backupPolicyTemplateName: - description: policy can inherit from backup config and override some - fields. - pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ - type: string - backupStatusUpdates: - description: define how to update metadata for backup status. - items: - properties: - containerName: - description: which container name that kubectl can execute. - type: string - path: - description: 'specify the json path of backup object for patch. - example: manifests.backupLog -- means patch the backup json - path of status.manifests.backupLog.' - type: string - script: - description: the shell Script commands to collect backup status - metadata. The script must exist in the container of ContainerName - and the output format must be set to JSON. Note that outputting - to stderr may cause the result format to not be in JSON. - type: string - updateStage: - description: 'when to update the backup status, pre: before - backup, post: after backup' - enum: - - pre - - post - type: string - type: object - type: array - backupToolName: - description: which backup tool to perform database backup, only support - one tool. - pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ - type: string - backupType: - default: snapshot - description: Backup ComponentDefRef. full or incremental or snapshot. - if unset, default is snapshot. - enum: - - full - - incremental - - snapshot - type: string - backupsHistoryLimit: - default: 7 - description: The number of automatic backups to retain. Value must - be non-negative integer. 0 means NO limit on the number of backups. - format: int32 - type: integer - hooks: - description: execute hook commands for backup. + full: + description: the policy for full backup. properties: - containerName: - description: which container can exec command - type: string - image: - description: exec command with image - type: string - postCommands: - description: post backup to perform commands + backupStatusUpdates: + description: define how to update metadata for backup status. items: - type: string - type: array - preCommands: - description: pre backup to perform commands - items: - type: string + properties: + containerName: + description: which container name that kubectl can execute. + type: string + path: + description: 'specify the json path of backup object for + patch. example: manifests.backupLog -- means patch the + backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect backup + status metadata. The script must exist in the container + of ContainerName and the output format must be set to + JSON. Note that outputting to stderr may cause the result + format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: before + backup, post: after backup' + enum: + - pre + - post + type: string + type: object type: array - type: object - onFailAttempted: - description: count of backup stop retries on fail. - format: int32 - type: integer - remoteVolume: - description: array of remote volumes from CSI driver definition. - properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount by - volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent disk - resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount on - the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the - blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must be a - filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob - disks per storage account Dedicated: single blob disk per - storage account Managed: azure managed data disk (only - in managed availability set). defaults to shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host that - shares a pod's lifetime + backupToolName: + description: which backup tool to perform database backup, only + support one tool. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. Value + must be non-negative integer. 0 means NO limit on the number + of backups. + format: int32 + type: integer + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + remoteVolume: + description: array of remote volumes from CSI driver definition. properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the path - to key ring for User, default is /etc/ceph/user.secret More - info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine and + then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is to + mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the readOnly + setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent + disk resource in AWS (Amazon EBS volume). More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID type: object - user: - description: 'user is optional: User is the rados user name, - default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and mounted - on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. More - info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret object - containing parameters used to connect to OpenStack.' + azureDisk: + description: azureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in + the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + description: fsType is Filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to + shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name type: string + required: + - secretName + - shareName type: object - volumeID: - description: 'volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate - this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect - the file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair in - the Data field of the referenced ConfigMap will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the - ConfigMap, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to set - permissions on this file. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: path is the relative path of the file to - map the key to. May not be an absolute path. May not - contain the path element '..'. May not start with - the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or its - keys must be defined - type: boolean - type: object - csi: - description: csi (Container Storage Interface) represents ephemeral - storage that is handled by certain external CSI drivers (Beta - feature). - properties: - driver: - description: driver is the name of the CSI driver that handles - this volume. Consult with your admin for the correct name - as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". If - not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem to - apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the secret - object containing sensitive information to pass to the CSI - driver to complete the CSI NodePublishVolume and NodeUnpublishVolume - calls. This field is optional, and may be empty if no secret - is required. If the secret object contains more than one - secret, all secret references are passed. + cephfs: + description: cephFS represents a Ceph FS mount on the host + that shares a pod's lifetime properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + monitors: + description: 'monitors is Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' type: string + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile is the + path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is reference + to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is optional: User is the rados user + name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors type: object - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. + cinder: + description: 'cinder represents a cinder volume attached and + mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a secret + object containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volumeID used to identify the volume in + cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod - that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Defaults to 0644. Directories within - the path are not affected by this setting. This might be - in conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are supported.' + configMap: + description: configMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value pair + in the Data field of the referenced ConfigMap will be + projected into the volume as a file whose name is the + key and content is the value. If specified, the listed + keys will be projected into the specified paths, and + unlisted keys will not be present. If a key is specified + which is not present in the ConfigMap, the volume setup + will error unless it is marked optional. Paths must + be relative and may not contain the '..' path or start + with '..'. + items: + description: Maps a string key to a path within a volume. properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". + key: + description: key is the key to project. type: string - fieldPath: - description: Path of the field to select in the - specified API version. + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. type: string required: - - fieldPath + - key + - path type: object - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name - of the file to be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 encoded. The - first item of the relative path must not start with - ''..''' + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap or + its keys must be defined + type: boolean + type: object + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external CSI + drivers (Beta feature). + properties: + driver: + description: driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated + CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to the + secret object containing sensitive information to pass + to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the secret + object contains more than one secret, all secret references + are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: type: string - resourceFieldRef: - description: 'Selects a resource of the container: only - resources limits and requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are currently supported.' + description: volumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Defaults to 0644. Directories within the path are not + affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON + requires decimal values for mode bits. If not + specified, the volume defaultMode will be used. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object required: - - resource + - path type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that shares - a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which means - to use the node''s default medium. Must be an empty string - (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is also - applicable for memory medium. The maximum usage on memory - medium EmptyDir would be the minimum value between the SizeLimit - specified here and the sum of memory limits of all containers - in a pod. The default is nil which means that the limit - is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled by - a cluster storage driver. The volume's lifecycle is tied to - the pod that defines it - it will be created before the pod - starts, and deleted when the pod is removed. \n Use this if: - a) the volume is only needed while the pod runs, b) features - of normal volumes like restoring from snapshot or capacity tracking - are needed, c) the storage driver is specified through a storage - class, and d) the storage driver supports dynamic volume provisioning - through a PersistentVolumeClaim (see EphemeralVolumeSource for - more information on the connection between this volume type - and PersistentVolumeClaim). \n Use PersistentVolumeClaim or - one of the vendor-specific APIs for volumes that persist for - longer than the lifecycle of an individual pod. \n Use CSI for - light-weight local ephemeral volumes if the CSI driver is meant - to be used that way - see the documentation of the driver for - more information. \n A pod can use both types of ephemeral volumes - and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC to - provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC will - be deleted together with the pod. The name of the PVC will - be `-` where `` is the - name from the `PodSpec.Volumes` array entry. Pod validation - will reject the pod if the concatenated name is not valid - for a PVC (for example, too long). \n An existing PVC with - that name that is not owned by the pod will *not* be used - for the pod to avoid using an unrelated volume by mistake. - Starting the pod is then blocked until the unrelated PVC - is removed. If such a pre-created PVC is meant to be used - by the pod, the PVC has to updated with an owner reference - to the pod once the pod exists. Normally this should not - be necessary, but it may be useful when manually reconstructing - a broken cluster. \n This field is read-only and no changes - will be made by Kubernetes to the PVC after it has been - created. \n Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations that will - be copied into the PVC when creating it. No other fields - are allowed and will be rejected during validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the PVC - that gets created from this template. The same fields - as in a PersistentVolumeClaim are also valid here. + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory that + shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage medium + should back this directory. The default is "" which + means to use the node''s default medium. Must be an + empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local storage + required for this EmptyDir volume. The size limit is + also applicable for memory medium. The maximum usage + on memory medium EmptyDir would be the minimum value + between the SizeLimit specified here and the sum of + memory limits of all containers in a pod. The default + is nil which means that the limit is undefined. More + info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use + this if: a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot + or capacity tracking are needed, c) the storage driver is + specified through a storage class, and d) the storage driver + supports dynamic volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information on the connection + between this volume type and PersistentVolumeClaim). \n + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. \n Use CSI for light-weight local + ephemeral volumes if the CSI driver is meant to be used + that way - see the documentation of the driver for more + information. \n A pod can use both types of ephemeral volumes + and persistent volumes at the same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC + to provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of + the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). + \n An existing PVC with that name that is not owned + by the pod will *not* be used for the pod to avoid using + an unrelated volume by mistake. Starting the pod is + then blocked until the unrelated PVC is removed. If + such a pre-created PVC is meant to be used by the pod, + the PVC has to updated with an owner reference to the + pod once the pod exists. Normally this should not be + necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no + changes will be made by Kubernetes to the PVC after + it has been created. \n Required, must not be nil." properties: - accessModes: - description: 'accessModes contains the desired access - modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to specify - either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If the - provisioner or an external controller can support - the specified data source, it will create a new - volume based on the contents of the specified data - source. When the AnyVolumeDataSource feature gate - is enabled, dataSource contents will be copied to - dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API group. - For any other third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - required: - - kind - - name - type: object - dataSourceRef: - description: 'dataSourceRef specifies the object from - which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a - non-empty API group (non core object) or a PersistentVolumeClaim - object. When this field is specified, volume binding - will only succeed if the type of the specified object - matches some installed volume populator or dynamic - provisioner. This field will replace the functionality - of the dataSource field and as such if both fields - are non-empty, they must have the same value. For - backwards compatibility, when namespace isn''t specified - in dataSourceRef, both fields (dataSource and dataSourceRef) - will be set to the same value automatically if one - of them is empty and the other is non-empty. When - namespace is specified in dataSourceRef, dataSource - isn''t set to the same value and must be empty. - There are three important differences between dataSource - and dataSourceRef: * While dataSource only allows - two specific types of objects, dataSourceRef allows - any non-core object, as well as PersistentVolumeClaim - objects. * While dataSource ignores disallowed values - (dropping them), dataSourceRef preserves all values, - and generates an error if a disallowed value is - specified. * While dataSource only allows local - objects, dataSourceRef allows objects in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource - feature gate to be enabled. (Alpha) Using the namespace - field of dataSourceRef requires the CrossNamespaceVolumeDataSource - feature gate to be enabled.' + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No + other fields are allowed and will be rejected during + validation. properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API group. - For any other third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object name: - description: Name is the name of resource being - referenced type: string namespace: - description: Namespace is the namespace of resource - being referenced Note that when a namespace - is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept the - reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires the - CrossNamespaceVolumeDataSource feature gate - to be enabled. type: string - required: - - kind - - name type: object - resources: - description: 'resources represents the minimum resources - the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity recorded - in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the + PVC that gets created from this template. The same + fields as in a PersistentVolumeClaim are also valid + here. properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are used - by this container. \n This is an alpha field - and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." + accessModes: + description: 'accessModes contains the desired + access modes the volume should have. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: - description: ResourceClaim references one entry - in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of - one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes - that resource available inside a container. - type: string - required: - - name - type: object + type: string type: array - x-kubernetes-list-map-keys: + dataSource: + description: 'dataSource field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If + the provisioner or an external controller can + support the specified data source, it will create + a new volume based on the contents of the specified + data source. When the AnyVolumeDataSource feature + gate is enabled, dataSource contents will be + copied to dataSourceRef, and dataSourceRef contents + will be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, + then dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount - of compute resources required. If Requests is - omitted for a container, it defaults to Limits - if that is explicitly specified, otherwise to - an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + dataSourceRef: + description: 'dataSourceRef specifies the object + from which to populate the volume with data, + if a non-empty volume is desired. This may be + any object from a non-empty API group (non core + object) or a PersistentVolumeClaim object. When + this field is specified, volume binding will + only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the dataSource field and as such if both + fields are non-empty, they must have the same + value. For backwards compatibility, when namespace + isn''t specified in dataSourceRef, both fields + (dataSource and dataSourceRef) will be set to + the same value automatically if one of them + is empty and the other is non-empty. When namespace + is specified in dataSourceRef, dataSource isn''t + set to the same value and must be empty. There + are three important differences between dataSource + and dataSourceRef: * While dataSource only allows + two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed + values (dropping them), dataSourceRef preserves + all values, and generates an error if a disallowed + value is specified. * While dataSource only + allows local objects, dataSourceRef allows objects + in any namespaces. (Beta) Using this field requires + the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef + requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: Namespace is the namespace of + resource being referenced Note that when + a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept + the reference. See the ReferenceGrant documentation + for details. (Alpha) This field requires + the CrossNamespaceVolumeDataSource feature + gate to be enabled. + type: string + required: + - kind + - name type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement is - a selector that contains values, a key, and - an operator that relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and DoesNotExist. + resources: + description: 'resources represents the minimum + resources the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity + recorded in the status field of the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are + used by this container. \n This is an alpha + field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. If - the operator is Exists or DoesNotExist, - the values array must be empty. This array - is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is "In", - and the values array contains only "value". - The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object type: object + storageClassName: + description: 'storageClassName is the name of + the StorageClass required by the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string type: object - storageClassName: - description: 'storageClassName is the name of the - StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem is - implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to - the PersistentVolume backing this claim. - type: string + required: + - spec type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached - to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and lun - must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource that - is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for this - volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends - on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' type: object - readOnly: - description: 'readOnly is Optional: defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information to - pass to the plugin scripts. This may be empty if no secret - object is specified. If the secret object contains more - than one secret, all secrets are passed to the plugin scripts.' + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to + the pod. properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising the machine' type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs and + lun must be set, but not both simultaneously.' + items: + type: string + type: array type: object - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached to a - kubelet's host machine. This depends on the Flocker control - service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored as - metadata -> name on the dataset for Flocker should be considered - as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume that - you want to mount. Tip: Ensure that the filesystem type - is supported by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount by - volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property empty). More - info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource in - GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision a - container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir into - the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, the - volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the host - that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More info: - https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + flexVolume: + description: flexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The default filesystem + depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is reference + to the secret object containing sensitive information + to pass to the plugin scripts. This may be empty if + no secret object is specified. If the secret object + contains more than one secret, all secrets are passed + to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: datasetName is Name of the dataset stored + as metadata -> name on the dataset for Flocker should + be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is to + mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource + in GCE. Used to identify the disk in GCE. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir + into the Pod''s container.' + properties: + directory: + description: directory is the target directory name. Must + not contain or start with '..'. If '.' is supplied, + the volume directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to + false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'path of the directory on the host. If the + path is a symlink, it will follow the link to the real + path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support + iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name that + uses an iSCSI transport. Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal List. + The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: targetPortal is iSCSI Target Portal. The + Portal is either an IP or ip_addr:port if the port is + other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean + nfs: + description: 'nfs represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export + to be mounted with read-only permissions. Defaults to + false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: readOnly Will force the ReadOnly setting + in VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this + setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced ConfigMap + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the ConfigMap, the + volume setup will error unless it is marked + optional. Paths must be relative and may not + contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and 0777 + or a decimal value between 0 and 511. + YAML accepts both octal and decimal + values, JSON requires decimal values + for mode bits. If not specified, the + volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the + container: only resources limits and + requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) are + currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced Secret + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the Secret, the volume + setup will error unless it is marked optional. + Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional field specify whether + the Secret or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to project + properties: + audience: + description: audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if the + token is older than 80 percent of its time + to live or if the token is older than 24 hours.Defaults + to 1 hour and must be at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative to the + mount point of the file to project the token + into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default is + no group + type: string + readOnly: + description: readOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults to + false. + type: boolean + registry: + description: registry represents a single or multiple + Quobyte Registry services specified as a string as host:port + pair (multiple entries are separated with commas) which + acts as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume in + the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: user to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'image is the rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default is + rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. Default + is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is the rados user name. Default is + admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret for ScaleIO + user and other sensitive information. If this is not + provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage + for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system + as configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume already + created in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value pair + in the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and + content is the value. If specified, the listed keys + will be projected into the specified paths, and unlisted + keys will not be present. If a key is specified which + is not present in the Secret, the volume setup will + error unless it is marked optional. Paths must be relative + and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether the Secret + or its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret in + the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: volumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope of the + volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within StorageOS + for tighter integration. Set VolumeName to any name + to override the default behaviour. Set to "default" + if you are not using namespaces within StorageOS. Namespaces + that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fsType is filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath + type: object required: - - endpoints - - path + - name type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' + target: + description: target database cluster for backup. properties: - path: - description: 'path of the directory on the host. If the path - is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" More - info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string + labelsSelector: + description: labelsSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + secret: + description: secret is used to connect to the target database + cluster. If not set, secret will be inherited from backup + policy template. if still not set, the controller will check + if any system account for dataprotection has been created. + properties: + name: + description: the secret name + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + passwordKey: + default: password + description: passwordKey the map key of the password in + the connection credential secret + type: string + usernameKey: + default: username + description: usernameKey the map key of the user in the + connection credential secret + type: string + required: + - name + type: object required: - - path + - labelsSelector type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that is - attached to a kubelet''s host machine and then exposed to the - pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + required: + - remoteVolume + - target + type: object + incremental: + description: the policy for incremental backup. + properties: + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can execute. + type: string + path: + description: 'specify the json path of backup object for + patch. example: manifests.backupLog -- means patch the + backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect backup + status metadata. The script must exist in the container + of ContainerName and the output format must be set to + JSON. Note that outputting to stderr may cause the result + format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: before + backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupToolName: + description: which backup tool to perform database backup, only + support one tool. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. Value + must be non-negative integer. 0 means NO limit on the number + of backups. + format: int32 + type: integer + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + remoteVolume: + description: array of remote volumes from CSI driver definition. properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI - Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, - new iSCSI interface : will be - created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that uses - an iSCSI transport. Defaults to 'default' (tcp). + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine and + then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is to + mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the readOnly + setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent + disk resource in AWS (Amazon EBS volume). More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in + the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + description: fsType is Filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to + shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host + that shares a pod's lifetime + properties: + monitors: + description: 'monitors is Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile is the + path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is reference + to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is optional: User is the rados user + name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'cinder represents a cinder volume attached and + mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a secret + object containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volumeID used to identify the volume in + cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value pair + in the Data field of the referenced ConfigMap will be + projected into the volume as a file whose name is the + key and content is the value. If specified, the listed + keys will be projected into the specified paths, and + unlisted keys will not be present. If a key is specified + which is not present in the ConfigMap, the volume setup + will error unless it is marked optional. Paths must + be relative and may not contain the '..' path or start + with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap or + its keys must be defined + type: boolean + type: object + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external CSI + drivers (Beta feature). + properties: + driver: + description: driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated + CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to the + secret object containing sensitive information to pass + to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the secret + object contains more than one secret, all secret references + are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Defaults to 0644. Directories within the path are not + affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON + requires decimal values for mode bits. If not + specified, the volume defaultMode will be used. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory that + shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage medium + should back this directory. The default is "" which + means to use the node''s default medium. Must be an + empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local storage + required for this EmptyDir volume. The size limit is + also applicable for memory medium. The maximum usage + on memory medium EmptyDir would be the minimum value + between the SizeLimit specified here and the sum of + memory limits of all containers in a pod. The default + is nil which means that the limit is undefined. More + info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use + this if: a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot + or capacity tracking are needed, c) the storage driver is + specified through a storage class, and d) the storage driver + supports dynamic volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information on the connection + between this volume type and PersistentVolumeClaim). \n + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. \n Use CSI for light-weight local + ephemeral volumes if the CSI driver is meant to be used + that way - see the documentation of the driver for more + information. \n A pod can use both types of ephemeral volumes + and persistent volumes at the same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC + to provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of + the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). + \n An existing PVC with that name that is not owned + by the pod will *not* be used for the pod to avoid using + an unrelated volume by mistake. Starting the pod is + then blocked until the unrelated PVC is removed. If + such a pre-created PVC is meant to be used by the pod, + the PVC has to updated with an owner reference to the + pod once the pod exists. Normally this should not be + necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no + changes will be made by Kubernetes to the PVC after + it has been created. \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No + other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the + PVC that gets created from this template. The same + fields as in a PersistentVolumeClaim are also valid + here. + properties: + accessModes: + description: 'accessModes contains the desired + access modes the volume should have. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If + the provisioner or an external controller can + support the specified data source, it will create + a new volume based on the contents of the specified + data source. When the AnyVolumeDataSource feature + gate is enabled, dataSource contents will be + copied to dataSourceRef, and dataSourceRef contents + will be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, + then dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'dataSourceRef specifies the object + from which to populate the volume with data, + if a non-empty volume is desired. This may be + any object from a non-empty API group (non core + object) or a PersistentVolumeClaim object. When + this field is specified, volume binding will + only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the dataSource field and as such if both + fields are non-empty, they must have the same + value. For backwards compatibility, when namespace + isn''t specified in dataSourceRef, both fields + (dataSource and dataSourceRef) will be set to + the same value automatically if one of them + is empty and the other is non-empty. When namespace + is specified in dataSourceRef, dataSource isn''t + set to the same value and must be empty. There + are three important differences between dataSource + and dataSourceRef: * While dataSource only allows + two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed + values (dropping them), dataSourceRef preserves + all values, and generates an error if a disallowed + value is specified. * While dataSource only + allows local objects, dataSourceRef allows objects + in any namespaces. (Beta) Using this field requires + the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef + requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: Namespace is the namespace of + resource being referenced Note that when + a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept + the reference. See the ReferenceGrant documentation + for details. (Alpha) This field requires + the CrossNamespaceVolumeDataSource feature + gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum + resources the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity + recorded in the status field of the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are + used by this container. \n This is an alpha + field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'storageClassName is the name of + the StorageClass required by the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to + the pod. + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising the machine' + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs and + lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: flexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The default filesystem + depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is reference + to the secret object containing sensitive information + to pass to the plugin scripts. This may be empty if + no secret object is specified. If the secret object + contains more than one secret, all secrets are passed + to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: datasetName is Name of the dataset stored + as metadata -> name on the dataset for Flocker should + be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is to + mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource + in GCE. Used to identify the disk in GCE. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir + into the Pod''s container.' + properties: + directory: + description: directory is the target directory name. Must + not contain or start with '..'. If '.' is supplied, + the volume directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to + false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'path of the directory on the host. If the + path is a symlink, it will follow the link to the real + path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support + iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name that + uses an iSCSI transport. Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal List. + The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: targetPortal is iSCSI Target Portal. The + Portal is either an IP or ip_addr:port if the port is + other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. The - portal is either an IP or ip_addr:port if the port is other - than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication + nfs: + description: 'nfs represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export + to be mounted with read-only permissions. Defaults to + false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string + readOnly: + description: readOnly Will force the ReadOnly setting + in VolumeMounts. Default false. + type: boolean + required: + - claimName type: object - targetPortal: - description: targetPortal is iSCSI Target Portal. The Portal - is either an IP or ip_addr:port if the port is other than - default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: 'name of the volume. Must be a DNS_LABEL and unique - within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - nfs: - description: 'nfs represents an NFS mount on the host that shares - a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export to be - mounted with read-only permissions. Defaults to false. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of the - NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents a reference - to a PersistentVolumeClaim in the same namespace. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Directories within the path are not - affected by this setting. This might be in conflict with - other options that affect the file mode, like fsGroup, and - the result can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along with - other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file whose - name is the key and content is the value. If specified, - the listed keys will be projected into the specified - paths, and unlisted keys will not be present. - If a key is specified which is not present in - the ConfigMap, the volume setup will error unless - it is marked optional. Paths must be relative - and may not contain the '..' path or start with - '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. If - not specified, the volume defaultMode will - be used. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this + setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along + with other supported volume types properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing the - pod field - properties: - fieldRef: - description: 'Required: Selects a field of - the pod: only annotations, labels, name - and namespace are supported.' + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced ConfigMap + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the ConfigMap, the + volume setup will error unless it is marked + optional. Paths must be relative and may not + contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. properties: - apiVersion: - description: Version of the schema the - FieldPath is written in terms of, defaults - to "v1". + key: + description: key is the key to project. type: string - fieldPath: - description: Path of the field to select - in the specified API version. + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. type: string required: - - fieldPath + - key + - path type: object - mode: - description: 'Optional: mode bits used to - set permissions on this file, must be an - octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both - octal and decimal values, JSON requires - decimal values for mode bits. If not specified, - the volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. Must - not be absolute or contain the ''..'' path. - Must be utf-8 encoded. The first item of - the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and 0777 + or a decimal value between 0 and 511. + YAML accepts both octal and decimal + values, JSON requires decimal values + for mode bits. If not specified, the + volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the + container: only resources limits and + requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) are + currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced Secret + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the Secret, the volume + setup will error unless it is marked optional. + Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' + key: + description: key is the key to project. type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format - of the exposed resources, defaults to - "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. type: string required: - - resource + - key + - path type: object - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret data - to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file whose - name is the key and content is the value. If specified, - the listed keys will be projected into the specified - paths, and unlisted keys will not be present. - If a key is specified which is not present in - the Secret, the volume setup will error unless - it is marked optional. Paths must be relative - and may not contain the '..' path or start with - '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. If - not specified, the volume defaultMode will - be used. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional field specify whether the - Secret or its key must be defined - type: boolean + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional field specify whether + the Secret or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to project + properties: + audience: + description: audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if the + token is older than 80 percent of its time + to live or if the token is older than 24 hours.Defaults + to 1 hour and must be at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative to the + mount point of the file to project the token + into. + type: string + required: + - path + type: object type: object - serviceAccountToken: - description: serviceAccountToken is information about - the serviceAccountToken data to project + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default is + no group + type: string + readOnly: + description: readOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults to + false. + type: boolean + registry: + description: registry represents a single or multiple + Quobyte Registry services specified as a string as host:port + pair (multiple entries are separated with commas) which + acts as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume in + the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: user to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'image is the rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default is + rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. Default + is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is the rados user name. Default is + admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret for ScaleIO + user and other sensitive information. If this is not + provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage + for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system + as configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume already + created in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value pair + in the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and + content is the value. If specified, the listed keys + will be projected into the specified paths, and unlisted + keys will not be present. If a key is specified which + is not present in the Secret, the volume setup will + error unless it is marked optional. Paths must be relative + and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. properties: - audience: - description: audience is the intended audience of - the token. A recipient of a token must identify - itself with an identifier specified in the audience - of the token, and otherwise should reject the - token. The audience defaults to the identifier - of the apiserver. + key: + description: key is the key to project. type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account token. - As the token approaches expiration, the kubelet - volume plugin will proactively rotate the service - account token. The kubelet will start trying to - rotate the token if the token is older than 80 - percent of its time to live or if the token is - older than 24 hours.Defaults to 1 hour and must - be at least 10 minutes. - format: int64 + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 type: integer path: - description: path is the path relative to the mount - point of the file to project the token into. + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. type: string required: + - key - path type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host that - shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is no group - type: string - readOnly: - description: readOnly here will force the Quobyte volume to - be mounted with read-only permissions. Defaults to false. - type: boolean - registry: - description: registry represents a single or multiple Quobyte - Registry services specified as a string as host:port pair - (multiple entries are separated with commas) which acts - as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in the - Backend Used with dynamically provisioned Quobyte volumes, - value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to serivceaccount - user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - image: - description: 'image is the rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. More - info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication secret - for RBDUser. If provided overrides keyring. Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: array + optional: + description: optional field specify whether the Secret + or its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret in + the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: volumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope of the + volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within StorageOS + for tighter integration. Set VolumeName to any name + to override the default behaviour. Set to "default" + if you are not using namespaces within StorageOS. Namespaces + that do not pre-exist within StorageOS will be created. type: string type: object - user: - description: 'user is the rados user name. Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API - Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection - Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not provided, - Login operation will fail. + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + fsType: + description: fsType is filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath type: object - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage for - a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system as configured - in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already created - in the ScaleIO system that is associated with this volume - source. - type: string required: - - gateway - - secretRef - - system + - name type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + target: + description: target database cluster for backup. properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect - the file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair in - the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the - Secret, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to set - permissions on this file. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: path is the relative path of the file to - map the key to. May not be an absolute path. May not - contain the path element '..'. May not start with - the string '..'. + labelsSelector: + description: labelsSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret or - its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in the - pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for obtaining - the StorageOS API credentials. If not specified, default - values will be attempted. + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + secret: + description: secret is used to connect to the target database + cluster. If not set, secret will be inherited from backup + policy template. if still not set, the controller will check + if any system account for dataprotection has been created. properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + description: the secret name + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + passwordKey: + default: password + description: passwordKey the map key of the password in + the connection credential secret type: string + usernameKey: + default: username + description: usernameKey the map key of the user in the + connection credential secret + type: string + required: + - name type: object - volumeName: - description: volumeName is the human-readable name of the - StorageOS volume. Volume names are only unique within a - namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the volume - within StorageOS. If no namespace is specified then the - Pod's namespace will be used. This allows the Kubernetes - name scoping to be mirrored within StorageOS for tighter - integration. Set VolumeName to any name to override the - default behaviour. Set to "default" if you are not using - namespaces within StorageOS. Namespaces that do not pre-exist - within StorageOS will be created. - type: string + required: + - labelsSelector type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine + required: + - remoteVolume + - target + type: object + schedule: + description: schedule policy for backup. + properties: + baseBackup: + description: schedule policy for base backup. properties: - fsType: - description: fsType is filesystem type to mount. Must be a - filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management - (SPBM) profile ID associated with the StoragePolicyName. + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. + enable: + description: enable or disable the schedule. + type: boolean + type: + description: the type of base backup, only support full and + snapshot. + enum: + - full + - snapshot type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk + required: + - cronExpression + - enable + - type + type: object + incremental: + description: schedule policy for incremental backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - - volumePath + - cronExpression + - enable type: object - required: - - name type: object - schedule: - description: The schedule in Cron format, the timezone is in UTC. - see https://en.wikipedia.org/wiki/Cron. - type: string - target: - description: database cluster service + snapshot: + description: the policy for snapshot backup. properties: - labelsSelector: - description: LabelSelector is used to find matching pods. Pods - that match this label selector are counted to determine the - number of pods in their corresponding topology domain. + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can execute. + type: string + path: + description: 'specify the json path of backup object for + patch. example: manifests.backupLog -- means patch the + backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect backup + status metadata. The script must exist in the container + of ContainerName and the output format must be set to + JSON. Note that outputting to stderr may cause the result + format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: before + backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. Value + must be non-negative integer. 0 means NO limit on the number + of backups. + format: int32 + type: integer + hooks: + description: execute hook commands for backup. properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. + containerName: + description: which container can exec command + type: string + image: + description: exec command with image + type: string + postCommands: + description: post backup to perform commands items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If - the operator is In or NotIn, the values array must - be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object + type: string type: array - matchLabels: - additionalProperties: + preCommands: + description: pre backup to perform commands + items: type: string - description: matchLabels is a map of {key,value} pairs. A - single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field is "key", - the operator is "In", and the values array contains only - "value". The requirements are ANDed. - type: object + type: array type: object - x-kubernetes-preserve-unknown-fields: true - secret: - description: Secret is used to connect to the target database - cluster. If not set, secret will be inherited from backup policy - template. if still not set, the controller will check if any - system account for dataprotection has been created. + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + target: + description: target database cluster for backup. properties: - name: - description: the secret name - pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ - type: string - passwordKeyword: - description: PasswordKeyword the map keyword of the password - in the connection credential secret - type: string - userKeyword: - description: UserKeyword the map keyword of the user in the - connection credential secret - type: string + labelsSelector: + description: labelsSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + secret: + description: secret is used to connect to the target database + cluster. If not set, secret will be inherited from backup + policy template. if still not set, the controller will check + if any system account for dataprotection has been created. + properties: + name: + description: the secret name + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + passwordKey: + default: password + description: passwordKey the map key of the password in + the connection credential secret + type: string + usernameKey: + default: username + description: usernameKey the map key of the user in the + connection credential secret + type: string + required: + - name + type: object required: - - name + - labelsSelector type: object required: - - labelsSelector + - target type: object ttl: - description: TTL is a time.Duration-parseable string describing how - long the Backup should be retained for. + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. if + not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ type: string - required: - - remoteVolume - - target type: object status: description: BackupPolicyStatus defines the observed state of BackupPolicy @@ -1749,22 +3666,25 @@ spec: description: the reason if backup policy check failed. type: string lastScheduleTime: - description: Information when was the last time the job was successfully + description: information when was the last time the job was successfully scheduled. format: date-time type: string lastSuccessfulTime: - description: Information when was the last time the job successfully + description: information when was the last time the job successfully completed. format: date-time type: string + observedGeneration: + description: observedGeneration is the most recent generation observed + for this BackupPolicy. It corresponds to the Cluster's generation, + which is updated on mutation by the API Server. + format: int64 + type: integer phase: - description: 'backup policy phase valid value: available, failed, - new.' + description: 'backup policy phase valid value: Available, Failed.' enum: - - New - Available - - InProgress - Failed type: string type: object diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicytemplates.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicytemplates.yaml deleted file mode 100644 index ea15b74c6..000000000 --- a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicytemplates.yaml +++ /dev/null @@ -1,144 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null - name: backuppolicytemplates.dataprotection.kubeblocks.io -spec: - group: dataprotection.kubeblocks.io - names: - categories: - - kubeblocks - kind: BackupPolicyTemplate - listKind: BackupPolicyTemplateList - plural: backuppolicytemplates - singular: backuppolicytemplate - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: BackupPolicyTemplate is the Schema for the BackupPolicyTemplates - API (defined by provider) - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BackupPolicyTemplateSpec defines the desired state of BackupPolicyTemplate - properties: - backupStatusUpdates: - description: define how to update metadata for backup status. - items: - properties: - containerName: - description: which container name that kubectl can execute. - type: string - path: - description: 'specify the json path of backup object for patch. - example: manifests.backupLog -- means patch the backup json - path of status.manifests.backupLog.' - type: string - script: - description: the shell Script commands to collect backup status - metadata. The script must exist in the container of ContainerName - and the output format must be set to JSON. Note that outputting - to stderr may cause the result format to not be in JSON. - type: string - updateStage: - description: 'when to update the backup status, pre: before - backup, post: after backup' - enum: - - pre - - post - type: string - type: object - type: array - backupToolName: - description: which backup tool to perform database backup, only support - one tool. - pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ - type: string - credentialKeyword: - description: CredentialKeyword determines backupTool connection credential - keyword in secret. the backupTool gets the credentials according - to the user and password keyword defined by secret - properties: - passwordKeyword: - default: password - description: PasswordKeyword the map keyword of the password in - the connection credential secret - type: string - userKeyword: - default: username - description: UserKeyword the map keyword of the user in the connection - credential secret - type: string - type: object - hooks: - description: execute hook commands for backup. - properties: - containerName: - description: which container can exec command - type: string - image: - description: exec command with image - type: string - postCommands: - description: post backup to perform commands - items: - type: string - type: array - preCommands: - description: pre backup to perform commands - items: - type: string - type: array - type: object - onFailAttempted: - description: limit count of backup stop retries on fail. if unset, - retry unlimit attempted. - format: int32 - type: integer - schedule: - description: The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron. - type: string - ttl: - description: TTL is a time.Duration-parseable string describing how - long the Backup should be retained for. - type: string - required: - - backupToolName - type: object - status: - description: BackupPolicyTemplateStatus defines the observed state of - BackupPolicyTemplate - properties: - failureReason: - type: string - phase: - description: BackupPolicyTemplatePhase defines phases for BackupPolicyTemplate - CR. - enum: - - New - - Available - - InProgress - - Failed - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml index 188fb3fb3..4542c6957 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml @@ -72,10 +72,6 @@ spec: parentBackupName: description: if backupType is incremental, parentBackupName is required. type: string - ttl: - description: ttl is a time.Duration-parsable string describing how - long the Backup should be retained for. - type: string required: - backupPolicyName - backupType diff --git a/config/crd/bases/dataprotection.kubeblocks.io_restorejobs.yaml b/config/crd/bases/dataprotection.kubeblocks.io_restorejobs.yaml index 84334be2b..b282429e6 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_restorejobs.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_restorejobs.yaml @@ -59,7 +59,7 @@ spec: description: the target database workload to restore properties: labelsSelector: - description: LabelSelector is used to find matching pods. Pods + description: labelsSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. properties: @@ -106,7 +106,7 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true secret: - description: Secret is used to connect to the target database + description: secret is used to connect to the target database cluster. If not set, secret will be inherited from backup policy template. if still not set, the controller will check if any system account for dataprotection has been created. @@ -115,14 +115,16 @@ spec: description: the secret name pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string - passwordKeyword: - description: PasswordKeyword the map keyword of the password - in the connection credential secret - type: string - userKeyword: - description: UserKeyword the map keyword of the user in the + passwordKey: + default: password + description: passwordKey the map key of the password in the connection credential secret type: string + usernameKey: + default: username + description: usernameKey the map key of the user in the connection + credential secret + type: string required: - name type: object diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 7de82f64b..e95ab06ea 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,6 +2,7 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: +- bases/apps.kubeblocks.io_backuppolicytemplates.yaml - bases/apps.kubeblocks.io_clusters.yaml - bases/apps.kubeblocks.io_clusterdefinitions.yaml - bases/apps.kubeblocks.io_clusterversions.yaml @@ -11,7 +12,6 @@ resources: - bases/dataprotection.kubeblocks.io_backuppolicies.yaml - bases/dataprotection.kubeblocks.io_backups.yaml - bases/dataprotection.kubeblocks.io_restorejobs.yaml -- bases/dataprotection.kubeblocks.io_backuppolicytemplates.yaml - bases/extensions.kubeblocks.io_addons.yaml - bases/apps.kubeblocks.io_classfamilies.yaml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/patches/cainjection_in_dataprotection_backuppolicytemplates.yaml b/config/crd/patches/cainjection_in_apps_backuppolicytemplates.yaml similarity index 81% rename from config/crd/patches/cainjection_in_dataprotection_backuppolicytemplates.yaml rename to config/crd/patches/cainjection_in_apps_backuppolicytemplates.yaml index 8340a38e6..7600d148d 100644 --- a/config/crd/patches/cainjection_in_dataprotection_backuppolicytemplates.yaml +++ b/config/crd/patches/cainjection_in_apps_backuppolicytemplates.yaml @@ -4,4 +4,4 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: backuppolicytemplates.dataprotection.kubeblocks.io + name: backuppolicytemplates.apps.kubeblocks.io diff --git a/config/crd/patches/webhook_in_dataprotection_backuppolicytemplates.yaml b/config/crd/patches/webhook_in_apps_backuppolicytemplates.yaml similarity index 85% rename from config/crd/patches/webhook_in_dataprotection_backuppolicytemplates.yaml rename to config/crd/patches/webhook_in_apps_backuppolicytemplates.yaml index 2dea48c6a..c6cba6e35 100644 --- a/config/crd/patches/webhook_in_dataprotection_backuppolicytemplates.yaml +++ b/config/crd/patches/webhook_in_apps_backuppolicytemplates.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: backuppolicytemplates.dataprotection.kubeblocks.io + name: backuppolicytemplates.apps.kubeblocks.io spec: conversion: strategy: Webhook diff --git a/config/rbac/dataprotection_backuppolicytemplate_editor_role.yaml b/config/rbac/apps_backuppolicytemplate_editor_role.yaml similarity index 85% rename from config/rbac/dataprotection_backuppolicytemplate_editor_role.yaml rename to config/rbac/apps_backuppolicytemplate_editor_role.yaml index 0c5f1c24e..ff9688280 100644 --- a/config/rbac/dataprotection_backuppolicytemplate_editor_role.yaml +++ b/config/rbac/apps_backuppolicytemplate_editor_role.yaml @@ -5,7 +5,7 @@ metadata: name: backuppolicytemplate-editor-role rules: - apiGroups: - - dataprotection.kubeblocks.io + - apps.kubeblocks.io resources: - backuppolicytemplates verbs: @@ -17,7 +17,7 @@ rules: - update - watch - apiGroups: - - dataprotection.kubeblocks.io + - apps.kubeblocks.io resources: - backuppolicytemplates/status verbs: diff --git a/config/rbac/dataprotection_backuppolicytemplate_viewer_role.yaml b/config/rbac/apps_backuppolicytemplate_viewer_role.yaml similarity index 83% rename from config/rbac/dataprotection_backuppolicytemplate_viewer_role.yaml rename to config/rbac/apps_backuppolicytemplate_viewer_role.yaml index b2f779fe8..aa2625708 100644 --- a/config/rbac/dataprotection_backuppolicytemplate_viewer_role.yaml +++ b/config/rbac/apps_backuppolicytemplate_viewer_role.yaml @@ -5,7 +5,7 @@ metadata: name: backuppolicytemplate-viewer-role rules: - apiGroups: - - dataprotection.kubeblocks.io + - apps.kubeblocks.io resources: - backuppolicytemplates verbs: @@ -13,7 +13,7 @@ rules: - list - watch - apiGroups: - - dataprotection.kubeblocks.io + - apps.kubeblocks.io resources: - backuppolicytemplates/status verbs: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e2747ca6a..7bd0f58ca 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -103,6 +103,13 @@ rules: - statefulsets/status verbs: - get +- apiGroups: + - apps.kubeblocks.io + resources: + - backuppolicytemplates + verbs: + - get + - list - apiGroups: - apps.kubeblocks.io resources: @@ -496,32 +503,6 @@ rules: - get - patch - update -- apiGroups: - - dataprotection.kubeblocks.io - resources: - - backuppolicytemplates - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - dataprotection.kubeblocks.io - resources: - - backuppolicytemplates/finalizers - verbs: - - update -- apiGroups: - - dataprotection.kubeblocks.io - resources: - - backuppolicytemplates/status - verbs: - - get - - patch - - update - apiGroups: - dataprotection.kubeblocks.io resources: diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index 018abcf9e..9ed4a84e2 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -47,6 +47,8 @@ import ( // +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=clusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=clusters/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=backuppolicytemplates,verbs=get;list + // owned K8s core API resources controller-gen RBAC marker // full access on core API resources // +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete;deletecollection @@ -91,7 +93,7 @@ import ( // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch // dataprotection get list and delete -// +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicies,verbs=get;list;delete;deletecollection +// +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicies,verbs=get;list;create;update;patch;delete;deletecollection // +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backups,verbs=get;list;delete;deletecollection // classfamily get list @@ -132,12 +134,12 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if re, ok := err.(lifecycle.RequeueError); ok { return intctrlutil.RequeueAfter(re.RequeueAfter(), reqCtx.Log, re.Reason()) } - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + return intctrlutil.RequeueWithError(err, reqCtx.Log, "") } planBuilder := lifecycle.NewClusterPlanBuilder(reqCtx, r.Client, req, r.Recorder) if err := planBuilder.Init(); err != nil { - return requeueError(err) + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } else if err := planBuilder.Validate(); err != nil { return requeueError(err) } else if plan, err := planBuilder.Build(); err != nil { diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 7a00b4c6c..2e75c6be0 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -51,6 +51,8 @@ import ( testk8s "github.com/apecloud/kubeblocks/internal/testutil/k8s" ) +const backupPolicyTPLName = "test-backup-policy-template-mysql" + var _ = Describe("Cluster Controller", func() { const ( clusterDefName = "test-clusterdef" @@ -601,16 +603,19 @@ var _ = Describe("Cluster Controller", func() { Expect(testCtx.Cli.Get(testCtx.Ctx, clusterKey, cluster)).Should(Succeed()) initialGeneration := int(cluster.Status.ObservedGeneration) + By("Checking backup policy created from backup policy template") + policyName := lifecycle.GenerateBackupPolicyName(clusterKey.Name, mysqlCompType) + Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKey{Name: policyName, Namespace: clusterKey.Namespace}, + &dataprotectionv1alpha1.BackupPolicy{}, true)).Should(Succeed()) + By("Set HorizontalScalePolicy") Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(clusterDefObj), func(clusterDef *appsv1alpha1.ClusterDefinition) { clusterDef.Spec.ComponentDefs[0].HorizontalScalePolicy = - &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot} + &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, + BackupPolicyTemplateName: backupPolicyTPLName} })()).ShouldNot(HaveOccurred()) - By("Creating a BackupPolicyTemplate") - createBackupPolicyTpl(clusterDefObj) - By("Mocking all components' PVCs to bound") for _, comp := range clusterObj.Spec.ComponentSpecs { for i := 0; i < int(comp.Replicas); i++ { @@ -1129,7 +1134,7 @@ var _ = Describe("Cluster Controller", func() { }, }, Spec: dataprotectionv1alpha1.BackupSpec{ - BackupPolicyName: "test-backup-policy", + BackupPolicyName: lifecycle.GenerateBackupPolicyName(clusterKey.Name, mysqlCompType), BackupType: "snapshot", }, } @@ -1140,16 +1145,12 @@ var _ = Describe("Cluster Controller", func() { g.Expect(backup.Status.Phase).Should(Equal(dataprotectionv1alpha1.BackupFailed)) })).Should(Succeed()) - By("Creating a BackupPolicyTemplate") - createBackupPolicyTpl(clusterDefObj) - By("Set HorizontalScalePolicy") Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(clusterDefObj), func(clusterDef *appsv1alpha1.ClusterDefinition) { clusterDef.Spec.ComponentDefs[0].HorizontalScalePolicy = - &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, BackupTemplateSelector: map[string]string{ - clusterDefLabelKey: clusterDefObj.Name, - }} + &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, + BackupPolicyTemplateName: backupPolicyTPLName} })()).ShouldNot(HaveOccurred()) By(fmt.Sprintf("Changing replicas to %d", updatedReplicas)) @@ -1213,6 +1214,9 @@ var _ = Describe("Cluster Controller", func() { AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). AddComponent(nginxCompType).AddContainerShort("nginx", testapps.NginxImage). Create(&testCtx).GetObject() + + By("Creating a BackupPolicyTemplate") + createBackupPolicyTpl(clusterDefObj) }) It("should create all sub-resources successfully", func() { @@ -1243,6 +1247,9 @@ var _ = Describe("Cluster Controller", func() { clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() + + By("Creating a BackupPolicyTemplate") + createBackupPolicyTpl(clusterDefObj) }) It("should delete cluster resources immediately if deleting cluster with WipeOut termination policy", func() { @@ -1305,6 +1312,9 @@ var _ = Describe("Cluster Controller", func() { clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() + + By("Creating a BackupPolicyTemplate") + createBackupPolicyTpl(clusterDefObj) }) It("Should success with one leader pod and two follower pods", func() { @@ -1695,17 +1705,10 @@ var _ = Describe("Cluster Controller", func() { func createBackupPolicyTpl(clusterDefObj *appsv1alpha1.ClusterDefinition) { By("Creating a BackupPolicyTemplate") - backupPolicyTplKey := types.NamespacedName{Name: "test-backup-policy-template-mysql"} - backupPolicyTpl := &dataprotectionv1alpha1.BackupPolicyTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Name: backupPolicyTplKey.Name, - Labels: map[string]string{ - clusterDefLabelKey: clusterDefObj.Name, - }, - }, - Spec: dataprotectionv1alpha1.BackupPolicyTemplateSpec{ - BackupToolName: "mysql-xtrabackup", - }, - } - Expect(testCtx.CreateObj(testCtx.Ctx, backupPolicyTpl)).Should(Succeed()) + testapps.NewBackupPolicyTemplateFactory(backupPolicyTPLName). + AddLabels(clusterDefLabelKey, clusterDefObj.Name). + AddBackupPolicy(clusterDefObj.Spec.ComponentDefs[0].Name). + AddSnapshotPolicy(). + SetClusterDefRef(clusterDefObj.Name). + SetTargetRole("leader").Create(&testCtx) } diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index af7999a45..cea8ed5f5 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -63,6 +63,9 @@ var _ = Describe("OpsRequest Controller", func() { // delete cluster(and all dependent sub-resources), clusterversion and clusterdef testapps.ClearClusterResources(&testCtx) testapps.ClearResources(&testCtx, intctrlutil.StorageClassSignature, ml) + + // non-namespaced + testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, ml) } BeforeEach(func() { @@ -254,7 +257,8 @@ var _ = Describe("OpsRequest Controller", func() { clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). AddComponent(testapps.ConsensusMySQLComponent, mysqlCompType). AddHorizontalScalePolicy(appsv1alpha1.HorizontalScalePolicy{ - Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, + Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, + BackupPolicyTemplateName: backupPolicyTPLName, }).Create(&testCtx).GetObject() By("Create a clusterVersion obj") diff --git a/controllers/apps/systemaccount_controller.go b/controllers/apps/systemaccount_controller.go index 21d002727..5b3ecb33a 100644 --- a/controllers/apps/systemaccount_controller.go +++ b/controllers/apps/systemaccount_controller.go @@ -84,12 +84,6 @@ const ( kbAccountEndPointEnvName = "KB_ACCOUNT_ENDPOINT" ) -// username and password are keys in created secrets for others to refer to. -const ( - accountNameForSecret = "username" - accountPasswdForSecret = "password" -) - // ENABLE_DEBUG_SYSACCOUNTS is used for debug only. const ( systemAccountsDebugMode string = "ENABLE_DEBUG_SYSACCOUNTS" diff --git a/controllers/apps/systemaccount_util.go b/controllers/apps/systemaccount_util.go index 0f6d5c8fc..b5c6abcff 100644 --- a/controllers/apps/systemaccount_util.go +++ b/controllers/apps/systemaccount_util.go @@ -217,8 +217,8 @@ func renderJob(engine *customizedEngine, key componentUniqueKey, statement []str func renderSecretWithPwd(key componentUniqueKey, username, passwd string) *corev1.Secret { secretData := map[string][]byte{} - secretData[accountNameForSecret] = []byte(username) - secretData[accountPasswdForSecret] = []byte(passwd) + secretData[constant.AccountNameForSecret] = []byte(username) + secretData[constant.AccountPasswdForSecret] = []byte(passwd) ml := getLabelsForSecretsAndJobs(key) ml[constant.ClusterAccountLabelKey] = username diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index d6810e785..217f7920e 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -34,7 +34,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/tools/record" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" @@ -128,6 +127,30 @@ func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { return b.Complete(r) } +func (r *BackupReconciler) getBackupPolicyAndValidate( + reqCtx intctrlutil.RequestCtx, + backup *dataprotectionv1alpha1.Backup) (*dataprotectionv1alpha1.BackupPolicy, error) { + // get referenced backup policy + backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} + backupPolicyNameSpaceName := types.NamespacedName{ + Namespace: reqCtx.Req.Namespace, + Name: backup.Spec.BackupPolicyName, + } + if err := r.Get(reqCtx.Ctx, backupPolicyNameSpaceName, backupPolicy); err != nil { + return nil, err + } + + if len(backupPolicy.Name) == 0 { + return nil, fmt.Errorf("backup policy %s not found", backupPolicyNameSpaceName) + } + + // validate backup spec + if err := backup.Spec.Validate(backupPolicy); err != nil { + return nil, err + } + return backupPolicy, nil +} + func (r *BackupReconciler) doNewPhaseAction( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) (ctrl.Result, error) { @@ -143,30 +166,28 @@ func (r *BackupReconciler) doNewPhaseAction( return intctrlutil.Reconciled() } - // update labels - backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} - backupPolicyNameSpaceName := types.NamespacedName{ - Namespace: reqCtx.Req.Namespace, - Name: backup.Spec.BackupPolicyName, - } - if err := r.Get(reqCtx.Ctx, backupPolicyNameSpaceName, backupPolicy); err != nil { - r.Recorder.Eventf(backup, corev1.EventTypeWarning, "CreatingBackup", - "Unable to get backupPolicy for backup %s.", backupPolicyNameSpaceName) + backupPolicy, err := r.getBackupPolicyAndValidate(reqCtx, backup) + if err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) } - if backupPolicy.Status.Phase != dataprotectionv1alpha1.ConfigAvailable { - if backupPolicy.Status.Phase == dataprotectionv1alpha1.ConfigFailed { - // REVIEW/TODO: need avoid using dynamic error string, this is bad for - // error type checking (errors.Is) - err := fmt.Errorf("backupPolicy %s status is failed", backupPolicy.Name) - return r.updateStatusIfFailed(reqCtx, backup, err) + + // TODO: get pod with matching labels to do backup. + var targetCluster dataprotectionv1alpha1.TargetCluster + switch backup.Spec.BackupType { + case dataprotectionv1alpha1.BackupTypeSnapshot: + targetCluster = backupPolicy.Spec.Snapshot.Target + default: + commonPolicy := backupPolicy.Spec.GetCommonPolicy(backup.Spec.BackupType) + if commonPolicy == nil { + return r.updateStatusIfFailed(reqCtx, backup, fmt.Errorf("backup type %s not supported", backup.Spec.BackupType)) } - // requeue to wait backupPolicy available - return intctrlutil.RequeueAfter(reconcileInterval, reqCtx.Log, "") + // save the backup message for restore + backup.Status.RemoteVolume = &commonPolicy.RemoteVolume + backup.Status.BackupToolName = commonPolicy.BackupToolName + targetCluster = commonPolicy.Target } - // TODO: get pod with matching labels to do backup. - target, err := r.getTargetPod(reqCtx, backup, backupPolicy.Spec.Target.LabelsSelector.MatchLabels) + target, err := r.getTargetPod(reqCtx, backup, targetCluster.LabelsSelector.MatchLabels) if err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) } @@ -177,19 +198,15 @@ func (r *BackupReconciler) doNewPhaseAction( return intctrlutil.Reconciled() } - // save the backup message for restore - backup.Status.RemoteVolume = &backupPolicy.Spec.RemoteVolume - backup.Status.BackupToolName = backupPolicy.Spec.BackupToolName - // update Phase to InProgress backup.Status.Phase = dataprotectionv1alpha1.BackupInProgress backup.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now().UTC()} - if backup.Spec.TTL != nil { + if backupPolicy.Spec.TTL != nil { backup.Status.Expiration = &metav1.Time{ - Time: backup.Status.StartTimestamp.Add(backup.Spec.TTL.Duration), + Time: backup.Status.StartTimestamp.Add(dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.TTL)), } } - if err := r.Client.Status().Patch(reqCtx.Ctx, backup, patch); err != nil { + if err = r.Client.Status().Patch(reqCtx.Ctx, backup, patch); err != nil { return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } return intctrlutil.Reconciled() @@ -198,24 +215,29 @@ func (r *BackupReconciler) doNewPhaseAction( func (r *BackupReconciler) doInProgressPhaseAction( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) (ctrl.Result, error) { + backupPolicy, err := r.getBackupPolicyAndValidate(reqCtx, backup) + if err != nil { + return r.updateStatusIfFailed(reqCtx, backup, err) + } patch := client.MergeFrom(backup.DeepCopy()) if backup.Spec.BackupType == dataprotectionv1alpha1.BackupTypeSnapshot { // 1. create and ensure pre-command job completed // 2. create and ensure volume snapshot ready // 3. create and ensure post-command job completed - isOK, err := r.createPreCommandJobAndEnsure(reqCtx, backup) + isOK, err := r.createPreCommandJobAndEnsure(reqCtx, backup, backupPolicy.Spec.Snapshot) if err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) } if !isOK { return intctrlutil.RequeueAfter(reconcileInterval, reqCtx.Log, "") } - if err = r.createUpdatesJobs(reqCtx, backup, dataprotectionv1alpha1.PRE); err != nil { + if err = r.createUpdatesJobs(reqCtx, backup, backupPolicy.Spec.Snapshot, dataprotectionv1alpha1.PRE); err != nil { r.Recorder.Event(backup, corev1.EventTypeNormal, "CreatedPreUpdatesJob", err.Error()) } - if err = r.createVolumeSnapshot(reqCtx, backup); err != nil { + if err = r.createVolumeSnapshot(reqCtx, backup, backupPolicy.Spec.Snapshot); err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) } + key := types.NamespacedName{Namespace: reqCtx.Req.Namespace, Name: backup.Name} isOK, err = r.ensureVolumeSnapshotReady(reqCtx, key) if err != nil { @@ -227,7 +249,7 @@ func (r *BackupReconciler) doInProgressPhaseAction( msg := fmt.Sprintf("Created volumeSnapshot %s ready.", key.Name) r.Recorder.Event(backup, corev1.EventTypeNormal, "CreatedVolumeSnapshot", msg) - isOK, err = r.createPostCommandJobAndEnsure(reqCtx, backup) + isOK, err = r.createPostCommandJobAndEnsure(reqCtx, backup, backupPolicy.Spec.Snapshot) if err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) } @@ -236,7 +258,7 @@ func (r *BackupReconciler) doInProgressPhaseAction( } // Failure MetadataCollectionJob does not affect the backup status. - if err = r.createUpdatesJobs(reqCtx, backup, dataprotectionv1alpha1.POST); err != nil { + if err = r.createUpdatesJobs(reqCtx, backup, backupPolicy.Spec.Snapshot, dataprotectionv1alpha1.POST); err != nil { r.Recorder.Event(backup, corev1.EventTypeNormal, "CreatedPostUpdatesJob", err.Error()) } @@ -250,7 +272,12 @@ func (r *BackupReconciler) doInProgressPhaseAction( } else { // 1. create and ensure backup tool job finished // 2. get job phase and update - err := r.createBackupToolJob(reqCtx, backup) + commonPolicy := backupPolicy.Spec.GetCommonPolicy(backup.Spec.BackupType) + if commonPolicy == nil { + // TODO: add error type + return r.updateStatusIfFailed(reqCtx, backup, fmt.Errorf("not found the %s policy", backup.Spec.BackupType)) + } + err = r.createBackupToolJob(reqCtx, backup, commonPolicy) if err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) } @@ -314,7 +341,7 @@ func (r *BackupReconciler) updateStatusIfFailed(reqCtx intctrlutil.RequestCtx, return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } -// patchBackupLabelsAndAnnotations patch backup labels and the annotations include cluster snapshot. +// patchBackupLabelsAndAnnotations patches backup labels and the annotations include cluster snapshot. func (r *BackupReconciler) patchBackupLabelsAndAnnotations( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, @@ -344,9 +371,10 @@ func (r *BackupReconciler) patchBackupLabelsAndAnnotations( } func (r *BackupReconciler) createPreCommandJobAndEnsure(reqCtx intctrlutil.RequestCtx, - backup *dataprotectionv1alpha1.Backup) (bool, error) { + backup *dataprotectionv1alpha1.Backup, + snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy) (bool, error) { - emptyCmd, err := r.ensureEmptyHooksCommand(reqCtx, backup, true) + emptyCmd, err := r.ensureEmptyHooksCommand(snapshotPolicy, true) if err != nil { return false, err } @@ -357,16 +385,17 @@ func (r *BackupReconciler) createPreCommandJobAndEnsure(reqCtx intctrlutil.Reque mgrNS := viper.GetString(constant.CfgKeyCtrlrMgrNS) key := types.NamespacedName{Namespace: mgrNS, Name: backup.Name + "-pre"} - if err := r.createHooksCommandJob(reqCtx, backup, key, true); err != nil { + if err := r.createHooksCommandJob(reqCtx, backup, snapshotPolicy, key, true); err != nil { return false, err } return r.ensureBatchV1JobCompleted(reqCtx, key) } func (r *BackupReconciler) createPostCommandJobAndEnsure(reqCtx intctrlutil.RequestCtx, - backup *dataprotectionv1alpha1.Backup) (bool, error) { + backup *dataprotectionv1alpha1.Backup, + snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy) (bool, error) { - emptyCmd, err := r.ensureEmptyHooksCommand(reqCtx, backup, false) + emptyCmd, err := r.ensureEmptyHooksCommand(snapshotPolicy, false) if err != nil { return false, err } @@ -377,7 +406,7 @@ func (r *BackupReconciler) createPostCommandJobAndEnsure(reqCtx intctrlutil.Requ mgrNS := viper.GetString(constant.CfgKeyCtrlrMgrNS) key := types.NamespacedName{Namespace: mgrNS, Name: backup.Name + "-post"} - if err := r.createHooksCommandJob(reqCtx, backup, key, false); err != nil { + if err = r.createHooksCommandJob(reqCtx, backup, snapshotPolicy, key, false); err != nil { return false, err } return r.ensureBatchV1JobCompleted(reqCtx, key) @@ -405,7 +434,8 @@ func (r *BackupReconciler) ensureBatchV1JobCompleted( func (r *BackupReconciler) createVolumeSnapshot( reqCtx intctrlutil.RequestCtx, - backup *dataprotectionv1alpha1.Backup) error { + backup *dataprotectionv1alpha1.Backup, + snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy) error { snap := &snapshotv1.VolumeSnapshot{} exists, err := intctrlutil.CheckResourceExists(reqCtx.Ctx, r.Client, reqCtx.Req.NamespacedName, snap) @@ -429,7 +459,7 @@ func (r *BackupReconciler) createVolumeSnapshot( return err } - targetPVCs, err := r.getTargetPVCs(reqCtx, backup, backupPolicy.Spec.Target.LabelsSelector.MatchLabels) + targetPVCs, err := r.getTargetPVCs(reqCtx, backup, snapshotPolicy.Target.LabelsSelector.MatchLabels) if err != nil { return err } @@ -493,7 +523,9 @@ func (r *BackupReconciler) ensureVolumeSnapshotReady(reqCtx intctrlutil.RequestC } func (r *BackupReconciler) createUpdatesJobs(reqCtx intctrlutil.RequestCtx, - backup *dataprotectionv1alpha1.Backup, stage dataprotectionv1alpha1.BackupStatusUpdateStage) error { + backup *dataprotectionv1alpha1.Backup, + snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy, + stage dataprotectionv1alpha1.BackupStatusUpdateStage) error { // get backup policy backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} backupPolicyNameSpaceName := types.NamespacedName{ @@ -504,11 +536,11 @@ func (r *BackupReconciler) createUpdatesJobs(reqCtx intctrlutil.RequestCtx, reqCtx.Log.V(1).Error(err, "Unable to get backupPolicy for backup.", "backupPolicy", backupPolicyNameSpaceName) return err } - for _, update := range backupPolicy.Spec.BackupStatusUpdates { + for _, update := range snapshotPolicy.BackupStatusUpdates { if update.UpdateStage != stage { continue } - if err := r.createMetadataCollectionJob(reqCtx, backup, update); err != nil { + if err := r.createMetadataCollectionJob(reqCtx, backup, snapshotPolicy.BasePolicy, update); err != nil { return err } } @@ -516,7 +548,9 @@ func (r *BackupReconciler) createUpdatesJobs(reqCtx intctrlutil.RequestCtx, } func (r *BackupReconciler) createMetadataCollectionJob(reqCtx intctrlutil.RequestCtx, - backup *dataprotectionv1alpha1.Backup, updateInfo dataprotectionv1alpha1.BackupStatusUpdate) error { + backup *dataprotectionv1alpha1.Backup, + basePolicy dataprotectionv1alpha1.BasePolicy, + updateInfo dataprotectionv1alpha1.BackupStatusUpdate) error { mgrNS := viper.GetString(constant.CfgKeyCtrlrMgrNS) key := types.NamespacedName{Namespace: mgrNS, Name: backup.Name + "-" + strings.ToLower(updateInfo.Path)} job := &batchv1.Job{} @@ -528,7 +562,7 @@ func (r *BackupReconciler) createMetadataCollectionJob(reqCtx intctrlutil.Reques } // build job and create - jobPodSpec, err := r.buildMetadataCollectionPodSpec(reqCtx, backup, updateInfo) + jobPodSpec, err := r.buildMetadataCollectionPodSpec(reqCtx, backup, basePolicy, updateInfo) if err != nil { return err } @@ -542,7 +576,8 @@ func (r *BackupReconciler) createMetadataCollectionJob(reqCtx intctrlutil.Reques func (r *BackupReconciler) createBackupToolJob( reqCtx intctrlutil.RequestCtx, - backup *dataprotectionv1alpha1.Backup) error { + backup *dataprotectionv1alpha1.Backup, + commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) error { key := types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name} job := batchv1.Job{} @@ -555,7 +590,7 @@ func (r *BackupReconciler) createBackupToolJob( return nil } - toolPodSpec, err := r.buildBackupToolPodSpec(reqCtx, backup) + toolPodSpec, err := r.buildBackupToolPodSpec(reqCtx, backup, commonPolicy) if err != nil { return err } @@ -570,38 +605,16 @@ func (r *BackupReconciler) createBackupToolJob( // ensureEmptyHooksCommand determines whether it has empty commands in the hooks func (r *BackupReconciler) ensureEmptyHooksCommand( - reqCtx intctrlutil.RequestCtx, - backup *dataprotectionv1alpha1.Backup, + snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy, preCommand bool) (bool, error) { - - // get backup policy - backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} - backupPolicyKey := types.NamespacedName{ - Namespace: reqCtx.Req.Namespace, - Name: backup.Spec.BackupPolicyName, - } - - policyExists, err := intctrlutil.CheckResourceExists(reqCtx.Ctx, r.Client, backupPolicyKey, backupPolicy) - if err != nil { - msg := fmt.Sprintf("Failed to get backupPolicy %s .", backupPolicyKey.Name) - r.Recorder.Event(backup, corev1.EventTypeWarning, "BackupPolicyFailed", msg) - return false, err - } - - if !policyExists { - msg := fmt.Sprintf("Not Found backupPolicy %s .", backupPolicyKey.Name) - r.Recorder.Event(backup, corev1.EventTypeWarning, "BackupPolicyFailed", msg) - return false, errors.New(msg) - } - // return true directly, means hooks commands is empty, skip subsequent hook jobs. - if backupPolicy.Spec.Hooks == nil { + if snapshotPolicy.Hooks == nil { return true, nil } - commands := backupPolicy.Spec.Hooks.PostCommands + commands := snapshotPolicy.Hooks.PostCommands if preCommand { - commands = backupPolicy.Spec.Hooks.PreCommands + commands = snapshotPolicy.Hooks.PreCommands } if len(commands) == 0 { return true, nil @@ -612,6 +625,7 @@ func (r *BackupReconciler) ensureEmptyHooksCommand( func (r *BackupReconciler) createHooksCommandJob( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, + snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy, key types.NamespacedName, preCommand bool) error { @@ -625,7 +639,7 @@ func (r *BackupReconciler) createHooksCommandJob( return nil } - jobPodSpec, err := r.buildSnapshotPodSpec(reqCtx, backup, preCommand) + jobPodSpec, err := r.buildSnapshotPodSpec(reqCtx, backup, snapshotPolicy, preCommand) if err != nil { return err } @@ -750,6 +764,9 @@ func (r *BackupReconciler) deleteExternalResources(reqCtx intctrlutil.RequestCtx return nil } +// getTargetPod gets the target pod by label selector. +// if the backup has obtained the target pod from label selector, it will be set to the annotations. +// then get the pod from this annotation to ensure that the same pod is picked in following up . func (r *BackupReconciler) getTargetPod(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, labels map[string]string) (*corev1.Pod, error) { if targetPodName, ok := backup.Annotations[dataProtectionBackupTargetPodKey]; ok { @@ -814,34 +831,22 @@ func (r *BackupReconciler) getTargetPVCs(reqCtx intctrlutil.RequestCtx, return allPVCs, nil } -func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) (corev1.PodSpec, error) { +func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, + backup *dataprotectionv1alpha1.Backup, + commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) (corev1.PodSpec, error) { podSpec := corev1.PodSpec{} - logger := reqCtx.Log - - // get backup policy - backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} - backupPolicyNameSpaceName := types.NamespacedName{ - Namespace: reqCtx.Req.Namespace, - Name: backup.Spec.BackupPolicyName, - } - - if err := r.Get(reqCtx.Ctx, backupPolicyNameSpaceName, backupPolicy); err != nil { - logger.Error(err, "Unable to get backupPolicy for backup.", "backupPolicy", backupPolicyNameSpaceName) - return podSpec, err - } - // get backup tool backupTool := &dataprotectionv1alpha1.BackupTool{} backupToolNameSpaceName := types.NamespacedName{ Namespace: reqCtx.Req.Namespace, - Name: backupPolicy.Spec.BackupToolName, + Name: commonPolicy.BackupToolName, } if err := r.Client.Get(reqCtx.Ctx, backupToolNameSpaceName, backupTool); err != nil { - logger.Error(err, "Unable to get backupTool for backup.", "BackupTool", backupToolNameSpaceName) + reqCtx.Log.Error(err, "Unable to get backupTool for backup.", "BackupTool", backupToolNameSpaceName) return podSpec, err } - - clusterPod, err := r.getTargetPod(reqCtx, backup, backupPolicy.Spec.Target.LabelsSelector.MatchLabels) + // TODO: check if pvc exists + clusterPod, err := r.getTargetPod(reqCtx, backup, commonPolicy.Target.LabelsSelector.MatchLabels) if err != nil { return podSpec, err } @@ -860,18 +865,6 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, Value: strings.Join(hostDNS, "."), } - envDBUser := corev1.EnvVar{ - Name: "DB_USER", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: backupPolicy.Spec.Target.Secret.Name, - }, - Key: backupPolicy.Spec.Target.Secret.UserKeyword, - }, - }, - } - container := corev1.Container{} container.Name = backup.Name container.Command = []string{"sh", "-c"} @@ -884,10 +877,10 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, remoteBackupPath := "/backupdata" // TODO(dsj): mount multi remote backup volumes - randomVolumeName := fmt.Sprintf("%s-%s", backupPolicy.Spec.RemoteVolume.Name, rand.String(6)) - backupPolicy.Spec.RemoteVolume.Name = randomVolumeName + remoteVolumeName := fmt.Sprintf("backup-%s", commonPolicy.RemoteVolume.Name) + commonPolicy.RemoteVolume.Name = remoteVolumeName remoteVolumeMount := corev1.VolumeMount{ - Name: randomVolumeName, + Name: remoteVolumeName, MountPath: remoteBackupPath, } container.VolumeMounts = clusterPod.Spec.Containers[0].VolumeMounts @@ -898,18 +891,6 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, AllowPrivilegeEscalation: &allowPrivilegeEscalation, RunAsUser: &runAsUser} - envDBPassword := corev1.EnvVar{ - Name: "DB_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: backupPolicy.Spec.Target.Secret.Name, - }, - Key: backupPolicy.Spec.Target.Secret.PasswordKeyword, - }, - }, - } - envBackupName := corev1.EnvVar{ Name: "BACKUP_NAME", Value: backup.Name, @@ -929,14 +910,41 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, Value: remoteBackupPath + "/$(BACKUP_DIR_PREFIX)", } - container.Env = []corev1.EnvVar{envDBHost, envDBUser, envDBPassword, envBackupName, envBackupDirPrefix, envBackupDir} + container.Env = []corev1.EnvVar{envDBHost, envBackupName, envBackupDirPrefix, envBackupDir} + if commonPolicy.Target.Secret != nil { + envDBUser := corev1.EnvVar{ + Name: "DB_USER", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: commonPolicy.Target.Secret.Name, + }, + Key: commonPolicy.Target.Secret.UsernameKey, + }, + }, + } + + envDBPassword := corev1.EnvVar{ + Name: "DB_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: commonPolicy.Target.Secret.Name, + }, + Key: commonPolicy.Target.Secret.PasswordKey, + }, + }, + } + container.Env = append(container.Env, envDBUser, envDBPassword) + } + // merge env from backup tool. container.Env = append(container.Env, backupTool.Spec.Env...) podSpec.Containers = []corev1.Container{container} podSpec.Volumes = clusterPod.Spec.Volumes - podSpec.Volumes = append(podSpec.Volumes, backupPolicy.Spec.RemoteVolume) + podSpec.Volumes = append(podSpec.Volumes, commonPolicy.RemoteVolume) podSpec.RestartPolicy = corev1.RestartPolicyNever // the pod of job needs to be scheduled on the same node as the workload pod, because it needs to share one pvc @@ -949,22 +957,11 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, func (r *BackupReconciler) buildSnapshotPodSpec( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, + snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy, preCommand bool) (corev1.PodSpec, error) { podSpec := corev1.PodSpec{} - logger := reqCtx.Log - - // get backup policy - backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} - backupPolicyNameSpaceName := types.NamespacedName{ - Namespace: reqCtx.Req.Namespace, - Name: backup.Spec.BackupPolicyName, - } - if err := r.Get(reqCtx.Ctx, backupPolicyNameSpaceName, backupPolicy); err != nil { - logger.Error(err, "Unable to get backupPolicy for backup.", "backupPolicy", backupPolicyNameSpaceName) - return podSpec, err - } - clusterPod, err := r.getTargetPod(reqCtx, backup, backupPolicy.Spec.Target.LabelsSelector.MatchLabels) + clusterPod, err := r.getTargetPod(reqCtx, backup, snapshotPolicy.Target.LabelsSelector.MatchLabels) if err != nil { return podSpec, err } @@ -972,13 +969,13 @@ func (r *BackupReconciler) buildSnapshotPodSpec( container := corev1.Container{} container.Name = backup.Name container.Command = []string{"kubectl", "exec", "-n", backup.Namespace, - "-i", clusterPod.Name, "-c", backupPolicy.Spec.Hooks.ContainerName, "--", "sh", "-c"} + "-i", clusterPod.Name, "-c", snapshotPolicy.Hooks.ContainerName, "--", "sh", "-c"} if preCommand { - container.Args = backupPolicy.Spec.Hooks.PreCommands + container.Args = snapshotPolicy.Hooks.PreCommands } else { - container.Args = backupPolicy.Spec.Hooks.PostCommands + container.Args = snapshotPolicy.Hooks.PostCommands } - container.Image = backupPolicy.Spec.Hooks.Image + container.Image = snapshotPolicy.Hooks.Image if container.Image == "" { container.Image = viper.GetString(constant.KBToolsImage) container.ImagePullPolicy = corev1.PullPolicy(viper.GetString(constant.KBImagePullPolicy)) @@ -1031,22 +1028,10 @@ func addTolerations(podSpec *corev1.PodSpec) (err error) { func (r *BackupReconciler) buildMetadataCollectionPodSpec( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, + basePolicy dataprotectionv1alpha1.BasePolicy, updateInfo dataprotectionv1alpha1.BackupStatusUpdate) (corev1.PodSpec, error) { podSpec := corev1.PodSpec{} - logger := reqCtx.Log - - // get backup policy - backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} - backupPolicyNameSpaceName := types.NamespacedName{ - Namespace: reqCtx.Req.Namespace, - Name: backup.Spec.BackupPolicyName, - } - - if err := r.Get(reqCtx.Ctx, backupPolicyNameSpaceName, backupPolicy); err != nil { - logger.Error(err, "Unable to get backupPolicy for backup.", "backupPolicy", backupPolicyNameSpaceName) - return podSpec, err - } - targetPod, err := r.getTargetPod(reqCtx, backup, backupPolicy.Spec.Target.LabelsSelector.MatchLabels) + targetPod, err := r.getTargetPod(reqCtx, backup, basePolicy.Target.LabelsSelector.MatchLabels) if err != nil { return podSpec, err } diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index 6d640e5c7..13f72f1ac 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -43,7 +43,7 @@ var _ = Describe("Backup Controller test", func() { const backupRemoteVolumeName = "backup-remote-volume" const backupRemotePVCName = "backup-remote-pvc" const defaultSchedule = "0 3 * * *" - const defaultTTL = "168h0m0s" + const defaultTTL = "7d" const backupName = "test-backup-job" viper.SetDefault(constant.CfgKeyCtrlrMgrNS, testCtx.DefaultNamespace) @@ -129,14 +129,19 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backupPolicy from backupTool: " + backupTool.Name) _ = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). - SetBackupToolName(backupTool.Name). - SetSchedule(defaultSchedule). SetTTL(defaultTTL). + AddSnapshotPolicy(). + SetSchedule(defaultSchedule, true). AddMatchLabels(constant.AppInstanceLabelKey, clusterName). AddMatchLabels(constant.RoleLabelKey, "leader"). SetTargetSecretName(clusterName). AddHookPreCommand("touch /data/mysql/.restore;sync"). AddHookPostCommand("rm -f /data/mysql/.restore;sync"). + AddFullPolicy(). + SetBackupToolName(backupTool.Name). + AddMatchLabels(constant.AppInstanceLabelKey, clusterName). + AddMatchLabels(constant.RoleLabelKey, "leader"). + SetTargetSecretName(clusterName). SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). Create(&testCtx).GetObject() }) @@ -147,7 +152,6 @@ var _ = Describe("Backup Controller test", func() { BeforeEach(func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dataprotectionv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() @@ -198,7 +202,6 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dataprotectionv1alpha1.BackupTypeSnapshot). Create(&testCtx).GetObject() @@ -297,7 +300,6 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dataprotectionv1alpha1.BackupTypeSnapshot). Create(&testCtx).GetObject() @@ -312,7 +314,6 @@ var _ = Describe("Backup Controller test", func() { It("should fail without pvc", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dataprotectionv1alpha1.BackupTypeSnapshot). Create(&testCtx).GetObject() @@ -343,19 +344,17 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backupPolicy from backupTool: " + backupTool.Name) _ = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). + AddFullPolicy(). SetBackupToolName(backupTool.Name). - SetSchedule(defaultSchedule). + SetSchedule(defaultSchedule, true). SetTTL(defaultTTL). AddMatchLabels(constant.AppInstanceLabelKey, clusterName). SetTargetSecretName(clusterName). - AddHookPreCommand("touch /data/mysql/.restore;sync"). - AddHookPostCommand("rm -f /data/mysql/.restore;sync"). SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). Create(&testCtx).GetObject() By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dataprotectionv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() @@ -378,7 +377,6 @@ var _ = Describe("Backup Controller test", func() { BeforeEach(func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dataprotectionv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() @@ -410,7 +408,7 @@ func patchVolumeSnapshotStatus(key types.NamespacedName, readyToUse bool) { func patchBackupPolicySpecBackupStatusUpdates(key types.NamespacedName) { Eventually(testapps.GetAndChangeObj(&testCtx, key, func(fetched *dataprotectionv1alpha1.BackupPolicy) { - fetched.Spec.BackupStatusUpdates = []dataprotectionv1alpha1.BackupStatusUpdate{ + fetched.Spec.Snapshot.BackupStatusUpdates = []dataprotectionv1alpha1.BackupStatusUpdate{ { Path: "manifests.backupLog", ContainerName: "postgresql", diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index 3698baab9..ea5b12f3f 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -18,16 +18,16 @@ package dataprotection import ( "context" - "embed" "encoding/json" "fmt" + "reflect" "sort" - "time" "github.com/leaanthony/debme" "github.com/spf13/viper" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -38,7 +38,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" @@ -51,30 +50,10 @@ type BackupPolicyReconciler struct { Recorder record.EventRecorder } -type backupPolicyOptions struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - MgrNamespace string `json:"mgrNamespace"` - Cluster string `json:"cluster"` - Schedule string `json:"schedule"` - BackupType string `json:"backupType"` - TTL *metav1.Duration `json:"ttl,omitempty"` - ServiceAccount string `json:"serviceAccount"` -} - -var ( - //go:embed cue/* - cueTemplates embed.FS -) - // +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicies,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicies/status,verbs=get;update;patch // +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicies/finalizers,verbs=update -// +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicytemplates,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicytemplates/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicytemplates/finalizers,verbs=update - // +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs/status,verbs=get // +kubebuilder:rbac:groups=batch,resources=cronjobs/finalizers,verbs=update;patch @@ -113,235 +92,192 @@ func (r *BackupPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request return *res, err } - switch backupPolicy.Status.Phase { - case "", dataprotectionv1alpha1.ConfigNew: - return r.doNewPhaseAction(reqCtx, backupPolicy) - case dataprotectionv1alpha1.ConfigInProgress: - return r.doInProgressPhaseAction(reqCtx, backupPolicy) - case dataprotectionv1alpha1.ConfigAvailable: - return r.doAvailablePhaseAction(reqCtx, backupPolicy) - default: - return intctrlutil.Reconciled() + // try to remove expired or oldest backups, triggered by cronjob controller + if err = r.removeExpiredBackups(reqCtx); err != nil { + return r.patchStatusFailed(reqCtx, backupPolicy, "RemoveExpiredBackupsFailed", err) } -} -func (r *BackupPolicyReconciler) doNewPhaseAction( - reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) (ctrl.Result, error) { - // update status phase - patch := client.MergeFrom(backupPolicy.DeepCopy()) - backupPolicy.Status.Phase = dataprotectionv1alpha1.ConfigInProgress - if err := r.Client.Status().Patch(reqCtx.Ctx, backupPolicy, patch); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + if err = r.handleSnapshotPolicy(reqCtx, backupPolicy); err != nil { + return r.patchStatusFailed(reqCtx, backupPolicy, "HandleSnapshotPolicyFailed", err) } - return intctrlutil.RequeueAfter(reconcileInterval, reqCtx.Log, "") -} -func (r *BackupPolicyReconciler) doInProgressPhaseAction( - reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) (ctrl.Result, error) { - // update default value from viper config if necessary - patch := client.MergeFrom(backupPolicy.DeepCopy()) - - if len(backupPolicy.Spec.Schedule) == 0 { - schedule := viper.GetString("DP_BACKUP_SCHEDULE") - if len(schedule) > 0 { - backupPolicy.Spec.Schedule = schedule - } - } - if backupPolicy.Spec.TTL == nil { - ttlString := viper.GetString("DP_BACKUP_TTL") - if len(ttlString) > 0 { - ttl, err := time.ParseDuration(ttlString) - if err == nil { - backupPolicy.Spec.TTL = &metav1.Duration{Duration: ttl} - } - } - } - for k, v := range backupPolicy.Spec.Target.LabelsSelector.MatchLabels { - if backupPolicy.Labels == nil { - backupPolicy.SetLabels(map[string]string{}) - } - backupPolicy.Labels[k] = v + if err = r.handleFullPolicy(reqCtx, backupPolicy); err != nil { + return r.patchStatusFailed(reqCtx, backupPolicy, "HandleFullPolicyFailed", err) } - if backupPolicy.Spec.Target.Secret == nil { - backupPolicy.Spec.Target.Secret = &dataprotectionv1alpha1.BackupPolicySecret{} + if err = r.handleIncrementalPolicy(reqCtx, backupPolicy); err != nil { + return r.patchStatusFailed(reqCtx, backupPolicy, "HandleIncrementalPolicyFailed", err) } - // merge backup policy template spec - if err := r.mergeBackupPolicyTemplate(reqCtx, backupPolicy); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } + return r.patchStatusAvailable(reqCtx, backupPolicy) +} - if err := r.fillSecretName(reqCtx, backupPolicy, true); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - // fill remaining fields - r.fillDefaultValueIfRequired(backupPolicy) +// SetupWithManager sets up the controller with the Manager. +func (r *BackupPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&dataprotectionv1alpha1.BackupPolicy{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: viper.GetInt(maxConcurDataProtectionReconKey), + }). + Complete(r) +} - if err := r.Client.Patch(reqCtx.Ctx, backupPolicy, patch); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } +func (r *BackupPolicyReconciler) deleteExternalResources(reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { + // delete cronjob resource + cronjob := &batchv1.CronJob{} - // if backup policy is available, try to remove expired or oldest backups - if backupPolicy.Status.Phase == dataprotectionv1alpha1.ConfigAvailable { - if err := r.removeExpiredBackups(reqCtx); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + for _, v := range []dataprotectionv1alpha1.BackupType{dataprotectionv1alpha1.BackupTypeFull, + dataprotectionv1alpha1.BackupTypeIncremental, dataprotectionv1alpha1.BackupTypeSnapshot} { + key := types.NamespacedName{ + Namespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), + Name: r.getCronJobName(backupPolicy.Name, backupPolicy.Namespace, v), } - if err := r.removeOldestBackups(reqCtx, backupPolicy); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + if err := r.Client.Get(reqCtx.Ctx, key, cronjob); err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err } - return intctrlutil.Reconciled() - } - // create cronjob from cue template. - if err := r.createCronJobIfNeeded(reqCtx, backupPolicy); err != nil { - r.Recorder.Eventf(backupPolicy, corev1.EventTypeWarning, "CreatingBackupPolicy", - "Failed to create cronjob %s.", err.Error()) - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - - // update status phase - backupPolicy.Status.Phase = dataprotectionv1alpha1.ConfigAvailable - if err := r.Client.Status().Patch(reqCtx.Ctx, backupPolicy, patch); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + // TODO: checks backupPolicy's uuid to ensure the cronjob is created by this backupPolicy + if err := r.removeCronJobFinalizer(reqCtx, cronjob); err != nil { + return err + } + if err := intctrlutil.BackgroundDeleteObject(r.Client, reqCtx.Ctx, cronjob); err != nil { + // failed delete k8s job, return error info. + return err + } } - return intctrlutil.RequeueAfter(reconcileInterval, reqCtx.Log, "") + return nil } -func (r *BackupPolicyReconciler) doAvailablePhaseAction( - reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) (ctrl.Result, error) { - // patch cronjob if backup policy spec patched - if err := r.patchCronJob(reqCtx, backupPolicy); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - - // try to remove expired or oldest backups, triggered by cronjob controller - if err := r.removeExpiredBackups(reqCtx); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - if err := r.removeOldestBackups(reqCtx, backupPolicy); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") +// patchStatusAvailable patches backup policy status phase to available. +func (r *BackupPolicyReconciler) patchStatusAvailable(reqCtx intctrlutil.RequestCtx, + backupPolicy *dataprotectionv1alpha1.BackupPolicy) (ctrl.Result, error) { + // update status phase + if backupPolicy.Status.Phase != dataprotectionv1alpha1.PolicyAvailable || + backupPolicy.Status.ObservedGeneration != backupPolicy.Generation { + patch := client.MergeFrom(backupPolicy.DeepCopy()) + backupPolicy.Status.Phase = dataprotectionv1alpha1.PolicyAvailable + backupPolicy.Status.FailureReason = "" + if err := r.Client.Status().Patch(reqCtx.Ctx, backupPolicy, patch); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } } return intctrlutil.Reconciled() } -func (r *BackupPolicyReconciler) mergeBackupPolicyTemplate( - reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - if backupPolicy.Spec.BackupPolicyTemplateName == "" { - return nil - } - template := &dataprotectionv1alpha1.BackupPolicyTemplate{} - key := types.NamespacedName{Namespace: backupPolicy.Namespace, Name: backupPolicy.Spec.BackupPolicyTemplateName} - if err := r.Client.Get(reqCtx.Ctx, key, template); err != nil { - r.Recorder.Eventf(backupPolicy, corev1.EventTypeWarning, "BackupPolicyTemplateFailed", - "Failed to get backupPolicyTemplateName: %s, reason: %s", key.Name, err.Error()) - return err - } - - if backupPolicy.Spec.BackupToolName == "" { - backupPolicy.Spec.BackupToolName = template.Spec.BackupToolName +// patchStatusFailed patches backup policy status phase to failed. +func (r *BackupPolicyReconciler) patchStatusFailed(reqCtx intctrlutil.RequestCtx, + backupPolicy *dataprotectionv1alpha1.BackupPolicy, + reason string, + err error) (ctrl.Result, error) { + backupPolicyDeepCopy := backupPolicy.DeepCopy() + backupPolicy.Status.Phase = dataprotectionv1alpha1.PolicyFailed + backupPolicy.Status.FailureReason = err.Error() + if !reflect.DeepEqual(backupPolicy.Status, backupPolicyDeepCopy.Status) { + if patchErr := r.Client.Status().Patch(reqCtx.Ctx, backupPolicy, client.MergeFrom(backupPolicyDeepCopy)); patchErr != nil { + return intctrlutil.RequeueWithError(patchErr, reqCtx.Log, "") + } } + r.Recorder.Event(backupPolicy, corev1.EventTypeWarning, reason, err.Error()) + return intctrlutil.RequeueWithError(err, reqCtx.Log, "") +} - // if template.Spec.CredentialKeyword is nil, use system account; else use root conn secret - useSysAcct := template.Spec.CredentialKeyword == nil - if err := r.fillSecretName(reqCtx, backupPolicy, useSysAcct); err != nil { +func (r *BackupPolicyReconciler) removeExpiredBackups(reqCtx intctrlutil.RequestCtx) error { + backups := dataprotectionv1alpha1.BackupList{} + if err := r.Client.List(reqCtx.Ctx, &backups, + client.InNamespace(reqCtx.Req.Namespace)); err != nil { return err } - - if template.Spec.CredentialKeyword != nil { - if backupPolicy.Spec.Target.Secret.UserKeyword == "" { - backupPolicy.Spec.Target.Secret.UserKeyword = template.Spec.CredentialKeyword.UserKeyword + now := metav1.Now() + for _, item := range backups.Items { + // ignore retained backup. + if item.GetLabels()[constant.BackupProtectionLabelKey] == constant.BackupRetain { + continue } - if backupPolicy.Spec.Target.Secret.PasswordKeyword == "" { - backupPolicy.Spec.Target.Secret.PasswordKeyword = template.Spec.CredentialKeyword.PasswordKeyword + if item.Status.Expiration != nil && item.Status.Expiration.Before(&now) { + if err := intctrlutil.BackgroundDeleteObject(r.Client, reqCtx.Ctx, &item); err != nil { + // failed delete backups, return error info. + return err + } } } - if backupPolicy.Spec.TTL == nil { - backupPolicy.Spec.TTL = template.Spec.TTL - } - if backupPolicy.Spec.Schedule == "" { - backupPolicy.Spec.Schedule = template.Spec.Schedule - } - if backupPolicy.Spec.Hooks == nil { - backupPolicy.Spec.Hooks = template.Spec.Hooks - } - if backupPolicy.Spec.OnFailAttempted == 0 { - backupPolicy.Spec.OnFailAttempted = template.Spec.OnFailAttempted - } - if len(backupPolicy.Spec.BackupStatusUpdates) == 0 { - backupPolicy.Spec.BackupStatusUpdates = template.Spec.BackupStatusUpdates - } return nil } -func (r *BackupPolicyReconciler) fillDefaultValueIfRequired(backupPolicy *dataprotectionv1alpha1.BackupPolicy) { - // set required parameter default values if template is empty - if backupPolicy.Spec.Target.Secret.UserKeyword == "" { - backupPolicy.Spec.Target.Secret.UserKeyword = "username" - } - if backupPolicy.Spec.Target.Secret.PasswordKeyword == "" { - backupPolicy.Spec.Target.Secret.PasswordKeyword = "password" - } -} - -// fillSecretName fills secret name if it is empty. -// If BackupPolicy.Sect.Target.Secret is not nil, use secret specified in BackupPolicy. -// Otherwise, lookup BackupPolicyTemplate and check if username and password are specified. -// If so, use root connection secret; otherwise, try system account before root connection. -func (r *BackupPolicyReconciler) fillSecretName(reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy, useSysAccount bool) error { - if len(backupPolicy.Spec.Target.Secret.Name) > 0 { +// removeOldestBackups removes old backups according to backupsHistoryLimit policy. +func (r *BackupPolicyReconciler) removeOldestBackups(reqCtx intctrlutil.RequestCtx, + backupPolicyName string, + backupType dataprotectionv1alpha1.BackupType, + backupsHistoryLimit int32) error { + if backupsHistoryLimit == 0 { return nil } - // get cluster name from labels - instanceName := backupPolicy.Spec.Target.LabelsSelector.MatchLabels[constant.AppInstanceLabelKey] - if len(instanceName) == 0 { - // REVIEW/TODO: need avoid using dynamic error string, this is bad for - // error type checking (errors.Is) - return fmt.Errorf("failed to get instance name from labels: %v", backupPolicy.Spec.Target.LabelsSelector.MatchLabels) - } - var labels map[string]string - if useSysAccount { - labels = map[string]string{ - constant.AppInstanceLabelKey: instanceName, - constant.ClusterAccountLabelKey: (string)(appsv1alpha1.DataprotectionAccount), - } - } else { - labels = map[string]string{ - constant.AppInstanceLabelKey: instanceName, - constant.AppManagedByLabelKey: constant.AppName, - } + matchLabels := map[string]string{ + dataProtectionLabelBackupPolicyKey: backupPolicyName, + dataProtectionLabelBackupTypeKey: string(backupType), + dataProtectionLabelAutoBackupKey: "true", } - - secrets := corev1.SecretList{} - if err := r.Client.List(reqCtx.Ctx, &secrets, client.MatchingLabels(labels)); err != nil { + backups := dataprotectionv1alpha1.BackupList{} + if err := r.Client.List(reqCtx.Ctx, &backups, + client.InNamespace(reqCtx.Req.Namespace), + client.MatchingLabels(matchLabels)); err != nil { return err } - if len(secrets.Items) > 0 { - backupPolicy.Spec.Target.Secret.Name = secrets.Items[0].GetName() + // filter final state backups only + backupItems := []dataprotectionv1alpha1.Backup{} + for _, item := range backups.Items { + if item.Status.Phase == dataprotectionv1alpha1.BackupCompleted || + item.Status.Phase == dataprotectionv1alpha1.BackupFailed { + backupItems = append(backupItems, item) + } + } + numToDelete := len(backupItems) - int(backupsHistoryLimit) + if numToDelete <= 0 { return nil } - // REVIEW/TODO: need avoid using dynamic error string, this is bad for - // error type checking (errors.Is) - return fmt.Errorf("no secret found for backup policy %s", backupPolicy.GetName()) + sort.Sort(byBackupStartTime(backupItems)) + for i := 0; i < numToDelete; i++ { + if err := intctrlutil.BackgroundDeleteObject(r.Client, reqCtx.Ctx, &backupItems[i]); err != nil { + // failed delete backups, return error info. + return err + } + } + return nil +} + +func (r *BackupPolicyReconciler) getCronJobName(backupPolicyName, backupPolicyNamespace string, backupType dataprotectionv1alpha1.BackupType) string { + return fmt.Sprintf("%s-%s-%s", backupPolicyName, backupPolicyNamespace, string(backupType)) } -func (r *BackupPolicyReconciler) buildCronJob(backupPolicy *dataprotectionv1alpha1.BackupPolicy) (*batchv1.CronJob, error) { +// buildCronJob builds cronjob from backup policy. +func (r *BackupPolicyReconciler) buildCronJob( + backupPolicy *dataprotectionv1alpha1.BackupPolicy, + target dataprotectionv1alpha1.TargetCluster, + cronExpression string, + backType dataprotectionv1alpha1.BackupType) (*batchv1.CronJob, error) { tplFile := "cronjob.cue" cueFS, _ := debme.FS(cueTemplates, "cue") cueTpl, err := intctrlutil.NewCUETplFromBytes(cueFS.ReadFile(tplFile)) if err != nil { return nil, err } + var ttl metav1.Duration + if backupPolicy.Spec.TTL != nil { + ttl = metav1.Duration{Duration: dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.TTL)} + } cueValue := intctrlutil.NewCUEBuilder(*cueTpl) options := backupPolicyOptions{ - Name: backupPolicy.Name, - Namespace: backupPolicy.Namespace, - Cluster: backupPolicy.Spec.Target.LabelsSelector.MatchLabels[constant.AppInstanceLabelKey], - Schedule: backupPolicy.Spec.Schedule, - TTL: backupPolicy.Spec.TTL, - BackupType: backupPolicy.Spec.BackupType, - ServiceAccount: viper.GetString("KUBEBLOCKS_SERVICEACCOUNT_NAME"), - MgrNamespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), + Name: r.getCronJobName(backupPolicy.Name, backupPolicy.Namespace, backType), + BackupPolicyName: backupPolicy.Name, + Namespace: backupPolicy.Namespace, + Cluster: target.LabelsSelector.MatchLabels[constant.AppInstanceLabelKey], + Schedule: cronExpression, + TTL: ttl, + BackupType: string(backType), + ServiceAccount: viper.GetString("KUBEBLOCKS_SERVICEACCOUNT_NAME"), + MgrNamespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), } backupPolicyOptionsByte, err := json.Marshal(options) if err != nil { @@ -356,12 +292,12 @@ func (r *BackupPolicyReconciler) buildCronJob(backupPolicy *dataprotectionv1alph return nil, err } - cronjob := batchv1.CronJob{} - if err = json.Unmarshal(cronjobByte, &cronjob); err != nil { + cronjob := &batchv1.CronJob{} + if err = json.Unmarshal(cronjobByte, cronjob); err != nil { return nil, err } - controllerutil.AddFinalizer(&cronjob, dataProtectionFinalizerName) + controllerutil.AddFinalizer(cronjob, dataProtectionFinalizerName) // set labels for k, v := range backupPolicy.Labels { @@ -370,144 +306,118 @@ func (r *BackupPolicyReconciler) buildCronJob(backupPolicy *dataprotectionv1alph } cronjob.Labels[k] = v } - return &cronjob, nil + cronjob.Labels[dataProtectionLabelBackupPolicyKey] = backupPolicy.Name + cronjob.Labels[dataProtectionLabelBackupTypeKey] = string(backType) + return cronjob, nil } -func (r *BackupPolicyReconciler) removeExpiredBackups(reqCtx intctrlutil.RequestCtx) error { - backups := dataprotectionv1alpha1.BackupList{} - if err := r.Client.List(reqCtx.Ctx, &backups, - client.InNamespace(reqCtx.Req.Namespace)); err != nil { +func (r *BackupPolicyReconciler) removeCronJobFinalizer(reqCtx intctrlutil.RequestCtx, cronjob *batchv1.CronJob) error { + patch := client.MergeFrom(cronjob.DeepCopy()) + controllerutil.RemoveFinalizer(cronjob, dataProtectionFinalizerName) + return r.Patch(reqCtx.Ctx, cronjob, patch) +} + +// reconcileCronJob will create/delete/patch cronjob according to cronExpression and policy changes. +func (r *BackupPolicyReconciler) reconcileCronJob(reqCtx intctrlutil.RequestCtx, + backupPolicy *dataprotectionv1alpha1.BackupPolicy, + basePolicy dataprotectionv1alpha1.BasePolicy, + cronExpression string, + backType dataprotectionv1alpha1.BackupType) error { + cronjobProto, err := r.buildCronJob(backupPolicy, basePolicy.Target, cronExpression, backType) + if err != nil { return err } - now := metav1.Now() - for _, item := range backups.Items { - // ignore retained backup. - if item.GetLabels()[constant.BackupProtectionLabelKey] == constant.BackupRetain { - continue - } - if item.Status.Expiration != nil && item.Status.Expiration.Before(&now) { - if err := intctrlutil.BackgroundDeleteObject(r.Client, reqCtx.Ctx, &item); err != nil { - // failed delete backups, return error info. + cronJob := &batchv1.CronJob{} + if err = r.Client.Get(reqCtx.Ctx, client.ObjectKey{Name: cronjobProto.Name, + Namespace: cronjobProto.Namespace}, cronJob); err != nil && !apierrors.IsNotFound(err) { + return err + } + + if len(cronExpression) == 0 { + if len(cronJob.Name) != 0 { + // delete the old cronjob. + if err = r.removeCronJobFinalizer(reqCtx, cronJob); err != nil { return err } + return r.Client.Delete(reqCtx.Ctx, cronJob) } + // if no cron expression, return + return nil } - return nil -} -func buildBackupLabelsForRemove(backupPolicy *dataprotectionv1alpha1.BackupPolicy) map[string]string { - return map[string]string{ - constant.AppInstanceLabelKey: backupPolicy.Labels[constant.AppInstanceLabelKey], - dataProtectionLabelAutoBackupKey: "true", + if len(cronJob.Name) == 0 { + // if no cronjob, create it. + return r.Client.Create(reqCtx.Ctx, cronjobProto) } + // sync the cronjob with the current backup policy configuration. + patch := client.MergeFrom(cronJob.DeepCopy()) + cronJob.Spec.JobTemplate.Spec.BackoffLimit = &basePolicy.OnFailAttempted + cronJob.Spec.JobTemplate.Spec.Template = cronjobProto.Spec.JobTemplate.Spec.Template + cronJob.Spec.Schedule = cronExpression + return r.Client.Patch(reqCtx.Ctx, cronJob, patch) } -func (r *BackupPolicyReconciler) removeOldestBackups(reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - if backupPolicy.Spec.BackupsHistoryLimit == 0 { - return nil - } - - backups := dataprotectionv1alpha1.BackupList{} - if err := r.Client.List(reqCtx.Ctx, &backups, - client.InNamespace(reqCtx.Req.Namespace), - client.MatchingLabels(buildBackupLabelsForRemove(backupPolicy))); err != nil { +// handlePolicy the common function to handle backup policy. +func (r *BackupPolicyReconciler) handlePolicy(reqCtx intctrlutil.RequestCtx, + backupPolicy *dataprotectionv1alpha1.BackupPolicy, + basePolicy dataprotectionv1alpha1.BasePolicy, + cronExpression string, + backType dataprotectionv1alpha1.BackupType) error { + // create/delete/patch cronjob workload + if err := r.reconcileCronJob(reqCtx, backupPolicy, basePolicy, + cronExpression, backType); err != nil { return err } - // filter final state backups only - backupItems := []dataprotectionv1alpha1.Backup{} - for _, item := range backups.Items { - if item.Status.Phase == dataprotectionv1alpha1.BackupCompleted || - item.Status.Phase == dataprotectionv1alpha1.BackupFailed { - backupItems = append(backupItems, item) - } - } - numToDelete := len(backupItems) - int(backupPolicy.Spec.BackupsHistoryLimit) - if numToDelete <= 0 { - return nil - } - sort.Sort(byBackupStartTime(backupItems)) - for i := 0; i < numToDelete; i++ { - if err := intctrlutil.BackgroundDeleteObject(r.Client, reqCtx.Ctx, &backupItems[i]); err != nil { - // failed delete backups, return error info. - return err - } - } - return nil + return r.removeOldestBackups(reqCtx, backupPolicy.Name, backType, basePolicy.BackupsHistoryLimit) } -// SetupWithManager sets up the controller with the Manager. -func (r *BackupPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&dataprotectionv1alpha1.BackupPolicy{}). - WithOptions(controller.Options{ - MaxConcurrentReconciles: viper.GetInt(maxConcurDataProtectionReconKey), - }). - Complete(r) -} - -func (r *BackupPolicyReconciler) deleteExternalResources(reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - // delete cronjob resource - cronjob := &batchv1.CronJob{} - - key := types.NamespacedName{ - Namespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), - Name: backupPolicy.Name, - } - if err := r.Client.Get(reqCtx.Ctx, key, cronjob); err != nil { - return client.IgnoreNotFound(err) - } - if controllerutil.ContainsFinalizer(cronjob, dataProtectionFinalizerName) { - patch := client.MergeFrom(cronjob.DeepCopy()) - controllerutil.RemoveFinalizer(cronjob, dataProtectionFinalizerName) - if err := r.Patch(reqCtx.Ctx, cronjob, patch); err != nil { - return err - } +// handleSnapshotPolicy handles snapshot policy. +func (r *BackupPolicyReconciler) handleSnapshotPolicy( + reqCtx intctrlutil.RequestCtx, + backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { + if backupPolicy.Spec.Snapshot == nil { + // TODO delete cronjob if exists + return nil } - if err := intctrlutil.BackgroundDeleteObject(r.Client, reqCtx.Ctx, cronjob); err != nil { - // failed delete k8s job, return error info. - return err + var cronExpression string + schedule := backupPolicy.Spec.Schedule.BaseBackup + if schedule != nil && schedule.Enable && schedule.Type == dataprotectionv1alpha1.BaseBackupTypeSnapshot { + cronExpression = schedule.CronExpression } - - return nil + return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Snapshot.BasePolicy, + cronExpression, dataprotectionv1alpha1.BackupTypeSnapshot) } -// createCronJobIfNeeded create cronjob spec if backup policy set schedule -func (r *BackupPolicyReconciler) createCronJobIfNeeded( +// handleFullPolicy handles full policy. +func (r *BackupPolicyReconciler) handleFullPolicy( reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - if backupPolicy.Spec.Schedule == "" { - r.Recorder.Eventf(backupPolicy, corev1.EventTypeNormal, "BackupPolicy", - "Backups will not be automatically scheduled due to lack of schedule configuration.") + if backupPolicy.Spec.Full == nil { + // TODO delete cronjob if exists return nil } - - // create cronjob from cue template. - cronjob, err := r.buildCronJob(backupPolicy) - if err != nil { - return err + var cronExpression string + schedule := backupPolicy.Spec.Schedule.BaseBackup + if schedule != nil && schedule.Enable && schedule.Type == dataprotectionv1alpha1.BaseBackupTypeFull { + cronExpression = schedule.CronExpression } - if err = r.Client.Create(reqCtx.Ctx, cronjob); err != nil { - // ignore already exists. - return client.IgnoreAlreadyExists(err) - } - return nil + return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Full.BasePolicy, + cronExpression, dataprotectionv1alpha1.BackupTypeFull) } -// patchCronJob patch cronjob spec if backup policy patched -func (r *BackupPolicyReconciler) patchCronJob( +// handleIncrementalPolicy handles incremental policy. +func (r *BackupPolicyReconciler) handleIncrementalPolicy( reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - - cronJob := &batchv1.CronJob{} - if err := r.Client.Get(reqCtx.Ctx, reqCtx.Req.NamespacedName, cronJob); err != nil { - return client.IgnoreNotFound(err) + if backupPolicy.Spec.Incremental == nil { + // TODO delete cronjob if exists + return nil } - patch := client.MergeFrom(cronJob.DeepCopy()) - cronJob, err := r.buildCronJob(backupPolicy) - if err != nil { - return err + var cronExpression string + schedule := backupPolicy.Spec.Schedule.Incremental + if schedule != nil && schedule.Enable { + cronExpression = schedule.CronExpression } - cronJob.Spec.Schedule = backupPolicy.Spec.Schedule - cronJob.Spec.JobTemplate.Spec.BackoffLimit = &backupPolicy.Spec.OnFailAttempted - return r.Client.Patch(reqCtx.Ctx, cronJob, patch) + return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Incremental.BasePolicy, + cronExpression, dataprotectionv1alpha1.BackupTypeIncremental) } diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index ca29c20cf..fc5a38187 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -17,7 +17,7 @@ limitations under the License. package dataprotection import ( - "context" + "fmt" "time" . "github.com/onsi/ginkgo/v2" @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/generics" @@ -43,16 +42,14 @@ var _ = Describe("Backup Policy Controller", func() { const containerName = "mysql" const defaultPVCSize = "1Gi" const backupPolicyName = "test-backup-policy" - const backupPolicyTplName = "test-backup-policy-template" + // const backupPolicyTplName = "test-backup-policy-template" const backupRemoteVolumeName = "backup-remote-volume" const backupRemotePVCName = "backup-remote-pvc" const defaultSchedule = "0 3 * * *" - const defaultTTL = "168h0m0s" + const defaultTTL = "7d" const backupNamePrefix = "test-backup-job-" const mgrNamespace = "kube-system" - viper.SetDefault("DP_BACKUP_SCHEDULE", "0 3 * * *") - viper.SetDefault("DP_BACKUP_TTL", "168h0m0s") viper.SetDefault(constant.CfgKeyCtrlrMgrNS, testCtx.DefaultNamespace) cleanEnv := func() { @@ -72,13 +69,13 @@ var _ = Describe("Backup Policy Controller", func() { testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.JobSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.CronJobSignature, inNS, ml) + testapps.ClearResources(&testCtx, intctrlutil.SecretSignature, inNS, ml) testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS, ml) // mgr namespaced inMgrNS := client.InNamespace(mgrNamespace) testapps.ClearResources(&testCtx, intctrlutil.CronJobSignature, inMgrNS, ml) // non-namespaced testapps.ClearResources(&testCtx, intctrlutil.BackupToolSignature, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, ml) } BeforeEach(func() { @@ -110,15 +107,15 @@ var _ = Describe("Backup Policy Controller", func() { When("creating backup policy with default settings", func() { var backupToolName string - var cronjobKey types.NamespacedName - - BeforeEach(func() { - viper.Set(constant.CfgKeyCtrlrMgrNS, mgrNamespace) - cronjobKey = types.NamespacedName{ - Name: backupPolicyName, + getCronjobKey := func(backupType dpv1alpha1.BackupType) types.NamespacedName { + return types.NamespacedName{ + Name: fmt.Sprintf("%s-%s-%s", backupPolicyName, testCtx.DefaultNamespace, backupType), Namespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), } + } + BeforeEach(func() { + viper.Set(constant.CfgKeyCtrlrMgrNS, mgrNamespace) By("By creating a backupTool") backupTool := testapps.CreateCustomizedObj(&testCtx, "backup/backuptool.yaml", &dpv1alpha1.BackupTool{}, testapps.RandomizedObjName()) @@ -135,9 +132,10 @@ var _ = Describe("Backup Policy Controller", func() { BeforeEach(func() { By("By creating a backupPolicy from backupTool: " + backupToolName) backupPolicy = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). + AddFullPolicy(). SetBackupToolName(backupToolName). SetBackupsHistoryLimit(1). - SetSchedule(defaultSchedule). + SetSchedule(defaultSchedule, true). SetTTL(defaultTTL). AddMatchLabels(constant.AppInstanceLabelKey, clusterName). SetTargetSecretName(clusterName). @@ -149,9 +147,9 @@ var _ = Describe("Backup Policy Controller", func() { }) It("should success", func() { Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) + g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) })).Should(Succeed()) - Eventually(testapps.CheckObj(&testCtx, cronjobKey, func(g Gomega, fetched *batchv1.CronJob) { + Eventually(testapps.CheckObj(&testCtx, getCronjobKey(dpv1alpha1.BackupTypeFull), func(g Gomega, fetched *batchv1.CronJob) { g.Expect(fetched.Spec.Schedule).To(Equal(defaultSchedule)) })).Should(Succeed()) }) @@ -165,28 +163,26 @@ var _ = Describe("Backup Policy Controller", func() { } autoBackupLabel := map[string]string{ - constant.AppInstanceLabelKey: backupPolicy.Labels[constant.AppInstanceLabelKey], - dataProtectionLabelAutoBackupKey: "true", + dataProtectionLabelAutoBackupKey: "true", + dataProtectionLabelBackupPolicyKey: backupPolicyName, + dataProtectionLabelBackupTypeKey: string(dpv1alpha1.BaseBackupTypeFull), } By("create a expired backup") backupExpired := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dpv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() By("create 1st limit backup") backupOutLimit1 := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dpv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() By("create 2nd limit backup") backupOutLimit2 := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dpv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() @@ -229,11 +225,11 @@ var _ = Describe("Backup Policy Controller", func() { patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupOutLimit2)) // trigger the backup policy controller through update cronjob - patchCronJobStatus(cronjobKey) + patchCronJobStatus(getCronjobKey(dpv1alpha1.BackupTypeFull)) By("retain the latest backup") Eventually(testapps.GetListLen(&testCtx, intctrlutil.BackupSignature, - client.MatchingLabels(backupPolicy.Spec.Target.LabelsSelector.MatchLabels), + client.MatchingLabels(backupPolicy.Spec.Full.Target.LabelsSelector.MatchLabels), client.InNamespace(backupPolicy.Namespace))).Should(Equal(1)) }) }) @@ -255,13 +251,7 @@ var _ = Describe("Backup Policy Controller", func() { }) It("should success", func() { Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) - })).Should(Succeed()) - }) - It("should success with empty viper config", func() { - viper.SetDefault("DP_BACKUP_SCHEDULE", "") - Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) + g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) })).Should(Succeed()) }) }) @@ -272,8 +262,9 @@ var _ = Describe("Backup Policy Controller", func() { BeforeEach(func() { By("By creating a backupPolicy from backupTool: " + backupToolName) backupPolicy = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). + AddSnapshotPolicy(). SetBackupToolName(backupToolName). - SetSchedule("invalid schedule"). + SetSchedule("invalid schedule", true). AddMatchLabels(constant.AppInstanceLabelKey, clusterName). SetTargetSecretName(clusterName). AddHookPreCommand("touch /data/mysql/.restore;sync"). @@ -284,127 +275,17 @@ var _ = Describe("Backup Policy Controller", func() { }) It("should failed", func() { Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).NotTo(Equal(dpv1alpha1.ConfigAvailable)) + g.Expect(fetched.Status.Phase).NotTo(Equal(dpv1alpha1.PolicyAvailable)) })).Should(Succeed()) }) }) - Context("creates a backup policy with backup policy template", func() { - var backupPolicyKey types.NamespacedName - var backupPolicy *dpv1alpha1.BackupPolicy - BeforeEach(func() { - viper.SetDefault("DP_BACKUP_SCHEDULE", nil) - viper.SetDefault("DP_BACKUP_TTL", nil) - By("By creating a backupPolicyTemplate") - template := testapps.NewBackupPolicyTemplateFactory(backupPolicyTplName). - SetBackupToolName(backupToolName). - SetSchedule(defaultSchedule). - SetTTL(defaultTTL). - SetCredentialKeyword("username", "password"). - AddHookPreCommand("touch /data/mysql/.restore;sync"). - AddHookPostCommand("rm -f /data/mysql/.restore;sync"). - Create(&testCtx).GetObject() - - By("By creating a backupPolicy from backupTool: " + backupToolName) - backupPolicy = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). - SetBackupPolicyTplName(template.Name). - AddMatchLabels(constant.AppInstanceLabelKey, clusterName). - SetTargetSecretName(clusterName). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). - Create(&testCtx).GetObject() - backupPolicyKey = client.ObjectKeyFromObject(backupPolicy) - }) - It("should success", func() { - Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) - })).Should(Succeed()) - }) - }) - - Context("creates a backup policy with nil pointer credentialKeyword in backupPolicyTemplate", func() { - var backupPolicyKey types.NamespacedName - var backupPolicy *dpv1alpha1.BackupPolicy - BeforeEach(func() { - viper.SetDefault("DP_BACKUP_SCHEDULE", nil) - viper.SetDefault("DP_BACKUP_TTL", nil) - By("By creating a backupPolicyTemplate") - template := testapps.NewBackupPolicyTemplateFactory(backupPolicyTplName). - SetBackupToolName(backupToolName). - SetSchedule(defaultSchedule). - SetTTL(defaultTTL). - Create(&testCtx).GetObject() - - By("By creating a backupPolicy from backupTool: " + backupToolName) - backupPolicy = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). - SetBackupPolicyTplName(template.Name). - AddMatchLabels(constant.AppInstanceLabelKey, clusterName). - SetTargetSecretName(clusterName). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). - Create(&testCtx).GetObject() - backupPolicyKey = client.ObjectKeyFromObject(backupPolicy) - }) - It("should success", func() { - Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) - })).Should(Succeed()) - }) - }) - - Context("creates a backup policy with empty secret", func() { - var ( - backupSecretName = "backup-secret" - rootSecretName = "root-secret" - secretsMap map[string]*corev1.Secret - ) - - // delete secrets before test starts - cleanSecrets := func() { - // delete rest mocked objects - inNS := client.InNamespace(testCtx.DefaultNamespace) - ml := client.HasLabels{testCtx.TestObjLabelKey} - // delete secret created for backup policy - testapps.ClearResources(&testCtx, intctrlutil.SecretSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, inNS, ml) - } - - fakeSecret := func(name string, labels map[string]string) *corev1.Secret { - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: testCtx.DefaultNamespace, - Labels: labels, - }, - } - } - - BeforeEach(func() { - secretsMap = make(map[string]*corev1.Secret) - // mock two secrets for backup policy, one for backup account, one for root conn - secretsMap[backupSecretName] = fakeSecret(backupSecretName, map[string]string{ - constant.AppInstanceLabelKey: clusterName, - constant.ClusterAccountLabelKey: (string)(appsv1alpha1.DataprotectionAccount), - }) - secretsMap[rootSecretName] = fakeSecret(rootSecretName, map[string]string{ - constant.AppInstanceLabelKey: clusterName, - constant.AppManagedByLabelKey: constant.AppName, - }) - - cleanSecrets() - }) - - AfterEach(cleanSecrets) - + Context("creating a backupPolicy with secret", func() { It("creating a backupPolicy with secret", func() { - // create two secrets - for _, v := range secretsMap { - Expect(testCtx.CreateObj(context.Background(), v)).Should(Succeed()) - Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKeyFromObject(v), &corev1.Secret{}, true)).Should(Succeed()) - } - By("By creating a backupPolicy with empty secret") randomSecretName := testCtx.GetRandomStr() backupPolicy := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). + AddFullPolicy(). SetBackupToolName(backupToolName). AddMatchLabels(constant.AppInstanceLabelKey, clusterName). SetTargetSecretName(randomSecretName). @@ -413,94 +294,8 @@ var _ = Describe("Backup Policy Controller", func() { SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName).Create(&testCtx).GetObject() backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) - g.Expect(fetched.Spec.Target.Secret.Name).To(Equal(randomSecretName)) - })).Should(Succeed()) - }) - - It("creating a backupPolicy with secrets missing", func() { - By("By creating a backupPolicy with empty secret") - backupPolicy := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). - SetBackupToolName(backupToolName). - AddMatchLabels(constant.AppInstanceLabelKey, clusterName). - AddHookPreCommand("touch /data/mysql/.restore;sync"). - AddHookPostCommand("rm -f /data/mysql/.restore;sync"). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName).Create(&testCtx).GetObject() - backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) - By("Secrets missing, the backup policy should never be `ConfigAvailable`") - Consistently(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).NotTo(Equal(dpv1alpha1.ConfigAvailable)) - }), time.Millisecond*100).Should(Succeed()) - }) - - It("creating a backupPolicy uses default secret", func() { - // create two secrets - for _, v := range secretsMap { - Expect(testCtx.CreateObj(context.Background(), v)).Should(Succeed()) - Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKeyFromObject(v), &corev1.Secret{}, true)).Should(Succeed()) - } - - By("By creating a backupPolicy with empty secret") - backupPolicy := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). - SetBackupToolName(backupToolName). - AddMatchLabels(constant.AppInstanceLabelKey, clusterName). - AddHookPreCommand("touch /data/mysql/.restore;sync"). - AddHookPostCommand("rm -f /data/mysql/.restore;sync"). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName).Create(&testCtx).GetObject() - backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) - Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) - g.Expect(fetched.Spec.Target.Secret.Name).To(Equal(backupSecretName)) - })).Should(Succeed()) - }) - - It("create backup policy with tempate and specify credential keyword", func() { - for _, v := range secretsMap { - Expect(testCtx.CreateObj(context.Background(), v)).Should(Succeed()) - Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKeyFromObject(v), &corev1.Secret{}, true)).Should(Succeed()) - } - // create template - template := testapps.NewBackupPolicyTemplateFactory(backupPolicyTplName). - SetBackupToolName(backupToolName). - SetCredentialKeyword("username", "password"). - Create(&testCtx).GetObject() - - // create backup policy - By("By creating a backupPolicy from backupTool: " + backupToolName) - backupPolicy := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). - SetBackupPolicyTplName(template.Name). - AddMatchLabels(constant.AppInstanceLabelKey, clusterName). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). - Create(&testCtx).GetObject() - backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) - Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) - g.Expect(fetched.Spec.Target.Secret.Name).To(Equal(rootSecretName)) - })).Should(Succeed()) - }) - - It("create backup policy with tempate but without default credential keyword", func() { - // create two secrets - for _, v := range secretsMap { - Expect(testCtx.CreateObj(context.Background(), v)).Should(Succeed()) - Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKeyFromObject(v), &corev1.Secret{}, true)).Should(Succeed()) - } - // create template - template := testapps.NewBackupPolicyTemplateFactory(backupPolicyTplName). - SetBackupToolName(backupToolName). - Create(&testCtx).GetObject() - - // create backup policy - By("By creating a backupPolicy from backupTool: " + backupToolName) - backupPolicy := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). - SetBackupPolicyTplName(template.Name). - AddMatchLabels(constant.AppInstanceLabelKey, clusterName). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). - Create(&testCtx).GetObject() - backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) - Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) - g.Expect(fetched.Spec.Target.Secret.Name).To(Equal(backupSecretName)) + g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) + g.Expect(fetched.Spec.Full.Target.Secret.Name).To(Equal(randomSecretName)) })).Should(Succeed()) }) }) diff --git a/controllers/dataprotection/cronjob_controller.go b/controllers/dataprotection/cronjob_controller.go index e4ebac321..8e000d6c5 100644 --- a/controllers/dataprotection/cronjob_controller.go +++ b/controllers/dataprotection/cronjob_controller.go @@ -67,7 +67,7 @@ func (r *CronJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct backupPolicyKey := types.NamespacedName{ Namespace: cronJob.Annotations["kubeblocks.io/backup-namespace"], - Name: cronJob.Name, + Name: cronJob.Labels[dataProtectionLabelBackupPolicyKey], } if err = r.Client.Get(reqCtx.Ctx, backupPolicyKey, backupPolicy); err != nil { return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") diff --git a/controllers/dataprotection/cue/cronjob.cue b/controllers/dataprotection/cue/cronjob.cue index 823c6a0f4..d43674418 100644 --- a/controllers/dataprotection/cue/cronjob.cue +++ b/controllers/dataprotection/cue/cronjob.cue @@ -13,14 +13,15 @@ // limitations under the License. options: { - name: string - namespace: string - mgrNamespace: string - cluster: string - schedule: string - backupType: string - ttl: string - serviceAccount: string + name: string + backupPolicyName: string + namespace: string + mgrNamespace: string + cluster: string + schedule: string + backupType: string + ttl: string + serviceAccount: string } cronjob: { @@ -63,9 +64,8 @@ metadata: name: backup-\(options.namespace)-\(options.cluster)-$(date -u +'%Y%m%d%H%M%S') namespace: \(options.namespace) spec: - backupPolicyName: \(options.name) + backupPolicyName: \(options.backupPolicyName) backupType: \(options.backupType) - ttl: \(options.ttl) EOF """, ] diff --git a/controllers/dataprotection/restorejob_controller.go b/controllers/dataprotection/restorejob_controller.go index 1b0c09765..e0d164b4d 100644 --- a/controllers/dataprotection/restorejob_controller.go +++ b/controllers/dataprotection/restorejob_controller.go @@ -239,28 +239,21 @@ func (r *RestoreJobReconciler) buildPodSpec(reqCtx intctrlutil.RequestCtx, resto return podSpec, err } - // get backup policy - backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} - backupPolicyNameSpaceName := types.NamespacedName{ - Namespace: reqCtx.Req.Namespace, - Name: backup.Spec.BackupPolicyName, - } - if err := r.Client.Get(reqCtx.Ctx, backupPolicyNameSpaceName, backupPolicy); err != nil { - logger.Error(err, "Unable to get backupPolicy for backup.", "BackupPolicy", backupPolicyNameSpaceName) - return podSpec, err - } - // get backup tool backupTool := &dataprotectionv1alpha1.BackupTool{} backupToolNameSpaceName := types.NamespacedName{ Namespace: reqCtx.Req.Namespace, - Name: backupPolicy.Spec.BackupToolName, + Name: backup.Status.BackupToolName, } if err := r.Client.Get(reqCtx.Ctx, backupToolNameSpaceName, backupTool); err != nil { logger.Error(err, "Unable to get backupTool for backup.", "BackupTool", backupToolNameSpaceName) return podSpec, err } + if backup.Status.RemoteVolume == nil { + return podSpec, nil + } + container := corev1.Container{} container.Name = restoreJob.Name container.Command = []string{"sh", "-c"} @@ -274,7 +267,7 @@ func (r *RestoreJobReconciler) buildPodSpec(reqCtx intctrlutil.RequestCtx, resto // add remote volumeMounts remoteVolumeMount := corev1.VolumeMount{} - remoteVolumeMount.Name = backupPolicy.Spec.RemoteVolume.Name + remoteVolumeMount.Name = backup.Status.RemoteVolume.Name remoteVolumeMount.MountPath = "/data" container.VolumeMounts = append(container.VolumeMounts, remoteVolumeMount) @@ -299,7 +292,7 @@ func (r *RestoreJobReconciler) buildPodSpec(reqCtx intctrlutil.RequestCtx, resto podSpec.Volumes = restoreJob.Spec.TargetVolumes // add remote volumes - podSpec.Volumes = append(podSpec.Volumes, backupPolicy.Spec.RemoteVolume) + podSpec.Volumes = append(podSpec.Volumes, *backup.Status.RemoteVolume) // TODO(dsj): mount readonly remote volumes for restore. // podSpec.Volumes[0].PersistentVolumeClaim.ReadOnly = true diff --git a/controllers/dataprotection/restorejob_controller_test.go b/controllers/dataprotection/restorejob_controller_test.go index 35117828b..6fcdba409 100644 --- a/controllers/dataprotection/restorejob_controller_test.go +++ b/controllers/dataprotection/restorejob_controller_test.go @@ -28,12 +28,16 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/generics" testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) var _ = Describe("RestoreJob Controller", func() { - + const ( + clusterName = "mycluster" + compName = "cluster" + ) cleanEnv := func() { // must wait until resources deleted and no longer exist before the testcases start, // otherwise if later it needs to create some new resource objects with the same name, @@ -46,6 +50,7 @@ var _ = Describe("RestoreJob Controller", func() { ml := client.HasLabels{testCtx.TestObjLabelKey} // namespaced testapps.ClearResources(&testCtx, intctrlutil.StatefulSetSignature, inNS, ml) + testapps.ClearResources(&testCtx, intctrlutil.PodSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.RestoreJobSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) @@ -75,7 +80,6 @@ var _ = Describe("RestoreJob Controller", func() { return testapps.NewBackupFactory(testCtx.DefaultNamespace, "backup-job-"). WithRandomName().SetBackupPolicyName(backupPolicy). SetBackupType(dataprotectionv1alpha1.BackupTypeFull). - SetTTL("168h0m0s"). Create(&testCtx).GetObject() } @@ -83,10 +87,11 @@ var _ = Describe("RestoreJob Controller", func() { By("By assure an backupPolicy obj") return testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, "backup-policy-"). WithRandomName(). - SetSchedule("0 3 * * *"). - SetTTL("168h0m0s"). + AddFullPolicy(). + AddMatchLabels(constant.AppInstanceLabelKey, clusterName). + SetSchedule("0 3 * * *", true). + SetTTL("7d"). SetBackupToolName(backupTool). - SetBackupPolicyTplName("backup-config-mysql"). SetTargetSecretName("mycluster-cluster-secret"). SetRemoteVolumePVC("backup-remote-volume", "backup-host-path-pvc"). Create(&testCtx).GetObject() @@ -110,8 +115,9 @@ var _ = Describe("RestoreJob Controller", func() { assureStatefulSetObj := func() *appsv1.StatefulSet { By("By assure an stateful obj") - return testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, "mycluster", "mycluster", "replicasets"). - AddAppInstanceLabel("mycluster"). + return testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, clusterName, clusterName, compName). + SetReplicas(3). + AddAppInstanceLabel(clusterName). AddContainer(corev1.Container{Name: "mysql", Image: testapps.ApeCloudMySQLImage}). AddVolumeClaimTemplate(corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: testapps.DataVolumeName}, @@ -144,73 +150,51 @@ var _ = Describe("RestoreJob Controller", func() { Expect(k8sClient.Status().Patch(ctx, &k8sJob, patch)).Should(Succeed()) } + testRestoreJob := func(withResources ...bool) { + By("By creating a statefulset and pod") + sts := assureStatefulSetObj() + testapps.MockConsensusComponentPods(testCtx, sts, clusterName, compName) + + By("By creating a backupTool") + backupTool := assureBackupToolObj(withResources...) + + By("By creating a backupPolicy from backupTool: " + backupTool.Name) + backupPolicy := assureBackupPolicyObj(backupTool.Name) + + By("By creating a backup from backupPolicy: " + backupPolicy.Name) + backup := assureBackupObj(backupPolicy.Name) + + By("By creating a restoreJob from backup: " + backup.Name) + toCreate := assureRestoreJobObj(backup.Name) + key := types.NamespacedName{ + Name: toCreate.Name, + Namespace: toCreate.Namespace, + } + backupKey := types.NamespacedName{Name: backup.Name, Namespace: backup.Namespace} + Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dataprotectionv1alpha1.Backup) { + g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupInProgress)) + })).Should(Succeed()) + + patchBackupStatus(dataprotectionv1alpha1.BackupCompleted, backupKey) + + patchK8sJobStatus(batchv1.JobComplete, key) + + result := &dataprotectionv1alpha1.RestoreJob{} + Eventually(func() bool { + Expect(k8sClient.Get(ctx, key, result)).Should(Succeed()) + return result.Status.Phase == dataprotectionv1alpha1.RestoreJobCompleted || + result.Status.Phase == dataprotectionv1alpha1.RestoreJobFailed + }).Should(BeTrue()) + Expect(result.Status.Phase).Should(Equal(dataprotectionv1alpha1.RestoreJobCompleted)) + } + Context("When creating restoreJob", func() { It("Should success with no error", func() { - - By("By creating a statefulset") - _ = assureStatefulSetObj() - - By("By creating a backupTool") - backupTool := assureBackupToolObj() - - By("By creating a backupPolicy from backupTool: " + backupTool.Name) - backupPolicy := assureBackupPolicyObj(backupTool.Name) - - By("By creating a backup from backupPolicy: " + backupPolicy.Name) - backup := assureBackupObj(backupPolicy.Name) - - By("By creating a restoreJob from backup: " + backup.Name) - toCreate := assureRestoreJobObj(backup.Name) - key := types.NamespacedName{ - Name: toCreate.Name, - Namespace: toCreate.Namespace, - } - - patchBackupStatus(dataprotectionv1alpha1.BackupCompleted, types.NamespacedName{Name: backup.Name, Namespace: backup.Namespace}) - - patchK8sJobStatus(batchv1.JobComplete, types.NamespacedName{Name: toCreate.Name, Namespace: toCreate.Namespace}) - - result := &dataprotectionv1alpha1.RestoreJob{} - Eventually(func() bool { - Expect(k8sClient.Get(ctx, key, result)).Should(Succeed()) - return result.Status.Phase == dataprotectionv1alpha1.RestoreJobCompleted || - result.Status.Phase == dataprotectionv1alpha1.RestoreJobFailed - }).Should(BeTrue()) - Expect(result.Status.Phase).Should(Equal(dataprotectionv1alpha1.RestoreJobCompleted)) + testRestoreJob() }) It("Without backupTool resources should success with no error", func() { - - By("By creating a statefulset") - _ = assureStatefulSetObj() - - By("By creating a backupTool") - backupTool := assureBackupToolObj(true) - - By("By creating a backupPolicy from backupTool: " + backupTool.Name) - backupPolicy := assureBackupPolicyObj(backupTool.Name) - - By("By creating a backup from backupPolicy: " + backupPolicy.Name) - backup := assureBackupObj(backupPolicy.Name) - - By("By creating a restoreJob from backup: " + backup.Name) - toCreate := assureRestoreJobObj(backup.Name) - key := types.NamespacedName{ - Name: toCreate.Name, - Namespace: toCreate.Namespace, - } - - patchBackupStatus(dataprotectionv1alpha1.BackupCompleted, types.NamespacedName{Name: backup.Name, Namespace: backup.Namespace}) - - patchK8sJobStatus(batchv1.JobComplete, types.NamespacedName{Name: toCreate.Name, Namespace: toCreate.Namespace}) - - result := &dataprotectionv1alpha1.RestoreJob{} - Eventually(func() bool { - Expect(k8sClient.Get(ctx, key, result)).Should(Succeed()) - return result.Status.Phase == dataprotectionv1alpha1.RestoreJobCompleted || - result.Status.Phase == dataprotectionv1alpha1.RestoreJobFailed - }).Should(BeTrue()) - Expect(result.Status.Phase).Should(Equal(dataprotectionv1alpha1.RestoreJobCompleted)) + testRestoreJob(true) }) }) diff --git a/controllers/dataprotection/suite_test.go b/controllers/dataprotection/suite_test.go index 6358e66f4..b1d04d347 100644 --- a/controllers/dataprotection/suite_test.go +++ b/controllers/dataprotection/suite_test.go @@ -113,7 +113,6 @@ var _ = BeforeSuite(func() { Expect(k8sClient).NotTo(BeNil()) uncachedObjects := []client.Object{ - &dataprotectionv1alpha1.BackupPolicyTemplate{}, &dataprotectionv1alpha1.BackupPolicy{}, &dataprotectionv1alpha1.BackupTool{}, &dataprotectionv1alpha1.Backup{}, diff --git a/controllers/dataprotection/type.go b/controllers/dataprotection/type.go index 0d8c1840a..795aedd7f 100644 --- a/controllers/dataprotection/type.go +++ b/controllers/dataprotection/type.go @@ -17,10 +17,12 @@ limitations under the License. package dataprotection import ( + "embed" "runtime" "time" "github.com/spf13/viper" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -30,6 +32,7 @@ const ( maxConcurDataProtectionReconKey = "MAXCONCURRENTRECONCILES_DATAPROTECTION" // label keys + dataProtectionLabelBackupPolicyKey = "dataprotection.kubeblocks.io/backup-policy" dataProtectionLabelBackupTypeKey = "dataprotection.kubeblocks.io/backup-type" dataProtectionLabelAutoBackupKey = "dataprotection.kubeblocks.io/autobackup" dataProtectionLabelBackupNameKey = "backups.dataprotection.kubeblocks.io/name" @@ -45,3 +48,20 @@ var reconcileInterval = time.Second func init() { viper.SetDefault(maxConcurDataProtectionReconKey, runtime.NumCPU()*2) } + +var ( + //go:embed cue/* + cueTemplates embed.FS +) + +type backupPolicyOptions struct { + Name string `json:"name"` + BackupPolicyName string `json:"backupPolicyName"` + Namespace string `json:"namespace"` + MgrNamespace string `json:"mgrNamespace"` + Cluster string `json:"cluster"` + Schedule string `json:"schedule"` + BackupType string `json:"backupType"` + TTL metav1.Duration `json:"ttl,omitempty"` + ServiceAccount string `json:"serviceAccount"` +} diff --git a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml index 7c708ccc1..62bfdb543 100644 --- a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml @@ -1,21 +1,28 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-mysql-scale + name: apecloud-mysql-backup-policy-template labels: - clusterdefinition.kubeblocks.io/name: apecloud-mysql + clusterdefinition.kubeblocks.io/name: apecloud-mysql-scale {{- include "apecloud-mysql.labels" . | nindent 4 }} spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: xtrabackup-mysql-scale - ttl: 168h0m0s - hooks: - containerName: mysql - preCommands: - - "touch /data/mysql/data/.restore_new_cluster; sync" - postCommands: - - "rm -f /data/mysql/data/.restore_new_cluster; sync" - - credentialKeyword: - userKeyword: username - passwordKeyword: password + clusterDefinitionRef: apecloud-mysql-scale + backupPolicies: + - componentDefRef: mysql + ttl: 7d + schedule: + baseBackup: + type: snapshot + enable: false + cronExpression: "0 18 * * 0" + snapshot: + hooks: + containerName: mysql + preCommands: + - "touch /data/mysql/data/.restore_new_cluster; sync" + postCommands: + - "rm -f /data/mysql/data/.restore_new_cluster; sync" + target: + role: leader + full: + backupToolName: xtrabackup-apecloud-mysql-scale \ No newline at end of file diff --git a/deploy/apecloud-mysql-scale/templates/backuptool.yaml b/deploy/apecloud-mysql-scale/templates/backuptool.yaml index 26266043f..90dd00b73 100644 --- a/deploy/apecloud-mysql-scale/templates/backuptool.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuptool.yaml @@ -1,7 +1,7 @@ apiVersion: dataprotection.kubeblocks.io/v1alpha1 kind: BackupTool metadata: - name: xtrabackup-mysql-scale + name: xtrabackup-apecloud-mysql-scale labels: clusterdefinition.kubeblocks.io/name: apecloud-mysql {{- include "apecloud-mysql.labels" . | nindent 4 }} @@ -22,17 +22,21 @@ spec: restoreCommands: - > set -e; - mkdir -p /tmp/data/ && cd /tmp/data; - xbstream -x < /${BACKUP_DIR}/${BACKUP_NAME}.xbstream; - xtrabackup --decompress --target-dir=/tmp/data/; - xtrabackup --prepare --target-dir=/tmp/data/; - find . -name "*.qp"|xargs rm -f; - rm -rf ${DATA_DIR}/*; - rm -rf ${DATA_DIR}/.xtrabackup_restore_new_cluster; - xtrabackup --move-back --target-dir=/tmp/data/ --datadir=${DATA_DIR}/; - touch ${DATA_DIR}/.xtrabackup_restore_new_cluster; - rm -rf /tmp/data/; - chmod -R 0777 ${DATA_DIR}; + mkdir -p ${DATA_DIR} + res=`ls -A ${DATA_DIR}` + if [ ! -z ${res} ]; then + echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." + exit 1 + fi + mkdir -p /tmp/data/ && cd /tmp/data + xbstream -x < /${BACKUP_DIR}/${BACKUP_NAME}.xbstream + xtrabackup --decompress --target-dir=/tmp/data/ + xtrabackup --prepare --target-dir=/tmp/data/ + find . -name "*.qp"|xargs rm -f + xtrabackup --move-back --target-dir=/tmp/data/ --datadir=${DATA_DIR}/ + touch ${DATA_DIR}/.xtrabackup_restore_new_cluster + rm -rf /tmp/data/ + chmod -R 0777 ${DATA_DIR} incrementalRestoreCommands: [] logical: restoreCommands: [] diff --git a/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml b/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml index e76bb9f15..66e147036 100644 --- a/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml @@ -73,8 +73,7 @@ spec: targetPort: delvedebug horizontalScalePolicy: type: Snapshot - backupTemplateSelector: - "clusterdefinition.kubeblocks.io/name": apecloud-mysql-scale + backupPolicyTemplateName: apecloud-mysql-backup-policy-template podSpec: containers: - name: mysql diff --git a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml index f23c990ff..7b2449d51 100644 --- a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml @@ -1,21 +1,28 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-mysql + name: apecloud-mysql-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: apecloud-mysql {{- include "apecloud-mysql.labels" . | nindent 4 }} spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: xtrabackup-mysql - ttl: 168h0m0s - hooks: - containerName: mysql - preCommands: - - "touch /data/mysql/data/.restore_new_cluster; sync" - postCommands: - - "rm -f /data/mysql/data/.restore_new_cluster; sync" - - credentialKeyword: - userKeyword: username - passwordKeyword: password + clusterDefinitionRef: apecloud-mysql + backupPolicies: + - componentDefRef: mysql + ttl: 7d + schedule: + baseBackup: + type: snapshot + enable: false + cronExpression: "0 18 * * 0" + snapshot: + hooks: + containerName: mysql + preCommands: + - "touch /data/mysql/data/.restore_new_cluster; sync" + postCommands: + - "rm -f /data/mysql/data/.restore_new_cluster; sync" + target: + role: leader + full: + backupToolName: xtrabackup-apecloud-mysql \ No newline at end of file diff --git a/deploy/apecloud-mysql/templates/backuptool.yaml b/deploy/apecloud-mysql/templates/backuptool.yaml index 5b0cd01c4..cc84fe2f2 100644 --- a/deploy/apecloud-mysql/templates/backuptool.yaml +++ b/deploy/apecloud-mysql/templates/backuptool.yaml @@ -1,7 +1,7 @@ apiVersion: dataprotection.kubeblocks.io/v1alpha1 kind: BackupTool metadata: - name: xtrabackup-mysql + name: xtrabackup-apecloud-mysql labels: clusterdefinition.kubeblocks.io/name: apecloud-mysql {{- include "apecloud-mysql.labels" . | nindent 4 }} diff --git a/deploy/apecloud-mysql/templates/clusterdefinition.yaml b/deploy/apecloud-mysql/templates/clusterdefinition.yaml index a7fca264e..92d8fbf67 100644 --- a/deploy/apecloud-mysql/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql/templates/clusterdefinition.yaml @@ -60,8 +60,7 @@ spec: targetPort: mysql horizontalScalePolicy: type: Snapshot - backupTemplateSelector: - "clusterdefinition.kubeblocks.io/name": apecloud-mysql + backupPolicyTemplateName: apecloud-mysql-backup-policy-template volumeTypes: - name: data type: data diff --git a/deploy/helm/config/rbac/role.yaml b/deploy/helm/config/rbac/role.yaml index e2747ca6a..7bd0f58ca 100644 --- a/deploy/helm/config/rbac/role.yaml +++ b/deploy/helm/config/rbac/role.yaml @@ -103,6 +103,13 @@ rules: - statefulsets/status verbs: - get +- apiGroups: + - apps.kubeblocks.io + resources: + - backuppolicytemplates + verbs: + - get + - list - apiGroups: - apps.kubeblocks.io resources: @@ -496,32 +503,6 @@ rules: - get - patch - update -- apiGroups: - - dataprotection.kubeblocks.io - resources: - - backuppolicytemplates - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - dataprotection.kubeblocks.io - resources: - - backuppolicytemplates/finalizers - verbs: - - update -- apiGroups: - - dataprotection.kubeblocks.io - resources: - - backuppolicytemplates/status - verbs: - - get - - patch - - update - apiGroups: - dataprotection.kubeblocks.io resources: diff --git a/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml b/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml new file mode 100644 index 000000000..e88cfd8e7 --- /dev/null +++ b/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml @@ -0,0 +1,412 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: backuppolicytemplates.apps.kubeblocks.io +spec: + group: apps.kubeblocks.io + names: + categories: + - kubeblocks + kind: BackupPolicyTemplate + listKind: BackupPolicyTemplateList + plural: backuppolicytemplates + shortNames: + - bpt + singular: backuppolicytemplate + scope: Cluster + versions: + - additionalPrinterColumns: + - description: ClusterDefinition referenced by cluster. + jsonPath: .spec.clusterDefinitionRef + name: CLUSTER-DEFINITION + type: string + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: BackupPolicyTemplate is the Schema for the BackupPolicyTemplates + API (defined by provider) + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupPolicyTemplateSpec defines the desired state of BackupPolicyTemplate + properties: + backupPolicies: + description: backupPolicies is a list of backup policy template for + the specified componentDefinition. + items: + properties: + componentDefRef: + description: componentDefRef references componentDef defined + in ClusterDefinition spec. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + full: + description: the policy for full backup. + properties: + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can + execute. + type: string + path: + description: 'specify the json path of backup object + for patch. example: manifests.backupLog -- means + patch the backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect + backup status metadata. The script must exist in + the container of ContainerName and the output format + must be set to JSON. Note that outputting to stderr + may cause the result format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: + before backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupToolName: + description: which backup tool to perform database backup, + only support one tool. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. + Value must be non-negative integer. 0 means NO limit on + the number of backups. + format: int32 + type: integer + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + target: + description: target instance for backup. + properties: + account: + description: refer to spec.componentDef.systemAccounts.accounts[*].name + in ClusterDefinition. the secret created by this account + will be used to connect the database. if not set, + the secret created by spec.ConnectionCredential of + the ClusterDefinition will be used. it will be transformed + to a secret for BackupPolicy's target secret. + type: string + connectionCredentialKey: + description: connectionCredentialKey defines connection + credential key in secret which created by spec.ConnectionCredential + of the ClusterDefinition. it will be ignored when + "account" is set. + properties: + passwordKey: + description: the key of password in the ConnectionCredential + secret. if not set, the default key is "password". + type: string + usernameKey: + description: the key of username in the ConnectionCredential + secret. if not set, the default key is "username". + type: string + type: object + role: + description: 'select instance of corresponding role + for backup, role are: - the name of Leader/Follower/Leaner + for Consensus component. - primary or secondary for + Replication component. finally, invalid role of the + component will be ignored. such as if workload type + is Replication and component''s replicas is 1, the + secondary role is invalid. and it also will be ignored + when component is Stateful/Stateless. the role will + be transformed to a role LabelSelector for BackupPolicy''s + target attribute.' + type: string + type: object + type: object + incremental: + description: the policy for incremental backup. + properties: + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can + execute. + type: string + path: + description: 'specify the json path of backup object + for patch. example: manifests.backupLog -- means + patch the backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect + backup status metadata. The script must exist in + the container of ContainerName and the output format + must be set to JSON. Note that outputting to stderr + may cause the result format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: + before backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupToolName: + description: which backup tool to perform database backup, + only support one tool. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. + Value must be non-negative integer. 0 means NO limit on + the number of backups. + format: int32 + type: integer + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + target: + description: target instance for backup. + properties: + account: + description: refer to spec.componentDef.systemAccounts.accounts[*].name + in ClusterDefinition. the secret created by this account + will be used to connect the database. if not set, + the secret created by spec.ConnectionCredential of + the ClusterDefinition will be used. it will be transformed + to a secret for BackupPolicy's target secret. + type: string + connectionCredentialKey: + description: connectionCredentialKey defines connection + credential key in secret which created by spec.ConnectionCredential + of the ClusterDefinition. it will be ignored when + "account" is set. + properties: + passwordKey: + description: the key of password in the ConnectionCredential + secret. if not set, the default key is "password". + type: string + usernameKey: + description: the key of username in the ConnectionCredential + secret. if not set, the default key is "username". + type: string + type: object + role: + description: 'select instance of corresponding role + for backup, role are: - the name of Leader/Follower/Leaner + for Consensus component. - primary or secondary for + Replication component. finally, invalid role of the + component will be ignored. such as if workload type + is Replication and component''s replicas is 1, the + secondary role is invalid. and it also will be ignored + when component is Stateful/Stateless. the role will + be transformed to a role LabelSelector for BackupPolicy''s + target attribute.' + type: string + type: object + type: object + schedule: + description: schedule policy for backup. + properties: + baseBackup: + description: schedule policy for base backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. + type: string + enable: + description: enable or disable the schedule. + type: boolean + type: + description: the type of base backup, only support full + and snapshot. + enum: + - full + - snapshot + type: string + required: + - cronExpression + - enable + - type + type: object + incremental: + description: schedule policy for incremental backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. + type: string + enable: + description: enable or disable the schedule. + type: boolean + required: + - cronExpression + - enable + type: object + type: object + snapshot: + description: the policy for snapshot backup. + properties: + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can + execute. + type: string + path: + description: 'specify the json path of backup object + for patch. example: manifests.backupLog -- means + patch the backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect + backup status metadata. The script must exist in + the container of ContainerName and the output format + must be set to JSON. Note that outputting to stderr + may cause the result format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: + before backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. + Value must be non-negative integer. 0 means NO limit on + the number of backups. + format: int32 + type: integer + hooks: + description: execute hook commands for backup. + properties: + containerName: + description: which container can exec command + type: string + image: + description: exec command with image + type: string + postCommands: + description: post backup to perform commands + items: + type: string + type: array + preCommands: + description: pre backup to perform commands + items: + type: string + type: array + type: object + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + target: + description: target instance for backup. + properties: + account: + description: refer to spec.componentDef.systemAccounts.accounts[*].name + in ClusterDefinition. the secret created by this account + will be used to connect the database. if not set, + the secret created by spec.ConnectionCredential of + the ClusterDefinition will be used. it will be transformed + to a secret for BackupPolicy's target secret. + type: string + connectionCredentialKey: + description: connectionCredentialKey defines connection + credential key in secret which created by spec.ConnectionCredential + of the ClusterDefinition. it will be ignored when + "account" is set. + properties: + passwordKey: + description: the key of password in the ConnectionCredential + secret. if not set, the default key is "password". + type: string + usernameKey: + description: the key of username in the ConnectionCredential + secret. if not set, the default key is "username". + type: string + type: object + role: + description: 'select instance of corresponding role + for backup, role are: - the name of Leader/Follower/Leaner + for Consensus component. - primary or secondary for + Replication component. finally, invalid role of the + component will be ignored. such as if workload type + is Replication and component''s replicas is 1, the + secondary role is invalid. and it also will be ignored + when component is Stateful/Stateless. the role will + be transformed to a role LabelSelector for BackupPolicy''s + target attribute.' + type: string + type: object + type: object + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + required: + - componentDefRef + type: object + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - componentDefRef + x-kubernetes-list-type: map + clusterDefinitionRef: + description: clusterDefinitionRef references ClusterDefinition name, + this is an immutable attribute. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + required: + - backupPolicies + - clusterDefinitionRef + type: object + status: + description: BackupPolicyTemplateStatus defines the observed state of + BackupPolicyTemplate + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml index 2916d11b2..d146df534 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml @@ -286,12 +286,10 @@ spec: description: horizontalScalePolicy controls the behavior of horizontal scale. properties: - backupTemplateSelector: - additionalProperties: - type: string - description: backupTemplateSelector defines the label selector - for finding associated BackupTemplate API object. - type: object + backupPolicyTemplateName: + description: BackupPolicyTemplateName reference the backup + policy template. + type: string type: default: None description: 'type controls what kind of data synchronization @@ -299,10 +297,10 @@ spec: Snapshot}. The default policy is `None`. None: Default policy, do nothing. Snapshot: Do native volume snapshot before scaling and restore to newly scaled pods. Prefer - backup job to create snapshot if `BackupTemplateSelector` - can find a template. Notice that ''Snapshot'' policy will - only take snapshot on one volumeMount, default is the - first volumeMount of first container (i.e. clusterdefinition.spec.components.podSpec.containers[0].volumeMounts[0]), + backup job to create snapshot if can find a backupPolicy + from ''BackupPolicyTemplateName''. Notice that ''Snapshot'' + policy will only take snapshot on one volumeMount, default + is the first volumeMount of first container (i.e. clusterdefinition.spec.components.podSpec.containers[0].volumeMounts[0]), since take multiple snapshots at one time might cause consistency problem.' enum: diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml index 6e359243d..870a98b38 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -14,6 +14,8 @@ spec: kind: BackupPolicy listKind: BackupPolicyList plural: backuppolicies + shortNames: + - bp singular: backuppolicy scope: Namespaced versions: @@ -21,9 +23,6 @@ spec: - jsonPath: .status.phase name: STATUS type: string - - jsonPath: .spec.schedule - name: SCHEDULE - type: string - jsonPath: .status.lastScheduleTime name: LAST SCHEDULE type: string @@ -51,1696 +50,3614 @@ spec: spec: description: BackupPolicySpec defines the desired state of BackupPolicy properties: - backupPolicyTemplateName: - description: policy can inherit from backup config and override some - fields. - pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ - type: string - backupStatusUpdates: - description: define how to update metadata for backup status. - items: - properties: - containerName: - description: which container name that kubectl can execute. - type: string - path: - description: 'specify the json path of backup object for patch. - example: manifests.backupLog -- means patch the backup json - path of status.manifests.backupLog.' - type: string - script: - description: the shell Script commands to collect backup status - metadata. The script must exist in the container of ContainerName - and the output format must be set to JSON. Note that outputting - to stderr may cause the result format to not be in JSON. - type: string - updateStage: - description: 'when to update the backup status, pre: before - backup, post: after backup' - enum: - - pre - - post - type: string - type: object - type: array - backupToolName: - description: which backup tool to perform database backup, only support - one tool. - pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ - type: string - backupType: - default: snapshot - description: Backup ComponentDefRef. full or incremental or snapshot. - if unset, default is snapshot. - enum: - - full - - incremental - - snapshot - type: string - backupsHistoryLimit: - default: 7 - description: The number of automatic backups to retain. Value must - be non-negative integer. 0 means NO limit on the number of backups. - format: int32 - type: integer - hooks: - description: execute hook commands for backup. + full: + description: the policy for full backup. properties: - containerName: - description: which container can exec command - type: string - image: - description: exec command with image - type: string - postCommands: - description: post backup to perform commands + backupStatusUpdates: + description: define how to update metadata for backup status. items: - type: string - type: array - preCommands: - description: pre backup to perform commands - items: - type: string + properties: + containerName: + description: which container name that kubectl can execute. + type: string + path: + description: 'specify the json path of backup object for + patch. example: manifests.backupLog -- means patch the + backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect backup + status metadata. The script must exist in the container + of ContainerName and the output format must be set to + JSON. Note that outputting to stderr may cause the result + format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: before + backup, post: after backup' + enum: + - pre + - post + type: string + type: object type: array - type: object - onFailAttempted: - description: count of backup stop retries on fail. - format: int32 - type: integer - remoteVolume: - description: array of remote volumes from CSI driver definition. - properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount by - volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent disk - resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount on - the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the - blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must be a - filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob - disks per storage account Dedicated: single blob disk per - storage account Managed: azure managed data disk (only - in managed availability set). defaults to shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host that - shares a pod's lifetime + backupToolName: + description: which backup tool to perform database backup, only + support one tool. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. Value + must be non-negative integer. 0 means NO limit on the number + of backups. + format: int32 + type: integer + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + remoteVolume: + description: array of remote volumes from CSI driver definition. properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the path - to key ring for User, default is /etc/ceph/user.secret More - info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine and + then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is to + mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the readOnly + setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent + disk resource in AWS (Amazon EBS volume). More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID type: object - user: - description: 'user is optional: User is the rados user name, - default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and mounted - on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. More - info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret object - containing parameters used to connect to OpenStack.' + azureDisk: + description: azureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in + the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + description: fsType is Filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to + shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name type: string + required: + - secretName + - shareName type: object - volumeID: - description: 'volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate - this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect - the file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair in - the Data field of the referenced ConfigMap will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the - ConfigMap, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to set - permissions on this file. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: path is the relative path of the file to - map the key to. May not be an absolute path. May not - contain the path element '..'. May not start with - the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or its - keys must be defined - type: boolean - type: object - csi: - description: csi (Container Storage Interface) represents ephemeral - storage that is handled by certain external CSI drivers (Beta - feature). - properties: - driver: - description: driver is the name of the CSI driver that handles - this volume. Consult with your admin for the correct name - as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". If - not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem to - apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the secret - object containing sensitive information to pass to the CSI - driver to complete the CSI NodePublishVolume and NodeUnpublishVolume - calls. This field is optional, and may be empty if no secret - is required. If the secret object contains more than one - secret, all secret references are passed. + cephfs: + description: cephFS represents a Ceph FS mount on the host + that shares a pod's lifetime properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + monitors: + description: 'monitors is Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' type: string + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile is the + path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is reference + to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is optional: User is the rados user + name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors type: object - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. + cinder: + description: 'cinder represents a cinder volume attached and + mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a secret + object containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volumeID used to identify the volume in + cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod - that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Defaults to 0644. Directories within - the path are not affected by this setting. This might be - in conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are supported.' + configMap: + description: configMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value pair + in the Data field of the referenced ConfigMap will be + projected into the volume as a file whose name is the + key and content is the value. If specified, the listed + keys will be projected into the specified paths, and + unlisted keys will not be present. If a key is specified + which is not present in the ConfigMap, the volume setup + will error unless it is marked optional. Paths must + be relative and may not contain the '..' path or start + with '..'. + items: + description: Maps a string key to a path within a volume. properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". + key: + description: key is the key to project. type: string - fieldPath: - description: Path of the field to select in the - specified API version. + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. type: string required: - - fieldPath + - key + - path type: object - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name - of the file to be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 encoded. The - first item of the relative path must not start with - ''..''' + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap or + its keys must be defined + type: boolean + type: object + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external CSI + drivers (Beta feature). + properties: + driver: + description: driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated + CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to the + secret object containing sensitive information to pass + to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the secret + object contains more than one secret, all secret references + are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: type: string - resourceFieldRef: - description: 'Selects a resource of the container: only - resources limits and requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are currently supported.' + description: volumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Defaults to 0644. Directories within the path are not + affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON + requires decimal values for mode bits. If not + specified, the volume defaultMode will be used. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object required: - - resource + - path type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that shares - a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which means - to use the node''s default medium. Must be an empty string - (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is also - applicable for memory medium. The maximum usage on memory - medium EmptyDir would be the minimum value between the SizeLimit - specified here and the sum of memory limits of all containers - in a pod. The default is nil which means that the limit - is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled by - a cluster storage driver. The volume's lifecycle is tied to - the pod that defines it - it will be created before the pod - starts, and deleted when the pod is removed. \n Use this if: - a) the volume is only needed while the pod runs, b) features - of normal volumes like restoring from snapshot or capacity tracking - are needed, c) the storage driver is specified through a storage - class, and d) the storage driver supports dynamic volume provisioning - through a PersistentVolumeClaim (see EphemeralVolumeSource for - more information on the connection between this volume type - and PersistentVolumeClaim). \n Use PersistentVolumeClaim or - one of the vendor-specific APIs for volumes that persist for - longer than the lifecycle of an individual pod. \n Use CSI for - light-weight local ephemeral volumes if the CSI driver is meant - to be used that way - see the documentation of the driver for - more information. \n A pod can use both types of ephemeral volumes - and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC to - provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC will - be deleted together with the pod. The name of the PVC will - be `-` where `` is the - name from the `PodSpec.Volumes` array entry. Pod validation - will reject the pod if the concatenated name is not valid - for a PVC (for example, too long). \n An existing PVC with - that name that is not owned by the pod will *not* be used - for the pod to avoid using an unrelated volume by mistake. - Starting the pod is then blocked until the unrelated PVC - is removed. If such a pre-created PVC is meant to be used - by the pod, the PVC has to updated with an owner reference - to the pod once the pod exists. Normally this should not - be necessary, but it may be useful when manually reconstructing - a broken cluster. \n This field is read-only and no changes - will be made by Kubernetes to the PVC after it has been - created. \n Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations that will - be copied into the PVC when creating it. No other fields - are allowed and will be rejected during validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the PVC - that gets created from this template. The same fields - as in a PersistentVolumeClaim are also valid here. + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory that + shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage medium + should back this directory. The default is "" which + means to use the node''s default medium. Must be an + empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local storage + required for this EmptyDir volume. The size limit is + also applicable for memory medium. The maximum usage + on memory medium EmptyDir would be the minimum value + between the SizeLimit specified here and the sum of + memory limits of all containers in a pod. The default + is nil which means that the limit is undefined. More + info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use + this if: a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot + or capacity tracking are needed, c) the storage driver is + specified through a storage class, and d) the storage driver + supports dynamic volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information on the connection + between this volume type and PersistentVolumeClaim). \n + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. \n Use CSI for light-weight local + ephemeral volumes if the CSI driver is meant to be used + that way - see the documentation of the driver for more + information. \n A pod can use both types of ephemeral volumes + and persistent volumes at the same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC + to provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of + the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). + \n An existing PVC with that name that is not owned + by the pod will *not* be used for the pod to avoid using + an unrelated volume by mistake. Starting the pod is + then blocked until the unrelated PVC is removed. If + such a pre-created PVC is meant to be used by the pod, + the PVC has to updated with an owner reference to the + pod once the pod exists. Normally this should not be + necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no + changes will be made by Kubernetes to the PVC after + it has been created. \n Required, must not be nil." properties: - accessModes: - description: 'accessModes contains the desired access - modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to specify - either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If the - provisioner or an external controller can support - the specified data source, it will create a new - volume based on the contents of the specified data - source. When the AnyVolumeDataSource feature gate - is enabled, dataSource contents will be copied to - dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API group. - For any other third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - required: - - kind - - name - type: object - dataSourceRef: - description: 'dataSourceRef specifies the object from - which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a - non-empty API group (non core object) or a PersistentVolumeClaim - object. When this field is specified, volume binding - will only succeed if the type of the specified object - matches some installed volume populator or dynamic - provisioner. This field will replace the functionality - of the dataSource field and as such if both fields - are non-empty, they must have the same value. For - backwards compatibility, when namespace isn''t specified - in dataSourceRef, both fields (dataSource and dataSourceRef) - will be set to the same value automatically if one - of them is empty and the other is non-empty. When - namespace is specified in dataSourceRef, dataSource - isn''t set to the same value and must be empty. - There are three important differences between dataSource - and dataSourceRef: * While dataSource only allows - two specific types of objects, dataSourceRef allows - any non-core object, as well as PersistentVolumeClaim - objects. * While dataSource ignores disallowed values - (dropping them), dataSourceRef preserves all values, - and generates an error if a disallowed value is - specified. * While dataSource only allows local - objects, dataSourceRef allows objects in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource - feature gate to be enabled. (Alpha) Using the namespace - field of dataSourceRef requires the CrossNamespaceVolumeDataSource - feature gate to be enabled.' + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No + other fields are allowed and will be rejected during + validation. properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API group. - For any other third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object name: - description: Name is the name of resource being - referenced type: string namespace: - description: Namespace is the namespace of resource - being referenced Note that when a namespace - is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept the - reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires the - CrossNamespaceVolumeDataSource feature gate - to be enabled. type: string - required: - - kind - - name type: object - resources: - description: 'resources represents the minimum resources - the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity recorded - in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the + PVC that gets created from this template. The same + fields as in a PersistentVolumeClaim are also valid + here. properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are used - by this container. \n This is an alpha field - and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." + accessModes: + description: 'accessModes contains the desired + access modes the volume should have. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' items: - description: ResourceClaim references one entry - in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of - one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes - that resource available inside a container. - type: string - required: - - name - type: object + type: string type: array - x-kubernetes-list-map-keys: + dataSource: + description: 'dataSource field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If + the provisioner or an external controller can + support the specified data source, it will create + a new volume based on the contents of the specified + data source. When the AnyVolumeDataSource feature + gate is enabled, dataSource contents will be + copied to dataSourceRef, and dataSourceRef contents + will be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, + then dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount - of compute resources required. If Requests is - omitted for a container, it defaults to Limits - if that is explicitly specified, otherwise to - an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + dataSourceRef: + description: 'dataSourceRef specifies the object + from which to populate the volume with data, + if a non-empty volume is desired. This may be + any object from a non-empty API group (non core + object) or a PersistentVolumeClaim object. When + this field is specified, volume binding will + only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the dataSource field and as such if both + fields are non-empty, they must have the same + value. For backwards compatibility, when namespace + isn''t specified in dataSourceRef, both fields + (dataSource and dataSourceRef) will be set to + the same value automatically if one of them + is empty and the other is non-empty. When namespace + is specified in dataSourceRef, dataSource isn''t + set to the same value and must be empty. There + are three important differences between dataSource + and dataSourceRef: * While dataSource only allows + two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed + values (dropping them), dataSourceRef preserves + all values, and generates an error if a disallowed + value is specified. * While dataSource only + allows local objects, dataSourceRef allows objects + in any namespaces. (Beta) Using this field requires + the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef + requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: Namespace is the namespace of + resource being referenced Note that when + a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept + the reference. See the ReferenceGrant documentation + for details. (Alpha) This field requires + the CrossNamespaceVolumeDataSource feature + gate to be enabled. + type: string + required: + - kind + - name type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement is - a selector that contains values, a key, and - an operator that relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and DoesNotExist. + resources: + description: 'resources represents the minimum + resources the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity + recorded in the status field of the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are + used by this container. \n This is an alpha + field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. If - the operator is Exists or DoesNotExist, - the values array must be empty. This array - is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is "In", - and the values array contains only "value". - The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object type: object + storageClassName: + description: 'storageClassName is the name of + the StorageClass required by the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string type: object - storageClassName: - description: 'storageClassName is the name of the - StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem is - implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to - the PersistentVolume backing this claim. - type: string + required: + - spec type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached - to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and lun - must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource that - is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for this - volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends - on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' type: object - readOnly: - description: 'readOnly is Optional: defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information to - pass to the plugin scripts. This may be empty if no secret - object is specified. If the secret object contains more - than one secret, all secrets are passed to the plugin scripts.' + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to + the pod. properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising the machine' type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs and + lun must be set, but not both simultaneously.' + items: + type: string + type: array type: object - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached to a - kubelet's host machine. This depends on the Flocker control - service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored as - metadata -> name on the dataset for Flocker should be considered - as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume that - you want to mount. Tip: Ensure that the filesystem type - is supported by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount by - volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property empty). More - info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource in - GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision a - container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir into - the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, the - volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the host - that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More info: - https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + flexVolume: + description: flexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The default filesystem + depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is reference + to the secret object containing sensitive information + to pass to the plugin scripts. This may be empty if + no secret object is specified. If the secret object + contains more than one secret, all secrets are passed + to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: datasetName is Name of the dataset stored + as metadata -> name on the dataset for Flocker should + be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is to + mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource + in GCE. Used to identify the disk in GCE. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir + into the Pod''s container.' + properties: + directory: + description: directory is the target directory name. Must + not contain or start with '..'. If '.' is supplied, + the volume directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to + false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'path of the directory on the host. If the + path is a symlink, it will follow the link to the real + path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support + iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name that + uses an iSCSI transport. Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal List. + The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: targetPortal is iSCSI Target Portal. The + Portal is either an IP or ip_addr:port if the port is + other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean + nfs: + description: 'nfs represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export + to be mounted with read-only permissions. Defaults to + false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: readOnly Will force the ReadOnly setting + in VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this + setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced ConfigMap + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the ConfigMap, the + volume setup will error unless it is marked + optional. Paths must be relative and may not + contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and 0777 + or a decimal value between 0 and 511. + YAML accepts both octal and decimal + values, JSON requires decimal values + for mode bits. If not specified, the + volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the + container: only resources limits and + requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) are + currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced Secret + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the Secret, the volume + setup will error unless it is marked optional. + Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional field specify whether + the Secret or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to project + properties: + audience: + description: audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if the + token is older than 80 percent of its time + to live or if the token is older than 24 hours.Defaults + to 1 hour and must be at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative to the + mount point of the file to project the token + into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default is + no group + type: string + readOnly: + description: readOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults to + false. + type: boolean + registry: + description: registry represents a single or multiple + Quobyte Registry services specified as a string as host:port + pair (multiple entries are separated with commas) which + acts as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume in + the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: user to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'image is the rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default is + rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. Default + is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is the rados user name. Default is + admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret for ScaleIO + user and other sensitive information. If this is not + provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage + for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system + as configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume already + created in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value pair + in the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and + content is the value. If specified, the listed keys + will be projected into the specified paths, and unlisted + keys will not be present. If a key is specified which + is not present in the Secret, the volume setup will + error unless it is marked optional. Paths must be relative + and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether the Secret + or its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret in + the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: volumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope of the + volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within StorageOS + for tighter integration. Set VolumeName to any name + to override the default behaviour. Set to "default" + if you are not using namespaces within StorageOS. Namespaces + that do not pre-exist within StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fsType is filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath + type: object required: - - endpoints - - path + - name type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' + target: + description: target database cluster for backup. properties: - path: - description: 'path of the directory on the host. If the path - is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" More - info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string + labelsSelector: + description: labelsSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + secret: + description: secret is used to connect to the target database + cluster. If not set, secret will be inherited from backup + policy template. if still not set, the controller will check + if any system account for dataprotection has been created. + properties: + name: + description: the secret name + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + passwordKey: + default: password + description: passwordKey the map key of the password in + the connection credential secret + type: string + usernameKey: + default: username + description: usernameKey the map key of the user in the + connection credential secret + type: string + required: + - name + type: object required: - - path + - labelsSelector type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that is - attached to a kubelet''s host machine and then exposed to the - pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + required: + - remoteVolume + - target + type: object + incremental: + description: the policy for incremental backup. + properties: + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can execute. + type: string + path: + description: 'specify the json path of backup object for + patch. example: manifests.backupLog -- means patch the + backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect backup + status metadata. The script must exist in the container + of ContainerName and the output format must be set to + JSON. Note that outputting to stderr may cause the result + format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: before + backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupToolName: + description: which backup tool to perform database backup, only + support one tool. + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. Value + must be non-negative integer. 0 means NO limit on the number + of backups. + format: int32 + type: integer + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + remoteVolume: + description: array of remote volumes from CSI driver definition. properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI - Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, - new iSCSI interface : will be - created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that uses - an iSCSI transport. Defaults to 'default' (tcp). + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine and + then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is to + mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the readOnly + setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent + disk resource in AWS (Amazon EBS volume). More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: None, + Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in + the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the blob + storage + type: string + fsType: + description: fsType is Filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed data + disk (only in managed availability set). defaults to + shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service mount + on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that contains + Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host + that shares a pod's lifetime + properties: + monitors: + description: 'monitors is Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile is the + path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is reference + to the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is optional: User is the rados user + name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'cinder represents a cinder volume attached and + mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a secret + object containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volumeID used to identify the volume in + cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value pair + in the Data field of the referenced ConfigMap will be + projected into the volume as a file whose name is the + key and content is the value. If specified, the listed + keys will be projected into the specified paths, and + unlisted keys will not be present. If a key is specified + which is not present in the ConfigMap, the volume setup + will error unless it is marked optional. Paths must + be relative and may not contain the '..' path or start + with '..'. + items: + description: Maps a string key to a path within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap or + its keys must be defined + type: boolean + type: object + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external CSI + drivers (Beta feature). + properties: + driver: + description: driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the associated + CSI driver which will determine the default filesystem + to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to the + secret object containing sensitive information to pass + to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the secret + object contains more than one secret, all secret references + are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific properties + that are passed to the CSI driver. Consult your driver's + documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created files + by default. Must be a Optional: mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Defaults to 0644. Directories within the path are not + affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the pod: + only annotations, labels, name and namespace are + supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used to set permissions + on this file, must be an octal value between 0000 + and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON + requires decimal values for mode bits. If not + specified, the volume defaultMode will be used. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative path + name of the file to be created. Must not be absolute + or contain the ''..'' path. Must be utf-8 encoded. + The first item of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory that + shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage medium + should back this directory. The default is "" which + means to use the node''s default medium. Must be an + empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local storage + required for this EmptyDir volume. The size limit is + also applicable for memory medium. The maximum usage + on memory medium EmptyDir would be the minimum value + between the SizeLimit specified here and the sum of + memory limits of all containers in a pod. The default + is nil which means that the limit is undefined. More + info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is tied + to the pod that defines it - it will be created before the + pod starts, and deleted when the pod is removed. \n Use + this if: a) the volume is only needed while the pod runs, + b) features of normal volumes like restoring from snapshot + or capacity tracking are needed, c) the storage driver is + specified through a storage class, and d) the storage driver + supports dynamic volume provisioning through a PersistentVolumeClaim + (see EphemeralVolumeSource for more information on the connection + between this volume type and PersistentVolumeClaim). \n + Use PersistentVolumeClaim or one of the vendor-specific + APIs for volumes that persist for longer than the lifecycle + of an individual pod. \n Use CSI for light-weight local + ephemeral volumes if the CSI driver is meant to be used + that way - see the documentation of the driver for more + information. \n A pod can use both types of ephemeral volumes + and persistent volumes at the same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC + to provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the PVC + will be deleted together with the pod. The name of + the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array + entry. Pod validation will reject the pod if the concatenated + name is not valid for a PVC (for example, too long). + \n An existing PVC with that name that is not owned + by the pod will *not* be used for the pod to avoid using + an unrelated volume by mistake. Starting the pod is + then blocked until the unrelated PVC is removed. If + such a pre-created PVC is meant to be used by the pod, + the PVC has to updated with an owner reference to the + pod once the pod exists. Normally this should not be + necessary, but it may be useful when manually reconstructing + a broken cluster. \n This field is read-only and no + changes will be made by Kubernetes to the PVC after + it has been created. \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations that + will be copied into the PVC when creating it. No + other fields are allowed and will be rejected during + validation. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the + PVC that gets created from this template. The same + fields as in a PersistentVolumeClaim are also valid + here. + properties: + accessModes: + description: 'accessModes contains the desired + access modes the volume should have. More info: + https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If + the provisioner or an external controller can + support the specified data source, it will create + a new volume based on the contents of the specified + data source. When the AnyVolumeDataSource feature + gate is enabled, dataSource contents will be + copied to dataSourceRef, and dataSourceRef contents + will be copied to dataSource when dataSourceRef.namespace + is not specified. If the namespace is specified, + then dataSourceRef will not be copied to dataSource.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'dataSourceRef specifies the object + from which to populate the volume with data, + if a non-empty volume is desired. This may be + any object from a non-empty API group (non core + object) or a PersistentVolumeClaim object. When + this field is specified, volume binding will + only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the dataSource field and as such if both + fields are non-empty, they must have the same + value. For backwards compatibility, when namespace + isn''t specified in dataSourceRef, both fields + (dataSource and dataSourceRef) will be set to + the same value automatically if one of them + is empty and the other is non-empty. When namespace + is specified in dataSourceRef, dataSource isn''t + set to the same value and must be empty. There + are three important differences between dataSource + and dataSourceRef: * While dataSource only allows + two specific types of objects, dataSourceRef + allows any non-core object, as well as PersistentVolumeClaim + objects. * While dataSource ignores disallowed + values (dropping them), dataSourceRef preserves + all values, and generates an error if a disallowed + value is specified. * While dataSource only + allows local objects, dataSourceRef allows objects + in any namespaces. (Beta) Using this field requires + the AnyVolumeDataSource feature gate to be enabled. + (Alpha) Using the namespace field of dataSourceRef + requires the CrossNamespaceVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + namespace: + description: Namespace is the namespace of + resource being referenced Note that when + a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant + object is required in the referent namespace + to allow that namespace's owner to accept + the reference. See the ReferenceGrant documentation + for details. (Alpha) This field requires + the CrossNamespaceVolumeDataSource feature + gate to be enabled. + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum + resources the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than previous + value but must still be higher than capacity + recorded in the status field of the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + claims: + description: "Claims lists the names of resources, + defined in spec.resourceClaims, that are + used by this container. \n This is an alpha + field and requires enabling the DynamicResourceAllocation + feature gate. \n This field is immutable." + items: + description: ResourceClaim references one + entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name + of one entry in pod.spec.resourceClaims + of the Pod where this field is used. + It makes that resource available inside + a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'storageClassName is the name of + the StorageClass required by the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem + is implied when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that is + attached to a kubelet's host machine and then exposed to + the pod. + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising the machine' + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs and + lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: flexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use for + this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The default filesystem + depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds extra + command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is reference + to the secret object containing sensitive information + to pass to the plugin scripts. This may be empty if + no secret object is specified. If the secret object + contains more than one secret, all secrets are passed + to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: datasetName is Name of the dataset stored + as metadata -> name on the dataset for Flocker should + be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. This + is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is to + mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource + in GCE. Used to identify the disk in GCE. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the EmptyDir + into the Pod''s container.' + properties: + directory: + description: directory is the target directory name. Must + not contain or start with '..'. If '.' is supplied, + the volume directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount on the + host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs volume + to be mounted with read-only permissions. Defaults to + false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file or directory + on the host machine that is directly exposed to the container. + This is generally used for system agents or other privileged + things that are allowed to see the host machine. Most containers + will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host directory + mounts and who can/can not mount host directories as read/write.' + properties: + path: + description: 'path of the directory on the host. If the + path is a symlink, it will follow the link to the real + path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support + iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support iSCSI + Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name that + uses an iSCSI transport. Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal List. + The portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI target + and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: targetPortal is iSCSI Target Portal. The + Portal is either an IP or ip_addr:port if the port is + other than default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. The - portal is either an IP or ip_addr:port if the port is other - than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication + nfs: + description: 'nfs represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export + to be mounted with read-only permissions. Defaults to + false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string + readOnly: + description: readOnly Will force the ReadOnly setting + in VolumeMounts. Default false. + type: boolean + required: + - claimName type: object - targetPortal: - description: targetPortal is iSCSI Target Portal. The Portal - is either an IP or ip_addr:port if the port is other than - default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: 'name of the volume. Must be a DNS_LABEL and unique - within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - nfs: - description: 'nfs represents an NFS mount on the host that shares - a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export to be - mounted with read-only permissions. Defaults to false. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of the - NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents a reference - to a PersistentVolumeClaim in the same namespace. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Directories within the path are not - affected by this setting. This might be in conflict with - other options that affect the file mode, like fsGroup, and - the result can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along with - other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file whose - name is the key and content is the value. If specified, - the listed keys will be projected into the specified - paths, and unlisted keys will not be present. - If a key is specified which is not present in - the ConfigMap, the volume setup will error unless - it is marked optional. Paths must be relative - and may not contain the '..' path or start with - '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. If - not specified, the volume defaultMode will - be used. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host machine + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this + setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along + with other supported volume types properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing the - pod field - properties: - fieldRef: - description: 'Required: Selects a field of - the pod: only annotations, labels, name - and namespace are supported.' + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced ConfigMap + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the ConfigMap, the + volume setup will error unless it is marked + optional. Paths must be relative and may not + contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. properties: - apiVersion: - description: Version of the schema the - FieldPath is written in terms of, defaults - to "v1". + key: + description: key is the key to project. type: string - fieldPath: - description: Path of the field to select - in the specified API version. + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. type: string required: - - fieldPath + - key + - path type: object - mode: - description: 'Optional: mode bits used to - set permissions on this file, must be an - octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both - octal and decimal values, JSON requires - decimal values for mode bits. If not specified, - the volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. Must - not be absolute or contain the ''..'' path. - Must be utf-8 encoded. The first item of - the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: downwardAPI information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and 0777 + or a decimal value between 0 and 511. + YAML accepts both octal and decimal + values, JSON requires decimal values + for mode bits. If not specified, the + volume defaultMode will be used. This + might be in conflict with other options + that affect the file mode, like fsGroup, + and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. + Must not be absolute or contain the + ''..'' path. Must be utf-8 encoded. + The first item of the relative path + must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the + container: only resources limits and + requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) are + currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to + select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced Secret + will be projected into the volume as a file + whose name is the key and content is the value. + If specified, the listed keys will be projected + into the specified paths, and unlisted keys + will not be present. If a key is specified + which is not present in the Secret, the volume + setup will error unless it is marked optional. + Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' + key: + description: key is the key to project. type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format - of the exposed resources, defaults to - "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' + mode: + description: 'mode is Optional: mode bits + used to set permissions on this file. + Must be an octal value between 0000 + and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, like + fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May not + be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. type: string required: - - resource + - key + - path type: object - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret data - to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file whose - name is the key and content is the value. If specified, - the listed keys will be projected into the specified - paths, and unlisted keys will not be present. - If a key is specified which is not present in - the Secret, the volume setup will error unless - it is marked optional. Paths must be relative - and may not contain the '..' path or start with - '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. If - not specified, the volume defaultMode will - be used. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional field specify whether the - Secret or its key must be defined - type: boolean + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional field specify whether + the Secret or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to project + properties: + audience: + description: audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if the + token is older than 80 percent of its time + to live or if the token is older than 24 hours.Defaults + to 1 hour and must be at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative to the + mount point of the file to project the token + into. + type: string + required: + - path + type: object type: object - serviceAccountToken: - description: serviceAccountToken is information about - the serviceAccountToken data to project + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default is + no group + type: string + readOnly: + description: readOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults to + false. + type: boolean + registry: + description: registry represents a single or multiple + Quobyte Registry services specified as a string as host:port + pair (multiple entries are separated with commas) which + acts as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume in + the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: user to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'image is the rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default is + rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. Default + is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is the rados user name. Default is + admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret for ScaleIO + user and other sensitive information. If this is not + provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage + for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool associated + with the protection domain. + type: string + system: + description: system is the name of the storage system + as configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume already + created in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in conflict + with other options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value pair + in the Data field of the referenced Secret will be projected + into the volume as a file whose name is the key and + content is the value. If specified, the listed keys + will be projected into the specified paths, and unlisted + keys will not be present. If a key is specified which + is not present in the Secret, the volume setup will + error unless it is marked optional. Paths must be relative + and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a volume. properties: - audience: - description: audience is the intended audience of - the token. A recipient of a token must identify - itself with an identifier specified in the audience - of the token, and otherwise should reject the - token. The audience defaults to the identifier - of the apiserver. + key: + description: key is the key to project. type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account token. - As the token approaches expiration, the kubelet - volume plugin will proactively rotate the service - account token. The kubelet will start trying to - rotate the token if the token is older than 80 - percent of its time to live or if the token is - older than 24 hours.Defaults to 1 hour and must - be at least 10 minutes. - format: int64 + mode: + description: 'mode is Optional: mode bits used to + set permissions on this file. Must be an octal + value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for + mode bits. If not specified, the volume defaultMode + will be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 type: integer path: - description: path is the path relative to the mount - point of the file to project the token into. + description: path is the relative path of the file + to map the key to. May not be an absolute path. + May not contain the path element '..'. May not + start with the string '..'. type: string required: + - key - path type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host that - shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is no group - type: string - readOnly: - description: readOnly here will force the Quobyte volume to - be mounted with read-only permissions. Defaults to false. - type: boolean - registry: - description: registry represents a single or multiple Quobyte - Registry services specified as a string as host:port pair - (multiple entries are separated with commas) which acts - as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in the - Backend Used with dynamically provisioned Quobyte volumes, - value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to serivceaccount - user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - image: - description: 'image is the rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. More - info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication secret - for RBDUser. If provided overrides keyring. Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: array + optional: + description: optional field specify whether the Secret + or its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret in + the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + fsType: + description: fsType is the filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: volumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope of the + volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within StorageOS + for tighter integration. Set VolumeName to any name + to override the default behaviour. Set to "default" + if you are not using namespaces within StorageOS. Namespaces + that do not pre-exist within StorageOS will be created. type: string type: object - user: - description: 'user is the rados user name. Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API - Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection - Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not provided, - Login operation will fail. + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + fsType: + description: fsType is filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. type: string + storagePolicyName: + description: storagePolicyName is the storage Policy Based + Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies vSphere + volume vmdk + type: string + required: + - volumePath type: object - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage for - a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system as configured - in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already created - in the ScaleIO system that is associated with this volume - source. - type: string required: - - gateway - - secretRef - - system + - name type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + target: + description: target database cluster for backup. properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect - the file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair in - the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the - Secret, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to set - permissions on this file. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: path is the relative path of the file to - map the key to. May not be an absolute path. May not - contain the path element '..'. May not start with - the string '..'. + labelsSelector: + description: labelsSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret or - its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in the - pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for obtaining - the StorageOS API credentials. If not specified, default - values will be attempted. + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + secret: + description: secret is used to connect to the target database + cluster. If not set, secret will be inherited from backup + policy template. if still not set, the controller will check + if any system account for dataprotection has been created. properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' + description: the secret name + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + passwordKey: + default: password + description: passwordKey the map key of the password in + the connection credential secret type: string + usernameKey: + default: username + description: usernameKey the map key of the user in the + connection credential secret + type: string + required: + - name type: object - volumeName: - description: volumeName is the human-readable name of the - StorageOS volume. Volume names are only unique within a - namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the volume - within StorageOS. If no namespace is specified then the - Pod's namespace will be used. This allows the Kubernetes - name scoping to be mirrored within StorageOS for tighter - integration. Set VolumeName to any name to override the - default behaviour. Set to "default" if you are not using - namespaces within StorageOS. Namespaces that do not pre-exist - within StorageOS will be created. - type: string + required: + - labelsSelector type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine + required: + - remoteVolume + - target + type: object + schedule: + description: schedule policy for backup. + properties: + baseBackup: + description: schedule policy for base backup. properties: - fsType: - description: fsType is filesystem type to mount. Must be a - filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management - (SPBM) profile ID associated with the StoragePolicyName. + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. + enable: + description: enable or disable the schedule. + type: boolean + type: + description: the type of base backup, only support full and + snapshot. + enum: + - full + - snapshot type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk + required: + - cronExpression + - enable + - type + type: object + incremental: + description: schedule policy for incremental backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - - volumePath + - cronExpression + - enable type: object - required: - - name type: object - schedule: - description: The schedule in Cron format, the timezone is in UTC. - see https://en.wikipedia.org/wiki/Cron. - type: string - target: - description: database cluster service + snapshot: + description: the policy for snapshot backup. properties: - labelsSelector: - description: LabelSelector is used to find matching pods. Pods - that match this label selector are counted to determine the - number of pods in their corresponding topology domain. + backupStatusUpdates: + description: define how to update metadata for backup status. + items: + properties: + containerName: + description: which container name that kubectl can execute. + type: string + path: + description: 'specify the json path of backup object for + patch. example: manifests.backupLog -- means patch the + backup json path of status.manifests.backupLog.' + type: string + script: + description: the shell Script commands to collect backup + status metadata. The script must exist in the container + of ContainerName and the output format must be set to + JSON. Note that outputting to stderr may cause the result + format to not be in JSON. + type: string + updateStage: + description: 'when to update the backup status, pre: before + backup, post: after backup' + enum: + - pre + - post + type: string + type: object + type: array + backupsHistoryLimit: + default: 7 + description: the number of automatic backups to retain. Value + must be non-negative integer. 0 means NO limit on the number + of backups. + format: int32 + type: integer + hooks: + description: execute hook commands for backup. properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. + containerName: + description: which container can exec command + type: string + image: + description: exec command with image + type: string + postCommands: + description: post backup to perform commands items: - description: A label selector requirement is a selector - that contains values, a key, and an operator that relates - the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: operator represents a key's relationship - to a set of values. Valid operators are In, NotIn, - Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If - the operator is In or NotIn, the values array must - be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced - during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object + type: string type: array - matchLabels: - additionalProperties: + preCommands: + description: pre backup to perform commands + items: type: string - description: matchLabels is a map of {key,value} pairs. A - single {key,value} in the matchLabels map is equivalent - to an element of matchExpressions, whose key field is "key", - the operator is "In", and the values array contains only - "value". The requirements are ANDed. - type: object + type: array type: object - x-kubernetes-preserve-unknown-fields: true - secret: - description: Secret is used to connect to the target database - cluster. If not set, secret will be inherited from backup policy - template. if still not set, the controller will check if any - system account for dataprotection has been created. + onFailAttempted: + description: count of backup stop retries on fail. + format: int32 + type: integer + target: + description: target database cluster for backup. properties: - name: - description: the secret name - pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ - type: string - passwordKeyword: - description: PasswordKeyword the map keyword of the password - in the connection credential secret - type: string - userKeyword: - description: UserKeyword the map keyword of the user in the - connection credential secret - type: string + labelsSelector: + description: labelsSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + secret: + description: secret is used to connect to the target database + cluster. If not set, secret will be inherited from backup + policy template. if still not set, the controller will check + if any system account for dataprotection has been created. + properties: + name: + description: the secret name + pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ + type: string + passwordKey: + default: password + description: passwordKey the map key of the password in + the connection credential secret + type: string + usernameKey: + default: username + description: usernameKey the map key of the user in the + connection credential secret + type: string + required: + - name + type: object required: - - name + - labelsSelector type: object required: - - labelsSelector + - target type: object ttl: - description: TTL is a time.Duration-parseable string describing how - long the Backup should be retained for. + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. if + not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ type: string - required: - - remoteVolume - - target type: object status: description: BackupPolicyStatus defines the observed state of BackupPolicy @@ -1749,22 +3666,25 @@ spec: description: the reason if backup policy check failed. type: string lastScheduleTime: - description: Information when was the last time the job was successfully + description: information when was the last time the job was successfully scheduled. format: date-time type: string lastSuccessfulTime: - description: Information when was the last time the job successfully + description: information when was the last time the job successfully completed. format: date-time type: string + observedGeneration: + description: observedGeneration is the most recent generation observed + for this BackupPolicy. It corresponds to the Cluster's generation, + which is updated on mutation by the API Server. + format: int64 + type: integer phase: - description: 'backup policy phase valid value: available, failed, - new.' + description: 'backup policy phase valid value: Available, Failed.' enum: - - New - Available - - InProgress - Failed type: string type: object diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicytemplates.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicytemplates.yaml deleted file mode 100644 index ea15b74c6..000000000 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicytemplates.yaml +++ /dev/null @@ -1,144 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null - name: backuppolicytemplates.dataprotection.kubeblocks.io -spec: - group: dataprotection.kubeblocks.io - names: - categories: - - kubeblocks - kind: BackupPolicyTemplate - listKind: BackupPolicyTemplateList - plural: backuppolicytemplates - singular: backuppolicytemplate - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: BackupPolicyTemplate is the Schema for the BackupPolicyTemplates - API (defined by provider) - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BackupPolicyTemplateSpec defines the desired state of BackupPolicyTemplate - properties: - backupStatusUpdates: - description: define how to update metadata for backup status. - items: - properties: - containerName: - description: which container name that kubectl can execute. - type: string - path: - description: 'specify the json path of backup object for patch. - example: manifests.backupLog -- means patch the backup json - path of status.manifests.backupLog.' - type: string - script: - description: the shell Script commands to collect backup status - metadata. The script must exist in the container of ContainerName - and the output format must be set to JSON. Note that outputting - to stderr may cause the result format to not be in JSON. - type: string - updateStage: - description: 'when to update the backup status, pre: before - backup, post: after backup' - enum: - - pre - - post - type: string - type: object - type: array - backupToolName: - description: which backup tool to perform database backup, only support - one tool. - pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ - type: string - credentialKeyword: - description: CredentialKeyword determines backupTool connection credential - keyword in secret. the backupTool gets the credentials according - to the user and password keyword defined by secret - properties: - passwordKeyword: - default: password - description: PasswordKeyword the map keyword of the password in - the connection credential secret - type: string - userKeyword: - default: username - description: UserKeyword the map keyword of the user in the connection - credential secret - type: string - type: object - hooks: - description: execute hook commands for backup. - properties: - containerName: - description: which container can exec command - type: string - image: - description: exec command with image - type: string - postCommands: - description: post backup to perform commands - items: - type: string - type: array - preCommands: - description: pre backup to perform commands - items: - type: string - type: array - type: object - onFailAttempted: - description: limit count of backup stop retries on fail. if unset, - retry unlimit attempted. - format: int32 - type: integer - schedule: - description: The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron. - type: string - ttl: - description: TTL is a time.Duration-parseable string describing how - long the Backup should be retained for. - type: string - required: - - backupToolName - type: object - status: - description: BackupPolicyTemplateStatus defines the observed state of - BackupPolicyTemplate - properties: - failureReason: - type: string - phase: - description: BackupPolicyTemplatePhase defines phases for BackupPolicyTemplate - CR. - enum: - - New - - Available - - InProgress - - Failed - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml index 188fb3fb3..4542c6957 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml @@ -72,10 +72,6 @@ spec: parentBackupName: description: if backupType is incremental, parentBackupName is required. type: string - ttl: - description: ttl is a time.Duration-parsable string describing how - long the Backup should be retained for. - type: string required: - backupPolicyName - backupType diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_restorejobs.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_restorejobs.yaml index 84334be2b..b282429e6 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_restorejobs.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_restorejobs.yaml @@ -59,7 +59,7 @@ spec: description: the target database workload to restore properties: labelsSelector: - description: LabelSelector is used to find matching pods. Pods + description: labelsSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. properties: @@ -106,7 +106,7 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true secret: - description: Secret is used to connect to the target database + description: secret is used to connect to the target database cluster. If not set, secret will be inherited from backup policy template. if still not set, the controller will check if any system account for dataprotection has been created. @@ -115,14 +115,16 @@ spec: description: the secret name pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string - passwordKeyword: - description: PasswordKeyword the map keyword of the password - in the connection credential secret - type: string - userKeyword: - description: UserKeyword the map keyword of the user in the + passwordKey: + default: password + description: passwordKey the map key of the password in the connection credential secret type: string + usernameKey: + default: username + description: usernameKey the map key of the user in the connection + credential secret + type: string required: - name type: object diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml index bac706b88..c63c8079d 100644 --- a/deploy/helm/templates/deployment.yaml +++ b/deploy/helm/templates/deployment.yaml @@ -77,14 +77,6 @@ spec: - name: VOLUMESNAPSHOT value: "true" {{- end }} - {{- if .Values.dataProtection.backupSchedule }} - - name: DP_BACKUP_SCHEDULE - value: {{ .Values.dataProtection.backupSchedule }} - {{- end }} - {{- if .Values.dataProtection.backupTTL }} - - name: DP_BACKUP_TTL - value: {{ .Values.dataProtection.backupTTL }} - {{- end }} {{- if .Values.admissionWebhooks.enabled }} - name: ENABLE_WEBHOOKS value: "true" diff --git a/deploy/helm/templates/rbac/dataprotection_backuppolicytemplate_editor_role.yaml b/deploy/helm/templates/rbac/apps_backuppolicytemplate_editor_role.yaml similarity index 88% rename from deploy/helm/templates/rbac/dataprotection_backuppolicytemplate_editor_role.yaml rename to deploy/helm/templates/rbac/apps_backuppolicytemplate_editor_role.yaml index 2153e9ac4..ae96ce974 100644 --- a/deploy/helm/templates/rbac/dataprotection_backuppolicytemplate_editor_role.yaml +++ b/deploy/helm/templates/rbac/apps_backuppolicytemplate_editor_role.yaml @@ -7,7 +7,7 @@ metadata: {{- include "kubeblocks.labels" . | nindent 4 }} rules: - apiGroups: - - dataprotection.kubeblocks.io + - apps.kubeblocks.io resources: - backuppolicytemplates verbs: @@ -19,7 +19,7 @@ rules: - update - watch - apiGroups: - - dataprotection.kubeblocks.io + - apps.kubeblocks.io resources: - backuppolicytemplates/status verbs: diff --git a/deploy/helm/templates/rbac/dataprotection_backuppolicytemplate_viewer_role.yaml b/deploy/helm/templates/rbac/apps_backuppolicytemplate_viewer_role.yaml similarity index 86% rename from deploy/helm/templates/rbac/dataprotection_backuppolicytemplate_viewer_role.yaml rename to deploy/helm/templates/rbac/apps_backuppolicytemplate_viewer_role.yaml index 0a7bbc579..b8dac3a90 100644 --- a/deploy/helm/templates/rbac/dataprotection_backuppolicytemplate_viewer_role.yaml +++ b/deploy/helm/templates/rbac/apps_backuppolicytemplate_viewer_role.yaml @@ -7,7 +7,7 @@ metadata: {{- include "kubeblocks.labels" . | nindent 4 }} rules: - apiGroups: - - dataprotection.kubeblocks.io + - apps.kubeblocks.io resources: - backuppolicytemplates verbs: @@ -15,7 +15,7 @@ rules: - list - watch - apiGroups: - - dataprotection.kubeblocks.io + - apps.kubeblocks.io resources: - backuppolicytemplates/status verbs: diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index a9902fe04..8f8c05fd7 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -246,16 +246,6 @@ dataProtection: ## @param dataProtection.enableVolumeSnapshot - set this to true if cluster does have snapshot.storage.k8s.io API installed ## enableVolumeSnapshot: false - ## @param dataProtection.backupSchedule -- set backup policy schedule time - ## backupSchedule is in Cron format, the timezone is in UTC. see https://en.wikipedia.org/wiki/Cron. - ## Example: 0 2 * * * -- backup job will be scheduled to start at 2:00 each day. - ## - backupSchedule: "" - ## @param dataProtection.backupTTL -- set backup time to live. - ## backupTTL is a time.Duration-parseable string describing how long. - ## Example: 168h0m0s -- the backup will expire in 7 days. - ## - backupTTL: "" ## Addon controller settings, this will require cluster-admin clusterrole. addonController: diff --git a/deploy/milvus/templates/backuppolicytemplate.yaml b/deploy/milvus/templates/backuppolicytemplate.yaml index 38c1d98c3..e6a22571a 100644 --- a/deploy/milvus/templates/backuppolicytemplate.yaml +++ b/deploy/milvus/templates/backuppolicytemplate.yaml @@ -1,15 +1,22 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-milvus + name: milvus-standalone-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: milvus-standalone {{- include "milvus.labels" . | nindent 4 }} spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: volumesnapshot - ttl: 168h0m0s - - credentialKeyword: - userKeyword: username - passwordKeyword: password + clusterDefinitionRef: milvus-standalone + backupPolicies: + - componentDefRef: milvus + ttl: 7d + schedule: + baseBackup: + type: snapshot + enable: false + cronExpression: "0 18 * * 0" + snapshot: + target: + connectionCredentialKey: + passwordKey: password + usernameKey: username \ No newline at end of file diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index 48a8be568..1c31aaa47 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -1,25 +1,24 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-postgresql + name: postgresql-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: postgresql {{- include "postgresql.labels" . | nindent 4 }} spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: volumesnapshot - ttl: 168h0m0s - - credentialKeyword: - userKeyword: username - passwordKeyword: password - - hooks: - containerName: postgresql - preCommands: - - psql -c "CHECKPOINT" - backupStatusUpdates: - - path: manifests.backupLog - containerName: postgresql - script: /scripts/backup-log-collector.sh - updateStage: pre + clusterDefinitionRef: postgresql + backupPolicies: + - componentDefRef: postgresql + ttl: 7d + schedule: + baseBackup: + type: snapshot + enable: false + cronExpression: "0 18 * * 0" + snapshot: + target: + connectionCredentialKey: + passwordKey: password + usernameKey: username + full: + backupToolName: postgres-basebackup \ No newline at end of file diff --git a/deploy/qdrant/templates/backuppolicytemplate.yaml b/deploy/qdrant/templates/backuppolicytemplate.yaml index b07273534..d26692c41 100644 --- a/deploy/qdrant/templates/backuppolicytemplate.yaml +++ b/deploy/qdrant/templates/backuppolicytemplate.yaml @@ -1,15 +1,22 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-qdrant + name: qdrant-standalone-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: qdrant-standalone {{- include "qdrant.labels" . | nindent 4 }} spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: volumesnapshot - ttl: 168h0m0s - - credentialKeyword: - userKeyword: username - passwordKeyword: password + clusterDefinitionRef: qdrant-standalone + backupPolicies: + - componentDefRef: qdrant + ttl: 7d + schedule: + baseBackup: + type: snapshot + enable: false + cronExpression: "0 18 * * 0" + snapshot: + target: + connectionCredentialKey: + passwordKey: password + usernameKey: username \ No newline at end of file diff --git a/deploy/redis/templates/backuppolicytemplate.yaml b/deploy/redis/templates/backuppolicytemplate.yaml index f1b2b7e29..f707cabf2 100644 --- a/deploy/redis/templates/backuppolicytemplate.yaml +++ b/deploy/redis/templates/backuppolicytemplate.yaml @@ -1,12 +1,22 @@ - -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-redis + name: redis-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: redis {{- include "redis.labels" . | nindent 4 }} spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: volumesnapshot - ttl: 168h0m0s + clusterDefinitionRef: redis + backupPolicies: + - componentDefRef: redis + ttl: 7d + schedule: + baseBackup: + type: snapshot + enable: false + cronExpression: "0 18 * * 0" + snapshot: + target: + connectionCredentialKey: + passwordKey: password + usernameKey: username \ No newline at end of file diff --git a/deploy/weaviate/templates/backuppolicytemplate.yaml b/deploy/weaviate/templates/backuppolicytemplate.yaml index 778b6181d..65c8d9648 100644 --- a/deploy/weaviate/templates/backuppolicytemplate.yaml +++ b/deploy/weaviate/templates/backuppolicytemplate.yaml @@ -1,15 +1,22 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-weaviate + name: weaviate-standalone-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: weaviate-standalone {{- include "weaviate.labels" . | nindent 4 }} spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: volumesnapshot - ttl: 168h0m0s - - credentialKeyword: - userKeyword: username - passwordKeyword: password + clusterDefinitionRef: weaviate-standalone + backupPolicies: + - componentDefRef: weaviate + ttl: 7d + schedule: + baseBackup: + type: snapshot + enable: false + cronExpression: "0 18 * * 0" + snapshot: + target: + connectionCredentialKey: + passwordKey: password + usernameKey: username \ No newline at end of file diff --git a/docs/release_notes/v0.5.0/v0.5.0.md b/docs/release_notes/v0.5.0/v0.5.0.md index e62720bb7..ea16a0f17 100644 --- a/docs/release_notes/v0.5.0/v0.5.0.md +++ b/docs/release_notes/v0.5.0/v0.5.0.md @@ -35,3 +35,11 @@ Thanks to everyone who made this release possible! ## Breaking changes + +- Breaking changes between v0.5 and v0.4. Uninstall v0.4 before installing v0.5. + - Move the backupPolicyTemplate API from dataprotection group to apps group. + Before installing v0.5, please ensure that the resources have been cleaned up: + ``` + kubectl delete backuppolicytemplates.dataprotection.kubeblocks.io --all + kubectl delete backuppolicies.dataprotection.kubeblocks.io --all + ``` \ No newline at end of file diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index cbbf95f9e..e1d393c68 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -65,13 +65,15 @@ Cluster command. * [kbcli cluster describe-ops](kbcli_cluster_describe-ops.md) - Show details of a specific OpsRequest. * [kbcli cluster diff-config](kbcli_cluster_diff-config.md) - Show the difference in parameters between the two submitted OpsRequest. * [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. +* [kbcli cluster edit-backup-policy](kbcli_cluster_edit-backup-policy.md) - Edit backup policy * [kbcli cluster explain-config](kbcli_cluster_explain-config.md) - List the constraint for supported configuration params. * [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account * [kbcli cluster hscale](kbcli_cluster_hscale.md) - Horizontally scale the specified components in the cluster. * [kbcli cluster list](kbcli_cluster_list.md) - List clusters. * [kbcli cluster list-accounts](kbcli_cluster_list-accounts.md) - List accounts for a cluster -* [kbcli cluster list-backups](kbcli_cluster_list-backups.md) - List backups. +* [kbcli cluster list-backup](kbcli_cluster_list-backup.md) - List backups. +* [kbcli cluster list-backup-policy](kbcli_cluster_list-backup-policy.md) - List backups policies. * [kbcli cluster list-components](kbcli_cluster_list-components.md) - List cluster components. * [kbcli cluster list-events](kbcli_cluster_list-events.md) - List cluster events. * [kbcli cluster list-instances](kbcli_cluster_list-instances.md) - List cluster instances. diff --git a/docs/user_docs/cli/kbcli_backup-config.md b/docs/user_docs/cli/kbcli_backup-config.md index c9d673175..9461f0c94 100644 --- a/docs/user_docs/cli/kbcli_backup-config.md +++ b/docs/user_docs/cli/kbcli_backup-config.md @@ -16,12 +16,6 @@ kbcli backup-config [flags] # If you have already installed a snapshot-controller, only enable the snapshot backup feature kbcli backup-config --set dataProtection.enableVolumeSnapshot=true - - # Schedule automatic backup at 18:00 every day (UTC timezone) - kbcli backup-config --set dataProtection.backupSchedule="0 18 * * *" - - # Set automatic backup retention for 7 days - kbcli backup-config --set dataProtection.backupTTL="168h0m0s" ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index 359bc280f..9fffbb637 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -53,13 +53,15 @@ Cluster command. * [kbcli cluster describe-ops](kbcli_cluster_describe-ops.md) - Show details of a specific OpsRequest. * [kbcli cluster diff-config](kbcli_cluster_diff-config.md) - Show the difference in parameters between the two submitted OpsRequest. * [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. +* [kbcli cluster edit-backup-policy](kbcli_cluster_edit-backup-policy.md) - Edit backup policy * [kbcli cluster explain-config](kbcli_cluster_explain-config.md) - List the constraint for supported configuration params. * [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account * [kbcli cluster hscale](kbcli_cluster_hscale.md) - Horizontally scale the specified components in the cluster. * [kbcli cluster list](kbcli_cluster_list.md) - List clusters. * [kbcli cluster list-accounts](kbcli_cluster_list-accounts.md) - List accounts for a cluster -* [kbcli cluster list-backups](kbcli_cluster_list-backups.md) - List backups. +* [kbcli cluster list-backup](kbcli_cluster_list-backup.md) - List backups. +* [kbcli cluster list-backup-policy](kbcli_cluster_list-backup-policy.md) - List backups policies. * [kbcli cluster list-components](kbcli_cluster_list-components.md) - List cluster components. * [kbcli cluster list-events](kbcli_cluster_list-events.md) - List cluster events. * [kbcli cluster list-instances](kbcli_cluster_list-instances.md) - List cluster instances. diff --git a/docs/user_docs/cli/kbcli_cluster_backup.md b/docs/user_docs/cli/kbcli_cluster_backup.md index b6fece97e..b49f3a867 100644 --- a/docs/user_docs/cli/kbcli_cluster_backup.md +++ b/docs/user_docs/cli/kbcli_cluster_backup.md @@ -18,11 +18,10 @@ kbcli cluster backup [flags] ### Options ``` - --backup-name string Backup name - --backup-type string Backup type (default "snapshot") - -h, --help help for backup - --role string backup on cluster role - --ttl string Time to live (default "168h0m0s") + --backup-name string Backup name + --backup-policy string Backup policy name, this flag will be ignored when backup-type is snapshot + --backup-type string Backup type (default "snapshot") + -h, --help help for backup ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_edit-backup-policy.md b/docs/user_docs/cli/kbcli_cluster_edit-backup-policy.md new file mode 100644 index 000000000..96c31087b --- /dev/null +++ b/docs/user_docs/cli/kbcli_cluster_edit-backup-policy.md @@ -0,0 +1,72 @@ +--- +title: kbcli cluster edit-backup-policy +--- + +Edit backup policy + +``` +kbcli cluster edit-backup-policy +``` + +### Examples + +``` + # edit backup policy + kbcli cluster edit-backup-policy + + # using short cmd to edit backup policy + kbcli cluster edit-bp +``` + +### Options + +``` + --allow-missing-template-keys If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats. (default true) + --field-manager string Name of the manager used to track field ownership. (default "kubectl-edit") + -f, --filename strings Filename, directory, or URL to files to use to edit the resource + -h, --help help for edit-backup-policy + -k, --kustomize string Process the kustomization directory. This flag can't be used together with -f or -R. + -o, --output string Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file). + --output-patch Output the patch if the resource is edited. + -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. + --save-config If true, the configuration of current object will be saved in its annotation. Otherwise, the annotation will be unchanged. This flag is useful when you want to perform kubectl apply on this object in the future. + --show-managed-fields If true, keep the managedFields when printing objects in JSON or YAML format. + --subresource string If specified, edit will operate on the subresource of the requested object. Must be one of [status]. This flag is alpha and may change in the future. + --template string Template string or path to template file to use when -o=go-template, -o=go-template-file. The template format is golang templates [http://golang.org/pkg/text/template/#pkg-overview]. + --validate string[="strict"] Must be one of: strict (or true), warn, ignore (or false). + "true" or "strict" will use a schema to validate the input and fail the request if invalid. It will perform server side validation if ServerSideFieldValidation is enabled on the api-server, but will fall back to less reliable client-side validation if not. + "warn" will warn about unknown or duplicate fields without blocking the request if server-side field validation is enabled on the API server, and behave as "ignore" otherwise. + "false" or "ignore" will not perform any schema validation, silently dropping any unknown or duplicate fields. (default "strict") + --windows-line-endings Defaults to the line ending native to your platform. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli cluster](kbcli_cluster.md) - Cluster command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md b/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md new file mode 100644 index 000000000..f0adc443c --- /dev/null +++ b/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md @@ -0,0 +1,60 @@ +--- +title: kbcli cluster list-backup-policy +--- + +List backups policies. + +``` +kbcli cluster list-backup-policy [flags] +``` + +### Examples + +``` + # list all backup policy + kbcli cluster list-backup-policy + + # using short cmd to list backup policy of specified cluster + kbcli cluster list-bp +``` + +### Options + +``` + -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. + -h, --help help for list-backup-policy + -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. + --show-labels When printing, show all labels as the last column (default hide labels column) +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli cluster](kbcli_cluster.md) - Cluster command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_cluster_list-backup.md b/docs/user_docs/cli/kbcli_cluster_list-backup.md new file mode 100644 index 000000000..d760da28d --- /dev/null +++ b/docs/user_docs/cli/kbcli_cluster_list-backup.md @@ -0,0 +1,58 @@ +--- +title: kbcli cluster list-backup +--- + +List backups. + +``` +kbcli cluster list-backup [flags] +``` + +### Examples + +``` + # list all backup + kbcli cluster list-backup +``` + +### Options + +``` + -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. + -h, --help help for list-backup + --name string The backup name to get the details. + -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. + --show-labels When printing, show all labels as the last column (default hide labels column) +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli cluster](kbcli_cluster.md) - Cluster command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/internal/cli/cmd/backupconfig/backup_config.go b/internal/cli/cmd/backupconfig/backup_config.go index 36368c948..7b8a25ea8 100644 --- a/internal/cli/cmd/backupconfig/backup_config.go +++ b/internal/cli/cmd/backupconfig/backup_config.go @@ -33,12 +33,6 @@ var backupConfigExample = templates.Examples(` # If you have already installed a snapshot-controller, only enable the snapshot backup feature kbcli backup-config --set dataProtection.enableVolumeSnapshot=true - - # Schedule automatic backup at 18:00 every day (UTC timezone) - kbcli backup-config --set dataProtection.backupSchedule="0 18 * * *" - - # Set automatic backup retention for 7 days - kbcli backup-config --set dataProtection.backupTTL="168h0m0s" `) // NewBackupConfigCmd creates the backup-config command diff --git a/internal/cli/cmd/cluster/cluster.go b/internal/cli/cmd/cluster/cluster.go index 857fdc4eb..796722792 100644 --- a/internal/cli/cmd/cluster/cluster.go +++ b/internal/cli/cmd/cluster/cluster.go @@ -78,6 +78,8 @@ func NewClusterCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr { Message: "Backup/Restore Commands:", Commands: []*cobra.Command{ + NewListBackupPolicyCmd(f, streams), + NewLEditBackupPolicyCmd(f, streams), NewCreateBackupCmd(f, streams), NewListBackupCmd(f, streams), NewDeleteBackupCmd(f, streams), diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 1ac44056e..343610b2a 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -29,7 +29,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/duration" "k8s.io/apimachinery/pkg/util/json" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -42,6 +41,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/cluster" "github.com/apecloud/kubeblocks/internal/cli/create" "github.com/apecloud/kubeblocks/internal/cli/delete" + "github.com/apecloud/kubeblocks/internal/cli/edit" "github.com/apecloud/kubeblocks/internal/cli/list" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" @@ -50,13 +50,27 @@ import ( ) var ( + listBackupPolicyExample = templates.Examples(` + # list all backup policy + kbcli cluster list-backup-policy + + # using short cmd to list backup policy of specified cluster + kbcli cluster list-bp + `) + editExample = templates.Examples(` + # edit backup policy + kbcli cluster edit-backup-policy + + # using short cmd to edit backup policy + kbcli cluster edit-bp + `) createBackupExample = templates.Examples(` # create a backup kbcli cluster backup cluster-name `) listBackupExample = templates.Examples(` # list all backup - kbcli cluster list-backups + kbcli cluster list-backup `) deleteBackupExample = templates.Examples(` # delete a backup named backup-name @@ -76,21 +90,13 @@ var ( `) ) +const annotationTrueValue = "true" + type CreateBackupOptions struct { BackupType string `json:"backupType"` BackupName string `json:"backupName"` Role string `json:"role,omitempty"` BackupPolicy string `json:"backupPolicy"` - TTL string `json:"ttl,omitempty"` - create.BaseOptions -} - -type CreateBackupPolicyOptions struct { - ClusterName string `json:"clusterName,omitempty"` - TTL string `json:"ttl,omitempty"` - ConnectionSecret string `json:"connectionSecret,omitempty"` - PolicyTemplate string `json:"policyTemplate,omitempty"` - Role string `json:"role,omitempty"` create.BaseOptions } @@ -100,6 +106,11 @@ type CreateVolumeSnapshotClassOptions struct { create.BaseOptions } +type ListBackupOptions struct { + *list.ListOptions + BackupName string +} + func (o *CreateVolumeSnapshotClassOptions) Complete() error { objs, err := o.Dynamic. Resource(types.StorageClassGVR()). @@ -112,7 +123,7 @@ func (o *CreateVolumeSnapshotClassOptions) Complete() error { if annotations == nil { continue } - if annotations["storageclass.kubernetes.io/is-default-class"] == "true" { + if annotations["storageclass.kubernetes.io/is-default-class"] == annotationTrueValue { o.Driver, _, _ = unstructured.NestedString(sc.Object, "provisioner") o.Name = "default-vsc" } @@ -137,7 +148,7 @@ func (o *CreateVolumeSnapshotClassOptions) Create() error { continue } // skip creation if default volumesnapshotclass exists. - if annotations["snapshot.storage.kubernetes.io/is-default-class"] == "true" { + if annotations["snapshot.storage.kubernetes.io/is-default-class"] == annotationTrueValue { return nil } } @@ -161,7 +172,6 @@ func (o *CreateBackupOptions) Complete() error { if len(o.BackupName) == 0 { o.BackupName = strings.Join([]string{"backup", o.Namespace, o.Name, time.Now().Format("20060102150405")}, "-") } - return nil } @@ -170,120 +180,68 @@ func (o *CreateBackupOptions) Validate() error { if o.Name == "" { return fmt.Errorf("missing cluster name") } - - connectionSecret, err := o.getConnectionSecret() - if err != nil { - return err - } - - backupPolicyTemplate, err := o.getDefaultBackupPolicyTemplate() - if err != nil { - return err + if o.BackupPolicy == "" { + return o.completeDefaultBackupPolicy() } + // check if backup policy exists + _, err := o.Dynamic.Resource(types.BackupPolicyGVR()).Namespace(o.Namespace).Get(context.TODO(), o.BackupPolicy, metav1.GetOptions{}) + // TODO: check if pvc exists + return err +} - role, err := o.getDefaultRole() +// completeDefaultBackupPolicy completes the default backup policy. +func (o *CreateBackupOptions) completeDefaultBackupPolicy() error { + defaultBackupPolicyName, err := o.getDefaultBackupPolicy() if err != nil { return err } - // apply backup policy - policyOptions := CreateBackupPolicyOptions{ - TTL: o.TTL, - ClusterName: o.Name, - ConnectionSecret: connectionSecret, - PolicyTemplate: backupPolicyTemplate, - BaseOptions: o.BaseOptions, - } - if role != "" { - policyOptions.Role = role - } - policyOptions.Name = "backup-policy-" + o.Namespace + "-" + o.Name - inputs := create.Inputs{ - CueTemplateName: "backuppolicy_template.cue", - ResourceName: types.ResourceBackupPolicies, - Group: types.DPAPIGroup, - Version: types.DPAPIVersion, - BaseOptionsObj: &policyOptions.BaseOptions, - Options: policyOptions, - } - - // cluster backup do 2 following things: - // 1. create or apply the backupPolicy, cause backupJob reference to a backupPolicy, - // and backupPolicy reference to the cluster. - // so it need apply the backupPolicy after the first backupPolicy created. - // 2. create a backupJob. - if err := policyOptions.BaseOptions.Run(inputs); err != nil && !apierrors.IsAlreadyExists(err) { - return err - } - o.BackupPolicy = policyOptions.Name - + o.BackupPolicy = defaultBackupPolicyName return nil } -func (o *CreateBackupOptions) getConnectionSecret() (string, error) { - // find secret from cluster label - opts := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s,%s=%s", - constant.AppInstanceLabelKey, o.Name, - constant.AppManagedByLabelKey, constant.AppName), - } - gvr := schema.GroupVersionResource{Version: "v1", Resource: "secrets"} - secretObjs, err := o.Dynamic.Resource(gvr).Namespace(o.Namespace).List(context.TODO(), opts) - if err != nil { - return "", err - } - if len(secretObjs.Items) == 0 { - return "", fmt.Errorf("not found connection credential for cluster %s", o.Name) - } - return secretObjs.Items[0].GetName(), nil -} - -func (o *CreateBackupOptions) getDefaultBackupPolicyTemplate() (string, error) { +func (o *CreateBackupOptions) getDefaultBackupPolicy() (string, error) { clusterObj, err := o.Dynamic.Resource(types.ClusterGVR()).Namespace(o.Namespace).Get(context.TODO(), o.Name, metav1.GetOptions{}) if err != nil { return "", err } - // find backupPolicyTemplate from cluster label + // TODO: support multiple components backup, add --componentDef flag opts := metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s", - constant.ClusterDefLabelKey, clusterObj.GetLabels()[constant.ClusterDefLabelKey]), + constant.AppInstanceLabelKey, clusterObj.GetName()), } objs, err := o.Dynamic. - Resource(types.BackupPolicyTemplateGVR()). + Resource(types.BackupPolicyGVR()). List(context.TODO(), opts) if err != nil { return "", err } if len(objs.Items) == 0 { - return "", fmt.Errorf("not found any backupPolicyTemplate for cluster %s", o.Name) - } - return objs.Items[0].GetName(), nil -} - -func (o *CreateBackupOptions) getDefaultRole() (string, error) { - gvr := schema.GroupVersionResource{Version: "v1", Resource: "pods"} - opts := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", - constant.AppInstanceLabelKey, o.Name), + return "", fmt.Errorf(`not found any backup policy for cluster "%s"`, o.Name) } - objs, err := o.Dynamic.Resource(gvr).Namespace(o.Namespace).List(context.TODO(), opts) - if err != nil { - return "", err + var defaultBackupPolicies []unstructured.Unstructured + for _, obj := range objs.Items { + if obj.GetAnnotations()[constant.DefaultBackupPolicyAnnotationKey] == annotationTrueValue { + defaultBackupPolicies = append(defaultBackupPolicies, obj) + } } - if len(objs.Items) == 0 { - return "", fmt.Errorf("not found any pods for cluster %s", o.Name) + if len(defaultBackupPolicies) == 0 { + return "", fmt.Errorf(`not found any default backup policy for cluster "%s"`, o.Name) } - pod := objs.Items[0] - // TODO(dsj):(hack fix) apecloud-mysql just support backup snapshot on the leader pod, - // backup snapshot on the follower will be support at the next version. - if o.BackupType == "snapshot" && pod.GetLabels()[constant.WorkloadTypeLabelKey] == string(appsv1alpha1.Consensus) { - return "leader", nil + if len(defaultBackupPolicies) > 1 { + return "", fmt.Errorf(`cluster "%s" has multiple default backup policies`, o.Name) } - return "", nil + return defaultBackupPolicies[0].GetName(), nil } func NewCreateBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &CreateBackupOptions{BaseOptions: create.BaseOptions{IOStreams: streams}} + customOutPut := func(opt *create.BaseOptions) { + output := fmt.Sprintf("Backup %s created successfully, you can view the progress:", opt.Name) + printer.PrintLine(output) + nextLine := fmt.Sprintf("\tkbcli cluster list-backup --name=%s -n %s", opt.Name, opt.Namespace) + printer.PrintLine(nextLine) + } inputs := create.Inputs{ Use: "backup", Short: "Create a backup.", @@ -297,12 +255,12 @@ func NewCreateBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) Factory: f, Complete: o.Complete, Validate: o.Validate, + CustomOutPut: customOutPut, ResourceNameGVRForCompletion: types.ClusterGVR(), BuildFlags: func(cmd *cobra.Command) { cmd.Flags().StringVar(&o.BackupType, "backup-type", "snapshot", "Backup type") cmd.Flags().StringVar(&o.BackupName, "backup-name", "", "Backup name") - cmd.Flags().StringVar(&o.Role, "role", "", "backup on cluster role") - cmd.Flags().StringVar(&o.TTL, "ttl", "168h0m0s", "Time to live") + cmd.Flags().StringVar(&o.BackupPolicy, "backup-policy", "", "Backup policy name, this flag will be ignored when backup-type is snapshot") }, } return create.BuildCommand(inputs) @@ -321,7 +279,7 @@ func getClusterNameMap(dClient dynamic.Interface, o *list.ListOptions) (map[stri return clusterMap, nil } -func printBackupList(o *list.ListOptions) error { +func printBackupList(o ListBackupOptions) error { dynamic, err := o.Factory.DynamicClient() if err != nil { return err @@ -339,7 +297,7 @@ func printBackupList(o *list.ListOptions) error { return nil } - clusterNameMap, err := getClusterNameMap(dynamic, o) + clusterNameMap, err := getClusterNameMap(dynamic, o.ListOptions) if err != nil { return err } @@ -361,6 +319,13 @@ func printBackupList(o *list.ListOptions) error { if backup.Status.Duration != nil { durationStr = duration.HumanDuration(backup.Status.Duration.Duration) } + if len(o.BackupName) > 0 { + if o.BackupName == obj.GetName() { + tbl.AddRow(backup.Name, clusterName, backup.Spec.BackupType, backup.Status.Phase, backup.Status.TotalSize, + durationStr, util.TimeFormat(&backup.CreationTimestamp), util.TimeFormat(backup.Status.CompletionTimestamp)) + } + continue + } tbl.AddRow(backup.Name, clusterName, backup.Spec.BackupType, backup.Status.Phase, backup.Status.TotalSize, durationStr, util.TimeFormat(&backup.CreationTimestamp), util.TimeFormat(backup.Status.CompletionTimestamp)) } @@ -369,21 +334,22 @@ func printBackupList(o *list.ListOptions) error { } func NewListBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := list.NewListOptions(f, streams, types.BackupGVR()) + o := &ListBackupOptions{ListOptions: list.NewListOptions(f, streams, types.OpsGVR())} cmd := &cobra.Command{ - Use: "list-backups", + Use: "list-backup", Short: "List backups.", - Aliases: []string{"ls-backups"}, + Aliases: []string{"ls-backup"}, Example: listBackupExample, ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.LabelSelector = util.BuildLabelSelectorByNames(o.LabelSelector, args) o.Names = nil util.CheckErr(o.Complete()) - util.CheckErr(printBackupList(o)) + util.CheckErr(printBackupList(*o)) }, } o.AddFlags(cmd) + cmd.Flags().StringVar(&o.BackupName, "name", "", "The backup name to get the details.") return cmd } @@ -591,3 +557,74 @@ func completeForDeleteRestore(o *delete.DeleteOptions, args []string) error { o.ConfirmedNames = o.Names return nil } + +func NewListBackupPolicyCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := list.NewListOptions(f, streams, types.OpsGVR()) + cmd := &cobra.Command{ + Use: "list-backup-policy", + Short: "List backups policies.", + Aliases: []string{"list-bp"}, + Example: listBackupPolicyExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.LabelSelector = util.BuildLabelSelectorByNames(o.LabelSelector, args) + o.Names = nil + util.CheckErr(o.Complete()) + util.CheckErr(printBackupPolicyList(*o)) + }, + } + o.AddFlags(cmd) + return cmd +} + +// printBackupPolicyList prints the backup policy list. +func printBackupPolicyList(o list.ListOptions) error { + dynamic, err := o.Factory.DynamicClient() + if err != nil { + return err + } + backupPolicyList, err := dynamic.Resource(types.BackupPolicyGVR()).Namespace(o.Namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: o.LabelSelector, + FieldSelector: o.FieldSelector, + }) + if err != nil { + return err + } + + if len(backupPolicyList.Items) == 0 { + o.PrintNotFoundResources() + return nil + } + + tbl := printer.NewTablePrinter(o.Out) + tbl.SetHeader("NAME", "DEFAULT", "CLUSTER", "CREATE-TIME") + for _, obj := range backupPolicyList.Items { + defaultPolicy, ok := obj.GetAnnotations()[constant.DefaultBackupPolicyAnnotationKey] + if !ok { + defaultPolicy = "false" + } + createTime := obj.GetCreationTimestamp() + tbl.AddRow(obj.GetName(), defaultPolicy, obj.GetLabels()[constant.AppInstanceLabelKey], util.TimeFormat(&createTime)) + } + tbl.Print() + return nil +} + +func NewLEditBackupPolicyCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := edit.NewEditOptions(f, streams, types.BackupPolicyGVR()) + cmd := &cobra.Command{ + Use: "edit-backup-policy", + DisableFlagsInUseLine: true, + Aliases: []string{"edit-bp"}, + Short: "Edit backup policy", + Example: editExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.BackupPolicyGVR()), + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Complete(cmd, args)) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.AddFlags(cmd) + return cmd +} diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index 34f2e0430..5cc3b611f 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -20,10 +20,12 @@ import ( "bytes" "context" "fmt" + "strings" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -35,6 +37,7 @@ import ( clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/create" "github.com/apecloud/kubeblocks/internal/cli/delete" "github.com/apecloud/kubeblocks/internal/cli/list" @@ -45,10 +48,12 @@ import ( ) var _ = Describe("DataProtection", func() { + const policyName = "policy" var streams genericclioptions.IOStreams var tf *cmdtesting.TestFactory + var out *bytes.Buffer BeforeEach(func() { - streams, _, _, _ = genericclioptions.NewTestIOStreams() + streams, _, out, _ = genericclioptions.NewTestIOStreams() tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) tf.Client = &clientfake.RESTClient{} }) @@ -58,6 +63,39 @@ var _ = Describe("DataProtection", func() { }) Context("backup", func() { + + initClient := func(policies ...*dataprotectionv1alpha1.BackupPolicy) { + clusterDef := testing.FakeClusterDef() + cluster := testing.FakeCluster(testing.ClusterName, testing.Namespace) + clusterDefLabel := map[string]string{ + constant.ClusterDefLabelKey: clusterDef.Name, + } + cluster.SetLabels(clusterDefLabel) + pods := testing.FakePods(1, testing.Namespace, testing.ClusterName) + objects := []runtime.Object{ + cluster, clusterDef, &pods.Items[0], + } + for _, v := range policies { + objects = append(objects, v) + } + tf.FakeDynamicClient = fake.NewSimpleDynamicClient(scheme.Scheme, objects...) + } + + It("list-backup-policy", func() { + By("fake client") + defaultBackupPolicy := testing.FakeBackupPolicy(policyName, testing.ClusterName) + policy2 := testing.FakeBackupPolicy("policy1", testing.ClusterName) + initClient(defaultBackupPolicy, policy2) + + By("test list-backup-policy cmd") + cmd := NewListBackupPolicyCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + cmd.Run(cmd, nil) + Expect(out.String()).Should(ContainSubstring(defaultBackupPolicy.Name)) + Expect(out.String()).Should(ContainSubstring("true")) + Expect(len(strings.Split(strings.Trim(out.String(), "\n"), "\n"))).Should(Equal(3)) + }) + It("validate create backup", func() { By("without cluster name") o := &CreateBackupOptions{ @@ -69,34 +107,35 @@ var _ = Describe("DataProtection", func() { o.IOStreams = streams Expect(o.Validate()).To(MatchError("missing cluster name")) - By("not found connection secret") + By("test without default backupPolicy") o.Name = testing.ClusterName - Expect(o.Validate()).Should(HaveOccurred()) + o.Namespace = testing.Namespace + initClient() + o.Dynamic = tf.FakeDynamicClient + Expect(o.Validate()).Should(MatchError(fmt.Errorf(`not found any backup policy for cluster "%s"`, testing.ClusterName))) + + By("test with two default backupPolicy") + defaultBackupPolicy := testing.FakeBackupPolicy(policyName, testing.ClusterName) + initClient(defaultBackupPolicy, testing.FakeBackupPolicy("policy2", testing.ClusterName)) + o.Dynamic = tf.FakeDynamicClient + Expect(o.Validate()).Should(MatchError(fmt.Errorf(`cluster "%s" has multiple default backup policies`, o.Name))) + + By("test with one default backupPolicy") + initClient(defaultBackupPolicy) + o.Dynamic = tf.FakeDynamicClient + Expect(o.Validate()).Should(Succeed()) }) It("run backup command", func() { - clusterDef := testing.FakeClusterDef() - cluster := testing.FakeCluster(testing.ClusterName, testing.Namespace) - clusterDefLabel := map[string]string{ - constant.ClusterDefLabelKey: clusterDef.Name, - } - cluster.SetLabels(clusterDefLabel) - - template := testing.FakeBackupPolicyTemplate() - template.SetLabels(clusterDefLabel) - - secrets := testing.FakeSecrets(testing.Namespace, testing.ClusterName) - pods := testing.FakePods(1, testing.Namespace, testing.ClusterName) - tf.FakeDynamicClient = fake.NewSimpleDynamicClient( - scheme.Scheme, &secrets.Items[0], cluster, clusterDef, template, &pods.Items[0]) - tf.Client = &clientfake.RESTClient{} + defaultBackupPolicy := testing.FakeBackupPolicy(policyName, testing.ClusterName) + initClient(defaultBackupPolicy) + By("test with specified backupPolicy") cmd := NewCreateBackupCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) // must succeed otherwise exit 1 and make test fails - _ = cmd.Flags().Set("backup-type", "snapshot") + _ = cmd.Flags().Set("backup-policy", defaultBackupPolicy.Name) cmd.Run(cmd, []string{testing.ClusterName}) }) - }) It("delete-backup", func() { @@ -129,7 +168,7 @@ var _ = Describe("DataProtection", func() { Expect(cmd).ShouldNot(BeNil()) By("test list-backup cmd with no backup") tf.FakeDynamicClient = testing.FakeDynamicClient() - o := list.NewListOptions(tf, streams, types.BackupGVR()) + o := ListBackupOptions{ListOptions: list.NewListOptions(tf, streams, types.BackupGVR())} Expect(printBackupList(o)).Should(Succeed()) Expect(o.ErrOut.(*bytes.Buffer).String()).Should(ContainSubstring("No backups found")) @@ -186,15 +225,13 @@ var _ = Describe("DataProtection", func() { constant.ClusterDefLabelKey: clusterDef.Name, } cluster.SetLabels(clusterDefLabel) - - template := testing.FakeBackupPolicyTemplate() - template.SetLabels(clusterDefLabel) + backupPolicy := testing.FakeBackupPolicy("backPolicy", cluster.Name) pods := testing.FakePods(1, testing.Namespace, clusterName) tf.FakeDynamicClient = fake.NewSimpleDynamicClient( - scheme.Scheme, &secrets.Items[0], &pods.Items[0], cluster, template) + scheme.Scheme, &secrets.Items[0], &pods.Items[0], cluster, backupPolicy) tf.FakeDynamicClient = fake.NewSimpleDynamicClient( - scheme.Scheme, &secrets.Items[0], &pods.Items[0], clusterDef, cluster, template) + scheme.Scheme, &secrets.Items[0], &pods.Items[0], clusterDef, cluster, backupPolicy) tf.Client = &clientfake.RESTClient{} // create backup cmd := NewCreateBackupCmd(tf, streams) diff --git a/internal/cli/cmd/kubeblocks/status.go b/internal/cli/cmd/kubeblocks/status.go index da9889134..f963a3ef7 100644 --- a/internal/cli/cmd/kubeblocks/status.go +++ b/internal/cli/cmd/kubeblocks/status.go @@ -64,7 +64,6 @@ var ( } kubeBlocksGlobalCustomResources = []schema.GroupVersionResource{ - types.BackupPolicyTemplateGVR(), types.BackupToolGVR(), types.ClusterDefGVR(), types.ClusterVersionGVR(), diff --git a/internal/cli/create/template/backup_template.cue b/internal/cli/create/template/backup_template.cue index 2967978d7..67064647c 100644 --- a/internal/cli/create/template/backup_template.cue +++ b/internal/cli/create/template/backup_template.cue @@ -18,7 +18,6 @@ options: { namespace: string backupType: string backupPolicy: string - ttl: string } // required, k8s api resource content @@ -35,6 +34,5 @@ content: { spec: { backupType: options.backupType backupPolicyName: options.backupPolicy - ttl: options.ttl } } diff --git a/internal/cli/create/template/backuppolicy_template.cue b/internal/cli/create/template/backuppolicy_template.cue deleted file mode 100644 index 0da18ea4b..000000000 --- a/internal/cli/create/template/backuppolicy_template.cue +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright ApeCloud, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// required, command line input options for parameters and flags -options: { - name: string - namespace: string - clusterName: string - ttl: string - connectionSecret: string - policyTemplate: string - role: string -} - -// required, k8s api resource content -content: { - apiVersion: "dataprotection.kubeblocks.io/v1alpha1" - kind: "BackupPolicy" - metadata: { - name: options.name - namespace: options.namespace - } - spec: { - backupPolicyTemplateName: options.policyTemplate - target: { - labelsSelector: { - matchLabels: { - "app.kubernetes.io/instance": options.clusterName - if options.role != _|_ { - "kubeblocks.io/role": options.role - } - } - } - secret: { - name: options.connectionSecret - } - } - remoteVolume: { - name: "backup-remote-volume" - persistentVolumeClaim: { - claimName: "backup-s3-pvc" - } - } - ttl: options.ttl - } -} diff --git a/internal/cli/edit/edit.go b/internal/cli/edit/edit.go new file mode 100644 index 000000000..edc89c737 --- /dev/null +++ b/internal/cli/edit/edit.go @@ -0,0 +1,72 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edit + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/cmd/util/editor" + + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +type EditOptions struct { + editor.EditOptions + Factory cmdutil.Factory + // Name are the resource name + Name string + GVR schema.GroupVersionResource +} + +func NewEditOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, + gvr schema.GroupVersionResource) *EditOptions { + return &EditOptions{ + Factory: f, + GVR: gvr, + EditOptions: *editor.NewEditOptions(editor.NormalEditMode, streams), + } +} + +func (o *EditOptions) AddFlags(cmd *cobra.Command) { + // bind flag structs + o.RecordFlags.AddFlags(cmd) + o.PrintFlags.AddFlags(cmd) + + usage := "to use to edit the resource" + cmdutil.AddFilenameOptionFlags(cmd, &o.FilenameOptions, usage) + cmdutil.AddValidateFlags(cmd) + cmd.Flags().BoolVarP(&o.OutputPatch, "output-patch", "", o.OutputPatch, "Output the patch if the resource is edited.") + cmd.Flags().BoolVar(&o.WindowsLineEndings, "windows-line-endings", o.WindowsLineEndings, + "Defaults to the line ending native to your platform.") + cmdutil.AddFieldManagerFlagVar(cmd, &o.FieldManager, "kubectl-edit") + cmdutil.AddApplyAnnotationVarFlags(cmd, &o.ApplyAnnotation) + cmdutil.AddSubresourceFlags(cmd, &o.Subresource, "If specified, edit will operate on the subresource of the requested object.", editor.SupportedSubresources...) +} + +func (o *EditOptions) Complete(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return fmt.Errorf("missing the name") + } + if len(args) > 0 { + o.Name = args[0] + } + return o.EditOptions.Complete(o.Factory, []string{util.GVRToString(o.GVR), o.Name}, cmd) +} diff --git a/internal/cli/edit/edit_test.go b/internal/cli/edit/edit_test.go new file mode 100644 index 000000000..6d31d7a8b --- /dev/null +++ b/internal/cli/edit/edit_test.go @@ -0,0 +1,75 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edit + +import ( + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/types" +) + +var _ = Describe("List", func() { + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + ) + + mockClient := func() *corev1.PodList { + pods, _, _ := cmdtesting.TestData() + tf = cmdtesting.NewTestFactory().WithNamespace("test") + defer tf.Cleanup() + + codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) + tf.UnstructuredClient = &fake.RESTClient{ + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + Resp: &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, pods)}, + } + return pods + } + + AfterEach(func() { + tf.Cleanup() + }) + + It("test edit", func() { + pods := mockClient() + o := NewEditOptions(tf, streams, schema.GroupVersionResource{Group: "", Resource: "pods", Version: types.K8sCoreAPIVersion}) + cmd := &cobra.Command{ + Use: "edit-test", + Short: "edit test.", + Run: func(cmd *cobra.Command, args []string) { + + }, + } + o.AddFlags(cmd) + podName := pods.Items[0].Name + Expect(o.Complete(cmd, []string{})).Should(MatchError("missing the name")) + Expect(o.Complete(cmd, []string{podName})).ShouldNot(HaveOccurred()) + }) +}) diff --git a/internal/cli/edit/suite_test.go b/internal/cli/edit/suite_test.go new file mode 100644 index 000000000..621f29747 --- /dev/null +++ b/internal/cli/edit/suite_test.go @@ -0,0 +1,29 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edit + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestList(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Edit Suite") +} diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index 1461d4f7f..c41a9c7f7 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -53,7 +53,6 @@ const ( KubeBlocksChartName = "fake-kubeblocks" KubeBlocksChartURL = "fake-kubeblocks-chart-url" BackupToolName = "fake-backup-tool" - BackupTemplateName = "fake-backup-policy-template" ) func GetRandomStr() string { @@ -288,14 +287,21 @@ func FakeBackupTool() *dpv1alpha1.BackupTool { return tool } -func FakeBackupPolicyTemplate() *dpv1alpha1.BackupPolicyTemplate { - template := &dpv1alpha1.BackupPolicyTemplate{ +func FakeBackupPolicy(backupPolicyName, clusterName string) *dpv1alpha1.BackupPolicy { + template := &dpv1alpha1.BackupPolicy{ TypeMeta: metav1.TypeMeta{ APIVersion: fmt.Sprintf("%s/%s", types.DPAPIGroup, types.DPAPIVersion), - Kind: types.KindBackupPolicyTemplate, + Kind: types.KindBackupPolicy, }, ObjectMeta: metav1.ObjectMeta{ - Name: BackupTemplateName, + Name: backupPolicyName, + Namespace: Namespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: clusterName, + }, + Annotations: map[string]string{ + constant.DefaultBackupPolicyAnnotationKey: "true", + }, }, } return template diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index 324700fef..609c1549b 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -79,7 +79,7 @@ const ( KindConfigConstraint = "ConfigConstraint" KindBackup = "Backup" KindRestoreJob = "RestoreJob" - KindBackupPolicyTemplate = "BackupPolicyTemplate" + KindBackupPolicy = "BackupPolicy" KindOps = "OpsRequest" ) @@ -186,8 +186,8 @@ func BackupGVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: DPAPIGroup, Version: DPAPIVersion, Resource: ResourceBackups} } -func BackupPolicyTemplateGVR() schema.GroupVersionResource { - return schema.GroupVersionResource{Group: DPAPIGroup, Version: DPAPIVersion, Resource: ResourceBackupPolicyTemplates} +func BackupPolicyGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: DPAPIGroup, Version: DPAPIVersion, Resource: ResourceBackupPolicies} } func BackupToolGVR() schema.GroupVersionResource { diff --git a/internal/cli/util/util_test.go b/internal/cli/util/util_test.go index c474e83b5..db2562f00 100644 --- a/internal/cli/util/util_test.go +++ b/internal/cli/util/util_test.go @@ -84,7 +84,7 @@ var _ = Describe("util", func() { "spec": map[string]interface{}{ "backupPolicyName": "backup-policy-demo", "backupType": "full", - "ttl": "168h0m0s", + "ttl": "7d", }, }, } diff --git a/internal/constant/const.go b/internal/constant/const.go index 7f78ebab1..7c6fcdc5d 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -75,13 +75,15 @@ const ( ClassProviderLabelKey = "class.kubeblocks.io/provider" // kubeblocks.io annotations - OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster - ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile - RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart - SnapShotForStartAnnotationKey = "kubeblocks.io/snapshot-for-start" - RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. - ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. - LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" + OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster + ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile + RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart + SnapShotForStartAnnotationKey = "kubeblocks.io/snapshot-for-start" + RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. + ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. + LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" + DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" + BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" // ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl ConfigurationTplLabelPrefixKey = "config.kubeblocks.io/tpl" @@ -200,3 +202,9 @@ const ( const ( KBReplicationSetPrimaryPodName = "KB_PRIMARY_POD_NAME" ) + +// username and password are keys in created secrets for others to refer to. +const ( + AccountNameForSecret = "username" + AccountPasswdForSecret = "password" +) diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index a5bd768f7..84f2f0e7b 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -535,22 +535,6 @@ func BuildEnvConfig(params BuilderParams, reqCtx intctrlutil.RequestCtx, cli cli return &config, nil } -func BuildBackupPolicy(sts *appsv1.StatefulSet, - template *dataprotectionv1alpha1.BackupPolicyTemplate, - backupKey types.NamespacedName) (*dataprotectionv1alpha1.BackupPolicy, error) { - backupKey.Name = backupKey.Name + "-" + randomString(6) - backupPolicy := dataprotectionv1alpha1.BackupPolicy{} - if err := buildFromCUE("backup_policy_template.cue", map[string]any{ - "sts": sts, - "backup_key": backupKey, - "template": template.Name, - }, "backup_policy", &backupPolicy); err != nil { - return nil, err - } - - return &backupPolicy, nil -} - func BuildBackup(sts *appsv1.StatefulSet, backupPolicyName string, backupKey types.NamespacedName) (*dataprotectionv1alpha1.Backup, error) { diff --git a/internal/controller/builder/builder_test.go b/internal/controller/builder/builder_test.go index fc1830815..ba488c008 100644 --- a/internal/controller/builder/builder_test.go +++ b/internal/controller/builder/builder_test.go @@ -34,7 +34,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" cfgcm "github.com/apecloud/kubeblocks/internal/configuration/config_manager" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/component" @@ -184,16 +183,6 @@ var _ = Describe("builder", func() { return ¶ms } - newBackupPolicyTemplate := func() *dataprotectionv1alpha1.BackupPolicyTemplate { - return testapps.NewBackupPolicyTemplateFactory("backup-policy-template-mysql"). - SetBackupToolName("mysql-xtrabackup"). - SetSchedule("0 2 * * *"). - SetTTL("168h0m0s"). - AddHookPreCommand("touch /data/mysql/.restore;sync"). - AddHookPostCommand("rm -f /data/mysql/.restore;sync"). - Create(&testCtx).GetObject() - } - Context("has helper function which builds specific object from cue template", func() { It("builds PVC correctly", func() { snapshotName := "test-snapshot-name" @@ -412,18 +401,6 @@ var _ = Describe("builder", func() { checkEnvValues() }) - It("builds BackupPolicy correctly", func() { - sts := newStsObj() - backupPolicyTemplate := newBackupPolicyTemplate() - backupKey := types.NamespacedName{ - Namespace: "default", - Name: "test-backup", - } - policy, err := BuildBackupPolicy(sts, backupPolicyTemplate, backupKey) - Expect(err).Should(BeNil()) - Expect(policy).ShouldNot(BeNil()) - }) - It("builds BackupJob correctly", func() { sts := newStsObj() backupJobKey := types.NamespacedName{ diff --git a/internal/controller/builder/cue/backup_policy_template.cue b/internal/controller/builder/cue/backup_policy_template.cue deleted file mode 100644 index 89d01f75c..000000000 --- a/internal/controller/builder/cue/backup_policy_template.cue +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright ApeCloud, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -sts: { - metadata: { - labels: { - "app.kubernetes.io/instance": string - } - namespace: string - } -} -backup_key: { - Name: string - Namespace: string -} -template: string -backup_policy: { - apiVersion: "dataprotection.kubeblocks.io/v1alpha1" - kind: "BackupPolicy" - metadata: { - name: backup_key.Name - // generateName: "\(backup_key.Name)-" - namespace: backup_key.Namespace - labels: { - "apps.kubeblocks.io/managed-by": "cluster" - for k, v in sts.metadata.labels { - "\(k)": "\(v)" - } - } - } - spec: { - "backupPolicyTemplateName": template - "target": { - "labelsSelector": { - "matchLabels": { - "app.kubernetes.io/instance": sts.metadata.labels["app.kubernetes.io/instance"] - "apps.kubeblocks.io/component-name": sts.metadata.labels["apps.kubeblocks.io/component-name"] - } - } - "secret": { - "name": "wesql-cluster" - } - } - "remoteVolume": { - "name": "backup-remote-volume" - "persistentVolumeClaim": { - "claimName": "backup-s3-pvc" - } - } - "hooks": { - "preCommands": [ - "touch /data/mysql/data/.restore; sync", - ] - "postCommands": [ - ] - } - "onFailAttempted": 3 - } -} diff --git a/internal/controller/builder/cue/conn_credential_template.cue b/internal/controller/builder/cue/conn_credential_template.cue index 6f8508711..a6a0643c3 100644 --- a/internal/controller/builder/cue/conn_credential_template.cue +++ b/internal/controller/builder/cue/conn_credential_template.cue @@ -39,7 +39,7 @@ secret: { "app.kubernetes.io/instance": cluster.metadata.name "app.kubernetes.io/managed-by": "kubeblocks" if clusterdefinition.spec.type != _|_ { - "apps.kubeblocks.io/cluster-type": clusterdefinition.spec.type + "apps.kubeblocks.io/cluster-type": clusterdefinition.spec.type } } } diff --git a/internal/controller/component/component.go b/internal/controller/component/component.go index 550704263..90c29eafa 100644 --- a/internal/controller/component/component.go +++ b/internal/controller/component/component.go @@ -59,6 +59,7 @@ func BuildComponent( Issuer: clusterCompSpec.Issuer, VolumeTypes: clusterCompDefObj.VolumeTypes, CustomLabelSpecs: clusterCompDefObj.CustomLabelSpecs, + ComponentDef: clusterCompSpec.ComponentDefRef, } // resolve component.ConfigTemplates diff --git a/internal/controller/component/restore_utils_test.go b/internal/controller/component/restore_utils_test.go index 2d5e5488c..7bfd0a575 100644 --- a/internal/controller/component/restore_utils_test.go +++ b/internal/controller/component/restore_utils_test.go @@ -36,7 +36,6 @@ import ( var _ = Describe("probe_utils", func() { const backupPolicyName = "test-backup-policy" - const defaultTTL = "168h0m0s" const backupName = "test-backup-job" var backupToolName string @@ -81,7 +80,6 @@ var _ = Describe("probe_utils", func() { Log: logger, } backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dataprotectionv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() @@ -107,7 +105,6 @@ var _ = Describe("probe_utils", func() { Log: logger, } backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyName). SetBackupType(dataprotectionv1alpha1.BackupTypeSnapshot). Create(&testCtx).GetObject() diff --git a/internal/controller/component/type.go b/internal/controller/component/type.go index fbaaa91d6..5723e08d7 100644 --- a/internal/controller/component/type.go +++ b/internal/controller/component/type.go @@ -53,6 +53,7 @@ type SynthesizedComponent struct { Issuer *v1alpha1.Issuer `json:"issuer,omitempty"` VolumeTypes []v1alpha1.VolumeTypeSpec `json:"VolumeTypes,omitempty"` CustomLabelSpecs []v1alpha1.CustomLabelSpec `json:"customLabelSpecs,omitempty"` + ComponentDef string `json:"componentDef,omitempty"` } // GetPrimaryIndex provides PrimaryIndex value getter, if PrimaryIndex is diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 14e40c12d..8d53cd0d8 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -201,6 +201,8 @@ func (c *clusterPlanBuilder) Build() (graph.Plan, error) { &clusterTransformer{cc: *cr, cli: c.cli, ctx: c.ctx}, // tls certs secret &tlsCertsTransformer{cr: *cr, cli: roClient, ctx: c.ctx}, + // transform backupPolicy tpl to backuppolicy.dataprotection.kubeblocks.io + &backupPolicyTPLTransformer{cr: *cr, cli: c.cli, ctx: c.ctx}, // add our finalizer to all objects &ownershipTransformer{finalizer: dbClusterFinalizerName}, // make all workload objects depending on credential secret diff --git a/internal/controller/lifecycle/transform_utils.go b/internal/controller/lifecycle/transform_utils.go index f0b80327f..efba6a66f 100644 --- a/internal/controller/lifecycle/transform_utils.go +++ b/internal/controller/lifecycle/transform_utils.go @@ -29,6 +29,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" types2 "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/graph" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" @@ -151,8 +152,10 @@ func getBackupObjects(reqCtx intctrlutil.RequestCtx, // get backup tool backupTool := &dataprotectionv1alpha1.BackupTool{} - if err := cli.Get(reqCtx.Ctx, types.NamespacedName{Name: backup.Status.BackupToolName}, backupTool); err != nil { - return nil, nil, err + if backup.Spec.BackupType != dataprotectionv1alpha1.BackupTypeSnapshot { + if err := cli.Get(reqCtx.Ctx, types.NamespacedName{Name: backup.Status.BackupToolName}, backupTool); err != nil { + return nil, nil, err + } } return backup, backupTool, nil } @@ -161,3 +164,26 @@ func isTypeOf[T interface{}](obj client.Object) bool { _, ok := obj.(T) return ok } + +// getBackupPolicyFromTemplate gets backup policy from template policy template. +func getBackupPolicyFromTemplate(reqCtx intctrlutil.RequestCtx, + cli types2.ReadonlyClient, + cluster *appsv1alpha1.Cluster, + componentDef, + backupPolicyTemplateName string) (*dataprotectionv1alpha1.BackupPolicy, error) { + backupPolicyList := &dataprotectionv1alpha1.BackupPolicyList{} + if err := cli.List(reqCtx.Ctx, backupPolicyList, + client.InNamespace(cluster.Namespace), + client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.Name, + constant.KBAppComponentDefRefLabelKey: componentDef, + }); err != nil { + return nil, err + } + for _, backupPolicy := range backupPolicyList.Items { + if backupPolicy.Annotations[constant.BackupPolicyTemplateAnnotationKey] == backupPolicyTemplateName { + return &backupPolicy, nil + } + } + return nil, nil +} diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go new file mode 100644 index 000000000..fa70d38b8 --- /dev/null +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -0,0 +1,313 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "fmt" + + "golang.org/x/exp/slices" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" + types2 "github.com/apecloud/kubeblocks/internal/controller/client" + "github.com/apecloud/kubeblocks/internal/controller/graph" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" +) + +// backupPolicyTPLTransformer transforms the backup policy template to the backup policy. +type backupPolicyTPLTransformer struct { + cr clusterRefResources + cli types2.ReadonlyClient + ctx intctrlutil.RequestCtx +} + +func (r *backupPolicyTPLTransformer) Transform(dag *graph.DAG) error { + clusterDefName := r.cr.cd.Name + backupPolicyTPLs := &appsv1alpha1.BackupPolicyTemplateList{} + if err := r.cli.List(r.ctx.Ctx, backupPolicyTPLs, client.MatchingLabels{constant.ClusterDefLabelKey: clusterDefName}); err != nil { + return err + } + if len(backupPolicyTPLs.Items) == 0 { + return nil + } + rootVertex, err := findRootVertex(dag) + if err != nil { + return err + } + origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) + for _, tpl := range backupPolicyTPLs.Items { + for _, v := range tpl.Spec.BackupPolicies { + compDef := r.cr.cd.GetComponentDefByName(v.ComponentDefRef) + if compDef == nil { + return intctrlutil.NewNotFound("componentDef %s not found in ClusterDefinition: %s ", v.ComponentDefRef, clusterDefName) + } + // build the backup policy from the template. + backupPolicy := r.transformBackupPolicy(v, origCluster, compDef.WorkloadType, tpl.Name) + vertex := &lifecycleVertex{obj: backupPolicy} + dag.AddVertex(vertex) + dag.Connect(rootVertex, vertex) + } + } + return nil +} + +// transformBackupPolicy transform backup policy template to backup policy. +func (r *backupPolicyTPLTransformer) transformBackupPolicy(policyTPL appsv1alpha1.BackupPolicy, + cluster *appsv1alpha1.Cluster, + workloadType appsv1alpha1.WorkloadType, + tplName string) *dataprotectionv1alpha1.BackupPolicy { + backupPolicyName := GenerateBackupPolicyName(cluster.Name, policyTPL.ComponentDefRef) + backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} + if err := r.cli.Get(r.ctx.Ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: backupPolicyName}, backupPolicy); err != nil && !apierrors.IsNotFound(err) { + return nil + } + if len(backupPolicy.Name) == 0 { + // build a new backup policy from the backup policy template. + return r.buildBackupPolicy(policyTPL, cluster, workloadType, tplName) + } + // sync the existing backup policy with the cluster changes + r.syncBackupPolicy(backupPolicy, cluster, policyTPL, workloadType, tplName) + return backupPolicy +} + +// syncBackupPolicy syncs labels and annotations of the backup policy with the cluster changes. +func (r *backupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotectionv1alpha1.BackupPolicy, + cluster *appsv1alpha1.Cluster, + policyTPL appsv1alpha1.BackupPolicy, + workloadType appsv1alpha1.WorkloadType, + tplName string) { + // update labels and annotations of the backup policy. + if backupPolicy.Annotations == nil { + backupPolicy.Annotations = map[string]string{} + } + backupPolicy.Annotations[constant.DefaultBackupPolicyAnnotationKey] = "true" + backupPolicy.Annotations[constant.BackupPolicyTemplateAnnotationKey] = tplName + if backupPolicy.Labels == nil { + backupPolicy.Labels = map[string]string{} + } + backupPolicy.Labels[constant.AppInstanceLabelKey] = cluster.Name + backupPolicy.Labels[constant.KBAppComponentDefRefLabelKey] = policyTPL.ComponentDefRef + + // only update the role labelSelector of the backup target instance when component workload is Replication/Consensus. + if !slices.Contains([]appsv1alpha1.WorkloadType{appsv1alpha1.Replication, appsv1alpha1.Consensus}, workloadType) { + return + } + component := r.getFirstComponent(cluster, policyTPL.ComponentDefRef) + if component == nil { + return + } + // convert role labelSelector based on the replicas of the component automatically. + syncTheRoleLabel := func(target dataprotectionv1alpha1.TargetCluster, + basePolicy appsv1alpha1.BasePolicy) dataprotectionv1alpha1.TargetCluster { + role := basePolicy.Target.Role + if len(role) == 0 { + return target + } + if target.LabelsSelector == nil || target.LabelsSelector.MatchLabels == nil { + target.LabelsSelector = &metav1.LabelSelector{MatchLabels: map[string]string{}} + } + if component.Replicas == 1 { + // if replicas is 1, remove the role label selector. + delete(target.LabelsSelector.MatchLabels, constant.RoleLabelKey) + } else { + target.LabelsSelector.MatchLabels[constant.RoleLabelKey] = role + } + return target + } + if backupPolicy.Spec.Snapshot != nil { + backupPolicy.Spec.Snapshot.Target = syncTheRoleLabel(backupPolicy.Spec.Snapshot.Target, + policyTPL.Snapshot.BasePolicy) + } + if backupPolicy.Spec.Full != nil { + backupPolicy.Spec.Full.Target = syncTheRoleLabel(backupPolicy.Spec.Full.Target, + policyTPL.Full.BasePolicy) + } + if backupPolicy.Spec.Incremental != nil { + backupPolicy.Spec.Incremental.Target = syncTheRoleLabel(backupPolicy.Spec.Incremental.Target, + policyTPL.Incremental.BasePolicy) + } +} + +// buildBackupPolicy builds a new backup policy from the backup policy template. +func (r *backupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.BackupPolicy, + cluster *appsv1alpha1.Cluster, + workloadType appsv1alpha1.WorkloadType, + tplName string) *dataprotectionv1alpha1.BackupPolicy { + backupPolicy := &dataprotectionv1alpha1.BackupPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: GenerateBackupPolicyName(cluster.Name, policyTPL.ComponentDefRef), + Namespace: cluster.Namespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: cluster.Name, + constant.KBAppComponentDefRefLabelKey: policyTPL.ComponentDefRef, + }, + Annotations: map[string]string{ + constant.DefaultBackupPolicyAnnotationKey: "true", + constant.BackupPolicyTemplateAnnotationKey: tplName, + }, + }, + } + bpSpec := backupPolicy.Spec + bpSpec.TTL = policyTPL.TTL + bpSpec.Schedule.BaseBackup = r.convertBaseBackupSchedulePolicy(policyTPL.Schedule.BaseBackup) + bpSpec.Schedule.Incremental = r.convertSchedulePolicy(policyTPL.Schedule.Incremental) + component := r.getFirstComponent(cluster, policyTPL.ComponentDefRef) + if component != nil { + bpSpec.Full = r.convertCommonPolicy(policyTPL.Full, cluster.Name, *component, workloadType) + bpSpec.Incremental = r.convertCommonPolicy(policyTPL.Incremental, cluster.Name, *component, workloadType) + bpSpec.Snapshot = r.convertSnapshotPolicy(policyTPL.Snapshot, cluster.Name, *component, workloadType) + } + backupPolicy.Spec = bpSpec + return backupPolicy +} + +// getFirstComponent returns the first component name of the componentDefRef. +func (r *backupPolicyTPLTransformer) getFirstComponent(cluster *appsv1alpha1.Cluster, + componentDefRef string) *appsv1alpha1.ClusterComponentSpec { + for _, v := range cluster.Spec.ComponentSpecs { + if v.ComponentDefRef == componentDefRef { + return &v + } + } + return nil +} + +// convertSchedulePolicy converts the schedulePolicy from backupPolicyTemplate. +func (r *backupPolicyTPLTransformer) convertSchedulePolicy(sp *appsv1alpha1.SchedulePolicy) *dataprotectionv1alpha1.SchedulePolicy { + if sp == nil { + return nil + } + return &dataprotectionv1alpha1.SchedulePolicy{ + Enable: sp.Enable, + CronExpression: sp.CronExpression, + } +} + +// convertBaseBackupSchedulePolicy converts the baseBackupSchedulePolicy from backupPolicyTemplate. +func (r *backupPolicyTPLTransformer) convertBaseBackupSchedulePolicy(sp *appsv1alpha1.BaseBackupSchedulePolicy) *dataprotectionv1alpha1.BaseBackupSchedulePolicy { + if sp == nil { + return nil + } + schedulePolicy := r.convertSchedulePolicy(&sp.SchedulePolicy) + return &dataprotectionv1alpha1.BaseBackupSchedulePolicy{ + Type: dataprotectionv1alpha1.BaseBackupType(sp.Type), + SchedulePolicy: *schedulePolicy, + } +} + +// convertBasePolicy converts the basePolicy from backupPolicyTemplate. +func (r *backupPolicyTPLTransformer) convertBasePolicy(bp appsv1alpha1.BasePolicy, + clusterName string, + component appsv1alpha1.ClusterComponentSpec, + workloadType appsv1alpha1.WorkloadType) dataprotectionv1alpha1.BasePolicy { + basePolicy := dataprotectionv1alpha1.BasePolicy{ + Target: dataprotectionv1alpha1.TargetCluster{ + LabelsSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + constant.AppInstanceLabelKey: clusterName, + constant.KBAppComponentLabelKey: component.Name, + }, + }, + }, + BackupsHistoryLimit: bp.BackupsHistoryLimit, + OnFailAttempted: bp.OnFailAttempted, + } + if len(bp.BackupStatusUpdates) != 0 { + backupStatusUpdates := make([]dataprotectionv1alpha1.BackupStatusUpdate, len(bp.BackupStatusUpdates)) + for i, v := range bp.BackupStatusUpdates { + backupStatusUpdates[i] = dataprotectionv1alpha1.BackupStatusUpdate{ + Path: v.Path, + ContainerName: v.ContainerName, + Script: v.Script, + UpdateStage: dataprotectionv1alpha1.BackupStatusUpdateStage(v.UpdateStage), + } + } + basePolicy.BackupStatusUpdates = backupStatusUpdates + } + switch workloadType { + case appsv1alpha1.Replication, appsv1alpha1.Consensus: + if len(bp.Target.Role) > 0 && component.Replicas > 1 { + // the role only works when the component has multiple replicas. + basePolicy.Target.LabelsSelector.MatchLabels[constant.RoleLabelKey] = bp.Target.Role + } + } + // build the target secret. + if len(bp.Target.Account) > 0 { + basePolicy.Target.Secret = &dataprotectionv1alpha1.BackupPolicySecret{ + Name: fmt.Sprintf("%s-%s-%s", clusterName, component.Name, bp.Target.Account), + PasswordKey: constant.AccountPasswdForSecret, + UsernameKey: constant.AccountNameForSecret, + } + } else { + basePolicy.Target.Secret = &dataprotectionv1alpha1.BackupPolicySecret{ + Name: fmt.Sprintf("%s-conn-credential", clusterName), + } + connectionCredentialKey := bp.Target.ConnectionCredentialKey + if connectionCredentialKey.PasswordKey != nil { + basePolicy.Target.Secret.PasswordKey = *connectionCredentialKey.PasswordKey + } + if connectionCredentialKey.UsernameKey != nil { + basePolicy.Target.Secret.UsernameKey = *connectionCredentialKey.UsernameKey + } + } + return basePolicy +} + +// convertBaseBackupSchedulePolicy converts the snapshotPolicy from backupPolicyTemplate. +func (r *backupPolicyTPLTransformer) convertSnapshotPolicy(sp *appsv1alpha1.SnapshotPolicy, + clusterName string, + component appsv1alpha1.ClusterComponentSpec, + workloadType appsv1alpha1.WorkloadType) *dataprotectionv1alpha1.SnapshotPolicy { + if sp == nil { + return nil + } + snapshotPolicy := &dataprotectionv1alpha1.SnapshotPolicy{ + BasePolicy: r.convertBasePolicy(sp.BasePolicy, clusterName, component, workloadType), + } + if sp.Hooks != nil { + snapshotPolicy.Hooks = &dataprotectionv1alpha1.BackupPolicyHook{ + PreCommands: sp.Hooks.PreCommands, + PostCommands: sp.Hooks.PostCommands, + ContainerName: sp.Hooks.ContainerName, + Image: sp.Hooks.Image, + } + } + return snapshotPolicy +} + +// convertBaseBackupSchedulePolicy converts the commonPolicy from backupPolicyTemplate. +func (r *backupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.CommonBackupPolicy, + clusterName string, + component appsv1alpha1.ClusterComponentSpec, + workloadType appsv1alpha1.WorkloadType) *dataprotectionv1alpha1.CommonBackupPolicy { + if bp == nil { + return nil + } + return &dataprotectionv1alpha1.CommonBackupPolicy{ + BackupToolName: bp.BackupToolName, + BasePolicy: r.convertBasePolicy(bp.BasePolicy, clusterName, component, workloadType), + } +} + +// GenerateBackupPolicyName generates the backup policy name which is created from backup policy template. +func GenerateBackupPolicyName(clusterName, componentDef string) string { + return fmt.Sprintf("%s-%s-backup-policy", clusterName, componentDef) +} diff --git a/internal/controller/lifecycle/transformer_object_action.go b/internal/controller/lifecycle/transformer_object_action.go index e20d62702..42ea4d1fe 100644 --- a/internal/controller/lifecycle/transformer_object_action.go +++ b/internal/controller/lifecycle/transformer_object_action.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" client2 "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/graph" @@ -48,6 +49,7 @@ func ownKinds() []client.ObjectList { &corev1.ConfigMapList{}, &corev1.PersistentVolumeClaimList{}, &policyv1.PodDisruptionBudgetList{}, + &dataprotectionv1alpha1.BackupPolicyList{}, } } diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 015c26dcf..d83e3b34a 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -344,7 +344,8 @@ func doBackup(reqCtx intctrlutil.RequestCtx, snapshotKey, stsObj, vcts, - component.HorizontalScalePolicy.BackupTemplateSelector, + component.ComponentDef, + component.HorizontalScalePolicy.BackupPolicyTemplateName, dag, root); err != nil { return err @@ -569,22 +570,21 @@ func doSnapshot(cli types2.ReadonlyClient, snapshotKey types.NamespacedName, stsObj *appsv1.StatefulSet, vcts []corev1.PersistentVolumeClaimTemplate, - backupTemplateSelector map[string]string, + componentDef, + backupPolicyTemplateName string, dag *graph.DAG, root graph.Vertex) error { ctx := reqCtx.Ctx - ml := client.MatchingLabels(backupTemplateSelector) - backupPolicyTemplateList := dataprotectionv1alpha1.BackupPolicyTemplateList{} - // find backuppolicytemplate by clusterdefinition - if err := cli.List(ctx, &backupPolicyTemplateList, ml); err != nil { + backupPolicyTemplate := &appsv1alpha1.BackupPolicyTemplate{} + if err := cli.Get(ctx, client.ObjectKey{Name: backupPolicyTemplateName}, backupPolicyTemplate); err != nil && !apierrors.IsNotFound(err) { return err } - if len(backupPolicyTemplateList.Items) > 0 { + if len(backupPolicyTemplate.Name) > 0 { // if there is backuppolicytemplate created by provider // create backupjob CR, will ignore error if already exists - err := createBackup(reqCtx, cli, stsObj, &backupPolicyTemplateList.Items[0], snapshotKey, cluster, dag, root) + err := createBackup(reqCtx, cli, stsObj, componentDef, backupPolicyTemplateName, snapshotKey, cluster, dag, root) if err != nil { return err } @@ -676,39 +676,15 @@ func checkedCreatePVCFromSnapshot(cli types2.ReadonlyClient, func createBackup(reqCtx intctrlutil.RequestCtx, cli types2.ReadonlyClient, sts *appsv1.StatefulSet, - backupPolicyTemplate *dataprotectionv1alpha1.BackupPolicyTemplate, + componentDef, + backupPolicyTemplateName string, backupKey types.NamespacedName, cluster *appsv1alpha1.Cluster, dag *graph.DAG, root graph.Vertex) error { ctx := reqCtx.Ctx - createBackupPolicy := func() (backupPolicyName string, Vertex *lifecycleVertex, err error) { - backupPolicyName = "" - backupPolicyList := dataprotectionv1alpha1.BackupPolicyList{} - ml := getBackupMatchingLabels(cluster.Name, sts.Labels[constant.KBAppComponentLabelKey]) - if err = cli.List(ctx, &backupPolicyList, ml); err != nil { - return - } - if len(backupPolicyList.Items) > 0 { - backupPolicyName = backupPolicyList.Items[0].Name - return - } - backupPolicy, err := builder.BuildBackupPolicy(sts, backupPolicyTemplate, backupKey) - if err != nil { - return - } - if err = controllerutil.SetControllerReference(cluster, backupPolicy, scheme); err != nil { - return - } - backupPolicyName = backupPolicy.Name - vertex := &lifecycleVertex{obj: backupPolicy, action: actionPtr(CREATE)} - dag.AddVertex(vertex) - dag.Connect(root, vertex) - return - } - - createBackup := func(backupPolicyName string, policyVertex *lifecycleVertex) error { + createBackup := func(backupPolicyName string) error { backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} if err := cli.Get(ctx, client.ObjectKey{Namespace: backupKey.Namespace, Name: backupPolicyName}, backupPolicy); err != nil && !apierrors.IsNotFound(err) { return err @@ -744,12 +720,14 @@ func createBackup(reqCtx intctrlutil.RequestCtx, dag.Connect(root, vertex) return nil } - - backupPolicyName, policyVertex, err := createBackupPolicy() + backupPolicy, err := getBackupPolicyFromTemplate(reqCtx, cli, cluster, componentDef, backupPolicyTemplateName) if err != nil { return err } - if err := createBackup(backupPolicyName, policyVertex); err != nil { + if backupPolicy == nil { + return intctrlutil.NewNotFound("not found any backup policy created by %s", backupPolicyTemplateName) + } + if err = createBackup(backupPolicy.Name); err != nil { return err } diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go index ba172bab4..fae154a4a 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go @@ -215,7 +215,7 @@ var _ = Describe("sts horizontal scaling test", func() { // // By("creating a backup as user do") // backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - // SetTTL("168h0m0s"). + // SetTTL("7d"). // SetBackupPolicyName(backupPolicyName). // SetBackupType(dataprotectionv1alpha1.BackupTypeSnapshot). // AddAppInstanceLabel(clusterName). diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index ed4318430..6d25bd996 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -37,10 +37,15 @@ func (v *Error) Error() string { type ErrorType string const ( - // ErrorTypeBackupNotCompleted is used to report backup not completed. - ErrorTypeBackupNotCompleted ErrorType = "BackupNotCompleted" // ErrorWaitCacheRefresh waits for synchronization of the corresponding object cache in client-go from ApiServer. ErrorWaitCacheRefresh = "WaitCacheRefresh" + + // ErrorTypeBackupNotCompleted is used to report backup not completed. + ErrorTypeBackupNotCompleted ErrorType = "BackupNotCompleted" + // ErrorTypeBackupPolicyFailed backup policy failed. + ErrorTypeBackupPolicyFailed = "BackupPolicyFailed" + // ErrorTypeNotFound not found any resource. + ErrorTypeNotFound = "NotFound" ) var ErrFailedToAddFinalizer = errors.New("failed to add finalizer") @@ -66,3 +71,16 @@ func IsTargetError(err error, errorType ErrorType) bool { } return false } + +// NewNotFound returns a new Error with ErrorTypeNotFound. +func NewNotFound(format string, a ...any) *Error { + return &Error{ + Type: ErrorTypeNotFound, + Message: fmt.Sprintf(format, a...), + } +} + +// IsNotFound returns true if the specified error is the error type of ErrorTypeNotFound. +func IsNotFound(err error) bool { + return IsTargetError(err, ErrorTypeNotFound) +} diff --git a/internal/generics/type.go b/internal/generics/type.go index 1095efa6f..6aeb47f77 100644 --- a/internal/generics/type.go +++ b/internal/generics/type.go @@ -83,7 +83,7 @@ var OpsRequestSignature = func(_ appsv1alpha1.OpsRequest, _ appsv1alpha1.OpsRequ var ConfigConstraintSignature = func(_ appsv1alpha1.ConfigConstraint, _ appsv1alpha1.ConfigConstraintList) { } -var BackupPolicyTemplateSignature = func(_ dataprotectionv1alpha1.BackupPolicyTemplate, _ dataprotectionv1alpha1.BackupPolicyTemplateList) { +var BackupPolicyTemplateSignature = func(_ appsv1alpha1.BackupPolicyTemplate, _ appsv1alpha1.BackupPolicyTemplateList) { } var BackupPolicySignature = func(_ dataprotectionv1alpha1.BackupPolicy, _ dataprotectionv1alpha1.BackupPolicyList) { } diff --git a/internal/testutil/apps/backup_factory.go b/internal/testutil/apps/backup_factory.go index 35e62bfd5..6409e6a20 100644 --- a/internal/testutil/apps/backup_factory.go +++ b/internal/testutil/apps/backup_factory.go @@ -17,11 +17,6 @@ limitations under the License. package apps import ( - "time" - - "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" ) @@ -47,13 +42,3 @@ func (factory *MockBackupFactory) SetBackupType(backupType dataprotectionv1alpha factory.get().Spec.BackupType = backupType return factory } - -func (factory *MockBackupFactory) SetTTL(duration string) *MockBackupFactory { - du, err := time.ParseDuration(duration) - gomega.Expect(err).Should(gomega.Succeed()) - - var d metav1.Duration - d.Duration = du - factory.get().Spec.TTL = &d - return factory -} diff --git a/internal/testutil/apps/backuppolicy_factory.go b/internal/testutil/apps/backuppolicy_factory.go index 691988d99..027dec473 100644 --- a/internal/testutil/apps/backuppolicy_factory.go +++ b/internal/testutil/apps/backuppolicy_factory.go @@ -17,9 +17,6 @@ limitations under the License. package apps import ( - "time" - - "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,97 +25,181 @@ import ( type MockBackupPolicyFactory struct { BaseFactory[dataprotectionv1alpha1.BackupPolicy, *dataprotectionv1alpha1.BackupPolicy, MockBackupPolicyFactory] + backupType dataprotectionv1alpha1.BackupType } func NewBackupPolicyFactory(namespace, name string) *MockBackupPolicyFactory { f := &MockBackupPolicyFactory{} f.init(namespace, name, - &dataprotectionv1alpha1.BackupPolicy{ - Spec: dataprotectionv1alpha1.BackupPolicySpec{ - Target: dataprotectionv1alpha1.TargetCluster{ - LabelsSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{}, - }, - }, - Hooks: &dataprotectionv1alpha1.BackupPolicyHook{}, - }, - }, f) + &dataprotectionv1alpha1.BackupPolicy{}, f) return f } -func (factory *MockBackupPolicyFactory) SetBackupPolicyTplName(backupPolicyTplName string) *MockBackupPolicyFactory { - factory.get().Spec.BackupPolicyTemplateName = backupPolicyTplName +func (factory *MockBackupPolicyFactory) setBasePolicyField(setField func(basePolicy *dataprotectionv1alpha1.BasePolicy)) { + var basePolicy *dataprotectionv1alpha1.BasePolicy + switch factory.backupType { + case dataprotectionv1alpha1.BackupTypeFull: + basePolicy = &factory.get().Spec.Full.BasePolicy + case dataprotectionv1alpha1.BackupTypeIncremental: + basePolicy = &factory.get().Spec.Incremental.BasePolicy + case dataprotectionv1alpha1.BackupTypeSnapshot: + basePolicy = &factory.get().Spec.Snapshot.BasePolicy + } + if basePolicy == nil { + // ignore + return + } + setField(basePolicy) +} + +func (factory *MockBackupPolicyFactory) setCommonPolicyField(setField func(commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy)) { + var commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy + switch factory.backupType { + case dataprotectionv1alpha1.BackupTypeFull: + commonPolicy = factory.get().Spec.Full + case dataprotectionv1alpha1.BackupTypeIncremental: + commonPolicy = factory.get().Spec.Incremental + } + if commonPolicy == nil { + // ignore + return + } + setField(commonPolicy) +} + +func (factory *MockBackupPolicyFactory) setScheduleField(setField func(schedulePolicy *dataprotectionv1alpha1.SchedulePolicy)) { + var schedulePolicy *dataprotectionv1alpha1.SchedulePolicy + switch factory.backupType { + case dataprotectionv1alpha1.BackupTypeFull, dataprotectionv1alpha1.BackupTypeSnapshot: + factory.get().Spec.Schedule.BaseBackup = &dataprotectionv1alpha1.BaseBackupSchedulePolicy{ + SchedulePolicy: dataprotectionv1alpha1.SchedulePolicy{}, + Type: dataprotectionv1alpha1.BaseBackupType(factory.backupType), + } + schedulePolicy = &factory.get().Spec.Schedule.BaseBackup.SchedulePolicy + case dataprotectionv1alpha1.BackupTypeIncremental: + schedulePolicy = &dataprotectionv1alpha1.SchedulePolicy{} + factory.get().Spec.Schedule.Incremental = schedulePolicy + } + if schedulePolicy == nil { + // ignore + return + } + setField(schedulePolicy) +} + +func (factory *MockBackupPolicyFactory) AddSnapshotPolicy() *MockBackupPolicyFactory { + factory.get().Spec.Snapshot = &dataprotectionv1alpha1.SnapshotPolicy{ + Hooks: &dataprotectionv1alpha1.BackupPolicyHook{}, + } + factory.backupType = dataprotectionv1alpha1.BackupTypeSnapshot + return factory +} + +func (factory *MockBackupPolicyFactory) AddFullPolicy() *MockBackupPolicyFactory { + factory.get().Spec.Full = &dataprotectionv1alpha1.CommonBackupPolicy{} + factory.backupType = dataprotectionv1alpha1.BackupTypeFull + return factory +} + +func (factory *MockBackupPolicyFactory) AddIncrementalPolicy() *MockBackupPolicyFactory { + factory.get().Spec.Incremental = &dataprotectionv1alpha1.CommonBackupPolicy{} + factory.backupType = dataprotectionv1alpha1.BackupTypeIncremental return factory } func (factory *MockBackupPolicyFactory) SetBackupToolName(backupToolName string) *MockBackupPolicyFactory { - factory.get().Spec.BackupToolName = backupToolName + factory.setCommonPolicyField(func(commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) { + commonPolicy.BackupToolName = backupToolName + }) return factory } -func (factory *MockBackupPolicyFactory) SetSchedule(schedule string) *MockBackupPolicyFactory { - factory.get().Spec.Schedule = schedule +func (factory *MockBackupPolicyFactory) SetSchedule(schedule string, enable bool) *MockBackupPolicyFactory { + factory.setScheduleField(func(schedulePolicy *dataprotectionv1alpha1.SchedulePolicy) { + schedulePolicy.Enable = enable + schedulePolicy.CronExpression = schedule + }) return factory } func (factory *MockBackupPolicyFactory) SetTTL(duration string) *MockBackupPolicyFactory { - du, err := time.ParseDuration(duration) - gomega.Expect(err).Should(gomega.Succeed()) - - var d metav1.Duration - d.Duration = du - factory.get().Spec.TTL = &d + factory.get().Spec.TTL = &duration return factory } func (factory *MockBackupPolicyFactory) SetBackupsHistoryLimit(backupsHistoryLimit int32) *MockBackupPolicyFactory { - factory.get().Spec.BackupsHistoryLimit = backupsHistoryLimit + factory.setBasePolicyField(func(basePolicy *dataprotectionv1alpha1.BasePolicy) { + basePolicy.BackupsHistoryLimit = backupsHistoryLimit + }) return factory } func (factory *MockBackupPolicyFactory) AddMatchLabels(keyAndValues ...string) *MockBackupPolicyFactory { + matchLabels := make(map[string]string) for k, v := range WithMap(keyAndValues...) { - factory.get().Spec.Target.LabelsSelector.MatchLabels[k] = v + matchLabels[k] = v } + factory.setBasePolicyField(func(basePolicy *dataprotectionv1alpha1.BasePolicy) { + basePolicy.Target.LabelsSelector = &metav1.LabelSelector{ + MatchLabels: matchLabels, + } + }) return factory } func (factory *MockBackupPolicyFactory) SetTargetSecretName(name string) *MockBackupPolicyFactory { - factory.get().Spec.Target.Secret = &dataprotectionv1alpha1.BackupPolicySecret{} - factory.get().Spec.Target.Secret.Name = name + factory.setBasePolicyField(func(basePolicy *dataprotectionv1alpha1.BasePolicy) { + basePolicy.Target.Secret = &dataprotectionv1alpha1.BackupPolicySecret{Name: name} + }) return factory } func (factory *MockBackupPolicyFactory) SetHookContainerName(containerName string) *MockBackupPolicyFactory { - factory.get().Spec.Hooks.ContainerName = containerName + snapshotPolicy := factory.get().Spec.Snapshot + if snapshotPolicy == nil { + return factory + } + snapshotPolicy.Hooks.ContainerName = containerName return factory } func (factory *MockBackupPolicyFactory) AddHookPreCommand(preCommand string) *MockBackupPolicyFactory { - preCommands := &factory.get().Spec.Hooks.PreCommands + snapshotPolicy := factory.get().Spec.Snapshot + if snapshotPolicy == nil { + return factory + } + preCommands := &snapshotPolicy.Hooks.PreCommands *preCommands = append(*preCommands, preCommand) return factory } func (factory *MockBackupPolicyFactory) AddHookPostCommand(postCommand string) *MockBackupPolicyFactory { - postCommands := &factory.get().Spec.Hooks.PostCommands + snapshotPolicy := factory.get().Spec.Snapshot + if snapshotPolicy == nil { + return factory + } + postCommands := &snapshotPolicy.Hooks.PostCommands *postCommands = append(*postCommands, postCommand) return factory } func (factory *MockBackupPolicyFactory) SetRemoteVolume(volume corev1.Volume) *MockBackupPolicyFactory { - factory.get().Spec.RemoteVolume = volume + factory.setCommonPolicyField(func(commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) { + commonPolicy.RemoteVolume = volume + }) return factory } func (factory *MockBackupPolicyFactory) SetRemoteVolumePVC(volumeName, pvcName string) *MockBackupPolicyFactory { - factory.get().Spec.RemoteVolume = corev1.Volume{ - Name: volumeName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcName, + factory.setCommonPolicyField(func(commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) { + commonPolicy.RemoteVolume = corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, }, - }, - } + } + }) return factory } diff --git a/internal/testutil/apps/backuppolicytemplate_factory.go b/internal/testutil/apps/backuppolicytemplate_factory.go index 94e9b8b7c..4fe0a19a2 100644 --- a/internal/testutil/apps/backuppolicytemplate_factory.go +++ b/internal/testutil/apps/backuppolicytemplate_factory.go @@ -17,70 +17,188 @@ limitations under the License. package apps import ( - "time" - - "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" ) type MockBackupPolicyTemplateFactory struct { - BaseFactory[dataprotectionv1alpha1.BackupPolicyTemplate, *dataprotectionv1alpha1.BackupPolicyTemplate, MockBackupPolicyTemplateFactory] + BaseFactory[appsv1alpha1.BackupPolicyTemplate, *appsv1alpha1.BackupPolicyTemplate, MockBackupPolicyTemplateFactory] + backupType dataprotectionv1alpha1.BackupType } func NewBackupPolicyTemplateFactory(name string) *MockBackupPolicyTemplateFactory { f := &MockBackupPolicyTemplateFactory{} f.init("", name, - &dataprotectionv1alpha1.BackupPolicyTemplate{ - Spec: dataprotectionv1alpha1.BackupPolicyTemplateSpec{ - Hooks: &dataprotectionv1alpha1.BackupPolicyHook{}, - }, - }, f) + &appsv1alpha1.BackupPolicyTemplate{}, + f) return f } -func (factory *MockBackupPolicyTemplateFactory) SetBackupToolName(backupToolName string) *MockBackupPolicyTemplateFactory { - factory.get().Spec.BackupToolName = backupToolName +func (factory *MockBackupPolicyTemplateFactory) SetClusterDefRef(clusterDefRef string) *MockBackupPolicyTemplateFactory { + factory.get().Spec.ClusterDefRef = clusterDefRef return factory } -func (factory *MockBackupPolicyTemplateFactory) SetSchedule(schedule string) *MockBackupPolicyTemplateFactory { - factory.get().Spec.Schedule = schedule +func (factory *MockBackupPolicyTemplateFactory) getLastBackupPolicy() *appsv1alpha1.BackupPolicy { + l := len(factory.get().Spec.BackupPolicies) + if l == 0 { + return nil + } + backupPolicies := factory.get().Spec.BackupPolicies + return &backupPolicies[l-1] +} + +func (factory *MockBackupPolicyTemplateFactory) AddBackupPolicy(componentDef string) *MockBackupPolicyTemplateFactory { + factory.get().Spec.BackupPolicies = append(factory.get().Spec.BackupPolicies, appsv1alpha1.BackupPolicy{ + ComponentDefRef: componentDef, + }) return factory } func (factory *MockBackupPolicyTemplateFactory) SetTTL(duration string) *MockBackupPolicyTemplateFactory { - du, err := time.ParseDuration(duration) - gomega.Expect(err).Should(gomega.Succeed()) + factory.getLastBackupPolicy().TTL = &duration + return factory +} + +func (factory *MockBackupPolicyTemplateFactory) setBasePolicyField(setField func(basePolicy *appsv1alpha1.BasePolicy)) { + backupPolicy := factory.getLastBackupPolicy() + var basePolicy *appsv1alpha1.BasePolicy + switch factory.backupType { + case dataprotectionv1alpha1.BackupTypeFull: + basePolicy = &backupPolicy.Full.BasePolicy + case dataprotectionv1alpha1.BackupTypeIncremental: + basePolicy = &backupPolicy.Incremental.BasePolicy + case dataprotectionv1alpha1.BackupTypeSnapshot: + basePolicy = &backupPolicy.Snapshot.BasePolicy + } + if basePolicy == nil { + // ignore + return + } + setField(basePolicy) +} + +func (factory *MockBackupPolicyTemplateFactory) setCommonPolicyField(setField func(commonPolicy *appsv1alpha1.CommonBackupPolicy)) { + backupPolicy := factory.getLastBackupPolicy() + var commonPolicy *appsv1alpha1.CommonBackupPolicy + switch factory.backupType { + case dataprotectionv1alpha1.BackupTypeFull: + commonPolicy = backupPolicy.Full + case dataprotectionv1alpha1.BackupTypeIncremental: + commonPolicy = backupPolicy.Incremental + } + if commonPolicy == nil { + // ignore + return + } + setField(commonPolicy) +} + +func (factory *MockBackupPolicyTemplateFactory) setScheduleField(setField func(schedulePolicy *appsv1alpha1.SchedulePolicy)) { + backupPolicy := factory.getLastBackupPolicy() + var schedulePolicy *appsv1alpha1.SchedulePolicy + switch factory.backupType { + case dataprotectionv1alpha1.BackupTypeFull, dataprotectionv1alpha1.BackupTypeSnapshot: + backupPolicy.Schedule.BaseBackup = &appsv1alpha1.BaseBackupSchedulePolicy{ + SchedulePolicy: appsv1alpha1.SchedulePolicy{}, + Type: appsv1alpha1.BaseBackupType(factory.backupType), + } + schedulePolicy = &backupPolicy.Schedule.BaseBackup.SchedulePolicy + case dataprotectionv1alpha1.BackupTypeIncremental: + schedulePolicy = &appsv1alpha1.SchedulePolicy{} + backupPolicy.Schedule.Incremental = schedulePolicy + } + if schedulePolicy == nil { + // ignore + return + } + setField(schedulePolicy) +} + +func (factory *MockBackupPolicyTemplateFactory) AddSnapshotPolicy() *MockBackupPolicyTemplateFactory { + backupPolicy := factory.getLastBackupPolicy() + backupPolicy.Snapshot = &appsv1alpha1.SnapshotPolicy{ + Hooks: &appsv1alpha1.BackupPolicyHook{}, + } + factory.backupType = dataprotectionv1alpha1.BackupTypeSnapshot + return factory +} + +func (factory *MockBackupPolicyTemplateFactory) AddFullPolicy() *MockBackupPolicyTemplateFactory { + backupPolicy := factory.getLastBackupPolicy() + backupPolicy.Full = &appsv1alpha1.CommonBackupPolicy{} + factory.backupType = dataprotectionv1alpha1.BackupTypeFull + return factory +} - var d metav1.Duration - d.Duration = du - factory.get().Spec.TTL = &d +func (factory *MockBackupPolicyTemplateFactory) AddIncrementalPolicy() *MockBackupPolicyTemplateFactory { + backupPolicy := factory.getLastBackupPolicy() + backupPolicy.Incremental = &appsv1alpha1.CommonBackupPolicy{} + factory.backupType = dataprotectionv1alpha1.BackupTypeIncremental return factory } func (factory *MockBackupPolicyTemplateFactory) SetHookContainerName(containerName string) *MockBackupPolicyTemplateFactory { - factory.get().Spec.Hooks.ContainerName = containerName + backupPolicy := factory.getLastBackupPolicy() + if backupPolicy.Snapshot == nil { + return factory + } + backupPolicy.Snapshot.Hooks.ContainerName = containerName return factory } func (factory *MockBackupPolicyTemplateFactory) AddHookPreCommand(preCommand string) *MockBackupPolicyTemplateFactory { - preCommands := &factory.get().Spec.Hooks.PreCommands + backupPolicy := factory.getLastBackupPolicy() + if backupPolicy.Snapshot == nil { + return factory + } + preCommands := &backupPolicy.Snapshot.Hooks.PreCommands *preCommands = append(*preCommands, preCommand) return factory } func (factory *MockBackupPolicyTemplateFactory) AddHookPostCommand(postCommand string) *MockBackupPolicyTemplateFactory { - postCommands := &factory.get().Spec.Hooks.PostCommands + backupPolicy := factory.getLastBackupPolicy() + if backupPolicy.Snapshot == nil { + return factory + } + postCommands := &backupPolicy.Snapshot.Hooks.PostCommands *postCommands = append(*postCommands, postCommand) return factory } -func (factory *MockBackupPolicyTemplateFactory) SetCredentialKeyword(userKeyword, passwdKeyword string) *MockBackupPolicyTemplateFactory { - factory.get().Spec.CredentialKeyword = &dataprotectionv1alpha1.BackupPolicyCredentialKeyword{ - UserKeyword: userKeyword, - PasswordKeyword: passwdKeyword, - } +func (factory *MockBackupPolicyTemplateFactory) SetSchedule(schedule string, enable bool) *MockBackupPolicyTemplateFactory { + factory.setScheduleField(func(schedulePolicy *appsv1alpha1.SchedulePolicy) { + schedulePolicy.Enable = enable + schedulePolicy.CronExpression = schedule + }) + return factory +} + +func (factory *MockBackupPolicyTemplateFactory) SetBackupsHistoryLimit(backupsHistoryLimit int32) *MockBackupPolicyTemplateFactory { + factory.setBasePolicyField(func(basePolicy *appsv1alpha1.BasePolicy) { + basePolicy.BackupsHistoryLimit = backupsHistoryLimit + }) + return factory +} + +func (factory *MockBackupPolicyTemplateFactory) SetBackupToolName(backupToolName string) *MockBackupPolicyTemplateFactory { + factory.setCommonPolicyField(func(commonPolicy *appsv1alpha1.CommonBackupPolicy) { + commonPolicy.BackupToolName = backupToolName + }) + return factory +} + +func (factory *MockBackupPolicyTemplateFactory) SetTargetRole(role string) *MockBackupPolicyTemplateFactory { + factory.setBasePolicyField(func(basePolicy *appsv1alpha1.BasePolicy) { + basePolicy.Target.Role = role + }) + return factory +} + +func (factory *MockBackupPolicyTemplateFactory) SetTargetAccount(account string) *MockBackupPolicyTemplateFactory { + factory.setBasePolicyField(func(basePolicy *appsv1alpha1.BasePolicy) { + basePolicy.Target.Account = account + }) return factory } diff --git a/internal/testutil/apps/restorejob_factory.go b/internal/testutil/apps/restorejob_factory.go index fba1d561c..a8ff3d9f2 100644 --- a/internal/testutil/apps/restorejob_factory.go +++ b/internal/testutil/apps/restorejob_factory.go @@ -55,8 +55,7 @@ func (factory *MockRestoreJobFactory) AddTargetMatchLabels(keyAndValues ...strin } func (factory *MockRestoreJobFactory) SetTargetSecretName(name string) *MockRestoreJobFactory { - factory.get().Spec.Target.Secret = &dataprotectionv1alpha1.BackupPolicySecret{} - factory.get().Spec.Target.Secret.Name = name + factory.get().Spec.Target.Secret = &dataprotectionv1alpha1.BackupPolicySecret{Name: name} return factory } diff --git a/test/integration/backup_mysql_test.go b/test/integration/backup_mysql_test.go index 6e18d32f3..24a407991 100644 --- a/test/integration/backup_mysql_test.go +++ b/test/integration/backup_mysql_test.go @@ -44,8 +44,6 @@ var _ = Describe("MySQL data protection function", func() { const backupPolicyName = "test-backup-policy" const backupRemoteVolumeName = "backup-remote-volume" const backupRemotePVCName = "backup-remote-pvc" - const defaultSchedule = "0 3 * * *" - const defaultTTL = "168h0m0s" const backupName = "test-backup-job" // Cleanups @@ -66,7 +64,6 @@ var _ = Describe("MySQL data protection function", func() { testapps.ClearResources(&testCtx, intctrlutil.OpsRequestSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.ConfigMapSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupToolSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.RestoreJobSignature, inNS, ml) @@ -127,17 +124,11 @@ var _ = Describe("MySQL data protection function", func() { backupTool := testapps.CreateCustomizedObj(&testCtx, "backup/backuptool.yaml", &dpv1alpha1.BackupTool{}, testapps.RandomizedObjName()) - By("By creating a backupPolicyTemplate from backupTool: " + backupTool.Name) - _ = testapps.NewBackupPolicyTemplateFactory(backupPolicyTemplateName). - SetBackupToolName(backupTool.Name). - SetSchedule(defaultSchedule). - SetTTL(defaultTTL). - Create(&testCtx).GetObject() - By("By creating a backupPolicy from backupPolicyTemplate: " + backupPolicyTemplateName) backupPolicyObj := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). WithRandomName(). - SetBackupPolicyTplName(backupPolicyTemplateName). + AddFullPolicy(). + SetBackupToolName(backupTool.Name). AddMatchLabels(constant.AppInstanceLabelKey, clusterKey.Name). SetTargetSecretName(component.GenerateConnCredential(clusterKey.Name)). SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). @@ -153,13 +144,12 @@ var _ = Describe("MySQL data protection function", func() { By("By check backupPolicy available") Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, backupPolicy *dpv1alpha1.BackupPolicy) { - g.Expect(backupPolicy.Status.Phase).To(Equal(dpv1alpha1.ConfigAvailable)) + g.Expect(backupPolicy.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) })).Should(Succeed()) By("By creating a backup from backupPolicy: " + backupPolicyKey.Name) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). WithRandomName(). - SetTTL(defaultTTL). SetBackupPolicyName(backupPolicyKey.Name). SetBackupType(dpv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() From 7a9dcce05ca6ff9426da8fcbbf6f812089df39f7 Mon Sep 17 00:00:00 2001 From: xingran Date: Wed, 12 Apr 2023 18:22:12 +0800 Subject: [PATCH 004/439] chore: support redis sentinel role changed event after failover (#2517) --- .../internal/binding/postgres/postgres.go | 2 - cmd/probe/internal/binding/redis/redis.go | 6 +++ .../internal/binding/redis/redis_test.go | 4 +- cmd/probe/internal/binding/types.go | 5 +++ .../replicationset/replication_set.go | 4 ++ .../replication_set_switch_utils.go | 29 +++++++++++++++ .../replicationset/replication_set_utils.go | 37 ++++++++++++++++--- .../components/util/stateful_set_utils.go | 19 ++++++++++ .../util/stateful_set_utils_test.go | 5 +++ deploy/redis-cluster/values.yaml | 2 +- deploy/redis/templates/clusterdefinition.yaml | 5 +++ 11 files changed, 108 insertions(+), 10 deletions(-) diff --git a/cmd/probe/internal/binding/postgres/postgres.go b/cmd/probe/internal/binding/postgres/postgres.go index 14b55a7ff..afc94e61d 100644 --- a/cmd/probe/internal/binding/postgres/postgres.go +++ b/cmd/probe/internal/binding/postgres/postgres.go @@ -39,8 +39,6 @@ import ( const ( connectionURLKey = "url" commandSQLKey = "sql" - PRIMARY = "primary" - SECONDARY = "secondary" listUserTpl = ` SELECT usename AS userName, valuntil Date: Wed, 12 Apr 2023 18:51:22 +0800 Subject: [PATCH 005/439] feat: support yaml config for mongo and tidyup configuration (#2444) Co-authored-by: sophon-zt --- .../apps/configuration/config_annotation.go | 3 +- .../apps/configuration/reconfigure_policy.go | 9 +- .../configuration/rolling_upgrade_policy.go | 3 +- docs/user_docs/cli/cli.md | 2 +- docs/user_docs/cli/kbcli_cluster.md | 2 +- internal/cli/cmd/cluster/config_util.go | 3 +- internal/cli/cmd/cluster/config_wrapper.go | 3 +- internal/configuration/config.go | 84 ++-------- .../config_manager/reload_util.go | 17 +-- internal/configuration/config_patch.go | 85 +++++++++++ .../{util.go => config_patch_option.go} | 0 ...il_test.go => config_patch_option_test.go} | 0 internal/configuration/config_patch_test.go | 121 +++++++++++++++ internal/configuration/config_patch_util.go | 10 +- internal/configuration/config_test.go | 64 -------- .../configuration/config_validate_test.go | 8 + .../configuration/container/container_kill.go | 3 +- internal/configuration/cue_visitor.go | 53 +++++-- internal/configuration/cue_visitor_test.go | 50 +++--- internal/configuration/reconfigure_util.go | 40 ++++- .../configuration/reconfigure_util_test.go | 14 +- internal/configuration/util/file_util.go | 34 +++++ internal/configuration/{ => util}/hash.go | 7 +- .../configuration/{ => util}/hash_test.go | 2 +- internal/configuration/{ => util}/jsonpath.go | 14 +- internal/configuration/{ => util}/math.go | 2 +- .../configuration/{ => util}/math_test.go | 2 +- internal/configuration/{ => util}/set.go | 2 +- internal/configuration/{ => util}/set_test.go | 2 +- .../configuration/{ => util}/unstructured.go | 36 +++-- .../{ => util}/unstructured_test.go | 9 +- internal/controller/plan/prepare.go | 3 +- internal/unstructured/viper_wrap.go | 2 +- internal/unstructured/viper_wrap_test.go | 29 ---- internal/unstructured/yaml_config.go | 143 ++++++++++++++++++ internal/unstructured/yaml_config_test.go | 54 +++++++ test/testdata/cue_testdata/mongod.conf | 21 +++ test/testdata/cue_testdata/mongod.cue | 37 +++++ 38 files changed, 693 insertions(+), 280 deletions(-) create mode 100644 internal/configuration/config_patch.go rename internal/configuration/{util.go => config_patch_option.go} (100%) rename internal/configuration/{util_test.go => config_patch_option_test.go} (100%) create mode 100644 internal/configuration/config_patch_test.go create mode 100644 internal/configuration/util/file_util.go rename internal/configuration/{ => util}/hash.go (84%) rename internal/configuration/{ => util}/hash_test.go (98%) rename internal/configuration/{ => util}/jsonpath.go (77%) rename internal/configuration/{ => util}/math.go (97%) rename internal/configuration/{ => util}/math_test.go (98%) rename internal/configuration/{ => util}/set.go (98%) rename internal/configuration/{ => util}/set_test.go (99%) rename internal/configuration/{ => util}/unstructured.go (82%) rename internal/configuration/{ => util}/unstructured_test.go (89%) create mode 100644 internal/unstructured/yaml_config.go create mode 100644 internal/unstructured/yaml_config_test.go create mode 100644 test/testdata/cue_testdata/mongod.conf create mode 100644 test/testdata/cue_testdata/mongod.cue diff --git a/controllers/apps/configuration/config_annotation.go b/controllers/apps/configuration/config_annotation.go index e56636c58..0f4a1322d 100644 --- a/controllers/apps/configuration/config_annotation.go +++ b/controllers/apps/configuration/config_annotation.go @@ -25,6 +25,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/configuration/util" "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -86,7 +87,7 @@ func updateAppliedConfigs(cli client.Client, ctx intctrlutil.RequestCtx, config } config.ObjectMeta.Annotations[constant.LastAppliedConfigAnnotation] = string(configData) - hash, err := cfgcore.ComputeHash(config.Data) + hash, err := util.ComputeHash(config.Data) if err != nil { return false, err } diff --git a/controllers/apps/configuration/reconfigure_policy.go b/controllers/apps/configuration/reconfigure_policy.go index a0d42bff7..096f7aa87 100644 --- a/controllers/apps/configuration/reconfigure_policy.go +++ b/controllers/apps/configuration/reconfigure_policy.go @@ -29,6 +29,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" cfgproto "github.com/apecloud/kubeblocks/internal/configuration/proto" + "github.com/apecloud/kubeblocks/internal/configuration/util" "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -134,7 +135,7 @@ func (param *reconfigureParams) getConfigKey() string { } func (param *reconfigureParams) getTargetVersionHash() string { - hash, err := cfgcore.ComputeHash(param.ConfigMap.Data) + hash, err := util.ComputeHash(param.ConfigMap.Data) if err != nil { param.Ctx.Log.Error(err, "failed to cal configuration version!") return "" @@ -163,9 +164,9 @@ func (param *reconfigureParams) maxRollingReplicas() int32 { if isPercent { r = int32(math.Floor(float64(v) * float64(replicas) / 100)) } else { - r = int32(cfgcore.Min(v, param.getTargetReplicas())) + r = int32(util.Min(v, param.getTargetReplicas())) } - return cfgcore.Max(r, defaultRolling) + return util.Max(r, defaultRolling) } func (param *reconfigureParams) getTargetReplicas() int { @@ -174,7 +175,7 @@ func (param *reconfigureParams) getTargetReplicas() int { func (param *reconfigureParams) podMinReadySeconds() int32 { minReadySeconds := param.ComponentUnits[0].Spec.MinReadySeconds - return cfgcore.Max(minReadySeconds, viper.GetInt32(constant.PodMinReadySecondsEnv)) + return util.Max(minReadySeconds, viper.GetInt32(constant.PodMinReadySecondsEnv)) } func RegisterPolicy(policy appsv1alpha1.UpgradePolicy, action reconfigurePolicy) { diff --git a/controllers/apps/configuration/rolling_upgrade_policy.go b/controllers/apps/configuration/rolling_upgrade_policy.go index 71341d026..075c919d6 100644 --- a/controllers/apps/configuration/rolling_upgrade_policy.go +++ b/controllers/apps/configuration/rolling_upgrade_policy.go @@ -26,6 +26,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/configuration/util" "github.com/apecloud/kubeblocks/internal/constant" podutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -163,7 +164,7 @@ func markDynamicCursor(pods []corev1.Pod, podsStats *componentPodStats, configKe podsStats.updated[pod.Name] = pod } - podWindows.begin = cfgcore.Max[int](podWindows.end-int(rollingReplicas), 0) + podWindows.begin = util.Max[int](podWindows.end-int(rollingReplicas), 0) for i := podWindows.begin; i < podWindows.end; i++ { pod := &pods[i] if podutil.IsMatchConfigVersion(pod, configKey, currentVersion) { diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index e1d393c68..ee80129a3 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -64,8 +64,8 @@ Cluster command. * [kbcli cluster describe-config](kbcli_cluster_describe-config.md) - Show details of a specific reconfiguring. * [kbcli cluster describe-ops](kbcli_cluster_describe-ops.md) - Show details of a specific OpsRequest. * [kbcli cluster diff-config](kbcli_cluster_diff-config.md) - Show the difference in parameters between the two submitted OpsRequest. -* [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. * [kbcli cluster edit-backup-policy](kbcli_cluster_edit-backup-policy.md) - Edit backup policy +* [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. * [kbcli cluster explain-config](kbcli_cluster_explain-config.md) - List the constraint for supported configuration params. * [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index 9fffbb637..dc0df42c4 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -52,8 +52,8 @@ Cluster command. * [kbcli cluster describe-config](kbcli_cluster_describe-config.md) - Show details of a specific reconfiguring. * [kbcli cluster describe-ops](kbcli_cluster_describe-ops.md) - Show details of a specific OpsRequest. * [kbcli cluster diff-config](kbcli_cluster_diff-config.md) - Show the difference in parameters between the two submitted OpsRequest. -* [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. * [kbcli cluster edit-backup-policy](kbcli_cluster_edit-backup-policy.md) - Edit backup policy +* [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. * [kbcli cluster explain-config](kbcli_cluster_explain-config.md) - List the constraint for supported configuration params. * [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account diff --git a/internal/cli/cmd/cluster/config_util.go b/internal/cli/cmd/cluster/config_util.go index 8d03756f7..f76274a05 100644 --- a/internal/cli/cmd/cluster/config_util.go +++ b/internal/cli/cmd/cluster/config_util.go @@ -33,6 +33,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + cfgutil "github.com/apecloud/kubeblocks/internal/configuration/util" ) type configEditContext struct { @@ -67,7 +68,7 @@ func (c *configEditContext) prepare() error { val, ok := cmObj.Data[c.configKey] if !ok { - return makeNotFoundConfigFileErr(c.configKey, c.configSpecName, cfgcore.ToSet(cmObj.Data).AsSlice()) + return makeNotFoundConfigFileErr(c.configKey, c.configSpecName, cfgutil.ToSet(cmObj.Data).AsSlice()) } c.original = val diff --git a/internal/cli/cmd/cluster/config_wrapper.go b/internal/cli/cmd/cluster/config_wrapper.go index 9543683c8..5b7292c51 100644 --- a/internal/cli/cmd/cluster/config_wrapper.go +++ b/internal/cli/cmd/cluster/config_wrapper.go @@ -26,6 +26,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + cfgutil "github.com/apecloud/kubeblocks/internal/configuration/util" ) type configWrapper struct { @@ -92,7 +93,7 @@ func (w *configWrapper) ValidateRequiredParam() error { // step3: validate fileKey exist. if _, ok := cmObj.Data[w.configKey]; !ok { - return makeNotFoundConfigFileErr(w.configKey, w.configSpecName, cfgcore.ToSet(cmObj.Data).AsSlice()) + return makeNotFoundConfigFileErr(w.configKey, w.configSpecName, cfgutil.ToSet(cmObj.Data).AsSlice()) } // TODO support all config file update. diff --git a/internal/configuration/config.go b/internal/configuration/config.go index cd50ee978..f1ec520c8 100644 --- a/internal/configuration/config.go +++ b/internal/configuration/config.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/configuration/util" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" "github.com/apecloud/kubeblocks/internal/unstructured" ) @@ -238,51 +239,6 @@ type ConfigPatchInfo struct { LastVersion *cfgWrapper } -func (c *cfgWrapper) Diff(target *cfgWrapper) (*ConfigPatchInfo, error) { - fromOMap := ToSet(c.indexer) - fromNMap := ToSet(target.indexer) - - addSet := Difference(fromNMap, fromOMap) - deleteSet := Difference(fromOMap, fromNMap) - updateSet := Difference(fromOMap, deleteSet) - - reconfigureInfo := &ConfigPatchInfo{ - IsModify: false, - AddConfig: make(map[string]interface{}, addSet.Length()), - DeleteConfig: make(map[string]interface{}, deleteSet.Length()), - UpdateConfig: make(map[string][]byte, updateSet.Length()), - - Target: target, - LastVersion: c, - } - - for elem := range addSet.Iter() { - reconfigureInfo.AddConfig[elem] = target.indexer[elem].GetAllParameters() - reconfigureInfo.IsModify = true - } - - for elem := range deleteSet.Iter() { - reconfigureInfo.DeleteConfig[elem] = c.indexer[elem].GetAllParameters() - reconfigureInfo.IsModify = true - } - - for elem := range updateSet.Iter() { - old := c.indexer[elem] - new := target.indexer[elem] - - patch, err := jsonPatch(old.GetAllParameters(), new.GetAllParameters()) - if err != nil { - return nil, err - } - if len(patch) > len(emptyJSON) { - reconfigureInfo.UpdateConfig[elem] = patch - reconfigureInfo.IsModify = true - } - } - - return reconfigureInfo, nil -} - func NewCfgOptions(filename string, options ...Option) CfgOpOption { context := CfgOpOption{ FileName: filename, @@ -305,6 +261,13 @@ func WithFormatterConfig(formatConfig *appsv1alpha1.FormatterConfig) Option { } } +func NestedPrefixField(formatConfig *appsv1alpha1.FormatterConfig) string { + if formatConfig != nil && formatConfig.Format == appsv1alpha1.Ini && formatConfig.IniConfig != nil { + return formatConfig.IniConfig.SectionName + } + return "" +} + func (c *cfgWrapper) Query(jsonpath string, option CfgOpOption) ([]byte, error) { if option.AllSearch && c.fileCount > 1 { return c.queryAllCfg(jsonpath, option) @@ -323,7 +286,7 @@ func (c *cfgWrapper) Query(jsonpath string, option CfgOpOption) ([]byte, error) } } - return retrievalWithJSONPath(cfg.GetAllParameters(), jsonpath) + return util.RetrievalWithJSONPath(cfg.GetAllParameters(), jsonpath) } func (c *cfgWrapper) queryAllCfg(jsonpath string, option CfgOpOption) ([]byte, error) { @@ -332,7 +295,7 @@ func (c *cfgWrapper) queryAllCfg(jsonpath string, option CfgOpOption) ([]byte, e for filename, v := range c.indexer { tops[filename] = v.GetAllParameters() } - return retrievalWithJSONPath(tops, jsonpath) + return util.RetrievalWithJSONPath(tops, jsonpath) } func (c cfgWrapper) getConfigObject(option CfgOpOption) unstructured.ConfigObject { @@ -363,37 +326,12 @@ func FromCMKeysSelector(keys []string) *set.LinkedHashSetString { return cmKeySet } -func CreateMergePatch(oldVersion, newVersion interface{}, option CfgOption) (*ConfigPatchInfo, error) { - - ok, err := compareWithConfig(oldVersion, newVersion, option) - if err != nil { - return nil, err - } else if ok { - return &ConfigPatchInfo{IsModify: false}, err - } - - old, err := NewConfigLoader(withOption(option, oldVersion)) - if err != nil { - return nil, WrapError(err, "failed to create config: [%s]", oldVersion) - } - - new, err := NewConfigLoader(withOption(option, newVersion)) - if err != nil { - return nil, WrapError(err, "failed to create config: [%s]", oldVersion) - } - - return old.Diff(new.cfgWrapper) -} - func GenerateVisualizedParamsList(configPatch *ConfigPatchInfo, formatConfig *appsv1alpha1.FormatterConfig, sets *set.LinkedHashSetString) []VisualizedParam { if !configPatch.IsModify { return nil } - var trimPrefix = "" - if formatConfig != nil && formatConfig.Format == appsv1alpha1.Ini && formatConfig.IniConfig != nil { - trimPrefix = formatConfig.IniConfig.SectionName - } + var trimPrefix = NestedPrefixField(formatConfig) r := make([]VisualizedParam, 0) r = append(r, generateUpdateParam(configPatch.UpdateConfig, trimPrefix, sets)...) diff --git a/internal/configuration/config_manager/reload_util.go b/internal/configuration/config_manager/reload_util.go index 2d54ba547..d4957a16e 100644 --- a/internal/configuration/config_manager/reload_util.go +++ b/internal/configuration/config_manager/reload_util.go @@ -30,6 +30,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" cfgutil "github.com/apecloud/kubeblocks/internal/configuration" cfgcontainer "github.com/apecloud/kubeblocks/internal/configuration/container" + "github.com/apecloud/kubeblocks/internal/configuration/util" "github.com/apecloud/kubeblocks/internal/gotemplate" ) @@ -126,11 +127,11 @@ func createUpdatedParamsPatch(newVersion []string, oldVersion []string, formatCf } logger.V(1).Info(fmt.Sprintf("new version files: %v, old version files: %v", newVersion, oldVersion)) - oldData, err := fromConfigFiles(oldVersion) + oldData, err := util.FromConfigFiles(oldVersion) if err != nil { return nil, err } - newData, err := fromConfigFiles(newVersion) + newData, err := util.FromConfigFiles(newVersion) if err != nil { return nil, err } @@ -153,18 +154,6 @@ func createUpdatedParamsPatch(newVersion []string, oldVersion []string, formatCf return r, nil } -func fromConfigFiles(files []string) (map[string]string, error) { - m := make(map[string]string) - for _, file := range files { - b, err := os.ReadFile(file) - if err != nil { - return nil, err - } - m[filepath.Base(file)] = string(b) - } - return m, nil -} - func resolveLink(path string) (string, error) { logger.V(1).Info(fmt.Sprintf("resolveLink : %s", path)) diff --git a/internal/configuration/config_patch.go b/internal/configuration/config_patch.go new file mode 100644 index 000000000..1759a71e6 --- /dev/null +++ b/internal/configuration/config_patch.go @@ -0,0 +1,85 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configuration + +import "github.com/apecloud/kubeblocks/internal/configuration/util" + +func CreateMergePatch(oldVersion, newVersion interface{}, option CfgOption) (*ConfigPatchInfo, error) { + + ok, err := compareWithConfig(oldVersion, newVersion, option) + if err != nil { + return nil, err + } else if ok { + return &ConfigPatchInfo{IsModify: false}, err + } + + old, err := NewConfigLoader(withOption(option, oldVersion)) + if err != nil { + return nil, WrapError(err, "failed to create config: [%s]", oldVersion) + } + + new, err := NewConfigLoader(withOption(option, newVersion)) + if err != nil { + return nil, WrapError(err, "failed to create config: [%s]", oldVersion) + } + return difference(old.cfgWrapper, new.cfgWrapper) +} + +func difference(base *cfgWrapper, target *cfgWrapper) (*ConfigPatchInfo, error) { + fromOMap := util.ToSet(base.indexer) + fromNMap := util.ToSet(target.indexer) + + addSet := util.Difference(fromNMap, fromOMap) + deleteSet := util.Difference(fromOMap, fromNMap) + updateSet := util.Difference(fromOMap, deleteSet) + + reconfigureInfo := &ConfigPatchInfo{ + IsModify: false, + AddConfig: make(map[string]interface{}, addSet.Length()), + DeleteConfig: make(map[string]interface{}, deleteSet.Length()), + UpdateConfig: make(map[string][]byte, updateSet.Length()), + + Target: target, + LastVersion: base, + } + + for elem := range addSet.Iter() { + reconfigureInfo.AddConfig[elem] = target.indexer[elem].GetAllParameters() + reconfigureInfo.IsModify = true + } + + for elem := range deleteSet.Iter() { + reconfigureInfo.DeleteConfig[elem] = base.indexer[elem].GetAllParameters() + reconfigureInfo.IsModify = true + } + + for elem := range updateSet.Iter() { + old := base.indexer[elem] + new := target.indexer[elem] + + patch, err := util.JSONPatch(old.GetAllParameters(), new.GetAllParameters()) + if err != nil { + return nil, err + } + if len(patch) > len(emptyJSON) { + reconfigureInfo.UpdateConfig[elem] = patch + reconfigureInfo.IsModify = true + } + } + + return reconfigureInfo, nil +} diff --git a/internal/configuration/util.go b/internal/configuration/config_patch_option.go similarity index 100% rename from internal/configuration/util.go rename to internal/configuration/config_patch_option.go diff --git a/internal/configuration/util_test.go b/internal/configuration/config_patch_option_test.go similarity index 100% rename from internal/configuration/util_test.go rename to internal/configuration/config_patch_option_test.go diff --git a/internal/configuration/config_patch_test.go b/internal/configuration/config_patch_test.go new file mode 100644 index 000000000..e2b647c52 --- /dev/null +++ b/internal/configuration/config_patch_test.go @@ -0,0 +1,121 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configuration + +import ( + "context" + "testing" + + "github.com/ghodss/yaml" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/log" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" +) + +func TestConfigPatch(t *testing.T) { + + cfg, err := NewConfigLoader(CfgOption{ + Type: CfgRawType, + Log: log.FromContext(context.Background()), + CfgType: appsv1alpha1.Ini, + RawData: []byte(iniConfig), + }) + + if err != nil { + t.Fatalf("new config loader failed [%v]", err) + } + + ctx := NewCfgOptions("", + func(ctx *CfgOpOption) { + // filter mysqld + ctx.IniContext = &IniContext{ + SectionName: "mysqld", + } + }) + + // ctx := NewCfgOptions("$..slow_query_log_file", "") + + result, err := cfg.Query("$..slow_query_log_file", NewCfgOptions("")) + require.Nil(t, err) + require.NotNil(t, result) + require.Equal(t, "[\"/data/mysql/mysqld-slow.log\"]", string(result)) + + require.Nil(t, + cfg.MergeFrom(map[string]interface{}{ + "slow_query_log": 1, + "server-id": 2, + "socket": "xxxxxxxxxxxxxxx", + }, ctx)) + + content, err := cfg.ToCfgContent() + require.NotNil(t, content) + require.Nil(t, err) + + newContent, exist := content[cfg.name] + require.True(t, exist) + patch, err := CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option) + require.Nil(t, err) + log.Log.Info("patch : %v", patch) + require.True(t, patch.IsModify) + require.Equal(t, string(patch.UpdateConfig["raw"]), `{"mysqld":{"server-id":"2","socket":"xxxxxxxxxxxxxxx"}}`) + + { + require.Nil(t, + cfg.MergeFrom(map[string]interface{}{ + "server-id": 1, + "socket": "/data/mysql/tmp/mysqld.sock", + }, ctx)) + content, err := cfg.ToCfgContent() + require.Nil(t, err) + newContent := content[cfg.name] + // CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option) + patch, err := CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option) + require.Nil(t, err) + log.Log.Info("patch : %v", patch) + require.False(t, patch.IsModify) + } +} + +func TestYamlConfigPatch(t *testing.T) { + yamlContext := ` +net: + port: 2000 + bindIp: + type: "string" + trim: "whitespace" + tls: + mode: requireTLS + certificateKeyFilePassword: + type: "string" + digest: b08519162ba332985ac18204851949611ef73835ec99067b85723e10113f5c26 + digest_key: 6d795365637265744b65795374756666 +` + + patchOption := CfgOption{ + Type: CfgTplType, + CfgType: appsv1alpha1.YAML, + } + patch, err := CreateMergePatch(&ConfigResource{ConfigData: map[string]string{"test": ""}}, &ConfigResource{ConfigData: map[string]string{"test": yamlContext}}, patchOption) + require.Nil(t, err) + + yb, err := yaml.YAMLToJSON([]byte(yamlContext)) + require.Nil(t, err) + + require.Nil(t, err) + require.Equal(t, yb, patch.UpdateConfig["test"]) +} diff --git a/internal/configuration/config_patch_util.go b/internal/configuration/config_patch_util.go index 9700bb62c..7f5accb3d 100644 --- a/internal/configuration/config_patch_util.go +++ b/internal/configuration/config_patch_util.go @@ -22,9 +22,9 @@ import ( "github.com/StudioSol/set" "sigs.k8s.io/controller-runtime/pkg/log" - "github.com/apecloud/kubeblocks/internal/unstructured" - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/configuration/util" + "github.com/apecloud/kubeblocks/internal/unstructured" ) // CreateConfigPatch creates a patch for configuration files with difference version. @@ -49,10 +49,10 @@ func CreateConfigPatch(oldVersion, newVersion map[string]string, format appsv1al func checkExcludeConfigDifference(oldVersion map[string]string, newVersion map[string]string, keys []string) bool { keySet := set.NewLinkedHashSetString(keys...) - leftOldKey := Difference(ToSet(oldVersion), keySet) - leftNewKey := Difference(ToSet(newVersion), keySet) + leftOldKey := util.Difference(util.ToSet(oldVersion), keySet) + leftNewKey := util.Difference(util.ToSet(newVersion), keySet) - if !EqSet(leftOldKey, leftNewKey) { + if !util.EqSet(leftOldKey, leftNewKey) { return true } diff --git a/internal/configuration/config_test.go b/internal/configuration/config_test.go index b9466c759..61b76c403 100644 --- a/internal/configuration/config_test.go +++ b/internal/configuration/config_test.go @@ -64,70 +64,6 @@ socket=/data/mysql/tmp/mysqld.sock host=localhost ` -func TestRawConfig(t *testing.T) { - - cfg, err := NewConfigLoader(CfgOption{ - Type: CfgRawType, - Log: log.FromContext(context.Background()), - CfgType: appsv1alpha1.Ini, - RawData: []byte(iniConfig), - }) - - if err != nil { - t.Fatalf("new config loader failed [%v]", err) - } - - ctx := NewCfgOptions("", - func(ctx *CfgOpOption) { - // filter mysqld - ctx.IniContext = &IniContext{ - SectionName: "mysqld", - } - }) - - // ctx := NewCfgOptions("$..slow_query_log_file", "") - - result, err := cfg.Query("$..slow_query_log_file", NewCfgOptions("")) - require.Nil(t, err) - require.NotNil(t, result) - require.Equal(t, "[\"/data/mysql/mysqld-slow.log\"]", string(result)) - - require.Nil(t, - cfg.MergeFrom(map[string]interface{}{ - "slow_query_log": 1, - "server-id": 2, - "socket": "xxxxxxxxxxxxxxx", - }, ctx)) - - content, err := cfg.ToCfgContent() - require.NotNil(t, content) - require.Nil(t, err) - - newContent, exist := content[cfg.name] - require.True(t, exist) - patch, err := CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option) - require.Nil(t, err) - log.Log.Info("patch : %v", patch) - require.True(t, patch.IsModify) - require.Equal(t, string(patch.UpdateConfig["raw"]), `{"mysqld":{"server-id":"2","socket":"xxxxxxxxxxxxxxx"}}`) - - { - require.Nil(t, - cfg.MergeFrom(map[string]interface{}{ - "server-id": 1, - "socket": "/data/mysql/tmp/mysqld.sock", - }, ctx)) - content, err := cfg.ToCfgContent() - require.Nil(t, err) - newContent := content[cfg.name] - // CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option) - patch, err := CreateMergePatch([]byte(iniConfig), []byte(newContent), cfg.Option) - require.Nil(t, err) - log.Log.Info("patch : %v", patch) - require.False(t, patch.IsModify) - } -} - func TestConfigMapConfig(t *testing.T) { cfg, err := NewConfigLoader(CfgOption{ Type: CfgCmType, diff --git a/internal/configuration/config_validate_test.go b/internal/configuration/config_validate_test.go index eb0960799..1b1bda052 100644 --- a/internal/configuration/config_validate_test.go +++ b/internal/configuration/config_validate_test.go @@ -57,6 +57,14 @@ func TestSchemaValidatorWithCue(t *testing.T) { args args err error }{{ + name: "mongod_test", + args: args{ + cueFile: "cue_testdata/mongod.cue", + configFile: "cue_testdata/mongod.conf", + format: appsv1alpha1.YAML, + }, + err: nil, + }, { name: "test_wesql", args: args{ cueFile: "cue_testdata/wesql.cue", diff --git a/internal/configuration/container/container_kill.go b/internal/configuration/container/container_kill.go index 65564ca3f..cc7b176fc 100644 --- a/internal/configuration/container/container_kill.go +++ b/internal/configuration/container/container_kill.go @@ -34,6 +34,7 @@ import ( runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/configuration/util" ) const ( @@ -73,7 +74,7 @@ func (d *dockerContainer) Kill(ctx context.Context, containerIDs []string, signa } errs := make([]error, 0, len(containerIDs)) - d.logger.Debugf("all docker container: %v", cfgcore.ToSet(allContainer).AsSlice()) + d.logger.Debugf("all docker container: %v", util.ToSet(allContainer).AsSlice()) for _, containerID := range containerIDs { d.logger.Infof("stopping docker container: %s", containerID) container, ok := allContainer[containerID] diff --git a/internal/configuration/cue_visitor.go b/internal/configuration/cue_visitor.go index e7b973495..28ec3f32a 100644 --- a/internal/configuration/cue_visitor.go +++ b/internal/configuration/cue_visitor.go @@ -25,6 +25,8 @@ import ( "cuelang.org/go/cue" "github.com/spf13/viper" "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/apecloud/kubeblocks/internal/configuration/util" ) var disableAutoTransfer = viper.GetBool("DISABLE_AUTO_TRANSFER") @@ -45,7 +47,7 @@ func (c *cueTypeExtractor) Visit(val cue.Value) { c.fieldTypes = make(map[string]CueType) c.fieldUnits = make(map[string]string) } - c.visitStruct(val) + c.visitStruct(val, "") } func (c *cueTypeExtractor) visitValue(x cue.Value, path string) { @@ -69,13 +71,20 @@ func (c *cueTypeExtractor) visitValue(x cue.Value, path string) { c.visitList(x, path) case k&cue.StructKind == cue.StructKind: c.addFieldType(path, StructType) - c.visitStruct(x) + c.visitStruct(x, path) default: log.Log.Info(fmt.Sprintf("cannot convert value of type %s", k.String())) } } -func (c *cueTypeExtractor) visitStruct(v cue.Value) { +func (c *cueTypeExtractor) visitStruct(v cue.Value, parentPath string) { + joinFieldPath := func(path string, name string) string { + if path == "" || strings.HasPrefix(path, "#") { + return name + } + return path + "." + name + } + switch op, v := v.Expr(); op { // SelectorOp refer of other struct type case cue.NoOp, cue.SelectorOp: @@ -90,7 +99,7 @@ func (c *cueTypeExtractor) visitStruct(v cue.Value) { for itr, _ := v.Fields(cue.Optional(true), cue.Definitions(true)); itr.Next(); { name := itr.Label() - c.visitValue(itr.Value(), name) + c.visitValue(itr.Value(), joinFieldPath(parentPath, name)) } } @@ -104,7 +113,7 @@ func (c *cueTypeExtractor) visitList(v cue.Value, path string) { count := 0 for i, _ := v.List(); i.Next(); count++ { - c.visitValue(i.Value(), fmt.Sprintf("%s_%d", path, count)) + c.visitValue(i.Value(), path) } } @@ -119,7 +128,21 @@ func (c *cueTypeExtractor) addFieldUnits(path string, t CueType, base string) { } } -func transNumberOrBoolType(t CueType, obj reflect.Value, fn UpdateFn, expand string, trimString bool) error { +func (c *cueTypeExtractor) hasFieldType(parent string, cur string) (string, bool) { + fieldRef := cur + if parent != "" { + fieldRef = parent + "." + cur + } + if _, exist := c.fieldTypes[fieldRef]; exist { + return fieldRef, true + } + if _, exist := c.fieldTypes[cur]; exist { + return cur, true + } + return "", false +} + +func transNumberOrBoolType(t CueType, obj reflect.Value, fn util.UpdateFn, expand string, trimString bool) error { switch t { case IntType: return processTypeTrans[int](obj, strconv.Atoi, fn, trimString) @@ -145,7 +168,7 @@ func transNumberOrBoolType(t CueType, obj reflect.Value, fn UpdateFn, expand str return nil } -func trimStringQuotes(obj reflect.Value, fn UpdateFn) { +func trimStringQuotes(obj reflect.Value, fn util.UpdateFn) { if obj.Type().Kind() != reflect.String { return } @@ -160,7 +183,7 @@ func trimStringQuotes(obj reflect.Value, fn UpdateFn) { } } -func processTypeTrans[T int | int64 | float64 | float32 | bool](obj reflect.Value, transFn func(s string) (T, error), updateFn UpdateFn, trimString bool) error { +func processTypeTrans[T int | int64 | float64 | float32 | bool](obj reflect.Value, transFn func(s string) (T, error), updateFn util.UpdateFn, trimString bool) error { switch obj.Type().Kind() { case reflect.String: str := obj.String() @@ -188,17 +211,15 @@ func processCfgNotStringParam(data interface{}, context *cue.Context, tpl cue.Va context: context, } typeTransformer.Visit(tpl) - return UnstructuredObjectWalk(typeTransformer.data, - func(parent, cur string, obj reflect.Value, fn UpdateFn) error { + return util.UnstructuredObjectWalk(typeTransformer.data, + func(parent, cur string, obj reflect.Value, fn util.UpdateFn) error { if fn == nil || cur == "" || !obj.IsValid() { return nil } - if t, exist := typeTransformer.fieldTypes[cur]; exist { - err := transNumberOrBoolType(t, obj, fn, typeTransformer.fieldUnits[cur], trimString) - if err != nil { - return WrapError(err, "failed to type convertor, field[%s]", cur) - } + fieldPath, exist := typeTransformer.hasFieldType(parent, cur) + if !exist { + return nil } - return nil + return transNumberOrBoolType(typeTransformer.fieldTypes[fieldPath], obj, fn, typeTransformer.fieldUnits[fieldPath], trimString) }, false) } diff --git a/internal/configuration/cue_visitor_test.go b/internal/configuration/cue_visitor_test.go index 901f5a900..5a918337c 100644 --- a/internal/configuration/cue_visitor_test.go +++ b/internal/configuration/cue_visitor_test.go @@ -71,19 +71,28 @@ func TestCueTypeExtractorVisit(t *testing.T) { } `, fieldTypes: map[string]CueType{ - "#a": StructType, - "b": StructType, - "g": StructType, - "#c": StructType, - "e": IntType, - "f": StringType, - "#j": StructType, - "x": StringType, - "y": IntType, - "#n": StructType, - "m": StructType, - "d": StructType, - "j": NullableType, + "#a": StructType, + "b": StructType, + "g": StructType, + "#c": StructType, + "e": IntType, + "f": StringType, + "#j": StructType, + "x": StringType, + "y": IntType, + "#n": StructType, + "m": StructType, + "d": StructType, + "j": NullableType, + "b.e": IntType, + "b.f": StringType, + "g.x": StringType, + "g.y": IntType, + "g.m": StructType, + "g.m.d": StructType, + "g.m.j": NullableType, + "m.d": StructType, + "m.j": NullableType, }, }, }, { @@ -102,14 +111,13 @@ func TestCueTypeExtractorVisit(t *testing.T) { i:[int] }`, fieldTypes: map[string]CueType{ - "#a": StructType, - "b": IntType, - "c": StringType, - "d": StringType, - "e": StringType, - "g": StructType, - "i": ListType, - "i_0": IntType, + "#a": StructType, + "b": IntType, + "c": StringType, + "d": StringType, + "e": StringType, + "g": StructType, + "i": IntType, }, }, }, { diff --git a/internal/configuration/reconfigure_util.go b/internal/configuration/reconfigure_util.go index 9884ea695..98f353c21 100644 --- a/internal/configuration/reconfigure_util.go +++ b/internal/configuration/reconfigure_util.go @@ -22,40 +22,64 @@ import ( "github.com/StudioSol/set" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/configuration/util" "github.com/apecloud/kubeblocks/internal/constant" ) -func getUpdateParameterList(cfg *ConfigPatchInfo) ([]string, error) { +func getUpdateParameterList(cfg *ConfigPatchInfo, trimField string) ([]string, error) { params := make([]string, 0) - walkFn := func(parent, cur string, v reflect.Value, fn UpdateFn) error { + walkFn := func(parent, cur string, v reflect.Value, fn util.UpdateFn) error { if cur != "" { + if parent != "" { + cur = parent + "." + cur + } params = append(params, cur) } return nil } for _, diff := range cfg.UpdateConfig { + var err error var updatedParams any - if err := json.Unmarshal(diff, &updatedParams); err != nil { + if err = json.Unmarshal(diff, &updatedParams); err != nil { return nil, err } - if err := UnstructuredObjectWalk(updatedParams, walkFn, true); err != nil { + if updatedParams, err = trimNestedField(updatedParams, trimField); err != nil { + return nil, err + } + if err := util.UnstructuredObjectWalk(updatedParams, walkFn, true); err != nil { return nil, WrapError(err, "failed to walk params: [%s]", diff) } } return params, nil } +func trimNestedField(updatedParams any, trimField string) (any, error) { + if trimField == "" { + return updatedParams, nil + } + if m, ok := updatedParams.(map[string]interface{}); ok { + trimParams, found, err := unstructured.NestedFieldNoCopy(m, trimField) + if err != nil { + return nil, err + } + if found { + return trimParams, nil + } + } + return updatedParams, nil +} + // IsUpdateDynamicParameters is used to check whether the changed parameters require a restart func IsUpdateDynamicParameters(cc *appsv1alpha1.ConfigConstraintSpec, cfg *ConfigPatchInfo) (bool, error) { - // TODO(zt) how to process new or delete file if len(cfg.DeleteConfig) > 0 || len(cfg.AddConfig) > 0 { return false, nil } - params, err := getUpdateParameterList(cfg) + params, err := getUpdateParameterList(cfg, NestedPrefixField(cc.FormatterConfig)) if err != nil { return false, err } @@ -64,7 +88,7 @@ func IsUpdateDynamicParameters(cc *appsv1alpha1.ConfigConstraintSpec, cfg *Confi // if ConfigConstraint has StaticParameters, check updated parameter if len(cc.StaticParameters) > 0 { staticParams := set.NewLinkedHashSetString(cc.StaticParameters...) - union := Union(staticParams, updateParams) + union := util.Union(staticParams, updateParams) if union.Length() > 0 { return false, nil } @@ -77,7 +101,7 @@ func IsUpdateDynamicParameters(cc *appsv1alpha1.ConfigConstraintSpec, cfg *Confi // if ConfigConstraint has DynamicParameter, all updated param in dynamic params if len(cc.DynamicParameters) > 0 { dynamicParams := set.NewLinkedHashSetString(cc.DynamicParameters...) - union := Difference(updateParams, dynamicParams) + union := util.Difference(updateParams, dynamicParams) return union.Length() == 0, nil } diff --git a/internal/configuration/reconfigure_util_test.go b/internal/configuration/reconfigure_util_test.go index 1d7c4468e..956caf224 100644 --- a/internal/configuration/reconfigure_util_test.go +++ b/internal/configuration/reconfigure_util_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/configuration/util" ) func TestGetUpdateParameterList(t *testing.T) { @@ -43,14 +44,17 @@ func TestGetUpdateParameterList(t *testing.T) { ], "g": { "cd" : "abcd", - "msld" : "cakl" + "msld" : { + "cakl": 100, + "dg": "abcd" + } }} ` - params, err := getUpdateParameterList(newCfgDiffMeta(testData, nil, nil)) + expected := set.NewLinkedHashSetString("a", "f", "c", "xxx.test1", "xxx.test2", "g.msld.cakl", "g.msld.dg", "g.cd") + params, err := getUpdateParameterList(newCfgDiffMeta(testData, nil, nil), "") require.Nil(t, err) - require.True(t, EqSet( - set.NewLinkedHashSetString("a", "c_1", "c_0", "msld", "cd", "f", "test1", "test2"), - set.NewLinkedHashSetString(params...))) + require.True(t, util.EqSet(expected, + set.NewLinkedHashSetString(params...)), "param: %v, expected: %v", params, expected.AsSlice()) } func newCfgDiffMeta(testData string, add, delete map[string]interface{}) *ConfigPatchInfo { diff --git a/internal/configuration/util/file_util.go b/internal/configuration/util/file_util.go new file mode 100644 index 000000000..c50836f3a --- /dev/null +++ b/internal/configuration/util/file_util.go @@ -0,0 +1,34 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "os" + "path/filepath" +) + +func FromConfigFiles(files []string) (map[string]string, error) { + m := make(map[string]string) + for _, file := range files { + b, err := os.ReadFile(file) + if err != nil { + return nil, err + } + m[filepath.Base(file)] = string(b) + } + return m, nil +} diff --git a/internal/configuration/hash.go b/internal/configuration/util/hash.go similarity index 84% rename from internal/configuration/hash.go rename to internal/configuration/util/hash.go index 0e1135aad..6ee4d5808 100644 --- a/internal/configuration/hash.go +++ b/internal/configuration/util/hash.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import ( "bytes" @@ -23,19 +23,20 @@ import ( "hash/fnv" "io" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/rand" ) func ComputeHash(object interface{}) (string, error) { objString, err := json.Marshal(object) if err != nil { - return "", WrapError(err, "failed to compute hash.") + return "", errors.Wrap(err, "failed to compute hash.") } // hasher := sha1.New() hasher := fnv.New32() if _, err := io.Copy(hasher, bytes.NewReader(objString)); err != nil { - return "", WrapError(err, "failed to compute hash for sha256. [%s]", objString) + return "", errors.Wrapf(err, "failed to compute hash for sha256. [%s]", objString) } sha := hasher.Sum32() diff --git a/internal/configuration/hash_test.go b/internal/configuration/util/hash_test.go similarity index 98% rename from internal/configuration/hash_test.go rename to internal/configuration/util/hash_test.go index 935f2c9e4..9c00aa6a1 100644 --- a/internal/configuration/hash_test.go +++ b/internal/configuration/util/hash_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import "testing" diff --git a/internal/configuration/jsonpath.go b/internal/configuration/util/jsonpath.go similarity index 77% rename from internal/configuration/jsonpath.go rename to internal/configuration/util/jsonpath.go index 3a74df4f0..63b14701d 100644 --- a/internal/configuration/jsonpath.go +++ b/internal/configuration/util/jsonpath.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import ( "encoding/json" @@ -23,14 +23,13 @@ import ( jsonpatch "github.com/evanphx/json-patch" ) -func retrievalWithJSONPath(jsonObj interface{}, jsonpath string) ([]byte, error) { - - jsonbytes, err := json.Marshal(&jsonObj) +func RetrievalWithJSONPath(jsonObj interface{}, jsonpath string) ([]byte, error) { + jsonBytes, err := json.Marshal(&jsonObj) if err != nil { return nil, err } - res, err := jsonslice.Get(jsonbytes, jsonpath) + res, err := jsonslice.Get(jsonBytes, jsonpath) if err != nil { return res, err } @@ -39,11 +38,10 @@ func retrievalWithJSONPath(jsonObj interface{}, jsonpath string) ([]byte, error) if resLen > 2 && res[0] == '"' && res[resLen-1] == '"' { res = res[1 : resLen-1] } - return res, err } -func jsonPatch(originalJSON, modifiedJSON interface{}) ([]byte, error) { +func JSONPatch(originalJSON, modifiedJSON interface{}) ([]byte, error) { originalBytes, err := json.Marshal(originalJSON) if err != nil { return nil, err @@ -53,7 +51,5 @@ func jsonPatch(originalJSON, modifiedJSON interface{}) ([]byte, error) { if err != nil { return nil, err } - - // TODO(zt) It's a hack to do the logic, json object --> bytes, bytes --> json object return jsonpatch.CreateMergePatch(originalBytes, modifiedBytes) } diff --git a/internal/configuration/math.go b/internal/configuration/util/math.go similarity index 97% rename from internal/configuration/math.go rename to internal/configuration/util/math.go index c3f1d9ece..9af6b2c75 100644 --- a/internal/configuration/math.go +++ b/internal/configuration/util/math.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import "golang.org/x/exp/constraints" diff --git a/internal/configuration/math_test.go b/internal/configuration/util/math_test.go similarity index 98% rename from internal/configuration/math_test.go rename to internal/configuration/util/math_test.go index 8c20ba020..0cefd6d4f 100644 --- a/internal/configuration/math_test.go +++ b/internal/configuration/util/math_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import ( "reflect" diff --git a/internal/configuration/set.go b/internal/configuration/util/set.go similarity index 98% rename from internal/configuration/set.go rename to internal/configuration/util/set.go index 7aeb0b3c2..0238ce2ac 100644 --- a/internal/configuration/set.go +++ b/internal/configuration/util/set.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import "github.com/StudioSol/set" diff --git a/internal/configuration/set_test.go b/internal/configuration/util/set_test.go similarity index 99% rename from internal/configuration/set_test.go rename to internal/configuration/util/set_test.go index 4fc565f41..903812dca 100644 --- a/internal/configuration/set_test.go +++ b/internal/configuration/util/set_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import ( "testing" diff --git a/internal/configuration/unstructured.go b/internal/configuration/util/unstructured.go similarity index 82% rename from internal/configuration/unstructured.go rename to internal/configuration/util/unstructured.go index dc8d79dcb..704324522 100644 --- a/internal/configuration/unstructured.go +++ b/internal/configuration/util/unstructured.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import ( "fmt" @@ -46,7 +46,7 @@ type unstructuredAccessor struct { func (accessor *unstructuredAccessor) Visit(data interface{}) error { v := reflect.ValueOf(data) if !v.IsValid() { - return MakeError("invalid data type: %T", data) + return fmt.Errorf("invalid data type: %T", data) } return accessor.visitValueType(v, v.Type(), "", "", nil) } @@ -77,25 +77,25 @@ func (accessor *unstructuredAccessor) visitValueType(v reflect.Value, t reflect. implValue := v.Elem() return accessor.visitValueType(implValue, implValue.Type(), parent, cur, updateFn) case reflect.Struct: - return accessor.visitStruct(v, cur) + return accessor.visitStruct(v, joinFieldPath(parent, cur)) case reflect.Map: - return accessor.visitMap(v, t, cur) + return accessor.visitMap(v, t, joinFieldPath(parent, cur)) case reflect.Slice: - return accessor.visitArray(v, t.Elem(), cur) + return accessor.visitArray(v, t.Elem(), parent, cur) case reflect.Array: - return accessor.visitArray(v, t.Elem(), cur) + return accessor.visitArray(v, t.Elem(), parent, cur) case reflect.Pointer: return accessor.visitValueType(v, t.Elem(), parent, cur, updateFn) default: - return MakeError("not support type: %s", k) + return fmt.Errorf("not support type: %s", k) } } -func (accessor *unstructuredAccessor) visitArray(v reflect.Value, t reflect.Type, parent string) error { +func (accessor *unstructuredAccessor) visitArray(v reflect.Value, t reflect.Type, parent, cur string) error { n := v.Len() for i := 0; i < n; i++ { - index := fmt.Sprintf("%s_%d", parent, i) - if err := accessor.visitValueType(v.Index(i), t, parent, index, nil); err != nil { + // index := fmt.Sprintf("%s_%d", parent, i) + if err := accessor.visitValueType(v.Index(i), t, parent, cur, nil); err != nil { return err } } @@ -111,7 +111,7 @@ func (accessor *unstructuredAccessor) visitMap(v reflect.Value, t reflect.Type, switch k := t.Key().Kind(); k { case reflect.String: default: - return MakeError("not support key type: %s", k) + return fmt.Errorf("not support key type: %s", k) } t = t.Elem() @@ -157,6 +157,18 @@ func toString(key reflect.Value, kind reflect.Kind) string { } } +func joinFieldPath(parent, cur string) string { + if parent == "" { + return cur + } + + if cur == "" { + return parent + } + + return parent + "." + cur +} + func (accessor *unstructuredAccessor) visitStruct(v reflect.Value, parent string) error { - return MakeError("not support struct.") + return fmt.Errorf("not support struct") } diff --git a/internal/configuration/unstructured_test.go b/internal/configuration/util/unstructured_test.go similarity index 89% rename from internal/configuration/unstructured_test.go rename to internal/configuration/util/unstructured_test.go index a96a6e10a..eb724d194 100644 --- a/internal/configuration/unstructured_test.go +++ b/internal/configuration/util/unstructured_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package configuration +package util import ( "encoding/json" @@ -76,7 +76,7 @@ func TestUnstructuredObjectWalk(t *testing.T) { "g" : [ "e1","e2","e3"], "x" : [ 20,30] }}`, - expected: []string{"c", "d", "f", "x1", "x2", "x3", "x4"}, + expected: []string{"a.b.z.x1", "a.b.e.c", "a.b.e.d", "a.b.f", "a.b.z.x2", "a.b.z.x4", "a.b.z.x3", "a.g", "a.x"}, isStruct: false, }, }, { @@ -109,6 +109,9 @@ func TestUnstructuredObjectWalk(t *testing.T) { if cur == "" && parent != "" { res = append(res, parent) } else if cur != "" { + if parent != "" { + cur = parent + "." + cur + } res = append(res, cur) } return nil @@ -117,7 +120,7 @@ func TestUnstructuredObjectWalk(t *testing.T) { } if !tt.wantErr { - require.True(t, Contains(res, tt.args.expected)) + require.True(t, Contains(res, tt.args.expected), "res: %v, expected: %v", res, tt.args.expected) } }) } diff --git a/internal/controller/plan/prepare.go b/internal/controller/plan/prepare.go index 04efdfdcd..c32402902 100644 --- a/internal/controller/plan/prepare.go +++ b/internal/controller/plan/prepare.go @@ -34,6 +34,7 @@ import ( cfgutil "github.com/apecloud/kubeblocks/controllers/apps/configuration" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" cfgcm "github.com/apecloud/kubeblocks/internal/configuration/config_manager" + "github.com/apecloud/kubeblocks/internal/configuration/util" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/builder" "github.com/apecloud/kubeblocks/internal/controller/component" @@ -355,7 +356,7 @@ func updateResourceAnnotationsWithTemplate(obj client.Object, allTemplateAnnotat } // delete not exist configmap label - deletedLabels := cfgcore.MapKeyDifference(existLabels, allTemplateAnnotations) + deletedLabels := util.MapKeyDifference(existLabels, allTemplateAnnotations) for l := range deletedLabels.Iter() { delete(annotations, l) } diff --git a/internal/unstructured/viper_wrap.go b/internal/unstructured/viper_wrap.go index 6b251be6a..1a25acce3 100644 --- a/internal/unstructured/viper_wrap.go +++ b/internal/unstructured/viper_wrap.go @@ -37,7 +37,7 @@ type viperWrap struct { func init() { CfgObjectRegistry().RegisterConfigCreator(appsv1alpha1.Ini, createViper(appsv1alpha1.Ini)) - CfgObjectRegistry().RegisterConfigCreator(appsv1alpha1.YAML, createViper(appsv1alpha1.YAML)) + // CfgObjectRegistry().RegisterConfigCreator(appsv1alpha1.YAML, createViper(appsv1alpha1.YAML)) CfgObjectRegistry().RegisterConfigCreator(appsv1alpha1.JSON, createViper(appsv1alpha1.JSON)) CfgObjectRegistry().RegisterConfigCreator(appsv1alpha1.Dotenv, createViper(appsv1alpha1.Dotenv)) CfgObjectRegistry().RegisterConfigCreator(appsv1alpha1.HCL, createViper(appsv1alpha1.HCL)) diff --git a/internal/unstructured/viper_wrap_test.go b/internal/unstructured/viper_wrap_test.go index 5ad7ea87b..a20e41c2d 100644 --- a/internal/unstructured/viper_wrap_test.go +++ b/internal/unstructured/viper_wrap_test.go @@ -118,32 +118,3 @@ func TestJSONFormat(t *testing.T) { assert.EqualValues(t, jsonConfigObj.Get("name"), "test") } - -func TestYAMLFormat(t *testing.T) { - const yamlContext = ` -spec: - clusterref: pg - reconfigure: - componentname: postgresql - configurations: - - keys: - - key: postgresql.conf - parameters: - - key: max_connections - value: "2666" - name: postgresql-configuration -` - - yamlConfigObj, err := LoadConfig("yaml_test", yamlContext, appsv1alpha1.YAML) - assert.Nil(t, err) - - assert.EqualValues(t, yamlConfigObj.Get("spec.clusterRef"), "pg") - assert.EqualValues(t, yamlConfigObj.Get("spec.reconfigure.componentName"), "postgresql") - - dumpContext, err := yamlConfigObj.Marshal() - assert.Nil(t, err) - assert.EqualValues(t, dumpContext, yamlContext[1:]) // trim "\n" - - assert.Nil(t, yamlConfigObj.Update("spec.my_test", "100")) - assert.EqualValues(t, yamlConfigObj.Get("spec.my_test"), "100") -} diff --git a/internal/unstructured/yaml_config.go b/internal/unstructured/yaml_config.go new file mode 100644 index 000000000..0e9cc1029 --- /dev/null +++ b/internal/unstructured/yaml_config.go @@ -0,0 +1,143 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unstructured + +import ( + "strings" + + "github.com/spf13/cast" + "gopkg.in/yaml.v2" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" +) + +type yamlConfig struct { + name string + config map[string]any +} + +func init() { + CfgObjectRegistry().RegisterConfigCreator(appsv1alpha1.YAML, func(name string) ConfigObject { + return &yamlConfig{name: name} + }) +} + +func (y *yamlConfig) Update(key string, value any) error { + path := strings.Split(key, ".") + lastKey := path[len(path)-1] + deepestMap := checkAndCreateNestedPrefixMap(y.config, path[0:len(path)-1]) + deepestMap[lastKey] = value + return nil +} + +func (y *yamlConfig) Get(key string) any { + keys := strings.Split(key, ".") + return searchMap(y.config, keys) +} + +func (y *yamlConfig) GetString(key string) (string, error) { + v := y.Get(key) + if v != nil { + return cast.ToStringE(v) + } + return "", nil +} + +func (y *yamlConfig) GetAllParameters() map[string]any { + return y.config +} + +func (y *yamlConfig) SubConfig(key string) ConfigObject { + v := y.Get(key) + if m, ok := v.(map[string]any); ok { + return &yamlConfig{ + name: y.name, + config: m, + } + } + return nil +} + +func (y *yamlConfig) Marshal() (string, error) { + b, err := yaml.Marshal(y.config) + return string(b), err +} + +func (y *yamlConfig) Unmarshal(str string) error { + config := make(map[any]any) + err := yaml.Unmarshal([]byte(str), config) + if err != nil { + return err + } + y.config = transKeyString(config) + return nil +} + +func checkAndCreateNestedPrefixMap(m map[string]any, path []string) map[string]any { + for _, k := range path { + m2, ok := m[k] + // if the key is not exist, create a new map + if !ok { + m3 := make(map[string]any) + m[k] = m3 + m = m3 + continue + } + m3, ok := m2.(map[string]any) + // if the type is not map, replace with a new map + if !ok { + m3 = make(map[string]any) + m[k] = m3 + } + m = m3 + } + return m +} + +func searchMap(m map[string]any, path []string) any { + if len(path) == 0 { + return m + } + + next, ok := m[path[0]] + if !ok { + return nil + } + if len(path) == 1 { + return next + } + switch t := next.(type) { + default: + return nil + case map[any]any: + return searchMap(cast.ToStringMap(t), path[1:]) + case map[string]any: + return searchMap(t, path[1:]) + } +} + +func transKeyString(m map[any]any) map[string]any { + m2 := make(map[string]any, len(m)) + for k, v := range m { + if vi, ok := v.(map[any]any); ok { + m2[cast.ToString(k)] = transKeyString(vi) + } else { + m2[cast.ToString(k)] = v + } + } + return m2 +} diff --git a/internal/unstructured/yaml_config_test.go b/internal/unstructured/yaml_config_test.go new file mode 100644 index 000000000..d2ff68011 --- /dev/null +++ b/internal/unstructured/yaml_config_test.go @@ -0,0 +1,54 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package unstructured + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" +) + +func TestYAMLFormat(t *testing.T) { + const yamlContext = ` +spec: + clusterRef: pg + reconfigure: + componentName: postgresql + configurations: + - keys: + - key: postgresql.conf + parameters: + - key: max_connections + value: "2666" + name: postgresql-configuration +` + + yamlConfigObj, err := LoadConfig("yaml_test", yamlContext, appsv1alpha1.YAML) + assert.Nil(t, err) + + assert.EqualValues(t, yamlConfigObj.Get("spec.clusterRef"), "pg") + assert.EqualValues(t, yamlConfigObj.Get("spec.reconfigure.componentName"), "postgresql") + + dumpContext, err := yamlConfigObj.Marshal() + assert.Nil(t, err) + assert.EqualValues(t, dumpContext, yamlContext[1:]) // trim "\n" + + assert.Nil(t, yamlConfigObj.Update("spec.my_test", "100")) + assert.EqualValues(t, yamlConfigObj.Get("spec.my_test"), "100") +} diff --git a/test/testdata/cue_testdata/mongod.conf b/test/testdata/cue_testdata/mongod.conf new file mode 100644 index 000000000..c07e89dba --- /dev/null +++ b/test/testdata/cue_testdata/mongod.conf @@ -0,0 +1,21 @@ +storage: + dbPath: "/var/lib/mongo" +systemLog: + destination: file + path: "/var/log/mongodb/mongod.log" +net: + port: 2000 + bindIp: + __exec: "python /home/user/getIPAddresses.py" + type: "string" + trim: "whitespace" + digest: 85fed8997aac3f558e779625f2e51b4d142dff11184308dc6aca06cff26ee9ad + digest_key: 68656c6c30303030307365637265746d796f6c64667269656e64 + tls: + mode: requireTLS + certificateKeyFile: "/etc/tls/mongod.pem" + certificateKeyFilePassword: + __rest: "https://myrestserver.example.net/api/config/myCertKeyFilePassword" + type: "string" + digest: b08519162ba332985ac18204851949611ef73835ec99067b85723e10113f5c26 + digest_key: 6d795365637265744b65795374756666 \ No newline at end of file diff --git a/test/testdata/cue_testdata/mongod.cue b/test/testdata/cue_testdata/mongod.cue new file mode 100644 index 000000000..10bb6c66a --- /dev/null +++ b/test/testdata/cue_testdata/mongod.cue @@ -0,0 +1,37 @@ +#MongodParameter: { + net: { + port: int & >=0 & <=65535 + + unixDomainSocket: { + // Enables Unix Domain Sockets used for all network connections + enabled: bool | *false + pathPrefix: string + ... + } + tls: { + // Enables TLS used for all network connections + mode: string & "disabled" | "allowTLS" | "preferTLS" | "requireTLS" + + certificateKeyFile: string + CAFile: string + CRLFile: string + ... + } + ... + } + tls: { + // Enables TLS used for all network connections + mode: string & "disabled" | "allowTLS" | "preferTLS" | "requireTLS" + + certificateKeyFile: string + CAFile: string + CRLFile: string + ... + } + + ... +} + +// configuration require +configuration: #MongodParameter & { +} From 8f0a440713ea307abbf40a5dc9f97182a120022f Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Wed, 12 Apr 2023 19:11:00 +0800 Subject: [PATCH 006/439] chore: url package name needs to be consistent with chart name (#2539) --- deploy/helm/templates/addons/milvus-addon.yaml | 2 +- deploy/helm/templates/addons/qdrant-addon.yaml | 2 +- deploy/helm/templates/addons/weaviate-addon.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/helm/templates/addons/milvus-addon.yaml b/deploy/helm/templates/addons/milvus-addon.yaml index 40d9f022f..9ae726baf 100644 --- a/deploy/helm/templates/addons/milvus-addon.yaml +++ b/deploy/helm/templates/addons/milvus-addon.yaml @@ -15,7 +15,7 @@ spec: type: Helm helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/milvus-{{ default .Chart.Version .Values.versionOverride }}.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/milvus-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: autoInstall: false diff --git a/deploy/helm/templates/addons/qdrant-addon.yaml b/deploy/helm/templates/addons/qdrant-addon.yaml index 9f3b8ab10..38abb8625 100644 --- a/deploy/helm/templates/addons/qdrant-addon.yaml +++ b/deploy/helm/templates/addons/qdrant-addon.yaml @@ -15,7 +15,7 @@ spec: type: Helm helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/qdrant-{{ default .Chart.Version .Values.versionOverride }}.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/qdrant-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: autoInstall: false diff --git a/deploy/helm/templates/addons/weaviate-addon.yaml b/deploy/helm/templates/addons/weaviate-addon.yaml index 3d7891897..152b5fda2 100644 --- a/deploy/helm/templates/addons/weaviate-addon.yaml +++ b/deploy/helm/templates/addons/weaviate-addon.yaml @@ -15,7 +15,7 @@ spec: type: Helm helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/weaviate-{{ default .Chart.Version .Values.versionOverride }}.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/weaviate-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: autoInstall: false From 814ffbd1853aa9ddf8c7d5d918172261d1ae31e5 Mon Sep 17 00:00:00 2001 From: xingran Date: Wed, 12 Apr 2023 19:23:00 +0800 Subject: [PATCH 007/439] fix: kbcli support switchPolicy setup when create replicationSet cluster (#2540) --- .../templates/clusterdefinition.yaml | 2 + deploy/postgresql/templates/configmap.yaml | 1 + docs/user_docs/cli/kbcli_cluster_create.md | 3 ++ internal/cli/cmd/cluster/create.go | 45 +++++++++++++++---- internal/cli/cmd/cluster/create_test.go | 18 +++++++- 5 files changed, 59 insertions(+), 10 deletions(-) diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 30642c4d3..4b06d0895 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -182,6 +182,8 @@ spec: secretKeyRef: name: $(CONN_CREDENTIAL_SECRET_NAME) key: password + - name: PGUSER_STANDBY + value: standby - name: PGPASSWORD_STANDBY valueFrom: secretKeyRef: diff --git a/deploy/postgresql/templates/configmap.yaml b/deploy/postgresql/templates/configmap.yaml index f4965bf73..66c97e3ee 100644 --- a/deploy/postgresql/templates/configmap.yaml +++ b/deploy/postgresql/templates/configmap.yaml @@ -14,6 +14,7 @@ data: local all all trust host all all 127.0.0.1/32 trust host all all ::1/128 trust + local replication all trust host replication all 0.0.0.0/0 md5 host replication all ::/0 md5 kb_restore.conf: | diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index 424296190..d6ec2c9f0 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -38,6 +38,9 @@ kbcli cluster create [CLUSTER_NAME] [flags] # Create a cluster and set class to general-1c4g kbcli cluster create myclsuter --cluster-definition apecloud-mysql --set class=general-1c4g + # Create a cluster with replicationSet workloadType and set switchPolicy to Noop + kbcli cluster create myclsuter --cluster-definition postgresql --set switchPolicy=Noop + # Create a cluster and use a URL to set cluster resource kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file https://kubeblocks.io/yamls/my.yaml diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 03ae63a02..49a725e8f 100644 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -79,6 +79,9 @@ var clusterCreateExample = templates.Examples(` # Create a cluster and set class to general-1c4g kbcli cluster create myclsuter --cluster-definition apecloud-mysql --set class=general-1c4g + # Create a cluster with replicationSet workloadType and set switchPolicy to Noop + kbcli cluster create myclsuter --cluster-definition postgresql --set switchPolicy=Noop + # Create a cluster and use a URL to set cluster resource kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file https://kubeblocks.io/yamls/my.yaml @@ -107,13 +110,14 @@ const ( type setKey string const ( - keyType setKey = "type" - keyCPU setKey = "cpu" - keyClass setKey = "class" - keyMemory setKey = "memory" - keyReplicas setKey = "replicas" - keyStorage setKey = "storage" - keyUnknown setKey = "unknown" + keyType setKey = "type" + keyCPU setKey = "cpu" + keyClass setKey = "class" + keyMemory setKey = "memory" + keyReplicas setKey = "replicas" + keyStorage setKey = "storage" + keySwitchPolicy setKey = "switchPolicy" + keyUnknown setKey = "unknown" ) type envSet struct { @@ -459,7 +463,6 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map return v } } - // get value from environment variables env := setKeyEnvMap[key] val := viper.GetString(env.name) @@ -469,6 +472,27 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map return val } + buildSwitchPolicy := func(c *appsv1alpha1.ClusterComponentDefinition, compObj *appsv1alpha1.ClusterComponentSpec, sets map[setKey]string) error { + if c.WorkloadType != appsv1alpha1.Replication { + return nil + } + var switchPolicyType appsv1alpha1.SwitchPolicyType + switch getVal(keySwitchPolicy, sets) { + case "Noop", "": + switchPolicyType = appsv1alpha1.Noop + case "MaximumAvailability": + switchPolicyType = appsv1alpha1.MaximumAvailability + case "MaximumPerformance": + switchPolicyType = appsv1alpha1.MaximumDataProtection + default: + return fmt.Errorf("switchPolicy is illegal, only support Noop, MaximumAvailability, MaximumPerformance") + } + compObj.SwitchPolicy = &appsv1alpha1.ClusterSwitchPolicy{ + Type: switchPolicyType, + } + return nil + } + var comps []*appsv1alpha1.ClusterComponentSpec for _, c := range cd.Spec.ComponentDefs { sets := map[setKey]string{} @@ -509,6 +533,9 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map }, }}, } + if err := buildSwitchPolicy(&c, compObj, sets); err != nil { + return nil, err + } comps = append(comps, compObj) } return comps, nil @@ -518,7 +545,7 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map // specified in the set, use the cluster definition default component name. func buildCompSetsMap(values []string, cd *appsv1alpha1.ClusterDefinition) (map[string]map[setKey]string, error) { allSets := map[string]map[setKey]string{} - keys := []string{string(keyCPU), string(keyType), string(keyStorage), string(keyMemory), string(keyReplicas), string(keyClass)} + keys := []string{string(keyCPU), string(keyType), string(keyStorage), string(keyMemory), string(keyReplicas), string(keyClass), string(keySwitchPolicy)} parseKey := func(key string) setKey { for _, k := range keys { if strings.EqualFold(k, key) { diff --git a/internal/cli/cmd/cluster/create_test.go b/internal/cli/cmd/cluster/create_test.go index d2916ec2b..28f017683 100644 --- a/internal/cli/cmd/cluster/create_test.go +++ b/internal/cli/cmd/cluster/create_test.go @@ -181,6 +181,11 @@ var _ = Describe("create", func() { comps, err := buildClusterComp(cd, setsMap) Expect(err).Should(Succeed()) checkComponent(comps, "10Gi", 10, "10", "2Gi") + + setsMap[testing.ComponentDefName][keySwitchPolicy] = "invalid" + cd.Spec.ComponentDefs[0].WorkloadType = appsv1alpha1.Replication + _, err = buildClusterComp(cd, setsMap) + Expect(err).Should(HaveOccurred()) }) It("build component and set values map", func() { @@ -189,7 +194,8 @@ var _ = Describe("create", func() { var comps []appsv1alpha1.ClusterComponentDefinition for _, n := range compDefNames { comp := appsv1alpha1.ClusterComponentDefinition{ - Name: n, + Name: n, + WorkloadType: appsv1alpha1.Replication, } comps = append(comps, comp) } @@ -304,6 +310,16 @@ var _ = Describe("create", func() { }, true, }, + { + []string{"switchPolicy=MaximumAvailability"}, + []string{"my-comp"}, + map[string]map[setKey]string{ + "my-comp": { + keySwitchPolicy: "MaximumAvailability", + }, + }, + true, + }, } for _, t := range testCases { From e115d8f7ab5c7e0019fb7894746cadf2316f321d Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Wed, 12 Apr 2023 21:13:33 +0800 Subject: [PATCH 008/439] chore: tidy up clusterdefinition +listType=map markers (#2544) --- apis/apps/v1alpha1/clusterdefinition_types.go | 65 +++---------------- apis/apps/v1alpha1/type.go | 53 +++++++++++++++ ...apps.kubeblocks.io_clusterdefinitions.yaml | 14 ++-- .../apps.kubeblocks.io_clusterversions.yaml | 1 + ...apps.kubeblocks.io_clusterdefinitions.yaml | 14 ++-- .../apps.kubeblocks.io_clusterversions.yaml | 1 + .../controller/component/monitor_utils.go | 16 ++++- .../component/monitor_utils_test.go | 3 +- 8 files changed, 100 insertions(+), 67 deletions(-) diff --git a/apis/apps/v1alpha1/clusterdefinition_types.go b/apis/apps/v1alpha1/clusterdefinition_types.go index 88ed96c75..630f0eda1 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types.go +++ b/apis/apps/v1alpha1/clusterdefinition_types.go @@ -176,64 +176,11 @@ func (r ClusterDefinitionStatus) GetTerminalPhases() []Phase { return []Phase{AvailablePhase} } -type ComponentTemplateSpec struct { - // Specify the name of configuration template. - // +kubebuilder:validation:Required - // +kubebuilder:validation:MaxLength=63 - // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` - Name string `json:"name"` - - // Specify the name of the referenced the configuration template ConfigMap object. - // +kubebuilder:validation:Required - // +kubebuilder:validation:MaxLength=63 - // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` - TemplateRef string `json:"templateRef"` - - // Specify the namespace of the referenced the configuration template ConfigMap object. - // An empty namespace is equivalent to the "default" namespace. - // +kubebuilder:validation:MaxLength=63 - // +kubebuilder:default="default" - // +optional - Namespace string `json:"namespace,omitempty"` - - // volumeName is the volume name of PodTemplate, which the configuration file produced through the configuration template will be mounted to the corresponding volume. - // The volume name must be defined in podSpec.containers[*].volumeMounts. - // +kubebuilder:validation:Required - // +kubebuilder:validation:MaxLength=32 - VolumeName string `json:"volumeName"` - - // defaultMode is optional: mode bits used to set permissions on created files by default. - // Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. - // YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. - // Defaults to 0644. - // Directories within the path are not affected by this setting. - // This might be in conflict with other options that affect the file - // mode, like fsGroup, and the result can be other mode bits set. - // +optional - DefaultMode *int32 `json:"defaultMode,omitempty" protobuf:"varint,3,opt,name=defaultMode"` -} - -type ComponentConfigSpec struct { - ComponentTemplateSpec `json:",inline"` - - // Specify a list of keys. - // If empty, ConfigConstraint takes effect for all keys in configmap. - // +optional - Keys []string `json:"keys,omitempty"` - - // Specify the name of the referenced the configuration constraints object. - // +kubebuilder:validation:MaxLength=63 - // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` - // +optional - ConfigConstraintRef string `json:"constraintRef,omitempty"` -} - type ExporterConfig struct { // scrapePort is exporter port for Time Series Database to scrape metrics. // +kubebuilder:validation:Required - // +kubebuilder:validation:Maximum=65535 - // +kubebuilder:validation:Minimum=0 - ScrapePort int32 `json:"scrapePort"` + // +kubebuilder:validation:XIntOrString + ScrapePort intstr.IntOrString `json:"scrapePort"` // scrapePath is exporter url path for Time Series Database to scrape metrics. // +kubebuilder:validation:MaxLength=128 @@ -272,9 +219,9 @@ type LogConfig struct { type VolumeTypeSpec struct { // name definition is the same as the name of the VolumeMounts field in PodSpec.Container, // similar to the relations of Volumes[*].name and VolumesMounts[*].name in Pod.Spec. + // +kubebuilder:validation:Required // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` - // +optional - Name string `json:"name,omitempty"` + Name string `json:"name"` // type is in enum of {data, log}. // VolumeTypeData: the volume is for the persistent data storage. @@ -392,6 +339,8 @@ type ClusterComponentDefinition struct { // NOTE: // When volumeTypes is not defined, the backup function will not be supported, // even if a persistent volume has been specified. + // +listType=map + // +listMapKey=name // +optional VolumeTypes []VolumeTypeSpec `json:"volumeTypes,omitempty"` @@ -410,6 +359,7 @@ type ServiceSpec struct { // +listType=map // +listMapKey=port // +listMapKey=protocol + // +optional Ports []ServicePort `json:"ports,omitempty" patchStrategy:"merge" patchMergeKey:"port" protobuf:"bytes,1,rep,name=ports"` } @@ -462,6 +412,7 @@ type ServicePort struct { // This field is ignored for services with clusterIP=None, and should be // omitted or set equal to the 'port' field. // More info: https://kubernetes.io/docs/concepts/services-networking/service/#defining-a-service + // +kubebuilder:validation:XIntOrString // +optional TargetPort intstr.IntOrString `json:"targetPort,omitempty" protobuf:"bytes,4,opt,name=targetPort"` } diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index dce0c9c8d..7df51c6ec 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -30,6 +30,59 @@ const ( OpsRequestKind = "OpsRequestKind" ) +type ComponentTemplateSpec struct { + // Specify the name of configuration template. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + Name string `json:"name"` + + // Specify the name of the referenced the configuration template ConfigMap object. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + TemplateRef string `json:"templateRef"` + + // Specify the namespace of the referenced the configuration template ConfigMap object. + // An empty namespace is equivalent to the "default" namespace. + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:default="default" + // +optional + Namespace string `json:"namespace,omitempty"` + + // volumeName is the volume name of PodTemplate, which the configuration file produced through the configuration template will be mounted to the corresponding volume. + // The volume name must be defined in podSpec.containers[*].volumeMounts. + // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=32 + VolumeName string `json:"volumeName"` + + // defaultMode is optional: mode bits used to set permissions on created files by default. + // Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + // YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + // Defaults to 0644. + // Directories within the path are not affected by this setting. + // This might be in conflict with other options that affect the file + // mode, like fsGroup, and the result can be other mode bits set. + // +optional + DefaultMode *int32 `json:"defaultMode,omitempty" protobuf:"varint,3,opt,name=defaultMode"` +} + +type ComponentConfigSpec struct { + ComponentTemplateSpec `json:",inline"` + + // Specify a list of keys. + // If empty, ConfigConstraint takes effect for all keys in configmap. + // +listType=set + // +optional + Keys []string `json:"keys,omitempty"` + + // Specify the name of the referenced the configuration constraints object. + // +kubebuilder:validation:MaxLength=63 + // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` + // +optional + ConfigConstraintRef string `json:"constraintRef,omitempty"` +} + // ClusterPhase defines the Cluster CR .status.phase // +enum // +kubebuilder:validation:Enum={Running,Stopped,Failed,Abnormal,Creating,Updating} diff --git a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml index d146df534..49b645d13 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml @@ -94,6 +94,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: set name: description: Specify the name of configuration template. maxLength: 63 @@ -371,12 +372,12 @@ spec: maxLength: 128 type: string scrapePort: + anyOf: + - type: integer + - type: string description: scrapePort is exporter port for Time Series Database to scrape metrics. - format: int32 - maximum: 65535 - minimum: 0 - type: integer + x-kubernetes-int-or-string: true required: - scrapePort type: object @@ -8683,8 +8684,13 @@ spec: - data - log type: string + required: + - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map workloadType: description: workloadType defines type of the workload. Stateless is a stateless workload type used to describe stateless applications. diff --git a/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml index 517450462..f023b9d1f 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml @@ -100,6 +100,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: set name: description: Specify the name of configuration template. maxLength: 63 diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml index d146df534..49b645d13 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml @@ -94,6 +94,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: set name: description: Specify the name of configuration template. maxLength: 63 @@ -371,12 +372,12 @@ spec: maxLength: 128 type: string scrapePort: + anyOf: + - type: integer + - type: string description: scrapePort is exporter port for Time Series Database to scrape metrics. - format: int32 - maximum: 65535 - minimum: 0 - type: integer + x-kubernetes-int-or-string: true required: - scrapePort type: object @@ -8683,8 +8684,13 @@ spec: - data - log type: string + required: + - name type: object type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map workloadType: description: workloadType defines type of the workload. Stateless is a stateless workload type used to describe stateless applications. diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml index 517450462..f023b9d1f 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml @@ -100,6 +100,7 @@ spec: items: type: string type: array + x-kubernetes-list-type: set name: description: Specify the name of configuration template. maxLength: 63 diff --git a/internal/controller/component/monitor_utils.go b/internal/controller/component/monitor_utils.go index e7a36974a..f3bc1dd56 100644 --- a/internal/controller/component/monitor_utils.go +++ b/internal/controller/component/monitor_utils.go @@ -17,6 +17,8 @@ limitations under the License. package component import ( + "k8s.io/apimachinery/pkg/util/intstr" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" ) @@ -43,7 +45,19 @@ func buildMonitorConfig( component.Monitor = &MonitorConfig{ Enable: true, ScrapePath: monitorConfig.Exporter.ScrapePath, - ScrapePort: monitorConfig.Exporter.ScrapePort, + ScrapePort: monitorConfig.Exporter.ScrapePort.IntVal, + } + + if monitorConfig.Exporter.ScrapePort.Type == intstr.String { + portName := monitorConfig.Exporter.ScrapePort.StrVal + for _, c := range clusterCompDef.PodSpec.Containers { + for _, p := range c.Ports { + if p.Name == portName { + component.Monitor.ScrapePort = p.ContainerPort + break + } + } + } } return } diff --git a/internal/controller/component/monitor_utils_test.go b/internal/controller/component/monitor_utils_test.go index 01663dfbc..6d70f628e 100644 --- a/internal/controller/component/monitor_utils_test.go +++ b/internal/controller/component/monitor_utils_test.go @@ -19,6 +19,7 @@ package component import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/intstr" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" ) @@ -37,7 +38,7 @@ var _ = Describe("monitor_utils", func() { clusterCompDef.Monitor = &appsv1alpha1.MonitorConfig{ BuiltIn: false, Exporter: &appsv1alpha1.ExporterConfig{ - ScrapePort: 9144, + ScrapePort: intstr.FromInt(9144), ScrapePath: "/metrics", }, } From 9929fc887f1f4163a0e1762651ce6b5a3786fe89 Mon Sep 17 00:00:00 2001 From: xingran Date: Thu, 13 Apr 2023 10:22:34 +0800 Subject: [PATCH 009/439] fix: tiny up configMaps when cluster deleted (#2548) --- .../controller/lifecycle/cluster_plan_builder.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 8d53cd0d8..7b4ff8261 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -507,6 +507,9 @@ func (c *clusterPlanBuilder) handleClusterDeletion(cluster *appsv1alpha1.Cluster if err := c.deletePVCs(cluster); err != nil && !apierrors.IsNotFound(err) { return err } + if err := c.deleteConfigMaps(cluster); err != nil && !apierrors.IsNotFound(err) { + return err + } // The backup policy must be cleaned up when the cluster is deleted. // Automatic backup scheduling needs to be stopped at this point. if err := c.deleteBackupPolicies(cluster); err != nil && !apierrors.IsNotFound(err) { @@ -542,6 +545,15 @@ func (c *clusterPlanBuilder) deletePVCs(cluster *appsv1alpha1.Cluster) error { return nil } +func (c *clusterPlanBuilder) deleteConfigMaps(cluster *appsv1alpha1.Cluster) error { + inNS := client.InNamespace(cluster.Namespace) + ml := client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.GetName(), + constant.AppManagedByLabelKey: constant.AppName, + } + return c.cli.DeleteAllOf(c.ctx.Ctx, &corev1.ConfigMap{}, inNS, ml) +} + func (c *clusterPlanBuilder) deleteBackupPolicies(cluster *appsv1alpha1.Cluster) error { inNS := client.InNamespace(cluster.Namespace) ml := client.MatchingLabels{ From a12ae6420a3f8d68fad5fde706d62f2b6d5aa6a3 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Thu, 13 Apr 2023 11:21:58 +0800 Subject: [PATCH 010/439] feat: support mongodb (#2182) Co-authored-by: yunju.lly Co-authored-by: Ziang Guo --- apis/apps/v1alpha1/clusterdefinition_types.go | 6 +- .../v1alpha1/clusterdefinition_webhook.go | 4 +- .../clusterdefinition_webhook_test.go | 6 +- cmd/probe/internal/binding/mongodb/mongodb.go | 105 +- .../internal/binding/mongodb/mongodb_test.go | 8 +- ...apps.kubeblocks.io_clusterdefinitions.yaml | 14 +- config/probe/components/binding_mongodb.yaml | 2 +- .../apps/components/util/component_utils.go | 2 +- .../components/util/component_utils_test.go | 2 +- .../templates/clusterdefinition.yaml | 10 +- deploy/apecloud-mysql-scale/values.yaml | 4 +- .../templates/clusterdefinition.yaml | 8 +- deploy/apecloud-mysql/values.yaml | 2 +- .../templates/clusterdefinition.yaml | 8 +- deploy/clickhouse/values.yaml | 2 +- deploy/etcd/templates/clusterdefinition.yaml | 2 +- deploy/etcd/values.yaml | 2 +- ...apps.kubeblocks.io_clusterdefinitions.yaml | 14 +- .../mongodb-replicaset-overview.json | 5684 +++++++++++++++++ deploy/helm/values.yaml | 77 + deploy/mongodb-cluster/Chart.yaml | 2 +- deploy/mongodb-cluster/templates/cluster.yaml | 8 +- .../mongodb-cluster/templates/replicaset.yaml | 47 + deploy/mongodb-cluster/values.yaml | 53 +- deploy/mongodb/Chart.yaml | 2 +- deploy/mongodb/config/keyfile.tpl | 1 + deploy/mongodb/config/mongodb5.0-config.tpl | 63 + .../mongodb/scripts/replicaset-post-start.tpl | 52 + deploy/mongodb/scripts/replicaset-restore.tpl | 28 + deploy/mongodb/scripts/replicaset-setup.tpl | 20 + deploy/mongodb/templates/_helpers.tpl | 26 + .../templates/backuppolicytemplate.yaml | 15 + .../mongodb/templates/clusterdefinition.yaml | 130 +- deploy/mongodb/templates/clusterversion.yaml | 8 +- deploy/mongodb/templates/configmap.yaml | 75 - deploy/mongodb/templates/configtemplate.yaml | 11 + .../mongodb/templates/metrics-configmap.yaml | 8 + deploy/mongodb/templates/scriptstemplate.yaml | 41 + deploy/mongodb/values.yaml | 77 +- .../templates/clusterdefinition.yaml | 2 +- .../high-availability/high-availability.md | 6 +- internal/constant/const.go | 2 +- internal/controller/builder/builder.go | 13 +- internal/controller/builder/builder_test.go | 4 + internal/controller/component/probe_utils.go | 37 +- .../controller/component/probe_utils_test.go | 12 +- internal/testutil/apps/constant.go | 21 +- 47 files changed, 6506 insertions(+), 220 deletions(-) create mode 100644 deploy/helm/dashboards/mongodb-replicaset-overview.json create mode 100644 deploy/mongodb-cluster/templates/replicaset.yaml create mode 100644 deploy/mongodb/config/keyfile.tpl create mode 100644 deploy/mongodb/config/mongodb5.0-config.tpl create mode 100644 deploy/mongodb/scripts/replicaset-post-start.tpl create mode 100644 deploy/mongodb/scripts/replicaset-restore.tpl create mode 100644 deploy/mongodb/scripts/replicaset-setup.tpl create mode 100644 deploy/mongodb/templates/backuppolicytemplate.yaml delete mode 100644 deploy/mongodb/templates/configmap.yaml create mode 100644 deploy/mongodb/templates/configtemplate.yaml create mode 100644 deploy/mongodb/templates/metrics-configmap.yaml create mode 100644 deploy/mongodb/templates/scriptstemplate.yaml diff --git a/apis/apps/v1alpha1/clusterdefinition_types.go b/apis/apps/v1alpha1/clusterdefinition_types.go index 630f0eda1..ae412fdaa 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types.go +++ b/apis/apps/v1alpha1/clusterdefinition_types.go @@ -49,6 +49,8 @@ type ClusterDefinitionSpec struct { // `$(UUID_B64)` - generate a random UUID v4 BASE64 encoded string``. // `$(UUID_STR_B64)` - generate a random UUID v4 string then BASE64 encoded``. // `$(UUID_HEX)` - generate a random UUID v4 wth HEX representation``. + // `$(HEADLESS_SVC_FQDN)` - headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, + // where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute; // `$(SVC_FQDN)` - service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME).$(NAMESPACE).svc, // where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute; // `$(SVC_PORT_)` - a ServicePort's port value with specified port name, i.e, a servicePort JSON struct: @@ -493,13 +495,13 @@ type ClusterDefinitionProbes struct { // Probe for DB role changed check. // +optional - RoleChangedProbe *ClusterDefinitionProbe `json:"roleChangedProbe,omitempty"` + RoleProbe *ClusterDefinitionProbe `json:"roleProbe,omitempty"` // roleProbeTimeoutAfterPodsReady(in seconds), when all pods of the component are ready, // it will detect whether the application is available in the pod. // if pods exceed the InitializationTimeoutSeconds time without a role label, // this component will enter the Failed/Abnormal phase. - // Note that this configuration will only take effect if the component supports RoleChangedProbe + // Note that this configuration will only take effect if the component supports RoleProbe // and will not affect the life cycle of the pod. default values are 60 seconds. // +optional // +kubebuilder:validation:Minimum=30 diff --git a/apis/apps/v1alpha1/clusterdefinition_webhook.go b/apis/apps/v1alpha1/clusterdefinition_webhook.go index 2f0f1a662..b1bcccca8 100644 --- a/apis/apps/v1alpha1/clusterdefinition_webhook.go +++ b/apis/apps/v1alpha1/clusterdefinition_webhook.go @@ -57,13 +57,13 @@ func (r *ClusterDefinition) Default() { if probes == nil { continue } - if probes.RoleChangedProbe != nil { + if probes.RoleProbe != nil { // set default values if probes.RoleProbeTimeoutAfterPodsReady == 0 { probes.RoleProbeTimeoutAfterPodsReady = DefaultRoleProbeTimeoutAfterPodsReady } } else { - // if component does not support RoleChangedProbe, reset RoleProbeTimeoutAtPodsReady to zero + // if component does not support RoleProbe, reset RoleProbeTimeoutAtPodsReady to zero if probes.RoleProbeTimeoutAfterPodsReady != 0 { probes.RoleProbeTimeoutAfterPodsReady = 0 } diff --git a/apis/apps/v1alpha1/clusterdefinition_webhook_test.go b/apis/apps/v1alpha1/clusterdefinition_webhook_test.go index 9296eaee9..2007faafe 100644 --- a/apis/apps/v1alpha1/clusterdefinition_webhook_test.go +++ b/apis/apps/v1alpha1/clusterdefinition_webhook_test.go @@ -281,9 +281,9 @@ var _ = Describe("clusterDefinition webhook", func() { It("test mutating webhook", func() { clusterDef, _ := createTestClusterDefinitionObj3(clusterDefinitionName + "-mutating") - By("test set the default value to RoleProbeTimeoutAfterPodsReady when roleChangedProbe is not nil") + By("test set the default value to RoleProbeTimeoutAfterPodsReady when roleProbe is not nil") clusterDef.Spec.ComponentDefs[0].Probes = &ClusterDefinitionProbes{ - RoleChangedProbe: &ClusterDefinitionProbe{}, + RoleProbe: &ClusterDefinitionProbe{}, } Expect(testCtx.CreateObj(ctx, clusterDef)).Should(Succeed()) Eventually(func(g Gomega) int32 { @@ -291,7 +291,7 @@ var _ = Describe("clusterDefinition webhook", func() { return clusterDef.Spec.ComponentDefs[0].Probes.RoleProbeTimeoutAfterPodsReady }).Should(Equal(DefaultRoleProbeTimeoutAfterPodsReady)) - By("test set zero to RoleProbeTimeoutAfterPodsReady when roleChangedProbe is nil") + By("test set zero to RoleProbeTimeoutAfterPodsReady when roleProbe is nil") clusterDef.Spec.ComponentDefs[0].Probes = &ClusterDefinitionProbes{ RoleProbeTimeoutAfterPodsReady: 60, } diff --git a/cmd/probe/internal/binding/mongodb/mongodb.go b/cmd/probe/internal/binding/mongodb/mongodb.go index 79a992162..e09af94de 100644 --- a/cmd/probe/internal/binding/mongodb/mongodb.go +++ b/cmd/probe/internal/binding/mongodb/mongodb.go @@ -25,6 +25,7 @@ import ( "github.com/dapr/components-contrib/bindings" "github.com/dapr/kit/logger" + "github.com/spf13/viper" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" @@ -34,14 +35,13 @@ import ( . "github.com/apecloud/kubeblocks/cmd/probe/util" ) -// MongoDB is a binding implementation for MongoDB. -type MongoDB struct { +// MongoDBOperations is a binding implementation for MongoDB. +type MongoDBOperations struct { mongoDBMetadata mu sync.Mutex client *mongo.Client database *mongo.Database operationTimeout time.Duration - logger logger.Logger BaseOperations } @@ -105,7 +105,7 @@ const ( defaultTimeout = 5 * time.Second - defaultDBPort = 27018 + defaultDBPort = 27017 // mongodb://:/ connectionURIFormatWithAuthentication = "mongodb://%s:%s@%s/%s%s" @@ -124,62 +124,62 @@ const ( // NewMongoDB returns a new MongoDB Binding func NewMongoDB(logger logger.Logger) bindings.OutputBinding { - return &MongoDB{BaseOperations: BaseOperations{Logger: logger}} + return &MongoDBOperations{BaseOperations: BaseOperations{Logger: logger}} } // Init initializes the MongoDB Binding. -func (m *MongoDB) Init(metadata bindings.Metadata) error { - m.Logger.Debug("Initializing MongoDB binding") - m.BaseOperations.Init(metadata) +func (mongoOps *MongoDBOperations) Init(metadata bindings.Metadata) error { + mongoOps.Logger.Debug("Initializing MongoDB binding") + mongoOps.BaseOperations.Init(metadata) meta, err := getMongoDBMetaData(metadata) if err != nil { return err } - m.mongoDBMetadata = *meta + mongoOps.mongoDBMetadata = *meta - m.DBType = "mongodb" - m.InitIfNeed = m.initIfNeed - m.DBPort = m.GetRunningPort() - m.OperationMap[GetRoleOperation] = m.GetRoleOps + mongoOps.DBType = "mongodb" + mongoOps.InitIfNeed = mongoOps.initIfNeed + mongoOps.DBPort = mongoOps.GetRunningPort() + mongoOps.BaseOperations.GetRole = mongoOps.GetRole + mongoOps.OperationMap[GetRoleOperation] = mongoOps.GetRoleOps return nil } -func (m *MongoDB) Ping() error { - if err := m.client.Ping(context.Background(), nil); err != nil { - return fmt.Errorf("mongoDB store: error connecting to mongoDB at %s: %s", m.mongoDBMetadata.host, err) +func (mongoOps *MongoDBOperations) Ping() error { + if err := mongoOps.client.Ping(context.Background(), nil); err != nil { + return fmt.Errorf("mongoDB binding: error connecting to mongoDB at %s: %s", mongoOps.mongoDBMetadata.host, err) } return nil } -// InitIfNeed do the real init -func (m *MongoDB) initIfNeed() bool { - if m.database == nil { +func (mongoOps *MongoDBOperations) initIfNeed() bool { + if mongoOps.database == nil { go func() { - err := m.InitDelay() - m.Logger.Errorf("MongoDB connection init failed: %v", err) + err := mongoOps.InitDelay() + mongoOps.Logger.Errorf("MongoDB connection init failed: %v", err) }() return true } return false } -func (m *MongoDB) InitDelay() error { - m.mu.Lock() - defer m.mu.Unlock() - if m.database != nil { +func (mongoOps *MongoDBOperations) InitDelay() error { + mongoOps.mu.Lock() + defer mongoOps.mu.Unlock() + if mongoOps.database != nil { return nil } - m.operationTimeout = m.mongoDBMetadata.operationTimeout + mongoOps.operationTimeout = mongoOps.mongoDBMetadata.operationTimeout - client, err := getMongoDBClient(&m.mongoDBMetadata) + client, err := getMongoDBClient(&mongoOps.mongoDBMetadata) if err != nil { - m.Logger.Errorf("error in creating mongodb client: %s", err) + mongoOps.Logger.Errorf("error in creating mongodb client: %s", err) return err } if err = client.Ping(context.Background(), nil); err != nil { _ = client.Disconnect(context.Background()) - m.Logger.Errorf("error in connecting to mongodb, host: %s error: %s", m.mongoDBMetadata.host, err) + mongoOps.Logger.Errorf("error in connecting to mongodb, host: %s error: %s", mongoOps.mongoDBMetadata.host, err) return err } @@ -187,18 +187,22 @@ func (m *MongoDB) InitDelay() error { _, err = getReplSetStatus(context.Background(), db) if err != nil { _ = client.Disconnect(context.Background()) - m.Logger.Errorf("error in getting repl status from mongodb, error: %s", err) + mongoOps.Logger.Errorf("error in getting repl status from mongodb, error: %s", err) return err } - m.client = client - m.database = db + mongoOps.client = client + mongoOps.database = db return nil } -func (m *MongoDB) GetRunningPort() int { - uri := getMongoURI(&m.mongoDBMetadata) +func (mongoOps *MongoDBOperations) GetRunningPort() int { + if viper.IsSet("KB_SERVICE_PORT") { + return viper.GetInt("KB_SERVICE_PORT") + } + + uri := getMongoURI(&mongoOps.mongoDBMetadata) index := strings.Index(uri, "://") if index < 0 { return defaultDBPort @@ -213,12 +217,17 @@ func (m *MongoDB) GetRunningPort() int { if index < 0 { return defaultDBPort } - uri = uri[:index] + uri = uri[index:] index = strings.Index(uri, ":") if index < 0 { return defaultDBPort } - port, err := strconv.Atoi(uri[index+1:]) + portStr := uri[index+1:] + if viper.IsSet("KB_SERVICE_PORT") { + portStr = viper.GetString("KB_SERVICE_PORT") + } + + port, err := strconv.Atoi(portStr) if err != nil { return defaultDBPort } @@ -226,10 +235,10 @@ func (m *MongoDB) GetRunningPort() int { return port } -func (m *MongoDB) GetRole(ctx context.Context, request *bindings.InvokeRequest, response *bindings.InvokeResponse) (string, error) { - status, err := getReplSetStatus(ctx, m.database) +func (mongoOps *MongoDBOperations) GetRole(ctx context.Context, request *bindings.InvokeRequest, response *bindings.InvokeResponse) (string, error) { + status, err := getReplSetStatus(ctx, mongoOps.database) if err != nil { - m.Logger.Errorf("rs.status() error: %", err) + mongoOps.Logger.Errorf("rs.status() error: %", err) return "", err } for _, member := range status.Members { @@ -240,8 +249,8 @@ func (m *MongoDB) GetRole(ctx context.Context, request *bindings.InvokeRequest, return "", errors.New("role not found") } -func (m *MongoDB) GetRoleOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { - role, err := m.GetRole(ctx, req, resp) +func (mongoOps *MongoDBOperations) GetRoleOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { + role, err := mongoOps.GetRole(ctx, req, resp) if err != nil { return nil, err } @@ -250,7 +259,7 @@ func (m *MongoDB) GetRoleOps(ctx context.Context, req *bindings.InvokeRequest, r return opsRes, nil } -func (m *MongoDB) StatusCheck(ctx context.Context, cmd string, response *bindings.InvokeResponse) (OpsResult, error) { +func (mongoOps *MongoDBOperations) StatusCheck(ctx context.Context, cmd string, response *bindings.InvokeResponse) (OpsResult, error) { // TODO implement me when proposal is passed // proposal: https://infracreate.feishu.cn/wiki/wikcndch7lMZJneMnRqaTvhQpwb#doxcnOUyQ4Mu0KiUo232dOr5aad return nil, nil @@ -306,6 +315,10 @@ func getMongoDBMetaData(metadata bindings.Metadata) (*mongoDBMetadata, error) { meta.host = val } + if viper.IsSet("KB_SERVICE_PORT") { + meta.host = "localhost:" + viper.GetString("KB_SERVICE_PORT") + } + if val, ok := metadata.Properties[server]; ok && val != "" { meta.server = val } @@ -326,6 +339,14 @@ func getMongoDBMetaData(metadata bindings.Metadata) (*mongoDBMetadata, error) { meta.password = val } + if viper.IsSet("KB_SERVICE_USER") { + meta.username = viper.GetString("KB_SERVICE_USER") + } + + if viper.IsSet("KB_SERVICE_PASSWORD") { + meta.password = viper.GetString("KB_SERVICE_PASSWORD") + } + meta.databaseName = adminDatabase if val, ok := metadata.Properties[databaseName]; ok && val != "" { meta.databaseName = val diff --git a/cmd/probe/internal/binding/mongodb/mongodb_test.go b/cmd/probe/internal/binding/mongodb/mongodb_test.go index c9defa5da..f820f3d74 100644 --- a/cmd/probe/internal/binding/mongodb/mongodb_test.go +++ b/cmd/probe/internal/binding/mongodb/mongodb_test.go @@ -25,6 +25,8 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo/integration/mtest" + + . "github.com/apecloud/kubeblocks/cmd/probe/internal/binding" ) func TestGetMongoDBMetadata(t *testing.T) { @@ -200,9 +202,9 @@ func TestGetRole(t *testing.T) { }, }}, }) - m := &MongoDB{ - database: mt.Client.Database(adminDatabase), - logger: logger.NewLogger("mongodb-test"), + m := &MongoDBOperations{ + database: mt.Client.Database(adminDatabase), + BaseOperations: BaseOperations{Logger: logger.NewLogger("mongodb-test")}, } role, err := m.GetRole(context.Background(), &bindings.InvokeRequest{}, &bindings.InvokeResponse{}) if err != nil { diff --git a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml index 49b645d13..883f51f57 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml @@ -7915,7 +7915,7 @@ spec: probes: description: probes setting for healthy checks. properties: - roleChangedProbe: + roleProbe: description: Probe for DB role changed check. properties: commands: @@ -7962,9 +7962,8 @@ spec: exceed the InitializationTimeoutSeconds time without a role label, this component will enter the Failed/Abnormal phase. Note that this configuration will only take effect - if the component supports RoleChangedProbe and will not - affect the life cycle of the pod. default values are 60 - seconds. + if the component supports RoleProbe and will not affect + the life cycle of the pod. default values are 60 seconds. format: int32 minimum: 30 type: integer @@ -8724,9 +8723,12 @@ spec: - generate a random UUID v4 string. `$(UUID_B64)` - generate a random UUID v4 BASE64 encoded string``. `$(UUID_STR_B64)` - generate a random UUID v4 string then BASE64 encoded``. `$(UUID_HEX)` - generate - a random UUID v4 wth HEX representation``. `$(SVC_FQDN)` - service - FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME).$(NAMESPACE).svc, + a random UUID v4 wth HEX representation``. `$(HEADLESS_SVC_FQDN)` + - headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` + attribute; `$(SVC_FQDN)` - service FQDN placeholder, value pattern + - $(CLUSTER_NAME)-$(1ST_COMP_NAME).$(NAMESPACE).svc, where 1ST_COMP_NAME + is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute; `$(SVC_PORT_)` - a ServicePort''s port value with specified port name, i.e, a servicePort JSON struct: { "name": "mysql", "targetPort": "mysqlContainerPort", "port": 3306 }, and diff --git a/config/probe/components/binding_mongodb.yaml b/config/probe/components/binding_mongodb.yaml index 03290456f..9f686633e 100644 --- a/config/probe/components/binding_mongodb.yaml +++ b/config/probe/components/binding_mongodb.yaml @@ -7,6 +7,6 @@ spec: version: v1 metadata: - name: host - value: "127.0.0.1:27018" + value: "127.0.0.1:27017" - name: params value: "?directConnection=true" diff --git a/controllers/apps/components/util/component_utils.go b/controllers/apps/components/util/component_utils.go index 237d490ca..bbcb3da1f 100644 --- a/controllers/apps/components/util/component_utils.go +++ b/controllers/apps/components/util/component_utils.go @@ -115,7 +115,7 @@ func IsProbeTimeout(componentDef *appsv1alpha1.ClusterComponentDefinition, podsR return false } probes := componentDef.Probes - if probes == nil || probes.RoleChangedProbe == nil { + if probes == nil || probes.RoleProbe == nil { return false } roleProbeTimeout := time.Duration(appsv1alpha1.DefaultRoleProbeTimeoutAfterPodsReady) * time.Second diff --git a/controllers/apps/components/util/component_utils_test.go b/controllers/apps/components/util/component_utils_test.go index 3b49bd6d7..5c25e6126 100644 --- a/controllers/apps/components/util/component_utils_test.go +++ b/controllers/apps/components/util/component_utils_test.go @@ -46,7 +46,7 @@ func TestIsProbeTimeout(t *testing.T) { podsReadyTime := &metav1.Time{Time: time.Now().Add(-10 * time.Minute)} compDef := &appsv1alpha1.ClusterComponentDefinition{ Probes: &appsv1alpha1.ClusterDefinitionProbes{ - RoleChangedProbe: &appsv1alpha1.ClusterDefinitionProbe{}, + RoleProbe: &appsv1alpha1.ClusterDefinitionProbe{}, RoleProbeTimeoutAfterPodsReady: appsv1alpha1.DefaultRoleProbeTimeoutAfterPodsReady, }, } diff --git a/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml b/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml index 66e147036..e853f5973 100644 --- a/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml @@ -20,10 +20,10 @@ spec: - name: mysql characterType: mysql probes: - roleChangedProbe: - failureThreshold: {{ .Values.roleChangedProbe.failureThreshold }} - periodSeconds: {{ .Values.roleChangedProbe.periodSeconds }} - timeoutSeconds: {{ .Values.roleChangedProbe.timeoutSeconds }} + roleProbe: + failureThreshold: {{ .Values.roleProbe.failureThreshold }} + periodSeconds: {{ .Values.roleProbe.periodSeconds }} + timeoutSeconds: {{ .Values.roleProbe.timeoutSeconds }} monitor: builtIn: false exporterConfig: @@ -603,4 +603,4 @@ spec: --service_map 'grpc-vtgateservice' \ --pid_file $VTDATAROOT/vtgate.pid \ --mysql_auth_server_impl none - EOF \ No newline at end of file + EOF diff --git a/deploy/apecloud-mysql-scale/values.yaml b/deploy/apecloud-mysql-scale/values.yaml index 9eab84d14..811db0a32 100644 --- a/deploy/apecloud-mysql-scale/values.yaml +++ b/deploy/apecloud-mysql-scale/values.yaml @@ -61,7 +61,7 @@ logConfigs: slow: /data/mysql/log/mysqld-slowquery.log general: /data/mysql/log/mysqld.log -roleChangedProbe: +roleProbe: failureThreshold: 2 periodSeconds: 1 timeoutSeconds: 1 @@ -79,4 +79,4 @@ wesqlscale: registry: registry.cn-hangzhou.aliyuncs.com repository: apecloud/apecloud-mysql-scale tag: "latest" - pullPolicy: IfNotPresent \ No newline at end of file + pullPolicy: IfNotPresent diff --git a/deploy/apecloud-mysql/templates/clusterdefinition.yaml b/deploy/apecloud-mysql/templates/clusterdefinition.yaml index 92d8fbf67..98f77f5bc 100644 --- a/deploy/apecloud-mysql/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql/templates/clusterdefinition.yaml @@ -16,10 +16,10 @@ spec: - name: mysql characterType: mysql probes: - roleChangedProbe: - failureThreshold: {{ .Values.roleChangedProbe.failureThreshold }} - periodSeconds: {{ .Values.roleChangedProbe.periodSeconds }} - timeoutSeconds: {{ .Values.roleChangedProbe.timeoutSeconds }} + roleProbe: + failureThreshold: {{ .Values.roleProbe.failureThreshold }} + periodSeconds: {{ .Values.roleProbe.periodSeconds }} + timeoutSeconds: {{ .Values.roleProbe.timeoutSeconds }} monitor: builtIn: false exporterConfig: diff --git a/deploy/apecloud-mysql/values.yaml b/deploy/apecloud-mysql/values.yaml index 1e1e28e9b..735d0987d 100644 --- a/deploy/apecloud-mysql/values.yaml +++ b/deploy/apecloud-mysql/values.yaml @@ -61,7 +61,7 @@ logConfigs: slow: /data/mysql/log/mysqld-slowquery.log general: /data/mysql/log/mysqld.log -roleChangedProbe: +roleProbe: failureThreshold: 2 periodSeconds: 1 timeoutSeconds: 1 diff --git a/deploy/clickhouse/templates/clusterdefinition.yaml b/deploy/clickhouse/templates/clusterdefinition.yaml index 4fab723dc..efb208479 100644 --- a/deploy/clickhouse/templates/clusterdefinition.yaml +++ b/deploy/clickhouse/templates/clusterdefinition.yaml @@ -234,11 +234,11 @@ spec: workloadType: Stateful #Consensus characterType: zookeeper # probes: - # roleChangedProbe: + # roleProbe: # cmd: "stat | grep 'Leader'" - # failureThreshold: {{ .Values.zookeeper.roleChangedProbe.failureThreshold }} - # periodSeconds: {{ .Values.zookeeper.roleChangedProbe.periodSeconds }} - # successThreshold: {{ .Values.zookeeper.roleChangedProbe.successThreshold }} + # failureThreshold: {{ .Values.zookeeper.roleProbe.failureThreshold }} + # periodSeconds: {{ .Values.zookeeper.roleProbe.periodSeconds }} + # successThreshold: {{ .Values.zookeeper.roleProbe.successThreshold }} monitor: builtIn: false exporterConfig: diff --git a/deploy/clickhouse/values.yaml b/deploy/clickhouse/values.yaml index c610e4a0d..15f0983d0 100644 --- a/deploy/clickhouse/values.yaml +++ b/deploy/clickhouse/values.yaml @@ -131,7 +131,7 @@ zookeeper: tag: 3.8.0-debian-11-r47 logConfigs: {} - roleChangedProbe: + roleProbe: failureThreshold: 2 periodSeconds: 1 successThreshold: 1 diff --git a/deploy/etcd/templates/clusterdefinition.yaml b/deploy/etcd/templates/clusterdefinition.yaml index 445f6cac9..721d13539 100644 --- a/deploy/etcd/templates/clusterdefinition.yaml +++ b/deploy/etcd/templates/clusterdefinition.yaml @@ -19,7 +19,7 @@ spec: accessMode: ReadWrite updateStrategy: BestEffortParallel probes: - roleChangedProbe: + roleProbe: periodSeconds: 1 failureThreshold: 3 service: diff --git a/deploy/etcd/values.yaml b/deploy/etcd/values.yaml index e55273d44..cd7441ff1 100644 --- a/deploy/etcd/values.yaml +++ b/deploy/etcd/values.yaml @@ -8,7 +8,7 @@ clusterVersionOverride: "" nameOverride: "" fullnameOverride: "" -roleChangedProbe: +roleProbe: failureThreshold: 2 periodSeconds: 1 timeoutSeconds: 1 diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml index 49b645d13..883f51f57 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml @@ -7915,7 +7915,7 @@ spec: probes: description: probes setting for healthy checks. properties: - roleChangedProbe: + roleProbe: description: Probe for DB role changed check. properties: commands: @@ -7962,9 +7962,8 @@ spec: exceed the InitializationTimeoutSeconds time without a role label, this component will enter the Failed/Abnormal phase. Note that this configuration will only take effect - if the component supports RoleChangedProbe and will not - affect the life cycle of the pod. default values are 60 - seconds. + if the component supports RoleProbe and will not affect + the life cycle of the pod. default values are 60 seconds. format: int32 minimum: 30 type: integer @@ -8724,9 +8723,12 @@ spec: - generate a random UUID v4 string. `$(UUID_B64)` - generate a random UUID v4 BASE64 encoded string``. `$(UUID_STR_B64)` - generate a random UUID v4 string then BASE64 encoded``. `$(UUID_HEX)` - generate - a random UUID v4 wth HEX representation``. `$(SVC_FQDN)` - service - FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME).$(NAMESPACE).svc, + a random UUID v4 wth HEX representation``. `$(HEADLESS_SVC_FQDN)` + - headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` + attribute; `$(SVC_FQDN)` - service FQDN placeholder, value pattern + - $(CLUSTER_NAME)-$(1ST_COMP_NAME).$(NAMESPACE).svc, where 1ST_COMP_NAME + is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute; `$(SVC_PORT_)` - a ServicePort''s port value with specified port name, i.e, a servicePort JSON struct: { "name": "mysql", "targetPort": "mysqlContainerPort", "port": 3306 }, and diff --git a/deploy/helm/dashboards/mongodb-replicaset-overview.json b/deploy/helm/dashboards/mongodb-replicaset-overview.json new file mode 100644 index 000000000..6a25fa80d --- /dev/null +++ b/deploy/helm/dashboards/mongodb-replicaset-overview.json @@ -0,0 +1,5684 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dashboard for MongoDB ReplicaSet managed by KubeBlocks", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 7359, + "graphTooltip": 1, + "id": 10, + "links": [ + { + "asDropdown": false, + "icon": "cloud", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "ApeCloud", + "tooltip": "Improved productivity, cost-efficiency and business continuity.", + "type": "link", + "url": "https://kubeblocks.io/" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "KubeBlocks", + "tooltip": "An open-source and cloud-neutral DBaaS with Kubernetes.", + "type": "link", + "url": "https://github.com/apecloud/kubeblocks" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 117, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 119, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "10m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "exemplar": false, + "expr": "count(sum by(namespace)(mongodb_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}))", + "format": "time_series", + "instant": true, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "", + "range": false, + "refId": "A", + "step": 20 + } + ], + "title": "Total Namespaces", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 121, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "10m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "exemplar": false, + "expr": "count(sum by(namespace, app_kubernetes_io_instance)(mongodb_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}))", + "format": "time_series", + "instant": true, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{label_name}}", + "metric": "", + "range": false, + "refId": "A", + "step": 20 + } + ], + "title": "Total Clusters", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 123, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "10m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "exemplar": false, + "expr": "count(mongodb_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"} > 0)", + "format": "time_series", + "instant": true, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "", + "range": false, + "refId": "A", + "step": 20 + } + ], + "title": "Total Ups", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-red", + "value": 0.5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 125, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "10m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "exemplar": false, + "expr": "(count(mongodb_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"} <= 0)) or vector(0)", + "format": "time_series", + "instant": true, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "", + "range": false, + "refId": "A", + "step": 20 + } + ], + "title": "Total Downs", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": false, + "inspect": false + }, + "mappings": [ + { + "options": { + "0": { + "index": 2, + "text": "STARTUP" + }, + "1": { + "index": 1, + "text": "PRIMARY" + }, + "2": { + "index": 0, + "text": "SECONDARY" + }, + "3": { + "index": 3, + "text": "RECOVERING" + }, + "5": { + "index": 4, + "text": "STARTUP2" + }, + "6": { + "index": 5, + "text": "UNKNOWN" + }, + "7": { + "index": 6, + "text": "ARBITER" + }, + "8": { + "index": 7, + "text": "DOWN" + }, + "9": { + "index": 8, + "text": "ROLLBACK" + }, + "10": { + "index": 9, + "text": "REMOVED" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "MongoDBInstance: ${__data.fields.cluster} | ${__data.fields.instance}", + "url": "/d/pMEd7m0Mz/cadvisor-exporter?orgId=1&var-node=All&var-namespace=${__data.fields.namespace}&var-pod=${__data.fields.instance}&var-container=All" + } + ] + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "custom.filterable", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 60 + }, + { + "color": "dark-green", + "value": 120 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "lcd-gauge" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "namespace" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cluster" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 130, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "footer": { + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "instance" + } + ] + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "10m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "exemplar": false, + "expr": "sum by(namespace, app_kubernetes_io_instance, pod) (mongodb_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})", + "format": "table", + "instant": true, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "", + "range": false, + "refId": "A", + "step": 20 + }, + { + "calculatedInterval": "10m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "exemplar": false, + "expr": "sum by(namespace, app_kubernetes_io_instance, pod) (mongodb_instance_uptime_seconds{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "__auto", + "metric": "", + "range": false, + "refId": "B", + "step": 20 + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(namespace, app_kubernetes_io_instance, pod) (mongodb_mongod_replset_my_state{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + } + ], + "title": "Instance Resource", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "pod", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Value #A": true, + "Value #B": false, + "app_kubernetes_io_instance 2": true, + "app_kubernetes_io_instance 3": true, + "app_kubernetes_io_instance 4": true, + "namespace 2": true, + "namespace 3": true, + "namespace 4": true + }, + "indexByName": { + "Time 1": 3, + "Time 2": 5, + "Time 3": 10, + "Value #A": 4, + "Value #B": 9, + "Value #C": 8, + "app_kubernetes_io_instance 1": 1, + "app_kubernetes_io_instance 2": 6, + "app_kubernetes_io_instance 3": 11, + "namespace 1": 0, + "namespace 2": 7, + "namespace 3": 12, + "pod": 2 + }, + "renameByName": { + "Value #B": "uptime", + "Value #C": "ReplSet State", + "app_kubernetes_io_instance 1": "cluster", + "namespace 1": "namespace", + "pod": "instance" + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [], + "fields": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": false, + "inspect": false + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "instances" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 127, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "10m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "exemplar": false, + "expr": "count by(mongodb)(mongodb_version_info{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})", + "format": "table", + "instant": true, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "metric": "", + "range": false, + "refId": "A", + "step": 20 + } + ], + "title": "Cluster Versions", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": { + "Value": "instances", + "short_version": "version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "displayMode": "auto", + "filterable": false, + "inspect": false + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "instances" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 128, + "interval": "1m", + "links": [], + "maxDataPoints": 100, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "10m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "exemplar": false, + "expr": "count by(engine)(mongodb_mongod_storage_engine{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})", + "format": "table", + "instant": true, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{short_version}}", + "metric": "", + "range": false, + "refId": "A", + "step": 20 + } + ], + "title": "Cluster Engines", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "indexByName": {}, + "renameByName": { + "Value": "instances", + "short_version": "version" + } + } + } + ], + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 155, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by(namespace, app_kubernetes_io_instance, pod) (mongodb_mongod_replset_member_replication_lag{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "ReplSet Lag", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 156, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "time() - mongodb_mongod_replset_member_election_date{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "ReplSet Last Election", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 104, + "panels": [], + "title": "Memory & Network", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "mbytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + }, + "hiddenSeries": false, + "id": 75, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "mongodb_memory{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\", type=~\"resident|virtual\"}", + "hide": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}-{{type}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "MongoDB Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "mbytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 19 + }, + "hiddenSeries": false, + "id": 105, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_network_metrics_num_requests_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval])", + "hide": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "MongoDB NetworkRequests", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 131, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_ss_network_bytesOut{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_ss_network_bytesOut{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "hide": false, + "interval": "$__rate_interval", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "MongoDB NetWorkOut", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 132, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_ss_network_bytesIn{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_ss_network_bytesIn{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "hide": false, + "interval": "$__rate_interval", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "MongoDB NetworkIn", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 93, + "panels": [], + "title": "Operations", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 34 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_op_counters_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_op_counters_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - {{type}}", + "metric": "", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Query Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 34 + }, + "hiddenSeries": false, + "id": 133, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_op_counters_repl_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_op_counters_repl_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - {{type}}", + "metric": "", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Replica Query Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_metrics_document_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - {{state}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Document Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 10, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 41 + }, + "hiddenSeries": false, + "id": 163, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_metrics_query_executor_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_metrics_query_executor_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - {{state}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Document Query Executor", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 10, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 134, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_op_latencies_ops_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_op_latencies_ops_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - {{type}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "OpLatencies Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 10, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 48 + }, + "hiddenSeries": false, + "id": 135, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_op_latencies_latency_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{type}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "OpLatencies Latency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "ms", + "label": "", + "logBase": 10, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "mongodb_mongod_metrics_ttl_deleted_documents_total", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 55 + }, + "hiddenSeries": false, + "id": 160, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_metrics_ttl_deleted_documents_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_metrics_ttl_deleted_documents_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TTL Delete Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "short", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 10, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 137, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 81, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_connections{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\", state=~\"(current|available|totalCreated)\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - {{state}}", + "metric": "mongodb_mongod_metrics_repl_preload_indexes_num_total", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Collections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 138, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_mongod_metrics_cursor_open{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - {{state}}", + "metric": "mongodb_mongod_metrics_repl_preload_indexes_num_total", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cursors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Connections & Cursors", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 97, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 80 + }, + "hiddenSeries": false, + "id": 98, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_dbstats_dataSize{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\", database=~\"$database\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{database}}", + "metric": "mongodb_mongod_metrics_repl_preload_indexes_num_total", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "DataBase Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 80 + }, + "hiddenSeries": false, + "id": 101, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_dbstats_indexSize{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\", database=~\"$database\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{database}}", + "metric": "mongodb_mongod_metrics_repl_preload_indexes_num_total", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Index Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 87 + }, + "hiddenSeries": false, + "id": 99, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_dbstats_storageSize{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\", database=~\"$database\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{database}}", + "metric": "mongodb_mongod_metrics_repl_preload_indexes_num_total", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Storage Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 87 + }, + "hiddenSeries": false, + "id": 102, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_dbstats_totalSize{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\", database=~\"$database\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{database}}", + "metric": "mongodb_mongod_metrics_repl_preload_indexes_num_total", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 100, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_dbstats_indexes{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\", database=~\"$database\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{database}}", + "metric": "mongodb_mongod_metrics_repl_preload_indexes_num_total", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Indexes Num", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 94 + }, + "hiddenSeries": false, + "id": 139, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_dbstats_collections{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\", database=~\"$database\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{database}}", + "metric": "mongodb_mongod_metrics_repl_preload_indexes_num_total", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Collections Num", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "DB Stat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 95, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_oplog_stats_size{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oplog Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 162, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_replset_oplog_head_timestamp{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_replset_oplog_head_timestamp{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oplog Lag", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 80, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_mongod_metrics_repl_buffer_size_bytes{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - Used", + "range": true, + "refId": "A", + "step": 300 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_mongod_metrics_repl_buffer_max_size_bytes{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} - Max", + "range": true, + "refId": "B", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oplog Buffer Capacity", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 10, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 30 + }, + "hiddenSeries": false, + "id": 161, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_metrics_repl_buffer_count{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_metrics_repl_buffer_count{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oplog Buffered Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 37 + }, + "hiddenSeries": false, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_replset_oplog_head_timestamp{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_metrics_repl_preload_docs_total_milliseconds{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oplog Processing Time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ms", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "ms" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 37 + }, + "hiddenSeries": false, + "id": 79, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_metrics_repl_network_getmores_total_milliseconds{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_metrics_repl_network_getmores_total_milliseconds{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oplog Getmore Time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ms", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Shows the time range in the oplog and the oldest backed up operation.", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 44 + }, + "hiddenSeries": false, + "id": 165, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time()-mongodb_mongod_replset_oplog_tail_timestamp{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | Now to End", + "metric": "", + "range": true, + "refId": "J", + "step": 300 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_mongod_replset_oplog_head_timestamp{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}-mongodb_mongod_replset_oplog_tail_timestamp{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | Oplog Range", + "metric": "", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Oplog Recovery Window", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Oplog", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 107, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 110, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_asserts_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_asserts_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{type}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Assert Events", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Assert", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 141, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 41 + }, + "hiddenSeries": false, + "id": 112, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_ss_globalLock_activeClients_readers{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_ss_globalLock_activeClients_readers{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | readers", + "range": true, + "refId": "A", + "step": 300 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_ss_globalLock_activeClients_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_ss_globalLock_activeClients_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "hide": false, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | total", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_ss_globalLock_activeClients_writers{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_ss_globalLock_activeClients_writers{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "hide": false, + "interval": "", + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | writers", + "range": true, + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Active Clients", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 41 + }, + "hiddenSeries": false, + "id": 144, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_ss_globalLock_currentQueue{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_ss_globalLock_currentQueue{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{count_type}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Current Queue", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "GlobalLock", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 143, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is useful for write-heavy workloads to understand how long it takes to verify writes and how many concurrent writes are occurring.", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 146, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_metrics_get_last_error_wtime_total_milliseconds{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_metrics_get_last_error_wtime_total_milliseconds{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} ", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Write Time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "This is useful for write-heavy workloads to understand how long it takes to verify writes and how many concurrent writes are occurring.", + "editable": true, + "error": false, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 42 + }, + "hiddenSeries": false, + "id": 159, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_metrics_get_last_error_wtimeouts_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_metrics_get_last_error_wtimeouts_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | Timeouts", + "range": true, + "refId": "A", + "step": 300 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_metrics_get_last_error_wtime_num_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_metrics_get_last_error_wtime_num_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "hide": false, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | Total", + "range": true, + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Write Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "GetLastError", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 148, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 43 + }, + "hiddenSeries": false, + "id": 113, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "mongodb_mongod_wiredtiger_cache_bytes{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cache UsedSize", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 43 + }, + "hiddenSeries": false, + "id": 115, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_wiredtiger_cache_bytes_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_wiredtiger_cache_bytes_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{type}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cache R/W Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 50 + }, + "hiddenSeries": false, + "id": 153, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate (mongodb_mongod_wiredtiger_cache_evicted_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cache Evicted Page Num", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 50 + }, + "hiddenSeries": false, + "id": 149, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_mongod_wiredtiger_log_operations_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_mongod_wiredtiger_log_operations_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{type}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Log Operations", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "WiredTiger", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 151, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "fill": 2, + "fillGradient": 0, + "grid": { + "leftLogBase": 1, + "leftMin": 0, + "rightLogBase": 1 + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 44 + }, + "hiddenSeries": false, + "id": 152, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(mongodb_extra_info_page_faults_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[$__rate_interval]) or irate(mongodb_extra_info_page_faults_total{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\", pod=~\"$instance\"}[5m])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} | {{type}}", + "range": true, + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Page Faults", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "x-axis": true, + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "yaxes": [ + { + "format": "ops", + "label": "", + "logBase": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "title": "Extra Info", + "type": "row" + } + ], + "refresh": "1m", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "MongoDB", + "KubeBlocks" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(mongodb_mongod_replset_my_state, namespace)", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(mongodb_mongod_replset_my_state, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(mongodb_mongod_replset_my_state, app_kubernetes_io_instance)", + "hide": 0, + "includeAll": true, + "label": "Cluster", + "multi": true, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(mongodb_mongod_replset_my_state, app_kubernetes_io_instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(mongodb_mongod_replset_my_state{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}, pod)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(mongodb_mongod_replset_my_state{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\"}, pod)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(mongodb_dbstats_ok{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}, database)", + "hide": 0, + "includeAll": true, + "label": "Database", + "multi": true, + "name": "database", + "options": [], + "query": { + "query": "label_values(mongodb_dbstats_ok{namespace=~\"$namespace\",app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}, database)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "hidden": false, + "now": true, + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "MongoDB-ReplSet-KubeBlocks", + "uid": "7lzrQGNikKB", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 8f8c05fd7..df02cc1b0 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1075,6 +1075,82 @@ prometheus: summary: 'Redis replication broken' description: 'Redis instance lost a slave. (instance: {{ $labels.pod }})' + mongodb_replicaset_alert_rules.yaml: |- + groups: + - name: MongodbExporter + rules: + - alert: MongodbDown + expr: 'max_over_time(mongodb_up[1m]) == 0' + for: 0m + labels: + severity: critical + annotations: + summary: 'MongoDB is Down' + description: 'MongoDB instance is down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' + + - alert: MongodbReplicaMemberUnhealthy + expr: 'max_over_time(mongodb_rs_members_health[1m]) == 0' + for: 0m + labels: + severity: critical + annotations: + summary: 'Mongodb replica member is unhealthy' + description: 'MongoDB replica member is not healthy\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' + + - alert: MongodbReplicationLag + expr: '(mongodb_rs_members_optimeDate{member_state="PRIMARY"} - on (pod) group_right mongodb_rs_members_optimeDate{member_state="SECONDARY"}) / 1000 > 10' + for: 0m + labels: + severity: critical + annotations: + summary: 'MongoDB replication lag (> 10s)' + description: 'Mongodb replication lag is more than 10s\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' + + - alert: MongodbReplicationHeadroom + expr: 'sum(avg(mongodb_mongod_replset_oplog_head_timestamp - mongodb_mongod_replset_oplog_tail_timestamp)) - sum(avg(mongodb_rs_members_optimeDate{member_state="PRIMARY"} - on (pod) group_right mongodb_rs_members_optimeDate{member_state="SECONDARY"})) <= 0' + for: 0m + labels: + severity: critical + annotations: + summary: 'MongoDB replication headroom (< 0)' + description: 'MongoDB replication headroom is <= 0\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' + + - alert: MongodbNumberCursorsOpen + expr: 'mongodb_ss_metrics_cursor_open{csr_type="total"} > 10 * 1000' + for: 2m + labels: + severity: warning + annotations: + summary: 'MongoDB opened cursors num (> 10k)' + description: 'Too many cursors opened by MongoDB for clients (> 10k)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' + + - alert: MongodbCursorsTimeouts + expr: 'increase(mongodb_ss_metrics_cursor_timedOut[1m]) > 100' + for: 2m + labels: + severity: warning + annotations: + summary: 'MongoDB cursors timeouts (>100/minute)' + description: 'Too many cursors are timing out (> 100/minute)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' + + - alert: MongodbTooManyConnections + expr: 'avg by(pod) (rate(mongodb_ss_connections{conn_type="current"}[1m])) / avg by(pod) (sum (mongodb_ss_connections) by(pod)) * 100 > 80' + for: 2m + labels: + severity: warning + annotations: + summary: 'MongoDB too many connections (> 80%)' + description: 'Too many connections (> 80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' + + - alert: MongodbVirtualMemoryUsage + expr: '(sum(mongodb_ss_mem_virtual) BY (pod) / sum(mongodb_ss_mem_resident) BY (pod)) > 100' + for: 2m + labels: + severity: warning + annotations: + summary: MongoDB virtual memory usage high + description: "High memory usage: the quotient of (mem_virtual / mem_resident) is more than 100\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + kafka_alert_rules.yaml: |- group: - name: KafkaExporter @@ -1114,6 +1190,7 @@ prometheus: - /etc/config/postgresql_alert_rules.yml - /etc/config/redis_alert_rules.yml - /etc/config/kafka_alert_rules.yml + - /etc/config/mongodb_replicaset_alert_rules.yaml scrape_configs: - job_name: prometheus diff --git a/deploy/mongodb-cluster/Chart.yaml b/deploy/mongodb-cluster/Chart.yaml index 2a6b451d4..432ff137f 100644 --- a/deploy/mongodb-cluster/Chart.yaml +++ b/deploy/mongodb-cluster/Chart.yaml @@ -6,7 +6,7 @@ type: application version: 0.5.0-alpha.3 -appVersion: "6.0.3" +appVersion: "5.0.14" home: https://www.mongodb.com icon: https://bitnami.com/assets/stacks/mongodb/img/mongodb-stack-220x234.png diff --git a/deploy/mongodb-cluster/templates/cluster.yaml b/deploy/mongodb-cluster/templates/cluster.yaml index 4032cfc9d..b2097c645 100644 --- a/deploy/mongodb-cluster/templates/cluster.yaml +++ b/deploy/mongodb-cluster/templates/cluster.yaml @@ -1,7 +1,8 @@ +{{- if eq .Values.architecture "sharding" }} apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ .Release.Name }}-sharding labels: {{- include "mongodb-cluster.labels" . | nindent 4}} spec: @@ -21,7 +22,6 @@ spec: - name: shard-{{ $i }} componentDefRef: shard monitor: {{ $.Values.monitor.enabled }} - serviceType: {{ $.Values.service.type | default "ClusterIP" }} replicas: {{ .replicas | default "3" }} {{- with .tolerations }} tolerations: {{ .| toYaml | nindent 8 }} @@ -81,7 +81,6 @@ spec: - name: mongos-{{ $i }} componentDefRef: mongos monitor: {{ $.Values.monitor.enabled }} - serviceType: {{ $.Values.service.type | default "ClusterIP" }} replicas: {{ .replicas | default 1 }} {{- with .tolerations }} tolerations: {{ .| toYaml | nindent 8 }} @@ -97,4 +96,5 @@ spec: {{- end }} {{- $i := add1 $i }} {{- end }} - {{- end }} \ No newline at end of file + {{- end }} +{{- end }} diff --git a/deploy/mongodb-cluster/templates/replicaset.yaml b/deploy/mongodb-cluster/templates/replicaset.yaml new file mode 100644 index 000000000..5389319c6 --- /dev/null +++ b/deploy/mongodb-cluster/templates/replicaset.yaml @@ -0,0 +1,47 @@ +{{- if eq .Values.architecture "replicaset" }} +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + name: {{ .Release.Name }} + labels: + {{- include "mongodb-cluster.labels" . | nindent 4}} +spec: + clusterDefinitionRef: mongodb + clusterVersionRef: mongodb-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} + terminationPolicy: {{ .Values.terminationPolicy }} + affinity: + {{- with $.Values.topologyKeys }} + topologyKeys: {{ . | toYaml | nindent 6 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: {{ . | toYaml | nindent 4 }} + {{- end }} + componentSpecs: + - name: replicaset + componentDefRef: replicaset + monitor: {{ $.Values.monitor.enabled }} + replicas: {{ $.Values.replicaset.replicas }} + {{- with $.Values.replicaset.tolerations }} + tolerations: {{ .| toYaml | nindent 8 }} + {{- end }} + {{- with $.Values.replicaset.resources }} + resources: + limits: + cpu: {{ .limits.cpu | quote }} + memory: {{ .limits.memory | quote }} + requests: + cpu: {{ .requests.cpu | quote }} + memory: {{ .requests.memory | quote }} + {{- end }} + {{- if $.Values.replicaset.persistence.enabled }} + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + storageClassName: {{ $.Values.replicaset.persistence.data.storageClassName }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ $.Values.replicaset.persistence.data.size }} + {{- end }} +{{- end }} diff --git a/deploy/mongodb-cluster/values.yaml b/deploy/mongodb-cluster/values.yaml index 2e508ddaf..8b0007ece 100644 --- a/deploy/mongodb-cluster/values.yaml +++ b/deploy/mongodb-cluster/values.yaml @@ -1,3 +1,5 @@ +## @param architecture define MongoDB cluster topology architecture ( `replicaset` or `sharding`) +architecture: replicaset ## @param terminationPolicy define Cluster termination policy. One of DoNotTerminate, Halt, Delete, WipeOut. ## @@ -85,6 +87,52 @@ shard: ## tolerations: [ ] +replicaset: + ## @param replicaset.replicas Number of MongoDB replicas per replicaset to deploy + ## + replicas: 3 + ## MongoDB workload pod resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## @param replicaset.resources.limits The resources limits for the init container + ## @param replicaset.resources.requests The requested resources for the init container + ## + resources: { } + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + ## Enable persistence using Persistent Volume Claims + ## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/ + ## + persistence: + ## @param replicaset.persistence.enabled Enable persistence using Persistent Volume Claims + ## + enabled: true + ## `data` volume settings + ## + data: + ## @param replicaset.persistence.data.storageClassName Storage class of backing PVC + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + storageClassName: + ## @param replicaset.persistence.size Size of data volume + ## + size: 20Gi + ## @param replicaset.tolerations Tolerations for MongoDB pods assignment + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [ ] + configsvr: ## @param configsvr.replicas Number of MongoDB replicas per configsvr to deploy ## @@ -259,4 +307,7 @@ ingress: ## port: ## name: http ## - extraRules: [] \ No newline at end of file + extraRules: [] + +enabledLogs: + - running diff --git a/deploy/mongodb/Chart.yaml b/deploy/mongodb/Chart.yaml index 818006870..dac8a8181 100644 --- a/deploy/mongodb/Chart.yaml +++ b/deploy/mongodb/Chart.yaml @@ -6,7 +6,7 @@ type: application version: 0.5.0-alpha.3 -appVersion: "6.0.3" +appVersion: "5.0.14" home: https://www.mongodb.com icon: https://bitnami.com/assets/stacks/mongodb/img/mongodb-stack-220x234.png diff --git a/deploy/mongodb/config/keyfile.tpl b/deploy/mongodb/config/keyfile.tpl new file mode 100644 index 000000000..a6010f73a --- /dev/null +++ b/deploy/mongodb/config/keyfile.tpl @@ -0,0 +1 @@ +{{ randAscii 64 | b64enc }} diff --git a/deploy/mongodb/config/mongodb5.0-config.tpl b/deploy/mongodb/config/mongodb5.0-config.tpl new file mode 100644 index 000000000..bfc56e5a3 --- /dev/null +++ b/deploy/mongodb/config/mongodb5.0-config.tpl @@ -0,0 +1,63 @@ +# mongod.conf +# for documentation of all options, see: +# http://docs.mongodb.org/manual/reference/configuration-options/ + +{{- $log_root := getVolumePathByName ( index $.podSpec.containers 0 ) "log" }} +{{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} +{{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} +{{- $phy_memory := getContainerMemory ( index $.podSpec.containers 0 ) }} + +# require port +{{- $mongodb_port := 27017 }} +{{- if $mongodb_port_info }} +{{- $mongodb_port = $mongodb_port_info.containerPort }} +{{- end }} + +# where and how to store data. +storage: + dbPath: {{ $mongodb_root }}/db + journal: + enabled: true + directoryPerDB: true + +# where to write logging data. +systemLog: + destination: file + quiet: false + logAppend: true + logRotate: reopen + path: {{ $mongodb_root }}/logs/mongodb.log + verbosity: 0 + +# network interfaces +net: + port: {{ $mongodb_port }} + unixDomainSocket: + enabled: false + pathPrefix: {{ $mongodb_root }}/tmp + ipv6: false + bindIpAll: true + #bindIp: + +# replica set options +replication: + replSetName: replicaset + enableMajorityReadConcern: true + +# sharding options +#sharding: + #clusterRole: + +# process management options +processManagement: + fork: false + pidFilePath: {{ $mongodb_root }}/tmp/mongodb.pid + +# set parameter options +setParameter: + enableLocalhostAuthBypass: true + +# security options +security: + authorization: enabled + keyFile: /etc/mongodb/keyfile diff --git a/deploy/mongodb/scripts/replicaset-post-start.tpl b/deploy/mongodb/scripts/replicaset-post-start.tpl new file mode 100644 index 000000000..b7aa0f7dc --- /dev/null +++ b/deploy/mongodb/scripts/replicaset-post-start.tpl @@ -0,0 +1,52 @@ +#!/bin/sh +# usage: replicaset-post-start.sh type_name is_configsvr +# type_name: component.type, in uppercase +# is_configsvr: true or false, default false +{{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} +{{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} + +# require port +{{- $mongodb_port := 27017 }} +{{- if $mongodb_port_info }} +{{- $mongodb_port = $mongodb_port_info.containerPort }} +{{- end }} + +set -e +PORT={{ $mongodb_port }} +MONGODB_ROOT={{ $mongodb_root }} +INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); +INDEX=${INDEX#-}; +if [ $INDEX -ne 0 ]; then exit 0; fi + +until mongosh --quiet --port $PORT --eval "print('ready')"; do sleep 1; done + +RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); +RPL_SET_NAME=${RPL_SET_NAME%-}; + +TYPE_NAME=$1 +IS_CONFIGSVR=$2 +MEMBERS="" +i=0 +while [ $i -lt $(eval echo \$KB_"$TYPE_NAME"_N) ]; do + host=$(eval echo \$KB_"$TYPE_NAME"_"$i"_HOSTNAME) + host=$host"."$KB_NAMESPACE".svc.cluster.local" + until mongosh --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done + if [ $i -eq 0 ]; then + MEMBERS="{_id: $i, host: \"$host:$PORT\", priority:2}" + else + MEMBERS="$MEMBERS,{_id: $i, host: \"$host:$PORT\"}" + fi + i=$(( i + 1)) +done +CONFIGSVR="" +if [ ""$IS_CONFIGSVR = "true" ]; then CONFIGSVR="configsvr: true,"; fi + +until is_inited=$(mongosh --quiet --port $PORT --eval "rs.status().ok" -u root --password $MONGODB_ROOT_PASSWORD || mongosh --quiet --port $PORT --eval "try { rs.status().ok } catch (e) { 0 }") ; do sleep 1; done +if [ $is_inited -eq 1 ]; then + exit 0 +fi; +mongosh --quiet --port $PORT --eval "rs.initiate({_id: \"$RPL_SET_NAME\", $CONFIGSVR members: [$MEMBERS]})"; + +(until mongosh --quiet --port $PORT --eval "rs.isMaster().isWritablePrimary"|grep true; do sleep 1; done; +echo "create user"; +mongosh --quiet --port $PORT admin --eval "db.createUser({ user: \"$MONGODB_ROOT_USER\", pwd: \"$MONGODB_ROOT_PASSWORD\", roles: [{role: 'root', db: 'admin'}] })") /dev/null 2>&1 & diff --git a/deploy/mongodb/scripts/replicaset-restore.tpl b/deploy/mongodb/scripts/replicaset-restore.tpl new file mode 100644 index 000000000..eba4ca7b7 --- /dev/null +++ b/deploy/mongodb/scripts/replicaset-restore.tpl @@ -0,0 +1,28 @@ +#!/bin/sh + +set -e +PORT=27017 +MONGODB_ROOT=/data/mongodb +mkdir -p $MONGODB_ROOT/db +mkdir -p $MONGODB_ROOT/logs +mkdir -p $MONGODB_ROOT/tmp + +res=`ls -A ${DATA_DIR}` +if [ ! -z ${res} ]; then + echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." + exit 1 +fi +tar -xvf ${BACKUP_DIR}/${BACKUP_NAME}.tar.gz -C ${DATA_DIR}/../ +mv ${DATA_DIR}/../${BACKUP_NAME}/* ${DATA_DIR} +RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); +RPL_SET_NAME=${RPL_SET_NAME%-}; +MODE=$1 +mongod $MODE --bind_ip_all --port $PORT --dbpath $MONGODB_ROOT/db --directoryperdb --logpath $MONGODB_ROOT/logs/mongodb.log --logappend --pidfilepath $MONGODB_ROOT/tmp/mongodb.pid& +until mongosh --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done +PID=`cat $MONGODB_ROOT/tmp/mongodb.pid` + +mongosh --quiet --port $PORT local --eval "db.system.replset.deleteOne({})" +mongosh --quiet --port $PORT local --eval "db.system.replset.find()" +mongosh --quiet --port $PORT admin --eval 'db.dropUser("root", {w: "majority", wtimeout: 4000})' || true +kill $PID +wait $PID diff --git a/deploy/mongodb/scripts/replicaset-setup.tpl b/deploy/mongodb/scripts/replicaset-setup.tpl new file mode 100644 index 000000000..2bef290a3 --- /dev/null +++ b/deploy/mongodb/scripts/replicaset-setup.tpl @@ -0,0 +1,20 @@ +#!/bin/sh + +{{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} +{{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} + +# require port +{{- $mongodb_port := 27017 }} +{{- if $mongodb_port_info }} +{{- $mongodb_port = $mongodb_port_info.containerPort }} +{{- end }} + +PORT={{ $mongodb_port }} +MONGODB_ROOT={{ $mongodb_root }} +RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); +RPL_SET_NAME=${RPL_SET_NAME%-}; +mkdir -p $MONGODB_ROOT/db +mkdir -p $MONGODB_ROOT/logs +mkdir -p $MONGODB_ROOT/tmp +MODE=$1 +mongod $MODE --bind_ip_all --port $PORT --replSet $RPL_SET_NAME --config /etc/mongodb/mongodb.conf diff --git a/deploy/mongodb/templates/_helpers.tpl b/deploy/mongodb/templates/_helpers.tpl index 87b44f5ba..d2b8c7f68 100644 --- a/deploy/mongodb/templates/_helpers.tpl +++ b/deploy/mongodb/templates/_helpers.tpl @@ -49,3 +49,29 @@ Selector labels app.kubernetes.io/name: {{ include "mongodb.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} + + +{{/* +Return MongoDB service port +*/}} +{{- define "mongodb.service.port" -}} +{{- .Values.primary.service.ports.mongodb -}} +{{- end -}} + +{{/* +Return the name for a custom database to create +*/}} +{{- define "mongodb.database" -}} +{{- .Values.auth.database -}} +{{- end -}} + +{{/* +Get the password key. +*/}} +{{- define "mongodb.password" -}} +{{- if or (.Release.IsInstall) (not (lookup "apps.kubeblocks.io/v1alpha1" "ClusterDefinition" "" "mongodb")) -}} +{{ .Values.auth.password | default "$(RANDOM_PASSWD)"}} +{{- else -}} +{{ index (lookup "apps.kubeblocks.io/v1alpha1" "ClusterDefinition" "" "mongodb").spec.connectionCredential "password"}} +{{- end }} +{{- end }} diff --git a/deploy/mongodb/templates/backuppolicytemplate.yaml b/deploy/mongodb/templates/backuppolicytemplate.yaml new file mode 100644 index 000000000..c2bc88561 --- /dev/null +++ b/deploy/mongodb/templates/backuppolicytemplate.yaml @@ -0,0 +1,15 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupPolicyTemplate +metadata: + name: backup-policy-template-mongodb + labels: + clusterdefinition.kubeblocks.io/name: mongodb + {{- include "mongodb.labels" . | nindent 4 }} +spec: + # which backup tool to perform database backup, only support one tool. + backupToolName: volumesnapshot + ttl: 168h0m0s + + credentialKeyword: + userKeyword: username + passwordKeyword: password diff --git a/deploy/mongodb/templates/clusterdefinition.yaml b/deploy/mongodb/templates/clusterdefinition.yaml index a74a93036..a08280796 100644 --- a/deploy/mongodb/templates/clusterdefinition.yaml +++ b/deploy/mongodb/templates/clusterdefinition.yaml @@ -7,9 +7,131 @@ metadata: spec: type: mongodb connectionCredential: - username: admin - password: "" + username: root + password: {{ (include "mongodb.password" .) | quote }} + endpoint: "$(SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + host: "$(SVC_FQDN)" + port: "$(SVC_PORT_tcp-monogdb)" + headlessEndpoint: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + headlessHost: "$(POD_NAME_PREFIX)-0.$(HEADLESS_SVC_FQDN)" + headlessPort: "$(SVC_PORT_tcp-monogdb)" componentDefs: + - name: replicaset + characterType: mongodb + scriptSpecs: + - name: mongodb-scripts + templateRef: mongodb-scripts + volumeName: scripts + namespace: {{ .Release.Namespace }} + defaultMode: 493 + configSpecs: + - name: mongodb-config + templateRef: mongodb5.0-config-template + namespace: {{ .Release.Namespace }} + volumeName: mongodb-config + defaultMode: 256 + - name: mongodb-metrics-config + templateRef: mongodb-metrics-config + namespace: {{ .Release.Namespace }} + volumeName: mongodb-metrics-config + defaultMode: 0777 + monitor: + builtIn: false + exporterConfig: + scrapePath: /metrics + scrapePort: 9216 + logConfigs: + {{- range $name,$pattern := .Values.logConfigs }} + - name: {{ $name }} + filePathPattern: {{ $pattern }} + {{- end }} + workloadType: Consensus + consensusSpec: + leader: + name: "primary" + accessMode: ReadWrite + followers: + - name: "secondary" + accessMode: Readonly + updateStrategy: Serial + probes: + roleProbe: + periodSeconds: 2 + failureThreshold: 3 + service: + ports: + - protocol: TCP + port: 27017 + volumeTypes: + - name: data + type: data + podSpec: + containers: + - name: mongodb + ports: + - name: mongodb + protocol: TCP + containerPort: 27017 + command: + - /scripts/replicaset-setup.sh + env: + - name: MONGODB_ROOT_USER + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: username + - name: MONGODB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: password + lifecycle: + postStart: + exec: + command: + - /scripts/replicaset-post-start.sh + - REPLICASET + volumeMounts: + - mountPath: /data/mongodb + name: data + - mountPath: /etc/mongodb/mongodb.conf + name: mongodb-config + subPath: mongodb.conf + - mountPath: /etc/mongodb/keyfile + name: mongodb-config + subPath: keyfile + - name: scripts + mountPath: /scripts/replicaset-setup.sh + subPath: replicaset-setup.sh + - name: scripts + mountPath: /scripts/replicaset-post-start.sh + subPath: replicaset-post-start.sh + - name: metrics + image: {{ .Values.metrics.image.registry | default "docker.io" }}/{{ .Values.metrics.image.repository }}:{{ .Values.metrics.image.tag }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + securityContext: + runAsNonRoot: true + runAsUser: 1001 + env: + - name: MONGODB_ROOT_USER + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: username + - name: MONGODB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: password + command: + - "/bin/agamotto" + - "--config=/opt/conf/metrics-config.yaml" + ports: + - name: http-metrics + containerPort: 9216 + volumeMounts: + - name: mongodb-metrics-config + mountPath: /opt/conf - name: mongos scriptSpecs: - name: mongodb-scripts @@ -53,7 +175,7 @@ spec: accessMode: Readonly updateStrategy: Serial probes: - roleChangedProbe: + roleProbe: periodSeconds: 2 failureThreshold: 3 service: @@ -102,7 +224,7 @@ spec: accessMode: Readonly updateStrategy: BestEffortParallel probes: - roleChangedProbe: + roleProbe: periodSeconds: 2 failureThreshold: 3 service: diff --git a/deploy/mongodb/templates/clusterversion.yaml b/deploy/mongodb/templates/clusterversion.yaml index f0f435277..3e1cb149e 100644 --- a/deploy/mongodb/templates/clusterversion.yaml +++ b/deploy/mongodb/templates/clusterversion.yaml @@ -7,6 +7,12 @@ metadata: spec: clusterDefinitionRef: mongodb componentVersions: + - componentDefRef: replicaset + versionsContext: + containers: + - name: mongodb + image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} - componentDefRef: mongos versionsContext: containers: @@ -26,4 +32,4 @@ spec: image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} - name: agent image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} \ No newline at end of file + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} diff --git a/deploy/mongodb/templates/configmap.yaml b/deploy/mongodb/templates/configmap.yaml deleted file mode 100644 index 67926afc5..000000000 --- a/deploy/mongodb/templates/configmap.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: mongodb-scripts - labels: - {{- include "mongodb.labels" . | nindent 4 }} -data: - mongos-setup.sh: |- - #!/bin/sh - - PORT=27018 - CONFIG_SVR_NAME=$KB_CLUSTER_NAME"-configsvr" - DOMAIN=$CONFIG_SVR_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" - mongos --bind_ip_all --configdb $CONFIG_SVR_NAME/$CONFIG_SVR_NAME"-0."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-1."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-2."$DOMAIN:$PORT - replicaset-setup.sh: |- - #!/bin/sh - - RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); - RPL_SET_NAME=${RPL_SET_NAME%-}; - PORT=27018 - MODE=$1 - mongod $MODE --bind_ip_all --port $PORT --replSet $RPL_SET_NAME - replicaset-post-start.sh: |- - #!/bin/sh - # usage: replicaset-post-start.sh type_name is_configsvr - # type_name: component.type, in uppercase - # is_configsvr: true or false, default false - INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); - INDEX=${INDEX#-}; - if [ $INDEX -ne 0 ]; then exit 0; fi - - PORT=27018 - until mongosh --port $PORT --eval "print('ready')"; do sleep 1; done - - RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); - RPL_SET_NAME=${RPL_SET_NAME%-}; - - TYPE_NAME=$1 - IS_CONFIGSVR=$2 - MEMBERS="" - i=0 - while [ $i -lt $(eval echo \$KB_"$TYPE_NAME"_N) ]; do - if [ $i -ne 0 ]; then MEMBERS="$MEMBERS,"; fi - host=$(eval echo \$KB_"$TYPE_NAME"_"$i"_HOSTNAME) - host=$host"."$KB_NAMESPACE".svc.cluster.local" - until mongosh --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done - MEMBERS="$MEMBERS{_id: $i, host: \"$host:$PORT\"}" - i=$(( i + 1)) - done - CONFIGSVR="" - if [ $IS_CONFIGSVR = "true" ]; then CONFIGSVR="configsvr: true,"; fi - mongosh --port $PORT --eval "rs.initiate({_id: \"$RPL_SET_NAME\", $CONFIGSVR members: [$MEMBERS]})" - shard-agent.sh: |- - #!/bin/sh - - INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); - INDEX=${INDEX#-}; - if [ $INDEX -ne 0 ]; then - trap : TERM INT; (while true; do sleep 1000; done) & wait - fi - - # wait main container ready - PORT=27018 - until mongosh --port $PORT --eval "rs.status().ok"; do sleep 1; done - # add shard to mongos - SHARD_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); - SHARD_NAME=${SHARD_NAME%-}; - DOMAIN=$SHARD_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" - MONGOS_HOST=$KB_CLUSTER_NAME"-mongos" - MONGOS_PORT=27017 - SHARD_CONFIG=$SHARD_NAME/$SHARD_NAME"-0."$DOMAIN:$PORT,$SHARD_NAME"-1."$DOMAIN:$PORT,$SHARD_NAME"-2."$DOMAIN:$PORT - until mongosh --host $MONGOS_HOST --port $MONGOS_PORT --eval "print('service is ready')"; do sleep 1; done - mongosh --host $MONGOS_HOST --port $MONGOS_PORT --eval "sh.addShard(\"$SHARD_CONFIG\")" - - trap : TERM INT; (while true; do sleep 1000; done) & wait \ No newline at end of file diff --git a/deploy/mongodb/templates/configtemplate.yaml b/deploy/mongodb/templates/configtemplate.yaml new file mode 100644 index 000000000..a15bda17b --- /dev/null +++ b/deploy/mongodb/templates/configtemplate.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb5.0-config-template + labels: + {{- include "mongodb.labels" . | nindent 4 }} +data: + mongodb.conf: |- + {{- .Files.Get "config/mongodb5.0-config.tpl" | nindent 4 }} + keyfile: |- + {{- .Files.Get "config/keyfile.tpl" | nindent 4 }} diff --git a/deploy/mongodb/templates/metrics-configmap.yaml b/deploy/mongodb/templates/metrics-configmap.yaml new file mode 100644 index 000000000..1f305a643 --- /dev/null +++ b/deploy/mongodb/templates/metrics-configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-metrics-config + labels: + {{- include "mongodb.labels" . | nindent 4 }} +data: + metrics-config.yaml: {{ toYaml .Values.metrics.config | quote }} diff --git a/deploy/mongodb/templates/scriptstemplate.yaml b/deploy/mongodb/templates/scriptstemplate.yaml new file mode 100644 index 000000000..638a41d57 --- /dev/null +++ b/deploy/mongodb/templates/scriptstemplate.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-scripts + labels: + {{- include "mongodb.labels" . | nindent 4 }} +data: + mongos-setup.sh: |- + #!/bin/sh + + PORT=27018 + CONFIG_SVR_NAME=$KB_CLUSTER_NAME"-configsvr" + DOMAIN=$CONFIG_SVR_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" + mongos --bind_ip_all --configdb $CONFIG_SVR_NAME/$CONFIG_SVR_NAME"-0."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-1."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-2."$DOMAIN:$PORT + replicaset-setup.sh: |- + {{- .Files.Get "scripts/replicaset-setup.tpl" | nindent 4 }} + replicaset-post-start.sh: |- + {{- .Files.Get "scripts/replicaset-post-start.tpl" | nindent 4 }} + shard-agent.sh: |- + #!/bin/sh + + INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); + INDEX=${INDEX#-}; + if [ $INDEX -ne 0 ]; then + trap : TERM INT; (while true; do sleep 1000; done) & wait + fi + + # wait main container ready + PORT=27018 + until mongosh --quiet --port $PORT --eval "rs.status().ok"; do sleep 1; done + # add shard to mongos + SHARD_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); + SHARD_NAME=${SHARD_NAME%-}; + DOMAIN=$SHARD_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" + MONGOS_HOST=$KB_CLUSTER_NAME"-mongos" + MONGOS_PORT=27017 + SHARD_CONFIG=$SHARD_NAME/$SHARD_NAME"-0."$DOMAIN:$PORT,$SHARD_NAME"-1."$DOMAIN:$PORT,$SHARD_NAME"-2."$DOMAIN:$PORT + until mongosh --quiet --host $MONGOS_HOST --port $MONGOS_PORT --eval "print('service is ready')"; do sleep 1; done + mongosh --quiet --host $MONGOS_HOST --port $MONGOS_PORT --eval "sh.addShard(\"$SHARD_CONFIG\")" + + trap : TERM INT; (while true; do sleep 1000; done) & wait diff --git a/deploy/mongodb/values.yaml b/deploy/mongodb/values.yaml index df6f33cca..97cab9f36 100644 --- a/deploy/mongodb/values.yaml +++ b/deploy/mongodb/values.yaml @@ -13,7 +13,82 @@ nameOverride: "" fullnameOverride: "" replicaset: - roleChangedProbe: + roleProbe: failureThreshold: 2 periodSeconds: 1 timeoutSeconds: 1 + +## Authentication parameters +## +auth: + ## @param auth.password Password for the "mongodb" admin user, leave empty + ## for random generated password. + ## + password: + ## @param auth.database Name for a custom database to create + ## + database: "admin" + +logConfigs: + running: /data/mongodb/log/mongodb.log* + +metrics: + image: + registry: registry.cn-hangzhou.aliyuncs.com + repository: apecloud/agamotto + tag: 0.0.4 + pullPolicy: IfNotPresent + config: + extensions: + memory_ballast: + size_mib: 512 + health_check: + endpoint: 0.0.0.0:13133 + path: /health/status + check_collector_pipeline: + enabled: true + interval: 2m + exporter_failure_threshold: 5 + + receivers: + apecloudmongodb: + uri: mongodb://${env:MONGODB_ROOT_USER}:${env:MONGODB_ROOT_PASSWORD}@127.0.0.1:27017/admin?ssl=false&authSource=admin + # uri: mongodb://127.0.0.1:27017 + collect-all: true + collection_interval: 15s + direct-connect: true + global-conn-pool: false + log-level: info + compatible-mode: true + + processors: + batch: + timeout: 5s + memory_limiter: + limit_mib: 1024 + spike_limit_mib: 256 + check_interval: 10s + + exporters: + prometheus: + endpoint: 0.0.0.0:9216 + const_labels: [ ] + send_timestamps: false + metric_expiration: 30s + enable_open_metrics: false + resource_to_telemetry_conversion: + enabled: true + + service: + telemetry: + logs: + level: info + metrics: + address: 0.0.0.0:8888 + pipelines: + metrics: + receivers: [apecloudmongodb] + processors: [memory_limiter] + exporters: [prometheus] + + extensions: [memory_ballast, health_check] diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 4b06d0895..3d13d31d2 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -27,7 +27,7 @@ spec: selector: app.kubernetes.io/managed-by: kubeblocks probes: - roleChangedProbe: + roleProbe: failureThreshold: 2 periodSeconds: 1 timeoutSeconds: 1 diff --git a/docs/user_docs/kubeblocks-for-mysql/high-availability/high-availability.md b/docs/user_docs/kubeblocks-for-mysql/high-availability/high-availability.md index d0aac2a86..abb8f4872 100644 --- a/docs/user_docs/kubeblocks-for-mysql/high-availability/high-availability.md +++ b/docs/user_docs/kubeblocks-for-mysql/high-availability/high-availability.md @@ -20,10 +20,10 @@ The faults here are all simulated by deleting a pod. When there are sufficient r * Install a Kubernetes cluster and KubeBlocks, refer to [Install KubeBlocks](./../../installation/install-and-uninstall-kbcli-and-kubeblocks.md). * Create an ApeCloud MySQL Paxos Group, refer to [Create a MySQL cluster](./../cluster-management/create-and-connect-a-mysql-cluster.md). -* Run `kubectl get cd apecloud-mysql -o yaml` to check whether _rolechangedprobe_ is enabled in the ApeCloud MySQL Paxos Group (it is enabled by default). If the following configuration exists, it indicates that it is enabled: +* Run `kubectl get cd apecloud-mysql -o yaml` to check whether _roleprobe_ is enabled in the ApeCloud MySQL Paxos Group (it is enabled by default). If the following configuration exists, it indicates that it is enabled: ``` probes: - roleChangedProbe: + roleProbe: failureThreshold: 3 periodSeconds: 2 timeoutSeconds: 1 @@ -161,4 +161,4 @@ Therefore, whether exceptions occur to one leader and one follower or exceptions ***How the automatic recovery works*** - Every time the pod is deleted, recreation is triggered. And then ApeCloud MySQL automatically completes the cluster recovery and the election of a new leader. After the election of the leader is completed, KubeBlocks detects the new leader and updates the access link. This process takes less than 30 seconds. \ No newline at end of file + Every time the pod is deleted, recreation is triggered. And then ApeCloud MySQL automatically completes the cluster recovery and the election of a new leader. After the election of the leader is completed, KubeBlocks detects the new leader and updates the access link. This process takes less than 30 seconds. diff --git a/internal/constant/const.go b/internal/constant/const.go index 7c6fcdc5d..2ce158c6b 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -168,7 +168,7 @@ const ( ProbeGRPCPortName = "probe-grpc-port" RoleProbeContainerName = "kb-checkrole" StatusProbeContainerName = "kb-checkstatus" - RunningProbeContainerName = "kb-runningcheck" + RunningProbeContainerName = "kb-checkrunning" // the filedpath name used in event.InvolvedObject.FieldPath ProbeCheckRolePath = "spec.containers{" + RoleProbeContainerName + "}" diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index 84f2f0e7b..e423ac749 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -343,12 +343,13 @@ func BuildConnCredential(params BuilderParams) (*corev1.Secret, error) { uuidStrB64 := base64.RawStdEncoding.EncodeToString([]byte(strings.ReplaceAll(uuidStr, "-", ""))) uuidHex := hex.EncodeToString(uuidBytes) m := map[string]string{ - "$(RANDOM_PASSWD)": randomString(8), - "$(UUID)": uuidStr, - "$(UUID_B64)": uuidB64, - "$(UUID_STR_B64)": uuidStrB64, - "$(UUID_HEX)": uuidHex, - "$(SVC_FQDN)": fmt.Sprintf("%s-%s.%s.svc", params.Cluster.Name, params.Component.Name, params.Cluster.Namespace), + "$(RANDOM_PASSWD)": randomString(8), + "$(UUID)": uuidStr, + "$(UUID_B64)": uuidB64, + "$(UUID_STR_B64)": uuidStrB64, + "$(UUID_HEX)": uuidHex, + "$(SVC_FQDN)": fmt.Sprintf("%s-%s.%s.svc", params.Cluster.Name, params.Component.Name, params.Cluster.Namespace), + "$(HEADLESS_SVC_FQDN)": fmt.Sprintf("%s-%s-headless.%s.svc", params.Cluster.Name, params.Component.Name, params.Cluster.Namespace), } if len(params.Component.Services) > 0 { for _, p := range params.Component.Services[0].Spec.Ports { diff --git a/internal/controller/builder/builder_test.go b/internal/controller/builder/builder_test.go index ba488c008..99dee24a0 100644 --- a/internal/controller/builder/builder_test.go +++ b/internal/controller/builder/builder_test.go @@ -239,12 +239,15 @@ var _ = Describe("builder", func() { "UUID_B64", "UUID_STR_B64", "UUID_HEX", + "HEADLESS_SVC_FQDN", } { Expect(credential.StringData[v]).ShouldNot(BeEquivalentTo(fmt.Sprintf("$(%s)", v))) } Expect(credential.StringData["RANDOM_PASSWD"]).Should(HaveLen(8)) svcFQDN := fmt.Sprintf("%s-%s.%s.svc", params.Cluster.Name, params.Component.Name, params.Cluster.Namespace) + headlessSvcFQDN := fmt.Sprintf("%s-%s-headless.%s.svc", params.Cluster.Name, params.Component.Name, + params.Cluster.Namespace) var mysqlPort corev1.ServicePort var paxosPort corev1.ServicePort for _, s := range params.Component.Services[0].Spec.Ports { @@ -256,6 +259,7 @@ var _ = Describe("builder", func() { } } Expect(credential.StringData["SVC_FQDN"]).Should(Equal(svcFQDN)) + Expect(credential.StringData["HEADLESS_SVC_FQDN"]).Should(Equal(headlessSvcFQDN)) Expect(credential.StringData["tcpEndpoint"]).Should(Equal(fmt.Sprintf("tcp:%s:%d", svcFQDN, mysqlPort.Port))) Expect(credential.StringData["paxosEndpoint"]).Should(Equal(fmt.Sprintf("paxos:%s:%d", svcFQDN, paxosPort.Port))) diff --git a/internal/controller/component/probe_utils.go b/internal/controller/component/probe_utils.go index ca4562617..247b92c37 100644 --- a/internal/controller/component/probe_utils.go +++ b/internal/controller/component/probe_utils.go @@ -35,7 +35,9 @@ import ( const ( // http://localhost:/v1.0/bindings/ - roleObserveURIFormat = "http://localhost:%s/v1.0/bindings/%s" + checkRoleURIFormat = "http://localhost:%s/v1.0/bindings/%s" + checkRunningURIFormat = "/v1.0/bindings/%s?operation=checkRunning" + checkStatusURIFormat = "/v1.0/bindings/%s?operation=checkStatus" ) var ( @@ -66,21 +68,21 @@ func buildProbeContainers(reqCtx intctrlutil.RequestCtx, component *SynthesizedC return err } - if componentProbes.RoleChangedProbe != nil { + if componentProbes.RoleProbe != nil { roleChangedContainer := container.DeepCopy() - buildRoleChangedProbeContainer(component.CharacterType, roleChangedContainer, componentProbes.RoleChangedProbe, int(probeSvcHTTPPort)) + buildRoleProbeContainer(component.CharacterType, roleChangedContainer, componentProbes.RoleProbe, int(probeSvcHTTPPort)) probeContainers = append(probeContainers, *roleChangedContainer) } if componentProbes.StatusProbe != nil { statusProbeContainer := container.DeepCopy() - buildStatusProbeContainer(statusProbeContainer, componentProbes.StatusProbe, int(probeSvcHTTPPort)) + buildStatusProbeContainer(component.CharacterType, statusProbeContainer, componentProbes.StatusProbe, int(probeSvcHTTPPort)) probeContainers = append(probeContainers, *statusProbeContainer) } if componentProbes.RunningProbe != nil { runningProbeContainer := container.DeepCopy() - buildRunningProbeContainer(runningProbeContainer, componentProbes.RunningProbe, int(probeSvcHTTPPort)) + buildRunningProbeContainer(component.CharacterType, runningProbeContainer, componentProbes.RunningProbe, int(probeSvcHTTPPort)) probeContainers = append(probeContainers, *runningProbeContainer) } @@ -125,16 +127,13 @@ func buildProbeServiceContainer(component *SynthesizedComponent, container *core "--config", "/config/probe/config.yaml", "--components-path", "/config/probe/components"} - if len(component.Services) > 0 && len(component.Services[0].Spec.Ports) > 0 { - service := component.Services[0] - port := service.Spec.Ports[0] - dbPort := port.TargetPort.IntValue() - if dbPort == 0 { - dbPort = int(port.Port) - } + if len(component.PodSpec.Containers) > 0 && len(component.PodSpec.Containers[0].Ports) > 0 { + mainContainer := component.PodSpec.Containers[0] + port := mainContainer.Ports[0] + dbPort := port.ContainerPort container.Env = append(container.Env, corev1.EnvVar{ Name: constant.KBPrefix + "_SERVICE_PORT", - Value: strconv.Itoa(dbPort), + Value: strconv.Itoa(int(dbPort)), ValueFrom: nil, }) } @@ -182,13 +181,13 @@ func getComponentRoles(component *SynthesizedComponent) map[string]string { return roles } -func buildRoleChangedProbeContainer(characterType string, roleChangedContainer *corev1.Container, +func buildRoleProbeContainer(characterType string, roleChangedContainer *corev1.Container, probeSetting *appsv1alpha1.ClusterDefinitionProbe, probeSvcHTTPPort int) { roleChangedContainer.Name = constant.RoleProbeContainerName probe := roleChangedContainer.ReadinessProbe bindingType := strings.ToLower(characterType) svcPort := strconv.Itoa(probeSvcHTTPPort) - roleObserveURI := fmt.Sprintf(roleObserveURIFormat, svcPort, bindingType) + roleObserveURI := fmt.Sprintf(checkRoleURIFormat, svcPort, bindingType) probe.Exec.Command = []string{ "curl", "-X", "POST", "--max-time", strconv.Itoa(int(probeSetting.TimeoutSeconds)), @@ -203,12 +202,12 @@ func buildRoleChangedProbeContainer(characterType string, roleChangedContainer * roleChangedContainer.StartupProbe.TCPSocket.Port = intstr.FromInt(probeSvcHTTPPort) } -func buildStatusProbeContainer(statusProbeContainer *corev1.Container, +func buildStatusProbeContainer(characterType string, statusProbeContainer *corev1.Container, probeSetting *appsv1alpha1.ClusterDefinitionProbe, probeSvcHTTPPort int) { statusProbeContainer.Name = constant.StatusProbeContainerName probe := statusProbeContainer.ReadinessProbe httpGet := &corev1.HTTPGetAction{} - httpGet.Path = "/v1.0/bindings/probe?operation=checkStatus" + httpGet.Path = fmt.Sprintf(checkStatusURIFormat, characterType) httpGet.Port = intstr.FromInt(probeSvcHTTPPort) probe.Exec = nil probe.HTTPGet = httpGet @@ -218,12 +217,12 @@ func buildStatusProbeContainer(statusProbeContainer *corev1.Container, statusProbeContainer.StartupProbe.TCPSocket.Port = intstr.FromInt(probeSvcHTTPPort) } -func buildRunningProbeContainer(runningProbeContainer *corev1.Container, +func buildRunningProbeContainer(characterType string, runningProbeContainer *corev1.Container, probeSetting *appsv1alpha1.ClusterDefinitionProbe, probeSvcHTTPPort int) { runningProbeContainer.Name = constant.RunningProbeContainerName probe := runningProbeContainer.ReadinessProbe httpGet := &corev1.HTTPGetAction{} - httpGet.Path = "/v1.0/bindings/probe?operation=checkRunning" + httpGet.Path = fmt.Sprintf(checkRunningURIFormat, characterType) httpGet.Port = intstr.FromInt(probeSvcHTTPPort) probe.Exec = nil probe.HTTPGet = httpGet diff --git a/internal/controller/component/probe_utils_test.go b/internal/controller/component/probe_utils_test.go index 746bcae46..24388daab 100644 --- a/internal/controller/component/probe_utils_test.go +++ b/internal/controller/component/probe_utils_test.go @@ -72,9 +72,9 @@ var _ = Describe("probe_utils", func() { }, } component.Probes = &appsv1alpha1.ClusterDefinitionProbes{ - RunningProbe: &appsv1alpha1.ClusterDefinitionProbe{}, - StatusProbe: &appsv1alpha1.ClusterDefinitionProbe{}, - RoleChangedProbe: &appsv1alpha1.ClusterDefinitionProbe{}, + RunningProbe: &appsv1alpha1.ClusterDefinitionProbe{}, + StatusProbe: &appsv1alpha1.ClusterDefinitionProbe{}, + RoleProbe: &appsv1alpha1.ClusterDefinitionProbe{}, } component.PodSpec = &corev1.PodSpec{ Containers: []corev1.Container{}, @@ -92,7 +92,7 @@ var _ = Describe("probe_utils", func() { }) It("should build role changed probe container", func() { - buildRoleChangedProbeContainer("wesql", container, clusterDefProbe, probeServiceHTTPPort) + buildRoleProbeContainer("wesql", container, clusterDefProbe, probeServiceHTTPPort) Expect(container.ReadinessProbe.Exec.Command).ShouldNot(BeEmpty()) }) @@ -102,12 +102,12 @@ var _ = Describe("probe_utils", func() { }) It("should build status probe container", func() { - buildStatusProbeContainer(container, clusterDefProbe, probeServiceHTTPPort) + buildStatusProbeContainer("wesql", container, clusterDefProbe, probeServiceHTTPPort) Expect(container.ReadinessProbe.HTTPGet).ShouldNot(BeNil()) }) It("should build running probe container", func() { - buildRunningProbeContainer(container, clusterDefProbe, probeServiceHTTPPort) + buildRunningProbeContainer("wesql", container, clusterDefProbe, probeServiceHTTPPort) Expect(container.ReadinessProbe.HTTPGet).ShouldNot(BeNil()) }) }) diff --git a/internal/testutil/apps/constant.go b/internal/testutil/apps/constant.go index 40c805f62..17cbafd5a 100644 --- a/internal/testutil/apps/constant.go +++ b/internal/testutil/apps/constant.go @@ -73,15 +73,16 @@ var ( } defaultConnectionCredential = map[string]string{ - "username": "root", - "SVC_FQDN": "$(SVC_FQDN)", - "RANDOM_PASSWD": "$(RANDOM_PASSWD)", - "tcpEndpoint": "tcp:$(SVC_FQDN):$(SVC_PORT_mysql)", - "paxosEndpoint": "paxos:$(SVC_FQDN):$(SVC_PORT_paxos)", - "UUID": "$(UUID)", - "UUID_B64": "$(UUID_B64)", - "UUID_STR_B64": "$(UUID_STR_B64)", - "UUID_HEX": "$(UUID_HEX)", + "username": "root", + "SVC_FQDN": "$(SVC_FQDN)", + "HEADLESS_SVC_FQDN": "$(HEADLESS_SVC_FQDN)", + "RANDOM_PASSWD": "$(RANDOM_PASSWD)", + "tcpEndpoint": "tcp:$(SVC_FQDN):$(SVC_PORT_mysql)", + "paxosEndpoint": "paxos:$(SVC_FQDN):$(SVC_PORT_paxos)", + "UUID": "$(UUID)", + "UUID_B64": "$(UUID_B64)", + "UUID_STR_B64": "$(UUID_STR_B64)", + "UUID_HEX": "$(UUID_HEX)", } // defaultSvc value are corresponding to defaultMySQLContainer.Ports name mapping and @@ -187,7 +188,7 @@ var ( CharacterType: "mysql", ConsensusSpec: &defaultConsensusSpec, Probes: &appsv1alpha1.ClusterDefinitionProbes{ - RoleChangedProbe: &appsv1alpha1.ClusterDefinitionProbe{ + RoleProbe: &appsv1alpha1.ClusterDefinitionProbe{ FailureThreshold: 3, PeriodSeconds: 1, TimeoutSeconds: 5, From 55f7b9ec6dddff04094ad1e3ca4d835531556bf1 Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Thu, 13 Apr 2023 11:43:40 +0800 Subject: [PATCH 011/439] fix: fix bug for cli spinner (#2515) Co-authored-by: huyongqii --- internal/cli/printer/spinner.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/cli/printer/spinner.go b/internal/cli/printer/spinner.go index e7f23951d..ae306c3ad 100644 --- a/internal/cli/printer/spinner.go +++ b/internal/cli/printer/spinner.go @@ -19,8 +19,11 @@ package printer import ( "fmt" "io" + "os" + "os/signal" "runtime" "sync" + "syscall" "time" "github.com/briandowns/spinner" @@ -43,6 +46,17 @@ func Spinner(w io.Writer, fmtstr string, a ...any) func(result bool) { _ = s.Color("cyan") s.Suffix = fmt.Sprintf(" %s", msg) s.Start() + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + // Capture the interrupt signal, make the `spinner` program exit gracefully, and prevent the cursor from disappearing. + go func() { + <-c + s.Stop() + // Show cursor in terminal. + fmt.Fprintf(s.Writer, "\033[?25h") + os.Exit(0) + }() } return func(result bool) { From 268a68f2a8c8a609a58fd84f289daa7c9be76a61 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Thu, 13 Apr 2023 11:46:09 +0800 Subject: [PATCH 012/439] chore: update redis role probe (#2554) --- deploy/redis/templates/clusterdefinition.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/redis/templates/clusterdefinition.yaml b/deploy/redis/templates/clusterdefinition.yaml index 08f34a5c7..f0b76f20f 100644 --- a/deploy/redis/templates/clusterdefinition.yaml +++ b/deploy/redis/templates/clusterdefinition.yaml @@ -17,7 +17,7 @@ spec: workloadType: Replication characterType: redis probes: - roleChangedProbe: + roleProbe: failureThreshold: 2 periodSeconds: 2 timeoutSeconds: 1 @@ -258,4 +258,4 @@ spec: command: - sh - -c - - /scripts/redis-sentinel-ping.sh 1 \ No newline at end of file + - /scripts/redis-sentinel-ping.sh 1 From f866d2bd9af1e05c2e42f2dc052c40328993ba94 Mon Sep 17 00:00:00 2001 From: xingran Date: Thu, 13 Apr 2023 12:03:07 +0800 Subject: [PATCH 013/439] chore: optimize postgres secondary init should wait for primary to be ready (#2550) --- deploy/postgresql/templates/clusterdefinition.yaml | 2 +- deploy/postgresql/templates/scripts.yaml | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 3d13d31d2..8b888b31d 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -103,7 +103,7 @@ spec: readinessProbe: failureThreshold: 6 initialDelaySeconds: 10 - periodSeconds: 10 + periodSeconds: 30 successThreshold: 1 timeoutSeconds: 5 exec: diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index 45f775fad..c82c6f4b3 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -61,7 +61,10 @@ data: set -ex KB_PRIMARY_POD_NAME_PREFIX=${KB_PRIMARY_POD_NAME%%\.*} if [ "$KB_PRIMARY_POD_NAME_PREFIX" != "$KB_POD_NAME" ]; then - sleep 3 + # waiting for primary pod to be ready + until pg_isready -U {{ default "postgres" | quote }} -h $KB_PRIMARY_POD_NAME -p 5432; do + sleep 5 + done fi python3 /kb-scripts/generate_patroni_yaml.py tmp_patroni.yaml export SPILO_CONFIGURATION=$(cat tmp_patroni.yaml) From a42a3db0c59492e3edbb00ae1fe933e715ba1b6e Mon Sep 17 00:00:00 2001 From: shanshanying Date: Thu, 13 Apr 2023 12:05:54 +0800 Subject: [PATCH 014/439] fix: display one user role with the highest priv (#2535) --- cmd/probe/internal/binding/mysql/mysql.go | 29 ++-- .../internal/binding/mysql/mysql_test.go | 6 +- .../internal/binding/postgres/postgres.go | 46 ++++-- cmd/probe/internal/binding/redis/redis.go | 148 ++++++++++++------ .../internal/binding/redis/redis_test.go | 31 +++- cmd/probe/internal/binding/types.go | 38 ++++- cmd/probe/internal/binding/utils.go | 32 +++- go.mod | 2 +- .../lifecycle/transformer_fill_class.go | 16 ++ .../transformer_fix_cluster_labels.go | 16 ++ 10 files changed, 268 insertions(+), 96 deletions(-) diff --git a/cmd/probe/internal/binding/mysql/mysql.go b/cmd/probe/internal/binding/mysql/mysql.go index a52c5fb91..a8679b5a1 100644 --- a/cmd/probe/internal/binding/mysql/mysql.go +++ b/cmd/probe/internal/binding/mysql/mysql.go @@ -36,6 +36,7 @@ import ( "github.com/go-sql-driver/mysql" "github.com/pkg/errors" "github.com/spf13/viper" + "golang.org/x/exp/slices" . "github.com/apecloud/kubeblocks/cmd/probe/internal/binding" . "github.com/apecloud/kubeblocks/cmd/probe/util" @@ -79,8 +80,8 @@ const ( listUserTpl = "SELECT user AS userName, CASE password_expired WHEN 'N' THEN 'F' ELSE 'T' END as expired FROM mysql.user WHERE host = '%' and user <> 'root' and user not like 'kb%';" showGrantTpl = "SHOW GRANTS FOR '%s'@'%%';" getUserTpl = ` - SELECT user AS userName, CASE password_expired WHEN 'N' THEN 'F' ELSE 'T' END as expired - FROM mysql.user + SELECT user AS userName, CASE password_expired WHEN 'N' THEN 'F' ELSE 'T' END as expired + FROM mysql.user WHERE host = '%%' and user <> 'root' and user not like 'kb%%' and user ='%s';" ` createUserTpl = "CREATE USER '%s'@'%%' IDENTIFIED BY '%s';" @@ -522,16 +523,22 @@ func (mysqlOps *MysqlOperations) describeUserOps(ctx context.Context, req *bindi return nil, err } user := UserInfo{} - userRoles := make([]string, 0) + // only keep one role name of the highest privilege + userRoles := make([]RoleType, 0) for _, roleMap := range roles { for k, v := range roleMap { if len(user.UserName) == 0 { user.UserName = strings.TrimPrefix(strings.TrimSuffix(k, "@%"), "Grants for ") } - userRoles = append(userRoles, mysqlOps.inferRoleFromPriv(strings.TrimPrefix(v, "GRANT "))) + mysqlRoleType := mysqlOps.priv2Role(strings.TrimPrefix(v, "GRANT ")) + userRoles = append(userRoles, mysqlRoleType) } } - user.RoleName = strings.Join(userRoles, ",") + // sort roles by weight + slices.SortFunc(userRoles, SortRoleByWeight) + if len(userRoles) > 0 { + user.RoleName = (string)(userRoles[0]) + } if jsonData, err := json.Marshal([]UserInfo{user}); err != nil { return nil, err } else { @@ -618,7 +625,7 @@ func (mysqlOps *MysqlOperations) managePrivillege(ctx context.Context, req *bind object = UserInfo{} sqlTplRend = func(user UserInfo) string { // render sql stmts - roleDesc, _ := mysqlOps.renderRoleByName(user.RoleName) + roleDesc, _ := mysqlOps.role2Priv(user.RoleName) // update privilege sql := fmt.Sprintf(sqlTpl, roleDesc, user.UserName) return sql @@ -636,20 +643,20 @@ func (mysqlOps *MysqlOperations) managePrivillege(ctx context.Context, req *bind return ExecuteObject(ctx, mysqlOps, req, op, sqlTplRend, msgTplRend, object) } -func (mysqlOps *MysqlOperations) renderRoleByName(roleName string) (string, error) { - switch strings.ToLower(roleName) { +func (mysqlOps *MysqlOperations) role2Priv(roleName string) (string, error) { + roleType := String2RoleType(roleName) + switch roleType { case SuperUserRole: return superUserPriv, nil case ReadWriteRole: return readWritePriv, nil case ReadOnlyRole: return readOnlyRPriv, nil - default: - return "", fmt.Errorf("role name: %s is not supported", roleName) } + return "", fmt.Errorf("role name: %s is not supported", roleName) } -func (mysqlOps *MysqlOperations) inferRoleFromPriv(priv string) string { +func (mysqlOps *MysqlOperations) priv2Role(priv string) RoleType { if strings.HasPrefix(priv, readOnlyRPriv) { return ReadOnlyRole } diff --git a/cmd/probe/internal/binding/mysql/mysql_test.go b/cmd/probe/internal/binding/mysql/mysql_test.go index dd42e44b2..59b786ed5 100644 --- a/cmd/probe/internal/binding/mysql/mysql_test.go +++ b/cmd/probe/internal/binding/mysql/mysql_test.go @@ -499,7 +499,7 @@ func TestMySQLAccounts(t *testing.T) { assert.Equal(t, 1, len(users)) assert.Equal(t, userName, users[0].UserName) assert.NotEmpty(t, users[0].RoleName) - assert.Equal(t, users[0].RoleName, ReadOnlyRole) + assert.True(t, ReadOnlyRole.EqualTo(users[0].RoleName)) }) t.Run("List accounts", func(t *testing.T) { @@ -550,7 +550,7 @@ func TestMySQLAccounts(t *testing.T) { assert.Equal(t, ErrNoRoleName.Error(), result[RespTypMsg]) req.Metadata["roleName"] = roleName - roleDesc, err := mysqlOps.renderRoleByName(req.Metadata["roleName"]) + roleDesc, err := mysqlOps.role2Priv(req.Metadata["roleName"]) assert.Nil(t, err) grantRoleCmd := fmt.Sprintf("GRANT %s TO '%s'@'%%';", roleDesc, req.Metadata["userName"]) @@ -580,7 +580,7 @@ func TestMySQLAccounts(t *testing.T) { assert.Equal(t, ErrNoRoleName.Error(), result[RespTypMsg]) req.Metadata["roleName"] = roleName - roleDesc, err := mysqlOps.renderRoleByName(req.Metadata["roleName"]) + roleDesc, err := mysqlOps.role2Priv(req.Metadata["roleName"]) assert.Nil(t, err) revokeRoleCmd := fmt.Sprintf("REVOKE %s FROM '%s'@'%%';", roleDesc, req.Metadata["userName"]) diff --git a/cmd/probe/internal/binding/postgres/postgres.go b/cmd/probe/internal/binding/postgres/postgres.go index afc94e61d..9bb886811 100644 --- a/cmd/probe/internal/binding/postgres/postgres.go +++ b/cmd/probe/internal/binding/postgres/postgres.go @@ -21,7 +21,6 @@ import ( "encoding/json" "fmt" "strconv" - "strings" "sync" "time" @@ -30,6 +29,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/pkg/errors" "github.com/spf13/viper" + "golang.org/x/exp/slices" . "github.com/apecloud/kubeblocks/cmd/probe/internal/binding" . "github.com/apecloud/kubeblocks/cmd/probe/util" @@ -42,12 +42,12 @@ const ( listUserTpl = ` SELECT usename AS userName, valuntil 0 { + users[i].RoleName = string(roleTypes[0]) + } } if jsonData, err := json.Marshal(users); err != nil { return nil, err @@ -522,13 +534,13 @@ func pgUserRolesProcessor(data interface{}) (interface{}, error) { } } -func (pgOps *PostgresOperations) renderRoleByName(roleName string) (string, error) { - switch strings.ToLower(roleName) { +func (pgOps *PostgresOperations) role2PGRole(roleName string) (string, error) { + roleType := String2RoleType(roleName) + switch roleType { case ReadWriteRole: return "pg_write_all_data", nil case ReadOnlyRole: return "pg_read_all_data", nil - default: - return "", fmt.Errorf("role name: %s is not supported", roleName) } + return "", fmt.Errorf("role name: %s is not supported", roleName) } diff --git a/cmd/probe/internal/binding/redis/redis.go b/cmd/probe/internal/binding/redis/redis.go index 9b3072742..cbf2b7233 100644 --- a/cmd/probe/internal/binding/redis/redis.go +++ b/cmd/probe/internal/binding/redis/redis.go @@ -18,7 +18,6 @@ package redis import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -30,6 +29,10 @@ import ( bindings "github.com/dapr/components-contrib/bindings" "github.com/dapr/kit/logger" + // import this json-iterator package to replace the default + // to avoid the error: 'json: unsupported type: map[interface {}]interface {}' + json "github.com/json-iterator/go" + . "github.com/apecloud/kubeblocks/cmd/probe/internal/binding" rediscomponent "github.com/apecloud/kubeblocks/cmd/probe/internal/component/redis" . "github.com/apecloud/kubeblocks/cmd/probe/util" @@ -156,6 +159,9 @@ func (r *Redis) GetLogger() logger.Logger { // InternalQuery is used for internal query, implement BaseInternalOps interface. func (r *Redis) InternalQuery(ctx context.Context, cmd string) ([]byte, error) { redisArgs := tokenizeCmd2Args(cmd) + // Be aware of the result type. + // type of result could be string, []string, []interface{}, map[interface]interface{} + // it is not solely determined by the command, but also the redis version. result, err := r.query(ctx, redisArgs...) if err != nil { return nil, err @@ -271,48 +277,24 @@ func (r *Redis) listUsersOps(ctx context.Context, req *bindings.InvokeRequest, r func (r *Redis) describeUserOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { var ( object = UserInfo{} + profile map[string]string + err error dataProcessor = func(data interface{}) (interface{}, error) { - redisUserPrivContxt := []string{"commands", "keys", "channels", "selectors"} - redisUserInfoContext := []string{"flags", "passwords"} - - profile := make(map[string]string, 0) - results := make([]interface{}, 0) - err := json.Unmarshal(data.([]byte), &results) + // parse it to a map or an []interface + // try map first + profile, err = parseCommandAndKeyFromMap(data) if err != nil { - return nil, err - } - - var context string - for i := 0; i < len(results); i++ { - result := results[i] - switch result := result.(type) { - case string: - strVal := strings.TrimSpace(result) - if len(strVal) == 0 { - continue - } - if slices.Contains(redisUserInfoContext, strVal) { - i++ - continue - } - if slices.Contains(redisUserPrivContxt, strVal) { - context = strVal - } else { - profile[context] = strVal - } - case []interface{}: - selectors := make([]string, 0) - for _, sel := range result { - selectors = append(selectors, sel.(string)) - } - profile[context] = strings.Join(selectors, " ") + // try list + profile, err = parseCommandAndKeyFromList(data) + if err != nil { + return nil, err } } users := make([]UserInfo, 0) user := UserInfo{ UserName: object.UserName, - RoleName: redisPriv2RoleName(profile["commands"] + " " + profile["keys"]), + RoleName: (string)(r.priv2Role(profile["commands"] + " " + profile["keys"])), } users = append(users, user) if jsonData, err := json.Marshal(users); err != nil { @@ -336,6 +318,83 @@ func (r *Redis) describeUserOps(ctx context.Context, req *bindings.InvokeRequest return QueryObject(ctx, r, req, DescribeUserOp, cmdRender, dataProcessor, object) } +func parseCommandAndKeyFromList(data interface{}) (map[string]string, error) { + var ( + redisUserPrivContxt = []string{"commands", "keys", "channels", "selectors"} + redisUserInfoContext = []string{"flags", "passwords"} + ) + + profile := make(map[string]string, 0) + results := make([]interface{}, 0) + + err := json.Unmarshal(data.([]byte), &results) + if err != nil { + return nil, err + } + // parse line by line + var context string + for i := 0; i < len(results); i++ { + result := results[i] + switch result := result.(type) { + case string: + strVal := strings.TrimSpace(result) + if len(strVal) == 0 { + continue + } + if slices.Contains(redisUserInfoContext, strVal) { + i++ + continue + } + if slices.Contains(redisUserPrivContxt, strVal) { + context = strVal + } else { + profile[context] = strVal + } + case []interface{}: + selectors := make([]string, 0) + for _, sel := range result { + selectors = append(selectors, sel.(string)) + } + profile[context] = strings.Join(selectors, " ") + } + } + return profile, nil +} + +func parseCommandAndKeyFromMap(data interface{}) (map[string]string, error) { + var ( + redisUserPrivContxt = []string{"commands", "keys", "channels", "selectors"} + ) + + profile := make(map[string]string, 0) + results := make(map[string]interface{}, 0) + + err := json.Unmarshal(data.([]byte), &results) + if err != nil { + return nil, err + } + for k, v := range results { + // each key is string, and each v is eigher a string or a list of string + if !slices.Contains(redisUserPrivContxt, k) { + continue + } + + switch v := v.(type) { + case string: + profile[k] = v + case []interface{}: + selectors := make([]string, 0) + for _, sel := range v { + selectors = append(selectors, sel.(string)) + } + profile[k] = strings.Join(selectors, " ") + default: + return nil, fmt.Errorf("unknown data type: %v", v) + } + } + return profile, nil +} + func (r *Redis) createUserOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { var ( object = UserInfo{} @@ -399,7 +458,7 @@ func (r *Redis) managePrivillege(ctx context.Context, req *bindings.InvokeReques object = UserInfo{} cmdRend = func(user UserInfo) string { - command := roleName2RedisPriv(op, user.RoleName) + command := r.role2Priv(op, user.RoleName) return fmt.Sprintf("ACL SETUSER %s %s", user.UserName, command) } @@ -418,7 +477,7 @@ func (r *Redis) managePrivillege(ctx context.Context, req *bindings.InvokeReques return ExecuteObject(ctx, r, req, op, cmdRend, msgTplRend, object) } -func roleName2RedisPriv(op bindings.OperationKind, roleName string) string { +func (r *Redis) role2Priv(op bindings.OperationKind, roleName string) string { const ( grantPrefix = "+" revokePrefix = "-" @@ -430,22 +489,23 @@ func roleName2RedisPriv(op bindings.OperationKind, roleName string) string { prefix = revokePrefix } var command string - switch roleName { - case ReadOnlyRole: - command = fmt.Sprintf("-@all %s@read allkeys", prefix) - case ReadWriteRole: - command = fmt.Sprintf("-@all %s@write %s@read allkeys", prefix, prefix) + + roleType := String2RoleType(roleName) + switch roleType { case SuperUserRole: command = fmt.Sprintf("%s@all allkeys", prefix) + case ReadWriteRole: + command = fmt.Sprintf("-@all %s@write %s@read allkeys", prefix, prefix) + case ReadOnlyRole: + command = fmt.Sprintf("-@all %s@read allkeys", prefix) } return command } -func redisPriv2RoleName(commands string) string { +func (r *Redis) priv2Role(commands string) RoleType { if commands == "-@all" { return NoPrivileges } - switch commands { case "-@all +@read ~*": return ReadOnlyRole diff --git a/cmd/probe/internal/binding/redis/redis_test.go b/cmd/probe/internal/binding/redis/redis_test.go index f5ba6fe7d..4bb7c87e0 100644 --- a/cmd/probe/internal/binding/redis/redis_test.go +++ b/cmd/probe/internal/binding/redis/redis_test.go @@ -299,7 +299,7 @@ func TestRedisAccounts(t *testing.T) { testName: "validInput", testMetaData: map[string]string{ "userName": userName, - "roleName": roleName, + "roleName": (string)(roleName), }, expectEveType: RespEveSucc, }, @@ -307,7 +307,7 @@ func TestRedisAccounts(t *testing.T) { for _, ops := range []bindings.OperationKind{GrantUserRoleOp, RevokeUserRoleOp} { // mock exepctation - args := tokenizeCmd2Args(fmt.Sprintf("ACL SETUSER %s %s", userName, roleName2RedisPriv(ops, roleName))) + args := tokenizeCmd2Args(fmt.Sprintf("ACL SETUSER %s %s", userName, r.role2Priv(ops, (string)(roleName)))) mock.ExpectDo(args...).SetVal("ok") request := &bindings.InvokeRequest{ @@ -353,6 +353,15 @@ func TestRedisAccounts(t *testing.T) { "selectors", []interface{}{}, } + + userInfoMap = map[string]interface{}{ + "flags": []interface{}{"on"}, + "passwords": []interface{}{"mock-password"}, + "commands": "+@all", + "keys": "~*", + "channels": "", + "selectors": []interface{}{}, + } ) testCases := []redisTestCase{ @@ -383,10 +392,18 @@ func TestRedisAccounts(t *testing.T) { }, expectEveType: RespEveSucc, }, + { + testName: "validInputAsMap", + testMetaData: map[string]string{ + "userName": userName, + }, + expectEveType: RespEveSucc, + }, } mock.ExpectDo("ACL", "GETUSER", userName).RedisNil() mock.ExpectDo("ACL", "GETUSER", userName).SetVal(userInfo) + mock.ExpectDo("ACL", "GETUSER", userName).SetVal(userInfoMap) for _, accTest := range testCases { request.Metadata = accTest.testMetaData @@ -407,7 +424,7 @@ func TestRedisAccounts(t *testing.T) { assert.Len(t, users, 1) user := users[0] assert.Equal(t, userName, user.UserName) - assert.Equal(t, SuperUserRole, user.RoleName) + assert.True(t, SuperUserRole.EqualTo(user.RoleName)) } } mock.ClearExpect() @@ -464,7 +481,7 @@ func TestRedisAccounts(t *testing.T) { t.Run("RoleName Conversion", func(t *testing.T) { type roleTestCase struct { - roleName string + roleName RoleType redisPrivs string } grantTestCases := []roleTestCase{ @@ -482,12 +499,12 @@ func TestRedisAccounts(t *testing.T) { }, } for _, test := range grantTestCases { - cmd := roleName2RedisPriv(GrantUserRoleOp, test.roleName) + cmd := r.role2Priv(GrantUserRoleOp, (string)(test.roleName)) assert.Equal(t, test.redisPrivs, cmd) // allkeys -> ~* cmd = strings.Replace(cmd, "allkeys", "~*", 1) - inferredRole := redisPriv2RoleName(cmd) + inferredRole := r.priv2Role(cmd) assert.Equal(t, test.roleName, inferredRole) } @@ -506,7 +523,7 @@ func TestRedisAccounts(t *testing.T) { }, } for _, test := range revokeTestCases { - cmd := roleName2RedisPriv(RevokeUserRoleOp, test.roleName) + cmd := r.role2Priv(RevokeUserRoleOp, (string)(test.roleName)) assert.Equal(t, test.redisPrivs, cmd) } }) diff --git a/cmd/probe/internal/binding/types.go b/cmd/probe/internal/binding/types.go index 0fbf45efd..eb3f0be1d 100644 --- a/cmd/probe/internal/binding/types.go +++ b/cmd/probe/internal/binding/types.go @@ -18,6 +18,7 @@ package binding import ( "fmt" + "strings" ) const ( @@ -61,19 +62,42 @@ const ( RespEveSucc = "Success" RespEveFail = "Failed" - SuperUserRole string = "superuser" - ReadWriteRole string = "readwrite" - ReadOnlyRole string = "readonly" - NoPrivileges string = "" - CustomizedRole string = "customized" - InvalidRole string = "invalid" - PRIMARY = "primary" SECONDARY = "secondary" MASTER = "master" SLAVE = "slave" ) +type RoleType string + +const ( + SuperUserRole RoleType = "superuser" + ReadWriteRole RoleType = "readwrite" + ReadOnlyRole RoleType = "readonly" + NoPrivileges RoleType = "" + CustomizedRole RoleType = "customized" + InvalidRole RoleType = "invalid" +) + +func (r RoleType) EqualTo(role string) bool { + return strings.EqualFold(string(r), role) +} + +func (r RoleType) GetWeight() int32 { + switch r { + case SuperUserRole: + return 1 << 3 + case ReadWriteRole: + return 1 << 2 + case ReadOnlyRole: + return 1 << 1 + case CustomizedRole: + return 1 + default: + return 0 + } +} + const ( errMsgNoSQL = "no sql provided" errMsgNoUserName = "no username provided" diff --git a/cmd/probe/internal/binding/utils.go b/cmd/probe/internal/binding/utils.go index 118928fa6..913bdd754 100644 --- a/cmd/probe/internal/binding/utils.go +++ b/cmd/probe/internal/binding/utils.go @@ -20,11 +20,9 @@ import ( "context" "encoding/json" "fmt" - "strings" "time" "github.com/dapr/components-contrib/bindings" - "golang.org/x/exp/slices" ) type UserInfo struct { @@ -163,11 +161,13 @@ func UserNameAndRoleValidator(user UserInfo) error { if len(user.RoleName) == 0 { return ErrNoRoleName } - roles := []string{ReadOnlyRole, ReadWriteRole, SuperUserRole} - if !slices.Contains(roles, strings.ToLower(user.RoleName)) { - return ErrInvalidRoleName + roles := []RoleType{ReadOnlyRole, ReadWriteRole, SuperUserRole} + for _, role := range roles { + if role.EqualTo(user.RoleName) { + return nil + } } - return nil + return ErrInvalidRoleName } func getAndFormatNow() string { @@ -189,3 +189,23 @@ func opsTerminateOnErr(result OpsResult, metadata opsMetadata, err error) (OpsRe result[RespTypMeta] = metadata return result, nil } + +func SortRoleByWeight(r1, r2 RoleType) bool { + return int(r1.GetWeight()) > int(r2.GetWeight()) +} + +func String2RoleType(roleName string) RoleType { + if SuperUserRole.EqualTo(roleName) { + return SuperUserRole + } + if ReadWriteRole.EqualTo(roleName) { + return ReadWriteRole + } + if ReadOnlyRole.EqualTo(roleName) { + return ReadOnlyRole + } + if NoPrivileges.EqualTo(roleName) { + return NoPrivileges + } + return CustomizedRole +} diff --git a/go.mod b/go.mod index 1b2ec0e5f..92113b6c7 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/hashicorp/terraform-exec v0.18.0 github.com/jackc/pgx/v5 v5.2.0 github.com/jedib0t/go-pretty/v6 v6.4.4 + github.com/json-iterator/go v1.1.12 github.com/k3d-io/k3d/v5 v5.4.4 github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0 github.com/leaanthony/debme v1.2.1 @@ -224,7 +225,6 @@ require ( github.com/jmoiron/sqlx v1.3.5 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect diff --git a/internal/controller/lifecycle/transformer_fill_class.go b/internal/controller/lifecycle/transformer_fill_class.go index 55ee0c104..81f76b946 100644 --- a/internal/controller/lifecycle/transformer_fill_class.go +++ b/internal/controller/lifecycle/transformer_fill_class.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package lifecycle import ( diff --git a/internal/controller/lifecycle/transformer_fix_cluster_labels.go b/internal/controller/lifecycle/transformer_fix_cluster_labels.go index 83b04dc21..cff0c5875 100644 --- a/internal/controller/lifecycle/transformer_fix_cluster_labels.go +++ b/internal/controller/lifecycle/transformer_fix_cluster_labels.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package lifecycle import ( From 997faa12147d8fd43da9a5a9b3c38259b4be355e Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Thu, 13 Apr 2023 12:15:12 +0800 Subject: [PATCH 015/439] chore: add built-in variable HEADLESS_SVC_FQDN (#2555) --- docs/release_notes/v0.5.0/v0.5.0.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/release_notes/v0.5.0/v0.5.0.md b/docs/release_notes/v0.5.0/v0.5.0.md index ea16a0f17..21f1fa8b4 100644 --- a/docs/release_notes/v0.5.0/v0.5.0.md +++ b/docs/release_notes/v0.5.0/v0.5.0.md @@ -25,6 +25,9 @@ Thanks to everyone who made this release possible! #### Easy of Use +* ClusterDefinition API `spec.connectionCredential` add following built-in variables: + * Headless service FQDN `$(HEADLESS_SVC_FQDN)` placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute + #### Resource Isolation From 2ad6df3f7ce9475d62580b9dadd4ec40cb15b664 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Thu, 13 Apr 2023 15:43:10 +0800 Subject: [PATCH 016/439] fix: get version failed when use kubeconfig with only a certain namespace permission (#2551) --- internal/cli/cmd/cluster/operations.go | 6 +++++- internal/cli/cmd/kubeblocks/install.go | 8 ++++---- internal/cli/util/provider.go | 2 +- internal/cli/util/provider_test.go | 2 +- internal/cli/util/util.go | 14 -------------- internal/cli/util/version.go | 6 +++--- internal/cli/util/version_test.go | 4 ++-- 7 files changed, 16 insertions(+), 26 deletions(-) diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 86305f56d..73c488563 100644 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -229,7 +229,11 @@ func (o *OperationsOptions) validateExpose() error { } func (o *OperationsOptions) fillExpose() error { - provider, err := util.GetK8SProvider(o.Client) + version, err := util.GetK8sVersion(o.Client.Discovery()) + if err != nil { + return err + } + provider, err := util.GetK8sProvider(version, o.Client) if err != nil { return err } diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index 06f09ecef..85b199912 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -360,16 +360,16 @@ func (o *InstallOptions) preCheck(versionInfo map[util.AppName]string) error { return versionErr } - version := util.GetK8sVersion(k8sVersionStr) - if len(version) == 0 { + semVer := util.GetK8sSemVer(k8sVersionStr) + if len(semVer) == 0 { return versionErr } // output kubernetes version - fmt.Fprintf(o.Out, "Kubernetes version %s\n", ""+version) + fmt.Fprintf(o.Out, "Kubernetes version %s\n", ""+semVer) // disable or enable some features according to the kubernetes environment - provider, err := util.GetK8sProvider(version, o.Client) + provider, err := util.GetK8sProvider(k8sVersionStr, o.Client) if err != nil { return fmt.Errorf("failed to get kubernetes provider: %v", err) } diff --git a/internal/cli/util/provider.go b/internal/cli/util/provider.go index 266a99468..f1bb46aac 100644 --- a/internal/cli/util/provider.go +++ b/internal/cli/util/provider.go @@ -129,7 +129,7 @@ func GetK8sProviderFromVersion(version string) K8sProvider { return UnknownProvider } -func GetK8sVersion(version string) string { +func GetK8sSemVer(version string) string { removeFirstChart := func(v string) string { if len(v) == 0 { return v diff --git a/internal/cli/util/provider_test.go b/internal/cli/util/provider_test.go index 5e9568164..26b5b1552 100644 --- a/internal/cli/util/provider_test.go +++ b/internal/cli/util/provider_test.go @@ -125,7 +125,7 @@ var _ = Describe("provider util", func() { for _, c := range cases { By(c.description) - Expect(GetK8sVersion(c.version)).Should(Equal(c.expectVersion)) + Expect(GetK8sSemVer(c.version)).Should(Equal(c.expectVersion)) client := testing.FakeClientSet(c.nodes) p, err := GetK8sProvider(c.version, client) Expect(err).ShouldNot(HaveOccurred()) diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index ea1533671..3bdecbe3d 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -640,20 +640,6 @@ func GetExposeAnnotations(provider K8sProvider, exposeType ExposeType) (map[stri return annotations, nil } -func GetK8SProvider(client kubernetes.Interface) (K8sProvider, error) { - versionInfo, err := GetVersionInfo(client) - if err != nil { - return "", err - } - - versionErr := fmt.Errorf("failed to get kubernetes version") - k8sVersionStr, ok := versionInfo[KubernetesApp] - if !ok { - return "", versionErr - } - return GetK8sProvider(k8sVersionStr, client) -} - // BuildAddonReleaseName returns the release name of addon, its f func BuildAddonReleaseName(addon string) string { return fmt.Sprintf("%s-%s", types.AddonReleasePrefix, addon) diff --git a/internal/cli/util/version.go b/internal/cli/util/version.go index bcbcd116f..33d638f3c 100644 --- a/internal/cli/util/version.go +++ b/internal/cli/util/version.go @@ -49,7 +49,7 @@ func GetVersionInfo(client kubernetes.Interface) (map[AppName]string, error) { return versionInfo, nil } - if versionInfo[KubernetesApp], err = getK8sVersion(client.Discovery()); err != nil { + if versionInfo[KubernetesApp], err = GetK8sVersion(client.Discovery()); err != nil { return versionInfo, err } @@ -79,8 +79,8 @@ func getKubeBlocksVersion(client kubernetes.Interface) (string, error) { return v, nil } -// getK8sVersion get k8s server version -func getK8sVersion(discoveryClient discovery.DiscoveryInterface) (string, error) { +// GetK8sVersion get k8s server version +func GetK8sVersion(discoveryClient discovery.DiscoveryInterface) (string, error) { if discoveryClient == nil { return "", nil } diff --git a/internal/cli/util/version_test.go b/internal/cli/util/version_test.go index 838dbcbe8..67fe21c61 100644 --- a/internal/cli/util/version_test.go +++ b/internal/cli/util/version_test.go @@ -79,9 +79,9 @@ var _ = Describe("version util", func() { Expect(err).Should(Succeed()) }) - It("getK8sVersion", func() { + It("GetK8sVersion", func() { client := testing.FakeClientSet() - v, err := getK8sVersion(client.Discovery()) + v, err := GetK8sVersion(client.Discovery()) Expect(v).ShouldNot(BeEmpty()) Expect(err).Should(Succeed()) }) From 80968c59b7438a739de2be2d1264d24d45e63b08 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Thu, 13 Apr 2023 16:07:27 +0800 Subject: [PATCH 017/439] chore: set chatgpt plugin version to 0.1.0 (#2558) --- deploy/chatgpt-retrieval-plugin/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/chatgpt-retrieval-plugin/values.yaml b/deploy/chatgpt-retrieval-plugin/values.yaml index 282dbf940..6f16f2248 100644 --- a/deploy/chatgpt-retrieval-plugin/values.yaml +++ b/deploy/chatgpt-retrieval-plugin/values.yaml @@ -9,7 +9,7 @@ image: repository: apecloud/chatgpt-retrieval-plugin pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "arm64-latest" + tag: "0.1.0" imagePullSecrets: [] nameOverride: "" From ea9461b92b1191a37fb0aaff2f98df3ffb2fe2a5 Mon Sep 17 00:00:00 2001 From: xingran Date: Thu, 13 Apr 2023 17:06:01 +0800 Subject: [PATCH 018/439] chore: support redis maxmemory setup with request memory (#2563) --- deploy/redis-cluster/values.yaml | 15 ++++---- deploy/redis/config/redis7-config.tpl | 6 +++ internal/controller/plan/builtin_functions.go | 9 +++++ internal/controller/plan/config_template.go | 38 ++++++++++--------- internal/controllerutil/pod_utils.go | 10 +++++ 5 files changed, 52 insertions(+), 26 deletions(-) diff --git a/deploy/redis-cluster/values.yaml b/deploy/redis-cluster/values.yaml index 43d04c8b3..896990c2a 100644 --- a/deploy/redis-cluster/values.yaml +++ b/deploy/redis-cluster/values.yaml @@ -18,18 +18,17 @@ primaryIndex: 0 switchPolicy: type: Noop -resources: { } +resources: # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - - # limits: - # cpu: 500m - # memory: 2Gi - # requests: - # cpu: 100m - # memory: 1Gi + limits: + cpu: 500m + memory: 3Gi + requests: + cpu: 500m + memory: 1Gi persistence: enabled: true diff --git a/deploy/redis/config/redis7-config.tpl b/deploy/redis/config/redis7-config.tpl index 4f86d27fa..047dddc75 100644 --- a/deploy/redis/config/redis7-config.tpl +++ b/deploy/redis/config/redis7-config.tpl @@ -71,3 +71,9 @@ jemalloc-bg-thread yes enable-debug-command yes protected-mode no +# maxmemory +{{- $request_memory := getContainerRequestMemory ( index $.podSpec.containers 0 ) }} +{{- if gt $request_memory 0 }} +maxmemory {{ $request_memory }} +{{- end -}} + diff --git a/internal/controller/plan/builtin_functions.go b/internal/controller/plan/builtin_functions.go index 183c1e393..6baa4f215 100644 --- a/internal/controller/plan/builtin_functions.go +++ b/internal/controller/plan/builtin_functions.go @@ -211,6 +211,15 @@ func getContainerMemory(args interface{}) (int64, error) { return intctrlutil.GetMemorySize(*container), nil } +// getContainerRequestMemory for general built-in +func getContainerRequestMemory(args interface{}) (int64, error) { + container, err := fromJSONObject[corev1.Container](args) + if err != nil { + return 0, err + } + return intctrlutil.GetRequestMemorySize(*container), nil +} + // getArgByName for general built-in func getArgByName(args interface{}, argName string) string { // TODO Support parse command args diff --git a/internal/controller/plan/config_template.go b/internal/controller/plan/config_template.go index 88376f5e9..d360005a0 100644 --- a/internal/controller/plan/config_template.go +++ b/internal/controller/plan/config_template.go @@ -42,13 +42,14 @@ const ( // General Built-in functions const ( - builtInGetVolumeFunctionName = "getVolumePathByName" - builtInGetPvcFunctionName = "getPVCByName" - builtInGetEnvFunctionName = "getEnvByName" - builtInGetArgFunctionName = "getArgByName" - builtInGetPortFunctionName = "getPortByName" - builtInGetContainerFunctionName = "getContainerByName" - builtInGetContainerMemoryFunctionName = "getContainerMemory" + builtInGetVolumeFunctionName = "getVolumePathByName" + builtInGetPvcFunctionName = "getPVCByName" + builtInGetEnvFunctionName = "getEnvByName" + builtInGetArgFunctionName = "getArgByName" + builtInGetPortFunctionName = "getPortByName" + builtInGetContainerFunctionName = "getContainerByName" + builtInGetContainerMemoryFunctionName = "getContainerMemory" + builtInGetContainerRequestMemoryFunctionName = "getContainerRequestMemory" // BuiltinMysqlCalBufferFunctionName Mysql Built-in // TODO: This function migrate to configuration template @@ -172,17 +173,18 @@ func (c *configTemplateBuilder) injectBuiltInObjectsAndFunctions( func (c *configTemplateBuilder) injectBuiltInFunctions(component *component.SynthesizedComponent, task *intctrltypes.ReconcileTask) error { // TODO add built-in function c.builtInFunctions = &gotemplate.BuiltInObjectsFunc{ - builtInMysqlCalBufferFunctionName: calDBPoolSize, - builtInGetVolumeFunctionName: getVolumeMountPathByName, - builtInGetPvcFunctionName: getPVCByName, - builtInGetEnvFunctionName: wrapGetEnvByName(c, task), - builtInGetPortFunctionName: getPortByName, - builtInGetArgFunctionName: getArgByName, - builtInGetContainerFunctionName: getPodContainerByName, - builtInGetContainerMemoryFunctionName: getContainerMemory, - builtInGetCAFile: getCAFile, - builtInGetCertFile: getCertFile, - builtInGetKeyFile: getKeyFile, + builtInMysqlCalBufferFunctionName: calDBPoolSize, + builtInGetVolumeFunctionName: getVolumeMountPathByName, + builtInGetPvcFunctionName: getPVCByName, + builtInGetEnvFunctionName: wrapGetEnvByName(c, task), + builtInGetPortFunctionName: getPortByName, + builtInGetArgFunctionName: getArgByName, + builtInGetContainerFunctionName: getPodContainerByName, + builtInGetContainerMemoryFunctionName: getContainerMemory, + builtInGetContainerRequestMemoryFunctionName: getContainerRequestMemory, + builtInGetCAFile: getCAFile, + builtInGetCertFile: getCertFile, + builtInGetKeyFile: getKeyFile, } return nil } diff --git a/internal/controllerutil/pod_utils.go b/internal/controllerutil/pod_utils.go index a79c588e6..00b32dbe6 100644 --- a/internal/controllerutil/pod_utils.go +++ b/internal/controllerutil/pod_utils.go @@ -217,6 +217,16 @@ func GetMemorySize(container corev1.Container) int64 { return 0 } +// GetRequestMemorySize function description: +// if not Resource field, return 0 else Resources.Limits.memory +func GetRequestMemorySize(container corev1.Container) int64 { + requests := container.Resources.Requests + if val, ok := (requests)[corev1.ResourceMemory]; ok { + return val.Value() + } + return 0 +} + // PodIsReady check the pod is ready func PodIsReady(pod *corev1.Pod) bool { if pod.Status.Conditions == nil { From 5538108e89a9c1fb98ceb2305bb485c1e3e0ef43 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Thu, 13 Apr 2023 17:59:58 +0800 Subject: [PATCH 019/439] fix: update redis clusterdef for accounts config (#2566) --- deploy/redis/templates/clusterdefinition.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/redis/templates/clusterdefinition.yaml b/deploy/redis/templates/clusterdefinition.yaml index f0b76f20f..4b5772384 100644 --- a/deploy/redis/templates/clusterdefinition.yaml +++ b/deploy/redis/templates/clusterdefinition.yaml @@ -151,33 +151,33 @@ spec: - name: kbadmin provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) allcommands allkeys + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys - name: kbdataprotection provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) allcommands allkeys + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys - name: kbmonitoring provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) allkeys +get + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get - name: kbprobe provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) allkeys +get + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get - name: kbreplicator provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) +psync +replconf +ping + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) +psync +replconf +ping - name: redis-sentinel workloadType: Stateful characterType: redis From d3008da0e1d9c4793830ceca68eba08b7a9fa575 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Thu, 13 Apr 2023 19:05:29 +0800 Subject: [PATCH 020/439] chore: tidy up ClickHouse ClusterDefinition chart, fixes refactored API attribute names (#2556) --- apis/apps/v1alpha1/cluster_types.go | 5 +- .../bases/apps.kubeblocks.io_clusters.yaml | 5 +- .../configs/00_default_overrides.xml.tpl | 58 ++++++++++ .../ch-keeper_00_default_overrides.xml.tpl | 36 +++++++ deploy/clickhouse/templates/configmap.yaml | 4 +- deploy/clickhouse/values.yaml | 102 ------------------ .../crds/apps.kubeblocks.io_clusters.yaml | 5 +- 7 files changed, 105 insertions(+), 110 deletions(-) create mode 100644 deploy/clickhouse/configs/00_default_overrides.xml.tpl create mode 100644 deploy/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index 79dbd2aa4..40d11c66a 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -273,11 +273,10 @@ type ClusterSwitchPolicy struct { } type ClusterComponentVolumeClaimTemplate struct { - // Ref ClusterVersion.spec.components.containers.volumeMounts.name + // Reference `ClusterDefinition.spec.componentDefs.containers.volumeMounts.name`. // +kubebuilder:validation:Required Name string `json:"name"` // spec defines the desired characteristics of a volume requested by a pod author. - // +kubebuilder:pruning:PreserveUnknownFields // +optional Spec PersistentVolumeClaimSpec `json:"spec,omitempty"` } @@ -292,6 +291,7 @@ func (r *ClusterComponentVolumeClaimTemplate) toVolumeClaimTemplate() corev1.Per type PersistentVolumeClaimSpec struct { // accessModes contains the desired access modes the volume should have. // More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1 + // +kubebuilder:pruning:PreserveUnknownFields // +optional AccessModes []corev1.PersistentVolumeAccessMode `json:"accessModes,omitempty" protobuf:"bytes,1,rep,name=accessModes,casttype=PersistentVolumeAccessMode"` // resources represents the minimum resources the volume should have. @@ -299,6 +299,7 @@ type PersistentVolumeClaimSpec struct { // that are lower than previous value but must still be higher than capacity recorded in the // status field of the claim. // More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources + // +kubebuilder:pruning:PreserveUnknownFields // +optional Resources corev1.ResourceRequirements `json:"resources,omitempty" protobuf:"bytes,2,opt,name=resources"` // storageClassName is the name of the StorageClass required by the claim. diff --git a/config/crd/bases/apps.kubeblocks.io_clusters.yaml b/config/crd/bases/apps.kubeblocks.io_clusters.yaml index 68e463d91..6f8b269fa 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusters.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusters.yaml @@ -390,7 +390,7 @@ spec: items: properties: name: - description: Ref ClusterVersion.spec.components.containers.volumeMounts.name + description: Reference `ClusterDefinition.spec.componentDefs.containers.volumeMounts.name`. type: string spec: description: spec defines the desired characteristics @@ -402,6 +402,7 @@ spec: items: type: string type: array + x-kubernetes-preserve-unknown-fields: true resources: description: 'resources represents the minimum resources the volume should have. If RecoverVolumeExpansionFailure @@ -458,12 +459,12 @@ spec: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + x-kubernetes-preserve-unknown-fields: true storageClassName: description: 'storageClassName is the name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string type: object - x-kubernetes-preserve-unknown-fields: true required: - name type: object diff --git a/deploy/clickhouse/configs/00_default_overrides.xml.tpl b/deploy/clickhouse/configs/00_default_overrides.xml.tpl new file mode 100644 index 000000000..6c6861071 --- /dev/null +++ b/deploy/clickhouse/configs/00_default_overrides.xml.tpl @@ -0,0 +1,58 @@ +{{- $clusterName := $.cluster.metadata.name }} +{{- $namespace := $.cluster.metadata.namespace }} + + + + + + {{ $clusterName }} + + + + information + + + + +{{- range $.cluster.spec.componentSpecs }} + {{ $compIter := . }} + {{- if eq $compIter.componentDefRef "clickhouse" }} + + {{- $replicas := $compIter.replicas | int }} + {{- range $i, $_e := until $replicas }} + + {{ $clusterName }}-{{ $compIter.name }}-{{ $i }}.{{ $clusterName }}-{{ $compIter.name }}-headless.{{ $namespace }}.svc + 9000 + + {{- end }} + + {{- end }} +{{- end }} + + +{{- range $.cluster.spec.componentSpecs }} + {{ $compIter := . }} + {{- if or (eq $compIter.componentDefRef "zookeeper") (eq $compIter.componentDefRef "ch-keeper") }} + + + {{- $replicas := $compIter.replicas | int }} + {{- range $i, $_e := until $replicas }} + + {{ $clusterName }}-{{ $compIter.name }}-{{ $i }}.{{ $clusterName }}-{{ $compIter.name }}-headless.{{ $namespace }}.svc + 2181 + + {{- end }} + + {{- end }} +{{- end }} +{{- if $.component.monitor.enable }} + + + /metrics + + true + true + true + +{{- end }} + diff --git a/deploy/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl b/deploy/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl new file mode 100644 index 000000000..3bc150f35 --- /dev/null +++ b/deploy/clickhouse/configs/ch-keeper_00_default_overrides.xml.tpl @@ -0,0 +1,36 @@ +{{- $clusterName := $.cluster.metadata.name }} +{{- $namespace := $.cluster.metadata.namespace }} + + 0.0.0.0 + + + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + 10000 + 30000 + warning + + +{{- $replicas := $.component.replicas | int }} +{{- range $i, $e := until $replicas }} + + {{ $i | int | add1 }} + {{ $clusterName }}-{{ $.component.name }}-{{ $i }}.{{ $clusterName }}-{{ $.component.name }}-headless.{{ $namespace }}.svc + + +{{- end }} + + +{{- if $.component.monitor.enable }} + + + /metrics + + true + true + true + +{{- end }} + \ No newline at end of file diff --git a/deploy/clickhouse/templates/configmap.yaml b/deploy/clickhouse/templates/configmap.yaml index d075e14e6..6b2334c6c 100644 --- a/deploy/clickhouse/templates/configmap.yaml +++ b/deploy/clickhouse/templates/configmap.yaml @@ -11,7 +11,7 @@ metadata: {{- end }} data: 00_default_overrides.xml: | - {{- .Values.defaultConfigurationOverrides | nindent 4 }} + {{- .Files.Get "configs/00_default_overrides.xml.tpl" | nindent 4 }} --- apiVersion: v1 kind: ConfigMap @@ -26,7 +26,7 @@ metadata: {{- end }} data: 00_default_overrides.xml: | - {{- .Values.clickHouseKeeper.configuration | nindent 4 }} + {{- .Files.Get "configs/ch-keeper_00_default_overrides.xml.tpl" | nindent 4 }} --- {{- if .Values.zookeeper.configuration }} apiVersion: v1 diff --git a/deploy/clickhouse/values.yaml b/deploy/clickhouse/values.yaml index 15f0983d0..d7422c615 100644 --- a/deploy/clickhouse/values.yaml +++ b/deploy/clickhouse/values.yaml @@ -14,108 +14,6 @@ commonLabels: {} logConfigs: {} -## @param defaultConfigurationOverrides [string] Default configuration overrides (evaluated as a template) -## -defaultConfigurationOverrides: | - {{- $clusterName := $.cluster.metadata.name }} - {{- $namespace := $.cluster.metadata.namespace }} - - - - - - {{ $clusterName }} - - - - information - - - - - {{- range $.cluster.spec.components }} - {{ $compIter := . }} - {{- if eq $compIter.type "clickhouse" }} - - {{- $replicas := $compIter.replicas | int }} - {{- range $i, $_e := until $replicas }} - - {{ $clusterName }}-{{ $compIter.name }}-{{ $i }}.{{ $clusterName }}-{{ $compIter.name }}-headless.{{ $namespace }}.svc - 9000 - - {{- end }} - - {{- end }} - {{- end }} - - - {{- range $.cluster.spec.components }} - {{ $compIter := . }} - {{- if or (eq $compIter.type "zookeeper") (eq $compIter.type "ch-keeper") }} - - - {{- $replicas := $compIter.replicas | int }} - {{- range $i, $_e := until $replicas }} - - {{ $clusterName }}-{{ $compIter.name }}-{{ $i }}.{{ $clusterName }}-{{ $compIter.name }}-headless.{{ $namespace }}.svc - 2181 - - {{- end }} - - {{- end }} - {{- end }} - {{- if $.component.monitor.enable }} - - - /metrics - - true - true - true - - {{- end }} - - - -clickHouseKeeper: - configuration: | - {{- $clusterName := $.cluster.metadata.name }} - {{- $namespace := $.cluster.metadata.namespace }} - - 0.0.0.0 - - - 1 - /var/lib/clickhouse/coordination/log - /var/lib/clickhouse/coordination/snapshots - - 10000 - 30000 - warning - - - {{- $replicas := $.component.replicas | int }} - {{- range $i, $e := until $replicas }} - - {{ $i | int | add1 }} - {{ $clusterName }}-{{ $.component.name }}-{{ $i }}.{{ $clusterName }}-{{ $.component.name }}-headless.{{ $namespace }}.svc - - - {{- end }} - - - {{- if $.component.monitor.enable }} - - - /metrics - - true - true - true - - {{- end }} - - image: repository: docker.io/bitnami/clickhouse diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml index 68e463d91..6f8b269fa 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml @@ -390,7 +390,7 @@ spec: items: properties: name: - description: Ref ClusterVersion.spec.components.containers.volumeMounts.name + description: Reference `ClusterDefinition.spec.componentDefs.containers.volumeMounts.name`. type: string spec: description: spec defines the desired characteristics @@ -402,6 +402,7 @@ spec: items: type: string type: array + x-kubernetes-preserve-unknown-fields: true resources: description: 'resources represents the minimum resources the volume should have. If RecoverVolumeExpansionFailure @@ -458,12 +459,12 @@ spec: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' type: object type: object + x-kubernetes-preserve-unknown-fields: true storageClassName: description: 'storageClassName is the name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' type: string type: object - x-kubernetes-preserve-unknown-fields: true required: - name type: object From fe64a843067a49da229cc6301e6bcbf6cef1490c Mon Sep 17 00:00:00 2001 From: "zheyi.cqy" Date: Thu, 13 Apr 2023 19:42:54 +0800 Subject: [PATCH 021/439] feat: kbcli adds migration related commands (#2465) Co-authored-by: caiqy321 --- docs/release_notes/v0.5.0/v0.5.0.md | 1 + docs/user_docs/cli/cli.md | 12 + docs/user_docs/cli/kbcli.md | 1 + internal/cli/cmd/cli.go | 2 + internal/cli/cmd/migration/base.go | 224 ++++++++++++++ internal/cli/cmd/migration/base_test.go | 48 +++ internal/cli/cmd/migration/cmd_builder.go | 41 +++ .../cli/cmd/migration/cmd_builder_test.go | 23 ++ internal/cli/cmd/migration/create.go | 273 ++++++++++++++++++ internal/cli/cmd/migration/create_test.go | 163 +++++++++++ internal/cli/cmd/migration/describe.go | 271 +++++++++++++++++ internal/cli/cmd/migration/describe_test.go | 23 ++ internal/cli/cmd/migration/examples.go | 88 ++++++ internal/cli/cmd/migration/list.go | 31 ++ internal/cli/cmd/migration/list_test.go | 23 ++ internal/cli/cmd/migration/logs.go | 213 ++++++++++++++ internal/cli/cmd/migration/logs_test.go | 23 ++ internal/cli/cmd/migration/suite_test.go | 13 + internal/cli/cmd/migration/templates.go | 31 ++ internal/cli/cmd/migration/templates_test.go | 23 ++ internal/cli/cmd/migration/terminate.go | 38 +++ internal/cli/cmd/migration/terminate_test.go | 23 ++ .../create/template/migration_template.cue | 89 ++++++ .../migrationapi/migration_object_express.go | 93 ++++++ .../types/migrationapi/migrationtask_types.go | 153 ++++++++++ .../migrationapi/migrationtemplate_types.go | 103 +++++++ internal/cli/types/migrationapi/type.go | 208 +++++++++++++ internal/cli/types/types.go | 39 +++ internal/cli/util/util.go | 18 +- internal/cli/util/util_test.go | 3 + 30 files changed, 2293 insertions(+), 1 deletion(-) create mode 100644 internal/cli/cmd/migration/base.go create mode 100644 internal/cli/cmd/migration/base_test.go create mode 100644 internal/cli/cmd/migration/cmd_builder.go create mode 100644 internal/cli/cmd/migration/cmd_builder_test.go create mode 100644 internal/cli/cmd/migration/create.go create mode 100644 internal/cli/cmd/migration/create_test.go create mode 100644 internal/cli/cmd/migration/describe.go create mode 100644 internal/cli/cmd/migration/describe_test.go create mode 100644 internal/cli/cmd/migration/examples.go create mode 100644 internal/cli/cmd/migration/list.go create mode 100644 internal/cli/cmd/migration/list_test.go create mode 100644 internal/cli/cmd/migration/logs.go create mode 100644 internal/cli/cmd/migration/logs_test.go create mode 100644 internal/cli/cmd/migration/suite_test.go create mode 100644 internal/cli/cmd/migration/templates.go create mode 100644 internal/cli/cmd/migration/templates_test.go create mode 100644 internal/cli/cmd/migration/terminate.go create mode 100644 internal/cli/cmd/migration/terminate_test.go create mode 100644 internal/cli/create/template/migration_template.cue create mode 100644 internal/cli/types/migrationapi/migration_object_express.go create mode 100644 internal/cli/types/migrationapi/migrationtask_types.go create mode 100644 internal/cli/types/migrationapi/migrationtemplate_types.go create mode 100644 internal/cli/types/migrationapi/type.go diff --git a/docs/release_notes/v0.5.0/v0.5.0.md b/docs/release_notes/v0.5.0/v0.5.0.md index 21f1fa8b4..0dde6e548 100644 --- a/docs/release_notes/v0.5.0/v0.5.0.md +++ b/docs/release_notes/v0.5.0/v0.5.0.md @@ -14,6 +14,7 @@ Thanks to everyone who made this release possible! ### New Features #### PostgreSQL +- Support incremental migration from AWS RDS to KubeBlocks, support pre-check, full migration and incremental synchronization #### Redis diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index ee80129a3..6b1ca96cd 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -126,6 +126,18 @@ KubeBlocks operation commands. * [kbcli kubeblocks upgrade](kbcli_kubeblocks_upgrade.md) - Upgrade KubeBlocks. +## [migration](kbcli_migration.md) + +Data migration between two data sources. + +* [kbcli migration create](kbcli_migration_create.md) - Create a migration task. +* [kbcli migration describe](kbcli_migration_describe.md) - Show details of a specific migration task. +* [kbcli migration list](kbcli_migration_list.md) - List migration tasks. +* [kbcli migration logs](kbcli_migration_logs.md) - Access migration task log file. +* [kbcli migration templates](kbcli_migration_templates.md) - List migration templates. +* [kbcli migration terminate](kbcli_migration_terminate.md) - Delete migration task. + + ## [options](kbcli_options.md) Print the list of flags inherited by all commands. diff --git a/docs/user_docs/cli/kbcli.md b/docs/user_docs/cli/kbcli.md index 8935f3900..d8a45fd46 100644 --- a/docs/user_docs/cli/kbcli.md +++ b/docs/user_docs/cli/kbcli.md @@ -64,6 +64,7 @@ kbcli [flags] * [kbcli clusterversion](kbcli_clusterversion.md) - ClusterVersion command. * [kbcli dashboard](kbcli_dashboard.md) - List and open the KubeBlocks dashboards. * [kbcli kubeblocks](kbcli_kubeblocks.md) - KubeBlocks operation commands. +* [kbcli migration](kbcli_migration.md) - Data migration between two data sources. * [kbcli options](kbcli_options.md) - Print the list of flags inherited by all commands. * [kbcli playground](kbcli_playground.md) - Bootstrap a playground KubeBlocks in local host or cloud. * [kbcli version](kbcli_version.md) - Print the version information, include kubernetes, KubeBlocks and kbcli version. diff --git a/internal/cli/cmd/cli.go b/internal/cli/cmd/cli.go index 394fca51f..ea1a2a4e5 100644 --- a/internal/cli/cmd/cli.go +++ b/internal/cli/cmd/cli.go @@ -38,6 +38,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/cmd/clusterversion" "github.com/apecloud/kubeblocks/internal/cli/cmd/dashboard" "github.com/apecloud/kubeblocks/internal/cli/cmd/kubeblocks" + "github.com/apecloud/kubeblocks/internal/cli/cmd/migration" "github.com/apecloud/kubeblocks/internal/cli/cmd/options" "github.com/apecloud/kubeblocks/internal/cli/cmd/playground" "github.com/apecloud/kubeblocks/internal/cli/cmd/version" @@ -102,6 +103,7 @@ A Command Line Interface for KubeBlocks`, class.NewClassCommand(f, ioStreams), alert.NewAlertCmd(f, ioStreams), addon.NewAddonCmd(f, ioStreams), + migration.NewMigrationCmd(f, ioStreams), ) filters := []string{"options"} diff --git a/internal/cli/cmd/migration/base.go b/internal/cli/cmd/migration/base.go new file mode 100644 index 000000000..dc20d46ab --- /dev/null +++ b/internal/cli/cmd/migration/base.go @@ -0,0 +1,224 @@ +package migration + +import ( + "context" + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/apecloud/kubeblocks/internal/cli/types" + migrationv1 "github.com/apecloud/kubeblocks/internal/cli/types/migrationapi" +) + +const ( + MigrationTaskLabel = "datamigration.apecloud.io/migrationtask" + MigrationTaskStepAnnotation = "datamigration.apecloud.io/step" + SerialJobOrderAnnotation = "common.apecloud.io/serial_job_order" +) + +// Endpoint +// Todo: For the source or target is cluster in KubeBlocks. A better way is to get secret from {$clustername}-conn-credential, so the username, password, addresses can be omitted + +type EndpointModel struct { + UserName string `json:"userName"` + Password string `json:"password"` + Address string `json:"address"` + // +optional + Database string `json:"databaseName,omitempty"` +} + +func (e *EndpointModel) BuildFromStr(msgArr *[]string, endpointStr string) error { + if endpointStr == "" { + BuildErrorMsg(msgArr, "endpoint string can not be empty") + return nil + } + e.clear() + endpointStr = strings.TrimSpace(endpointStr) + accountURLPair := strings.Split(endpointStr, "@") + if len(accountURLPair) != 2 { + BuildErrorMsg(msgArr, "endpoint maybe does not contain account info") + return nil + } + accountPair := strings.Split(accountURLPair[0], ":") + if len(accountPair) != 2 { + BuildErrorMsg(msgArr, "the account info in endpoint is invalid, should be like \"user:123456\"") + return nil + } + e.UserName = accountPair[0] + e.Password = accountPair[1] + if strings.LastIndex(accountURLPair[1], "/") != -1 { + addressDatabasePair := strings.Split(accountURLPair[1], "/") + e.Address = strings.Join(addressDatabasePair[:len(addressDatabasePair)-1], "/") + e.Database = addressDatabasePair[len(addressDatabasePair)-1] + } else { + e.Address = accountURLPair[1] + } + return nil +} + +func (e *EndpointModel) clear() { + e.Address = "" + e.Password = "" + e.UserName = "" + e.Database = "" +} + +// Migration Object + +type MigrationObjectModel struct { + WhiteList []DBObjectExpress `json:"whiteList"` +} + +type DBObjectExpress struct { + SchemaName string `json:"schemaName"` + // +optional + IsAll bool `json:"isAll"` + // +optional + TableList []TableObjectExpress `json:"tableList"` +} + +type TableObjectExpress struct { + TableName string `json:"tableName"` + // +optional + IsAll bool `json:"isAll"` +} + +func (m *MigrationObjectModel) BuildFromStrs(errMsgArr *[]string, objStrs []string) error { + if len(objStrs) == 0 { + BuildErrorMsg(errMsgArr, "migration object can not be empty") + return nil + } + for _, str := range objStrs { + msg := "" + if str == "" { + msg = "the database or database.table in migration object can not be empty" + } + dbTablePair := strings.Split(str, ".") + if len(dbTablePair) > 2 { + msg = fmt.Sprintf("[%s] is not a valid database or database.table", str) + } + if msg != "" { + BuildErrorMsg(errMsgArr, msg) + return nil + } + if len(dbTablePair) == 1 { + m.WhiteList = append(m.WhiteList, DBObjectExpress{ + SchemaName: str, + IsAll: true, + }) + } else { + dbObjPoint, err := m.ContainSchema(dbTablePair[0]) + if err != nil { + return err + } + if dbObjPoint != nil { + dbObjPoint.TableList = append(dbObjPoint.TableList, TableObjectExpress{ + TableName: dbTablePair[1], + IsAll: true, + }) + } else { + m.WhiteList = append(m.WhiteList, DBObjectExpress{ + SchemaName: dbTablePair[0], + TableList: []TableObjectExpress{{ + TableName: dbTablePair[1], + IsAll: true, + }}, + }) + } + } + } + return nil +} + +func (m *MigrationObjectModel) ContainSchema(schemaName string) (*DBObjectExpress, error) { + for i := 0; i < len(m.WhiteList); i++ { + if m.WhiteList[i].SchemaName == schemaName { + return &m.WhiteList[i], nil + } + } + return nil, nil +} + +func CliStepChangeToStructure() (map[string]string, []string) { + validStepMap := map[string]string{ + migrationv1.CliStepPreCheck.String(): migrationv1.CliStepPreCheck.String(), + migrationv1.CliStepInitStruct.String(): migrationv1.CliStepInitStruct.String(), + migrationv1.CliStepInitData.String(): migrationv1.CliStepInitData.String(), + migrationv1.CliStepCdc.String(): migrationv1.CliStepCdc.String(), + } + validStepKey := make([]string, 0) + for k := range validStepMap { + validStepKey = append(validStepKey, k) + } + return validStepMap, validStepKey +} + +type TaskTypeEnum string + +const ( + Initialization TaskTypeEnum = "initialization" + InitializationAndCdc TaskTypeEnum = "initialization-and-cdc" // default value +) + +func (s TaskTypeEnum) String() string { + return string(s) +} + +func IsMigrationCrdValidWithDynamic(dynamic *dynamic.Interface) (bool, error) { + resource := types.CustomResourceDefinitionGVR() + if err := APIResource(dynamic, &resource, "migrationtasks.datamigration.apecloud.io", "", nil); err != nil { + return false, err + } + if err := APIResource(dynamic, &resource, "migrationtemplates.datamigration.apecloud.io", "", nil); err != nil { + return false, err + } + if err := APIResource(dynamic, &resource, "serialjobs.common.apecloud.io", "", nil); err != nil { + return false, err + } + return true, nil +} + +func IsMigrationCrdValidWithFactory(factory cmdutil.Factory) (bool, error) { + dynamic, err := factory.DynamicClient() + if err != nil { + return false, err + } + return IsMigrationCrdValidWithDynamic(&dynamic) +} + +func APIResource(dynamic *dynamic.Interface, resource *schema.GroupVersionResource, name string, namespace string, res interface{}) error { + obj, err := (*dynamic).Resource(*resource).Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{}, "") + if err != nil { + return err + } + if res != nil { + return runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, res) + } + return nil +} + +func BuildErrorMsg(msgArr *[]string, msg string) { + if *msgArr == nil { + *msgArr = make([]string, 1) + } + *msgArr = append(*msgArr, msg) +} + +func BuildInitializationStepsOrder(task *migrationv1.MigrationTask, template *migrationv1.MigrationTemplate) []string { + stepMap := make(map[string]string) + for _, taskStep := range task.Spec.Initialization.Steps { + stepMap[taskStep.String()] = taskStep.String() + } + resultArr := make([]string, 0) + for _, stepModel := range template.Spec.Initialization.Steps { + if stepMap[stepModel.Step.String()] != "" { + resultArr = append(resultArr, stepModel.Step.CliString()) + } + } + return resultArr +} diff --git a/internal/cli/cmd/migration/base_test.go b/internal/cli/cmd/migration/base_test.go new file mode 100644 index 000000000..3cbccbdf3 --- /dev/null +++ b/internal/cli/cmd/migration/base_test.go @@ -0,0 +1,48 @@ +package migration + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + v1alpha1 "github.com/apecloud/kubeblocks/internal/cli/types/migrationapi" +) + +var _ = Describe("base", func() { + + Context("Basic function validate", func() { + + It("CliStepChangeToStructure", func() { + resultMap, resultKeyArr := CliStepChangeToStructure() + Expect(len(resultMap)).Should(Equal(4)) + Expect(len(resultKeyArr)).Should(Equal(4)) + }) + + It("BuildInitializationStepsOrder", func() { + task := &v1alpha1.MigrationTask{ + Spec: v1alpha1.MigrationTaskSpec{ + Initialization: v1alpha1.InitializationConfig{ + Steps: []v1alpha1.StepEnum{ + v1alpha1.StepFullLoad, + v1alpha1.StepStructPreFullLoad, + }, + }, + }, + } + template := &v1alpha1.MigrationTemplate{ + Spec: v1alpha1.MigrationTemplateSpec{ + Initialization: v1alpha1.InitializationModel{ + Steps: []v1alpha1.StepModel{ + {Step: v1alpha1.StepStructPreFullLoad}, + {Step: v1alpha1.StepFullLoad}, + }, + }, + }, + } + arr := BuildInitializationStepsOrder(task, template) + Expect(len(arr)).Should(Equal(2)) + Expect(arr[0]).Should(Equal(v1alpha1.StepStructPreFullLoad.CliString())) + Expect(arr[1]).Should(Equal(v1alpha1.StepFullLoad.CliString())) + }) + }) + +}) diff --git a/internal/cli/cmd/migration/cmd_builder.go b/internal/cli/cmd/migration/cmd_builder.go new file mode 100644 index 000000000..343f889a4 --- /dev/null +++ b/internal/cli/cmd/migration/cmd_builder.go @@ -0,0 +1,41 @@ +package migration + +import ( + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" +) + +// NewMigrationCmd creates the cluster command +func NewMigrationCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "migration", + Short: "Data migration between two data sources.", + } + + groups := templates.CommandGroups{ + { + Message: "Basic Migration Commands:", + Commands: []*cobra.Command{ + NewMigrationCreateCmd(f, streams), + NewMigrationTemplatesCmd(f, streams), + NewMigrationListCmd(f, streams), + NewMigrationTerminateCmd(f, streams), + }, + }, + { + Message: "Migration Operation Commands:", + Commands: []*cobra.Command{ + NewMigrationDescribeCmd(f, streams), + NewMigrationLogsCmd(f, streams), + }, + }, + } + + // add subcommands + groups.Add(cmd) + templates.ActsAsRootCommand(cmd, nil, groups...) + + return cmd +} diff --git a/internal/cli/cmd/migration/cmd_builder_test.go b/internal/cli/cmd/migration/cmd_builder_test.go new file mode 100644 index 000000000..eb5ff5136 --- /dev/null +++ b/internal/cli/cmd/migration/cmd_builder_test.go @@ -0,0 +1,23 @@ +package migration + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" +) + +var _ = Describe("cmd_builder", func() { + + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + ) + + It("command build", func() { + cmd := NewMigrationCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + +}) diff --git a/internal/cli/cmd/migration/create.go b/internal/cli/cmd/migration/create.go new file mode 100644 index 000000000..4b668d009 --- /dev/null +++ b/internal/cli/cmd/migration/create.go @@ -0,0 +1,273 @@ +package migration + +import ( + "fmt" + "strings" + "time" + + "github.com/spf13/cobra" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/types" + migrationv1 "github.com/apecloud/kubeblocks/internal/cli/types/migrationapi" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var ( + AllStepsArr = []string{ + migrationv1.CliStepGlobal.String(), + migrationv1.CliStepPreCheck.String(), + migrationv1.CliStepCdc.String(), + migrationv1.CliStepInitStruct.String(), + migrationv1.CliStepInitData.String(), + } +) + +const ( + StringBoolTrue = "true" + StringBoolFalse = "false" +) + +type CreateMigrationOptions struct { + Template string `json:"template"` + TaskType string `json:"taskType,omitempty"` + Source string `json:"source"` + SourceEndpointModel EndpointModel `json:"sourceEndpointModel,omitempty"` + Sink string `json:"sink"` + SinkEndpointModel EndpointModel `json:"sinkEndpointModel,omitempty"` + MigrationObject []string `json:"migrationObject"` + MigrationObjectModel MigrationObjectModel `json:"migrationObjectModel,omitempty"` + Steps []string `json:"steps,omitempty"` + StepsModel []string `json:"stepsModel,omitempty"` + Tolerations []string `json:"tolerations,omitempty"` + TolerationModel map[string][]interface{} `json:"tolerationModel,omitempty"` + Resources []string `json:"resources,omitempty"` + ResourceModel map[string]interface{} `json:"resourceModel,omitempty"` + ServerID uint32 `json:"serverId,omitempty"` + create.BaseOptions +} + +func NewMigrationCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := &CreateMigrationOptions{BaseOptions: create.BaseOptions{IOStreams: streams}} + inputs := create.Inputs{ + Use: "create name", + Short: "Create a migration task.", + Example: CreateTemplate, + CueTemplateName: "migration_template.cue", + ResourceName: types.ResourceMigrationTasks, + Group: types.MigrationAPIGroup, + Version: types.MigrationAPIVersion, + BaseOptionsObj: &o.BaseOptions, + Options: o, + Factory: f, + Validate: o.Validate, + ResourceNameGVRForCompletion: types.MigrationTaskGVR(), + BuildFlags: func(cmd *cobra.Command) { + cmd.Flags().StringVar(&o.Template, "template", "", "Specify migration template, run \"kbcli migration templates\" to show all available migration templates") + cmd.Flags().StringVar(&o.Source, "source", "", "Set the source database information for migration.such as '{username}:{password}@{connection_address}:{connection_port}/[{database}]'") + cmd.Flags().StringVar(&o.Sink, "sink", "", "Set the sink database information for migration.such as '{username}:{password}@{connection_address}:{connection_port}/[{database}]") + cmd.Flags().StringSliceVar(&o.MigrationObject, "migration-object", []string{}, "Set the data objects that need to be migrated,such as '\"db1.table1\",\"db2\"'") + cmd.Flags().StringSliceVar(&o.Steps, "steps", []string{}, "Set up migration steps,such as: precheck=true,init-struct=true,init-data=true,cdc=true") + cmd.Flags().StringSliceVar(&o.Tolerations, "tolerations", []string{}, "Tolerations for migration, such as '\"key=engineType,value=pg,operator=Equal,effect=NoSchedule\"'") + cmd.Flags().StringSliceVar(&o.Resources, "resources", []string{}, "Resources limit for migration, such as '\"cpu=3000m,memory=3Gi\"'") + + util.CheckErr(cmd.MarkFlagRequired("template")) + util.CheckErr(cmd.MarkFlagRequired("source")) + util.CheckErr(cmd.MarkFlagRequired("sink")) + util.CheckErr(cmd.MarkFlagRequired("migration-object")) + }, + } + return create.BuildCommand(inputs) +} + +func (o *CreateMigrationOptions) Validate() error { + var err error + + _, err = IsMigrationCrdValidWithDynamic(&o.Dynamic) + if errors.IsNotFound(err) { + return fmt.Errorf("datamigration crd is not install") + } else if err != nil { + return err + } + + if o.Template == "" { + return fmt.Errorf("migration template is needed, use \"kbcli migration templates\" to check and special one") + } + + errMsgArr := make([]string, 0) + // Source + o.SourceEndpointModel = EndpointModel{} + if err = o.SourceEndpointModel.BuildFromStr(&errMsgArr, o.Source); err != nil { + return err + } + // Sink + o.SinkEndpointModel = EndpointModel{} + if err = o.SinkEndpointModel.BuildFromStr(&errMsgArr, o.Sink); err != nil { + return err + } + + // MigrationObject + if err = o.MigrationObjectModel.BuildFromStrs(&errMsgArr, o.MigrationObject); err != nil { + return err + } + + // Steps & taskType + if err = o.BuildWithSteps(&errMsgArr); err != nil { + return err + } + + // Tolerations + if err = o.BuildWithTolerations(); err != nil { + return err + } + + // Resources + if err = o.BuildWithResources(); err != nil { + return err + } + + // RuntimeParams + if err = o.BuildWithRuntimeParams(); err != nil { + return err + } + + // Log errors if necessary + if len(errMsgArr) > 0 { + return fmt.Errorf(strings.Join(errMsgArr, ";\n")) + } + return nil +} + +func (o *CreateMigrationOptions) BuildWithSteps(errMsgArr *[]string) error { + taskType := InitializationAndCdc.String() + validStepMap, validStepKey := CliStepChangeToStructure() + enableCdc, enablePreCheck, enableInitStruct, enableInitData := StringBoolTrue, StringBoolTrue, StringBoolTrue, StringBoolTrue + if len(o.Steps) > 0 { + for _, step := range o.Steps { + stepArr := strings.Split(step, "=") + if len(stepArr) != 2 { + BuildErrorMsg(errMsgArr, fmt.Sprintf("[%s] in steps setting is invalid", step)) + return nil + } + stepName := strings.ToLower(strings.TrimSpace(stepArr[0])) + enable := strings.ToLower(strings.TrimSpace(stepArr[1])) + if validStepMap[stepName] == "" { + BuildErrorMsg(errMsgArr, fmt.Sprintf("[%s] in steps settings is invalid, the name should be one of: (%s)", step, validStepKey)) + return nil + } + if enable != StringBoolTrue && enable != StringBoolFalse { + BuildErrorMsg(errMsgArr, fmt.Sprintf("[%s] in steps settings is invalid, the value should be one of: (true false)", step)) + return nil + } + switch stepName { + case migrationv1.CliStepCdc.String(): + enableCdc = enable + case migrationv1.CliStepPreCheck.String(): + enablePreCheck = enable + case migrationv1.CliStepInitStruct.String(): + enableInitStruct = enable + case migrationv1.CliStepInitData.String(): + enableInitData = enable + } + } + + if enableInitData != StringBoolTrue { + BuildErrorMsg(errMsgArr, "step init-data is needed") + return nil + } + if enableCdc == StringBoolTrue { + taskType = InitializationAndCdc.String() + } else { + taskType = Initialization.String() + } + } + o.TaskType = taskType + o.StepsModel = []string{} + if enablePreCheck == StringBoolTrue { + o.StepsModel = append(o.StepsModel, migrationv1.StepPreCheck.String()) + } + if enableInitStruct == StringBoolTrue { + o.StepsModel = append(o.StepsModel, migrationv1.StepStructPreFullLoad.String()) + } + if enableInitData == StringBoolTrue { + o.StepsModel = append(o.StepsModel, migrationv1.StepFullLoad.String()) + } + return nil +} + +func (o *CreateMigrationOptions) BuildWithTolerations() error { + o.TolerationModel = o.buildTolerationOrResources(o.Tolerations) + tmp := make([]interface{}, 0) + for _, step := range AllStepsArr { + if o.TolerationModel[step] == nil { + o.TolerationModel[step] = tmp + } + } + return nil +} + +func (o *CreateMigrationOptions) BuildWithResources() error { + o.ResourceModel = make(map[string]interface{}) + for k, v := range o.buildTolerationOrResources(o.Resources) { + if len(v) >= 1 { + o.ResourceModel[k] = v[0] + } + } + for _, step := range AllStepsArr { + if o.ResourceModel[step] == nil { + o.ResourceModel[step] = v1.ResourceList{} + } + } + return nil +} + +func (o *CreateMigrationOptions) BuildWithRuntimeParams() error { + template := migrationv1.MigrationTemplate{} + templateGvr := types.MigrationTemplateGVR() + if err := APIResource(&o.BaseOptions.Dynamic, &templateGvr, o.Template, "", &template); err != nil { + return err + } + + // Generate random serverId for MySQL type database.Possible values are between 10001 and 2^32-10001 + if template.Spec.Source.DBType == migrationv1.MigrationDBTypeMySQL { + o.ServerID = o.generateRandomMySQLServerID() + } else { + o.ServerID = 10001 + } + + return nil +} + +func (o *CreateMigrationOptions) buildTolerationOrResources(raws []string) map[string][]interface{} { + results := make(map[string][]interface{}) + for _, raw := range raws { + step := migrationv1.CliStepGlobal.String() + tmpMap := map[string]interface{}{} + rawLoop: + for _, entries := range strings.Split(raw, ",") { + parts := strings.SplitN(entries, "=", 2) + k := strings.TrimSpace(parts[0]) + v := strings.TrimSpace(parts[1]) + if k == "step" { + switch v { + case migrationv1.CliStepPreCheck.String(), migrationv1.CliStepCdc.String(), migrationv1.CliStepInitStruct.String(), migrationv1.CliStepInitData.String(): + step = v + } + continue rawLoop + } + tmpMap[k] = v + } + results[step] = append(results[step], tmpMap) + } + return results +} + +func (o *CreateMigrationOptions) generateRandomMySQLServerID() uint32 { + rand.Seed(time.Now().UnixNano()) + return uint32(rand.Int63nRange(10001, 1<<32-10001)) +} diff --git a/internal/cli/cmd/migration/create_test.go b/internal/cli/cmd/migration/create_test.go new file mode 100644 index 000000000..c6c1a5906 --- /dev/null +++ b/internal/cli/cmd/migration/create_test.go @@ -0,0 +1,163 @@ +package migration + +import ( + "bytes" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/kubernetes/scheme" + cmdTest "k8s.io/kubectl/pkg/cmd/testing" + + app "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/testing" + v1alpha1 "github.com/apecloud/kubeblocks/internal/cli/types/migrationapi" +) + +var ( + streams genericclioptions.IOStreams + out *bytes.Buffer + tf *cmdTest.TestFactory +) + +const ( + namespace = "test" +) + +var _ = Describe("create", func() { + o := &CreateMigrationOptions{} + + BeforeEach(func() { + streams, _, out, _ = genericclioptions.NewTestIOStreams() + tf = testing.NewTestFactory(namespace) + + _ = app.AddToScheme(scheme.Scheme) + + tf.Client = tf.UnstructuredClient + }) + + Context("Input params validate", func() { + var err error + errMsgArr := make([]string, 0, 3) + It("Endpoint with database", func() { + o.Source = "user:123456@127.0.0.1:5432/database" + err = o.SourceEndpointModel.BuildFromStr(&errMsgArr, o.Source) + Expect(err).ShouldNot(HaveOccurred()) + Expect(o.SourceEndpointModel.UserName).Should(Equal("user")) + Expect(o.SourceEndpointModel.Password).Should(Equal("123456")) + Expect(o.SourceEndpointModel.Address).Should(Equal("127.0.0.1:5432")) + Expect(o.SourceEndpointModel.Database).Should(Equal("database")) + Expect(len(errMsgArr)).Should(Equal(0)) + + o.Sink = "user:123456127.0.0.1:5432/database" + err = o.SinkEndpointModel.BuildFromStr(&errMsgArr, o.Sink) + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(errMsgArr)).Should(Equal(1)) + }) + + It("Endpoint with no database", func() { + o.Source = "user:123456@127.0.0.1:3306" + errMsgArr := make([]string, 0, 3) + err = o.SourceEndpointModel.BuildFromStr(&errMsgArr, o.Source) + Expect(err).ShouldNot(HaveOccurred()) + Expect(o.SourceEndpointModel.UserName).Should(Equal("user")) + Expect(o.SourceEndpointModel.Password).Should(Equal("123456")) + Expect(o.SourceEndpointModel.Address).Should(Equal("127.0.0.1:3306")) + Expect(o.SourceEndpointModel.Database).Should(BeEmpty()) + Expect(len(errMsgArr)).Should(Equal(0)) + + o.Sink = "user:123456127.0.0.1:3306" + err = o.SinkEndpointModel.BuildFromStr(&errMsgArr, o.Sink) + Expect(err).ShouldNot(HaveOccurred()) + Expect(len(errMsgArr)).Should(Equal(1)) + }) + + It("MigrationObject", func() { + o.MigrationObject = []string{"schema_public.table1", "schema2.table2_1", "schema2.table2_2", "schema3"} + err = o.MigrationObjectModel.BuildFromStrs(&errMsgArr, o.MigrationObject) + Expect(err).ShouldNot(HaveOccurred()) + for _, obj := range o.MigrationObjectModel.WhiteList { + Expect(obj.SchemaName).Should(BeElementOf("schema_public", "schema2", "schema3")) + switch obj.SchemaName { + case "schema_public": + Expect(len(obj.TableList)).Should(Equal(1)) + Expect(obj.TableList[0].TableName).Should(Equal("table1")) + Expect(obj.TableList[0].IsAll).Should(BeTrue()) + case "schema2": + Expect(len(obj.TableList)).Should(Equal(2)) + for _, tb := range obj.TableList { + Expect(tb.TableName).Should(BeElementOf("table2_1", "table2_2")) + Expect(tb.IsAll).Should(BeTrue()) + } + case "schema3": + Expect(obj.IsAll).Should(BeTrue()) + } + } + }) + + It("Steps", func() { + o.Steps = make([]string, 0) + err = o.BuildWithSteps(&errMsgArr) + Expect(err).ShouldNot(HaveOccurred()) + Expect(o.TaskType).Should(Equal(InitializationAndCdc.String())) + Expect(o.StepsModel).Should(ContainElements(v1alpha1.StepPreCheck.String(), v1alpha1.StepStructPreFullLoad.String(), v1alpha1.StepFullLoad.String())) + o.Steps = []string{"precheck=true", "init-struct=false", "cdc=false"} + err = o.BuildWithSteps(&errMsgArr) + Expect(err).ShouldNot(HaveOccurred()) + Expect(o.TaskType).Should(Equal(Initialization.String())) + Expect(o.StepsModel).Should(ContainElements(v1alpha1.StepPreCheck.String(), v1alpha1.StepFullLoad.String())) + }) + + It("Tolerations", func() { + o.Tolerations = []string{ + "step=global,key=engineType,value=pg,operator=Equal,effect=NoSchedule", + "step=init-data,key=engineType,value=pg1,operator=Equal,effect=NoSchedule", + "key=engineType,value=pg2,operator=Equal,effect=NoSchedule", + } + err = o.BuildWithTolerations() + Expect(err).ShouldNot(HaveOccurred()) + Expect(o.TolerationModel[v1alpha1.CliStepGlobal.String()]).ShouldNot(BeEmpty()) + Expect(o.TolerationModel[v1alpha1.CliStepInitData.String()]).ShouldNot(BeEmpty()) + Expect(len(o.TolerationModel[v1alpha1.CliStepInitData.String()])).Should(Equal(1)) + Expect(len(o.TolerationModel[v1alpha1.CliStepGlobal.String()])).Should(Equal(2)) + Expect(len(o.TolerationModel[v1alpha1.CliStepPreCheck.String()])).Should(Equal(0)) + }) + + It("Resources", func() { + o.Resources = []string{ + "step=global,cpu=1000m,memory=1Gi", + "step=init-data,cpu=2000m,memory=2Gi", + "cpu=3000m,memory=3Gi", + } + err = o.BuildWithResources() + Expect(err).ShouldNot(HaveOccurred()) + Expect(o.ResourceModel[v1alpha1.CliStepGlobal.String()]).ShouldNot(BeEmpty()) + Expect(o.ResourceModel[v1alpha1.CliStepInitData.String()]).ShouldNot(BeEmpty()) + Expect(o.ResourceModel[v1alpha1.CliStepPreCheck.String()]).Should(BeEmpty()) + }) + + It("RuntimeParams", func() { + type void struct{} + var setValue void + serverIDSet := make(map[uint32]void) + + loopCount := 0 + for loopCount < 1000 { + newServerID := o.generateRandomMySQLServerID() + Expect(newServerID >= 10001 && newServerID <= 1<<32-10001).Should(BeTrue()) + serverIDSet[newServerID] = setValue + + loopCount += 1 + } + Expect(len(serverIDSet) > 500).Should(BeTrue()) + }) + }) + + Context("Mock run", func() { + It("test", func() { + cmd := NewMigrationCreateCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + }) +}) diff --git a/internal/cli/cmd/migration/describe.go b/internal/cli/cmd/migration/describe.go new file mode 100644 index 000000000..f96d94efa --- /dev/null +++ b/internal/cli/cmd/migration/describe.go @@ -0,0 +1,271 @@ +package migration + +import ( + "context" + "fmt" + "io" + "sort" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/dynamic" + clientset "k8s.io/client-go/kubernetes" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/types" + v1alpha1 "github.com/apecloud/kubeblocks/internal/cli/types/migrationapi" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var ( + newTbl = func(out io.Writer, title string, header ...interface{}) *printer.TablePrinter { + fmt.Fprintln(out, title) + tbl := printer.NewTablePrinter(out) + tbl.SetHeader(header...) + return tbl + } +) + +type describeOptions struct { + factory cmdutil.Factory + client clientset.Interface + dynamic dynamic.Interface + namespace string + + // resource type and names + gvr schema.GroupVersionResource + names []string + + *v1alpha1.MigrationObjects + genericclioptions.IOStreams +} + +func newOptions(f cmdutil.Factory, streams genericclioptions.IOStreams) *describeOptions { + return &describeOptions{ + factory: f, + IOStreams: streams, + gvr: types.MigrationTaskGVR(), + } +} + +func NewMigrationDescribeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := newOptions(f, streams) + cmd := &cobra.Command{ + Use: "describe NAME", + Short: "Show details of a specific migration task.", + Example: DescribeExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.MigrationTaskGVR()), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(o.complete(args)) + util.CheckErr(o.run()) + }, + } + return cmd +} + +func (o *describeOptions) complete(args []string) error { + var err error + + if o.client, err = o.factory.KubernetesClientSet(); err != nil { + return err + } + + if o.dynamic, err = o.factory.DynamicClient(); err != nil { + return err + } + + if o.namespace, _, err = o.factory.ToRawKubeConfigLoader().Namespace(); err != nil { + return err + } + + _, err = IsMigrationCrdValidWithDynamic(&o.dynamic) + if errors.IsNotFound(err) { + return fmt.Errorf("datamigration crd is not install") + } else if err != nil { + return err + } + + if len(args) == 0 { + return fmt.Errorf("migration task name should be specified") + } + o.names = args + return nil +} + +func (o *describeOptions) run() error { + for _, name := range o.names { + if err := o.describeMigration(name); err != nil { + return err + } + } + return nil +} + +func (o *describeOptions) describeMigration(name string) error { + var err error + if o.MigrationObjects, err = getMigrationObjects(o, name); err != nil { + return err + } + + // MigrationTask Summary + showTaskSummary(o.Task, o.Out) + + // MigrationTask Config + showTaskConfig(o.Task, o.Out) + + // MigrationTemplate Summary + showTemplateSummary(o.Template, o.Out) + + // Initialization Detail + showInitialization(o.Task, o.Template, o.Jobs, o.Out) + + switch o.Task.Spec.TaskType { + case v1alpha1.InitializationAndCdc, v1alpha1.CDC: + // Cdc Detail + showCdc(o.Pods, o.Out) + + // Cdc Metrics + showCdcMetrics(o.Task, o.Out) + } + + fmt.Fprintln(o.Out) + + return nil +} + +func getMigrationObjects(o *describeOptions, taskName string) (*v1alpha1.MigrationObjects, error) { + obj := &v1alpha1.MigrationObjects{ + Task: &v1alpha1.MigrationTask{}, + Template: &v1alpha1.MigrationTemplate{}, + } + var err error + taskGvr := types.MigrationTaskGVR() + if err = APIResource(&o.dynamic, &taskGvr, taskName, o.namespace, obj.Task); err != nil { + return nil, err + } + templateGvr := types.MigrationTemplateGVR() + if err = APIResource(&o.dynamic, &templateGvr, obj.Task.Spec.Template, "", obj.Template); err != nil { + return nil, err + } + listOpts := func() metav1.ListOptions { + return metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", MigrationTaskLabel, taskName), + } + } + if obj.Jobs, err = o.client.BatchV1().Jobs(o.namespace).List(context.Background(), listOpts()); err != nil { + return nil, err + } + if obj.Pods, err = o.client.CoreV1().Pods(o.namespace).List(context.Background(), listOpts()); err != nil { + return nil, err + } + return obj, nil +} + +func showTaskSummary(task *v1alpha1.MigrationTask, out io.Writer) { + if task == nil { + return + } + title := fmt.Sprintf("Name: %s\t Status: %s", task.Name, task.Status.TaskStatus) + tbl := newTbl(out, title, "NAMESPACE", "CREATED-TIME", "START-TIME", "FINISHED-TIME") + tbl.AddRow(task.Namespace, util.TimeFormatWithDuration(&task.CreationTimestamp, time.Second), util.TimeFormatWithDuration(task.Status.StartTime, time.Second), util.TimeFormatWithDuration(task.Status.FinishTime, time.Second)) + tbl.Print() +} + +func showTaskConfig(task *v1alpha1.MigrationTask, out io.Writer) { + if task == nil { + return + } + tbl := newTbl(out, "\nMigration Config:") + tbl.AddRow("source", fmt.Sprintf("%s:%s@%s/%s", + task.Spec.SourceEndpoint.UserName, + task.Spec.SourceEndpoint.Password, + task.Spec.SourceEndpoint.Address, + task.Spec.SourceEndpoint.DatabaseName, + )) + tbl.AddRow("sink", fmt.Sprintf("%s:%s@%s/%s", + task.Spec.SinkEndpoint.UserName, + task.Spec.SinkEndpoint.Password, + task.Spec.SinkEndpoint.Address, + task.Spec.SinkEndpoint.DatabaseName, + )) + tbl.AddRow("migration objects", task.Spec.MigrationObj.String(true)) + tbl.Print() +} + +func showTemplateSummary(template *v1alpha1.MigrationTemplate, out io.Writer) { + if template == nil { + return + } + title := fmt.Sprintf("\nTemplate: %s\t", template.Name) + tbl := newTbl(out, title, "DATABASE-MAPPING", "STATUS") + tbl.AddRow(template.Spec.Description, template.Status.Phase) + tbl.Print() +} + +func showInitialization(task *v1alpha1.MigrationTask, template *v1alpha1.MigrationTemplate, jobList *batchv1.JobList, out io.Writer) { + if len(jobList.Items) == 0 { + return + } + sort.SliceStable(jobList.Items, func(i, j int) bool { + jobName1 := jobList.Items[i].Name + jobName2 := jobList.Items[j].Name + order1, _ := strconv.ParseInt(string([]byte(jobName1)[strings.LastIndex(jobName1, "-")+1:]), 10, 8) + order2, _ := strconv.ParseInt(string([]byte(jobName2)[strings.LastIndex(jobName2, "-")+1:]), 10, 8) + return order1 < order2 + }) + cliStepOrder := BuildInitializationStepsOrder(task, template) + tbl := newTbl(out, "\nInitialization:", "STEP", "NAMESPACE", "STATUS", "CREATED_TIME", "START-TIME", "FINISHED-TIME") + if len(cliStepOrder) != len(jobList.Items) { + return + } + for i, job := range jobList.Items { + tbl.AddRow(cliStepOrder[i], job.Namespace, getJobStatus(job.Status.Conditions), util.TimeFormatWithDuration(&job.CreationTimestamp, time.Second), util.TimeFormatWithDuration(job.Status.StartTime, time.Second), util.TimeFormatWithDuration(job.Status.CompletionTime, time.Second)) + } + tbl.Print() +} + +func showCdc(pods *v1.PodList, out io.Writer) { + if len(pods.Items) == 0 { + return + } + tbl := newTbl(out, "\nCdc:", "NAMESPACE", "STATUS", "CREATED_TIME", "START-TIME") + for _, pod := range pods.Items { + if pod.Annotations[MigrationTaskStepAnnotation] != v1alpha1.StepCdc.String() { + continue + } + tbl.AddRow(pod.Namespace, pod.Status.Phase, util.TimeFormatWithDuration(&pod.CreationTimestamp, time.Second), util.TimeFormatWithDuration(pod.Status.StartTime, time.Second)) + } + tbl.Print() +} + +func showCdcMetrics(task *v1alpha1.MigrationTask, out io.Writer) { + if task.Status.Cdc.Metrics == nil || len(task.Status.Cdc.Metrics) == 0 { + return + } + arr := make([]string, 0) + for mKey := range task.Status.Cdc.Metrics { + arr = append(arr, mKey) + } + tbl := newTbl(out, "\nCdc Metrics:") + for _, k := range arr { + tbl.AddRow(k, task.Status.Cdc.Metrics[k]) + } + tbl.Print() +} + +func getJobStatus(conditions []batchv1.JobCondition) string { + if len(conditions) == 0 { + return "-" + } else { + return string(conditions[len(conditions)-1].Type) + } +} diff --git a/internal/cli/cmd/migration/describe_test.go b/internal/cli/cmd/migration/describe_test.go new file mode 100644 index 000000000..9190b0953 --- /dev/null +++ b/internal/cli/cmd/migration/describe_test.go @@ -0,0 +1,23 @@ +package migration + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" +) + +var _ = Describe("describe", func() { + + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + ) + + It("command build", func() { + cmd := NewMigrationDescribeCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + +}) diff --git a/internal/cli/cmd/migration/examples.go b/internal/cli/cmd/migration/examples.go new file mode 100644 index 000000000..eaa2eb2c3 --- /dev/null +++ b/internal/cli/cmd/migration/examples.go @@ -0,0 +1,88 @@ +package migration + +import "k8s.io/kubectl/pkg/util/templates" + +// Cli Migration Command Examples +var ( + CreateTemplate = templates.Examples(` + # Create a migration task to migrate the entire database under mysql: mydb1 and mytable1 under database: mydb2 to the target mysql + kbcli migration create mytask --template apecloud-mysql2mysql + --source user:123456@127.0.0.1:3306 + --sink user:123456@127.0.0.1:3305 + --migration-object '"mydb1","mydb2.mytable1"' + + # Create a migration task to migrate the schema: myschema under database: mydb1 under PostgreSQL to the target PostgreSQL + kbcli migration create mytask --template apecloud-pg2pg + --source user:123456@127.0.0.1:3306/mydb1 + --sink user:123456@127.0.0.1:3305/mydb1 + --migration-object '"myschema"' + + # Use prechecks, data initialization, CDC, but do not perform structure initialization + kbcli migration create mytask --template apecloud-pg2pg + --source user:123456@127.0.0.1:3306/mydb1 + --sink user:123456@127.0.0.1:3305/mydb1 + --migration-object '"myschema"' + --steps precheck=true,init-struct=false,init-data=true,cdc=true + + # Create a migration task with two tolerations + kbcli migration create mytask --template apecloud-pg2pg + --source user:123456@127.0.0.1:3306/mydb1 + --sink user:123456@127.0.0.1:3305/mydb1 + --migration-object '"myschema"' + --tolerations '"step=global,key=engineType,value=pg,operator=Equal,effect=NoSchedule","step=init-data,key=diskType,value=ssd,operator=Equal,effect=NoSchedule"' + + # Limit resource usage when performing data initialization + kbcli migration create mytask --template apecloud-pg2pg + --source user:123456@127.0.0.1:3306/mydb1 + --sink user:123456@127.0.0.1:3305/mydb1 + --migration-object '"myschema"' + --resources '"step=init-data,cpu=1000m,memory=1Gi"' + `) + DescribeExample = templates.Examples(` + # describe a specified migration task + kbcli migration describe mytask + `) + ListExample = templates.Examples(` + # list all migration tasks + kbcli migration list + + # list a single migration task with specified NAME + kbcli migration list mytask + + # list a single migration task in YAML output format + kbcli migration list mytask -o yaml + + # list a single migration task in JSON output format + kbcli migration list mytask -o json + + # list a single migration task in wide output format + kbcli migration list mytask -o wide + `) + TemplateExample = templates.Examples(` + # list all migration templates + kbcli migration templates + + # list a single migration template with specified NAME + kbcli migration templates mytemplate + + # list a single migration template in YAML output format + kbcli migration templates mytemplate -o yaml + + # list a single migration template in JSON output format + kbcli migration templates mytemplate -o json + + # list a single migration template in wide output format + kbcli migration templates mytemplate -o wide + `) + DeleteExample = templates.Examples(` + # terminate a migration task named mytask and delete resources in k8s without affecting source and target data in database + kbcli migration terminate mytask + `) + LogsExample = templates.Examples(` + # Logs when returning to the "init-struct" step from the migration task mytask + kbcli migration logs mytask --step init-struct + + # Logs only the most recent 20 lines when returning to the "cdc" step from the migration task mytask + kbcli migration logs mytask --step cdc --tail=20 + `) +) diff --git a/internal/cli/cmd/migration/list.go b/internal/cli/cmd/migration/list.go new file mode 100644 index 000000000..67b248cba --- /dev/null +++ b/internal/cli/cmd/migration/list.go @@ -0,0 +1,31 @@ +package migration + +import ( + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/apecloud/kubeblocks/internal/cli/list" + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +func NewMigrationListCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := list.NewListOptions(f, streams, types.MigrationTaskGVR()) + cmd := &cobra.Command{ + Use: "list [NAME]", + Short: "List migration tasks.", + Example: ListExample, + Aliases: []string{"ls"}, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, o.GVR), + Run: func(cmd *cobra.Command, args []string) { + _, validErr := IsMigrationCrdValidWithFactory(o.Factory) + util.CheckErr(validErr) + o.Names = args + _, err := o.Run() + util.CheckErr(err) + }, + } + o.AddFlags(cmd) + return cmd +} diff --git a/internal/cli/cmd/migration/list_test.go b/internal/cli/cmd/migration/list_test.go new file mode 100644 index 000000000..5a28311d1 --- /dev/null +++ b/internal/cli/cmd/migration/list_test.go @@ -0,0 +1,23 @@ +package migration + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" +) + +var _ = Describe("list", func() { + + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + ) + + It("command build", func() { + cmd := NewMigrationListCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + +}) diff --git a/internal/cli/cmd/migration/logs.go b/internal/cli/cmd/migration/logs.go new file mode 100644 index 000000000..3d7347345 --- /dev/null +++ b/internal/cli/cmd/migration/logs.go @@ -0,0 +1,213 @@ +package migration + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + cmdlogs "k8s.io/kubectl/pkg/cmd/logs" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/polymorphichelpers" + + "github.com/apecloud/kubeblocks/internal/cli/exec" + "github.com/apecloud/kubeblocks/internal/cli/types" + migrationv1 "github.com/apecloud/kubeblocks/internal/cli/types/migrationapi" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +type LogsOptions struct { + taskName string + step string + Client *kubernetes.Clientset + Dynamic dynamic.Interface + *exec.ExecOptions + logOptions cmdlogs.LogsOptions +} + +func NewMigrationLogsCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + l := &LogsOptions{ + ExecOptions: exec.NewExecOptions(f, streams), + logOptions: cmdlogs.LogsOptions{ + Tail: -1, + IOStreams: streams, + }, + } + cmd := &cobra.Command{ + Use: "logs NAME", + Short: "Access migration task log file.", + Example: LogsExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.MigrationTaskGVR()), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(l.ExecOptions.Complete()) + util.CheckErr(l.complete(f, cmd, args)) + util.CheckErr(l.validate()) + util.CheckErr(l.runLogs()) + }, + } + l.addFlags(cmd) + return cmd +} + +func (o *LogsOptions) addFlags(cmd *cobra.Command) { + cmd.Flags().StringVar(&o.step, "step", "", "Specify the step. Allow values: precheck,init-struct,init-data,cdc") + + o.logOptions.AddFlags(cmd) +} + +// complete customs complete function for logs +func (o *LogsOptions) complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return fmt.Errorf("migration task name should be specified") + } + if len(args) > 0 { + o.taskName = args[0] + } + if o.step == "" { + return fmt.Errorf("migration task step should be specified") + } + var err error + o.logOptions.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace() + if err != nil { + return err + } + + o.Dynamic, err = f.DynamicClient() + if err != nil { + return err + } + + o.Client, err = f.KubernetesClientSet() + if err != nil { + return err + } + + _, err = IsMigrationCrdValidWithDynamic(&o.Dynamic) + if errors.IsNotFound(err) { + return fmt.Errorf("datamigration crd is not install") + } else if err != nil { + return err + } + + taskObj, err := o.getMigrationObjects(o.taskName) + if err != nil { + return fmt.Errorf("failed to find the migrationtask") + } + pod := o.getPodByStep(taskObj, strings.TrimSpace(o.step)) + if pod == nil { + return fmt.Errorf("migrationtask[%s] step[%s] 's pod not found", taskObj.Task.Name, o.step) + } + o.logOptions.RESTClientGetter = f + o.logOptions.LogsForObject = polymorphichelpers.LogsForObjectFn + o.logOptions.Object = pod + o.logOptions.Options, _ = o.logOptions.ToLogOptions() + o.Pod = pod + + return nil +} + +func (o *LogsOptions) validate() error { + if len(o.taskName) == 0 { + return fmt.Errorf("migration task name must be specified") + } + + if o.logOptions.LimitBytes < 0 { + return fmt.Errorf("--limit-bytes must be greater than 0") + } + if o.logOptions.Tail < -1 { + return fmt.Errorf("--tail must be greater than or equal to -1") + } + if len(o.logOptions.SinceTime) > 0 && o.logOptions.SinceSeconds != 0 { + return fmt.Errorf("at most one of `sinceTime` or `sinceSeconds` may be specified") + } + logsOptions, ok := o.logOptions.Options.(*corev1.PodLogOptions) + if !ok { + return fmt.Errorf("unexpected logs options object") + } + if logsOptions.SinceSeconds != nil && *logsOptions.SinceSeconds < int64(0) { + return fmt.Errorf("--since must be greater than 0") + } + if logsOptions.TailLines != nil && *logsOptions.TailLines < -1 { + return fmt.Errorf("--tail must be greater than or equal to -1") + } + return nil +} + +func (o *LogsOptions) getMigrationObjects(taskName string) (*migrationv1.MigrationObjects, error) { + obj := &migrationv1.MigrationObjects{ + Task: &migrationv1.MigrationTask{}, + Template: &migrationv1.MigrationTemplate{}, + } + var err error + taskGvr := types.MigrationTaskGVR() + if err = APIResource(&o.Dynamic, &taskGvr, taskName, o.logOptions.Namespace, obj.Task); err != nil { + return nil, err + } + templateGvr := types.MigrationTemplateGVR() + if err = APIResource(&o.Dynamic, &templateGvr, obj.Task.Spec.Template, "", obj.Template); err != nil { + return nil, err + } + listOpts := func() metav1.ListOptions { + return metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", MigrationTaskLabel, taskName), + } + } + if obj.Pods, err = o.Client.CoreV1().Pods(o.logOptions.Namespace).List(context.Background(), listOpts()); err != nil { + return nil, err + } + return obj, nil +} + +func (o *LogsOptions) runLogs() error { + requests, err := o.logOptions.LogsForObject(o.logOptions.RESTClientGetter, o.logOptions.Object, o.logOptions.Options, 60*time.Second, false) + if err != nil { + return err + } + for _, request := range requests { + if err := cmdlogs.DefaultConsumeRequest(request, o.Out); err != nil { + if !o.logOptions.IgnoreLogErrors { + return err + } + fmt.Fprintf(o.Out, "error: %v\n", err) + } + } + return nil +} + +func (o *LogsOptions) getPodByStep(taskObj *migrationv1.MigrationObjects, step string) *corev1.Pod { + if taskObj == nil || len(taskObj.Pods.Items) == 0 { + return nil + } + switch step { + case migrationv1.CliStepCdc.String(): + for _, pod := range taskObj.Pods.Items { + if pod.Annotations[MigrationTaskStepAnnotation] == migrationv1.StepCdc.String() { + return &pod + } + } + case migrationv1.CliStepPreCheck.String(), migrationv1.CliStepInitStruct.String(), migrationv1.CliStepInitData.String(): + stepArr := BuildInitializationStepsOrder(taskObj.Task, taskObj.Template) + orderNo := "-1" + for index, stepByTemplate := range stepArr { + if step == stepByTemplate { + orderNo = strconv.Itoa(index) + break + } + } + for _, pod := range taskObj.Pods.Items { + if pod.Annotations[SerialJobOrderAnnotation] != "" && + pod.Annotations[SerialJobOrderAnnotation] == orderNo { + return &pod + } + } + } + return nil +} diff --git a/internal/cli/cmd/migration/logs_test.go b/internal/cli/cmd/migration/logs_test.go new file mode 100644 index 000000000..14aa3024b --- /dev/null +++ b/internal/cli/cmd/migration/logs_test.go @@ -0,0 +1,23 @@ +package migration + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" +) + +var _ = Describe("logs", func() { + + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + ) + + It("command build", func() { + cmd := NewMigrationLogsCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + +}) diff --git a/internal/cli/cmd/migration/suite_test.go b/internal/cli/cmd/migration/suite_test.go new file mode 100644 index 000000000..e876871df --- /dev/null +++ b/internal/cli/cmd/migration/suite_test.go @@ -0,0 +1,13 @@ +package migration_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMigration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Migration Suite") +} diff --git a/internal/cli/cmd/migration/templates.go b/internal/cli/cmd/migration/templates.go new file mode 100644 index 000000000..5479c6b04 --- /dev/null +++ b/internal/cli/cmd/migration/templates.go @@ -0,0 +1,31 @@ +package migration + +import ( + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/apecloud/kubeblocks/internal/cli/list" + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +func NewMigrationTemplatesCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := list.NewListOptions(f, streams, types.MigrationTemplateGVR()) + cmd := &cobra.Command{ + Use: "templates [NAME]", + Short: "List migration templates.", + Example: TemplateExample, + Aliases: []string{"tp", "template"}, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, o.GVR), + Run: func(cmd *cobra.Command, args []string) { + _, validErr := IsMigrationCrdValidWithFactory(o.Factory) + util.CheckErr(validErr) + o.Names = args + _, err := o.Run() + util.CheckErr(err) + }, + } + o.AddFlags(cmd) + return cmd +} diff --git a/internal/cli/cmd/migration/templates_test.go b/internal/cli/cmd/migration/templates_test.go new file mode 100644 index 000000000..9d818c5ad --- /dev/null +++ b/internal/cli/cmd/migration/templates_test.go @@ -0,0 +1,23 @@ +package migration + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" +) + +var _ = Describe("templates", func() { + + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + ) + + It("command build", func() { + cmd := NewMigrationTemplatesCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + +}) diff --git a/internal/cli/cmd/migration/terminate.go b/internal/cli/cmd/migration/terminate.go new file mode 100644 index 000000000..beb8be287 --- /dev/null +++ b/internal/cli/cmd/migration/terminate.go @@ -0,0 +1,38 @@ +package migration + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/apecloud/kubeblocks/internal/cli/delete" + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +func NewMigrationTerminateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := delete.NewDeleteOptions(f, streams, types.MigrationTaskGVR()) + cmd := &cobra.Command{ + Use: "terminate NAME", + Short: "Delete migration task.", + Example: DeleteExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.MigrationTaskGVR()), + Run: func(cmd *cobra.Command, args []string) { + _, validErr := IsMigrationCrdValidWithFactory(o.Factory) + util.CheckErr(validErr) + util.CheckErr(deleteMigrationTask(o, args)) + }, + } + o.AddFlags(cmd) + return cmd +} + +func deleteMigrationTask(o *delete.DeleteOptions, args []string) error { + if len(args) == 0 { + return fmt.Errorf("missing migration task name") + } + o.Names = args + return o.Run() +} diff --git a/internal/cli/cmd/migration/terminate_test.go b/internal/cli/cmd/migration/terminate_test.go new file mode 100644 index 000000000..167e7d2c9 --- /dev/null +++ b/internal/cli/cmd/migration/terminate_test.go @@ -0,0 +1,23 @@ +package migration + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" +) + +var _ = Describe("terminate", func() { + + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + ) + + It("command build", func() { + cmd := NewMigrationTerminateCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + +}) diff --git a/internal/cli/create/template/migration_template.cue b/internal/cli/create/template/migration_template.cue new file mode 100644 index 000000000..074981680 --- /dev/null +++ b/internal/cli/create/template/migration_template.cue @@ -0,0 +1,89 @@ +// Copyright ApeCloud, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// required, command line input options for parameters and flags +options: { + name: string + namespace: string + teplate: string + taskType: string + source: string + sourceEndpointModel: {} + sink: string + sinkEndpointModel: {} + migrationObject: [...] + migrationObjectModel: {} + steps: [...] + stepsModel: [...] + tolerations: [...] + tolerationModel: {} + resources: [...] + resourceModel: {} + serverId: int +} + +// required, k8s api resource content +content: { + apiVersion: "datamigration.apecloud.io/v1alpha1" + kind: "MigrationTask" + metadata: { + name: options.name + namespace: options.namespace + } + spec: { + taskType: options.taskType + template: options.template + sourceEndpoint: options.sourceEndpointModel + sinkEndpoint: options.sinkEndpointModel + initialization: { + steps: options.stepsModel + config: { + preCheck: { + resource: { + limits: options.resourceModel["precheck"] + }, + tolerations: options.tolerationModel["precheck"] + } + initStruct: { + resource: { + limits: options.resourceModel["init-struct"] + }, + tolerations: options.tolerationModel["init-struct"] + } + initData: { + resource: { + limits: options.resourceModel["init-data"] + }, + tolerations: options.tolerationModel["init-data"] + } + } + } + cdc: { + config: { + resource: { + limits: options.resourceModel["cdc"] + }, + tolerations: options.tolerationModel["cdc"], + param: { + "extractor.server_id": options.serverId + } + } + } + migrationObj: options.migrationObjectModel + globalTolerations: options.tolerationModel["global"] + globalResources: { + limits: options.resourceModel["global"] + } + } +} diff --git a/internal/cli/types/migrationapi/migration_object_express.go b/internal/cli/types/migrationapi/migration_object_express.go new file mode 100644 index 000000000..c858ad480 --- /dev/null +++ b/internal/cli/types/migrationapi/migration_object_express.go @@ -0,0 +1,93 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "fmt" + "strings" +) + +type MigrationObjectExpress struct { + WhiteList []DBObjectExpress `json:"whiteList"` + // +optional + BlackList []DBObjectExpress `json:"blackList"` +} + +func (m *MigrationObjectExpress) String(isWhite bool) string { + expressArr := m.WhiteList + if !isWhite { + expressArr = m.BlackList + } + stringArr := make([]string, 0) + for _, db := range expressArr { + stringArr = append(stringArr, db.String()...) + } + return strings.Join(stringArr, ",") +} + +type DBObjectExpress struct { + SchemaName string `json:"schemaName"` + // +optional + SchemaMappingName string `json:"schemaMappingName"` + // +optional + IsAll bool `json:"isAll"` + // +optional + TableList []TableObjectExpress `json:"tableList"` + DxlOpConfig `json:""` +} + +func (d *DBObjectExpress) String() []string { + stringArr := make([]string, 0) + if d.IsAll { + stringArr = append(stringArr, d.SchemaName) + } else { + for _, tb := range d.TableList { + stringArr = append(stringArr, fmt.Sprintf("%s.%s", d.SchemaName, tb.TableName)) + } + } + return stringArr +} + +type TableObjectExpress struct { + TableName string `json:"tableName"` + // +optional + TableMappingName string `json:"tableMappingName"` + // +optional + IsAll bool `json:"isAll"` + // +optional + FieldList []FieldObjectExpress `json:"fieldList"` + DxlOpConfig `json:""` +} + +type FieldObjectExpress struct { + FieldName string `json:"fieldName"` + // +optional + FieldMappingName string `json:"fieldMappingName"` +} + +type DxlOpConfig struct { + // +optional + DmlOp []DMLOpEnum `json:"dmlOp"` + // +optional + DdlOp []DDLOpEnum `json:"ddlOp"` + // +optional + DclOp []DCLOpEnum `json:"dclOp"` +} + +func (op *DxlOpConfig) IsEmpty() bool { + return len(op.DmlOp) == 0 +} diff --git a/internal/cli/types/migrationapi/migrationtask_types.go b/internal/cli/types/migrationapi/migrationtask_types.go new file mode 100644 index 000000000..040a08f8b --- /dev/null +++ b/internal/cli/types/migrationapi/migrationtask_types.go @@ -0,0 +1,153 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// MigrationTaskSpec defines the desired state of MigrationTask +type MigrationTaskSpec struct { + TaskType TaskTypeEnum `json:"taskType,omitempty"` + Template string `json:"template"` + SourceEndpoint Endpoint `json:"sourceEndpoint,omitempty"` + SinkEndpoint Endpoint `json:"sinkEndpoint,omitempty"` + // +optional + Cdc CdcConfig `json:"cdc,omitempty"` + // +optional + Initialization InitializationConfig `json:"initialization,omitempty"` + MigrationObj MigrationObjectExpress `json:"migrationObj,omitempty"` + // +optional + IsForceDelete bool `json:"isForceDelete,omitempty"` + // +optional + GlobalTolerations []v1.Toleration `json:"globalTolerations,omitempty"` + // +optional + GlobalResources v1.ResourceRequirements `json:"globalResources,omitempty"` +} + +type Endpoint struct { + // +optional + EndpointType EndpointTypeEnum `json:"endpointType,omitempty"` + Address string `json:"address"` + // +optional + DatabaseName string `json:"databaseName,omitempty"` + // +optional + UserName string `json:"userName"` + // +optional + Password string `json:"password"` + // +optional + Secret UserPswSecret `json:"secret"` +} + +type UserPswSecret struct { + Name string `json:"name"` + // +optional + Namespace string `json:"namespace,omitempty"` + // +optional + UserKeyword string `json:"userKeyword,omitempty"` + // +optional + PasswordKeyword string `json:"passwordKeyword,omitempty"` +} + +type CdcConfig struct { + // +optional + Config BaseConfig `json:"config"` +} + +type InitializationConfig struct { + // +optional + Steps []StepEnum `json:"steps,omitempty"` + // +optional + Config map[StepEnum]BaseConfig `json:"config,omitempty"` +} + +type BaseConfig struct { + // +optional + Resource v1.ResourceRequirements `json:"resource,omitempty"` + // +optional + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Param IntOrStringMap `json:"param"` + // +optional + PersistentVolumeClaimName string `json:"persistentVolumeClaimName"` + // +optional + Metrics Metrics `json:"metrics,omitempty"` +} + +// MigrationTaskStatus defines the observed state of MigrationTask +type MigrationTaskStatus struct { + // +optional + TaskStatus TaskStatus `json:"taskStatus"` + // +optional + StartTime *metav1.Time `json:"startTime"` + // +optional + FinishTime *metav1.Time `json:"finishTime"` + // +optional + Cdc RunTimeStatus `json:"cdc"` + // +optional + Initialization RunTimeStatus `json:"initialization"` +} + +type RunTimeStatus struct { + // +optional + StartTime *metav1.Time `json:"startTime"` + // +optional + FinishTime *metav1.Time `json:"finishTime"` + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + RunTimeParam IntOrStringMap `json:"runTimeParam,omitempty"` + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Metrics IntOrStringMap `json:"metrics,omitempty"` + // +optional + FailedReason string `json:"failedReason,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:categories={dtplatform},scope=Cluster,shortName=mt +// +kubebuilder:printcolumn:name="TEMPLATE",type="string",JSONPath=".spec.template",description="spec migration template" +// +kubebuilder:printcolumn:name="STATUS",type="string",JSONPath=".status.taskStatus",description="status taskStatus" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" + +// MigrationTask is the Schema for the migrationTasks API +type MigrationTask struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec MigrationTaskSpec `json:"spec,omitempty"` + Status MigrationTaskStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// MigrationTaskList contains a list of MigrationTask +type MigrationTaskList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []MigrationTask `json:"items"` +} + +type Metrics struct { + IsDisable bool `json:"isDisable,omitempty"` + PeriodSeconds int32 `json:"periodSeconds,omitempty"` +} diff --git a/internal/cli/types/migrationapi/migrationtemplate_types.go b/internal/cli/types/migrationapi/migrationtemplate_types.go new file mode 100644 index 000000000..6119bf3f1 --- /dev/null +++ b/internal/cli/types/migrationapi/migrationtemplate_types.go @@ -0,0 +1,103 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// MigrationTemplateSpec defines the desired state of MigrationTemplate +type MigrationTemplateSpec struct { + TaskType []TaskTypeEnum `json:"taskType,omitempty"` + Source DBTypeSupport `json:"source"` + Sink DBTypeSupport `json:"target"` + Initialization InitializationModel `json:"initialization,omitempty"` + Cdc CdcModel `json:"cdc,omitempty"` + // +optional + Description string `json:"description,omitempty"` + // +optional + Decorator string `json:"decorator,omitempty"` +} + +type DBTypeSupport struct { + DBType DBTypeEnum `json:"dbType"` + DBVersion string `json:"dbVersion"` +} + +type InitializationModel struct { + // +optional + IsPositionPreparation bool `json:"isPositionPreparation,omitempty"` + Steps []StepModel `json:"steps,omitempty"` +} + +type StepModel struct { + Step StepEnum `json:"step"` + Container BasicContainerTemplate `json:"container"` + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Param IntOrStringMap `json:"param"` +} + +type CdcModel struct { + Container BasicContainerTemplate `json:"container"` + // +optional + Replicas *int32 `json:"replicas,omitempty"` + // +optional + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + Param IntOrStringMap `json:"param"` +} + +type BasicContainerTemplate struct { + Image string `json:"image"` + // +optional + Command []string `json:"command,omitempty"` + // +optional + Env []v1.EnvVar `json:"env,omitempty"` +} + +// MigrationTemplateStatus defines the observed state of MigrationTemplate +type MigrationTemplateStatus struct { + Phase Phase `json:"phase,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:categories={dtplatform},scope=Cluster,shortName=mtp +// +kubebuilder:printcolumn:name="DATABASE-MAPPING",type="string",JSONPath=".spec.description",description="the database mapping that supported" +// +kubebuilder:printcolumn:name="STATUS",type="string",JSONPath=".status.phase",description="the template status" +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp" + +// MigrationTemplate is the Schema for the migrationtemplates API +type MigrationTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec MigrationTemplateSpec `json:"spec,omitempty"` + Status MigrationTemplateStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// MigrationTemplateList contains a list of MigrationTemplate +type MigrationTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []MigrationTemplate `json:"items"` +} diff --git a/internal/cli/types/migrationapi/type.go b/internal/cli/types/migrationapi/type.go new file mode 100644 index 000000000..836b55a07 --- /dev/null +++ b/internal/cli/types/migrationapi/type.go @@ -0,0 +1,208 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "strings" + + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DBTypeEnum defines the MigrationTemplate CR .spec.Source.DbType or .spec.Sink.DbType +// +enum +// +kubebuilder:validation:Enum={MySQL, PostgreSQL} +type DBTypeEnum string + +const ( + MigrationDBTypeMySQL DBTypeEnum = "MySQL" // default value + MigrationDBTypePostgreSQL DBTypeEnum = "PostgreSQL" +) + +func (d DBTypeEnum) String() string { + return string(d) +} + +// TaskTypeEnum defines the MigrationTask CR .spec.taskType +// +enum +// +kubebuilder:validation:Enum={initialization,cdc,initialization-and-cdc,initialization-and-twoway-cdc} +type TaskTypeEnum string + +const ( + Initialization TaskTypeEnum = "initialization" + CDC TaskTypeEnum = "cdc" + InitializationAndCdc TaskTypeEnum = "initialization-and-cdc" // default value +) + +// EndpointTypeEnum defines the MigrationTask CR .spec.source.endpointType and .spec.sink.endpointType +// +enum +// +kubebuilder:validation:Enum={address} +type EndpointTypeEnum string + +const ( + AddressDirectConnect EndpointTypeEnum = "address" // default value +) + +// non-use yet + +type ConflictPolicyEnum string + +const ( + Ignore ConflictPolicyEnum = "ignore" // default in FullLoad + Override ConflictPolicyEnum = "override" // default in CDC +) + +// DMLOpEnum defines the MigrationTask CR .spec.migrationObj +// +enum +// +kubebuilder:validation:Enum={all,none,insert,update,delete} +type DMLOpEnum string + +const ( + AllDML DMLOpEnum = "all" + NoneDML DMLOpEnum = "none" + Insert DMLOpEnum = "insert" + Update DMLOpEnum = "update" + Delete DMLOpEnum = "delete" +) + +// DDLOpEnum defines the MigrationTask CR .spec.migrationObj +// +enum +// +kubebuilder:validation:Enum={all,none} +type DDLOpEnum string + +const ( + AllDDL DDLOpEnum = "all" + NoneDDL DDLOpEnum = "none" +) + +// DCLOpEnum defines the MigrationTask CR .spec.migrationObj +// +enum +// +kubebuilder:validation:Enum={all,none} +type DCLOpEnum string + +const ( + AllDCL DDLOpEnum = "all" + NoneDCL DDLOpEnum = "none" +) + +// TaskStatus defines the MigrationTask CR .status.taskStatus +// +enum +// +kubebuilder:validation:Enum={Prepare,InitPrepared,Init,InitFinished,Running,Cached,Pause,Done} +type TaskStatus string + +const ( + PrepareStatus TaskStatus = "Prepare" + InitPrepared TaskStatus = "InitPrepared" + InitStatus TaskStatus = "Init" + InitFinished TaskStatus = "InitFinished" + RunningStatus TaskStatus = "Running" + CachedStatus TaskStatus = "Cached" + PauseStatus TaskStatus = "Pause" + DoneStatus TaskStatus = "Done" +) + +// StepEnum defines the MigrationTask CR .spec.steps +// +enum +// +kubebuilder:validation:Enum={preCheck,initStruct,initData,initStructLater} +type StepEnum string + +const ( + StepPreCheck StepEnum = "preCheck" + StepStructPreFullLoad StepEnum = "initStruct" + StepFullLoad StepEnum = "initData" + StepStructAfterFullLoad StepEnum = "initStructLater" + StepInitialization StepEnum = "initialization" + StepPreDelete StepEnum = "preDelete" + StepCdc StepEnum = "cdc" +) + +func (s StepEnum) String() string { + return string(s) +} + +func (s StepEnum) LowerCaseString() string { + return strings.ToLower(s.String()) +} + +func (s StepEnum) CliString() string { + switch s { + case StepPreCheck: + return CliStepPreCheck.String() + case StepStructPreFullLoad: + return CliStepInitStruct.String() + case StepFullLoad: + return CliStepInitData.String() + case StepCdc: + return CliStepCdc.String() + default: + return "unknown" + } +} + +type CliStepEnum string + +const ( + CliStepGlobal CliStepEnum = "global" + CliStepPreCheck CliStepEnum = "precheck" + CliStepInitStruct CliStepEnum = "init-struct" + CliStepInitData CliStepEnum = "init-data" + CliStepCdc CliStepEnum = "cdc" +) + +func (s CliStepEnum) String() string { + return string(s) +} + +// Phase defines the MigrationTemplate CR .status.phase +// +enum +// +kubebuilder:validation:Enum={Available,Unavailable} +type Phase string + +const ( + AvailablePhase Phase = "Available" + UnavailablePhase Phase = "Unavailable" +) + +type MigrationObjects struct { + Task *MigrationTask + Template *MigrationTemplate + + Jobs *batchv1.JobList + Pods *v1.PodList +} + +// +k8s:deepcopy-gen=false + +type IntOrStringMap map[string]interface{} + +func (in *IntOrStringMap) DeepCopyInto(out *IntOrStringMap) { + if in == nil { + *out = nil + } else { + *out = runtime.DeepCopyJSON(*in) + } +} + +func (in *IntOrStringMap) DeepCopy() *IntOrStringMap { + if in == nil { + return nil + } + out := new(IntOrStringMap) + in.DeepCopyInto(out) + return out +} diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index 609c1549b..ad7b41b94 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -121,6 +121,21 @@ const ( ResourceAddons = "addons" ) +// Migration API group +const ( + MigrationAPIGroup = "datamigration.apecloud.io" + MigrationAPIVersion = "v1alpha1" + ResourceMigrationTasks = "migrationtasks" + ResourceMigrationTemplates = "migrationtemplates" +) + +// Crd Api group +const ( + CustomResourceDefinitionAPIGroup = "apiextensions.k8s.io" + CustomResourceDefinitionAPIVersion = "v1" + ResourceCustomResourceDefinition = "customresourcedefinitions" +) + const ( None = "" @@ -284,3 +299,27 @@ func ClusterRoleGVR() schema.GroupVersionResource { func ClusterRoleBindingGVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: RBACAPIGroup, Version: RBACAPIVersion, Resource: ClusterRoleBindings} } + +func MigrationTaskGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{ + Group: MigrationAPIGroup, + Version: MigrationAPIVersion, + Resource: ResourceMigrationTasks, + } +} + +func MigrationTemplateGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{ + Group: MigrationAPIGroup, + Version: MigrationAPIVersion, + Resource: ResourceMigrationTemplates, + } +} + +func CustomResourceDefinitionGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{ + Group: CustomResourceDefinitionAPIGroup, + Version: CustomResourceDefinitionAPIVersion, + Resource: ResourceCustomResourceDefinition, + } +} diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 3bdecbe3d..48a8b65c6 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -307,10 +307,15 @@ func OpenBrowser(url string) error { } func TimeFormat(t *metav1.Time) string { + return TimeFormatWithDuration(t, time.Minute) +} + +// TimeFormatWithDuration format time with specified precision +func TimeFormatWithDuration(t *metav1.Time, duration time.Duration) string { if t == nil || t.IsZero() { return "" } - return TimeTimeFormat(t.Time) + return TimeTimeFormatWithDuration(t.Time, duration) } func TimeTimeFormat(t time.Time) string { @@ -318,6 +323,17 @@ func TimeTimeFormat(t time.Time) string { return t.Format(layout) } +func TimeTimeFormatWithDuration(t time.Time, precision time.Duration) string { + layout := "Jan 02,2006 15:04 UTC-0700" + switch precision { + case time.Second: + layout = "Jan 02,2006 15:04:05 UTC-0700" + case time.Millisecond: + layout = "Jan 02,2006 15:04:05.000 UTC-0700" + } + return t.Format(layout) +} + // GetHumanReadableDuration returns a succinct representation of the provided startTime and endTime // with limited precision for consumption by humans. func GetHumanReadableDuration(startTime metav1.Time, endTime metav1.Time) string { diff --git a/internal/cli/util/util_test.go b/internal/cli/util/util_test.go index db2562f00..b8da7c6e0 100644 --- a/internal/cli/util/util_test.go +++ b/internal/cli/util/util_test.go @@ -138,6 +138,9 @@ var _ = Describe("util", func() { t, _ := time.Parse(time.RFC3339, "2023-01-04T01:00:00.000Z") metav1Time := metav1.Time{Time: t} Expect(TimeFormat(&metav1Time)).Should(Equal("Jan 04,2023 01:00 UTC+0000")) + Expect(TimeFormatWithDuration(&metav1Time, time.Minute)).Should(Equal("Jan 04,2023 01:00 UTC+0000")) + Expect(TimeFormatWithDuration(&metav1Time, time.Second)).Should(Equal("Jan 04,2023 01:00:00 UTC+0000")) + Expect(TimeFormatWithDuration(&metav1Time, time.Millisecond)).Should(Equal("Jan 04,2023 01:00:00.000 UTC+0000")) }) It("CheckEmpty", func() { From dffbbed79eac27cbe69188afc2107a167d59e955 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Thu, 13 Apr 2023 20:32:12 +0800 Subject: [PATCH 022/439] chore: update mongodb helm (#2575) --- .../templates/backuppolicytemplate.yaml | 28 ++++++--- deploy/mongodb/templates/backuptool.yaml | 61 +++++++++++++++++++ 2 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 deploy/mongodb/templates/backuptool.yaml diff --git a/deploy/mongodb/templates/backuppolicytemplate.yaml b/deploy/mongodb/templates/backuppolicytemplate.yaml index c2bc88561..df4016795 100644 --- a/deploy/mongodb/templates/backuppolicytemplate.yaml +++ b/deploy/mongodb/templates/backuppolicytemplate.yaml @@ -1,15 +1,25 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-mongodb + name: mongodb-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: mongodb {{- include "mongodb.labels" . | nindent 4 }} spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: volumesnapshot - ttl: 168h0m0s - - credentialKeyword: - userKeyword: username - passwordKeyword: password + clusterDefinitionRef: mongodb + backupPolicies: + - componentDefRef: replicaset + ttl: 7d + schedule: + baseBackup: + type: full + enable: false + cronExpression: "0 18 * * 0" + snapshot: + target: + role: leader + connectionCredentialKey: + passwordKey: password + usernameKey: username + full: + backupToolName: xtrabackup-apecloud-mysql diff --git a/deploy/mongodb/templates/backuptool.yaml b/deploy/mongodb/templates/backuptool.yaml new file mode 100644 index 000000000..9bad85a27 --- /dev/null +++ b/deploy/mongodb/templates/backuptool.yaml @@ -0,0 +1,61 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupTool +metadata: + name: mongodb-physical-backup-tool + labels: + clusterdefinition.kubeblocks.io/name: mongodb +spec: + image: mongo:5.0.14 + deployKind: job + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 128Mi + env: + - name: DATA_DIR + value: /data/mongodb/db + physical: + restoreCommands: + - | + set -e + mkdir -p ${DATA_DIR} + res=`ls -A ${DATA_DIR}` + if [ ! -z ${res} ]; then + echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." + exit 1 + fi + tar -xvf ${BACKUP_DIR}/${BACKUP_NAME}.tar.gz -C ${DATA_DIR}/../ + mv ${DATA_DIR}/../${BACKUP_NAME}/* ${DATA_DIR} + PORT=27017 + MONGODB_ROOT=/data/mongodb + RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); + RPL_SET_NAME=${RPL_SET_NAME%-}; + mkdir -p $MONGODB_ROOT/db + mkdir -p $MONGODB_ROOT/logs + mkdir -p $MONGODB_ROOT/tmp + MODE=$1 + mongod $MODE --bind_ip_all --port $PORT --dbpath $MONGODB_ROOT/db --directoryperdb --logpath $MONGODB_ROOT/logs/mongodb.log --logappend --pidfilepath $MONGODB_ROOT/tmp/mongodb.pid& + until mongosh --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done + PID=`cat $MONGODB_ROOT/tmp/mongodb.pid` + + mongosh --quiet --port $PORT local --eval "db.system.replset.deleteOne({})" + mongosh --quiet --port $PORT local --eval "db.system.replset.find()" + mongosh --quiet --port $PORT admin --eval 'db.dropUser("root", {w: "majority", wtimeout: 4000})' || true + kill $PID + wait $PID + incrementalRestoreCommands: [] + logical: + restoreCommands: [] + incrementalRestoreCommands: [] + backupCommands: + - | + set -e + mkdir -p ${BACKUP_DIR}/${BACKUP_NAME} + cp -R ${DATA_DIR}/* ${BACKUP_DIR}/${BACKUP_NAME}/ + cd ${BACKUP_DIR} + tar -czvf ${BACKUP_NAME}.tar.gz ./${BACKUP_NAME} + rm -rf ${BACKUP_DIR}/${BACKUP_NAME} + incrementalBackupCommands: [] From 9089ada47e150bdb4904ef0c801369fd668a75cc Mon Sep 17 00:00:00 2001 From: shanshanying Date: Fri, 14 Apr 2023 10:51:16 +0800 Subject: [PATCH 023/439] fix: kbcli rename flag component (#2577) --- .../cli/kbcli_cluster_create-account.md | 16 ++--- .../cli/kbcli_cluster_create-user.md | 62 ------------------- .../cli/kbcli_cluster_delete-account.md | 10 +-- .../cli/kbcli_cluster_delete-user.md | 57 ----------------- docs/user_docs/cli/kbcli_cluster_desc-user.md | 57 ----------------- .../cli/kbcli_cluster_describe-account.md | 10 +-- .../cli/kbcli_cluster_describe-config.md | 16 ++--- .../cli/kbcli_cluster_explain-config.md | 18 +++--- .../user_docs/cli/kbcli_cluster_grant-role.md | 12 ++-- .../cli/kbcli_cluster_list-accounts.md | 8 +-- .../user_docs/cli/kbcli_cluster_list-users.md | 62 ------------------- .../cli/kbcli_cluster_revoke-role.md | 12 ++-- internal/cli/cmd/accounts/base.go | 4 +- internal/cli/cmd/cluster/accounts.go | 16 ++--- internal/cli/cmd/cluster/config.go | 20 +++--- internal/cli/cmd/cluster/describe_ops.go | 10 +-- internal/cli/cmd/cluster/errors.go | 4 +- 17 files changed, 78 insertions(+), 316 deletions(-) delete mode 100644 docs/user_docs/cli/kbcli_cluster_create-user.md delete mode 100644 docs/user_docs/cli/kbcli_cluster_delete-user.md delete mode 100644 docs/user_docs/cli/kbcli_cluster_desc-user.md delete mode 100644 docs/user_docs/cli/kbcli_cluster_list-users.md diff --git a/docs/user_docs/cli/kbcli_cluster_create-account.md b/docs/user_docs/cli/kbcli_cluster_create-account.md index 59e4e935b..28a3330d3 100644 --- a/docs/user_docs/cli/kbcli_cluster_create-account.md +++ b/docs/user_docs/cli/kbcli_cluster_create-account.md @@ -12,21 +12,21 @@ kbcli cluster create-account [flags] ``` # create account - kbcli cluster create-account NAME --component-name COMPNAME --username NAME --password PASSWD + kbcli cluster create-account NAME --component COMPNAME --username NAME --password PASSWD # create account without password - kbcli cluster create-account NAME --component-name COMPNAME --username NAME + kbcli cluster create-account NAME --component COMPNAME --username NAME # create account with expired interval - kbcli cluster create-account NAME --component-name COMPNAME --username NAME --password PASSWD --expiredAt 2046-01-02T15:04:05Z + kbcli cluster create-account NAME --component COMPNAME --username NAME --password PASSWD --expiredAt 2046-01-02T15:04:05Z ``` ### Options ``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for create-account - -i, --instance string Specify the name of instance to be connected. - -p, --password string Optional. Specify the password of user. The default value is empty, which means a random password will be generated. - -u, --username string Required. Specify the name of user, which must be unique. + --component string Specify the name of component to be connected. If not specified, the first component will be used. + -h, --help help for create-account + -i, --instance string Specify the name of instance to be connected. + -p, --password string Optional. Specify the password of user. The default value is empty, which means a random password will be generated. + -u, --username string Required. Specify the name of user, which must be unique. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_create-user.md b/docs/user_docs/cli/kbcli_cluster_create-user.md deleted file mode 100644 index c7c4a05d6..000000000 --- a/docs/user_docs/cli/kbcli_cluster_create-user.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: kbcli cluster create-user ---- - -Create user for a cluster - -``` -kbcli cluster create-user [flags] -``` - -### Examples - -``` - # create user - kbcli cluster create-user NAME --component-name COMPNAME --username NAME --password PASSWD - # create user without password - kbcli cluster create-user NAME --component-name COMPNAME --username NAME - # create user with expired interval - kbcli cluster create-user NAME --component-name COMPNAME --username NAME --password PASSWD --expiredAt 2046-01-02T15:04:05Z -``` - -### Options - -``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for create-user - -i, --instance string Specify the name of instance to be connected. - -p, --password string Optional. Specify the password of user. The default value is empty, which means a random password will be generated. - -u, --username string Required. Specify the name of user, which must be unique. - --verbose Print verbose information. -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli cluster](kbcli_cluster.md) - Cluster command. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_cluster_delete-account.md b/docs/user_docs/cli/kbcli_cluster_delete-account.md index 171243fb0..0cd2b59ab 100644 --- a/docs/user_docs/cli/kbcli_cluster_delete-account.md +++ b/docs/user_docs/cli/kbcli_cluster_delete-account.md @@ -12,16 +12,16 @@ kbcli cluster delete-account [flags] ``` # delete account by name - kbcli cluster delete-account NAME --component-name COMPNAME --username NAME + kbcli cluster delete-account NAME --component COMPNAME --username NAME ``` ### Options ``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for delete-account - -i, --instance string Specify the name of instance to be connected. - -u, --username string Required. Specify the name of user + --component string Specify the name of component to be connected. If not specified, the first component will be used. + -h, --help help for delete-account + -i, --instance string Specify the name of instance to be connected. + -u, --username string Required. Specify the name of user ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_delete-user.md b/docs/user_docs/cli/kbcli_cluster_delete-user.md deleted file mode 100644 index b7ca04a75..000000000 --- a/docs/user_docs/cli/kbcli_cluster_delete-user.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: kbcli cluster delete-user ---- - -Delete user for a cluster - -``` -kbcli cluster delete-user [flags] -``` - -### Examples - -``` - # delete user by name - kbcli cluster delete-user NAME --component-name COMPNAME --username NAME -``` - -### Options - -``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for delete-user - -i, --instance string Specify the name of instance to be connected. - -u, --username string Required. Specify the name of user - --verbose Print verbose information. -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli cluster](kbcli_cluster.md) - Cluster command. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_cluster_desc-user.md b/docs/user_docs/cli/kbcli_cluster_desc-user.md deleted file mode 100644 index acd9c6c38..000000000 --- a/docs/user_docs/cli/kbcli_cluster_desc-user.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: kbcli cluster desc-user ---- - -Describe user roles and related information - -``` -kbcli cluster desc-user [flags] -``` - -### Examples - -``` - # describe user and show role information - kbcli cluster desc-user NAME --component-name COMPNAME--username NAME -``` - -### Options - -``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for desc-user - -i, --instance string Specify the name of instance to be connected. - -u, --username string Required. Specify the name of user - --verbose Print verbose information. -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli cluster](kbcli_cluster.md) - Cluster command. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_cluster_describe-account.md b/docs/user_docs/cli/kbcli_cluster_describe-account.md index 527ba7ecd..c45332b1f 100644 --- a/docs/user_docs/cli/kbcli_cluster_describe-account.md +++ b/docs/user_docs/cli/kbcli_cluster_describe-account.md @@ -12,16 +12,16 @@ kbcli cluster describe-account [flags] ``` # describe account and show role information - kbcli cluster describe-account NAME --component-name COMPNAME--username NAME + kbcli cluster describe-account NAME --component COMPNAME--username NAME ``` ### Options ``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for describe-account - -i, --instance string Specify the name of instance to be connected. - -u, --username string Required. Specify the name of user + --component string Specify the name of component to be connected. If not specified, the first component will be used. + -h, --help help for describe-account + -i, --instance string Specify the name of instance to be connected. + -u, --username string Required. Specify the name of user ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_describe-config.md b/docs/user_docs/cli/kbcli_cluster_describe-config.md index f7127fd60..6bc44de87 100644 --- a/docs/user_docs/cli/kbcli_cluster_describe-config.md +++ b/docs/user_docs/cli/kbcli_cluster_describe-config.md @@ -15,23 +15,23 @@ kbcli cluster describe-config [flags] kbcli cluster describe-config mycluster # describe a component, e.g. cluster name is mycluster, component name is mysql - kbcli cluster describe-config mycluster --component-name=mysql + kbcli cluster describe-config mycluster --component=mysql # describe all configuration files. - kbcli cluster describe-config mycluster --component-name=mysql --show-detail + kbcli cluster describe-config mycluster --component=mysql --show-detail # describe a content of configuration file. - kbcli cluster describe-config mycluster --component-name=mysql --config-file=my.cnf --show-detail + kbcli cluster describe-config mycluster --component=mysql --config-file=my.cnf --show-detail ``` ### Options ``` - --component-name string Specify the name of Component to be describe (e.g. for apecloud-mysql: --component-name=mysql). If the cluster has only one component, unset the parameter." - --config-file strings Specify the name of the configuration file to be describe (e.g. for mysql: --config-file=my.cnf). If unset, all files. - --config-specs strings Specify the name of the configuration template to be describe. (e.g. for apecloud-mysql: --config-specs=mysql-3node-tpl) - -h, --help help for describe-config - --show-detail If true, the content of the files specified by config-file will be printed. + --component string Specify the name of Component to be describe (e.g. for apecloud-mysql: --component=mysql). If the cluster has only one component, unset the parameter." + --config-file strings Specify the name of the configuration file to be describe (e.g. for mysql: --config-file=my.cnf). If unset, all files. + --config-specs strings Specify the name of the configuration template to be describe. (e.g. for apecloud-mysql: --config-specs=mysql-3node-tpl) + -h, --help help for describe-config + --show-detail If true, the content of the files specified by config-file will be printed. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_explain-config.md b/docs/user_docs/cli/kbcli_cluster_explain-config.md index 569b4d2d8..8abe9394f 100644 --- a/docs/user_docs/cli/kbcli_cluster_explain-config.md +++ b/docs/user_docs/cli/kbcli_cluster_explain-config.md @@ -15,24 +15,24 @@ kbcli cluster explain-config [flags] kbcli cluster explain-config mycluster # describe a specified configure template, e.g. cluster name is mycluster - kbcli cluster explain-config mycluster --component-name=mysql --config-specs=mysql-3node-tpl + kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl # describe a specified configure template, e.g. cluster name is mycluster - kbcli cluster explain-config mycluster --component-name=mysql --config-specs=mysql-3node-tpl --trunc-document=false --trunc-enum=false + kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl --trunc-document=false --trunc-enum=false # describe a specified parameters, e.g. cluster name is mycluster - kbcli cluster explain-config mycluster --component-name=mysql --config-specs=mysql-3node-tpl --param=sql_mode + kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl --param=sql_mode ``` ### Options ``` - --component-name string Specify the name of Component to be describe (e.g. for apecloud-mysql: --component-name=mysql). If the cluster has only one component, unset the parameter." - --config-specs strings Specify the name of the configuration template to be describe. (e.g. for apecloud-mysql: --config-specs=mysql-3node-tpl) - -h, --help help for explain-config - --param string Specify the name of parameter to be query. It clearly display the details of the parameter. - --trunc-document If the document length of the parameter is greater than 100, it will be truncated. - --trunc-enum If the value list length of the parameter is greater than 20, it will be truncated. (default true) + --component string Specify the name of Component to be describe (e.g. for apecloud-mysql: --component=mysql). If the cluster has only one component, unset the parameter." + --config-specs strings Specify the name of the configuration template to be describe. (e.g. for apecloud-mysql: --config-specs=mysql-3node-tpl) + -h, --help help for explain-config + --param string Specify the name of parameter to be query. It clearly display the details of the parameter. + --trunc-document If the document length of the parameter is greater than 100, it will be truncated. + --trunc-enum If the value list length of the parameter is greater than 20, it will be truncated. (default true) ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_grant-role.md b/docs/user_docs/cli/kbcli_cluster_grant-role.md index 8d1ad9c82..632e2b143 100644 --- a/docs/user_docs/cli/kbcli_cluster_grant-role.md +++ b/docs/user_docs/cli/kbcli_cluster_grant-role.md @@ -12,17 +12,17 @@ kbcli cluster grant-role [flags] ``` # grant role to user - kbcli cluster grant-role NAME --component-name COMPNAME --username NAME --role ROLENAME + kbcli cluster grant-role NAME --component COMPNAME --username NAME --role ROLENAME ``` ### Options ``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for grant-role - -i, --instance string Specify the name of instance to be connected. - -r, --role string Role name should be one of {SUPERUSER, READWRITE, READONLY} - -u, --username string Required. Specify the name of user. + --component string Specify the name of component to be connected. If not specified, the first component will be used. + -h, --help help for grant-role + -i, --instance string Specify the name of instance to be connected. + -r, --role string Role name should be one of {SUPERUSER, READWRITE, READONLY} + -u, --username string Required. Specify the name of user. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_list-accounts.md b/docs/user_docs/cli/kbcli_cluster_list-accounts.md index aa900d32a..5d5efcfbd 100644 --- a/docs/user_docs/cli/kbcli_cluster_list-accounts.md +++ b/docs/user_docs/cli/kbcli_cluster_list-accounts.md @@ -12,7 +12,7 @@ kbcli cluster list-accounts [flags] ``` # list all users from specified component of a cluster - kbcli cluster list-accounts NAME --component-name COMPNAME --show-connected-users + kbcli cluster list-accounts NAME --component COMPNAME --show-connected-users # list all users from cluster's one particular instance kbcli cluster list-accounts NAME -i INSTANCE @@ -21,9 +21,9 @@ kbcli cluster list-accounts [flags] ### Options ``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for list-accounts - -i, --instance string Specify the name of instance to be connected. + --component string Specify the name of component to be connected. If not specified, the first component will be used. + -h, --help help for list-accounts + -i, --instance string Specify the name of instance to be connected. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_list-users.md b/docs/user_docs/cli/kbcli_cluster_list-users.md deleted file mode 100644 index f1a7239c1..000000000 --- a/docs/user_docs/cli/kbcli_cluster_list-users.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: kbcli cluster list-users ---- - -List users for a cluster - -``` -kbcli cluster list-users [flags] -``` - -### Examples - -``` - # list all users from specified component of a cluster - kbcli cluster list-users NAME --component-name COMPNAME --show-connected-users - - # list all users of a cluster, by default the first component will be used - kbcli cluster list-users NAME --show-connected-users - - # list all users from cluster's one particular instance - kbcli cluster list-users NAME -i INSTANCE -``` - -### Options - -``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for list-users - -i, --instance string Specify the name of instance to be connected. - --verbose Print verbose information. -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli cluster](kbcli_cluster.md) - Cluster command. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_cluster_revoke-role.md b/docs/user_docs/cli/kbcli_cluster_revoke-role.md index 2a26d7ab8..454cc7bee 100644 --- a/docs/user_docs/cli/kbcli_cluster_revoke-role.md +++ b/docs/user_docs/cli/kbcli_cluster_revoke-role.md @@ -12,17 +12,17 @@ kbcli cluster revoke-role [flags] ``` # revoke role from user - kbcli cluster revoke-role NAME --component-name COMPNAME --role ROLENAME + kbcli cluster revoke-role NAME --component COMPNAME --role ROLENAME ``` ### Options ``` - --component-name string Specify the name of component to be connected. If not specified, the first component will be used. - -h, --help help for revoke-role - -i, --instance string Specify the name of instance to be connected. - -r, --role string Role name should be one of {SUPERUSER, READWRITE, READONLY} - -u, --username string Required. Specify the name of user. + --component string Specify the name of component to be connected. If not specified, the first component will be used. + -h, --help help for revoke-role + -i, --instance string Specify the name of instance to be connected. + -r, --role string Role name should be one of {SUPERUSER, READWRITE, READONLY} + -u, --username string Required. Specify the name of user. ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/accounts/base.go b/internal/cli/cmd/accounts/base.go index 9b029ce4a..c21052941 100644 --- a/internal/cli/cmd/accounts/base.go +++ b/internal/cli/cmd/accounts/base.go @@ -54,7 +54,7 @@ var ( errMissingRoleName = fmt.Errorf("please specify at least ONE role name") errInvalidRoleName = fmt.Errorf("invalid role name, should be one of [SUPERUSER, READWRITE, READONLY] ") errInvalidOp = fmt.Errorf("invalid operation") - errCompNameOrInstName = fmt.Errorf("please specify either --component-name or --instance, not both") + errCompNameOrInstName = fmt.Errorf("please specify either --component or --instance, not both") ) func NewAccountBaseOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, op bindings.OperationKind) *AccountBaseOptions { @@ -65,7 +65,7 @@ func NewAccountBaseOptions(f cmdutil.Factory, streams genericclioptions.IOStream } func (o *AccountBaseOptions) AddFlags(cmd *cobra.Command) { - cmd.Flags().StringVar(&o.ComponentName, "component-name", "", "Specify the name of component to be connected. If not specified, the first component will be used.") + cmd.Flags().StringVar(&o.ComponentName, "component", "", "Specify the name of component to be connected. If not specified, the first component will be used.") cmd.Flags().StringVarP(&o.PodName, "instance", "i", "", "Specify the name of instance to be connected.") } diff --git a/internal/cli/cmd/cluster/accounts.go b/internal/cli/cmd/cluster/accounts.go index 04f6b4fdd..d9162a005 100644 --- a/internal/cli/cmd/cluster/accounts.go +++ b/internal/cli/cmd/cluster/accounts.go @@ -31,37 +31,37 @@ import ( var ( createUserExamples = templates.Examples(` # create account - kbcli cluster create-account NAME --component-name COMPNAME --username NAME --password PASSWD + kbcli cluster create-account NAME --component COMPNAME --username NAME --password PASSWD # create account without password - kbcli cluster create-account NAME --component-name COMPNAME --username NAME + kbcli cluster create-account NAME --component COMPNAME --username NAME # create account with expired interval - kbcli cluster create-account NAME --component-name COMPNAME --username NAME --password PASSWD --expiredAt 2046-01-02T15:04:05Z + kbcli cluster create-account NAME --component COMPNAME --username NAME --password PASSWD --expiredAt 2046-01-02T15:04:05Z `) deleteUserExamples = templates.Examples(` # delete account by name - kbcli cluster delete-account NAME --component-name COMPNAME --username NAME + kbcli cluster delete-account NAME --component COMPNAME --username NAME `) descUserExamples = templates.Examples(` # describe account and show role information - kbcli cluster describe-account NAME --component-name COMPNAME--username NAME + kbcli cluster describe-account NAME --component COMPNAME--username NAME `) listUsersExample = templates.Examples(` # list all users from specified component of a cluster - kbcli cluster list-accounts NAME --component-name COMPNAME --show-connected-users + kbcli cluster list-accounts NAME --component COMPNAME --show-connected-users # list all users from cluster's one particular instance kbcli cluster list-accounts NAME -i INSTANCE `) grantRoleExamples = templates.Examples(` # grant role to user - kbcli cluster grant-role NAME --component-name COMPNAME --username NAME --role ROLENAME + kbcli cluster grant-role NAME --component COMPNAME --username NAME --role ROLENAME `) revokeRoleExamples = templates.Examples(` # revoke role from user - kbcli cluster revoke-role NAME --component-name COMPNAME --role ROLENAME + kbcli cluster revoke-role NAME --component COMPNAME --role ROLENAME `) ) diff --git a/internal/cli/cmd/cluster/config.go b/internal/cli/cmd/cluster/config.go index a919a12d8..4760b59f9 100644 --- a/internal/cli/cmd/cluster/config.go +++ b/internal/cli/cmd/cluster/config.go @@ -92,32 +92,32 @@ var ( kbcli cluster describe-config mycluster # describe a component, e.g. cluster name is mycluster, component name is mysql - kbcli cluster describe-config mycluster --component-name=mysql + kbcli cluster describe-config mycluster --component=mysql - # describe all configuration files. - kbcli cluster describe-config mycluster --component-name=mysql --show-detail + # describe all configuration files. + kbcli cluster describe-config mycluster --component=mysql --show-detail - # describe a content of configuration file. - kbcli cluster describe-config mycluster --component-name=mysql --config-file=my.cnf --show-detail`) + # describe a content of configuration file. + kbcli cluster describe-config mycluster --component=mysql --config-file=my.cnf --show-detail`) explainReconfigureExample = templates.Examples(` # describe a cluster, e.g. cluster name is mycluster kbcli cluster explain-config mycluster # describe a specified configure template, e.g. cluster name is mycluster - kbcli cluster explain-config mycluster --component-name=mysql --config-specs=mysql-3node-tpl + kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl # describe a specified configure template, e.g. cluster name is mycluster - kbcli cluster explain-config mycluster --component-name=mysql --config-specs=mysql-3node-tpl --trunc-document=false --trunc-enum=false + kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl --trunc-document=false --trunc-enum=false # describe a specified parameters, e.g. cluster name is mycluster - kbcli cluster explain-config mycluster --component-name=mysql --config-specs=mysql-3node-tpl --param=sql_mode`) + kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl --param=sql_mode`) diffConfigureExample = templates.Examples(` - # compare config files + # compare config files kbcli cluster diff-config opsrequest1 opsrequest2`) ) func (r *reconfigureOptions) addCommonFlags(cmd *cobra.Command) { - cmd.Flags().StringVar(&r.componentName, "component-name", "", "Specify the name of Component to be describe (e.g. for apecloud-mysql: --component-name=mysql). If the cluster has only one component, unset the parameter.\"") + cmd.Flags().StringVar(&r.componentName, "component", "", "Specify the name of Component to be describe (e.g. for apecloud-mysql: --component=mysql). If the cluster has only one component, unset the parameter.\"") cmd.Flags().StringSliceVar(&r.configSpecs, "config-specs", nil, "Specify the name of the configuration template to be describe. (e.g. for apecloud-mysql: --config-specs=mysql-3node-tpl)") } diff --git a/internal/cli/cmd/cluster/describe_ops.go b/internal/cli/cmd/cluster/describe_ops.go index 4f03a3a64..d75e384ed 100644 --- a/internal/cli/cmd/cluster/describe_ops.go +++ b/internal/cli/cmd/cluster/describe_ops.go @@ -231,7 +231,7 @@ func (o *describeOpsOptions) getRestartCommand(spec appsv1alpha1.OpsRequestSpec) componentNames[i] = v.ComponentName } return []string{ - fmt.Sprintf("kbcli cluster restart %s --component-names=%s", spec.ClusterRef, + fmt.Sprintf("kbcli cluster restart %s --components=%s", spec.ClusterRef, strings.Join(componentNames, ",")), } } @@ -268,7 +268,7 @@ func (o *describeOpsOptions) getVerticalScalingCommand(spec appsv1alpha1.OpsRequ commands := make([]string, len(componentNameSlice)) for i := range componentNameSlice { resource := resourceSlice[i].(corev1.ResourceRequirements) - commands[i] = fmt.Sprintf("kbcli cluster vertical-scale %s --component-names=%s", + commands[i] = fmt.Sprintf("kbcli cluster vertical-scale %s --components=%s", spec.ClusterRef, strings.Join(componentNameSlice[i], ",")) commands[i] += o.addResourceFlag("requests.cpu", resource.Requests.Cpu()) commands[i] += o.addResourceFlag("requests.memory", resource.Requests.Memory()) @@ -293,7 +293,7 @@ func (o *describeOpsOptions) getHorizontalScalingCommand(spec appsv1alpha1.OpsRe spec.HorizontalScalingList, convertObject, getCompName) commands := make([]string, len(componentNameSlice)) for i := range componentNameSlice { - commands[i] = fmt.Sprintf("kbcli cluster horizontal-scale %s --component-names=%s --replicas=%d", + commands[i] = fmt.Sprintf("kbcli cluster horizontal-scale %s --components=%s --replicas=%d", spec.ClusterRef, strings.Join(componentNameSlice[i], ","), replicasSlice[i].(int32)) } return commands @@ -313,7 +313,7 @@ func (o *describeOpsOptions) getVolumeExpansionCommand(spec appsv1alpha1.OpsRequ v.VolumeClaimTemplates, convertObject, getVCTName) for i := range vctNameSlice { storage := storageSlice[i].(resource.Quantity) - commands = append(commands, fmt.Sprintf("kbcli cluster volume-expand %s --component-names=%s --volume-claim-template-names=%s --storage=%s", + commands = append(commands, fmt.Sprintf("kbcli cluster volume-expand %s --components=%s --volume-claim-template-names=%s --storage=%s", spec.ClusterRef, v.ComponentName, strings.Join(vctNameSlice[i], ","), storage.String())) } } @@ -341,7 +341,7 @@ func (o *describeOpsOptions) getReconfiguringCommand(spec appsv1alpha1.OpsReques commandArgs = append(commandArgs, "cluster") commandArgs = append(commandArgs, "configure") commandArgs = append(commandArgs, spec.ClusterRef) - commandArgs = append(commandArgs, fmt.Sprintf("--component-names=%s", componentName)) + commandArgs = append(commandArgs, fmt.Sprintf("--components=%s", componentName)) commandArgs = append(commandArgs, fmt.Sprintf("--config-spec=%s", configuration.Name)) config := configuration.Keys[0] diff --git a/internal/cli/cmd/cluster/errors.go b/internal/cli/cmd/cluster/errors.go index 2e782da39..92b28622a 100644 --- a/internal/cli/cmd/cluster/errors.go +++ b/internal/cli/cmd/cluster/errors.go @@ -23,11 +23,11 @@ import ( var ( clusterNotExistErrMessage = "cluster[name=%s] is not exist. Please check that is spelled correctly." - componentNotExistErrMessage = "cluster[name=%s] does not has this component[name=%s]. Please check that --component-name is spelled correctly." + componentNotExistErrMessage = "cluster[name=%s] does not has this component[name=%s]. Please check that --component is spelled correctly." missingClusterArgErrMassage = "cluster name should be specified, using --help." missingUpdatedParametersErrMessage = "missing updated parameters, using --help." - multiComponentsErrorMessage = "when multi component exist, must specify which component to use. Please using --component-name" + multiComponentsErrorMessage = "when multi component exist, must specify which component to use. Please using --component" multiConfigTemplateErrorMessage = "when multi config template exist, must specify which config template to use. Please using --config-spec" multiConfigFileErrorMessage = "when multi config files exist, must specify which config file to update. Please using --config-file" From b265bc3657a4b541374e875a3f90116c5a1919f7 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Fri, 14 Apr 2023 11:28:47 +0800 Subject: [PATCH 024/439] chore: mongodb support configure or edit-config command for kbcli (#2578) --- deploy/mongodb/templates/clusterdefinition.yaml | 3 +++ deploy/mongodb/templates/configconstraint.yaml | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 deploy/mongodb/templates/configconstraint.yaml diff --git a/deploy/mongodb/templates/clusterdefinition.yaml b/deploy/mongodb/templates/clusterdefinition.yaml index a08280796..b2ca36283 100644 --- a/deploy/mongodb/templates/clusterdefinition.yaml +++ b/deploy/mongodb/templates/clusterdefinition.yaml @@ -29,6 +29,9 @@ spec: templateRef: mongodb5.0-config-template namespace: {{ .Release.Namespace }} volumeName: mongodb-config + constraintRef: mongodb-config-constraints + keys: + - mongodb.conf defaultMode: 256 - name: mongodb-metrics-config templateRef: mongodb-metrics-config diff --git a/deploy/mongodb/templates/configconstraint.yaml b/deploy/mongodb/templates/configconstraint.yaml new file mode 100644 index 000000000..292151749 --- /dev/null +++ b/deploy/mongodb/templates/configconstraint.yaml @@ -0,0 +1,13 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ConfigConstraint +metadata: + name: mongodb-config-constraints + labels: + {{- include "mongodb.labels" . | nindent 4 }} +spec: + configurationSchema: + cue: "" + + # mysql configuration file format + formatterConfig: + format: yaml \ No newline at end of file From eea96353f6eaf8709956833738f0fcd4e8f5060b Mon Sep 17 00:00:00 2001 From: Nayuta <111858489+nayutah@users.noreply.github.com> Date: Fri, 14 Apr 2023 13:43:55 +0800 Subject: [PATCH 025/439] chore: support helm create milvus instance (#2549) --- Makefile | 1 + deploy/milvus-cluster/Chart.yaml | 9 ++ deploy/milvus-cluster/templates/NOTES.txt | 2 + deploy/milvus-cluster/templates/_helpers.tpl | 62 ++++++++++++ deploy/milvus-cluster/templates/cluster.yaml | 101 +++++++++++++++++++ deploy/milvus-cluster/values.yaml | 29 ++++++ 6 files changed, 204 insertions(+) create mode 100644 deploy/milvus-cluster/Chart.yaml create mode 100644 deploy/milvus-cluster/templates/NOTES.txt create mode 100644 deploy/milvus-cluster/templates/_helpers.tpl create mode 100644 deploy/milvus-cluster/templates/cluster.yaml create mode 100644 deploy/milvus-cluster/values.yaml diff --git a/Makefile b/Makefile index 781fab822..77dd25c4e 100644 --- a/Makefile +++ b/Makefile @@ -388,6 +388,7 @@ bump-chart-ver: \ bump-single-chart-ver.redis \ bump-single-chart-ver.redis-cluster \ bump-single-chart-ver.milvus \ + bump-single-chart-ver.milvus-cluster \ bump-single-chart-ver.qdrant \ bump-single-chart-ver.qdrant-cluster \ bump-single-chart-ver.weaviate \ diff --git a/deploy/milvus-cluster/Chart.yaml b/deploy/milvus-cluster/Chart.yaml new file mode 100644 index 000000000..b20fdeb74 --- /dev/null +++ b/deploy/milvus-cluster/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: milvus-cluster +description: A Milvus cluster Helm chart for KubeBlocks. + +type: application + +version: 0.5.0-alpha.3 + +appVersion: "2.2.4" diff --git a/deploy/milvus-cluster/templates/NOTES.txt b/deploy/milvus-cluster/templates/NOTES.txt new file mode 100644 index 000000000..c3b3453e3 --- /dev/null +++ b/deploy/milvus-cluster/templates/NOTES.txt @@ -0,0 +1,2 @@ +1. Get the application URL by running these commands: + diff --git a/deploy/milvus-cluster/templates/_helpers.tpl b/deploy/milvus-cluster/templates/_helpers.tpl new file mode 100644 index 000000000..d6843da01 --- /dev/null +++ b/deploy/milvus-cluster/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "milvus.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "milvus.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "milvus.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "milvus.labels" -}} +helm.sh/chart: {{ include "milvus.chart" . }} +{{ include "milvus.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "milvus.selectorLabels" -}} +app.kubernetes.io/name: {{ include "milvus.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "milvus.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "milvus.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/milvus-cluster/templates/cluster.yaml b/deploy/milvus-cluster/templates/cluster.yaml new file mode 100644 index 000000000..fdac7bee9 --- /dev/null +++ b/deploy/milvus-cluster/templates/cluster.yaml @@ -0,0 +1,101 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + name: {{ .Release.Name }} + labels: {{ include "milvus.labels" . | nindent 4 }} +spec: + clusterDefinitionRef: milvus-standalone # ref clusterdefinition.name + clusterVersionRef: milvus-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} # ref clusterversion.name + terminationPolicy: {{ .Values.terminationPolicy }} + affinity: + {{- with .Values.topologyKeys }} + topologyKeys: {{ . | toYaml | nindent 6 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: {{ . | toYaml | nindent 4 }} + {{- end }} + componentSpecs: + - name: milvus # user-defined + componentDefRef: milvus # ref clusterdefinition components.name + monitor: {{ .Values.monitor.enabled | default false }} + replicas: {{ .Values.replicaCount | default 1 }} + {{- with .Values.resources }} + resources: + {{- with .limits }} + limits: + cpu: {{ .cpu | quote }} + memory: {{ .memory | quote }} + {{- end }} + {{- with .requests }} + requests: + cpu: {{ .cpu | quote }} + memory: {{ .memory | quote }} + {{- end }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + storageClassName: {{ .Values.persistence.data.storageClassName }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.data.size }} + {{- end }} + - name: etcd # user-defined + componentDefRef: etcd # ref clusterdefinition components.name + monitor: {{ .Values.monitor.enabled | default false }} + replicas: {{ .Values.replicaCount | default 1 }} + {{- with .Values.resources }} + resources: + {{- with .limits }} + limits: + cpu: {{ .cpu | quote }} + memory: {{ .memory | quote }} + {{- end }} + {{- with .requests }} + requests: + cpu: {{ .cpu | quote }} + memory: {{ .memory | quote }} + {{- end }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + storageClassName: {{ .Values.persistence.data.storageClassName }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.data.size }} + {{- end }} + - name: minio # user-defined + componentDefRef: minio # ref clusterdefinition components.name + monitor: {{ .Values.monitor.enabled | default false }} + replicas: {{ .Values.replicaCount | default 1 }} + {{- with .Values.resources }} + resources: + {{- with .limits }} + limits: + cpu: {{ .cpu | quote }} + memory: {{ .memory | quote }} + {{- end }} + {{- with .requests }} + requests: + cpu: {{ .cpu | quote }} + memory: {{ .memory | quote }} + {{- end }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + storageClassName: {{ .Values.persistence.data.storageClassName }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.data.size }} + {{- end }} \ No newline at end of file diff --git a/deploy/milvus-cluster/values.yaml b/deploy/milvus-cluster/values.yaml new file mode 100644 index 000000000..3f14e3c39 --- /dev/null +++ b/deploy/milvus-cluster/values.yaml @@ -0,0 +1,29 @@ +# Default values for wesqlcluster. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 +terminationPolicy: Delete + +clusterVersionOverride: "" + +monitor: + enabled: false + +resources: { } + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + + # limits: + # cpu: 500m + # memory: 2Gi + # requests: + # cpu: 100m + # memory: 1Gi +persistence: + enabled: true + data: + storageClassName: + size: 10Gi From 245fcbffb300fec2e8f9688872004a08b0ff17a7 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Fri, 14 Apr 2023 14:08:30 +0800 Subject: [PATCH 026/439] chore: set secrets GITLAB_ACCESS_TOKEN to env (#2581) --- .github/workflows/cicd-pull-request.yml | 4 ++-- .github/workflows/cicd-push.yml | 5 +++-- .github/workflows/release-publish.yml | 5 +++-- .github/workflows/release-sync.yml | 5 +++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index c3122f053..c0c98b1b7 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -8,7 +8,7 @@ env: GITLAB_GO_CACHE_PROJECT_ID: 98800 GO_CACHE: "go-cache" GO_CACHE_DIR: "/root/.cache" - + GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }} jobs: trigger-mode: @@ -43,7 +43,7 @@ jobs: --project-id ${{ env.GITLAB_GO_CACHE_PROJECT_ID }} \ --tag-name ${{ env.GO_CACHE }} \ --asset-name ${{ env.GO_CACHE }}.tar.gz \ - --access-token ${{ secrets.GITLAB_ACCESS_TOKEN }} + --access-token ${{ env.GITLAB_ACCESS_TOKEN }} - name: Extract ${{ env.GO_CACHE }} uses: a7ul/tar-action@v1.1.3 diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index abfcf92c7..c99189f7f 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -13,6 +13,7 @@ env: GITLAB_GO_CACHE_PROJECT_ID: 98800 GO_CACHE: "go-cache" GO_CACHE_DIR: "/root/.cache" + GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }} jobs: @@ -116,7 +117,7 @@ jobs: --project-id ${{ env.GITLAB_GO_CACHE_PROJECT_ID }} \ --tag-name ${{ env.GO_CACHE }} \ --asset-name ${{ env.GO_CACHE }}.tar.gz \ - --access-token ${{ secrets.GITLAB_ACCESS_TOKEN }} + --access-token ${{ env.GITLAB_ACCESS_TOKEN }} - name: Extract ${{ env.GO_CACHE }} if: ${{ github.ref_name != 'main' }} @@ -175,7 +176,7 @@ jobs: --project-id ${{ env.GITLAB_GO_CACHE_PROJECT_ID }} \ --tag-name ${{ env.GO_CACHE }} \ --asset-path ${{ env.GO_CACHE }}.tar.gz \ - --access-token ${{ secrets.GITLAB_ACCESS_TOKEN }} + --access-token ${{ env.GITLAB_ACCESS_TOKEN }} check-image: needs: trigger-mode diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 6ad92b85e..8e48bb56b 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -12,6 +12,7 @@ env: CLI_NAME: 'kbcli' CLI_REPO: 'apecloud/kbcli' GITLAB_KBCLI_PROJECT_ID: 85948 + GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }} jobs: create-release-kbcli: @@ -34,7 +35,7 @@ jobs: --type 1 \ --project-id ${{ env.GITLAB_KBCLI_PROJECT_ID }} \ --tag-name ${{ env.TAG_NAME }} \ - --access-token ${{ secrets.GITLAB_ACCESS_TOKEN }} + --access-token ${{ env.GITLAB_ACCESS_TOKEN }} upload-release-assert: @@ -134,4 +135,4 @@ jobs: --tag-name ${{ env.TAG_NAME }} \ --asset-path ./bin/${{ env.ASSET_NAME }} \ --asset-name ${{ env.ASSET_NAME }} \ - --access-token ${{ secrets.GITLAB_ACCESS_TOKEN }} + --access-token ${{ env.GITLAB_ACCESS_TOKEN }} diff --git a/.github/workflows/release-sync.yml b/.github/workflows/release-sync.yml index 91d3eee1a..dd4ed8236 100644 --- a/.github/workflows/release-sync.yml +++ b/.github/workflows/release-sync.yml @@ -8,6 +8,7 @@ on: env: CLI_REPO: 'apecloud/kbcli' GITLAB_KBCLI_PROJECT_ID: 85948 + GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }} jobs: update-release-kbcli: @@ -28,13 +29,13 @@ jobs: --type 5 \ --tag-name $LATEST_RELEASE_TAG \ --github-repo ${{ env.CLI_REPO }} \ - --github-token ${{ secrets.PERSONAL_ACCESS_TOKEN }} + --github-token ${{ env.PERSONAL_ACCESS_TOKEN }} bash ${{ github.workspace }}/.github/utils/release_gitlab.sh \ --type 4 \ --tag-name $LATEST_RELEASE_TAG \ --project-id ${{ env.GITLAB_KBCLI_PROJECT_ID }} \ - --access-token ${{ secrets.GITLAB_ACCESS_TOKEN }} + --access-token ${{ env.GITLAB_ACCESS_TOKEN }} echo release_version=$LATEST_RELEASE_TAG >> $GITHUB_OUTPUT From adfaef8ebd86d0022549a3243670315a8abca532 Mon Sep 17 00:00:00 2001 From: ToKliar <52400562+ToKliar@users.noreply.github.com> Date: Fri, 14 Apr 2023 14:35:54 +0800 Subject: [PATCH 027/439] feat: support more metrics about replication for postgresql exporter (#2568) --- deploy/postgresql/values.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/deploy/postgresql/values.yaml b/deploy/postgresql/values.yaml index 33cf5582a..1030cab15 100644 --- a/deploy/postgresql/values.yaml +++ b/deploy/postgresql/values.yaml @@ -130,6 +130,20 @@ metrics: - start_time_seconds: usage: "GAUGE" description: "Time at which postmaster started" + + pg_replication: + query: | + SELECT + (case when (not pg_is_in_recovery() or pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()) then 0 else greatest (0, extract(epoch from (now() - pg_last_xact_replay_timestamp()))) end) as lag, + (case when pg_is_in_recovery() then 0 else 1 end) as is_master + master: true + metrics: + - lag: + usage: "GAUGE" + description: "Replication lag behind master in seconds" + - is_master: + usage: "GAUGE" + description: "Instance is master or slave" pg_stat_user_tables: query: | From a8daa482585c716906d2eff99767b3a22c138765 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Fri, 14 Apr 2023 14:43:16 +0800 Subject: [PATCH 028/439] chore: add the new API ComponentClassDefinition and rename ClassFamily to ComponentResourceConstraint (#2459) --- PROJECT | 10 +- apis/apps/v1alpha1/cluster_types.go | 14 + apis/apps/v1alpha1/cluster_webhook.go | 18 ++ .../componentclassdefinition_types.go | 177 ++++++++++++ ...o => componentresourceconstraint_types.go} | 67 ++--- ...componentresourceconstraint_types_test.go} | 19 +- apis/apps/v1alpha1/opsrequest_types.go | 9 +- apis/apps/v1alpha1/opsrequest_webhook.go | 76 +++++ apis/apps/v1alpha1/type.go | 48 ++++ cmd/manager/main.go | 9 + .../bases/apps.kubeblocks.io_clusters.yaml | 13 + ...beblocks.io_componentclassdefinitions.yaml | 261 ++++++++++++++++++ ...locks.io_componentresourceconstraints.yaml | 26 +- .../bases/apps.kubeblocks.io_opsrequests.yaml | 6 + config/crd/kustomization.yaml | 9 +- ...on_in_apps_componentclassdefinitions.yaml} | 2 +- ..._in_apps_componentresourceconstraints.yaml | 7 + ...ok_in_apps_componentclassdefinitions.yaml} | 2 +- ..._in_apps_componentresourceconstraints.yaml | 16 ++ ...componentclassdefinition_editor_role.yaml} | 10 +- ...componentclassdefinition_viewer_role.yaml} | 10 +- ...mponentresourceconstraint_editor_role.yaml | 31 +++ ...mponentresourceconstraint_viewer_role.yaml | 27 ++ config/rbac/role.yaml | 42 ++- ...pps_v1alpha1_componentclassdefinition.yaml | 12 + ...v1alpha1_componentresourceconstraint.yaml} | 8 +- controllers/apps/class_controller.go | 91 ++++++ controllers/apps/class_controller_test.go | 69 +++++ controllers/apps/cluster_controller.go | 11 +- controllers/apps/operations/suite_test.go | 2 + .../apps/operations/vertical_scaling.go | 16 +- controllers/apps/suite_test.go | 9 + deploy/apecloud-mysql/templates/class.yaml | 112 ++++---- deploy/helm/config/rbac/role.yaml | 42 ++- .../crds/apps.kubeblocks.io_clusters.yaml | 13 + ...beblocks.io_componentclassdefinitions.yaml | 261 ++++++++++++++++++ ...locks.io_componentresourceconstraints.yaml | 26 +- .../crds/apps.kubeblocks.io_opsrequests.yaml | 6 + ...ily.yaml => componentclassconstraint.yaml} | 16 +- docs/user_docs/cli/kbcli_class_create.md | 6 +- docs/user_docs/cli/kbcli_class_list.md | 2 +- docs/user_docs/cli/kbcli_cluster_create.md | 6 +- docs/user_docs/cli/kbcli_cluster_vscale.md | 3 + internal/class/class_utils.go | 225 ++++++--------- internal/class/class_utils_test.go | 32 ++- internal/class/types.go | 86 +----- internal/class/types_test.go | 42 +-- internal/cli/cmd/class/class.go | 5 - internal/cli/cmd/class/create.go | 145 +++++----- internal/cli/cmd/class/create_test.go | 45 +-- internal/cli/cmd/class/list.go | 50 ++-- internal/cli/cmd/class/list_test.go | 36 +-- internal/cli/cmd/class/suite_test.go | 31 ++- internal/cli/cmd/class/template.go | 20 +- internal/cli/cmd/cluster/create.go | 59 ++-- internal/cli/cmd/cluster/operations.go | 3 + .../template/cluster_operations_template.cue | 4 + internal/cli/testing/fake.go | 23 +- internal/cli/testing/testdata/class.yaml | 132 +++++---- .../cli/testing/testdata/custom_class.yaml | 64 ++--- ....yaml => resource-constraint-general.yaml} | 8 +- ...resource-constraint-memory-optimized.yaml} | 8 +- internal/cli/types/types.go | 48 ++-- .../lifecycle/transformer_fill_class.go | 79 +++--- internal/generics/type.go | 2 + .../apps/componentclassdefinition_factory.go | 74 +++++ 66 files changed, 2006 insertions(+), 835 deletions(-) create mode 100644 apis/apps/v1alpha1/componentclassdefinition_types.go rename apis/apps/v1alpha1/{classfamily_types.go => componentresourceconstraint_types.go} (75%) rename apis/apps/v1alpha1/{classfamily_types_test.go => componentresourceconstraint_types_test.go} (79%) create mode 100644 config/crd/bases/apps.kubeblocks.io_componentclassdefinitions.yaml rename deploy/helm/crds/apps.kubeblocks.io_classfamilies.yaml => config/crd/bases/apps.kubeblocks.io_componentresourceconstraints.yaml (92%) rename config/crd/patches/{cainjection_in_apps_classfamilies.yaml => cainjection_in_apps_componentclassdefinitions.yaml} (82%) create mode 100644 config/crd/patches/cainjection_in_apps_componentresourceconstraints.yaml rename config/crd/patches/{webhook_in_apps_classfamilies.yaml => webhook_in_apps_componentclassdefinitions.yaml} (87%) create mode 100644 config/crd/patches/webhook_in_apps_componentresourceconstraints.yaml rename config/rbac/{apps_classfamily_editor_role.yaml => apps_componentclassdefinition_editor_role.yaml} (66%) rename config/rbac/{apps_classfamily_viewer_role.yaml => apps_componentclassdefinition_viewer_role.yaml} (64%) create mode 100644 config/rbac/apps_componentresourceconstraint_editor_role.yaml create mode 100644 config/rbac/apps_componentresourceconstraint_viewer_role.yaml create mode 100644 config/samples/apps_v1alpha1_componentclassdefinition.yaml rename config/samples/{apps_v1alpha1_classfamily.yaml => apps_v1alpha1_componentresourceconstraint.yaml} (53%) create mode 100644 controllers/apps/class_controller.go create mode 100644 controllers/apps/class_controller_test.go create mode 100644 deploy/helm/crds/apps.kubeblocks.io_componentclassdefinitions.yaml rename config/crd/bases/apps.kubeblocks.io_classfamilies.yaml => deploy/helm/crds/apps.kubeblocks.io_componentresourceconstraints.yaml (92%) rename deploy/helm/templates/class/{classfamily.yaml => componentclassconstraint.yaml} (64%) rename internal/cli/testing/testdata/{classfamily-general.yaml => resource-constraint-general.yaml} (68%) rename internal/cli/testing/testdata/{classfamily-memory-optimized.yaml => resource-constraint-memory-optimized.yaml} (59%) create mode 100644 internal/testutil/apps/componentclassdefinition_factory.go diff --git a/PROJECT b/PROJECT index 847d7728e..22a8bca83 100644 --- a/PROJECT +++ b/PROJECT @@ -128,7 +128,15 @@ resources: namespaced: true domain: kubeblocks.io group: apps - kind: ClassFamily + kind: ComponentResourceConstraint + path: github.com/apecloud/kubeblocks/apis/apps/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + domain: kubeblocks.io + group: apps + kind: ComponentClassDefinition path: github.com/apecloud/kubeblocks/apis/apps/v1alpha1 version: v1alpha1 version: "3" diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index 40d11c66a..427b8b8aa 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -112,6 +112,10 @@ type ClusterComponentSpec struct { // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` ComponentDefRef string `json:"componentDefRef"` + // classDefRef reference class defined in ComponentClassDefinition. + // +optional + ClassDefRef *ClassDefRef `json:"classDefRef,omitempty"` + // monitor which is a switch to enable monitoring, default is false // KubeBlocks provides an extension mechanism to support component level monitoring, // which will scrape metrics auto or manually from servers in component and export @@ -428,6 +432,16 @@ type ClusterComponentService struct { Annotations map[string]string `json:"annotations,omitempty"` } +type ClassDefRef struct { + // name refers to the name of the ComponentClassDefinition. + // +optional + Name string `json:"name,omitempty"` + + // class refers to the name of the class that is defined in the ComponentClassDefinition. + // +kubebuilder:validation:Required + Class string `json:"class"` +} + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:categories={kubeblocks,all} diff --git a/apis/apps/v1alpha1/cluster_webhook.go b/apis/apps/v1alpha1/cluster_webhook.go index d7ac7cf69..3461186f4 100644 --- a/apis/apps/v1alpha1/cluster_webhook.go +++ b/apis/apps/v1alpha1/cluster_webhook.go @@ -213,6 +213,10 @@ func (r *Cluster) validateComponents(allErrs *field.ErrorList, clusterDef *Clust componentMap[v.Name] = v } + compClasses, err := getClasses(clusterDef.Name) + if err != nil { + return + } for i, v := range r.Spec.ComponentSpecs { if _, ok := componentDefMap[v.ComponentDefRef]; !ok { invalidComponentDefs = append(invalidComponentDefs, v.ComponentDefRef) @@ -220,6 +224,19 @@ func (r *Cluster) validateComponents(allErrs *field.ErrorList, clusterDef *Clust componentNameMap[v.Name] = struct{}{} r.validateComponentResources(allErrs, v.Resources, i) + + if classes, ok := compClasses[v.ComponentDefRef]; ok { + if v.ClassDefRef.Class != "" { + if _, ok = classes[v.ClassDefRef.Class]; !ok { + *allErrs = append(*allErrs, field.Invalid(field.NewPath(fmt.Sprintf("spec.components[%d].classDefRef", i)), v.ClassDefRef.Class, "can not find the specified class")) + return + } + } + if err = validateMatchingClass(classes, v.Resources); err != nil { + *allErrs = append(*allErrs, field.Invalid(field.NewPath(fmt.Sprintf("spec.components[%d].resources", i)), v.Resources.String(), err.Error())) + return + } + } } r.validatePrimaryIndex(allErrs) @@ -243,6 +260,7 @@ func (r *Cluster) validateComponentResources(allErrs *field.ErrorList, resources if invalidValue, err := compareRequestsAndLimits(resources); err != nil { *allErrs = append(*allErrs, field.Invalid(field.NewPath(fmt.Sprintf("spec.components[%d].resources.requests", index)), invalidValue, err.Error())) } + } func (r *Cluster) validateComponentTLSSettings(allErrs *field.ErrorList) { diff --git a/apis/apps/v1alpha1/componentclassdefinition_types.go b/apis/apps/v1alpha1/componentclassdefinition_types.go new file mode 100644 index 000000000..4347d6e72 --- /dev/null +++ b/apis/apps/v1alpha1/componentclassdefinition_types.go @@ -0,0 +1,177 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ComponentClassDefinitionSpec defines the desired state of ComponentClassDefinition +type ComponentClassDefinitionSpec struct { + // group defines a list of class series that conform to the same constraint. + // +optional + Groups []ComponentClassGroup `json:"groups,omitempty"` +} + +type ComponentClassGroup struct { + // resourceConstraintRef reference to the resource constraint object name, indicates that the series + // defined below all conform to the constraint. + // +kubebuilder:validation:Required + ResourceConstraintRef string `json:"resourceConstraintRef"` + + // template is a class definition template that uses the Go template syntax and allows for variable declaration. + // When defining a class in Series, specifying the variable's value is sufficient, as the complete class + // definition will be generated through rendering the template. + // + // For example: + // template: | + // cpu: "{{ or .cpu 1 }}" + // memory: "{{ or .memory 4 }}Gi" + // storage: + // - name: data + // size: "{{ or .dataStorageSize 10 }}Gi" + // - name: log + // size: "{{ or .logStorageSize 1 }}Gi" + // + // +optional + Template string `json:"template,omitempty"` + + // vars defines the variables declared in the template and will be used to generating the complete class definition by + // render the template. + // +listType=set + // +optional + Vars []string `json:"vars,omitempty"` + + // series is a series of class definitions. + // +optional + Series []ComponentClassSeries `json:"series,omitempty"` +} + +type ComponentClassSeries struct { + // namingTemplate is a template that uses the Go template syntax and allows for referencing variables defined + // in ComponentClassGroup.Template. This enables dynamic generation of class names. + // For example: + // name: "general-{{ .cpu }}c{{ .memory }}g" + // +optional + NamingTemplate string `json:"namingTemplate,omitempty"` + + // classes are definitions of classes that come in two forms. In the first form, only ComponentClass.Args + // need to be defined, and the complete class definition is generated by rendering the ComponentClassGroup.Template + // and Name. In the second form, the Name, CPU, Memory, and Volumes must be defined. + // +optional + Classes []ComponentClass `json:"classes,omitempty"` +} + +type ComponentClass struct { + // name is the class name + // +optional + Name string `json:"name,omitempty"` + + // args are variable's value + // +optional + Args []string `json:"args,omitempty"` + + // the CPU of the class + // +optional + CPU resource.Quantity `json:"cpu,omitempty"` + + // the memory of the class + // +optional + Memory resource.Quantity `json:"memory,omitempty"` + + // the volumes of the class + // +listType=map + // +listMapKey=name + // +optional + Volumes []Volume `json:"volumes,omitempty"` + + // the variants of the class in different clouds. + // +listType=map + // +listMapKey=provider + // +optional + Variants []ProviderComponentClassDef `json:"variants,omitempty"` +} + +type ProviderComponentClassDef struct { + // cloud provider name + // +kubebuilder:validation:Required + Provider string `json:"provider"` + + // cloud provider specific variables + // +optional + Args []string `json:"args,omitempty"` +} + +type Volume struct { + // The volume name, etc. data, log. + // +kubebuilder:validation:Required + Name string `json:"name"` + + // The size of the volume. + // +kubebuilder:validation:Required + Size resource.Quantity `json:"size"` + + // The StorageClass name of the volume. + // +optional + StorageClassName *string `json:"storageCLassName,omitempty"` +} + +// ComponentClassDefinitionStatus defines the observed state of ComponentClassDefinition +type ComponentClassDefinitionStatus struct { + // observedGeneration is the most recent generation observed for this + // ComponentClassDefinition. It corresponds to the ComponentClassDefinition's generation, which is + // updated on mutation by the API Server. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // classes is the list of classes that have been observed for this ComponentClassDefinition + Classes []ComponentClassInstance `json:"classes,omitempty"` +} + +type ComponentClassInstance struct { + ComponentClass `json:",inline"` + + // resourceConstraintRef reference to the resource constraint object name. + ResourceConstraintRef string `json:"resourceConstraintRef,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:categories={kubeblocks},scope=Cluster,shortName=ccd + +// ComponentClassDefinition is the Schema for the componentclassdefinitions API +type ComponentClassDefinition struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ComponentClassDefinitionSpec `json:"spec,omitempty"` + Status ComponentClassDefinitionStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// ComponentClassDefinitionList contains a list of ComponentClassDefinition +type ComponentClassDefinitionList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ComponentClassDefinition `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ComponentClassDefinition{}, &ComponentClassDefinitionList{}) +} diff --git a/apis/apps/v1alpha1/classfamily_types.go b/apis/apps/v1alpha1/componentresourceconstraint_types.go similarity index 75% rename from apis/apps/v1alpha1/classfamily_types.go rename to apis/apps/v1alpha1/componentresourceconstraint_types.go index e833e3019..d36fda7cc 100644 --- a/apis/apps/v1alpha1/classfamily_types.go +++ b/apis/apps/v1alpha1/componentresourceconstraint_types.go @@ -24,20 +24,20 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// ClassFamilySpec defines the desired state of ClassFamily -type ClassFamilySpec struct { - // Class family models, generally, a model is a specific constraint for CPU, memory and their relation. - Models []ClassFamilyModel `json:"models,omitempty"` +// ComponentResourceConstraintSpec defines the desired state of ComponentResourceConstraint +type ComponentResourceConstraintSpec struct { + // Component resource constraints + Constraints []ResourceConstraint `json:"constraints,omitempty"` } -type ClassFamilyModel struct { +type ResourceConstraint struct { // The constraint for vcpu cores. // +kubebuilder:validation:Required - CPU CPUConstraint `json:"cpu,omitempty"` + CPU CPUConstraint `json:"cpu"` // The constraint for memory size. // +kubebuilder:validation:Required - Memory MemoryConstraint `json:"memory,omitempty"` + Memory MemoryConstraint `json:"memory"` } type CPUConstraint struct { @@ -92,34 +92,31 @@ type MemoryConstraint struct { } // +kubebuilder:object:root=true -// +kubebuilder:resource:categories={kubeblocks,all},scope=Cluster,shortName=cf +// +kubebuilder:resource:categories={kubeblocks,all},scope=Cluster,shortName=crc -// ClassFamily is the Schema for the classfamilies API -type ClassFamily struct { +// ComponentResourceConstraint is the Schema for the componentresourceconstraints API +type ComponentResourceConstraint struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` - Spec ClassFamilySpec `json:"spec,omitempty"` + Spec ComponentResourceConstraintSpec `json:"spec,omitempty"` } // +kubebuilder:object:root=true -// ClassFamilyList contains a list of ClassFamily -type ClassFamilyList struct { +// ComponentResourceConstraintList contains a list of ComponentResourceConstraint +type ComponentResourceConstraintList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` - Items []ClassFamily `json:"items"` + Items []ComponentResourceConstraint `json:"items"` } func init() { - SchemeBuilder.Register(&ClassFamily{}, &ClassFamilyList{}) + SchemeBuilder.Register(&ComponentResourceConstraint{}, &ComponentResourceConstraintList{}) } -// ValidateCPU validate if the CPU matches the class family model constraint -func (m *ClassFamilyModel) ValidateCPU(cpu resource.Quantity) bool { - if m == nil { - return false - } +// ValidateCPU validate if the CPU matches the resource constraints +func (m ResourceConstraint) ValidateCPU(cpu resource.Quantity) bool { if m.CPU.Min != nil && m.CPU.Min.Cmp(cpu) > 0 { return false } @@ -132,12 +129,8 @@ func (m *ClassFamilyModel) ValidateCPU(cpu resource.Quantity) bool { return true } -// ValidateMemory validate if the memory matches the class family model constraint -func (m *ClassFamilyModel) ValidateMemory(cpu *resource.Quantity, memory *resource.Quantity) bool { - if m == nil { - return false - } - +// ValidateMemory validate if the memory matches the resource constraints +func (m ResourceConstraint) ValidateMemory(cpu *resource.Quantity, memory *resource.Quantity) bool { if memory == nil { return true } @@ -158,17 +151,13 @@ func (m *ClassFamilyModel) ValidateMemory(cpu *resource.Quantity, memory *resour return true } -// ValidateResourceRequirements validate if the resource matches the class family model constraints -func (m *ClassFamilyModel) ValidateResourceRequirements(r *corev1.ResourceRequirements) bool { +// ValidateResourceRequirements validate if the resource matches the resource constraints +func (m ResourceConstraint) ValidateResourceRequirements(r *corev1.ResourceRequirements) bool { var ( cpu = r.Requests.Cpu() memory = r.Requests.Memory() ) - if m == nil { - return false - } - if cpu.IsZero() && memory.IsZero() { return true } @@ -184,16 +173,16 @@ func (m *ClassFamilyModel) ValidateResourceRequirements(r *corev1.ResourceRequir return true } -// FindMatchingModels find all class family models that resource matches -func (c *ClassFamily) FindMatchingModels(r *corev1.ResourceRequirements) []ClassFamilyModel { +// FindMatchingConstraints find all constraints that resource matches +func (c *ComponentResourceConstraint) FindMatchingConstraints(r *corev1.ResourceRequirements) []ResourceConstraint { if c == nil { return nil } - var models []ClassFamilyModel - for _, model := range c.Spec.Models { - if model.ValidateResourceRequirements(r) { - models = append(models, model) + var constraints []ResourceConstraint + for _, constraint := range c.Spec.Constraints { + if constraint.ValidateResourceRequirements(r) { + constraints = append(constraints, constraint) } } - return models + return constraints } diff --git a/apis/apps/v1alpha1/classfamily_types_test.go b/apis/apps/v1alpha1/componentresourceconstraint_types_test.go similarity index 79% rename from apis/apps/v1alpha1/classfamily_types_test.go rename to apis/apps/v1alpha1/componentresourceconstraint_types_test.go index be491451d..b767a3243 100644 --- a/apis/apps/v1alpha1/classfamily_types_test.go +++ b/apis/apps/v1alpha1/componentresourceconstraint_types_test.go @@ -25,15 +25,14 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" ) -const classFamilyBytes = ` +const resourceConstraints = ` # API scope: cluster -# ClusterClassFamily apiVersion: "apps.kubeblocks.io/v1alpha1" -kind: "ClassFamily" +kind: "ComponentResourceConstraint" metadata: - name: kb-class-family-general + name: kb-resource-constraint-general spec: - models: + constraints: - cpu: min: 0.5 max: 128 @@ -53,11 +52,11 @@ spec: maxPerCPU: 8Gi ` -func TestClassFamily_ValidateResourceRequirements(t *testing.T) { - var cf ClassFamily - err := yaml.Unmarshal([]byte(classFamilyBytes), &cf) +func TestResourceConstraints_ValidateResourceRequirements(t *testing.T) { + var cf ComponentResourceConstraint + err := yaml.Unmarshal([]byte(resourceConstraints), &cf) if err != nil { - panic("Failed to unmarshal class family: %v" + err.Error()) + panic("Failed to unmarshal resource constraints: %v" + err.Error()) } cases := []struct { cpu string @@ -78,6 +77,6 @@ func TestClassFamily_ValidateResourceRequirements(t *testing.T) { corev1.ResourceMemory: resource.MustParse(item.memory), }, } - assert.Equal(t, item.expect, len(cf.FindMatchingModels(requirements)) > 0) + assert.Equal(t, item.expect, len(cf.FindMatchingConstraints(requirements)) > 0) } } diff --git a/apis/apps/v1alpha1/opsrequest_types.go b/apis/apps/v1alpha1/opsrequest_types.go index 4e540b2ac..ad0dd73dd 100644 --- a/apis/apps/v1alpha1/opsrequest_types.go +++ b/apis/apps/v1alpha1/opsrequest_types.go @@ -106,9 +106,12 @@ type VerticalScaling struct { ComponentOps `json:",inline"` // resources specifies the computing resource size of verticalScaling. - // +kubebuilder:validation:Required // +kubebuilder:pruning:PreserveUnknownFields corev1.ResourceRequirements `json:",inline"` + + // class specifies the class name of the component + // +optional + Class string `json:"class,omitempty"` } // VolumeExpansion defines the variables of volume expansion operation. @@ -300,6 +303,10 @@ type LastComponentConfiguration struct { // +optional corev1.ResourceRequirements `json:",inline,omitempty"` + // the last class name of the component. + // +optional + Class string `json:"class,omitempty"` + // volumeClaimTemplates records the last volumeClaimTemplates of the component. // +optional VolumeClaimTemplates []OpsRequestVolumeClaimTemplate `json:"volumeClaimTemplates,omitempty"` diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index db701388b..39cee415e 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -224,10 +225,25 @@ func (r *OpsRequest) validateVerticalScaling(cluster *Cluster) error { return notEmptyError("spec.verticalScaling") } + compClasses, err := getClasses(cluster.Spec.ClusterDefRef) + if err != nil { + return nil + } + + getComponent := func(name string) *ClusterComponentSpec { + for _, comp := range cluster.Spec.ComponentSpecs { + if comp.Name == name { + return &comp + } + } + return nil + } + // validate resources is legal and get component name slice componentNames := make([]string, len(verticalScalingList)) for i, v := range verticalScalingList { componentNames[i] = v.ComponentName + if invalidValue, err := validateVerticalResourceList(v.Requests); err != nil { return invalidValueError(invalidValue, err.Error()) } @@ -237,6 +253,20 @@ func (r *OpsRequest) validateVerticalScaling(cluster *Cluster) error { if invalidValue, err := compareRequestsAndLimits(v.ResourceRequirements); err != nil { return invalidValueError(invalidValue, err.Error()) } + comp := getComponent(v.ComponentName) + if comp == nil { + continue + } + if classes, ok := compClasses[comp.ComponentDefRef]; ok { + if comp.ClassDefRef.Class != "" { + if _, ok = classes[comp.ClassDefRef.Class]; !ok { + return field.Invalid(field.NewPath(fmt.Sprintf("spec.components[%d].classDefRef", i)), comp.ClassDefRef.Class, err.Error()) + } + } + if err = validateMatchingClass(classes, v.ResourceRequirements); err != nil { + return fmt.Errorf("can not find matching class for component %s", v.ComponentName) + } + } } return r.checkComponentExistence(cluster, componentNames) } @@ -452,9 +482,55 @@ func validateVerticalResourceList(resourceList map[corev1.ResourceName]resource. return string(k), fmt.Errorf("resource key is not cpu or memory or hugepages- ") } } + return "", nil } +func getClasses(clusterDef string) (map[string]map[string]*ComponentClassInstance, error) { + ml := []client.ListOption{ + client.MatchingLabels{"clusterdefinition.kubeblocks.io/name": clusterDef}, + } + var classDefinitionList ComponentClassDefinitionList + if err := webhookMgr.client.List(context.Background(), &classDefinitionList, ml...); err != nil { + return nil, err + } + var ( + componentClasses = make(map[string]map[string]*ComponentClassInstance) + ) + for _, classDefinition := range classDefinitionList.Items { + componentType := classDefinition.GetLabels()["apps.kubeblocks.io/component-def-ref"] + if componentType == "" { + return nil, fmt.Errorf("failed to find component type") + } + classes := make(map[string]*ComponentClassInstance) + for idx := range classDefinition.Status.Classes { + cls := classDefinition.Status.Classes[idx] + classes[cls.Name] = &cls + } + if _, ok := componentClasses[componentType]; !ok { + componentClasses[componentType] = classes + } else { + for k, v := range classes { + if _, exists := componentClasses[componentType][k]; exists { + return nil, fmt.Errorf("duplicate component class %s", k) + } + componentClasses[componentType][k] = v + } + } + } + return componentClasses, nil +} + +func validateMatchingClass(classes map[string]*ComponentClassInstance, resource corev1.ResourceRequirements) error { + if cls := chooseComponentClasses(classes, resource.Requests); cls == nil { + return fmt.Errorf("can not find matching class with specified requests") + } + if cls := chooseComponentClasses(classes, resource.Limits); cls == nil { + return fmt.Errorf("can not find matching class with specified limits") + } + return nil +} + func notEmptyError(target string) error { return fmt.Errorf(`"%s" can not be empty`, target) } diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index 7df51c6ec..1eac9213c 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -18,6 +18,10 @@ limitations under the License. package v1alpha1 import ( + "sort" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -510,3 +514,47 @@ func RegisterWebhookManager(mgr manager.Manager) { } type ComponentNameSet map[string]struct{} + +func chooseComponentClasses(classes map[string]*ComponentClassInstance, filters map[corev1.ResourceName]resource.Quantity) *ComponentClassInstance { + var candidates []*ComponentClassInstance + for _, cls := range classes { + cpu, ok := filters[corev1.ResourceCPU] + if ok && !cpu.Equal(cls.CPU) { + continue + } + memory, ok := filters[corev1.ResourceMemory] + if ok && !memory.Equal(cls.Memory) { + continue + } + candidates = append(candidates, cls) + } + if len(candidates) == 0 { + return nil + } + sort.Sort(byClassCPUAndMemory(candidates)) + return candidates[0] +} + +var _ sort.Interface = byClassCPUAndMemory{} + +type byClassCPUAndMemory []*ComponentClassInstance + +func (b byClassCPUAndMemory) Len() int { + return len(b) +} + +func (b byClassCPUAndMemory) Less(i, j int) bool { + if out := b[i].CPU.Cmp(b[j].CPU); out != 0 { + return out < 0 + } + + if out := b[i].Memory.Cmp(b[j].Memory); out != 0 { + return out < 0 + } + + return false +} + +func (b byClassCPUAndMemory) Swap(i, j int) { + b[i], b[j] = b[j], b[i] +} diff --git a/cmd/manager/main.go b/cmd/manager/main.go index e13d6ff2a..a8b9755ed 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -391,6 +391,15 @@ func main() { os.Exit(1) } + if err = (&appscontrollers.ComponentClassReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("class-controller"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Class") + os.Exit(1) + } + if viper.GetBool("enable_webhooks") { appsv1alpha1.RegisterWebhookManager(mgr) diff --git a/config/crd/bases/apps.kubeblocks.io_clusters.yaml b/config/crd/bases/apps.kubeblocks.io_clusters.yaml index 6f8b269fa..b41dc82b0 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusters.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusters.yaml @@ -156,6 +156,19 @@ spec: type: array x-kubernetes-list-type: set type: object + classDefRef: + description: classDefRef reference class defined in ComponentClassDefinition. + properties: + class: + description: class refers to the name of the class that + is defined in the ComponentClassDefinition. + type: string + name: + description: name refers to the name of the ComponentClassDefinition. + type: string + required: + - class + type: object componentDefRef: description: componentDefRef reference componentDef defined in ClusterDefinition spec. diff --git a/config/crd/bases/apps.kubeblocks.io_componentclassdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentclassdefinitions.yaml new file mode 100644 index 000000000..37deea269 --- /dev/null +++ b/config/crd/bases/apps.kubeblocks.io_componentclassdefinitions.yaml @@ -0,0 +1,261 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: componentclassdefinitions.apps.kubeblocks.io +spec: + group: apps.kubeblocks.io + names: + categories: + - kubeblocks + kind: ComponentClassDefinition + listKind: ComponentClassDefinitionList + plural: componentclassdefinitions + shortNames: + - ccd + singular: componentclassdefinition + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ComponentClassDefinition is the Schema for the componentclassdefinitions + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ComponentClassDefinitionSpec defines the desired state of + ComponentClassDefinition + properties: + groups: + description: group defines a list of class series that conform to + the same constraint. + items: + properties: + resourceConstraintRef: + description: resourceConstraintRef reference to the resource + constraint object name, indicates that the series defined + below all conform to the constraint. + type: string + series: + description: series is a series of class definitions. + items: + properties: + classes: + description: classes are definitions of classes that come + in two forms. In the first form, only ComponentClass.Args + need to be defined, and the complete class definition + is generated by rendering the ComponentClassGroup.Template + and Name. In the second form, the Name, CPU, Memory, + and Volumes must be defined. + items: + properties: + args: + description: args are variable's value + items: + type: string + type: array + cpu: + anyOf: + - type: integer + - type: string + description: the CPU of the class + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: the memory of the class + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name is the class name + type: string + variants: + description: the variants of the class in different + clouds. + items: + properties: + args: + description: cloud provider specific variables + items: + type: string + type: array + provider: + description: cloud provider name + type: string + required: + - provider + type: object + type: array + x-kubernetes-list-map-keys: + - provider + x-kubernetes-list-type: map + volumes: + description: the volumes of the class + items: + properties: + name: + description: The volume name, etc. data, log. + type: string + size: + anyOf: + - type: integer + - type: string + description: The size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageCLassName: + description: The StorageClass name of the + volume. + type: string + required: + - name + - size + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: array + namingTemplate: + description: 'namingTemplate is a template that uses the + Go template syntax and allows for referencing variables + defined in ComponentClassGroup.Template. This enables + dynamic generation of class names. For example: name: + "general-{{ .cpu }}c{{ .memory }}g"' + type: string + type: object + type: array + template: + description: "template is a class definition template that uses + the Go template syntax and allows for variable declaration. + When defining a class in Series, specifying the variable's + value is sufficient, as the complete class definition will + be generated through rendering the template. \n For example: + template: | cpu: \"{{ or .cpu 1 }}\" memory: \"{{ or .memory + 4 }}Gi\" storage: - name: data size: \"{{ or .dataStorageSize + 10 }}Gi\" - name: log size: \"{{ or .logStorageSize 1 }}Gi\"" + type: string + vars: + description: vars defines the variables declared in the template + and will be used to generating the complete class definition + by render the template. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - resourceConstraintRef + type: object + type: array + type: object + status: + description: ComponentClassDefinitionStatus defines the observed state + of ComponentClassDefinition + properties: + classes: + description: classes is the list of classes that have been observed + for this ComponentClassDefinition + items: + properties: + args: + description: args are variable's value + items: + type: string + type: array + cpu: + anyOf: + - type: integer + - type: string + description: the CPU of the class + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: the memory of the class + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name is the class name + type: string + resourceConstraintRef: + description: resourceConstraintRef reference to the resource + constraint object name. + type: string + variants: + description: the variants of the class in different clouds. + items: + properties: + args: + description: cloud provider specific variables + items: + type: string + type: array + provider: + description: cloud provider name + type: string + required: + - provider + type: object + type: array + x-kubernetes-list-map-keys: + - provider + x-kubernetes-list-type: map + volumes: + description: the volumes of the class + items: + properties: + name: + description: The volume name, etc. data, log. + type: string + size: + anyOf: + - type: integer + - type: string + description: The size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageCLassName: + description: The StorageClass name of the volume. + type: string + required: + - name + - size + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: array + observedGeneration: + description: observedGeneration is the most recent generation observed + for this ComponentClassDefinition. It corresponds to the ComponentClassDefinition's + generation, which is updated on mutation by the API Server. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deploy/helm/crds/apps.kubeblocks.io_classfamilies.yaml b/config/crd/bases/apps.kubeblocks.io_componentresourceconstraints.yaml similarity index 92% rename from deploy/helm/crds/apps.kubeblocks.io_classfamilies.yaml rename to config/crd/bases/apps.kubeblocks.io_componentresourceconstraints.yaml index 1c4f66d32..5f00d7070 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_classfamilies.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentresourceconstraints.yaml @@ -5,25 +5,26 @@ metadata: annotations: controller-gen.kubebuilder.io/version: v0.9.0 creationTimestamp: null - name: classfamilies.apps.kubeblocks.io + name: componentresourceconstraints.apps.kubeblocks.io spec: group: apps.kubeblocks.io names: categories: - kubeblocks - all - kind: ClassFamily - listKind: ClassFamilyList - plural: classfamilies + kind: ComponentResourceConstraint + listKind: ComponentResourceConstraintList + plural: componentresourceconstraints shortNames: - - cf - singular: classfamily + - crc + singular: componentresourceconstraint scope: Cluster versions: - name: v1alpha1 schema: openAPIV3Schema: - description: ClassFamily is the Schema for the classfamilies API + description: ComponentResourceConstraint is the Schema for the componentresourceconstraints + API properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -38,11 +39,11 @@ spec: metadata: type: object spec: - description: ClassFamilySpec defines the desired state of ClassFamily + description: ComponentResourceConstraintSpec defines the desired state + of ComponentResourceConstraint properties: - models: - description: Class family models, generally, a model is a specific - constraint for CPU, memory and their relation. + constraints: + description: Component resource constraints items: properties: cpu: @@ -133,6 +134,9 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + required: + - cpu + - memory type: object type: array type: object diff --git a/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml b/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml index 0ab01c7f7..b0679d64b 100644 --- a/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml +++ b/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml @@ -277,6 +277,9 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + class: + description: class specifies the class name of the component + type: string componentName: description: componentName cluster component name. type: string @@ -539,6 +542,9 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + class: + description: the last class name of the component. + type: string limits: additionalProperties: anyOf: diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index e95ab06ea..6bd5ab6db 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -13,7 +13,8 @@ resources: - bases/dataprotection.kubeblocks.io_backups.yaml - bases/dataprotection.kubeblocks.io_restorejobs.yaml - bases/extensions.kubeblocks.io_addons.yaml -- bases/apps.kubeblocks.io_classfamilies.yaml +- bases/apps.kubeblocks.io_componentresourceconstraints.yaml +- bases/apps.kubeblocks.io_componentclassdefinitions.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -33,7 +34,8 @@ patchesStrategicMerge: #- patches/webhook_in_hostpreflights.yaml #- patches/webhook_in_preflights.yaml #- patches/webhook_in_addons.yaml -#- patches/webhook_in_classfamilies.yaml +#- patches/webhook_in_componentresourceconstraints.yaml +#- patches/webhook_in_componentclassdefinitions.yaml #+kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. @@ -52,7 +54,8 @@ patchesStrategicMerge: #- patches/cainjection_in_hostpreflights.yaml #- patches/cainjection_in_preflights.yaml #- patches/cainjection_in_addonspecs.yaml -#- patches/cainjection_in_classfamilies.yaml +#- patches/cainjection_in_componentresourceconstraints.yaml +#- patches/cainjection_in_componentclassdefinitions.yaml #+kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_apps_classfamilies.yaml b/config/crd/patches/cainjection_in_apps_componentclassdefinitions.yaml similarity index 82% rename from config/crd/patches/cainjection_in_apps_classfamilies.yaml rename to config/crd/patches/cainjection_in_apps_componentclassdefinitions.yaml index bc1f3ff5c..128d2cbe3 100644 --- a/config/crd/patches/cainjection_in_apps_classfamilies.yaml +++ b/config/crd/patches/cainjection_in_apps_componentclassdefinitions.yaml @@ -4,4 +4,4 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: classfamilies.apps.kubeblocks.io + name: componentclassdefinitions.apps.kubeblocks.io diff --git a/config/crd/patches/cainjection_in_apps_componentresourceconstraints.yaml b/config/crd/patches/cainjection_in_apps_componentresourceconstraints.yaml new file mode 100644 index 000000000..611d9c6cb --- /dev/null +++ b/config/crd/patches/cainjection_in_apps_componentresourceconstraints.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: componentresourceconstraints.apps.kubeblocks.io diff --git a/config/crd/patches/webhook_in_apps_classfamilies.yaml b/config/crd/patches/webhook_in_apps_componentclassdefinitions.yaml similarity index 87% rename from config/crd/patches/webhook_in_apps_classfamilies.yaml rename to config/crd/patches/webhook_in_apps_componentclassdefinitions.yaml index 1667132fe..46abc75c9 100644 --- a/config/crd/patches/webhook_in_apps_classfamilies.yaml +++ b/config/crd/patches/webhook_in_apps_componentclassdefinitions.yaml @@ -2,7 +2,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: classfamilies.apps.kubeblocks.io + name: componentclassdefinitions.apps.kubeblocks.io spec: conversion: strategy: Webhook diff --git a/config/crd/patches/webhook_in_apps_componentresourceconstraints.yaml b/config/crd/patches/webhook_in_apps_componentresourceconstraints.yaml new file mode 100644 index 000000000..ea37a59b6 --- /dev/null +++ b/config/crd/patches/webhook_in_apps_componentresourceconstraints.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: componentresourceconstraints.apps.kubeblocks.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/rbac/apps_classfamily_editor_role.yaml b/config/rbac/apps_componentclassdefinition_editor_role.yaml similarity index 66% rename from config/rbac/apps_classfamily_editor_role.yaml rename to config/rbac/apps_componentclassdefinition_editor_role.yaml index 5a06154b4..0faf3fbe7 100644 --- a/config/rbac/apps_classfamily_editor_role.yaml +++ b/config/rbac/apps_componentclassdefinition_editor_role.yaml @@ -1,20 +1,20 @@ -# permissions for end users to edit classfamilies. +# permissions for end users to edit componentclassdefinitions. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: classfamily-editor-role + app.kubernetes.io/instance: componentclassdefinition-editor-role app.kubernetes.io/component: rbac app.kubernetes.io/created-by: kubeblocks app.kubernetes.io/part-of: kubeblocks app.kubernetes.io/managed-by: kustomize - name: classfamily-editor-role + name: componentclassdefinition-editor-role rules: - apiGroups: - apps.kubeblocks.io resources: - - classfamilies + - componentclassdefinitions verbs: - create - delete @@ -26,6 +26,6 @@ rules: - apiGroups: - apps.kubeblocks.io resources: - - classfamilies/status + - componentclassdefinitions/status verbs: - get diff --git a/config/rbac/apps_classfamily_viewer_role.yaml b/config/rbac/apps_componentclassdefinition_viewer_role.yaml similarity index 64% rename from config/rbac/apps_classfamily_viewer_role.yaml rename to config/rbac/apps_componentclassdefinition_viewer_role.yaml index d82810999..29e1d8f8d 100644 --- a/config/rbac/apps_classfamily_viewer_role.yaml +++ b/config/rbac/apps_componentclassdefinition_viewer_role.yaml @@ -1,20 +1,20 @@ -# permissions for end users to view classfamilies. +# permissions for end users to view componentclassdefinitions. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: app.kubernetes.io/name: clusterrole - app.kubernetes.io/instance: classfamily-viewer-role + app.kubernetes.io/instance: componentclassdefinition-viewer-role app.kubernetes.io/component: rbac app.kubernetes.io/created-by: kubeblocks app.kubernetes.io/part-of: kubeblocks app.kubernetes.io/managed-by: kustomize - name: classfamily-viewer-role + name: componentclassdefinition-viewer-role rules: - apiGroups: - apps.kubeblocks.io resources: - - classfamilies + - componentclassdefinitions verbs: - get - list @@ -22,6 +22,6 @@ rules: - apiGroups: - apps.kubeblocks.io resources: - - classfamilies/status + - componentclassdefinitions/status verbs: - get diff --git a/config/rbac/apps_componentresourceconstraint_editor_role.yaml b/config/rbac/apps_componentresourceconstraint_editor_role.yaml new file mode 100644 index 000000000..ff4e58c98 --- /dev/null +++ b/config/rbac/apps_componentresourceconstraint_editor_role.yaml @@ -0,0 +1,31 @@ +# permissions for end users to edit componentresourceconstraints +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: componentresourceconstraint-editor-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: kubeblocks + app.kubernetes.io/part-of: kubeblocks + app.kubernetes.io/managed-by: kustomize + name: componentresourceconstraint-editor-role +rules: +- apiGroups: + - apps.kubeblocks.io + resources: + - componentresourceconstraints + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.kubeblocks.io + resources: + - componentresourceconstraints/status + verbs: + - get diff --git a/config/rbac/apps_componentresourceconstraint_viewer_role.yaml b/config/rbac/apps_componentresourceconstraint_viewer_role.yaml new file mode 100644 index 000000000..feae2809c --- /dev/null +++ b/config/rbac/apps_componentresourceconstraint_viewer_role.yaml @@ -0,0 +1,27 @@ +# permissions for end users to view componentresourceconstraints +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: clusterrole + app.kubernetes.io/instance: componentresourceconstraint-viewer-role + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: kubeblocks + app.kubernetes.io/part-of: kubeblocks + app.kubernetes.io/managed-by: kustomize + name: componentresourceconstraint-viewer-role +rules: +- apiGroups: + - apps.kubeblocks.io + resources: + - componentresourceconstraints + verbs: + - get + - list + - watch +- apiGroups: + - apps.kubeblocks.io + resources: + - componentresourceconstraints/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7bd0f58ca..3981b5213 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -110,14 +110,6 @@ rules: verbs: - get - list -- apiGroups: - - apps.kubeblocks.io - resources: - - classfamilies - verbs: - - get - - list - - watch - apiGroups: - apps.kubeblocks.io resources: @@ -196,6 +188,40 @@ rules: - get - patch - update +- apiGroups: + - apps.kubeblocks.io + resources: + - componentclassdefinitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.kubeblocks.io + resources: + - componentclassdefinitions/finalizers + verbs: + - update +- apiGroups: + - apps.kubeblocks.io + resources: + - componentclassdefinitions/status + verbs: + - get + - patch + - update +- apiGroups: + - apps.kubeblocks.io + resources: + - componentresourceconstraints + verbs: + - get + - list + - watch - apiGroups: - apps.kubeblocks.io resources: diff --git a/config/samples/apps_v1alpha1_componentclassdefinition.yaml b/config/samples/apps_v1alpha1_componentclassdefinition.yaml new file mode 100644 index 000000000..4e7209c0d --- /dev/null +++ b/config/samples/apps_v1alpha1_componentclassdefinition.yaml @@ -0,0 +1,12 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ComponentClassDefinition +metadata: + labels: + app.kubernetes.io/name: componentclassdefinition + app.kubernetes.io/instance: componentclassdefinition-sample + app.kubernetes.io/part-of: kubeblocks + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: kubeblocks + name: componentclassdefinition-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/apps_v1alpha1_classfamily.yaml b/config/samples/apps_v1alpha1_componentresourceconstraint.yaml similarity index 53% rename from config/samples/apps_v1alpha1_classfamily.yaml rename to config/samples/apps_v1alpha1_componentresourceconstraint.yaml index af312721a..5e9f61c08 100644 --- a/config/samples/apps_v1alpha1_classfamily.yaml +++ b/config/samples/apps_v1alpha1_componentresourceconstraint.yaml @@ -1,12 +1,12 @@ apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClassFamily +kind: ComponentResourceConstraint metadata: labels: - app.kubernetes.io/name: classfamily - app.kubernetes.io/instance: classfamily-sample + app.kubernetes.io/name: componentresourceconstraint + app.kubernetes.io/instance: componentresourceconstraint-sample app.kubernetes.io/part-of: kubeblocks app.kuberentes.io/managed-by: kustomize app.kubernetes.io/created-by: kubeblocks - name: classfamily-sample + name: componentresourceconstraint-sample spec: # TODO(user): Add fields here diff --git a/controllers/apps/class_controller.go b/controllers/apps/class_controller.go new file mode 100644 index 000000000..7a0623a06 --- /dev/null +++ b/controllers/apps/class_controller.go @@ -0,0 +1,91 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apps + +import ( + "context" + + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/class" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" +) + +// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=componentclassdefinitions,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=componentclassdefinitions/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=componentclassdefinitions/finalizers,verbs=update + +type ComponentClassReconciler struct { + client.Client + Scheme *k8sruntime.Scheme + Recorder record.EventRecorder +} + +func (r *ComponentClassReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + reqCtx := intctrlutil.RequestCtx{ + Ctx: ctx, + Req: req, + Log: log.FromContext(ctx).WithValues("classDefinition", req.NamespacedName), + Recorder: r.Recorder, + } + + classDefinition := &appsv1alpha1.ComponentClassDefinition{} + if err := r.Client.Get(reqCtx.Ctx, reqCtx.Req.NamespacedName, classDefinition); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } + + res, err := intctrlutil.HandleCRDeletion(reqCtx, r, classDefinition, dbClusterFinalizerName, func() (*ctrl.Result, error) { + // TODO validate if existing cluster reference classes being deleted + return nil, nil + }) + if res != nil { + return *res, err + } + + if classDefinition.Status.ObservedGeneration == classDefinition.Generation { + return intctrlutil.Reconciled() + } + + classInstances, err := class.ParseComponentClasses(*classDefinition) + if err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "parse component classes failed") + } + + patch := client.MergeFrom(classDefinition.DeepCopy()) + var classList []appsv1alpha1.ComponentClassInstance + for _, v := range classInstances { + classList = append(classList, *v) + } + classDefinition.Status.Classes = classList + classDefinition.Status.ObservedGeneration = classDefinition.Generation + if err = r.Client.Status().Patch(ctx, classDefinition, patch); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "patch component class status failed") + } + + return intctrlutil.Reconciled() +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ComponentClassReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr).For(&appsv1alpha1.ComponentClassDefinition{}).Complete(r) +} diff --git a/controllers/apps/class_controller_test.go b/controllers/apps/class_controller_test.go new file mode 100644 index 000000000..b436ad0c6 --- /dev/null +++ b/controllers/apps/class_controller_test.go @@ -0,0 +1,69 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apps + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + intctrlutil "github.com/apecloud/kubeblocks/internal/generics" + testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" +) + +var _ = Describe("", func() { + + var componentClassDefinition *v1alpha1.ComponentClassDefinition + + cleanEnv := func() { + // must wait until resources deleted and no longer exist before the testcases start, + // otherwise if later it needs to create some new resource objects with the same name, + // in race conditions, it will find the existence of old objects, resulting failure to + // create the new objects. + By("clean resources") + + // delete rest mocked objects + ml := client.HasLabels{testCtx.TestObjLabelKey} + testapps.ClearResources(&testCtx, intctrlutil.ComponentResourceConstraintSignature, ml) + testapps.ClearResources(&testCtx, intctrlutil.ComponentClassDefinitionSignature, ml) + } + + BeforeEach(cleanEnv) + + AfterEach(cleanEnv) + + It("Class should exist in status", func() { + var ( + clsName = "test" + ) + class := v1alpha1.ComponentClass{ + Name: clsName, + CPU: resource.MustParse("1"), + Memory: resource.MustParse("2Gi"), + } + componentClassDefinition = testapps.NewComponentClassDefinitionFactory("custom", "apecloud-mysql", "mysql"). + AddClass(class).Create(&testCtx).GetObject() + key := client.ObjectKeyFromObject(componentClassDefinition) + Eventually(testapps.CheckObj(&testCtx, key, func(g Gomega, pobj *v1alpha1.ComponentClassDefinition) { + g.Expect(pobj.Status.Classes).ShouldNot(BeEmpty()) + g.Expect(pobj.Status.Classes[0].Name).Should(Equal(clsName)) + })).Should(Succeed()) + }) +}) diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index 9ed4a84e2..0deca83e5 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -30,10 +30,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/source" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" @@ -96,8 +93,8 @@ import ( // +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backuppolicies,verbs=get;list;create;update;patch;delete;deletecollection // +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backups,verbs=get;list;delete;deletecollection -// classfamily get list -// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=classfamilies,verbs=get;list;watch +// componentresourceconstraint get list +// +kubebuilder:rbac:groups=apps.kubeblocks.io,resources=componentresourceconstraints,verbs=get;list;watch // ClusterReconciler reconciles a Cluster object type ClusterReconciler struct { @@ -168,10 +165,6 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { if viper.GetBool("VOLUMESNAPSHOT") { b.Owns(&snapshotv1.VolumeSnapshot{}, builder.OnlyMetadata, builder.Predicates{}) } - b.Watches(&source.Kind{Type: &appsv1alpha1.ClassFamily{}}, - &handler.EnqueueRequestForObject{}, - builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { return true })), - ) return b.Complete(r) } diff --git a/controllers/apps/operations/suite_test.go b/controllers/apps/operations/suite_test.go index 6edd7760a..30d6d6e14 100644 --- a/controllers/apps/operations/suite_test.go +++ b/controllers/apps/operations/suite_test.go @@ -118,6 +118,8 @@ var _ = BeforeSuite(func() { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + appsv1alpha1.RegisterWebhookManager(k8sManager) + testCtx = testutil.NewDefaultTestContext(ctx, k8sClient, testEnv) go func() { diff --git a/controllers/apps/operations/vertical_scaling.go b/controllers/apps/operations/vertical_scaling.go index 1913d98b5..4d23e5817 100644 --- a/controllers/apps/operations/vertical_scaling.go +++ b/controllers/apps/operations/vertical_scaling.go @@ -55,10 +55,16 @@ func (vs verticalScalingHandler) ActionStartedCondition(opsRequest *appsv1alpha1 func (vs verticalScalingHandler) Action(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes *OpsResource) error { verticalScalingMap := opsRes.OpsRequest.Spec.ToVerticalScalingListToMap() for index, component := range opsRes.Cluster.Spec.ComponentSpecs { - if verticalScaling, ok := verticalScalingMap[component.Name]; ok { + verticalScaling, ok := verticalScalingMap[component.Name] + if !ok { + continue + } + if verticalScaling.Class != "" { + component.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: verticalScaling.Class} + } else { component.Resources = verticalScaling.ResourceRequirements - opsRes.Cluster.Spec.ComponentSpecs[index] = component } + opsRes.Cluster.Spec.ComponentSpecs[index] = component } return cli.Update(reqCtx.Ctx, opsRes.Cluster) } @@ -77,9 +83,13 @@ func (vs verticalScalingHandler) SaveLastConfiguration(reqCtx intctrlutil.Reques if _, ok := componentNameSet[v.Name]; !ok { continue } - lastComponentInfo[v.Name] = appsv1alpha1.LastComponentConfiguration{ + lastConfiguration := appsv1alpha1.LastComponentConfiguration{ ResourceRequirements: v.Resources, } + if v.ClassDefRef != nil { + lastConfiguration.Class = v.ClassDefRef.Class + } + lastComponentInfo[v.Name] = lastConfiguration } opsRes.OpsRequest.Status.LastConfiguration.Components = lastComponentInfo return nil diff --git a/controllers/apps/suite_test.go b/controllers/apps/suite_test.go index 3e8b1cfcc..6f5298e5a 100644 --- a/controllers/apps/suite_test.go +++ b/controllers/apps/suite_test.go @@ -210,8 +210,17 @@ var _ = BeforeSuite(func() { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + err = (&ComponentClassReconciler{ + Client: k8sManager.GetClient(), + Scheme: k8sManager.GetScheme(), + Recorder: k8sManager.GetEventRecorderFor("class-controller"), + }).SetupWithManager(k8sManager) + Expect(err).ToNot(HaveOccurred()) + testCtx = testutil.NewDefaultTestContext(ctx, k8sClient, testEnv) + appsv1alpha1.RegisterWebhookManager(k8sManager) + go func() { defer GinkgoRecover() err = k8sManager.Start(ctx) diff --git a/deploy/apecloud-mysql/templates/class.yaml b/deploy/apecloud-mysql/templates/class.yaml index af0db9a7c..098dc7093 100644 --- a/deploy/apecloud-mysql/templates/class.yaml +++ b/deploy/apecloud-mysql/templates/class.yaml @@ -1,63 +1,63 @@ -apiVersion: v1 -kind: ConfigMap +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ComponentClassDefinition metadata: name: kb.classes.default.apecloud-mysql.mysql labels: - class.kubeblocks.io/level: component class.kubeblocks.io/provider: kubeblocks apps.kubeblocks.io/component-def-ref: mysql clusterdefinition.kubeblocks.io/name: apecloud-mysql -data: - families-20230223162700: | - - family: kb-class-family-general - template: | - cpu: {{ printf "{{ or .cpu 1 }}" }} - memory: {{ printf "{{ or .memory 4 }}Gi" }} - storage: - - name: data - size: {{ printf "{{ or .dataStorageSize 10 }}Gi" }} - - name: log - size: {{ printf "{{ or .logStorageSize 1 }}Gi" }} - vars: [cpu, memory, dataStorageSize, logStorageSize] - series: - - name: {{ printf "general-{{ .cpu }}c{{ .memory }}g" }} - classes: - - args: [1, 1, 100, 10] - - args: [2, 2, 100, 10] - - args: [2, 4, 100, 10] - - args: [2, 8, 100, 10] - - args: [4, 16, 100, 10] - - args: [8, 32, 100, 10] - - args: [16, 64, 200, 10] - - args: [32, 128, 200, 10] - - args: [64, 256, 200, 10] - - args: [128, 512, 200, 10] +spec: + groups: + - resourceConstraintRef: kb-resource-constraint-general + template: | + cpu: {{ printf "{{ or .cpu 1 }}" }} + memory: {{ printf "{{ or .memory 4 }}Gi" }} + volumes: + - name: data + size: {{ printf "{{ or .dataStorageSize 10 }}Gi" }} + - name: log + size: {{ printf "{{ or .logStorageSize 1 }}Gi" }} + vars: [ cpu, memory, dataStorageSize, logStorageSize ] + series: + - namingTemplate: {{ printf "general-{{ .cpu }}c{{ .memory }}g" }} + classes: + - args: [ "0.5", "0.5", "10", "1" ] + - args: [ "1", "1", "10", "1" ] + - args: [ "2", "2", "10", "1" ] + - args: [ "2", "4", "10", "1" ] + - args: [ "2", "8", "10", "1" ] + - args: [ "4", "16", "10", "1" ] + - args: [ "8", "32", "10", "1" ] + - args: [ "16", "64", "20", "1" ] + - args: [ "32", "128", "20", "1" ] + - args: [ "64", "256", "20", "1" ] + - args: [ "128", "512", "20", "1" ] - - family: kb-class-family-memory-optimized - template: | - cpu: {{ printf "{{ or .cpu 1 }}" }} - memory: {{ printf "{{ or .memory 8 }}Gi" }} - storage: - - name: data - size: {{ printf "{{ or .dataStorageSize 10 }}Gi" }} - - name: log - size: {{ printf "{{ or .logStorageSize 1 }}Gi" }} - vars: [cpu, memory, dataStorageSize, logStorageSize] - series: - - name: {{ printf "mo-{{ .cpu }}c{{ .memory }}g" }} - classes: - # 1:8 - - args: [2, 16, 100, 10] - - args: [4, 32, 100, 10] - - args: [8, 64, 100, 10] - - args: [12, 96, 100, 10] - - args: [24, 192, 200, 10] - - args: [48, 384, 200, 10] - # 1:16 - - args: [2, 32, 100, 10] - - args: [4, 64, 100, 10] - - args: [8, 128, 100, 10] - - args: [16, 256, 100, 10] - - args: [32, 512, 200, 10] - - args: [48, 768, 200, 10] - - args: [64, 1024, 200, 10] \ No newline at end of file + - resourceConstraintRef: kb-resource-constraint-memory-optimized + template: | + cpu: {{ printf "{{ or .cpu 1 }}" }} + memory: {{ printf "{{ or .memory 8 }}Gi" }} + volumes: + - name: data + size: {{ printf "{{ or .dataStorageSize 10 }}Gi" }} + - name: log + size: {{ printf "{{ or .logStorageSize 1 }}Gi" }} + vars: [ cpu, memory, dataStorageSize, logStorageSize ] + series: + - namingTemplate: {{ printf "mo-{{ .cpu }}c{{ .memory }}g" }} + classes: + # 1:8 + - args: [ "2", "16", "10", "1" ] + - args: [ "4", "32", "10", "1" ] + - args: [ "8", "64", "10", "1" ] + - args: [ "12", "96", "10", "1" ] + - args: [ "24", "192", "20", "1" ] + - args: [ "48", "384", "20", "1" ] + # 1:16 + - args: [ "2", "32", "10", "1" ] + - args: [ "4", "64", "10", "1" ] + - args: [ "8", "128", "10", "1" ] + - args: [ "16", "256", "10", "1" ] + - args: [ "32", "512", "20", "1" ] + - args: [ "48", "768", "20", "1" ] + - args: [ "64", "1024", "20", "1" ] diff --git a/deploy/helm/config/rbac/role.yaml b/deploy/helm/config/rbac/role.yaml index 7bd0f58ca..3981b5213 100644 --- a/deploy/helm/config/rbac/role.yaml +++ b/deploy/helm/config/rbac/role.yaml @@ -110,14 +110,6 @@ rules: verbs: - get - list -- apiGroups: - - apps.kubeblocks.io - resources: - - classfamilies - verbs: - - get - - list - - watch - apiGroups: - apps.kubeblocks.io resources: @@ -196,6 +188,40 @@ rules: - get - patch - update +- apiGroups: + - apps.kubeblocks.io + resources: + - componentclassdefinitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.kubeblocks.io + resources: + - componentclassdefinitions/finalizers + verbs: + - update +- apiGroups: + - apps.kubeblocks.io + resources: + - componentclassdefinitions/status + verbs: + - get + - patch + - update +- apiGroups: + - apps.kubeblocks.io + resources: + - componentresourceconstraints + verbs: + - get + - list + - watch - apiGroups: - apps.kubeblocks.io resources: diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml index 6f8b269fa..b41dc82b0 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml @@ -156,6 +156,19 @@ spec: type: array x-kubernetes-list-type: set type: object + classDefRef: + description: classDefRef reference class defined in ComponentClassDefinition. + properties: + class: + description: class refers to the name of the class that + is defined in the ComponentClassDefinition. + type: string + name: + description: name refers to the name of the ComponentClassDefinition. + type: string + required: + - class + type: object componentDefRef: description: componentDefRef reference componentDef defined in ClusterDefinition spec. diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentclassdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentclassdefinitions.yaml new file mode 100644 index 000000000..37deea269 --- /dev/null +++ b/deploy/helm/crds/apps.kubeblocks.io_componentclassdefinitions.yaml @@ -0,0 +1,261 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: componentclassdefinitions.apps.kubeblocks.io +spec: + group: apps.kubeblocks.io + names: + categories: + - kubeblocks + kind: ComponentClassDefinition + listKind: ComponentClassDefinitionList + plural: componentclassdefinitions + shortNames: + - ccd + singular: componentclassdefinition + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ComponentClassDefinition is the Schema for the componentclassdefinitions + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ComponentClassDefinitionSpec defines the desired state of + ComponentClassDefinition + properties: + groups: + description: group defines a list of class series that conform to + the same constraint. + items: + properties: + resourceConstraintRef: + description: resourceConstraintRef reference to the resource + constraint object name, indicates that the series defined + below all conform to the constraint. + type: string + series: + description: series is a series of class definitions. + items: + properties: + classes: + description: classes are definitions of classes that come + in two forms. In the first form, only ComponentClass.Args + need to be defined, and the complete class definition + is generated by rendering the ComponentClassGroup.Template + and Name. In the second form, the Name, CPU, Memory, + and Volumes must be defined. + items: + properties: + args: + description: args are variable's value + items: + type: string + type: array + cpu: + anyOf: + - type: integer + - type: string + description: the CPU of the class + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: the memory of the class + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name is the class name + type: string + variants: + description: the variants of the class in different + clouds. + items: + properties: + args: + description: cloud provider specific variables + items: + type: string + type: array + provider: + description: cloud provider name + type: string + required: + - provider + type: object + type: array + x-kubernetes-list-map-keys: + - provider + x-kubernetes-list-type: map + volumes: + description: the volumes of the class + items: + properties: + name: + description: The volume name, etc. data, log. + type: string + size: + anyOf: + - type: integer + - type: string + description: The size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageCLassName: + description: The StorageClass name of the + volume. + type: string + required: + - name + - size + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: array + namingTemplate: + description: 'namingTemplate is a template that uses the + Go template syntax and allows for referencing variables + defined in ComponentClassGroup.Template. This enables + dynamic generation of class names. For example: name: + "general-{{ .cpu }}c{{ .memory }}g"' + type: string + type: object + type: array + template: + description: "template is a class definition template that uses + the Go template syntax and allows for variable declaration. + When defining a class in Series, specifying the variable's + value is sufficient, as the complete class definition will + be generated through rendering the template. \n For example: + template: | cpu: \"{{ or .cpu 1 }}\" memory: \"{{ or .memory + 4 }}Gi\" storage: - name: data size: \"{{ or .dataStorageSize + 10 }}Gi\" - name: log size: \"{{ or .logStorageSize 1 }}Gi\"" + type: string + vars: + description: vars defines the variables declared in the template + and will be used to generating the complete class definition + by render the template. + items: + type: string + type: array + x-kubernetes-list-type: set + required: + - resourceConstraintRef + type: object + type: array + type: object + status: + description: ComponentClassDefinitionStatus defines the observed state + of ComponentClassDefinition + properties: + classes: + description: classes is the list of classes that have been observed + for this ComponentClassDefinition + items: + properties: + args: + description: args are variable's value + items: + type: string + type: array + cpu: + anyOf: + - type: integer + - type: string + description: the CPU of the class + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: the memory of the class + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: name is the class name + type: string + resourceConstraintRef: + description: resourceConstraintRef reference to the resource + constraint object name. + type: string + variants: + description: the variants of the class in different clouds. + items: + properties: + args: + description: cloud provider specific variables + items: + type: string + type: array + provider: + description: cloud provider name + type: string + required: + - provider + type: object + type: array + x-kubernetes-list-map-keys: + - provider + x-kubernetes-list-type: map + volumes: + description: the volumes of the class + items: + properties: + name: + description: The volume name, etc. data, log. + type: string + size: + anyOf: + - type: integer + - type: string + description: The size of the volume. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageCLassName: + description: The StorageClass name of the volume. + type: string + required: + - name + - size + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + type: array + observedGeneration: + description: observedGeneration is the most recent generation observed + for this ComponentClassDefinition. It corresponds to the ComponentClassDefinition's + generation, which is updated on mutation by the API Server. + format: int64 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/apps.kubeblocks.io_classfamilies.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentresourceconstraints.yaml similarity index 92% rename from config/crd/bases/apps.kubeblocks.io_classfamilies.yaml rename to deploy/helm/crds/apps.kubeblocks.io_componentresourceconstraints.yaml index 1c4f66d32..5f00d7070 100644 --- a/config/crd/bases/apps.kubeblocks.io_classfamilies.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentresourceconstraints.yaml @@ -5,25 +5,26 @@ metadata: annotations: controller-gen.kubebuilder.io/version: v0.9.0 creationTimestamp: null - name: classfamilies.apps.kubeblocks.io + name: componentresourceconstraints.apps.kubeblocks.io spec: group: apps.kubeblocks.io names: categories: - kubeblocks - all - kind: ClassFamily - listKind: ClassFamilyList - plural: classfamilies + kind: ComponentResourceConstraint + listKind: ComponentResourceConstraintList + plural: componentresourceconstraints shortNames: - - cf - singular: classfamily + - crc + singular: componentresourceconstraint scope: Cluster versions: - name: v1alpha1 schema: openAPIV3Schema: - description: ClassFamily is the Schema for the classfamilies API + description: ComponentResourceConstraint is the Schema for the componentresourceconstraints + API properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -38,11 +39,11 @@ spec: metadata: type: object spec: - description: ClassFamilySpec defines the desired state of ClassFamily + description: ComponentResourceConstraintSpec defines the desired state + of ComponentResourceConstraint properties: - models: - description: Class family models, generally, a model is a specific - constraint for CPU, memory and their relation. + constraints: + description: Component resource constraints items: properties: cpu: @@ -133,6 +134,9 @@ spec: pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object + required: + - cpu + - memory type: object type: array type: object diff --git a/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml b/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml index 0ab01c7f7..b0679d64b 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml @@ -277,6 +277,9 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + class: + description: class specifies the class name of the component + type: string componentName: description: componentName cluster component name. type: string @@ -539,6 +542,9 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + class: + description: the last class name of the component. + type: string limits: additionalProperties: anyOf: diff --git a/deploy/helm/templates/class/classfamily.yaml b/deploy/helm/templates/class/componentclassconstraint.yaml similarity index 64% rename from deploy/helm/templates/class/classfamily.yaml rename to deploy/helm/templates/class/componentclassconstraint.yaml index 14ad1038f..ddf6c38fc 100644 --- a/deploy/helm/templates/class/classfamily.yaml +++ b/deploy/helm/templates/class/componentclassconstraint.yaml @@ -1,11 +1,11 @@ apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClassFamily +kind: ComponentResourceConstraint metadata: - name: kb-class-family-general + name: kb-resource-constraint-general labels: - classfamily.kubeblocks.io/provider: kubeblocks + resourceconstraint.kubeblocks.io/provider: kubeblocks spec: - models: + constraints: - cpu: min: "0.5" max: 2 @@ -25,13 +25,13 @@ spec: --- apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClassFamily +kind: ComponentResourceConstraint metadata: - name: kb-class-family-memory-optimized + name: kb-resource-constraint-memory-optimized labels: - classfamily.kubeblocks.io/provider: kubeblocks + resourceconstraint.kubeblocks.io/provider: kubeblocks spec: - models: + constraints: - cpu: slots: [2, 4, 8, 12, 24, 48] memory: diff --git a/docs/user_docs/cli/kbcli_class_create.md b/docs/user_docs/cli/kbcli_class_create.md index 46dcf7849..7c493e43d 100644 --- a/docs/user_docs/cli/kbcli_class_create.md +++ b/docs/user_docs/cli/kbcli_class_create.md @@ -11,8 +11,8 @@ kbcli class create [NAME] [flags] ### Examples ``` - # Create a class following class family kubeblocks-general-classes for component mysql in cluster definition apecloud-mysql, which have 1 cpu core, 2Gi memory and storage is 10Gi - kbcli class create custom-1c2g --cluster-definition apecloud-mysql --type mysql --class-family kubeblocks-general-classes --cpu 1 --memory 2Gi --storage name=data,size=10Gi + # Create a class following constraint kb-resource-constraint-general for component mysql in cluster definition apecloud-mysql, which have 1 cpu core, 2Gi memory and storage is 10Gi + kbcli class create custom-1c2g --cluster-definition apecloud-mysql --type mysql --constraint kb-resource-constraint-general --cpu 1 --memory 2Gi --storage name=data,size=10Gi # Create classes for component mysql in cluster definition apecloud-mysql, where classes is defined in file kbcli class create --cluster-definition apecloud-mysql --type mysql --file ./classes.yaml @@ -21,8 +21,8 @@ kbcli class create [NAME] [flags] ### Options ``` - --class-family string Specify class family --cluster-definition string Specify cluster definition, run "kbcli clusterdefinition list" to show all available cluster definition + --constraint string Specify resource constraint --cpu string Specify component cpu cores --file string Specify file path which contains YAML definition of class -h, --help help for create diff --git a/docs/user_docs/cli/kbcli_class_list.md b/docs/user_docs/cli/kbcli_class_list.md index 798d52e85..c44de63f0 100644 --- a/docs/user_docs/cli/kbcli_class_list.md +++ b/docs/user_docs/cli/kbcli_class_list.md @@ -18,7 +18,7 @@ kbcli class list [flags] ### Options ``` - --cluster-definition string Specify cluster definition, run "kbcli cluster-definition list" to show all available cluster definition + --cluster-definition string Specify cluster definition, run "kbcli clusterdefinition list" to show all available cluster definition -h, --help help for list ``` diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index d6ec2c9f0..a8bc7a166 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -35,8 +35,8 @@ kbcli cluster create [CLUSTER_NAME] [flags] # Create a cluster and set cpu to 1 core, memory to 1Gi, storage size to 20Gi and replicas to 3 kbcli cluster create mycluster --cluster-definition apecloud-mysql --set cpu=1,memory=1Gi,storage=20Gi,replicas=3 - # Create a cluster and set class to general-1c4g - kbcli cluster create myclsuter --cluster-definition apecloud-mysql --set class=general-1c4g + # Create a cluster and set the class to general-1c1g, valid classes can be found by executing the command "kbcli class list --cluster-definition=" + kbcli cluster create myclsuter --cluster-definition apecloud-mysql --set class=general-1c1g # Create a cluster with replicationSet workloadType and set switchPolicy to Noop kbcli cluster create myclsuter --cluster-definition postgresql --set switchPolicy=Noop @@ -72,7 +72,7 @@ kbcli cluster create [CLUSTER_NAME] [flags] --monitor Set monitor enabled and inject metrics exporter (default true) --node-labels stringToString Node label selector (default []) --pod-anti-affinity string Pod anti-affinity type, one of: (Preferred, Required) (default "Preferred") - --set stringArray Set the cluster resource including cpu, memory, replicas and storage, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi) + --set stringArray Set the cluster resource including cpu, memory, replicas and storage, or you can just specify the class, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi or --set class=general-1c1g) -f, --set-file string Use yaml file, URL, or stdin to set the cluster resource --tenancy string Tenancy options, one of: (SharedNode, DedicatedNode) (default "SharedNode") --termination-policy string Termination policy, one of: (DoNotTerminate, Halt, Delete, WipeOut) (default "Delete") diff --git a/docs/user_docs/cli/kbcli_cluster_vscale.md b/docs/user_docs/cli/kbcli_cluster_vscale.md index 1817bc4fe..964192be7 100644 --- a/docs/user_docs/cli/kbcli_cluster_vscale.md +++ b/docs/user_docs/cli/kbcli_cluster_vscale.md @@ -13,6 +13,9 @@ kbcli cluster vscale [flags] ``` # scale the computing resources of specified components, separate with commas when more than one kbcli cluster vscale --components= --cpu=500m --memory=500Mi + + # scale the computing resources of specified components by class, available classes can be get by executing the command "kbcli class list --cluster-definition " + kbcli cluster vscale --components= --set class=general-1c4g ``` ### Options diff --git a/internal/class/class_utils.go b/internal/class/class_utils.go index dfe420dd0..f44e89f2e 100644 --- a/internal/class/class_utils.go +++ b/internal/class/class_utils.go @@ -20,10 +20,8 @@ import ( "context" "fmt" "sort" - "strconv" "strings" "text/template" - "time" "github.com/ghodss/yaml" corev1 "k8s.io/api/core/v1" @@ -31,27 +29,26 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/constant" ) -// GetCustomClassConfigMapName Returns the name of the ConfigMap containing the custom classes -func GetCustomClassConfigMapName(cdName string, componentName string) string { +// GetCustomClassObjectName Returns the name of the ComponentClassDefinition object containing the custom classes +func GetCustomClassObjectName(cdName string, componentName string) string { return fmt.Sprintf("kb.classes.custom.%s.%s", cdName, componentName) } // ChooseComponentClasses Choose the classes to be used for a given component with some constraints -func ChooseComponentClasses(classes map[string]*ComponentClass, filters map[string]resource.Quantity) *ComponentClass { - var candidates []*ComponentClass +func ChooseComponentClasses(classes map[string]*v1alpha1.ComponentClassInstance, filters map[corev1.ResourceName]resource.Quantity) *v1alpha1.ComponentClassInstance { + var candidates []*v1alpha1.ComponentClassInstance for _, cls := range classes { - cpu, ok := filters[corev1.ResourceCPU.String()] + cpu, ok := filters[corev1.ResourceCPU] if ok && !cpu.Equal(cls.CPU) { continue } - memory, ok := filters[corev1.ResourceMemory.String()] + memory, ok := filters[corev1.ResourceMemory] if ok && !memory.Equal(cls.Memory) { continue } @@ -64,21 +61,49 @@ func ChooseComponentClasses(classes map[string]*ComponentClass, filters map[stri return candidates[0] } -func GetClassFamilies(dynamic dynamic.Interface) (map[string]*v1alpha1.ClassFamily, error) { - objs, err := dynamic.Resource(types.ClassFamilyGVR()).Namespace("").List(context.TODO(), metav1.ListOptions{ - //LabelSelector: types.ClassFamilyProviderLabelKey, +func GetClasses(classDefinitionList v1alpha1.ComponentClassDefinitionList) (map[string]map[string]*v1alpha1.ComponentClassInstance, error) { + var ( + componentClasses = make(map[string]map[string]*v1alpha1.ComponentClassInstance) + ) + for _, classDefinition := range classDefinitionList.Items { + componentType := classDefinition.GetLabels()["apps.kubeblocks.io/component-def-ref"] + if componentType == "" { + return nil, fmt.Errorf("failed to find component type") + } + classes := make(map[string]*v1alpha1.ComponentClassInstance) + for idx := range classDefinition.Status.Classes { + cls := classDefinition.Status.Classes[idx] + classes[cls.Name] = &cls + } + if _, ok := componentClasses[componentType]; !ok { + componentClasses[componentType] = classes + } else { + for k, v := range classes { + if _, exists := componentClasses[componentType][k]; exists { + return nil, fmt.Errorf("duplicate component class %s", k) + } + componentClasses[componentType][k] = v + } + } + } + + return componentClasses, nil +} +func GetResourceConstraints(dynamic dynamic.Interface) (map[string]*v1alpha1.ComponentResourceConstraint, error) { + objs, err := dynamic.Resource(types.ComponentResourceConstraintGVR()).List(context.TODO(), metav1.ListOptions{ + //LabelSelector: types.ResourceConstraintProviderLabelKey, }) if err != nil { return nil, err } - var classFamilyList v1alpha1.ClassFamilyList - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(objs.UnstructuredContent(), &classFamilyList); err != nil { + var constraintsList v1alpha1.ComponentResourceConstraintList + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(objs.UnstructuredContent(), &constraintsList); err != nil { return nil, err } - result := make(map[string]*v1alpha1.ClassFamily) - for _, cf := range classFamilyList.Items { - if _, ok := cf.GetLabels()[types.ClassFamilyProviderLabelKey]; !ok { + result := make(map[string]*v1alpha1.ComponentResourceConstraint) + for _, cf := range constraintsList.Items { + if _, ok := cf.GetLabels()[types.ResourceConstraintProviderLabelKey]; !ok { continue } result[cf.GetName()] = &cf @@ -86,83 +111,26 @@ func GetClassFamilies(dynamic dynamic.Interface) (map[string]*v1alpha1.ClassFami return result, nil } -// GetClasses Get all classes, including kubeblocks default classes and user custom classes -func GetClasses(client kubernetes.Interface, cdName string) (map[string]map[string]*ComponentClass, error) { +// ListClassesByClusterDefinition Get all classes, including kubeblocks default classes and user custom classes +func ListClassesByClusterDefinition(client dynamic.Interface, cdName string) (map[string]map[string]*v1alpha1.ComponentClassInstance, error) { selector := fmt.Sprintf("%s=%s,%s", constant.ClusterDefLabelKey, cdName, types.ClassProviderLabelKey) - cmList, err := client.CoreV1().ConfigMaps(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{ + objs, err := client.Resource(types.ComponentClassDefinitionGVR()).Namespace("").List(context.TODO(), metav1.ListOptions{ LabelSelector: selector, }) if err != nil { return nil, err } - return ParseClasses(cmList) -} - -func ParseClasses(cmList *corev1.ConfigMapList) (map[string]map[string]*ComponentClass, error) { - var ( - componentClasses = make(map[string]map[string]*ComponentClass) - ) - for _, cm := range cmList.Items { - if _, ok := cm.GetLabels()[types.ClassProviderLabelKey]; !ok { - continue - } - level := cm.GetLabels()[types.ClassLevelLabelKey] - switch level { - case "component": - componentType := cm.GetLabels()[constant.KBAppComponentDefRefLabelKey] - if componentType == "" { - return nil, fmt.Errorf("failed to find component type") - } - classes, err := ParseComponentClasses(cm.Data) - if err != nil { - return nil, err - } - if _, ok := componentClasses[componentType]; !ok { - componentClasses[componentType] = classes - } else { - for k, v := range classes { - if _, exists := componentClasses[componentType][k]; exists { - return nil, fmt.Errorf("duplicate component class %s", k) - } - componentClasses[componentType][k] = v - } - } - case "cluster": - // TODO - default: - return nil, fmt.Errorf("invalid class level: %s", level) - } + var classDefinitionList v1alpha1.ComponentClassDefinitionList + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(objs.UnstructuredContent(), &classDefinitionList); err != nil { + return nil, err } - - return componentClasses, nil + return GetClasses(classDefinitionList) } -type classVersion int64 - -// ParseComponentClasses parse configmap.data to component classes -func ParseComponentClasses(data map[string]string) (map[string]*ComponentClass, error) { - versions := make(map[classVersion][]*ComponentClassFamilyDef) - - for k, v := range data { - // ConfigMap data key follows the format: families-[version] - // version is the timestamp in unix microseconds which class is created - parts := strings.SplitAfterN(k, "-", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("invalid key: %s", k) - } - version, err := strconv.ParseInt(parts[1], 10, 64) - if err != nil { - return nil, fmt.Errorf("invalid key: %s", k) - } - var families []*ComponentClassFamilyDef - if err := yaml.Unmarshal([]byte(v), &families); err != nil { - return nil, err - } - versions[classVersion(version)] = families - } - - genClassDef := func(nameTpl string, bodyTpl string, vars []string, args []string) (ComponentClassDef, error) { - var def ComponentClassDef +// ParseComponentClasses parse ComponentClassDefinition to component classes +func ParseComponentClasses(classDefinition v1alpha1.ComponentClassDefinition) (map[string]*v1alpha1.ComponentClassInstance, error) { + genClass := func(nameTpl string, bodyTpl string, vars []string, args []string) (v1alpha1.ComponentClass, error) { + var result v1alpha1.ComponentClass values := make(map[string]interface{}) for index, key := range vars { values[key] = args[index] @@ -170,69 +138,65 @@ func ParseComponentClasses(data map[string]string) (map[string]*ComponentClass, classStr, err := renderTemplate(bodyTpl, values) if err != nil { - return def, err + return result, err } - if err = yaml.Unmarshal([]byte(classStr), &def); err != nil { - return def, err + if err = yaml.Unmarshal([]byte(classStr), &result); err != nil { + return result, err } - def.Name, err = renderTemplate(nameTpl, values) + name, err := renderTemplate(nameTpl, values) if err != nil { - return def, err + return result, err } - return def, nil + result.Name = name + return result, nil } - parser := func(family *ComponentClassFamilyDef, series ComponentClassSeriesDef, class ComponentClassDef) (*ComponentClass, error) { - var ( - err error - def = class - ) - + parser := func(group v1alpha1.ComponentClassGroup, series v1alpha1.ComponentClassSeries, class v1alpha1.ComponentClass) (*v1alpha1.ComponentClassInstance, error) { if len(class.Args) > 0 { - def, err = genClassDef(series.Name, family.Template, family.Vars, class.Args) + cls, err := genClass(series.NamingTemplate, group.Template, group.Vars, class.Args) if err != nil { return nil, err } - if class.Name != "" { - def.Name = class.Name + if class.Name == "" && cls.Name != "" { + class.Name = cls.Name } - } - - result := &ComponentClass{ - Name: def.Name, - Family: family.Family, - CPU: resource.MustParse(def.CPU), - Memory: resource.MustParse(def.Memory), - } - - for _, disk := range def.Storage { - result.Storage = append(result.Storage, &Disk{ - Name: disk.Name, - Class: disk.Class, - Size: resource.MustParse(disk.Size), + class.CPU = cls.CPU + class.Memory = cls.Memory + class.Volumes = cls.Volumes + } + result := &v1alpha1.ComponentClassInstance{ + ComponentClass: v1alpha1.ComponentClass{ + Name: class.Name, + CPU: class.CPU, + Memory: class.Memory, + }, + ResourceConstraintRef: group.ResourceConstraintRef, + } + for _, volume := range class.Volumes { + result.Volumes = append(result.Volumes, v1alpha1.Volume{ + Name: volume.Name, + StorageClassName: volume.StorageClassName, + Size: volume.Size, }) } - return result, nil } - result := make(map[string]*ComponentClass) - for _, families := range versions { - for _, family := range families { - for _, series := range family.Series { - for _, class := range series.Classes { - out, err := parser(family, series, class) - if err != nil { - return nil, err - } - if _, exists := result[out.Name]; exists { - return nil, fmt.Errorf("duplicate component class name: %s", out.Name) - } - result[out.Name] = out + result := make(map[string]*v1alpha1.ComponentClassInstance) + for _, group := range classDefinition.Spec.Groups { + for _, series := range group.Series { + for _, class := range series.Classes { + out, err := parser(group, series, class) + if err != nil { + return nil, err + } + if _, exists := result[out.Name]; exists { + return nil, fmt.Errorf("duplicate component class name: %s", out.Name) } + result[out.Name] = out } } } @@ -250,8 +214,3 @@ func renderTemplate(tpl string, values map[string]interface{}) (string, error) { } return buf.String(), nil } - -// BuildClassDefinitionVersion generate the key in the configmap data field -func BuildClassDefinitionVersion() string { - return fmt.Sprintf("version-%s", time.Now().Format("20060102150405")) -} diff --git a/internal/class/class_utils_test.go b/internal/class/class_utils_test.go index f68ec1fb0..a0631e8fb 100644 --- a/internal/class/class_utils_test.go +++ b/internal/class/class_utils_test.go @@ -22,7 +22,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" ) var _ = Describe("utils", func() { @@ -30,18 +34,22 @@ var _ = Describe("utils", func() { cpuMin = 1 cpuMax = 64 scales = []int{4, 8, 16} - classes map[string]*ComponentClass + classes map[string]*v1alpha1.ComponentClassInstance ) - genComponentClasses := func(cpuMin int, cpuMax int, scales []int) map[string]*ComponentClass { - results := make(map[string]*ComponentClass) + genComponentClasses := func(cpuMin int, cpuMax int, scales []int) map[string]*v1alpha1.ComponentClassInstance { + results := make(map[string]*v1alpha1.ComponentClassInstance) for cpu := cpuMin; cpu <= cpuMax; cpu++ { for _, scale := range scales { - name := fmt.Sprintf("cpu-%d-scale-%d", cpu, scale) - results[name] = &ComponentClass{ - Name: name, - CPU: resource.MustParse(fmt.Sprintf("%d", cpu)), - Memory: resource.MustParse(fmt.Sprintf("%dGi", cpu*scale)), + var ( + clsName = fmt.Sprintf("cpu-%d-scale-%d", cpu, scale) + ) + results[clsName] = &v1alpha1.ComponentClassInstance{ + ComponentClass: v1alpha1.ComponentClass{ + Name: clsName, + CPU: resource.MustParse(fmt.Sprintf("%d", cpu)), + Memory: resource.MustParse(fmt.Sprintf("%dGi", cpu*scale)), + }, } } } @@ -56,13 +64,13 @@ var _ = Describe("utils", func() { // Add any teardown steps that needs to be executed after each test }) - buildFilters := func(cpu string, memory string) map[string]resource.Quantity { - result := make(map[string]resource.Quantity) + buildFilters := func(cpu string, memory string) map[corev1.ResourceName]resource.Quantity { + result := make(map[corev1.ResourceName]resource.Quantity) if cpu != "" { - result["cpu"] = resource.MustParse(cpu) + result[corev1.ResourceCPU] = resource.MustParse(cpu) } if memory != "" { - result["memory"] = resource.MustParse(memory) + result[corev1.ResourceMemory] = resource.MustParse(memory) } return result } diff --git a/internal/class/types.go b/internal/class/types.go index dc43a38e2..84d79a912 100644 --- a/internal/class/types.go +++ b/internal/class/types.go @@ -17,9 +17,7 @@ limitations under the License. package class import ( - "fmt" "sort" - "strings" "gopkg.in/inf.v0" "k8s.io/apimachinery/pkg/api/resource" @@ -27,7 +25,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" ) -func GetMinCPUAndMemory(model appsv1alpha1.ClassFamilyModel) (*resource.Quantity, *resource.Quantity) { +func GetMinCPUAndMemory(model appsv1alpha1.ResourceConstraint) (*resource.Quantity, *resource.Quantity) { var ( minCPU resource.Quantity minMemory resource.Quantity @@ -50,20 +48,22 @@ func GetMinCPUAndMemory(model appsv1alpha1.ClassFamilyModel) (*resource.Quantity return &minCPU, &minMemory } -type ClassModelWithFamilyName struct { - Family string - Model appsv1alpha1.ClassFamilyModel +type ConstraintWithName struct { + Name string + Constraint appsv1alpha1.ResourceConstraint } -type ByModelList []ClassModelWithFamilyName +var _ sort.Interface = ByConstraintList{} -func (m ByModelList) Len() int { +type ByConstraintList []ConstraintWithName + +func (m ByConstraintList) Len() int { return len(m) } -func (m ByModelList) Less(i, j int) bool { - cpu1, mem1 := GetMinCPUAndMemory(m[i].Model) - cpu2, mem2 := GetMinCPUAndMemory(m[j].Model) +func (m ByConstraintList) Less(i, j int) bool { + cpu1, mem1 := GetMinCPUAndMemory(m[i].Constraint) + cpu2, mem2 := GetMinCPUAndMemory(m[j].Constraint) switch cpu1.Cmp(*cpu2) { case 1: return false @@ -79,21 +79,13 @@ func (m ByModelList) Less(i, j int) bool { return false } -func (m ByModelList) Swap(i, j int) { +func (m ByConstraintList) Swap(i, j int) { m[i], m[j] = m[j], m[i] } -type ComponentClass struct { - Name string `json:"name,omitempty"` - CPU resource.Quantity `json:"cpu,omitempty"` - Memory resource.Quantity `json:"memory,omitempty"` - Storage []*Disk `json:"storage,omitempty"` - Family string `json:"-"` -} - var _ sort.Interface = ByClassCPUAndMemory{} -type ByClassCPUAndMemory []*ComponentClass +type ByClassCPUAndMemory []*appsv1alpha1.ComponentClassInstance func (b ByClassCPUAndMemory) Len() int { return len(b) @@ -114,55 +106,3 @@ func (b ByClassCPUAndMemory) Less(i, j int) bool { func (b ByClassCPUAndMemory) Swap(i, j int) { b[i], b[j] = b[j], b[i] } - -type Filters map[string]resource.Quantity - -func (f Filters) String() string { - var result []string - for k, v := range f { - result = append(result, fmt.Sprintf("%s=%v", k, v.Value())) - } - return strings.Join(result, ",") -} - -type Disk struct { - Name string `json:"name,omitempty"` - Size resource.Quantity `json:"size,omitempty"` - Class string `json:"class,omitempty"` -} - -func (d Disk) String() string { - return fmt.Sprintf("%s=%s", d.Name, d.Size.String()) -} - -type ProviderComponentClassDef struct { - Provider string `json:"provider,omitempty"` - Args []string `json:"args,omitempty"` -} - -type DiskDef struct { - Name string `json:"name,omitempty"` - Size string `json:"size,omitempty"` - Class string `json:"class,omitempty"` -} - -type ComponentClassDef struct { - Name string `json:"name,omitempty"` - CPU string `json:"cpu,omitempty"` - Memory string `json:"memory,omitempty"` - Storage []DiskDef `json:"storage,omitempty"` - Args []string `json:"args,omitempty"` - Variants []ProviderComponentClassDef `json:"variants,omitempty"` -} - -type ComponentClassSeriesDef struct { - Name string `json:"name,omitempty"` - Classes []ComponentClassDef `json:"classes,omitempty"` -} - -type ComponentClassFamilyDef struct { - Family string `json:"family"` - Template string `json:"template,omitempty"` - Vars []string `json:"vars,omitempty"` - Series []ComponentClassSeriesDef `json:"series,omitempty"` -} diff --git a/internal/class/types_test.go b/internal/class/types_test.go index ad65a631b..3f4c018b4 100644 --- a/internal/class/types_test.go +++ b/internal/class/types_test.go @@ -27,15 +27,14 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" ) -const classFamilyBytes = ` +const resourceConstraintBytes = ` # API scope: cluster -# ClusterClassFamily apiVersion: "apps.kubeblocks.io/v1alpha1" -kind: "ClassFamily" +kind: "ComponentResourceConstraint" metadata: - name: kb-class-family-general + name: kb-resource-constraint-general spec: - models: + constraints: - cpu: min: 0.5 max: 128 @@ -55,11 +54,16 @@ spec: maxPerCPU: 8Gi ` -func TestClassFamily_ByClassCPUAndMemory(t *testing.T) { - buildClass := func(cpu string, memory string) *ComponentClass { - return &ComponentClass{CPU: resource.MustParse(cpu), Memory: resource.MustParse(memory)} +func TestResourceConstraint_ByClassCPUAndMemory(t *testing.T) { + buildClass := func(cpu string, memory string) *appsv1alpha1.ComponentClassInstance { + return &appsv1alpha1.ComponentClassInstance{ + ComponentClass: appsv1alpha1.ComponentClass{ + CPU: resource.MustParse(cpu), + Memory: resource.MustParse(memory), + }, + } } - classes := []*ComponentClass{ + classes := []*appsv1alpha1.ComponentClassInstance{ buildClass("1", "2Gi"), buildClass("1", "1Gi"), buildClass("2", "0.5Gi"), @@ -68,24 +72,24 @@ func TestClassFamily_ByClassCPUAndMemory(t *testing.T) { } sort.Sort(ByClassCPUAndMemory(classes)) candidate := classes[0] - if candidate.CPU != resource.MustParse("0.5") || candidate.Memory != resource.MustParse("10Gi") { + if !candidate.CPU.Equal(resource.MustParse("0.5")) || !candidate.Memory.Equal(resource.MustParse("10Gi")) { t.Errorf("case failed") } } -func TestClassFamily_ModelList(t *testing.T) { - var cf appsv1alpha1.ClassFamily - err := yaml.Unmarshal([]byte(classFamilyBytes), &cf) +func TestResourceConstraint_ConstraintList(t *testing.T) { + var cf appsv1alpha1.ComponentResourceConstraint + err := yaml.Unmarshal([]byte(resourceConstraintBytes), &cf) if err != nil { - panic("Failed to unmarshal class family: %v" + err.Error()) + panic("Failed to unmarshal resource constraint: %v" + err.Error()) } - var models []ClassModelWithFamilyName - for _, model := range cf.Spec.Models { - models = append(models, ClassModelWithFamilyName{Family: cf.Name, Model: model}) + var constraints []ConstraintWithName + for _, constraint := range cf.Spec.Constraints { + constraints = append(constraints, ConstraintWithName{Name: cf.Name, Constraint: constraint}) } resource.MustParse("200Mi") - sort.Sort(ByModelList(models)) - cpu, memory := GetMinCPUAndMemory(models[0].Model) + sort.Sort(ByConstraintList(constraints)) + cpu, memory := GetMinCPUAndMemory(constraints[0].Constraint) assert.Equal(t, cpu.Cmp(resource.MustParse("0.1")) == 0, true) assert.Equal(t, memory.Cmp(resource.MustParse("20Mi")) == 0, true) } diff --git a/internal/cli/cmd/class/class.go b/internal/cli/cmd/class/class.go index 4b10d796f..27e999017 100644 --- a/internal/cli/cmd/class/class.go +++ b/internal/cli/cmd/class/class.go @@ -22,11 +22,6 @@ import ( cmdutil "k8s.io/kubectl/pkg/cmd/util" ) -const ( - CustomClassNamespace = "kube-system" - CMDataKeyDefinition = "definition" -) - func NewClassCommand(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { cmd := &cobra.Command{ Use: "class", diff --git a/internal/cli/cmd/class/create.go b/internal/cli/cmd/class/create.go index ac82d22fa..47d57f341 100644 --- a/internal/cli/cmd/class/create.go +++ b/internal/cli/cmd/class/create.go @@ -28,12 +28,14 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" + "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/class" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" @@ -44,10 +46,9 @@ type CreateOptions struct { genericclioptions.IOStreams Factory cmdutil.Factory - client kubernetes.Interface dynamic dynamic.Interface ClusterDefRef string - ClassFamily string + Constraint string ComponentType string ClassName string CPU string @@ -57,8 +58,8 @@ type CreateOptions struct { } var classCreateExamples = templates.Examples(` - # Create a class following class family kubeblocks-general-classes for component mysql in cluster definition apecloud-mysql, which have 1 cpu core, 2Gi memory and storage is 10Gi - kbcli class create custom-1c2g --cluster-definition apecloud-mysql --type mysql --class-family kubeblocks-general-classes --cpu 1 --memory 2Gi --storage name=data,size=10Gi + # Create a class following constraint kb-resource-constraint-general for component mysql in cluster definition apecloud-mysql, which have 1 cpu core, 2Gi memory and storage is 10Gi + kbcli class create custom-1c2g --cluster-definition apecloud-mysql --type mysql --constraint kb-resource-constraint-general --cpu 1 --memory 2Gi --storage name=data,size=10Gi # Create classes for component mysql in cluster definition apecloud-mysql, where classes is defined in file kbcli class create --cluster-definition apecloud-mysql --type mysql --file ./classes.yaml @@ -81,7 +82,7 @@ func NewCreateCommand(f cmdutil.Factory, streams genericclioptions.IOStreams) *c cmd.Flags().StringVar(&o.ComponentType, "type", "", "Specify component type") util.CheckErr(cmd.MarkFlagRequired("type")) - cmd.Flags().StringVar(&o.ClassFamily, "class-family", "", "Specify class family") + cmd.Flags().StringVar(&o.Constraint, "constraint", "", "Specify resource constraint") cmd.Flags().StringVar(&o.CPU, corev1.ResourceCPU.String(), "", "Specify component cpu cores") cmd.Flags().StringVar(&o.Memory, corev1.ResourceMemory.String(), "", "Specify component memory size") cmd.Flags().StringArrayVar(&o.Storage, corev1.ResourceStorage.String(), []string{}, "Specify component storage disks") @@ -116,38 +117,29 @@ func (o *CreateOptions) validate(args []string) error { func (o *CreateOptions) complete(f cmdutil.Factory) error { var err error - if o.client, err = f.KubernetesClientSet(); err != nil { - return err - } - if o.dynamic, err = f.DynamicClient(); err != nil { - return err - } - return nil + o.dynamic, err = f.DynamicClient() + return err } func (o *CreateOptions) run() error { - componentClasses, err := class.GetClasses(o.client, o.ClusterDefRef) + componentClasses, err := class.ListClassesByClusterDefinition(o.dynamic, o.ClusterDefRef) if err != nil { return err } classes, ok := componentClasses[o.ComponentType] if !ok { - classes = make(map[string]*class.ComponentClass) + classes = make(map[string]*v1alpha1.ComponentClassInstance) } - families, err := class.GetClassFamilies(o.dynamic) + constraints, err := class.GetResourceConstraints(o.dynamic) if err != nil { return err } var ( - // new class definition version key - cmK = class.BuildClassDefinitionVersion() - // new class definition version value - cmV string - // newly created class names - classNames []string + classNames []string + componentClassGroups []v1alpha1.ComponentClassGroup ) if o.File != "" { @@ -155,105 +147,126 @@ func (o *CreateOptions) run() error { if err != nil { return err } - newClasses, err := class.ParseComponentClasses(map[string]string{cmK: string(data)}) + if err := yaml.Unmarshal(data, &componentClassGroups); err != nil { + return err + } + classDefinition := v1alpha1.ComponentClassDefinition{ + Spec: v1alpha1.ComponentClassDefinitionSpec{Groups: componentClassGroups}, + } + newClasses, err := class.ParseComponentClasses(classDefinition) if err != nil { return err } for name, cls := range newClasses { - if _, ok = families[cls.Family]; !ok { - return fmt.Errorf("family %s is not found", cls.Family) + if _, ok = constraints[cls.ResourceConstraintRef]; !ok { + return fmt.Errorf("resource constraint %s is not found", cls.ResourceConstraintRef) } if _, ok = classes[name]; ok { return fmt.Errorf("class name conflicted %s", name) } classNames = append(classNames, name) } - cmV = string(data) } else { if _, ok = classes[o.ClassName]; ok { return fmt.Errorf("class name conflicted %s", o.ClassName) } - if _, ok = families[o.ClassFamily]; !ok { - return fmt.Errorf("family %s is not found", o.ClassFamily) + if _, ok = constraints[o.Constraint]; !ok { + return fmt.Errorf("resource constraint %s is not found", o.Constraint) } - def, err := o.buildClassFamilyDef() + cls, err := o.buildClass() if err != nil { return err } - data, err := yaml.Marshal([]*class.ComponentClassFamilyDef{def}) - if err != nil { - return err + componentClassGroups = []v1alpha1.ComponentClassGroup{ + { + ResourceConstraintRef: o.Constraint, + Series: []v1alpha1.ComponentClassSeries{ + { + Classes: []v1alpha1.ComponentClass{*cls}, + }, + }, + }, } - cmV = string(data) classNames = append(classNames, o.ClassName) } - cmName := class.GetCustomClassConfigMapName(o.ClusterDefRef, o.ComponentType) - cm, err := o.client.CoreV1().ConfigMaps(CustomClassNamespace).Get(context.TODO(), cmName, metav1.GetOptions{}) + objName := class.GetCustomClassObjectName(o.ClusterDefRef, o.ComponentType) + obj, err := o.dynamic.Resource(types.ComponentClassDefinitionGVR()).Get(context.TODO(), objName, metav1.GetOptions{}) if err != nil && !errors.IsNotFound(err) { return err } + var classDefinition v1alpha1.ComponentClassDefinition if err == nil { - cm.Data[cmK] = cmV - if _, err = o.client.CoreV1().ConfigMaps(cm.GetNamespace()).Update(context.TODO(), cm, metav1.UpdateOptions{}); err != nil { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &classDefinition); err != nil { + return err + } + classDefinition.Spec.Groups = append(classDefinition.Spec.Groups, componentClassGroups...) + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&classDefinition) + if err != nil { + return err + } + if _, err = o.dynamic.Resource(types.ComponentClassDefinitionGVR()).Update( + context.Background(), &unstructured.Unstructured{Object: unstructuredMap}, metav1.UpdateOptions{}); err != nil { return err } } else { - cm = &corev1.ConfigMap{ + gvr := types.ComponentClassDefinitionGVR() + classDefinition = v1alpha1.ComponentClassDefinition{ + TypeMeta: metav1.TypeMeta{ + Kind: types.KindComponentClassDefinition, + APIVersion: gvr.Group + "/" + gvr.Version, + }, ObjectMeta: metav1.ObjectMeta{ - Name: class.GetCustomClassConfigMapName(o.ClusterDefRef, o.ComponentType), - Namespace: CustomClassNamespace, + Name: class.GetCustomClassObjectName(o.ClusterDefRef, o.ComponentType), Labels: map[string]string{ constant.ClusterDefLabelKey: o.ClusterDefRef, types.ClassProviderLabelKey: "user", - types.ClassLevelLabelKey: "component", constant.KBAppComponentDefRefLabelKey: o.ComponentType, }, }, - Data: map[string]string{cmK: cmV}, + Spec: v1alpha1.ComponentClassDefinitionSpec{ + Groups: componentClassGroups, + }, + } + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&classDefinition) + if err != nil { + return err } - if _, err = o.client.CoreV1().ConfigMaps(CustomClassNamespace).Create(context.TODO(), cm, metav1.CreateOptions{}); err != nil { + if _, err = o.dynamic.Resource(types.ComponentClassDefinitionGVR()).Create( + context.Background(), &unstructured.Unstructured{Object: unstructuredMap}, metav1.CreateOptions{}); err != nil { return err } } - _, _ = fmt.Fprintf(o.Out, "Successfully created class [%s].", strings.Join(classNames, ",")) + _, _ = fmt.Fprintf(o.Out, "Successfully created class [%s].\n", strings.Join(classNames, ",")) return nil } -func (o *CreateOptions) buildClassFamilyDef() (*class.ComponentClassFamilyDef, error) { - clsDef := class.ComponentClassDef{Name: o.ClassName, CPU: o.CPU, Memory: o.Memory} - for _, disk := range o.Storage { - kvs := strings.Split(disk, ",") - def := class.DiskDef{} +func (o *CreateOptions) buildClass() (*v1alpha1.ComponentClass, error) { + cls := v1alpha1.ComponentClass{Name: o.ClassName, CPU: resource.MustParse(o.CPU), Memory: resource.MustParse(o.Memory)} + for _, item := range o.Storage { + kvs := strings.Split(item, ",") + volume := v1alpha1.Volume{} for _, kv := range kvs { parts := strings.Split(kv, "=") if len(parts) != 2 { - return nil, fmt.Errorf("invalid storage disk: %s", disk) + return nil, fmt.Errorf("invalid storage item: %s", item) } switch parts[0] { case "name": - def.Name = parts[1] + volume.Name = parts[1] case "size": - def.Size = parts[1] + volume.Size = resource.MustParse(parts[1]) case "class": - def.Class = parts[1] + volume.StorageClassName = &parts[1] default: - return nil, fmt.Errorf("invalid storage disk: %s", disk) + return nil, fmt.Errorf("invalid storage item: %s", item) } } - // validate disk size - if _, err := resource.ParseQuantity(def.Size); err != nil { - return nil, fmt.Errorf("invalid disk size: %s", disk) - } - if def.Name == "" { - return nil, fmt.Errorf("invalid disk name: %s", disk) + if volume.Name == "" { + return nil, fmt.Errorf("invalid item name: %s", item) } - clsDef.Storage = append(clsDef.Storage, def) - } - def := &class.ComponentClassFamilyDef{ - Family: o.ClassFamily, - Series: []class.ComponentClassSeriesDef{{Classes: []class.ComponentClassDef{clsDef}}}, + cls.Volumes = append(cls.Volumes, volume) } - return def, nil + return &cls, nil } diff --git a/internal/cli/cmd/class/create_test.go b/internal/cli/cmd/class/create_test.go index f529487de..e79cf7b88 100644 --- a/internal/cli/cmd/class/create_test.go +++ b/internal/cli/cmd/class/create_test.go @@ -18,19 +18,12 @@ package class import ( "bytes" - "net/http" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/cli-runtime/pkg/resource" "k8s.io/client-go/kubernetes/scheme" - clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -40,7 +33,6 @@ import ( var _ = Describe("create", func() { var ( o *CreateOptions - cd *appsv1alpha1.ClusterDefinition out *bytes.Buffer tf *cmdtesting.TestFactory streams genericclioptions.IOStreams @@ -53,42 +45,15 @@ var _ = Describe("create", func() { } BeforeEach(func() { - cd = testing.FakeClusterDef() - streams, _, out, _ = genericclioptions.NewTestIOStreams() tf = testing.NewTestFactory(namespace) - - codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) - httpResp := func(obj runtime.Object) *http.Response { - return &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, obj)} - } - cms := testing.FakeComponentClassDef(cd, classDef) - - resources := map[string]runtime.Object{ - "/api/v1/configmaps": cms, - } - - tf.UnstructuredClient = &clientfake.RESTClient{ - GroupVersion: schema.GroupVersion{Group: "core", Version: "v1"}, - NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, - Client: clientfake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { - if req.Method == "POST" { - return httpResp(&corev1.ConfigMap{}), nil - } - resource, ok := resources[req.URL.Path] - if !ok { - return nil, errors.NewNotFound(schema.GroupResource{}, req.URL.Path) - } - return httpResp(resource), nil - }), - } - tf.Client = tf.UnstructuredClient - tf.FakeDynamicClient = testing.FakeDynamicClient(&generalClassFamily, &memoryOptimizedClassFamily, cd) + _ = appsv1alpha1.AddToScheme(scheme.Scheme) + tf.FakeDynamicClient = testing.FakeDynamicClient(&classDef, &generalResourceConstraint, &memoryOptimizedResourceConstraint) o = &CreateOptions{ Factory: tf, IOStreams: streams, - ClusterDefRef: cd.Name, + ClusterDefRef: "apecloud-mysql", ComponentType: testing.ComponentDefName, } Expect(o.complete(tf)).ShouldNot(HaveOccurred()) @@ -106,7 +71,7 @@ var _ = Describe("create", func() { Context("with resource arguments", func() { It("should fail if required arguments is missing", func() { - o.ClassFamily = generalClassFamily.Name + o.Constraint = generalResourceConstraint.Name fillResources(o, "", "48Gi", nil) Expect(o.validate([]string{"general-12c48g"})).Should(HaveOccurred()) fillResources(o, "12", "", nil) @@ -116,7 +81,7 @@ var _ = Describe("create", func() { }) It("should succeed with required arguments", func() { - o.ClassFamily = generalClassFamily.Name + o.Constraint = generalResourceConstraint.Name fillResources(o, "12", "48Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) Expect(o.validate([]string{"general-12c48g"})).ShouldNot(HaveOccurred()) Expect(o.run()).ShouldNot(HaveOccurred()) diff --git a/internal/cli/cmd/class/list.go b/internal/cli/cmd/class/list.go index aae922ea2..eee764c6c 100644 --- a/internal/cli/cmd/class/list.go +++ b/internal/cli/cmd/class/list.go @@ -23,10 +23,11 @@ import ( "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/client-go/kubernetes" + "k8s.io/client-go/dynamic" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/class" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/util" @@ -35,7 +36,7 @@ import ( type ListOptions struct { ClusterDefRef string Factory cmdutil.Factory - client *kubernetes.Clientset + dynamic dynamic.Interface genericclioptions.IOStreams } @@ -55,59 +56,56 @@ func NewListCommand(f cmdutil.Factory, streams genericclioptions.IOStreams) *cob util.CheckErr(o.run()) }, } - cmd.Flags().StringVar(&o.ClusterDefRef, "cluster-definition", "", "Specify cluster definition, run \"kbcli cluster-definition list\" to show all available cluster definition") + cmd.Flags().StringVar(&o.ClusterDefRef, "cluster-definition", "", "Specify cluster definition, run \"kbcli clusterdefinition list\" to show all available cluster definition") util.CheckErr(cmd.MarkFlagRequired("cluster-definition")) return cmd } func (o *ListOptions) complete(f cmdutil.Factory) error { var err error - o.client, err = f.KubernetesClientSet() - if err != nil { - return err - } + o.dynamic, err = f.DynamicClient() return err } func (o *ListOptions) run() error { - componentClasses, err := class.GetClasses(o.client, o.ClusterDefRef) + componentClasses, err := class.ListClassesByClusterDefinition(o.dynamic, o.ClusterDefRef) if err != nil { return err } - familyClassMap := make(map[string]map[string][]*class.ComponentClass) + constraintClassMap := make(map[string]map[string][]*appsv1alpha1.ComponentClassInstance) for compName, items := range componentClasses { for _, item := range items { - if _, ok := familyClassMap[item.Family]; !ok { - familyClassMap[item.Family] = make(map[string][]*class.ComponentClass) + if _, ok := constraintClassMap[item.ResourceConstraintRef]; !ok { + constraintClassMap[item.ResourceConstraintRef] = make(map[string][]*appsv1alpha1.ComponentClassInstance) } - familyClassMap[item.Family][compName] = append(familyClassMap[item.Family][compName], item) + constraintClassMap[item.ResourceConstraintRef][compName] = append(constraintClassMap[item.ResourceConstraintRef][compName], item) } } - var familyNames []string - for name := range familyClassMap { - familyNames = append(familyNames, name) + var constraintNames []string + for name := range constraintClassMap { + constraintNames = append(constraintNames, name) } - sort.Strings(familyNames) - for _, family := range familyNames { - for compName, classes := range familyClassMap[family] { - o.printClassFamily(family, compName, classes) + sort.Strings(constraintNames) + for _, constraintName := range constraintNames { + for compName, classes := range constraintClassMap[constraintName] { + o.printClass(constraintName, compName, classes) } _, _ = fmt.Fprint(o.Out, "\n") } return nil } -func (o *ListOptions) printClassFamily(family string, compName string, classes []*class.ComponentClass) { +func (o *ListOptions) printClass(constraintName string, compName string, classes []*appsv1alpha1.ComponentClassInstance) { tbl := printer.NewTablePrinter(o.Out) - _, _ = fmt.Fprintf(o.Out, "\nFamily %s:\n", family) + _, _ = fmt.Fprintf(o.Out, "\nConstraint %s:\n", constraintName) tbl.SetHeader("COMPONENT", "CLASS", "CPU", "MEMORY", "STORAGE") sort.Sort(class.ByClassCPUAndMemory(classes)) - for _, class := range classes { - var disks []string - for _, disk := range class.Storage { - disks = append(disks, disk.String()) + for _, cls := range classes { + var volumes []string + for _, volume := range cls.Volumes { + volumes = append(volumes, fmt.Sprintf("%s=%s", volume.Name, volume.Size.String())) } - tbl.AddRow(compName, class.Name, class.CPU.String(), class.Memory.String(), strings.Join(disks, ",")) + tbl.AddRow(compName, cls.Name, cls.CPU.String(), cls.Memory.String(), strings.Join(volumes, ",")) } tbl.Print() } diff --git a/internal/cli/cmd/class/list_test.go b/internal/cli/cmd/class/list_test.go index c872514bf..be2c3b773 100644 --- a/internal/cli/cmd/class/list_test.go +++ b/internal/cli/cmd/class/list_test.go @@ -18,18 +18,12 @@ package class import ( "bytes" - "net/http" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/cli-runtime/pkg/resource" "k8s.io/client-go/kubernetes/scheme" - clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -38,34 +32,16 @@ import ( var _ = Describe("list", func() { var ( - cd *appsv1alpha1.ClusterDefinition out *bytes.Buffer tf *cmdtesting.TestFactory streams genericclioptions.IOStreams ) BeforeEach(func() { - cd = testing.FakeClusterDef() - streams, _, out, _ = genericclioptions.NewTestIOStreams() tf = testing.NewTestFactory(namespace) - - _ = corev1.AddToScheme(scheme.Scheme) - codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) - httpResp := func(obj runtime.Object) *http.Response { - return &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, obj)} - } - - tf.UnstructuredClient = &clientfake.RESTClient{ - GroupVersion: schema.GroupVersion{Group: "core", Version: "v1"}, - NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, - Client: clientfake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { - return map[string]*http.Response{ - "/api/v1/configmaps": httpResp(testing.FakeComponentClassDef(cd, classDef)), - }[req.URL.Path], nil - }), - } - tf.Client = tf.UnstructuredClient + _ = appsv1alpha1.AddToScheme(scheme.Scheme) + tf.FakeDynamicClient = testing.FakeDynamicClient(&classDef) }) AfterEach(func() { @@ -75,10 +51,10 @@ var _ = Describe("list", func() { It("should succeed", func() { cmd := NewListCommand(tf, streams) Expect(cmd).ShouldNot(BeNil()) - cmd.Run(cmd, []string{"--cluster-definition", cd.GetName()}) + _ = cmd.Flags().Set("cluster-definition", "apecloud-mysql") + cmd.Run(cmd, []string{}) Expect(out.String()).To(ContainSubstring("general-1c1g")) - Expect(out.String()).To(ContainSubstring(testing.ComponentDefName)) - Expect(out.String()).To(ContainSubstring(generalClassFamily.Name)) - Expect(out.String()).To(ContainSubstring(memoryOptimizedClassFamily.Name)) + Expect(out.String()).To(ContainSubstring("mysql")) + Expect(out.String()).To(ContainSubstring(generalResourceConstraint.Name)) }) }) diff --git a/internal/cli/cmd/class/suite_test.go b/internal/cli/cmd/class/suite_test.go index ef9544a90..907e4eacf 100644 --- a/internal/cli/cmd/class/suite_test.go +++ b/internal/cli/cmd/class/suite_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes/scheme" @@ -30,35 +31,35 @@ import ( ) const ( - namespace = "test" - testDefaultClassDefsPath = "../../testing/testdata/class.yaml" - testCustomClassDefsPath = "../../testing/testdata/custom_class.yaml" - testGeneralClassFamilyPath = "../../testing/testdata/classfamily-general.yaml" - testMemoryOptimizedClassFamilyPath = "../../testing/testdata/classfamily-memory-optimized.yaml" + namespace = "test" + testDefaultClassDefsPath = "../../testing/testdata/class.yaml" + testCustomClassDefsPath = "../../testing/testdata/custom_class.yaml" + testGeneralResourceConstraintPath = "../../testing/testdata/resource-constraint-general.yaml" + testMemoryOptimizedResourceConstraintPath = "../../testing/testdata/resource-constraint-memory-optimized.yaml" ) var ( - classDef []byte - generalFamilyDef []byte - memoryOptimizedFamilyDef []byte - generalClassFamily appsv1alpha1.ClassFamily - memoryOptimizedClassFamily appsv1alpha1.ClassFamily + classDef appsv1alpha1.ComponentClassDefinition + generalResourceConstraint appsv1alpha1.ComponentResourceConstraint + memoryOptimizedResourceConstraint appsv1alpha1.ComponentResourceConstraint ) var _ = BeforeSuite(func() { var err error - classDef, err = os.ReadFile(testDefaultClassDefsPath) + classDefBytes, err := os.ReadFile(testDefaultClassDefsPath) + Expect(err).ShouldNot(HaveOccurred()) + err = yaml.Unmarshal(classDefBytes, &classDef) Expect(err).ShouldNot(HaveOccurred()) - generalFamilyDef, err = os.ReadFile(testGeneralClassFamilyPath) + generalResourceConstraintBytes, err := os.ReadFile(testGeneralResourceConstraintPath) Expect(err).ShouldNot(HaveOccurred()) - err = yaml.Unmarshal(generalFamilyDef, &generalClassFamily) + err = yaml.Unmarshal(generalResourceConstraintBytes, &generalResourceConstraint) Expect(err).ShouldNot(HaveOccurred()) - memoryOptimizedFamilyDef, err = os.ReadFile(testMemoryOptimizedClassFamilyPath) + memoryOptimizedResourceConstraintBytes, err := os.ReadFile(testMemoryOptimizedResourceConstraintPath) Expect(err).ShouldNot(HaveOccurred()) - err = yaml.Unmarshal(memoryOptimizedFamilyDef, &memoryOptimizedClassFamily) + err = yaml.Unmarshal(memoryOptimizedResourceConstraintBytes, &memoryOptimizedResourceConstraint) Expect(err).ShouldNot(HaveOccurred()) err = appsv1alpha1.AddToScheme(scheme.Scheme) diff --git a/internal/cli/cmd/class/template.go b/internal/cli/cmd/class/template.go index 3b25e59aa..9e288b86f 100644 --- a/internal/cli/cmd/class/template.go +++ b/internal/cli/cmd/class/template.go @@ -27,29 +27,27 @@ import ( ) const ComponentClassTemplate = ` -- # class family name, such as general, memory-optimized, cpu-optimized etc. - family: kb-class-family-general - # class schema template, you can set default resource values here +- resourceConstraintRef: kb-resource-constraint-general + # class template, you can declare variables and set default values here template: | cpu: "{{ or .cpu 1 }}" memory: "{{ or .memory 4 }}Gi" - storage: + volumes: - name: data size: "{{ or .dataStorageSize 10 }}Gi" - name: log size: "{{ or .logStorageSize 1 }}Gi" - # class schema template variables + # template variables used to define classes vars: [cpu, memory, dataStorageSize, logStorageSize] series: - - # class name generator, you can reference variables in class schema template - # it's also ok to define static class name in following class definitions - name: "custom-{{ .cpu }}c{{ .memory }}g" + - # class naming template, you can reference variables in class template + # it's also ok to define static class name in the following class definitions + namingTemplate: "custom-{{ .cpu }}c{{ .memory }}g" # class definitions, we support two kinds of class definitions: - # 1. define arguments for class schema variables, class schema will be dynamically generated - # 2. statically define complete class schema + # 1. define values for template variables and the full class definition will be dynamically rendered + # 2. statically define the complete class classes: - # arguments for dynamically generated class - args: [1, 1, 100, 10] ` diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 49a725e8f..df74761be 100644 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -76,8 +76,8 @@ var clusterCreateExample = templates.Examples(` # Create a cluster and set cpu to 1 core, memory to 1Gi, storage size to 20Gi and replicas to 3 kbcli cluster create mycluster --cluster-definition apecloud-mysql --set cpu=1,memory=1Gi,storage=20Gi,replicas=3 - # Create a cluster and set class to general-1c4g - kbcli cluster create myclsuter --cluster-definition apecloud-mysql --set class=general-1c4g + # Create a cluster and set the class to general-1c1g, valid classes can be found by executing the command "kbcli class list --cluster-definition=" + kbcli cluster create myclsuter --cluster-definition apecloud-mysql --set class=general-1c1g # Create a cluster with replicationSet workloadType and set switchPolicy to Noop kbcli cluster create myclsuter --cluster-definition postgresql --set switchPolicy=Noop @@ -212,7 +212,7 @@ func setBackup(o *CreateOptions, components []map[string]interface{}) error { func (o *CreateOptions) Validate() error { if o.ClusterDefRef == "" { - return fmt.Errorf("a valid cluster definition is needed, use --cluster-definition to specify one, run \"kbcli cluster-definition list\" to show all cluster definition") + return fmt.Errorf("a valid cluster definition is needed, use --cluster-definition to specify one, run \"kbcli clusterdefinition list\" to show all cluster definition") } if o.TerminationPolicy == "" { @@ -311,38 +311,10 @@ func (o *CreateOptions) buildComponents() ([]map[string]interface{}, error) { } components = append(components, comp) } - - if err = o.buildClassMappings(componentObjs, compSets); err != nil { - return nil, err - } } return components, nil } -func (o *CreateOptions) buildClassMappings(components []*appsv1alpha1.ClusterComponentSpec, setsMap map[string]map[setKey]string) error { - classMappings := make(map[string]string) - for _, comp := range components { - sets, ok := setsMap[comp.ComponentDefRef] - if !ok { - continue - } - class, ok := sets[keyClass] - if !ok { - continue - } - classMappings[comp.Name] = class - } - bytes, err := json.Marshal(classMappings) - if err != nil { - return err - } - if o.Annotations == nil { - o.Annotations = make(map[string]string) - } - o.Annotations[types.ComponentClassAnnotationKey] = string(bytes) - return nil -} - // MultipleSourceComponents get component data from multiple source, such as stdin, URI and local file func MultipleSourceComponents(fileName string, in io.Reader) ([]byte, error) { var data io.Reader @@ -385,7 +357,7 @@ func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra cmd.Flags().StringVar(&o.ClusterDefRef, "cluster-definition", "", "Specify cluster definition, run \"kbcli cd list\" to show all available cluster definitions") cmd.Flags().StringVar(&o.ClusterVersionRef, "cluster-version", "", "Specify cluster version, run \"kbcli cv list\" to show all available cluster versions, use the latest version if not specified") cmd.Flags().StringVarP(&o.SetFile, "set-file", "f", "", "Use yaml file, URL, or stdin to set the cluster resource") - cmd.Flags().StringArrayVar(&o.Values, "set", []string{}, "Set the cluster resource including cpu, memory, replicas and storage, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi)") + cmd.Flags().StringArrayVar(&o.Values, "set", []string{}, "Set the cluster resource including cpu, memory, replicas and storage, or you can just specify the class, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi or --set class=general-1c1g)") cmd.Flags().StringVar(&o.Backup, "backup", "", "Set a source backup to restore data") // add updatable flags @@ -507,19 +479,26 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map } replicas := int32(setReplicas) - resourceList := corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(getVal(keyCPU, sets)), - corev1.ResourceMemory: resource.MustParse(getVal(keyMemory, sets)), - } compObj := &appsv1alpha1.ClusterComponentSpec{ Name: c.Name, ComponentDefRef: c.Name, Replicas: replicas, - Resources: corev1.ResourceRequirements{ + } + + // class has higher priority than other resource related parameters + className := getVal(keyClass, sets) + if className != "" { + compObj.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: className} + } else { + resourceList := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(getVal(keyCPU, sets)), + corev1.ResourceMemory: resource.MustParse(getVal(keyMemory, sets)), + } + compObj.Resources = corev1.ResourceRequirements{ Requests: resourceList, Limits: resourceList, - }, - VolumeClaimTemplates: []appsv1alpha1.ClusterComponentVolumeClaimTemplate{{ + } + compObj.VolumeClaimTemplates = []appsv1alpha1.ClusterComponentVolumeClaimTemplate{{ Name: "data", Spec: appsv1alpha1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{ @@ -531,7 +510,7 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map }, }, }, - }}, + }} } if err := buildSwitchPolicy(&c, compObj, sets); err != nil { return nil, err diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 73c488563..8eb1b92b6 100644 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -347,6 +347,9 @@ func NewUpgradeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr var verticalScalingExample = templates.Examples(` # scale the computing resources of specified components, separate with commas when more than one kbcli cluster vscale --components= --cpu=500m --memory=500Mi + + # scale the computing resources of specified components by class, available classes can be get by executing the command "kbcli class list --cluster-definition " + kbcli cluster vscale --components= --set class=general-1c4g `) // NewVerticalScalingCmd creates a vertical scaling command diff --git a/internal/cli/create/template/cluster_operations_template.cue b/internal/cli/create/template/cluster_operations_template.cue index eb02b11e9..518814cc5 100644 --- a/internal/cli/create/template/cluster_operations_template.cue +++ b/internal/cli/create/template/cluster_operations_template.cue @@ -24,6 +24,7 @@ options: { componentNames: [...string] cpu: string memory: string + class: string replicas: int storage: string vctNames: [...string] @@ -88,6 +89,9 @@ content: { if options.type == "VerticalScaling" { verticalScaling: [ for _, cName in options.componentNames { componentName: cName + if options.class != "" { + class: options.class + } requests: { if options.memory != "" { memory: options.memory diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index c41a9c7f7..e2f2abcbb 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -36,18 +36,16 @@ import ( ) const ( - ClusterName = "fake-cluster-name" - Namespace = "fake-namespace" - ClusterVersionName = "fake-cluster-version" - ClusterDefName = "fake-cluster-definition" - ComponentName = "fake-component-name" - ComponentDefName = "fake-component-type" - NodeName = "fake-node-name" - SecretName = "fake-secret-conn-credential" - StorageClassName = "fake-storage-class" - PVCName = "fake-pvc" - GeneralClassFamily = "kb-class-family-general" - MemoryOptimizedClassFamily = "kb-class-family-memory-optimized" + ClusterName = "fake-cluster-name" + Namespace = "fake-namespace" + ClusterVersionName = "fake-cluster-version" + ClusterDefName = "fake-cluster-definition" + ComponentName = "fake-component-name" + ComponentDefName = "fake-component-type" + NodeName = "fake-node-name" + SecretName = "fake-secret-conn-credential" + StorageClassName = "fake-storage-class" + PVCName = "fake-pvc" KubeBlocksRepoName = "fake-kubeblocks-repo" KubeBlocksChartName = "fake-kubeblocks" @@ -259,7 +257,6 @@ func FakeComponentClassDef(clusterDef *appsv1alpha1.ClusterDefinition, def []byt cm := &corev1.ConfigMap{} cm.Name = fmt.Sprintf("fake-kubeblocks-classes-%s", ComponentName) cm.SetLabels(map[string]string{ - types.ClassLevelLabelKey: "component", constant.KBAppComponentDefRefLabelKey: ComponentDefName, types.ClassProviderLabelKey: "kubeblocks", constant.ClusterDefLabelKey: clusterDef.Name, diff --git a/internal/cli/testing/testdata/class.yaml b/internal/cli/testing/testdata/class.yaml index 9e1c965fd..7ad0f25ca 100644 --- a/internal/cli/testing/testdata/class.yaml +++ b/internal/cli/testing/testdata/class.yaml @@ -1,61 +1,81 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ComponentClassDefinition +metadata: + name: kb.classes.default.apecloud-mysql.mysql + labels: + class.kubeblocks.io/provider: kubeblocks + apps.kubeblocks.io/component-def-ref: mysql + clusterdefinition.kubeblocks.io/name: apecloud-mysql +spec: + groups: + - # resource constraint name, such as general, memory-optimized, cpu-optimized etc. + resourceConstraintRef: kb-resource-constraint-general + # class schema template, you can set default resource values here + template: | + cpu: "{{ or .cpu 1 }}" + memory: "{{ or .memory 4 }}Gi" + volumes: + - name: data + size: "{{ or .dataStorageSize 10 }}Gi" + - name: log + size: "{{ or .logStorageSize 1 }}Gi" + # class schema template variables + vars: [ cpu, memory, dataStorageSize, logStorageSize ] + series: + - # class name generator, you can reference variables in class schema template + # it's also ok to define static class name in following class definitions + namingTemplate: "general-{{ .cpu }}c{{ .memory }}g" -- # class family name, such as general, memory-optimized, cpu-optimized etc. - family: kb-class-family-general - # class schema template, you can set default resource values here - template: | - cpu: "{{ or .cpu 1 }}" - memory: "{{ or .memory 4 }}Gi" - storage: - - name: data - size: "{{ or .dataStorageSize 10 }}Gi" - - name: log - size: "{{ or .logStorageSize 1 }}Gi" - # class schema template variables - vars: [cpu, memory, dataStorageSize, logStorageSize] - series: - - # class name generator, you can reference variables in class schema template - # it's also ok to define static class name in following class definitions - name: "general-{{ .cpu }}c{{ .memory }}g" - - # class definitions, we support two kinds of class definitions: - # 1. define arguments for class schema variables, class schema will be dynamically generated - # 2. statically define complete class schema - classes: - - args: [1, 1, 100, 10] - - args: [2, 2, 100, 10] - - args: [2, 4, 100, 10] - - args: [2, 8, 100, 10] - - args: [4, 16, 100, 10] - - args: [8, 32, 100, 10] - - args: [16, 64, 200, 10] - - args: [32, 128, 200, 10] - - args: [64, 256, 200, 10] - - args: [128, 512, 200, 10] + # class definitions, we support two kinds of class definitions: + # 1. define arguments for class schema variables, class schema will be dynamically generated + # 2. statically define complete class schema + classes: + - args: [ "1", "1", "10", "1" ] + - args: [ "2", "2", "10", "1" ] + - args: [ "2", "4", "10", "1" ] + - args: [ "2", "8", "10", "1" ] + - args: [ "4", "16", "10", "1" ] + - args: [ "8", "32", "10", "1" ] + - args: [ "16", "64", "20", "1" ] + - args: [ "32", "128", "20", "1" ] + - args: [ "64", "256", "20", "1" ] + - args: [ "128", "512", "20", "1" ] -- family: kb-class-family-memory-optimized - template: | - cpu: "{{ or .cpu 1 }}" - memory: "{{ or .memory 8 }}Gi" + - resourceConstraintRef: kb-resource-constraint-memory-optimized + template: | + cpu: "{{ or .cpu 1 }}" + memory: "{{ or .memory 8 }}Gi" + volumes: + - name: data + size: "{{ or .dataStorageSize 10 }}Gi" + - name: log + size: "{{ or .logStorageSize 1 }}Gi" + vars: [ cpu, memory, dataStorageSize, logStorageSize ] + series: + - namingTemplate: "mo-{{ .cpu }}c{{ .memory }}g" + classes: + - args: [ "2", "16", "10", "1" ] + - args: [ "4", "32", "10", "1" ] + - args: [ "8", "64", "10", "1" ] + - args: [ "12", "96", "10", "1" ] + - args: [ "24", "192", "20", "1" ] + - args: [ "48", "384", "20", "1" ] + - args: [ "2", "32", "10", "1" ] + - args: [ "4", "64", "10", "1" ] + - args: [ "8", "128", "10", "1" ] + - args: [ "16", "256", "10", "1" ] + - args: [ "32", "512", "20", "1" ] + - args: [ "48", "768", "20", "1" ] + - args: [ "64", "1024", "20", "1" ] + - args: [ "128", "2048", "20", "1" ] +status: + classes: + - name: general-1c1g + resourceConstraintRef: kb-resource-constraint-general + cpu: 1 + memory: 1Gi storage: - name: data - size: "{{ or .dataStorageSize 10 }}Gi" + size: 100Gi - name: log - size: "{{ or .logStorageSize 1 }}Gi" - vars: [cpu, memory, dataStorageSize, logStorageSize] - series: - - name: "mo-{{ .cpu }}c{{ .memory }}g" - classes: - - args: [2, 16, 100, 10] - - args: [4, 32, 100, 10] - - args: [8, 64, 100, 10] - - args: [12, 96, 100, 10] - - args: [24, 192, 200, 10] - - args: [48, 384, 200, 10] - - args: [2, 32, 100, 10] - - args: [4, 64, 100, 10] - - args: [8, 128, 100, 10] - - args: [16, 256, 100, 10] - - args: [32, 512, 200, 10] - - args: [48, 768, 200, 10] - - args: [64, 1024, 200, 10] - - args: [128, 2048, 200, 10] + size: 10Gi \ No newline at end of file diff --git a/internal/cli/testing/testdata/custom_class.yaml b/internal/cli/testing/testdata/custom_class.yaml index b51a653ea..c227ce5cc 100644 --- a/internal/cli/testing/testdata/custom_class.yaml +++ b/internal/cli/testing/testdata/custom_class.yaml @@ -1,33 +1,33 @@ -- family: kb-class-family-general - template: | - cpu: "{{ or .cpu 1 }}" - memory: "{{ or .memory 4 }}Gi" - storage: - - name: data - size: "{{ or .dataStorageSize 10 }}Gi" - - name: log - size: "{{ or .logStorageSize 1 }}Gi" - vars: [cpu, memory, dataStorageSize, logStorageSize] - series: - - name: "custom-{{ .cpu }}c{{ .memory }}g" - classes: - - args: [1, 1, 100, 10] - - name: custom-200c400g - cpu: 200 - memory: 400Gi + - resourceConstraintRef: kb-resource-constraint-general + template: | + cpu: "{{ or .cpu 1 }}" + memory: "{{ or .memory 4 }}Gi" + volumes: + - name: data + size: "{{ or .dataStorageSize 10 }}Gi" + - name: log + size: "{{ or .logStorageSize 1 }}Gi" + vars: [cpu, memory, dataStorageSize, logStorageSize] + series: + - namingTemplate: "custom-{{ .cpu }}c{{ .memory }}g" + classes: + - args: ["1", "1", "100", "10"] + - name: custom-200c400g + cpu: 200 + memory: 400Gi -- family: kb-class-family-memory-optimized - template: | - cpu: "{{ or .cpu 1 }}" - memory: "{{ or .memory 4 }}Gi" - storage: - - name: data - size: "{{ or .dataStorageSize 10 }}Gi" - - name: log - size: "{{ or .logStorageSize 1 }}Gi" - vars: [cpu, memory, dataStorageSize, logStorageSize] - series: - - name: "custom-{{ .cpu }}c{{ .memory }}g" - classes: - - args: [1, 32, 100, 10] - - args: [2, 64, 100, 10] + - resourceConstraintRef: kb-resource-constraint-memory-optimized + template: | + cpu: "{{ or .cpu 1 }}" + memory: "{{ or .memory 4 }}Gi" + volumes: + - name: data + size: "{{ or .dataStorageSize 10 }}Gi" + - name: log + size: "{{ or .logStorageSize 1 }}Gi" + vars: [cpu, memory, dataStorageSize, logStorageSize] + series: + - namingTemplate: "custom-{{ .cpu }}c{{ .memory }}g" + classes: + - args: ["1", "32", "100", "10"] + - args: ["2", "64", "100", "10"] diff --git a/internal/cli/testing/testdata/classfamily-general.yaml b/internal/cli/testing/testdata/resource-constraint-general.yaml similarity index 68% rename from internal/cli/testing/testdata/classfamily-general.yaml rename to internal/cli/testing/testdata/resource-constraint-general.yaml index 0ba574d26..a6f300221 100644 --- a/internal/cli/testing/testdata/classfamily-general.yaml +++ b/internal/cli/testing/testdata/resource-constraint-general.yaml @@ -1,11 +1,11 @@ apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClassFamily +kind: ComponentResourceConstraint metadata: - name: kb-class-family-general + name: kb-resource-constraint-general labels: - classfamily.kubeblocks.io/provider: kubeblocks + resourceconstraint.kubeblocks.io/provider: kubeblocks spec: - models: + constraints: - cpu: min: 0.5 max: 2 diff --git a/internal/cli/testing/testdata/classfamily-memory-optimized.yaml b/internal/cli/testing/testdata/resource-constraint-memory-optimized.yaml similarity index 59% rename from internal/cli/testing/testdata/classfamily-memory-optimized.yaml rename to internal/cli/testing/testdata/resource-constraint-memory-optimized.yaml index b02f488b9..a0acf512a 100644 --- a/internal/cli/testing/testdata/classfamily-memory-optimized.yaml +++ b/internal/cli/testing/testdata/resource-constraint-memory-optimized.yaml @@ -1,11 +1,11 @@ apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClassFamily +kind: ComponentResourceConstraint metadata: - name: kb-class-family-memory-optimized + name: kb-resource-constraint-memory-optimized labels: - classfamily.kubeblocks.io/provider: kubeblocks + resourceconstraint.kubeblocks.io/provider: kubeblocks spec: - models: + constraints: - cpu: slots: [2, 4, 8, 12, 24, 48] memory: diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index ad7b41b94..3b4c85079 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -65,22 +65,24 @@ const ( // Apps API group const ( - AppsAPIGroup = "apps.kubeblocks.io" - AppsAPIVersion = "v1alpha1" - ResourceClusters = "clusters" - ResourceClusterDefs = "clusterdefinitions" - ResourceClusterVersions = "clusterversions" - ResourceOpsRequests = "opsrequests" - ResourceConfigConstraintVersions = "configconstraints" - ResourceClassFamily = "classfamilies" - KindCluster = "Cluster" - KindClusterDef = "ClusterDefinition" - KindClusterVersion = "ClusterVersion" - KindConfigConstraint = "ConfigConstraint" - KindBackup = "Backup" - KindRestoreJob = "RestoreJob" - KindBackupPolicy = "BackupPolicy" - KindOps = "OpsRequest" + AppsAPIGroup = "apps.kubeblocks.io" + AppsAPIVersion = "v1alpha1" + ResourceClusters = "clusters" + ResourceClusterDefs = "clusterdefinitions" + ResourceClusterVersions = "clusterversions" + ResourceOpsRequests = "opsrequests" + ResourceConfigConstraintVersions = "configconstraints" + ResourceComponentResourceConstraint = "componentresourceconstraints" + ResourceComponentClassDefinition = "componentclassdefinitions" + KindCluster = "Cluster" + KindComponentClassDefinition = "ComponentClassDefinition" + KindClusterDef = "ClusterDefinition" + KindClusterVersion = "ClusterVersion" + KindConfigConstraint = "ConfigConstraint" + KindBackup = "Backup" + KindRestoreJob = "RestoreJob" + KindBackupPolicy = "BackupPolicy" + KindOps = "OpsRequest" ) // K8S rbac API group @@ -97,10 +99,8 @@ const ( ServiceHAVIPTypeAnnotationValue = "private-ip" ServiceFloatingIPAnnotationKey = "service.kubernetes.io/kubeblocks-havip-floating-ip" - ClassLevelLabelKey = "class.kubeblocks.io/level" - ClassProviderLabelKey = "class.kubeblocks.io/provider" - ClassFamilyProviderLabelKey = "classfamily.kubeblocks.io/provider" - ComponentClassAnnotationKey = "cluster.kubeblocks.io/component-class" + ClassProviderLabelKey = "class.kubeblocks.io/provider" + ResourceConstraintProviderLabelKey = "resourceconstraint.kubeblocks.io/provider" ) // DataProtection API group @@ -217,8 +217,12 @@ func AddonGVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: ExtensionsAPIGroup, Version: ExtensionsAPIVersion, Resource: ResourceAddons} } -func ClassFamilyGVR() schema.GroupVersionResource { - return schema.GroupVersionResource{Group: AppsAPIGroup, Version: AppsAPIVersion, Resource: ResourceClassFamily} +func ComponentResourceConstraintGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: AppsAPIGroup, Version: AppsAPIVersion, Resource: ResourceComponentResourceConstraint} +} + +func ComponentClassDefinitionGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: AppsAPIGroup, Version: AppsAPIVersion, Resource: ResourceComponentClassDefinition} } func CRDGVR() schema.GroupVersionResource { diff --git a/internal/controller/lifecycle/transformer_fill_class.go b/internal/controller/lifecycle/transformer_fill_class.go index 81f76b946..886ea129a 100644 --- a/internal/controller/lifecycle/transformer_fill_class.go +++ b/internal/controller/lifecycle/transformer_fill_class.go @@ -17,7 +17,6 @@ limitations under the License. package lifecycle import ( - "encoding/json" "fmt" "sort" @@ -50,63 +49,58 @@ func (r *fillClass) Transform(dag *graph.DAG) error { func (r *fillClass) fillClass(reqCtx intctrlutil.RequestCtx, cluster *appsv1alpha1.Cluster, clusterDefinition appsv1alpha1.ClusterDefinition) error { var ( - value = cluster.GetAnnotations()[constant.ClassAnnotationKey] - componentClassMapping = make(map[string]string) - cmList corev1.ConfigMapList + classDefinitionList appsv1alpha1.ComponentClassDefinitionList ) - if value != "" { - if err := json.Unmarshal([]byte(value), &componentClassMapping); err != nil { - return err - } - } - cmLabels := []client.ListOption{ + ml := []client.ListOption{ client.MatchingLabels{constant.ClusterDefLabelKey: clusterDefinition.Name}, - client.HasLabels{constant.ClassProviderLabelKey}, } - if err := r.cli.List(reqCtx.Ctx, &cmList, cmLabels...); err != nil { + if err := r.cli.List(reqCtx.Ctx, &classDefinitionList, ml...); err != nil { return err } - compClasses, err := class.ParseClasses(&cmList) + compClasses, err := class.GetClasses(classDefinitionList) if err != nil { return err } - var classFamilyList appsv1alpha1.ClassFamilyList - if err = r.cli.List(reqCtx.Ctx, &classFamilyList); err != nil { + var constraintList appsv1alpha1.ComponentResourceConstraintList + if err = r.cli.List(reqCtx.Ctx, &constraintList); err != nil { return err } - // TODO use this function to get matched class families if class is not specified and component has no classes - _ = func(comp appsv1alpha1.ClusterComponentSpec) *class.ComponentClass { - var candidates []class.ClassModelWithFamilyName - for _, family := range classFamilyList.Items { - models := family.FindMatchingModels(&comp.Resources) - for _, model := range models { - candidates = append(candidates, class.ClassModelWithFamilyName{Family: family.Name, Model: model}) + // TODO use this function to get matched resource constraints if class is not specified and component has no classes + _ = func(comp appsv1alpha1.ClusterComponentSpec) *appsv1alpha1.ComponentClassInstance { + var candidates []class.ConstraintWithName + for _, item := range constraintList.Items { + constraints := item.FindMatchingConstraints(&comp.Resources) + for _, constraint := range constraints { + candidates = append(candidates, class.ConstraintWithName{Name: item.Name, Constraint: constraint}) } } if len(candidates) == 0 { return nil } - sort.Sort(class.ByModelList(candidates)) + sort.Sort(class.ByConstraintList(candidates)) candidate := candidates[0] - cpu, memory := class.GetMinCPUAndMemory(candidate.Model) - cls := &class.ComponentClass{ - Name: fmt.Sprintf("%s-%vc%vg", candidate.Family, cpu.AsDec().String(), memory.AsDec().String()), - CPU: *cpu, - Memory: *memory, + cpu, memory := class.GetMinCPUAndMemory(candidate.Constraint) + name := fmt.Sprintf("%s-%vc%vg", candidate.Name, cpu.AsDec().String(), memory.AsDec().String()) + cls := &appsv1alpha1.ComponentClassInstance{ + ComponentClass: appsv1alpha1.ComponentClass{ + Name: name, + CPU: *cpu, + Memory: *memory, + }, } return cls } - matchComponentClass := func(comp appsv1alpha1.ClusterComponentSpec, classes map[string]*class.ComponentClass) *class.ComponentClass { - filters := class.Filters(make(map[string]resource.Quantity)) + matchComponentClass := func(comp appsv1alpha1.ClusterComponentSpec, classes map[string]*appsv1alpha1.ComponentClassInstance) *appsv1alpha1.ComponentClassInstance { + filters := make(map[corev1.ResourceName]resource.Quantity) if !comp.Resources.Requests.Cpu().IsZero() { - filters[corev1.ResourceCPU.String()] = *comp.Resources.Requests.Cpu() + filters[corev1.ResourceCPU] = *comp.Resources.Requests.Cpu() } if !comp.Resources.Requests.Memory().IsZero() { - filters[corev1.ResourceMemory.String()] = *comp.Resources.Requests.Memory() + filters[corev1.ResourceMemory] = *comp.Resources.Requests.Memory() } return class.ChooseComponentClasses(classes, filters) } @@ -114,14 +108,13 @@ func (r *fillClass) fillClass(reqCtx intctrlutil.RequestCtx, cluster *appsv1alph for idx, comp := range cluster.Spec.ComponentSpecs { classes := compClasses[comp.ComponentDefRef] - var cls *class.ComponentClass - className, ok := componentClassMapping[comp.Name] - // TODO another case if len(classFamilyList.Items) > 0, use matchClassFamilies to find matching class family: + var cls *appsv1alpha1.ComponentClassInstance + // TODO another case if len(constraintList.Items) > 0, use matchClassFamilies to find matching resource constraint: switch { - case ok: - cls = classes[className] + case comp.ClassDefRef != nil && comp.ClassDefRef.Class != "": + cls = classes[comp.ClassDefRef.Class] if cls == nil { - return fmt.Errorf("unknown component class %s", className) + return fmt.Errorf("unknown component class %s", comp.ClassDefRef.Class) } case classes != nil: cls = matchComponentClass(comp, classes) @@ -133,7 +126,7 @@ func (r *fillClass) fillClass(reqCtx intctrlutil.RequestCtx, cluster *appsv1alph // TODO reconsider handling policy for this case continue } - componentClassMapping[comp.Name] = cls.Name + comp.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: cls.Name} requests := corev1.ResourceList{ corev1.ResourceCPU: cls.CPU, corev1.ResourceMemory: cls.Memory, @@ -152,17 +145,17 @@ func (r *fillClass) fillClass(reqCtx intctrlutil.RequestCtx, cluster *appsv1alph return nil } -func buildVolumeClaimByClass(cls *class.ComponentClass) []appsv1alpha1.ClusterComponentVolumeClaimTemplate { +func buildVolumeClaimByClass(cls *appsv1alpha1.ComponentClassInstance) []appsv1alpha1.ClusterComponentVolumeClaimTemplate { var volumes []appsv1alpha1.ClusterComponentVolumeClaimTemplate - for _, disk := range cls.Storage { + for _, volume := range cls.Volumes { volume := appsv1alpha1.ClusterComponentVolumeClaimTemplate{ - Name: disk.Name, + Name: volume.Name, Spec: appsv1alpha1.PersistentVolumeClaimSpec{ // TODO define access mode in class AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: disk.Size, + corev1.ResourceStorage: volume.Size, }, }, }, diff --git a/internal/generics/type.go b/internal/generics/type.go index 6aeb47f77..a69c4446b 100644 --- a/internal/generics/type.go +++ b/internal/generics/type.go @@ -95,6 +95,8 @@ var RestoreJobSignature = func(_ dataprotectionv1alpha1.RestoreJob, _ dataprotec } var AddonSignature = func(_ extensionsv1alpha1.Addon, _ extensionsv1alpha1.AddonList) { } +var ComponentResourceConstraintSignature = func(_ appsv1alpha1.ComponentResourceConstraint, _ appsv1alpha1.ComponentResourceConstraintList) {} +var ComponentClassDefinitionSignature = func(_ appsv1alpha1.ComponentClassDefinition, _ appsv1alpha1.ComponentClassDefinitionList) {} func ToGVK(object client.Object) schema.GroupVersionKind { t := reflect.TypeOf(object) diff --git a/internal/testutil/apps/componentclassdefinition_factory.go b/internal/testutil/apps/componentclassdefinition_factory.go new file mode 100644 index 000000000..74d9811b4 --- /dev/null +++ b/internal/testutil/apps/componentclassdefinition_factory.go @@ -0,0 +1,74 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apps + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" +) + +const classTemplate = ` +cpu: "{{ or .cpu 1 }}" +memory: "{{ or .memory 4 }}Gi" +volumes: +- name: data + size: "{{ or .dataStorageSize 10 }}Gi" +- name: log + size: "{{ or .logStorageSize 1 }}Gi" +` + +type MockComponentClassDefinitionFactory struct { + BaseFactory[appsv1alpha1.ComponentClassDefinition, *appsv1alpha1.ComponentClassDefinition, MockComponentClassDefinitionFactory] +} + +func NewComponentClassDefinitionFactory(name, clusterDefinitionRef, componentType string) *MockComponentClassDefinitionFactory { + f := &MockComponentClassDefinitionFactory{} + f.init("", name, &appsv1alpha1.ComponentClassDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + constant.ClassProviderLabelKey: "kubeblocks", + constant.ClusterDefLabelKey: clusterDefinitionRef, + constant.KBAppComponentDefRefLabelKey: componentType, + }, + }, + Spec: appsv1alpha1.ComponentClassDefinitionSpec{ + Groups: []appsv1alpha1.ComponentClassGroup{ + { + ResourceConstraintRef: "kube-resource-constraint-general", + Template: classTemplate, + Vars: []string{"cpu", "memory", "dataStorageSize", "logStorageSize"}, + Series: []appsv1alpha1.ComponentClassSeries{ + { + NamingTemplate: "general-{{ .cpu }}c{{ .memory }}g", + }, + }, + }, + }, + }, + }, f) + return f +} + +func (factory *MockComponentClassDefinitionFactory) AddClass(cls appsv1alpha1.ComponentClass) *MockComponentClassDefinitionFactory { + classes := factory.get().Spec.Groups[0].Series[0].Classes + classes = append(classes, cls) + factory.get().Spec.Groups[0].Series[0].Classes = classes + return factory +} From 51998a32ae8dde276be1d89547c1849720019bf2 Mon Sep 17 00:00:00 2001 From: diankuizhao Date: Fri, 14 Apr 2023 14:48:42 +0800 Subject: [PATCH 029/439] fix: BackupPolicyTemplate name of mysql-scale error (#2583) --- deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml index 62bfdb543..664060d00 100644 --- a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: apecloud-mysql-backup-policy-template + name: apecloud-mysql-scale-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: apecloud-mysql-scale {{- include "apecloud-mysql.labels" . | nindent 4 }} From 1f75849f16578c64cbe6b976693e7621866aa2b6 Mon Sep 17 00:00:00 2001 From: free6om Date: Fri, 14 Apr 2023 14:51:20 +0800 Subject: [PATCH 030/439] fix: KB_MYSQL_FOLLOWERS env inconsistent with cluster status after scale-in (#2565) --- .../consensusset/consensus_set_utils.go | 51 +++++++++++-------- .../consensusset/consensus_set_utils_test.go | 44 ++++++++++++++++ 2 files changed, 73 insertions(+), 22 deletions(-) diff --git a/controllers/apps/components/consensusset/consensus_set_utils.go b/controllers/apps/components/consensusset/consensus_set_utils.go index 8c5dc181a..39bf14857 100644 --- a/controllers/apps/components/consensusset/consensus_set_utils.go +++ b/controllers/apps/components/consensusset/consensus_set_utils.go @@ -400,28 +400,7 @@ func updateConsensusRoleInfo(ctx context.Context, componentDef *appsv1alpha1.ClusterComponentDefinition, componentName string, pods []corev1.Pod) error { - leader := "" - followers := "" - for _, pod := range pods { - role := pod.Labels[constant.RoleLabelKey] - // mapping role label to consensus member - roleMap := composeConsensusRoleMap(componentDef) - memberExt, ok := roleMap[role] - if !ok { - continue - } - switch memberExt.consensusRole { - case roleLeader: - leader = pod.Name - case roleFollower: - if len(followers) > 0 { - followers += "," - } - followers += pod.Name - case roleLearner: - // TODO: CT - } - } + leader, followers := composeRoleEnv(componentDef, pods) ml := client.MatchingLabels{ constant.AppInstanceLabelKey: cluster.GetName(), @@ -458,3 +437,31 @@ func updateConsensusRoleInfo(ctx context.Context, return nil } + +func composeRoleEnv(componentDef *appsv1alpha1.ClusterComponentDefinition, pods []corev1.Pod) (leader, followers string) { + leader, followers = "", "" + for _, pod := range pods { + if !intctrlutil.PodIsReadyWithLabel(pod) { + continue + } + role := pod.Labels[constant.RoleLabelKey] + // mapping role label to consensus member + roleMap := composeConsensusRoleMap(componentDef) + memberExt, ok := roleMap[role] + if !ok { + continue + } + switch memberExt.consensusRole { + case roleLeader: + leader = pod.Name + case roleFollower: + if len(followers) > 0 { + followers += "," + } + followers += pod.Name + case roleLearner: + // TODO: CT + } + } + return +} diff --git a/controllers/apps/components/consensusset/consensus_set_utils_test.go b/controllers/apps/components/consensusset/consensus_set_utils_test.go index c2ea47a42..583a49cef 100644 --- a/controllers/apps/components/consensusset/consensus_set_utils_test.go +++ b/controllers/apps/components/consensusset/consensus_set_utils_test.go @@ -19,6 +19,7 @@ package consensusset import ( "strconv" "testing" + "time" "github.com/stretchr/testify/assert" apps "k8s.io/api/apps/v1" @@ -163,3 +164,46 @@ func TestSortPods(t *testing.T) { }) } } + +func TestComposeRoleEnv(t *testing.T) { + componentDef := &appsv1alpha1.ClusterComponentDefinition{ + WorkloadType: appsv1alpha1.Consensus, + ConsensusSpec: &appsv1alpha1.ConsensusSetSpec{ + Leader: appsv1alpha1.ConsensusMember{ + Name: "leader", + AccessMode: appsv1alpha1.ReadWrite, + }, + Followers: []appsv1alpha1.ConsensusMember{ + { + Name: "follower", + AccessMode: appsv1alpha1.Readonly, + }, + }, + }, + } + + set := testk8s.NewFakeStatefulSet("foo", 3) + pods := make([]v1.Pod, 0) + for i := 0; i < 5; i++ { + pod := testk8s.NewFakeStatefulSetPod(set, i) + pod.Status.Conditions = []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + } + pod.Labels = map[string]string{constant.RoleLabelKey: "follower"} + pods = append(pods, *pod) + } + pods[0].Labels = map[string]string{constant.RoleLabelKey: "leader"} + leader, followers := composeRoleEnv(componentDef, pods) + assert.Equal(t, "foo-0", leader) + assert.Equal(t, "foo-1,foo-2,foo-3,foo-4", followers) + + dt := time.Now() + pods[3].DeletionTimestamp = &metav1.Time{Time: dt} + pods[4].DeletionTimestamp = &metav1.Time{Time: dt} + leader, followers = composeRoleEnv(componentDef, pods) + assert.Equal(t, "foo-0", leader) + assert.Equal(t, "foo-1,foo-2", followers) +} From 8fa93c58da8ba2cc774b0df8d17c3a6a03f9253e Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Fri, 14 Apr 2023 15:48:28 +0800 Subject: [PATCH 031/439] chore: fix release sync error (#2593) --- .github/workflows/release-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-sync.yml b/.github/workflows/release-sync.yml index dd4ed8236..f29b4ebe3 100644 --- a/.github/workflows/release-sync.yml +++ b/.github/workflows/release-sync.yml @@ -29,7 +29,7 @@ jobs: --type 5 \ --tag-name $LATEST_RELEASE_TAG \ --github-repo ${{ env.CLI_REPO }} \ - --github-token ${{ env.PERSONAL_ACCESS_TOKEN }} + --github-token ${{ secrets.PERSONAL_ACCESS_TOKEN }} bash ${{ github.workspace }}/.github/utils/release_gitlab.sh \ --type 4 \ From 36295805acee6542ac0c9e511d8c8c68692be6b0 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Fri, 14 Apr 2023 16:56:25 +0800 Subject: [PATCH 032/439] fix: refactor connect cluster (#2514) Co-authored-by: shanshanying --- docs/user_docs/cli/kbcli_cluster_connect.md | 16 +- docs/user_docs/cli/kbcli_migration.md | 48 +++ docs/user_docs/cli/kbcli_migration_create.md | 90 ++++++ .../user_docs/cli/kbcli_migration_describe.md | 53 ++++ docs/user_docs/cli/kbcli_migration_list.md | 69 ++++ docs/user_docs/cli/kbcli_migration_logs.md | 72 +++++ .../cli/kbcli_migration_templates.md | 69 ++++ .../cli/kbcli_migration_terminate.md | 59 ++++ internal/cli/cluster/helper.go | 131 +++++++- internal/cli/cmd/accounts/base.go | 70 ++-- internal/cli/cmd/accounts/base_test.go | 4 +- internal/cli/cmd/accounts/create_test.go | 2 +- internal/cli/cmd/accounts/delete_test.go | 2 +- internal/cli/cmd/accounts/describe_test.go | 2 +- internal/cli/cmd/accounts/grant_test.go | 2 +- internal/cli/cmd/accounts/list_test.go | 4 +- internal/cli/cmd/accounts/util.go | 88 ----- internal/cli/cmd/cluster/connect.go | 300 ++++++++++-------- internal/cli/cmd/cluster/connect_test.go | 84 ++--- internal/cli/cmd/cluster/logs.go | 2 +- .../create/template/migration_template.cue | 124 ++++---- internal/cli/engine/types.go | 17 - internal/cli/util/util.go | 19 ++ internal/sqlchannel/client.go | 14 +- internal/{cli => sqlchannel}/engine/client.go | 0 internal/{cli => sqlchannel}/engine/engine.go | 31 +- .../{cli => sqlchannel}/engine/engine_test.go | 25 +- internal/{cli => sqlchannel}/engine/mysql.go | 18 +- .../{cli => sqlchannel}/engine/mysql_test.go | 0 .../{cli => sqlchannel}/engine/postgresql.go | 18 +- internal/{cli => sqlchannel}/engine/redis.go | 23 +- .../{cli => sqlchannel}/engine/suite_test.go | 0 32 files changed, 1014 insertions(+), 442 deletions(-) create mode 100644 docs/user_docs/cli/kbcli_migration.md create mode 100644 docs/user_docs/cli/kbcli_migration_create.md create mode 100644 docs/user_docs/cli/kbcli_migration_describe.md create mode 100644 docs/user_docs/cli/kbcli_migration_list.md create mode 100644 docs/user_docs/cli/kbcli_migration_logs.md create mode 100644 docs/user_docs/cli/kbcli_migration_templates.md create mode 100644 docs/user_docs/cli/kbcli_migration_terminate.md delete mode 100644 internal/cli/engine/types.go rename internal/{cli => sqlchannel}/engine/client.go (100%) rename internal/{cli => sqlchannel}/engine/engine.go (92%) rename internal/{cli => sqlchannel}/engine/engine_test.go (62%) rename internal/{cli => sqlchannel}/engine/mysql.go (91%) rename internal/{cli => sqlchannel}/engine/mysql_test.go (100%) rename internal/{cli => sqlchannel}/engine/postgresql.go (91%) rename internal/{cli => sqlchannel}/engine/redis.go (68%) rename internal/{cli => sqlchannel}/engine/suite_test.go (100%) diff --git a/docs/user_docs/cli/kbcli_cluster_connect.md b/docs/user_docs/cli/kbcli_cluster_connect.md index 2565e2ab5..905d2fb0e 100644 --- a/docs/user_docs/cli/kbcli_cluster_connect.md +++ b/docs/user_docs/cli/kbcli_cluster_connect.md @@ -14,9 +14,15 @@ kbcli cluster connect (NAME | -i INSTANCE-NAME) [flags] # connect to a specified cluster, default connect to the leader or primary instance kbcli cluster connect mycluster + # connect to cluster as user + kbcli cluster connect mycluster --as-user myuser + # connect to a specified instance kbcli cluster connect -i mycluster-instance-0 + # connect to a specified component + kbcli cluster connect mycluster --component mycomponent + # show cli connection example kbcli cluster connect mycluster --show-example --client=cli @@ -30,10 +36,12 @@ kbcli cluster connect (NAME | -i INSTANCE-NAME) [flags] ### Options ``` - --client string Which client connection example should be output, only valid if --show-example is true. - -h, --help help for connect - -i, --instance string The instance name to connect. - --show-example Show how to connect to cluster or instance from different client. + --as-user string Connect to cluster as user + --client string Which client connection example should be output, only valid if --show-example is true. + --component string The component to connect. If not specified, the first component will be used. + -h, --help help for connect + -i, --instance string The instance name to connect. + --show-example Show how to connect to cluster or instance from different client. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_migration.md b/docs/user_docs/cli/kbcli_migration.md new file mode 100644 index 000000000..3cf6987d9 --- /dev/null +++ b/docs/user_docs/cli/kbcli_migration.md @@ -0,0 +1,48 @@ +--- +title: kbcli migration +--- + +Data migration between two data sources. + +### Options + +``` + -h, --help help for migration +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + + +* [kbcli migration create](kbcli_migration_create.md) - Create a migration task. +* [kbcli migration describe](kbcli_migration_describe.md) - Show details of a specific migration task. +* [kbcli migration list](kbcli_migration_list.md) - List migration tasks. +* [kbcli migration logs](kbcli_migration_logs.md) - Access migration task log file. +* [kbcli migration templates](kbcli_migration_templates.md) - List migration templates. +* [kbcli migration terminate](kbcli_migration_terminate.md) - Delete migration task. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_migration_create.md b/docs/user_docs/cli/kbcli_migration_create.md new file mode 100644 index 000000000..bf8445804 --- /dev/null +++ b/docs/user_docs/cli/kbcli_migration_create.md @@ -0,0 +1,90 @@ +--- +title: kbcli migration create +--- + +Create a migration task. + +``` +kbcli migration create name [flags] +``` + +### Examples + +``` + # Create a migration task to migrate the entire database under mysql: mydb1 and mytable1 under database: mydb2 to the target mysql + kbcli migration create mytask --template apecloud-mysql2mysql + --source user:123456@127.0.0.1:3306 + --sink user:123456@127.0.0.1:3305 + --migration-object '"mydb1","mydb2.mytable1"' + + # Create a migration task to migrate the schema: myschema under database: mydb1 under PostgreSQL to the target PostgreSQL + kbcli migration create mytask --template apecloud-pg2pg + --source user:123456@127.0.0.1:3306/mydb1 + --sink user:123456@127.0.0.1:3305/mydb1 + --migration-object '"myschema"' + + # Use prechecks, data initialization, CDC, but do not perform structure initialization + kbcli migration create mytask --template apecloud-pg2pg + --source user:123456@127.0.0.1:3306/mydb1 + --sink user:123456@127.0.0.1:3305/mydb1 + --migration-object '"myschema"' + --steps precheck=true,init-struct=false,init-data=true,cdc=true + + # Create a migration task with two tolerations + kbcli migration create mytask --template apecloud-pg2pg + --source user:123456@127.0.0.1:3306/mydb1 + --sink user:123456@127.0.0.1:3305/mydb1 + --migration-object '"myschema"' + --tolerations '"step=global,key=engineType,value=pg,operator=Equal,effect=NoSchedule","step=init-data,key=diskType,value=ssd,operator=Equal,effect=NoSchedule"' + + # Limit resource usage when performing data initialization + kbcli migration create mytask --template apecloud-pg2pg + --source user:123456@127.0.0.1:3306/mydb1 + --sink user:123456@127.0.0.1:3305/mydb1 + --migration-object '"myschema"' + --resources '"step=init-data,cpu=1000m,memory=1Gi"' +``` + +### Options + +``` + -h, --help help for create + --migration-object strings Set the data objects that need to be migrated,such as '"db1.table1","db2"' + --resources strings Resources limit for migration, such as '"cpu=3000m,memory=3Gi"' + --sink string Set the sink database information for migration.such as '{username}:{password}@{connection_address}:{connection_port}/[{database}] + --source string Set the source database information for migration.such as '{username}:{password}@{connection_address}:{connection_port}/[{database}]' + --steps strings Set up migration steps,such as: precheck=true,init-struct=true,init-data=true,cdc=true + --template string Specify migration template, run "kbcli migration templates" to show all available migration templates + --tolerations strings Tolerations for migration, such as '"key=engineType,value=pg,operator=Equal,effect=NoSchedule"' +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli migration](kbcli_migration.md) - Data migration between two data sources. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_migration_describe.md b/docs/user_docs/cli/kbcli_migration_describe.md new file mode 100644 index 000000000..73432c7c0 --- /dev/null +++ b/docs/user_docs/cli/kbcli_migration_describe.md @@ -0,0 +1,53 @@ +--- +title: kbcli migration describe +--- + +Show details of a specific migration task. + +``` +kbcli migration describe NAME [flags] +``` + +### Examples + +``` + # describe a specified migration task + kbcli migration describe mytask +``` + +### Options + +``` + -h, --help help for describe +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli migration](kbcli_migration.md) - Data migration between two data sources. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_migration_list.md b/docs/user_docs/cli/kbcli_migration_list.md new file mode 100644 index 000000000..9fdbe1219 --- /dev/null +++ b/docs/user_docs/cli/kbcli_migration_list.md @@ -0,0 +1,69 @@ +--- +title: kbcli migration list +--- + +List migration tasks. + +``` +kbcli migration list [NAME] [flags] +``` + +### Examples + +``` + # list all migration tasks + kbcli migration list + + # list a single migration task with specified NAME + kbcli migration list mytask + + # list a single migration task in YAML output format + kbcli migration list mytask -o yaml + + # list a single migration task in JSON output format + kbcli migration list mytask -o json + + # list a single migration task in wide output format + kbcli migration list mytask -o wide +``` + +### Options + +``` + -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. + -h, --help help for list + -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. + --show-labels When printing, show all labels as the last column (default hide labels column) +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli migration](kbcli_migration.md) - Data migration between two data sources. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_migration_logs.md b/docs/user_docs/cli/kbcli_migration_logs.md new file mode 100644 index 000000000..2048907c9 --- /dev/null +++ b/docs/user_docs/cli/kbcli_migration_logs.md @@ -0,0 +1,72 @@ +--- +title: kbcli migration logs +--- + +Access migration task log file. + +``` +kbcli migration logs NAME [flags] +``` + +### Examples + +``` + # Logs when returning to the "init-struct" step from the migration task mytask + kbcli migration logs mytask --step init-struct + + # Logs only the most recent 20 lines when returning to the "cdc" step from the migration task mytask + kbcli migration logs mytask --step cdc --tail=20 +``` + +### Options + +``` + --all-containers Get all containers' logs in the pod(s). + -c, --container string Print the logs of this container + -f, --follow Specify if the logs should be streamed. + -h, --help help for logs + --ignore-errors If watching / following pod logs, allow for any errors that occur to be non-fatal + --insecure-skip-tls-verify-backend Skip verifying the identity of the kubelet that logs are requested from. In theory, an attacker could provide invalid log content back. You might want to use this if your kubelet serving certificates have expired. + --limit-bytes int Maximum bytes of logs to return. Defaults to no limit. + --max-log-requests int Specify maximum number of concurrent logs to follow when using by a selector. Defaults to 5. + --pod-running-timeout duration The length of time (like 5s, 2m, or 3h, higher than zero) to wait until at least one pod is running (default 20s) + --prefix Prefix each log line with the log source (pod name and container name) + -p, --previous If true, print the logs for the previous instance of the container in a pod if it exists. + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. + --since duration Only return logs newer than a relative duration like 5s, 2m, or 3h. Defaults to all logs. Only one of since-time / since may be used. + --since-time string Only return logs after a specific date (RFC3339). Defaults to all logs. Only one of since-time / since may be used. + --step string Specify the step. Allow values: precheck,init-struct,init-data,cdc + --tail int Lines of recent log file to display. Defaults to -1 with no selector, showing all log lines otherwise 10, if a selector is provided. (default -1) + --timestamps Include timestamps on each line in the log output +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli migration](kbcli_migration.md) - Data migration between two data sources. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_migration_templates.md b/docs/user_docs/cli/kbcli_migration_templates.md new file mode 100644 index 000000000..7f5bb59b8 --- /dev/null +++ b/docs/user_docs/cli/kbcli_migration_templates.md @@ -0,0 +1,69 @@ +--- +title: kbcli migration templates +--- + +List migration templates. + +``` +kbcli migration templates [NAME] [flags] +``` + +### Examples + +``` + # list all migration templates + kbcli migration templates + + # list a single migration template with specified NAME + kbcli migration templates mytemplate + + # list a single migration template in YAML output format + kbcli migration templates mytemplate -o yaml + + # list a single migration template in JSON output format + kbcli migration templates mytemplate -o json + + # list a single migration template in wide output format + kbcli migration templates mytemplate -o wide +``` + +### Options + +``` + -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. + -h, --help help for templates + -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. + --show-labels When printing, show all labels as the last column (default hide labels column) +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli migration](kbcli_migration.md) - Data migration between two data sources. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_migration_terminate.md b/docs/user_docs/cli/kbcli_migration_terminate.md new file mode 100644 index 000000000..bcec9fa01 --- /dev/null +++ b/docs/user_docs/cli/kbcli_migration_terminate.md @@ -0,0 +1,59 @@ +--- +title: kbcli migration terminate +--- + +Delete migration task. + +``` +kbcli migration terminate NAME [flags] +``` + +### Examples + +``` + # terminate a migration task named mytask and delete resources in k8s without affecting source and target data in database + kbcli migration terminate mytask +``` + +### Options + +``` + -A, --all-namespaces If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. + --auto-approve Skip interactive approval before deleting + --force If true, immediately remove resources from API and bypass graceful deletion. Note that immediate deletion of some resources may result in inconsistency or data loss and requires confirmation. + --grace-period int Period of time in seconds given to the resource to terminate gracefully. Ignored if negative. Set to 1 for immediate shutdown. Can only be set to 0 when --force is true (force deletion). (default -1) + -h, --help help for terminate + --now If true, resources are signaled for immediate shutdown (same as --grace-period=1). + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli migration](kbcli_migration.md) - Data migration between two data sources. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/internal/cli/cluster/helper.go b/internal/cli/cluster/helper.go index 53e5ef68e..304946ae1 100644 --- a/internal/cli/cluster/helper.go +++ b/internal/cli/cluster/helper.go @@ -35,27 +35,41 @@ import ( "github.com/apecloud/kubeblocks/internal/constant" ) +const ( + ComponentNameEmpty = "" +) + // GetSimpleInstanceInfos return simple instance info that only contains instance name and role, the default // instance should be the first element in the returned array. -func GetSimpleInstanceInfos(dynamic dynamic.Interface, name string, namespace string) []*InstanceInfo { +func GetSimpleInstanceInfos(dynamic dynamic.Interface, name, namespace string) []*InstanceInfo { + return GetSimpleInstanceInfosForComponent(dynamic, name, ComponentNameEmpty, namespace) +} + +// GetSimpleInstanceInfosForComponent return simple instance info that only contains instance name and role, +func GetSimpleInstanceInfosForComponent(dynamic dynamic.Interface, name, componentName, namespace string) []*InstanceInfo { // if cluster status contains what we need, return directly - if infos := getInstanceInfoFromStatus(dynamic, name, namespace); len(infos) > 0 { + if infos := getInstanceInfoFromStatus(dynamic, name, componentName, namespace); len(infos) > 0 { return infos } // if cluster status does not contain what we need, try to list all pods and build instance info - return getInstanceInfoByList(dynamic, name, namespace) + return getInstanceInfoByList(dynamic, name, componentName, namespace) } // getInstancesInfoFromCluster get instances info from cluster status -func getInstanceInfoFromStatus(dynamic dynamic.Interface, name string, namespace string) []*InstanceInfo { +func getInstanceInfoFromStatus(dynamic dynamic.Interface, name, componentName, namespace string) []*InstanceInfo { var infos []*InstanceInfo cluster, err := GetClusterByName(dynamic, name, namespace) if err != nil { return nil } // travel all components, check type - for _, c := range cluster.Status.Components { + for compName, c := range cluster.Status.Components { + // filter by component name + if len(componentName) > 0 && compName != componentName { + continue + } + var info *InstanceInfo // workload type is Consensus if c.ConsensusSetStatus != nil { @@ -101,15 +115,22 @@ func getInstanceInfoFromStatus(dynamic dynamic.Interface, name string, namespace } // getInstanceInfoByList get instances info by list all pods -func getInstanceInfoByList(dynamic dynamic.Interface, name string, namespace string) []*InstanceInfo { +func getInstanceInfoByList(dynamic dynamic.Interface, name, componentName, namespace string) []*InstanceInfo { var infos []*InstanceInfo + // filter by cluster name + lables := util.BuildLabelSelectorByNames("", []string{name}) + // filter by component name + if len(componentName) > 0 { + lables = util.BuildComponentNameLables(lables, []string{componentName}) + } + objs, err := dynamic.Resource(schema.GroupVersionResource{Group: corev1.GroupName, Version: types.K8sCoreAPIVersion, Resource: "pods"}). - Namespace(namespace).List(context.TODO(), metav1.ListOptions{ - LabelSelector: util.BuildLabelSelectorByNames("", []string{name}), - }) + Namespace(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: lables}) + if err != nil { return nil } + for _, o := range objs.Items { infos = append(infos, &InstanceInfo{Name: o.GetName()}) } @@ -313,3 +334,95 @@ func findLatestVersion(versions *appsv1alpha1.ClusterVersionList) *appsv1alpha1. } return version } + +type CompInfo struct { + Component *appsv1alpha1.ClusterComponentSpec + ComponentStatus *appsv1alpha1.ClusterComponentStatus + ComponentDef *appsv1alpha1.ClusterComponentDefinition +} + +func (info *CompInfo) InferPodName() (string, error) { + if info.ComponentStatus == nil { + return "", fmt.Errorf("component status is missing") + } + if info.ComponentStatus.Phase != appsv1alpha1.RunningClusterCompPhase || !*info.ComponentStatus.PodsReady { + return "", fmt.Errorf("component is not ready, please try later") + } + if info.ComponentStatus.ConsensusSetStatus != nil { + return info.ComponentStatus.ConsensusSetStatus.Leader.Pod, nil + } + if info.ComponentStatus.ReplicationSetStatus != nil { + return info.ComponentStatus.ReplicationSetStatus.Primary.Pod, nil + } + return "", fmt.Errorf("cannot infer the pod to connect, please specify the pod name explicitly by `--instance` flag") +} + +func FillCompInfoByName(ctx context.Context, dynamic dynamic.Interface, namespace, clusterName, componentName string) (*CompInfo, error) { + cluster, err := GetClusterByName(dynamic, clusterName, namespace) + if err != nil { + return nil, err + } + if cluster.Status.Phase != appsv1alpha1.RunningClusterPhase { + return nil, fmt.Errorf("cluster %s is not running, please try later", clusterName) + } + + compInfo := &CompInfo{} + // fill component + if len(componentName) == 0 { + compInfo.Component = &cluster.Spec.ComponentSpecs[0] + } else { + compInfo.Component = cluster.Spec.GetComponentByName(componentName) + } + + if compInfo.Component == nil { + return nil, fmt.Errorf("component %s not found in cluster %s", componentName, clusterName) + } + // fill component status + for name, compStatus := range cluster.Status.Components { + if name == compInfo.Component.Name { + compInfo.ComponentStatus = &compStatus + break + } + } + if compInfo.ComponentStatus == nil { + return nil, fmt.Errorf("componentStatus %s not found in cluster %s", componentName, clusterName) + } + + // find cluster def + clusterDef, err := GetClusterDefByName(dynamic, cluster.Spec.ClusterDefRef) + if err != nil { + return nil, err + } + // find component def by reference + for _, compDef := range clusterDef.Spec.ComponentDefs { + if compDef.Name == compInfo.Component.ComponentDefRef { + compInfo.ComponentDef = &compDef + break + } + } + if compInfo.ComponentDef == nil { + return nil, fmt.Errorf("componentDef %s not found in clusterDef %s", compInfo.Component.ComponentDefRef, clusterDef.Name) + } + return compInfo, nil +} + +func GetPodClusterName(pod *corev1.Pod) string { + if pod.Labels == nil { + return "" + } + return pod.Labels[constant.AppInstanceLabelKey] +} + +func GetPodComponentName(pod *corev1.Pod) string { + if pod.Labels == nil { + return "" + } + return pod.Labels[constant.KBAppComponentLabelKey] +} + +func GetPodWorkloadType(pod *corev1.Pod) string { + if pod.Labels == nil { + return "" + } + return pod.Labels[constant.WorkloadTypeLabelKey] +} diff --git a/internal/cli/cmd/accounts/base.go b/internal/cli/cmd/accounts/base.go index c21052941..41d61d398 100644 --- a/internal/cli/cmd/accounts/base.go +++ b/internal/cli/cmd/accounts/base.go @@ -29,6 +29,7 @@ import ( klog "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" + clusterutil "github.com/apecloud/kubeblocks/internal/cli/cluster" "github.com/apecloud/kubeblocks/internal/cli/exec" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/util" @@ -36,7 +37,6 @@ import ( ) type AccountBaseOptions struct { - Namespace string ClusterName string CharType string ComponentName string @@ -49,12 +49,13 @@ type AccountBaseOptions struct { } var ( - errClusterNameNum = fmt.Errorf("please specify ONE cluster-name at a time") - errMissingUserName = fmt.Errorf("please specify username") - errMissingRoleName = fmt.Errorf("please specify at least ONE role name") - errInvalidRoleName = fmt.Errorf("invalid role name, should be one of [SUPERUSER, READWRITE, READONLY] ") - errInvalidOp = fmt.Errorf("invalid operation") - errCompNameOrInstName = fmt.Errorf("please specify either --component or --instance, not both") + errClusterNameNum = fmt.Errorf("please specify ONE cluster-name at a time") + errMissingUserName = fmt.Errorf("please specify username") + errMissingRoleName = fmt.Errorf("please specify at least ONE role name") + errInvalidRoleName = fmt.Errorf("invalid role name, should be one of [SUPERUSER, READWRITE, READONLY] ") + errInvalidOp = fmt.Errorf("invalid operation") + errCompNameOrInstName = fmt.Errorf("please specify either --component or --instance, not both") + errClusterNameorInstName = fmt.Errorf("specify either cluster name or --instance") ) func NewAccountBaseOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, op bindings.OperationKind) *AccountBaseOptions { @@ -70,24 +71,28 @@ func (o *AccountBaseOptions) AddFlags(cmd *cobra.Command) { } func (o *AccountBaseOptions) Validate(args []string) error { - if len(args) != 1 { + if len(args) > 1 { return errClusterNameNum - } else { - o.ClusterName = args[0] } - if len(o.PodName) > 0 && len(o.ComponentName) > 0 { - return errCompNameOrInstName + if len(o.PodName) > 0 { + if len(o.ComponentName) > 0 { + return errCompNameOrInstName + } + if len(args) > 0 { + return errClusterNameorInstName + } + } else if len(args) == 0 { + return errClusterNameorInstName + } + if len(args) == 1 { + o.ClusterName = args[0] } return nil } func (o *AccountBaseOptions) Complete(f cmdutil.Factory) error { var err error - o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace() - if err != nil { - return err - } err = o.ExecOptions.Complete() if err != nil { return err @@ -96,27 +101,36 @@ func (o *AccountBaseOptions) Complete(f cmdutil.Factory) error { ctx, cancelFn := context.WithCancel(context.Background()) defer cancelFn() - compInfo, err := fillCompInfoByName(ctx, o.ExecOptions.Dynamic, o.Namespace, o.ClusterName, o.ComponentName) + if len(o.PodName) > 0 { + // get pod by name + o.Pod, err = o.ExecOptions.Client.CoreV1().Pods(o.Namespace).Get(ctx, o.PodName, metav1.GetOptions{}) + if err != nil { + return err + } + o.ClusterName = clusterutil.GetPodClusterName(o.Pod) + o.ComponentName = clusterutil.GetPodComponentName(o.Pod) + } + + compInfo, err := clusterutil.FillCompInfoByName(ctx, o.ExecOptions.Dynamic, o.Namespace, o.ClusterName, o.ComponentName) if err != nil { return err } // fill component name if len(o.ComponentName) == 0 { - o.ComponentName = compInfo.comp.Name + o.ComponentName = compInfo.Component.Name } // fill character type - o.CharType = compInfo.compDef.CharacterType + o.CharType = compInfo.ComponentDef.CharacterType - // fill pod name if len(o.PodName) == 0 { - if o.PodName, err = compInfo.inferPodName(); err != nil { + if o.PodName, err = compInfo.InferPodName(); err != nil { + return err + } + // get pod by name + o.Pod, err = o.ExecOptions.Client.CoreV1().Pods(o.Namespace).Get(ctx, o.PodName, metav1.GetOptions{}) + if err != nil { return err } - } - // get pod by name - o.Pod, err = o.ExecOptions.Client.CoreV1().Pods(o.Namespace).Get(ctx, o.PodName, metav1.GetOptions{}) - if err != nil { - return err } o.ExecOptions.Pod = o.Pod @@ -171,13 +185,13 @@ func (o *AccountBaseOptions) Run(f cmdutil.Factory, streams genericclioptions.IO func (o *AccountBaseOptions) Do() (sqlchannel.SQLChannelResponse, error) { klog.V(1).Info(fmt.Sprintf("connect to cluster %s, component %s, instance %s\n", o.ClusterName, o.ComponentName, o.PodName)) response := sqlchannel.SQLChannelResponse{} - sqlClient, err := sqlchannel.NewHTTPClientWithPod(o.ExecOptions, o.Pod, o.CharType) + sqlClient, err := sqlchannel.NewHTTPClientWithChannelPod(o.Pod, o.CharType) if err != nil { return response, err } request := sqlchannel.SQLChannelRequest{Operation: (string)(o.AccountOp), Metadata: o.RequestMeta} - response, err = sqlClient.SendRequest(request) + response, err = sqlClient.SendRequest(o.ExecOptions, request) return response, err } diff --git a/internal/cli/cmd/accounts/base_test.go b/internal/cli/cmd/accounts/base_test.go index 041088afd..646984561 100644 --- a/internal/cli/cmd/accounts/base_test.go +++ b/internal/cli/cmd/accounts/base_test.go @@ -91,7 +91,7 @@ var _ = Describe("Base Account Options", func() { o := NewAccountBaseOptions(tf, streams, sqlchannel.CreateUserOp) Expect(o).ShouldNot(BeNil()) args := []string{} - Expect(o.Validate(args)).Should(MatchError(errClusterNameNum)) + Expect(o.Validate(args)).Should(MatchError(errClusterNameorInstName)) // add two elements By("add two args") @@ -105,7 +105,7 @@ var _ = Describe("Base Account Options", func() { // set pod name o.PodName = "testpod" - Expect(o.Validate(args)).Should(Succeed()) + Expect(o.Validate(args)).Should(MatchError(errClusterNameorInstName)) // set component name as well o.ComponentName = "testcomponent" Expect(o.Validate(args)).Should(MatchError(errCompNameOrInstName)) diff --git a/internal/cli/cmd/accounts/create_test.go b/internal/cli/cmd/accounts/create_test.go index 04a5c2c79..eed815064 100644 --- a/internal/cli/cmd/accounts/create_test.go +++ b/internal/cli/cmd/accounts/create_test.go @@ -88,7 +88,7 @@ var _ = Describe("Create Account Options", func() { o := NewCreateUserOptions(tf, streams) Expect(o).ShouldNot(BeNil()) args := []string{} - Expect(o.Validate(args)).Should(MatchError(errClusterNameNum)) + Expect(o.Validate(args)).Should(MatchError(errClusterNameorInstName)) // add two elements By("add two args") diff --git a/internal/cli/cmd/accounts/delete_test.go b/internal/cli/cmd/accounts/delete_test.go index 197d38971..f764a08fe 100644 --- a/internal/cli/cmd/accounts/delete_test.go +++ b/internal/cli/cmd/accounts/delete_test.go @@ -90,7 +90,7 @@ var _ = Describe("Delete Account Options", func() { o := NewDeleteUserOptions(tf, streams) Expect(o).ShouldNot(BeNil()) args := []string{} - Expect(o.Validate(args)).Should(MatchError(errClusterNameNum)) + Expect(o.Validate(args)).Should(MatchError(errClusterNameorInstName)) // add one element By("add one more args, should fail") diff --git a/internal/cli/cmd/accounts/describe_test.go b/internal/cli/cmd/accounts/describe_test.go index f0584f624..83657c495 100644 --- a/internal/cli/cmd/accounts/describe_test.go +++ b/internal/cli/cmd/accounts/describe_test.go @@ -88,7 +88,7 @@ var _ = Describe("Describe Account Options", func() { o := NewDescribeUserOptions(tf, streams) Expect(o).ShouldNot(BeNil()) args := []string{} - Expect(o.Validate(args)).Should(MatchError(errClusterNameNum)) + Expect(o.Validate(args)).Should(MatchError(errClusterNameorInstName)) // add one element By("add one more args, should fail") diff --git a/internal/cli/cmd/accounts/grant_test.go b/internal/cli/cmd/accounts/grant_test.go index ed7ecb64f..b1c778103 100644 --- a/internal/cli/cmd/accounts/grant_test.go +++ b/internal/cli/cmd/accounts/grant_test.go @@ -94,7 +94,7 @@ var _ = Describe("Grant Account Options", func() { o := NewGrantOptions(tf, streams, op) Expect(o).ShouldNot(BeNil()) args := []string{} - Expect(o.Validate(args)).Should(MatchError(errClusterNameNum)) + Expect(o.Validate(args)).Should(MatchError(errClusterNameorInstName)) // add one element By("add one more args, should fail") diff --git a/internal/cli/cmd/accounts/list_test.go b/internal/cli/cmd/accounts/list_test.go index 130a9f254..90165739c 100644 --- a/internal/cli/cmd/accounts/list_test.go +++ b/internal/cli/cmd/accounts/list_test.go @@ -88,7 +88,7 @@ var _ = Describe("List Account Options", func() { o := NewListUserOptions(tf, streams) Expect(o).ShouldNot(BeNil()) args := []string{} - Expect(o.Validate(args)).Should(MatchError(errClusterNameNum)) + Expect(o.Validate(args)).Should(MatchError(errClusterNameorInstName)) // add two elements By("add two args") @@ -102,7 +102,7 @@ var _ = Describe("List Account Options", func() { // set pod name o.PodName = "pod1" - Expect(o.Validate(args)).Should(Succeed()) + Expect(o.Validate(args)).Should(MatchError(errClusterNameorInstName)) // set component name o.ComponentName = "foo-component" Expect(o.Validate(args)).Should(MatchError(errCompNameOrInstName)) diff --git a/internal/cli/cmd/accounts/util.go b/internal/cli/cmd/accounts/util.go index 2ccf2c356..c676d5c18 100644 --- a/internal/cli/cmd/accounts/util.go +++ b/internal/cli/cmd/accounts/util.go @@ -17,16 +17,7 @@ limitations under the License. package accounts import ( - "context" "encoding/json" - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/dynamic" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/internal/cli/types" ) func struct2Map(obj interface{}) (map[string]interface{}, error) { @@ -40,82 +31,3 @@ func struct2Map(obj interface{}) (map[string]interface{}, error) { } return m, nil } - -type compInfo struct { - comp *appsv1alpha1.ClusterComponentSpec - compStatus *appsv1alpha1.ClusterComponentStatus - compDef *appsv1alpha1.ClusterComponentDefinition -} - -func (info *compInfo) inferPodName() (string, error) { - if info.compStatus == nil { - return "", fmt.Errorf("component status is missing") - } - if info.compStatus.Phase != appsv1alpha1.RunningClusterCompPhase || !*info.compStatus.PodsReady { - return "", fmt.Errorf("component is not ready, please try later") - } - if info.compStatus.ConsensusSetStatus != nil { - return info.compStatus.ConsensusSetStatus.Leader.Pod, nil - } - if info.compStatus.ReplicationSetStatus != nil { - return info.compStatus.ReplicationSetStatus.Primary.Pod, nil - } - return "", fmt.Errorf("cannot infer the pod to connect, please specify the pod name explicitly by `--instance` flag") -} - -func fillCompInfoByName(ctx context.Context, dynamic dynamic.Interface, namespace, clusterName, componentName string) (*compInfo, error) { - // find cluster - obj, err := dynamic.Resource(types.ClusterGVR()).Namespace(namespace).Get(ctx, clusterName, metav1.GetOptions{}) - if err != nil { - return nil, err - } - cluster := &appsv1alpha1.Cluster{} - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, cluster); err != nil { - return nil, err - } - if cluster.Status.Phase != appsv1alpha1.RunningClusterPhase { - return nil, fmt.Errorf("cluster %s is not running, please try later", clusterName) - } - - compInfo := &compInfo{} - // fill component - if len(componentName) == 0 { - compInfo.comp = &cluster.Spec.ComponentSpecs[0] - } else { - compInfo.comp = cluster.Spec.GetComponentByName(componentName) - } - if compInfo.comp == nil { - return nil, fmt.Errorf("component %s not found in cluster %s", componentName, clusterName) - } - // fill component status - for name, compStatus := range cluster.Status.Components { - if name == compInfo.comp.Name { - compInfo.compStatus = &compStatus - break - } - } - if compInfo.compStatus == nil { - return nil, fmt.Errorf("componentStatus %s not found in cluster %s", componentName, clusterName) - } - - // find cluster def - obj, err = dynamic.Resource(types.ClusterDefGVR()).Namespace(metav1.NamespaceAll).Get(ctx, cluster.Spec.ClusterDefRef, metav1.GetOptions{}) - if err != nil { - return nil, err - } - clusterDef := &appsv1alpha1.ClusterDefinition{} - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, clusterDef); err != nil { - return nil, err - } - // find component def by reference - for _, compDef := range clusterDef.Spec.ComponentDefs { - if compDef.Name == compInfo.comp.ComponentDefRef { - compInfo.compDef = &compDef - break - } - } - if compInfo.compDef == nil { - return nil, fmt.Errorf("componentDef %s not found in clusterDef %s", compInfo.comp.ComponentDefRef, clusterDef.Name) - } - return compInfo, nil -} diff --git a/internal/cli/cmd/cluster/connect.go b/internal/cli/cmd/cluster/connect.go index fc8cf3fa7..f5f8317f1 100644 --- a/internal/cli/cmd/cluster/connect.go +++ b/internal/cli/cmd/cluster/connect.go @@ -20,31 +20,40 @@ import ( "context" "fmt" "strings" + "syscall" "github.com/spf13/cobra" + "golang.org/x/crypto/ssh/terminal" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" computil "github.com/apecloud/kubeblocks/controllers/apps/components/util" "github.com/apecloud/kubeblocks/internal/cli/cluster" - "github.com/apecloud/kubeblocks/internal/cli/engine" "github.com/apecloud/kubeblocks/internal/cli/exec" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/sqlchannel/engine" ) var connectExample = templates.Examples(` # connect to a specified cluster, default connect to the leader or primary instance kbcli cluster connect mycluster + # connect to cluster as user + kbcli cluster connect mycluster --as-user myuser + # connect to a specified instance kbcli cluster connect -i mycluster-instance-0 + # connect to a specified component + kbcli cluster connect mycluster --component mycomponent + # show cli connection example kbcli cluster connect mycluster --show-example --client=cli @@ -55,7 +64,9 @@ var connectExample = templates.Examples(` kbcli cluster connect mycluster --show-example`) type ConnectOptions struct { - name string + clusterName string + componentName string + clientType string showExample bool engine engine.Interface @@ -63,6 +74,15 @@ type ConnectOptions struct { privateEndPoint bool svc *corev1.Service + component *appsv1alpha1.ClusterComponentSpec + componentDef *appsv1alpha1.ClusterComponentDefinition + targetCluster *appsv1alpha1.Cluster + targetClusterDef *appsv1alpha1.ClusterDefinition + + // assume user , who has access to the cluster + userName string + userPasswd string + *exec.ExecOptions } @@ -75,17 +95,22 @@ func NewConnectCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr Example: connectExample, ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.ExecOptions.Complete()) + util.CheckErr(o.validate(args)) + util.CheckErr(o.complete()) if o.showExample { - util.CheckErr(o.runShowExample(args)) + util.CheckErr(o.runShowExample()) } else { - util.CheckErr(o.connect(args)) + util.CheckErr(o.connect()) } }, } cmd.Flags().StringVarP(&o.PodName, "instance", "i", "", "The instance name to connect.") + cmd.Flags().StringVar(&o.componentName, "component", "", "The component to connect. If not specified, the first component will be used.") cmd.Flags().BoolVar(&o.showExample, "show-example", false, "Show how to connect to cluster or instance from different client.") cmd.Flags().StringVar(&o.clientType, "client", "", "Which client connection example should be output, only valid if --show-example is true.") + + cmd.Flags().StringVar(&o.userName, "as-user", "", "Connect to cluster as user") + util.CheckErr(cmd.RegisterFlagCompletionFunc("client", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { var types []string for _, t := range engine.ClientTypes() { @@ -98,28 +123,22 @@ func NewConnectCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr return cmd } -func (o *ConnectOptions) runShowExample(args []string) error { - if len(args) > 1 { - return fmt.Errorf("only support to connect one cluster") - } - - if len(args) == 0 { - return fmt.Errorf("cluster name should be specified when --show-example is true") - } - - o.name = args[0] - +func (o *ConnectOptions) runShowExample() error { // get connection info info, err := o.getConnectionInfo() if err != nil { return err } + // make sure engine is initialized + if o.engine == nil { + return fmt.Errorf("engine is not initialized yet") + } // if cluster does not have public endpoints, tell user to use port-forward command and // connect cluster from local host if o.privateEndPoint { fmt.Fprintf(o.Out, "# cluster %s does not have public endpoints, you can run following command and connect cluster from local host\n"+ - "kubectl port-forward service/%s %s:%s\n\n", o.name, o.svc.Name, info.Port, info.Port) + "kubectl port-forward service/%s %s:%s\n\n", o.clusterName, o.svc.Name, info.Port, info.Port) info.Host = "127.0.0.1" } @@ -127,59 +146,161 @@ func (o *ConnectOptions) runShowExample(args []string) error { return nil } -// connect create parameters for connecting cluster and connect -func (o *ConnectOptions) connect(args []string) error { +func (o *ConnectOptions) validate(args []string) error { if len(args) > 1 { return fmt.Errorf("only support to connect one cluster") } - if len(args) == 0 && len(o.PodName) == 0 { - return fmt.Errorf("cluster name or instance name should be specified") + // cluster name and pod instance are mutual exclusive + if len(o.PodName) > 0 { + if len(args) > 0 { + return fmt.Errorf("specify either cluster name or instance name, not both") + } + if len(o.componentName) > 0 { + return fmt.Errorf("component name is valid only when cluster name is specified") + } + } else if len(args) == 0 { + return fmt.Errorf("either cluster name or instance name should be specified") } + // set custer name if len(args) > 0 { - o.name = args[0] + o.clusterName = args[0] } - // get target pod name, if not specified, find default pod from cluster - if len(o.PodName) == 0 { - if err := o.getTargetPod(); err != nil { + // validate user name and password + if len(o.userName) > 0 { + // read password from stdin + fmt.Print("Password: ") + if bytePassword, err := terminal.ReadPassword(syscall.Stdin); err != nil { return err + } else { + o.userPasswd = string(bytePassword) } } + return nil +} - // get the pod object - pod, err := o.Client.CoreV1().Pods(o.Namespace).Get(context.TODO(), o.PodName, metav1.GetOptions{}) - if err != nil { +func (o *ConnectOptions) complete() error { + var err error + if err = o.ExecOptions.Complete(); err != nil { return err } + // opt 1. specified pod name + // 1.1 get pod by name + if len(o.PodName) > 0 { + if o.Pod, err = o.Client.CoreV1().Pods(o.Namespace).Get(context.Background(), o.PodName, metav1.GetOptions{}); err != nil { + return err + } + o.clusterName = cluster.GetPodClusterName(o.Pod) + o.componentName = cluster.GetPodComponentName(o.Pod) + } - // cluster name is not specified, get from pod label - if o.name == "" { - if name, ok := pod.Annotations[constant.AppInstanceLabelKey]; !ok { - return fmt.Errorf("failed to find the cluster to which the instance belongs") - } else { - o.name = name + // cannot infer characterType from pod directly (neither from pod annotation nor pod label) + // so we have to get cluster definition first to get characterType + // opt 2. specified cluster name + // 2.1 get cluster by name + if o.targetCluster, err = cluster.GetClusterByName(o.Dynamic, o.clusterName, o.Namespace); err != nil { + return err + } + // get cluster def + if o.targetClusterDef, err = cluster.GetClusterDefByName(o.Dynamic, o.targetCluster.Spec.ClusterDefRef); err != nil { + return err + } + + // 2.2 fill component name, use the first component by default + if len(o.componentName) == 0 { + o.component = &o.targetCluster.Spec.ComponentSpecs[0] + o.componentName = o.component.Name + } else { + // verify component + if o.component = o.targetCluster.Spec.GetComponentByName(o.componentName); o.component == nil { + return fmt.Errorf("failed to get component %s. Check the list of components use: \n\tkbcli cluster list-components %s -n %s", o.componentName, o.clusterName, o.Namespace) } } - info, err := o.getConnectionInfo() - if err != nil { + // 2.3 get character type + if o.componentDef = o.targetClusterDef.GetComponentDefByName(o.component.ComponentDefRef); o.componentDef == nil { + return fmt.Errorf("failed to get component def :%s", o.component.ComponentDefRef) + } + + // 2.4. get pod to connect, make sure o.clusterName, o.componentName are set before this step + if len(o.PodName) == 0 { + if err = o.getTargetPod(); err != nil { + return err + } + if o.Pod, err = o.Client.CoreV1().Pods(o.Namespace).Get(context.TODO(), o.PodName, metav1.GetOptions{}); err != nil { + return err + } + } + return nil +} + +// connect create parameters for connecting cluster and connect +func (o *ConnectOptions) connect() error { + if o.componentDef == nil { + return fmt.Errorf("component def is not initialized") + } + + var err error + + if o.engine, err = engine.New(o.componentDef.CharacterType); err != nil { return err } - o.Command = buildCommand(info) - o.Pod = pod + var authInfo *engine.AuthInfo + if len(o.userName) > 0 { + authInfo = &engine.AuthInfo{} + authInfo.UserName = o.userName + authInfo.UserPasswd = o.userPasswd + } else if authInfo, err = o.getAuthInfo(); err != nil { + return err + } + + o.ExecOptions.ContainerName = o.engine.Container() + o.ExecOptions.Command = o.engine.ConnectCommand(authInfo) + if klog.V(1).Enabled() { + fmt.Fprintf(o.Out, "connect with cmd: %s", o.ExecOptions.Command) + } return o.ExecOptions.Run() } +func (o *ConnectOptions) getAuthInfo() (*engine.AuthInfo, error) { + // select secrets by labels, prefer admin account + labels := fmt.Sprintf("%s=%s,%s=%s,%s=%s", + constant.AppInstanceLabelKey, o.clusterName, + constant.KBAppComponentLabelKey, o.componentName, + constant.ClusterAccountLabelKey, (string)(appsv1alpha1.AdminAccount), + ) + + secrets, err := o.Client.CoreV1().Secrets(o.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labels}) + if err != nil { + return nil, fmt.Errorf("failed to list secrets for cluster %s, component %s, err %v", o.clusterName, o.componentName, err) + } + if len(secrets.Items) == 0 { + return nil, nil + } + return &engine.AuthInfo{ + UserName: string(secrets.Items[0].Data["username"]), + UserPasswd: string(secrets.Items[0].Data["password"]), + }, nil +} + func (o *ConnectOptions) getTargetPod() error { - infos := cluster.GetSimpleInstanceInfos(o.Dynamic, o.name, o.Namespace) + // guarantee cluster name and component name are set + if len(o.clusterName) == 0 { + return fmt.Errorf("cluster name is not set yet") + } + if len(o.componentName) == 0 { + return fmt.Errorf("component name is not set yet") + } + + // get instantces for given cluster name and component name + infos := cluster.GetSimpleInstanceInfosForComponent(o.Dynamic, o.clusterName, o.componentName, o.Namespace) if len(infos) == 0 || infos[0].Name == computil.ComponentStatusDefaultPodName { return fmt.Errorf("failed to find the instance to connect, please check cluster status") } - // first element is the default instance to connect o.PodName = infos[0].Name // print instance info that we connect @@ -202,11 +323,16 @@ func (o *ConnectOptions) getTargetPod() error { } func (o *ConnectOptions) getConnectionInfo() (*engine.ConnectionInfo, error) { + // make sure component and componentDef are set before this step + if o.component == nil || o.componentDef == nil { + return nil, fmt.Errorf("failed to get component or component def") + } + info := &engine.ConnectionInfo{} getter := cluster.ObjectsGetter{ Client: o.Client, Dynamic: o.Dynamic, - Name: o.name, + Name: o.clusterName, Namespace: o.Namespace, GetOptions: cluster.GetOptions{ WithClusterDef: true, @@ -230,9 +356,7 @@ func (o *ConnectOptions) getConnectionInfo() (*engine.ConnectionInfo, error) { // TODO: now the primary component is the first component, that may not be correct, // maybe show all components connection info in the future. - primaryCompDef := objs.ClusterDef.Spec.ComponentDefs[0] - primaryComp := cluster.FindClusterComp(objs.Cluster, primaryCompDef.Name) - internalSvcs, externalSvcs := cluster.GetComponentServices(objs.Services, primaryComp) + internalSvcs, externalSvcs := cluster.GetComponentServices(objs.Services, o.component) switch { case len(externalSvcs) > 0: // cluster has public endpoint @@ -250,14 +374,7 @@ func (o *ConnectOptions) getConnectionInfo() (*engine.ConnectionInfo, error) { return nil, fmt.Errorf("failed to find any cluster endpoints") } - info.Command, info.Args, err = getCompCommandArgs(&primaryCompDef) - if err != nil { - return nil, err - } - - // get engine - o.engine, err = engine.New(objs.ClusterDef.Spec.ComponentDefs[0].CharacterType) - if err != nil { + if o.engine, err = engine.New(o.componentDef.CharacterType); err != nil { return nil, err } @@ -297,6 +414,7 @@ func getUserAndPassword(clusterDef *appsv1alpha1.ClusterDefinition, secrets *cor for i, s := range secrets.Items { if strings.Contains(s.Name, "conn-credential") { secret = secrets.Items[i] + break } } user, err = getSecretVal(&secret, "username") @@ -308,81 +426,3 @@ func getUserAndPassword(clusterDef *appsv1alpha1.ClusterDefinition, secrets *cor password, err = getSecretVal(&secret, passwordKey) return user, password, err } - -func getCompCommandArgs(compDef *appsv1alpha1.ClusterComponentDefinition) ([]string, []string, error) { - failErr := fmt.Errorf("failed to find the connection command") - if compDef == nil || compDef.SystemAccounts == nil || - compDef.SystemAccounts.CmdExecutorConfig == nil { - return nil, nil, failErr - } - - execCfg := compDef.SystemAccounts.CmdExecutorConfig - command := execCfg.Command - if len(command) == 0 { - return nil, nil, failErr - } - return command, execCfg.Args, nil -} - -// buildCommand build connection command by SystemAccounts.CmdExecutorConfig. -// CLI should not be coupled to a specific engine, so read command info from -// clusterDefinition, but now these information is used to create system -// accounts, we need to do some special handling. -// -// TODO: Refactoring using command channel -// examples of info.Args are: -// mysql : -// command: -// - mysql -// args: -// - -u$(MYSQL_ROOT_USER) -// - -p$(MYSQL_ROOT_PASSWORD) -// - -h -// - $(KB_ACCOUNT_ENDPOINT) -// - -e -// - $(KB_ACCOUNT_STATEMENT) -// but in redis, it looks like following: -// redis : -// command: -// - sh -// - -c -// args: -// - "redis-cli -h $(KB_ACCOUNT_ENDPOINT) $(KB_ACCOUNT_STATEMENT)" -func buildCommand(info *engine.ConnectionInfo) []string { - result := make([]string, 0) - var extraCmd string - // prepare commands - if len(info.Command) == 1 { - // append [sh -c] - result = append(result, "sh", "-c") - extraCmd = info.Command[0] - } else { - result = append(result, info.Command...) - } - // prepare args - args := buildArgs(info.Args, extraCmd) - result = append(result, args) - return result -} - -func buildArgs(args []string, extraCmd string) string { - result := make([]string, 0) - if len(extraCmd) > 0 { - result = append(result, extraCmd) - } - for i := 0; i < len(args); i++ { - arg := args[i] - // skip command - if arg == "-c" || arg == "-e" { - i++ - continue - } - - arg = strings.Replace(arg, "$(KB_ACCOUNT_ENDPOINT)", "127.0.0.1", 1) - arg = strings.Replace(arg, "$(KB_ACCOUNT_STATEMENT)", "", 1) - arg = strings.Replace(arg, "(", "", 1) - arg = strings.Replace(arg, ")", "", 1) - result = append(result, strings.TrimSpace(arg)) - } - return strings.Join(result, " ") -} diff --git a/internal/cli/cmd/cluster/connect_test.go b/internal/cli/cmd/cluster/connect_test.go index 01ab437b0..18b69e86e 100644 --- a/internal/cli/cmd/cluster/connect_test.go +++ b/internal/cli/cmd/cluster/connect_test.go @@ -31,7 +31,6 @@ import ( clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" - "github.com/apecloud/kubeblocks/internal/cli/engine" "github.com/apecloud/kubeblocks/internal/cli/exec" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" @@ -84,33 +83,53 @@ var _ = Describe("connection", func() { Expect(cmd).ShouldNot(BeNil()) }) - It("connection", func() { + It("validate", func() { o := &ConnectOptions{ExecOptions: exec.NewExecOptions(tf, streams)} By("specified more than one cluster") - Expect(o.connect([]string{"c1", "c2"})).Should(HaveOccurred()) + Expect(o.validate([]string{"c1", "c2"})).Should(HaveOccurred()) By("without cluster name") - Expect(o.connect(nil)).Should(HaveOccurred()) + Expect(o.validate(nil)).Should(HaveOccurred()) + + Expect(o.validate([]string{clusterName})).Should(Succeed()) + + // set instance name and cluster name, should fail + o.PodName = "test-pod-0" + Expect(o.validate([]string{clusterName})).Should(HaveOccurred()) + o.componentName = "test-component" + Expect(o.validate([]string{})).Should(HaveOccurred()) + + // unset pod name + o.PodName = "" + Expect(o.validate([]string{clusterName})).Should(Succeed()) + // unset component name as well + o.componentName = "" + Expect(o.validate([]string{clusterName})).Should(Succeed()) + }) - By("specify cluster name") - Expect(o.ExecOptions.Complete()).Should(Succeed()) - _ = o.connect([]string{clusterName}) + It("complete by cluster name", func() { + o := &ConnectOptions{ExecOptions: exec.NewExecOptions(tf, streams)} + Expect(o.validate([]string{clusterName})).Should(Succeed()) + Expect(o.complete()).Should(Succeed()) Expect(o.Pod).ShouldNot(BeNil()) }) - It("show example", func() { + It("complete by pod name", func() { o := &ConnectOptions{ExecOptions: exec.NewExecOptions(tf, streams)} - Expect(o.ExecOptions.Complete()).Should(Succeed()) - - By("without args") - Expect(o.runShowExample(nil)).Should(HaveOccurred()) + o.PodName = "test-pod-0" + Expect(o.validate([]string{})).Should(Succeed()) + Expect(o.complete()).Should(Succeed()) + Expect(o.Pod).ShouldNot(BeNil()) + }) - By("specify more than one cluster") - Expect(o.runShowExample([]string{"c1", "c2"})).Should(HaveOccurred()) + It("show example", func() { + o := &ConnectOptions{ExecOptions: exec.NewExecOptions(tf, streams)} + Expect(o.validate([]string{clusterName})).Should(Succeed()) + Expect(o.complete()).Should(Succeed()) By("specify one cluster") - Expect(o.runShowExample([]string{clusterName})).Should(Succeed()) + Expect(o.runShowExample()).Should(Succeed()) }) It("getUserAndPassword", func() { @@ -131,41 +150,6 @@ var _ = Describe("connection", func() { Expect(u).Should(Equal(user)) Expect(p).Should(Equal(password)) }) - - It("build connect command", func() { - type argsCases struct { - command []string - args []string - expect []string - } - - testCases := []argsCases{ - { - command: []string{"mysql"}, - args: []string{"-h$(KB_ACCOUNT_ENDPOINT)", "-e", "$(KB_ACCOUNT_STATEMENT)"}, - expect: []string{"sh", "-c", "mysql -h127.0.0.1"}, - }, - { - command: []string{"psql"}, - args: []string{"-h$(KB_ACCOUNT_ENDPOINT)", "-c", "$(KB_ACCOUNT_STATEMENT)"}, - expect: []string{"sh", "-c", "psql -h127.0.0.1"}, - }, - { - command: []string{"sh", "-c"}, - args: []string{"redis-cli -h $(KB_ACCOUNT_ENDPOINT) $(KB_ACCOUNT_STATEMENT)"}, - expect: []string{"sh", "-c", "redis-cli -h 127.0.0.1"}, - }, - } - - for _, testCase := range testCases { - info := &engine.ConnectionInfo{ - Command: testCase.command, - Args: testCase.args, - } - result := buildCommand(info) - Expect(result).Should(BeEquivalentTo(testCase.expect)) - } - }) }) func mockPod() *corev1.Pod { diff --git a/internal/cli/cmd/cluster/logs.go b/internal/cli/cmd/cluster/logs.go index b897bcfaf..c4eceb113 100644 --- a/internal/cli/cmd/cluster/logs.go +++ b/internal/cli/cmd/cluster/logs.go @@ -49,7 +49,7 @@ var ( # Display only the most recent 20 lines from cluster mycluster with default primary instance (stdout) kbcli cluster logs mycluster --tail=20 - # Display stdout info of specific instance my-instance-0 (cluster name comes from annotation app.kubernetes.io/instance) + # Display stdout info of specific instance my-instance-0 (cluster name comes from annotation app.kubernetes.io/instance) kbcli cluster logs --instance my-instance-0 # Return snapshot logs from cluster mycluster with specific instance my-instance-0 (stdout) diff --git a/internal/cli/create/template/migration_template.cue b/internal/cli/create/template/migration_template.cue index 074981680..718f81e3f 100644 --- a/internal/cli/create/template/migration_template.cue +++ b/internal/cli/create/template/migration_template.cue @@ -14,76 +14,76 @@ // required, command line input options for parameters and flags options: { - name: string - namespace: string - teplate: string - taskType: string - source: string + name: string + namespace: string + teplate: string + taskType: string + source: string sourceEndpointModel: {} - sink: string + sink: string sinkEndpointModel: {} - migrationObject: [...] + migrationObject: [...] migrationObjectModel: {} - steps: [...] - stepsModel: [...] - tolerations: [...] - tolerationModel: {} - resources: [...] - resourceModel: {} - serverId: int + steps: [...] + stepsModel: [...] + tolerations: [...] + tolerationModel: {} + resources: [...] + resourceModel: {} + serverId: int } // required, k8s api resource content content: { apiVersion: "datamigration.apecloud.io/v1alpha1" - kind: "MigrationTask" + kind: "MigrationTask" metadata: { - name: options.name - namespace: options.namespace - } + name: options.name + namespace: options.namespace + } spec: { - taskType: options.taskType - template: options.template - sourceEndpoint: options.sourceEndpointModel - sinkEndpoint: options.sinkEndpointModel - initialization: { - steps: options.stepsModel - config: { - preCheck: { - resource: { - limits: options.resourceModel["precheck"] - }, - tolerations: options.tolerationModel["precheck"] - } - initStruct: { - resource: { - limits: options.resourceModel["init-struct"] - }, - tolerations: options.tolerationModel["init-struct"] - } - initData: { - resource: { - limits: options.resourceModel["init-data"] - }, - tolerations: options.tolerationModel["init-data"] - } - } - } - cdc: { - config: { - resource: { - limits: options.resourceModel["cdc"] - }, - tolerations: options.tolerationModel["cdc"], - param: { - "extractor.server_id": options.serverId - } - } - } - migrationObj: options.migrationObjectModel - globalTolerations: options.tolerationModel["global"] - globalResources: { - limits: options.resourceModel["global"] - } - } + taskType: options.taskType + template: options.template + sourceEndpoint: options.sourceEndpointModel + sinkEndpoint: options.sinkEndpointModel + initialization: { + steps: options.stepsModel + config: { + preCheck: { + resource: { + limits: options.resourceModel["precheck"] + } + tolerations: options.tolerationModel["precheck"] + } + initStruct: { + resource: { + limits: options.resourceModel["init-struct"] + } + tolerations: options.tolerationModel["init-struct"] + } + initData: { + resource: { + limits: options.resourceModel["init-data"] + } + tolerations: options.tolerationModel["init-data"] + } + } + } + cdc: { + config: { + resource: { + limits: options.resourceModel["cdc"] + } + tolerations: options.tolerationModel["cdc"] + param: { + "extractor.server_id": options.serverId + } + } + } + migrationObj: options.migrationObjectModel + globalTolerations: options.tolerationModel["global"] + globalResources: { + limits: options.resourceModel["global"] + } + } } diff --git a/internal/cli/engine/types.go b/internal/cli/engine/types.go deleted file mode 100644 index 15ae743d8..000000000 --- a/internal/cli/engine/types.go +++ /dev/null @@ -1,17 +0,0 @@ -/* -Copyright ApeCloud, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package engine diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 48a8b65c6..a7d45717d 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -669,3 +669,22 @@ func CombineLabels(labels map[string]string) string { } return strings.TrimSuffix(labelStr, ",") } + +func BuildComponentNameLables(prefix string, names []string) string { + return buildLableSelectors(prefix, constant.KBAppComponentLabelKey, names) +} + +// BuildLableSelectors build the label selector by given lable key, the label selector is +// like "label-key in (name1, name2)" +func buildLableSelectors(prefix string, key string, names []string) string { + if len(names) == 0 { + return prefix + } + + label := fmt.Sprintf("%s in (%s)", key, strings.Join(names, ",")) + if len(prefix) == 0 { + return label + } else { + return prefix + "," + label + } +} diff --git a/internal/sqlchannel/client.go b/internal/sqlchannel/client.go index ec0860353..d7cf8d4e0 100644 --- a/internal/sqlchannel/client.go +++ b/internal/sqlchannel/client.go @@ -169,11 +169,10 @@ type OperationHTTPClient struct { httpRequestPrefix string RequestTimeout time.Duration containerName string - exec *exec.ExecOptions } -// NewHTTPClientWithPod create a new OperationHTTPClient with pod -func NewHTTPClientWithPod(exec *exec.ExecOptions, pod *corev1.Pod, characterType string) (*OperationHTTPClient, error) { +// NewHTTPClientWithChannelPod create a new OperationHTTPClient with sqlchannel container +func NewHTTPClientWithChannelPod(pod *corev1.Pod, characterType string) (*OperationHTTPClient, error) { var ( err error ) @@ -199,13 +198,12 @@ func NewHTTPClientWithPod(exec *exec.ExecOptions, pod *corev1.Pod, characterType httpRequestPrefix: fmt.Sprintf(HTTPRequestPrefx, port, characterType), RequestTimeout: 10 * time.Second, containerName: container, - exec: exec, } return client, nil } // SendRequest exec sql operation, this is a blocking operation and it will use pod EXEC subresource to send an http request to the probe pod -func (cli *OperationHTTPClient) SendRequest(request SQLChannelRequest) (SQLChannelResponse, error) { +func (cli *OperationHTTPClient) SendRequest(exec *exec.ExecOptions, request SQLChannelRequest) (SQLChannelResponse, error) { var ( response = SQLChannelResponse{} strBuffer bytes.Buffer @@ -216,12 +214,12 @@ func (cli *OperationHTTPClient) SendRequest(request SQLChannelRequest) (SQLChann if jsonData, err := json.Marshal(request); err != nil { return response, err } else { - cli.exec.ContainerName = cli.containerName - cli.exec.Command = []string{"sh", "-c", cli.httpRequestPrefix + " -d '" + string(jsonData) + "'"} + exec.ContainerName = cli.containerName + exec.Command = []string{"sh", "-c", cli.httpRequestPrefix + " -d '" + string(jsonData) + "'"} } // redirect output to strBuffer to be parsed later - if err = cli.exec.RunWithRedirect(&strBuffer, &errBuffer); err != nil { + if err = exec.RunWithRedirect(&strBuffer, &errBuffer); err != nil { return response, err } diff --git a/internal/cli/engine/client.go b/internal/sqlchannel/engine/client.go similarity index 100% rename from internal/cli/engine/client.go rename to internal/sqlchannel/engine/client.go diff --git a/internal/cli/engine/engine.go b/internal/sqlchannel/engine/engine.go similarity index 92% rename from internal/cli/engine/engine.go rename to internal/sqlchannel/engine/engine.go index 4559c3090..2d7f45dff 100644 --- a/internal/cli/engine/engine.go +++ b/internal/sqlchannel/engine/engine.go @@ -22,29 +22,24 @@ import ( "strings" ) -// ClusterDefinition ComponentDefRef Const Define const ( stateMysql = "mysql" statePostgreSQL = "postgresql" stateRedis = "redis" ) +// AuthInfo is the authentication information for the database +type AuthInfo struct { + UserName string + UserPasswd string +} + type Interface interface { - ConnectCommand() []string + ConnectCommand(info *AuthInfo) []string Container() string ConnectExample(info *ConnectionInfo, client string) string } -type ConnectionInfo struct { - Host string - User string - Password string - Database string - Port string - Command []string - Args []string -} - type EngineInfo struct { Client string Container string @@ -53,8 +48,6 @@ type EngineInfo struct { Database string } -type buildConnectExample func(info *ConnectionInfo) string - func New(typeName string) (Interface, error) { switch typeName { case stateMysql: @@ -68,6 +61,16 @@ func New(typeName string) (Interface, error) { } } +type ConnectionInfo struct { + Host string + User string + Password string + Database string + Port string +} + +type buildConnectExample func(info *ConnectionInfo) string + func buildExample(info *ConnectionInfo, client string, examples map[ClientType]buildConnectExample) string { // if client is not specified, output all examples if len(client) == 0 { diff --git a/internal/cli/engine/engine_test.go b/internal/sqlchannel/engine/engine_test.go similarity index 62% rename from internal/cli/engine/engine_test.go rename to internal/sqlchannel/engine/engine_test.go index 81f2254e3..bbbd0510c 100644 --- a/internal/cli/engine/engine_test.go +++ b/internal/sqlchannel/engine/engine_test.go @@ -23,17 +23,20 @@ import ( var _ = Describe("Engine", func() { It("new mysql engine", func() { - typeName := stateMysql - engine, _ := New(typeName) - Expect(engine).ShouldNot(BeNil()) - - url := engine.ConnectCommand() - Expect(len(url)).Should(Equal(3)) - - url = engine.ConnectCommand() - Expect(len(url)).Should(Equal(3)) - - Expect(engine.Container()).Should(Equal("mysql")) + for _, typeName := range []string{stateMysql, statePostgreSQL, stateRedis} { + engine, _ := New(typeName) + Expect(engine).ShouldNot(BeNil()) + + url := engine.ConnectCommand(nil) + Expect(len(url)).Should(Equal(3)) + + url = engine.ConnectCommand(nil) + Expect(len(url)).Should(Equal(3)) + // it is a tricky way to check the container name + // for the moment, we only support mysql, postgresql and redis + // and the container name is the same as the state name + Expect(engine.Container()).Should(Equal(typeName)) + } }) It("new unknown engine", func() { diff --git a/internal/cli/engine/mysql.go b/internal/sqlchannel/engine/mysql.go similarity index 91% rename from internal/cli/engine/mysql.go rename to internal/sqlchannel/engine/mysql.go index facb2b933..494a286b5 100644 --- a/internal/cli/engine/mysql.go +++ b/internal/sqlchannel/engine/mysql.go @@ -34,6 +34,7 @@ func newMySQL() *mysql { Client: "mysql", Container: "mysql", PasswordEnv: "$MYSQL_ROOT_PASSWORD", + UserEnv: "$MYSQL_ROOT_USER", Database: "mysql", }, examples: map[ClientType]buildConnectExample{ @@ -246,8 +247,21 @@ DATABASE_URL='mysql://%s:%s@%s:%s/%s' } } -func (m *mysql) ConnectCommand() []string { - mysqlCmd := []string{"MYSQL_PWD=" + m.info.PasswordEnv, m.info.Client} +func (m *mysql) ConnectCommand(connectInfo *AuthInfo) []string { + userName := m.info.UserEnv + userPass := m.info.PasswordEnv + + if connectInfo != nil { + userName = connectInfo.UserName + userPass = connectInfo.UserPasswd + } + + // avoid use env variables + // MYSQL_PWD is deprecated as of MySQL 8.0; expect it to be removed in a future version of MySQL. + // ref to mysql manual for more details. + // https://dev.mysql.com/doc/refman/8.0/en/environment-variables.html + mysqlCmd := []string{fmt.Sprintf("%s -u%s -p%s", m.info.Client, userName, userPass)} + return []string{"sh", "-c", strings.Join(mysqlCmd, " ")} } diff --git a/internal/cli/engine/mysql_test.go b/internal/sqlchannel/engine/mysql_test.go similarity index 100% rename from internal/cli/engine/mysql_test.go rename to internal/sqlchannel/engine/mysql_test.go diff --git a/internal/cli/engine/postgresql.go b/internal/sqlchannel/engine/postgresql.go similarity index 91% rename from internal/cli/engine/postgresql.go rename to internal/sqlchannel/engine/postgresql.go index 24912774c..5602121eb 100644 --- a/internal/cli/engine/postgresql.go +++ b/internal/sqlchannel/engine/postgresql.go @@ -33,8 +33,8 @@ func newPostgreSQL() *postgresql { info: EngineInfo{ Client: "psql", Container: "postgresql", - PasswordEnv: "$POSTGRES_PASSWORD", - UserEnv: "$POSTGRES_USER", + PasswordEnv: "$PGPASSWORD", + UserEnv: "$PGUSER", Database: "postgres", }, examples: map[ClientType]buildConnectExample{ @@ -228,8 +228,18 @@ DATABASE_URL='postgresql://%s:%s@%s:%s/%s' } } -func (m *postgresql) ConnectCommand() []string { - cmd := []string{"PGPASSWORD=" + m.info.PasswordEnv, "PGUSER=" + m.info.UserEnv, m.info.Client} +func (m *postgresql) ConnectCommand(connectInfo *AuthInfo) []string { + userName := m.info.UserEnv + userPass := m.info.PasswordEnv + + if connectInfo != nil { + userName = connectInfo.UserName + userPass = connectInfo.UserPasswd + } + + // pls refer to PostgreSQL documentation for more details + // https://www.postgresql.org/docs/current/libpq-envars.html + cmd := []string{fmt.Sprintf("PGUSER=%s PGPASSWORD=%s PGDATABASE=%s %s", userName, userPass, m.info.Database, m.info.Client)} return []string{"sh", "-c", strings.Join(cmd, " ")} } diff --git a/internal/cli/engine/redis.go b/internal/sqlchannel/engine/redis.go similarity index 68% rename from internal/cli/engine/redis.go rename to internal/sqlchannel/engine/redis.go index dc831857d..aee485801 100644 --- a/internal/cli/engine/redis.go +++ b/internal/sqlchannel/engine/redis.go @@ -16,6 +16,8 @@ limitations under the License. package engine +import "strings" + type redis struct { info EngineInfo examples map[ClientType]buildConnectExample @@ -23,19 +25,28 @@ type redis struct { func newRedis() *redis { return &redis{ - info: EngineInfo{}, + info: EngineInfo{ + Client: "redis-cli", + Container: "redis", + }, examples: map[ClientType]buildConnectExample{}, } } -func (r redis) ConnectCommand() []string { - // TODO implement me - panic("implement me") +func (r redis) ConnectCommand(connectInfo *AuthInfo) []string { + redisCmd := []string{ + "redis-cli", + } + + if connectInfo != nil { + redisCmd = append(redisCmd, "--user", connectInfo.UserName) + redisCmd = append(redisCmd, "--pass", connectInfo.UserPasswd) + } + return []string{"sh", "-c", strings.Join(redisCmd, " ")} } func (r redis) Container() string { - // TODO implement me - panic("implement me") + return r.info.Container } func (r redis) ConnectExample(info *ConnectionInfo, client string) string { diff --git a/internal/cli/engine/suite_test.go b/internal/sqlchannel/engine/suite_test.go similarity index 100% rename from internal/cli/engine/suite_test.go rename to internal/sqlchannel/engine/suite_test.go From 565378786fcba1149fdbc32e5f17209a36a62aef Mon Sep 17 00:00:00 2001 From: shanshanying Date: Fri, 14 Apr 2023 18:01:00 +0800 Subject: [PATCH 033/439] fix: caseinsentive superuser role in postgres (#2600) --- .../internal/binding/postgres/postgres.go | 2 +- .../binding/postgres/postgres_test.go | 31 +++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/cmd/probe/internal/binding/postgres/postgres.go b/cmd/probe/internal/binding/postgres/postgres.go index 9bb886811..750c9a5a2 100644 --- a/cmd/probe/internal/binding/postgres/postgres.go +++ b/cmd/probe/internal/binding/postgres/postgres.go @@ -428,7 +428,7 @@ func (pgOps *PostgresOperations) managePrivillege(ctx context.Context, req *bind object = UserInfo{} sqlTplRend = func(user UserInfo) string { - if SuperUserRole.EqualTo(user.UserName) { + if SuperUserRole.EqualTo(user.RoleName) { if op == GrantUserRoleOp { return "ALTER USER " + user.UserName + " WITH SUPERUSER;" } else { diff --git a/cmd/probe/internal/binding/postgres/postgres_test.go b/cmd/probe/internal/binding/postgres/postgres_test.go index f99b7c19a..2f4ab7db3 100644 --- a/cmd/probe/internal/binding/postgres/postgres_test.go +++ b/cmd/probe/internal/binding/postgres/postgres_test.go @@ -194,10 +194,10 @@ func TestPostgresIntegration(t *testing.T) { } // SETUP TESTS, run as `postgre` to manage accounts -// 1. exprot PGUSER=potgres -// 2. exprot PGPASSWORD= +// 1. export PGUSER=potgres +// 2. export PGPASSWORD= // 4. export POSTGRES_TEST_CONN_URL="postgres://${PGUSER}:${PGPASSWORD}@localhost:5432/postgres" -// 5. `go test -v -count=1 ./bindings/postgres -run ^TestPostgresIntegrationAccounts` +// 5. `go test -v -count=1 ./cmd/probe/internal/binding/postgres -run ^TestPostgresIntegrationAccounts` func TestPostgresIntegrationAccounts(t *testing.T) { url := os.Getenv("POSTGRES_TEST_CONN_URL") if url == "" { @@ -280,9 +280,16 @@ func TestPostgresIntegrationAccounts(t *testing.T) { res, err = b.Invoke(ctx, req) assertResponse(t, res, err, RespEveFail) - req.Metadata["roleName"] = roleName - res, err = b.Invoke(ctx, req) - assertResponse(t, res, err, RespEveSucc) + for _, roleType := range []RoleType{ReadOnlyRole, ReadWriteRole, SuperUserRole} { + roleStr := (string)(roleType) + req.Metadata["roleName"] = roleStr + res, err = b.Invoke(ctx, req) + assertResponse(t, res, err, RespEveSucc) + + req.Metadata["roleName"] = strings.ToUpper(roleStr) + res, err = b.Invoke(ctx, req) + assertResponse(t, res, err, RespEveSucc) + } // revoke role req = &bindings.InvokeRequest{ @@ -300,10 +307,16 @@ func TestPostgresIntegrationAccounts(t *testing.T) { res, err = b.Invoke(ctx, req) assertResponse(t, res, err, RespEveFail) - req.Metadata["roleName"] = roleName - res, err = b.Invoke(ctx, req) - assertResponse(t, res, err, RespEveSucc) + for _, roleType := range []RoleType{ReadOnlyRole, ReadWriteRole, SuperUserRole} { + roleStr := (string)(roleType) + req.Metadata["roleName"] = roleStr + res, err = b.Invoke(ctx, req) + assertResponse(t, res, err, RespEveSucc) + req.Metadata["roleName"] = strings.ToUpper(roleStr) + res, err = b.Invoke(ctx, req) + assertResponse(t, res, err, RespEveSucc) + } // delete user req = &bindings.InvokeRequest{ Operation: DeleteUserOp, From c98dfe91c0588d4d18031b45c6bc585801e64c4a Mon Sep 17 00:00:00 2001 From: xingran Date: Fri, 14 Apr 2023 19:02:22 +0800 Subject: [PATCH 034/439] chore: remove postgresql readiness probe because k8s event limit when failover (#2607) --- deploy/postgresql/templates/clusterdefinition.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 8b888b31d..82cae01f6 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -100,20 +100,6 @@ spec: runAsUser: 0 command: - /kb-scripts/setup.sh - readinessProbe: - failureThreshold: 6 - initialDelaySeconds: 10 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 5 - exec: - command: - - /bin/sh - - -c - - -ee - - | - exec pg_isready -U {{ default "postgres" | quote }} -h 127.0.0.1 -p 5432 - [ -f /postgresql/tmp/.initialized ] || [ -f /postgresql/.initialized ] volumeMounts: - name: dshm mountPath: /dev/shm From 3d679a5db45dfaeefcecbbc08910b7a2053024bb Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Fri, 14 Apr 2023 19:17:00 +0800 Subject: [PATCH 035/439] chore: upgrade go version to 1.20 (#2574) --- .github/localflows/github-action-locally.sh | 2 +- .github/workflows/cicd-merge.yml | 2 +- .github/workflows/cicd-pull-request.yml | 4 ++-- .github/workflows/cicd-push.yml | 6 +++--- .github/workflows/release-image.yml | 4 ++-- .github/workflows/release-publish.yml | 2 +- Makefile | 4 ++-- cmd/probe/README.md | 2 +- cmd/reloader/README.md | 2 +- cmd/reloader/tools/README.md | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile-dev | 2 +- docker/Dockerfile-tools | 2 +- docs/DEVELOPING.md | 2 +- go.mod | 8 ++------ go.sum | 4 ++-- internal/cli/util/util.go | 5 ----- 17 files changed, 23 insertions(+), 32 deletions(-) diff --git a/.github/localflows/github-action-locally.sh b/.github/localflows/github-action-locally.sh index 77ca6fb80..d28cff034 100755 --- a/.github/localflows/github-action-locally.sh +++ b/.github/localflows/github-action-locally.sh @@ -6,4 +6,4 @@ if ! [ -x "$(which act)" ]; then fi # run act -act --reuse --platform self-hosted=jashbook/golang-lint:1.19-latest --workflows .github/localflows/cicd-local.yml +act --reuse --platform self-hosted=jashbook/golang-lint:1.20-latest --workflows .github/localflows/cicd-local.yml diff --git a/.github/workflows/cicd-merge.yml b/.github/workflows/cicd-merge.yml index 9a6730bf7..fae19bc6e 100644 --- a/.github/workflows/cicd-merge.yml +++ b/.github/workflows/cicd-merge.yml @@ -23,7 +23,7 @@ jobs: - name: setup Go uses: actions/setup-go@v3 with: - go-version: '1.19' + go-version: '1.20' - name: start minikube run: | diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index c0c98b1b7..51e2f34e0 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -73,7 +73,7 @@ jobs: MAKE_OPS: "build-manager-image" IMG: "apecloud/kubeblocks" VERSION: "check" - GO_VERSION: 1.19 + GO_VERSION: 1.20 secrets: inherit check-tools-image: @@ -86,7 +86,7 @@ jobs: MAKE_OPS: "build-tools-image" IMG: "apecloud/kubeblocks-tools" VERSION: "check" - GO_VERSION: 1.19 + GO_VERSION: 1.20 secrets: inherit check-helm: diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index c99189f7f..3661c19b2 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -81,7 +81,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: 1.19 + go-version: 1.20 - name: Check cli doc id: check-cli-doc @@ -187,7 +187,7 @@ jobs: MAKE_OPS: "build-manager-image" IMG: "apecloud/kubeblocks" VERSION: "check" - GO_VERSION: 1.19 + GO_VERSION: 1.20 secrets: inherit check-tools-image: @@ -200,7 +200,7 @@ jobs: MAKE_OPS: "build-tools-image" IMG: "apecloud/kubeblocks-tools" VERSION: "check" - GO_VERSION: 1.19 + GO_VERSION: 1.20 secrets: inherit check-helm: diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 2c8095b78..d9be93882 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -44,7 +44,7 @@ jobs: MAKE_OPS: "push-manager-image" IMG: "apecloud/kubeblocks" VERSION: "${{ needs.image-tag.outputs.tag-name }}" - GO_VERSION: 1.19 + GO_VERSION: 1.20 secrets: inherit release-tools-image: @@ -55,5 +55,5 @@ jobs: MAKE_OPS: "push-tools-image" IMG: "apecloud/kubeblocks-tools" VERSION: "${{ needs.image-tag.outputs.tag-name }}" - GO_VERSION: 1.19 + GO_VERSION: 1.20 secrets: inherit diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 8e48bb56b..79197d165 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -8,7 +8,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} TAG_NAME: ${{ github.ref_name }} - GO_VERSION: '1.19' + GO_VERSION: '1.20' CLI_NAME: 'kbcli' CLI_REPO: 'apecloud/kbcli' GITLAB_KBCLI_PROJECT_ID: 85948 diff --git a/Makefile b/Makefile index 77dd25c4e..b088b04dd 100644 --- a/Makefile +++ b/Makefile @@ -182,7 +182,7 @@ mod-vendor: module ## Run go mod vendor against go modules. .PHONY: module module: ## Run go mod tidy->verify against go modules. - $(GO) mod tidy -compat=1.19 + $(GO) mod tidy -compat=1.20 $(GO) mod verify TEST_PACKAGES ?= ./internal/... ./apis/... ./controllers/... ./cmd/... @@ -333,7 +333,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi .PHONY: reviewable reviewable: generate build-checks test check-license-header ## Run code checks to proceed with PR reviews. - $(GO) mod tidy -compat=1.19 + $(GO) mod tidy -compat=1.20 .PHONY: check-diff check-diff: reviewable ## Run git code diff checker. diff --git a/cmd/probe/README.md b/cmd/probe/README.md index 88d92a8c4..f845b64f2 100644 --- a/cmd/probe/README.md +++ b/cmd/probe/README.md @@ -17,7 +17,7 @@ You can get started with Probe, by any of the following methods: ## 2.1 Build -Compiler `Go 1.19+` (Generics Programming Support), checking the [Go Installation](https://go.dev/doc/install) to see how to install Go on your platform. +Compiler `Go 1.20+` (Generics Programming Support), checking the [Go Installation](https://go.dev/doc/install) to see how to install Go on your platform. Use `go build` to build and produce the `probe` binary file. The executable is produced under current directory. diff --git a/cmd/reloader/README.md b/cmd/reloader/README.md index 1ce1b4e52..3b7c1214f 100644 --- a/cmd/reloader/README.md +++ b/cmd/reloader/README.md @@ -15,7 +15,7 @@ You can get started with Reloader, by any of the following methods: ## 2.1 Build -Compiler `Go 1.19+` (Generics Programming Support), checking the [Go Installation](https://go.dev/doc/install) to see how to install Go on your platform. +Compiler `Go 1.20+` (Generics Programming Support), checking the [Go Installation](https://go.dev/doc/install) to see how to install Go on your platform. Use `make reloader` to build and produce the `reloader` binary file. The executable is produced under current directory. diff --git a/cmd/reloader/tools/README.md b/cmd/reloader/tools/README.md index 8f17ed74f..4aa3325fe 100644 --- a/cmd/reloader/tools/README.md +++ b/cmd/reloader/tools/README.md @@ -11,7 +11,7 @@ You can get started with cue-helper, by the following methods: ## 2.1 Build -Compiler `Go 1.19+` (Generics Programming Support), checking the [Go Installation](https://go.dev/doc/install) to see how to install Go on your platform. +Compiler `Go 1.20+` (Generics Programming Support), checking the [Go Installation](https://go.dev/doc/install) to see how to install Go on your platform. Use `make cue-helper` to build and produce the `cue-helper` binary file. The executable is produced under current directory. diff --git a/docker/Dockerfile b/docker/Dockerfile index 23d12f5b3..e6082854b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM --platform=${BUILDPLATFORM} golang:1.19 as builder +FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ## docker buildx buid injected build-args: #BUILDPLATFORM — matches the current machine. (e.g. linux/amd64) diff --git a/docker/Dockerfile-dev b/docker/Dockerfile-dev index d96661027..8ed81f28e 100644 --- a/docker/Dockerfile-dev +++ b/docker/Dockerfile-dev @@ -1,7 +1,7 @@ # Based on https://github.com/microsoft/vscode-dev-containers/tree/v0.245.2/containers/go/.devcontainer/base.Dockerfile # [Choice] Go version: 1, 1.19, 1.18, etc -ARG GOVERSION=1.19 +ARG GOVERSION=1.20 FROM golang:${GOVERSION}-bullseye # Copy library scripts to execute diff --git a/docker/Dockerfile-tools b/docker/Dockerfile-tools index 3c7f90e1b..3c10dbff6 100644 --- a/docker/Dockerfile-tools +++ b/docker/Dockerfile-tools @@ -1,6 +1,6 @@ # Build the kubeblocks tools binaries # includes kbcli, kubectl, and manager tools. -FROM --platform=${BUILDPLATFORM} golang:1.19 as builder +FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ## docker buildx buid injected build-args: #BUILDPLATFORM — matches the current machine. (e.g. linux/amd64) diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 1dc766e08..fb06aba1f 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -46,7 +46,7 @@ To build `KubeBlocks` on your own host, needs to install the following tools: - Make #### Install Go -Download and install [Go 1.19 or later](https://go.dev/doc/install). +Download and install [Go 1.20 or later](https://go.dev/doc/install). #### Install Make `KubeBlocks` uses `make` for a variety of build and test actions, and needs to be installed as appropriate for your platform: diff --git a/go.mod b/go.mod index 92113b6c7..e16a93f44 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/apecloud/kubeblocks -go 1.19 +go 1.20 require ( cuelang.org/go v0.4.3 @@ -342,7 +342,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect - go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect + go4.org/unsafe/assume-no-moving-gc v0.0.0-20230221090011-e4bae7ad2296 // indirect golang.org/x/mod v0.7.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sys v0.5.0 // indirect @@ -369,11 +369,7 @@ require ( ) replace ( - // github.com/google/certificate-transparency-go => github.com/google/certificate-transparency-go v1.1.3 - // github.com/coreos/etcd => github.com/coreos/etcd v3.5.5+incompatible - github.com/hashicorp/terraform => github.com/apecloud/terraform v1.3.0-20220927 github.com/spf13/afero => github.com/spf13/afero v1.2.2 go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.10.0 go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.10.0 -// go.etcd.io/etcd/v3 => go.etcd.io/etcd/v3 v3.5.5+incompatible ) diff --git a/go.sum b/go.sum index b1c5304f0..2f448ca9e 100644 --- a/go.sum +++ b/go.sum @@ -1883,8 +1883,8 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE= go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20230221090011-e4bae7ad2296 h1:QJ/xcIANMLApehfgPCHnfK1hZiaMmbaTVmPv7DAoTbo= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20230221090011-e4bae7ad2296/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index a7d45717d..45dfc39e6 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -38,7 +38,6 @@ import ( "time" "github.com/go-logr/logr" - "github.com/google/uuid" "github.com/pkg/errors" "golang.org/x/crypto/ssh" corev1 "k8s.io/api/core/v1" @@ -205,10 +204,6 @@ func DoWithRetry(ctx context.Context, logger logr.Logger, operation func() error return err } -func GenRequestID() string { - return uuid.New().String() -} - func PrintGoTemplate(wr io.Writer, tpl string, values interface{}) error { tmpl, err := template.New("output").Parse(tpl) if err != nil { From 4e3fe1a34ab958fd2a38a7c94be62d22c91abd3e Mon Sep 17 00:00:00 2001 From: dingben Date: Fri, 14 Apr 2023 19:19:27 +0800 Subject: [PATCH 036/439] feat: kbcli support cluster label (#2567) --- docs/user_docs/cli/cli.md | 1 + docs/user_docs/cli/kbcli_cluster.md | 1 + docs/user_docs/cli/kbcli_cluster_label.md | 73 +++++ internal/cli/cluster/printer.go | 17 ++ internal/cli/cmd/cluster/cluster.go | 1 + internal/cli/cmd/cluster/label.go | 340 ++++++++++++++++++++++ internal/cli/cmd/cluster/label_test.go | 81 ++++++ internal/cli/util/util.go | 10 +- 8 files changed, 521 insertions(+), 3 deletions(-) create mode 100644 docs/user_docs/cli/kbcli_cluster_label.md create mode 100644 internal/cli/cmd/cluster/label.go create mode 100644 internal/cli/cmd/cluster/label_test.go diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index 6b1ca96cd..337805f38 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -70,6 +70,7 @@ Cluster command. * [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account * [kbcli cluster hscale](kbcli_cluster_hscale.md) - Horizontally scale the specified components in the cluster. +* [kbcli cluster label](kbcli_cluster_label.md) - Update the labels on cluster * [kbcli cluster list](kbcli_cluster_list.md) - List clusters. * [kbcli cluster list-accounts](kbcli_cluster_list-accounts.md) - List accounts for a cluster * [kbcli cluster list-backup](kbcli_cluster_list-backup.md) - List backups. diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index dc0df42c4..9e900b0e6 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -58,6 +58,7 @@ Cluster command. * [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account * [kbcli cluster hscale](kbcli_cluster_hscale.md) - Horizontally scale the specified components in the cluster. +* [kbcli cluster label](kbcli_cluster_label.md) - Update the labels on cluster * [kbcli cluster list](kbcli_cluster_list.md) - List clusters. * [kbcli cluster list-accounts](kbcli_cluster_list-accounts.md) - List accounts for a cluster * [kbcli cluster list-backup](kbcli_cluster_list-backup.md) - List backups. diff --git a/docs/user_docs/cli/kbcli_cluster_label.md b/docs/user_docs/cli/kbcli_cluster_label.md new file mode 100644 index 000000000..928139be7 --- /dev/null +++ b/docs/user_docs/cli/kbcli_cluster_label.md @@ -0,0 +1,73 @@ +--- +title: kbcli cluster label +--- + +Update the labels on cluster + +``` +kbcli cluster label NAME [flags] +``` + +### Examples + +``` + # list label for clusters with specified name + kbcli cluster label mycluster --list + + # add label 'env' and value 'dev' for clusters with specified name + kbcli cluster label mycluster env=dev + + # add label 'env' and value 'dev' for all clusters + kbcli cluster label env=dev --all + + # add label 'env' and value 'dev' for the clusters that match the selector + kbcli cluster label env=dev -l type=mysql + + # update cluster with the label 'env' with value 'test', overwriting any existing value + kbcli cluster label mycluster --overwrite env=test + + # delete label env for clusters with specified name + kbcli cluster label mycluster env- +``` + +### Options + +``` + --all Select all cluster + --dry-run string[="unchanged"] Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for label + --list If true, display the labels of the clusters + --overwrite If true, allow labels to be overwritten, otherwise reject label updates that overwrite existing labels. + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli cluster](kbcli_cluster.md) - Cluster command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/internal/cli/cluster/printer.go b/internal/cli/cluster/printer.go index 16cb8ccc9..722fcd445 100644 --- a/internal/cli/cluster/printer.go +++ b/internal/cli/cluster/printer.go @@ -18,6 +18,7 @@ package cluster import ( "io" + "strings" corev1 "k8s.io/api/core/v1" @@ -33,6 +34,7 @@ const ( PrintInstances PrintType = "instances" PrintComponents PrintType = "components" PrintEvents PrintType = "events" + PrintLabels PrintType = "label" ) type PrinterOptions struct { @@ -86,6 +88,11 @@ var mapTblInfo = map[PrintType]tblInfo{ addRow: AddEventRow, getOptions: GetOptions{WithClusterDef: true, WithPod: true, WithEvent: true}, }, + PrintLabels: { + header: []interface{}{"NAME", "NAMESPACE"}, + addRow: AddLabelRow, + getOptions: GetOptions{}, + }, } // Printer prints cluster info @@ -124,6 +131,16 @@ func (p *Printer) GetterOptions() GetOptions { return p.getOptions } +func AddLabelRow(tbl *printer.TablePrinter, objs *ClusterObjects, opt *PrinterOptions) { + c := objs.GetClusterInfo() + info := []interface{}{c.Name, c.Namespace} + if opt.ShowLabels { + labels := strings.ReplaceAll(c.Labels, ",", "\n") + info = append(info, labels) + } + tbl.AddRow(info...) +} + func AddComponentRow(tbl *printer.TablePrinter, objs *ClusterObjects, opt *PrinterOptions) { components := objs.GetComponentInfo() for _, c := range components { diff --git a/internal/cli/cmd/cluster/cluster.go b/internal/cli/cmd/cluster/cluster.go index 796722792..fc6cc4824 100644 --- a/internal/cli/cmd/cluster/cluster.go +++ b/internal/cli/cmd/cluster/cluster.go @@ -50,6 +50,7 @@ func NewClusterCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr NewListInstancesCmd(f, streams), NewListComponentsCmd(f, streams), NewListEventsCmd(f, streams), + NewLabelCmd(f, streams), NewDeleteCmd(f, streams), }, }, diff --git a/internal/cli/cmd/cluster/label.go b/internal/cli/cmd/cluster/label.go new file mode 100644 index 000000000..e0a8294d2 --- /dev/null +++ b/internal/cli/cmd/cluster/label.go @@ -0,0 +1,340 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "encoding/json" + "fmt" + "strings" + + jsonpatch "github.com/evanphx/json-patch" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ktypes "k8s.io/apimachinery/pkg/types" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/cluster" + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var ( + labelExample = templates.Examples(` + # list label for clusters with specified name + kbcli cluster label mycluster --list + + # add label 'env' and value 'dev' for clusters with specified name + kbcli cluster label mycluster env=dev + + # add label 'env' and value 'dev' for all clusters + kbcli cluster label env=dev --all + + # add label 'env' and value 'dev' for the clusters that match the selector + kbcli cluster label env=dev -l type=mysql + + # update cluster with the label 'env' with value 'test', overwriting any existing value + kbcli cluster label mycluster --overwrite env=test + + # delete label env for clusters with specified name + kbcli cluster label mycluster env-`) +) + +type LabelOptions struct { + Factory cmdutil.Factory + GVR schema.GroupVersionResource + + // Common user flags + overwrite bool + all bool + list bool + selector string + + // results of arg parsing + resources []string + newLabels map[string]string + removeLabels []string + + namespace string + enforceNamespace bool + dryRunStrategy cmdutil.DryRunStrategy + dryRunVerifier *resource.QueryParamVerifier + builder *resource.Builder + unstructuredClientForMapping func(mapping *meta.RESTMapping) (resource.RESTClient, error) + + genericclioptions.IOStreams +} + +func NewLabelOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, gvr schema.GroupVersionResource) *LabelOptions { + return &LabelOptions{ + Factory: f, + GVR: gvr, + IOStreams: streams, + } +} + +func NewLabelCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewLabelOptions(f, streams, types.ClusterGVR()) + cmd := &cobra.Command{ + Use: "label NAME", + Short: "Update the labels on cluster", + Example: labelExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, o.GVR), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(o.complete(cmd, args)) + util.CheckErr(o.validate()) + util.CheckErr(o.run()) + }, + } + + cmd.Flags().BoolVar(&o.overwrite, "overwrite", o.overwrite, "If true, allow labels to be overwritten, otherwise reject label updates that overwrite existing labels.") + cmd.Flags().BoolVar(&o.all, "all", o.all, "Select all cluster") + cmd.Flags().BoolVar(&o.list, "list", o.list, "If true, display the labels of the clusters") + cmdutil.AddDryRunFlag(cmd) + cmdutil.AddLabelSelectorFlagVar(cmd, &o.selector) + + return cmd +} + +func (o *LabelOptions) complete(cmd *cobra.Command, args []string) error { + var err error + + o.dryRunStrategy, err = cmdutil.GetDryRunStrategy(cmd) + if err != nil { + return err + } + + // parse resources and labels + resources, labelArgs, err := cmdutil.GetResourcesAndPairs(args, "label") + if err != nil { + return err + } + o.resources = resources + o.newLabels, o.removeLabels, err = parseLabels(labelArgs) + if err != nil { + return err + } + + o.namespace, o.enforceNamespace, err = o.Factory.ToRawKubeConfigLoader().Namespace() + if err != nil { + return nil + } + o.builder = o.Factory.NewBuilder() + o.unstructuredClientForMapping = o.Factory.UnstructuredClientForMapping + dynamicClient, err := o.Factory.DynamicClient() + if err != nil { + return err + } + o.dryRunVerifier = resource.NewQueryParamVerifier(dynamicClient, o.Factory.OpenAPIGetter(), resource.QueryParamDryRun) + return nil +} + +func (o *LabelOptions) validate() error { + if o.all && len(o.selector) > 0 { + return fmt.Errorf("cannot set --all and --selector at the same time") + } + + if !o.all && len(o.selector) == 0 && len(o.resources) == 0 { + return fmt.Errorf("at least one cluster is required") + } + + if len(o.newLabels) < 1 && len(o.removeLabels) < 1 && !o.list { + return fmt.Errorf("at least one label update is required") + } + return nil +} + +func (o *LabelOptions) run() error { + r := o.builder. + Unstructured(). + NamespaceParam(o.namespace).DefaultNamespace(). + LabelSelector(o.selector). + ResourceTypeOrNameArgs(o.all, append([]string{util.GVRToString(o.GVR)}, o.resources...)...). + ContinueOnError(). + Latest(). + Flatten(). + Do() + + if err := r.Err(); err != nil { + return err + } + + infos, err := r.Infos() + if err != nil { + return err + } + + if len(infos) == 0 { + return fmt.Errorf("no clusters found") + } + + for _, info := range infos { + obj := info.Object + oldData, err := json.Marshal(obj) + if err != nil { + return err + } + + if o.dryRunStrategy == cmdutil.DryRunClient || o.list { + err = labelFunc(obj, o.overwrite, o.newLabels, o.removeLabels) + if err != nil { + return err + } + } else { + name, namespace := info.Name, info.Namespace + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + for _, label := range o.removeLabels { + if _, ok := accessor.GetLabels()[label]; !ok { + fmt.Fprintf(o.Out, "label %q not found.\n", label) + } + } + + if err := labelFunc(obj, o.overwrite, o.newLabels, o.removeLabels); err != nil { + return err + } + + newObj, err := json.Marshal(obj) + if err != nil { + return err + } + patchBytes, err := jsonpatch.CreateMergePatch(oldData, newObj) + createPatch := err == nil + mapping := info.ResourceMapping() + client, err := o.unstructuredClientForMapping(mapping) + if err != nil { + return err + } + helper := resource.NewHelper(client, mapping). + DryRun(o.dryRunStrategy == cmdutil.DryRunServer) + if createPatch { + _, err = helper.Patch(namespace, name, ktypes.MergePatchType, patchBytes, nil) + } else { + _, err = helper.Replace(namespace, name, false, obj) + } + if err != nil { + return err + } + } + } + + if o.list { + dynamic, err := o.Factory.DynamicClient() + if err != nil { + return err + } + + client, err := o.Factory.KubernetesClientSet() + if err != nil { + return err + } + + opt := &cluster.PrinterOptions{ + ShowLabels: true, + } + + p := cluster.NewPrinter(o.IOStreams.Out, cluster.PrintLabels, opt) + for _, info := range infos { + if err = addRow(dynamic, client, info.Namespace, info.Name, p); err != nil { + return err + } + } + p.Print() + } + + return nil +} + +func parseLabels(spec []string) (map[string]string, []string, error) { + labels := map[string]string{} + var remove []string + for _, labelSpec := range spec { + switch { + case strings.Contains(labelSpec, "="): + parts := strings.Split(labelSpec, "=") + if len(parts) != 2 { + return nil, nil, fmt.Errorf("invalid label spec: %s", labelSpec) + } + labels[parts[0]] = parts[1] + case strings.HasSuffix(labelSpec, "-"): + remove = append(remove, labelSpec[:len(labelSpec)-1]) + default: + return nil, nil, fmt.Errorf("unknown label spec: %s", labelSpec) + } + } + for _, removeLabel := range remove { + if _, found := labels[removeLabel]; found { + return nil, nil, fmt.Errorf("can not both modify and remove label in the same command") + } + } + return labels, remove, nil +} + +func validateNoOverwrites(obj runtime.Object, labels map[string]string) error { + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + + objLabels := accessor.GetLabels() + if objLabels == nil { + return nil + } + + for key := range labels { + if _, found := objLabels[key]; found { + return fmt.Errorf("'%s' already has a value (%s), and --overwrite is false", key, objLabels[key]) + } + } + return nil +} + +func labelFunc(obj runtime.Object, overwrite bool, labels map[string]string, remove []string) error { + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + if !overwrite { + if err := validateNoOverwrites(obj, labels); err != nil { + return err + } + } + + objLabels := accessor.GetLabels() + if objLabels == nil { + objLabels = make(map[string]string) + } + + for key, value := range labels { + objLabels[key] = value + } + for _, label := range remove { + delete(objLabels, label) + } + accessor.SetLabels(objLabels) + + return nil +} diff --git a/internal/cli/cmd/cluster/label_test.go b/internal/cli/cmd/cluster/label_test.go new file mode 100644 index 000000000..3093eaa63 --- /dev/null +++ b/internal/cli/cmd/cluster/label_test.go @@ -0,0 +1,81 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/types" +) + +var _ = Describe("cluster label", func() { + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + ) + + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace("default") + }) + + AfterEach(func() { + tf.Cleanup() + }) + + It("label command", func() { + cmd := NewLabelCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + + Context("complete", func() { + var o *LabelOptions + var cmd *cobra.Command + var args []string + BeforeEach(func() { + cmd = NewLabelCmd(tf, streams) + o = NewLabelOptions(tf, streams, types.ClusterDefGVR()) + args = []string{"c1", "env=dev"} + }) + + It("args is empty", func() { + Expect(o.complete(cmd, nil)).Should(Succeed()) + Expect(o.validate()).Should(HaveOccurred()) + }) + + It("cannot set --all and --selector at the same time", func() { + o.all = true + o.selector = "status=unhealthy" + Expect(o.complete(cmd, args)).Should(Succeed()) + Expect(o.validate()).Should(HaveOccurred()) + }) + + It("at least one label update is required", func() { + Expect(o.complete(cmd, []string{"c1"})).Should(Succeed()) + Expect(o.validate()).Should(HaveOccurred()) + }) + + It("can not both modify and remove label in the same command", func() { + Expect(o.complete(cmd, []string{"c1", "env=dev", "env-"})).Should(HaveOccurred()) + }) + }) +}) diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 45dfc39e6..6fdc7f55c 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -658,11 +658,15 @@ func BuildAddonReleaseName(addon string) string { // CombineLabels combines labels into a string func CombineLabels(labels map[string]string) string { - var labelStr string + var labelStr []string for k, v := range labels { - labelStr += fmt.Sprintf("%s=%s,", k, v) + labelStr = append(labelStr, fmt.Sprintf("%s=%s", k, v)) } - return strings.TrimSuffix(labelStr, ",") + + // sort labelStr to make sure the order is stable + sort.Strings(labelStr) + + return strings.Join(labelStr, ",") } func BuildComponentNameLables(prefix string, names []string) string { From 44abe31d40d7f91cb4d0906533a961e51d87eb4a Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Sat, 15 Apr 2023 16:17:57 +0800 Subject: [PATCH 037/439] feat: add tpltool to render ComponentTemplateSpec for developer (#1774) (#2377) --- cmd/cmd.mk | 17 +++ cmd/tpl/README.md | 90 +++++++++++++ cmd/tpl/app/helm_helper.go | 119 +++++++++++++++++ cmd/tpl/app/k8s_resource.go | 74 +++++++++++ cmd/tpl/app/mock_client.go | 119 +++++++++++++++++ cmd/tpl/app/util.go | 94 ++++++++++++++ cmd/tpl/app/workflow.go | 250 ++++++++++++++++++++++++++++++++++++ cmd/tpl/main.go | 137 ++++++++++++++++++++ 8 files changed, 900 insertions(+) create mode 100644 cmd/tpl/README.md create mode 100644 cmd/tpl/app/helm_helper.go create mode 100644 cmd/tpl/app/k8s_resource.go create mode 100644 cmd/tpl/app/mock_client.go create mode 100644 cmd/tpl/app/util.go create mode 100644 cmd/tpl/app/workflow.go create mode 100644 cmd/tpl/main.go diff --git a/cmd/cmd.mk b/cmd/cmd.mk index 3e70d72c3..2906e6ef7 100644 --- a/cmd/cmd.mk +++ b/cmd/cmd.mk @@ -31,6 +31,23 @@ reloader: test-go-generate build-checks ## Build reloader related binaries clean-reloader: ## Clean bin/reloader. rm -f bin/reloader +## tpltool cmd + +CONFIG_TOOL_LD_FLAGS = "-s -w" + +bin/tpltool.%: ## Cross build bin/tpltool.$(OS).$(ARCH) . + GOOS=$(word 2,$(subst ., ,$@)) GOARCH=$(word 3,$(subst ., ,$@)) $(GO) build -ldflags=${CONFIG_TOOL_LD_FLAGS} -o $@ ./cmd/tpl/main.go + +.PHONY: tpltool +tpltool: OS=$(shell $(GO) env GOOS) +tpltool: ARCH=$(shell $(GO) env GOARCH) +tpltool: build-checks ## Build tpltool related binaries + $(MAKE) bin/tpltool.${OS}.${ARCH} + mv bin/tpltool.${OS}.${ARCH} bin/tpltool + +.PHONY: clean-tpltool +clean-tpltool: ## Clean bin/tpltool. + rm -f bin/tpltool ## cue-helper cmd diff --git a/cmd/tpl/README.md b/cmd/tpl/README.md new file mode 100644 index 000000000..639e1138c --- /dev/null +++ b/cmd/tpl/README.md @@ -0,0 +1,90 @@ +

tpltool

+ +# 1. Introduction + +Welcome to tpltool - a developer tool integrated with Kubeblocks that can help developers quickly generate rendered configurations or scripts based on Helm templates, and discover errors in the template before creating the database cluster. + +# 2. Getting Started + +You can get started with tpltool, by any of the following methods: +* Build `reloader` from sources + +## 2.1 Build + +Compiler `Go 1.19+` (Generics Programming Support), checking the [Go Installation](https://go.dev/doc/install) to see how to install Go on your platform. + +Use `make tpltool` to build and produce the `tpltool` binary file. The executable is produced under current directory. + +```shell +$ cd kubeblocks +$ make tpltool +``` + +## 2.2 Run + +You can run the following command to start tpltool once built + +```shell +tpltool Provides a mechanism to rendered template for ComponentConfigSpec and ComponentScriptSpec in the ClusterComponentDefinition. + +Usage: + tpltool [flags] + +Flags: + -a, --all template all config/script specs + --clean specify whether to clear the output dir + + # specify the cluster yaml + --cluster string the cluster yaml file + # specify the clusterdefinition yaml + --cluster-definition string the cluster definition yaml file + + --component-name string specify the component name of the clusterdefinition + --config-spec string specify the config spec to be rendered + + # for mock cluster yaml + --cpu string specify the cpu of the component + --memory string specify the memory of the component + --volume-name string specify the data volume name of the component + + --helm string specify the helm template dir of the component + --helm-output string specify the helm template output dir + + -o, --output-dir string specify the output directory + -r, --replicas int32 specify the replicas of the component (default 1) + +``` + +```shell + +# the first way + +$ ./bin/tpltool --helm ./deploy/apecloud-mysql --output-dir ./rendered_output --clean --cpu=200 --memory=10G --config-spec mysql-consensusset-config +wrote ./temp_helm_template_output/apecloud-mysql/templates/configmap.yaml +wrote ./temp_helm_template_output/apecloud-mysql/templates/configmap.yaml +wrote ./temp_helm_template_output/apecloud-mysql/templates/scripts.yaml +wrote ./temp_helm_template_output/apecloud-mysql/templates/backuppolicytemplate.yaml +wrote ./temp_helm_template_output/apecloud-mysql/templates/backuptool.yaml +wrote ./temp_helm_template_output/apecloud-mysql/templates/clusterdefinition.yaml +wrote ./temp_helm_template_output/apecloud-mysql/templates/clusterversion.yaml +wrote ./temp_helm_template_output/apecloud-mysql/templates/configconstraint.yaml + + +2023-04-02T23:25:07+08:00 INFO tpltool rendering template: +2023-04-02T23:25:07+08:00 INFO tpltool config spec: mysql-consensusset-config, template name: mysql8.0-config-template in the component[mysql] +2023-04-02T23:25:07+08:00 INFO dump rendering template spec: mysql-consensusset-config, output directory: rendered_output/cluster-HVevhr-mysql-Qsq-mysql-config + +$ ls rendered_output/cluster-HVevhr-mysql-Qsq-mysql-config +my.cnf + + +# the second way +# helm template deploy/apecloud-mysql --output-dir ${helm_template_output} +$ ./bin/tpltool --helm-output ${helm_template_output} -a + +``` + + +# 7. License + +Reloader is under the Apache 2.0 license. See the [LICENSE](../../LICENSE) file for details. diff --git a/cmd/tpl/app/helm_helper.go b/cmd/tpl/app/helm_helper.go new file mode 100644 index 000000000..ab898892b --- /dev/null +++ b/cmd/tpl/app/helm_helper.go @@ -0,0 +1,119 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "bytes" + "os" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/internal/generics" +) + +func scanDirectoryPath(rootPath string) ([]string, error) { + dirs, err := os.ReadDir(rootPath) + if err != nil { + return nil, err + } + resourceList := make([]string, 0) + for _, d := range dirs { + if d.IsDir() { + subDirectory, err := scanDirectoryPath(filepath.Join(rootPath, d.Name())) + if err != nil { + return nil, err + } + resourceList = append(resourceList, subDirectory...) + continue + } + if filepath.Ext(d.Name()) != ".yaml" { + continue + } + resourceList = append(resourceList, filepath.Join(rootPath, d.Name())) + } + return resourceList, nil +} + +func getResourceMeta(yamlBytes []byte) (metav1.TypeMeta, error) { + type k8sObj struct { + metav1.TypeMeta `json:",inline"` + } + var o k8sObj + err := yaml.Unmarshal(yamlBytes, &o) + if err != nil { + return metav1.TypeMeta{}, err + } + return o.TypeMeta, nil +} + +func CreateObjectsFromDirectory(rootPath string) ([]client.Object, error) { + allObjs := make([]client.Object, 0) + + // create cr from yaml + resourceList, err := scanDirectoryPath(rootPath) + if err != nil { + return nil, err + } + for _, resourceFile := range resourceList { + yamlBytes, err := os.ReadFile(resourceFile) + if err != nil { + return nil, err + } + objects, err := createObjectsFromYaml(yamlBytes) + if err != nil { + return nil, err + } + allObjs = append(allObjs, objects...) + } + return allObjs, nil +} + +func createObjectsFromYaml(yamlBytes []byte) ([]client.Object, error) { + objects := make([]client.Object, 0) + for _, doc := range bytes.Split(yamlBytes, []byte("---")) { + if len(bytes.TrimSpace(doc)) == 0 { + continue + } + meta, err := getResourceMeta(doc) + if err != nil { + return nil, err + } + switch meta.Kind { + case kindFromResource(corev1.ConfigMap{}): + objects = append(objects, CreateTypedObjectFromYamlByte(doc, generics.ConfigMapSignature)) + case kindFromResource(corev1.Secret{}): + objects = append(objects, CreateTypedObjectFromYamlByte(doc, generics.SecretSignature)) + case kindFromResource(appsv1alpha1.ConfigConstraint{}): + objects = append(objects, CreateTypedObjectFromYamlByte(doc, generics.ConfigConstraintSignature)) + case kindFromResource(appsv1alpha1.ClusterDefinition{}): + objects = append(objects, CreateTypedObjectFromYamlByte(doc, generics.ClusterDefinitionSignature)) + case kindFromResource(appsv1alpha1.ClusterVersion{}): + objects = append(objects, CreateTypedObjectFromYamlByte(doc, generics.ClusterVersionSignature)) + case kindFromResource(appsv1alpha1.BackupPolicyTemplate{}): + objects = append(objects, CreateTypedObjectFromYamlByte(doc, generics.BackupPolicyTemplateSignature)) + case kindFromResource(dataprotectionv1alpha1.BackupTool{}): + objects = append(objects, CreateTypedObjectFromYamlByte(doc, generics.BackupToolSignature)) + } + } + return objects, nil +} diff --git a/cmd/tpl/app/k8s_resource.go b/cmd/tpl/app/k8s_resource.go new file mode 100644 index 000000000..f9248c293 --- /dev/null +++ b/cmd/tpl/app/k8s_resource.go @@ -0,0 +1,74 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "bytes" + "os" + + "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/internal/generics" +) + +func CustomizedObjFromYaml[T generics.Object, PT generics.PObject[T], + L generics.ObjList[T], PL generics.PObjList[T, L]](filePath string, signature func(T, L)) (PT, error) { + objList, err := CustomizedObjectListFromYaml[T, PT, L, PL](filePath, signature) + if err != nil { + return nil, err + } + if len(objList) == 0 { + return nil, nil + } + return objList[0], nil +} + +func CustomizedObjectListFromYaml[T generics.Object, PT generics.PObject[T], + L generics.ObjList[T], PL generics.PObjList[T, L]](yamlfile string, signature func(T, L)) ([]PT, error) { + objBytes, err := os.ReadFile(yamlfile) + if err != nil { + return nil, err + } + objList := make([]PT, 0) + for _, doc := range bytes.Split(objBytes, []byte("---")) { + if len(bytes.TrimSpace(doc)) == 0 { + continue + } + objList = append(objList, CreateTypedObjectFromYamlByte[T, PT, L, PL](doc, signature)) + } + return objList, nil +} + +func CreateTypedObjectFromYamlByte[T generics.Object, PT generics.PObject[T], + L generics.ObjList[T], PL generics.PObjList[T, L]](yamlBytes []byte, _ func(T, L)) PT { + var obj PT + if err := yaml.Unmarshal(yamlBytes, &obj); err != nil { + return nil + } + return obj +} + +func GetResourceObjectWithType[T generics.Object, PT generics.PObject[T], + L generics.ObjList[T], PL generics.PObjList[T, L]](objects []client.Object, _ func(T, L)) PT { + for _, object := range objects { + if cd, ok := object.(PT); ok { + return cd + } + } + return nil +} diff --git a/cmd/tpl/app/mock_client.go b/cmd/tpl/app/mock_client.go new file mode 100644 index 000000000..dcb00fb56 --- /dev/null +++ b/cmd/tpl/app/mock_client.go @@ -0,0 +1,119 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + testutil "github.com/apecloud/kubeblocks/internal/testutil/k8s" +) + +type mockClient struct { + objects map[client.ObjectKey]client.Object + kindObjectList map[string][]runtime.Object +} + +func newMockClient(objs []client.Object) client.Client { + return &mockClient{ + objects: fromObjects(objs), + kindObjectList: splitRuntimeObject(objs), + } +} + +func fromObjects(objs []client.Object) map[client.ObjectKey]client.Object { + r := make(map[client.ObjectKey]client.Object) + for _, obj := range objs { + if obj != nil { + r[client.ObjectKeyFromObject(obj)] = obj + } + } + return r +} + +func splitRuntimeObject(objects []client.Object) map[string][]runtime.Object { + r := make(map[string][]runtime.Object) + for _, object := range objects { + kind := object.GetObjectKind().GroupVersionKind().Kind + if _, ok := r[kind]; !ok { + r[kind] = make([]runtime.Object, 0) + } + r[kind] = append(r[kind], object) + } + return r +} + +func (m *mockClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + objKey := key + if object, ok := m.objects[objKey]; ok { + testutil.SetGetReturnedObject(obj, object) + return nil + } + objKey.Namespace = "" + if object, ok := m.objects[objKey]; ok { + testutil.SetGetReturnedObject(obj, object) + } + return nil +} + +func (m *mockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + r := m.kindObjectList[list.GetObjectKind().GroupVersionKind().Kind] + if r != nil { + return testutil.SetListReturnedObjects(list, r) + } + return nil +} + +func (m mockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return cfgcore.MakeError("not support") +} + +func (m mockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + return cfgcore.MakeError("not support") +} + +func (m mockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + return cfgcore.MakeError("not support") +} + +func (m mockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return cfgcore.MakeError("not support") +} + +func (m mockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + return cfgcore.MakeError("not support") +} + +func (m mockClient) Status() client.SubResourceWriter { + panic("implement me") +} + +func (m mockClient) SubResource(subResource string) client.SubResourceClient { + panic("implement me") +} + +func (m mockClient) Scheme() *runtime.Scheme { + panic("implement me") +} + +func (m mockClient) RESTMapper() meta.RESTMapper { + panic("implement me") +} diff --git a/cmd/tpl/app/util.go b/cmd/tpl/app/util.go new file mode 100644 index 000000000..e69942322 --- /dev/null +++ b/cmd/tpl/app/util.go @@ -0,0 +1,94 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "reflect" + + "github.com/sethvargo/go-password/password" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" +) + +type RenderedOptions struct { + ConfigSpec string + AllConfigSpecs bool + + // mock cluster object + Name string + Namespace string + + Replicas int32 + DataVolumeName string + ComponentName string + + CPU string + Memory string +} + +func mockClusterObject(clusterDefObj *appsv1alpha1.ClusterDefinition, renderedOpts RenderedOptions, clusterVersion *appsv1alpha1.ClusterVersion) *appsv1alpha1.Cluster { + cvReference := "" + if clusterVersion != nil { + cvReference = clusterVersion.Name + } + factory := testapps.NewClusterFactory(renderedOpts.Namespace, renderedOpts.Name, clusterDefObj.Name, cvReference) + for _, component := range clusterDefObj.Spec.ComponentDefs { + factory.AddComponent(component.CharacterType+"-"+RandomString(3), component.Name) + factory.SetReplicas(renderedOpts.Replicas) + if renderedOpts.DataVolumeName != "" { + pvcSpec := testapps.NewPVCSpec("10Gi") + factory.AddVolumeClaimTemplate(renderedOpts.DataVolumeName, pvcSpec) + } + if renderedOpts.CPU != "" || renderedOpts.Memory != "" { + factory.SetResources(fromResource(renderedOpts)) + } + } + return factory.GetObject() +} + +func fromResource(opts RenderedOptions) corev1.ResourceRequirements { + cpu := opts.CPU + memory := opts.Memory + if cpu == "" { + cpu = "1" + } + if memory == "" { + memory = "1Gi" + } + return corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "cpu": resource.MustParse(cpu), + "memory": resource.MustParse(memory), + }, + } +} + +func kindFromResource[T any](resource T) string { + t := reflect.TypeOf(resource) + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t.Name() +} + +func RandomString(n int) string { + s, _ := password.Generate(n, 0, 0, false, false) + return s +} diff --git a/cmd/tpl/app/workflow.go b/cmd/tpl/app/workflow.go new file mode 100644 index 000000000..eec943c4f --- /dev/null +++ b/cmd/tpl/app/workflow.go @@ -0,0 +1,250 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + "fmt" + "os" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/controller/builder" + "github.com/apecloud/kubeblocks/internal/controller/component" + "github.com/apecloud/kubeblocks/internal/controller/plan" + intctrltypes "github.com/apecloud/kubeblocks/internal/controller/types" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" + "github.com/apecloud/kubeblocks/internal/generics" +) + +type componentedConfigSpec struct { + component string + configSpec appsv1alpha1.ComponentTemplateSpec +} + +type templateRenderWorkflow struct { + renderedOpts RenderedOptions + clusterYaml string + clusterDefObj *appsv1alpha1.ClusterDefinition + localObjects []client.Object + + clusterDefComponents []appsv1alpha1.ClusterComponentDefinition +} + +func (w *templateRenderWorkflow) Do(outputDir string) error { + var err error + var cluster *appsv1alpha1.Cluster + var configSpecs []componentedConfigSpec + + cli := newMockClient(w.localObjects) + ctx := intctrlutil.RequestCtx{ + Ctx: context.Background(), + Log: log.Log.WithName("ctool"), + } + + if cluster, err = w.createClusterObject(); err != nil { + return err + } + ctx.Log.V(1).Info(fmt.Sprintf("cluster object : %v", cluster)) + + if configSpecs, err = w.getRenderedConfigSpec(); err != nil { + return err + } + + ctx.Log.Info("rendering template:") + for _, tplSpec := range configSpecs { + ctx.Log.Info(fmt.Sprintf("config spec: %s, template name: %s in the component[%s]", + tplSpec.configSpec.Name, + tplSpec.configSpec.TemplateRef, + tplSpec.component)) + } + + cache := make(map[string]*intctrltypes.ReconcileTask) + for _, configSpec := range configSpecs { + task, ok := cache[configSpec.component] + if !ok { + param, err := createComponentParams(w, ctx, cli, configSpec.component, cluster) + if err != nil { + return err + } + cache[configSpec.component] = param + task = param + } + if err := renderTemplates(configSpec.configSpec, outputDir, task); err != nil { + return err + } + } + return nil +} + +func (w *templateRenderWorkflow) Prepare(ctx intctrlutil.RequestCtx, componentType string, cluster *appsv1alpha1.Cluster) (*component.SynthesizedComponent, error) { + clusterCompDef := w.clusterDefObj.GetComponentDefByName(componentType) + clusterCompSpecMap := cluster.Spec.GetDefNameMappingComponents() + clusterCompSpec := clusterCompSpecMap[componentType] + + if clusterCompDef == nil || len(clusterCompSpec) == 0 { + return nil, cfgcore.MakeError("component[%s] is not defined in cluster definition", componentType) + } + + return component.BuildComponent(ctx, *cluster, *w.clusterDefObj, *clusterCompDef, clusterCompSpec[0]), nil +} + +func (w *templateRenderWorkflow) getRenderedConfigSpec() ([]componentedConfigSpec, error) { + foundSpec := func(com appsv1alpha1.ClusterComponentDefinition, specName string) (appsv1alpha1.ComponentTemplateSpec, bool) { + for _, spec := range com.ConfigSpecs { + if spec.Name == specName { + return spec.ComponentTemplateSpec, true + } + } + for _, spec := range com.ScriptSpecs { + if spec.Name == specName { + return spec, true + } + } + return appsv1alpha1.ComponentTemplateSpec{}, false + } + + if w.renderedOpts.ConfigSpec != "" { + for _, com := range w.clusterDefComponents { + if spec, ok := foundSpec(com, w.renderedOpts.ConfigSpec); ok { + return []componentedConfigSpec{{com.Name, spec}}, nil + } + } + return nil, cfgcore.MakeError("config spec[%s] is not found", w.renderedOpts.ConfigSpec) + } + + if !w.renderedOpts.AllConfigSpecs { + return nil, cfgcore.MakeError("config spec[%s] is not found", w.renderedOpts.ConfigSpec) + } + configSpecs := make([]componentedConfigSpec, 0) + for _, com := range w.clusterDefComponents { + for _, configSpec := range com.ConfigSpecs { + configSpecs = append(configSpecs, componentedConfigSpec{com.Name, configSpec.ComponentTemplateSpec}) + } + for _, configSpec := range com.ScriptSpecs { + configSpecs = append(configSpecs, componentedConfigSpec{com.Name, configSpec}) + } + } + return configSpecs, nil +} + +func (w *templateRenderWorkflow) createClusterObject() (*appsv1alpha1.Cluster, error) { + if w.clusterYaml != "" { + return CustomizedObjFromYaml(w.clusterYaml, generics.ClusterSignature) + } + + clusterVersionObj := GetResourceObjectWithType(w.localObjects, generics.ClusterVersionSignature) + return mockClusterObject(w.clusterDefObj, w.renderedOpts, clusterVersionObj), nil +} + +func NewWorkflowTemplateRender(helmTemplateDir string, opts RenderedOptions) (*templateRenderWorkflow, error) { + if _, err := os.Stat(helmTemplateDir); err != nil { + panic("cluster definition yaml file is required") + } + + allObjects, err := CreateObjectsFromDirectory(helmTemplateDir) + if err != nil { + return nil, err + } + + clusterDefObj := GetResourceObjectWithType(allObjects, generics.ClusterDefinitionSignature) + if clusterDefObj == nil { + return nil, cfgcore.MakeError("cluster definition object is not found in helm template directory[%s]", helmTemplateDir) + } + // hack apiserver auto filefield + checkAndFillPortProtocol(clusterDefObj.Spec.ComponentDefs) + + components := clusterDefObj.Spec.ComponentDefs + if opts.ComponentName != "" { + component := clusterDefObj.GetComponentDefByName(opts.ComponentName) + if component == nil { + return nil, cfgcore.MakeError("component[%s] is not defined in cluster definition", opts.ComponentName) + } + components = []appsv1alpha1.ClusterComponentDefinition{*component} + } + return &templateRenderWorkflow{ + renderedOpts: opts, + clusterDefObj: clusterDefObj, + localObjects: allObjects, + clusterDefComponents: components, + }, nil +} + +func checkAndFillPortProtocol(clusterDefComponents []appsv1alpha1.ClusterComponentDefinition) { + // fix failed to BuildHeadlessSvc + // failed to render workflow: cue: marshal error: service.spec.ports.0.protocol: undefined field: protocol + for i := range clusterDefComponents { + for j := range clusterDefComponents[i].PodSpec.Containers { + container := &clusterDefComponents[i].PodSpec.Containers[j] + for k := range container.Ports { + port := &container.Ports[k] + if port.Protocol == "" { + port.Protocol = corev1.ProtocolTCP + } + } + } + } +} + +func renderTemplates(configSpec appsv1alpha1.ComponentTemplateSpec, outputDir string, task *intctrltypes.ReconcileTask) error { + cfgName := cfgcore.GetComponentCfgName(task.Cluster.Name, task.Component.Name, configSpec.Name) + output := filepath.Join(outputDir, cfgName) + log.Log.Info(fmt.Sprintf("dump rendering template spec: %s, output directory: %s", configSpec.Name, output)) + + if err := os.MkdirAll(output, 0755); err != nil { + return err + } + + var ok bool + var cm *corev1.ConfigMap + for _, obj := range *task.Resources { + if cm, ok = obj.(*corev1.ConfigMap); !ok || cm.Name != cfgName { + continue + } + for file, val := range cm.Data { + if err := os.WriteFile(filepath.Join(output, file), []byte(val), 0755); err != nil { + return err + } + } + break + } + return nil +} + +func createComponentParams(w *templateRenderWorkflow, ctx intctrlutil.RequestCtx, cli client.Client, componentType string, cluster *appsv1alpha1.Cluster) (*intctrltypes.ReconcileTask, error) { + component, err := w.Prepare(ctx, componentType, cluster) + if err != nil { + return nil, err + } + clusterVersionObj := GetResourceObjectWithType(w.localObjects, generics.ClusterVersionSignature) + task := intctrltypes.InitReconcileTask(w.clusterDefObj, clusterVersionObj, cluster, component) + secret, err := builder.BuildConnCredential(task.GetBuilderParams()) + if err != nil { + return nil, err + } + // must make sure secret resources are created before workloads resources + task.AppendResource(secret) + if err := plan.PrepareComponentResources(ctx, cli, task); err != nil { + return nil, err + } + return task, nil +} diff --git a/cmd/tpl/main.go b/cmd/tpl/main.go new file mode 100644 index 000000000..6497da8ff --- /dev/null +++ b/cmd/tpl/main.go @@ -0,0 +1,137 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/spf13/pflag" + corezap "go.uber.org/zap" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + cfgcontainer "github.com/apecloud/kubeblocks/internal/configuration/container" + + "github.com/apecloud/kubeblocks/cmd/tpl/app" +) + +var clusterYaml string +var clusterDefYaml string + +var outputDir string +var clearOutputDir bool +var helmOutputDir string +var helmTemplateDir string + +var opts app.RenderedOptions + +func installFlags() { + pflag.StringVar(&clusterYaml, "cluster", "", "the cluster yaml file") + pflag.StringVar(&clusterDefYaml, "cluster-definition", "", "the cluster definition yaml file") + + pflag.StringVarP(&outputDir, "output-dir", "o", "", "specify the output directory") + + pflag.StringVar(&opts.ConfigSpec, "config-spec", "", "specify the config spec to be rendered") + pflag.BoolVarP(&opts.AllConfigSpecs, "all", "a", false, "template all config specs") + + // mock cluster object + pflag.Int32VarP(&opts.Replicas, "replicas", "r", 1, "specify the replicas of the component") + pflag.StringVar(&opts.DataVolumeName, "volume-name", "", "specify the data volume name of the component") + pflag.StringVar(&opts.ComponentName, "component-name", "", "specify the component name of the clusterdefinition") + pflag.StringVar(&helmTemplateDir, "helm", "", "specify the helm template dir") + pflag.StringVar(&helmOutputDir, "helm-output", "", "specify the helm template output dir") + pflag.StringVar(&opts.CPU, "cpu", "", "specify the cpu of the component") + pflag.StringVar(&opts.Memory, "memory", "", "specify the memory of the component") + pflag.BoolVar(&clearOutputDir, "clean", false, "specify whether to clear the output dir") + + opts := zap.Options{ + Development: true, + Level: func() *corezap.AtomicLevel { + lvl := corezap.NewAtomicLevelAt(corezap.InfoLevel) + return &lvl + }(), + } + + opts.BindFlags(flag.CommandLine) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + // NOTES: + // zap is "Blazing fast, structured, leveled logging in Go.", DON'T event try + // to refactor this logging lib to anything else. Check FAQ - https://github.com/uber-go/zap/blob/master/FAQ.md + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) +} + +func main() { + + opts = app.RenderedOptions{ + // for mock cluster object + Namespace: "default", + Name: "cluster-" + app.RandomString(6), + } + installFlags() + + if err := checkAndHelmTemplate(); err != nil { + fmt.Printf("failed to exec helm template: %v", err) + os.Exit(-1) + } + + if helmOutputDir == "" { + fmt.Printf("helm template dir is empty") + os.Exit(-1) + } + + workflow, err := app.NewWorkflowTemplateRender(helmOutputDir, opts) + if err != nil { + fmt.Printf("failed to create workflow: %v", err) + os.Exit(-1) + } + + if clearOutputDir && outputDir != "" { + _ = os.RemoveAll(outputDir) + } + if outputDir == "" { + outputDir = filepath.Join("./output", app.RandomString(6)) + } + + if err := workflow.Do(outputDir); err != nil { + fmt.Printf("failed to render workflow: %v", err) + os.Exit(-1) + } +} + +func checkAndHelmTemplate() error { + if helmTemplateDir != "" || helmOutputDir == "" { + helmOutputDir = filepath.Join("./helm-output", app.RandomString(6)) + } + + if helmTemplateDir == "" || helmOutputDir == "" { + return nil + } + cmd := exec.Command("helm", "template", helmTemplateDir, "--output-dir", helmOutputDir) + stdout, err := cfgcontainer.ExecShellCommand(cmd) + if err != nil { + fmt.Println(err) + os.Exit(-1) + } + fmt.Println(stdout) + return nil +} From 26c84e5e095b06f7b6bf7885de8e091dcf854ab1 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Sat, 15 Apr 2023 18:58:16 +0800 Subject: [PATCH 038/439] chore: optimize the description of expose command (#2603) --- docs/user_docs/cli/cli.md | 2 +- docs/user_docs/cli/kbcli_cluster.md | 2 +- docs/user_docs/cli/kbcli_cluster_expose.md | 2 +- internal/cli/cmd/cluster/operations.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index 337805f38..383dbb872 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -67,7 +67,7 @@ Cluster command. * [kbcli cluster edit-backup-policy](kbcli_cluster_edit-backup-policy.md) - Edit backup policy * [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. * [kbcli cluster explain-config](kbcli_cluster_explain-config.md) - List the constraint for supported configuration params. -* [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster. +* [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster with a new endpoint and the new endpoint can be found by executing the command 'kbcli cluster describe '. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account * [kbcli cluster hscale](kbcli_cluster_hscale.md) - Horizontally scale the specified components in the cluster. * [kbcli cluster label](kbcli_cluster_label.md) - Update the labels on cluster diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index 9e900b0e6..b3af4261f 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -55,7 +55,7 @@ Cluster command. * [kbcli cluster edit-backup-policy](kbcli_cluster_edit-backup-policy.md) - Edit backup policy * [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. * [kbcli cluster explain-config](kbcli_cluster_explain-config.md) - List the constraint for supported configuration params. -* [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster. +* [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster with a new endpoint and the new endpoint can be found by executing the command 'kbcli cluster describe '. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account * [kbcli cluster hscale](kbcli_cluster_hscale.md) - Horizontally scale the specified components in the cluster. * [kbcli cluster label](kbcli_cluster_label.md) - Update the labels on cluster diff --git a/docs/user_docs/cli/kbcli_cluster_expose.md b/docs/user_docs/cli/kbcli_cluster_expose.md index 1ce7171e0..9cdb467b2 100644 --- a/docs/user_docs/cli/kbcli_cluster_expose.md +++ b/docs/user_docs/cli/kbcli_cluster_expose.md @@ -2,7 +2,7 @@ title: kbcli cluster expose --- -Expose a cluster. +Expose a cluster with a new endpoint and the new endpoint can be found by executing the command 'kbcli cluster describe '. ``` kbcli cluster expose [flags] diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 8eb1b92b6..0b1414594 100644 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -430,7 +430,7 @@ func NewExposeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra o := newBaseOperationsOptions(streams, appsv1alpha1.ExposeType, true) inputs := buildOperationsInputs(f, o) inputs.Use = "expose" - inputs.Short = "Expose a cluster." + inputs.Short = "Expose a cluster with a new endpoint and the new endpoint can be found by executing the command 'kbcli cluster describe '." inputs.Example = exposeExamples inputs.BuildFlags = func(cmd *cobra.Command) { o.buildCommonFlags(cmd) From be124e894916796b915db35749227336f4f83d3b Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Sun, 16 Apr 2023 14:24:50 +0800 Subject: [PATCH 039/439] fix: getClasses panic when webhook is disabled (#2602) --- apis/apps/v1alpha1/cluster_webhook.go | 7 ++++--- apis/apps/v1alpha1/opsrequest_webhook.go | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/apis/apps/v1alpha1/cluster_webhook.go b/apis/apps/v1alpha1/cluster_webhook.go index 3461186f4..7d72584d4 100644 --- a/apis/apps/v1alpha1/cluster_webhook.go +++ b/apis/apps/v1alpha1/cluster_webhook.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -174,7 +175,7 @@ func (r *Cluster) validate() error { allErrs = append(allErrs, field.Invalid(field.NewPath("spec.clusterDefinitionRef"), r.Spec.ClusterDefRef, err.Error())) } else { - r.validateComponents(&allErrs, clusterDef) + r.validateComponents(&allErrs, webhookMgr.client, clusterDef) } if len(allErrs) > 0 { @@ -199,7 +200,7 @@ func (r *Cluster) validateClusterVersionRef(allErrs *field.ErrorList) { } // ValidateComponents validate spec.components is legal -func (r *Cluster) validateComponents(allErrs *field.ErrorList, clusterDef *ClusterDefinition) { +func (r *Cluster) validateComponents(allErrs *field.ErrorList, k8sClient client.Client, clusterDef *ClusterDefinition) { var ( // invalid component slice invalidComponentDefs = make([]string, 0) @@ -213,7 +214,7 @@ func (r *Cluster) validateComponents(allErrs *field.ErrorList, clusterDef *Clust componentMap[v.Name] = v } - compClasses, err := getClasses(clusterDef.Name) + compClasses, err := getClasses(context.Background(), k8sClient, clusterDef.Name) if err != nil { return } diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index 39cee415e..0406751fe 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -176,7 +176,7 @@ func (r *OpsRequest) validateOps(ctx context.Context, case UpgradeType: return r.validateUpgrade(ctx, k8sClient) case VerticalScalingType: - return r.validateVerticalScaling(cluster) + return r.validateVerticalScaling(ctx, k8sClient, cluster) case HorizontalScalingType: return r.validateHorizontalScaling(ctx, k8sClient, cluster) case VolumeExpansionType: @@ -219,13 +219,13 @@ func (r *OpsRequest) validateUpgrade(ctx context.Context, } // validateVerticalScaling validates api when spec.type is VerticalScaling -func (r *OpsRequest) validateVerticalScaling(cluster *Cluster) error { +func (r *OpsRequest) validateVerticalScaling(ctx context.Context, k8sCLient client.Client, cluster *Cluster) error { verticalScalingList := r.Spec.VerticalScalingList if len(verticalScalingList) == 0 { return notEmptyError("spec.verticalScaling") } - compClasses, err := getClasses(cluster.Spec.ClusterDefRef) + compClasses, err := getClasses(ctx, k8sCLient, cluster.Spec.ClusterDefRef) if err != nil { return nil } @@ -486,12 +486,12 @@ func validateVerticalResourceList(resourceList map[corev1.ResourceName]resource. return "", nil } -func getClasses(clusterDef string) (map[string]map[string]*ComponentClassInstance, error) { +func getClasses(ctx context.Context, k8sClient client.Client, clusterDef string) (map[string]map[string]*ComponentClassInstance, error) { ml := []client.ListOption{ client.MatchingLabels{"clusterdefinition.kubeblocks.io/name": clusterDef}, } var classDefinitionList ComponentClassDefinitionList - if err := webhookMgr.client.List(context.Background(), &classDefinitionList, ml...); err != nil { + if err := k8sClient.List(ctx, &classDefinitionList, ml...); err != nil { return nil, err } var ( From 3271752045f49263fc52d0db8f85b767954f7acb Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Sun, 16 Apr 2023 15:19:42 +0800 Subject: [PATCH 040/439] chore: support xengine for apecloud-mysql (#2608) --- .../config/mysql8-config-constraint.cue | 5 +- .../config/mysql8-config-effect-scope.yaml | 1 + .../apecloud-mysql/config/mysql8-config.tpl | 55 +++++++++++++++++++ internal/controller/plan/builtin_functions.go | 9 +++ internal/controller/plan/config_template.go | 2 + 5 files changed, 71 insertions(+), 1 deletion(-) diff --git a/deploy/apecloud-mysql/config/mysql8-config-constraint.cue b/deploy/apecloud-mysql/config/mysql8-config-constraint.cue index 24108dc7f..c63b7c9d1 100644 --- a/deploy/apecloud-mysql/config/mysql8-config-constraint.cue +++ b/deploy/apecloud-mysql/config/mysql8-config-constraint.cue @@ -165,6 +165,9 @@ // Write a core file if mysqld dies. "core-file"?: string & "0" | "1" | "OFF" | "ON" + // xengine enable + "xengine"?: string & "0" | "1" | "OFF" | "ON" + // Abort a recursive common table expression if it does more than this number of iterations. cte_max_recursion_depth: int & >=0 & <=4294967295 | *1000 @@ -181,7 +184,7 @@ default_password_lifetime: int & >=0 & <=65535 | *0 // The default storage engine (table type). - default_storage_engine?: string & "InnoDB" | "MRG_MYISAM" | "BLACKHOLE" | "CSV" | "MEMORY" | "FEDERATED" | "ARCHIVE" | "MyISAM" + default_storage_engine?: string & "InnoDB" | "MRG_MYISAM" | "BLACKHOLE" | "CSV" | "MEMORY" | "FEDERATED" | "ARCHIVE" | "MyISAM" | "xengine" // Server current time zone default_time_zone?: string diff --git a/deploy/apecloud-mysql/config/mysql8-config-effect-scope.yaml b/deploy/apecloud-mysql/config/mysql8-config-effect-scope.yaml index 498cdaf09..d3bd9b7bd 100644 --- a/deploy/apecloud-mysql/config/mysql8-config-effect-scope.yaml +++ b/deploy/apecloud-mysql/config/mysql8-config-effect-scope.yaml @@ -5,6 +5,7 @@ ## if all the changed parameters are in the dynamicParameters list, this change executes reload without process restart. ## if the above two conditions are not met, by default, parameter change operation follow the rule for using staticParameters. staticParameters: + - xengine - allow-suspicious-udfs - auto_generate_certs - back_log diff --git a/deploy/apecloud-mysql/config/mysql8-config.tpl b/deploy/apecloud-mysql/config/mysql8-config.tpl index 2aa1d7ec1..133a24773 100644 --- a/deploy/apecloud-mysql/config/mysql8-config.tpl +++ b/deploy/apecloud-mysql/config/mysql8-config.tpl @@ -6,6 +6,7 @@ {{- $mysql_port_info := getPortByName ( index $.podSpec.containers 0 ) "mysql" }} {{- $pool_buffer_size := ( callBufferSizeByResource ( index $.podSpec.containers 0 ) ) }} {{- $phy_memory := getContainerMemory ( index $.podSpec.containers 0 ) }} +{{- $phy_cpu := getContainerCPU ( index $.podSpec.containers 0 ) }} {{- if $pool_buffer_size }} innodb_buffer_pool_size={{ $pool_buffer_size }} @@ -166,6 +167,60 @@ ssl_cert={{ $cert_file }} ssl_key={{ $key_file }} {{- end }} +## xengine base config +#default_storage_engine=xengine +default_tmp_storage_engine=innodb +xengine=0 + +# log_error_verbosity=3 +# binlog_format=ROW + +## non classes config + +loose_xengine_datadir={{ $data_root }}/xengine +loose_xengine_wal_dir={{ $data_root }}/xengine +loose_xengine_flush_log_at_trx_commit=1 +loose_xengine_enable_2pc=1 +loose_xengine_batch_group_slot_array_size=5 +loose_xengine_batch_group_max_group_size=15 +loose_xengine_batch_group_max_leader_wait_time_us=50 +loose_xengine_block_size=16384 +loose_xengine_disable_auto_compactions=0 +loose_xengine_dump_memtable_limit_size=0 + +loose_xengine_min_write_buffer_number_to_merge=1 +loose_xengine_level0_file_num_compaction_trigger=64 +loose_xengine_level0_layer_num_compaction_trigger=2 +loose_xengine_level1_extent s_major_compaction_trigger=1000 +loose_xengine_level2_usage_percent=70 +loose_xengine_flush_delete_percent=70 +loose_xengine_compaction_delete_percent=50 +loose_xengine_flush_delete_percent_trigger=700000 +loose_xengine_flush_delete_record_trigger=700000 +loose_xengine_scan_add_blocks_limit=100 + +loose_xengine_compression_per_level=kZSTD:kZSTD:kZSTD + + +## classes classes config + +{{- if gt $phy_memory 0 }} +{{- $phy_memory := div $phy_memory ( mul 1024 1024 ) }} +#loose_xengine_write_buffer_size={{ min ( max 32 ( mulf $phy_memory 0.01 ) ) 256 | int | mul 1024 1024 }} +#loose_xengine_db_write_buffer_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} +#loose_xengine_db_total_write_buffer_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} +#loose_xengine_block_cache_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} +#loose_xengine_row_cache_size={{ mulf $phy_memory 0.1 | int | mul 1024 1024 }} +#loose_xengine_max_total_wal_size={{ min ( mulf $phy_memory 0.3 ) ( mul 12 1024 ) | int | mul 1024 1024 }} +{{- end }} + +{{- if gt $phy_cpu 0 }} +loose_xengine_max_background_flushes={{ max 1 ( min ( div $phy_cpu 2 ) 8 ) | int }} +loose_xengine_base_background_compactions={{ max 1 ( min ( div $phy_cpu 2 ) 8 ) | int }} +loose_xengine_max_background_compactions={{ max 1 (min ( div $phy_cpu 2 ) 12 ) | int }} +{{- end }} + + [client] port={{ $mysql_port }} socket=/var/run/mysqld/mysqld.sock \ No newline at end of file diff --git a/internal/controller/plan/builtin_functions.go b/internal/controller/plan/builtin_functions.go index 6baa4f215..a7a9d67b0 100644 --- a/internal/controller/plan/builtin_functions.go +++ b/internal/controller/plan/builtin_functions.go @@ -202,6 +202,15 @@ func getPVCByName(args []interface{}, volumeName string) (interface{}, error) { return nil, nil } +// getContainerCPU for general built-in +func getContainerCPU(args interface{}) (int64, error) { + container, err := fromJSONObject[corev1.Container](args) + if err != nil { + return 0, err + } + return intctrlutil.GetCoreNum(*container), nil +} + // getContainerMemory for general built-in func getContainerMemory(args interface{}) (int64, error) { container, err := fromJSONObject[corev1.Container](args) diff --git a/internal/controller/plan/config_template.go b/internal/controller/plan/config_template.go index d360005a0..29fd794aa 100644 --- a/internal/controller/plan/config_template.go +++ b/internal/controller/plan/config_template.go @@ -48,6 +48,7 @@ const ( builtInGetArgFunctionName = "getArgByName" builtInGetPortFunctionName = "getPortByName" builtInGetContainerFunctionName = "getContainerByName" + builtInGetContainerCPUFunctionName = "getContainerCPU" builtInGetContainerMemoryFunctionName = "getContainerMemory" builtInGetContainerRequestMemoryFunctionName = "getContainerRequestMemory" @@ -180,6 +181,7 @@ func (c *configTemplateBuilder) injectBuiltInFunctions(component *component.Synt builtInGetPortFunctionName: getPortByName, builtInGetArgFunctionName: getArgByName, builtInGetContainerFunctionName: getPodContainerByName, + builtInGetContainerCPUFunctionName: getContainerCPU, builtInGetContainerMemoryFunctionName: getContainerMemory, builtInGetContainerRequestMemoryFunctionName: getContainerRequestMemory, builtInGetCAFile: getCAFile, From 3692ffb8e47e3eec593b345c1787d80288f02dc3 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Mon, 17 Apr 2023 00:00:01 +0800 Subject: [PATCH 041/439] chore: refactor replicationset component should create single workload component (#2533) Co-authored-by: wangyelei --- controllers/apps/cluster_controller_test.go | 478 +++++++----------- controllers/apps/cluster_status_utils_test.go | 49 +- .../apps/clusterdefinition_controller_test.go | 10 +- .../apps/clusterversion_controller_test.go | 6 +- controllers/apps/components/component.go | 8 +- .../apps/components/component_status_test.go | 24 +- .../components/consensusset/consensus_set.go | 32 +- .../components/consensusset/consensus_test.go | 2 +- .../components/deployment_controller_test.go | 14 +- .../replicationset/replication_set.go | 129 ++--- .../replicationset/replication_set_switch.go | 32 +- .../replication_set_switch_test.go | 59 +-- .../replication_set_switch_utils.go | 14 +- .../replication_set_switch_utils_test.go | 54 +- .../replicationset/replication_set_test.go | 123 ++--- .../replicationset/replication_set_utils.go | 184 +------ .../replication_set_utils_test.go | 166 +++--- .../apps/components/stateful/stateful.go | 35 +- .../apps/components/stateful/stateful_test.go | 15 +- .../stateful_set_controller_test.go | 24 +- .../apps/components/stateless/stateless.go | 13 +- .../components/stateless/stateless_test.go | 8 +- .../apps/components/types/component.go | 8 + .../apps/components/util/component_utils.go | 9 +- .../components/util/stateful_set_utils.go | 31 +- .../util/stateful_set_utils_test.go | 4 +- .../apps/configuration/config_util_test.go | 9 +- .../configconstraint_controller_test.go | 9 +- .../reconfigurerequest_controller_test.go | 13 +- .../apps/operations/horizontal_scaling.go | 2 +- .../operations/horizontal_scaling_test.go | 12 +- controllers/apps/operations/ops_manager.go | 5 +- .../apps/operations/ops_progress_util.go | 73 +-- .../apps/operations/ops_progress_util_test.go | 8 +- controllers/apps/operations/ops_util.go | 20 +- controllers/apps/operations/ops_util_test.go | 2 +- controllers/apps/operations/reconfigure.go | 6 +- controllers/apps/operations/restart.go | 2 +- controllers/apps/operations/start.go | 2 +- controllers/apps/operations/stop.go | 2 +- controllers/apps/operations/upgrade.go | 2 +- .../apps/operations/vertical_scaling.go | 2 +- .../apps/operations/volume_expansion.go | 4 +- .../apps/operations/volume_expansion_test.go | 9 +- .../operations/volume_expansion_updater.go | 4 +- controllers/apps/opsrequest_controller.go | 12 +- .../apps/opsrequest_controller_test.go | 124 ++--- .../apps/systemaccount_controller_test.go | 48 +- controllers/apps/systemaccount_util_test.go | 18 +- controllers/apps/tls_utils_test.go | 26 +- .../redis/scripts/redis-sentinel-setup.sh.tpl | 5 +- internal/cli/cmd/cluster/config_ops_test.go | 22 +- internal/controller/builder/builder_test.go | 18 +- .../builder/cue/statefulset_template.cue | 11 +- .../component/affinity_utils_test.go | 17 +- .../controller/component/component_test.go | 14 +- .../lifecycle/cluster_plan_builder.go | 2 - .../transformer_backup_policy_tpl.go | 13 +- .../lifecycle/transformer_cluster_status.go | 4 +- .../transformer_rpl_set_horizontal_scaling.go | 99 ---- internal/controller/plan/prepare.go | 74 +-- internal/controller/plan/prepare_test.go | 211 ++++---- .../apps/cluster_consensus_test_util.go | 4 +- internal/testutil/apps/cluster_factory.go | 4 +- .../apps/cluster_replication_test_util.go | 26 +- internal/testutil/apps/cluster_util.go | 24 +- internal/testutil/apps/clusterdef_factory.go | 16 +- .../testutil/apps/clusterversion_factory.go | 4 +- internal/testutil/apps/common_util.go | 10 +- internal/testutil/apps/constant.go | 2 +- internal/testutil/apps/native_object_util.go | 3 +- test/integration/backup_mysql_test.go | 10 +- test/integration/controller_suite_test.go | 12 +- test/integration/mysql_ha_test.go | 10 +- test/integration/mysql_scale_test.go | 19 +- test/integration/redis_hscale_test.go | 10 +- 76 files changed, 960 insertions(+), 1629 deletions(-) delete mode 100644 internal/controller/lifecycle/transformer_rpl_set_horizontal_scaling.go diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 2e75c6be0..676aad584 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -55,17 +55,19 @@ const backupPolicyTPLName = "test-backup-policy-template-mysql" var _ = Describe("Cluster Controller", func() { const ( - clusterDefName = "test-clusterdef" - clusterVersionName = "test-clusterversion" - clusterNamePrefix = "test-cluster" - mysqlCompType = "replicasets" - mysqlCompName = "mysql" - nginxCompType = "proxy" - nginxCompName = "nginx" - consensusCompName = "consensus" - consensusCompType = "consensus" - leader = "leader" - follower = "follower" + clusterDefName = "test-clusterdef" + clusterVersionName = "test-clusterversion" + clusterNamePrefix = "test-cluster" + statelessCompName = "stateless" + statelessCompDefName = "stateless" + statefulCompName = "stateful" + statefulCompDefName = "stateful" + consensusCompName = "consensus" + consensusCompDefName = "consensus" + replicationCompName = "replication" + replicationCompDefName = "replication" + leader = "leader" + follower = "follower" ) var ( @@ -91,7 +93,7 @@ var _ = Describe("Cluster Controller", func() { ml := client.HasLabels{testCtx.TestObjLabelKey} // namespaced testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.PodSignature, inNS, ml) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PodSignature, true, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupSignature, inNS, ml) // non-namespaced @@ -128,13 +130,13 @@ var _ = Describe("Cluster Controller", func() { By("Creating a cluster") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(mysqlCompName, mysqlCompType).SetReplicas(3). - AddComponent(nginxCompName, nginxCompType).SetReplicas(3). + AddComponent(replicationCompName, replicationCompDefName).SetReplicas(3). + AddComponent(statelessCompName, statelessCompDefName).SetReplicas(3). WithRandomName().Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) By("Check deployment workload has been created") Eventually(testapps.GetListLen(&testCtx, intctrlutil.DeploymentSignature, @@ -194,7 +196,7 @@ var _ = Describe("Cluster Controller", func() { By("Make sure the cluster controller has set the cluster status to Running") for i, comp := range clusterObj.Spec.ComponentSpecs { - if comp.ComponentDefRef != mysqlCompType || comp.Name != mysqlCompName { + if comp.ComponentDefRef != replicationCompDefName || comp.Name != replicationCompName { continue } stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), clusterObj.Spec.ComponentSpecs[i].Name) @@ -262,14 +264,14 @@ var _ = Describe("Cluster Controller", func() { By("Creating a cluster with two LoadBalancer services") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(mysqlCompName, mysqlCompType).SetReplicas(1). + AddComponent(replicationCompName, replicationCompDefName).SetReplicas(1). AddService(testapps.ServiceVPCName, corev1.ServiceTypeLoadBalancer). AddService(testapps.ServiceInternetName, corev1.ServiceTypeLoadBalancer). WithRandomName().Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) expectServices := map[string]ExpectService{ testapps.ServiceHeadlessName: {svcType: corev1.ServiceTypeClusterIP, headless: true}, @@ -277,14 +279,14 @@ var _ = Describe("Cluster Controller", func() { testapps.ServiceVPCName: {svcType: corev1.ServiceTypeLoadBalancer, headless: false}, testapps.ServiceInternetName: {svcType: corev1.ServiceTypeLoadBalancer, headless: false}, } - Eventually(func(g Gomega) { validateCompSvcList(g, mysqlCompName, mysqlCompType, expectServices) }).Should(Succeed()) + Eventually(func(g Gomega) { validateCompSvcList(g, replicationCompName, replicationCompDefName, expectServices) }).Should(Succeed()) By("Delete a LoadBalancer service") deleteService := testapps.ServiceVPCName delete(expectServices, deleteService) Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { for idx, comp := range cluster.Spec.ComponentSpecs { - if comp.ComponentDefRef != mysqlCompType || comp.Name != mysqlCompName { + if comp.ComponentDefRef != replicationCompDefName || comp.Name != replicationCompName { continue } var services []appsv1alpha1.ClusterComponentService @@ -298,13 +300,13 @@ var _ = Describe("Cluster Controller", func() { return } })()).ShouldNot(HaveOccurred()) - Eventually(func(g Gomega) { validateCompSvcList(g, mysqlCompName, mysqlCompType, expectServices) }).Should(Succeed()) + Eventually(func(g Gomega) { validateCompSvcList(g, replicationCompName, replicationCompDefName, expectServices) }).Should(Succeed()) By("Add the deleted LoadBalancer service back") expectServices[deleteService] = ExpectService{svcType: corev1.ServiceTypeLoadBalancer, headless: false} Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { for idx, comp := range cluster.Spec.ComponentSpecs { - if comp.ComponentDefRef != mysqlCompType || comp.Name != mysqlCompName { + if comp.ComponentDefRef != replicationCompDefName || comp.Name != replicationCompName { continue } comp.Services = append(comp.Services, appsv1alpha1.ClusterComponentService{ @@ -315,20 +317,20 @@ var _ = Describe("Cluster Controller", func() { return } })()).ShouldNot(HaveOccurred()) - Eventually(func(g Gomega) { validateCompSvcList(g, mysqlCompName, mysqlCompType, expectServices) }).Should(Succeed()) + Eventually(func(g Gomega) { validateCompSvcList(g, replicationCompName, replicationCompDefName, expectServices) }).Should(Succeed()) } checkAllServicesCreate := func() { By("Creating a cluster") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(mysqlCompName, mysqlCompType).SetReplicas(1). - AddComponent(nginxCompName, nginxCompType).SetReplicas(3). + AddComponent(replicationCompName, replicationCompDefName).SetReplicas(1). + AddComponent(statelessCompName, statelessCompDefName).SetReplicas(3). WithRandomName().Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName, nginxCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName, statelessCompName) By("Checking proxy services") nginxExpectServices := map[string]ExpectService{ @@ -336,18 +338,20 @@ var _ = Describe("Cluster Controller", func() { testapps.ServiceHeadlessName: {svcType: corev1.ServiceTypeClusterIP, headless: true}, testapps.ServiceDefaultName: {svcType: corev1.ServiceTypeClusterIP, headless: false}, } - Eventually(func(g Gomega) { validateCompSvcList(g, nginxCompName, nginxCompType, nginxExpectServices) }).Should(Succeed()) + Eventually(func(g Gomega) { validateCompSvcList(g, statelessCompName, statelessCompDefName, nginxExpectServices) }).Should(Succeed()) By("Checking mysql services") mysqlExpectServices := map[string]ExpectService{ testapps.ServiceHeadlessName: {svcType: corev1.ServiceTypeClusterIP, headless: true}, testapps.ServiceDefaultName: {svcType: corev1.ServiceTypeClusterIP, headless: false}, } - Eventually(func(g Gomega) { validateCompSvcList(g, mysqlCompName, mysqlCompType, mysqlExpectServices) }).Should(Succeed()) + Eventually(func(g Gomega) { + validateCompSvcList(g, replicationCompName, replicationCompDefName, mysqlExpectServices) + }).Should(Succeed()) By("Make sure the cluster controller has set the cluster status to Running") for i, comp := range clusterObj.Spec.ComponentSpecs { - if comp.ComponentDefRef != mysqlCompType || comp.Name != mysqlCompName { + if comp.ComponentDefRef != replicationCompDefName || comp.Name != replicationCompName { continue } stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), clusterObj.Spec.ComponentSpecs[i].Name) @@ -424,8 +428,8 @@ var _ = Describe("Cluster Controller", func() { if len(cluster.Spec.ComponentSpecs) == 0 { cluster.Spec.ComponentSpecs = []appsv1alpha1.ClusterComponentSpec{ { - Name: mysqlCompName, - ComponentDefRef: mysqlCompType, + Name: replicationCompName, + ComponentDefRef: replicationCompDefName, Replicas: replicas, }} } else { @@ -595,26 +599,74 @@ var _ = Describe("Cluster Controller", func() { }).Should(BeEquivalentTo(updatedReplicas)) } - horizontalScale := func(updatedReplicas int) { - + // @argument componentDefsWithHScalePolicy assign ClusterDefinition.spec.componentDefs[].horizontalScalePolicy for + // the matching names. If not provided, will set 1st ClusterDefinition.spec.componentDefs[0].horizontalScalePolicy. + horizontalScale := func(updatedReplicas int, componentDefsWithHScalePolicy ...string) { viper.Set("VOLUMESNAPSHOT", true) - cluster := &appsv1alpha1.Cluster{} Expect(testCtx.Cli.Get(testCtx.Ctx, clusterKey, cluster)).Should(Succeed()) initialGeneration := int(cluster.Status.ObservedGeneration) - By("Checking backup policy created from backup policy template") - policyName := lifecycle.GenerateBackupPolicyName(clusterKey.Name, mysqlCompType) - Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKey{Name: policyName, Namespace: clusterKey.Namespace}, - &dataprotectionv1alpha1.BackupPolicy{}, true)).Should(Succeed()) + // REVIEW/TODO: (chantu) + // ought to have HorizontalScalePolicy setup during ClusterDefinition object creation, + // following implementation is rather hack-ish. By("Set HorizontalScalePolicy") Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(clusterDefObj), func(clusterDef *appsv1alpha1.ClusterDefinition) { - clusterDef.Spec.ComponentDefs[0].HorizontalScalePolicy = - &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, - BackupPolicyTemplateName: backupPolicyTPLName} + // assign 1st component + if len(componentDefsWithHScalePolicy) == 0 && len(clusterDef.Spec.ComponentDefs) > 0 { + componentDefsWithHScalePolicy = []string{ + clusterDef.Spec.ComponentDefs[0].Name, + } + } + for i, compDef := range clusterDef.Spec.ComponentDefs { + if !slices.Contains(componentDefsWithHScalePolicy, compDef.Name) { + continue + } + + By("Checking backup policy created from backup policy template") + policyName := lifecycle.DeriveBackupPolicyName(clusterKey.Name, compDef.Name) + // REVIEW/TODO: (chantu) + // caught following error, it appears that BackupPolicy is statically setup or only work with 1st + // componentDefs? + // + // Unexpected error: + // <*errors.StatusError | 0x140023b5b80>: { + // ErrStatus: { + // TypeMeta: {Kind: "", APIVersion: ""}, + // ListMeta: { + // SelfLink: "", + // ResourceVersion: "", + // Continue: "", + // RemainingItemCount: nil, + // }, + // Status: "Failure", + // Message: "backuppolicies.dataprotection.kubeblocks.io \"test-clusterstqcba-consensus-backup-policy\" not found", + // Reason: "NotFound", + // Details: { + // Name: "test-clusterstqcba-consensus-backup-policy", + // Group: "dataprotection.kubeblocks.io", + // Kind: "backuppolicies", + // UID: "", + // Causes: nil, + // RetryAfterSeconds: 0, + // }, + // Code: 404, + // }, + // } + // backuppolicies.dataprotection.kubeblocks.io "test-clusterstqcba-consensus-backup-policy" not found + // occurred + clusterDef.Spec.ComponentDefs[i].HorizontalScalePolicy = + &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, + BackupPolicyTemplateName: backupPolicyTPLName} + + Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKey{Name: policyName, Namespace: clusterKey.Namespace}, + &dataprotectionv1alpha1.BackupPolicy{}, true)).Should(Succeed()) + + } })()).ShouldNot(HaveOccurred()) + // By("Mocking all components' PVCs to bound") for _, comp := range clusterObj.Spec.ComponentSpecs { @@ -624,7 +676,10 @@ var _ = Describe("Cluster Controller", func() { Name: getPVCName(comp.Name, i), } createPVC(clusterKey.Name, pvcKey.Name, comp.Name) - Eventually(testapps.CheckObjExists(&testCtx, pvcKey, &corev1.PersistentVolumeClaim{}, true)).Should(Succeed()) + Eventually(testapps.CheckObjExists(&testCtx, pvcKey, + &corev1.PersistentVolumeClaim{}, true)).Should(Succeed()) + // REVIEW/TODO: (chantu) + // why using Eventually for change object status? Eventually(testapps.GetAndChangeObjStatus(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound })).Should(Succeed()) @@ -635,9 +690,11 @@ var _ = Describe("Cluster Controller", func() { } By("Checking cluster status and the number of replicas changed") - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(initialGeneration + len(clusterObj.Spec.ComponentSpecs))) + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)). + Should(BeEquivalentTo(initialGeneration + len(clusterObj.Spec.ComponentSpecs))) for i := range clusterObj.Spec.ComponentSpecs { - stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), clusterObj.Spec.ComponentSpecs[i].Name) + stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), + clusterObj.Spec.ComponentSpecs[i].Name) for _, v := range stsList.Items { Expect(testapps.ChangeObjStatus(&testCtx, &v, func() { testk8s.MockStatefulSetReady(&v) @@ -654,14 +711,14 @@ var _ = Describe("Cluster Controller", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(replicationCompName, replicationCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). SetReplicas(initialReplicas). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) // REVIEW: this test flow, wait for running phase? horizontalScale(int(updatedReplicas)) @@ -671,26 +728,29 @@ var _ = Describe("Cluster Controller", func() { initialReplicas := int32(1) updatedReplicas := int32(3) - secondMysqlCompName := mysqlCompName + "1" - By("Creating a multi components cluster with VolumeClaimTemplate") pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(statefulCompName, statefulCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). SetReplicas(initialReplicas). - AddComponent(secondMysqlCompName, mysqlCompType). + AddComponent(consensusCompName, consensusCompDefName). + AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). + SetReplicas(initialReplicas). + AddComponent(replicationCompName, replicationCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). SetReplicas(initialReplicas). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName, secondMysqlCompName) + waitForCreatingResourceCompletely(clusterKey, statefulCompName, consensusCompName, replicationCompName) - // REVIEW: this test flow, wait for running phase? - horizontalScale(int(updatedReplicas)) + // REVIEW: (chantu) + // 1. this test flow, wait for running phase? + // 2. following horizontalScale only work with statefulCompDefName? + horizontalScale(int(updatedReplicas), statefulCompDefName, consensusCompDefName, replicationCompDefName) } testVerticalScale := func() { @@ -715,14 +775,14 @@ var _ = Describe("Cluster Controller", func() { By("Create cluster and waiting for the cluster initialized") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(replicationCompName, replicationCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). SetReplicas(replicas). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) By("Checking the replicas") stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) @@ -733,7 +793,7 @@ var _ = Describe("Cluster Controller", func() { for i := 0; i < replicas; i++ { pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: getPVCName(mysqlCompName, i), + Name: getPVCName(replicationCompName, i), Namespace: clusterKey.Namespace, Labels: map[string]string{ constant.AppInstanceLabelKey: clusterKey.Name, @@ -753,14 +813,14 @@ var _ = Describe("Cluster Controller", func() { })()).ShouldNot(HaveOccurred()) By("mock pods/sts of component are available") - testapps.MockConsensusComponentPods(testCtx, sts, clusterObj.Name, mysqlCompName) + testapps.MockConsensusComponentPods(testCtx, sts, clusterObj.Name, replicationCompName) Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { testk8s.MockStatefulSetReady(sts) })).ShouldNot(HaveOccurred()) By("Checking the resize operation finished") Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, replicationCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) By("Checking PVCs are resized") @@ -770,7 +830,7 @@ var _ = Describe("Cluster Controller", func() { pvc := &corev1.PersistentVolumeClaim{} pvcKey := types.NamespacedName{ Namespace: clusterKey.Namespace, - Name: getPVCName(mysqlCompName, int(i)), + Name: getPVCName(replicationCompName, int(i)), } Expect(k8sClient.Get(testCtx.Ctx, pvcKey, pvc)).Should(Succeed()) Expect(pvc.Spec.Resources.Requests[corev1.ResourceStorage]).To(Equal(newStorageValue)) @@ -794,13 +854,13 @@ var _ = Describe("Cluster Controller", func() { clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(mysqlCompName, mysqlCompType).SetReplicas(3). + AddComponent(replicationCompName, replicationCompDefName).SetReplicas(3). WithRandomName().SetClusterAffinity(affinity). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) By("Checking the Affinity and TopologySpreadConstraints") Eventually(func(g Gomega) { @@ -832,12 +892,12 @@ var _ = Describe("Cluster Controller", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().SetClusterAffinity(affinity). - AddComponent(mysqlCompName, mysqlCompType).SetComponentAffinity(compAffinity). + AddComponent(replicationCompName, replicationCompDefName).SetComponentAffinity(compAffinity). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) By("Checking the Affinity and the TopologySpreadConstraints") Eventually(func(g Gomega) { @@ -863,13 +923,13 @@ var _ = Describe("Cluster Controller", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType).SetReplicas(1). + AddComponent(replicationCompName, replicationCompDefName).SetReplicas(1). AddClusterToleration(toleration). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) By("Checking the tolerations") Eventually(func(g Gomega) { @@ -903,12 +963,12 @@ var _ = Describe("Cluster Controller", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().AddClusterToleration(toleration). - AddComponent(mysqlCompName, mysqlCompType).AddComponentToleration(compToleration). + AddComponent(replicationCompName, replicationCompDefName).AddComponentToleration(compToleration). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) By("Checking the tolerations") Eventually(func(g Gomega) { @@ -967,13 +1027,13 @@ var _ = Describe("Cluster Controller", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(replicationCompName, replicationCompDefName). SetReplicas(replicas).AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) var stsList *appsv1.StatefulSetList var sts *appsv1.StatefulSet @@ -1050,53 +1110,7 @@ var _ = Describe("Cluster Controller", func() { }).Should(Succeed()) By("Waiting the component be running") - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) - } - - mockPodsForReplicationTest := func(cluster *appsv1alpha1.Cluster, stsList []appsv1.StatefulSet) []corev1.Pod { - componentName := cluster.Spec.ComponentSpecs[0].Name - clusterName := cluster.Name - pods := make([]corev1.Pod, 0) - for _, sts := range stsList { - t := true - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: sts.Name + "-0", - Namespace: testCtx.DefaultNamespace, - Annotations: map[string]string{}, - Labels: map[string]string{ - constant.RoleLabelKey: sts.Labels[constant.RoleLabelKey], - constant.AppInstanceLabelKey: clusterName, - constant.KBAppComponentLabelKey: componentName, - appsv1.ControllerRevisionHashLabelKey: sts.Status.UpdateRevision, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "apps/v1", - Kind: constant.StatefulSetKind, - Controller: &t, - BlockOwnerDeletion: &t, - Name: sts.Name, - UID: sts.GetUID(), - }, - }, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "mock-container", - Image: "mock-container", - }}, - }, - } - for k, v := range sts.Spec.Template.Labels { - pod.ObjectMeta.Labels[k] = v - } - for k, v := range sts.Spec.Template.Annotations { - pod.ObjectMeta.Annotations[k] = v - } - pods = append(pods, *pod) - } - return pods + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, replicationCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) } testBackupError := func() { @@ -1107,17 +1121,16 @@ var _ = Describe("Cluster Controller", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(replicationCompName, replicationCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). SetReplicas(initialReplicas). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) // REVIEW: this test flow, should wait/fake still Running phase? - By("Creating backup") backupKey := types.NamespacedName{ Namespace: testCtx.DefaultNamespace, @@ -1129,12 +1142,12 @@ var _ = Describe("Cluster Controller", func() { Namespace: backupKey.Namespace, Labels: map[string]string{ constant.AppInstanceLabelKey: clusterKey.Name, - constant.KBAppComponentLabelKey: mysqlCompName, + constant.KBAppComponentLabelKey: replicationCompName, constant.KBManagedByKey: "cluster", }, }, Spec: dataprotectionv1alpha1.BackupSpec{ - BackupPolicyName: lifecycle.GenerateBackupPolicyName(clusterKey.Name, mysqlCompType), + BackupPolicyName: lifecycle.DeriveBackupPolicyName(clusterKey.Name, replicationCompDefName), BackupType: "snapshot", }, } @@ -1171,8 +1184,8 @@ var _ = Describe("Cluster Controller", func() { } updateClusterAnnotation := func(cluster *appsv1alpha1.Cluster) { - Expect(testapps.ChangeObj(&testCtx, cluster, func() { - cluster.Annotations = map[string]string{ + Expect(testapps.ChangeObj(&testCtx, cluster, func(lcluster *appsv1alpha1.Cluster) { + lcluster.Annotations = map[string]string{ "time": time.Now().Format(time.RFC3339), } })).ShouldNot(HaveOccurred()) @@ -1184,7 +1197,7 @@ var _ = Describe("Cluster Controller", func() { BeforeEach(func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, replicationCompDefName). Create(&testCtx).GetObject() }) @@ -1192,12 +1205,12 @@ var _ = Describe("Cluster Controller", func() { By("Creating a cluster") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, ""). - AddComponent(mysqlCompName, mysqlCompType).SetReplicas(3). + AddComponent(replicationCompName, replicationCompDefName).SetReplicas(3). WithRandomName().Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) }) }) @@ -1205,14 +1218,16 @@ var _ = Describe("Cluster Controller", func() { BeforeEach(func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). - AddComponent(testapps.StatelessNginxComponent, nginxCompType). + AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). + AddComponentDef(testapps.ConsensusMySQLComponent, consensusCompDefName). + AddComponentDef(testapps.StatefulMySQLComponent, replicationCompDefName). + AddComponentDef(testapps.StatelessNginxComponent, statelessCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - AddComponent(nginxCompType).AddContainerShort("nginx", testapps.NginxImage). + AddComponent(replicationCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(statelessCompDefName).AddContainerShort("nginx", testapps.NginxImage). Create(&testCtx).GetObject() By("Creating a BackupPolicyTemplate") @@ -1236,16 +1251,16 @@ var _ = Describe("Cluster Controller", func() { }) }) - Context("when creating cluster with workloadType=stateful component", func() { + When("when creating cluster with workloadType=stateful component", func() { BeforeEach(func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, replicationCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(replicationCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() By("Creating a BackupPolicyTemplate") @@ -1301,16 +1316,16 @@ var _ = Describe("Cluster Controller", func() { }) }) - Context("when creating cluster with workloadType=consensus component", func() { + When("when creating cluster with workloadType=consensus component", func() { BeforeEach(func() { By("Create a clusterDef obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ConsensusMySQLComponent, mysqlCompType). + AddComponentDef(testapps.ConsensusMySQLComponent, replicationCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(replicationCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() By("Creating a BackupPolicyTemplate") @@ -1379,34 +1394,34 @@ var _ = Describe("Cluster Controller", func() { g.Expect(tmpBackup.Status.Phase).Should(Equal(dataprotectionv1alpha1.BackupCompleted)) })).Should(Succeed()) By("creating cluster with backup") - restoreFromBackup := fmt.Sprintf(`{"%s":"%s"}`, mysqlCompName, backupName) + restoreFromBackup := fmt.Sprintf(`{"%s":"%s"}`, replicationCompName, backupName) clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(replicationCompName, replicationCompDefName). SetReplicas(3). AddAnnotations(constant.RestoreFromBackUpAnnotationKey, restoreFromBackup).Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, mysqlCompName) + waitForCreatingResourceCompletely(clusterKey, replicationCompName) stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) sts := stsList.Items[0] Expect(sts.Spec.Template.Spec.InitContainers).Should(HaveLen(1)) By("mock pod/sts are available and wait for component enter running phase") - testapps.MockConsensusComponentPods(testCtx, &sts, clusterObj.Name, mysqlCompName) + testapps.MockConsensusComponentPods(testCtx, &sts, clusterObj.Name, replicationCompName) Expect(testapps.ChangeObjStatus(&testCtx, &sts, func() { testk8s.MockStatefulSetReady(&sts) })).ShouldNot(HaveOccurred()) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, replicationCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) By("remove init container after all components are Running") Eventually(testapps.GetClusterObservedGeneration(&testCtx, client.ObjectKeyFromObject(clusterObj))).Should(BeEquivalentTo(1)) Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterObj), clusterObj)).Should(Succeed()) Expect(testapps.ChangeObjStatus(&testCtx, clusterObj, func() { clusterObj.Status.Components = map[string]appsv1alpha1.ClusterComponentStatus{ - mysqlCompName: {Phase: appsv1alpha1.RunningClusterCompPhase}, + replicationCompName: {Phase: appsv1alpha1.RunningClusterCompPhase}, } })).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(&sts), func(g Gomega, tmpSts *appsv1.StatefulSet) { @@ -1423,28 +1438,28 @@ var _ = Describe("Cluster Controller", func() { }) }) - Context("when creating cluster with workloadType=replication component", func() { + When("when creating cluster with workloadType=replication component", func() { BeforeEach(func() { By("Create a clusterDefinition obj with replication componentDefRef.") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompType). + AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj with replication componentDefRef.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompDefName). AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() }) // REVIEW/TODO: following test always failed at cluster.phase.observerGeneration=1 // with cluster.phase.phase=creating - It("Should success with primary sts and secondary sts", func() { + It("Should success with primary pod and secondary pod", func() { By("Mock a cluster obj with replication componentDefRef.") pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). SetReplicas(testapps.DefaultReplicationReplicas). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). @@ -1455,146 +1470,18 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely(clusterKey, testapps.DefaultRedisCompName) By("Checking statefulSet number") - var stsList *appsv1.StatefulSetList - Eventually(func(g Gomega) { - stsList = testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - g.Expect(stsList.Items).Should(HaveLen(testapps.DefaultReplicationReplicas)) - }).Should(Succeed()) - - By("Checking statefulSet role label") - for _, sts := range stsList.Items { - if strings.HasSuffix(sts.Name, fmt.Sprintf("%s-%s", clusterObj.Name, testapps.DefaultRedisCompName)) { - Expect(sts.Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replicationset.Primary)) - } else { - Expect(sts.Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replicationset.Secondary)) - } - } + stsList := testk8s.ListAndCheckStatefulSetCount(&testCtx, clusterKey, 1) + sts := &stsList.Items[0] - By("Checking statefulSet template volumes mount") - for _, sts := range stsList.Items { - Expect(sts.Spec.VolumeClaimTemplates).Should(BeEmpty()) - for _, volume := range sts.Spec.Template.Spec.Volumes { - if volume.Name == testapps.DataVolumeName { - Expect(strings.HasPrefix(volume.VolumeSource.PersistentVolumeClaim.ClaimName, testapps.DataVolumeName+"-"+clusterKey.Name)).Should(BeTrue()) - } - } - Expect(testapps.ChangeObjStatus(&testCtx, &sts, func() { - testk8s.MockStatefulSetReady(&sts) - })).ShouldNot(HaveOccurred()) - podName := sts.Name + "-0" - testapps.MockReplicationComponentStsPod(testCtx, &sts, clusterObj.Name, testapps.DefaultRedisCompName, podName, sts.Labels[constant.RoleLabelKey]) - } - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - }) - - It("Should successfully doing volume expansion", func() { - storageClassName := "test-storage" - pvcSpec := testapps.NewPVCSpec("1Gi") - pvcSpec.StorageClassName = &storageClassName - updatedPVCSpec := testapps.NewPVCSpec("2Gi") - updatedPVCSpec.StorageClassName = &storageClassName - - By("Mock a cluster obj with replication componentDefRef.") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). - SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). - SetReplicas(testapps.DefaultReplicationReplicas). - AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - - By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, testapps.DefaultRedisCompName) - - // REVIEW: this test flow, should wait/fake still Running phase? - - By("Checking statefulset count") - stsList := testk8s.ListAndCheckStatefulSetCount(&testCtx, clusterKey, testapps.DefaultReplicationReplicas) - - By("Creating mock pods in StatefulSet") - pods := mockPodsForReplicationTest(clusterObj, stsList.Items) - for _, pod := range pods { - Expect(testCtx.CreateObj(testCtx.Ctx, &pod)).Should(Succeed()) - pod.Status.Conditions = []corev1.PodCondition{{ - Type: corev1.PodReady, - Status: corev1.ConditionTrue, - }} - Expect(testCtx.Cli.Status().Update(testCtx.Ctx, &pod)).Should(Succeed()) - } - - By("Checking pod count and ready") - Eventually(func(g Gomega) { - podList := testk8s.ListAndCheckPodCountWithComponent(&testCtx, clusterKey, testapps.DefaultRedisCompName, testapps.DefaultReplicationReplicas) - for _, pod := range podList.Items { - g.Expect(len(pod.Status.Conditions) > 0).Should(BeTrue()) - g.Expect(pod.Status.Conditions[0].Status).Should(Equal(corev1.ConditionTrue)) - } - }).Should(Succeed()) - - By("Mocking statefulset status to ready") - for _, sts := range stsList.Items { - sts.Status.ObservedGeneration = sts.Generation - sts.Status.AvailableReplicas = 1 - sts.Status.Replicas = 1 - sts.Status.ReadyReplicas = 1 - err := testCtx.Cli.Status().Update(testCtx.Ctx, &sts) - Expect(err).ShouldNot(HaveOccurred()) + Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { + testk8s.MockStatefulSetReady(sts) + })).ShouldNot(HaveOccurred()) + for i := int32(0); i < *sts.Spec.Replicas; i++ { + podName := fmt.Sprintf("%s-%d", sts.Name, i) + testapps.MockReplicationComponentStsPod(nil, testCtx, sts, clusterObj.Name, + testapps.DefaultRedisCompName, podName, replicationset.DefaultRole(i)) } - - By("Checking reconcile succeeded") - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - - By("Creating storageclass") - _ = testapps.CreateStorageClass(testCtx, storageClassName, true) - - pvcList := &corev1.PersistentVolumeClaimList{} - - By("Mocking PVCs status to bound") - Expect(testCtx.Cli.List(testCtx.Ctx, pvcList, client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - constant.KBAppComponentLabelKey: testapps.DefaultRedisCompName, - }, client.InNamespace(clusterKey.Namespace))).Should(Succeed()) - Expect(pvcList.Items).Should(HaveLen(testapps.DefaultReplicationReplicas)) - for _, pvc := range pvcList.Items { - pvc.Status.Phase = corev1.ClaimBound - Expect(testCtx.Cli.Status().Update(testCtx.Ctx, &pvc)).Should(Succeed()) - } - - By("Checking PVCs status bound") - Eventually(func(g Gomega) { - g.Expect(testCtx.Cli.List(testCtx.Ctx, pvcList, client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - constant.KBAppComponentLabelKey: testapps.DefaultRedisCompName, - }, client.InNamespace(clusterKey.Namespace))).Should(Succeed()) - Expect(pvcList.Items).Should(HaveLen(testapps.DefaultReplicationReplicas)) - for _, pvc := range pvcList.Items { - g.Expect(pvc.Status.Phase).Should(Equal(corev1.ClaimBound)) - } - }).Should(Succeed()) - - By("Updating PVC volume size") - patch := client.MergeFrom(clusterObj.DeepCopy()) - componentSpec := clusterObj.Spec.GetComponentByName(testapps.DefaultRedisCompName) - componentSpec.VolumeClaimTemplates[0].Spec = updatedPVCSpec - Expect(testCtx.Cli.Patch(ctx, clusterObj, patch)).Should(Succeed()) - - By("Waiting cluster update reconcile succeed") - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) - - By("Checking pvc volume size") - Eventually(func(g Gomega) { - g.Expect(testCtx.Cli.List(testCtx.Ctx, pvcList, client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - constant.KBAppComponentLabelKey: testapps.DefaultRedisCompName, - }, client.InNamespace(clusterKey.Namespace))).Should(Succeed()) - g.Expect(len(pvcList.Items) == testapps.DefaultReplicationReplicas).To(BeTrue()) - for _, pvc := range pvcList.Items { - g.Expect(pvc.Spec.Resources.Requests[corev1.ResourceStorage]).Should(BeEquivalentTo(updatedPVCSpec.Resources.Requests[corev1.ResourceStorage])) - } - }).Should(Succeed()) }) }) @@ -1602,7 +1489,7 @@ var _ = Describe("Cluster Controller", func() { It("test cluster conditions", func() { By("init cluster") cluster := testapps.CreateConsensusMysqlCluster(testCtx, clusterDefNameRand, - clusterVersionNameRand, clusterNameRand, consensusCompType, consensusCompName) + clusterVersionNameRand, clusterNameRand, consensusCompDefName, consensusCompName) clusterKey := client.ObjectKeyFromObject(cluster) By("mock pvc created") @@ -1636,8 +1523,8 @@ var _ = Describe("Cluster Controller", func() { // })).Should(Succeed()) By("test when clusterVersion not Available") - _ = testapps.CreateConsensusMysqlClusterDef(testCtx, clusterDefNameRand, consensusCompType) - clusterVersion := testapps.CreateConsensusMysqlClusterVersion(testCtx, clusterDefNameRand, clusterVersionNameRand, consensusCompType) + _ = testapps.CreateConsensusMysqlClusterDef(testCtx, clusterDefNameRand, consensusCompDefName) + clusterVersion := testapps.CreateConsensusMysqlClusterVersion(testCtx, clusterDefNameRand, clusterVersionNameRand, consensusCompDefName) clusterVersionKey := client.ObjectKeyFromObject(clusterVersion) // mock clusterVersion unavailable Expect(testapps.GetAndChangeObj(&testCtx, clusterVersionKey, func(clusterVersion *appsv1alpha1.ClusterVersion) { @@ -1705,10 +1592,17 @@ var _ = Describe("Cluster Controller", func() { func createBackupPolicyTpl(clusterDefObj *appsv1alpha1.ClusterDefinition) { By("Creating a BackupPolicyTemplate") - testapps.NewBackupPolicyTemplateFactory(backupPolicyTPLName). + bpt := testapps.NewBackupPolicyTemplateFactory(backupPolicyTPLName). AddLabels(clusterDefLabelKey, clusterDefObj.Name). - AddBackupPolicy(clusterDefObj.Spec.ComponentDefs[0].Name). - AddSnapshotPolicy(). - SetClusterDefRef(clusterDefObj.Name). - SetTargetRole("leader").Create(&testCtx) + SetClusterDefRef(clusterDefObj.Name) + for _, v := range clusterDefObj.Spec.ComponentDefs { + bpt = bpt.AddBackupPolicy(v.Name).AddSnapshotPolicy() + switch v.WorkloadType { + case appsv1alpha1.Consensus: + bpt.SetTargetRole("leader") + case appsv1alpha1.Replication: + bpt.SetTargetRole("primary") + } + } + bpt.Create(&testCtx) } diff --git a/controllers/apps/cluster_status_utils_test.go b/controllers/apps/cluster_status_utils_test.go index 374c24ae7..f04353332 100644 --- a/controllers/apps/cluster_status_utils_test.go +++ b/controllers/apps/cluster_status_utils_test.go @@ -51,49 +51,46 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { // in race conditions, it will find the existence of old objects, resulting failure to // create the new objects. By("clean resources") - testapps.ClearClusterResources(&testCtx) inNS := client.InNamespace(testCtx.DefaultNamespace) ml := client.HasLabels{testCtx.TestObjLabelKey} - // testapps.ClearResources(&testCtx, intctrlutil.StatefulSetSignature, inNS, ml) - // testapps.ClearResources(&testCtx, intctrlutil.DeploymentSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.PodSignature, inNS, ml) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PodSignature, true, inNS, ml) } BeforeEach(cleanEnv) AfterEach(cleanEnv) - const statefulMySQLCompType = "stateful" + const statefulMySQLCompDefName = "stateful" const statefulMySQLCompName = "mysql1" - const consensusMySQLCompType = "consensus" + const consensusMySQLCompDefName = "consensus" const consensusMySQLCompName = "mysql2" - const nginxCompType = "stateless" + const statelessCompDefName = "stateless" const nginxCompName = "nginx" createClusterDef := func() { _ = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, statefulMySQLCompType). - AddComponent(testapps.ConsensusMySQLComponent, consensusMySQLCompType). - AddComponent(testapps.StatelessNginxComponent, nginxCompType). + AddComponentDef(testapps.StatefulMySQLComponent, statefulMySQLCompDefName). + AddComponentDef(testapps.ConsensusMySQLComponent, consensusMySQLCompDefName). + AddComponentDef(testapps.StatelessNginxComponent, statelessCompDefName). Create(&testCtx) } createClusterVersion := func() { _ = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(statefulMySQLCompType).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - AddComponent(consensusMySQLCompType).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - AddComponent(nginxCompType).AddContainerShort(testapps.DefaultNginxContainerName, testapps.NginxImage). + AddComponent(statefulMySQLCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponent(consensusMySQLCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponent(statelessCompDefName).AddContainerShort(testapps.DefaultNginxContainerName, testapps.NginxImage). Create(&testCtx) } createCluster := func() *appsv1alpha1.Cluster { return testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(statefulMySQLCompName, statefulMySQLCompType).SetReplicas(3). - AddComponent(consensusMySQLCompName, consensusMySQLCompType).SetReplicas(3). - AddComponent(nginxCompName, nginxCompType).SetReplicas(3). + AddComponent(statefulMySQLCompName, statefulMySQLCompDefName).SetReplicas(3). + AddComponent(consensusMySQLCompName, consensusMySQLCompDefName).SetReplicas(3). + AddComponent(nginxCompName, statelessCompDefName).SetReplicas(3). Create(&testCtx).GetObject() } @@ -281,7 +278,7 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { By("test the cluster phase when cluster only contains a component of Stateful workload, and the component is Failed or Abnormal") clusterObj := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(statefulMySQLCompName, statefulMySQLCompType).SetReplicas(3).GetObject() + AddComponent(statefulMySQLCompName, statefulMySQLCompDefName).SetReplicas(3).GetObject() // mock Stateful component is Failed and expect cluster phase is FailedPhase testHandleClusterPhaseWhenCompsNotReady(clusterObj, appsv1alpha1.FailedClusterCompPhase, appsv1alpha1.FailedClusterPhase) @@ -301,9 +298,9 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { g.Expect(tmpCluster.Status.Components).Should(HaveLen(len(tmpCluster.Spec.ComponentSpecs))) })).Should(Succeed()) - changeAndCheckComponents := func(changeFunc func(), expectObservedGeneration int64, checkFun func(Gomega, *appsv1alpha1.Cluster)) { - Expect(testapps.ChangeObj(&testCtx, cluster, func() { - changeFunc() + changeAndCheckComponents := func(changeFunc func(*appsv1alpha1.Cluster), expectObservedGeneration int64, checkFun func(Gomega, *appsv1alpha1.Cluster)) { + Expect(testapps.ChangeObj(&testCtx, cluster, func(lcluster *appsv1alpha1.Cluster) { + changeFunc(lcluster) })).ShouldNot(HaveOccurred()) // wait for cluster controller reconciles to complete. Eventually(testapps.GetClusterObservedGeneration(&testCtx, client.ObjectKeyFromObject(cluster))).Should(Equal(expectObservedGeneration)) @@ -313,8 +310,8 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { By("delete consensus component") consensusClusterComponent := cluster.Spec.ComponentSpecs[2] changeAndCheckComponents( - func() { - cluster.Spec.ComponentSpecs = cluster.Spec.ComponentSpecs[:2] + func(lcluster *appsv1alpha1.Cluster) { + lcluster.Spec.ComponentSpecs = cluster.Spec.ComponentSpecs[:2] }, 2, func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { g.Expect(tmpCluster.Status.Components).Should(HaveLen(2)) @@ -324,8 +321,8 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { By("add consensus component") consensusClusterComponent.Name = "consensus1" changeAndCheckComponents( - func() { - cluster.Spec.ComponentSpecs = append(cluster.Spec.ComponentSpecs, consensusClusterComponent) + func(lcluster *appsv1alpha1.Cluster) { + lcluster.Spec.ComponentSpecs = append(cluster.Spec.ComponentSpecs, consensusClusterComponent) }, 3, func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { _, isExist := tmpCluster.Status.Components[consensusClusterComponent.Name] @@ -336,8 +333,8 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { By("modify consensus component name") modifyConsensusName := "consensus2" changeAndCheckComponents( - func() { - cluster.Spec.ComponentSpecs[2].Name = modifyConsensusName + func(lcluster *appsv1alpha1.Cluster) { + lcluster.Spec.ComponentSpecs[2].Name = modifyConsensusName }, 4, func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { _, isExist := tmpCluster.Status.Components[modifyConsensusName] diff --git a/controllers/apps/clusterdefinition_controller_test.go b/controllers/apps/clusterdefinition_controller_test.go index 8def6cde6..994762042 100644 --- a/controllers/apps/clusterdefinition_controller_test.go +++ b/controllers/apps/clusterdefinition_controller_test.go @@ -36,7 +36,7 @@ var _ = Describe("ClusterDefinition Controller", func() { const clusterDefName = "test-clusterdef" const clusterVersionName = "test-clusterversion" - const statefulCompType = "replicasets" + const statefulCompDefName = "replicasets" const configVolumeName = "mysql-config" @@ -80,12 +80,12 @@ var _ = Describe("ClusterDefinition Controller", func() { BeforeEach(func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, statefulCompType). + AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) @@ -142,13 +142,13 @@ var _ = Describe("ClusterDefinition Controller", func() { BeforeEach(func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, statefulCompType). + AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). AddConfigTemplate(cmName, cmName, cmName, testCtx.DefaultNamespace, configVolumeName). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/clusterversion_controller_test.go b/controllers/apps/clusterversion_controller_test.go index 35fe7538d..0fbe0606d 100644 --- a/controllers/apps/clusterversion_controller_test.go +++ b/controllers/apps/clusterversion_controller_test.go @@ -34,7 +34,7 @@ var _ = Describe("test clusterVersion controller", func() { clusterDefName = "mysql-definition-" + randomStr ) - const statefulCompType = "stateful" + const statefulCompDefName = "stateful" cleanEnv := func() { // must wait until resources deleted and no longer exist before the testcases start, @@ -54,7 +54,7 @@ var _ = Describe("test clusterVersion controller", func() { It("test clusterVersion controller", func() { By("create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(statefulCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() By("wait for clusterVersion phase is unavailable when clusterDef is not found") @@ -65,7 +65,7 @@ var _ = Describe("test clusterVersion controller", func() { By("create a clusterDefinition obj") testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, statefulCompType). + AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). Create(&testCtx).GetObject() By("wait for clusterVersion phase is available") diff --git a/controllers/apps/components/component.go b/controllers/apps/components/component.go index 213c9e67a..6115c67ca 100644 --- a/controllers/apps/components/component.go +++ b/controllers/apps/components/component.go @@ -59,13 +59,13 @@ func NewComponentByType( } switch componentDef.WorkloadType { case appsv1alpha1.Consensus: - return consensusset.NewConsensusSet(cli, cluster, component, componentDef) + return consensusset.NewConsensusComponent(cli, cluster, component, componentDef) case appsv1alpha1.Replication: - return replicationset.NewReplicationSet(cli, cluster, component, componentDef) + return replicationset.NewReplicationComponent(cli, cluster, component, componentDef) case appsv1alpha1.Stateful: - return stateful.NewStateful(cli, cluster, component, componentDef) + return stateful.NewStatefulComponent(cli, cluster, component, componentDef) case appsv1alpha1.Stateless: - return stateless.NewStateless(cli, cluster, component, componentDef) + return stateless.NewStatelessComponent(cli, cluster, component, componentDef) default: panic("unknown workload type") } diff --git a/controllers/apps/components/component_status_test.go b/controllers/apps/components/component_status_test.go index 38bfa79ef..f6d1bf41e 100644 --- a/controllers/apps/components/component_status_test.go +++ b/controllers/apps/components/component_status_test.go @@ -39,8 +39,8 @@ import ( var _ = Describe("ComponentStatusSynchronizer", func() { const ( - compName = "comp" - compType = "comp" + compName = "comp" + compDefName = "comp" ) var ( @@ -85,11 +85,11 @@ var _ = Describe("ComponentStatusSynchronizer", func() { BeforeEach(func() { clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatelessNginxComponent, compType). + AddComponentDef(testapps.StatelessNginxComponent, compDefName). GetObject() cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(compName, compType). + AddComponent(compName, compDefName). SetReplicas(1). GetObject() @@ -198,11 +198,11 @@ var _ = Describe("ComponentStatusSynchronizer", func() { BeforeEach(func() { clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, compType). + AddComponentDef(testapps.StatefulMySQLComponent, compDefName). GetObject() cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(compName, compType). + AddComponent(compName, compDefName). SetReplicas(int32(3)). GetObject() @@ -324,11 +324,11 @@ var _ = Describe("ComponentStatusSynchronizer", func() { BeforeEach(func() { clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ConsensusMySQLComponent, compType). + AddComponentDef(testapps.ConsensusMySQLComponent, compDefName). Create(&testCtx).GetObject() cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(compName, compType). + AddComponent(compName, compDefName). SetReplicas(int32(3)). Create(&testCtx).GetObject() @@ -450,11 +450,11 @@ var _ = Describe("ComponentStatusSynchronizer", func() { BeforeEach(func() { clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, compType). + AddComponentDef(testapps.ReplicationRedisComponent, compDefName). GetObject() cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(compName, compType). + AddComponent(compName, compDefName). SetReplicas(2). GetObject() @@ -589,7 +589,7 @@ func mockContainerError(pod *corev1.Pod) error { } func setPodRole(pod *corev1.Pod, role string) error { - return testapps.ChangeObj(&testCtx, pod, func() { - pod.Labels[constant.RoleLabelKey] = role + return testapps.ChangeObj(&testCtx, pod, func(lpod *corev1.Pod) { + lpod.Labels[constant.RoleLabelKey] = role }) } diff --git a/controllers/apps/components/consensusset/consensus_set.go b/controllers/apps/components/consensusset/consensus_set.go index eb1022a82..a97e73cbc 100644 --- a/controllers/apps/components/consensusset/consensus_set.go +++ b/controllers/apps/components/consensusset/consensus_set.go @@ -26,6 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/controllers/apps/components/stateful" "github.com/apecloud/kubeblocks/controllers/apps/components/types" "github.com/apecloud/kubeblocks/controllers/apps/components/util" opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" @@ -34,10 +35,7 @@ import ( ) type ConsensusSet struct { - Cli client.Client - Cluster *appsv1alpha1.Cluster - Component *appsv1alpha1.ClusterComponentSpec - componentDef *appsv1alpha1.ClusterComponentDefinition + stateful.Stateful } var _ types.Component = &ConsensusSet{} @@ -65,11 +63,7 @@ func (r *ConsensusSet) IsRunning(ctx context.Context, obj client.Object) (bool, } func (r *ConsensusSet) PodsReady(ctx context.Context, obj client.Object) (bool, error) { - if obj == nil { - return false, nil - } - sts := util.ConvertToStatefulSet(obj) - return util.StatefulSetPodsAreReady(sts, r.Component.Replicas), nil + return r.Stateful.PodsReady(ctx, obj) } func (r *ConsensusSet) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { @@ -95,7 +89,7 @@ func (r *ConsensusSet) HandleProbeTimeoutWhenPodsReady(ctx context.Context, reco if compStatus.PodsReadyTime == nil { return true, nil } - if !util.IsProbeTimeout(r.componentDef, compStatus.PodsReadyTime) { + if !util.IsProbeTimeout(r.ComponentDef, compStatus.PodsReadyTime) { return true, nil } @@ -111,7 +105,7 @@ func (r *ConsensusSet) HandleProbeTimeoutWhenPodsReady(ctx context.Context, reco patch := client.MergeFrom(cluster.DeepCopy()) for _, pod := range podList.Items { role := pod.Labels[constant.RoleLabelKey] - if role == r.componentDef.ConsensusSpec.Leader.Name { + if role == r.ComponentDef.ConsensusSpec.Leader.Name { isFailed = false } if role == "" { @@ -158,7 +152,7 @@ func (r *ConsensusSet) GetPhaseWhenPodsNotReady(ctx context.Context, var ( existLatestRevisionFailedPod bool leaderIsReady bool - consensusSpec = r.componentDef.ConsensusSpec + consensusSpec = r.ComponentDef.ConsensusSpec ) for _, v := range podList.Items { // if the pod is terminating, ignore it @@ -261,18 +255,20 @@ func (r *ConsensusSet) HandleUpdate(ctx context.Context, obj client.Object) erro } return nil } -func NewConsensusSet( +func NewConsensusComponent( cli client.Client, cluster *appsv1alpha1.Cluster, component *appsv1alpha1.ClusterComponentSpec, - componentDef appsv1alpha1.ClusterComponentDefinition) (*ConsensusSet, error) { + componentDef appsv1alpha1.ClusterComponentDefinition) (types.Component, error) { if err := util.ComponentRuntimeReqArgsCheck(cli, cluster, component); err != nil { return nil, err } return &ConsensusSet{ - Cli: cli, - Cluster: cluster, - Component: component, - componentDef: &componentDef, + Stateful: stateful.Stateful{ + Cli: cli, + Cluster: cluster, + Component: component, + ComponentDef: &componentDef, + }, }, nil } diff --git a/controllers/apps/components/consensusset/consensus_test.go b/controllers/apps/components/consensusset/consensus_test.go index 59e7afcee..efd51bc17 100644 --- a/controllers/apps/components/consensusset/consensus_test.go +++ b/controllers/apps/components/consensusset/consensus_test.go @@ -104,7 +104,7 @@ var _ = Describe("Consensus Component", func() { component := cluster.Spec.GetComponentByName(componentName) By("test pods are not ready") - consensusComponent, err := NewConsensusSet(k8sClient, cluster, component, *componentDef) + consensusComponent, err := NewConsensusComponent(k8sClient, cluster, component, *componentDef) Expect(err).Should(Succeed()) sts.Status.AvailableReplicas = *sts.Spec.Replicas - 1 podsReady, _ := consensusComponent.PodsReady(ctx, sts) diff --git a/controllers/apps/components/deployment_controller_test.go b/controllers/apps/components/deployment_controller_test.go index 71fe5c967..6463788a4 100644 --- a/controllers/apps/components/deployment_controller_test.go +++ b/controllers/apps/components/deployment_controller_test.go @@ -44,9 +44,9 @@ var _ = Describe("Deployment Controller", func() { ) const ( - namespace = "default" - statelessCompName = "stateless" - statelessCompType = "stateless" + namespace = "default" + statelessCompName = "stateless" + statelessCompDefName = "stateless" ) cleanAll := func() { @@ -74,11 +74,11 @@ var _ = Describe("Deployment Controller", func() { Context("test controller", func() { It("", func() { testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatelessNginxComponent, statelessCompType). + AddComponentDef(testapps.StatelessNginxComponent, statelessCompDefName). Create(&testCtx).GetObject() cluster := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(statelessCompName, statelessCompType).SetReplicas(2).Create(&testCtx).GetObject() + AddComponent(statelessCompName, statelessCompDefName).SetReplicas(2).Create(&testCtx).GetObject() By("patch cluster to Running") Expect(testapps.ChangeObjStatus(&testCtx, cluster, func() { @@ -123,8 +123,8 @@ var _ = Describe("Deployment Controller", func() { } })).Should(Succeed()) // mark deployment to reconcile - Expect(testapps.ChangeObj(&testCtx, deploy, func() { - deploy.Annotations = map[string]string{ + Expect(testapps.ChangeObj(&testCtx, deploy, func(ldeploy *appsv1.Deployment) { + ldeploy.Annotations = map[string]string{ "reconcile": "1", } })).Should(Succeed()) diff --git a/controllers/apps/components/replicationset/replication_set.go b/controllers/apps/components/replicationset/replication_set.go index c470a7c88..fb7df0e85 100644 --- a/controllers/apps/components/replicationset/replication_set.go +++ b/controllers/apps/components/replicationset/replication_set.go @@ -22,11 +22,11 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/controllers/apps/components/stateful" "github.com/apecloud/kubeblocks/controllers/apps/components/types" "github.com/apecloud/kubeblocks/controllers/apps/components/util" "github.com/apecloud/kubeblocks/internal/constant" @@ -35,10 +35,7 @@ import ( // ReplicationSet is a component object used by Cluster, ClusterComponentDefinition and ClusterComponentSpec type ReplicationSet struct { - Cli client.Client - Cluster *appsv1alpha1.Cluster - Component *appsv1alpha1.ClusterComponentSpec - componentDef *appsv1alpha1.ClusterComponentDefinition + stateful.Stateful } var _ types.Component = &ReplicationSet{} @@ -46,26 +43,17 @@ var _ types.Component = &ReplicationSet{} // IsRunning is the implementation of the type Component interface method, // which is used to check whether the replicationSet component is running normally. func (r *ReplicationSet) IsRunning(ctx context.Context, obj client.Object) (bool, error) { - var componentStsList = &appsv1.StatefulSetList{} var componentStatusIsRunning = true sts := util.ConvertToStatefulSet(obj) - if err := util.GetObjectListByComponentName(ctx, r.Cli, *r.Cluster, - componentStsList, sts.Labels[constant.KBAppComponentLabelKey]); err != nil { + isRevisionConsistent, err := util.IsStsAndPodsRevisionConsistent(ctx, r.Cli, sts) + if err != nil { return false, err } - var availableReplicas int32 - for _, stsObj := range componentStsList.Items { - isRevisionConsistent, err := util.IsStsAndPodsRevisionConsistent(ctx, r.Cli, sts) - if err != nil { - return false, err - } - stsIsReady := util.StatefulSetOfComponentIsReady(&stsObj, isRevisionConsistent, nil) - availableReplicas += stsObj.Status.AvailableReplicas - if !stsIsReady { - return false, nil - } + stsIsReady := util.StatefulSetOfComponentIsReady(sts, isRevisionConsistent, nil) + if !stsIsReady { + return false, nil } - if availableReplicas < r.Component.Replicas { + if sts.Status.AvailableReplicas < r.Component.Replicas { componentStatusIsRunning = false } return componentStatusIsRunning, nil @@ -74,21 +62,7 @@ func (r *ReplicationSet) IsRunning(ctx context.Context, obj client.Object) (bool // PodsReady is the implementation of the type Component interface method, // which is used to check whether all the pods of replicationSet component is ready. func (r *ReplicationSet) PodsReady(ctx context.Context, obj client.Object) (bool, error) { - var podsReady = true - var componentStsList = &appsv1.StatefulSetList{} - sts := util.ConvertToStatefulSet(obj) - if err := util.GetObjectListByComponentName(ctx, r.Cli, *r.Cluster, componentStsList, - sts.Labels[constant.KBAppComponentLabelKey]); err != nil { - return false, err - } - var availableReplicas int32 - for _, stsObj := range componentStsList.Items { - availableReplicas += stsObj.Status.AvailableReplicas - } - if availableReplicas < r.Component.Replicas { - podsReady = false - } - return podsReady, nil + return r.Stateful.PodsReady(ctx, obj) } // PodIsAvailable is the implementation of the type Component interface method, @@ -109,30 +83,29 @@ func (r *ReplicationSet) HandleProbeTimeoutWhenPodsReady(ctx context.Context, re // GetPhaseWhenPodsNotReady is the implementation of the type Component interface method, // when the pods of replicationSet are not ready, calculate the component phase is Failed or Abnormal. // if return an empty phase, means the pods of component are ready and skips it. -func (r *ReplicationSet) GetPhaseWhenPodsNotReady(ctx context.Context, componentName string) (appsv1alpha1.ClusterComponentPhase, error) { - componentStsList := &appsv1.StatefulSetList{} - podList, err := util.GetCompRelatedObjectList(ctx, r.Cli, *r.Cluster, componentName, componentStsList) - if err != nil || len(componentStsList.Items) == 0 { +func (r *ReplicationSet) GetPhaseWhenPodsNotReady(ctx context.Context, + componentName string) (appsv1alpha1.ClusterComponentPhase, error) { + stsList := &appsv1.StatefulSetList{} + podList, err := util.GetCompRelatedObjectList(ctx, r.Cli, *r.Cluster, + componentName, stsList) + if err != nil || len(stsList.Items) == 0 { return "", err } - podCount, componentReplicas := len(podList.Items), r.Component.Replicas - if podCount == 0 { + stsObj := stsList.Items[0] + podCount := len(podList.Items) + componentReplicas := r.Component.Replicas + if podCount == 0 || stsObj.Status.AvailableReplicas == 0 { return util.GetPhaseWithNoAvailableReplicas(componentReplicas), nil } + // get the statefulSet of component var ( - stsMap = make(map[string]appsv1.StatefulSet) - availableReplicas int32 - primaryIsReady bool existLatestRevisionFailedPod bool + primaryIsReady bool needPatch bool compStatus = r.Cluster.Status.Components[componentName] ) - for _, v := range componentStsList.Items { - stsMap[v.Name] = v - availableReplicas += v.Status.AvailableReplicas - } for _, v := range podList.Items { - // if the pod is terminating, ignore the warning event. + // if the pod is terminating, ignore it if v.DeletionTimestamp != nil { return "", nil } @@ -145,13 +118,10 @@ func (r *ReplicationSet) GetPhaseWhenPodsNotReady(ctx context.Context, component compStatus.SetObjectMessage(v.Kind, v.Name, "empty label for pod, please check.") needPatch = true } - controllerRef := metav1.GetControllerOf(&v) - stsObj := stsMap[controllerRef.Name] if !intctrlutil.PodIsReady(&v) && intctrlutil.PodIsControlledByLatestRevision(&v, &stsObj) { existLatestRevisionFailedPod = true } } - // REVIEW: this isn't a get function, where r.Cluster.Status.Components is being updated. // patch abnormal reason to cluster.status.ComponentDefs. if needPatch { @@ -162,35 +132,32 @@ func (r *ReplicationSet) GetPhaseWhenPodsNotReady(ctx context.Context, component } } return util.GetCompPhaseByConditions(existLatestRevisionFailedPod, primaryIsReady, - componentReplicas, int32(podCount), availableReplicas), nil + componentReplicas, int32(podCount), stsObj.Status.AvailableReplicas), nil } // HandleUpdate is the implementation of the type Component interface method, handles replicationSet workload Pod updates. func (r *ReplicationSet) HandleUpdate(ctx context.Context, obj client.Object) error { - var componentStsList = &appsv1.StatefulSetList{} - var podList []*corev1.Pod sts := util.ConvertToStatefulSet(obj) - if err := util.GetObjectListByComponentName(ctx, r.Cli, *r.Cluster, componentStsList, - sts.Labels[constant.KBAppComponentLabelKey]); err != nil { + if sts.Generation != sts.Status.ObservedGeneration { + return nil + } + podList, err := util.GetPodListByStatefulSet(ctx, r.Cli, sts) + if err != nil { return err } - for _, sts := range componentStsList.Items { - if sts.Generation != sts.Status.ObservedGeneration { - continue - } - pod, err := getAndCheckReplicationPodByStatefulSet(ctx, r.Cli, &sts) - if err != nil { - return err - } + for _, pod := range podList { // if there is no role label on the Pod, it needs to be updated with statefulSet's role label. if v, ok := pod.Labels[constant.RoleLabelKey]; !ok || v == "" { - if err := updateObjRoleLabel(ctx, r.Cli, *pod, sts.Labels[constant.RoleLabelKey]); err != nil { + _, o := util.ParseParentNameAndOrdinal(pod.Name) + role := string(Secondary) + if o == r.Component.GetPrimaryIndex() { + role = string(Primary) + } + if err := updateObjRoleLabel(ctx, r.Cli, pod, role); err != nil { return err } - } else { - podList = append(podList, pod) } - if err := util.DeleteStsPods(ctx, r.Cli, &sts); err != nil { + if err := util.DeleteStsPods(ctx, r.Cli, sts); err != nil { return err } } @@ -209,19 +176,29 @@ func (r *ReplicationSet) HandleUpdate(ctx context.Context, obj client.Object) er return r.Cli.Status().Patch(ctx, r.Cluster, client.MergeFrom(clusterDeepCopy)) } -// NewReplicationSet creates a new ReplicationSet object. -func NewReplicationSet( +func DefaultRole(i int32) string { + role := string(Secondary) + if i == 0 { + role = string(Primary) + } + return role +} + +// NewReplicationComponent creates a new ReplicationSet object. +func NewReplicationComponent( cli client.Client, cluster *appsv1alpha1.Cluster, component *appsv1alpha1.ClusterComponentSpec, - componentDef appsv1alpha1.ClusterComponentDefinition) (*ReplicationSet, error) { + componentDef appsv1alpha1.ClusterComponentDefinition) (types.Component, error) { if err := util.ComponentRuntimeReqArgsCheck(cli, cluster, component); err != nil { return nil, err } return &ReplicationSet{ - Cli: cli, - Cluster: cluster, - Component: component, - componentDef: &componentDef, + Stateful: stateful.Stateful{ + Cli: cli, + Cluster: cluster, + Component: component, + ComponentDef: &componentDef, + }, }, nil } diff --git a/controllers/apps/components/replicationset/replication_set_switch.go b/controllers/apps/components/replicationset/replication_set_switch.go index 469262879..3723cd61c 100644 --- a/controllers/apps/components/replicationset/replication_set_switch.go +++ b/controllers/apps/components/replicationset/replication_set_switch.go @@ -371,9 +371,9 @@ func (s *Switch) updateRoleLabel() error { // initSwitchInstance initializes the switchInstance object without detection info according to the pod list under the component, // and the detection information will be filled in the detection phase. func (s *Switch) initSwitchInstance(oldPrimaryIndex, newPrimaryIndex int32) error { - var stsList = &appsv1.StatefulSetList{} + var podList = &corev1.PodList{} if err := utils.GetObjectListByComponentName(s.SwitchResource.Ctx, s.SwitchResource.Cli, - *s.SwitchResource.Cluster, stsList, s.SwitchResource.CompSpec.Name); err != nil { + *s.SwitchResource.Cluster, podList, s.SwitchResource.CompSpec.Name); err != nil { return err } if s.SwitchInstance == nil { @@ -383,35 +383,15 @@ func (s *Switch) initSwitchInstance(oldPrimaryIndex, newPrimaryIndex int32) erro SecondariesRole: make([]*SwitchRoleInfo, 0), } } - for _, sts := range stsList.Items { - pod, err := getAndCheckReplicationPodByStatefulSet(s.SwitchResource.Ctx, s.SwitchResource.Cli, &sts) - if err != nil { - return err - } + for _, pod := range podList.Items { sri := &SwitchRoleInfo{ - Pod: pod, + Pod: &pod, HealthDetectInfo: nil, RoleDetectInfo: nil, LagDetectInfo: nil, } - - // because the first sts is named differently than the other sts, special handling is required here. - // TODO: The following code is not very elegant, and it is recommended to be optimized in the future. - clusterCompName := fmt.Sprintf("%s-%s", s.SwitchResource.Cluster.GetName(), s.SwitchResource.CompSpec.Name) - if sts.GetName() == clusterCompName { - if oldPrimaryIndex == 0 { - s.SwitchInstance.OldPrimaryRole = sri - continue - } - if newPrimaryIndex == 0 { - s.SwitchInstance.CandidatePrimaryRole = sri - continue - } - s.SwitchInstance.SecondariesRole = append(s.SwitchInstance.SecondariesRole, sri) - continue - } - - switch int32(utils.GetOrdinalSts(&sts)) { + _, o := utils.ParseParentNameAndOrdinal(pod.Name) + switch o { case oldPrimaryIndex: s.SwitchInstance.OldPrimaryRole = sri case newPrimaryIndex: diff --git a/controllers/apps/components/replicationset/replication_set_switch_test.go b/controllers/apps/components/replicationset/replication_set_switch_test.go index 2008dcc19..fa4a5ab23 100644 --- a/controllers/apps/components/replicationset/replication_set_switch_test.go +++ b/controllers/apps/components/replicationset/replication_set_switch_test.go @@ -17,10 +17,11 @@ limitations under the License. package replicationset import ( + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -98,7 +99,7 @@ var _ = Describe("ReplicationSet Switch", func() { By("Creating a cluster with replication workloadType.") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). SetReplicas(testapps.DefaultReplicationReplicas). SetPrimaryIndex(DefaultPrimaryIndexDiffWithStsOrdinal). SetSwitchPolicy(clusterSwitchPolicy). @@ -110,38 +111,28 @@ var _ = Describe("ReplicationSet Switch", func() { Image: testapps.DefaultRedisImageName, ImagePullPolicy: corev1.PullIfNotPresent, } - stsList := make([]*appsv1.StatefulSet, 0) - for k, v := range map[string]string{ - clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-0": string(Primary), - clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-1": string(Secondary), - clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-2": string(Secondary), - clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-3": string(Secondary), - } { - sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, k, clusterObj.Name, testapps.DefaultRedisCompName). - AddContainer(container). - AddAppInstanceLabel(clusterObj.Name). - AddAppComponentLabel(testapps.DefaultRedisCompName). - AddAppManangedByLabel(). - AddRoleLabel(v). - SetReplicas(1). - Create(&testCtx).GetObject() - isStsPrimary, err := checkObjRoleLabelIsPrimary(sts) - if v == string(Primary) { - Expect(err).To(Succeed()) - Expect(isStsPrimary).Should(BeTrue()) - } else { - Expect(err).To(Succeed()) - Expect(isStsPrimary).ShouldNot(BeTrue()) - } - stsList = append(stsList, sts) - } + + replicationSetSts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, + clusterObj.Name+"-"+testapps.DefaultRedisCompName, clusterObj.Name, testapps.DefaultRedisCompName). + AddContainer(container). + AddAppInstanceLabel(clusterObj.Name). + AddAppComponentLabel(testapps.DefaultRedisCompName). + AddAppManangedByLabel(). + SetReplicas(4). + Create(&testCtx).GetObject() By("Creating Pods of replication workloadType.") - for _, sts := range stsList { - _ = testapps.NewPodFactory(testCtx.DefaultNamespace, sts.Name+"-0"). + for i := int32(0); i < *replicationSetSts.Spec.Replicas; i++ { + podBuilder := testapps.NewPodFactory(testCtx.DefaultNamespace, + fmt.Sprintf("%s-%d", replicationSetSts.Name, i)). AddContainer(container). - AddLabelsInMap(sts.Labels). - Create(&testCtx).GetObject() + AddLabelsInMap(replicationSetSts.Labels) + if i == 0 { + podBuilder.AddRoleLabel(string(Primary)) + } else { + podBuilder.AddRoleLabel(string(Secondary)) + } + _ = podBuilder.Create(&testCtx).GetObject() } clusterComponentSpec := &clusterObj.Spec.ComponentSpecs[0] @@ -162,7 +153,7 @@ var _ = Describe("ReplicationSet Switch", func() { Expect(s.SwitchStatus.SwitchPhaseStatus).Should(Equal(SwitchPhaseStatusSucceed)) By("Test switch election with multi secondaries should be successful, and the candidate primary should be the priorityPod.") - priorityPod := clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-2-0" + priorityPod := clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-3" for _, sri := range s.SwitchInstance.SecondariesRole { if sri.Pod.Name != priorityPod { sri.LagDetectInfo = &lagNotZero @@ -289,13 +280,13 @@ var _ = Describe("ReplicationSet Switch", func() { } By("Create a clusterDefinition obj with replication workloadType.") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompType). + AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). AddReplicationSpec(mockReplicationSpec). Create(&testCtx).GetObject() By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(testapps.DefaultRedisCompType).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/components/replicationset/replication_set_switch_utils.go b/controllers/apps/components/replicationset/replication_set_switch_utils.go index ac507f947..dd7ce19a3 100644 --- a/controllers/apps/components/replicationset/replication_set_switch_utils.go +++ b/controllers/apps/components/replicationset/replication_set_switch_utils.go @@ -481,20 +481,14 @@ func CheckPrimaryIndexChanged(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster, compName string, - specPrimaryIndex int32) (bool, int32, error) { + currentPrimaryIndex int32) (bool, int32, error) { // get the statefulSet object whose current role label is primary - primarySts, err := getReplicationSetPrimaryObj(ctx, cli, cluster, generics.StatefulSetSignature, compName) + pod, err := getReplicationSetPrimaryObj(ctx, cli, cluster, generics.PodSignature, compName) if err != nil { return false, -1, err } - - clusterCompName := fmt.Sprintf("%s-%s", cluster.GetName(), compName) - if primarySts.GetName() == clusterCompName { - return specPrimaryIndex != 0, 0, nil - } - - currentPrimaryIndex := int32(util.GetOrdinalSts(primarySts)) - return specPrimaryIndex != currentPrimaryIndex, currentPrimaryIndex, nil + _, o := util.ParseParentNameAndOrdinal(pod.Name) + return currentPrimaryIndex != o, o, nil } // syncPrimaryIndex syncs cluster.spec.componentSpecs.[x].primaryIndex when failover occurs and switchPolicy is Noop. diff --git a/controllers/apps/components/replicationset/replication_set_switch_utils_test.go b/controllers/apps/components/replicationset/replication_set_switch_utils_test.go index f284cdbc7..a437bcebe 100644 --- a/controllers/apps/components/replicationset/replication_set_switch_utils_test.go +++ b/controllers/apps/components/replicationset/replication_set_switch_utils_test.go @@ -17,10 +17,11 @@ limitations under the License. package replicationset import ( + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -75,7 +76,7 @@ var _ = Describe("ReplicationSet Switch Util", func() { By("Creating a cluster with replication workloadType.") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). SetReplicas(testapps.DefaultReplicationReplicas). SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). SetSwitchPolicy(clusterSwitchPolicy). @@ -87,35 +88,24 @@ var _ = Describe("ReplicationSet Switch Util", func() { Image: testapps.DefaultRedisImageName, ImagePullPolicy: corev1.PullIfNotPresent, } - stsList := make([]*appsv1.StatefulSet, 0) - for k, v := range map[string]string{ - string(Primary): clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-0", - string(Secondary): clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-1", - } { - sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, v, clusterObj.Name, testapps.DefaultRedisCompName). - AddContainer(container). - AddAppInstanceLabel(clusterObj.Name). - AddAppComponentLabel(testapps.DefaultRedisCompName). - AddAppManangedByLabel(). - AddRoleLabel(k). - SetReplicas(1). - Create(&testCtx).GetObject() - isStsPrimary, err := checkObjRoleLabelIsPrimary(sts) - if k == string(Primary) { - Expect(err).To(Succeed()) - Expect(isStsPrimary).Should(BeTrue()) - } else { - Expect(err).To(Succeed()) - Expect(isStsPrimary).ShouldNot(BeTrue()) - } - stsList = append(stsList, sts) - } + sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, + clusterObj.Name+"-"+testapps.DefaultRedisCompName, + clusterObj.Name, + testapps.DefaultRedisCompName). + AddContainer(container). + AddAppInstanceLabel(clusterObj.Name). + AddAppComponentLabel(testapps.DefaultRedisCompName). + AddAppManangedByLabel(). + SetReplicas(2). + Create(&testCtx).GetObject() By("Creating Pods of replication workloadType.") - for _, sts := range stsList { - _ = testapps.NewPodFactory(testCtx.DefaultNamespace, sts.Name+"-0"). + + for i := int32(0); i < *sts.Spec.Replicas; i++ { + _ = testapps.NewPodFactory(testCtx.DefaultNamespace, fmt.Sprintf("%s-%d", sts.Name, i)). AddContainer(container). AddLabelsInMap(sts.Labels). + AddRoleLabel(DefaultRole(i)). Create(&testCtx).GetObject() } clusterComponentSpec := &clusterObj.Spec.ComponentSpecs[0] @@ -131,12 +121,14 @@ var _ = Describe("ReplicationSet Switch Util", func() { Expect(err).Should(Succeed()) By("Test update cluster component primaryIndex should be successful.") - testapps.UpdateClusterCompSpecPrimaryIndex(&testCtx, clusterObj, clusterComponentSpec.Name, &DefaultPrimaryIndexDiffWithStsOrdinal) + testapps.UpdateClusterCompSpecPrimaryIndex(&testCtx, clusterObj, clusterComponentSpec.Name, + &DefaultPrimaryIndexDiffWithStsOrdinal) By("Test new Switch obj and init SwitchInstance should be successful.") clusterObj.Spec.ComponentSpecs[0].PrimaryIndex = &DefaultPrimaryIndexDiffWithStsOrdinal clusterComponentSpec.PrimaryIndex = &DefaultPrimaryIndexDiffWithStsOrdinal - s := newSwitch(testCtx.Ctx, k8sClient, clusterObj, &clusterDefObj.Spec.ComponentDefs[0], clusterComponentSpec, nil, nil, nil, nil, nil) + s := newSwitch(testCtx.Ctx, k8sClient, clusterObj, &clusterDefObj.Spec.ComponentDefs[0], clusterComponentSpec, + nil, nil, nil, nil, nil) err = s.initSwitchInstance(DefaultReplicationPrimaryIndex, DefaultPrimaryIndexDiffWithStsOrdinal) Expect(err).Should(Succeed()) @@ -206,13 +198,13 @@ var _ = Describe("ReplicationSet Switch Util", func() { } By("Create a clusterDefinition obj with replication workloadType.") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompType). + AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). AddReplicationSpec(replicationSpec). Create(&testCtx).GetObject() By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(testapps.DefaultRedisCompType).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/components/replicationset/replication_set_test.go b/controllers/apps/components/replicationset/replication_set_test.go index a758cfd16..fef095806 100644 --- a/controllers/apps/components/replicationset/replication_set_test.go +++ b/controllers/apps/components/replicationset/replication_set_test.go @@ -72,18 +72,18 @@ var _ = Describe("Replication Component", func() { By("Create a clusterDefinition obj with replication workloadType.") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompType). + AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(testapps.DefaultRedisCompType).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() By("Creating a cluster with replication workloadType.") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). SetReplicas(testapps.DefaultReplicationReplicas). Create(&testCtx).GetObject() @@ -96,80 +96,62 @@ var _ = Describe("Replication Component", func() { } })).Should(Succeed()) - By("Creating two statefulSets of replication workloadType.") + By("Creating statefulSet of replication workloadType.") + replicas := int32(2) status := appsv1.StatefulSetStatus{ - AvailableReplicas: 1, + AvailableReplicas: replicas, ObservedGeneration: 1, - Replicas: 1, - ReadyReplicas: 1, - UpdatedReplicas: 1, + Replicas: replicas, + ReadyReplicas: replicas, + UpdatedReplicas: replicas, CurrentRevision: controllerRivision, UpdateRevision: controllerRivision, } - var ( - primarySts *appsv1.StatefulSet - secondarySts *appsv1.StatefulSet - ) - for k, v := range map[string]string{ - string(Primary): clusterObj.Name + "-" + testapps.DefaultRedisCompName, - string(Secondary): clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-1", - } { - sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, v, clusterObj.Name, testapps.DefaultRedisCompName). - AddContainer(corev1.Container{Name: testapps.DefaultRedisContainerName, Image: testapps.DefaultRedisImageName}). - AddAppInstanceLabel(clusterObj.Name). - AddAppComponentLabel(testapps.DefaultRedisCompName). - AddAppManangedByLabel(). - AddRoleLabel(k). - SetReplicas(1). - Create(&testCtx).GetObject() - isStsPrimary, err := checkObjRoleLabelIsPrimary(sts) - if k == string(Primary) { - Expect(err).To(Succeed()) - Expect(isStsPrimary).Should(BeTrue()) - primarySts = sts - } else { - Expect(err).To(Succeed()) - Expect(isStsPrimary).ShouldNot(BeTrue()) - secondarySts = sts - } - Expect(sts.Spec.VolumeClaimTemplates).Should(BeEmpty()) - } + replicationSetSts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, + clusterObj.Name+"-"+testapps.DefaultRedisCompName, clusterObj.Name, testapps.DefaultRedisCompName). + AddContainer(corev1.Container{Name: testapps.DefaultRedisContainerName, Image: testapps.DefaultRedisImageName}). + AddAppInstanceLabel(clusterObj.Name). + AddAppComponentLabel(testapps.DefaultRedisCompName). + AddAppManangedByLabel(). + SetReplicas(replicas). + Create(&testCtx).GetObject() + + Expect(replicationSetSts.Spec.VolumeClaimTemplates).Should(BeEmpty()) compDefName := clusterObj.Spec.GetComponentDefRefName(testapps.DefaultRedisCompName) componentDef := clusterDefObj.GetComponentDefByName(compDefName) component := clusterObj.Spec.GetComponentByName(testapps.DefaultRedisCompName) - replicationComponent, err := NewReplicationSet(k8sClient, clusterObj, component, *componentDef) + replicationComponent, err := NewReplicationComponent(k8sClient, clusterObj, component, *componentDef) Expect(err).Should(Succeed()) var podList []*corev1.Pod - for _, availableReplica := range []int32{0, 1} { - status.AvailableReplicas = availableReplica - primarySts.Status = status - testk8s.PatchStatefulSetStatus(&testCtx, primarySts.Name, status) - secondarySts.Status = status - testk8s.PatchStatefulSetStatus(&testCtx, secondarySts.Name, status) - // Create pod of the statefulset - if availableReplica == 1 { - sts1Pod := testapps.MockReplicationComponentPods(testCtx, primarySts, clusterObj.Name, testapps.DefaultRedisCompName, string(Primary)) - podList = append(podList, sts1Pod...) - sts2Pod := testapps.MockReplicationComponentPods(testCtx, secondarySts, clusterObj.Name, testapps.DefaultRedisCompName, string(Secondary)) - podList = append(podList, sts2Pod...) - } - podsReady, _ := replicationComponent.PodsReady(ctx, primarySts) - isRunning, _ := replicationComponent.IsRunning(ctx, primarySts) - if availableReplica == 1 { + for _, availableReplica := range []int32{0, replicas} { + status.AvailableReplicas = availableReplica + replicationSetSts.Status = status + testk8s.PatchStatefulSetStatus(&testCtx, replicationSetSts.Name, status) + + if availableReplica > 0 { + // Create pods of the statefulset + stsPods := testapps.MockReplicationComponentPods(nil, testCtx, replicationSetSts, clusterObj.Name, + testapps.DefaultRedisCompName, map[int32]string{ + 0: string(Primary), + 1: string(Secondary), + }) + podList = append(podList, stsPods...) By("Testing pods are ready") - Expect(podsReady == true).Should(BeTrue()) - + podsReady, _ := replicationComponent.PodsReady(ctx, replicationSetSts) + Expect(podsReady).Should(BeTrue()) By("Testing component is running") - Expect(isRunning == true).Should(BeTrue()) + isRunning, _ := replicationComponent.IsRunning(ctx, replicationSetSts) + Expect(isRunning).Should(BeTrue()) } else { + podsReady, _ := replicationComponent.PodsReady(ctx, replicationSetSts) By("Testing pods are not ready") - Expect(podsReady == false).Should(BeTrue()) - + Expect(podsReady).Should(BeFalse()) By("Testing component is not running") - Expect(isRunning == false).Should(BeTrue()) + isRunning, _ := replicationComponent.IsRunning(ctx, replicationSetSts) + Expect(isRunning).Should(BeFalse()) } } @@ -183,10 +165,9 @@ var _ = Describe("Replication Component", func() { By("Testing component phase when pods not ready") // mock secondary pod is not ready. - Expect(testapps.ChangeObjStatus(&testCtx, secondarySts, func() { - secondarySts.Status.AvailableReplicas = 0 - })).Should(Succeed()) testk8s.UpdatePodStatusNotReady(ctx, testCtx, podList[1].Name) + status.AvailableReplicas -= 1 + testk8s.PatchStatefulSetStatus(&testCtx, replicationSetSts.Name, status) phase, _ := replicationComponent.GetPhaseWhenPodsNotReady(ctx, testapps.DefaultRedisCompName) Expect(phase).Should(Equal(appsv1alpha1.AbnormalClusterCompPhase)) @@ -196,8 +177,8 @@ var _ = Describe("Replication Component", func() { Expect(phase).Should(Equal(appsv1alpha1.FailedClusterCompPhase)) // mock pod label is empty - Expect(testapps.ChangeObj(&testCtx, primaryPod, func() { - primaryPod.Labels[constant.RoleLabelKey] = "" + Expect(testapps.ChangeObj(&testCtx, primaryPod, func(lpod *corev1.Pod) { + lpod.Labels[constant.RoleLabelKey] = "" })).Should(Succeed()) _, _ = replicationComponent.GetPhaseWhenPodsNotReady(ctx, testapps.DefaultRedisCompName) Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(clusterObj), @@ -208,19 +189,9 @@ var _ = Describe("Replication Component", func() { })).Should(Succeed()) By("Checking if the pod is not updated when statefulset is not updated") - Expect(replicationComponent.HandleUpdate(ctx, primarySts)).To(Succeed()) - primaryStsPodList, err := util.GetPodListByStatefulSet(ctx, k8sClient, primarySts) - Expect(err).To(Succeed()) - Expect(len(primaryStsPodList)).To(Equal(1)) - Expect(util.IsStsAndPodsRevisionConsistent(ctx, k8sClient, primarySts)).Should(BeTrue()) - - By("Checking if the pod is deleted when statefulset is updated") - status.UpdateRevision = "new-mock-revision" - testk8s.PatchStatefulSetStatus(&testCtx, primarySts.Name, status) - Expect(replicationComponent.HandleUpdate(ctx, primarySts)).To(Succeed()) - primaryStsPodList, err = util.GetPodListByStatefulSet(ctx, k8sClient, primarySts) + Expect(replicationComponent.HandleUpdate(ctx, replicationSetSts)).To(Succeed()) Expect(err).To(Succeed()) - Expect(len(primaryStsPodList)).To(Equal(0)) + Expect(util.IsStsAndPodsRevisionConsistent(ctx, k8sClient, replicationSetSts)).Should(BeTrue()) }) }) }) diff --git a/controllers/apps/components/replicationset/replication_set_utils.go b/controllers/apps/components/replicationset/replication_set_utils.go index 524cba152..6e1521388 100644 --- a/controllers/apps/components/replicationset/replication_set_utils.go +++ b/controllers/apps/components/replicationset/replication_set_utils.go @@ -20,15 +20,12 @@ import ( "context" "fmt" "reflect" - "sort" "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/controllers/apps/components/util" @@ -45,155 +42,17 @@ const ( DBClusterFinalizerName = "cluster.kubeblocks.io/finalizer" ) -// HandleReplicationSet handles the replication workload life cycle process, including horizontal scaling, etc. -func HandleReplicationSet(ctx context.Context, - cli client.Client, - cluster *appsv1alpha1.Cluster, - stsList []*appsv1.StatefulSet) error { - if cluster == nil { - return util.ErrReqClusterObj - } - // handle replication workload horizontal scaling - if err := handleReplicationSetHorizontalScale(ctx, cli, cluster, stsList); err != nil { - return err - } - return nil -} - -// handleReplicationSetHorizontalScale handles changes of replication workload replicas and synchronizes cluster status. -func handleReplicationSetHorizontalScale(ctx context.Context, - cli client.Client, - cluster *appsv1alpha1.Cluster, - stsList []*appsv1.StatefulSet) error { - - // handle StatefulSets including delete sts when pod number larger than cluster.component[i].replicas - // delete the StatefulSets with the largest sequence number which is not the primary role - clusterCompReplicasMap := make(map[string]int32, len(cluster.Spec.ComponentSpecs)) - for _, clusterComp := range cluster.Spec.ComponentSpecs { - clusterCompReplicasMap[clusterComp.Name] = clusterComp.Replicas - } - - // compOwnsStsMap is used to divide stsList into sts list under each replicationSet component according to componentLabelKey - compOwnsStsMap := make(map[string][]*appsv1.StatefulSet) - for _, stsObj := range stsList { - compName := stsObj.Labels[constant.KBAppComponentLabelKey] - compDef, err := filterReplicationWorkload(ctx, cli, cluster, compName) - if err != nil { - return err - } - if compDef == nil { - continue - } - compOwnsStsMap[compName] = append(compOwnsStsMap[compName], stsObj) - } - - // stsToDeleteMap is used to record the count of statefulsets to be deleted when horizontal scale-in - stsToDeleteMap := make(map[string]int32) - for compName := range compOwnsStsMap { - if int32(len(compOwnsStsMap[compName])) > clusterCompReplicasMap[compName] { - stsToDeleteMap[compName] = int32(len(compOwnsStsMap[compName])) - clusterCompReplicasMap[compName] - } - } - if len(stsToDeleteMap) > 0 { - if err := doHorizontalScaleDown(ctx, cli, cluster, compOwnsStsMap, clusterCompReplicasMap, stsToDeleteMap); err != nil { - return err - } - } - return nil -} - -// handleComponentIsStopped checks the component status is stopped and updates it. -func handleComponentIsStopped(cluster *appsv1alpha1.Cluster) { - for _, clusterComp := range cluster.Spec.ComponentSpecs { - if clusterComp.Replicas == int32(0) { - replicationStatus := cluster.Status.Components[clusterComp.Name] - replicationStatus.Phase = appsv1alpha1.StoppedClusterCompPhase - cluster.Status.SetComponentStatus(clusterComp.Name, replicationStatus) - } - } -} - -func doHorizontalScaleDown(ctx context.Context, - cli client.Client, - cluster *appsv1alpha1.Cluster, - compOwnsStsMap map[string][]*appsv1.StatefulSet, - clusterCompReplicasMap map[string]int32, - stsToDeleteMap map[string]int32) error { - // remove cluster status and delete sts when horizontal scale-in - for compName, stsToDelCount := range stsToDeleteMap { - // list all statefulSets by cluster and componentKey label - var componentStsList = &appsv1.StatefulSetList{} - err := util.GetObjectListByComponentName(ctx, cli, *cluster, componentStsList, compName) - if err != nil { - return err - } - if int32(len(compOwnsStsMap[compName])) != int32(len(componentStsList.Items)) { - return fmt.Errorf("statefulset total number has changed") - } - dos := make([]*appsv1.StatefulSet, 0) - partition := int32(len(componentStsList.Items)) - stsToDelCount - componentReplicas := clusterCompReplicasMap[compName] - var primarySts *appsv1.StatefulSet - for _, sts := range componentStsList.Items { - // if current primary statefulSet ordinal is larger than target number replica, return err - stsIsPrimary, err := checkObjRoleLabelIsPrimary(&sts) - primarySts = &sts - if err != nil { - return err - } - // check if the current primary statefulSet ordinal is larger than target replicas number of component when the target number is not 0. - if int32(util.GetOrdinalSts(&sts)) >= partition && stsIsPrimary && componentReplicas != 0 { - return fmt.Errorf("current primary statefulset ordinal is larger than target number replicas, can not be reduce, please switchover first") - } - dos = append(dos, sts.DeepCopy()) - } - - // sort the statefulSets by their ordinals desc - sort.Sort(util.DescendingOrdinalSts(dos)) - - if err = RemoveReplicationSetClusterStatus(cli, ctx, cluster, dos[:stsToDelCount], componentReplicas); err != nil { - return err - } - for i := int32(0); i < stsToDelCount; i++ { - err = cli.Delete(ctx, dos[i]) - if err == nil { - patch := client.MergeFrom(dos[i].DeepCopy()) - controllerutil.RemoveFinalizer(dos[i], DBClusterFinalizerName) - if err = cli.Patch(ctx, dos[i], patch); err != nil { - return err - } - continue - } - if apierrors.IsNotFound(err) { - continue - } - return err - } - // reconcile the primary statefulSet after deleting other sts to make component phase is correct. - if componentReplicas != 0 { - if err = util.MarkPrimaryStsToReconcile(ctx, cli, primarySts); err != nil { - return client.IgnoreNotFound(err) - } - } - } - - // if component replicas is 0, handle replication component status after scaling down the replicas. - handleComponentIsStopped(cluster) - return nil -} - // syncReplicationSetClusterStatus syncs replicationSet pod status to cluster.status.component[componentName].ReplicationStatus. func syncReplicationSetClusterStatus( cli client.Client, ctx context.Context, cluster *appsv1alpha1.Cluster, - podList []*corev1.Pod) error { + podList []corev1.Pod) error { if len(podList) == 0 { return nil } - // update cluster status - componentName, componentDef, err := util.GetComponentInfoByPod(ctx, cli, *cluster, podList[0]) + componentName, componentDef, err := util.GetComponentInfoByPod(ctx, cli, *cluster, &podList[0]) if err != nil { return err } @@ -214,7 +73,7 @@ func syncReplicationSetClusterStatus( } // syncReplicationSetStatus syncs the target pod info in cluster.status.components. -func syncReplicationSetStatus(replicationStatus *appsv1alpha1.ReplicationSetStatus, podList []*corev1.Pod) error { +func syncReplicationSetStatus(replicationStatus *appsv1alpha1.ReplicationSetStatus, podList []corev1.Pod) error { for _, pod := range podList { role := pod.Labels[constant.RoleLabelKey] if role == "" { @@ -396,18 +255,6 @@ func filterReplicationWorkload(ctx context.Context, return compDef, nil } -// getAndCheckReplicationPodByStatefulSet checks the number of replication statefulSet equal 1 and returns it. -func getAndCheckReplicationPodByStatefulSet(ctx context.Context, cli client.Client, stsObj *appsv1.StatefulSet) (*corev1.Pod, error) { - podList, err := util.GetPodListByStatefulSet(ctx, cli, stsObj) - if err != nil { - return nil, err - } - if len(podList) != 1 { - return nil, fmt.Errorf("pod number in statefulset %s is not 1", stsObj.Name) - } - return &podList[0], nil -} - // HandleReplicationSetRoleChangeEvent handles the role change event of the replication workload when switchPolicy is Noop. func HandleReplicationSetRoleChangeEvent(cli client.Client, reqCtx intctrlutil.RequestCtx, @@ -438,40 +285,23 @@ func HandleReplicationSetRoleChangeEvent(cli client.Client, return err } // pod is old primary and newRole is secondary, it means that the old primary needs to be changed to secondary, - // we do not deal with this situation because We will only change the old primary to secondary when the new primary changes from secondary to primary, + // we do not deal with this situation because only change the old primary to secondary when the new primary + // changes from secondary to primary, // this is to avoid simultaneous occurrence of two primary or no primary at the same time if oldPrimaryPod.Name == pod.Name { - reqCtx.Log.Info("pod is old primary and new role is secondary, do not deal with this situation", "podName", pod.Name, "newRole", newRole) + reqCtx.Log.Info("pod is old primary and new role is secondary, do not deal with this situation", + "podName", pod.Name, "newRole", newRole) return nil } - // pod is old secondary and newRole is primary - oldPrimarySts, err := util.GetPodOwnerReferencesSts(reqCtx.Ctx, cli, oldPrimaryPod) - if err != nil { - return err - } // update old primary pod to secondary if err := updateObjRoleLabel(reqCtx.Ctx, cli, *oldPrimaryPod, string(Secondary)); err != nil { return err } - // update old primary sts to secondary - if oldPrimarySts != nil { - if err := updateObjRoleLabel(reqCtx.Ctx, cli, *oldPrimarySts, string(Secondary)); err != nil { - return err - } - } - newPrimarySts, err := util.GetPodOwnerReferencesSts(reqCtx.Ctx, cli, pod) - if err != nil { - return err - } // update secondary pod to primary if err := updateObjRoleLabel(reqCtx.Ctx, cli, *pod, string(Primary)); err != nil { return err } - // update secondary sts to primary - if newPrimarySts != nil { - return updateObjRoleLabel(reqCtx.Ctx, cli, *newPrimarySts, string(Primary)) - } return nil } diff --git a/controllers/apps/components/replicationset/replication_set_utils_test.go b/controllers/apps/components/replicationset/replication_set_utils_test.go index 4312e39f8..c502c03ab 100644 --- a/controllers/apps/components/replicationset/replication_set_utils_test.go +++ b/controllers/apps/components/replicationset/replication_set_utils_test.go @@ -18,10 +18,10 @@ package replicationset import ( "context" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -62,7 +62,7 @@ var _ = Describe("ReplicationSet Util", func() { inNS := client.InNamespace(testCtx.DefaultNamespace) ml := client.HasLabels{testCtx.TestObjLabelKey} // namespaced resources - testapps.ClearResources(&testCtx, generics.StatefulSetSignature, inNS, ml) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.StatefulSetSignature, true, inNS, ml) testapps.ClearResources(&testCtx, generics.PodSignature, inNS, ml, client.GracePeriodSeconds(0)) } @@ -75,7 +75,7 @@ var _ = Describe("ReplicationSet Util", func() { By("Creating a cluster with replication workloadType.") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). SetReplicas(testapps.DefaultReplicationReplicas). SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). Create(&testCtx).GetObject() @@ -86,66 +86,31 @@ var _ = Describe("ReplicationSet Util", func() { Image: testapps.DefaultRedisImageName, ImagePullPolicy: corev1.PullIfNotPresent, } - stsList := make([]*appsv1.StatefulSet, 0) - secondaryName := clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-1" - for k, v := range map[string]string{ - string(Primary): clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-0", - string(Secondary): secondaryName, - } { - sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, v, clusterObj.Name, testapps.DefaultRedisCompName). - AddFinalizers([]string{DBClusterFinalizerName}). - AddContainer(container). - AddAppInstanceLabel(clusterObj.Name). - AddAppComponentLabel(testapps.DefaultRedisCompName). - AddAppManangedByLabel(). - AddRoleLabel(k). - SetReplicas(1). - Create(&testCtx).GetObject() - isStsPrimary, err := checkObjRoleLabelIsPrimary(sts) - if k == string(Primary) { - Expect(err).To(Succeed()) - Expect(isStsPrimary).Should(BeTrue()) - } else { - Expect(err).To(Succeed()) - Expect(isStsPrimary).ShouldNot(BeTrue()) - } - stsList = append(stsList, sts) - } + sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, + clusterObj.Name+"-"+testapps.DefaultRedisCompName, clusterObj.Name, testapps.DefaultRedisCompName). + AddFinalizers([]string{DBClusterFinalizerName}). + AddContainer(container). + AddAppInstanceLabel(clusterObj.Name). + AddAppComponentLabel(testapps.DefaultRedisCompName). + AddAppManangedByLabel(). + SetReplicas(2). + Create(&testCtx).GetObject() By("Creating Pods of replication workloadType.") - for _, sts := range stsList { - _ = testapps.NewPodFactory(testCtx.DefaultNamespace, sts.Name+"-0"). + for i := int32(0); i < *sts.Spec.Replicas; i++ { + _ = testapps.NewPodFactory(testCtx.DefaultNamespace, fmt.Sprintf("%s-%d", sts.Name, i)). AddContainer(container). AddLabelsInMap(sts.Labels). + AddRoleLabel(DefaultRole(i)). Create(&testCtx).GetObject() } - - By("Test ReplicationSet pod number of sts equals 1") - _, err := getAndCheckReplicationPodByStatefulSet(ctx, k8sClient, stsList[0]) - Expect(err).Should(Succeed()) - - By("Test handleReplicationSet success when stsList count equal cluster.replicas.") - err = HandleReplicationSet(ctx, k8sClient, clusterObj, stsList) - Expect(err).Should(Succeed()) - - By("Test handleReplicationSet scale-in return err when remove Finalizer after delete the sts") - clusterObj.Spec.ComponentSpecs[0].Replicas = testapps.DefaultReplicationReplicas - 1 - Expect(HandleReplicationSet(ctx, k8sClient, clusterObj, stsList)).Should(Succeed()) - Eventually(testapps.GetListLen(&testCtx, generics.StatefulSetSignature, - client.InNamespace(testCtx.DefaultNamespace))).Should(Equal(1)) - - By("Test handleReplicationSet scale replicas to 0") - clusterObj.Spec.ComponentSpecs[0].Replicas = 0 - Expect(HandleReplicationSet(ctx, k8sClient, clusterObj, stsList[:1])).Should(Succeed()) - Eventually(testapps.GetListLen(&testCtx, generics.StatefulSetSignature, client.InNamespace(testCtx.DefaultNamespace))).Should(Equal(0)) - Expect(clusterObj.Status.Components[testapps.DefaultRedisCompName].Phase).Should(Equal(appsv1alpha1.StoppedClusterCompPhase)) } testNeedUpdateReplicationSetStatus := func() { By("Creating a cluster with replication workloadType.") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType).Create(&testCtx).GetObject() + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName).Create(&testCtx).GetObject() By("init replicationSet cluster status") patch := client.MergeFrom(clusterObj.DeepCopy()) @@ -155,14 +120,14 @@ var _ = Describe("ReplicationSet Util", func() { Phase: appsv1alpha1.RunningClusterCompPhase, ReplicationSetStatus: &appsv1alpha1.ReplicationSetStatus{ Primary: appsv1alpha1.ReplicationMemberStatus{ - Pod: clusterObj.Name + testapps.DefaultRedisCompName + "-0-0", + Pod: clusterObj.Name + testapps.DefaultRedisCompName + "-0", }, Secondaries: []appsv1alpha1.ReplicationMemberStatus{ { - Pod: clusterObj.Name + testapps.DefaultRedisCompName + "-1-0", + Pod: clusterObj.Name + testapps.DefaultRedisCompName + "-1", }, { - Pod: clusterObj.Name + testapps.DefaultRedisCompName + "-2-0", + Pod: clusterObj.Name + testapps.DefaultRedisCompName + "-2", }, }, }, @@ -171,35 +136,35 @@ var _ = Describe("ReplicationSet Util", func() { Expect(k8sClient.Status().Patch(context.Background(), clusterObj, patch)).Should(Succeed()) By("testing sync cluster status with add pod") - var podList []*corev1.Pod - sts := testk8s.NewFakeStatefulSet(clusterObj.Name+testapps.DefaultRedisCompName+"-3", 3) - pod := testapps.NewPodFactory(testCtx.DefaultNamespace, sts.Name+"-0"). - AddContainer(corev1.Container{Name: testapps.DefaultRedisContainerName, Image: testapps.DefaultRedisImageName}). - AddRoleLabel(string(Secondary)). - Create(&testCtx).GetObject() - podList = append(podList, pod) + + var podList []corev1.Pod + sts := testk8s.NewFakeStatefulSet(clusterObj.Name+testapps.DefaultRedisCompName, 4) + + for i := int32(0); i < *sts.Spec.Replicas; i++ { + pod := testapps.NewPodFactory(testCtx.DefaultNamespace, fmt.Sprintf("%s-%d", sts.Name, i)). + AddContainer(corev1.Container{Name: testapps.DefaultRedisContainerName, Image: testapps.DefaultRedisImageName}). + AddRoleLabel(DefaultRole(i)). + Create(&testCtx).GetObject() + podList = append(podList, *pod) + } err := syncReplicationSetStatus(clusterObj.Status.Components[testapps.DefaultRedisCompName].ReplicationSetStatus, podList) Expect(err).Should(Succeed()) Expect(len(clusterObj.Status.Components[testapps.DefaultRedisCompName].ReplicationSetStatus.Secondaries)).Should(Equal(3)) By("testing sync cluster status with remove pod") var podRemoveList []corev1.Pod - sts = testk8s.NewFakeStatefulSet(clusterObj.Name+testapps.DefaultRedisCompName+"-2", 3) - pod = testapps.NewPodFactory(testCtx.DefaultNamespace, sts.Name+"-0"). - AddContainer(corev1.Container{Name: testapps.DefaultRedisContainerName, Image: testapps.DefaultRedisImageName}). - AddRoleLabel(string(Secondary)). - Create(&testCtx).GetObject() - podRemoveList = append(podRemoveList, *pod) + *sts.Spec.Replicas -= 1 + podRemoveList = append(podRemoveList, podList[len(podList)-1]) Expect(removeTargetPodsInfoInStatus(clusterObj.Status.Components[testapps.DefaultRedisCompName].ReplicationSetStatus, podRemoveList, clusterObj.Spec.ComponentSpecs[0].Replicas)).Should(Succeed()) - Expect(len(clusterObj.Status.Components[testapps.DefaultRedisCompName].ReplicationSetStatus.Secondaries)).Should(Equal(2)) + Expect(clusterObj.Status.Components[testapps.DefaultRedisCompName].ReplicationSetStatus.Secondaries).Should(HaveLen(2)) } testGeneratePVCFromVolumeClaimTemplates := func() { By("Creating a cluster with replication workloadType.") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). SetReplicas(testapps.DefaultReplicationReplicas). SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). Create(&testCtx).GetObject() @@ -234,7 +199,7 @@ var _ = Describe("ReplicationSet Util", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). SetReplicas(testapps.DefaultReplicationReplicas). SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). SetSwitchPolicy(clusterSwitchPolicy). @@ -246,57 +211,46 @@ var _ = Describe("ReplicationSet Util", func() { Image: testapps.DefaultRedisImageName, ImagePullPolicy: corev1.PullIfNotPresent, } - stsList := make([]*appsv1.StatefulSet, 0) - secondaryName := clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-1" - for k, v := range map[string]string{ - string(Primary): clusterObj.Name + "-" + testapps.DefaultRedisCompName + "-0", - string(Secondary): secondaryName, - } { - sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, v, clusterObj.Name, testapps.DefaultRedisCompName). - AddContainer(container). - AddAppInstanceLabel(clusterObj.Name). - AddAppComponentLabel(testapps.DefaultRedisCompName). - AddAppManangedByLabel(). - AddRoleLabel(k). - SetReplicas(1). - Create(&testCtx).GetObject() - isStsPrimary, err := checkObjRoleLabelIsPrimary(sts) - if k == string(Primary) { - Expect(err).To(Succeed()) - Expect(isStsPrimary).Should(BeTrue()) - } else { - Expect(err).To(Succeed()) - Expect(isStsPrimary).ShouldNot(BeTrue()) - } - stsList = append(stsList, sts) - } + sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, + clusterObj.Name+"-"+testapps.DefaultRedisCompName, clusterObj.Name, testapps.DefaultRedisCompName). + AddContainer(container). + AddAppInstanceLabel(clusterObj.Name). + AddAppComponentLabel(testapps.DefaultRedisCompName). + AddAppManangedByLabel(). + SetReplicas(2). + Create(&testCtx).GetObject() By("Creating Pods of replication workloadType.") var ( - primaryPod *corev1.Pod - secondaryPod *corev1.Pod + primaryPod *corev1.Pod + secondaryPods []*corev1.Pod ) - for _, sts := range stsList { - pod := testapps.NewPodFactory(testCtx.DefaultNamespace, sts.Name+"-0"). + for i := int32(0); i < *sts.Spec.Replicas; i++ { + pod := testapps.NewPodFactory(testCtx.DefaultNamespace, fmt.Sprintf("%s-%d", sts.Name, i)). AddContainer(container). AddLabelsInMap(sts.Labels). + AddRoleLabel(DefaultRole(i)). Create(&testCtx).GetObject() - if sts.Labels[constant.RoleLabelKey] == string(Primary) { + if pod.Labels[constant.RoleLabelKey] == string(Primary) { primaryPod = pod } else { - secondaryPod = pod + secondaryPods = append(secondaryPods, pod) } } + Expect(primaryPod).ShouldNot(BeNil()) + Expect(secondaryPods).ShouldNot(BeEmpty()) + By("Test update replicationSet pod role label with event driver, secondary change to primary.") reqCtx := intctrlutil.RequestCtx{ Ctx: testCtx.Ctx, Log: log.FromContext(ctx).WithValues("event", testCtx.DefaultNamespace), } - err := HandleReplicationSetRoleChangeEvent(k8sClient, reqCtx, clusterObj, testapps.DefaultRedisCompName, secondaryPod, string(Primary)) - Expect(err).Should(Succeed()) + Expect(HandleReplicationSetRoleChangeEvent(k8sClient, reqCtx, clusterObj, testapps.DefaultRedisCompName, + secondaryPods[0], string(Primary))).ShouldNot(HaveOccurred()) + By("Test when secondary change to primary, the old primary label has been updated at the same time, so return nil directly.") - err = HandleReplicationSetRoleChangeEvent(k8sClient, reqCtx, clusterObj, testapps.DefaultRedisCompName, primaryPod, string(Secondary)) - Expect(err).Should(BeNil()) + Expect(HandleReplicationSetRoleChangeEvent(k8sClient, reqCtx, clusterObj, testapps.DefaultRedisCompName, + primaryPod, string(Secondary))).ShouldNot(HaveOccurred()) } // Scenarios @@ -305,12 +259,12 @@ var _ = Describe("ReplicationSet Util", func() { BeforeEach(func() { By("Create a clusterDefinition obj with replication workloadType.") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompType). + AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(testapps.DefaultRedisCompType).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/components/stateful/stateful.go b/controllers/apps/components/stateful/stateful.go index 9647ab4e6..ff4cc2a21 100644 --- a/controllers/apps/components/stateful/stateful.go +++ b/controllers/apps/components/stateful/stateful.go @@ -33,36 +33,31 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -type Stateful struct { - Cli client.Client - Cluster *appsv1alpha1.Cluster - Component *appsv1alpha1.ClusterComponentSpec - componentDef *appsv1alpha1.ClusterComponentDefinition -} +type Stateful types.ComponentBase var _ types.Component = &Stateful{} -func (stateful *Stateful) IsRunning(ctx context.Context, obj client.Object) (bool, error) { +func (r *Stateful) IsRunning(ctx context.Context, obj client.Object) (bool, error) { if obj == nil { return false, nil } sts := util.ConvertToStatefulSet(obj) - isRevisionConsistent, err := util.IsStsAndPodsRevisionConsistent(ctx, stateful.Cli, sts) + isRevisionConsistent, err := util.IsStsAndPodsRevisionConsistent(ctx, r.Cli, sts) if err != nil { return false, err } - return util.StatefulSetOfComponentIsReady(sts, isRevisionConsistent, &stateful.Component.Replicas), nil + return util.StatefulSetOfComponentIsReady(sts, isRevisionConsistent, &r.Component.Replicas), nil } -func (stateful *Stateful) PodsReady(ctx context.Context, obj client.Object) (bool, error) { +func (r *Stateful) PodsReady(ctx context.Context, obj client.Object) (bool, error) { if obj == nil { return false, nil } sts := util.ConvertToStatefulSet(obj) - return util.StatefulSetPodsAreReady(sts, stateful.Component.Replicas), nil + return util.StatefulSetPodsAreReady(sts, r.Component.Replicas), nil } -func (stateful *Stateful) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { +func (r *Stateful) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { if pod == nil { return false } @@ -70,14 +65,14 @@ func (stateful *Stateful) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) } // HandleProbeTimeoutWhenPodsReady the Stateful component has no role detection, empty implementation here. -func (stateful *Stateful) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { +func (r *Stateful) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { return false, nil } // GetPhaseWhenPodsNotReady gets the component phase when the pods of component are not ready. -func (stateful *Stateful) GetPhaseWhenPodsNotReady(ctx context.Context, componentName string) (appsv1alpha1.ClusterComponentPhase, error) { +func (r *Stateful) GetPhaseWhenPodsNotReady(ctx context.Context, componentName string) (appsv1alpha1.ClusterComponentPhase, error) { stsList := &appsv1.StatefulSetList{} - podList, err := util.GetCompRelatedObjectList(ctx, stateful.Cli, *stateful.Cluster, componentName, stsList) + podList, err := util.GetCompRelatedObjectList(ctx, r.Cli, *r.Cluster, componentName, stsList) if err != nil || len(stsList.Items) == 0 { return "", err } @@ -87,20 +82,20 @@ func (stateful *Stateful) GetPhaseWhenPodsNotReady(ctx context.Context, componen return !intctrlutil.PodIsReady(pod) && intctrlutil.PodIsControlledByLatestRevision(pod, sts) } stsObj := stsList.Items[0] - return util.GetComponentPhaseWhenPodsNotReady(podList, &stsObj, stateful.Component.Replicas, + return util.GetComponentPhaseWhenPodsNotReady(podList, &stsObj, r.Component.Replicas, stsObj.Status.AvailableReplicas, checkExistFailedPodOfLatestRevision), nil } -func (stateful *Stateful) HandleUpdate(ctx context.Context, obj client.Object) error { +func (r *Stateful) HandleUpdate(ctx context.Context, obj client.Object) error { return nil } -func NewStateful( +func NewStatefulComponent( cli client.Client, cluster *appsv1alpha1.Cluster, component *appsv1alpha1.ClusterComponentSpec, componentDef appsv1alpha1.ClusterComponentDefinition, -) (*Stateful, error) { +) (types.Component, error) { if err := util.ComponentRuntimeReqArgsCheck(cli, cluster, component); err != nil { return nil, err } @@ -108,6 +103,6 @@ func NewStateful( Cli: cli, Cluster: cluster, Component: component, - componentDef: &componentDef, + ComponentDef: &componentDef, }, nil } diff --git a/controllers/apps/components/stateful/stateful_test.go b/controllers/apps/components/stateful/stateful_test.go index 5abd6114a..784d0dafa 100644 --- a/controllers/apps/components/stateful/stateful_test.go +++ b/controllers/apps/components/stateful/stateful_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -86,20 +87,20 @@ var _ = Describe("Stateful Component", func() { sts := &stsList.Items[0] clusterComponent := cluster.Spec.GetComponentByName(statefulCompName) componentDef := clusterDef.GetComponentDefByName(clusterComponent.ComponentDefRef) - stateful, err := NewStateful(k8sClient, cluster, clusterComponent, *componentDef) + stateful, err := NewStatefulComponent(k8sClient, cluster, clusterComponent, *componentDef) Expect(err).Should(Succeed()) phase, _ := stateful.GetPhaseWhenPodsNotReady(ctx, statefulCompName) Expect(phase == appsv1alpha1.FailedClusterCompPhase).Should(BeTrue()) By("test pods are not ready") - updateRevison := fmt.Sprintf("%s-%s-%s", clusterName, statefulCompName, "6fdd48d9cd") + updateRevision := fmt.Sprintf("%s-%s-%s", clusterName, statefulCompName, "6fdd48d9cd") Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { availableReplicas := *sts.Spec.Replicas - 1 sts.Status.AvailableReplicas = availableReplicas sts.Status.ReadyReplicas = availableReplicas sts.Status.Replicas = availableReplicas sts.Status.ObservedGeneration = 1 - sts.Status.UpdateRevision = updateRevison + sts.Status.UpdateRevision = updateRevision })).Should(Succeed()) podsReady, _ := stateful.PodsReady(ctx, sts) Expect(podsReady == false).Should(BeTrue()) @@ -121,14 +122,14 @@ var _ = Describe("Stateful Component", func() { By("not ready pod is not controlled by latest revision, should return empty string") // mock pod is not controlled by latest revision - Expect(testapps.ChangeObj(&testCtx, pod, func() { - pod.Labels[appsv1.ControllerRevisionHashLabelKey] = fmt.Sprintf("%s-%s-%s", clusterName, statefulCompName, "5wdsd8d9fs") + Expect(testapps.ChangeObj(&testCtx, pod, func(lpod *corev1.Pod) { + lpod.Labels[appsv1.ControllerRevisionHashLabelKey] = fmt.Sprintf("%s-%s-%s", clusterName, statefulCompName, "5wdsd8d9fs") })).Should(Succeed()) phase, _ = stateful.GetPhaseWhenPodsNotReady(ctx, statefulCompName) Expect(len(phase) == 0).Should(BeTrue()) // reset updateRevision - Expect(testapps.ChangeObj(&testCtx, pod, func() { - pod.Labels[appsv1.ControllerRevisionHashLabelKey] = updateRevison + Expect(testapps.ChangeObj(&testCtx, pod, func(lpod *corev1.Pod) { + lpod.Labels[appsv1.ControllerRevisionHashLabelKey] = updateRevision })).Should(Succeed()) By("test pod is available") diff --git a/controllers/apps/components/stateful_set_controller_test.go b/controllers/apps/components/stateful_set_controller_test.go index e03ae6343..530723000 100644 --- a/controllers/apps/components/stateful_set_controller_test.go +++ b/controllers/apps/components/stateful_set_controller_test.go @@ -101,16 +101,16 @@ var _ = Describe("StatefulSet Controller", func() { By("Mock a pod without role label and it will wait for HandleProbeTimeoutWhenPodsReady") leaderPod := pods[0] - Expect(testapps.ChangeObj(&testCtx, leaderPod, func() { - delete(leaderPod.Labels, constant.RoleLabelKey) + Expect(testapps.ChangeObj(&testCtx, leaderPod, func(lpod *corev1.Pod) { + delete(lpod.Labels, constant.RoleLabelKey) })).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(leaderPod), func(g Gomega, pod *corev1.Pod) { g.Expect(pod.Labels[constant.RoleLabelKey] == "").Should(BeTrue()) })).Should(Succeed()) By("mock restart component to trigger reconcile of StatefulSet controller") - Expect(testapps.ChangeObj(&testCtx, sts, func() { - sts.Spec.Template.Annotations = map[string]string{ + Expect(testapps.ChangeObj(&testCtx, sts, func(lsts *appsv1.StatefulSet) { + lsts.Spec.Template.Annotations = map[string]string{ constant.RestartAnnotationKey: time.Now().Format(time.RFC3339), } })).Should(Succeed()) @@ -131,8 +131,8 @@ var _ = Describe("StatefulSet Controller", func() { })).Should(Succeed()) By("add leader role label for leaderPod to mock consensus component to be Running") - Expect(testapps.ChangeObj(&testCtx, leaderPod, func() { - leaderPod.Labels[constant.RoleLabelKey] = "leader" + Expect(testapps.ChangeObj(&testCtx, leaderPod, func(lpod *corev1.Pod) { + lpod.Labels[constant.RoleLabelKey] = "leader" })).Should(Succeed()) return pods } @@ -155,8 +155,8 @@ var _ = Describe("StatefulSet Controller", func() { } })).Should(Succeed()) _ = testapps.CreateRestartOpsRequest(testCtx, clusterName, opsRequestName, []string{consensusCompName}) - Expect(testapps.ChangeObj(&testCtx, cluster, func() { - cluster.Annotations = map[string]string{ + Expect(testapps.ChangeObj(&testCtx, cluster, func(lcluster *appsv1alpha1.Cluster) { + lcluster.Annotations = map[string]string{ constant.OpsRequestAnnotationKey: fmt.Sprintf(`[{"name":"%s","clusterPhase":"Updating"}]`, opsRequestName), } })).Should(Succeed()) @@ -180,12 +180,12 @@ var _ = Describe("StatefulSet Controller", func() { })()).Should(Succeed()) By("mock stop operation and processed successfully") - Expect(testapps.ChangeObj(&testCtx, cluster, func() { - cluster.Spec.ComponentSpecs[0].Replicas = 0 + Expect(testapps.ChangeObj(&testCtx, cluster, func(lcluster *appsv1alpha1.Cluster) { + lcluster.Spec.ComponentSpecs[0].Replicas = 0 })).Should(Succeed()) - Expect(testapps.ChangeObj(&testCtx, sts, func() { + Expect(testapps.ChangeObj(&testCtx, sts, func(lsts *appsv1.StatefulSet) { replicas := int32(0) - sts.Spec.Replicas = &replicas + lsts.Spec.Replicas = &replicas })).Should(Succeed()) Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { testk8s.MockStatefulSetReady(sts) diff --git a/controllers/apps/components/stateless/stateless.go b/controllers/apps/components/stateless/stateless.go index 2ce6decf6..43f668060 100644 --- a/controllers/apps/components/stateless/stateless.go +++ b/controllers/apps/components/stateless/stateless.go @@ -42,12 +42,7 @@ import ( // is at least the minimum available pods that need to run for the deployment. const NewRSAvailableReason = "NewReplicaSetAvailable" -type Stateless struct { - Cli client.Client - Cluster *appsv1alpha1.Cluster - Component *appsv1alpha1.ClusterComponentSpec - componentDef *appsv1alpha1.ClusterComponentDefinition -} +type Stateless types.ComponentBase var _ types.Component = &Stateless{} @@ -104,11 +99,11 @@ func (stateless *Stateless) HandleUpdate(ctx context.Context, obj client.Object) return nil } -func NewStateless( +func NewStatelessComponent( cli client.Client, cluster *appsv1alpha1.Cluster, component *appsv1alpha1.ClusterComponentSpec, - componentDef appsv1alpha1.ClusterComponentDefinition) (*Stateless, error) { + componentDef appsv1alpha1.ClusterComponentDefinition) (types.Component, error) { if err := util.ComponentRuntimeReqArgsCheck(cli, cluster, component); err != nil { return nil, err } @@ -116,7 +111,7 @@ func NewStateless( Cli: cli, Cluster: cluster, Component: component, - componentDef: &componentDef, + ComponentDef: &componentDef, }, nil } diff --git a/controllers/apps/components/stateless/stateless_test.go b/controllers/apps/components/stateless/stateless_test.go index b7b5c2428..edb548068 100644 --- a/controllers/apps/components/stateless/stateless_test.go +++ b/controllers/apps/components/stateless/stateless_test.go @@ -43,7 +43,7 @@ var _ = Describe("Stateful Component", func() { ) const ( statelessCompName = "stateless" - statelessCompDefRef = "stateless" + statelessCompDefName = "stateless" defaultMinReadySeconds = 10 ) @@ -71,14 +71,14 @@ var _ = Describe("Stateful Component", func() { It("Stateless Component test", func() { By(" init cluster, deployment") clusterDef := testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatelessNginxComponent, statelessCompDefRef). + AddComponentDef(testapps.StatelessNginxComponent, statelessCompDefName). Create(&testCtx).GetObject() cluster := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(statelessCompName, statelessCompDefRef).SetReplicas(2).Create(&testCtx).GetObject() + AddComponent(statelessCompName, statelessCompDefName).SetReplicas(2).Create(&testCtx).GetObject() deploy := testapps.MockStatelessComponentDeploy(testCtx, clusterName, statelessCompName) clusterComponent := cluster.Spec.GetComponentByName(statelessCompName) componentDef := clusterDef.GetComponentDefByName(clusterComponent.ComponentDefRef) - statelessComponent, err := NewStateless(k8sClient, cluster, clusterComponent, *componentDef) + statelessComponent, err := NewStatelessComponent(k8sClient, cluster, clusterComponent, *componentDef) Expect(err).Should(Succeed()) By("test pods number of deploy is 0 ") phase, _ := statelessComponent.GetPhaseWhenPodsNotReady(ctx, statelessCompName) diff --git a/controllers/apps/components/types/component.go b/controllers/apps/components/types/component.go index cfb0e78b8..073a55376 100644 --- a/controllers/apps/components/types/component.go +++ b/controllers/apps/components/types/component.go @@ -58,6 +58,14 @@ type Component interface { HandleUpdate(ctx context.Context, obj client.Object) error } +// ComponentBase is a common component base struct +type ComponentBase struct { + Cli client.Client + Cluster *appsv1alpha1.Cluster + Component *appsv1alpha1.ClusterComponentSpec + ComponentDef *appsv1alpha1.ClusterComponentDefinition +} + const ( // RoleProbeTimeoutReason the event reason when all pods of the component role probe timed out. RoleProbeTimeoutReason = "RoleProbeTimeout" diff --git a/controllers/apps/components/util/component_utils.go b/controllers/apps/components/util/component_utils.go index bbcb3da1f..1632f0173 100644 --- a/controllers/apps/components/util/component_utils.go +++ b/controllers/apps/components/util/component_utils.go @@ -136,20 +136,23 @@ func GetComponentPhase(isFailed, isAbnormal bool) appsv1alpha1.ClusterComponentP } // GetObjectListByComponentName gets k8s workload list with component -func GetObjectListByComponentName(ctx context.Context, cli client2.ReadonlyClient, cluster appsv1alpha1.Cluster, objectList client.ObjectList, componentName string) error { +func GetObjectListByComponentName(ctx context.Context, cli client2.ReadonlyClient, cluster appsv1alpha1.Cluster, + objectList client.ObjectList, componentName string) error { matchLabels := GetComponentMatchLabels(cluster.Name, componentName) inNamespace := client.InNamespace(cluster.Namespace) return cli.List(ctx, objectList, client.MatchingLabels(matchLabels), inNamespace) } // GetObjectListByCustomLabels gets k8s workload list with custom labels -func GetObjectListByCustomLabels(ctx context.Context, cli client.Client, cluster appsv1alpha1.Cluster, objectList client.ObjectList, matchLabels client.ListOption) error { +func GetObjectListByCustomLabels(ctx context.Context, cli client.Client, cluster appsv1alpha1.Cluster, + objectList client.ObjectList, matchLabels client.ListOption) error { inNamespace := client.InNamespace(cluster.Namespace) return cli.List(ctx, objectList, matchLabels, inNamespace) } // GetComponentDefByCluster gets component from ClusterDefinition with compDefName -func GetComponentDefByCluster(ctx context.Context, cli client2.ReadonlyClient, cluster appsv1alpha1.Cluster, compDefName string) (*appsv1alpha1.ClusterComponentDefinition, error) { +func GetComponentDefByCluster(ctx context.Context, cli client2.ReadonlyClient, cluster appsv1alpha1.Cluster, + compDefName string) (*appsv1alpha1.ClusterComponentDefinition, error) { clusterDef := &appsv1alpha1.ClusterDefinition{} if err := cli.Get(ctx, client.ObjectKey{Name: cluster.Spec.ClusterDefRef}, clusterDef); err != nil { return nil, err diff --git a/controllers/apps/components/util/stateful_set_utils.go b/controllers/apps/components/util/stateful_set_utils.go index 0e0bc15fa..06d7fe77d 100644 --- a/controllers/apps/components/util/stateful_set_utils.go +++ b/controllers/apps/components/util/stateful_set_utils.go @@ -123,40 +123,19 @@ func ConvertToStatefulSet(obj client.Object) *appsv1.StatefulSet { return nil } -// Len is the implementation of the sort.Interface, calculate the length of the list of DescendingOrdinalSts. -func (dos DescendingOrdinalSts) Len() int { - return len(dos) -} - -// Swap is the implementation of the sort.Interface, exchange two items in DescendingOrdinalSts. -func (dos DescendingOrdinalSts) Swap(i, j int) { - dos[i], dos[j] = dos[j], dos[i] -} - -// Less is the implementation of the sort.Interface, sort the size of the statefulSet ordinal in descending order. -func (dos DescendingOrdinalSts) Less(i, j int) bool { - return GetOrdinalSts(dos[i]) > GetOrdinalSts(dos[j]) -} - -// GetOrdinalSts gets StatefulSet's ordinal. If StatefulSet has no ordinal, -1 is returned. -func GetOrdinalSts(sts *appsv1.StatefulSet) int { - _, ordinal := getParentNameAndOrdinalSts(sts) - return ordinal -} - -// getParentNameAndOrdinalSts gets the name of cluster-component and StatefulSet's ordinal as extracted from its Name. If +// ParseParentNameAndOrdinal gets the name of cluster-component and StatefulSet's ordinal as extracted from its Name. If // the StatefulSet's Name was not match a statefulSetRegex, its parent is considered to be empty string, // and its ordinal is considered to be -1. -func getParentNameAndOrdinalSts(sts *appsv1.StatefulSet) (string, int) { +func ParseParentNameAndOrdinal(s string) (string, int32) { parent := "" - ordinal := -1 - subMatches := statefulSetRegex.FindStringSubmatch(sts.Name) + ordinal := int32(-1) + subMatches := statefulSetRegex.FindStringSubmatch(s) if len(subMatches) < 3 { return parent, ordinal } parent = subMatches[1] if i, err := strconv.ParseInt(subMatches[2], 10, 32); err == nil { - ordinal = int(i) + ordinal = int32(i) } return parent, ordinal } diff --git a/controllers/apps/components/util/stateful_set_utils_test.go b/controllers/apps/components/util/stateful_set_utils_test.go index 1d5844cbc..50963e260 100644 --- a/controllers/apps/components/util/stateful_set_utils_test.go +++ b/controllers/apps/components/util/stateful_set_utils_test.go @@ -130,7 +130,7 @@ var _ = Describe("StatefulSet utils test", func() { Create(&testCtx).GetObject() By("Creating pods by the StatefulSet") - testapps.MockReplicationComponentPods(testCtx, sts, clusterName, testapps.DefaultRedisCompName, role) + testapps.MockReplicationComponentPods(nil, testCtx, sts, clusterName, testapps.DefaultRedisCompName, nil) Expect(IsStsAndPodsRevisionConsistent(testCtx.Ctx, k8sClient, sts)).Should(BeTrue()) By("Updating the StatefulSet's UpdateRevision") @@ -152,7 +152,7 @@ var _ = Describe("StatefulSet utils test", func() { Expect(len(podList)).To(Equal(0)) By("Creating new pods by StatefulSet with new UpdateRevision") - testapps.MockReplicationComponentPods(testCtx, sts, clusterName, testapps.DefaultRedisCompName, role) + testapps.MockReplicationComponentPods(nil, testCtx, sts, clusterName, testapps.DefaultRedisCompName, nil) Expect(IsStsAndPodsRevisionConsistent(testCtx.Ctx, k8sClient, sts)).Should(BeTrue()) }) }) diff --git a/controllers/apps/configuration/config_util_test.go b/controllers/apps/configuration/config_util_test.go index 33868a45f..72e38a06e 100644 --- a/controllers/apps/configuration/config_util_test.go +++ b/controllers/apps/configuration/config_util_test.go @@ -38,11 +38,8 @@ import ( var _ = Describe("ConfigWrapper util test", func() { const clusterDefName = "test-clusterdef" const clusterVersionName = "test-clusterversion" - - const statefulCompType = "replicasets" - + const statefulCompDefName = "replicasets" const configSpecName = "mysql-config-tpl" - const configVolumeName = "mysql-config" var ( @@ -98,13 +95,13 @@ var _ = Describe("ConfigWrapper util test", func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, statefulCompType). + AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). AddConfigTemplate(configSpecName, configMapObj.Name, configConstraintObj.Name, testCtx.DefaultNamespace, configVolumeName). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompType). + AddComponent(statefulCompDefName). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/configuration/configconstraint_controller_test.go b/controllers/apps/configuration/configconstraint_controller_test.go index 0e087b79f..5212fef6e 100644 --- a/controllers/apps/configuration/configconstraint_controller_test.go +++ b/controllers/apps/configuration/configconstraint_controller_test.go @@ -36,11 +36,8 @@ import ( var _ = Describe("ConfigConstraint Controller", func() { const clusterDefName = "test-clusterdef" const clusterVersionName = "test-clusterversion" - - const statefulCompType = "replicasets" - + const statefulCompDefName = "replicasets" const configSpecName = "mysql-config-tpl" - const configVolumeName = "mysql-config" cleanEnv := func() { @@ -81,7 +78,7 @@ var _ = Describe("ConfigConstraint Controller", func() { By("Create a clusterDefinition obj") clusterDefObj := testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, statefulCompType). + AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). AddConfigTemplate(configSpecName, configmap.Name, constraint.Name, testCtx.DefaultNamespace, configVolumeName). AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(configSpecName), configmap.Name, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). @@ -89,7 +86,7 @@ var _ = Describe("ConfigConstraint Controller", func() { By("Create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompType). + AddComponent(statefulCompDefName). AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(configSpecName), configmap.Name, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). Create(&testCtx).GetObject() diff --git a/controllers/apps/configuration/reconfigurerequest_controller_test.go b/controllers/apps/configuration/reconfigurerequest_controller_test.go index e48bb3f8d..7fd984aa4 100644 --- a/controllers/apps/configuration/reconfigurerequest_controller_test.go +++ b/controllers/apps/configuration/reconfigurerequest_controller_test.go @@ -36,16 +36,11 @@ var _ = Describe("Reconfigure Controller", func() { const clusterDefName = "test-clusterdef" const clusterVersionName = "test-clusterversion" const clusterName = "test-cluster" - - const statefulCompType = "replicasets" + const statefulCompDefName = "replicasets" const statefulCompName = "mysql" - const statefulSetName = "mysql-statefulset" - const configSpecName = "mysql-config-tpl" - const configVolumeName = "mysql-config" - const cmName = "mysql-tree-node-template-8.0" var ctx = context.Background() @@ -96,7 +91,7 @@ var _ = Describe("Reconfigure Controller", func() { By("Create a clusterDefinition obj") clusterDefObj := testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, statefulCompType). + AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). AddConfigTemplate(configSpecName, configmap.Name, constraint.Name, testCtx.DefaultNamespace, configVolumeName). AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(configSpecName), configmap.Name, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). @@ -104,7 +99,7 @@ var _ = Describe("Reconfigure Controller", func() { By("Create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompType). + AddComponent(statefulCompDefName). AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(configSpecName), configmap.Name, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). Create(&testCtx).GetObject() @@ -112,7 +107,7 @@ var _ = Describe("Reconfigure Controller", func() { By("Creating a cluster") clusterObj := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(statefulCompName, statefulCompType).Create(&testCtx).GetObject() + AddComponent(statefulCompName, statefulCompDefName).Create(&testCtx).GetObject() container := corev1.Container{ Name: "mock-container", diff --git a/controllers/apps/operations/horizontal_scaling.go b/controllers/apps/operations/horizontal_scaling.go index 7720c3778..a9473dc5f 100644 --- a/controllers/apps/operations/horizontal_scaling.go +++ b/controllers/apps/operations/horizontal_scaling.go @@ -79,7 +79,7 @@ func (hs horizontalScalingOpsHandler) ReconcileAction(reqCtx intctrlutil.Request compStatus *appsv1alpha1.OpsRequestComponentStatus) (int32, int32, error) { return handleComponentProgressForScalingReplicas(reqCtx, cli, opsRes, pgRes, compStatus, hs.getExpectReplicas) } - return ReconcileActionWithComponentOps(reqCtx, cli, opsRes, "", handleComponentProgress) + return reconcileActionWithComponentOps(reqCtx, cli, opsRes, "", handleComponentProgress) } // GetRealAffectedComponentMap gets the real affected component map for the operation diff --git a/controllers/apps/operations/horizontal_scaling_test.go b/controllers/apps/operations/horizontal_scaling_test.go index 11ec55172..01d8827c9 100644 --- a/controllers/apps/operations/horizontal_scaling_test.go +++ b/controllers/apps/operations/horizontal_scaling_test.go @@ -93,10 +93,10 @@ var _ = Describe("HorizontalScaling OpsRequest", func() { Expect(err).ShouldNot(HaveOccurred()) By("test GetOpsRequestAnnotation function") - Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func() { + Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func(lcluster *appsv1alpha1.Cluster) { opsAnnotationString := fmt.Sprintf(`[{"name":"%s","clusterPhase":"Updating"},{"name":"test-not-exists-ops","clusterPhase":"Updating"}]`, opsRes.OpsRequest.Name) - opsRes.Cluster.Annotations = map[string]string{ + lcluster.Annotations = map[string]string{ constant.OpsRequestAnnotationKey: opsAnnotationString, } })).ShouldNot(HaveOccurred()) @@ -104,15 +104,15 @@ var _ = Describe("HorizontalScaling OpsRequest", func() { Expect(err.Error()).Should(ContainSubstring("existing OpsRequest:")) // reset cluster annotation - Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func() { - opsRes.Cluster.Annotations = map[string]string{} + Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func(lcluster *appsv1alpha1.Cluster) { + lcluster.Annotations = map[string]string{} })).ShouldNot(HaveOccurred()) By("Test HorizontalScaling with scale up replicax") initClusterForOps(opsRes) expectClusterComponentReplicas := int32(2) - Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func() { - opsRes.Cluster.Spec.ComponentSpecs[1].Replicas = expectClusterComponentReplicas + Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func(lcluster *appsv1alpha1.Cluster) { + lcluster.Spec.ComponentSpecs[1].Replicas = expectClusterComponentReplicas })).ShouldNot(HaveOccurred()) // mock pod created according to horizontalScaling replicas diff --git a/controllers/apps/operations/ops_manager.go b/controllers/apps/operations/ops_manager.go index 9142b85f7..697d10945 100644 --- a/controllers/apps/operations/ops_manager.go +++ b/controllers/apps/operations/ops_manager.go @@ -56,7 +56,7 @@ func (opsMgr *OpsManager) Do(reqCtx intctrlutil.RequestCtx, cli client.Client, o // validate OpsRequest.spec if err = opsRequest.Validate(reqCtx.Ctx, cli, opsRes.Cluster, true); err != nil { - if patchErr := PatchValidateErrorCondition(reqCtx.Ctx, cli, opsRes, err.Error()); patchErr != nil { + if patchErr := patchValidateErrorCondition(reqCtx.Ctx, cli, opsRes, err.Error()); patchErr != nil { return nil, patchErr } return nil, err @@ -101,7 +101,8 @@ func (opsMgr *OpsManager) Reconcile(reqCtx intctrlutil.RequestCtx, cli client.Cl return 0, patchOpsHandlerNotSupported(reqCtx.Ctx, cli, opsRes) } opsRes.ToClusterPhase = opsBehaviour.ToClusterPhase - if opsRequestPhase, requeueAfter, err = opsBehaviour.OpsHandler.ReconcileAction(reqCtx, cli, opsRes); err != nil && !isOpsRequestFailedPhase(opsRequestPhase) { + if opsRequestPhase, requeueAfter, err = opsBehaviour.OpsHandler.ReconcileAction(reqCtx, cli, opsRes); err != nil && + !isOpsRequestFailedPhase(opsRequestPhase) { // if the opsRequest phase is Failed, skipped return requeueAfter, err } diff --git a/controllers/apps/operations/ops_progress_util.go b/controllers/apps/operations/ops_progress_util.go index 70b8c83fa..ac2390ce2 100644 --- a/controllers/apps/operations/ops_progress_util.go +++ b/controllers/apps/operations/ops_progress_util.go @@ -35,8 +35,8 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -// GetProgressObjectKey gets progress object key from the object of client.Object. -func GetProgressObjectKey(kind, name string) string { +// getProgressObjectKey gets progress object key from the object of client.Object. +func getProgressObjectKey(kind, name string) string { return fmt.Sprintf("%s/%s", kind, name) } @@ -46,11 +46,11 @@ func isCompletedProgressStatus(status appsv1alpha1.ProgressStatus) bool { appsv1alpha1.FailedProgressStatus}, status) } -// SetComponentStatusProgressDetail sets the corresponding progressDetail in progressDetails to newProgressDetail. +// setComponentStatusProgressDetail sets the corresponding progressDetail in progressDetails to newProgressDetail. // progressDetails must be non-nil. // 1. the startTime and endTime will be filled automatically. // 2. if the progressDetail of the specified objectKey does not exist, it will be appended to the progressDetails. -func SetComponentStatusProgressDetail( +func setComponentStatusProgressDetail( recorder record.EventRecorder, opsRequest *appsv1alpha1.OpsRequest, progressDetails *[]appsv1alpha1.ProgressStatusDetail, @@ -58,7 +58,7 @@ func SetComponentStatusProgressDetail( if progressDetails == nil { return } - existingProgressDetail := FindStatusProgressDetail(*progressDetails, newProgressDetail.ObjectKey) + existingProgressDetail := findStatusProgressDetail(*progressDetails, newProgressDetail.ObjectKey) if existingProgressDetail == nil { updateProgressDetailTime(&newProgressDetail) *progressDetails = append(*progressDetails, newProgressDetail) @@ -79,8 +79,8 @@ func SetComponentStatusProgressDetail( sendProgressDetailEvent(recorder, opsRequest, newProgressDetail) } -// FindStatusProgressDetail finds the progressDetail of the specified objectKey in progressDetails. -func FindStatusProgressDetail(progressDetails []appsv1alpha1.ProgressStatusDetail, +// findStatusProgressDetail finds the progressDetail of the specified objectKey in progressDetails. +func findStatusProgressDetail(progressDetails []appsv1alpha1.ProgressStatusDetail, objectKey string) *appsv1alpha1.ProgressStatusDetail { for i := range progressDetails { if progressDetails[i].ObjectKey == objectKey { @@ -139,7 +139,7 @@ func updateProgressDetailTime(progressDetail *appsv1alpha1.ProgressStatusDetail) func convertPodObjectKeyMap(podList *corev1.PodList) map[string]struct{} { podObjectKeyMap := map[string]struct{}{} for _, v := range podList.Items { - objectKey := GetProgressObjectKey(v.Kind, v.Name) + objectKey := getProgressObjectKey(v.Kind, v.Name) podObjectKeyMap[objectKey] = struct{}{} } return podObjectKeyMap @@ -200,7 +200,7 @@ func handleStatelessProgress(reqCtx intctrlutil.RequestCtx, return 0, intctrlutil.NewError(intctrlutil.ErrorWaitCacheRefresh, "wait for the pods of deployment to be synchronized") } - currComponent, err := stateless.NewStateless(cli, opsRes.Cluster, + currComponent, err := stateless.NewStatelessComponent(cli, opsRes.Cluster, pgRes.clusterComponent, *pgRes.clusterComponentDef) if err != nil { return 0, err @@ -219,7 +219,7 @@ func handleStatelessProgress(reqCtx intctrlutil.RequestCtx, opsRequest := opsRes.OpsRequest opsStartTime := opsRequest.Status.StartTimestamp for _, v := range podList.Items { - objectKey := GetProgressObjectKey(v.Kind, v.Name) + objectKey := getProgressObjectKey(v.Kind, v.Name) progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey} if podIsPendingDuringOperation(opsStartTime, &v, compStatus.Phase) { handlePendingProgressDetail(opsRes, compStatus, progressDetail) @@ -261,7 +261,7 @@ func handleStatefulSetProgress(reqCtx intctrlutil.RequestCtx, opsStartTime := opsRequest.Status.StartTimestamp var completedCount int32 for _, v := range podList.Items { - objectKey := GetProgressObjectKey(v.Kind, v.Name) + objectKey := getProgressObjectKey(v.Kind, v.Name) progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey} if podIsPendingDuringOperation(opsStartTime, &v, compStatus.Phase) { handlePendingProgressDetail(opsRes, compStatus, progressDetail) @@ -284,7 +284,7 @@ func handlePendingProgressDetail(opsRes *OpsResource, progressDetail appsv1alpha1.ProgressStatusDetail, ) { progressDetail.Status = appsv1alpha1.PendingProgressStatus - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) } @@ -296,7 +296,7 @@ func handleSucceedProgressDetail(opsRes *OpsResource, ) { progressDetail.SetStatusAndMessage(appsv1alpha1.SucceedProgressStatus, getProgressSucceedMessage(pgRes.opsMessageKey, progressDetail.ObjectKey, pgRes.clusterComponent.Name)) - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) } @@ -321,7 +321,7 @@ func handleFailedOrProcessingProgressDetail(opsRes *OpsResource, progressDetail.SetStatusAndMessage(appsv1alpha1.ProcessingProgressStatus, getProgressProcessingMessage(pgRes.opsMessageKey, progressDetail.ObjectKey, componentName)) } - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) return completedCount } @@ -387,39 +387,44 @@ func getComponentLastReplicas(opsRequest *appsv1alpha1.OpsRequest, componentName } // handleComponentProgressDetails handles the component progressDetails when scale the replicas. +// @return expectProgressCount, +// @return completedCount +// @return error func handleComponentProgressForScalingReplicas(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes *OpsResource, pgRes progressResource, compStatus *appsv1alpha1.OpsRequestComponentStatus, - getExpectReplicas func(opsRequest *appsv1alpha1.OpsRequest, componentName string) *int32) (expectProgressCount int32, completedCount int32, err error) { + getExpectReplicas func(opsRequest *appsv1alpha1.OpsRequest, componentName string) *int32) (int32, int32, error) { var ( - podList *corev1.PodList - clusterComponent = pgRes.clusterComponent - opsRequest = opsRes.OpsRequest - isScaleOut bool + podList *corev1.PodList + clusterComponent = pgRes.clusterComponent + opsRequest = opsRes.OpsRequest + isScaleOut bool + expectProgressCount int32 + completedCount int32 + err error ) if clusterComponent == nil || pgRes.clusterComponentDef == nil { - return + return 0, 0, nil } expectReplicas := getExpectReplicas(opsRequest, clusterComponent.Name) if expectReplicas == nil { - return + return 0, 0, nil } lastComponentReplicas := getComponentLastReplicas(opsRequest, clusterComponent.Name) if lastComponentReplicas == nil { - return + return 0, 0, nil } // if replicas are not changed, return if *lastComponentReplicas == *expectReplicas { - return + return 0, 0, nil } if podList, err = util.GetComponentPodList(reqCtx.Ctx, cli, *opsRes.Cluster, clusterComponent.Name); err != nil { - return + return 0, 0, err } if compStatus.Phase == appsv1alpha1.RunningClusterCompPhase && pgRes.clusterComponent.Replicas != int32(len(podList.Items)) { - err = intctrlutil.NewError(intctrlutil.ErrorWaitCacheRefresh, "wait for the pods of component to be synchronized") - return + return 0, 0, intctrlutil.NewError(intctrlutil.ErrorWaitCacheRefresh, "wait for the pods of component to be synchronized") } dValue := *expectReplicas - *lastComponentReplicas if dValue > 0 { @@ -431,7 +436,7 @@ func handleComponentProgressForScalingReplicas(reqCtx intctrlutil.RequestCtx, if !isScaleOut { completedCount, err = handleScaleDownProgress(opsRes, pgRes, podList, compStatus) expectProgressCount = getFinalExpectCount(compStatus, expectProgressCount) - return + return expectProgressCount, completedCount, err } completedCount, err = handleScaleOutProgress(reqCtx, cli, opsRes, pgRes, podList, compStatus) // if the workload type is Stateless, remove the progressDetails of the expired pods. @@ -466,13 +471,13 @@ func handleScaleOutProgress(reqCtx intctrlutil.RequestCtx, if v.CreationTimestamp.Before(&opsRes.OpsRequest.Status.StartTimestamp) { continue } - objectKey := GetProgressObjectKey(v.Kind, v.Name) + objectKey := getProgressObjectKey(v.Kind, v.Name) progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey} if currComponent.PodIsAvailable(&v, minReadySeconds) { completedCount += 1 message := fmt.Sprintf("Successfully created pod: %s in Component: %s", objectKey, componentName) progressDetail.SetStatusAndMessage(appsv1alpha1.SucceedProgressStatus, message) - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) continue } @@ -486,7 +491,7 @@ func handleScaleOutProgress(reqCtx intctrlutil.RequestCtx, } else { progressDetail.SetStatusAndMessage(appsv1alpha1.ProcessingProgressStatus, "Start to create pod: "+objectKey) } - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) } return completedCount, nil @@ -501,7 +506,7 @@ func handleScaleDownProgress( podMap := map[string]struct{}{} // record the deleting pod progressDetail for _, v := range podList.Items { - objectKey := GetProgressObjectKey(constant.PodKind, v.Name) + objectKey := getProgressObjectKey(constant.PodKind, v.Name) podMap[objectKey] = struct{}{} if v.DeletionTimestamp.IsZero() { continue @@ -511,13 +516,13 @@ func handleScaleDownProgress( Status: appsv1alpha1.ProcessingProgressStatus, Message: fmt.Sprintf("Start to delete pod: %s in Component: %s", objectKey, pgRes.clusterComponent.Name), } - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) } lastComponentConfigs := opsRes.OpsRequest.Status.LastConfiguration.Components[pgRes.clusterComponent.Name] lastComponentPodNames := lastComponentConfigs.TargetResources[appsv1alpha1.PodsCompResourceKey] for _, v := range lastComponentPodNames { - objectKey := GetProgressObjectKey(constant.PodKind, v) + objectKey := getProgressObjectKey(constant.PodKind, v) if _, ok := podMap[objectKey]; ok { continue } @@ -528,7 +533,7 @@ func handleScaleDownProgress( Message: fmt.Sprintf("Successfully deleted pod: %s in Component: %s", objectKey, pgRes.clusterComponent.Name), } completedCount += 1 - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) } return completedCount, nil diff --git a/controllers/apps/operations/ops_progress_util_test.go b/controllers/apps/operations/ops_progress_util_test.go index 19d5a9dbf..328c82dd8 100644 --- a/controllers/apps/operations/ops_progress_util_test.go +++ b/controllers/apps/operations/ops_progress_util_test.go @@ -166,8 +166,8 @@ var _ = Describe("Ops ProgressDetails", func() { By("create horizontalScaling operation to test the progressDetails when scaling up the replicas ") initClusterForOps(opsRes) expectClusterComponentReplicas := int32(2) - Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func() { - opsRes.Cluster.Spec.ComponentSpecs[1].Replicas = expectClusterComponentReplicas + Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func(lcluster *appsv1alpha1.Cluster) { + lcluster.Spec.ComponentSpecs[1].Replicas = expectClusterComponentReplicas })).ShouldNot(HaveOccurred()) opsRes.OpsRequest = createHorizontalScaling(clusterName, 3) // update ops phase to Running first @@ -193,9 +193,9 @@ var _ = Describe("Ops ProgressDetails", func() { }) func getProgressDetailStatus(opsRes *OpsResource, componentName string, pod *corev1.Pod) appsv1alpha1.ProgressStatus { - objectKey := GetProgressObjectKey(pod.Kind, pod.Name) + objectKey := getProgressObjectKey(pod.Kind, pod.Name) progressDetails := opsRes.OpsRequest.Status.Components[componentName].ProgressDetails - progressDetail := FindStatusProgressDetail(progressDetails, objectKey) + progressDetail := findStatusProgressDetail(progressDetails, objectKey) var status appsv1alpha1.ProgressStatus if progressDetail != nil { status = progressDetail.Status diff --git a/controllers/apps/operations/ops_util.go b/controllers/apps/operations/ops_util.go index b097ee6f5..b099c1c3e 100644 --- a/controllers/apps/operations/ops_util.go +++ b/controllers/apps/operations/ops_util.go @@ -42,10 +42,10 @@ type handleStatusProgressWithComponent func(reqCtx intctrlutil.RequestCtx, type handleReconfigureOpsStatus func(cmStatus *appsv1alpha1.ConfigurationStatus) error -// ReconcileActionWithComponentOps will be performed when action is done and loops till OpsRequest.status.phase is Succeed/Failed. +// reconcileActionWithComponentOps will be performed when action is done and loops till OpsRequest.status.phase is Succeed/Failed. // if OpsRequest.spec.componentOps is not null, you can use it to OpsBehaviour.ReconcileAction. // return the OpsRequest.status.phase -func ReconcileActionWithComponentOps(reqCtx intctrlutil.RequestCtx, +func reconcileActionWithComponentOps(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes *OpsResource, opsMessageKey string, @@ -55,7 +55,7 @@ func ReconcileActionWithComponentOps(reqCtx intctrlutil.RequestCtx, return "", 0, nil } opsRequestPhase := appsv1alpha1.OpsRunningPhase - clusterDef, err := GetClusterDefByName(reqCtx.Ctx, cli, + clusterDef, err := getClusterDefByName(reqCtx.Ctx, cli, opsRes.Cluster.Spec.ClusterDefRef) if err != nil { return opsRequestPhase, 0, err @@ -137,8 +137,8 @@ func opsRequestIsComponent(opsRes OpsResource) bool { opsRes.Cluster.Status.ObservedGeneration >= opsRes.OpsRequest.Status.ClusterGeneration } -// GetClusterDefByName gets the ClusterDefinition object by the name. -func GetClusterDefByName(ctx context.Context, cli client.Client, clusterDefName string) (*appsv1alpha1.ClusterDefinition, error) { +// getClusterDefByName gets the ClusterDefinition object by the name. +func getClusterDefByName(ctx context.Context, cli client.Client, clusterDefName string) (*appsv1alpha1.ClusterDefinition, error) { clusterDef := &appsv1alpha1.ClusterDefinition{} if err := cli.Get(ctx, client.ObjectKey{Name: clusterDefName}, clusterDef); err != nil { return nil, err @@ -151,7 +151,7 @@ func opsRequestIsCompleted(phase appsv1alpha1.OpsPhase) bool { return slices.Index([]appsv1alpha1.OpsPhase{appsv1alpha1.OpsFailedPhase, appsv1alpha1.OpsSucceedPhase}, phase) != -1 } -func PatchOpsStatusWithOpsDeepCopy(ctx context.Context, +func patchOpsStatusWithOpsDeepCopy(ctx context.Context, cli client.Client, opsRes *OpsResource, opsRequestDeepCopy *appsv1alpha1.OpsRequest, @@ -192,7 +192,7 @@ func PatchOpsStatus(ctx context.Context, opsRes *OpsResource, phase appsv1alpha1.OpsPhase, condition ...*metav1.Condition) error { - return PatchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsRes.OpsRequest.DeepCopy(), phase, condition...) + return patchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsRes.OpsRequest.DeepCopy(), phase, condition...) } // PatchClusterNotFound patches ClusterNotFound condition to the OpsRequest.status.conditions. @@ -209,8 +209,8 @@ func patchOpsHandlerNotSupported(ctx context.Context, cli client.Client, opsRes return PatchOpsStatus(ctx, cli, opsRes, appsv1alpha1.OpsFailedPhase, condition) } -// PatchValidateErrorCondition patches ValidateError condition to the OpsRequest.status.conditions. -func PatchValidateErrorCondition(ctx context.Context, cli client.Client, opsRes *OpsResource, errMessage string) error { +// patchValidateErrorCondition patches ValidateError condition to the OpsRequest.status.conditions. +func patchValidateErrorCondition(ctx context.Context, cli client.Client, opsRes *OpsResource, errMessage string) error { condition := appsv1alpha1.NewValidateFailedCondition(appsv1alpha1.ReasonValidateFailed, errMessage) return PatchOpsStatus(ctx, cli, opsRes, appsv1alpha1.OpsFailedPhase, condition) } @@ -247,7 +247,7 @@ func patchOpsRequestToCreating(ctx context.Context, var condition *metav1.Condition validatePassCondition := appsv1alpha1.NewValidatePassedCondition(opsRes.OpsRequest.Name) condition = opsHandler.ActionStartedCondition(opsRes.OpsRequest) - return PatchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsDeepCoy, appsv1alpha1.OpsCreatingPhase, validatePassCondition, condition) + return patchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsDeepCoy, appsv1alpha1.OpsCreatingPhase, validatePassCondition, condition) } // patchClusterStatusAndRecordEvent records the ops event in the cluster and diff --git a/controllers/apps/operations/ops_util_test.go b/controllers/apps/operations/ops_util_test.go index eb2352d3a..c73e7ba92 100644 --- a/controllers/apps/operations/ops_util_test.go +++ b/controllers/apps/operations/ops_util_test.go @@ -64,7 +64,7 @@ var _ = Describe("OpsUtil functions", func() { By("Test the functions in ops_util.go") opsRes.OpsRequest = createHorizontalScaling(clusterName, 1) - Expect(PatchValidateErrorCondition(ctx, k8sClient, opsRes, "validate error")).Should(Succeed()) + Expect(patchValidateErrorCondition(ctx, k8sClient, opsRes, "validate error")).Should(Succeed()) Expect(patchOpsHandlerNotSupported(ctx, k8sClient, opsRes)).Should(Succeed()) Expect(isOpsRequestFailedPhase(appsv1alpha1.OpsFailedPhase)).Should(BeTrue()) Expect(PatchClusterNotFound(ctx, k8sClient, opsRes)).Should(Succeed()) diff --git a/controllers/apps/operations/reconfigure.go b/controllers/apps/operations/reconfigure.go index 56b39928e..ff652a496 100644 --- a/controllers/apps/operations/reconfigure.go +++ b/controllers/apps/operations/reconfigure.go @@ -94,21 +94,21 @@ func (r *reconfigureAction) Handle(eventContext cfgcore.ConfigEventContext, last switch phase { case appsv1alpha1.OpsSucceedPhase: // only update the condition of the opsRequest. - return PatchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsDeepCopy, appsv1alpha1.OpsRunningPhase, + return patchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsDeepCopy, appsv1alpha1.OpsRunningPhase, appsv1alpha1.NewReconfigureRunningCondition(opsRequest, appsv1alpha1.ReasonReconfigureSucceed, eventContext.ConfigSpecName, formatConfigPatchToMessage(eventContext.ConfigPatch, &eventContext.PolicyStatus)), appsv1alpha1.NewSucceedCondition(opsRequest)) case appsv1alpha1.OpsFailedPhase: - return PatchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsDeepCopy, appsv1alpha1.OpsRunningPhase, + return patchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsDeepCopy, appsv1alpha1.OpsRunningPhase, appsv1alpha1.NewReconfigureRunningCondition(opsRequest, appsv1alpha1.ReasonReconfigureFailed, eventContext.ConfigSpecName, formatConfigPatchToMessage(eventContext.ConfigPatch, &eventContext.PolicyStatus)), appsv1alpha1.NewReconfigureFailedCondition(opsRequest, cfgError)) default: - return PatchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsDeepCopy, appsv1alpha1.OpsRunningPhase, + return patchOpsStatusWithOpsDeepCopy(ctx, cli, opsRes, opsDeepCopy, appsv1alpha1.OpsRunningPhase, appsv1alpha1.NewReconfigureRunningCondition(opsRequest, appsv1alpha1.ReasonReconfigureRunning, eventContext.ConfigSpecName)) diff --git a/controllers/apps/operations/restart.go b/controllers/apps/operations/restart.go index 342ab10f6..7ea87d42f 100644 --- a/controllers/apps/operations/restart.go +++ b/controllers/apps/operations/restart.go @@ -69,7 +69,7 @@ func (r restartOpsHandler) Action(reqCtx intctrlutil.RequestCtx, cli client.Clie // ReconcileAction will be performed when action is done and loops till OpsRequest.status.phase is Succeed/Failed. // the Reconcile function for volume expansion opsRequest. func (r restartOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes *OpsResource) (appsv1alpha1.OpsPhase, time.Duration, error) { - return ReconcileActionWithComponentOps(reqCtx, cli, opsRes, "restart", handleComponentStatusProgress) + return reconcileActionWithComponentOps(reqCtx, cli, opsRes, "restart", handleComponentStatusProgress) } // GetRealAffectedComponentMap gets the real affected component map for the operation diff --git a/controllers/apps/operations/start.go b/controllers/apps/operations/start.go index 36b0fdb2a..15941f1e3 100644 --- a/controllers/apps/operations/start.go +++ b/controllers/apps/operations/start.go @@ -90,7 +90,7 @@ func (start StartOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCtx, cli compStatus *appsv1alpha1.OpsRequestComponentStatus) (int32, int32, error) { return handleComponentProgressForScalingReplicas(reqCtx, cli, opsRes, pgRes, compStatus, getExpectReplicas) } - return ReconcileActionWithComponentOps(reqCtx, cli, opsRes, "", handleComponentProgress) + return reconcileActionWithComponentOps(reqCtx, cli, opsRes, "", handleComponentProgress) } // SaveLastConfiguration records last configuration to the OpsRequest.status.lastConfiguration diff --git a/controllers/apps/operations/stop.go b/controllers/apps/operations/stop.go index 00d027923..71ba56bcb 100644 --- a/controllers/apps/operations/stop.go +++ b/controllers/apps/operations/stop.go @@ -86,7 +86,7 @@ func (stop StopOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCtx, cli cl compStatus *appsv1alpha1.OpsRequestComponentStatus) (int32, int32, error) { return handleComponentProgressForScalingReplicas(reqCtx, cli, opsRes, pgRes, compStatus, getExpectReplicas) } - return ReconcileActionWithComponentOps(reqCtx, cli, opsRes, "", handleComponentProgress) + return reconcileActionWithComponentOps(reqCtx, cli, opsRes, "", handleComponentProgress) } // SaveLastConfiguration records last configuration to the OpsRequest.status.lastConfiguration diff --git a/controllers/apps/operations/upgrade.go b/controllers/apps/operations/upgrade.go index e04fc914c..e96849409 100644 --- a/controllers/apps/operations/upgrade.go +++ b/controllers/apps/operations/upgrade.go @@ -60,7 +60,7 @@ func (u upgradeOpsHandler) Action(reqCtx intctrlutil.RequestCtx, cli client.Clie // ReconcileAction will be performed when action is done and loops till OpsRequest.status.phase is Succeed/Failed. // the Reconcile function for upgrade opsRequest. func (u upgradeOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes *OpsResource) (appsv1alpha1.OpsPhase, time.Duration, error) { - return ReconcileActionWithComponentOps(reqCtx, cli, opsRes, "upgrade", handleComponentStatusProgress) + return reconcileActionWithComponentOps(reqCtx, cli, opsRes, "upgrade", handleComponentStatusProgress) } // GetRealAffectedComponentMap gets the real affected component map for the operation diff --git a/controllers/apps/operations/vertical_scaling.go b/controllers/apps/operations/vertical_scaling.go index 4d23e5817..b61aa8d4b 100644 --- a/controllers/apps/operations/vertical_scaling.go +++ b/controllers/apps/operations/vertical_scaling.go @@ -72,7 +72,7 @@ func (vs verticalScalingHandler) Action(reqCtx intctrlutil.RequestCtx, cli clien // ReconcileAction will be performed when action is done and loops till OpsRequest.status.phase is Succeed/Failed. // the Reconcile function for vertical scaling opsRequest. func (vs verticalScalingHandler) ReconcileAction(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes *OpsResource) (appsv1alpha1.OpsPhase, time.Duration, error) { - return ReconcileActionWithComponentOps(reqCtx, cli, opsRes, "vertical scale", handleComponentStatusProgress) + return reconcileActionWithComponentOps(reqCtx, cli, opsRes, "vertical scale", handleComponentStatusProgress) } // SaveLastConfiguration records last configuration to the OpsRequest.status.lastConfiguration diff --git a/controllers/apps/operations/volume_expansion.go b/controllers/apps/operations/volume_expansion.go index 4e7f079ac..64b3787e0 100644 --- a/controllers/apps/operations/volume_expansion.go +++ b/controllers/apps/operations/volume_expansion.go @@ -309,7 +309,7 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti completedCount += 1 message := fmt.Sprintf("Successfully expand volume: %s in Component: %s ", objectKey, componentName) progressDetail.SetStatusAndMessage(appsv1alpha1.SucceedProgressStatus, message) - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) continue } if ve.pvcIsResizing(&v) { @@ -319,7 +319,7 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti message := fmt.Sprintf("Waiting for an external controller to process the pvc: %s in Component: %s ", objectKey, componentName) progressDetail.SetStatusAndMessage(appsv1alpha1.PendingProgressStatus, message) } - SetComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) if ve.isExpansionCompleted(progressDetail.Status) { completedCount += 1 } diff --git a/controllers/apps/operations/volume_expansion_test.go b/controllers/apps/operations/volume_expansion_test.go index 4ed9f598e..9d3542525 100644 --- a/controllers/apps/operations/volume_expansion_test.go +++ b/controllers/apps/operations/volume_expansion_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -177,7 +178,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(newOps), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) { progressDetails := tmpOps.Status.Components[consensusCompName].ProgressDetails g.Expect(len(progressDetails) > 0).Should(BeTrue()) - progressDetail := FindStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcName)) + progressDetail := findStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcName)) g.Expect(progressDetail.Status == appsv1alpha1.FailedProgressStatus).Should(BeTrue()) })).Should(Succeed()) } @@ -214,7 +215,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(newOps), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) { progressDetails := tmpOps.Status.Components[consensusCompName].ProgressDetails - progressDetail := FindStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcName)) + progressDetail := findStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcName)) g.Expect(progressDetail != nil && progressDetail.Status == appsv1alpha1.ProcessingProgressStatus).Should(BeTrue()) })).Should(Succeed()) @@ -262,8 +263,8 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { clusterVersionName, clusterName, "consensus", consensusCompName) // init storageClass sc := testapps.CreateStorageClass(testCtx, storageClassName, true) - Expect(testapps.ChangeObj(&testCtx, sc, func() { - sc.Annotations = map[string]string{storage.IsDefaultStorageClassAnnotation: "true"} + Expect(testapps.ChangeObj(&testCtx, sc, func(lsc *storagev1.StorageClass) { + lsc.Annotations = map[string]string{storage.IsDefaultStorageClassAnnotation: "true"} })).ShouldNot(HaveOccurred()) opsRes := &OpsResource{ diff --git a/controllers/apps/operations/volume_expansion_updater.go b/controllers/apps/operations/volume_expansion_updater.go index 7b873afd2..2548683a6 100644 --- a/controllers/apps/operations/volume_expansion_updater.go +++ b/controllers/apps/operations/volume_expansion_updater.go @@ -181,7 +181,7 @@ func (pvcEventHandler PersistentVolumeClaimEventHandler) handlePVCFailedStatusOn } // save the failed message to the progressDetail. objectKey := getPVCProgressObjectKey(pvc.Name) - progressDetail := FindStatusProgressDetail(component.ProgressDetails, objectKey) + progressDetail := findStatusProgressDetail(component.ProgressDetails, objectKey) if progressDetail == nil || progressDetail.Message != event.Message { isChanged = true } @@ -192,7 +192,7 @@ func (pvcEventHandler PersistentVolumeClaimEventHandler) handlePVCFailedStatusOn Message: event.Message, } - SetComponentStatusProgressDetail(recorder, opsRequest, &component.ProgressDetails, *progressDetail) + setComponentStatusProgressDetail(recorder, opsRequest, &component.ProgressDetails, *progressDetail) compsStatus[cName] = component break } diff --git a/controllers/apps/opsrequest_controller.go b/controllers/apps/opsrequest_controller.go index 2ecc597c7..03cd3b218 100644 --- a/controllers/apps/opsrequest_controller.go +++ b/controllers/apps/opsrequest_controller.go @@ -63,7 +63,7 @@ func (r *OpsRequestReconciler) Reconcile(ctx context.Context, req ctrl.Request) opsCtrlHandler := &opsControllerHandler{} return opsCtrlHandler.Handle(reqCtx, &operations.OpsResource{Recorder: r.Recorder}, r.fetchOpsRequest, - r.handleDeleteEvent, + r.handleDeletion, r.fetchCluster, r.addClusterLabelAndSetOwnerReference, r.handleOpsRequestByPhase, @@ -85,7 +85,7 @@ func (r *OpsRequestReconciler) fetchOpsRequest(reqCtx intctrlutil.RequestCtx, op return intctrlutil.ResultToP(intctrlutil.RequeueWithError(err, reqCtx.Log, "")) } // if the opsRequest is not found, we need to check if this opsRequest is deleted abnormally - if err = r.handleOpsDeletedDuringRunning(reqCtx); err != nil { + if err = r.handleOpsReqDeletedDuringRunning(reqCtx); err != nil { return intctrlutil.ResultToP(intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "")) } return intctrlutil.ResultToP(intctrlutil.Reconciled()) @@ -94,8 +94,8 @@ func (r *OpsRequestReconciler) fetchOpsRequest(reqCtx intctrlutil.RequestCtx, op return nil, nil } -// handleDeleteEvent handles the delete event of the OpsRequest. -func (r *OpsRequestReconciler) handleDeleteEvent(reqCtx intctrlutil.RequestCtx, opsRes *operations.OpsResource) (*ctrl.Result, error) { +// handleDeletion handles the delete event of the OpsRequest. +func (r *OpsRequestReconciler) handleDeletion(reqCtx intctrlutil.RequestCtx, opsRes *operations.OpsResource) (*ctrl.Result, error) { if opsRes.OpsRequest.Status.Phase == appsv1alpha1.OpsRunningPhase { return nil, nil } @@ -223,8 +223,8 @@ func (r *OpsRequestReconciler) doOpsRequestAction(reqCtx intctrlutil.RequestCtx, return intctrlutil.ResultToP(intctrlutil.Reconciled()) } -// handleOpsDeletedDuringRunning handles the cluster annotation if the OpsRequest is deleted during running. -func (r *OpsRequestReconciler) handleOpsDeletedDuringRunning(reqCtx intctrlutil.RequestCtx) error { +// handleOpsReqDeletedDuringRunning handles the cluster annotation if the OpsRequest is deleted during running. +func (r *OpsRequestReconciler) handleOpsReqDeletedDuringRunning(reqCtx intctrlutil.RequestCtx) error { clusterList := &appsv1alpha1.ClusterList{} if err := r.Client.List(reqCtx.Ctx, clusterList, client.InNamespace(reqCtx.Req.Namespace)); err != nil { return err diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index cea8ed5f5..ff6c8605d 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/controllers/apps/components/replicationset" opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/lifecycle" @@ -44,8 +45,7 @@ var _ = Describe("OpsRequest Controller", func() { const clusterDefName = "test-clusterdef" const clusterVersionName = "test-clusterversion" const clusterNamePrefix = "test-cluster" - - const mysqlCompType = "consensus" + const mysqlCompDefName = "consensus" const mysqlCompName = "mysql" const defaultMinReadySeconds = 10 @@ -126,7 +126,7 @@ var _ = Describe("OpsRequest Controller", func() { })()).ShouldNot(HaveOccurred()) } - testVerticalScaleCPUAndMemory := func(workloadType testapps.ComponentTplType) { + testVerticalScaleCPUAndMemory := func(workloadType testapps.ComponentDefTplType) { const opsName = "mysql-verticalscaling" By("Create a cluster obj") @@ -142,7 +142,7 @@ var _ = Describe("OpsRequest Controller", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). SetReplicas(1). SetResources(resources). Create(&testCtx).GetObject() @@ -207,11 +207,11 @@ var _ = Describe("OpsRequest Controller", func() { checkLatestOpsHasProcessed(clusterKey) By("patch opsrequest controller to run") - Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func() { - if verticalScalingOpsRequest.Annotations == nil { - verticalScalingOpsRequest.Annotations = map[string]string{} + Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func(lopsReq *appsv1alpha1.OpsRequest) { + if lopsReq.Annotations == nil { + lopsReq.Annotations = map[string]string{} } - verticalScalingOpsRequest.Annotations[constant.ReconcileAnnotationKey] = time.Now().Format(time.RFC3339Nano) + lopsReq.Annotations[constant.ReconcileAnnotationKey] = time.Now().Format(time.RFC3339Nano) })).ShouldNot(HaveOccurred()) By("check VerticalScalingOpsRequest succeed") @@ -224,8 +224,8 @@ var _ = Describe("OpsRequest Controller", func() { })).Should(Succeed()) By("check OpsRequest reclaimed after ttl") - Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func() { - verticalScalingOpsRequest.Spec.TTLSecondsAfterSucceed = 1 + Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func(lopsReq *appsv1alpha1.OpsRequest) { + lopsReq.Spec.TTLSecondsAfterSucceed = 1 })).ShouldNot(HaveOccurred()) Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKeyFromObject(verticalScalingOpsRequest), verticalScalingOpsRequest, false)).Should(Succeed()) @@ -233,16 +233,17 @@ var _ = Describe("OpsRequest Controller", func() { // Scenarios + // TODO: should focus on OpsRequest control actions, and iterator through all component workload types. Context("with Cluster which has MySQL StatefulSet", func() { BeforeEach(func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) @@ -255,7 +256,7 @@ var _ = Describe("OpsRequest Controller", func() { BeforeEach(func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ConsensusMySQLComponent, mysqlCompType). + AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). AddHorizontalScalePolicy(appsv1alpha1.HorizontalScalePolicy{ Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, BackupPolicyTemplateName: backupPolicyTPLName, @@ -263,7 +264,7 @@ var _ = Describe("OpsRequest Controller", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) @@ -286,7 +287,7 @@ var _ = Describe("OpsRequest Controller", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). SetReplicas(replicas). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). Create(&testCtx).GetObject() @@ -337,8 +338,8 @@ var _ = Describe("OpsRequest Controller", func() { By("delete h-scale ops") testapps.DeleteObject(&testCtx, opsKey, ops) - Expect(testapps.ChangeObj(&testCtx, ops, func() { - ops.Finalizers = []string{} + Expect(testapps.ChangeObj(&testCtx, ops, func(lopsReq *appsv1alpha1.OpsRequest) { + lopsReq.SetFinalizers([]string{}) })).ShouldNot(HaveOccurred()) By("reset replicas to 1 and cluster should reconcile to Running") @@ -357,14 +358,18 @@ var _ = Describe("OpsRequest Controller", func() { Eventually(testapps.GetListLen(&testCtx, intctrlutil.StatefulSetSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterObj.Name, - }, client.InNamespace(clusterObj.Namespace))).Should(BeEquivalentTo(2)) - stsList = testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), testapps.DefaultRedisCompName) - for _, v := range stsList.Items { - Expect(testapps.ChangeObjStatus(&testCtx, &v, func() { - testk8s.MockStatefulSetReady(&v) - })).ShouldNot(HaveOccurred()) - podName := v.Name + "-0" - pod := testapps.MockReplicationComponentStsPod(testCtx, &v, clusterObj.Name, testapps.DefaultRedisCompName, podName, v.Labels[constant.RoleLabelKey]) + }, client.InNamespace(clusterObj.Namespace))).Should(BeEquivalentTo(1)) + stsList = testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), + testapps.DefaultRedisCompName) + Expect(stsList.Items).Should(HaveLen(1)) + sts := &stsList.Items[0] + Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { + testk8s.MockStatefulSetReady(sts) + })).ShouldNot(HaveOccurred()) + for i := int32(0); i < *sts.Spec.Replicas; i++ { + podName := fmt.Sprintf("%s-%d", sts.Name, i) + pod := testapps.MockReplicationComponentStsPod(nil, testCtx, sts, clusterObj.Name, + testapps.DefaultRedisCompName, podName, replicationset.DefaultRole(i)) podList = append(podList, pod) } } @@ -374,12 +379,13 @@ var _ = Describe("OpsRequest Controller", func() { storageClassName := "standard" testapps.CreateStorageClass(testCtx, storageClassName, true) clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompType). + AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(testapps.DefaultRedisCompType).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, + testapps.DefaultRedisImageName). Create(&testCtx).GetObject() By("Creating a cluster with replication workloadType.") @@ -387,68 +393,15 @@ var _ = Describe("OpsRequest Controller", func() { pvcSpec.StorageClassName = &storageClassName clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec).SetPrimaryIndex(0). SetReplicas(testapps.DefaultReplicationReplicas). Create(&testCtx).GetObject() // mock sts ready and create pod createStsPodAndMockStsReady() // wait for cluster to running - Eventually(testapps.GetClusterPhase(&testCtx, client.ObjectKeyFromObject(clusterObj))).Should(Equal(appsv1alpha1.RunningClusterPhase)) - }) - - It("test stop/start ops", func() { - By("Create a stop ops") - stopOpsName := "stop-ops" + testCtx.GetRandomStr() - stopOps := testapps.NewOpsRequestObj(stopOpsName, clusterObj.Namespace, - clusterObj.Name, appsv1alpha1.StopType) - Expect(testCtx.CreateObj(testCtx.Ctx, stopOps)).Should(Succeed()) - - clusterKey = client.ObjectKeyFromObject(clusterObj) - opsKey := client.ObjectKeyFromObject(stopOps) - Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) - // mock deleting pod - for _, pod := range podList { - testk8s.MockPodIsTerminating(ctx, testCtx, pod) - } - // reconcile opsRequest - Expect(testapps.ChangeObj(&testCtx, stopOps, func() { - stopOps.Annotations = map[string]string{ - constant.ReconcileAnnotationKey: time.Now().Format(time.RFC3339Nano), - } - })).ShouldNot(HaveOccurred()) - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.StoppedClusterPhase)) - - By("should be Running before pods are not deleted successfully") - Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) - checkLatestOpsIsProcessing(clusterKey, stopOps.Spec.Type) - // mock pod deleted successfully - for _, pod := range podList { - Expect(testapps.ChangeObj(&testCtx, pod, func() { - pod.Finalizers = make([]string, 0) - })).ShouldNot(HaveOccurred()) - } - By("ops phase should be Succeed") - // reconcile opsRequest - Expect(testapps.ChangeObj(&testCtx, stopOps, func() { - stopOps.Annotations = map[string]string{ - constant.ReconcileAnnotationKey: time.Now().Format(time.RFC3339Nano), - } - })).ShouldNot(HaveOccurred()) - Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsSucceedPhase)) - checkLatestOpsHasProcessed(clusterKey) - - By("test start ops") - startOpsName := "start-ops" + testCtx.GetRandomStr() - startOps := testapps.NewOpsRequestObj(startOpsName, clusterObj.Namespace, - clusterObj.Name, appsv1alpha1.StartType) - opsKey = client.ObjectKeyFromObject(startOps) - Expect(testCtx.CreateObj(testCtx.Ctx, startOps)).Should(Succeed()) - Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) - // mock sts ready and create pod - createStsPodAndMockStsReady() - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsSucceedPhase)) + Eventually(testapps.GetClusterPhase(&testCtx, client.ObjectKeyFromObject(clusterObj))). + Should(Equal(appsv1alpha1.RunningClusterPhase)) }) It("delete Running opsRequest", func() { @@ -480,8 +433,8 @@ var _ = Describe("OpsRequest Controller", func() { By("delete the Running ops") testapps.DeleteObject(&testCtx, opsKey, volumeExpandOps) - Expect(testapps.ChangeObj(&testCtx, volumeExpandOps, func() { - volumeExpandOps.Finalizers = []string{} + Expect(testapps.ChangeObj(&testCtx, volumeExpandOps, func(lopsReq *appsv1alpha1.OpsRequest) { + lopsReq.SetFinalizers([]string{}) })).ShouldNot(HaveOccurred()) By("check the cluster annotation") @@ -490,6 +443,5 @@ var _ = Describe("OpsRequest Controller", func() { g.Expect(opsSlice).Should(HaveLen(0)) })).Should(Succeed()) }) - }) }) diff --git a/controllers/apps/systemaccount_controller_test.go b/controllers/apps/systemaccount_controller_test.go index 53c88a341..bbf588303 100644 --- a/controllers/apps/systemaccount_controller_test.go +++ b/controllers/apps/systemaccount_controller_test.go @@ -39,15 +39,15 @@ import ( var _ = Describe("SystemAccount Controller", func() { const ( - clusterDefName = "test-clusterdef" - clusterVersionName = "test-clusterversion" - clusterNamePrefix = "test-cluster" - mysqlCompType = "replicasets" - mysqlCompTypeWOSysAcct = "wo-sysacct" - mysqlCompName = "mysql" - mysqlCompNameWOSysAcct = "wo-sysacct" - orphanFinalizerName = "orphan" - clusterEndPointsSize = 3 + clusterDefName = "test-clusterdef" + clusterVersionName = "test-clusterversion" + clusterNamePrefix = "test-cluster" + mysqlCompDefName = "replicasets" + mysqlCompTypeWOSysAcctDefName = "wo-sysacct" + mysqlCompName = "mysql" + mysqlCompNameWOSysAcct = "wo-sysacct" + orphanFinalizerName = "orphan" + clusterEndPointsSize = 3 ) /** @@ -190,14 +190,14 @@ var _ = Describe("SystemAccount Controller", func() { By("Create a clusterDefinition obj") systemAccount := mockSystemAccountsSpec() clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). AddSystemAccountSpec(systemAccount). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompTypeWOSysAcct). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompTypeWOSysAcctDefName). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). AddComponent(mysqlCompNameWOSysAcct).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() @@ -285,12 +285,12 @@ var _ = Describe("SystemAccount Controller", func() { mysqlTestCases = map[string]*sysAcctTestCase{ "wesql-no-accts": { componentName: mysqlCompNameWOSysAcct, - componentDefRef: mysqlCompTypeWOSysAcct, + componentDefRef: mysqlCompTypeWOSysAcctDefName, accounts: []appsv1alpha1.AccountName{}, }, "wesql-with-accts": { componentName: mysqlCompName, - componentDefRef: mysqlCompType, + componentDefRef: mysqlCompDefName, accounts: getAllSysAccounts(), }, } @@ -436,7 +436,9 @@ var _ = Describe("SystemAccount Controller", func() { jobs := &batchv1.JobList{} g.Expect(k8sClient.List(ctx, jobs, client.InNamespace(cluster.Namespace), ml)).To(Succeed()) for _, job := range jobs.Items { - g.Expect(testapps.ChangeObj(&testCtx, &job, func() { controllerutil.RemoveFinalizer(&job, orphanFinalizerName) })).To(Succeed()) + g.Expect(testapps.ChangeObj(&testCtx, &job, func(ljob *batchv1.Job) { + controllerutil.RemoveFinalizer(ljob, orphanFinalizerName) + })).To(Succeed()) } g.Expect(len(jobs.Items)).To(Equal(0), "Verify all jobs completed and deleted") }).Should(Succeed()) @@ -484,12 +486,12 @@ var _ = Describe("SystemAccount Controller", func() { mysqlTestCases = map[string]*sysAcctTestCase{ "wesql-with-accts": { componentName: mysqlCompName, - componentDefRef: mysqlCompType, + componentDefRef: mysqlCompDefName, accounts: getAllSysAccounts(), }, "wesql-with-accts-dup": { componentName: mysqlCompName, - componentDefRef: mysqlCompType, + componentDefRef: mysqlCompDefName, accounts: getAllSysAccounts(), }, } @@ -588,12 +590,12 @@ var _ = Describe("SystemAccount Controller", func() { mysqlTestCases = map[string]*sysAcctTestCase{ "wesql-with-accts": { componentName: mysqlCompName, - componentDefRef: mysqlCompType, + componentDefRef: mysqlCompDefName, accounts: getAllSysAccounts(), }, "wesql-with-accts-dup": { componentName: mysqlCompName, - componentDefRef: mysqlCompType, + componentDefRef: mysqlCompDefName, accounts: getAllSysAccounts(), }, } @@ -672,7 +674,9 @@ var _ = Describe("SystemAccount Controller", func() { tmpJob := &batchv1.Job{} g.Expect(k8sClient.Get(ctx, jobKey, tmpJob)).To(Succeed()) g.Expect(len(tmpJob.ObjectMeta.Finalizers)).To(BeEquivalentTo(1)) - g.Expect(testapps.ChangeObj(&testCtx, tmpJob, func() { controllerutil.RemoveFinalizer(tmpJob, orphanFinalizerName) })).To(Succeed()) + g.Expect(testapps.ChangeObj(&testCtx, tmpJob, func(ljob *batchv1.Job) { + controllerutil.RemoveFinalizer(ljob, orphanFinalizerName) + })).To(Succeed()) }).Should(Succeed()) By("Verify jobs size decreased and secrets size increased") @@ -710,7 +714,9 @@ var _ = Describe("SystemAccount Controller", func() { err := k8sClient.Get(ctx, jobKey, tmpJob) g.Expect(err).To(Succeed()) g.Expect(len(tmpJob.ObjectMeta.Finalizers)).To(BeEquivalentTo(1)) - g.Expect(testapps.ChangeObj(&testCtx, tmpJob, func() { controllerutil.RemoveFinalizer(tmpJob, orphanFinalizerName) })).To(Succeed()) + g.Expect(testapps.ChangeObj(&testCtx, tmpJob, func(ljob *batchv1.Job) { + controllerutil.RemoveFinalizer(ljob, orphanFinalizerName) + })).To(Succeed()) }).Should(Succeed()) By("Verify jobs size decreased and secrets size increased") diff --git a/controllers/apps/systemaccount_util_test.go b/controllers/apps/systemaccount_util_test.go index 8d77fa352..c659d3cf5 100644 --- a/controllers/apps/systemaccount_util_test.go +++ b/controllers/apps/systemaccount_util_test.go @@ -146,20 +146,20 @@ func TestRenderJob(t *testing.T) { clusterDefName = "test-clusterdef" clusterVersionName = "test-clusterversion" clusterNamePrefix = "test-cluster" - mysqlCompType = "replicasets" + mysqlCompDefName = "replicasets" mysqlCompName = "mysql" ) systemAccount := mockSystemAccountsSpec() clusterDef := testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). AddSystemAccountSpec(systemAccount). GetObject() assert.NotNil(t, clusterDef) assert.NotNil(t, clusterDef.Spec.ComponentDefs[0].SystemAccounts) cluster := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDef.Name, clusterVersionName). - AddComponent(mysqlCompType, mysqlCompName).GetObject() + AddComponent(mysqlCompDefName, mysqlCompName).GetObject() assert.NotNil(t, cluster) if cluster.Annotations == nil { cluster.Annotations = make(map[string]string, 0) @@ -306,20 +306,20 @@ func TestAccountDebugMode(t *testing.T) { func TestRenderCreationStmt(t *testing.T) { var ( - clusterDefName = "test-clusterdef" - clusterName = "test-cluster" - mysqlCompType = "replicasets" - mysqlCompName = "mysql" + clusterDefName = "test-clusterdef" + clusterName = "test-cluster" + mysqlCompDefName = "replicasets" + mysqlCompName = "mysql" ) systemAccount := mockSystemAccountsSpec() clusterDef := testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). AddSystemAccountSpec(systemAccount). GetObject() assert.NotNil(t, clusterDef) - compDef := clusterDef.GetComponentDefByName(mysqlCompType) + compDef := clusterDef.GetComponentDefByName(mysqlCompDefName) assert.NotNil(t, compDef.SystemAccounts) accountsSetting := compDef.SystemAccounts diff --git a/controllers/apps/tls_utils_test.go b/controllers/apps/tls_utils_test.go index 6c02e6b30..e3899f621 100644 --- a/controllers/apps/tls_utils_test.go +++ b/controllers/apps/tls_utils_test.go @@ -38,13 +38,13 @@ import ( var _ = Describe("TLS self-signed cert function", func() { const ( - clusterDefName = "test-clusterdef-tls" - clusterVersionName = "test-clusterversion-tls" - clusterNamePrefix = "test-cluster" - statefulCompType = "replicasets" - statefulCompName = "mysql" - mysqlContainerName = "mysql" - configSpecName = "mysql-config-tpl" + clusterDefName = "test-clusterdef-tls" + clusterVersionName = "test-clusterversion-tls" + clusterNamePrefix = "test-cluster" + statefulCompDefName = "replicasets" + statefulCompName = "mysql" + mysqlContainerName = "mysql" + configSpecName = "mysql-config-tpl" ) ctx := context.Background() @@ -89,14 +89,14 @@ var _ = Describe("TLS self-signed cert function", func() { By("Create a clusterDef obj") testapps.NewClusterDefFactory(clusterDefName). SetConnectionCredential(map[string]string{"username": "root", "password": ""}, nil). - AddComponent(testapps.ConsensusMySQLComponent, statefulCompType). + AddComponentDef(testapps.ConsensusMySQLComponent, statefulCompDefName). AddConfigTemplate(configSpecName, configMapObj.Name, configConstraintObj.Name, testCtx.DefaultNamespace, testapps.ConfVolumeName). AddContainerEnv(mysqlContainerName, corev1.EnvVar{Name: "MYSQL_ALLOW_EMPTY_PASSWORD", Value: "yes"}). CheckedCreate(&testCtx).GetObject() By("Create a clusterVersion obj") testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(statefulCompType).AddContainerShort(mysqlContainerName, testapps.ApeCloudMySQLImage). + AddComponent(statefulCompDefName).AddContainerShort(mysqlContainerName, testapps.ApeCloudMySQLImage). CheckedCreate(&testCtx).GetObject() }) @@ -124,7 +124,7 @@ var _ = Describe("TLS self-signed cert function", func() { // clusterObj := testapps.NewClusterFactory(testCtx.DefaultNamespace, // clusterNamePrefix, clusterDefName, clusterVersionName). // WithRandomName(). - // AddComponent(statefulCompName, statefulCompType). + // AddComponentDef(statefulCompName, statefulCompDefName). // SetReplicas(3). // SetTLS(true). // SetIssuer(tlsIssuer). @@ -230,7 +230,7 @@ var _ = Describe("TLS self-signed cert function", func() { By("create cluster obj") clusterObj := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefName, clusterVersionName). WithRandomName(). - AddComponent(statefulCompName, statefulCompType). + AddComponent(statefulCompName, statefulCompDefName). SetReplicas(3). SetTLS(true). SetIssuer(tlsIssuer). @@ -258,7 +258,7 @@ var _ = Describe("TLS self-signed cert function", func() { // By("create cluster obj") // clusterObj := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefName, clusterVersionName). // WithRandomName(). - // AddComponent(statefulCompName, statefulCompType). + // AddComponentDef(statefulCompName, statefulCompDefName). // SetReplicas(3). // SetTLS(true). // SetIssuer(tlsIssuer). @@ -281,7 +281,7 @@ var _ = Describe("TLS self-signed cert function", func() { By("create cluster with tls disabled") clusterObj := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefName, clusterVersionName). WithRandomName(). - AddComponent(statefulCompName, statefulCompType). + AddComponent(statefulCompName, statefulCompDefName). SetReplicas(3). SetTLS(false). Create(&testCtx). diff --git a/deploy/redis/scripts/redis-sentinel-setup.sh.tpl b/deploy/redis/scripts/redis-sentinel-setup.sh.tpl index 440d5cec9..8f28c0fe9 100644 --- a/deploy/redis/scripts/redis-sentinel-setup.sh.tpl +++ b/deploy/redis/scripts/redis-sentinel-setup.sh.tpl @@ -20,10 +20,7 @@ set -ex {{- end }} {{- end }} {{- /* build primary pod message, because currently does not support cross-component acquisition of environment variables, the service of the redis master node is assembled here through specific rules */}} -{{- $primary_pod = printf "%s-%s-0.%s-%s-headless.%s.svc" $clusterName $redis_component.name $clusterName $redis_component.name $namespace }} -{{- if ne $primary_index 0 }} - {{- $primary_pod = printf "%s-%s-%d-0.%s-%s-headless.%s.svc" $clusterName $redis_component.name $primary_index $clusterName $redis_component.name $namespace }} -{{- end }} +{{- $primary_pod = printf "%s-%s-%d.%s-%s-headless.%s.svc" $clusterName $redis_component.name $primary_index $clusterName $redis_component.name $namespace }} {{- $sentinel_monitor := printf "%s-%s %s" $clusterName $sentinel_component.name $primary_pod }} cat>/etc/sentinel/redis-sentinel.conf<" for subsequent sts workload - if stsIndex != 0 { - sts.ObjectMeta.Name = fmt.Sprintf("%s-%d", sts.ObjectMeta.Name, stsIndex) - } - if stsIndex == task.Component.GetPrimaryIndex() { - sts.Labels[constant.RoleLabelKey] = string(replicationset.Primary) - } else { - sts.Labels[constant.RoleLabelKey] = string(replicationset.Secondary) - } sts.Spec.UpdateStrategy.Type = appsv1.OnDeleteStatefulSetStrategyType - // build replicationSet persistentVolumeClaim manually - if err := buildReplicationSetPVC(task, sts); err != nil { - return sts, err - } return sts, nil } -// buildReplicationSetPVC builds replicationSet persistentVolumeClaim manually, -// replicationSet does not manage pvc through volumeClaimTemplate defined on statefulSet, -// the purpose is convenient to convert between workloadTypes in the future (TODO). -func buildReplicationSetPVC(task *intctrltypes.ReconcileTask, sts *appsv1.StatefulSet) error { - // generate persistentVolumeClaim objects used by replicationSet's pod from component.VolumeClaimTemplates - // TODO: The pvc objects involved in all processes in the KubeBlocks will be reconstructed into a unified generation method - pvcMap := replicationset.GeneratePVCFromVolumeClaimTemplates(sts, task.Component.VolumeClaimTemplates) - for pvcTplName, pvc := range pvcMap { - builder.BuildPersistentVolumeClaimLabels(sts, pvc, task.Component, pvcTplName) - task.AppendResource(pvc) - } - - // binding persistentVolumeClaim to podSpec.Volumes - podSpec := &sts.Spec.Template.Spec - if podSpec == nil { - return nil - } - podVolumes := podSpec.Volumes - for _, pvc := range pvcMap { - volumeName := strings.Split(pvc.Name, "-")[0] - podVolumes, _ = intctrlutil.CreateOrUpdateVolume(podVolumes, volumeName, func(volumeName string) corev1.Volume { - return corev1.Volume{ - Name: volumeName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvc.Name, - }, - }, - } - }, nil) - } - podSpec.Volumes = podVolumes - return nil -} - // buildCfg generate volumes for PodTemplate, volumeMount for container, rendered configTemplate and scriptTemplate, // and generate configManager sidecar for the reconfigure operation. // TODO rename this function, this function name is not very reasonable, but there is no suitable name. diff --git a/internal/controller/plan/prepare_test.go b/internal/controller/plan/prepare_test.go index ae6861e16..95a155a95 100644 --- a/internal/controller/plan/prepare_test.go +++ b/internal/controller/plan/prepare_test.go @@ -34,6 +34,15 @@ import ( testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) +const ( + mysqlCompDefName = "replicasets" + mysqlCompName = "mysql" + nginxCompDefName = "nginx" + nginxCompName = "nginx" + redisCompDefName = "replicasets" + redisCompName = "redis" +) + var _ = Describe("Cluster Controller", func() { cleanEnv := func() { @@ -73,21 +82,17 @@ var _ = Describe("Cluster Controller", func() { ) Context("with Deployment workload", func() { - const ( - nginxCompType = "nginx" - nginxCompName = "nginx" - ) BeforeEach(func() { clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatelessNginxComponent, nginxCompType). + AddComponentDef(testapps.StatelessNginxComponent, nginxCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(nginxCompType). + AddComponent(nginxCompDefName). AddContainerShort("nginx", testapps.NginxImage). GetObject() cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDef.Name, clusterVersion.Name). - AddComponent(nginxCompType, nginxCompName). + AddComponent(nginxCompDefName, nginxCompName). GetObject() }) @@ -116,22 +121,18 @@ var _ = Describe("Cluster Controller", func() { }) Context("with Stateful workload and without config template", func() { - const ( - mysqlCompType = "replicasets" - mysqlCompName = "mysql" - ) BeforeEach(func() { clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompType). + AddComponent(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). GetObject() pvcSpec := testapps.NewPVCSpec("1Gi") cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDef.Name, clusterVersion.Name). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). GetObject() }) @@ -165,10 +166,6 @@ var _ = Describe("Cluster Controller", func() { }) Context("with Stateful workload and with config template", func() { - const ( - mysqlCompType = "replicasets" - mysqlCompName = "mysql" - ) BeforeEach(func() { cm := testapps.CreateCustomizedObj(&testCtx, "config/config-template.yaml", &corev1.ConfigMap{}, testCtx.UseDefaultNamespace()) @@ -177,17 +174,17 @@ var _ = Describe("Cluster Controller", func() { &appsv1alpha1.ConfigConstraint{}) clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). AddConfigTemplate(cm.Name, cm.Name, cfgTpl.Name, testCtx.DefaultNamespace, "mysql-config"). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompType). + AddComponent(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). GetObject() pvcSpec := testapps.NewPVCSpec("1Gi") cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDef.Name, clusterVersion.Name). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). GetObject() }) @@ -217,10 +214,6 @@ var _ = Describe("Cluster Controller", func() { }) Context("with Stateful workload and with config template and with config volume mount", func() { - const ( - mysqlCompType = "replicasets" - mysqlCompName = "mysql" - ) BeforeEach(func() { cm := testapps.CreateCustomizedObj(&testCtx, "config/config-template.yaml", &corev1.ConfigMap{}, testCtx.UseDefaultNamespace()) @@ -229,18 +222,18 @@ var _ = Describe("Cluster Controller", func() { &appsv1alpha1.ConfigConstraint{}) clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). AddConfigTemplate(cm.Name, cm.Name, cfgTpl.Name, testCtx.DefaultNamespace, "mysql-config"). AddContainerVolumeMounts("mysql", []corev1.VolumeMount{{Name: "mysql-config", MountPath: "/mnt/config"}}). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompType). + AddComponent(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). GetObject() pvcSpec := testapps.NewPVCSpec("1Gi") cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDef.Name, clusterVersion.Name). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). GetObject() }) @@ -278,12 +271,6 @@ var _ = Describe("Cluster Controller", func() { // for test GetContainerWithVolumeMount Context("with Consensus workload and with external service", func() { - const ( - mysqlCompType = "replicasets" - mysqlCompName = "mysql" - nginxCompType = "proxy" - ) - var ( clusterDef *appsv1alpha1.ClusterDefinition clusterVersion *appsv1alpha1.ClusterVersion @@ -292,19 +279,19 @@ var _ = Describe("Cluster Controller", func() { BeforeEach(func() { clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ConsensusMySQLComponent, mysqlCompType). - AddComponent(testapps.StatelessNginxComponent, nginxCompType). + AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). + AddComponentDef(testapps.StatelessNginxComponent, nginxCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompType). + AddComponent(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - AddComponent(nginxCompType). + AddComponent(nginxCompDefName). AddContainerShort("nginx", testapps.NginxImage). GetObject() pvcSpec := testapps.NewPVCSpec("1Gi") cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDef.Name, clusterVersion.Name). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). GetObject() }) @@ -335,12 +322,6 @@ var _ = Describe("Cluster Controller", func() { // for test GetContainerWithVolumeMount Context("with Replications workload without pvc", func() { - const ( - redisCompType = "replicasets" - redisCompName = "redis" - nginxCompType = "proxy" - ) - var ( clusterDef *appsv1alpha1.ClusterDefinition clusterVersion *appsv1alpha1.ClusterVersion @@ -349,24 +330,24 @@ var _ = Describe("Cluster Controller", func() { BeforeEach(func() { clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, redisCompType). - AddComponent(testapps.StatelessNginxComponent, nginxCompType). + AddComponentDef(testapps.ReplicationRedisComponent, redisCompDefName). + AddComponentDef(testapps.StatelessNginxComponent, nginxCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(redisCompType). + AddComponent(redisCompDefName). AddContainerShort("redis", testapps.DefaultRedisImageName). - AddComponent(nginxCompType). + AddComponent(nginxCompDefName). AddContainerShort("nginx", testapps.NginxImage). GetObject() cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDef.Name, clusterVersion.Name). - AddComponent(redisCompName, redisCompType). + AddComponent(redisCompName, redisCompDefName). SetReplicas(2). SetPrimaryIndex(0). GetObject() }) - It("should construct env, headless service, statefuset objects for each replica, besides an external service object", func() { + It("should construct env, headless service, statefuset object, besides an external service object", func() { reqCtx := intctrlutil.RequestCtx{ Ctx: ctx, Log: logger, @@ -382,79 +363,71 @@ var _ = Describe("Cluster Controller", func() { Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) resources := *task.Resources - Expect(len(resources)).Should(Equal(7)) + // REVIEW: (free6om) + // missing connection credential, TLS secret objs check? + Expect(resources).Should(HaveLen(4)) Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("Service")) Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("StatefulSet")) - Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[4]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[5]).String()).Should(ContainSubstring("StatefulSet")) - Expect(reflect.TypeOf(resources[6]).String()).Should(ContainSubstring("Service")) + Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("Service")) }) }) - // for test GetContainerWithVolumeMount - Context("with Replications workload with pvc", func() { - const ( - redisCompType = "replicasets" - redisCompName = "redis" - nginxCompType = "proxy" - ) - - var ( - clusterDef *appsv1alpha1.ClusterDefinition - clusterVersion *appsv1alpha1.ClusterVersion - cluster *appsv1alpha1.Cluster - ) - - BeforeEach(func() { - clusterDef = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, redisCompType). - AddComponent(testapps.StatelessNginxComponent, nginxCompType). - GetObject() - clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(redisCompType). - AddContainerShort("redis", testapps.DefaultRedisImageName). - AddComponent(nginxCompType). - AddContainerShort("nginx", testapps.NginxImage). - GetObject() - pvcSpec := testapps.NewPVCSpec("1Gi") - cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, - clusterDef.Name, clusterVersion.Name). - AddComponent(redisCompName, redisCompType). - SetReplicas(2). - SetPrimaryIndex(0). - AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - GetObject() - }) - - It("should construct pvc objects for each replica", func() { - reqCtx := intctrlutil.RequestCtx{ - Ctx: ctx, - Log: logger, - } - component := component.BuildComponent( - reqCtx, - *cluster, - *clusterDef, - clusterDef.Spec.ComponentDefs[0], - cluster.Spec.ComponentSpecs[0], - &clusterVersion.Spec.ComponentVersions[0]) - task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) - Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) - - resources := *task.Resources - Expect(len(resources)).Should(Equal(9)) - Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("PersistentVolumeClaim")) - Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("StatefulSet")) - Expect(reflect.TypeOf(resources[4]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[5]).String()).Should(ContainSubstring("PersistentVolumeClaim")) - Expect(reflect.TypeOf(resources[6]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[7]).String()).Should(ContainSubstring("StatefulSet")) - Expect(reflect.TypeOf(resources[8]).String()).Should(ContainSubstring("Service")) - }) - }) + // TODO: (free6om) + // uncomment following test case until pre-provisoned PVC work begin + // // for test GetContainerWithVolumeMount + // Context("with Replications workload with pvc", func() { + // var ( + // clusterDef *appsv1alpha1.ClusterDefinition + // clusterVersion *appsv1alpha1.ClusterVersion + // cluster *appsv1alpha1.Cluster + // ) + // + // BeforeEach(func() { + // clusterDef = testapps.NewClusterDefFactory(clusterDefName). + // AddComponentDef(testapps.ReplicationRedisComponent, redisCompDefName). + // AddComponentDef(testapps.StatelessNginxComponent, nginxCompDefName). + // GetObject() + // clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). + // AddComponent(redisCompDefName). + // AddContainerShort("redis", testapps.DefaultRedisImageName). + // AddComponent(nginxCompDefName). + // AddContainerShort("nginx", testapps.NginxImage). + // GetObject() + // pvcSpec := testapps.NewPVCSpec("1Gi") + // cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, + // clusterDef.Name, clusterVersion.Name). + // AddComponent(redisCompName, redisCompDefName). + // SetReplicas(2). + // SetPrimaryIndex(0). + // AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). + // GetObject() + // }) + // + // It("should construct pvc objects for each replica", func() { + // reqCtx := intctrlutil.RequestCtx{ + // Ctx: ctx, + // Log: logger, + // } + // component := component.BuildComponent( + // reqCtx, + // *cluster, + // *clusterDef, + // clusterDef.Spec.ComponentDefs[0], + // cluster.Spec.ComponentSpecs[0], + // &clusterVersion.Spec.ComponentVersions[0]) + // task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) + // Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) + // + // resources := *task.Resources + // Expect(resources).Should(HaveLen(6)) + // Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) + // Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("Service")) + // Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("PersistentVolumeClaim")) + // Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("PersistentVolumeClaim")) + // Expect(reflect.TypeOf(resources[4]).String()).Should(ContainSubstring("StatefulSet")) + // Expect(reflect.TypeOf(resources[5]).String()).Should(ContainSubstring("Service")) + // }) + // }) }) diff --git a/internal/testutil/apps/cluster_consensus_test_util.go b/internal/testutil/apps/cluster_consensus_test_util.go index 1f5c4a3a1..e5d207ff1 100644 --- a/internal/testutil/apps/cluster_consensus_test_util.go +++ b/internal/testutil/apps/cluster_consensus_test_util.go @@ -66,9 +66,9 @@ func CreateConsensusMysqlCluster( } // CreateConsensusMysqlClusterDef creates a mysql clusterDefinition with a component of ConsensusSet type. -func CreateConsensusMysqlClusterDef(testCtx testutil.TestContext, clusterDefName, workloadType string) *appsv1alpha1.ClusterDefinition { +func CreateConsensusMysqlClusterDef(testCtx testutil.TestContext, clusterDefName, componentDefName string) *appsv1alpha1.ClusterDefinition { filePathPattern := "/data/mysql/log/mysqld.err" - return NewClusterDefFactory(clusterDefName).AddComponent(ConsensusMySQLComponent, workloadType). + return NewClusterDefFactory(clusterDefName).AddComponentDef(ConsensusMySQLComponent, componentDefName). AddLogConfig(errorLogName, filePathPattern).Create(&testCtx).GetObject() } diff --git a/internal/testutil/apps/cluster_factory.go b/internal/testutil/apps/cluster_factory.go index d6b517243..04c3683d8 100644 --- a/internal/testutil/apps/cluster_factory.go +++ b/internal/testutil/apps/cluster_factory.go @@ -55,10 +55,10 @@ func (factory *MockClusterFactory) AddClusterToleration(toleration corev1.Tolera return factory } -func (factory *MockClusterFactory) AddComponent(compName string, compType string) *MockClusterFactory { +func (factory *MockClusterFactory) AddComponent(compName string, compDefName string) *MockClusterFactory { comp := appsv1alpha1.ClusterComponentSpec{ Name: compName, - ComponentDefRef: compType, + ComponentDefRef: compDefName, } factory.get().Spec.ComponentSpecs = append(factory.get().Spec.ComponentSpecs, comp) return factory diff --git a/internal/testutil/apps/cluster_replication_test_util.go b/internal/testutil/apps/cluster_replication_test_util.go index 924d4b4ee..45322b22c 100644 --- a/internal/testutil/apps/cluster_replication_test_util.go +++ b/internal/testutil/apps/cluster_replication_test_util.go @@ -32,6 +32,7 @@ import ( // MockReplicationComponentStsPod mocks to create pod of the replication StatefulSet, just using in envTest func MockReplicationComponentStsPod( + g gomega.Gomega, testCtx testutil.TestContext, sts *appsv1.StatefulSet, clusterName, @@ -54,20 +55,35 @@ func MockReplicationComponentStsPod( Status: corev1.ConditionTrue, }, } - gomega.Expect(testCtx.Cli.Status().Patch(context.Background(), pod, patch)).Should(gomega.Succeed()) + if g != nil { + g.Expect(testCtx.Cli.Status().Patch(context.Background(), pod, patch)).Should(gomega.Succeed()) + } else { + gomega.Expect(testCtx.Cli.Status().Patch(context.Background(), pod, patch)).Should(gomega.Succeed()) + } return pod } -// MockReplicationComponentPods mocks to create pods of the component, just using in envTest +// MockReplicationComponentPods mocks to create pods of the component, just using in envTest. If roleByIdx is empty, +// will have implicit pod-0 being "primary" role and others to "secondary" role. func MockReplicationComponentPods( + g gomega.Gomega, testCtx testutil.TestContext, sts *appsv1.StatefulSet, clusterName, compName string, - podRole string) []*corev1.Pod { + roleByIdx map[int32]string) []*corev1.Pod { + var pods []*corev1.Pod - podName := fmt.Sprintf("%s-0", sts.Name) - pods = append(pods, MockReplicationComponentStsPod(testCtx, sts, clusterName, compName, podName, podRole)) + for i := int32(0); i < *sts.Spec.Replicas; i++ { + podName := fmt.Sprintf("%s-%d", sts.Name, i) + role := "secondary" + if podRole, ok := roleByIdx[i]; ok && podRole != "" { + role = podRole + } else if i == 0 { + role = "primary" + } + pods = append(pods, MockReplicationComponentStsPod(g, testCtx, sts, clusterName, compName, podName, role)) + } return pods } diff --git a/internal/testutil/apps/cluster_util.go b/internal/testutil/apps/cluster_util.go index 94dc37d16..b45d355a7 100644 --- a/internal/testutil/apps/cluster_util.go +++ b/internal/testutil/apps/cluster_util.go @@ -35,23 +35,23 @@ func InitClusterWithHybridComps( clusterDefName, clusterVersionName, clusterName, - statelessComp, - statefulComp, - consensusComp string) (*appsv1alpha1.ClusterDefinition, *appsv1alpha1.ClusterVersion, *appsv1alpha1.Cluster) { + statelessCompDefName, + statefulCompDefName, + consensusCompDefName string) (*appsv1alpha1.ClusterDefinition, *appsv1alpha1.ClusterVersion, *appsv1alpha1.Cluster) { clusterDef := NewClusterDefFactory(clusterDefName). - AddComponent(StatelessNginxComponent, statelessComp). - AddComponent(ConsensusMySQLComponent, consensusComp). - AddComponent(StatefulMySQLComponent, statefulComp). + AddComponentDef(StatelessNginxComponent, statelessCompDefName). + AddComponentDef(ConsensusMySQLComponent, consensusCompDefName). + AddComponentDef(StatefulMySQLComponent, statefulCompDefName). Create(&testCtx).GetObject() clusterVersion := NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(statelessComp).AddContainerShort(DefaultNginxContainerName, NginxImage). - AddComponent(consensusComp).AddContainerShort(DefaultMySQLContainerName, NginxImage). - AddComponent(statefulComp).AddContainerShort(DefaultMySQLContainerName, NginxImage). + AddComponent(statelessCompDefName).AddContainerShort(DefaultNginxContainerName, NginxImage). + AddComponent(consensusCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). + AddComponent(statefulCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). Create(&testCtx).GetObject() cluster := NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(statelessComp, statelessComp).SetReplicas(1). - AddComponent(consensusComp, consensusComp).SetReplicas(3). - AddComponent(statefulComp, statefulComp).SetReplicas(3). + AddComponent(statelessCompDefName, statelessCompDefName).SetReplicas(1). + AddComponent(consensusCompDefName, consensusCompDefName).SetReplicas(3). + AddComponent(statefulCompDefName, statefulCompDefName).SetReplicas(3). Create(&testCtx).GetObject() return clusterDef, clusterVersion, cluster } diff --git a/internal/testutil/apps/clusterdef_factory.go b/internal/testutil/apps/clusterdef_factory.go index 7c8add98f..35ae0f550 100644 --- a/internal/testutil/apps/clusterdef_factory.go +++ b/internal/testutil/apps/clusterdef_factory.go @@ -22,13 +22,13 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" ) -type ComponentTplType string +type ComponentDefTplType string const ( - StatefulMySQLComponent ComponentTplType = "stateful-mysql" - ConsensusMySQLComponent ComponentTplType = "consensus-mysql" - ReplicationRedisComponent ComponentTplType = "replication-redis" - StatelessNginxComponent ComponentTplType = "stateless-nginx" + StatefulMySQLComponent ComponentDefTplType = "stateful-mysql" + ConsensusMySQLComponent ComponentDefTplType = "consensus-mysql" + ReplicationRedisComponent ComponentDefTplType = "replication-redis" + StatelessNginxComponent ComponentDefTplType = "stateless-nginx" ) type MockClusterDefFactory struct { @@ -49,12 +49,12 @@ func NewClusterDefFactory(name string) *MockClusterDefFactory { func NewClusterDefFactoryWithConnCredential(name string) *MockClusterDefFactory { f := NewClusterDefFactory(name) - f.AddComponent(StatefulMySQLComponent, "conn-cred") + f.AddComponentDef(StatefulMySQLComponent, "conn-cred") f.SetConnectionCredential(defaultConnectionCredential, &defaultSvcSpec) return f } -func (factory *MockClusterDefFactory) AddComponent(tplType ComponentTplType, newName string) *MockClusterDefFactory { +func (factory *MockClusterDefFactory) AddComponentDef(tplType ComponentDefTplType, compDefName string) *MockClusterDefFactory { var component *appsv1alpha1.ClusterComponentDefinition switch tplType { case StatefulMySQLComponent: @@ -68,7 +68,7 @@ func (factory *MockClusterDefFactory) AddComponent(tplType ComponentTplType, new } factory.get().Spec.ComponentDefs = append(factory.get().Spec.ComponentDefs, *component) comp := factory.getLastCompDef() - comp.Name = newName + comp.Name = compDefName return factory } diff --git a/internal/testutil/apps/clusterversion_factory.go b/internal/testutil/apps/clusterversion_factory.go index 3f506d557..e25d737cd 100644 --- a/internal/testutil/apps/clusterversion_factory.go +++ b/internal/testutil/apps/clusterversion_factory.go @@ -38,9 +38,9 @@ func NewClusterVersionFactory(name, cdRef string) *MockClusterVersionFactory { return f } -func (factory *MockClusterVersionFactory) AddComponent(compType string) *MockClusterVersionFactory { +func (factory *MockClusterVersionFactory) AddComponent(compDefName string) *MockClusterVersionFactory { comp := appsv1alpha1.ClusterComponentVersion{ - ComponentDefRef: compType, + ComponentDefRef: compDefName, } factory.get().Spec.ComponentVersions = append(factory.get().Spec.ComponentVersions, comp) return factory diff --git a/internal/testutil/apps/common_util.go b/internal/testutil/apps/common_util.go index 19d91b701..8658dfa5a 100644 --- a/internal/testutil/apps/common_util.go +++ b/internal/testutil/apps/common_util.go @@ -61,9 +61,9 @@ func ResetToIgnoreFinalizers() { // })).Should(Succeed()) func ChangeObj[T intctrlutil.Object, PT intctrlutil.PObject[T]](testCtx *testutil.TestContext, - pobj PT, action func()) error { + pobj PT, action func(PT)) error { patch := client.MergeFrom(PT(pobj.DeepCopy())) - action() + action(pobj) return testCtx.Cli.Patch(testCtx.Ctx, pobj, patch) } @@ -95,7 +95,9 @@ func GetAndChangeObj[T intctrlutil.Object, PT intctrlutil.PObject[T]]( if err := testCtx.Cli.Get(testCtx.Ctx, namespacedName, pobj); err != nil { return err } - return ChangeObj(testCtx, pobj, func() { action(pobj) }) + return ChangeObj(testCtx, pobj, func(lobj PT) { + action(lobj) + }) } } @@ -322,7 +324,7 @@ func ClearResourcesWithRemoveFinalizerOption[T intctrlutil.Object, PT intctrluti finalizers := pobj.GetFinalizers() if len(finalizers) > 0 { if removeFinalizer { - g.Expect(ChangeObj(testCtx, pobj, func() { + g.Expect(ChangeObj(testCtx, pobj, func(lobj PT) { pobj.SetFinalizers([]string{}) })).To(gomega.Succeed()) } else { diff --git a/internal/testutil/apps/constant.go b/internal/testutil/apps/constant.go index 17cbafd5a..b8ca81e63 100644 --- a/internal/testutil/apps/constant.go +++ b/internal/testutil/apps/constant.go @@ -48,7 +48,7 @@ const ( DefaultNginxContainerName = "nginx" RedisType = "state.redis" - DefaultRedisCompType = "redis" + DefaultRedisCompDefName = "redis" DefaultRedisCompName = "redis-rsts" DefaultRedisImageName = "redis:7.0.5" DefaultRedisContainerName = "redis" diff --git a/internal/testutil/apps/native_object_util.go b/internal/testutil/apps/native_object_util.go index e1c07ff1b..daa08adc1 100644 --- a/internal/testutil/apps/native_object_util.go +++ b/internal/testutil/apps/native_object_util.go @@ -65,7 +65,8 @@ func NewPVC(size string) corev1.PersistentVolumeClaimSpec { } } -func CreateStorageClass(testCtx testutil.TestContext, storageClassName string, allowVolumeExpansion bool) *storagev1.StorageClass { +func CreateStorageClass(testCtx testutil.TestContext, storageClassName string, + allowVolumeExpansion bool) *storagev1.StorageClass { storageClass := &storagev1.StorageClass{ ObjectMeta: metav1.ObjectMeta{ Name: storageClassName, diff --git a/test/integration/backup_mysql_test.go b/test/integration/backup_mysql_test.go index 24a407991..88d11d25d 100644 --- a/test/integration/backup_mysql_test.go +++ b/test/integration/backup_mysql_test.go @@ -36,10 +36,8 @@ var _ = Describe("MySQL data protection function", func() { const clusterVersionName = "test-clusterversion" const clusterNamePrefix = "test-cluster" const scriptConfigName = "test-cluster-mysql-scripts" - - const mysqlCompType = "replicasets" + const mysqlCompDefName = "replicasets" const mysqlCompName = "mysql" - const backupPolicyTemplateName = "test-backup-policy-template" const backupPolicyName = "test-backup-policy" const backupRemoteVolumeName = "backup-remote-volume" @@ -92,13 +90,13 @@ var _ = Describe("MySQL data protection function", func() { By("Create a clusterDef obj") mode := int32(0755) clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ConsensusMySQLComponent, mysqlCompType). + AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponent(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() By("Create a cluster obj") @@ -106,7 +104,7 @@ var _ = Describe("MySQL data protection function", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). SetReplicas(1). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). Create(&testCtx).GetObject() diff --git a/test/integration/controller_suite_test.go b/test/integration/controller_suite_test.go index 1a922b817..69c7902bf 100644 --- a/test/integration/controller_suite_test.go +++ b/test/integration/controller_suite_test.go @@ -127,18 +127,14 @@ func CreateSimpleConsensusMySQLClusterWithConfig( mysqlConfigConstraintPath, mysqlScriptsPath string) ( *appsv1alpha1.ClusterDefinition, *appsv1alpha1.ClusterVersion, *appsv1alpha1.Cluster) { - const mysqlCompName = "mysql" - const mysqlCompType = "mysql" - + const mysqlCompDefName = "mysql" const mysqlConfigName = "mysql-component-config" const mysqlConfigConstraintName = "mysql8.0-config-constraints" const mysqlScriptsConfigName = "apecloud-mysql-scripts" - const mysqlDataVolumeName = "data" const mysqlConfigVolumeName = "mysql-config" const mysqlScriptsVolumeName = "scripts" - const mysqlErrorFilePath = "/data/mysql/log/mysqld-error.log" const mysqlGeneralFilePath = "/data/mysql/log/mysqld.log" const mysqlSlowlogFilePath = "/data/mysql/log/mysqld-slowquery.log" @@ -185,7 +181,7 @@ func CreateSimpleConsensusMySQLClusterWithConfig( mode := int32(0755) clusterDefObj := testapps.NewClusterDefFactory(clusterDefName). SetConnectionCredential(map[string]string{"username": "root", "password": ""}, nil). - AddComponent(testapps.ConsensusMySQLComponent, mysqlCompType). + AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). AddConfigTemplate(mysqlConfigName, configmap.Name, constraint.Name, testCtx.DefaultNamespace, mysqlConfigVolumeName). AddScriptTemplate(mysqlScriptsConfigName, mysqlScriptsConfigName, @@ -203,7 +199,7 @@ func CreateSimpleConsensusMySQLClusterWithConfig( By("Create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType). + AddComponent(mysqlCompDefName). AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(mysqlConfigName), configmap.Name, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). @@ -220,7 +216,7 @@ func CreateSimpleConsensusMySQLClusterWithConfig( } clusterObj := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). SetReplicas(3). SetEnabledLogs("error", "general", "slow"). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). diff --git a/test/integration/mysql_ha_test.go b/test/integration/mysql_ha_test.go index 94930d643..011fd11c9 100644 --- a/test/integration/mysql_ha_test.go +++ b/test/integration/mysql_ha_test.go @@ -41,10 +41,8 @@ var _ = Describe("MySQL High-Availability function", func() { const clusterVersionName = "test-clusterversion" const clusterNamePrefix = "test-cluster" const scriptConfigName = "test-cluster-mysql-scripts" - - const mysqlCompType = "replicasets" + const mysqlCompDefName = "replicasets" const mysqlCompName = "mysql" - const leader = "leader" const follower = "follower" @@ -112,7 +110,7 @@ var _ = Describe("MySQL High-Availability function", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). SetReplicas(3).AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) @@ -193,14 +191,14 @@ var _ = Describe("MySQL High-Availability function", func() { mode := int32(0755) clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). SetConnectionCredential(map[string]string{"username": "root", "password": ""}, nil). - AddComponent(testapps.ConsensusMySQLComponent, mysqlCompType). + AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). AddContainerEnv(testapps.DefaultMySQLContainerName, corev1.EnvVar{Name: "MYSQL_ALLOW_EMPTY_PASSWORD", Value: "yes"}). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponent(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) diff --git a/test/integration/mysql_scale_test.go b/test/integration/mysql_scale_test.go index 2cfe6bf2b..b267b364a 100644 --- a/test/integration/mysql_scale_test.go +++ b/test/integration/mysql_scale_test.go @@ -41,8 +41,7 @@ var _ = Describe("MySQL Scaling function", func() { const clusterVersionName = "test-clusterversion" const clusterNamePrefix = "test-cluster" const scriptConfigName = "test-cluster-mysql-scripts" - - const mysqlCompType = "replicasets" + const mysqlCompDefName = "replicasets" const mysqlCompName = "mysql" // Cleanups @@ -93,7 +92,7 @@ var _ = Describe("MySQL Scaling function", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). SetResources(resources).SetReplicas(1). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) @@ -135,8 +134,8 @@ var _ = Describe("MySQL Scaling function", func() { })).Should(Succeed()) By("check OpsRequest reclaimed after ttl") - Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func() { - verticalScalingOpsRequest.Spec.TTLSecondsAfterSucceed = 1 + Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func(lopsReq *appsv1alpha1.OpsRequest) { + lopsReq.Spec.TTLSecondsAfterSucceed = 1 })).Should(Succeed()) By("OpsRequest reclaimed after ttl") @@ -163,7 +162,7 @@ var _ = Describe("MySQL Scaling function", func() { logPvcSpec.StorageClassName = &defaultStorageClass.Name clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(mysqlCompName, mysqlCompType). + AddComponent(mysqlCompName, mysqlCompDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, dataPvcSpec). AddVolumeClaimTemplate(testapps.LogVolumeName, logPvcSpec). Create(&testCtx).GetObject() @@ -226,13 +225,13 @@ var _ = Describe("MySQL Scaling function", func() { By("Create a clusterDef obj") mode := int32(0755) clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponent(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) @@ -254,13 +253,13 @@ var _ = Describe("MySQL Scaling function", func() { By("Create a clusterDef obj") mode := int32(0755) clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ConsensusMySQLComponent, mysqlCompType). + AddComponentDef(testapps.ConsensusMySQLComponent, mysqlCompDefName). AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). Create(&testCtx).GetObject() By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompType).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponent(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) diff --git a/test/integration/redis_hscale_test.go b/test/integration/redis_hscale_test.go index 5fec39ab2..d57f60d70 100644 --- a/test/integration/redis_hscale_test.go +++ b/test/integration/redis_hscale_test.go @@ -91,7 +91,7 @@ var _ = Describe("Redis Horizontal Scale function", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). SetReplicas(replicas).AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). Create(&testCtx).GetObject() @@ -145,8 +145,8 @@ var _ = Describe("Redis Horizontal Scale function", func() { for _, newReplicas := range []int32{4, 2, 7, 1} { By(fmt.Sprintf("horizontal scale out to %d", newReplicas)) - Expect(testapps.ChangeObj(&testCtx, clusterObj, func() { - clusterObj.Spec.ComponentSpecs[0].Replicas = newReplicas + Expect(testapps.ChangeObj(&testCtx, clusterObj, func(lcluster *appsv1alpha1.Cluster) { + lcluster.Spec.ComponentSpecs[0].Replicas = newReplicas })).Should(Succeed()) By("Wait for the cluster to be running") @@ -191,7 +191,7 @@ var _ = Describe("Redis Horizontal Scale function", func() { By("Create a clusterDefinition obj with replication workloadType.") mode := int32(0755) clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponent(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompType). + AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). AddConfigTemplate(primaryConfigName, primaryConfigName, "", testCtx.DefaultNamespace, string(replicationset.Primary)). AddConfigTemplate(secondaryConfigName, secondaryConfigName, "", testCtx.DefaultNamespace, string(replicationset.Secondary)). @@ -201,7 +201,7 @@ var _ = Describe("Redis Horizontal Scale function", func() { By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(testapps.DefaultRedisCompType). + AddComponent(testapps.DefaultRedisCompDefName). AddInitContainerShort(testapps.DefaultRedisInitContainerName, testapps.DefaultRedisImageName). AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() From 7eb7b35f7697d64ffc3de4e237a66977f20a7edb Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Mon, 17 Apr 2023 08:04:48 +0800 Subject: [PATCH 042/439] chore: refactore controllers/apps/components/* packages and structs naming (#2618) --- controllers/apps/cluster_controller_test.go | 4 +-- controllers/apps/components/component.go | 8 +++--- .../consensus.go} | 26 ++++++++--------- .../consensus_test.go | 2 +- .../consensus_utils.go} | 2 +- .../consensus_utils_test.go} | 2 +- .../{consensusset => consensus}/suite_test.go | 4 +-- .../replication.go} | 28 +++++++++---------- .../replication_switch.go} | 2 +- .../replication_switch_test.go} | 2 +- .../replication_switch_utils.go} | 2 +- .../replication_switch_utils_test.go} | 2 +- .../replication_test.go} | 2 +- .../replication_utils.go} | 2 +- .../replication_utils_test.go} | 2 +- .../suite_test.go | 2 +- .../apps/components/stateful/stateful.go | 18 ++++++------ .../apps/components/stateless/stateless.go | 18 ++++++------ controllers/apps/configuration/policy_util.go | 4 +-- .../apps/opsrequest_controller_test.go | 4 +-- controllers/k8score/event_controller.go | 8 +++--- internal/controller/plan/prepare.go | 8 +++--- test/integration/redis_hscale_test.go | 18 ++++++------ 23 files changed, 85 insertions(+), 85 deletions(-) rename controllers/apps/components/{consensusset/consensus_set.go => consensus/consensus.go} (90%) rename controllers/apps/components/{consensusset => consensus}/consensus_test.go (99%) rename controllers/apps/components/{consensusset/consensus_set_utils.go => consensus/consensus_utils.go} (99%) rename controllers/apps/components/{consensusset/consensus_set_utils_test.go => consensus/consensus_utils_test.go} (99%) rename controllers/apps/components/{consensusset => consensus}/suite_test.go (97%) rename controllers/apps/components/{replicationset/replication_set.go => replication/replication.go} (87%) rename controllers/apps/components/{replicationset/replication_set_switch.go => replication/replication_switch.go} (99%) rename controllers/apps/components/{replicationset/replication_set_switch_test.go => replication/replication_switch_test.go} (99%) rename controllers/apps/components/{replicationset/replication_set_switch_utils.go => replication/replication_switch_utils.go} (99%) rename controllers/apps/components/{replicationset/replication_set_switch_utils_test.go => replication/replication_switch_utils_test.go} (99%) rename controllers/apps/components/{replicationset/replication_set_test.go => replication/replication_test.go} (99%) rename controllers/apps/components/{replicationset/replication_set_utils.go => replication/replication_utils.go} (99%) rename controllers/apps/components/{replicationset/replication_set_utils_test.go => replication/replication_utils_test.go} (99%) rename controllers/apps/components/{replicationset => replication}/suite_test.go (99%) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 676aad584..816175982 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -42,7 +42,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/replicationset" + "github.com/apecloud/kubeblocks/controllers/apps/components/replication" "github.com/apecloud/kubeblocks/controllers/apps/components/util" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/lifecycle" @@ -1479,7 +1479,7 @@ var _ = Describe("Cluster Controller", func() { for i := int32(0); i < *sts.Spec.Replicas; i++ { podName := fmt.Sprintf("%s-%d", sts.Name, i) testapps.MockReplicationComponentStsPod(nil, testCtx, sts, clusterObj.Name, - testapps.DefaultRedisCompName, podName, replicationset.DefaultRole(i)) + testapps.DefaultRedisCompName, podName, replication.DefaultRole(i)) } Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) }) diff --git a/controllers/apps/components/component.go b/controllers/apps/components/component.go index 6115c67ca..8e26347af 100644 --- a/controllers/apps/components/component.go +++ b/controllers/apps/components/component.go @@ -26,8 +26,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/consensusset" - "github.com/apecloud/kubeblocks/controllers/apps/components/replicationset" + "github.com/apecloud/kubeblocks/controllers/apps/components/consensus" + "github.com/apecloud/kubeblocks/controllers/apps/components/replication" "github.com/apecloud/kubeblocks/controllers/apps/components/stateful" "github.com/apecloud/kubeblocks/controllers/apps/components/stateless" "github.com/apecloud/kubeblocks/controllers/apps/components/types" @@ -59,9 +59,9 @@ func NewComponentByType( } switch componentDef.WorkloadType { case appsv1alpha1.Consensus: - return consensusset.NewConsensusComponent(cli, cluster, component, componentDef) + return consensus.NewConsensusComponent(cli, cluster, component, componentDef) case appsv1alpha1.Replication: - return replicationset.NewReplicationComponent(cli, cluster, component, componentDef) + return replication.NewReplicationComponent(cli, cluster, component, componentDef) case appsv1alpha1.Stateful: return stateful.NewStatefulComponent(cli, cluster, component, componentDef) case appsv1alpha1.Stateless: diff --git a/controllers/apps/components/consensusset/consensus_set.go b/controllers/apps/components/consensus/consensus.go similarity index 90% rename from controllers/apps/components/consensusset/consensus_set.go rename to controllers/apps/components/consensus/consensus.go index a97e73cbc..3efcd5693 100644 --- a/controllers/apps/components/consensusset/consensus_set.go +++ b/controllers/apps/components/consensus/consensus.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package consensusset +package consensus import ( "context" @@ -34,13 +34,13 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -type ConsensusSet struct { - stateful.Stateful +type ConsensusComponent struct { + stateful.StatefulComponent } -var _ types.Component = &ConsensusSet{} +var _ types.Component = &ConsensusComponent{} -func (r *ConsensusSet) IsRunning(ctx context.Context, obj client.Object) (bool, error) { +func (r *ConsensusComponent) IsRunning(ctx context.Context, obj client.Object) (bool, error) { if obj == nil { return false, nil } @@ -62,18 +62,18 @@ func (r *ConsensusSet) IsRunning(ctx context.Context, obj client.Object) (bool, return util.StatefulSetOfComponentIsReady(sts, isRevisionConsistent, &r.Component.Replicas), nil } -func (r *ConsensusSet) PodsReady(ctx context.Context, obj client.Object) (bool, error) { - return r.Stateful.PodsReady(ctx, obj) +func (r *ConsensusComponent) PodsReady(ctx context.Context, obj client.Object) (bool, error) { + return r.StatefulComponent.PodsReady(ctx, obj) } -func (r *ConsensusSet) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { +func (r *ConsensusComponent) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { if pod == nil { return false } return intctrlutil.PodIsReadyWithLabel(*pod) } -func (r *ConsensusSet) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { +func (r *ConsensusComponent) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { var ( compStatus appsv1alpha1.ClusterComponentStatus ok bool @@ -134,7 +134,7 @@ func (r *ConsensusSet) HandleProbeTimeoutWhenPodsReady(ctx context.Context, reco return false, opsutil.MarkRunningOpsRequestAnnotation(ctx, r.Cli, cluster) } -func (r *ConsensusSet) GetPhaseWhenPodsNotReady(ctx context.Context, +func (r *ConsensusComponent) GetPhaseWhenPodsNotReady(ctx context.Context, componentName string) (appsv1alpha1.ClusterComponentPhase, error) { stsList := &appsv1.StatefulSetList{} podList, err := util.GetCompRelatedObjectList(ctx, r.Cli, *r.Cluster, @@ -172,7 +172,7 @@ func (r *ConsensusSet) GetPhaseWhenPodsNotReady(ctx context.Context, componentReplicas, int32(podCount), stsObj.Status.AvailableReplicas), nil } -func (r *ConsensusSet) HandleUpdate(ctx context.Context, obj client.Object) error { +func (r *ConsensusComponent) HandleUpdate(ctx context.Context, obj client.Object) error { if r == nil { return nil } @@ -263,8 +263,8 @@ func NewConsensusComponent( if err := util.ComponentRuntimeReqArgsCheck(cli, cluster, component); err != nil { return nil, err } - return &ConsensusSet{ - Stateful: stateful.Stateful{ + return &ConsensusComponent{ + StatefulComponent: stateful.StatefulComponent{ Cli: cli, Cluster: cluster, Component: component, diff --git a/controllers/apps/components/consensusset/consensus_test.go b/controllers/apps/components/consensus/consensus_test.go similarity index 99% rename from controllers/apps/components/consensusset/consensus_test.go rename to controllers/apps/components/consensus/consensus_test.go index efd51bc17..45c53b3d0 100644 --- a/controllers/apps/components/consensusset/consensus_test.go +++ b/controllers/apps/components/consensus/consensus_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package consensusset +package consensus import ( "fmt" diff --git a/controllers/apps/components/consensusset/consensus_set_utils.go b/controllers/apps/components/consensus/consensus_utils.go similarity index 99% rename from controllers/apps/components/consensusset/consensus_set_utils.go rename to controllers/apps/components/consensus/consensus_utils.go index 39bf14857..55820c00c 100644 --- a/controllers/apps/components/consensusset/consensus_set_utils.go +++ b/controllers/apps/components/consensus/consensus_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package consensusset +package consensus import ( "context" diff --git a/controllers/apps/components/consensusset/consensus_set_utils_test.go b/controllers/apps/components/consensus/consensus_utils_test.go similarity index 99% rename from controllers/apps/components/consensusset/consensus_set_utils_test.go rename to controllers/apps/components/consensus/consensus_utils_test.go index 583a49cef..90aeaa29a 100644 --- a/controllers/apps/components/consensusset/consensus_set_utils_test.go +++ b/controllers/apps/components/consensus/consensus_utils_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package consensusset +package consensus import ( "strconv" diff --git a/controllers/apps/components/consensusset/suite_test.go b/controllers/apps/components/consensus/suite_test.go similarity index 97% rename from controllers/apps/components/consensusset/suite_test.go rename to controllers/apps/components/consensus/suite_test.go index e48fe879e..5f686bc8a 100644 --- a/controllers/apps/components/consensusset/suite_test.go +++ b/controllers/apps/components/consensus/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package consensusset +package consensus import ( "context" @@ -55,7 +55,7 @@ func init() { func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) - RunSpecs(t, "ConsensusSet Controller Suite") + RunSpecs(t, "Consensus Controller Suite") } var _ = BeforeSuite(func() { diff --git a/controllers/apps/components/replicationset/replication_set.go b/controllers/apps/components/replication/replication.go similarity index 87% rename from controllers/apps/components/replicationset/replication_set.go rename to controllers/apps/components/replication/replication.go index fb7df0e85..6a325be57 100644 --- a/controllers/apps/components/replicationset/replication_set.go +++ b/controllers/apps/components/replication/replication.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( "context" @@ -33,16 +33,16 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -// ReplicationSet is a component object used by Cluster, ClusterComponentDefinition and ClusterComponentSpec -type ReplicationSet struct { - stateful.Stateful +// ReplicationComponent is a component object used by Cluster, ClusterComponentDefinition and ClusterComponentSpec +type ReplicationComponent struct { + stateful.StatefulComponent } -var _ types.Component = &ReplicationSet{} +var _ types.Component = &ReplicationComponent{} // IsRunning is the implementation of the type Component interface method, // which is used to check whether the replicationSet component is running normally. -func (r *ReplicationSet) IsRunning(ctx context.Context, obj client.Object) (bool, error) { +func (r *ReplicationComponent) IsRunning(ctx context.Context, obj client.Object) (bool, error) { var componentStatusIsRunning = true sts := util.ConvertToStatefulSet(obj) isRevisionConsistent, err := util.IsStsAndPodsRevisionConsistent(ctx, r.Cli, sts) @@ -61,13 +61,13 @@ func (r *ReplicationSet) IsRunning(ctx context.Context, obj client.Object) (bool // PodsReady is the implementation of the type Component interface method, // which is used to check whether all the pods of replicationSet component is ready. -func (r *ReplicationSet) PodsReady(ctx context.Context, obj client.Object) (bool, error) { - return r.Stateful.PodsReady(ctx, obj) +func (r *ReplicationComponent) PodsReady(ctx context.Context, obj client.Object) (bool, error) { + return r.StatefulComponent.PodsReady(ctx, obj) } // PodIsAvailable is the implementation of the type Component interface method, // Check whether the status of a Pod of the replicationSet is ready, including the role label on the Pod -func (r *ReplicationSet) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { +func (r *ReplicationComponent) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { if pod == nil { return false } @@ -76,14 +76,14 @@ func (r *ReplicationSet) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) // HandleProbeTimeoutWhenPodsReady is the implementation of the type Component interface method, // and replicationSet does not need to do role probe detection, so it returns false directly. -func (r *ReplicationSet) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { +func (r *ReplicationComponent) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { return false, nil } // GetPhaseWhenPodsNotReady is the implementation of the type Component interface method, // when the pods of replicationSet are not ready, calculate the component phase is Failed or Abnormal. // if return an empty phase, means the pods of component are ready and skips it. -func (r *ReplicationSet) GetPhaseWhenPodsNotReady(ctx context.Context, +func (r *ReplicationComponent) GetPhaseWhenPodsNotReady(ctx context.Context, componentName string) (appsv1alpha1.ClusterComponentPhase, error) { stsList := &appsv1.StatefulSetList{} podList, err := util.GetCompRelatedObjectList(ctx, r.Cli, *r.Cluster, @@ -136,7 +136,7 @@ func (r *ReplicationSet) GetPhaseWhenPodsNotReady(ctx context.Context, } // HandleUpdate is the implementation of the type Component interface method, handles replicationSet workload Pod updates. -func (r *ReplicationSet) HandleUpdate(ctx context.Context, obj client.Object) error { +func (r *ReplicationComponent) HandleUpdate(ctx context.Context, obj client.Object) error { sts := util.ConvertToStatefulSet(obj) if sts.Generation != sts.Status.ObservedGeneration { return nil @@ -193,8 +193,8 @@ func NewReplicationComponent( if err := util.ComponentRuntimeReqArgsCheck(cli, cluster, component); err != nil { return nil, err } - return &ReplicationSet{ - Stateful: stateful.Stateful{ + return &ReplicationComponent{ + StatefulComponent: stateful.StatefulComponent{ Cli: cli, Cluster: cluster, Component: component, diff --git a/controllers/apps/components/replicationset/replication_set_switch.go b/controllers/apps/components/replication/replication_switch.go similarity index 99% rename from controllers/apps/components/replicationset/replication_set_switch.go rename to controllers/apps/components/replication/replication_switch.go index 3723cd61c..09cca1991 100644 --- a/controllers/apps/components/replicationset/replication_set_switch.go +++ b/controllers/apps/components/replication/replication_switch.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( "context" diff --git a/controllers/apps/components/replicationset/replication_set_switch_test.go b/controllers/apps/components/replication/replication_switch_test.go similarity index 99% rename from controllers/apps/components/replicationset/replication_set_switch_test.go rename to controllers/apps/components/replication/replication_switch_test.go index fa4a5ab23..8552b7e11 100644 --- a/controllers/apps/components/replicationset/replication_set_switch_test.go +++ b/controllers/apps/components/replication/replication_switch_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( "fmt" diff --git a/controllers/apps/components/replicationset/replication_set_switch_utils.go b/controllers/apps/components/replication/replication_switch_utils.go similarity index 99% rename from controllers/apps/components/replicationset/replication_set_switch_utils.go rename to controllers/apps/components/replication/replication_switch_utils.go index dd7ce19a3..e3b73a535 100644 --- a/controllers/apps/components/replicationset/replication_set_switch_utils.go +++ b/controllers/apps/components/replication/replication_switch_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( "context" diff --git a/controllers/apps/components/replicationset/replication_set_switch_utils_test.go b/controllers/apps/components/replication/replication_switch_utils_test.go similarity index 99% rename from controllers/apps/components/replicationset/replication_set_switch_utils_test.go rename to controllers/apps/components/replication/replication_switch_utils_test.go index a437bcebe..d8fe0e303 100644 --- a/controllers/apps/components/replicationset/replication_set_switch_utils_test.go +++ b/controllers/apps/components/replication/replication_switch_utils_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( "fmt" diff --git a/controllers/apps/components/replicationset/replication_set_test.go b/controllers/apps/components/replication/replication_test.go similarity index 99% rename from controllers/apps/components/replicationset/replication_set_test.go rename to controllers/apps/components/replication/replication_test.go index fef095806..7a43c13fa 100644 --- a/controllers/apps/components/replicationset/replication_set_test.go +++ b/controllers/apps/components/replication/replication_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( . "github.com/onsi/ginkgo/v2" diff --git a/controllers/apps/components/replicationset/replication_set_utils.go b/controllers/apps/components/replication/replication_utils.go similarity index 99% rename from controllers/apps/components/replicationset/replication_set_utils.go rename to controllers/apps/components/replication/replication_utils.go index 6e1521388..f411b5f68 100644 --- a/controllers/apps/components/replicationset/replication_set_utils.go +++ b/controllers/apps/components/replication/replication_utils.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( "context" diff --git a/controllers/apps/components/replicationset/replication_set_utils_test.go b/controllers/apps/components/replication/replication_utils_test.go similarity index 99% rename from controllers/apps/components/replicationset/replication_set_utils_test.go rename to controllers/apps/components/replication/replication_utils_test.go index c502c03ab..8e7eebcef 100644 --- a/controllers/apps/components/replicationset/replication_set_utils_test.go +++ b/controllers/apps/components/replication/replication_utils_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( "context" diff --git a/controllers/apps/components/replicationset/suite_test.go b/controllers/apps/components/replication/suite_test.go similarity index 99% rename from controllers/apps/components/replicationset/suite_test.go rename to controllers/apps/components/replication/suite_test.go index 8e8d7e579..0d610b790 100644 --- a/controllers/apps/components/replicationset/suite_test.go +++ b/controllers/apps/components/replication/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package replicationset +package replication import ( "context" diff --git a/controllers/apps/components/stateful/stateful.go b/controllers/apps/components/stateful/stateful.go index ff4cc2a21..e59917d80 100644 --- a/controllers/apps/components/stateful/stateful.go +++ b/controllers/apps/components/stateful/stateful.go @@ -33,11 +33,11 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -type Stateful types.ComponentBase +type StatefulComponent types.ComponentBase -var _ types.Component = &Stateful{} +var _ types.Component = &StatefulComponent{} -func (r *Stateful) IsRunning(ctx context.Context, obj client.Object) (bool, error) { +func (r *StatefulComponent) IsRunning(ctx context.Context, obj client.Object) (bool, error) { if obj == nil { return false, nil } @@ -49,7 +49,7 @@ func (r *Stateful) IsRunning(ctx context.Context, obj client.Object) (bool, erro return util.StatefulSetOfComponentIsReady(sts, isRevisionConsistent, &r.Component.Replicas), nil } -func (r *Stateful) PodsReady(ctx context.Context, obj client.Object) (bool, error) { +func (r *StatefulComponent) PodsReady(ctx context.Context, obj client.Object) (bool, error) { if obj == nil { return false, nil } @@ -57,7 +57,7 @@ func (r *Stateful) PodsReady(ctx context.Context, obj client.Object) (bool, erro return util.StatefulSetPodsAreReady(sts, r.Component.Replicas), nil } -func (r *Stateful) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { +func (r *StatefulComponent) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { if pod == nil { return false } @@ -65,12 +65,12 @@ func (r *Stateful) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { } // HandleProbeTimeoutWhenPodsReady the Stateful component has no role detection, empty implementation here. -func (r *Stateful) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { +func (r *StatefulComponent) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { return false, nil } // GetPhaseWhenPodsNotReady gets the component phase when the pods of component are not ready. -func (r *Stateful) GetPhaseWhenPodsNotReady(ctx context.Context, componentName string) (appsv1alpha1.ClusterComponentPhase, error) { +func (r *StatefulComponent) GetPhaseWhenPodsNotReady(ctx context.Context, componentName string) (appsv1alpha1.ClusterComponentPhase, error) { stsList := &appsv1.StatefulSetList{} podList, err := util.GetCompRelatedObjectList(ctx, r.Cli, *r.Cluster, componentName, stsList) if err != nil || len(stsList.Items) == 0 { @@ -86,7 +86,7 @@ func (r *Stateful) GetPhaseWhenPodsNotReady(ctx context.Context, componentName s stsObj.Status.AvailableReplicas, checkExistFailedPodOfLatestRevision), nil } -func (r *Stateful) HandleUpdate(ctx context.Context, obj client.Object) error { +func (r *StatefulComponent) HandleUpdate(ctx context.Context, obj client.Object) error { return nil } @@ -99,7 +99,7 @@ func NewStatefulComponent( if err := util.ComponentRuntimeReqArgsCheck(cli, cluster, component); err != nil { return nil, err } - return &Stateful{ + return &StatefulComponent{ Cli: cli, Cluster: cluster, Component: component, diff --git a/controllers/apps/components/stateless/stateless.go b/controllers/apps/components/stateless/stateless.go index 43f668060..728f13df0 100644 --- a/controllers/apps/components/stateless/stateless.go +++ b/controllers/apps/components/stateless/stateless.go @@ -42,18 +42,18 @@ import ( // is at least the minimum available pods that need to run for the deployment. const NewRSAvailableReason = "NewReplicaSetAvailable" -type Stateless types.ComponentBase +type StatelessComponent types.ComponentBase -var _ types.Component = &Stateless{} +var _ types.Component = &StatelessComponent{} -func (stateless *Stateless) IsRunning(ctx context.Context, obj client.Object) (bool, error) { +func (stateless *StatelessComponent) IsRunning(ctx context.Context, obj client.Object) (bool, error) { if stateless == nil { return false, nil } return stateless.PodsReady(ctx, obj) } -func (stateless *Stateless) PodsReady(ctx context.Context, obj client.Object) (bool, error) { +func (stateless *StatelessComponent) PodsReady(ctx context.Context, obj client.Object) (bool, error) { if stateless == nil { return false, nil } @@ -64,7 +64,7 @@ func (stateless *Stateless) PodsReady(ctx context.Context, obj client.Object) (b return deploymentIsReady(deploy, &stateless.Component.Replicas), nil } -func (stateless *Stateless) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { +func (stateless *StatelessComponent) PodIsAvailable(pod *corev1.Pod, minReadySeconds int32) bool { if stateless == nil || pod == nil { return false } @@ -72,13 +72,13 @@ func (stateless *Stateless) PodIsAvailable(pod *corev1.Pod, minReadySeconds int3 } // HandleProbeTimeoutWhenPodsReady the stateless component has no role detection, empty implementation here. -func (stateless *Stateless) HandleProbeTimeoutWhenPodsReady(ctx context.Context, +func (stateless *StatelessComponent) HandleProbeTimeoutWhenPodsReady(ctx context.Context, recorder record.EventRecorder) (bool, error) { return false, nil } // GetPhaseWhenPodsNotReady gets the component phase when the pods of component are not ready. -func (stateless *Stateless) GetPhaseWhenPodsNotReady(ctx context.Context, +func (stateless *StatelessComponent) GetPhaseWhenPodsNotReady(ctx context.Context, componentName string) (appsv1alpha1.ClusterComponentPhase, error) { deployList := &appsv1.DeploymentList{} podList, err := util.GetCompRelatedObjectList(ctx, stateless.Cli, *stateless.Cluster, componentName, deployList) @@ -95,7 +95,7 @@ func (stateless *Stateless) GetPhaseWhenPodsNotReady(ctx context.Context, deploy.Status.AvailableReplicas, checkExistFailedPodOfNewRS), nil } -func (stateless *Stateless) HandleUpdate(ctx context.Context, obj client.Object) error { +func (stateless *StatelessComponent) HandleUpdate(ctx context.Context, obj client.Object) error { return nil } @@ -107,7 +107,7 @@ func NewStatelessComponent( if err := util.ComponentRuntimeReqArgsCheck(cli, cluster, component); err != nil { return nil, err } - return &Stateless{ + return &StatelessComponent{ Cli: cli, Cluster: cluster, Component: component, diff --git a/controllers/apps/configuration/policy_util.go b/controllers/apps/configuration/policy_util.go index 4a1f9f0e5..be71357ae 100644 --- a/controllers/apps/configuration/policy_util.go +++ b/controllers/apps/configuration/policy_util.go @@ -26,7 +26,7 @@ import ( "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" - "github.com/apecloud/kubeblocks/controllers/apps/components/consensusset" + "github.com/apecloud/kubeblocks/controllers/apps/components/consensus" "github.com/apecloud/kubeblocks/controllers/apps/components/util" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" cfgproto "github.com/apecloud/kubeblocks/internal/configuration/proto" @@ -158,7 +158,7 @@ func getConsensusPods(params reconfigureParams) ([]corev1.Pod, error) { } // sort pods - consensusset.SortPods(pods, consensusset.ComposeRolePriorityMap(*params.Component)) + consensus.SortPods(pods, consensus.ComposeRolePriorityMap(*params.Component)) r := make([]corev1.Pod, 0, len(pods)) for i := len(pods); i > 0; i-- { r = append(r, pods[i-1:i]...) diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index ff6c8605d..93bcf84b0 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -32,7 +32,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/replicationset" + "github.com/apecloud/kubeblocks/controllers/apps/components/replication" opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/lifecycle" @@ -369,7 +369,7 @@ var _ = Describe("OpsRequest Controller", func() { for i := int32(0); i < *sts.Spec.Replicas; i++ { podName := fmt.Sprintf("%s-%d", sts.Name, i) pod := testapps.MockReplicationComponentStsPod(nil, testCtx, sts, clusterObj.Name, - testapps.DefaultRedisCompName, podName, replicationset.DefaultRole(i)) + testapps.DefaultRedisCompName, podName, replication.DefaultRole(i)) podList = append(podList, pod) } } diff --git a/controllers/k8score/event_controller.go b/controllers/k8score/event_controller.go index 3a35969fc..be157f0bd 100644 --- a/controllers/k8score/event_controller.go +++ b/controllers/k8score/event_controller.go @@ -32,8 +32,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/consensusset" - "github.com/apecloud/kubeblocks/controllers/apps/components/replicationset" + "github.com/apecloud/kubeblocks/controllers/apps/components/consensus" + "github.com/apecloud/kubeblocks/controllers/apps/components/replication" componentutil "github.com/apecloud/kubeblocks/controllers/apps/components/util" "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" @@ -182,9 +182,9 @@ func handleRoleChangedEvent(cli client.Client, reqCtx intctrlutil.RequestCtx, re } switch componentDef.WorkloadType { case appsv1alpha1.Consensus: - return role, consensusset.UpdateConsensusSetRoleLabel(cli, reqCtx, componentDef, pod, role) + return role, consensus.UpdateConsensusSetRoleLabel(cli, reqCtx, componentDef, pod, role) case appsv1alpha1.Replication: - return role, replicationset.HandleReplicationSetRoleChangeEvent(cli, reqCtx, cluster, compName, pod, role) + return role, replication.HandleReplicationSetRoleChangeEvent(cli, reqCtx, cluster, compName, pod, role) } return role, nil } diff --git a/internal/controller/plan/prepare.go b/internal/controller/plan/prepare.go index 0cc3e46ae..be925766f 100644 --- a/internal/controller/plan/prepare.go +++ b/internal/controller/plan/prepare.go @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/replicationset" + "github.com/apecloud/kubeblocks/controllers/apps/components/replication" componentutil "github.com/apecloud/kubeblocks/controllers/apps/components/util" cfgutil "github.com/apecloud/kubeblocks/controllers/apps/configuration" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" @@ -154,13 +154,13 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, // If the statefulSets already exists, check whether there is an HA switching and the HA process is prioritized to handle. // TODO(xingran) After refactoring, HA switching will be handled in the replicationSet controller. if len(existStsList.Items) > 0 { - primaryIndexChanged, _, err := replicationset.CheckPrimaryIndexChanged(reqCtx.Ctx, cli, task.Cluster, + primaryIndexChanged, _, err := replication.CheckPrimaryIndexChanged(reqCtx.Ctx, cli, task.Cluster, task.Component.Name, task.Component.GetPrimaryIndex()) if err != nil { return err } if primaryIndexChanged { - if err := replicationset.HandleReplicationSetHASwitch(reqCtx.Ctx, cli, task.Cluster, + if err := replication.HandleReplicationSetHASwitch(reqCtx.Ctx, cli, task.Cluster, componentutil.GetClusterComponentSpecByName(*task.Cluster, task.Component.Name)); err != nil { return err } @@ -196,7 +196,7 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, case appsv1alpha1.Consensus: addLeaderSelectorLabels(svc, task.Component) case appsv1alpha1.Replication: - svc.Spec.Selector[constant.RoleLabelKey] = string(replicationset.Primary) + svc.Spec.Selector[constant.RoleLabelKey] = string(replication.Primary) } task.AppendResource(svc) } diff --git a/test/integration/redis_hscale_test.go b/test/integration/redis_hscale_test.go index d57f60d70..efca18596 100644 --- a/test/integration/redis_hscale_test.go +++ b/test/integration/redis_hscale_test.go @@ -29,7 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/replicationset" + "github.com/apecloud/kubeblocks/controllers/apps/components/replication" "github.com/apecloud/kubeblocks/controllers/apps/components/util" "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/generics" @@ -110,9 +110,9 @@ var _ = Describe("Redis Horizontal Scale function", func() { By("Checking statefulSet role label") for _, sts := range stsList.Items { if strings.HasSuffix(sts.Name, strconv.Itoa(testapps.DefaultReplicationPrimaryIndex)) { - Expect(sts.Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replicationset.Primary)) + Expect(sts.Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replication.Primary)) } else { - Expect(sts.Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replicationset.Secondary)) + Expect(sts.Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replication.Secondary)) } } @@ -122,9 +122,9 @@ var _ = Describe("Redis Horizontal Scale function", func() { Expect(err).To(Succeed()) Expect(len(podList)).Should(BeEquivalentTo(1)) if strings.HasSuffix(sts.Name, strconv.Itoa(testapps.DefaultReplicationPrimaryIndex)) { - Expect(podList[0].Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replicationset.Primary)) + Expect(podList[0].Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replication.Primary)) } else { - Expect(podList[0].Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replicationset.Secondary)) + Expect(podList[0].Labels[constant.RoleLabelKey]).Should(BeEquivalentTo(replication.Secondary)) } } @@ -179,11 +179,11 @@ var _ = Describe("Redis Horizontal Scale function", func() { replicationRedisConfigVolumeMounts := []corev1.VolumeMount{ { - Name: string(replicationset.Primary), + Name: string(replication.Primary), MountPath: "/etc/conf/primary", }, { - Name: string(replicationset.Secondary), + Name: string(replication.Secondary), MountPath: "/etc/conf/secondary", }, } @@ -193,8 +193,8 @@ var _ = Describe("Redis Horizontal Scale function", func() { clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). AddScriptTemplate(scriptConfigName, scriptConfigName, testCtx.DefaultNamespace, testapps.ScriptsVolumeName, &mode). - AddConfigTemplate(primaryConfigName, primaryConfigName, "", testCtx.DefaultNamespace, string(replicationset.Primary)). - AddConfigTemplate(secondaryConfigName, secondaryConfigName, "", testCtx.DefaultNamespace, string(replicationset.Secondary)). + AddConfigTemplate(primaryConfigName, primaryConfigName, "", testCtx.DefaultNamespace, string(replication.Primary)). + AddConfigTemplate(secondaryConfigName, secondaryConfigName, "", testCtx.DefaultNamespace, string(replication.Secondary)). AddInitContainerVolumeMounts(testapps.DefaultRedisInitContainerName, replicationRedisConfigVolumeMounts). AddContainerVolumeMounts(testapps.DefaultRedisContainerName, replicationRedisConfigVolumeMounts). Create(&testCtx).GetObject() From a12cb9fc31a6485c58f43369d9f883f6b2c9e3f7 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Mon, 17 Apr 2023 10:00:30 +0800 Subject: [PATCH 043/439] chore: support postgresql 14.7 instead of 15.2 (#2613) --- deploy/postgresql-cluster/Chart.yaml | 2 +- deploy/postgresql/Chart.yaml | 2 +- deploy/postgresql/values.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/postgresql-cluster/Chart.yaml b/deploy/postgresql-cluster/Chart.yaml index 56c51fcc3..cd0988253 100644 --- a/deploy/postgresql-cluster/Chart.yaml +++ b/deploy/postgresql-cluster/Chart.yaml @@ -6,4 +6,4 @@ type: application version: 0.5.0-alpha.3 -appVersion: "15.2.0" +appVersion: "14.7.0" diff --git a/deploy/postgresql/Chart.yaml b/deploy/postgresql/Chart.yaml index a2c45dc38..decbedd15 100644 --- a/deploy/postgresql/Chart.yaml +++ b/deploy/postgresql/Chart.yaml @@ -6,7 +6,7 @@ type: application version: 0.5.0-alpha.3 -appVersion: "15.2.0" +appVersion: "14.7.0" home: https://kubeblocks.io/ icon: https://github.com/apecloud/kubeblocks/raw/main/img/logo.png diff --git a/deploy/postgresql/values.yaml b/deploy/postgresql/values.yaml index 1030cab15..96a9c62c6 100644 --- a/deploy/postgresql/values.yaml +++ b/deploy/postgresql/values.yaml @@ -11,7 +11,7 @@ image: registry: registry.cn-hangzhou.aliyuncs.com repository: apecloud/spilo - tag: 15.2.0 + tag: 14.7.0 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' From 0dee08209c02419733c33e26eb5bfc0426435908 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Mon, 17 Apr 2023 10:21:02 +0800 Subject: [PATCH 044/439] chore: kbcli support mongodb (#2580) Co-authored-by: Shanshan.Ying --- .../mongodb/templates/clusterdefinition.yaml | 130 --------------- deploy/mongodb/templates/clusterversion.yaml | 20 --- .../templates/sharding-clusterdefinition.yaml | 148 ++++++++++++++++++ .../templates/sharding-clusterversion.yaml | 29 ++++ internal/sqlchannel/engine/engine.go | 3 + internal/sqlchannel/engine/mongodb.go | 64 ++++++++ 6 files changed, 244 insertions(+), 150 deletions(-) create mode 100644 deploy/mongodb/templates/sharding-clusterdefinition.yaml create mode 100644 deploy/mongodb/templates/sharding-clusterversion.yaml create mode 100644 internal/sqlchannel/engine/mongodb.go diff --git a/deploy/mongodb/templates/clusterdefinition.yaml b/deploy/mongodb/templates/clusterdefinition.yaml index b2ca36283..dc2e6a5e9 100644 --- a/deploy/mongodb/templates/clusterdefinition.yaml +++ b/deploy/mongodb/templates/clusterdefinition.yaml @@ -135,133 +135,3 @@ spec: volumeMounts: - name: mongodb-metrics-config mountPath: /opt/conf - - name: mongos - scriptSpecs: - - name: mongodb-scripts - templateRef: mongodb-scripts - volumeName: scripts - namespace: {{ .Release.Namespace }} - defaultMode: 493 - workloadType: Stateless - service: - ports: - - name: mongos - port: 27017 - targetPort: mongos - podSpec: - containers: - - name: mongos - ports: - - name: mongos - containerPort: 27017 - command: - - /scripts/mongos-setup.sh - volumeMounts: - - name: scripts - mountPath: /scripts/mongos-setup.sh - subPath: mongos-setup.sh - - name: configsvr - scriptSpecs: - - name: mongodb-scripts - templateRef: mongodb-scripts - volumeName: scripts - namespace: {{ .Release.Namespace }} - defaultMode: 493 - characterType: mongodb - workloadType: Consensus - consensusSpec: - leader: - name: "primary" - accessMode: ReadWrite - followers: - - name: "secondary" - accessMode: Readonly - updateStrategy: Serial - probes: - roleProbe: - periodSeconds: 2 - failureThreshold: 3 - service: - ports: - - name: configsvr - port: 27018 - targetPort: configsvr - podSpec: - containers: - - name: configsvr - ports: - - name: configsvr - containerPort: 27018 - command: - - /scripts/replicaset-setup.sh - - --configsvr - lifecycle: - postStart: - exec: - command: - - /scripts/replicaset-post-start.sh - - CONFIGSVR - - "true" - volumeMounts: - - name: scripts - mountPath: /scripts/replicaset-setup.sh - subPath: replicaset-setup.sh - - name: scripts - mountPath: /scripts/replicaset-post-start.sh - subPath: replicaset-post-start.sh - - name: shard - scriptSpecs: - - name: mongodb-scripts - templateRef: mongodb-scripts - volumeName: scripts - namespace: {{ .Release.Namespace }} - defaultMode: 493 - characterType: mongodb - workloadType: Consensus - consensusSpec: - leader: - name: "primary" - accessMode: ReadWrite - followers: - - name: "secondary" - accessMode: Readonly - updateStrategy: BestEffortParallel - probes: - roleProbe: - periodSeconds: 2 - failureThreshold: 3 - service: - ports: - - name: shard - port: 27018 - targetPort: shard - podSpec: - containers: - - name: shard - ports: - - name: shard - containerPort: 27018 - command: - - /scripts/replicaset-setup.sh - - --shardsvr - lifecycle: - postStart: - exec: - command: - - /scripts/replicaset-post-start.sh - - SHARD - - "false" - volumeMounts: - - name: scripts - mountPath: /scripts/replicaset-setup.sh - subPath: replicaset-setup.sh - - name: scripts - mountPath: /scripts/replicaset-post-start.sh - subPath: replicaset-post-start.sh - - name: agent - command: - - /scripts/shard-agent.sh - volumeMounts: - - name: scripts - mountPath: /scripts/shard-agent.sh - subPath: shard-agent.sh diff --git a/deploy/mongodb/templates/clusterversion.yaml b/deploy/mongodb/templates/clusterversion.yaml index 3e1cb149e..4a15db2db 100644 --- a/deploy/mongodb/templates/clusterversion.yaml +++ b/deploy/mongodb/templates/clusterversion.yaml @@ -13,23 +13,3 @@ spec: - name: mongodb image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} - - componentDefRef: mongos - versionsContext: - containers: - - name: mongos - image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} - - componentDefRef: configsvr - versionsContext: - containers: - - name: configsvr - image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} - - componentDefRef: shard - versionsContext: - containers: - - name: shard - image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} - - name: agent - image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} - imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} diff --git a/deploy/mongodb/templates/sharding-clusterdefinition.yaml b/deploy/mongodb/templates/sharding-clusterdefinition.yaml new file mode 100644 index 000000000..2caed0dfd --- /dev/null +++ b/deploy/mongodb/templates/sharding-clusterdefinition.yaml @@ -0,0 +1,148 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterDefinition +metadata: + name: mongodb-sharding + labels: + {{- include "mongodb.labels" . | nindent 4 }} +spec: + type: mongodb + connectionCredential: + username: root + password: {{ (include "mongodb.password" .) | quote }} + endpoint: "$(SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + host: "$(SVC_FQDN)" + port: "$(SVC_PORT_tcp-monogdb)" + headlessEndpoint: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + headlessHost: "$(POD_NAME_PREFIX)-0.$(HEADLESS_SVC_FQDN)" + headlessPort: "$(SVC_PORT_tcp-monogdb)" + componentDefs: + - name: mongos + scriptSpecs: + - name: mongodb-scripts + templateRef: mongodb-scripts + volumeName: scripts + namespace: {{ .Release.Namespace }} + defaultMode: 493 + workloadType: Stateless + service: + ports: + - name: mongos + port: 27017 + targetPort: mongos + podSpec: + containers: + - name: mongos + ports: + - name: mongos + containerPort: 27017 + command: + - /scripts/mongos-setup.sh + volumeMounts: + - name: scripts + mountPath: /scripts/mongos-setup.sh + subPath: mongos-setup.sh + - name: configsvr + scriptSpecs: + - name: mongodb-scripts + templateRef: mongodb-scripts + volumeName: scripts + namespace: {{ .Release.Namespace }} + defaultMode: 493 + characterType: mongodb + workloadType: Consensus + consensusSpec: + leader: + name: "primary" + accessMode: ReadWrite + followers: + - name: "secondary" + accessMode: Readonly + updateStrategy: Serial + probes: + roleProbe: + periodSeconds: 2 + failureThreshold: 3 + service: + ports: + - name: configsvr + port: 27018 + targetPort: configsvr + podSpec: + containers: + - name: configsvr + ports: + - name: configsvr + containerPort: 27018 + command: + - /scripts/replicaset-setup.sh + - --configsvr + lifecycle: + postStart: + exec: + command: + - /scripts/replicaset-post-start.sh + - CONFIGSVR + - "true" + volumeMounts: + - name: scripts + mountPath: /scripts/replicaset-setup.sh + subPath: replicaset-setup.sh + - name: scripts + mountPath: /scripts/replicaset-post-start.sh + subPath: replicaset-post-start.sh + - name: shard + scriptSpecs: + - name: mongodb-scripts + templateRef: mongodb-scripts + volumeName: scripts + namespace: {{ .Release.Namespace }} + defaultMode: 493 + characterType: mongodb + workloadType: Consensus + consensusSpec: + leader: + name: "primary" + accessMode: ReadWrite + followers: + - name: "secondary" + accessMode: Readonly + updateStrategy: BestEffortParallel + probes: + roleProbe: + periodSeconds: 2 + failureThreshold: 3 + service: + ports: + - name: shard + port: 27018 + targetPort: shard + podSpec: + containers: + - name: shard + ports: + - name: shard + containerPort: 27018 + command: + - /scripts/replicaset-setup.sh + - --shardsvr + lifecycle: + postStart: + exec: + command: + - /scripts/replicaset-post-start.sh + - SHARD + - "false" + volumeMounts: + - name: scripts + mountPath: /scripts/replicaset-setup.sh + subPath: replicaset-setup.sh + - name: scripts + mountPath: /scripts/replicaset-post-start.sh + subPath: replicaset-post-start.sh + - name: agent + command: + - /scripts/shard-agent.sh + volumeMounts: + - name: scripts + mountPath: /scripts/shard-agent.sh + subPath: shard-agent.sh diff --git a/deploy/mongodb/templates/sharding-clusterversion.yaml b/deploy/mongodb/templates/sharding-clusterversion.yaml new file mode 100644 index 000000000..609e0a512 --- /dev/null +++ b/deploy/mongodb/templates/sharding-clusterversion.yaml @@ -0,0 +1,29 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterVersion +metadata: + name: mongodb-sharding-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} + labels: + {{- include "mongodb.labels" . | nindent 4 }} +spec: + clusterDefinitionRef: mongodb-sharding + componentVersions: + - componentDefRef: mongos + versionsContext: + containers: + - name: mongos + image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + - componentDefRef: configsvr + versionsContext: + containers: + - name: configsvr + image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + - componentDefRef: shard + versionsContext: + containers: + - name: shard + image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + - name: agent + image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} diff --git a/internal/sqlchannel/engine/engine.go b/internal/sqlchannel/engine/engine.go index 2d7f45dff..09078783d 100644 --- a/internal/sqlchannel/engine/engine.go +++ b/internal/sqlchannel/engine/engine.go @@ -26,6 +26,7 @@ const ( stateMysql = "mysql" statePostgreSQL = "postgresql" stateRedis = "redis" + stateMongoDB = "mongodb" ) // AuthInfo is the authentication information for the database @@ -56,6 +57,8 @@ func New(typeName string) (Interface, error) { return newPostgreSQL(), nil case stateRedis: return newRedis(), nil + case stateMongoDB: + return newMongoDB(), nil default: return nil, fmt.Errorf("unsupported engine type: %s", typeName) } diff --git a/internal/sqlchannel/engine/mongodb.go b/internal/sqlchannel/engine/mongodb.go new file mode 100644 index 000000000..a17ca94dc --- /dev/null +++ b/internal/sqlchannel/engine/mongodb.go @@ -0,0 +1,64 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package engine + +import ( + "fmt" + "strings" +) + +type mongodb struct { + info EngineInfo + examples map[ClientType]buildConnectExample +} + +func newMongoDB() *mongodb { + return &mongodb{ + info: EngineInfo{ + Client: "mongosh", + Container: "mongodb", + UserEnv: "$MONGODB_ROOT_USER", + PasswordEnv: "$MONGODB_ROOT_PASSWORD", + }, + examples: map[ClientType]buildConnectExample{}, + } +} + +func (r mongodb) ConnectCommand(connectInfo *AuthInfo) []string { + userName := r.info.UserEnv + userPass := r.info.PasswordEnv + + if connectInfo != nil { + userName = connectInfo.UserName + userPass = connectInfo.UserPasswd + } + + mongodbCmd := []string{fmt.Sprintf("%s -u %s -p %s", r.info.Client, userName, userPass)} + + return []string{"sh", "-c", strings.Join(mongodbCmd, " ")} +} + +func (r mongodb) Container() string { + return r.info.Container +} + +func (r mongodb) ConnectExample(info *ConnectionInfo, client string) string { + // TODO implement me + panic("implement me") +} + +var _ Interface = &mongodb{} From 11d8e7af8b607b70779e24b1d2ac1e5cb866658a Mon Sep 17 00:00:00 2001 From: xingran Date: Mon, 17 Apr 2023 10:22:47 +0800 Subject: [PATCH 045/439] fix: postgresql cluster with patroni stop and start failed fix (#2614) --- controllers/apps/operations/stop.go | 11 ++++++++++- .../lifecycle/cluster_plan_builder.go | 17 +++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/controllers/apps/operations/stop.go b/controllers/apps/operations/stop.go index 71ba56bcb..6591ec658 100644 --- a/controllers/apps/operations/stop.go +++ b/controllers/apps/operations/stop.go @@ -25,6 +25,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/controller/lifecycle" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -84,7 +85,15 @@ func (stop StopOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCtx, cli cl opsRes *OpsResource, pgRes progressResource, compStatus *appsv1alpha1.OpsRequestComponentStatus) (int32, int32, error) { - return handleComponentProgressForScalingReplicas(reqCtx, cli, opsRes, pgRes, compStatus, getExpectReplicas) + expectProgressCount, completedCount, err := handleComponentProgressForScalingReplicas(reqCtx, cli, opsRes, pgRes, compStatus, getExpectReplicas) + if err != nil { + return expectProgressCount, completedCount, err + } + // TODO: delete the configmaps of the cluster should be removed from the opsRequest after refactor. + if err := lifecycle.DeleteConfigMaps(reqCtx.Ctx, cli, opsRes.Cluster); err != nil { + return expectProgressCount, completedCount, err + } + return expectProgressCount, completedCount, nil } return reconcileActionWithComponentOps(reqCtx, cli, opsRes, "", handleComponentProgress) } diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 951ca2d3d..e91b9bf6a 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -17,6 +17,7 @@ limitations under the License. package lifecycle import ( + "context" "errors" "fmt" "reflect" @@ -544,12 +545,7 @@ func (c *clusterPlanBuilder) deletePVCs(cluster *appsv1alpha1.Cluster) error { } func (c *clusterPlanBuilder) deleteConfigMaps(cluster *appsv1alpha1.Cluster) error { - inNS := client.InNamespace(cluster.Namespace) - ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.GetName(), - constant.AppManagedByLabelKey: constant.AppName, - } - return c.cli.DeleteAllOf(c.ctx.Ctx, &corev1.ConfigMap{}, inNS, ml) + return DeleteConfigMaps(c.ctx.Ctx, c.cli, cluster) } func (c *clusterPlanBuilder) deleteBackupPolicies(cluster *appsv1alpha1.Cluster) error { @@ -583,3 +579,12 @@ func (c *clusterPlanBuilder) deleteBackups(cluster *appsv1alpha1.Cluster) error } return nil } + +func DeleteConfigMaps(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster) error { + inNS := client.InNamespace(cluster.Namespace) + ml := client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.GetName(), + constant.AppManagedByLabelKey: constant.AppName, + } + return cli.DeleteAllOf(ctx, &corev1.ConfigMap{}, inNS, ml) +} From 5c5629830086346e61f7878c579e468322e7a0a7 Mon Sep 17 00:00:00 2001 From: linghan-hub <56351212+linghan-hub@users.noreply.github.com> Date: Mon, 17 Apr 2023 10:24:46 +0800 Subject: [PATCH 046/439] chore: add e2e test cases (#2612) --- Makefile | 3 + deploy/apecloud-mysql-cluster/Chart.yaml | 2 +- .../apecloud-mysql-scale-cluster/Chart.yaml | 2 +- deploy/apecloud-mysql-scale/Chart.yaml | 2 +- deploy/apecloud-mysql/Chart.yaml | 2 +- deploy/chatgpt-retrieval-plugin/Chart.yaml | 2 +- deploy/clickhouse-cluster/Chart.yaml | 2 +- deploy/clickhouse/Chart.yaml | 2 +- deploy/helm/Chart.yaml | 4 +- deploy/kafka-cluster/Chart.yaml | 2 +- deploy/kafka/Chart.yaml | 2 +- deploy/milvus/Chart.yaml | 2 +- deploy/mongodb-cluster/Chart.yaml | 2 +- deploy/mongodb/Chart.yaml | 2 +- deploy/nyancat/Chart.yaml | 4 +- deploy/postgresql-cluster/Chart.yaml | 2 +- deploy/postgresql/Chart.yaml | 2 +- deploy/qdrant-cluster/Chart.yaml | 2 +- deploy/qdrant/Chart.yaml | 2 +- deploy/redis-cluster/Chart.yaml | 2 +- deploy/redis/Chart.yaml | 2 +- deploy/weaviate-cluster/Chart.yaml | 2 +- deploy/weaviate/Chart.yaml | 2 +- test/e2e/Makefile | 2 +- .../smoketest/mongodb/00_mongodbcluster.yaml | 692 ++++++++++++++++++ .../smoketest/mongodb/01_vexpand.yaml | 12 + .../testdata/smoketest/mongodb/02_stop.yaml | 10 + .../testdata/smoketest/mongodb/03_start.yaml | 10 + .../testdata/smoketest/mongodb/04_vscale.yaml | 12 + .../smoketest/mongodb/05_restart.yaml | 10 + .../postgresql/00_postgresqlcluster.yaml | 10 +- .../smoketest/postgresql/03_stop.yaml | 10 + .../smoketest/postgresql/04_start.yaml | 10 + .../postgresql/{03_cv.yaml => 05_cv.yaml} | 6 +- .../smoketest/postgresql/06_backuppolicy.yaml | 25 - .../{04_upgrade.yaml => 06_upgrade.yaml} | 2 +- .../{05_restart.yaml => 07_restart.yaml} | 0 .../08_backup_snapshot.yaml | 5 +- .../08_backup_snapshot_restore.yaml | 30 - .../smoketest/postgresql/09_backup_full.yaml | 8 + .../10_backup_sbapshot_restore.yaml | 23 + .../postgresql/11_backup_full_restore.yaml | 23 + .../smoketest/redis/00_rediscluster.yaml | 312 +++++++- .../testdata/smoketest/redis/01_vscale.yaml | 2 +- .../smoketest/redis/02_hscale_up.yaml | 4 +- .../smoketest/redis/03_hscale_down.yaml | 4 +- .../e2e/testdata/smoketest/redis/04_stop.yaml | 10 + .../testdata/smoketest/redis/05_start.yaml | 10 + .../redis/{04_cv.yaml => 06_cv.yaml} | 2 +- .../{05_upgrade.yaml => 07_upgrade.yaml} | 2 +- .../{06_restart.yaml => 08_restart.yaml} | 0 .../testdata/smoketest/redis/09_vexpand.yaml | 12 + .../smoketest/wesql/00_wesqlcluster.yaml | 2 +- .../smoketest/wesql/01_custom_class.yaml | 20 + .../wesql/{01_vscale.yaml => 02_vscale.yaml} | 8 +- .../wesql/{02_hscale.yaml => 03_hscale.yaml} | 0 .../{03_vexpand.yaml => 04_vexpand.yaml} | 0 .../wesql/{04_cv.yaml => 05_cv.yaml} | 0 .../{05_upgrade.yaml => 06_upgrade.yaml} | 0 .../wesql/07_backuppolicy_snapshot.yaml | 24 - .../e2e/testdata/smoketest/wesql/07_stop.yaml | 10 + .../testdata/smoketest/wesql/08_start.yaml | 10 + .../{06_restart.yaml => 09_restart.yaml} | 0 .../11_backup_snapshot.yaml} | 5 +- .../smoketest/wesql/11_backuppolicy_full.yaml | 26 - .../smoketest/wesql/12_backup_full.yaml | 5 +- .../wesql/13_backup_full_restore.yaml | 32 - .../wesql/13_backup_snapshot_restore.yaml | 23 + ...store.yaml => 14_backup_full_restore.yaml} | 8 +- 69 files changed, 1246 insertions(+), 236 deletions(-) create mode 100644 test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml create mode 100644 test/e2e/testdata/smoketest/mongodb/01_vexpand.yaml create mode 100644 test/e2e/testdata/smoketest/mongodb/02_stop.yaml create mode 100644 test/e2e/testdata/smoketest/mongodb/03_start.yaml create mode 100644 test/e2e/testdata/smoketest/mongodb/04_vscale.yaml create mode 100644 test/e2e/testdata/smoketest/mongodb/05_restart.yaml create mode 100644 test/e2e/testdata/smoketest/postgresql/03_stop.yaml create mode 100644 test/e2e/testdata/smoketest/postgresql/04_start.yaml rename test/e2e/testdata/smoketest/postgresql/{03_cv.yaml => 05_cv.yaml} (61%) delete mode 100644 test/e2e/testdata/smoketest/postgresql/06_backuppolicy.yaml rename test/e2e/testdata/smoketest/postgresql/{04_upgrade.yaml => 06_upgrade.yaml} (75%) rename test/e2e/testdata/smoketest/postgresql/{05_restart.yaml => 07_restart.yaml} (100%) rename test/e2e/testdata/smoketest/{wesql => postgresql}/08_backup_snapshot.yaml (72%) delete mode 100644 test/e2e/testdata/smoketest/postgresql/08_backup_snapshot_restore.yaml create mode 100644 test/e2e/testdata/smoketest/postgresql/09_backup_full.yaml create mode 100644 test/e2e/testdata/smoketest/postgresql/10_backup_sbapshot_restore.yaml create mode 100644 test/e2e/testdata/smoketest/postgresql/11_backup_full_restore.yaml create mode 100644 test/e2e/testdata/smoketest/redis/04_stop.yaml create mode 100644 test/e2e/testdata/smoketest/redis/05_start.yaml rename test/e2e/testdata/smoketest/redis/{04_cv.yaml => 06_cv.yaml} (91%) rename test/e2e/testdata/smoketest/redis/{05_upgrade.yaml => 07_upgrade.yaml} (77%) rename test/e2e/testdata/smoketest/redis/{06_restart.yaml => 08_restart.yaml} (100%) create mode 100644 test/e2e/testdata/smoketest/redis/09_vexpand.yaml create mode 100644 test/e2e/testdata/smoketest/wesql/01_custom_class.yaml rename test/e2e/testdata/smoketest/wesql/{01_vscale.yaml => 02_vscale.yaml} (62%) rename test/e2e/testdata/smoketest/wesql/{02_hscale.yaml => 03_hscale.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{03_vexpand.yaml => 04_vexpand.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{04_cv.yaml => 05_cv.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{05_upgrade.yaml => 06_upgrade.yaml} (100%) delete mode 100644 test/e2e/testdata/smoketest/wesql/07_backuppolicy_snapshot.yaml create mode 100644 test/e2e/testdata/smoketest/wesql/07_stop.yaml create mode 100644 test/e2e/testdata/smoketest/wesql/08_start.yaml rename test/e2e/testdata/smoketest/wesql/{06_restart.yaml => 09_restart.yaml} (100%) rename test/e2e/testdata/smoketest/{postgresql/07_backup_snapshot.yaml => wesql/11_backup_snapshot.yaml} (72%) delete mode 100644 test/e2e/testdata/smoketest/wesql/11_backuppolicy_full.yaml delete mode 100644 test/e2e/testdata/smoketest/wesql/13_backup_full_restore.yaml create mode 100644 test/e2e/testdata/smoketest/wesql/13_backup_snapshot_restore.yaml rename test/e2e/testdata/smoketest/wesql/{09_backup_snapshot_restore.yaml => 14_backup_full_restore.yaml} (76%) diff --git a/Makefile b/Makefile index b088b04dd..f863421c4 100644 --- a/Makefile +++ b/Makefile @@ -749,6 +749,9 @@ render-smoke-testdata-manifests: ## Update E2E test dataset $(HELM) template mycluster deploy/postgresql-cluster > test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml $(HELM) template mycluster deploy/redis > test/e2e/testdata/smoketest/redis/00_rediscluster.yaml $(HELM) template mycluster deploy/redis-cluster >> test/e2e/testdata/smoketest/redis/00_rediscluster.yaml + $(HELM) template mycluster deploy/mongodb > test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml + $(HELM) template mycluster deploy/mongodb-cluster >> test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml + .PHONY: test-e2e test-e2e: helm-package render-smoke-testdata-manifests ## Run E2E tests. diff --git a/deploy/apecloud-mysql-cluster/Chart.yaml b/deploy/apecloud-mysql-cluster/Chart.yaml index 262d694e3..df86189ff 100644 --- a/deploy/apecloud-mysql-cluster/Chart.yaml +++ b/deploy/apecloud-mysql-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: An ApeCloud MySQL Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "8.0.30" diff --git a/deploy/apecloud-mysql-scale-cluster/Chart.yaml b/deploy/apecloud-mysql-scale-cluster/Chart.yaml index 5634d4b3e..8e71ff5c1 100644 --- a/deploy/apecloud-mysql-scale-cluster/Chart.yaml +++ b/deploy/apecloud-mysql-scale-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: An ApeCloud MySQL-Scale Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/deploy/apecloud-mysql-scale/Chart.yaml b/deploy/apecloud-mysql-scale/Chart.yaml index 165ca3995..09528eb08 100644 --- a/deploy/apecloud-mysql-scale/Chart.yaml +++ b/deploy/apecloud-mysql-scale/Chart.yaml @@ -5,7 +5,7 @@ description: ApeCloud MySQL-Scale is ApeCloud MySQL proxy. ApeCloud MySQL-Scale type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/deploy/apecloud-mysql/Chart.yaml b/deploy/apecloud-mysql/Chart.yaml index 736d7bd5b..efa0019f6 100644 --- a/deploy/apecloud-mysql/Chart.yaml +++ b/deploy/apecloud-mysql/Chart.yaml @@ -9,7 +9,7 @@ description: ApeCloud MySQL is fully compatible with MySQL syntax and supports s type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "8.0.30" diff --git a/deploy/chatgpt-retrieval-plugin/Chart.yaml b/deploy/chatgpt-retrieval-plugin/Chart.yaml index 347c5625d..1b84d6178 100644 --- a/deploy/chatgpt-retrieval-plugin/Chart.yaml +++ b/deploy/chatgpt-retrieval-plugin/Chart.yaml @@ -5,7 +5,7 @@ description: A demo application for ChatGPT plugin. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: 0.1.0 diff --git a/deploy/clickhouse-cluster/Chart.yaml b/deploy/clickhouse-cluster/Chart.yaml index 5e3e0cdba..de06b63b9 100644 --- a/deploy/clickhouse-cluster/Chart.yaml +++ b/deploy/clickhouse-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A ClickHouse cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: 22.9.4 diff --git a/deploy/clickhouse/Chart.yaml b/deploy/clickhouse/Chart.yaml index d7c59ce9d..7b963e2bc 100644 --- a/deploy/clickhouse/Chart.yaml +++ b/deploy/clickhouse/Chart.yaml @@ -9,7 +9,7 @@ annotations: type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: 22.9.4 diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml index b3bb0da0f..f4e6bc6fe 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.5.0-alpha.3 +appVersion: 0.5.0-alpha.8 kubeVersion: '>=1.22.0-0' diff --git a/deploy/kafka-cluster/Chart.yaml b/deploy/kafka-cluster/Chart.yaml index c854cb9b5..96f668ff9 100644 --- a/deploy/kafka-cluster/Chart.yaml +++ b/deploy/kafka-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A Kafka server cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: 3.4.0 diff --git a/deploy/kafka/Chart.yaml b/deploy/kafka/Chart.yaml index d8b5b22dc..f3a99dcdc 100644 --- a/deploy/kafka/Chart.yaml +++ b/deploy/kafka/Chart.yaml @@ -11,7 +11,7 @@ annotations: type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: 3.4.0 diff --git a/deploy/milvus/Chart.yaml b/deploy/milvus/Chart.yaml index 40702d561..a00be2d6f 100644 --- a/deploy/milvus/Chart.yaml +++ b/deploy/milvus/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 # This is the version number of milvus appVersion: "2.2.4" diff --git a/deploy/mongodb-cluster/Chart.yaml b/deploy/mongodb-cluster/Chart.yaml index 432ff137f..6fdd0b479 100644 --- a/deploy/mongodb-cluster/Chart.yaml +++ b/deploy/mongodb-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A MongoDB cluster Helm chart for KubeBlocks type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "5.0.14" diff --git a/deploy/mongodb/Chart.yaml b/deploy/mongodb/Chart.yaml index dac8a8181..b3504b6a1 100644 --- a/deploy/mongodb/Chart.yaml +++ b/deploy/mongodb/Chart.yaml @@ -4,7 +4,7 @@ description: MongoDB is a document database designed for ease of application dev type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "5.0.14" diff --git a/deploy/nyancat/Chart.yaml b/deploy/nyancat/Chart.yaml index 68a4c6259..4e385afeb 100644 --- a/deploy/nyancat/Chart.yaml +++ b/deploy/nyancat/Chart.yaml @@ -4,8 +4,8 @@ description: A demo application for showing database cluster availability. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 -appVersion: 0.5.0-alpha.3 +appVersion: 0.5.0-alpha.8 kubeVersion: '>=1.22.0-0' diff --git a/deploy/postgresql-cluster/Chart.yaml b/deploy/postgresql-cluster/Chart.yaml index cd0988253..de247d6c2 100644 --- a/deploy/postgresql-cluster/Chart.yaml +++ b/deploy/postgresql-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A PostgreSQL (with Patroni HA) cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "14.7.0" diff --git a/deploy/postgresql/Chart.yaml b/deploy/postgresql/Chart.yaml index decbedd15..dfb1814c1 100644 --- a/deploy/postgresql/Chart.yaml +++ b/deploy/postgresql/Chart.yaml @@ -4,7 +4,7 @@ description: A PostgreSQL (with Patroni HA) cluster definition Helm chart for Ku type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "14.7.0" diff --git a/deploy/qdrant-cluster/Chart.yaml b/deploy/qdrant-cluster/Chart.yaml index 84fbf4dbd..2e6b6c598 100644 --- a/deploy/qdrant-cluster/Chart.yaml +++ b/deploy/qdrant-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A Qdrant cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "1.1.0" diff --git a/deploy/qdrant/Chart.yaml b/deploy/qdrant/Chart.yaml index 8898897de..3e30e30cc 100644 --- a/deploy/qdrant/Chart.yaml +++ b/deploy/qdrant/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version. -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 # This is the version number of qdrant. appVersion: "1.1.0" diff --git a/deploy/redis-cluster/Chart.yaml b/deploy/redis-cluster/Chart.yaml index 55783b46c..ccb95e45c 100644 --- a/deploy/redis-cluster/Chart.yaml +++ b/deploy/redis-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: An Redis Replication Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "7.0.6" diff --git a/deploy/redis/Chart.yaml b/deploy/redis/Chart.yaml index 5950941a2..9762619ce 100644 --- a/deploy/redis/Chart.yaml +++ b/deploy/redis/Chart.yaml @@ -4,7 +4,7 @@ description: A Redis cluster definition Helm chart for Kubernetes type: application -version: 0.5.0-alpha.3 +version: 0.5.0-alpha.8 appVersion: "7.0.6" diff --git a/deploy/weaviate-cluster/Chart.yaml b/deploy/weaviate-cluster/Chart.yaml index a9395d6dd..c59c9159c 100644 --- a/deploy/weaviate-cluster/Chart.yaml +++ b/deploy/weaviate-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A weaviate cluster Helm chart for KubeBlocks. type: application -version: 0.1.0 +version: 0.5.0-alpha.8 appVersion: "1.18.0" diff --git a/deploy/weaviate/Chart.yaml b/deploy/weaviate/Chart.yaml index c198dced1..466cb1a70 100644 --- a/deploy/weaviate/Chart.yaml +++ b/deploy/weaviate/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version. -version: 0.1.0 +version: 0.5.0-alpha.8 # This is the version number of weaviate. appVersion: "1.18.0" diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 169399316..2eaef082a 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -33,7 +33,7 @@ endif .PHONY: run run: ginkgo ## Run end-to-end tests. #ACK_GINKGO_DEPRECATIONS=$(GINKGO_VERSION) $(GINKGO) run . - $(GINKGO) test -process -ginkgo.v . --json-report=report.json -- --VERSION=$(VERSION) + $(GINKGO) test -process -ginkgo.v . -- --VERSION=$(VERSION) --ginkgo.json-report=report.json build: ginkgo ## Run ginkgo build e2e test suite binary. $(GINKGO) build . diff --git a/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml b/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml new file mode 100644 index 000000000..d786498ea --- /dev/null +++ b/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml @@ -0,0 +1,692 @@ +--- +# Source: mongodb/templates/configtemplate.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb5.0-config-template + labels: + helm.sh/chart: mongodb-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +data: + mongodb.conf: |- + # mongod.conf + # for documentation of all options, see: + # http://docs.mongodb.org/manual/reference/configuration-options/ + + {{- $log_root := getVolumePathByName ( index $.podSpec.containers 0 ) "log" }} + {{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} + {{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} + {{- $phy_memory := getContainerMemory ( index $.podSpec.containers 0 ) }} + + # require port + {{- $mongodb_port := 27017 }} + {{- if $mongodb_port_info }} + {{- $mongodb_port = $mongodb_port_info.containerPort }} + {{- end }} + + # where and how to store data. + storage: + dbPath: {{ $mongodb_root }}/db + journal: + enabled: true + directoryPerDB: true + + # where to write logging data. + systemLog: + destination: file + quiet: false + logAppend: true + logRotate: reopen + path: {{ $mongodb_root }}/logs/mongodb.log + verbosity: 0 + + # network interfaces + net: + port: {{ $mongodb_port }} + unixDomainSocket: + enabled: false + pathPrefix: {{ $mongodb_root }}/tmp + ipv6: false + bindIpAll: true + #bindIp: + + # replica set options + replication: + replSetName: replicaset + enableMajorityReadConcern: true + + # sharding options + #sharding: + #clusterRole: + + # process management options + processManagement: + fork: false + pidFilePath: {{ $mongodb_root }}/tmp/mongodb.pid + + # set parameter options + setParameter: + enableLocalhostAuthBypass: true + + # security options + security: + authorization: enabled + keyFile: /etc/mongodb/keyfile + + keyfile: |- + {{ randAscii 64 | b64enc }} +--- +# Source: mongodb/templates/metrics-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-metrics-config + labels: + helm.sh/chart: mongodb-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +data: + metrics-config.yaml: "exporters:\n prometheus:\n const_labels: []\n enable_open_metrics: false\n endpoint: 0.0.0.0:9216\n metric_expiration: 30s\n resource_to_telemetry_conversion:\n enabled: true\n send_timestamps: false\nextensions:\n health_check:\n check_collector_pipeline:\n enabled: true\n exporter_failure_threshold: 5\n interval: 2m\n endpoint: 0.0.0.0:13133\n path: /health/status\n memory_ballast:\n size_mib: 512\nprocessors:\n batch:\n timeout: 5s\n memory_limiter:\n check_interval: 10s\n limit_mib: 1024\n spike_limit_mib: 256\nreceivers:\n apecloudmongodb:\n collect-all: true\n collection_interval: 15s\n compatible-mode: true\n direct-connect: true\n global-conn-pool: false\n log-level: info\n uri: mongodb://${env:MONGODB_ROOT_USER}:${env:MONGODB_ROOT_PASSWORD}@127.0.0.1:27017/admin?ssl=false&authSource=admin\nservice:\n extensions:\n - memory_ballast\n - health_check\n pipelines:\n metrics:\n exporters:\n - prometheus\n processors:\n - memory_limiter\n receivers:\n - apecloudmongodb\n telemetry:\n logs:\n level: info\n metrics:\n address: 0.0.0.0:8888" +--- +# Source: mongodb/templates/scriptstemplate.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-scripts + labels: + helm.sh/chart: mongodb-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +data: + mongos-setup.sh: |- + #!/bin/sh + + PORT=27018 + CONFIG_SVR_NAME=$KB_CLUSTER_NAME"-configsvr" + DOMAIN=$CONFIG_SVR_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" + mongos --bind_ip_all --configdb $CONFIG_SVR_NAME/$CONFIG_SVR_NAME"-0."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-1."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-2."$DOMAIN:$PORT + replicaset-setup.sh: |- + #!/bin/sh + + {{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} + {{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} + + # require port + {{- $mongodb_port := 27017 }} + {{- if $mongodb_port_info }} + {{- $mongodb_port = $mongodb_port_info.containerPort }} + {{- end }} + + PORT={{ $mongodb_port }} + MONGODB_ROOT={{ $mongodb_root }} + RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); + RPL_SET_NAME=${RPL_SET_NAME%-}; + mkdir -p $MONGODB_ROOT/db + mkdir -p $MONGODB_ROOT/logs + mkdir -p $MONGODB_ROOT/tmp + MODE=$1 + mongod $MODE --bind_ip_all --port $PORT --replSet $RPL_SET_NAME --config /etc/mongodb/mongodb.conf + + replicaset-post-start.sh: |- + #!/bin/sh + # usage: replicaset-post-start.sh type_name is_configsvr + # type_name: component.type, in uppercase + # is_configsvr: true or false, default false + {{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} + {{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} + + # require port + {{- $mongodb_port := 27017 }} + {{- if $mongodb_port_info }} + {{- $mongodb_port = $mongodb_port_info.containerPort }} + {{- end }} + + set -e + PORT={{ $mongodb_port }} + MONGODB_ROOT={{ $mongodb_root }} + INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); + INDEX=${INDEX#-}; + if [ $INDEX -ne 0 ]; then exit 0; fi + + until mongosh --quiet --port $PORT --eval "print('ready')"; do sleep 1; done + + RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); + RPL_SET_NAME=${RPL_SET_NAME%-}; + + TYPE_NAME=$1 + IS_CONFIGSVR=$2 + MEMBERS="" + i=0 + while [ $i -lt $(eval echo \$KB_"$TYPE_NAME"_N) ]; do + host=$(eval echo \$KB_"$TYPE_NAME"_"$i"_HOSTNAME) + host=$host"."$KB_NAMESPACE".svc.cluster.local" + until mongosh --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done + if [ $i -eq 0 ]; then + MEMBERS="{_id: $i, host: \"$host:$PORT\", priority:2}" + else + MEMBERS="$MEMBERS,{_id: $i, host: \"$host:$PORT\"}" + fi + i=$(( i + 1)) + done + CONFIGSVR="" + if [ ""$IS_CONFIGSVR = "true" ]; then CONFIGSVR="configsvr: true,"; fi + + until is_inited=$(mongosh --quiet --port $PORT --eval "rs.status().ok" -u root --password $MONGODB_ROOT_PASSWORD || mongosh --quiet --port $PORT --eval "try { rs.status().ok } catch (e) { 0 }") ; do sleep 1; done + if [ $is_inited -eq 1 ]; then + exit 0 + fi; + mongosh --quiet --port $PORT --eval "rs.initiate({_id: \"$RPL_SET_NAME\", $CONFIGSVR members: [$MEMBERS]})"; + + (until mongosh --quiet --port $PORT --eval "rs.isMaster().isWritablePrimary"|grep true; do sleep 1; done; + echo "create user"; + mongosh --quiet --port $PORT admin --eval "db.createUser({ user: \"$MONGODB_ROOT_USER\", pwd: \"$MONGODB_ROOT_PASSWORD\", roles: [{role: 'root', db: 'admin'}] })") /dev/null 2>&1 & + + shard-agent.sh: |- + #!/bin/sh + + INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); + INDEX=${INDEX#-}; + if [ $INDEX -ne 0 ]; then + trap : TERM INT; (while true; do sleep 1000; done) & wait + fi + + # wait main container ready + PORT=27018 + until mongosh --quiet --port $PORT --eval "rs.status().ok"; do sleep 1; done + # add shard to mongos + SHARD_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); + SHARD_NAME=${SHARD_NAME%-}; + DOMAIN=$SHARD_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" + MONGOS_HOST=$KB_CLUSTER_NAME"-mongos" + MONGOS_PORT=27017 + SHARD_CONFIG=$SHARD_NAME/$SHARD_NAME"-0."$DOMAIN:$PORT,$SHARD_NAME"-1."$DOMAIN:$PORT,$SHARD_NAME"-2."$DOMAIN:$PORT + until mongosh --quiet --host $MONGOS_HOST --port $MONGOS_PORT --eval "print('service is ready')"; do sleep 1; done + mongosh --quiet --host $MONGOS_HOST --port $MONGOS_PORT --eval "sh.addShard(\"$SHARD_CONFIG\")" + + trap : TERM INT; (while true; do sleep 1000; done) & wait +--- +# Source: mongodb/templates/backuppolicytemplate.yaml +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: BackupPolicyTemplate +metadata: + name: mongodb-backup-policy-template + labels: + clusterdefinition.kubeblocks.io/name: mongodb + helm.sh/chart: mongodb-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +spec: + clusterDefinitionRef: mongodb + backupPolicies: + - componentDefRef: replicaset + ttl: 7d + schedule: + baseBackup: + type: full + enable: false + cronExpression: "0 18 * * 0" + snapshot: + target: + role: leader + connectionCredentialKey: + passwordKey: password + usernameKey: username + full: + backupToolName: xtrabackup-apecloud-mysql +--- +# Source: mongodb/templates/backuptool.yaml +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupTool +metadata: + name: mongodb-physical-backup-tool + labels: + clusterdefinition.kubeblocks.io/name: mongodb +spec: + image: mongo:5.0.14 + deployKind: job + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: "1" + memory: 128Mi + env: + - name: DATA_DIR + value: /data/mongodb/db + physical: + restoreCommands: + - | + set -e + mkdir -p ${DATA_DIR} + res=`ls -A ${DATA_DIR}` + if [ ! -z ${res} ]; then + echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." + exit 1 + fi + tar -xvf ${BACKUP_DIR}/${BACKUP_NAME}.tar.gz -C ${DATA_DIR}/../ + mv ${DATA_DIR}/../${BACKUP_NAME}/* ${DATA_DIR} + PORT=27017 + MONGODB_ROOT=/data/mongodb + RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); + RPL_SET_NAME=${RPL_SET_NAME%-}; + mkdir -p $MONGODB_ROOT/db + mkdir -p $MONGODB_ROOT/logs + mkdir -p $MONGODB_ROOT/tmp + MODE=$1 + mongod $MODE --bind_ip_all --port $PORT --dbpath $MONGODB_ROOT/db --directoryperdb --logpath $MONGODB_ROOT/logs/mongodb.log --logappend --pidfilepath $MONGODB_ROOT/tmp/mongodb.pid& + until mongosh --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done + PID=`cat $MONGODB_ROOT/tmp/mongodb.pid` + + mongosh --quiet --port $PORT local --eval "db.system.replset.deleteOne({})" + mongosh --quiet --port $PORT local --eval "db.system.replset.find()" + mongosh --quiet --port $PORT admin --eval 'db.dropUser("root", {w: "majority", wtimeout: 4000})' || true + kill $PID + wait $PID + incrementalRestoreCommands: [] + logical: + restoreCommands: [] + incrementalRestoreCommands: [] + backupCommands: + - | + set -e + mkdir -p ${BACKUP_DIR}/${BACKUP_NAME} + cp -R ${DATA_DIR}/* ${BACKUP_DIR}/${BACKUP_NAME}/ + cd ${BACKUP_DIR} + tar -czvf ${BACKUP_NAME}.tar.gz ./${BACKUP_NAME} + rm -rf ${BACKUP_DIR}/${BACKUP_NAME} + incrementalBackupCommands: [] +--- +# Source: mongodb/templates/clusterdefinition.yaml +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterDefinition +metadata: + name: mongodb + labels: + helm.sh/chart: mongodb-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +spec: + type: mongodb + connectionCredential: + username: root + password: "$(RANDOM_PASSWD)" + endpoint: "$(SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + host: "$(SVC_FQDN)" + port: "$(SVC_PORT_tcp-monogdb)" + headlessEndpoint: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + headlessHost: "$(POD_NAME_PREFIX)-0.$(HEADLESS_SVC_FQDN)" + headlessPort: "$(SVC_PORT_tcp-monogdb)" + componentDefs: + - name: replicaset + characterType: mongodb + scriptSpecs: + - name: mongodb-scripts + templateRef: mongodb-scripts + volumeName: scripts + namespace: default + defaultMode: 493 + configSpecs: + - name: mongodb-config + templateRef: mongodb5.0-config-template + namespace: default + volumeName: mongodb-config + constraintRef: mongodb-config-constraints + keys: + - mongodb.conf + defaultMode: 256 + - name: mongodb-metrics-config + templateRef: mongodb-metrics-config + namespace: default + volumeName: mongodb-metrics-config + defaultMode: 0777 + monitor: + builtIn: false + exporterConfig: + scrapePath: /metrics + scrapePort: 9216 + logConfigs: + - name: running + filePathPattern: /data/mongodb/log/mongodb.log* + workloadType: Consensus + consensusSpec: + leader: + name: "primary" + accessMode: ReadWrite + followers: + - name: "secondary" + accessMode: Readonly + updateStrategy: Serial + probes: + roleProbe: + periodSeconds: 2 + failureThreshold: 3 + service: + ports: + - protocol: TCP + port: 27017 + volumeTypes: + - name: data + type: data + podSpec: + containers: + - name: mongodb + ports: + - name: mongodb + protocol: TCP + containerPort: 27017 + command: + - /scripts/replicaset-setup.sh + env: + - name: MONGODB_ROOT_USER + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: username + - name: MONGODB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: password + lifecycle: + postStart: + exec: + command: + - /scripts/replicaset-post-start.sh + - REPLICASET + volumeMounts: + - mountPath: /data/mongodb + name: data + - mountPath: /etc/mongodb/mongodb.conf + name: mongodb-config + subPath: mongodb.conf + - mountPath: /etc/mongodb/keyfile + name: mongodb-config + subPath: keyfile + - name: scripts + mountPath: /scripts/replicaset-setup.sh + subPath: replicaset-setup.sh + - name: scripts + mountPath: /scripts/replicaset-post-start.sh + subPath: replicaset-post-start.sh + - name: metrics + image: registry.cn-hangzhou.aliyuncs.com/apecloud/agamotto:0.0.4 + imagePullPolicy: "IfNotPresent" + securityContext: + runAsNonRoot: true + runAsUser: 1001 + env: + - name: MONGODB_ROOT_USER + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: username + - name: MONGODB_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: password + command: + - "/bin/agamotto" + - "--config=/opt/conf/metrics-config.yaml" + ports: + - name: http-metrics + containerPort: 9216 + volumeMounts: + - name: mongodb-metrics-config + mountPath: /opt/conf + - name: mongos + scriptSpecs: + - name: mongodb-scripts + templateRef: mongodb-scripts + volumeName: scripts + namespace: default + defaultMode: 493 + workloadType: Stateless + service: + ports: + - name: mongos + port: 27017 + targetPort: mongos + podSpec: + containers: + - name: mongos + ports: + - name: mongos + containerPort: 27017 + command: + - /scripts/mongos-setup.sh + volumeMounts: + - name: scripts + mountPath: /scripts/mongos-setup.sh + subPath: mongos-setup.sh + - name: configsvr + scriptSpecs: + - name: mongodb-scripts + templateRef: mongodb-scripts + volumeName: scripts + namespace: default + defaultMode: 493 + characterType: mongodb + workloadType: Consensus + consensusSpec: + leader: + name: "primary" + accessMode: ReadWrite + followers: + - name: "secondary" + accessMode: Readonly + updateStrategy: Serial + probes: + roleProbe: + periodSeconds: 2 + failureThreshold: 3 + service: + ports: + - name: configsvr + port: 27018 + targetPort: configsvr + podSpec: + containers: + - name: configsvr + ports: + - name: configsvr + containerPort: 27018 + command: + - /scripts/replicaset-setup.sh + - --configsvr + lifecycle: + postStart: + exec: + command: + - /scripts/replicaset-post-start.sh + - CONFIGSVR + - "true" + volumeMounts: + - name: scripts + mountPath: /scripts/replicaset-setup.sh + subPath: replicaset-setup.sh + - name: scripts + mountPath: /scripts/replicaset-post-start.sh + subPath: replicaset-post-start.sh + - name: shard + scriptSpecs: + - name: mongodb-scripts + templateRef: mongodb-scripts + volumeName: scripts + namespace: default + defaultMode: 493 + characterType: mongodb + workloadType: Consensus + consensusSpec: + leader: + name: "primary" + accessMode: ReadWrite + followers: + - name: "secondary" + accessMode: Readonly + updateStrategy: BestEffortParallel + probes: + roleProbe: + periodSeconds: 2 + failureThreshold: 3 + service: + ports: + - name: shard + port: 27018 + targetPort: shard + podSpec: + containers: + - name: shard + ports: + - name: shard + containerPort: 27018 + command: + - /scripts/replicaset-setup.sh + - --shardsvr + lifecycle: + postStart: + exec: + command: + - /scripts/replicaset-post-start.sh + - SHARD + - "false" + volumeMounts: + - name: scripts + mountPath: /scripts/replicaset-setup.sh + subPath: replicaset-setup.sh + - name: scripts + mountPath: /scripts/replicaset-post-start.sh + subPath: replicaset-post-start.sh + - name: agent + command: + - /scripts/shard-agent.sh + volumeMounts: + - name: scripts + mountPath: /scripts/shard-agent.sh + subPath: shard-agent.sh +--- +# Source: mongodb/templates/clusterversion.yaml +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterVersion +metadata: + name: mongodb-5.0.14 + labels: + helm.sh/chart: mongodb-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +spec: + clusterDefinitionRef: mongodb + componentVersions: + - componentDefRef: replicaset + versionsContext: + containers: + - name: mongodb + image: mongo:5.0.14 + imagePullPolicy: IfNotPresent + - componentDefRef: mongos + versionsContext: + containers: + - name: mongos + image: mongo:5.0.14 + imagePullPolicy: IfNotPresent + - componentDefRef: configsvr + versionsContext: + containers: + - name: configsvr + image: mongo:5.0.14 + imagePullPolicy: IfNotPresent + - componentDefRef: shard + versionsContext: + containers: + - name: shard + image: mongo:5.0.14 + - name: agent + image: mongo:5.0.14 + imagePullPolicy: IfNotPresent +--- +# Source: mongodb/templates/configconstraint.yaml +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ConfigConstraint +metadata: + name: mongodb-config-constraints + labels: + helm.sh/chart: mongodb-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +spec: + configurationSchema: + cue: "" + + # mysql configuration file format + formatterConfig: + format: yaml +--- +# Source: mongodb-cluster/templates/replicaset.yaml +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + name: mycluster + labels: + helm.sh/chart: mongodb-cluster-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb-cluster + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +spec: + clusterDefinitionRef: mongodb + clusterVersionRef: mongodb-5.0.14 + terminationPolicy: Halt + affinity: + topologyKeys: + - kubernetes.io/hostname + componentSpecs: + - name: replicaset + componentDefRef: replicaset + monitor: false + replicas: 3 + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + storageClassName: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +--- +# Source: mongodb-cluster/templates/tests/test-connection.yaml +apiVersion: v1 +kind: Pod +metadata: + name: "mycluster-mongodb-cluster-test-connection" + labels: + helm.sh/chart: mongodb-cluster-0.5.0-alpha.8 + app.kubernetes.io/name: mongodb-cluster + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['mycluster-mongodb-cluster:'] + restartPolicy: Never diff --git a/test/e2e/testdata/smoketest/mongodb/01_vexpand.yaml b/test/e2e/testdata/smoketest/mongodb/01_vexpand.yaml new file mode 100644 index 000000000..0e1633071 --- /dev/null +++ b/test/e2e/testdata/smoketest/mongodb/01_vexpand.yaml @@ -0,0 +1,12 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-vexpand +spec: + clusterRef: mycluster + type: VolumeExpansion + volumeExpansion: + - componentName: replicaset + volumeClaimTemplates: + - name: data + storage: "2Gi" \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/mongodb/02_stop.yaml b/test/e2e/testdata/smoketest/mongodb/02_stop.yaml new file mode 100644 index 000000000..dc42f742d --- /dev/null +++ b/test/e2e/testdata/smoketest/mongodb/02_stop.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-stop +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 27017 + type: Stop + restart: + - componentName: replicaset \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/mongodb/03_start.yaml b/test/e2e/testdata/smoketest/mongodb/03_start.yaml new file mode 100644 index 000000000..c3eddb51a --- /dev/null +++ b/test/e2e/testdata/smoketest/mongodb/03_start.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-start +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 27017 + type: Start + restart: + - componentName: replicaset \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/mongodb/04_vscale.yaml b/test/e2e/testdata/smoketest/mongodb/04_vscale.yaml new file mode 100644 index 000000000..ae3e4c701 --- /dev/null +++ b/test/e2e/testdata/smoketest/mongodb/04_vscale.yaml @@ -0,0 +1,12 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-vscale +spec: + clusterRef: mycluster + type: VerticalScaling + verticalScaling: + - componentName: replicaset + requests: + cpu: "500m" + memory: 500Mi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/mongodb/05_restart.yaml b/test/e2e/testdata/smoketest/mongodb/05_restart.yaml new file mode 100644 index 000000000..bf4a126b9 --- /dev/null +++ b/test/e2e/testdata/smoketest/mongodb/05_restart.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-restart +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 27017 + type: Restart + restart: + - componentName: replicaset \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml b/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml index f43e24127..6e2b20f89 100644 --- a/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml +++ b/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml @@ -5,22 +5,24 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: pgcluster-0.5.0-alpha.3 + helm.sh/chart: pgcluster-0.5.0-alpha.8 app.kubernetes.io/name: pgcluster app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "14.7.0" + app.kubernetes.io/version: "15.2.0" app.kubernetes.io/managed-by: Helm spec: clusterDefinitionRef: postgresql # ref clusterdefinition.name - clusterVersionRef: postgresql-14.7.0 # ref clusterversion.name + clusterVersionRef: postgresql-15.2.0 # ref clusterversion.name terminationPolicy: Delete affinity: componentSpecs: - name: postgresql # user-defined - componentDefRef: pg-replication # ref clusterdefinition components.name + componentDefRef: postgresql # ref clusterdefinition components.name monitor: false replicas: 2 primaryIndex: 0 + switchPolicy: + type: Noop enabledLogs: ["running"] volumeClaimTemplates: - name: data # ref clusterdefinition components.containers.volumeMounts.name diff --git a/test/e2e/testdata/smoketest/postgresql/03_stop.yaml b/test/e2e/testdata/smoketest/postgresql/03_stop.yaml new file mode 100644 index 000000000..c3220f818 --- /dev/null +++ b/test/e2e/testdata/smoketest/postgresql/03_stop.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-stop +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 5432 + type: Stop + restart: + - componentName: postgresql \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/04_start.yaml b/test/e2e/testdata/smoketest/postgresql/04_start.yaml new file mode 100644 index 000000000..200178e44 --- /dev/null +++ b/test/e2e/testdata/smoketest/postgresql/04_start.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-start +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 5432 + type: Start + restart: + - componentName: postgresql \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/03_cv.yaml b/test/e2e/testdata/smoketest/postgresql/05_cv.yaml similarity index 61% rename from test/e2e/testdata/smoketest/postgresql/03_cv.yaml rename to test/e2e/testdata/smoketest/postgresql/05_cv.yaml index b87f3bbcc..47bb38480 100644 --- a/test/e2e/testdata/smoketest/postgresql/03_cv.yaml +++ b/test/e2e/testdata/smoketest/postgresql/05_cv.yaml @@ -1,12 +1,12 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterVersion metadata: - name: postgresql-14.7.0-latest + name: postgresql-15.2.0-latest spec: clusterDefinitionRef: postgresql componentVersions: - - componentDefRef: pg-replication + - componentDefRef: postgresql versionsContext: containers: - name: postgresql - image: docker.io/apecloud/postgresql:14.7.0 + image: docker.io/apecloud/postgresql:15.2.0 diff --git a/test/e2e/testdata/smoketest/postgresql/06_backuppolicy.yaml b/test/e2e/testdata/smoketest/postgresql/06_backuppolicy.yaml deleted file mode 100644 index a9a2a82be..000000000 --- a/test/e2e/testdata/smoketest/postgresql/06_backuppolicy.yaml +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 -kind: BackupPolicy -metadata: - name: backup-policy-mycluster -spec: - backupToolName: xtrabackup-postgresql - hooks: - postCommands: - - rm -f /data/mysql/data/.restore_new_cluster; sync - preCommands: - - touch /data/mysql/data/.restore_new_cluster; sync - remoteVolume: - name: backup-remote-volume - persistentVolumeClaim: - claimName: backup-host-path-pvc - schedule: 0 3 * * * - target: - labelsSelector: - matchLabels: - app.kubernetes.io/instance: mycluster - secret: - name: mycluster-conn-credential - ttl: 168h0m0s - - diff --git a/test/e2e/testdata/smoketest/postgresql/04_upgrade.yaml b/test/e2e/testdata/smoketest/postgresql/06_upgrade.yaml similarity index 75% rename from test/e2e/testdata/smoketest/postgresql/04_upgrade.yaml rename to test/e2e/testdata/smoketest/postgresql/06_upgrade.yaml index abf304c68..dd9c6fde5 100644 --- a/test/e2e/testdata/smoketest/postgresql/04_upgrade.yaml +++ b/test/e2e/testdata/smoketest/postgresql/06_upgrade.yaml @@ -6,4 +6,4 @@ spec: clusterRef: mycluster type: Upgrade upgrade: - clusterVersionRef: postgresql-14.7.0-latest \ No newline at end of file + clusterVersionRef: postgresql-15.2.0-latest \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/05_restart.yaml b/test/e2e/testdata/smoketest/postgresql/07_restart.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/05_restart.yaml rename to test/e2e/testdata/smoketest/postgresql/07_restart.yaml diff --git a/test/e2e/testdata/smoketest/wesql/08_backup_snapshot.yaml b/test/e2e/testdata/smoketest/postgresql/08_backup_snapshot.yaml similarity index 72% rename from test/e2e/testdata/smoketest/wesql/08_backup_snapshot.yaml rename to test/e2e/testdata/smoketest/postgresql/08_backup_snapshot.yaml index 65b22b227..74998c0fa 100644 --- a/test/e2e/testdata/smoketest/wesql/08_backup_snapshot.yaml +++ b/test/e2e/testdata/smoketest/postgresql/08_backup_snapshot.yaml @@ -6,6 +6,5 @@ metadata: dataprotection.kubeblocks.io/backup-type: snapshot name: backup-sbapshot-mycluster spec: - backupPolicyName: backup-policy-mycluster - backupType: snapshot - ttl: 168h0m0s \ No newline at end of file + backupPolicyName: mycluster-postgresql-backup-policy + backupType: snapshot \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/08_backup_snapshot_restore.yaml b/test/e2e/testdata/smoketest/postgresql/08_backup_snapshot_restore.yaml deleted file mode 100644 index 83df02f3d..000000000 --- a/test/e2e/testdata/smoketest/postgresql/08_backup_snapshot_restore.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: Cluster -metadata: - name: snapshot-mycluster -spec: - clusterDefinitionRef: postgresql - clusterVersionRef: postgresql-14.7.0 - terminationPolicy: WipeOut - affinity: - topologyKeys: - - kubernetes.io/hostname - componentSpecs: - - name: postgresql - componentDefRef: pg-replication - monitor: false - replicas: 2 - enabledLogs: ["running"] - volumeClaimTemplates: - - name: data - spec: - storageClassName: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 11Gi - dataSource: - apiGroup: snapshot.storage.k8s.io - kind: VolumeSnapshot - name: backup-sbapshot-mycluster \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/09_backup_full.yaml b/test/e2e/testdata/smoketest/postgresql/09_backup_full.yaml new file mode 100644 index 000000000..846d9de30 --- /dev/null +++ b/test/e2e/testdata/smoketest/postgresql/09_backup_full.yaml @@ -0,0 +1,8 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: Backup +metadata: + name: backup-full + namespace: default +spec: + backupPolicyName: mycluster-postgresql-backup-policy + backupType: full diff --git a/test/e2e/testdata/smoketest/postgresql/10_backup_sbapshot_restore.yaml b/test/e2e/testdata/smoketest/postgresql/10_backup_sbapshot_restore.yaml new file mode 100644 index 000000000..241ac3509 --- /dev/null +++ b/test/e2e/testdata/smoketest/postgresql/10_backup_sbapshot_restore.yaml @@ -0,0 +1,23 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + name: mycluster-sbapshot + annotations: + kubeblocks.io/restore-from-backup: "{\"postgresql\":\"backup-sbapshot-mycluster\"}" +spec: + clusterDefinitionRef: postgresql + clusterVersionRef: postgresql-15.2.0 + terminationPolicy: WipeOut + componentSpecs: + - name: wesql + componentDefRef: mysql + monitor: false + replicas: 1 + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/11_backup_full_restore.yaml b/test/e2e/testdata/smoketest/postgresql/11_backup_full_restore.yaml new file mode 100644 index 000000000..d91be7614 --- /dev/null +++ b/test/e2e/testdata/smoketest/postgresql/11_backup_full_restore.yaml @@ -0,0 +1,23 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + name: mycluster-sbapshot + annotations: + kubeblocks.io/restore-from-backup: "{\"postgresql\":\"backup-full-mycluster\"}" +spec: + clusterDefinitionRef: postgresql + clusterVersionRef: postgresql-15.2.0 + terminationPolicy: WipeOut + componentSpecs: + - name: wesql + componentDefRef: mysql + monitor: false + replicas: 1 + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml index dfe5e35a2..4cbc35a2b 100644 --- a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml +++ b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml @@ -5,10 +5,10 @@ kind: ConfigMap metadata: name: redis7-config-template labels: - helm.sh/chart: redis-0.5.0-alpha.3 + helm.sh/chart: redis-0.5.0-alpha.8 app.kubernetes.io/name: redis app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.5" + app.kubernetes.io/version: "7.0.6" app.kubernetes.io/managed-by: Helm data: redis.conf: |- @@ -82,6 +82,14 @@ data: aof-rewrite-incremental-fsync yes rdb-save-incremental-fsync yes jemalloc-bg-thread yes + enable-debug-command yes + protected-mode no + + # maxmemory + {{- $request_memory := getContainerRequestMemory ( index $.podSpec.containers 0 ) }} + {{- if gt $request_memory 0 }} + maxmemory {{ $request_memory }} + {{- end -}} --- # Source: redis/templates/scripts.yaml apiVersion: v1 @@ -100,23 +108,121 @@ data: until redis-cli -h $KB_PRIMARY_POD_NAME -p 6379 ping; do sleep 1; done redis-cli -h 127.0.0.1 -p 6379 replicaof $KB_PRIMARY_POD_NAME 6379 || exit 1 fi + redis-start.sh: | + #!/bin/sh + set -ex + echo "include /etc/conf/redis.conf" >> /etc/redis/redis.conf + echo "replica-announce-ip $KB_POD_FQDN" >> /etc/redis/redis.conf + exec redis-server /etc/redis/redis.conf \ + --loadmodule /opt/redis-stack/lib/redisearch.so ${REDISEARCH_ARGS} \ + --loadmodule /opt/redis-stack/lib/redisgraph.so ${REDISGRAPH_ARGS} \ + --loadmodule /opt/redis-stack/lib/redistimeseries.so ${REDISTIMESERIES_ARGS} \ + --loadmodule /opt/redis-stack/lib/rejson.so ${REDISJSON_ARGS} \ + --loadmodule /opt/redis-stack/lib/redisbloom.so ${REDISBLOOM_ARGS} + redis-sentinel-setup.sh: |- + #!/bin/sh + set -ex + {{- $clusterName := $.cluster.metadata.name }} + {{- $namespace := $.cluster.metadata.namespace }} + {{- /* find redis-sentinel component */}} + {{- $sentinel_component := fromJson "{}" }} + {{- $redis_component := fromJson "{}" }} + {{- $primary_index := 0 }} + {{- $primary_pod := "" }} + {{- range $i, $e := $.cluster.spec.componentSpecs }} + {{- if eq $e.componentDefRef "redis-sentinel" }} + {{- $sentinel_component = $e }} + {{- else if eq $e.componentDefRef "redis" }} + {{- $redis_component = $e }} + {{- if index $e "primaryIndex" }} + {{- if ne ($e.primaryIndex | int) 0 }} + {{- $primary_index = ($e.primaryIndex | int) }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- /* build primary pod message, because currently does not support cross-component acquisition of environment variables, the service of the redis master node is assembled here through specific rules */}} + {{- $primary_pod = printf "%s-%s-0.%s-%s-headless.%s.svc" $clusterName $redis_component.name $clusterName $redis_component.name $namespace }} + {{- if ne $primary_index 0 }} + {{- $primary_pod = printf "%s-%s-%d-0.%s-%s-headless.%s.svc" $clusterName $redis_component.name $primary_index $clusterName $redis_component.name $namespace }} + {{- end }} + {{- $sentinel_monitor := printf "%s-%s %s" $clusterName $sentinel_component.name $primary_pod }} + cat>/etc/sentinel/redis-sentinel.conf<> /etc/sentinel/redis-sentinel.conf + exec redis-server /etc/sentinel/redis-sentinel.conf --sentinel + echo "Start sentinel succeeded!" + redis-sentinel-ping.sh: |- + #!/bin/sh + set -ex + response=$( + timeout -s 3 $1 \ + redis-cli \ + -h localhost \ + -p 26379 \ + ping + ) + if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 + fi + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi --- # Source: redis/templates/backuppolicytemplate.yaml -apiVersion: dataprotection.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: backup-policy-template-redis + name: redis-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: redis - helm.sh/chart: redis-0.5.0-alpha.3 + helm.sh/chart: redis-0.5.0-alpha.8 app.kubernetes.io/name: redis app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.5" + app.kubernetes.io/version: "7.0.6" app.kubernetes.io/managed-by: Helm spec: - # which backup tool to perform database backup, only support one tool. - backupToolName: volumesnapshot - ttl: 168h0m0s + clusterDefinitionRef: redis + backupPolicies: + - componentDefRef: redis + ttl: 7d + schedule: + baseBackup: + type: snapshot + enable: false + cronExpression: "0 18 * * 0" + snapshot: + target: + connectionCredentialKey: + passwordKey: password + usernameKey: username --- # Source: redis/templates/clusterdefinition.yaml apiVersion: apps.kubeblocks.io/v1alpha1 @@ -124,12 +230,13 @@ kind: ClusterDefinition metadata: name: redis labels: - helm.sh/chart: redis-0.5.0-alpha.3 + helm.sh/chart: redis-0.5.0-alpha.8 app.kubernetes.io/name: redis app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.5" + app.kubernetes.io/version: "7.0.6" app.kubernetes.io/managed-by: Helm spec: + type: redis connectionCredential: username: "" password: "" @@ -140,6 +247,11 @@ spec: - name: redis workloadType: Replication characterType: redis + probes: + roleProbe: + failureThreshold: 2 + periodSeconds: 2 + timeoutSeconds: 1 replicationSpec: switchPolicies: - type: MaximumAvailability @@ -181,8 +293,12 @@ spec: - redis-cli -h $(KB_SWITCH_ROLE_ENDPOINT) -p 6379 $(KB_SWITCH_DEMOTE_STATEMENT) service: ports: - - protocol: TCP + - name: redis port: 6379 + targetPort: redis + - name: metrics + port: 9121 + targetPort: metrics configSpecs: - name: redis-replication-config templateRef: redis7-config-template @@ -209,7 +325,6 @@ spec: podSpec: containers: - name: redis - image: redis:7.0.5 ports: - name: redis containerPort: 6379 @@ -220,7 +335,9 @@ spec: mountPath: /etc/conf - name: scripts mountPath: /scripts - args: [ "/etc/conf/redis.conf" ] + - name: redis-conf + mountPath: /etc/redis + command: ["/scripts/redis-start.sh"] lifecycle: postStart: exec: @@ -235,15 +352,14 @@ spec: ports: - name: metrics containerPort: 9121 - protocol: TCP livenessProbe: httpGet: path: / - port: 9121 + port: metrics readinessProbe: httpGet: path: / - port: 9121 + port: metrics systemAccounts: # Seems redis-cli has its own mechanism to parse input tokens and there is no elegent way # to pass $(KB_ACCOUNT_STATEMENT) to redis-cli without causing parsing error. @@ -264,44 +380,125 @@ spec: - name: kbadmin provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) allcommands allkeys + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys - name: kbdataprotection provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) allcommands allkeys + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys - name: kbmonitoring provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) allkeys +get + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get - name: kbprobe provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) allkeys +get + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get - name: kbreplicator provisionPolicy: type: CreateByStmt - scope: AnyPods + scope: AllPods statements: - creation: ACL SETUSER $(USERNAME) ON >$(PASSWD) +psync +replconf +ping + creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) +psync +replconf +ping + - name: redis-sentinel + workloadType: Stateful + characterType: redis + service: + ports: + - name: redis-sentinel + targetPort: redis-sentinel + port: 26379 + configSpecs: + - name: redis-replication-config + templateRef: redis7-config-template + constraintRef: redis7-config-constraints + namespace: default + volumeName: redis-config + scriptSpecs: + - name: redis-scripts + templateRef: redis-scripts + namespace: default + volumeName: scripts + defaultMode: 493 + volumeTypes: + - name: data + type: data + podSpec: + initContainers: + - name: init-redis-sentinel + imagePullPolicy: IfNotPresent + volumeMounts: + - name: data + mountPath: /data + - name: redis-config + mountPath: /etc/conf + - name: sentinel-conf + mountPath: /etc/sentinel + - name: scripts + mountPath: /scripts + command: [ "/scripts/redis-sentinel-setup.sh" ] + containers: + - name: redis-sentinel + imagePullPolicy: IfNotPresent + ports: + - containerPort: 26379 + name: redis-sentinel + volumeMounts: + - name: data + mountPath: /data + - name: redis-config + mountPath: /etc/conf + - name: sentinel-conf + mountPath: /etc/sentinel + - name: scripts + mountPath: /scripts + command: + - /bin/bash + args: + - -c + - | + set -ex + /scripts/redis-sentinel-start.sh + livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + exec: + command: + - sh + - -c + - /scripts/redis-sentinel-ping.sh 5 + readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 5 + exec: + command: + - sh + - -c + - /scripts/redis-sentinel-ping.sh 1 --- # Source: redis/templates/clusterversion.yaml apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterVersion metadata: - name: redis-7.0.5 + name: redis-7.0.6 labels: - helm.sh/chart: redis-0.5.0-alpha.3 + helm.sh/chart: redis-0.5.0-alpha.8 app.kubernetes.io/name: redis app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.5" + app.kubernetes.io/version: "7.0.6" app.kubernetes.io/managed-by: Helm spec: clusterDefinitionRef: redis @@ -310,8 +507,18 @@ spec: versionsContext: containers: - name: redis - image: redis:7.0.5 + image: redis/redis-stack-server:7.0.6-RC8 imagePullPolicy: IfNotPresent + - componentDefRef: redis-sentinel + versionsContext: + initContainers: + - name: init-redis-sentinel + image: redis/redis-stack-server:7.0.6-RC8 + imagePullPolicy: IfNotPresent + containers: + - name: redis-sentinel + image: redis/redis-stack-server:7.0.6-RC8 + imagePullPolicy: IfNotPresent --- # Source: redis/templates/configconstraint.yaml apiVersion: apps.kubeblocks.io/v1alpha1 @@ -319,10 +526,10 @@ kind: ConfigConstraint metadata: name: redis7-config-constraints labels: - helm.sh/chart: redis-0.5.0-alpha.3 + helm.sh/chart: redis-0.5.0-alpha.8 app.kubernetes.io/name: redis app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.5" + app.kubernetes.io/version: "7.0.6" app.kubernetes.io/managed-by: Helm spec: @@ -475,6 +682,10 @@ spec: "zset-max-listpack-value": int | *64 + "protected-mode"?: string & "yes" | "no" + + "enable-debug-command"?: string & "yes" | "no" | "local" + ... } @@ -500,27 +711,52 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: redis-cluster-0.5.0-alpha.3 + helm.sh/chart: redis-cluster-0.5.0-alpha.8 app.kubernetes.io/name: redis-cluster app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.5" + app.kubernetes.io/version: "7.0.6" app.kubernetes.io/managed-by: Helm spec: clusterDefinitionRef: redis # ref clusterDefinition.name - clusterVersionRef: redis-7.0.5 # ref clusterVersion.name + clusterVersionRef: redis-7.0.6 # ref clusterVersion.name terminationPolicy: Delete affinity: topologyKeys: - kubernetes.io/hostname componentSpecs: - - name: redis-repl # user-defined + - name: redis # user-defined componentDefRef: redis # ref clusterDefinition componentDefs.name monitor: false enabledLogs: ["running"] replicas: 2 primaryIndex: 0 switchPolicy: - type: MaximumAvailability + type: Noop + resources: + limits: + cpu: "500m" + memory: "3Gi" + requests: + cpu: "500m" + memory: "1Gi" + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - name: redis-sentinel # user-defined + componentDefRef: redis-sentinel # ref clusterDefinition componentDefs.name + replicas: 3 + resources: + limits: + cpu: "500m" + memory: "3Gi" + requests: + cpu: "500m" + memory: "1Gi" volumeClaimTemplates: - name: data # ref clusterdefinition components.containers.volumeMounts.name spec: diff --git a/test/e2e/testdata/smoketest/redis/01_vscale.yaml b/test/e2e/testdata/smoketest/redis/01_vscale.yaml index 18ddbd8de..12f474466 100644 --- a/test/e2e/testdata/smoketest/redis/01_vscale.yaml +++ b/test/e2e/testdata/smoketest/redis/01_vscale.yaml @@ -6,7 +6,7 @@ spec: clusterRef: mycluster type: VerticalScaling verticalScaling: - - componentName: redis-repl + - componentName: redis requests: memory: "500Mi" cpu: "500m" diff --git a/test/e2e/testdata/smoketest/redis/02_hscale_up.yaml b/test/e2e/testdata/smoketest/redis/02_hscale_up.yaml index e7897b2e7..3a268beb1 100644 --- a/test/e2e/testdata/smoketest/redis/02_hscale_up.yaml +++ b/test/e2e/testdata/smoketest/redis/02_hscale_up.yaml @@ -1,10 +1,10 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: OpsRequest metadata: - name: ops-hscale + name: ops-hscale-up spec: clusterRef: mycluster type: HorizontalScaling horizontalScaling: - - componentName: redis-repl + - componentName: redis replicas: 3 \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/03_hscale_down.yaml b/test/e2e/testdata/smoketest/redis/03_hscale_down.yaml index feeebb615..bca2e520d 100644 --- a/test/e2e/testdata/smoketest/redis/03_hscale_down.yaml +++ b/test/e2e/testdata/smoketest/redis/03_hscale_down.yaml @@ -1,10 +1,10 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: OpsRequest metadata: - name: ops-hscale + name: ops-hscale-down spec: clusterRef: mycluster type: HorizontalScaling horizontalScaling: - - componentName: redis-repl + - componentName: redis replicas: 2 \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/04_stop.yaml b/test/e2e/testdata/smoketest/redis/04_stop.yaml new file mode 100644 index 000000000..c8f7bbaf2 --- /dev/null +++ b/test/e2e/testdata/smoketest/redis/04_stop.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-stop +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 3600 + type: Stop + restart: + - componentName: redis \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/05_start.yaml b/test/e2e/testdata/smoketest/redis/05_start.yaml new file mode 100644 index 000000000..83ebbd60b --- /dev/null +++ b/test/e2e/testdata/smoketest/redis/05_start.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-start +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 3600 + type: Start + restart: + - componentName: redis \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/04_cv.yaml b/test/e2e/testdata/smoketest/redis/06_cv.yaml similarity index 91% rename from test/e2e/testdata/smoketest/redis/04_cv.yaml rename to test/e2e/testdata/smoketest/redis/06_cv.yaml index a66882dfd..8c5d177d8 100644 --- a/test/e2e/testdata/smoketest/redis/04_cv.yaml +++ b/test/e2e/testdata/smoketest/redis/06_cv.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterVersion metadata: - name: redis-7.0.5-latest + name: redis-7.0.6-latest spec: clusterDefinitionRef: redis componentVersions: diff --git a/test/e2e/testdata/smoketest/redis/05_upgrade.yaml b/test/e2e/testdata/smoketest/redis/07_upgrade.yaml similarity index 77% rename from test/e2e/testdata/smoketest/redis/05_upgrade.yaml rename to test/e2e/testdata/smoketest/redis/07_upgrade.yaml index d4935e0b7..43ca5a22a 100644 --- a/test/e2e/testdata/smoketest/redis/05_upgrade.yaml +++ b/test/e2e/testdata/smoketest/redis/07_upgrade.yaml @@ -6,4 +6,4 @@ spec: clusterRef: mycluster type: Upgrade upgrade: - clusterVersionRef: redis-7.0.5-latest \ No newline at end of file + clusterVersionRef: redis-7.0.6-latest \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/06_restart.yaml b/test/e2e/testdata/smoketest/redis/08_restart.yaml similarity index 100% rename from test/e2e/testdata/smoketest/redis/06_restart.yaml rename to test/e2e/testdata/smoketest/redis/08_restart.yaml diff --git a/test/e2e/testdata/smoketest/redis/09_vexpand.yaml b/test/e2e/testdata/smoketest/redis/09_vexpand.yaml new file mode 100644 index 000000000..ba32370ce --- /dev/null +++ b/test/e2e/testdata/smoketest/redis/09_vexpand.yaml @@ -0,0 +1,12 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-vexpand +spec: + clusterRef: mycluster + type: VolumeExpansion + volumeExpansion: + - componentName: postgresql + volumeClaimTemplates: + - name: data + storage: "11Gi" \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml b/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml index 0de7e31dd..ec30ef49f 100644 --- a/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml +++ b/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml @@ -5,7 +5,7 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: apecloud-mysql-cluster-0.5.0-alpha.3 + helm.sh/chart: apecloud-mysql-cluster-0.5.0-alpha.8 app.kubernetes.io/name: apecloud-mysql-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "8.0.30" diff --git a/test/e2e/testdata/smoketest/wesql/01_custom_class.yaml b/test/e2e/testdata/smoketest/wesql/01_custom_class.yaml new file mode 100644 index 000000000..f149b6b15 --- /dev/null +++ b/test/e2e/testdata/smoketest/wesql/01_custom_class.yaml @@ -0,0 +1,20 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ComponentClassDefinition +metadata: + name: custom-class + labels: + class.kubeblocks.io/provider: kubeblocks + apps.kubeblocks.io/component-def-ref: mysql + clusterdefinition.kubeblocks.io/name: apecloud-mysql +spec: + groups: + - resourceConstraintRef: kb-resource-constraint-general + template: | + cpu: "{{ or .cpu 1 }}" + memory: "{{ or .memory 4 }}Gi" + vars: [ cpu, memory] + series: + - namingTemplate: "general-{{ .cpu }}c{{ .memory }}g" + classes: + - args: [ "1", "4" ] + - args: [ "1", "16"] \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/01_vscale.yaml b/test/e2e/testdata/smoketest/wesql/02_vscale.yaml similarity index 62% rename from test/e2e/testdata/smoketest/wesql/01_vscale.yaml rename to test/e2e/testdata/smoketest/wesql/02_vscale.yaml index 59b66036f..d869d2093 100644 --- a/test/e2e/testdata/smoketest/wesql/01_vscale.yaml +++ b/test/e2e/testdata/smoketest/wesql/02_vscale.yaml @@ -7,9 +7,5 @@ spec: type: VerticalScaling verticalScaling: - componentName: mysql - requests: - memory: "500Mi" - cpu: "0.5" - limits: - memory: "1000Mi" - cpu: "1" \ No newline at end of file + class: general-1c4g + diff --git a/test/e2e/testdata/smoketest/wesql/02_hscale.yaml b/test/e2e/testdata/smoketest/wesql/03_hscale.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/02_hscale.yaml rename to test/e2e/testdata/smoketest/wesql/03_hscale.yaml diff --git a/test/e2e/testdata/smoketest/wesql/03_vexpand.yaml b/test/e2e/testdata/smoketest/wesql/04_vexpand.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/03_vexpand.yaml rename to test/e2e/testdata/smoketest/wesql/04_vexpand.yaml diff --git a/test/e2e/testdata/smoketest/wesql/04_cv.yaml b/test/e2e/testdata/smoketest/wesql/05_cv.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/04_cv.yaml rename to test/e2e/testdata/smoketest/wesql/05_cv.yaml diff --git a/test/e2e/testdata/smoketest/wesql/05_upgrade.yaml b/test/e2e/testdata/smoketest/wesql/06_upgrade.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/05_upgrade.yaml rename to test/e2e/testdata/smoketest/wesql/06_upgrade.yaml diff --git a/test/e2e/testdata/smoketest/wesql/07_backuppolicy_snapshot.yaml b/test/e2e/testdata/smoketest/wesql/07_backuppolicy_snapshot.yaml deleted file mode 100644 index 66fd6c2b9..000000000 --- a/test/e2e/testdata/smoketest/wesql/07_backuppolicy_snapshot.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 -kind: BackupPolicy -metadata: - name: backup-policy-mycluster -spec: - backupToolName: xtrabackup-mysql - hooks: - postCommands: - - rm -f /data/mysql/data/.restore_new_cluster; sync - preCommands: - - touch /data/mysql/data/.restore_new_cluster; sync - remoteVolume: - name: backup-remote-volume - persistentVolumeClaim: - claimName: backup-host-path-pvc - schedule: 0 3 * * * - target: - labelsSelector: - matchLabels: - app.kubernetes.io/instance: mycluster - secret: - name: mycluster-conn-credential - ttl: 168h0m0s - diff --git a/test/e2e/testdata/smoketest/wesql/07_stop.yaml b/test/e2e/testdata/smoketest/wesql/07_stop.yaml new file mode 100644 index 000000000..8b039e4d6 --- /dev/null +++ b/test/e2e/testdata/smoketest/wesql/07_stop.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-stop +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 3600 + type: Stop + restart: + - componentName: mysql \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/08_start.yaml b/test/e2e/testdata/smoketest/wesql/08_start.yaml new file mode 100644 index 000000000..406a567bd --- /dev/null +++ b/test/e2e/testdata/smoketest/wesql/08_start.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-start +spec: + clusterRef: mycluster + ttlSecondsAfterSucceed: 3600 + type: Start + restart: + - componentName: mysql \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/06_restart.yaml b/test/e2e/testdata/smoketest/wesql/09_restart.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/06_restart.yaml rename to test/e2e/testdata/smoketest/wesql/09_restart.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/07_backup_snapshot.yaml b/test/e2e/testdata/smoketest/wesql/11_backup_snapshot.yaml similarity index 72% rename from test/e2e/testdata/smoketest/postgresql/07_backup_snapshot.yaml rename to test/e2e/testdata/smoketest/wesql/11_backup_snapshot.yaml index 65b22b227..749c83fed 100644 --- a/test/e2e/testdata/smoketest/postgresql/07_backup_snapshot.yaml +++ b/test/e2e/testdata/smoketest/wesql/11_backup_snapshot.yaml @@ -6,6 +6,5 @@ metadata: dataprotection.kubeblocks.io/backup-type: snapshot name: backup-sbapshot-mycluster spec: - backupPolicyName: backup-policy-mycluster - backupType: snapshot - ttl: 168h0m0s \ No newline at end of file + backupPolicyName: mycluster-mysql-backup-policy + backupType: snapshot \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/11_backuppolicy_full.yaml b/test/e2e/testdata/smoketest/wesql/11_backuppolicy_full.yaml deleted file mode 100644 index 587b00f2c..000000000 --- a/test/e2e/testdata/smoketest/wesql/11_backuppolicy_full.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 -kind: BackupPolicy -metadata: - name: backup-policy-mycluster-full -spec: - backupToolName: xtrabackup-mysql - backupType: full - hooks: - postCommands: - - rm -f /data/mysql/data/.restore_new_cluster; sync - preCommands: - - touch /data/mysql/data/.restore_new_cluster; sync - remoteVolume: - name: backup-remote-volume - persistentVolumeClaim: - claimName: backup-host-path-pvc - schedule: 0 3 * * * - target: - labelsSelector: - matchLabels: - app.kubernetes.io/instance: mycluster - secret: - name: mycluster-conn-credential - ttl: 168h0m0s - - diff --git a/test/e2e/testdata/smoketest/wesql/12_backup_full.yaml b/test/e2e/testdata/smoketest/wesql/12_backup_full.yaml index 31aaeb249..3b3b25e2e 100644 --- a/test/e2e/testdata/smoketest/wesql/12_backup_full.yaml +++ b/test/e2e/testdata/smoketest/wesql/12_backup_full.yaml @@ -6,6 +6,5 @@ metadata: dataprotection.kubeblocks.io/backup-type: full name: backup-full-mycluster spec: - backupPolicyName: backup-policy-mycluster-full - backupType: full - ttl: 168h0m0s \ No newline at end of file + backupPolicyName: mycluster-mysql-backup-policy + backupType: full \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/13_backup_full_restore.yaml b/test/e2e/testdata/smoketest/wesql/13_backup_full_restore.yaml deleted file mode 100644 index 09abd4007..000000000 --- a/test/e2e/testdata/smoketest/wesql/13_backup_full_restore.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: Cluster -metadata: - name: snapshot-mycluster - annotations: - kubeblocks.io/restore-from-backup: "{\"wesql\":\"backup-policy-mycluster-full\"}" -spec: - clusterDefinitionRef: apecloud-mysql - clusterVersionRef: ac-mysql-8.0.30 - terminationPolicy: WipeOut - affinity: - topologyKeys: - - kubernetes.io/hostname - componentSpecs: - - name: mysql - componentDefRef: mysql - monitor: false - replicas: 3 - enabledLogs: [ "slow","error" ] - volumeClaimTemplates: - - name: data - spec: - storageClassName: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi - dataSource: - apiGroup: snapshot.storage.k8s.io - kind: VolumeSnapshot - name: backup-full-mycluster \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/13_backup_snapshot_restore.yaml b/test/e2e/testdata/smoketest/wesql/13_backup_snapshot_restore.yaml new file mode 100644 index 000000000..cb92e7076 --- /dev/null +++ b/test/e2e/testdata/smoketest/wesql/13_backup_snapshot_restore.yaml @@ -0,0 +1,23 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + name: mycluster-sbapshot + annotations: + kubeblocks.io/restore-from-backup: "{\"mysql\":\"backup-sbapshot-mycluster\"}" +spec: + clusterDefinitionRef: postgresql + clusterVersionRef: postgresql-15.2.0 + terminationPolicy: WipeOut + componentSpecs: + - name: wesql + componentDefRef: mysql + monitor: false + replicas: 1 + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/09_backup_snapshot_restore.yaml b/test/e2e/testdata/smoketest/wesql/14_backup_full_restore.yaml similarity index 76% rename from test/e2e/testdata/smoketest/wesql/09_backup_snapshot_restore.yaml rename to test/e2e/testdata/smoketest/wesql/14_backup_full_restore.yaml index c8c7748ce..cd8866635 100644 --- a/test/e2e/testdata/smoketest/wesql/09_backup_snapshot_restore.yaml +++ b/test/e2e/testdata/smoketest/wesql/14_backup_full_restore.yaml @@ -2,6 +2,8 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: name: snapshot-mycluster + annotations: + kubeblocks.io/restore-from-backup: "{\"mysql\":\"backup-full-mycluster\"}" spec: clusterDefinitionRef: apecloud-mysql clusterVersionRef: ac-mysql-8.0.30 @@ -23,8 +25,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 2Gi - dataSource: - apiGroup: snapshot.storage.k8s.io - kind: VolumeSnapshot - name: backup-sbapshot-mycluster \ No newline at end of file + storage: 2Gi \ No newline at end of file From 0e3f824c82063a177075ad0dec99f97787c98a1f Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 17 Apr 2023 10:30:29 +0800 Subject: [PATCH 047/439] fix: trim single quotes for the parameters value in the pg config file (#2523) (#2527) --- deploy/postgresql/scripts/patroni-reload.tpl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy/postgresql/scripts/patroni-reload.tpl b/deploy/postgresql/scripts/patroni-reload.tpl index 6669eea7b..0e0380f69 100644 --- a/deploy/postgresql/scripts/patroni-reload.tpl +++ b/deploy/postgresql/scripts/patroni-reload.tpl @@ -1,12 +1,14 @@ {{- $bootstrap := $.Files.Get "bootstrap.yaml" | fromYamlArray }} {{- $command := "reload" }} -{{- range $pk, $_ := $.arg0 }} +{{- $trimParams := dict }} +{{- range $pk, $val := $.arg0 }} + {{- /* trim single quotes for value in the pg config file */}} + {{- set $trimParams $pk ( $val | trimAll "'" ) }} {{- if has $pk $bootstrap }} {{- $command = "restart" }} - {{ break }} {{- end }} {{- end }} -{{ $params := dict "parameters" $.arg0 }} +{{ $params := dict "parameters" $trimParams }} {{- $err := execSql ( dict "postgresql" $params | toJson ) "config" }} {{- if $err }} {{- failed $err }} From dcb1af841fc2c53c1ef545eac48010dde49a1a8c Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 17 Apr 2023 10:43:04 +0800 Subject: [PATCH 048/439] fix: config change does not take effect (#2511) (#2543) --- .../apps/configuration/config_annotation.go | 1 - .../reconfigurerequest_controller_test.go | 5 +- controllers/apps/tls_utils_test.go | 7 +- internal/configuration/reconfigure_util.go | 34 ++++++++- .../configuration/reconfigure_util_test.go | 51 +++++++++++++ internal/constant/const.go | 3 + .../lifecycle/transformer_config.go | 14 ++-- internal/controller/plan/template_wrapper.go | 72 +++++++++++++------ 8 files changed, 151 insertions(+), 36 deletions(-) diff --git a/controllers/apps/configuration/config_annotation.go b/controllers/apps/configuration/config_annotation.go index 0f4a1322d..a37de3310 100644 --- a/controllers/apps/configuration/config_annotation.go +++ b/controllers/apps/configuration/config_annotation.go @@ -104,7 +104,6 @@ func updateAppliedConfigs(cli client.Client, ctx intctrlutil.RequestCtx, config // delete reconfigure-policy delete(config.ObjectMeta.Annotations, constant.UpgradePolicyAnnotationKey) - delete(config.ObjectMeta.Annotations, constant.KBParameterUpdateSourceAnnotationKey) if err := cli.Patch(ctx.Ctx, config, patch); err != nil { return false, err } diff --git a/controllers/apps/configuration/reconfigurerequest_controller_test.go b/controllers/apps/configuration/reconfigurerequest_controller_test.go index 7fd984aa4..9a878776c 100644 --- a/controllers/apps/configuration/reconfigurerequest_controller_test.go +++ b/controllers/apps/configuration/reconfigurerequest_controller_test.go @@ -83,7 +83,10 @@ var _ = Describe("Reconfigure Controller", func() { constant.CMConfigurationConstraintsNameLabelKey, cmName, constant.CMConfigurationSpecProviderLabelKey, configSpecName, constant.CMConfigurationTypeLabelKey, constant.ConfigInstanceType, - )) + ), + testapps.WithAnnotations(constant.KBParameterUpdateSourceAnnotationKey, + constant.ReconfigureManagerSource, + constant.CMInsEnableRerenderTemplateKey, "true")) constraint := testapps.CreateCustomizedObj(&testCtx, "resources/mysql-config-constraint.yaml", diff --git a/controllers/apps/tls_utils_test.go b/controllers/apps/tls_utils_test.go index e3899f621..54cb1b6f2 100644 --- a/controllers/apps/tls_utils_test.go +++ b/controllers/apps/tls_utils_test.go @@ -30,6 +30,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/plan" "github.com/apecloud/kubeblocks/internal/generics" testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" @@ -80,7 +81,8 @@ var _ = Describe("TLS self-signed cert function", func() { configMapObj := testapps.CheckedCreateCustomizedObj(&testCtx, "resources/mysql-tls-config-template.yaml", &corev1.ConfigMap{}, - testCtx.UseDefaultNamespace()) + testCtx.UseDefaultNamespace(), + testapps.WithAnnotations(constant.CMInsEnableRerenderTemplateKey, "true")) configConstraintObj := testapps.CheckedCreateCustomizedObj(&testCtx, "resources/mysql-config-constraint.yaml", @@ -293,6 +295,9 @@ var _ = Describe("TLS self-signed cert function", func() { Expect(k8sClient.Get(ctx, types.NamespacedName{Name: clusterDefName, Namespace: testCtx.DefaultNamespace}, cd)).Should(Succeed()) cmName := cfgcore.GetInstanceCMName(&sts, &cd.Spec.ComponentDefs[0].ConfigSpecs[0].ComponentTemplateSpec) cmKey := client.ObjectKey{Namespace: sts.Namespace, Name: cmName} + Eventually(testapps.GetAndChangeObj(&testCtx, cmKey, func(cm *corev1.ConfigMap) { + cm.Annotations[constant.CMInsEnableRerenderTemplateKey] = "true" + })).Should(Succeed()) hasTLSSettings := func() bool { cm := &corev1.ConfigMap{} Expect(k8sClient.Get(ctx, cmKey, cm)).Should(Succeed()) diff --git a/internal/configuration/reconfigure_util.go b/internal/configuration/reconfigure_util.go index 98f353c21..919057cc0 100644 --- a/internal/configuration/reconfigure_util.go +++ b/internal/configuration/reconfigure_util.go @@ -23,6 +23,7 @@ import ( "github.com/StudioSol/set" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/configuration/util" @@ -123,10 +124,17 @@ func IsParametersUpdateFromManager(cm *corev1.ConfigMap) bool { // IsNotUserReconfigureOperation is used to check whether the parameters are updated from operation func IsNotUserReconfigureOperation(cm *corev1.ConfigMap) bool { labels := cm.GetLabels() - if labels == nil { - return true + annotations := cm.GetAnnotations() + if labels == nil || annotations == nil { + return false + } + if _, ok := annotations[constant.CMInsEnableRerenderTemplateKey]; !ok { + return false } lastReconfigurePhase := labels[constant.CMInsLastReconfigurePhaseKey] + if annotations[constant.KBParameterUpdateSourceAnnotationKey] != constant.ReconfigureManagerSource { + return false + } return lastReconfigurePhase == "" || ReconfigureCreatedPhase == lastReconfigurePhase } @@ -142,3 +150,25 @@ func SetParametersUpdateSource(cm *corev1.ConfigMap, source string) { annotation[constant.KBParameterUpdateSourceAnnotationKey] = source cm.SetAnnotations(annotation) } + +func IsSchedulableConfigResource(object client.Object) bool { + var requiredLabels = []string{ + constant.AppNameLabelKey, + constant.AppInstanceLabelKey, + constant.KBAppComponentLabelKey, + constant.CMConfigurationTemplateNameLabelKey, + constant.CMConfigurationTypeLabelKey, + constant.CMConfigurationSpecProviderLabelKey, + } + + labels := object.GetLabels() + if len(labels) == 0 { + return false + } + for _, label := range requiredLabels { + if _, ok := labels[label]; !ok { + return false + } + } + return true +} diff --git a/internal/configuration/reconfigure_util_test.go b/internal/configuration/reconfigure_util_test.go index 956caf224..4935bc174 100644 --- a/internal/configuration/reconfigure_util_test.go +++ b/internal/configuration/reconfigure_util_test.go @@ -21,9 +21,13 @@ import ( "github.com/StudioSol/set" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/configuration/util" + "github.com/apecloud/kubeblocks/internal/constant" ) func TestGetUpdateParameterList(t *testing.T) { @@ -183,3 +187,50 @@ func TestIsUpdateDynamicParameters(t *testing.T) { }) } } + +func TestIsSchedulableConfigResource(t *testing.T) { + tests := []struct { + name string + object client.Object + want bool + }{{ + name: "test", + object: &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{}}, + want: false, + }, { + name: "test", + object: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constant.AppNameLabelKey: "test", + constant.AppInstanceLabelKey: "test", + constant.KBAppComponentLabelKey: "component", + }, + }, + }, + want: false, + }, { + name: "test", + object: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + constant.AppNameLabelKey: "test", + constant.AppInstanceLabelKey: "test", + constant.KBAppComponentLabelKey: "component", + constant.CMConfigurationTemplateNameLabelKey: "test_config_template", + constant.CMConfigurationConstraintsNameLabelKey: "test_config_constraint", + constant.CMConfigurationSpecProviderLabelKey: "for_test_config", + constant.CMConfigurationTypeLabelKey: constant.ConfigInstanceType, + }, + }, + }, + want: true, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsSchedulableConfigResource(tt.object); got != tt.want { + t.Errorf("IsSchedulableConfigResource() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/constant/const.go b/internal/constant/const.go index 2ce158c6b..8df2aa2d8 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -114,6 +114,9 @@ const ( // CMInsLastReconfigurePhaseKey defines the current phase CMInsLastReconfigurePhaseKey = "config.kubeblocks.io/last-applied-reconfigure-phase" + // CMInsEnableRerenderTemplateKey is used to enable rerender template + CMInsEnableRerenderTemplateKey = "config.kubeblocks.io/enable-rerender" + // configuration finalizer ConfigurationTemplateFinalizerName = "config.kubeblocks.io/finalizer" diff --git a/internal/controller/lifecycle/transformer_config.go b/internal/controller/lifecycle/transformer_config.go index 18d2e774b..ede95c6ba 100644 --- a/internal/controller/lifecycle/transformer_config.go +++ b/internal/controller/lifecycle/transformer_config.go @@ -19,6 +19,7 @@ package lifecycle import ( corev1 "k8s.io/api/core/v1" + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" "github.com/apecloud/kubeblocks/internal/controller/graph" ) @@ -26,17 +27,12 @@ import ( type configTransformer struct{} func (c *configTransformer) Transform(dag *graph.DAG) error { - cmVertices := findAll[*corev1.ConfigMap](dag) - isConfig := func(cm *corev1.ConfigMap) bool { - // TODO: we should find a way to know if cm is a true config - // TODO: the main problem is we can't separate script from config, - // TODO: as componentDef.ConfigSpec defines them in same way - return false - } - for _, vertex := range cmVertices { + for _, vertex := range findAll[*corev1.ConfigMap](dag) { v, _ := vertex.(*lifecycleVertex) cm, _ := v.obj.(*corev1.ConfigMap) - if isConfig(cm) { + // Note: Disable updating of the config resources. + // Labels and Annotations have the necessary meta information for controller. + if cfgcore.IsSchedulableConfigResource(cm) { v.immutable = true } } diff --git a/internal/controller/plan/template_wrapper.go b/internal/controller/plan/template_wrapper.go index 174908b93..c531b86c0 100644 --- a/internal/controller/plan/template_wrapper.go +++ b/internal/controller/plan/template_wrapper.go @@ -18,6 +18,8 @@ package plan import ( "context" + "encoding/json" + "reflect" "strings" corev1 "k8s.io/api/core/v1" @@ -62,7 +64,7 @@ func newTemplateRenderWrapper(templateBuilder *configTemplateBuilder, cluster *a } } -func (wrapper *renderWrapper) enableRerenderTemplateSpec(cfgCMName string, task *intctrltypes.ReconcileTask) (bool, error) { +func (wrapper *renderWrapper) checkRerenderTemplateSpec(cfgCMName string, task *intctrltypes.ReconcileTask) (bool, *corev1.ConfigMap, error) { cmKey := client.ObjectKey{ Name: cfgCMName, Namespace: wrapper.cluster.Namespace, @@ -71,48 +73,50 @@ func (wrapper *renderWrapper) enableRerenderTemplateSpec(cfgCMName string, task cmObj := &corev1.ConfigMap{} localObject := task.GetLocalResourceWithObjectKey(cmKey, generics.ToGVK(cmObj)) if localObject != nil { - return false, nil + if cm, ok := localObject.(*corev1.ConfigMap); ok { + return false, cm, nil + } } cmErr := wrapper.cli.Get(wrapper.ctx, cmKey, cmObj) if cmErr != nil && !apierrors.IsNotFound(cmErr) { // An unexpected error occurs - return false, cmErr + return false, nil, cmErr } if cmErr != nil { // Config is not exists - return true, nil + return true, nil, nil } // Config is exists - return cfgcore.IsNotUserReconfigureOperation(cmObj), nil + return cfgcore.IsNotUserReconfigureOperation(cmObj), cmObj, nil } func (wrapper *renderWrapper) renderConfigTemplate(task *intctrltypes.ReconcileTask) error { - var err error - var enableRerender bool - scheme, _ := appsv1alpha1.SchemeBuilder.Build() for _, configSpec := range task.Component.ConfigTemplates { cmName := cfgcore.GetComponentCfgName(task.Cluster.Name, task.Component.Name, configSpec.Name) - if enableRerender, err = wrapper.enableRerenderTemplateSpec(cmName, task); err != nil { + enableRerender, origCMObj, err := wrapper.checkRerenderTemplateSpec(cmName, task) + if err != nil { return err } if !enableRerender { - wrapper.addVolumeMountMeta(configSpec.ComponentTemplateSpec, cmName) + wrapper.addVolumeMountMeta(configSpec.ComponentTemplateSpec, origCMObj) continue } - // Generate ConfigMap objects for config files - cm, err := generateConfigMapFromTpl(wrapper.templateBuilder, cmName, configSpec.ConfigConstraintRef, configSpec.ComponentTemplateSpec, + newCMObj, err := generateConfigMapFromTpl(wrapper.templateBuilder, cmName, configSpec.ConfigConstraintRef, configSpec.ComponentTemplateSpec, wrapper.params, wrapper.ctx, wrapper.cli, func(m map[string]string) error { return validateRenderedData(m, configSpec, wrapper.ctx, wrapper.cli) }) if err != nil { return err } - updateCMConfigSpecLabels(cm, configSpec) - if err := wrapper.addRenderedObject(configSpec.ComponentTemplateSpec, cm, scheme); err != nil { + if err := wrapper.checkAndPatchConfigResource(origCMObj, newCMObj.Data); err != nil { + return err + } + updateCMConfigSpecLabels(newCMObj, configSpec) + if err := wrapper.addRenderedObject(configSpec.ComponentTemplateSpec, newCMObj, scheme); err != nil { return err } } @@ -123,11 +127,12 @@ func (wrapper *renderWrapper) renderScriptTemplate(task *intctrltypes.ReconcileT scheme, _ := appsv1alpha1.SchemeBuilder.Build() for _, templateSpec := range task.Component.ScriptTemplates { cmName := cfgcore.GetComponentCfgName(task.Cluster.Name, task.Component.Name, templateSpec.Name) - if task.GetLocalResourceWithObjectKey(client.ObjectKey{ + object := task.GetLocalResourceWithObjectKey(client.ObjectKey{ Name: cmName, Namespace: wrapper.cluster.Namespace, - }, generics.ToGVK(&corev1.ConfigMap{})) != nil { - wrapper.addVolumeMountMeta(templateSpec, cmName) + }, generics.ToGVK(&corev1.ConfigMap{})) + if object != nil { + wrapper.addVolumeMountMeta(templateSpec, object) continue } @@ -151,14 +156,37 @@ func (wrapper *renderWrapper) addRenderedObject(templateSpec appsv1alpha1.Compon } cfgcore.SetParametersUpdateSource(cm, constant.ReconfigureManagerSource) - wrapper.renderedObjs = append(wrapper.renderedObjs, cm) - wrapper.addVolumeMountMeta(templateSpec, cm.Name) + wrapper.addVolumeMountMeta(templateSpec, cm) return nil } -func (wrapper *renderWrapper) addVolumeMountMeta(templateSpec appsv1alpha1.ComponentTemplateSpec, cmName string) { - wrapper.volumes[cmName] = templateSpec - wrapper.templateAnnotations[cfgcore.GenerateTPLUniqLabelKeyWithConfig(templateSpec.Name)] = cmName +func (wrapper *renderWrapper) addVolumeMountMeta(templateSpec appsv1alpha1.ComponentTemplateSpec, object client.Object) { + wrapper.volumes[object.GetName()] = templateSpec + wrapper.renderedObjs = append(wrapper.renderedObjs, object) + wrapper.templateAnnotations[cfgcore.GenerateTPLUniqLabelKeyWithConfig(templateSpec.Name)] = object.GetName() +} + +func (wrapper *renderWrapper) checkAndPatchConfigResource(origCMObj *corev1.ConfigMap, newData map[string]string) error { + if origCMObj == nil { + return nil + } + if reflect.DeepEqual(origCMObj.Data, newData) { + return nil + } + + patch := client.MergeFrom(origCMObj.DeepCopy()) + origCMObj.Data = newData + if origCMObj.Annotations == nil { + origCMObj.Annotations = make(map[string]string) + } + cfgcore.SetParametersUpdateSource(origCMObj, constant.ReconfigureManagerSource) + rawData, err := json.Marshal(origCMObj.Data) + if err != nil { + return err + } + + origCMObj.Annotations[corev1.LastAppliedConfigAnnotation] = string(rawData) + return wrapper.cli.Patch(wrapper.ctx, origCMObj, patch) } func updateCMConfigSpecLabels(cm *corev1.ConfigMap, configSpec appsv1alpha1.ComponentConfigSpec) { From de0df179908bf293205d98c0cb4ee576fd9a1b86 Mon Sep 17 00:00:00 2001 From: ToKliar <52400562+ToKliar@users.noreply.github.com> Date: Mon, 17 Apr 2023 11:17:18 +0800 Subject: [PATCH 049/439] feat: add replication metrics in postgresql dashboard (#2610) (#2611) --- .../helm/dashboards/postgresql-overview.json | 591 +++++++++++++++++- 1 file changed, 557 insertions(+), 34 deletions(-) diff --git a/deploy/helm/dashboards/postgresql-overview.json b/deploy/helm/dashboards/postgresql-overview.json index e4276ff32..bd409740a 100644 --- a/deploy/helm/dashboards/postgresql-overview.json +++ b/deploy/helm/dashboards/postgresql-overview.json @@ -1495,8 +1495,7 @@ "mode": "absolute", "steps": [ { - "color": "dark-green", - "value": null + "color": "dark-green" } ] }, @@ -1709,7 +1708,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 2 }, "id": 413, "links": [], @@ -1818,7 +1817,7 @@ "h": 8, "w": 12, "x": 12, - "y": 31 + "y": 2 }, "id": 414, "links": [], @@ -1941,7 +1940,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 3 }, "id": 394, "links": [], @@ -2050,7 +2049,7 @@ "h": 8, "w": 12, "x": 12, - "y": 32 + "y": 3 }, "id": 395, "links": [], @@ -2159,7 +2158,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 11 }, "id": 396, "links": [], @@ -2268,7 +2267,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 11 }, "id": 397, "links": [], @@ -2377,7 +2376,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 19 }, "id": 398, "links": [], @@ -2500,7 +2499,7 @@ "h": 8, "w": 12, "x": 0, - "y": 33 + "y": 4 }, "id": 432, "links": [], @@ -2611,7 +2610,7 @@ "h": 8, "w": 12, "x": 12, - "y": 33 + "y": 4 }, "id": 434, "links": [], @@ -2722,7 +2721,7 @@ "h": 8, "w": 12, "x": 0, - "y": 41 + "y": 12 }, "id": 433, "links": [], @@ -2833,7 +2832,7 @@ "h": 8, "w": 12, "x": 12, - "y": 41 + "y": 12 }, "id": 435, "links": [], @@ -2944,7 +2943,7 @@ "h": 8, "w": 12, "x": 0, - "y": 49 + "y": 20 }, "id": 436, "links": [], @@ -3055,7 +3054,7 @@ "h": 8, "w": 12, "x": 12, - "y": 49 + "y": 20 }, "id": 437, "links": [], @@ -3180,7 +3179,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 5 }, "id": 389, "links": [], @@ -3289,7 +3288,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 5 }, "id": 390, "links": [], @@ -3398,7 +3397,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 13 }, "id": 391, "links": [], @@ -3507,7 +3506,7 @@ "h": 8, "w": 12, "x": 12, - "y": 42 + "y": 13 }, "id": 423, "links": [], @@ -3652,7 +3651,7 @@ "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 6 }, "id": 92, "links": [], @@ -3761,7 +3760,7 @@ "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 6 }, "id": 384, "links": [], @@ -3870,7 +3869,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 14 }, "id": 385, "links": [], @@ -3979,7 +3978,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 14 }, "id": 386, "links": [], @@ -4186,7 +4185,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 7 }, "id": 408, "links": [], @@ -4295,7 +4294,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 7 }, "id": 409, "links": [], @@ -4404,7 +4403,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 15 }, "id": 401, "links": [], @@ -4513,7 +4512,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 15 }, "id": 402, "links": [], @@ -4622,7 +4621,7 @@ "h": 8, "w": 12, "x": 0, - "y": 52 + "y": 23 }, "id": 403, "links": [], @@ -4807,7 +4806,7 @@ "h": 8, "w": 12, "x": 12, - "y": 52 + "y": 23 }, "id": 410, "links": [], @@ -4935,7 +4934,7 @@ "h": 8, "w": 12, "x": 0, - "y": 60 + "y": 31 }, "id": 415, "links": [], @@ -5077,7 +5076,7 @@ "h": 8, "w": 12, "x": 0, - "y": 37 + "y": 16 }, "id": 406, "links": [], @@ -5186,7 +5185,7 @@ "h": 8, "w": 12, "x": 12, - "y": 37 + "y": 16 }, "id": 407, "links": [], @@ -5309,7 +5308,7 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 17 }, "id": 418, "links": [], @@ -5357,6 +5356,530 @@ ], "title": "Database Size", "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 459, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 486, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "2m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "expr": "sum(pg_replication_is_master{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}) by (namespace, app_kubernetes_io_instance, pod)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} ", + "metric": "", + "range": true, + "refId": "A", + "step": 20 + } + ], + "title": "Master Role", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 484, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "2m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "expr": "sum (pg_replication_slots_pg_wal_lsn_diff{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\"}) by (namespace, app_kubernetes_io_instance, slot_name)", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{slot_name}} ", + "metric": "", + "range": true, + "refId": "A", + "step": 20 + } + ], + "title": "Replication Lag Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 483, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "2m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "expr": "sum(pg_replication_lag{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}) by (namespace, app_kubernetes_io_instance, pod) * 1000", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}} ", + "metric": "", + "range": true, + "refId": "A", + "step": 20 + } + ], + "title": "Replication Lag Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 463, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(pg_replication_slots_active{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\"}) by (namespace, app_kubernetes_io_instance)", + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Replication Slots", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 485, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "calculatedInterval": "2m", + "datasource": { + "uid": "$datasource" + }, + "datasourceErrors": {}, + "editorMode": "code", + "errors": {}, + "expr": "(time() - sum(pg_stat_replication_reply_time{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\"}) by (namespace, app_kubernetes_io_instance, application_name)) < bool 2000", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{application_name}} ", + "metric": "", + "range": true, + "refId": "A", + "step": 20 + } + ], + "title": "Replication Status", + "type": "timeseries" + } + ], + "title": "Replication", + "type": "row" } ], "refresh": "", @@ -5569,4 +6092,4 @@ "uid": "5UxloIJVk", "version": 1, "weekStart": "" -} +} \ No newline at end of file From 70f2d0ccf1a7428e76173e1787aa77afa00824ac Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Mon, 17 Apr 2023 11:22:23 +0800 Subject: [PATCH 050/439] chore: fix release image error (#2620) --- .github/workflows/cicd-pull-request.yml | 4 ++-- .github/workflows/cicd-push.yml | 6 +++--- .github/workflows/release-image.yml | 4 ++-- .github/workflows/release-publish.yml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index 51e2f34e0..e57c90d37 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -73,7 +73,7 @@ jobs: MAKE_OPS: "build-manager-image" IMG: "apecloud/kubeblocks" VERSION: "check" - GO_VERSION: 1.20 + GO_VERSION: "1.20" secrets: inherit check-tools-image: @@ -86,7 +86,7 @@ jobs: MAKE_OPS: "build-tools-image" IMG: "apecloud/kubeblocks-tools" VERSION: "check" - GO_VERSION: 1.20 + GO_VERSION: "1.20" secrets: inherit check-helm: diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index 3661c19b2..5f7037653 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -81,7 +81,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: 1.20 + go-version: "1.20" - name: Check cli doc id: check-cli-doc @@ -187,7 +187,7 @@ jobs: MAKE_OPS: "build-manager-image" IMG: "apecloud/kubeblocks" VERSION: "check" - GO_VERSION: 1.20 + GO_VERSION: "1.20" secrets: inherit check-tools-image: @@ -200,7 +200,7 @@ jobs: MAKE_OPS: "build-tools-image" IMG: "apecloud/kubeblocks-tools" VERSION: "check" - GO_VERSION: 1.20 + GO_VERSION: "1.20" secrets: inherit check-helm: diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index d9be93882..04df61eb1 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -44,7 +44,7 @@ jobs: MAKE_OPS: "push-manager-image" IMG: "apecloud/kubeblocks" VERSION: "${{ needs.image-tag.outputs.tag-name }}" - GO_VERSION: 1.20 + GO_VERSION: "1.20" secrets: inherit release-tools-image: @@ -55,5 +55,5 @@ jobs: MAKE_OPS: "push-tools-image" IMG: "apecloud/kubeblocks-tools" VERSION: "${{ needs.image-tag.outputs.tag-name }}" - GO_VERSION: 1.20 + GO_VERSION: "1.20" secrets: inherit diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 79197d165..b4a09985a 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -8,7 +8,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} TAG_NAME: ${{ github.ref_name }} - GO_VERSION: '1.20' + GO_VERSION: "1.20" CLI_NAME: 'kbcli' CLI_REPO: 'apecloud/kbcli' GITLAB_KBCLI_PROJECT_ID: 85948 From 87db72c8f1675103281389dab67b8e45f1c07020 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Mon, 17 Apr 2023 13:24:44 +0800 Subject: [PATCH 051/439] fix: cannot use syscall.Stdin (variable of type "syscall".Handle) (#2621) --- internal/cli/cmd/cluster/connect.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cli/cmd/cluster/connect.go b/internal/cli/cmd/cluster/connect.go index f5f8317f1..3c55dc184 100644 --- a/internal/cli/cmd/cluster/connect.go +++ b/internal/cli/cmd/cluster/connect.go @@ -19,8 +19,8 @@ package cluster import ( "context" "fmt" + "os" "strings" - "syscall" "github.com/spf13/cobra" "golang.org/x/crypto/ssh/terminal" @@ -172,7 +172,7 @@ func (o *ConnectOptions) validate(args []string) error { if len(o.userName) > 0 { // read password from stdin fmt.Print("Password: ") - if bytePassword, err := terminal.ReadPassword(syscall.Stdin); err != nil { + if bytePassword, err := terminal.ReadPassword(int(os.Stdin.Fd())); err != nil { return err } else { o.userPasswd = string(bytePassword) From 1ee11291ea83ab75e16364ff9a3e39e132153d39 Mon Sep 17 00:00:00 2001 From: Nayuta <111858489+nayutah@users.noreply.github.com> Date: Mon, 17 Apr 2023 13:50:37 +0800 Subject: [PATCH 052/439] feat: Support/docs for chatgpt plugin (#2579) --- .../templates/deployment.yaml | 8 --- .../addons/chatgpt-retrieval-plugin.yaml | 39 ------------ .../helm/templates/addons/milvus-addon.yaml | 2 +- .../helm/templates/addons/qdrant-addon.yaml | 2 +- deploy/helm/templates/addons/redis-addon.yaml | 2 +- .../helm/templates/addons/weaviate-addon.yaml | 2 +- .../kubeblocks-for-gptplugin/Installation.md | 63 +++++++++++++++++++ .../kubeblocks-for-gptplugin/Introduction.md | 13 ++++ 8 files changed, 80 insertions(+), 51 deletions(-) delete mode 100644 deploy/helm/templates/addons/chatgpt-retrieval-plugin.yaml create mode 100644 docs/user_docs/kubeblocks-for-gptplugin/Installation.md create mode 100644 docs/user_docs/kubeblocks-for-gptplugin/Introduction.md diff --git a/deploy/chatgpt-retrieval-plugin/templates/deployment.yaml b/deploy/chatgpt-retrieval-plugin/templates/deployment.yaml index f3e250980..845e40034 100644 --- a/deploy/chatgpt-retrieval-plugin/templates/deployment.yaml +++ b/deploy/chatgpt-retrieval-plugin/templates/deployment.yaml @@ -44,14 +44,6 @@ spec: - name: http containerPort: 8080 protocol: TCP - livenessProbe: - httpGet: - path: /docs - port: http - readinessProbe: - httpGet: - path: /docs - port: http resources: {{- toYaml .Values.resources | nindent 12 }} env: diff --git a/deploy/helm/templates/addons/chatgpt-retrieval-plugin.yaml b/deploy/helm/templates/addons/chatgpt-retrieval-plugin.yaml deleted file mode 100644 index de7826305..000000000 --- a/deploy/helm/templates/addons/chatgpt-retrieval-plugin.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: extensions.kubeblocks.io/v1alpha1 -kind: Addon -metadata: - name: chatgpt-retrieval-plugin - labels: - {{- include "kubeblocks.labels" . | nindent 4 }} - "kubeblocks.io/provider": apecloud - {{- if .Values.keepAddons }} - annotations: - helm.sh/resource-policy: keep - {{- end }} -spec: - description: 'Deploys a ChatGPT Retrieval Plugin application in a cluster. - ChatGPT Retrieval Plugin is an application for personalizing your ChatGPT dialogue through your private data.' - type: Helm - - helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/chatgpt-retrieval-plugin-{{ default .Chart.Version .Values.versionOverride }}.tgz - valuesMapping: - valueMap: - replicaCount: replicaCount - - jsonMap: - tolerations: tolerations - - resources: - cpu: - requests: resources.requests.cpu - limits: resources.limits.cpu - memory: - requests: resources.requests.memory - limits: resources.limits.memory - - defaultInstallValues: - - replicas: 1 - - installable: - autoInstall: false - diff --git a/deploy/helm/templates/addons/milvus-addon.yaml b/deploy/helm/templates/addons/milvus-addon.yaml index 9ae726baf..614eeb8e9 100644 --- a/deploy/helm/templates/addons/milvus-addon.yaml +++ b/deploy/helm/templates/addons/milvus-addon.yaml @@ -18,7 +18,7 @@ spec: chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/milvus-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: - autoInstall: false + autoInstall: true defaultInstallValues: - enabled: true diff --git a/deploy/helm/templates/addons/qdrant-addon.yaml b/deploy/helm/templates/addons/qdrant-addon.yaml index 38abb8625..61cee3c9f 100644 --- a/deploy/helm/templates/addons/qdrant-addon.yaml +++ b/deploy/helm/templates/addons/qdrant-addon.yaml @@ -18,7 +18,7 @@ spec: chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/qdrant-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: - autoInstall: false + autoInstall: true defaultInstallValues: - enabled: true diff --git a/deploy/helm/templates/addons/redis-addon.yaml b/deploy/helm/templates/addons/redis-addon.yaml index 8a18ef5cf..da725257a 100644 --- a/deploy/helm/templates/addons/redis-addon.yaml +++ b/deploy/helm/templates/addons/redis-addon.yaml @@ -19,7 +19,7 @@ spec: chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/redis-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: - autoInstall: false + autoInstall: true defaultInstallValues: - enabled: true diff --git a/deploy/helm/templates/addons/weaviate-addon.yaml b/deploy/helm/templates/addons/weaviate-addon.yaml index 152b5fda2..a86a78742 100644 --- a/deploy/helm/templates/addons/weaviate-addon.yaml +++ b/deploy/helm/templates/addons/weaviate-addon.yaml @@ -18,7 +18,7 @@ spec: chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/weaviate-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: - autoInstall: false + autoInstall: true defaultInstallValues: - enabled: true diff --git a/docs/user_docs/kubeblocks-for-gptplugin/Installation.md b/docs/user_docs/kubeblocks-for-gptplugin/Installation.md new file mode 100644 index 000000000..d6bb5e733 --- /dev/null +++ b/docs/user_docs/kubeblocks-for-gptplugin/Installation.md @@ -0,0 +1,63 @@ +## Installation of ChatGPT Retrieval Plugin on KubeBlocks + +### Requirements from OpenAI +Two prerequisites are needed for running the Plugin: +The Plugin authorities from OpenAI, if none, you can join the [waiting list here](https://openai.com/waitlist/plugins). +An OPENAI_API_KEY for the Plugin to call OpenAI embedding APIs. + +### Requirements from KubeBlocks +Kbcli & the lastest KubeBlocks edition are installed. +TODO: the installation part of kbcli & kb. + +### Installation +#### Step 1: Create a vector database with kbcli +1. List Addons of KubeBlocks, each vector database is an addon in KubeBlocks +```shell +kbcli addon list +``` +2. If not enabled, enable it with +```shell +kbcli addon enable qdrant +``` +waiting for the addon from 'enabling' to 'enabled' +3. When enabled successfully, you can check it with +```shell +kbcli addon list +kubectl get clusterdefintion + +NAME MAIN-COMPONENT-NAME STATUS AGE +qdrant-standalone qdrant Available 6m14s +``` +4. create a qdrant cluster +```shell +kbcli cluster create --cluster-definition=qdrant-standalone + +Warning: cluster version is not specified, use the recently created ClusterVersion qdrant-1.1.0 +Cluster lilac26 created +``` +a qdrant standalone cluster is created successfully +#### Step 2: Start the plugin with qdrant as store +with helm to install +```shell +helm install gptplugin +--set datastore.DATASTORE=qdrant +--set datastore.QDRANT_COLLECTION=document_chunks +--set datastore.QDRANT_URL=http://lilac26-qdrant-headless.default.svc.cluster.local +--set datastore.QDRANT_PORT=6333 +--set datastore.BEARER_TOKEN=your_bearer_token +--set datastore.OPENAI_API_KEY=your_openai_api_key +--set website.url=your_website_url + +kubectl get pods +NAME READY STATUS RESTARTS AGE +gptplugin-chatgpt-retrieval-plugin-647d85498d-jd2bj 1/1 Running 0 10m +``` + +#### Step 3: Port-forward the Plugin Portal +```shell +kubectl port-forward pod/gptplugin-chatgpt-retrieval-plugin-647d85498d-jd2bj 8081:8080 +``` +In your web browser, open the plugin portal with +```shell +http://127.0.0.1:8081/docs +``` diff --git a/docs/user_docs/kubeblocks-for-gptplugin/Introduction.md b/docs/user_docs/kubeblocks-for-gptplugin/Introduction.md new file mode 100644 index 000000000..6717b3ff3 --- /dev/null +++ b/docs/user_docs/kubeblocks-for-gptplugin/Introduction.md @@ -0,0 +1,13 @@ +## Introduction to KubeBlocks for ChatGPT Retrieval Plugin +### ChatGPT Retrieval Plugin +[ChatGPT Retrieval Plugin](https://github.com/openai/chatgpt-retrieval-plugin) is the official plugin from OpenAI, it provides a flexible solution for semantic search and retrieval of personal or organizational documents using natural language queries. +The purpose of Plugin is to extend the OpenAI abilities from a public large model to a hybrid model with private data. It manages a private documents store to promise the privacy and safety, meanwhile, it can be accessed through a set of query APIs from the remote OpenAI GPT Chatbox. +The remote Chatbox is responsible for the dialogue with user, it processes the natural language query, retrieves data from both of OpenAI large model and private store, combines results to provide a better answer. +With the boost of private and domain data, the results could be more reasonable, accurate and personal. +It is a good way to balance the data privacy and utilization. + +### Plugin on KubeBlocks +KubeBlocks makes two major improvements over ChatGPT Retrieval Plugin: +KubeBlocks provides a solid vector database for Plugin, and relieves the burden of database management. Right now, KubeBlocks support a wide range of vector databases, such as Postgres, Redis, Milvus, Qdrant and Weaviate. The support of other vector databases is on the way. +KubeBlocks also builds multi-arch images for Plugin, integrates the Plugin as a native Cluster inside, makes the APIs, Secrets, Env vars configurable through command line & Helm Charts. +These improvements achieve a better experience for building your own Plugin. From 96e22da7a4a630587d9d42a313451d5cd7a6f541 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Mon, 17 Apr 2023 14:00:14 +0800 Subject: [PATCH 053/439] feat: support pitr (#1738) --- apis/apps/v1alpha1/opsrequest_types.go | 40 ++ .../v1alpha1/backuptool_types.go | 5 + .../bases/apps.kubeblocks.io_opsrequests.yaml | 40 ++ ...aprotection.kubeblocks.io_backuptools.yaml | 7 + .../crds/apps.kubeblocks.io_opsrequests.yaml | 40 ++ ...aprotection.kubeblocks.io_backuptools.yaml | 7 + .../templates/backuppolicytemplate.yaml | 11 +- .../postgresql/templates/backuptool-pitr.yaml | 54 ++ deploy/postgresql/templates/configmap.yaml | 6 + deploy/postgresql/templates/scripts.yaml | 32 +- internal/constant/const.go | 21 +- internal/controller/builder/builder.go | 20 + .../builder/cue/pitr_job_template.cue | 32 + .../lifecycle/cluster_plan_builder.go | 22 + .../lifecycle/transformer_cluster.go | 4 + .../lifecycle/transformer_cluster_status.go | 10 + internal/controller/plan/pitr.go | 564 ++++++++++++++++++ internal/controller/plan/pitr_test.go | 240 ++++++++ internal/testutil/apps/backup_factory.go | 5 + .../apps/backuppolicytemplate_factory.go | 5 + internal/testutil/apps/cluster_factory.go | 16 + internal/testutil/apps/cluster_util.go | 2 + test/testdata/backup/pitr_backuptool.yaml | 25 + 23 files changed, 1197 insertions(+), 11 deletions(-) create mode 100644 deploy/postgresql/templates/backuptool-pitr.yaml create mode 100644 internal/controller/builder/cue/pitr_job_template.cue create mode 100644 internal/controller/plan/pitr.go create mode 100644 internal/controller/plan/pitr_test.go create mode 100644 test/testdata/backup/pitr_backuptool.yaml diff --git a/apis/apps/v1alpha1/opsrequest_types.go b/apis/apps/v1alpha1/opsrequest_types.go index ad0dd73dd..a40bb1f13 100644 --- a/apis/apps/v1alpha1/opsrequest_types.go +++ b/apis/apps/v1alpha1/opsrequest_types.go @@ -85,6 +85,10 @@ type OpsRequestSpec struct { // +listType=map // +listMapKey=componentName ExposeList []Expose `json:"expose,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"componentName"` + + // cluster RestoreFrom backup or point in time + // +optional + RestoreFrom *RestoreFromSpec `json:"restoreFrom,omitempty"` } // ComponentOps defines the common variables of component scope operations. @@ -223,6 +227,42 @@ type Expose struct { Services []ClusterComponentService `json:"services"` } +type RestoreFromSpec struct { + // use the backup name and component name for restore, support for multiple components' recovery. + // +optional + Backup []BackupRefSpec `json:"backup,omitempty"` + + // specified the point in time to recovery + // +optional + PointInTime *PointInTimeRefSpec `json:"pointInTime,omitempty"` +} + +type RefNamespaceName struct { + // specified the name + // +optional + Name string `json:"name,omitempty"` + + // specified the namespace + // +optional + Namespace string `json:"namespace,omitempty"` +} + +type BackupRefSpec struct { + // specify a reference backup to restore + // +optional + Ref RefNamespaceName `json:"ref,omitempty"` +} + +type PointInTimeRefSpec struct { + // specify the time point to restore, with UTC as the time zone. + // +optional + Time *metav1.Time `json:"time,omitempty"` + + // specify a reference source cluster to restore + // +optional + Ref RefNamespaceName `json:"ref,omitempty"` +} + // OpsRequestStatus defines the observed state of OpsRequest type OpsRequestStatus struct { diff --git a/apis/dataprotection/v1alpha1/backuptool_types.go b/apis/dataprotection/v1alpha1/backuptool_types.go index 9e7115d39..ec51b0817 100644 --- a/apis/dataprotection/v1alpha1/backuptool_types.go +++ b/apis/dataprotection/v1alpha1/backuptool_types.go @@ -32,6 +32,11 @@ type BackupToolSpec struct { // +kubebuilder:default=job DeployKind string `json:"deployKind,omitempty"` + // the type of backup tool, file or pitr + // +kubebuilder:validation:Enum={file,pitr} + // +kubebuilder:default=file + Type string `json:"type,omitempty"` + // Compute Resources required by this container. // Cannot be updated. // +kubebuilder:pruning:PreserveUnknownFields diff --git a/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml b/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml index b0679d64b..ee8985ced 100644 --- a/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml +++ b/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml @@ -223,6 +223,46 @@ spec: x-kubernetes-list-map-keys: - componentName x-kubernetes-list-type: map + restoreFrom: + description: cluster RestoreFrom backup or point in time + properties: + backup: + description: use the backup name and component name for restore, + support for multiple components' recovery. + items: + properties: + ref: + description: specify a reference backup to restore + properties: + name: + description: specified the name + type: string + namespace: + description: specified the namespace + type: string + type: object + type: object + type: array + pointInTime: + description: specified the point in time to recovery + properties: + ref: + description: specify a reference source cluster to restore + properties: + name: + description: specified the name + type: string + namespace: + description: specified the namespace + type: string + type: object + time: + description: specify the time point to restore, with UTC as + the time zone. + format: date-time + type: string + type: object + type: object ttlSecondsAfterSucceed: description: ttlSecondsAfterSucceed OpsRequest will be deleted after TTLSecondsAfterSucceed second when OpsRequest.status.phase is Succeed. diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backuptools.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backuptools.yaml index 3f84de80f..3fcd126fe 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backuptools.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backuptools.yaml @@ -286,6 +286,13 @@ spec: type: object type: object x-kubernetes-preserve-unknown-fields: true + type: + default: file + description: the type of backup tool, file or pitr + enum: + - file + - pitr + type: string required: - backupCommands - image diff --git a/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml b/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml index b0679d64b..ee8985ced 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml @@ -223,6 +223,46 @@ spec: x-kubernetes-list-map-keys: - componentName x-kubernetes-list-type: map + restoreFrom: + description: cluster RestoreFrom backup or point in time + properties: + backup: + description: use the backup name and component name for restore, + support for multiple components' recovery. + items: + properties: + ref: + description: specify a reference backup to restore + properties: + name: + description: specified the name + type: string + namespace: + description: specified the namespace + type: string + type: object + type: object + type: array + pointInTime: + description: specified the point in time to recovery + properties: + ref: + description: specify a reference source cluster to restore + properties: + name: + description: specified the name + type: string + namespace: + description: specified the namespace + type: string + type: object + time: + description: specify the time point to restore, with UTC as + the time zone. + format: date-time + type: string + type: object + type: object ttlSecondsAfterSucceed: description: ttlSecondsAfterSucceed OpsRequest will be deleted after TTLSecondsAfterSucceed second when OpsRequest.status.phase is Succeed. diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backuptools.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backuptools.yaml index 3f84de80f..3fcd126fe 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backuptools.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backuptools.yaml @@ -286,6 +286,13 @@ spec: type: object type: object x-kubernetes-preserve-unknown-fields: true + type: + default: file + description: the type of backup tool, file or pitr + enum: + - file + - pitr + type: string required: - backupCommands - image diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index 1c31aaa47..77f690774 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -20,5 +20,14 @@ spec: connectionCredentialKey: passwordKey: password usernameKey: username + hooks: + containerName: postgresql + preCommands: + - psql -c "SELECT txid_current();CHECKPOINT;" + backupStatusUpdates: + - path: manifests.backupLog + containerName: postgresql + script: /kb-scripts/backup-log-collector.sh + updateStage: pre full: - backupToolName: postgres-basebackup \ No newline at end of file + backupToolName: postgres-basebackup diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml new file mode 100644 index 000000000..c2e363cc6 --- /dev/null +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -0,0 +1,54 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupTool +metadata: + labels: + clusterdefinition.kubeblocks.io/name: postgresql + kubeblocks.io/backup-tool-type: pitr + {{- include "postgresql.labels" . | nindent 4 }} + name: postgres-pitr +spec: + backupCommands: [] + deployKind: job + env: + - name: VOLUME_DATA_DIR + value: /home/postgres/pgdata + - name: VOLUME_LOG_DIR + value: /home/postgres/pgwal + - name: RESTORE_DATA_DIR + value: "$(VOLUME_DATA_DIR)/kb_restore" + - name: PITR_DIR + value: "$(VOLUME_DATA_DIR)/pitr" + - name: DATA_DIR + value: "$(VOLUME_DATA_DIR)/pgroot/data" + - name: CONF_DIR + value: "$(VOLUME_DATA_DIR)/conf" + - name: LOG_DIR + value: "$(VOLUME_LOG_DIR)/pgroot/data/pg_wal" + - name: RECOVERY_TIME + value: $KB_RECOVERY_TIME + - name: TIME_FORMAT + value: 2006-01-02 15:04:05 MST + image: alpine:3.17 + logical: + restoreCommands: + - | + set -e; + rm -f ${CONF_DIR}/recovery.conf; + rm -rf ${DATA_DIR}.old; + rm -rf ${DATA_DIR}.failed; + physical: + restoreCommands: + - | + set -e; + mkdir -p ${PITR_DIR}; + cp -R $LOG_DIR ${PITR_DIR}/; + chmod 777 -R ${PITR_DIR}; + touch ${DATA_DIR}/recovery.signal; + mkdir -p ${CONF_DIR}; + chmod 777 -R ${CONF_DIR}; + mkdir -p ${RESTORE_DATA_DIR}; + echo "cp -R ${DATA_DIR}.old ${DATA_DIR}" > ${RESTORE_DATA_DIR}/kb_restore.sh; + echo -e "restore_command=mv ${PITR_DIR}/pg_wal/%f %p\nrecovery_target_time=${RECOVERY_TIME}\nrecovery_target_action=promote" > ${CONF_DIR}/recovery.conf; + mv ${DATA_DIR} ${DATA_DIR}.old; + sync; + type: pitr \ No newline at end of file diff --git a/deploy/postgresql/templates/configmap.yaml b/deploy/postgresql/templates/configmap.yaml index 66c97e3ee..00a089f49 100644 --- a/deploy/postgresql/templates/configmap.yaml +++ b/deploy/postgresql/templates/configmap.yaml @@ -25,3 +25,9 @@ data: recovery_conf: restore_command: cp /home/postgres/pgdata/pgroot/arch/%f %p recovery_target_timeline: latest + kb_pitr.conf: | + method: kb_restore_from_time + kb_restore_from_time: + command: sh /home/postgres/pgdata/kb_restore/kb_restore.sh + keep_existing_recovery_conf: false + recovery_conf: {} \ No newline at end of file diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index c82c6f4b3..c48dfe448 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -32,6 +32,19 @@ data: if line and not line.startswith('#'): ret.append(line) return ret + def postgresql_conf_to_dict(file_path): + with open(file_path, 'r') as f: + content = f.read() + lines = content.splitlines() + result = {} + for line in lines: + if line.startswith('#'): + continue + if '=' not in line: + continue + key, value = line.split('=', 1) + result[key.strip()] = value.strip() + return result def main(filename): restore_dir = os.environ.get('RESTORE_DATA_DIR', '') local_config = yaml.safe_load( @@ -52,6 +65,14 @@ data: local_config['bootstrap'] = {} with open('/home/postgres/conf/kb_restore.conf', 'r') as f: local_config['bootstrap'].update(yaml.safe_load(f)) + # point in time recovery(PITR) + data_dir = os.environ.get('PGDATA', '') + if os.path.isfile("/home/postgres/pgdata/conf/recovery.conf"): + with open('/home/postgres/conf/kb_pitr.conf', 'r') as f: + pitr_config = yaml.safe_load(f) + re_config = postgresql_conf_to_dict("/home/postgres/pgdata/conf/recovery.conf") + pitr_config[pitr_config['method']]['recovery_conf'].update(re_config) + local_config['bootstrap'].update(pitr_config) write_file(yaml.dump(local_config, default_flow_style=False), filename, True) if __name__ == '__main__': main(sys.argv[1]) @@ -69,4 +90,13 @@ data: python3 /kb-scripts/generate_patroni_yaml.py tmp_patroni.yaml export SPILO_CONFIGURATION=$(cat tmp_patroni.yaml) # export SCOPE="$KB_CLUSTER_NAME-$KB_CLUSTER_NAME" - exec /launch.sh init \ No newline at end of file + exec /launch.sh init + backup-log-collector.sh: | + #!/bin/bash + set -o errexit + set -o nounset + LOG_START_TIME=$(pg_waldump $(ls -tr $PGDATA/pg_wal/ | grep '[[:digit:]]$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + LOG_STOP_TIME=$(pg_waldump $(ls -t $PGDATA/pg_wal/ | grep '[[:digit:]]$'|head -n 1) --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + LOG_START_TIME=$(date -d "$LOG_START_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') + LOG_STOP_TIME=$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') + printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}" \ No newline at end of file diff --git a/internal/constant/const.go b/internal/constant/const.go index 8df2aa2d8..5db40ecd4 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -73,17 +73,20 @@ const ( VolumeTypeLabelKey = "kubeblocks.io/volume-type" KBManagedByKey = "apps.kubeblocks.io/managed-by" // KBManagedByKey marks resources that auto created during operation ClassProviderLabelKey = "class.kubeblocks.io/provider" + BackupToolTypeLabelKey = "kubeblocks.io/backup-tool-type" // kubeblocks.io annotations - OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster - ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile - RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart - SnapShotForStartAnnotationKey = "kubeblocks.io/snapshot-for-start" - RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. - ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. - LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" - DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" - BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" + OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster + ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile + RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart + SnapShotForStartAnnotationKey = "kubeblocks.io/snapshot-for-start" + RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. + ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. + LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" + DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" + BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" + RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. + RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. // ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl ConfigurationTplLabelPrefixKey = "config.kubeblocks.io/tpl" diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index e423ac749..0497d1e32 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -707,3 +707,23 @@ func BuildBackupManifestsJob(key types.NamespacedName, backup *dataprotectionv1a } return job, nil } + +func BuildPITRJob(name string, cluster *appsv1alpha1.Cluster, image string, command []string, args []string, + volumes []corev1.Volume, volumeMounts []corev1.VolumeMount, env []corev1.EnvVar) (*batchv1.Job, error) { + const tplFile = "pitr_job_template.cue" + job := &batchv1.Job{} + if err := buildFromCUE(tplFile, map[string]any{ + "job.metadata.name": name, + "job.metadata.namespace": cluster.Namespace, + "job.spec.template.spec.volumes": volumes, + "container.image": image, + "container.command": command, + "container.args": args, + "container.volumeMounts": volumeMounts, + "container.env": env, + }, "job", job); err != nil { + return nil, err + } + + return job, nil +} diff --git a/internal/controller/builder/cue/pitr_job_template.cue b/internal/controller/builder/cue/pitr_job_template.cue new file mode 100644 index 000000000..cebeecb4c --- /dev/null +++ b/internal/controller/builder/cue/pitr_job_template.cue @@ -0,0 +1,32 @@ +container: { + name: "pitr" + image: string + imagePullPolicy: "IfNotPresent" + command: [...] + args: [...] + volumeMounts: [...] + env: [...] +} + +job: { + apiVersion: "batch/v1" + kind: "Job" + metadata: { + name: string + namespace: string + labels: { + "app.kubernetes.io/managed-by": "kubeblocks" + } + } + spec: { + template: { + spec: { + containers: [container] + volumes: [...] + restartPolicy: "OnFailure" + securityContext: + runAsUser: 0 + } + } + } +} diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index e91b9bf6a..79fb493b5 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -24,6 +24,7 @@ import ( snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -521,6 +522,9 @@ func (c *clusterPlanBuilder) handleClusterDeletion(cluster *appsv1alpha1.Cluster return err } } + if err := c.deleteJobs(cluster); err != nil && !apierrors.IsNotFound(err) { + return err + } } return nil } @@ -580,6 +584,24 @@ func (c *clusterPlanBuilder) deleteBackups(cluster *appsv1alpha1.Cluster) error return nil } +func (c *clusterPlanBuilder) deleteJobs(cluster *appsv1alpha1.Cluster) error { + inNS := client.InNamespace(cluster.Namespace) + ml := client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.GetName(), + } + // clean jobs + jobList := batchv1.JobList{} + if err := c.cli.List(c.ctx.Ctx, &jobList, inNS, ml); err != nil { + return err + } + for _, job := range jobList.Items { + if err := intctrlutil.BackgroundDeleteObject(c.cli, c.ctx.Ctx, &job); err != nil { + return err + } + } + return nil +} + func DeleteConfigMaps(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster) error { inNS := client.InNamespace(cluster.Namespace) ml := client.MatchingLabels{ diff --git a/internal/controller/lifecycle/transformer_cluster.go b/internal/controller/lifecycle/transformer_cluster.go index 2a40432a9..1bb550080 100644 --- a/internal/controller/lifecycle/transformer_cluster.go +++ b/internal/controller/lifecycle/transformer_cluster.go @@ -96,6 +96,10 @@ func (c *clusterTransformer) Transform(dag *graph.DAG) error { return err } } + if err = plan.DoPITRPrepare(c.ctx.Ctx, c.cli, cluster, synthesizedComp); err != nil { + return err + } + return plan.PrepareComponentResources(c.ctx, c.cli, &iParams) } diff --git a/internal/controller/lifecycle/transformer_cluster_status.go b/internal/controller/lifecycle/transformer_cluster_status.go index 2e59a1983..68103a6f0 100644 --- a/internal/controller/lifecycle/transformer_cluster_status.go +++ b/internal/controller/lifecycle/transformer_cluster_status.go @@ -34,6 +34,7 @@ import ( "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/component" "github.com/apecloud/kubeblocks/internal/controller/graph" + "github.com/apecloud/kubeblocks/internal/controller/plan" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -136,6 +137,15 @@ func (c *clusterStatusTransformer) Transform(dag *graph.DAG) error { return err } c.cleanupAnnotationsAfterRunning(cluster) + + if shouldRequeue, err := plan.DoPITRIfNeed(c.ctx.Ctx, c.cli, cluster); err != nil { + return err + } else if shouldRequeue { + return &realRequeueError{reason: "waiting pitr job", requeueAfter: requeueDuration} + } + if err = plan.DoPITRCleanup(c.ctx.Ctx, c.cli, cluster); err != nil { + return err + } } return nil diff --git a/internal/controller/plan/pitr.go b/internal/controller/plan/pitr.go new file mode 100644 index 000000000..fbbeb69d4 --- /dev/null +++ b/internal/controller/plan/pitr.go @@ -0,0 +1,564 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + "github.com/spf13/viper" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/controllers/apps/components/util" + "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/controller/builder" + "github.com/apecloud/kubeblocks/internal/controller/component" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" +) + +// PointInTimeRecoveryManager pitr manager functions +// 1. get the latest base backup +// 2. get the next earliest backup +// 3. add log pvc by datasource volume snapshot +// 4. create init container to prepare log +// 5. run recovery jobs +// 6. cleanup +type PointInTimeRecoveryManager struct { + client.Client + Ctx context.Context + Cluster *appsv1alpha1.Cluster + + // private + namespace string + restoreTime *metav1.Time + sourceCluster string +} + +const ( + initContainerName = "pitr-for-pause" +) + +// DoPITRPrepare prepares init container and pvc before point in time recovery +func DoPITRPrepare(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster, component *component.SynthesizedComponent) error { + if cluster.Status.ObservedGeneration >= 1 { + return nil + } + + // build pitr init container to wait prepare data + // prepare data if PITR needed + pitrMgr := PointInTimeRecoveryManager{ + Cluster: cluster, + Client: cli, + Ctx: ctx, + } + return pitrMgr.doPrepare(component) +} + +// DoPITRIfNeed checks if run restore job and copy data for point in time recovery +func DoPITRIfNeed(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster) (shouldRequeue bool, err error) { + if cluster.Status.ObservedGeneration != 1 { + return false, nil + } + pitrMgr := PointInTimeRecoveryManager{ + Cluster: cluster, + Client: cli, + Ctx: ctx, + } + return pitrMgr.doRecoveryJob() +} + +// DoPITRCleanup cleanup the resources and annotations after point in time recovery +func DoPITRCleanup(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster) error { + if cluster.Status.ObservedGeneration < 1 { + return nil + } + pitrMgr := PointInTimeRecoveryManager{ + Cluster: cluster, + Client: cli, + Ctx: ctx, + } + if need, err := pitrMgr.checkAndInit(); err != nil { + return err + } else if !need { + return nil + } + // clean up job + if err := pitrMgr.cleanupScriptsJob(); err != nil { + return err + } + // clean cluster annotations + if err := pitrMgr.cleanupClusterAnnotations(); err != nil { + return err + } + return nil +} + +// doRecoveryJob runs a physical recovery job before cluster service start +func (p *PointInTimeRecoveryManager) doRecoveryJob() (shouldRequeue bool, err error) { + if need, err := p.checkAndInit(); err != nil { + return false, err + } else if !need { + return false, nil + } + + // mount the data+log pvc, and run scripts job to prepare data + if err = p.runRecoveryJob(); err != nil { + if err.Error() == "waiting PVC Bound" { + return true, nil + } + return false, err + } + + // check job done + if !p.ensureJobDone() { + return true, nil + } + + // remove init container + for _, componentSpec := range p.Cluster.Spec.ComponentSpecs { + if err = p.removeStsInitContainer(p.Cluster, componentSpec.Name); err != nil { + return false, err + } + } + + return false, nil + +} + +// doPrepare prepares init container and pvc before point in time recovery +func (p *PointInTimeRecoveryManager) doPrepare(component *component.SynthesizedComponent) error { + if need, err := p.checkAndInit(); err != nil { + return err + } else if !need { + return nil + } + // prepare init container + container := corev1.Container{} + container.Name = initContainerName + container.Image = viper.GetString(constant.KBToolsImage) + container.Command = []string{"sleep", "infinity"} + component.PodSpec.InitContainers = append(component.PodSpec.InitContainers, container) + + // prepare data pvc + if len(component.VolumeClaimTemplates) == 0 { + return errors.New("not found data pvc") + } + latestBackup, err := p.getLatestBaseBackup() + if err != nil { + return err + } + + vct := component.VolumeClaimTemplates[0] + snapshotAPIGroup := snapshotv1.GroupName + vct.Spec.DataSource = &corev1.TypedLocalObjectReference{ + APIGroup: &snapshotAPIGroup, + Kind: constant.VolumeSnapshotKind, + Name: latestBackup.Name, + } + component.VolumeClaimTemplates[0] = vct + return nil +} + +func (p *PointInTimeRecoveryManager) listCompletedBackups() (backupItems []dpv1alpha1.Backup, err error) { + backups := dpv1alpha1.BackupList{} + if err := p.Client.List(p.Ctx, &backups, + client.InNamespace(p.namespace), + client.MatchingLabels(map[string]string{ + constant.AppInstanceLabelKey: p.sourceCluster, + }), + ); err != nil { + return nil, err + } + + backupItems = []dpv1alpha1.Backup{} + for _, b := range backups.Items { + if b.Status.Phase == dpv1alpha1.BackupCompleted && b.Status.Manifests != nil && b.Status.Manifests.BackupLog != nil { + backupItems = append(backupItems, b) + } + } + return backupItems, nil +} + +// getSortedBackups sorts by StopTime +func (p *PointInTimeRecoveryManager) getSortedBackups(reverse bool) ([]dpv1alpha1.Backup, error) { + backups, err := p.listCompletedBackups() + if err != nil { + return backups, err + } + sort.Slice(backups, func(i, j int) bool { + if reverse { + i, j = j, i + } + if backups[i].Status.Manifests.BackupLog.StopTime == nil && backups[j].Status.Manifests.BackupLog.StopTime != nil { + return false + } + if backups[i].Status.Manifests.BackupLog.StopTime != nil && backups[j].Status.Manifests.BackupLog.StopTime == nil { + return true + } + if backups[i].Status.Manifests.BackupLog.StopTime.Equal(backups[j].Status.Manifests.BackupLog.StopTime) { + return backups[i].Name < backups[j].Name + } + return backups[i].Status.Manifests.BackupLog.StopTime.Before(backups[j].Status.Manifests.BackupLog.StopTime) + }) + return backups, nil +} + +// getLatestBaseBackup gets the latest baseBackup +func (p *PointInTimeRecoveryManager) getLatestBaseBackup() (*dpv1alpha1.Backup, error) { + // 1. sort backups by completed timestamp + backups, err := p.getSortedBackups(true) + if err != nil { + return nil, err + } + + // 2. get the latest backup object + var latestBackup *dpv1alpha1.Backup + for _, item := range backups { + if item.Status.Manifests.BackupLog.StopTime != nil && + p.restoreTime.After(item.Status.Manifests.BackupLog.StopTime.Time) { + latestBackup = &item + break + } + } + if latestBackup == nil { + return nil, errors.New("can not found latest base backup") + } + + return latestBackup, nil +} + +func (p *PointInTimeRecoveryManager) getNextBackup() (*dpv1alpha1.Backup, error) { + // 1. sort backups by reverse completed timestamp + backups, err := p.getSortedBackups(false) + if err != nil { + return nil, err + } + + // 2. get the next earliest backup object + var nextBackup *dpv1alpha1.Backup + for _, item := range backups { + if p.restoreTime.Before(item.Status.Manifests.BackupLog.StopTime) { + nextBackup = &item + break + } + } + if nextBackup == nil { + return nil, errors.New("can not found next earliest base backup") + } + + return nextBackup, nil +} + +// checkAndInit checks if cluster need to be restored, return value: true: need, false: no need +func (p *PointInTimeRecoveryManager) checkAndInit() (need bool, err error) { + // check args if pitr supported + cluster := p.Cluster + if cluster.Annotations[constant.RestoreFromTimeAnnotationKey] == "" { + return false, nil + } + restoreTimeStr := cluster.Annotations[constant.RestoreFromTimeAnnotationKey] + sourceCuster := cluster.Annotations[constant.RestoreFromSrcClusterAnnotationKey] + if sourceCuster == "" { + return false, errors.New("need specify a source cluster name to recovery") + } + restoreTime := &metav1.Time{} + if err = restoreTime.UnmarshalQueryParameter(restoreTimeStr); err != nil { + return false, err + } + vctCount := 0 + for _, item := range cluster.Spec.ComponentSpecs { + vctCount += len(item.VolumeClaimTemplates) + } + if vctCount == 0 { + return false, errors.New("not support pitr without any volume claim templates") + } + + // init args + p.restoreTime = restoreTime + p.sourceCluster = sourceCuster + p.namespace = cluster.Namespace + return true, nil +} + +func getVolumeMount(spec *dpv1alpha1.BackupToolSpec) (string, string) { + dataVolumeMount := "/data" + logVolumeMount := "/log" + tag := 0 + // TODO: hack it because the mount path is not explicitly specified in cluster definition + for _, env := range spec.Env { + if env.Name == "VOLUME_DATA_DIR" { + dataVolumeMount = env.Value + tag++ + } else if env.Name == "VOLUME_LOG_DIR" { + logVolumeMount = env.Value + tag++ + } + if tag >= 2 { + break + } + } + return dataVolumeMount, logVolumeMount +} + +func (p *PointInTimeRecoveryManager) getRecoveryInfo() (*dpv1alpha1.BackupToolSpec, error) { + // get scripts from backup template + toolList := dpv1alpha1.BackupToolList{} + // TODO: The reference PITR backup tool needs a stronger reference relationship, for now use label references + if err := p.Client.List(p.Ctx, &toolList, + client.MatchingLabels{ + constant.ClusterDefLabelKey: p.Cluster.Spec.ClusterDefRef, + constant.BackupToolTypeLabelKey: "pitr", + }); err != nil { + + return nil, err + } + if len(toolList.Items) == 0 { + return nil, errors.New("not support recovery because of non-existed pitr backupTool") + } + spec := &toolList.Items[0].Spec + timeFormat := time.RFC3339 + envTimeEnvIdx := -1 + for i, env := range spec.Env { + if env.Value == "$KB_RECOVERY_TIME" { + envTimeEnvIdx = i + } else if env.Name == "TIME_FORMAT" { + timeFormat = env.Value + } + } + if envTimeEnvIdx != -1 { + spec.Env[envTimeEnvIdx].Value = p.restoreTime.Time.UTC().Format(timeFormat) + } + + return spec, nil +} + +func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, err error) { + objs = make([]client.Object, 0) + recoveryInfo, err := p.getRecoveryInfo() + if err != nil { + return objs, err + } + for _, componentSpec := range p.Cluster.Spec.ComponentSpecs { + if len(componentSpec.VolumeClaimTemplates) == 0 { + continue + } + + commonLabels := map[string]string{ + constant.AppManagedByLabelKey: constant.AppName, + constant.AppInstanceLabelKey: p.Cluster.Name, + constant.KBAppComponentLabelKey: componentSpec.Name, + } + sts := &appsv1.StatefulSet{} + sts.SetLabels(commonLabels) + vct := corev1.PersistentVolumeClaimTemplate{} + vct.Name = componentSpec.VolumeClaimTemplates[0].Name + vct.Spec = componentSpec.VolumeClaimTemplates[0].Spec.ToV1PersistentVolumeClaimSpec() + + // get data dir pvc name + dataPVCList := corev1.PersistentVolumeClaimList{} + dataPVCLabels := map[string]string{ + constant.AppInstanceLabelKey: p.Cluster.Name, + constant.KBAppComponentLabelKey: componentSpec.Name, + constant.VolumeTypeLabelKey: string(appsv1alpha1.VolumeTypeData), + } + if err = p.Client.List(p.Ctx, &dataPVCList, + client.InNamespace(p.namespace), + client.MatchingLabels(dataPVCLabels)); err != nil { + return objs, err + } + if len(dataPVCList.Items) == 0 { + return objs, errors.New("not found data pvc") + } + for i, dataPVC := range dataPVCList.Items { + if dataPVC.Status.Phase != corev1.ClaimBound { + return objs, errors.New("waiting PVC Bound") + } + + nextBackup, err := p.getNextBackup() + if err != nil { + return objs, err + } + pitrPVCName := fmt.Sprintf("pitr-%s-%s-%d", p.Cluster.Name, componentSpec.Name, i) + pitrPVCKey := types.NamespacedName{ + Namespace: p.namespace, + Name: pitrPVCName, + } + pitrPVC, err := builder.BuildPVCFromSnapshot(sts, vct, pitrPVCKey, nextBackup.Name, nil) + if err != nil { + return objs, err + } + volumes := []corev1.Volume{ + {Name: "data", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: dataPVC.Name}}}, + {Name: "log", VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pitrPVCName}}}, + } + dataVolumeMount, logVolumeMount := getVolumeMount(recoveryInfo) + volumeMounts := []corev1.VolumeMount{ + {Name: "data", MountPath: dataVolumeMount}, + {Name: "log", MountPath: logVolumeMount}, + } + + // render the job cue template + image := recoveryInfo.Image + if image == "" { + image = viper.GetString(constant.KBToolsImage) + } + jobName := fmt.Sprintf("pitr-phy-%s-%s-%d", p.Cluster.Name, componentSpec.Name, i) + job, err := builder.BuildPITRJob(jobName, p.Cluster, image, []string{"sh", "-c"}, + recoveryInfo.Physical.RestoreCommands, volumes, volumeMounts, recoveryInfo.Env) + job.SetLabels(commonLabels) + if err != nil { + return objs, err + } + // create logic restore job + if p.Cluster.Status.Phase == appsv1alpha1.RunningClusterPhase && len(recoveryInfo.Logical.RestoreCommands) > 0 { + logicJobName := fmt.Sprintf("pitr-logic-%s-%s-%d", p.Cluster.Name, componentSpec.Name, i) + logicJob, err := builder.BuildPITRJob(logicJobName, p.Cluster, image, []string{"sh", "-c"}, + recoveryInfo.Logical.RestoreCommands, volumes, volumeMounts, recoveryInfo.Env) + if err != nil { + return objs, err + } + logicJob.SetLabels(commonLabels) + objs = append(objs, logicJob) + } + // collect pvcs and jobs for later deletion + objs = append(objs, pitrPVC) + objs = append(objs, job) + } + } + return objs, nil +} + +func (p *PointInTimeRecoveryManager) runRecoveryJob() error { + objs, err := p.buildResourceObjs() + if err != nil { + return err + } + + for _, obj := range objs { + err = p.Client.Create(p.Ctx, obj) + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + } + return nil +} + +func (p *PointInTimeRecoveryManager) checkJobDone(key client.ObjectKey) (bool, error) { + result := &batchv1.Job{} + if err := p.Client.Get(p.Ctx, key, result); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + // if err is NOT "not found", that means unknown error. + return false, err + } + if result.Status.Conditions != nil && len(result.Status.Conditions) > 0 { + jobStatusCondition := result.Status.Conditions[0] + if jobStatusCondition.Type == batchv1.JobComplete { + return true, nil + } else if jobStatusCondition.Type == batchv1.JobFailed { + return true, errors.New(jobStatusCondition.Reason) + } + } + // if found, return true + return false, nil +} + +func (p *PointInTimeRecoveryManager) ensureJobDone() bool { + var jobObj *batchv1.Job + var ok bool + objs, err := p.buildResourceObjs() + if err != nil { + return false + } + for _, obj := range objs { + if jobObj, ok = obj.(*batchv1.Job); !ok { + continue + } + if done, err := p.checkJobDone(client.ObjectKeyFromObject(jobObj)); err != nil { + return false + } else if !done { + return false + } + } + return true +} + +func (p *PointInTimeRecoveryManager) cleanupScriptsJob() error { + objs, err := p.buildResourceObjs() + if err != nil { + return err + } + if p.Cluster.Status.Phase == appsv1alpha1.RunningClusterPhase { + for _, obj := range objs { + if err := intctrlutil.BackgroundDeleteObject(p.Client, p.Ctx, obj); err != nil { + return err + } + } + } + return nil +} + +func (p *PointInTimeRecoveryManager) cleanupClusterAnnotations() error { + if p.Cluster.Status.Phase == appsv1alpha1.RunningClusterPhase && p.Cluster.Annotations != nil { + cluster := p.Cluster + patch := client.MergeFrom(cluster.DeepCopy()) + delete(cluster.Annotations, constant.RestoreFromSrcClusterAnnotationKey) + delete(cluster.Annotations, constant.RestoreFromTimeAnnotationKey) + return p.Client.Patch(p.Ctx, cluster, patch) + } + return nil +} + +// removeStsInitContainerForRestore removes the statefulSet's init container after recovery job completed. +func (p *PointInTimeRecoveryManager) removeStsInitContainer( + cluster *appsv1alpha1.Cluster, + componentName string) error { + // get the sts list of component + stsList := &appsv1.StatefulSetList{} + if err := util.GetObjectListByComponentName(p.Ctx, p.Client, *cluster, stsList, componentName); err != nil { + return err + } + for _, sts := range stsList.Items { + initContainers := sts.Spec.Template.Spec.InitContainers + updateInitContainers := make([]corev1.Container, 0) + for _, c := range initContainers { + if c.Name != initContainerName { + updateInitContainers = append(updateInitContainers, c) + } + } + sts.Spec.Template.Spec.InitContainers = updateInitContainers + if err := p.Client.Update(p.Ctx, &sts); err != nil { + return err + } + } + return nil +} diff --git a/internal/controller/plan/pitr_test.go b/internal/controller/plan/pitr_test.go new file mode 100644 index 000000000..d67d4e8ce --- /dev/null +++ b/internal/controller/plan/pitr_test.go @@ -0,0 +1,240 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plan + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/controller/component" + "github.com/apecloud/kubeblocks/internal/generics" + testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" +) + +var _ = Describe("PITR Functions", func() { + const defaultTTL = "7d" + const backupName = "test-backup-job" + const sourceCluster = "source-cluster" + + var ( + randomStr = testCtx.GetRandomStr() + clusterName = "cluster-for-pitr-" + randomStr + backupToolName string + ) + + cleanEnv := func() { + // must wait until resources deleted and no longer exist before the testcases start, + // otherwise if later it needs to create some new resource objects with the same name, + // in race conditions, it will find the existence of old objects, resulting failure to + // create the new objects. + By("clean resources") + + // delete cluster(and all dependent sub-resources), clusterversion and clusterdef + testapps.ClearClusterResources(&testCtx) + inNS := client.InNamespace(testCtx.DefaultNamespace) + ml := client.HasLabels{testCtx.TestObjLabelKey} + testapps.ClearResources(&testCtx, generics.BackupSignature, inNS, ml) + testapps.ClearResources(&testCtx, generics.BackupPolicySignature, inNS, ml) + testapps.ClearResources(&testCtx, generics.JobSignature, inNS, ml) + testapps.ClearResources(&testCtx, generics.CronJobSignature, inNS, ml) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PersistentVolumeClaimSignature, true, inNS, ml) + // + // non-namespaced + testapps.ClearResources(&testCtx, generics.BackupPolicyTemplateSignature, ml) + } + + BeforeEach(cleanEnv) + + AfterEach(cleanEnv) + + Context("Test PITR", func() { + const ( + clusterDefName = "test-clusterdef" + clusterVersionName = "test-clusterversion" + mysqlCompType = "replicasets" + mysqlCompName = "mysql" + nginxCompType = "proxy" + ) + + var ( + clusterDef *appsv1alpha1.ClusterDefinition + clusterVersion *appsv1alpha1.ClusterVersion + cluster *appsv1alpha1.Cluster + synthesizedComponent *component.SynthesizedComponent + pvc *corev1.PersistentVolumeClaim + ) + + BeforeEach(func() { + clusterDef = testapps.NewClusterDefFactory(clusterDefName). + AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompType). + AddComponentDef(testapps.StatelessNginxComponent, nginxCompType). + Create(&testCtx).GetObject() + clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). + AddComponent(mysqlCompType). + AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponent(nginxCompType). + AddInitContainerShort("nginx-init", testapps.NginxImage). + AddContainerShort("nginx", testapps.NginxImage). + Create(&testCtx).GetObject() + pvcSpec := testapps.NewPVCSpec("1Gi") + cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, + clusterDef.Name, clusterVersion.Name). + AddComponent(mysqlCompName, mysqlCompType). + AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). + AddRestorePointInTime(metav1.Time{Time: metav1.Now().Time}, sourceCluster). + Create(&testCtx).GetObject() + + By("By mocking a pvc") + pvc = testapps.NewPersistentVolumeClaimFactory( + testCtx.DefaultNamespace, "data-"+clusterName+"-"+mysqlCompName+"-0", clusterName, mysqlCompName, "data"). + SetStorage("1Gi"). + Create(&testCtx).GetObject() + + By("By creating backup tool: ") + backupSelfDefineObj := &dpv1alpha1.BackupTool{} + backupSelfDefineObj.SetLabels(map[string]string{ + constant.BackupToolTypeLabelKey: "pitr", + constant.ClusterDefLabelKey: clusterDefName, + }) + backupTool := testapps.CreateCustomizedObj(&testCtx, "backup/pitr_backuptool.yaml", + backupSelfDefineObj, testapps.RandomizedObjName()) + backupToolName = backupTool.Name + + backupObj := dpv1alpha1.BackupToolList{} + Expect(testCtx.Cli.List(testCtx.Ctx, &backupObj)).Should(Succeed()) + + By("By creating backup policyTemplate: ") + backupTplLabels := map[string]string{ + constant.ClusterDefLabelKey: clusterDefName, + } + _ = testapps.NewBackupPolicyTemplateFactory("backup-policy-template"). + WithRandomName().SetLabels(backupTplLabels). + AddBackupPolicy(mysqlCompName). + SetClusterDefRef(clusterDefName). + SetBackupToolName(backupToolName). + SetSchedule("0 * * * *", true). + AddSnapshotPolicy(). + SetTTL(defaultTTL). + Create(&testCtx).GetObject() + + clusterCompDefObj := clusterDef.Spec.ComponentDefs[0] + synthesizedComponent = &component.SynthesizedComponent{ + PodSpec: clusterCompDefObj.PodSpec, + Probes: clusterCompDefObj.Probes, + LogConfigs: clusterCompDefObj.LogConfigs, + HorizontalScalePolicy: clusterCompDefObj.HorizontalScalePolicy, + VolumeClaimTemplates: cluster.Spec.ComponentSpecs[0].ToVolumeClaimTemplates(), + } + + By("By creating earlier backup: ") + now := metav1.Now() + backupLabels := map[string]string{ + constant.AppInstanceLabelKey: sourceCluster, + } + backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). + WithRandomName().SetLabels(backupLabels). + SetBackupPolicyName("test-fake"). + SetBackupType(dpv1alpha1.BackupTypeFull). + Create(&testCtx).GetObject() + earlierStartTime := &metav1.Time{Time: now.Add(-time.Hour * 3)} + earlierStopTime := &metav1.Time{Time: now.Add(-time.Hour * 2)} + backupStatus := dpv1alpha1.BackupStatus{ + Phase: dpv1alpha1.BackupCompleted, + StartTimestamp: earlierStartTime, + CompletionTimestamp: earlierStopTime, + Manifests: &dpv1alpha1.ManifestsStatus{ + BackupLog: &dpv1alpha1.BackupLogStatus{ + StartTime: earlierStartTime, + StopTime: earlierStopTime, + }, + }, + } + backupStatus.CompletionTimestamp = &metav1.Time{Time: now.Add(-time.Hour * 2)} + patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backup)) + + By("By creating latest backup: ") + latestStartTime := &metav1.Time{Time: now.Add(-time.Hour * 3)} + latestStopTime := &metav1.Time{Time: now.Add(time.Hour * 2)} + backupNext := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). + WithRandomName().SetLabels(backupLabels). + SetBackupPolicyName("test-fake"). + SetBackupType(dpv1alpha1.BackupTypeFull). + Create(&testCtx).GetObject() + backupStatus = dpv1alpha1.BackupStatus{ + Phase: dpv1alpha1.BackupCompleted, + StartTimestamp: latestStartTime, + CompletionTimestamp: latestStopTime, + Manifests: &dpv1alpha1.ManifestsStatus{ + BackupLog: &dpv1alpha1.BackupLogStatus{ + StartTime: latestStartTime, + StopTime: latestStopTime, + }, + }, + } + patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupNext)) + }) + + It("Test PITR prepare", func() { + Expect(DoPITRPrepare(ctx, testCtx.Cli, cluster, synthesizedComponent)).Should(Succeed()) + Expect(synthesizedComponent.PodSpec.InitContainers).ShouldNot(BeEmpty()) + }) + It("Test PITR job run and cleanup", func() { + By("when data pvc is pending") + cluster.Status.ObservedGeneration = 1 + shouldRequeue, err := DoPITRIfNeed(ctx, testCtx.Cli, cluster) + Expect(err).Should(Succeed()) + Expect(shouldRequeue).Should(BeTrue()) + By("when data pvc is bound") + Eventually(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(pvc), func(fetched *corev1.PersistentVolumeClaim) { + fetched.Status.Phase = corev1.ClaimBound + })).Should(Succeed()) + _, err = DoPITRIfNeed(ctx, testCtx.Cli, cluster) + Expect(err).Should(Succeed()) + By("when job is completed") + jobName := fmt.Sprintf("pitr-phy-%s-%s-0", clusterName, mysqlCompName) + jobKey := types.NamespacedName{Namespace: cluster.Namespace, Name: jobName} + Eventually(testapps.GetAndChangeObjStatus(&testCtx, jobKey, func(fetched *batchv1.Job) { + fetched.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete}} + })).Should(Succeed()) + Eventually(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(cluster), func(fetched *appsv1alpha1.Cluster) { + fetched.Status.Phase = appsv1alpha1.RunningClusterPhase + })).Should(Succeed()) + _, err = DoPITRIfNeed(ctx, testCtx.Cli, cluster) + Expect(err).Should(Succeed()) + By("cleanup pitr job") + Expect(DoPITRCleanup(ctx, testCtx.Cli, cluster)).Should(Succeed()) + }) + }) +}) + +func patchBackupStatus(status dpv1alpha1.BackupStatus, key types.NamespacedName) { + Eventually(testapps.GetAndChangeObjStatus(&testCtx, key, func(fetched *dpv1alpha1.Backup) { + fetched.Status = status + })).Should(Succeed()) +} diff --git a/internal/testutil/apps/backup_factory.go b/internal/testutil/apps/backup_factory.go index 6409e6a20..bd3e260dc 100644 --- a/internal/testutil/apps/backup_factory.go +++ b/internal/testutil/apps/backup_factory.go @@ -42,3 +42,8 @@ func (factory *MockBackupFactory) SetBackupType(backupType dataprotectionv1alpha factory.get().Spec.BackupType = backupType return factory } + +func (factory *MockBackupFactory) SetLabels(labels map[string]string) *MockBackupFactory { + factory.get().SetLabels(labels) + return factory +} diff --git a/internal/testutil/apps/backuppolicytemplate_factory.go b/internal/testutil/apps/backuppolicytemplate_factory.go index 4fe0a19a2..46868011c 100644 --- a/internal/testutil/apps/backuppolicytemplate_factory.go +++ b/internal/testutil/apps/backuppolicytemplate_factory.go @@ -202,3 +202,8 @@ func (factory *MockBackupPolicyTemplateFactory) SetTargetAccount(account string) }) return factory } + +func (factory *MockBackupPolicyTemplateFactory) SetLabels(labels map[string]string) *MockBackupPolicyTemplateFactory { + factory.get().SetLabels(labels) + return factory +} diff --git a/internal/testutil/apps/cluster_factory.go b/internal/testutil/apps/cluster_factory.go index 04c3683d8..8566c3e0d 100644 --- a/internal/testutil/apps/cluster_factory.go +++ b/internal/testutil/apps/cluster_factory.go @@ -17,9 +17,13 @@ limitations under the License. package apps import ( + "time" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" ) type MockClusterFactory struct { @@ -191,3 +195,15 @@ func (factory *MockClusterFactory) AddService(serviceName string, serviceType co factory.get().Spec.ComponentSpecs = comps return factory } + +func (factory *MockClusterFactory) AddRestorePointInTime(restoreTime metav1.Time, sourceCluster string) *MockClusterFactory { + annotations := factory.get().Annotations + if annotations == nil { + annotations = map[string]string{} + } + annotations[constant.RestoreFromTimeAnnotationKey] = restoreTime.Format(time.RFC3339) + annotations[constant.RestoreFromSrcClusterAnnotationKey] = sourceCluster + + factory.get().Annotations = annotations + return factory +} diff --git a/internal/testutil/apps/cluster_util.go b/internal/testutil/apps/cluster_util.go index b45d355a7..32746f482 100644 --- a/internal/testutil/apps/cluster_util.go +++ b/internal/testutil/apps/cluster_util.go @@ -48,10 +48,12 @@ func InitClusterWithHybridComps( AddComponent(consensusCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). AddComponent(statefulCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). Create(&testCtx).GetObject() + pvcSpec := NewPVCSpec("1Gi") cluster := NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). AddComponent(statelessCompDefName, statelessCompDefName).SetReplicas(1). AddComponent(consensusCompDefName, consensusCompDefName).SetReplicas(3). AddComponent(statefulCompDefName, statefulCompDefName).SetReplicas(3). + AddVolumeClaimTemplate(DataVolumeName, pvcSpec). Create(&testCtx).GetObject() return clusterDef, clusterVersion, cluster } diff --git a/test/testdata/backup/pitr_backuptool.yaml b/test/testdata/backup/pitr_backuptool.yaml new file mode 100644 index 000000000..19e9a2c07 --- /dev/null +++ b/test/testdata/backup/pitr_backuptool.yaml @@ -0,0 +1,25 @@ +apiVersion: dataprotection.kubeblocks.io/v1alpha1 +kind: BackupTool +metadata: + name: backup-tool- +spec: + image: registry.cn-hangzhou.aliyuncs.com/apecloud/percona-xtrabackup + deployKind: job + type: pitr + env: + - name: DATA_DIR + value: /var/lib/mysql + - name: RECOVERY_TIME + value: $KB_RECOVERY_TIME + physical: + restoreCommands: + - | + echo $RECOVERY_TIME + incrementalRestoreCommands: [] + logical: + restoreCommands: + - | + echo $RECOVERY_TIME + incrementalRestoreCommands: [] + backupCommands: [] + incrementalBackupCommands: [] \ No newline at end of file From 02adf66262e702cb659c8ad4d8c44e10687cc681 Mon Sep 17 00:00:00 2001 From: xingran Date: Mon, 17 Apr 2023 14:20:03 +0800 Subject: [PATCH 054/439] fix: replicationSet h-scale failed with single statefulSet and without h-scale-policy (#2630) --- .../lifecycle/transformer_sts_horizontal_scaling.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index d83e3b34a..ff65d53f4 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -164,6 +164,10 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { return err } if !allPVCsExist { + if comp.HorizontalScalePolicy == nil { + vertex.immutable = false + return nil + } // do backup according to component's horizontal scale policy if err := doBackup(s.ctx, s.cli, comp, snapshotKey, dag, rootVertex, vertex); err != nil { return err From c9903ed152062162328482c1b9b08e822c031d58 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Mon, 17 Apr 2023 14:34:20 +0800 Subject: [PATCH 055/439] chore: improve pvc usage for full/incremental backup (#2546) --- apis/dataprotection/v1alpha1/backup_types.go | 3 +- .../v1alpha1/backuppolicy_types.go | 27 +- apis/dataprotection/v1alpha1/types.go | 10 + ...otection.kubeblocks.io_backuppolicies.yaml | 3212 +---------------- .../dataprotection.kubeblocks.io_backups.yaml | 1532 +------- controllers/apps/cluster_controller_test.go | 4 +- .../dataprotection/backup_controller.go | 66 +- .../dataprotection/backup_controller_test.go | 12 +- .../dataprotection/backuppolicy_controller.go | 32 +- .../backuppolicy_controller_test.go | 37 +- .../dataprotection/restorejob_controller.go | 17 +- .../restorejob_controller_test.go | 2 +- controllers/dataprotection/type.go | 3 +- .../apecloud-mysql/templates/backuptool.yaml | 7 +- deploy/csi-s3/templates/_helpers.tpl | 10 + deploy/csi-s3/templates/attacher.yaml | 12 +- deploy/csi-s3/templates/provisioner.yaml | 12 +- deploy/csi-s3/templates/storageclass.yaml | 2 +- deploy/csi-s3/values.yaml | 10 +- ...otection.kubeblocks.io_backuppolicies.yaml | 3212 +---------------- .../dataprotection.kubeblocks.io_backups.yaml | 1532 +------- .../helm/templates/addons/csi-s3-addon.yaml | 9 - deploy/helm/templates/configmap.yaml | 25 + deploy/helm/templates/deployment.yaml | 4 +- deploy/helm/values.yaml | 9 +- docs/user_docs/cli/cli.md | 7 +- docs/user_docs/cli/kbcli.md | 1 - docs/user_docs/cli/kbcli_kubeblocks.md | 1 + internal/cli/cmd/backupconfig/suite_test.go | 29 - internal/cli/cmd/cli.go | 2 - .../backup_config.go => kubeblocks/config.go} | 40 +- .../config_test.go} | 9 +- internal/cli/cmd/kubeblocks/kubeblocks.go | 1 + internal/constant/const.go | 6 + .../controller/component/restore_utils.go | 19 +- .../component/restore_utils_test.go | 4 +- .../transformer_backup_policy_tpl.go | 18 +- .../testutil/apps/backuppolicy_factory.go | 34 +- test/integration/backup_mysql_test.go | 3 +- 39 files changed, 469 insertions(+), 9506 deletions(-) create mode 100644 deploy/csi-s3/templates/_helpers.tpl create mode 100644 deploy/helm/templates/configmap.yaml delete mode 100644 internal/cli/cmd/backupconfig/suite_test.go rename internal/cli/cmd/{backupconfig/backup_config.go => kubeblocks/config.go} (52%) rename internal/cli/cmd/{backupconfig/backup_config_test.go => kubeblocks/config_test.go} (92%) diff --git a/apis/dataprotection/v1alpha1/backup_types.go b/apis/dataprotection/v1alpha1/backup_types.go index a855ca849..97ad98c0f 100644 --- a/apis/dataprotection/v1alpha1/backup_types.go +++ b/apis/dataprotection/v1alpha1/backup_types.go @@ -19,7 +19,6 @@ package v1alpha1 import ( "fmt" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -77,7 +76,7 @@ type BackupStatus struct { // remoteVolume saves the backup data. // +optional - RemoteVolume *corev1.Volume `json:"remoteVolume,omitempty"` + PersistentVolumeClaimName string `json:"persistentVolumeClaimName,omitempty"` // backupToolName referenced backup tool name. // +optional diff --git a/apis/dataprotection/v1alpha1/backuppolicy_types.go b/apis/dataprotection/v1alpha1/backuppolicy_types.go index 931cc0bbb..33ba451b7 100644 --- a/apis/dataprotection/v1alpha1/backuppolicy_types.go +++ b/apis/dataprotection/v1alpha1/backuppolicy_types.go @@ -21,7 +21,7 @@ import ( "strings" "time" - corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -88,9 +88,9 @@ type SnapshotPolicy struct { type CommonBackupPolicy struct { BasePolicy `json:",inline"` - // array of remote volumes from CSI driver definition. + // refer to PersistentVolumeClaim and the backup data will be stored in the corresponding persistent volume. // +kubebuilder:validation:Required - RemoteVolume corev1.Volume `json:"remoteVolume"` + PersistentVolumeClaim PersistentVolumeClaim `json:"persistentVolumeClaim"` // which backup tool to perform database backup, only support one tool. // +kubebuilder:validation:Required @@ -98,6 +98,27 @@ type CommonBackupPolicy struct { BackupToolName string `json:"backupToolName,omitempty"` } +type PersistentVolumeClaim struct { + // the name of the PersistentVolumeClaim. + Name string `json:"name"` + + // storageClassName is the name of the StorageClass required by the claim. + // +optional + StorageClassName *string `json:"storageClassName,omitempty"` + + // initCapacity represents the init storage size of the PersistentVolumeClaim which should be created if not exist. + // and the default value is 100Gi if it is empty. + // +optional + InitCapacity resource.Quantity `json:"initCapacity,omitempty"` + + // createPolicy defines the policy for creating the PersistentVolumeClaim, enum values: + // - Never: do nothing if the PersistentVolumeClaim not exist. + // - IfNotPresent: create the PersistentVolumeClaim if not present and the accessModes only contains 'ReadWriteMany'. + // +kubebuilder:default=IfNotPresent + // +optional + CreatePolicy CreatePVCPolicy `json:"createPolicy"` +} + type BasePolicy struct { // target database cluster for backup. // +kubebuilder:validation:Required diff --git a/apis/dataprotection/v1alpha1/types.go b/apis/dataprotection/v1alpha1/types.go index 4c459db01..ecfae0849 100644 --- a/apis/dataprotection/v1alpha1/types.go +++ b/apis/dataprotection/v1alpha1/types.go @@ -49,6 +49,16 @@ const ( BaseBackupTypeSnapshot BaseBackupType = "snapshot" ) +// CreatePVCPolicy the policy how to create the PersistentVolumeClaim for backup. +// +enum +// +kubebuilder:validation:Enum={IfNotPresent,Never} +type CreatePVCPolicy string + +const ( + CreatePVCPolicyNever CreatePVCPolicy = "Never" + CreatePVCPolicyIfNotPresent CreatePVCPolicy = "IfNotPresent" +) + // BackupPolicyPhase defines phases for BackupPolicy CR. // +enum // +kubebuilder:validation:Enum={Available,Failed} diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml index 870a98b38..9e4b7bd9a 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -97,1585 +97,37 @@ spec: description: count of backup stop retries on fail. format: int32 type: integer - remoteVolume: - description: array of remote volumes from CSI driver definition. + persistentVolumeClaim: + description: refer to PersistentVolumeClaim and the backup data + will be stored in the corresponding persistent volume. properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk - resource that is attached to a kubelet''s host machine and - then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume - that you want to mount. If omitted, the default is to - mount by volume name. Examples: For volume /dev/sda1, - you specify the partition as "1". Similarly, the volume - partition for /dev/sda is "0" (or you can leave the - property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent - disk resource in AWS (Amazon EBS volume). More info: - https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount - on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in - the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob - storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple - blob disks per storage account Dedicated: single blob - disk per storage account Managed: azure managed data - disk (only in managed availability set). defaults to - shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host - that shares a pod's lifetime - properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the - path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - user: - description: 'user is optional: User is the rados user - name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and - mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating - system. Examples: "ext4", "xfs", "ntfs". Implicitly - inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret - object containing parameters used to connect to OpenStack.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - volumeID: - description: 'volumeID used to identify the volume in - cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should - populate this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used - to set permissions on created files by default. Must - be an octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within the path - are not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair - in the Data field of the referenced ConfigMap will be - projected into the volume as a file whose name is the - key and content is the value. If specified, the listed - keys will be projected into the specified paths, and - unlisted keys will not be present. If a key is specified - which is not present in the ConfigMap, the volume setup - will error unless it is marked optional. Paths must - be relative and may not contain the '..' path or start - with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for - mode bits. If not specified, the volume defaultMode - will be used. This might be in conflict with other - options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or - its keys must be defined - type: boolean - type: object - csi: - description: csi (Container Storage Interface) represents - ephemeral storage that is handled by certain external CSI - drivers (Beta feature). - properties: - driver: - description: driver is the name of the CSI driver that - handles this volume. Consult with your admin for the - correct name as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem - to apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the - secret object containing sensitive information to pass - to the CSI driver to complete the CSI NodePublishVolume - and NodeUnpublishVolume calls. This field is optional, - and may be empty if no secret is required. If the secret - object contains more than one secret, all secret references - are passed. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the - pod that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set - permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode bits. - Defaults to 0644. Directories within the path are not - affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are - supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in - the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. If not - specified, the volume defaultMode will be used. - This might be in conflict with other options that - affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path - name of the file to be created. Must not be absolute - or contain the ''..'' path. Must be utf-8 encoded. - The first item of the relative path must not start - with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of - the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that - shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which - means to use the node''s default medium. Must be an - empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is - also applicable for memory medium. The maximum usage - on memory medium EmptyDir would be the minimum value - between the SizeLimit specified here and the sum of - memory limits of all containers in a pod. The default - is nil which means that the limit is undefined. More - info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled - by a cluster storage driver. The volume's lifecycle is tied - to the pod that defines it - it will be created before the - pod starts, and deleted when the pod is removed. \n Use - this if: a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot - or capacity tracking are needed, c) the storage driver is - specified through a storage class, and d) the storage driver - supports dynamic volume provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information on the connection - between this volume type and PersistentVolumeClaim). \n - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. \n Use CSI for light-weight local - ephemeral volumes if the CSI driver is meant to be used - that way - see the documentation of the driver for more - information. \n A pod can use both types of ephemeral volumes - and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC - to provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC - will be deleted together with the pod. The name of - the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated - name is not valid for a PVC (for example, too long). - \n An existing PVC with that name that is not owned - by the pod will *not* be used for the pod to avoid using - an unrelated volume by mistake. Starting the pod is - then blocked until the unrelated PVC is removed. If - such a pre-created PVC is meant to be used by the pod, - the PVC has to updated with an owner reference to the - pod once the pod exists. Normally this should not be - necessary, but it may be useful when manually reconstructing - a broken cluster. \n This field is read-only and no - changes will be made by Kubernetes to the PVC after - it has been created. \n Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations that - will be copied into the PVC when creating it. No - other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the - PVC that gets created from this template. The same - fields as in a PersistentVolumeClaim are also valid - here. - properties: - accessModes: - description: 'accessModes contains the desired - access modes the volume should have. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to - specify either: * An existing VolumeSnapshot - object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If - the provisioner or an external controller can - support the specified data source, it will create - a new volume based on the contents of the specified - data source. When the AnyVolumeDataSource feature - gate is enabled, dataSource contents will be - copied to dataSourceRef, and dataSourceRef contents - will be copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the - resource being referenced. If APIGroup is - not specified, the specified Kind must be - in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - required: - - kind - - name - type: object - dataSourceRef: - description: 'dataSourceRef specifies the object - from which to populate the volume with data, - if a non-empty volume is desired. This may be - any object from a non-empty API group (non core - object) or a PersistentVolumeClaim object. When - this field is specified, volume binding will - only succeed if the type of the specified object - matches some installed volume populator or dynamic - provisioner. This field will replace the functionality - of the dataSource field and as such if both - fields are non-empty, they must have the same - value. For backwards compatibility, when namespace - isn''t specified in dataSourceRef, both fields - (dataSource and dataSourceRef) will be set to - the same value automatically if one of them - is empty and the other is non-empty. When namespace - is specified in dataSourceRef, dataSource isn''t - set to the same value and must be empty. There - are three important differences between dataSource - and dataSourceRef: * While dataSource only allows - two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim - objects. * While dataSource ignores disallowed - values (dropping them), dataSourceRef preserves - all values, and generates an error if a disallowed - value is specified. * While dataSource only - allows local objects, dataSourceRef allows objects - in any namespaces. (Beta) Using this field requires - the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef - requires the CrossNamespaceVolumeDataSource - feature gate to be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the - resource being referenced. If APIGroup is - not specified, the specified Kind must be - in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - namespace: - description: Namespace is the namespace of - resource being referenced Note that when - a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept - the reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires - the CrossNamespaceVolumeDataSource feature - gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the minimum - resources the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity - recorded in the status field of the claim. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are - used by this container. \n This is an alpha - field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name - of one entry in pod.spec.resourceClaims - of the Pod where this field is used. - It makes that resource available inside - a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum - amount of compute resources allowed. More - info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If - Requests is omitted for a container, it - defaults to Limits if that is explicitly - specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, a - key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - storageClassName: - description: 'storageClassName is the name of - the StorageClass required by the claim. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem - is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference - to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is - attached to a kubelet's host machine and then exposed to - the pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. TODO: how do we prevent - errors in the filesystem from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and - lun must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource - that is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for - this volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". The default filesystem - depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: 'readOnly is Optional: defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information - to pass to the plugin scripts. This may be empty if - no secret object is specified. If the secret object - contains more than one secret, all secrets are passed - to the plugin scripts.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached - to a kubelet's host machine. This depends on the Flocker - control service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored - as metadata -> name on the dataset for Flocker should - be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume - that you want to mount. If omitted, the default is to - mount by volume name. Examples: For volume /dev/sda1, - you specify the partition as "1". Similarly, the volume - partition for /dev/sda is "0" (or you can leave the - property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource - in GCE. Used to identify the disk in GCE. More info: - https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision - a container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir - into the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, - the volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More - info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to - false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' - properties: - path: - description: 'path of the directory on the host. If the - path is a symlink, it will follow the link to the real - path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that - is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support - iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator - Name. If initiatorName is specified with iscsiInterface - simultaneously, new iSCSI interface : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that - uses an iSCSI transport. Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. - The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - targetPortal: - description: targetPortal is iSCSI Target Portal. The - Portal is either an IP or ip_addr:port if the port is - other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object + createPolicy: + default: IfNotPresent + description: 'createPolicy defines the policy for creating + the PersistentVolumeClaim, enum values: - Never: do nothing + if the PersistentVolumeClaim not exist. - IfNotPresent: + create the PersistentVolumeClaim if not present and the + accessModes only contains ''ReadWriteMany''.' + enum: + - IfNotPresent + - Never + type: string + initCapacity: + anyOf: + - type: integer + - type: string + description: initCapacity represents the init storage size + of the PersistentVolumeClaim which should be created if + not exist. and the default value is 100Gi if it is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true name: - description: 'name of the volume. Must be a DNS_LABEL and - unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: the name of the PersistentVolumeClaim. + type: string + storageClassName: + description: storageClassName is the name of the StorageClass + required by the claim. type: string - nfs: - description: 'nfs represents an NFS mount on the host that - shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export - to be mounted with read-only permissions. Defaults to - false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of - the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents - a reference to a PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting - in VolumeMounts. Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to - mount Must be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set - permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this - setting. This might be in conflict with other options - that affect the file mode, like fsGroup, and the result - can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along - with other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the ConfigMap, the - volume setup will error unless it is marked - optional. Paths must be relative and may not - contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used - to set permissions on this file, must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. - YAML accepts both octal and decimal - values, JSON requires decimal values - for mode bits. If not specified, the - volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. - Must not be absolute or contain the - ''..'' path. Must be utf-8 encoded. - The first item of the relative path - must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and - requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are - currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the Secret, the volume - setup will error unless it is marked optional. - Paths must be relative and may not contain - the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional field specify whether - the Secret or its key must be defined - type: boolean - type: object - serviceAccountToken: - description: serviceAccountToken is information - about the serviceAccountToken data to project - properties: - audience: - description: audience is the intended audience - of the token. A recipient of a token must - identify itself with an identifier specified - in the audience of the token, and otherwise - should reject the token. The audience defaults - to the identifier of the apiserver. - type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account - token. As the token approaches expiration, - the kubelet volume plugin will proactively - rotate the service account token. The kubelet - will start trying to rotate the token if the - token is older than 80 percent of its time - to live or if the token is older than 24 hours.Defaults - to 1 hour and must be at least 10 minutes. - format: int64 - type: integer - path: - description: path is the path relative to the - mount point of the file to project the token - into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host - that shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is - no group - type: string - readOnly: - description: readOnly here will force the Quobyte volume - to be mounted with read-only permissions. Defaults to - false. - type: boolean - registry: - description: registry represents a single or multiple - Quobyte Registry services specified as a string as host:port - pair (multiple entries are separated with commas) which - acts as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in - the Backend Used with dynamically provisioned Quobyte - volumes, value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to - serivceaccount user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on - the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - image: - description: 'image is the rados image name. More info: - https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is - rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication - secret for RBDUser. If provided overrides keyring. Default - is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - user: - description: 'user is the rados user name. Default is - admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume - attached and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO - API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO - Protection Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not - provided, Login operation will fail. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage - for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system - as configured in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already - created in the ScaleIO system that is associated with - this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used - to set permissions on created files by default. Must - be an octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within the path - are not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair - in the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and - content is the value. If specified, the listed keys - will be projected into the specified paths, and unlisted - keys will not be present. If a key is specified which - is not present in the Secret, the volume setup will - error unless it is marked optional. Paths must be relative - and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for - mode bits. If not specified, the volume defaultMode - will be used. This might be in conflict with other - options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret - or its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in - the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for - obtaining the StorageOS API credentials. If not specified, - default values will be attempted. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - volumeName: - description: volumeName is the human-readable name of - the StorageOS volume. Volume names are only unique - within a namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the - volume within StorageOS. If no namespace is specified - then the Pod's namespace will be used. This allows - the Kubernetes name scoping to be mirrored within StorageOS - for tighter integration. Set VolumeName to any name - to override the default behaviour. Set to "default" - if you are not using namespaces within StorageOS. Namespaces - that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fsType is filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based - Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object required: - name type: object @@ -1757,7 +209,7 @@ spec: - labelsSelector type: object required: - - remoteVolume + - persistentVolumeClaim - target type: object incremental: @@ -1807,1585 +259,37 @@ spec: description: count of backup stop retries on fail. format: int32 type: integer - remoteVolume: - description: array of remote volumes from CSI driver definition. + persistentVolumeClaim: + description: refer to PersistentVolumeClaim and the backup data + will be stored in the corresponding persistent volume. properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk - resource that is attached to a kubelet''s host machine and - then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume - that you want to mount. If omitted, the default is to - mount by volume name. Examples: For volume /dev/sda1, - you specify the partition as "1". Similarly, the volume - partition for /dev/sda is "0" (or you can leave the - property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent - disk resource in AWS (Amazon EBS volume). More info: - https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount - on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in - the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob - storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple - blob disks per storage account Dedicated: single blob - disk per storage account Managed: azure managed data - disk (only in managed availability set). defaults to - shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host - that shares a pod's lifetime - properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the - path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - user: - description: 'user is optional: User is the rados user - name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and - mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating - system. Examples: "ext4", "xfs", "ntfs". Implicitly - inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret - object containing parameters used to connect to OpenStack.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - volumeID: - description: 'volumeID used to identify the volume in - cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should - populate this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used - to set permissions on created files by default. Must - be an octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within the path - are not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair - in the Data field of the referenced ConfigMap will be - projected into the volume as a file whose name is the - key and content is the value. If specified, the listed - keys will be projected into the specified paths, and - unlisted keys will not be present. If a key is specified - which is not present in the ConfigMap, the volume setup - will error unless it is marked optional. Paths must - be relative and may not contain the '..' path or start - with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for - mode bits. If not specified, the volume defaultMode - will be used. This might be in conflict with other - options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or - its keys must be defined - type: boolean - type: object - csi: - description: csi (Container Storage Interface) represents - ephemeral storage that is handled by certain external CSI - drivers (Beta feature). - properties: - driver: - description: driver is the name of the CSI driver that - handles this volume. Consult with your admin for the - correct name as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem - to apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the - secret object containing sensitive information to pass - to the CSI driver to complete the CSI NodePublishVolume - and NodeUnpublishVolume calls. This field is optional, - and may be empty if no secret is required. If the secret - object contains more than one secret, all secret references - are passed. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the - pod that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set - permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode bits. - Defaults to 0644. Directories within the path are not - affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are - supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in - the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. If not - specified, the volume defaultMode will be used. - This might be in conflict with other options that - affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path - name of the file to be created. Must not be absolute - or contain the ''..'' path. Must be utf-8 encoded. - The first item of the relative path must not start - with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of - the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that - shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which - means to use the node''s default medium. Must be an - empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is - also applicable for memory medium. The maximum usage - on memory medium EmptyDir would be the minimum value - between the SizeLimit specified here and the sum of - memory limits of all containers in a pod. The default - is nil which means that the limit is undefined. More - info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled - by a cluster storage driver. The volume's lifecycle is tied - to the pod that defines it - it will be created before the - pod starts, and deleted when the pod is removed. \n Use - this if: a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot - or capacity tracking are needed, c) the storage driver is - specified through a storage class, and d) the storage driver - supports dynamic volume provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information on the connection - between this volume type and PersistentVolumeClaim). \n - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. \n Use CSI for light-weight local - ephemeral volumes if the CSI driver is meant to be used - that way - see the documentation of the driver for more - information. \n A pod can use both types of ephemeral volumes - and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC - to provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC - will be deleted together with the pod. The name of - the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated - name is not valid for a PVC (for example, too long). - \n An existing PVC with that name that is not owned - by the pod will *not* be used for the pod to avoid using - an unrelated volume by mistake. Starting the pod is - then blocked until the unrelated PVC is removed. If - such a pre-created PVC is meant to be used by the pod, - the PVC has to updated with an owner reference to the - pod once the pod exists. Normally this should not be - necessary, but it may be useful when manually reconstructing - a broken cluster. \n This field is read-only and no - changes will be made by Kubernetes to the PVC after - it has been created. \n Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations that - will be copied into the PVC when creating it. No - other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the - PVC that gets created from this template. The same - fields as in a PersistentVolumeClaim are also valid - here. - properties: - accessModes: - description: 'accessModes contains the desired - access modes the volume should have. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to - specify either: * An existing VolumeSnapshot - object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If - the provisioner or an external controller can - support the specified data source, it will create - a new volume based on the contents of the specified - data source. When the AnyVolumeDataSource feature - gate is enabled, dataSource contents will be - copied to dataSourceRef, and dataSourceRef contents - will be copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the - resource being referenced. If APIGroup is - not specified, the specified Kind must be - in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - required: - - kind - - name - type: object - dataSourceRef: - description: 'dataSourceRef specifies the object - from which to populate the volume with data, - if a non-empty volume is desired. This may be - any object from a non-empty API group (non core - object) or a PersistentVolumeClaim object. When - this field is specified, volume binding will - only succeed if the type of the specified object - matches some installed volume populator or dynamic - provisioner. This field will replace the functionality - of the dataSource field and as such if both - fields are non-empty, they must have the same - value. For backwards compatibility, when namespace - isn''t specified in dataSourceRef, both fields - (dataSource and dataSourceRef) will be set to - the same value automatically if one of them - is empty and the other is non-empty. When namespace - is specified in dataSourceRef, dataSource isn''t - set to the same value and must be empty. There - are three important differences between dataSource - and dataSourceRef: * While dataSource only allows - two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim - objects. * While dataSource ignores disallowed - values (dropping them), dataSourceRef preserves - all values, and generates an error if a disallowed - value is specified. * While dataSource only - allows local objects, dataSourceRef allows objects - in any namespaces. (Beta) Using this field requires - the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef - requires the CrossNamespaceVolumeDataSource - feature gate to be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the - resource being referenced. If APIGroup is - not specified, the specified Kind must be - in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - namespace: - description: Namespace is the namespace of - resource being referenced Note that when - a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept - the reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires - the CrossNamespaceVolumeDataSource feature - gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the minimum - resources the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity - recorded in the status field of the claim. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are - used by this container. \n This is an alpha - field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name - of one entry in pod.spec.resourceClaims - of the Pod where this field is used. - It makes that resource available inside - a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum - amount of compute resources allowed. More - info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If - Requests is omitted for a container, it - defaults to Limits if that is explicitly - specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, a - key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - storageClassName: - description: 'storageClassName is the name of - the StorageClass required by the claim. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem - is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference - to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is - attached to a kubelet's host machine and then exposed to - the pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. TODO: how do we prevent - errors in the filesystem from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and - lun must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource - that is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for - this volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". The default filesystem - depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: 'readOnly is Optional: defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information - to pass to the plugin scripts. This may be empty if - no secret object is specified. If the secret object - contains more than one secret, all secrets are passed - to the plugin scripts.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached - to a kubelet's host machine. This depends on the Flocker - control service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored - as metadata -> name on the dataset for Flocker should - be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume - that you want to mount. If omitted, the default is to - mount by volume name. Examples: For volume /dev/sda1, - you specify the partition as "1". Similarly, the volume - partition for /dev/sda is "0" (or you can leave the - property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource - in GCE. Used to identify the disk in GCE. More info: - https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision - a container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir - into the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, - the volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More - info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to - false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' - properties: - path: - description: 'path of the directory on the host. If the - path is a symlink, it will follow the link to the real - path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that - is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support - iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator - Name. If initiatorName is specified with iscsiInterface - simultaneously, new iSCSI interface : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that - uses an iSCSI transport. Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. - The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - targetPortal: - description: targetPortal is iSCSI Target Portal. The - Portal is either an IP or ip_addr:port if the port is - other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object + createPolicy: + default: IfNotPresent + description: 'createPolicy defines the policy for creating + the PersistentVolumeClaim, enum values: - Never: do nothing + if the PersistentVolumeClaim not exist. - IfNotPresent: + create the PersistentVolumeClaim if not present and the + accessModes only contains ''ReadWriteMany''.' + enum: + - IfNotPresent + - Never + type: string + initCapacity: + anyOf: + - type: integer + - type: string + description: initCapacity represents the init storage size + of the PersistentVolumeClaim which should be created if + not exist. and the default value is 100Gi if it is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true name: - description: 'name of the volume. Must be a DNS_LABEL and - unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: the name of the PersistentVolumeClaim. + type: string + storageClassName: + description: storageClassName is the name of the StorageClass + required by the claim. type: string - nfs: - description: 'nfs represents an NFS mount on the host that - shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export - to be mounted with read-only permissions. Defaults to - false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of - the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents - a reference to a PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting - in VolumeMounts. Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to - mount Must be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set - permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this - setting. This might be in conflict with other options - that affect the file mode, like fsGroup, and the result - can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along - with other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the ConfigMap, the - volume setup will error unless it is marked - optional. Paths must be relative and may not - contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used - to set permissions on this file, must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. - YAML accepts both octal and decimal - values, JSON requires decimal values - for mode bits. If not specified, the - volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. - Must not be absolute or contain the - ''..'' path. Must be utf-8 encoded. - The first item of the relative path - must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and - requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are - currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the Secret, the volume - setup will error unless it is marked optional. - Paths must be relative and may not contain - the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional field specify whether - the Secret or its key must be defined - type: boolean - type: object - serviceAccountToken: - description: serviceAccountToken is information - about the serviceAccountToken data to project - properties: - audience: - description: audience is the intended audience - of the token. A recipient of a token must - identify itself with an identifier specified - in the audience of the token, and otherwise - should reject the token. The audience defaults - to the identifier of the apiserver. - type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account - token. As the token approaches expiration, - the kubelet volume plugin will proactively - rotate the service account token. The kubelet - will start trying to rotate the token if the - token is older than 80 percent of its time - to live or if the token is older than 24 hours.Defaults - to 1 hour and must be at least 10 minutes. - format: int64 - type: integer - path: - description: path is the path relative to the - mount point of the file to project the token - into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host - that shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is - no group - type: string - readOnly: - description: readOnly here will force the Quobyte volume - to be mounted with read-only permissions. Defaults to - false. - type: boolean - registry: - description: registry represents a single or multiple - Quobyte Registry services specified as a string as host:port - pair (multiple entries are separated with commas) which - acts as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in - the Backend Used with dynamically provisioned Quobyte - volumes, value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to - serivceaccount user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on - the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - image: - description: 'image is the rados image name. More info: - https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is - rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication - secret for RBDUser. If provided overrides keyring. Default - is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - user: - description: 'user is the rados user name. Default is - admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume - attached and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO - API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO - Protection Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not - provided, Login operation will fail. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage - for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system - as configured in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already - created in the ScaleIO system that is associated with - this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used - to set permissions on created files by default. Must - be an octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within the path - are not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair - in the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and - content is the value. If specified, the listed keys - will be projected into the specified paths, and unlisted - keys will not be present. If a key is specified which - is not present in the Secret, the volume setup will - error unless it is marked optional. Paths must be relative - and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for - mode bits. If not specified, the volume defaultMode - will be used. This might be in conflict with other - options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret - or its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in - the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for - obtaining the StorageOS API credentials. If not specified, - default values will be attempted. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - volumeName: - description: volumeName is the human-readable name of - the StorageOS volume. Volume names are only unique - within a namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the - volume within StorageOS. If no namespace is specified - then the Pod's namespace will be used. This allows - the Kubernetes name scoping to be mirrored within StorageOS - for tighter integration. Set VolumeName to any name - to override the default behaviour. Set to "default" - if you are not using namespaces within StorageOS. Namespaces - that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fsType is filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based - Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object required: - name type: object @@ -3467,7 +371,7 @@ spec: - labelsSelector type: object required: - - remoteVolume + - persistentVolumeClaim - target type: object schedule: diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml index 4542c6957..062b596c3 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml @@ -166,6 +166,9 @@ spec: parentBackupName: description: record parentBackupName if backupType is incremental. type: string + persistentVolumeClaimName: + description: remoteVolume saves the backup data. + type: string phase: description: BackupPhase The current phase. Valid values are New, InProgress, Completed, Failed. @@ -175,1535 +178,6 @@ spec: - Completed - Failed type: string - remoteVolume: - description: remoteVolume saves the backup data. - properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount by - volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent disk - resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount on - the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the - blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must be a - filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob - disks per storage account Dedicated: single blob disk per - storage account Managed: azure managed data disk (only - in managed availability set). defaults to shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host that - shares a pod's lifetime - properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the path - to key ring for User, default is /etc/ceph/user.secret More - info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - user: - description: 'user is optional: User is the rados user name, - default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and mounted - on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. More - info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret object - containing parameters used to connect to OpenStack.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - volumeID: - description: 'volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate - this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect - the file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair in - the Data field of the referenced ConfigMap will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the - ConfigMap, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to set - permissions on this file. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: path is the relative path of the file to - map the key to. May not be an absolute path. May not - contain the path element '..'. May not start with - the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or its - keys must be defined - type: boolean - type: object - csi: - description: csi (Container Storage Interface) represents ephemeral - storage that is handled by certain external CSI drivers (Beta - feature). - properties: - driver: - description: driver is the name of the CSI driver that handles - this volume. Consult with your admin for the correct name - as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". If - not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem to - apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the secret - object containing sensitive information to pass to the CSI - driver to complete the CSI NodePublishVolume and NodeUnpublishVolume - calls. This field is optional, and may be empty if no secret - is required. If the secret object contains more than one - secret, all secret references are passed. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod - that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Defaults to 0644. Directories within - the path are not affected by this setting. This might be - in conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name - of the file to be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 encoded. The - first item of the relative path must not start with - ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: only - resources limits and requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that shares - a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which means - to use the node''s default medium. Must be an empty string - (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is also - applicable for memory medium. The maximum usage on memory - medium EmptyDir would be the minimum value between the SizeLimit - specified here and the sum of memory limits of all containers - in a pod. The default is nil which means that the limit - is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled by - a cluster storage driver. The volume's lifecycle is tied to - the pod that defines it - it will be created before the pod - starts, and deleted when the pod is removed. \n Use this if: - a) the volume is only needed while the pod runs, b) features - of normal volumes like restoring from snapshot or capacity tracking - are needed, c) the storage driver is specified through a storage - class, and d) the storage driver supports dynamic volume provisioning - through a PersistentVolumeClaim (see EphemeralVolumeSource for - more information on the connection between this volume type - and PersistentVolumeClaim). \n Use PersistentVolumeClaim or - one of the vendor-specific APIs for volumes that persist for - longer than the lifecycle of an individual pod. \n Use CSI for - light-weight local ephemeral volumes if the CSI driver is meant - to be used that way - see the documentation of the driver for - more information. \n A pod can use both types of ephemeral volumes - and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC to - provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC will - be deleted together with the pod. The name of the PVC will - be `-` where `` is the - name from the `PodSpec.Volumes` array entry. Pod validation - will reject the pod if the concatenated name is not valid - for a PVC (for example, too long). \n An existing PVC with - that name that is not owned by the pod will *not* be used - for the pod to avoid using an unrelated volume by mistake. - Starting the pod is then blocked until the unrelated PVC - is removed. If such a pre-created PVC is meant to be used - by the pod, the PVC has to updated with an owner reference - to the pod once the pod exists. Normally this should not - be necessary, but it may be useful when manually reconstructing - a broken cluster. \n This field is read-only and no changes - will be made by Kubernetes to the PVC after it has been - created. \n Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations that will - be copied into the PVC when creating it. No other fields - are allowed and will be rejected during validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the PVC - that gets created from this template. The same fields - as in a PersistentVolumeClaim are also valid here. - properties: - accessModes: - description: 'accessModes contains the desired access - modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to specify - either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If the - provisioner or an external controller can support - the specified data source, it will create a new - volume based on the contents of the specified data - source. When the AnyVolumeDataSource feature gate - is enabled, dataSource contents will be copied to - dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API group. - For any other third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - required: - - kind - - name - type: object - dataSourceRef: - description: 'dataSourceRef specifies the object from - which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a - non-empty API group (non core object) or a PersistentVolumeClaim - object. When this field is specified, volume binding - will only succeed if the type of the specified object - matches some installed volume populator or dynamic - provisioner. This field will replace the functionality - of the dataSource field and as such if both fields - are non-empty, they must have the same value. For - backwards compatibility, when namespace isn''t specified - in dataSourceRef, both fields (dataSource and dataSourceRef) - will be set to the same value automatically if one - of them is empty and the other is non-empty. When - namespace is specified in dataSourceRef, dataSource - isn''t set to the same value and must be empty. - There are three important differences between dataSource - and dataSourceRef: * While dataSource only allows - two specific types of objects, dataSourceRef allows - any non-core object, as well as PersistentVolumeClaim - objects. * While dataSource ignores disallowed values - (dropping them), dataSourceRef preserves all values, - and generates an error if a disallowed value is - specified. * While dataSource only allows local - objects, dataSourceRef allows objects in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource - feature gate to be enabled. (Alpha) Using the namespace - field of dataSourceRef requires the CrossNamespaceVolumeDataSource - feature gate to be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API group. - For any other third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - namespace: - description: Namespace is the namespace of resource - being referenced Note that when a namespace - is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept the - reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires the - CrossNamespaceVolumeDataSource feature gate - to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the minimum resources - the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity recorded - in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are used - by this container. \n This is an alpha field - and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one entry - in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of - one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes - that resource available inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount - of compute resources required. If Requests is - omitted for a container, it defaults to Limits - if that is explicitly specified, otherwise to - an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement is - a selector that contains values, a key, and - an operator that relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. If - the operator is Exists or DoesNotExist, - the values array must be empty. This array - is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is "In", - and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - storageClassName: - description: 'storageClassName is the name of the - StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem is - implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to - the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached - to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and lun - must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource that - is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for this - volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends - on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: 'readOnly is Optional: defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information to - pass to the plugin scripts. This may be empty if no secret - object is specified. If the secret object contains more - than one secret, all secrets are passed to the plugin scripts.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached to a - kubelet's host machine. This depends on the Flocker control - service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored as - metadata -> name on the dataset for Flocker should be considered - as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume that - you want to mount. Tip: Ensure that the filesystem type - is supported by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount by - volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property empty). More - info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource in - GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision a - container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir into - the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, the - volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the host - that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More info: - https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' - properties: - path: - description: 'path of the directory on the host. If the path - is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" More - info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that is - attached to a kubelet''s host machine and then exposed to the - pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI - Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, - new iSCSI interface : will be - created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that uses - an iSCSI transport. Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. The - portal is either an IP or ip_addr:port if the port is other - than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - targetPortal: - description: targetPortal is iSCSI Target Portal. The Portal - is either an IP or ip_addr:port if the port is other than - default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: 'name of the volume. Must be a DNS_LABEL and unique - within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - nfs: - description: 'nfs represents an NFS mount on the host that shares - a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export to be - mounted with read-only permissions. Defaults to false. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of the - NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents a reference - to a PersistentVolumeClaim in the same namespace. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Directories within the path are not - affected by this setting. This might be in conflict with - other options that affect the file mode, like fsGroup, and - the result can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along with - other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file whose - name is the key and content is the value. If specified, - the listed keys will be projected into the specified - paths, and unlisted keys will not be present. - If a key is specified which is not present in - the ConfigMap, the volume setup will error unless - it is marked optional. Paths must be relative - and may not contain the '..' path or start with - '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. If - not specified, the volume defaultMode will - be used. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing the - pod field - properties: - fieldRef: - description: 'Required: Selects a field of - the pod: only annotations, labels, name - and namespace are supported.' - properties: - apiVersion: - description: Version of the schema the - FieldPath is written in terms of, defaults - to "v1". - type: string - fieldPath: - description: Path of the field to select - in the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used to - set permissions on this file, must be an - octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both - octal and decimal values, JSON requires - decimal values for mode bits. If not specified, - the volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. Must - not be absolute or contain the ''..'' path. - Must be utf-8 encoded. The first item of - the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format - of the exposed resources, defaults to - "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret data - to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file whose - name is the key and content is the value. If specified, - the listed keys will be projected into the specified - paths, and unlisted keys will not be present. - If a key is specified which is not present in - the Secret, the volume setup will error unless - it is marked optional. Paths must be relative - and may not contain the '..' path or start with - '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. If - not specified, the volume defaultMode will - be used. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional field specify whether the - Secret or its key must be defined - type: boolean - type: object - serviceAccountToken: - description: serviceAccountToken is information about - the serviceAccountToken data to project - properties: - audience: - description: audience is the intended audience of - the token. A recipient of a token must identify - itself with an identifier specified in the audience - of the token, and otherwise should reject the - token. The audience defaults to the identifier - of the apiserver. - type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account token. - As the token approaches expiration, the kubelet - volume plugin will proactively rotate the service - account token. The kubelet will start trying to - rotate the token if the token is older than 80 - percent of its time to live or if the token is - older than 24 hours.Defaults to 1 hour and must - be at least 10 minutes. - format: int64 - type: integer - path: - description: path is the path relative to the mount - point of the file to project the token into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host that - shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is no group - type: string - readOnly: - description: readOnly here will force the Quobyte volume to - be mounted with read-only permissions. Defaults to false. - type: boolean - registry: - description: registry represents a single or multiple Quobyte - Registry services specified as a string as host:port pair - (multiple entries are separated with commas) which acts - as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in the - Backend Used with dynamically provisioned Quobyte volumes, - value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to serivceaccount - user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - image: - description: 'image is the rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. More - info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication secret - for RBDUser. If provided overrides keyring. Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - user: - description: 'user is the rados user name. Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API - Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection - Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not provided, - Login operation will fail. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage for - a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system as configured - in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already created - in the ScaleIO system that is associated with this volume - source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect - the file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair in - the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the - Secret, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to set - permissions on this file. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: path is the relative path of the file to - map the key to. May not be an absolute path. May not - contain the path element '..'. May not start with - the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret or - its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in the - pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for obtaining - the StorageOS API credentials. If not specified, default - values will be attempted. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - volumeName: - description: volumeName is the human-readable name of the - StorageOS volume. Volume names are only unique within a - namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the volume - within StorageOS. If no namespace is specified then the - Pod's namespace will be used. This allows the Kubernetes - name scoping to be mirrored within StorageOS for tighter - integration. Set VolumeName to any name to override the - default behaviour. Set to "default" if you are not using - namespaces within StorageOS. Namespaces that do not pre-exist - within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fsType is filesystem type to mount. Must be a - filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management - (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object startTimestamp: description: Date/time when the backup started being processed. format: date-time diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 816175982..808c9ae69 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1383,9 +1383,7 @@ var _ = Describe("Cluster Controller", func() { By("mocking backup status completed, we don't need backup reconcile here") Expect(testapps.ChangeObjStatus(&testCtx, backup, func() { backup.Status.BackupToolName = backupTool.Name - backup.Status.RemoteVolume = &corev1.Volume{ - Name: "backup-pvc", - } + backup.Status.PersistentVolumeClaimName = "backup-pvc" backup.Status.Phase = dataprotectionv1alpha1.BackupCompleted })).ShouldNot(HaveOccurred()) By("checking backup status completed") diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 217f7920e..1fff54aa3 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -31,6 +31,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -179,12 +180,15 @@ func (r *BackupReconciler) doNewPhaseAction( default: commonPolicy := backupPolicy.Spec.GetCommonPolicy(backup.Spec.BackupType) if commonPolicy == nil { - return r.updateStatusIfFailed(reqCtx, backup, fmt.Errorf("backup type %s not supported", backup.Spec.BackupType)) + return r.updateStatusIfFailed(reqCtx, backup, intctrlutil.NewNotFound(`backup type "%s" not supported in the backupPolicy "%s"`, backup.Spec.BackupType, backupPolicy.Name)) } // save the backup message for restore - backup.Status.RemoteVolume = &commonPolicy.RemoteVolume + backup.Status.PersistentVolumeClaimName = commonPolicy.PersistentVolumeClaim.Name backup.Status.BackupToolName = commonPolicy.BackupToolName targetCluster = commonPolicy.Target + if err = r.handlePersistentVolumeClaim(reqCtx, backupPolicy.Name, commonPolicy); err != nil { + return r.updateStatusIfFailed(reqCtx, backup, err) + } } target, err := r.getTargetPod(reqCtx, backup, targetCluster.LabelsSelector.MatchLabels) @@ -212,6 +216,51 @@ func (r *BackupReconciler) doNewPhaseAction( return intctrlutil.Reconciled() } +// handlePersistentVolumeClaim handles the persistent volume claim for the backup, the rules are as follows +// - if CreatePolicy is "Never", it will check if the pvc exists. if not exist, will report an error. +// - if CreatePolicy is "IfNotPresent" and the pvc not exists, will create the pvc automatically. +func (r *BackupReconciler) handlePersistentVolumeClaim(reqCtx intctrlutil.RequestCtx, + backupPolicyName string, + commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) error { + pvc := &corev1.PersistentVolumeClaim{} + pvcConfig := commonPolicy.PersistentVolumeClaim + if err := r.Client.Get(reqCtx.Ctx, client.ObjectKey{Namespace: reqCtx.Req.Namespace, + Name: pvcConfig.Name}, pvc); err != nil && !apierrors.IsNotFound(err) { + return err + } + if len(pvc.Name) > 0 { + return nil + } + if pvcConfig.CreatePolicy == dataprotectionv1alpha1.CreatePVCPolicyNever { + return intctrlutil.NewNotFound(`persistent volume claim "%s" not found`, pvcConfig.Name) + } + pvc = &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcConfig.Name, + Namespace: reqCtx.Req.Namespace, + Annotations: map[string]string{ + dataProtectionAnnotationCreateByPolicyKey: "true", + dataProtectionLabelBackupPolicyKey: backupPolicyName, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: pvcConfig.StorageClassName, + Resources: corev1.ResourceRequirements{ + Requests: map[corev1.ResourceName]resource.Quantity{ + corev1.ResourceStorage: pvcConfig.InitCapacity, + }, + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + }, + } + // add a finalizer + controllerutil.AddFinalizer(pvc, dataProtectionFinalizerName) + err := r.Client.Create(reqCtx.Ctx, pvc) + return client.IgnoreAlreadyExists(err) +} + func (r *BackupReconciler) doInProgressPhaseAction( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) (ctrl.Result, error) { @@ -877,8 +926,15 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, remoteBackupPath := "/backupdata" // TODO(dsj): mount multi remote backup volumes - remoteVolumeName := fmt.Sprintf("backup-%s", commonPolicy.RemoteVolume.Name) - commonPolicy.RemoteVolume.Name = remoteVolumeName + remoteVolumeName := fmt.Sprintf("backup-%s", commonPolicy.PersistentVolumeClaim.Name) + remoteVolume := corev1.Volume{ + Name: remoteVolumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: commonPolicy.PersistentVolumeClaim.Name, + }, + }, + } remoteVolumeMount := corev1.VolumeMount{ Name: remoteVolumeName, MountPath: remoteBackupPath, @@ -944,7 +1000,7 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, podSpec.Containers = []corev1.Container{container} podSpec.Volumes = clusterPod.Spec.Volumes - podSpec.Volumes = append(podSpec.Volumes, commonPolicy.RemoteVolume) + podSpec.Volumes = append(podSpec.Volumes, remoteVolume) podSpec.RestartPolicy = corev1.RestartPolicyNever // the pod of job needs to be scheduled on the same node as the workload pod, because it needs to share one pvc diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index 13f72f1ac..b3dd47c9c 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -40,7 +40,6 @@ var _ = Describe("Backup Controller test", func() { const componentName = "replicasets-primary" const containerName = "mysql" const backupPolicyName = "test-backup-policy" - const backupRemoteVolumeName = "backup-remote-volume" const backupRemotePVCName = "backup-remote-pvc" const defaultSchedule = "0 3 * * *" const defaultTTL = "7d" @@ -142,7 +141,7 @@ var _ = Describe("Backup Controller test", func() { AddMatchLabels(constant.AppInstanceLabelKey, clusterName). AddMatchLabels(constant.RoleLabelKey, "leader"). SetTargetSecretName(clusterName). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). + SetPVC(backupRemotePVCName). Create(&testCtx).GetObject() }) @@ -350,7 +349,7 @@ var _ = Describe("Backup Controller test", func() { SetTTL(defaultTTL). AddMatchLabels(constant.AppInstanceLabelKey, clusterName). SetTargetSecretName(clusterName). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). + SetPVC(backupRemotePVCName). Create(&testCtx).GetObject() By("By creating a backup from backupPolicy: " + backupPolicyName) @@ -362,6 +361,13 @@ var _ = Describe("Backup Controller test", func() { }) It("should succeed after job completes", func() { + + By("Check pvc created by backup") + Eventually(testapps.CheckObjExists(&testCtx, types.NamespacedName{ + Name: backupRemotePVCName, + Namespace: testCtx.DefaultNamespace, + }, &corev1.PersistentVolumeClaim{}, true)).Should(Succeed()) + patchK8sJobStatus(backupKey, batchv1.JobComplete) By("Check backup job completed") diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index ea5b12f3f..b28159c50 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -28,6 +28,7 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -84,6 +85,8 @@ func (r *BackupPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } + originBackupPolicy := backupPolicy.DeepCopy() + // handle finalizer res, err := intctrlutil.HandleCRDeletion(reqCtx, r, backupPolicy, dataProtectionFinalizerName, func() (*ctrl.Result, error) { return nil, r.deleteExternalResources(reqCtx, backupPolicy) @@ -109,7 +112,7 @@ func (r *BackupPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request return r.patchStatusFailed(reqCtx, backupPolicy, "HandleIncrementalPolicyFailed", err) } - return r.patchStatusAvailable(reqCtx, backupPolicy) + return r.patchStatusAvailable(reqCtx, originBackupPolicy, backupPolicy) } // SetupWithManager sets up the controller with the Manager. @@ -153,7 +156,13 @@ func (r *BackupPolicyReconciler) deleteExternalResources(reqCtx intctrlutil.Requ // patchStatusAvailable patches backup policy status phase to available. func (r *BackupPolicyReconciler) patchStatusAvailable(reqCtx intctrlutil.RequestCtx, + originBackupPolicy, backupPolicy *dataprotectionv1alpha1.BackupPolicy) (ctrl.Result, error) { + if !reflect.DeepEqual(originBackupPolicy.Spec, backupPolicy.Spec) { + if err := r.Client.Update(reqCtx.Ctx, backupPolicy); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } + } // update status phase if backupPolicy.Status.Phase != dataprotectionv1alpha1.PolicyAvailable || backupPolicy.Status.ObservedGeneration != backupPolicy.Generation { @@ -401,6 +410,7 @@ func (r *BackupPolicyReconciler) handleFullPolicy( if schedule != nil && schedule.Enable && schedule.Type == dataprotectionv1alpha1.BaseBackupTypeFull { cronExpression = schedule.CronExpression } + r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Full) return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Full.BasePolicy, cronExpression, dataprotectionv1alpha1.BackupTypeFull) } @@ -418,6 +428,26 @@ func (r *BackupPolicyReconciler) handleIncrementalPolicy( if schedule != nil && schedule.Enable { cronExpression = schedule.CronExpression } + r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Incremental) return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Incremental.BasePolicy, cronExpression, dataprotectionv1alpha1.BackupTypeIncremental) } + +// setGlobalPersistentVolumeClaim sets global config of pvc to common policy. +func (r *BackupPolicyReconciler) setGlobalPersistentVolumeClaim(backupPolicy *dataprotectionv1alpha1.CommonBackupPolicy) { + pvcCfg := backupPolicy.PersistentVolumeClaim + globalPVCName := viper.GetString(constant.CfgKeyBackupPVCName) + if len(pvcCfg.Name) == 0 && globalPVCName != "" { + backupPolicy.PersistentVolumeClaim.Name = globalPVCName + } + + globalStorageClass := viper.GetString(constant.CfgKeyBackupPVCStorageClass) + if pvcCfg.StorageClassName == nil && globalStorageClass != "" { + backupPolicy.PersistentVolumeClaim.StorageClassName = &globalStorageClass + } + + globalInitCapacity := viper.GetString(constant.CfgKeyBackupPVCInitCapacity) + if pvcCfg.InitCapacity.IsZero() && globalInitCapacity != "" { + backupPolicy.PersistentVolumeClaim.InitCapacity = resource.MustParse(globalInitCapacity) + } +} diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index fc5a38187..88448228c 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -42,8 +42,6 @@ var _ = Describe("Backup Policy Controller", func() { const containerName = "mysql" const defaultPVCSize = "1Gi" const backupPolicyName = "test-backup-policy" - // const backupPolicyTplName = "test-backup-policy-template" - const backupRemoteVolumeName = "backup-remote-volume" const backupRemotePVCName = "backup-remote-pvc" const defaultSchedule = "0 3 * * *" const defaultTTL = "7d" @@ -141,7 +139,7 @@ var _ = Describe("Backup Policy Controller", func() { SetTargetSecretName(clusterName). AddHookPreCommand("touch /data/mysql/.restore;sync"). AddHookPostCommand("rm -f /data/mysql/.restore;sync"). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). + SetPVC(backupRemotePVCName). Create(&testCtx).GetObject() backupPolicyKey = client.ObjectKeyFromObject(backupPolicy) }) @@ -245,7 +243,7 @@ var _ = Describe("Backup Policy Controller", func() { SetTargetSecretName(clusterName). AddHookPreCommand("touch /data/mysql/.restore;sync"). AddHookPostCommand("rm -f /data/mysql/.restore;sync"). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). + SetPVC(backupRemotePVCName). Create(&testCtx).GetObject() backupPolicyKey = client.ObjectKeyFromObject(backupPolicy) }) @@ -269,7 +267,7 @@ var _ = Describe("Backup Policy Controller", func() { SetTargetSecretName(clusterName). AddHookPreCommand("touch /data/mysql/.restore;sync"). AddHookPostCommand("rm -f /data/mysql/.restore;sync"). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). + SetPVC(backupRemotePVCName). Create(&testCtx).GetObject() backupPolicyKey = client.ObjectKeyFromObject(backupPolicy) }) @@ -291,7 +289,8 @@ var _ = Describe("Backup Policy Controller", func() { SetTargetSecretName(randomSecretName). AddHookPreCommand("touch /data/mysql/.restore;sync"). AddHookPostCommand("rm -f /data/mysql/.restore;sync"). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName).Create(&testCtx).GetObject() + SetPVC(backupRemotePVCName). + Create(&testCtx).GetObject() backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) @@ -299,6 +298,32 @@ var _ = Describe("Backup Policy Controller", func() { })).Should(Succeed()) }) }) + + Context("creating a backupPolicy with global backup config", func() { + It("ccreating a backupPolicy with global backup config", func() { + By("By creating a backupPolicy with empty secret") + pvcName := "backup-data" + pvcInitCapacity := "10Gi" + pvcStorageClass := "standard" + viper.SetDefault(constant.CfgKeyBackupPVCName, pvcName) + viper.SetDefault(constant.CfgKeyBackupPVCInitCapacity, pvcInitCapacity) + viper.SetDefault(constant.CfgKeyBackupPVCStorageClass, pvcStorageClass) + backupPolicy := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). + AddFullPolicy(). + SetBackupToolName(backupToolName). + AddMatchLabels(constant.AppInstanceLabelKey, clusterName). + AddHookPreCommand("touch /data/mysql/.restore;sync"). + AddHookPostCommand("rm -f /data/mysql/.restore;sync"). + Create(&testCtx).GetObject() + backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) + Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { + g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) + g.Expect(fetched.Spec.Full.PersistentVolumeClaim.Name).To(Equal(pvcName)) + g.Expect(*fetched.Spec.Full.PersistentVolumeClaim.StorageClassName).To(Equal(pvcStorageClass)) + g.Expect(fetched.Spec.Full.PersistentVolumeClaim.InitCapacity.String()).To(Equal(pvcInitCapacity)) + })).Should(Succeed()) + }) + }) }) }) diff --git a/controllers/dataprotection/restorejob_controller.go b/controllers/dataprotection/restorejob_controller.go index e0d164b4d..8ed9d24b7 100644 --- a/controllers/dataprotection/restorejob_controller.go +++ b/controllers/dataprotection/restorejob_controller.go @@ -18,6 +18,7 @@ package dataprotection import ( "context" + "fmt" "github.com/spf13/viper" appv1 "k8s.io/api/apps/v1" @@ -250,7 +251,7 @@ func (r *RestoreJobReconciler) buildPodSpec(reqCtx intctrlutil.RequestCtx, resto return podSpec, err } - if backup.Status.RemoteVolume == nil { + if len(backup.Status.PersistentVolumeClaimName) == 0 { return podSpec, nil } @@ -265,9 +266,19 @@ func (r *RestoreJobReconciler) buildPodSpec(reqCtx intctrlutil.RequestCtx, resto container.VolumeMounts = restoreJob.Spec.TargetVolumeMounts + // add the volumeMounts with backup volume + restoreVolumeName := fmt.Sprintf("restore-%s", backup.Status.PersistentVolumeClaimName) + remoteVolume := corev1.Volume{ + Name: restoreVolumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: backup.Status.PersistentVolumeClaimName, + }, + }, + } // add remote volumeMounts remoteVolumeMount := corev1.VolumeMount{} - remoteVolumeMount.Name = backup.Status.RemoteVolume.Name + remoteVolumeMount.Name = restoreVolumeName remoteVolumeMount.MountPath = "/data" container.VolumeMounts = append(container.VolumeMounts, remoteVolumeMount) @@ -292,7 +303,7 @@ func (r *RestoreJobReconciler) buildPodSpec(reqCtx intctrlutil.RequestCtx, resto podSpec.Volumes = restoreJob.Spec.TargetVolumes // add remote volumes - podSpec.Volumes = append(podSpec.Volumes, *backup.Status.RemoteVolume) + podSpec.Volumes = append(podSpec.Volumes, remoteVolume) // TODO(dsj): mount readonly remote volumes for restore. // podSpec.Volumes[0].PersistentVolumeClaim.ReadOnly = true diff --git a/controllers/dataprotection/restorejob_controller_test.go b/controllers/dataprotection/restorejob_controller_test.go index 6fcdba409..b25b7a7a6 100644 --- a/controllers/dataprotection/restorejob_controller_test.go +++ b/controllers/dataprotection/restorejob_controller_test.go @@ -93,7 +93,7 @@ var _ = Describe("RestoreJob Controller", func() { SetTTL("7d"). SetBackupToolName(backupTool). SetTargetSecretName("mycluster-cluster-secret"). - SetRemoteVolumePVC("backup-remote-volume", "backup-host-path-pvc"). + SetPVC("backup-host-path-pvc"). Create(&testCtx).GetObject() } diff --git a/controllers/dataprotection/type.go b/controllers/dataprotection/type.go index 795aedd7f..a4bec458d 100644 --- a/controllers/dataprotection/type.go +++ b/controllers/dataprotection/type.go @@ -38,7 +38,8 @@ const ( dataProtectionLabelBackupNameKey = "backups.dataprotection.kubeblocks.io/name" dataProtectionLabelRestoreJobNameKey = "restorejobs.dataprotection.kubeblocks.io/name" - dataProtectionBackupTargetPodKey = "dataprotection.kubeblocks.io/target-pod-name" + dataProtectionBackupTargetPodKey = "dataprotection.kubeblocks.io/target-pod-name" + dataProtectionAnnotationCreateByPolicyKey = "dataprotection.kubeblocks.io/created-by-policy" // error status errorJobFailed = "JobFailed" ) diff --git a/deploy/apecloud-mysql/templates/backuptool.yaml b/deploy/apecloud-mysql/templates/backuptool.yaml index cc84fe2f2..eb42c15ac 100644 --- a/deploy/apecloud-mysql/templates/backuptool.yaml +++ b/deploy/apecloud-mysql/templates/backuptool.yaml @@ -13,7 +13,7 @@ spec: cpu: "1" memory: 2Gi requests: - cpu: "1" + cpu: "500m" memory: 128Mi env: - name: DATA_DIR @@ -42,5 +42,8 @@ spec: restoreCommands: [] incrementalRestoreCommands: [] backupCommands: - - xtrabackup --compress --backup --safe-slave-backup --slave-info --stream=xbstream --host=${DB_HOST} --user=${DB_USER} --password=${DB_PASSWORD} --datadir=${DATA_DIR} > /${BACKUP_DIR}/${BACKUP_NAME}.xbstream + - | + set -e + mkdir -p ${BACKUP_DIR} + xtrabackup --compress --backup --safe-slave-backup --slave-info --stream=xbstream --host=${DB_HOST} --user=${DB_USER} --password=${DB_PASSWORD} --datadir=${DATA_DIR} > ${BACKUP_DIR}/${BACKUP_NAME}.xbstream incrementalBackupCommands: [] diff --git a/deploy/csi-s3/templates/_helpers.tpl b/deploy/csi-s3/templates/_helpers.tpl new file mode 100644 index 000000000..e84ca23ce --- /dev/null +++ b/deploy/csi-s3/templates/_helpers.tpl @@ -0,0 +1,10 @@ +{{/* +Expand the mountOptions of the storageClass. +*/}} +{{- define "storageClass.mountOptions" -}} +{{- if .Values.secret.region }} +{{- printf "%s --region %s" .Values.storageClass.mountOptions .Values.secret.region }} +{{- else }} +{{- .Values.storageClass.mountOptions }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/deploy/csi-s3/templates/attacher.yaml b/deploy/csi-s3/templates/attacher.yaml index b849393e0..8cf0d05fa 100644 --- a/deploy/csi-s3/templates/attacher.yaml +++ b/deploy/csi-s3/templates/attacher.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: csi-attacher-sa + name: csi-attacher-sa-{{ .Values.storageClass.name }} namespace: {{ .Release.Namespace }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: external-attacher-runner + name: external-attacher-runner-{{ .Values.storageClass.name }} rules: - apiGroups: [""] resources: ["secrets"] @@ -34,14 +34,14 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: csi-attacher-role + name: csi-attacher-role-{{ .Values.storageClass.name }} subjects: - kind: ServiceAccount - name: csi-attacher-sa + name: csi-attacher-sa-{{ .Values.storageClass.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: external-attacher-runner + name: external-attacher-runner-{{ .Values.storageClass.name }} apiGroup: rbac.authorization.k8s.io --- # needed for StatefulSet @@ -75,7 +75,7 @@ spec: labels: app: csi-attacher-s3 spec: - serviceAccount: csi-attacher-sa + serviceAccount: csi-attacher-sa-{{ .Values.storageClass.name }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} diff --git a/deploy/csi-s3/templates/provisioner.yaml b/deploy/csi-s3/templates/provisioner.yaml index db0b9b585..8d5d8ba9e 100644 --- a/deploy/csi-s3/templates/provisioner.yaml +++ b/deploy/csi-s3/templates/provisioner.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: csi-provisioner-sa + name: csi-provisioner-sa-{{ .Values.storageClass.name }} namespace: {{ .Release.Namespace }} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: external-provisioner-runner + name: external-provisioner-runner-{{ .Values.storageClass.name }} rules: - apiGroups: [""] resources: ["secrets"] @@ -28,14 +28,14 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: csi-provisioner-role + name: csi-provisioner-role-{{ .Values.storageClass.name }} subjects: - kind: ServiceAccount - name: csi-provisioner-sa + name: csi-provisioner-sa-{{ .Values.storageClass.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: external-provisioner-runner + name: external-provisioner-runner-{{ .Values.storageClass.name }} apiGroup: rbac.authorization.k8s.io --- kind: Service @@ -68,7 +68,7 @@ spec: labels: app: csi-provisioner-s3 spec: - serviceAccount: csi-provisioner-sa + serviceAccount: csi-provisioner-sa-{{ .Values.storageClass.name }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} diff --git a/deploy/csi-s3/templates/storageclass.yaml b/deploy/csi-s3/templates/storageclass.yaml index e40d69939..8c0d6fbb3 100644 --- a/deploy/csi-s3/templates/storageclass.yaml +++ b/deploy/csi-s3/templates/storageclass.yaml @@ -10,7 +10,7 @@ metadata: provisioner: ru.yandex.s3.csi parameters: mounter: "{{ .Values.storageClass.mounter }}" - options: "{{ .Values.storageClass.mountOptions }}" + options: "{{ include "storageClass.mountOptions" . }}" {{- if .Values.storageClass.singleBucket }} bucket: "{{ .Values.storageClass.singleBucket }}" {{- end }} diff --git a/deploy/csi-s3/values.yaml b/deploy/csi-s3/values.yaml index 91e589777..4989b4489 100644 --- a/deploy/csi-s3/values.yaml +++ b/deploy/csi-s3/values.yaml @@ -21,7 +21,7 @@ storageClass: # GeeseFS mount options mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" # Volume reclaim policy - reclaimPolicy: Delete + reclaimPolicy: Retain # Annotations for the storage class # Example: # annotations: @@ -38,7 +38,10 @@ secret: # S3 Secret Key secretKey: "" # Endpoint + # For AWS set it to "https://s3..amazonaws.com", for example https://s3.eu-central-1.amazonaws.com + # In China set it to "https://s3..amazonaws.com.cn", for example https://s3.cn-north-1.amazonaws.com.cn endpoint: https://storage.yandexcloud.net + region: "" tolerations: - key: kb-controller @@ -58,3 +61,8 @@ affinity: operator: In values: - "true" + - key: kubernetes.io/arch + operator: In + values: + - "amd64" + diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml index 870a98b38..9e4b7bd9a 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -97,1585 +97,37 @@ spec: description: count of backup stop retries on fail. format: int32 type: integer - remoteVolume: - description: array of remote volumes from CSI driver definition. + persistentVolumeClaim: + description: refer to PersistentVolumeClaim and the backup data + will be stored in the corresponding persistent volume. properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk - resource that is attached to a kubelet''s host machine and - then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume - that you want to mount. If omitted, the default is to - mount by volume name. Examples: For volume /dev/sda1, - you specify the partition as "1". Similarly, the volume - partition for /dev/sda is "0" (or you can leave the - property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent - disk resource in AWS (Amazon EBS volume). More info: - https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount - on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in - the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob - storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple - blob disks per storage account Dedicated: single blob - disk per storage account Managed: azure managed data - disk (only in managed availability set). defaults to - shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host - that shares a pod's lifetime - properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the - path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - user: - description: 'user is optional: User is the rados user - name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and - mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating - system. Examples: "ext4", "xfs", "ntfs". Implicitly - inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret - object containing parameters used to connect to OpenStack.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - volumeID: - description: 'volumeID used to identify the volume in - cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should - populate this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used - to set permissions on created files by default. Must - be an octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within the path - are not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair - in the Data field of the referenced ConfigMap will be - projected into the volume as a file whose name is the - key and content is the value. If specified, the listed - keys will be projected into the specified paths, and - unlisted keys will not be present. If a key is specified - which is not present in the ConfigMap, the volume setup - will error unless it is marked optional. Paths must - be relative and may not contain the '..' path or start - with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for - mode bits. If not specified, the volume defaultMode - will be used. This might be in conflict with other - options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or - its keys must be defined - type: boolean - type: object - csi: - description: csi (Container Storage Interface) represents - ephemeral storage that is handled by certain external CSI - drivers (Beta feature). - properties: - driver: - description: driver is the name of the CSI driver that - handles this volume. Consult with your admin for the - correct name as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem - to apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the - secret object containing sensitive information to pass - to the CSI driver to complete the CSI NodePublishVolume - and NodeUnpublishVolume calls. This field is optional, - and may be empty if no secret is required. If the secret - object contains more than one secret, all secret references - are passed. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the - pod that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set - permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode bits. - Defaults to 0644. Directories within the path are not - affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are - supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in - the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. If not - specified, the volume defaultMode will be used. - This might be in conflict with other options that - affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path - name of the file to be created. Must not be absolute - or contain the ''..'' path. Must be utf-8 encoded. - The first item of the relative path must not start - with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of - the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that - shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which - means to use the node''s default medium. Must be an - empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is - also applicable for memory medium. The maximum usage - on memory medium EmptyDir would be the minimum value - between the SizeLimit specified here and the sum of - memory limits of all containers in a pod. The default - is nil which means that the limit is undefined. More - info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled - by a cluster storage driver. The volume's lifecycle is tied - to the pod that defines it - it will be created before the - pod starts, and deleted when the pod is removed. \n Use - this if: a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot - or capacity tracking are needed, c) the storage driver is - specified through a storage class, and d) the storage driver - supports dynamic volume provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information on the connection - between this volume type and PersistentVolumeClaim). \n - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. \n Use CSI for light-weight local - ephemeral volumes if the CSI driver is meant to be used - that way - see the documentation of the driver for more - information. \n A pod can use both types of ephemeral volumes - and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC - to provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC - will be deleted together with the pod. The name of - the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated - name is not valid for a PVC (for example, too long). - \n An existing PVC with that name that is not owned - by the pod will *not* be used for the pod to avoid using - an unrelated volume by mistake. Starting the pod is - then blocked until the unrelated PVC is removed. If - such a pre-created PVC is meant to be used by the pod, - the PVC has to updated with an owner reference to the - pod once the pod exists. Normally this should not be - necessary, but it may be useful when manually reconstructing - a broken cluster. \n This field is read-only and no - changes will be made by Kubernetes to the PVC after - it has been created. \n Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations that - will be copied into the PVC when creating it. No - other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the - PVC that gets created from this template. The same - fields as in a PersistentVolumeClaim are also valid - here. - properties: - accessModes: - description: 'accessModes contains the desired - access modes the volume should have. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to - specify either: * An existing VolumeSnapshot - object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If - the provisioner or an external controller can - support the specified data source, it will create - a new volume based on the contents of the specified - data source. When the AnyVolumeDataSource feature - gate is enabled, dataSource contents will be - copied to dataSourceRef, and dataSourceRef contents - will be copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the - resource being referenced. If APIGroup is - not specified, the specified Kind must be - in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - required: - - kind - - name - type: object - dataSourceRef: - description: 'dataSourceRef specifies the object - from which to populate the volume with data, - if a non-empty volume is desired. This may be - any object from a non-empty API group (non core - object) or a PersistentVolumeClaim object. When - this field is specified, volume binding will - only succeed if the type of the specified object - matches some installed volume populator or dynamic - provisioner. This field will replace the functionality - of the dataSource field and as such if both - fields are non-empty, they must have the same - value. For backwards compatibility, when namespace - isn''t specified in dataSourceRef, both fields - (dataSource and dataSourceRef) will be set to - the same value automatically if one of them - is empty and the other is non-empty. When namespace - is specified in dataSourceRef, dataSource isn''t - set to the same value and must be empty. There - are three important differences between dataSource - and dataSourceRef: * While dataSource only allows - two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim - objects. * While dataSource ignores disallowed - values (dropping them), dataSourceRef preserves - all values, and generates an error if a disallowed - value is specified. * While dataSource only - allows local objects, dataSourceRef allows objects - in any namespaces. (Beta) Using this field requires - the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef - requires the CrossNamespaceVolumeDataSource - feature gate to be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the - resource being referenced. If APIGroup is - not specified, the specified Kind must be - in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - namespace: - description: Namespace is the namespace of - resource being referenced Note that when - a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept - the reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires - the CrossNamespaceVolumeDataSource feature - gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the minimum - resources the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity - recorded in the status field of the claim. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are - used by this container. \n This is an alpha - field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name - of one entry in pod.spec.resourceClaims - of the Pod where this field is used. - It makes that resource available inside - a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum - amount of compute resources allowed. More - info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If - Requests is omitted for a container, it - defaults to Limits if that is explicitly - specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, a - key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - storageClassName: - description: 'storageClassName is the name of - the StorageClass required by the claim. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem - is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference - to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is - attached to a kubelet's host machine and then exposed to - the pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. TODO: how do we prevent - errors in the filesystem from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and - lun must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource - that is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for - this volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". The default filesystem - depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: 'readOnly is Optional: defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information - to pass to the plugin scripts. This may be empty if - no secret object is specified. If the secret object - contains more than one secret, all secrets are passed - to the plugin scripts.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached - to a kubelet's host machine. This depends on the Flocker - control service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored - as metadata -> name on the dataset for Flocker should - be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume - that you want to mount. If omitted, the default is to - mount by volume name. Examples: For volume /dev/sda1, - you specify the partition as "1". Similarly, the volume - partition for /dev/sda is "0" (or you can leave the - property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource - in GCE. Used to identify the disk in GCE. More info: - https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision - a container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir - into the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, - the volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More - info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to - false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' - properties: - path: - description: 'path of the directory on the host. If the - path is a symlink, it will follow the link to the real - path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that - is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support - iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator - Name. If initiatorName is specified with iscsiInterface - simultaneously, new iSCSI interface : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that - uses an iSCSI transport. Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. - The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - targetPortal: - description: targetPortal is iSCSI Target Portal. The - Portal is either an IP or ip_addr:port if the port is - other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object + createPolicy: + default: IfNotPresent + description: 'createPolicy defines the policy for creating + the PersistentVolumeClaim, enum values: - Never: do nothing + if the PersistentVolumeClaim not exist. - IfNotPresent: + create the PersistentVolumeClaim if not present and the + accessModes only contains ''ReadWriteMany''.' + enum: + - IfNotPresent + - Never + type: string + initCapacity: + anyOf: + - type: integer + - type: string + description: initCapacity represents the init storage size + of the PersistentVolumeClaim which should be created if + not exist. and the default value is 100Gi if it is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true name: - description: 'name of the volume. Must be a DNS_LABEL and - unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: the name of the PersistentVolumeClaim. + type: string + storageClassName: + description: storageClassName is the name of the StorageClass + required by the claim. type: string - nfs: - description: 'nfs represents an NFS mount on the host that - shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export - to be mounted with read-only permissions. Defaults to - false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of - the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents - a reference to a PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting - in VolumeMounts. Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to - mount Must be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set - permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this - setting. This might be in conflict with other options - that affect the file mode, like fsGroup, and the result - can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along - with other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the ConfigMap, the - volume setup will error unless it is marked - optional. Paths must be relative and may not - contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used - to set permissions on this file, must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. - YAML accepts both octal and decimal - values, JSON requires decimal values - for mode bits. If not specified, the - volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. - Must not be absolute or contain the - ''..'' path. Must be utf-8 encoded. - The first item of the relative path - must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and - requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are - currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the Secret, the volume - setup will error unless it is marked optional. - Paths must be relative and may not contain - the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional field specify whether - the Secret or its key must be defined - type: boolean - type: object - serviceAccountToken: - description: serviceAccountToken is information - about the serviceAccountToken data to project - properties: - audience: - description: audience is the intended audience - of the token. A recipient of a token must - identify itself with an identifier specified - in the audience of the token, and otherwise - should reject the token. The audience defaults - to the identifier of the apiserver. - type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account - token. As the token approaches expiration, - the kubelet volume plugin will proactively - rotate the service account token. The kubelet - will start trying to rotate the token if the - token is older than 80 percent of its time - to live or if the token is older than 24 hours.Defaults - to 1 hour and must be at least 10 minutes. - format: int64 - type: integer - path: - description: path is the path relative to the - mount point of the file to project the token - into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host - that shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is - no group - type: string - readOnly: - description: readOnly here will force the Quobyte volume - to be mounted with read-only permissions. Defaults to - false. - type: boolean - registry: - description: registry represents a single or multiple - Quobyte Registry services specified as a string as host:port - pair (multiple entries are separated with commas) which - acts as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in - the Backend Used with dynamically provisioned Quobyte - volumes, value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to - serivceaccount user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on - the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - image: - description: 'image is the rados image name. More info: - https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is - rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication - secret for RBDUser. If provided overrides keyring. Default - is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - user: - description: 'user is the rados user name. Default is - admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume - attached and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO - API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO - Protection Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not - provided, Login operation will fail. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage - for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system - as configured in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already - created in the ScaleIO system that is associated with - this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used - to set permissions on created files by default. Must - be an octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within the path - are not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair - in the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and - content is the value. If specified, the listed keys - will be projected into the specified paths, and unlisted - keys will not be present. If a key is specified which - is not present in the Secret, the volume setup will - error unless it is marked optional. Paths must be relative - and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for - mode bits. If not specified, the volume defaultMode - will be used. This might be in conflict with other - options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret - or its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in - the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for - obtaining the StorageOS API credentials. If not specified, - default values will be attempted. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - volumeName: - description: volumeName is the human-readable name of - the StorageOS volume. Volume names are only unique - within a namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the - volume within StorageOS. If no namespace is specified - then the Pod's namespace will be used. This allows - the Kubernetes name scoping to be mirrored within StorageOS - for tighter integration. Set VolumeName to any name - to override the default behaviour. Set to "default" - if you are not using namespaces within StorageOS. Namespaces - that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fsType is filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based - Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object required: - name type: object @@ -1757,7 +209,7 @@ spec: - labelsSelector type: object required: - - remoteVolume + - persistentVolumeClaim - target type: object incremental: @@ -1807,1585 +259,37 @@ spec: description: count of backup stop retries on fail. format: int32 type: integer - remoteVolume: - description: array of remote volumes from CSI driver definition. + persistentVolumeClaim: + description: refer to PersistentVolumeClaim and the backup data + will be stored in the corresponding persistent volume. properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk - resource that is attached to a kubelet''s host machine and - then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume - that you want to mount. If omitted, the default is to - mount by volume name. Examples: For volume /dev/sda1, - you specify the partition as "1". Similarly, the volume - partition for /dev/sda is "0" (or you can leave the - property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent - disk resource in AWS (Amazon EBS volume). More info: - https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount - on the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in - the blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob - storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple - blob disks per storage account Dedicated: single blob - disk per storage account Managed: azure managed data - disk (only in managed availability set). defaults to - shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host - that shares a pod's lifetime - properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the - path to key ring for User, default is /etc/ceph/user.secret - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - user: - description: 'user is optional: User is the rados user - name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and - mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating - system. Examples: "ext4", "xfs", "ntfs". Implicitly - inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret - object containing parameters used to connect to OpenStack.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - volumeID: - description: 'volumeID used to identify the volume in - cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should - populate this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used - to set permissions on created files by default. Must - be an octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within the path - are not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair - in the Data field of the referenced ConfigMap will be - projected into the volume as a file whose name is the - key and content is the value. If specified, the listed - keys will be projected into the specified paths, and - unlisted keys will not be present. If a key is specified - which is not present in the ConfigMap, the volume setup - will error unless it is marked optional. Paths must - be relative and may not contain the '..' path or start - with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for - mode bits. If not specified, the volume defaultMode - will be used. This might be in conflict with other - options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or - its keys must be defined - type: boolean - type: object - csi: - description: csi (Container Storage Interface) represents - ephemeral storage that is handled by certain external CSI - drivers (Beta feature). - properties: - driver: - description: driver is the name of the CSI driver that - handles this volume. Consult with your admin for the - correct name as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". - If not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem - to apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the - secret object containing sensitive information to pass - to the CSI driver to complete the CSI NodePublishVolume - and NodeUnpublishVolume calls. This field is optional, - and may be empty if no secret is required. If the secret - object contains more than one secret, all secret references - are passed. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the - pod that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set - permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode bits. - Defaults to 0644. Directories within the path are not - affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are - supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in - the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. If not - specified, the volume defaultMode will be used. - This might be in conflict with other options that - affect the file mode, like fsGroup, and the result - can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path - name of the file to be created. Must not be absolute - or contain the ''..'' path. Must be utf-8 encoded. - The first item of the relative path must not start - with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of - the exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that - shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which - means to use the node''s default medium. Must be an - empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is - also applicable for memory medium. The maximum usage - on memory medium EmptyDir would be the minimum value - between the SizeLimit specified here and the sum of - memory limits of all containers in a pod. The default - is nil which means that the limit is undefined. More - info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled - by a cluster storage driver. The volume's lifecycle is tied - to the pod that defines it - it will be created before the - pod starts, and deleted when the pod is removed. \n Use - this if: a) the volume is only needed while the pod runs, - b) features of normal volumes like restoring from snapshot - or capacity tracking are needed, c) the storage driver is - specified through a storage class, and d) the storage driver - supports dynamic volume provisioning through a PersistentVolumeClaim - (see EphemeralVolumeSource for more information on the connection - between this volume type and PersistentVolumeClaim). \n - Use PersistentVolumeClaim or one of the vendor-specific - APIs for volumes that persist for longer than the lifecycle - of an individual pod. \n Use CSI for light-weight local - ephemeral volumes if the CSI driver is meant to be used - that way - see the documentation of the driver for more - information. \n A pod can use both types of ephemeral volumes - and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC - to provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC - will be deleted together with the pod. The name of - the PVC will be `-` where `` is the name from the `PodSpec.Volumes` array - entry. Pod validation will reject the pod if the concatenated - name is not valid for a PVC (for example, too long). - \n An existing PVC with that name that is not owned - by the pod will *not* be used for the pod to avoid using - an unrelated volume by mistake. Starting the pod is - then blocked until the unrelated PVC is removed. If - such a pre-created PVC is meant to be used by the pod, - the PVC has to updated with an owner reference to the - pod once the pod exists. Normally this should not be - necessary, but it may be useful when manually reconstructing - a broken cluster. \n This field is read-only and no - changes will be made by Kubernetes to the PVC after - it has been created. \n Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations that - will be copied into the PVC when creating it. No - other fields are allowed and will be rejected during - validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the - PVC that gets created from this template. The same - fields as in a PersistentVolumeClaim are also valid - here. - properties: - accessModes: - description: 'accessModes contains the desired - access modes the volume should have. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to - specify either: * An existing VolumeSnapshot - object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If - the provisioner or an external controller can - support the specified data source, it will create - a new volume based on the contents of the specified - data source. When the AnyVolumeDataSource feature - gate is enabled, dataSource contents will be - copied to dataSourceRef, and dataSourceRef contents - will be copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the - resource being referenced. If APIGroup is - not specified, the specified Kind must be - in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - required: - - kind - - name - type: object - dataSourceRef: - description: 'dataSourceRef specifies the object - from which to populate the volume with data, - if a non-empty volume is desired. This may be - any object from a non-empty API group (non core - object) or a PersistentVolumeClaim object. When - this field is specified, volume binding will - only succeed if the type of the specified object - matches some installed volume populator or dynamic - provisioner. This field will replace the functionality - of the dataSource field and as such if both - fields are non-empty, they must have the same - value. For backwards compatibility, when namespace - isn''t specified in dataSourceRef, both fields - (dataSource and dataSourceRef) will be set to - the same value automatically if one of them - is empty and the other is non-empty. When namespace - is specified in dataSourceRef, dataSource isn''t - set to the same value and must be empty. There - are three important differences between dataSource - and dataSourceRef: * While dataSource only allows - two specific types of objects, dataSourceRef - allows any non-core object, as well as PersistentVolumeClaim - objects. * While dataSource ignores disallowed - values (dropping them), dataSourceRef preserves - all values, and generates an error if a disallowed - value is specified. * While dataSource only - allows local objects, dataSourceRef allows objects - in any namespaces. (Beta) Using this field requires - the AnyVolumeDataSource feature gate to be enabled. - (Alpha) Using the namespace field of dataSourceRef - requires the CrossNamespaceVolumeDataSource - feature gate to be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the - resource being referenced. If APIGroup is - not specified, the specified Kind must be - in the core API group. For any other third-party - types, APIGroup is required. - type: string - kind: - description: Kind is the type of resource - being referenced - type: string - name: - description: Name is the name of resource - being referenced - type: string - namespace: - description: Namespace is the namespace of - resource being referenced Note that when - a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept - the reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires - the CrossNamespaceVolumeDataSource feature - gate to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the minimum - resources the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity - recorded in the status field of the claim. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are - used by this container. \n This is an alpha - field and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one - entry in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name - of one entry in pod.spec.resourceClaims - of the Pod where this field is used. - It makes that resource available inside - a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum - amount of compute resources allowed. More - info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum - amount of compute resources required. If - Requests is omitted for a container, it - defaults to Limits if that is explicitly - specified, otherwise to an implementation-defined - value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of - label selector requirements. The requirements - are ANDed. - items: - description: A label selector requirement - is a selector that contains values, a - key, and an operator that relates the - key and values. - properties: - key: - description: key is the label key that - the selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and - DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. - If the operator is Exists or DoesNotExist, - the values array must be empty. This - array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is - "In", and the values array contains only - "value". The requirements are ANDed. - type: object - type: object - storageClassName: - description: 'storageClassName is the name of - the StorageClass required by the claim. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem - is implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference - to the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is - attached to a kubelet's host machine and then exposed to - the pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. TODO: how do we prevent - errors in the filesystem from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and - lun must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource - that is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for - this volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". The default filesystem - depends on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: 'readOnly is Optional: defaults to false - (read/write). ReadOnly here will force the ReadOnly - setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information - to pass to the plugin scripts. This may be empty if - no secret object is specified. If the secret object - contains more than one secret, all secrets are passed - to the plugin scripts.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached - to a kubelet's host machine. This depends on the Flocker - control service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored - as metadata -> name on the dataset for Flocker should - be considered as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - partition: - description: 'partition is the partition in the volume - that you want to mount. If omitted, the default is to - mount by volume name. Examples: For volume /dev/sda1, - you specify the partition as "1". Similarly, the volume - partition for /dev/sda is "0" (or you can leave the - property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource - in GCE. Used to identify the disk in GCE. More info: - https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision - a container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir - into the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, - the volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More - info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to - false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' - properties: - path: - description: 'path of the directory on the host. If the - path is a symlink, it will follow the link to the real - path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that - is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support - iSCSI Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator - Name. If initiatorName is specified with iscsiInterface - simultaneously, new iSCSI interface : will be created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that - uses an iSCSI transport. Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. - The portal is either an IP or ip_addr:port if the port - is other than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - targetPortal: - description: targetPortal is iSCSI Target Portal. The - Portal is either an IP or ip_addr:port if the port is - other than default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object + createPolicy: + default: IfNotPresent + description: 'createPolicy defines the policy for creating + the PersistentVolumeClaim, enum values: - Never: do nothing + if the PersistentVolumeClaim not exist. - IfNotPresent: + create the PersistentVolumeClaim if not present and the + accessModes only contains ''ReadWriteMany''.' + enum: + - IfNotPresent + - Never + type: string + initCapacity: + anyOf: + - type: integer + - type: string + description: initCapacity represents the init storage size + of the PersistentVolumeClaim which should be created if + not exist. and the default value is 100Gi if it is empty. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true name: - description: 'name of the volume. Must be a DNS_LABEL and - unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: the name of the PersistentVolumeClaim. + type: string + storageClassName: + description: storageClassName is the name of the StorageClass + required by the claim. type: string - nfs: - description: 'nfs represents an NFS mount on the host that - shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. - More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export - to be mounted with read-only permissions. Defaults to - false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of - the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents - a reference to a PersistentVolumeClaim in the same namespace. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. - More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting - in VolumeMounts. Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to - mount Must be a filesystem type supported by the host - operating system. Ex. "ext4", "xfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set - permissions on created files by default. Must be an - octal value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and decimal - values, JSON requires decimal values for mode bits. - Directories within the path are not affected by this - setting. This might be in conflict with other options - that affect the file mode, like fsGroup, and the result - can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along - with other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the ConfigMap, the - volume setup will error unless it is marked - optional. Paths must be relative and may not - contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing - the pod field - properties: - fieldRef: - description: 'Required: Selects a field - of the pod: only annotations, labels, - name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema - the FieldPath is written in terms - of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to - select in the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used - to set permissions on this file, must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. - YAML accepts both octal and decimal - values, JSON requires decimal values - for mode bits. If not specified, the - volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. - Must not be absolute or contain the - ''..'' path. Must be utf-8 encoded. - The first item of the relative path - must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the - container: only resources limits and - requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are - currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output - format of the exposed resources, - defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to - select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file - whose name is the key and content is the value. - If specified, the listed keys will be projected - into the specified paths, and unlisted keys - will not be present. If a key is specified - which is not present in the Secret, the volume - setup will error unless it is marked optional. - Paths must be relative and may not contain - the '..' path or start with '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. - Must be an octal value between 0000 - and 0777 or a decimal value between - 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal - values for mode bits. If not specified, - the volume defaultMode will be used. - This might be in conflict with other - options that affect the file mode, like - fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path - of the file to map the key to. May not - be an absolute path. May not contain - the path element '..'. May not start - with the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, - kind, uid?' - type: string - optional: - description: optional field specify whether - the Secret or its key must be defined - type: boolean - type: object - serviceAccountToken: - description: serviceAccountToken is information - about the serviceAccountToken data to project - properties: - audience: - description: audience is the intended audience - of the token. A recipient of a token must - identify itself with an identifier specified - in the audience of the token, and otherwise - should reject the token. The audience defaults - to the identifier of the apiserver. - type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account - token. As the token approaches expiration, - the kubelet volume plugin will proactively - rotate the service account token. The kubelet - will start trying to rotate the token if the - token is older than 80 percent of its time - to live or if the token is older than 24 hours.Defaults - to 1 hour and must be at least 10 minutes. - format: int64 - type: integer - path: - description: path is the path relative to the - mount point of the file to project the token - into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host - that shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is - no group - type: string - readOnly: - description: readOnly here will force the Quobyte volume - to be mounted with read-only permissions. Defaults to - false. - type: boolean - registry: - description: registry represents a single or multiple - Quobyte Registry services specified as a string as host:port - pair (multiple entries are separated with commas) which - acts as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in - the Backend Used with dynamically provisioned Quobyte - volumes, value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to - serivceaccount user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on - the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from - compromising the machine' - type: string - image: - description: 'image is the rados image name. More info: - https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is - rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication - secret for RBDUser. If provided overrides keyring. Default - is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - user: - description: 'user is the rados user name. Default is - admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume - attached and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO - API Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO - Protection Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not - provided, Login operation will fail. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage - for a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system - as configured in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already - created in the ScaleIO system that is associated with - this volume source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used - to set permissions on created files by default. Must - be an octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for mode - bits. Defaults to 0644. Directories within the path - are not affected by this setting. This might be in conflict - with other options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair - in the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and - content is the value. If specified, the listed keys - will be projected into the specified paths, and unlisted - keys will not be present. If a key is specified which - is not present in the Secret, the volume setup will - error unless it is marked optional. Paths must be relative - and may not contain the '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to - set permissions on this file. Must be an octal - value between 0000 and 0777 or a decimal value - between 0 and 511. YAML accepts both octal and - decimal values, JSON requires decimal values for - mode bits. If not specified, the volume defaultMode - will be used. This might be in conflict with other - options that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of the file - to map the key to. May not be an absolute path. - May not contain the path element '..'. May not - start with the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret - or its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in - the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for - obtaining the StorageOS API credentials. If not specified, - default values will be attempted. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - type: object - volumeName: - description: volumeName is the human-readable name of - the StorageOS volume. Volume names are only unique - within a namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the - volume within StorageOS. If no namespace is specified - then the Pod's namespace will be used. This allows - the Kubernetes name scoping to be mirrored within StorageOS - for tighter integration. Set VolumeName to any name - to override the default behaviour. Set to "default" - if you are not using namespaces within StorageOS. Namespaces - that do not pre-exist within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fsType is filesystem type to mount. Must - be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred - to be "ext4" if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based - Management (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object required: - name type: object @@ -3467,7 +371,7 @@ spec: - labelsSelector type: object required: - - remoteVolume + - persistentVolumeClaim - target type: object schedule: diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml index 4542c6957..062b596c3 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml @@ -166,6 +166,9 @@ spec: parentBackupName: description: record parentBackupName if backupType is incremental. type: string + persistentVolumeClaimName: + description: remoteVolume saves the backup data. + type: string phase: description: BackupPhase The current phase. Valid values are New, InProgress, Completed, Failed. @@ -175,1535 +178,6 @@ spec: - Completed - Failed type: string - remoteVolume: - description: remoteVolume saves the backup data. - properties: - awsElasticBlockStore: - description: 'awsElasticBlockStore represents an AWS Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount by - volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property empty).' - format: int32 - type: integer - readOnly: - description: 'readOnly value true will force the readOnly - setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: boolean - volumeID: - description: 'volumeID is unique ID of the persistent disk - resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' - type: string - required: - - volumeID - type: object - azureDisk: - description: azureDisk represents an Azure Data Disk mount on - the host and bind mount to the pod. - properties: - cachingMode: - description: 'cachingMode is the Host Caching mode: None, - Read Only, Read Write.' - type: string - diskName: - description: diskName is the Name of the data disk in the - blob storage - type: string - diskURI: - description: diskURI is the URI of data disk in the blob storage - type: string - fsType: - description: fsType is Filesystem type to mount. Must be a - filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - kind: - description: 'kind expected values are Shared: multiple blob - disks per storage account Dedicated: single blob disk per - storage account Managed: azure managed data disk (only - in managed availability set). defaults to shared' - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - description: azureFile represents an Azure File Service mount - on the host and bind mount to the pod. - properties: - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretName: - description: secretName is the name of secret that contains - Azure Storage Account Name and Key - type: string - shareName: - description: shareName is the azure share Name - type: string - required: - - secretName - - shareName - type: object - cephfs: - description: cephFS represents a Ceph FS mount on the host that - shares a pod's lifetime - properties: - monitors: - description: 'monitors is Required: Monitors is a collection - of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - items: - type: string - type: array - path: - description: 'path is Optional: Used as the mounted root, - rather than the full Ceph tree, default is /' - type: string - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: boolean - secretFile: - description: 'secretFile is Optional: SecretFile is the path - to key ring for User, default is /etc/ceph/user.secret More - info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - secretRef: - description: 'secretRef is Optional: SecretRef is reference - to the authentication secret for User, default is empty. - More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - user: - description: 'user is optional: User is the rados user name, - default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' - type: string - required: - - monitors - type: object - cinder: - description: 'cinder represents a cinder volume attached and mounted - on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Examples: "ext4", "xfs", "ntfs". Implicitly inferred to - be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - readOnly: - description: 'readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. More - info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: boolean - secretRef: - description: 'secretRef is optional: points to a secret object - containing parameters used to connect to OpenStack.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - volumeID: - description: 'volumeID used to identify the volume in cinder. - More info: https://examples.k8s.io/mysql-cinder-pd/README.md' - type: string - required: - - volumeID - type: object - configMap: - description: configMap represents a configMap that should populate - this volume - properties: - defaultMode: - description: 'defaultMode is optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect - the file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - items: - description: items if unspecified, each key-value pair in - the Data field of the referenced ConfigMap will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the - ConfigMap, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to set - permissions on this file. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: path is the relative path of the file to - map the key to. May not be an absolute path. May not - contain the path element '..'. May not start with - the string '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - optional: - description: optional specify whether the ConfigMap or its - keys must be defined - type: boolean - type: object - csi: - description: csi (Container Storage Interface) represents ephemeral - storage that is handled by certain external CSI drivers (Beta - feature). - properties: - driver: - description: driver is the name of the CSI driver that handles - this volume. Consult with your admin for the correct name - as registered in the cluster. - type: string - fsType: - description: fsType to mount. Ex. "ext4", "xfs", "ntfs". If - not provided, the empty value is passed to the associated - CSI driver which will determine the default filesystem to - apply. - type: string - nodePublishSecretRef: - description: nodePublishSecretRef is a reference to the secret - object containing sensitive information to pass to the CSI - driver to complete the CSI NodePublishVolume and NodeUnpublishVolume - calls. This field is optional, and may be empty if no secret - is required. If the secret object contains more than one - secret, all secret references are passed. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - readOnly: - description: readOnly specifies a read-only configuration - for the volume. Defaults to false (read/write). - type: boolean - volumeAttributes: - additionalProperties: - type: string - description: volumeAttributes stores driver-specific properties - that are passed to the CSI driver. Consult your driver's - documentation for supported values. - type: object - required: - - driver - type: object - downwardAPI: - description: downwardAPI represents downward API about the pod - that should populate this volume - properties: - defaultMode: - description: 'Optional: mode bits to use on created files - by default. Must be a Optional: mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Defaults to 0644. Directories within - the path are not affected by this setting. This might be - in conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits set.' - format: int32 - type: integer - items: - description: Items is a list of downward API volume file - items: - description: DownwardAPIVolumeFile represents information - to create the file containing the pod field - properties: - fieldRef: - description: 'Required: Selects a field of the pod: - only annotations, labels, name and namespace are supported.' - properties: - apiVersion: - description: Version of the schema the FieldPath - is written in terms of, defaults to "v1". - type: string - fieldPath: - description: Path of the field to select in the - specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used to set permissions - on this file, must be an octal value between 0000 - and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative path name - of the file to be created. Must not be absolute or - contain the ''..'' path. Must be utf-8 encoded. The - first item of the relative path must not start with - ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: only - resources limits and requests (limits.cpu, limits.memory, - requests.cpu and requests.memory) are currently supported.' - properties: - containerName: - description: 'Container name: required for volumes, - optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format of the - exposed resources, defaults to "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - description: 'emptyDir represents a temporary directory that shares - a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - properties: - medium: - description: 'medium represents what type of storage medium - should back this directory. The default is "" which means - to use the node''s default medium. Must be an empty string - (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' - type: string - sizeLimit: - anyOf: - - type: integer - - type: string - description: 'sizeLimit is the total amount of local storage - required for this EmptyDir volume. The size limit is also - applicable for memory medium. The maximum usage on memory - medium EmptyDir would be the minimum value between the SizeLimit - specified here and the sum of memory limits of all containers - in a pod. The default is nil which means that the limit - is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - ephemeral: - description: "ephemeral represents a volume that is handled by - a cluster storage driver. The volume's lifecycle is tied to - the pod that defines it - it will be created before the pod - starts, and deleted when the pod is removed. \n Use this if: - a) the volume is only needed while the pod runs, b) features - of normal volumes like restoring from snapshot or capacity tracking - are needed, c) the storage driver is specified through a storage - class, and d) the storage driver supports dynamic volume provisioning - through a PersistentVolumeClaim (see EphemeralVolumeSource for - more information on the connection between this volume type - and PersistentVolumeClaim). \n Use PersistentVolumeClaim or - one of the vendor-specific APIs for volumes that persist for - longer than the lifecycle of an individual pod. \n Use CSI for - light-weight local ephemeral volumes if the CSI driver is meant - to be used that way - see the documentation of the driver for - more information. \n A pod can use both types of ephemeral volumes - and persistent volumes at the same time." - properties: - volumeClaimTemplate: - description: "Will be used to create a stand-alone PVC to - provision the volume. The pod in which this EphemeralVolumeSource - is embedded will be the owner of the PVC, i.e. the PVC will - be deleted together with the pod. The name of the PVC will - be `-` where `` is the - name from the `PodSpec.Volumes` array entry. Pod validation - will reject the pod if the concatenated name is not valid - for a PVC (for example, too long). \n An existing PVC with - that name that is not owned by the pod will *not* be used - for the pod to avoid using an unrelated volume by mistake. - Starting the pod is then blocked until the unrelated PVC - is removed. If such a pre-created PVC is meant to be used - by the pod, the PVC has to updated with an owner reference - to the pod once the pod exists. Normally this should not - be necessary, but it may be useful when manually reconstructing - a broken cluster. \n This field is read-only and no changes - will be made by Kubernetes to the PVC after it has been - created. \n Required, must not be nil." - properties: - metadata: - description: May contain labels and annotations that will - be copied into the PVC when creating it. No other fields - are allowed and will be rejected during validation. - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: The specification for the PersistentVolumeClaim. - The entire content is copied unchanged into the PVC - that gets created from this template. The same fields - as in a PersistentVolumeClaim are also valid here. - properties: - accessModes: - description: 'accessModes contains the desired access - modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' - items: - type: string - type: array - dataSource: - description: 'dataSource field can be used to specify - either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) - * An existing PVC (PersistentVolumeClaim) If the - provisioner or an external controller can support - the specified data source, it will create a new - volume based on the contents of the specified data - source. When the AnyVolumeDataSource feature gate - is enabled, dataSource contents will be copied to - dataSourceRef, and dataSourceRef contents will be - copied to dataSource when dataSourceRef.namespace - is not specified. If the namespace is specified, - then dataSourceRef will not be copied to dataSource.' - properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API group. - For any other third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - required: - - kind - - name - type: object - dataSourceRef: - description: 'dataSourceRef specifies the object from - which to populate the volume with data, if a non-empty - volume is desired. This may be any object from a - non-empty API group (non core object) or a PersistentVolumeClaim - object. When this field is specified, volume binding - will only succeed if the type of the specified object - matches some installed volume populator or dynamic - provisioner. This field will replace the functionality - of the dataSource field and as such if both fields - are non-empty, they must have the same value. For - backwards compatibility, when namespace isn''t specified - in dataSourceRef, both fields (dataSource and dataSourceRef) - will be set to the same value automatically if one - of them is empty and the other is non-empty. When - namespace is specified in dataSourceRef, dataSource - isn''t set to the same value and must be empty. - There are three important differences between dataSource - and dataSourceRef: * While dataSource only allows - two specific types of objects, dataSourceRef allows - any non-core object, as well as PersistentVolumeClaim - objects. * While dataSource ignores disallowed values - (dropping them), dataSourceRef preserves all values, - and generates an error if a disallowed value is - specified. * While dataSource only allows local - objects, dataSourceRef allows objects in any namespaces. - (Beta) Using this field requires the AnyVolumeDataSource - feature gate to be enabled. (Alpha) Using the namespace - field of dataSourceRef requires the CrossNamespaceVolumeDataSource - feature gate to be enabled.' - properties: - apiGroup: - description: APIGroup is the group for the resource - being referenced. If APIGroup is not specified, - the specified Kind must be in the core API group. - For any other third-party types, APIGroup is - required. - type: string - kind: - description: Kind is the type of resource being - referenced - type: string - name: - description: Name is the name of resource being - referenced - type: string - namespace: - description: Namespace is the namespace of resource - being referenced Note that when a namespace - is specified, a gateway.networking.k8s.io/ReferenceGrant - object is required in the referent namespace - to allow that namespace's owner to accept the - reference. See the ReferenceGrant documentation - for details. (Alpha) This field requires the - CrossNamespaceVolumeDataSource feature gate - to be enabled. - type: string - required: - - kind - - name - type: object - resources: - description: 'resources represents the minimum resources - the volume should have. If RecoverVolumeExpansionFailure - feature is enabled users are allowed to specify - resource requirements that are lower than previous - value but must still be higher than capacity recorded - in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' - properties: - claims: - description: "Claims lists the names of resources, - defined in spec.resourceClaims, that are used - by this container. \n This is an alpha field - and requires enabling the DynamicResourceAllocation - feature gate. \n This field is immutable." - items: - description: ResourceClaim references one entry - in PodSpec.ResourceClaims. - properties: - name: - description: Name must match the name of - one entry in pod.spec.resourceClaims of - the Pod where this field is used. It makes - that resource available inside a container. - type: string - required: - - name - type: object - type: array - x-kubernetes-list-map-keys: - - name - x-kubernetes-list-type: map - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount - of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount - of compute resources required. If Requests is - omitted for a container, it defaults to Limits - if that is explicitly specified, otherwise to - an implementation-defined value. More info: - https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' - type: object - type: object - selector: - description: selector is a label query over volumes - to consider for binding. - properties: - matchExpressions: - description: matchExpressions is a list of label - selector requirements. The requirements are - ANDed. - items: - description: A label selector requirement is - a selector that contains values, a key, and - an operator that relates the key and values. - properties: - key: - description: key is the label key that the - selector applies to. - type: string - operator: - description: operator represents a key's - relationship to a set of values. Valid - operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string - values. If the operator is In or NotIn, - the values array must be non-empty. If - the operator is Exists or DoesNotExist, - the values array must be empty. This array - is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} - pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, - whose key field is "key", the operator is "In", - and the values array contains only "value". - The requirements are ANDed. - type: object - type: object - storageClassName: - description: 'storageClassName is the name of the - StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' - type: string - volumeMode: - description: volumeMode defines what type of volume - is required by the claim. Value of Filesystem is - implied when not included in claim spec. - type: string - volumeName: - description: volumeName is the binding reference to - the PersistentVolume backing this claim. - type: string - type: object - required: - - spec - type: object - type: object - fc: - description: fc represents a Fibre Channel resource that is attached - to a kubelet's host machine and then exposed to the pod. - properties: - fsType: - description: 'fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. TODO: how do we prevent errors in the filesystem - from compromising the machine' - type: string - lun: - description: 'lun is Optional: FC target lun number' - format: int32 - type: integer - readOnly: - description: 'readOnly is Optional: Defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - targetWWNs: - description: 'targetWWNs is Optional: FC target worldwide - names (WWNs)' - items: - type: string - type: array - wwids: - description: 'wwids Optional: FC volume world wide identifiers - (wwids) Either wwids or combination of targetWWNs and lun - must be set, but not both simultaneously.' - items: - type: string - type: array - type: object - flexVolume: - description: flexVolume represents a generic volume resource that - is provisioned/attached using an exec based plugin. - properties: - driver: - description: driver is the name of the driver to use for this - volume. - type: string - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". The default filesystem depends - on FlexVolume script. - type: string - options: - additionalProperties: - type: string - description: 'options is Optional: this field holds extra - command options if any.' - type: object - readOnly: - description: 'readOnly is Optional: defaults to false (read/write). - ReadOnly here will force the ReadOnly setting in VolumeMounts.' - type: boolean - secretRef: - description: 'secretRef is Optional: secretRef is reference - to the secret object containing sensitive information to - pass to the plugin scripts. This may be empty if no secret - object is specified. If the secret object contains more - than one secret, all secrets are passed to the plugin scripts.' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - required: - - driver - type: object - flocker: - description: flocker represents a Flocker volume attached to a - kubelet's host machine. This depends on the Flocker control - service being running - properties: - datasetName: - description: datasetName is Name of the dataset stored as - metadata -> name on the dataset for Flocker should be considered - as deprecated - type: string - datasetUUID: - description: datasetUUID is the UUID of the dataset. This - is unique identifier of a Flocker dataset - type: string - type: object - gcePersistentDisk: - description: 'gcePersistentDisk represents a GCE Disk resource - that is attached to a kubelet''s host machine and then exposed - to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - properties: - fsType: - description: 'fsType is filesystem type of the volume that - you want to mount. Tip: Ensure that the filesystem type - is supported by the host operating system. Examples: "ext4", - "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. - More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - partition: - description: 'partition is the partition in the volume that - you want to mount. If omitted, the default is to mount by - volume name. Examples: For volume /dev/sda1, you specify - the partition as "1". Similarly, the volume partition for - /dev/sda is "0" (or you can leave the property empty). More - info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - format: int32 - type: integer - pdName: - description: 'pdName is unique name of the PD resource in - GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' - type: boolean - required: - - pdName - type: object - gitRepo: - description: 'gitRepo represents a git repository at a particular - revision. DEPRECATED: GitRepo is deprecated. To provision a - container with a git repo, mount an EmptyDir into an InitContainer - that clones the repo using git, then mount the EmptyDir into - the Pod''s container.' - properties: - directory: - description: directory is the target directory name. Must - not contain or start with '..'. If '.' is supplied, the - volume directory will be the git repository. Otherwise, - if specified, the volume will contain the git repository - in the subdirectory with the given name. - type: string - repository: - description: repository is the URL - type: string - revision: - description: revision is the commit hash for the specified - revision. - type: string - required: - - repository - type: object - glusterfs: - description: 'glusterfs represents a Glusterfs mount on the host - that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' - properties: - endpoints: - description: 'endpoints is the endpoint name that details - Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - path: - description: 'path is the Glusterfs volume path. More info: - https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: string - readOnly: - description: 'readOnly here will force the Glusterfs volume - to be mounted with read-only permissions. Defaults to false. - More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' - type: boolean - required: - - endpoints - - path - type: object - hostPath: - description: 'hostPath represents a pre-existing file or directory - on the host machine that is directly exposed to the container. - This is generally used for system agents or other privileged - things that are allowed to see the host machine. Most containers - will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath - --- TODO(jonesdl) We need to restrict who can use host directory - mounts and who can/can not mount host directories as read/write.' - properties: - path: - description: 'path of the directory on the host. If the path - is a symlink, it will follow the link to the real path. - More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - type: - description: 'type for HostPath Volume Defaults to "" More - info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' - type: string - required: - - path - type: object - iscsi: - description: 'iscsi represents an ISCSI Disk resource that is - attached to a kubelet''s host machine and then exposed to the - pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' - properties: - chapAuthDiscovery: - description: chapAuthDiscovery defines whether support iSCSI - Discovery CHAP authentication - type: boolean - chapAuthSession: - description: chapAuthSession defines whether support iSCSI - Session CHAP authentication - type: boolean - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - initiatorName: - description: initiatorName is the custom iSCSI Initiator Name. - If initiatorName is specified with iscsiInterface simultaneously, - new iSCSI interface : will be - created for the connection. - type: string - iqn: - description: iqn is the target iSCSI Qualified Name. - type: string - iscsiInterface: - description: iscsiInterface is the interface Name that uses - an iSCSI transport. Defaults to 'default' (tcp). - type: string - lun: - description: lun represents iSCSI Target Lun number. - format: int32 - type: integer - portals: - description: portals is the iSCSI Target Portal List. The - portal is either an IP or ip_addr:port if the port is other - than default (typically TCP ports 860 and 3260). - items: - type: string - type: array - readOnly: - description: readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. - type: boolean - secretRef: - description: secretRef is the CHAP Secret for iSCSI target - and initiator authentication - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - targetPortal: - description: targetPortal is iSCSI Target Portal. The Portal - is either an IP or ip_addr:port if the port is other than - default (typically TCP ports 860 and 3260). - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - description: 'name of the volume. Must be a DNS_LABEL and unique - within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - nfs: - description: 'nfs represents an NFS mount on the host that shares - a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - properties: - path: - description: 'path that is exported by the NFS server. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - readOnly: - description: 'readOnly here will force the NFS export to be - mounted with read-only permissions. Defaults to false. More - info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: boolean - server: - description: 'server is the hostname or IP address of the - NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - description: 'persistentVolumeClaimVolumeSource represents a reference - to a PersistentVolumeClaim in the same namespace. More info: - https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - properties: - claimName: - description: 'claimName is the name of a PersistentVolumeClaim - in the same namespace as the pod using this volume. More - info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' - type: string - readOnly: - description: readOnly Will force the ReadOnly setting in VolumeMounts. - Default false. - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - description: photonPersistentDisk represents a PhotonController - persistent disk attached and mounted on kubelets host machine - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - pdID: - description: pdID is the ID that identifies Photon Controller - persistent disk - type: string - required: - - pdID - type: object - portworxVolume: - description: portworxVolume represents a portworx volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fSType represents the filesystem type to mount - Must be a filesystem type supported by the host operating - system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - volumeID: - description: volumeID uniquely identifies a Portworx volume - type: string - required: - - volumeID - type: object - projected: - description: projected items for all in one resources secrets, - configmaps, and downward API - properties: - defaultMode: - description: defaultMode are the mode bits used to set permissions - on created files by default. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON requires decimal - values for mode bits. Directories within the path are not - affected by this setting. This might be in conflict with - other options that affect the file mode, like fsGroup, and - the result can be other mode bits set. - format: int32 - type: integer - sources: - description: sources is the list of volume projections - items: - description: Projection that may be projected along with - other supported volume types - properties: - configMap: - description: configMap information about the configMap - data to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced ConfigMap - will be projected into the volume as a file whose - name is the key and content is the value. If specified, - the listed keys will be projected into the specified - paths, and unlisted keys will not be present. - If a key is specified which is not present in - the ConfigMap, the volume setup will error unless - it is marked optional. Paths must be relative - and may not contain the '..' path or start with - '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. If - not specified, the volume defaultMode will - be used. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional specify whether the ConfigMap - or its keys must be defined - type: boolean - type: object - downwardAPI: - description: downwardAPI information about the downwardAPI - data to project - properties: - items: - description: Items is a list of DownwardAPIVolume - file - items: - description: DownwardAPIVolumeFile represents - information to create the file containing the - pod field - properties: - fieldRef: - description: 'Required: Selects a field of - the pod: only annotations, labels, name - and namespace are supported.' - properties: - apiVersion: - description: Version of the schema the - FieldPath is written in terms of, defaults - to "v1". - type: string - fieldPath: - description: Path of the field to select - in the specified API version. - type: string - required: - - fieldPath - type: object - mode: - description: 'Optional: mode bits used to - set permissions on this file, must be an - octal value between 0000 and 0777 or a decimal - value between 0 and 511. YAML accepts both - octal and decimal values, JSON requires - decimal values for mode bits. If not specified, - the volume defaultMode will be used. This - might be in conflict with other options - that affect the file mode, like fsGroup, - and the result can be other mode bits set.' - format: int32 - type: integer - path: - description: 'Required: Path is the relative - path name of the file to be created. Must - not be absolute or contain the ''..'' path. - Must be utf-8 encoded. The first item of - the relative path must not start with ''..''' - type: string - resourceFieldRef: - description: 'Selects a resource of the container: - only resources limits and requests (limits.cpu, - limits.memory, requests.cpu and requests.memory) - are currently supported.' - properties: - containerName: - description: 'Container name: required - for volumes, optional for env vars' - type: string - divisor: - anyOf: - - type: integer - - type: string - description: Specifies the output format - of the exposed resources, defaults to - "1" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - resource: - description: 'Required: resource to select' - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - description: secret information about the secret data - to project - properties: - items: - description: items if unspecified, each key-value - pair in the Data field of the referenced Secret - will be projected into the volume as a file whose - name is the key and content is the value. If specified, - the listed keys will be projected into the specified - paths, and unlisted keys will not be present. - If a key is specified which is not present in - the Secret, the volume setup will error unless - it is marked optional. Paths must be relative - and may not contain the '..' path or start with - '..'. - items: - description: Maps a string key to a path within - a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits - used to set permissions on this file. Must - be an octal value between 0000 and 0777 - or a decimal value between 0 and 511. YAML - accepts both octal and decimal values, JSON - requires decimal values for mode bits. If - not specified, the volume defaultMode will - be used. This might be in conflict with - other options that affect the file mode, - like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - path: - description: path is the relative path of - the file to map the key to. May not be an - absolute path. May not contain the path - element '..'. May not start with the string - '..'. - type: string - required: - - key - - path - type: object - type: array - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, - uid?' - type: string - optional: - description: optional field specify whether the - Secret or its key must be defined - type: boolean - type: object - serviceAccountToken: - description: serviceAccountToken is information about - the serviceAccountToken data to project - properties: - audience: - description: audience is the intended audience of - the token. A recipient of a token must identify - itself with an identifier specified in the audience - of the token, and otherwise should reject the - token. The audience defaults to the identifier - of the apiserver. - type: string - expirationSeconds: - description: expirationSeconds is the requested - duration of validity of the service account token. - As the token approaches expiration, the kubelet - volume plugin will proactively rotate the service - account token. The kubelet will start trying to - rotate the token if the token is older than 80 - percent of its time to live or if the token is - older than 24 hours.Defaults to 1 hour and must - be at least 10 minutes. - format: int64 - type: integer - path: - description: path is the path relative to the mount - point of the file to project the token into. - type: string - required: - - path - type: object - type: object - type: array - type: object - quobyte: - description: quobyte represents a Quobyte mount on the host that - shares a pod's lifetime - properties: - group: - description: group to map volume access to Default is no group - type: string - readOnly: - description: readOnly here will force the Quobyte volume to - be mounted with read-only permissions. Defaults to false. - type: boolean - registry: - description: registry represents a single or multiple Quobyte - Registry services specified as a string as host:port pair - (multiple entries are separated with commas) which acts - as the central registry for volumes - type: string - tenant: - description: tenant owning the given Quobyte volume in the - Backend Used with dynamically provisioned Quobyte volumes, - value is set by the plugin - type: string - user: - description: user to map volume access to Defaults to serivceaccount - user - type: string - volume: - description: volume is a string that references an already - created Quobyte volume by name. - type: string - required: - - registry - - volume - type: object - rbd: - description: 'rbd represents a Rados Block Device mount on the - host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' - properties: - fsType: - description: 'fsType is the filesystem type of the volume - that you want to mount. Tip: Ensure that the filesystem - type is supported by the host operating system. Examples: - "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd - TODO: how do we prevent errors in the filesystem from compromising - the machine' - type: string - image: - description: 'image is the rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - keyring: - description: 'keyring is the path to key ring for RBDUser. - Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - monitors: - description: 'monitors is a collection of Ceph monitors. More - info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - items: - type: string - type: array - pool: - description: 'pool is the rados pool name. Default is rbd. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - readOnly: - description: 'readOnly here will force the ReadOnly setting - in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: boolean - secretRef: - description: 'secretRef is name of the authentication secret - for RBDUser. If provided overrides keyring. Default is nil. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - user: - description: 'user is the rados user name. Default is admin. - More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' - type: string - required: - - image - - monitors - type: object - scaleIO: - description: scaleIO represents a ScaleIO persistent volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Default is "xfs". - type: string - gateway: - description: gateway is the host address of the ScaleIO API - Gateway. - type: string - protectionDomain: - description: protectionDomain is the name of the ScaleIO Protection - Domain for the configured storage. - type: string - readOnly: - description: readOnly Defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef references to the secret for ScaleIO - user and other sensitive information. If this is not provided, - Login operation will fail. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - sslEnabled: - description: sslEnabled Flag enable/disable SSL communication - with Gateway, default false - type: boolean - storageMode: - description: storageMode indicates whether the storage for - a volume should be ThickProvisioned or ThinProvisioned. - Default is ThinProvisioned. - type: string - storagePool: - description: storagePool is the ScaleIO Storage Pool associated - with the protection domain. - type: string - system: - description: system is the name of the storage system as configured - in ScaleIO. - type: string - volumeName: - description: volumeName is the name of a volume already created - in the ScaleIO system that is associated with this volume - source. - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - description: 'secret represents a secret that should populate - this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - properties: - defaultMode: - description: 'defaultMode is Optional: mode bits used to set - permissions on created files by default. Must be an octal - value between 0000 and 0777 or a decimal value between 0 - and 511. YAML accepts both octal and decimal values, JSON - requires decimal values for mode bits. Defaults to 0644. - Directories within the path are not affected by this setting. - This might be in conflict with other options that affect - the file mode, like fsGroup, and the result can be other - mode bits set.' - format: int32 - type: integer - items: - description: items If unspecified, each key-value pair in - the Data field of the referenced Secret will be projected - into the volume as a file whose name is the key and content - is the value. If specified, the listed keys will be projected - into the specified paths, and unlisted keys will not be - present. If a key is specified which is not present in the - Secret, the volume setup will error unless it is marked - optional. Paths must be relative and may not contain the - '..' path or start with '..'. - items: - description: Maps a string key to a path within a volume. - properties: - key: - description: key is the key to project. - type: string - mode: - description: 'mode is Optional: mode bits used to set - permissions on this file. Must be an octal value between - 0000 and 0777 or a decimal value between 0 and 511. - YAML accepts both octal and decimal values, JSON requires - decimal values for mode bits. If not specified, the - volume defaultMode will be used. This might be in - conflict with other options that affect the file mode, - like fsGroup, and the result can be other mode bits - set.' - format: int32 - type: integer - path: - description: path is the relative path of the file to - map the key to. May not be an absolute path. May not - contain the path element '..'. May not start with - the string '..'. - type: string - required: - - key - - path - type: object - type: array - optional: - description: optional field specify whether the Secret or - its keys must be defined - type: boolean - secretName: - description: 'secretName is the name of the secret in the - pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' - type: string - type: object - storageos: - description: storageOS represents a StorageOS volume attached - and mounted on Kubernetes nodes. - properties: - fsType: - description: fsType is the filesystem type to mount. Must - be a filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - readOnly: - description: readOnly defaults to false (read/write). ReadOnly - here will force the ReadOnly setting in VolumeMounts. - type: boolean - secretRef: - description: secretRef specifies the secret to use for obtaining - the StorageOS API credentials. If not specified, default - values will be attempted. - properties: - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Add other useful fields. apiVersion, kind, uid?' - type: string - type: object - volumeName: - description: volumeName is the human-readable name of the - StorageOS volume. Volume names are only unique within a - namespace. - type: string - volumeNamespace: - description: volumeNamespace specifies the scope of the volume - within StorageOS. If no namespace is specified then the - Pod's namespace will be used. This allows the Kubernetes - name scoping to be mirrored within StorageOS for tighter - integration. Set VolumeName to any name to override the - default behaviour. Set to "default" if you are not using - namespaces within StorageOS. Namespaces that do not pre-exist - within StorageOS will be created. - type: string - type: object - vsphereVolume: - description: vsphereVolume represents a vSphere volume attached - and mounted on kubelets host machine - properties: - fsType: - description: fsType is filesystem type to mount. Must be a - filesystem type supported by the host operating system. - Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" - if unspecified. - type: string - storagePolicyID: - description: storagePolicyID is the storage Policy Based Management - (SPBM) profile ID associated with the StoragePolicyName. - type: string - storagePolicyName: - description: storagePolicyName is the storage Policy Based - Management (SPBM) profile name. - type: string - volumePath: - description: volumePath is the path that identifies vSphere - volume vmdk - type: string - required: - - volumePath - type: object - required: - - name - type: object startTimestamp: description: Date/time when the backup started being processed. format: date-time diff --git a/deploy/helm/templates/addons/csi-s3-addon.yaml b/deploy/helm/templates/addons/csi-s3-addon.yaml index 1a5819226..f22d125d6 100644 --- a/deploy/helm/templates/addons/csi-s3-addon.yaml +++ b/deploy/helm/templates/addons/csi-s3-addon.yaml @@ -26,12 +26,3 @@ spec: installable: autoInstall: {{ get ( get ( .Values | toYaml | fromYaml ) "csi-s3" ) "enabled" }} - selectors: - - key: KubeGitVersion - operator: Contains - values: - - eks - - aliyun - - gke - - tke - - aks diff --git a/deploy/helm/templates/configmap.yaml b/deploy/helm/templates/configmap.yaml new file mode 100644 index 000000000..f1be0b870 --- /dev/null +++ b/deploy/helm/templates/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "kubeblocks.fullname" . }}-manager-config + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} +data: + config.yaml: | + # the global pvc name which persistent volume claim to store the backup data. + # will replace the pvc name when it is empty in the backup policy. + BACKUP_PVC_NAME: "{{ .Values.dataProtection.backupPVCName }}" + + # the init capacity of pvc for creating the pvc, e.g. 10Gi. + # will replace the init capacity when it is empty in the backup policy. + BACKUP_PVC_INIT_CAPACITY: "{{ .Values.dataProtection.backupPVCInitCapacity }}" + + # the pvc storage class name. + # will replace the storageClassName when it is nil in the backup policy. + BACKUP_PVC_STORAGE_CLASS: "{{ .Values.dataProtection.backupPVCStorageClassName }}" + + # the pvc create policy. + # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. + # otherwise, using "Never" policy. + # only affect the backupPolicy automatically created by Kubeblocks. + BACKUP_PVC_CREATE_POLICY: "{{ .Values.dataProtection.backupPVCCreatePolicy }}" \ No newline at end of file diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml index c63c8079d..dc758226d 100644 --- a/deploy/helm/templates/deployment.yaml +++ b/deploy/helm/templates/deployment.yaml @@ -123,6 +123,8 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: + - mountPath: /etc/kubeblocks + name: manager-config {{- if .Values.admissionWebhooks.enabled }} - mountPath: /tmp/k8s-webhook-server/serving-certs name: cert @@ -154,7 +156,7 @@ spec: volumes: - name: manager-config configMap: - name: manager-config + name: {{ include "kubeblocks.fullname" . }}-manager-config {{- if .Values.admissionWebhooks.enabled }} - name: cert secret: diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index df02cc1b0..1366ac989 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -244,8 +244,15 @@ admissionWebhooks: ## dataProtection: ## @param dataProtection.enableVolumeSnapshot - set this to true if cluster does have snapshot.storage.k8s.io API installed - ## + ## @param dataProtection.backupPVCName - set the default pvc to store the file for backup + ## @param dataProtection.backupPVCInitCapacity - set the default pvc initCapacity if the pvc need to be created by backup controller + ## @param dataProtection.backupPVCStorageClassName - set the default pvc storageClassName if the pvc need to be created by backup controller + ## @param dataProtection.backupPVCCreatePolicy - set the default create policy of the pvc, optional values: IfNotPresent, Never enableVolumeSnapshot: false + backupPVCName: "" + backupPVCInitCapacity: "" + backupPVCStorageClassName: "" + backupPVCCreatePolicy: "" ## Addon controller settings, this will require cluster-admin clusterrole. addonController: diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index 383dbb872..a0b412463 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -23,12 +23,6 @@ Manage alert receiver, include add, list and delete receiver. * [kbcli alert list-receivers](kbcli_alert_list-receivers.md) - List all alert receivers. -## [backup-config](kbcli_backup-config.md) - -KubeBlocks backup config. - - - ## [bench](kbcli_bench.md) Run a benchmark. @@ -119,6 +113,7 @@ List and open the KubeBlocks dashboards. KubeBlocks operation commands. +* [kbcli kubeblocks config](kbcli_kubeblocks_config.md) - KubeBlocks config. * [kbcli kubeblocks install](kbcli_kubeblocks_install.md) - Install KubeBlocks. * [kbcli kubeblocks list-versions](kbcli_kubeblocks_list-versions.md) - List KubeBlocks versions. * [kbcli kubeblocks preflight](kbcli_kubeblocks_preflight.md) - Run and retrieve preflight checks for KubeBlocks. diff --git a/docs/user_docs/cli/kbcli.md b/docs/user_docs/cli/kbcli.md index d8a45fd46..084fb938e 100644 --- a/docs/user_docs/cli/kbcli.md +++ b/docs/user_docs/cli/kbcli.md @@ -56,7 +56,6 @@ kbcli [flags] * [kbcli addon](kbcli_addon.md) - Addon command. * [kbcli alert](kbcli_alert.md) - Manage alert receiver, include add, list and delete receiver. -* [kbcli backup-config](kbcli_backup-config.md) - KubeBlocks backup config. * [kbcli bench](kbcli_bench.md) - Run a benchmark. * [kbcli class](kbcli_class.md) - Manage classes * [kbcli cluster](kbcli_cluster.md) - Cluster command. diff --git a/docs/user_docs/cli/kbcli_kubeblocks.md b/docs/user_docs/cli/kbcli_kubeblocks.md index 058612ee4..71eb55914 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks.md +++ b/docs/user_docs/cli/kbcli_kubeblocks.md @@ -37,6 +37,7 @@ KubeBlocks operation commands. ### SEE ALSO +* [kbcli kubeblocks config](kbcli_kubeblocks_config.md) - KubeBlocks config. * [kbcli kubeblocks install](kbcli_kubeblocks_install.md) - Install KubeBlocks. * [kbcli kubeblocks list-versions](kbcli_kubeblocks_list-versions.md) - List KubeBlocks versions. * [kbcli kubeblocks preflight](kbcli_kubeblocks_preflight.md) - Run and retrieve preflight checks for KubeBlocks. diff --git a/internal/cli/cmd/backupconfig/suite_test.go b/internal/cli/cmd/backupconfig/suite_test.go deleted file mode 100644 index 92317831c..000000000 --- a/internal/cli/cmd/backupconfig/suite_test.go +++ /dev/null @@ -1,29 +0,0 @@ -/* -Copyright ApeCloud, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package backupconfig - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestApps(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "BackupConfig Suite") -} diff --git a/internal/cli/cmd/cli.go b/internal/cli/cmd/cli.go index ea1a2a4e5..e5387b7cf 100644 --- a/internal/cli/cmd/cli.go +++ b/internal/cli/cmd/cli.go @@ -30,7 +30,6 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/cmd/addon" "github.com/apecloud/kubeblocks/internal/cli/cmd/alert" - "github.com/apecloud/kubeblocks/internal/cli/cmd/backupconfig" "github.com/apecloud/kubeblocks/internal/cli/cmd/bench" "github.com/apecloud/kubeblocks/internal/cli/cmd/class" "github.com/apecloud/kubeblocks/internal/cli/cmd/cluster" @@ -96,7 +95,6 @@ A Command Line Interface for KubeBlocks`, bench.NewBenchCmd(), options.NewCmdOptions(ioStreams.Out), version.NewVersionCmd(f), - backupconfig.NewBackupConfigCmd(f, ioStreams), dashboard.NewDashboardCmd(f, ioStreams), clusterversion.NewClusterVersionCmd(f, ioStreams), clusterdefinition.NewClusterDefinitionCmd(f, ioStreams), diff --git a/internal/cli/cmd/backupconfig/backup_config.go b/internal/cli/cmd/kubeblocks/config.go similarity index 52% rename from internal/cli/cmd/backupconfig/backup_config.go rename to internal/cli/cmd/kubeblocks/config.go index 7b8a25ea8..657e0caac 100644 --- a/internal/cli/cmd/backupconfig/backup_config.go +++ b/internal/cli/cmd/kubeblocks/config.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backupconfig +package kubeblocks import ( "github.com/spf13/cobra" @@ -22,35 +22,53 @@ import ( cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" - "github.com/apecloud/kubeblocks/internal/cli/cmd/kubeblocks" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" ) var backupConfigExample = templates.Examples(` # Enable the snapshot-controller and volume snapshot, to support snapshot backup. - kbcli backup-config --set snapshot-controller.enabled=true - + kbcli kubeblocks config --set snapshot-controller.enabled=true + + Options Parameters: # If you have already installed a snapshot-controller, only enable the snapshot backup feature - kbcli backup-config --set dataProtection.enableVolumeSnapshot=true + dataProtection.enableVolumeSnapshot=true + + # the global pvc name which persistent volume claim to store the backup data. + # will replace the pvc name when it is empty in the backup policy. + dataProtection.backupPVCName=backup-data + + # the init capacity of pvc for creating the pvc, e.g. 10Gi. + # will replace the init capacity when it is empty in the backup policy. + dataProtection.backupPVCInitCapacity=100Gi + + # the pvc storage class name. + # will replace the storageClassName when it is nil in the backup policy. + dataProtection.backupPVCStorageClassName=csi-s3 + + # the pvc create policy. + # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. + # otherwise, using "Never" policy. only affect the backupPolicy automatically created by Kubeblocks. + dataProtection.backupPVCCreatePolicy=Never `) -// NewBackupConfigCmd creates the backup-config command -func NewBackupConfigCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := &kubeblocks.InstallOptions{ - Options: kubeblocks.Options{ +// NewConfigCmd creates the config command +func NewConfigCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := &InstallOptions{ + Options: Options{ IOStreams: streams, }, } cmd := &cobra.Command{ - Use: "backup-config", - Short: "KubeBlocks backup config.", + Use: "config", + Short: "KubeBlocks config.", Example: backupConfigExample, Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { util.CheckErr(o.Complete(f, cmd)) util.CheckErr(o.Upgrade()) + // TODO: post handle after the config updates }, } helm.AddValueOptionsFlags(cmd.Flags(), &o.ValueOpts) diff --git a/internal/cli/cmd/backupconfig/backup_config_test.go b/internal/cli/cmd/kubeblocks/config_test.go similarity index 92% rename from internal/cli/cmd/backupconfig/backup_config_test.go rename to internal/cli/cmd/kubeblocks/config_test.go index d67a57fe7..48879a836 100644 --- a/internal/cli/cmd/backupconfig/backup_config_test.go +++ b/internal/cli/cmd/kubeblocks/config_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package backupconfig +package kubeblocks import ( . "github.com/onsi/ginkgo/v2" @@ -25,7 +25,6 @@ import ( clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" - "github.com/apecloud/kubeblocks/internal/cli/cmd/kubeblocks" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util/helm" @@ -60,8 +59,8 @@ var _ = Describe("backupconfig", func() { return deploy } - o := &kubeblocks.InstallOptions{ - Options: kubeblocks.Options{ + o := &InstallOptions{ + Options: Options{ IOStreams: streams, HelmCfg: helm.NewFakeConfig(testing.Namespace), Namespace: "default", @@ -72,7 +71,7 @@ var _ = Describe("backupconfig", func() { Monitor: true, ValueOpts: values.Options{Values: []string{"snapshot-controller.enabled=true"}}, } - cmd := NewBackupConfigCmd(tf, streams) + cmd := NewConfigCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) Expect(o.Install()).Should(Succeed()) }) diff --git a/internal/cli/cmd/kubeblocks/kubeblocks.go b/internal/cli/cmd/kubeblocks/kubeblocks.go index a97f0bf82..117e1d567 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks.go @@ -35,6 +35,7 @@ func NewKubeBlocksCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *c newUninstallCmd(f, streams), newListVersionsCmd(streams), newStatusCmd(f, streams), + NewConfigCmd(f, streams), ) // add preflight cmd cmd.AddCommand(NewPreflightCmd(f, streams)) diff --git a/internal/constant/const.go b/internal/constant/const.go index 5db40ecd4..7c6b8870f 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -24,6 +24,10 @@ const ( CfgKeyCtrlrMgrNodeSelector = "CM_NODE_SELECTOR" CfgKeyCtrlrMgrTolerations = "CM_TOLERATIONS" CfgKeyCtrlrReconcileRetryDurationMS = "CM_RECON_RETRY_DURATION_MS" // accept time + CfgKeyBackupPVCName = "BACKUP_PVC_NAME" // the global pvc name which persistent volume claim to store the backup data + CfgKeyBackupPVCInitCapacity = "BACKUP_PVC_INIT_CAPACITY" // the init capacity of pvc for creating the pvc, e.g. 10Gi. + CfgKeyBackupPVCStorageClass = "BACKUP_PVC_STORAGE_CLASS" // the pvc storage class name. + CfgKeyBackupPVCCreatePolicy = "BACKUP_PVC_CREATE_POLICY" // the pvc create policy. support "IfNotPresent" or "Never" // addon config keys CfgKeyAddonJobTTL = "ADDON_JOB_TTL" @@ -214,3 +218,5 @@ const ( AccountNameForSecret = "username" AccountPasswdForSecret = "password" ) + +const DefaultBackupPvcInitCapacity = "100Gi" diff --git a/internal/controller/component/restore_utils.go b/internal/controller/component/restore_utils.go index 4d27874cc..356e44576 100644 --- a/internal/controller/component/restore_utils.go +++ b/internal/controller/component/restore_utils.go @@ -93,8 +93,8 @@ func buildInitContainerWithFullBackup( if component.PodSpec == nil || len(component.PodSpec.Containers) == 0 { return nil } - if backup.Status.RemoteVolume == nil { - return fmt.Errorf("remote volume can not be empty in Backup.status.remoteVolume") + if len(backup.Status.PersistentVolumeClaimName) == 0 { + return fmt.Errorf("persistentVolumeClaimName can not be empty in Backup.status") } container := corev1.Container{} container.Name = GetRestoredInitContainerName(backup.Name) @@ -106,10 +106,17 @@ func buildInitContainerWithFullBackup( } container.VolumeMounts = component.PodSpec.Containers[0].VolumeMounts // add the volumeMounts with backup volume - randomVolumeName := fmt.Sprintf("%s-%s", component.Name, backup.Status.RemoteVolume.Name) - backup.Status.RemoteVolume.Name = randomVolumeName + backupVolumeName := fmt.Sprintf("%s-%s", component.Name, backup.Status.PersistentVolumeClaimName) + remoteVolume := corev1.Volume{ + Name: backupVolumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: backup.Status.PersistentVolumeClaimName, + }, + }, + } remoteVolumeMount := corev1.VolumeMount{} - remoteVolumeMount.Name = randomVolumeName + remoteVolumeMount.Name = backupVolumeName remoteVolumeMount.MountPath = "/" + backup.Name container.VolumeMounts = append(container.VolumeMounts, remoteVolumeMount) @@ -131,7 +138,7 @@ func buildInitContainerWithFullBackup( // merge env from backup tool. container.Env = append(container.Env, backupTool.Spec.Env...) // add volume of backup data - component.PodSpec.Volumes = append(component.PodSpec.Volumes, *backup.Status.RemoteVolume) + component.PodSpec.Volumes = append(component.PodSpec.Volumes, remoteVolume) component.PodSpec.InitContainers = append(component.PodSpec.InitContainers, container) return nil } diff --git a/internal/controller/component/restore_utils_test.go b/internal/controller/component/restore_utils_test.go index 7bfd0a575..d83702e1e 100644 --- a/internal/controller/component/restore_utils_test.go +++ b/internal/controller/component/restore_utils_test.go @@ -67,9 +67,7 @@ var _ = Describe("probe_utils", func() { updateBackupStatus := func(backup *dataprotectionv1alpha1.Backup, backupToolName string, expectPhase dataprotectionv1alpha1.BackupPhase) { Expect(testapps.ChangeObjStatus(&testCtx, backup, func() { backup.Status.BackupToolName = backupToolName - backup.Status.RemoteVolume = &corev1.Volume{ - Name: "backup-pvc", - } + backup.Status.PersistentVolumeClaimName = "backup-pvc" backup.Status.Phase = expectPhase })).Should(Succeed()) } diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index 9d326fbdc..d293ce0e4 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -19,8 +19,10 @@ package lifecycle import ( "fmt" + "github.com/spf13/viper" "golang.org/x/exp/slices" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -306,9 +308,23 @@ func (r *backupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.Common if bp == nil { return nil } + defaultCreatePolicy := dataprotectionv1alpha1.CreatePVCPolicyIfNotPresent + globalCreatePolicy := viper.GetString(constant.CfgKeyBackupPVCCreatePolicy) + if len(globalCreatePolicy) != 0 { + defaultCreatePolicy = dataprotectionv1alpha1.CreatePVCPolicy(globalCreatePolicy) + } + defaultInitCapacity := constant.DefaultBackupPvcInitCapacity + globalInitCapacity := viper.GetString(constant.CfgKeyBackupPVCInitCapacity) + if len(globalInitCapacity) != 0 { + defaultInitCapacity = globalInitCapacity + } return &dataprotectionv1alpha1.CommonBackupPolicy{ BackupToolName: bp.BackupToolName, - BasePolicy: r.convertBasePolicy(bp.BasePolicy, clusterName, component, workloadType), + PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ + InitCapacity: resource.MustParse(defaultInitCapacity), + CreatePolicy: defaultCreatePolicy, + }, + BasePolicy: r.convertBasePolicy(bp.BasePolicy, clusterName, component, workloadType), } } diff --git a/internal/testutil/apps/backuppolicy_factory.go b/internal/testutil/apps/backuppolicy_factory.go index 027dec473..3b455b135 100644 --- a/internal/testutil/apps/backuppolicy_factory.go +++ b/internal/testutil/apps/backuppolicy_factory.go @@ -17,10 +17,11 @@ limitations under the License. package apps import ( - corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" ) type MockBackupPolicyFactory struct { @@ -96,13 +97,21 @@ func (factory *MockBackupPolicyFactory) AddSnapshotPolicy() *MockBackupPolicyFac } func (factory *MockBackupPolicyFactory) AddFullPolicy() *MockBackupPolicyFactory { - factory.get().Spec.Full = &dataprotectionv1alpha1.CommonBackupPolicy{} + factory.get().Spec.Full = &dataprotectionv1alpha1.CommonBackupPolicy{ + PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ + CreatePolicy: dataprotectionv1alpha1.CreatePVCPolicyIfNotPresent, + }, + } factory.backupType = dataprotectionv1alpha1.BackupTypeFull return factory } func (factory *MockBackupPolicyFactory) AddIncrementalPolicy() *MockBackupPolicyFactory { - factory.get().Spec.Incremental = &dataprotectionv1alpha1.CommonBackupPolicy{} + factory.get().Spec.Incremental = &dataprotectionv1alpha1.CommonBackupPolicy{ + PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ + CreatePolicy: dataprotectionv1alpha1.CreatePVCPolicyIfNotPresent, + }, + } factory.backupType = dataprotectionv1alpha1.BackupTypeIncremental return factory } @@ -183,23 +192,10 @@ func (factory *MockBackupPolicyFactory) AddHookPostCommand(postCommand string) * return factory } -func (factory *MockBackupPolicyFactory) SetRemoteVolume(volume corev1.Volume) *MockBackupPolicyFactory { - factory.setCommonPolicyField(func(commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) { - commonPolicy.RemoteVolume = volume - }) - return factory -} - -func (factory *MockBackupPolicyFactory) SetRemoteVolumePVC(volumeName, pvcName string) *MockBackupPolicyFactory { +func (factory *MockBackupPolicyFactory) SetPVC(pvcName string) *MockBackupPolicyFactory { factory.setCommonPolicyField(func(commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) { - commonPolicy.RemoteVolume = corev1.Volume{ - Name: volumeName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcName, - }, - }, - } + commonPolicy.PersistentVolumeClaim.Name = pvcName + commonPolicy.PersistentVolumeClaim.InitCapacity = resource.MustParse(constant.DefaultBackupPvcInitCapacity) }) return factory } diff --git a/test/integration/backup_mysql_test.go b/test/integration/backup_mysql_test.go index 88d11d25d..3bf16c836 100644 --- a/test/integration/backup_mysql_test.go +++ b/test/integration/backup_mysql_test.go @@ -40,7 +40,6 @@ var _ = Describe("MySQL data protection function", func() { const mysqlCompName = "mysql" const backupPolicyTemplateName = "test-backup-policy-template" const backupPolicyName = "test-backup-policy" - const backupRemoteVolumeName = "backup-remote-volume" const backupRemotePVCName = "backup-remote-pvc" const backupName = "test-backup-job" @@ -129,7 +128,7 @@ var _ = Describe("MySQL data protection function", func() { SetBackupToolName(backupTool.Name). AddMatchLabels(constant.AppInstanceLabelKey, clusterKey.Name). SetTargetSecretName(component.GenerateConnCredential(clusterKey.Name)). - SetRemoteVolumePVC(backupRemoteVolumeName, backupRemotePVCName). + SetPVC(backupRemotePVCName). Create(&testCtx).GetObject() backupPolicyKey := client.ObjectKeyFromObject(backupPolicyObj) From 8add3da530cc1c9972939e994daf05d07472151e Mon Sep 17 00:00:00 2001 From: chantu Date: Mon, 17 Apr 2023 14:44:13 +0800 Subject: [PATCH 056/439] fix: mysql setup read topology from leader (#2590) --- deploy/apecloud-mysql/templates/scripts.yaml | 48 +++++++++---------- .../template/cluster_operations_template.cue | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index 8c46f63a2..a2c420171 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -11,15 +11,13 @@ data: followers=$KB_MYSQL_FOLLOWERS echo "leader=$leader" echo "followers=$followers" - sub_follower=`echo "$followers" | grep "$KB_POD_NAME"` echo "KB_POD_NAME=$KB_POD_NAME" - echo "sub_follower=$sub_follower" - if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" -o ! -z "$sub_follower" ]; then - echo "no need to call add" + idx=${KB_POD_NAME##*-} + host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) + echo "host=$host" + if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then + echo "no leader or self is leader, no need to call add." else - idx=${KB_POD_NAME##*-} - host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) - echo "host=$host" leader_idx=${leader##*-} leader_host=$(eval echo \$KB_MYSQL_"$leader_idx"_HOSTNAME) if [ ! -z $leader_host ]; then @@ -28,18 +26,24 @@ data: if [ ! -z $MYSQL_ROOT_PASSWORD ]; then password_flag="-p$MYSQL_ROOT_PASSWORD" fi - echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.add_learner('$host:13306');\" >> /tmp/setup_error.log 2>&1 " - mysql $host_flag -uroot $password_flag -e "call dbms_consensus.add_learner('$host:13306');" >> /tmp/setup_error.log 2>&1 - code=$? - echo "exit code: $code" - if [ $code -ne 0 ]; then - cat /tmp/setup_error.log - already_exists=`cat /tmp/setup_error.log | grep "Target node already exists"` - if [ -z "$already_exists" ]; then - exit $code - fi - fi - /scripts/upgrade-learner.sh & + echo "mysql $host_flag -uroot $password_flag -e \"select * from information_schema.wesql_cluster_global;\" 2>&1" + topology=`mysql $host_flag -uroot $password_flag -e "select * from information_schema.wesql_cluster_global;" 2>&1` + echo "topology=$topology" + in_topology=`echo $topology | grep "$KB_POD_NAME"` + if [ -z "$in_topology" ]; then + echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.add_learner('$host:13306');\" >> /tmp/setup_error.log 2>&1 " + mysql $host_flag -uroot $password_flag -e "call dbms_consensus.add_learner('$host:13306');" >> /tmp/setup_error.log 2>&1 + code=$? + echo "exit code: $code" + if [ $code -ne 0 ]; then + cat /tmp/setup_error.log + already_exists=`cat /tmp/setup_error.log | grep "Target node already exists"` + if [ -z "$already_exists" ]; then + exit $code + fi + fi + /scripts/upgrade-learner.sh & + fi fi cluster_info=""; for (( i=0; i< $KB_MYSQL_N; i++ )); do @@ -54,13 +58,9 @@ data: cluster_info="$cluster_info$host:13306#1N"; fi done; - idx=${KB_POD_NAME##*-} - echo $idx - host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) cluster_info="$cluster_info@$(($idx+1))"; echo "cluster_info=$cluster_info"; - mkdir -p /data/mysql/data; - mkdir -p /data/mysql/log; + mkdir -p /data/mysql/data /data/mysql/log chmod +777 -R /data/mysql; echo "KB_MYSQL_RECREATE=$KB_MYSQL_RECREATE" if [ "$KB_MYSQL_RECREATE" == "true" ]; then diff --git a/internal/cli/create/template/cluster_operations_template.cue b/internal/cli/create/template/cluster_operations_template.cue index 518814cc5..53240c4a3 100644 --- a/internal/cli/create/template/cluster_operations_template.cue +++ b/internal/cli/create/template/cluster_operations_template.cue @@ -24,7 +24,7 @@ options: { componentNames: [...string] cpu: string memory: string - class: string + class: string replicas: int storage: string vctNames: [...string] From a9270761bffe3e7d4f84b347983e59b07565471d Mon Sep 17 00:00:00 2001 From: shaojiang Date: Mon, 17 Apr 2023 15:00:08 +0800 Subject: [PATCH 057/439] feat: kbcli add pitr support commands. (#2456) --- apis/dataprotection/v1alpha1/backup_types.go | 43 ++++++ docs/user_docs/cli/kbcli_cluster_restore.md | 9 +- internal/cli/cluster/cluster.go | 31 +++++ internal/cli/cluster/types.go | 4 + internal/cli/cmd/cluster/create_test.go | 2 +- internal/cli/cmd/cluster/dataprotection.go | 125 ++++++++++++++++-- .../cli/cmd/cluster/dataprotection_test.go | 69 ++++++++-- internal/cli/cmd/cluster/describe.go | 73 +++++++++- internal/cli/cmd/cluster/describe_test.go | 38 ++++++ internal/cli/testing/fake.go | 3 + internal/cli/util/util.go | 12 +- 11 files changed, 382 insertions(+), 27 deletions(-) diff --git a/apis/dataprotection/v1alpha1/backup_types.go b/apis/dataprotection/v1alpha1/backup_types.go index 97ad98c0f..e79ec0f48 100644 --- a/apis/dataprotection/v1alpha1/backup_types.go +++ b/apis/dataprotection/v1alpha1/backup_types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( "fmt" + "sort" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -206,3 +207,45 @@ func (r *BackupSpec) Validate(backupPolicy *BackupPolicy) error { } return nil } + +// GetRecoverableTimeRange return the recoverable time range array +func GetRecoverableTimeRange(backups []Backup) []BackupLogStatus { + // filter backups with backupLog + backupsWithLog := make([]Backup, 0) + for _, b := range backups { + if b.Status.Phase == BackupCompleted && + b.Status.Manifests != nil && b.Status.Manifests.BackupLog != nil { + backupsWithLog = append(backupsWithLog, b) + } + } + if len(backupsWithLog) == 0 { + return nil + } + sort.Slice(backups, func(i, j int) bool { + if backups[i].Status.StartTimestamp == nil && backups[j].Status.StartTimestamp != nil { + return false + } + if backups[i].Status.StartTimestamp != nil && backups[j].Status.StartTimestamp == nil { + return true + } + if backups[i].Status.StartTimestamp.Equal(backups[j].Status.StartTimestamp) { + return backups[i].Name < backups[j].Name + } + return backups[i].Status.StartTimestamp.Before(backups[j].Status.StartTimestamp) + }) + result := make([]BackupLogStatus, 0) + start, end := backupsWithLog[0].Status.Manifests.BackupLog.StopTime, backupsWithLog[0].Status.Manifests.BackupLog.StopTime + + for i := 1; i < len(backupsWithLog); i++ { + b := backupsWithLog[i].Status.Manifests.BackupLog + if b.StartTime.Before(end) || b.StartTime.Equal(end) { + if b.StopTime.After(end.Time) { + end = b.StopTime + } + } else { + result = append(result, BackupLogStatus{StartTime: start, StopTime: end}) + start, end = b.StopTime, b.StopTime + } + } + return append(result, BackupLogStatus{StartTime: start, StopTime: end}) +} diff --git a/docs/user_docs/cli/kbcli_cluster_restore.md b/docs/user_docs/cli/kbcli_cluster_restore.md index 605bed030..d41594a45 100644 --- a/docs/user_docs/cli/kbcli_cluster_restore.md +++ b/docs/user_docs/cli/kbcli_cluster_restore.md @@ -13,13 +13,18 @@ kbcli cluster restore [flags] ``` # restore a new cluster from a backup kbcli cluster restore new-cluster-name --backup backup-name + + # restore a new cluster from point in time + kbcli cluster restore new-cluster-name --restore-to-time "Apr 13,2023 18:40:35 UTC+0800" --source-cluster mycluster ``` ### Options ``` - --backup string Backup name - -h, --help help for restore + --backup string Backup name + -h, --help help for restore + --restore-to-time string point in time recovery(PITR) + --source-cluster string source cluster name ``` ### Options inherited from parent commands diff --git a/internal/cli/cluster/cluster.go b/internal/cli/cluster/cluster.go index 1128ac0eb..241f73e7c 100644 --- a/internal/cli/cluster/cluster.go +++ b/internal/cli/cluster/cluster.go @@ -46,6 +46,7 @@ type GetOptions struct { WithSecret bool WithPod bool WithEvent bool + WithDataProtection bool } type ObjectsGetter struct { @@ -63,6 +64,24 @@ func NewClusterObjects() *ClusterObjects { } } +func listResources[T any](dynamic dynamic.Interface, gvr schema.GroupVersionResource, ns string, opts metav1.ListOptions, items *[]T) error { + if *items == nil { + *items = []T{} + } + obj, err := dynamic.Resource(gvr).Namespace(ns).List(context.TODO(), opts) + if err != nil { + return err + } + for _, i := range obj.Items { + var object T + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(i.Object, &object); err != nil { + return err + } + *items = append(*items, object) + } + return nil +} + // Get all kubernetes objects belonging to the database cluster func (o *ObjectsGetter) Get() (*ClusterObjects, error) { var err error @@ -189,6 +208,18 @@ func (o *ObjectsGetter) Get() (*ClusterObjects, error) { } } } + if o.WithDataProtection { + dplistOpts := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", + constant.AppInstanceLabelKey, o.Name), + } + if err := listResources(o.Dynamic, types.BackupPolicyGVR(), o.Namespace, dplistOpts, &objs.BackupPolicies); err != nil { + return nil, err + } + if err := listResources(o.Dynamic, types.BackupGVR(), o.Namespace, dplistOpts, &objs.Backups); err != nil { + return nil, err + } + } return objs, nil } diff --git a/internal/cli/cluster/types.go b/internal/cli/cluster/types.go index 9c9f17dc3..15b536c83 100644 --- a/internal/cli/cluster/types.go +++ b/internal/cli/cluster/types.go @@ -20,6 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" ) type ClusterObjects struct { @@ -34,6 +35,9 @@ type ClusterObjects struct { Nodes []*corev1.Node ConfigMaps *corev1.ConfigMapList Events *corev1.EventList + + BackupPolicies []dpv1alpha1.BackupPolicy + Backups []dpv1alpha1.Backup } type ClusterInfo struct { diff --git a/internal/cli/cmd/cluster/create_test.go b/internal/cli/cmd/cluster/create_test.go index 28f017683..ea97ed1f4 100644 --- a/internal/cli/cmd/cluster/create_test.go +++ b/internal/cli/cmd/cluster/create_test.go @@ -365,7 +365,7 @@ var _ = Describe("create", func() { Expect(setBackup(o, components).Error()).Should(ContainSubstring("is not completed")) By("test backup is completed") - mockBackupInfo(dynamic, backupName, clusterName) + mockBackupInfo(dynamic, backupName, clusterName, nil) Expect(setBackup(o, components)).Should(Succeed()) }) }) diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 343610b2a..d3fa60944 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -87,6 +87,9 @@ var ( createRestoreExample = templates.Examples(` # restore a new cluster from a backup kbcli cluster restore new-cluster-name --backup backup-name + + # restore a new cluster from point in time + kbcli cluster restore new-cluster-name --restore-to-time "Apr 13,2023 18:40:35 UTC+0800" --source-cluster mycluster `) ) @@ -394,6 +397,12 @@ func completeForDeleteBackup(o *delete.DeleteOptions, args []string) error { type CreateRestoreOptions struct { // backup name to restore in creation Backup string `json:"backup,omitempty"` + + // point in time recovery args + RestoreTime *time.Time `json:"restoreTime,omitempty"` + RestoreTimeStr string `json:"restoreTimeStr,omitempty"` + SourceCluster string `json:"sourceCluster,omitempty"` + create.BaseOptions } @@ -415,7 +424,16 @@ func (o *CreateRestoreOptions) getClusterObject(backup *dataprotectionv1alpha1.B } func (o *CreateRestoreOptions) Run() error { - // get backup job + if o.Backup != "" { + return o.runRestoreFromBackup() + } else if o.RestoreTime != nil { + return o.runPITR() + } + return nil +} + +func (o *CreateRestoreOptions) runRestoreFromBackup() error { + // get backup backup := &dataprotectionv1alpha1.Backup{} if err := cluster.GetK8SClientObject(o.Dynamic, backup, types.BackupGVR(), o.Namespace, o.Backup); err != nil { return err @@ -427,21 +445,25 @@ func (o *CreateRestoreOptions) Run() error { return errors.Errorf(`missing source cluster in backup "%s", "app.kubernetes.io/instance" is empty in labels.`, o.Backup) } // get the cluster object and set the annotation for restore - cluster, err := o.getClusterObject(backup) + clusterObj, err := o.getClusterObject(backup) if err != nil { return err } - restoreAnnotation, err := getRestoreFromBackupAnnotation(backup, len(cluster.Spec.ComponentSpecs), cluster.Spec.ComponentSpecs[0].Name) + restoreAnnotation, err := getRestoreFromBackupAnnotation(backup, len(clusterObj.Spec.ComponentSpecs), clusterObj.Spec.ComponentSpecs[0].Name) if err != nil { return err } - cluster.Status = appsv1alpha1.ClusterStatus{} - cluster.ObjectMeta = metav1.ObjectMeta{ - Namespace: cluster.Namespace, + clusterObj.ObjectMeta = metav1.ObjectMeta{ + Namespace: clusterObj.Namespace, Name: o.Name, Annotations: map[string]string{constant.RestoreFromBackUpAnnotationKey: restoreAnnotation}, } + return o.createCluster(clusterObj) +} + +func (o *CreateRestoreOptions) createCluster(cluster *appsv1alpha1.Cluster) error { clusterGVR := types.ClusterGVR() + cluster.Status = appsv1alpha1.ClusterStatus{} cluster.TypeMeta = metav1.TypeMeta{ Kind: types.KindCluster, APIVersion: clusterGVR.Group + "/" + clusterGVR.Version, @@ -461,7 +483,93 @@ func (o *CreateRestoreOptions) Run() error { return nil } +func (o *CreateRestoreOptions) runPITR() error { + objs, err := o.Dynamic.Resource(types.BackupGVR()).Namespace(o.Namespace). + List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", + constant.AppInstanceLabelKey, o.SourceCluster), + }) + if err != nil { + return err + } + backup := &dataprotectionv1alpha1.Backup{} + + // no need check items len because it is validated by o.validateRestoreTime(). + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(objs.Items[0].Object, backup); err != nil { + return err + } + // TODO: use opsRequest to create cluster. + // get the cluster object and set the annotation for restore + clusterObj, err := o.getClusterObject(backup) + if err != nil { + return err + } + clusterObj.ObjectMeta = metav1.ObjectMeta{ + Namespace: clusterObj.Namespace, + Name: o.Name, + Annotations: map[string]string{ + // TODO: use constant annotation key + "kubeblocks.io/restore-from-time": o.RestoreTime.Format(time.RFC3339), + "kubeblocks.io/restore-from-source-cluster": o.SourceCluster, + }, + } + return o.createCluster(clusterObj) +} + +func isTimeInRange(t time.Time, start time.Time, end time.Time) bool { + return !t.Before(start) && !t.After(end) +} + +func (o *CreateRestoreOptions) validateRestoreTime() error { + if o.RestoreTimeStr == "" && o.SourceCluster == "" { + return nil + } + if o.RestoreTimeStr == "" && o.SourceCluster == "" { + return fmt.Errorf("--source-cluster must be specified if specified --restore-to-time") + } + restoreTime, err := util.TimeParse(o.RestoreTimeStr, time.Second) + if err != nil { + return err + } + o.RestoreTime = &restoreTime + objs, err := o.Dynamic.Resource(types.BackupGVR()).Namespace(o.Namespace). + List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", + constant.AppInstanceLabelKey, o.SourceCluster), + }) + if err != nil { + return err + } + backups := make([]dataprotectionv1alpha1.Backup, 0) + for _, i := range objs.Items { + obj := dataprotectionv1alpha1.Backup{} + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(i.Object, &obj); err != nil { + return err + } + if obj.Status.Phase != dataprotectionv1alpha1.BackupCompleted || + obj.Status.Manifests == nil || obj.Status.Manifests.BackupLog == nil { + continue + } + backups = append(backups, obj) + } + recoverableTime := dataprotectionv1alpha1.GetRecoverableTimeRange(backups) + for _, i := range recoverableTime { + if isTimeInRange(restoreTime, i.StartTime.Time, i.StopTime.Time) { + return nil + } + } + return fmt.Errorf("restore-to-time is out of time range, you can view the recoverable time: \n"+ + "\tkbcli cluster describe %s -n %s", o.SourceCluster, o.Namespace) +} + func (o *CreateRestoreOptions) Validate() error { + if o.Backup == "" && o.RestoreTimeStr == "" { + return fmt.Errorf("must be specified one of the --backup or --restore-to-time") + } + if err := o.validateRestoreTime(); err != nil { + return err + } + if o.Name == "" { name, err := generateClusterName(o.Dynamic, o.Namespace) if err != nil { @@ -472,9 +580,6 @@ func (o *CreateRestoreOptions) Validate() error { } o.Name = name } - if o.Backup == "" { - return fmt.Errorf("backup name should be specified by --backup") - } return nil } @@ -498,6 +603,8 @@ func NewCreateRestoreCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) }, } cmd.Flags().StringVar(&o.Backup, "backup", "", "Backup name") + cmd.Flags().StringVar(&o.RestoreTimeStr, "restore-to-time", "", "point in time recovery(PITR)") + cmd.Flags().StringVar(&o.SourceCluster, "source-cluster", "", "source cluster name") return cmd } diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index 5cc3b611f..8239d2c31 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -242,7 +242,7 @@ var _ = Describe("DataProtection", func() { By("restore new cluster from source cluster which is not deleted") // mock backup is ok - mockBackupInfo(tf.FakeDynamicClient, backupName, clusterName) + mockBackupInfo(tf.FakeDynamicClient, backupName, clusterName, nil) cmdRestore := NewCreateRestoreCmd(tf, streams) Expect(cmdRestore != nil).To(BeTrue()) _ = cmdRestore.Flags().Set("backup", backupName) @@ -250,7 +250,7 @@ var _ = Describe("DataProtection", func() { By("restore new cluster from source cluster which is deleted") // mock cluster is not lived in kubernetes - mockBackupInfo(tf.FakeDynamicClient, backupName, "deleted-cluster") + mockBackupInfo(tf.FakeDynamicClient, backupName, "deleted-cluster", nil) cmdRestore.Run(nil, []string{newClusterName + "1"}) By("run restore cmd with cluster spec.affinity=nil") @@ -259,21 +259,72 @@ var _ = Describe("DataProtection", func() { k8sapitypes.MergePatchType, patchCluster, metav1.PatchOptions{}) cmdRestore.Run(nil, []string{newClusterName + "-with-nil-affinity"}) }) + + It("restore-to-time", func() { + timestamp := time.Now().Format("20060102150405") + backupName := "backup-test-" + timestamp + clusterName := "source-cluster-" + timestamp + secrets := testing.FakeSecrets(testing.Namespace, clusterName) + clusterDef := testing.FakeClusterDef() + cluster := testing.FakeCluster(clusterName, testing.Namespace) + clusterDefLabel := map[string]string{ + constant.ClusterDefLabelKey: clusterDef.Name, + } + cluster.SetLabels(clusterDefLabel) + backupPolicy := testing.FakeBackupPolicy("backPolicy", cluster.Name) + backup := testing.FakeBackup("backup-base") + + pods := testing.FakePods(1, testing.Namespace, clusterName) + tf.FakeDynamicClient = fake.NewSimpleDynamicClient( + scheme.Scheme, &secrets.Items[0], &pods.Items[0], cluster, backupPolicy, backup) + tf.Client = &clientfake.RESTClient{} + // create backup + cmd := NewCreateBackupCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + _ = cmd.Flags().Set("backup-type", "snapshot") + _ = cmd.Flags().Set("backup-name", backupName) + cmd.Run(nil, []string{clusterName}) + + By("restore new cluster from source cluster which is not deleted") + // mock backup is ok + now := metav1.Now() + baseManifests := map[string]any{ + "backupLog": map[string]any{ + "startTime": now.Add(-time.Minute).Format(time.RFC3339), + "stopTime": now.Add(-time.Second).Format(time.RFC3339), + }, + } + mockBackupInfo(tf.FakeDynamicClient, backup.Name, clusterName, baseManifests) + + manifests := map[string]any{ + "backupLog": map[string]any{ + "startTime": now.Add(-time.Minute).Format(time.RFC3339), + "stopTime": now.Add(time.Minute).Format(time.RFC3339), + }, + } + mockBackupInfo(tf.FakeDynamicClient, backupName, clusterName, manifests) + cmdRestore := NewCreateRestoreCmd(tf, streams) + Expect(cmdRestore != nil).To(BeTrue()) + _ = cmdRestore.Flags().Set("restore-to-time", util.TimeFormatWithDuration(&now, time.Second)) + _ = cmdRestore.Flags().Set("source-cluster", clusterName) + cmdRestore.Run(nil, []string{}) + }) }) -func mockBackupInfo(dynamic dynamic.Interface, backupName, clusterName string) { +func mockBackupInfo(dynamic dynamic.Interface, backupName, clusterName string, manifests map[string]any) { clusterString := fmt.Sprintf(`{"metadata":{"name":"deleted-cluster","namespace":"%s"},"spec":{"clusterDefinitionRef":"apecloud-mysql","clusterVersionRef":"ac-mysql-8.0.30","componentSpecs":[{"name":"mysql","componentDefRef":"mysql","replicas":1}]}}`, testing.Namespace) backupStatus := &unstructured.Unstructured{ - Object: map[string]interface{}{ - "status": map[string]interface{}{ - "phase": "Completed", + Object: map[string]any{ + "status": map[string]any{ + "phase": "Completed", + "manifests": manifests, }, - "metadata": map[string]interface{}{ + "metadata": map[string]any{ "name": backupName, - "annotations": map[string]interface{}{ + "annotations": map[string]any{ constant.ClusterSnapshotAnnotationKey: clusterString, }, - "labels": map[string]interface{}{ + "labels": map[string]any{ constant.AppInstanceLabelKey: clusterName, constant.KBAppComponentLabelKey: "test", }, diff --git a/internal/cli/cmd/cluster/describe.go b/internal/cli/cmd/cluster/describe.go index 80669a7ab..f2116947a 100644 --- a/internal/cli/cmd/cluster/describe.go +++ b/internal/cli/cmd/cluster/describe.go @@ -20,9 +20,11 @@ import ( "fmt" "io" "strings" + "time" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" @@ -31,6 +33,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/cluster" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" @@ -126,11 +129,12 @@ func (o *describeOptions) describeCluster(name string) error { Name: name, Namespace: o.namespace, GetOptions: cluster.GetOptions{ - WithClusterDef: true, - WithService: true, - WithPod: true, - WithEvent: true, - WithPVC: true, + WithClusterDef: true, + WithService: true, + WithPod: true, + WithEvent: true, + WithPVC: true, + WithDataProtection: true, }, } @@ -155,6 +159,9 @@ func (o *describeOptions) describeCluster(name string) error { // images showImages(comps, o.Out) + // data protection info + showDataProtection(o.BackupPolicies, o.Backups, o.Out) + // events showEvents(o.Events, o.Cluster.Name, o.Cluster.Namespace, o.Out) fmt.Fprintln(o.Out) @@ -235,3 +242,59 @@ func showEndpoints(c *appsv1alpha1.Cluster, svcList *corev1.ServiceList, out io. } tbl.Print() } + +func showDataProtection(backupPolicies []dpv1alpha1.BackupPolicy, backups []dpv1alpha1.Backup, out io.Writer) { + if len(backupPolicies) == 0 { + return + } + tbl := newTbl(out, "\nData Protection:", "AUTO-BACKUP", "BACKUP-SCHEDULE", "TYPE", "BACKUP-TTL", "LAST-SCHEDULE", "RECOVERABLE-TIME") + for _, policy := range backupPolicies { + if policy.Status.Phase != dpv1alpha1.PolicyAvailable { + continue + } + ttlString := printer.NoneString + backupSchedule := printer.NoneString + backupType := printer.NoneString + scheduleEnable := "Disabled" + if policy.Spec.Schedule.BaseBackup != nil { + if policy.Spec.Schedule.BaseBackup.Enable { + scheduleEnable = "Enabled" + } + backupSchedule = policy.Spec.Schedule.BaseBackup.CronExpression + backupType = string(policy.Spec.Schedule.BaseBackup.Type) + + } + if policy.Spec.TTL != nil { + ttlString = *policy.Spec.TTL + } + lastScheduleTime := printer.NoneString + if policy.Status.LastScheduleTime != nil { + lastScheduleTime = util.TimeFormat(policy.Status.LastScheduleTime) + } + + tbl.AddRow(scheduleEnable, backupSchedule, backupType, ttlString, lastScheduleTime, getBackupRecoverableTime(backups)) + } + tbl.Print() +} + +// getBackupRecoverableTime return the recoverable time range string +func getBackupRecoverableTime(backups []dpv1alpha1.Backup) string { + recoverabelTime := dpv1alpha1.GetRecoverableTimeRange(backups) + var result string + for _, i := range recoverabelTime { + result = addTimeRange(result, i.StartTime, i.StopTime) + } + if result == "" { + return printer.NoneString + } + return result +} + +func addTimeRange(result string, start, end *metav1.Time) string { + if result != "" { + result += ", " + } + result += fmt.Sprintf("%s ~ %s", util.TimeFormatWithDuration(start, time.Second), + util.TimeFormatWithDuration(end, time.Second)) + return result +} diff --git a/internal/cli/cmd/cluster/describe_test.go b/internal/cli/cmd/cluster/describe_test.go index 95cc3dacd..a6ff75c5f 100644 --- a/internal/cli/cmd/cluster/describe_test.go +++ b/internal/cli/cmd/cluster/describe_test.go @@ -20,10 +20,12 @@ import ( "bytes" "net/http" "strings" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -32,6 +34,7 @@ import ( clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" ) @@ -111,4 +114,39 @@ var _ = Describe("Expose", func() { secondEvent := strs[4] Expect(strings.Compare(firstEvent, secondEvent) < 0).Should(BeTrue()) }) + + It("showDataProtections", func() { + out := &bytes.Buffer{} + fakeBackupPolicies := []dpv1alpha1.BackupPolicy{ + *testing.FakeBackupPolicy("backup-policy-test", "test-cluster"), + } + fakeBackups := []dpv1alpha1.Backup{ + *testing.FakeBackup("backup-test"), + *testing.FakeBackup("backup-test2"), + } + now := metav1.Now() + fakeBackups[0].Status = dpv1alpha1.BackupStatus{ + Phase: dpv1alpha1.BackupCompleted, + Manifests: &dpv1alpha1.ManifestsStatus{ + BackupLog: &dpv1alpha1.BackupLogStatus{ + StartTime: &now, + StopTime: &now, + }, + }, + } + after := metav1.Time{Time: now.Add(time.Hour)} + fakeBackups[1].Status = dpv1alpha1.BackupStatus{ + Phase: dpv1alpha1.BackupCompleted, + Manifests: &dpv1alpha1.ManifestsStatus{ + BackupLog: &dpv1alpha1.BackupLogStatus{ + StartTime: &now, + StopTime: &after, + }, + }, + } + showDataProtection(fakeBackupPolicies, fakeBackups, out) + strs := strings.Split(out.String(), "\n") + + Expect(strs).ShouldNot(BeEmpty()) + }) }) diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index e2f2abcbb..0e78c2976 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -300,6 +300,9 @@ func FakeBackupPolicy(backupPolicyName, clusterName string) *dpv1alpha1.BackupPo constant.DefaultBackupPolicyAnnotationKey: "true", }, }, + Status: dpv1alpha1.BackupPolicyStatus{ + Phase: dpv1alpha1.PolicyAvailable, + }, } return template } diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 6fdc7f55c..1d060d61d 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -318,7 +318,7 @@ func TimeTimeFormat(t time.Time) string { return t.Format(layout) } -func TimeTimeFormatWithDuration(t time.Time, precision time.Duration) string { +func timeLayout(precision time.Duration) string { layout := "Jan 02,2006 15:04 UTC-0700" switch precision { case time.Second: @@ -326,9 +326,19 @@ func TimeTimeFormatWithDuration(t time.Time, precision time.Duration) string { case time.Millisecond: layout = "Jan 02,2006 15:04:05.000 UTC-0700" } + return layout +} + +func TimeTimeFormatWithDuration(t time.Time, precision time.Duration) string { + layout := timeLayout(precision) return t.Format(layout) } +func TimeParse(t string, precision time.Duration) (time.Time, error) { + layout := timeLayout(precision) + return time.Parse(layout, t) +} + // GetHumanReadableDuration returns a succinct representation of the provided startTime and endTime // with limited precision for consumption by humans. func GetHumanReadableDuration(startTime metav1.Time, endTime metav1.Time) string { From 91edc158e8ad0736d3b2aa9652cc9840846c5d3f Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 17 Apr 2023 16:33:42 +0800 Subject: [PATCH 058/439] chore: adjust kbcli example-commands prompt (#2631) Co-authored-by: sophon-zt --- docs/user_docs/cli/kbcli_cluster_configure.md | 3 ++- docs/user_docs/cli/kbcli_cluster_explain-config.md | 2 +- internal/cli/cmd/cluster/cluster.go | 7 ++++++- internal/cli/cmd/cluster/config.go | 2 +- internal/cli/cmd/cluster/config_ops.go | 3 ++- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/user_docs/cli/kbcli_cluster_configure.md b/docs/user_docs/cli/kbcli_cluster_configure.md index 009e132d1..23c046b34 100644 --- a/docs/user_docs/cli/kbcli_cluster_configure.md +++ b/docs/user_docs/cli/kbcli_cluster_configure.md @@ -14,8 +14,9 @@ kbcli cluster configure [flags] # update component params kbcli cluster configure --component= --config-spec= --config-file= --set max_connections=1000,general_log=OFF + # if only one component, and one config spec, and one config file, simplify the use of configure. e.g: # update mysql max_connections, cluster name is mycluster - kbcli cluster configure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=2000 + kbcli cluster configure mycluster --set max_connections=2000 ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_explain-config.md b/docs/user_docs/cli/kbcli_cluster_explain-config.md index 8abe9394f..82988a4d3 100644 --- a/docs/user_docs/cli/kbcli_cluster_explain-config.md +++ b/docs/user_docs/cli/kbcli_cluster_explain-config.md @@ -21,7 +21,7 @@ kbcli cluster explain-config [flags] kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl --trunc-document=false --trunc-enum=false # describe a specified parameters, e.g. cluster name is mycluster - kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl --param=sql_mode + kbcli cluster explain-config mycluster --param=sql_mode ``` ### Options diff --git a/internal/cli/cmd/cluster/cluster.go b/internal/cli/cmd/cluster/cluster.go index fc6cc4824..cf0e8f7ff 100644 --- a/internal/cli/cmd/cluster/cluster.go +++ b/internal/cli/cmd/cluster/cluster.go @@ -68,9 +68,14 @@ func NewClusterCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr NewDescribeOpsCmd(f, streams), NewListOpsCmd(f, streams), NewDeleteOpsCmd(f, streams), + NewExposeCmd(f, streams), + }, + }, + { + Message: "Cluster Configuration Operation Commands:", + Commands: []*cobra.Command{ NewReconfigureCmd(f, streams), NewEditConfigureCmd(f, streams), - NewExposeCmd(f, streams), NewDescribeReconfigureCmd(f, streams), NewExplainReconfigureCmd(f, streams), NewDiffConfigureCmd(f, streams), diff --git a/internal/cli/cmd/cluster/config.go b/internal/cli/cmd/cluster/config.go index 4760b59f9..67c5c3050 100644 --- a/internal/cli/cmd/cluster/config.go +++ b/internal/cli/cmd/cluster/config.go @@ -110,7 +110,7 @@ var ( kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl --trunc-document=false --trunc-enum=false # describe a specified parameters, e.g. cluster name is mycluster - kbcli cluster explain-config mycluster --component=mysql --config-specs=mysql-3node-tpl --param=sql_mode`) + kbcli cluster explain-config mycluster --param=sql_mode`) diffConfigureExample = templates.Examples(` # compare config files kbcli cluster diff-config opsrequest1 opsrequest2`) diff --git a/internal/cli/cmd/cluster/config_ops.go b/internal/cli/cmd/cluster/config_ops.go index ff48df16c..ccae51428 100644 --- a/internal/cli/cmd/cluster/config_ops.go +++ b/internal/cli/cmd/cluster/config_ops.go @@ -52,8 +52,9 @@ var ( # update component params kbcli cluster configure --component= --config-spec= --config-file= --set max_connections=1000,general_log=OFF + # if only one component, and one config spec, and one config file, simplify the use of configure. e.g: # update mysql max_connections, cluster name is mycluster - kbcli cluster configure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=2000 + kbcli cluster configure mycluster --set max_connections=2000 `) ) From 9195bb6e3bf8515707ac93210889af5faa020db3 Mon Sep 17 00:00:00 2001 From: diankuizhao Date: Mon, 17 Apr 2023 18:18:24 +0800 Subject: [PATCH 059/439] fix: adjust vtgate healthcheck options (#2650) --- .../apecloud-mysql-scale/templates/clusterdefinition.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml b/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml index e853f5973..34e70aba7 100644 --- a/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml @@ -587,10 +587,10 @@ spec: $TOPOLOGY_FLAGS \ --alsologtostderr \ --gateway_initial_tablet_timeout 30s \ - --healthcheck_timeout 1s \ + --healthcheck_timeout 2s \ --srv_topo_timeout 1s \ - --grpc_keepalive_time 1s \ - --grpc_keepalive_timeout 2s \ + --grpc_keepalive_time 10s \ + --grpc_keepalive_timeout 10s \ --log_dir $VTDATAROOT \ --log_queries_to_file $VTDATAROOT/vtgate_querylog.txt \ --port $web_port \ From 08e310ab5ada6e1a729f42440ce84efb6ffd03af Mon Sep 17 00:00:00 2001 From: wangyelei Date: Mon, 17 Apr 2023 18:28:28 +0800 Subject: [PATCH 060/439] chore: fix not found kbcli_kubeblocks_config.md (#2647) --- docs/user_docs/cli/kbcli_kubeblocks_config.md | 79 +++++++++++++++++++ .../kubeblocks-for-gptplugin/Installation.md | 2 +- 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 docs/user_docs/cli/kbcli_kubeblocks_config.md diff --git a/docs/user_docs/cli/kbcli_kubeblocks_config.md b/docs/user_docs/cli/kbcli_kubeblocks_config.md new file mode 100644 index 000000000..5f47f30f4 --- /dev/null +++ b/docs/user_docs/cli/kbcli_kubeblocks_config.md @@ -0,0 +1,79 @@ +--- +title: kbcli kubeblocks config +--- + +KubeBlocks config. + +``` +kbcli kubeblocks config [flags] +``` + +### Examples + +``` + # Enable the snapshot-controller and volume snapshot, to support snapshot backup. + kbcli kubeblocks config --set snapshot-controller.enabled=true + + Options Parameters: + # If you have already installed a snapshot-controller, only enable the snapshot backup feature + dataProtection.enableVolumeSnapshot=true + + # the global pvc name which persistent volume claim to store the backup data. + # will replace the pvc name when it is empty in the backup policy. + dataProtection.backupPVCName=backup-data + + # the init capacity of pvc for creating the pvc, e.g. 10Gi. + # will replace the init capacity when it is empty in the backup policy. + dataProtection.backupPVCInitCapacity=100Gi + + # the pvc storage class name. + # will replace the storageClassName when it is nil in the backup policy. + dataProtection.backupPVCStorageClassName=csi-s3 + + # the pvc create policy. + # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. + # otherwise, using "Never" policy. only affect the backupPolicy automatically created by Kubeblocks. + dataProtection.backupPVCCreatePolicy=Never +``` + +### Options + +``` + -h, --help help for config + --set stringArray Set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) + --set-file stringArray Set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2) + --set-json stringArray Set JSON values on the command line (can specify multiple or separate values with commas: key1=jsonval1,key2=jsonval2) + --set-string stringArray Set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) + -f, --values strings Specify values in a YAML file or a URL (can specify multiple) +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli kubeblocks](kbcli_kubeblocks.md) - KubeBlocks operation commands. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/kubeblocks-for-gptplugin/Installation.md b/docs/user_docs/kubeblocks-for-gptplugin/Installation.md index d6bb5e733..482ec19f3 100644 --- a/docs/user_docs/kubeblocks-for-gptplugin/Installation.md +++ b/docs/user_docs/kubeblocks-for-gptplugin/Installation.md @@ -6,7 +6,7 @@ The Plugin authorities from OpenAI, if none, you can join the [waiting list here An OPENAI_API_KEY for the Plugin to call OpenAI embedding APIs. ### Requirements from KubeBlocks -Kbcli & the lastest KubeBlocks edition are installed. +Kbcli & the latest KubeBlocks edition are installed. TODO: the installation part of kbcli & kb. ### Installation From 9a3d2b9f17eb0467de6a145db4faba57d38631f6 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Mon, 17 Apr 2023 18:51:22 +0800 Subject: [PATCH 061/439] fix: probe pg checkrole (#2638) --- cmd/probe/internal/binding/base.go | 1 - cmd/probe/internal/binding/postgres/postgres.go | 5 ++--- .../middleware/http/probe/checks_middleware.go | 5 ++++- .../templates/tests/test-connection.yaml | 15 --------------- 4 files changed, 6 insertions(+), 20 deletions(-) delete mode 100644 deploy/mongodb-cluster/templates/tests/test-connection.yaml diff --git a/cmd/probe/internal/binding/base.go b/cmd/probe/internal/binding/base.go index a61e84a0c..807c524c8 100644 --- a/cmd/probe/internal/binding/base.go +++ b/cmd/probe/internal/binding/base.go @@ -137,7 +137,6 @@ func (ops *BaseOperations) Invoke(ctx context.Context, req *bindings.InvokeReque return nil, errors.Errorf("invoke request required") } - ops.Logger.Debugf("request operation: %v", req.Operation) startTime := time.Now() resp := &bindings.InvokeResponse{ Metadata: map[string]string{ diff --git a/cmd/probe/internal/binding/postgres/postgres.go b/cmd/probe/internal/binding/postgres/postgres.go index 750c9a5a2..e0e37a5a8 100644 --- a/cmd/probe/internal/binding/postgres/postgres.go +++ b/cmd/probe/internal/binding/postgres/postgres.go @@ -212,11 +212,10 @@ func (pgOps *PostgresOperations) GetRole(ctx context.Context, request *bindings. return "", err } } - pgOps.OriRole = PRIMARY if isRecovery { - pgOps.OriRole = SECONDARY + return SECONDARY, nil } - return pgOps.OriRole, nil + return PRIMARY, nil } func (pgOps *PostgresOperations) ExecOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { diff --git a/cmd/probe/internal/middleware/http/probe/checks_middleware.go b/cmd/probe/internal/middleware/http/probe/checks_middleware.go index 940b913ae..2b599765c 100644 --- a/cmd/probe/internal/middleware/http/probe/checks_middleware.go +++ b/cmd/probe/internal/middleware/http/probe/checks_middleware.go @@ -96,10 +96,13 @@ func (m *Middleware) GetHandler(metadata middleware.Metadata) (func(next fasthtt code := ctx.Response.Header.Peek(statusCodeHeader) statusCode, err := strconv.Atoi(string(code)) if err == nil { + // header has a statusCodeHeader ctx.Response.Header.SetStatusCode(statusCode) m.logger.Infof("response abnormal: %v", ctx.Response.String()) + } else { + // header has no statusCodeHeader + m.logger.Infof("response: %v", ctx.Response.String()) } - m.logger.Infof("response: %v", ctx.Response.String()) } }, nil } diff --git a/deploy/mongodb-cluster/templates/tests/test-connection.yaml b/deploy/mongodb-cluster/templates/tests/test-connection.yaml deleted file mode 100644 index 1259080a5..000000000 --- a/deploy/mongodb-cluster/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "mongodb-cluster.fullname" . }}-test-connection" - labels: - {{- include "mongodb-cluster.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "mongodb-cluster.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never From 2804df1d08de99b51284a7ccd4c8271a28a89cad Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Mon, 17 Apr 2023 19:07:11 +0800 Subject: [PATCH 062/439] chore: enhance class validating (#2625) --- .../componentresourceconstraint_types.go | 13 +++ .../componentresourceconstraint_types_test.go | 29 ++++-- controllers/apps/class_controller.go | 25 ++++++ controllers/apps/class_controller_test.go | 21 +++-- internal/class/class_utils.go | 7 +- internal/cli/cmd/class/create.go | 29 +++--- internal/cli/cmd/class/create_test.go | 10 +-- internal/cli/cmd/migration/base.go | 16 ++++ internal/cli/cmd/migration/base_test.go | 16 ++++ internal/cli/cmd/migration/cmd_builder.go | 16 ++++ .../cli/cmd/migration/cmd_builder_test.go | 16 ++++ internal/cli/cmd/migration/create.go | 16 ++++ internal/cli/cmd/migration/create_test.go | 16 ++++ internal/cli/cmd/migration/describe.go | 16 ++++ internal/cli/cmd/migration/describe_test.go | 16 ++++ internal/cli/cmd/migration/examples.go | 16 ++++ internal/cli/cmd/migration/list.go | 16 ++++ internal/cli/cmd/migration/list_test.go | 16 ++++ internal/cli/cmd/migration/logs.go | 16 ++++ internal/cli/cmd/migration/logs_test.go | 16 ++++ internal/cli/cmd/migration/suite_test.go | 16 ++++ internal/cli/cmd/migration/templates.go | 16 ++++ internal/cli/cmd/migration/templates_test.go | 16 ++++ internal/cli/cmd/migration/terminate.go | 16 ++++ internal/cli/cmd/migration/terminate_test.go | 16 ++++ .../cli/testing/testdata/custom_class.yaml | 10 +-- .../apps/componentclassdefinition_factory.go | 43 ++++----- .../componentresourceconstraint_factory.go | 88 +++++++++++++++++++ internal/testutil/apps/constant.go | 38 ++++++-- test/testdata/cue_testdata/mongod.cue | 14 +++ 30 files changed, 544 insertions(+), 71 deletions(-) create mode 100644 internal/testutil/apps/componentresourceconstraint_factory.go diff --git a/apis/apps/v1alpha1/componentresourceconstraint_types.go b/apis/apps/v1alpha1/componentresourceconstraint_types.go index d36fda7cc..2c80e071d 100644 --- a/apis/apps/v1alpha1/componentresourceconstraint_types.go +++ b/apis/apps/v1alpha1/componentresourceconstraint_types.go @@ -186,3 +186,16 @@ func (c *ComponentResourceConstraint) FindMatchingConstraints(r *corev1.Resource } return constraints } + +func (c *ComponentResourceConstraint) MatchClass(class *ComponentClassInstance) bool { + request := corev1.ResourceList{ + corev1.ResourceCPU: class.CPU, + corev1.ResourceMemory: class.Memory, + } + resource := &corev1.ResourceRequirements{ + Limits: request, + Requests: request, + } + constraints := c.FindMatchingConstraints(resource) + return len(constraints) > 0 +} diff --git a/apis/apps/v1alpha1/componentresourceconstraint_types_test.go b/apis/apps/v1alpha1/componentresourceconstraint_types_test.go index b767a3243..d70a66c25 100644 --- a/apis/apps/v1alpha1/componentresourceconstraint_types_test.go +++ b/apis/apps/v1alpha1/componentresourceconstraint_types_test.go @@ -52,12 +52,17 @@ spec: maxPerCPU: 8Gi ` -func TestResourceConstraints_ValidateResourceRequirements(t *testing.T) { - var cf ComponentResourceConstraint - err := yaml.Unmarshal([]byte(resourceConstraints), &cf) - if err != nil { +var ( + cf ComponentResourceConstraint +) + +func init() { + if err := yaml.Unmarshal([]byte(resourceConstraints), &cf); err != nil { panic("Failed to unmarshal resource constraints: %v" + err.Error()) } +} + +func TestResourceConstraints(t *testing.T) { cases := []struct { cpu string memory string @@ -71,12 +76,24 @@ func TestResourceConstraints_ValidateResourceRequirements(t *testing.T) { } for _, item := range cases { + var ( + cpu = resource.MustParse(item.cpu) + memory = resource.MustParse(item.memory) + ) requirements := &corev1.ResourceRequirements{ Requests: map[corev1.ResourceName]resource.Quantity{ - corev1.ResourceCPU: resource.MustParse(item.cpu), - corev1.ResourceMemory: resource.MustParse(item.memory), + corev1.ResourceCPU: cpu, + corev1.ResourceMemory: memory, }, } assert.Equal(t, item.expect, len(cf.FindMatchingConstraints(requirements)) > 0) + + class := &ComponentClassInstance{ + ComponentClass: ComponentClass{ + CPU: cpu, + Memory: memory, + }, + } + assert.Equal(t, item.expect, cf.MatchClass(class)) } } diff --git a/controllers/apps/class_controller.go b/controllers/apps/class_controller.go index 7a0623a06..833e7046f 100644 --- a/controllers/apps/class_controller.go +++ b/controllers/apps/class_controller.go @@ -18,6 +18,7 @@ package apps import ( "context" + "fmt" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" @@ -28,6 +29,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/class" + "github.com/apecloud/kubeblocks/internal/cli/types" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -54,6 +56,22 @@ func (r *ComponentClassReconciler) Reconcile(ctx context.Context, req reconcile. return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } + ml := []client.ListOption{ + client.HasLabels{types.ResourceConstraintProviderLabelKey}, + } + constraintsList := &appsv1alpha1.ComponentResourceConstraintList{} + if err := r.Client.List(reqCtx.Ctx, constraintsList, ml...); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } + constraintsMap := make(map[string]appsv1alpha1.ComponentResourceConstraint) + for idx := range constraintsList.Items { + cf := constraintsList.Items[idx] + if _, ok := cf.GetLabels()[types.ResourceConstraintProviderLabelKey]; !ok { + continue + } + constraintsMap[cf.GetName()] = cf + } + res, err := intctrlutil.HandleCRDeletion(reqCtx, r, classDefinition, dbClusterFinalizerName, func() (*ctrl.Result, error) { // TODO validate if existing cluster reference classes being deleted return nil, nil @@ -74,6 +92,13 @@ func (r *ComponentClassReconciler) Reconcile(ctx context.Context, req reconcile. patch := client.MergeFrom(classDefinition.DeepCopy()) var classList []appsv1alpha1.ComponentClassInstance for _, v := range classInstances { + constraint, ok := constraintsMap[v.ResourceConstraintRef] + if !ok { + return intctrlutil.CheckedRequeueWithError(nil, reqCtx.Log, fmt.Sprintf("resource constraint %s not found", v.ResourceConstraintRef)) + } + if !constraint.MatchClass(v) { + return intctrlutil.CheckedRequeueWithError(nil, reqCtx.Log, fmt.Sprintf("class %s does not conform to constraint %s", v.Name, v.ResourceConstraintRef)) + } classList = append(classList, *v) } classDefinition.Status.Classes = classList diff --git a/controllers/apps/class_controller_test.go b/controllers/apps/class_controller_test.go index b436ad0c6..5f4d5d0f8 100644 --- a/controllers/apps/class_controller_test.go +++ b/controllers/apps/class_controller_test.go @@ -52,14 +52,23 @@ var _ = Describe("", func() { It("Class should exist in status", func() { var ( clsName = "test" + class = v1alpha1.ComponentClass{ + Name: clsName, + CPU: resource.MustParse("1"), + Memory: resource.MustParse("1Gi"), + } ) - class := v1alpha1.ComponentClass{ - Name: clsName, - CPU: resource.MustParse("1"), - Memory: resource.MustParse("2Gi"), - } + + constraint := testapps.NewComponentResourceConstraintFactory(testapps.DefaultGeneralResourceConstraintName). + AddConstraints(testapps.ResourceConstraintNormal). + AddConstraints(testapps.ResourceConstraintSpecial). + Create(&testCtx).GetObject() + componentClassDefinition = testapps.NewComponentClassDefinitionFactory("custom", "apecloud-mysql", "mysql"). - AddClass(class).Create(&testCtx).GetObject() + AddClassGroup(constraint.Name). + AddClasses([]v1alpha1.ComponentClass{class}). + Create(&testCtx).GetObject() + key := client.ObjectKeyFromObject(componentClassDefinition) Eventually(testapps.CheckObj(&testCtx, key, func(g Gomega, pobj *v1alpha1.ComponentClassDefinition) { g.Expect(pobj.Status.Classes).ShouldNot(BeEmpty()) diff --git a/internal/class/class_utils.go b/internal/class/class_utils.go index f44e89f2e..f43f3ed47 100644 --- a/internal/class/class_utils.go +++ b/internal/class/class_utils.go @@ -89,6 +89,8 @@ func GetClasses(classDefinitionList v1alpha1.ComponentClassDefinitionList) (map[ return componentClasses, nil } + +// GetResourceConstraints get all resource constraints func GetResourceConstraints(dynamic dynamic.Interface) (map[string]*v1alpha1.ComponentResourceConstraint, error) { objs, err := dynamic.Resource(types.ComponentResourceConstraintGVR()).List(context.TODO(), metav1.ListOptions{ //LabelSelector: types.ResourceConstraintProviderLabelKey, @@ -102,7 +104,8 @@ func GetResourceConstraints(dynamic dynamic.Interface) (map[string]*v1alpha1.Com } result := make(map[string]*v1alpha1.ComponentResourceConstraint) - for _, cf := range constraintsList.Items { + for idx := range constraintsList.Items { + cf := constraintsList.Items[idx] if _, ok := cf.GetLabels()[types.ResourceConstraintProviderLabelKey]; !ok { continue } @@ -111,7 +114,7 @@ func GetResourceConstraints(dynamic dynamic.Interface) (map[string]*v1alpha1.Com return result, nil } -// ListClassesByClusterDefinition Get all classes, including kubeblocks default classes and user custom classes +// ListClassesByClusterDefinition get all classes, including kubeblocks default classes and user custom classes func ListClassesByClusterDefinition(client dynamic.Interface, cdName string) (map[string]map[string]*v1alpha1.ComponentClassInstance, error) { selector := fmt.Sprintf("%s=%s,%s", constant.ClusterDefLabelKey, cdName, types.ClassProviderLabelKey) objs, err := client.Resource(types.ComponentClassDefinitionGVR()).Namespace("").List(context.TODO(), metav1.ListOptions{ diff --git a/internal/cli/cmd/class/create.go b/internal/cli/cmd/class/create.go index 47d57f341..c0ad548fc 100644 --- a/internal/cli/cmd/class/create.go +++ b/internal/cli/cmd/class/create.go @@ -138,7 +138,7 @@ func (o *CreateOptions) run() error { } var ( - classNames []string + classInstances []*v1alpha1.ComponentClassInstance componentClassGroups []v1alpha1.ComponentClassGroup ) @@ -157,14 +157,8 @@ func (o *CreateOptions) run() error { if err != nil { return err } - for name, cls := range newClasses { - if _, ok = constraints[cls.ResourceConstraintRef]; !ok { - return fmt.Errorf("resource constraint %s is not found", cls.ResourceConstraintRef) - } - if _, ok = classes[name]; ok { - return fmt.Errorf("class name conflicted %s", name) - } - classNames = append(classNames, name) + for _, cls := range newClasses { + classInstances = append(classInstances, cls) } } else { if _, ok = classes[o.ClassName]; ok { @@ -187,7 +181,22 @@ func (o *CreateOptions) run() error { }, }, } - classNames = append(classNames, o.ClassName) + classInstances = append(classInstances, &v1alpha1.ComponentClassInstance{ComponentClass: *cls, ResourceConstraintRef: o.Constraint}) + } + + var classNames []string + for _, item := range classInstances { + constraint, ok := constraints[item.ResourceConstraintRef] + if !ok { + return fmt.Errorf("resource constraint %s is not found", item.ResourceConstraintRef) + } + if _, ok = classes[item.Name]; ok { + return fmt.Errorf("class name conflicted %s", item.Name) + } + if !constraint.MatchClass(item) { + return fmt.Errorf("class %s does not conform to constraint %s", item.Name, item.ResourceConstraintRef) + } + classNames = append(classNames, item.Name) } objName := class.GetCustomClassObjectName(o.ClusterDefRef, o.ComponentType) diff --git a/internal/cli/cmd/class/create_test.go b/internal/cli/cmd/class/create_test.go index e79cf7b88..8fbfb4063 100644 --- a/internal/cli/cmd/class/create_test.go +++ b/internal/cli/cmd/class/create_test.go @@ -82,8 +82,8 @@ var _ = Describe("create", func() { It("should succeed with required arguments", func() { o.Constraint = generalResourceConstraint.Name - fillResources(o, "12", "48Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) - Expect(o.validate([]string{"general-12c48g"})).ShouldNot(HaveOccurred()) + fillResources(o, "2", "8Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) + Expect(o.validate([]string{"general-2c8g"})).ShouldNot(HaveOccurred()) Expect(o.run()).ShouldNot(HaveOccurred()) Expect(out.String()).Should(ContainSubstring(o.ClassName)) }) @@ -100,10 +100,10 @@ var _ = Describe("create", func() { o.File = testCustomClassDefsPath Expect(o.run()).ShouldNot(HaveOccurred()) Expect(out.String()).Should(ContainSubstring("custom-1c1g")) - Expect(out.String()).Should(ContainSubstring("custom-200c400g")) + Expect(out.String()).Should(ContainSubstring("custom-4c16g")) // memory optimized classes - Expect(out.String()).Should(ContainSubstring("custom-1c32g")) - Expect(out.String()).Should(ContainSubstring("custom-2c64g")) + Expect(out.String()).Should(ContainSubstring("custom-2c16g")) + Expect(out.String()).Should(ContainSubstring("custom-4c64g")) }) }) diff --git a/internal/cli/cmd/migration/base.go b/internal/cli/cmd/migration/base.go index dc20d46ab..b16e0e673 100644 --- a/internal/cli/cmd/migration/base.go +++ b/internal/cli/cmd/migration/base.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/base_test.go b/internal/cli/cmd/migration/base_test.go index 3cbccbdf3..04cd0a40d 100644 --- a/internal/cli/cmd/migration/base_test.go +++ b/internal/cli/cmd/migration/base_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/cmd_builder.go b/internal/cli/cmd/migration/cmd_builder.go index 343f889a4..b3c55da41 100644 --- a/internal/cli/cmd/migration/cmd_builder.go +++ b/internal/cli/cmd/migration/cmd_builder.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/cmd_builder_test.go b/internal/cli/cmd/migration/cmd_builder_test.go index eb5ff5136..7656131a2 100644 --- a/internal/cli/cmd/migration/cmd_builder_test.go +++ b/internal/cli/cmd/migration/cmd_builder_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/create.go b/internal/cli/cmd/migration/create.go index 4b668d009..3a99c9f94 100644 --- a/internal/cli/cmd/migration/create.go +++ b/internal/cli/cmd/migration/create.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/create_test.go b/internal/cli/cmd/migration/create_test.go index c6c1a5906..9a0407877 100644 --- a/internal/cli/cmd/migration/create_test.go +++ b/internal/cli/cmd/migration/create_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/describe.go b/internal/cli/cmd/migration/describe.go index f96d94efa..bc9033501 100644 --- a/internal/cli/cmd/migration/describe.go +++ b/internal/cli/cmd/migration/describe.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/describe_test.go b/internal/cli/cmd/migration/describe_test.go index 9190b0953..d8c0dd62b 100644 --- a/internal/cli/cmd/migration/describe_test.go +++ b/internal/cli/cmd/migration/describe_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/examples.go b/internal/cli/cmd/migration/examples.go index eaa2eb2c3..0a61280a9 100644 --- a/internal/cli/cmd/migration/examples.go +++ b/internal/cli/cmd/migration/examples.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import "k8s.io/kubectl/pkg/util/templates" diff --git a/internal/cli/cmd/migration/list.go b/internal/cli/cmd/migration/list.go index 67b248cba..1f1a25e67 100644 --- a/internal/cli/cmd/migration/list.go +++ b/internal/cli/cmd/migration/list.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/list_test.go b/internal/cli/cmd/migration/list_test.go index 5a28311d1..0f34095d6 100644 --- a/internal/cli/cmd/migration/list_test.go +++ b/internal/cli/cmd/migration/list_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/logs.go b/internal/cli/cmd/migration/logs.go index 3d7347345..35b1c0d02 100644 --- a/internal/cli/cmd/migration/logs.go +++ b/internal/cli/cmd/migration/logs.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/logs_test.go b/internal/cli/cmd/migration/logs_test.go index 14aa3024b..e2ecdefac 100644 --- a/internal/cli/cmd/migration/logs_test.go +++ b/internal/cli/cmd/migration/logs_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/suite_test.go b/internal/cli/cmd/migration/suite_test.go index e876871df..e2ac143ec 100644 --- a/internal/cli/cmd/migration/suite_test.go +++ b/internal/cli/cmd/migration/suite_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration_test import ( diff --git a/internal/cli/cmd/migration/templates.go b/internal/cli/cmd/migration/templates.go index 5479c6b04..951370ae5 100644 --- a/internal/cli/cmd/migration/templates.go +++ b/internal/cli/cmd/migration/templates.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/templates_test.go b/internal/cli/cmd/migration/templates_test.go index 9d818c5ad..3c8e538e0 100644 --- a/internal/cli/cmd/migration/templates_test.go +++ b/internal/cli/cmd/migration/templates_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/terminate.go b/internal/cli/cmd/migration/terminate.go index beb8be287..22e188d57 100644 --- a/internal/cli/cmd/migration/terminate.go +++ b/internal/cli/cmd/migration/terminate.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/cmd/migration/terminate_test.go b/internal/cli/cmd/migration/terminate_test.go index 167e7d2c9..0dafce7d0 100644 --- a/internal/cli/cmd/migration/terminate_test.go +++ b/internal/cli/cmd/migration/terminate_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package migration import ( diff --git a/internal/cli/testing/testdata/custom_class.yaml b/internal/cli/testing/testdata/custom_class.yaml index c227ce5cc..e47ff3ea3 100644 --- a/internal/cli/testing/testdata/custom_class.yaml +++ b/internal/cli/testing/testdata/custom_class.yaml @@ -12,9 +12,9 @@ - namingTemplate: "custom-{{ .cpu }}c{{ .memory }}g" classes: - args: ["1", "1", "100", "10"] - - name: custom-200c400g - cpu: 200 - memory: 400Gi + - name: custom-4c16g + cpu: 4 + memory: 16Gi - resourceConstraintRef: kb-resource-constraint-memory-optimized template: | @@ -29,5 +29,5 @@ series: - namingTemplate: "custom-{{ .cpu }}c{{ .memory }}g" classes: - - args: ["1", "32", "100", "10"] - - args: ["2", "64", "100", "10"] + - args: ["2", "16", "100", "10"] + - args: ["4", "64", "100", "10"] diff --git a/internal/testutil/apps/componentclassdefinition_factory.go b/internal/testutil/apps/componentclassdefinition_factory.go index 74d9811b4..166ea24a1 100644 --- a/internal/testutil/apps/componentclassdefinition_factory.go +++ b/internal/testutil/apps/componentclassdefinition_factory.go @@ -23,16 +23,6 @@ import ( "github.com/apecloud/kubeblocks/internal/constant" ) -const classTemplate = ` -cpu: "{{ or .cpu 1 }}" -memory: "{{ or .memory 4 }}Gi" -volumes: -- name: data - size: "{{ or .dataStorageSize 10 }}Gi" -- name: log - size: "{{ or .logStorageSize 1 }}Gi" -` - type MockComponentClassDefinitionFactory struct { BaseFactory[appsv1alpha1.ComponentClassDefinition, *appsv1alpha1.ComponentClassDefinition, MockComponentClassDefinitionFactory] } @@ -48,27 +38,24 @@ func NewComponentClassDefinitionFactory(name, clusterDefinitionRef, componentTyp constant.KBAppComponentDefRefLabelKey: componentType, }, }, - Spec: appsv1alpha1.ComponentClassDefinitionSpec{ - Groups: []appsv1alpha1.ComponentClassGroup{ - { - ResourceConstraintRef: "kube-resource-constraint-general", - Template: classTemplate, - Vars: []string{"cpu", "memory", "dataStorageSize", "logStorageSize"}, - Series: []appsv1alpha1.ComponentClassSeries{ - { - NamingTemplate: "general-{{ .cpu }}c{{ .memory }}g", - }, - }, - }, - }, - }, }, f) return f } -func (factory *MockComponentClassDefinitionFactory) AddClass(cls appsv1alpha1.ComponentClass) *MockComponentClassDefinitionFactory { - classes := factory.get().Spec.Groups[0].Series[0].Classes - classes = append(classes, cls) - factory.get().Spec.Groups[0].Series[0].Classes = classes +func (factory *MockComponentClassDefinitionFactory) AddClassGroup(constraintRef string) *MockComponentClassDefinitionFactory { + groups := factory.get().Spec.Groups + group := classGroupTemplate + group.ResourceConstraintRef = constraintRef + groups = append(groups, group) + factory.get().Spec.Groups = groups + return factory +} + +func (factory *MockComponentClassDefinitionFactory) AddClasses(classes []appsv1alpha1.ComponentClass) *MockComponentClassDefinitionFactory { + groups := factory.get().Spec.Groups + if len(groups) > 0 { + groups[len(groups)-1].Series[0].Classes = classes + } + factory.get().Spec.Groups = groups return factory } diff --git a/internal/testutil/apps/componentresourceconstraint_factory.go b/internal/testutil/apps/componentresourceconstraint_factory.go new file mode 100644 index 000000000..1c067755e --- /dev/null +++ b/internal/testutil/apps/componentresourceconstraint_factory.go @@ -0,0 +1,88 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apps + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" +) + +type ResourceConstraintTplType string + +const ( + ResourceConstraintSpecial ResourceConstraintTplType = "special" + ResourceConstraintNormal ResourceConstraintTplType = "normal" + + specialConstraintTemplate = ` +- cpu: + min: 0.5 + max: 2 + step: 0.5 + memory: + sizePerCPU: 1Gi +- cpu: + min: 2 + max: 2 + memory: + sizePerCPU: 2Gi +` + normalConstraintTemplate = ` +- cpu: + slots: [2, 4, 8, 16, 24, 32, 48, 64, 96, 128] + memory: + sizePerCPU: 4Gi +` +) + +type MockComponentResourceConstraintFactory struct { + BaseFactory[appsv1alpha1.ComponentResourceConstraint, *appsv1alpha1.ComponentResourceConstraint, MockComponentResourceConstraintFactory] +} + +func NewComponentResourceConstraintFactory(name string) *MockComponentResourceConstraintFactory { + f := &MockComponentResourceConstraintFactory{} + f.init("", name, &appsv1alpha1.ComponentResourceConstraint{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "resourceconstraint.kubeblocks.io/provider": "kubeblocks", + }, + }, + }, f) + return f +} + +func (factory *MockComponentResourceConstraintFactory) AddConstraints(constraintTplType ResourceConstraintTplType) *MockComponentResourceConstraintFactory { + var ( + tpl string + newConstraints []appsv1alpha1.ResourceConstraint + constraints = factory.get().Spec.Constraints + ) + switch constraintTplType { + case ResourceConstraintSpecial: + tpl = specialConstraintTemplate + case ResourceConstraintNormal: + tpl = normalConstraintTemplate + } + if err := yaml.Unmarshal([]byte(tpl), &newConstraints); err != nil { + panic(err) + } + constraints = append(constraints, newConstraints...) + factory.get().Spec.Constraints = constraints + return factory +} diff --git a/internal/testutil/apps/constant.go b/internal/testutil/apps/constant.go index b8ca81e63..bd92b921e 100644 --- a/internal/testutil/apps/constant.go +++ b/internal/testutil/apps/constant.go @@ -25,15 +25,17 @@ import ( ) const ( - KubeBlocks = "kubeblocks" - LogVolumeName = "log" - ConfVolumeName = "conf" - DataVolumeName = "data" - ScriptsVolumeName = "scripts" - ServiceDefaultName = "" - ServiceHeadlessName = "headless" - ServiceVPCName = "a-vpc-lb-service-for-app" - ServiceInternetName = "a-internet-lb-service-for-app" + KubeBlocks = "kubeblocks" + LogVolumeName = "log" + ConfVolumeName = "conf" + DataVolumeName = "data" + ScriptsVolumeName = "scripts" + ServiceDefaultName = "" + ServiceHeadlessName = "headless" + ServiceVPCName = "a-vpc-lb-service-for-app" + ServiceInternetName = "a-internet-lb-service-for-app" + DefaultGeneralResourceConstraintName = "kb-resource-constraint-general" + DefaultMemoryOptimizedResourceConstraintName = "kb-resource-constraint-memory-optimized" ReplicationPodRoleVolume = "pod-role" ReplicationRoleLabelFieldPath = "metadata.labels['kubeblocks.io/role']" @@ -286,4 +288,22 @@ var ( Containers: []corev1.Container{defaultRedisContainer}, }, } + + classGroupTemplate = appsv1alpha1.ComponentClassGroup{ + Template: ` +cpu: "{{ or .cpu 1 }}" +memory: "{{ or .memory 4 }}Gi" +volumes: +- name: data + size: "{{ or .dataStorageSize 10 }}Gi" +- name: log + size: "{{ or .logStorageSize 1 }}Gi" +`, + Vars: []string{"cpu", "memory", "dataStorageSize", "logStorageSize"}, + Series: []appsv1alpha1.ComponentClassSeries{ + { + NamingTemplate: "custom-{{ .cpu }}c{{ .memory }}g", + }, + }, + } ) diff --git a/test/testdata/cue_testdata/mongod.cue b/test/testdata/cue_testdata/mongod.cue index 10bb6c66a..4543b77b9 100644 --- a/test/testdata/cue_testdata/mongod.cue +++ b/test/testdata/cue_testdata/mongod.cue @@ -1,3 +1,17 @@ +// Copyright ApeCloud, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #MongodParameter: { net: { port: int & >=0 & <=65535 From 1fd0ec35e8cb873d1d9b431cccc906a9e5d7f29f Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Mon, 17 Apr 2023 19:15:26 +0800 Subject: [PATCH 063/439] fix: add preflight for gke (#2609) --- .../kubeblocks/data/gke_hostpreflight.yaml | 20 +++++++++ .../cmd/kubeblocks/data/gke_preflight.yaml | 44 +++++++++++++++++++ internal/cli/cmd/kubeblocks/preflight.go | 9 ++++ 3 files changed, 73 insertions(+) create mode 100644 internal/cli/cmd/kubeblocks/data/gke_hostpreflight.yaml create mode 100644 internal/cli/cmd/kubeblocks/data/gke_preflight.yaml diff --git a/internal/cli/cmd/kubeblocks/data/gke_hostpreflight.yaml b/internal/cli/cmd/kubeblocks/data/gke_hostpreflight.yaml new file mode 100644 index 000000000..1a8ccc863 --- /dev/null +++ b/internal/cli/cmd/kubeblocks/data/gke_hostpreflight.yaml @@ -0,0 +1,20 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: HostPreflight +metadata: + name: host-utility +spec: + collectors: + analyzers: + extendCollectors: + - hostUtility : + collectorName: gcloud-cli + utilityName: gcloud + extendAnalyzers: + - hostUtility: + checkName: gcloudCli-Check + collectorName: gcloud-cli + outcomes: + - pass: + message: gcloud-cli has been installed + - warn: + message: gcloud-cli isn't installed \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml b/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml new file mode 100644 index 000000000..1b47740c6 --- /dev/null +++ b/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml @@ -0,0 +1,44 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: Preflight +metadata: + name: kubeblocks_preflight +spec: + collectors: + - clusterInfo: {} + analyzers: + - clusterVersion: + checkName: GKE-Version + outcomes: + - fail: + when: "< 1.22.0" + message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0. + uri: https://www.kubernetes.io + - pass: + when: ">= 1.22.0" + message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes. + uri: https://www.kubernetes.io + - nodeResources: + checkName: At-Least-3-Nodes + outcomes: + - warn: + when: "count() < 3" + message: This application requires at least 3 nodes + - pass: + message: This cluster has enough nodes. + extendAnalyzers: + - clusterAccess: + checkName: Check-K8S-Access + outcomes: + - fail: + message: k8s cluster access fail + - pass: + message: k8s cluster access ok + - storageClass: + checkName: Required-PREMIUM-SC + storageClassType: "pd-ssd" + provisioner: "pd.csi.storage.gke.io" + outcomes: + - fail: + message: The premium storage class was not found + - pass: + message: premium is the presence, and all good on storage classes \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/preflight.go b/internal/cli/cmd/kubeblocks/preflight.go index a6452a1e1..c94ee2e4f 100644 --- a/internal/cli/cmd/kubeblocks/preflight.go +++ b/internal/cli/cmd/kubeblocks/preflight.go @@ -76,6 +76,8 @@ var ( const ( EKSHostPreflight = "data/eks_hostpreflight.yaml" EKSPreflight = "data/eks_preflight.yaml" + GKEHostPreflight = "data/gke_hostpreflight.yaml" + GKEPreflight = "data/gke_preflight.yaml" ) // PreflightOptions declares the arguments accepted by the preflight command @@ -131,6 +133,13 @@ func LoadVendorCheckYaml(vendorName util.K8sProvider) ([][]byte, error) { if data, err := defaultVendorYamlData.ReadFile(EKSPreflight); err == nil { yamlDataList = append(yamlDataList, data) } + case util.GKEProvider: + if data, err := defaultVendorYamlData.ReadFile(GKEHostPreflight); err == nil { + yamlDataList = append(yamlDataList, data) + } + if data, err := defaultVendorYamlData.ReadFile(GKEPreflight); err == nil { + yamlDataList = append(yamlDataList, data) + } case util.UnknownProvider: fallthrough default: From 9470dc5923ab67c3f44387e0d241000a92943f32 Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Mon, 17 Apr 2023 19:44:53 +0800 Subject: [PATCH 064/439] fix: fixed check storage class failed error (#2545) --- externalapis/preflight/v1beta2/type.go | 18 +++ .../cmd/kubeblocks/data/eks_preflight.yaml | 19 +-- internal/preflight/analyzer/analyze_result.go | 78 ++++++++++++ .../preflight/analyzer/analyze_result_test.go | 78 ++++++++++++ internal/preflight/analyzer/analyzer.go | 2 + .../preflight/analyzer/kb_storage_class.go | 85 +++++++++++++ .../analyzer/kb_storage_class_test.go | 119 ++++++++++++++++++ 7 files changed, 390 insertions(+), 9 deletions(-) create mode 100644 internal/preflight/analyzer/analyze_result.go create mode 100644 internal/preflight/analyzer/analyze_result_test.go create mode 100644 internal/preflight/analyzer/kb_storage_class.go create mode 100644 internal/preflight/analyzer/kb_storage_class_test.go diff --git a/externalapis/preflight/v1beta2/type.go b/externalapis/preflight/v1beta2/type.go index 896d0a2ee..11994badb 100644 --- a/externalapis/preflight/v1beta2/type.go +++ b/externalapis/preflight/v1beta2/type.go @@ -32,6 +32,9 @@ type ExtendAnalyze struct { // clusterAccess is to determine the accessibility of target k8s cluster // +optional ClusterAccess *ClusterAccessAnalyze `json:"clusterAccess,omitempty"` + // StorageClass is to determine the correctness of target storage class + // +optional + StorageClass *KBStorageClassAnalyze `json:"storageClass,omitempty"` } type HostUtility struct { @@ -81,6 +84,21 @@ type ClusterRegionAnalyze struct { RegionNames []string `json:"regionNames"` } +// KBStorageClassAnalyze replaces default storageClassAnalyze in preflight +type KBStorageClassAnalyze struct { + // analyzeMeta is defined in troubleshoot.sh + troubleshoot.AnalyzeMeta `json:",inline"` + // outcomes are expected user defined results. + // +kubebuilder:validation:Required + Outcomes []*troubleshoot.Outcome `json:"outcomes"` + // Parameters is a set of parameters including type and fsType... + // +kubebuilder:validation:Required + StorageClassType string `json:"storageClassType"` + // provisioner is the provisioner of storageClass + // +optional + Provisioner string `json:"provisioner,omitempty"` +} + type ExtendHostAnalyze struct { // hostUtility is to analyze the presence of target utility. // +optional diff --git a/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml b/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml index d50337f1e..35b6324bc 100644 --- a/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml @@ -25,14 +25,6 @@ spec: message: This application requires at least 3 nodes - pass: message: This cluster has enough nodes. - - storageClass: - checkName: Required-GP3-SC - storageClassName: "gp3" - outcomes: - - fail: - message: The gp3 storage class was not found - - pass: - message: gp3 is the presence, and all good on storage classes - deploymentStatus: checkName: AWS-Load-Balancer-Check name: aws-load-balancer-controller @@ -56,4 +48,13 @@ spec: - fail: message: k8s cluster access fail - pass: - message: k8s cluster access ok \ No newline at end of file + message: k8s cluster access ok + - storageClass: + checkName: Required-GP3-SC + storageClassType: "gp3" + provisioner: "ebs.csi.aws.com" + outcomes: + - fail: + message: The gp3 storage class was not found + - pass: + message: gp3 is the presence, and all good on storage classes \ No newline at end of file diff --git a/internal/preflight/analyzer/analyze_result.go b/internal/preflight/analyzer/analyze_result.go new file mode 100644 index 000000000..5444aaf39 --- /dev/null +++ b/internal/preflight/analyzer/analyze_result.go @@ -0,0 +1,78 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package analyzer + +import ( + analyze "github.com/replicatedhq/troubleshoot/pkg/analyze" + troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" +) + +func newAnalyzeResult(title string, resultType string, outcomes []*troubleshoot.Outcome) *analyze.AnalyzeResult { + for _, outcome := range outcomes { + if outcome == nil { + continue + } + switch resultType { + case PassType: + if outcome.Pass != nil { + return newPassAnalyzeResult(title, outcome) + } + case WarnType: + if outcome.Warn != nil { + return newWarnAnalyzeResult(title, outcome) + } + case FailType: + if outcome.Fail != nil { + return newFailAnalyzeResult(title, outcome) + } + default: + return newFailedResultWithMessage(title, IncorrectOutcomeType) + } + } + return newFailedResultWithMessage(title, MissingOutcomeMessage) +} + +func newFailAnalyzeResult(titile string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { + return &analyze.AnalyzeResult{ + Title: titile, + IsFail: true, + Message: outcome.Fail.Message, + URI: outcome.Fail.URI, + } +} + +func newWarnAnalyzeResult(titile string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { + return &analyze.AnalyzeResult{ + Title: titile, + IsWarn: true, + Message: outcome.Warn.Message, + URI: outcome.Warn.URI, + } +} + +func newPassAnalyzeResult(titile string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { + return &analyze.AnalyzeResult{ + Title: titile, + IsPass: true, + Message: outcome.Pass.Message, + URI: outcome.Pass.URI, + } +} + +func newFailedResultWithMessage(title, message string) *analyze.AnalyzeResult { + return newFailAnalyzeResult(title, &troubleshoot.Outcome{Fail: &troubleshoot.SingleOutcome{Message: message}}) +} diff --git a/internal/preflight/analyzer/analyze_result_test.go b/internal/preflight/analyzer/analyze_result_test.go new file mode 100644 index 000000000..5d83f54c0 --- /dev/null +++ b/internal/preflight/analyzer/analyze_result_test.go @@ -0,0 +1,78 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package analyzer + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" +) + +var _ = Describe("kb_storage_class_test", func() { + var ( + outcomes []*troubleshoot.Outcome + ) + Context("analyze storage class test", func() { + BeforeEach(func() { + outcomes = []*troubleshoot.Outcome{ + { + Pass: &troubleshoot.SingleOutcome{ + + Message: "analyze storage class success", + }, + Fail: &troubleshoot.SingleOutcome{ + Message: "analyze storage class fail", + }, + Warn: &troubleshoot.SingleOutcome{ + Message: "warn message", + }, + }, + } + }) + It("AnalyzeResult test, and expected that fail is true", func() { + Eventually(func(g Gomega) { + res := newAnalyzeResult("test", FailType, outcomes) + g.Expect(res.IsFail).Should(BeTrue()) + g.Expect(res.Message).Should(Equal(outcomes[0].Fail.Message)) + }).Should(Succeed()) + }) + + It("AnalyzeResult test, and expected that warn is true", func() { + Eventually(func(g Gomega) { + res := newAnalyzeResult("test", WarnType, outcomes) + g.Expect(res.IsWarn).Should(BeTrue()) + g.Expect(res.Message).Should(Equal(outcomes[0].Warn.Message)) + }).Should(Succeed()) + }) + It("AnalyzeResult test, and expected that pass is true", func() { + Eventually(func(g Gomega) { + res := newAnalyzeResult("test", PassType, outcomes) + g.Expect(res.IsPass).Should(BeTrue()) + g.Expect(res.Message).Should(Equal(outcomes[0].Pass.Message)) + }).Should(Succeed()) + }) + It("AnalyzeResult with message test, and expected that fail is true", func() { + Eventually(func(g Gomega) { + message := "test" + res := newFailedResultWithMessage("test", message) + g.Expect(res.IsFail).Should(BeTrue()) + g.Expect(res.Message).Should(Equal(message)) + }).Should(Succeed()) + }) + }) +}) diff --git a/internal/preflight/analyzer/analyzer.go b/internal/preflight/analyzer/analyzer.go index df7a18e13..48514e361 100644 --- a/internal/preflight/analyzer/analyzer.go +++ b/internal/preflight/analyzer/analyzer.go @@ -39,6 +39,8 @@ func GetAnalyzer(analyzer *preflightv1beta2.ExtendAnalyze) (KBAnalyzer, bool) { switch { case analyzer.ClusterAccess != nil: return &AnalyzeClusterAccess{analyzer: analyzer.ClusterAccess}, true + case analyzer.StorageClass != nil: + return &AnalyzeStorageClassByKb{analyzer: analyzer.StorageClass}, true default: return nil, false } diff --git a/internal/preflight/analyzer/kb_storage_class.go b/internal/preflight/analyzer/kb_storage_class.go new file mode 100644 index 000000000..d85de289d --- /dev/null +++ b/internal/preflight/analyzer/kb_storage_class.go @@ -0,0 +1,85 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package analyzer + +import ( + "encoding/json" + "fmt" + + analyze "github.com/replicatedhq/troubleshoot/pkg/analyze" + storagev1beta1 "k8s.io/api/storage/v1beta1" + + preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" + "github.com/apecloud/kubeblocks/internal/preflight/util" +) + +const ( + StorageClassPath = "cluster-resources/storage-classes.json" + MissingOutcomeMessage = "there is a missing outcome message" + IncorrectOutcomeType = "there is an incorrect outcome type" + PassType = "Pass" + WarnType = "Warn" + FailType = "Fail" +) + +type AnalyzeStorageClassByKb struct { + analyzer *preflightv1beta2.KBStorageClassAnalyze +} + +func (a *AnalyzeStorageClassByKb) Title() string { + return util.TitleOrDefault(a.analyzer.AnalyzeMeta, "Kubeblocks Storage Class") +} + +func (a *AnalyzeStorageClassByKb) GetAnalyzer() *preflightv1beta2.KBStorageClassAnalyze { + return a.analyzer +} + +func (a *AnalyzeStorageClassByKb) IsExcluded() (bool, error) { + return util.IsExcluded(a.analyzer.Exclude) +} + +func (a *AnalyzeStorageClassByKb) Analyze(getFile GetCollectedFileContents, findFiles GetChildCollectedFileContents) ([]*analyze.AnalyzeResult, error) { + result, err := a.analyzeStorageClass(a.analyzer, getFile, findFiles) + if err != nil { + return []*analyze.AnalyzeResult{result}, err + } + result.Strict = a.analyzer.Strict.BoolOrDefaultFalse() + return []*analyze.AnalyzeResult{result}, nil +} + +func (a *AnalyzeStorageClassByKb) analyzeStorageClass(analyzer *preflightv1beta2.KBStorageClassAnalyze, getFile GetCollectedFileContents, findFiles GetChildCollectedFileContents) (*analyze.AnalyzeResult, error) { + storageClassesData, err := getFile(StorageClassPath) + if err != nil { + return newFailedResultWithMessage(a.Title(), fmt.Sprintf("get jsonfile failed, err:%v", err)), err + } + var storageClasses storagev1beta1.StorageClassList + if err = json.Unmarshal(storageClassesData, &storageClasses); err != nil { + return newFailedResultWithMessage(a.Title(), fmt.Sprintf("get jsonfile failed, err:%v", err)), err + } + + for _, storageClass := range storageClasses.Items { + if storageClass.Parameters["type"] != analyzer.StorageClassType { + continue + } + if storageClass.Provisioner == "" || (storageClass.Provisioner == analyzer.Provisioner) { + return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil + } + } + return newAnalyzeResult(a.Title(), FailType, a.analyzer.Outcomes), nil +} + +var _ KBAnalyzer = &AnalyzeStorageClassByKb{} diff --git a/internal/preflight/analyzer/kb_storage_class_test.go b/internal/preflight/analyzer/kb_storage_class_test.go new file mode 100644 index 000000000..b7106f1c2 --- /dev/null +++ b/internal/preflight/analyzer/kb_storage_class_test.go @@ -0,0 +1,119 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package analyzer + +import ( + "encoding/json" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + storagev1beta1 "k8s.io/api/storage/v1beta1" + + preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" +) + +var ( + clusterResources = storagev1beta1.StorageClassList{ + Items: []storagev1beta1.StorageClass{ + { + Provisioner: "ebs.csi.aws.com", + Parameters: map[string]string{"type": "gp3"}, + }, + }, + } +) + +var _ = Describe("kb_storage_class_test", func() { + var ( + analyzer AnalyzeStorageClassByKb + ) + Context("analyze storage class test", func() { + BeforeEach(func() { + analyzer = AnalyzeStorageClassByKb{ + analyzer: &preflightv1beta2.KBStorageClassAnalyze{ + Provisioner: "ebs.csi.aws.com", + StorageClassType: "gp3", + Outcomes: []*troubleshoot.Outcome{ + { + Pass: &troubleshoot.SingleOutcome{ + + Message: "analyze storage class success", + }, + Fail: &troubleshoot.SingleOutcome{ + Message: "analyze storage class fail", + }, + }, + }}} + }) + It("Analyze test, and get file failed", func() { + Eventually(func(g Gomega) { + getCollectedFileContents := func(filename string) ([]byte, error) { + return nil, errors.New("get file failed") + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).To(HaveOccurred()) + g.Expect(res[0].IsFail).Should(BeTrue()) + g.Expect(res[0].IsPass).Should(BeFalse()) + }).Should(Succeed()) + }) + + It("Analyze test, and return of get file is not clusterResource", func() { + Eventually(func(g Gomega) { + getCollectedFileContents := func(filename string) ([]byte, error) { + return []byte("test"), nil + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).To(HaveOccurred()) + g.Expect(res[0].IsFail).Should(BeTrue()) + g.Expect(res[0].IsPass).Should(BeFalse()) + }).Should(Succeed()) + }) + + It("Analyze test, and analyzer result is expected that pass is true", func() { + Eventually(func(g Gomega) { + g.Expect(analyzer.IsExcluded()).Should(BeFalse()) + b, err := json.Marshal(clusterResources) + g.Expect(err).NotTo(HaveOccurred()) + getCollectedFileContents := func(filename string) ([]byte, error) { + return b, nil + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res[0].IsPass).Should(BeTrue()) + g.Expect(res[0].IsFail).Should(BeFalse()) + }).Should(Succeed()) + }) + It("Analyze test, and analyzer result is expected that fail is true", func() { + Eventually(func(g Gomega) { + g.Expect(analyzer.IsExcluded()).Should(BeFalse()) + clusterResources.Items[0].Provisioner = "apecloud" + b, err := json.Marshal(clusterResources) + g.Expect(err).NotTo(HaveOccurred()) + getCollectedFileContents := func(filename string) ([]byte, error) { + return b, nil + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res[0].IsPass).Should(BeFalse()) + g.Expect(res[0].IsFail).Should(BeTrue()) + }).Should(Succeed()) + }) + }) +}) From 41cece54a7a4fd2edf489f1d3d7f882e6855d740 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Mon, 17 Apr 2023 21:31:14 +0800 Subject: [PATCH 065/439] chore: fix no space left on device (#2664) --- .github/workflows/cicd-pull-request.yml | 17 ------ .github/workflows/cicd-push.yml | 70 +++++++++++-------------- 2 files changed, 32 insertions(+), 55 deletions(-) diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index e57c90d37..498badfdc 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -35,23 +35,6 @@ jobs: runs-on: [ self-hosted, eks-fargate-runner ] steps: - uses: apecloud/checkout@main - - - name: Download ${{ env.GO_CACHE }} - run: | - bash .github/utils/release_gitlab.sh \ - --type 6 \ - --project-id ${{ env.GITLAB_GO_CACHE_PROJECT_ID }} \ - --tag-name ${{ env.GO_CACHE }} \ - --asset-name ${{ env.GO_CACHE }}.tar.gz \ - --access-token ${{ env.GITLAB_ACCESS_TOKEN }} - - - name: Extract ${{ env.GO_CACHE }} - uses: a7ul/tar-action@v1.1.3 - with: - command: x - cwd: ${{ env.GO_CACHE_DIR }} - files: ${{ env.GO_CACHE }}.tar.gz - - name: make mod-vendor and lint run: | mkdir -p ./bin diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index 5f7037653..a0ae408d1 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -109,24 +109,6 @@ jobs: if: contains(needs.trigger-mode.outputs.trigger-mode, '[test]') steps: - uses: apecloud/checkout@main - - name: Download ${{ env.GO_CACHE }} - if: ${{ github.ref_name != 'main' }} - run: | - bash .github/utils/release_gitlab.sh \ - --type 6 \ - --project-id ${{ env.GITLAB_GO_CACHE_PROJECT_ID }} \ - --tag-name ${{ env.GO_CACHE }} \ - --asset-name ${{ env.GO_CACHE }}.tar.gz \ - --access-token ${{ env.GITLAB_ACCESS_TOKEN }} - - - name: Extract ${{ env.GO_CACHE }} - if: ${{ github.ref_name != 'main' }} - uses: a7ul/tar-action@v1.1.3 - with: - command: x - cwd: ${{ env.GO_CACHE_DIR }} - files: ${{ env.GO_CACHE }}.tar.gz - - name: make manifests check run: | mkdir -p ./bin @@ -158,26 +140,6 @@ jobs: name: codecov-report verbose: true - - name: Compress ${{ env.GO_CACHE }} - if: ${{ github.ref_name == 'main' }} - uses: a7ul/tar-action@v1.1.3 - with: - command: c - cwd: ${{ env.GO_CACHE_DIR }} - files: | - ./ - outPath: ${{ env.GO_CACHE }}.tar.gz - - - name: Upload ${{ env.GO_CACHE }} to gitlab - if: ${{ github.ref_name == 'main' }} - run: | - bash .github/utils/release_gitlab.sh \ - --type 5 \ - --project-id ${{ env.GITLAB_GO_CACHE_PROJECT_ID }} \ - --tag-name ${{ env.GO_CACHE }} \ - --asset-path ${{ env.GO_CACHE }}.tar.gz \ - --access-token ${{ env.GITLAB_ACCESS_TOKEN }} - check-image: needs: trigger-mode if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') && github.ref_name != 'main' }} @@ -224,3 +186,35 @@ jobs: BRANCH_NAME: "master" WORKFLOW_ID: "deploy.yml" secrets: inherit + + build-kbcli: + needs: trigger-mode + runs-on: ubuntu-latest + if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[cli]') && github.ref_name != 'main' }} + strategy: + matrix: + os: [linux-amd64, linux-arm64, darwin-amd64, darwin-arm64, windows-amd64] + steps: + - uses: actions/checkout@v3 + - name: install lib + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libbtrfs-dev \ + libdevmapper-dev + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + + - name: make generate + run: make generate + + - name: build cli + run: | + CLI_OS_ARCH=`bash .github/utils/utils.sh \ + --tag-name ${{ matrix.os }} \ + --type 2` + + make bin/kbcli.$CLI_OS_ARCH From 4b0cca076e505dd27bbded840e3fc7c670e6d6a5 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 17 Apr 2023 22:21:03 +0800 Subject: [PATCH 066/439] chore: Adjust the shared phase of configconstraint to a separate phase (#2209) --- apis/apps/v1alpha1/configconstraint_types.go | 9 ++++++--- apis/apps/v1alpha1/type.go | 11 +++++++++++ .../apps.kubeblocks.io_configconstraints.yaml | 14 +++++--------- .../apps/clusterdefinition_controller_test.go | 2 +- controllers/apps/configuration/config_util.go | 9 ++++++++- .../apps/configuration/config_util_test.go | 6 +++--- .../configconstraint_controller.go | 17 +++++++++++------ .../configconstraint_controller_test.go | 6 ++++++ .../apps.kubeblocks.io_configconstraints.yaml | 14 +++++--------- 9 files changed, 56 insertions(+), 32 deletions(-) diff --git a/apis/apps/v1alpha1/configconstraint_types.go b/apis/apps/v1alpha1/configconstraint_types.go index fa0b9eb5d..7e35cd873 100644 --- a/apis/apps/v1alpha1/configconstraint_types.go +++ b/apis/apps/v1alpha1/configconstraint_types.go @@ -66,10 +66,9 @@ type ConfigConstraintSpec struct { // ConfigConstraintStatus defines the observed state of ConfigConstraint. type ConfigConstraintStatus struct { - // phase is status of configuration template, when set to AvailablePhase, it can be referenced by ClusterDefinition or ClusterVersion. - // +kubebuilder:validation:Enum={Available,Unavailable,Deleting} + // phase is status of configuration template, when set to CCAvailablePhase, it can be referenced by ClusterDefinition or ClusterVersion. // +optional - Phase Phase `json:"phase,omitempty"` + Phase ConfigConstraintPhase `json:"phase,omitempty"` // message field describes the reasons of abnormal status. // +optional @@ -82,6 +81,10 @@ type ConfigConstraintStatus struct { ObservedGeneration int64 `json:"observedGeneration,omitempty"` } +func (cs ConfigConstraintStatus) IsConfigConstraintTerminalPhases() bool { + return cs.Phase == CCAvailablePhase +} + type CustomParametersValidation struct { // schema provides a way for providers to validate the changed parameters through json. // +kubebuilder:validation:Schemaless diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index 1eac9213c..58786882b 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -147,6 +147,17 @@ const ( UnavailablePhase Phase = "Unavailable" ) +// ConfigConstraintPhase defines the ConfigConstraint CR .status.phase +// +enum +// +kubebuilder:validation:Enum={Available,Unavailable, Deleting} +type ConfigConstraintPhase string + +const ( + CCAvailablePhase ConfigConstraintPhase = "Available" + CCUnavailablePhase ConfigConstraintPhase = "Unavailable" + CCDeletingPhase ConfigConstraintPhase = "Deleting" +) + // OpsPhase defines opsRequest phase. // +enum // +kubebuilder:validation:Enum={Pending,Creating,Running,Failed,Succeed} diff --git a/config/crd/bases/apps.kubeblocks.io_configconstraints.yaml b/config/crd/bases/apps.kubeblocks.io_configconstraints.yaml index 5a6a5d3b0..c73a6b1d2 100644 --- a/config/crd/bases/apps.kubeblocks.io_configconstraints.yaml +++ b/config/crd/bases/apps.kubeblocks.io_configconstraints.yaml @@ -284,16 +284,12 @@ spec: format: int64 type: integer phase: - allOf: - - enum: - - Available - - Unavailable - - enum: - - Available - - Unavailable - - Deleting description: phase is status of configuration template, when set to - AvailablePhase, it can be referenced by ClusterDefinition or ClusterVersion. + CCAvailablePhase, it can be referenced by ClusterDefinition or ClusterVersion. + enum: + - Available + - Unavailable + - Deleting type: string type: object type: object diff --git a/controllers/apps/clusterdefinition_controller_test.go b/controllers/apps/clusterdefinition_controller_test.go index 994762042..8e7b1e6e9 100644 --- a/controllers/apps/clusterdefinition_controller_test.go +++ b/controllers/apps/clusterdefinition_controller_test.go @@ -133,7 +133,7 @@ var _ = Describe("ClusterDefinition Controller", func() { cfgTpl := testapps.CreateCustomizedObj(&testCtx, "config/config-constraint.yaml", &appsv1alpha1.ConfigConstraint{}) Expect(testapps.ChangeObjStatus(&testCtx, cfgTpl, func() { - cfgTpl.Status.Phase = appsv1alpha1.AvailablePhase + cfgTpl.Status.Phase = appsv1alpha1.CCAvailablePhase })).Should(Succeed()) return cm } diff --git a/controllers/apps/configuration/config_util.go b/controllers/apps/configuration/config_util.go index 1f7bd6ec3..939f2a9a9 100644 --- a/controllers/apps/configuration/config_util.go +++ b/controllers/apps/configuration/config_util.go @@ -361,13 +361,20 @@ func validateConfigTemplate(cli client.Client, ctx intctrlutil.RequestCtx, confi } func validateConfigConstraintStatus(ccStatus appsv1alpha1.ConfigConstraintStatus) bool { - return ccStatus.Phase == appsv1alpha1.AvailablePhase + return ccStatus.Phase == appsv1alpha1.CCAvailablePhase } func usingComponentConfigSpec(annotations map[string]string, key, value string) bool { return len(annotations) != 0 && annotations[key] == value } +func updateConfigConstraintStatus(cli client.Client, ctx intctrlutil.RequestCtx, configConstraint *appsv1alpha1.ConfigConstraint, phase appsv1alpha1.ConfigConstraintPhase) error { + patch := client.MergeFrom(configConstraint.DeepCopy()) + configConstraint.Status.Phase = phase + configConstraint.Status.ObservedGeneration = configConstraint.Generation + return cli.Status().Patch(ctx.Ctx, configConstraint, patch) +} + func getAssociatedComponentsByConfigmap(stsList *appv1.StatefulSetList, cfg client.ObjectKey, configSpecName string) ([]appv1.StatefulSet, []string) { managerContainerName := constant.ConfigSidecarName stsLen := len(stsList.Items) diff --git a/controllers/apps/configuration/config_util_test.go b/controllers/apps/configuration/config_util_test.go index 72e38a06e..9c441f0be 100644 --- a/controllers/apps/configuration/config_util_test.go +++ b/controllers/apps/configuration/config_util_test.go @@ -115,7 +115,7 @@ var _ = Describe("ConfigWrapper util test", func() { Context("clusterdefinition CR test", func() { It("Should success without error", func() { availableTPL := configConstraintObj.DeepCopy() - availableTPL.Status.Phase = appsv1alpha1.AvailablePhase + availableTPL.Status.Phase = appsv1alpha1.CCAvailablePhase k8sMockClient.MockPatchMethod(testutil.WithSucceed()) k8sMockClient.MockListMethod(testutil.WithSucceed()) @@ -187,7 +187,7 @@ var _ = Describe("ConfigWrapper util test", func() { Expect(err).Should(Succeed()) availableTPL := configConstraintObj.DeepCopy() - availableTPL.Status.Phase = appsv1alpha1.AvailablePhase + availableTPL.Status.Phase = appsv1alpha1.CCAvailablePhase k8sMockClient.MockGetMethod(testutil.WithGetReturned(testutil.WithConstructSequenceResult( map[client.ObjectKey][]testutil.MockGetReturned{ @@ -230,7 +230,7 @@ var _ = Describe("ConfigWrapper util test", func() { It("Should success without error", func() { updateAVTemplates() availableTPL := configConstraintObj.DeepCopy() - availableTPL.Status.Phase = appsv1alpha1.AvailablePhase + availableTPL.Status.Phase = appsv1alpha1.CCAvailablePhase k8sMockClient.MockPatchMethod(testutil.WithSucceed()) k8sMockClient.MockListMethod(testutil.WithSucceed()) diff --git a/controllers/apps/configuration/configconstraint_controller.go b/controllers/apps/configuration/configconstraint_controller.go index 9a66e041a..d386d799f 100644 --- a/controllers/apps/configuration/configconstraint_controller.go +++ b/controllers/apps/configuration/configconstraint_controller.go @@ -71,6 +71,14 @@ func (r *ConfigConstraintReconciler) Reconcile(ctx context.Context, req ctrl.Req r.Recorder.Event(configConstraint, corev1.EventTypeWarning, "ExistsReferencedResources", "cannot be deleted because of existing referencing ClusterDefinition or ClusterVersion.") } + if configConstraint.Status.Phase != appsv1alpha1.CCDeletingPhase { + err := updateConfigConstraintStatus(r.Client, reqCtx, configConstraint, appsv1alpha1.CCDeletingPhase) + // if fail to update ConfigConstraint status, return error, + // so that it can be retried + if err != nil { + return nil, err + } + } if res, err := intctrlutil.ValidateReferenceCR(reqCtx, r.Client, configConstraint, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(configConstraint.GetName()), recordEvent, &appsv1alpha1.ClusterDefinitionList{}, @@ -83,7 +91,7 @@ func (r *ConfigConstraintReconciler) Reconcile(ctx context.Context, req ctrl.Req return *res, err } - if configConstraint.Status.ObservedGeneration == configConstraint.Generation { + if configConstraint.Status.ObservedGeneration == configConstraint.Generation && configConstraint.Status.IsConfigConstraintTerminalPhases() { return intctrlutil.Reconciled() } @@ -96,14 +104,11 @@ func (r *ConfigConstraintReconciler) Reconcile(ctx context.Context, req ctrl.Req return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "failed to generate openAPISchema") } - statusPatch := client.MergeFrom(configConstraint.DeepCopy()) - configConstraint.Status.ObservedGeneration = configConstraint.Generation - configConstraint.Status.Phase = appsv1alpha1.AvailablePhase - if err = r.Client.Status().Patch(reqCtx.Ctx, configConstraint, statusPatch); err != nil { + err = updateConfigConstraintStatus(r.Client, reqCtx, configConstraint, appsv1alpha1.CCAvailablePhase) + if err != nil { return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } intctrlutil.RecordCreatedEvent(r.Recorder, configConstraint) - return ctrl.Result{}, nil } diff --git a/controllers/apps/configuration/configconstraint_controller_test.go b/controllers/apps/configuration/configconstraint_controller_test.go index 5212fef6e..0db707b05 100644 --- a/controllers/apps/configuration/configconstraint_controller_test.go +++ b/controllers/apps/configuration/configconstraint_controller_test.go @@ -105,6 +105,12 @@ var _ = Describe("ConfigConstraint Controller", func() { log.Log.Info("expect that ConfigConstraint is not deleted.") Consistently(testapps.CheckObjExists(&testCtx, constraintKey, &appsv1alpha1.ConfigConstraint{}, true)).Should(Succeed()) + By("check ConfigConstraint status should be deleting") + Eventually(testapps.CheckObj(&testCtx, constraintKey, + func(g Gomega, tpl *appsv1alpha1.ConfigConstraint) { + g.Expect(tpl.Status.Phase).To(BeEquivalentTo(appsv1alpha1.CCDeletingPhase)) + })).Should(Succeed()) + By("By delete referencing clusterdefinition and clusterversion") Expect(k8sClient.Delete(testCtx.Ctx, clusterVersionObj)).Should(Succeed()) Expect(k8sClient.Delete(testCtx.Ctx, clusterDefObj)).Should(Succeed()) diff --git a/deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml b/deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml index 5a6a5d3b0..c73a6b1d2 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_configconstraints.yaml @@ -284,16 +284,12 @@ spec: format: int64 type: integer phase: - allOf: - - enum: - - Available - - Unavailable - - enum: - - Available - - Unavailable - - Deleting description: phase is status of configuration template, when set to - AvailablePhase, it can be referenced by ClusterDefinition or ClusterVersion. + CCAvailablePhase, it can be referenced by ClusterDefinition or ClusterVersion. + enum: + - Available + - Unavailable + - Deleting type: string type: object type: object From 9b3af1c24fbf634e1804b9b634fea0531d6e6a51 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Tue, 18 Apr 2023 11:16:25 +0800 Subject: [PATCH 067/439] chore: remove `-A` option for cd/cv list (#2657) --- docs/user_docs/cli/kbcli_clusterdefinition_list.md | 1 - docs/user_docs/cli/kbcli_clusterversion_list.md | 1 - internal/cli/cmd/cluster/dataprotection_test.go | 3 +-- internal/cli/cmd/clusterdefinition/clusterdefinition.go | 2 +- internal/cli/cmd/clusterdefinition/clusterdefinition_test.go | 3 +-- internal/cli/cmd/clusterversion/clusterversion.go | 2 +- internal/cli/cmd/clusterversion/clusterversion_test.go | 3 +-- 7 files changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/user_docs/cli/kbcli_clusterdefinition_list.md b/docs/user_docs/cli/kbcli_clusterdefinition_list.md index 901814591..770e0c342 100644 --- a/docs/user_docs/cli/kbcli_clusterdefinition_list.md +++ b/docs/user_docs/cli/kbcli_clusterdefinition_list.md @@ -18,7 +18,6 @@ kbcli clusterdefinition list [flags] ### Options ``` - -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. -h, --help help for list -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. diff --git a/docs/user_docs/cli/kbcli_clusterversion_list.md b/docs/user_docs/cli/kbcli_clusterversion_list.md index 6124e60e0..64affc368 100644 --- a/docs/user_docs/cli/kbcli_clusterversion_list.md +++ b/docs/user_docs/cli/kbcli_clusterversion_list.md @@ -18,7 +18,6 @@ kbcli clusterversion list [flags] ### Options ``` - -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. -h, --help help for list -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index 8239d2c31..671ea6f2d 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -63,7 +63,6 @@ var _ = Describe("DataProtection", func() { }) Context("backup", func() { - initClient := func(policies ...*dataprotectionv1alpha1.BackupPolicy) { clusterDef := testing.FakeClusterDef() cluster := testing.FakeCluster(testing.ClusterName, testing.Namespace) @@ -78,7 +77,7 @@ var _ = Describe("DataProtection", func() { for _, v := range policies { objects = append(objects, v) } - tf.FakeDynamicClient = fake.NewSimpleDynamicClient(scheme.Scheme, objects...) + tf.FakeDynamicClient = testing.FakeDynamicClient(objects...) } It("list-backup-policy", func() { diff --git a/internal/cli/cmd/clusterdefinition/clusterdefinition.go b/internal/cli/cmd/clusterdefinition/clusterdefinition.go index c8c853c33..045fec933 100644 --- a/internal/cli/cmd/clusterdefinition/clusterdefinition.go +++ b/internal/cli/cmd/clusterdefinition/clusterdefinition.go @@ -56,6 +56,6 @@ func NewListCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C util.CheckErr(err) }, } - o.AddFlags(cmd) + o.AddFlags(cmd, true) return cmd } diff --git a/internal/cli/cmd/clusterdefinition/clusterdefinition_test.go b/internal/cli/cmd/clusterdefinition/clusterdefinition_test.go index 7c11eed58..5c9db2eae 100644 --- a/internal/cli/cmd/clusterdefinition/clusterdefinition_test.go +++ b/internal/cli/cmd/clusterdefinition/clusterdefinition_test.go @@ -24,13 +24,12 @@ import ( ) var _ = Describe("clusterdefinition", func() { - const namespace = "test" var streams genericclioptions.IOStreams var tf *cmdtesting.TestFactory BeforeEach(func() { streams, _, _, _ = genericclioptions.NewTestIOStreams() - tf = cmdtesting.NewTestFactory().WithNamespace(namespace) + tf = cmdtesting.NewTestFactory() }) AfterEach(func() { diff --git a/internal/cli/cmd/clusterversion/clusterversion.go b/internal/cli/cmd/clusterversion/clusterversion.go index f77ff4509..d83f83042 100644 --- a/internal/cli/cmd/clusterversion/clusterversion.go +++ b/internal/cli/cmd/clusterversion/clusterversion.go @@ -56,6 +56,6 @@ func NewListCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C util.CheckErr(err) }, } - o.AddFlags(cmd) + o.AddFlags(cmd, true) return cmd } diff --git a/internal/cli/cmd/clusterversion/clusterversion_test.go b/internal/cli/cmd/clusterversion/clusterversion_test.go index 2225a0b49..501054cd5 100644 --- a/internal/cli/cmd/clusterversion/clusterversion_test.go +++ b/internal/cli/cmd/clusterversion/clusterversion_test.go @@ -24,13 +24,12 @@ import ( ) var _ = Describe("clusterversion", func() { - const namespace = "test" var streams genericclioptions.IOStreams var tf *cmdtesting.TestFactory BeforeEach(func() { streams, _, _, _ = genericclioptions.NewTestIOStreams() - tf = cmdtesting.NewTestFactory().WithNamespace(namespace) + tf = cmdtesting.NewTestFactory() }) AfterEach(func() { From 02c5c47e673a8c706a36833305181b53ae1b04b0 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Tue, 18 Apr 2023 11:37:23 +0800 Subject: [PATCH 068/439] chore: some typos (#2652) --- .github/utils/typos.toml | 1 - apis/apps/v1alpha1/cluster_webhook_test.go | 4 ++-- apis/apps/v1alpha1/clusterdefinition_types.go | 6 +++--- apis/apps/v1alpha1/clusterdefinition_webhook.go | 2 +- .../apps/v1alpha1/clusterversion_webhook_test.go | 2 +- apis/apps/v1alpha1/opsrequest_conditions.go | 2 +- apis/apps/v1alpha1/opsrequest_conditions_test.go | 2 +- apis/apps/v1alpha1/opsrequest_webhook.go | 4 ++-- apis/apps/v1alpha1/type.go | 2 +- apis/extensions/v1alpha1/addon_types_test.go | 6 +++--- cmd/reloader/container_killer/killer.go | 12 ++++++------ .../apps.kubeblocks.io_clusterdefinitions.yaml | 6 +++--- controllers/apps/cluster_controller_test.go | 6 +++--- controllers/apps/components/component_status.go | 2 +- .../components/consensus/consensus_utils_test.go | 2 +- .../components/stateful_set_controller_test.go | 2 +- .../apps/components/util/stateful_set_utils.go | 2 +- .../apps/operations/volume_expansion_updater.go | 2 +- controllers/apps/opsrequest_controller_test.go | 2 +- controllers/apps/systemaccount_controller.go | 10 +++++----- .../apps/systemaccount_controller_test.go | 10 +++++----- controllers/apps/systemaccount_util.go | 2 +- controllers/extensions/addon_controller.go | 4 ++-- deploy/apecloud-mysql-scale/Chart.yaml | 2 +- .../config/mysql8-config-constraint.cue | 6 +++--- .../config/mysql8-config.tpl | 2 +- .../config/mysql8-config-constraint.cue | 6 +++--- deploy/apecloud-mysql/config/mysql8-config.tpl | 2 +- .../apps.kubeblocks.io_clusterdefinitions.yaml | 6 +++--- deploy/helm/templates/addons/nyancat-addon.yaml | 2 +- deploy/kafka/scripts/libkafka.sh | 2 +- deploy/weaviate/values.yaml | 2 +- docker/Dockerfile | 2 +- docker/Dockerfile-tools | 2 +- docker/custom-scripts/kubectl-helm-debian.sh | 2 +- docker/library-scripts/go-debian.sh | 2 +- docker/library-scripts/kubectl-helm-debian.sh | 2 +- docs/release_notes/v0.1.0/v0.1.0.md | 4 ++-- .../api/lifecycle-management/ops-request-api.md | 2 +- hack/install_cli.ps1 | 4 ++-- internal/cli/cluster/helper.go | 6 +++--- internal/cli/cmd/accounts/base.go | 8 ++++---- internal/cli/cmd/accounts/create.go | 2 +- internal/cli/cmd/accounts/grant.go | 8 ++++---- internal/cli/cmd/cluster/cluster.go | 2 +- internal/cli/cmd/cluster/dataprotection.go | 2 +- internal/cli/cmd/playground/init.go | 2 +- internal/cli/util/util.go | 8 ++++---- ...r_updater.go => dynamic_parameter_updater.go} | 0 .../config_manager/handler_util_test.go | 2 +- .../{constrant.go => constraint.go} | 0 .../container/container_kill_test.go | 4 ++-- .../controller/component/affinity_utils_test.go | 12 ++++++------ internal/controller/component/component_test.go | 2 +- internal/controller/plan/builtin_env_test.go | 2 +- internal/controller/plan/config_template_test.go | 2 +- internal/preflight/analyzer/analyze_result.go | 12 ++++++------ internal/testutil/k8s/k8sclient_util.go | 6 +++--- staticcheck.conf | 2 +- test/e2e/testdata/smoketest/playgroundtest.go | 16 ++++++++-------- test/testdata/addon/addon.yaml | 2 +- test/testdata/cue_testdata/mysql.cue | 6 +++--- test/testdata/cue_testdata/mysql_for_cli.cue | 6 +++--- test/testdata/cue_testdata/mysql_openapi.cue | 6 +++--- test/testdata/cue_testdata/mysql_openapi.json | 2 +- test/testdata/cue_testdata/mysql_openapi_v2.cue | 2 +- test/testdata/cue_testdata/mysql_openapi_v2.json | 2 +- test/testdata/cue_testdata/mysql_simple.cue | 4 ++-- test/testdata/cue_testdata/wesql.cnf | 2 +- test/testdata/cue_testdata/wesql.cue | 6 +++--- .../resources/mysql-config-constraint.yaml | 6 +++--- .../mysql-consensus-config-constraint.yaml | 6 +++--- .../mysql-consensus-config-template.yaml | 2 +- 73 files changed, 146 insertions(+), 147 deletions(-) rename internal/configuration/config_manager/{dynamic_paramter_updater.go => dynamic_parameter_updater.go} (100%) rename internal/configuration/{constrant.go => constraint.go} (100%) diff --git a/.github/utils/typos.toml b/.github/utils/typos.toml index 862285ef0..8f9113a2a 100644 --- a/.github/utils/typos.toml +++ b/.github/utils/typos.toml @@ -1,6 +1,5 @@ [default.extend-identifiers] # *sigh* this just isn't worth the cost of fixing -ClusterPhaseMisMatch = "ClusterPhaseMisMatch" [default.extend-words] # Don't correct the "AKS" diff --git a/apis/apps/v1alpha1/cluster_webhook_test.go b/apis/apps/v1alpha1/cluster_webhook_test.go index 9488f3334..ca2d91c67 100644 --- a/apis/apps/v1alpha1/cluster_webhook_test.go +++ b/apis/apps/v1alpha1/cluster_webhook_test.go @@ -309,7 +309,7 @@ spec: return cluster, err } -func createTestReplicationSetCluster(clusterDefinitionName, clusterVerisonName, clusterName string) (*Cluster, error) { +func createTestReplicationSetCluster(clusterDefinitionName, clusterVersionName, clusterName string) (*Cluster, error) { clusterYaml := fmt.Sprintf(` apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster @@ -333,7 +333,7 @@ spec: resources: requests: storage: 1Gi -`, clusterName, clusterDefinitionName, clusterVerisonName) +`, clusterName, clusterDefinitionName, clusterVersionName) cluster := &Cluster{} err := yaml.Unmarshal([]byte(clusterYaml), cluster) cluster.Spec.TerminationPolicy = WipeOut diff --git a/apis/apps/v1alpha1/clusterdefinition_types.go b/apis/apps/v1alpha1/clusterdefinition_types.go index ae412fdaa..6e9886a7c 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types.go +++ b/apis/apps/v1alpha1/clusterdefinition_types.go @@ -43,13 +43,13 @@ type ClusterDefinitionSpec struct { ComponentDefs []ClusterComponentDefinition `json:"componentDefs" patchStrategy:"merge,retainKeys" patchMergeKey:"name"` // Connection credential template used for creating a connection credential - // secret for cluster.apps.kubeblock.io object. Built-in objects are: + // secret for cluster.apps.kubeblocks.io object. Built-in objects are: // `$(RANDOM_PASSWD)` - random 8 characters. // `$(UUID)` - generate a random UUID v4 string. // `$(UUID_B64)` - generate a random UUID v4 BASE64 encoded string``. // `$(UUID_STR_B64)` - generate a random UUID v4 string then BASE64 encoded``. - // `$(UUID_HEX)` - generate a random UUID v4 wth HEX representation``. - // `$(HEADLESS_SVC_FQDN)` - headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, + // `$(UUID_HEX)` - generate a random UUID v4 HEX representation``. + // `$(HEADLESS_SVC_FQDN)` - headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, // where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute; // `$(SVC_FQDN)` - service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME).$(NAMESPACE).svc, // where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute; diff --git a/apis/apps/v1alpha1/clusterdefinition_webhook.go b/apis/apps/v1alpha1/clusterdefinition_webhook.go index b1bcccca8..d8ed420ad 100644 --- a/apis/apps/v1alpha1/clusterdefinition_webhook.go +++ b/apis/apps/v1alpha1/clusterdefinition_webhook.go @@ -234,7 +234,7 @@ func (r *SystemAccountSpec) validateSysAccounts(allErrs *field.ErrorList) { if _, exists := accountName[sysAccount.Name]; exists { *allErrs = append(*allErrs, field.Invalid(field.NewPath("spec.components[*].systemAccounts.accounts"), - sysAccount.Name, "duplicated system account names are not allowd.")) + sysAccount.Name, "duplicated system account names are not allowed.")) continue } else { accountName[sysAccount.Name] = true diff --git a/apis/apps/v1alpha1/clusterversion_webhook_test.go b/apis/apps/v1alpha1/clusterversion_webhook_test.go index 2921c93a3..afcfff505 100644 --- a/apis/apps/v1alpha1/clusterversion_webhook_test.go +++ b/apis/apps/v1alpha1/clusterversion_webhook_test.go @@ -58,7 +58,7 @@ var _ = Describe("clusterVersion webhook", func() { Expect(testCtx.CreateObj(ctx, clusterDef)).Should(Succeed()) Eventually(func() bool { - By("By testing component name is not found in cluserDefinition") + By("By testing component name is not found in clusterDefinition") clusterVersion.Spec.ComponentVersions[1].ComponentDefRef = "proxy1" Expect(testCtx.CheckedCreateObj(ctx, clusterVersion)).ShouldNot(Succeed()) diff --git a/apis/apps/v1alpha1/opsrequest_conditions.go b/apis/apps/v1alpha1/opsrequest_conditions.go index df9e58233..f7c33b056 100644 --- a/apis/apps/v1alpha1/opsrequest_conditions.go +++ b/apis/apps/v1alpha1/opsrequest_conditions.go @@ -48,7 +48,7 @@ const ( ReasonReconfigureNoChanged = "ReconfigureNoChanged" ReasonReconfigureSucceed = "ReconfigureSucceed" ReasonReconfigureRunning = "ReconfigureRunning" - ReasonClusterPhaseMisMatch = "ClusterPhaseMisMatch" + ReasonClusterPhaseMismatch = "ClusterPhaseMismatch" ReasonOpsTypeNotSupported = "OpsTypeNotSupported" ReasonValidateFailed = "ValidateFailed" ReasonClusterNotFound = "ClusterNotFound" diff --git a/apis/apps/v1alpha1/opsrequest_conditions_test.go b/apis/apps/v1alpha1/opsrequest_conditions_test.go index 9d25dd37b..9a1f924ef 100644 --- a/apis/apps/v1alpha1/opsrequest_conditions_test.go +++ b/apis/apps/v1alpha1/opsrequest_conditions_test.go @@ -33,7 +33,7 @@ func TestNewAllCondition(t *testing.T) { NewSucceedCondition(opsRequest) NewVerticalScalingCondition(opsRequest) NewUpgradingCondition(opsRequest) - NewValidateFailedCondition(ReasonClusterPhaseMisMatch, "fail") + NewValidateFailedCondition(ReasonClusterPhaseMismatch, "fail") NewFailedCondition(opsRequest, nil) NewFailedCondition(opsRequest, errors.New("opsRequest run failed")) diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index 0406751fe..05d7862ee 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -219,13 +219,13 @@ func (r *OpsRequest) validateUpgrade(ctx context.Context, } // validateVerticalScaling validates api when spec.type is VerticalScaling -func (r *OpsRequest) validateVerticalScaling(ctx context.Context, k8sCLient client.Client, cluster *Cluster) error { +func (r *OpsRequest) validateVerticalScaling(ctx context.Context, k8sClient client.Client, cluster *Cluster) error { verticalScalingList := r.Spec.VerticalScalingList if len(verticalScalingList) == 0 { return notEmptyError("spec.verticalScaling") } - compClasses, err := getClasses(ctx, k8sCLient, cluster.Spec.ClusterDefRef) + compClasses, err := getClasses(ctx, k8sClient, cluster.Spec.ClusterDefRef) if err != nil { return nil } diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index 58786882b..f16bdf80e 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -309,7 +309,7 @@ type OpsRecorder struct { type ProvisionPolicyType string const ( - // CreateByStmt will create account w.r.t. deleteion and creation statement given by provider. + // CreateByStmt will create account w.r.t. deletion and creation statement given by provider. CreateByStmt ProvisionPolicyType = "CreateByStmt" // ReferToExisting will not create account, but create a secret by copying data from referred secret file. ReferToExisting ProvisionPolicyType = "ReferToExisting" diff --git a/apis/extensions/v1alpha1/addon_types_test.go b/apis/extensions/v1alpha1/addon_types_test.go index 28c91e78e..f8ebdd66b 100644 --- a/apis/extensions/v1alpha1/addon_types_test.go +++ b/apis/extensions/v1alpha1/addon_types_test.go @@ -266,7 +266,7 @@ func TestHelmInstallSpecBuildMergedValues(t *testing.T) { }, } - bulidInstallSpecItem := func() AddonInstallSpecItem { + buildInstallSpecItem := func() AddonInstallSpecItem { toleration := []map[string]string{ { "key": "taint-key", @@ -296,11 +296,11 @@ func TestHelmInstallSpecBuildMergedValues(t *testing.T) { } installSpec := AddonInstallSpec{ - AddonInstallSpecItem: bulidInstallSpecItem(), + AddonInstallSpecItem: buildInstallSpecItem(), ExtraItems: []AddonInstallExtraItem{ { Name: "extra", - AddonInstallSpecItem: bulidInstallSpecItem(), + AddonInstallSpecItem: buildInstallSpecItem(), }, }, } diff --git a/cmd/reloader/container_killer/killer.go b/cmd/reloader/container_killer/killer.go index 2b3564240..1e8e48002 100644 --- a/cmd/reloader/container_killer/killer.go +++ b/cmd/reloader/container_killer/killer.go @@ -42,17 +42,17 @@ var logger logr.Logger func main() { var containerRuntime cfgutil.CRIType var runtimeEndpoint string - var contaienrID []string + var containerID []string pflag.StringVar((*string)(&containerRuntime), "container-runtime", "auto", "the config set cri runtime type.") pflag.StringVar(&runtimeEndpoint, "runtime-endpoint", runtimeEndpoint, "the config set cri runtime endpoint.") - pflag.StringArrayVar(&contaienrID, - "container-id", contaienrID, "the container-id killed.") + pflag.StringArrayVar(&containerID, + "container-id", containerID, "the container-id killed.") pflag.Parse() - if len(contaienrID) == 0 { + if len(containerID) == 0 { fmt.Fprintf(os.Stderr, "require container-id!\n\n") pflag.Usage() os.Exit(-1) @@ -76,7 +76,7 @@ func main() { logger.Error(err, "failed to init killer") } - if err := killer.Kill(context.Background(), contaienrID, viper.GetString(cfgutil.KillContainerSignalEnvName), nil); err != nil { - logger.Error(err, fmt.Sprintf("failed to kill container[%s]", contaienrID)) + if err := killer.Kill(context.Background(), containerID, viper.GetString(cfgutil.KillContainerSignalEnvName), nil); err != nil { + logger.Error(err, fmt.Sprintf("failed to kill container[%s]", containerID)) } } diff --git a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml index 883f51f57..a7f8ced49 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml @@ -8718,13 +8718,13 @@ spec: additionalProperties: type: string description: 'Connection credential template used for creating a connection - credential secret for cluster.apps.kubeblock.io object. Built-in + credential secret for cluster.apps.kubeblocks.io object. Built-in objects are: `$(RANDOM_PASSWD)` - random 8 characters. `$(UUID)` - generate a random UUID v4 string. `$(UUID_B64)` - generate a random UUID v4 BASE64 encoded string``. `$(UUID_STR_B64)` - generate a random UUID v4 string then BASE64 encoded``. `$(UUID_HEX)` - generate - a random UUID v4 wth HEX representation``. `$(HEADLESS_SVC_FQDN)` - - headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, + a random UUID v4 HEX representation``. `$(HEADLESS_SVC_FQDN)` - + headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute; `$(SVC_FQDN)` - service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME).$(NAMESPACE).svc, where 1ST_COMP_NAME diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 808c9ae69..eac2bf038 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -839,7 +839,7 @@ var _ = Describe("Cluster Controller", func() { testClusterAffinity := func() { const topologyKey = "testTopologyKey" - const lableKey = "testNodeLabelKey" + const labelKey = "testNodeLabelKey" const labelValue = "testLabelValue" By("Creating a cluster with Affinity") @@ -847,7 +847,7 @@ var _ = Describe("Cluster Controller", func() { PodAntiAffinity: appsv1alpha1.Required, TopologyKeys: []string{topologyKey}, NodeLabels: map[string]string{ - lableKey: labelValue, + labelKey: labelValue, }, Tenancy: appsv1alpha1.SharedNode, } @@ -866,7 +866,7 @@ var _ = Describe("Cluster Controller", func() { Eventually(func(g Gomega) { stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) podSpec := stsList.Items[0].Spec.Template.Spec - g.Expect(podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal(lableKey)) + g.Expect(podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal(labelKey)) g.Expect(podSpec.TopologySpreadConstraints[0].WhenUnsatisfiable).To(Equal(corev1.DoNotSchedule)) g.Expect(podSpec.TopologySpreadConstraints[0].TopologyKey).To(Equal(topologyKey)) g.Expect(podSpec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution).Should(HaveLen(1)) diff --git a/controllers/apps/components/component_status.go b/controllers/apps/components/component_status.go index 8a90270e6..c7b25a0c9 100644 --- a/controllers/apps/components/component_status.go +++ b/controllers/apps/components/component_status.go @@ -143,7 +143,7 @@ func (cs *ComponentStatusSynchronizer) Update(ctx context.Context, obj client.Ob } // hasFailedAndTimedOutPod returns whether the pod of components is still failed after a PodFailedTimeout period. -// if return ture, component phase will be set to Failed/Abnormal. +// if return true, component phase will be set to Failed/Abnormal. func (cs *ComponentStatusSynchronizer) hasFailedAndTimedOutPod() (hasFailedAndTimedoutPod bool, hasFailedPod bool) { // init a new ComponentMessageMap to store the message of failed pods message := appsv1alpha1.ComponentMessageMap{} diff --git a/controllers/apps/components/consensus/consensus_utils_test.go b/controllers/apps/components/consensus/consensus_utils_test.go index 90aeaa29a..20ce8473a 100644 --- a/controllers/apps/components/consensus/consensus_utils_test.go +++ b/controllers/apps/components/consensus/consensus_utils_test.go @@ -69,7 +69,7 @@ func TestInitClusterComponentStatusIfNeed(t *testing.T) { } if len(cluster.Status.Components) == 0 { - t.Errorf("cluster.Status.ComponentDefs[*] not intialized properly") + t.Errorf("cluster.Status.ComponentDefs[*] not initialized properly") } if _, ok := cluster.Status.Components[componentName]; !ok { t.Errorf("cluster.Status.ComponentDefs[componentName] not initialized properly") diff --git a/controllers/apps/components/stateful_set_controller_test.go b/controllers/apps/components/stateful_set_controller_test.go index 530723000..07679a0ab 100644 --- a/controllers/apps/components/stateful_set_controller_test.go +++ b/controllers/apps/components/stateful_set_controller_test.go @@ -127,7 +127,7 @@ var _ = Describe("StatefulSet Controller", func() { g.Expect(compStatus.Phase).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) // original expecting value RebootingPhase g.Expect(compStatus.PodsReady).ShouldNot(BeNil()) g.Expect(*compStatus.PodsReady).Should(BeTrue()) - // REVIEW/TODO: ought add extra condtion check for RebootingPhase + // REVIEW/TODO: ought add extra condition check for RebootingPhase })).Should(Succeed()) By("add leader role label for leaderPod to mock consensus component to be Running") diff --git a/controllers/apps/components/util/stateful_set_utils.go b/controllers/apps/components/util/stateful_set_utils.go index 06d7fe77d..4a33557b3 100644 --- a/controllers/apps/components/util/stateful_set_utils.go +++ b/controllers/apps/components/util/stateful_set_utils.go @@ -49,7 +49,7 @@ func IsMemberOf(set *appsv1.StatefulSet, pod *corev1.Pod) bool { return getParentName(pod) == set.Name } -// IsStsAndPodsRevisionConsistent checks if StatefulSet and pods of the StatefuleSet have the same revison, +// IsStsAndPodsRevisionConsistent checks if StatefulSet and pods of the StatefulSet have the same revision. func IsStsAndPodsRevisionConsistent(ctx context.Context, cli client.Client, sts *appsv1.StatefulSet) (bool, error) { pods, err := GetPodListByStatefulSet(ctx, cli, sts) if err != nil { diff --git a/controllers/apps/operations/volume_expansion_updater.go b/controllers/apps/operations/volume_expansion_updater.go index 2548683a6..1f5621103 100644 --- a/controllers/apps/operations/volume_expansion_updater.go +++ b/controllers/apps/operations/volume_expansion_updater.go @@ -96,7 +96,7 @@ func handleClusterVolumeExpandingPhase(ctx context.Context, cluster.Status.SetComponentStatus(k, v) } } - // REVIEW: a single component status affect cluser level status? + // REVIEW: a single component status affect cluster level status? cluster.Status.Phase = appsv1alpha1.RunningClusterPhase return cli.Status().Patch(ctx, cluster, patch) } diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index 93bcf84b0..aa057388f 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -107,7 +107,7 @@ var _ = Describe("OpsRequest Controller", func() { mockSetClusterStatusPhaseToRunning := func(namespacedName types.NamespacedName) { Expect(testapps.GetAndChangeObjStatus(&testCtx, namespacedName, func(fetched *appsv1alpha1.Cluster) { - // TODO: whould be better to have hint for cluster.status.phase is mocked, + // TODO: would be better to have hint for cluster.status.phase is mocked, // i.e., add annotation info for the mocked context fetched.Status.Phase = appsv1alpha1.RunningClusterPhase if len(fetched.Status.Components) == 0 { diff --git a/controllers/apps/systemaccount_controller.go b/controllers/apps/systemaccount_controller.go index 5b3ecb33a..7ab7a6285 100644 --- a/controllers/apps/systemaccount_controller.go +++ b/controllers/apps/systemaccount_controller.go @@ -49,8 +49,8 @@ type SystemAccountReconciler struct { SecretMapStore *secretMapStore } -// jobCompleditionPredicate implements a default delete predicate function on job deletion. -type jobCompletitionPredicate struct { +// jobCompletionPredicate implements a default delete predicate function on job deletion. +type jobCompletionPredicate struct { predicate.Funcs reconciler *SystemAccountReconciler Log logr.Logger @@ -90,7 +90,7 @@ const ( ) // compile-time assert that the local data object satisfies the phases data interface. -var _ predicate.Predicate = &jobCompletitionPredicate{} +var _ predicate.Predicate = &jobCompletionPredicate{} // compile-time assert that the local data object satisfies the phases data interface. var _ predicate.Predicate = &clusterDeletionPredicate{} @@ -255,7 +255,7 @@ func (r *SystemAccountReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Watches(&source.Kind{Type: &batchv1.Job{}}, &handler.EnqueueRequestForObject{}, - builder.WithPredicates(&jobCompletitionPredicate{reconciler: r, Log: log.FromContext(context.TODO())})). + builder.WithPredicates(&jobCompletionPredicate{reconciler: r, Log: log.FromContext(context.TODO())})). Complete(r) } @@ -371,7 +371,7 @@ func (r *SystemAccountReconciler) getAccountFacts(reqCtx intctrlutil.RequestCtx, // Delete implements default DeleteEvent filter on job deletion. // If the job for creating account completes successfully, corresponding secret will be created. -func (r *jobCompletitionPredicate) Delete(e event.DeleteEvent) bool { +func (r *jobCompletionPredicate) Delete(e event.DeleteEvent) bool { if e.Object == nil { return false } diff --git a/controllers/apps/systemaccount_controller_test.go b/controllers/apps/systemaccount_controller_test.go index bbf588303..a0f7ca414 100644 --- a/controllers/apps/systemaccount_controller_test.go +++ b/controllers/apps/systemaccount_controller_test.go @@ -56,7 +56,7 @@ var _ = Describe("SystemAccount Controller", func() { * 2. create two clusters, one cluster for each component, and verify * a) the number of secrets, jobs, and cached secrets are as expected * b) secret will be created, once corresponding job succeeds. - * c) secrets, deleted accidentially, will be re-created during next cluster reconciliation round. + * c) secrets, deleted accidentally, will be re-created during next cluster reconciliation round. * * Each test case, used in following IT(integration test), consists of two parts: * a) how to build the test cluster, and @@ -186,7 +186,7 @@ var _ = Describe("SystemAccount Controller", func() { } initSysAccountTestsAndCluster := func(testCases map[string]*sysAcctTestCase) (clustersMap map[string]types.NamespacedName) { - // create clusterdef and cluster verions, but not clusters + // create clusterdef and cluster versions, but not clusters By("Create a clusterDefinition obj") systemAccount := mockSystemAccountsSpec() clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). @@ -346,7 +346,7 @@ var _ = Describe("SystemAccount Controller", func() { return len(systemAccountReconciler.SecretMapStore.ListKeys()) }).Should(BeEquivalentTo(cachedSecretNum)) - By("Verify all jobs created have their lables set correctly") + By("Verify all jobs created have their labels set correctly") // get all jobs Eventually(func(g Gomega) { // all jobs matching filter `ml` should be a job for sys account. @@ -450,8 +450,8 @@ var _ = Describe("SystemAccount Controller", func() { g.Expect(len(secrets.Items)).To(BeEquivalentTo(secretsNum + cachedSecretNum)) }).Should(Succeed()) - By("Verify all secrets created have their finalizer and lables set correctly") - // get all secrets, and check their lables and finalizer + By("Verify all secrets created have their finalizer and labels set correctly") + // get all secrets, and check their labels and finalizer Eventually(func(g Gomega) { // get secrets matching filter secretsForAcct := &corev1.SecretList{} diff --git a/controllers/apps/systemaccount_util.go b/controllers/apps/systemaccount_util.go index b5c6abcff..666a80ac9 100644 --- a/controllers/apps/systemaccount_util.go +++ b/controllers/apps/systemaccount_util.go @@ -306,7 +306,7 @@ func getCreationStmtForAccount(key componentUniqueKey, passConfig appsv1alpha1.P accountConfig appsv1alpha1.SystemAccountConfig) ([]string, *corev1.Secret) { // generated password with mixedcases = true passwd, _ := password.Generate((int)(passConfig.Length), (int)(passConfig.NumDigits), (int)(passConfig.NumSymbols), false, false) - // refine pasword to upper or lower cases w.r.t configuration + // refine password to upper or lower cases w.r.t configuration switch passConfig.LetterCase { case appsv1alpha1.UpperCases: passwd = strings.ToUpper(passwd) diff --git a/controllers/extensions/addon_controller.go b/controllers/extensions/addon_controller.go index 92457a8eb..f748697f1 100644 --- a/controllers/extensions/addon_controller.go +++ b/controllers/extensions/addon_controller.go @@ -137,14 +137,14 @@ func (r *AddonReconciler) SetupWithManager(mgr ctrl.Manager) error { // TODO: replace with controller-idioms's adopt lib // Watches(&source.Kind{Type: &batchv1.Job{}}, // &handler.EnqueueRequestForObject{}, - // builder.WithPredicates(&jobCompletitionPredicate{reconciler: r, Log: log.FromContext(context.TODO())})). + // builder.WithPredicates(&jobCompletionPredicate{reconciler: r, Log: log.FromContext(context.TODO())})). WithOptions(controller.Options{ MaxConcurrentReconciles: viper.GetInt(maxConcurrentReconcilesKey), }). Complete(r) } -// type jobCompletitionPredicate struct { +// type jobCompletionPredicate struct { // predicate.Funcs // reconciler *AddonReconciler // Log logr.Logger diff --git a/deploy/apecloud-mysql-scale/Chart.yaml b/deploy/apecloud-mysql-scale/Chart.yaml index 09528eb08..51126d007 100644 --- a/deploy/apecloud-mysql-scale/Chart.yaml +++ b/deploy/apecloud-mysql-scale/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: apecloud-mysql-scale description: ApeCloud MySQL-Scale is ApeCloud MySQL proxy. ApeCloud MySQL-Scale can support - apecloud mysql failover automatically and read write seperation. + apecloud mysql failover automatically and read write separation. type: application diff --git a/deploy/apecloud-mysql-scale/config/mysql8-config-constraint.cue b/deploy/apecloud-mysql-scale/config/mysql8-config-constraint.cue index 24108dc7f..e2fc384c5 100644 --- a/deploy/apecloud-mysql-scale/config/mysql8-config-constraint.cue +++ b/deploy/apecloud-mysql-scale/config/mysql8-config-constraint.cue @@ -83,7 +83,7 @@ // If this variable is enabled (the default), transactions are committed in the same order they are written to the binary log. If disabled, transactions may be committed in parallel. binlog_order_commits?: string & "0" | "1" | "OFF" | "ON" - // Whether the server logs full or minmal rows with row-based replication. + // Whether the server logs full or minimal rows with row-based replication. binlog_row_image?: string & "FULL" | "MINIMAL" | "NOBLOB" // Controls whether metadata is logged using FULL or MINIMAL format. FULL causes all metadata to be logged; MINIMAL means that only metadata actually required by slave is logged. Default: MINIMAL. @@ -1526,8 +1526,8 @@ // For SQL window functions, determines whether to enable inversion optimization for moving window frames also for floating values. windowing_use_high_precision: string & "0" | "1" | "OFF" | "ON" | *"1" - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } diff --git a/deploy/apecloud-mysql-scale/config/mysql8-config.tpl b/deploy/apecloud-mysql-scale/config/mysql8-config.tpl index a1b8a2a12..43a59d898 100644 --- a/deploy/apecloud-mysql-scale/config/mysql8-config.tpl +++ b/deploy/apecloud-mysql-scale/config/mysql8-config.tpl @@ -143,7 +143,7 @@ log_bin_index=mysql-bin.index max_binlog_size=134217728 log_replica_updates=1 # binlog_rows_query_log_events=ON #AWS not set -# binlog_transaction_dependency_tracking=WRITESET #Defautl Commit Order, Aws not set +# binlog_transaction_dependency_tracking=WRITESET #Default Commit Order, Aws not set # replay log # relay_log_info_repository=TABLE diff --git a/deploy/apecloud-mysql/config/mysql8-config-constraint.cue b/deploy/apecloud-mysql/config/mysql8-config-constraint.cue index c63b7c9d1..3cc107f31 100644 --- a/deploy/apecloud-mysql/config/mysql8-config-constraint.cue +++ b/deploy/apecloud-mysql/config/mysql8-config-constraint.cue @@ -83,7 +83,7 @@ // If this variable is enabled (the default), transactions are committed in the same order they are written to the binary log. If disabled, transactions may be committed in parallel. binlog_order_commits?: string & "0" | "1" | "OFF" | "ON" - // Whether the server logs full or minmal rows with row-based replication. + // Whether the server logs full or minimal rows with row-based replication. binlog_row_image?: string & "FULL" | "MINIMAL" | "NOBLOB" // Controls whether metadata is logged using FULL or MINIMAL format. FULL causes all metadata to be logged; MINIMAL means that only metadata actually required by slave is logged. Default: MINIMAL. @@ -1529,8 +1529,8 @@ // For SQL window functions, determines whether to enable inversion optimization for moving window frames also for floating values. windowing_use_high_precision: string & "0" | "1" | "OFF" | "ON" | *"1" - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } diff --git a/deploy/apecloud-mysql/config/mysql8-config.tpl b/deploy/apecloud-mysql/config/mysql8-config.tpl index 133a24773..eacd9acf3 100644 --- a/deploy/apecloud-mysql/config/mysql8-config.tpl +++ b/deploy/apecloud-mysql/config/mysql8-config.tpl @@ -144,7 +144,7 @@ log_bin_index=mysql-bin.index max_binlog_size=134217728 log_replica_updates=1 # binlog_rows_query_log_events=ON #AWS not set -# binlog_transaction_dependency_tracking=WRITESET #Defautl Commit Order, Aws not set +# binlog_transaction_dependency_tracking=WRITESET #Default Commit Order, Aws not set # replay log # relay_log_info_repository=TABLE diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml index 883f51f57..a7f8ced49 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml @@ -8718,13 +8718,13 @@ spec: additionalProperties: type: string description: 'Connection credential template used for creating a connection - credential secret for cluster.apps.kubeblock.io object. Built-in + credential secret for cluster.apps.kubeblocks.io object. Built-in objects are: `$(RANDOM_PASSWD)` - random 8 characters. `$(UUID)` - generate a random UUID v4 string. `$(UUID_B64)` - generate a random UUID v4 BASE64 encoded string``. `$(UUID_STR_B64)` - generate a random UUID v4 string then BASE64 encoded``. `$(UUID_HEX)` - generate - a random UUID v4 wth HEX representation``. `$(HEADLESS_SVC_FQDN)` - - headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, + a random UUID v4 HEX representation``. `$(HEADLESS_SVC_FQDN)` - + headless service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME)-headless.$(NAMESPACE).svc, where 1ST_COMP_NAME is the 1st component that provide `ClusterDefinition.spec.componentDefs[].service` attribute; `$(SVC_FQDN)` - service FQDN placeholder, value pattern - $(CLUSTER_NAME)-$(1ST_COMP_NAME).$(NAMESPACE).svc, where 1ST_COMP_NAME diff --git a/deploy/helm/templates/addons/nyancat-addon.yaml b/deploy/helm/templates/addons/nyancat-addon.yaml index 4fde96f1e..bf28d5cf1 100644 --- a/deploy/helm/templates/addons/nyancat-addon.yaml +++ b/deploy/helm/templates/addons/nyancat-addon.yaml @@ -11,7 +11,7 @@ metadata: {{- end }} spec: description: 'Deploys a nyancat application in a cluster. - Nyancat is a demo application for showing database cluster availibility.' + Nyancat is a demo application for showing database cluster availability.' type: Helm helm: diff --git a/deploy/kafka/scripts/libkafka.sh b/deploy/kafka/scripts/libkafka.sh index 025dc88c9..cf8182aa2 100644 --- a/deploy/kafka/scripts/libkafka.sh +++ b/deploy/kafka/scripts/libkafka.sh @@ -616,7 +616,7 @@ kafka_configure_internal_communications() { if [[ -n "$KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL" ]]; then kafka_server_conf_set sasl.mechanism.inter.broker.protocol "$KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL" else - error "When using SASL for inter broker comunication the mechanism should be provided at KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL" + error "When using SASL for inter broker communication the mechanism should be provided at KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL" exit 1 fi fi diff --git a/deploy/weaviate/values.yaml b/deploy/weaviate/values.yaml index b69d52ff0..764cc2975 100644 --- a/deploy/weaviate/values.yaml +++ b/deploy/weaviate/values.yaml @@ -61,7 +61,7 @@ resources: {} # memory: '1Gi' -# Add a service account ot the Weaviate pods if you need Weaviate to have permissions to +# Add a service account to the Weaviate pods if you need Weaviate to have permissions to # access kubernetes resources or cloud provider resources. For example for it to have # access to a backup up bucket, or if you want to restrict Weaviate pod in any way. # By default, use the default ServiceAccount diff --git a/docker/Dockerfile b/docker/Dockerfile index e6082854b..12d7db318 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # Build the manager binary FROM --platform=${BUILDPLATFORM} golang:1.20 as builder -## docker buildx buid injected build-args: +## docker buildx build injected build-args: #BUILDPLATFORM — matches the current machine. (e.g. linux/amd64) #BUILDOS — os component of BUILDPLATFORM, e.g. linux #BUILDARCH — e.g. amd64, arm64, riscv64 diff --git a/docker/Dockerfile-tools b/docker/Dockerfile-tools index 3c10dbff6..1fa3b38fb 100644 --- a/docker/Dockerfile-tools +++ b/docker/Dockerfile-tools @@ -2,7 +2,7 @@ # includes kbcli, kubectl, and manager tools. FROM --platform=${BUILDPLATFORM} golang:1.20 as builder -## docker buildx buid injected build-args: +## docker buildx build injected build-args: #BUILDPLATFORM — matches the current machine. (e.g. linux/amd64) #BUILDOS — os component of BUILDPLATFORM, e.g. linux #BUILDARCH — e.g. amd64, arm64, riscv64 diff --git a/docker/custom-scripts/kubectl-helm-debian.sh b/docker/custom-scripts/kubectl-helm-debian.sh index e0ee37dc4..0bf9d8f02 100644 --- a/docker/custom-scripts/kubectl-helm-debian.sh +++ b/docker/custom-scripts/kubectl-helm-debian.sh @@ -26,7 +26,7 @@ docker/#!/usr/bin/env bash # Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/kubectl-helm.md # Maintainer: The VS Code and Codespaces Teams # -# Syntax: ./kubectl-helm-debian.sh [kubectl verison] [Helm version] [minikube version] [kubectl SHA256] [Helm SHA256] [minikube SHA256] +# Syntax: ./kubectl-helm-debian.sh [kubectl version] [Helm version] [minikube version] [kubectl SHA256] [Helm SHA256] [minikube SHA256] set -e diff --git a/docker/library-scripts/go-debian.sh b/docker/library-scripts/go-debian.sh index 0e3400d92..f109b289f 100644 --- a/docker/library-scripts/go-debian.sh +++ b/docker/library-scripts/go-debian.sh @@ -156,7 +156,7 @@ fi usermod -a -G golang "${USERNAME}" mkdir -p "${TARGET_GOROOT}" "${TARGET_GOPATH}" if [ "${TARGET_GO_VERSION}" != "none" ] && ! type go > /dev/null 2>&1; then - # Use a temporary locaiton for gpg keys to avoid polluting image + # Use a temporary location for gpg keys to avoid polluting image export GNUPGHOME="/tmp/tmp-gnupg" mkdir -p ${GNUPGHOME} chmod 700 ${GNUPGHOME} diff --git a/docker/library-scripts/kubectl-helm-debian.sh b/docker/library-scripts/kubectl-helm-debian.sh index 6e4a7ca2c..0c1bd32a5 100644 --- a/docker/library-scripts/kubectl-helm-debian.sh +++ b/docker/library-scripts/kubectl-helm-debian.sh @@ -26,7 +26,7 @@ docker/#!/usr/bin/env bash # Docs: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/kubectl-helm.md # Maintainer: The VS Code and Codespaces Teams # -# Syntax: ./kubectl-helm-debian.sh [kubectl verison] [Helm version] [minikube version] [kubectl SHA256] [Helm SHA256] [minikube SHA256] +# Syntax: ./kubectl-helm-debian.sh [kubectl version] [Helm version] [minikube version] [kubectl SHA256] [Helm SHA256] [minikube SHA256] set -e diff --git a/docs/release_notes/v0.1.0/v0.1.0.md b/docs/release_notes/v0.1.0/v0.1.0.md index ede0ea97d..761de440a 100644 --- a/docs/release_notes/v0.1.0/v0.1.0.md +++ b/docs/release_notes/v0.1.0/v0.1.0.md @@ -105,10 +105,10 @@ Thanks to everyone who made this release possible! - migrate KubeBlocks core driver operator ([#78](https://github.com/apecloud/kubeblocks/pull/78), @nashtsai) - support appVersion, clusterDefinition and cluster CR validating webhook ([#83](https://github.com/apecloud/kubeblocks/pull/83), @wangyelei) - fix ci-test and add badges ([#88](https://github.com/apecloud/kubeblocks/pull/88), @JashBook) -- Feature/unified dbcluser lifecycle ([#89](https://github.com/apecloud/kubeblocks/pull/89), @lynnleelhl) +- Feature/unified dbcluster lifecycle ([#89](https://github.com/apecloud/kubeblocks/pull/89), @lynnleelhl) - fix release publish ([#95](https://github.com/apecloud/kubeblocks/pull/95), @JashBook) - Support/csi driver volume testing ([#99](https://github.com/apecloud/kubeblocks/pull/99), @nashtsai) -- Feature/unified dbcluser lifecycle ([#100](https://github.com/apecloud/kubeblocks/pull/100), @lynnleelhl) +- Feature/unified dbcluster lifecycle ([#100](https://github.com/apecloud/kubeblocks/pull/100), @lynnleelhl) - add Cluster Status handling. ([#101](https://github.com/apecloud/kubeblocks/pull/101), @wangyelei) - Refactor/container to podspec ([#102](https://github.com/apecloud/kubeblocks/pull/102), @lynnleelhl) - CICD add staticcheck ([#106](https://github.com/apecloud/kubeblocks/pull/106), @JashBook) diff --git a/docs/user_docs/api/lifecycle-management/ops-request-api.md b/docs/user_docs/api/lifecycle-management/ops-request-api.md index 4b6d2b213..8af07a31c 100644 --- a/docs/user_docs/api/lifecycle-management/ops-request-api.md +++ b/docs/user_docs/api/lifecycle-management/ops-request-api.md @@ -256,7 +256,7 @@ It corresponds to `metadata.generation`. | HorizontalScalingStarted | HorizontalScaling started. | | VolumeExpandingStarted | VolumeExpanding started. | | UpgradingStarted | Upgrade started. | -| ClusterPhaseMisMatch | The cluster status mismatches. | +| ClusterPhaseMismatch | The cluster status mismatches. | | OpsTypeNotSupported | The cluster does not support this operation. | | ClusterExistOtherOperation | Another mutually exclusive operation is running.| | ClusterNotFound | The specified cluster is not found. | diff --git a/hack/install_cli.ps1 b/hack/install_cli.ps1 index b742c426c..e617a3405 100644 --- a/hack/install_cli.ps1 +++ b/hack/install_cli.ps1 @@ -107,10 +107,10 @@ function downloadFile { $timer.Interval = 500 Register-ObjectEvent -InputObject $timer -EventName Elapsed -SourceIdentifier "TimerElapsed" -Action { - $precent = $Global:Data.SourceArgs.ProgressPercentage + $percent = $Global:Data.SourceArgs.ProgressPercentage $totalBytes = $Global:Data.SourceArgs.TotalBytesToReceive $receivedBytes = $Global:Data.SourceArgs.BytesReceived - if ($precent -ne $null) { + if ($percent -ne $null) { $downloadProgress = [Math]::Round(($receivedBytes / $totalBytes) * 100, 2) $status = "Downloaded {0} of {1} bytes" -f $receivedBytes, $totalBytes Write-Progress -Activity "Downloading kbcli..." -Status $status -PercentComplete $downloadProgress diff --git a/internal/cli/cluster/helper.go b/internal/cli/cluster/helper.go index 304946ae1..332324ccf 100644 --- a/internal/cli/cluster/helper.go +++ b/internal/cli/cluster/helper.go @@ -118,14 +118,14 @@ func getInstanceInfoFromStatus(dynamic dynamic.Interface, name, componentName, n func getInstanceInfoByList(dynamic dynamic.Interface, name, componentName, namespace string) []*InstanceInfo { var infos []*InstanceInfo // filter by cluster name - lables := util.BuildLabelSelectorByNames("", []string{name}) + labels := util.BuildLabelSelectorByNames("", []string{name}) // filter by component name if len(componentName) > 0 { - lables = util.BuildComponentNameLables(lables, []string{componentName}) + labels = util.BuildComponentNameLabels(labels, []string{componentName}) } objs, err := dynamic.Resource(schema.GroupVersionResource{Group: corev1.GroupName, Version: types.K8sCoreAPIVersion, Resource: "pods"}). - Namespace(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: lables}) + Namespace(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labels}) if err != nil { return nil diff --git a/internal/cli/cmd/accounts/base.go b/internal/cli/cmd/accounts/base.go index 41d61d398..4f67cb92a 100644 --- a/internal/cli/cmd/accounts/base.go +++ b/internal/cli/cmd/accounts/base.go @@ -26,7 +26,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" - klog "k8s.io/klog/v2" + "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" clusterutil "github.com/apecloud/kubeblocks/internal/cli/cluster" @@ -222,14 +222,14 @@ func (o *AccountBaseOptions) printUserInfo(response sqlchannel.SQLChannelRespons o.printGeneralInfo(response) return nil } - // decode user info from metatdata + // decode user info from metadata users := []sqlchannel.UserInfo{} err := json.Unmarshal([]byte(response.Message), &users) if err != nil { return err } - // render user info with username and pasword expired boolean + // render user info with username and password expired boolean tblPrinter := o.newTblPrinterWithStyle("USER INFO", []interface{}{"USERNAME", "EXPIRED"}) for _, user := range users { tblPrinter.AddRow(user.UserName, user.Expired) @@ -245,7 +245,7 @@ func (o *AccountBaseOptions) printRoleInfo(response sqlchannel.SQLChannelRespons return nil } - // decode role info from metatdata + // decode role info from metadata users := []sqlchannel.UserInfo{} err := json.Unmarshal([]byte(response.Message), &users) if err != nil { diff --git a/internal/cli/cmd/accounts/create.go b/internal/cli/cmd/accounts/create.go index 3581c6c36..63e6d6617 100644 --- a/internal/cli/cmd/accounts/create.go +++ b/internal/cli/cmd/accounts/create.go @@ -63,7 +63,7 @@ func (o *CreateUserOptions) Complete(f cmdutil.Factory) error { if len(o.info.Password) == 0 { o.info.Password, _ = password.Generate(10, 2, 0, false, false) } - // encode user info to metatdata + // encode user info to metadata o.RequestMeta, err = struct2Map(o.info) return err } diff --git a/internal/cli/cmd/accounts/grant.go b/internal/cli/cmd/accounts/grant.go index 6980264a4..b70c9130d 100644 --- a/internal/cli/cmd/accounts/grant.go +++ b/internal/cli/cmd/accounts/grant.go @@ -23,7 +23,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/exp/slices" "k8s.io/cli-runtime/pkg/genericclioptions" - klog "k8s.io/klog/v2" + "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "github.com/apecloud/kubeblocks/internal/sqlchannel" @@ -50,7 +50,7 @@ func (o *GrantOptions) AddFlags(cmd *cobra.Command) { cmd.Flags().StringVarP(&o.info.RoleName, "role", "r", "", "Role name should be one of {SUPERUSER, READWRITE, READONLY}") } -func (o GrantOptions) Validate(args []string) error { +func (o *GrantOptions) Validate(args []string) error { if err := o.AccountBaseOptions.Validate(args); err != nil { return err } @@ -67,8 +67,8 @@ func (o GrantOptions) Validate(args []string) error { } func (o *GrantOptions) validRoleName() error { - candiates := []string{sqlchannel.SuperUserRole, sqlchannel.ReadWriteRole, sqlchannel.ReadOnlyRole} - if slices.Contains(candiates, strings.ToLower(o.info.RoleName)) { + candidates := []string{sqlchannel.SuperUserRole, sqlchannel.ReadWriteRole, sqlchannel.ReadOnlyRole} + if slices.Contains(candidates, strings.ToLower(o.info.RoleName)) { return nil } return errInvalidRoleName diff --git a/internal/cli/cmd/cluster/cluster.go b/internal/cli/cmd/cluster/cluster.go index cf0e8f7ff..3e26725ea 100644 --- a/internal/cli/cmd/cluster/cluster.go +++ b/internal/cli/cmd/cluster/cluster.go @@ -85,7 +85,7 @@ func NewClusterCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr Message: "Backup/Restore Commands:", Commands: []*cobra.Command{ NewListBackupPolicyCmd(f, streams), - NewLEditBackupPolicyCmd(f, streams), + NewEditBackupPolicyCmd(f, streams), NewCreateBackupCmd(f, streams), NewListBackupCmd(f, streams), NewDeleteBackupCmd(f, streams), diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index d3fa60944..92ab21873 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -717,7 +717,7 @@ func printBackupPolicyList(o list.ListOptions) error { return nil } -func NewLEditBackupPolicyCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { +func NewEditBackupPolicyCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := edit.NewEditOptions(f, streams, types.BackupPolicyGVR()) cmd := &cobra.Command{ Use: "edit-backup-policy", diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index b050070c1..6dc08bec9 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -272,7 +272,7 @@ func (o *initOptions) confirmInitNewKubeCluster() error { `) fmt.Fprintf(o.Out, ` -The whole process wll take about %s, please wait patiently, +The whole process will take about %s, please wait patiently, if it takes a long time, please check the network environment and try again. `, printer.BoldRed("20 minutes")) diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 1d060d61d..a62539d8b 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -679,13 +679,13 @@ func CombineLabels(labels map[string]string) string { return strings.Join(labelStr, ",") } -func BuildComponentNameLables(prefix string, names []string) string { - return buildLableSelectors(prefix, constant.KBAppComponentLabelKey, names) +func BuildComponentNameLabels(prefix string, names []string) string { + return buildLabelSelectors(prefix, constant.KBAppComponentLabelKey, names) } -// BuildLableSelectors build the label selector by given lable key, the label selector is +// buildLabelSelectors build the label selector by given label key, the label selector is // like "label-key in (name1, name2)" -func buildLableSelectors(prefix string, key string, names []string) string { +func buildLabelSelectors(prefix string, key string, names []string) string { if len(names) == 0 { return prefix } diff --git a/internal/configuration/config_manager/dynamic_paramter_updater.go b/internal/configuration/config_manager/dynamic_parameter_updater.go similarity index 100% rename from internal/configuration/config_manager/dynamic_paramter_updater.go rename to internal/configuration/config_manager/dynamic_parameter_updater.go diff --git a/internal/configuration/config_manager/handler_util_test.go b/internal/configuration/config_manager/handler_util_test.go index 3dd158c6a..35837cc47 100644 --- a/internal/configuration/config_manager/handler_util_test.go +++ b/internal/configuration/config_manager/handler_util_test.go @@ -92,7 +92,7 @@ func TestIsSupportReload(t *testing.T) { } } -var _ = Describe("Hander Util Test", func() { +var _ = Describe("Handler Util Test", func() { var mockK8sCli *testutil.K8sClientMockHelper diff --git a/internal/configuration/constrant.go b/internal/configuration/constraint.go similarity index 100% rename from internal/configuration/constrant.go rename to internal/configuration/constraint.go diff --git a/internal/configuration/container/container_kill_test.go b/internal/configuration/container/container_kill_test.go index cea6e0809..93660237a 100644 --- a/internal/configuration/container/container_kill_test.go +++ b/internal/configuration/container/container_kill_test.go @@ -128,7 +128,7 @@ func TestDockerContainerKill(t *testing.T) { // mock ContainerKill failed cli.EXPECT().ContainerKill(gomock.Any(), gomock.Any(), gomock.Any()). - Return(cfgcore.MakeError("faield to kill docker container!")) + Return(cfgcore.MakeError("failed to kill docker container!")) // mock ContainerKill success cli.EXPECT().ContainerKill(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil).AnyTimes() @@ -140,7 +140,7 @@ func TestDockerContainerKill(t *testing.T) { require.Nil(t, docker.Kill(context.Background(), []string{"76f9c2ae8cf47bfa43b97626e3c95045cb3b82c50019ab759801ab52e3acff55"}, "", nil)) require.ErrorContains(t, docker.Kill(context.Background(), []string{"76f9c2ae8cf47bfa43b97626e3c95045cb3b82c50019ab759801ab52e3acff55"}, "", nil), - "faield to kill docker container") + "failed to kill docker container") require.Nil(t, docker.Kill(context.Background(), []string{"76f9c2ae8cf47bfa43b97626e3c95045cb3b82c50019ab759801ab52e3acff55"}, "", nil)) } diff --git a/internal/controller/component/affinity_utils_test.go b/internal/controller/component/affinity_utils_test.go index 4352419d9..c62187650 100644 --- a/internal/controller/component/affinity_utils_test.go +++ b/internal/controller/component/affinity_utils_test.go @@ -44,7 +44,7 @@ var _ = Describe("affinity utils", func() { Context("with PodAntiAffinity set to Required", func() { const topologyKey = "testTopologyKey" - const lableKey = "testNodeLabelKey" + const labelKey = "testNodeLabelKey" const labelValue = "testLabelValue" BeforeEach(func() { @@ -60,7 +60,7 @@ var _ = Describe("affinity utils", func() { PodAntiAffinity: appsv1alpha1.Required, TopologyKeys: []string{topologyKey}, NodeLabels: map[string]string{ - lableKey: labelValue, + labelKey: labelValue, }, } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, @@ -85,7 +85,7 @@ var _ = Describe("affinity utils", func() { It("should have correct Affinity and TopologySpreadConstraints", func() { affinity := buildPodAffinity(clusterObj, clusterObj.Spec.Affinity, component) - Expect(affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).Should(Equal(lableKey)) + Expect(affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).Should(Equal(labelKey)) Expect(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey).Should(Equal(topologyKey)) Expect(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution).Should(BeEmpty()) @@ -101,7 +101,7 @@ var _ = Describe("affinity utils", func() { Context("with PodAntiAffinity set to Preferred", func() { const topologyKey = "testTopologyKey" - const lableKey = "testNodeLabelKey" + const labelKey = "testNodeLabelKey" const labelValue = "testLabelValue" BeforeEach(func() { @@ -117,7 +117,7 @@ var _ = Describe("affinity utils", func() { PodAntiAffinity: appsv1alpha1.Preferred, TopologyKeys: []string{topologyKey}, NodeLabels: map[string]string{ - lableKey: labelValue, + labelKey: labelValue, }, } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, @@ -143,7 +143,7 @@ var _ = Describe("affinity utils", func() { It("should have correct Affinity and TopologySpreadConstraints", func() { affinity := buildPodAffinity(clusterObj, clusterObj.Spec.Affinity, component) - Expect(affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).Should(Equal(lableKey)) + Expect(affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).Should(Equal(labelKey)) Expect(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution).Should(BeEmpty()) Expect(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight).ShouldNot(BeNil()) Expect(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.TopologyKey).Should(Equal(topologyKey)) diff --git a/internal/controller/component/component_test.go b/internal/controller/component/component_test.go index 7a1ca708b..dfd7f63d6 100644 --- a/internal/controller/component/component_test.go +++ b/internal/controller/component/component_test.go @@ -88,7 +88,7 @@ var _ = Describe("component module", func() { &clusterVersion.Spec.ComponentVersions[0]) Expect(component).ShouldNot(BeNil()) - By("leave clusterVersion.versionCtx empty initContains and conainers") + By("leave clusterVersion.versionCtx empty initContains and containers") clusterVersion.Spec.ComponentVersions[0].VersionsCtx.Containers = nil clusterVersion.Spec.ComponentVersions[0].VersionsCtx.InitContainers = nil component = BuildComponent( diff --git a/internal/controller/plan/builtin_env_test.go b/internal/controller/plan/builtin_env_test.go index c4bf0c374..386008879 100644 --- a/internal/controller/plan/builtin_env_test.go +++ b/internal/controller/plan/builtin_env_test.go @@ -188,7 +188,7 @@ bootstrap: }, }, { - Name: "invalid_contaienr", + Name: "invalid_container", }, }, } diff --git a/internal/controller/plan/config_template_test.go b/internal/controller/plan/config_template_test.go index eef8b270a..67928a2b9 100644 --- a/internal/controller/plan/config_template_test.go +++ b/internal/controller/plan/config_template_test.go @@ -133,7 +133,7 @@ single_thread_memory = 294912 }, }, { - Name: "invalid_contaienr", + Name: "invalid_container", }, }, Volumes: []corev1.Volume{ diff --git a/internal/preflight/analyzer/analyze_result.go b/internal/preflight/analyzer/analyze_result.go index 5444aaf39..64f2bb8ab 100644 --- a/internal/preflight/analyzer/analyze_result.go +++ b/internal/preflight/analyzer/analyze_result.go @@ -46,27 +46,27 @@ func newAnalyzeResult(title string, resultType string, outcomes []*troubleshoot. return newFailedResultWithMessage(title, MissingOutcomeMessage) } -func newFailAnalyzeResult(titile string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { +func newFailAnalyzeResult(title string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { return &analyze.AnalyzeResult{ - Title: titile, + Title: title, IsFail: true, Message: outcome.Fail.Message, URI: outcome.Fail.URI, } } -func newWarnAnalyzeResult(titile string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { +func newWarnAnalyzeResult(title string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { return &analyze.AnalyzeResult{ - Title: titile, + Title: title, IsWarn: true, Message: outcome.Warn.Message, URI: outcome.Warn.URI, } } -func newPassAnalyzeResult(titile string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { +func newPassAnalyzeResult(title string, outcome *troubleshoot.Outcome) *analyze.AnalyzeResult { return &analyze.AnalyzeResult{ - Title: titile, + Title: title, IsPass: true, Message: outcome.Pass.Message, URI: outcome.Pass.URI, diff --git a/internal/testutil/k8s/k8sclient_util.go b/internal/testutil/k8s/k8sclient_util.go index 47a19df72..76935a988 100644 --- a/internal/testutil/k8s/k8sclient_util.go +++ b/internal/testutil/k8s/k8sclient_util.go @@ -263,16 +263,16 @@ type MockGetReturned struct { func WithConstructSequenceResult(mockObjs map[client.ObjectKey][]MockGetReturned) HandleGetReturnedObject { sequenceAccessCounter := make(map[client.ObjectKey]int, len(mockObjs)) return func(key client.ObjectKey, obj client.Object) error { - accessableSequence, ok := mockObjs[key] + accessibleSequence, ok := mockObjs[key] if !ok { return fmt.Errorf("not exist: %v", key) } index := sequenceAccessCounter[key] - mockReturned := accessableSequence[index] + mockReturned := accessibleSequence[index] if mockReturned.Err == nil { SetGetReturnedObject(obj, mockReturned.Object) } - if index < len(accessableSequence)-1 { + if index < len(accessibleSequence)-1 { sequenceAccessCounter[key]++ } return mockReturned.Err diff --git a/staticcheck.conf b/staticcheck.conf index 7e997d1fe..b8afabf96 100644 --- a/staticcheck.conf +++ b/staticcheck.conf @@ -1,7 +1,7 @@ # This is config file for staticcheck. # Check https://staticcheck.io/docs/checks/ for check ID. -# If you need to add ignored checks, pls also add explaination in comments. +# If you need to add ignored checks, pls also add explanation in comments. checks = ["all", "-ST1000", "-SA1019", "-ST1001"] diff --git a/test/e2e/testdata/smoketest/playgroundtest.go b/test/e2e/testdata/smoketest/playgroundtest.go index 5c042dacf..29ca3c5fe 100644 --- a/test/e2e/testdata/smoketest/playgroundtest.go +++ b/test/e2e/testdata/smoketest/playgroundtest.go @@ -64,15 +64,15 @@ func UninstallKubeblocks() { }) Context("KubeBlocks uninstall", func() { It("delete mycluster", func() { - commond := "kbcli cluster delete mycluster --auto-approve" - log.Println(commond) - result := e2eutil.ExecuteCommand(commond) + command := "kbcli cluster delete mycluster --auto-approve" + log.Println(command) + result := e2eutil.ExecuteCommand(command) Expect(result).Should(BeTrue()) }) It("check mycluster and pod", func() { - commond := "kbcli cluster list -A" + command := "kbcli cluster list -A" Eventually(func(g Gomega) { - cluster := e2eutil.ExecCommand(commond) + cluster := e2eutil.ExecCommand(command) g.Expect(e2eutil.StringStrip(cluster)).Should(Equal("Noclusterfound")) }, time.Second*10, time.Second*1).Should(Succeed()) cmd := "kbcli cluster list-instances" @@ -107,11 +107,11 @@ func PlaygroundDestroy() { } func checkPlaygroundCluster() { - commond := "kubectl get pod -n default -l 'app.kubernetes.io/instance in (mycluster)'| grep mycluster |" + + command := "kubectl get pod -n default -l 'app.kubernetes.io/instance in (mycluster)'| grep mycluster |" + " awk '{print $3}'" - log.Println(commond) + log.Println(command) Eventually(func(g Gomega) { - podStatus := e2eutil.ExecCommand(commond) + podStatus := e2eutil.ExecCommand(command) log.Println(e2eutil.StringStrip(podStatus)) g.Expect(e2eutil.StringStrip(podStatus)).Should(Equal("Running")) }, time.Second*180, time.Second*1).Should(Succeed()) diff --git a/test/testdata/addon/addon.yaml b/test/testdata/addon/addon.yaml index 29baf41e7..a129352da 100644 --- a/test/testdata/addon/addon.yaml +++ b/test/testdata/addon/addon.yaml @@ -33,7 +33,7 @@ spec: # via YAML contents reside in secret.data. secretRefs: # - name: - # namepsace: + # namespace: # key: setValues: [ ] setJSONValues: [ ] diff --git a/test/testdata/cue_testdata/mysql.cue b/test/testdata/cue_testdata/mysql.cue index d87fd990b..97b3ca39a 100644 --- a/test/testdata/cue_testdata/mysql.cue +++ b/test/testdata/cue_testdata/mysql.cue @@ -27,12 +27,12 @@ mysqld: { // [0|1|2] default: 2 innodb_autoinc_lock_mode?: int & 0 | 1 | 2 | *2 - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } -// ingore client parameter validate +// ignore client parameter validate // mysql client: a set of name/value pairs. client?: { [string]: string diff --git a/test/testdata/cue_testdata/mysql_for_cli.cue b/test/testdata/cue_testdata/mysql_for_cli.cue index db743d4f6..8baf9f57a 100644 --- a/test/testdata/cue_testdata/mysql_for_cli.cue +++ b/test/testdata/cue_testdata/mysql_for_cli.cue @@ -23,13 +23,13 @@ binlog_stmt_cache_size?: int & >=4096 & <=16777216 | *2097152 // [0|1|2] default: 2 innodb_autoinc_lock_mode?: int & 0 | 1 | 2 | *2 - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } mysqld: #MysqlParameter & { } -// ingore client parameter validate +// ignore client parameter validate // mysql client: a set of name/value pairs. client?: { [string]: string diff --git a/test/testdata/cue_testdata/mysql_openapi.cue b/test/testdata/cue_testdata/mysql_openapi.cue index 6fd1e1778..d9613fb8d 100644 --- a/test/testdata/cue_testdata/mysql_openapi.cue +++ b/test/testdata/cue_testdata/mysql_openapi.cue @@ -28,12 +28,12 @@ // [0|1|2] default: 2 innodb_autoinc_lock_mode?: int & 0 | 1 | 2 | *2 - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } - // ingore client parameter validate + // ignore client parameter validate // mysql client: a set of name/value pairs. client?: { [string]: string diff --git a/test/testdata/cue_testdata/mysql_openapi.json b/test/testdata/cue_testdata/mysql_openapi.json index 6a7cb5298..21e963937 100644 --- a/test/testdata/cue_testdata/mysql_openapi.json +++ b/test/testdata/cue_testdata/mysql_openapi.json @@ -9,7 +9,7 @@ ], "properties": { "client": { - "description": "ingore client parameter validate\nmysql client: a set of name/value pairs.", + "description": "ignore client parameter validate\nmysql client: a set of name/value pairs.", "type": "object", "additionalProperties": { "type": "string" diff --git a/test/testdata/cue_testdata/mysql_openapi_v2.cue b/test/testdata/cue_testdata/mysql_openapi_v2.cue index f42bd584b..82b2b2ce3 100644 --- a/test/testdata/cue_testdata/mysql_openapi_v2.cue +++ b/test/testdata/cue_testdata/mysql_openapi_v2.cue @@ -41,7 +41,7 @@ // mysql config validator mysqlld: #SectionParameter - // ingore client parameter validate + // ignore client parameter validate // mysql client: a set of name/value pairs. client?: { [string]: string diff --git a/test/testdata/cue_testdata/mysql_openapi_v2.json b/test/testdata/cue_testdata/mysql_openapi_v2.json index 7fbf6de6e..bf1fadbb4 100644 --- a/test/testdata/cue_testdata/mysql_openapi_v2.json +++ b/test/testdata/cue_testdata/mysql_openapi_v2.json @@ -9,7 +9,7 @@ ], "properties": { "client": { - "description": "ingore client parameter validate\nmysql client: a set of name/value pairs.", + "description": "ignore client parameter validate\nmysql client: a set of name/value pairs.", "type": "object", "additionalProperties": { "type": "string" diff --git a/test/testdata/cue_testdata/mysql_simple.cue b/test/testdata/cue_testdata/mysql_simple.cue index e1a9faca9..a5e2154e3 100644 --- a/test/testdata/cue_testdata/mysql_simple.cue +++ b/test/testdata/cue_testdata/mysql_simple.cue @@ -25,8 +25,8 @@ // [0|1|2] default: 2 innodb_autoinc_lock_mode?: int & 0 | 1 | 2 | *2 - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } diff --git a/test/testdata/cue_testdata/wesql.cnf b/test/testdata/cue_testdata/wesql.cnf index 9afaf7281..58c2f1c27 100644 --- a/test/testdata/cue_testdata/wesql.cnf +++ b/test/testdata/cue_testdata/wesql.cnf @@ -105,7 +105,7 @@ log_bin_index=mysql-bin.index max_binlog_size=134217728 log_replica_updates=1 # binlog_rows_query_log_events=ON #AWS not set -# binlog_transaction_dependency_tracking=WRITESET #Defautl Commit Order, Aws not set +# binlog_transaction_dependency_tracking=WRITESET #Default Commit Order, Aws not set # replay log # relay_log_info_repository=TABLE diff --git a/test/testdata/cue_testdata/wesql.cue b/test/testdata/cue_testdata/wesql.cue index 24108dc7f..e2fc384c5 100644 --- a/test/testdata/cue_testdata/wesql.cue +++ b/test/testdata/cue_testdata/wesql.cue @@ -83,7 +83,7 @@ // If this variable is enabled (the default), transactions are committed in the same order they are written to the binary log. If disabled, transactions may be committed in parallel. binlog_order_commits?: string & "0" | "1" | "OFF" | "ON" - // Whether the server logs full or minmal rows with row-based replication. + // Whether the server logs full or minimal rows with row-based replication. binlog_row_image?: string & "FULL" | "MINIMAL" | "NOBLOB" // Controls whether metadata is logged using FULL or MINIMAL format. FULL causes all metadata to be logged; MINIMAL means that only metadata actually required by slave is logged. Default: MINIMAL. @@ -1526,8 +1526,8 @@ // For SQL window functions, determines whether to enable inversion optimization for moving window frames also for floating values. windowing_use_high_precision: string & "0" | "1" | "OFF" | "ON" | *"1" - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } diff --git a/test/testdata/resources/mysql-config-constraint.yaml b/test/testdata/resources/mysql-config-constraint.yaml index b5faedf86..cfb5a9c98 100644 --- a/test/testdata/resources/mysql-config-constraint.yaml +++ b/test/testdata/resources/mysql-config-constraint.yaml @@ -28,13 +28,13 @@ spec: binlog_stmt_cache_size?: int & >= 4096 & <= 16777216 | *2097152 // [0|1|2] default: 2 innodb_autoinc_lock_mode?: int & 0 | 1 | 2 | *2 - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } mysqld: #MysqlParameter - // ingore client parameter validate + // ignore client parameter validate // mysql client: a set of name/value pairs. client?: { [string]: string diff --git a/test/testdata/resources/mysql-consensus-config-constraint.yaml b/test/testdata/resources/mysql-consensus-config-constraint.yaml index 783ddaf2e..a584148d8 100644 --- a/test/testdata/resources/mysql-consensus-config-constraint.yaml +++ b/test/testdata/resources/mysql-consensus-config-constraint.yaml @@ -111,7 +111,7 @@ spec: // If this variable is enabled (the default), transactions are committed in the same order they are written to the binary log. If disabled, transactions may be committed in parallel. binlog_order_commits?: string & "0" | "1" | "OFF" | "ON" - // Whether the server logs full or minmal rows with row-based replication. + // Whether the server logs full or minimal rows with row-based replication. binlog_row_image?: string & "FULL" | "MINIMAL" | "NOBLOB" // Controls whether metadata is logged using FULL or MINIMAL format. FULL causes all metadata to be logged; MINIMAL means that only metadata actually required by slave is logged. Default: MINIMAL. @@ -1554,8 +1554,8 @@ spec: // For SQL window functions, determines whether to enable inversion optimization for moving window frames also for floating values. windowing_use_high_precision: string & "0" | "1" | "OFF" | "ON" | *"1" - // other parmeters - // reference mysql parmeters + // other parameters + // reference mysql parameters ... } diff --git a/test/testdata/resources/mysql-consensus-config-template.yaml b/test/testdata/resources/mysql-consensus-config-template.yaml index 315b040f8..767f28b23 100644 --- a/test/testdata/resources/mysql-consensus-config-template.yaml +++ b/test/testdata/resources/mysql-consensus-config-template.yaml @@ -155,7 +155,7 @@ data: max_binlog_size=134217728 log_replica_updates=1 # binlog_rows_query_log_events=ON #AWS not set - # binlog_transaction_dependency_tracking=WRITESET #Defautl Commit Order, Aws not set + # binlog_transaction_dependency_tracking=WRITESET #Default Commit Order, Aws not set # replay log # relay_log_info_repository=TABLE From bee2190ac2291b67a14075844d3c1deeac679112 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 18 Apr 2023 11:46:27 +0800 Subject: [PATCH 069/439] feat: support the auto create pv by template for backup, csi-oss addon and support kbcli kubeblocks describe-config (#2660) Co-authored-by: wangyelei --- .../v1alpha1/backuppolicy_types.go | 20 ++- ...otection.kubeblocks.io_backuppolicies.yaml | 44 +++++- .../dataprotection/backup_controller.go | 72 ++++++++-- .../dataprotection/backup_controller_test.go | 108 ++++++++++++-- controllers/dataprotection/type.go | 3 + .../templates/backuptool.yaml | 5 +- deploy/csi-oss/Chart.yaml | 8 ++ deploy/csi-oss/README.md | 32 +++++ deploy/csi-oss/templates/oss-plugin.yaml | 135 ++++++++++++++++++ deploy/csi-oss/templates/pv-template.yaml | 31 ++++ deploy/csi-oss/templates/rbac.yaml | 97 +++++++++++++ deploy/csi-oss/templates/secret.yaml | 8 ++ deploy/csi-oss/values.yaml | 47 ++++++ ...otection.kubeblocks.io_backuppolicies.yaml | 44 +++++- .../helm/templates/addons/csi-oss-addon.yaml | 27 ++++ .../helm/templates/addons/csi-oss-values.yaml | 13 ++ deploy/helm/templates/configmap.yaml | 18 ++- deploy/helm/values.yaml | 4 + docs/user_docs/cli/cli.md | 1 + docs/user_docs/cli/kbcli_kubeblocks.md | 1 + docs/user_docs/cli/kbcli_kubeblocks_config.md | 16 ++- .../cli/kbcli_kubeblocks_describe-config.md | 53 +++++++ internal/cli/cmd/kubeblocks/config.go | 114 ++++++++++++++- internal/cli/cmd/kubeblocks/config_test.go | 69 +++++++-- internal/cli/cmd/kubeblocks/kubeblocks.go | 1 + internal/cli/util/version.go | 6 +- internal/constant/const.go | 12 +- .../transformer_backup_policy_tpl.go | 15 +- internal/controllerutil/errors.go | 3 +- 29 files changed, 943 insertions(+), 64 deletions(-) create mode 100644 deploy/csi-oss/Chart.yaml create mode 100644 deploy/csi-oss/README.md create mode 100644 deploy/csi-oss/templates/oss-plugin.yaml create mode 100644 deploy/csi-oss/templates/pv-template.yaml create mode 100644 deploy/csi-oss/templates/rbac.yaml create mode 100644 deploy/csi-oss/templates/secret.yaml create mode 100644 deploy/csi-oss/values.yaml create mode 100644 deploy/helm/templates/addons/csi-oss-addon.yaml create mode 100644 deploy/helm/templates/addons/csi-oss-values.yaml create mode 100644 docs/user_docs/cli/kbcli_kubeblocks_describe-config.md diff --git a/apis/dataprotection/v1alpha1/backuppolicy_types.go b/apis/dataprotection/v1alpha1/backuppolicy_types.go index 33ba451b7..b2d7f4ade 100644 --- a/apis/dataprotection/v1alpha1/backuppolicy_types.go +++ b/apis/dataprotection/v1alpha1/backuppolicy_types.go @@ -112,13 +112,31 @@ type PersistentVolumeClaim struct { InitCapacity resource.Quantity `json:"initCapacity,omitempty"` // createPolicy defines the policy for creating the PersistentVolumeClaim, enum values: - // - Never: do nothing if the PersistentVolumeClaim not exist. + // - Never: do nothing if the PersistentVolumeClaim not exists. // - IfNotPresent: create the PersistentVolumeClaim if not present and the accessModes only contains 'ReadWriteMany'. // +kubebuilder:default=IfNotPresent // +optional CreatePolicy CreatePVCPolicy `json:"createPolicy"` + + // persistentVolumeConfigMap references the configmap which contains a persistentVolume template. + // key must be "persistentVolume" and value is the "PersistentVolume" struct. + // support the following built-in Objects: + // - $(GENERATE_NAME): generate a specific format "pvcName-pvcNamespace". + // if the PersistentVolumeClaim not exists and CreatePolicy is "IfNotPresent", the controller + // will create it by this template. this is a mutually exclusive setting with "storageClassName". + // +optional + PersistentVolumeConfigMap *PersistentVolumeConfigMap `json:"persistentVolumeConfigMap,omitempty"` } +type PersistentVolumeConfigMap struct { + // the name of the persistentVolume ConfigMap. + // +kubebuilder:validation:Required + Name string `json:"name"` + + // the namespace of the persistentVolume ConfigMap. + // +kubebuilder:validation:Required + Namespace string `json:"namespace"` +} type BasePolicy struct { // target database cluster for backup. // +kubebuilder:validation:Required diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml index 9e4b7bd9a..46a131acb 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -105,7 +105,7 @@ spec: default: IfNotPresent description: 'createPolicy defines the policy for creating the PersistentVolumeClaim, enum values: - Never: do nothing - if the PersistentVolumeClaim not exist. - IfNotPresent: + if the PersistentVolumeClaim not exists. - IfNotPresent: create the PersistentVolumeClaim if not present and the accessModes only contains ''ReadWriteMany''.' enum: @@ -124,6 +124,26 @@ spec: name: description: the name of the PersistentVolumeClaim. type: string + persistentVolumeConfigMap: + description: 'persistentVolumeConfigMap references the configmap + which contains a persistentVolume template. key must be + "persistentVolume" and value is the "PersistentVolume" struct. + support the following built-in Objects: - $(GENERATE_NAME): + generate a specific format "pvcName-pvcNamespace". if the + PersistentVolumeClaim not exists and CreatePolicy is "IfNotPresent", + the controller will create it by this template. this is + a mutually exclusive setting with "storageClassName".' + properties: + name: + description: the name of the persistentVolume ConfigMap. + type: string + namespace: + description: the namespace of the persistentVolume ConfigMap. + type: string + required: + - name + - namespace + type: object storageClassName: description: storageClassName is the name of the StorageClass required by the claim. @@ -267,7 +287,7 @@ spec: default: IfNotPresent description: 'createPolicy defines the policy for creating the PersistentVolumeClaim, enum values: - Never: do nothing - if the PersistentVolumeClaim not exist. - IfNotPresent: + if the PersistentVolumeClaim not exists. - IfNotPresent: create the PersistentVolumeClaim if not present and the accessModes only contains ''ReadWriteMany''.' enum: @@ -286,6 +306,26 @@ spec: name: description: the name of the PersistentVolumeClaim. type: string + persistentVolumeConfigMap: + description: 'persistentVolumeConfigMap references the configmap + which contains a persistentVolume template. key must be + "persistentVolume" and value is the "PersistentVolume" struct. + support the following built-in Objects: - $(GENERATE_NAME): + generate a specific format "pvcName-pvcNamespace". if the + PersistentVolumeClaim not exists and CreatePolicy is "IfNotPresent", + the controller will create it by this template. this is + a mutually exclusive setting with "storageClassName".' + properties: + name: + description: the name of the persistentVolume ConfigMap. + type: string + namespace: + description: the namespace of the persistentVolume ConfigMap. + type: string + required: + - name + - namespace + type: object storageClassName: description: storageClassName is the name of the StorageClass required by the claim. diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 1fff54aa3..b235392b5 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -35,6 +35,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/tools/record" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" @@ -222,8 +223,11 @@ func (r *BackupReconciler) doNewPhaseAction( func (r *BackupReconciler) handlePersistentVolumeClaim(reqCtx intctrlutil.RequestCtx, backupPolicyName string, commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) error { - pvc := &corev1.PersistentVolumeClaim{} pvcConfig := commonPolicy.PersistentVolumeClaim + if len(pvcConfig.Name) == 0 { + return fmt.Errorf("the persistentVolumeClaim name of this policy is empty") + } + pvc := &corev1.PersistentVolumeClaim{} if err := r.Client.Get(reqCtx.Ctx, client.ObjectKey{Namespace: reqCtx.Req.Namespace, Name: pvcConfig.Name}, pvc); err != nil && !apierrors.IsNotFound(err) { return err @@ -234,14 +238,26 @@ func (r *BackupReconciler) handlePersistentVolumeClaim(reqCtx intctrlutil.Reques if pvcConfig.CreatePolicy == dataprotectionv1alpha1.CreatePVCPolicyNever { return intctrlutil.NewNotFound(`persistent volume claim "%s" not found`, pvcConfig.Name) } - pvc = &corev1.PersistentVolumeClaim{ + if pvcConfig.PersistentVolumeConfigMap != nil && + (pvcConfig.StorageClassName == nil || *pvcConfig.StorageClassName == "") { + // if the storageClassName is empty and the PersistentVolumeConfigMap is not empty, + // will create the persistentVolume with the template + if err := r.createPersistentVolumeWithTemplate(reqCtx, backupPolicyName, &pvcConfig); err != nil { + return err + } + } + return r.createPVCWithStorageClassName(reqCtx, backupPolicyName, pvcConfig) +} + +// createPVCWithStorageClassName creates the persistent volume claim with the storageClassName. +func (r *BackupReconciler) createPVCWithStorageClassName(reqCtx intctrlutil.RequestCtx, + backupPolicyName string, + pvcConfig dataprotectionv1alpha1.PersistentVolumeClaim) error { + pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: pvcConfig.Name, - Namespace: reqCtx.Req.Namespace, - Annotations: map[string]string{ - dataProtectionAnnotationCreateByPolicyKey: "true", - dataProtectionLabelBackupPolicyKey: backupPolicyName, - }, + Name: pvcConfig.Name, + Namespace: reqCtx.Req.Namespace, + Annotations: r.buildAutoCreationAnnotations(backupPolicyName), }, Spec: corev1.PersistentVolumeClaimSpec{ StorageClassName: pvcConfig.StorageClassName, @@ -261,6 +277,46 @@ func (r *BackupReconciler) handlePersistentVolumeClaim(reqCtx intctrlutil.Reques return client.IgnoreAlreadyExists(err) } +// createPersistentVolumeWithTemplate creates the persistent volume with the template. +func (r *BackupReconciler) createPersistentVolumeWithTemplate(reqCtx intctrlutil.RequestCtx, + backupPolicyName string, + pvcConfig *dataprotectionv1alpha1.PersistentVolumeClaim) error { + pvConfig := pvcConfig.PersistentVolumeConfigMap + configMap := &corev1.ConfigMap{} + if err := r.Client.Get(reqCtx.Ctx, client.ObjectKey{Namespace: pvConfig.Namespace, + Name: pvConfig.Name}, configMap); err != nil { + return err + } + pvTemplate := configMap.Data[persistentVolumeTemplateKey] + if pvTemplate == "" { + return intctrlutil.NewNotFound("the persistentVolume template is empty in the configMap %s/%s", pvConfig.Namespace, pvConfig.Name) + } + pvName := fmt.Sprintf("%s-%s", pvcConfig.Name, reqCtx.Req.Namespace) + pvTemplate = strings.ReplaceAll(pvTemplate, "$(GENERATE_NAME)", pvName) + pv := &corev1.PersistentVolume{} + if err := yaml.Unmarshal([]byte(pvTemplate), pv); err != nil { + return err + } + pv.Name = pvName + pv.Spec.ClaimRef = &corev1.ObjectReference{ + Namespace: reqCtx.Req.Namespace, + Name: pvcConfig.Name, + } + pv.Annotations = r.buildAutoCreationAnnotations(backupPolicyName) + // set the storageClassName to empty for the persistentVolumeClaim to avoid the dynamic provisioning + emptyStorageClassName := "" + pvcConfig.StorageClassName = &emptyStorageClassName + controllerutil.AddFinalizer(pv, dataProtectionFinalizerName) + return r.Client.Create(reqCtx.Ctx, pv) +} + +func (r *BackupReconciler) buildAutoCreationAnnotations(backupPolicyName string) map[string]string { + return map[string]string{ + dataProtectionAnnotationCreateByPolicyKey: "true", + dataProtectionLabelBackupPolicyKey: backupPolicyName, + } +} + func (r *BackupReconciler) doInProgressPhaseAction( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) (ctrl.Result, error) { diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index b3dd47c9c..16b374502 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -19,8 +19,11 @@ package dataprotection import ( "strings" + "github.com/ghodss/yaml" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/viper" @@ -64,7 +67,7 @@ var _ = Describe("Backup Controller test", func() { testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.JobSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.CronJobSignature, inNS, ml) - testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS, ml) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS) // // non-namespaced testapps.ClearResources(&testCtx, intctrlutil.BackupToolSignature, ml) @@ -329,9 +332,18 @@ var _ = Describe("Backup Controller test", func() { }) }) - When("without backupTool resources", func() { + When("with backupTool resources", func() { Context("creates a full backup", func() { var backupKey types.NamespacedName + var backupPolicy *dataprotectionv1alpha1.BackupPolicy + createBackup := func(backupName string) { + By("By creating a backup from backupPolicy: " + backupPolicyName) + backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). + SetBackupPolicyName(backupPolicyName). + SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + Create(&testCtx).GetObject() + backupKey = client.ObjectKeyFromObject(backup) + } BeforeEach(func() { By("By creating a backupTool") @@ -342,7 +354,7 @@ var _ = Describe("Backup Controller test", func() { }) By("By creating a backupPolicy from backupTool: " + backupTool.Name) - _ = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). + backupPolicy = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). AddFullPolicy(). SetBackupToolName(backupTool.Name). SetSchedule(defaultSchedule, true). @@ -352,28 +364,98 @@ var _ = Describe("Backup Controller test", func() { SetPVC(backupRemotePVCName). Create(&testCtx).GetObject() - By("By creating a backup from backupPolicy: " + backupPolicyName) - backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). - Create(&testCtx).GetObject() - backupKey = client.ObjectKeyFromObject(backup) }) It("should succeed after job completes", func() { + createBackup(backupName) + patchK8sJobStatus(backupKey, batchv1.JobComplete) + By("Check backup job completed") + Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dataprotectionv1alpha1.Backup) { + g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupCompleted)) + })).Should(Succeed()) + }) - By("Check pvc created by backup") + It("creates pvc if the specified pvc not exists", func() { + createBackup(backupName) + By("Check pvc created by backup controller") Eventually(testapps.CheckObjExists(&testCtx, types.NamespacedName{ Name: backupRemotePVCName, Namespace: testCtx.DefaultNamespace, }, &corev1.PersistentVolumeClaim{}, true)).Should(Succeed()) + }) - patchK8sJobStatus(backupKey, batchv1.JobComplete) + It("creates pvc if the specified pvc not exists", func() { + By("set persistentVolumeConfigmap") + configMapName := "pv-template-configmap" + Expect(testapps.ChangeObj(&testCtx, backupPolicy, func(tmpObj *dataprotectionv1alpha1.BackupPolicy) { + tmpObj.Spec.Full.PersistentVolumeClaim.PersistentVolumeConfigMap = &dataprotectionv1alpha1.PersistentVolumeConfigMap{ + Name: configMapName, + Namespace: testCtx.DefaultNamespace, + } + })).Should(Succeed()) - By("Check backup job completed") + By("create backup with non existent configmap of pv template") + createBackup(backupName) Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dataprotectionv1alpha1.Backup) { - g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupCompleted)) + g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupFailed)) + g.Expect(fetched.Status.FailureReason).To(ContainSubstring("not found")) + })).Should(Succeed()) + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: testCtx.DefaultNamespace, + }, + Data: map[string]string{}, + } + Expect(testCtx.CreateObj(ctx, configMap)).Should(Succeed()) + + By("create backup with the configmap not contains the key 'persistentVolume'") + createBackup(backupName + "1") + Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dataprotectionv1alpha1.Backup) { + g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupFailed)) + g.Expect(fetched.Status.FailureReason).To(ContainSubstring("the persistentVolume template is empty in the configMap")) + })).Should(Succeed()) + + By("create backup with the configmap contains the key 'persistentVolume'") + Expect(testapps.ChangeObj(&testCtx, configMap, func(tmpObj *corev1.ConfigMap) { + pv := corev1.PersistentVolume{ + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + }, + Capacity: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: "kubeblocks.com", + FSType: "ext4", + VolumeHandle: pvcName, + }, + }, + }, + } + pvString, _ := yaml.Marshal(pv) + tmpObj.Data = map[string]string{ + "persistentVolume": string(pvString), + } })).Should(Succeed()) + createBackup(backupName + "2") + Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dataprotectionv1alpha1.Backup) { + g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupInProgress)) + })).Should(Succeed()) + + By("check pvc and pv created by backup controller") + Eventually(testapps.CheckObjExists(&testCtx, types.NamespacedName{ + Name: backupRemotePVCName, + Namespace: testCtx.DefaultNamespace, + }, &corev1.PersistentVolumeClaim{}, true)).Should(Succeed()) + Eventually(testapps.CheckObjExists(&testCtx, types.NamespacedName{ + Name: backupRemotePVCName + "-" + testCtx.DefaultNamespace, + Namespace: testCtx.DefaultNamespace, + }, &corev1.PersistentVolume{}, true)).Should(Succeed()) + }) }) }) diff --git a/controllers/dataprotection/type.go b/controllers/dataprotection/type.go index a4bec458d..332140f33 100644 --- a/controllers/dataprotection/type.go +++ b/controllers/dataprotection/type.go @@ -42,6 +42,9 @@ const ( dataProtectionAnnotationCreateByPolicyKey = "dataprotection.kubeblocks.io/created-by-policy" // error status errorJobFailed = "JobFailed" + + // the key of persistentVolumeTemplate in the configmap. + persistentVolumeTemplateKey = "persistentVolume" ) var reconcileInterval = time.Second diff --git a/deploy/apecloud-mysql-scale/templates/backuptool.yaml b/deploy/apecloud-mysql-scale/templates/backuptool.yaml index 90dd00b73..577790874 100644 --- a/deploy/apecloud-mysql-scale/templates/backuptool.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuptool.yaml @@ -42,5 +42,8 @@ spec: restoreCommands: [] incrementalRestoreCommands: [] backupCommands: - - xtrabackup --compress --backup --safe-slave-backup --slave-info --stream=xbstream --host=${DB_HOST} --user=${DB_USER} --password=${DB_PASSWORD} --datadir=${DATA_DIR} > /${BACKUP_DIR}/${BACKUP_NAME}.xbstream + - | + set -e + mkdir -p ${BACKUP_DIR} + xtrabackup --compress --backup --safe-slave-backup --slave-info --stream=xbstream --host=${DB_HOST} --user=${DB_USER} --password=${DB_PASSWORD} --datadir=${DATA_DIR} > /${BACKUP_DIR}/${BACKUP_NAME}.xbstream incrementalBackupCommands: [] diff --git a/deploy/csi-oss/Chart.yaml b/deploy/csi-oss/Chart.yaml new file mode 100644 index 000000000..e8fe9c492 --- /dev/null +++ b/deploy/csi-oss/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +appVersion: 1.1.7 +description: Container Storage Interface (CSI) driver for oss volumes +home: https://github.com/kubernetes-sigs/alibaba-cloud-csi-driver +keywords: +- oss +name: csi-oss +version: 1.1.7 diff --git a/deploy/csi-oss/README.md b/deploy/csi-oss/README.md new file mode 100644 index 000000000..e53782901 --- /dev/null +++ b/deploy/csi-oss/README.md @@ -0,0 +1,32 @@ +# Helm chart for csi-oss + +This chart adds oss volume support to your cluster. + +## Install chart + +- Helm 2.x: `helm install [--set secret.akId=... --set secret.akSecret=... ...] --namespace kube-system --name csi-oss .` +- Helm 3.x: `helm install [--set secret.akId=... --set secret.akSecret=... ...] --namespace kube-system csi-oss` + +After installation succeeds, you can get a status of Chart: `helm status csi-oss`. + +## Delete Chart + +- Helm 2.x: `helm delete --purge csi-oss` +- Helm 3.x: `helm uninstall csi-oss --namespace kube-system` + +## Configuration + +By default, this chart creates a secret and a configmap with persistentVolume. You should at least set `secret.akId`, `secret.akSecret` and `storageConfig.bucket` +to your [Alibaba OSS CSI-DRIVER](https://github.com/kubernetes-sigs/alibaba-cloud-csi-driver/blob/master/docs/oss.md) keys for it to work. + +The following table lists all configuration parameters and their default values. + +| Parameter | Description | Default | +| ---------------------------- |------------------------------------------------------------------------------------|--------------------------------------------------------| +| `storageConfig.endpoint` | Mount OSS access domain name | oss-cn-hangzhou.aliyuncs.com | +| `storageConfig.bucket` | The OSS bucket that needs to be mounted | | +| `storageConfig.path` | Indicates the directory structure of the relative bucket root file during mounting | / | +| `storageConfig.otherOpts` | Support for inputting customized parameters when mounting OSS | -o max_stat_cache_size=0 -o allow_other | +| `secret.name` | Name of the secret | csi-oss-secret | +| `secret.akId` | OSS Access Key | | +| `secret.akSecret` | OSS Secret Key | | \ No newline at end of file diff --git a/deploy/csi-oss/templates/oss-plugin.yaml b/deploy/csi-oss/templates/oss-plugin.yaml new file mode 100644 index 000000000..69a58fc9e --- /dev/null +++ b/deploy/csi-oss/templates/oss-plugin.yaml @@ -0,0 +1,135 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: ossplugin.csi.alibabacloud.com +spec: + attachRequired: false + podInfoOnMount: true +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: oss-csi-plugin + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: oss-csi-plugin + template: + metadata: + labels: + app: oss-csi-plugin + spec: + tolerations: + - operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + nodeSelector: + beta.kubernetes.io/os: linux + serviceAccount: csi-admin + priorityClassName: system-node-critical + hostNetwork: true + hostPID: true + containers: + - name: oss-driver-registrar + image: {{ .Values.images.registrar }} + imagePullPolicy: Always + args: + - "--v=5" + - "--csi-address=/var/lib/kubelet/csi-plugins/ossplugin.csi.alibabacloud.com/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/csi-plugins/ossplugin.csi.alibabacloud.com/csi.sock" + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet/ + - name: registration-dir + mountPath: /registration + - name: csi-plugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{ .Values.images.csi }} + imagePullPolicy: "Always" + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=2" + - "--driver=oss" + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/csi-plugins/driverplugin.csi.alibabacloud.com-replace/csi.sock + - name: MAX_VOLUMES_PERNODE + value: "15" + - name: SERVICE_TYPE + value: "plugin" + livenessProbe: + httpGet: + path: /healthz + port: healthz + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + ports: + - name: healthz + containerPort: 11260 + protocol: TCP + volumeMounts: + - name: kubelet-dir + mountPath: /var/lib/kubelet/ + mountPropagation: "Bidirectional" + - name: etc + mountPath: /host/etc + - name: host-log + mountPath: /var/log/ + - name: ossconnectordir + mountPath: /host/usr/ + - name: container-dir + mountPath: /var/lib/container + mountPropagation: "Bidirectional" + - name: host-dev + mountPath: /dev + mountPropagation: "HostToContainer" + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry + type: DirectoryOrCreate + - name: container-dir + hostPath: + path: /var/lib/container + type: DirectoryOrCreate + - name: kubelet-dir + hostPath: + path: /var/lib/kubelet + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-log + hostPath: + path: /var/log/ + - name: etc + hostPath: + path: /etc + - name: ossconnectordir + hostPath: + path: /usr/ + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate \ No newline at end of file diff --git a/deploy/csi-oss/templates/pv-template.yaml b/deploy/csi-oss/templates/pv-template.yaml new file mode 100644 index 000000000..a58278c03 --- /dev/null +++ b/deploy/csi-oss/templates/pv-template.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: oss-persistent-volume-template + labels: + kubeblocks.io/persistent-volume-template: "true" +data: + persistentVolume: | + apiVersion: v1 + kind: PersistentVolume + metadata: + name: $(GENERATE_NAME) + labels: + alicloud-pvname: $(GENERATE_NAME) + spec: + capacity: + storage: 100Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + csi: + driver: ossplugin.csi.alibabacloud.com + volumeHandle: $(GENERATE_NAME) + nodePublishSecretRef: + name: {{ .Values.secret.name }} + namespace: {{ .Release.Namespace }} + volumeAttributes: + bucket: "{{ .Values.storageConfig.bucket }}" + url: "{{ .Values.storageConfig.endpoint }}" + otherOpts: "{{ .Values.storageConfig.otherOpts }}" + path: "{{ .Values.storageConfig.path }}" \ No newline at end of file diff --git a/deploy/csi-oss/templates/rbac.yaml b/deploy/csi-oss/templates/rbac.yaml new file mode 100644 index 000000000..9b4507db7 --- /dev/null +++ b/deploy/csi-oss/templates/rbac.yaml @@ -0,0 +1,97 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-admin + namespace: {{ .Release.Namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: alicloud-csi-plugin +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch", "create"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "get", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "list", "watch", "delete", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: alicloud-csi-plugin +subjects: + - kind: ServiceAccount + name: csi-admin + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: alicloud-csi-plugin + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/csi-oss/templates/secret.yaml b/deploy/csi-oss/templates/secret.yaml new file mode 100644 index 000000000..e5b9d4fb3 --- /dev/null +++ b/deploy/csi-oss/templates/secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + namespace: {{ .Release.Namespace }} + name: {{ .Values.secret.name }} +stringData: + akId: {{ required "akId required, please input it!" .Values.secret.akId }} + akSecret: {{ required "akSecret required, please input it!" .Values.secret.akSecret }} \ No newline at end of file diff --git a/deploy/csi-oss/values.yaml b/deploy/csi-oss/values.yaml new file mode 100644 index 000000000..8ec7f7a58 --- /dev/null +++ b/deploy/csi-oss/values.yaml @@ -0,0 +1,47 @@ +--- +images: + registrar: registry.cn-hangzhou.aliyuncs.com/acs/csi-node-driver-registrar:v1.2.0 + # Main image + csi: registry.cn-hangzhou.aliyuncs.com/acs/csi-plugin:v1.18.8.47-906bd535-aliyun + +storageConfig: + # Endpoint of the oss service, e.g. oss-cn-hangzhou.aliyuncs.com + endpoint: oss-cn-hangzhou.aliyuncs.com + # oss bucket + bucket: "" + # mount path of the oss bucket + path: "/" + # mount options + otherOpts: "-o max_stat_cache_size=0 -o allow_other" + +secret: + # Name of the secret + name: csi-oss-secret + # AccessKey ID + akId: "" + # AccessKey Secret + akSecret: "" + +tolerations: + - key: kb-controller + operator: Equal + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: "Exists" + +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kb-controller + operator: In + values: + - "true" + - key: kubernetes.io/arch + operator: In + values: + - "amd64" + diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml index 9e4b7bd9a..46a131acb 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -105,7 +105,7 @@ spec: default: IfNotPresent description: 'createPolicy defines the policy for creating the PersistentVolumeClaim, enum values: - Never: do nothing - if the PersistentVolumeClaim not exist. - IfNotPresent: + if the PersistentVolumeClaim not exists. - IfNotPresent: create the PersistentVolumeClaim if not present and the accessModes only contains ''ReadWriteMany''.' enum: @@ -124,6 +124,26 @@ spec: name: description: the name of the PersistentVolumeClaim. type: string + persistentVolumeConfigMap: + description: 'persistentVolumeConfigMap references the configmap + which contains a persistentVolume template. key must be + "persistentVolume" and value is the "PersistentVolume" struct. + support the following built-in Objects: - $(GENERATE_NAME): + generate a specific format "pvcName-pvcNamespace". if the + PersistentVolumeClaim not exists and CreatePolicy is "IfNotPresent", + the controller will create it by this template. this is + a mutually exclusive setting with "storageClassName".' + properties: + name: + description: the name of the persistentVolume ConfigMap. + type: string + namespace: + description: the namespace of the persistentVolume ConfigMap. + type: string + required: + - name + - namespace + type: object storageClassName: description: storageClassName is the name of the StorageClass required by the claim. @@ -267,7 +287,7 @@ spec: default: IfNotPresent description: 'createPolicy defines the policy for creating the PersistentVolumeClaim, enum values: - Never: do nothing - if the PersistentVolumeClaim not exist. - IfNotPresent: + if the PersistentVolumeClaim not exists. - IfNotPresent: create the PersistentVolumeClaim if not present and the accessModes only contains ''ReadWriteMany''.' enum: @@ -286,6 +306,26 @@ spec: name: description: the name of the PersistentVolumeClaim. type: string + persistentVolumeConfigMap: + description: 'persistentVolumeConfigMap references the configmap + which contains a persistentVolume template. key must be + "persistentVolume" and value is the "PersistentVolume" struct. + support the following built-in Objects: - $(GENERATE_NAME): + generate a specific format "pvcName-pvcNamespace". if the + PersistentVolumeClaim not exists and CreatePolicy is "IfNotPresent", + the controller will create it by this template. this is + a mutually exclusive setting with "storageClassName".' + properties: + name: + description: the name of the persistentVolume ConfigMap. + type: string + namespace: + description: the namespace of the persistentVolume ConfigMap. + type: string + required: + - name + - namespace + type: object storageClassName: description: storageClassName is the name of the StorageClass required by the claim. diff --git a/deploy/helm/templates/addons/csi-oss-addon.yaml b/deploy/helm/templates/addons/csi-oss-addon.yaml new file mode 100644 index 000000000..e8e43f62c --- /dev/null +++ b/deploy/helm/templates/addons/csi-oss-addon.yaml @@ -0,0 +1,27 @@ +apiVersion: extensions.kubeblocks.io/v1alpha1 +kind: Addon +metadata: + name: csi-oss + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} + "kubeblocks.io/provider": community + {{- if .Values.keepAddons }} + annotations: + helm.sh/resource-policy: keep + {{- end }} +spec: + description: Container Storage Interface (CSI) driver for oss volumes + type: Helm + + helm: + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/csi-oss-{{ default .Chart.Version .Values.versionOverride }}.tgz + installValues: + configMapRefs: + - name: csi-oss-chart-kubeblocks-values + key: values-kubeblocks-override.yaml + + defaultInstallValues: + - enabled: true + + installable: + autoInstall: {{ get ( get ( .Values | toYaml | fromYaml ) "csi-oss" ) "enabled" }} diff --git a/deploy/helm/templates/addons/csi-oss-values.yaml b/deploy/helm/templates/addons/csi-oss-values.yaml new file mode 100644 index 000000000..0174bdb4a --- /dev/null +++ b/deploy/helm/templates/addons/csi-oss-values.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: csi-oss-chart-kubeblocks-values + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} + {{- if .Values.keepAddons }} + annotations: + helm.sh/resource-policy: keep + {{- end }} +data: + values-kubeblocks-override.yaml: |- + {{- get ( .Values | toYaml | fromYaml ) "csi-oss" | toYaml | nindent 4 }} \ No newline at end of file diff --git a/deploy/helm/templates/configmap.yaml b/deploy/helm/templates/configmap.yaml index f1be0b870..fd8b8fb01 100644 --- a/deploy/helm/templates/configmap.yaml +++ b/deploy/helm/templates/configmap.yaml @@ -18,8 +18,16 @@ data: # will replace the storageClassName when it is nil in the backup policy. BACKUP_PVC_STORAGE_CLASS: "{{ .Values.dataProtection.backupPVCStorageClassName }}" - # the pvc create policy. - # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. - # otherwise, using "Never" policy. - # only affect the backupPolicy automatically created by Kubeblocks. - BACKUP_PVC_CREATE_POLICY: "{{ .Values.dataProtection.backupPVCCreatePolicy }}" \ No newline at end of file + # the pvc create policy. + # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. + # otherwise, using "Never" policy. + # only affect the backupPolicy automatically created by KubeBs-locks. + BACKUP_PVC_CREATE_POLICY: "{{ .Values.dataProtection.backupPVCCreatePolicy }}" + + # the configmap name of the pv template. if the csi-driver not support dynamic provisioning, + # you can provide a configmap which contains key "persistentVolume" and value of the persistentVolume struct. + # only effective when storageClass is empty. + BACKUP_PV_CONFIGMAP_NAME: "{{ .Values.dataProtection.backupPVConfigMapName }}" + + # the configmap namespace of the pv template. + BACKUP_PV_CONFIGMAP_NAMESPACE: "{{ .Values.dataProtection.backupPVConfigMapNamespace }}" \ No newline at end of file diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 1366ac989..c5636528f 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -248,11 +248,15 @@ dataProtection: ## @param dataProtection.backupPVCInitCapacity - set the default pvc initCapacity if the pvc need to be created by backup controller ## @param dataProtection.backupPVCStorageClassName - set the default pvc storageClassName if the pvc need to be created by backup controller ## @param dataProtection.backupPVCCreatePolicy - set the default create policy of the pvc, optional values: IfNotPresent, Never + ## @param dataProtection.backupPVConfigMapName - set the default configmap name which contains key "persistentVolume" and value of the persistentVolume struct. + ## @param dataProtection.backupPVConfigMapNamespace - set the default configmap namespace of pv template. enableVolumeSnapshot: false backupPVCName: "" backupPVCInitCapacity: "" backupPVCStorageClassName: "" backupPVCCreatePolicy: "" + backupPVConfigMapName: "" + backupPVConfigMapNamespace: "" ## Addon controller settings, this will require cluster-admin clusterrole. addonController: diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index a0b412463..fbf3f5d79 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -114,6 +114,7 @@ List and open the KubeBlocks dashboards. KubeBlocks operation commands. * [kbcli kubeblocks config](kbcli_kubeblocks_config.md) - KubeBlocks config. +* [kbcli kubeblocks describe-config](kbcli_kubeblocks_describe-config.md) - describe KubeBlocks config. * [kbcli kubeblocks install](kbcli_kubeblocks_install.md) - Install KubeBlocks. * [kbcli kubeblocks list-versions](kbcli_kubeblocks_list-versions.md) - List KubeBlocks versions. * [kbcli kubeblocks preflight](kbcli_kubeblocks_preflight.md) - Run and retrieve preflight checks for KubeBlocks. diff --git a/docs/user_docs/cli/kbcli_kubeblocks.md b/docs/user_docs/cli/kbcli_kubeblocks.md index 71eb55914..43b075fbf 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks.md +++ b/docs/user_docs/cli/kbcli_kubeblocks.md @@ -38,6 +38,7 @@ KubeBlocks operation commands. * [kbcli kubeblocks config](kbcli_kubeblocks_config.md) - KubeBlocks config. +* [kbcli kubeblocks describe-config](kbcli_kubeblocks_describe-config.md) - describe KubeBlocks config. * [kbcli kubeblocks install](kbcli_kubeblocks_install.md) - Install KubeBlocks. * [kbcli kubeblocks list-versions](kbcli_kubeblocks_list-versions.md) - List KubeBlocks versions. * [kbcli kubeblocks preflight](kbcli_kubeblocks_preflight.md) - Run and retrieve preflight checks for KubeBlocks. diff --git a/docs/user_docs/cli/kbcli_kubeblocks_config.md b/docs/user_docs/cli/kbcli_kubeblocks_config.md index 5f47f30f4..96e0ada17 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks_config.md +++ b/docs/user_docs/cli/kbcli_kubeblocks_config.md @@ -19,21 +19,27 @@ kbcli kubeblocks config [flags] dataProtection.enableVolumeSnapshot=true # the global pvc name which persistent volume claim to store the backup data. - # will replace the pvc name when it is empty in the backup policy. + # replace the pvc name when it is empty in the backup policy. dataProtection.backupPVCName=backup-data # the init capacity of pvc for creating the pvc, e.g. 10Gi. - # will replace the init capacity when it is empty in the backup policy. + # replace the init capacity when it is empty in the backup policy. dataProtection.backupPVCInitCapacity=100Gi - # the pvc storage class name. - # will replace the storageClassName when it is nil in the backup policy. + # the pvc storage class name. replace the storageClassName when it is nil in the backup policy. dataProtection.backupPVCStorageClassName=csi-s3 # the pvc create policy. # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. - # otherwise, using "Never" policy. only affect the backupPolicy automatically created by Kubeblocks. + # otherwise, using "Never" policy. only affect the backupPolicy automatically created by KubeBlocks. dataProtection.backupPVCCreatePolicy=Never + + # the configmap name of the pv template. if the csi-driver not support dynamic provisioning, + # you can provide a configmap which contains key "persistentVolume" and value of the persistentVolume struct. + dataProtection.backupPVConfigMapName=pv-template + + # the configmap namespace of the pv template. + dataProtection.backupPVConfigMapNamespace=default ``` ### Options diff --git a/docs/user_docs/cli/kbcli_kubeblocks_describe-config.md b/docs/user_docs/cli/kbcli_kubeblocks_describe-config.md new file mode 100644 index 000000000..4d71a4895 --- /dev/null +++ b/docs/user_docs/cli/kbcli_kubeblocks_describe-config.md @@ -0,0 +1,53 @@ +--- +title: kbcli kubeblocks describe-config +--- + +describe KubeBlocks config. + +``` +kbcli kubeblocks describe-config [flags] +``` + +### Examples + +``` + # Describe the KubeBlocks config. + kbcli kubeblocks describe-config +``` + +### Options + +``` + -h, --help help for describe-config +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli kubeblocks](kbcli_kubeblocks.md) - KubeBlocks operation commands. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/internal/cli/cmd/kubeblocks/config.go b/internal/cli/cmd/kubeblocks/config.go index 657e0caac..5b66ca756 100644 --- a/internal/cli/cmd/kubeblocks/config.go +++ b/internal/cli/cmd/kubeblocks/config.go @@ -17,15 +17,27 @@ limitations under the License. package kubeblocks import ( + "context" + "fmt" + "sort" + "github.com/spf13/cobra" + "golang.org/x/exp/maps" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/cli-runtime/pkg/genericclioptions" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" + "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" ) +const configKey = "config.yaml" + var backupConfigExample = templates.Examples(` # Enable the snapshot-controller and volume snapshot, to support snapshot backup. kbcli kubeblocks config --set snapshot-controller.enabled=true @@ -35,23 +47,34 @@ var backupConfigExample = templates.Examples(` dataProtection.enableVolumeSnapshot=true # the global pvc name which persistent volume claim to store the backup data. - # will replace the pvc name when it is empty in the backup policy. + # replace the pvc name when it is empty in the backup policy. dataProtection.backupPVCName=backup-data # the init capacity of pvc for creating the pvc, e.g. 10Gi. - # will replace the init capacity when it is empty in the backup policy. + # replace the init capacity when it is empty in the backup policy. dataProtection.backupPVCInitCapacity=100Gi - # the pvc storage class name. - # will replace the storageClassName when it is nil in the backup policy. + # the pvc storage class name. replace the storageClassName when it is nil in the backup policy. dataProtection.backupPVCStorageClassName=csi-s3 # the pvc create policy. # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. - # otherwise, using "Never" policy. only affect the backupPolicy automatically created by Kubeblocks. + # otherwise, using "Never" policy. only affect the backupPolicy automatically created by KubeBlocks. dataProtection.backupPVCCreatePolicy=Never + + # the configmap name of the pv template. if the csi-driver not support dynamic provisioning, + # you can provide a configmap which contains key "persistentVolume" and value of the persistentVolume struct. + dataProtection.backupPVConfigMapName=pv-template + + # the configmap namespace of the pv template. + dataProtection.backupPVConfigMapNamespace=default `) +var describeConfigExample = templates.Examples(` + # Describe the KubeBlocks config. + kbcli kubeblocks describe-config +`) + // NewConfigCmd creates the config command func NewConfigCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &InstallOptions{ @@ -74,3 +97,84 @@ func NewConfigCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra helm.AddValueOptionsFlags(cmd.Flags(), &o.ValueOpts) return cmd } + +func NewDescribeConfigCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := &InstallOptions{ + Options: Options{ + IOStreams: streams, + }, + } + + cmd := &cobra.Command{ + Use: "describe-config", + Short: "describe KubeBlocks config.", + Example: describeConfigExample, + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(o.Complete(f, cmd)) + util.CheckErr(describeConfig(o)) + }, + } + return cmd +} + +func describeConfig(o *InstallOptions) error { + config := map[string]string{} + // get KubeBlocks configmap + configMap, err := getKubeBlocksConfigMap(o) + if err != nil { + return err + } + if configMap != nil { + values := configMap.Data[configKey] + if len(values) != 0 { + if err = yaml.Unmarshal([]byte(values), &config); err != nil { + return err + } + } + } + // get the KubeBlocks config from the deployment env. + // variables with the same name in env will overwrite variables in the configmap. + deploy, err := util.GetKubeBlocksDeploy(o.Client) + if err != nil { + return err + } + if deploy != nil { + containers := deploy.Spec.Template.Spec.Containers + if len(containers) > 0 { + for _, env := range containers[0].Env { + if env.ValueFrom != nil { + continue + } + config[env.Name] = env.Value + } + } + } + // in alphabetical order by variable name + keys := maps.Keys(config) + sort.Strings(keys) + for _, k := range keys { + line := fmt.Sprintf("%s=%v", k, config[k]) + printer.PrintLine(line) + } + return nil +} + +// getKubeBlocksConfigMap get the configmap of the KubeBlocks. +func getKubeBlocksConfigMap(o *InstallOptions) (*corev1.ConfigMap, error) { + configMapList, err := o.Client.CoreV1().ConfigMaps(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=" + types.KubeBlocksChartName, + }) + if err != nil { + return nil, err + } + configMapName := fmt.Sprintf("%s-manager-config", types.KubeBlocksChartName) + var configMap *corev1.ConfigMap + for _, v := range configMapList.Items { + if v.Name == configMapName { + configMap = &v + break + } + } + return configMap, nil +} diff --git a/internal/cli/cmd/kubeblocks/config_test.go b/internal/cli/cmd/kubeblocks/config_test.go index 48879a836..814f18656 100644 --- a/internal/cli/cmd/kubeblocks/config_test.go +++ b/internal/cli/cmd/kubeblocks/config_test.go @@ -17,10 +17,13 @@ limitations under the License. package kubeblocks import ( + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "helm.sh/helm/v3/pkg/cli/values" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/cli-runtime/pkg/genericclioptions" clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" @@ -35,6 +38,42 @@ var _ = Describe("backupconfig", func() { var streams genericclioptions.IOStreams var tf *cmdtesting.TestFactory + mockDeploy := func() *appsv1.Deployment { + deploy := &appsv1.Deployment{} + deploy.SetLabels(map[string]string{ + "app.kubernetes.io/name": types.KubeBlocksChartName, + "app.kubernetes.io/version": "0.3.0", + }) + deploy.Spec.Template.Spec.Containers = []corev1.Container{ + { + Name: "kb", + Env: []corev1.EnvVar{ + { + Name: "CM_NAMESPACE", + Value: "default", + }, + { + Name: "VOLUMESNAPSHOT", + Value: "true", + }, + }, + }, + } + return deploy + } + + mockConfigMap := func() *corev1.ConfigMap { + configmap := &corev1.ConfigMap{} + configmap.Name = fmt.Sprintf("%s-manager-config", types.KubeBlocksChartName) + configmap.SetLabels(map[string]string{ + "app.kubernetes.io/name": types.KubeBlocksChartName, + }) + configmap.Data = map[string]string{ + "config.yaml": `BACKUP_PVC_NAME: "test-pvc"`, + } + return configmap + } + BeforeEach(func() { streams, _, _, _ = genericclioptions.NewTestIOStreams() tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) @@ -49,16 +88,7 @@ var _ = Describe("backupconfig", func() { tf.Cleanup() }) - It("run cmd", func() { - mockDeploy := func() *appsv1.Deployment { - deploy := &appsv1.Deployment{} - deploy.SetLabels(map[string]string{ - "app.kubernetes.io/name": types.KubeBlocksChartName, - "app.kubernetes.io/version": "0.3.0", - }) - return deploy - } - + It("run config cmd", func() { o := &InstallOptions{ Options: Options{ IOStreams: streams, @@ -75,4 +105,23 @@ var _ = Describe("backupconfig", func() { Expect(cmd).ShouldNot(BeNil()) Expect(o.Install()).Should(Succeed()) }) + + It("run describe config cmd", func() { + o := &InstallOptions{ + Options: Options{ + IOStreams: streams, + HelmCfg: helm.NewFakeConfig(testing.Namespace), + Namespace: "default", + Client: testing.FakeClientSet(mockDeploy(), mockConfigMap()), + }, + } + cmd := NewDescribeConfigCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + done := testing.Capture() + Expect(describeConfig(o)).Should(Succeed()) + capturedOutput, err := done() + Expect(err).Should(Succeed()) + Expect(capturedOutput).Should(ContainSubstring("VOLUMESNAPSHOT=true")) + Expect(capturedOutput).Should(ContainSubstring("BACKUP_PVC_NAME=test-pvc")) + }) }) diff --git a/internal/cli/cmd/kubeblocks/kubeblocks.go b/internal/cli/cmd/kubeblocks/kubeblocks.go index 117e1d567..04c419d1b 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks.go @@ -36,6 +36,7 @@ func NewKubeBlocksCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *c newListVersionsCmd(streams), newStatusCmd(f, streams), NewConfigCmd(f, streams), + NewDescribeConfigCmd(f, streams), ) // add preflight cmd cmd.AddCommand(NewPreflightCmd(f, streams)) diff --git a/internal/cli/util/version.go b/internal/cli/util/version.go index 33d638f3c..14fa74a29 100644 --- a/internal/cli/util/version.go +++ b/internal/cli/util/version.go @@ -62,7 +62,7 @@ func GetVersionInfo(client kubernetes.Interface) (map[AppName]string, error) { // getKubeBlocksVersion get KubeBlocks version func getKubeBlocksVersion(client kubernetes.Interface) (string, error) { - deploy, err := getKubeBlocksDeploy(client) + deploy, err := GetKubeBlocksDeploy(client) if err != nil || deploy == nil { return "", err } @@ -96,9 +96,9 @@ func GetK8sVersion(discoveryClient discovery.DiscoveryInterface) (string, error) return "", nil } -// getKubeBlocksDeploy get KubeBlocks deployments, now one kubernetes cluster +// GetKubeBlocksDeploy gets KubeBlocks deployments, now one kubernetes cluster // only support one KubeBlocks -func getKubeBlocksDeploy(client kubernetes.Interface) (*appsv1.Deployment, error) { +func GetKubeBlocksDeploy(client kubernetes.Interface) (*appsv1.Deployment, error) { deploys, err := client.AppsV1().Deployments(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{ LabelSelector: "app.kubernetes.io/name=" + types.KubeBlocksChartName, }) diff --git a/internal/constant/const.go b/internal/constant/const.go index 7c6b8870f..a8200be76 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -23,11 +23,13 @@ const ( CfgKeyCtrlrMgrAffinity = "CM_AFFINITY" CfgKeyCtrlrMgrNodeSelector = "CM_NODE_SELECTOR" CfgKeyCtrlrMgrTolerations = "CM_TOLERATIONS" - CfgKeyCtrlrReconcileRetryDurationMS = "CM_RECON_RETRY_DURATION_MS" // accept time - CfgKeyBackupPVCName = "BACKUP_PVC_NAME" // the global pvc name which persistent volume claim to store the backup data - CfgKeyBackupPVCInitCapacity = "BACKUP_PVC_INIT_CAPACITY" // the init capacity of pvc for creating the pvc, e.g. 10Gi. - CfgKeyBackupPVCStorageClass = "BACKUP_PVC_STORAGE_CLASS" // the pvc storage class name. - CfgKeyBackupPVCCreatePolicy = "BACKUP_PVC_CREATE_POLICY" // the pvc create policy. support "IfNotPresent" or "Never" + CfgKeyCtrlrReconcileRetryDurationMS = "CM_RECON_RETRY_DURATION_MS" // accept time + CfgKeyBackupPVCName = "BACKUP_PVC_NAME" // the global pvc name which persistent volume claim to store the backup data + CfgKeyBackupPVCInitCapacity = "BACKUP_PVC_INIT_CAPACITY" // the init capacity of pvc for creating the pvc, e.g. 10Gi. + CfgKeyBackupPVCStorageClass = "BACKUP_PVC_STORAGE_CLASS" // the pvc storage class name. + CfgKeyBackupPVCCreatePolicy = "BACKUP_PVC_CREATE_POLICY" // the pvc create policy. support "IfNotPresent" or "Never" + CfgKeyBackupPVConfigmapName = "BACKUP_PV_CONFIGMAP_NAME" // the configmap name which contains a persistentVolume template. + CfgKeyBackupPVConfigmapNamespace = "BACKUP_PV_CONFIGMAP_NAMESPACE" // the configmap namespace which contains a persistentVolume template. // addon config keys CfgKeyAddonJobTTL = "ADDON_JOB_TTL" diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index d293ce0e4..d54fad92b 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -318,11 +318,22 @@ func (r *backupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.Common if len(globalInitCapacity) != 0 { defaultInitCapacity = globalInitCapacity } + // set the persistent volume configmap infos if these variables exist. + globalPVConfigMapName := viper.GetString(constant.CfgKeyBackupPVConfigmapName) + globalPVConfigMapNamespace := viper.GetString(constant.CfgKeyBackupPVConfigmapNamespace) + var persistentVolumeConfigMap *dataprotectionv1alpha1.PersistentVolumeConfigMap + if globalPVConfigMapName != "" && globalPVConfigMapNamespace != "" { + persistentVolumeConfigMap = &dataprotectionv1alpha1.PersistentVolumeConfigMap{ + Name: globalPVConfigMapName, + Namespace: globalPVConfigMapNamespace, + } + } return &dataprotectionv1alpha1.CommonBackupPolicy{ BackupToolName: bp.BackupToolName, PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ - InitCapacity: resource.MustParse(defaultInitCapacity), - CreatePolicy: defaultCreatePolicy, + InitCapacity: resource.MustParse(defaultInitCapacity), + CreatePolicy: defaultCreatePolicy, + PersistentVolumeConfigMap: persistentVolumeConfigMap, }, BasePolicy: r.convertBasePolicy(bp.BasePolicy, clusterName, component, workloadType), } diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index 6d25bd996..c23f4c583 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -42,8 +42,7 @@ const ( // ErrorTypeBackupNotCompleted is used to report backup not completed. ErrorTypeBackupNotCompleted ErrorType = "BackupNotCompleted" - // ErrorTypeBackupPolicyFailed backup policy failed. - ErrorTypeBackupPolicyFailed = "BackupPolicyFailed" + // ErrorTypeNotFound not found any resource. ErrorTypeNotFound = "NotFound" ) From 18e32653399696330c1b906c5ed61031fc50e0be Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Tue, 18 Apr 2023 11:46:48 +0800 Subject: [PATCH 070/439] fix: the restart logic of the configure operator is inconsistent with the judged by kbcli (#2646) --- internal/cli/cmd/cluster/config_edit.go | 3 ++- internal/cli/cmd/cluster/config_ops.go | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/cli/cmd/cluster/config_edit.go b/internal/cli/cmd/cluster/config_edit.go index 073aea666..d81e77b68 100644 --- a/internal/cli/cmd/cluster/config_edit.go +++ b/internal/cli/cmd/cluster/config_edit.go @@ -34,6 +34,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/prompt" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + cfgcm "github.com/apecloud/kubeblocks/internal/configuration/config_manager" ) type editConfigOptions struct { @@ -114,7 +115,7 @@ func (o *editConfigOptions) Run(fn func(info *cfgcore.ConfigPatchInfo, cc *appsv } confirmPrompt := confirmApplyReconfigurePrompt - if !dynamicUpdated { + if !dynamicUpdated || !cfgcm.IsSupportReload(configConstraint.Spec.ReloadOptions) { confirmPrompt = restartConfirmPrompt } yes, err := o.confirmReconfigure(confirmPrompt) diff --git a/internal/cli/cmd/cluster/config_ops.go b/internal/cli/cmd/cluster/config_ops.go index ccae51428..e076d8e63 100644 --- a/internal/cli/cmd/cluster/config_ops.go +++ b/internal/cli/cmd/cluster/config_ops.go @@ -33,6 +33,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/prompt" cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + cfgcm "github.com/apecloud/kubeblocks/internal/configuration/config_manager" ) type configOpsOptions struct { @@ -129,6 +130,10 @@ func (o *configOpsOptions) checkChangedParamsAndDoubleConfirm(cc *appsv1alpha1.C return r } + if !cfgcm.IsSupportReload(cc.ReloadOptions) { + return o.confirmReconfigureWithRestart() + } + configPatch, _, err := cfgcore.CreateConfigPatch(mockEmptyData(data), data, cc.FormatterConfig.Format, tpl.Keys, false) if err != nil { return err From 46e42708b4c4e80e8584d8707542f969bc848a6d Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Tue, 18 Apr 2023 12:43:22 +0800 Subject: [PATCH 071/439] fix: kbcli will validate the cluster's name length when create (#2616) --- apis/apps/v1alpha1/cluster_types.go | 1 + .../bases/apps.kubeblocks.io_clusters.yaml | 1 + .../bases/apps.kubeblocks.io_opsrequests.yaml | 2 + .../crds/apps.kubeblocks.io_clusters.yaml | 1 + .../crds/apps.kubeblocks.io_opsrequests.yaml | 2 + internal/cli/cmd/cluster/cluster_test.go | 71 +++++++++++++++++++ internal/cli/cmd/cluster/create.go | 3 + internal/testutil/apps/constant.go | 4 +- 8 files changed, 83 insertions(+), 2 deletions(-) diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index 427b8b8aa..206e652fc 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -404,6 +404,7 @@ type TLSSecretRef struct { type ClusterComponentService struct { // Service name // +kubebuilder:validation:Required + // +kubebuilder:validation:MaxLength=15 Name string `json:"name"` // serviceType determines how the Service is exposed. Valid diff --git a/config/crd/bases/apps.kubeblocks.io_clusters.yaml b/config/crd/bases/apps.kubeblocks.io_clusters.yaml index b41dc82b0..0bd9a3df9 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusters.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusters.yaml @@ -310,6 +310,7 @@ spec: type: object name: description: Service name + maxLength: 15 type: string serviceType: default: ClusterIP diff --git a/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml b/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml index ee8985ced..1df66f6a6 100644 --- a/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml +++ b/config/crd/bases/apps.kubeblocks.io_opsrequests.yaml @@ -82,6 +82,7 @@ spec: type: object name: description: Service name + maxLength: 15 type: string serviceType: default: ClusterIP @@ -625,6 +626,7 @@ spec: type: object name: description: Service name + maxLength: 15 type: string serviceType: default: ClusterIP diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml index b41dc82b0..0bd9a3df9 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml @@ -310,6 +310,7 @@ spec: type: object name: description: Service name + maxLength: 15 type: string serviceType: default: ClusterIP diff --git a/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml b/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml index ee8985ced..1df66f6a6 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_opsrequests.yaml @@ -82,6 +82,7 @@ spec: type: object name: description: Service name + maxLength: 15 type: string serviceType: default: ClusterIP @@ -625,6 +626,7 @@ spec: type: object name: description: Service name + maxLength: 15 type: string serviceType: default: ClusterIP diff --git a/internal/cli/cmd/cluster/cluster_test.go b/internal/cli/cmd/cluster/cluster_test.go index c43f6fa4c..65f27d7ff 100644 --- a/internal/cli/cmd/cluster/cluster_test.go +++ b/internal/cli/cmd/cluster/cluster_test.go @@ -132,6 +132,77 @@ var _ = Describe("Cluster", func() { }) }) + Context("create validate", func() { + var o *CreateOptions + BeforeEach(func() { + o = &CreateOptions{ + ClusterDefRef: testing.ClusterDefName, + ClusterVersionRef: testing.ClusterVersionName, + SetFile: testComponentPath, + UpdatableFlags: UpdatableFlags{ + TerminationPolicy: "Delete", + }, + BaseOptions: create.BaseOptions{ + Namespace: "default", + Name: "mycluster", + Dynamic: tf.FakeDynamicClient, + IOStreams: streams, + }, + } + }) + + It("can validate whether the ClusterDefRef is null when create a new cluster ", func() { + Expect(o.ClusterDefRef).ShouldNot(BeEmpty()) + Expect(o.Validate()).Should(Succeed()) + o.ClusterDefRef = "" + Expect(o.Validate()).Should(HaveOccurred()) + }) + + It("can validate whether the TerminationPolicy is null when create a new cluster ", func() { + Expect(o.TerminationPolicy).ShouldNot(BeEmpty()) + Expect(o.Validate()).Should(Succeed()) + o.TerminationPolicy = "" + Expect(o.Validate()).Should(HaveOccurred()) + }) + + It("can validate whether the ClusterVersionRef is null and can't get latest version from client when create a new cluster ", func() { + Expect(o.ClusterVersionRef).ShouldNot(BeEmpty()) + Expect(o.Validate()).Should(Succeed()) + o.ClusterVersionRef = "" + Expect(o.Validate()).Should(Succeed()) + }) + + It("can validate whether --set and --set-file both are specified when create a new cluster ", func() { + Expect(o.SetFile).ShouldNot(BeEmpty()) + Expect(o.Values).Should(BeNil()) + Expect(o.Validate()).Should(Succeed()) + o.Values = []string{"notEmpty"} + Expect(o.Validate()).Should(HaveOccurred()) + }) + + It("can validate whether the name is not specified and fail to generate a random cluster name when create a new cluster ", func() { + Expect(o.Name).ShouldNot(BeEmpty()) + Expect(o.Validate()).Should(Succeed()) + o.Name = "" + Expect(o.Validate()).Should(Succeed()) + }) + + It("can validate whether the name is not longer than 16 characters when create a new cluster", func() { + Expect(len(o.Name)).Should(BeNumerically("<=", 16)) + Expect(o.Validate()).Should(Succeed()) + moreThan16 := 17 + bytes := make([]byte, 0) + var clusterNameMoreThan16 string + for i := 0; i < moreThan16; i++ { + bytes = append(bytes, byte(i%26+'a')) + } + clusterNameMoreThan16 = string(bytes) + Expect(len(clusterNameMoreThan16)).Should(BeNumerically(">", 16)) + o.Name = clusterNameMoreThan16 + Expect(o.Validate()).Should(HaveOccurred()) + }) + }) + It("delete", func() { cmd := NewDeleteCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index df74761be..eb13d7469 100644 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -243,6 +243,9 @@ func (o *CreateOptions) Validate() error { } o.Name = name } + if len(o.Name) > 16 { + return fmt.Errorf("cluster name should be less than 16 characters") + } return nil } diff --git a/internal/testutil/apps/constant.go b/internal/testutil/apps/constant.go index bd92b921e..c9c197c4d 100644 --- a/internal/testutil/apps/constant.go +++ b/internal/testutil/apps/constant.go @@ -32,8 +32,8 @@ const ( ScriptsVolumeName = "scripts" ServiceDefaultName = "" ServiceHeadlessName = "headless" - ServiceVPCName = "a-vpc-lb-service-for-app" - ServiceInternetName = "a-internet-lb-service-for-app" + ServiceVPCName = "vpc-lb" + ServiceInternetName = "internet-lb" DefaultGeneralResourceConstraintName = "kb-resource-constraint-general" DefaultMemoryOptimizedResourceConstraintName = "kb-resource-constraint-memory-optimized" From e51856a68737bb5d7649e9536d5162f3b579a68c Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 18 Apr 2023 13:29:53 +0800 Subject: [PATCH 072/439] chore: fix helm kubeblocks failed (#2679) --- deploy/helm/values.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index c5636528f..795cfcf05 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1664,6 +1664,23 @@ csi-s3: singleBucket: "" mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 --region cn-northwest-1" +csi-oss: + enabled: false + ## @param csi-oss.secret.akId -- oss Access Key. + ## @param csi-oss.secret.akSecret -- oss Secret Key. + secret: + akId: "" + akSecret: "" + ## @param csi-oss.storageConfig.bucket -- oss bucket name. + ## @param csi-oss.storageConfig.endpoint -- endpoint of the oss service, e.g. oss-cn-hangzhou.aliyuncs.com. + ## @param csi-oss.storageConfig.path -- mount path of the oss bucket. + ## @param csi-oss.storageConfig.otherOpts -- mount options. + storageConfig: + bucket: "" + endpoint: "oss-cn-hangzhou.aliyuncs.com" + path: "/" + otherOpts: "-o max_stat_cache_size=0 -o allow_other" + alertmanager-webhook-adaptor: ## Linkage with prometheus.enabled ## From 8c74c0ca55c689761b3754d07e1241b544bf0b1f Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Tue, 18 Apr 2023 13:31:22 +0800 Subject: [PATCH 073/439] chore: improve cluster create examples (#2641) --- docs/user_docs/cli/kbcli_cluster_create.md | 8 ++++---- internal/cli/cmd/cluster/create.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index a8bc7a166..152b827e9 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -36,13 +36,13 @@ kbcli cluster create [CLUSTER_NAME] [flags] kbcli cluster create mycluster --cluster-definition apecloud-mysql --set cpu=1,memory=1Gi,storage=20Gi,replicas=3 # Create a cluster and set the class to general-1c1g, valid classes can be found by executing the command "kbcli class list --cluster-definition=" - kbcli cluster create myclsuter --cluster-definition apecloud-mysql --set class=general-1c1g + kbcli cluster create mycluster --cluster-definition apecloud-mysql --set class=general-1c1g # Create a cluster with replicationSet workloadType and set switchPolicy to Noop - kbcli cluster create myclsuter --cluster-definition postgresql --set switchPolicy=Noop + kbcli cluster create mycluster --cluster-definition postgresql --set switchPolicy=Noop # Create a cluster and use a URL to set cluster resource - kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file https://kubeblocks.io/yamls/my.yaml + kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file https://kubeblocks.io/yamls/apecloud-mysql.yaml # Create a cluster and load cluster resource set from stdin cat << EOF | kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file - @@ -58,7 +58,7 @@ kbcli cluster create [CLUSTER_NAME] [flags] kbcli cluster create --cluster-definition apecloud-mysql --tolerations '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule","key=diskType,value=ssd,operator=Equal,effect=NoSchedule"' # Create a cluster, with each pod runs on their own dedicated node - kbcli cluster create --tenancy=DedicatedNode + kbcli cluster create --cluster-definition apecloud-mysql --tenancy=DedicatedNode ``` ### Options diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index eb13d7469..3aa788f17 100644 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -77,13 +77,13 @@ var clusterCreateExample = templates.Examples(` kbcli cluster create mycluster --cluster-definition apecloud-mysql --set cpu=1,memory=1Gi,storage=20Gi,replicas=3 # Create a cluster and set the class to general-1c1g, valid classes can be found by executing the command "kbcli class list --cluster-definition=" - kbcli cluster create myclsuter --cluster-definition apecloud-mysql --set class=general-1c1g + kbcli cluster create mycluster --cluster-definition apecloud-mysql --set class=general-1c1g # Create a cluster with replicationSet workloadType and set switchPolicy to Noop - kbcli cluster create myclsuter --cluster-definition postgresql --set switchPolicy=Noop + kbcli cluster create mycluster --cluster-definition postgresql --set switchPolicy=Noop # Create a cluster and use a URL to set cluster resource - kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file https://kubeblocks.io/yamls/my.yaml + kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file https://kubeblocks.io/yamls/apecloud-mysql.yaml # Create a cluster and load cluster resource set from stdin cat << EOF | kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file - @@ -99,7 +99,7 @@ var clusterCreateExample = templates.Examples(` kbcli cluster create --cluster-definition apecloud-mysql --tolerations '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule","key=diskType,value=ssd,operator=Equal,effect=NoSchedule"' # Create a cluster, with each pod runs on their own dedicated node - kbcli cluster create --tenancy=DedicatedNode + kbcli cluster create --cluster-definition apecloud-mysql --tenancy=DedicatedNode `) const ( From 2758a152deaf4194a454479ac22d344de63385c4 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Tue, 18 Apr 2023 14:17:11 +0800 Subject: [PATCH 074/439] chore: refactor cli version (#2672) --- internal/cli/cmd/kubeblocks/install.go | 16 ++++---- internal/cli/cmd/kubeblocks/install_test.go | 17 ++++----- .../cli/cmd/kubeblocks/kubeblocks_objects.go | 3 +- internal/cli/cmd/kubeblocks/preflight.go | 2 +- internal/cli/cmd/kubeblocks/uninstall.go | 2 +- internal/cli/cmd/kubeblocks/upgrade.go | 10 ++--- internal/cli/cmd/playground/init.go | 4 -- internal/cli/cmd/version/version.go | 12 +++--- internal/cli/types/types.go | 3 ++ internal/cli/util/provider.go | 15 ++++---- internal/cli/util/version.go | 32 ++++++++-------- internal/cli/util/version_test.go | 38 +++++++++---------- 12 files changed, 72 insertions(+), 82 deletions(-) diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index 85b199912..0ea596d79 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -155,13 +155,13 @@ func (o *Options) Complete(f cmdutil.Factory, cmd *cobra.Command) error { func (o *InstallOptions) Install() error { // check if KubeBlocks has been installed - versionInfo, err := util.GetVersionInfo(o.Client) + v, err := util.GetVersionInfo(o.Client) if err != nil { return err } - if v := versionInfo[util.KubeBlocksApp]; len(v) > 0 { - printer.Warning(o.Out, "KubeBlocks %s already exists, repeated installation is not supported.\n\n", v) + if v.KubeBlocks != "" { + printer.Warning(o.Out, "KubeBlocks %s already exists, repeated installation is not supported.\n\n", v.KubeBlocks) fmt.Fprintln(o.Out, "If you want to upgrade it, please use \"kbcli kubeblocks upgrade\".") return nil } @@ -177,7 +177,7 @@ func (o *InstallOptions) Install() error { return err } - if err = o.preCheck(versionInfo); err != nil { + if err = o.preCheck(v); err != nil { return err } @@ -341,7 +341,7 @@ func (o *InstallOptions) waitAddonsEnabled() error { return nil } -func (o *InstallOptions) preCheck(versionInfo map[util.AppName]string) error { +func (o *InstallOptions) preCheck(v util.Version) error { if !o.Check { return nil } @@ -355,8 +355,8 @@ func (o *InstallOptions) preCheck(versionInfo map[util.AppName]string) error { } versionErr := fmt.Errorf("failed to get kubernetes version") - k8sVersionStr, ok := versionInfo[util.KubernetesApp] - if !ok { + k8sVersionStr := v.Kubernetes + if k8sVersionStr == "" { return versionErr } @@ -378,7 +378,7 @@ func (o *InstallOptions) preCheck(versionInfo map[util.AppName]string) error { } // check kbcli version, now do nothing - fmt.Fprintf(o.Out, "kbcli version %s\n", versionInfo[util.KBCLIApp]) + fmt.Fprintf(o.Out, "kbcli version %s\n", v.Cli) return nil } diff --git a/internal/cli/cmd/kubeblocks/install_test.go b/internal/cli/cmd/kubeblocks/install_test.go index 4e257c490..826176431 100644 --- a/internal/cli/cmd/kubeblocks/install_test.go +++ b/internal/cli/cmd/kubeblocks/install_test.go @@ -88,7 +88,7 @@ var _ = Describe("kubeblocks install", func() { CreateNamespace: true, } Expect(o.Install()).Should(HaveOccurred()) - Expect(len(o.ValueOpts.Values)).To(Equal(1)) + Expect(o.ValueOpts.Values).Should(HaveLen(1)) Expect(o.ValueOpts.Values[0]).To(Equal(fmt.Sprintf(kMonitorParam, true))) Expect(o.installChart()).Should(HaveOccurred()) o.printNotes() @@ -120,18 +120,15 @@ var _ = Describe("kubeblocks install", func() { Check: true, } By("kubernetes version is empty") - versionInfo := map[util.AppName]string{} - Expect(o.preCheck(versionInfo).Error()).Should(ContainSubstring("failed to get kubernetes version")) - - versionInfo[util.KubernetesApp] = "" - Expect(o.preCheck(versionInfo).Error()).Should(ContainSubstring("failed to get kubernetes version")) + v := util.Version{} + Expect(o.preCheck(v).Error()).Should(ContainSubstring("failed to get kubernetes version")) By("kubernetes is provided by cloud provider") - versionInfo[util.KubernetesApp] = "v1.25.0-eks" - Expect(o.preCheck(versionInfo)).Should(Succeed()) + v.Kubernetes = "v1.25.0-eks" + Expect(o.preCheck(v)).Should(Succeed()) By("kubernetes is not provided by cloud provider") - versionInfo[util.KubernetesApp] = "v1.25.0" - Expect(o.preCheck(versionInfo)).Should(Succeed()) + v.Kubernetes = "v1.25.0" + Expect(o.preCheck(v)).Should(Succeed()) }) }) diff --git a/internal/cli/cmd/kubeblocks/kubeblocks_objects.go b/internal/cli/cmd/kubeblocks/kubeblocks_objects.go index a5ccc51b7..6181ed6a9 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks_objects.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks_objects.go @@ -35,7 +35,6 @@ import ( extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/types" - "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/constant" ) @@ -102,7 +101,7 @@ func getKBObjects(dynamic dynamic.Interface, namespace string, addons []*extensi } result := &unstructured.UnstructuredList{} for _, obj := range objs.Items { - if !strings.Contains(obj.GetName(), strings.ToLower(string(util.KubeBlocksApp))) { + if !strings.Contains(obj.GetName(), strings.ToLower(types.KubeBlocksName)) { continue } result.Items = append(result.Items, obj) diff --git a/internal/cli/cmd/kubeblocks/preflight.go b/internal/cli/cmd/kubeblocks/preflight.go index c94ee2e4f..0a25f0914 100644 --- a/internal/cli/cmd/kubeblocks/preflight.go +++ b/internal/cli/cmd/kubeblocks/preflight.go @@ -160,7 +160,7 @@ func (p *PreflightOptions) complete(factory cmdutil.Factory, args []string) erro if err != nil { return errors.New("get k8s version of server failed, and please check your k8s accessibility") } - vendorName, err := util.GetK8sProvider(versionInfo[util.KubernetesApp], clientSet) + vendorName, err := util.GetK8sProvider(versionInfo.Kubernetes, clientSet) if err != nil { return errors.New("get k8s cloud provider failed, and please check your k8s accessibility") } diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index e702f3572..dbcc8360b 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -179,7 +179,7 @@ func (o *UninstallOptions) Uninstall() error { Name: types.KubeBlocksChartName, Namespace: o.Namespace, } - printSpinner(newSpinner("Uninstall helm release "+types.KubeBlocksChartName+" "+v[util.KubeBlocksApp]), + printSpinner(newSpinner("Uninstall helm release "+types.KubeBlocksReleaseName+" "+v.KubeBlocks), chart.Uninstall(o.HelmCfg)) // remove repo diff --git a/internal/cli/cmd/kubeblocks/upgrade.go b/internal/cli/cmd/kubeblocks/upgrade.go index 0bdfc037d..755c73dd7 100644 --- a/internal/cli/cmd/kubeblocks/upgrade.go +++ b/internal/cli/cmd/kubeblocks/upgrade.go @@ -87,23 +87,23 @@ func (o *InstallOptions) Upgrade() error { } // check if KubeBlocks has been installed - versionInfo, err := util.GetVersionInfo(o.Client) + v, err := util.GetVersionInfo(o.Client) if err != nil { return err } - v := versionInfo[util.KubeBlocksApp] - if len(v) == 0 { + kbVersion := v.KubeBlocks + if kbVersion == "" { return errors.New("KubeBlocks does not exist, try to run \"kbcli kubeblocks install\" to install") } - if v == o.Version && helm.ValueOptsIsEmpty(&o.ValueOpts) { + if kbVersion == o.Version && helm.ValueOptsIsEmpty(&o.ValueOpts) { fmt.Fprintf(o.Out, "Current version %s is the same as the upgraded version, no need to upgrade.\n", o.Version) return nil } fmt.Fprintf(o.Out, "Current KubeBlocks version %s.\n", v) - if err = o.preCheck(versionInfo); err != nil { + if err = o.preCheck(v); err != nil { return err } diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index 6dc08bec9..6bad7f071 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -417,10 +417,6 @@ func (o *initOptions) installKubeBlocks(k8sClusterName string) error { "snapshot-controller.enabled=true", "csi-hostpath-driver.enabled=true", - // enable aws loadbalancer controller addon automatically on playground - "aws-loadbalancer-controller.enabled=true", - fmt.Sprintf("aws-loadbalancer-controller.clusterName=%s", k8sClusterName), - // disable the persistent volume of prometheus, if not, the prometheus // will dependent the hostpath csi driver ready to create persistent // volume, but the order of addon installation is not guaranteed that diff --git a/internal/cli/cmd/version/version.go b/internal/cli/cmd/version/version.go index 695e8ec75..bc9b71f62 100644 --- a/internal/cli/cmd/version/version.go +++ b/internal/cli/cmd/version/version.go @@ -52,14 +52,14 @@ func (o *versionOptions) Run(f cmdutil.Factory) { klog.V(1).Infof("failed to get clientset: %v", err) } - versionInfo, _ := util.GetVersionInfo(client) - if v := versionInfo[util.KubernetesApp]; len(v) > 0 { - fmt.Printf("Kubernetes: %s\n", v) + v, _ := util.GetVersionInfo(client) + if v.Kubernetes != "" { + fmt.Printf("Kubernetes: %s\n", v.Kubernetes) } - if v := versionInfo[util.KubeBlocksApp]; len(v) > 0 { - fmt.Printf("KubeBlocks: %s\n", v) + if v.KubeBlocks != "" { + fmt.Printf("KubeBlocks: %s\n", v.KubeBlocks) } - fmt.Printf("kbcli: %s\n", versionInfo[util.KBCLIApp]) + fmt.Printf("kbcli: %s\n", v.Cli) if o.verbose { fmt.Printf(" BuildDate: %s\n", version.BuildDate) fmt.Printf(" GitCommit: %s\n", version.GitCommit) diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index 3b4c85079..a50b42686 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -144,6 +144,9 @@ const ( ) var ( + // KubeBlocksName is the name of KubeBlocks project + KubeBlocksName = "kubeblocks" + // KubeBlocksRepoName helm repo name for kubeblocks KubeBlocksRepoName = "kubeblocks" diff --git a/internal/cli/util/provider.go b/internal/cli/util/provider.go index f1bb46aac..b8c1b3138 100644 --- a/internal/cli/util/provider.go +++ b/internal/cli/util/provider.go @@ -81,17 +81,18 @@ var ( // GetK8sProvider returns the k8s provider func GetK8sProvider(version string, client kubernetes.Interface) (K8sProvider, error) { - nodes, err := client.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) - if err != nil { - return UnknownProvider, err - } - - provider := GetK8sProviderFromNodes(nodes) + // get provider from version first + provider := GetK8sProviderFromVersion(version) if provider != UnknownProvider { return provider, nil } - return GetK8sProviderFromVersion(version), nil + // if provider is unknown, get provider from node + nodes, err := client.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + return UnknownProvider, err + } + return GetK8sProviderFromNodes(nodes), nil } // GetK8sProviderFromNodes get k8s provider from node.spec.providerID diff --git a/internal/cli/util/version.go b/internal/cli/util/version.go index 14fa74a29..e7f759e82 100644 --- a/internal/cli/util/version.go +++ b/internal/cli/util/version.go @@ -30,34 +30,32 @@ import ( "github.com/apecloud/kubeblocks/version" ) -type AppName string - -const ( - KubernetesApp AppName = "Kubernetes" - KubeBlocksApp AppName = "KubeBlocks" - KBCLIApp AppName = "kbcli" -) +type Version struct { + KubeBlocks string + Kubernetes string + Cli string +} -// GetVersionInfo get application version include KubeBlocks, CLI and kubernetes -func GetVersionInfo(client kubernetes.Interface) (map[AppName]string, error) { +// GetVersionInfo get version include KubeBlocks, CLI and kubernetes +func GetVersionInfo(client kubernetes.Interface) (Version, error) { var err error - versionInfo := map[AppName]string{ - KBCLIApp: version.GetVersion(), + version := Version{ + Cli: version.GetVersion(), } if client == nil || reflect.ValueOf(client).IsNil() { - return versionInfo, nil + return version, nil } - if versionInfo[KubernetesApp], err = GetK8sVersion(client.Discovery()); err != nil { - return versionInfo, err + if version.Kubernetes, err = GetK8sVersion(client.Discovery()); err != nil { + return version, err } - if versionInfo[KubeBlocksApp], err = getKubeBlocksVersion(client); err != nil { - return versionInfo, err + if version.KubeBlocks, err = getKubeBlocksVersion(client); err != nil { + return version, err } - return versionInfo, nil + return version, nil } // getKubeBlocksVersion get KubeBlocks version diff --git a/internal/cli/util/version_test.go b/internal/cli/util/version_test.go index 67fe21c61..90439f2f3 100644 --- a/internal/cli/util/version_test.go +++ b/internal/cli/util/version_test.go @@ -29,42 +29,38 @@ const kbVersion = "0.3.0" var _ = Describe("version util", func() { It("get version info when client is nil", func() { - info, err := GetVersionInfo(nil) + v, err := GetVersionInfo(nil) Expect(err).Should(Succeed()) - Expect(info).ShouldNot(BeEmpty()) - Expect(info[KubeBlocksApp]).Should(BeEmpty()) - Expect(info[KubernetesApp]).Should(BeEmpty()) - Expect(info[KBCLIApp]).ShouldNot(BeEmpty()) + Expect(v.KubeBlocks).Should(BeEmpty()) + Expect(v.Kubernetes).Should(BeEmpty()) + Expect(v.Cli).ShouldNot(BeEmpty()) }) It("get version info when client variable is a nil pointer", func() { var client *kubernetes.Clientset - info, err := GetVersionInfo(client) + v, err := GetVersionInfo(client) Expect(err).Should(Succeed()) - Expect(info).ShouldNot(BeEmpty()) - Expect(info[KubeBlocksApp]).Should(BeEmpty()) - Expect(info[KubernetesApp]).Should(BeEmpty()) - Expect(info[KBCLIApp]).ShouldNot(BeEmpty()) + Expect(v.KubeBlocks).Should(BeEmpty()) + Expect(v.Kubernetes).Should(BeEmpty()) + Expect(v.Cli).ShouldNot(BeEmpty()) }) - It("get version info when KubeBlocks is deployed", func() { + It("get vsion info when KubeBlocks is deployed", func() { client := testing.FakeClientSet(testing.FakeKBDeploy(kbVersion)) - info, err := GetVersionInfo(client) + v, err := GetVersionInfo(client) Expect(err).Should(Succeed()) - Expect(info).ShouldNot(BeEmpty()) - Expect(info[KubeBlocksApp]).Should(Equal(kbVersion)) - Expect(info[KubernetesApp]).ShouldNot(BeEmpty()) - Expect(info[KBCLIApp]).ShouldNot(BeEmpty()) + Expect(v.KubeBlocks).Should(Equal(kbVersion)) + Expect(v.Kubernetes).ShouldNot(BeEmpty()) + Expect(v.Cli).ShouldNot(BeEmpty()) }) It("get version info when KubeBlocks is not deployed", func() { client := testing.FakeClientSet() - info, err := GetVersionInfo(client) + v, err := GetVersionInfo(client) Expect(err).Should(Succeed()) - Expect(info).ShouldNot(BeEmpty()) - Expect(info[KubeBlocksApp]).Should(BeEmpty()) - Expect(info[KubernetesApp]).ShouldNot(BeEmpty()) - Expect(info[KBCLIApp]).ShouldNot(BeEmpty()) + Expect(v.KubeBlocks).Should(BeEmpty()) + Expect(v.Kubernetes).ShouldNot(BeEmpty()) + Expect(v.Cli).ShouldNot(BeEmpty()) }) It("getKubeBlocksVersion", func() { From c9f35883128d7cc07589f58f63c63242468c364c Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Tue, 18 Apr 2023 14:19:56 +0800 Subject: [PATCH 075/439] feat: support mongodb backup (#2682) --- deploy/mongodb/scripts/replicaset-post-start.tpl | 1 - deploy/mongodb/templates/backuppolicytemplate.yaml | 7 +++++-- deploy/mongodb/templates/backuptool.yaml | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deploy/mongodb/scripts/replicaset-post-start.tpl b/deploy/mongodb/scripts/replicaset-post-start.tpl index b7aa0f7dc..a485e48f7 100644 --- a/deploy/mongodb/scripts/replicaset-post-start.tpl +++ b/deploy/mongodb/scripts/replicaset-post-start.tpl @@ -11,7 +11,6 @@ {{- $mongodb_port = $mongodb_port_info.containerPort }} {{- end }} -set -e PORT={{ $mongodb_port }} MONGODB_ROOT={{ $mongodb_root }} INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); diff --git a/deploy/mongodb/templates/backuppolicytemplate.yaml b/deploy/mongodb/templates/backuppolicytemplate.yaml index df4016795..a2377e97b 100644 --- a/deploy/mongodb/templates/backuppolicytemplate.yaml +++ b/deploy/mongodb/templates/backuppolicytemplate.yaml @@ -17,9 +17,12 @@ spec: cronExpression: "0 18 * * 0" snapshot: target: - role: leader + role: primary connectionCredentialKey: passwordKey: password usernameKey: username full: - backupToolName: xtrabackup-apecloud-mysql + backupToolName: mongodb-physical-backup-tool + backupsHistoryLimit: 7 + target: + role: primary diff --git a/deploy/mongodb/templates/backuptool.yaml b/deploy/mongodb/templates/backuptool.yaml index 9bad85a27..537c8fe45 100644 --- a/deploy/mongodb/templates/backuptool.yaml +++ b/deploy/mongodb/templates/backuptool.yaml @@ -23,7 +23,7 @@ spec: set -e mkdir -p ${DATA_DIR} res=`ls -A ${DATA_DIR}` - if [ ! -z ${res} ]; then + if [ ! -z "${res}" ]; then echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." exit 1 fi From 3c7ed0642ac8784985fcf35651cb5cea1fa6d169 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 18 Apr 2023 14:57:39 +0800 Subject: [PATCH 076/439] chore: fix viper not watch the changes of config file (#2681) --- cmd/manager/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index a8b9755ed..b1f6d1fae 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -26,6 +26,7 @@ import ( // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. + "github.com/fsnotify/fsnotify" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -181,6 +182,10 @@ func main() { setupLog.Info("unable read in config, errors ignored") } setupLog.Info(fmt.Sprintf("config file: %s", viper.GetViper().ConfigFileUsed())) + viper.OnConfigChange(func(e fsnotify.Event) { + setupLog.Info(fmt.Sprintf("config file changed: %s", e.Name)) + }) + viper.WatchConfig() metricsAddr = viper.GetString(metricsAddrFlagKey.viperName()) probeAddr = viper.GetString(probeAddrFlagKey.viperName()) From e858acc10703046a29ef33c909a4301c81d918b1 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Tue, 18 Apr 2023 15:47:54 +0800 Subject: [PATCH 077/439] fix: Redis configuration verification does not take effect (#2644) --- cmd/tpl/app/k8s_resource.go | 14 +++++++------- cmd/tpl/app/workflow.go | 6 +++--- deploy/redis/config/redis7-config-constraint.cue | 3 +++ internal/cli/cmd/cluster/config_edit.go | 8 ++++++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/tpl/app/k8s_resource.go b/cmd/tpl/app/k8s_resource.go index f9248c293..c67b87b9b 100644 --- a/cmd/tpl/app/k8s_resource.go +++ b/cmd/tpl/app/k8s_resource.go @@ -27,8 +27,8 @@ import ( ) func CustomizedObjFromYaml[T generics.Object, PT generics.PObject[T], - L generics.ObjList[T], PL generics.PObjList[T, L]](filePath string, signature func(T, L)) (PT, error) { - objList, err := CustomizedObjectListFromYaml[T, PT, L, PL](filePath, signature) + L generics.ObjList[T]](filePath string, signature func(T, L)) (PT, error) { + objList, err := CustomizedObjectListFromYaml[T, PT, L](filePath, signature) if err != nil { return nil, err } @@ -39,7 +39,7 @@ func CustomizedObjFromYaml[T generics.Object, PT generics.PObject[T], } func CustomizedObjectListFromYaml[T generics.Object, PT generics.PObject[T], - L generics.ObjList[T], PL generics.PObjList[T, L]](yamlfile string, signature func(T, L)) ([]PT, error) { + L generics.ObjList[T]](yamlfile string, signature func(T, L)) ([]PT, error) { objBytes, err := os.ReadFile(yamlfile) if err != nil { return nil, err @@ -49,13 +49,13 @@ func CustomizedObjectListFromYaml[T generics.Object, PT generics.PObject[T], if len(bytes.TrimSpace(doc)) == 0 { continue } - objList = append(objList, CreateTypedObjectFromYamlByte[T, PT, L, PL](doc, signature)) + objList = append(objList, CreateTypedObjectFromYamlByte[T, PT, L](doc, signature)) } return objList, nil } func CreateTypedObjectFromYamlByte[T generics.Object, PT generics.PObject[T], - L generics.ObjList[T], PL generics.PObjList[T, L]](yamlBytes []byte, _ func(T, L)) PT { + L generics.ObjList[T]](yamlBytes []byte, _ func(T, L)) PT { var obj PT if err := yaml.Unmarshal(yamlBytes, &obj); err != nil { return nil @@ -63,8 +63,8 @@ func CreateTypedObjectFromYamlByte[T generics.Object, PT generics.PObject[T], return obj } -func GetResourceObjectWithType[T generics.Object, PT generics.PObject[T], - L generics.ObjList[T], PL generics.PObjList[T, L]](objects []client.Object, _ func(T, L)) PT { +func GetTypedResourceObjectBySignature[T generics.Object, PT generics.PObject[T], + L generics.ObjList[T]](objects []client.Object, _ func(T, L)) PT { for _, object := range objects { if cd, ok := object.(PT); ok { return cd diff --git a/cmd/tpl/app/workflow.go b/cmd/tpl/app/workflow.go index eec943c4f..2ccd29bf3 100644 --- a/cmd/tpl/app/workflow.go +++ b/cmd/tpl/app/workflow.go @@ -152,7 +152,7 @@ func (w *templateRenderWorkflow) createClusterObject() (*appsv1alpha1.Cluster, e return CustomizedObjFromYaml(w.clusterYaml, generics.ClusterSignature) } - clusterVersionObj := GetResourceObjectWithType(w.localObjects, generics.ClusterVersionSignature) + clusterVersionObj := GetTypedResourceObjectBySignature(w.localObjects, generics.ClusterVersionSignature) return mockClusterObject(w.clusterDefObj, w.renderedOpts, clusterVersionObj), nil } @@ -166,7 +166,7 @@ func NewWorkflowTemplateRender(helmTemplateDir string, opts RenderedOptions) (*t return nil, err } - clusterDefObj := GetResourceObjectWithType(allObjects, generics.ClusterDefinitionSignature) + clusterDefObj := GetTypedResourceObjectBySignature(allObjects, generics.ClusterDefinitionSignature) if clusterDefObj == nil { return nil, cfgcore.MakeError("cluster definition object is not found in helm template directory[%s]", helmTemplateDir) } @@ -235,7 +235,7 @@ func createComponentParams(w *templateRenderWorkflow, ctx intctrlutil.RequestCtx if err != nil { return nil, err } - clusterVersionObj := GetResourceObjectWithType(w.localObjects, generics.ClusterVersionSignature) + clusterVersionObj := GetTypedResourceObjectBySignature(w.localObjects, generics.ClusterVersionSignature) task := intctrltypes.InitReconcileTask(w.clusterDefObj, clusterVersionObj, cluster, component) secret, err := builder.BuildConnCredential(task.GetBuilderParams()) if err != nil { diff --git a/deploy/redis/config/redis7-config-constraint.cue b/deploy/redis/config/redis7-config-constraint.cue index 46974f0f1..d36547ebe 100644 --- a/deploy/redis/config/redis7-config-constraint.cue +++ b/deploy/redis/config/redis7-config-constraint.cue @@ -148,3 +148,6 @@ ... } + +configuration: #RedisParameter & { +} \ No newline at end of file diff --git a/internal/cli/cmd/cluster/config_edit.go b/internal/cli/cmd/cluster/config_edit.go index d81e77b68..d3e5af1e7 100644 --- a/internal/cli/cmd/cluster/config_edit.go +++ b/internal/cli/cmd/cluster/config_edit.go @@ -125,6 +125,14 @@ func (o *editConfigOptions) Run(fn func(info *cfgcore.ConfigPatchInfo, cc *appsv if !yes { return nil } + + validatedData := map[string]string{ + o.CfgFile: cfgEditContext.getEdited(), + } + options := cfgcore.WithKeySelector(wrapper.ConfigSpec().Keys) + if err = cfgcore.NewConfigValidator(&configConstraint.Spec, options).Validate(validatedData); err != nil { + return cfgcore.WrapError(err, "failed to validate edited config") + } return fn(configPatch, &configConstraint.Spec) } From 6fa12d47fd4212d535e3743a00a339b30da94920 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 18 Apr 2023 17:47:45 +0800 Subject: [PATCH 078/439] chore: fix ut unstable error (#2695) --- .../dataprotection/backup_controller_test.go | 5 +-- .../backuppolicy_controller_test.go | 33 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index 16b374502..ff5593a96 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -17,6 +17,7 @@ limitations under the License. package dataprotection import ( + "fmt" "strings" "github.com/ghodss/yaml" @@ -68,7 +69,6 @@ var _ = Describe("Backup Controller test", func() { testapps.ClearResources(&testCtx, intctrlutil.JobSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.CronJobSignature, inNS, ml) testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS) - // // non-namespaced testapps.ClearResources(&testCtx, intctrlutil.BackupToolSignature, ml) } @@ -346,6 +346,7 @@ var _ = Describe("Backup Controller test", func() { } BeforeEach(func() { + viper.SetDefault(constant.CfgKeyBackupPVCStorageClass, "") By("By creating a backupTool") backupTool := testapps.CreateCustomizedObj(&testCtx, "backup/backuptool.yaml", &dataprotectionv1alpha1.BackupTool{}, testapps.RandomizedObjName(), @@ -398,7 +399,7 @@ var _ = Describe("Backup Controller test", func() { createBackup(backupName) Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dataprotectionv1alpha1.Backup) { g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupFailed)) - g.Expect(fetched.Status.FailureReason).To(ContainSubstring("not found")) + g.Expect(fetched.Status.FailureReason).To(ContainSubstring(fmt.Sprintf(`ConfigMap "%s" not found`, configMapName))) })).Should(Succeed()) configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index 88448228c..4ad815918 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -68,7 +68,7 @@ var _ = Describe("Backup Policy Controller", func() { testapps.ClearResources(&testCtx, intctrlutil.JobSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.CronJobSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.SecretSignature, inNS, ml) - testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS, ml) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS) // mgr namespaced inMgrNS := client.InNamespace(mgrNamespace) testapps.ClearResources(&testCtx, intctrlutil.CronJobSignature, inMgrNS, ml) @@ -185,38 +185,37 @@ var _ = Describe("Backup Policy Controller", func() { SetBackupType(dpv1alpha1.BackupTypeFull). Create(&testCtx).GetObject() - By("mock jobs completed") + By("waiting expired backup completed") backupExpiredKey := client.ObjectKeyFromObject(backupExpired) patchK8sJobStatus(backupExpiredKey, batchv1.JobComplete) - backupOutLimit1Key := client.ObjectKeyFromObject(backupOutLimit1) - patchK8sJobStatus(backupOutLimit1Key, batchv1.JobComplete) - backupOutLimit2Key := client.ObjectKeyFromObject(backupOutLimit2) - patchK8sJobStatus(backupOutLimit2Key, batchv1.JobComplete) - - By("waiting expired backup completed") Eventually(testapps.CheckObj(&testCtx, backupExpiredKey, func(g Gomega, fetched *dpv1alpha1.Backup) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.BackupCompleted)) })).Should(Succeed()) + By("mock update expired backup status to expire") + backupStatus.Expiration = &metav1.Time{Time: now.Add(-time.Hour * 24)} + backupStatus.StartTimestamp = backupStatus.Expiration + patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupExpired)) + By("waiting 1st limit backup completed") + backupOutLimit1Key := client.ObjectKeyFromObject(backupOutLimit1) + patchK8sJobStatus(backupOutLimit1Key, batchv1.JobComplete) Eventually(testapps.CheckObj(&testCtx, backupOutLimit1Key, func(g Gomega, fetched *dpv1alpha1.Backup) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.BackupCompleted)) })).Should(Succeed()) + By("mock update 1st limit backup NOT to expire") + backupStatus.Expiration = &metav1.Time{Time: now.Add(time.Hour * 24)} + backupStatus.StartTimestamp = &metav1.Time{Time: now.Add(time.Hour)} + patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupOutLimit1)) + By("waiting 2nd limit backup completed") + backupOutLimit2Key := client.ObjectKeyFromObject(backupOutLimit2) + patchK8sJobStatus(backupOutLimit2Key, batchv1.JobComplete) Eventually(testapps.CheckObj(&testCtx, backupOutLimit2Key, func(g Gomega, fetched *dpv1alpha1.Backup) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.BackupCompleted)) })).Should(Succeed()) - - By("mock update expired backup status to expire") - backupStatus.Expiration = &metav1.Time{Time: now.Add(-time.Hour * 24)} - backupStatus.StartTimestamp = backupStatus.Expiration - patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupExpired)) - By("mock update 1st limit backup NOT to expire") - backupStatus.Expiration = &metav1.Time{Time: now.Add(time.Hour * 24)} - backupStatus.StartTimestamp = &metav1.Time{Time: now.Add(time.Hour)} - patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupOutLimit1)) By("mock update 2nd limit backup NOT to expire") backupStatus.Expiration = &metav1.Time{Time: now.Add(time.Hour * 24)} backupStatus.StartTimestamp = &metav1.Time{Time: now.Add(time.Hour * 2)} From 66273f04bbd12c3f21ec5a056c513bd7900fb46d Mon Sep 17 00:00:00 2001 From: free6om Date: Tue, 18 Apr 2023 19:13:47 +0800 Subject: [PATCH 079/439] chore: ut for nil backup policy (#2654) --- controllers/apps/cluster_controller_test.go | 37 ++++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index eac2bf038..2e957b4f6 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -504,7 +504,7 @@ var _ = Describe("Cluster Controller", func() { return pods } - horizontalScaleComp := func(updatedReplicas int, comp *appsv1alpha1.ClusterComponentSpec) { + horizontalScaleComp := func(updatedReplicas int, comp *appsv1alpha1.ClusterComponentSpec, policy *appsv1alpha1.HorizontalScalePolicy) { By("Mocking components' PVCs to bound") for i := 0; i < int(comp.Replicas); i++ { pvcKey := types.NamespacedName{ @@ -537,6 +537,19 @@ var _ = Describe("Cluster Controller", func() { By(fmt.Sprintf("Changing replicas to %d", updatedReplicas)) changeCompReplicas(clusterKey, int32(updatedReplicas), comp) + checkUpdatedStsReplicas := func() { + By("Checking updated sts replicas") + Eventually(func() int32 { + stsList = testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, clusterKey, comp.Name) + return *stsList.Items[0].Spec.Replicas + }).Should(BeEquivalentTo(updatedReplicas)) + } + + if policy == nil { + checkUpdatedStsReplicas() + return + } + By("Checking Backup created") Eventually(testapps.GetListLen(&testCtx, intctrlutil.BackupSignature, client.MatchingLabels{ @@ -592,11 +605,7 @@ var _ = Describe("Cluster Controller", func() { }, client.InNamespace(clusterKey.Namespace))).Should(Equal(0)) Eventually(testapps.CheckObjExists(&testCtx, snapshotKey, &snapshotv1.VolumeSnapshot{}, false)).Should(Succeed()) - By("Checking updated sts replicas") - Eventually(func() int32 { - stsList = testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, clusterKey, comp.Name) - return *stsList.Items[0].Spec.Replicas - }).Should(BeEquivalentTo(updatedReplicas)) + checkUpdatedStsReplicas() } // @argument componentDefsWithHScalePolicy assign ClusterDefinition.spec.componentDefs[].horizontalScalePolicy for @@ -685,8 +694,17 @@ var _ = Describe("Cluster Controller", func() { })).Should(Succeed()) } } - for i := range clusterObj.Spec.ComponentSpecs { - horizontalScaleComp(updatedReplicas, &clusterObj.Spec.ComponentSpecs[i]) + + By("Get the latest cluster def") + Expect(k8sClient.Get(testCtx.Ctx, client.ObjectKeyFromObject(clusterDefObj), clusterDefObj)).Should(Succeed()) + for i, comp := range clusterObj.Spec.ComponentSpecs { + var policy *appsv1alpha1.HorizontalScalePolicy + for _, componentDef := range clusterDefObj.Spec.ComponentDefs { + if componentDef.Name == comp.ComponentDefRef { + policy = componentDef.HorizontalScalePolicy + } + } + horizontalScaleComp(updatedReplicas, &clusterObj.Spec.ComponentSpecs[i], policy) } By("Checking cluster status and the number of replicas changed") @@ -747,10 +765,11 @@ var _ = Describe("Cluster Controller", func() { By("Waiting for the cluster controller to create resources completely") waitForCreatingResourceCompletely(clusterKey, statefulCompName, consensusCompName, replicationCompName) + // statefulCompDefName not in componentDefsWithHScalePolicy, for nil backup policy test // REVIEW: (chantu) // 1. this test flow, wait for running phase? // 2. following horizontalScale only work with statefulCompDefName? - horizontalScale(int(updatedReplicas), statefulCompDefName, consensusCompDefName, replicationCompDefName) + horizontalScale(int(updatedReplicas), consensusCompDefName, replicationCompDefName) } testVerticalScale := func() { From 09c33efa5f18b3792aba70783d9e66f2c84ed7f1 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Tue, 18 Apr 2023 20:24:46 +0800 Subject: [PATCH 080/439] fix: failed to build for windows (#2690) --- .../configuration/config_manager/signal.go | 2 - .../config_manager/signal_test.go | 2 + .../config_manager/signal_unknown.go | 30 ------------- .../config_manager/signal_windows.go | 42 +++++++++++++++++++ 4 files changed, 44 insertions(+), 32 deletions(-) delete mode 100644 internal/configuration/config_manager/signal_unknown.go create mode 100644 internal/configuration/config_manager/signal_windows.go diff --git a/internal/configuration/config_manager/signal.go b/internal/configuration/config_manager/signal.go index c90069f88..1f09ef7ba 100644 --- a/internal/configuration/config_manager/signal.go +++ b/internal/configuration/config_manager/signal.go @@ -1,5 +1,3 @@ -//go:build linux || darwin - /* Copyright ApeCloud, Inc. diff --git a/internal/configuration/config_manager/signal_test.go b/internal/configuration/config_manager/signal_test.go index 3b08a35d9..9733cec92 100644 --- a/internal/configuration/config_manager/signal_test.go +++ b/internal/configuration/config_manager/signal_test.go @@ -1,3 +1,5 @@ +//go:build linux || darwin + /* Copyright ApeCloud, Inc. diff --git a/internal/configuration/config_manager/signal_unknown.go b/internal/configuration/config_manager/signal_unknown.go deleted file mode 100644 index aaa1feebe..000000000 --- a/internal/configuration/config_manager/signal_unknown.go +++ /dev/null @@ -1,30 +0,0 @@ -//go:build !linux && !darwin - -/* -Copyright ApeCloud, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package configmanager - -import ( - "os" - "runtime" - - cfgcore "github.com/apecloud/kubeblocks/internal/configuration" -) - -func sendSignal(pid PID, sig os.Signal) error { - return cfgcore.MakeError("not support os: ", runtime.GOOS) -} diff --git a/internal/configuration/config_manager/signal_windows.go b/internal/configuration/config_manager/signal_windows.go new file mode 100644 index 000000000..dfe9684c5 --- /dev/null +++ b/internal/configuration/config_manager/signal_windows.go @@ -0,0 +1,42 @@ +//go:build windows + +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configmanager + +import ( + "os" + "syscall" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" +) + +var allUnixSignals = map[appsv1alpha1.SignalType]os.Signal{ + appsv1alpha1.SIGHUP: syscall.SIGHUP, + appsv1alpha1.SIGINT: syscall.SIGINT, + appsv1alpha1.SIGQUIT: syscall.SIGQUIT, + appsv1alpha1.SIGILL: syscall.SIGILL, + appsv1alpha1.SIGTRAP: syscall.SIGTRAP, + appsv1alpha1.SIGABRT: syscall.SIGABRT, + appsv1alpha1.SIGBUS: syscall.SIGBUS, + appsv1alpha1.SIGFPE: syscall.SIGFPE, + appsv1alpha1.SIGKILL: syscall.SIGKILL, + appsv1alpha1.SIGSEGV: syscall.SIGSEGV, + appsv1alpha1.SIGPIPE: syscall.SIGPIPE, + appsv1alpha1.SIGALRM: syscall.SIGALRM, + appsv1alpha1.SIGTERM: syscall.SIGTERM, +} From f3f97afd330051fb7143dd28ac3fc94310d35f14 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 19 Apr 2023 10:12:24 +0800 Subject: [PATCH 081/439] chore: rename backupToolName and revert list-backups command (#2702) --- .../templates/backuppolicytemplate.yaml | 2 +- .../templates/backuptool.yaml | 2 +- .../templates/backuppolicytemplate.yaml | 2 +- .../apecloud-mysql/templates/backuptool.yaml | 2 +- docs/release_notes/v0.5.0/v0.5.0.md | 6 ++ docs/user_docs/cli/cli.md | 2 +- docs/user_docs/cli/kbcli_cluster.md | 2 +- .../cli/kbcli_cluster_list-backup.md | 58 ------------------- .../cli/kbcli_cluster_list-backups.md | 1 + internal/cli/cmd/cluster/dataprotection.go | 6 +- 10 files changed, 16 insertions(+), 67 deletions(-) delete mode 100644 docs/user_docs/cli/kbcli_cluster_list-backup.md diff --git a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml index 664060d00..acf514eb5 100644 --- a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml @@ -25,4 +25,4 @@ spec: target: role: leader full: - backupToolName: xtrabackup-apecloud-mysql-scale \ No newline at end of file + backupToolName: xtrabackup-for-apecloud-mysql-scale \ No newline at end of file diff --git a/deploy/apecloud-mysql-scale/templates/backuptool.yaml b/deploy/apecloud-mysql-scale/templates/backuptool.yaml index 577790874..b358a9a40 100644 --- a/deploy/apecloud-mysql-scale/templates/backuptool.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuptool.yaml @@ -1,7 +1,7 @@ apiVersion: dataprotection.kubeblocks.io/v1alpha1 kind: BackupTool metadata: - name: xtrabackup-apecloud-mysql-scale + name: xtrabackup-for-apecloud-mysql-scale labels: clusterdefinition.kubeblocks.io/name: apecloud-mysql {{- include "apecloud-mysql.labels" . | nindent 4 }} diff --git a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml index 7b2449d51..19b61c66f 100644 --- a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml @@ -25,4 +25,4 @@ spec: target: role: leader full: - backupToolName: xtrabackup-apecloud-mysql \ No newline at end of file + backupToolName: xtrabackup-for-apecloud-mysql \ No newline at end of file diff --git a/deploy/apecloud-mysql/templates/backuptool.yaml b/deploy/apecloud-mysql/templates/backuptool.yaml index eb42c15ac..e2e9f3402 100644 --- a/deploy/apecloud-mysql/templates/backuptool.yaml +++ b/deploy/apecloud-mysql/templates/backuptool.yaml @@ -1,7 +1,7 @@ apiVersion: dataprotection.kubeblocks.io/v1alpha1 kind: BackupTool metadata: - name: xtrabackup-apecloud-mysql + name: xtrabackup-for-apecloud-mysql labels: clusterdefinition.kubeblocks.io/name: apecloud-mysql {{- include "apecloud-mysql.labels" . | nindent 4 }} diff --git a/docs/release_notes/v0.5.0/v0.5.0.md b/docs/release_notes/v0.5.0/v0.5.0.md index 0dde6e548..e9b259d7f 100644 --- a/docs/release_notes/v0.5.0/v0.5.0.md +++ b/docs/release_notes/v0.5.0/v0.5.0.md @@ -46,4 +46,10 @@ Thanks to everyone who made this release possible! ``` kubectl delete backuppolicytemplates.dataprotection.kubeblocks.io --all kubectl delete backuppolicies.dataprotection.kubeblocks.io --all + ``` + - redefines the phase of cluster and component. + Before installing v0.5, please ensure that the resources have been cleaned up: + ``` + kubectl delete clusters.apps.kubeblocks.io --all + kubectl delete opsrequets.apps.kubeblocks.io --all ``` \ No newline at end of file diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index fbf3f5d79..4435d9d60 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -67,8 +67,8 @@ Cluster command. * [kbcli cluster label](kbcli_cluster_label.md) - Update the labels on cluster * [kbcli cluster list](kbcli_cluster_list.md) - List clusters. * [kbcli cluster list-accounts](kbcli_cluster_list-accounts.md) - List accounts for a cluster -* [kbcli cluster list-backup](kbcli_cluster_list-backup.md) - List backups. * [kbcli cluster list-backup-policy](kbcli_cluster_list-backup-policy.md) - List backups policies. +* [kbcli cluster list-backups](kbcli_cluster_list-backups.md) - List backups. * [kbcli cluster list-components](kbcli_cluster_list-components.md) - List cluster components. * [kbcli cluster list-events](kbcli_cluster_list-events.md) - List cluster events. * [kbcli cluster list-instances](kbcli_cluster_list-instances.md) - List cluster instances. diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index b3af4261f..b5d138e24 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -61,8 +61,8 @@ Cluster command. * [kbcli cluster label](kbcli_cluster_label.md) - Update the labels on cluster * [kbcli cluster list](kbcli_cluster_list.md) - List clusters. * [kbcli cluster list-accounts](kbcli_cluster_list-accounts.md) - List accounts for a cluster -* [kbcli cluster list-backup](kbcli_cluster_list-backup.md) - List backups. * [kbcli cluster list-backup-policy](kbcli_cluster_list-backup-policy.md) - List backups policies. +* [kbcli cluster list-backups](kbcli_cluster_list-backups.md) - List backups. * [kbcli cluster list-components](kbcli_cluster_list-components.md) - List cluster components. * [kbcli cluster list-events](kbcli_cluster_list-events.md) - List cluster events. * [kbcli cluster list-instances](kbcli_cluster_list-instances.md) - List cluster instances. diff --git a/docs/user_docs/cli/kbcli_cluster_list-backup.md b/docs/user_docs/cli/kbcli_cluster_list-backup.md deleted file mode 100644 index d760da28d..000000000 --- a/docs/user_docs/cli/kbcli_cluster_list-backup.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: kbcli cluster list-backup ---- - -List backups. - -``` -kbcli cluster list-backup [flags] -``` - -### Examples - -``` - # list all backup - kbcli cluster list-backup -``` - -### Options - -``` - -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. - -h, --help help for list-backup - --name string The backup name to get the details. - -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) - -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. - --show-labels When printing, show all labels as the last column (default hide labels column) -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli cluster](kbcli_cluster.md) - Cluster command. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_cluster_list-backups.md b/docs/user_docs/cli/kbcli_cluster_list-backups.md index c0d34dabe..c8f15ba45 100644 --- a/docs/user_docs/cli/kbcli_cluster_list-backups.md +++ b/docs/user_docs/cli/kbcli_cluster_list-backups.md @@ -20,6 +20,7 @@ kbcli cluster list-backups [flags] ``` -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. -h, --help help for list-backups + --name string The backup name to get the details. -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. --show-labels When printing, show all labels as the last column (default hide labels column) diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 92ab21873..48b7dab96 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -70,7 +70,7 @@ var ( `) listBackupExample = templates.Examples(` # list all backup - kbcli cluster list-backup + kbcli cluster list-backups `) deleteBackupExample = templates.Examples(` # delete a backup named backup-name @@ -242,7 +242,7 @@ func NewCreateBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) customOutPut := func(opt *create.BaseOptions) { output := fmt.Sprintf("Backup %s created successfully, you can view the progress:", opt.Name) printer.PrintLine(output) - nextLine := fmt.Sprintf("\tkbcli cluster list-backup --name=%s -n %s", opt.Name, opt.Namespace) + nextLine := fmt.Sprintf("\tkbcli cluster list-backups --name=%s -n %s", opt.Name, opt.Namespace) printer.PrintLine(nextLine) } inputs := create.Inputs{ @@ -339,7 +339,7 @@ func printBackupList(o ListBackupOptions) error { func NewListBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &ListBackupOptions{ListOptions: list.NewListOptions(f, streams, types.OpsGVR())} cmd := &cobra.Command{ - Use: "list-backup", + Use: "list-backups", Short: "List backups.", Aliases: []string{"ls-backup"}, Example: listBackupExample, From dea844e4b8a302d20aa13f973c75eb7994e66e31 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Wed, 19 Apr 2023 10:46:47 +0800 Subject: [PATCH 082/439] chore: improve cli examples and use (#2704) --- docs/user_docs/cli/cli.md | 2 +- docs/user_docs/cli/kbcli_cluster.md | 2 +- docs/user_docs/cli/kbcli_cluster_configure.md | 4 ++-- docs/user_docs/cli/kbcli_cluster_create.md | 2 +- docs/user_docs/cli/kbcli_cluster_edit-config.md | 5 +---- docs/user_docs/cli/kbcli_cluster_expose.md | 4 ++-- docs/user_docs/cli/kbcli_cluster_list-backup-policy.md | 2 +- internal/cli/cmd/cluster/config_edit.go | 9 +++++---- internal/cli/cmd/cluster/config_ops.go | 4 ++-- internal/cli/cmd/cluster/create.go | 2 +- internal/cli/cmd/cluster/dataprotection.go | 2 +- internal/cli/cmd/cluster/operations.go | 6 +++--- 12 files changed, 21 insertions(+), 23 deletions(-) diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index 4435d9d60..a0467e026 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -61,7 +61,7 @@ Cluster command. * [kbcli cluster edit-backup-policy](kbcli_cluster_edit-backup-policy.md) - Edit backup policy * [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. * [kbcli cluster explain-config](kbcli_cluster_explain-config.md) - List the constraint for supported configuration params. -* [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster with a new endpoint and the new endpoint can be found by executing the command 'kbcli cluster describe '. +* [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster with a new endpoint, the new endpoint can be found by executing 'kbcli cluster describe NAME'. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account * [kbcli cluster hscale](kbcli_cluster_hscale.md) - Horizontally scale the specified components in the cluster. * [kbcli cluster label](kbcli_cluster_label.md) - Update the labels on cluster diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index b5d138e24..215d34109 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -55,7 +55,7 @@ Cluster command. * [kbcli cluster edit-backup-policy](kbcli_cluster_edit-backup-policy.md) - Edit backup policy * [kbcli cluster edit-config](kbcli_cluster_edit-config.md) - Edit the config file of the component. * [kbcli cluster explain-config](kbcli_cluster_explain-config.md) - List the constraint for supported configuration params. -* [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster with a new endpoint and the new endpoint can be found by executing the command 'kbcli cluster describe '. +* [kbcli cluster expose](kbcli_cluster_expose.md) - Expose a cluster with a new endpoint, the new endpoint can be found by executing 'kbcli cluster describe NAME'. * [kbcli cluster grant-role](kbcli_cluster_grant-role.md) - Grant role to account * [kbcli cluster hscale](kbcli_cluster_hscale.md) - Horizontally scale the specified components in the cluster. * [kbcli cluster label](kbcli_cluster_label.md) - Update the labels on cluster diff --git a/docs/user_docs/cli/kbcli_cluster_configure.md b/docs/user_docs/cli/kbcli_cluster_configure.md index 23c046b34..396630cb2 100644 --- a/docs/user_docs/cli/kbcli_cluster_configure.md +++ b/docs/user_docs/cli/kbcli_cluster_configure.md @@ -5,14 +5,14 @@ title: kbcli cluster configure Reconfigure parameters with the specified components in the cluster. ``` -kbcli cluster configure [flags] +kbcli cluster configure NAME --set key=value[,key=value] [--component=component-name] [--config-spec=config-spec-name] [--config-file=config-file] [flags] ``` ### Examples ``` # update component params - kbcli cluster configure --component= --config-spec= --config-file= --set max_connections=1000,general_log=OFF + kbcli cluster configure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=1000,general_log=OFF # if only one component, and one config spec, and one config file, simplify the use of configure. e.g: # update mysql max_connections, cluster name is mycluster diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index 152b827e9..90ad8727a 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -5,7 +5,7 @@ title: kbcli cluster create Create a cluster. ``` -kbcli cluster create [CLUSTER_NAME] [flags] +kbcli cluster create [NAME] [flags] ``` ### Examples diff --git a/docs/user_docs/cli/kbcli_cluster_edit-config.md b/docs/user_docs/cli/kbcli_cluster_edit-config.md index 201cc04d6..3071485ab 100644 --- a/docs/user_docs/cli/kbcli_cluster_edit-config.md +++ b/docs/user_docs/cli/kbcli_cluster_edit-config.md @@ -5,15 +5,12 @@ title: kbcli cluster edit-config Edit the config file of the component. ``` -kbcli cluster edit-config [flags] +kbcli cluster edit-config NAME [--component=component-name] [--config-spec=config-spec-name] [--config-file=config-file] [flags] ``` ### Examples ``` - # edit config for component - kbcli cluster edit-config [--component=] [--config-spec=] [--config-file=] - # update mysql max_connections, cluster name is mycluster kbcli cluster edit-config mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf ``` diff --git a/docs/user_docs/cli/kbcli_cluster_expose.md b/docs/user_docs/cli/kbcli_cluster_expose.md index 9cdb467b2..21c96fcc8 100644 --- a/docs/user_docs/cli/kbcli_cluster_expose.md +++ b/docs/user_docs/cli/kbcli_cluster_expose.md @@ -2,10 +2,10 @@ title: kbcli cluster expose --- -Expose a cluster with a new endpoint and the new endpoint can be found by executing the command 'kbcli cluster describe '. +Expose a cluster with a new endpoint, the new endpoint can be found by executing 'kbcli cluster describe NAME'. ``` -kbcli cluster expose [flags] +kbcli cluster expose NAME --enable=[true|false] --type=[vpc|internet] [flags] ``` ### Examples diff --git a/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md b/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md index f0adc443c..5379ea1f2 100644 --- a/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md +++ b/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md @@ -15,7 +15,7 @@ kbcli cluster list-backup-policy [flags] kbcli cluster list-backup-policy # using short cmd to list backup policy of specified cluster - kbcli cluster list-bp + kbcli cluster list-bp mycluster ``` ### Options diff --git a/internal/cli/cmd/cluster/config_edit.go b/internal/cli/cmd/cluster/config_edit.go index d3e5af1e7..f88335ba0 100644 --- a/internal/cli/cmd/cluster/config_edit.go +++ b/internal/cli/cmd/cluster/config_edit.go @@ -44,13 +44,14 @@ type editConfigOptions struct { replaceFile bool } -var editConfigExample = templates.Examples(` - # edit config for component - kbcli cluster edit-config [--component=] [--config-spec=] [--config-file=] +var ( + editConfigUse = "edit-config NAME [--component=component-name] [--config-spec=config-spec-name] [--config-file=config-file]" + editConfigExample = templates.Examples(` # update mysql max_connections, cluster name is mycluster kbcli cluster edit-config mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf `) +) func (o *editConfigOptions) Run(fn func(info *cfgcore.ConfigPatchInfo, cc *appsv1alpha1.ConfigConstraintSpec) error) error { wrapper := o.wrapper @@ -163,7 +164,7 @@ func NewEditConfigureCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) OperationsOptions: newBaseOperationsOptions(streams, appsv1alpha1.ReconfiguringType, false), }} inputs := buildOperationsInputs(f, editOptions.OperationsOptions) - inputs.Use = "edit-config" + inputs.Use = editConfigUse inputs.Short = "Edit the config file of the component." inputs.Example = editConfigExample inputs.BuildFlags = func(cmd *cobra.Command) { diff --git a/internal/cli/cmd/cluster/config_ops.go b/internal/cli/cmd/cluster/config_ops.go index e076d8e63..c0c398a9a 100644 --- a/internal/cli/cmd/cluster/config_ops.go +++ b/internal/cli/cmd/cluster/config_ops.go @@ -51,7 +51,7 @@ type configOpsOptions struct { var ( createReconfigureExample = templates.Examples(` # update component params - kbcli cluster configure --component= --config-spec= --config-file= --set max_connections=1000,general_log=OFF + kbcli cluster configure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=1000,general_log=OFF # if only one component, and one config spec, and one config file, simplify the use of configure. e.g: # update mysql max_connections, cluster name is mycluster @@ -206,7 +206,7 @@ func NewReconfigureCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) * OperationsOptions: newBaseOperationsOptions(streams, appsv1alpha1.ReconfiguringType, false), } inputs := buildOperationsInputs(f, o.OperationsOptions) - inputs.Use = "configure" + inputs.Use = "configure NAME --set key=value[,key=value] [--component=component-name] [--config-spec=config-spec-name] [--config-file=config-file]" inputs.Short = "Reconfigure parameters with the specified components in the cluster." inputs.Example = createReconfigureExample inputs.BuildFlags = func(cmd *cobra.Command) { diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 3aa788f17..745a7dd32 100644 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -345,7 +345,7 @@ func MultipleSourceComponents(fileName string, in io.Reader) ([]byte, error) { func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &CreateOptions{BaseOptions: create.BaseOptions{IOStreams: streams}} inputs := create.Inputs{ - Use: "create [CLUSTER_NAME]", + Use: "create [NAME]", Short: "Create a cluster.", Example: clusterCreateExample, CueTemplateName: CueTemplateName, diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 48b7dab96..8033d1255 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -55,7 +55,7 @@ var ( kbcli cluster list-backup-policy # using short cmd to list backup policy of specified cluster - kbcli cluster list-bp + kbcli cluster list-bp mycluster `) editExample = templates.Examples(` # edit backup policy diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 0b1414594..782900937 100644 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -425,12 +425,12 @@ var ( `) ) -// NewExposeCmd creates a Expose command +// NewExposeCmd creates an expose command func NewExposeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := newBaseOperationsOptions(streams, appsv1alpha1.ExposeType, true) inputs := buildOperationsInputs(f, o) - inputs.Use = "expose" - inputs.Short = "Expose a cluster with a new endpoint and the new endpoint can be found by executing the command 'kbcli cluster describe '." + inputs.Use = "expose NAME --enable=[true|false] --type=[vpc|internet]" + inputs.Short = "Expose a cluster with a new endpoint, the new endpoint can be found by executing 'kbcli cluster describe NAME'." inputs.Example = exposeExamples inputs.BuildFlags = func(cmd *cobra.Command) { o.buildCommonFlags(cmd) From 0d6c4ce889e15ee23a9d24c4afea33a8f472b5b7 Mon Sep 17 00:00:00 2001 From: xingran Date: Wed, 19 Apr 2023 11:51:10 +0800 Subject: [PATCH 083/439] fix: replicationSet cluster stop failed fix (#2691) --- controllers/apps/components/replication/replication.go | 3 +++ internal/controller/plan/prepare.go | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/controllers/apps/components/replication/replication.go b/controllers/apps/components/replication/replication.go index 6a325be57..e607036dc 100644 --- a/controllers/apps/components/replication/replication.go +++ b/controllers/apps/components/replication/replication.go @@ -145,6 +145,9 @@ func (r *ReplicationComponent) HandleUpdate(ctx context.Context, obj client.Obje if err != nil { return err } + if len(podList) == 0 { + return nil + } for _, pod := range podList { // if there is no role label on the Pod, it needs to be updated with statefulSet's role label. if v, ok := pod.Labels[constant.RoleLabelKey]; !ok || v == "" { diff --git a/internal/controller/plan/prepare.go b/internal/controller/plan/prepare.go index be925766f..a21bf20e4 100644 --- a/internal/controller/plan/prepare.go +++ b/internal/controller/plan/prepare.go @@ -145,15 +145,15 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, return err } case appsv1alpha1.Replication: - // get the number of existing statefulsets under the current component - var existStsList = &appsv1.StatefulSetList{} - if err := componentutil.GetObjectListByComponentName(reqCtx.Ctx, cli, *task.Cluster, existStsList, task.Component.Name); err != nil { + // get the number of existing pods under the current component + var existPodList = &corev1.PodList{} + if err := componentutil.GetObjectListByComponentName(reqCtx.Ctx, cli, *task.Cluster, existPodList, task.Component.Name); err != nil { return err } - // If the statefulSets already exists, check whether there is an HA switching and the HA process is prioritized to handle. + // If the Pods already exists, check whether there is an HA switching and the HA process is prioritized to handle. // TODO(xingran) After refactoring, HA switching will be handled in the replicationSet controller. - if len(existStsList.Items) > 0 { + if len(existPodList.Items) > 0 { primaryIndexChanged, _, err := replication.CheckPrimaryIndexChanged(reqCtx.Ctx, cli, task.Cluster, task.Component.Name, task.Component.GetPrimaryIndex()) if err != nil { From b44b103e3f76aca1e979509c8e27c78d94ca3d7e Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:58:35 +0800 Subject: [PATCH 084/439] fix: configure pg does not take effect (#2633) (#2703) --- deploy/apecloud-mysql/config/mysql8-config.tpl | 12 ++++++------ deploy/postgresql/config/pg14-config.tpl | 2 +- deploy/postgresql/templates/clusterdefinition.yaml | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/deploy/apecloud-mysql/config/mysql8-config.tpl b/deploy/apecloud-mysql/config/mysql8-config.tpl index eacd9acf3..7663a8a7a 100644 --- a/deploy/apecloud-mysql/config/mysql8-config.tpl +++ b/deploy/apecloud-mysql/config/mysql8-config.tpl @@ -206,12 +206,12 @@ loose_xengine_compression_per_level=kZSTD:kZSTD:kZSTD {{- if gt $phy_memory 0 }} {{- $phy_memory := div $phy_memory ( mul 1024 1024 ) }} -#loose_xengine_write_buffer_size={{ min ( max 32 ( mulf $phy_memory 0.01 ) ) 256 | int | mul 1024 1024 }} -#loose_xengine_db_write_buffer_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} -#loose_xengine_db_total_write_buffer_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} -#loose_xengine_block_cache_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} -#loose_xengine_row_cache_size={{ mulf $phy_memory 0.1 | int | mul 1024 1024 }} -#loose_xengine_max_total_wal_size={{ min ( mulf $phy_memory 0.3 ) ( mul 12 1024 ) | int | mul 1024 1024 }} +loose_xengine_write_buffer_size={{ min ( max 32 ( mulf $phy_memory 0.01 ) ) 256 | int | mul 1024 1024 }} +loose_xengine_db_write_buffer_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} +loose_xengine_db_total_write_buffer_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} +loose_xengine_block_cache_size={{ mulf $phy_memory 0.3 | int | mul 1024 1024 }} +loose_xengine_row_cache_size={{ mulf $phy_memory 0.1 | int | mul 1024 1024 }} +loose_xengine_max_total_wal_size={{ min ( mulf $phy_memory 0.3 ) ( mul 12 1024 ) | int | mul 1024 1024 }} {{- end }} {{- if gt $phy_cpu 0 }} diff --git a/deploy/postgresql/config/pg14-config.tpl b/deploy/postgresql/config/pg14-config.tpl index c38b79027..38aa38697 100644 --- a/deploy/postgresql/config/pg14-config.tpl +++ b/deploy/postgresql/config/pg14-config.tpl @@ -89,7 +89,7 @@ pg_stat_statements.track_utility = 'False' random_page_cost = '1.1' #auto generated shared_buffers = '{{ printf "%d%s" $shared_buffers $buffer_unit }}' -shared_preload_libraries = 'pg_stat_statements, auto_explain' +# shared_preload_libraries = 'pg_stat_statements,auto_explain,bg_mon,pgextwlist,pg_auth_mon,set_user,pg_cron,pg_stat_kcache' superuser_reserved_connections = '10' temp_file_limit = '100GB' #timescaledb.max_background_workers = '6' diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 82cae01f6..6848e679f 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -131,6 +131,8 @@ spec: value: '{"app.kubernetes.io/instance":"$(KB_CLUSTER_NAME)","apps.kubeblocks.io/component-name":"$(KB_COMP_NAME)","app.kubernetes.io/managed-by":"kubeblocks"}' - name: RESTORE_DATA_DIR value: /home/postgres/pgdata/kb_restore + - name: KB_PG_CONFIG_PATH + value: /home/postgres/conf/postgresql.conf - name: SPILO_CONFIGURATION value: | ## https://github.com/zalando/patroni#yaml-configuration bootstrap: From 2e793d43e6950fdb0c8e035a360a761d59d11915 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 19 Apr 2023 15:06:40 +0800 Subject: [PATCH 085/439] fix: command in kbcli cluster describe-ops is incorrect (#2719) --- internal/cli/cmd/cluster/describe_ops.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cli/cmd/cluster/describe_ops.go b/internal/cli/cmd/cluster/describe_ops.go index d75e384ed..9309f4870 100644 --- a/internal/cli/cmd/cluster/describe_ops.go +++ b/internal/cli/cmd/cluster/describe_ops.go @@ -268,7 +268,7 @@ func (o *describeOpsOptions) getVerticalScalingCommand(spec appsv1alpha1.OpsRequ commands := make([]string, len(componentNameSlice)) for i := range componentNameSlice { resource := resourceSlice[i].(corev1.ResourceRequirements) - commands[i] = fmt.Sprintf("kbcli cluster vertical-scale %s --components=%s", + commands[i] = fmt.Sprintf("kbcli cluster vscale %s --components=%s", spec.ClusterRef, strings.Join(componentNameSlice[i], ",")) commands[i] += o.addResourceFlag("requests.cpu", resource.Requests.Cpu()) commands[i] += o.addResourceFlag("requests.memory", resource.Requests.Memory()) @@ -293,7 +293,7 @@ func (o *describeOpsOptions) getHorizontalScalingCommand(spec appsv1alpha1.OpsRe spec.HorizontalScalingList, convertObject, getCompName) commands := make([]string, len(componentNameSlice)) for i := range componentNameSlice { - commands[i] = fmt.Sprintf("kbcli cluster horizontal-scale %s --components=%s --replicas=%d", + commands[i] = fmt.Sprintf("kbcli cluster hscale %s --components=%s --replicas=%d", spec.ClusterRef, strings.Join(componentNameSlice[i], ","), replicasSlice[i].(int32)) } return commands From c7eba333a07e93abb5a3c95756aca638b833cf54 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Wed, 19 Apr 2023 15:29:10 +0800 Subject: [PATCH 086/439] chore: sqlchannel add test (#2694) --- internal/controller/component/probe_utils.go | 1 - internal/sqlchannel/client_test.go | 23 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/internal/controller/component/probe_utils.go b/internal/controller/component/probe_utils.go index 247b92c37..559ee3b3d 100644 --- a/internal/controller/component/probe_utils.go +++ b/internal/controller/component/probe_utils.go @@ -122,7 +122,6 @@ func buildProbeServiceContainer(component *SynthesizedComponent, container *core container.Command = []string{"probe", "--app-id", "batch-sdk", "--dapr-http-port", strconv.Itoa(probeSvcHTTPPort), "--dapr-grpc-port", strconv.Itoa(probeSvcGRPCPort), - "--app-protocol", "http", "--log-level", logLevel, "--config", "/config/probe/config.yaml", "--components-path", "/config/probe/components"} diff --git a/internal/sqlchannel/client_test.go b/internal/sqlchannel/client_test.go index 38a0045ec..ad0505664 100644 --- a/internal/sqlchannel/client_test.go +++ b/internal/sqlchannel/client_test.go @@ -18,12 +18,15 @@ package sqlchannel import ( "context" + "encoding/json" "fmt" "net" + "os" "strings" "testing" "time" + dapr "github.com/dapr/go-sdk/client" pb "github.com/dapr/go-sdk/dapr/proto/runtime/v1" "google.golang.org/grpc" corev1 "k8s.io/api/core/v1" @@ -85,6 +88,26 @@ func TestNewClientWithPod(t *testing.T) { }) } +func TestGPRC(t *testing.T) { + url := os.Getenv("PROBE_GRPC_URL") + if url == "" { + t.SkipNow() + } + req := &dapr.InvokeBindingRequest{ + Name: "mongodb", + Operation: "getRole", + Data: []byte(""), + Metadata: map[string]string{}, + } + cli, _ := dapr.NewClientWithAddress(url) + resp, _ := cli.InvokeBinding(context.Background(), req) + t.Logf("probe response metadata: %v", resp.Metadata) + result := map[string]string{} + _ = json.Unmarshal(resp.Data, &result) + t.Logf("probe response data: %v", result) + +} + func TestGetRole(t *testing.T) { port, closer := newTCPServer(t, 50001) defer closer() From 7fe14d72edf52933e4fc14bfd640b823c4b27daa Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 19 Apr 2023 15:29:43 +0800 Subject: [PATCH 087/439] fix: vscale command is incorrect in kbcli describe-ops (#2728) --- internal/cli/cmd/cluster/describe_ops.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/cli/cmd/cluster/describe_ops.go b/internal/cli/cmd/cluster/describe_ops.go index 9309f4870..a317b13c3 100644 --- a/internal/cli/cmd/cluster/describe_ops.go +++ b/internal/cli/cmd/cluster/describe_ops.go @@ -270,10 +270,8 @@ func (o *describeOpsOptions) getVerticalScalingCommand(spec appsv1alpha1.OpsRequ resource := resourceSlice[i].(corev1.ResourceRequirements) commands[i] = fmt.Sprintf("kbcli cluster vscale %s --components=%s", spec.ClusterRef, strings.Join(componentNameSlice[i], ",")) - commands[i] += o.addResourceFlag("requests.cpu", resource.Requests.Cpu()) - commands[i] += o.addResourceFlag("requests.memory", resource.Requests.Memory()) - commands[i] += o.addResourceFlag("limits.cpu", resource.Limits.Cpu()) - commands[i] += o.addResourceFlag("limits.memory", resource.Limits.Memory()) + commands[i] += o.addResourceFlag("cpu", resource.Limits.Cpu()) + commands[i] += o.addResourceFlag("memory", resource.Limits.Memory()) } return commands } From 164db7d99a80ba00d5422fcb773f5b8a873440d1 Mon Sep 17 00:00:00 2001 From: free6om Date: Wed, 19 Apr 2023 15:39:57 +0800 Subject: [PATCH 088/439] fix: h-scale pvc unexpected deletion (#2680) --- .../controller/lifecycle/transform_utils.go | 32 ++ .../lifecycle/transformer_object_action.go | 37 +-- .../transformer_sts_horizontal_scaling.go | 75 ++++- ...transformer_sts_horizontal_scaling_test.go | 312 +++++------------- 4 files changed, 188 insertions(+), 268 deletions(-) diff --git a/internal/controller/lifecycle/transform_utils.go b/internal/controller/lifecycle/transform_utils.go index efba6a66f..456b267ca 100644 --- a/internal/controller/lifecycle/transform_utils.go +++ b/internal/controller/lifecycle/transform_utils.go @@ -17,7 +17,9 @@ limitations under the License. package lifecycle import ( + "context" "fmt" + "reflect" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -187,3 +189,33 @@ func getBackupPolicyFromTemplate(reqCtx intctrlutil.RequestCtx, } return nil, nil } + +// read all objects owned by our cluster +func readCacheSnapshot(ctx context.Context, cli types2.ReadonlyClient, cluster appsv1alpha1.Cluster, kinds ...client.ObjectList) (clusterSnapshot, error) { + // list what kinds of object cluster owns + snapshot := make(clusterSnapshot) + ml := client.MatchingLabels{constant.AppInstanceLabelKey: cluster.GetName()} + inNS := client.InNamespace(cluster.Namespace) + for _, list := range kinds { + if err := cli.List(ctx, list, inNS, ml); err != nil { + return nil, err + } + // reflect get list.Items + items := reflect.ValueOf(list).Elem().FieldByName("Items") + l := items.Len() + for i := 0; i < l; i++ { + // get the underlying object + object := items.Index(i).Addr().Interface().(client.Object) + // put to snapshot if owned by our cluster + if isOwnerOf(&cluster, object, scheme) { + name, err := getGVKName(object, scheme) + if err != nil { + return nil, err + } + snapshot[*name] = object + } + } + } + + return snapshot, nil +} diff --git a/internal/controller/lifecycle/transformer_object_action.go b/internal/controller/lifecycle/transformer_object_action.go index 42ea4d1fe..fa66a1a43 100644 --- a/internal/controller/lifecycle/transformer_object_action.go +++ b/internal/controller/lifecycle/transformer_object_action.go @@ -17,7 +17,6 @@ limitations under the License. package lifecycle import ( - "reflect" "strings" appsv1 "k8s.io/api/apps/v1" @@ -28,7 +27,6 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" - "github.com/apecloud/kubeblocks/internal/constant" client2 "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/graph" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" @@ -47,43 +45,11 @@ func ownKinds() []client.ObjectList { &corev1.ServiceList{}, &corev1.SecretList{}, &corev1.ConfigMapList{}, - &corev1.PersistentVolumeClaimList{}, &policyv1.PodDisruptionBudgetList{}, &dataprotectionv1alpha1.BackupPolicyList{}, } } -// read all objects owned by our cluster -func (c *objectActionTransformer) readCacheSnapshot(cluster appsv1alpha1.Cluster) (clusterSnapshot, error) { - // list what kinds of object cluster owns - kinds := ownKinds() - snapshot := make(clusterSnapshot) - ml := client.MatchingLabels{constant.AppInstanceLabelKey: cluster.GetName()} - inNS := client.InNamespace(cluster.Namespace) - for _, list := range kinds { - if err := c.cli.List(c.ctx.Ctx, list, inNS, ml); err != nil { - return nil, err - } - // reflect get list.Items - items := reflect.ValueOf(list).Elem().FieldByName("Items") - l := items.Len() - for i := 0; i < l; i++ { - // get the underlying object - object := items.Index(i).Addr().Interface().(client.Object) - // put to snapshot if owned by our cluster - if isOwnerOf(&cluster, object, scheme) { - name, err := getGVKName(object, scheme) - if err != nil { - return nil, err - } - snapshot[*name] = object - } - } - } - - return snapshot, nil -} - func (c *objectActionTransformer) Transform(dag *graph.DAG) error { rootVertex, err := findRootVertex(dag) if err != nil { @@ -92,7 +58,7 @@ func (c *objectActionTransformer) Transform(dag *graph.DAG) error { origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) // get the old objects snapshot - oldSnapshot, err := c.readCacheSnapshot(*origCluster) + oldSnapshot, err := readCacheSnapshot(c.ctx.Ctx, c.cli, *origCluster, ownKinds()...) if err != nil { return err } @@ -133,6 +99,7 @@ func (c *objectActionTransformer) Transform(dag *graph.DAG) error { v.action = actionPtr(UPDATE) } } + deleteOrphanVertices := func() { for name := range deleteSet { v := &lifecycleVertex{ diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index ff65d53f4..29a68a0a5 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -29,6 +29,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -56,10 +57,6 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) cluster, _ := rootVertex.obj.(*appsv1alpha1.Cluster) - if isClusterDeleting(*origCluster) { - return nil - } - handleHorizontalScaling := func(vertex *lifecycleVertex) error { stsObj, _ := vertex.oriObj.(*appsv1.StatefulSet) stsProto, _ := vertex.obj.(*appsv1.StatefulSet) @@ -243,17 +240,71 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { return nil } - - vertices := findAll[*appsv1.StatefulSet](dag) - for _, vertex := range vertices { - v, _ := vertex.(*lifecycleVertex) - if v.obj == nil || v.oriObj == nil { - continue + findPVCsToBeDeleted := func(pvcSnapshot clusterSnapshot) []*corev1.PersistentVolumeClaim { + stsToBeDeleted := make([]*appsv1.StatefulSet, 0) + // list sts to be deleted + for _, vertex := range dag.Vertices() { + v, _ := vertex.(*lifecycleVertex) + // find sts to be deleted + if sts, ok := v.obj.(*appsv1.StatefulSet); ok && (v.action != nil && *v.action == DELETE) { + stsToBeDeleted = append(stsToBeDeleted, sts) + } } - if err := handleHorizontalScaling(v); err != nil { - return err + // compose all pvc names that owned by sts to be deleted + pvcNameSet := sets.New[string]() + for _, sts := range stsToBeDeleted { + for _, template := range sts.Spec.VolumeClaimTemplates { + for i := 0; i < int(*sts.Spec.Replicas); i++ { + name := fmt.Sprintf("%s-%s-%d", template.Name, sts.Name, i) + pvcNameSet.Insert(name) + } + } + } + // pvcs that not owned by any deleting sts should be filtered + orphanPVCs := make([]*corev1.PersistentVolumeClaim, 0) + for _, obj := range pvcSnapshot { + pvc, _ := obj.(*corev1.PersistentVolumeClaim) + if pvcNameSet.Has(pvc.Name) { + orphanPVCs = append(orphanPVCs, pvc) + } + } + return orphanPVCs + } + + // if cluster is deleting, no need h-scale + if !isClusterDeleting(*origCluster) { + vertices := findAll[*appsv1.StatefulSet](dag) + for _, vertex := range vertices { + v, _ := vertex.(*lifecycleVertex) + if v.obj == nil || v.oriObj == nil { + continue + } + if err := handleHorizontalScaling(v); err != nil { + return err + } } } + + // find all pvcs that should be deleted when parent sts is deleting: + // 1. cluster is deleting + // 2. component is deleting by a cluster Update + // + // why handle pvc deletion here? + // two types of pvc should be handled: generated by sts and by our h-scale transformer. + // by sts: we only handle the pvc deletion which occurs in cluster deletion. + // by h-scale transformer: we handle the pvc creation and deletion, the creation is handled in h-scale funcs. + // so all in all, here we should only handle the pvc deletion of both types. + oldSnapshot, err := readCacheSnapshot(s.ctx.Ctx, s.cli, *cluster, &corev1.PersistentVolumeClaimList{}) + if err != nil { + return err + } + pvcs := findPVCsToBeDeleted(oldSnapshot) + for _, pvc := range pvcs { + vertex := &lifecycleVertex{obj: pvc, action: actionPtr(DELETE)} + dag.AddVertex(vertex) + dag.Connect(rootVertex, vertex) + } + return nil } diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go index fae154a4a..a2439b062 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go @@ -17,229 +17,99 @@ limitations under the License. package lifecycle import ( + "context" + + "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/internal/controller/graph" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" + "github.com/apecloud/kubeblocks/internal/testutil/apps" + testutil "github.com/apecloud/kubeblocks/internal/testutil/k8s" ) var _ = Describe("sts horizontal scaling test", func() { - // TODO: refactor the following ut - // - // ctx := context.Background() - // newReqCtx := func() intctrlutil.RequestCtx { - // reqCtx := intctrlutil.RequestCtx{ - // Ctx: ctx, - // Log: logger, - // Recorder: clusterRecorder, - // } - // return reqCtx - // } - // - // newVolumeSnapshot := func(clusterName, componentName string) *snapshotv1.VolumeSnapshot { - // vsYAML := ` - // - // apiVersion: snapshot.storage.k8s.io/v1 - // kind: VolumeSnapshot - // metadata: - // - // labels: - // app.kubernetes.io/name: mysql-apecloud-mysql - // backupjobs.dataprotection.kubeblocks.io/name: wesql-01-replicasets-scaling-qf6cr - // backuppolicies.dataprotection.kubeblocks.io/name: wesql-01-replicasets-scaling-hcxps - // dataprotection.kubeblocks.io/backup-type: snapshot - // name: test-volume-snapshot - // namespace: default - // - // spec: - // - // source: - // persistentVolumeClaimName: data-wesql-01-replicasets-0 - // volumeSnapshotClassName: csi-aws-ebs-snapclass - // - // ` - // - // vs := snapshotv1.VolumeSnapshot{} - // Expect(yaml.Unmarshal([]byte(vsYAML), &vs)).ShouldNot(HaveOccurred()) - // labels := map[string]string{ - // constant.KBManagedByKey: "cluster", - // constant.AppInstanceLabelKey: clusterName, - // constant.KBAppComponentLabelKey: componentName, - // } - // for k, v := range labels { - // vs.Labels[k] = v - // } - // return &vs - // } - // - // Context("with HorizontalScalePolicy set to CloneFromSnapshot and VolumeSnapshot exists", func() { - // It("determines return value of doBackup according to whether VolumeSnapshot is ReadyToUse", func() { - // By("prepare cluster and construct component") - // reqCtx := newReqCtx() - // cluster, clusterDef, clusterVersion, _ := newAllFieldsClusterObj(nil, nil, false) - // component := component.BuildComponent( - // reqCtx, - // *cluster, - // *clusterDef, - // clusterDef.Spec.ComponentDefs[0], - // cluster.Spec.ComponentSpecs[0], - // &clusterVersion.Spec.ComponentVersions[0]) - // Expect(component).ShouldNot(BeNil()) - // component.HorizontalScalePolicy = &appsv1alpha1.HorizontalScalePolicy{ - // Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, - // VolumeMountsName: "data", - // } - // - // By("prepare VolumeSnapshot and set ReadyToUse to true") - // vs := newVolumeSnapshot(cluster.Name, mysqlCompName) - // Expect(testCtx.CreateObj(ctx, vs)).Should(Succeed()) - // Expect(testapps.ChangeObjStatus(&testCtx, vs, func() { - // t := true - // vs.Status = &snapshotv1.VolumeSnapshotStatus{ReadyToUse: &t} - // })).Should(Succeed()) - // - // // prepare doBackup input parameters - // snapshotKey := types.NamespacedName{ - // Namespace: "default", - // Name: "test-snapshot", - // } - // sts := newStsObj() - // stsProto := *sts.DeepCopy() - // r := int32(3) - // stsProto.Spec.Replicas = &r - // - // By("doBackup should return requeue=false") - // shouldRequeue, err := doBackup(reqCtx, k8sClient, cluster, component, sts, &stsProto, snapshotKey) - // Expect(err).ShouldNot(HaveOccurred()) - // Expect(shouldRequeue).Should(BeFalse()) - // - // By("Set ReadyToUse to nil, doBackup should return requeue=true") - // Expect(testapps.ChangeObjStatus(&testCtx, vs, func() { - // vs.Status = &snapshotv1.VolumeSnapshotStatus{ReadyToUse: nil} - // })).Should(Succeed()) - // shouldRequeue, err = doBackup(reqCtx, k8sClient, cluster, component, sts, &stsProto, snapshotKey) - // Expect(err).ShouldNot(HaveOccurred()) - // Expect(shouldRequeue).Should(BeTrue()) - // }) - // - // // REIVEW: this test seems always failed - // It("should do backup to create volumesnapshot when there exists a deleting volumesnapshot", func() { - // By("prepare cluster and construct component") - // reqCtx := newReqCtx() - // cluster, clusterDef, clusterVersion, _ := newAllFieldsClusterObj(nil, nil, false) - // component := component.BuildComponent( - // reqCtx, - // *cluster, - // *clusterDef, - // clusterDef.Spec.ComponentDefs[0], - // cluster.Spec.ComponentSpecs[0], - // &clusterVersion.Spec.ComponentVersions[0]) - // Expect(component).ShouldNot(BeNil()) - // component.HorizontalScalePolicy = &appsv1alpha1.HorizontalScalePolicy{ - // Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, - // VolumeMountsName: "data", - // } - // - // By("prepare VolumeSnapshot and set finalizer to prevent it from deletion") - // vs := newVolumeSnapshot(cluster.Name, mysqlCompName) - // Expect(testCtx.CreateObj(ctx, vs)).Should(Succeed()) - // Expect(testapps.ChangeObj(&testCtx, vs, func() { - // vs.Finalizers = append(vs.Finalizers, "test-finalizer") - // })).Should(Succeed()) - // - // By("deleting volume snapshot") - // Expect(k8sClient.Delete(ctx, vs)).Should(Succeed()) - // - // By("checking DeletionTimestamp exists") - // Eventually(func(g Gomega) { - // tmpVS := snapshotv1.VolumeSnapshot{} - // g.Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: vs.Namespace, Name: vs.Name}, &tmpVS)).Should(Succeed()) - // g.Expect(tmpVS.DeletionTimestamp).ShouldNot(BeNil()) - // }).Should(Succeed()) - // - // // prepare doBackup input parameters - // snapshotKey := types.NamespacedName{ - // Namespace: "default", - // Name: "test-snapshot", - // } - // sts := newStsObj() - // stsProto := *sts.DeepCopy() - // r := int32(3) - // stsProto.Spec.Replicas = &r - // - // By("doBackup should create volumesnapshot and return requeue=true") - // shouldRequeue, err := doBackup(reqCtx, k8sClient, cluster, component, sts, &stsProto, snapshotKey) - // Expect(err).ShouldNot(HaveOccurred()) - // Expect(shouldRequeue).Should(BeTrue()) - // - // newVS := snapshotv1.VolumeSnapshot{} - // By("checking volumesnapshot created by doBackup exists") - // Eventually(func(g Gomega) { - // g.Expect(k8sClient.Get(ctx, snapshotKey, &newVS)).Should(Succeed()) - // }).Should(Succeed()) - // - // By("mocking volumesnapshot status ready") - // Expect(testapps.ChangeObjStatus(&testCtx, &newVS, func() { - // t := true - // newVS.Status = &snapshotv1.VolumeSnapshotStatus{ReadyToUse: &t} - // })).Should(Succeed()) - // - // By("do backup again, this time should create pvcs") - // shouldRequeue, err = doBackup(reqCtx, k8sClient, cluster, component, sts, &stsProto, snapshotKey) - // - // By("checking not requeue, since create pvc is the last step of doBackup") - // Expect(shouldRequeue).Should(BeFalse()) - // Expect(err).ShouldNot(HaveOccurred()) - // - // By("checking pvcs reference right volumesnapshot") - // Eventually(func(g Gomega) { - // for i := *stsProto.Spec.Replicas - 1; i > *sts.Spec.Replicas; i-- { - // pvc := &corev1.PersistentVolumeClaim{} - // g.Expect(k8sClient.Get(ctx, - // types.NamespacedName{ - // Namespace: cluster.Namespace, - // Name: fmt.Sprintf("%s-%s-%d", testapps.DataVolumeName, sts.Name, i)}, - // pvc)).Should(Succeed()) - // g.Expect(pvc.Spec.DataSource.Name).Should(Equal(snapshotKey.Name)) - // } - // }).Should(Succeed()) - // - // By("remove finalizer to clean up") - // Expect(testapps.ChangeObj(&testCtx, vs, func() { - // vs.SetFinalizers(vs.Finalizers[:len(vs.Finalizers)-1]) - // })).Should(Succeed()) - // }) - // }) - // - // Context("backup test", func() { - // It("should not delete backups not created by lifecycle", func() { - // backupPolicyName := "test-backup-policy" - // backupName := "test-backup-job" - // - // By("creating a backup as user do") - // backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - // SetTTL("7d"). - // SetBackupPolicyName(backupPolicyName). - // SetBackupType(dataprotectionv1alpha1.BackupTypeSnapshot). - // AddAppInstanceLabel(clusterName). - // AddAppComponentLabel(mysqlCompName). - // AddAppManangedByLabel(). - // Create(&testCtx).GetObject() - // backupKey := client.ObjectKeyFromObject(backup) - // - // By("checking backup exists") - // Eventually(func(g Gomega) { - // tmpBackup := dataprotectionv1alpha1.Backup{} - // g.Expect(k8sClient.Get(ctx, backupKey, &tmpBackup)).Should(Succeed()) - // g.Expect(tmpBackup.Labels[constant.AppInstanceLabelKey]).NotTo(Equal("")) - // g.Expect(tmpBackup.Labels[constant.KBAppComponentLabelKey]).NotTo(Equal("")) - // }).Should(Succeed()) - // - // By("call deleteBackup in lifecycle which should only delete backups created by itself") - // Expect(deleteBackup(ctx, k8sClient, clusterName, mysqlCompName)) - // - // By("checking backup does not be deleted") - // Consistently(func(g Gomega) { - // tmpBackup := dataprotectionv1alpha1.Backup{} - // Expect(k8sClient.Get(ctx, backupKey, &tmpBackup)).Should(Succeed()) - // }).Should(Succeed()) - // }) - // }) + When("h-scale with cluster reconcile", func() { + It("should not delete pvcs generated by h-scale transformer", func() { + var ( + namespace = "default" + clusterDefName = "sts-h-scale-cluster-def" + componentDefName = "foo" + clusterVerName = "sts-h-scale-cluster-ver" + clusterName = "sts-h-scale-cluster" + componentName = "bar" + volumeName = "data" + stsName = clusterName + "-" + componentName + pvcNameBase = volumeName + "-" + stsName + "-" + ) + By("prepare cd, cv, cluster, sts and pvcs") + cd := apps.NewClusterDefFactory(clusterDefName). + AddComponentDef(apps.ConsensusMySQLComponent, componentDefName). + GetObject() + cv := apps.NewClusterVersionFactory(clusterVerName, cd.Name). + AddComponent(componentDefName). + GetObject() + cluster := apps.NewClusterFactory(namespace, clusterName, cd.Name, cv.Name). + AddComponent(componentName, componentDefName). + SetReplicas(3). + AddVolumeClaimTemplate(volumeName, apps.NewPVCSpec("1G")). + GetObject() + template := cluster.Spec.ComponentSpecs[0].ToVolumeClaimTemplates()[0] + sts := apps.NewStatefulSetFactory(namespace, stsName, cluster.Name, componentName). + SetReplicas(3). + AddVolumeClaimTemplate(corev1.PersistentVolumeClaim{ObjectMeta: template.ObjectMeta, Spec: template.Spec}). + GetObject() + origSts := sts.DeepCopy() + pvc1 := apps.NewPersistentVolumeClaimFactory(namespace, pvcNameBase+"1", cluster.Name, componentName, volumeName). + AddAppInstanceLabel(clusterName). + GetObject() + Expect(intctrlutil.SetOwnership(cluster, pvc1, scheme, dbClusterFinalizerName)).Should(Succeed()) + pvc2 := pvc1.DeepCopy() + pvc2.Name = pvcNameBase + "2" + Expect(intctrlutil.SetOwnership(cluster, pvc2, scheme, dbClusterFinalizerName)).Should(Succeed()) + + By("prepare params for transformer") + reqCtx := intctrlutil.RequestCtx{ + Ctx: context.Background(), + } + ctrl, k8sMock := testutil.SetupK8sMock() + defer ctrl.Finish() + cr := clusterRefResources{cd: *cd, cv: *cv} + + transformer := &stsHorizontalScalingTransformer{ctx: reqCtx, cli: k8sMock, cr: cr} + + By("prepare initial DAG with sts.action=UPDATE") + dag := graph.NewDAG() + rootVertex := &lifecycleVertex{obj: cluster, oriObj: cluster.DeepCopy(), action: actionPtr(STATUS)} + dag.AddVertex(rootVertex) + stsVertex := &lifecycleVertex{obj: sts, oriObj: origSts, action: actionPtr(UPDATE)} + dag.AddVertex(stsVertex) + dag.Connect(rootVertex, stsVertex) + By("mock client.List pvcs") + k8sMock.EXPECT(). + List(gomock.Any(), &corev1.PersistentVolumeClaimList{}, gomock.Any()). + DoAndReturn( + func(_ context.Context, list *corev1.PersistentVolumeClaimList, _ ...client.ListOption) error { + list.Items = []corev1.PersistentVolumeClaim{ + *pvc1, + *pvc2, + } + return nil + }).AnyTimes() + + By("do transform") + Expect(transformer.Transform(dag)).Should(Succeed()) + Expect(len(findAll[*corev1.PersistentVolumeClaim](dag))).Should(Equal(0)) + + By("prepare initial DAG with sts.action=DELETE") + stsVertex.action = actionPtr(DELETE) + + By("do transform") + Expect(transformer.Transform(dag)).Should(Succeed()) + Expect(len(findAll[*corev1.PersistentVolumeClaim](dag))).Should(Equal(2)) + }) + }) }) From e49de203de0cac2aef68adaf6b9635d7bda27393 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Wed, 19 Apr 2023 15:42:17 +0800 Subject: [PATCH 089/439] chore: useraccount update flag name (#2722) --- docs/user_docs/cli/kbcli_cluster_create-account.md | 2 +- docs/user_docs/cli/kbcli_cluster_delete-account.md | 2 +- docs/user_docs/cli/kbcli_cluster_describe-account.md | 2 +- docs/user_docs/cli/kbcli_cluster_grant-role.md | 2 +- docs/user_docs/cli/kbcli_cluster_revoke-role.md | 2 +- internal/cli/cmd/accounts/create.go | 2 +- internal/cli/cmd/accounts/delete.go | 2 +- internal/cli/cmd/accounts/describe.go | 2 +- internal/cli/cmd/accounts/grant.go | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/user_docs/cli/kbcli_cluster_create-account.md b/docs/user_docs/cli/kbcli_cluster_create-account.md index 28a3330d3..7b0078f75 100644 --- a/docs/user_docs/cli/kbcli_cluster_create-account.md +++ b/docs/user_docs/cli/kbcli_cluster_create-account.md @@ -25,8 +25,8 @@ kbcli cluster create-account [flags] --component string Specify the name of component to be connected. If not specified, the first component will be used. -h, --help help for create-account -i, --instance string Specify the name of instance to be connected. + --name string Required. Specify the name of user, which must be unique. -p, --password string Optional. Specify the password of user. The default value is empty, which means a random password will be generated. - -u, --username string Required. Specify the name of user, which must be unique. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_delete-account.md b/docs/user_docs/cli/kbcli_cluster_delete-account.md index 0cd2b59ab..72b32eb66 100644 --- a/docs/user_docs/cli/kbcli_cluster_delete-account.md +++ b/docs/user_docs/cli/kbcli_cluster_delete-account.md @@ -21,7 +21,7 @@ kbcli cluster delete-account [flags] --component string Specify the name of component to be connected. If not specified, the first component will be used. -h, --help help for delete-account -i, --instance string Specify the name of instance to be connected. - -u, --username string Required. Specify the name of user + --name string Required. Specify the name of user ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_describe-account.md b/docs/user_docs/cli/kbcli_cluster_describe-account.md index c45332b1f..793c90020 100644 --- a/docs/user_docs/cli/kbcli_cluster_describe-account.md +++ b/docs/user_docs/cli/kbcli_cluster_describe-account.md @@ -21,7 +21,7 @@ kbcli cluster describe-account [flags] --component string Specify the name of component to be connected. If not specified, the first component will be used. -h, --help help for describe-account -i, --instance string Specify the name of instance to be connected. - -u, --username string Required. Specify the name of user + --name string Required. Specify the name of user ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_grant-role.md b/docs/user_docs/cli/kbcli_cluster_grant-role.md index 632e2b143..5826f516e 100644 --- a/docs/user_docs/cli/kbcli_cluster_grant-role.md +++ b/docs/user_docs/cli/kbcli_cluster_grant-role.md @@ -21,8 +21,8 @@ kbcli cluster grant-role [flags] --component string Specify the name of component to be connected. If not specified, the first component will be used. -h, --help help for grant-role -i, --instance string Specify the name of instance to be connected. + --name string Required. Specify the name of user. -r, --role string Role name should be one of {SUPERUSER, READWRITE, READONLY} - -u, --username string Required. Specify the name of user. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_revoke-role.md b/docs/user_docs/cli/kbcli_cluster_revoke-role.md index 454cc7bee..1d8bf3933 100644 --- a/docs/user_docs/cli/kbcli_cluster_revoke-role.md +++ b/docs/user_docs/cli/kbcli_cluster_revoke-role.md @@ -21,8 +21,8 @@ kbcli cluster revoke-role [flags] --component string Specify the name of component to be connected. If not specified, the first component will be used. -h, --help help for revoke-role -i, --instance string Specify the name of instance to be connected. + --name string Required. Specify the name of user. -r, --role string Role name should be one of {SUPERUSER, READWRITE, READONLY} - -u, --username string Required. Specify the name of user. ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/accounts/create.go b/internal/cli/cmd/accounts/create.go index 63e6d6617..0d6e9eecd 100644 --- a/internal/cli/cmd/accounts/create.go +++ b/internal/cli/cmd/accounts/create.go @@ -38,7 +38,7 @@ func NewCreateUserOptions(f cmdutil.Factory, streams genericclioptions.IOStreams func (o *CreateUserOptions) AddFlags(cmd *cobra.Command) { o.AccountBaseOptions.AddFlags(cmd) - cmd.Flags().StringVarP(&o.info.UserName, "username", "u", "", "Required. Specify the name of user, which must be unique.") + cmd.Flags().StringVar(&o.info.UserName, "name", "", "Required. Specify the name of user, which must be unique.") cmd.Flags().StringVarP(&o.info.Password, "password", "p", "", "Optional. Specify the password of user. The default value is empty, which means a random password will be generated.") // TODO:@shanshan add expire flag if needed // cmd.Flags().DurationVar(&o.info.ExpireAt, "expire", 0, "Optional. Specify the expire time of password. The default value is 0, which means the user will never expire.") diff --git a/internal/cli/cmd/accounts/delete.go b/internal/cli/cmd/accounts/delete.go index 1e13bb110..2f7d95cb6 100644 --- a/internal/cli/cmd/accounts/delete.go +++ b/internal/cli/cmd/accounts/delete.go @@ -38,7 +38,7 @@ func NewDeleteUserOptions(f cmdutil.Factory, streams genericclioptions.IOStreams func (o *DeleteUserOptions) AddFlags(cmd *cobra.Command) { o.AccountBaseOptions.AddFlags(cmd) - cmd.Flags().StringVarP(&o.info.UserName, "username", "u", "", "Required. Specify the name of user") + cmd.Flags().StringVar(&o.info.UserName, "name", "", "Required. Specify the name of user") } func (o *DeleteUserOptions) Validate(args []string) error { diff --git a/internal/cli/cmd/accounts/describe.go b/internal/cli/cmd/accounts/describe.go index 5ed56c42e..3754b296d 100644 --- a/internal/cli/cmd/accounts/describe.go +++ b/internal/cli/cmd/accounts/describe.go @@ -37,7 +37,7 @@ func NewDescribeUserOptions(f cmdutil.Factory, streams genericclioptions.IOStrea func (o *DescribeUserOptions) AddFlags(cmd *cobra.Command) { o.AccountBaseOptions.AddFlags(cmd) - cmd.Flags().StringVarP(&o.info.UserName, "username", "u", "", "Required. Specify the name of user") + cmd.Flags().StringVar(&o.info.UserName, "name", "", "Required. Specify the name of user") } func (o DescribeUserOptions) Validate(args []string) error { diff --git a/internal/cli/cmd/accounts/grant.go b/internal/cli/cmd/accounts/grant.go index b70c9130d..c64f8f75d 100644 --- a/internal/cli/cmd/accounts/grant.go +++ b/internal/cli/cmd/accounts/grant.go @@ -46,7 +46,7 @@ func NewGrantOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, op func (o *GrantOptions) AddFlags(cmd *cobra.Command) { o.AccountBaseOptions.AddFlags(cmd) - cmd.Flags().StringVarP(&o.info.UserName, "username", "u", "", "Required. Specify the name of user.") + cmd.Flags().StringVar(&o.info.UserName, "name", "", "Required. Specify the name of user.") cmd.Flags().StringVarP(&o.info.RoleName, "role", "r", "", "Role name should be one of {SUPERUSER, READWRITE, READONLY}") } From 76e16d76400168f65418601e4ed435da53af5069 Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Wed, 19 Apr 2023 19:07:36 +0800 Subject: [PATCH 090/439] chore: update the powershell script to specify the version in remote (#2677) --- hack/install_cli.ps1 | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/hack/install_cli.ps1 b/hack/install_cli.ps1 index e617a3405..29befbfb8 100644 --- a/hack/install_cli.ps1 +++ b/hack/install_cli.ps1 @@ -1,3 +1,6 @@ +param ( + [string]$v +) # kbcli filename $CLI_FILENAME = "kbcli" @@ -10,8 +13,6 @@ $GITLAB_REPO = "85948" $GITLAB = "https://jihulab.com/api/v4/projects" $COUNTRY_CODE = "" -Import-Module Microsoft.PowerShell.Utility - function getCountryCode() { return (Invoke-WebRequest -Uri "https://ifconfig.io/country_code" -UseBasicParsing | Select-Object -ExpandProperty Content).Trim() } @@ -63,7 +64,7 @@ function getLatestRelease { $webClient = New-Object System.Net.WebClient $isDownLoaded = $False $Data = -$timeout = New-TimeSpan -Seconds 60 + function downloadFile { param ( $LATEST_RELEASE_TAG @@ -74,9 +75,8 @@ function downloadFile { $DOWNLOAD_BASE = "$GITLAB/$GITLAB_REPO/packages/generic/kubeblocks" } $DOWNLOAD_URL = "${DOWNLOAD_BASE}/${LATEST_RELEASE_TAG}/${CLI_ARTIFACT}" + # Check the Resource - # Write-Host DOWNLOAD_URL = $DOWNLOAD_URL - $webRequest = [System.Net.HttpWebRequest]::Create($DOWNLOAD_URL) $webRequest.Method = "HEAD" try { @@ -86,6 +86,7 @@ function downloadFile { Write-Host "Resource not found." exit 1 } + # Create the temp directory $CLI_TMP_ROOT = New-Item -ItemType Directory -Path (Join-Path $env:TEMP "kbcli-install-$(Get-Date -Format 'yyyyMMddHHmmss')") $Global:ARTIFACT_TMP_FILE = Join-Path $CLI_TMP_ROOT $CLI_ARTIFACT @@ -191,15 +192,15 @@ checkExistingCLI $COUNTRY_CODE = getCountryCode $ret_val -if (-not $args) { +if (-not $v) { Write-Host "Getting the latest kbcli ..." $ret_val = getLatestRelease } -elseif ($args[0] -match "^v.*$") { - $ret_val = $args[0] +elseif ($v -match "^v.*$") { + $ret_val = $v } else { - $ret_val = "v" + $args[0] + $ret_val = "v" + $v } $CLI_TMP_ROOT = downloadFile $ret_val @@ -213,6 +214,4 @@ try { } cleanup -installCompleted - - +installCompleted \ No newline at end of file From 46d1a4f8886a08f028bb911218f10fc5fb779f24 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 19 Apr 2023 19:35:19 +0800 Subject: [PATCH 091/439] chore: fix backup data directory is not hierarchical (#2739) --- apis/dataprotection/v1alpha1/backup_types.go | 4 -- .../dataprotection.kubeblocks.io_backups.yaml | 3 -- .../dataprotection/backup_controller.go | 37 +++++++++++-------- .../dataprotection/backup_controller_test.go | 3 ++ .../templates/backuptool.yaml | 4 +- .../apecloud-mysql/templates/backuptool.yaml | 2 +- .../dataprotection.kubeblocks.io_backups.yaml | 3 -- internal/constant/const.go | 3 +- .../controller/component/restore_utils.go | 7 +++- .../transformer_backup_policy_tpl.go | 28 +++++++------- 10 files changed, 51 insertions(+), 43 deletions(-) diff --git a/apis/dataprotection/v1alpha1/backup_types.go b/apis/dataprotection/v1alpha1/backup_types.go index e79ec0f48..96b0147b0 100644 --- a/apis/dataprotection/v1alpha1/backup_types.go +++ b/apis/dataprotection/v1alpha1/backup_types.go @@ -134,10 +134,6 @@ type BackupSnapshotStatus struct { } type BackupToolManifestsStatus struct { - // backupToolName referenced backup tool name. - // +optional - BackupToolName string `json:"backupToolName,omitempty"` - // filePath records the file path of backup. // +optional FilePath string `json:"filePath,omitempty"` diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml index 062b596c3..19138c606 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml @@ -137,9 +137,6 @@ spec: CheckPoint: description: backup check point, for incremental backup. type: string - backupToolName: - description: backupToolName referenced backup tool name. - type: string checkSum: description: checksum of backup file, generated by md5 or sha1 or sha256 diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index b235392b5..70e6dda46 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -317,6 +317,14 @@ func (r *BackupReconciler) buildAutoCreationAnnotations(backupPolicyName string) } } +// getBackupPathPrefix gets the backup path prefix. +func (r *BackupReconciler) getBackupPathPrefix(backupNamespace, pathPrefix string) string { + if strings.TrimSpace(pathPrefix) == "" || strings.HasPrefix(pathPrefix, "/") { + return fmt.Sprintf("/%s%s", backupNamespace, pathPrefix) + } + return fmt.Sprintf("/%s/%s", backupNamespace, pathPrefix) +} + func (r *BackupReconciler) doInProgressPhaseAction( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) (ctrl.Result, error) { @@ -382,7 +390,8 @@ func (r *BackupReconciler) doInProgressPhaseAction( // TODO: add error type return r.updateStatusIfFailed(reqCtx, backup, fmt.Errorf("not found the %s policy", backup.Spec.BackupType)) } - err = r.createBackupToolJob(reqCtx, backup, commonPolicy) + pathPrefix := r.getBackupPathPrefix(backup.Namespace, backupPolicy.Annotations[constant.BackupDataPathPrefixAnnotationKey]) + err = r.createBackupToolJob(reqCtx, backup, commonPolicy, pathPrefix) if err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) } @@ -403,6 +412,11 @@ func (r *BackupReconciler) doInProgressPhaseAction( // update Phase to in Completed backup.Status.Phase = dataprotectionv1alpha1.BackupCompleted backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now().UTC()} + backup.Status.Manifests = &dataprotectionv1alpha1.ManifestsStatus{ + BackupTool: &dataprotectionv1alpha1.BackupToolManifestsStatus{ + FilePath: pathPrefix, + }, + } } else if jobStatusConditions[0].Type == batchv1.JobFailed { backup.Status.Phase = dataprotectionv1alpha1.BackupFailed backup.Status.FailureReason = job.Status.Conditions[0].Reason @@ -682,7 +696,8 @@ func (r *BackupReconciler) createMetadataCollectionJob(reqCtx intctrlutil.Reques func (r *BackupReconciler) createBackupToolJob( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, - commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) error { + commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy, + pathPrefix string) error { key := types.NamespacedName{Namespace: backup.Namespace, Name: backup.Name} job := batchv1.Job{} @@ -695,7 +710,7 @@ func (r *BackupReconciler) createBackupToolJob( return nil } - toolPodSpec, err := r.buildBackupToolPodSpec(reqCtx, backup, commonPolicy) + toolPodSpec, err := r.buildBackupToolPodSpec(reqCtx, backup, commonPolicy, pathPrefix) if err != nil { return err } @@ -938,7 +953,8 @@ func (r *BackupReconciler) getTargetPVCs(reqCtx intctrlutil.RequestCtx, func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, - commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) (corev1.PodSpec, error) { + commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy, + pathPrefix string) (corev1.PodSpec, error) { podSpec := corev1.PodSpec{} // get backup tool backupTool := &dataprotectionv1alpha1.BackupTool{} @@ -1008,21 +1024,12 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, Value: backup.Name, } - envBackupDirPrefix := corev1.EnvVar{ - Name: "BACKUP_DIR_PREFIX", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - } - envBackupDir := corev1.EnvVar{ Name: "BACKUP_DIR", - Value: remoteBackupPath + "/$(BACKUP_DIR_PREFIX)", + Value: remoteBackupPath + pathPrefix, } - container.Env = []corev1.EnvVar{envDBHost, envBackupName, envBackupDirPrefix, envBackupDir} + container.Env = []corev1.EnvVar{envDBHost, envBackupName, envBackupDir} if commonPolicy.Target.Secret != nil { envDBUser := corev1.EnvVar{ Name: "DB_USER", diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index ff5593a96..28385b1b5 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -336,6 +336,7 @@ var _ = Describe("Backup Controller test", func() { Context("creates a full backup", func() { var backupKey types.NamespacedName var backupPolicy *dataprotectionv1alpha1.BackupPolicy + var pathPrefix = "/mysql/backup" createBackup := func(backupName string) { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). @@ -356,6 +357,7 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backupPolicy from backupTool: " + backupTool.Name) backupPolicy = testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). + AddAnnotations(constant.BackupDataPathPrefixAnnotationKey, pathPrefix). AddFullPolicy(). SetBackupToolName(backupTool.Name). SetSchedule(defaultSchedule, true). @@ -373,6 +375,7 @@ var _ = Describe("Backup Controller test", func() { By("Check backup job completed") Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dataprotectionv1alpha1.Backup) { g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupCompleted)) + g.Expect(fetched.Status.Manifests.BackupTool.FilePath).To(Equal(fmt.Sprintf("/%s%s", backupKey.Namespace, pathPrefix))) })).Should(Succeed()) }) diff --git a/deploy/apecloud-mysql-scale/templates/backuptool.yaml b/deploy/apecloud-mysql-scale/templates/backuptool.yaml index b358a9a40..341844543 100644 --- a/deploy/apecloud-mysql-scale/templates/backuptool.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuptool.yaml @@ -29,7 +29,7 @@ spec: exit 1 fi mkdir -p /tmp/data/ && cd /tmp/data - xbstream -x < /${BACKUP_DIR}/${BACKUP_NAME}.xbstream + xbstream -x < ${BACKUP_DIR}/${BACKUP_NAME}.xbstream xtrabackup --decompress --target-dir=/tmp/data/ xtrabackup --prepare --target-dir=/tmp/data/ find . -name "*.qp"|xargs rm -f @@ -45,5 +45,5 @@ spec: - | set -e mkdir -p ${BACKUP_DIR} - xtrabackup --compress --backup --safe-slave-backup --slave-info --stream=xbstream --host=${DB_HOST} --user=${DB_USER} --password=${DB_PASSWORD} --datadir=${DATA_DIR} > /${BACKUP_DIR}/${BACKUP_NAME}.xbstream + xtrabackup --compress --backup --safe-slave-backup --slave-info --stream=xbstream --host=${DB_HOST} --user=${DB_USER} --password=${DB_PASSWORD} --datadir=${DATA_DIR} > ${BACKUP_DIR}/${BACKUP_NAME}.xbstream incrementalBackupCommands: [] diff --git a/deploy/apecloud-mysql/templates/backuptool.yaml b/deploy/apecloud-mysql/templates/backuptool.yaml index e2e9f3402..0a7d5396c 100644 --- a/deploy/apecloud-mysql/templates/backuptool.yaml +++ b/deploy/apecloud-mysql/templates/backuptool.yaml @@ -29,7 +29,7 @@ spec: exit 1 fi mkdir -p /tmp/data/ && cd /tmp/data - xbstream -x < /${BACKUP_DIR}/${BACKUP_NAME}.xbstream + xbstream -x < ${BACKUP_DIR}/${BACKUP_NAME}.xbstream xtrabackup --decompress --target-dir=/tmp/data/ xtrabackup --prepare --target-dir=/tmp/data/ find . -name "*.qp"|xargs rm -f diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml index 062b596c3..19138c606 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml @@ -137,9 +137,6 @@ spec: CheckPoint: description: backup check point, for incremental backup. type: string - backupToolName: - description: backupToolName referenced backup tool name. - type: string checkSum: description: checksum of backup file, generated by md5 or sha1 or sha256 diff --git a/internal/constant/const.go b/internal/constant/const.go index a8200be76..c1cb636bb 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -89,7 +89,8 @@ const ( RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" - DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" + DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" // DefaultBackupPolicyAnnotationKey specifies the default backup policy. + BackupDataPathPrefixAnnotationKey = "dataprotection.kubeblocks.io/path-prefix" // BackupDataPathPrefixAnnotationKey specifies the backup data path prefix. BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. diff --git a/internal/controller/component/restore_utils.go b/internal/controller/component/restore_utils.go index 356e44576..4c2b842d6 100644 --- a/internal/controller/component/restore_utils.go +++ b/internal/controller/component/restore_utils.go @@ -126,6 +126,11 @@ func buildInitContainerWithFullBackup( AllowPrivilegeEscalation: &allowPrivilegeEscalation, RunAsUser: &runAsUser} + backupDataPath := fmt.Sprintf("/%s/%s", backup.Name, backup.Namespace) + manifests := backup.Status.Manifests + if manifests != nil && manifests.BackupTool != nil { + backupDataPath = fmt.Sprintf("/%s%s", backup.Name, manifests.BackupTool.FilePath) + } // build env for restore container.Env = []corev1.EnvVar{ { @@ -133,7 +138,7 @@ func buildInitContainerWithFullBackup( Value: backup.Name, }, { Name: "BACKUP_DIR", - Value: fmt.Sprintf("/%s/%s", backup.Name, backup.Namespace), + Value: backupDataPath, }} // merge env from backup tool. container.Env = append(container.Env, backupTool.Spec.Env...) diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index d54fad92b..98a146908 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -63,6 +63,9 @@ func (r *backupPolicyTPLTransformer) Transform(dag *graph.DAG) error { } // build the backup policy from the template. backupPolicy := r.transformBackupPolicy(v, origCluster, compDef.WorkloadType, tpl.Name) + if backupPolicy == nil { + continue + } vertex := &lifecycleVertex{obj: backupPolicy} dag.AddVertex(vertex) dag.Connect(rootVertex, vertex) @@ -108,12 +111,9 @@ func (r *backupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti backupPolicy.Labels[constant.AppInstanceLabelKey] = cluster.Name backupPolicy.Labels[constant.KBAppComponentDefRefLabelKey] = policyTPL.ComponentDefRef - // REVIEW/TODO: (wangyelei) - // 1. following is rather hack-ish, as Backup target criteria has no direct relation with workloadType, - // need extra attributes for the target selector. - // 2. need to update workloadType API attributes documentation for current design implementation. - // // only update the role labelSelector of the backup target instance when component workload is Replication/Consensus. + // because the replicas of component will change, such as 2->1. then if the target role is 'follower' and replicas is 1, + // the target instance can not be found. so we sync the label selector automatically. if !slices.Contains([]appsv1alpha1.WorkloadType{appsv1alpha1.Replication, appsv1alpha1.Consensus}, workloadType) { return } @@ -158,6 +158,10 @@ func (r *backupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba cluster *appsv1alpha1.Cluster, workloadType appsv1alpha1.WorkloadType, tplName string) *dataprotectionv1alpha1.BackupPolicy { + component := r.getFirstComponent(cluster, policyTPL.ComponentDefRef) + if component == nil { + return nil + } backupPolicy := &dataprotectionv1alpha1.BackupPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: DeriveBackupPolicyName(cluster.Name, policyTPL.ComponentDefRef), @@ -169,6 +173,7 @@ func (r *backupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba Annotations: map[string]string{ constant.DefaultBackupPolicyAnnotationKey: "true", constant.BackupPolicyTemplateAnnotationKey: tplName, + constant.BackupDataPathPrefixAnnotationKey: fmt.Sprintf("/%s-%s/%s", cluster.Name, cluster.UID, component.Name), }, }, } @@ -176,12 +181,9 @@ func (r *backupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba bpSpec.TTL = policyTPL.TTL bpSpec.Schedule.BaseBackup = r.convertBaseBackupSchedulePolicy(policyTPL.Schedule.BaseBackup) bpSpec.Schedule.Incremental = r.convertSchedulePolicy(policyTPL.Schedule.Incremental) - component := r.getFirstComponent(cluster, policyTPL.ComponentDefRef) - if component != nil { - bpSpec.Full = r.convertCommonPolicy(policyTPL.Full, cluster.Name, *component, workloadType) - bpSpec.Incremental = r.convertCommonPolicy(policyTPL.Incremental, cluster.Name, *component, workloadType) - bpSpec.Snapshot = r.convertSnapshotPolicy(policyTPL.Snapshot, cluster.Name, *component, workloadType) - } + bpSpec.Full = r.convertCommonPolicy(policyTPL.Full, cluster.Name, *component, workloadType) + bpSpec.Incremental = r.convertCommonPolicy(policyTPL.Incremental, cluster.Name, *component, workloadType) + bpSpec.Snapshot = r.convertSnapshotPolicy(policyTPL.Snapshot, cluster.Name, *component, workloadType) backupPolicy.Spec = bpSpec return backupPolicy } @@ -310,8 +312,8 @@ func (r *backupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.Common } defaultCreatePolicy := dataprotectionv1alpha1.CreatePVCPolicyIfNotPresent globalCreatePolicy := viper.GetString(constant.CfgKeyBackupPVCCreatePolicy) - if len(globalCreatePolicy) != 0 { - defaultCreatePolicy = dataprotectionv1alpha1.CreatePVCPolicy(globalCreatePolicy) + if dataprotectionv1alpha1.CreatePVCPolicy(globalCreatePolicy) == dataprotectionv1alpha1.CreatePVCPolicyNever { + defaultCreatePolicy = dataprotectionv1alpha1.CreatePVCPolicyNever } defaultInitCapacity := constant.DefaultBackupPvcInitCapacity globalInitCapacity := viper.GetString(constant.CfgKeyBackupPVCInitCapacity) From bdc3b5255bc6d8cd2136b1fe47b86a510917b7d6 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Wed, 19 Apr 2023 21:25:10 +0800 Subject: [PATCH 092/439] fix: systemaccount supports update strategy (#2710) --- apis/apps/v1alpha1/clusterdefinition_types.go | 5 + cmd/probe/internal/binding/mysql/mysql.go | 32 +- .../internal/binding/mysql/mysql_test.go | 26 ++ .../internal/binding/postgres/postgres.go | 41 ++- .../binding/postgres/postgres_test.go | 8 + cmd/probe/internal/binding/redis/redis.go | 28 ++ .../internal/binding/redis/redis_test.go | 25 ++ cmd/probe/util/const.go | 13 +- ...apps.kubeblocks.io_clusterdefinitions.yaml | 11 +- controllers/apps/systemaccount_controller.go | 289 ++++++++++-------- controllers/apps/systemaccount_util.go | 23 +- controllers/apps/systemaccount_util_test.go | 9 +- .../templates/clusterdefinition.yaml | 26 +- ...apps.kubeblocks.io_clusterdefinitions.yaml | 11 +- .../templates/clusterdefinition.yaml | 10 +- .../redis/config/redis7-config-constraint.cue | 2 +- deploy/redis/templates/clusterdefinition.yaml | 5 + .../builder/cue/statefulset_template.cue | 4 +- internal/sqlchannel/client.go | 39 ++- internal/sqlchannel/client_test.go | 147 ++++++--- internal/sqlchannel/types.go | 13 +- 21 files changed, 542 insertions(+), 225 deletions(-) diff --git a/apis/apps/v1alpha1/clusterdefinition_types.go b/apis/apps/v1alpha1/clusterdefinition_types.go index 6e9886a7c..92620f2c1 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types.go +++ b/apis/apps/v1alpha1/clusterdefinition_types.go @@ -152,7 +152,12 @@ type ProvisionStatements struct { // creation specifies statement how to create this account with required privileges. // +kubebuilder:validation:Required CreationStatement string `json:"creation"` + // update specifies statement how to update account's password. + // +kubebuilder:validation:Required + UpdateStatement string `json:"update,omitempty"` // deletion specifies statement how to delete this account. + // Used in combination with `CreateionStatement` to delete the account before create it. + // For instance, one usually uses `drop user if exists` statement followed by `create user` statement to create an account. // +optional DeletionStatement string `json:"deletion,omitempty"` } diff --git a/cmd/probe/internal/binding/mysql/mysql.go b/cmd/probe/internal/binding/mysql/mysql.go index a8679b5a1..b402e5ef3 100644 --- a/cmd/probe/internal/binding/mysql/mysql.go +++ b/cmd/probe/internal/binding/mysql/mysql.go @@ -84,10 +84,11 @@ const ( FROM mysql.user WHERE host = '%%' and user <> 'root' and user not like 'kb%%' and user ='%s';" ` - createUserTpl = "CREATE USER '%s'@'%%' IDENTIFIED BY '%s';" - deleteUserTpl = "DROP USER IF EXISTS '%s'@'%%';" - grantTpl = "GRANT %s TO '%s'@'%%';" - revokeTpl = "REVOKE %s FROM '%s'@'%%';" + createUserTpl = "CREATE USER '%s'@'%%' IDENTIFIED BY '%s';" + deleteUserTpl = "DROP USER IF EXISTS '%s'@'%%';" + grantTpl = "GRANT %s TO '%s'@'%%';" + revokeTpl = "REVOKE %s FROM '%s'@'%%';" + listSystemAccountsTpl = "SELECT user AS userName FROM mysql.user WHERE host = '%' and user like 'kb%';" ) var ( @@ -130,6 +131,7 @@ func (mysqlOps *MysqlOperations) Init(metadata bindings.Metadata) error { mysqlOps.RegisterOperation(DescribeUserOp, mysqlOps.describeUserOps) mysqlOps.RegisterOperation(GrantUserRoleOp, mysqlOps.grantUserRoleOps) mysqlOps.RegisterOperation(RevokeUserRoleOp, mysqlOps.revokeUserRoleOps) + mysqlOps.RegisterOperation(ListSystemAccountsOp, mysqlOps.listSystemAccountsOps) return nil } @@ -507,6 +509,28 @@ func (mysqlOps *MysqlOperations) listUsersOps(ctx context.Context, req *bindings return QueryObject(ctx, mysqlOps, req, ListUsersOp, sqlTplRend, nil, UserInfo{}) } +func (mysqlOps *MysqlOperations) listSystemAccountsOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { + sqlTplRend := func(user UserInfo) string { + return listSystemAccountsTpl + } + dataProcessor := func(data interface{}) (interface{}, error) { + var users []UserInfo + if err := json.Unmarshal(data.([]byte), &users); err != nil { + return nil, err + } + userNames := make([]string, 0) + for _, user := range users { + userNames = append(userNames, user.UserName) + } + if jsonData, err := json.Marshal(userNames); err != nil { + return nil, err + } else { + return string(jsonData), nil + } + } + return QueryObject(ctx, mysqlOps, req, ListSystemAccountsOp, sqlTplRend, dataProcessor, UserInfo{}) +} + func (mysqlOps *MysqlOperations) describeUserOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { var ( object = UserInfo{} diff --git a/cmd/probe/internal/binding/mysql/mysql_test.go b/cmd/probe/internal/binding/mysql/mysql_test.go index 59b786ed5..92697854f 100644 --- a/cmd/probe/internal/binding/mysql/mysql_test.go +++ b/cmd/probe/internal/binding/mysql/mysql_test.go @@ -589,6 +589,32 @@ func TestMySQLAccounts(t *testing.T) { assert.Nil(t, err) assert.Equal(t, RespEveSucc, result[RespTypEve], result[RespTypMsg]) }) + t.Run("List System Accounts", func(t *testing.T) { + var err error + var result OpsResult + + req := &bindings.InvokeRequest{} + req.Operation = CreateUserOp + req.Metadata = map[string]string{} + + col1 := sqlmock.NewColumn("userName").OfType("STRING", "turning") + + rows := sqlmock.NewRowsWithColumnDefinition(col1). + AddRow("kbadmin") + + stmt := "SELECT user AS userName FROM mysql.user WHERE host = '%' and user like 'kb%';" + mock.ExpectQuery(regexp.QuoteMeta(stmt)).WillReturnRows(rows) + + result, err = mysqlOps.listSystemAccountsOps(ctx, req, resp) + assert.Nil(t, err) + assert.Equal(t, RespEveSucc, result[RespTypEve], result[RespTypMsg]) + data := result[RespTypMsg].(string) + users := []string{} + err = json.Unmarshal([]byte(data), &users) + assert.Nil(t, err) + assert.Equal(t, 1, len(users)) + assert.Equal(t, "kbadmin", users[0]) + }) } func mockDatabase(t *testing.T) (*MysqlOperations, sqlmock.Sqlmock, error) { viper.SetDefault("KB_SERVICE_ROLES", "{\"follower\":\"Readonly\",\"leader\":\"ReadWrite\"}") diff --git a/cmd/probe/internal/binding/postgres/postgres.go b/cmd/probe/internal/binding/postgres/postgres.go index e0e37a5a8..2b3f5babd 100644 --- a/cmd/probe/internal/binding/postgres/postgres.go +++ b/cmd/probe/internal/binding/postgres/postgres.go @@ -69,10 +69,11 @@ const ( FROM pg_user WHERE usename = '%s'; ` - createUserTpl = "CREATE USER %s WITH PASSWORD '%s';" - dropUserTpl = "DROP USER IF EXISTS %s;" - grantTpl = "GRANT %s TO %s;" - revokeTpl = "REVOKE %s FROM %s;" + createUserTpl = "CREATE USER %s WITH PASSWORD '%s';" + dropUserTpl = "DROP USER IF EXISTS %s;" + grantTpl = "GRANT %s TO %s;" + revokeTpl = "REVOKE %s FROM %s;" + listSystemAccountsTpl = "SELECT rolname FROM pg_catalog.pg_roles WHERE pg_roles.rolname LIKE 'kb%'" ) var ( @@ -124,6 +125,7 @@ func (pgOps *PostgresOperations) Init(metadata bindings.Metadata) error { pgOps.RegisterOperation(DescribeUserOp, pgOps.describeUserOps) pgOps.RegisterOperation(GrantUserRoleOp, pgOps.grantUserRoleOps) pgOps.RegisterOperation(RevokeUserRoleOp, pgOps.revokeUserRoleOps) + pgOps.RegisterOperation(ListSystemAccountsOp, pgOps.listSystemAccountsOps) return nil } @@ -460,10 +462,39 @@ func (pgOps *PostgresOperations) listUsersOps(ctx context.Context, req *bindings return listUserTpl } ) - return QueryObject(ctx, pgOps, req, opsKind, sqlTplRend, pgUserRolesProcessor, UserInfo{}) } +func (pgOps *PostgresOperations) listSystemAccountsOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { + var ( + opsKind = ListUsersOp + sqlTplRend = func(user UserInfo) string { + return listSystemAccountsTpl + } + ) + dataProcessor := func(data interface{}) (interface{}, error) { + type roleInfo struct { + Rolname string `json:"rolname"` + } + var roles []roleInfo + if err := json.Unmarshal(data.([]byte), &roles); err != nil { + return nil, err + } + + roleNames := make([]string, 0) + for _, role := range roles { + roleNames = append(roleNames, role.Rolname) + } + if jsonData, err := json.Marshal(roleNames); err != nil { + return nil, err + } else { + return string(jsonData), nil + } + } + + return QueryObject(ctx, pgOps, req, opsKind, sqlTplRend, dataProcessor, UserInfo{}) +} + func (pgOps *PostgresOperations) describeUserOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { var ( object = UserInfo{} diff --git a/cmd/probe/internal/binding/postgres/postgres_test.go b/cmd/probe/internal/binding/postgres/postgres_test.go index 2f4ab7db3..11fb9767c 100644 --- a/cmd/probe/internal/binding/postgres/postgres_test.go +++ b/cmd/probe/internal/binding/postgres/postgres_test.go @@ -264,6 +264,14 @@ func TestPostgresIntegrationAccounts(t *testing.T) { res, err = b.Invoke(ctx, req) assertResponse(t, res, err, RespEveSucc) + // list system users + req = &bindings.InvokeRequest{ + Operation: ListSystemAccountsOp, + Metadata: map[string]string{}, + } + res, err = b.Invoke(ctx, req) + assertResponse(t, res, err, RespEveSucc) + // grant role req = &bindings.InvokeRequest{ Operation: GrantUserRoleOp, diff --git a/cmd/probe/internal/binding/redis/redis.go b/cmd/probe/internal/binding/redis/redis.go index cbf2b7233..4d4d590df 100644 --- a/cmd/probe/internal/binding/redis/redis.go +++ b/cmd/probe/internal/binding/redis/redis.go @@ -89,6 +89,7 @@ func (r *Redis) Init(meta bindings.Metadata) (err error) { r.RegisterOperation(DescribeUserOp, r.describeUserOps) r.RegisterOperation(GrantUserRoleOp, r.grantUserRoleOps) r.RegisterOperation(RevokeUserRoleOp, r.revokeUserRoleOps) + r.RegisterOperation(ListSystemAccountsOp, r.listSystemAccountsOps) return nil } @@ -274,6 +275,33 @@ func (r *Redis) listUsersOps(ctx context.Context, req *bindings.InvokeRequest, r return QueryObject(ctx, r, req, ListUsersOp, cmdRender, dataProcessor, UserInfo{}) } +func (r *Redis) listSystemAccountsOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { + dataProcessor := func(data interface{}) (interface{}, error) { + // data is an array of interface{} of string + results := make([]string, 0) + err := json.Unmarshal(data.([]byte), &results) + if err != nil { + return nil, err + } + sysetmUsers := make([]string, 0) + for _, user := range results { + if slices.Contains(redisPreDefinedUsers, user) { + sysetmUsers = append(sysetmUsers, user) + } + } + if jsonData, err := json.Marshal(sysetmUsers); err != nil { + return nil, err + } else { + return string(jsonData), nil + } + } + cmdRender := func(user UserInfo) string { + return "ACL USERS" + } + + return QueryObject(ctx, r, req, ListUsersOp, cmdRender, dataProcessor, UserInfo{}) +} + func (r *Redis) describeUserOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { var ( object = UserInfo{} diff --git a/cmd/probe/internal/binding/redis/redis_test.go b/cmd/probe/internal/binding/redis/redis_test.go index 4bb7c87e0..46b6bbac2 100644 --- a/cmd/probe/internal/binding/redis/redis_test.go +++ b/cmd/probe/internal/binding/redis/redis_test.go @@ -527,6 +527,31 @@ func TestRedisAccounts(t *testing.T) { assert.Equal(t, test.redisPrivs, cmd) } }) + // list accounts + t.Run("List System Accounts", func(t *testing.T) { + mock.ExpectDo("ACL", "USERS").SetVal([]string{"ape", "default", "kbadmin"}) + + response, err := r.Invoke(ctx, &bindings.InvokeRequest{ + Operation: ListSystemAccountsOp, + }) + + assert.Nil(t, err) + assert.NotNil(t, response) + assert.NotNil(t, response.Data) + // parse result + opsResult := OpsResult{} + _ = json.Unmarshal(response.Data, &opsResult) + assert.Equal(t, RespEveSucc, opsResult[RespTypEve], opsResult[RespTypMsg]) + + users := []string{} + err = json.Unmarshal([]byte(opsResult[RespTypMsg].(string)), &users) + assert.Nil(t, err) + assert.NotEmpty(t, users) + assert.Len(t, users, 2) + assert.Contains(t, users, "kbadmin") + assert.Contains(t, users, "default") + mock.ClearExpect() + }) } func mockRedisOps(t *testing.T) (*Redis, redismock.ClientMock) { diff --git a/cmd/probe/util/const.go b/cmd/probe/util/const.go index b4629dae1..97d61e55e 100644 --- a/cmd/probe/util/const.go +++ b/cmd/probe/util/const.go @@ -29,12 +29,13 @@ const ( CloseOperation bindings.OperationKind = "close" // actions for cluster accounts management - ListUsersOp bindings.OperationKind = "listUsers" - CreateUserOp bindings.OperationKind = "createUser" - DeleteUserOp bindings.OperationKind = "deleteUser" - DescribeUserOp bindings.OperationKind = "describeUser" - GrantUserRoleOp bindings.OperationKind = "grantUserRole" - RevokeUserRoleOp bindings.OperationKind = "revokeUserRole" + ListUsersOp bindings.OperationKind = "listUsers" + CreateUserOp bindings.OperationKind = "createUser" + DeleteUserOp bindings.OperationKind = "deleteUser" + DescribeUserOp bindings.OperationKind = "describeUser" + GrantUserRoleOp bindings.OperationKind = "grantUserRole" + RevokeUserRoleOp bindings.OperationKind = "revokeUserRole" + ListSystemAccountsOp bindings.OperationKind = "listSystemAccounts" // actions for cluster roles management OperationNotImplemented = "NotImplemented" diff --git a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml index a7f8ced49..d4ad71843 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml @@ -8455,7 +8455,16 @@ spec: type: string deletion: description: deletion specifies statement - how to delete this account. + how to delete this account. Used in combination + with `CreateionStatement` to delete the + account before create it. For instance, + one usually uses `drop user if exists` statement + followed by `create user` statement to create + an account. + type: string + update: + description: update specifies statement how + to update account's password. type: string required: - creation diff --git a/controllers/apps/systemaccount_controller.go b/controllers/apps/systemaccount_controller.go index 7ab7a6285..7d2e1551d 100644 --- a/controllers/apps/systemaccount_controller.go +++ b/controllers/apps/systemaccount_controller.go @@ -18,6 +18,7 @@ package apps import ( "context" + "fmt" "github.com/go-logr/logr" "github.com/spf13/viper" @@ -26,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -39,6 +41,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" + "github.com/apecloud/kubeblocks/internal/sqlchannel" ) // SystemAccountReconciler reconciles a SystemAccount object. @@ -49,28 +52,22 @@ type SystemAccountReconciler struct { SecretMapStore *secretMapStore } -// jobCompletionPredicate implements a default delete predicate function on job deletion. -type jobCompletionPredicate struct { - predicate.Funcs - reconciler *SystemAccountReconciler - Log logr.Logger -} - -// clusterDeletionPredicate implements a default delete predication function on cluster deletion. -// It is used to clean cached secrets from SystemAccountReconciler.SecretMapStore -type clusterDeletionPredicate struct { - predicate.Funcs - reconciler *SystemAccountReconciler - clusterLog logr.Logger -} - // componentUniqueKey is used internally to uniquely identify a component, by namespace-clusterName-componentName. type componentUniqueKey struct { namespace string clusterName string componentName string + characterType string } +// updateStrategy is used to specify the update strategy for a component. +type updateStrategy int8 + +const ( + inPlaceUpdate updateStrategy = 1 + reCreate updateStrategy = 2 +) + // SysAccountDeletion and SysAccountCreation are used as event reasons. const ( SysAcctDelete = "SysAcctDelete" @@ -89,12 +86,6 @@ const ( systemAccountsDebugMode string = "ENABLE_DEBUG_SYSACCOUNTS" ) -// compile-time assert that the local data object satisfies the phases data interface. -var _ predicate.Predicate = &jobCompletionPredicate{} - -// compile-time assert that the local data object satisfies the phases data interface. -var _ predicate.Predicate = &clusterDeletionPredicate{} - var ( // systemAccountLog is a logger for use during runtime systemAccountLog logr.Logger @@ -160,31 +151,39 @@ func (r *SystemAccountReconciler) Reconcile(ctx context.Context, req ctrl.Reques processAccountsForComponent := func(compDef *appsv1alpha1.ClusterComponentDefinition, compDecl *appsv1alpha1.ClusterComponentSpec, svcEP *corev1.Endpoints, headlessEP *corev1.Endpoints) error { var ( - err error - toCreate appsv1alpha1.KBAccountType - detectedFacts appsv1alpha1.KBAccountType - engine *customizedEngine - compKey = componentUniqueKey{ + err error + toCreate appsv1alpha1.KBAccountType + detectedK8SFacts appsv1alpha1.KBAccountType + detectedEngineFacts appsv1alpha1.KBAccountType + engine *customizedEngine + compKey = componentUniqueKey{ namespace: cluster.Namespace, clusterName: cluster.Name, componentName: compDecl.Name, + characterType: compDef.CharacterType, } ) // expectations: collect accounts from default setting, cluster and cluster definition. toCreate = getDefaultAccounts() - // facts: accounts have been created. - detectedFacts, err = r.getAccountFacts(reqCtx, compKey) - if err != nil { + // facts: accounts have been created, in form of k8s secrets. + if detectedK8SFacts, err = r.getAccountFacts(reqCtx, compKey); err != nil { reqCtx.Log.Error(err, "failed to get secrets") return err } // toCreate = account to create - account exists - toCreate &= toCreate ^ detectedFacts + // (toCreate \intersect detectedEngineFacts) means the set of account exists in engine but not in k8s, and should be updated or altered, not re-created. + toCreate &= toCreate ^ detectedK8SFacts if toCreate == 0 { return nil } + // facts: accounts have been created in engine. + if detectedEngineFacts, err = r.getEngineFacts(reqCtx, compKey); err != nil { + reqCtx.Log.Error(err, "failed to get accounts", "cluster", cluster.Name, "component", compDecl.Name) + // we don't return error here, because we can still create accounts in k8s and will give it a try. + } + // replace KubeBlocks ENVs. replaceEnvsValues(cluster.Name, compDef.SystemAccounts) @@ -194,13 +193,18 @@ func (r *SystemAccountReconciler) Reconcile(ctx context.Context, req ctrl.Reques continue } + strategy := reCreate + if detectedEngineFacts&accountID != 0 { + strategy = inPlaceUpdate + } + switch account.ProvisionPolicy.Type { case appsv1alpha1.CreateByStmt: if engine == nil { execConfig := compDef.SystemAccounts.CmdExecutorConfig engine = newCustomizedEngine(execConfig, cluster, compDecl.Name) } - if err := r.createByStmt(reqCtx, cluster, compDef, compKey, engine, account, svcEP, headlessEP); err != nil { + if err := r.createByStmt(reqCtx, cluster, compDef, compKey, engine, account, svcEP, headlessEP, strategy); err != nil { return err } case appsv1alpha1.ReferToExisting: @@ -251,11 +255,9 @@ func (r *SystemAccountReconciler) Reconcile(ctx context.Context, req ctrl.Reques func (r *SystemAccountReconciler) SetupWithManager(mgr ctrl.Manager) error { r.SecretMapStore = newSecretMapStore() return ctrl.NewControllerManagedBy(mgr). - For(&appsv1alpha1.Cluster{}, builder.WithPredicates(&clusterDeletionPredicate{reconciler: r, clusterLog: systemAccountLog.WithName("clusterDeletionPredicate")})). + For(&appsv1alpha1.Cluster{}, r.clusterDeletionHander()). Owns(&corev1.Secret{}). - Watches(&source.Kind{Type: &batchv1.Job{}}, - &handler.EnqueueRequestForObject{}, - builder.WithPredicates(&jobCompletionPredicate{reconciler: r, Log: log.FromContext(context.TODO())})). + Watches(&source.Kind{Type: &batchv1.Job{}}, r.jobCompletionHander()). Complete(r) } @@ -265,14 +267,14 @@ func (r *SystemAccountReconciler) createByStmt(reqCtx intctrlutil.RequestCtx, compKey componentUniqueKey, engine *customizedEngine, account appsv1alpha1.SystemAccountConfig, - svcEP *corev1.Endpoints, headlessEP *corev1.Endpoints) error { + svcEP *corev1.Endpoints, headlessEP *corev1.Endpoints, strategy updateStrategy) error { // render statements scheme, _ := appsv1alpha1.SchemeBuilder.Build() policy := account.ProvisionPolicy - stmts, secret := getCreationStmtForAccount(compKey, compDef.SystemAccounts.PasswordConfig, account) + stmts, secret := getCreationStmtForAccount(compKey, compDef.SystemAccounts.PasswordConfig, account, strategy) - uprefErr := controllerutil.SetOwnerReference(cluster, secret, scheme) + uprefErr := controllerutil.SetControllerReference(cluster, secret, scheme) if uprefErr != nil { return uprefErr } @@ -283,7 +285,7 @@ func (r *SystemAccountReconciler) createByStmt(reqCtx intctrlutil.RequestCtx, // before create job, we adjust job's attributes, such as labels, tolerations w.r.t cluster info. calibrateJobMetaAndSpec(job, cluster, compKey, account.Name) // update owner reference - if err := controllerutil.SetOwnerReference(cluster, job, scheme); err != nil { + if err := controllerutil.SetControllerReference(cluster, job, scheme); err != nil { return err } // create job @@ -308,7 +310,7 @@ func (r *SystemAccountReconciler) createByReferingToExisting(reqCtx intctrlutil. } // and make a copy of it newSecret := renderSecretByCopy(key, (string)(account.Name), secret) - if uprefErr := controllerutil.SetOwnerReference(cluster, newSecret, scheme); uprefErr != nil { + if uprefErr := controllerutil.SetControllerReference(cluster, newSecret, scheme); uprefErr != nil { return uprefErr } @@ -369,32 +371,46 @@ func (r *SystemAccountReconciler) getAccountFacts(reqCtx intctrlutil.RequestCtx, return detectedFacts, nil } -// Delete implements default DeleteEvent filter on job deletion. -// If the job for creating account completes successfully, corresponding secret will be created. -func (r *jobCompletionPredicate) Delete(e event.DeleteEvent) bool { - if e.Object == nil { - return false +func (r *SystemAccountReconciler) getEngineFacts(reqCtx intctrlutil.RequestCtx, key componentUniqueKey) (appsv1alpha1.KBAccountType, error) { + // get pods for this cluster-component, by lable + ml := getLabelsForSecretsAndJobs(key) + pods := &corev1.PodList{} + if err := r.Client.List(reqCtx.Ctx, pods, client.InNamespace(key.namespace), ml); err != nil { + return appsv1alpha1.KBAccountInvalid, err } - job, ok := e.Object.(*batchv1.Job) - if !ok { - return false + if len(pods.Items) == 0 { + return appsv1alpha1.KBAccountInvalid, fmt.Errorf("no pods available for cluster: %s, component %s", key.clusterName, key.componentName) + } + // find the first running pod + var target *corev1.Pod + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodRunning { + target = &pod + } + } + if target == nil { + return appsv1alpha1.KBAccountInvalid, fmt.Errorf("no pod is running for cluster: %s, component %s", key.clusterName, key.componentName) } - ml := job.ObjectMeta.Labels - accountName, ok := ml[constant.ClusterAccountLabelKey] - if !ok { - return false + sqlChanClient, err := sqlchannel.NewClientWithPod(target, key.characterType) + if err != nil { + return appsv1alpha1.KBAccountInvalid, err } - clusterName, ok := ml[constant.AppInstanceLabelKey] - if !ok { - return false + accounts, err := sqlChanClient.GetSystemAccounts() + if err != nil { + return appsv1alpha1.KBAccountInvalid, err } - componentName, ok := ml[constant.KBAppComponentLabelKey] - if !ok { - return false + accountsID := appsv1alpha1.KBAccountInvalid + for _, acc := range accounts { + updateFacts((appsv1alpha1.AccountName(acc)), &accountsID) } + return accountsID, nil +} - containsJobCondition := func(jobConditions []batchv1.JobCondition, +func (r *SystemAccountReconciler) jobCompletionHander() *handler.Funcs { + logger := systemAccountLog.WithName("jobCompletionHandler") + + containsJobCondition := func(job batchv1.Job, jobConditions []batchv1.JobCondition, jobCondType batchv1.JobConditionType, jobCondStatus corev1.ConditionStatus) bool { for _, jobCond := range job.Status.Conditions { if jobCond.Type == jobCondType && jobCond.Status == jobCondStatus { @@ -404,78 +420,105 @@ func (r *jobCompletionPredicate) Delete(e event.DeleteEvent) bool { return false } - // job failed, reconcile - if !containsJobCondition(job.Status.Conditions, batchv1.JobComplete, corev1.ConditionTrue) { - return true - } + return &handler.Funcs{ + DeleteFunc: func(e event.DeleteEvent, q workqueue.RateLimitingInterface) { + if e.Object == nil { + return + } - // job for cluster-component-account succeeded - // create secret for this account - compKey := componentUniqueKey{ - namespace: job.Namespace, - clusterName: clusterName, - componentName: componentName, - } - key := concatSecretName(compKey, accountName) - entry, ok, err := r.reconciler.SecretMapStore.getSecret(key) - if err != nil || !ok { - return false - } + job, ok := e.Object.(*batchv1.Job) + if !ok { + return + } - err = r.reconciler.Client.Create(context.TODO(), entry.value) - if err != nil { - r.Log.Error(err, "failed to create secret, will try later", "secret key", key) - return false - } - clusterKey := types.NamespacedName{Namespace: job.Namespace, Name: clusterName} - cluster := &appsv1alpha1.Cluster{} - if err := r.reconciler.Client.Get(context.TODO(), clusterKey, cluster); err != nil { - r.Log.Error(err, "failed to get cluster", "cluster key", clusterKey) - return false - } else { - r.reconciler.Recorder.Eventf(cluster, corev1.EventTypeNormal, SysAcctCreate, - "Created Accounts for cluster: %s, component: %s, accounts: %s", cluster.Name, componentName, accountName) - // delete secret from cache store - if err = r.reconciler.SecretMapStore.deleteSecret(key); err != nil { - r.Log.Error(err, "failed to delete secret by key", "secret key", key) - } + ml := job.ObjectMeta.Labels + accountName := ml[constant.ClusterAccountLabelKey] + clusterName := ml[constant.AppInstanceLabelKey] + componentName := ml[constant.KBAppComponentLabelKey] + if len(accountName) == 0 || len(clusterName) == 0 || len(componentName) == 0 { + return + } + + // job failed, reconcile + if !containsJobCondition(*job, job.Status.Conditions, batchv1.JobComplete, corev1.ConditionTrue) { + return + } + + // job for cluster-component-account succeeded + // create secret for this account + compKey := componentUniqueKey{ + namespace: job.Namespace, + clusterName: clusterName, + componentName: componentName, + } + + key := concatSecretName(compKey, accountName) + entry, ok, err := r.SecretMapStore.getSecret(key) + if err != nil || !ok { + return + } + + err = r.Client.Create(context.TODO(), entry.value) + if err != nil { + logger.Error(err, "failed to create secret, will try later", "secret key", key) + return + } + + clusterKey := types.NamespacedName{Namespace: job.Namespace, Name: clusterName} + cluster := &appsv1alpha1.Cluster{} + if err := r.Client.Get(context.TODO(), clusterKey, cluster); err != nil { + logger.Error(err, "failed to get cluster", "cluster key", clusterKey) + } else { + r.Recorder.Eventf(cluster, corev1.EventTypeNormal, SysAcctCreate, + "Created Accounts for cluster: %s, component: %s, accounts: %s", cluster.Name, componentName, accountName) + // delete secret from cache store + if err = r.SecretMapStore.deleteSecret(key); err != nil { + logger.Error(err, "failed to delete secret by key", "secret key", key) + } + } + }, } - return false } // Delete removes cached entries from SystemAccountReconciler.SecretMapStore -func (r *clusterDeletionPredicate) Delete(e event.DeleteEvent) bool { - if e.Object == nil { - return false - } - cluster, ok := e.Object.(*appsv1alpha1.Cluster) - if !ok { - return false - } - - // for each component from the cluster, delete cached secrets - for _, comp := range cluster.Spec.ComponentSpecs { - compKey := componentUniqueKey{ - namespace: cluster.Namespace, - clusterName: cluster.Name, - componentName: comp.Name, - } - for _, accName := range getAllSysAccounts() { - key := concatSecretName(compKey, string(accName)) - // delete left-over secrets, and ignore errors if it has been removed. - _, exists, err := r.reconciler.SecretMapStore.getSecret(key) - if err != nil { - r.clusterLog.Error(err, "failed to get secrets", "secret key", key) - continue +func (r *SystemAccountReconciler) clusterDeletionHander() builder.Predicates { + logger := systemAccountLog.WithName("clusterDeletionHandler") + predicate := predicate.Funcs{ + DeleteFunc: func(e event.DeleteEvent) bool { + if e.Object == nil { + return false } - if !exists { - continue + cluster, ok := e.Object.(*appsv1alpha1.Cluster) + if !ok { + return false } - err = r.reconciler.SecretMapStore.deleteSecret(key) - if err != nil { - r.clusterLog.Error(err, "failed to delete secrets", "secret key", key) + + // for each component from the cluster, delete cached secrets + for _, comp := range cluster.Spec.ComponentSpecs { + compKey := componentUniqueKey{ + namespace: cluster.Namespace, + clusterName: cluster.Name, + componentName: comp.Name, + } + for _, accName := range getAllSysAccounts() { + key := concatSecretName(compKey, string(accName)) + // delete left-over secrets, and ignore errors if it has been removed. + _, exists, err := r.SecretMapStore.getSecret(key) + if err != nil { + logger.Error(err, "failed to get secrets", "secret key", key) + continue + } + if !exists { + continue + } + err = r.SecretMapStore.deleteSecret(key) + if err != nil { + logger.Error(err, "failed to delete secrets", "secret key", key) + } + } } - } + return false + }, } - return false + return builder.WithPredicates(predicate) } diff --git a/controllers/apps/systemaccount_util.go b/controllers/apps/systemaccount_util.go index 666a80ac9..89a19aa81 100644 --- a/controllers/apps/systemaccount_util.go +++ b/controllers/apps/systemaccount_util.go @@ -245,8 +245,7 @@ func renderSecret(key componentUniqueKey, username string, labels client.Matchin return secret } -func retrieveEndpoints(scope appsv1alpha1.ProvisionScope, - svcEP *corev1.Endpoints, headlessEP *corev1.Endpoints) []string { +func retrieveEndpoints(scope appsv1alpha1.ProvisionScope, svcEP *corev1.Endpoints, headlessEP *corev1.Endpoints) []string { // parse endpoints endpoints := make([]string, 0) if scope == appsv1alpha1.AnyPods { @@ -303,7 +302,7 @@ func concatSecretName(key componentUniqueKey, username string) string { } func getCreationStmtForAccount(key componentUniqueKey, passConfig appsv1alpha1.PasswordConfig, - accountConfig appsv1alpha1.SystemAccountConfig) ([]string, *corev1.Secret) { + accountConfig appsv1alpha1.SystemAccountConfig, strategy updateStrategy) ([]string, *corev1.Secret) { // generated password with mixedcases = true passwd, _ := password.Generate((int)(passConfig.Length), (int)(passConfig.NumDigits), (int)(passConfig.NumSymbols), false, false) // refine password to upper or lower cases w.r.t configuration @@ -319,15 +318,21 @@ func getCreationStmtForAccount(key componentUniqueKey, passConfig appsv1alpha1.P namedVars := getEnvReplacementMapForAccount(userName, passwd) execStmts := make([]string, 0) - // drop if exists + create if not exists + statements := accountConfig.ProvisionPolicy.Statements - if len(statements.DeletionStatement) > 0 { - stmt := componetutil.ReplaceNamedVars(namedVars, statements.DeletionStatement, -1, true) + if strategy == inPlaceUpdate { + // use update statement + stmt := componetutil.ReplaceNamedVars(namedVars, statements.UpdateStatement, -1, true) + execStmts = append(execStmts, stmt) + } else { + // drop if exists + create if not exists + if len(statements.DeletionStatement) > 0 { + stmt := componetutil.ReplaceNamedVars(namedVars, statements.DeletionStatement, -1, true) + execStmts = append(execStmts, stmt) + } + stmt := componetutil.ReplaceNamedVars(namedVars, statements.CreationStatement, -1, true) execStmts = append(execStmts, stmt) } - stmt := componetutil.ReplaceNamedVars(namedVars, statements.CreationStatement, -1, true) - execStmts = append(execStmts, stmt) - secret := renderSecretWithPwd(key, userName, passwd) return execStmts, secret } diff --git a/controllers/apps/systemaccount_util_test.go b/controllers/apps/systemaccount_util_test.go index c659d3cf5..aae830404 100644 --- a/controllers/apps/systemaccount_util_test.go +++ b/controllers/apps/systemaccount_util_test.go @@ -81,6 +81,7 @@ func mockCreateByStmtSystemAccount(name appsv1alpha1.AccountName) appsv1alpha1.S Type: appsv1alpha1.CreateByStmt, Statements: &appsv1alpha1.ProvisionStatements{ CreationStatement: "CREATE USER IF NOT EXISTS $(USERNAME) IDENTIFIED BY \"$(PASSWD)\";", + UpdateStatement: "ALTER USER $(USERNAME) IDENTIFIED BY \"$(PASSWD)\";", DeletionStatement: "DROP USER IF EXISTS $(USERNAME);", }, }, @@ -195,7 +196,7 @@ func TestRenderJob(t *testing.T) { for _, acc := range accountsSetting.Accounts { switch acc.ProvisionPolicy.Type { case appsv1alpha1.CreateByStmt: - creationStmt, secrets := getCreationStmtForAccount(compKey, accountsSetting.PasswordConfig, acc) + creationStmt, secrets := getCreationStmtForAccount(compKey, accountsSetting.PasswordConfig, acc, reCreate) // make sure all variables have been replaced for _, stmt := range creationStmt { assert.False(t, strings.Contains(stmt, "$(USERNAME)")) @@ -340,13 +341,17 @@ func TestRenderCreationStmt(t *testing.T) { account.ProvisionPolicy.Statements.DeletionStatement = "" } - stmts, secret := getCreationStmtForAccount(compKey, compDef.SystemAccounts.PasswordConfig, account) + stmts, secret := getCreationStmtForAccount(compKey, compDef.SystemAccounts.PasswordConfig, account, reCreate) if toss == 1 { assert.Equal(t, 1, len(stmts)) } else { assert.Equal(t, 2, len(stmts)) } assert.NotNil(t, secret) + + stmts, secret = getCreationStmtForAccount(compKey, compDef.SystemAccounts.PasswordConfig, account, inPlaceUpdate) + assert.Equal(t, 1, len(stmts)) + assert.NotNil(t, secret) } } } diff --git a/deploy/apecloud-mysql/templates/clusterdefinition.yaml b/deploy/apecloud-mysql/templates/clusterdefinition.yaml index 98f77f5bc..f9a726b29 100644 --- a/deploy/apecloud-mysql/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql/templates/clusterdefinition.yaml @@ -203,33 +203,33 @@ spec: type: CreateByStmt scope: AnyPods statements: - creation: CREATE USER IF NOT EXISTS $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT ALL PRIVILEGES ON *.* TO $(USERNAME); - deletion: DROP USER IF EXISTS $(USERNAME); + creation: CREATE USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT ALL PRIVILEGES ON *.* TO $(USERNAME); + update: ALTER USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; - name: kbdataprotection provisionPolicy: type: CreateByStmt scope: AnyPods statements: - creation: CREATE USER IF NOT EXISTS $(USERNAME) IDENTIFIED BY '$(PASSWD)';GRANT RELOAD, LOCK TABLES, PROCESS, REPLICATION CLIENT ON *.* TO $(USERNAME); GRANT LOCK TABLES,RELOAD,PROCESS,REPLICATION CLIENT, SUPER,SELECT,EVENT,TRIGGER,SHOW VIEW ON *.* TO $(USERNAME); - deletion: DROP USER IF EXISTS $(USERNAME); + creation: CREATE USER $(USERNAME) IDENTIFIED BY '$(PASSWD)';GRANT RELOAD, LOCK TABLES, PROCESS, REPLICATION CLIENT ON *.* TO $(USERNAME); GRANT LOCK TABLES,RELOAD,PROCESS,REPLICATION CLIENT, SUPER,SELECT,EVENT,TRIGGER,SHOW VIEW ON *.* TO $(USERNAME); + update: ALTER USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; - name: kbprobe provisionPolicy: type: CreateByStmt scope: AnyPods - statements: - creation: CREATE USER IF NOT EXISTS $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT REPLICATION CLIENT, PROCESS ON *.* TO $(USERNAME); GRANT SELECT ON performance_schema.* TO $(USERNAME); - deletion: DROP USER IF EXISTS $(USERNAME); + statements: + creation: CREATE USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT REPLICATION CLIENT, PROCESS ON *.* TO $(USERNAME); GRANT SELECT ON performance_schema.* TO $(USERNAME); + update: ALTER USER (USERNAME) IDENTIFIED BY '$(PASSWD)'; - name: kbmonitoring provisionPolicy: type: CreateByStmt scope: AnyPods - statements: - creation: CREATE USER IF NOT EXISTS $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT REPLICATION CLIENT, PROCESS ON *.* TO $(USERNAME); GRANT SELECT ON performance_schema.* TO $(USERNAME); - deletion: DROP USER IF EXISTS $(USERNAME); + statements: + creation: CREATE USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT REPLICATION CLIENT, PROCESS ON *.* TO $(USERNAME); GRANT SELECT ON performance_schema.* TO $(USERNAME); + update: ALTER USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; - name: kbreplicator provisionPolicy: type: CreateByStmt scope: AnyPods - statements: - creation: CREATE USER IF NOT EXISTS $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT REPLICATION SLAVE ON *.* TO $(USERNAME) WITH GRANT OPTION; - deletion: DROP USER IF EXISTS $(USERNAME); \ No newline at end of file + statements: + creation: CREATE USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT REPLICATION SLAVE ON *.* TO $(USERNAME) WITH GRANT OPTION; + update: ALTER USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; \ No newline at end of file diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml index a7f8ced49..d4ad71843 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml @@ -8455,7 +8455,16 @@ spec: type: string deletion: description: deletion specifies statement - how to delete this account. + how to delete this account. Used in combination + with `CreateionStatement` to delete the + account before create it. For instance, + one usually uses `drop user if exists` statement + followed by `create user` statement to create + an account. + type: string + update: + description: update specifies statement how + to update account's password. type: string required: - creation diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 6848e679f..2ff1a7a14 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -276,32 +276,32 @@ spec: scope: AnyPods statements: creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)'; - deletion: DROP USER IF EXISTS $(USERNAME); + update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; - name: kbdataprotection provisionPolicy: type: CreateByStmt scope: AnyPods statements: creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)'; - deletion: DROP USER IF EXISTS $(USERNAME); + update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; - name: kbprobe provisionPolicy: type: CreateByStmt scope: AnyPods statements: creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; GRANT pg_monitor TO $(USERNAME); - deletion: DROP USER IF EXISTS $(USERNAME); + update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; - name: kbmonitoring provisionPolicy: type: CreateByStmt scope: AnyPods statements: creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; GRANT pg_monitor TO $(USERNAME); - deletion: DROP USER IF EXISTS $(USERNAME); + update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; - name: kbreplicator provisionPolicy: type: CreateByStmt scope: AnyPods statements: creation: CREATE USER $(USERNAME) WITH REPLICATION PASSWORD '$(PASSWD)'; - deletion: DROP USER IF EXISTS $(USERNAME); + update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; diff --git a/deploy/redis/config/redis7-config-constraint.cue b/deploy/redis/config/redis7-config-constraint.cue index d36547ebe..cef8d638f 100644 --- a/deploy/redis/config/redis7-config-constraint.cue +++ b/deploy/redis/config/redis7-config-constraint.cue @@ -150,4 +150,4 @@ } configuration: #RedisParameter & { -} \ No newline at end of file +} diff --git a/deploy/redis/templates/clusterdefinition.yaml b/deploy/redis/templates/clusterdefinition.yaml index 4b5772384..4f89f11c9 100644 --- a/deploy/redis/templates/clusterdefinition.yaml +++ b/deploy/redis/templates/clusterdefinition.yaml @@ -154,30 +154,35 @@ spec: scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbdataprotection provisionPolicy: type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbmonitoring provisionPolicy: type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbprobe provisionPolicy: type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbreplicator provisionPolicy: type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) +psync +replconf +ping + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: redis-sentinel workloadType: Stateful characterType: redis diff --git a/internal/controller/builder/cue/statefulset_template.cue b/internal/controller/builder/cue/statefulset_template.cue index d3c054a6f..d3a7ce4e4 100644 --- a/internal/controller/builder/cue/statefulset_template.cue +++ b/internal/controller/builder/cue/statefulset_template.cue @@ -52,8 +52,8 @@ statefulset: { "app.kubernetes.io/managed-by": "kubeblocks" "apps.kubeblocks.io/component-name": "\(component.name)" } - serviceName: "\(cluster.metadata.name)-\(component.name)-headless" - replicas: component.replicas + serviceName: "\(cluster.metadata.name)-\(component.name)-headless" + replicas: component.replicas minReadySeconds: 10 podManagementPolicy: "Parallel" template: { diff --git a/internal/sqlchannel/client.go b/internal/sqlchannel/client.go index d7cf8d4e0..1340c6191 100644 --- a/internal/sqlchannel/client.go +++ b/internal/sqlchannel/client.go @@ -46,12 +46,6 @@ type OperationResult struct { respTime time.Time } -type Order struct { - OrderID int `json:"orderid"` - Customer string `json:"customer"` - Price float64 `json:"price"` -} - func NewClientWithPod(pod *corev1.Pod, characterType string) (*OperationClient, error) { if characterType == "" { return nil, fmt.Errorf("pod %v chacterType must be set", pod.Name) @@ -61,6 +55,7 @@ func NewClientWithPod(pod *corev1.Pod, characterType string) (*OperationClient, if ip == "" { return nil, fmt.Errorf("pod %v has no ip", pod.Name) } + port, err := intctrlutil.GetProbeGRPCPort(pod) if err != nil { return nil, err @@ -93,6 +88,7 @@ func (cli *OperationClient) GetRole() (string, error) { Data: []byte(""), Metadata: map[string]string{}, } + resp, err := cli.InvokeComponentInRoutine(ctxWithReconcileTimeout, req) if err != nil { return "", err @@ -106,6 +102,37 @@ func (cli *OperationClient) GetRole() (string, error) { return result["role"], nil } +// GetSystemAccounts list all system accounts created +func (cli *OperationClient) GetSystemAccounts() ([]string, error) { + ctxWithReconcileTimeout, cancel := context.WithTimeout(context.Background(), cli.ReconcileTimeout) + defer cancel() + + // Request sql channel via Dapr SDK + req := &dapr.InvokeBindingRequest{ + Name: cli.CharacterType, + Operation: string(ListSystemAccountsOp), + } + + if resp, err := cli.InvokeComponentInRoutine(ctxWithReconcileTimeout, req); err != nil { + return nil, err + } else { + sqlResponse := SQLChannelResponse{} + if err = json.Unmarshal(resp.Data, &sqlResponse); err != nil { + return nil, err + } + if sqlResponse.Event == RespEveFail { + return nil, fmt.Errorf("get system accounts error: %s", sqlResponse.Message) + } else { + result := []string{} + if err = json.Unmarshal(([]byte)(sqlResponse.Message), &result); err != nil { + return nil, err + } else { + return result, err + } + } + } +} + func (cli *OperationClient) InvokeComponentInRoutine(ctxWithReconcileTimeout context.Context, req *dapr.InvokeBindingRequest) (*dapr.BindingEvent, error) { ch := make(chan *OperationResult, 1) go cli.InvokeComponent(ctxWithReconcileTimeout, req, ch) diff --git a/internal/sqlchannel/client_test.go b/internal/sqlchannel/client_test.go index ad0505664..9032379fa 100644 --- a/internal/sqlchannel/client_test.go +++ b/internal/sqlchannel/client_test.go @@ -35,8 +35,39 @@ import ( testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) +type testDaprServer struct { + pb.UnimplementedDaprServer + state map[string][]byte + configurationSubscriptionID map[string]chan struct{} + cachedRequest map[string]*pb.InvokeBindingResponse +} + +var _ pb.DaprServer = &testDaprServer{} + +func (s *testDaprServer) InvokeBinding(ctx context.Context, req *pb.InvokeBindingRequest) (*pb.InvokeBindingResponse, error) { + time.Sleep(100 * time.Millisecond) + darpRequest := dapr.InvokeBindingRequest{Name: req.Name, Operation: req.Operation, Data: req.Data, Metadata: req.Metadata} + resp, ok := s.cachedRequest[GetMapKeyFromRequest(&darpRequest)] + if ok { + return resp, nil + } else { + return nil, fmt.Errorf("unexpected request") + } +} + +func (s *testDaprServer) ExepctRequest(req *pb.InvokeBindingRequest, resp *pb.InvokeBindingResponse) { + darpRequest := dapr.InvokeBindingRequest{Name: req.Name, Operation: req.Operation, Data: req.Data, Metadata: req.Metadata} + s.cachedRequest[GetMapKeyFromRequest(&darpRequest)] = resp +} + func TestNewClientWithPod(t *testing.T) { - port, closer := newTCPServer(t, 50001) + daprServer := &testDaprServer{ + state: make(map[string][]byte), + configurationSubscriptionID: map[string]chan struct{}{}, + cachedRequest: make(map[string]*pb.InvokeBindingResponse), + } + + port, closer := newTCPServer(t, daprServer, 50001) defer closer() podName := "pod-for-sqlchannel-test" pod := testapps.NewPodFactory("default", podName). @@ -76,7 +107,7 @@ func TestNewClientWithPod(t *testing.T) { podWithoutGRPCPort.Spec.Containers[0].Ports = podWithoutGRPCPort.Spec.Containers[0].Ports[:1] _, err := NewClientWithPod(podWithoutGRPCPort, "mysql") if err == nil { - t.Errorf("new sql channel client unexpection") + t.Errorf("new sql channel client union") } }) @@ -109,27 +140,18 @@ func TestGPRC(t *testing.T) { } func TestGetRole(t *testing.T) { - port, closer := newTCPServer(t, 50001) - defer closer() - podName := "pod-for-sqlchannel-test" - pod := testapps.NewPodFactory("default", podName). - AddContainer(corev1.Container{Name: testapps.DefaultNginxContainerName, Image: testapps.NginxImage}).GetObject() - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{{ - ContainerPort: int32(3501), - Name: intctrlutil.ProbeHTTPPortName, - Protocol: "TCP", - }, - { - ContainerPort: int32(port), - Name: intctrlutil.ProbeGRPCPortName, - Protocol: "TCP", - }, - } - pod.Status.PodIP = "127.0.0.1" - cli, err := NewClientWithPod(pod, "mysql") + daprServer, cli, closer, err := initSQLChannelClient(t) if err != nil { t.Errorf("new sql channel client error: %v", err) } + defer closer() + + daprServer.ExepctRequest(&pb.InvokeBindingRequest{ + Name: "mysql", + Operation: "getRole", + }, &pb.InvokeBindingResponse{ + Data: []byte("{\"role\": \"leader\"}"), + }) t.Run("ResponseInTime", func(t *testing.T) { cli.ReconcileTimeout = 1 * time.Second @@ -169,7 +191,42 @@ func TestGetRole(t *testing.T) { }) } -func newTCPServer(t *testing.T, port int) (int, func()) { +func TestSystemAccounts(t *testing.T) { + daprServer, cli, closer, err := initSQLChannelClient(t) + if err != nil { + t.Errorf("new sql channel client error: %v", err) + } + defer closer() + + roleNames, _ := json.Marshal([]string{"kbadmin", "kbprobe"}) + sqlResponse := SQLChannelResponse{ + Event: RespEveSucc, + Message: string(roleNames), + } + respData, _ := json.Marshal(sqlResponse) + resp := &pb.InvokeBindingResponse{ + Data: respData, + } + + daprServer.ExepctRequest(&pb.InvokeBindingRequest{ + Name: "mysql", + Operation: string(ListSystemAccountsOp), + }, resp) + + t.Run("ResponseByCache", func(t *testing.T) { + cli.ReconcileTimeout = 200 * time.Millisecond + _, err := cli.GetSystemAccounts() + + if err != nil { + t.Errorf("return reps in cache: %v", err) + } + if len(cli.cache) != 0 { + t.Errorf("cache should be cleared: %v", cli.cache) + } + }) +} + +func newTCPServer(t *testing.T, daprServer pb.DaprServer, port int) (int, func()) { var l net.Listener for i := 0; i < 3; i++ { l, _ = net.Listen("tcp", fmt.Sprintf(":%v", port)) @@ -182,10 +239,7 @@ func newTCPServer(t *testing.T, port int) (int, func()) { t.Errorf("couldn't start listening") } s := grpc.NewServer() - pb.RegisterDaprServer(s, &testDaprServer{ - state: make(map[string][]byte), - configurationSubscriptionID: map[string]chan struct{}{}, - }) + pb.RegisterDaprServer(s, daprServer) go func() { if err := s.Serve(l); err != nil && err.Error() != "closed" { @@ -200,22 +254,33 @@ func newTCPServer(t *testing.T, port int) (int, func()) { return port, closer } -type testDaprServer struct { - pb.UnimplementedDaprServer - state map[string][]byte - configurationSubscriptionID map[string]chan struct{} -} +func initSQLChannelClient(t *testing.T) (*testDaprServer, *OperationClient, func(), error) { + daprServer := &testDaprServer{ + state: make(map[string][]byte), + configurationSubscriptionID: map[string]chan struct{}{}, + cachedRequest: make(map[string]*pb.InvokeBindingResponse), + } -func (s *testDaprServer) InvokeBinding(ctx context.Context, req *pb.InvokeBindingRequest) (*pb.InvokeBindingResponse, error) { - time.Sleep(100 * time.Millisecond) - if req.Data == nil { - return &pb.InvokeBindingResponse{ - Data: []byte("{\"role\": \"leader\"}"), - Metadata: map[string]string{"k1": "v1", "k2": "v2"}, - }, nil + port, closer := newTCPServer(t, daprServer, 50001) + podName := "pod-for-sqlchannel-test" + pod := testapps.NewPodFactory("default", podName). + AddContainer(corev1.Container{Name: testapps.DefaultNginxContainerName, Image: testapps.NginxImage}).GetObject() + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ + { + ContainerPort: int32(3501), + Name: intctrlutil.ProbeHTTPPortName, + Protocol: "TCP", + }, + { + ContainerPort: int32(port), + Name: intctrlutil.ProbeGRPCPortName, + Protocol: "TCP", + }, + } + pod.Status.PodIP = "127.0.0.1" + cli, err := NewClientWithPod(pod, "mysql") + if err != nil { + t.Errorf("new sql channel client error: %v", err) } - return &pb.InvokeBindingResponse{ - Data: req.Data, - Metadata: req.Metadata, - }, nil + return daprServer, cli, closer, err } diff --git a/internal/sqlchannel/types.go b/internal/sqlchannel/types.go index ee000c4aa..f6dfd09cd 100644 --- a/internal/sqlchannel/types.go +++ b/internal/sqlchannel/types.go @@ -35,12 +35,13 @@ const ( InvalidRole string = "invalid" // actions for cluster accounts management - ListUsersOp bindings.OperationKind = "listUsers" - CreateUserOp bindings.OperationKind = "createUser" - DeleteUserOp bindings.OperationKind = "deleteUser" - DescribeUserOp bindings.OperationKind = "describeUser" - GrantUserRoleOp bindings.OperationKind = "grantUserRole" - RevokeUserRoleOp bindings.OperationKind = "revokeUserRole" + ListUsersOp bindings.OperationKind = "listUsers" + CreateUserOp bindings.OperationKind = "createUser" + DeleteUserOp bindings.OperationKind = "deleteUser" + DescribeUserOp bindings.OperationKind = "describeUser" + GrantUserRoleOp bindings.OperationKind = "grantUserRole" + RevokeUserRoleOp bindings.OperationKind = "revokeUserRole" + ListSystemAccountsOp bindings.OperationKind = "listSystemAccounts" HTTPRequestPrefx string = "curl -X POST -H 'Content-Type: application/json' http://localhost:%d/v1.0/bindings/%s" ) From 9f3a450edee27ddbcd83ce689399cff983df1419 Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Wed, 19 Apr 2023 21:35:39 +0800 Subject: [PATCH 093/439] fix: add preflight for ack(aliyun) and tke(tencent) (#2708) --- .../kubeblocks/data/ack_hostpreflight.yaml | 20 +++++++++ .../cmd/kubeblocks/data/ack_preflight.yaml | 44 +++++++++++++++++++ .../kubeblocks/data/tke_hostpreflight.yaml | 20 +++++++++ .../cmd/kubeblocks/data/tke_preflight.yaml | 44 +++++++++++++++++++ internal/cli/cmd/kubeblocks/preflight.go | 18 ++++++++ 5 files changed, 146 insertions(+) create mode 100644 internal/cli/cmd/kubeblocks/data/ack_hostpreflight.yaml create mode 100644 internal/cli/cmd/kubeblocks/data/ack_preflight.yaml create mode 100644 internal/cli/cmd/kubeblocks/data/tke_hostpreflight.yaml create mode 100644 internal/cli/cmd/kubeblocks/data/tke_preflight.yaml diff --git a/internal/cli/cmd/kubeblocks/data/ack_hostpreflight.yaml b/internal/cli/cmd/kubeblocks/data/ack_hostpreflight.yaml new file mode 100644 index 000000000..a41ffdc63 --- /dev/null +++ b/internal/cli/cmd/kubeblocks/data/ack_hostpreflight.yaml @@ -0,0 +1,20 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: HostPreflight +metadata: + name: host-utility +spec: + collectors: + analyzers: + extendCollectors: + - hostUtility : + collectorName: aliyun-cli + utilityName: aliyun + extendAnalyzers: + - hostUtility: + checkName: aliyunCli-Check + collectorName: aliyun-cli + outcomes: + - pass: + message: aliyun-cli has been installed + - warn: + message: aliyun-cli isn't installed \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml b/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml new file mode 100644 index 000000000..5a233595e --- /dev/null +++ b/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml @@ -0,0 +1,44 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: Preflight +metadata: + name: kubeblocks_preflight +spec: + collectors: + - clusterInfo: {} + analyzers: + - clusterVersion: + checkName: GKE-Version + outcomes: + - fail: + when: "< 1.22.0" + message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0. + uri: https://www.kubernetes.io + - pass: + when: ">= 1.22.0" + message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes. + uri: https://www.kubernetes.io + - nodeResources: + checkName: At-Least-3-Nodes + outcomes: + - warn: + when: "count() < 3" + message: This application requires at least 3 nodes + - pass: + message: This cluster has enough nodes. + extendAnalyzers: + - clusterAccess: + checkName: Check-K8S-Access + outcomes: + - fail: + message: k8s cluster access fail + - pass: + message: k8s cluster access ok + - storageClass: + checkName: Required-Cloud-SSD-SC + storageClassType: "cloud_ssd" + provisioner: "diskplugin.csi.alibabacloud.com" + outcomes: + - fail: + message: The cloud_ssd storage class was not found + - pass: + message: cloud_ssd is the presence, and all good on storage classes \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/tke_hostpreflight.yaml b/internal/cli/cmd/kubeblocks/data/tke_hostpreflight.yaml new file mode 100644 index 000000000..acd2238f1 --- /dev/null +++ b/internal/cli/cmd/kubeblocks/data/tke_hostpreflight.yaml @@ -0,0 +1,20 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: HostPreflight +metadata: + name: host-utility +spec: + collectors: + analyzers: + extendCollectors: + - hostUtility : + collectorName: txcloud-cli + utilityName: tccli + extendAnalyzers: + - hostUtility: + checkName: txcloudCli-Check + collectorName: txcloud-cli + outcomes: + - pass: + message: txcloud-cli has been installed + - warn: + message: txcloud-cli isn't installed \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml b/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml new file mode 100644 index 000000000..95519e753 --- /dev/null +++ b/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml @@ -0,0 +1,44 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: Preflight +metadata: + name: kubeblocks_preflight +spec: + collectors: + - clusterInfo: {} + analyzers: + - clusterVersion: + checkName: GKE-Version + outcomes: + - fail: + when: "< 1.22.0" + message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0. + uri: https://www.kubernetes.io + - pass: + when: ">= 1.22.0" + message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes. + uri: https://www.kubernetes.io + - nodeResources: + checkName: At-Least-3-Nodes + outcomes: + - warn: + when: "count() < 3" + message: This application requires at least 3 nodes + - pass: + message: This cluster has enough nodes. + extendAnalyzers: + - clusterAccess: + checkName: Check-K8S-Access + outcomes: + - fail: + message: k8s cluster access fail + - pass: + message: k8s cluster access ok + - storageClass: + checkName: Required-CBS-SC + storageClassType: "cbs" + provisioner: "com.tencent.cloud.csi.cbs" + outcomes: + - fail: + message: The cbs storage class was not found + - pass: + message: cbs is the presence, and all good on storage classes \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/preflight.go b/internal/cli/cmd/kubeblocks/preflight.go index 0a25f0914..10b9d958f 100644 --- a/internal/cli/cmd/kubeblocks/preflight.go +++ b/internal/cli/cmd/kubeblocks/preflight.go @@ -78,6 +78,10 @@ const ( EKSPreflight = "data/eks_preflight.yaml" GKEHostPreflight = "data/gke_hostpreflight.yaml" GKEPreflight = "data/gke_preflight.yaml" + ACKHostPreflight = "data/ack_hostpreflight.yaml" + ACKPreflight = "data/ack_preflight.yaml" + TKEHostPreflight = "data/tke_hostpreflight.yaml" + TKEPreflight = "data/tke_preflight.yaml" ) // PreflightOptions declares the arguments accepted by the preflight command @@ -140,6 +144,20 @@ func LoadVendorCheckYaml(vendorName util.K8sProvider) ([][]byte, error) { if data, err := defaultVendorYamlData.ReadFile(GKEPreflight); err == nil { yamlDataList = append(yamlDataList, data) } + case util.ACKProvider: + if data, err := defaultVendorYamlData.ReadFile(ACKHostPreflight); err == nil { + yamlDataList = append(yamlDataList, data) + } + if data, err := defaultVendorYamlData.ReadFile(ACKPreflight); err == nil { + yamlDataList = append(yamlDataList, data) + } + case util.TKEProvider: + if data, err := defaultVendorYamlData.ReadFile(TKEHostPreflight); err == nil { + yamlDataList = append(yamlDataList, data) + } + if data, err := defaultVendorYamlData.ReadFile(TKEPreflight); err == nil { + yamlDataList = append(yamlDataList, data) + } case util.UnknownProvider: fallthrough default: From a7387ffff87eeac518e9752cfc03bf0a186420f9 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Wed, 19 Apr 2023 22:02:56 +0800 Subject: [PATCH 094/439] chore: add .github/utils/bugs_not_in_current_milestone.sh for settings bugs with no milestone to current milestone (#2727) --- .../utils/bugs_not_in_current_milestone.sh | 26 ++ .github/utils/feature_triage.sh | 48 +-- .github/utils/functions.bash | 87 ++++++ .github/utils/generate_release_notes.py | 281 +++++++++--------- .github/utils/get_release_version.py | 61 ++-- .github/utils/gh_env | 12 + .github/utils/requirements.txt | 1 + 7 files changed, 320 insertions(+), 196 deletions(-) create mode 100755 .github/utils/bugs_not_in_current_milestone.sh create mode 100644 .github/utils/functions.bash create mode 100644 .github/utils/gh_env create mode 100644 .github/utils/requirements.txt diff --git a/.github/utils/bugs_not_in_current_milestone.sh b/.github/utils/bugs_not_in_current_milestone.sh new file mode 100755 index 000000000..9675b4ccf --- /dev/null +++ b/.github/utils/bugs_not_in_current_milestone.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. + +. ./gh_env +. ./functions.bash + +gh_get_issues "none" "kind/bug" + +rows=$(echo ${last_issue_list}| jq -r '. | sort_by(.state,.number)| .[].number') + + +printf "%s | %s | %s \n" "Issue Title" "Assignees" "Issue State" +echo "---|---|---" +for row in $rows +do + issue_id=$(echo $row | awk -F "," '{print $1}') + gh_get_issue_body ${issue_id} + printf "[%s](%s) #%s | %s | %s\n" "${last_issue_title}" "${last_issue_url}" "${issue_id}" "${last_issue_assignees_printable}" "${last_issue_state}" + + gh_update_issue_milestone ${issue_id} +done \ No newline at end of file diff --git a/.github/utils/feature_triage.sh b/.github/utils/feature_triage.sh index 41044eebd..1eaea81d0 100755 --- a/.github/utils/feature_triage.sh +++ b/.github/utils/feature_triage.sh @@ -4,44 +4,26 @@ set -o errexit set -o nounset set -o pipefail -REMOTE_URL=$(git config --get remote.origin.url) -OWNER=$(dirname ${REMOTE_URL} | awk -F ":" '{print $2}') -REPO=$(basename -s .git ${REMOTE_URL}) -MILESTONE_ID=${MILESTONE_ID:-5} +# requires `git`, `gh`, and `jq` commands, ref. https://cli.github.com/manual/installation for installation guides. -# GH list issues API ref: https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues -ISSUE_LIST=$(gh api \ - --header 'Accept: application/vnd.github+json' \ - --method GET \ - /repos/${OWNER}/${REPO}/issues \ - -F per_page=100 \ - -f milestone=${MILESTONE_ID} \ - -f labels=kind/feature \ - -f state=all) +. ./gh_env +. ./functions.bash -ROWS=$(echo ${ISSUE_LIST}| jq -r '. | sort_by(.state,.number)| .[].number') +gh_get_issues ${MILESTONE_ID} "kind/feature" "all" +rows=$(echo ${last_issue_list}| jq -r '. | sort_by(.state,.number)| .[].number') + +echo $rows printf "%s | %s | %s | %s | %s | %s\n" "Feature Title" "Assignees" "Issue State" "Code PR Merge Status" "Feature Doc. Status" "Extra Notes" echo "---|---|---|---|---|---" -for ROW in $ROWS +for row in $rows do - ISSUE_ID=$(echo $ROW | awk -F "," '{print $1}') - # GH get issue API ref: https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#get-an-issue - ISSUE_BODY=$(gh api \ - --header 'Accept: application/vnd.github+json' \ - --method GET \ - /repos/${OWNER}/${REPO}/issues/${ISSUE_ID}) - URL=$(echo $ISSUE_BODY| jq -r '.url') - TITLE=$(echo $ISSUE_BODY| jq -r '.title') - ASSIGNEES=$(echo $ISSUE_BODY| jq -r '.assignees[]?.login') - ASSIGNEES_PRINTABLE= - for ASSIGNEE in $ASSIGNEES - do - ASSIGNEES_PRINTABLE="${ASSIGNEES_PRINTABLE},${ASSIGNEE}" - done - ASSIGNEES_PRINTABLE=${ASSIGNEES_PRINTABLE#,} - STATE=$(echo $ISSUE_BODY| jq -r '.state') - PR=$(echo $ISSUE_BODY| jq -r '.pull_request?.url') - printf "[%s](%s) #%s | %s | %s | | | \n" "$TITLE" $URL $ISSUE_ID "$ASSIGNEES_PRINTABLE" "$STATE" + issue_id=$(echo $row | awk -F "," '{print $1}') + gh_get_issue_body ${issue_id} + pr_url=$(echo $last_issue_body| jq -r '.pull_request?.url') + if [ "$pr_url" == "null" ]; then + pr_url="N/A" + fi + printf "[%s](%s) #%s | %s | %s | %s| | \n" "${last_issue_title}" "${last_issue_url}" "${issue_id}" "${last_issue_assignees_printable}" "${last_issue_state}" "${pr_url}" done \ No newline at end of file diff --git a/.github/utils/functions.bash b/.github/utils/functions.bash new file mode 100644 index 000000000..55acad50d --- /dev/null +++ b/.github/utils/functions.bash @@ -0,0 +1,87 @@ +# bash functions + +# requires `gh` command, ref. https://cli.github.com/manual/installation for installation guides. + +gh_get_issues () { + # @arg milestone - Milestone ID, if the string none is passed, issues without milestones are returned. + # @arg state - Can be one of: open, closed, all; Default: open. + # @arg page - Cardinal value; Default: 1 + # @result $last_issue_list - contains JSON result + declare milestone="$1" labels="$2" state="${3:-open}" page="${4:-1}" + + # GH list issues API ref: https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues + local cmd="gh api \ + --method GET \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + /repos/${OWNER}/${REPO}/issues \ + -F per_page=100 \ + -F page=${page} \ + -f milestone=${milestone} \ + -f labels=${labels} \ + -f state=${state}" + echo $cmd + last_issue_list=`eval ${cmd}` +} + + +gh_get_issue_body() { + # @arg issue_id - Github issue ID + # @result last_issue_body + # @result last_issue_url + # @result last_issue_title + # @result last_issue_state + # @result last_issue_assignees + # @result last_issue_assignees_printable + declare issue_id="$1" + + local issue_body=$(gh api \ + --method GET \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + /repos/${OWNER}/${REPO}/issues/${issue_id}) + local url=$(echo ${issue_body} | jq -r '.url') + local title=$(echo ${issue_body} | jq -r '.title') + local assignees=$(echo ${issue_body} | jq -r '.assignees[]?.login') + local assignees_printable= + for assignee in ${assignees} + do + assignees_printable="${assignees_printable},${assignee}" + done + local assignees_printable=${assignees_printable#,} + local state=$(echo ${issue_body}| jq -r '.state') + + last_issue_body="${issue_body}" + last_issue_url="${url}" + last_issue_title="${title}" + last_issue_state="${state}" + last_issue_assignees=${assignees} + last_issue_assignees_printable=${assignees_printable} +} + +gh_update_issue_milestone() { + # @arg issue_id - Github issue ID + # @arg milestone - Milestone ID, if the string none is passed, issues without milestones are returned. + # @result last_issue_resp + declare issue_id="$1" milestone_id="${2:-}" + + if [ -z "$milestone_id" ]; then + milestone_id=${MILESTONE_ID} + fi + + local req_data="{\"milestone\":$milestone_id}" + echo "req_data=$req_data" + + local gh_token=$(gh auth token) + + local resp=$(curl \ + --location \ + --request PATCH \ + --header 'Accept: application/vnd.github+json' \ + --header 'X-GitHub-Api-Version: 2022-11-28' \ + --header "Authorization: Bearer ${gh_token}" \ + --data "${req_data}" \ + https://api.github.com/repos/${OWNER}/${REPO}/issues/${issue_id}) + + last_issue_resp=${resp} +} \ No newline at end of file diff --git a/.github/utils/generate_release_notes.py b/.github/utils/generate_release_notes.py index e01c63c9e..f5c726892 100755 --- a/.github/utils/generate_release_notes.py +++ b/.github/utils/generate_release_notes.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding:utf-8 -*- # generate release note for milestone @@ -15,139 +15,146 @@ from github import Github -releaseIssueRegex = "^v(.*) Release Planning$" -majorReleaseRegex = "^([0-9]+\.[0-9]+)\.[0-9]+.*$" -milestoneRegex = "https://github.com/apecloud/kubeblocks/milestone/([0-9]+)" - -githubToken = os.getenv("GITHUB_TOKEN") - -changeTypes = [ - "New Features", - "Bug Fixes", - "Miscellaneous" -] - - -def get_change_priority(name): - if name in changeTypes: - return changeTypes.index(name) - return len(changeTypes) - - -changes = [] -warnings = [] -changeLines = [] -breakingChangeLines = [] - -gh = Github(githubToken) - -# get milestone issue -issues = [i for i in gh.get_repo("apecloud/kubeblocks").get_issues(state='open') if - re.search(releaseIssueRegex, i.title)] -issues = sorted(issues, key=lambda i: i.id) - -if len(issues) == 0: - print("FATAL: failed to find issue for release.") - sys.exit(0) - -if len(issues) > 1: - print("WARNING: found more than one issue for release, so first issue created will be picked: {}". - format([i.title for i in issues])) - -issue = issues[0] -print("Found issue: {}".format(issue.title)) - -# get release version from issue name -releaseVersion = re.search(releaseIssueRegex, issue.title).group(1) -print("Generating release notes for KubeBlocks {}".format(releaseVersion)) - -# Set REL_VERSION -if os.getenv("GITHUB_ENV"): - with open(os.getenv("GITHUB_ENV"), "a") as githubEnv: - githubEnv.write("REL_VERSION={}\n".format(releaseVersion)) - githubEnv.write("REL_BRANCH=release-{}\n".format(re.search(majorReleaseRegex, releaseVersion).group(1))) - -releaseNotePath = "docs/release_notes/v{}.md".format(releaseVersion) - -# get milestone -repoMilestones = re.findall(milestoneRegex, issue.body) -if len(repoMilestones) == 0: - print("FATAL: failed to find milestone in release issue body") - sys.exit(0) -if len(repoMilestones) > 1: - print("WARNING: found more than one milestone in release issue body, first milestone will be picked: {}". - format([i for i in repoMilestones])) - -# find all issues and PRs in milestone -repo = gh.get_repo(f"apecloud/kubeblocks") -milestone = repo.get_milestone(int(repoMilestones[0])) -issueOrPRs = [i for i in repo.get_issues(milestone, state="closed")] -print("Detected {} issues or pull requests".format(len(issueOrPRs))) - -# find all contributors and build changes -allContributors = set() -for issueOrPR in issueOrPRs: - url = issueOrPR.html_url - try: - # only a PR can be converted to a PR object, otherwise will throw error. - pr = issueOrPR.as_pull_request() - except: - continue - if not pr.merged: - continue - contributor = "@" + str(pr.user.login) - # Auto generate a release note - note = pr.title.strip() - changeType = "Miscellaneous" - title = note.split(":") - if len(title) > 1: - prefix = title[0].strip().lower() - if prefix in ("feat", "feature"): - changeType = "New Features" - elif prefix in ("fix", "bug"): - changeType = "Bug Fixes" - note = title[1].strip() - changes.append((changeType, pr, note, contributor, url)) - allContributors.add(contributor) - -lastSubtitle = "" -# generate changes for release notes -for change in sorted(changes, key=lambda c: (get_change_priority(c[0]), c[1].id)): - subtitle = change[0] - if lastSubtitle != subtitle: - lastSubtitle = subtitle - changeLines.append("\n### " + subtitle) - breakingChange = 'breaking-change' in [label.name for label in change[1].labels] - changeUrl = " ([#" + str(change[1].number) + "](" + change[4] + ")" - changeAuthor = ", " + change[3] + ")" - changeLines.append("- " + change[2] + changeUrl + changeAuthor) - if breakingChange: - breakingChangeLines.append("- " + change[2] + changeUrl + changeAuthor) - -if len(breakingChangeLines) > 0: - warnings.append("> **Note: This release contains a few [breaking changes](#breaking-changes).**") - -# generate release note from template -template = '' -releaseNoteTemplatePath = "docs/release_notes/template.md" -with open(releaseNoteTemplatePath, "r") as file: - template = file.read() - -changeText = "\n".join(changeLines) -breakingChangeText = "None." -if len(breakingChangeLines) > 0: - breakingChangeText = '\n'.join(breakingChangeLines) -warningsText = '' -if len(warnings) > 0: - warningsText = '\n'.join(warnings) - -with open(releaseNotePath, 'w') as file: - file.write(Template(template).safe_substitute( - kubeblocks_version=releaseVersion, - kubeblocks_changes=changeText, - kubeblocks_breaking_changes=breakingChangeText, - warnings=warningsText, - kubeblocks_contributors=', '.join(sorted(list(allContributors), key=str.casefold)), - today=date.today().strftime("%Y-%m-%d"))) - -print("Done") +RELEASE_ISSUE_RANGE = "^v(.*) Release Planning$" +MAJOR_RELEASE_REGEX = "^([0-9]+\.[0-9]+)\.[0-9]+.*$" +MILESTONE_REGEX = "https://github.com/apecloud/kubeblocks/milestone/([0-9]+)" +CHANGE_TYPES : list[str] = ["New Features", "Bug Fixes", "Miscellaneous"] + + +def get_change_priority(name: str) -> int: + if name in CHANGE_TYPES: + return CHANGE_TYPES.index(name) + return len(CHANGE_TYPES) + + +def main(argv: list[str]) -> None: + changes = [] + warnings = [] + change_lines = [] + breaking_change_lines = [] + gh_env = os.getenv("GITHUB_ENV") + gh = Github(os.getenv("GITHUB_TOKEN")) + + # get milestone issue + issues = [ + i + for i in gh.get_repo("apecloud/kubeblocks").get_issues(state="open") + if re.search(RELEASE_ISSUE_RANGE, i.title) + ] + issues = sorted(issues, key=lambda i: i.id) + + if len(issues) == 0: + print("FATAL: failed to find issue for release.") + sys.exit(0) + + if len(issues) > 1: + print(f"WARNING: found more than one issue for release, so first issue created will be picked: {[i.title for i in issues]}") + + issue = issues[0] + print(f"Found issue: {issue.title}") + + # get release version from issue name + release_version = re.search(RELEASE_ISSUE_RANGE, issue.title).group(1) + print(f"Generating release notes for KubeBlocks {release_version}") + + # Set REL_VERSION + if gh_env: + with open(gh_env, "a") as f: + f.write(f"REL_VERSION={release_version}\n") + f.write(f"REL_BRANCH=release-{re.search(MAJOR_RELEASE_REGEX, release_version).group(1)}\n") + + release_note_path = f"docs/release_notes/v{release_version}.md" + + # get milestone + repo_milestones = re.findall(MILESTONE_REGEX, issue.body) + if len(repo_milestones) == 0: + print("FATAL: failed to find milestone in release issue body") + sys.exit(0) + if len(repo_milestones) > 1: + print(f"WARNING: found more than one milestone in release issue body, first milestone will be picked: {[i for i in repo_milestones]}") + + # find all issues and PRs in milestone + repo = gh.get_repo(f"apecloud/kubeblocks") + milestone = repo.get_milestone(int(repo_milestones[0])) + issue_or_prs = [i for i in repo.get_issues(milestone, state="closed")] + print(f"Detected {len(issue_or_prs)} issues or pull requests") + + # find all contributors and build changes + allContributors = set() + for issue_or_pr in issue_or_prs: + url = issue_or_pr.html_url + try: + # only a PR can be converted to a PR object, otherwise will throw error. + pr = issue_or_pr.as_pull_request() + except: + continue + if not pr.merged: + continue + contributor = "@" + str(pr.user.login) + # Auto generate a release note + note = pr.title.strip() + change_type = "Miscellaneous" + title = note.split(":") + if len(title) > 1: + prefix = title[0].strip().lower() + if prefix in ("feat", "feature"): + change_type = "New Features" + elif prefix in ("fix", "bug"): + change_type = "Bug Fixes" + note = title[1].strip() + changes.append((change_type, pr, note, contributor, url)) + allContributors.add(contributor) + + last_subtitle = "" + # generate changes for release notes + for change in sorted(changes, key=lambda c: (get_change_priority(c[0]), c[1].id)): + subtitle = change[0] + if last_subtitle != subtitle: + last_subtitle = subtitle + change_lines.append("\n### " + subtitle) + breaking_change = "breaking-change" in [label.name for label in change[1].labels] + change_url = " ([#" + str(change[1].number) + "](" + change[4] + ")" + change_author = ", " + change[3] + ")" + change_lines.append("- " + change[2] + change_url + change_author) + if breaking_change: + breaking_change_lines.append("- " + change[2] + change_url + change_author) + + if len(breaking_change_lines) > 0: + warnings.append( + "> **Note: This release contains a few [breaking changes](#breaking-changes).**" + ) + + # generate release note from template + template = "" + release_note_template_path = "docs/release_notes/template.md" + with open(release_note_template_path, "r") as file: + template = file.read() + + change_text = "\n".join(change_lines) + breaking_change_text = "None." + if len(breaking_change_lines) > 0: + breaking_change_text = "\n".join(breaking_change_lines) + + warnings_text = "" + if len(warnings) > 0: + warnings_text = "\n".join(warnings) + + with open(release_note_path, "w") as file: + file.write( + Template(template).safe_substitute( + kubeblocks_version=release_version, + kubeblocks_changes=change_text, + kubeblocks_breaking_changes=breaking_change_text, + warnings=warnings_text, + kubeblocks_contributors=", ".join( + sorted(list(allContributors), key=str.casefold) + ), + today=date.today().strftime("%Y-%m-%d"), + ) + ) + + print("Done") + + +if __name__ == "__main__": + main(sys.argv) diff --git a/.github/utils/get_release_version.py b/.github/utils/get_release_version.py index e1bcf4177..622259c9b 100755 --- a/.github/utils/get_release_version.py +++ b/.github/utils/get_release_version.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding:utf-8 -*- # Get release version from git tag and set the parsed version to @@ -7,31 +7,40 @@ import os import sys +from typing import TypeAlias -gitRef = os.getenv("GITHUB_REF") -tagRefPrefix = "refs/tags/v" - -with open(os.getenv("GITHUB_ENV"), "a") as githubEnv: - if gitRef is None or not gitRef.startswith(tagRefPrefix): - print("This is not a release tag") - sys.exit(1) - - releaseVersion = gitRef[len(tagRefPrefix):] - releaseNotePath = "docs/release_notes/v{}/v{}.md".format(releaseVersion,releaseVersion) - - if gitRef.find("-alpha.") > 0: - print("Alpha release build from {} ...".format(gitRef)) - elif gitRef.find("-beta.") > 0: - print("Beta release build from {} ...".format(gitRef)) - elif gitRef.find("-rc.") > 0: - print("Release Candidate build from {} ...".format(gitRef)) - else: - print("Checking if {} exists".format(releaseNotePath)) - if os.path.exists(releaseNotePath): - print("Found {}".format(releaseNotePath)) - githubEnv.write("WITH_RELEASE_NOTES=true\n") +OptStr : TypeAlias = str | None + +def main(argv: list[str]) -> None: + git_ref = os.getenv("GITHUB_REF") + tag_ref_prefix = "refs/tags/v" + github_env : str = str(os.getenv("GITHUB_ENV")) + + with open(github_env, "a") as github_env_f: + if git_ref is None or not git_ref.startswith(tag_ref_prefix): + print("This is not a release tag") + sys.exit(1) + + release_version = git_ref[len(tag_ref_prefix) :] + release_note_path = f"docs/release_notes/v{release_version}/v{release_version}.md" + + if git_ref.find("-alpha.") > 0: + print(f"Alpha release build from {git_ref} ...") + elif git_ref.find("-beta.") > 0: + print(f"Beta release build from {git_ref} ...") + elif git_ref.find("-rc.") > 0: + print(f"Release Candidate build from {git_ref} ...") else: - print("{} is not found".format(releaseNotePath)) - print("Release build from {} ...".format(gitRef)) + print(f"Checking if {release_note_path} exists") + if os.path.exists(release_note_path): + print(f"Found {release_note_path}") + github_env_f.write("WITH_RELEASE_NOTES=true\n") + else: + print("{} is not found".format(release_note_path)) + print(f"Release build from {git_ref} ...") + + github_env_f.write(f"REL_VERSION={release_version}\n") + - githubEnv.write("REL_VERSION={}\n".format(releaseVersion)) +if __name__ == "__main__": + main(sys.argv) diff --git a/.github/utils/gh_env b/.github/utils/gh_env new file mode 100644 index 000000000..46eeded15 --- /dev/null +++ b/.github/utils/gh_env @@ -0,0 +1,12 @@ +export REMOTE_URL=$(git config --get remote.origin.url) +export OWNER=$(dirname ${REMOTE_URL} | awk -F ":" '{print $2}') +prefix='^//' +if [[ $OWNER =~ $prefix ]]; then +export OWNER="${OWNER#*//github.com/}" +fi +export REPO=$(basename -s .git ${REMOTE_URL}) +export MILESTONE_ID=${MILESTONE_ID:-5} + +echo "OWNER=${OWNER}" +echo "REPO=${REPO}" +echo "MILESTONE_ID=${MILESTONE_ID}" \ No newline at end of file diff --git a/.github/utils/requirements.txt b/.github/utils/requirements.txt new file mode 100644 index 000000000..b393fb08a --- /dev/null +++ b/.github/utils/requirements.txt @@ -0,0 +1 @@ +PyGithub \ No newline at end of file From 26801b49f903b330a4ef8f3f9145f3ec1df78284 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Wed, 19 Apr 2023 22:23:59 +0800 Subject: [PATCH 095/439] fix: add restart parameters which must be always passed to pg as command line options (#2732) --- .../postgresql/scripts/restart-parameter.yaml | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/deploy/postgresql/scripts/restart-parameter.yaml b/deploy/postgresql/scripts/restart-parameter.yaml index eabd4c5ae..f73fb4e94 100644 --- a/deploy/postgresql/scripts/restart-parameter.yaml +++ b/deploy/postgresql/scripts/restart-parameter.yaml @@ -3,18 +3,31 @@ - autovacuum_freeze_max_age - autovacuum_max_workers - autovacuum_multixact_freeze_max_age +- bg_mon.history_buckets +- bonjour +- bonjour_name +- cluster_name - config_file - cron.database_name +- cron.enable_superuser_jobs +- cron.host - cron.log_run - cron.log_statement - cron.max_running_jobs +- cron.timezone - cron.use_background_workers - data_directory +- data_sync_retry +- dynamic_shared_memory_type +- event_source +- external_pid_file - hba_file +- hot_standby - huge_pages - huge_page_size - ident_file - ignore_invalid_pages +- jit_provider - listen_addresses - logging_collector - max_connections @@ -36,10 +49,21 @@ - port - postgis.gdal_enabled_drivers - recovery_init_sync_method +- recovery_target +- recovery_target_action +- recovery_target_inclusive +- recovery_target_lsn +- recovery_target_name +- recovery_target_time +- recovery_target_timeline +- recovery_target_xid - session_preload_libraries - shared_buffers +- shared_memory_type - shared_preload_libraries - superuser_reserved_connections +- timescaledb.bgw_launcher_poll_time +- timescaledb.max_background_workers - track_activity_query_size - track_commit_timestamp - unix_socket_directories @@ -47,4 +71,8 @@ - unix_socket_permissions - wal_buffers - wal_compression -- wal_decode_buffer_size \ No newline at end of file +- wal_decode_buffer_size +- wal_level +- wal_log_hints +- wal_keep_segments +- wal_keep_size \ No newline at end of file From a07321518482cff0bf4c6e0144f9fa6fcd1f1fc4 Mon Sep 17 00:00:00 2001 From: dingben Date: Wed, 19 Apr 2023 22:56:16 +0800 Subject: [PATCH 096/439] fix: cli cluster subcommand help info is wrong (#2706) --- internal/cli/cmd/cli.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/cli/cmd/cli.go b/internal/cli/cmd/cli.go index e5387b7cf..3ab6fc94d 100644 --- a/internal/cli/cmd/cli.go +++ b/internal/cli/cmd/cli.go @@ -91,7 +91,6 @@ A Command Line Interface for KubeBlocks`, cmd.AddCommand( playground.NewPlaygroundCmd(ioStreams), kubeblocks.NewKubeBlocksCmd(f, ioStreams), - cluster.NewClusterCmd(f, ioStreams), bench.NewBenchCmd(), options.NewCmdOptions(ioStreams.Out), version.NewVersionCmd(f), @@ -107,6 +106,15 @@ A Command Line Interface for KubeBlocks`, filters := []string{"options"} templates.ActsAsRootCommand(cmd, filters, []templates.CommandGroup{}...) + helpFunc := cmd.HelpFunc() + usageFunc := cmd.UsageFunc() + + // clusterCmd set its own usage and help function and its subcommand will inherit it, + // so we need to set its subcommand's usage and help function back to the root command + clusterCmd := cluster.NewClusterCmd(f, ioStreams) + registerUsageAndHelpFuncForSubCommand(clusterCmd, helpFunc, usageFunc) + cmd.AddCommand(clusterCmd) + utilcomp.SetFactoryForCompletion(f) registerCompletionFuncForGlobalFlags(cmd, f) @@ -159,3 +167,10 @@ func registerCompletionFuncForGlobalFlags(cmd *cobra.Command, f cmdutil.Factory) return utilcomp.ListUsersInConfig(toComplete), cobra.ShellCompDirectiveNoFileComp })) } + +func registerUsageAndHelpFuncForSubCommand(cmd *cobra.Command, helpFunc func(*cobra.Command, []string), usageFunc func(command *cobra.Command) error) { + for _, subCmd := range cmd.Commands() { + subCmd.SetHelpFunc(helpFunc) + subCmd.SetUsageFunc(usageFunc) + } +} From b2467b802b317ddfb10f097ae5540d13e4b88519 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Wed, 19 Apr 2023 23:05:31 +0800 Subject: [PATCH 097/439] fix: vscale example typo (#2723) --- docs/user_docs/cli/kbcli_cluster_vscale.md | 2 +- internal/cli/cmd/cluster/operations.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user_docs/cli/kbcli_cluster_vscale.md b/docs/user_docs/cli/kbcli_cluster_vscale.md index 964192be7..9a16ec88a 100644 --- a/docs/user_docs/cli/kbcli_cluster_vscale.md +++ b/docs/user_docs/cli/kbcli_cluster_vscale.md @@ -15,7 +15,7 @@ kbcli cluster vscale [flags] kbcli cluster vscale --components= --cpu=500m --memory=500Mi # scale the computing resources of specified components by class, available classes can be get by executing the command "kbcli class list --cluster-definition " - kbcli cluster vscale --components= --set class=general-1c4g + kbcli cluster vscale --components= --class= ``` ### Options diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 782900937..cf78b842f 100644 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -349,7 +349,7 @@ var verticalScalingExample = templates.Examples(` kbcli cluster vscale --components= --cpu=500m --memory=500Mi # scale the computing resources of specified components by class, available classes can be get by executing the command "kbcli class list --cluster-definition " - kbcli cluster vscale --components= --set class=general-1c4g + kbcli cluster vscale --components= --class= `) // NewVerticalScalingCmd creates a vertical scaling command From e511776faecb92a668e9b9ddfd6e3a976c4e60a3 Mon Sep 17 00:00:00 2001 From: dingben Date: Wed, 19 Apr 2023 23:16:36 +0800 Subject: [PATCH 098/439] feat: cli support to auto complete more than one args (#2716) --- internal/cli/util/completion.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/cli/util/completion.go b/internal/cli/util/completion.go index ceccb7e03..69e0df96f 100644 --- a/internal/cli/util/completion.go +++ b/internal/cli/util/completion.go @@ -24,5 +24,19 @@ import ( ) func ResourceNameCompletionFunc(f cmdutil.Factory, gvr schema.GroupVersionResource) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { - return utilcomp.ResourceNameCompletionFunc(f, GVRToString(gvr)) + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + comps := utilcomp.CompGetResource(f, cmd, GVRToString(gvr), toComplete) + seen := make(map[string]bool) + + var availableComps []string + for _, arg := range args { + seen[arg] = true + } + for _, comp := range comps { + if !seen[comp] { + availableComps = append(availableComps, comp) + } + } + return availableComps, cobra.ShellCompDirectiveNoFileComp + } } From 470fdc99c185dfba6a9b8722f850d0e43eefe4a3 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Thu, 20 Apr 2023 09:33:50 +0800 Subject: [PATCH 099/439] fix: validating cpu step (#2675) --- .../componentresourceconstraint_types.go | 3 + .../componentresourceconstraint_types_test.go | 54 ++++++++++++-- apis/apps/v1alpha1/opsrequest_webhook.go | 2 +- controllers/apps/cluster_controller_test.go | 6 +- docs/user_docs/cli/kbcli_class_create.md | 4 +- internal/class/class_utils.go | 24 +++++-- internal/class/class_utils_test.go | 51 +++++++++++++ internal/class/suite_test.go | 14 ++++ internal/cli/cmd/class/create.go | 4 +- internal/cli/cmd/class/create_test.go | 71 ++++++++++++------- internal/cli/cmd/class/list.go | 30 +++++++- internal/cli/cmd/class/list_test.go | 45 +++++++++++- internal/cli/testing/testdata/class.yaml | 13 +--- .../builder/cue/pitr_job_template.cue | 14 ++++ 14 files changed, 278 insertions(+), 57 deletions(-) create mode 100644 internal/class/suite_test.go diff --git a/apis/apps/v1alpha1/componentresourceconstraint_types.go b/apis/apps/v1alpha1/componentresourceconstraint_types.go index 2c80e071d..d7fd4e487 100644 --- a/apis/apps/v1alpha1/componentresourceconstraint_types.go +++ b/apis/apps/v1alpha1/componentresourceconstraint_types.go @@ -123,6 +123,9 @@ func (m ResourceConstraint) ValidateCPU(cpu resource.Quantity) bool { if m.CPU.Max != nil && m.CPU.Max.Cmp(cpu) < 0 { return false } + if m.CPU.Step != nil && inf.NewDec(1, 0).QuoExact(cpu.AsDec(), m.CPU.Step.AsDec()).Scale() != 0 { + return false + } if m.CPU.Slots != nil && slices.Index(m.CPU.Slots, cpu) < 0 { return false } diff --git a/apis/apps/v1alpha1/componentresourceconstraint_types_test.go b/apis/apps/v1alpha1/componentresourceconstraint_types_test.go index d70a66c25..cf1e34633 100644 --- a/apis/apps/v1alpha1/componentresourceconstraint_types_test.go +++ b/apis/apps/v1alpha1/componentresourceconstraint_types_test.go @@ -64,15 +64,59 @@ func init() { func TestResourceConstraints(t *testing.T) { cases := []struct { + desc string cpu string memory string expect bool }{ - {cpu: "0.5", memory: "2Gi", expect: true}, - {cpu: "0.2", memory: "40Mi", expect: true}, - {cpu: "1", memory: "6Gi", expect: true}, - {cpu: "2", memory: "20Gi", expect: false}, - {cpu: "2", memory: "6Gi", expect: false}, + { + desc: "test memory constraint with sizePerCPU", + cpu: "0.5", + memory: "2Gi", + expect: true, + }, + { + desc: "test memory constraint with unit Mi", + cpu: "0.2", + memory: "40Mi", + expect: true, + }, + { + desc: "test memory constraint with minPerCPU and maxPerCPU", + cpu: "1", + memory: "6Gi", + expect: true, + }, + { + desc: "test cpu with decimal", + cpu: "0.3", + memory: "1.2Gi", + expect: true, + }, + { + desc: "test CPU with invalid step", + cpu: "100.6", + memory: "402.4Gi", + expect: false, + }, + { + desc: "test CPU with invalid step", + cpu: "1.05", + memory: "4.2Gi", + expect: false, + }, + { + desc: "test invalid memory", + cpu: "2", + memory: "20Gi", + expect: false, + }, + { + desc: "test invalid memory", + cpu: "2", + memory: "6Gi", + expect: false, + }, } for _, item := range cases { diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index 05d7862ee..ca9c7c6fa 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -264,7 +264,7 @@ func (r *OpsRequest) validateVerticalScaling(ctx context.Context, k8sClient clie } } if err = validateMatchingClass(classes, v.ResourceRequirements); err != nil { - return fmt.Errorf("can not find matching class for component %s", v.ComponentName) + return errors.Wrapf(err, "can not find matching class for component %s", v.ComponentName) } } } diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 2e957b4f6..664b88019 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1270,7 +1270,7 @@ var _ = Describe("Cluster Controller", func() { }) }) - When("when creating cluster with workloadType=stateful component", func() { + When("creating cluster with workloadType=stateful component", func() { BeforeEach(func() { By("Create a clusterDefinition obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). @@ -1335,7 +1335,7 @@ var _ = Describe("Cluster Controller", func() { }) }) - When("when creating cluster with workloadType=consensus component", func() { + When("creating cluster with workloadType=consensus component", func() { BeforeEach(func() { By("Create a clusterDef obj") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). @@ -1455,7 +1455,7 @@ var _ = Describe("Cluster Controller", func() { }) }) - When("when creating cluster with workloadType=replication component", func() { + When("creating cluster with workloadType=replication component", func() { BeforeEach(func() { By("Create a clusterDefinition obj with replication componentDefRef.") clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). diff --git a/docs/user_docs/cli/kbcli_class_create.md b/docs/user_docs/cli/kbcli_class_create.md index 7c493e43d..53e924d66 100644 --- a/docs/user_docs/cli/kbcli_class_create.md +++ b/docs/user_docs/cli/kbcli_class_create.md @@ -11,8 +11,8 @@ kbcli class create [NAME] [flags] ### Examples ``` - # Create a class following constraint kb-resource-constraint-general for component mysql in cluster definition apecloud-mysql, which have 1 cpu core, 2Gi memory and storage is 10Gi - kbcli class create custom-1c2g --cluster-definition apecloud-mysql --type mysql --constraint kb-resource-constraint-general --cpu 1 --memory 2Gi --storage name=data,size=10Gi + # Create a class following constraint kb-resource-constraint-general for component mysql in cluster definition apecloud-mysql, which have 1 cpu core, 1Gi memory and storage is 10Gi + kbcli class create custom-1c1g --cluster-definition apecloud-mysql --type mysql --constraint kb-resource-constraint-general --cpu 1 --memory 1Gi --storage name=data,size=10Gi # Create classes for component mysql in cluster definition apecloud-mysql, where classes is defined in file kbcli class create --cluster-definition apecloud-mysql --type mysql --file ./classes.yaml diff --git a/internal/class/class_utils.go b/internal/class/class_utils.go index f43f3ed47..7ca02cee3 100644 --- a/internal/class/class_utils.go +++ b/internal/class/class_utils.go @@ -63,17 +63,29 @@ func ChooseComponentClasses(classes map[string]*v1alpha1.ComponentClassInstance, func GetClasses(classDefinitionList v1alpha1.ComponentClassDefinitionList) (map[string]map[string]*v1alpha1.ComponentClassInstance, error) { var ( + compTypeLabel = "apps.kubeblocks.io/component-def-ref" componentClasses = make(map[string]map[string]*v1alpha1.ComponentClassInstance) ) for _, classDefinition := range classDefinitionList.Items { - componentType := classDefinition.GetLabels()["apps.kubeblocks.io/component-def-ref"] + componentType := classDefinition.GetLabels()[compTypeLabel] if componentType == "" { - return nil, fmt.Errorf("failed to find component type") + return nil, fmt.Errorf("can not find component type label %s", compTypeLabel) } - classes := make(map[string]*v1alpha1.ComponentClassInstance) - for idx := range classDefinition.Status.Classes { - cls := classDefinition.Status.Classes[idx] - classes[cls.Name] = &cls + var ( + err error + classes = make(map[string]*v1alpha1.ComponentClassInstance) + ) + if classDefinition.GetGeneration() != 0 && + classDefinition.Status.ObservedGeneration == classDefinition.GetGeneration() { + for idx := range classDefinition.Status.Classes { + cls := classDefinition.Status.Classes[idx] + classes[cls.Name] = &cls + } + } else { + classes, err = ParseComponentClasses(classDefinition) + if err != nil { + return nil, err + } } if _, ok := componentClasses[componentType]; !ok { componentClasses[componentType] = classes diff --git a/internal/class/class_utils_test.go b/internal/class/class_utils_test.go index a0631e8fb..9771d1cd4 100644 --- a/internal/class/class_utils_test.go +++ b/internal/class/class_utils_test.go @@ -27,6 +27,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) var _ = Describe("utils", func() { @@ -104,4 +105,54 @@ var _ = Describe("utils", func() { Expect(class).Should(BeNil()) }) }) + + Context("get classes", func() { + It("should succeed", func() { + var ( + err error + specClassName = "general-1c1g" + statusClassName = "general-100c100g" + compClasses map[string]map[string]*v1alpha1.ComponentClassInstance + compType = "mysql" + ) + + classDef := testapps.NewComponentClassDefinitionFactory("custom", "apecloud-mysql", compType). + AddClassGroup(testapps.DefaultGeneralResourceConstraintName). + AddClasses([]v1alpha1.ComponentClass{{ + Name: specClassName, + CPU: resource.MustParse("1"), + Memory: resource.MustParse("1Gi"), + }}). + GetObject() + + By("class definition status is out of date") + classDef.SetGeneration(1) + classDef.Status.ObservedGeneration = 0 + classDef.Status.Classes = []v1alpha1.ComponentClassInstance{ + { + ComponentClass: v1alpha1.ComponentClass{ + Name: statusClassName, + CPU: resource.MustParse("100"), + Memory: resource.MustParse("100Gi"), + }, + ResourceConstraintRef: "", + }, + } + compClasses, err = GetClasses(v1alpha1.ComponentClassDefinitionList{ + Items: []v1alpha1.ComponentClassDefinition{*classDef}, + }) + Expect(err).ShouldNot(HaveOccurred()) + Expect(compClasses[compType][specClassName]).ShouldNot(BeNil()) + Expect(compClasses[compType][statusClassName]).Should(BeNil()) + + By("class definition status is in sync with the class definition spec") + classDef.Status.ObservedGeneration = 1 + compClasses, err = GetClasses(v1alpha1.ComponentClassDefinitionList{ + Items: []v1alpha1.ComponentClassDefinition{*classDef}, + }) + Expect(err).ShouldNot(HaveOccurred()) + Expect(compClasses[compType][specClassName]).Should(BeNil()) + Expect(compClasses[compType][statusClassName]).ShouldNot(BeNil()) + }) + }) }) diff --git a/internal/class/suite_test.go b/internal/class/suite_test.go new file mode 100644 index 000000000..68df942fd --- /dev/null +++ b/internal/class/suite_test.go @@ -0,0 +1,14 @@ +package class + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestClass(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Class Test Suite") +} diff --git a/internal/cli/cmd/class/create.go b/internal/cli/cmd/class/create.go index c0ad548fc..37d0f5faa 100644 --- a/internal/cli/cmd/class/create.go +++ b/internal/cli/cmd/class/create.go @@ -58,8 +58,8 @@ type CreateOptions struct { } var classCreateExamples = templates.Examples(` - # Create a class following constraint kb-resource-constraint-general for component mysql in cluster definition apecloud-mysql, which have 1 cpu core, 2Gi memory and storage is 10Gi - kbcli class create custom-1c2g --cluster-definition apecloud-mysql --type mysql --constraint kb-resource-constraint-general --cpu 1 --memory 2Gi --storage name=data,size=10Gi + # Create a class following constraint kb-resource-constraint-general for component mysql in cluster definition apecloud-mysql, which have 1 cpu core, 1Gi memory and storage is 10Gi + kbcli class create custom-1c1g --cluster-definition apecloud-mysql --type mysql --constraint kb-resource-constraint-general --cpu 1 --memory 1Gi --storage name=data,size=10Gi # Create classes for component mysql in cluster definition apecloud-mysql, where classes is defined in file kbcli class create --cluster-definition apecloud-mysql --type mysql --file ./classes.yaml diff --git a/internal/cli/cmd/class/create_test.go b/internal/cli/cmd/class/create_test.go index 8fbfb4063..cbeb56760 100644 --- a/internal/cli/cmd/class/create_test.go +++ b/internal/cli/cmd/class/create_test.go @@ -18,6 +18,7 @@ package class import ( "bytes" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -32,16 +33,18 @@ import ( var _ = Describe("create", func() { var ( - o *CreateOptions - out *bytes.Buffer - tf *cmdtesting.TestFactory - streams genericclioptions.IOStreams + createOptions *CreateOptions + out *bytes.Buffer + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams ) fillResources := func(o *CreateOptions, cpu string, memory string, storage []string) { o.CPU = cpu o.Memory = memory o.Storage = storage + o.ClassName = fmt.Sprintf("custom-%s-%s", cpu, memory) + o.Constraint = generalResourceConstraint.Name } BeforeEach(func() { @@ -50,13 +53,13 @@ var _ = Describe("create", func() { _ = appsv1alpha1.AddToScheme(scheme.Scheme) tf.FakeDynamicClient = testing.FakeDynamicClient(&classDef, &generalResourceConstraint, &memoryOptimizedResourceConstraint) - o = &CreateOptions{ + createOptions = &CreateOptions{ Factory: tf, IOStreams: streams, ClusterDefRef: "apecloud-mysql", - ComponentType: testing.ComponentDefName, + ComponentType: "mysql", } - Expect(o.complete(tf)).ShouldNot(HaveOccurred()) + Expect(createOptions.complete(tf)).ShouldNot(HaveOccurred()) }) AfterEach(func() { @@ -71,34 +74,54 @@ var _ = Describe("create", func() { Context("with resource arguments", func() { It("should fail if required arguments is missing", func() { - o.Constraint = generalResourceConstraint.Name - fillResources(o, "", "48Gi", nil) - Expect(o.validate([]string{"general-12c48g"})).Should(HaveOccurred()) - fillResources(o, "12", "", nil) - Expect(o.validate([]string{"general-12c48g"})).Should(HaveOccurred()) - fillResources(o, "12", "48g", nil) - Expect(o.validate([]string{})).Should(HaveOccurred()) + fillResources(createOptions, "", "48Gi", nil) + Expect(createOptions.validate([]string{"general-12c48g"})).Should(HaveOccurred()) + fillResources(createOptions, "12", "", nil) + Expect(createOptions.validate([]string{"general-12c48g"})).Should(HaveOccurred()) + fillResources(createOptions, "12", "48g", nil) + Expect(createOptions.validate([]string{})).Should(HaveOccurred()) }) It("should succeed with required arguments", func() { - o.Constraint = generalResourceConstraint.Name - fillResources(o, "2", "8Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) - Expect(o.validate([]string{"general-2c8g"})).ShouldNot(HaveOccurred()) - Expect(o.run()).ShouldNot(HaveOccurred()) - Expect(out.String()).Should(ContainSubstring(o.ClassName)) + fillResources(createOptions, "96", "384Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) + Expect(createOptions.validate([]string{"general-96c384g"})).ShouldNot(HaveOccurred()) + Expect(createOptions.run()).ShouldNot(HaveOccurred()) + Expect(out.String()).Should(ContainSubstring(createOptions.ClassName)) + }) + + It("should fail if constraint not exist", func() { + fillResources(createOptions, "2", "8Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) + createOptions.Constraint = "constraint-not-exist" + Expect(createOptions.run()).Should(HaveOccurred()) + }) + + It("should fail if not conform to constraint", func() { + By("memory not conform to constraint") + fillResources(createOptions, "2", "9Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) + Expect(createOptions.run()).Should(HaveOccurred()) + + By("cpu with invalid step") + fillResources(createOptions, "0.6", "0.6Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) + Expect(createOptions.run()).Should(HaveOccurred()) }) It("should fail if class name is conflicted", func() { - o.ClassName = "general-1c1g" - fillResources(o, "1", "1Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) - Expect(o.run()).Should(HaveOccurred()) + fillResources(createOptions, "1", "1Gi", []string{"name=data,size=10Gi", "name=log,size=1Gi"}) + createOptions.ClassName = "general-1c1g" + Expect(createOptions.run()).Should(HaveOccurred()) + + fillResources(createOptions, "0.5", "0.5Gi", []string{}) + Expect(createOptions.run()).ShouldNot(HaveOccurred()) + + fillResources(createOptions, "0.5", "0.5Gi", []string{}) + Expect(createOptions.run()).Should(HaveOccurred()) }) }) Context("with class definitions file", func() { It("should succeed", func() { - o.File = testCustomClassDefsPath - Expect(o.run()).ShouldNot(HaveOccurred()) + createOptions.File = testCustomClassDefsPath + Expect(createOptions.run()).ShouldNot(HaveOccurred()) Expect(out.String()).Should(ContainSubstring("custom-1c1g")) Expect(out.String()).Should(ContainSubstring("custom-4c16g")) // memory optimized classes diff --git a/internal/cli/cmd/class/list.go b/internal/cli/cmd/class/list.go index eee764c6c..45f3313f9 100644 --- a/internal/cli/cmd/class/list.go +++ b/internal/cli/cmd/class/list.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" cmdutil "k8s.io/kubectl/pkg/cmd/util" @@ -105,7 +106,34 @@ func (o *ListOptions) printClass(constraintName string, compName string, classes for _, volume := range cls.Volumes { volumes = append(volumes, fmt.Sprintf("%s=%s", volume.Name, volume.Size.String())) } - tbl.AddRow(compName, cls.Name, cls.CPU.String(), cls.Memory.String(), strings.Join(volumes, ",")) + tbl.AddRow(compName, cls.Name, cls.CPU.String(), normalizeMemory(cls.Memory), strings.Join(volumes, ",")) } tbl.Print() } + +func normalizeMemory(mem resource.Quantity) string { + if !strings.HasSuffix(mem.String(), "m") { + return mem.String() + } + + var ( + value float64 + suffix string + bytes = float64(mem.MilliValue()) / 1000 + ) + switch { + case bytes < 1024: + value = bytes / 1024 + suffix = "Ki" + case bytes < 1024*1024: + value = bytes / 1024 / 1024 + suffix = "Mi" + case bytes < 1024*1024*1024: + value = bytes / 1024 / 1024 / 1024 + suffix = "Gi" + default: + value = bytes / 1024 / 1024 / 1024 / 1024 + suffix = "Ti" + } + return strings.TrimRight(fmt.Sprintf("%.3f", value), "0") + suffix +} diff --git a/internal/cli/cmd/class/list_test.go b/internal/cli/cmd/class/list_test.go index be2c3b773..550e3f15b 100644 --- a/internal/cli/cmd/class/list_test.go +++ b/internal/cli/cmd/class/list_test.go @@ -21,7 +21,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/kubernetes/scheme" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" @@ -57,4 +57,47 @@ var _ = Describe("list", func() { Expect(out.String()).To(ContainSubstring("mysql")) Expect(out.String()).To(ContainSubstring(generalResourceConstraint.Name)) }) + + It("memory should be normalized", func() { + cases := []struct { + memory string + normalized string + }{ + { + memory: "0.2Gi", + normalized: "0.2Gi", + }, + { + memory: "0.2Mi", + normalized: "0.2Mi", + }, + { + memory: "0.2Ki", + normalized: "0.2Ki", + }, + { + memory: "1024Mi", + normalized: "1Gi", + }, + { + memory: "1025Mi", + normalized: "1025Mi", + }, + { + memory: "1023Mi", + normalized: "1023Mi", + }, + { + memory: "1Gi", + normalized: "1Gi", + }, + { + memory: "512Mi", + normalized: "512Mi", + }, + } + for _, item := range cases { + Expect(normalizeMemory(resource.MustParse(item.memory))).Should(Equal(item.normalized)) + } + }) }) diff --git a/internal/cli/testing/testdata/class.yaml b/internal/cli/testing/testdata/class.yaml index 7ad0f25ca..e3b65a898 100644 --- a/internal/cli/testing/testdata/class.yaml +++ b/internal/cli/testing/testdata/class.yaml @@ -67,15 +67,4 @@ spec: - args: [ "32", "512", "20", "1" ] - args: [ "48", "768", "20", "1" ] - args: [ "64", "1024", "20", "1" ] - - args: [ "128", "2048", "20", "1" ] -status: - classes: - - name: general-1c1g - resourceConstraintRef: kb-resource-constraint-general - cpu: 1 - memory: 1Gi - storage: - - name: data - size: 100Gi - - name: log - size: 10Gi \ No newline at end of file + - args: [ "128", "2048", "20", "1" ] \ No newline at end of file diff --git a/internal/controller/builder/cue/pitr_job_template.cue b/internal/controller/builder/cue/pitr_job_template.cue index cebeecb4c..755b597d4 100644 --- a/internal/controller/builder/cue/pitr_job_template.cue +++ b/internal/controller/builder/cue/pitr_job_template.cue @@ -1,3 +1,17 @@ +// Copyright ApeCloud, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + container: { name: "pitr" image: string From 8d3c84e3073d438b6d687ca33eea3bd82fb15c7a Mon Sep 17 00:00:00 2001 From: xingran Date: Thu, 20 Apr 2023 10:47:39 +0800 Subject: [PATCH 100/439] chore: remove default postgresql metrics service and optmize redis sentinel configuration (#2742) --- deploy/postgresql/templates/clusterdefinition.yaml | 3 --- deploy/redis/scripts/redis-sentinel-setup.sh.tpl | 8 ++++---- deploy/redis/templates/clusterdefinition.yaml | 3 --- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 2ff1a7a14..a9b52b658 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -66,9 +66,6 @@ spec: - name: tcp-postgresql port: 5432 targetPort: tcp-postgresql - - name: http-metrics-postgresql - port: 9187 - targetPort: http-metrics volumeTypes: - name: data type: data diff --git a/deploy/redis/scripts/redis-sentinel-setup.sh.tpl b/deploy/redis/scripts/redis-sentinel-setup.sh.tpl index 8f28c0fe9..e1afcef92 100644 --- a/deploy/redis/scripts/redis-sentinel-setup.sh.tpl +++ b/deploy/redis/scripts/redis-sentinel-setup.sh.tpl @@ -21,14 +21,14 @@ set -ex {{- end }} {{- /* build primary pod message, because currently does not support cross-component acquisition of environment variables, the service of the redis master node is assembled here through specific rules */}} {{- $primary_pod = printf "%s-%s-%d.%s-%s-headless.%s.svc" $clusterName $redis_component.name $primary_index $clusterName $redis_component.name $namespace }} -{{- $sentinel_monitor := printf "%s-%s %s" $clusterName $sentinel_component.name $primary_pod }} +{{- $sentinel_monitor := printf "%s-%s %s" $clusterName $redis_component.name $primary_pod }} cat>/etc/sentinel/redis-sentinel.conf< Date: Thu, 20 Apr 2023 13:08:15 +0800 Subject: [PATCH 101/439] fix: mongodb support addon (#2758) --- .../helm/templates/addons/mongodb-addon.yaml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 deploy/helm/templates/addons/mongodb-addon.yaml diff --git a/deploy/helm/templates/addons/mongodb-addon.yaml b/deploy/helm/templates/addons/mongodb-addon.yaml new file mode 100644 index 000000000..7edbb59b4 --- /dev/null +++ b/deploy/helm/templates/addons/mongodb-addon.yaml @@ -0,0 +1,25 @@ +apiVersion: extensions.kubeblocks.io/v1alpha1 +kind: Addon +metadata: + name: mongodb + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} + "kubeblocks.io/provider": community + {{- if .Values.keepAddons }} + annotations: + helm.sh/resource-policy: keep + {{- end }} +spec: + description: 'MongoDB is a document database designed for ease of application development and scaling.' + + type: Helm + + helm: + # chartLocationURL: https://github.com/apecloud/helm-charts/releases/download/mongodb-{{ default .Chart.Version .Values.versionOverride }}/mongodb-{{ default .Chart.Version .Values.versionOverride }}.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/mongodb-{{ default .Chart.Version .Values.versionOverride }}.tgz + + installable: + autoInstall: true + + defaultInstallValues: + - enabled: true From 424459c0eaa5acec16547932adc1732b7a81fbdf Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Thu, 20 Apr 2023 13:32:48 +0800 Subject: [PATCH 102/439] chore: add .github/utils/current_milestone_bugs.sh script (#2750) --- .../utils/bugs_not_in_current_milestone.sh | 34 +++++--- .github/utils/current_milestone_bugs.sh | 46 ++++++++++ .github/utils/feature_triage.sh | 47 ++++++---- .github/utils/functions.bash | 57 ++++++++---- .github/utils/generate_release_notes.py | 7 +- .github/utils/get_release_version.py | 1 - .github/utils/gh_env | 6 +- .github/workflows/pull-request-check.yml | 2 +- docs/release_notes/template.md | 35 ++++++++ docs/release_notes/v0.5.0.md | 86 +++++++++++++++++++ 10 files changed, 273 insertions(+), 48 deletions(-) create mode 100755 .github/utils/current_milestone_bugs.sh create mode 100644 docs/release_notes/template.md create mode 100644 docs/release_notes/v0.5.0.md diff --git a/.github/utils/bugs_not_in_current_milestone.sh b/.github/utils/bugs_not_in_current_milestone.sh index 9675b4ccf..e6663b23f 100755 --- a/.github/utils/bugs_not_in_current_milestone.sh +++ b/.github/utils/bugs_not_in_current_milestone.sh @@ -9,18 +9,28 @@ set -o pipefail . ./gh_env . ./functions.bash -gh_get_issues "none" "kind/bug" - -rows=$(echo ${last_issue_list}| jq -r '. | sort_by(.state,.number)| .[].number') - - +process_issue_rows() { + for ((i = 0; i < ${item_count}; i++)) + do + local issue_body=$(echo ${last_issue_list} | jq -r ".[${i}]") + local issue_id=$(echo ${issue_body} | jq -r ".number") + local url=$(echo ${issue_body} | jq -r '.html_url') + local title=$(echo ${issue_body} | jq -r '.title') + local assignees=$(echo ${issue_body} | jq -r '.assignees[]?.login') + local state=$(echo ${issue_body}| jq -r '.state') + printf "[%s](%s) #%s | %s | %s | %s \n" "${title}" "${url}" "${issue_id}" "$(join_by , ${assignees})" "${state}" + gh_update_issue_milestone ${issue_id} + done +} + +item_count=100 +page=1 printf "%s | %s | %s \n" "Issue Title" "Assignees" "Issue State" echo "---|---|---" -for row in $rows -do - issue_id=$(echo $row | awk -F "," '{print $1}') - gh_get_issue_body ${issue_id} - printf "[%s](%s) #%s | %s | %s\n" "${last_issue_title}" "${last_issue_url}" "${issue_id}" "${last_issue_assignees_printable}" "${last_issue_state}" - - gh_update_issue_milestone ${issue_id} +while [ "${item_count}" == "100" ] +do + gh_get_issues "none" "kind/bug" "open" ${page} + item_count=$(echo ${last_issue_list} | jq -r '. | length') + process_issue_rows + page=$((page+1)) done \ No newline at end of file diff --git a/.github/utils/current_milestone_bugs.sh b/.github/utils/current_milestone_bugs.sh new file mode 100755 index 000000000..064fce660 --- /dev/null +++ b/.github/utils/current_milestone_bugs.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. + +. ./gh_env +. ./functions.bash + +LABELS=${LABELS:-'kind/bug,bug'} #"severity/critical,severity/major,severity/minor,severity/normal" + +print_issue_rows() { + for ((i = 0; i < ${item_count}; i++)) + do + local issue_body=$(echo ${last_issue_list} | jq -r ".[${i}]") + local issue_id=$(echo ${issue_body} | jq -r ".number") + local url=$(echo ${issue_body} | jq -r '.html_url') + local title=$(echo ${issue_body} | jq -r '.title') + local assignees=$(echo ${issue_body} | jq -r '.assignees[]?.login') + local state=$(echo ${issue_body}| jq -r '.state') + local labels=$(echo ${issue_body} | jq -r '.labels[]?.name') + printf "[%s](%s) #%s | %s | %s | %s \n" "${title}" "${url}" "${issue_id}" "$(join_by , ${assignees})" "${state}" "$(join_by , ${labels})" + done +} + +count_total=0 +item_count=100 +page=1 +echo "" +printf "%s | %s | %s | %s \n" "Issue Title" "Assignees" "Issue State" "Labels" +echo "---|---|---|---" +while [ "${item_count}" == "100" ] +do + gh_get_issues ${MILESTONE_ID} "${LABELS}" "open" ${page} + item_count=$(echo ${last_issue_list} | jq -r '. | length') + print_issue_rows + page=$((page+1)) + count_total=$((count_total + item_count)) +done + +if [ -n "$DEBUG" ]; then +echo "" +echo "total items: ${count_total}" +fi diff --git a/.github/utils/feature_triage.sh b/.github/utils/feature_triage.sh index 1eaea81d0..e5ff3f3b0 100755 --- a/.github/utils/feature_triage.sh +++ b/.github/utils/feature_triage.sh @@ -9,21 +9,38 @@ set -o pipefail . ./gh_env . ./functions.bash -gh_get_issues ${MILESTONE_ID} "kind/feature" "all" - -rows=$(echo ${last_issue_list}| jq -r '. | sort_by(.state,.number)| .[].number') - -echo $rows +print_issue_rows() { + for ((i = 0; i < ${item_count}; i++)) + do + local issue_body=$(echo ${last_issue_list} | jq -r ".[${i}]") + local issue_id=$(echo ${issue_body} | jq -r ".number") + local url=$(echo ${issue_body} | jq -r '.html_url') + local title=$(echo ${issue_body} | jq -r '.title') + local assignees=$(echo ${issue_body} | jq -r '.assignees[]?.login') + local state=$(echo ${issue_body}| jq -r '.state') + local labels=$(echo ${issue_body} | jq -r '.labels[]?.name') + pr_url=$(echo ${issue_body} | jq -r '.pull_request?.url') + if [ "$pr_url" == "null" ]; then + pr_url="N/A" + fi + printf "[%s](%s) #%s | %s | %s | %s| | \n" "${title}" "${url}" "${issue_id}" "$(join_by , ${assignees})" "${state}" "${pr_url}" + done +} +count_total=0 +item_count=100 +page=1 +echo "" printf "%s | %s | %s | %s | %s | %s\n" "Feature Title" "Assignees" "Issue State" "Code PR Merge Status" "Feature Doc. Status" "Extra Notes" echo "---|---|---|---|---|---" -for row in $rows -do - issue_id=$(echo $row | awk -F "," '{print $1}') - gh_get_issue_body ${issue_id} - pr_url=$(echo $last_issue_body| jq -r '.pull_request?.url') - if [ "$pr_url" == "null" ]; then - pr_url="N/A" - fi - printf "[%s](%s) #%s | %s | %s | %s| | \n" "${last_issue_title}" "${last_issue_url}" "${issue_id}" "${last_issue_assignees_printable}" "${last_issue_state}" "${pr_url}" -done \ No newline at end of file +while [ "${item_count}" == "100" ] +do + gh_get_issues ${MILESTONE_ID} "kind/feature" "all" ${page} + item_count=$(echo ${last_issue_list} | jq -r '. | length') + print_issue_rows + page=$((page+1)) + count_total=$((count_total + item_count)) +done + +echo "" +echo "total items: ${count_total}" diff --git a/.github/utils/functions.bash b/.github/utils/functions.bash index 55acad50d..45c84e83f 100644 --- a/.github/utils/functions.bash +++ b/.github/utils/functions.bash @@ -1,13 +1,40 @@ # bash functions +DEBUG=${DEBUG:-} # requires `gh` command, ref. https://cli.github.com/manual/installation for installation guides. gh_get_issues () { # @arg milestone - Milestone ID, if the string none is passed, issues without milestones are returned. + # @arg labels - A list of comma separated label names, processed as OR query. # @arg state - Can be one of: open, closed, all; Default: open. # @arg page - Cardinal value; Default: 1 # @result $last_issue_list - contains JSON result - declare milestone="$1" labels="$2" state="${3:-open}" page="${4:-1}" + declare milestone="$1" labels="$2" state="${3:-open}" page="${4:-1}" + local label_filter="" + IFS=',' read -ra label_items <<< "${labels}" + for i in "${label_items[@]}"; do + label_filter="${label_filter} -f labels=${i}" + done + _gh_get_issues ${milestone} "${label_filter}" ${state} ${page} +} + +gh_get_issues_with_and_labels () { + # @arg milestone - Milestone ID, if the string none is passed, issues without milestones are returned. + # @arg labels - A list of comma separated label names, processed as AND query. + # @arg state - Can be one of: open, closed, all; Default: open. + # @arg page - Cardinal value; Default: 1 + # @result $last_issue_list - contains JSON result + declare milestone="$1" labels="$2" state="${3:-open}" page="${4:-1}" + _gh_get_issues ${milestone} "-f labels=${labels}" ${state} ${page} +} + +_gh_get_issues () { + # @arg milestone - Milestone ID, if the string none is passed, issues without milestones are returned. + # @arg label_filter - Label fileter query params. + # @arg state - Can be one of: open, closed, all; Default: open. + # @arg page - Cardinal value; Default: 1 + # @result $last_issue_list - contains JSON result + declare milestone="$1" label_filter="$2" state="${3:-open}" page="${4:-1}" # GH list issues API ref: https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues local cmd="gh api \ @@ -18,10 +45,10 @@ gh_get_issues () { -F per_page=100 \ -F page=${page} \ -f milestone=${milestone} \ - -f labels=${labels} \ + ${label_filter} \ -f state=${state}" - echo $cmd - last_issue_list=`eval ${cmd}` + if [ -n "$DEBUG" ]; then echo $cmd; fi + last_issue_list=`eval ${cmd} 2> /dev/null` } @@ -31,8 +58,7 @@ gh_get_issue_body() { # @result last_issue_url # @result last_issue_title # @result last_issue_state - # @result last_issue_assignees - # @result last_issue_assignees_printable + # @result last_issue_assignees - multi-lines items declare issue_id="$1" local issue_body=$(gh api \ @@ -43,20 +69,12 @@ gh_get_issue_body() { local url=$(echo ${issue_body} | jq -r '.url') local title=$(echo ${issue_body} | jq -r '.title') local assignees=$(echo ${issue_body} | jq -r '.assignees[]?.login') - local assignees_printable= - for assignee in ${assignees} - do - assignees_printable="${assignees_printable},${assignee}" - done - local assignees_printable=${assignees_printable#,} local state=$(echo ${issue_body}| jq -r '.state') - last_issue_body="${issue_body}" last_issue_url="${url}" last_issue_title="${title}" last_issue_state="${state}" last_issue_assignees=${assignees} - last_issue_assignees_printable=${assignees_printable} } gh_update_issue_milestone() { @@ -70,10 +88,10 @@ gh_update_issue_milestone() { fi local req_data="{\"milestone\":$milestone_id}" - echo "req_data=$req_data" - local gh_token=$(gh auth token) + if [ -n "$DEBUG" ]; then echo "req_data=$req_data"; fi + local gh_token=$(gh auth token) local resp=$(curl \ --location \ --request PATCH \ @@ -84,4 +102,11 @@ gh_update_issue_milestone() { https://api.github.com/repos/${OWNER}/${REPO}/issues/${issue_id}) last_issue_resp=${resp} +} + +function join_by { + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi } \ No newline at end of file diff --git a/.github/utils/generate_release_notes.py b/.github/utils/generate_release_notes.py index f5c726892..6159fbea8 100755 --- a/.github/utils/generate_release_notes.py +++ b/.github/utils/generate_release_notes.py @@ -127,8 +127,11 @@ def main(argv: list[str]) -> None: # generate release note from template template = "" release_note_template_path = "docs/release_notes/template.md" - with open(release_note_template_path, "r") as file: - template = file.read() + try: + with open(release_note_template_path, "r") as file: + template = file.read() + except FileNotFoundError as e: + print(f"template {release_note_template_path} not found, IGNORED") change_text = "\n".join(change_lines) breaking_change_text = "None." diff --git a/.github/utils/get_release_version.py b/.github/utils/get_release_version.py index 622259c9b..eccfda834 100755 --- a/.github/utils/get_release_version.py +++ b/.github/utils/get_release_version.py @@ -9,7 +9,6 @@ import sys from typing import TypeAlias -OptStr : TypeAlias = str | None def main(argv: list[str]) -> None: git_ref = os.getenv("GITHUB_REF") diff --git a/.github/utils/gh_env b/.github/utils/gh_env index 46eeded15..0c7370c9c 100644 --- a/.github/utils/gh_env +++ b/.github/utils/gh_env @@ -1,3 +1,5 @@ +DEBUG=${DEBUG:-} + export REMOTE_URL=$(git config --get remote.origin.url) export OWNER=$(dirname ${REMOTE_URL} | awk -F ":" '{print $2}') prefix='^//' @@ -7,6 +9,8 @@ fi export REPO=$(basename -s .git ${REMOTE_URL}) export MILESTONE_ID=${MILESTONE_ID:-5} +if [ -n "$DEBUG" ]; then echo "OWNER=${OWNER}" echo "REPO=${REPO}" -echo "MILESTONE_ID=${MILESTONE_ID}" \ No newline at end of file +echo "MILESTONE_ID=${MILESTONE_ID}" +fi \ No newline at end of file diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml index 8fcb706ba..e67785dc0 100644 --- a/.github/workflows/pull-request-check.yml +++ b/.github/workflows/pull-request-check.yml @@ -15,7 +15,7 @@ jobs: - name: check branch name uses: apecloud/check-branch-name@v0.1.0 with: - branch_pattern: 'feature/|bugfix/|release/|hotfix/|support/|dependabot/' + branch_pattern: 'feature/|bugfix/|release/|releasing/|hotfix/|support/|dependabot/' comment_for_invalid_branch_name: 'This branch name is not following the standards: feature/|bugfix/|release/|hotfix/|support/|dependabot/' fail_if_invalid_branch_name: 'true' ignore_branch_pattern: 'main|master' diff --git a/docs/release_notes/template.md b/docs/release_notes/template.md new file mode 100644 index 000000000..c81747402 --- /dev/null +++ b/docs/release_notes/template.md @@ -0,0 +1,35 @@ +# KubeBlocks $kubeblocks_version ($today) + +We're happy to announce the release of KubeBlocks $kubeblocks_version! 🚀 🎉 🎈 + +We would like to extend our appreciation to all contributors who helped make this release happen. + +**Breaking changes** + + +**Highlights** + + +**Known issues and limitations** + * Limitations of cluster's horizontal scale operation: + * Only support VolumeSnapshot API to make a clone of Cluster's PV for syncing data when horizontal scaling. + * Only 1st pod container and 1st volume mount associated PV will be processed for VolumeSnapshot, do assure that data volume is placed in 1st pod container's 1st volume mount. + * Unused PVCs will be deleted in 30 minutes after scale in. + +If you're new to KubeBlocks, visit the [getting started](https://github.com/apecloud/kubeblocks/blob/v$kubeblocks_version/docs/user_docs/quick_start_guide.md) page and get a quick start with KubeBlocks. + +$warnings + +See [this](#upgrading-to-kubeblocks-$kubeblocks_version) section to upgrade KubeBlocks to version $kubeblocks_version. + +## Acknowledgements + +Thanks to everyone who made this release possible! + +$kubeblocks_contributors + +## What's Changed +$kubeblocks_changes + +## Upgrading to KubeBlocks $kubeblocks_version + diff --git a/docs/release_notes/v0.5.0.md b/docs/release_notes/v0.5.0.md new file mode 100644 index 000000000..7b4e04f97 --- /dev/null +++ b/docs/release_notes/v0.5.0.md @@ -0,0 +1,86 @@ +# KubeBlocks 0.5.0 (2023-04-20) + +We're happy to announce the release of KubeBlocks 0.5.0! 🚀 🎉 🎈 + +We would like to extend our appreciation to all contributors who helped make this release happen. + +**Breaking changes** + + +**Highlights** + + +**Known issues and limitations** + * Limitations of cluster's horizontal scale operation: + * Only support VolumeSnapshot API to make a clone of Cluster's PV for syncing data when horizontal scaling. + * Only 1st pod container and 1st volume mount associated PV will be processed for VolumeSnapshot, do assure that data volume is placed in 1st pod container's 1st volume mount. + * Unused PVCs will be deleted in 30 minutes after scale in. + +If you're new to KubeBlocks, visit the [getting started](https://github.com/apecloud/kubeblocks/blob/v0.5.0/docs/user_docs/quick_start_guide.md) page and get a quick start with KubeBlocks. + +> **Note: This release contains a few [breaking changes](#breaking-changes).** + +See [this](#upgrading-to-kubeblocks-0.5.0) section to upgrade KubeBlocks to version 0.5.0. + +## Acknowledgements + +Thanks to everyone who made this release possible! + +@1aal, @free6om, @heng4fun, @iziang, @ldming, @nayutah, @sophon-zt, @TalktoCrystal, @xuriwuyun, @Y-Rookie, @ZhaoDiankui + +## What's Changed + +### New Features +- support Redis snapshot backup and restore ([#1886](https://github.com/apecloud/kubeblocks/pull/1886), @heng4fun) +- sql channel support postgres ([#1898](https://github.com/apecloud/kubeblocks/pull/1898), @xuriwuyun) +- sql channel support pg checkstatus ([#2043](https://github.com/apecloud/kubeblocks/pull/2043), @xuriwuyun) +- support vitess ([#2116](https://github.com/apecloud/kubeblocks/pull/2116), @ZhaoDiankui) +- support mongodb ([#2182](https://github.com/apecloud/kubeblocks/pull/2182), @xuriwuyun) +- cli playground supports more cloud provider ([#2241](https://github.com/apecloud/kubeblocks/pull/2241), @ldming) +- support milvus standalone mode ([#2310](https://github.com/apecloud/kubeblocks/pull/2310), @nayutah) +- highly available Postgresql using our own image that support pgvector ([#2406](https://github.com/apecloud/kubeblocks/pull/2406), @ldming) +- delete clusters and uninstall kubeblocks when playground destroy ([#2457](https://github.com/apecloud/kubeblocks/pull/2457), @ldming) +- support mongodb backup ([#2682](https://github.com/apecloud/kubeblocks/pull/2682), @xuriwuyun) + +### Bug Fixes +- cli playground use default kubeconfig file ([#2150](https://github.com/apecloud/kubeblocks/pull/2150), @ldming) +- update running check ([#2174](https://github.com/apecloud/kubeblocks/pull/2174), @xuriwuyun) +- set cluster default storage size to 20Gi ([#2254](https://github.com/apecloud/kubeblocks/pull/2254), @ldming) +- cli kubeblocks upgrade command output dashboard info ([#2290](https://github.com/apecloud/kubeblocks/pull/2290), @ldming) +- set default storage size to 10Gi for TKE ([#2317](https://github.com/apecloud/kubeblocks/pull/2317), @ldming) +- cli playground pull latest cloud provider repo ([#2373](https://github.com/apecloud/kubeblocks/pull/2373), @ldming) +- cli playground does not output error message when kubernetes cluster is not ready ([#2391](https://github.com/apecloud/kubeblocks/pull/2391), @ldming) +- github action uploads kbcli asset for windows and add powershell script to install on windows ([#2449](https://github.com/apecloud/kubeblocks/pull/2449), @1aal) +- trim single quotes for the parameters value in the pg config file (#2523) ([#2527](https://github.com/apecloud/kubeblocks/pull/2527), @sophon-zt) +- config change does not take effect (#2511) ([#2543](https://github.com/apecloud/kubeblocks/pull/2543), @sophon-zt) +- KB_MYSQL_FOLLOWERS env inconsistent with cluster status after scale-in ([#2565](https://github.com/apecloud/kubeblocks/pull/2565), @free6om) +- BackupPolicyTemplate name of mysql-scale error ([#2583](https://github.com/apecloud/kubeblocks/pull/2583), @ZhaoDiankui) +- probe pg checkrole ([#2638](https://github.com/apecloud/kubeblocks/pull/2638), @xuriwuyun) +- adjust vtgate healthcheck options ([#2650](https://github.com/apecloud/kubeblocks/pull/2650), @ZhaoDiankui) +- h-scale pvc unexpected deleting ([#2680](https://github.com/apecloud/kubeblocks/pull/2680), @free6om) +- support mongodb backup ([#2683](https://github.com/apecloud/kubeblocks/pull/2683), @xuriwuyun) +- replicationSet cluster stop failed fix ([#2691](https://github.com/apecloud/kubeblocks/pull/2691), @Y-Rookie) +- h-scale pvc unexpected deleting (#2680) ([#2730](https://github.com/apecloud/kubeblocks/pull/2730), @free6om) + +### Miscellaneous +- lifecycle dag ([#1571](https://github.com/apecloud/kubeblocks/pull/1571), @free6om) +- add cluster default webhook for `PrimaryIndex` ([#1677](https://github.com/apecloud/kubeblocks/pull/1677), @heng4fun) +- refactor labels usage ([#1696](https://github.com/apecloud/kubeblocks/pull/1696), @heng4fun) +- update probe mysql tests ([#1808](https://github.com/apecloud/kubeblocks/pull/1808), @xuriwuyun) +- update pg probe url ([#2115](https://github.com/apecloud/kubeblocks/pull/2115), @xuriwuyun) +- cli support to output addon install progress ([#2132](https://github.com/apecloud/kubeblocks/pull/2132), @ldming) +- rewrite overview ([#2266](https://github.com/apecloud/kubeblocks/pull/2266), @TalktoCrystal) +- move loadbalancer sub-module to a separate repo https ([#2354](https://github.com/apecloud/kubeblocks/pull/2354), @iziang) +- use gitlab helm repo if failed to get ip location ([#2421](https://github.com/apecloud/kubeblocks/pull/2421), @ldming) +- update redis role probe ([#2554](https://github.com/apecloud/kubeblocks/pull/2554), @xuriwuyun) +- update mongodb helm ([#2575](https://github.com/apecloud/kubeblocks/pull/2575), @xuriwuyun) +- kbcli support mongodb ([#2580](https://github.com/apecloud/kubeblocks/pull/2580), @xuriwuyun) +- support xengine for apecloud-mysql ([#2608](https://github.com/apecloud/kubeblocks/pull/2608), @sophon-zt) +- support postgresql 14.7 instead of 15.2 ([#2613](https://github.com/apecloud/kubeblocks/pull/2613), @ldming) +- improve cluster create examples ([#2641](https://github.com/apecloud/kubeblocks/pull/2641), @ldming) +- ut for nil backup policy ([#2654](https://github.com/apecloud/kubeblocks/pull/2654), @free6om) +- sqlchannel add test ([#2694](https://github.com/apecloud/kubeblocks/pull/2694), @xuriwuyun) +- configure does not take effect ([#2735](https://github.com/apecloud/kubeblocks/pull/2735), @sophon-zt) + +## Upgrading to KubeBlocks 0.5.0 + From 1a372f910d98185c0cfe736b5517dcae692573bd Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Thu, 20 Apr 2023 13:47:23 +0800 Subject: [PATCH 103/439] chore: cli set default cluster value based on workload and component type (#2743) --- internal/cli/cmd/cluster/create.go | 52 +++++++++++++++++++++---- internal/cli/cmd/cluster/create_test.go | 13 ++++--- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 745a7dd32..b1aad7796 100644 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -431,13 +431,37 @@ func setEnableAllLogs(c *appsv1alpha1.Cluster, cd *appsv1alpha1.ClusterDefinitio } func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map[setKey]string) ([]*appsv1alpha1.ClusterComponentSpec, error) { - getVal := func(key setKey, sets map[setKey]string) string { + // get value from set values and environment variables, the second return value is + // true if the value is from environment variables + getVal := func(c *appsv1alpha1.ClusterComponentDefinition, key setKey, sets map[setKey]string) string { // get value from set values if sets != nil { if v := sets[key]; len(v) > 0 { return v } } + + // HACK: if user does not set by command flag, for replicationSet workload, + // set replicas to 2, for redis sentinel, set replicas to 3, cpu and memory + // to 200M and 200Mi + // TODO: use more graceful way to set default value + if c.WorkloadType == appsv1alpha1.Replication { + if key == keyReplicas { + return "2" + } + } + + if c.CharacterType == "redis" && c.Name == "redis-sentinel" { + switch key { + case keyReplicas: + return "3" + case keyCPU: + return "200m" + case keyMemory: + return "200Mi" + } + } + // get value from environment variables env := setKeyEnvMap[key] val := viper.GetString(env.name) @@ -452,7 +476,7 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map return nil } var switchPolicyType appsv1alpha1.SwitchPolicyType - switch getVal(keySwitchPolicy, sets) { + switch getVal(c, keySwitchPolicy, sets) { case "Noop", "": switchPolicyType = appsv1alpha1.Noop case "MaximumAvailability": @@ -476,7 +500,7 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map } // get replicas - setReplicas, err := strconv.Atoi(getVal(keyReplicas, sets)) + setReplicas, err := strconv.Atoi(getVal(&c, keyReplicas, sets)) if err != nil { return nil, fmt.Errorf("repicas is illegal " + err.Error()) } @@ -489,13 +513,13 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map } // class has higher priority than other resource related parameters - className := getVal(keyClass, sets) + className := getVal(&c, keyClass, sets) if className != "" { compObj.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: className} } else { resourceList := corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(getVal(keyCPU, sets)), - corev1.ResourceMemory: resource.MustParse(getVal(keyMemory, sets)), + corev1.ResourceCPU: resource.MustParse(getVal(&c, keyCPU, sets)), + corev1.ResourceMemory: resource.MustParse(getVal(&c, keyMemory, sets)), } compObj.Resources = corev1.ResourceRequirements{ Requests: resourceList, @@ -509,13 +533,13 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(getVal(keyStorage, sets)), + corev1.ResourceStorage: resource.MustParse(getVal(&c, keyStorage, sets)), }, }, }, }} } - if err := buildSwitchPolicy(&c, compObj, sets); err != nil { + if err = buildSwitchPolicy(&c, compObj, sets); err != nil { return nil, err } comps = append(comps, compObj) @@ -575,6 +599,18 @@ func buildCompSetsMap(values []string, cd *appsv1alpha1.ClusterDefinition) (map[ return nil, err } compDefName = name + } else { + // check the type is a valid component definition name + valid := false + for _, c := range cd.Spec.ComponentDefs { + if c.Name == compDefName { + valid = true + break + } + } + if !valid { + return nil, fmt.Errorf("the type \"%s\" is not a valid component definition name", compDefName) + } } // if already set by other value, later values override earlier values diff --git a/internal/cli/cmd/cluster/create_test.go b/internal/cli/cmd/cluster/create_test.go index ea97ed1f4..644ecbf28 100644 --- a/internal/cli/cmd/cluster/create_test.go +++ b/internal/cli/cmd/cluster/create_test.go @@ -292,8 +292,8 @@ var _ = Describe("create", func() { true, }, { - []string{"type=comp1,cpu=1,memory=2Gi,class=general-2c4g", "type=comp2,storage=10Gi,cpu=2,class=mo-1c8g"}, - []string{"my-comp"}, + []string{"type=comp1,cpu=1,memory=2Gi,class=general-2c4g", "type=comp2,storage=10Gi,cpu=2,class=mo-1c8g,replicas=3"}, + []string{"comp1", "comp2"}, map[string]map[setKey]string{ "comp1": { keyType: "comp1", @@ -302,10 +302,11 @@ var _ = Describe("create", func() { keyClass: "general-2c4g", }, "comp2": { - keyType: "comp2", - keyCPU: "2", - keyStorage: "10Gi", - keyClass: "mo-1c8g", + keyType: "comp2", + keyCPU: "2", + keyStorage: "10Gi", + keyClass: "mo-1c8g", + keyReplicas: "3", }, }, true, From b4c362ff7726da3213d24c05bd1bf8efb0be4cc3 Mon Sep 17 00:00:00 2001 From: "yunju.lly" Date: Thu, 20 Apr 2023 15:07:07 +0800 Subject: [PATCH 104/439] fix: make cluster cmd 'kbcli cluster update --enabled-all-logs' effect (#2663) --- .../apecloud-mysql/config/mysql8-config.tpl | 13 +- deploy/mongodb/config/mongodb5.0-config.tpl | 4 +- deploy/mongodb/values.yaml | 2 +- deploy/postgresql/config/pg14-config.tpl | 2 + deploy/redis/config/redis7-config.tpl | 2 + internal/cli/cluster/helper.go | 16 ++ internal/cli/cluster/helper_test.go | 20 ++ internal/cli/cmd/cluster/list_logs.go | 3 +- internal/cli/cmd/cluster/list_logs_test.go | 1 + internal/cli/cmd/cluster/update.go | 178 +++++++++++++++++- internal/cli/cmd/cluster/update_test.go | 176 +++++++++++++++++ internal/cli/testing/fake.go | 51 +++++ internal/cli/util/util.go | 65 +++++++ internal/cli/util/util_test.go | 13 ++ internal/configuration/config_patch_util.go | 28 +++ .../configuration/config_patch_util_test.go | 64 +++++++ internal/controller/plan/builtin_functions.go | 21 +++ internal/controller/plan/config_template.go | 17 +- internal/gotemplate/tpl_engine.go | 4 + 19 files changed, 646 insertions(+), 34 deletions(-) diff --git a/deploy/apecloud-mysql/config/mysql8-config.tpl b/deploy/apecloud-mysql/config/mysql8-config.tpl index 7663a8a7a..53b7e6c98 100644 --- a/deploy/apecloud-mysql/config/mysql8-config.tpl +++ b/deploy/apecloud-mysql/config/mysql8-config.tpl @@ -90,28 +90,25 @@ mysqlx=0 datadir={{ $data_root }}/data +{{ block "logsBlock" . }} log_statements_unsafe_for_binlog=OFF log_error_verbosity=2 log_output=FILE {{- if hasKey $.component "enabledLogs" }} {{- if mustHas "error" $.component.enabledLogs }} -# Mysql error log -log_error={{ $data_root }}/log/mysqld-error.log +log_error=/data/mysql/log/mysqld-error.log {{- end }} - {{- if mustHas "slow" $.component.enabledLogs }} -# MySQL Slow log slow_query_log=ON long_query_time=5 -slow_query_log_file={{ $data_root }}/log/mysqld-slowquery.log +slow_query_log_file=/data/mysql/log/mysqld-slowquery.log {{- end }} - {{- if mustHas "general" $.component.enabledLogs }} -# SQL access log, default off general_log=ON -general_log_file={{ $data_root }}/log/mysqld.log +general_log_file=/data/mysql/log/mysqld.log {{- end }} {{- end }} +{{ end }} #innodb innodb_doublewrite_batch_size=16 diff --git a/deploy/mongodb/config/mongodb5.0-config.tpl b/deploy/mongodb/config/mongodb5.0-config.tpl index bfc56e5a3..de5c5a187 100644 --- a/deploy/mongodb/config/mongodb5.0-config.tpl +++ b/deploy/mongodb/config/mongodb5.0-config.tpl @@ -21,13 +21,15 @@ storage: directoryPerDB: true # where to write logging data. +{{ block "logsBlock" . }} systemLog: destination: file quiet: false logAppend: true logRotate: reopen - path: {{ $mongodb_root }}/logs/mongodb.log + path: /data/mongodb/logs/mongodb.log verbosity: 0 +{{ end }} # network interfaces net: diff --git a/deploy/mongodb/values.yaml b/deploy/mongodb/values.yaml index 97cab9f36..313f1731c 100644 --- a/deploy/mongodb/values.yaml +++ b/deploy/mongodb/values.yaml @@ -30,7 +30,7 @@ auth: database: "admin" logConfigs: - running: /data/mongodb/log/mongodb.log* + running: /data/mongodb/logs/mongodb.log* metrics: image: diff --git a/deploy/postgresql/config/pg14-config.tpl b/deploy/postgresql/config/pg14-config.tpl index 38aa38697..812346da1 100644 --- a/deploy/postgresql/config/pg14-config.tpl +++ b/deploy/postgresql/config/pg14-config.tpl @@ -57,9 +57,11 @@ idle_in_transaction_session_timeout = '1h' listen_addresses = '0.0.0.0' log_autovacuum_min_duration = '1s' log_checkpoints = 'True' +{{ block "logsBlock" . }} log_destination = 'csvlog' log_directory = 'log' log_filename = 'postgresql-%Y-%m-%d.log' +{{ end }} log_lock_waits = 'True' log_min_duration_statement = '100' log_replication_commands = 'True' diff --git a/deploy/redis/config/redis7-config.tpl b/deploy/redis/config/redis7-config.tpl index 047dddc75..d7d8ebada 100644 --- a/deploy/redis/config/redis7-config.tpl +++ b/deploy/redis/config/redis7-config.tpl @@ -5,8 +5,10 @@ timeout 0 tcp-keepalive 300 daemonize no pidfile /var/run/redis_6379.pid +{{ block "logsBlock" . }} loglevel notice logfile "/data/running.log" +{{ end }} databases 16 always-show-logo no set-proc-title yes diff --git a/internal/cli/cluster/helper.go b/internal/cli/cluster/helper.go index 332324ccf..0df2e539d 100644 --- a/internal/cli/cluster/helper.go +++ b/internal/cli/cluster/helper.go @@ -426,3 +426,19 @@ func GetPodWorkloadType(pod *corev1.Pod) string { } return pod.Labels[constant.WorkloadTypeLabelKey] } + +func GetConfigMapByName(dynamic dynamic.Interface, namespace, name string) (*corev1.ConfigMap, error) { + cmObj := &corev1.ConfigMap{} + if err := GetK8SClientObject(dynamic, cmObj, types.ConfigmapGVR(), namespace, name); err != nil { + return nil, err + } + return cmObj, nil +} + +func GetConfigConstraintByName(dynamic dynamic.Interface, name string) (*appsv1alpha1.ConfigConstraint, error) { + ccObj := &appsv1alpha1.ConfigConstraint{} + if err := GetK8SClientObject(dynamic, ccObj, types.ConfigConstraintGVR(), "", name); err != nil { + return nil, err + } + return ccObj, nil +} diff --git a/internal/cli/cluster/helper_test.go b/internal/cli/cluster/helper_test.go index 129335fc6..e48320313 100644 --- a/internal/cli/cluster/helper_test.go +++ b/internal/cli/cluster/helper_test.go @@ -102,4 +102,24 @@ var _ = Describe("helper", func() { Expect(latestVer).ShouldNot(BeNil()) Expect(latestVer.Name).Should(Equal("now-version")) }) + + It("get configmap by name", func() { + cmName := "test-cm" + dynamic := testing.FakeDynamicClient(testing.FakeConfigMap(cmName)) + cm, err := GetConfigMapByName(dynamic, testing.Namespace, cmName) + Expect(err).Should(Succeed()) + Expect(cm).ShouldNot(BeNil()) + + cm, err = GetConfigMapByName(dynamic, testing.Namespace, cmName+"error") + Expect(err).Should(HaveOccurred()) + Expect(cm).Should(BeNil()) + }) + + It("get config constraint by name", func() { + ccName := "test-cc" + dynamic := testing.FakeDynamicClient(testing.FakeConfigConstraint(ccName)) + cm, err := GetConfigConstraintByName(dynamic, ccName) + Expect(err).Should(Succeed()) + Expect(cm).ShouldNot(BeNil()) + }) }) diff --git a/internal/cli/cmd/cluster/list_logs.go b/internal/cli/cmd/cluster/list_logs.go index 92c71f7e7..48ee0c912 100644 --- a/internal/cli/cmd/cluster/list_logs.go +++ b/internal/cli/cmd/cluster/list_logs.go @@ -143,7 +143,8 @@ func (o *ListLogsOptions) printListLogs(dataObj *cluster.ClusterObjects) error { tbl := printer.NewTablePrinter(o.Out) logFilesData := o.gatherLogFilesData(dataObj.Cluster, dataObj.ClusterDef, dataObj.Pods) if len(logFilesData) == 0 { - fmt.Fprintln(o.ErrOut, "No log files found. \nYou can enable the log feature when creating a cluster with option of \"--enable-all-logs=true\"") + fmt.Fprintf(o.ErrOut, "No log files found. You can enable the log feature with the kbcli command below.\n"+ + "kbcli cluster update %s --enable-all-logs=true --namespace %s\n", dataObj.Cluster.Name, dataObj.Cluster.Namespace) } else { tbl.SetHeader("INSTANCE", "LOG-TYPE", "FILE-PATH", "SIZE", "LAST-WRITTEN", "COMPONENT") for _, f := range logFilesData { diff --git a/internal/cli/cmd/cluster/list_logs_test.go b/internal/cli/cmd/cluster/list_logs_test.go index 6cdd10b15..3098f8914 100644 --- a/internal/cli/cmd/cluster/list_logs_test.go +++ b/internal/cli/cmd/cluster/list_logs_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/internal/cli/cmd/cluster/update.go b/internal/cli/cmd/cluster/update.go index 7cd8da50d..7424d94a7 100644 --- a/internal/cli/cmd/cluster/update.go +++ b/internal/cli/cmd/cluster/update.go @@ -17,13 +17,20 @@ limitations under the License. package cluster import ( + "bytes" + "context" "encoding/csv" + "encoding/json" "fmt" "strconv" "strings" + "text/template" + "github.com/google/uuid" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -36,6 +43,9 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/patch" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" + cfgcore "github.com/apecloud/kubeblocks/internal/configuration" + "github.com/apecloud/kubeblocks/internal/controller/plan" + "github.com/apecloud/kubeblocks/internal/gotemplate" ) var clusterUpdateExample = templates.Examples(` @@ -232,38 +242,190 @@ func (o *updateOptions) buildComponents(field string, val string) error { switch field { case "monitor": - return o.setMonitor(val) + return o.updateMonitor(val) case "enable-all-logs": - return o.setEnabledLog(val) + return o.updateEnabledLog(val) default: return nil } } -func (o *updateOptions) setEnabledLog(val string) error { +func (o *updateOptions) updateEnabledLog(val string) error { boolVal, err := strconv.ParseBool(val) if err != nil { return err } - // disable all monitor + // update --enabled-all-logs=false for all components if !boolVal { - for _, c := range o.cluster.Spec.ComponentSpecs { - c.EnabledLogs = nil + for index := range o.cluster.Spec.ComponentSpecs { + o.cluster.Spec.ComponentSpecs[index].EnabledLogs = nil } return nil } - // enable all monitor + // update --enabled-all-logs=true for all components cd, err := cluster.GetClusterDefByName(o.dynamic, o.cluster.Spec.ClusterDefRef) if err != nil { return err } + // set --enabled-all-logs at cluster components setEnableAllLogs(o.cluster, cd) + if err = o.reconfigureLogVariables(o.cluster, cd); err != nil { + return errors.Wrap(err, "reconfigure log variables of target cluster failed") + } return nil } -func (o *updateOptions) setMonitor(val string) error { +const logsBlockName = "logsBlock" +const logsTemplateName = "template-logs-block" +const topTPLLogsObject = "component" +const defaultSectionName = "default" + +// reconfigureLogVariables reconfigures the log variables of db kernel +func (o *updateOptions) reconfigureLogVariables(c *appsv1alpha1.Cluster, cd *appsv1alpha1.ClusterDefinition) error { + var ( + err error + keyName string + configSpec *appsv1alpha1.ComponentConfigSpec + configTemplate *corev1.ConfigMap + formatter *appsv1alpha1.FormatterConfig + logTPL *template.Template + logValue *gotemplate.TplValues + buf bytes.Buffer + logVariables map[string]string + unstructuredObj *unstructured.Unstructured + ) + for _, compSpec := range c.Spec.ComponentSpecs { + if configSpec, err = findFirstConfigSpec(c.Spec.ComponentSpecs, cd.Spec.ComponentDefs, compSpec.Name); err != nil { + return err + } + if configTemplate, formatter, err = findConfigTemplateInfo(o.dynamic, configSpec); err != nil { + return err + } + if keyName, logTPL, err = findLogsBlockTPL(configTemplate.Data); err != nil { + return err + } + if logValue, err = buildLogsTPLValues(&compSpec); err != nil { + return err + } + if err = logTPL.Execute(&buf, logValue); err != nil { + return err + } + formatter.FormatterOptions.IniConfig.SectionName = defaultSectionName + if logVariables, err = cfgcore.TransformConfigFileToKeyValueMap(keyName, formatter, buf.Bytes()); err != nil { + return err + } + // build OpsRequest and apply this OpsRequest + opsRequest := buildLogsReconfiguringOps(c.Name, c.Namespace, compSpec.Name, configSpec.Name, keyName, logVariables) + if unstructuredObj, err = util.ConvertObjToUnstructured(opsRequest); err != nil { + return err + } + if err = util.CreateResourceIfAbsent(o.dynamic, types.OpsGVR(), c.Namespace, unstructuredObj); err != nil { + return err + } + } + return nil +} + +func findFirstConfigSpec( + compSpecs []appsv1alpha1.ClusterComponentSpec, + cdCompSpecs []appsv1alpha1.ClusterComponentDefinition, + compName string) (*appsv1alpha1.ComponentConfigSpec, error) { + configSpecs, err := util.GetConfigTemplateListWithResource(compSpecs, cdCompSpecs, nil, compName, true) + if err != nil { + return nil, err + } + if len(configSpecs) == 0 { + return nil, errors.Errorf("no config template for component %s", compName) + } + return &configSpecs[0], nil +} + +func findConfigTemplateInfo(dynamic dynamic.Interface, configSpec *appsv1alpha1.ComponentConfigSpec) (*corev1.ConfigMap, *appsv1alpha1.FormatterConfig, error) { + if configSpec == nil { + return nil, nil, errors.New("configSpec is nil") + } + configTemplate, err := cluster.GetConfigMapByName(dynamic, configSpec.Namespace, configSpec.TemplateRef) + if err != nil { + return nil, nil, err + } + configConstraint, err := cluster.GetConfigConstraintByName(dynamic, configSpec.ConfigConstraintRef) + if err != nil { + return nil, nil, err + } + return configTemplate, configConstraint.Spec.FormatterConfig, nil +} + +func newConfigTemplateEngine() *template.Template { + customizedFuncMap := plan.BuiltInCustomFunctions(nil, nil) + engine := gotemplate.NewTplEngine(nil, customizedFuncMap, logsTemplateName, nil, context.TODO()) + return engine.GetTplEngine() +} + +func findLogsBlockTPL(confData map[string]string) (string, *template.Template, error) { + engine := newConfigTemplateEngine() + for key, value := range confData { + if !strings.Contains(value, logsBlockName) { + continue + } + tpl, err := engine.Parse(value) + if err != nil { + return key, nil, err + } + logTPL := tpl.Lookup(logsBlockName) + // find target logs template + if logTPL != nil { + return key, logTPL, nil + } + } + return "", nil, errors.New("no logs block template found") +} + +func buildLogsTPLValues(compSpec *appsv1alpha1.ClusterComponentSpec) (*gotemplate.TplValues, error) { + compMap := map[string]interface{}{} + bytesData, err := json.Marshal(compSpec) + if err != nil { + return nil, err + } + err = json.Unmarshal(bytesData, &compMap) + if err != nil { + return nil, err + } + value := gotemplate.TplValues{ + topTPLLogsObject: compMap, + } + return &value, nil +} + +func buildLogsReconfiguringOps(clusterName, namespace, compName, configName, keyName string, variables map[string]string) *appsv1alpha1.OpsRequest { + opsName := fmt.Sprintf("%s-%s", "logs-reconfigure", uuid.NewString()) + opsRequest := util.NewOpsRequestForReconfiguring(opsName, namespace, clusterName) + parameterPairs := make([]appsv1alpha1.ParameterPair, 0, len(variables)) + for key, value := range variables { + v := value + parameterPairs = append(parameterPairs, appsv1alpha1.ParameterPair{ + Key: key, + Value: &v, + }) + } + var keys []appsv1alpha1.ParameterConfig + keys = append(keys, appsv1alpha1.ParameterConfig{ + Key: keyName, + Parameters: parameterPairs, + }) + var configurations []appsv1alpha1.Configuration + configurations = append(configurations, appsv1alpha1.Configuration{ + Keys: keys, + Name: configName, + }) + reconfigure := opsRequest.Spec.Reconfigure + reconfigure.ComponentName = compName + reconfigure.Configurations = append(reconfigure.Configurations, configurations...) + return opsRequest +} + +func (o *updateOptions) updateMonitor(val string) error { boolVal, err := strconv.ParseBool(val) if err != nil { return err diff --git a/internal/cli/cmd/cluster/update_test.go b/internal/cli/cmd/cluster/update_test.go index ad89939aa..dae0d7deb 100644 --- a/internal/cli/cmd/cluster/update_test.go +++ b/internal/cli/cmd/cluster/update_test.go @@ -21,8 +21,11 @@ import ( . "github.com/onsi/gomega" "github.com/spf13/cobra" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/dynamic" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/patch" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" @@ -101,4 +104,177 @@ var _ = Describe("cluster update", func() { Expect(o.Patch).Should(ContainSubstring("k1")) }) }) + Context("logs variables reconfiguring tests", func() { + var ( + c *appsv1alpha1.Cluster + cd *appsv1alpha1.ClusterDefinition + myConfig string + ) + BeforeEach(func() { + c = testing.FakeCluster("c1", "default") + cd = testing.FakeClusterDef() + myConfig = ` +{{ block "logsBlock" . }} +log_statements_unsafe_for_binlog=OFF +log_error_verbosity=2 +log_output=FILE +{{- if hasKey $.component "enabledLogs" }} +{{- if mustHas "error" $.component.enabledLogs }} +log_error=/data/mysql/log/mysqld-error.log +{{- end }} +{{- if mustHas "slow" $.component.enabledLogs }} +slow_query_log=ON +long_query_time=5 +slow_query_log_file=/data/mysql/log/mysqld-slowquery.log +{{- end }} +{{- if mustHas "general" $.component.enabledLogs }} +general_log=ON +general_log_file=/data/mysql/log/mysqld.log +{{- end }} +{{- end }} +{{ end }} +` + }) + + It("findFirstConfigSpec tests", func() { + tests := []struct { + compSpecs []appsv1alpha1.ClusterComponentSpec + cdCompSpecs []appsv1alpha1.ClusterComponentDefinition + compName string + expectedErr bool + }{ + { + compSpecs: nil, + cdCompSpecs: nil, + compName: "name", + expectedErr: true, + }, + { + compSpecs: c.Spec.ComponentSpecs, + cdCompSpecs: cd.Spec.ComponentDefs, + compName: testing.ComponentName, + expectedErr: false, + }, + { + compSpecs: c.Spec.ComponentSpecs, + cdCompSpecs: cd.Spec.ComponentDefs, + compName: "error-name", + expectedErr: true, + }, + } + for _, test := range tests { + configSpec, err := findFirstConfigSpec(test.compSpecs, test.cdCompSpecs, test.compName) + if test.expectedErr { + Expect(err).Should(HaveOccurred()) + } else { + Expect(configSpec).ShouldNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + } + } + }) + + It("findConfigTemplateInfo tests", func() { + tests := []struct { + dynamic dynamic.Interface + configSpec *appsv1alpha1.ComponentConfigSpec + expectedErr bool + }{{ + dynamic: nil, + configSpec: nil, + expectedErr: true, + }, { + dynamic: testing.FakeDynamicClient(testing.FakeConfigMap("config-template")), + configSpec: &appsv1alpha1.ComponentConfigSpec{ + ComponentTemplateSpec: appsv1alpha1.ComponentTemplateSpec{ + TemplateRef: "config-template", + Namespace: testing.Namespace, + }, + }, + expectedErr: true, + }, { + dynamic: testing.FakeDynamicClient(testing.FakeConfigMap("config-template")), + configSpec: &appsv1alpha1.ComponentConfigSpec{ + ComponentTemplateSpec: appsv1alpha1.ComponentTemplateSpec{ + TemplateRef: "config-template", + Namespace: testing.Namespace, + }, + }, + expectedErr: true, + }, { + dynamic: testing.FakeDynamicClient(testing.FakeConfigMap("config-template"), testing.FakeConfigConstraint("config-constraint")), + configSpec: &appsv1alpha1.ComponentConfigSpec{ + ComponentTemplateSpec: appsv1alpha1.ComponentTemplateSpec{ + TemplateRef: "config-template", + + Namespace: testing.Namespace, + }, + ConfigConstraintRef: "config-constraint", + }, + expectedErr: false, + }} + for _, test := range tests { + cm, format, err := findConfigTemplateInfo(test.dynamic, test.configSpec) + if test.expectedErr { + Expect(err).Should(HaveOccurred()) + } else { + Expect(cm).ShouldNot(BeNil()) + Expect(format).ShouldNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + } + } + }) + + It("findLogsBlockTPL tests", func() { + tests := []struct { + confData map[string]string + keyName string + expectedErr bool + }{{ + confData: nil, + keyName: "", + expectedErr: true, + }, { + confData: map[string]string{ + "test.cnf": "test", + "my.cnf": "{{ logsBlock", + }, + keyName: "my.cnf", + expectedErr: true, + }, { + confData: map[string]string{ + "my.cnf": myConfig, + }, + keyName: "my.cnf", + expectedErr: false, + }, + } + for _, test := range tests { + key, tpl, err := findLogsBlockTPL(test.confData) + if test.expectedErr { + Expect(err).Should(HaveOccurred()) + } else { + Expect(key).Should(Equal(test.keyName)) + Expect(tpl).ShouldNot(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) + } + } + }) + + It("buildLogsTPLValues tests", func() { + configSpec := testing.FakeCluster("test", "test").Spec.ComponentSpecs[0] + tplValue, err := buildLogsTPLValues(&configSpec) + Expect(err).ShouldNot(HaveOccurred()) + Expect(tplValue).ShouldNot(BeNil()) + }) + + It("buildLogsReconfiguringOps tests", func() { + opsRequest := buildLogsReconfiguringOps("clusterName", "namespace", "compName", "configName", "keyName", map[string]string{"key1": "value1", "key2": "value2"}) + Expect(opsRequest).ShouldNot(BeNil()) + Expect(opsRequest.Spec.Reconfigure.ComponentName).Should(Equal("compName")) + Expect(opsRequest.Spec.Reconfigure.Configurations).Should(HaveLen(1)) + Expect(opsRequest.Spec.Reconfigure.Configurations[0].Keys).Should(HaveLen(1)) + Expect(opsRequest.Spec.Reconfigure.Configurations[0].Keys[0].Parameters).Should(HaveLen(2)) + }) + + }) }) diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index 0e78c2976..4c01fa34c 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -243,10 +243,32 @@ func FakeClusterDef() *appsv1alpha1.ClusterDefinition { PasswordConfig: appsv1alpha1.PasswordConfig{}, Accounts: []appsv1alpha1.SystemAccountConfig{}, }, + ConfigSpecs: []appsv1alpha1.ComponentConfigSpec{ + { + ComponentTemplateSpec: appsv1alpha1.ComponentTemplateSpec{ + Name: "mysql-consensusset-config", + TemplateRef: "mysql8.0-config-template", + Namespace: Namespace, + VolumeName: "mysql-config", + }, + ConfigConstraintRef: "mysql8.0-config-constraints", + }, + }, }, { Name: fmt.Sprintf("%s-%d", ComponentDefName, 1), CharacterType: "mysql", + ConfigSpecs: []appsv1alpha1.ComponentConfigSpec{ + { + ComponentTemplateSpec: appsv1alpha1.ComponentTemplateSpec{ + Name: "mysql-consensusset-config", + TemplateRef: "mysql8.0-config-template", + Namespace: Namespace, + VolumeName: "mysql-config", + }, + ConfigConstraintRef: "mysql8.0-config-constraints", + }, + }, }, } return clusterDef @@ -473,3 +495,32 @@ func FakeAddon(name string) *extensionsv1alpha1.Addon { addon.SetCreationTimestamp(metav1.Now()) return addon } + +func FakeConfigMap(cmName string) *corev1.ConfigMap { + cm := &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "ConfigMap", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: Namespace, + }, + Data: map[string]string{ + "fake": "fake", + }, + } + return cm +} + +func FakeConfigConstraint(ccName string) *appsv1alpha1.ConfigConstraint { + cm := &appsv1alpha1.ConfigConstraint{ + ObjectMeta: metav1.ObjectMeta{ + Name: ccName, + }, + Spec: appsv1alpha1.ConfigConstraintSpec{ + FormatterConfig: &appsv1alpha1.FormatterConfig{}, + }, + } + return cm +} diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index a62539d8b..4a4d3f52f 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -21,6 +21,7 @@ import ( "crypto/rand" "crypto/rsa" "crypto/x509" + "encoding/json" "encoding/pem" "fmt" "io" @@ -42,11 +43,13 @@ import ( "golang.org/x/crypto/ssh" corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" apiruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + k8sapitypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/duration" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" @@ -697,3 +700,65 @@ func buildLabelSelectors(prefix string, key string, names []string) string { return prefix + "," + label } } + +// NewOpsRequestForReconfiguring returns a new common OpsRequest for Reconfiguring operation +func NewOpsRequestForReconfiguring(opsName, namespace, clusterName string) *appsv1alpha1.OpsRequest { + return &appsv1alpha1.OpsRequest{ + TypeMeta: metav1.TypeMeta{ + APIVersion: fmt.Sprintf("%s/%s", types.AppsAPIGroup, types.AppsAPIVersion), + Kind: types.KindOps, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: opsName, + Namespace: namespace, + }, + Spec: appsv1alpha1.OpsRequestSpec{ + ClusterRef: clusterName, + Type: appsv1alpha1.ReconfiguringType, + Reconfigure: &appsv1alpha1.Reconfigure{}, + }, + } +} +func ConvertObjToUnstructured(obj any) (*unstructured.Unstructured, error) { + var ( + contentBytes []byte + err error + unstructuredObj = &unstructured.Unstructured{} + ) + + if contentBytes, err = json.Marshal(obj); err != nil { + return nil, err + } + if err = json.Unmarshal(contentBytes, unstructuredObj); err != nil { + return nil, err + } + return unstructuredObj, nil +} + +func CreateResourceIfAbsent( + dynamic dynamic.Interface, + gvr schema.GroupVersionResource, + namespace string, + unstructuredObj *unstructured.Unstructured) error { + objectName, isFound, err := unstructured.NestedString(unstructuredObj.Object, "metadata", "name") + if !isFound || err != nil { + return err + } + objectByte, err := json.Marshal(unstructuredObj) + if err != nil { + return err + } + if _, err = dynamic.Resource(gvr).Namespace(namespace).Patch( + context.TODO(), objectName, k8sapitypes.MergePatchType, + objectByte, metav1.PatchOptions{}); err != nil { + if apierrors.IsNotFound(err) { + if _, err = dynamic.Resource(gvr).Namespace(namespace).Create( + context.TODO(), unstructuredObj, metav1.CreateOptions{}); err != nil { + return err + } + } else { + return err + } + } + return nil +} diff --git a/internal/cli/util/util_test.go b/internal/cli/util/util_test.go index b8da7c6e0..4f12d8d0b 100644 --- a/internal/cli/util/util_test.go +++ b/internal/cli/util/util_test.go @@ -252,4 +252,17 @@ var _ = Describe("util", func() { It("get helm chart repo url", func() { Expect(GetHelmChartRepoURL()).ShouldNot(BeEmpty()) }) + + It("new OpsRequest for Reconfiguring ", func() { + Expect(NewOpsRequestForReconfiguring("logs", "test", "cluster")).ShouldNot(BeNil()) + }) + + It("convert obj to unstructured ", func() { + unstructuredObj, err := ConvertObjToUnstructured(testing.FakeConfigMap("cm-test")) + Expect(err).ShouldNot(HaveOccurred()) + Expect(unstructuredObj.Object).Should(HaveLen(4)) + + _, err = ConvertObjToUnstructured(struct{ name string }{name: "test"}) + Expect(err).Should(HaveOccurred()) + }) }) diff --git a/internal/configuration/config_patch_util.go b/internal/configuration/config_patch_util.go index 7f5accb3d..8ed50a234 100644 --- a/internal/configuration/config_patch_util.go +++ b/internal/configuration/config_patch_util.go @@ -82,3 +82,31 @@ func LoadRawConfigObject(data map[string]string, formatConfig *appsv1alpha1.Form } return r, nil } + +// TransformConfigFileToKeyValueMap transforms a config file which formed by appsv1alpha1.CfgFileFormat format to a map in which the key is config name and the value is config value。 +// sectionName means the desired section of config file, such as [mysqld] section. +// If config file has no section structure, sectionName should be default to get all values in this config file. +func TransformConfigFileToKeyValueMap(fileName string, formatterConfig *appsv1alpha1.FormatterConfig, configData []byte) (map[string]string, error) { + oldData := map[string]string{ + fileName: "", + } + newData := map[string]string{ + fileName: string(configData), + } + keys := []string{fileName} + patchInfo, _, err := CreateConfigPatch(oldData, newData, formatterConfig.Format, keys, false) + if err != nil { + return nil, err + } + params := GenerateVisualizedParamsList(patchInfo, formatterConfig, nil) + result := make(map[string]string) + for _, param := range params { + if param.Key != fileName { + continue + } + for _, kv := range param.Parameters { + result[kv.Key] = kv.Value + } + } + return result, nil +} diff --git a/internal/configuration/config_patch_util_test.go b/internal/configuration/config_patch_util_test.go index 72c856699..905aaeca7 100644 --- a/internal/configuration/config_patch_util_test.go +++ b/internal/configuration/config_patch_util_test.go @@ -17,6 +17,7 @@ limitations under the License. package configuration import ( + "reflect" "testing" "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -348,3 +349,66 @@ func TestLoadRawConfigObject(t *testing.T) { }) } } + +func TestTransformConfigFileToKeyValueMap(t *testing.T) { + mysqlConfig := ` +[mysqld] +key_buffer_size=16777216 +log_error=/data/mysql/logs/mysql.log +` + mongodbConfig := ` +systemLog: + logRotate: reopen + path: /data/mongodb/logs/mongodb.log + verbosity: 0 +` + tests := []struct { + name string + fileName string + formatConfig *v1alpha1.FormatterConfig + configData []byte + expected map[string]string + }{{ + name: "mysql-test", + fileName: "my.cnf", + formatConfig: &v1alpha1.FormatterConfig{ + Format: v1alpha1.Ini, + FormatterOptions: v1alpha1.FormatterOptions{ + IniConfig: &v1alpha1.IniConfig{ + SectionName: "mysqld", + }, + }, + }, + configData: []byte(mysqlConfig), + expected: map[string]string{ + "key_buffer_size": "16777216", + "log_error": "/data/mysql/logs/mysql.log", + }, + }, { + name: "mongodb-test", + fileName: "mongodb.conf", + formatConfig: &v1alpha1.FormatterConfig{ + Format: v1alpha1.YAML, + FormatterOptions: v1alpha1.FormatterOptions{ + IniConfig: &v1alpha1.IniConfig{ + SectionName: "default", + }, + }, + }, + configData: []byte(mongodbConfig), + expected: map[string]string{ + "systemLog.logRotate": "reopen", + "systemLog.path": "/data/mongodb/logs/mongodb.log", + "systemLog.verbosity": "0", + }, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res, _ := TransformConfigFileToKeyValueMap(tt.fileName, tt.formatConfig, tt.configData) + if !reflect.DeepEqual(res, tt.expected) { + t.Errorf("TransformConfigFileToKeyValueMap() res = %v, res %v", res, tt.expected) + return + } + }) + } +} diff --git a/internal/controller/plan/builtin_functions.go b/internal/controller/plan/builtin_functions.go index a7a9d67b0..bcfc36ce5 100644 --- a/internal/controller/plan/builtin_functions.go +++ b/internal/controller/plan/builtin_functions.go @@ -24,7 +24,9 @@ import ( corev1 "k8s.io/api/core/v1" "github.com/apecloud/kubeblocks/internal/controller/builder" + intctrltypes "github.com/apecloud/kubeblocks/internal/controller/types" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" + "github.com/apecloud/kubeblocks/internal/gotemplate" ) func toJSONObject[T corev1.VolumeSource | corev1.Container | corev1.ContainerPort](obj T) (interface{}, error) { @@ -264,3 +266,22 @@ func getCertFile() string { func getKeyFile() string { return builder.MountPath + "/" + builder.KeyName } + +// BuiltInCustomFunctions builds a map of customized functions for KubeBlocks +func BuiltInCustomFunctions(c *configTemplateBuilder, task *intctrltypes.ReconcileTask) *gotemplate.BuiltInObjectsFunc { + return &gotemplate.BuiltInObjectsFunc{ + builtInMysqlCalBufferFunctionName: calDBPoolSize, + builtInGetVolumeFunctionName: getVolumeMountPathByName, + builtInGetPvcFunctionName: getPVCByName, + builtInGetEnvFunctionName: wrapGetEnvByName(c, task), + builtInGetPortFunctionName: getPortByName, + builtInGetArgFunctionName: getArgByName, + builtInGetContainerFunctionName: getPodContainerByName, + builtInGetContainerCPUFunctionName: getContainerCPU, + builtInGetContainerMemoryFunctionName: getContainerMemory, + builtInGetContainerRequestMemoryFunctionName: getContainerRequestMemory, + builtInGetCAFile: getCAFile, + builtInGetCertFile: getCertFile, + builtInGetKeyFile: getKeyFile, + } +} diff --git a/internal/controller/plan/config_template.go b/internal/controller/plan/config_template.go index 29fd794aa..31fce219c 100644 --- a/internal/controller/plan/config_template.go +++ b/internal/controller/plan/config_template.go @@ -173,21 +173,8 @@ func (c *configTemplateBuilder) injectBuiltInObjectsAndFunctions( func (c *configTemplateBuilder) injectBuiltInFunctions(component *component.SynthesizedComponent, task *intctrltypes.ReconcileTask) error { // TODO add built-in function - c.builtInFunctions = &gotemplate.BuiltInObjectsFunc{ - builtInMysqlCalBufferFunctionName: calDBPoolSize, - builtInGetVolumeFunctionName: getVolumeMountPathByName, - builtInGetPvcFunctionName: getPVCByName, - builtInGetEnvFunctionName: wrapGetEnvByName(c, task), - builtInGetPortFunctionName: getPortByName, - builtInGetArgFunctionName: getArgByName, - builtInGetContainerFunctionName: getPodContainerByName, - builtInGetContainerCPUFunctionName: getContainerCPU, - builtInGetContainerMemoryFunctionName: getContainerMemory, - builtInGetContainerRequestMemoryFunctionName: getContainerRequestMemory, - builtInGetCAFile: getCAFile, - builtInGetCertFile: getCertFile, - builtInGetKeyFile: getKeyFile, - } + c.builtInFunctions = BuiltInCustomFunctions(c, task) + // other logic here return nil } diff --git a/internal/gotemplate/tpl_engine.go b/internal/gotemplate/tpl_engine.go index 23a068853..1e81d4e2f 100644 --- a/internal/gotemplate/tpl_engine.go +++ b/internal/gotemplate/tpl_engine.go @@ -70,6 +70,10 @@ type TplEngine struct { ctx context.Context } +func (t *TplEngine) GetTplEngine() *template.Template { + return t.tpl +} + func (t *TplEngine) Render(context string) (string, error) { var buf strings.Builder tpl, err := t.tpl.Parse(context) From 6c96d4ff7907a7726c067054781313846f61cc0e Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Thu, 20 Apr 2023 15:20:49 +0800 Subject: [PATCH 105/439] chore: upgrade spilo fix pg config (#2771) --- deploy/postgresql/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/postgresql/values.yaml b/deploy/postgresql/values.yaml index 96a9c62c6..cc18b2d0d 100644 --- a/deploy/postgresql/values.yaml +++ b/deploy/postgresql/values.yaml @@ -11,7 +11,7 @@ image: registry: registry.cn-hangzhou.aliyuncs.com repository: apecloud/spilo - tag: 14.7.0 + tag: 14.7.1 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' From 94358668fffa7e532f4bf041d406b721686060e0 Mon Sep 17 00:00:00 2001 From: "zheyi.cqy" Date: Thu, 20 Apr 2023 16:03:18 +0800 Subject: [PATCH 106/439] fix: migration kbcli describe metrics with sorting. & cdc status display logic optimization (#2777) --- internal/cli/cmd/migration/describe.go | 26 ++++++++++++--- internal/cli/cmd/migration/describe_test.go | 37 +++++++++++++++++++++ internal/cli/types/migrationapi/type.go | 6 ++-- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/internal/cli/cmd/migration/describe.go b/internal/cli/cmd/migration/describe.go index bc9033501..ce9d41310 100644 --- a/internal/cli/cmd/migration/describe.go +++ b/internal/cli/cmd/migration/describe.go @@ -26,6 +26,7 @@ import ( "time" "github.com/spf13/cobra" + appv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -147,7 +148,7 @@ func (o *describeOptions) describeMigration(name string) error { switch o.Task.Spec.TaskType { case v1alpha1.InitializationAndCdc, v1alpha1.CDC: // Cdc Detail - showCdc(o.Pods, o.Out) + showCdc(o.StatefulSets, o.Pods, o.Out) // Cdc Metrics showCdcMetrics(o.Task, o.Out) @@ -183,6 +184,9 @@ func getMigrationObjects(o *describeOptions, taskName string) (*v1alpha1.Migrati if obj.Pods, err = o.client.CoreV1().Pods(o.namespace).List(context.Background(), listOpts()); err != nil { return nil, err } + if obj.StatefulSets, err = o.client.AppsV1().StatefulSets(o.namespace).List(context.Background(), listOpts()); err != nil { + return nil, err + } return obj, nil } @@ -249,8 +253,8 @@ func showInitialization(task *v1alpha1.MigrationTask, template *v1alpha1.Migrati tbl.Print() } -func showCdc(pods *v1.PodList, out io.Writer) { - if len(pods.Items) == 0 { +func showCdc(statefulSets *appv1.StatefulSetList, pods *v1.PodList, out io.Writer) { + if len(pods.Items) == 0 || len(statefulSets.Items) == 0 { return } tbl := newTbl(out, "\nCdc:", "NAMESPACE", "STATUS", "CREATED_TIME", "START-TIME") @@ -258,7 +262,7 @@ func showCdc(pods *v1.PodList, out io.Writer) { if pod.Annotations[MigrationTaskStepAnnotation] != v1alpha1.StepCdc.String() { continue } - tbl.AddRow(pod.Namespace, pod.Status.Phase, util.TimeFormatWithDuration(&pod.CreationTimestamp, time.Second), util.TimeFormatWithDuration(pod.Status.StartTime, time.Second)) + tbl.AddRow(pod.Namespace, getCdcStatus(&statefulSets.Items[0], &pod), util.TimeFormatWithDuration(&pod.CreationTimestamp, time.Second), util.TimeFormatWithDuration(pod.Status.StartTime, time.Second)) } tbl.Print() } @@ -271,6 +275,7 @@ func showCdcMetrics(task *v1alpha1.MigrationTask, out io.Writer) { for mKey := range task.Status.Cdc.Metrics { arr = append(arr, mKey) } + sort.Strings(arr) tbl := newTbl(out, "\nCdc Metrics:") for _, k := range arr { tbl.AddRow(k, task.Status.Cdc.Metrics[k]) @@ -285,3 +290,16 @@ func getJobStatus(conditions []batchv1.JobCondition) string { return string(conditions[len(conditions)-1].Type) } } + +func getCdcStatus(statefulSet *appv1.StatefulSet, cdcPod *v1.Pod) v1.PodPhase { + if cdcPod.Status.Phase == v1.PodRunning && + statefulSet.Status.Replicas > statefulSet.Status.AvailableReplicas { + if time.Now().Unix()-statefulSet.CreationTimestamp.Time.Unix() < 10*60 { + return v1.PodPending + } else { + return v1.PodFailed + } + } else { + return cdcPod.Status.Phase + } +} diff --git a/internal/cli/cmd/migration/describe_test.go b/internal/cli/cmd/migration/describe_test.go index d8c0dd62b..af7b1bcb2 100644 --- a/internal/cli/cmd/migration/describe_test.go +++ b/internal/cli/cmd/migration/describe_test.go @@ -17,9 +17,14 @@ limitations under the License. package migration import ( + "time" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" ) @@ -36,4 +41,36 @@ var _ = Describe("describe", func() { Expect(cmd).ShouldNot(BeNil()) }) + It("func test", func() { + sts := appv1.StatefulSet{ + Status: appv1.StatefulSetStatus{ + Replicas: 1, + }, + } + pod := corev1.Pod{} + + sts.Status.AvailableReplicas = 0 + pod.Status.Phase = corev1.PodFailed + Expect(getCdcStatus(&sts, &pod)).Should(Equal(corev1.PodFailed)) + + sts.Status.AvailableReplicas = 1 + pod.Status.Phase = corev1.PodPending + Expect(getCdcStatus(&sts, &pod)).Should(Equal(corev1.PodPending)) + + sts.Status.AvailableReplicas = 1 + pod.Status.Phase = corev1.PodRunning + Expect(getCdcStatus(&sts, &pod)).Should(Equal(corev1.PodRunning)) + + sts.Status.AvailableReplicas = 0 + t1, _ := time.ParseDuration("-30m") + sts.CreationTimestamp = v1.NewTime(time.Now().Add(t1)) + pod.Status.Phase = corev1.PodRunning + Expect(getCdcStatus(&sts, &pod)).Should(Equal(corev1.PodFailed)) + + sts.Status.AvailableReplicas = 0 + sts.CreationTimestamp = v1.NewTime(time.Now()) + pod.Status.Phase = corev1.PodRunning + Expect(getCdcStatus(&sts, &pod)).Should(Equal(corev1.PodPending)) + }) + }) diff --git a/internal/cli/types/migrationapi/type.go b/internal/cli/types/migrationapi/type.go index 836b55a07..623a5550e 100644 --- a/internal/cli/types/migrationapi/type.go +++ b/internal/cli/types/migrationapi/type.go @@ -19,6 +19,7 @@ package v1alpha1 import ( "strings" + appv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -182,8 +183,9 @@ type MigrationObjects struct { Task *MigrationTask Template *MigrationTemplate - Jobs *batchv1.JobList - Pods *v1.PodList + Jobs *batchv1.JobList + Pods *v1.PodList + StatefulSets *appv1.StatefulSetList } // +k8s:deepcopy-gen=false From 321fdc3375e18a67cf93019a991550a1b0ea8e3f Mon Sep 17 00:00:00 2001 From: shaojiang Date: Thu, 20 Apr 2023 16:09:22 +0800 Subject: [PATCH 107/439] fix: enhance PITR function (#2744) --- apis/dataprotection/v1alpha1/backup_types.go | 31 ++-- .../dataprotection/backup_controller.go | 46 ++++-- .../dataprotection/backuppolicy_controller.go | 15 +- controllers/dataprotection/cue/cronjob.cue | 56 +++++++- controllers/dataprotection/type.go | 1 + deploy/postgresql/config/pg14-config.tpl | 4 +- .../templates/backuppolicytemplate.yaml | 16 ++- .../postgresql/templates/backuptool-pitr.yaml | 48 +++++-- deploy/postgresql/templates/backuptool.yaml | 2 +- deploy/postgresql/templates/configmap.yaml | 4 +- deploy/postgresql/templates/scripts.yaml | 14 +- internal/cli/cmd/cluster/dataprotection.go | 4 - .../cli/cmd/cluster/dataprotection_test.go | 44 +++--- internal/constant/const.go | 1 + .../transformer_backup_policy_tpl.go | 6 +- internal/controller/plan/pitr.go | 132 +++++++++--------- internal/controller/plan/pitr_test.go | 57 +++++--- internal/testutil/apps/backup_factory.go | 18 +++ 18 files changed, 322 insertions(+), 177 deletions(-) diff --git a/apis/dataprotection/v1alpha1/backup_types.go b/apis/dataprotection/v1alpha1/backup_types.go index 96b0147b0..efbc0a7f8 100644 --- a/apis/dataprotection/v1alpha1/backup_types.go +++ b/apis/dataprotection/v1alpha1/backup_types.go @@ -207,14 +207,20 @@ func (r *BackupSpec) Validate(backupPolicy *BackupPolicy) error { // GetRecoverableTimeRange return the recoverable time range array func GetRecoverableTimeRange(backups []Backup) []BackupLogStatus { // filter backups with backupLog - backupsWithLog := make([]Backup, 0) + baseBackups := make([]Backup, 0) + var incrementalBackup *Backup for _, b := range backups { - if b.Status.Phase == BackupCompleted && - b.Status.Manifests != nil && b.Status.Manifests.BackupLog != nil { - backupsWithLog = append(backupsWithLog, b) + if b.Status.Manifests == nil || b.Status.Manifests.BackupLog == nil || + b.Status.Manifests.BackupLog.StopTime == nil { + continue + } + if b.Spec.BackupType == BackupTypeIncremental { + incrementalBackup = &b + } else if b.Spec.BackupType != BackupTypeIncremental && b.Status.Phase == BackupCompleted { + baseBackups = append(baseBackups, b) } } - if len(backupsWithLog) == 0 { + if len(baseBackups) == 0 { return nil } sort.Slice(backups, func(i, j int) bool { @@ -230,18 +236,9 @@ func GetRecoverableTimeRange(backups []Backup) []BackupLogStatus { return backups[i].Status.StartTimestamp.Before(backups[j].Status.StartTimestamp) }) result := make([]BackupLogStatus, 0) - start, end := backupsWithLog[0].Status.Manifests.BackupLog.StopTime, backupsWithLog[0].Status.Manifests.BackupLog.StopTime - - for i := 1; i < len(backupsWithLog); i++ { - b := backupsWithLog[i].Status.Manifests.BackupLog - if b.StartTime.Before(end) || b.StartTime.Equal(end) { - if b.StopTime.After(end.Time) { - end = b.StopTime - } - } else { - result = append(result, BackupLogStatus{StartTime: start, StopTime: end}) - start, end = b.StopTime, b.StopTime - } + start, end := baseBackups[0].Status.Manifests.BackupLog.StopTime, baseBackups[0].Status.Manifests.BackupLog.StopTime + if incrementalBackup != nil && start.Before(incrementalBackup.Status.Manifests.BackupLog.StopTime) { + end = incrementalBackup.Status.Manifests.BackupLog.StopTime } return append(result, BackupLogStatus{StartTime: start, StopTime: end}) } diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 70e6dda46..1307f5741 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -211,6 +211,7 @@ func (r *BackupReconciler) doNewPhaseAction( Time: backup.Status.StartTimestamp.Add(dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.TTL)), } } + if err = r.Client.Status().Patch(reqCtx.Ctx, backup, patch); err != nil { return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } @@ -344,7 +345,7 @@ func (r *BackupReconciler) doInProgressPhaseAction( if !isOK { return intctrlutil.RequeueAfter(reconcileInterval, reqCtx.Log, "") } - if err = r.createUpdatesJobs(reqCtx, backup, backupPolicy.Spec.Snapshot, dataprotectionv1alpha1.PRE); err != nil { + if err = r.createUpdatesJobs(reqCtx, backup, &backupPolicy.Spec.Snapshot.BasePolicy, dataprotectionv1alpha1.PRE); err != nil { r.Recorder.Event(backup, corev1.EventTypeNormal, "CreatedPreUpdatesJob", err.Error()) } if err = r.createVolumeSnapshot(reqCtx, backup, backupPolicy.Spec.Snapshot); err != nil { @@ -371,7 +372,7 @@ func (r *BackupReconciler) doInProgressPhaseAction( } // Failure MetadataCollectionJob does not affect the backup status. - if err = r.createUpdatesJobs(reqCtx, backup, backupPolicy.Spec.Snapshot, dataprotectionv1alpha1.POST); err != nil { + if err = r.createUpdatesJobs(reqCtx, backup, &backupPolicy.Spec.Snapshot.BasePolicy, dataprotectionv1alpha1.POST); err != nil { r.Recorder.Event(backup, corev1.EventTypeNormal, "CreatedPostUpdatesJob", err.Error()) } @@ -390,6 +391,10 @@ func (r *BackupReconciler) doInProgressPhaseAction( // TODO: add error type return r.updateStatusIfFailed(reqCtx, backup, fmt.Errorf("not found the %s policy", backup.Spec.BackupType)) } + // createUpdatesJobs should not affect the backup status, just need to record events when the run fails + if err = r.createUpdatesJobs(reqCtx, backup, &commonPolicy.BasePolicy, dataprotectionv1alpha1.PRE); err != nil { + r.Recorder.Event(backup, corev1.EventTypeNormal, "CreatedPreUpdatesJob", err.Error()) + } pathPrefix := r.getBackupPathPrefix(backup.Namespace, backupPolicy.Annotations[constant.BackupDataPathPrefixAnnotationKey]) err = r.createBackupToolJob(reqCtx, backup, commonPolicy, pathPrefix) if err != nil { @@ -407,20 +412,33 @@ func (r *BackupReconciler) doInProgressPhaseAction( if err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) } + // createUpdatesJobs should not affect the backup status, just need to record events when the run fails + if err = r.createUpdatesJobs(reqCtx, backup, &commonPolicy.BasePolicy, dataprotectionv1alpha1.POST); err != nil { + r.Recorder.Event(backup, corev1.EventTypeNormal, "CreatedPostUpdatesJob", err.Error()) + } jobStatusConditions := job.Status.Conditions if jobStatusConditions[0].Type == batchv1.JobComplete { // update Phase to in Completed backup.Status.Phase = dataprotectionv1alpha1.BackupCompleted backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now().UTC()} - backup.Status.Manifests = &dataprotectionv1alpha1.ManifestsStatus{ - BackupTool: &dataprotectionv1alpha1.BackupToolManifestsStatus{ - FilePath: pathPrefix, - }, + if backup.Status.Manifests == nil { + backup.Status.Manifests = &dataprotectionv1alpha1.ManifestsStatus{} } + if backup.Status.Manifests.BackupTool == nil { + backup.Status.Manifests.BackupTool = &dataprotectionv1alpha1.BackupToolManifestsStatus{} + } + backup.Status.Manifests.BackupTool.FilePath = pathPrefix } else if jobStatusConditions[0].Type == batchv1.JobFailed { backup.Status.Phase = dataprotectionv1alpha1.BackupFailed backup.Status.FailureReason = job.Status.Conditions[0].Reason } + if backup.Spec.BackupType == dataprotectionv1alpha1.BackupTypeIncremental { + if backup.Status.Manifests != nil && + backup.Status.Manifests.BackupLog != nil && + backup.Status.Manifests.BackupLog.StartTime == nil { + backup.Status.Manifests.BackupLog.StartTime = backup.Status.Manifests.BackupLog.StopTime + } + } } // finally, update backup status @@ -643,7 +661,7 @@ func (r *BackupReconciler) ensureVolumeSnapshotReady(reqCtx intctrlutil.RequestC func (r *BackupReconciler) createUpdatesJobs(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, - snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy, + basePolicy *dataprotectionv1alpha1.BasePolicy, stage dataprotectionv1alpha1.BackupStatusUpdateStage) error { // get backup policy backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} @@ -655,11 +673,11 @@ func (r *BackupReconciler) createUpdatesJobs(reqCtx intctrlutil.RequestCtx, reqCtx.Log.V(1).Error(err, "Unable to get backupPolicy for backup.", "backupPolicy", backupPolicyNameSpaceName) return err } - for _, update := range snapshotPolicy.BackupStatusUpdates { + for _, update := range basePolicy.BackupStatusUpdates { if update.UpdateStage != stage { continue } - if err := r.createMetadataCollectionJob(reqCtx, backup, snapshotPolicy.BasePolicy, update); err != nil { + if err := r.createMetadataCollectionJob(reqCtx, backup, basePolicy, update); err != nil { return err } } @@ -668,10 +686,14 @@ func (r *BackupReconciler) createUpdatesJobs(reqCtx intctrlutil.RequestCtx, func (r *BackupReconciler) createMetadataCollectionJob(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, - basePolicy dataprotectionv1alpha1.BasePolicy, + basePolicy *dataprotectionv1alpha1.BasePolicy, updateInfo dataprotectionv1alpha1.BackupStatusUpdate) error { mgrNS := viper.GetString(constant.CfgKeyCtrlrMgrNS) - key := types.NamespacedName{Namespace: mgrNS, Name: backup.Name + "-" + strings.ToLower(updateInfo.Path)} + jobName := backup.Name + if len(backup.Name) > 30 { + jobName = backup.Name[:30] + } + key := types.NamespacedName{Namespace: mgrNS, Name: jobName + "-" + strings.ToLower(updateInfo.Path)} job := &batchv1.Job{} // check if job is created if exists, err := intctrlutil.CheckResourceExists(reqCtx.Ctx, r.Client, key, job); err != nil { @@ -1147,7 +1169,7 @@ func addTolerations(podSpec *corev1.PodSpec) (err error) { func (r *BackupReconciler) buildMetadataCollectionPodSpec( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, - basePolicy dataprotectionv1alpha1.BasePolicy, + basePolicy *dataprotectionv1alpha1.BasePolicy, updateInfo dataprotectionv1alpha1.BackupStatusUpdate) (corev1.PodSpec, error) { podSpec := corev1.PodSpec{} targetPod, err := r.getTargetPod(reqCtx, backup, basePolicy.Target.LabelsSelector.MatchLabels) diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index b28159c50..5b8e2f960 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -257,7 +257,11 @@ func (r *BackupPolicyReconciler) removeOldestBackups(reqCtx intctrlutil.RequestC } func (r *BackupPolicyReconciler) getCronJobName(backupPolicyName, backupPolicyNamespace string, backupType dataprotectionv1alpha1.BackupType) string { - return fmt.Sprintf("%s-%s-%s", backupPolicyName, backupPolicyNamespace, string(backupType)) + name := fmt.Sprintf("%s-%s", backupPolicyName, backupPolicyNamespace) + if len(name) > 30 { + name = name[:30] + } + return fmt.Sprintf("%s-%s", name, string(backupType)) } // buildCronJob builds cronjob from backup policy. @@ -287,6 +291,7 @@ func (r *BackupPolicyReconciler) buildCronJob( BackupType: string(backType), ServiceAccount: viper.GetString("KUBEBLOCKS_SERVICEACCOUNT_NAME"), MgrNamespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), + Image: viper.GetString(constant.KBToolsImage), } backupPolicyOptionsByte, err := json.Marshal(options) if err != nil { @@ -295,8 +300,11 @@ func (r *BackupPolicyReconciler) buildCronJob( if err = cueValue.Fill("options", backupPolicyOptionsByte); err != nil { return nil, err } - - cronjobByte, err := cueValue.Lookup("cronjob") + cuePath := "cronjob" + if backType == dataprotectionv1alpha1.BackupTypeIncremental { + cuePath = "cronjob_incremental" + } + cronjobByte, err := cueValue.Lookup(cuePath) if err != nil { return nil, err } @@ -420,7 +428,6 @@ func (r *BackupPolicyReconciler) handleIncrementalPolicy( reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { if backupPolicy.Spec.Incremental == nil { - // TODO delete cronjob if exists return nil } var cronExpression string diff --git a/controllers/dataprotection/cue/cronjob.cue b/controllers/dataprotection/cue/cronjob.cue index d43674418..7763b9a7c 100644 --- a/controllers/dataprotection/cue/cronjob.cue +++ b/controllers/dataprotection/cue/cronjob.cue @@ -22,6 +22,7 @@ options: { backupType: string ttl: string serviceAccount: string + image: string } cronjob: { @@ -37,7 +38,7 @@ cronjob: { } spec: { schedule: options.schedule - successfulJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 0 failedJobsHistoryLimit: 1 concurrencyPolicy: "Forbid" jobTemplate: spec: template: spec: { @@ -45,7 +46,7 @@ cronjob: { serviceAccountName: options.serviceAccount containers: [{ name: "backup-policy" - image: "appscode/kubectl:1.25" + image: options.image imagePullPolicy: "IfNotPresent" command: [ "sh", @@ -73,3 +74,54 @@ EOF } } } + +cronjob_incremental: { + apiVersion: "batch/v1" + kind: "CronJob" + metadata: { + name: options.name + namespace: options.mgrNamespace + annotations: + "kubeblocks.io/backup-namespace": options.namespace + labels: + "app.kubernetes.io/managed-by": "kubeblocks" + } + spec: { + schedule: options.schedule + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 1 + concurrencyPolicy: "Forbid" + jobTemplate: spec: template: spec: { + restartPolicy: "Never" + serviceAccountName: options.serviceAccount + containers: [{ + name: "backup-policy" + image: options.image + imagePullPolicy: "IfNotPresent" + command: [ + "sh", + "-c", + ] + args: [ + """ +kubectl apply -f - < ${RESTORE_DATA_DIR}/kb_restore.sh; - echo -e "restore_command=mv ${PITR_DIR}/pg_wal/%f %p\nrecovery_target_time=${RECOVERY_TIME}\nrecovery_target_action=promote" > ${CONF_DIR}/recovery.conf; + mkdir -p ${RESTORE_SCRIPT_DIR}; + echo "#!/bin/bash" > ${RESTORE_SCRIPT_DIR}/kb_restore.sh; + echo "[[ -d '${DATA_DIR}.old' ]] && mv -f ${DATA_DIR}.old ${DATA_DIR};" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; + echo "[[ -d '${DATA_DIR}.failed/data.old' ]] && mv -f ${DATA_DIR}.failed/data.old ${DATA_DIR};" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; + chmod +x ${RESTORE_SCRIPT_DIR}/kb_restore.sh; + echo "restore_command='mv ${PITR_DIR}/%f %p'\nrecovery_target_time='${RECOVERY_TIME}'\nrecovery_target_action='promote'" > ${CONF_DIR}/recovery.conf; mv ${DATA_DIR} ${DATA_DIR}.old; sync; + backupCommands: + - | + set -e; + EXPIRED_INCR_LOG=${REMOTE_LOG_DIR}/$(date -d"7 day ago" +%Y%m%d); + if [ -d ${EXPIRED_INCR_LOG} ]; then rm -rf ${EXPIRED_INCR_LOG}; fi + TODAY_INCR_LOG=${REMOTE_LOG_DIR}/$(date +%Y%m%d); + mkdir -p ${TODAY_INCR_LOG}; + for i in $(find ${ARCHIVE_LOG_DIR} -name "*.gz"); do + mv -f ${i} ${TODAY_INCR_LOG}/; + done + if [ -d ${LOG_DIR} ]; then + cd ${LOG_DIR}; + LATEST_LOG=$(ls -t . | grep '[[:digit:]]$\|.partial$'|head -n 1); + gzip -kqc ${LATEST_LOG} > ${TODAY_INCR_LOG}/${LATEST_LOG}.gz; + fi + type: pitr \ No newline at end of file diff --git a/deploy/postgresql/templates/backuptool.yaml b/deploy/postgresql/templates/backuptool.yaml index 90e618194..7f7f4f30a 100644 --- a/deploy/postgresql/templates/backuptool.yaml +++ b/deploy/postgresql/templates/backuptool.yaml @@ -6,7 +6,7 @@ metadata: clusterdefinition.kubeblocks.io/name: postgresql {{- include "postgresql.labels" . | nindent 4 }} spec: - image: registry.cn-hangzhou.aliyuncs.com/apecloud/postgresql:14.7.0 + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} deployKind: job resources: limits: diff --git a/deploy/postgresql/templates/configmap.yaml b/deploy/postgresql/templates/configmap.yaml index 00a089f49..880083f0f 100644 --- a/deploy/postgresql/templates/configmap.yaml +++ b/deploy/postgresql/templates/configmap.yaml @@ -20,7 +20,7 @@ data: kb_restore.conf: | method: kb_restore_from_backup kb_restore_from_backup: - command: sh /home/postgres/pgdata/kb_restore/kb_restore.sh + command: bash /home/postgres/pgdata/kb_restore/kb_restore.sh keep_existing_recovery_conf: false recovery_conf: restore_command: cp /home/postgres/pgdata/pgroot/arch/%f %p @@ -28,6 +28,6 @@ data: kb_pitr.conf: | method: kb_restore_from_time kb_restore_from_time: - command: sh /home/postgres/pgdata/kb_restore/kb_restore.sh + command: bash /home/postgres/pgdata/kb_restore/kb_restore.sh keep_existing_recovery_conf: false recovery_conf: {} \ No newline at end of file diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index c48dfe448..920d663e1 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -43,7 +43,7 @@ data: if '=' not in line: continue key, value = line.split('=', 1) - result[key.strip()] = value.strip() + result[key.strip()] = value.strip().strip("'") return result def main(filename): restore_dir = os.environ.get('RESTORE_DATA_DIR', '') @@ -55,6 +55,11 @@ data: postgresql['config_dir'] = '/home/postgres/pgdata/conf' postgresql['custom_conf'] = '/home/postgres/conf/postgresql.conf' # TODO add local postgresql.parameters + if not 'parameters' in postgresql: + postgresql['parameters'] = {} + parameters = postgresql_conf_to_dict("/home/postgres/conf/postgresql.conf") + if 'archive_command' in parameters: + postgresql['parameters']['archive_command'] = parameters['archive_command'] # add pg_hba.conf with open('/home/postgres/conf/pg_hba.conf', 'r') as f: lines = read_file_lines(f) @@ -95,8 +100,9 @@ data: #!/bin/bash set -o errexit set -o nounset - LOG_START_TIME=$(pg_waldump $(ls -tr $PGDATA/pg_wal/ | grep '[[:digit:]]$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') - LOG_STOP_TIME=$(pg_waldump $(ls -t $PGDATA/pg_wal/ | grep '[[:digit:]]$'|head -n 1) --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + SHOW_START_TIME=$1 + LOG_START_TIME=$(pg_waldump $(ls -tr $PGDATA/pg_wal/ | grep '[[:digit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + LOG_STOP_TIME=$(pg_waldump $(ls -t $PGDATA/pg_wal/ | grep '[[:digit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') LOG_START_TIME=$(date -d "$LOG_START_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') LOG_STOP_TIME=$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') - printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}" \ No newline at end of file + [[ $SHOW_START_TIME == "false" ]] && printf "{\"stopTime\": \"$LOG_STOP_TIME\"}" || printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}" \ No newline at end of file diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 8033d1255..400a74b4c 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -546,10 +546,6 @@ func (o *CreateRestoreOptions) validateRestoreTime() error { if err = runtime.DefaultUnstructuredConverter.FromUnstructured(i.Object, &obj); err != nil { return err } - if obj.Status.Phase != dataprotectionv1alpha1.BackupCompleted || - obj.Status.Manifests == nil || obj.Status.Manifests.BackupLog == nil { - continue - } backups = append(backups, obj) } recoverableTime := dataprotectionv1alpha1.GetRecoverableTimeRange(backups) diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index 671ea6f2d..36d68d99e 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -45,6 +45,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/constant" + testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) var _ = Describe("DataProtection", func() { @@ -271,37 +272,30 @@ var _ = Describe("DataProtection", func() { } cluster.SetLabels(clusterDefLabel) backupPolicy := testing.FakeBackupPolicy("backPolicy", cluster.Name) - backup := testing.FakeBackup("backup-base") + backupTypeMeta := testing.FakeBackup("backup-none").TypeMeta + backupLabels := map[string]string{ + constant.AppInstanceLabelKey: clusterName, + constant.KBAppComponentLabelKey: "test", + } + now := metav1.Now() + baseBackup := testapps.NewBackupFactory(testing.Namespace, "backup-base"). + SetBackupType(dataprotectionv1alpha1.BackupTypeSnapshot). + SetBackLog(now.Add(-time.Minute), now.Add(-time.Second)). + SetLabels(backupLabels).GetObject() + baseBackup.TypeMeta = backupTypeMeta + baseBackup.Status.Phase = dataprotectionv1alpha1.BackupCompleted + incrBackup := testapps.NewBackupFactory(testing.Namespace, backupName). + SetBackupType(dataprotectionv1alpha1.BackupTypeIncremental). + SetBackLog(now.Add(-time.Minute), now.Add(time.Minute)). + SetLabels(backupLabels).GetObject() + incrBackup.TypeMeta = backupTypeMeta pods := testing.FakePods(1, testing.Namespace, clusterName) tf.FakeDynamicClient = fake.NewSimpleDynamicClient( - scheme.Scheme, &secrets.Items[0], &pods.Items[0], cluster, backupPolicy, backup) + scheme.Scheme, &secrets.Items[0], &pods.Items[0], cluster, backupPolicy, baseBackup, incrBackup) tf.Client = &clientfake.RESTClient{} - // create backup - cmd := NewCreateBackupCmd(tf, streams) - Expect(cmd).ShouldNot(BeNil()) - _ = cmd.Flags().Set("backup-type", "snapshot") - _ = cmd.Flags().Set("backup-name", backupName) - cmd.Run(nil, []string{clusterName}) By("restore new cluster from source cluster which is not deleted") - // mock backup is ok - now := metav1.Now() - baseManifests := map[string]any{ - "backupLog": map[string]any{ - "startTime": now.Add(-time.Minute).Format(time.RFC3339), - "stopTime": now.Add(-time.Second).Format(time.RFC3339), - }, - } - mockBackupInfo(tf.FakeDynamicClient, backup.Name, clusterName, baseManifests) - - manifests := map[string]any{ - "backupLog": map[string]any{ - "startTime": now.Add(-time.Minute).Format(time.RFC3339), - "stopTime": now.Add(time.Minute).Format(time.RFC3339), - }, - } - mockBackupInfo(tf.FakeDynamicClient, backupName, clusterName, manifests) cmdRestore := NewCreateRestoreCmd(tf, streams) Expect(cmdRestore != nil).To(BeTrue()) _ = cmdRestore.Flags().Set("restore-to-time", util.TimeFormatWithDuration(&now, time.Second)) diff --git a/internal/constant/const.go b/internal/constant/const.go index c1cb636bb..c81d3de56 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -80,6 +80,7 @@ const ( KBManagedByKey = "apps.kubeblocks.io/managed-by" // KBManagedByKey marks resources that auto created during operation ClassProviderLabelKey = "class.kubeblocks.io/provider" BackupToolTypeLabelKey = "kubeblocks.io/backup-tool-type" + BackupTypeLabelKeyKey = "dataprotection.kubeblocks.io/backup-type" // kubeblocks.io annotations OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index 98a146908..73c9ad41f 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -139,15 +139,15 @@ func (r *backupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti } return target } - if backupPolicy.Spec.Snapshot != nil { + if backupPolicy.Spec.Snapshot != nil && policyTPL.Snapshot != nil { backupPolicy.Spec.Snapshot.Target = syncTheRoleLabel(backupPolicy.Spec.Snapshot.Target, policyTPL.Snapshot.BasePolicy) } - if backupPolicy.Spec.Full != nil { + if backupPolicy.Spec.Full != nil && policyTPL.Full != nil { backupPolicy.Spec.Full.Target = syncTheRoleLabel(backupPolicy.Spec.Full.Target, policyTPL.Full.BasePolicy) } - if backupPolicy.Spec.Incremental != nil { + if backupPolicy.Spec.Incremental != nil && policyTPL.Incremental != nil { backupPolicy.Spec.Incremental.Target = syncTheRoleLabel(backupPolicy.Spec.Incremental.Target, policyTPL.Incremental.BasePolicy) } diff --git a/internal/controller/plan/pitr.go b/internal/controller/plan/pitr.go index fbbeb69d4..deaeddfd9 100644 --- a/internal/controller/plan/pitr.go +++ b/internal/controller/plan/pitr.go @@ -62,6 +62,7 @@ type PointInTimeRecoveryManager struct { const ( initContainerName = "pitr-for-pause" + backupVolumePATH = "/backupdata" ) // DoPITRPrepare prepares init container and pvc before point in time recovery @@ -173,13 +174,19 @@ func (p *PointInTimeRecoveryManager) doPrepare(component *component.SynthesizedC if err != nil { return err } + if latestBackup.Spec.BackupType == dpv1alpha1.BackupTypeSnapshot { + return p.doPrepareSnapshotBackup(component, latestBackup) + } + return nil +} +func (p *PointInTimeRecoveryManager) doPrepareSnapshotBackup(component *component.SynthesizedComponent, backup *dpv1alpha1.Backup) error { vct := component.VolumeClaimTemplates[0] snapshotAPIGroup := snapshotv1.GroupName vct.Spec.DataSource = &corev1.TypedLocalObjectReference{ APIGroup: &snapshotAPIGroup, Kind: constant.VolumeSnapshotKind, - Name: latestBackup.Name, + Name: backup.Name, } component.VolumeClaimTemplates[0] = vct return nil @@ -231,7 +238,7 @@ func (p *PointInTimeRecoveryManager) getSortedBackups(reverse bool) ([]dpv1alpha // getLatestBaseBackup gets the latest baseBackup func (p *PointInTimeRecoveryManager) getLatestBaseBackup() (*dpv1alpha1.Backup, error) { - // 1. sort backups by completed timestamp + // 1. sort reverse backups backups, err := p.getSortedBackups(true) if err != nil { return nil, err @@ -253,28 +260,6 @@ func (p *PointInTimeRecoveryManager) getLatestBaseBackup() (*dpv1alpha1.Backup, return latestBackup, nil } -func (p *PointInTimeRecoveryManager) getNextBackup() (*dpv1alpha1.Backup, error) { - // 1. sort backups by reverse completed timestamp - backups, err := p.getSortedBackups(false) - if err != nil { - return nil, err - } - - // 2. get the next earliest backup object - var nextBackup *dpv1alpha1.Backup - for _, item := range backups { - if p.restoreTime.Before(item.Status.Manifests.BackupLog.StopTime) { - nextBackup = &item - break - } - } - if nextBackup == nil { - return nil, errors.New("can not found next earliest base backup") - } - - return nextBackup, nil -} - // checkAndInit checks if cluster need to be restored, return value: true: need, false: no need func (p *PointInTimeRecoveryManager) checkAndInit() (need bool, err error) { // check args if pitr supported @@ -306,27 +291,19 @@ func (p *PointInTimeRecoveryManager) checkAndInit() (need bool, err error) { return true, nil } -func getVolumeMount(spec *dpv1alpha1.BackupToolSpec) (string, string) { +func getVolumeMount(spec *dpv1alpha1.BackupToolSpec) string { dataVolumeMount := "/data" - logVolumeMount := "/log" - tag := 0 // TODO: hack it because the mount path is not explicitly specified in cluster definition for _, env := range spec.Env { if env.Name == "VOLUME_DATA_DIR" { dataVolumeMount = env.Value - tag++ - } else if env.Name == "VOLUME_LOG_DIR" { - logVolumeMount = env.Value - tag++ - } - if tag >= 2 { break } } - return dataVolumeMount, logVolumeMount + return dataVolumeMount } -func (p *PointInTimeRecoveryManager) getRecoveryInfo() (*dpv1alpha1.BackupToolSpec, error) { +func (p *PointInTimeRecoveryManager) getRecoveryInfo(componentName string) (*dpv1alpha1.BackupToolSpec, error) { // get scripts from backup template toolList := dpv1alpha1.BackupToolList{} // TODO: The reference PITR backup tool needs a stronger reference relationship, for now use label references @@ -341,6 +318,10 @@ func (p *PointInTimeRecoveryManager) getRecoveryInfo() (*dpv1alpha1.BackupToolSp if len(toolList.Items) == 0 { return nil, errors.New("not support recovery because of non-existed pitr backupTool") } + incrementalBackup, err := p.getIncrementalBackup(componentName) + if err != nil { + return nil, err + } spec := &toolList.Items[0].Spec timeFormat := time.RFC3339 envTimeEnvIdx := -1 @@ -354,16 +335,52 @@ func (p *PointInTimeRecoveryManager) getRecoveryInfo() (*dpv1alpha1.BackupToolSp if envTimeEnvIdx != -1 { spec.Env[envTimeEnvIdx].Value = p.restoreTime.Time.UTC().Format(timeFormat) } - + backupDIR := incrementalBackup.Name + if incrementalBackup.Status.Manifests != nil && incrementalBackup.Status.Manifests.BackupTool != nil { + backupDIR = incrementalBackup.Status.Manifests.BackupTool.FilePath + } + headEnv := []corev1.EnvVar{ + {Name: "BACKUP_DIR", Value: backupVolumePATH + "/" + backupDIR}, + {Name: "BACKUP_NAME", Value: incrementalBackup.Name}} + spec.Env = append(headEnv, spec.Env...) return spec, nil } -func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, err error) { - objs = make([]client.Object, 0) - recoveryInfo, err := p.getRecoveryInfo() +func (p *PointInTimeRecoveryManager) getIncrementalBackup(componentName string) (*dpv1alpha1.Backup, error) { + incrementalBackupList := dpv1alpha1.BackupList{} + if err := p.Client.List(p.Ctx, &incrementalBackupList, + client.MatchingLabels{ + constant.AppInstanceLabelKey: p.sourceCluster, + constant.KBAppComponentLabelKey: componentName, + constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeIncremental), + }); err != nil { + return nil, err + } + if len(incrementalBackupList.Items) == 0 { + return nil, errors.New("not found incremental backups") + } + return &incrementalBackupList.Items[0], nil +} + +func (p *PointInTimeRecoveryManager) getIncrementalPVC(componentName string) (*corev1.PersistentVolumeClaim, error) { + incrementalBackup, err := p.getIncrementalBackup(componentName) if err != nil { - return objs, err + return nil, err + } + pvcKey := types.NamespacedName{ + Name: incrementalBackup.Status.PersistentVolumeClaimName, + Namespace: incrementalBackup.Namespace, } + pvc := corev1.PersistentVolumeClaim{} + if err := p.Client.Get(p.Ctx, pvcKey, &pvc); err != nil { + return nil, err + } + return &pvc, nil +} + +func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, err error) { + objs = make([]client.Object, 0) + for _, componentSpec := range p.Cluster.Spec.ComponentSpecs { if len(componentSpec.VolumeClaimTemplates) == 0 { continue @@ -374,12 +391,6 @@ func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, constant.AppInstanceLabelKey: p.Cluster.Name, constant.KBAppComponentLabelKey: componentSpec.Name, } - sts := &appsv1.StatefulSet{} - sts.SetLabels(commonLabels) - vct := corev1.PersistentVolumeClaimTemplate{} - vct.Name = componentSpec.VolumeClaimTemplates[0].Name - vct.Spec = componentSpec.VolumeClaimTemplates[0].Spec.ToV1PersistentVolumeClaimSpec() - // get data dir pvc name dataPVCList := corev1.PersistentVolumeClaimList{} dataPVCLabels := map[string]string{ @@ -395,34 +406,28 @@ func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, if len(dataPVCList.Items) == 0 { return objs, errors.New("not found data pvc") } + recoveryInfo, err := p.getRecoveryInfo(componentSpec.Name) + if err != nil { + return objs, err + } + incrementalPVC, err := p.getIncrementalPVC(componentSpec.Name) + if err != nil { + return objs, err + } + dataVolumeMount := getVolumeMount(recoveryInfo) for i, dataPVC := range dataPVCList.Items { if dataPVC.Status.Phase != corev1.ClaimBound { return objs, errors.New("waiting PVC Bound") } - - nextBackup, err := p.getNextBackup() - if err != nil { - return objs, err - } - pitrPVCName := fmt.Sprintf("pitr-%s-%s-%d", p.Cluster.Name, componentSpec.Name, i) - pitrPVCKey := types.NamespacedName{ - Namespace: p.namespace, - Name: pitrPVCName, - } - pitrPVC, err := builder.BuildPVCFromSnapshot(sts, vct, pitrPVCKey, nextBackup.Name, nil) - if err != nil { - return objs, err - } volumes := []corev1.Volume{ {Name: "data", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: dataPVC.Name}}}, {Name: "log", VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pitrPVCName}}}, + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: incrementalPVC.Name}}}, } - dataVolumeMount, logVolumeMount := getVolumeMount(recoveryInfo) volumeMounts := []corev1.VolumeMount{ {Name: "data", MountPath: dataVolumeMount}, - {Name: "log", MountPath: logVolumeMount}, + {Name: "log", MountPath: backupVolumePATH}, } // render the job cue template @@ -449,7 +454,6 @@ func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, objs = append(objs, logicJob) } // collect pvcs and jobs for later deletion - objs = append(objs, pitrPVC) objs = append(objs, job) } } diff --git a/internal/controller/plan/pitr_test.go b/internal/controller/plan/pitr_test.go index d67d4e8ce..c8ff773a0 100644 --- a/internal/controller/plan/pitr_test.go +++ b/internal/controller/plan/pitr_test.go @@ -152,52 +152,67 @@ var _ = Describe("PITR Functions", func() { VolumeClaimTemplates: cluster.Spec.ComponentSpecs[0].ToVolumeClaimTemplates(), } - By("By creating earlier backup: ") + By("By creating base backup: ") now := metav1.Now() backupLabels := map[string]string{ - constant.AppInstanceLabelKey: sourceCluster, + constant.AppInstanceLabelKey: sourceCluster, + constant.KBAppComponentLabelKey: mysqlCompName, + constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeSnapshot), } backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). WithRandomName().SetLabels(backupLabels). SetBackupPolicyName("test-fake"). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeSnapshot). Create(&testCtx).GetObject() - earlierStartTime := &metav1.Time{Time: now.Add(-time.Hour * 3)} - earlierStopTime := &metav1.Time{Time: now.Add(-time.Hour * 2)} + baseStartTime := &metav1.Time{Time: now.Add(-time.Hour * 3)} + baseStopTime := &metav1.Time{Time: now.Add(-time.Hour * 2)} backupStatus := dpv1alpha1.BackupStatus{ Phase: dpv1alpha1.BackupCompleted, - StartTimestamp: earlierStartTime, - CompletionTimestamp: earlierStopTime, + StartTimestamp: baseStartTime, + CompletionTimestamp: baseStopTime, Manifests: &dpv1alpha1.ManifestsStatus{ BackupLog: &dpv1alpha1.BackupLogStatus{ - StartTime: earlierStartTime, - StopTime: earlierStopTime, + StartTime: baseStartTime, + StopTime: baseStopTime, }, }, } backupStatus.CompletionTimestamp = &metav1.Time{Time: now.Add(-time.Hour * 2)} patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backup)) - By("By creating latest backup: ") - latestStartTime := &metav1.Time{Time: now.Add(-time.Hour * 3)} - latestStopTime := &metav1.Time{Time: now.Add(time.Hour * 2)} - backupNext := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). - WithRandomName().SetLabels(backupLabels). + By("By creating remote pvc: ") + remotePVC := testapps.NewPersistentVolumeClaimFactory( + testCtx.DefaultNamespace, "remote-pvc", clusterName, mysqlCompName, "log"). + SetStorage("1Gi"). + Create(&testCtx).GetObject() + + By("By creating incremental backup: ") + incrBackupLabels := map[string]string{ + constant.AppInstanceLabelKey: sourceCluster, + constant.KBAppComponentLabelKey: mysqlCompName, + constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeIncremental), + } + incrStartTime := &metav1.Time{Time: now.Add(-time.Hour * 3)} + incrStopTime := &metav1.Time{Time: now.Add(time.Hour * 2)} + backupIncr := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). + WithRandomName().SetLabels(incrBackupLabels). SetBackupPolicyName("test-fake"). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeIncremental). Create(&testCtx).GetObject() backupStatus = dpv1alpha1.BackupStatus{ - Phase: dpv1alpha1.BackupCompleted, - StartTimestamp: latestStartTime, - CompletionTimestamp: latestStopTime, + Phase: dpv1alpha1.BackupCompleted, + StartTimestamp: incrStartTime, + CompletionTimestamp: incrStopTime, + PersistentVolumeClaimName: remotePVC.Name, Manifests: &dpv1alpha1.ManifestsStatus{ BackupLog: &dpv1alpha1.BackupLogStatus{ - StartTime: latestStartTime, - StopTime: latestStopTime, + StartTime: incrStartTime, + StopTime: incrStopTime, }, }, } - patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupNext)) + patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupIncr)) + }) It("Test PITR prepare", func() { diff --git a/internal/testutil/apps/backup_factory.go b/internal/testutil/apps/backup_factory.go index bd3e260dc..9b2bd55a5 100644 --- a/internal/testutil/apps/backup_factory.go +++ b/internal/testutil/apps/backup_factory.go @@ -17,6 +17,10 @@ limitations under the License. package apps import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" ) @@ -47,3 +51,17 @@ func (factory *MockBackupFactory) SetLabels(labels map[string]string) *MockBacku factory.get().SetLabels(labels) return factory } + +func (factory *MockBackupFactory) SetBackLog(startTime, stopTime time.Time) *MockBackupFactory { + manitests := factory.get().Status.Manifests + if manitests == nil { + manitests = &dataprotectionv1alpha1.ManifestsStatus{} + } + if manitests.BackupLog == nil { + manitests.BackupLog = &dataprotectionv1alpha1.BackupLogStatus{} + } + manitests.BackupLog.StartTime = &metav1.Time{Time: startTime} + manitests.BackupLog.StopTime = &metav1.Time{Time: stopTime} + factory.get().Status.Manifests = manitests + return factory +} From 6d34306347a0990a1a1da9940ed9f2bf401bc31d Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Thu, 20 Apr 2023 16:25:14 +0800 Subject: [PATCH 108/439] fix: add flag for playground destroy to skip delete cluster and uninstall kubeblocks (#2765) Co-authored-by: ldming --- docs/user_docs/cli/kbcli_playground_destroy.md | 3 ++- internal/cli/cmd/playground/destroy.go | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/user_docs/cli/kbcli_playground_destroy.md b/docs/user_docs/cli/kbcli_playground_destroy.md index 923e4dbdc..5cb02a067 100644 --- a/docs/user_docs/cli/kbcli_playground_destroy.md +++ b/docs/user_docs/cli/kbcli_playground_destroy.md @@ -18,7 +18,8 @@ kbcli playground destroy [flags] ### Options ``` - -h, --help help for destroy + -h, --help help for destroy + --purge Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks. (default true) ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/playground/destroy.go b/internal/cli/cmd/playground/destroy.go index 72870977e..371ef4cad 100644 --- a/internal/cli/cmd/playground/destroy.go +++ b/internal/cli/cmd/playground/destroy.go @@ -55,6 +55,10 @@ var ( type destroyOptions struct { genericclioptions.IOStreams baseOptions + + // purge resources, before destroy kubernetes cluster we should delete cluster and + // uninstall KubeBlocks + purge bool } func newDestroyCmd(streams genericclioptions.IOStreams) *cobra.Command { @@ -70,6 +74,9 @@ func newDestroyCmd(streams genericclioptions.IOStreams) *cobra.Command { util.CheckErr(o.destroy()) }, } + + cmd.Flags().BoolVar(&o.purge, "purge", true, "Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks.") + return cmd } @@ -121,10 +128,8 @@ func (o *destroyOptions) destroyLocal() error { func (o *destroyOptions) destroyCloud() error { var err error - // start to destroy cluster - printer.Warning(o.Out, `This action will uninstall KubeBlocks and delete the kubernetes cluster, - there may be residual resources, please confirm and manually clean up related - resources after this action. + printer.Warning(o.Out, `This action will destroy the kubernetes cluster, there may be residual resources, + please confirm and manually clean up related resources after this action. `) @@ -187,6 +192,11 @@ func (o *destroyOptions) destroyCloud() error { func (o *destroyOptions) deleteClustersAndUninstallKB() error { var err error + if !o.purge { + klog.V(1).Infof("Skip to delete all clusters created by KubeBlocks and uninstall KubeBlocks") + return nil + } + if o.prevCluster.KubeConfig == "" { fmt.Fprintf(o.Out, "No kubeconfig found for kubernetes cluster %s in %s \n", o.prevCluster.ClusterName, o.stateFilePath) From b317422dd374fd1946843bcbafa3a4cdfdd3c36f Mon Sep 17 00:00:00 2001 From: wangyelei Date: Thu, 20 Apr 2023 17:22:35 +0800 Subject: [PATCH 109/439] chore: fix restore pg failed (#2792) --- deploy/postgresql/templates/backuptool.yaml | 2 +- deploy/postgresql/templates/scripts.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/deploy/postgresql/templates/backuptool.yaml b/deploy/postgresql/templates/backuptool.yaml index 7f7f4f30a..795e3ec26 100644 --- a/deploy/postgresql/templates/backuptool.yaml +++ b/deploy/postgresql/templates/backuptool.yaml @@ -36,7 +36,7 @@ spec: echo "mkdir -p ${DATA_DIR}/../arch" >> kb_restore.sh echo "mv -f ${TMP_DATA_DIR}/* ${DATA_DIR}/" >> kb_restore.sh echo "mv -f ${TMP_ARCH_DATA_DIR}/* ${DATA_DIR}/../arch" >> kb_restore.sh - echo "rm -rf ${RESTORE_DATA_DIR}" >> kb_restore.sh + echo "rm -rf ${RESTORE_DATA_DIR}/*" >> kb_restore.sh # extract the data file to the temporary data directory mkdir -p ${TMP_DATA_DIR} && mkdir -p ${TMP_ARCH_DATA_DIR} diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index 920d663e1..a1b930a88 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -92,6 +92,9 @@ data: sleep 5 done fi + if [ -f ${RESTORE_DATA_DIR}/kb_restore.signal ]; then + chown -R postgres ${RESTORE_DATA_DIR} + fi python3 /kb-scripts/generate_patroni_yaml.py tmp_patroni.yaml export SPILO_CONFIGURATION=$(cat tmp_patroni.yaml) # export SCOPE="$KB_CLUSTER_NAME-$KB_CLUSTER_NAME" From ac077b9a31d27664f818fd4c102a43c3eb0b3c16 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Thu, 20 Apr 2023 17:27:18 +0800 Subject: [PATCH 110/439] fix: sqlchannel-parse-unsupportd-binding-response (#2790) --- internal/sqlchannel/client.go | 34 ++++++++++++++++++++++--- internal/sqlchannel/client_test.go | 41 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/internal/sqlchannel/client.go b/internal/sqlchannel/client.go index 1340c6191..b6514b775 100644 --- a/internal/sqlchannel/client.go +++ b/internal/sqlchannel/client.go @@ -196,6 +196,7 @@ type OperationHTTPClient struct { httpRequestPrefix string RequestTimeout time.Duration containerName string + characterType string } // NewHTTPClientWithChannelPod create a new OperationHTTPClient with sqlchannel container @@ -225,6 +226,7 @@ func NewHTTPClientWithChannelPod(pod *corev1.Pod, characterType string) (*Operat httpRequestPrefix: fmt.Sprintf(HTTPRequestPrefx, port, characterType), RequestTimeout: 10 * time.Second, containerName: container, + characterType: characterType, } return client, nil } @@ -232,10 +234,10 @@ func NewHTTPClientWithChannelPod(pod *corev1.Pod, characterType string) (*Operat // SendRequest exec sql operation, this is a blocking operation and it will use pod EXEC subresource to send an http request to the probe pod func (cli *OperationHTTPClient) SendRequest(exec *exec.ExecOptions, request SQLChannelRequest) (SQLChannelResponse, error) { var ( - response = SQLChannelResponse{} strBuffer bytes.Buffer errBuffer bytes.Buffer err error + response = SQLChannelResponse{} ) if jsonData, err := json.Marshal(request); err != nil { @@ -249,9 +251,35 @@ func (cli *OperationHTTPClient) SendRequest(exec *exec.ExecOptions, request SQLC if err = exec.RunWithRedirect(&strBuffer, &errBuffer); err != nil { return response, err } + return parseResponse(strBuffer.Bytes(), request.Operation, cli.characterType) +} - if err = json.Unmarshal(strBuffer.Bytes(), &response); err != nil { +type errorResponse struct { + ErrorCode string `json:"errorCode"` + Message string `json:"message"` +} + +func parseResponse(data []byte, operation string, charType string) (SQLChannelResponse, error) { + // conver to errorResponse first, and check error code + // if error code is not empty, it means the request failed + errorResponse := errorResponse{} + response := SQLChannelResponse{} + if err := json.Unmarshal(data, &errorResponse); err != nil { return response, err + } else if len(errorResponse.ErrorCode) > 0 { + return SQLChannelResponse{ + Event: RespEveFail, + Message: fmt.Sprintf("Operation `%s` on component of type `%s` is not supported yet.", operation, charType), + Metadata: SQLChannelMeta{ + Operation: operation, + StartTime: time.Now(), + EndTime: time.Now(), + Extra: errorResponse.Message, + }, + }, nil } - return response, nil + + // conver it to SQLChannelResponse + err := json.Unmarshal(data, &response) + return response, err } diff --git a/internal/sqlchannel/client_test.go b/internal/sqlchannel/client_test.go index 9032379fa..4592f8ce1 100644 --- a/internal/sqlchannel/client_test.go +++ b/internal/sqlchannel/client_test.go @@ -28,6 +28,7 @@ import ( dapr "github.com/dapr/go-sdk/client" pb "github.com/dapr/go-sdk/dapr/proto/runtime/v1" + "github.com/stretchr/testify/assert" "google.golang.org/grpc" corev1 "k8s.io/api/core/v1" @@ -226,6 +227,46 @@ func TestSystemAccounts(t *testing.T) { }) } +func TestParseSqlChannelResult(t *testing.T) { + t.Run("Binding Not Supported", func(t *testing.T) { + result := ` + {"errorCode":"ERR_INVOKE_OUTPUT_BINDING","message":"error when invoke output binding mongodb: binding mongodb does not support operation listUsers. supported operations:checkRunning checkRole getRole"} + ` + sqlResposne, err := parseResponse(([]byte)(result), "listUsers", "mongodb") + assert.Nil(t, err) + assert.Equal(t, sqlResposne.Event, RespEveFail) + assert.Contains(t, sqlResposne.Message, "not supported") + }) + + t.Run("Binding Exec Failed", func(t *testing.T) { + result := ` + {"event":"Failed","message":"db not ready"} + ` + sqlResposne, err := parseResponse(([]byte)(result), "listUsers", "mongodb") + assert.Nil(t, err) + assert.Equal(t, sqlResposne.Event, RespEveFail) + assert.Contains(t, sqlResposne.Message, "db not ready") + }) + + t.Run("Binding Exec Success", func(t *testing.T) { + result := ` + {"event":"Success","message":"[]"} + ` + sqlResposne, err := parseResponse(([]byte)(result), "listUsers", "mongodb") + assert.Nil(t, err) + assert.Equal(t, sqlResposne.Event, RespEveSucc) + }) + + t.Run("Invalid Resonse Format", func(t *testing.T) { + // msg cannot be parsed to json + result := ` + {"event":"Success","message":"[] + ` + _, err := parseResponse(([]byte)(result), "listUsers", "mongodb") + assert.NotNil(t, err) + }) +} + func newTCPServer(t *testing.T, daprServer pb.DaprServer, port int) (int, func()) { var l net.Listener for i := 0; i < 3; i++ { From 580564bb468115e477d2a844c7be7386ecadad32 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Thu, 20 Apr 2023 18:28:30 +0800 Subject: [PATCH 111/439] chore: disable hook when uninstall KubeBlocks (#2795) --- internal/cli/cmd/kubeblocks/uninstall.go | 4 ++++ internal/cli/util/helm/helm.go | 2 ++ 2 files changed, 6 insertions(+) diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index dbcc8360b..d52a89d5c 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -178,6 +178,10 @@ func (o *UninstallOptions) Uninstall() error { chart := helm.InstallOpts{ Name: types.KubeBlocksChartName, Namespace: o.Namespace, + + // KubeBlocks chart has a hook to delete addons, but we have already deleted addons, + // and that webhook may fail, so we need to disable hooks. + DisableHooks: true, } printSpinner(newSpinner("Uninstall helm release "+types.KubeBlocksReleaseName+" "+v.KubeBlocks), chart.Uninstall(o.HelmCfg)) diff --git a/internal/cli/util/helm/helm.go b/internal/cli/util/helm/helm.go index a67775d6f..62470eb7f 100644 --- a/internal/cli/util/helm/helm.go +++ b/internal/cli/util/helm/helm.go @@ -66,6 +66,7 @@ type InstallOpts struct { ValueOpts *values.Options Timeout time.Duration Atomic bool + DisableHooks bool } type Option func(*cli.EnvSettings) @@ -282,6 +283,7 @@ func (i *InstallOpts) tryUninstall(cfg *action.Configuration) error { client := action.NewUninstall(cfg) client.Wait = i.Wait client.Timeout = defaultTimeout + client.DisableHooks = i.DisableHooks // Create context and prepare the handle of SIGTERM ctx := context.Background() From e83668d54f1540008ca176dc2e429c2c71cf57a5 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Thu, 20 Apr 2023 19:33:11 +0800 Subject: [PATCH 112/439] fix: incremental backup update failed (#2802) --- controllers/dataprotection/cue/cronjob.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/dataprotection/cue/cronjob.cue b/controllers/dataprotection/cue/cronjob.cue index 7763b9a7c..3545b1f5b 100644 --- a/controllers/dataprotection/cue/cronjob.cue +++ b/controllers/dataprotection/cue/cronjob.cue @@ -118,7 +118,7 @@ spec: backupPolicyName: \(options.backupPolicyName) backupType: \(options.backupType) EOF -kubectl patch backup/\(options.name) --subresource=status --type=merge --patch '{"status": {"phase": "New"}}'; +kubectl -n \(options.namespace) patch backup/\(options.name) --subresource=status --type=merge --patch '{"status": {"phase": "New"}}'; """, ] }] From 399b42999f940cbd0580cf63aba904c9a0c3328d Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Thu, 20 Apr 2023 19:40:14 +0800 Subject: [PATCH 113/439] chore: add .github/utils/ releasing scripts (#2801) --- .github/utils/create-releasing-pr.sh | 30 +++++++++++++++++++++ .github/utils/functions.bash | 3 ++- .github/utils/gh_env | 9 ++++++- .github/utils/merge-releasing-pr.sh | 33 ++++++++++++++++++++++++ .github/workflows/pull-request-check.yml | 2 +- 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100755 .github/utils/create-releasing-pr.sh create mode 100755 .github/utils/merge-releasing-pr.sh diff --git a/.github/utils/create-releasing-pr.sh b/.github/utils/create-releasing-pr.sh new file mode 100755 index 000000000..2faf3073f --- /dev/null +++ b/.github/utils/create-releasing-pr.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. + +. ./gh_env +. ./functions.bash + +echo "Creating ${PR_TITLE}" + +result=$(gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${OWNER}/${REPO}/pulls \ + -f title="${PR_TITLE}" \ + -f head="${HEAD_BRANCH}" \ + -f base="${BASE_BRANCH}" 1> /dev/null) + +if [ "$?" != "0" ]; then + echo "error: ${result}" + exit 1 +else + echo "PR created" +fi + + diff --git a/.github/utils/functions.bash b/.github/utils/functions.bash index 45c84e83f..4993917db 100644 --- a/.github/utils/functions.bash +++ b/.github/utils/functions.bash @@ -109,4 +109,5 @@ function join_by { if shift 2; then printf %s "$f" "${@/#/$d}" fi -} \ No newline at end of file +} + diff --git a/.github/utils/gh_env b/.github/utils/gh_env index 0c7370c9c..bbf6c6636 100644 --- a/.github/utils/gh_env +++ b/.github/utils/gh_env @@ -1,5 +1,12 @@ DEBUG=${DEBUG:-} +export MILESTONE_ID=${MILESTONE_ID:-5} + +export HEAD_BRANCH=${HEAD_BRANCH:-'release-0.5'} +export BASE_BRANCH=${BASE_BRANCH:-'releasing-0.5'} +export PR_TITLE=${PR_TITLE:-"${BASE_BRANCH} tracker PR (no-need-to-review)"} + + export REMOTE_URL=$(git config --get remote.origin.url) export OWNER=$(dirname ${REMOTE_URL} | awk -F ":" '{print $2}') prefix='^//' @@ -7,7 +14,7 @@ if [[ $OWNER =~ $prefix ]]; then export OWNER="${OWNER#*//github.com/}" fi export REPO=$(basename -s .git ${REMOTE_URL}) -export MILESTONE_ID=${MILESTONE_ID:-5} + if [ -n "$DEBUG" ]; then echo "OWNER=${OWNER}" diff --git a/.github/utils/merge-releasing-pr.sh b/.github/utils/merge-releasing-pr.sh new file mode 100755 index 000000000..31720487d --- /dev/null +++ b/.github/utils/merge-releasing-pr.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. + +. ./gh_env +. ./functions.bash + +echo "Merging ${PR_TITLE}" + + +pr_info=$(gh pr list --repo ${OWNER}/${REPO} --base ${BASE_BRANCH} --json "number,url,mergeStateStatus,mergeable" ) +pr_number=$(echo ${pr_info} | jq -r '.[0].number') +pr_merge_status=$(echo ${pr_info} | jq -r '.[0].mergeStateStatus') +pr_mergeable=$(echo ${pr_info} | jq -r '.[0].mergeable') + +echo "pr_number=${pr_number}" +echo "pr_merge_status=${pr_merge_status}" +echo "pr_mergeable=${pr_mergeable}" + +if [ "${pr_merge_status}" == "CLEAN" ] && [ "${pr_mergeable}" == "MERGEABLE" ]; then + gh pr --repo apecloud/kubeblocks merge ${pr_number} --merge +# # gh API ref: https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#merge-a-pull-request +# gh api \ +# --method PUT \ +# --header "Accept: application/vnd.github+json" \ +# --header "X-GitHub-Api-Version: 2022-11-28" \ +# /repos/${OWNER}/${REPO}/pulls/${pr_number}/merge \ +# -f merge_method=merge +fi \ No newline at end of file diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml index e67785dc0..a5fec0123 100644 --- a/.github/workflows/pull-request-check.yml +++ b/.github/workflows/pull-request-check.yml @@ -16,7 +16,7 @@ jobs: uses: apecloud/check-branch-name@v0.1.0 with: branch_pattern: 'feature/|bugfix/|release/|releasing/|hotfix/|support/|dependabot/' - comment_for_invalid_branch_name: 'This branch name is not following the standards: feature/|bugfix/|release/|hotfix/|support/|dependabot/' + comment_for_invalid_branch_name: 'This branch name is not following the standards: feature/|bugfix/|release/|releasing/|hotfix/|support/|dependabot/' fail_if_invalid_branch_name: 'true' ignore_branch_pattern: 'main|master' From ac2dfbff29d799513f751a6a18a53d646621e80e Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Thu, 20 Apr 2023 20:18:05 +0800 Subject: [PATCH 114/439] chore: add ci test to release-version.yml (#2807) Co-authored-by: Nash Tsai --- .github/workflows/pull-request-check.yml | 4 ++-- .github/workflows/release-version.yml | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml index a5fec0123..1c2e17506 100644 --- a/.github/workflows/pull-request-check.yml +++ b/.github/workflows/pull-request-check.yml @@ -15,8 +15,8 @@ jobs: - name: check branch name uses: apecloud/check-branch-name@v0.1.0 with: - branch_pattern: 'feature/|bugfix/|release/|releasing/|hotfix/|support/|dependabot/' - comment_for_invalid_branch_name: 'This branch name is not following the standards: feature/|bugfix/|release/|releasing/|hotfix/|support/|dependabot/' + branch_pattern: 'feature/|bugfix/|release/|hotfix/|support/|release/|release-|releasing/|releasing-|dependabot/' + comment_for_invalid_branch_name: 'This branch name is not following the standards: feature/|bugfix/|hotfix/|support/|release/|release-|releasing/|releasing-|dependabot/' fail_if_invalid_branch_name: 'true' ignore_branch_pattern: 'main|master' diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml index f08b4db5e..a2e44ee71 100644 --- a/.github/workflows/release-version.yml +++ b/.github/workflows/release-version.yml @@ -7,6 +7,10 @@ on: description: 'The tag name of release' required: true default: '' + release_type: + description: 'The type of release (1: release 2: package)' + required: true + default: '1' run-name: ref_name:${{ github.ref_name }} release_version:${{ inputs.release_version }} @@ -15,7 +19,21 @@ env: jobs: + release-test: + runs-on: [ self-hosted, eks-fargate-runner ] + steps: + - if: ${{ inputs.release_type == '1' }} + uses: apecloud/checkout@main + - if: ${{ inputs.release_type == '1' }} + name: vendor lint test + run: | + mkdir -p ./bin + cp -r /go/bin/controller-gen ./bin/controller-gen + cp -r /go/bin/setup-envtest ./bin/setup-envtest + make mod-vendor lint test + release-version: + needs: release-test runs-on: ubuntu-latest steps: - name: checkout branch ${{ github.ref_name }} From 8c6f63c36be247f83730b69310a9b14ce229782d Mon Sep 17 00:00:00 2001 From: "yunju.lly" Date: Thu, 20 Apr 2023 22:24:49 +0800 Subject: [PATCH 115/439] fix: default value of enable-all-logs is set to false (#2788) --- docs/user_docs/cli/kbcli_cluster_create.md | 2 +- docs/user_docs/cli/kbcli_cluster_update.md | 2 +- internal/cli/cmd/cluster/create.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index 90ad8727a..62b445561 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -67,7 +67,7 @@ kbcli cluster create [NAME] [flags] --backup string Set a source backup to restore data --cluster-definition string Specify cluster definition, run "kbcli cd list" to show all available cluster definitions --cluster-version string Specify cluster version, run "kbcli cv list" to show all available cluster versions, use the latest version if not specified - --enable-all-logs Enable advanced application all log extraction, and true will ignore enabledLogs of component level (default true) + --enable-all-logs Enable advanced application all log extraction, and true will ignore enabledLogs of component level, default is false -h, --help help for create --monitor Set monitor enabled and inject metrics exporter (default true) --node-labels stringToString Node label selector (default []) diff --git a/docs/user_docs/cli/kbcli_cluster_update.md b/docs/user_docs/cli/kbcli_cluster_update.md index d94cf8803..c3b95e2bd 100644 --- a/docs/user_docs/cli/kbcli_cluster_update.md +++ b/docs/user_docs/cli/kbcli_cluster_update.md @@ -32,7 +32,7 @@ kbcli cluster update NAME [flags] ``` --allow-missing-template-keys If true, ignore any errors in templates when a field or map key is missing in the template. Only applies to golang and jsonpath output formats. (default true) --dry-run string[="unchanged"] Must be "none", "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --enable-all-logs Enable advanced application all log extraction, and true will ignore enabledLogs of component level (default true) + --enable-all-logs Enable advanced application all log extraction, and true will ignore enabledLogs of component level, default is false -h, --help help for update --monitor Set monitor enabled and inject metrics exporter (default true) --node-labels stringToString Node label selector (default []) diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index b1aad7796..8fb5f1277 100644 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -659,7 +659,7 @@ func generateClusterName(dynamic dynamic.Interface, namespace string) (string, e func (f *UpdatableFlags) addFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&f.PodAntiAffinity, "pod-anti-affinity", "Preferred", "Pod anti-affinity type, one of: (Preferred, Required)") cmd.Flags().BoolVar(&f.Monitor, "monitor", true, "Set monitor enabled and inject metrics exporter") - cmd.Flags().BoolVar(&f.EnableAllLogs, "enable-all-logs", true, "Enable advanced application all log extraction, and true will ignore enabledLogs of component level") + cmd.Flags().BoolVar(&f.EnableAllLogs, "enable-all-logs", false, "Enable advanced application all log extraction, and true will ignore enabledLogs of component level, default is false") cmd.Flags().StringVar(&f.TerminationPolicy, "termination-policy", "Delete", "Termination policy, one of: (DoNotTerminate, Halt, Delete, WipeOut)") cmd.Flags().StringArrayVar(&f.TopologyKeys, "topology-keys", nil, "Topology keys for affinity") cmd.Flags().StringToStringVar(&f.NodeLabels, "node-labels", nil, "Node label selector") From 6c96ca95d91cdf37cca1d3b638c3f03aa0505d82 Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Thu, 20 Apr 2023 22:26:13 +0800 Subject: [PATCH 116/439] fix: decouple code dependency when adding preflight (#2776) --- internal/cli/cmd/kubeblocks/preflight.go | 65 ++++++------------- internal/cli/cmd/kubeblocks/preflight_test.go | 8 +++ 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/internal/cli/cmd/kubeblocks/preflight.go b/internal/cli/cmd/kubeblocks/preflight.go index 10b9d958f..eee1f6ce0 100644 --- a/internal/cli/cmd/kubeblocks/preflight.go +++ b/internal/cli/cmd/kubeblocks/preflight.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "os/signal" + "strings" "github.com/ahmetalpbalkan/go-cursor" "github.com/fatih/color" @@ -54,6 +55,9 @@ const ( flagDebug = "debug" flagNamespace = "namespace" flagVerbose = "verbose" + + PreflightPattern = "data/%s_preflight.yaml" + HostPreflightPattern = "data/%s_hostpreflight.yaml" ) var ( @@ -73,17 +77,6 @@ var ( kbcli kubeblocks preflight preflight-check.yaml --interactive=true`) ) -const ( - EKSHostPreflight = "data/eks_hostpreflight.yaml" - EKSPreflight = "data/eks_preflight.yaml" - GKEHostPreflight = "data/gke_hostpreflight.yaml" - GKEPreflight = "data/gke_preflight.yaml" - ACKHostPreflight = "data/ack_hostpreflight.yaml" - ACKPreflight = "data/ack_preflight.yaml" - TKEHostPreflight = "data/tke_hostpreflight.yaml" - TKEPreflight = "data/tke_preflight.yaml" -) - // PreflightOptions declares the arguments accepted by the preflight command type PreflightOptions struct { factory cmdutil.Factory @@ -129,40 +122,14 @@ func NewPreflightCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *co func LoadVendorCheckYaml(vendorName util.K8sProvider) ([][]byte, error) { var yamlDataList [][]byte - switch vendorName { - case util.EKSProvider: - if data, err := defaultVendorYamlData.ReadFile(EKSHostPreflight); err == nil { - yamlDataList = append(yamlDataList, data) - } - if data, err := defaultVendorYamlData.ReadFile(EKSPreflight); err == nil { - yamlDataList = append(yamlDataList, data) - } - case util.GKEProvider: - if data, err := defaultVendorYamlData.ReadFile(GKEHostPreflight); err == nil { - yamlDataList = append(yamlDataList, data) - } - if data, err := defaultVendorYamlData.ReadFile(GKEPreflight); err == nil { - yamlDataList = append(yamlDataList, data) - } - case util.ACKProvider: - if data, err := defaultVendorYamlData.ReadFile(ACKHostPreflight); err == nil { - yamlDataList = append(yamlDataList, data) - } - if data, err := defaultVendorYamlData.ReadFile(ACKPreflight); err == nil { - yamlDataList = append(yamlDataList, data) - } - case util.TKEProvider: - if data, err := defaultVendorYamlData.ReadFile(TKEHostPreflight); err == nil { - yamlDataList = append(yamlDataList, data) - } - if data, err := defaultVendorYamlData.ReadFile(TKEPreflight); err == nil { - yamlDataList = append(yamlDataList, data) - } - case util.UnknownProvider: - fallthrough - default: - fmt.Println("unsupported k8s provider, and the validation of provider will coming soon") - return yamlDataList, errors.New("no supported provider") + if data, err := defaultVendorYamlData.ReadFile(newPreflightPath(vendorName)); err == nil { + yamlDataList = append(yamlDataList, data) + } + if data, err := defaultVendorYamlData.ReadFile(newHostPreflightPath(vendorName)); err == nil { + yamlDataList = append(yamlDataList, data) + } + if len(yamlDataList) == 0 { + return yamlDataList, errors.New("unsupported k8s provider, and the validation of provider will coming soon") } return yamlDataList, nil } @@ -279,3 +246,11 @@ func CollectProgress(ctx context.Context, progressCh <-chan interface{}, verbose } } } + +func newPreflightPath(vendorName util.K8sProvider) string { + return fmt.Sprintf(PreflightPattern, strings.ToLower(string(vendorName))) +} + +func newHostPreflightPath(vendorName util.K8sProvider) string { + return fmt.Sprintf(HostPreflightPattern, strings.ToLower(string(vendorName))) +} diff --git a/internal/cli/cmd/kubeblocks/preflight_test.go b/internal/cli/cmd/kubeblocks/preflight_test.go index ce0d860f9..f21944588 100644 --- a/internal/cli/cmd/kubeblocks/preflight_test.go +++ b/internal/cli/cmd/kubeblocks/preflight_test.go @@ -129,4 +129,12 @@ var _ = Describe("Preflight API Test", func() { Expect(err).NotTo(HaveOccurred()) Expect(len(res)).Should(Equal(2)) }) + It("newPreflightPath test, and expect success", func() { + res := newPreflightPath("test") + Expect(res).Should(Equal("data/test_preflight.yaml")) + }) + It("newHostPreflightPath test, and expect success", func() { + res := newHostPreflightPath("test") + Expect(res).Should(Equal("data/test_hostpreflight.yaml")) + }) }) From 4bfdfa31045e081289ff881db1255a18982529fe Mon Sep 17 00:00:00 2001 From: linghan-hub <56351212+linghan-hub@users.noreply.github.com> Date: Fri, 21 Apr 2023 01:29:48 +0800 Subject: [PATCH 117/439] chore: e2e support playground init multiple cloud provider test (#2667) --- Makefile | 3 +- deploy/apecloud-mysql-cluster/Chart.yaml | 2 +- .../apecloud-mysql-scale-cluster/Chart.yaml | 2 +- deploy/apecloud-mysql-scale/Chart.yaml | 2 +- deploy/apecloud-mysql/Chart.yaml | 2 +- deploy/chatgpt-retrieval-plugin/Chart.yaml | 2 +- deploy/clickhouse-cluster/Chart.yaml | 2 +- deploy/clickhouse/Chart.yaml | 2 +- deploy/helm/Chart.yaml | 4 +- deploy/kafka-cluster/Chart.yaml | 2 +- deploy/kafka/Chart.yaml | 2 +- deploy/milvus-cluster/Chart.yaml | 2 +- deploy/milvus/Chart.yaml | 2 +- deploy/mongodb-cluster/Chart.yaml | 2 +- deploy/mongodb/Chart.yaml | 2 +- deploy/nyancat/Chart.yaml | 4 +- deploy/postgresql-cluster/Chart.yaml | 2 +- deploy/postgresql/Chart.yaml | 2 +- deploy/qdrant-cluster/Chart.yaml | 2 +- deploy/qdrant/Chart.yaml | 2 +- deploy/redis-cluster/Chart.yaml | 2 +- deploy/redis/Chart.yaml | 2 +- deploy/weaviate-cluster/Chart.yaml | 2 +- deploy/weaviate/Chart.yaml | 2 +- test/e2e/Makefile | 12 ++- test/e2e/e2e_suite_test.go | 14 +++ .../smoketest/mongodb/00_mongodbcluster.yaml | 92 ++++++++++++------- test/e2e/testdata/smoketest/playgroundtest.go | 56 ++++++++--- .../postgresql/00_postgresqlcluster.yaml | 6 +- .../smoketest/postgresql/05_hscale_up.yaml | 10 ++ .../smoketest/postgresql/06_hscale_down.yaml | 10 ++ .../postgresql/{05_cv.yaml => 07_cv.yaml} | 4 +- .../{06_upgrade.yaml => 08_upgrade.yaml} | 2 +- .../smoketest/postgresql/09_backup_full.yaml | 8 -- .../{07_restart.yaml => 09_restart.yaml} | 0 ..._snapshot.yaml => 10_backup_snapshot.yaml} | 0 .../postgresql/11_backup_full_restore.yaml | 23 ----- ...e.yaml => 11_backup_sbapshot_restore.yaml} | 2 +- .../smoketest/redis/00_rediscluster.yaml | 38 ++++---- test/e2e/testdata/smoketest/smoketestrun.go | 3 +- .../smoketest/wesql/00_wesqlcluster.yaml | 2 +- ...01_componentresourceconstraint_custom.yaml | 42 +++++++++ ...custom_class.yaml => 02_custom_class.yaml} | 5 +- .../wesql/{02_vscale.yaml => 03_vscale.yaml} | 2 +- .../{03_hscale.yaml => 04_hscale_up.yaml} | 2 +- .../smoketest/wesql/05_hscale_down.yaml | 10 ++ .../{04_vexpand.yaml => 06_vexpand.yaml} | 0 .../wesql/{05_cv.yaml => 07_cv.yaml} | 0 .../{06_upgrade.yaml => 08_upgrade.yaml} | 0 .../wesql/{07_stop.yaml => 09_stop.yaml} | 0 .../wesql/{08_start.yaml => 10_start.yaml} | 0 .../{09_restart.yaml => 11_restart.yaml} | 0 .../smoketest/wesql/12_backup_full.yaml | 10 -- ...0_reconfigure.yaml => 12_reconfigure.yaml} | 0 ..._snapshot.yaml => 13_backup_snapshot.yaml} | 0 .../wesql/14_backup_full_restore.yaml | 28 ------ ...e.yaml => 14_backup_snapshot_restore.yaml} | 4 +- test/e2e/types.go | 4 + test/e2e/util/smoke_util.go | 31 +++++++ 59 files changed, 295 insertions(+), 178 deletions(-) create mode 100644 test/e2e/testdata/smoketest/postgresql/05_hscale_up.yaml create mode 100644 test/e2e/testdata/smoketest/postgresql/06_hscale_down.yaml rename test/e2e/testdata/smoketest/postgresql/{05_cv.yaml => 07_cv.yaml} (72%) rename test/e2e/testdata/smoketest/postgresql/{06_upgrade.yaml => 08_upgrade.yaml} (75%) delete mode 100644 test/e2e/testdata/smoketest/postgresql/09_backup_full.yaml rename test/e2e/testdata/smoketest/postgresql/{07_restart.yaml => 09_restart.yaml} (100%) rename test/e2e/testdata/smoketest/postgresql/{08_backup_snapshot.yaml => 10_backup_snapshot.yaml} (100%) delete mode 100644 test/e2e/testdata/smoketest/postgresql/11_backup_full_restore.yaml rename test/e2e/testdata/smoketest/postgresql/{10_backup_sbapshot_restore.yaml => 11_backup_sbapshot_restore.yaml} (93%) create mode 100644 test/e2e/testdata/smoketest/wesql/01_componentresourceconstraint_custom.yaml rename test/e2e/testdata/smoketest/wesql/{01_custom_class.yaml => 02_custom_class.yaml} (80%) rename test/e2e/testdata/smoketest/wesql/{02_vscale.yaml => 03_vscale.yaml} (86%) rename test/e2e/testdata/smoketest/wesql/{03_hscale.yaml => 04_hscale_up.yaml} (89%) create mode 100644 test/e2e/testdata/smoketest/wesql/05_hscale_down.yaml rename test/e2e/testdata/smoketest/wesql/{04_vexpand.yaml => 06_vexpand.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{05_cv.yaml => 07_cv.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{06_upgrade.yaml => 08_upgrade.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{07_stop.yaml => 09_stop.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{08_start.yaml => 10_start.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{09_restart.yaml => 11_restart.yaml} (100%) delete mode 100644 test/e2e/testdata/smoketest/wesql/12_backup_full.yaml rename test/e2e/testdata/smoketest/wesql/{10_reconfigure.yaml => 12_reconfigure.yaml} (100%) rename test/e2e/testdata/smoketest/wesql/{11_backup_snapshot.yaml => 13_backup_snapshot.yaml} (100%) delete mode 100644 test/e2e/testdata/smoketest/wesql/14_backup_full_restore.yaml rename test/e2e/testdata/smoketest/wesql/{13_backup_snapshot_restore.yaml => 14_backup_snapshot_restore.yaml} (86%) diff --git a/Makefile b/Makefile index f863421c4..c82759753 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ SKIP_GO_GEN ?= true CHART_PATH = deploy/helm WEBHOOK_CERT_DIR ?= /tmp/k8s-webhook-server/serving-certs + # Go setup export GO111MODULE = auto # export GOPROXY = https://proxy.golang.org @@ -755,7 +756,7 @@ render-smoke-testdata-manifests: ## Update E2E test dataset .PHONY: test-e2e test-e2e: helm-package render-smoke-testdata-manifests ## Run E2E tests. - $(MAKE) -e VERSION=$(VERSION) -C test/e2e run + $(MAKE) -e VERSION=$(VERSION) PROVIDER=$(PROVIDER) REGION=$(REGION) SECRET_ID=$(SECRET_ID) SECRET_KEY=$(SECRET_KEY) -C test/e2e run # NOTE: include must be placed at the end include docker/docker.mk diff --git a/deploy/apecloud-mysql-cluster/Chart.yaml b/deploy/apecloud-mysql-cluster/Chart.yaml index df86189ff..8bfb608a6 100644 --- a/deploy/apecloud-mysql-cluster/Chart.yaml +++ b/deploy/apecloud-mysql-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: An ApeCloud MySQL Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "8.0.30" diff --git a/deploy/apecloud-mysql-scale-cluster/Chart.yaml b/deploy/apecloud-mysql-scale-cluster/Chart.yaml index 8e71ff5c1..fb76be8cf 100644 --- a/deploy/apecloud-mysql-scale-cluster/Chart.yaml +++ b/deploy/apecloud-mysql-scale-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: An ApeCloud MySQL-Scale Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/deploy/apecloud-mysql-scale/Chart.yaml b/deploy/apecloud-mysql-scale/Chart.yaml index 51126d007..793b37ff2 100644 --- a/deploy/apecloud-mysql-scale/Chart.yaml +++ b/deploy/apecloud-mysql-scale/Chart.yaml @@ -5,7 +5,7 @@ description: ApeCloud MySQL-Scale is ApeCloud MySQL proxy. ApeCloud MySQL-Scale type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/deploy/apecloud-mysql/Chart.yaml b/deploy/apecloud-mysql/Chart.yaml index efa0019f6..ce9ec5be7 100644 --- a/deploy/apecloud-mysql/Chart.yaml +++ b/deploy/apecloud-mysql/Chart.yaml @@ -9,7 +9,7 @@ description: ApeCloud MySQL is fully compatible with MySQL syntax and supports s type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "8.0.30" diff --git a/deploy/chatgpt-retrieval-plugin/Chart.yaml b/deploy/chatgpt-retrieval-plugin/Chart.yaml index 1b84d6178..054472a09 100644 --- a/deploy/chatgpt-retrieval-plugin/Chart.yaml +++ b/deploy/chatgpt-retrieval-plugin/Chart.yaml @@ -5,7 +5,7 @@ description: A demo application for ChatGPT plugin. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: 0.1.0 diff --git a/deploy/clickhouse-cluster/Chart.yaml b/deploy/clickhouse-cluster/Chart.yaml index de06b63b9..b86db763f 100644 --- a/deploy/clickhouse-cluster/Chart.yaml +++ b/deploy/clickhouse-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A ClickHouse cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: 22.9.4 diff --git a/deploy/clickhouse/Chart.yaml b/deploy/clickhouse/Chart.yaml index 7b963e2bc..c67a16ef2 100644 --- a/deploy/clickhouse/Chart.yaml +++ b/deploy/clickhouse/Chart.yaml @@ -9,7 +9,7 @@ annotations: type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: 22.9.4 diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml index f4e6bc6fe..751fbdb3f 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.5.0-alpha.8 +appVersion: 0.5.0-beta.2 kubeVersion: '>=1.22.0-0' diff --git a/deploy/kafka-cluster/Chart.yaml b/deploy/kafka-cluster/Chart.yaml index 96f668ff9..634fd6623 100644 --- a/deploy/kafka-cluster/Chart.yaml +++ b/deploy/kafka-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A Kafka server cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: 3.4.0 diff --git a/deploy/kafka/Chart.yaml b/deploy/kafka/Chart.yaml index f3a99dcdc..481ce2d98 100644 --- a/deploy/kafka/Chart.yaml +++ b/deploy/kafka/Chart.yaml @@ -11,7 +11,7 @@ annotations: type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: 3.4.0 diff --git a/deploy/milvus-cluster/Chart.yaml b/deploy/milvus-cluster/Chart.yaml index b20fdeb74..ca05710fa 100644 --- a/deploy/milvus-cluster/Chart.yaml +++ b/deploy/milvus-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A Milvus cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.3 +version: 0.5.0-beta.2 appVersion: "2.2.4" diff --git a/deploy/milvus/Chart.yaml b/deploy/milvus/Chart.yaml index a00be2d6f..0e264027c 100644 --- a/deploy/milvus/Chart.yaml +++ b/deploy/milvus/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 # This is the version number of milvus appVersion: "2.2.4" diff --git a/deploy/mongodb-cluster/Chart.yaml b/deploy/mongodb-cluster/Chart.yaml index 6fdd0b479..5485a468e 100644 --- a/deploy/mongodb-cluster/Chart.yaml +++ b/deploy/mongodb-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A MongoDB cluster Helm chart for KubeBlocks type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "5.0.14" diff --git a/deploy/mongodb/Chart.yaml b/deploy/mongodb/Chart.yaml index b3504b6a1..6eed23d60 100644 --- a/deploy/mongodb/Chart.yaml +++ b/deploy/mongodb/Chart.yaml @@ -4,7 +4,7 @@ description: MongoDB is a document database designed for ease of application dev type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "5.0.14" diff --git a/deploy/nyancat/Chart.yaml b/deploy/nyancat/Chart.yaml index 4e385afeb..a9774cec9 100644 --- a/deploy/nyancat/Chart.yaml +++ b/deploy/nyancat/Chart.yaml @@ -4,8 +4,8 @@ description: A demo application for showing database cluster availability. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 -appVersion: 0.5.0-alpha.8 +appVersion: 0.5.0-beta.2 kubeVersion: '>=1.22.0-0' diff --git a/deploy/postgresql-cluster/Chart.yaml b/deploy/postgresql-cluster/Chart.yaml index de247d6c2..51f863120 100644 --- a/deploy/postgresql-cluster/Chart.yaml +++ b/deploy/postgresql-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A PostgreSQL (with Patroni HA) cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "14.7.0" diff --git a/deploy/postgresql/Chart.yaml b/deploy/postgresql/Chart.yaml index dfb1814c1..997434337 100644 --- a/deploy/postgresql/Chart.yaml +++ b/deploy/postgresql/Chart.yaml @@ -4,7 +4,7 @@ description: A PostgreSQL (with Patroni HA) cluster definition Helm chart for Ku type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "14.7.0" diff --git a/deploy/qdrant-cluster/Chart.yaml b/deploy/qdrant-cluster/Chart.yaml index 2e6b6c598..75ec59a5a 100644 --- a/deploy/qdrant-cluster/Chart.yaml +++ b/deploy/qdrant-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A Qdrant cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "1.1.0" diff --git a/deploy/qdrant/Chart.yaml b/deploy/qdrant/Chart.yaml index 3e30e30cc..1cb33f31b 100644 --- a/deploy/qdrant/Chart.yaml +++ b/deploy/qdrant/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version. -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 # This is the version number of qdrant. appVersion: "1.1.0" diff --git a/deploy/redis-cluster/Chart.yaml b/deploy/redis-cluster/Chart.yaml index ccb95e45c..79eeb886b 100644 --- a/deploy/redis-cluster/Chart.yaml +++ b/deploy/redis-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: An Redis Replication Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "7.0.6" diff --git a/deploy/redis/Chart.yaml b/deploy/redis/Chart.yaml index 9762619ce..438d3da4c 100644 --- a/deploy/redis/Chart.yaml +++ b/deploy/redis/Chart.yaml @@ -4,7 +4,7 @@ description: A Redis cluster definition Helm chart for Kubernetes type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "7.0.6" diff --git a/deploy/weaviate-cluster/Chart.yaml b/deploy/weaviate-cluster/Chart.yaml index c59c9159c..052298f7b 100644 --- a/deploy/weaviate-cluster/Chart.yaml +++ b/deploy/weaviate-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A weaviate cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 appVersion: "1.18.0" diff --git a/deploy/weaviate/Chart.yaml b/deploy/weaviate/Chart.yaml index 466cb1a70..02368fba3 100644 --- a/deploy/weaviate/Chart.yaml +++ b/deploy/weaviate/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version. -version: 0.5.0-alpha.8 +version: 0.5.0-beta.2 # This is the version number of weaviate. appVersion: "1.18.0" diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 2eaef082a..6de2cbd9e 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -30,10 +30,20 @@ ginkgo: # Make sure ginkgo is in $GOPATH/bin ifeq ($(origin VERSION), command line) VERSION ?= $(VERSION) endif +ifeq ($(origin VERSION), command line) + PROVIDER ?= $(PROVIDER) +else + PROVIDER ?= "" +endif +ifeq ($(origin REGION), command line) + REGION ?= $(REGION) +else + REGION ?= "" +endif .PHONY: run run: ginkgo ## Run end-to-end tests. #ACK_GINKGO_DEPRECATIONS=$(GINKGO_VERSION) $(GINKGO) run . - $(GINKGO) test -process -ginkgo.v . -- --VERSION=$(VERSION) --ginkgo.json-report=report.json + $(GINKGO) test -process -ginkgo.v . -- -VERSION=$(VERSION) -PROVIDER=$(PROVIDER) -REGION=$(REGION) -SECRET_ID=$(SECRET_ID) -SECRET_KEY=$(SECRET_KEY) --ginkgo.json-report=report.json build: ginkgo ## Run ginkgo build e2e test suite binary. $(GINKGO) build . diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ad5a9c4c3..1b03d2ddd 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -48,10 +48,18 @@ var cfg *rest.Config var testEnv *envtest.Environment var TC *TestClient var version string +var provider string +var region string +var secretID string +var secretKey string func init() { viper.AutomaticEnv() flag.StringVar(&version, "VERSION", "", "kubeblocks test version") + flag.StringVar(&provider, "PROVIDER", "", "kubeblocks test cloud-provider") + flag.StringVar(®ion, "REGION", "", "kubeblocks test region") + flag.StringVar(&secretID, "SECRET_ID", "", "cloud-provider SECRET_ID") + flag.StringVar(&secretKey, "SECRET_KEY", "", "cloud-provider SECRET_KEY") } func TestE2e(t *testing.T) { @@ -91,6 +99,12 @@ var _ = BeforeSuite(func() { } log.Println("kb version:" + version) Version = version + if len(provider) > 0 && len(region) > 0 && len(secretID) > 0 && len(secretKey) > 0 { + Provider = provider + Region = region + SecretID = secretID + SecretKey = secretKey + } if viper.GetBool("ENABLE_DEBUG_LOG") { logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true), func(o *zap.Options) { o.TimeEncoder = zapcore.ISO8601TimeEncoder diff --git a/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml b/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml index d786498ea..5e62f3bbf 100644 --- a/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml +++ b/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml @@ -5,7 +5,7 @@ kind: ConfigMap metadata: name: mongodb5.0-config-template labels: - helm.sh/chart: mongodb-0.5.0-alpha.8 + helm.sh/chart: mongodb-0.5.0-beta.2 app.kubernetes.io/name: mongodb app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -35,13 +35,15 @@ data: directoryPerDB: true # where to write logging data. + {{ block "logsBlock" . }} systemLog: destination: file quiet: false logAppend: true logRotate: reopen - path: {{ $mongodb_root }}/logs/mongodb.log + path: /data/mongodb/logs/mongodb.log verbosity: 0 + {{ end }} # network interfaces net: @@ -85,7 +87,7 @@ kind: ConfigMap metadata: name: mongodb-metrics-config labels: - helm.sh/chart: mongodb-0.5.0-alpha.8 + helm.sh/chart: mongodb-0.5.0-beta.2 app.kubernetes.io/name: mongodb app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -99,7 +101,7 @@ kind: ConfigMap metadata: name: mongodb-scripts labels: - helm.sh/chart: mongodb-0.5.0-alpha.8 + helm.sh/chart: mongodb-0.5.0-beta.2 app.kubernetes.io/name: mongodb app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -148,7 +150,6 @@ data: {{- $mongodb_port = $mongodb_port_info.containerPort }} {{- end }} - set -e PORT={{ $mongodb_port }} MONGODB_ROOT={{ $mongodb_root }} INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); @@ -219,7 +220,7 @@ metadata: name: mongodb-backup-policy-template labels: clusterdefinition.kubeblocks.io/name: mongodb - helm.sh/chart: mongodb-0.5.0-alpha.8 + helm.sh/chart: mongodb-0.5.0-beta.2 app.kubernetes.io/name: mongodb app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -236,12 +237,15 @@ spec: cronExpression: "0 18 * * 0" snapshot: target: - role: leader + role: primary connectionCredentialKey: passwordKey: password usernameKey: username full: - backupToolName: xtrabackup-apecloud-mysql + backupToolName: mongodb-physical-backup-tool + backupsHistoryLimit: 7 + target: + role: primary --- # Source: mongodb/templates/backuptool.yaml apiVersion: dataprotection.kubeblocks.io/v1alpha1 @@ -269,7 +273,7 @@ spec: set -e mkdir -p ${DATA_DIR} res=`ls -A ${DATA_DIR}` - if [ ! -z ${res} ]; then + if [ ! -z "${res}" ]; then echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." exit 1 fi @@ -312,7 +316,7 @@ kind: ClusterDefinition metadata: name: mongodb labels: - helm.sh/chart: mongodb-0.5.0-alpha.8 + helm.sh/chart: mongodb-0.5.0-beta.2 app.kubernetes.io/name: mongodb app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -358,7 +362,7 @@ spec: scrapePort: 9216 logConfigs: - name: running - filePathPattern: /data/mongodb/log/mongodb.log* + filePathPattern: /data/mongodb/logs/mongodb.log* workloadType: Consensus consensusSpec: leader: @@ -446,6 +450,30 @@ spec: volumeMounts: - name: mongodb-metrics-config mountPath: /opt/conf +--- +# Source: mongodb/templates/sharding-clusterdefinition.yaml +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterDefinition +metadata: + name: mongodb-sharding + labels: + helm.sh/chart: mongodb-0.5.0-beta.2 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +spec: + type: mongodb + connectionCredential: + username: root + password: "$(RANDOM_PASSWD)" + endpoint: "$(SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + host: "$(SVC_FQDN)" + port: "$(SVC_PORT_tcp-monogdb)" + headlessEndpoint: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + headlessHost: "$(POD_NAME_PREFIX)-0.$(HEADLESS_SVC_FQDN)" + headlessPort: "$(SVC_PORT_tcp-monogdb)" + componentDefs: - name: mongos scriptSpecs: - name: mongodb-scripts @@ -583,7 +611,7 @@ kind: ClusterVersion metadata: name: mongodb-5.0.14 labels: - helm.sh/chart: mongodb-0.5.0-alpha.8 + helm.sh/chart: mongodb-0.5.0-beta.2 app.kubernetes.io/name: mongodb app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -597,6 +625,21 @@ spec: - name: mongodb image: mongo:5.0.14 imagePullPolicy: IfNotPresent +--- +# Source: mongodb/templates/sharding-clusterversion.yaml +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterVersion +metadata: + name: mongodb-sharding-5.0.14 + labels: + helm.sh/chart: mongodb-0.5.0-beta.2 + app.kubernetes.io/name: mongodb + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "5.0.14" + app.kubernetes.io/managed-by: Helm +spec: + clusterDefinitionRef: mongodb-sharding + componentVersions: - componentDefRef: mongos versionsContext: containers: @@ -624,7 +667,7 @@ kind: ConfigConstraint metadata: name: mongodb-config-constraints labels: - helm.sh/chart: mongodb-0.5.0-alpha.8 + helm.sh/chart: mongodb-0.5.0-beta.2 app.kubernetes.io/name: mongodb app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -643,7 +686,7 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: mongodb-cluster-0.5.0-alpha.8 + helm.sh/chart: mongodb-cluster-0.5.0-beta.2 app.kubernetes.io/name: mongodb-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -669,24 +712,3 @@ spec: resources: requests: storage: 20Gi ---- -# Source: mongodb-cluster/templates/tests/test-connection.yaml -apiVersion: v1 -kind: Pod -metadata: - name: "mycluster-mongodb-cluster-test-connection" - labels: - helm.sh/chart: mongodb-cluster-0.5.0-alpha.8 - app.kubernetes.io/name: mongodb-cluster - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['mycluster-mongodb-cluster:'] - restartPolicy: Never diff --git a/test/e2e/testdata/smoketest/playgroundtest.go b/test/e2e/testdata/smoketest/playgroundtest.go index 29ca3c5fe..be021faff 100644 --- a/test/e2e/testdata/smoketest/playgroundtest.go +++ b/test/e2e/testdata/smoketest/playgroundtest.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + . "github.com/apecloud/kubeblocks/test/e2e" e2eutil "github.com/apecloud/kubeblocks/test/e2e/util" ) @@ -45,10 +46,37 @@ func PlaygroundInit() { } }) It("kbcli playground init", func() { - cmd := "kbcli playground init" - log.Println(cmd) - init := e2eutil.ExecuteCommand(cmd) - log.Println(init) + var cmd string + if len(Provider) > 0 && len(Region) > 0 && len(SecretID) > 0 && len(SecretKey) > 0 { + var id, key string + if Provider == "aws" { + id = "export AWS_ACCESS_KEY_ID=" + SecretID + key = "export AWS_SECRET_ACCESS_KEY=" + SecretKey + } else if Provider == "tencentcloud" { + id = "export TENCENTCLOUD_SECRET_ID=" + SecretID + key = "export TENCENTCLOUD_SECRET_KEY" + SecretKey + } else if Provider == "alicloud" { + id = "export ALICLOUD_ACCESS_KEY=" + SecretID + key = "export ALICLOUD_SECRET_KEY=" + SecretKey + } else { + log.Println("not support " + Provider + " cloud-provider") + } + idCmd := e2eutil.ExecuteCommand(id) + log.Println(idCmd) + keyCmd := e2eutil.ExecuteCommand(key) + log.Println(keyCmd) + cmd = "kbcli playground init --cloud-provider " + Provider + " --region " + Region + output, err := e2eutil.Check(cmd, "yes\n") + if err != nil { + log.Fatalf("Command execution failure: %v\n", err) + } + log.Println("Command execution result：", output) + } else { + cmd = "kbcli playground init" + log.Println(cmd) + init := e2eutil.ExecuteCommand(cmd) + log.Println(init) + } }) It("check kbcli playground cluster and pod status", func() { checkPlaygroundCluster() @@ -64,15 +92,15 @@ func UninstallKubeblocks() { }) Context("KubeBlocks uninstall", func() { It("delete mycluster", func() { - command := "kbcli cluster delete mycluster --auto-approve" - log.Println(command) - result := e2eutil.ExecuteCommand(command) + commond := "kbcli cluster delete mycluster --auto-approve" + log.Println(commond) + result := e2eutil.ExecuteCommand(commond) Expect(result).Should(BeTrue()) }) It("check mycluster and pod", func() { - command := "kbcli cluster list -A" + commond := "kbcli cluster list -A" Eventually(func(g Gomega) { - cluster := e2eutil.ExecCommand(command) + cluster := e2eutil.ExecCommand(commond) g.Expect(e2eutil.StringStrip(cluster)).Should(Equal("Noclusterfound")) }, time.Second*10, time.Second*1).Should(Succeed()) cmd := "kbcli cluster list-instances" @@ -107,19 +135,19 @@ func PlaygroundDestroy() { } func checkPlaygroundCluster() { - command := "kubectl get pod -n default -l 'app.kubernetes.io/instance in (mycluster)'| grep mycluster |" + + commond := "kubectl get pod -n default -l 'app.kubernetes.io/instance in (mycluster)'| grep mycluster |" + " awk '{print $3}'" - log.Println(command) + log.Println(commond) Eventually(func(g Gomega) { - podStatus := e2eutil.ExecCommand(command) - log.Println(e2eutil.StringStrip(podStatus)) + podStatus := e2eutil.ExecCommand(commond) + log.Println("podStatus is " + e2eutil.StringStrip(podStatus)) g.Expect(e2eutil.StringStrip(podStatus)).Should(Equal("Running")) }, time.Second*180, time.Second*1).Should(Succeed()) cmd := "kbcli cluster list | grep mycluster | awk '{print $6}'" log.Println(cmd) Eventually(func(g Gomega) { clusterStatus := e2eutil.ExecCommand(cmd) - log.Println(e2eutil.StringStrip(clusterStatus)) + log.Println("clusterStatus is " + e2eutil.StringStrip(clusterStatus)) g.Expect(e2eutil.StringStrip(clusterStatus)).Should(Equal("Running")) }, time.Second*360, time.Second*1).Should(Succeed()) } diff --git a/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml b/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml index 6e2b20f89..a6fe12e21 100644 --- a/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml +++ b/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml @@ -5,14 +5,14 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: pgcluster-0.5.0-alpha.8 + helm.sh/chart: pgcluster-0.5.0-beta.2 app.kubernetes.io/name: pgcluster app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "15.2.0" + app.kubernetes.io/version: "14.7.0" app.kubernetes.io/managed-by: Helm spec: clusterDefinitionRef: postgresql # ref clusterdefinition.name - clusterVersionRef: postgresql-15.2.0 # ref clusterversion.name + clusterVersionRef: postgresql-14.7.0 # ref clusterversion.name terminationPolicy: Delete affinity: componentSpecs: diff --git a/test/e2e/testdata/smoketest/postgresql/05_hscale_up.yaml b/test/e2e/testdata/smoketest/postgresql/05_hscale_up.yaml new file mode 100644 index 000000000..d156a4069 --- /dev/null +++ b/test/e2e/testdata/smoketest/postgresql/05_hscale_up.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-hscale-up +spec: + clusterRef: mycluster + type: HorizontalScaling + horizontalScaling: + - componentName: postgresql + replicas: 3 \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/06_hscale_down.yaml b/test/e2e/testdata/smoketest/postgresql/06_hscale_down.yaml new file mode 100644 index 000000000..5cfc80700 --- /dev/null +++ b/test/e2e/testdata/smoketest/postgresql/06_hscale_down.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-hscale-down +spec: + clusterRef: mycluster + type: HorizontalScaling + horizontalScaling: + - componentName: postgresql + replicas: 2 \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/05_cv.yaml b/test/e2e/testdata/smoketest/postgresql/07_cv.yaml similarity index 72% rename from test/e2e/testdata/smoketest/postgresql/05_cv.yaml rename to test/e2e/testdata/smoketest/postgresql/07_cv.yaml index 47bb38480..a60746a07 100644 --- a/test/e2e/testdata/smoketest/postgresql/05_cv.yaml +++ b/test/e2e/testdata/smoketest/postgresql/07_cv.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterVersion metadata: - name: postgresql-15.2.0-latest + name: postgresql-14.7.0-latest spec: clusterDefinitionRef: postgresql componentVersions: @@ -9,4 +9,4 @@ spec: versionsContext: containers: - name: postgresql - image: docker.io/apecloud/postgresql:15.2.0 + image: docker.io/apecloud/postgresql:14.7.0 diff --git a/test/e2e/testdata/smoketest/postgresql/06_upgrade.yaml b/test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml similarity index 75% rename from test/e2e/testdata/smoketest/postgresql/06_upgrade.yaml rename to test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml index dd9c6fde5..abf304c68 100644 --- a/test/e2e/testdata/smoketest/postgresql/06_upgrade.yaml +++ b/test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml @@ -6,4 +6,4 @@ spec: clusterRef: mycluster type: Upgrade upgrade: - clusterVersionRef: postgresql-15.2.0-latest \ No newline at end of file + clusterVersionRef: postgresql-14.7.0-latest \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/09_backup_full.yaml b/test/e2e/testdata/smoketest/postgresql/09_backup_full.yaml deleted file mode 100644 index 846d9de30..000000000 --- a/test/e2e/testdata/smoketest/postgresql/09_backup_full.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 -kind: Backup -metadata: - name: backup-full - namespace: default -spec: - backupPolicyName: mycluster-postgresql-backup-policy - backupType: full diff --git a/test/e2e/testdata/smoketest/postgresql/07_restart.yaml b/test/e2e/testdata/smoketest/postgresql/09_restart.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/07_restart.yaml rename to test/e2e/testdata/smoketest/postgresql/09_restart.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/08_backup_snapshot.yaml b/test/e2e/testdata/smoketest/postgresql/10_backup_snapshot.yaml similarity index 100% rename from test/e2e/testdata/smoketest/postgresql/08_backup_snapshot.yaml rename to test/e2e/testdata/smoketest/postgresql/10_backup_snapshot.yaml diff --git a/test/e2e/testdata/smoketest/postgresql/11_backup_full_restore.yaml b/test/e2e/testdata/smoketest/postgresql/11_backup_full_restore.yaml deleted file mode 100644 index d91be7614..000000000 --- a/test/e2e/testdata/smoketest/postgresql/11_backup_full_restore.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: Cluster -metadata: - name: mycluster-sbapshot - annotations: - kubeblocks.io/restore-from-backup: "{\"postgresql\":\"backup-full-mycluster\"}" -spec: - clusterDefinitionRef: postgresql - clusterVersionRef: postgresql-15.2.0 - terminationPolicy: WipeOut - componentSpecs: - - name: wesql - componentDefRef: mysql - monitor: false - replicas: 1 - volumeClaimTemplates: - - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/10_backup_sbapshot_restore.yaml b/test/e2e/testdata/smoketest/postgresql/11_backup_sbapshot_restore.yaml similarity index 93% rename from test/e2e/testdata/smoketest/postgresql/10_backup_sbapshot_restore.yaml rename to test/e2e/testdata/smoketest/postgresql/11_backup_sbapshot_restore.yaml index 241ac3509..89aeb5629 100644 --- a/test/e2e/testdata/smoketest/postgresql/10_backup_sbapshot_restore.yaml +++ b/test/e2e/testdata/smoketest/postgresql/11_backup_sbapshot_restore.yaml @@ -6,7 +6,7 @@ metadata: kubeblocks.io/restore-from-backup: "{\"postgresql\":\"backup-sbapshot-mycluster\"}" spec: clusterDefinitionRef: postgresql - clusterVersionRef: postgresql-15.2.0 + clusterVersionRef: postgresql-14.7.0 terminationPolicy: WipeOut componentSpecs: - name: wesql diff --git a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml index 4cbc35a2b..aabbb40b9 100644 --- a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml +++ b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml @@ -5,7 +5,7 @@ kind: ConfigMap metadata: name: redis7-config-template labels: - helm.sh/chart: redis-0.5.0-alpha.8 + helm.sh/chart: redis-0.5.0-beta.2 app.kubernetes.io/name: redis app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "7.0.6" @@ -19,8 +19,10 @@ data: tcp-keepalive 300 daemonize no pidfile /var/run/redis_6379.pid + {{ block "logsBlock" . }} loglevel notice logfile "/data/running.log" + {{ end }} databases 16 always-show-logo no set-proc-title yes @@ -142,19 +144,16 @@ data: {{- end }} {{- end }} {{- /* build primary pod message, because currently does not support cross-component acquisition of environment variables, the service of the redis master node is assembled here through specific rules */}} - {{- $primary_pod = printf "%s-%s-0.%s-%s-headless.%s.svc" $clusterName $redis_component.name $clusterName $redis_component.name $namespace }} - {{- if ne $primary_index 0 }} - {{- $primary_pod = printf "%s-%s-%d-0.%s-%s-headless.%s.svc" $clusterName $redis_component.name $primary_index $clusterName $redis_component.name $namespace }} - {{- end }} - {{- $sentinel_monitor := printf "%s-%s %s" $clusterName $sentinel_component.name $primary_pod }} + {{- $primary_pod = printf "%s-%s-%d.%s-%s-headless.%s.svc" $clusterName $redis_component.name $primary_index $clusterName $redis_component.name $namespace }} + {{- $sentinel_monitor := printf "%s-%s %s" $clusterName $redis_component.name $primary_pod }} cat>/etc/sentinel/redis-sentinel.conf<$(PASSWD) allcommands allkeys + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbdataprotection provisionPolicy: type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbmonitoring provisionPolicy: type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbprobe provisionPolicy: type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbreplicator provisionPolicy: type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) +psync +replconf +ping + update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: redis-sentinel workloadType: Stateful characterType: redis @@ -495,7 +496,7 @@ kind: ClusterVersion metadata: name: redis-7.0.6 labels: - helm.sh/chart: redis-0.5.0-alpha.8 + helm.sh/chart: redis-0.5.0-beta.2 app.kubernetes.io/name: redis app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "7.0.6" @@ -526,7 +527,7 @@ kind: ConfigConstraint metadata: name: redis7-config-constraints labels: - helm.sh/chart: redis-0.5.0-alpha.8 + helm.sh/chart: redis-0.5.0-beta.2 app.kubernetes.io/name: redis app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "7.0.6" @@ -689,6 +690,9 @@ spec: ... } + configuration: #RedisParameter & { + } + ## require db instance restart staticParameters: @@ -711,7 +715,7 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: redis-cluster-0.5.0-alpha.8 + helm.sh/chart: redis-cluster-0.5.0-beta.2 app.kubernetes.io/name: redis-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "7.0.6" diff --git a/test/e2e/testdata/smoketest/smoketestrun.go b/test/e2e/testdata/smoketest/smoketestrun.go index 3429ef371..5536344e6 100644 --- a/test/e2e/testdata/smoketest/smoketestrun.go +++ b/test/e2e/testdata/smoketest/smoketestrun.go @@ -125,6 +125,7 @@ func runTestCases(files []string) { b := e2eutil.OpsYaml(file, "apply") Expect(b).Should(BeTrue()) Eventually(func(g Gomega) { + e2eutil.WaitTime(100000) podStatusResult := e2eutil.CheckPodStatus() for _, result := range podStatusResult { g.Expect(result).Should(BeTrue()) @@ -133,7 +134,7 @@ func runTestCases(files []string) { Eventually(func(g Gomega) { clusterStatusResult := e2eutil.CheckClusterStatus() g.Expect(clusterStatusResult).Should(BeTrue()) - }, time.Second*180, time.Second*1).Should(Succeed()) + }, time.Second*240, time.Second*1).Should(Succeed()) } if len(files) > 0 { diff --git a/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml b/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml index ec30ef49f..dd7f6c390 100644 --- a/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml +++ b/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml @@ -5,7 +5,7 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: apecloud-mysql-cluster-0.5.0-alpha.8 + helm.sh/chart: apecloud-mysql-cluster-0.5.0-beta.2 app.kubernetes.io/name: apecloud-mysql-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "8.0.30" diff --git a/test/e2e/testdata/smoketest/wesql/01_componentresourceconstraint_custom.yaml b/test/e2e/testdata/smoketest/wesql/01_componentresourceconstraint_custom.yaml new file mode 100644 index 000000000..85c63d336 --- /dev/null +++ b/test/e2e/testdata/smoketest/wesql/01_componentresourceconstraint_custom.yaml @@ -0,0 +1,42 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ComponentResourceConstraint +metadata: + annotations: + meta.helm.sh/release-name: kubeblocks + meta.helm.sh/release-namespace: default + labels: + app.kubernetes.io/managed-by: Helm + resourceconstraint.kubeblocks.io/provider: kubeblocks + name: kb-resource-constraint-custom +spec: + constraints: + - cpu: + max: 2 + min: "0.1" + step: "0.1" + memory: + sizePerCPU: 1Gi + - cpu: + max: 2 + min: "0.1" + step: "0.1" + memory: + sizePerCPU: 2Gi + - cpu: + max: 2 + min: "0.1" + step: "0.1" + memory: + sizePerCPU: 3Gi + - cpu: + max: 2 + min: "0.1" + step: "0.1" + memory: + sizePerCPU: 4Gi + - cpu: + max: 2 + min: "0.1" + step: "0.1" + memory: + sizePerCPU: 5Gi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/01_custom_class.yaml b/test/e2e/testdata/smoketest/wesql/02_custom_class.yaml similarity index 80% rename from test/e2e/testdata/smoketest/wesql/01_custom_class.yaml rename to test/e2e/testdata/smoketest/wesql/02_custom_class.yaml index f149b6b15..6529601dd 100644 --- a/test/e2e/testdata/smoketest/wesql/01_custom_class.yaml +++ b/test/e2e/testdata/smoketest/wesql/02_custom_class.yaml @@ -8,7 +8,7 @@ metadata: clusterdefinition.kubeblocks.io/name: apecloud-mysql spec: groups: - - resourceConstraintRef: kb-resource-constraint-general + - resourceConstraintRef: kb-resource-constraint-custom template: | cpu: "{{ or .cpu 1 }}" memory: "{{ or .memory 4 }}Gi" @@ -16,5 +16,4 @@ spec: series: - namingTemplate: "general-{{ .cpu }}c{{ .memory }}g" classes: - - args: [ "1", "4" ] - - args: [ "1", "16"] \ No newline at end of file + - args: [ "0.2", "0.6"] \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/02_vscale.yaml b/test/e2e/testdata/smoketest/wesql/03_vscale.yaml similarity index 86% rename from test/e2e/testdata/smoketest/wesql/02_vscale.yaml rename to test/e2e/testdata/smoketest/wesql/03_vscale.yaml index d869d2093..e88906fe4 100644 --- a/test/e2e/testdata/smoketest/wesql/02_vscale.yaml +++ b/test/e2e/testdata/smoketest/wesql/03_vscale.yaml @@ -7,5 +7,5 @@ spec: type: VerticalScaling verticalScaling: - componentName: mysql - class: general-1c4g + class: general-0.2c0.6g diff --git a/test/e2e/testdata/smoketest/wesql/03_hscale.yaml b/test/e2e/testdata/smoketest/wesql/04_hscale_up.yaml similarity index 89% rename from test/e2e/testdata/smoketest/wesql/03_hscale.yaml rename to test/e2e/testdata/smoketest/wesql/04_hscale_up.yaml index 592bdb325..1b50f73f9 100644 --- a/test/e2e/testdata/smoketest/wesql/03_hscale.yaml +++ b/test/e2e/testdata/smoketest/wesql/04_hscale_up.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: OpsRequest metadata: - name: ops-hscale + name: ops-hscale-up spec: clusterRef: mycluster type: HorizontalScaling diff --git a/test/e2e/testdata/smoketest/wesql/05_hscale_down.yaml b/test/e2e/testdata/smoketest/wesql/05_hscale_down.yaml new file mode 100644 index 000000000..85fb23836 --- /dev/null +++ b/test/e2e/testdata/smoketest/wesql/05_hscale_down.yaml @@ -0,0 +1,10 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: ops-hscale-down +spec: + clusterRef: mycluster + type: HorizontalScaling + horizontalScaling: + - componentName: mysql + replicas: 3 \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/04_vexpand.yaml b/test/e2e/testdata/smoketest/wesql/06_vexpand.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/04_vexpand.yaml rename to test/e2e/testdata/smoketest/wesql/06_vexpand.yaml diff --git a/test/e2e/testdata/smoketest/wesql/05_cv.yaml b/test/e2e/testdata/smoketest/wesql/07_cv.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/05_cv.yaml rename to test/e2e/testdata/smoketest/wesql/07_cv.yaml diff --git a/test/e2e/testdata/smoketest/wesql/06_upgrade.yaml b/test/e2e/testdata/smoketest/wesql/08_upgrade.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/06_upgrade.yaml rename to test/e2e/testdata/smoketest/wesql/08_upgrade.yaml diff --git a/test/e2e/testdata/smoketest/wesql/07_stop.yaml b/test/e2e/testdata/smoketest/wesql/09_stop.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/07_stop.yaml rename to test/e2e/testdata/smoketest/wesql/09_stop.yaml diff --git a/test/e2e/testdata/smoketest/wesql/08_start.yaml b/test/e2e/testdata/smoketest/wesql/10_start.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/08_start.yaml rename to test/e2e/testdata/smoketest/wesql/10_start.yaml diff --git a/test/e2e/testdata/smoketest/wesql/09_restart.yaml b/test/e2e/testdata/smoketest/wesql/11_restart.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/09_restart.yaml rename to test/e2e/testdata/smoketest/wesql/11_restart.yaml diff --git a/test/e2e/testdata/smoketest/wesql/12_backup_full.yaml b/test/e2e/testdata/smoketest/wesql/12_backup_full.yaml deleted file mode 100644 index 3b3b25e2e..000000000 --- a/test/e2e/testdata/smoketest/wesql/12_backup_full.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: dataprotection.kubeblocks.io/v1alpha1 -kind: Backup -metadata: - labels: - app.kubernetes.io/instance: mycluster - dataprotection.kubeblocks.io/backup-type: full - name: backup-full-mycluster -spec: - backupPolicyName: mycluster-mysql-backup-policy - backupType: full \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/10_reconfigure.yaml b/test/e2e/testdata/smoketest/wesql/12_reconfigure.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/10_reconfigure.yaml rename to test/e2e/testdata/smoketest/wesql/12_reconfigure.yaml diff --git a/test/e2e/testdata/smoketest/wesql/11_backup_snapshot.yaml b/test/e2e/testdata/smoketest/wesql/13_backup_snapshot.yaml similarity index 100% rename from test/e2e/testdata/smoketest/wesql/11_backup_snapshot.yaml rename to test/e2e/testdata/smoketest/wesql/13_backup_snapshot.yaml diff --git a/test/e2e/testdata/smoketest/wesql/14_backup_full_restore.yaml b/test/e2e/testdata/smoketest/wesql/14_backup_full_restore.yaml deleted file mode 100644 index cd8866635..000000000 --- a/test/e2e/testdata/smoketest/wesql/14_backup_full_restore.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: Cluster -metadata: - name: snapshot-mycluster - annotations: - kubeblocks.io/restore-from-backup: "{\"mysql\":\"backup-full-mycluster\"}" -spec: - clusterDefinitionRef: apecloud-mysql - clusterVersionRef: ac-mysql-8.0.30 - terminationPolicy: WipeOut - affinity: - topologyKeys: - - kubernetes.io/hostname - componentSpecs: - - name: mysql - componentDefRef: mysql - monitor: false - replicas: 3 - enabledLogs: [ "slow","error" ] - volumeClaimTemplates: - - name: data - spec: - storageClassName: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 2Gi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/wesql/13_backup_snapshot_restore.yaml b/test/e2e/testdata/smoketest/wesql/14_backup_snapshot_restore.yaml similarity index 86% rename from test/e2e/testdata/smoketest/wesql/13_backup_snapshot_restore.yaml rename to test/e2e/testdata/smoketest/wesql/14_backup_snapshot_restore.yaml index cb92e7076..b467267be 100644 --- a/test/e2e/testdata/smoketest/wesql/13_backup_snapshot_restore.yaml +++ b/test/e2e/testdata/smoketest/wesql/14_backup_snapshot_restore.yaml @@ -5,8 +5,8 @@ metadata: annotations: kubeblocks.io/restore-from-backup: "{\"mysql\":\"backup-sbapshot-mycluster\"}" spec: - clusterDefinitionRef: postgresql - clusterVersionRef: postgresql-15.2.0 + clusterDefinitionRef: apecloud-mysql + clusterVersionRef: ac-mysql-8.0.30-latest terminationPolicy: WipeOut componentSpecs: - name: wesql diff --git a/test/e2e/types.go b/test/e2e/types.go index d73f50e98..5af6572c9 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -28,3 +28,7 @@ var Ctx context.Context var Cancel context.CancelFunc var Logger logr.Logger var Version string +var Provider string +var Region string +var SecretID string +var SecretKey string diff --git a/test/e2e/util/smoke_util.go b/test/e2e/util/smoke_util.go index 2ebc2ca72..76129e749 100644 --- a/test/e2e/util/smoke_util.go +++ b/test/e2e/util/smoke_util.go @@ -18,9 +18,11 @@ package util import ( "bufio" + "bytes" "io" "log" "os" + executil "os/exec" "path/filepath" "strings" "sync" @@ -286,3 +288,32 @@ func CheckKbcliExists() error { _, err := exec.New().LookPath("kbcli") return err } + +func Check(command string, input string) (string, error) { + cmd := executil.Command("bash", "-c", command) + + var output bytes.Buffer + cmd.Stdout = &output + + inPipe, err := cmd.StdinPipe() + if err != nil { + return "", err + } + + err = cmd.Start() + if err != nil { + return "", err + } + + _, e := io.WriteString(inPipe, input) + if e != nil { + return "", e + } + + err = cmd.Wait() + if err != nil { + return "", err + } + + return output.String(), nil +} From 5acd27862fae2d14d2268d3f227a20278a59e826 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Fri, 21 Apr 2023 09:47:12 +0800 Subject: [PATCH 118/439] chore: check package version (#2810) --- .github/utils/utils.sh | 25 ++++++++++++++++++++ .github/workflows/package-version.yml | 33 +++++++++++++++++++++++++++ .github/workflows/release-version.yml | 10 ++------ 3 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/package-version.yml diff --git a/.github/utils/utils.sh b/.github/utils/utils.sh index 7be50ef4d..5953d730d 100644 --- a/.github/utils/utils.sh +++ b/.github/utils/utils.sh @@ -16,6 +16,7 @@ Usage: $(basename "$0") 4) get latest release tag 5) update release latest 6) get the ci trigger mode + 7) check package version -tn, --tag-name Release tag name -gr, --github-repo Github Repo -gt, --github-token Github token @@ -53,6 +54,9 @@ main() { 6) get_trigger_mode ;; + 7) + check_package_version + ;; *) show_help break @@ -165,4 +169,25 @@ get_trigger_mode() { echo $TRIGGER_MODE } +check_package_version() { + exit_status=0 + beta_tag="v"*"."*"."*"-beta."* + rc_tag="v"*"."*"."*"-rc."* + release_tag="v"*"."*"."* + not_release_tag="v"*"."*"."*"-"* + if [[ "$TAG_NAME" == $release_tag && "$TAG_NAME" != $not_release_tag ]]; then + echo "::error title=Release Version Not Allow::$(tput -T xterm setaf 1) $TAG_NAME does not allow packaging.$(tput -T xterm sgr0)" + exit_status=1 + elif [[ "$TAG_NAME" == $beta_tag ]]; then + echo "::error title=Beta Version Not Allow::$(tput -T xterm setaf 1) $TAG_NAME does not allow packaging.$(tput -T xterm sgr0)" + exit_status=1 + elif [[ "$TAG_NAME" == $rc_tag ]]; then + echo "::error title=Release Candidate Version Not Allow::$(tput -T xterm setaf 1) $TAG_NAME does not allow packaging.$(tput -T xterm sgr0)" + exit_status=1 + else + echo "$(tput -T xterm setaf 2)Version allows packaging$(tput -T xterm sgr0)" + fi + exit $exit_status +} + main "$@" diff --git a/.github/workflows/package-version.yml b/.github/workflows/package-version.yml new file mode 100644 index 000000000..afd1b2346 --- /dev/null +++ b/.github/workflows/package-version.yml @@ -0,0 +1,33 @@ +name: PACKAGE-VERSION + +on: + workflow_dispatch: + inputs: + release_version: + description: 'The tag name of release' + required: true + default: '' + +run-name: ref_name:${{ github.ref_name }} release_version:${{ inputs.release_version }} + +env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + + +jobs: + package-version: + runs-on: ubuntu-latest + steps: + - name: checkout branch ${{ github.ref_name }} + uses: actions/checkout@v3 + + - name: package check + run: | + bash .github/utils/utils.sh --type 7 --tag-name "${{ inputs.release_version }}" + + - name: push tag + uses: mathieudutour/github-tag-action@v6.1 + with: + custom_tag: ${{ inputs.release_version }} + github_token: ${{ env.GITHUB_TOKEN }} + tag_prefix: "" diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml index a2e44ee71..0b1469c08 100644 --- a/.github/workflows/release-version.yml +++ b/.github/workflows/release-version.yml @@ -7,10 +7,6 @@ on: description: 'The tag name of release' required: true default: '' - release_type: - description: 'The type of release (1: release 2: package)' - required: true - default: '1' run-name: ref_name:${{ github.ref_name }} release_version:${{ inputs.release_version }} @@ -22,10 +18,8 @@ jobs: release-test: runs-on: [ self-hosted, eks-fargate-runner ] steps: - - if: ${{ inputs.release_type == '1' }} - uses: apecloud/checkout@main - - if: ${{ inputs.release_type == '1' }} - name: vendor lint test + - uses: apecloud/checkout@main + - name: vendor lint test run: | mkdir -p ./bin cp -r /go/bin/controller-gen ./bin/controller-gen From e1ad20f40ac2ec72f04f96387f6aa3c8134c570d Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Fri, 21 Apr 2023 10:34:53 +0800 Subject: [PATCH 119/439] fix: vscale by resource does not work (#2748) --- .../componentclassdefinition_types.go | 9 ++ controllers/apps/class_controller_test.go | 21 +-- .../apps/operations/vertical_scaling.go | 4 +- .../apps/operations/vertical_scaling_test.go | 46 ++++--- .../apps/opsrequest_controller_test.go | 120 +++++++++++++----- internal/class/class_utils_test.go | 9 +- internal/class/suite_test.go | 16 +++ .../lifecycle/transformer_fill_class.go | 5 +- internal/testutil/apps/cluster_factory.go | 9 ++ .../apps/componentclassdefinition_factory.go | 26 ++-- .../componentresourceconstraint_factory.go | 29 +++-- internal/testutil/apps/constant.go | 70 ++++++---- 12 files changed, 246 insertions(+), 118 deletions(-) diff --git a/apis/apps/v1alpha1/componentclassdefinition_types.go b/apis/apps/v1alpha1/componentclassdefinition_types.go index 4347d6e72..bac5dcfe5 100644 --- a/apis/apps/v1alpha1/componentclassdefinition_types.go +++ b/apis/apps/v1alpha1/componentclassdefinition_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -175,3 +176,11 @@ type ComponentClassDefinitionList struct { func init() { SchemeBuilder.Register(&ComponentClassDefinition{}, &ComponentClassDefinitionList{}) } + +func (r *ComponentClass) ToResourceRequirements() corev1.ResourceRequirements { + requests := corev1.ResourceList{ + corev1.ResourceCPU: r.CPU, + corev1.ResourceMemory: r.Memory, + } + return corev1.ResourceRequirements{Requests: requests, Limits: requests} +} diff --git a/controllers/apps/class_controller_test.go b/controllers/apps/class_controller_test.go index 5f4d5d0f8..23a9b26aa 100644 --- a/controllers/apps/class_controller_test.go +++ b/controllers/apps/class_controller_test.go @@ -19,8 +19,6 @@ package apps import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/resource" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -50,29 +48,18 @@ var _ = Describe("", func() { AfterEach(cleanEnv) It("Class should exist in status", func() { - var ( - clsName = "test" - class = v1alpha1.ComponentClass{ - Name: clsName, - CPU: resource.MustParse("1"), - Memory: resource.MustParse("1Gi"), - } - ) - - constraint := testapps.NewComponentResourceConstraintFactory(testapps.DefaultGeneralResourceConstraintName). - AddConstraints(testapps.ResourceConstraintNormal). - AddConstraints(testapps.ResourceConstraintSpecial). + constraint := testapps.NewComponentResourceConstraintFactory(testapps.DefaultResourceConstraintName). + AddConstraints(testapps.GeneralResourceConstraint). Create(&testCtx).GetObject() componentClassDefinition = testapps.NewComponentClassDefinitionFactory("custom", "apecloud-mysql", "mysql"). - AddClassGroup(constraint.Name). - AddClasses([]v1alpha1.ComponentClass{class}). + AddClasses(constraint.Name, []string{testapps.Class1c1gName}). Create(&testCtx).GetObject() key := client.ObjectKeyFromObject(componentClassDefinition) Eventually(testapps.CheckObj(&testCtx, key, func(g Gomega, pobj *v1alpha1.ComponentClassDefinition) { g.Expect(pobj.Status.Classes).ShouldNot(BeEmpty()) - g.Expect(pobj.Status.Classes[0].Name).Should(Equal(clsName)) + g.Expect(pobj.Status.Classes[0].Name).Should(Equal(testapps.Class1c1gName)) })).Should(Succeed()) }) }) diff --git a/controllers/apps/operations/vertical_scaling.go b/controllers/apps/operations/vertical_scaling.go index b61aa8d4b..a76e2ad41 100644 --- a/controllers/apps/operations/vertical_scaling.go +++ b/controllers/apps/operations/vertical_scaling.go @@ -62,6 +62,8 @@ func (vs verticalScalingHandler) Action(reqCtx intctrlutil.RequestCtx, cli clien if verticalScaling.Class != "" { component.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: verticalScaling.Class} } else { + // clear old class ref + component.ClassDefRef = &appsv1alpha1.ClassDefRef{} component.Resources = verticalScaling.ResourceRequirements } opsRes.Cluster.Spec.ComponentSpecs[index] = component @@ -104,7 +106,7 @@ func (vs verticalScalingHandler) GetRealAffectedComponentMap(opsRequest *appsv1a if !ok { continue } - if !reflect.DeepEqual(currVs.ResourceRequirements, v.ResourceRequirements) { + if !reflect.DeepEqual(currVs.ResourceRequirements, v.ResourceRequirements) || currVs.Class != v.Class { realChangedMap[k] = struct{}{} } } diff --git a/controllers/apps/operations/vertical_scaling_test.go b/controllers/apps/operations/vertical_scaling_test.go index 393e6e475..70305f071 100644 --- a/controllers/apps/operations/vertical_scaling_test.go +++ b/controllers/apps/operations/vertical_scaling_test.go @@ -60,8 +60,8 @@ var _ = Describe("VerticalScaling OpsRequest", func() { AfterEach(cleanEnv) Context("Test OpsRequest", func() { - It("Test verticalScaling OpsRequest", func() { + testVerticalScaling := func(verticalScaling []appsv1alpha1.VerticalScaling) { By("init operations resources ") reqCtx := intctrlutil.RequestCtx{Ctx: ctx} opsRes, _, _ := initOperationsResources(clusterDefinitionName, clusterVersionName, clusterName) @@ -69,7 +69,26 @@ var _ = Describe("VerticalScaling OpsRequest", func() { By("create VerticalScaling ops") ops := testapps.NewOpsRequestObj("verticalscaling-ops-"+randomStr, testCtx.DefaultNamespace, clusterName, appsv1alpha1.VerticalScalingType) - ops.Spec.VerticalScalingList = []appsv1alpha1.VerticalScaling{ + + ops.Spec.VerticalScalingList = verticalScaling + opsRes.OpsRequest = testapps.CreateOpsRequest(ctx, testCtx, ops) + By("test save last configuration and OpsRequest phase is Running") + _, err := GetOpsManager().Do(reqCtx, k8sClient, opsRes) + Expect(err).ShouldNot(HaveOccurred()) + Eventually(testapps.GetOpsRequestPhase(&testCtx, client.ObjectKeyFromObject(ops))).Should(Equal(appsv1alpha1.OpsCreatingPhase)) + + By("test vertical scale action function") + vsHandler := verticalScalingHandler{} + Expect(vsHandler.Action(reqCtx, k8sClient, opsRes)).Should(Succeed()) + _, _, err = vsHandler.ReconcileAction(reqCtx, k8sClient, opsRes) + Expect(err == nil).Should(BeTrue()) + + By("test GetRealAffectedComponentMap function") + Expect(len(vsHandler.GetRealAffectedComponentMap(opsRes.OpsRequest))).Should(Equal(1)) + } + + It("vertical scaling by resource", func() { + verticalScaling := []appsv1alpha1.VerticalScaling{ { ComponentOps: appsv1alpha1.ComponentOps{ComponentName: consensusComp}, ResourceRequirements: corev1.ResourceRequirements{ @@ -84,20 +103,17 @@ var _ = Describe("VerticalScaling OpsRequest", func() { }, }, } - opsRes.OpsRequest = testapps.CreateOpsRequest(ctx, testCtx, ops) - By("test save last configuration and OpsRequest phase is Running") - _, err := GetOpsManager().Do(reqCtx, k8sClient, opsRes) - Expect(err).ShouldNot(HaveOccurred()) - Eventually(testapps.GetOpsRequestPhase(&testCtx, client.ObjectKeyFromObject(ops))).Should(Equal(appsv1alpha1.OpsCreatingPhase)) - - By("test vertical scale action function") - vsHandler := verticalScalingHandler{} - Expect(vsHandler.Action(reqCtx, k8sClient, opsRes)).Should(Succeed()) - _, _, err = vsHandler.ReconcileAction(reqCtx, k8sClient, opsRes) - Expect(err == nil).Should(BeTrue()) + testVerticalScaling(verticalScaling) + }) - By("test GetRealAffectedComponentMap function") - Expect(len(vsHandler.GetRealAffectedComponentMap(opsRes.OpsRequest))).Should(Equal(1)) + It("vertical scaling by class", func() { + verticalScaling := []appsv1alpha1.VerticalScaling{ + { + ComponentOps: appsv1alpha1.ComponentOps{ComponentName: consensusComp}, + Class: testapps.Class1c1gName, + }, + } + testVerticalScaling(verticalScaling) }) }) }) diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index aa057388f..74dea9af8 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -66,6 +66,8 @@ var _ = Describe("OpsRequest Controller", func() { // non-namespaced testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, ml) + testapps.ClearResources(&testCtx, intctrlutil.ComponentResourceConstraintSignature, ml) + testapps.ClearResources(&testCtx, intctrlutil.ComponentClassDefinitionSignature, ml) } BeforeEach(func() { @@ -126,26 +128,42 @@ var _ = Describe("OpsRequest Controller", func() { })()).ShouldNot(HaveOccurred()) } - testVerticalScaleCPUAndMemory := func(workloadType testapps.ComponentDefTplType) { + type resourceContext struct { + class *appsv1alpha1.ComponentClass + resource corev1.ResourceRequirements + } + + type verticalScalingContext struct { + description string + source resourceContext + target resourceContext + } + + testVerticalScaleCPUAndMemory := func(workloadType testapps.ComponentDefTplType, scalingCtx verticalScalingContext) { const opsName = "mysql-verticalscaling" + By(scalingCtx.description) + + By("Create class related objects") + constraint := testapps.NewComponentResourceConstraintFactory(testapps.DefaultResourceConstraintName). + AddConstraints(testapps.GeneralResourceConstraint). + Create(&testCtx).GetObject() + + testapps.NewComponentClassDefinitionFactory("custom", clusterDefObj.Name, mysqlCompDefName). + AddClasses(constraint.Name, []string{testapps.Class1c1gName, testapps.Class2c4gName}). + Create(&testCtx) + By("Create a cluster obj") - resources := corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - "cpu": resource.MustParse("800m"), - "memory": resource.MustParse("512Mi"), - }, - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("500m"), - "memory": resource.MustParse("256Mi"), - }, - } - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterFactory := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(mysqlCompName, mysqlCompDefName). - SetReplicas(1). - SetResources(resources). - Create(&testCtx).GetObject() + SetReplicas(1) + if scalingCtx.source.class != nil { + clusterFactory.SetClassDefRef(&appsv1alpha1.ClassDefRef{Class: scalingCtx.source.class.Name}) + } else { + clusterFactory.SetResources(scalingCtx.source.resource) + } + clusterObj = clusterFactory.Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster enter running phase") @@ -173,16 +191,20 @@ var _ = Describe("OpsRequest Controller", func() { opsKey := types.NamespacedName{Name: opsName, Namespace: testCtx.DefaultNamespace} verticalScalingOpsRequest := testapps.NewOpsRequestObj(opsKey.Name, opsKey.Namespace, clusterObj.Name, appsv1alpha1.VerticalScalingType) - verticalScalingOpsRequest.Spec.VerticalScalingList = []appsv1alpha1.VerticalScaling{ - { - ComponentOps: appsv1alpha1.ComponentOps{ComponentName: mysqlCompName}, - ResourceRequirements: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - "cpu": resource.MustParse("400m"), - "memory": resource.MustParse("300Mi"), - }, + if scalingCtx.target.class != nil { + verticalScalingOpsRequest.Spec.VerticalScalingList = []appsv1alpha1.VerticalScaling{ + { + ComponentOps: appsv1alpha1.ComponentOps{ComponentName: mysqlCompName}, + Class: scalingCtx.target.class.Name, + }, + } + } else { + verticalScalingOpsRequest.Spec.VerticalScalingList = []appsv1alpha1.VerticalScaling{ + { + ComponentOps: appsv1alpha1.ComponentOps{ComponentName: mysqlCompName}, + ResourceRequirements: scalingCtx.target.resource, }, - }, + } } Expect(testCtx.CreateObj(testCtx.Ctx, verticalScalingOpsRequest)).Should(Succeed()) @@ -218,9 +240,17 @@ var _ = Describe("OpsRequest Controller", func() { Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsSucceedPhase)) By("check cluster resource requirements changed") + var targetRequests corev1.ResourceList + if scalingCtx.target.class != nil { + targetRequests = corev1.ResourceList{ + corev1.ResourceCPU: scalingCtx.target.class.CPU, + corev1.ResourceMemory: scalingCtx.target.class.Memory, + } + } else { + targetRequests = scalingCtx.target.resource.Requests + } Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, fetched *appsv1alpha1.Cluster) { - g.Expect(fetched.Spec.ComponentSpecs[0].Resources.Requests).To(Equal( - verticalScalingOpsRequest.Spec.VerticalScalingList[0].Requests)) + g.Expect(fetched.Spec.ComponentSpecs[0].Resources.Requests).To(Equal(targetRequests)) })).Should(Succeed()) By("check OpsRequest reclaimed after ttl") @@ -247,8 +277,36 @@ var _ = Describe("OpsRequest Controller", func() { Create(&testCtx).GetObject() }) - It("issue an VerticalScalingOpsRequest should change Cluster's resource requirements successfully", func() { - testVerticalScaleCPUAndMemory(testapps.StatefulMySQLComponent) + It("create cluster by class, vertical scaling by resource", func() { + ctx := verticalScalingContext{ + source: resourceContext{class: &testapps.Class1c1g}, + target: resourceContext{resource: testapps.Class2c4g.ToResourceRequirements()}, + } + testVerticalScaleCPUAndMemory(testapps.StatefulMySQLComponent, ctx) + }) + + It("create cluster by class, vertical scaling by class", func() { + ctx := verticalScalingContext{ + source: resourceContext{class: &testapps.Class1c1g}, + target: resourceContext{class: &testapps.Class2c4g}, + } + testVerticalScaleCPUAndMemory(testapps.StatefulMySQLComponent, ctx) + }) + + It("create cluster by resource, vertical scaling by class", func() { + ctx := verticalScalingContext{ + source: resourceContext{resource: testapps.Class1c1g.ToResourceRequirements()}, + target: resourceContext{class: &testapps.Class2c4g}, + } + testVerticalScaleCPUAndMemory(testapps.StatefulMySQLComponent, ctx) + }) + + It("create cluster by resource, vertical scaling by resource", func() { + ctx := verticalScalingContext{ + source: resourceContext{resource: testapps.Class1c1g.ToResourceRequirements()}, + target: resourceContext{resource: testapps.Class2c4g.ToResourceRequirements()}, + } + testVerticalScaleCPUAndMemory(testapps.StatefulMySQLComponent, ctx) }) }) @@ -269,7 +327,11 @@ var _ = Describe("OpsRequest Controller", func() { }) It("issue an VerticalScalingOpsRequest should change Cluster's resource requirements successfully", func() { - testVerticalScaleCPUAndMemory(testapps.ConsensusMySQLComponent) + ctx := verticalScalingContext{ + source: resourceContext{class: &testapps.Class1c1g}, + target: resourceContext{resource: testapps.Class2c4g.ToResourceRequirements()}, + } + testVerticalScaleCPUAndMemory(testapps.ConsensusMySQLComponent, ctx) }) It("HorizontalScaling when not support snapshot", func() { diff --git a/internal/class/class_utils_test.go b/internal/class/class_utils_test.go index 9771d1cd4..60470d137 100644 --- a/internal/class/class_utils_test.go +++ b/internal/class/class_utils_test.go @@ -110,19 +110,14 @@ var _ = Describe("utils", func() { It("should succeed", func() { var ( err error - specClassName = "general-1c1g" + specClassName = testapps.Class1c1gName statusClassName = "general-100c100g" compClasses map[string]map[string]*v1alpha1.ComponentClassInstance compType = "mysql" ) classDef := testapps.NewComponentClassDefinitionFactory("custom", "apecloud-mysql", compType). - AddClassGroup(testapps.DefaultGeneralResourceConstraintName). - AddClasses([]v1alpha1.ComponentClass{{ - Name: specClassName, - CPU: resource.MustParse("1"), - Memory: resource.MustParse("1Gi"), - }}). + AddClasses(testapps.DefaultResourceConstraintName, []string{specClassName}). GetObject() By("class definition status is out of date") diff --git a/internal/class/suite_test.go b/internal/class/suite_test.go index 68df942fd..00be59db9 100644 --- a/internal/class/suite_test.go +++ b/internal/class/suite_test.go @@ -1,3 +1,19 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package class import ( diff --git a/internal/controller/lifecycle/transformer_fill_class.go b/internal/controller/lifecycle/transformer_fill_class.go index 886ea129a..969527da1 100644 --- a/internal/controller/lifecycle/transformer_fill_class.go +++ b/internal/controller/lifecycle/transformer_fill_class.go @@ -148,7 +148,7 @@ func (r *fillClass) fillClass(reqCtx intctrlutil.RequestCtx, cluster *appsv1alph func buildVolumeClaimByClass(cls *appsv1alpha1.ComponentClassInstance) []appsv1alpha1.ClusterComponentVolumeClaimTemplate { var volumes []appsv1alpha1.ClusterComponentVolumeClaimTemplate for _, volume := range cls.Volumes { - volume := appsv1alpha1.ClusterComponentVolumeClaimTemplate{ + volumes = append(volumes, appsv1alpha1.ClusterComponentVolumeClaimTemplate{ Name: volume.Name, Spec: appsv1alpha1.PersistentVolumeClaimSpec{ // TODO define access mode in class @@ -159,8 +159,7 @@ func buildVolumeClaimByClass(cls *appsv1alpha1.ComponentClassInstance) []appsv1a }, }, }, - } - volumes = append(volumes, volume) + }) } return volumes } diff --git a/internal/testutil/apps/cluster_factory.go b/internal/testutil/apps/cluster_factory.go index 8566c3e0d..a241097ac 100644 --- a/internal/testutil/apps/cluster_factory.go +++ b/internal/testutil/apps/cluster_factory.go @@ -104,6 +104,15 @@ func (factory *MockClusterFactory) SetEnabledLogs(logName ...string) *MockCluste return factory } +func (factory *MockClusterFactory) SetClassDefRef(classDefRef *appsv1alpha1.ClassDefRef) *MockClusterFactory { + comps := factory.get().Spec.ComponentSpecs + if len(comps) > 0 { + comps[len(comps)-1].ClassDefRef = classDefRef + } + factory.get().Spec.ComponentSpecs = comps + return factory +} + func (factory *MockClusterFactory) AddComponentToleration(toleration corev1.Toleration) *MockClusterFactory { comps := factory.get().Spec.ComponentSpecs if len(comps) > 0 { diff --git a/internal/testutil/apps/componentclassdefinition_factory.go b/internal/testutil/apps/componentclassdefinition_factory.go index 166ea24a1..2044b7f05 100644 --- a/internal/testutil/apps/componentclassdefinition_factory.go +++ b/internal/testutil/apps/componentclassdefinition_factory.go @@ -42,20 +42,20 @@ func NewComponentClassDefinitionFactory(name, clusterDefinitionRef, componentTyp return f } -func (factory *MockComponentClassDefinitionFactory) AddClassGroup(constraintRef string) *MockComponentClassDefinitionFactory { - groups := factory.get().Spec.Groups - group := classGroupTemplate - group.ResourceConstraintRef = constraintRef - groups = append(groups, group) - factory.get().Spec.Groups = groups - return factory -} - -func (factory *MockComponentClassDefinitionFactory) AddClasses(classes []appsv1alpha1.ComponentClass) *MockComponentClassDefinitionFactory { - groups := factory.get().Spec.Groups - if len(groups) > 0 { - groups[len(groups)-1].Series[0].Classes = classes +func (factory *MockComponentClassDefinitionFactory) AddClasses(constraintRef string, classNames []string) *MockComponentClassDefinitionFactory { + var classes []appsv1alpha1.ComponentClass + for _, name := range classNames { + classes = append(classes, DefaultClasses[name]) } + groups := factory.get().Spec.Groups + groups = append(groups, appsv1alpha1.ComponentClassGroup{ + ResourceConstraintRef: constraintRef, + Series: []appsv1alpha1.ComponentClassSeries{ + { + Classes: classes, + }, + }, + }) factory.get().Spec.Groups = groups return factory } diff --git a/internal/testutil/apps/componentresourceconstraint_factory.go b/internal/testutil/apps/componentresourceconstraint_factory.go index 1c067755e..879be88be 100644 --- a/internal/testutil/apps/componentresourceconstraint_factory.go +++ b/internal/testutil/apps/componentresourceconstraint_factory.go @@ -26,10 +26,10 @@ import ( type ResourceConstraintTplType string const ( - ResourceConstraintSpecial ResourceConstraintTplType = "special" - ResourceConstraintNormal ResourceConstraintTplType = "normal" + GeneralResourceConstraint ResourceConstraintTplType = "general" + MemoryOptimizedResourceConstraint ResourceConstraintTplType = "memory-optimized" - specialConstraintTemplate = ` + generalResourceConstraintTemplate = ` - cpu: min: 0.5 max: 2 @@ -41,13 +41,24 @@ const ( max: 2 memory: sizePerCPU: 2Gi -` - normalConstraintTemplate = ` - cpu: slots: [2, 4, 8, 16, 24, 32, 48, 64, 96, 128] memory: sizePerCPU: 4Gi ` + + memoryResourceConstraintTemplate = ` +- cpu: + slots: [2, 4, 8, 12, 24, 48] + memory: + sizePerCPU: 8Gi +- cpu: + min: 2 + max: 128 + step: 2 + memory: + sizePerCPU: 16Gi +` ) type MockComponentResourceConstraintFactory struct { @@ -74,10 +85,10 @@ func (factory *MockComponentResourceConstraintFactory) AddConstraints(constraint constraints = factory.get().Spec.Constraints ) switch constraintTplType { - case ResourceConstraintSpecial: - tpl = specialConstraintTemplate - case ResourceConstraintNormal: - tpl = normalConstraintTemplate + case GeneralResourceConstraint: + tpl = generalResourceConstraintTemplate + case MemoryOptimizedResourceConstraint: + tpl = memoryResourceConstraintTemplate } if err := yaml.Unmarshal([]byte(tpl), &newConstraints); err != nil { panic(err) diff --git a/internal/testutil/apps/constant.go b/internal/testutil/apps/constant.go index c9c197c4d..b83c69ce4 100644 --- a/internal/testutil/apps/constant.go +++ b/internal/testutil/apps/constant.go @@ -18,6 +18,7 @@ package apps import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -25,17 +26,15 @@ import ( ) const ( - KubeBlocks = "kubeblocks" - LogVolumeName = "log" - ConfVolumeName = "conf" - DataVolumeName = "data" - ScriptsVolumeName = "scripts" - ServiceDefaultName = "" - ServiceHeadlessName = "headless" - ServiceVPCName = "vpc-lb" - ServiceInternetName = "internet-lb" - DefaultGeneralResourceConstraintName = "kb-resource-constraint-general" - DefaultMemoryOptimizedResourceConstraintName = "kb-resource-constraint-memory-optimized" + KubeBlocks = "kubeblocks" + LogVolumeName = "log" + ConfVolumeName = "conf" + DataVolumeName = "data" + ScriptsVolumeName = "scripts" + ServiceDefaultName = "" + ServiceHeadlessName = "headless" + ServiceVPCName = "vpc-lb" + ServiceInternetName = "internet-lb" ReplicationPodRoleVolume = "pod-role" ReplicationRoleLabelFieldPath = "metadata.labels['kubeblocks.io/role']" @@ -55,6 +54,10 @@ const ( DefaultRedisImageName = "redis:7.0.5" DefaultRedisContainerName = "redis" DefaultRedisInitContainerName = "redis-init-container" + + Class1c1gName = "general-1c1g" + Class2c4gName = "general-2c4g" + DefaultResourceConstraintName = "kb-resource-constraint" ) var ( @@ -289,21 +292,40 @@ var ( }, } - classGroupTemplate = appsv1alpha1.ComponentClassGroup{ - Template: ` -cpu: "{{ or .cpu 1 }}" -memory: "{{ or .memory 4 }}Gi" -volumes: -- name: data - size: "{{ or .dataStorageSize 10 }}Gi" -- name: log - size: "{{ or .logStorageSize 1 }}Gi" -`, - Vars: []string{"cpu", "memory", "dataStorageSize", "logStorageSize"}, - Series: []appsv1alpha1.ComponentClassSeries{ + Class1c1g = appsv1alpha1.ComponentClass{ + Name: Class1c1gName, + CPU: resource.MustParse("1"), + Memory: resource.MustParse("1Gi"), + Volumes: []appsv1alpha1.Volume{ + { + Name: "data", + Size: resource.MustParse("20Gi"), + }, + { + Name: "log", + Size: resource.MustParse("10Gi"), + }, + }, + } + + Class2c4g = appsv1alpha1.ComponentClass{ + Name: Class2c4gName, + CPU: resource.MustParse("2"), + Memory: resource.MustParse("4Gi"), + Volumes: []appsv1alpha1.Volume{ + { + Name: "data", + Size: resource.MustParse("20Gi"), + }, { - NamingTemplate: "custom-{{ .cpu }}c{{ .memory }}g", + Name: "log", + Size: resource.MustParse("10Gi"), }, }, } + + DefaultClasses = map[string]appsv1alpha1.ComponentClass{ + Class1c1gName: Class1c1g, + Class2c4gName: Class2c4g, + } ) From 2cf31cdccb39820fca474237bd0e78292f412a96 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Fri, 21 Apr 2023 14:06:38 +0800 Subject: [PATCH 120/439] chore: unify the path of backup dir (#2819) --- controllers/dataprotection/backup_controller.go | 9 +++++---- controllers/dataprotection/backup_controller_test.go | 2 +- deploy/helm/values.yaml | 2 +- deploy/mongodb/templates/backuptool.yaml | 3 +-- deploy/postgresql/templates/backuptool-pitr.yaml | 8 +++----- deploy/postgresql/templates/backuptool.yaml | 8 ++++---- 6 files changed, 15 insertions(+), 17 deletions(-) diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 1307f5741..5f4bb994b 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -319,11 +319,12 @@ func (r *BackupReconciler) buildAutoCreationAnnotations(backupPolicyName string) } // getBackupPathPrefix gets the backup path prefix. -func (r *BackupReconciler) getBackupPathPrefix(backupNamespace, pathPrefix string) string { +func (r *BackupReconciler) getBackupPathPrefix(req ctrl.Request, pathPrefix string) string { + pathPrefix = strings.TrimRight(pathPrefix, "/") if strings.TrimSpace(pathPrefix) == "" || strings.HasPrefix(pathPrefix, "/") { - return fmt.Sprintf("/%s%s", backupNamespace, pathPrefix) + return fmt.Sprintf("/%s%s/%s", req.Namespace, pathPrefix, req.Name) } - return fmt.Sprintf("/%s/%s", backupNamespace, pathPrefix) + return fmt.Sprintf("/%s/%s/%s", req.Namespace, pathPrefix, req.Name) } func (r *BackupReconciler) doInProgressPhaseAction( @@ -395,7 +396,7 @@ func (r *BackupReconciler) doInProgressPhaseAction( if err = r.createUpdatesJobs(reqCtx, backup, &commonPolicy.BasePolicy, dataprotectionv1alpha1.PRE); err != nil { r.Recorder.Event(backup, corev1.EventTypeNormal, "CreatedPreUpdatesJob", err.Error()) } - pathPrefix := r.getBackupPathPrefix(backup.Namespace, backupPolicy.Annotations[constant.BackupDataPathPrefixAnnotationKey]) + pathPrefix := r.getBackupPathPrefix(reqCtx.Req, backupPolicy.Annotations[constant.BackupDataPathPrefixAnnotationKey]) err = r.createBackupToolJob(reqCtx, backup, commonPolicy, pathPrefix) if err != nil { return r.updateStatusIfFailed(reqCtx, backup, err) diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index 28385b1b5..9df8730db 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -375,7 +375,7 @@ var _ = Describe("Backup Controller test", func() { By("Check backup job completed") Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *dataprotectionv1alpha1.Backup) { g.Expect(fetched.Status.Phase).To(Equal(dataprotectionv1alpha1.BackupCompleted)) - g.Expect(fetched.Status.Manifests.BackupTool.FilePath).To(Equal(fmt.Sprintf("/%s%s", backupKey.Namespace, pathPrefix))) + g.Expect(fetched.Status.Manifests.BackupTool.FilePath).To(Equal(fmt.Sprintf("/%s%s/%s", backupKey.Namespace, pathPrefix, backupKey.Name))) })).Should(Succeed()) }) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 795cfcf05..3f70d6fa1 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1662,7 +1662,7 @@ csi-s3: storageClass: create: true singleBucket: "" - mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 --region cn-northwest-1" + mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" csi-oss: enabled: false diff --git a/deploy/mongodb/templates/backuptool.yaml b/deploy/mongodb/templates/backuptool.yaml index 537c8fe45..9cff5e9c1 100644 --- a/deploy/mongodb/templates/backuptool.yaml +++ b/deploy/mongodb/templates/backuptool.yaml @@ -12,7 +12,7 @@ spec: cpu: "1" memory: 2Gi requests: - cpu: "1" + cpu: "500m" memory: 128Mi env: - name: DATA_DIR @@ -52,7 +52,6 @@ spec: incrementalRestoreCommands: [] backupCommands: - | - set -e mkdir -p ${BACKUP_DIR}/${BACKUP_NAME} cp -R ${DATA_DIR}/* ${BACKUP_DIR}/${BACKUP_NAME}/ cd ${BACKUP_DIR} diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml index d3d9a4c84..ae4092cb9 100644 --- a/deploy/postgresql/templates/backuptool-pitr.yaml +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -27,8 +27,6 @@ spec: value: $(VOLUME_DATA_DIR)/pgroot/arcwal - name: LOG_DIR value: $(VOLUME_DATA_DIR)/pgroot/data/pg_wal - - name: REMOTE_LOG_DIR - value: $(BACKUP_DIR)/$(BACKUP_NAME) image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} logical: restoreCommands: @@ -41,7 +39,7 @@ spec: set -e; mkdir -p ${PITR_DIR}; cd ${PITR_DIR} - for i in $(find ${REMOTE_LOG_DIR} -name "*.gz"); do + for i in $(find ${BACKUP_DIR} -name "*.gz"); do cp ${i} $(basename $i) gzip -df $(basename $i); done @@ -60,9 +58,9 @@ spec: backupCommands: - | set -e; - EXPIRED_INCR_LOG=${REMOTE_LOG_DIR}/$(date -d"7 day ago" +%Y%m%d); + EXPIRED_INCR_LOG=${BACKUP_DIR}/$(date -d"7 day ago" +%Y%m%d); if [ -d ${EXPIRED_INCR_LOG} ]; then rm -rf ${EXPIRED_INCR_LOG}; fi - TODAY_INCR_LOG=${REMOTE_LOG_DIR}/$(date +%Y%m%d); + TODAY_INCR_LOG=${BACKUP_DIR}/$(date +%Y%m%d); mkdir -p ${TODAY_INCR_LOG}; for i in $(find ${ARCHIVE_LOG_DIR} -name "*.gz"); do mv -f ${i} ${TODAY_INCR_LOG}/; diff --git a/deploy/postgresql/templates/backuptool.yaml b/deploy/postgresql/templates/backuptool.yaml index 795e3ec26..1028bf0bf 100644 --- a/deploy/postgresql/templates/backuptool.yaml +++ b/deploy/postgresql/templates/backuptool.yaml @@ -13,7 +13,7 @@ spec: cpu: "1" memory: 2Gi requests: - cpu: "1" + cpu: "500m" memory: 128Mi env: - name: RESTORE_DATA_DIR @@ -41,7 +41,7 @@ spec: # extract the data file to the temporary data directory mkdir -p ${TMP_DATA_DIR} && mkdir -p ${TMP_ARCH_DATA_DIR} rm -rf ${TMP_ARCH_DATA_DIR}/* && rm -rf ${TMP_DATA_DIR}/* - cd ${BACKUP_DIR}/${BACKUP_NAME} + cd ${BACKUP_DIR} tar -xvf base.tar.gz -C ${TMP_DATA_DIR}/ tar -xvf pg_wal.tar.gz -C ${TMP_ARCH_DATA_DIR}/ echo "done!" @@ -52,6 +52,6 @@ spec: backupCommands: - > set -e; - mkdir -p ${BACKUP_DIR}/${BACKUP_NAME}/; - echo ${DB_PASSWORD} | pg_basebackup -Ft -Pv -Xs -z -D ${BACKUP_DIR}/${BACKUP_NAME} -Z5 -h ${DB_HOST} -U standby -W; + mkdir -p ${BACKUP_DIR}; + echo ${DB_PASSWORD} | pg_basebackup -Ft -Pv -Xs -z -D ${BACKUP_DIR} -Z5 -h ${DB_HOST} -U standby -W; incrementalBackupCommands: [] From 32b22d04e2773bafdd50bfc4b147ac26f9f0fd1f Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Fri, 21 Apr 2023 14:27:33 +0800 Subject: [PATCH 121/439] feat: enable golang race detector in makefile (#2799) Co-authored-by: huyongqii Co-authored-by: huyongqii --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index c82759753..84d3e7a35 100644 --- a/Makefile +++ b/Makefile @@ -208,6 +208,10 @@ test-fast: envtest .PHONY: test test: manifests generate test-go-generate fmt vet add-k8s-host test-fast ## Run tests. if existing k8s cluster is k3d or minikube, specify EXISTING_CLUSTER_TYPE. +.PHONY: race +race: + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GO) test -race $(TEST_PACKAGES) + .PHONY: test-integration test-integration: manifests generate fmt vet envtest add-k8s-host ## Run tests. if existing k8s cluster is k3d or minikube, specify EXISTING_CLUSTER_TYPE. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" $(GO) test ./test/integration From 3c719709e9b84417147cc38b3198f07bf0e3d1c9 Mon Sep 17 00:00:00 2001 From: "zheyi.cqy" Date: Fri, 21 Apr 2023 15:51:04 +0800 Subject: [PATCH 122/439] chore: migration support add-on (#2826) --- .../templates/addons/migration-addon.yaml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 deploy/helm/templates/addons/migration-addon.yaml diff --git a/deploy/helm/templates/addons/migration-addon.yaml b/deploy/helm/templates/addons/migration-addon.yaml new file mode 100644 index 000000000..e324816a7 --- /dev/null +++ b/deploy/helm/templates/addons/migration-addon.yaml @@ -0,0 +1,24 @@ +apiVersion: extensions.kubeblocks.io/v1alpha1 +kind: Addon +metadata: + name: migration + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} + "kubeblocks.io/provider": community + {{- if .Values.keepAddons }} + annotations: + helm.sh/resource-policy: keep + {{- end }} +spec: + description: 'Migration is a tool for migrating data between two databases.' + + type: Helm + + helm: + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/dt-platform-0.1.0-test.3.tgz + + installable: + autoInstall: false + + defaultInstallValues: + - enabled: true From 843dc1028b8fb3a3178aa41069d4dca844db1df4 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Fri, 21 Apr 2023 16:34:37 +0800 Subject: [PATCH 123/439] fix: set driver for bench config (#2804) --- internal/cli/cmd/bench/tpcc.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/cli/cmd/bench/tpcc.go b/internal/cli/cmd/bench/tpcc.go index fac7c0560..a4a48995c 100644 --- a/internal/cli/cmd/bench/tpcc.go +++ b/internal/cli/cmd/bench/tpcc.go @@ -113,6 +113,7 @@ func executeTpcc(action string) error { tpccConfig.DBName = dbName tpccConfig.Threads = threads tpccConfig.Isolation = isolationLevel + tpccConfig.Driver = driver switch tpccConfig.OutputType { case "csv", "CSV": From 98086ec30d452086647c713938a6a7dc63bb4e4e Mon Sep 17 00:00:00 2001 From: "yunju.lly" Date: Fri, 21 Apr 2023 17:04:33 +0800 Subject: [PATCH 124/439] fix(cli): cluster update enable-all-logs panic (#2829) --- deploy/postgresql/config/pg14-config.tpl | 12 +++++++++--- deploy/postgresql/values.yaml | 2 +- internal/cli/cmd/cluster/update.go | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deploy/postgresql/config/pg14-config.tpl b/deploy/postgresql/config/pg14-config.tpl index 49f64f6dc..0288fcf3b 100644 --- a/deploy/postgresql/config/pg14-config.tpl +++ b/deploy/postgresql/config/pg14-config.tpl @@ -57,16 +57,22 @@ idle_in_transaction_session_timeout = '1h' listen_addresses = '0.0.0.0' log_autovacuum_min_duration = '1s' log_checkpoints = 'True' -{{ block "logsBlock" . }} + +{{- block "logsBlock" . }} +{{- if hasKey $.component "enabledLogs" }} +{{- if mustHas "running" $.component.enabledLogs }} +logging_collector = 'True' log_destination = 'csvlog' log_directory = 'log' log_filename = 'postgresql-%Y-%m-%d.log' -{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + log_lock_waits = 'True' log_min_duration_statement = '100' log_replication_commands = 'True' log_statement = 'ddl' -logging_collector = 'True' #maintenance_work_mem = '3952MB' max_connections = '{{ $max_connections }}' max_locks_per_transaction = '128' diff --git a/deploy/postgresql/values.yaml b/deploy/postgresql/values.yaml index cc18b2d0d..a15dee0c3 100644 --- a/deploy/postgresql/values.yaml +++ b/deploy/postgresql/values.yaml @@ -435,4 +435,4 @@ metrics: usage: "COUNTER" description: "Total amount of WAL generated by the statement in bytes" logConfigs: - running: /postgresql/data/log/postgresql-* \ No newline at end of file + running: /home/postgres/pgdata/pgroot/data/log/postgresql-* \ No newline at end of file diff --git a/internal/cli/cmd/cluster/update.go b/internal/cli/cmd/cluster/update.go index 7424d94a7..3810b269e 100644 --- a/internal/cli/cmd/cluster/update.go +++ b/internal/cli/cmd/cluster/update.go @@ -312,7 +312,7 @@ func (o *updateOptions) reconfigureLogVariables(c *appsv1alpha1.Cluster, cd *app if err = logTPL.Execute(&buf, logValue); err != nil { return err } - formatter.FormatterOptions.IniConfig.SectionName = defaultSectionName + formatter.FormatterOptions = appsv1alpha1.FormatterOptions{IniConfig: &appsv1alpha1.IniConfig{SectionName: defaultSectionName}} if logVariables, err = cfgcore.TransformConfigFileToKeyValueMap(keyName, formatter, buf.Bytes()); err != nil { return err } From 3ca2c918a90fd1a0ad991cbc428f573b59fa59f6 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Fri, 21 Apr 2023 17:30:10 +0800 Subject: [PATCH 125/439] fix: use S3 as the backup storage source execute full file backup Failed (#2832) --- deploy/csi-oss/templates/oss-plugin.yaml | 4 +++- deploy/csi-oss/values.yaml | 9 ++------- deploy/csi-s3/templates/csi-s3.yaml | 4 ++++ deploy/csi-s3/values.yaml | 3 +++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/deploy/csi-oss/templates/oss-plugin.yaml b/deploy/csi-oss/templates/oss-plugin.yaml index 69a58fc9e..baea8be37 100644 --- a/deploy/csi-oss/templates/oss-plugin.yaml +++ b/deploy/csi-oss/templates/oss-plugin.yaml @@ -20,8 +20,10 @@ spec: labels: app: oss-csi-plugin spec: + {{- with .Values.daemonSetTolerations }} tolerations: - - operator: Exists + {{- toYaml . | nindent 8 }} + {{- end }} affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/deploy/csi-oss/values.yaml b/deploy/csi-oss/values.yaml index 8ec7f7a58..871e46a92 100644 --- a/deploy/csi-oss/values.yaml +++ b/deploy/csi-oss/values.yaml @@ -22,13 +22,8 @@ secret: # AccessKey Secret akSecret: "" -tolerations: - - key: kb-controller - operator: Equal - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/master - operator: "Exists" +daemonSetTolerations: + - operator: Exists affinity: nodeAffinity: diff --git a/deploy/csi-s3/templates/csi-s3.yaml b/deploy/csi-s3/templates/csi-s3.yaml index 1b02cc003..84e507e1c 100644 --- a/deploy/csi-s3/templates/csi-s3.yaml +++ b/deploy/csi-s3/templates/csi-s3.yaml @@ -53,6 +53,10 @@ spec: app: csi-s3 spec: serviceAccount: csi-s3 + {{- with .Values.daemonSetTolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} hostNetwork: true containers: - name: driver-registrar diff --git a/deploy/csi-s3/values.yaml b/deploy/csi-s3/values.yaml index 4989b4489..97de65b28 100644 --- a/deploy/csi-s3/values.yaml +++ b/deploy/csi-s3/values.yaml @@ -51,6 +51,9 @@ tolerations: - key: node-role.kubernetes.io/master operator: "Exists" +daemonSetTolerations: + - operator: Exists + affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: From 749f9e55a503ce503d3756e383e0ea7866d1e32b Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Fri, 21 Apr 2023 18:45:24 +0800 Subject: [PATCH 126/439] feat: cli create support dry run (#2636) Co-authored-by: huyongqii Co-authored-by: huyongqii Co-authored-by: huyongqii Co-authored-by: huyongqii Co-authored-by: huyongqii --- docs/user_docs/cli/kbcli_cluster_configure.md | 16 ++- docs/user_docs/cli/kbcli_cluster_create.md | 25 ++++ .../cli/kbcli_cluster_edit-config.md | 18 +-- docs/user_docs/cli/kbcli_cluster_expose.md | 14 +- docs/user_docs/cli/kbcli_cluster_hscale.md | 12 +- docs/user_docs/cli/kbcli_cluster_restart.md | 10 +- docs/user_docs/cli/kbcli_cluster_start.md | 8 +- docs/user_docs/cli/kbcli_cluster_stop.md | 8 +- docs/user_docs/cli/kbcli_cluster_upgrade.md | 10 +- .../cli/kbcli_cluster_volume-expand.md | 2 + docs/user_docs/cli/kbcli_cluster_vscale.md | 16 ++- internal/cli/cmd/cluster/create.go | 27 +++- internal/cli/cmd/cluster/operations.go | 5 + internal/cli/create/create.go | 106 +++++++++++++-- internal/cli/create/create_test.go | 121 ++++++++++++++++++ internal/cli/printer/format.go | 4 + 16 files changed, 343 insertions(+), 59 deletions(-) mode change 100644 => 100755 internal/cli/cmd/cluster/create.go mode change 100644 => 100755 internal/cli/cmd/cluster/operations.go mode change 100644 => 100755 internal/cli/create/create.go mode change 100644 => 100755 internal/cli/create/create_test.go mode change 100644 => 100755 internal/cli/printer/format.go diff --git a/docs/user_docs/cli/kbcli_cluster_configure.md b/docs/user_docs/cli/kbcli_cluster_configure.md index 396630cb2..9ef2591b8 100644 --- a/docs/user_docs/cli/kbcli_cluster_configure.md +++ b/docs/user_docs/cli/kbcli_cluster_configure.md @@ -22,13 +22,15 @@ kbcli cluster configure NAME --set key=value[,key=value] [--component=component- ### Options ``` - --component string Specify the name of Component to be updated. If the cluster has only one component, unset the parameter. - --config-file string Specify the name of the configuration file to be updated (e.g. for mysql: --config-file=my.cnf). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. - --config-spec string Specify the name of the configuration template to be updated (e.g. for apecloud-mysql: --config-spec=mysql-3node-tpl). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. - -h, --help help for configure - --name string OpsRequest name. if not specified, it will be randomly generated - --set strings Specify updated parameter list. For details about the parameters, refer to kbcli sub command: 'kbcli cluster describe-config'. - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --component string Specify the name of Component to be updated. If the cluster has only one component, unset the parameter. + --config-file string Specify the name of the configuration file to be updated (e.g. for mysql: --config-file=my.cnf). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. + --config-spec string Specify the name of the configuration template to be updated (e.g. for apecloud-mysql: --config-spec=mysql-3node-tpl). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for configure + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --set strings Specify updated parameter list. For details about the parameters, refer to kbcli sub command: 'kbcli cluster describe-config'. + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index 62b445561..7a7bf79b3 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -17,6 +17,12 @@ kbcli cluster create [NAME] [flags] # --cluster-definition is required, if --cluster-version is not specified, will use the most recently created version kbcli cluster create mycluster --cluster-definition apecloud-mysql + # Output resource information in YAML format, but do not create resources. + kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run=client -o yaml + + # Output resource information in YAML format, the information will be sent to the server, but the resource will not be actually created. + kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run=server -o yaml + # Create a cluster and set termination policy DoNotTerminate that will prevent the cluster from being deleted kbcli cluster create mycluster --cluster-definition apecloud-mysql --termination-policy DoNotTerminate @@ -64,6 +70,7 @@ kbcli cluster create [NAME] [flags] ### Options ``` +<<<<<<< HEAD --backup string Set a source backup to restore data --cluster-definition string Specify cluster definition, run "kbcli cd list" to show all available cluster definitions --cluster-version string Specify cluster version, run "kbcli cv list" to show all available cluster versions, use the latest version if not specified @@ -78,6 +85,24 @@ kbcli cluster create [NAME] [flags] --termination-policy string Termination policy, one of: (DoNotTerminate, Halt, Delete, WipeOut) (default "Delete") --tolerations strings Tolerations for cluster, such as '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule"' --topology-keys stringArray Topology keys for affinity +======= + --backup string Set a source backup to restore data + --cluster-definition string Specify cluster definition, run "kbcli cd list" to show all available cluster definitions + --cluster-version string Specify cluster version, run "kbcli cv list" to show all available cluster versions, use the latest version if not specified + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --enable-all-logs Enable advanced application all log extraction, and true will ignore enabledLogs of component level (default true) + -h, --help help for create + --monitor Set monitor enabled and inject metrics exporter (default true) + --node-labels stringToString Node label selector (default []) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --pod-anti-affinity string Pod anti-affinity type, one of: (Preferred, Required) (default "Preferred") + --set stringArray Set the cluster resource including cpu, memory, replicas and storage, or you can just specify the class, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi or --set class=general-1c1g) + -f, --set-file string Use yaml file, URL, or stdin to set the cluster resource + --tenancy string Tenancy options, one of: (SharedNode, DedicatedNode) (default "SharedNode") + --termination-policy string Termination policy, one of: (DoNotTerminate, Halt, Delete, WipeOut) (default "Delete") + --tolerations strings Tolerations for cluster, such as '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule"' + --topology-keys stringArray Topology keys for affinity +>>>>>>> 8158f9bf149e7acb13c56693a6c2e8165f1d5bca ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_edit-config.md b/docs/user_docs/cli/kbcli_cluster_edit-config.md index 3071485ab..324dfe71f 100644 --- a/docs/user_docs/cli/kbcli_cluster_edit-config.md +++ b/docs/user_docs/cli/kbcli_cluster_edit-config.md @@ -18,14 +18,16 @@ kbcli cluster edit-config NAME [--component=component-name] [--config-spec=confi ### Options ``` - --component string Specify the name of Component to be updated. If the cluster has only one component, unset the parameter. - --config-file string Specify the name of the configuration file to be updated (e.g. for mysql: --config-file=my.cnf). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. - --config-spec string Specify the name of the configuration template to be updated (e.g. for apecloud-mysql: --config-spec=mysql-3node-tpl). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. - -h, --help help for edit-config - --name string OpsRequest name. if not specified, it will be randomly generated - --replace Specify whether to replace the config file. Default to false. - --set strings Specify updated parameter list. For details about the parameters, refer to kbcli sub command: 'kbcli cluster describe-config'. - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --component string Specify the name of Component to be updated. If the cluster has only one component, unset the parameter. + --config-file string Specify the name of the configuration file to be updated (e.g. for mysql: --config-file=my.cnf). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. + --config-spec string Specify the name of the configuration template to be updated (e.g. for apecloud-mysql: --config-spec=mysql-3node-tpl). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for edit-config + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --replace Specify whether to replace the config file. Default to false. + --set strings Specify updated parameter list. For details about the parameters, refer to kbcli sub command: 'kbcli cluster describe-config'. + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_expose.md b/docs/user_docs/cli/kbcli_cluster_expose.md index 21c96fcc8..281bb5867 100644 --- a/docs/user_docs/cli/kbcli_cluster_expose.md +++ b/docs/user_docs/cli/kbcli_cluster_expose.md @@ -24,12 +24,14 @@ kbcli cluster expose NAME --enable=[true|false] --type=[vpc|internet] [flags] ### Options ``` - --components strings Component names to this operations - --enable string Enable or disable the expose, values can be true or false - -h, --help help for expose - --name string OpsRequest name. if not specified, it will be randomly generated - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed - --type string Expose type, currently supported types are 'vpc', 'internet' + --components strings Component names to this operations + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --enable string Enable or disable the expose, values can be true or false + -h, --help help for expose + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --type string Expose type, currently supported types are 'vpc', 'internet' ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_hscale.md b/docs/user_docs/cli/kbcli_cluster_hscale.md index 4854da0f8..fd5f82147 100644 --- a/docs/user_docs/cli/kbcli_cluster_hscale.md +++ b/docs/user_docs/cli/kbcli_cluster_hscale.md @@ -18,11 +18,13 @@ kbcli cluster hscale [flags] ### Options ``` - --components strings Component names to this operations - -h, --help help for hscale - --name string OpsRequest name. if not specified, it will be randomly generated - --replicas int Replicas with the specified components - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --components strings Component names to this operations + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for hscale + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --replicas int Replicas with the specified components + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_restart.md b/docs/user_docs/cli/kbcli_cluster_restart.md index 884a18a07..76f10b971 100644 --- a/docs/user_docs/cli/kbcli_cluster_restart.md +++ b/docs/user_docs/cli/kbcli_cluster_restart.md @@ -21,10 +21,12 @@ kbcli cluster restart [flags] ### Options ``` - --components strings Component names to this operations - -h, --help help for restart - --name string OpsRequest name. if not specified, it will be randomly generated - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --components strings Component names to this operations + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for restart + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_start.md b/docs/user_docs/cli/kbcli_cluster_start.md index ddee7e7b4..215f5dae2 100644 --- a/docs/user_docs/cli/kbcli_cluster_start.md +++ b/docs/user_docs/cli/kbcli_cluster_start.md @@ -18,9 +18,11 @@ kbcli cluster start [flags] ### Options ``` - -h, --help help for start - --name string OpsRequest name. if not specified, it will be randomly generated - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for start + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_stop.md b/docs/user_docs/cli/kbcli_cluster_stop.md index 2fd02badb..911721b01 100644 --- a/docs/user_docs/cli/kbcli_cluster_stop.md +++ b/docs/user_docs/cli/kbcli_cluster_stop.md @@ -18,9 +18,11 @@ kbcli cluster stop [flags] ### Options ``` - -h, --help help for stop - --name string OpsRequest name. if not specified, it will be randomly generated - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for stop + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_upgrade.md b/docs/user_docs/cli/kbcli_cluster_upgrade.md index fa2e58dcd..46cc32be4 100644 --- a/docs/user_docs/cli/kbcli_cluster_upgrade.md +++ b/docs/user_docs/cli/kbcli_cluster_upgrade.md @@ -18,10 +18,12 @@ kbcli cluster upgrade [flags] ### Options ``` - --cluster-version string Reference cluster version (required) - -h, --help help for upgrade - --name string OpsRequest name. if not specified, it will be randomly generated - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --cluster-version string Reference cluster version (required) + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for upgrade + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_cluster_volume-expand.md b/docs/user_docs/cli/kbcli_cluster_volume-expand.md index e536aa408..3f6f75507 100644 --- a/docs/user_docs/cli/kbcli_cluster_volume-expand.md +++ b/docs/user_docs/cli/kbcli_cluster_volume-expand.md @@ -20,8 +20,10 @@ kbcli cluster volume-expand [flags] ``` --components strings Component names to this operations + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for volume-expand --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) --storage string Volume storage size (required) --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed -t, --volume-claim-templates strings VolumeClaimTemplate names in components (required) diff --git a/docs/user_docs/cli/kbcli_cluster_vscale.md b/docs/user_docs/cli/kbcli_cluster_vscale.md index 9a16ec88a..6bf3d593e 100644 --- a/docs/user_docs/cli/kbcli_cluster_vscale.md +++ b/docs/user_docs/cli/kbcli_cluster_vscale.md @@ -21,13 +21,15 @@ kbcli cluster vscale [flags] ### Options ``` - --class string Component class - --components strings Component names to this operations - --cpu string Requested and limited size of component cpu - -h, --help help for vscale - --memory string Requested and limited size of component memory - --name string OpsRequest name. if not specified, it will be randomly generated - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed + --class string Component class + --components strings Component names to this operations + --cpu string Requested and limited size of component cpu + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for vscale + --memory string Requested and limited size of component memory + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go old mode 100644 new mode 100755 index 8fb5f1277..b8aedb212 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -58,6 +58,12 @@ var clusterCreateExample = templates.Examples(` # --cluster-definition is required, if --cluster-version is not specified, will use the most recently created version kbcli cluster create mycluster --cluster-definition apecloud-mysql + # Output resource information in YAML format, but do not create resources. + kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run=client -o yaml + + # Output resource information in YAML format, the information will be sent to the server, but the resource will not be actually created. + kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run=server -o yaml + # Create a cluster and set termination policy DoNotTerminate that will prevent the cluster from being deleted kbcli cluster create mycluster --cluster-definition apecloud-mysql --termination-policy DoNotTerminate @@ -362,10 +368,14 @@ func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra cmd.Flags().StringVarP(&o.SetFile, "set-file", "f", "", "Use yaml file, URL, or stdin to set the cluster resource") cmd.Flags().StringArrayVar(&o.Values, "set", []string{}, "Set the cluster resource including cpu, memory, replicas and storage, or you can just specify the class, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi or --set class=general-1c1g)") cmd.Flags().StringVar(&o.Backup, "backup", "", "Set a source backup to restore data") - + cmd.Flags().String("dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" // add updatable flags o.UpdatableFlags.addFlags(cmd) + // add print flags + printer.AddOutputFlagForCreate(cmd, &o.Format) + // set required flag util.CheckErr(cmd.MarkFlagRequired("cluster-definition")) @@ -388,6 +398,21 @@ func registerFlagCompletionFunc(cmd *cobra.Command, f cmdutil.Factory) { func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return utilcomp.CompGetResource(f, cmd, util.GVRToString(types.ClusterVersionGVR()), toComplete), cobra.ShellCompDirectiveNoFileComp })) + + var formatsWithDesc = map[string]string{ + "JSON": "Output result in JSON format", + "YAML": "Output result in YAML format", + } + util.CheckErr(cmd.RegisterFlagCompletionFunc("output", + func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + var names []string + for format, desc := range formatsWithDesc { + if strings.HasPrefix(format, toComplete) { + names = append(names, fmt.Sprintf("%s\t%s", format, desc)) + } + } + return names, cobra.ShellCompDirectiveNoFileComp + })) } // PreCreate before commit yaml to k8s, make changes on Unstructured yaml diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go old mode 100644 new mode 100755 index cf78b842f..0664150cf --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -94,8 +94,13 @@ func newBaseOperationsOptions(streams genericclioptions.IOStreams, opsType appsv // buildCommonFlags build common flags for operations command func (o *OperationsOptions) buildCommonFlags(cmd *cobra.Command) { + // add print flags + printer.AddOutputFlagForCreate(cmd, &o.Format) + cmd.Flags().StringVar(&o.OpsRequestName, "name", "", "OpsRequest name. if not specified, it will be randomly generated ") cmd.Flags().IntVar(&o.TTLSecondsAfterSucceed, "ttlSecondsAfterSucceed", 0, "Time to live after the OpsRequest succeed") + cmd.Flags().String("dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" if o.HasComponentNamesFlag { cmd.Flags().StringSliceVar(&o.ComponentNames, "components", nil, " Component names to this operations") } diff --git a/internal/cli/create/create.go b/internal/cli/create/create.go old mode 100644 new mode 100755 index 480492770..d4116a57a --- a/internal/cli/create/create.go +++ b/internal/cli/create/create.go @@ -29,16 +29,21 @@ import ( "github.com/leaanthony/debme" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" k8sapitypes "k8s.io/apimachinery/pkg/types" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/printers" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/scheme" + "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" ) @@ -75,6 +80,9 @@ type Inputs struct { // Group of Version, default is v1alpha1 Version string + // Command of input + Cmd *cobra.Command + // Factory Factory cmdutil.Factory @@ -108,6 +116,10 @@ type BaseOptions struct { Client kubernetes.Interface `json:"-"` + ToPrinter func(*meta.RESTMapping, bool) (printers.ResourcePrinterFunc, error) `json:"-"` + + Format printer.Format `json:"-"` + // Quiet minimize unnecessary output Quiet bool @@ -129,6 +141,7 @@ func BuildCommand(inputs Inputs) *cobra.Command { util.CheckErr(inputs.BaseOptionsObj.Run(inputs)) }, } + inputs.Cmd = cmd if inputs.BuildFlags != nil { inputs.BuildFlags(cmd) } @@ -157,6 +170,24 @@ func (o *BaseOptions) Complete(inputs Inputs, args []string) error { return err } + o.ToPrinter = func(mapping *meta.RESTMapping, withNamespace bool) (printers.ResourcePrinterFunc, error) { + var p printers.ResourcePrinter + switch o.Format { + case printer.JSON: + p = &printers.JSONPrinter{} + case printer.YAML: + p = &printers.YAMLPrinter{} + default: + return nil, genericclioptions.NoCompatiblePrinterError{AllowedFormats: []string{"JSON", "YAML"}} + } + + p, err = printers.NewTypeSetter(scheme.Scheme).WrapToPrinter(p, nil) + if err != nil { + return nil, err + } + return p.PrintObj, nil + } + // do custom options complete if inputs.Complete != nil { if err = inputs.Complete(); err != nil { @@ -215,21 +246,43 @@ func (o *BaseOptions) Run(inputs Inputs) error { if len(version) == 0 { version = types.AppsAPIVersion } - // create k8s resource - gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: inputs.ResourceName} - if unstructuredObj, err = o.Dynamic.Resource(gvr).Namespace(o.Namespace).Create(context.TODO(), unstructuredObj, metav1.CreateOptions{}); err != nil { + + previewObj := unstructuredObj + dryRunStrategy, err := GetDryRunStrategy(inputs.Cmd) + if err != nil { return err } - o.Name = unstructuredObj.GetName() - if o.Quiet { - return nil + + if dryRunStrategy != DryRunClient { + gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: inputs.ResourceName} + createOptions := metav1.CreateOptions{} + + if dryRunStrategy == DryRunServer { + createOptions.DryRun = []string{metav1.DryRunAll} + } + // create k8s resource + previewObj, err = o.Dynamic.Resource(gvr).Namespace(o.Namespace).Create(context.TODO(), previewObj, createOptions) + if err != nil { + return err + } + if dryRunStrategy != DryRunServer { + o.Name = unstructuredObj.GetName() + if o.Quiet { + return nil + } + if inputs.CustomOutPut != nil { + inputs.CustomOutPut(o) + } else { + fmt.Fprintf(o.Out, "%s %s created\n", unstructuredObj.GetKind(), unstructuredObj.GetName()) + } + return nil + } } - if inputs.CustomOutPut != nil { - inputs.CustomOutPut(o) - } else { - fmt.Fprintf(o.Out, "%s %s created\n", unstructuredObj.GetKind(), unstructuredObj.GetName()) + printer, err := o.ToPrinter(nil, false) + if err != nil { + return err } - return nil + return printer.PrintObj(previewObj, o.Out) } // RunAsApply execute command. the options of parameter contain the command flags and args. @@ -333,3 +386,34 @@ func convertContentToUnstructured(cueValue cue.Value) (*unstructured.Unstructure } return unstructuredObj, nil } + +type DryRunStrategy int + +const ( + // DryRunNone indicates the client will make all mutating calls + DryRunNone DryRunStrategy = iota + DryRunClient + DryRunServer +) + +func GetDryRunStrategy(cmd *cobra.Command) (DryRunStrategy, error) { + if cmd == nil { + return DryRunNone, nil + } + dryRunFlag, err := cmd.Flags().GetString("dry-run") + if err != nil { + return DryRunNone, nil + } + switch dryRunFlag { + case cmd.Flag("dry-run").NoOptDefVal: + return DryRunClient, nil + case "client": + return DryRunClient, nil + case "server": + return DryRunServer, nil + case "none": + return DryRunNone, nil + default: + return DryRunNone, fmt.Errorf(`invalid dry-run value (%v). Must be "none", "server", or "client"`, dryRunFlag) + } +} diff --git a/internal/cli/create/create_test.go b/internal/cli/create/create_test.go old mode 100644 new mode 100755 index 7d59a0d96..c05a38438 --- a/internal/cli/create/create_test.go +++ b/internal/cli/create/create_test.go @@ -21,10 +21,14 @@ import ( . "github.com/onsi/gomega" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/printers" clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + "k8s.io/kubectl/pkg/scheme" + "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" ) @@ -86,6 +90,123 @@ var _ = Describe("Create", func() { Expect(baseOptions.Run(inputs)).Should(Succeed()) }) + It("test create dry-run", func() { + clusterOptions := map[string]interface{}{ + "name": "test", + "namespace": testing.Namespace, + "clusterDefRef": "test-def", + "clusterVersionRef": "test-clusterversion-ref", + "components": []string{}, + "terminationPolicy": "Halt", + } + + inputs := Inputs{ + CueTemplateName: "create_template_test.cue", + ResourceName: types.ResourceClusters, + BaseOptionsObj: &baseOptions, + Options: clusterOptions, + Factory: tf, + Validate: func() error { + return nil + }, + Complete: func() error { + baseOptions.ToPrinter = func(mapping *meta.RESTMapping, withNamespace bool) (printers.ResourcePrinterFunc, error) { + var p printers.ResourcePrinter + var err error + switch baseOptions.Format { + case printer.JSON: + p = &printers.JSONPrinter{} + case printer.YAML: + p = &printers.YAMLPrinter{} + default: + return nil, genericclioptions.NoCompatiblePrinterError{AllowedFormats: []string{"JOSN", "YAML"}} + } + + p, err = printers.NewTypeSetter(scheme.Scheme).WrapToPrinter(p, nil) + if err != nil { + return nil, err + } + return p.PrintObj, nil + } + return nil + }, + BuildFlags: func(cmd *cobra.Command) { + cmd.Flags().StringVar(&baseOptions.Namespace, "clusterDefRef", "", "cluster definition") + cmd.Flags().String("dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" + printer.AddOutputFlagForCreate(cmd, &baseOptions.Format) + }, + } + cmd := BuildCommand(inputs) + inputs.Cmd = cmd + + testCases := []struct { + clusterName string + isUseDryRun bool + mode string + dryRunStrateg DryRunStrategy + success bool + }{ + { // test do not use dry-run strategy + "test1", + false, + "", + DryRunNone, + true, + }, + { // test no parameter strategy + "test2", + true, + "unchanged", + DryRunClient, + true, + }, + { // test client strategy + "test3", + true, + "client", + DryRunClient, + true, + }, + { // test server strategy + "test4", + true, + "server", + DryRunServer, + true, + }, + { // test error parameter + "test5", + true, + "ape", + DryRunServer, + false, + }, + } + + for _, t := range testCases { + clusterOptions["name"] = t.clusterName + Expect(cmd).ShouldNot(BeNil()) + Expect(cmd.Flags().Lookup("clusterDefRef")).ShouldNot(BeNil()) + Expect(cmd.Flags().Lookup("dry-run")).ShouldNot(BeNil()) + Expect(cmd.Flags().Lookup("output")).ShouldNot(BeNil()) + if t.isUseDryRun { + Expect(cmd.Flags().Set("dry-run", t.mode)).Should(Succeed()) + } + + Expect(baseOptions.Complete(inputs, []string{})).Should(Succeed()) + Expect(baseOptions.Validate(inputs)).Should(Succeed()) + + dryRunStrateg, _ := GetDryRunStrategy(cmd) + if t.success { + Expect(dryRunStrateg == t.dryRunStrateg).Should(BeTrue()) + Expect(baseOptions.Run(inputs)).Should(Succeed()) + } else { + Expect(dryRunStrateg == t.dryRunStrateg).Should(BeFalse()) + } + } + }) + It("test Create runAsApply", func() { clusterOptions := map[string]interface{}{ "name": "test-apply", diff --git a/internal/cli/printer/format.go b/internal/cli/printer/format.go old mode 100644 new mode 100755 index b63a3960e..aac2024c3 --- a/internal/cli/printer/format.go +++ b/internal/cli/printer/format.go @@ -88,6 +88,10 @@ func AddOutputFlag(cmd *cobra.Command, varRef *Format) { })) } +func AddOutputFlagForCreate(cmd *cobra.Command, varRef *Format) { + cmd.Flags().VarP(newOutputValue(YAML, varRef), "output", "o", "prints the output in the specified format. Allowed values: JSON and YAML") +} + type outputValue Format func newOutputValue(defaultValue Format, p *Format) *outputValue { From 65bb0811962752bb81678350a455fa3dc1a7d462 Mon Sep 17 00:00:00 2001 From: dingben Date: Fri, 21 Apr 2023 19:39:14 +0800 Subject: [PATCH 127/439] feat: kbcli support binary plugin like kubectl (#2637) --- cmd/cli/main.go | 2 +- docs/user_docs/cli/cli.md | 9 + docs/user_docs/cli/kbcli.md | 1 + docs/user_docs/cli/kbcli_plugin.md | 49 ++++ docs/user_docs/cli/kbcli_plugin_list.md | 53 ++++ go.mod | 5 + go.sum | 14 + internal/cli/cmd/cli.go | 38 +++ internal/cli/cmd/plugin/plugin.go | 275 ++++++++++++++++++ internal/cli/cmd/plugin/plugin_test.go | 239 +++++++++++++++ internal/cli/cmd/plugin/testdata/kbcli-foo | 10 + .../cli/cmd/plugin/testdata/kbcli-version | 4 + internal/cli/cmd/plugin/testdata/kubectl-foo | 10 + .../cli/cmd/plugin/testdata/kubectl-version | 4 + 14 files changed, 712 insertions(+), 1 deletion(-) create mode 100644 docs/user_docs/cli/kbcli_plugin.md create mode 100644 docs/user_docs/cli/kbcli_plugin_list.md create mode 100644 internal/cli/cmd/plugin/plugin.go create mode 100644 internal/cli/cmd/plugin/plugin_test.go create mode 100644 internal/cli/cmd/plugin/testdata/kbcli-foo create mode 100644 internal/cli/cmd/plugin/testdata/kbcli-version create mode 100644 internal/cli/cmd/plugin/testdata/kubectl-foo create mode 100644 internal/cli/cmd/plugin/testdata/kubectl-version diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 14ccda619..540d7e45a 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -24,7 +24,7 @@ import ( ) func main() { - cmd := cmd.NewCliCmd() + cmd := cmd.NewDefaultCliCmd() if err := cli.RunNoErrOutput(cmd); err != nil { util.CheckErr(err) } diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index a0467e026..e168330e6 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -150,6 +150,15 @@ Bootstrap a playground KubeBlocks in local host or cloud. * [kbcli playground init](kbcli_playground_init.md) - Bootstrap a kubernetes cluster and install KubeBlocks for playground. +## [plugin](kbcli_plugin.md) + +Provides utilities for interacting with plugins. + + Plugins provide extended functionality that is not part of the major command-line distribution. + +* [kbcli plugin list](kbcli_plugin_list.md) - List all visible plugin executables on a user's PATH + + ## [version](kbcli_version.md) Print the version information, include kubernetes, KubeBlocks and kbcli version. diff --git a/docs/user_docs/cli/kbcli.md b/docs/user_docs/cli/kbcli.md index 084fb938e..981c94a1b 100644 --- a/docs/user_docs/cli/kbcli.md +++ b/docs/user_docs/cli/kbcli.md @@ -66,6 +66,7 @@ kbcli [flags] * [kbcli migration](kbcli_migration.md) - Data migration between two data sources. * [kbcli options](kbcli_options.md) - Print the list of flags inherited by all commands. * [kbcli playground](kbcli_playground.md) - Bootstrap a playground KubeBlocks in local host or cloud. +* [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. * [kbcli version](kbcli_version.md) - Print the version information, include kubernetes, KubeBlocks and kbcli version. #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/docs/user_docs/cli/kbcli_plugin.md b/docs/user_docs/cli/kbcli_plugin.md new file mode 100644 index 000000000..f9627d808 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin.md @@ -0,0 +1,49 @@ +--- +title: kbcli plugin +--- + +Provides utilities for interacting with plugins. + +### Synopsis + +Provides utilities for interacting with plugins. + + Plugins provide extended functionality that is not part of the major command-line distribution. + +### Options + +``` + -h, --help help for plugin +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + + +* [kbcli plugin list](kbcli_plugin_list.md) - List all visible plugin executables on a user's PATH + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_list.md b/docs/user_docs/cli/kbcli_plugin_list.md new file mode 100644 index 000000000..945ec1445 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_list.md @@ -0,0 +1,53 @@ +--- +title: kbcli plugin list +--- + +List all visible plugin executables on a user's PATH + +``` +kbcli plugin list +``` + +### Examples + +``` + # List all available plugins file on a user's PATH. + kbcli plugin list +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/go.mod b/go.mod index e16a93f44..930b48d23 100644 --- a/go.mod +++ b/go.mod @@ -143,6 +143,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/daviddengcn/go-colortext v1.0.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 // indirect @@ -234,6 +235,7 @@ require ( github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.7 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect + github.com/lithammer/dedent v1.1.0 // indirect github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.7 // indirect @@ -264,6 +266,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/mpvl/unique v0.0.0-20150818121801-cbe035fff7de // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect @@ -361,10 +364,12 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 // indirect k8s.io/apiserver v0.26.1 // indirect + k8s.io/component-helpers v0.26.0 // indirect oras.land/oras-go v1.2.2 // indirect periph.io/x/host/v3 v3.8.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/kustomize/api v0.12.1 // indirect + sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) diff --git a/go.sum b/go.sum index 2f448ca9e..9b4e2d619 100644 --- a/go.sum +++ b/go.sum @@ -595,6 +595,8 @@ github.com/dapr/kit v0.0.3/go.mod h1:+vh2UIRT0KzFm5YJWfj7az4XVSdodys1OCz1WzNe1Eo github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE= +github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5na0fBt4/CRe7Lrji0= github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= @@ -894,6 +896,11 @@ github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlS github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= +github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= +github.com/golangplus/bytes v1.0.0/go.mod h1:AdRaCFwmc/00ZzELMWb01soso6W1R/++O1XL80yAn+A= +github.com/golangplus/fmt v1.0.0/go.mod h1:zpM0OfbMCjPtd2qkTD/jX2MgiFCqklhSUFyDW44gVQE= +github.com/golangplus/testing v1.0.0 h1:+ZeeiKZENNOMkTTELoSySazi+XaEhVO0mb+eanrSEUQ= +github.com/golangplus/testing v1.0.0/go.mod h1:ZDreixUV3YzhoVraIDyOzHrr76p6NUh6k/pPg/Q3gYA= github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= github.com/goodhosts/hostsfile v0.1.1 h1:SqRUTFOshOCon0ZSXDrW1bkKZvs4+5pRgYFWySdaLno= github.com/goodhosts/hostsfile v0.1.1/go.mod h1:lXcUP8xO4WR5vvuQ3F/N0bMQoclOtYKEEUnyY2jTusY= @@ -1216,6 +1223,8 @@ github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e h1:hz4quJkaJWDo+xW+G6wTF6d6/95QvJ+o2D0+bB/tJ1U= github.com/longhorn/go-iscsi-helper v0.0.0-20210330030558-49a327fb024e/go.mod h1:9z/y9glKmWEdV50tjlUPxFwi1goQfIrrsoZbnMyIZbY= @@ -1366,6 +1375,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= @@ -2684,6 +2694,8 @@ k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGw k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= +k8s.io/component-helpers v0.26.0 h1:KNgwqs3EUdK0HLfW4GhnbD+q/Zl9U021VfIU7qoVYFk= +k8s.io/component-helpers v0.26.0/go.mod h1:jHN01qS/Jdj95WCbTe9S2VZ9yxpxXNY488WjF+yW4fo= k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= @@ -2731,6 +2743,8 @@ sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s= +sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 h1:cDW6AVMl6t/SLuQaezMET8hgnadZGIAr8tUrxFVOrpg= +sigs.k8s.io/kustomize/kustomize/v4 v4.5.7/go.mod h1:VSNKEH9D9d9bLiWEGbS6Xbg/Ih0tgQalmPvntzRxZ/Q= sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/internal/cli/cmd/cli.go b/internal/cli/cmd/cli.go index 3ab6fc94d..b70cd9700 100644 --- a/internal/cli/cmd/cli.go +++ b/internal/cli/cmd/cli.go @@ -19,11 +19,13 @@ package cmd import ( "fmt" "os" + "strings" "github.com/spf13/cobra" "github.com/spf13/viper" "k8s.io/cli-runtime/pkg/genericclioptions" cliflag "k8s.io/component-base/cli/flag" + kccmd "k8s.io/kubectl/pkg/cmd" cmdutil "k8s.io/kubectl/pkg/cmd/util" utilcomp "k8s.io/kubectl/pkg/util/completion" "k8s.io/kubectl/pkg/util/templates" @@ -40,6 +42,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/cmd/migration" "github.com/apecloud/kubeblocks/internal/cli/cmd/options" "github.com/apecloud/kubeblocks/internal/cli/cmd/playground" + "github.com/apecloud/kubeblocks/internal/cli/cmd/plugin" "github.com/apecloud/kubeblocks/internal/cli/cmd/version" "github.com/apecloud/kubeblocks/internal/cli/util" ) @@ -48,6 +51,40 @@ const ( cliName = "kbcli" ) +func NewDefaultCliCmd() *cobra.Command { + cmd := NewCliCmd() + + pluginHandler := kccmd.NewDefaultPluginHandler(plugin.ValidPluginFilenamePrefixes) + + if len(os.Args) > 1 { + cmdPathPieces := os.Args[1:] + + // only look for suitable extension executables if + // the specified command does not already exist + if _, _, err := cmd.Find(cmdPathPieces); err != nil { + var cmdName string + for _, arg := range cmdPathPieces { + if !strings.HasPrefix(arg, "-") { + cmdName = arg + break + } + } + + switch cmdName { + case "help", cobra.ShellCompRequestCmd, cobra.ShellCompNoDescRequestCmd: + // Don't search for a plugin + default: + if err := kccmd.HandlePluginCommand(pluginHandler, cmdPathPieces); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + } + } + } + + return cmd +} + func NewCliCmd() *cobra.Command { cmd := &cobra.Command{ Use: cliName, @@ -101,6 +138,7 @@ A Command Line Interface for KubeBlocks`, alert.NewAlertCmd(f, ioStreams), addon.NewAddonCmd(f, ioStreams), migration.NewMigrationCmd(f, ioStreams), + plugin.NewPluginCmd(ioStreams), ) filters := []string{"options"} diff --git a/internal/cli/cmd/plugin/plugin.go b/internal/cli/cmd/plugin/plugin.go new file mode 100644 index 000000000..7105b4ef8 --- /dev/null +++ b/internal/cli/cmd/plugin/plugin.go @@ -0,0 +1,275 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/printer" +) + +var ( + pluginLong = templates.LongDesc(` + Provides utilities for interacting with plugins. + + Plugins provide extended functionality that is not part of the major command-line distribution. + `) + + pluginListExample = templates.Examples(` + # List all available plugins file on a user's PATH. + kbcli plugin list + `) + + ValidPluginFilenamePrefixes = []string{"kbcli", "kubectl"} +) + +func NewPluginCmd(streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "plugin", + Short: "Provides utilities for interacting with plugins.", + Long: pluginLong, + } + + cmd.AddCommand(NewPluginListCmd(streams)) + return cmd +} + +type PluginListOptions struct { + Verifier PathVerifier + + PluginPaths []string + + genericclioptions.IOStreams +} + +func NewPluginListCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &PluginListOptions{ + IOStreams: streams, + } + cmd := &cobra.Command{ + Use: "list", + DisableFlagsInUseLine: true, + Short: "List all visible plugin executables on a user's PATH", + Example: pluginListExample, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Complete(cmd)) + cmdutil.CheckErr(o.Run()) + }, + } + return cmd +} + +func (o *PluginListOptions) Complete(cmd *cobra.Command) error { + o.Verifier = &CommandOverrideVerifier{ + root: cmd.Root(), + seenPlugins: map[string]string{}, + } + + o.PluginPaths = filepath.SplitList(os.Getenv("PATH")) + return nil +} + +func (o *PluginListOptions) Run() error { + plugins, pluginErrors := o.ListPlugins() + + if len(plugins) == 0 { + pluginErrors = append(pluginErrors, fmt.Errorf("error: unable to find any kbcli or kubectl plugins in your PATH")) + } + + pluginWarnings := 0 + p := NewPluginPrinter(o.IOStreams.Out) + errMsg := "" + for _, pluginPath := range plugins { + name := filepath.Base(pluginPath) + path := filepath.Dir(pluginPath) + if errs := o.Verifier.Verify(pluginPath); len(errs) != 0 { + for _, err := range errs { + errMsg += fmt.Sprintf("%s\n", err) + pluginWarnings++ + } + } + addRow(name, path, p) + } + p.Print() + klog.V(1).Info(errMsg) + + if pluginWarnings > 0 { + if pluginWarnings == 1 { + pluginErrors = append(pluginErrors, fmt.Errorf("error: one plugin warining was found")) + } else { + pluginErrors = append(pluginErrors, fmt.Errorf("error: %d plugin warnings were found", pluginWarnings)) + } + } + if len(pluginErrors) > 0 { + errs := bytes.NewBuffer(nil) + for _, e := range pluginErrors { + fmt.Fprintln(errs, e) + } + return fmt.Errorf("%s", errs.String()) + } + + return nil +} + +func (o *PluginListOptions) ListPlugins() ([]string, []error) { + var plugins []string + var errors []error + + for _, dir := range uniquePathsList(o.PluginPaths) { + if len(strings.TrimSpace(dir)) == 0 { + continue + } + + files, err := os.ReadDir(dir) + if err != nil { + if _, ok := err.(*os.PathError); ok { + klog.V(1).Info("Unable read directory %q from your PATH: %v. Skipping...\n", dir, err) + continue + } + + errors = append(errors, fmt.Errorf("error: unable to read directory %q in your PATH: %v", dir, err)) + continue + } + + for _, f := range files { + if f.IsDir() { + continue + } + if !hasValidPrefix(f.Name(), ValidPluginFilenamePrefixes) { + continue + } + + plugins = append(plugins, filepath.Join(dir, f.Name())) + } + } + + return plugins, errors +} + +// PathVerifier receives a path and determines if it is valid or not. +type PathVerifier interface { + Verify(path string) []error +} + +type CommandOverrideVerifier struct { + root *cobra.Command + seenPlugins map[string]string +} + +// Verify implements PathVerifier and determines if a given path +// is valid depending on whether it overwrites an existing +// kbcli command path, or a previously seen plugin. +func (v *CommandOverrideVerifier) Verify(path string) []error { + if v.root == nil { + return []error{fmt.Errorf("unable to verify path with nil root")} + } + + // extract the plugin binary name + binName := filepath.Base(path) + + cmdPath := strings.Split(binName, "-") + if len(cmdPath) > 1 { + // the first argument is always "kbcli" or "kubectl" for a plugin binary + cmdPath = cmdPath[1:] + } + + var errors []error + if isExec, err := isExecutable(path); err == nil && !isExec { + errors = append(errors, fmt.Errorf("warning: %q identified as a kbcli or kubectl plugin, but it is not executable", path)) + } else if err != nil { + errors = append(errors, fmt.Errorf("error: unable to indentify %s as an executable file: %v", path, err)) + } + + if existingPath, ok := v.seenPlugins[binName]; ok { + errors = append(errors, fmt.Errorf("warning: %s is overshadowed by a similarly named plugin: %s", path, existingPath)) + } else { + v.seenPlugins[binName] = path + } + + if cmd, _, err := v.root.Find(cmdPath); err == nil { + errors = append(errors, fmt.Errorf("warning: %q overwrites existing kbcli command: %q", path, cmd.CommandPath())) + } + + return errors +} + +func isExecutable(fullPath string) (bool, error) { + info, err := os.Stat(fullPath) + if err != nil { + return false, err + } + + if runtime.GOOS == "windows" { + fileExt := strings.ToLower(filepath.Ext(fullPath)) + + switch fileExt { + case ".bat", ".cmd", ".com", ".exe", ".ps1": + return true, nil + } + return false, nil + } + + if m := info.Mode(); !m.IsDir() && m&0111 != 0 { + return true, nil + } + + return false, nil +} + +func uniquePathsList(paths []string) []string { + var newPaths []string + seen := map[string]bool{} + + for _, path := range paths { + if !seen[path] { + newPaths = append(newPaths, path) + seen[path] = true + } + } + return newPaths +} + +func hasValidPrefix(filepath string, validPrefixes []string) bool { + for _, prefix := range validPrefixes { + if strings.HasPrefix(filepath, prefix+"-") { + return true + } + } + return false +} + +func NewPluginPrinter(out io.Writer) *printer.TablePrinter { + t := printer.NewTablePrinter(out) + t.SetHeader("NAME", "PATH") + return t +} + +func addRow(name, path string, p *printer.TablePrinter) { + p.AddRow(name, path) +} diff --git a/internal/cli/cmd/plugin/plugin_test.go b/internal/cli/cmd/plugin/plugin_test.go new file mode 100644 index 000000000..6129d04d8 --- /dev/null +++ b/internal/cli/cmd/plugin/plugin_test.go @@ -0,0 +1,239 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugin + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "k8s.io/cli-runtime/pkg/genericclioptions" +) + +func TestPluginPathsAreUnaltered(t *testing.T) { + tempDir1, err := os.MkdirTemp(os.TempDir(), "test-cmd-plugins1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tempDir2, err := os.MkdirTemp(os.TempDir(), "test-cmd-plugins2") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // cleanup + defer func() { + if err := os.RemoveAll(tempDir1); err != nil { + panic(fmt.Errorf("unexpected cleanup error: %v", err)) + } + if err := os.RemoveAll(tempDir2); err != nil { + panic(fmt.Errorf("unexpected cleanup error: %v", err)) + } + }() + + ioStreams, _, _, errOut := genericclioptions.NewTestIOStreams() + verifier := newFakePluginPathVerifier() + pluginPaths := []string{tempDir1, tempDir2} + o := &PluginListOptions{ + Verifier: verifier, + IOStreams: ioStreams, + + PluginPaths: pluginPaths, + } + + // write at least one valid plugin file + if _, err := os.CreateTemp(tempDir1, "kbcli-"); err != nil { + t.Fatalf("unexpected error %v", err) + } + if _, err := os.CreateTemp(tempDir2, "kubectl-"); err != nil { + t.Fatalf("unexpected error %v", err) + } + + if err := o.Run(); err != nil { + t.Fatalf("unexpected error %v - %v", err, errOut.String()) + } + + // ensure original paths remain unaltered + if len(verifier.seenUnsorted) != len(pluginPaths) { + t.Fatalf("saw unexpected plugin paths. Expecting %v, got %v", pluginPaths, verifier.seenUnsorted) + } + for actual := range verifier.seenUnsorted { + if !strings.HasPrefix(verifier.seenUnsorted[actual], pluginPaths[actual]) { + t.Fatalf("expected PATH slice to be unaltered. Expecting %v, but got %v", pluginPaths[actual], verifier.seenUnsorted[actual]) + } + } +} + +func TestPluginPathsAreValid(t *testing.T) { + tempDir, err := os.MkdirTemp(os.TempDir(), "test-cmd-plugins") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // cleanup + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + panic(fmt.Errorf("unexpected cleanup error: %v", err)) + } + }() + + tc := []struct { + name string + pluginPaths []string + pluginFile func() (*os.File, error) + verifier *fakePluginPathVerifier + expectVerifyErrors []error + expectErr string + expectErrOut string + expectOut string + }{ + { + name: "ensure no plugins found if no files begin with kubectl- prefix", + pluginPaths: []string{tempDir}, + verifier: newFakePluginPathVerifier(), + pluginFile: func() (*os.File, error) { + return os.CreateTemp(tempDir, "notkbcli-") + }, + expectErr: "error: unable to find any kbcli or kubectl plugins in your PATH\n", + expectOut: "NAME", + }, + { + name: "ensure de-duplicated plugin-paths slice", + pluginPaths: []string{tempDir, tempDir}, + verifier: newFakePluginPathVerifier(), + pluginFile: func() (*os.File, error) { + return os.CreateTemp(tempDir, "kbcli-") + }, + expectOut: "NAME", + }, + { + name: "ensure no errors when empty string or blank path are specified", + pluginPaths: []string{tempDir, "", " "}, + verifier: newFakePluginPathVerifier(), + pluginFile: func() (*os.File, error) { + return os.CreateTemp(tempDir, "kbcli-") + }, + expectOut: "NAME", + }, + } + + for _, test := range tc { + t.Run(test.name, func(t *testing.T) { + ioStreams, _, out, errOut := genericclioptions.NewTestIOStreams() + o := &PluginListOptions{ + Verifier: test.verifier, + IOStreams: ioStreams, + + PluginPaths: test.pluginPaths, + } + + // create files + if test.pluginFile != nil { + if _, err := test.pluginFile(); err != nil { + t.Fatalf("unexpected error creating plugin file: %v", err) + } + } + + for _, expected := range test.expectVerifyErrors { + for _, actual := range test.verifier.errors { + if expected != actual { + t.Fatalf("unexpected error: expected %v, but got %v", expected, actual) + } + } + } + + err := o.Run() + switch { + case err == nil && len(test.expectErr) > 0: + t.Fatalf("unexpected non-error: expected %v, but got nothing", test.expectErr) + case err != nil && len(test.expectErr) == 0: + t.Fatalf("unexpected error: expected nothing, but got %v", err.Error()) + case err != nil && err.Error() != test.expectErr: + t.Fatalf("unexpected error: expected %v, but got %v", test.expectErr, err.Error()) + } + + if len(test.expectErrOut) == 0 && errOut.Len() > 0 { + t.Fatalf("unexpected error output: expected nothing, but got %v", errOut.String()) + } else if len(test.expectErrOut) > 0 && !strings.Contains(errOut.String(), test.expectErrOut) { + t.Fatalf("unexpected error output: expected to contain %v, but got %v", test.expectErrOut, errOut.String()) + } + + if len(test.expectOut) > 0 && !strings.Contains(out.String(), test.expectOut) { + t.Fatalf("unexpected output: expected to contain %v, but got %v", test.expectOut, out.String()) + } + }) + } +} + +func TestListPlugins(t *testing.T) { + pluginPath, _ := filepath.Abs("./testdata") + expectPlugins := []string{ + filepath.Join(pluginPath, "kbcli-foo"), + filepath.Join(pluginPath, "kbcli-version"), + filepath.Join(pluginPath, "kubectl-foo"), + filepath.Join(pluginPath, "kubectl-version"), + } + + verifier := newFakePluginPathVerifier() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() + pluginPaths := []string{pluginPath} + + o := &PluginListOptions{ + Verifier: verifier, + IOStreams: ioStreams, + + PluginPaths: pluginPaths, + } + + plugins, errs := o.ListPlugins() + if len(errs) > 0 { + t.Fatalf("unexpected errors: %v", errs) + } + + if !reflect.DeepEqual(expectPlugins, plugins) { + t.Fatalf("saw unexpected plugins. Expecting %v, got %v", expectPlugins, plugins) + } +} + +type duplicatePathError struct { + path string +} + +func (d *duplicatePathError) Error() string { + return fmt.Sprintf("path %q already visited", d.path) +} + +type fakePluginPathVerifier struct { + errors []error + seen map[string]bool + seenUnsorted []string +} + +func (f *fakePluginPathVerifier) Verify(path string) []error { + if f.seen[path] { + err := &duplicatePathError{path} + f.errors = append(f.errors, err) + return []error{err} + } + f.seen[path] = true + f.seenUnsorted = append(f.seenUnsorted, path) + return nil +} + +func newFakePluginPathVerifier() *fakePluginPathVerifier { + return &fakePluginPathVerifier{seen: make(map[string]bool)} +} diff --git a/internal/cli/cmd/plugin/testdata/kbcli-foo b/internal/cli/cmd/plugin/testdata/kbcli-foo new file mode 100644 index 000000000..41b6b7b74 --- /dev/null +++ b/internal/cli/cmd/plugin/testdata/kbcli-foo @@ -0,0 +1,10 @@ +#!/bin/bash + +# optional argument handling +if [[ "$1" == "version" ]] +then + echo "1.0.0" + exit 0 +fi + +echo "I am a plugin named kbcli-foo" diff --git a/internal/cli/cmd/plugin/testdata/kbcli-version b/internal/cli/cmd/plugin/testdata/kbcli-version new file mode 100644 index 000000000..b745356fe --- /dev/null +++ b/internal/cli/cmd/plugin/testdata/kbcli-version @@ -0,0 +1,4 @@ +!/bin/bash + +# This plugin is a no-op and is used to test plugins +# that overshadow existing kbcli commands \ No newline at end of file diff --git a/internal/cli/cmd/plugin/testdata/kubectl-foo b/internal/cli/cmd/plugin/testdata/kubectl-foo new file mode 100644 index 000000000..71dd6e54d --- /dev/null +++ b/internal/cli/cmd/plugin/testdata/kubectl-foo @@ -0,0 +1,10 @@ +#!/bin/bash + +# optional argument handling +if [[ "$1" == "version" ]] +then + echo "1.0.0" + exit 0 +fi + +echo "I am a plugin named kubectl-foo" diff --git a/internal/cli/cmd/plugin/testdata/kubectl-version b/internal/cli/cmd/plugin/testdata/kubectl-version new file mode 100644 index 000000000..e12a64322 --- /dev/null +++ b/internal/cli/cmd/plugin/testdata/kubectl-version @@ -0,0 +1,4 @@ +!/bin/bash + +# This plugin is a no-op and is used to test plugins +# that overshadow existing kbcli commands From 4a59b21c9ef2b1a20c9d7637411908b4c907d864 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Fri, 21 Apr 2023 22:02:04 +0800 Subject: [PATCH 128/439] chore: update docs/release_notes/v0.5.0/v0.5.0.md for CRD changes info (#2643) --- apis/apps/v1alpha1/cluster_types.go | 6 +- .../bases/apps.kubeblocks.io_clusters.yaml | 8 +- .../crds/apps.kubeblocks.io_clusters.yaml | 8 +- docs/release_notes/v0.5.0/v0.5.0.md | 75 ++++++++++++++++++- 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index 206e652fc..e70cf9be5 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -170,11 +170,11 @@ type ClusterComponentSpec struct { // +optional SwitchPolicy *ClusterSwitchPolicy `json:"switchPolicy,omitempty"` - // tls should be enabled or not + // Enable or disable TLS certs. // +optional TLS bool `json:"tls,omitempty"` - // issuer who provides tls certs + // issuer defines provider context for TLS certs. // required when TLS enabled // +optional Issuer *Issuer `json:"issuer,omitempty"` @@ -376,7 +376,7 @@ type Issuer struct { // +kubebuilder:validation:Required Name IssuerName `json:"name"` - // secretRef, Tls certs Secret reference + // secretRef, TLS certs Secret reference // required when from is UserProvided // +optional SecretRef *TLSSecretRef `json:"secretRef,omitempty"` diff --git a/config/crd/bases/apps.kubeblocks.io_clusters.yaml b/config/crd/bases/apps.kubeblocks.io_clusters.yaml index 0bd9a3df9..3942699b1 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusters.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusters.yaml @@ -185,8 +185,8 @@ spec: type: array x-kubernetes-list-type: set issuer: - description: issuer who provides tls certs required when TLS - enabled + description: issuer defines provider context for TLS certs. + required when TLS enabled properties: name: default: KubeBlocks @@ -198,7 +198,7 @@ spec: - UserProvided type: string secretRef: - description: secretRef, Tls certs Secret reference required + description: secretRef, TLS certs Secret reference required when from is UserProvided properties: ca: @@ -353,7 +353,7 @@ spec: type: string type: object tls: - description: tls should be enabled or not + description: Enable or disable TLS certs. type: boolean tolerations: description: Component tolerations will override ClusterSpec.Tolerations diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml index 0bd9a3df9..3942699b1 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml @@ -185,8 +185,8 @@ spec: type: array x-kubernetes-list-type: set issuer: - description: issuer who provides tls certs required when TLS - enabled + description: issuer defines provider context for TLS certs. + required when TLS enabled properties: name: default: KubeBlocks @@ -198,7 +198,7 @@ spec: - UserProvided type: string secretRef: - description: secretRef, Tls certs Secret reference required + description: secretRef, TLS certs Secret reference required when from is UserProvided properties: ca: @@ -353,7 +353,7 @@ spec: type: string type: object tls: - description: tls should be enabled or not + description: Enable or disable TLS certs. type: boolean tolerations: description: Component tolerations will override ClusterSpec.Tolerations diff --git a/docs/release_notes/v0.5.0/v0.5.0.md b/docs/release_notes/v0.5.0/v0.5.0.md index e9b259d7f..e07383137 100644 --- a/docs/release_notes/v0.5.0/v0.5.0.md +++ b/docs/release_notes/v0.5.0/v0.5.0.md @@ -10,9 +10,81 @@ Thanks to everyone who made this release possible! ## What's Changed +- New APIs: + - backuppolicytemplates.apps.kubeblocks.io + - componentclassdefinitions.apps.kubeblocks.io + - componentresourceconstraints.apps.kubeblocks.io + +- Deleted APIs: + - backuppolicytemplates.dataprotection.kubeblocks.io + +- New API attributes: + - clusterdefinitions.apps.kubeblocks.io API + - spec.type + - spec.componentDefs.customLabelSpecs + - clusters.apps.kubeblocks.io API + - spec.componentSpecs.classDefRef + - configconstraints.apps.kubeblocks.io API + - spec.reloadOptions.shellTrigger.namespace + - spec.reloadOptions.shellTrigger.scriptConfigMapRef + - spec.reloadOptions.tplScriptTrigger.sync + - spec.selector + - opsrequests.apps.kubeblocks.io API + - spec.restoreFrom + - spec.verticalScaling.class + - status.reconfiguringStatus.configurationStatus.updatePolicy + - backuppolicies.dataprotection.kubeblocks.io API + - spec.full + - backups.dataprotection.kubeblocks.io + - status.manifests + - backuptools.dataprotection.kubeblocks.io + - spec.type + +- Renamed API attributes: + - clusterdefinitions.apps.kubeblocks.io API + - spec.componentDefs.horizontalScalePolicy.backupTemplateSelector -> spec.componentDefs.horizontalScalePolicy.backupPolicyTemplateName + - spec.componentDefs.probe.roleChangedProbe -> spec.componentDefs.probe.roleProbe + - restorejobs.dataprotection.kubeblocks.io API + - spec.target.secret.passwordKeyword -> spec.target.secret.passwordKey + - spec.target.secret.userKeyword -> spec.target.secret.usernameKey + - addons.extensions.kubeblocks.io API + - spec.helm.installValues.secretsRefs -> spec.helm.installValues.secretRefs + +- Deleted API attributes: + - opsrequests.apps.kubeblocks.io API + - status.observedGeneration + - backuppolicies.dataprotection.kubeblocks.io API + - spec.backupPolicyTemplateName + - spec.backupToolName + - spec.backupType + - spec.backupsHistoryLimit + - spec.hooks + - backups.dataprotection.kubeblocks.io API + - spec.ttl + - status.CheckPoint + - status.checkSum + - addons.extensions.kubeblocks.io API + - spec.helm.valuesMapping.jsonMap.additionalProperties + - spec.helm.valuesMapping.valueMap.additionalProperties + - spec.helm.valuesMapping.extras.jsonMap.additionalProperties + - spec.helm.valuesMapping.extras.valueMap.additionalProperties + + +- Updates API Status info: + - clusters.apps.kubeblocks.io API + - status.components.phase valid values are Running, Stopped, Failed, Abnormal, Creating, Updating; REMOVED phases are SpecUpdating, Deleting, Deleted, VolumeExpanding, Reconfiguring, HorizontalScaling, VerticalScaling, VersionUpgrading, Rebooting, Stopping, Starting. + - status.phase valid values are Running, Stopped, Failed, Abnormal, Creating, Updating; REMOVED phases are ConditionsError, SpecUpdating, Deleting, Deleted, VolumeExpanding, Reconfiguring, HorizontalScaling, VerticalScaling, VersionUpgrading, Rebooting, Stopping, Starting. + - opsrequests.apps.kubeblocks.io API + - status.components.phase valid values are Running, Stopped, Failed, Abnormal, Creating, Updating; REMOVED phases are SpecUpdating, Deleting, Deleted, VolumeExpanding, Reconfiguring, HorizontalScaling, VerticalScaling, VersionUpgrading, Rebooting, Stopping, Starting, Exposing. + - status.phase added 'Creating' phase. + + + ### New Features + + #### PostgreSQL - Support incremental migration from AWS RDS to KubeBlocks, support pre-check, full migration and incremental synchronization @@ -52,4 +124,5 @@ Thanks to everyone who made this release possible! ``` kubectl delete clusters.apps.kubeblocks.io --all kubectl delete opsrequets.apps.kubeblocks.io --all - ``` \ No newline at end of file + ``` +- `addons.extensions.kubeblocks.io` API deleted `spec.helm.valuesMapping.jsonMap.additionalProperties`, `spec.helm.valuesMapping.valueMap.additionalProperties`, `spec.helm.valuesMapping.extras.jsonMap.additionalProperties` and `spec.helm.valuesMapping.extras.valueMap.additionalProperties` attributes that was introduced by CRD generator, all existing Addons API YAML shouldn't have referenced these attributes. From ed5c5fe8f443bd84a09cd4a5a723b910d07ef450 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Fri, 21 Apr 2023 22:33:16 +0800 Subject: [PATCH 129/439] chore: add .github/utils/sanitize_release_body.sh script (#2837) --- .../utils/bugs_not_in_current_milestone.sh | 5 +- .github/utils/create-releasing-pr.sh | 30 ---------- .github/utils/create_releasing_pr.sh | 21 +++++++ .github/utils/current_milestone_bugs.sh | 5 +- .github/utils/feature_triage.sh | 5 +- .github/utils/functions.bash | 1 - .github/utils/gh_env | 4 +- ....py => is_rc_or_stable_release_version.py} | 22 ++++--- ...-releasing-pr.sh => merge_releasing_pr.sh} | 5 +- .github/utils/sanitize_release_body.sh | 58 +++++++++++++++++++ .github/workflows/pull-request-check.yml | 2 +- .github/workflows/release-create.yml | 10 +++- 12 files changed, 116 insertions(+), 52 deletions(-) delete mode 100755 .github/utils/create-releasing-pr.sh create mode 100755 .github/utils/create_releasing_pr.sh rename .github/utils/{get_release_version.py => is_rc_or_stable_release_version.py} (89%) rename .github/utils/{merge-releasing-pr.sh => merge_releasing_pr.sh} (93%) create mode 100755 .github/utils/sanitize_release_body.sh diff --git a/.github/utils/bugs_not_in_current_milestone.sh b/.github/utils/bugs_not_in_current_milestone.sh index e6663b23f..782d93ef5 100755 --- a/.github/utils/bugs_not_in_current_milestone.sh +++ b/.github/utils/bugs_not_in_current_milestone.sh @@ -6,8 +6,9 @@ set -o pipefail # requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. -. ./gh_env -. ./functions.bash +worddir=$(dirname $0) +. ${worddir}/gh_env +. ${worddir}/functions.bash process_issue_rows() { for ((i = 0; i < ${item_count}; i++)) diff --git a/.github/utils/create-releasing-pr.sh b/.github/utils/create-releasing-pr.sh deleted file mode 100755 index 2faf3073f..000000000 --- a/.github/utils/create-releasing-pr.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. - -. ./gh_env -. ./functions.bash - -echo "Creating ${PR_TITLE}" - -result=$(gh api \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/${OWNER}/${REPO}/pulls \ - -f title="${PR_TITLE}" \ - -f head="${HEAD_BRANCH}" \ - -f base="${BASE_BRANCH}" 1> /dev/null) - -if [ "$?" != "0" ]; then - echo "error: ${result}" - exit 1 -else - echo "PR created" -fi - - diff --git a/.github/utils/create_releasing_pr.sh b/.github/utils/create_releasing_pr.sh new file mode 100755 index 000000000..27cbd66ef --- /dev/null +++ b/.github/utils/create_releasing_pr.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. + +worddir=$(dirname $0) +. ${worddir}/gh_env +. ${worddir}/functions.bash + +set -x + +git stash +git switch ${BASE_BRANCH} +git pull +git merge origin/${HEAD_BRANCH} + +echo "Creating ${PR_TITLE}" +gh pr create --head ${HEAD_BRANCH} --base ${BASE_BRANCH} --title "${PR_TITLE}" --body "" \ No newline at end of file diff --git a/.github/utils/current_milestone_bugs.sh b/.github/utils/current_milestone_bugs.sh index 064fce660..21622c97e 100755 --- a/.github/utils/current_milestone_bugs.sh +++ b/.github/utils/current_milestone_bugs.sh @@ -6,8 +6,9 @@ set -o pipefail # requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. -. ./gh_env -. ./functions.bash +worddir=$(dirname $0) +. ${worddir}/gh_env +. ${worddir}/functions.bash LABELS=${LABELS:-'kind/bug,bug'} #"severity/critical,severity/major,severity/minor,severity/normal" diff --git a/.github/utils/feature_triage.sh b/.github/utils/feature_triage.sh index e5ff3f3b0..e174ccaee 100755 --- a/.github/utils/feature_triage.sh +++ b/.github/utils/feature_triage.sh @@ -6,8 +6,9 @@ set -o pipefail # requires `git`, `gh`, and `jq` commands, ref. https://cli.github.com/manual/installation for installation guides. -. ./gh_env -. ./functions.bash +worddir=$(dirname $0) +. ${worddir}/gh_env +. ${worddir}/functions.bash print_issue_rows() { for ((i = 0; i < ${item_count}; i++)) diff --git a/.github/utils/functions.bash b/.github/utils/functions.bash index 4993917db..de1e22c24 100644 --- a/.github/utils/functions.bash +++ b/.github/utils/functions.bash @@ -110,4 +110,3 @@ function join_by { printf %s "$f" "${@/#/$d}" fi } - diff --git a/.github/utils/gh_env b/.github/utils/gh_env index bbf6c6636..683cc28f7 100644 --- a/.github/utils/gh_env +++ b/.github/utils/gh_env @@ -2,8 +2,8 @@ DEBUG=${DEBUG:-} export MILESTONE_ID=${MILESTONE_ID:-5} -export HEAD_BRANCH=${HEAD_BRANCH:-'release-0.5'} -export BASE_BRANCH=${BASE_BRANCH:-'releasing-0.5'} +export BASE_BRANCH=${BASE_BRANCH:-'release-0.5'} +export HEAD_BRANCH=${HEAD_BRANCH:-'releasing-0.5'} export PR_TITLE=${PR_TITLE:-"${BASE_BRANCH} tracker PR (no-need-to-review)"} diff --git a/.github/utils/get_release_version.py b/.github/utils/is_rc_or_stable_release_version.py similarity index 89% rename from .github/utils/get_release_version.py rename to .github/utils/is_rc_or_stable_release_version.py index eccfda834..304c287eb 100755 --- a/.github/utils/get_release_version.py +++ b/.github/utils/is_rc_or_stable_release_version.py @@ -15,6 +15,7 @@ def main(argv: list[str]) -> None: tag_ref_prefix = "refs/tags/v" github_env : str = str(os.getenv("GITHUB_ENV")) + with open(github_env, "a") as github_env_f: if git_ref is None or not git_ref.startswith(tag_ref_prefix): print("This is not a release tag") @@ -23,13 +24,7 @@ def main(argv: list[str]) -> None: release_version = git_ref[len(tag_ref_prefix) :] release_note_path = f"docs/release_notes/v{release_version}/v{release_version}.md" - if git_ref.find("-alpha.") > 0: - print(f"Alpha release build from {git_ref} ...") - elif git_ref.find("-beta.") > 0: - print(f"Beta release build from {git_ref} ...") - elif git_ref.find("-rc.") > 0: - print(f"Release Candidate build from {git_ref} ...") - else: + def set_with_rel_note_to_true() -> None print(f"Checking if {release_note_path} exists") if os.path.exists(release_note_path): print(f"Found {release_note_path}") @@ -38,6 +33,19 @@ def main(argv: list[str]) -> None: print("{} is not found".format(release_note_path)) print(f"Release build from {git_ref} ...") + + if git_ref.find("-alpha.") > 0: + print(f"Alpha release build from {git_ref} ...") + print(f"IGNORED") + elif git_ref.find("-beta.") > 0: + print(f"Beta release build from {git_ref} ...") + print(f"IGNORED") + elif git_ref.find("-rc.") > 0: + print(f"Release Candidate build from {git_ref} ...") + set_with_rel_note_to_true() + else: + set_with_rel_note_to_true() + github_env_f.write(f"REL_VERSION={release_version}\n") diff --git a/.github/utils/merge-releasing-pr.sh b/.github/utils/merge_releasing_pr.sh similarity index 93% rename from .github/utils/merge-releasing-pr.sh rename to .github/utils/merge_releasing_pr.sh index 31720487d..cd835c617 100755 --- a/.github/utils/merge-releasing-pr.sh +++ b/.github/utils/merge_releasing_pr.sh @@ -6,8 +6,9 @@ set -o pipefail # requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. -. ./gh_env -. ./functions.bash +worddir=$(dirname $0) +. ${worddir}/gh_env +. ${worddir}/functions.bash echo "Merging ${PR_TITLE}" diff --git a/.github/utils/sanitize_release_body.sh b/.github/utils/sanitize_release_body.sh new file mode 100755 index 000000000..98f32fe72 --- /dev/null +++ b/.github/utils/sanitize_release_body.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. + +worddir=$(dirname $0) +. ${worddir}/gh_env +. ${worddir}/functions.bash + +TAG=${TAG:-} +GITHUB_REF=${GITHUB_REF:-} +tag_ref_prefix="refs/tags/" + +if [ -z "$TAG" ] && [ -n "$GITHUB_REF" ]; then +TAG=${GITHUB_REF#"${tag_ref_prefix}"} +fi + +if [ -z "$TAG" ]; then + if [ -n "$DEBUG" ]; then echo "EMPTY TAG, NOOP"; fi + exit 0 +fi + +if [ -n "$DEBUG" ]; then +set -x +fi +echo "Processing tag ${TAG}" +rel_body=$(gh release \ + --repo ${OWNER}/${REPO} view "${TAG}" \ + --json 'body') + +rel_body_text=$(echo ${rel_body} | jq -r '.body') + +if [ -n "$DEBUG" ]; then echo $rel_body_text; fi + + +# set -o noglob +IFS=$'\r\n' rel_items=($rel_body_text) +# set +o noglob + +final_rel_notes="" +for val in "${rel_items[@]}"; +do + if [[ $val == "**Full Changelog**"* ]]; then + final_rel_notes="${final_rel_notes}\r\n\r\n${val}" + continue + fi + # ignore line if contain ${PR_TITLE} + if [[ $val != "* ${PR_TITLE}"* ]];then + final_rel_notes="${final_rel_notes}${val}\r\n" + fi +done + +if [ -n "$DEBUG" ]; then echo -e $final_rel_notes; fi + +# gh release --repo ${OWNER}/${REPO} edit ${TAG} --notes "$(echo -e ${final_rel_notes})" diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml index 1c2e17506..e5a477a45 100644 --- a/.github/workflows/pull-request-check.yml +++ b/.github/workflows/pull-request-check.yml @@ -15,7 +15,7 @@ jobs: - name: check branch name uses: apecloud/check-branch-name@v0.1.0 with: - branch_pattern: 'feature/|bugfix/|release/|hotfix/|support/|release/|release-|releasing/|releasing-|dependabot/' + branch_pattern: 'feature/|bugfix/|hotfix/|support/|release/|release-|releasing/|releasing-|dependabot/' comment_for_invalid_branch_name: 'This branch name is not following the standards: feature/|bugfix/|hotfix/|support/|release/|release-|releasing/|releasing-|dependabot/' fail_if_invalid_branch_name: 'true' ignore_branch_pattern: 'main|master' diff --git a/.github/workflows/release-create.yml b/.github/workflows/release-create.yml index c35f5ea56..da1e14593 100644 --- a/.github/workflows/release-create.yml +++ b/.github/workflows/release-create.yml @@ -13,17 +13,18 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v3 - name: Parse release version and set REL_VERSION - run: python ./.github/utils/get_release_version.py - - name: release without release notes + run: python ./.github/utils/is_rc_or_stable_release_version.py + - name: release pre-release without release notes uses: softprops/action-gh-release@v1 if: not ${{ env.WITH_RELEASE_NOTES }} with: + # body_path: ./docs/release_notes/v${{ env.REL_VERSION }}/v${{ env.REL_VERSION }}.md token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} name: KubeBlocks v${{ env.REL_VERSION }} generate_release_notes: true tag_name: v${{ env.REL_VERSION }} prerelease: true - - name: release with release notes + - name: release RC with release notes uses: softprops/action-gh-release@v1 if: ${{ env.WITH_RELEASE_NOTES }} with: @@ -32,3 +33,6 @@ jobs: name: KubeBlocks v${{ env.REL_VERSION }} tag_name: v${{ env.REL_VERSION }} prerelease: true + - name: sanitized release body + if: not ${{ env.WITH_RELEASE_NOTES }} + run: ./.github/utils/sanitize_release_body.sh \ No newline at end of file From a126c722cd31f63aa5e8282a9b94aae2223733db Mon Sep 17 00:00:00 2001 From: zjx20 Date: Sun, 23 Apr 2023 09:29:22 +0800 Subject: [PATCH 130/439] feat: delete correlated backup files when deleting a Backup object (#2797) --- controllers/apps/cluster_controller_test.go | 2 +- .../dataprotection/backup_controller.go | 181 +++++++++++++++--- .../dataprotection/backup_controller_test.go | 65 ++++++- .../dataprotection/backuppolicy_controller.go | 2 - .../dataprotection/restorejob_controller.go | 2 - controllers/dataprotection/type.go | 2 - internal/testutil/apps/backup_factory.go | 16 +- 7 files changed, 229 insertions(+), 41 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 664b88019..ee4a20a6f 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -95,7 +95,7 @@ var _ = Describe("Cluster Controller", func() { testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS, ml) testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PodSignature, true, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupSignature, inNS, ml) + testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) // non-namespaced testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupToolSignature, ml) diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 5f4bb994b..272de3660 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -52,6 +52,16 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) +var ( + errJobFailed = errors.New("job failed") + errDeletingBackupFiles = errors.New("deleting backup files") +) + +const ( + backupPathBase = "/backupdata" + deleteBackupFilesJobNamePrefix = "delete-backup-files-" +) + // BackupReconciler reconciles a Backup object type BackupReconciler struct { client.Client @@ -74,8 +84,6 @@ type BackupReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) - // NOTES: // setup common request context reqCtx := intctrlutil.RequestCtx{ @@ -93,7 +101,11 @@ func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // handle finalizer res, err := intctrlutil.HandleCRDeletion(reqCtx, r, backup, dataProtectionFinalizerName, func() (*ctrl.Result, error) { - return nil, r.deleteExternalResources(reqCtx, backup) + err := r.deleteExternalResources(reqCtx, backup) + if errors.Is(err, errDeletingBackupFiles) { + return intctrlutil.ResultToP(intctrlutil.Requeue(reqCtx.Log, "deleting backup files")) + } + return nil, err }) if res != nil { return *res, err @@ -563,7 +575,7 @@ func (r *BackupReconciler) ensureBatchV1JobCompleted( if jobStatusConditions[0].Type == batchv1.JobComplete { return true, nil } else if jobStatusConditions[0].Type == batchv1.JobFailed { - return false, errors.New(errorJobFailed) + return false, errJobFailed } } } @@ -716,6 +728,66 @@ func (r *BackupReconciler) createMetadataCollectionJob(reqCtx intctrlutil.Reques return client.IgnoreAlreadyExists(r.Client.Create(reqCtx.Ctx, job)) } +func (r *BackupReconciler) createDeleteBackupFileJob( + reqCtx intctrlutil.RequestCtx, + jobKey types.NamespacedName, + backup *dataprotectionv1alpha1.Backup, + backupPVCName string, + backupFilePath string) error { + + // build container + container := corev1.Container{} + container.Name = backup.Name + container.Command = []string{"sh", "-c"} + container.Args = []string{fmt.Sprintf("rm -rf %s%s", backupPathBase, backupFilePath)} + container.Image = viper.GetString(constant.KBToolsImage) + container.ImagePullPolicy = corev1.PullPolicy(viper.GetString(constant.KBImagePullPolicy)) + + allowPrivilegeEscalation := false + runAsUser := int64(0) + container.SecurityContext = &corev1.SecurityContext{ + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + RunAsUser: &runAsUser, + } + + // build pod + podSpec := corev1.PodSpec{ + Containers: []corev1.Container{container}, + RestartPolicy: corev1.RestartPolicyNever, + } + + // mount the backup volume to the pod + r.appendBackupVolumeMount(backupPVCName, &podSpec, &podSpec.Containers[0]) + + if err := addTolerations(&podSpec); err != nil { + return err + } + + // build job + backOffLimit := int32(3) + ttlSecondsAfterSuccess := int32(600) + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: jobKey.Namespace, + Name: jobKey.Name, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: jobKey.Namespace, + Name: jobKey.Name, + }, + Spec: podSpec, + }, + BackoffLimit: &backOffLimit, + TTLSecondsAfterFinished: &ttlSecondsAfterSuccess, + }, + } + + reqCtx.Log.V(1).Info("create a job from delete backup files", "job", job) + return client.IgnoreAlreadyExists(r.Client.Create(reqCtx.Ctx, job)) +} + func (r *BackupReconciler) createBackupToolJob( reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, @@ -897,7 +969,59 @@ func (r *BackupReconciler) deleteReferenceVolumeSnapshot(reqCtx intctrlutil.Requ return nil } +func (r *BackupReconciler) deleteBackupFiles(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) error { + if backup.Spec.BackupType == dataprotectionv1alpha1.BackupTypeSnapshot { + // no file to delete for this type + return nil + } + if backup.Status.Phase == dataprotectionv1alpha1.BackupNew || + backup.Status.Phase == dataprotectionv1alpha1.BackupFailed { + // nothing to delete + return nil + } + + jobName := deleteBackupFilesJobNamePrefix + backup.Name + jobKey := types.NamespacedName{Namespace: backup.Namespace, Name: jobName} + job := batchv1.Job{} + exists, err := intctrlutil.CheckResourceExists(reqCtx.Ctx, r.Client, jobKey, &job) + if err != nil { + return err + } + // create job for deleting backup files + if !exists { + pvcName := backup.Status.PersistentVolumeClaimName + if pvcName == "" { + reqCtx.Log.Info("skip deleting backup files because PersistentVolumeClaimName empty", + "backup", backup.Name) + return nil + } + + backupFilePath := "" + if backup.Status.Manifests != nil && backup.Status.Manifests.BackupTool != nil { + backupFilePath = backup.Status.Manifests.BackupTool.FilePath + } + if backupFilePath == "" || !strings.Contains(backupFilePath, backup.Name) { + // For compatibility: the FilePath field was changed from time to time, + // and it may not contain the backup name as a path component if the Backup object + // was created in a previous version. In this case, it's dangerous to execute + // the deletion command. For example, files belongs to other Backups can be deleted as well. + reqCtx.Log.Info("skip deleting backup files because backupFilePath is invalid", + "backupFilePath", backupFilePath, "backup", backup.Name) + return nil + } + // the job will run in the background + if err = r.createDeleteBackupFileJob(reqCtx, jobKey, backup, pvcName, backupFilePath); err != nil { + return err + } + } + + return nil +} + func (r *BackupReconciler) deleteExternalResources(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) error { + if err := r.deleteBackupFiles(reqCtx, backup); err != nil { + return err + } if err := r.deleteReferenceBatchV1Jobs(reqCtx, backup); err != nil { return err } @@ -974,6 +1098,28 @@ func (r *BackupReconciler) getTargetPVCs(reqCtx intctrlutil.RequestCtx, return allPVCs, nil } +func (r *BackupReconciler) appendBackupVolumeMount( + pvcName string, + podSpec *corev1.PodSpec, + container *corev1.Container) { + // TODO(dsj): mount multi remote backup volumes + remoteVolumeName := fmt.Sprintf("backup-%s", pvcName) + remoteVolume := corev1.Volume{ + Name: remoteVolumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + }, + }, + } + remoteVolumeMount := corev1.VolumeMount{ + Name: remoteVolumeName, + MountPath: backupPathBase, + } + podSpec.Volumes = append(podSpec.Volumes, remoteVolume) + container.VolumeMounts = append(container.VolumeMounts, remoteVolumeMount) +} + func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy, @@ -1017,25 +1163,8 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, if backupTool.Spec.Resources != nil { container.Resources = *backupTool.Spec.Resources } - - remoteBackupPath := "/backupdata" - - // TODO(dsj): mount multi remote backup volumes - remoteVolumeName := fmt.Sprintf("backup-%s", commonPolicy.PersistentVolumeClaim.Name) - remoteVolume := corev1.Volume{ - Name: remoteVolumeName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: commonPolicy.PersistentVolumeClaim.Name, - }, - }, - } - remoteVolumeMount := corev1.VolumeMount{ - Name: remoteVolumeName, - MountPath: remoteBackupPath, - } container.VolumeMounts = clusterPod.Spec.Containers[0].VolumeMounts - container.VolumeMounts = append(container.VolumeMounts, remoteVolumeMount) + allowPrivilegeEscalation := false runAsUser := int64(0) container.SecurityContext = &corev1.SecurityContext{ @@ -1049,7 +1178,7 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, envBackupDir := corev1.EnvVar{ Name: "BACKUP_DIR", - Value: remoteBackupPath + pathPrefix, + Value: backupPathBase + pathPrefix, } container.Env = []corev1.EnvVar{envDBHost, envBackupName, envBackupDir} @@ -1084,11 +1213,13 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, container.Env = append(container.Env, backupTool.Spec.Env...) podSpec.Containers = []corev1.Container{container} - podSpec.Volumes = clusterPod.Spec.Volumes - podSpec.Volumes = append(podSpec.Volumes, remoteVolume) podSpec.RestartPolicy = corev1.RestartPolicyNever + // mount the backup volume to the pod of backup tool + pvcName := commonPolicy.PersistentVolumeClaim.Name + r.appendBackupVolumeMount(pvcName, &podSpec, &podSpec.Containers[0]) + // the pod of job needs to be scheduled on the same node as the workload pod, because it needs to share one pvc // see: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename podSpec.NodeName = clusterPod.Spec.NodeName diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index 9df8730db..85e103912 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -177,7 +177,7 @@ var _ = Describe("Backup Controller test", func() { })).Should(Succeed()) By("Check backup job is deleted after completed") - Eventually(testapps.CheckObjExists(&testCtx, backupKey, &batchv1.Job{}, false)) + Eventually(testapps.CheckObjExists(&testCtx, backupKey, &batchv1.Job{}, false)).Should(Succeed()) }) It("should fail after job fails", func() { @@ -190,6 +190,69 @@ var _ = Describe("Backup Controller test", func() { }) }) + Context("deletes a full backup", func() { + var backupKey types.NamespacedName + + BeforeEach(func() { + By("creating a backup from backupPolicy: " + backupPolicyName) + backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). + SetBackupPolicyName(backupPolicyName). + SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + Create(&testCtx).GetObject() + backupKey = client.ObjectKeyFromObject(backup) + + By("waiting for finalizers to be added") + Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, backup *dataprotectionv1alpha1.Backup) { + g.Expect(backup.GetFinalizers()).ToNot(BeEmpty()) + })).Should(Succeed()) + + By("setting backup file path") + Eventually(testapps.ChangeObjStatus(&testCtx, backup, func() { + if backup.Status.Manifests == nil { + backup.Status.Manifests = &dataprotectionv1alpha1.ManifestsStatus{} + } + if backup.Status.Manifests.BackupTool == nil { + backup.Status.Manifests.BackupTool = &dataprotectionv1alpha1.BackupToolManifestsStatus{} + } + backup.Status.Manifests.BackupTool.FilePath = "/" + backupName + })).Should(Succeed()) + }) + + It("should create a Job for deleting backup files", func() { + By("deleting a Backup object") + testapps.DeleteObject(&testCtx, backupKey, &dataprotectionv1alpha1.Backup{}) + + By("checking new created Job") + jobKey := types.NamespacedName{ + Namespace: testCtx.DefaultNamespace, + Name: deleteBackupFilesJobNamePrefix + backupName, + } + Eventually(testapps.CheckObjExists(&testCtx, jobKey, + &batchv1.Job{}, true)).Should(Succeed()) + volumeName := "backup-" + backupRemotePVCName + Eventually(testapps.CheckObj(&testCtx, jobKey, func(g Gomega, job *batchv1.Job) { + Expect(job.Spec.Template.Spec.Volumes). + Should(ContainElement(corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: backupRemotePVCName, + }, + }, + })) + Expect(job.Spec.Template.Spec.Containers[0].VolumeMounts). + Should(ContainElement(corev1.VolumeMount{ + Name: volumeName, + MountPath: backupPathBase, + })) + })).Should(Succeed()) + + By("checking Backup object, it should be deleted") + Eventually(testapps.CheckObjExists(&testCtx, backupKey, + &dataprotectionv1alpha1.Backup{}, false)).Should(Succeed()) + }) + }) + Context("creates a snapshot backup", func() { var backupKey types.NamespacedName diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index 5b8e2f960..93a633853 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -69,8 +69,6 @@ type BackupPolicyReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile func (r *BackupPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) - // NOTES: // setup common request context reqCtx := intctrlutil.RequestCtx{ diff --git a/controllers/dataprotection/restorejob_controller.go b/controllers/dataprotection/restorejob_controller.go index 8ed9d24b7..6c00988bb 100644 --- a/controllers/dataprotection/restorejob_controller.go +++ b/controllers/dataprotection/restorejob_controller.go @@ -61,8 +61,6 @@ type RestoreJobReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile func (r *RestoreJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) - // NOTES: // setup common request context reqCtx := intctrlutil.RequestCtx{ diff --git a/controllers/dataprotection/type.go b/controllers/dataprotection/type.go index 8163fd0de..5d52b2f5c 100644 --- a/controllers/dataprotection/type.go +++ b/controllers/dataprotection/type.go @@ -40,8 +40,6 @@ const ( dataProtectionBackupTargetPodKey = "dataprotection.kubeblocks.io/target-pod-name" dataProtectionAnnotationCreateByPolicyKey = "dataprotection.kubeblocks.io/created-by-policy" - // error status - errorJobFailed = "JobFailed" // the key of persistentVolumeTemplate in the configmap. persistentVolumeTemplateKey = "persistentVolume" diff --git a/internal/testutil/apps/backup_factory.go b/internal/testutil/apps/backup_factory.go index 9b2bd55a5..d88a5d3f1 100644 --- a/internal/testutil/apps/backup_factory.go +++ b/internal/testutil/apps/backup_factory.go @@ -53,15 +53,15 @@ func (factory *MockBackupFactory) SetLabels(labels map[string]string) *MockBacku } func (factory *MockBackupFactory) SetBackLog(startTime, stopTime time.Time) *MockBackupFactory { - manitests := factory.get().Status.Manifests - if manitests == nil { - manitests = &dataprotectionv1alpha1.ManifestsStatus{} + manifests := factory.get().Status.Manifests + if manifests == nil { + manifests = &dataprotectionv1alpha1.ManifestsStatus{} } - if manitests.BackupLog == nil { - manitests.BackupLog = &dataprotectionv1alpha1.BackupLogStatus{} + if manifests.BackupLog == nil { + manifests.BackupLog = &dataprotectionv1alpha1.BackupLogStatus{} } - manitests.BackupLog.StartTime = &metav1.Time{Time: startTime} - manitests.BackupLog.StopTime = &metav1.Time{Time: stopTime} - factory.get().Status.Manifests = manitests + manifests.BackupLog.StartTime = &metav1.Time{Time: startTime} + manifests.BackupLog.StopTime = &metav1.Time{Time: stopTime} + factory.get().Status.Manifests = manifests return factory } From 36bb595f86e4cb4b04fb36d10edbe793ceb9faa8 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Sun, 23 Apr 2023 10:25:14 +0800 Subject: [PATCH 131/439] chore: refine kbcli for mongodb (#2834) --- .../helm/dashboards/mongodb-replicaset-overview.json | 4 ++-- deploy/mongodb/scripts/replicaset-post-start.tpl | 3 +++ deploy/mongodb/templates/backuppolicytemplate.yaml | 2 +- deploy/mongodb/templates/clusterdefinition.yaml | 9 +++++---- deploy/mongodb/templates/clusterversion.yaml | 2 +- .../mongodb/templates/sharding-clusterdefinition.yaml | 10 ++++++---- deploy/mongodb/values.yaml | 9 ++++----- internal/cli/cmd/cluster/create.go | 6 ++++++ 8 files changed, 28 insertions(+), 17 deletions(-) diff --git a/deploy/helm/dashboards/mongodb-replicaset-overview.json b/deploy/helm/dashboards/mongodb-replicaset-overview.json index 6a25fa80d..1d370d879 100644 --- a/deploy/helm/dashboards/mongodb-replicaset-overview.json +++ b/deploy/helm/dashboards/mongodb-replicaset-overview.json @@ -1008,7 +1008,7 @@ ], "yaxes": [ { - "format": "s", + "format": "ns", "label": "", "logBase": 1, "min": 0, @@ -5681,4 +5681,4 @@ "uid": "7lzrQGNikKB", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/deploy/mongodb/scripts/replicaset-post-start.tpl b/deploy/mongodb/scripts/replicaset-post-start.tpl index a485e48f7..88ef808b6 100644 --- a/deploy/mongodb/scripts/replicaset-post-start.tpl +++ b/deploy/mongodb/scripts/replicaset-post-start.tpl @@ -44,7 +44,10 @@ until is_inited=$(mongosh --quiet --port $PORT --eval "rs.status().ok" -u root - if [ $is_inited -eq 1 ]; then exit 0 fi; +sleep 10 +set -e mongosh --quiet --port $PORT --eval "rs.initiate({_id: \"$RPL_SET_NAME\", $CONFIGSVR members: [$MEMBERS]})"; +set +e (until mongosh --quiet --port $PORT --eval "rs.isMaster().isWritablePrimary"|grep true; do sleep 1; done; echo "create user"; diff --git a/deploy/mongodb/templates/backuppolicytemplate.yaml b/deploy/mongodb/templates/backuppolicytemplate.yaml index a2377e97b..177dd0fbd 100644 --- a/deploy/mongodb/templates/backuppolicytemplate.yaml +++ b/deploy/mongodb/templates/backuppolicytemplate.yaml @@ -8,7 +8,7 @@ metadata: spec: clusterDefinitionRef: mongodb backupPolicies: - - componentDefRef: replicaset + - componentDefRef: mongodb ttl: 7d schedule: baseBackup: diff --git a/deploy/mongodb/templates/clusterdefinition.yaml b/deploy/mongodb/templates/clusterdefinition.yaml index dc2e6a5e9..6a5d3eb38 100644 --- a/deploy/mongodb/templates/clusterdefinition.yaml +++ b/deploy/mongodb/templates/clusterdefinition.yaml @@ -16,7 +16,7 @@ spec: headlessHost: "$(POD_NAME_PREFIX)-0.$(HEADLESS_SVC_FQDN)" headlessPort: "$(SVC_PORT_tcp-monogdb)" componentDefs: - - name: replicaset + - name: mongodb characterType: mongodb scriptSpecs: - name: mongodb-scripts @@ -59,8 +59,9 @@ spec: updateStrategy: Serial probes: roleProbe: - periodSeconds: 2 - failureThreshold: 3 + failureThreshold: {{ .Values.roleProbe.failureThreshold }} + periodSeconds: {{ .Values.roleProbe.periodSeconds }} + timeoutSeconds: {{ .Values.roleProbe.timeoutSeconds }} service: ports: - protocol: TCP @@ -93,7 +94,7 @@ spec: exec: command: - /scripts/replicaset-post-start.sh - - REPLICASET + - MONGODB volumeMounts: - mountPath: /data/mongodb name: data diff --git a/deploy/mongodb/templates/clusterversion.yaml b/deploy/mongodb/templates/clusterversion.yaml index 4a15db2db..9eb4ef83a 100644 --- a/deploy/mongodb/templates/clusterversion.yaml +++ b/deploy/mongodb/templates/clusterversion.yaml @@ -7,7 +7,7 @@ metadata: spec: clusterDefinitionRef: mongodb componentVersions: - - componentDefRef: replicaset + - componentDefRef: mongodb versionsContext: containers: - name: mongodb diff --git a/deploy/mongodb/templates/sharding-clusterdefinition.yaml b/deploy/mongodb/templates/sharding-clusterdefinition.yaml index 2caed0dfd..96d4dc01e 100644 --- a/deploy/mongodb/templates/sharding-clusterdefinition.yaml +++ b/deploy/mongodb/templates/sharding-clusterdefinition.yaml @@ -60,8 +60,9 @@ spec: updateStrategy: Serial probes: roleProbe: - periodSeconds: 2 - failureThreshold: 3 + failureThreshold: {{ .Values.roleProbe.failureThreshold }} + periodSeconds: {{ .Values.roleProbe.periodSeconds }} + timeoutSeconds: {{ .Values.roleProbe.timeoutSeconds }} service: ports: - name: configsvr @@ -109,8 +110,9 @@ spec: updateStrategy: BestEffortParallel probes: roleProbe: - periodSeconds: 2 - failureThreshold: 3 + failureThreshold: {{ .Values.roleProbe.failureThreshold }} + periodSeconds: {{ .Values.roleProbe.periodSeconds }} + timeoutSeconds: {{ .Values.roleProbe.timeoutSeconds }} service: ports: - name: shard diff --git a/deploy/mongodb/values.yaml b/deploy/mongodb/values.yaml index 313f1731c..ae4db120f 100644 --- a/deploy/mongodb/values.yaml +++ b/deploy/mongodb/values.yaml @@ -12,11 +12,10 @@ clusterVersionOverride: "" nameOverride: "" fullnameOverride: "" -replicaset: - roleProbe: - failureThreshold: 2 - periodSeconds: 1 - timeoutSeconds: 1 +roleProbe: + failureThreshold: 3 + periodSeconds: 2 + timeoutSeconds: 1 ## Authentication parameters ## diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index b8aedb212..1a80e2f27 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -475,6 +475,12 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map return "2" } } + // the default replicas is 3 if not set by command flag, for Consensus workload + if c.WorkloadType == appsv1alpha1.Consensus { + if key == keyReplicas { + return "3" + } + } if c.CharacterType == "redis" && c.Name == "redis-sentinel" { switch key { From 265d5bd317ed9a7566688cdb0c919932b8d4f7fd Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Fri, 21 Apr 2023 22:41:39 +0800 Subject: [PATCH 132/439] chore: fixed scripts typo --- .github/utils/bugs_not_in_current_milestone.sh | 6 +++--- .github/utils/create_releasing_pr.sh | 6 +++--- .github/utils/current_milestone_bugs.sh | 6 +++--- .github/utils/feature_triage.sh | 6 +++--- .github/utils/merge_releasing_pr.sh | 13 +++---------- .github/utils/sanitize_release_body.sh | 6 +++--- 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/.github/utils/bugs_not_in_current_milestone.sh b/.github/utils/bugs_not_in_current_milestone.sh index 782d93ef5..1c0176fd8 100755 --- a/.github/utils/bugs_not_in_current_milestone.sh +++ b/.github/utils/bugs_not_in_current_milestone.sh @@ -6,9 +6,9 @@ set -o pipefail # requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. -worddir=$(dirname $0) -. ${worddir}/gh_env -. ${worddir}/functions.bash +workdir=$(dirname $0) +. ${workdir}/gh_env +. ${workdir}/functions.bash process_issue_rows() { for ((i = 0; i < ${item_count}; i++)) diff --git a/.github/utils/create_releasing_pr.sh b/.github/utils/create_releasing_pr.sh index 27cbd66ef..8fa1bc550 100755 --- a/.github/utils/create_releasing_pr.sh +++ b/.github/utils/create_releasing_pr.sh @@ -6,9 +6,9 @@ set -o pipefail # requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. -worddir=$(dirname $0) -. ${worddir}/gh_env -. ${worddir}/functions.bash +workdir=$(dirname $0) +. ${workdir}/gh_env +. ${workdir}/functions.bash set -x diff --git a/.github/utils/current_milestone_bugs.sh b/.github/utils/current_milestone_bugs.sh index 21622c97e..ff1a294d2 100755 --- a/.github/utils/current_milestone_bugs.sh +++ b/.github/utils/current_milestone_bugs.sh @@ -6,9 +6,9 @@ set -o pipefail # requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. -worddir=$(dirname $0) -. ${worddir}/gh_env -. ${worddir}/functions.bash +workdir=$(dirname $0) +. ${workdir}/gh_env +. ${workdir}/functions.bash LABELS=${LABELS:-'kind/bug,bug'} #"severity/critical,severity/major,severity/minor,severity/normal" diff --git a/.github/utils/feature_triage.sh b/.github/utils/feature_triage.sh index e174ccaee..997ec7189 100755 --- a/.github/utils/feature_triage.sh +++ b/.github/utils/feature_triage.sh @@ -6,9 +6,9 @@ set -o pipefail # requires `git`, `gh`, and `jq` commands, ref. https://cli.github.com/manual/installation for installation guides. -worddir=$(dirname $0) -. ${worddir}/gh_env -. ${worddir}/functions.bash +workdir=$(dirname $0) +. ${workdir}/gh_env +. ${workdir}/functions.bash print_issue_rows() { for ((i = 0; i < ${item_count}; i++)) diff --git a/.github/utils/merge_releasing_pr.sh b/.github/utils/merge_releasing_pr.sh index cd835c617..bf200414e 100755 --- a/.github/utils/merge_releasing_pr.sh +++ b/.github/utils/merge_releasing_pr.sh @@ -6,9 +6,9 @@ set -o pipefail # requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. -worddir=$(dirname $0) -. ${worddir}/gh_env -. ${worddir}/functions.bash +workdir=$(dirname $0) +. ${workdir}/gh_env +. ${workdir}/functions.bash echo "Merging ${PR_TITLE}" @@ -24,11 +24,4 @@ echo "pr_mergeable=${pr_mergeable}" if [ "${pr_merge_status}" == "CLEAN" ] && [ "${pr_mergeable}" == "MERGEABLE" ]; then gh pr --repo apecloud/kubeblocks merge ${pr_number} --merge -# # gh API ref: https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#merge-a-pull-request -# gh api \ -# --method PUT \ -# --header "Accept: application/vnd.github+json" \ -# --header "X-GitHub-Api-Version: 2022-11-28" \ -# /repos/${OWNER}/${REPO}/pulls/${pr_number}/merge \ -# -f merge_method=merge fi \ No newline at end of file diff --git a/.github/utils/sanitize_release_body.sh b/.github/utils/sanitize_release_body.sh index 98f32fe72..bf273208a 100755 --- a/.github/utils/sanitize_release_body.sh +++ b/.github/utils/sanitize_release_body.sh @@ -6,9 +6,9 @@ set -o pipefail # requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. -worddir=$(dirname $0) -. ${worddir}/gh_env -. ${worddir}/functions.bash +workdir=$(dirname $0) +. ${workdir}/gh_env +. ${workdir}/functions.bash TAG=${TAG:-} GITHUB_REF=${GITHUB_REF:-} From 1eb0381bec70088f97994d737b47130092bdecb1 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sat, 22 Apr 2023 16:21:54 +0800 Subject: [PATCH 133/439] chore: tidy up .github/workflows/*.sh use 'gh pr' cmd --- .github/utils/create_releasing_pr.sh | 2 +- .github/utils/merge_releasing_pr.sh | 52 ++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/.github/utils/create_releasing_pr.sh b/.github/utils/create_releasing_pr.sh index 8fa1bc550..4d17de757 100755 --- a/.github/utils/create_releasing_pr.sh +++ b/.github/utils/create_releasing_pr.sh @@ -18,4 +18,4 @@ git pull git merge origin/${HEAD_BRANCH} echo "Creating ${PR_TITLE}" -gh pr create --head ${HEAD_BRANCH} --base ${BASE_BRANCH} --title "${PR_TITLE}" --body "" \ No newline at end of file +gh pr create --head ${HEAD_BRANCH} --base ${BASE_BRANCH} --title "${PR_TITLE}" --body "" --label "releasing-task" \ No newline at end of file diff --git a/.github/utils/merge_releasing_pr.sh b/.github/utils/merge_releasing_pr.sh index bf200414e..09af066d9 100755 --- a/.github/utils/merge_releasing_pr.sh +++ b/.github/utils/merge_releasing_pr.sh @@ -10,18 +10,48 @@ workdir=$(dirname $0) . ${workdir}/gh_env . ${workdir}/functions.bash -echo "Merging ${PR_TITLE}" +get_pr_status() { + pr_info=$(gh pr --repo ${OWNER}/${REPO} view ${pr_number} --json "mergeStateStatus,mergeable") + pr_merge_status=$(echo ${pr_info} | jq -r '.mergeStateStatus') + pr_mergeable=$(echo ${pr_info} | jq -r '.mergeable') + if [ -n "$DEBUG" ]; then + echo "pr_number=${pr_number}" + echo "pr_merge_status=${pr_merge_status}" + echo "pr_mergeable=${pr_mergeable}" + fi +} +echo "Merging ${PR_TITLE}" -pr_info=$(gh pr list --repo ${OWNER}/${REPO} --base ${BASE_BRANCH} --json "number,url,mergeStateStatus,mergeable" ) +retry_times=0 +pr_info=$(gh pr list --repo ${OWNER}/${REPO} --base ${BASE_BRANCH} --json "number" ) pr_number=$(echo ${pr_info} | jq -r '.[0].number') -pr_merge_status=$(echo ${pr_info} | jq -r '.[0].mergeStateStatus') -pr_mergeable=$(echo ${pr_info} | jq -r '.[0].mergeable') - -echo "pr_number=${pr_number}" -echo "pr_merge_status=${pr_merge_status}" -echo "pr_mergeable=${pr_mergeable}" +get_pr_status + + +if [ "${pr_mergeable}" == "MERGEABLE" ]; then + if [ "${pr_merge_status}" == "BLOCKED" ]; then + echo "Approve PR #${pr_number}" + gh pr --repo ${OWNER}/${REPO} comment ${pr_number} --body "/approve" + sleep 5 + get_pr_status + fi + + if [ "${pr_merge_status}" == "UNSTABLE" ]; then + retry_times=100 + while [ $retry_times -gt 0 ] + do + ((retry_times--)) + sleep 5 + get_pr_status + done + fi + + if [ "${pr_merge_status}" == "CLEAN" ]; then + echo "Merging PR #${pr_number}" + # gh pr --repo ${OWNER}/${REPO} merge ${pr_number} --rebase + exit 0 + fi +fi +exit 1 -if [ "${pr_merge_status}" == "CLEAN" ] && [ "${pr_mergeable}" == "MERGEABLE" ]; then - gh pr --repo apecloud/kubeblocks merge ${pr_number} --merge -fi \ No newline at end of file From 04170cf5d5eb35055cdc0f074283d14cc6295477 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sat, 22 Apr 2023 16:22:06 +0800 Subject: [PATCH 134/439] chore: add merge-releasing-branch job --- .github/workflows/release-version.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml index 0b1469c08..535bbcde5 100644 --- a/.github/workflows/release-version.yml +++ b/.github/workflows/release-version.yml @@ -15,6 +15,13 @@ env: jobs: + merge-releasing-branch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Auto merge releasing PR + run: ./.github/utils/merge_releasing_pr.sh + release-test: runs-on: [ self-hosted, eks-fargate-runner ] steps: @@ -32,7 +39,6 @@ jobs: steps: - name: checkout branch ${{ github.ref_name }} uses: actions/checkout@v3 - - name: push tag uses: mathieudutour/github-tag-action@v6.1 with: From 945f50ea7b6c4e126a82bb6fe1460db1c68f3ad9 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sun, 23 Apr 2023 10:50:49 +0800 Subject: [PATCH 135/439] chore: added checking handling for merge_releasing_pr.sh --- .github/utils/merge_releasing_pr.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/utils/merge_releasing_pr.sh b/.github/utils/merge_releasing_pr.sh index 09af066d9..efe3f7e9e 100755 --- a/.github/utils/merge_releasing_pr.sh +++ b/.github/utils/merge_releasing_pr.sh @@ -25,10 +25,14 @@ echo "Merging ${PR_TITLE}" retry_times=0 pr_info=$(gh pr list --repo ${OWNER}/${REPO} --base ${BASE_BRANCH} --json "number" ) +pr_len=$(echo ${pr_info} | jq -r '. | length') +if [ "${pr_len}" == "0" ]; then +exit 0 +fi + pr_number=$(echo ${pr_info} | jq -r '.[0].number') get_pr_status - if [ "${pr_mergeable}" == "MERGEABLE" ]; then if [ "${pr_merge_status}" == "BLOCKED" ]; then echo "Approve PR #${pr_number}" From 7179eeab4071210f6a8f4eb685b9e923d75114c9 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sun, 23 Apr 2023 10:53:42 +0800 Subject: [PATCH 136/439] chore: added checking handling for merge_releasing_pr.sh --- .github/utils/merge_releasing_pr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/utils/merge_releasing_pr.sh b/.github/utils/merge_releasing_pr.sh index efe3f7e9e..a07446985 100755 --- a/.github/utils/merge_releasing_pr.sh +++ b/.github/utils/merge_releasing_pr.sh @@ -53,7 +53,7 @@ if [ "${pr_mergeable}" == "MERGEABLE" ]; then if [ "${pr_merge_status}" == "CLEAN" ]; then echo "Merging PR #${pr_number}" - # gh pr --repo ${OWNER}/${REPO} merge ${pr_number} --rebase + gh pr --repo ${OWNER}/${REPO} merge ${pr_number} --rebase exit 0 fi fi From 36b0dc0642fa135c49aed5eaf102415f996a598a Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sun, 23 Apr 2023 11:08:04 +0800 Subject: [PATCH 137/439] Support/fix ci scripts typo (#2843) From 70828b6b81316e94f9ecfde5e7d75c6c6ab0a251 Mon Sep 17 00:00:00 2001 From: free6om Date: Sun, 23 Apr 2023 11:38:30 +0800 Subject: [PATCH 138/439] chore: lifecycle chain in cluster controller (#2604) an enhancement to PR #1571 with following improvements: 1. add `AddTransformers`/`AddParallelTransformers` interface and move plan `Build` stage back to cluster controller 2. handle deletion first to support fast&force deletion 3. emit events after cluster status updated (in the `Execute` stage) 4. sample ut in `transformer_sts_horizontal_scaling_test.go` to show how to test a transformer without `envtest` 5. more docs and other minor improvements --- controllers/apps/cluster_controller.go | 80 ++- controllers/apps/operations/stop.go | 14 +- docs/user_docs/cli/kbcli_cluster_create.md | 19 +- internal/controller/graph/doc.go | 16 +- internal/controller/graph/plan_builder.go | 13 +- internal/controller/graph/transformer.go | 42 +- internal/controller/graph/validator.go | 33 -- .../lifecycle/cluster_plan_builder.go | 512 +++++------------- .../lifecycle/cluster_plan_utils.go | 14 - .../lifecycle/cluster_status_conditions.go | 17 + .../controller/lifecycle/transform_types.go | 7 - .../controller/lifecycle/transform_utils.go | 36 +- .../transformer_backup_policy_tpl.go | 45 +- .../lifecycle/transformer_cluster.go | 50 +- .../lifecycle/transformer_cluster_deletion.go | 119 ++++ .../lifecycle/transformer_cluster_status.go | 337 +++++++----- .../lifecycle/transformer_config.go | 8 +- .../lifecycle/transformer_credential.go | 8 +- .../lifecycle/transformer_fill_class.go | 32 +- ...ster_labels.go => transformer_fix_meta.go} | 25 +- .../controller/lifecycle/transformer_init.go | 36 +- .../lifecycle/transformer_object_action.go | 37 +- .../lifecycle/transformer_ownership.go | 10 +- .../controller/lifecycle/transformer_pitr.go | 49 ++ .../transformer_sts_horizontal_scaling.go | 203 +++---- ...transformer_sts_horizontal_scaling_test.go | 23 +- .../lifecycle/transformer_sts_pvc.go | 21 +- .../lifecycle/transformer_tls_certs.go | 24 +- ...sformer_validate_and_load_ref_resources.go | 88 +++ .../transformer_validate_enable_logs.go | 43 ++ .../lifecycle/transformer_workloads_last.go | 52 ++ ..._terminate.go => transformers_parallel.go} | 44 +- .../lifecycle/validator_enable_logs.go | 31 -- internal/controller/plan/tls_utils.go | 56 +- 34 files changed, 1175 insertions(+), 969 deletions(-) delete mode 100644 internal/controller/graph/validator.go create mode 100644 internal/controller/lifecycle/transformer_cluster_deletion.go rename internal/controller/lifecycle/{transformer_fix_cluster_labels.go => transformer_fix_meta.go} (60%) create mode 100644 internal/controller/lifecycle/transformer_pitr.go create mode 100644 internal/controller/lifecycle/transformer_validate_and_load_ref_resources.go create mode 100644 internal/controller/lifecycle/transformer_validate_enable_logs.go create mode 100644 internal/controller/lifecycle/transformer_workloads_last.go rename internal/controller/lifecycle/{transformer_do_not_terminate.go => transformers_parallel.go} (52%) delete mode 100644 internal/controller/lifecycle/validator_enable_logs.go diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index 0deca83e5..a8f16aec7 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -134,16 +134,82 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return intctrlutil.RequeueWithError(err, reqCtx.Log, "") } - planBuilder := lifecycle.NewClusterPlanBuilder(reqCtx, r.Client, req, r.Recorder) + // the cluster reconciliation loop is a 3-stage model: plan Init, plan Build and plan Execute + // Init stage + planBuilder := lifecycle.NewClusterPlanBuilder(reqCtx, r.Client, req) if err := planBuilder.Init(); err != nil { return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } else if err := planBuilder.Validate(); err != nil { - return requeueError(err) - } else if plan, err := planBuilder.Build(); err != nil { - return requeueError(err) - } else if err = plan.Execute(); err != nil { - return requeueError(err) } + + // Build stage + // what you should do in most cases is writing your transformer. + // + // here are the how-to tips: + // 1. one transformer for one scenario + // 2. try not to modify the current transformers, make a new one + // 3. transformers are independent with each-other, with some exceptions. + // Which means transformers' order is not important in most cases. + // If you don't know where to put your transformer, append it to the end and that would be ok. + // 4. don't use client.Client for object write, use client.ReadonlyClient for object read. + // If you do need to create/update/delete object, make your intent operation a lifecycleVertex and put it into the DAG. + // + // TODO: transformers are vertices, theirs' dependencies are edges, make plan Build stage a DAG. + plan, errBuild := planBuilder. + AddTransformer( + // handle deletion + // handle cluster deletion first + &lifecycle.ClusterDeletionTransformer{}, + // fix meta + // fix finalizer and cd&cv labels + &lifecycle.FixMetaTransformer{}, + // validate + // validate cd & cv's existence and availability + &lifecycle.ValidateAndLoadRefResourcesTransformer{}, + // validate config + &lifecycle.ValidateEnableLogsTransformer{}, + // fix spec + // fill class related info + &lifecycle.FillClassTransformer{}, + // generate objects + // cluster to K8s objects and put them into dag + &lifecycle.ClusterTransformer{Client: r.Client}, + // tls certs secret + &lifecycle.TLSCertsTransformer{}, + // transform backupPolicy tpl to backuppolicy.dataprotection.kubeblocks.io + &lifecycle.BackupPolicyTPLTransformer{}, + // add our finalizer to all objects + &lifecycle.OwnershipTransformer{}, + // make all workload objects depending on credential secret + &lifecycle.CredentialTransformer{}, + // make all workload objects depending on all none workload objects + &lifecycle.WorkloadsLastTransformer{}, + // make config configmap immutable + &lifecycle.ConfigTransformer{}, + // read old snapshot from cache, and generate diff plan + &lifecycle.ObjectActionTransformer{}, + // day-2 ops + // horizontal scaling + &lifecycle.StsHorizontalScalingTransformer{}, + // stateful set pvc Update + &lifecycle.StsPVCTransformer{}, + // update cluster status + &lifecycle.ClusterStatusTransformer{}, + // handle PITR + &lifecycle.PITRTransformer{Client: r.Client}, + // always safe to put your transformer below + ). + Build() + + // Execute stage + // errBuild not nil means build stage partial success or validation error + // execute the plan first, delay error handling + if errExec := plan.Execute(); errExec != nil { + return requeueError(errExec) + } + if errBuild != nil { + return requeueError(errBuild) + } + return intctrlutil.Reconciled() } diff --git a/controllers/apps/operations/stop.go b/controllers/apps/operations/stop.go index 6591ec658..9211ce0cd 100644 --- a/controllers/apps/operations/stop.go +++ b/controllers/apps/operations/stop.go @@ -17,15 +17,16 @@ limitations under the License. package operations import ( + "context" "encoding/json" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" - "github.com/apecloud/kubeblocks/internal/controller/lifecycle" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -90,7 +91,7 @@ func (stop StopOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCtx, cli cl return expectProgressCount, completedCount, err } // TODO: delete the configmaps of the cluster should be removed from the opsRequest after refactor. - if err := lifecycle.DeleteConfigMaps(reqCtx.Ctx, cli, opsRes.Cluster); err != nil { + if err := deleteConfigMaps(reqCtx.Ctx, cli, opsRes.Cluster); err != nil { return expectProgressCount, completedCount, err } return expectProgressCount, completedCount, nil @@ -134,3 +135,12 @@ func getCompMapFromLastConfiguration(opsRequest *appsv1alpha1.OpsRequest) realAf } return realChangedMap } + +func deleteConfigMaps(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster) error { + inNS := client.InNamespace(cluster.Namespace) + ml := client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.GetName(), + constant.AppManagedByLabelKey: constant.AppName, + } + return cli.DeleteAllOf(ctx, &corev1.ConfigMap{}, inNS, ml) +} diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index 7a7bf79b3..9a3c488c1 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -70,27 +70,11 @@ kbcli cluster create [NAME] [flags] ### Options ``` -<<<<<<< HEAD - --backup string Set a source backup to restore data - --cluster-definition string Specify cluster definition, run "kbcli cd list" to show all available cluster definitions - --cluster-version string Specify cluster version, run "kbcli cv list" to show all available cluster versions, use the latest version if not specified - --enable-all-logs Enable advanced application all log extraction, and true will ignore enabledLogs of component level, default is false - -h, --help help for create - --monitor Set monitor enabled and inject metrics exporter (default true) - --node-labels stringToString Node label selector (default []) - --pod-anti-affinity string Pod anti-affinity type, one of: (Preferred, Required) (default "Preferred") - --set stringArray Set the cluster resource including cpu, memory, replicas and storage, or you can just specify the class, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi or --set class=general-1c1g) - -f, --set-file string Use yaml file, URL, or stdin to set the cluster resource - --tenancy string Tenancy options, one of: (SharedNode, DedicatedNode) (default "SharedNode") - --termination-policy string Termination policy, one of: (DoNotTerminate, Halt, Delete, WipeOut) (default "Delete") - --tolerations strings Tolerations for cluster, such as '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule"' - --topology-keys stringArray Topology keys for affinity -======= --backup string Set a source backup to restore data --cluster-definition string Specify cluster definition, run "kbcli cd list" to show all available cluster definitions --cluster-version string Specify cluster version, run "kbcli cv list" to show all available cluster versions, use the latest version if not specified --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --enable-all-logs Enable advanced application all log extraction, and true will ignore enabledLogs of component level (default true) + --enable-all-logs Enable advanced application all log extraction, and true will ignore enabledLogs of component level, default is false -h, --help help for create --monitor Set monitor enabled and inject metrics exporter (default true) --node-labels stringToString Node label selector (default []) @@ -102,7 +86,6 @@ kbcli cluster create [NAME] [flags] --termination-policy string Termination policy, one of: (DoNotTerminate, Halt, Delete, WipeOut) (default "Delete") --tolerations strings Tolerations for cluster, such as '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule"' --topology-keys stringArray Topology keys for affinity ->>>>>>> 8158f9bf149e7acb13c56693a6c2e8165f1d5bca ``` ### Options inherited from parent commands diff --git a/internal/controller/graph/doc.go b/internal/controller/graph/doc.go index a29985523..7657d6f50 100644 --- a/internal/controller/graph/doc.go +++ b/internal/controller/graph/doc.go @@ -15,22 +15,24 @@ limitations under the License. */ /* -Package graph tries to model the controller reconciliation loop in a more structure way. -It structures the reconciliation loop to 4 stage: Init, Validate, Build and Execute. +Package graph tries to model the controller reconciliation loop in a more structured way. +It structures the reconciliation loop to 3 stage: Init, Build and Execute. # Initialization Stage the Init stage is for meta loading, object query etc. Try loading infos that used in the following stages. -# Validation Stage +# Building Stage -Validating everything (object spec is legal, resources in K8s cluster are enough etc.) in this stage -to make sure the following Build and Execute stages can go well. +## Validation -# Building Stage +The first part of Building is Validation, +which Validates everything (object spec is legal, resources in K8s cluster are enough etc.) +to make sure the following Build and Execute parts can go well. -The Build stage's target is to generate an execution plan. +## Building +The Building part's target is to generate an execution plan. The plan is composed by a DAG which represents the actions that should be taken on all K8s native objects owned by the controller, a group of Transformers which transform the initial DAG to the final one, and a WalkFunc which does the real action when the final DAG is walked through. diff --git a/internal/controller/graph/plan_builder.go b/internal/controller/graph/plan_builder.go index 7620bbc95..d9f29d3ec 100644 --- a/internal/controller/graph/plan_builder.go +++ b/internal/controller/graph/plan_builder.go @@ -18,12 +18,23 @@ package graph // PlanBuilder builds a Plan by applying a group of Transformer to an empty DAG. type PlanBuilder interface { + // Init loads the primary object to be reconciled, and does meta initialization Init() error - Validate() error + + // AddTransformer adds transformers to the builder in sequence order. + // And the transformers will be executed in the add order. + AddTransformer(transformer ...Transformer) PlanBuilder + + // AddParallelTransformer adds transformers to the builder. + // And the transformers will be executed in parallel. + AddParallelTransformer(transformer ...Transformer) PlanBuilder + + // Build runs all the transformers added by AddTransformer and/or AddParallelTransformer. Build() (Plan, error) } // Plan defines the final actions should be executed. type Plan interface { + // Execute the plan Execute() error } diff --git a/internal/controller/graph/transformer.go b/internal/controller/graph/transformer.go index 03646013e..ad39880b3 100644 --- a/internal/controller/graph/transformer.go +++ b/internal/controller/graph/transformer.go @@ -16,18 +16,52 @@ limitations under the License. package graph +import ( + "context" + "errors" + + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + + "github.com/apecloud/kubeblocks/internal/controller/client" +) + +// TransformContext is used by Transformer.Transform +type TransformContext interface { + GetContext() context.Context + GetClient() client.ReadonlyClient + GetRecorder() record.EventRecorder + GetLogger() logr.Logger +} + // Transformer transforms a DAG to a new version type Transformer interface { - Transform(dag *DAG) error + Transform(ctx TransformContext, dag *DAG) error } +// TransformerChain chains a group Transformer together type TransformerChain []Transformer -func (t *TransformerChain) ApplyTo(dag *DAG) error { +// ErrFastReturn is used to stop the Transformer chain for some purpose. +// Use it in Transformer.Transform when all jobs have done and no need to run following transformers +var ErrFastReturn = errors.New("fast return") + +// ApplyTo applies TransformerChain t to dag +func (t *TransformerChain) ApplyTo(ctx TransformContext, dag *DAG) error { + if t == nil { + return nil + } for _, transformer := range *t { - if err := transformer.Transform(dag); err != nil { - return err + if err := transformer.Transform(ctx, dag); err != nil { + return fastReturnErrorToNil(err) } } return nil } + +func fastReturnErrorToNil(err error) error { + if err == ErrFastReturn { + return nil + } + return err +} diff --git a/internal/controller/graph/validator.go b/internal/controller/graph/validator.go deleted file mode 100644 index 3272c2c64..000000000 --- a/internal/controller/graph/validator.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright ApeCloud, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package graph - -// Validator validate everything is ok before Build the plan -type Validator interface { - Validate() error -} - -type ValidatorChain []Validator - -func (v *ValidatorChain) WalkThrough() error { - for _, validator := range *v { - if err := validator.Validate(); err != nil { - return err - } - } - return nil -} diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 79fb493b5..0065f1292 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -22,306 +22,177 @@ import ( "fmt" "reflect" - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" "github.com/apecloud/kubeblocks/internal/constant" - types2 "github.com/apecloud/kubeblocks/internal/controller/client" + roclient "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/graph" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) +// TODO: cluster plan builder can be abstracted as a common flow + +// ClusterTransformContext a graph.TransformContext implementation for Cluster reconciliation +type ClusterTransformContext struct { + context.Context + Client roclient.ReadonlyClient + record.EventRecorder + logr.Logger + Cluster *appsv1alpha1.Cluster + OrigCluster *appsv1alpha1.Cluster + ClusterDef *appsv1alpha1.ClusterDefinition + ClusterVer *appsv1alpha1.ClusterVersion +} + // clusterPlanBuilder a graph.PlanBuilder implementation for Cluster reconciliation type clusterPlanBuilder struct { - ctx intctrlutil.RequestCtx - cli client.Client - req ctrl.Request - recorder record.EventRecorder - cluster *appsv1alpha1.Cluster - originCluster appsv1alpha1.Cluster + req ctrl.Request + cli client.Client + transCtx *ClusterTransformContext + transformers graph.TransformerChain } // clusterPlan a graph.Plan implementation for Cluster reconciliation type clusterPlan struct { - ctx intctrlutil.RequestCtx - cli client.Client - recorder record.EventRecorder dag *graph.DAG walkFunc graph.WalkFunc - cluster *appsv1alpha1.Cluster + cli client.Client + transCtx *ClusterTransformContext } +var _ graph.TransformContext = &ClusterTransformContext{} var _ graph.PlanBuilder = &clusterPlanBuilder{} var _ graph.Plan = &clusterPlan{} -func (c *clusterPlanBuilder) Init() error { - cluster := &appsv1alpha1.Cluster{} - if err := c.cli.Get(c.ctx.Ctx, c.req.NamespacedName, cluster); err != nil { - return err - } - c.cluster = cluster - c.originCluster = *cluster.DeepCopy() - // handles the cluster phase and ops condition first to indicates what the current cluster is doing. - c.handleClusterPhase() - c.handleLatestOpsRequestProcessingCondition() - return nil +// TransformContext implementation + +func (c *ClusterTransformContext) GetContext() context.Context { + return c.Context } -// updateClusterPhase handles the cluster phase and ops condition first to indicates what the current cluster is doing. -func (c *clusterPlanBuilder) handleClusterPhase() { - clusterPhase := c.cluster.Status.Phase - if isClusterUpdating(*c.cluster) { - if clusterPhase == "" { - c.cluster.Status.Phase = appsv1alpha1.CreatingClusterPhase - } else if clusterPhase != appsv1alpha1.CreatingClusterPhase { - c.cluster.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase - } - } +func (c *ClusterTransformContext) GetClient() roclient.ReadonlyClient { + return c.Client } -// updateLatestOpsRequestProcessingCondition handles the latest opsRequest processing condition. -func (c *clusterPlanBuilder) handleLatestOpsRequestProcessingCondition() { - opsRecords, _ := opsutil.GetOpsRequestSliceFromCluster(c.cluster) - if len(opsRecords) == 0 { - return - } - ops := opsRecords[0] - opsBehaviour, ok := appsv1alpha1.OpsRequestBehaviourMapper[ops.Type] - if !ok { - return - } - opsCondition := newOpsRequestProcessingCondition(ops.Name, string(ops.Type), opsBehaviour.ProcessingReasonInClusterCondition) - oldCondition := meta.FindStatusCondition(c.cluster.Status.Conditions, opsCondition.Type) - if oldCondition == nil { - // if this condition not exists, insert it to the first position. - opsCondition.LastTransitionTime = metav1.Now() - c.cluster.Status.Conditions = append([]metav1.Condition{opsCondition}, c.cluster.Status.Conditions...) - } else { - meta.SetStatusCondition(&c.cluster.Status.Conditions, opsCondition) - } +func (c *ClusterTransformContext) GetRecorder() record.EventRecorder { + return c.EventRecorder } -func (c *clusterPlanBuilder) Validate() error { - var err error - defer func() { - if err != nil { - _ = c.updateClusterStatusWithCondition(newFailedProvisioningStartedCondition(err.Error(), ReasonPreCheckFailed)) - } - }() +func (c *ClusterTransformContext) GetLogger() logr.Logger { + return c.Logger +} - validateExistence := func(key client.ObjectKey, object client.Object) error { - err = c.cli.Get(c.ctx.Ctx, key, object) - if err != nil { - return newRequeueError(requeueDuration, err.Error()) - } - return nil - } +// PlanBuilder implementation - // validate cd & cv existences - cd := &appsv1alpha1.ClusterDefinition{} - if err = validateExistence(types.NamespacedName{Name: c.cluster.Spec.ClusterDefRef}, cd); err != nil { +func (c *clusterPlanBuilder) Init() error { + cluster := &appsv1alpha1.Cluster{} + if err := c.cli.Get(c.transCtx.Context, c.req.NamespacedName, cluster); err != nil { return err } - var cv *appsv1alpha1.ClusterVersion - if len(c.cluster.Spec.ClusterVersionRef) > 0 { - cv = &appsv1alpha1.ClusterVersion{} - if err = validateExistence(types.NamespacedName{Name: c.cluster.Spec.ClusterVersionRef}, cv); err != nil { - return err - } - } - - // validate cd & cv availability - if cd.Status.Phase != appsv1alpha1.AvailablePhase || (cv != nil && cv.Status.Phase != appsv1alpha1.AvailablePhase) { - message := fmt.Sprintf("ref resource is unavailable, this problem needs to be solved first. cd: %v, cv: %v", cd, cv) - err = errors.New(message) - return newRequeueError(requeueDuration, message) - } - - // validate logs - // and a sample validator chain - chain := &graph.ValidatorChain{ - &enableLogsValidator{cluster: c.cluster, clusterDef: cd}, - } - if err = chain.WalkThrough(); err != nil { - return newRequeueError(requeueDuration, err.Error()) - } + c.transCtx.Cluster = cluster + c.transCtx.OrigCluster = cluster.DeepCopy() + c.transformers = append(c.transformers, &initTransformer{ + cluster: c.transCtx.Cluster, + originCluster: c.transCtx.OrigCluster, + }) return nil } -func (c *clusterPlanBuilder) handleProvisionStartedCondition() { - // set provisioning cluster condition - condition := newProvisioningStartedCondition(c.cluster.Name, c.cluster.Generation) - oldCondition := meta.FindStatusCondition(c.cluster.Status.Conditions, condition.Type) - if conditionIsChanged(oldCondition, condition) { - meta.SetStatusCondition(&c.cluster.Status.Conditions, condition) - c.recorder.Event(c.cluster, corev1.EventTypeNormal, condition.Reason, condition.Message) - } +func (c *clusterPlanBuilder) AddTransformer(transformer ...graph.Transformer) graph.PlanBuilder { + c.transformers = append(c.transformers, transformer...) + return c +} + +func (c *clusterPlanBuilder) AddParallelTransformer(transformer ...graph.Transformer) graph.PlanBuilder { + c.transformers = append(c.transformers, &ParallelTransformers{transformers: transformer}) + return c } -// Build only cluster Creation, Update and Deletion supported. +// Build runs all transformers to generate a plan func (c *clusterPlanBuilder) Build() (graph.Plan, error) { - // set provisioning cluster condition - c.handleProvisionStartedCondition() var err error defer func() { - if err != nil { - _ = c.updateClusterStatusWithCondition(newFailedApplyResourcesCondition(err.Error())) + // set apply resource condition + // if cluster is being deleted, no need to set apply resource condition + if isClusterDeleting(*c.transCtx.Cluster) { + return + } + preCheckCondition := meta.FindStatusCondition(c.transCtx.Cluster.Status.Conditions, appsv1alpha1.ConditionTypeProvisioningStarted) + if preCheckCondition == nil { + // this should not happen + return + } + // if pre-check failed, this is a fast return, no need to set apply resource condition + if preCheckCondition.Status != metav1.ConditionTrue { + return } + setApplyResourceCondition(&c.transCtx.Cluster.Status.Conditions, c.transCtx.Cluster.Generation, err) }() - var cr *clusterRefResources - cr, err = c.getClusterRefResources() - if err != nil { - return nil, err - } - var roClient types2.ReadonlyClient = delegateClient{Client: c.cli} - - // TODO: remove all cli & ctx fields from transformers, keep them in pure-dag-manipulation form - // build transformer chain - chain := &graph.TransformerChain{ - // init dag, that is put cluster vertex into dag - &initTransformer{cluster: c.cluster, originCluster: &c.originCluster}, - // fill class related info - &fillClass{cc: *cr, cli: c.cli, ctx: c.ctx}, - // fix cd&cv labels of cluster - &fixClusterLabelsTransformer{}, - // cluster to K8s objects and put them into dag - &clusterTransformer{cc: *cr, cli: c.cli, ctx: c.ctx}, - // tls certs secret - &tlsCertsTransformer{cr: *cr, cli: roClient, ctx: c.ctx}, - // transform backupPolicy tpl to backuppolicy.dataprotection.kubeblocks.io - &backupPolicyTPLTransformer{cr: *cr, cli: c.cli, ctx: c.ctx}, - // add our finalizer to all objects - &ownershipTransformer{finalizer: dbClusterFinalizerName}, - // make all workload objects depending on credential secret - &credentialTransformer{}, - // make config configmap immutable - &configTransformer{}, - // read old snapshot from cache, and generate diff plan - &objectActionTransformer{cli: roClient, ctx: c.ctx}, - // handle TerminationPolicyType=DoNotTerminate - &doNotTerminateTransformer{}, - // horizontal scaling - &stsHorizontalScalingTransformer{cr: *cr, cli: roClient, ctx: c.ctx}, - // stateful set pvc Update - &stsPVCTransformer{cli: c.cli, ctx: c.ctx}, - // finally, update cluster status - newClusterStatusTransformer(c.ctx, c.cli, c.recorder, *cr), - } - // new a DAG and apply chain on it, after that we should get the final Plan dag := graph.NewDAG() - if err = chain.ApplyTo(dag); err != nil { - return nil, err - } + err = c.transformers.ApplyTo(c.transCtx, dag) + // log for debug + c.transCtx.Logger.Info(fmt.Sprintf("DAG: %s", dag)) - c.ctx.Log.Info(fmt.Sprintf("DAG: %s", dag)) // we got the execution plan plan := &clusterPlan{ - ctx: c.ctx, - cli: c.cli, - recorder: c.recorder, dag: dag, walkFunc: c.defaultWalkFunc, - cluster: c.cluster, - } - return plan, nil -} - -func (c *clusterPlanBuilder) updateClusterStatusWithCondition(condition metav1.Condition) error { - oldCondition := meta.FindStatusCondition(c.cluster.Status.Conditions, condition.Type) - meta.SetStatusCondition(&c.cluster.Status.Conditions, condition) - if !reflect.DeepEqual(c.cluster.Status, c.originCluster.Status) { - if err := c.cli.Status().Patch(c.ctx.Ctx, c.cluster, client.MergeFrom(c.originCluster.DeepCopy())); err != nil { - return err - } - } - // Normal events are only sent once. - if !conditionIsChanged(oldCondition, condition) && condition.Status == metav1.ConditionTrue { - return nil - } - eventType := corev1.EventTypeWarning - if condition.Status == metav1.ConditionTrue { - eventType = corev1.EventTypeNormal + cli: c.cli, + transCtx: c.transCtx, } - c.recorder.Event(c.cluster, eventType, condition.Reason, condition.Message) - return nil + return plan, err } -// NewClusterPlanBuilder returns a clusterPlanBuilder powered PlanBuilder -// TODO: change ctx to context.Context -func NewClusterPlanBuilder(ctx intctrlutil.RequestCtx, cli client.Client, req ctrl.Request, recorder record.EventRecorder) graph.PlanBuilder { - return &clusterPlanBuilder{ - ctx: ctx, - cli: cli, - req: req, - recorder: recorder, - } -} +// Plan implementation func (p *clusterPlan) Execute() error { err := p.dag.WalkReverseTopoOrder(p.walkFunc) if err != nil { - if hErr := p.handleDAGWalkError(err); hErr != nil { + if hErr := p.handlePlanExecutionError(err); hErr != nil { return hErr } } return err } -func (p *clusterPlan) handleDAGWalkError(err error) error { +func (p *clusterPlan) handlePlanExecutionError(err error) error { condition := newFailedApplyResourcesCondition(err.Error()) - meta.SetStatusCondition(&p.cluster.Status.Conditions, condition) - p.recorder.Event(p.cluster, corev1.EventTypeWarning, condition.Reason, condition.Message) - rootVertex, _ := findRootVertex(p.dag) - if rootVertex == nil { - return nil - } - originCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) - if originCluster == nil || reflect.DeepEqual(originCluster.Status, p.cluster.Status) { - return nil - } - return p.cli.Status().Patch(p.ctx.Ctx, p.cluster, client.MergeFrom(originCluster.DeepCopy())) + meta.SetStatusCondition(&p.transCtx.Cluster.Status.Conditions, condition) + p.transCtx.EventRecorder.Event(p.transCtx.Cluster, corev1.EventTypeWarning, condition.Reason, condition.Message) + return p.cli.Status().Patch(p.transCtx.Context, p.transCtx.Cluster, client.MergeFrom(p.transCtx.OrigCluster.DeepCopy())) } -func (c *clusterPlanBuilder) getClusterRefResources() (*clusterRefResources, error) { - cluster := c.cluster - cd := &appsv1alpha1.ClusterDefinition{} - if err := c.cli.Get(c.ctx.Ctx, types.NamespacedName{ - Name: cluster.Spec.ClusterDefRef, - }, cd); err != nil { - return nil, err - } - cv := &appsv1alpha1.ClusterVersion{} - if len(cluster.Spec.ClusterVersionRef) > 0 { - if err := c.cli.Get(c.ctx.Ctx, types.NamespacedName{ - Name: cluster.Spec.ClusterVersionRef, - }, cv); err != nil { - return nil, err - } - } +// Do the real works - cc := &clusterRefResources{ - cd: *cd, - cv: *cv, +// NewClusterPlanBuilder returns a clusterPlanBuilder powered PlanBuilder +func NewClusterPlanBuilder(ctx intctrlutil.RequestCtx, cli client.Client, req ctrl.Request) graph.PlanBuilder { + return &clusterPlanBuilder{ + req: req, + cli: cli, + transCtx: &ClusterTransformContext{ + Context: ctx.Ctx, + Client: cli, + EventRecorder: ctx.Recorder, + Logger: ctx.Log, + }, } - return cc, nil } +// TODO: retry strategy on error func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { node, ok := vertex.(*lifecycleVertex) if !ok { @@ -330,25 +201,13 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { if node.action == nil { return errors.New("node action can't be nil") } - updateComponentPhaseIfNeeded := func(orig, curr client.Object) { - switch orig.(type) { - case *appsv1.StatefulSet, *appsv1.Deployment: - componentName := orig.GetLabels()[constant.KBAppComponentLabelKey] - origSpec := reflect.ValueOf(orig).Elem().FieldByName("Spec").Interface() - newSpec := reflect.ValueOf(curr).Elem().FieldByName("Spec").Interface() - if !reflect.DeepEqual(origSpec, newSpec) { - // sync component phase - updateComponentPhaseWithOperation(c.cluster, componentName) - } - } - } // cluster object has more business to do, handle them here if _, ok := node.obj.(*appsv1alpha1.Cluster); ok { cluster := node.obj.(*appsv1alpha1.Cluster).DeepCopy() origCluster := node.oriObj.(*appsv1alpha1.Cluster) switch *node.action { // cluster.meta and cluster.spec might change - case CREATE, UPDATE, STATUS: + case STATUS: if !reflect.DeepEqual(cluster.ObjectMeta, origCluster.ObjectMeta) || !reflect.DeepEqual(cluster.Spec, origCluster.Spec) { // TODO: we should Update instead of Patch cluster object, @@ -361,23 +220,20 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { // return err // } patch := client.MergeFrom(origCluster.DeepCopy()) - if err := c.cli.Patch(c.ctx.Ctx, cluster, patch); err != nil { - c.ctx.Log.Error(err, fmt.Sprintf("patch %T error, orig: %v, curr: %v", origCluster, origCluster, cluster)) + if err := c.cli.Patch(c.transCtx.Context, cluster, patch); err != nil { + // log for debug + // TODO:(free6om) make error message smaller when refactor done. + c.transCtx.Logger.Error(err, fmt.Sprintf("patch %T error, orig: %v, curr: %v", origCluster, origCluster, cluster)) return err } } - case DELETE: - if err := c.handleClusterDeletion(cluster); err != nil { - return err - } - if cluster.Spec.TerminationPolicy == appsv1alpha1.DoNotTerminate { - return nil - } + case CREATE, UPDATE: + return fmt.Errorf("cluster can't be created or updated: %s", cluster.Name) } } switch *node.action { case CREATE: - err := c.cli.Create(c.ctx.Ctx, node.obj) + err := c.cli.Create(c.transCtx.Context, node.obj) if err != nil && !apierrors.IsAlreadyExists(err) { return err } @@ -389,47 +245,36 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { if err != nil { return err } - err = c.cli.Update(c.ctx.Ctx, o) + err = c.cli.Update(c.transCtx.Context, o) if err != nil && !apierrors.IsNotFound(err) { - c.ctx.Log.Error(err, fmt.Sprintf("update %T error, orig: %v, curr: %v", o, node.oriObj, o)) + c.transCtx.Logger.Error(err, fmt.Sprintf("update %T error: %s", o, node.oriObj.GetName())) return err } - // TODO: find a better comparison way that knows whether fields are updated before calling the Update func - updateComponentPhaseIfNeeded(node.oriObj, o) case DELETE: if controllerutil.RemoveFinalizer(node.obj, dbClusterFinalizerName) { - err := c.cli.Update(c.ctx.Ctx, node.obj) + err := c.cli.Update(c.transCtx.Context, node.obj) if err != nil && !apierrors.IsNotFound(err) { - c.ctx.Log.Error(err, fmt.Sprintf("delete %T error, orig: %v, curr: %v", node.obj, node.oriObj, node.obj)) + c.transCtx.Logger.Error(err, fmt.Sprintf("delete %T error: %s", node.obj, node.obj.GetName())) return err } } - if node.isOrphan { - err := c.cli.Delete(c.ctx.Ctx, node.obj) + // delete secondary objects + if _, ok := node.obj.(*appsv1alpha1.Cluster); !ok { + err := c.cli.Delete(c.transCtx.Context, node.obj) if err != nil && !apierrors.IsNotFound(err) { return err } } - // TODO: delete backup objects created in scale-out - // TODO: should manage backup objects in a better way - if isTypeOf[*snapshotv1.VolumeSnapshot](node.obj) || - isTypeOf[*dataprotectionv1alpha1.BackupPolicy](node.obj) || - isTypeOf[*dataprotectionv1alpha1.Backup](node.obj) { - _ = c.cli.Delete(c.ctx.Ctx, node.obj) - } - case STATUS: - if node.immutable { - return nil - } patch := client.MergeFrom(node.oriObj) - if err := c.cli.Status().Patch(c.ctx.Ctx, node.obj, patch); err != nil { + if err := c.cli.Status().Patch(c.transCtx.Context, node.obj, patch); err != nil { return err } - for _, postHandle := range node.postHandleAfterStatusPatch { - if err := postHandle(); err != nil { - return err - } + // handle condition and phase changing triggered events + if newCluster, ok := node.obj.(*appsv1alpha1.Cluster); ok { + oldCluster, _ := node.oriObj.(*appsv1alpha1.Cluster) + c.emitConditionUpdatingEvent(oldCluster.Status.Conditions, newCluster.Status.Conditions) + c.emitPhaseUpdatingEvent(oldCluster.Status.Phase, newCluster.Status.Phase) } } return nil @@ -440,7 +285,7 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec stsObj := origObj.DeepCopy() componentName := stsObj.Labels[constant.KBAppComponentLabelKey] if *stsObj.Spec.Replicas != *stsProto.Spec.Replicas { - c.recorder.Eventf(c.cluster, + c.transCtx.EventRecorder.Eventf(c.transCtx.Cluster, corev1.EventTypeNormal, "HorizontalScale", "Start horizontal scale component %s from %d to %d", @@ -498,115 +343,42 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec return node.obj, nil } -func (c *clusterPlanBuilder) handleClusterDeletion(cluster *appsv1alpha1.Cluster) error { - switch cluster.Spec.TerminationPolicy { - case appsv1alpha1.DoNotTerminate: - c.recorder.Eventf(cluster, corev1.EventTypeWarning, "DoNotTerminate", "spec.terminationPolicy %s is preventing deletion.", cluster.Spec.TerminationPolicy) - return nil - case appsv1alpha1.Delete, appsv1alpha1.WipeOut: - if err := c.deletePVCs(cluster); err != nil && !apierrors.IsNotFound(err) { - return err - } - if err := c.deleteConfigMaps(cluster); err != nil && !apierrors.IsNotFound(err) { - return err +func (c *clusterPlanBuilder) emitConditionUpdatingEvent(oldConditions, newConditions []metav1.Condition) { + for _, newCondition := range newConditions { + oldCondition := meta.FindStatusCondition(oldConditions, newCondition.Type) + // filtered in cluster creation + if oldCondition == nil && newCondition.Status == metav1.ConditionFalse { + return } - // The backup policy must be cleaned up when the cluster is deleted. - // Automatic backup scheduling needs to be stopped at this point. - if err := c.deleteBackupPolicies(cluster); err != nil && !apierrors.IsNotFound(err) { - return err - } - if cluster.Spec.TerminationPolicy == appsv1alpha1.WipeOut { - // TODO check whether delete backups together with cluster is allowed - // wipe out all backups - if err := c.deleteBackups(cluster); err != nil && !apierrors.IsNotFound(err) { - return err + if !reflect.DeepEqual(oldCondition, &newCondition) { + eType := corev1.EventTypeNormal + if newCondition.Status == metav1.ConditionFalse { + eType = corev1.EventTypeWarning } - } - if err := c.deleteJobs(cluster); err != nil && !apierrors.IsNotFound(err) { - return err + c.transCtx.EventRecorder.Event(c.transCtx.Cluster, eType, newCondition.Reason, newCondition.Message) } } - return nil } -func (c *clusterPlanBuilder) deletePVCs(cluster *appsv1alpha1.Cluster) error { - // it's possible at time of external resource deletion, cluster definition has already been deleted. - ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.GetName(), - } - inNS := client.InNamespace(cluster.Namespace) - - pvcList := &corev1.PersistentVolumeClaimList{} - if err := c.cli.List(c.ctx.Ctx, pvcList, inNS, ml); err != nil { - return err - } - for _, pvc := range pvcList.Items { - if err := c.cli.Delete(c.ctx.Ctx, &pvc); err != nil { - return err - } - } - return nil -} - -func (c *clusterPlanBuilder) deleteConfigMaps(cluster *appsv1alpha1.Cluster) error { - return DeleteConfigMaps(c.ctx.Ctx, c.cli, cluster) -} - -func (c *clusterPlanBuilder) deleteBackupPolicies(cluster *appsv1alpha1.Cluster) error { - inNS := client.InNamespace(cluster.Namespace) - ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.GetName(), - } - // clean backupPolicies - return c.cli.DeleteAllOf(c.ctx.Ctx, &dataprotectionv1alpha1.BackupPolicy{}, inNS, ml) -} - -func (c *clusterPlanBuilder) deleteBackups(cluster *appsv1alpha1.Cluster) error { - inNS := client.InNamespace(cluster.Namespace) - ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.GetName(), - } - // clean backups - backups := &dataprotectionv1alpha1.BackupList{} - if err := c.cli.List(c.ctx.Ctx, backups, inNS, ml); err != nil { - return err - } - for _, backup := range backups.Items { - // check backup delete protection label - deleteProtection, exists := backup.GetLabels()[constant.BackupProtectionLabelKey] - // not found backup-protection or value is Delete, delete it. - if !exists || deleteProtection == constant.BackupDelete { - if err := c.cli.Delete(c.ctx.Ctx, &backup); err != nil { - return err - } - } +func (c *clusterPlanBuilder) emitPhaseUpdatingEvent(oldPhase, newPhase appsv1alpha1.ClusterPhase) { + if oldPhase == newPhase { + return } - return nil -} -func (c *clusterPlanBuilder) deleteJobs(cluster *appsv1alpha1.Cluster) error { - inNS := client.InNamespace(cluster.Namespace) - ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.GetName(), + cluster := c.transCtx.Cluster + eType := corev1.EventTypeNormal + message := "" + switch newPhase { + case appsv1alpha1.RunningClusterPhase: + message = fmt.Sprintf("Cluster: %s is ready, current phase is %s", cluster.Name, newPhase) + case appsv1alpha1.StoppedClusterPhase: + message = fmt.Sprintf("Cluster: %s stopped successfully.", cluster.Name) + case appsv1alpha1.FailedClusterPhase, appsv1alpha1.AbnormalClusterPhase: + message = fmt.Sprintf("Cluster: %s is %s, check according to the components message", cluster.Name, newPhase) + eType = corev1.EventTypeWarning } - // clean jobs - jobList := batchv1.JobList{} - if err := c.cli.List(c.ctx.Ctx, &jobList, inNS, ml); err != nil { - return err - } - for _, job := range jobList.Items { - if err := intctrlutil.BackgroundDeleteObject(c.cli, c.ctx.Ctx, &job); err != nil { - return err - } - } - return nil -} - -func DeleteConfigMaps(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster) error { - inNS := client.InNamespace(cluster.Namespace) - ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.GetName(), - constant.AppManagedByLabelKey: constant.AppName, + if len(message) > 0 { + c.transCtx.EventRecorder.Event(cluster, eType, string(newPhase), message) + _ = opsutil.MarkRunningOpsRequestAnnotation(c.transCtx.Context, c.cli, cluster) } - return cli.DeleteAllOf(ctx, &corev1.ConfigMap{}, inNS, ml) } diff --git a/internal/controller/lifecycle/cluster_plan_utils.go b/internal/controller/lifecycle/cluster_plan_utils.go index 13ea656e1..283bbde41 100644 --- a/internal/controller/lifecycle/cluster_plan_utils.go +++ b/internal/controller/lifecycle/cluster_plan_utils.go @@ -20,8 +20,6 @@ import ( "strings" "golang.org/x/exp/maps" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" ) // mergeAnnotations keeps the original annotations. @@ -56,15 +54,3 @@ func mergeServiceAnnotations(originalAnnotations, targetAnnotations map[string]s maps.Copy(tmpAnnotations, targetAnnotations) return tmpAnnotations } - -// updateComponentPhaseWithOperation if workload of component changes, should update the component phase. -func updateComponentPhaseWithOperation(cluster *appsv1alpha1.Cluster, componentName string) { - componentPhase := appsv1alpha1.SpecReconcilingClusterCompPhase - if cluster.Status.Phase == appsv1alpha1.CreatingClusterPhase { - componentPhase = appsv1alpha1.CreatingClusterCompPhase - } - compStatus := cluster.Status.Components[componentName] - // synchronous component phase is consistent with cluster phase - compStatus.Phase = componentPhase - cluster.Status.SetComponentStatus(componentName, compStatus) -} diff --git a/internal/controller/lifecycle/cluster_status_conditions.go b/internal/controller/lifecycle/cluster_status_conditions.go index bb24b8d61..31d614e5b 100644 --- a/internal/controller/lifecycle/cluster_status_conditions.go +++ b/internal/controller/lifecycle/cluster_status_conditions.go @@ -23,6 +23,7 @@ import ( "golang.org/x/exp/maps" "golang.org/x/exp/slices" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -49,6 +50,14 @@ func conditionIsChanged(oldCondition *metav1.Condition, newCondition metav1.Cond return !reflect.DeepEqual(oldCondition, &newCondition) } +func setProvisioningStartedCondition(conditions *[]metav1.Condition, clusterName string, clusterGeneration int64, err error) { + condition := newProvisioningStartedCondition(clusterName, clusterGeneration) + if err != nil { + condition = newFailedProvisioningStartedCondition(err.Error(), ReasonPreCheckFailed) + } + meta.SetStatusCondition(conditions, condition) +} + // newProvisioningStartedCondition creates the provisioning started condition in cluster conditions. func newProvisioningStartedCondition(clusterName string, clusterGeneration int64) metav1.Condition { return metav1.Condition{ @@ -70,6 +79,14 @@ func newFailedProvisioningStartedCondition(message, reason string) metav1.Condit } } +func setApplyResourceCondition(conditions *[]metav1.Condition, clusterGeneration int64, err error) { + condition := newApplyResourcesCondition(clusterGeneration) + if err != nil { + condition = newFailedApplyResourcesCondition(err.Error()) + } + meta.SetStatusCondition(conditions, condition) +} + // newApplyResourcesCondition creates a condition when applied resources succeed. func newApplyResourcesCondition(clusterGeneration int64) metav1.Condition { return metav1.Condition{ diff --git a/internal/controller/lifecycle/transform_types.go b/internal/controller/lifecycle/transform_types.go index 695041b4a..2fc14e600 100644 --- a/internal/controller/lifecycle/transform_types.go +++ b/internal/controller/lifecycle/transform_types.go @@ -70,11 +70,6 @@ type gvkName struct { ns, name string } -type clusterRefResources struct { - cd appsv1alpha1.ClusterDefinition - cv appsv1alpha1.ClusterVersion -} - // lifecycleVertex describes expected object spec and how to reach it // obj always represents the expected part: new object in Create/Update action and old object in Delete action // oriObj is set in Update action @@ -89,8 +84,6 @@ type lifecycleVertex struct { immutable bool isOrphan bool action *Action - // postHandleAfterStatusPatch is called after the object status has changed - postHandleAfterStatusPatch []func() error } func (v lifecycleVertex) String() string { diff --git a/internal/controller/lifecycle/transform_utils.go b/internal/controller/lifecycle/transform_utils.go index 456b267ca..839ff2183 100644 --- a/internal/controller/lifecycle/transform_utils.go +++ b/internal/controller/lifecycle/transform_utils.go @@ -22,6 +22,9 @@ import ( "reflect" "time" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -138,35 +141,28 @@ func isClusterUpdating(cluster appsv1alpha1.Cluster) bool { func isClusterStatusUpdating(cluster appsv1alpha1.Cluster) bool { return !isClusterDeleting(cluster) && !isClusterUpdating(cluster) - // return cluster.Status.ObservedGeneration == cluster.Generation && - // slices.Contains(appsv1alpha1.GetClusterTerminalPhases(), cluster.Status.Phase) } -func getBackupObjects(reqCtx intctrlutil.RequestCtx, +func getBackupObjects(ctx context.Context, cli types2.ReadonlyClient, namespace string, backupName string) (*dataprotectionv1alpha1.Backup, *dataprotectionv1alpha1.BackupTool, error) { // get backup backup := &dataprotectionv1alpha1.Backup{} - if err := cli.Get(reqCtx.Ctx, types.NamespacedName{Name: backupName, Namespace: namespace}, backup); err != nil { + if err := cli.Get(ctx, types.NamespacedName{Name: backupName, Namespace: namespace}, backup); err != nil { return nil, nil, err } // get backup tool backupTool := &dataprotectionv1alpha1.BackupTool{} if backup.Spec.BackupType != dataprotectionv1alpha1.BackupTypeSnapshot { - if err := cli.Get(reqCtx.Ctx, types.NamespacedName{Name: backup.Status.BackupToolName}, backupTool); err != nil { + if err := cli.Get(ctx, types.NamespacedName{Name: backup.Status.BackupToolName}, backupTool); err != nil { return nil, nil, err } } return backup, backupTool, nil } -func isTypeOf[T interface{}](obj client.Object) bool { - _, ok := obj.(T) - return ok -} - // getBackupPolicyFromTemplate gets backup policy from template policy template. func getBackupPolicyFromTemplate(reqCtx intctrlutil.RequestCtx, cli types2.ReadonlyClient, @@ -190,14 +186,26 @@ func getBackupPolicyFromTemplate(reqCtx intctrlutil.RequestCtx, return nil, nil } +func ownKinds() []client.ObjectList { + return []client.ObjectList{ + &appsv1.StatefulSetList{}, + &appsv1.DeploymentList{}, + &corev1.ServiceList{}, + &corev1.SecretList{}, + &corev1.ConfigMapList{}, + &policyv1.PodDisruptionBudgetList{}, + &dataprotectionv1alpha1.BackupPolicyList{}, + } +} + // read all objects owned by our cluster -func readCacheSnapshot(ctx context.Context, cli types2.ReadonlyClient, cluster appsv1alpha1.Cluster, kinds ...client.ObjectList) (clusterSnapshot, error) { +func readCacheSnapshot(transCtx *ClusterTransformContext, cluster appsv1alpha1.Cluster, kinds ...client.ObjectList) (clusterSnapshot, error) { // list what kinds of object cluster owns snapshot := make(clusterSnapshot) ml := client.MatchingLabels{constant.AppInstanceLabelKey: cluster.GetName()} inNS := client.InNamespace(cluster.Namespace) for _, list := range kinds { - if err := cli.List(ctx, list, inNS, ml); err != nil { + if err := transCtx.Client.List(transCtx.Context, list, inNS, ml); err != nil { return nil, err } // reflect get list.Items @@ -207,7 +215,9 @@ func readCacheSnapshot(ctx context.Context, cli types2.ReadonlyClient, cluster a // get the underlying object object := items.Index(i).Addr().Interface().(client.Object) // put to snapshot if owned by our cluster - if isOwnerOf(&cluster, object, scheme) { + // pvcs created by sts don't have cluster in ownerReferences + _, isPVC := object.(*corev1.PersistentVolumeClaim) + if isPVC || isOwnerOf(&cluster, object, scheme) { name, err := getGVKName(object, scheme) if err != nil { return nil, err diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index 73c9ad41f..29693d002 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -29,22 +29,18 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" - types2 "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/graph" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -// backupPolicyTPLTransformer transforms the backup policy template to the backup policy. -type backupPolicyTPLTransformer struct { - cr clusterRefResources - cli types2.ReadonlyClient - ctx intctrlutil.RequestCtx -} +// BackupPolicyTPLTransformer transforms the backup policy template to the backup policy. +type BackupPolicyTPLTransformer struct{} -func (r *backupPolicyTPLTransformer) Transform(dag *graph.DAG) error { - clusterDefName := r.cr.cd.Name +func (r *BackupPolicyTPLTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + clusterDefName := transCtx.ClusterDef.Name backupPolicyTPLs := &appsv1alpha1.BackupPolicyTemplateList{} - if err := r.cli.List(r.ctx.Ctx, backupPolicyTPLs, client.MatchingLabels{constant.ClusterDefLabelKey: clusterDefName}); err != nil { + if err := transCtx.Client.List(transCtx.Context, backupPolicyTPLs, client.MatchingLabels{constant.ClusterDefLabelKey: clusterDefName}); err != nil { return err } if len(backupPolicyTPLs.Items) == 0 { @@ -54,15 +50,15 @@ func (r *backupPolicyTPLTransformer) Transform(dag *graph.DAG) error { if err != nil { return err } - origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) + origCluster := transCtx.OrigCluster for _, tpl := range backupPolicyTPLs.Items { for _, v := range tpl.Spec.BackupPolicies { - compDef := r.cr.cd.GetComponentDefByName(v.ComponentDefRef) + compDef := transCtx.ClusterDef.GetComponentDefByName(v.ComponentDefRef) if compDef == nil { return intctrlutil.NewNotFound("componentDef %s not found in ClusterDefinition: %s ", v.ComponentDefRef, clusterDefName) } // build the backup policy from the template. - backupPolicy := r.transformBackupPolicy(v, origCluster, compDef.WorkloadType, tpl.Name) + backupPolicy := r.transformBackupPolicy(transCtx, v, origCluster, compDef.WorkloadType, tpl.Name) if backupPolicy == nil { continue } @@ -75,13 +71,14 @@ func (r *backupPolicyTPLTransformer) Transform(dag *graph.DAG) error { } // transformBackupPolicy transform backup policy template to backup policy. -func (r *backupPolicyTPLTransformer) transformBackupPolicy(policyTPL appsv1alpha1.BackupPolicy, +func (r *BackupPolicyTPLTransformer) transformBackupPolicy(transCtx *ClusterTransformContext, + policyTPL appsv1alpha1.BackupPolicy, cluster *appsv1alpha1.Cluster, workloadType appsv1alpha1.WorkloadType, tplName string) *dataprotectionv1alpha1.BackupPolicy { backupPolicyName := DeriveBackupPolicyName(cluster.Name, policyTPL.ComponentDefRef) backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} - if err := r.cli.Get(r.ctx.Ctx, client.ObjectKey{Namespace: cluster.Namespace, Name: backupPolicyName}, backupPolicy); err != nil && !apierrors.IsNotFound(err) { + if err := transCtx.Client.Get(transCtx.Context, client.ObjectKey{Namespace: cluster.Namespace, Name: backupPolicyName}, backupPolicy); err != nil && !apierrors.IsNotFound(err) { return nil } if len(backupPolicy.Name) == 0 { @@ -94,7 +91,7 @@ func (r *backupPolicyTPLTransformer) transformBackupPolicy(policyTPL appsv1alpha } // syncBackupPolicy syncs labels and annotations of the backup policy with the cluster changes. -func (r *backupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotectionv1alpha1.BackupPolicy, +func (r *BackupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotectionv1alpha1.BackupPolicy, cluster *appsv1alpha1.Cluster, policyTPL appsv1alpha1.BackupPolicy, workloadType appsv1alpha1.WorkloadType, @@ -154,7 +151,7 @@ func (r *backupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti } // buildBackupPolicy builds a new backup policy from the backup policy template. -func (r *backupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.BackupPolicy, +func (r *BackupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.BackupPolicy, cluster *appsv1alpha1.Cluster, workloadType appsv1alpha1.WorkloadType, tplName string) *dataprotectionv1alpha1.BackupPolicy { @@ -189,7 +186,7 @@ func (r *backupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba } // getFirstComponent returns the first component name of the componentDefRef. -func (r *backupPolicyTPLTransformer) getFirstComponent(cluster *appsv1alpha1.Cluster, +func (r *BackupPolicyTPLTransformer) getFirstComponent(cluster *appsv1alpha1.Cluster, componentDefRef string) *appsv1alpha1.ClusterComponentSpec { for _, v := range cluster.Spec.ComponentSpecs { if v.ComponentDefRef == componentDefRef { @@ -200,7 +197,7 @@ func (r *backupPolicyTPLTransformer) getFirstComponent(cluster *appsv1alpha1.Clu } // convertSchedulePolicy converts the schedulePolicy from backupPolicyTemplate. -func (r *backupPolicyTPLTransformer) convertSchedulePolicy(sp *appsv1alpha1.SchedulePolicy) *dataprotectionv1alpha1.SchedulePolicy { +func (r *BackupPolicyTPLTransformer) convertSchedulePolicy(sp *appsv1alpha1.SchedulePolicy) *dataprotectionv1alpha1.SchedulePolicy { if sp == nil { return nil } @@ -211,7 +208,7 @@ func (r *backupPolicyTPLTransformer) convertSchedulePolicy(sp *appsv1alpha1.Sche } // convertBaseBackupSchedulePolicy converts the baseBackupSchedulePolicy from backupPolicyTemplate. -func (r *backupPolicyTPLTransformer) convertBaseBackupSchedulePolicy(sp *appsv1alpha1.BaseBackupSchedulePolicy) *dataprotectionv1alpha1.BaseBackupSchedulePolicy { +func (r *BackupPolicyTPLTransformer) convertBaseBackupSchedulePolicy(sp *appsv1alpha1.BaseBackupSchedulePolicy) *dataprotectionv1alpha1.BaseBackupSchedulePolicy { if sp == nil { return nil } @@ -223,7 +220,7 @@ func (r *backupPolicyTPLTransformer) convertBaseBackupSchedulePolicy(sp *appsv1a } // convertBasePolicy converts the basePolicy from backupPolicyTemplate. -func (r *backupPolicyTPLTransformer) convertBasePolicy(bp appsv1alpha1.BasePolicy, +func (r *BackupPolicyTPLTransformer) convertBasePolicy(bp appsv1alpha1.BasePolicy, clusterName string, component appsv1alpha1.ClusterComponentSpec, workloadType appsv1alpha1.WorkloadType) dataprotectionv1alpha1.BasePolicy { @@ -281,7 +278,7 @@ func (r *backupPolicyTPLTransformer) convertBasePolicy(bp appsv1alpha1.BasePolic } // convertBaseBackupSchedulePolicy converts the snapshotPolicy from backupPolicyTemplate. -func (r *backupPolicyTPLTransformer) convertSnapshotPolicy(sp *appsv1alpha1.SnapshotPolicy, +func (r *BackupPolicyTPLTransformer) convertSnapshotPolicy(sp *appsv1alpha1.SnapshotPolicy, clusterName string, component appsv1alpha1.ClusterComponentSpec, workloadType appsv1alpha1.WorkloadType) *dataprotectionv1alpha1.SnapshotPolicy { @@ -303,7 +300,7 @@ func (r *backupPolicyTPLTransformer) convertSnapshotPolicy(sp *appsv1alpha1.Snap } // convertBaseBackupSchedulePolicy converts the commonPolicy from backupPolicyTemplate. -func (r *backupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.CommonBackupPolicy, +func (r *BackupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.CommonBackupPolicy, clusterName string, component appsv1alpha1.ClusterComponentSpec, workloadType appsv1alpha1.WorkloadType) *dataprotectionv1alpha1.CommonBackupPolicy { @@ -345,3 +342,5 @@ func (r *backupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.Common func DeriveBackupPolicyName(clusterName, componentDef string) string { return fmt.Sprintf("%s-%s-backup-policy", clusterName, componentDef) } + +var _ graph.Transformer = &BackupPolicyTPLTransformer{} diff --git a/internal/controller/lifecycle/transformer_cluster.go b/internal/controller/lifecycle/transformer_cluster.go index 1bb550080..0b4f8500f 100644 --- a/internal/controller/lifecycle/transformer_cluster.go +++ b/internal/controller/lifecycle/transformer_cluster.go @@ -32,24 +32,17 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -// clusterTransformer transforms a Cluster to a K8s objects DAG +// ClusterTransformer builds a Cluster into K8s objects and put them into a DAG // TODO: remove cli and ctx, we should read all objects needed, and then do pure objects computation // TODO: only replication set left -type clusterTransformer struct { - cc clusterRefResources - cli client.Client - ctx intctrlutil.RequestCtx +type ClusterTransformer struct { + client.Client } -func (c *clusterTransformer) Transform(dag *graph.DAG) error { - rootVertex, err := findRootVertex(dag) - if err != nil { - return err - } - origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) - cluster, _ := rootVertex.obj.(*appsv1alpha1.Cluster) - - // return fast when cluster is deleting +func (c *ClusterTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + origCluster := transCtx.OrigCluster + cluster := transCtx.Cluster if isClusterDeleting(*origCluster) { return nil } @@ -59,8 +52,8 @@ func (c *clusterTransformer) Transform(dag *graph.DAG) error { resourcesQueue := make([]client.Object, 0, 3) task := intctrltypes.ReconcileTask{ Cluster: cluster, - ClusterDefinition: &c.cc.cd, - ClusterVersion: &c.cc.cv, + ClusterDefinition: transCtx.ClusterDef, + ClusterVersion: transCtx.ClusterVer, Resources: &resourcesQueue, } @@ -70,9 +63,14 @@ func (c *clusterTransformer) Transform(dag *graph.DAG) error { } clusterCompSpecMap := cluster.Spec.GetDefNameMappingComponents() - clusterCompVerMap := c.cc.cv.Spec.GetDefNameMappingComponents() + clusterCompVerMap := transCtx.ClusterVer.Spec.GetDefNameMappingComponents() process1stComp := true + reqCtx := intctrlutil.RequestCtx{ + Ctx: transCtx.Context, + Log: transCtx.Logger, + Recorder: transCtx.EventRecorder, + } // TODO: should move credential secrets creation from system_account_controller & here into credential_transformer, // TODO: as those secrets are owned by the cluster prepareComp := func(synthesizedComp *component.SynthesizedComponent) error { @@ -88,7 +86,7 @@ func (c *clusterTransformer) Transform(dag *graph.DAG) error { // build info that needs to be restored from backup backupSourceName := clusterBackupResourceMap[synthesizedComp.Name] if len(backupSourceName) > 0 { - backup, backupTool, err := getBackupObjects(c.ctx, c.cli, cluster.Namespace, backupSourceName) + backup, backupTool, err := getBackupObjects(transCtx.Context, c.Client, cluster.Namespace, backupSourceName) if err != nil { return err } @@ -96,19 +94,19 @@ func (c *clusterTransformer) Transform(dag *graph.DAG) error { return err } } - if err = plan.DoPITRPrepare(c.ctx.Ctx, c.cli, cluster, synthesizedComp); err != nil { + if err = plan.DoPITRPrepare(transCtx.Context, c.Client, cluster, synthesizedComp); err != nil { return err } - return plan.PrepareComponentResources(c.ctx, c.cli, &iParams) + return plan.PrepareComponentResources(reqCtx, c.Client, &iParams) } - for _, compDef := range c.cc.cd.Spec.ComponentDefs { + for _, compDef := range transCtx.ClusterDef.Spec.ComponentDefs { compDefName := compDef.Name compVer := clusterCompVerMap[compDefName] compSpecs := clusterCompSpecMap[compDefName] for _, compSpec := range compSpecs { - if err := prepareComp(component.BuildComponent(c.ctx, *cluster, c.cc.cd, compDef, compSpec, compVer)); err != nil { + if err := prepareComp(component.BuildComponent(reqCtx, *cluster, *transCtx.ClusterDef, compDef, compSpec, compVer)); err != nil { return err } } @@ -116,12 +114,16 @@ func (c *clusterTransformer) Transform(dag *graph.DAG) error { // replication set will create duplicate env configmap and headless service // dedup them + root, err := findRootVertex(dag) + if err != nil { + return err + } objects := deDupResources(*task.Resources) // now task.Resources to DAG vertices for _, object := range objects { vertex := &lifecycleVertex{obj: object} dag.AddVertex(vertex) - dag.Connect(rootVertex, vertex) + dag.Connect(root, vertex) } return nil } @@ -163,3 +165,5 @@ func getClusterBackupSourceMap(cluster *appsv1alpha1.Cluster) (map[string]string err := json.Unmarshal([]byte(compBackupMapString), &compBackupMap) return compBackupMap, err } + +var _ graph.Transformer = &ClusterTransformer{} diff --git a/internal/controller/lifecycle/transformer_cluster_deletion.go b/internal/controller/lifecycle/transformer_cluster_deletion.go new file mode 100644 index 000000000..acc050ab5 --- /dev/null +++ b/internal/controller/lifecycle/transformer_cluster_deletion.go @@ -0,0 +1,119 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "strings" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/controller/graph" +) + +// ClusterDeletionTransformer handles cluster deletion +type ClusterDeletionTransformer struct{} + +func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + cluster := transCtx.OrigCluster + if !isClusterDeleting(*cluster) { + return nil + } + + // list all kinds to be deleted based on v1alpha1.TerminationPolicyType + kinds := make([]client.ObjectList, 0) + switch cluster.Spec.TerminationPolicy { + case v1alpha1.DoNotTerminate: + transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeWarning, "DoNotTerminate", "spec.terminationPolicy %s is preventing deletion.", cluster.Spec.TerminationPolicy) + return graph.ErrFastReturn + case v1alpha1.Halt: + kinds = kindsForHalt() + case v1alpha1.Delete: + kinds = kindsForDelete() + case v1alpha1.WipeOut: + kinds = kindsForWipeOut() + } + + transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeNormal, constant.ReasonDeletingCR, "Deleting %s: %s", + strings.ToLower(cluster.GetObjectKind().GroupVersionKind().Kind), cluster.GetName()) + + // list all objects owned by this cluster in cache, and delete them all + // there is chance that objects leak occurs because of cache stale + // ignore the problem currently + // TODO: GC the leaked objects + snapshot, err := readCacheSnapshot(transCtx, *cluster, kinds...) + if err != nil { + return err + } + root, err := findRootVertex(dag) + if err != nil { + return err + } + for _, object := range snapshot { + vertex := &lifecycleVertex{obj: object, action: actionPtr(DELETE)} + dag.AddVertex(vertex) + dag.Connect(root, vertex) + } + root.action = actionPtr(DELETE) + + // fast return, that is stopping the plan.Build() stage and jump to plan.Execute() directly + return graph.ErrFastReturn +} + +func kindsForDoNotTerminate() []client.ObjectList { + return []client.ObjectList{} +} + +func kindsForHalt() []client.ObjectList { + kinds := kindsForDoNotTerminate() + kindsPlus := []client.ObjectList{ + &appsv1.StatefulSetList{}, + &appsv1.DeploymentList{}, + &corev1.ServiceList{}, + &corev1.SecretList{}, + &corev1.ConfigMapList{}, + &policyv1.PodDisruptionBudgetList{}, + } + return append(kinds, kindsPlus...) +} + +func kindsForDelete() []client.ObjectList { + kinds := kindsForHalt() + kindsPlus := []client.ObjectList{ + &corev1.PersistentVolumeClaimList{}, + &dataprotectionv1alpha1.BackupPolicyList{}, + &batchv1.JobList{}, + } + return append(kinds, kindsPlus...) +} + +func kindsForWipeOut() []client.ObjectList { + kinds := kindsForDelete() + kindsPlus := []client.ObjectList{ + &dataprotectionv1alpha1.BackupList{}, + } + return append(kinds, kindsPlus...) +} + +var _ graph.Transformer = &ClusterDeletionTransformer{} diff --git a/internal/controller/lifecycle/transformer_cluster_status.go b/internal/controller/lifecycle/transformer_cluster_status.go index 68103a6f0..6ea61685f 100644 --- a/internal/controller/lifecycle/transformer_cluster_status.go +++ b/internal/controller/lifecycle/transformer_cluster_status.go @@ -17,7 +17,6 @@ limitations under the License. package lifecycle import ( - "fmt" "reflect" "golang.org/x/exp/slices" @@ -25,16 +24,12 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/util" opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/component" "github.com/apecloud/kubeblocks/internal/controller/graph" - "github.com/apecloud/kubeblocks/internal/controller/plan" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -48,11 +43,7 @@ const ( clusterExistFailedOrAbnormal // cluster exists failed or abnormal component ) -type clusterStatusTransformer struct { - cc clusterRefResources - cli client.Client - ctx intctrlutil.RequestCtx - recorder record.EventRecorder +type ClusterStatusTransformer struct { // phaseSyncLevel defines a phase synchronization level to indicate how to handle cluster phase. phaseSyncLevel phaseSyncLevel // existsAbnormalOrFailed indicates whether the cluster exists abnormal or failed component. @@ -63,129 +54,185 @@ type clusterStatusTransformer struct { replicasNotReadyCompNames map[string]struct{} } -func newClusterStatusTransformer(ctx intctrlutil.RequestCtx, - cli client.Client, - recorder record.EventRecorder, - cc clusterRefResources) *clusterStatusTransformer { - return &clusterStatusTransformer{ - ctx: ctx, - cc: cc, - cli: cli, - recorder: recorder, - phaseSyncLevel: clusterPhaseNoChange, - notReadyCompNames: map[string]struct{}{}, - replicasNotReadyCompNames: map[string]struct{}{}, - } -} -func (c *clusterStatusTransformer) Transform(dag *graph.DAG) error { +func (t *ClusterStatusTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + origCluster := transCtx.OrigCluster + cluster := transCtx.Cluster rootVertex, err := findRootVertex(dag) if err != nil { return err } - origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) - cluster, _ := rootVertex.obj.(*appsv1alpha1.Cluster) - updateComponentPhase := func() { - vertices := findAllNot[*appsv1alpha1.Cluster](dag) + updateObservedGeneration := func() { + cluster.Status.ObservedGeneration = cluster.Generation + cluster.Status.ClusterDefGeneration = transCtx.ClusterDef.Generation + } + + updateClusterPhase := func() { + clusterPhase := cluster.Status.Phase + if clusterPhase == "" { + cluster.Status.Phase = appsv1alpha1.CreatingClusterPhase + } else if clusterPhase != appsv1alpha1.CreatingClusterPhase { + cluster.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase + } + } + + isStorageUpdated := func(oldSts, newSts *appsv1.StatefulSet) bool { + if oldSts == nil || newSts == nil { + return false + } + for _, oldVct := range oldSts.Spec.VolumeClaimTemplates { + var newVct *corev1.PersistentVolumeClaim + for _, v := range newSts.Spec.VolumeClaimTemplates { + if v.Name == oldVct.Name { + newVct = &v + break + } + } + if newVct == nil { + continue + } + if oldVct.Spec.Resources.Requests[corev1.ResourceStorage] != newVct.Spec.Resources.Requests[corev1.ResourceStorage] { + return true + } + } + return false + } + + updateComponentsPhase := func() { + vertices := findAll[*appsv1.StatefulSet](dag) + deployVertices := findAll[*appsv1.Deployment](dag) + vertices = append(vertices, deployVertices...) for _, vertex := range vertices { v, _ := vertex.(*lifecycleVertex) - if v.immutable || v.action == nil || *v.action != CREATE { + if v.immutable || v.action == nil { continue } - switch v.obj.(type) { - case *appsv1.StatefulSet, *appsv1.Deployment: + if *v.action == CREATE { updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) + continue } + if *v.action != UPDATE { + continue + } + oldSpec := reflect.ValueOf(v.oriObj).Elem().FieldByName("Spec") + newSpec := reflect.ValueOf(v.obj).Elem().FieldByName("Spec") + + // compare replicas + // oldReplicas := oldSpec.FieldByName("Replicas").Interface() + // newReplicas := newSpec.FieldByName("Replicas").Interface() + // if !reflect.DeepEqual(oldReplicas, newReplicas) { + // updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) + // continue + // } + // compare cpu & memory + oldResources := oldSpec.FieldByName("Template"). + FieldByName("Spec"). + FieldByName("Containers"). + Index(0). + FieldByName("Resources").Interface() + newResources := newSpec.FieldByName("Template"). + FieldByName("Spec"). + FieldByName("Containers"). + Index(0). + FieldByName("Resources").Interface() + if !reflect.DeepEqual(oldResources, newResources) { + updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) + continue + } + // compare sts storage + if _, ok := v.obj.(*appsv1.StatefulSet); ok { + oldSts, _ := v.oriObj.(*appsv1.StatefulSet) + newSts, _ := v.obj.(*appsv1.StatefulSet) + if !isStorageUpdated(oldSts, newSts) { + continue + } + } + updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) } } + initClusterStatusParams := func() { + t.phaseSyncLevel = clusterPhaseNoChange + t.notReadyCompNames = map[string]struct{}{} + t.replicasNotReadyCompNames = map[string]struct{}{} + } + switch { case isClusterDeleting(*origCluster): // if cluster is deleting, set root(cluster) vertex.action to DELETE rootVertex.action = actionPtr(DELETE) case isClusterUpdating(*origCluster): - c.ctx.Log.Info("update cluster status after applying resources ") - defer func() { - // update components' phase in cluster.status - updateComponentPhase() - rootVertex.action = actionPtr(STATUS) - rootVertex.immutable = reflect.DeepEqual(cluster.Status, origCluster.Status) - }() - cluster.Status.ObservedGeneration = cluster.Generation - cluster.Status.ClusterDefGeneration = c.cc.cd.Generation - applyResourcesCondition := newApplyResourcesCondition(cluster.Generation) - oldApplyCondition := meta.FindStatusCondition(cluster.Status.Conditions, applyResourcesCondition.Type) - if !conditionIsChanged(oldApplyCondition, applyResourcesCondition) { - return nil - } - meta.SetStatusCondition(&cluster.Status.Conditions, applyResourcesCondition) - rootVertex.postHandleAfterStatusPatch = append(rootVertex.postHandleAfterStatusPatch, func() error { - c.recorder.Event(cluster, corev1.EventTypeNormal, applyResourcesCondition.Reason, applyResourcesCondition.Message) - return nil - }) + transCtx.Logger.Info("update cluster status after applying resources ") + updateObservedGeneration() + updateClusterPhase() + updateComponentsPhase() + // update components' phase in cluster.status + rootVertex.action = actionPtr(STATUS) case isClusterStatusUpdating(*origCluster): - defer func() { - rootVertex.action = actionPtr(STATUS) - rootVertex.immutable = reflect.DeepEqual(cluster.Status, origCluster.Status) - }() + initClusterStatusParams() + defer func() { rootVertex.action = actionPtr(STATUS) }() // checks if the controller is handling the garbage of restore. - if err := c.handleGarbageOfRestoreBeforeRunning(cluster); err != nil { + if err := t.handleGarbageOfRestoreBeforeRunning(transCtx, cluster, dag); err != nil { return err } // reconcile the phase and conditions of the Cluster.status - if err := c.reconcileClusterStatus(cluster, rootVertex); err != nil { - return err - } - c.cleanupAnnotationsAfterRunning(cluster) - - if shouldRequeue, err := plan.DoPITRIfNeed(c.ctx.Ctx, c.cli, cluster); err != nil { - return err - } else if shouldRequeue { - return &realRequeueError{reason: "waiting pitr job", requeueAfter: requeueDuration} - } - if err = plan.DoPITRCleanup(c.ctx.Ctx, c.cli, cluster); err != nil { + if err := t.reconcileClusterStatus(transCtx, dag, cluster); err != nil { return err } + t.cleanupAnnotationsAfterRunning(cluster) } return nil } +// updateComponentPhaseWithOperation if workload of component changes, should update the component phase. +func updateComponentPhaseWithOperation(cluster *appsv1alpha1.Cluster, componentName string) { + componentPhase := appsv1alpha1.SpecReconcilingClusterCompPhase + if cluster.Status.Phase == appsv1alpha1.CreatingClusterPhase { + componentPhase = appsv1alpha1.CreatingClusterCompPhase + } + compStatus := cluster.Status.Components[componentName] + // synchronous component phase is consistent with cluster phase + compStatus.Phase = componentPhase + cluster.Status.SetComponentStatus(componentName, compStatus) +} + // reconcileClusterStatus reconciles phase and conditions of the Cluster.status. -func (c *clusterStatusTransformer) reconcileClusterStatus(cluster *appsv1alpha1.Cluster, rootVertex *lifecycleVertex) error { +func (t *ClusterStatusTransformer) reconcileClusterStatus(transCtx *ClusterTransformContext, dag *graph.DAG, cluster *appsv1alpha1.Cluster) error { if len(cluster.Status.Components) == 0 { return nil } // removes the invalid component of status.components which is deleted from spec.components. - c.removeInvalidCompStatus(cluster) + t.removeInvalidCompStatus(cluster) // do analysis of Cluster.Status.component and update the results to status synchronizer. - c.doAnalysisAndUpdateSynchronizer(cluster) + t.doAnalysisAndUpdateSynchronizer(dag, cluster) // sync the LatestOpsRequestProcessed condition. - c.syncOpsRequestProcessedCondition(cluster, rootVertex) + t.syncOpsRequestProcessedCondition(cluster) // handle the ready condition. - c.syncReadyConditionForCluster(cluster, rootVertex) + t.syncReadyConditionForCluster(cluster) // sync the cluster phase. - switch c.phaseSyncLevel { + switch t.phaseSyncLevel { case clusterIsRunning: if cluster.Status.Phase != appsv1alpha1.RunningClusterPhase { - c.syncClusterPhaseToRunning(cluster, rootVertex) + t.syncClusterPhaseToRunning(cluster) } case clusterIsStopped: if cluster.Status.Phase != appsv1alpha1.StoppedClusterPhase { - c.syncClusterPhaseToStopped(cluster, rootVertex) + t.syncClusterPhaseToStopped(cluster) } case clusterExistFailedOrAbnormal: - c.handleExistAbnormalOrFailed(cluster, rootVertex) + t.handleExistAbnormalOrFailed(transCtx, cluster) } return nil } // removeInvalidCompStatus removes the invalid component of status.components which is deleted from spec.components. -func (c *clusterStatusTransformer) removeInvalidCompStatus(cluster *appsv1alpha1.Cluster) { +func (t *ClusterStatusTransformer) removeInvalidCompStatus(cluster *appsv1alpha1.Cluster) { // remove the invalid component in status.components when the component is deleted from spec.components. tmpCompsStatus := map[string]appsv1alpha1.ClusterComponentStatus{} compsStatus := cluster.Status.Components @@ -199,7 +246,7 @@ func (c *clusterStatusTransformer) removeInvalidCompStatus(cluster *appsv1alpha1 } // doAnalysisAndUpdateSynchronizer analyses the Cluster.Status.Components and updates the results to the synchronizer. -func (c *clusterStatusTransformer) doAnalysisAndUpdateSynchronizer(cluster *appsv1alpha1.Cluster) { +func (t *ClusterStatusTransformer) doAnalysisAndUpdateSynchronizer(dag *graph.DAG, cluster *appsv1alpha1.Cluster) { var ( runningCompCount int stoppedCompCount int @@ -207,33 +254,53 @@ func (c *clusterStatusTransformer) doAnalysisAndUpdateSynchronizer(cluster *apps // analysis the status of components and calculate the cluster phase. for k, v := range cluster.Status.Components { if v.PodsReady == nil || !*v.PodsReady { - c.replicasNotReadyCompNames[k] = struct{}{} - c.notReadyCompNames[k] = struct{}{} + t.replicasNotReadyCompNames[k] = struct{}{} + t.notReadyCompNames[k] = struct{}{} } switch v.Phase { case appsv1alpha1.AbnormalClusterCompPhase, appsv1alpha1.FailedClusterCompPhase: - c.existsAbnormalOrFailed, c.notReadyCompNames[k] = true, struct{}{} + t.existsAbnormalOrFailed, t.notReadyCompNames[k] = true, struct{}{} case appsv1alpha1.RunningClusterCompPhase: - runningCompCount += 1 + if !isComponentInHorizontalScaling(dag, k) { + runningCompCount += 1 + } case appsv1alpha1.StoppedClusterCompPhase: stoppedCompCount += 1 } } - if c.existsAbnormalOrFailed { - c.phaseSyncLevel = clusterExistFailedOrAbnormal + if t.existsAbnormalOrFailed { + t.phaseSyncLevel = clusterExistFailedOrAbnormal return } switch len(cluster.Status.Components) { case runningCompCount: - c.phaseSyncLevel = clusterIsRunning + t.phaseSyncLevel = clusterIsRunning case stoppedCompCount: // cluster is Stopped when cluster is not Running and all components are Stopped or Running - c.phaseSyncLevel = clusterIsStopped + t.phaseSyncLevel = clusterIsStopped } } +func isComponentInHorizontalScaling(dag *graph.DAG, componentName string) bool { + stsVertices := findAll[*appsv1.StatefulSet](dag) + for _, v := range stsVertices { + vertex, _ := v.(*lifecycleVertex) + if vertex.action == nil || *vertex.action != UPDATE { + continue + } + name := vertex.obj.GetLabels()[constant.KBAppComponentLabelKey] + if name != componentName { + continue + } + oldSts, _ := vertex.oriObj.(*appsv1.StatefulSet) + newSts, _ := vertex.obj.(*appsv1.StatefulSet) + return *oldSts.Spec.Replicas != *newSts.Spec.Replicas + } + return false +} + // handleOpsRequestProcessedCondition syncs the condition that OpsRequest has been processed. -func (c *clusterStatusTransformer) syncOpsRequestProcessedCondition(cluster *appsv1alpha1.Cluster, rootVertex *lifecycleVertex) { +func (t *ClusterStatusTransformer) syncOpsRequestProcessedCondition(cluster *appsv1alpha1.Cluster) { opsCondition := meta.FindStatusCondition(cluster.Status.Conditions, appsv1alpha1.ConditionTypeLatestOpsRequestProcessed) if opsCondition == nil || opsCondition.Status == metav1.ConditionTrue { return @@ -248,77 +315,44 @@ func (c *clusterStatusTransformer) syncOpsRequestProcessedCondition(cluster *app return } meta.SetStatusCondition(&cluster.Status.Conditions, processedCondition) - rootVertex.postHandleAfterStatusPatch = append(rootVertex.postHandleAfterStatusPatch, func() error { - // send an event when all pods of the components are ready. - c.recorder.Event(cluster, corev1.EventTypeNormal, processedCondition.Reason, processedCondition.Message) - return nil - }) } // syncReadyConditionForCluster syncs the cluster conditions with ClusterReady and ReplicasReady type. -func (c *clusterStatusTransformer) syncReadyConditionForCluster(cluster *appsv1alpha1.Cluster, rootVertex *lifecycleVertex) { - if len(c.replicasNotReadyCompNames) == 0 { - oldReplicasReadyCondition := meta.FindStatusCondition(cluster.Status.Conditions, appsv1alpha1.ConditionTypeReplicasReady) +func (t *ClusterStatusTransformer) syncReadyConditionForCluster(cluster *appsv1alpha1.Cluster) { + if len(t.replicasNotReadyCompNames) == 0 { // if all replicas of cluster are ready, set ReasonAllReplicasReady to status.conditions readyCondition := newAllReplicasPodsReadyConditions() - if oldReplicasReadyCondition == nil || oldReplicasReadyCondition.Status == metav1.ConditionFalse { - rootVertex.postHandleAfterStatusPatch = append(rootVertex.postHandleAfterStatusPatch, func() error { - // send an event when all pods of the components are ready. - c.recorder.Event(cluster, corev1.EventTypeNormal, readyCondition.Reason, readyCondition.Message) - return nil - }) - } meta.SetStatusCondition(&cluster.Status.Conditions, readyCondition) } else { - meta.SetStatusCondition(&cluster.Status.Conditions, newReplicasNotReadyCondition(c.replicasNotReadyCompNames)) + meta.SetStatusCondition(&cluster.Status.Conditions, newReplicasNotReadyCondition(t.replicasNotReadyCompNames)) } - if len(c.notReadyCompNames) > 0 { - meta.SetStatusCondition(&cluster.Status.Conditions, newComponentsNotReadyCondition(c.notReadyCompNames)) + if len(t.notReadyCompNames) > 0 { + meta.SetStatusCondition(&cluster.Status.Conditions, newComponentsNotReadyCondition(t.notReadyCompNames)) } } // syncClusterPhaseToRunning syncs the cluster phase to Running. -func (c *clusterStatusTransformer) syncClusterPhaseToRunning(cluster *appsv1alpha1.Cluster, rootVertex *lifecycleVertex) { +func (t *ClusterStatusTransformer) syncClusterPhaseToRunning(cluster *appsv1alpha1.Cluster) { cluster.Status.Phase = appsv1alpha1.RunningClusterPhase meta.SetStatusCondition(&cluster.Status.Conditions, newClusterReadyCondition(cluster.Name)) - rootVertex.postHandleAfterStatusPatch = append(rootVertex.postHandleAfterStatusPatch, func() error { - message := fmt.Sprintf("Cluster: %s is ready, current phase is Running", cluster.Name) - c.recorder.Event(cluster, corev1.EventTypeNormal, string(appsv1alpha1.RunningClusterPhase), message) - return opsutil.MarkRunningOpsRequestAnnotation(c.ctx.Ctx, c.cli, cluster) - }) } // syncClusterToStopped syncs the cluster phase to Stopped. -func (c *clusterStatusTransformer) syncClusterPhaseToStopped(cluster *appsv1alpha1.Cluster, rootVertex *lifecycleVertex) { +func (t *ClusterStatusTransformer) syncClusterPhaseToStopped(cluster *appsv1alpha1.Cluster) { cluster.Status.Phase = appsv1alpha1.StoppedClusterPhase - rootVertex.postHandleAfterStatusPatch = append(rootVertex.postHandleAfterStatusPatch, func() error { - message := fmt.Sprintf("Cluster: %s stopped successfully.", cluster.Name) - c.recorder.Event(cluster, corev1.EventTypeNormal, string(cluster.Status.Phase), message) - return opsutil.MarkRunningOpsRequestAnnotation(c.ctx.Ctx, c.cli, cluster) - }) } // handleExistAbnormalOrFailed handles the cluster status when some components are not ready. -func (c *clusterStatusTransformer) handleExistAbnormalOrFailed(cluster *appsv1alpha1.Cluster, rootVertex *lifecycleVertex) { - oldPhase := cluster.Status.Phase +func (t *ClusterStatusTransformer) handleExistAbnormalOrFailed(transCtx *ClusterTransformContext, cluster *appsv1alpha1.Cluster) { componentMap, clusterAvailabilityEffectMap, _ := getComponentRelatedInfo(cluster, - c.cc.cd, "") + *transCtx.ClusterDef, "") // handle the cluster status when some components are not ready. handleClusterPhaseWhenCompsNotReady(cluster, componentMap, clusterAvailabilityEffectMap) - currPhase := cluster.Status.Phase - if slices.Contains(appsv1alpha1.GetClusterFailedPhases(), currPhase) && oldPhase != currPhase { - rootVertex.postHandleAfterStatusPatch = append(rootVertex.postHandleAfterStatusPatch, func() error { - message := fmt.Sprintf("Cluster: %s is %s, check according to the components message", - cluster.Name, currPhase) - c.recorder.Event(cluster, corev1.EventTypeWarning, string(cluster.Status.Phase), message) - return opsutil.MarkRunningOpsRequestAnnotation(c.ctx.Ctx, c.cli, cluster) - }) - } } // cleanupAnnotationsAfterRunning cleans up the cluster annotations after cluster is Running. -func (c *clusterStatusTransformer) cleanupAnnotationsAfterRunning(cluster *appsv1alpha1.Cluster) { +func (t *ClusterStatusTransformer) cleanupAnnotationsAfterRunning(cluster *appsv1alpha1.Cluster) { if !slices.Contains(appsv1alpha1.GetClusterTerminalPhases(), cluster.Status.Phase) { return } @@ -332,7 +366,7 @@ func (c *clusterStatusTransformer) cleanupAnnotationsAfterRunning(cluster *appsv // handleRestoreGarbageBeforeRunning handles the garbage for restore before cluster phase changes to Running. // @return ErrNoOps if no operation // Deprecated: to be removed by PITR feature. -func (c *clusterStatusTransformer) handleGarbageOfRestoreBeforeRunning(cluster *appsv1alpha1.Cluster) error { +func (t *ClusterStatusTransformer) handleGarbageOfRestoreBeforeRunning(transCtx *ClusterTransformContext, cluster *appsv1alpha1.Cluster, dag *graph.DAG) error { clusterBackupResourceMap, err := getClusterBackupSourceMap(cluster) if err != nil { return err @@ -347,22 +381,34 @@ func (c *clusterStatusTransformer) handleGarbageOfRestoreBeforeRunning(cluster * } } // remove the garbage for restore if the cluster restores from backup. - return c.removeGarbageWithRestore(cluster, clusterBackupResourceMap) + return t.removeGarbageWithRestore(transCtx, cluster, clusterBackupResourceMap, dag) } // REVIEW: this handling is rather hackish, call for refactor. // removeGarbageWithRestore removes the garbage for restore when all components are Running. // @return ErrNoOps if no operation // Deprecated: -func (c *clusterStatusTransformer) removeGarbageWithRestore( +func (t *ClusterStatusTransformer) removeGarbageWithRestore( + transCtx *ClusterTransformContext, cluster *appsv1alpha1.Cluster, - clusterBackupResourceMap map[string]string) error { + clusterBackupResourceMap map[string]string, + dag *graph.DAG) error { var ( err error ) + vertices := findAll[*appsv1.StatefulSet](dag) for k, v := range clusterBackupResourceMap { + // get the vertex list which contains sts owned by componentName + vertexList := make([]graph.Vertex, 0) + for _, vertex := range vertices { + v, _ := vertex.(*lifecycleVertex) + labels := v.obj.GetLabels() + if labels != nil && labels[constant.KBAppComponentLabelKey] == k { + vertexList = append(vertexList, vertex) + } + } // remove the init container for restore - if _, err = c.removeStsInitContainerForRestore(cluster, k, v); err != nil { + if _, err = t.removeStsInitContainerForRestore(cluster, k, v, vertexList); err != nil { return err } } @@ -370,17 +416,15 @@ func (c *clusterStatusTransformer) removeGarbageWithRestore( } // removeStsInitContainerForRestore removes the statefulSet's init container which restores data from backup. -func (c *clusterStatusTransformer) removeStsInitContainerForRestore( +func (t *ClusterStatusTransformer) removeStsInitContainerForRestore( cluster *appsv1alpha1.Cluster, componentName, - backupName string) (bool, error) { - // get the sts list of component - stsList := &appsv1.StatefulSetList{} - if err := util.GetObjectListByComponentName(c.ctx.Ctx, c.cli, *cluster, stsList, componentName); err != nil { - return false, err - } + backupName string, + vertexList []graph.Vertex) (bool, error) { var doRemoveInitContainers bool - for _, sts := range stsList.Items { + for _, vertex := range vertexList { + v, _ := vertex.(*lifecycleVertex) + sts, _ := v.obj.(*appsv1.StatefulSet) initContainers := sts.Spec.Template.Spec.InitContainers restoreInitContainerName := component.GetRestoredInitContainerName(backupName) restoreInitContainerIndex, _ := intctrlutil.GetContainerByName(initContainers, restoreInitContainerName) @@ -390,8 +434,9 @@ func (c *clusterStatusTransformer) removeStsInitContainerForRestore( doRemoveInitContainers = true initContainers = append(initContainers[:restoreInitContainerIndex], initContainers[restoreInitContainerIndex+1:]...) sts.Spec.Template.Spec.InitContainers = initContainers - if err := c.cli.Update(c.ctx.Ctx, &sts); err != nil { - return false, err + if v.oriObj != nil { + v.immutable = false + v.action = actionPtr(UPDATE) } } if doRemoveInitContainers { @@ -477,3 +522,5 @@ func getComponentRelatedInfo(cluster *appsv1alpha1.Cluster, clusterDef appsv1alp } return componentMap, clusterAvailabilityEffectMap, componentDef } + +var _ graph.Transformer = &ClusterStatusTransformer{} diff --git a/internal/controller/lifecycle/transformer_config.go b/internal/controller/lifecycle/transformer_config.go index ede95c6ba..9938051b6 100644 --- a/internal/controller/lifecycle/transformer_config.go +++ b/internal/controller/lifecycle/transformer_config.go @@ -23,10 +23,10 @@ import ( "github.com/apecloud/kubeblocks/internal/controller/graph" ) -// configTransformer makes all config related ConfigMaps immutable -type configTransformer struct{} +// ConfigTransformer makes all config related ConfigMaps immutable +type ConfigTransformer struct{} -func (c *configTransformer) Transform(dag *graph.DAG) error { +func (c *ConfigTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { for _, vertex := range findAll[*corev1.ConfigMap](dag) { v, _ := vertex.(*lifecycleVertex) cm, _ := v.obj.(*corev1.ConfigMap) @@ -38,3 +38,5 @@ func (c *configTransformer) Transform(dag *graph.DAG) error { } return nil } + +var _ graph.Transformer = &ConfigTransformer{} diff --git a/internal/controller/lifecycle/transformer_credential.go b/internal/controller/lifecycle/transformer_credential.go index 4f58e5796..794391625 100644 --- a/internal/controller/lifecycle/transformer_credential.go +++ b/internal/controller/lifecycle/transformer_credential.go @@ -23,10 +23,10 @@ import ( "github.com/apecloud/kubeblocks/internal/controller/graph" ) -// credentialTransformer puts the credential Secret at the beginning of the DAG -type credentialTransformer struct{} +// CredentialTransformer puts the credential Secret at the beginning of the DAG +type CredentialTransformer struct{} -func (c *credentialTransformer) Transform(dag *graph.DAG) error { +func (c *CredentialTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { var secretVertices, noneRootVertices []graph.Vertex secretVertices = findAll[*corev1.Secret](dag) noneRootVertices = findAllNot[*appsv1alpha1.Cluster](dag) @@ -43,3 +43,5 @@ func (c *credentialTransformer) Transform(dag *graph.DAG) error { } return nil } + +var _ graph.Transformer = &CredentialTransformer{} diff --git a/internal/controller/lifecycle/transformer_fill_class.go b/internal/controller/lifecycle/transformer_fill_class.go index 969527da1..d5440745e 100644 --- a/internal/controller/lifecycle/transformer_fill_class.go +++ b/internal/controller/lifecycle/transformer_fill_class.go @@ -28,26 +28,24 @@ import ( "github.com/apecloud/kubeblocks/internal/class" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/graph" - intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -// fixClusterLabelsTransformer fill the class related info to cluster -type fillClass struct { - cc clusterRefResources - cli client.Client - ctx intctrlutil.RequestCtx -} +// FillClassTransformer fill the class related info to cluster +type FillClassTransformer struct{} -func (r *fillClass) Transform(dag *graph.DAG) error { - rootVertex, err := findRootVertex(dag) - if err != nil { - return err +func (r *FillClassTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + cluster := transCtx.Cluster + if isClusterDeleting(*cluster) { + return nil } - cluster, _ := rootVertex.obj.(*appsv1alpha1.Cluster) - return r.fillClass(r.ctx, cluster, r.cc.cd) + return r.fillClass(transCtx) } -func (r *fillClass) fillClass(reqCtx intctrlutil.RequestCtx, cluster *appsv1alpha1.Cluster, clusterDefinition appsv1alpha1.ClusterDefinition) error { +func (r *FillClassTransformer) fillClass(transCtx *ClusterTransformContext) error { + cluster := transCtx.Cluster + clusterDefinition := transCtx.ClusterDef + var ( classDefinitionList appsv1alpha1.ComponentClassDefinitionList ) @@ -55,7 +53,7 @@ func (r *fillClass) fillClass(reqCtx intctrlutil.RequestCtx, cluster *appsv1alph ml := []client.ListOption{ client.MatchingLabels{constant.ClusterDefLabelKey: clusterDefinition.Name}, } - if err := r.cli.List(reqCtx.Ctx, &classDefinitionList, ml...); err != nil { + if err := transCtx.Client.List(transCtx.Context, &classDefinitionList, ml...); err != nil { return err } compClasses, err := class.GetClasses(classDefinitionList) @@ -64,7 +62,7 @@ func (r *fillClass) fillClass(reqCtx intctrlutil.RequestCtx, cluster *appsv1alph } var constraintList appsv1alpha1.ComponentResourceConstraintList - if err = r.cli.List(reqCtx.Ctx, &constraintList); err != nil { + if err = transCtx.Client.List(transCtx.Context, &constraintList); err != nil { return err } @@ -163,3 +161,5 @@ func buildVolumeClaimByClass(cls *appsv1alpha1.ComponentClassInstance) []appsv1a } return volumes } + +var _ graph.Transformer = &FillClassTransformer{} diff --git a/internal/controller/lifecycle/transformer_fix_cluster_labels.go b/internal/controller/lifecycle/transformer_fix_meta.go similarity index 60% rename from internal/controller/lifecycle/transformer_fix_cluster_labels.go rename to internal/controller/lifecycle/transformer_fix_meta.go index cff0c5875..82aa9b29c 100644 --- a/internal/controller/lifecycle/transformer_fix_cluster_labels.go +++ b/internal/controller/lifecycle/transformer_fix_meta.go @@ -17,19 +17,25 @@ limitations under the License. package lifecycle import ( - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/apecloud/kubeblocks/internal/controller/graph" ) -// fixClusterLabelsTransformer should patch the label first to prevent the label from being modified by the user. -type fixClusterLabelsTransformer struct{} +type FixMetaTransformer struct{} + +func (t *FixMetaTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + cluster := transCtx.Cluster -func (f *fixClusterLabelsTransformer) Transform(dag *graph.DAG) error { - rootVertex, err := findRootVertex(dag) - if err != nil { - return err + // The object is not being deleted, so if it does not have our finalizer, + // then lets add the finalizer and update the object. This is equivalent + // registering our finalizer. + if !controllerutil.ContainsFinalizer(cluster, dbClusterFinalizerName) { + controllerutil.AddFinalizer(cluster, dbClusterFinalizerName) } - cluster, _ := rootVertex.obj.(*appsv1alpha1.Cluster) + + // patch the label to prevent the label from being modified by the user. labels := cluster.Labels if labels == nil { labels = map[string]string{} @@ -43,5 +49,8 @@ func (f *fixClusterLabelsTransformer) Transform(dag *graph.DAG) error { labels[clusterDefLabelKey] = cdName labels[clusterVersionLabelKey] = cvName cluster.Labels = labels + return nil } + +var _ graph.Transformer = &FixMetaTransformer{} diff --git a/internal/controller/lifecycle/transformer_init.go b/internal/controller/lifecycle/transformer_init.go index e8f753a1b..a194ef3a6 100644 --- a/internal/controller/lifecycle/transformer_init.go +++ b/internal/controller/lifecycle/transformer_init.go @@ -17,7 +17,11 @@ limitations under the License. package lifecycle import ( + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" "github.com/apecloud/kubeblocks/internal/controller/graph" ) @@ -26,9 +30,37 @@ type initTransformer struct { originCluster *appsv1alpha1.Cluster } -func (i *initTransformer) Transform(dag *graph.DAG) error { +func (t *initTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { // put the cluster object first, it will be root vertex of DAG - rootVertex := &lifecycleVertex{obj: i.cluster, oriObj: i.originCluster} + rootVertex := &lifecycleVertex{obj: t.cluster, oriObj: t.originCluster, action: actionPtr(STATUS)} dag.AddVertex(rootVertex) + + if !isClusterDeleting(*t.cluster) { + t.handleLatestOpsRequestProcessingCondition() + } return nil } + +// updateLatestOpsRequestProcessingCondition handles the latest opsRequest processing condition. +func (t *initTransformer) handleLatestOpsRequestProcessingCondition() { + opsRecords, _ := opsutil.GetOpsRequestSliceFromCluster(t.cluster) + if len(opsRecords) == 0 { + return + } + ops := opsRecords[0] + opsBehaviour, ok := appsv1alpha1.OpsRequestBehaviourMapper[ops.Type] + if !ok { + return + } + opsCondition := newOpsRequestProcessingCondition(ops.Name, string(ops.Type), opsBehaviour.ProcessingReasonInClusterCondition) + oldCondition := meta.FindStatusCondition(t.cluster.Status.Conditions, opsCondition.Type) + if oldCondition == nil { + // if this condition not exists, insert it to the first position. + opsCondition.LastTransitionTime = metav1.Now() + t.cluster.Status.Conditions = append([]metav1.Condition{opsCondition}, t.cluster.Status.Conditions...) + } else { + meta.SetStatusCondition(&t.cluster.Status.Conditions, opsCondition) + } +} + +var _ graph.Transformer = &initTransformer{} diff --git a/internal/controller/lifecycle/transformer_object_action.go b/internal/controller/lifecycle/transformer_object_action.go index fa66a1a43..d7d725f66 100644 --- a/internal/controller/lifecycle/transformer_object_action.go +++ b/internal/controller/lifecycle/transformer_object_action.go @@ -19,46 +19,27 @@ package lifecycle import ( "strings" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" - client2 "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/graph" - intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -// objectActionTransformer reads all Vertex.Obj in cache and compute the diff DAG. -type objectActionTransformer struct { - cli client2.ReadonlyClient - ctx intctrlutil.RequestCtx -} +// ObjectActionTransformer reads all Vertex.Obj in cache and compute the diff DAG. +type ObjectActionTransformer struct{} -func ownKinds() []client.ObjectList { - return []client.ObjectList{ - &appsv1.StatefulSetList{}, - &appsv1.DeploymentList{}, - &corev1.ServiceList{}, - &corev1.SecretList{}, - &corev1.ConfigMapList{}, - &policyv1.PodDisruptionBudgetList{}, - &dataprotectionv1alpha1.BackupPolicyList{}, - } -} +func (t *ObjectActionTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + origCluster := transCtx.OrigCluster -func (c *objectActionTransformer) Transform(dag *graph.DAG) error { - rootVertex, err := findRootVertex(dag) + // get the old objects snapshot + oldSnapshot, err := readCacheSnapshot(transCtx, *origCluster, ownKinds()...) if err != nil { return err } - origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) - // get the old objects snapshot - oldSnapshot, err := readCacheSnapshot(c.ctx.Ctx, c.cli, *origCluster, ownKinds()...) + rootVertex, err := findRootVertex(dag) if err != nil { return err } @@ -164,3 +145,5 @@ func (c *objectActionTransformer) Transform(dag *graph.DAG) error { return nil } + +var _ graph.Transformer = &ObjectActionTransformer{} diff --git a/internal/controller/lifecycle/transformer_ownership.go b/internal/controller/lifecycle/transformer_ownership.go index b59a08758..ae59fdb81 100644 --- a/internal/controller/lifecycle/transformer_ownership.go +++ b/internal/controller/lifecycle/transformer_ownership.go @@ -24,12 +24,10 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -// ownershipTransformer add finalizer to all none cluster objects -type ownershipTransformer struct { - finalizer string -} +// OwnershipTransformer add finalizer to all none cluster objects +type OwnershipTransformer struct{} -func (f *ownershipTransformer) Transform(dag *graph.DAG) error { +func (f *OwnershipTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { rootVertex, err := findRootVertex(dag) if err != nil { return err @@ -45,3 +43,5 @@ func (f *ownershipTransformer) Transform(dag *graph.DAG) error { } return nil } + +var _ graph.Transformer = &OwnershipTransformer{} diff --git a/internal/controller/lifecycle/transformer_pitr.go b/internal/controller/lifecycle/transformer_pitr.go new file mode 100644 index 000000000..e54c48998 --- /dev/null +++ b/internal/controller/lifecycle/transformer_pitr.go @@ -0,0 +1,49 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/internal/controller/graph" + "github.com/apecloud/kubeblocks/internal/controller/plan" +) + +type PITRTransformer struct { + client.Client +} + +func (t *PITRTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + cluster := transCtx.Cluster + // handle PITR only when cluster is in status reconciliation stage + if !isClusterStatusUpdating(*cluster) { + return nil + } + // TODO: (free6om) refactor: remove client.Client + if shouldRequeue, err := plan.DoPITRIfNeed(transCtx.Context, t.Client, cluster); err != nil { + return err + } else if shouldRequeue { + return &realRequeueError{reason: "waiting pitr job", requeueAfter: requeueDuration} + } + if err := plan.DoPITRCleanup(transCtx.Context, t.Client, cluster); err != nil { + return err + } + return nil +} + +var _ graph.Transformer = &PITRTransformer{} diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 29a68a0a5..282babed1 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -18,12 +18,13 @@ package lifecycle import ( "context" + "errors" "fmt" "strings" "time" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" - "github.com/pkg/errors" + "github.com/spf13/viper" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -37,19 +38,21 @@ import ( dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/builder" - types2 "github.com/apecloud/kubeblocks/internal/controller/client" + roclient "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/component" "github.com/apecloud/kubeblocks/internal/controller/graph" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -type stsHorizontalScalingTransformer struct { - cr clusterRefResources - cli types2.ReadonlyClient - ctx intctrlutil.RequestCtx -} +type StsHorizontalScalingTransformer struct{} -func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { +func (t *StsHorizontalScalingTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + reqCtx := intctrlutil.RequestCtx{ + Ctx: transCtx.Context, + Log: transCtx.Logger, + Recorder: transCtx.EventRecorder, + } rootVertex, err := findRootVertex(dag) if err != nil { return err @@ -60,9 +63,6 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { handleHorizontalScaling := func(vertex *lifecycleVertex) error { stsObj, _ := vertex.oriObj.(*appsv1.StatefulSet) stsProto, _ := vertex.obj.(*appsv1.StatefulSet) - if *stsObj.Spec.Replicas == *stsProto.Spec.Replicas { - return nil - } key := client.ObjectKey{ Namespace: stsProto.GetNamespace(), @@ -74,18 +74,20 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { } // find component of current statefulset componentName := stsObj.Labels[constant.KBAppComponentLabelKey] - components := mergeComponentsList(s.ctx, + components := mergeComponentsList(reqCtx, *cluster, - s.cr.cd, - s.cr.cd.Spec.ComponentDefs, + *transCtx.ClusterDef, + transCtx.ClusterDef.Spec.ComponentDefs, cluster.Spec.ComponentSpecs) comp := getComponent(components, componentName) if comp == nil { - s.ctx.Recorder.Eventf(cluster, - corev1.EventTypeWarning, - "HorizontalScaleFailed", - "component %s not found", - componentName) + if *stsObj.Spec.Replicas != *stsProto.Spec.Replicas { + transCtx.EventRecorder.Eventf(cluster, + corev1.EventTypeWarning, + "HorizontalScaleFailed", + "component %s not found", + componentName) + } return nil } cleanCronJobs := func() error { @@ -99,7 +101,7 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { cronJobKey := pvcKey cronJobKey.Name = "delete-pvc-" + pvcKey.Name cronJob := &batchv1.CronJob{} - if err := s.cli.Get(s.ctx.Ctx, cronJobKey, cronJob); err != nil { + if err := transCtx.Client.Get(transCtx.Context, cronJobKey, cronJob); err != nil { return client.IgnoreNotFound(err) } v := &lifecycleVertex{ @@ -121,9 +123,9 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { Name: fmt.Sprintf("%s-%s-%d", vct.Name, stsObj.Name, i), } // check pvc existence - pvcExists, err := isPVCExists(s.cli, s.ctx.Ctx, pvcKey) + pvcExists, err := isPVCExists(transCtx.Client, transCtx.Context, pvcKey) if err != nil { - return true, err + return false, err } if !pvcExists { return false, nil @@ -136,20 +138,41 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { checkAllPVCBoundIfNeeded := func() (bool, error) { if comp.HorizontalScalePolicy == nil || comp.HorizontalScalePolicy.Type != appsv1alpha1.HScaleDataClonePolicyFromSnapshot || - !isSnapshotAvailable(s.cli, s.ctx.Ctx) { + !isSnapshotAvailable(transCtx.Client, transCtx.Context) { return true, nil } - return isAllPVCBound(s.cli, s.ctx.Ctx, stsObj) + return isAllPVCBound(transCtx.Client, transCtx.Context, stsProto) } cleanBackupResourcesIfNeeded := func() error { if comp.HorizontalScalePolicy == nil || comp.HorizontalScalePolicy.Type != appsv1alpha1.HScaleDataClonePolicyFromSnapshot || - !isSnapshotAvailable(s.cli, s.ctx.Ctx) { + !isSnapshotAvailable(transCtx.Client, transCtx.Context) { return nil } // if all pvc bounded, clean backup resources - return deleteSnapshot(s.cli, s.ctx, snapshotKey, cluster, comp, dag, rootVertex) + return deleteSnapshot(transCtx.Client, reqCtx, snapshotKey, cluster, comp, dag, rootVertex) + } + + emitHorizontalScalingEvent := func() { + if cluster.Status.Components == nil { + return + } + if *stsObj.Spec.Replicas == *stsProto.Spec.Replicas { + return + } + if componentStatus, ok := cluster.Status.Components[componentName]; ok { + if componentStatus.Phase == appsv1alpha1.SpecReconcilingClusterCompPhase { + return + } + transCtx.EventRecorder.Eventf(cluster, + corev1.EventTypeNormal, + "HorizontalScale", + "Start horizontal scale component %s from %d to %d", + comp.Name, + *stsObj.Spec.Replicas, + *stsProto.Spec.Replicas) + } } scaleOut := func() error { @@ -166,19 +189,25 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { return nil } // do backup according to component's horizontal scale policy - if err := doBackup(s.ctx, s.cli, comp, snapshotKey, dag, rootVertex, vertex); err != nil { + vertex.immutable = true + if err := doBackup(reqCtx, transCtx.Client, comp, snapshotKey, dag, rootVertex, vertex); err != nil { return err } - vertex.immutable = true return nil } - // check all pvc bound, requeue if not all ready + // pvcs are ready, stateful_set.replicas should be updated + vertex.immutable = false + + return nil + } + + postScaleOut := func() error { + // check all pvc bound, wait next reconciliation if not all ready allPVCBounded, err := checkAllPVCBoundIfNeeded() if err != nil { return err } if !allPVCBounded { - vertex.immutable = true return nil } // clean backup resources. @@ -186,12 +215,7 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { if err := cleanBackupResourcesIfNeeded(); err != nil { return err } - - // pvcs are ready, stateful_set.replicas should be updated - vertex.immutable = false - return nil - } scaleIn := func() error { @@ -199,6 +223,7 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { if *stsProto.Spec.Replicas == 0 || len(stsObj.Spec.VolumeClaimTemplates) == 0 { return nil } + for i := *stsProto.Spec.Replicas; i < *stsObj.Spec.Replicas; i++ { for _, vct := range stsObj.Spec.VolumeClaimTemplates { pvcKey := types.NamespacedName{ @@ -206,14 +231,26 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { Name: fmt.Sprintf("%s-%s-%d", vct.Name, stsObj.Name, i), } // create cronjob to delete pvc after 30 minutes - if err := checkedCreateDeletePVCCronJob(s.cli, s.ctx, pvcKey, stsObj, cluster, dag, rootVertex); err != nil { + if err := checkedCreateDeletePVCCronJob(transCtx.Client, reqCtx, pvcKey, stsObj, cluster, dag, rootVertex); err != nil { return err } } } return nil } + updateClusterPhase := func() { + if *stsObj.Spec.Replicas == *stsProto.Spec.Replicas { + return + } + clusterPhase := cluster.Status.Phase + if clusterPhase == "" { + cluster.Status.Phase = appsv1alpha1.CreatingClusterPhase + } else if clusterPhase != appsv1alpha1.CreatingClusterPhase { + cluster.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase + } + } + updateClusterPhase() // when horizontal scaling up, sometimes db needs backup to sync data from master, // log is not reliable enough since it can be recycled var err error @@ -227,15 +264,10 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { if err != nil { return err } + emitHorizontalScalingEvent() - if *stsObj.Spec.Replicas != *stsProto.Spec.Replicas { - s.ctx.Recorder.Eventf(cluster, - corev1.EventTypeNormal, - "HorizontalScale", - "Start horizontal scale component %s from %d to %d", - comp.Name, - *stsObj.Spec.Replicas, - *stsProto.Spec.Replicas) + if err = postScaleOut(); err != nil { + return err } return nil @@ -294,7 +326,7 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { // by sts: we only handle the pvc deletion which occurs in cluster deletion. // by h-scale transformer: we handle the pvc creation and deletion, the creation is handled in h-scale funcs. // so all in all, here we should only handle the pvc deletion of both types. - oldSnapshot, err := readCacheSnapshot(s.ctx.Ctx, s.cli, *cluster, &corev1.PersistentVolumeClaimList{}) + oldSnapshot, err := readCacheSnapshot(transCtx, *cluster, &corev1.PersistentVolumeClaimList{}) if err != nil { return err } @@ -308,7 +340,7 @@ func (s *stsHorizontalScalingTransformer) Transform(dag *graph.DAG) error { return nil } -func isPVCExists(cli types2.ReadonlyClient, +func isPVCExists(cli roclient.ReadonlyClient, ctx context.Context, pvcKey types.NamespacedName) (bool, error) { pvc := corev1.PersistentVolumeClaim{} @@ -346,7 +378,7 @@ func getComponent(componentList []component.SynthesizedComponent, name string) * } func doBackup(reqCtx intctrlutil.RequestCtx, - cli types2.ReadonlyClient, + cli roclient.ReadonlyClient, component *component.SynthesizedComponent, snapshotKey types.NamespacedName, dag *graph.DAG, @@ -377,7 +409,7 @@ func doBackup(reqCtx intctrlutil.RequestCtx, "HorizontalScaleFailed", "volume snapshot not support") // TODO: add ut - return errors.Errorf("volume snapshot not support") + return fmt.Errorf("volume snapshot not support") } vcts := component.VolumeClaimTemplates if len(vcts) == 0 { @@ -461,7 +493,7 @@ func doBackup(reqCtx intctrlutil.RequestCtx, } // TODO: handle unfinished jobs from previous scale in -func checkedCreateDeletePVCCronJob(cli types2.ReadonlyClient, +func checkedCreateDeletePVCCronJob(cli roclient.ReadonlyClient, reqCtx intctrlutil.RequestCtx, pvcKey types.NamespacedName, stsObj *appsv1.StatefulSet, @@ -501,13 +533,16 @@ func timeToSchedule(t time.Time) string { } // check volume snapshot available -func isSnapshotAvailable(cli types2.ReadonlyClient, ctx context.Context) bool { +func isSnapshotAvailable(cli roclient.ReadonlyClient, ctx context.Context) bool { + if !viper.GetBool("VOLUMESNAPSHOT") { + return false + } vsList := snapshotv1.VolumeSnapshotList{} getVSErr := cli.List(ctx, &vsList) return getVSErr == nil } -func isAllPVCBound(cli types2.ReadonlyClient, +func isAllPVCBound(cli roclient.ReadonlyClient, ctx context.Context, stsObj *appsv1.StatefulSet) (bool, error) { if len(stsObj.Spec.VolumeClaimTemplates) == 0 { @@ -521,7 +556,7 @@ func isAllPVCBound(cli types2.ReadonlyClient, pvc := corev1.PersistentVolumeClaim{} // check pvc existence if err := cli.Get(ctx, pvcKey, &pvc); err != nil { - return false, err + return false, client.IgnoreNotFound(err) } if pvc.Status.Phase != corev1.ClaimBound { return false, nil @@ -530,7 +565,7 @@ func isAllPVCBound(cli types2.ReadonlyClient, return true, nil } -func deleteSnapshot(cli types2.ReadonlyClient, +func deleteSnapshot(cli roclient.ReadonlyClient, reqCtx intctrlutil.RequestCtx, snapshotKey types.NamespacedName, cluster *appsv1alpha1.Cluster, @@ -554,41 +589,18 @@ func deleteSnapshot(cli types2.ReadonlyClient, } // deleteBackup will delete all backup related resources created during horizontal scaling, -func deleteBackup(ctx context.Context, cli types2.ReadonlyClient, clusterName string, componentName string, dag *graph.DAG, root graph.Vertex) error { - +func deleteBackup(ctx context.Context, cli roclient.ReadonlyClient, clusterName string, componentName string, dag *graph.DAG, root graph.Vertex) error { ml := getBackupMatchingLabels(clusterName, componentName) - - deleteBackupPolicy := func() error { - backupPolicyList := dataprotectionv1alpha1.BackupPolicyList{} - if err := cli.List(ctx, &backupPolicyList, ml); err != nil { - return err - } - for _, backupPolicy := range backupPolicyList.Items { - vertex := &lifecycleVertex{obj: &backupPolicy, oriObj: &backupPolicy, action: actionPtr(DELETE)} - dag.AddVertex(vertex) - dag.Connect(root, vertex) - } - return nil - } - - deleteRelatedBackups := func() error { - backupList := dataprotectionv1alpha1.BackupList{} - if err := cli.List(ctx, &backupList, ml); err != nil { - return err - } - for _, backup := range backupList.Items { - vertex := &lifecycleVertex{obj: &backup, oriObj: &backup, action: actionPtr(DELETE)} - dag.AddVertex(vertex) - dag.Connect(root, vertex) - } - return nil - } - - if err := deleteBackupPolicy(); err != nil { + backupList := dataprotectionv1alpha1.BackupList{} + if err := cli.List(ctx, &backupList, ml); err != nil { return err } - - return deleteRelatedBackups() + for _, backup := range backupList.Items { + vertex := &lifecycleVertex{obj: &backup, oriObj: &backup, action: actionPtr(DELETE)} + dag.AddVertex(vertex) + dag.Connect(root, vertex) + } + return nil } func getBackupMatchingLabels(clusterName string, componentName string) client.MatchingLabels { @@ -600,7 +612,7 @@ func getBackupMatchingLabels(clusterName string, componentName string) client.Ma } // check snapshot existence -func isVolumeSnapshotExists(cli types2.ReadonlyClient, +func isVolumeSnapshotExists(cli roclient.ReadonlyClient, ctx context.Context, cluster *appsv1alpha1.Cluster, component *component.SynthesizedComponent) (bool, error) { @@ -619,7 +631,7 @@ func isVolumeSnapshotExists(cli types2.ReadonlyClient, return false, nil } -func doSnapshot(cli types2.ReadonlyClient, +func doSnapshot(cli roclient.ReadonlyClient, reqCtx intctrlutil.RequestCtx, cluster *appsv1alpha1.Cluster, snapshotKey types.NamespacedName, @@ -657,18 +669,13 @@ func doSnapshot(cli types2.ReadonlyClient, dag.AddVertex(vertex) dag.Connect(root, vertex) - scheme, _ := appsv1alpha1.SchemeBuilder.Build() - // TODO: SetOwnership - if err := controllerutil.SetControllerReference(cluster, snapshot, scheme); err != nil { - return err - } reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "VolumeSnapshotCreate", "Create volumesnapshot/%s", snapshotKey.Name) } return nil } // check snapshot ready to use -func isVolumeSnapshotReadyToUse(cli types2.ReadonlyClient, +func isVolumeSnapshotReadyToUse(cli roclient.ReadonlyClient, ctx context.Context, cluster *appsv1alpha1.Cluster, component *component.SynthesizedComponent) (bool, error) { @@ -690,7 +697,7 @@ func isVolumeSnapshotReadyToUse(cli types2.ReadonlyClient, return *status.ReadyToUse, nil } -func checkedCreatePVCFromSnapshot(cli types2.ReadonlyClient, +func checkedCreatePVCFromSnapshot(cli roclient.ReadonlyClient, ctx context.Context, pvcKey types.NamespacedName, cluster *appsv1alpha1.Cluster, @@ -711,7 +718,7 @@ func checkedCreatePVCFromSnapshot(cli types2.ReadonlyClient, return err } if len(vsList.Items) == 0 { - return errors.Errorf("volumesnapshot not found in cluster %s component %s", cluster.Name, component.Name) + return fmt.Errorf("volumesnapshot not found in cluster %s component %s", cluster.Name, component.Name) } // exclude volumes that are deleting vsName := "" @@ -729,7 +736,7 @@ func checkedCreatePVCFromSnapshot(cli types2.ReadonlyClient, // createBackup create backup resources required to do backup, func createBackup(reqCtx intctrlutil.RequestCtx, - cli types2.ReadonlyClient, + cli roclient.ReadonlyClient, sts *appsv1.StatefulSet, componentDef, backupPolicyTemplateName string, @@ -758,7 +765,7 @@ func createBackup(reqCtx intctrlutil.RequestCtx, if backupList.Items[0].Status.Phase == dataprotectionv1alpha1.BackupFailed { reqCtx.Recorder.Eventf(cluster, corev1.EventTypeWarning, "HorizontalScaleFailed", "backup %s status failed", backupKey.Name) - return errors.Errorf("cluster %s h-scale failed, backup error: %s", + return fmt.Errorf("cluster %s h-scale failed, backup error: %s", cluster.Name, backupList.Items[0].Status.FailureReason) } return nil @@ -811,3 +818,5 @@ func createPVCFromSnapshot(vct corev1.PersistentVolumeClaimTemplate, dag.Connect(root, vertex) return nil } + +var _ graph.Transformer = &StsHorizontalScalingTransformer{} diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go index a2439b062..09e8fbb11 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" "github.com/apecloud/kubeblocks/internal/controller/graph" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" @@ -72,14 +73,18 @@ var _ = Describe("sts horizontal scaling test", func() { Expect(intctrlutil.SetOwnership(cluster, pvc2, scheme, dbClusterFinalizerName)).Should(Succeed()) By("prepare params for transformer") - reqCtx := intctrlutil.RequestCtx{ - Ctx: context.Background(), - } ctrl, k8sMock := testutil.SetupK8sMock() defer ctrl.Finish() - cr := clusterRefResources{cd: *cd, cv: *cv} - - transformer := &stsHorizontalScalingTransformer{ctx: reqCtx, cli: k8sMock, cr: cr} + ctx := context.Background() + transCtx := &ClusterTransformContext{ + Context: ctx, + Client: k8sMock, + Logger: log.FromContext(ctx).WithValues("transformer", "h-scale"), + ClusterDef: cd, + ClusterVer: cv, + Cluster: cluster, + OrigCluster: cluster.DeepCopy(), + } By("prepare initial DAG with sts.action=UPDATE") dag := graph.NewDAG() @@ -100,15 +105,17 @@ var _ = Describe("sts horizontal scaling test", func() { return nil }).AnyTimes() + transformer := &StsHorizontalScalingTransformer{} + By("do transform") - Expect(transformer.Transform(dag)).Should(Succeed()) + Expect(transformer.Transform(transCtx, dag)).Should(Succeed()) Expect(len(findAll[*corev1.PersistentVolumeClaim](dag))).Should(Equal(0)) By("prepare initial DAG with sts.action=DELETE") stsVertex.action = actionPtr(DELETE) By("do transform") - Expect(transformer.Transform(dag)).Should(Succeed()) + Expect(transformer.Transform(transCtx, dag)).Should(Succeed()) Expect(len(findAll[*corev1.PersistentVolumeClaim](dag))).Should(Equal(2)) }) }) diff --git a/internal/controller/lifecycle/transformer_sts_pvc.go b/internal/controller/lifecycle/transformer_sts_pvc.go index 0385c62d6..939017369 100644 --- a/internal/controller/lifecycle/transformer_sts_pvc.go +++ b/internal/controller/lifecycle/transformer_sts_pvc.go @@ -23,23 +23,14 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - types2 "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/graph" - intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -type stsPVCTransformer struct { - cli types2.ReadonlyClient - ctx intctrlutil.RequestCtx -} +type StsPVCTransformer struct{} -func (s *stsPVCTransformer) Transform(dag *graph.DAG) error { - rootVertex, err := findRootVertex(dag) - if err != nil { - return err - } - origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) +func (t *StsPVCTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + origCluster := transCtx.OrigCluster if isClusterDeleting(*origCluster) { return nil @@ -75,7 +66,7 @@ func (s *stsPVCTransformer) Transform(dag *graph.DAG) error { Namespace: stsObj.Namespace, Name: fmt.Sprintf("%s-%s-%d", vct.Name, stsObj.Name, i), } - if err := s.cli.Get(s.ctx.Ctx, pvcKey, pvc); err != nil { + if err := transCtx.Client.Get(transCtx.Context, pvcKey, pvc); err != nil { return err } obj := pvc.DeepCopy() @@ -103,3 +94,5 @@ func (s *stsPVCTransformer) Transform(dag *graph.DAG) error { } return nil } + +var _ graph.Transformer = &StsPVCTransformer{} diff --git a/internal/controller/lifecycle/transformer_tls_certs.go b/internal/controller/lifecycle/transformer_tls_certs.go index c6a1adc92..41fc71f32 100644 --- a/internal/controller/lifecycle/transformer_tls_certs.go +++ b/internal/controller/lifecycle/transformer_tls_certs.go @@ -23,25 +23,17 @@ import ( corev1 "k8s.io/api/core/v1" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/graph" "github.com/apecloud/kubeblocks/internal/controller/plan" - intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -type tlsCertsTransformer struct { - cr clusterRefResources - cli client.ReadonlyClient - ctx intctrlutil.RequestCtx -} +// TLSCertsTransformer handles tls certs provisioning or validation +type TLSCertsTransformer struct{} -func (t *tlsCertsTransformer) Transform(dag *graph.DAG) error { - rootVertex, err := findRootVertex(dag) - if err != nil { - return err - } - origCluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) - cluster, _ := rootVertex.obj.(*appsv1alpha1.Cluster) +func (t *TLSCertsTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx := ctx.(*ClusterTransformContext) + origCluster := transCtx.OrigCluster + cluster := transCtx.Cluster // return fast when cluster is deleting if isClusterDeleting(*origCluster) { return nil @@ -58,7 +50,7 @@ func (t *tlsCertsTransformer) Transform(dag *graph.DAG) error { switch comp.Issuer.Name { case appsv1alpha1.IssuerUserProvided: - if err := plan.CheckTLSSecretRef(t.ctx, t.cli, cluster.Namespace, comp.Issuer.SecretRef); err != nil { + if err := plan.CheckTLSSecretRef(transCtx.Context, transCtx.Client, cluster.Namespace, comp.Issuer.SecretRef); err != nil { return err } case appsv1alpha1.IssuerKubeBlocks: @@ -82,3 +74,5 @@ func (t *tlsCertsTransformer) Transform(dag *graph.DAG) error { return nil } + +var _ graph.Transformer = &TLSCertsTransformer{} diff --git a/internal/controller/lifecycle/transformer_validate_and_load_ref_resources.go b/internal/controller/lifecycle/transformer_validate_and_load_ref_resources.go new file mode 100644 index 000000000..045fda8a8 --- /dev/null +++ b/internal/controller/lifecycle/transformer_validate_and_load_ref_resources.go @@ -0,0 +1,88 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "errors" + "fmt" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/controller/graph" +) + +// ValidateAndLoadRefResourcesTransformer handles referenced resources'(cd & cv) validation and load them into context +type ValidateAndLoadRefResourcesTransformer struct{} + +func (t *ValidateAndLoadRefResourcesTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + cluster := transCtx.Cluster + if isClusterDeleting(*cluster) { + return nil + } + + var err error + defer func() { + setProvisioningStartedCondition(&cluster.Status.Conditions, cluster.Name, cluster.Generation, err) + }() + + validateExistence := func(key client.ObjectKey, object client.Object) error { + err = transCtx.Client.Get(transCtx.Context, key, object) + if err != nil { + return newRequeueError(requeueDuration, err.Error()) + } + return nil + } + + // validate cd & cv's existence + // if we can't get the referenced cd & cv, set provisioning condition failed, and jump to plan.Execute() + cd := &appsv1alpha1.ClusterDefinition{} + if err = validateExistence(types.NamespacedName{Name: cluster.Spec.ClusterDefRef}, cd); err != nil { + return err + } + var cv *appsv1alpha1.ClusterVersion + if len(cluster.Spec.ClusterVersionRef) > 0 { + cv = &appsv1alpha1.ClusterVersion{} + if err = validateExistence(types.NamespacedName{Name: cluster.Spec.ClusterVersionRef}, cv); err != nil { + return err + } + } + + // validate cd & cv's availability + // if wrong phase, set provisioning condition failed, and jump to plan.Execute() + if cd.Status.Phase != appsv1alpha1.AvailablePhase || (cv != nil && cv.Status.Phase != appsv1alpha1.AvailablePhase) { + message := fmt.Sprintf("ref resource is unavailable, this problem needs to be solved first. cd: %s", cd.Name) + if cv != nil { + message = fmt.Sprintf("%s, cv: %s", message, cv.Name) + } + err = errors.New(message) + return newRequeueError(requeueDuration, message) + } + + // inject cd & cv into the shared ctx + transCtx.ClusterDef = cd + transCtx.ClusterVer = cv + if cv == nil { + transCtx.ClusterVer = &appsv1alpha1.ClusterVersion{} + } + + return nil +} + +var _ graph.Transformer = &ValidateAndLoadRefResourcesTransformer{} diff --git a/internal/controller/lifecycle/transformer_validate_enable_logs.go b/internal/controller/lifecycle/transformer_validate_enable_logs.go new file mode 100644 index 000000000..dc137d1bc --- /dev/null +++ b/internal/controller/lifecycle/transformer_validate_enable_logs.go @@ -0,0 +1,43 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "github.com/apecloud/kubeblocks/internal/controller/graph" +) + +// ValidateEnableLogsTransformer validate config and send warning event log necessarily +type ValidateEnableLogsTransformer struct{} + +func (e *ValidateEnableLogsTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + cluster := transCtx.Cluster + if isClusterDeleting(*cluster) { + return nil + } + + // validate config and send warning event log necessarily + err := cluster.Spec.ValidateEnabledLogs(transCtx.ClusterDef) + setProvisioningStartedCondition(&cluster.Status.Conditions, cluster.Name, cluster.Generation, err) + if err != nil { + return newRequeueError(requeueDuration, err.Error()) + } + + return nil +} + +var _ graph.Transformer = &ValidateEnableLogsTransformer{} diff --git a/internal/controller/lifecycle/transformer_workloads_last.go b/internal/controller/lifecycle/transformer_workloads_last.go new file mode 100644 index 000000000..6910bfb92 --- /dev/null +++ b/internal/controller/lifecycle/transformer_workloads_last.go @@ -0,0 +1,52 @@ +/* +Copyright ApeCloud, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lifecycle + +import ( + "github.com/apecloud/kubeblocks/internal/constant" + appv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/sets" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/controller/graph" +) + +// WorkloadsLastTransformer have workload objects placed last +type WorkloadsLastTransformer struct{} + +func (c *WorkloadsLastTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + workloadKinds := sets.New(constant.StatefulSetKind, constant.DeploymentKind) + var workloadsVertices, noneRootVertices []graph.Vertex + + workloadsVertices = findAll[*appv1.StatefulSet](dag) + workloadsVertices = append(workloadsVertices, findAll[*appv1.Deployment](dag)) + noneRootVertices = findAllNot[*appsv1alpha1.Cluster](dag) + + for _, workloadV := range workloadsVertices { + workload, _ := workloadV.(*lifecycleVertex) + for _, vertex := range noneRootVertices { + v, _ := vertex.(*lifecycleVertex) + // connect all workloads vertices to all none workloads vertices + if !workloadKinds.Has(v.obj.GetObjectKind().GroupVersionKind().Kind) { + dag.Connect(workload, vertex) + } + } + } + return nil +} + +var _ graph.Transformer = &WorkloadsLastTransformer{} diff --git a/internal/controller/lifecycle/transformer_do_not_terminate.go b/internal/controller/lifecycle/transformers_parallel.go similarity index 52% rename from internal/controller/lifecycle/transformer_do_not_terminate.go rename to internal/controller/lifecycle/transformers_parallel.go index d68a7ddcd..8b2487eac 100644 --- a/internal/controller/lifecycle/transformer_do_not_terminate.go +++ b/internal/controller/lifecycle/transformers_parallel.go @@ -17,29 +17,33 @@ limitations under the License. package lifecycle import ( - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "fmt" + "sync" + "github.com/apecloud/kubeblocks/internal/controller/graph" ) -type doNotTerminateTransformer struct{} - -func (d *doNotTerminateTransformer) Transform(dag *graph.DAG) error { - rootVertex, err := findRootVertex(dag) - if err != nil { - return err - } - cluster, _ := rootVertex.oriObj.(*appsv1alpha1.Cluster) +type ParallelTransformers struct { + transformers []graph.Transformer +} - if cluster.DeletionTimestamp.IsZero() { - return nil +func (t *ParallelTransformers) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + var group sync.WaitGroup + var errs error + for _, transformer := range t.transformers { + transformer := transformer + group.Add(1) + go func() { + err := transformer.Transform(ctx, dag) + if err != nil { + // TODO: sync.Mutex errs + errs = fmt.Errorf("%v; %v", errs, err) + } + group.Done() + }() } - if cluster.Spec.TerminationPolicy != appsv1alpha1.DoNotTerminate { - return nil - } - vertices := findAllNot[*appsv1alpha1.Cluster](dag) - for _, vertex := range vertices { - v, _ := vertex.(*lifecycleVertex) - v.immutable = true - } - return nil + group.Wait() + return errs } + +var _ graph.Transformer = &ParallelTransformers{} diff --git a/internal/controller/lifecycle/validator_enable_logs.go b/internal/controller/lifecycle/validator_enable_logs.go deleted file mode 100644 index 52c59f50f..000000000 --- a/internal/controller/lifecycle/validator_enable_logs.go +++ /dev/null @@ -1,31 +0,0 @@ -/* -Copyright ApeCloud, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package lifecycle - -import ( - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" -) - -type enableLogsValidator struct { - cluster *appsv1alpha1.Cluster - clusterDef *appsv1alpha1.ClusterDefinition -} - -func (e *enableLogsValidator) Validate() error { - // validate config and send warning event log necessarily - return e.cluster.Spec.ValidateEnabledLogs(e.clusterDef) -} diff --git a/internal/controller/plan/tls_utils.go b/internal/controller/plan/tls_utils.go index f76bfd068..279e5dd93 100644 --- a/internal/controller/plan/tls_utils.go +++ b/internal/controller/plan/tls_utils.go @@ -18,6 +18,7 @@ package plan import ( "bytes" + "context" "strings" "text/template" @@ -26,64 +27,13 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/controller-runtime/pkg/client" dbaasv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - componentutil "github.com/apecloud/kubeblocks/controllers/apps/components/util" "github.com/apecloud/kubeblocks/internal/controller/builder" client2 "github.com/apecloud/kubeblocks/internal/controller/client" "github.com/apecloud/kubeblocks/internal/controller/component" - "github.com/apecloud/kubeblocks/internal/controllerutil" ) -func CreateOrCheckTLSCerts(reqCtx controllerutil.RequestCtx, - cli client.Client, - cluster *dbaasv1alpha1.Cluster, -) (*v1.Secret, error) { - if cluster == nil { - return nil, componentutil.ErrReqClusterObj - } - - for _, comp := range cluster.Spec.ComponentSpecs { - if !comp.TLS { - continue - } - // REVIEW/TODO: should do spec validation during validation stage - if comp.Issuer == nil { - return nil, errors.New("issuer shouldn't be nil when tls enabled") - } - switch comp.Issuer.Name { - case dbaasv1alpha1.IssuerUserProvided: - if err := CheckTLSSecretRef(reqCtx, cli, cluster.Namespace, comp.Issuer.SecretRef); err != nil { - return nil, err - } - case dbaasv1alpha1.IssuerKubeBlocks: - return createTLSSecret(reqCtx, cli, cluster, comp.Name) - } - } - return nil, nil -} - -// func deleteTLSSecrets(reqCtx controllerutil.RequestCtx, cli client.Client, secretList []v1.Secret) { -// for _, secret := range secretList { -// err := cli.Delete(reqCtx.Ctx, &secret) -// if err != nil { -// reqCtx.Log.Info("delete tls secret error", "err", err) -// } -// } -// } - -func createTLSSecret(reqCtx controllerutil.RequestCtx, - cli client.Client, - cluster *dbaasv1alpha1.Cluster, - componentName string) (*v1.Secret, error) { - secret, err := ComposeTLSSecret(cluster.Namespace, cluster.Name, componentName) - if err != nil { - return nil, err - } - return secret, nil -} - // ComposeTLSSecret compose a TSL secret object. // REVIEW/TODO: // 1. missing public function doc @@ -131,14 +81,14 @@ func buildFromTemplate(tpl string, vars interface{}) (string, error) { return b.String(), nil } -func CheckTLSSecretRef(reqCtx controllerutil.RequestCtx, cli client2.ReadonlyClient, namespace string, +func CheckTLSSecretRef(ctx context.Context, cli client2.ReadonlyClient, namespace string, secretRef *dbaasv1alpha1.TLSSecretRef) error { if secretRef == nil { return errors.New("issuer.secretRef shouldn't be nil when issuer is UserProvided") } secret := &v1.Secret{} - if err := cli.Get(reqCtx.Ctx, types.NamespacedName{Namespace: namespace, Name: secretRef.Name}, secret); err != nil { + if err := cli.Get(ctx, types.NamespacedName{Namespace: namespace, Name: secretRef.Name}, secret); err != nil { return err } if secret.Data == nil { From 99b0f672d14a2f2ef89ed2e8059f55972dfcebc5 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sun, 23 Apr 2023 12:32:09 +0800 Subject: [PATCH 139/439] chore: tidy up *_releasing_pr.sh scripts (#2848) --- .github/utils/create_releasing_pr.sh | 5 +++-- .github/utils/merge_releasing_pr.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/utils/create_releasing_pr.sh b/.github/utils/create_releasing_pr.sh index 4d17de757..1e7a44def 100755 --- a/.github/utils/create_releasing_pr.sh +++ b/.github/utils/create_releasing_pr.sh @@ -13,9 +13,10 @@ workdir=$(dirname $0) set -x git stash -git switch ${BASE_BRANCH} +git switch ${HEAD_BRANCH} +git pull +git merge origin/${BASE_BRANCH} git pull -git merge origin/${HEAD_BRANCH} echo "Creating ${PR_TITLE}" gh pr create --head ${HEAD_BRANCH} --base ${BASE_BRANCH} --title "${PR_TITLE}" --body "" --label "releasing-task" \ No newline at end of file diff --git a/.github/utils/merge_releasing_pr.sh b/.github/utils/merge_releasing_pr.sh index a07446985..d75dd55f2 100755 --- a/.github/utils/merge_releasing_pr.sh +++ b/.github/utils/merge_releasing_pr.sh @@ -24,7 +24,7 @@ get_pr_status() { echo "Merging ${PR_TITLE}" retry_times=0 -pr_info=$(gh pr list --repo ${OWNER}/${REPO} --base ${BASE_BRANCH} --json "number" ) +pr_info=$(gh pr list --repo ${OWNER}/${REPO} --head ${HEAD_BRANCH} --base ${BASE_BRANCH} --json "number" ) pr_len=$(echo ${pr_info} | jq -r '. | length') if [ "${pr_len}" == "0" ]; then exit 0 From 8b07df43da6b836862293b5afb306ae261f8b494 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sun, 23 Apr 2023 13:09:43 +0800 Subject: [PATCH 140/439] chore: update releasing PR title to chore(release) (#2854) --- .github/utils/create_releasing_pr.sh | 3 ++- .github/utils/gh_env | 2 +- .github/utils/merge_releasing_pr.sh | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/utils/create_releasing_pr.sh b/.github/utils/create_releasing_pr.sh index 1e7a44def..bc7c1a9f6 100755 --- a/.github/utils/create_releasing_pr.sh +++ b/.github/utils/create_releasing_pr.sh @@ -15,8 +15,9 @@ set -x git stash git switch ${HEAD_BRANCH} git pull -git merge origin/${BASE_BRANCH} +git rebase origin/${BASE_BRANCH} git pull +git push echo "Creating ${PR_TITLE}" gh pr create --head ${HEAD_BRANCH} --base ${BASE_BRANCH} --title "${PR_TITLE}" --body "" --label "releasing-task" \ No newline at end of file diff --git a/.github/utils/gh_env b/.github/utils/gh_env index 683cc28f7..18b51b01f 100644 --- a/.github/utils/gh_env +++ b/.github/utils/gh_env @@ -4,7 +4,7 @@ export MILESTONE_ID=${MILESTONE_ID:-5} export BASE_BRANCH=${BASE_BRANCH:-'release-0.5'} export HEAD_BRANCH=${HEAD_BRANCH:-'releasing-0.5'} -export PR_TITLE=${PR_TITLE:-"${BASE_BRANCH} tracker PR (no-need-to-review)"} +export PR_TITLE=${PR_TITLE:-"chore(releasing): ${BASE_BRANCH} tracker PR (no-need-to-review)"} export REMOTE_URL=$(git config --get remote.origin.url) diff --git a/.github/utils/merge_releasing_pr.sh b/.github/utils/merge_releasing_pr.sh index d75dd55f2..5e713be99 100755 --- a/.github/utils/merge_releasing_pr.sh +++ b/.github/utils/merge_releasing_pr.sh @@ -43,7 +43,7 @@ if [ "${pr_mergeable}" == "MERGEABLE" ]; then if [ "${pr_merge_status}" == "UNSTABLE" ]; then retry_times=100 - while [ $retry_times -gt 0 ] + while [ $retry_times -gt 0 ] && [ "${pr_merge_status}" == "UNSTABLE" ] do ((retry_times--)) sleep 5 @@ -53,6 +53,7 @@ if [ "${pr_mergeable}" == "MERGEABLE" ]; then if [ "${pr_merge_status}" == "CLEAN" ]; then echo "Merging PR #${pr_number}" + set -x gh pr --repo ${OWNER}/${REPO} merge ${pr_number} --rebase exit 0 fi From c1a9f9bfff875ba04aa12ba74d8f76169106f614 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Sun, 23 Apr 2023 13:24:30 +0800 Subject: [PATCH 141/439] fix: move kbcli volumesnapshotclass creation into snapshot-controller (#2844) --- .../addons/snapshot-controller-addon.yaml | 33 +++++++++- deploy/helm/values.yaml | 7 +++ internal/cli/cmd/cluster/dataprotection.go | 62 ------------------- internal/cli/cmd/kubeblocks/install.go | 46 -------------- internal/cli/cmd/kubeblocks/install_test.go | 18 ------ internal/cli/cmd/kubeblocks/upgrade.go | 5 -- 6 files changed, 38 insertions(+), 133 deletions(-) diff --git a/deploy/helm/templates/addons/snapshot-controller-addon.yaml b/deploy/helm/templates/addons/snapshot-controller-addon.yaml index 04a734ed9..ba4be98ee 100644 --- a/deploy/helm/templates/addons/snapshot-controller-addon.yaml +++ b/deploy/helm/templates/addons/snapshot-controller-addon.yaml @@ -26,6 +26,7 @@ spec: valuesMapping: valueMap: replicaCount: replicaCount + storageClass: volumeSnapshotClasses[0].driver jsonMap: tolerations: tolerations @@ -39,8 +40,36 @@ spec: limits: resources.limits.memory defaultInstallValues: - - replicas: 1 + - enabled: {{ get ( get ( .Values | toYaml | fromYaml ) "snapshot-controller" ) "enabled" }} + - storageClass: ebs.csi.aws.com + selectors: + - key: KubeGitVersion + operator: Contains + values: + - eks + - storageClass: diskplugin.csi.alibabacloud.com + selectors: + - key: KubeGitVersion + operator: Contains + values: + - aliyun + - storageClass: pd.csi.storage.gke.io + selectors: + - key: KubeGitVersion + operator: Contains + values: + - gke + - storageClass: disk.csi.azure.com + selectors: + - key: KubeGitVersion + operator: Contains + values: + - aks installable: autoInstall: {{ get ( get ( .Values | toYaml | fromYaml ) "snapshot-controller" ) "enabled" }} - + selectors: + - key: KubeGitVersion + operator: DoesNotContain + values: + - tke \ No newline at end of file diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 3f70d6fa1..d96d67d10 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1625,6 +1625,13 @@ snapshot-controller: value: "true" effect: NoSchedule + volumeSnapshotClasses: + - name: default-vsc + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + driver: hostpath.csi.k8s.io + deletionPolicy: Delete + affinity: nodeAffinity: preferredDuringSchedulingIgnoredDuringExecution: diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 400a74b4c..eae902072 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -103,73 +103,11 @@ type CreateBackupOptions struct { create.BaseOptions } -type CreateVolumeSnapshotClassOptions struct { - Driver string `json:"driver"` - Name string `json:"name"` - create.BaseOptions -} - type ListBackupOptions struct { *list.ListOptions BackupName string } -func (o *CreateVolumeSnapshotClassOptions) Complete() error { - objs, err := o.Dynamic. - Resource(types.StorageClassGVR()). - List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - for _, sc := range objs.Items { - annotations := sc.GetAnnotations() - if annotations == nil { - continue - } - if annotations["storageclass.kubernetes.io/is-default-class"] == annotationTrueValue { - o.Driver, _, _ = unstructured.NestedString(sc.Object, "provisioner") - o.Name = "default-vsc" - } - } - // warning if not found default storage class - if o.Driver == "" { - return fmt.Errorf("no default StorageClass found, snapshot-controller may not work") - } - return nil -} - -func (o *CreateVolumeSnapshotClassOptions) Create() error { - objs, err := o.Dynamic. - Resource(types.VolumeSnapshotClassGVR()). - List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return err - } - for _, vsc := range objs.Items { - annotations := vsc.GetAnnotations() - if annotations == nil { - continue - } - // skip creation if default volumesnapshotclass exists. - if annotations["snapshot.storage.kubernetes.io/is-default-class"] == annotationTrueValue { - return nil - } - } - - inputs := create.Inputs{ - CueTemplateName: "volumesnapshotclass_template.cue", - ResourceName: "volumesnapshotclasses", - Group: "snapshot.storage.k8s.io", - Version: types.K8sCoreAPIVersion, - BaseOptionsObj: &o.BaseOptions, - Options: o, - } - if err := o.BaseOptions.Run(inputs); err != nil { - return err - } - return nil -} - func (o *CreateBackupOptions) Complete() error { // generate backupName if len(o.BackupName) == 0 { diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index 0ea596d79..6362fe1c4 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -42,7 +42,6 @@ import ( "k8s.io/kubectl/pkg/util/templates" extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" - "github.com/apecloud/kubeblocks/internal/cli/cmd/cluster" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" @@ -206,11 +205,6 @@ func (o *InstallOptions) Install() error { return err } - // create VolumeSnapshotClass - if err = o.createVolumeSnapshotClass(); err != nil { - return err - } - if !o.Quiet { fmt.Fprintf(o.Out, "\nKubeBlocks %s installed to namespace %s SUCCESSFULLY!\n", o.Version, o.HelmCfg.Namespace()) @@ -463,46 +457,6 @@ Note: Monitoring add-ons are not installed. } } -func (o *InstallOptions) createVolumeSnapshotClass() error { - createFunc := func() error { - options := cluster.CreateVolumeSnapshotClassOptions{} - options.BaseOptions.Dynamic = o.Dynamic - options.BaseOptions.IOStreams = o.IOStreams - options.BaseOptions.Quiet = true - - spinner := printer.Spinner(o.Out, "%-50s", "Configure VolumeSnapshotClass") - defer spinner(false) - - if err := options.Complete(); err != nil { - return err - } - if err := options.Create(); err != nil { - return err - } - spinner(true) - return nil - } - - var sets []string - for _, set := range o.ValueOpts.Values { - splitSet := strings.Split(set, ",") - sets = append(sets, splitSet...) - } - for _, set := range sets { - if set != "snapshot-controller.enabled=true" { - continue - } - - if err := createFunc(); err != nil { - return err - } else { - // only need to create once - return nil - } - } - return nil -} - func (o *InstallOptions) buildChart() *helm.InstallOpts { return &helm.InstallOpts{ Name: types.KubeBlocksChartName, diff --git a/internal/cli/cmd/kubeblocks/install_test.go b/internal/cli/cmd/kubeblocks/install_test.go index 826176431..9135080d3 100644 --- a/internal/cli/cmd/kubeblocks/install_test.go +++ b/internal/cli/cmd/kubeblocks/install_test.go @@ -23,7 +23,6 @@ import ( . "github.com/onsi/gomega" "github.com/spf13/cobra" - "helm.sh/helm/v3/pkg/cli/values" "k8s.io/cli-runtime/pkg/genericclioptions" clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" @@ -94,23 +93,6 @@ var _ = Describe("kubeblocks install", func() { o.printNotes() }) - It("create volumeSnapshotClass", func() { - o := &InstallOptions{ - Options: Options{ - IOStreams: streams, - HelmCfg: helm.NewFakeConfig(namespace), - Namespace: "default", - Client: testing.FakeClientSet(), - Dynamic: testing.FakeDynamicClient(testing.FakeVolumeSnapshotClass()), - }, - Version: version.DefaultKubeBlocksVersion, - Monitor: true, - CreateNamespace: true, - ValueOpts: values.Options{Values: []string{"snapshot-controller.enabled=true"}}, - } - Expect(o.createVolumeSnapshotClass()).Should(HaveOccurred()) - }) - It("preCheck", func() { o := &InstallOptions{ Options: Options{ diff --git a/internal/cli/cmd/kubeblocks/upgrade.go b/internal/cli/cmd/kubeblocks/upgrade.go index 755c73dd7..534d44317 100644 --- a/internal/cli/cmd/kubeblocks/upgrade.go +++ b/internal/cli/cmd/kubeblocks/upgrade.go @@ -130,11 +130,6 @@ func (o *InstallOptions) Upgrade() error { // successfully upgraded spinner(true) - // create VolumeSnapshotClass - if err = o.createVolumeSnapshotClass(); err != nil { - return err - } - if !o.Quiet { fmt.Fprintf(o.Out, "\nKubeBlocks has been upgraded %s SUCCESSFULLY!\n", msg) // set monitor to true, so that we can print notes with monitor From 066fcac499ec9b0ebafd684fa759c16c852455a8 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Sun, 23 Apr 2023 13:50:17 +0800 Subject: [PATCH 142/439] chore: refactor backup error (#2858) --- .../dataprotection/backup_controller.go | 30 ++++++------- internal/controllerutil/errors.go | 43 ++++++++++++++++--- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 272de3660..970d763fc 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -52,11 +52,6 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) -var ( - errJobFailed = errors.New("job failed") - errDeletingBackupFiles = errors.New("deleting backup files") -) - const ( backupPathBase = "/backupdata" deleteBackupFilesJobNamePrefix = "delete-backup-files-" @@ -101,11 +96,7 @@ func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr // handle finalizer res, err := intctrlutil.HandleCRDeletion(reqCtx, r, backup, dataProtectionFinalizerName, func() (*ctrl.Result, error) { - err := r.deleteExternalResources(reqCtx, backup) - if errors.Is(err, errDeletingBackupFiles) { - return intctrlutil.ResultToP(intctrlutil.Requeue(reqCtx.Log, "deleting backup files")) - } - return nil, err + return nil, r.deleteExternalResources(reqCtx, backup) }) if res != nil { return *res, err @@ -155,7 +146,7 @@ func (r *BackupReconciler) getBackupPolicyAndValidate( } if len(backupPolicy.Name) == 0 { - return nil, fmt.Errorf("backup policy %s not found", backupPolicyNameSpaceName) + return nil, intctrlutil.NewNotFound(`backup policy "%s" not found`, backupPolicyNameSpaceName) } // validate backup spec @@ -193,7 +184,7 @@ func (r *BackupReconciler) doNewPhaseAction( default: commonPolicy := backupPolicy.Spec.GetCommonPolicy(backup.Spec.BackupType) if commonPolicy == nil { - return r.updateStatusIfFailed(reqCtx, backup, intctrlutil.NewNotFound(`backup type "%s" not supported in the backupPolicy "%s"`, backup.Spec.BackupType, backupPolicy.Name)) + return r.updateStatusIfFailed(reqCtx, backup, intctrlutil.NewBackupNotSupported(string(backup.Spec.BackupType), backupPolicy.Name)) } // save the backup message for restore backup.Status.PersistentVolumeClaimName = commonPolicy.PersistentVolumeClaim.Name @@ -238,7 +229,7 @@ func (r *BackupReconciler) handlePersistentVolumeClaim(reqCtx intctrlutil.Reques commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy) error { pvcConfig := commonPolicy.PersistentVolumeClaim if len(pvcConfig.Name) == 0 { - return fmt.Errorf("the persistentVolumeClaim name of this policy is empty") + return intctrlutil.NewBackupPVCNameIsEmpty(backupPolicyName) } pvc := &corev1.PersistentVolumeClaim{} if err := r.Client.Get(reqCtx.Ctx, client.ObjectKey{Namespace: reqCtx.Req.Namespace, @@ -302,7 +293,7 @@ func (r *BackupReconciler) createPersistentVolumeWithTemplate(reqCtx intctrlutil } pvTemplate := configMap.Data[persistentVolumeTemplateKey] if pvTemplate == "" { - return intctrlutil.NewNotFound("the persistentVolume template is empty in the configMap %s/%s", pvConfig.Namespace, pvConfig.Name) + return intctrlutil.NewBackupPVTemplateNotFound(pvConfig.Namespace, pvConfig.Name) } pvName := fmt.Sprintf("%s-%s", pvcConfig.Name, reqCtx.Req.Namespace) pvTemplate = strings.ReplaceAll(pvTemplate, "$(GENERATE_NAME)", pvName) @@ -481,8 +472,13 @@ func (r *BackupReconciler) doCompletedPhaseAction( func (r *BackupReconciler) updateStatusIfFailed(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup, err error) (ctrl.Result, error) { patch := client.MergeFrom(backup.DeepCopy()) - r.Recorder.Eventf(backup, corev1.EventTypeWarning, "FailedCreatedBackup", - "Failed creating backup, error: %s", err.Error()) + controllerErr := intctrlutil.ToControllerError(err) + if controllerErr != nil { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, string(controllerErr.Type), err.Error()) + } else { + r.Recorder.Eventf(backup, corev1.EventTypeWarning, "FailedCreatedBackup", + "Failed creating backup, error: %s", err.Error()) + } backup.Status.Phase = dataprotectionv1alpha1.BackupFailed backup.Status.FailureReason = err.Error() if errUpdate := r.Client.Status().Patch(reqCtx.Ctx, backup, patch); errUpdate != nil { @@ -575,7 +571,7 @@ func (r *BackupReconciler) ensureBatchV1JobCompleted( if jobStatusConditions[0].Type == batchv1.JobComplete { return true, nil } else if jobStatusConditions[0].Type == batchv1.JobFailed { - return false, errJobFailed + return false, intctrlutil.NewBackupJobFailed(job.Name) } } } diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index c23f4c583..091a16714 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -38,13 +38,16 @@ type ErrorType string const ( // ErrorWaitCacheRefresh waits for synchronization of the corresponding object cache in client-go from ApiServer. - ErrorWaitCacheRefresh = "WaitCacheRefresh" - - // ErrorTypeBackupNotCompleted is used to report backup not completed. - ErrorTypeBackupNotCompleted ErrorType = "BackupNotCompleted" - + ErrorWaitCacheRefresh ErrorType = "WaitCacheRefresh" // ErrorTypeNotFound not found any resource. - ErrorTypeNotFound = "NotFound" + ErrorTypeNotFound ErrorType = "NotFound" + + // ErrorType for backup + ErrorTypeBackupNotSupported ErrorType = "BackupNotSupported" // this backup type not supported + ErrorTypeBackupPVTemplateNotFound ErrorType = "BackupPVTemplateNotFound" // this pv template not found + ErrorTypeBackupNotCompleted ErrorType = "BackupNotCompleted" // report backup not completed. + ErrorTypeBackupPVCNameIsEmpty ErrorType = "BackupPVCNameIsEmpty" // pvc name for backup is empty + ErrorTypeBackupJobFailed ErrorType = "BackupJobFailed" // backup job failed ) var ErrFailedToAddFinalizer = errors.New("failed to add finalizer") @@ -71,6 +74,14 @@ func IsTargetError(err error, errorType ErrorType) bool { return false } +// ToControllerError converts the error to the Controller error. +func ToControllerError(err error) *Error { + if tmpErr, ok := err.(*Error); ok || errors.As(err, &tmpErr) { + return tmpErr + } + return nil +} + // NewNotFound returns a new Error with ErrorTypeNotFound. func NewNotFound(format string, a ...any) *Error { return &Error{ @@ -83,3 +94,23 @@ func NewNotFound(format string, a ...any) *Error { func IsNotFound(err error) bool { return IsTargetError(err, ErrorTypeNotFound) } + +// NewBackupNotSupported returns a new Error with ErrorTypeBackupNotSupported. +func NewBackupNotSupported(backupType, backupPolicyName string) *Error { + return NewErrorf(ErrorTypeBackupNotSupported, `backup type "%s" not supported by backup policy "%s"`, backupType, backupPolicyName) +} + +// NewBackupPVTemplateNotFound returns a new Error with ErrorTypeBackupPVTemplateNotFound. +func NewBackupPVTemplateNotFound(cmName, cmNamespace string) *Error { + return NewErrorf(ErrorTypeBackupPVTemplateNotFound, `"the persistentVolume template is empty in the configMap %s/%s", pvConfig.Namespace, pvConfig.Name`, cmNamespace, cmName) +} + +// NewBackupPVCNameIsEmpty returns a new Error with ErrorTypeBackupPVCNameIsEmpty. +func NewBackupPVCNameIsEmpty(backupPolicyName string) *Error { + return NewErrorf(ErrorTypeBackupPVCNameIsEmpty, `the persistentVolumeClaim name of this policy "%s" is empty`, backupPolicyName) +} + +// NewBackupJobFailed returns a new Error with ErrorTypeBackupJobFailed. +func NewBackupJobFailed(jobName string) *Error { + return NewErrorf(ErrorTypeBackupJobFailed, `backup job "%s" failed`, jobName) +} From 960712d916b8450b42883bc727654b1d1075bd3b Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Sun, 23 Apr 2023 13:51:34 +0800 Subject: [PATCH 143/439] chore: ignore check PR release-* from releasing-* (#2856) --- .github/workflows/pull-request-check.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml index e5a477a45..055e956db 100644 --- a/.github/workflows/pull-request-check.yml +++ b/.github/workflows/pull-request-check.yml @@ -10,13 +10,14 @@ env: jobs: pr-check: name: PR Pre-Check + if: ${{ !(startsWith(github.head_ref, 'releasing-') && startsWith(github.base_ref, 'release-')) }} runs-on: ubuntu-latest steps: - name: check branch name uses: apecloud/check-branch-name@v0.1.0 with: - branch_pattern: 'feature/|bugfix/|hotfix/|support/|release/|release-|releasing/|releasing-|dependabot/' - comment_for_invalid_branch_name: 'This branch name is not following the standards: feature/|bugfix/|hotfix/|support/|release/|release-|releasing/|releasing-|dependabot/' + branch_pattern: 'feature/|bugfix/|release/|hotfix/|support/|releasing/|dependabot/' + comment_for_invalid_branch_name: 'This branch name is not following the standards: feature/|bugfix/|release/|hotfix/|support/|releasing/|dependabot/' fail_if_invalid_branch_name: 'true' ignore_branch_pattern: 'main|master' From 413077d3f0a114bd5bcfa65b11a57052c2ea8647 Mon Sep 17 00:00:00 2001 From: zjx20 Date: Sun, 23 Apr 2023 13:53:31 +0800 Subject: [PATCH 144/439] fix: fix an unstable UT case in cluster_controller_test.go (#2851) --- controllers/apps/cluster_controller_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index ee4a20a6f..661274d2b 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1400,11 +1400,11 @@ var _ = Describe("Cluster Controller", func() { g.Expect(tmpBackup.Status.Phase).Should(Equal(dataprotectionv1alpha1.BackupFailed)) })).Should(Succeed()) By("mocking backup status completed, we don't need backup reconcile here") - Expect(testapps.ChangeObjStatus(&testCtx, backup, func() { + Eventually(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(backup), func(backup *dataprotectionv1alpha1.Backup) { backup.Status.BackupToolName = backupTool.Name backup.Status.PersistentVolumeClaimName = "backup-pvc" backup.Status.Phase = dataprotectionv1alpha1.BackupCompleted - })).ShouldNot(HaveOccurred()) + })).Should(Succeed()) By("checking backup status completed") Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(backup), func(g Gomega, tmpBackup *dataprotectionv1alpha1.Backup) { From c4c61759c656b43a79c9b84d984d13f6409b8703 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Sun, 23 Apr 2023 14:15:56 +0800 Subject: [PATCH 145/439] chore: improve cli create cluster output message (#2831) --- internal/cli/cmd/cluster/create.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 1a80e2f27..e8c081ef2 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -231,7 +231,7 @@ func (o *CreateOptions) Validate() error { return err } o.ClusterVersionRef = version - printer.Warning(o.Out, "cluster version is not specified, use the recently created ClusterVersion %s\n", o.ClusterVersionRef) + fmt.Fprintf(o.Out, "Info: --cluster-version is not specified, ClusterVersion %s is applied by default\n", o.ClusterVersionRef) } if len(o.Values) > 0 && len(o.SetFile) > 0 { From 97c69295052effc2bd058874ea3862a671156eb8 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Sun, 23 Apr 2023 14:36:33 +0800 Subject: [PATCH 146/439] chore: fix is_rc_or_stable_release_version error (#2863) --- .github/utils/is_rc_or_stable_release_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/utils/is_rc_or_stable_release_version.py b/.github/utils/is_rc_or_stable_release_version.py index 304c287eb..9bd43b97e 100755 --- a/.github/utils/is_rc_or_stable_release_version.py +++ b/.github/utils/is_rc_or_stable_release_version.py @@ -24,7 +24,7 @@ def main(argv: list[str]) -> None: release_version = git_ref[len(tag_ref_prefix) :] release_note_path = f"docs/release_notes/v{release_version}/v{release_version}.md" - def set_with_rel_note_to_true() -> None + def set_with_rel_note_to_true() -> None: print(f"Checking if {release_note_path} exists") if os.path.exists(release_note_path): print(f"Found {release_note_path}") From 2a47fb9f36e50e6a6a3e7039fe50cd7eb77163ff Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Sun, 23 Apr 2023 14:43:08 +0800 Subject: [PATCH 147/439] fix: remove class validating webhook as webhook is almost disabled (#2812) --- apis/apps/v1alpha1/cluster_webhook.go | 23 +-- apis/apps/v1alpha1/opsrequest_webhook.go | 78 +-------- apis/apps/v1alpha1/type.go | 48 ------ .../apps/opsrequest_controller_test.go | 7 +- deploy/apecloud-mysql/templates/class.yaml | 30 ++-- internal/class/class_utils.go | 31 +++- internal/class/class_utils_test.go | 119 ++++++++++++- internal/cli/cmd/cluster/cluster_test.go | 158 +++++++++++++++--- internal/cli/cmd/cluster/create.go | 74 +++++--- internal/cli/cmd/cluster/create_test.go | 10 +- internal/cli/cmd/cluster/operations.go | 48 +++++- internal/cli/cmd/cluster/operations_test.go | 39 ++++- internal/cli/create/create.go | 6 - internal/cli/testing/fake.go | 29 ++-- .../testdata/component_with_class_1c1g.yaml | 16 ++ .../component_with_invalid_class.yaml | 16 ++ .../component_with_invalid_resource.yaml | 18 ++ .../component_with_resource_1c1g.yaml | 18 ++ .../lifecycle/transformer_fill_class.go | 32 +--- 19 files changed, 526 insertions(+), 274 deletions(-) create mode 100644 internal/cli/testing/testdata/component_with_class_1c1g.yaml create mode 100644 internal/cli/testing/testdata/component_with_invalid_class.yaml create mode 100644 internal/cli/testing/testdata/component_with_invalid_resource.yaml create mode 100644 internal/cli/testing/testdata/component_with_resource_1c1g.yaml diff --git a/apis/apps/v1alpha1/cluster_webhook.go b/apis/apps/v1alpha1/cluster_webhook.go index 7d72584d4..d7ac7cf69 100644 --- a/apis/apps/v1alpha1/cluster_webhook.go +++ b/apis/apps/v1alpha1/cluster_webhook.go @@ -28,7 +28,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" ) @@ -175,7 +174,7 @@ func (r *Cluster) validate() error { allErrs = append(allErrs, field.Invalid(field.NewPath("spec.clusterDefinitionRef"), r.Spec.ClusterDefRef, err.Error())) } else { - r.validateComponents(&allErrs, webhookMgr.client, clusterDef) + r.validateComponents(&allErrs, clusterDef) } if len(allErrs) > 0 { @@ -200,7 +199,7 @@ func (r *Cluster) validateClusterVersionRef(allErrs *field.ErrorList) { } // ValidateComponents validate spec.components is legal -func (r *Cluster) validateComponents(allErrs *field.ErrorList, k8sClient client.Client, clusterDef *ClusterDefinition) { +func (r *Cluster) validateComponents(allErrs *field.ErrorList, clusterDef *ClusterDefinition) { var ( // invalid component slice invalidComponentDefs = make([]string, 0) @@ -214,10 +213,6 @@ func (r *Cluster) validateComponents(allErrs *field.ErrorList, k8sClient client. componentMap[v.Name] = v } - compClasses, err := getClasses(context.Background(), k8sClient, clusterDef.Name) - if err != nil { - return - } for i, v := range r.Spec.ComponentSpecs { if _, ok := componentDefMap[v.ComponentDefRef]; !ok { invalidComponentDefs = append(invalidComponentDefs, v.ComponentDefRef) @@ -225,19 +220,6 @@ func (r *Cluster) validateComponents(allErrs *field.ErrorList, k8sClient client. componentNameMap[v.Name] = struct{}{} r.validateComponentResources(allErrs, v.Resources, i) - - if classes, ok := compClasses[v.ComponentDefRef]; ok { - if v.ClassDefRef.Class != "" { - if _, ok = classes[v.ClassDefRef.Class]; !ok { - *allErrs = append(*allErrs, field.Invalid(field.NewPath(fmt.Sprintf("spec.components[%d].classDefRef", i)), v.ClassDefRef.Class, "can not find the specified class")) - return - } - } - if err = validateMatchingClass(classes, v.Resources); err != nil { - *allErrs = append(*allErrs, field.Invalid(field.NewPath(fmt.Sprintf("spec.components[%d].resources", i)), v.Resources.String(), err.Error())) - return - } - } } r.validatePrimaryIndex(allErrs) @@ -261,7 +243,6 @@ func (r *Cluster) validateComponentResources(allErrs *field.ErrorList, resources if invalidValue, err := compareRequestsAndLimits(resources); err != nil { *allErrs = append(*allErrs, field.Invalid(field.NewPath(fmt.Sprintf("spec.components[%d].resources.requests", index)), invalidValue, err.Error())) } - } func (r *Cluster) validateComponentTLSSettings(allErrs *field.ErrorList) { diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index ca9c7c6fa..e3c17b1a5 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -31,7 +31,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/validation/field" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -176,7 +175,7 @@ func (r *OpsRequest) validateOps(ctx context.Context, case UpgradeType: return r.validateUpgrade(ctx, k8sClient) case VerticalScalingType: - return r.validateVerticalScaling(ctx, k8sClient, cluster) + return r.validateVerticalScaling(cluster) case HorizontalScalingType: return r.validateHorizontalScaling(ctx, k8sClient, cluster) case VolumeExpansionType: @@ -219,26 +218,12 @@ func (r *OpsRequest) validateUpgrade(ctx context.Context, } // validateVerticalScaling validates api when spec.type is VerticalScaling -func (r *OpsRequest) validateVerticalScaling(ctx context.Context, k8sClient client.Client, cluster *Cluster) error { +func (r *OpsRequest) validateVerticalScaling(cluster *Cluster) error { verticalScalingList := r.Spec.VerticalScalingList if len(verticalScalingList) == 0 { return notEmptyError("spec.verticalScaling") } - compClasses, err := getClasses(ctx, k8sClient, cluster.Spec.ClusterDefRef) - if err != nil { - return nil - } - - getComponent := func(name string) *ClusterComponentSpec { - for _, comp := range cluster.Spec.ComponentSpecs { - if comp.Name == name { - return &comp - } - } - return nil - } - // validate resources is legal and get component name slice componentNames := make([]string, len(verticalScalingList)) for i, v := range verticalScalingList { @@ -253,20 +238,6 @@ func (r *OpsRequest) validateVerticalScaling(ctx context.Context, k8sClient clie if invalidValue, err := compareRequestsAndLimits(v.ResourceRequirements); err != nil { return invalidValueError(invalidValue, err.Error()) } - comp := getComponent(v.ComponentName) - if comp == nil { - continue - } - if classes, ok := compClasses[comp.ComponentDefRef]; ok { - if comp.ClassDefRef.Class != "" { - if _, ok = classes[comp.ClassDefRef.Class]; !ok { - return field.Invalid(field.NewPath(fmt.Sprintf("spec.components[%d].classDefRef", i)), comp.ClassDefRef.Class, err.Error()) - } - } - if err = validateMatchingClass(classes, v.ResourceRequirements); err != nil { - return errors.Wrapf(err, "can not find matching class for component %s", v.ComponentName) - } - } } return r.checkComponentExistence(cluster, componentNames) } @@ -486,51 +457,6 @@ func validateVerticalResourceList(resourceList map[corev1.ResourceName]resource. return "", nil } -func getClasses(ctx context.Context, k8sClient client.Client, clusterDef string) (map[string]map[string]*ComponentClassInstance, error) { - ml := []client.ListOption{ - client.MatchingLabels{"clusterdefinition.kubeblocks.io/name": clusterDef}, - } - var classDefinitionList ComponentClassDefinitionList - if err := k8sClient.List(ctx, &classDefinitionList, ml...); err != nil { - return nil, err - } - var ( - componentClasses = make(map[string]map[string]*ComponentClassInstance) - ) - for _, classDefinition := range classDefinitionList.Items { - componentType := classDefinition.GetLabels()["apps.kubeblocks.io/component-def-ref"] - if componentType == "" { - return nil, fmt.Errorf("failed to find component type") - } - classes := make(map[string]*ComponentClassInstance) - for idx := range classDefinition.Status.Classes { - cls := classDefinition.Status.Classes[idx] - classes[cls.Name] = &cls - } - if _, ok := componentClasses[componentType]; !ok { - componentClasses[componentType] = classes - } else { - for k, v := range classes { - if _, exists := componentClasses[componentType][k]; exists { - return nil, fmt.Errorf("duplicate component class %s", k) - } - componentClasses[componentType][k] = v - } - } - } - return componentClasses, nil -} - -func validateMatchingClass(classes map[string]*ComponentClassInstance, resource corev1.ResourceRequirements) error { - if cls := chooseComponentClasses(classes, resource.Requests); cls == nil { - return fmt.Errorf("can not find matching class with specified requests") - } - if cls := chooseComponentClasses(classes, resource.Limits); cls == nil { - return fmt.Errorf("can not find matching class with specified limits") - } - return nil -} - func notEmptyError(target string) error { return fmt.Errorf(`"%s" can not be empty`, target) } diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index f16bdf80e..e0b8d17d0 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -18,10 +18,6 @@ limitations under the License. package v1alpha1 import ( - "sort" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -525,47 +521,3 @@ func RegisterWebhookManager(mgr manager.Manager) { } type ComponentNameSet map[string]struct{} - -func chooseComponentClasses(classes map[string]*ComponentClassInstance, filters map[corev1.ResourceName]resource.Quantity) *ComponentClassInstance { - var candidates []*ComponentClassInstance - for _, cls := range classes { - cpu, ok := filters[corev1.ResourceCPU] - if ok && !cpu.Equal(cls.CPU) { - continue - } - memory, ok := filters[corev1.ResourceMemory] - if ok && !memory.Equal(cls.Memory) { - continue - } - candidates = append(candidates, cls) - } - if len(candidates) == 0 { - return nil - } - sort.Sort(byClassCPUAndMemory(candidates)) - return candidates[0] -} - -var _ sort.Interface = byClassCPUAndMemory{} - -type byClassCPUAndMemory []*ComponentClassInstance - -func (b byClassCPUAndMemory) Len() int { - return len(b) -} - -func (b byClassCPUAndMemory) Less(i, j int) bool { - if out := b[i].CPU.Cmp(b[j].CPU); out != 0 { - return out < 0 - } - - if out := b[i].Memory.Cmp(b[j].Memory); out != 0 { - return out < 0 - } - - return false -} - -func (b byClassCPUAndMemory) Swap(i, j int) { - b[i], b[j] = b[j], b[i] -} diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index 74dea9af8..3fb7b9d5b 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -134,16 +134,13 @@ var _ = Describe("OpsRequest Controller", func() { } type verticalScalingContext struct { - description string - source resourceContext - target resourceContext + source resourceContext + target resourceContext } testVerticalScaleCPUAndMemory := func(workloadType testapps.ComponentDefTplType, scalingCtx verticalScalingContext) { const opsName = "mysql-verticalscaling" - By(scalingCtx.description) - By("Create class related objects") constraint := testapps.NewComponentResourceConstraintFactory(testapps.DefaultResourceConstraintName). AddConstraints(testapps.GeneralResourceConstraint). diff --git a/deploy/apecloud-mysql/templates/class.yaml b/deploy/apecloud-mysql/templates/class.yaml index 098dc7093..d6bb20db7 100644 --- a/deploy/apecloud-mysql/templates/class.yaml +++ b/deploy/apecloud-mysql/templates/class.yaml @@ -21,13 +21,13 @@ spec: series: - namingTemplate: {{ printf "general-{{ .cpu }}c{{ .memory }}g" }} classes: - - args: [ "0.5", "0.5", "10", "1" ] - - args: [ "1", "1", "10", "1" ] - - args: [ "2", "2", "10", "1" ] - - args: [ "2", "4", "10", "1" ] - - args: [ "2", "8", "10", "1" ] - - args: [ "4", "16", "10", "1" ] - - args: [ "8", "32", "10", "1" ] + - args: [ "0.5", "0.5", "20", "1" ] + - args: [ "1", "1", "20", "1" ] + - args: [ "2", "2", "20", "1" ] + - args: [ "2", "4", "20", "1" ] + - args: [ "2", "8", "20", "1" ] + - args: [ "4", "16", "20", "1" ] + - args: [ "8", "32", "20", "1" ] - args: [ "16", "64", "20", "1" ] - args: [ "32", "128", "20", "1" ] - args: [ "64", "256", "20", "1" ] @@ -47,17 +47,17 @@ spec: - namingTemplate: {{ printf "mo-{{ .cpu }}c{{ .memory }}g" }} classes: # 1:8 - - args: [ "2", "16", "10", "1" ] - - args: [ "4", "32", "10", "1" ] - - args: [ "8", "64", "10", "1" ] - - args: [ "12", "96", "10", "1" ] + - args: [ "2", "16", "20", "1" ] + - args: [ "4", "32", "20", "1" ] + - args: [ "8", "64", "20", "1" ] + - args: [ "12", "96", "20", "1" ] - args: [ "24", "192", "20", "1" ] - args: [ "48", "384", "20", "1" ] # 1:16 - - args: [ "2", "32", "10", "1" ] - - args: [ "4", "64", "10", "1" ] - - args: [ "8", "128", "10", "1" ] - - args: [ "16", "256", "10", "1" ] + - args: [ "2", "32", "20", "1" ] + - args: [ "4", "64", "20", "1" ] + - args: [ "8", "128", "20", "1" ] + - args: [ "16", "256", "20", "1" ] - args: [ "32", "512", "20", "1" ] - args: [ "48", "768", "20", "1" ] - args: [ "64", "1024", "20", "1" ] diff --git a/internal/class/class_utils.go b/internal/class/class_utils.go index 7ca02cee3..de5e696e0 100644 --- a/internal/class/class_utils.go +++ b/internal/class/class_utils.go @@ -25,7 +25,6 @@ import ( "github.com/ghodss/yaml" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" @@ -35,21 +34,41 @@ import ( "github.com/apecloud/kubeblocks/internal/constant" ) +// ValidateComponentClass check if component classDefRef or resource is invalid +func ValidateComponentClass(comp *v1alpha1.ClusterComponentSpec, compClasses map[string]map[string]*v1alpha1.ComponentClassInstance) (*v1alpha1.ComponentClassInstance, error) { + classes := compClasses[comp.ComponentDefRef] + var cls *v1alpha1.ComponentClassInstance + switch { + case comp.ClassDefRef != nil && comp.ClassDefRef.Class != "": + if classes == nil { + return nil, fmt.Errorf("can not find classes for component %s", comp.ComponentDefRef) + } + cls = classes[comp.ClassDefRef.Class] + if cls == nil { + return nil, fmt.Errorf("unknown component class %s", comp.ClassDefRef.Class) + } + case classes != nil: + cls = ChooseComponentClasses(classes, comp.Resources.Requests) + if cls == nil { + return nil, fmt.Errorf("can not find matching class for component %s", comp.Name) + } + } + return cls, nil +} + // GetCustomClassObjectName Returns the name of the ComponentClassDefinition object containing the custom classes func GetCustomClassObjectName(cdName string, componentName string) string { return fmt.Sprintf("kb.classes.custom.%s.%s", cdName, componentName) } // ChooseComponentClasses Choose the classes to be used for a given component with some constraints -func ChooseComponentClasses(classes map[string]*v1alpha1.ComponentClassInstance, filters map[corev1.ResourceName]resource.Quantity) *v1alpha1.ComponentClassInstance { +func ChooseComponentClasses(classes map[string]*v1alpha1.ComponentClassInstance, resources corev1.ResourceList) *v1alpha1.ComponentClassInstance { var candidates []*v1alpha1.ComponentClassInstance for _, cls := range classes { - cpu, ok := filters[corev1.ResourceCPU] - if ok && !cpu.Equal(cls.CPU) { + if !resources.Cpu().IsZero() && !resources.Cpu().Equal(cls.CPU) { continue } - memory, ok := filters[corev1.ResourceMemory] - if ok && !memory.Equal(cls.Memory) { + if !resources.Memory().IsZero() && !resources.Memory().Equal(cls.Memory) { continue } candidates = append(candidates, cls) diff --git a/internal/class/class_utils_test.go b/internal/class/class_utils_test.go index 60470d137..4c7a8c3cb 100644 --- a/internal/class/class_utils_test.go +++ b/internal/class/class_utils_test.go @@ -18,6 +18,7 @@ package class import ( "fmt" + "reflect" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -65,8 +66,8 @@ var _ = Describe("utils", func() { // Add any teardown steps that needs to be executed after each test }) - buildFilters := func(cpu string, memory string) map[corev1.ResourceName]resource.Quantity { - result := make(map[corev1.ResourceName]resource.Quantity) + buildResourceList := func(cpu string, memory string) corev1.ResourceList { + result := make(corev1.ResourceList) if cpu != "" { result[corev1.ResourceCPU] = resource.MustParse(cpu) } @@ -76,32 +77,136 @@ var _ = Describe("utils", func() { return result } + Context("validate component class", func() { + var ( + specClassName = testapps.Class1c1gName + comp1Name = "component-have-class-definition" + comp2Name = "component-does-not-have-class-definition" + compClasses map[string]map[string]*v1alpha1.ComponentClassInstance + ) + + BeforeEach(func() { + var err error + classDef := testapps.NewComponentClassDefinitionFactory("custom", "apecloud-mysql", comp1Name). + AddClasses(testapps.DefaultResourceConstraintName, []string{specClassName}). + GetObject() + compClasses, err = GetClasses(v1alpha1.ComponentClassDefinitionList{ + Items: []v1alpha1.ComponentClassDefinition{ + *classDef, + }, + }) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("should succeed if component have class definition and with valid classDefRef", func() { + comp := &v1alpha1.ClusterComponentSpec{ + ComponentDefRef: comp1Name, + ClassDefRef: &v1alpha1.ClassDefRef{Class: specClassName}, + } + cls, err := ValidateComponentClass(comp, compClasses) + Expect(err).ShouldNot(HaveOccurred()) + Expect(reflect.DeepEqual(cls.ComponentClass, testapps.Class1c1g)).Should(BeTrue()) + }) + + It("should fail if component have class definition and with invalid classDefRef", func() { + comp := &v1alpha1.ClusterComponentSpec{ + ComponentDefRef: comp1Name, + ClassDefRef: &v1alpha1.ClassDefRef{Class: "class-not-exists"}, + } + _, err := ValidateComponentClass(comp, compClasses) + Expect(err).Should(HaveOccurred()) + }) + + It("should succeed if component have class definition and with valid resource", func() { + comp := &v1alpha1.ClusterComponentSpec{ + ComponentDefRef: comp1Name, + Resources: corev1.ResourceRequirements{ + Requests: buildResourceList("1", "1Gi"), + }, + } + cls, err := ValidateComponentClass(comp, compClasses) + Expect(err).ShouldNot(HaveOccurred()) + Expect(reflect.DeepEqual(cls.ComponentClass, testapps.Class1c1g)).Should(BeTrue()) + }) + + It("should fail if component have class definition and with invalid resource", func() { + comp := &v1alpha1.ClusterComponentSpec{ + ComponentDefRef: comp1Name, + Resources: corev1.ResourceRequirements{ + Requests: buildResourceList("100", "200Gi"), + }, + } + _, err := ValidateComponentClass(comp, compClasses) + Expect(err).Should(HaveOccurred()) + }) + + It("should succeed if component does not have class definition and without classDefRef", func() { + comp := &v1alpha1.ClusterComponentSpec{ + ComponentDefRef: comp2Name, + } + cls, err := ValidateComponentClass(comp, compClasses) + Expect(err).ShouldNot(HaveOccurred()) + Expect(cls).Should(BeNil()) + }) + + It("should fail if component does not have class definition and with classDefRef", func() { + comp := &v1alpha1.ClusterComponentSpec{ + ComponentDefRef: comp2Name, + ClassDefRef: &v1alpha1.ClassDefRef{Class: specClassName}, + } + _, err := ValidateComponentClass(comp, compClasses) + Expect(err).Should(HaveOccurred()) + }) + + It("should succeed if component does not have class definition and without classDefRef", func() { + comp := &v1alpha1.ClusterComponentSpec{ + ComponentDefRef: comp2Name, + Resources: corev1.ResourceRequirements{ + Requests: buildResourceList("100", "200Gi"), + }, + } + cls, err := ValidateComponentClass(comp, compClasses) + Expect(err).ShouldNot(HaveOccurred()) + Expect(cls).Should(BeNil()) + }) + }) + Context("sort component classes", func() { + It("should match minial class if cpu and memory are empty", func() { + class := ChooseComponentClasses(classes, buildResourceList("", "")) + Expect(class).ShouldNot(BeNil()) + Expect(class.CPU.String()).Should(Equal("1")) + Expect(class.Memory.String()).Should(Equal("4Gi")) + }) + It("should match one class by cpu and memory", func() { - class := ChooseComponentClasses(classes, buildFilters("1", "4Gi")) + class := ChooseComponentClasses(classes, buildResourceList("1", "4Gi")) + Expect(class).ShouldNot(BeNil()) Expect(class.CPU.String()).Should(Equal("1")) Expect(class.Memory.String()).Should(Equal("4Gi")) }) It("match multiple classes by cpu", func() { - class := ChooseComponentClasses(classes, buildFilters("1", "")) + class := ChooseComponentClasses(classes, buildResourceList("1", "")) + Expect(class).ShouldNot(BeNil()) Expect(class.CPU.String()).Should(Equal("1")) Expect(class.Memory.String()).Should(Equal("4Gi")) }) It("match multiple classes by memory", func() { - class := ChooseComponentClasses(classes, buildFilters("", "16Gi")) + class := ChooseComponentClasses(classes, buildResourceList("", "16Gi")) + Expect(class).ShouldNot(BeNil()) Expect(class.CPU.String()).Should(Equal("1")) Expect(class.Memory.String()).Should(Equal("16Gi")) }) It("not match any classes by cpu", func() { - class := ChooseComponentClasses(classes, buildFilters(fmt.Sprintf("%d", cpuMax+1), "")) + class := ChooseComponentClasses(classes, buildResourceList(fmt.Sprintf("%d", cpuMax+1), "")) Expect(class).Should(BeNil()) }) It("not match any classes by memory", func() { - class := ChooseComponentClasses(classes, buildFilters("", "1Pi")) + class := ChooseComponentClasses(classes, buildResourceList("", "1Pi")) Expect(class).Should(BeNil()) }) }) diff --git a/internal/cli/cmd/cluster/cluster_test.go b/internal/cli/cmd/cluster/cluster_test.go index 65f27d7ff..d072aa6a1 100644 --- a/internal/cli/cmd/cluster/cluster_test.go +++ b/internal/cli/cmd/cluster/cluster_test.go @@ -17,7 +17,7 @@ limitations under the License. package cluster import ( - "os" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -29,11 +29,17 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/create" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" + testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) var _ = Describe("Cluster", func() { - const testComponentPath = "../../testing/testdata/component.yaml" - const testClassDefsPath = "../../testing/testdata/class.yaml" + const ( + testComponentPath = "../../testing/testdata/component.yaml" + testComponentWithClassPath = "../../testing/testdata/component_with_class_1c1g.yaml" + testComponentWithInvalidClassPath = "../../testing/testdata/component_with_invalid_class.yaml" + testComponentWithResourcePath = "../../testing/testdata/component_with_resource_1c1g.yaml" + testComponentWithInvalidResourcePath = "../../testing/testdata/component_with_invalid_resource.yaml" + ) var streams genericclioptions.IOStreams var tf *cmdtesting.TestFactory @@ -80,14 +86,20 @@ var _ = Describe("Cluster", func() { cmd.Run(nil, []string{"test1"}) }) - It("run", func() { + }) + + Context("run", func() { + var o *CreateOptions + + BeforeEach(func() { clusterDef := testing.FakeClusterDef() - tf.FakeDynamicClient = testing.FakeDynamicClient(clusterDef) - data, err := os.ReadFile(testClassDefsPath) - Expect(err).NotTo(HaveOccurred()) - clientSet := testing.FakeClientSet(testing.FakeComponentClassDef(clusterDef, data)) - o := &CreateOptions{ - BaseOptions: create.BaseOptions{IOStreams: streams, Name: "test", Dynamic: tf.FakeDynamicClient, ClientSet: clientSet}, + tf.FakeDynamicClient = testing.FakeDynamicClient( + clusterDef, + testing.FakeComponentClassDef(fmt.Sprintf("custom-%s", testing.ComponentDefName), clusterDef.Name, testing.ComponentDefName), + testing.FakeComponentClassDef("custom-mysql", clusterDef.Name, "mysql"), + ) + o = &CreateOptions{ + BaseOptions: create.BaseOptions{IOStreams: streams, Name: "test", Dynamic: tf.FakeDynamicClient}, SetFile: "", ClusterDefRef: testing.ClusterDefName, ClusterVersionRef: "cluster-version", @@ -99,36 +111,138 @@ var _ = Describe("Cluster", func() { Tenancy: string(appsv1alpha1.SharedNode), }, } + o.TerminationPolicy = "WipeOut" + }) + + Run := func() { + inputs := create.Inputs{ + ResourceName: types.ResourceClusters, + CueTemplateName: CueTemplateName, + Options: o, + Factory: tf, + } + + Expect(o.BaseOptions.Complete(inputs, []string{"test"})).Should(Succeed()) + Expect(o.Namespace).To(Equal("default")) + Expect(o.Name).To(Equal("test")) + Expect(o.Run(inputs)).Should(Succeed()) + } + + It("validate tolerations", func() { Expect(len(o.TolerationsRaw)).Should(Equal(1)) Expect(o.Complete()).Should(Succeed()) Expect(len(o.Tolerations)).Should(Equal(1)) + }) + + It("validate termination policy should be set", func() { + o.TerminationPolicy = "" Expect(o.Validate()).Should(HaveOccurred()) + }) - o.TerminationPolicy = "WipeOut" + It("should succeed if component with valid class", func() { + o.Values = []string{fmt.Sprintf("type=%s,class=%s", testing.ComponentDefName, testapps.Class1c1gName)} + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Run() + }) + + It("should fail if component with invalid class", func() { + o.Values = []string{fmt.Sprintf("type=%s,class=class-not-exists", testing.ComponentDefName)} + Expect(o.Complete()).Should(HaveOccurred()) + }) + + It("should succeed if component with resource matching to one class", func() { + o.Values = []string{fmt.Sprintf("type=%s,cpu=1,memory=1Gi", testing.ComponentDefName)} + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Run() + }) + + It("should succeed if component with resource equivalent to class", func() { + o.Values = []string{fmt.Sprintf("type=%s,cpu=1000m,memory=1024Mi", testing.ComponentDefName)} + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Run() + }) + + It("should fail if component with resource not matching to any class", func() { + o.Values = []string{fmt.Sprintf("type=%s,cpu=1,memory=2Gi", testing.ComponentDefName)} + Expect(o.Complete()).Should(HaveOccurred()) + }) + + It("should succeed if component with cpu matching one class", func() { + o.Values = []string{fmt.Sprintf("type=%s,cpu=1", testing.ComponentDefName)} + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Run() + }) + + It("should fail if component with cpu not matching to any class", func() { + o.Values = []string{fmt.Sprintf("type=%s,cpu=3", testing.ComponentDefName)} + Expect(o.Complete()).Should(HaveOccurred()) + }) + + It("should succeed if component with memory matching one class", func() { + o.Values = []string{fmt.Sprintf("type=%s,memory=1Gi", testing.ComponentDefName)} + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Run() + }) + + It("should fail if component with memory not matching any class", func() { + o.Values = []string{fmt.Sprintf("type=%s,memory=7Gi", testing.ComponentDefName)} + Expect(o.Complete()).Should(HaveOccurred()) + }) + + It("should succeed if component don't have class definition", func() { + o.Values = []string{fmt.Sprintf("type=%s,cpu=3,memory=7Gi", testing.ExtraComponentDefName)} + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Run() + }) + + It("should fail if create cluster by file not existing", func() { o.SetFile = "test.yaml" - Expect(o.Complete()).ShouldNot(Succeed()) + Expect(o.Complete()).Should(HaveOccurred()) + }) + It("should succeed if create cluster by empty file", func() { o.SetFile = "" Expect(o.Complete()).Should(Succeed()) Expect(o.Validate()).Should(Succeed()) + Run() + }) + It("should succeed if create cluster by file without class and resource", func() { o.SetFile = testComponentPath Expect(o.Complete()).Should(Succeed()) Expect(o.Validate()).Should(Succeed()) + Run() + }) - inputs := create.Inputs{ - ResourceName: types.ResourceClusters, - CueTemplateName: CueTemplateName, - Options: o, - Factory: tf, - } + It("should succeed if create cluster by file with class", func() { + o.SetFile = testComponentWithClassPath + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Run() + }) - Expect(o.BaseOptions.Complete(inputs, []string{"test"})).Should(Succeed()) - Expect(o.Namespace).To(Equal("default")) - Expect(o.Name).To(Equal("test")) + It("should succeed if create cluster by file with resource", func() { + o.SetFile = testComponentWithResourcePath + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Run() + }) - Expect(o.Run(inputs)).Should(Succeed()) + It("should fail if create cluster by file with class not exists", func() { + o.SetFile = testComponentWithInvalidClassPath + Expect(o.Complete()).Should(HaveOccurred()) + }) + + It("should fail if create cluster by file with resource not matching to any class", func() { + o.SetFile = testComponentWithInvalidResourcePath + Expect(o.Complete()).Should(HaveOccurred()) }) }) diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index e8c081ef2..1da2540ee 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -43,6 +43,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + "github.com/apecloud/kubeblocks/internal/class" "github.com/apecloud/kubeblocks/internal/cli/cluster" "github.com/apecloud/kubeblocks/internal/cli/create" "github.com/apecloud/kubeblocks/internal/cli/printer" @@ -282,6 +283,11 @@ func (o *CreateOptions) buildComponents() ([]map[string]interface{}, error) { err error ) + componentClasses, err := class.ListClassesByClusterDefinition(o.Dynamic, o.ClusterDefRef) + if err != nil { + return nil, err + } + // build components from file components := o.ComponentSpecs if len(o.SetFile) > 0 { @@ -294,6 +300,15 @@ func (o *CreateOptions) buildComponents() ([]map[string]interface{}, error) { if err = json.Unmarshal(componentByte, &components); err != nil { return nil, err } + for _, item := range components { + var comp appsv1alpha1.ClusterComponentSpec + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(item, &comp); err != nil { + return nil, err + } + if _, err = class.ValidateComponentClass(&comp, componentClasses); err != nil { + return nil, err + } + } return components, nil } @@ -309,11 +324,14 @@ func (o *CreateOptions) buildComponents() ([]map[string]interface{}, error) { return nil, err } - componentObjs, err := buildClusterComp(cd, compSets) + componentObjs, err := buildClusterComp(cd, compSets, componentClasses) if err != nil { return nil, err } for _, compObj := range componentObjs { + if _, err = class.ValidateComponentClass(compObj, componentClasses); err != nil { + return nil, err + } comp, err := runtime.DefaultUnstructuredConverter.ToUnstructured(compObj) if err != nil { return nil, err @@ -455,7 +473,7 @@ func setEnableAllLogs(c *appsv1alpha1.Cluster, cd *appsv1alpha1.ClusterDefinitio } } -func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map[setKey]string) ([]*appsv1alpha1.ClusterComponentSpec, error) { +func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map[setKey]string, componentClasses map[string]map[string]*appsv1alpha1.ComponentClassInstance) ([]*appsv1alpha1.ClusterComponentSpec, error) { // get value from set values and environment variables, the second return value is // true if the value is from environment variables getVal := func(c *appsv1alpha1.ClusterComponentDefinition, key setKey, sets map[setKey]string) string { @@ -544,32 +562,44 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map } // class has higher priority than other resource related parameters - className := getVal(&c, keyClass, sets) - if className != "" { - compObj.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: className} + resourceList := make(corev1.ResourceList) + if _, ok := componentClasses[c.Name]; ok { + if className := getVal(&c, keyClass, sets); className != "" { + compObj.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: className} + } else { + if cpu, ok := sets[keyCPU]; ok { + resourceList[corev1.ResourceCPU] = resource.MustParse(cpu) + } + if mem, ok := sets[keyMemory]; ok { + resourceList[corev1.ResourceMemory] = resource.MustParse(mem) + } + } } else { - resourceList := corev1.ResourceList{ + if className := getVal(&c, keyClass, sets); className != "" { + return nil, fmt.Errorf("can not find class %s for component type %s", className, c.Name) + } + resourceList = corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(getVal(&c, keyCPU, sets)), corev1.ResourceMemory: resource.MustParse(getVal(&c, keyMemory, sets)), } - compObj.Resources = corev1.ResourceRequirements{ - Requests: resourceList, - Limits: resourceList, - } - compObj.VolumeClaimTemplates = []appsv1alpha1.ClusterComponentVolumeClaimTemplate{{ - Name: "data", - Spec: appsv1alpha1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteOnce, - }, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(getVal(&c, keyStorage, sets)), - }, + } + compObj.Resources = corev1.ResourceRequirements{ + Requests: resourceList, + Limits: resourceList, + } + compObj.VolumeClaimTemplates = []appsv1alpha1.ClusterComponentVolumeClaimTemplate{{ + Name: "data", + Spec: appsv1alpha1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(getVal(&c, keyStorage, sets)), }, }, - }} - } + }, + }} if err = buildSwitchPolicy(&c, compObj, sets); err != nil { return nil, err } diff --git a/internal/cli/cmd/cluster/create_test.go b/internal/cli/cmd/cluster/create_test.go index 644ecbf28..76b322458 100644 --- a/internal/cli/cmd/cluster/create_test.go +++ b/internal/cli/cmd/cluster/create_test.go @@ -55,6 +55,8 @@ func getResource(res corev1.ResourceRequirements, name corev1.ResourceName) inte } var _ = Describe("create", func() { + var componentClasses map[string]map[string]*appsv1alpha1.ComponentClassInstance + Context("setMonitor", func() { var components []map[string]interface{} BeforeEach(func() { @@ -150,7 +152,7 @@ var _ = Describe("create", func() { It("build default cluster component without environment", func() { dynamic := testing.FakeDynamicClient(testing.FakeClusterDef()) cd, _ := cluster.GetClusterDefByName(dynamic, testing.ClusterDefName) - comps, err := buildClusterComp(cd, nil) + comps, err := buildClusterComp(cd, nil, componentClasses) Expect(err).ShouldNot(HaveOccurred()) checkComponent(comps, "20Gi", 1, "1", "1Gi") }) @@ -162,7 +164,7 @@ var _ = Describe("create", func() { viper.Set("CLUSTER_DEFAULT_MEMORY", "2Gi") dynamic := testing.FakeDynamicClient(testing.FakeClusterDef()) cd, _ := cluster.GetClusterDefByName(dynamic, testing.ClusterDefName) - comps, err := buildClusterComp(cd, nil) + comps, err := buildClusterComp(cd, nil, componentClasses) Expect(err).ShouldNot(HaveOccurred()) checkComponent(comps, "5Gi", 1, "2", "2Gi") }) @@ -178,13 +180,13 @@ var _ = Describe("create", func() { keyReplicas: "10", }, } - comps, err := buildClusterComp(cd, setsMap) + comps, err := buildClusterComp(cd, setsMap, componentClasses) Expect(err).Should(Succeed()) checkComponent(comps, "10Gi", 10, "10", "2Gi") setsMap[testing.ComponentDefName][keySwitchPolicy] = "invalid" cd.Spec.ComponentDefs[0].WorkloadType = appsv1alpha1.Replication - _, err = buildClusterComp(cd, setsMap) + _, err = buildClusterComp(cd, setsMap, componentClasses) Expect(err).Should(HaveOccurred()) }) diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 0664150cf..152a1fb78 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -23,6 +23,7 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -31,6 +32,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/class" "github.com/apecloud/kubeblocks/internal/cli/cluster" "github.com/apecloud/kubeblocks/internal/cli/create" "github.com/apecloud/kubeblocks/internal/cli/delete" @@ -162,6 +164,42 @@ func (o *OperationsOptions) validateVolumeExpansion() error { return nil } +func (o *OperationsOptions) validateVScale(cluster *appsv1alpha1.Cluster) error { + componentClasses, err := class.ListClassesByClusterDefinition(o.Dynamic, cluster.Spec.ClusterDefRef) + if err != nil { + return err + } + + fillClassParams := func(comp *appsv1alpha1.ClusterComponentSpec) { + if o.Class != "" { + comp.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: o.Class} + } + + requests := make(corev1.ResourceList) + if o.CPU != "" { + requests[corev1.ResourceCPU] = resource.MustParse(o.CPU) + } + if o.Memory != "" { + requests[corev1.ResourceMemory] = resource.MustParse(o.Memory) + } + requests.DeepCopyInto(&comp.Resources.Requests) + requests.DeepCopyInto(&comp.Resources.Limits) + } + + for _, name := range o.ComponentNames { + for _, comp := range cluster.Spec.ComponentSpecs { + if comp.Name != name { + continue + } + fillClassParams(&comp) + if _, err = class.ValidateComponentClass(&comp, componentClasses); err != nil { + return err + } + } + } + return nil +} + // Validate command flags or args is legal func (o *OperationsOptions) Validate() error { if o.Name == "" { @@ -169,10 +207,14 @@ func (o *OperationsOptions) Validate() error { } // check if cluster exist - _, err := o.Dynamic.Resource(types.ClusterGVR()).Namespace(o.Namespace).Get(context.TODO(), o.Name, metav1.GetOptions{}) + unstructuredObj, err := o.Dynamic.Resource(types.ClusterGVR()).Namespace(o.Namespace).Get(context.TODO(), o.Name, metav1.GetOptions{}) if err != nil { return err } + var cluster appsv1alpha1.Cluster + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.Object, &cluster); err != nil { + return err + } // common validate for componentOps if o.HasComponentNamesFlag && len(o.ComponentNames) == 0 { @@ -188,6 +230,10 @@ func (o *OperationsOptions) Validate() error { if err := o.validateUpgrade(); err != nil { return err } + case appsv1alpha1.VerticalScalingType: + if err := o.validateVScale(&cluster); err != nil { + return err + } } if o.RequireConfirm { return delete.Confirm([]string{o.Name}, o.In) diff --git a/internal/cli/cmd/cluster/operations_test.go b/internal/cli/cmd/cluster/operations_test.go index eba9f5c00..7f6399770 100644 --- a/internal/cli/cmd/cluster/operations_test.go +++ b/internal/cli/cmd/cluster/operations_test.go @@ -30,6 +30,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" + testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) var _ = Describe("operations", func() { @@ -52,8 +53,11 @@ var _ = Describe("operations", func() { clusterWithOneComp.Spec.ComponentSpecs = []appsv1alpha1.ClusterComponentSpec{ clusterWithOneComp.Spec.ComponentSpecs[0], } + classDef := testapps.NewComponentClassDefinitionFactory("custom", clusterWithOneComp.Spec.ClusterDefRef, testing.ComponentDefName). + AddClasses(testapps.DefaultResourceConstraintName, []string{testapps.Class1c1gName}). + GetObject() tf.FakeDynamicClient = testing.FakeDynamicClient(testing.FakeClusterDef(), - testing.FakeClusterVersion(), clusterWithTwoComps, clusterWithOneComp) + testing.FakeClusterVersion(), clusterWithTwoComps, clusterWithOneComp, classDef) tf.Client = &clientfake.RESTClient{} }) @@ -105,6 +109,39 @@ var _ = Describe("operations", func() { Expect(o.Validate()).Should(Succeed()) }) + It("Vscale Ops", func() { + o := initCommonOperationOps(appsv1alpha1.VerticalScalingType, clusterName1, true) + By("test CompleteComponentsFlag function") + o.ComponentNames = nil + By("expect to auto complete components when cluster has only one component") + Expect(o.CompleteComponentsFlag()).Should(Succeed()) + Expect(o.ComponentNames[0]).Should(Equal(testing.ComponentName)) + + By("validate invalid class") + o.Class = "class-not-exists" + in.Write([]byte(o.Name + "\n")) + Expect(o.Validate()).Should(HaveOccurred()) + + By("expect to validate success with class") + o.Class = testapps.Class1c1gName + in.Write([]byte(o.Name + "\n")) + Expect(o.Validate()).ShouldNot(HaveOccurred()) + + By("validate invalid resource") + o.Class = "" + o.CPU = "100" + o.Memory = "100Gi" + in.Write([]byte(o.Name + "\n")) + Expect(o.Validate()).Should(HaveOccurred()) + + By("expect to validate success with resource") + o.Class = "" + o.CPU = "1" + o.Memory = "1Gi" + in.Write([]byte(o.Name + "\n")) + Expect(o.Validate()).ShouldNot(HaveOccurred()) + }) + It("Hscale Ops", func() { o := initCommonOperationOps(appsv1alpha1.HorizontalScalingType, clusterName1, true) By("test CompleteComponentsFlag function") diff --git a/internal/cli/create/create.go b/internal/cli/create/create.go index d4116a57a..d8acc8498 100755 --- a/internal/cli/create/create.go +++ b/internal/cli/create/create.go @@ -123,8 +123,6 @@ type BaseOptions struct { // Quiet minimize unnecessary output Quiet bool - ClientSet kubernetes.Interface - genericclioptions.IOStreams } @@ -166,10 +164,6 @@ func (o *BaseOptions) Complete(inputs Inputs, args []string) error { return err } - if o.ClientSet, err = inputs.Factory.KubernetesClientSet(); err != nil { - return err - } - o.ToPrinter = func(mapping *meta.RESTMapping, withNamespace bool) (printers.ResourcePrinterFunc, error) { var p printers.ResourcePrinter switch o.Format { diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index 4c01fa34c..44bbbb5a5 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -33,6 +33,7 @@ import ( extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/constant" + testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) const ( @@ -53,6 +54,10 @@ const ( BackupToolName = "fake-backup-tool" ) +var ( + ExtraComponentDefName = fmt.Sprintf("%s-%d", ComponentDefName, 1) +) + func GetRandomStr() string { seq, _ := password.Generate(6, 2, 0, true, true) return seq @@ -256,7 +261,7 @@ func FakeClusterDef() *appsv1alpha1.ClusterDefinition { }, }, { - Name: fmt.Sprintf("%s-%d", ComponentDefName, 1), + Name: ExtraComponentDefName, CharacterType: "mysql", ConfigSpecs: []appsv1alpha1.ComponentConfigSpec{ { @@ -274,18 +279,16 @@ func FakeClusterDef() *appsv1alpha1.ClusterDefinition { return clusterDef } -func FakeComponentClassDef(clusterDef *appsv1alpha1.ClusterDefinition, def []byte) *corev1.ConfigMapList { - result := &corev1.ConfigMapList{} - cm := &corev1.ConfigMap{} - cm.Name = fmt.Sprintf("fake-kubeblocks-classes-%s", ComponentName) - cm.SetLabels(map[string]string{ - constant.KBAppComponentDefRefLabelKey: ComponentDefName, - types.ClassProviderLabelKey: "kubeblocks", - constant.ClusterDefLabelKey: clusterDef.Name, - }) - cm.Data = map[string]string{"families-20230223162700": string(def)} - result.Items = append(result.Items, *cm) - return result +func FakeComponentClassDef(name string, clusterDefRef string, componentDefRef string) *appsv1alpha1.ComponentClassDefinition { + constraint := testapps.NewComponentResourceConstraintFactory(testapps.DefaultResourceConstraintName). + AddConstraints(testapps.GeneralResourceConstraint). + GetObject() + + componentClassDefinition := testapps.NewComponentClassDefinitionFactory(name, clusterDefRef, componentDefRef). + AddClasses(constraint.Name, []string{testapps.Class1c1gName, testapps.Class2c4gName}). + GetObject() + + return componentClassDefinition } func FakeClusterVersion() *appsv1alpha1.ClusterVersion { diff --git a/internal/cli/testing/testdata/component_with_class_1c1g.yaml b/internal/cli/testing/testdata/component_with_class_1c1g.yaml new file mode 100644 index 000000000..a4d035abf --- /dev/null +++ b/internal/cli/testing/testdata/component_with_class_1c1g.yaml @@ -0,0 +1,16 @@ +- name: test + componentDefRef: mysql + monitor: true + enabledLogs: [error, slow] + replicas: 1 + classDefRef: + class: general-1c1g + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumeMode: Filesystem diff --git a/internal/cli/testing/testdata/component_with_invalid_class.yaml b/internal/cli/testing/testdata/component_with_invalid_class.yaml new file mode 100644 index 000000000..23a158ecb --- /dev/null +++ b/internal/cli/testing/testdata/component_with_invalid_class.yaml @@ -0,0 +1,16 @@ +- name: test + componentDefRef: mysql + monitor: true + enabledLogs: [error, slow] + replicas: 1 + classDefRef: + class: class-not-exists + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumeMode: Filesystem diff --git a/internal/cli/testing/testdata/component_with_invalid_resource.yaml b/internal/cli/testing/testdata/component_with_invalid_resource.yaml new file mode 100644 index 000000000..8141020ba --- /dev/null +++ b/internal/cli/testing/testdata/component_with_invalid_resource.yaml @@ -0,0 +1,18 @@ +- name: test + componentDefRef: mysql + monitor: true + enabledLogs: [error, slow] + replicas: 1 + resources: + requests: + cpu: 3 + memory: 7Gi + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumeMode: Filesystem diff --git a/internal/cli/testing/testdata/component_with_resource_1c1g.yaml b/internal/cli/testing/testdata/component_with_resource_1c1g.yaml new file mode 100644 index 000000000..479d00ec3 --- /dev/null +++ b/internal/cli/testing/testdata/component_with_resource_1c1g.yaml @@ -0,0 +1,18 @@ +- name: test + componentDefRef: mysql + monitor: true + enabledLogs: [error, slow] + replicas: 1 + resources: + requests: + cpu: 1 + memory: 1Gi + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumeMode: Filesystem diff --git a/internal/controller/lifecycle/transformer_fill_class.go b/internal/controller/lifecycle/transformer_fill_class.go index d5440745e..c85c2aaf5 100644 --- a/internal/controller/lifecycle/transformer_fill_class.go +++ b/internal/controller/lifecycle/transformer_fill_class.go @@ -21,7 +21,6 @@ import ( "sort" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -92,38 +91,16 @@ func (r *FillClassTransformer) fillClass(transCtx *ClusterTransformContext) erro return cls } - matchComponentClass := func(comp appsv1alpha1.ClusterComponentSpec, classes map[string]*appsv1alpha1.ComponentClassInstance) *appsv1alpha1.ComponentClassInstance { - filters := make(map[corev1.ResourceName]resource.Quantity) - if !comp.Resources.Requests.Cpu().IsZero() { - filters[corev1.ResourceCPU] = *comp.Resources.Requests.Cpu() - } - if !comp.Resources.Requests.Memory().IsZero() { - filters[corev1.ResourceMemory] = *comp.Resources.Requests.Memory() - } - return class.ChooseComponentClasses(classes, filters) - } - for idx, comp := range cluster.Spec.ComponentSpecs { - classes := compClasses[comp.ComponentDefRef] - - var cls *appsv1alpha1.ComponentClassInstance - // TODO another case if len(constraintList.Items) > 0, use matchClassFamilies to find matching resource constraint: - switch { - case comp.ClassDefRef != nil && comp.ClassDefRef.Class != "": - cls = classes[comp.ClassDefRef.Class] - if cls == nil { - return fmt.Errorf("unknown component class %s", comp.ClassDefRef.Class) - } - case classes != nil: - cls = matchComponentClass(comp, classes) - if cls == nil { - return fmt.Errorf("can not find matching class for component %s", comp.Name) - } + cls, err := class.ValidateComponentClass(&comp, compClasses) + if err != nil { + return err } if cls == nil { // TODO reconsider handling policy for this case continue } + comp.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: cls.Name} requests := corev1.ResourceList{ corev1.ResourceCPU: cls.CPU, @@ -131,6 +108,7 @@ func (r *FillClassTransformer) fillClass(transCtx *ClusterTransformContext) erro } requests.DeepCopyInto(&comp.Resources.Requests) requests.DeepCopyInto(&comp.Resources.Limits) + var volumes []appsv1alpha1.ClusterComponentVolumeClaimTemplate if len(comp.VolumeClaimTemplates) > 0 { volumes = comp.VolumeClaimTemplates From 121e8307323f827fb894d79a988ac8456375166a Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Sun, 23 Apr 2023 14:45:26 +0800 Subject: [PATCH 148/439] chore: improve cli cluster describe (#2860) --- internal/cli/cluster/cluster.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/cli/cluster/cluster.go b/internal/cli/cluster/cluster.go index 241f73e7c..56eb4603f 100644 --- a/internal/cli/cluster/cluster.go +++ b/internal/cli/cluster/cluster.go @@ -390,6 +390,11 @@ func getResourceInfo(reqs, limits corev1.ResourceList) (string, string) { res := types.None limit, req := limits[name], reqs[name] + // if request is empty and limit is not, set limit to request + if util.ResourceIsEmpty(&req) && !util.ResourceIsEmpty(&limit) { + req = limit + } + // if both limit and request are empty, only output none if !util.ResourceIsEmpty(&limit) || !util.ResourceIsEmpty(&req) { res = fmt.Sprintf("%s / %s", req.String(), limit.String()) From f612c495ad5d3c479ed8006d987d81affdfc45f0 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Sun, 23 Apr 2023 15:30:17 +0800 Subject: [PATCH 149/439] chore: merge_releasing_branch.sh (#2867) --- .github/utils/merge_releasing_branch.sh | 17 +++++++++++++++++ .github/workflows/cicd-push.yml | 5 +++++ .github/workflows/release-create.yml | 3 +++ 3 files changed, 25 insertions(+) create mode 100644 .github/utils/merge_releasing_branch.sh diff --git a/.github/utils/merge_releasing_branch.sh b/.github/utils/merge_releasing_branch.sh new file mode 100644 index 000000000..d5b9631b6 --- /dev/null +++ b/.github/utils/merge_releasing_branch.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. + +workdir=$(dirname $0) +. ${workdir}/gh_env +. ${workdir}/functions.bash + +set -x + +git switch ${BASE_BRANCH} +git merge origin/${HEAD_BRANCH} +git push \ No newline at end of file diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index a0ae408d1..51579bbbf 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -32,6 +32,11 @@ jobs: echo $TRIGGER_MODE echo trigger_mode=$TRIGGER_MODE >> $GITHUB_OUTPUT + - name: merge releasing to release + if: ${{ startsWith(github.ref_name, 'releasing-') }} + run: | + bash .github/utils/merge_releasing_branch.sh + pre-push: needs: trigger-mode runs-on: ubuntu-latest diff --git a/.github/workflows/release-create.yml b/.github/workflows/release-create.yml index da1e14593..8a59dc887 100644 --- a/.github/workflows/release-create.yml +++ b/.github/workflows/release-create.yml @@ -5,6 +5,9 @@ on: tags: - v* +env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + jobs: publish: name: create a release From 8a6018e601ad8b28c22d74152368f0f1af5ab616 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Sun, 23 Apr 2023 15:32:03 +0800 Subject: [PATCH 150/439] fix: remove kbcli useless cmds: list-restore delete-restore (#2859) --- docs/user_docs/cli/cli.md | 2 - docs/user_docs/cli/kbcli_cluster.md | 2 - docs/user_docs/cli/kbcli_cluster_backup.md | 9 +++ .../cli/kbcli_cluster_delete-restore.md | 60 --------------- .../cli/kbcli_cluster_list-restores.md | 57 -------------- internal/cli/cmd/cluster/cluster.go | 2 - internal/cli/cmd/cluster/dataprotection.go | 76 +++---------------- .../cli/cmd/cluster/dataprotection_test.go | 30 -------- 8 files changed, 19 insertions(+), 219 deletions(-) delete mode 100644 docs/user_docs/cli/kbcli_cluster_delete-restore.md delete mode 100644 docs/user_docs/cli/kbcli_cluster_list-restores.md diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index e168330e6..a034a0a77 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -52,7 +52,6 @@ Cluster command. * [kbcli cluster delete-account](kbcli_cluster_delete-account.md) - Delete account for a cluster * [kbcli cluster delete-backup](kbcli_cluster_delete-backup.md) - Delete a backup. * [kbcli cluster delete-ops](kbcli_cluster_delete-ops.md) - Delete an OpsRequest. -* [kbcli cluster delete-restore](kbcli_cluster_delete-restore.md) - Delete a restore job. * [kbcli cluster describe](kbcli_cluster_describe.md) - Show details of a specific cluster. * [kbcli cluster describe-account](kbcli_cluster_describe-account.md) - Describe account roles and related information * [kbcli cluster describe-config](kbcli_cluster_describe-config.md) - Show details of a specific reconfiguring. @@ -74,7 +73,6 @@ Cluster command. * [kbcli cluster list-instances](kbcli_cluster_list-instances.md) - List cluster instances. * [kbcli cluster list-logs](kbcli_cluster_list-logs.md) - List supported log files in cluster. * [kbcli cluster list-ops](kbcli_cluster_list-ops.md) - List all opsRequests. -* [kbcli cluster list-restores](kbcli_cluster_list-restores.md) - List all restore jobs. * [kbcli cluster logs](kbcli_cluster_logs.md) - Access cluster log file. * [kbcli cluster restart](kbcli_cluster_restart.md) - Restart the specified components in the cluster. * [kbcli cluster restore](kbcli_cluster_restore.md) - Restore a new cluster from backup. diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index 215d34109..48a8cd37e 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -46,7 +46,6 @@ Cluster command. * [kbcli cluster delete-account](kbcli_cluster_delete-account.md) - Delete account for a cluster * [kbcli cluster delete-backup](kbcli_cluster_delete-backup.md) - Delete a backup. * [kbcli cluster delete-ops](kbcli_cluster_delete-ops.md) - Delete an OpsRequest. -* [kbcli cluster delete-restore](kbcli_cluster_delete-restore.md) - Delete a restore job. * [kbcli cluster describe](kbcli_cluster_describe.md) - Show details of a specific cluster. * [kbcli cluster describe-account](kbcli_cluster_describe-account.md) - Describe account roles and related information * [kbcli cluster describe-config](kbcli_cluster_describe-config.md) - Show details of a specific reconfiguring. @@ -68,7 +67,6 @@ Cluster command. * [kbcli cluster list-instances](kbcli_cluster_list-instances.md) - List cluster instances. * [kbcli cluster list-logs](kbcli_cluster_list-logs.md) - List supported log files in cluster. * [kbcli cluster list-ops](kbcli_cluster_list-ops.md) - List all opsRequests. -* [kbcli cluster list-restores](kbcli_cluster_list-restores.md) - List all restore jobs. * [kbcli cluster logs](kbcli_cluster_logs.md) - Access cluster log file. * [kbcli cluster restart](kbcli_cluster_restart.md) - Restart the specified components in the cluster. * [kbcli cluster restore](kbcli_cluster_restore.md) - Restore a new cluster from backup. diff --git a/docs/user_docs/cli/kbcli_cluster_backup.md b/docs/user_docs/cli/kbcli_cluster_backup.md index b49f3a867..4234d7bff 100644 --- a/docs/user_docs/cli/kbcli_cluster_backup.md +++ b/docs/user_docs/cli/kbcli_cluster_backup.md @@ -13,6 +13,15 @@ kbcli cluster backup [flags] ``` # create a backup kbcli cluster backup cluster-name + + # create a snapshot backup + kbcli cluster backup cluster-name --backup-type snapshot + + # create a full backup + kbcli cluster backup cluster-name --backup-type full + + # create a backup with specified backup policy + kbcli cluster backup cluster-name --backup-policy ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_delete-restore.md b/docs/user_docs/cli/kbcli_cluster_delete-restore.md deleted file mode 100644 index ca3c38294..000000000 --- a/docs/user_docs/cli/kbcli_cluster_delete-restore.md +++ /dev/null @@ -1,60 +0,0 @@ ---- -title: kbcli cluster delete-restore ---- - -Delete a restore job. - -``` -kbcli cluster delete-restore [flags] -``` - -### Examples - -``` - # delete a restore named restore-name - kbcli cluster delete-restore cluster-name --name restore-name -``` - -### Options - -``` - -A, --all-namespaces If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. - --auto-approve Skip interactive approval before deleting - --force If true, immediately remove resources from API and bypass graceful deletion. Note that immediate deletion of some resources may result in inconsistency or data loss and requires confirmation. - --grace-period int Period of time in seconds given to the resource to terminate gracefully. Ignored if negative. Set to 1 for immediate shutdown. Can only be set to 0 when --force is true (force deletion). (default -1) - -h, --help help for delete-restore - --name strings Restore names - --now If true, resources are signaled for immediate shutdown (same as --grace-period=1). - -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli cluster](kbcli_cluster.md) - Cluster command. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_cluster_list-restores.md b/docs/user_docs/cli/kbcli_cluster_list-restores.md deleted file mode 100644 index b2f4d7448..000000000 --- a/docs/user_docs/cli/kbcli_cluster_list-restores.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: kbcli cluster list-restores ---- - -List all restore jobs. - -``` -kbcli cluster list-restores [flags] -``` - -### Examples - -``` - # list all restore - kbcli cluster list-restore -``` - -### Options - -``` - -A, --all-namespace If present, list the requested object(s) across all namespaces. Namespace in current context is ignored even if specified with --namespace. - -h, --help help for list-restores - -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) - -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. - --show-labels When printing, show all labels as the last column (default hide labels column) -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli cluster](kbcli_cluster.md) - Cluster command. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/internal/cli/cmd/cluster/cluster.go b/internal/cli/cmd/cluster/cluster.go index 3e26725ea..a213e981c 100644 --- a/internal/cli/cmd/cluster/cluster.go +++ b/internal/cli/cmd/cluster/cluster.go @@ -90,8 +90,6 @@ func NewClusterCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr NewListBackupCmd(f, streams), NewDeleteBackupCmd(f, streams), NewCreateRestoreCmd(f, streams), - NewListRestoreCmd(f, streams), - NewDeleteRestoreCmd(f, streams), }, }, { diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index eae902072..06a3e36e9 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -67,6 +67,15 @@ var ( createBackupExample = templates.Examples(` # create a backup kbcli cluster backup cluster-name + + # create a snapshot backup + kbcli cluster backup cluster-name --backup-type snapshot + + # create a full backup + kbcli cluster backup cluster-name --backup-type full + + # create a backup with specified backup policy + kbcli cluster backup cluster-name --backup-policy `) listBackupExample = templates.Examples(` # list all backup @@ -76,14 +85,6 @@ var ( # delete a backup named backup-name kbcli cluster delete-backup cluster-name --name backup-name `) - listRestoreExample = templates.Examples(` - # list all restore - kbcli cluster list-restore - `) - deleteRestoreExample = templates.Examples(` - # delete a restore named restore-name - kbcli cluster delete-restore cluster-name --name restore-name - `) createRestoreExample = templates.Examples(` # restore a new cluster from a backup kbcli cluster restore new-cluster-name --backup backup-name @@ -152,7 +153,7 @@ func (o *CreateBackupOptions) getDefaultBackupPolicy() (string, error) { constant.AppInstanceLabelKey, clusterObj.GetName()), } objs, err := o.Dynamic. - Resource(types.BackupPolicyGVR()). + Resource(types.BackupPolicyGVR()).Namespace(o.Namespace). List(context.TODO(), opts) if err != nil { return "", err @@ -542,63 +543,6 @@ func NewCreateRestoreCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) return cmd } -func NewListRestoreCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := list.NewListOptions(f, streams, types.RestoreJobGVR()) - cmd := &cobra.Command{ - Use: "list-restores", - Short: "List all restore jobs.", - Aliases: []string{"ls-restores"}, - Example: listRestoreExample, - ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), - Run: func(cmd *cobra.Command, args []string) { - o.LabelSelector = util.BuildLabelSelectorByNames(o.LabelSelector, args) - o.Names = nil - _, err := o.Run() - util.CheckErr(err) - }, - } - o.AddFlags(cmd) - return cmd -} - -func NewDeleteRestoreCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := delete.NewDeleteOptions(f, streams, types.RestoreJobGVR()) - cmd := &cobra.Command{ - Use: "delete-restore", - Short: "Delete a restore job.", - Example: deleteRestoreExample, - ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(completeForDeleteRestore(o, args)) - util.CheckErr(o.Run()) - }, - } - cmd.Flags().StringSliceVar(&o.Names, "name", []string{}, "Restore names") - o.AddFlags(cmd) - return cmd -} - -// completeForDeleteRestore complete cmd for delete restore -func completeForDeleteRestore(o *delete.DeleteOptions, args []string) error { - if len(args) == 0 { - return errors.New("Missing cluster name") - } - if len(args) > 1 { - return errors.New("Only supported delete the restore of one cluster") - } - if !o.Force && len(o.Names) == 0 { - return errors.New("Missing --name as restore name.") - } - if o.Force && len(o.Names) == 0 { - // do force action, if specified --force and not specified --name, all restores with the cluster will be deleted - // if no specify restore name and cluster name is specified. it will delete all restores with the cluster - o.LabelSelector = util.BuildLabelSelectorByNames(o.LabelSelector, args) - o.ConfirmedNames = args - } - o.ConfirmedNames = o.Names - return nil -} - func NewListBackupPolicyCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := list.NewListOptions(f, streams, types.OpsGVR()) cmd := &cobra.Command{ diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index 36d68d99e..4bafedac1 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -183,36 +183,6 @@ var _ = Describe("DataProtection", func() { Expect(o.Out.(*bytes.Buffer).String()).Should(ContainSubstring("apecloud-mysql (deleted)")) }) - It("delete-restore", func() { - By("test delete-restore cmd") - cmd := NewDeleteRestoreCmd(tf, streams) - Expect(cmd).ShouldNot(BeNil()) - - args := []string{"test1"} - clusterLabel := util.BuildLabelSelectorByNames("", args) - - By("test delete-restore with cluster") - o := delete.NewDeleteOptions(tf, streams, types.BackupGVR()) - Expect(completeForDeleteRestore(o, args)).Should(HaveOccurred()) - - By("test delete-restore with cluster and force") - o.Force = true - Expect(completeForDeleteRestore(o, args)).Should(Succeed()) - Expect(o.LabelSelector == clusterLabel).Should(BeTrue()) - - By("test delete-restore with cluster and force and labels") - o.Force = true - customLabel := "test=test" - o.LabelSelector = customLabel - Expect(completeForDeleteRestore(o, args)).Should(Succeed()) - Expect(o.LabelSelector == customLabel+","+clusterLabel).Should(BeTrue()) - }) - - It("list-restore", func() { - cmd := NewListRestoreCmd(tf, streams) - Expect(cmd).ShouldNot(BeNil()) - }) - It("restore", func() { timestamp := time.Now().Format("20060102150405") backupName := "backup-test-" + timestamp From ab54adbece7dfed85d9f4e61d21b95e04db2b5b9 Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Sun, 23 Apr 2023 15:37:48 +0800 Subject: [PATCH 151/439] refactor: redefine preflight api (#2775) --- internal/cli/cmd/kubeblocks/kubeblocks.go | 3 +- internal/cli/cmd/kubeblocks/preflight.go | 6 ++-- internal/preflight/collect.go | 41 ++++++++++------------- internal/preflight/collect_test.go | 4 +-- 4 files changed, 23 insertions(+), 31 deletions(-) diff --git a/internal/cli/cmd/kubeblocks/kubeblocks.go b/internal/cli/cmd/kubeblocks/kubeblocks.go index 04c419d1b..33ee0c830 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks.go @@ -37,8 +37,7 @@ func NewKubeBlocksCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *c newStatusCmd(f, streams), NewConfigCmd(f, streams), NewDescribeConfigCmd(f, streams), + NewPreflightCmd(f, streams), ) - // add preflight cmd - cmd.AddCommand(NewPreflightCmd(f, streams)) return cmd } diff --git a/internal/cli/cmd/kubeblocks/preflight.go b/internal/cli/cmd/kubeblocks/preflight.go index eee1f6ce0..db097e08d 100644 --- a/internal/cli/cmd/kubeblocks/preflight.go +++ b/internal/cli/cmd/kubeblocks/preflight.go @@ -134,10 +134,10 @@ func LoadVendorCheckYaml(vendorName util.K8sProvider) ([][]byte, error) { return yamlDataList, nil } -func (p *PreflightOptions) complete(factory cmdutil.Factory, args []string) error { +func (p *PreflightOptions) complete(f cmdutil.Factory, args []string) error { // default no args, and run default validating vendor if len(args) == 0 { - clientSet, err := factory.KubernetesClientSet() + clientSet, err := f.KubernetesClientSet() if err != nil { return errors.New("init k8s client failed, and please check kubeconfig") } @@ -202,7 +202,7 @@ func (p *PreflightOptions) run() error { return err } // 2. collect data - collectResults, err = kbpreflight.CollectPreflight(ctx, kbPreflight, kbHostPreflight, progressCh) + collectResults, err = kbpreflight.CollectPreflight(p.factory, ctx, kbPreflight, kbHostPreflight, progressCh) if err != nil { return err } diff --git a/internal/preflight/collect.go b/internal/preflight/collect.go index 734aac1a3..e492c378f 100644 --- a/internal/preflight/collect.go +++ b/internal/preflight/collect.go @@ -25,27 +25,25 @@ import ( "github.com/pkg/errors" troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" pkgcollector "github.com/replicatedhq/troubleshoot/pkg/collect" - "github.com/replicatedhq/troubleshoot/pkg/constants" - "github.com/replicatedhq/troubleshoot/pkg/k8sutil" "github.com/replicatedhq/troubleshoot/pkg/logger" "github.com/replicatedhq/troubleshoot/pkg/preflight" "github.com/spf13/viper" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/client-go/kubernetes" + cmdutil "k8s.io/kubectl/pkg/cmd/util" preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" kbcollector "github.com/apecloud/kubeblocks/internal/preflight/collector" ) -func CollectPreflight(ctx context.Context, kbPreflight *preflightv1beta2.Preflight, kbHostPreflight *preflightv1beta2.HostPreflight, progressCh chan interface{}) ([]preflight.CollectResult, error) { +func CollectPreflight(f cmdutil.Factory, ctx context.Context, kbPreflight *preflightv1beta2.Preflight, kbHostPreflight *preflightv1beta2.HostPreflight, progressCh chan interface{}) ([]preflight.CollectResult, error) { var ( collectResults []preflight.CollectResult err error ) // deal with preflight if kbPreflight != nil && (len(kbPreflight.Spec.ExtendCollectors) > 0 || len(kbPreflight.Spec.Collectors) > 0) { - res, err := CollectClusterData(ctx, kbPreflight, progressCh) + res, err := CollectClusterData(ctx, kbPreflight, f, progressCh) if err != nil { return collectResults, errors.Wrap(err, "failed to collect data in cluster") } @@ -61,7 +59,7 @@ func CollectPreflight(ctx context.Context, kbPreflight *preflightv1beta2.Preflig collectResults = append(collectResults, *res) } if len(kbHostPreflight.Spec.RemoteCollectors) > 0 { - res, err := CollectRemoteData(ctx, kbHostPreflight, progressCh) + res, err := CollectRemoteData(ctx, kbHostPreflight, f, progressCh) if err != nil { return collectResults, errors.Wrap(err, "failed to collect data remotely") } @@ -126,19 +124,23 @@ func CollectHost(ctx context.Context, opts preflight.CollectOpts, collectors []p } // CollectClusterData transforms the specs of Preflight to Collector, and sets the collectOpts, such as restConfig, Namespace, and ProgressChan -func CollectClusterData(ctx context.Context, kbPreflight *preflightv1beta2.Preflight, progressCh chan interface{}) (*preflight.CollectResult, error) { +func CollectClusterData(ctx context.Context, kbPreflight *preflightv1beta2.Preflight, f cmdutil.Factory, progressCh chan interface{}) (*preflight.CollectResult, error) { + var err error v := viper.GetViper() - restConfig, err := k8sutil.GetRESTConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to convert kube flags to rest config") - } - collectOpts := preflight.CollectOpts{ Namespace: v.GetString("namespace"), IgnorePermissionErrors: v.GetBool("collect-without-permissions"), ProgressChan: progressCh, - KubernetesRestConfig: restConfig, + } + + if collectOpts.KubernetesRestConfig, err = f.ToRESTConfig(); err != nil { + return nil, errors.Wrap(err, "failed to instantiate Kubernetes restconfig") + } + + k8sClient, err := f.KubernetesClientSet() + if err != nil { + return nil, errors.Wrap(err, "failed to instantiate Kubernetes client") } if v.GetString("since") != "" || v.GetString("since-time") != "" { @@ -159,14 +161,6 @@ func CollectClusterData(ctx context.Context, kbPreflight *preflightv1beta2.Prefl collectSpecs = pkgcollector.DedupCollectors(collectSpecs) collectSpecs = pkgcollector.EnsureClusterResourcesFirst(collectSpecs) - collectOpts.KubernetesRestConfig.QPS = constants.DEFAULT_CLIENT_QPS - collectOpts.KubernetesRestConfig.Burst = constants.DEFAULT_CLIENT_BURST - // collectOpts.KubernetesRestConfig.UserAgent = fmt.Sprintf("%s/%s", constants.DEFAULT_CLIENT_USER_AGENT, version.Version()) - - k8sClient, err := kubernetes.NewForConfig(collectOpts.KubernetesRestConfig) - if err != nil { - return nil, errors.Wrap(err, "failed to instantiate Kubernetes client") - } var collectors []pkgcollector.Collector allCollectorsMap := make(map[reflect.Type][]pkgcollector.Collector) for _, collectSpec := range collectSpecs { @@ -299,14 +293,13 @@ func CollectCluster(ctx context.Context, opts preflight.CollectOpts, allCollecto } collectResult.AllCollectedData = allCollectedData - return collectResult, nil } -func CollectRemoteData(ctx context.Context, preflightSpec *preflightv1beta2.HostPreflight, progressCh chan interface{}) (*preflight.CollectResult, error) { +func CollectRemoteData(ctx context.Context, preflightSpec *preflightv1beta2.HostPreflight, f cmdutil.Factory, progressCh chan interface{}) (*preflight.CollectResult, error) { v := viper.GetViper() - restConfig, err := k8sutil.GetRESTConfig() + restConfig, err := f.ToRESTConfig() if err != nil { return nil, errors.Wrap(err, "failed to convert kube flags to rest config") } diff --git a/internal/preflight/collect_test.go b/internal/preflight/collect_test.go index cd1ac9717..1d9640366 100644 --- a/internal/preflight/collect_test.go +++ b/internal/preflight/collect_test.go @@ -97,7 +97,7 @@ var _ = Describe("collect_test", func() { g.Expect(<-progressCh).NotTo(BeNil()) } }() - results, err := CollectPreflight(context.TODO(), preflight, hostPreflight, progressCh) + results, err := CollectPreflight(tf, context.TODO(), preflight, hostPreflight, progressCh) g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(results)).Should(BeNumerically(">=", 3)) }).WithTimeout(timeOut).Should(Succeed()) @@ -126,7 +126,7 @@ var _ = Describe("collect_test", func() { g.Expect(<-progressCh).NotTo(BeNil()) } }() - collectResult, err := CollectRemoteData(context.TODO(), &preflightv1beta2.HostPreflight{}, progressCh) + collectResult, err := CollectRemoteData(context.TODO(), &preflightv1beta2.HostPreflight{}, tf, progressCh) g.Expect(err).NotTo(HaveOccurred()) g.Expect(collectResult).NotTo(BeNil()) }).WithTimeout(timeOut).Should(Succeed()) From 8e39b9c21185580770e0622ef8a213dc9cad6698 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Sun, 23 Apr 2023 17:37:57 +0800 Subject: [PATCH 152/439] auto enable snapshot-controller (#2871) --- deploy/helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index d96d67d10..d16c1c990 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1608,7 +1608,7 @@ grafana: snapshot-controller: ## @param snapshot-controller.enabled -- Enable snapshot-controller chart. ## - enabled: false + enabled: true ## @param snapshot-controller.replicaCount -- Number of replicas to deploy. ## replicaCount: 1 From 33c9ebeef1b15d21373b27e279315da312a33d1c Mon Sep 17 00:00:00 2001 From: wangyelei Date: Sun, 23 Apr 2023 17:59:57 +0800 Subject: [PATCH 153/439] fix: hscale bug (#2872) --- controllers/apps/components/component_status.go | 8 +++++++- controllers/apps/operations/ops_util.go | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/controllers/apps/components/component_status.go b/controllers/apps/components/component_status.go index c7b25a0c9..aa98065f5 100644 --- a/controllers/apps/components/component_status.go +++ b/controllers/apps/components/component_status.go @@ -183,7 +183,7 @@ func (cs *ComponentStatusSynchronizer) updateComponentsPhase( if !componentIsRunning { // if no operation is running in cluster or failed pod timed out, // means the component is Failed or Abnormal. - if slices.Contains(appsv1alpha1.GetClusterUpRunningPhases(), cs.cluster.Status.Phase) || hasFailedPodTimedOut { + if clusterUpRunning(cs.cluster) || hasFailedPodTimedOut { if phase, err := cs.component.GetPhaseWhenPodsNotReady(ctx, componentName); err != nil { return err } else if phase != "" { @@ -255,3 +255,9 @@ func isContainerFailedAndTimedOut(pod *corev1.Pod, podConditionType corev1.PodCo } return time.Now().After(containerReadyCondition.LastTransitionTime.Add(types.PodContainerFailedTimeout)) } + +// clusterUpRunning checks if the cluster is up running, includes the partially running. +func clusterUpRunning(cluster *appsv1alpha1.Cluster) bool { + return cluster.Status.ObservedGeneration != cluster.Generation && + slices.Contains(appsv1alpha1.GetClusterUpRunningPhases(), cluster.Status.Phase) +} diff --git a/controllers/apps/operations/ops_util.go b/controllers/apps/operations/ops_util.go index b099c1c3e..5109ab7d7 100644 --- a/controllers/apps/operations/ops_util.go +++ b/controllers/apps/operations/ops_util.go @@ -78,7 +78,7 @@ func reconcileActionWithComponentOps(reqCtx intctrlutil.RequestCtx, if opsRequest.Status.Components == nil { opsRequest.Status.Components = map[string]appsv1alpha1.OpsRequestComponentStatus{} } - opsIsCompleted := opsRequestIsComponent(*opsRes) + opsIsCompleted := opsRequestHasProcessed(*opsRes) for k, v := range opsRes.Cluster.Status.Components { if _, ok = componentNameMap[k]; !ok && !checkAllClusterComponent { continue @@ -131,8 +131,8 @@ func reconcileActionWithComponentOps(reqCtx intctrlutil.RequestCtx, return appsv1alpha1.OpsSucceedPhase, 0, nil } -// opsRequestIsComponent checks if the opsRequest is completed. -func opsRequestIsComponent(opsRes OpsResource) bool { +// opsRequestHasProcessed checks if the opsRequest has processed. +func opsRequestHasProcessed(opsRes OpsResource) bool { return opsRes.ToClusterPhase != opsRes.Cluster.Status.Phase && opsRes.Cluster.Status.ObservedGeneration >= opsRes.OpsRequest.Status.ClusterGeneration } From 4713e9e85ce819603b2ac93f327562f85c52ef29 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Sun, 23 Apr 2023 18:01:11 +0800 Subject: [PATCH 154/439] chore: fix merge_releasing_branch.sh (#2877) --- .github/utils/merge_releasing_branch.sh | 2 +- .github/workflows/cicd-push.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/utils/merge_releasing_branch.sh b/.github/utils/merge_releasing_branch.sh index d5b9631b6..29903b7c3 100644 --- a/.github/utils/merge_releasing_branch.sh +++ b/.github/utils/merge_releasing_branch.sh @@ -13,5 +13,5 @@ workdir=$(dirname $0) set -x git switch ${BASE_BRANCH} -git merge origin/${HEAD_BRANCH} +git merge ${HEAD_BRANCH} git push \ No newline at end of file diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index 51579bbbf..899ee428a 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -24,7 +24,7 @@ jobs: steps: - uses: actions/checkout@v3 with: - fetch-depth: 2 + fetch-depth: 0 - name: Get trigger mode id: get_trigger_mode run: | From 5e8d73d3ecda1e5fd756690154594bae93cccd88 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Sun, 23 Apr 2023 19:35:10 +0800 Subject: [PATCH 155/439] chore: build tools image dockerfile add testdata (#2878) --- docker/Dockerfile-tools | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile-tools b/docker/Dockerfile-tools index 1fa3b38fb..f5c3354a0 100644 --- a/docker/Dockerfile-tools +++ b/docker/Dockerfile-tools @@ -41,6 +41,7 @@ COPY externalapis/ externalapis/ COPY version/ version/ COPY cmd/cli/ cmd/cli/ COPY apis/ apis/ +COPY test/testdata/testdata.go test/testdata/testdata.go # Download binaries RUN curl -fsSL https://dl.k8s.io/v1.26.3/kubernetes-client-${TARGETOS}-${TARGETARCH}.tar.gz | tar -zxv From a7e0017864b0b2b2eb6244d313c9f43c3a83e0f4 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Mon, 24 Apr 2023 10:21:55 +0800 Subject: [PATCH 156/439] chore: rename aws-loadbalancer-controller to aws-load-balancer-controller and add affinity (#2870) --- .../addons/aws-loadbalancer-controller-addon.yaml | 8 ++++---- .../addons/aws-loadbalancer-controller-values.yaml | 4 ++-- deploy/helm/values.yaml | 12 +++++++++++- internal/cli/cmd/playground/init.go | 6 +++--- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/deploy/helm/templates/addons/aws-loadbalancer-controller-addon.yaml b/deploy/helm/templates/addons/aws-loadbalancer-controller-addon.yaml index 14137410d..a0786248a 100644 --- a/deploy/helm/templates/addons/aws-loadbalancer-controller-addon.yaml +++ b/deploy/helm/templates/addons/aws-loadbalancer-controller-addon.yaml @@ -1,7 +1,7 @@ apiVersion: extensions.kubeblocks.io/v1alpha1 kind: Addon metadata: - name: aws-loadbalancer-controller + name: aws-load-balancer-controller labels: {{- include "kubeblocks.labels" . | nindent 4 }} {{- if .Values.keepAddons }} @@ -18,11 +18,11 @@ spec: installValues: configMapRefs: - - name: aws-loadbalancer-controller-chart-kubeblocks-values + - name: aws-load-balancer-controller-chart-kubeblocks-values key: values-kubeblocks-override.yaml setValues: - - clusterName={{ index .Values "aws-loadbalancer-controller" "clusterName" }} + - clusterName={{ index .Values "aws-load-balancer-controller" "clusterName" }} valuesMapping: valueMap: @@ -43,7 +43,7 @@ spec: - replicas: 1 installable: - autoInstall: {{ index .Values "aws-loadbalancer-controller" "enabled" }} + autoInstall: {{ index .Values "aws-load-balancer-controller" "enabled" }} selectors: - key: KubeGitVersion operator: Contains diff --git a/deploy/helm/templates/addons/aws-loadbalancer-controller-values.yaml b/deploy/helm/templates/addons/aws-loadbalancer-controller-values.yaml index 8c272438d..4a37b0e7f 100644 --- a/deploy/helm/templates/addons/aws-loadbalancer-controller-values.yaml +++ b/deploy/helm/templates/addons/aws-loadbalancer-controller-values.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: aws-loadbalancer-controller-chart-kubeblocks-values + name: aws-load-balancer-controller-chart-kubeblocks-values labels: {{- include "kubeblocks.labels" . | nindent 4 }} {{- if .Values.keepAddons }} @@ -10,4 +10,4 @@ metadata: {{- end }} data: values-kubeblocks-override.yaml: |- - {{- get ( .Values | toYaml | fromYaml ) "aws-loadbalancer-controller" | toYaml | nindent 4 }} \ No newline at end of file + {{- get ( .Values | toYaml | fromYaml ) "aws-load-balancer-controller" | toYaml | nindent 4 }} \ No newline at end of file diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index d16c1c990..3b21fe746 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1752,7 +1752,7 @@ csi-hostpath-driver: create: true default: true -aws-loadbalancer-controller: +aws-load-balancer-controller: clusterName: "" enabled: false replicaCount: 1 @@ -1764,3 +1764,13 @@ aws-loadbalancer-controller: serviceAccount: create: true name: kubeblocks-service-account-aws-load-balancer-controller + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kb-controller + operator: In + values: + - "true" diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index 6bad7f071..d96bea81c 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -427,9 +427,9 @@ func (o *initOptions) installKubeBlocks(k8sClusterName string) error { "prometheus.alertmanager.statefulSet.enabled=false") } else if o.cloudProvider == cp.AWS { insOpts.ValueOpts.Values = append(insOpts.ValueOpts.Values, - // enable aws loadbalancer controller addon automatically on playground - "aws-loadbalancer-controller.enabled=true", - fmt.Sprintf("aws-loadbalancer-controller.clusterName=%s", k8sClusterName), + // enable aws-load-balancer-controller addon automatically on playground + "aws-load-balancer-controller.enabled=true", + fmt.Sprintf("aws-load-balancer-controller.clusterName=%s", k8sClusterName), ) } From 5ba9b39716940cd75859ee443b4b0602ff828d9e Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Mon, 24 Apr 2023 10:23:25 +0800 Subject: [PATCH 157/439] chore: optimize expose validation (#2865) --- internal/cli/cmd/cluster/operations.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 152a1fb78..584bad611 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -234,6 +234,10 @@ func (o *OperationsOptions) Validate() error { if err := o.validateVScale(&cluster); err != nil { return err } + case appsv1alpha1.ExposeType: + if err := o.validateExpose(); err != nil { + return err + } } if o.RequireConfirm { return delete.Confirm([]string{o.Name}, o.In) @@ -292,6 +296,10 @@ func (o *OperationsOptions) fillExpose() error { return fmt.Errorf("unknown k8s provider") } + if err = o.CompleteComponentsFlag(); err != nil { + return err + } + // default expose to internet exposeType := util.ExposeType(o.ExposeType) if exposeType == "" { @@ -313,14 +321,6 @@ func (o *OperationsOptions) fillExpose() error { return err } - if len(o.ComponentNames) == 0 { - if len(cluster.Spec.ComponentSpecs) == 1 { - o.ComponentNames = append(o.ComponentNames, cluster.Spec.ComponentSpecs[0].Name) - } else { - return fmt.Errorf("please specify --components") - } - } - compMap := make(map[string]appsv1alpha1.ClusterComponentSpec) for _, compSpec := range cluster.Spec.ComponentSpecs { compMap[compSpec.Name] = compSpec @@ -495,7 +495,6 @@ func NewExposeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra })) _ = cmd.MarkFlagRequired("enable") } - inputs.Validate = o.validateExpose inputs.Complete = o.fillExpose return create.BuildCommand(inputs) } From 7b31aa99052be15b87eccdcac31b63e4457aa6fd Mon Sep 17 00:00:00 2001 From: yijing <107018199+ahjing99@users.noreply.github.com> Date: Mon, 24 Apr 2023 10:31:32 +0800 Subject: [PATCH 158/439] chore: Update Copyright and license (#2833) Co-authored-by: huangzhangshu --- LICENSE | 863 ++++++++++++++---- NOTICE | 8 + .../v1alpha1/backuppolicytemplate_types.go | 23 +- apis/apps/v1alpha1/cluster_types.go | 23 +- apis/apps/v1alpha1/cluster_types_test.go | 23 +- apis/apps/v1alpha1/cluster_webhook.go | 23 +- apis/apps/v1alpha1/cluster_webhook_test.go | 23 +- apis/apps/v1alpha1/clusterdefinition_types.go | 23 +- .../v1alpha1/clusterdefinition_types_test.go | 23 +- .../v1alpha1/clusterdefinition_webhook.go | 23 +- .../clusterdefinition_webhook_test.go | 23 +- apis/apps/v1alpha1/clusterversion_types.go | 23 +- .../v1alpha1/clusterversion_types_test.go | 23 +- apis/apps/v1alpha1/clusterversion_webhook.go | 23 +- .../v1alpha1/clusterversion_webhook_test.go | 23 +- .../componentclassdefinition_types.go | 23 +- .../componentresourceconstraint_types.go | 23 +- .../componentresourceconstraint_types_test.go | 23 +- apis/apps/v1alpha1/configconstraint_types.go | 23 +- apis/apps/v1alpha1/groupversion_info.go | 23 +- apis/apps/v1alpha1/opsrequest_conditions.go | 23 +- .../v1alpha1/opsrequest_conditions_test.go | 23 +- apis/apps/v1alpha1/opsrequest_types.go | 23 +- apis/apps/v1alpha1/opsrequest_types_test.go | 23 +- apis/apps/v1alpha1/opsrequest_webhook.go | 23 +- apis/apps/v1alpha1/opsrequest_webhook_test.go | 23 +- apis/apps/v1alpha1/type.go | 23 +- apis/apps/v1alpha1/webhook_suite_test.go | 23 +- apis/dataprotection/v1alpha1/backup_types.go | 23 +- .../v1alpha1/backup_types_test.go | 23 +- .../v1alpha1/backuppolicy_types.go | 23 +- .../v1alpha1/backuptool_types.go | 23 +- .../v1alpha1/groupversion_info.go | 23 +- .../v1alpha1/restorejob_types.go | 23 +- apis/dataprotection/v1alpha1/types.go | 23 +- apis/extensions/v1alpha1/addon_types.go | 23 +- apis/extensions/v1alpha1/addon_types_test.go | 23 +- apis/extensions/v1alpha1/groupversion_info.go | 23 +- apis/extensions/v1alpha1/type.go | 23 +- cmd/cli/main.go | 23 +- cmd/cmd.mk | 26 +- cmd/manager/main.go | 23 +- cmd/probe/internal/binding/base.go | 23 +- cmd/probe/internal/binding/base_test.go | 23 +- cmd/probe/internal/binding/etcd/etcd.go | 23 +- cmd/probe/internal/binding/etcd/etcd_test.go | 23 +- cmd/probe/internal/binding/mongodb/mongodb.go | 27 +- .../internal/binding/mongodb/mongodb_test.go | 27 +- cmd/probe/internal/binding/mysql/mysql.go | 23 +- .../internal/binding/mysql/mysql_test.go | 23 +- .../internal/binding/postgres/postgres.go | 23 +- .../binding/postgres/postgres_test.go | 23 +- cmd/probe/internal/binding/redis/redis.go | 23 +- .../internal/binding/redis/redis_test.go | 23 +- cmd/probe/internal/binding/types.go | 23 +- cmd/probe/internal/binding/utils.go | 23 +- .../internal/component/redis/metadata.go | 23 +- cmd/probe/internal/component/redis/redis.go | 23 +- .../internal/component/redis/redis_test.go | 23 +- .../internal/component/redis/settings.go | 23 +- .../http/probe/checks_middleware.go | 23 +- .../http/probe/checks_middleware_test.go | 23 +- cmd/probe/main.go | 23 +- cmd/probe/probe.proto | 23 +- cmd/probe/util/const.go | 23 +- cmd/reloader/app/cmd.go | 23 +- cmd/reloader/app/flags.go | 23 +- cmd/reloader/app/proxy.go | 23 +- cmd/reloader/container_killer/killer.go | 23 +- cmd/reloader/main.go | 23 +- cmd/reloader/tools/cue_auto_generator.go | 23 +- cmd/tpl/app/helm_helper.go | 23 +- cmd/tpl/app/k8s_resource.go | 23 +- cmd/tpl/app/mock_client.go | 23 +- cmd/tpl/app/util.go | 23 +- cmd/tpl/app/workflow.go | 23 +- cmd/tpl/main.go | 23 +- controllers/apps/class_controller.go | 23 +- controllers/apps/class_controller_test.go | 23 +- controllers/apps/cluster_controller.go | 23 +- controllers/apps/cluster_controller_test.go | 23 +- controllers/apps/cluster_status_utils.go | 23 +- controllers/apps/cluster_status_utils_test.go | 23 +- .../apps/clusterdefinition_controller.go | 23 +- .../apps/clusterdefinition_controller_test.go | 23 +- controllers/apps/clusterversion_controller.go | 23 +- .../apps/clusterversion_controller_test.go | 23 +- controllers/apps/components/component.go | 23 +- .../apps/components/component_status.go | 23 +- .../apps/components/component_status_test.go | 23 +- .../apps/components/consensus/consensus.go | 23 +- .../components/consensus/consensus_test.go | 23 +- .../components/consensus/consensus_utils.go | 23 +- .../consensus/consensus_utils_test.go | 23 +- .../apps/components/consensus/suite_test.go | 23 +- .../apps/components/deployment_controller.go | 23 +- .../components/deployment_controller_test.go | 23 +- controllers/apps/components/pod_controller.go | 23 +- .../apps/components/pod_controller_test.go | 23 +- .../components/replication/replication.go | 23 +- .../replication/replication_switch.go | 23 +- .../replication/replication_switch_test.go | 23 +- .../replication/replication_switch_utils.go | 23 +- .../replication_switch_utils_test.go | 23 +- .../replication/replication_test.go | 23 +- .../replication/replication_utils.go | 23 +- .../replication/replication_utils_test.go | 23 +- .../apps/components/replication/suite_test.go | 23 +- .../apps/components/stateful/stateful.go | 23 +- .../apps/components/stateful/stateful_test.go | 23 +- .../apps/components/stateful/suite_test.go | 23 +- .../components/stateful_set_controller.go | 23 +- .../stateful_set_controller_test.go | 23 +- .../apps/components/stateless/stateless.go | 23 +- .../components/stateless/stateless_test.go | 23 +- .../apps/components/stateless/suite_test.go | 23 +- controllers/apps/components/suite_test.go | 23 +- .../apps/components/types/component.go | 23 +- .../apps/components/util/component_utils.go | 23 +- .../components/util/component_utils_test.go | 23 +- controllers/apps/components/util/plan.go | 23 +- controllers/apps/components/util/plan_test.go | 23 +- .../components/util/stateful_set_utils.go | 24 +- .../util/stateful_set_utils_test.go | 24 +- .../apps/components/util/suite_test.go | 23 +- .../apps/configuration/config_annotation.go | 23 +- controllers/apps/configuration/config_util.go | 23 +- .../apps/configuration/config_util_test.go | 23 +- .../configconstraint_controller.go | 23 +- .../configconstraint_controller_test.go | 23 +- .../configuration/parallel_upgrade_policy.go | 23 +- .../parallel_upgrade_policy_test.go | 23 +- controllers/apps/configuration/policy_util.go | 23 +- .../apps/configuration/policy_util_test.go | 23 +- .../apps/configuration/reconfigure_policy.go | 23 +- .../reconfigurerequest_controller.go | 23 +- .../reconfigurerequest_controller_test.go | 23 +- .../configuration/rolling_upgrade_policy.go | 23 +- .../rolling_upgrade_policy_test.go | 23 +- .../apps/configuration/simple_policy.go | 23 +- .../apps/configuration/simple_policy_test.go | 23 +- controllers/apps/configuration/suite_test.go | 23 +- .../apps/configuration/sync_upgrade_policy.go | 23 +- .../configuration/sync_upgrade_policy_test.go | 23 +- controllers/apps/const.go | 23 +- controllers/apps/operations/expose.go | 23 +- controllers/apps/operations/expose_test.go | 23 +- .../apps/operations/horizontal_scaling.go | 23 +- .../operations/horizontal_scaling_test.go | 23 +- controllers/apps/operations/ops_manager.go | 23 +- .../apps/operations/ops_progress_util.go | 23 +- .../apps/operations/ops_progress_util_test.go | 23 +- controllers/apps/operations/ops_util.go | 23 +- controllers/apps/operations/ops_util_test.go | 23 +- controllers/apps/operations/reconfigure.go | 23 +- .../apps/operations/reconfigure_test.go | 23 +- .../apps/operations/reconfigure_util.go | 23 +- .../apps/operations/reconfigure_util_test.go | 23 +- controllers/apps/operations/restart.go | 23 +- controllers/apps/operations/restart_test.go | 23 +- controllers/apps/operations/start.go | 23 +- controllers/apps/operations/start_test.go | 23 +- controllers/apps/operations/stop.go | 23 +- controllers/apps/operations/stop_test.go | 23 +- controllers/apps/operations/suite_test.go | 23 +- controllers/apps/operations/type.go | 23 +- controllers/apps/operations/upgrade.go | 23 +- controllers/apps/operations/upgrade_test.go | 23 +- .../apps/operations/util/common_util.go | 23 +- .../apps/operations/util/common_util_test.go | 23 +- .../apps/operations/util/suite_test.go | 23 +- .../apps/operations/vertical_scaling.go | 23 +- .../apps/operations/vertical_scaling_test.go | 23 +- .../apps/operations/volume_expansion.go | 23 +- .../apps/operations/volume_expansion_test.go | 23 +- .../operations/volume_expansion_updater.go | 23 +- controllers/apps/opsrequest_controller.go | 23 +- .../apps/opsrequest_controller_test.go | 23 +- controllers/apps/suite_test.go | 23 +- controllers/apps/systemaccount_controller.go | 23 +- .../apps/systemaccount_controller_test.go | 23 +- controllers/apps/systemaccount_util.go | 23 +- controllers/apps/systemaccount_util_test.go | 23 +- controllers/apps/tls_utils_test.go | 23 +- controllers/apps/utils.go | 23 +- .../dataprotection/backup_controller.go | 23 +- .../dataprotection/backup_controller_test.go | 23 +- .../dataprotection/backuppolicy_controller.go | 23 +- .../backuppolicy_controller_test.go | 23 +- .../dataprotection/backuptool_controller.go | 23 +- .../dataprotection/cronjob_controller.go | 23 +- controllers/dataprotection/cue/cronjob.cue | 23 +- .../dataprotection/restorejob_controller.go | 23 +- .../restorejob_controller_test.go | 23 +- controllers/dataprotection/suite_test.go | 23 +- controllers/dataprotection/type.go | 23 +- controllers/dataprotection/utils.go | 23 +- controllers/extensions/addon_controller.go | 23 +- .../extensions/addon_controller_stages.go | 23 +- .../extensions/addon_controller_test.go | 23 +- controllers/extensions/const.go | 23 +- controllers/extensions/suite_test.go | 23 +- controllers/k8score/const.go | 23 +- controllers/k8score/event_controller.go | 23 +- controllers/k8score/event_controller_test.go | 23 +- controllers/k8score/event_utils.go | 23 +- controllers/k8score/pvc_controller.go | 23 +- controllers/k8score/pvc_controller_test.go | 23 +- controllers/k8score/suite_test.go | 23 +- .../config/mysql8-config-constraint.cue | 23 +- .../config/mysql8-config-constraint.cue | 23 +- .../config/pg14-config-constraint.cue | 23 +- .../redis/config/redis7-config-constraint.cue | 23 +- docker/docker.mk | 26 +- .../preflight/v1beta2/groupversion_info.go | 23 +- .../preflight/v1beta2/hostpreflight_types.go | 23 +- .../preflight/v1beta2/preflight_types.go | 26 +- externalapis/preflight/v1beta2/type.go | 26 +- hack/boilerplate.cue.txt | 23 +- hack/boilerplate.go.txt | 23 +- hack/docgen/cli/main.go | 23 +- hack/install_cli.sh | 23 +- hack/install_cli_docker.sh | 23 +- hack/license/header-check.sh | 29 +- internal/class/class_utils.go | 23 +- internal/class/class_utils_test.go | 23 +- internal/class/suite_test.go | 23 +- internal/class/types.go | 23 +- internal/class/types_test.go | 23 +- internal/cli/cloudprovider/interface.go | 23 +- internal/cli/cloudprovider/k3d.go | 23 +- internal/cli/cloudprovider/k3d_test.go | 23 +- internal/cli/cloudprovider/provider.go | 23 +- internal/cli/cloudprovider/provider_test.go | 23 +- internal/cli/cloudprovider/suite_test.go | 23 +- internal/cli/cloudprovider/terraform.go | 23 +- internal/cli/cloudprovider/terraform_test.go | 23 +- internal/cli/cloudprovider/types.go | 23 +- internal/cli/cluster/cluster.go | 23 +- internal/cli/cluster/cluster_test.go | 23 +- internal/cli/cluster/helper.go | 23 +- internal/cli/cluster/helper_test.go | 23 +- internal/cli/cluster/name_generator.go | 23 +- internal/cli/cluster/name_generator_test.go | 23 +- internal/cli/cluster/printer.go | 23 +- internal/cli/cluster/printer_test.go | 23 +- internal/cli/cluster/suite_test.go | 23 +- internal/cli/cluster/types.go | 23 +- internal/cli/cmd/accounts/base.go | 23 +- internal/cli/cmd/accounts/base_test.go | 23 +- internal/cli/cmd/accounts/create.go | 23 +- internal/cli/cmd/accounts/create_test.go | 23 +- internal/cli/cmd/accounts/delete.go | 23 +- internal/cli/cmd/accounts/delete_test.go | 23 +- internal/cli/cmd/accounts/describe.go | 23 +- internal/cli/cmd/accounts/describe_test.go | 23 +- internal/cli/cmd/accounts/grant.go | 23 +- internal/cli/cmd/accounts/grant_test.go | 23 +- internal/cli/cmd/accounts/list.go | 23 +- internal/cli/cmd/accounts/list_test.go | 23 +- internal/cli/cmd/accounts/suite_test.go | 23 +- internal/cli/cmd/accounts/util.go | 23 +- internal/cli/cmd/addon/addon.go | 23 +- internal/cli/cmd/addon/addon_test.go | 23 +- internal/cli/cmd/addon/suite_test.go | 23 +- internal/cli/cmd/alert/add_receiver.go | 23 +- internal/cli/cmd/alert/add_receiver_test.go | 23 +- internal/cli/cmd/alert/alert_test.go | 23 +- internal/cli/cmd/alert/alter.go | 23 +- internal/cli/cmd/alert/delete_receiver.go | 23 +- .../cli/cmd/alert/delete_receiver_test.go | 23 +- internal/cli/cmd/alert/list_receivers.go | 23 +- internal/cli/cmd/alert/list_receivers_test.go | 23 +- internal/cli/cmd/alert/suite_test.go | 23 +- internal/cli/cmd/alert/types.go | 23 +- internal/cli/cmd/alert/util.go | 23 +- internal/cli/cmd/alert/util_test.go | 23 +- internal/cli/cmd/bench/bench.go | 23 +- internal/cli/cmd/bench/bench_test.go | 23 +- internal/cli/cmd/bench/suite_test.go | 23 +- internal/cli/cmd/bench/tpcc.go | 23 +- internal/cli/cmd/bench/util.go | 23 +- internal/cli/cmd/class/class.go | 23 +- internal/cli/cmd/class/class_test.go | 23 +- internal/cli/cmd/class/create.go | 23 +- internal/cli/cmd/class/create_test.go | 23 +- internal/cli/cmd/class/list.go | 23 +- internal/cli/cmd/class/list_test.go | 23 +- internal/cli/cmd/class/suite_test.go | 23 +- internal/cli/cmd/class/template.go | 23 +- internal/cli/cmd/class/template_test.go | 23 +- internal/cli/cmd/cli.go | 23 +- internal/cli/cmd/cluster/accounts.go | 23 +- internal/cli/cmd/cluster/cluster.go | 23 +- internal/cli/cmd/cluster/cluster_test.go | 23 +- internal/cli/cmd/cluster/config.go | 23 +- internal/cli/cmd/cluster/config_edit.go | 23 +- internal/cli/cmd/cluster/config_ops.go | 23 +- internal/cli/cmd/cluster/config_ops_test.go | 23 +- internal/cli/cmd/cluster/config_util.go | 23 +- internal/cli/cmd/cluster/config_util_test.go | 23 +- internal/cli/cmd/cluster/config_wrapper.go | 23 +- internal/cli/cmd/cluster/connect.go | 23 +- internal/cli/cmd/cluster/connect_test.go | 23 +- internal/cli/cmd/cluster/create.go | 23 +- internal/cli/cmd/cluster/create_test.go | 23 +- internal/cli/cmd/cluster/dataprotection.go | 23 +- .../cli/cmd/cluster/dataprotection_test.go | 23 +- internal/cli/cmd/cluster/delete.go | 23 +- internal/cli/cmd/cluster/delete_ops.go | 23 +- internal/cli/cmd/cluster/describe.go | 23 +- internal/cli/cmd/cluster/describe_ops.go | 23 +- internal/cli/cmd/cluster/describe_ops_test.go | 23 +- internal/cli/cmd/cluster/describe_test.go | 23 +- internal/cli/cmd/cluster/errors.go | 23 +- internal/cli/cmd/cluster/label.go | 23 +- internal/cli/cmd/cluster/label_test.go | 23 +- internal/cli/cmd/cluster/list.go | 23 +- internal/cli/cmd/cluster/list_logs.go | 23 +- internal/cli/cmd/cluster/list_logs_test.go | 23 +- internal/cli/cmd/cluster/list_ops.go | 23 +- internal/cli/cmd/cluster/list_ops_test.go | 24 +- internal/cli/cmd/cluster/list_test.go | 23 +- internal/cli/cmd/cluster/logs.go | 23 +- internal/cli/cmd/cluster/logs_test.go | 23 +- internal/cli/cmd/cluster/operations.go | 23 +- internal/cli/cmd/cluster/operations_test.go | 23 +- internal/cli/cmd/cluster/suite_test.go | 23 +- internal/cli/cmd/cluster/update.go | 23 +- internal/cli/cmd/cluster/update_test.go | 23 +- .../clusterdefinition/clusterdefinition.go | 23 +- .../clusterdefinition_test.go | 23 +- .../cli/cmd/clusterdefinition/suite_test.go | 23 +- .../cli/cmd/clusterversion/clusterversion.go | 23 +- .../cmd/clusterversion/clusterversion_test.go | 23 +- internal/cli/cmd/clusterversion/suite_test.go | 23 +- internal/cli/cmd/dashboard/dashboard.go | 23 +- internal/cli/cmd/dashboard/dashboard_test.go | 23 +- internal/cli/cmd/dashboard/suite_test.go | 23 +- internal/cli/cmd/kubeblocks/config.go | 23 +- internal/cli/cmd/kubeblocks/config_test.go | 23 +- internal/cli/cmd/kubeblocks/install.go | 23 +- internal/cli/cmd/kubeblocks/install_test.go | 23 +- internal/cli/cmd/kubeblocks/kubeblocks.go | 23 +- .../cli/cmd/kubeblocks/kubeblocks_objects.go | 23 +- .../cmd/kubeblocks/kubeblocks_objects_test.go | 23 +- .../cli/cmd/kubeblocks/kubeblocks_test.go | 23 +- internal/cli/cmd/kubeblocks/list_versions.go | 23 +- .../cli/cmd/kubeblocks/list_versions_test.go | 23 +- internal/cli/cmd/kubeblocks/preflight.go | 23 +- internal/cli/cmd/kubeblocks/preflight_test.go | 23 +- internal/cli/cmd/kubeblocks/status.go | 23 +- internal/cli/cmd/kubeblocks/status_test.go | 23 +- internal/cli/cmd/kubeblocks/suite_test.go | 23 +- internal/cli/cmd/kubeblocks/uninstall.go | 23 +- internal/cli/cmd/kubeblocks/uninstall_test.go | 23 +- internal/cli/cmd/kubeblocks/upgrade.go | 23 +- internal/cli/cmd/kubeblocks/upgrade_test.go | 23 +- internal/cli/cmd/kubeblocks/util.go | 23 +- internal/cli/cmd/kubeblocks/util_test.go | 23 +- internal/cli/cmd/migration/base.go | 23 +- internal/cli/cmd/migration/base_test.go | 23 +- internal/cli/cmd/migration/cmd_builder.go | 23 +- .../cli/cmd/migration/cmd_builder_test.go | 23 +- internal/cli/cmd/migration/create.go | 23 +- internal/cli/cmd/migration/create_test.go | 23 +- internal/cli/cmd/migration/describe.go | 23 +- internal/cli/cmd/migration/describe_test.go | 23 +- internal/cli/cmd/migration/examples.go | 23 +- internal/cli/cmd/migration/list.go | 23 +- internal/cli/cmd/migration/list_test.go | 23 +- internal/cli/cmd/migration/logs.go | 23 +- internal/cli/cmd/migration/logs_test.go | 23 +- internal/cli/cmd/migration/suite_test.go | 23 +- internal/cli/cmd/migration/templates.go | 23 +- internal/cli/cmd/migration/templates_test.go | 23 +- internal/cli/cmd/migration/terminate.go | 23 +- internal/cli/cmd/migration/terminate_test.go | 23 +- internal/cli/cmd/options/options.go | 23 +- internal/cli/cmd/options/options_test.go | 23 +- internal/cli/cmd/playground/base.go | 23 +- internal/cli/cmd/playground/destroy.go | 23 +- internal/cli/cmd/playground/destroy_test.go | 23 +- internal/cli/cmd/playground/init.go | 23 +- internal/cli/cmd/playground/init_test.go | 23 +- internal/cli/cmd/playground/kubeconfig.go | 23 +- .../cli/cmd/playground/kubeconfig_test.go | 23 +- internal/cli/cmd/playground/palyground.go | 23 +- .../cli/cmd/playground/playground_test.go | 23 +- internal/cli/cmd/playground/suite_test.go | 23 +- internal/cli/cmd/playground/types.go | 23 +- internal/cli/cmd/playground/util.go | 23 +- internal/cli/cmd/playground/util_test.go | 23 +- internal/cli/cmd/version/suite_test.go | 23 +- internal/cli/cmd/version/version.go | 23 +- internal/cli/cmd/version/version_test.go | 23 +- internal/cli/create/create.go | 23 +- internal/cli/create/create_test.go | 23 +- internal/cli/create/suite_test.go | 23 +- .../cli/create/template/backup_template.cue | 23 +- .../template/cluster_operations_template.cue | 23 +- .../cli/create/template/cluster_template.cue | 23 +- .../create/template/create_template_test.cue | 23 +- .../create/template/migration_template.cue | 23 +- .../template/volumesnapshotclass_template.cue | 23 +- internal/cli/delete/delete.go | 23 +- internal/cli/delete/delete_test.go | 23 +- internal/cli/delete/suite_test.go | 23 +- internal/cli/edit/edit.go | 23 +- internal/cli/edit/edit_test.go | 23 +- internal/cli/edit/suite_test.go | 23 +- internal/cli/exec/exec.go | 23 +- internal/cli/exec/exec_test.go | 23 +- internal/cli/exec/suite_test.go | 23 +- internal/cli/list/list.go | 23 +- internal/cli/list/list_test.go | 23 +- internal/cli/list/suite_test.go | 23 +- internal/cli/patch/patch.go | 23 +- internal/cli/patch/patch_test.go | 23 +- internal/cli/patch/suite_test.go | 23 +- internal/cli/printer/describe.go | 23 +- internal/cli/printer/describe_test.go | 23 +- internal/cli/printer/format.go | 23 +- internal/cli/printer/format_test.go | 23 +- internal/cli/printer/helper.go | 23 +- internal/cli/printer/printer.go | 23 +- internal/cli/printer/printer_test.go | 23 +- internal/cli/printer/spinner.go | 23 +- internal/cli/printer/spinner_test.go | 23 +- internal/cli/testing/client.go | 23 +- internal/cli/testing/factory.go | 23 +- internal/cli/testing/factory_test.go | 23 +- internal/cli/testing/fake.go | 23 +- internal/cli/testing/fake_test.go | 23 +- internal/cli/testing/printer.go | 23 +- internal/cli/testing/suite_test.go | 23 +- .../migrationapi/migration_object_express.go | 23 +- .../types/migrationapi/migrationtask_types.go | 23 +- .../migrationapi/migrationtemplate_types.go | 23 +- internal/cli/types/migrationapi/type.go | 23 +- internal/cli/types/types.go | 23 +- internal/cli/util/completion.go | 23 +- internal/cli/util/error.go | 23 +- internal/cli/util/error_test.go | 23 +- internal/cli/util/git.go | 23 +- internal/cli/util/helm/config.go | 23 +- internal/cli/util/helm/errors.go | 23 +- internal/cli/util/helm/helm.go | 23 +- internal/cli/util/helm/helm_test.go | 23 +- internal/cli/util/helm/suite_test.go | 23 +- internal/cli/util/prompt/prompt.go | 23 +- internal/cli/util/prompt/prompt_test.go | 23 +- internal/cli/util/provider.go | 23 +- internal/cli/util/provider_test.go | 23 +- internal/cli/util/suite_test.go | 23 +- internal/cli/util/util.go | 23 +- internal/cli/util/util_test.go | 23 +- internal/cli/util/version.go | 23 +- internal/cli/util/version_test.go | 23 +- internal/configuration/config.go | 23 +- .../configuration/config_manager/builder.go | 23 +- .../config_manager/builder_test.go | 23 +- .../dynamic_parameter_updater.go | 23 +- .../configuration/config_manager/files.go | 23 +- .../config_manager/files_test.go | 23 +- .../configuration/config_manager/handler.go | 23 +- .../config_manager/handler_test.go | 23 +- .../config_manager/handler_util.go | 23 +- .../config_manager/handler_util_test.go | 23 +- .../config_manager/reload_util.go | 23 +- .../config_manager/reload_util_test.go | 23 +- .../configuration/config_manager/signal.go | 23 +- .../config_manager/signal_darwin.go | 23 +- .../config_manager/signal_linux.go | 23 +- .../config_manager/signal_test.go | 23 +- .../config_manager/signal_windows.go | 23 +- .../config_manager/suite_test.go | 23 +- .../config_manager/volume_watcher.go | 23 +- .../config_manager/volume_watcher_test.go | 23 +- internal/configuration/config_patch.go | 23 +- internal/configuration/config_patch_option.go | 23 +- .../configuration/config_patch_option_test.go | 23 +- internal/configuration/config_patch_test.go | 23 +- internal/configuration/config_patch_util.go | 23 +- .../configuration/config_patch_util_test.go | 23 +- internal/configuration/config_query.go | 23 +- internal/configuration/config_query_test.go | 23 +- internal/configuration/config_test.go | 23 +- internal/configuration/config_util.go | 23 +- internal/configuration/config_util_test.go | 23 +- internal/configuration/config_validate.go | 23 +- .../configuration/config_validate_test.go | 29 +- internal/configuration/configtemplate_util.go | 23 +- .../configuration/configtemplate_util_test.go | 23 +- internal/configuration/constraint.go | 23 +- .../configuration/container/container_kill.go | 23 +- .../container/container_kill_test.go | 23 +- .../configuration/container/container_util.go | 23 +- .../container/container_util_test.go | 23 +- .../configuration/container/mocks/generate.go | 23 +- internal/configuration/container/type.go | 23 +- internal/configuration/cue_gen_openapi.go | 23 +- .../configuration/cue_gen_openapi_test.go | 23 +- internal/configuration/cue_util.go | 23 +- internal/configuration/cue_util_test.go | 23 +- internal/configuration/cue_visitor.go | 23 +- internal/configuration/cue_visitor_test.go | 23 +- internal/configuration/cuelang_expansion.go | 23 +- internal/configuration/error.go | 23 +- internal/configuration/proto/generate.go | 23 +- .../configuration/proto/mocks/generate.go | 23 +- internal/configuration/reconfigure_util.go | 23 +- .../configuration/reconfigure_util_test.go | 23 +- internal/configuration/suite_test.go | 23 +- internal/configuration/type.go | 23 +- internal/configuration/util/file_util.go | 23 +- internal/configuration/util/hash.go | 23 +- internal/configuration/util/hash_test.go | 23 +- internal/configuration/util/jsonpath.go | 23 +- internal/configuration/util/math.go | 23 +- internal/configuration/util/math_test.go | 23 +- internal/configuration/util/set.go | 23 +- internal/configuration/util/set_test.go | 23 +- internal/configuration/util/unstructured.go | 23 +- .../configuration/util/unstructured_test.go | 23 +- internal/constant/const.go | 23 +- internal/controller/builder/builder.go | 23 +- internal/controller/builder/builder_test.go | 23 +- .../builder/cue/backup_job_template.cue | 23 +- .../builder/cue/backup_manifests_template.cue | 23 +- .../builder/cue/config_manager_sidecar.cue | 23 +- .../builder/cue/config_template.cue | 23 +- .../builder/cue/conn_credential_template.cue | 23 +- .../cue/delete_pvc_cron_job_template.cue | 23 +- .../builder/cue/deployment_template.cue | 23 +- .../builder/cue/env_config_template.cue | 23 +- .../builder/cue/headless_service_template.cue | 23 +- .../controller/builder/cue/pdb_template.cue | 23 +- .../builder/cue/pitr_job_template.cue | 23 +- .../controller/builder/cue/pvc_template.cue | 23 +- .../builder/cue/service_template.cue | 23 +- .../builder/cue/snapshot_template.cue | 23 +- .../builder/cue/statefulset_template.cue | 23 +- .../builder/cue/tls_certs_secret_template.cue | 23 +- internal/controller/builder/suite_test.go | 23 +- internal/controller/client/readonly_client.go | 23 +- .../controller/component/affinity_utils.go | 23 +- .../component/affinity_utils_test.go | 23 +- internal/controller/component/component.go | 23 +- .../controller/component/component_test.go | 23 +- .../component/cue/probe_template.cue | 23 +- .../controller/component/monitor_utils.go | 23 +- .../component/monitor_utils_test.go | 23 +- internal/controller/component/port_utils.go | 23 +- .../controller/component/port_utils_test.go | 23 +- internal/controller/component/probe_utils.go | 23 +- .../controller/component/probe_utils_test.go | 23 +- .../controller/component/restore_utils.go | 23 +- .../component/restore_utils_test.go | 23 +- internal/controller/component/suite_test.go | 23 +- internal/controller/component/type.go | 23 +- internal/controller/graph/dag.go | 23 +- internal/controller/graph/dag_test.go | 23 +- internal/controller/graph/doc.go | 23 +- internal/controller/graph/plan_builder.go | 23 +- internal/controller/graph/transformer.go | 23 +- .../lifecycle/cluster_plan_builder.go | 23 +- .../lifecycle/cluster_plan_utils.go | 23 +- .../lifecycle/cluster_plan_utils_test.go | 23 +- .../lifecycle/cluster_status_conditions.go | 23 +- internal/controller/lifecycle/suite_test.go | 23 +- .../controller/lifecycle/transform_types.go | 23 +- .../controller/lifecycle/transform_utils.go | 23 +- .../lifecycle/transform_utils_test.go | 23 +- .../transformer_backup_policy_tpl.go | 23 +- .../lifecycle/transformer_cluster.go | 23 +- .../lifecycle/transformer_cluster_status.go | 23 +- .../lifecycle/transformer_config.go | 23 +- .../lifecycle/transformer_credential.go | 23 +- .../lifecycle/transformer_fill_class.go | 23 +- .../lifecycle/transformer_fix_meta.go | 23 +- .../controller/lifecycle/transformer_init.go | 23 +- .../lifecycle/transformer_object_action.go | 23 +- .../lifecycle/transformer_ownership.go | 23 +- .../transformer_sts_horizontal_scaling.go | 23 +- ...transformer_sts_horizontal_scaling_test.go | 23 +- .../lifecycle/transformer_sts_pvc.go | 23 +- .../lifecycle/transformer_tls_certs.go | 23 +- .../lifecycle/transformers_parallel.go | 23 +- internal/controller/plan/builtin_env.go | 23 +- internal/controller/plan/builtin_env_test.go | 23 +- internal/controller/plan/builtin_functions.go | 23 +- internal/controller/plan/config_template.go | 23 +- .../controller/plan/config_template_test.go | 23 +- internal/controller/plan/pitr.go | 23 +- internal/controller/plan/pitr_test.go | 23 +- internal/controller/plan/prepare.go | 23 +- internal/controller/plan/prepare_test.go | 23 +- internal/controller/plan/suite_test.go | 23 +- internal/controller/plan/template_wrapper.go | 23 +- internal/controller/plan/tls_utils.go | 23 +- internal/controller/types/task.go | 23 +- internal/controllerutil/container_util.go | 23 +- internal/controllerutil/controller_common.go | 23 +- .../controllerutil/controller_common_test.go | 23 +- internal/controllerutil/cue_value.go | 23 +- internal/controllerutil/cue_value_test.go | 23 +- internal/controllerutil/errors.go | 23 +- internal/controllerutil/errors_test.go | 23 +- internal/controllerutil/pod_utils.go | 23 +- internal/controllerutil/pod_utils_test.go | 23 +- internal/controllerutil/suite_test.go | 23 +- internal/controllerutil/task.go | 23 +- internal/controllerutil/type.go | 23 +- internal/controllerutil/types_util.go | 23 +- internal/controllerutil/volume_util.go | 23 +- internal/controllerutil/volume_util_test.go | 23 +- internal/generics/type.go | 23 +- internal/gotemplate/functional.go | 23 +- internal/gotemplate/suite_test.go | 23 +- internal/gotemplate/tpl_engine.go | 23 +- internal/gotemplate/tpl_engine_test.go | 23 +- internal/preflight/analyze.go | 23 +- internal/preflight/analyze_test.go | 23 +- internal/preflight/analyzer/access.go | 23 +- internal/preflight/analyzer/access_test.go | 23 +- internal/preflight/analyzer/analyze_result.go | 23 +- .../preflight/analyzer/analyze_result_test.go | 23 +- internal/preflight/analyzer/analyzer.go | 23 +- internal/preflight/analyzer/anzlyzer_test.go | 23 +- internal/preflight/analyzer/host_analyzer.go | 23 +- .../preflight/analyzer/host_analyzer_test.go | 23 +- internal/preflight/analyzer/host_region.go | 23 +- .../preflight/analyzer/host_region_test.go | 23 +- internal/preflight/analyzer/host_utility.go | 23 +- .../preflight/analyzer/host_utility_test.go | 23 +- .../preflight/analyzer/kb_storage_class.go | 23 +- .../analyzer/kb_storage_class_test.go | 23 +- internal/preflight/analyzer/suite_test.go | 23 +- internal/preflight/collect.go | 23 +- internal/preflight/collect_test.go | 23 +- .../preflight/collector/host_collector.go | 23 +- .../collector/host_collector_test.go | 23 +- internal/preflight/collector/host_region.go | 23 +- .../preflight/collector/host_region_test.go | 23 +- internal/preflight/collector/host_utility.go | 23 +- .../preflight/collector/host_utility_test.go | 23 +- internal/preflight/collector/suite_test.go | 23 +- internal/preflight/concat_spec.go | 23 +- internal/preflight/concat_spec_test.go | 23 +- internal/preflight/interactive/interactive.go | 23 +- internal/preflight/load_spec.go | 23 +- internal/preflight/load_spec_test.go | 23 +- internal/preflight/suite_test.go | 23 +- internal/preflight/testing/fake.go | 23 +- internal/preflight/testing/fake_test.go | 23 +- internal/preflight/testing/suite_test.go | 23 +- internal/preflight/text_results.go | 23 +- internal/preflight/text_results_test.go | 23 +- internal/preflight/util/schema.go | 26 +- internal/preflight/util/suite_test.go | 23 +- internal/preflight/util/util.go | 23 +- internal/preflight/util/util_test.go | 23 +- internal/sqlchannel/client.go | 23 +- internal/sqlchannel/client_test.go | 23 +- internal/sqlchannel/engine/client.go | 23 +- internal/sqlchannel/engine/engine.go | 23 +- internal/sqlchannel/engine/engine_test.go | 23 +- internal/sqlchannel/engine/mongodb.go | 23 +- internal/sqlchannel/engine/mysql.go | 23 +- internal/sqlchannel/engine/mysql_test.go | 23 +- internal/sqlchannel/engine/postgresql.go | 23 +- internal/sqlchannel/engine/redis.go | 23 +- internal/sqlchannel/engine/suite_test.go | 23 +- internal/sqlchannel/types.go | 23 +- internal/testutil/apps/backup_factory.go | 23 +- .../testutil/apps/backuppolicy_factory.go | 23 +- .../apps/backuppolicytemplate_factory.go | 23 +- internal/testutil/apps/base_factory.go | 23 +- .../apps/cluster_consensus_test_util.go | 23 +- internal/testutil/apps/cluster_factory.go | 23 +- .../apps/cluster_replication_test_util.go | 23 +- .../apps/cluster_stateless_test_util.go | 23 +- internal/testutil/apps/cluster_util.go | 23 +- internal/testutil/apps/clusterdef_factory.go | 23 +- .../testutil/apps/clusterversion_factory.go | 23 +- internal/testutil/apps/common_util.go | 23 +- .../apps/componentclassdefinition_factory.go | 23 +- .../componentresourceconstraint_factory.go | 23 +- internal/testutil/apps/constant.go | 23 +- internal/testutil/apps/deployment_factoy.go | 23 +- internal/testutil/apps/native_object_util.go | 23 +- internal/testutil/apps/opsrequest_util.go | 23 +- internal/testutil/apps/pod_factory.go | 23 +- internal/testutil/apps/pvc_factoy.go | 23 +- internal/testutil/apps/restorejob_factory.go | 23 +- internal/testutil/apps/statefulset_factoy.go | 23 +- internal/testutil/k8s/deployment_util.go | 23 +- internal/testutil/k8s/k8sclient_util.go | 23 +- internal/testutil/k8s/mocks/generate.go | 23 +- internal/testutil/k8s/statefulset_util.go | 23 +- internal/testutil/k8s/storage_util.go | 23 +- internal/testutil/k8s/tunnel_util.go | 23 +- internal/testutil/type.go | 23 +- internal/unstructured/config_object.go | 23 +- internal/unstructured/redis/lexer.go | 23 +- internal/unstructured/redis/parser_fsm.go | 23 +- .../unstructured/redis/parser_fsm_test.go | 23 +- internal/unstructured/redis/rune_util.go | 23 +- internal/unstructured/redis/rune_util_test.go | 23 +- internal/unstructured/redis_config.go | 23 +- internal/unstructured/redis_config_test.go | 23 +- internal/unstructured/type.go | 23 +- internal/unstructured/viper_util.go | 23 +- internal/unstructured/viper_wrap.go | 23 +- internal/unstructured/viper_wrap_test.go | 23 +- internal/unstructured/xml_config.go | 23 +- internal/unstructured/xml_config_test.go | 23 +- internal/unstructured/yaml_config.go | 23 +- internal/unstructured/yaml_config_test.go | 23 +- internal/webhook/pod_admission.go | 23 +- internal/webhook/webhook.go | 23 +- test/e2e/e2e_suite_test.go | 23 +- test/e2e/envcheck/envcheck.go | 23 +- test/e2e/installation/installcheck.go | 23 +- test/e2e/testdata/smoketest/playgroundtest.go | 23 +- .../smoketest/redis/00_rediscluster.yaml | 23 +- test/e2e/testdata/smoketest/smoketestrun.go | 23 +- test/e2e/types.go | 23 +- test/e2e/util/client.go | 23 +- test/e2e/util/common.go | 23 +- test/e2e/util/smoke_util.go | 23 +- test/integration/backup_mysql_test.go | 24 +- test/integration/controller_suite_test.go | 23 +- test/integration/mysql_ha_test.go | 23 +- test/integration/mysql_reconfigure_test.go | 23 +- test/integration/mysql_scale_test.go | 23 +- test/integration/redis_hscale_test.go | 23 +- test/testdata/cue_testdata/clickhouse.cue | 23 +- test/testdata/cue_testdata/mongod.cue | 23 +- test/testdata/cue_testdata/mysql.cue | 23 +- test/testdata/cue_testdata/mysql_for_cli.cue | 23 +- test/testdata/cue_testdata/mysql_openapi.cue | 23 +- .../cue_testdata/mysql_openapi_v2.cue | 23 +- test/testdata/cue_testdata/mysql_simple.cue | 23 +- test/testdata/cue_testdata/pg14.cue | 23 +- test/testdata/cue_testdata/wesql.cue | 23 +- .../mysql-consensus-config-constraint.yaml | 23 +- test/testdata/testdata.go | 23 +- tools/tools.go | 23 +- version/version.go | 23 +- 751 files changed, 10432 insertions(+), 7705 deletions(-) create mode 100644 NOTICE diff --git a/LICENSE b/LICENSE index d64569567..be3f7b28e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,202 +1,661 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..6951952b0 --- /dev/null +++ b/NOTICE @@ -0,0 +1,8 @@ +KubeBlocks Project, (C) 2022-2023 ApeCloud Co., Ltd. + +This product includes software developed at ApeCloud Co., Ltd. + +The KubeBlocks project contains unmodified/modified subcomponents too with +separate copyright notices and license terms. Your use of the source +code for these subcomponents is subject to the terms and conditions +of GNU Affero General Public License 3.0. \ No newline at end of file diff --git a/apis/apps/v1alpha1/backuppolicytemplate_types.go b/apis/apps/v1alpha1/backuppolicytemplate_types.go index 3ccf5fa9a..d92439096 100644 --- a/apis/apps/v1alpha1/backuppolicytemplate_types.go +++ b/apis/apps/v1alpha1/backuppolicytemplate_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index e70cf9be5..b955e7d10 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/cluster_types_test.go b/apis/apps/v1alpha1/cluster_types_test.go index 6245259f6..b1d06de0d 100644 --- a/apis/apps/v1alpha1/cluster_types_test.go +++ b/apis/apps/v1alpha1/cluster_types_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/cluster_webhook.go b/apis/apps/v1alpha1/cluster_webhook.go index d7ac7cf69..bbb6880d1 100644 --- a/apis/apps/v1alpha1/cluster_webhook.go +++ b/apis/apps/v1alpha1/cluster_webhook.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/cluster_webhook_test.go b/apis/apps/v1alpha1/cluster_webhook_test.go index ca2d91c67..ecfa2e020 100644 --- a/apis/apps/v1alpha1/cluster_webhook_test.go +++ b/apis/apps/v1alpha1/cluster_webhook_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/clusterdefinition_types.go b/apis/apps/v1alpha1/clusterdefinition_types.go index 92620f2c1..8d0a4ffd8 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types.go +++ b/apis/apps/v1alpha1/clusterdefinition_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/clusterdefinition_types_test.go b/apis/apps/v1alpha1/clusterdefinition_types_test.go index 778f6fcc8..4a58b1daf 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types_test.go +++ b/apis/apps/v1alpha1/clusterdefinition_types_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/clusterdefinition_webhook.go b/apis/apps/v1alpha1/clusterdefinition_webhook.go index d8ed420ad..ae1765a07 100644 --- a/apis/apps/v1alpha1/clusterdefinition_webhook.go +++ b/apis/apps/v1alpha1/clusterdefinition_webhook.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/clusterdefinition_webhook_test.go b/apis/apps/v1alpha1/clusterdefinition_webhook_test.go index 2007faafe..3b1b1e120 100644 --- a/apis/apps/v1alpha1/clusterdefinition_webhook_test.go +++ b/apis/apps/v1alpha1/clusterdefinition_webhook_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/clusterversion_types.go b/apis/apps/v1alpha1/clusterversion_types.go index 5e233c5d9..810e8ed73 100644 --- a/apis/apps/v1alpha1/clusterversion_types.go +++ b/apis/apps/v1alpha1/clusterversion_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/clusterversion_types_test.go b/apis/apps/v1alpha1/clusterversion_types_test.go index d98502cfb..8d8eb6051 100644 --- a/apis/apps/v1alpha1/clusterversion_types_test.go +++ b/apis/apps/v1alpha1/clusterversion_types_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/clusterversion_webhook.go b/apis/apps/v1alpha1/clusterversion_webhook.go index 41fd39a7f..ab4feed9a 100644 --- a/apis/apps/v1alpha1/clusterversion_webhook.go +++ b/apis/apps/v1alpha1/clusterversion_webhook.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/clusterversion_webhook_test.go b/apis/apps/v1alpha1/clusterversion_webhook_test.go index afcfff505..32870a622 100644 --- a/apis/apps/v1alpha1/clusterversion_webhook_test.go +++ b/apis/apps/v1alpha1/clusterversion_webhook_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/componentclassdefinition_types.go b/apis/apps/v1alpha1/componentclassdefinition_types.go index bac5dcfe5..a6a3a0ecd 100644 --- a/apis/apps/v1alpha1/componentclassdefinition_types.go +++ b/apis/apps/v1alpha1/componentclassdefinition_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/componentresourceconstraint_types.go b/apis/apps/v1alpha1/componentresourceconstraint_types.go index d7fd4e487..5c9c41b5a 100644 --- a/apis/apps/v1alpha1/componentresourceconstraint_types.go +++ b/apis/apps/v1alpha1/componentresourceconstraint_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/componentresourceconstraint_types_test.go b/apis/apps/v1alpha1/componentresourceconstraint_types_test.go index cf1e34633..738592b0a 100644 --- a/apis/apps/v1alpha1/componentresourceconstraint_types_test.go +++ b/apis/apps/v1alpha1/componentresourceconstraint_types_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/configconstraint_types.go b/apis/apps/v1alpha1/configconstraint_types.go index 7e35cd873..74ca16cb9 100644 --- a/apis/apps/v1alpha1/configconstraint_types.go +++ b/apis/apps/v1alpha1/configconstraint_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/groupversion_info.go b/apis/apps/v1alpha1/groupversion_info.go index fc705499f..b9e0a0a8a 100644 --- a/apis/apps/v1alpha1/groupversion_info.go +++ b/apis/apps/v1alpha1/groupversion_info.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ // Package v1alpha1 contains API Schema definitions for the apps v1alpha1 API group diff --git a/apis/apps/v1alpha1/opsrequest_conditions.go b/apis/apps/v1alpha1/opsrequest_conditions.go index f7c33b056..7a0182527 100644 --- a/apis/apps/v1alpha1/opsrequest_conditions.go +++ b/apis/apps/v1alpha1/opsrequest_conditions.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/opsrequest_conditions_test.go b/apis/apps/v1alpha1/opsrequest_conditions_test.go index 9a1f924ef..de6235a77 100644 --- a/apis/apps/v1alpha1/opsrequest_conditions_test.go +++ b/apis/apps/v1alpha1/opsrequest_conditions_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/opsrequest_types.go b/apis/apps/v1alpha1/opsrequest_types.go index a40bb1f13..9bdd24eb6 100644 --- a/apis/apps/v1alpha1/opsrequest_types.go +++ b/apis/apps/v1alpha1/opsrequest_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/opsrequest_types_test.go b/apis/apps/v1alpha1/opsrequest_types_test.go index d041caa7f..0f360a80a 100644 --- a/apis/apps/v1alpha1/opsrequest_types_test.go +++ b/apis/apps/v1alpha1/opsrequest_types_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index e3c17b1a5..dc103be49 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/opsrequest_webhook_test.go b/apis/apps/v1alpha1/opsrequest_webhook_test.go index d5ef49961..a885f9862 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook_test.go +++ b/apis/apps/v1alpha1/opsrequest_webhook_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index e0b8d17d0..599774e48 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ // Package v1alpha1 contains API Schema definitions for the apps v1alpha1 API group diff --git a/apis/apps/v1alpha1/webhook_suite_test.go b/apis/apps/v1alpha1/webhook_suite_test.go index a9d726bff..4a4e4b232 100644 --- a/apis/apps/v1alpha1/webhook_suite_test.go +++ b/apis/apps/v1alpha1/webhook_suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/dataprotection/v1alpha1/backup_types.go b/apis/dataprotection/v1alpha1/backup_types.go index efbc0a7f8..4f36ee442 100644 --- a/apis/dataprotection/v1alpha1/backup_types.go +++ b/apis/dataprotection/v1alpha1/backup_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/dataprotection/v1alpha1/backup_types_test.go b/apis/dataprotection/v1alpha1/backup_types_test.go index f78ed10c4..ea3520428 100644 --- a/apis/dataprotection/v1alpha1/backup_types_test.go +++ b/apis/dataprotection/v1alpha1/backup_types_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/dataprotection/v1alpha1/backuppolicy_types.go b/apis/dataprotection/v1alpha1/backuppolicy_types.go index b2d7f4ade..773197781 100644 --- a/apis/dataprotection/v1alpha1/backuppolicy_types.go +++ b/apis/dataprotection/v1alpha1/backuppolicy_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/dataprotection/v1alpha1/backuptool_types.go b/apis/dataprotection/v1alpha1/backuptool_types.go index ec51b0817..064ec4107 100644 --- a/apis/dataprotection/v1alpha1/backuptool_types.go +++ b/apis/dataprotection/v1alpha1/backuptool_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/dataprotection/v1alpha1/groupversion_info.go b/apis/dataprotection/v1alpha1/groupversion_info.go index 00eabd849..3e134f1c5 100644 --- a/apis/dataprotection/v1alpha1/groupversion_info.go +++ b/apis/dataprotection/v1alpha1/groupversion_info.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ // Package v1alpha1 contains API Schema definitions for the dataprotection v1alpha1 API group diff --git a/apis/dataprotection/v1alpha1/restorejob_types.go b/apis/dataprotection/v1alpha1/restorejob_types.go index 603c2ca4f..803a26b8e 100644 --- a/apis/dataprotection/v1alpha1/restorejob_types.go +++ b/apis/dataprotection/v1alpha1/restorejob_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/dataprotection/v1alpha1/types.go b/apis/dataprotection/v1alpha1/types.go index ecfae0849..78b50a700 100644 --- a/apis/dataprotection/v1alpha1/types.go +++ b/apis/dataprotection/v1alpha1/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/extensions/v1alpha1/addon_types.go b/apis/extensions/v1alpha1/addon_types.go index 0d981158b..9beb80be2 100644 --- a/apis/extensions/v1alpha1/addon_types.go +++ b/apis/extensions/v1alpha1/addon_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/extensions/v1alpha1/addon_types_test.go b/apis/extensions/v1alpha1/addon_types_test.go index f8ebdd66b..17c0019fc 100644 --- a/apis/extensions/v1alpha1/addon_types_test.go +++ b/apis/extensions/v1alpha1/addon_types_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/apis/extensions/v1alpha1/groupversion_info.go b/apis/extensions/v1alpha1/groupversion_info.go index ce8f37e15..a8ee4d428 100644 --- a/apis/extensions/v1alpha1/groupversion_info.go +++ b/apis/extensions/v1alpha1/groupversion_info.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ // Package v1alpha1 contains API Schema definitions for the extensions v1alpha1 API group diff --git a/apis/extensions/v1alpha1/type.go b/apis/extensions/v1alpha1/type.go index a52106c16..cbf2457bf 100644 --- a/apis/extensions/v1alpha1/type.go +++ b/apis/extensions/v1alpha1/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 540d7e45a..82f10f6fe 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package main diff --git a/cmd/cmd.mk b/cmd/cmd.mk index 2906e6ef7..90c532e24 100644 --- a/cmd/cmd.mk +++ b/cmd/cmd.mk @@ -1,14 +1,20 @@ # -# Copyright ApeCloud, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +#Copyright (C) 2022-2023 ApeCloud Co., Ltd +# +#This file is part of KubeBlocks project +# +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, either version 3 of the License, or +#(at your option) any later version. +# +#This program is distributed in the hope that it will be useful +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU Affero General Public License for more details. +# +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . # ##@ Sub-commands diff --git a/cmd/manager/main.go b/cmd/manager/main.go index b1f6d1fae..84c31ce6e 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package main diff --git a/cmd/probe/internal/binding/base.go b/cmd/probe/internal/binding/base.go index 807c524c8..4c2c592e9 100644 --- a/cmd/probe/internal/binding/base.go +++ b/cmd/probe/internal/binding/base.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package binding diff --git a/cmd/probe/internal/binding/base_test.go b/cmd/probe/internal/binding/base_test.go index 8f7a499fb..05eba33f1 100644 --- a/cmd/probe/internal/binding/base_test.go +++ b/cmd/probe/internal/binding/base_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package binding diff --git a/cmd/probe/internal/binding/etcd/etcd.go b/cmd/probe/internal/binding/etcd/etcd.go index 7c672e108..6a4e9ff11 100644 --- a/cmd/probe/internal/binding/etcd/etcd.go +++ b/cmd/probe/internal/binding/etcd/etcd.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package etcd diff --git a/cmd/probe/internal/binding/etcd/etcd_test.go b/cmd/probe/internal/binding/etcd/etcd_test.go index 98b9c9a39..76e0f2d26 100644 --- a/cmd/probe/internal/binding/etcd/etcd_test.go +++ b/cmd/probe/internal/binding/etcd/etcd_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package etcd diff --git a/cmd/probe/internal/binding/mongodb/mongodb.go b/cmd/probe/internal/binding/mongodb/mongodb.go index e09af94de..155ecd36d 100644 --- a/cmd/probe/internal/binding/mongodb/mongodb.go +++ b/cmd/probe/internal/binding/mongodb/mongodb.go @@ -1,15 +1,20 @@ /* -Copyright ApeCloud, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package mongodb diff --git a/cmd/probe/internal/binding/mongodb/mongodb_test.go b/cmd/probe/internal/binding/mongodb/mongodb_test.go index f820f3d74..be84403e4 100644 --- a/cmd/probe/internal/binding/mongodb/mongodb_test.go +++ b/cmd/probe/internal/binding/mongodb/mongodb_test.go @@ -1,15 +1,20 @@ /* -Copyright ApeCloud, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package mongodb diff --git a/cmd/probe/internal/binding/mysql/mysql.go b/cmd/probe/internal/binding/mysql/mysql.go index b402e5ef3..b14b63e6f 100644 --- a/cmd/probe/internal/binding/mysql/mysql.go +++ b/cmd/probe/internal/binding/mysql/mysql.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package mysql diff --git a/cmd/probe/internal/binding/mysql/mysql_test.go b/cmd/probe/internal/binding/mysql/mysql_test.go index 92697854f..f8fb3fab6 100644 --- a/cmd/probe/internal/binding/mysql/mysql_test.go +++ b/cmd/probe/internal/binding/mysql/mysql_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package mysql diff --git a/cmd/probe/internal/binding/postgres/postgres.go b/cmd/probe/internal/binding/postgres/postgres.go index 2b3f5babd..3f4c64d05 100644 --- a/cmd/probe/internal/binding/postgres/postgres.go +++ b/cmd/probe/internal/binding/postgres/postgres.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package postgres diff --git a/cmd/probe/internal/binding/postgres/postgres_test.go b/cmd/probe/internal/binding/postgres/postgres_test.go index 11fb9767c..af49fced8 100644 --- a/cmd/probe/internal/binding/postgres/postgres_test.go +++ b/cmd/probe/internal/binding/postgres/postgres_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package postgres diff --git a/cmd/probe/internal/binding/redis/redis.go b/cmd/probe/internal/binding/redis/redis.go index 4d4d590df..4f8d5489c 100644 --- a/cmd/probe/internal/binding/redis/redis.go +++ b/cmd/probe/internal/binding/redis/redis.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/cmd/probe/internal/binding/redis/redis_test.go b/cmd/probe/internal/binding/redis/redis_test.go index 46b6bbac2..b1f067c63 100644 --- a/cmd/probe/internal/binding/redis/redis_test.go +++ b/cmd/probe/internal/binding/redis/redis_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/cmd/probe/internal/binding/types.go b/cmd/probe/internal/binding/types.go index eb3f0be1d..15d6ed1a4 100644 --- a/cmd/probe/internal/binding/types.go +++ b/cmd/probe/internal/binding/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package binding diff --git a/cmd/probe/internal/binding/utils.go b/cmd/probe/internal/binding/utils.go index 913bdd754..6d0441baf 100644 --- a/cmd/probe/internal/binding/utils.go +++ b/cmd/probe/internal/binding/utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package binding diff --git a/cmd/probe/internal/component/redis/metadata.go b/cmd/probe/internal/component/redis/metadata.go index 0d8d8237a..3f4c621e9 100644 --- a/cmd/probe/internal/component/redis/metadata.go +++ b/cmd/probe/internal/component/redis/metadata.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/cmd/probe/internal/component/redis/redis.go b/cmd/probe/internal/component/redis/redis.go index 4c69272fe..5924f4b80 100644 --- a/cmd/probe/internal/component/redis/redis.go +++ b/cmd/probe/internal/component/redis/redis.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/cmd/probe/internal/component/redis/redis_test.go b/cmd/probe/internal/component/redis/redis_test.go index 02a1429f3..3b3bed11e 100644 --- a/cmd/probe/internal/component/redis/redis_test.go +++ b/cmd/probe/internal/component/redis/redis_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/cmd/probe/internal/component/redis/settings.go b/cmd/probe/internal/component/redis/settings.go index 355c9132b..7355eed7a 100644 --- a/cmd/probe/internal/component/redis/settings.go +++ b/cmd/probe/internal/component/redis/settings.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/cmd/probe/internal/middleware/http/probe/checks_middleware.go b/cmd/probe/internal/middleware/http/probe/checks_middleware.go index 2b599765c..15a0b7142 100644 --- a/cmd/probe/internal/middleware/http/probe/checks_middleware.go +++ b/cmd/probe/internal/middleware/http/probe/checks_middleware.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package probe diff --git a/cmd/probe/internal/middleware/http/probe/checks_middleware_test.go b/cmd/probe/internal/middleware/http/probe/checks_middleware_test.go index 123c3f7a4..3c252389d 100644 --- a/cmd/probe/internal/middleware/http/probe/checks_middleware_test.go +++ b/cmd/probe/internal/middleware/http/probe/checks_middleware_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package probe diff --git a/cmd/probe/main.go b/cmd/probe/main.go index 577001c07..b5ec2e935 100644 --- a/cmd/probe/main.go +++ b/cmd/probe/main.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package main diff --git a/cmd/probe/probe.proto b/cmd/probe/probe.proto index e26fa2c6f..b272e85a0 100644 --- a/cmd/probe/probe.proto +++ b/cmd/probe/probe.proto @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ syntax = "proto3"; diff --git a/cmd/probe/util/const.go b/cmd/probe/util/const.go index 97d61e55e..d7db61434 100644 --- a/cmd/probe/util/const.go +++ b/cmd/probe/util/const.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/cmd/reloader/app/cmd.go b/cmd/reloader/app/cmd.go index 6947a6a2a..62e365601 100644 --- a/cmd/reloader/app/cmd.go +++ b/cmd/reloader/app/cmd.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package app diff --git a/cmd/reloader/app/flags.go b/cmd/reloader/app/flags.go index 4217c11f0..2a73931eb 100644 --- a/cmd/reloader/app/flags.go +++ b/cmd/reloader/app/flags.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package app diff --git a/cmd/reloader/app/proxy.go b/cmd/reloader/app/proxy.go index cb2d440a5..ecc33928b 100644 --- a/cmd/reloader/app/proxy.go +++ b/cmd/reloader/app/proxy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package app diff --git a/cmd/reloader/container_killer/killer.go b/cmd/reloader/container_killer/killer.go index 1e8e48002..89c8183a7 100644 --- a/cmd/reloader/container_killer/killer.go +++ b/cmd/reloader/container_killer/killer.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package main diff --git a/cmd/reloader/main.go b/cmd/reloader/main.go index fe3b23eb1..52695d4b2 100644 --- a/cmd/reloader/main.go +++ b/cmd/reloader/main.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package main diff --git a/cmd/reloader/tools/cue_auto_generator.go b/cmd/reloader/tools/cue_auto_generator.go index db3180e31..2fb55c568 100644 --- a/cmd/reloader/tools/cue_auto_generator.go +++ b/cmd/reloader/tools/cue_auto_generator.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package main diff --git a/cmd/tpl/app/helm_helper.go b/cmd/tpl/app/helm_helper.go index ab898892b..43d2c2d18 100644 --- a/cmd/tpl/app/helm_helper.go +++ b/cmd/tpl/app/helm_helper.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package app diff --git a/cmd/tpl/app/k8s_resource.go b/cmd/tpl/app/k8s_resource.go index c67b87b9b..85b07de3a 100644 --- a/cmd/tpl/app/k8s_resource.go +++ b/cmd/tpl/app/k8s_resource.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package app diff --git a/cmd/tpl/app/mock_client.go b/cmd/tpl/app/mock_client.go index dcb00fb56..50d67970d 100644 --- a/cmd/tpl/app/mock_client.go +++ b/cmd/tpl/app/mock_client.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package app diff --git a/cmd/tpl/app/util.go b/cmd/tpl/app/util.go index e69942322..334e210a0 100644 --- a/cmd/tpl/app/util.go +++ b/cmd/tpl/app/util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package app diff --git a/cmd/tpl/app/workflow.go b/cmd/tpl/app/workflow.go index 2ccd29bf3..b9ec44d0e 100644 --- a/cmd/tpl/app/workflow.go +++ b/cmd/tpl/app/workflow.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package app diff --git a/cmd/tpl/main.go b/cmd/tpl/main.go index 6497da8ff..c89959351 100644 --- a/cmd/tpl/main.go +++ b/cmd/tpl/main.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package main diff --git a/controllers/apps/class_controller.go b/controllers/apps/class_controller.go index 833e7046f..b0e5be064 100644 --- a/controllers/apps/class_controller.go +++ b/controllers/apps/class_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/class_controller_test.go b/controllers/apps/class_controller_test.go index 23a9b26aa..e1c11594c 100644 --- a/controllers/apps/class_controller_test.go +++ b/controllers/apps/class_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index a8f16aec7..ec6ed65e7 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 661274d2b..b1be1c0cd 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/cluster_status_utils.go b/controllers/apps/cluster_status_utils.go index 7d16d3a3f..4367a3592 100644 --- a/controllers/apps/cluster_status_utils.go +++ b/controllers/apps/cluster_status_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/cluster_status_utils_test.go b/controllers/apps/cluster_status_utils_test.go index f04353332..b1de0ba9b 100644 --- a/controllers/apps/cluster_status_utils_test.go +++ b/controllers/apps/cluster_status_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/clusterdefinition_controller.go b/controllers/apps/clusterdefinition_controller.go index c6775d30c..17a206a6d 100644 --- a/controllers/apps/clusterdefinition_controller.go +++ b/controllers/apps/clusterdefinition_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/clusterdefinition_controller_test.go b/controllers/apps/clusterdefinition_controller_test.go index 8e7b1e6e9..f2d5ba227 100644 --- a/controllers/apps/clusterdefinition_controller_test.go +++ b/controllers/apps/clusterdefinition_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/clusterversion_controller.go b/controllers/apps/clusterversion_controller.go index b261f3581..eff70b96c 100644 --- a/controllers/apps/clusterversion_controller.go +++ b/controllers/apps/clusterversion_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/clusterversion_controller_test.go b/controllers/apps/clusterversion_controller_test.go index 0fbe0606d..d20c711e9 100644 --- a/controllers/apps/clusterversion_controller_test.go +++ b/controllers/apps/clusterversion_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/components/component.go b/controllers/apps/components/component.go index 8e26347af..d5250240d 100644 --- a/controllers/apps/components/component.go +++ b/controllers/apps/components/component.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/component_status.go b/controllers/apps/components/component_status.go index aa98065f5..7c95b1bfd 100644 --- a/controllers/apps/components/component_status.go +++ b/controllers/apps/components/component_status.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/component_status_test.go b/controllers/apps/components/component_status_test.go index f6d1bf41e..c2fc47651 100644 --- a/controllers/apps/components/component_status_test.go +++ b/controllers/apps/components/component_status_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/consensus/consensus.go b/controllers/apps/components/consensus/consensus.go index 3efcd5693..11804922b 100644 --- a/controllers/apps/components/consensus/consensus.go +++ b/controllers/apps/components/consensus/consensus.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package consensus diff --git a/controllers/apps/components/consensus/consensus_test.go b/controllers/apps/components/consensus/consensus_test.go index 45c53b3d0..43bc89de3 100644 --- a/controllers/apps/components/consensus/consensus_test.go +++ b/controllers/apps/components/consensus/consensus_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package consensus diff --git a/controllers/apps/components/consensus/consensus_utils.go b/controllers/apps/components/consensus/consensus_utils.go index 55820c00c..e06c2143b 100644 --- a/controllers/apps/components/consensus/consensus_utils.go +++ b/controllers/apps/components/consensus/consensus_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package consensus diff --git a/controllers/apps/components/consensus/consensus_utils_test.go b/controllers/apps/components/consensus/consensus_utils_test.go index 20ce8473a..c9c93175c 100644 --- a/controllers/apps/components/consensus/consensus_utils_test.go +++ b/controllers/apps/components/consensus/consensus_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package consensus diff --git a/controllers/apps/components/consensus/suite_test.go b/controllers/apps/components/consensus/suite_test.go index 5f686bc8a..78161f968 100644 --- a/controllers/apps/components/consensus/suite_test.go +++ b/controllers/apps/components/consensus/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package consensus diff --git a/controllers/apps/components/deployment_controller.go b/controllers/apps/components/deployment_controller.go index 47fe745a3..2bd4b22c4 100644 --- a/controllers/apps/components/deployment_controller.go +++ b/controllers/apps/components/deployment_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/deployment_controller_test.go b/controllers/apps/components/deployment_controller_test.go index 6463788a4..89861ba31 100644 --- a/controllers/apps/components/deployment_controller_test.go +++ b/controllers/apps/components/deployment_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/pod_controller.go b/controllers/apps/components/pod_controller.go index ea74f949f..38ca74f42 100644 --- a/controllers/apps/components/pod_controller.go +++ b/controllers/apps/components/pod_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/pod_controller_test.go b/controllers/apps/components/pod_controller_test.go index 02e406972..d8214e9d0 100644 --- a/controllers/apps/components/pod_controller_test.go +++ b/controllers/apps/components/pod_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/replication/replication.go b/controllers/apps/components/replication/replication.go index e607036dc..64ab198e0 100644 --- a/controllers/apps/components/replication/replication.go +++ b/controllers/apps/components/replication/replication.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/replication/replication_switch.go b/controllers/apps/components/replication/replication_switch.go index 09cca1991..412a59516 100644 --- a/controllers/apps/components/replication/replication_switch.go +++ b/controllers/apps/components/replication/replication_switch.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/replication/replication_switch_test.go b/controllers/apps/components/replication/replication_switch_test.go index 8552b7e11..3e2f4ef9e 100644 --- a/controllers/apps/components/replication/replication_switch_test.go +++ b/controllers/apps/components/replication/replication_switch_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/replication/replication_switch_utils.go b/controllers/apps/components/replication/replication_switch_utils.go index e3b73a535..6821a398b 100644 --- a/controllers/apps/components/replication/replication_switch_utils.go +++ b/controllers/apps/components/replication/replication_switch_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/replication/replication_switch_utils_test.go b/controllers/apps/components/replication/replication_switch_utils_test.go index d8fe0e303..aec407797 100644 --- a/controllers/apps/components/replication/replication_switch_utils_test.go +++ b/controllers/apps/components/replication/replication_switch_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/replication/replication_test.go b/controllers/apps/components/replication/replication_test.go index 7a43c13fa..d083700a2 100644 --- a/controllers/apps/components/replication/replication_test.go +++ b/controllers/apps/components/replication/replication_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/replication/replication_utils.go b/controllers/apps/components/replication/replication_utils.go index f411b5f68..21a15a444 100644 --- a/controllers/apps/components/replication/replication_utils.go +++ b/controllers/apps/components/replication/replication_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/replication/replication_utils_test.go b/controllers/apps/components/replication/replication_utils_test.go index 8e7eebcef..e74143e37 100644 --- a/controllers/apps/components/replication/replication_utils_test.go +++ b/controllers/apps/components/replication/replication_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/replication/suite_test.go b/controllers/apps/components/replication/suite_test.go index 0d610b790..fda81689e 100644 --- a/controllers/apps/components/replication/suite_test.go +++ b/controllers/apps/components/replication/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package replication diff --git a/controllers/apps/components/stateful/stateful.go b/controllers/apps/components/stateful/stateful.go index e59917d80..0a006f26e 100644 --- a/controllers/apps/components/stateful/stateful.go +++ b/controllers/apps/components/stateful/stateful.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package stateful diff --git a/controllers/apps/components/stateful/stateful_test.go b/controllers/apps/components/stateful/stateful_test.go index 784d0dafa..4bf237ddb 100644 --- a/controllers/apps/components/stateful/stateful_test.go +++ b/controllers/apps/components/stateful/stateful_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package stateful diff --git a/controllers/apps/components/stateful/suite_test.go b/controllers/apps/components/stateful/suite_test.go index c26a77511..fb40c857b 100644 --- a/controllers/apps/components/stateful/suite_test.go +++ b/controllers/apps/components/stateful/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package stateful diff --git a/controllers/apps/components/stateful_set_controller.go b/controllers/apps/components/stateful_set_controller.go index 5a8d146f3..75821c8b3 100644 --- a/controllers/apps/components/stateful_set_controller.go +++ b/controllers/apps/components/stateful_set_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/stateful_set_controller_test.go b/controllers/apps/components/stateful_set_controller_test.go index 07679a0ab..45a7a850d 100644 --- a/controllers/apps/components/stateful_set_controller_test.go +++ b/controllers/apps/components/stateful_set_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/stateless/stateless.go b/controllers/apps/components/stateless/stateless.go index 728f13df0..294ea5a11 100644 --- a/controllers/apps/components/stateless/stateless.go +++ b/controllers/apps/components/stateless/stateless.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package stateless diff --git a/controllers/apps/components/stateless/stateless_test.go b/controllers/apps/components/stateless/stateless_test.go index edb548068..2ad9b1519 100644 --- a/controllers/apps/components/stateless/stateless_test.go +++ b/controllers/apps/components/stateless/stateless_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package stateless diff --git a/controllers/apps/components/stateless/suite_test.go b/controllers/apps/components/stateless/suite_test.go index a68bb40ea..4b6f6179f 100644 --- a/controllers/apps/components/stateless/suite_test.go +++ b/controllers/apps/components/stateless/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package stateless diff --git a/controllers/apps/components/suite_test.go b/controllers/apps/components/suite_test.go index d6adda579..a4d404caa 100644 --- a/controllers/apps/components/suite_test.go +++ b/controllers/apps/components/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package components diff --git a/controllers/apps/components/types/component.go b/controllers/apps/components/types/component.go index 073a55376..53b8e2a5e 100644 --- a/controllers/apps/components/types/component.go +++ b/controllers/apps/components/types/component.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package types diff --git a/controllers/apps/components/util/component_utils.go b/controllers/apps/components/util/component_utils.go index 1632f0173..6f810f681 100644 --- a/controllers/apps/components/util/component_utils.go +++ b/controllers/apps/components/util/component_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/components/util/component_utils_test.go b/controllers/apps/components/util/component_utils_test.go index 5c25e6126..28dcd4d7a 100644 --- a/controllers/apps/components/util/component_utils_test.go +++ b/controllers/apps/components/util/component_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/components/util/plan.go b/controllers/apps/components/util/plan.go index e0ce54d59..e0c5b8b27 100644 --- a/controllers/apps/components/util/plan.go +++ b/controllers/apps/components/util/plan.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/components/util/plan_test.go b/controllers/apps/components/util/plan_test.go index a2e76553c..fd9ab1189 100644 --- a/controllers/apps/components/util/plan_test.go +++ b/controllers/apps/components/util/plan_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/components/util/stateful_set_utils.go b/controllers/apps/components/util/stateful_set_utils.go index 4a33557b3..c76b80c81 100644 --- a/controllers/apps/components/util/stateful_set_utils.go +++ b/controllers/apps/components/util/stateful_set_utils.go @@ -1,18 +1,20 @@ /* -Copyright ApeCloud, Inc. -Copyright 2016 The Kubernetes Authors. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/components/util/stateful_set_utils_test.go b/controllers/apps/components/util/stateful_set_utils_test.go index 50963e260..c5f5c39cf 100644 --- a/controllers/apps/components/util/stateful_set_utils_test.go +++ b/controllers/apps/components/util/stateful_set_utils_test.go @@ -1,18 +1,20 @@ /* -Copyright ApeCloud, Inc. -Copyright 2016 The Kubernetes Authors. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/components/util/suite_test.go b/controllers/apps/components/util/suite_test.go index e31ef3f96..08de72226 100644 --- a/controllers/apps/components/util/suite_test.go +++ b/controllers/apps/components/util/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/configuration/config_annotation.go b/controllers/apps/configuration/config_annotation.go index a37de3310..8e4bd28e8 100644 --- a/controllers/apps/configuration/config_annotation.go +++ b/controllers/apps/configuration/config_annotation.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/config_util.go b/controllers/apps/configuration/config_util.go index 939f2a9a9..196ad52d3 100644 --- a/controllers/apps/configuration/config_util.go +++ b/controllers/apps/configuration/config_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/config_util_test.go b/controllers/apps/configuration/config_util_test.go index 9c441f0be..6535b55bc 100644 --- a/controllers/apps/configuration/config_util_test.go +++ b/controllers/apps/configuration/config_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/configconstraint_controller.go b/controllers/apps/configuration/configconstraint_controller.go index d386d799f..60078c36d 100644 --- a/controllers/apps/configuration/configconstraint_controller.go +++ b/controllers/apps/configuration/configconstraint_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/configconstraint_controller_test.go b/controllers/apps/configuration/configconstraint_controller_test.go index 0db707b05..54d0750ac 100644 --- a/controllers/apps/configuration/configconstraint_controller_test.go +++ b/controllers/apps/configuration/configconstraint_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/parallel_upgrade_policy.go b/controllers/apps/configuration/parallel_upgrade_policy.go index c32f39bec..7970fb812 100644 --- a/controllers/apps/configuration/parallel_upgrade_policy.go +++ b/controllers/apps/configuration/parallel_upgrade_policy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/parallel_upgrade_policy_test.go b/controllers/apps/configuration/parallel_upgrade_policy_test.go index beb490a55..b1a13ed63 100644 --- a/controllers/apps/configuration/parallel_upgrade_policy_test.go +++ b/controllers/apps/configuration/parallel_upgrade_policy_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/policy_util.go b/controllers/apps/configuration/policy_util.go index be71357ae..8a3e305d4 100644 --- a/controllers/apps/configuration/policy_util.go +++ b/controllers/apps/configuration/policy_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/policy_util_test.go b/controllers/apps/configuration/policy_util_test.go index ca5dc03da..b8ce5e5d6 100644 --- a/controllers/apps/configuration/policy_util_test.go +++ b/controllers/apps/configuration/policy_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/reconfigure_policy.go b/controllers/apps/configuration/reconfigure_policy.go index 096f7aa87..ebf7f51e2 100644 --- a/controllers/apps/configuration/reconfigure_policy.go +++ b/controllers/apps/configuration/reconfigure_policy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/reconfigurerequest_controller.go b/controllers/apps/configuration/reconfigurerequest_controller.go index 75740c037..fe4b3d492 100644 --- a/controllers/apps/configuration/reconfigurerequest_controller.go +++ b/controllers/apps/configuration/reconfigurerequest_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/reconfigurerequest_controller_test.go b/controllers/apps/configuration/reconfigurerequest_controller_test.go index 9a878776c..09acbe971 100644 --- a/controllers/apps/configuration/reconfigurerequest_controller_test.go +++ b/controllers/apps/configuration/reconfigurerequest_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/rolling_upgrade_policy.go b/controllers/apps/configuration/rolling_upgrade_policy.go index 075c919d6..2f045c39d 100644 --- a/controllers/apps/configuration/rolling_upgrade_policy.go +++ b/controllers/apps/configuration/rolling_upgrade_policy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/rolling_upgrade_policy_test.go b/controllers/apps/configuration/rolling_upgrade_policy_test.go index 67a41b109..65064cf2f 100644 --- a/controllers/apps/configuration/rolling_upgrade_policy_test.go +++ b/controllers/apps/configuration/rolling_upgrade_policy_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/simple_policy.go b/controllers/apps/configuration/simple_policy.go index 0c42141ba..69a6a14c0 100644 --- a/controllers/apps/configuration/simple_policy.go +++ b/controllers/apps/configuration/simple_policy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/simple_policy_test.go b/controllers/apps/configuration/simple_policy_test.go index f13751975..e6e0cfc06 100644 --- a/controllers/apps/configuration/simple_policy_test.go +++ b/controllers/apps/configuration/simple_policy_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/suite_test.go b/controllers/apps/configuration/suite_test.go index 481fb5d9c..72dc7841c 100644 --- a/controllers/apps/configuration/suite_test.go +++ b/controllers/apps/configuration/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/sync_upgrade_policy.go b/controllers/apps/configuration/sync_upgrade_policy.go index bbf8c98f9..da9fc14ea 100644 --- a/controllers/apps/configuration/sync_upgrade_policy.go +++ b/controllers/apps/configuration/sync_upgrade_policy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/configuration/sync_upgrade_policy_test.go b/controllers/apps/configuration/sync_upgrade_policy_test.go index 24ff2d9d0..f045380ec 100644 --- a/controllers/apps/configuration/sync_upgrade_policy_test.go +++ b/controllers/apps/configuration/sync_upgrade_policy_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/controllers/apps/const.go b/controllers/apps/const.go index 23eef527b..9ad4e9680 100644 --- a/controllers/apps/const.go +++ b/controllers/apps/const.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/operations/expose.go b/controllers/apps/operations/expose.go index c6cee17c6..a60f9e8c7 100644 --- a/controllers/apps/operations/expose.go +++ b/controllers/apps/operations/expose.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/expose_test.go b/controllers/apps/operations/expose_test.go index 6cc723ef9..4f4f4a738 100644 --- a/controllers/apps/operations/expose_test.go +++ b/controllers/apps/operations/expose_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/horizontal_scaling.go b/controllers/apps/operations/horizontal_scaling.go index a9473dc5f..3c7b030d2 100644 --- a/controllers/apps/operations/horizontal_scaling.go +++ b/controllers/apps/operations/horizontal_scaling.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/horizontal_scaling_test.go b/controllers/apps/operations/horizontal_scaling_test.go index 01d8827c9..8c61d7d53 100644 --- a/controllers/apps/operations/horizontal_scaling_test.go +++ b/controllers/apps/operations/horizontal_scaling_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/ops_manager.go b/controllers/apps/operations/ops_manager.go index 697d10945..db5b1e1ff 100644 --- a/controllers/apps/operations/ops_manager.go +++ b/controllers/apps/operations/ops_manager.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/ops_progress_util.go b/controllers/apps/operations/ops_progress_util.go index ac2390ce2..1986d0b99 100644 --- a/controllers/apps/operations/ops_progress_util.go +++ b/controllers/apps/operations/ops_progress_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/ops_progress_util_test.go b/controllers/apps/operations/ops_progress_util_test.go index 328c82dd8..4b8554c9f 100644 --- a/controllers/apps/operations/ops_progress_util_test.go +++ b/controllers/apps/operations/ops_progress_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/ops_util.go b/controllers/apps/operations/ops_util.go index 5109ab7d7..48cb63020 100644 --- a/controllers/apps/operations/ops_util.go +++ b/controllers/apps/operations/ops_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/ops_util_test.go b/controllers/apps/operations/ops_util_test.go index c73e7ba92..853e2c8b8 100644 --- a/controllers/apps/operations/ops_util_test.go +++ b/controllers/apps/operations/ops_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/reconfigure.go b/controllers/apps/operations/reconfigure.go index ff652a496..9989f0163 100644 --- a/controllers/apps/operations/reconfigure.go +++ b/controllers/apps/operations/reconfigure.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/reconfigure_test.go b/controllers/apps/operations/reconfigure_test.go index 03e828292..978fa25b8 100644 --- a/controllers/apps/operations/reconfigure_test.go +++ b/controllers/apps/operations/reconfigure_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/reconfigure_util.go b/controllers/apps/operations/reconfigure_util.go index 829403f17..aac795e5a 100644 --- a/controllers/apps/operations/reconfigure_util.go +++ b/controllers/apps/operations/reconfigure_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/reconfigure_util_test.go b/controllers/apps/operations/reconfigure_util_test.go index 887a7246f..6a44d88d8 100644 --- a/controllers/apps/operations/reconfigure_util_test.go +++ b/controllers/apps/operations/reconfigure_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/restart.go b/controllers/apps/operations/restart.go index 7ea87d42f..46dce02d2 100644 --- a/controllers/apps/operations/restart.go +++ b/controllers/apps/operations/restart.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/restart_test.go b/controllers/apps/operations/restart_test.go index 90517b9a1..b59e885f1 100644 --- a/controllers/apps/operations/restart_test.go +++ b/controllers/apps/operations/restart_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/start.go b/controllers/apps/operations/start.go index 15941f1e3..7212e7c38 100644 --- a/controllers/apps/operations/start.go +++ b/controllers/apps/operations/start.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/start_test.go b/controllers/apps/operations/start_test.go index 1012703e9..7c4533af3 100644 --- a/controllers/apps/operations/start_test.go +++ b/controllers/apps/operations/start_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/stop.go b/controllers/apps/operations/stop.go index 9211ce0cd..12785e90b 100644 --- a/controllers/apps/operations/stop.go +++ b/controllers/apps/operations/stop.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/stop_test.go b/controllers/apps/operations/stop_test.go index 9a2fca1ea..e28fbbeb4 100644 --- a/controllers/apps/operations/stop_test.go +++ b/controllers/apps/operations/stop_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/suite_test.go b/controllers/apps/operations/suite_test.go index 30d6d6e14..7637d9bdf 100644 --- a/controllers/apps/operations/suite_test.go +++ b/controllers/apps/operations/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/type.go b/controllers/apps/operations/type.go index 2e5dab449..9c70088de 100644 --- a/controllers/apps/operations/type.go +++ b/controllers/apps/operations/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/upgrade.go b/controllers/apps/operations/upgrade.go index e96849409..948bb0bb8 100644 --- a/controllers/apps/operations/upgrade.go +++ b/controllers/apps/operations/upgrade.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/upgrade_test.go b/controllers/apps/operations/upgrade_test.go index 871b56fb3..848f2039e 100644 --- a/controllers/apps/operations/upgrade_test.go +++ b/controllers/apps/operations/upgrade_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/util/common_util.go b/controllers/apps/operations/util/common_util.go index 0db958277..07eca565c 100644 --- a/controllers/apps/operations/util/common_util.go +++ b/controllers/apps/operations/util/common_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/operations/util/common_util_test.go b/controllers/apps/operations/util/common_util_test.go index 93333ef87..1c602bff9 100644 --- a/controllers/apps/operations/util/common_util_test.go +++ b/controllers/apps/operations/util/common_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/operations/util/suite_test.go b/controllers/apps/operations/util/suite_test.go index 7315aca9e..1789ac7e8 100644 --- a/controllers/apps/operations/util/suite_test.go +++ b/controllers/apps/operations/util/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/controllers/apps/operations/vertical_scaling.go b/controllers/apps/operations/vertical_scaling.go index a76e2ad41..181905087 100644 --- a/controllers/apps/operations/vertical_scaling.go +++ b/controllers/apps/operations/vertical_scaling.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/vertical_scaling_test.go b/controllers/apps/operations/vertical_scaling_test.go index 70305f071..77203115c 100644 --- a/controllers/apps/operations/vertical_scaling_test.go +++ b/controllers/apps/operations/vertical_scaling_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/volume_expansion.go b/controllers/apps/operations/volume_expansion.go index 64b3787e0..ddca837e3 100644 --- a/controllers/apps/operations/volume_expansion.go +++ b/controllers/apps/operations/volume_expansion.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/volume_expansion_test.go b/controllers/apps/operations/volume_expansion_test.go index 9d3542525..c4b154e5e 100644 --- a/controllers/apps/operations/volume_expansion_test.go +++ b/controllers/apps/operations/volume_expansion_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/operations/volume_expansion_updater.go b/controllers/apps/operations/volume_expansion_updater.go index 1f5621103..3c3a3a066 100644 --- a/controllers/apps/operations/volume_expansion_updater.go +++ b/controllers/apps/operations/volume_expansion_updater.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package operations diff --git a/controllers/apps/opsrequest_controller.go b/controllers/apps/opsrequest_controller.go index 03cd3b218..bb1879d6e 100644 --- a/controllers/apps/opsrequest_controller.go +++ b/controllers/apps/opsrequest_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index 3fb7b9d5b..9261cd07c 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/suite_test.go b/controllers/apps/suite_test.go index 6f5298e5a..279cb4926 100644 --- a/controllers/apps/suite_test.go +++ b/controllers/apps/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/systemaccount_controller.go b/controllers/apps/systemaccount_controller.go index 7d2e1551d..af2dd29b9 100644 --- a/controllers/apps/systemaccount_controller.go +++ b/controllers/apps/systemaccount_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/systemaccount_controller_test.go b/controllers/apps/systemaccount_controller_test.go index a0f7ca414..931eb1430 100644 --- a/controllers/apps/systemaccount_controller_test.go +++ b/controllers/apps/systemaccount_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/systemaccount_util.go b/controllers/apps/systemaccount_util.go index 89a19aa81..a18f2d303 100644 --- a/controllers/apps/systemaccount_util.go +++ b/controllers/apps/systemaccount_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/systemaccount_util_test.go b/controllers/apps/systemaccount_util_test.go index aae830404..b72b1e0db 100644 --- a/controllers/apps/systemaccount_util_test.go +++ b/controllers/apps/systemaccount_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/tls_utils_test.go b/controllers/apps/tls_utils_test.go index 54cb1b6f2..bd85778b5 100644 --- a/controllers/apps/tls_utils_test.go +++ b/controllers/apps/tls_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/apps/utils.go b/controllers/apps/utils.go index 199e98e1d..b95ab52b7 100644 --- a/controllers/apps/utils.go +++ b/controllers/apps/utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 970d763fc..f9e982124 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index 85e103912..a2b4d8fe1 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index 93a633853..272d1744d 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index 4ad815918..dd76d96ae 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/backuptool_controller.go b/controllers/dataprotection/backuptool_controller.go index 80bc59195..c6bb1c866 100644 --- a/controllers/dataprotection/backuptool_controller.go +++ b/controllers/dataprotection/backuptool_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/cronjob_controller.go b/controllers/dataprotection/cronjob_controller.go index 8e000d6c5..f69c92839 100644 --- a/controllers/dataprotection/cronjob_controller.go +++ b/controllers/dataprotection/cronjob_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/cue/cronjob.cue b/controllers/dataprotection/cue/cronjob.cue index 3545b1f5b..e20d136ea 100644 --- a/controllers/dataprotection/cue/cronjob.cue +++ b/controllers/dataprotection/cue/cronjob.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . options: { name: string diff --git a/controllers/dataprotection/restorejob_controller.go b/controllers/dataprotection/restorejob_controller.go index 6c00988bb..021e12c49 100644 --- a/controllers/dataprotection/restorejob_controller.go +++ b/controllers/dataprotection/restorejob_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/restorejob_controller_test.go b/controllers/dataprotection/restorejob_controller_test.go index b25b7a7a6..79a009aad 100644 --- a/controllers/dataprotection/restorejob_controller_test.go +++ b/controllers/dataprotection/restorejob_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/suite_test.go b/controllers/dataprotection/suite_test.go index b1d04d347..a435596f3 100644 --- a/controllers/dataprotection/suite_test.go +++ b/controllers/dataprotection/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/type.go b/controllers/dataprotection/type.go index 5d52b2f5c..07c0b14f8 100644 --- a/controllers/dataprotection/type.go +++ b/controllers/dataprotection/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/dataprotection/utils.go b/controllers/dataprotection/utils.go index f0c2cdbd6..e32c7e76e 100644 --- a/controllers/dataprotection/utils.go +++ b/controllers/dataprotection/utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dataprotection diff --git a/controllers/extensions/addon_controller.go b/controllers/extensions/addon_controller.go index f748697f1..8f8a7133d 100644 --- a/controllers/extensions/addon_controller.go +++ b/controllers/extensions/addon_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package extensions diff --git a/controllers/extensions/addon_controller_stages.go b/controllers/extensions/addon_controller_stages.go index c9fd7ff5f..4be2e54c9 100644 --- a/controllers/extensions/addon_controller_stages.go +++ b/controllers/extensions/addon_controller_stages.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package extensions diff --git a/controllers/extensions/addon_controller_test.go b/controllers/extensions/addon_controller_test.go index 295b0a0d4..cd0b4ca69 100644 --- a/controllers/extensions/addon_controller_test.go +++ b/controllers/extensions/addon_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package extensions diff --git a/controllers/extensions/const.go b/controllers/extensions/const.go index e4c56fe27..22c97ebde 100644 --- a/controllers/extensions/const.go +++ b/controllers/extensions/const.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package extensions diff --git a/controllers/extensions/suite_test.go b/controllers/extensions/suite_test.go index 1d3f78d25..b6fb96ae2 100644 --- a/controllers/extensions/suite_test.go +++ b/controllers/extensions/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package extensions diff --git a/controllers/k8score/const.go b/controllers/k8score/const.go index e6cc1fc94..31476e120 100644 --- a/controllers/k8score/const.go +++ b/controllers/k8score/const.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package k8score diff --git a/controllers/k8score/event_controller.go b/controllers/k8score/event_controller.go index be157f0bd..a0c661cf7 100644 --- a/controllers/k8score/event_controller.go +++ b/controllers/k8score/event_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package k8score diff --git a/controllers/k8score/event_controller_test.go b/controllers/k8score/event_controller_test.go index d85c4b239..89e3a3c29 100644 --- a/controllers/k8score/event_controller_test.go +++ b/controllers/k8score/event_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package k8score diff --git a/controllers/k8score/event_utils.go b/controllers/k8score/event_utils.go index 12443fd97..f3a275f1a 100644 --- a/controllers/k8score/event_utils.go +++ b/controllers/k8score/event_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package k8score diff --git a/controllers/k8score/pvc_controller.go b/controllers/k8score/pvc_controller.go index 86f775bed..db6b8698d 100644 --- a/controllers/k8score/pvc_controller.go +++ b/controllers/k8score/pvc_controller.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package k8score diff --git a/controllers/k8score/pvc_controller_test.go b/controllers/k8score/pvc_controller_test.go index bfe078868..c9ec9011f 100644 --- a/controllers/k8score/pvc_controller_test.go +++ b/controllers/k8score/pvc_controller_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package k8score diff --git a/controllers/k8score/suite_test.go b/controllers/k8score/suite_test.go index a287832fc..954c4060e 100644 --- a/controllers/k8score/suite_test.go +++ b/controllers/k8score/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package k8score diff --git a/deploy/apecloud-mysql-scale/config/mysql8-config-constraint.cue b/deploy/apecloud-mysql-scale/config/mysql8-config-constraint.cue index e2fc384c5..19104e267 100644 --- a/deploy/apecloud-mysql-scale/config/mysql8-config-constraint.cue +++ b/deploy/apecloud-mysql-scale/config/mysql8-config-constraint.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #MysqlParameter: { diff --git a/deploy/apecloud-mysql/config/mysql8-config-constraint.cue b/deploy/apecloud-mysql/config/mysql8-config-constraint.cue index 3cc107f31..a5744fae2 100644 --- a/deploy/apecloud-mysql/config/mysql8-config-constraint.cue +++ b/deploy/apecloud-mysql/config/mysql8-config-constraint.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #MysqlParameter: { diff --git a/deploy/postgresql/config/pg14-config-constraint.cue b/deploy/postgresql/config/pg14-config-constraint.cue index 1d735c145..775749618 100644 --- a/deploy/postgresql/config/pg14-config-constraint.cue +++ b/deploy/postgresql/config/pg14-config-constraint.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // PostgreSQL parameters: https://postgresqlco.nf/doc/en/param/ #PGParameter: { diff --git a/deploy/redis/config/redis7-config-constraint.cue b/deploy/redis/config/redis7-config-constraint.cue index cef8d638f..57b77d67e 100644 --- a/deploy/redis/config/redis7-config-constraint.cue +++ b/deploy/redis/config/redis7-config-constraint.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #RedisParameter: { diff --git a/docker/docker.mk b/docker/docker.mk index 2a61c35ab..01bded261 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -1,14 +1,20 @@ # -# Copyright ApeCloud, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +#Copyright (C) 2022-2023 ApeCloud Co., Ltd +# +#This file is part of KubeBlocks project +# +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, either version 3 of the License, or +#(at your option) any later version. +# +#This program is distributed in the hope that it will be useful +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU Affero General Public License for more details. +# +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . # # To use buildx: https://github.com/docker/buildx#docker-ce diff --git a/externalapis/preflight/v1beta2/groupversion_info.go b/externalapis/preflight/v1beta2/groupversion_info.go index 2157d11e6..359337880 100644 --- a/externalapis/preflight/v1beta2/groupversion_info.go +++ b/externalapis/preflight/v1beta2/groupversion_info.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ // Package v1beta2 contains API Schema definitions for the preflight v1beta2 API group diff --git a/externalapis/preflight/v1beta2/hostpreflight_types.go b/externalapis/preflight/v1beta2/hostpreflight_types.go index 2aff73568..402187116 100644 --- a/externalapis/preflight/v1beta2/hostpreflight_types.go +++ b/externalapis/preflight/v1beta2/hostpreflight_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1beta2 diff --git a/externalapis/preflight/v1beta2/preflight_types.go b/externalapis/preflight/v1beta2/preflight_types.go index 4e8995463..9a5b18b7e 100644 --- a/externalapis/preflight/v1beta2/preflight_types.go +++ b/externalapis/preflight/v1beta2/preflight_types.go @@ -1,14 +1,20 @@ /* -Copyright ApeCloud, Inc. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1beta2 diff --git a/externalapis/preflight/v1beta2/type.go b/externalapis/preflight/v1beta2/type.go index 11994badb..f759c2085 100644 --- a/externalapis/preflight/v1beta2/type.go +++ b/externalapis/preflight/v1beta2/type.go @@ -1,14 +1,20 @@ /* -Copyright ApeCloud, Inc. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1beta2 diff --git a/hack/boilerplate.cue.txt b/hack/boilerplate.cue.txt index 4500eda5d..b7a6f2184 100644 --- a/hack/boilerplate.cue.txt +++ b/hack/boilerplate.cue.txt @@ -1,14 +1,17 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 53620698c..8b1b64f82 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,16 +1,19 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ diff --git a/hack/docgen/cli/main.go b/hack/docgen/cli/main.go index 10bbe10a3..6118a7cec 100644 --- a/hack/docgen/cli/main.go +++ b/hack/docgen/cli/main.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package main diff --git a/hack/install_cli.sh b/hack/install_cli.sh index 47497dcba..348b1ad6d 100755 --- a/hack/install_cli.sh +++ b/hack/install_cli.sh @@ -1,17 +1,20 @@ #!/usr/bin/env bash -# Copyright ApeCloud, Inc. +#Copyright (C) 2022-2023 ApeCloud Co., Ltd # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#This file is part of KubeBlocks project # -# http://www.apache.org/licenses/LICENSE-2.0 +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, either version 3 of the License, or +#(at your option) any later version. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +#This program is distributed in the hope that it will be useful +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU Affero General Public License for more details. +# +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . : ${CLI_INSTALL_DIR:="/usr/local/bin"} : ${CLI_BREW_INSTALL_DIR:="/opt/homebrew/bin"} diff --git a/hack/install_cli_docker.sh b/hack/install_cli_docker.sh index 5ac423c55..0e9a20c0e 100755 --- a/hack/install_cli_docker.sh +++ b/hack/install_cli_docker.sh @@ -1,17 +1,20 @@ #!/usr/bin/env bash -# Copyright ApeCloud, Inc. +#Copyright (C) 2022-2023 ApeCloud Co., Ltd # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#This file is part of KubeBlocks project # -# http://www.apache.org/licenses/LICENSE-2.0 +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, either version 3 of the License, or +#(at your option) any later version. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +#This program is distributed in the hope that it will be useful +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU Affero General Public License for more details. +# +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . : ${CLI_INSTALL_DIR:="/usr/local/bin"} : ${CLI_BREW_INSTALL_DIR:="/opt/homebrew/bin"} diff --git a/hack/license/header-check.sh b/hack/license/header-check.sh index 40086345e..90216cfe2 100755 --- a/hack/license/header-check.sh +++ b/hack/license/header-check.sh @@ -1,23 +1,20 @@ #!/bin/bash -# Copyright ApeCloud, Inc. +#Copyright (C) 2022-2023 ApeCloud Co., Ltd # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#This file is part of KubeBlocks project # -# http://www.apache.org/licenses/LICENSE-2.0 +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU Affero General Public License as published by +#the Free Software Foundation, either version 3 of the License, or +#(at your option) any later version. # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Easy & Dumb header check for CI jobs, currently checks ".go" files only. +#This program is distributed in the hope that it will be useful +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +#GNU Affero General Public License for more details. # -# This will be called by the CI system (with no args) to perform checking and -# fail the job if headers are not correctly set. It can also be called with the -# 'fix' argument to automatically add headers to the missing files. +#You should have received a copy of the GNU Affero General Public License +#along with this program. If not, see . # # Check if headers are fine: # $ ./hack/header-check.sh @@ -32,7 +29,7 @@ FAIL=false for file in $(git ls-files | grep '\.cue\|\.go$' | grep -v vendor/); do echo -n "Header check: $file... " - if [[ -z $(cat ${file} | grep "Copyright ApeCloud, Inc.\|Code generated by") ]]; then + if [[ -z $(cat ${file} | grep "Copyright (C) 2022-2023 ApeCloud Co., Ltd\|Code generated by") ]]; then ERR=true fi if [ $ERR == true ]; then diff --git a/internal/class/class_utils.go b/internal/class/class_utils.go index de5e696e0..2c3c6ae1e 100644 --- a/internal/class/class_utils.go +++ b/internal/class/class_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/class/class_utils_test.go b/internal/class/class_utils_test.go index 4c7a8c3cb..52a418bbb 100644 --- a/internal/class/class_utils_test.go +++ b/internal/class/class_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/class/suite_test.go b/internal/class/suite_test.go index 00be59db9..e802330f4 100644 --- a/internal/class/suite_test.go +++ b/internal/class/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/class/types.go b/internal/class/types.go index 84d79a912..2dca907ed 100644 --- a/internal/class/types.go +++ b/internal/class/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/class/types_test.go b/internal/class/types_test.go index 3f4c018b4..b75c03be1 100644 --- a/internal/class/types_test.go +++ b/internal/class/types_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cloudprovider/interface.go b/internal/cli/cloudprovider/interface.go index 8ba91bdbc..74325abde 100644 --- a/internal/cli/cloudprovider/interface.go +++ b/internal/cli/cloudprovider/interface.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cloudprovider/k3d.go b/internal/cli/cloudprovider/k3d.go index 3561782fa..0a1f9a57d 100644 --- a/internal/cli/cloudprovider/k3d.go +++ b/internal/cli/cloudprovider/k3d.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cloudprovider/k3d_test.go b/internal/cli/cloudprovider/k3d_test.go index f3f16be67..9ab1dbc91 100644 --- a/internal/cli/cloudprovider/k3d_test.go +++ b/internal/cli/cloudprovider/k3d_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cloudprovider/provider.go b/internal/cli/cloudprovider/provider.go index abc89b6d1..75fad4179 100644 --- a/internal/cli/cloudprovider/provider.go +++ b/internal/cli/cloudprovider/provider.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cloudprovider/provider_test.go b/internal/cli/cloudprovider/provider_test.go index 92cf09059..4fe1e4ec3 100644 --- a/internal/cli/cloudprovider/provider_test.go +++ b/internal/cli/cloudprovider/provider_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cloudprovider/suite_test.go b/internal/cli/cloudprovider/suite_test.go index 64b5e67e4..e51e040b8 100644 --- a/internal/cli/cloudprovider/suite_test.go +++ b/internal/cli/cloudprovider/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cloudprovider/terraform.go b/internal/cli/cloudprovider/terraform.go index d52919e0f..933ecacbb 100644 --- a/internal/cli/cloudprovider/terraform.go +++ b/internal/cli/cloudprovider/terraform.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cloudprovider/terraform_test.go b/internal/cli/cloudprovider/terraform_test.go index 07cd4fda8..4b60b4597 100644 --- a/internal/cli/cloudprovider/terraform_test.go +++ b/internal/cli/cloudprovider/terraform_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cloudprovider/types.go b/internal/cli/cloudprovider/types.go index d42fb8bbc..6ed4f3876 100644 --- a/internal/cli/cloudprovider/types.go +++ b/internal/cli/cloudprovider/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cloudprovider diff --git a/internal/cli/cluster/cluster.go b/internal/cli/cluster/cluster.go index 56eb4603f..d3ce6f341 100644 --- a/internal/cli/cluster/cluster.go +++ b/internal/cli/cluster/cluster.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/cluster_test.go b/internal/cli/cluster/cluster_test.go index 1b87ed59f..1a588a467 100644 --- a/internal/cli/cluster/cluster_test.go +++ b/internal/cli/cluster/cluster_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/helper.go b/internal/cli/cluster/helper.go index 0df2e539d..50bebdf89 100644 --- a/internal/cli/cluster/helper.go +++ b/internal/cli/cluster/helper.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/helper_test.go b/internal/cli/cluster/helper_test.go index e48320313..f41cd4775 100644 --- a/internal/cli/cluster/helper_test.go +++ b/internal/cli/cluster/helper_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/name_generator.go b/internal/cli/cluster/name_generator.go index 1e1e2ec67..0bbc76a8c 100644 --- a/internal/cli/cluster/name_generator.go +++ b/internal/cli/cluster/name_generator.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/name_generator_test.go b/internal/cli/cluster/name_generator_test.go index cfc65bc0a..55581f7ac 100644 --- a/internal/cli/cluster/name_generator_test.go +++ b/internal/cli/cluster/name_generator_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/printer.go b/internal/cli/cluster/printer.go index 722fcd445..34d95d2eb 100644 --- a/internal/cli/cluster/printer.go +++ b/internal/cli/cluster/printer.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/printer_test.go b/internal/cli/cluster/printer_test.go index 7cf3d84c7..cd78cc995 100644 --- a/internal/cli/cluster/printer_test.go +++ b/internal/cli/cluster/printer_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/suite_test.go b/internal/cli/cluster/suite_test.go index 53c9e84b3..bc75d56a7 100644 --- a/internal/cli/cluster/suite_test.go +++ b/internal/cli/cluster/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cluster/types.go b/internal/cli/cluster/types.go index 15b536c83..80708e72c 100644 --- a/internal/cli/cluster/types.go +++ b/internal/cli/cluster/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/accounts/base.go b/internal/cli/cmd/accounts/base.go index 4f67cb92a..28e8cf207 100644 --- a/internal/cli/cmd/accounts/base.go +++ b/internal/cli/cmd/accounts/base.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/base_test.go b/internal/cli/cmd/accounts/base_test.go index 646984561..89079e5eb 100644 --- a/internal/cli/cmd/accounts/base_test.go +++ b/internal/cli/cmd/accounts/base_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/create.go b/internal/cli/cmd/accounts/create.go index 0d6e9eecd..0af447433 100644 --- a/internal/cli/cmd/accounts/create.go +++ b/internal/cli/cmd/accounts/create.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/create_test.go b/internal/cli/cmd/accounts/create_test.go index eed815064..0ee74b134 100644 --- a/internal/cli/cmd/accounts/create_test.go +++ b/internal/cli/cmd/accounts/create_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/delete.go b/internal/cli/cmd/accounts/delete.go index 2f7d95cb6..abb4d9e41 100644 --- a/internal/cli/cmd/accounts/delete.go +++ b/internal/cli/cmd/accounts/delete.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/delete_test.go b/internal/cli/cmd/accounts/delete_test.go index f764a08fe..fad99bdfc 100644 --- a/internal/cli/cmd/accounts/delete_test.go +++ b/internal/cli/cmd/accounts/delete_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/describe.go b/internal/cli/cmd/accounts/describe.go index 3754b296d..4941cc743 100644 --- a/internal/cli/cmd/accounts/describe.go +++ b/internal/cli/cmd/accounts/describe.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/describe_test.go b/internal/cli/cmd/accounts/describe_test.go index 83657c495..97e3841c3 100644 --- a/internal/cli/cmd/accounts/describe_test.go +++ b/internal/cli/cmd/accounts/describe_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/grant.go b/internal/cli/cmd/accounts/grant.go index c64f8f75d..5d925a950 100644 --- a/internal/cli/cmd/accounts/grant.go +++ b/internal/cli/cmd/accounts/grant.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/grant_test.go b/internal/cli/cmd/accounts/grant_test.go index b1c778103..582c4098e 100644 --- a/internal/cli/cmd/accounts/grant_test.go +++ b/internal/cli/cmd/accounts/grant_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/list.go b/internal/cli/cmd/accounts/list.go index 9e7392c2e..18bfeeb53 100644 --- a/internal/cli/cmd/accounts/list.go +++ b/internal/cli/cmd/accounts/list.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/list_test.go b/internal/cli/cmd/accounts/list_test.go index 90165739c..e2c971640 100644 --- a/internal/cli/cmd/accounts/list_test.go +++ b/internal/cli/cmd/accounts/list_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/suite_test.go b/internal/cli/cmd/accounts/suite_test.go index 1b9cf327b..3c07b6358 100644 --- a/internal/cli/cmd/accounts/suite_test.go +++ b/internal/cli/cmd/accounts/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/accounts/util.go b/internal/cli/cmd/accounts/util.go index c676d5c18..df1e7c942 100644 --- a/internal/cli/cmd/accounts/util.go +++ b/internal/cli/cmd/accounts/util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package accounts diff --git a/internal/cli/cmd/addon/addon.go b/internal/cli/cmd/addon/addon.go index 63285fa88..9a33d4878 100644 --- a/internal/cli/cmd/addon/addon.go +++ b/internal/cli/cmd/addon/addon.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package addon diff --git a/internal/cli/cmd/addon/addon_test.go b/internal/cli/cmd/addon/addon_test.go index 229a0830a..7dfd321cd 100644 --- a/internal/cli/cmd/addon/addon_test.go +++ b/internal/cli/cmd/addon/addon_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package addon diff --git a/internal/cli/cmd/addon/suite_test.go b/internal/cli/cmd/addon/suite_test.go index c641069f2..760d16b2b 100644 --- a/internal/cli/cmd/addon/suite_test.go +++ b/internal/cli/cmd/addon/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package addon diff --git a/internal/cli/cmd/alert/add_receiver.go b/internal/cli/cmd/alert/add_receiver.go index a0a3a9334..7815a940f 100644 --- a/internal/cli/cmd/alert/add_receiver.go +++ b/internal/cli/cmd/alert/add_receiver.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/add_receiver_test.go b/internal/cli/cmd/alert/add_receiver_test.go index 031279ec1..46ef9bf7c 100644 --- a/internal/cli/cmd/alert/add_receiver_test.go +++ b/internal/cli/cmd/alert/add_receiver_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/alert_test.go b/internal/cli/cmd/alert/alert_test.go index 4e49958bb..c4814f350 100644 --- a/internal/cli/cmd/alert/alert_test.go +++ b/internal/cli/cmd/alert/alert_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/alter.go b/internal/cli/cmd/alert/alter.go index 849dcfb60..0fb237605 100644 --- a/internal/cli/cmd/alert/alter.go +++ b/internal/cli/cmd/alert/alter.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/delete_receiver.go b/internal/cli/cmd/alert/delete_receiver.go index 0e6e37966..e6ad93bbe 100644 --- a/internal/cli/cmd/alert/delete_receiver.go +++ b/internal/cli/cmd/alert/delete_receiver.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/delete_receiver_test.go b/internal/cli/cmd/alert/delete_receiver_test.go index 9998a4041..4101bb2d2 100644 --- a/internal/cli/cmd/alert/delete_receiver_test.go +++ b/internal/cli/cmd/alert/delete_receiver_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/list_receivers.go b/internal/cli/cmd/alert/list_receivers.go index a2577e4cd..a2908ba0b 100644 --- a/internal/cli/cmd/alert/list_receivers.go +++ b/internal/cli/cmd/alert/list_receivers.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/list_receivers_test.go b/internal/cli/cmd/alert/list_receivers_test.go index 77509a4a4..266ba2ed0 100644 --- a/internal/cli/cmd/alert/list_receivers_test.go +++ b/internal/cli/cmd/alert/list_receivers_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/suite_test.go b/internal/cli/cmd/alert/suite_test.go index 7f39d3215..675cd8b53 100644 --- a/internal/cli/cmd/alert/suite_test.go +++ b/internal/cli/cmd/alert/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/types.go b/internal/cli/cmd/alert/types.go index df6eaec49..540f91121 100644 --- a/internal/cli/cmd/alert/types.go +++ b/internal/cli/cmd/alert/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/util.go b/internal/cli/cmd/alert/util.go index 0cc131a26..906012a99 100644 --- a/internal/cli/cmd/alert/util.go +++ b/internal/cli/cmd/alert/util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/alert/util_test.go b/internal/cli/cmd/alert/util_test.go index ad7627cbe..e6828b4c3 100644 --- a/internal/cli/cmd/alert/util_test.go +++ b/internal/cli/cmd/alert/util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package alert diff --git a/internal/cli/cmd/bench/bench.go b/internal/cli/cmd/bench/bench.go index b5f6a6620..2793d9629 100644 --- a/internal/cli/cmd/bench/bench.go +++ b/internal/cli/cmd/bench/bench.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package bench diff --git a/internal/cli/cmd/bench/bench_test.go b/internal/cli/cmd/bench/bench_test.go index 4db5fd80c..23f10b632 100644 --- a/internal/cli/cmd/bench/bench_test.go +++ b/internal/cli/cmd/bench/bench_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package bench diff --git a/internal/cli/cmd/bench/suite_test.go b/internal/cli/cmd/bench/suite_test.go index fae3f9201..61c76790c 100644 --- a/internal/cli/cmd/bench/suite_test.go +++ b/internal/cli/cmd/bench/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package bench diff --git a/internal/cli/cmd/bench/tpcc.go b/internal/cli/cmd/bench/tpcc.go index a4a48995c..69425f202 100644 --- a/internal/cli/cmd/bench/tpcc.go +++ b/internal/cli/cmd/bench/tpcc.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package bench diff --git a/internal/cli/cmd/bench/util.go b/internal/cli/cmd/bench/util.go index 2e91334e2..c834fc7cc 100644 --- a/internal/cli/cmd/bench/util.go +++ b/internal/cli/cmd/bench/util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package bench diff --git a/internal/cli/cmd/class/class.go b/internal/cli/cmd/class/class.go index 27e999017..bde026923 100644 --- a/internal/cli/cmd/class/class.go +++ b/internal/cli/cmd/class/class.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/class/class_test.go b/internal/cli/cmd/class/class_test.go index 58544ac58..c4d92e43a 100644 --- a/internal/cli/cmd/class/class_test.go +++ b/internal/cli/cmd/class/class_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/class/create.go b/internal/cli/cmd/class/create.go index 37d0f5faa..747acf1d0 100644 --- a/internal/cli/cmd/class/create.go +++ b/internal/cli/cmd/class/create.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/class/create_test.go b/internal/cli/cmd/class/create_test.go index cbeb56760..966154be9 100644 --- a/internal/cli/cmd/class/create_test.go +++ b/internal/cli/cmd/class/create_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/class/list.go b/internal/cli/cmd/class/list.go index 45f3313f9..dccff40a5 100644 --- a/internal/cli/cmd/class/list.go +++ b/internal/cli/cmd/class/list.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/class/list_test.go b/internal/cli/cmd/class/list_test.go index 550e3f15b..508f3d97c 100644 --- a/internal/cli/cmd/class/list_test.go +++ b/internal/cli/cmd/class/list_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/class/suite_test.go b/internal/cli/cmd/class/suite_test.go index 907e4eacf..a027135aa 100644 --- a/internal/cli/cmd/class/suite_test.go +++ b/internal/cli/cmd/class/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/class/template.go b/internal/cli/cmd/class/template.go index 9e288b86f..c30b12a0d 100644 --- a/internal/cli/cmd/class/template.go +++ b/internal/cli/cmd/class/template.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/class/template_test.go b/internal/cli/cmd/class/template_test.go index e2a239d3d..c9ad814e8 100644 --- a/internal/cli/cmd/class/template_test.go +++ b/internal/cli/cmd/class/template_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package class diff --git a/internal/cli/cmd/cli.go b/internal/cli/cmd/cli.go index b70cd9700..7f128585c 100644 --- a/internal/cli/cmd/cli.go +++ b/internal/cli/cmd/cli.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cmd diff --git a/internal/cli/cmd/cluster/accounts.go b/internal/cli/cmd/cluster/accounts.go index d9162a005..b56b5caab 100644 --- a/internal/cli/cmd/cluster/accounts.go +++ b/internal/cli/cmd/cluster/accounts.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/cluster.go b/internal/cli/cmd/cluster/cluster.go index a213e981c..1778d42d8 100644 --- a/internal/cli/cmd/cluster/cluster.go +++ b/internal/cli/cmd/cluster/cluster.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/cluster_test.go b/internal/cli/cmd/cluster/cluster_test.go index d072aa6a1..063b4b03d 100644 --- a/internal/cli/cmd/cluster/cluster_test.go +++ b/internal/cli/cmd/cluster/cluster_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/config.go b/internal/cli/cmd/cluster/config.go index 67c5c3050..aac7c2357 100644 --- a/internal/cli/cmd/cluster/config.go +++ b/internal/cli/cmd/cluster/config.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/config_edit.go b/internal/cli/cmd/cluster/config_edit.go index f88335ba0..151b09e11 100644 --- a/internal/cli/cmd/cluster/config_edit.go +++ b/internal/cli/cmd/cluster/config_edit.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/config_ops.go b/internal/cli/cmd/cluster/config_ops.go index c0c398a9a..915de1b58 100644 --- a/internal/cli/cmd/cluster/config_ops.go +++ b/internal/cli/cmd/cluster/config_ops.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/config_ops_test.go b/internal/cli/cmd/cluster/config_ops_test.go index 1f6445db6..4c32d2dfa 100644 --- a/internal/cli/cmd/cluster/config_ops_test.go +++ b/internal/cli/cmd/cluster/config_ops_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/config_util.go b/internal/cli/cmd/cluster/config_util.go index f76274a05..9a9d7695e 100644 --- a/internal/cli/cmd/cluster/config_util.go +++ b/internal/cli/cmd/cluster/config_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/config_util_test.go b/internal/cli/cmd/cluster/config_util_test.go index 0dfe2d71a..402d05076 100644 --- a/internal/cli/cmd/cluster/config_util_test.go +++ b/internal/cli/cmd/cluster/config_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/config_wrapper.go b/internal/cli/cmd/cluster/config_wrapper.go index 5b7292c51..3d785c342 100644 --- a/internal/cli/cmd/cluster/config_wrapper.go +++ b/internal/cli/cmd/cluster/config_wrapper.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/connect.go b/internal/cli/cmd/cluster/connect.go index 3c55dc184..c4683524d 100644 --- a/internal/cli/cmd/cluster/connect.go +++ b/internal/cli/cmd/cluster/connect.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/connect_test.go b/internal/cli/cmd/cluster/connect_test.go index 18b69e86e..d09eb309f 100644 --- a/internal/cli/cmd/cluster/connect_test.go +++ b/internal/cli/cmd/cluster/connect_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 1da2540ee..86d97cc1a 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/create_test.go b/internal/cli/cmd/cluster/create_test.go index 76b322458..b377ea004 100644 --- a/internal/cli/cmd/cluster/create_test.go +++ b/internal/cli/cmd/cluster/create_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 06a3e36e9..eae2723ea 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index 4bafedac1..f9de33a42 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/delete.go b/internal/cli/cmd/cluster/delete.go index 2ffb250ac..dda4985b2 100644 --- a/internal/cli/cmd/cluster/delete.go +++ b/internal/cli/cmd/cluster/delete.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/delete_ops.go b/internal/cli/cmd/cluster/delete_ops.go index ae22df2a5..e8264ad34 100644 --- a/internal/cli/cmd/cluster/delete_ops.go +++ b/internal/cli/cmd/cluster/delete_ops.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/describe.go b/internal/cli/cmd/cluster/describe.go index f2116947a..5b8fec2ce 100644 --- a/internal/cli/cmd/cluster/describe.go +++ b/internal/cli/cmd/cluster/describe.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/describe_ops.go b/internal/cli/cmd/cluster/describe_ops.go index a317b13c3..8e166124e 100644 --- a/internal/cli/cmd/cluster/describe_ops.go +++ b/internal/cli/cmd/cluster/describe_ops.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/describe_ops_test.go b/internal/cli/cmd/cluster/describe_ops_test.go index d5e927065..cbbb710bd 100644 --- a/internal/cli/cmd/cluster/describe_ops_test.go +++ b/internal/cli/cmd/cluster/describe_ops_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/describe_test.go b/internal/cli/cmd/cluster/describe_test.go index a6ff75c5f..7b99f538c 100644 --- a/internal/cli/cmd/cluster/describe_test.go +++ b/internal/cli/cmd/cluster/describe_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/errors.go b/internal/cli/cmd/cluster/errors.go index 92b28622a..6e588ae3f 100644 --- a/internal/cli/cmd/cluster/errors.go +++ b/internal/cli/cmd/cluster/errors.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/label.go b/internal/cli/cmd/cluster/label.go index e0a8294d2..457df3c32 100644 --- a/internal/cli/cmd/cluster/label.go +++ b/internal/cli/cmd/cluster/label.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/label_test.go b/internal/cli/cmd/cluster/label_test.go index 3093eaa63..1299a736f 100644 --- a/internal/cli/cmd/cluster/label_test.go +++ b/internal/cli/cmd/cluster/label_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/list.go b/internal/cli/cmd/cluster/list.go index f005148c6..25f751a3c 100644 --- a/internal/cli/cmd/cluster/list.go +++ b/internal/cli/cmd/cluster/list.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/list_logs.go b/internal/cli/cmd/cluster/list_logs.go index 48ee0c912..800305f90 100644 --- a/internal/cli/cmd/cluster/list_logs.go +++ b/internal/cli/cmd/cluster/list_logs.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/list_logs_test.go b/internal/cli/cmd/cluster/list_logs_test.go index 3098f8914..888c01b69 100644 --- a/internal/cli/cmd/cluster/list_logs_test.go +++ b/internal/cli/cmd/cluster/list_logs_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/list_ops.go b/internal/cli/cmd/cluster/list_ops.go index 045d5db59..6f1d2d94b 100644 --- a/internal/cli/cmd/cluster/list_ops.go +++ b/internal/cli/cmd/cluster/list_ops.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/list_ops_test.go b/internal/cli/cmd/cluster/list_ops_test.go index 97918f6bd..25867b333 100644 --- a/internal/cli/cmd/cluster/list_ops_test.go +++ b/internal/cli/cmd/cluster/list_ops_test.go @@ -1,18 +1,22 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ + package cluster import ( diff --git a/internal/cli/cmd/cluster/list_test.go b/internal/cli/cmd/cluster/list_test.go index fc8679273..5cbd7ebfd 100644 --- a/internal/cli/cmd/cluster/list_test.go +++ b/internal/cli/cmd/cluster/list_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/logs.go b/internal/cli/cmd/cluster/logs.go index c4eceb113..bd133bb61 100644 --- a/internal/cli/cmd/cluster/logs.go +++ b/internal/cli/cmd/cluster/logs.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/logs_test.go b/internal/cli/cmd/cluster/logs_test.go index 7835c7723..70126f7fe 100644 --- a/internal/cli/cmd/cluster/logs_test.go +++ b/internal/cli/cmd/cluster/logs_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 584bad611..09e223a43 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/operations_test.go b/internal/cli/cmd/cluster/operations_test.go index 7f6399770..7942ef593 100644 --- a/internal/cli/cmd/cluster/operations_test.go +++ b/internal/cli/cmd/cluster/operations_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/suite_test.go b/internal/cli/cmd/cluster/suite_test.go index 53c9e84b3..bc75d56a7 100644 --- a/internal/cli/cmd/cluster/suite_test.go +++ b/internal/cli/cmd/cluster/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/update.go b/internal/cli/cmd/cluster/update.go index 3810b269e..267944b19 100644 --- a/internal/cli/cmd/cluster/update.go +++ b/internal/cli/cmd/cluster/update.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/cluster/update_test.go b/internal/cli/cmd/cluster/update_test.go index dae0d7deb..ba9da4d2a 100644 --- a/internal/cli/cmd/cluster/update_test.go +++ b/internal/cli/cmd/cluster/update_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package cluster diff --git a/internal/cli/cmd/clusterdefinition/clusterdefinition.go b/internal/cli/cmd/clusterdefinition/clusterdefinition.go index 045fec933..d217e14c4 100644 --- a/internal/cli/cmd/clusterdefinition/clusterdefinition.go +++ b/internal/cli/cmd/clusterdefinition/clusterdefinition.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package clusterdefinition diff --git a/internal/cli/cmd/clusterdefinition/clusterdefinition_test.go b/internal/cli/cmd/clusterdefinition/clusterdefinition_test.go index 5c9db2eae..6673b181f 100644 --- a/internal/cli/cmd/clusterdefinition/clusterdefinition_test.go +++ b/internal/cli/cmd/clusterdefinition/clusterdefinition_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package clusterdefinition diff --git a/internal/cli/cmd/clusterdefinition/suite_test.go b/internal/cli/cmd/clusterdefinition/suite_test.go index 2136ee232..204a1b093 100644 --- a/internal/cli/cmd/clusterdefinition/suite_test.go +++ b/internal/cli/cmd/clusterdefinition/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package clusterdefinition diff --git a/internal/cli/cmd/clusterversion/clusterversion.go b/internal/cli/cmd/clusterversion/clusterversion.go index d83f83042..9be9d8480 100644 --- a/internal/cli/cmd/clusterversion/clusterversion.go +++ b/internal/cli/cmd/clusterversion/clusterversion.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package clusterversion diff --git a/internal/cli/cmd/clusterversion/clusterversion_test.go b/internal/cli/cmd/clusterversion/clusterversion_test.go index 501054cd5..7c9ba84ec 100644 --- a/internal/cli/cmd/clusterversion/clusterversion_test.go +++ b/internal/cli/cmd/clusterversion/clusterversion_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package clusterversion diff --git a/internal/cli/cmd/clusterversion/suite_test.go b/internal/cli/cmd/clusterversion/suite_test.go index 3a569110f..32a4136e3 100644 --- a/internal/cli/cmd/clusterversion/suite_test.go +++ b/internal/cli/cmd/clusterversion/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package clusterversion diff --git a/internal/cli/cmd/dashboard/dashboard.go b/internal/cli/cmd/dashboard/dashboard.go index feb0ae1c4..6fcdf73dd 100644 --- a/internal/cli/cmd/dashboard/dashboard.go +++ b/internal/cli/cmd/dashboard/dashboard.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dashboard diff --git a/internal/cli/cmd/dashboard/dashboard_test.go b/internal/cli/cmd/dashboard/dashboard_test.go index 050430a0b..f0dcf960c 100644 --- a/internal/cli/cmd/dashboard/dashboard_test.go +++ b/internal/cli/cmd/dashboard/dashboard_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dashboard diff --git a/internal/cli/cmd/dashboard/suite_test.go b/internal/cli/cmd/dashboard/suite_test.go index fa1d7380c..a42645afd 100644 --- a/internal/cli/cmd/dashboard/suite_test.go +++ b/internal/cli/cmd/dashboard/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package dashboard diff --git a/internal/cli/cmd/kubeblocks/config.go b/internal/cli/cmd/kubeblocks/config.go index 5b66ca756..c606e555b 100644 --- a/internal/cli/cmd/kubeblocks/config.go +++ b/internal/cli/cmd/kubeblocks/config.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/config_test.go b/internal/cli/cmd/kubeblocks/config_test.go index 814f18656..915d98f46 100644 --- a/internal/cli/cmd/kubeblocks/config_test.go +++ b/internal/cli/cmd/kubeblocks/config_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index 6362fe1c4..6f37a8459 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/install_test.go b/internal/cli/cmd/kubeblocks/install_test.go index 9135080d3..4e4687310 100644 --- a/internal/cli/cmd/kubeblocks/install_test.go +++ b/internal/cli/cmd/kubeblocks/install_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/kubeblocks.go b/internal/cli/cmd/kubeblocks/kubeblocks.go index 33ee0c830..256e8a913 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/kubeblocks_objects.go b/internal/cli/cmd/kubeblocks/kubeblocks_objects.go index 6181ed6a9..f9ee5fcd3 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks_objects.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks_objects.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/kubeblocks_objects_test.go b/internal/cli/cmd/kubeblocks/kubeblocks_objects_test.go index bc6f63c2f..692527304 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks_objects_test.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks_objects_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/kubeblocks_test.go b/internal/cli/cmd/kubeblocks/kubeblocks_test.go index 2b373fe64..002619112 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks_test.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/list_versions.go b/internal/cli/cmd/kubeblocks/list_versions.go index 799a035e7..68fbcb203 100644 --- a/internal/cli/cmd/kubeblocks/list_versions.go +++ b/internal/cli/cmd/kubeblocks/list_versions.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/list_versions_test.go b/internal/cli/cmd/kubeblocks/list_versions_test.go index 93bc75f0c..a485d31db 100644 --- a/internal/cli/cmd/kubeblocks/list_versions_test.go +++ b/internal/cli/cmd/kubeblocks/list_versions_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/preflight.go b/internal/cli/cmd/kubeblocks/preflight.go index db097e08d..88693972d 100644 --- a/internal/cli/cmd/kubeblocks/preflight.go +++ b/internal/cli/cmd/kubeblocks/preflight.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/preflight_test.go b/internal/cli/cmd/kubeblocks/preflight_test.go index f21944588..b60f3f036 100644 --- a/internal/cli/cmd/kubeblocks/preflight_test.go +++ b/internal/cli/cmd/kubeblocks/preflight_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/status.go b/internal/cli/cmd/kubeblocks/status.go index f963a3ef7..028047135 100644 --- a/internal/cli/cmd/kubeblocks/status.go +++ b/internal/cli/cmd/kubeblocks/status.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/status_test.go b/internal/cli/cmd/kubeblocks/status_test.go index 5df1db11f..583531cfa 100644 --- a/internal/cli/cmd/kubeblocks/status_test.go +++ b/internal/cli/cmd/kubeblocks/status_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/suite_test.go b/internal/cli/cmd/kubeblocks/suite_test.go index d53d1305b..c352a773a 100644 --- a/internal/cli/cmd/kubeblocks/suite_test.go +++ b/internal/cli/cmd/kubeblocks/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index d52a89d5c..1629ee7ba 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/uninstall_test.go b/internal/cli/cmd/kubeblocks/uninstall_test.go index 947f652c2..bb4f94880 100644 --- a/internal/cli/cmd/kubeblocks/uninstall_test.go +++ b/internal/cli/cmd/kubeblocks/uninstall_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/upgrade.go b/internal/cli/cmd/kubeblocks/upgrade.go index 534d44317..a49607073 100644 --- a/internal/cli/cmd/kubeblocks/upgrade.go +++ b/internal/cli/cmd/kubeblocks/upgrade.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/upgrade_test.go b/internal/cli/cmd/kubeblocks/upgrade_test.go index 65fb4ec2c..97a677d53 100644 --- a/internal/cli/cmd/kubeblocks/upgrade_test.go +++ b/internal/cli/cmd/kubeblocks/upgrade_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/util.go b/internal/cli/cmd/kubeblocks/util.go index ea5dae03b..a73834256 100644 --- a/internal/cli/cmd/kubeblocks/util.go +++ b/internal/cli/cmd/kubeblocks/util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/kubeblocks/util_test.go b/internal/cli/cmd/kubeblocks/util_test.go index 66bf3daea..86fddcf7d 100644 --- a/internal/cli/cmd/kubeblocks/util_test.go +++ b/internal/cli/cmd/kubeblocks/util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package kubeblocks diff --git a/internal/cli/cmd/migration/base.go b/internal/cli/cmd/migration/base.go index b16e0e673..03b3fa5cd 100644 --- a/internal/cli/cmd/migration/base.go +++ b/internal/cli/cmd/migration/base.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/base_test.go b/internal/cli/cmd/migration/base_test.go index 04cd0a40d..171555a08 100644 --- a/internal/cli/cmd/migration/base_test.go +++ b/internal/cli/cmd/migration/base_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/cmd_builder.go b/internal/cli/cmd/migration/cmd_builder.go index b3c55da41..57d276b84 100644 --- a/internal/cli/cmd/migration/cmd_builder.go +++ b/internal/cli/cmd/migration/cmd_builder.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/cmd_builder_test.go b/internal/cli/cmd/migration/cmd_builder_test.go index 7656131a2..dc1105d26 100644 --- a/internal/cli/cmd/migration/cmd_builder_test.go +++ b/internal/cli/cmd/migration/cmd_builder_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/create.go b/internal/cli/cmd/migration/create.go index 3a99c9f94..81967e8f5 100644 --- a/internal/cli/cmd/migration/create.go +++ b/internal/cli/cmd/migration/create.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/create_test.go b/internal/cli/cmd/migration/create_test.go index 9a0407877..1c7456852 100644 --- a/internal/cli/cmd/migration/create_test.go +++ b/internal/cli/cmd/migration/create_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/describe.go b/internal/cli/cmd/migration/describe.go index ce9d41310..4ea8a5d94 100644 --- a/internal/cli/cmd/migration/describe.go +++ b/internal/cli/cmd/migration/describe.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/describe_test.go b/internal/cli/cmd/migration/describe_test.go index af7b1bcb2..1ce4890cd 100644 --- a/internal/cli/cmd/migration/describe_test.go +++ b/internal/cli/cmd/migration/describe_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/examples.go b/internal/cli/cmd/migration/examples.go index 0a61280a9..7dd35f366 100644 --- a/internal/cli/cmd/migration/examples.go +++ b/internal/cli/cmd/migration/examples.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/list.go b/internal/cli/cmd/migration/list.go index 1f1a25e67..49b21abe2 100644 --- a/internal/cli/cmd/migration/list.go +++ b/internal/cli/cmd/migration/list.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/list_test.go b/internal/cli/cmd/migration/list_test.go index 0f34095d6..01be8a3b6 100644 --- a/internal/cli/cmd/migration/list_test.go +++ b/internal/cli/cmd/migration/list_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/logs.go b/internal/cli/cmd/migration/logs.go index 35b1c0d02..3a342fd4e 100644 --- a/internal/cli/cmd/migration/logs.go +++ b/internal/cli/cmd/migration/logs.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/logs_test.go b/internal/cli/cmd/migration/logs_test.go index e2ecdefac..d97bb04ef 100644 --- a/internal/cli/cmd/migration/logs_test.go +++ b/internal/cli/cmd/migration/logs_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/suite_test.go b/internal/cli/cmd/migration/suite_test.go index e2ac143ec..d682b3e74 100644 --- a/internal/cli/cmd/migration/suite_test.go +++ b/internal/cli/cmd/migration/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration_test diff --git a/internal/cli/cmd/migration/templates.go b/internal/cli/cmd/migration/templates.go index 951370ae5..f1bb2ca51 100644 --- a/internal/cli/cmd/migration/templates.go +++ b/internal/cli/cmd/migration/templates.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/templates_test.go b/internal/cli/cmd/migration/templates_test.go index 3c8e538e0..b7e5662a4 100644 --- a/internal/cli/cmd/migration/templates_test.go +++ b/internal/cli/cmd/migration/templates_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/terminate.go b/internal/cli/cmd/migration/terminate.go index 22e188d57..9c476b681 100644 --- a/internal/cli/cmd/migration/terminate.go +++ b/internal/cli/cmd/migration/terminate.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/migration/terminate_test.go b/internal/cli/cmd/migration/terminate_test.go index 0dafce7d0..bb443343a 100644 --- a/internal/cli/cmd/migration/terminate_test.go +++ b/internal/cli/cmd/migration/terminate_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package migration diff --git a/internal/cli/cmd/options/options.go b/internal/cli/cmd/options/options.go index fd6ba5ca3..08172cf77 100644 --- a/internal/cli/cmd/options/options.go +++ b/internal/cli/cmd/options/options.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package options diff --git a/internal/cli/cmd/options/options_test.go b/internal/cli/cmd/options/options_test.go index f8265f3fd..d10a6740c 100644 --- a/internal/cli/cmd/options/options_test.go +++ b/internal/cli/cmd/options/options_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package options diff --git a/internal/cli/cmd/playground/base.go b/internal/cli/cmd/playground/base.go index dfaea297e..3742e7637 100644 --- a/internal/cli/cmd/playground/base.go +++ b/internal/cli/cmd/playground/base.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/destroy.go b/internal/cli/cmd/playground/destroy.go index 371ef4cad..543397b99 100644 --- a/internal/cli/cmd/playground/destroy.go +++ b/internal/cli/cmd/playground/destroy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/destroy_test.go b/internal/cli/cmd/playground/destroy_test.go index f7cdcad37..0cfb5c819 100644 --- a/internal/cli/cmd/playground/destroy_test.go +++ b/internal/cli/cmd/playground/destroy_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index d96bea81c..bfbc499cc 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/init_test.go b/internal/cli/cmd/playground/init_test.go index f6ae12944..5fb84a097 100644 --- a/internal/cli/cmd/playground/init_test.go +++ b/internal/cli/cmd/playground/init_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/kubeconfig.go b/internal/cli/cmd/playground/kubeconfig.go index 91d67a1c4..71c0c0a66 100644 --- a/internal/cli/cmd/playground/kubeconfig.go +++ b/internal/cli/cmd/playground/kubeconfig.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/kubeconfig_test.go b/internal/cli/cmd/playground/kubeconfig_test.go index 0fd8ff4f3..b64a333ae 100644 --- a/internal/cli/cmd/playground/kubeconfig_test.go +++ b/internal/cli/cmd/playground/kubeconfig_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/palyground.go b/internal/cli/cmd/playground/palyground.go index d44077b9c..68f5543f6 100644 --- a/internal/cli/cmd/playground/palyground.go +++ b/internal/cli/cmd/playground/palyground.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/playground_test.go b/internal/cli/cmd/playground/playground_test.go index 21e3ce01c..021e57abe 100644 --- a/internal/cli/cmd/playground/playground_test.go +++ b/internal/cli/cmd/playground/playground_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/suite_test.go b/internal/cli/cmd/playground/suite_test.go index 4486a9b60..2f4af0b68 100644 --- a/internal/cli/cmd/playground/suite_test.go +++ b/internal/cli/cmd/playground/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/types.go b/internal/cli/cmd/playground/types.go index 90913fe24..4ec7c12ae 100644 --- a/internal/cli/cmd/playground/types.go +++ b/internal/cli/cmd/playground/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/util.go b/internal/cli/cmd/playground/util.go index d60993e87..fe491371f 100644 --- a/internal/cli/cmd/playground/util.go +++ b/internal/cli/cmd/playground/util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/playground/util_test.go b/internal/cli/cmd/playground/util_test.go index d3d59b9a2..fdbb49c07 100644 --- a/internal/cli/cmd/playground/util_test.go +++ b/internal/cli/cmd/playground/util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package playground diff --git a/internal/cli/cmd/version/suite_test.go b/internal/cli/cmd/version/suite_test.go index 0e19b0b64..43f4ae546 100644 --- a/internal/cli/cmd/version/suite_test.go +++ b/internal/cli/cmd/version/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package version diff --git a/internal/cli/cmd/version/version.go b/internal/cli/cmd/version/version.go index bc9b71f62..697b0a295 100644 --- a/internal/cli/cmd/version/version.go +++ b/internal/cli/cmd/version/version.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package version diff --git a/internal/cli/cmd/version/version_test.go b/internal/cli/cmd/version/version_test.go index 5bd9c9ac6..f3c1d9c25 100644 --- a/internal/cli/cmd/version/version_test.go +++ b/internal/cli/cmd/version/version_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package version diff --git a/internal/cli/create/create.go b/internal/cli/create/create.go index d8acc8498..15d3ac586 100755 --- a/internal/cli/create/create.go +++ b/internal/cli/create/create.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package create diff --git a/internal/cli/create/create_test.go b/internal/cli/create/create_test.go index c05a38438..ff9640ca0 100755 --- a/internal/cli/create/create_test.go +++ b/internal/cli/create/create_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package create diff --git a/internal/cli/create/suite_test.go b/internal/cli/create/suite_test.go index 6567bd8b3..b7857ebc9 100644 --- a/internal/cli/create/suite_test.go +++ b/internal/cli/create/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package create diff --git a/internal/cli/create/template/backup_template.cue b/internal/cli/create/template/backup_template.cue index 67064647c..85d37eabf 100644 --- a/internal/cli/create/template/backup_template.cue +++ b/internal/cli/create/template/backup_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // required, command line input options for parameters and flags options: { diff --git a/internal/cli/create/template/cluster_operations_template.cue b/internal/cli/create/template/cluster_operations_template.cue index 53240c4a3..3c1f9c3d3 100644 --- a/internal/cli/create/template/cluster_operations_template.cue +++ b/internal/cli/create/template/cluster_operations_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // required, command line input options for parameters and flags options: { diff --git a/internal/cli/create/template/cluster_template.cue b/internal/cli/create/template/cluster_template.cue index 6ccc528e0..75b5b7a92 100644 --- a/internal/cli/create/template/cluster_template.cue +++ b/internal/cli/create/template/cluster_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // required, command line input options for parameters and flags options: { diff --git a/internal/cli/create/template/create_template_test.cue b/internal/cli/create/template/create_template_test.cue index 4309a6db7..e487417b9 100644 --- a/internal/cli/create/template/create_template_test.cue +++ b/internal/cli/create/template/create_template_test.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // required, options for command line input for args and flags. options: { diff --git a/internal/cli/create/template/migration_template.cue b/internal/cli/create/template/migration_template.cue index 718f81e3f..83ffc3bc8 100644 --- a/internal/cli/create/template/migration_template.cue +++ b/internal/cli/create/template/migration_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // required, command line input options for parameters and flags options: { diff --git a/internal/cli/create/template/volumesnapshotclass_template.cue b/internal/cli/create/template/volumesnapshotclass_template.cue index 1730dca9f..81c446be8 100644 --- a/internal/cli/create/template/volumesnapshotclass_template.cue +++ b/internal/cli/create/template/volumesnapshotclass_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // required, command line input options for parameters and flags options: { diff --git a/internal/cli/delete/delete.go b/internal/cli/delete/delete.go index 2816e6eb0..b077d3662 100644 --- a/internal/cli/delete/delete.go +++ b/internal/cli/delete/delete.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package delete diff --git a/internal/cli/delete/delete_test.go b/internal/cli/delete/delete_test.go index 9959e7ab2..4ec5ef80b 100644 --- a/internal/cli/delete/delete_test.go +++ b/internal/cli/delete/delete_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package delete diff --git a/internal/cli/delete/suite_test.go b/internal/cli/delete/suite_test.go index e0268bd23..690969207 100644 --- a/internal/cli/delete/suite_test.go +++ b/internal/cli/delete/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package delete diff --git a/internal/cli/edit/edit.go b/internal/cli/edit/edit.go index edc89c737..59fcb3d0f 100644 --- a/internal/cli/edit/edit.go +++ b/internal/cli/edit/edit.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package edit diff --git a/internal/cli/edit/edit_test.go b/internal/cli/edit/edit_test.go index 6d31d7a8b..62b1593a7 100644 --- a/internal/cli/edit/edit_test.go +++ b/internal/cli/edit/edit_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package edit diff --git a/internal/cli/edit/suite_test.go b/internal/cli/edit/suite_test.go index 621f29747..0d5f0022f 100644 --- a/internal/cli/edit/suite_test.go +++ b/internal/cli/edit/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package edit diff --git a/internal/cli/exec/exec.go b/internal/cli/exec/exec.go index b59b9ca41..48e58f27a 100644 --- a/internal/cli/exec/exec.go +++ b/internal/cli/exec/exec.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package exec diff --git a/internal/cli/exec/exec_test.go b/internal/cli/exec/exec_test.go index cada2f46f..015556e53 100644 --- a/internal/cli/exec/exec_test.go +++ b/internal/cli/exec/exec_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package exec diff --git a/internal/cli/exec/suite_test.go b/internal/cli/exec/suite_test.go index da8c4d9ff..7533619ca 100644 --- a/internal/cli/exec/suite_test.go +++ b/internal/cli/exec/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package exec diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index b054c67e0..3c3209d6c 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package list diff --git a/internal/cli/list/list_test.go b/internal/cli/list/list_test.go index f4e3b8b84..fcf8b4cde 100644 --- a/internal/cli/list/list_test.go +++ b/internal/cli/list/list_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package list diff --git a/internal/cli/list/suite_test.go b/internal/cli/list/suite_test.go index d4f3fa80a..958d5b313 100644 --- a/internal/cli/list/suite_test.go +++ b/internal/cli/list/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package list diff --git a/internal/cli/patch/patch.go b/internal/cli/patch/patch.go index a021fb1e6..900a50cc8 100644 --- a/internal/cli/patch/patch.go +++ b/internal/cli/patch/patch.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package patch diff --git a/internal/cli/patch/patch_test.go b/internal/cli/patch/patch_test.go index d422904fe..987f04546 100644 --- a/internal/cli/patch/patch_test.go +++ b/internal/cli/patch/patch_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package patch diff --git a/internal/cli/patch/suite_test.go b/internal/cli/patch/suite_test.go index e0ed40d36..1cd0d5bfe 100644 --- a/internal/cli/patch/suite_test.go +++ b/internal/cli/patch/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package patch diff --git a/internal/cli/printer/describe.go b/internal/cli/printer/describe.go index 2c176a83b..671f12ec5 100644 --- a/internal/cli/printer/describe.go +++ b/internal/cli/printer/describe.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/printer/describe_test.go b/internal/cli/printer/describe_test.go index 97089930f..7a8683e07 100644 --- a/internal/cli/printer/describe_test.go +++ b/internal/cli/printer/describe_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/printer/format.go b/internal/cli/printer/format.go index aac2024c3..fbb56db4b 100755 --- a/internal/cli/printer/format.go +++ b/internal/cli/printer/format.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/printer/format_test.go b/internal/cli/printer/format_test.go index 2a3d16511..ef133ea14 100644 --- a/internal/cli/printer/format_test.go +++ b/internal/cli/printer/format_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/printer/helper.go b/internal/cli/printer/helper.go index 02c4cf64d..b54bb5a46 100644 --- a/internal/cli/printer/helper.go +++ b/internal/cli/printer/helper.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/printer/printer.go b/internal/cli/printer/printer.go index cf8b01736..0fe436b9e 100644 --- a/internal/cli/printer/printer.go +++ b/internal/cli/printer/printer.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/printer/printer_test.go b/internal/cli/printer/printer_test.go index 27e7d0410..0de5b490b 100644 --- a/internal/cli/printer/printer_test.go +++ b/internal/cli/printer/printer_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/printer/spinner.go b/internal/cli/printer/spinner.go index ae306c3ad..dacd218bd 100644 --- a/internal/cli/printer/spinner.go +++ b/internal/cli/printer/spinner.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/printer/spinner_test.go b/internal/cli/printer/spinner_test.go index 0de16208a..da7c1a59b 100644 --- a/internal/cli/printer/spinner_test.go +++ b/internal/cli/printer/spinner_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package printer diff --git a/internal/cli/testing/client.go b/internal/cli/testing/client.go index cd8551be8..147d0e2c9 100644 --- a/internal/cli/testing/client.go +++ b/internal/cli/testing/client.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/cli/testing/factory.go b/internal/cli/testing/factory.go index b4d658a80..d1e9edde4 100644 --- a/internal/cli/testing/factory.go +++ b/internal/cli/testing/factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/cli/testing/factory_test.go b/internal/cli/testing/factory_test.go index 8eccdcc64..891f82f60 100644 --- a/internal/cli/testing/factory_test.go +++ b/internal/cli/testing/factory_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index 44bbbb5a5..eabf961e0 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/cli/testing/fake_test.go b/internal/cli/testing/fake_test.go index ce2e20f1d..4be1ccab1 100644 --- a/internal/cli/testing/fake_test.go +++ b/internal/cli/testing/fake_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/cli/testing/printer.go b/internal/cli/testing/printer.go index f9ccd2a64..f4fa3c1ba 100644 --- a/internal/cli/testing/printer.go +++ b/internal/cli/testing/printer.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/cli/testing/suite_test.go b/internal/cli/testing/suite_test.go index c112d986b..730ff3d2e 100644 --- a/internal/cli/testing/suite_test.go +++ b/internal/cli/testing/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/cli/types/migrationapi/migration_object_express.go b/internal/cli/types/migrationapi/migration_object_express.go index c858ad480..00c4ded8d 100644 --- a/internal/cli/types/migrationapi/migration_object_express.go +++ b/internal/cli/types/migrationapi/migration_object_express.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/internal/cli/types/migrationapi/migrationtask_types.go b/internal/cli/types/migrationapi/migrationtask_types.go index 040a08f8b..9121ddd23 100644 --- a/internal/cli/types/migrationapi/migrationtask_types.go +++ b/internal/cli/types/migrationapi/migrationtask_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/internal/cli/types/migrationapi/migrationtemplate_types.go b/internal/cli/types/migrationapi/migrationtemplate_types.go index 6119bf3f1..66acc6e6a 100644 --- a/internal/cli/types/migrationapi/migrationtemplate_types.go +++ b/internal/cli/types/migrationapi/migrationtemplate_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/internal/cli/types/migrationapi/type.go b/internal/cli/types/migrationapi/type.go index 623a5550e..01aa75a8d 100644 --- a/internal/cli/types/migrationapi/type.go +++ b/internal/cli/types/migrationapi/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package v1alpha1 diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index a50b42686..f27622abd 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package types diff --git a/internal/cli/util/completion.go b/internal/cli/util/completion.go index 69e0df96f..deac3fa87 100644 --- a/internal/cli/util/completion.go +++ b/internal/cli/util/completion.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/error.go b/internal/cli/util/error.go index 3ff7810ce..530fa30e1 100644 --- a/internal/cli/util/error.go +++ b/internal/cli/util/error.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/error_test.go b/internal/cli/util/error_test.go index a8ae8a536..6a841050e 100644 --- a/internal/cli/util/error_test.go +++ b/internal/cli/util/error_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/git.go b/internal/cli/util/git.go index 520202abc..b511b9cdd 100644 --- a/internal/cli/util/git.go +++ b/internal/cli/util/git.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/helm/config.go b/internal/cli/util/helm/config.go index 74ca56e4e..8203e0719 100644 --- a/internal/cli/util/helm/config.go +++ b/internal/cli/util/helm/config.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package helm diff --git a/internal/cli/util/helm/errors.go b/internal/cli/util/helm/errors.go index a46e41af8..687872499 100644 --- a/internal/cli/util/helm/errors.go +++ b/internal/cli/util/helm/errors.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package helm diff --git a/internal/cli/util/helm/helm.go b/internal/cli/util/helm/helm.go index 62470eb7f..7706fd254 100644 --- a/internal/cli/util/helm/helm.go +++ b/internal/cli/util/helm/helm.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package helm diff --git a/internal/cli/util/helm/helm_test.go b/internal/cli/util/helm/helm_test.go index f6618c3a9..56addc920 100644 --- a/internal/cli/util/helm/helm_test.go +++ b/internal/cli/util/helm/helm_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package helm diff --git a/internal/cli/util/helm/suite_test.go b/internal/cli/util/helm/suite_test.go index 7570dba14..8c628ef8c 100644 --- a/internal/cli/util/helm/suite_test.go +++ b/internal/cli/util/helm/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package helm diff --git a/internal/cli/util/prompt/prompt.go b/internal/cli/util/prompt/prompt.go index 6b6b90f26..381f2bfb0 100644 --- a/internal/cli/util/prompt/prompt.go +++ b/internal/cli/util/prompt/prompt.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package prompt diff --git a/internal/cli/util/prompt/prompt_test.go b/internal/cli/util/prompt/prompt_test.go index 855580c98..d12b4b8c3 100644 --- a/internal/cli/util/prompt/prompt_test.go +++ b/internal/cli/util/prompt/prompt_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package prompt diff --git a/internal/cli/util/provider.go b/internal/cli/util/provider.go index b8c1b3138..1e330382c 100644 --- a/internal/cli/util/provider.go +++ b/internal/cli/util/provider.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/provider_test.go b/internal/cli/util/provider_test.go index 26b5b1552..0dd1adb18 100644 --- a/internal/cli/util/provider_test.go +++ b/internal/cli/util/provider_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/suite_test.go b/internal/cli/util/suite_test.go index 65af1d282..ab1fa6284 100644 --- a/internal/cli/util/suite_test.go +++ b/internal/cli/util/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 4a4d3f52f..ddf7d3712 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/util_test.go b/internal/cli/util/util_test.go index 4f12d8d0b..8588409e0 100644 --- a/internal/cli/util/util_test.go +++ b/internal/cli/util/util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/version.go b/internal/cli/util/version.go index e7f759e82..635885429 100644 --- a/internal/cli/util/version.go +++ b/internal/cli/util/version.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/cli/util/version_test.go b/internal/cli/util/version_test.go index 90439f2f3..94c2f130a 100644 --- a/internal/cli/util/version_test.go +++ b/internal/cli/util/version_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/config.go b/internal/configuration/config.go index f1ec520c8..073dfc896 100644 --- a/internal/configuration/config.go +++ b/internal/configuration/config.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_manager/builder.go b/internal/configuration/config_manager/builder.go index 31a872ed9..0abc784b6 100644 --- a/internal/configuration/config_manager/builder.go +++ b/internal/configuration/config_manager/builder.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/builder_test.go b/internal/configuration/config_manager/builder_test.go index 8e8ebf7cf..2804182e2 100644 --- a/internal/configuration/config_manager/builder_test.go +++ b/internal/configuration/config_manager/builder_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/dynamic_parameter_updater.go b/internal/configuration/config_manager/dynamic_parameter_updater.go index 0f1bf02fc..900ec5a37 100644 --- a/internal/configuration/config_manager/dynamic_parameter_updater.go +++ b/internal/configuration/config_manager/dynamic_parameter_updater.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/files.go b/internal/configuration/config_manager/files.go index 8343cc96f..2cd0166c1 100644 --- a/internal/configuration/config_manager/files.go +++ b/internal/configuration/config_manager/files.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/files_test.go b/internal/configuration/config_manager/files_test.go index 7b1c3089e..800b422a7 100644 --- a/internal/configuration/config_manager/files_test.go +++ b/internal/configuration/config_manager/files_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/handler.go b/internal/configuration/config_manager/handler.go index 4cdb19113..724b9d152 100644 --- a/internal/configuration/config_manager/handler.go +++ b/internal/configuration/config_manager/handler.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/handler_test.go b/internal/configuration/config_manager/handler_test.go index c74645e8e..d9c5707f5 100644 --- a/internal/configuration/config_manager/handler_test.go +++ b/internal/configuration/config_manager/handler_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/handler_util.go b/internal/configuration/config_manager/handler_util.go index 7dfd9dfa7..508a63d03 100644 --- a/internal/configuration/config_manager/handler_util.go +++ b/internal/configuration/config_manager/handler_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/handler_util_test.go b/internal/configuration/config_manager/handler_util_test.go index 35837cc47..572f19cba 100644 --- a/internal/configuration/config_manager/handler_util_test.go +++ b/internal/configuration/config_manager/handler_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/reload_util.go b/internal/configuration/config_manager/reload_util.go index d4957a16e..5e32a6fea 100644 --- a/internal/configuration/config_manager/reload_util.go +++ b/internal/configuration/config_manager/reload_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/reload_util_test.go b/internal/configuration/config_manager/reload_util_test.go index f02b2386f..b52ecd4fb 100644 --- a/internal/configuration/config_manager/reload_util_test.go +++ b/internal/configuration/config_manager/reload_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/signal.go b/internal/configuration/config_manager/signal.go index 1f09ef7ba..8c09ecb50 100644 --- a/internal/configuration/config_manager/signal.go +++ b/internal/configuration/config_manager/signal.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/signal_darwin.go b/internal/configuration/config_manager/signal_darwin.go index 13c859997..6775f098a 100644 --- a/internal/configuration/config_manager/signal_darwin.go +++ b/internal/configuration/config_manager/signal_darwin.go @@ -1,19 +1,22 @@ //go:build darwin /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/signal_linux.go b/internal/configuration/config_manager/signal_linux.go index 8fc2ae694..f8944342d 100644 --- a/internal/configuration/config_manager/signal_linux.go +++ b/internal/configuration/config_manager/signal_linux.go @@ -1,19 +1,22 @@ //go:build linux /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/signal_test.go b/internal/configuration/config_manager/signal_test.go index 9733cec92..0934a89e5 100644 --- a/internal/configuration/config_manager/signal_test.go +++ b/internal/configuration/config_manager/signal_test.go @@ -1,19 +1,22 @@ //go:build linux || darwin /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/signal_windows.go b/internal/configuration/config_manager/signal_windows.go index dfe9684c5..e82951eb8 100644 --- a/internal/configuration/config_manager/signal_windows.go +++ b/internal/configuration/config_manager/signal_windows.go @@ -1,19 +1,22 @@ //go:build windows /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/suite_test.go b/internal/configuration/config_manager/suite_test.go index 2997ffb9e..06d4cd58d 100644 --- a/internal/configuration/config_manager/suite_test.go +++ b/internal/configuration/config_manager/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/volume_watcher.go b/internal/configuration/config_manager/volume_watcher.go index 007f7e3bb..be9e99601 100644 --- a/internal/configuration/config_manager/volume_watcher.go +++ b/internal/configuration/config_manager/volume_watcher.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_manager/volume_watcher_test.go b/internal/configuration/config_manager/volume_watcher_test.go index 9cbac366d..951dceea1 100644 --- a/internal/configuration/config_manager/volume_watcher_test.go +++ b/internal/configuration/config_manager/volume_watcher_test.go @@ -1,19 +1,22 @@ //go:build linux || darwin /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configmanager diff --git a/internal/configuration/config_patch.go b/internal/configuration/config_patch.go index 1759a71e6..11356165a 100644 --- a/internal/configuration/config_patch.go +++ b/internal/configuration/config_patch.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_patch_option.go b/internal/configuration/config_patch_option.go index 63f356b55..153e7f1f7 100644 --- a/internal/configuration/config_patch_option.go +++ b/internal/configuration/config_patch_option.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_patch_option_test.go b/internal/configuration/config_patch_option_test.go index ebe8d6327..039409f42 100644 --- a/internal/configuration/config_patch_option_test.go +++ b/internal/configuration/config_patch_option_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_patch_test.go b/internal/configuration/config_patch_test.go index e2b647c52..bd16102cb 100644 --- a/internal/configuration/config_patch_test.go +++ b/internal/configuration/config_patch_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_patch_util.go b/internal/configuration/config_patch_util.go index 8ed50a234..e120a2da5 100644 --- a/internal/configuration/config_patch_util.go +++ b/internal/configuration/config_patch_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_patch_util_test.go b/internal/configuration/config_patch_util_test.go index 905aaeca7..1600a088e 100644 --- a/internal/configuration/config_patch_util_test.go +++ b/internal/configuration/config_patch_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_query.go b/internal/configuration/config_query.go index f88b15c8b..a1165478e 100644 --- a/internal/configuration/config_query.go +++ b/internal/configuration/config_query.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_query_test.go b/internal/configuration/config_query_test.go index cea2fdc8b..dc037cfb3 100644 --- a/internal/configuration/config_query_test.go +++ b/internal/configuration/config_query_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_test.go b/internal/configuration/config_test.go index 61b76c403..bbc45d1fc 100644 --- a/internal/configuration/config_test.go +++ b/internal/configuration/config_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_util.go b/internal/configuration/config_util.go index a46e17d36..92298bbf4 100644 --- a/internal/configuration/config_util.go +++ b/internal/configuration/config_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_util_test.go b/internal/configuration/config_util_test.go index 723ca45ba..70ea55a94 100644 --- a/internal/configuration/config_util_test.go +++ b/internal/configuration/config_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_validate.go b/internal/configuration/config_validate.go index d63be5222..0390fb2be 100644 --- a/internal/configuration/config_validate.go +++ b/internal/configuration/config_validate.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/config_validate_test.go b/internal/configuration/config_validate_test.go index 1b1bda052..7577f35b1 100644 --- a/internal/configuration/config_validate_test.go +++ b/internal/configuration/config_validate_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration @@ -105,11 +108,11 @@ func TestSchemaValidatorWithCue(t *testing.T) { }, err: errors.New(`failed to cue template render configure: [mysqld.innodb_autoinc_lock_mode: 3 errors in empty disjunction: mysqld.innodb_autoinc_lock_mode: conflicting values 0 and 100: - 28:35 + 31:35 mysqld.innodb_autoinc_lock_mode: conflicting values 1 and 100: - 28:39 + 31:39 mysqld.innodb_autoinc_lock_mode: conflicting values 2 and 100: - 28:43 + 31:43 ]`), }, { name: "configmap_key_filter", diff --git a/internal/configuration/configtemplate_util.go b/internal/configuration/configtemplate_util.go index 1bcc3b994..32b54a41a 100644 --- a/internal/configuration/configtemplate_util.go +++ b/internal/configuration/configtemplate_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/configtemplate_util_test.go b/internal/configuration/configtemplate_util_test.go index 2cd817d2f..7ec9423ff 100644 --- a/internal/configuration/configtemplate_util_test.go +++ b/internal/configuration/configtemplate_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/constraint.go b/internal/configuration/constraint.go index fa2dea63b..81655fdff 100644 --- a/internal/configuration/constraint.go +++ b/internal/configuration/constraint.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/container/container_kill.go b/internal/configuration/container/container_kill.go index cc7b176fc..e2f77654b 100644 --- a/internal/configuration/container/container_kill.go +++ b/internal/configuration/container/container_kill.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package container diff --git a/internal/configuration/container/container_kill_test.go b/internal/configuration/container/container_kill_test.go index 93660237a..ea446b2ee 100644 --- a/internal/configuration/container/container_kill_test.go +++ b/internal/configuration/container/container_kill_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package container diff --git a/internal/configuration/container/container_util.go b/internal/configuration/container/container_util.go index fd372a14b..445256471 100644 --- a/internal/configuration/container/container_util.go +++ b/internal/configuration/container/container_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package container diff --git a/internal/configuration/container/container_util_test.go b/internal/configuration/container/container_util_test.go index bef540157..ed433a080 100644 --- a/internal/configuration/container/container_util_test.go +++ b/internal/configuration/container/container_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package container diff --git a/internal/configuration/container/mocks/generate.go b/internal/configuration/container/mocks/generate.go index c3cfbcf1b..eebe99bcf 100644 --- a/internal/configuration/container/mocks/generate.go +++ b/internal/configuration/container/mocks/generate.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package mocks diff --git a/internal/configuration/container/type.go b/internal/configuration/container/type.go index 5843d2c1f..155fcd61f 100644 --- a/internal/configuration/container/type.go +++ b/internal/configuration/container/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package container diff --git a/internal/configuration/cue_gen_openapi.go b/internal/configuration/cue_gen_openapi.go index 9753b7ee5..adb7ceda1 100644 --- a/internal/configuration/cue_gen_openapi.go +++ b/internal/configuration/cue_gen_openapi.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/cue_gen_openapi_test.go b/internal/configuration/cue_gen_openapi_test.go index dc3d75689..5653e9fec 100644 --- a/internal/configuration/cue_gen_openapi_test.go +++ b/internal/configuration/cue_gen_openapi_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/cue_util.go b/internal/configuration/cue_util.go index 05e2a5b08..15531c7b1 100644 --- a/internal/configuration/cue_util.go +++ b/internal/configuration/cue_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/cue_util_test.go b/internal/configuration/cue_util_test.go index 0c6780eb4..2d3e9b1f5 100644 --- a/internal/configuration/cue_util_test.go +++ b/internal/configuration/cue_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/cue_visitor.go b/internal/configuration/cue_visitor.go index 28ec3f32a..2f4758a8b 100644 --- a/internal/configuration/cue_visitor.go +++ b/internal/configuration/cue_visitor.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/cue_visitor_test.go b/internal/configuration/cue_visitor_test.go index 5a918337c..64efcd012 100644 --- a/internal/configuration/cue_visitor_test.go +++ b/internal/configuration/cue_visitor_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/cuelang_expansion.go b/internal/configuration/cuelang_expansion.go index ec62b6b6e..6705bee83 100644 --- a/internal/configuration/cuelang_expansion.go +++ b/internal/configuration/cuelang_expansion.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/error.go b/internal/configuration/error.go index fc29e5495..abc6deb82 100644 --- a/internal/configuration/error.go +++ b/internal/configuration/error.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/proto/generate.go b/internal/configuration/proto/generate.go index 0440a1278..7b5436164 100644 --- a/internal/configuration/proto/generate.go +++ b/internal/configuration/proto/generate.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package proto diff --git a/internal/configuration/proto/mocks/generate.go b/internal/configuration/proto/mocks/generate.go index 293e42c8d..7d4b7442f 100644 --- a/internal/configuration/proto/mocks/generate.go +++ b/internal/configuration/proto/mocks/generate.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package mocks diff --git a/internal/configuration/reconfigure_util.go b/internal/configuration/reconfigure_util.go index 919057cc0..f67268f17 100644 --- a/internal/configuration/reconfigure_util.go +++ b/internal/configuration/reconfigure_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/reconfigure_util_test.go b/internal/configuration/reconfigure_util_test.go index 4935bc174..361effc89 100644 --- a/internal/configuration/reconfigure_util_test.go +++ b/internal/configuration/reconfigure_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/suite_test.go b/internal/configuration/suite_test.go index 57ecdbfd5..6be50e691 100644 --- a/internal/configuration/suite_test.go +++ b/internal/configuration/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/type.go b/internal/configuration/type.go index ec51c6762..a6c7ed966 100644 --- a/internal/configuration/type.go +++ b/internal/configuration/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package configuration diff --git a/internal/configuration/util/file_util.go b/internal/configuration/util/file_util.go index c50836f3a..62302546e 100644 --- a/internal/configuration/util/file_util.go +++ b/internal/configuration/util/file_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/hash.go b/internal/configuration/util/hash.go index 6ee4d5808..255fbf4bd 100644 --- a/internal/configuration/util/hash.go +++ b/internal/configuration/util/hash.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/hash_test.go b/internal/configuration/util/hash_test.go index 9c00aa6a1..947157f5a 100644 --- a/internal/configuration/util/hash_test.go +++ b/internal/configuration/util/hash_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/jsonpath.go b/internal/configuration/util/jsonpath.go index 63b14701d..acba9ff19 100644 --- a/internal/configuration/util/jsonpath.go +++ b/internal/configuration/util/jsonpath.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/math.go b/internal/configuration/util/math.go index 9af6b2c75..f9d80db51 100644 --- a/internal/configuration/util/math.go +++ b/internal/configuration/util/math.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/math_test.go b/internal/configuration/util/math_test.go index 0cefd6d4f..013388efd 100644 --- a/internal/configuration/util/math_test.go +++ b/internal/configuration/util/math_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/set.go b/internal/configuration/util/set.go index 0238ce2ac..642b42d5f 100644 --- a/internal/configuration/util/set.go +++ b/internal/configuration/util/set.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/set_test.go b/internal/configuration/util/set_test.go index 903812dca..1ef17456f 100644 --- a/internal/configuration/util/set_test.go +++ b/internal/configuration/util/set_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/unstructured.go b/internal/configuration/util/unstructured.go index 704324522..53f4bea76 100644 --- a/internal/configuration/util/unstructured.go +++ b/internal/configuration/util/unstructured.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/configuration/util/unstructured_test.go b/internal/configuration/util/unstructured_test.go index eb724d194..2f77f6994 100644 --- a/internal/configuration/util/unstructured_test.go +++ b/internal/configuration/util/unstructured_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/constant/const.go b/internal/constant/const.go index c81d3de56..16ba402da 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package constant diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index 0497d1e32..a85a1f23f 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package builder diff --git a/internal/controller/builder/builder_test.go b/internal/controller/builder/builder_test.go index fd75a85d3..87d6b2265 100644 --- a/internal/controller/builder/builder_test.go +++ b/internal/controller/builder/builder_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package builder diff --git a/internal/controller/builder/cue/backup_job_template.cue b/internal/controller/builder/cue/backup_job_template.cue index 9caaafb45..c91cf30ea 100644 --- a/internal/controller/builder/cue/backup_job_template.cue +++ b/internal/controller/builder/cue/backup_job_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . sts: { metadata: { diff --git a/internal/controller/builder/cue/backup_manifests_template.cue b/internal/controller/builder/cue/backup_manifests_template.cue index 852bed36f..88d7a7e65 100644 --- a/internal/controller/builder/cue/backup_manifests_template.cue +++ b/internal/controller/builder/cue/backup_manifests_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . backup: { metadata: { diff --git a/internal/controller/builder/cue/config_manager_sidecar.cue b/internal/controller/builder/cue/config_manager_sidecar.cue index 3c05392aa..c5f851151 100644 --- a/internal/controller/builder/cue/config_manager_sidecar.cue +++ b/internal/controller/builder/cue/config_manager_sidecar.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . template: { name: parameter.name diff --git a/internal/controller/builder/cue/config_template.cue b/internal/controller/builder/cue/config_template.cue index 22c88c9df..c3763c0e0 100644 --- a/internal/controller/builder/cue/config_template.cue +++ b/internal/controller/builder/cue/config_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . meta: { clusterDefinition: { diff --git a/internal/controller/builder/cue/conn_credential_template.cue b/internal/controller/builder/cue/conn_credential_template.cue index a6a0643c3..8f6a2f2b4 100644 --- a/internal/controller/builder/cue/conn_credential_template.cue +++ b/internal/controller/builder/cue/conn_credential_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . clusterdefinition: { metadata: { diff --git a/internal/controller/builder/cue/delete_pvc_cron_job_template.cue b/internal/controller/builder/cue/delete_pvc_cron_job_template.cue index 10311fbe8..762e3f368 100644 --- a/internal/controller/builder/cue/delete_pvc_cron_job_template.cue +++ b/internal/controller/builder/cue/delete_pvc_cron_job_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . pvc: { Name: string diff --git a/internal/controller/builder/cue/deployment_template.cue b/internal/controller/builder/cue/deployment_template.cue index 4804a1fbc..b2bb58cad 100644 --- a/internal/controller/builder/cue/deployment_template.cue +++ b/internal/controller/builder/cue/deployment_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/env_config_template.cue b/internal/controller/builder/cue/env_config_template.cue index 4219f84a6..dd2ef2cdc 100644 --- a/internal/controller/builder/cue/env_config_template.cue +++ b/internal/controller/builder/cue/env_config_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/headless_service_template.cue b/internal/controller/builder/cue/headless_service_template.cue index 29ff0078e..d81fcd8c7 100644 --- a/internal/controller/builder/cue/headless_service_template.cue +++ b/internal/controller/builder/cue/headless_service_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/pdb_template.cue b/internal/controller/builder/cue/pdb_template.cue index e658838f1..4bb702dbf 100644 --- a/internal/controller/builder/cue/pdb_template.cue +++ b/internal/controller/builder/cue/pdb_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/pitr_job_template.cue b/internal/controller/builder/cue/pitr_job_template.cue index 755b597d4..1ed22b8fd 100644 --- a/internal/controller/builder/cue/pitr_job_template.cue +++ b/internal/controller/builder/cue/pitr_job_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . container: { name: "pitr" diff --git a/internal/controller/builder/cue/pvc_template.cue b/internal/controller/builder/cue/pvc_template.cue index 848dde657..4b34a43e8 100644 --- a/internal/controller/builder/cue/pvc_template.cue +++ b/internal/controller/builder/cue/pvc_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . sts: { metadata: { diff --git a/internal/controller/builder/cue/service_template.cue b/internal/controller/builder/cue/service_template.cue index 530f3a9f1..27193553e 100644 --- a/internal/controller/builder/cue/service_template.cue +++ b/internal/controller/builder/cue/service_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/snapshot_template.cue b/internal/controller/builder/cue/snapshot_template.cue index 729e73d36..264b5c437 100644 --- a/internal/controller/builder/cue/snapshot_template.cue +++ b/internal/controller/builder/cue/snapshot_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . snapshot_key: { Name: string diff --git a/internal/controller/builder/cue/statefulset_template.cue b/internal/controller/builder/cue/statefulset_template.cue index d3a7ce4e4..57ca9925a 100644 --- a/internal/controller/builder/cue/statefulset_template.cue +++ b/internal/controller/builder/cue/statefulset_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/tls_certs_secret_template.cue b/internal/controller/builder/cue/tls_certs_secret_template.cue index 5faf8f4fc..521fe978f 100644 --- a/internal/controller/builder/cue/tls_certs_secret_template.cue +++ b/internal/controller/builder/cue/tls_certs_secret_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . pathedName: { namespace: string diff --git a/internal/controller/builder/suite_test.go b/internal/controller/builder/suite_test.go index e212b36d0..1cfaa18e4 100644 --- a/internal/controller/builder/suite_test.go +++ b/internal/controller/builder/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package builder diff --git a/internal/controller/client/readonly_client.go b/internal/controller/client/readonly_client.go index cba7c2b94..9a282f404 100644 --- a/internal/controller/client/readonly_client.go +++ b/internal/controller/client/readonly_client.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package client diff --git a/internal/controller/component/affinity_utils.go b/internal/controller/component/affinity_utils.go index 8b232f4ee..8dd97a4c3 100644 --- a/internal/controller/component/affinity_utils.go +++ b/internal/controller/component/affinity_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/affinity_utils_test.go b/internal/controller/component/affinity_utils_test.go index c62187650..7bb76d388 100644 --- a/internal/controller/component/affinity_utils_test.go +++ b/internal/controller/component/affinity_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/component.go b/internal/controller/component/component.go index 90c29eafa..421db0e62 100644 --- a/internal/controller/component/component.go +++ b/internal/controller/component/component.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/component_test.go b/internal/controller/component/component_test.go index dfd7f63d6..8e88a989a 100644 --- a/internal/controller/component/component_test.go +++ b/internal/controller/component/component_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/cue/probe_template.cue b/internal/controller/component/cue/probe_template.cue index 01732a8b8..0eae78919 100644 --- a/internal/controller/component/cue/probe_template.cue +++ b/internal/controller/component/cue/probe_template.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . probeContainer: { image: "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6" diff --git a/internal/controller/component/monitor_utils.go b/internal/controller/component/monitor_utils.go index f3bc1dd56..2f65b177a 100644 --- a/internal/controller/component/monitor_utils.go +++ b/internal/controller/component/monitor_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/monitor_utils_test.go b/internal/controller/component/monitor_utils_test.go index 6d70f628e..68f9cab01 100644 --- a/internal/controller/component/monitor_utils_test.go +++ b/internal/controller/component/monitor_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/port_utils.go b/internal/controller/component/port_utils.go index 00191d3ad..6708d0840 100644 --- a/internal/controller/component/port_utils.go +++ b/internal/controller/component/port_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/port_utils_test.go b/internal/controller/component/port_utils_test.go index 8a4853f7d..183c0acdd 100644 --- a/internal/controller/component/port_utils_test.go +++ b/internal/controller/component/port_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/probe_utils.go b/internal/controller/component/probe_utils.go index 559ee3b3d..f7e66cf3e 100644 --- a/internal/controller/component/probe_utils.go +++ b/internal/controller/component/probe_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/probe_utils_test.go b/internal/controller/component/probe_utils_test.go index 24388daab..b2b5ed0ec 100644 --- a/internal/controller/component/probe_utils_test.go +++ b/internal/controller/component/probe_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/restore_utils.go b/internal/controller/component/restore_utils.go index 4c2b842d6..71fc9cb7c 100644 --- a/internal/controller/component/restore_utils.go +++ b/internal/controller/component/restore_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/restore_utils_test.go b/internal/controller/component/restore_utils_test.go index d83702e1e..17cb5307f 100644 --- a/internal/controller/component/restore_utils_test.go +++ b/internal/controller/component/restore_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/suite_test.go b/internal/controller/component/suite_test.go index 429badf60..67cf9f7e3 100644 --- a/internal/controller/component/suite_test.go +++ b/internal/controller/component/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/component/type.go b/internal/controller/component/type.go index 5723e08d7..698972b05 100644 --- a/internal/controller/component/type.go +++ b/internal/controller/component/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package component diff --git a/internal/controller/graph/dag.go b/internal/controller/graph/dag.go index feaeeb2b4..27c178f6e 100644 --- a/internal/controller/graph/dag.go +++ b/internal/controller/graph/dag.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package graph diff --git a/internal/controller/graph/dag_test.go b/internal/controller/graph/dag_test.go index 471e39f58..e02f403df 100644 --- a/internal/controller/graph/dag_test.go +++ b/internal/controller/graph/dag_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package graph diff --git a/internal/controller/graph/doc.go b/internal/controller/graph/doc.go index 7657d6f50..57cd51e63 100644 --- a/internal/controller/graph/doc.go +++ b/internal/controller/graph/doc.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ /* diff --git a/internal/controller/graph/plan_builder.go b/internal/controller/graph/plan_builder.go index d9f29d3ec..9e6453a3d 100644 --- a/internal/controller/graph/plan_builder.go +++ b/internal/controller/graph/plan_builder.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package graph diff --git a/internal/controller/graph/transformer.go b/internal/controller/graph/transformer.go index ad39880b3..eeab6b511 100644 --- a/internal/controller/graph/transformer.go +++ b/internal/controller/graph/transformer.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package graph diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 0065f1292..2921975f1 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/cluster_plan_utils.go b/internal/controller/lifecycle/cluster_plan_utils.go index 283bbde41..b24e14c71 100644 --- a/internal/controller/lifecycle/cluster_plan_utils.go +++ b/internal/controller/lifecycle/cluster_plan_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/cluster_plan_utils_test.go b/internal/controller/lifecycle/cluster_plan_utils_test.go index fcd505921..01f16c688 100644 --- a/internal/controller/lifecycle/cluster_plan_utils_test.go +++ b/internal/controller/lifecycle/cluster_plan_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/cluster_status_conditions.go b/internal/controller/lifecycle/cluster_status_conditions.go index 31d614e5b..79060ad2a 100644 --- a/internal/controller/lifecycle/cluster_status_conditions.go +++ b/internal/controller/lifecycle/cluster_status_conditions.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/suite_test.go b/internal/controller/lifecycle/suite_test.go index 97dd91b8a..162b41439 100644 --- a/internal/controller/lifecycle/suite_test.go +++ b/internal/controller/lifecycle/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transform_types.go b/internal/controller/lifecycle/transform_types.go index 2fc14e600..14e6c02ee 100644 --- a/internal/controller/lifecycle/transform_types.go +++ b/internal/controller/lifecycle/transform_types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transform_utils.go b/internal/controller/lifecycle/transform_utils.go index 839ff2183..185b3a247 100644 --- a/internal/controller/lifecycle/transform_utils.go +++ b/internal/controller/lifecycle/transform_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transform_utils_test.go b/internal/controller/lifecycle/transform_utils_test.go index 52c78dd78..cb89fc441 100644 --- a/internal/controller/lifecycle/transform_utils_test.go +++ b/internal/controller/lifecycle/transform_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index 29693d002..e73b07fe8 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_cluster.go b/internal/controller/lifecycle/transformer_cluster.go index 0b4f8500f..7149977cb 100644 --- a/internal/controller/lifecycle/transformer_cluster.go +++ b/internal/controller/lifecycle/transformer_cluster.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_cluster_status.go b/internal/controller/lifecycle/transformer_cluster_status.go index 6ea61685f..3f3ce41a2 100644 --- a/internal/controller/lifecycle/transformer_cluster_status.go +++ b/internal/controller/lifecycle/transformer_cluster_status.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_config.go b/internal/controller/lifecycle/transformer_config.go index 9938051b6..0a2b01887 100644 --- a/internal/controller/lifecycle/transformer_config.go +++ b/internal/controller/lifecycle/transformer_config.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_credential.go b/internal/controller/lifecycle/transformer_credential.go index 794391625..cc1968b0f 100644 --- a/internal/controller/lifecycle/transformer_credential.go +++ b/internal/controller/lifecycle/transformer_credential.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_fill_class.go b/internal/controller/lifecycle/transformer_fill_class.go index c85c2aaf5..9105ecc48 100644 --- a/internal/controller/lifecycle/transformer_fill_class.go +++ b/internal/controller/lifecycle/transformer_fill_class.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_fix_meta.go b/internal/controller/lifecycle/transformer_fix_meta.go index 82aa9b29c..a89d7311d 100644 --- a/internal/controller/lifecycle/transformer_fix_meta.go +++ b/internal/controller/lifecycle/transformer_fix_meta.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_init.go b/internal/controller/lifecycle/transformer_init.go index a194ef3a6..f08ca86f0 100644 --- a/internal/controller/lifecycle/transformer_init.go +++ b/internal/controller/lifecycle/transformer_init.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_object_action.go b/internal/controller/lifecycle/transformer_object_action.go index d7d725f66..a468d8bf2 100644 --- a/internal/controller/lifecycle/transformer_object_action.go +++ b/internal/controller/lifecycle/transformer_object_action.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_ownership.go b/internal/controller/lifecycle/transformer_ownership.go index ae59fdb81..7e7706eb0 100644 --- a/internal/controller/lifecycle/transformer_ownership.go +++ b/internal/controller/lifecycle/transformer_ownership.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 282babed1..46aca15d9 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go index 09e8fbb11..1f07a2c2a 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_sts_pvc.go b/internal/controller/lifecycle/transformer_sts_pvc.go index 939017369..1df2bd3fe 100644 --- a/internal/controller/lifecycle/transformer_sts_pvc.go +++ b/internal/controller/lifecycle/transformer_sts_pvc.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_tls_certs.go b/internal/controller/lifecycle/transformer_tls_certs.go index 41fc71f32..bb5bdbdd3 100644 --- a/internal/controller/lifecycle/transformer_tls_certs.go +++ b/internal/controller/lifecycle/transformer_tls_certs.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformers_parallel.go b/internal/controller/lifecycle/transformers_parallel.go index 8b2487eac..ccda1723f 100644 --- a/internal/controller/lifecycle/transformers_parallel.go +++ b/internal/controller/lifecycle/transformers_parallel.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/plan/builtin_env.go b/internal/controller/plan/builtin_env.go index 18e8593cf..235d96b13 100644 --- a/internal/controller/plan/builtin_env.go +++ b/internal/controller/plan/builtin_env.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/builtin_env_test.go b/internal/controller/plan/builtin_env_test.go index 386008879..f90dbddd2 100644 --- a/internal/controller/plan/builtin_env_test.go +++ b/internal/controller/plan/builtin_env_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/builtin_functions.go b/internal/controller/plan/builtin_functions.go index bcfc36ce5..cfe2d8dba 100644 --- a/internal/controller/plan/builtin_functions.go +++ b/internal/controller/plan/builtin_functions.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/config_template.go b/internal/controller/plan/config_template.go index 31fce219c..65e055cee 100644 --- a/internal/controller/plan/config_template.go +++ b/internal/controller/plan/config_template.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/config_template_test.go b/internal/controller/plan/config_template_test.go index 67928a2b9..665eda861 100644 --- a/internal/controller/plan/config_template_test.go +++ b/internal/controller/plan/config_template_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/pitr.go b/internal/controller/plan/pitr.go index deaeddfd9..121eed9e1 100644 --- a/internal/controller/plan/pitr.go +++ b/internal/controller/plan/pitr.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/pitr_test.go b/internal/controller/plan/pitr_test.go index c8ff773a0..1aa46ff6a 100644 --- a/internal/controller/plan/pitr_test.go +++ b/internal/controller/plan/pitr_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/prepare.go b/internal/controller/plan/prepare.go index a21bf20e4..6980dc6e4 100644 --- a/internal/controller/plan/prepare.go +++ b/internal/controller/plan/prepare.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/prepare_test.go b/internal/controller/plan/prepare_test.go index 95a155a95..dde991fd0 100644 --- a/internal/controller/plan/prepare_test.go +++ b/internal/controller/plan/prepare_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/suite_test.go b/internal/controller/plan/suite_test.go index ce9cb549b..a13ce2d67 100644 --- a/internal/controller/plan/suite_test.go +++ b/internal/controller/plan/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/template_wrapper.go b/internal/controller/plan/template_wrapper.go index c531b86c0..4544d030f 100644 --- a/internal/controller/plan/template_wrapper.go +++ b/internal/controller/plan/template_wrapper.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/plan/tls_utils.go b/internal/controller/plan/tls_utils.go index 279e5dd93..bcd985bf8 100644 --- a/internal/controller/plan/tls_utils.go +++ b/internal/controller/plan/tls_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plan diff --git a/internal/controller/types/task.go b/internal/controller/types/task.go index dfa80c648..2351d5830 100644 --- a/internal/controller/types/task.go +++ b/internal/controller/types/task.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package types diff --git a/internal/controllerutil/container_util.go b/internal/controllerutil/container_util.go index 2571f6a12..0db587d2d 100644 --- a/internal/controllerutil/container_util.go +++ b/internal/controllerutil/container_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/controller_common.go b/internal/controllerutil/controller_common.go index 5bbca501c..a1df716a8 100644 --- a/internal/controllerutil/controller_common.go +++ b/internal/controllerutil/controller_common.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/controller_common_test.go b/internal/controllerutil/controller_common_test.go index 55e66bf8d..195e2e1e1 100644 --- a/internal/controllerutil/controller_common_test.go +++ b/internal/controllerutil/controller_common_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/cue_value.go b/internal/controllerutil/cue_value.go index f85ebce40..891bbcfda 100644 --- a/internal/controllerutil/cue_value.go +++ b/internal/controllerutil/cue_value.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/cue_value_test.go b/internal/controllerutil/cue_value_test.go index 86f3311a9..c2cb2e026 100644 --- a/internal/controllerutil/cue_value_test.go +++ b/internal/controllerutil/cue_value_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index 091a16714..bea523b14 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/errors_test.go b/internal/controllerutil/errors_test.go index 34fd04558..1a3dd0da3 100644 --- a/internal/controllerutil/errors_test.go +++ b/internal/controllerutil/errors_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/pod_utils.go b/internal/controllerutil/pod_utils.go index 00b32dbe6..e4be2d342 100644 --- a/internal/controllerutil/pod_utils.go +++ b/internal/controllerutil/pod_utils.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/pod_utils_test.go b/internal/controllerutil/pod_utils_test.go index 568b9fbc1..720203def 100644 --- a/internal/controllerutil/pod_utils_test.go +++ b/internal/controllerutil/pod_utils_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/suite_test.go b/internal/controllerutil/suite_test.go index e6cd2410a..f415dd8a9 100644 --- a/internal/controllerutil/suite_test.go +++ b/internal/controllerutil/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/task.go b/internal/controllerutil/task.go index fdcc5373e..95ed50ee2 100644 --- a/internal/controllerutil/task.go +++ b/internal/controllerutil/task.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/type.go b/internal/controllerutil/type.go index 02a999774..414cf1e18 100644 --- a/internal/controllerutil/type.go +++ b/internal/controllerutil/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/types_util.go b/internal/controllerutil/types_util.go index 50956dbb3..bd6d56d5c 100644 --- a/internal/controllerutil/types_util.go +++ b/internal/controllerutil/types_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/volume_util.go b/internal/controllerutil/volume_util.go index 10605b43d..e32d16ed0 100644 --- a/internal/controllerutil/volume_util.go +++ b/internal/controllerutil/volume_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/controllerutil/volume_util_test.go b/internal/controllerutil/volume_util_test.go index 3c8605683..c653be22e 100644 --- a/internal/controllerutil/volume_util_test.go +++ b/internal/controllerutil/volume_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package controllerutil diff --git a/internal/generics/type.go b/internal/generics/type.go index a69c4446b..5f0fa3400 100644 --- a/internal/generics/type.go +++ b/internal/generics/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package generics diff --git a/internal/gotemplate/functional.go b/internal/gotemplate/functional.go index d0bc082f6..a3f1fa436 100644 --- a/internal/gotemplate/functional.go +++ b/internal/gotemplate/functional.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package gotemplate diff --git a/internal/gotemplate/suite_test.go b/internal/gotemplate/suite_test.go index 3542ce4bb..d5c7f3d8e 100644 --- a/internal/gotemplate/suite_test.go +++ b/internal/gotemplate/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package gotemplate diff --git a/internal/gotemplate/tpl_engine.go b/internal/gotemplate/tpl_engine.go index 1e81d4e2f..951cab1eb 100644 --- a/internal/gotemplate/tpl_engine.go +++ b/internal/gotemplate/tpl_engine.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package gotemplate diff --git a/internal/gotemplate/tpl_engine_test.go b/internal/gotemplate/tpl_engine_test.go index a924ca300..5786ca924 100644 --- a/internal/gotemplate/tpl_engine_test.go +++ b/internal/gotemplate/tpl_engine_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package gotemplate diff --git a/internal/preflight/analyze.go b/internal/preflight/analyze.go index f75ed39fc..b0a461ca1 100644 --- a/internal/preflight/analyze.go +++ b/internal/preflight/analyze.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/analyze_test.go b/internal/preflight/analyze_test.go index 4dda0200f..35def024e 100644 --- a/internal/preflight/analyze_test.go +++ b/internal/preflight/analyze_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/analyzer/access.go b/internal/preflight/analyzer/access.go index c86704139..342d8be18 100644 --- a/internal/preflight/analyzer/access.go +++ b/internal/preflight/analyzer/access.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/access_test.go b/internal/preflight/analyzer/access_test.go index 4bec2d55b..ddbbbc8a2 100644 --- a/internal/preflight/analyzer/access_test.go +++ b/internal/preflight/analyzer/access_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/analyze_result.go b/internal/preflight/analyzer/analyze_result.go index 64f2bb8ab..e8cde05dd 100644 --- a/internal/preflight/analyzer/analyze_result.go +++ b/internal/preflight/analyzer/analyze_result.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/analyze_result_test.go b/internal/preflight/analyzer/analyze_result_test.go index 5d83f54c0..4987cd56f 100644 --- a/internal/preflight/analyzer/analyze_result_test.go +++ b/internal/preflight/analyzer/analyze_result_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/analyzer.go b/internal/preflight/analyzer/analyzer.go index 48514e361..7d8a47c9c 100644 --- a/internal/preflight/analyzer/analyzer.go +++ b/internal/preflight/analyzer/analyzer.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/anzlyzer_test.go b/internal/preflight/analyzer/anzlyzer_test.go index 94a525037..51f7ab3a0 100644 --- a/internal/preflight/analyzer/anzlyzer_test.go +++ b/internal/preflight/analyzer/anzlyzer_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/host_analyzer.go b/internal/preflight/analyzer/host_analyzer.go index ac23bfe04..a5fadf2c5 100644 --- a/internal/preflight/analyzer/host_analyzer.go +++ b/internal/preflight/analyzer/host_analyzer.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/host_analyzer_test.go b/internal/preflight/analyzer/host_analyzer_test.go index f8efa9ace..a1504ef15 100644 --- a/internal/preflight/analyzer/host_analyzer_test.go +++ b/internal/preflight/analyzer/host_analyzer_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/host_region.go b/internal/preflight/analyzer/host_region.go index ef2c2a484..b2900e672 100644 --- a/internal/preflight/analyzer/host_region.go +++ b/internal/preflight/analyzer/host_region.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/host_region_test.go b/internal/preflight/analyzer/host_region_test.go index d9df42ff6..9feb9d97f 100644 --- a/internal/preflight/analyzer/host_region_test.go +++ b/internal/preflight/analyzer/host_region_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/host_utility.go b/internal/preflight/analyzer/host_utility.go index 9994249b8..a2894cb8c 100644 --- a/internal/preflight/analyzer/host_utility.go +++ b/internal/preflight/analyzer/host_utility.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/host_utility_test.go b/internal/preflight/analyzer/host_utility_test.go index 0c9f8bc26..d1f198ecb 100644 --- a/internal/preflight/analyzer/host_utility_test.go +++ b/internal/preflight/analyzer/host_utility_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/kb_storage_class.go b/internal/preflight/analyzer/kb_storage_class.go index d85de289d..af602d537 100644 --- a/internal/preflight/analyzer/kb_storage_class.go +++ b/internal/preflight/analyzer/kb_storage_class.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/kb_storage_class_test.go b/internal/preflight/analyzer/kb_storage_class_test.go index b7106f1c2..57fa18011 100644 --- a/internal/preflight/analyzer/kb_storage_class_test.go +++ b/internal/preflight/analyzer/kb_storage_class_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/analyzer/suite_test.go b/internal/preflight/analyzer/suite_test.go index 28d34aa38..f509de773 100644 --- a/internal/preflight/analyzer/suite_test.go +++ b/internal/preflight/analyzer/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package analyzer diff --git a/internal/preflight/collect.go b/internal/preflight/collect.go index e492c378f..51d1d9e79 100644 --- a/internal/preflight/collect.go +++ b/internal/preflight/collect.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/collect_test.go b/internal/preflight/collect_test.go index 1d9640366..73c7c8911 100644 --- a/internal/preflight/collect_test.go +++ b/internal/preflight/collect_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/collector/host_collector.go b/internal/preflight/collector/host_collector.go index bb317f55c..912a4e219 100644 --- a/internal/preflight/collector/host_collector.go +++ b/internal/preflight/collector/host_collector.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package collector diff --git a/internal/preflight/collector/host_collector_test.go b/internal/preflight/collector/host_collector_test.go index 6e9303bb0..bab32bbeb 100644 --- a/internal/preflight/collector/host_collector_test.go +++ b/internal/preflight/collector/host_collector_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package collector diff --git a/internal/preflight/collector/host_region.go b/internal/preflight/collector/host_region.go index fed0ddb62..37e96bce0 100644 --- a/internal/preflight/collector/host_region.go +++ b/internal/preflight/collector/host_region.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package collector diff --git a/internal/preflight/collector/host_region_test.go b/internal/preflight/collector/host_region_test.go index ffafc2a18..4644c859f 100644 --- a/internal/preflight/collector/host_region_test.go +++ b/internal/preflight/collector/host_region_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package collector diff --git a/internal/preflight/collector/host_utility.go b/internal/preflight/collector/host_utility.go index bd99f029d..006ec8388 100644 --- a/internal/preflight/collector/host_utility.go +++ b/internal/preflight/collector/host_utility.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package collector diff --git a/internal/preflight/collector/host_utility_test.go b/internal/preflight/collector/host_utility_test.go index bdbd44575..187848d92 100644 --- a/internal/preflight/collector/host_utility_test.go +++ b/internal/preflight/collector/host_utility_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package collector diff --git a/internal/preflight/collector/suite_test.go b/internal/preflight/collector/suite_test.go index 0722717ca..04411e27c 100644 --- a/internal/preflight/collector/suite_test.go +++ b/internal/preflight/collector/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package collector diff --git a/internal/preflight/concat_spec.go b/internal/preflight/concat_spec.go index 06696079f..a7198a5f7 100644 --- a/internal/preflight/concat_spec.go +++ b/internal/preflight/concat_spec.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/concat_spec_test.go b/internal/preflight/concat_spec_test.go index 9e865f49d..8ef81b67b 100644 --- a/internal/preflight/concat_spec_test.go +++ b/internal/preflight/concat_spec_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/interactive/interactive.go b/internal/preflight/interactive/interactive.go index 31a718e0a..308e5e629 100644 --- a/internal/preflight/interactive/interactive.go +++ b/internal/preflight/interactive/interactive.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package interactive diff --git a/internal/preflight/load_spec.go b/internal/preflight/load_spec.go index 82b4637cb..9385022fd 100644 --- a/internal/preflight/load_spec.go +++ b/internal/preflight/load_spec.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/load_spec_test.go b/internal/preflight/load_spec_test.go index a8c8ebb9f..38dee1072 100644 --- a/internal/preflight/load_spec_test.go +++ b/internal/preflight/load_spec_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/suite_test.go b/internal/preflight/suite_test.go index 452b1c889..1db0fa069 100644 --- a/internal/preflight/suite_test.go +++ b/internal/preflight/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/testing/fake.go b/internal/preflight/testing/fake.go index f74af46cc..c64793822 100644 --- a/internal/preflight/testing/fake.go +++ b/internal/preflight/testing/fake.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/preflight/testing/fake_test.go b/internal/preflight/testing/fake_test.go index 488b89542..7c1f19802 100644 --- a/internal/preflight/testing/fake_test.go +++ b/internal/preflight/testing/fake_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/preflight/testing/suite_test.go b/internal/preflight/testing/suite_test.go index ba4c7f25e..760ab9b22 100644 --- a/internal/preflight/testing/suite_test.go +++ b/internal/preflight/testing/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testing diff --git a/internal/preflight/text_results.go b/internal/preflight/text_results.go index b4a4e95f2..780e43331 100644 --- a/internal/preflight/text_results.go +++ b/internal/preflight/text_results.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/text_results_test.go b/internal/preflight/text_results_test.go index c91f9118e..f91cf1e82 100644 --- a/internal/preflight/text_results_test.go +++ b/internal/preflight/text_results_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package preflight diff --git a/internal/preflight/util/schema.go b/internal/preflight/util/schema.go index dc8f93d76..4e8a97aec 100644 --- a/internal/preflight/util/schema.go +++ b/internal/preflight/util/schema.go @@ -1,14 +1,20 @@ /* -Copyright ApeCloud, Inc. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/preflight/util/suite_test.go b/internal/preflight/util/suite_test.go index 65af1d282..ab1fa6284 100644 --- a/internal/preflight/util/suite_test.go +++ b/internal/preflight/util/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/preflight/util/util.go b/internal/preflight/util/util.go index 74d2c3996..e25e89de7 100644 --- a/internal/preflight/util/util.go +++ b/internal/preflight/util/util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/preflight/util/util_test.go b/internal/preflight/util/util_test.go index b8a043489..d82e47e6e 100644 --- a/internal/preflight/util/util_test.go +++ b/internal/preflight/util/util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/internal/sqlchannel/client.go b/internal/sqlchannel/client.go index b6514b775..02682e293 100644 --- a/internal/sqlchannel/client.go +++ b/internal/sqlchannel/client.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package sqlchannel diff --git a/internal/sqlchannel/client_test.go b/internal/sqlchannel/client_test.go index 4592f8ce1..eef236016 100644 --- a/internal/sqlchannel/client_test.go +++ b/internal/sqlchannel/client_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package sqlchannel diff --git a/internal/sqlchannel/engine/client.go b/internal/sqlchannel/engine/client.go index 56381a639..a0f2936c4 100644 --- a/internal/sqlchannel/engine/client.go +++ b/internal/sqlchannel/engine/client.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine diff --git a/internal/sqlchannel/engine/engine.go b/internal/sqlchannel/engine/engine.go index 09078783d..38685cec7 100644 --- a/internal/sqlchannel/engine/engine.go +++ b/internal/sqlchannel/engine/engine.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine diff --git a/internal/sqlchannel/engine/engine_test.go b/internal/sqlchannel/engine/engine_test.go index bbbd0510c..56194bd81 100644 --- a/internal/sqlchannel/engine/engine_test.go +++ b/internal/sqlchannel/engine/engine_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine diff --git a/internal/sqlchannel/engine/mongodb.go b/internal/sqlchannel/engine/mongodb.go index a17ca94dc..a140c71be 100644 --- a/internal/sqlchannel/engine/mongodb.go +++ b/internal/sqlchannel/engine/mongodb.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine diff --git a/internal/sqlchannel/engine/mysql.go b/internal/sqlchannel/engine/mysql.go index 494a286b5..8a30f0bbe 100644 --- a/internal/sqlchannel/engine/mysql.go +++ b/internal/sqlchannel/engine/mysql.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine diff --git a/internal/sqlchannel/engine/mysql_test.go b/internal/sqlchannel/engine/mysql_test.go index 8f246fad7..6e2da5e63 100644 --- a/internal/sqlchannel/engine/mysql_test.go +++ b/internal/sqlchannel/engine/mysql_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine diff --git a/internal/sqlchannel/engine/postgresql.go b/internal/sqlchannel/engine/postgresql.go index 5602121eb..c1f7f045f 100644 --- a/internal/sqlchannel/engine/postgresql.go +++ b/internal/sqlchannel/engine/postgresql.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine diff --git a/internal/sqlchannel/engine/redis.go b/internal/sqlchannel/engine/redis.go index aee485801..be5a6b524 100644 --- a/internal/sqlchannel/engine/redis.go +++ b/internal/sqlchannel/engine/redis.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine diff --git a/internal/sqlchannel/engine/suite_test.go b/internal/sqlchannel/engine/suite_test.go index fa075bc86..647f05097 100644 --- a/internal/sqlchannel/engine/suite_test.go +++ b/internal/sqlchannel/engine/suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package engine_test diff --git a/internal/sqlchannel/types.go b/internal/sqlchannel/types.go index f6dfd09cd..2b23b5bbd 100644 --- a/internal/sqlchannel/types.go +++ b/internal/sqlchannel/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package sqlchannel diff --git a/internal/testutil/apps/backup_factory.go b/internal/testutil/apps/backup_factory.go index d88a5d3f1..5695d1d00 100644 --- a/internal/testutil/apps/backup_factory.go +++ b/internal/testutil/apps/backup_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/backuppolicy_factory.go b/internal/testutil/apps/backuppolicy_factory.go index 3b455b135..1ffc081f0 100644 --- a/internal/testutil/apps/backuppolicy_factory.go +++ b/internal/testutil/apps/backuppolicy_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/backuppolicytemplate_factory.go b/internal/testutil/apps/backuppolicytemplate_factory.go index 46868011c..d6acdaf1d 100644 --- a/internal/testutil/apps/backuppolicytemplate_factory.go +++ b/internal/testutil/apps/backuppolicytemplate_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/base_factory.go b/internal/testutil/apps/base_factory.go index 722c48291..e97c64684 100644 --- a/internal/testutil/apps/base_factory.go +++ b/internal/testutil/apps/base_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/cluster_consensus_test_util.go b/internal/testutil/apps/cluster_consensus_test_util.go index e5d207ff1..051874556 100644 --- a/internal/testutil/apps/cluster_consensus_test_util.go +++ b/internal/testutil/apps/cluster_consensus_test_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/cluster_factory.go b/internal/testutil/apps/cluster_factory.go index a241097ac..3a090a0ab 100644 --- a/internal/testutil/apps/cluster_factory.go +++ b/internal/testutil/apps/cluster_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/cluster_replication_test_util.go b/internal/testutil/apps/cluster_replication_test_util.go index 45322b22c..a09c87530 100644 --- a/internal/testutil/apps/cluster_replication_test_util.go +++ b/internal/testutil/apps/cluster_replication_test_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/cluster_stateless_test_util.go b/internal/testutil/apps/cluster_stateless_test_util.go index 4dcb35e13..480497952 100644 --- a/internal/testutil/apps/cluster_stateless_test_util.go +++ b/internal/testutil/apps/cluster_stateless_test_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/cluster_util.go b/internal/testutil/apps/cluster_util.go index 32746f482..938ab6ae2 100644 --- a/internal/testutil/apps/cluster_util.go +++ b/internal/testutil/apps/cluster_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/clusterdef_factory.go b/internal/testutil/apps/clusterdef_factory.go index 35ae0f550..0b47530ff 100644 --- a/internal/testutil/apps/clusterdef_factory.go +++ b/internal/testutil/apps/clusterdef_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/clusterversion_factory.go b/internal/testutil/apps/clusterversion_factory.go index e25d737cd..8d78985bf 100644 --- a/internal/testutil/apps/clusterversion_factory.go +++ b/internal/testutil/apps/clusterversion_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/common_util.go b/internal/testutil/apps/common_util.go index 8658dfa5a..04c1a6560 100644 --- a/internal/testutil/apps/common_util.go +++ b/internal/testutil/apps/common_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/componentclassdefinition_factory.go b/internal/testutil/apps/componentclassdefinition_factory.go index 2044b7f05..0b38b953d 100644 --- a/internal/testutil/apps/componentclassdefinition_factory.go +++ b/internal/testutil/apps/componentclassdefinition_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/componentresourceconstraint_factory.go b/internal/testutil/apps/componentresourceconstraint_factory.go index 879be88be..288e7327f 100644 --- a/internal/testutil/apps/componentresourceconstraint_factory.go +++ b/internal/testutil/apps/componentresourceconstraint_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/constant.go b/internal/testutil/apps/constant.go index b83c69ce4..bc9f0ad8d 100644 --- a/internal/testutil/apps/constant.go +++ b/internal/testutil/apps/constant.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/deployment_factoy.go b/internal/testutil/apps/deployment_factoy.go index 797df7615..75bcdf7e8 100644 --- a/internal/testutil/apps/deployment_factoy.go +++ b/internal/testutil/apps/deployment_factoy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/native_object_util.go b/internal/testutil/apps/native_object_util.go index daa08adc1..6224402ff 100644 --- a/internal/testutil/apps/native_object_util.go +++ b/internal/testutil/apps/native_object_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/opsrequest_util.go b/internal/testutil/apps/opsrequest_util.go index c442e61c0..a5ae3cd44 100644 --- a/internal/testutil/apps/opsrequest_util.go +++ b/internal/testutil/apps/opsrequest_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/pod_factory.go b/internal/testutil/apps/pod_factory.go index c0c0e2881..0b6d277cd 100644 --- a/internal/testutil/apps/pod_factory.go +++ b/internal/testutil/apps/pod_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/pvc_factoy.go b/internal/testutil/apps/pvc_factoy.go index 85dc44d6d..3e32cf06b 100644 --- a/internal/testutil/apps/pvc_factoy.go +++ b/internal/testutil/apps/pvc_factoy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/restorejob_factory.go b/internal/testutil/apps/restorejob_factory.go index a8ff3d9f2..1c46ab9b5 100644 --- a/internal/testutil/apps/restorejob_factory.go +++ b/internal/testutil/apps/restorejob_factory.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/apps/statefulset_factoy.go b/internal/testutil/apps/statefulset_factoy.go index c73919593..18366814f 100644 --- a/internal/testutil/apps/statefulset_factoy.go +++ b/internal/testutil/apps/statefulset_factoy.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package apps diff --git a/internal/testutil/k8s/deployment_util.go b/internal/testutil/k8s/deployment_util.go index 6c8363d85..6041bd7fb 100644 --- a/internal/testutil/k8s/deployment_util.go +++ b/internal/testutil/k8s/deployment_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testutil diff --git a/internal/testutil/k8s/k8sclient_util.go b/internal/testutil/k8s/k8sclient_util.go index 76935a988..d55bbabbd 100644 --- a/internal/testutil/k8s/k8sclient_util.go +++ b/internal/testutil/k8s/k8sclient_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testutil diff --git a/internal/testutil/k8s/mocks/generate.go b/internal/testutil/k8s/mocks/generate.go index 2acc5a69a..9ed9b29be 100644 --- a/internal/testutil/k8s/mocks/generate.go +++ b/internal/testutil/k8s/mocks/generate.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package mocks diff --git a/internal/testutil/k8s/statefulset_util.go b/internal/testutil/k8s/statefulset_util.go index 6af9cf6d8..6cdb97057 100644 --- a/internal/testutil/k8s/statefulset_util.go +++ b/internal/testutil/k8s/statefulset_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testutil diff --git a/internal/testutil/k8s/storage_util.go b/internal/testutil/k8s/storage_util.go index b96ddb063..77f0fb013 100644 --- a/internal/testutil/k8s/storage_util.go +++ b/internal/testutil/k8s/storage_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testutil diff --git a/internal/testutil/k8s/tunnel_util.go b/internal/testutil/k8s/tunnel_util.go index f5f09eb06..66b5a1b5e 100644 --- a/internal/testutil/k8s/tunnel_util.go +++ b/internal/testutil/k8s/tunnel_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testutil diff --git a/internal/testutil/type.go b/internal/testutil/type.go index 8a2c1a961..887194433 100644 --- a/internal/testutil/type.go +++ b/internal/testutil/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testutil diff --git a/internal/unstructured/config_object.go b/internal/unstructured/config_object.go index 79999b21b..4c4a9ac2a 100644 --- a/internal/unstructured/config_object.go +++ b/internal/unstructured/config_object.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/redis/lexer.go b/internal/unstructured/redis/lexer.go index ed9be0442..996aff60d 100644 --- a/internal/unstructured/redis/lexer.go +++ b/internal/unstructured/redis/lexer.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/internal/unstructured/redis/parser_fsm.go b/internal/unstructured/redis/parser_fsm.go index 82adbb165..881354c16 100644 --- a/internal/unstructured/redis/parser_fsm.go +++ b/internal/unstructured/redis/parser_fsm.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/internal/unstructured/redis/parser_fsm_test.go b/internal/unstructured/redis/parser_fsm_test.go index a12336265..77c9b47c8 100644 --- a/internal/unstructured/redis/parser_fsm_test.go +++ b/internal/unstructured/redis/parser_fsm_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/internal/unstructured/redis/rune_util.go b/internal/unstructured/redis/rune_util.go index c60e4dfa9..38ea1e3e7 100644 --- a/internal/unstructured/redis/rune_util.go +++ b/internal/unstructured/redis/rune_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/internal/unstructured/redis/rune_util_test.go b/internal/unstructured/redis/rune_util_test.go index 027c7d2a4..27d94b825 100644 --- a/internal/unstructured/redis/rune_util_test.go +++ b/internal/unstructured/redis/rune_util_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package redis diff --git a/internal/unstructured/redis_config.go b/internal/unstructured/redis_config.go index 6d0e04fda..a12943e35 100644 --- a/internal/unstructured/redis_config.go +++ b/internal/unstructured/redis_config.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/redis_config_test.go b/internal/unstructured/redis_config_test.go index c7e87141a..7920726d6 100644 --- a/internal/unstructured/redis_config_test.go +++ b/internal/unstructured/redis_config_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/type.go b/internal/unstructured/type.go index 2ec4300e1..e9b8febe4 100644 --- a/internal/unstructured/type.go +++ b/internal/unstructured/type.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/viper_util.go b/internal/unstructured/viper_util.go index 5d94e8870..a69bd1b4a 100644 --- a/internal/unstructured/viper_util.go +++ b/internal/unstructured/viper_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/viper_wrap.go b/internal/unstructured/viper_wrap.go index 1a25acce3..36571089c 100644 --- a/internal/unstructured/viper_wrap.go +++ b/internal/unstructured/viper_wrap.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/viper_wrap_test.go b/internal/unstructured/viper_wrap_test.go index a20e41c2d..2ab1c9760 100644 --- a/internal/unstructured/viper_wrap_test.go +++ b/internal/unstructured/viper_wrap_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/xml_config.go b/internal/unstructured/xml_config.go index 267f99162..015886805 100644 --- a/internal/unstructured/xml_config.go +++ b/internal/unstructured/xml_config.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/xml_config_test.go b/internal/unstructured/xml_config_test.go index 4289d7e5b..c71af5c80 100644 --- a/internal/unstructured/xml_config_test.go +++ b/internal/unstructured/xml_config_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/yaml_config.go b/internal/unstructured/yaml_config.go index 0e9cc1029..cd489b388 100644 --- a/internal/unstructured/yaml_config.go +++ b/internal/unstructured/yaml_config.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/unstructured/yaml_config_test.go b/internal/unstructured/yaml_config_test.go index d2ff68011..289a90577 100644 --- a/internal/unstructured/yaml_config_test.go +++ b/internal/unstructured/yaml_config_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package unstructured diff --git a/internal/webhook/pod_admission.go b/internal/webhook/pod_admission.go index e1e68eab3..44846baa5 100644 --- a/internal/webhook/pod_admission.go +++ b/internal/webhook/pod_admission.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package webhook diff --git a/internal/webhook/webhook.go b/internal/webhook/webhook.go index 6640a7b93..068d891be 100644 --- a/internal/webhook/webhook.go +++ b/internal/webhook/webhook.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package webhook diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1b03d2ddd..cc231a5f5 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package e2e_test diff --git a/test/e2e/envcheck/envcheck.go b/test/e2e/envcheck/envcheck.go index 0aa9934dc..7bf1c4748 100644 --- a/test/e2e/envcheck/envcheck.go +++ b/test/e2e/envcheck/envcheck.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package envcheck diff --git a/test/e2e/installation/installcheck.go b/test/e2e/installation/installcheck.go index d03432a5d..805e45646 100644 --- a/test/e2e/installation/installcheck.go +++ b/test/e2e/installation/installcheck.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package installation diff --git a/test/e2e/testdata/smoketest/playgroundtest.go b/test/e2e/testdata/smoketest/playgroundtest.go index be021faff..01f5c8d92 100644 --- a/test/e2e/testdata/smoketest/playgroundtest.go +++ b/test/e2e/testdata/smoketest/playgroundtest.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package smoketest diff --git a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml index aabbb40b9..5720b38d1 100644 --- a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml +++ b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml @@ -539,19 +539,22 @@ spec: # ConfigurationSchema that impose restrictions on engine parameter's rule configurationSchema: cue: |- - // Copyright ApeCloud, Inc. + //Copyright (C) 2022-2023 ApeCloud Co., Ltd // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. - // You may obtain a copy of the License at + //This file is part of KubeBlocks project // - // http://www.apache.org/licenses/LICENSE-2.0 + //This program is free software: you can redistribute it and/or modify + //it under the terms of the GNU Affero General Public License as published by + //the Free Software Foundation, either version 3 of the License, or + //(at your option) any later version. // - // Unless required by applicable law or agreed to in writing, software - // distributed under the License is distributed on an "AS IS" BASIS, - // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - // See the License for the specific language governing permissions and - // limitations under the License. + //This program is distributed in the hope that it will be useful + //but WITHOUT ANY WARRANTY; without even the implied warranty of + //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + //GNU Affero General Public License for more details. + // + //You should have received a copy of the GNU Affero General Public License + //along with this program. If not, see . #RedisParameter: { diff --git a/test/e2e/testdata/smoketest/smoketestrun.go b/test/e2e/testdata/smoketest/smoketestrun.go index 5536344e6..3fefb65d8 100644 --- a/test/e2e/testdata/smoketest/smoketestrun.go +++ b/test/e2e/testdata/smoketest/smoketestrun.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package smoketest diff --git a/test/e2e/types.go b/test/e2e/types.go index 5af6572c9..d9c813d41 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package e2e diff --git a/test/e2e/util/client.go b/test/e2e/util/client.go index da3c368d4..24c96aed8 100644 --- a/test/e2e/util/client.go +++ b/test/e2e/util/client.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/test/e2e/util/common.go b/test/e2e/util/common.go index f6e21ca83..2c1eb74e1 100644 --- a/test/e2e/util/common.go +++ b/test/e2e/util/common.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ /* diff --git a/test/e2e/util/smoke_util.go b/test/e2e/util/smoke_util.go index 76129e749..74772c864 100644 --- a/test/e2e/util/smoke_util.go +++ b/test/e2e/util/smoke_util.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package util diff --git a/test/integration/backup_mysql_test.go b/test/integration/backup_mysql_test.go index 3bf16c836..c2fbfdd14 100644 --- a/test/integration/backup_mysql_test.go +++ b/test/integration/backup_mysql_test.go @@ -1,18 +1,22 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ + package appstest import ( diff --git a/test/integration/controller_suite_test.go b/test/integration/controller_suite_test.go index 69c7902bf..31a66d4fc 100644 --- a/test/integration/controller_suite_test.go +++ b/test/integration/controller_suite_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package appstest diff --git a/test/integration/mysql_ha_test.go b/test/integration/mysql_ha_test.go index 011fd11c9..29d94ddc1 100644 --- a/test/integration/mysql_ha_test.go +++ b/test/integration/mysql_ha_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package appstest diff --git a/test/integration/mysql_reconfigure_test.go b/test/integration/mysql_reconfigure_test.go index d0fc56576..2cf423078 100644 --- a/test/integration/mysql_reconfigure_test.go +++ b/test/integration/mysql_reconfigure_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package appstest diff --git a/test/integration/mysql_scale_test.go b/test/integration/mysql_scale_test.go index b267b364a..b0987431a 100644 --- a/test/integration/mysql_scale_test.go +++ b/test/integration/mysql_scale_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package appstest diff --git a/test/integration/redis_hscale_test.go b/test/integration/redis_hscale_test.go index efca18596..34dc613de 100644 --- a/test/integration/redis_hscale_test.go +++ b/test/integration/redis_hscale_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package appstest diff --git a/test/testdata/cue_testdata/clickhouse.cue b/test/testdata/cue_testdata/clickhouse.cue index dbc488290..ff024ef72 100644 --- a/test/testdata/cue_testdata/clickhouse.cue +++ b/test/testdata/cue_testdata/clickhouse.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #ProfilesParameter: { profiles: [string]: #ClickhouseParameter diff --git a/test/testdata/cue_testdata/mongod.cue b/test/testdata/cue_testdata/mongod.cue index 4543b77b9..f4477eb76 100644 --- a/test/testdata/cue_testdata/mongod.cue +++ b/test/testdata/cue_testdata/mongod.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #MongodParameter: { net: { diff --git a/test/testdata/cue_testdata/mysql.cue b/test/testdata/cue_testdata/mysql.cue index 97b3ca39a..35f5f6173 100644 --- a/test/testdata/cue_testdata/mysql.cue +++ b/test/testdata/cue_testdata/mysql.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // mysql config validator // mysql server param: a set of name/value pairs. diff --git a/test/testdata/cue_testdata/mysql_for_cli.cue b/test/testdata/cue_testdata/mysql_for_cli.cue index 8baf9f57a..418c61399 100644 --- a/test/testdata/cue_testdata/mysql_for_cli.cue +++ b/test/testdata/cue_testdata/mysql_for_cli.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // top level configuration type // mysql server param: a set of name/value pairs. diff --git a/test/testdata/cue_testdata/mysql_openapi.cue b/test/testdata/cue_testdata/mysql_openapi.cue index d9613fb8d..779a90028 100644 --- a/test/testdata/cue_testdata/mysql_openapi.cue +++ b/test/testdata/cue_testdata/mysql_openapi.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . // mysql config validator #MysqlParameter: { diff --git a/test/testdata/cue_testdata/mysql_openapi_v2.cue b/test/testdata/cue_testdata/mysql_openapi_v2.cue index 82b2b2ce3..57e2c7857 100644 --- a/test/testdata/cue_testdata/mysql_openapi_v2.cue +++ b/test/testdata/cue_testdata/mysql_openapi_v2.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #SectionParameter: { // [OFF|ON] default ON diff --git a/test/testdata/cue_testdata/mysql_simple.cue b/test/testdata/cue_testdata/mysql_simple.cue index a5e2154e3..d696fb31b 100644 --- a/test/testdata/cue_testdata/mysql_simple.cue +++ b/test/testdata/cue_testdata/mysql_simple.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #Section: { // SectionName is extract section name diff --git a/test/testdata/cue_testdata/pg14.cue b/test/testdata/cue_testdata/pg14.cue index 7e58446c0..687474175 100644 --- a/test/testdata/cue_testdata/pg14.cue +++ b/test/testdata/cue_testdata/pg14.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #PGPameter: { // PostgreSQL parameters: https://postgresqlco.nf/doc/en/param/ diff --git a/test/testdata/cue_testdata/wesql.cue b/test/testdata/cue_testdata/wesql.cue index e2fc384c5..19104e267 100644 --- a/test/testdata/cue_testdata/wesql.cue +++ b/test/testdata/cue_testdata/wesql.cue @@ -1,16 +1,19 @@ -// Copyright ApeCloud, Inc. +//Copyright (C) 2022-2023 ApeCloud Co., Ltd // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +//This file is part of KubeBlocks project // -// http://www.apache.org/licenses/LICENSE-2.0 +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . #MysqlParameter: { diff --git a/test/testdata/resources/mysql-consensus-config-constraint.yaml b/test/testdata/resources/mysql-consensus-config-constraint.yaml index a584148d8..2ff293225 100644 --- a/test/testdata/resources/mysql-consensus-config-constraint.yaml +++ b/test/testdata/resources/mysql-consensus-config-constraint.yaml @@ -26,19 +26,22 @@ spec: # schema: auto generate from mmmcue scripts # example: ../../internal/configuration/testdata/mysql_openapi.json cue: |- - // Copyright ApeCloud, Inc. + //Copyright (C) 2022-2023 ApeCloud Co., Ltd // - // Licensed under the Apache License, Version 2.0 (the "License"); - // you may not use this file except in compliance with the License. - // You may obtain a copy of the License at + //This file is part of KubeBlocks project // - // http://www.apache.org/licenses/LICENSE-2.0 + //This program is free software: you can redistribute it and/or modify + //it under the terms of the GNU Affero General Public License as published by + //the Free Software Foundation, either version 3 of the License, or + //(at your option) any later version. // - // Unless required by applicable law or agreed to in writing, software - // distributed under the License is distributed on an "AS IS" BASIS, - // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - // See the License for the specific language governing permissions and - // limitations under the License. + //This program is distributed in the hope that it will be useful + //but WITHOUT ANY WARRANTY; without even the implied warranty of + //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + //GNU Affero General Public License for more details. + // + //You should have received a copy of the GNU Affero General Public License + //along with this program. If not, see . #MysqlParameter: { diff --git a/test/testdata/testdata.go b/test/testdata/testdata.go index b0408255e..66c5cbb95 100644 --- a/test/testdata/testdata.go +++ b/test/testdata/testdata.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package testdata diff --git a/tools/tools.go b/tools/tools.go index 9108b71b2..6338bda98 100644 --- a/tools/tools.go +++ b/tools/tools.go @@ -2,19 +2,22 @@ // +build tools /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ // This package imports things required by build scripts, to force `go mod` to see them as dependencies diff --git a/version/version.go b/version/version.go index 754f87756..4629d8964 100644 --- a/version/version.go +++ b/version/version.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package version From a2eab6d758ebedb2eaea642ee34be54275310fa7 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Mon, 24 Apr 2023 11:12:23 +0800 Subject: [PATCH 159/439] chore: update kbcli accounts examples (#2884) Co-authored-by: shanshanying --- .../cli/kbcli_cluster_create-account.md | 12 +++-- .../cli/kbcli_cluster_delete-account.md | 6 ++- .../cli/kbcli_cluster_describe-account.md | 6 ++- .../user_docs/cli/kbcli_cluster_grant-role.md | 6 ++- .../cli/kbcli_cluster_list-accounts.md | 11 +++-- .../cli/kbcli_cluster_revoke-role.md | 6 ++- internal/cli/cmd/cluster/accounts.go | 47 +++++++++++++------ 7 files changed, 66 insertions(+), 28 deletions(-) diff --git a/docs/user_docs/cli/kbcli_cluster_create-account.md b/docs/user_docs/cli/kbcli_cluster_create-account.md index 7b0078f75..7a5b97770 100644 --- a/docs/user_docs/cli/kbcli_cluster_create-account.md +++ b/docs/user_docs/cli/kbcli_cluster_create-account.md @@ -11,12 +11,14 @@ kbcli cluster create-account [flags] ### Examples ``` - # create account - kbcli cluster create-account NAME --component COMPNAME --username NAME --password PASSWD + # create account with password + kbcli cluster create-account NAME --component COMPNAME --name USERNAME --password PASSWD # create account without password - kbcli cluster create-account NAME --component COMPNAME --username NAME - # create account with expired interval - kbcli cluster create-account NAME --component COMPNAME --username NAME --password PASSWD --expiredAt 2046-01-02T15:04:05Z + kbcli cluster create-account NAME --component COMPNAME --name USERNAME + # create account with default component + kbcli cluster create-account NAME --name USERNAME + # create account for instance + kbcli cluster create-account --instance INSTANCE --name USERNAME ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_delete-account.md b/docs/user_docs/cli/kbcli_cluster_delete-account.md index 72b32eb66..ea8bcd8f1 100644 --- a/docs/user_docs/cli/kbcli_cluster_delete-account.md +++ b/docs/user_docs/cli/kbcli_cluster_delete-account.md @@ -12,7 +12,11 @@ kbcli cluster delete-account [flags] ``` # delete account by name - kbcli cluster delete-account NAME --component COMPNAME --username NAME + kbcli cluster delete-account NAME --component COMPNAME --name USERNAME + # delete account with default component + kbcli cluster delete-account NAME --name USERNAME + # delete account for instance + kbcli cluster delete-account --instance INSTANCE --name USERNAME ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_describe-account.md b/docs/user_docs/cli/kbcli_cluster_describe-account.md index 793c90020..75597352b 100644 --- a/docs/user_docs/cli/kbcli_cluster_describe-account.md +++ b/docs/user_docs/cli/kbcli_cluster_describe-account.md @@ -12,7 +12,11 @@ kbcli cluster describe-account [flags] ``` # describe account and show role information - kbcli cluster describe-account NAME --component COMPNAME--username NAME + kbcli cluster describe-account NAME --component COMPNAME --name USERNAME + # describe account with default component + kbcli cluster delete-account NAME --name USERNAME + # describe account for instance + kbcli cluster describe-account --instance INSTANCE --name USERNAME ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_grant-role.md b/docs/user_docs/cli/kbcli_cluster_grant-role.md index 5826f516e..6dcb76fc5 100644 --- a/docs/user_docs/cli/kbcli_cluster_grant-role.md +++ b/docs/user_docs/cli/kbcli_cluster_grant-role.md @@ -12,7 +12,11 @@ kbcli cluster grant-role [flags] ``` # grant role to user - kbcli cluster grant-role NAME --component COMPNAME --username NAME --role ROLENAME + kbcli cluster grant-role NAME --component COMPNAME --name USERNAME --role ROLENAME + # grant role to user with default component + kbcli cluster grant-role NAME --name USERNAME --role ROLENAME + # grant role to user for instance + kbcli cluster grant-role --instance INSTANCE --name USERNAME --role ROLENAME ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_list-accounts.md b/docs/user_docs/cli/kbcli_cluster_list-accounts.md index 5d5efcfbd..1611e50c4 100644 --- a/docs/user_docs/cli/kbcli_cluster_list-accounts.md +++ b/docs/user_docs/cli/kbcli_cluster_list-accounts.md @@ -11,11 +11,12 @@ kbcli cluster list-accounts [flags] ### Examples ``` - # list all users from specified component of a cluster - kbcli cluster list-accounts NAME --component COMPNAME --show-connected-users - - # list all users from cluster's one particular instance - kbcli cluster list-accounts NAME -i INSTANCE + # list all users for component + kbcli cluster list-accounts NAME --component COMPNAME + # list all users with default component + kbcli cluster list-accounts NAME + # list all users from instance + kbcli cluster list-accounts --instance INSTANCE ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_revoke-role.md b/docs/user_docs/cli/kbcli_cluster_revoke-role.md index 1d8bf3933..014c7b10b 100644 --- a/docs/user_docs/cli/kbcli_cluster_revoke-role.md +++ b/docs/user_docs/cli/kbcli_cluster_revoke-role.md @@ -12,7 +12,11 @@ kbcli cluster revoke-role [flags] ``` # revoke role from user - kbcli cluster revoke-role NAME --component COMPNAME --role ROLENAME + kbcli cluster revoke-role NAME --component COMPNAME --name USERNAME --role ROLENAME + # revoke role from user with default component + kbcli cluster revoke-role NAME --name USERNAME --role ROLENAME + # revoke role from user for instance + kbcli cluster revoke-role --instance INSTANCE --name USERNAME --role ROLENAME ``` ### Options diff --git a/internal/cli/cmd/cluster/accounts.go b/internal/cli/cmd/cluster/accounts.go index b56b5caab..6f58c980b 100644 --- a/internal/cli/cmd/cluster/accounts.go +++ b/internal/cli/cmd/cluster/accounts.go @@ -33,38 +33,57 @@ import ( var ( createUserExamples = templates.Examples(` - # create account - kbcli cluster create-account NAME --component COMPNAME --username NAME --password PASSWD + # create account with password + kbcli cluster create-account NAME --component COMPNAME --name USERNAME --password PASSWD # create account without password - kbcli cluster create-account NAME --component COMPNAME --username NAME - # create account with expired interval - kbcli cluster create-account NAME --component COMPNAME --username NAME --password PASSWD --expiredAt 2046-01-02T15:04:05Z + kbcli cluster create-account NAME --component COMPNAME --name USERNAME + # create account with default component + kbcli cluster create-account NAME --name USERNAME + # create account for instance + kbcli cluster create-account --instance INSTANCE --name USERNAME `) deleteUserExamples = templates.Examples(` # delete account by name - kbcli cluster delete-account NAME --component COMPNAME --username NAME + kbcli cluster delete-account NAME --component COMPNAME --name USERNAME + # delete account with default component + kbcli cluster delete-account NAME --name USERNAME + # delete account for instance + kbcli cluster delete-account --instance INSTANCE --name USERNAME `) descUserExamples = templates.Examples(` # describe account and show role information - kbcli cluster describe-account NAME --component COMPNAME--username NAME + kbcli cluster describe-account NAME --component COMPNAME --name USERNAME + # describe account with default component + kbcli cluster delete-account NAME --name USERNAME + # describe account for instance + kbcli cluster describe-account --instance INSTANCE --name USERNAME `) listUsersExample = templates.Examples(` - # list all users from specified component of a cluster - kbcli cluster list-accounts NAME --component COMPNAME --show-connected-users - - # list all users from cluster's one particular instance - kbcli cluster list-accounts NAME -i INSTANCE + # list all users for component + kbcli cluster list-accounts NAME --component COMPNAME + # list all users with default component + kbcli cluster list-accounts NAME + # list all users from instance + kbcli cluster list-accounts --instance INSTANCE `) grantRoleExamples = templates.Examples(` # grant role to user - kbcli cluster grant-role NAME --component COMPNAME --username NAME --role ROLENAME + kbcli cluster grant-role NAME --component COMPNAME --name USERNAME --role ROLENAME + # grant role to user with default component + kbcli cluster grant-role NAME --name USERNAME --role ROLENAME + # grant role to user for instance + kbcli cluster grant-role --instance INSTANCE --name USERNAME --role ROLENAME `) revokeRoleExamples = templates.Examples(` # revoke role from user - kbcli cluster revoke-role NAME --component COMPNAME --role ROLENAME + kbcli cluster revoke-role NAME --component COMPNAME --name USERNAME --role ROLENAME + # revoke role from user with default component + kbcli cluster revoke-role NAME --name USERNAME --role ROLENAME + # revoke role from user for instance + kbcli cluster revoke-role --instance INSTANCE --name USERNAME --role ROLENAME `) ) From 47b559e622b049b89035a85188b6d87d706a6d37 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 24 Apr 2023 13:11:48 +0800 Subject: [PATCH 160/439] fix: faield to pg config template render (#2861) (#2887) --- deploy/postgresql/config/pg14-config.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/postgresql/config/pg14-config.tpl b/deploy/postgresql/config/pg14-config.tpl index 0288fcf3b..c89467e27 100644 --- a/deploy/postgresql/config/pg14-config.tpl +++ b/deploy/postgresql/config/pg14-config.tpl @@ -67,7 +67,7 @@ log_directory = 'log' log_filename = 'postgresql-%Y-%m-%d.log' {{ end -}} {{ end -}} -{{ end -}} +{{ end }} log_lock_waits = 'True' log_min_duration_statement = '100' From 99e6d62ee565dc09887dba8d04fea3bc900afd24 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 24 Apr 2023 14:51:47 +0800 Subject: [PATCH 161/439] fix: tpl tools failed to render config (#2890) --- cmd/tpl/app/mock_client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/tpl/app/mock_client.go b/cmd/tpl/app/mock_client.go index 50d67970d..539e65ed9 100644 --- a/cmd/tpl/app/mock_client.go +++ b/cmd/tpl/app/mock_client.go @@ -22,6 +22,8 @@ package app import ( "context" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -73,8 +75,9 @@ func (m *mockClient) Get(ctx context.Context, key client.ObjectKey, obj client.O objKey.Namespace = "" if object, ok := m.objects[objKey]; ok { testutil.SetGetReturnedObject(obj, object) + return nil } - return nil + return apierrors.NewNotFound(corev1.SchemeGroupVersion.WithResource("mock_resource").GroupResource(), key.String()) } func (m *mockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { From 1c14051bbc44e6e7d786b711d42ecd6d91339e2e Mon Sep 17 00:00:00 2001 From: shaojiang Date: Mon, 24 Apr 2023 16:52:49 +0800 Subject: [PATCH 162/439] fix: add volumesnapshot v1beta1 compatibility (#2892) --- cmd/manager/main.go | 3 + controllers/apps/cluster_controller.go | 7 +- .../dataprotection/backup_controller.go | 36 ++-- deploy/helm/templates/deployment.yaml | 4 + go.mod | 1 + go.sum | 2 + .../transformer_sts_horizontal_scaling.go | 15 +- internal/controllerutil/suite_test.go | 15 +- internal/controllerutil/volumesnapshot.go | 170 ++++++++++++++++++ .../controllerutil/volumesnapshot_test.go | 94 ++++++++++ 10 files changed, 327 insertions(+), 20 deletions(-) create mode 100644 internal/controllerutil/volumesnapshot.go create mode 100644 internal/controllerutil/volumesnapshot_test.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 84c31ce6e..43b206de9 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -30,6 +30,7 @@ import ( // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. "github.com/fsnotify/fsnotify" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -79,6 +80,7 @@ func init() { utilruntime.Must(appsv1alpha1.AddToScheme(scheme)) utilruntime.Must(dataprotectionv1alpha1.AddToScheme(scheme)) utilruntime.Must(snapshotv1.AddToScheme(scheme)) + utilruntime.Must(snapshotv1beta1.AddToScheme(scheme)) utilruntime.Must(extensionsv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme @@ -92,6 +94,7 @@ func init() { viper.SetDefault(constant.CfgKeyCtrlrReconcileRetryDurationMS, 100) viper.SetDefault("CERT_DIR", "/tmp/k8s-webhook-server/serving-certs") viper.SetDefault("VOLUMESNAPSHOT", false) + viper.SetDefault("VOLUMESNAPSHOT_API_BETA", false) viper.SetDefault(constant.KBToolsImage, "apecloud/kubeblocks-tools:latest") viper.SetDefault("PROBE_SERVICE_HTTP_PORT", 3501) viper.SetDefault("PROBE_SERVICE_GRPC_PORT", 50001) diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index ec6ed65e7..a2ff369d3 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -23,6 +23,7 @@ import ( "context" "time" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/viper" appsv1 "k8s.io/api/apps/v1" @@ -232,7 +233,11 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&dataprotectionv1alpha1.BackupPolicy{}). Owns(&dataprotectionv1alpha1.Backup{}) if viper.GetBool("VOLUMESNAPSHOT") { - b.Owns(&snapshotv1.VolumeSnapshot{}, builder.OnlyMetadata, builder.Predicates{}) + if intctrlutil.InVolumeSnapshotV1Beta1() { + b.Owns(&snapshotv1beta1.VolumeSnapshot{}, builder.OnlyMetadata, builder.Predicates{}) + } else { + b.Owns(&snapshotv1.VolumeSnapshot{}, builder.OnlyMetadata, builder.Predicates{}) + } } return b.Complete(r) } diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index f9e982124..d2c911a64 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -29,6 +29,7 @@ import ( "strings" "time" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/viper" batchv1 "k8s.io/api/batch/v1" @@ -63,9 +64,10 @@ const ( // BackupReconciler reconciles a Backup object type BackupReconciler struct { client.Client - Scheme *k8sruntime.Scheme - Recorder record.EventRecorder - clock clock.RealClock + Scheme *k8sruntime.Scheme + Recorder record.EventRecorder + clock clock.RealClock + snapshotCli *intctrlutil.VolumeSnapshotCompatClient } // +kubebuilder:rbac:groups=dataprotection.kubeblocks.io,resources=backups,verbs=get;list;watch;create;update;patch;delete @@ -90,6 +92,11 @@ func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr Log: log.FromContext(ctx).WithValues("backup", req.NamespacedName), Recorder: r.Recorder, } + // initialize snapshotCompatClient + r.snapshotCli = &intctrlutil.VolumeSnapshotCompatClient{ + Client: r.Client, + Ctx: ctx, + } // Get backup obj backup := &dataprotectionv1alpha1.Backup{} if err := r.Client.Get(reqCtx.Ctx, reqCtx.Req.NamespacedName, backup); err != nil { @@ -129,7 +136,11 @@ func (r *BackupReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&batchv1.Job{}) if viper.GetBool("VOLUMESNAPSHOT") { - b.Owns(&snapshotv1.VolumeSnapshot{}, builder.OnlyMetadata, builder.Predicates{}) + if intctrlutil.InVolumeSnapshotV1Beta1() { + b.Owns(&snapshotv1beta1.VolumeSnapshot{}, builder.OnlyMetadata, builder.Predicates{}) + } else { + b.Owns(&snapshotv1.VolumeSnapshot{}, builder.OnlyMetadata, builder.Predicates{}) + } } return b.Complete(r) @@ -386,7 +397,7 @@ func (r *BackupReconciler) doInProgressPhaseAction( backup.Status.Phase = dataprotectionv1alpha1.BackupCompleted backup.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now().UTC()} snap := &snapshotv1.VolumeSnapshot{} - exists, _ := intctrlutil.CheckResourceExists(reqCtx.Ctx, r.Client, key, snap) + exists, _ := r.snapshotCli.CheckResourceExists(key, snap) if exists { backup.Status.TotalSize = snap.Status.RestoreSize.String() } @@ -587,7 +598,7 @@ func (r *BackupReconciler) createVolumeSnapshot( snapshotPolicy *dataprotectionv1alpha1.SnapshotPolicy) error { snap := &snapshotv1.VolumeSnapshot{} - exists, err := intctrlutil.CheckResourceExists(reqCtx.Ctx, r.Client, reqCtx.Req.NamespacedName, snap) + exists, err := r.snapshotCli.CheckResourceExists(reqCtx.Req.NamespacedName, snap) if err != nil { return err } @@ -640,7 +651,7 @@ func (r *BackupReconciler) createVolumeSnapshot( } reqCtx.Log.V(1).Info("create a volumeSnapshot from backup", "snapshot", snap.Name) - if err = r.Client.Create(reqCtx.Ctx, snap); err != nil && !apierrors.IsAlreadyExists(err) { + if err = r.snapshotCli.Create(snap); err != nil && !apierrors.IsAlreadyExists(err) { return err } } @@ -653,7 +664,8 @@ func (r *BackupReconciler) ensureVolumeSnapshotReady(reqCtx intctrlutil.RequestC key types.NamespacedName) (bool, error) { snap := &snapshotv1.VolumeSnapshot{} - exists, err := intctrlutil.CheckResourceExists(reqCtx.Ctx, r.Client, key, snap) + // not found, continue creation + exists, err := r.snapshotCli.CheckResourceExists(key, snap) if err != nil { return false, err } @@ -948,20 +960,20 @@ func (r *BackupReconciler) deleteReferenceBatchV1Jobs(reqCtx intctrlutil.Request func (r *BackupReconciler) deleteReferenceVolumeSnapshot(reqCtx intctrlutil.RequestCtx, backup *dataprotectionv1alpha1.Backup) error { snaps := &snapshotv1.VolumeSnapshotList{} - if err := r.Client.List(reqCtx.Ctx, snaps, + if err := r.snapshotCli.List(snaps, client.InNamespace(reqCtx.Req.Namespace), client.MatchingLabels(buildBackupLabels(backup))); err != nil { return err } for _, i := range snaps.Items { if controllerutil.ContainsFinalizer(&i, dataProtectionFinalizerName) { - patch := client.MergeFrom(i.DeepCopy()) + patch := i.DeepCopy() controllerutil.RemoveFinalizer(&i, dataProtectionFinalizerName) - if err := r.Patch(reqCtx.Ctx, &i, patch); err != nil { + if err := r.snapshotCli.Patch(&i, patch); err != nil { return err } } - if err := intctrlutil.BackgroundDeleteObject(r.Client, reqCtx.Ctx, &i); err != nil { + if err := r.snapshotCli.Delete(&i); err != nil { return err } } diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml index dc758226d..5d8dbccf5 100644 --- a/deploy/helm/templates/deployment.yaml +++ b/deploy/helm/templates/deployment.yaml @@ -77,6 +77,10 @@ spec: - name: VOLUMESNAPSHOT value: "true" {{- end }} + {{- if .Capabilities.APIVersions.Has "snapshot.storage.k8s.io/v1beta1" }} + - name: VOLUMESNAPSHOT_API_BETA + value: "true" + {{- end }} {{- if .Values.admissionWebhooks.enabled }} - name: ENABLE_WEBHOOKS value: "true" diff --git a/go.mod b/go.mod index 930b48d23..94496b182 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/jedib0t/go-pretty/v6 v6.4.4 github.com/json-iterator/go v1.1.12 github.com/k3d-io/k3d/v5 v5.4.4 + github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0 github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0 github.com/leaanthony/debme v1.2.1 github.com/manifoldco/promptui v0.9.0 diff --git a/go.sum b/go.sum index 9b4e2d619..88ebb8e68 100644 --- a/go.sum +++ b/go.sum @@ -1198,6 +1198,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0 h1:OYDCOjVcx/5wNzlZ/At8otRibUlw0T6R0xOD31f32bw= +github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0/go.mod h1:Q7VUue/CIrKbtpBdF04a1yjGGgsMaDws1HUxtjzgnEY= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0 h1:cMM5AB37e9aRGjErygVT6EuBPB6s5a+l95OPERmSlVM= diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 46aca15d9..de9f9b6d2 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -541,7 +541,8 @@ func isSnapshotAvailable(cli roclient.ReadonlyClient, ctx context.Context) bool return false } vsList := snapshotv1.VolumeSnapshotList{} - getVSErr := cli.List(ctx, &vsList) + compatClient := intctrlutil.VolumeSnapshotCompatClient{ReadonlyClient: cli, Ctx: ctx} + getVSErr := compatClient.List(&vsList) return getVSErr == nil } @@ -581,7 +582,8 @@ func deleteSnapshot(cli roclient.ReadonlyClient, } reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "BackupJobDelete", "Delete backupJob/%s", snapshotKey.Name) vs := &snapshotv1.VolumeSnapshot{} - if err := cli.Get(ctx, snapshotKey, vs); err != nil { + compatClient := intctrlutil.VolumeSnapshotCompatClient{ReadonlyClient: cli, Ctx: ctx} + if err := compatClient.Get(snapshotKey, vs); err != nil { return client.IgnoreNotFound(err) } vertex := &lifecycleVertex{obj: vs, oriObj: vs, action: actionPtr(DELETE)} @@ -621,7 +623,8 @@ func isVolumeSnapshotExists(cli roclient.ReadonlyClient, component *component.SynthesizedComponent) (bool, error) { ml := getBackupMatchingLabels(cluster.Name, component.Name) vsList := snapshotv1.VolumeSnapshotList{} - if err := cli.List(ctx, &vsList, ml); err != nil { + compatClient := intctrlutil.VolumeSnapshotCompatClient{ReadonlyClient: cli, Ctx: ctx} + if err := compatClient.List(&vsList, ml); err != nil { return false, client.IgnoreNotFound(err) } for _, vs := range vsList.Items { @@ -684,7 +687,8 @@ func isVolumeSnapshotReadyToUse(cli roclient.ReadonlyClient, component *component.SynthesizedComponent) (bool, error) { ml := getBackupMatchingLabels(cluster.Name, component.Name) vsList := snapshotv1.VolumeSnapshotList{} - if err := cli.List(ctx, &vsList, ml); err != nil { + compatClient := intctrlutil.VolumeSnapshotCompatClient{ReadonlyClient: cli, Ctx: ctx} + if err := compatClient.List(&vsList, ml); err != nil { return false, client.IgnoreNotFound(err) } if len(vsList.Items) == 0 || vsList.Items[0].Status == nil { @@ -717,7 +721,8 @@ func checkedCreatePVCFromSnapshot(cli roclient.ReadonlyClient, } ml := getBackupMatchingLabels(cluster.Name, component.Name) vsList := snapshotv1.VolumeSnapshotList{} - if err := cli.List(ctx, &vsList, ml); err != nil { + compatClient := intctrlutil.VolumeSnapshotCompatClient{ReadonlyClient: cli, Ctx: ctx} + if err := compatClient.List(&vsList, ml); err != nil { return err } if len(vsList.Items) == 0 { diff --git a/internal/controllerutil/suite_test.go b/internal/controllerutil/suite_test.go index f415dd8a9..0f8dd32f5 100644 --- a/internal/controllerutil/suite_test.go +++ b/internal/controllerutil/suite_test.go @@ -21,9 +21,11 @@ package controllerutil import ( "context" + "go/build" "path/filepath" "testing" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -74,7 +76,12 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "config", "crd", "bases"), + // use VolumeSnapshot v1beta1 API CRDs. + filepath.Join(build.Default.GOPATH, "pkg", "mod", "github.com", "kubernetes-csi/external-snapshotter/", + "client/v3@v3.0.0", "config", "crd"), + }, ErrorIfCRDPathMissing: true, } @@ -85,8 +92,12 @@ var _ = BeforeSuite(func() { Expect(cfg).NotTo(BeNil()) // +kubebuilder:scaffold:scheme + scheme := scheme.Scheme - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + err = snapshotv1beta1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) }) diff --git a/internal/controllerutil/volumesnapshot.go b/internal/controllerutil/volumesnapshot.go new file mode 100644 index 000000000..ba39c9a98 --- /dev/null +++ b/internal/controllerutil/volumesnapshot.go @@ -0,0 +1,170 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controllerutil + +import ( + "context" + "encoding/json" + + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + "github.com/spf13/viper" + "sigs.k8s.io/controller-runtime/pkg/client" + + roclient "github.com/apecloud/kubeblocks/internal/controller/client" +) + +func InVolumeSnapshotV1Beta1() bool { + return viper.GetBool("VOLUMESNAPSHOT_API_BETA") +} + +// VolumeSnapshotCompatClient client is compatible both VolumeSnapshot v1 and v1beta1 +type VolumeSnapshotCompatClient struct { + client.Client + roclient.ReadonlyClient + Ctx context.Context +} + +func (c *VolumeSnapshotCompatClient) Create(snapshot *snapshotv1.VolumeSnapshot, opts ...client.CreateOption) error { + if InVolumeSnapshotV1Beta1() { + snapshotV1Beta1, err := convertV1ToV1beta1(snapshot) + if err != nil { + return err + } + return c.Client.Create(c.Ctx, snapshotV1Beta1, opts...) + } + return c.Client.Create(c.Ctx, snapshot, opts...) +} + +func (c *VolumeSnapshotCompatClient) Get(key client.ObjectKey, snapshot *snapshotv1.VolumeSnapshot, opts ...client.GetOption) error { + if c.ReadonlyClient == nil { + c.ReadonlyClient = c.Client + } + if InVolumeSnapshotV1Beta1() { + snapshotV1Beta1 := &snapshotv1beta1.VolumeSnapshot{} + err := c.ReadonlyClient.Get(c.Ctx, key, snapshotV1Beta1, opts...) + if err != nil { + return err + } + snap, err := convertV1Beta1ToV1(snapshotV1Beta1) + if err != nil { + return err + } + *snapshot = *snap + return nil + } + return c.ReadonlyClient.Get(c.Ctx, key, snapshot, opts...) +} + +func (c *VolumeSnapshotCompatClient) Delete(snapshot *snapshotv1.VolumeSnapshot, opts ...client.DeleteOption) error { + if InVolumeSnapshotV1Beta1() { + snapshotV1Beta1, err := convertV1ToV1beta1(snapshot) + if err != nil { + return err + } + return BackgroundDeleteObject(c.Client, c.Ctx, snapshotV1Beta1) + } + return BackgroundDeleteObject(c.Client, c.Ctx, snapshot) +} + +func (c *VolumeSnapshotCompatClient) Patch(snapshot *snapshotv1.VolumeSnapshot, deepCopy *snapshotv1.VolumeSnapshot, opts ...client.PatchOption) error { + if InVolumeSnapshotV1Beta1() { + snapshotV1Beta1, err := convertV1ToV1beta1(snapshot) + if err != nil { + return err + } + snapshotV1Beta1Patch, err := convertV1ToV1beta1(deepCopy) + if err != nil { + return err + } + patch := client.MergeFrom(snapshotV1Beta1Patch) + return c.Client.Patch(c.Ctx, snapshotV1Beta1, patch, opts...) + } + snapPatch := client.MergeFrom(deepCopy) + return c.Client.Patch(c.Ctx, snapshot, snapPatch, opts...) +} + +func (c *VolumeSnapshotCompatClient) List(snapshotList *snapshotv1.VolumeSnapshotList, opts ...client.ListOption) error { + if c.ReadonlyClient == nil { + c.ReadonlyClient = c.Client + } + if InVolumeSnapshotV1Beta1() { + snapshotV1Beta1List := &snapshotv1beta1.VolumeSnapshotList{} + err := c.ReadonlyClient.List(c.Ctx, snapshotV1Beta1List, opts...) + if err != nil { + return err + } + snaps, err := convertListV1Beta1ToV1(snapshotV1Beta1List) + if err != nil { + return err + } + *snapshotList = *snaps + return nil + } + return c.ReadonlyClient.List(c.Ctx, snapshotList, opts...) +} + +// CheckResourceExists checks whether resource exist or not. +func (c *VolumeSnapshotCompatClient) CheckResourceExists(key client.ObjectKey, obj *snapshotv1.VolumeSnapshot) (bool, error) { + if err := c.Get(key, obj); err != nil { + return false, client.IgnoreNotFound(err) + } + // if found, return true + return true, nil +} + +func convertV1ToV1beta1(snapshot *snapshotv1.VolumeSnapshot) (*snapshotv1beta1.VolumeSnapshot, error) { + v1beta1Snapshot := &snapshotv1beta1.VolumeSnapshot{} + snapshotBytes, err := json.Marshal(snapshot) + if err != nil { + return nil, err + } + if err := json.Unmarshal(snapshotBytes, v1beta1Snapshot); err != nil { + return nil, err + } + + return v1beta1Snapshot, nil +} + +func convertV1Beta1ToV1(snapshot *snapshotv1beta1.VolumeSnapshot) (*snapshotv1.VolumeSnapshot, error) { + v1Snapshot := &snapshotv1.VolumeSnapshot{} + snapshotBytes, err := json.Marshal(snapshot) + if err != nil { + return nil, err + } + if err := json.Unmarshal(snapshotBytes, v1Snapshot); err != nil { + return nil, err + } + + return v1Snapshot, nil +} + +func convertListV1Beta1ToV1(snapshots *snapshotv1beta1.VolumeSnapshotList) (*snapshotv1.VolumeSnapshotList, error) { + v1Snapshots := &snapshotv1.VolumeSnapshotList{} + snapshotBytes, err := json.Marshal(snapshots) + if err != nil { + return nil, err + } + if err := json.Unmarshal(snapshotBytes, v1Snapshots); err != nil { + return nil, err + } + + return v1Snapshots, nil +} diff --git a/internal/controllerutil/volumesnapshot_test.go b/internal/controllerutil/volumesnapshot_test.go new file mode 100644 index 000000000..d81c68555 --- /dev/null +++ b/internal/controllerutil/volumesnapshot_test.go @@ -0,0 +1,94 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package controllerutil + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" + "github.com/spf13/viper" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + rtclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("VolumeSnapshot compat client", func() { + const snapName = "test-volumesnapshot-name" + + var ( + pvcName = "test-pvc-name" + snapClassName = "test-vsc-name" + ) + + viper.SetDefault("VOLUMESNAPSHOT", "true") + viper.SetDefault("VOLUMESNAPSHOT_API_BETA", "true") + + It("test create/get/list/patch/delete", func() { + compatClient := VolumeSnapshotCompatClient{Client: k8sClient, Ctx: ctx} + snap := &snapshotv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: snapName, + Namespace: "default", + }, + Spec: snapshotv1.VolumeSnapshotSpec{ + Source: snapshotv1.VolumeSnapshotSource{ + PersistentVolumeClaimName: &pvcName, + }, + }, + } + snapKey := rtclient.ObjectKeyFromObject(snap) + snapGet := &snapshotv1.VolumeSnapshot{} + + By("create volumesnapshot") + // check object not found + exists, err := compatClient.CheckResourceExists(snapKey, snapGet) + Expect(err).Should(BeNil()) + Expect(exists).Should(BeFalse()) + // create + Expect(compatClient.Create(snap)).Should(Succeed()) + // check object exists + exists, err = compatClient.CheckResourceExists(snapKey, snapGet) + Expect(err).Should(BeNil()) + Expect(exists).Should(BeTrue()) + + By("get volumesnapshot") + Expect(compatClient.Get(snapKey, snapGet)).Should(Succeed()) + Expect(snapKey.Name).Should(Equal(snapName)) + + By("list volumesnapshots") + snapList := &snapshotv1.VolumeSnapshotList{} + Expect(compatClient.List(snapList)).Should(Succeed()) + Expect(snapList.Items).ShouldNot(BeEmpty()) + + By("patch volumesnapshot") + snapPatch := snap.DeepCopy() + snap.Spec.VolumeSnapshotClassName = &snapClassName + Expect(compatClient.Patch(snap, snapPatch)).Should(Succeed()) + Expect(compatClient.Get(snapKey, snapGet)).Should(Succeed()) + Expect(*snapGet.Spec.VolumeSnapshotClassName).Should(Equal(snapClassName)) + + By("delete volumesnapshot") + Expect(compatClient.Delete(snap)).Should(Succeed()) + Eventually(func() error { + return compatClient.Get(snapKey, snapGet) + }).Should(Satisfy(apierrors.IsNotFound)) + }) +}) From 0d2e923e081b0e8e749dbb11b259d08ff827e1a5 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Mon, 24 Apr 2023 17:54:11 +0800 Subject: [PATCH 163/439] fix: cli ops command output lost ops name (#2904) --- internal/cli/create/create.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cli/create/create.go b/internal/cli/create/create.go index 15d3ac586..cbf3669e4 100755 --- a/internal/cli/create/create.go +++ b/internal/cli/create/create.go @@ -263,14 +263,14 @@ func (o *BaseOptions) Run(inputs Inputs) error { return err } if dryRunStrategy != DryRunServer { - o.Name = unstructuredObj.GetName() + o.Name = previewObj.GetName() if o.Quiet { return nil } if inputs.CustomOutPut != nil { inputs.CustomOutPut(o) } else { - fmt.Fprintf(o.Out, "%s %s created\n", unstructuredObj.GetKind(), unstructuredObj.GetName()) + fmt.Fprintf(o.Out, "%s %s created\n", previewObj.GetKind(), previewObj.GetName()) } return nil } From 20d8d31ca8c55ddba1bf15afe476d969701d57e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BB=8A?= <39260018+kizuna-lek@users.noreply.github.com> Date: Mon, 24 Apr 2023 20:01:34 +0800 Subject: [PATCH 164/439] fix: add secure sql type (#2656) --- cmd/probe/internal/binding/mysql/mysql.go | 47 +++++++++++++++++-- .../internal/binding/mysql/mysql_test.go | 6 ++- cmd/probe/internal/binding/types.go | 8 ++++ go.mod | 2 +- 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/cmd/probe/internal/binding/mysql/mysql.go b/cmd/probe/internal/binding/mysql/mysql.go index b14b63e6f..463d8eb8c 100644 --- a/cmd/probe/internal/binding/mysql/mysql.go +++ b/cmd/probe/internal/binding/mysql/mysql.go @@ -276,15 +276,39 @@ func (mysqlOps *MysqlOperations) ExecOps(ctx context.Context, req *bindings.Invo func (mysqlOps *MysqlOperations) GetLagOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { result := OpsResult{} + slaveStatus := make([]SlaveStatus, 0) + var err error + + if mysqlOps.OriRole == "" { + mysqlOps.OriRole, err = mysqlOps.GetRole(ctx, req, resp) + if err != nil { + result["event"] = OperationFailed + result["message"] = err.Error() + return result, nil + } + } + if mysqlOps.OriRole == LEADER { + result["event"] = OperationSuccess + result["lag"] = 0 + result["message"] = "This is leader instance, leader has no lag" + return result, nil + } + sql := "show slave status" - _, err := mysqlOps.query(ctx, sql) + data, err := mysqlOps.query(ctx, sql) if err != nil { mysqlOps.Logger.Infof("GetLagOps error: %v", err) result["event"] = OperationFailed result["message"] = err.Error() } else { - result["event"] = OperationSuccess - result["lag"] = 0 + err = json.Unmarshal(data, &slaveStatus) + if err != nil { + result["event"] = OperationFailed + result["message"] = err.Error() + } else { + result["event"] = OperationSuccess + result["lag"] = slaveStatus[0].SecondsBehindMaster + } } return result, nil } @@ -459,7 +483,22 @@ func prepareValues(columnTypes []*sql.ColumnType) []interface{} { } values := make([]interface{}, len(columnTypes)) for i := range values { - values[i] = reflect.New(types[i]).Interface() + switch types[i].Kind() { + case reflect.String, reflect.Interface: + values[i] = &sql.NullString{} + case reflect.Bool: + values[i] = &sql.NullBool{} + case reflect.Float64: + values[i] = &sql.NullFloat64{} + case reflect.Int16, reflect.Uint16: + values[i] = &sql.NullInt16{} + case reflect.Int32, reflect.Uint32: + values[i] = &sql.NullInt32{} + case reflect.Int64, reflect.Uint64: + values[i] = &sql.NullInt64{} + default: + values[i] = reflect.New(types[i]).Interface() + } } return values } diff --git a/cmd/probe/internal/binding/mysql/mysql_test.go b/cmd/probe/internal/binding/mysql/mysql_test.go index f8fb3fab6..c7ce71952 100644 --- a/cmd/probe/internal/binding/mysql/mysql_test.go +++ b/cmd/probe/internal/binding/mysql/mysql_test.go @@ -166,6 +166,10 @@ func TestGetLagOps(t *testing.T) { col2 := sqlmock.NewColumn("ROLE").OfType("VARCHAR", "") col3 := sqlmock.NewColumn("SERVER_ID").OfType("INT", 0) rows := sqlmock.NewRowsWithColumnDefinition(col1, col2, col3).AddRow("wesql-main-1.wesql-main-headless:13306", "Follower", 1) + getRoleRows := sqlmock.NewRowsWithColumnDefinition(col1, col2, col3).AddRow("wesql-main-1.wesql-main-headless:13306", "Follower", 1) + if mysqlOps.OriRole == "" { + mock.ExpectQuery("select .* from information_schema.wesql_cluster_local").WillReturnRows(getRoleRows) + } mock.ExpectQuery("show slave status").WillReturnRows(rows) result, err := mysqlOps.GetLagOps(context.Background(), req, &bindings.InvokeResponse{}) @@ -363,7 +367,7 @@ func TestQuery(t *testing.T) { ret, err := mysqlOps.query(context.Background(), `SELECT * FROM foo WHERE id < 4`) assert.Nil(t, err) t.Logf("query result: %s", ret) - assert.Contains(t, string(ret), "\"id\":1") + assert.Contains(t, string(ret), "\"id\":\"1") var result []interface{} err = json.Unmarshal(ret, &result) assert.Nil(t, err) diff --git a/cmd/probe/internal/binding/types.go b/cmd/probe/internal/binding/types.go index 15d6ed1a4..edf458d06 100644 --- a/cmd/probe/internal/binding/types.go +++ b/cmd/probe/internal/binding/types.go @@ -69,6 +69,10 @@ const ( SECONDARY = "secondary" MASTER = "master" SLAVE = "slave" + LEADER = "Leader" + FOLLOWER = "Follower" + LEARNER = "Learner" + CANDIDATE = "Candidate" ) type RoleType string @@ -118,3 +122,7 @@ var ( ErrInvalidRoleName = fmt.Errorf(errMsgInvalidRoleName) ErrNoSuchUser = fmt.Errorf(errMsgNoSuchUser) ) + +type SlaveStatus struct { + SecondsBehindMaster int64 `json:"Seconds_Behind_Master"` +} diff --git a/go.mod b/go.mod index 94496b182..a1cf36398 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,6 @@ require ( github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 github.com/onsi/ginkgo/v2 v2.7.0 github.com/onsi/gomega v1.25.0 - github.com/opencontainers/image-spec v1.1.0-rc2 github.com/pingcap/go-tpc v1.0.9 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 @@ -272,6 +271,7 @@ require ( github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opencontainers/selinux v1.10.2 // indirect From 65292642241299821accf5deb343ff260521b217 Mon Sep 17 00:00:00 2001 From: xingran Date: Mon, 24 Apr 2023 20:04:07 +0800 Subject: [PATCH 165/439] fix: remove ownerReference when read cache snapshot to get all resource (#2911) --- .../controller/lifecycle/transform_utils.go | 52 ++----------------- 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/internal/controller/lifecycle/transform_utils.go b/internal/controller/lifecycle/transform_utils.go index 185b3a247..f0ceae12e 100644 --- a/internal/controller/lifecycle/transform_utils.go +++ b/internal/controller/lifecycle/transform_utils.go @@ -28,9 +28,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -86,43 +84,6 @@ func getGVKName(object client.Object, scheme *runtime.Scheme) (*gvkName, error) }, nil } -func isOwnerOf(owner, obj client.Object, scheme *runtime.Scheme) bool { - ro, ok := owner.(runtime.Object) - if !ok { - return false - } - gvk, err := apiutil.GVKForObject(ro, scheme) - if err != nil { - return false - } - ref := metav1.OwnerReference{ - APIVersion: gvk.GroupVersion().String(), - Kind: gvk.Kind, - UID: owner.GetUID(), - Name: owner.GetName(), - } - owners := obj.GetOwnerReferences() - referSameObject := func(a, b metav1.OwnerReference) bool { - aGV, err := schema.ParseGroupVersion(a.APIVersion) - if err != nil { - return false - } - - bGV, err := schema.ParseGroupVersion(b.APIVersion) - if err != nil { - return false - } - - return aGV.Group == bGV.Group && a.Kind == b.Kind && a.Name == b.Name - } - for _, ownerRef := range owners { - if referSameObject(ownerRef, ref) { - return true - } - } - return false -} - func actionPtr(action Action) *Action { return &action } @@ -217,16 +178,11 @@ func readCacheSnapshot(transCtx *ClusterTransformContext, cluster appsv1alpha1.C for i := 0; i < l; i++ { // get the underlying object object := items.Index(i).Addr().Interface().(client.Object) - // put to snapshot if owned by our cluster - // pvcs created by sts don't have cluster in ownerReferences - _, isPVC := object.(*corev1.PersistentVolumeClaim) - if isPVC || isOwnerOf(&cluster, object, scheme) { - name, err := getGVKName(object, scheme) - if err != nil { - return nil, err - } - snapshot[*name] = object + name, err := getGVKName(object, scheme) + if err != nil { + return nil, err } + snapshot[*name] = object } } From a6d233ca28e3cbec9d6eefbd58e87b70549e9de2 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Mon, 24 Apr 2023 20:59:36 +0800 Subject: [PATCH 166/439] chore: remove log volume in class definition for apecloud-mysql (#2900) --- deploy/apecloud-mysql/templates/class.yaml | 68 ++++++++++------------ internal/cli/testing/testdata/class.yaml | 68 ++++++++++------------ 2 files changed, 64 insertions(+), 72 deletions(-) diff --git a/deploy/apecloud-mysql/templates/class.yaml b/deploy/apecloud-mysql/templates/class.yaml index d6bb20db7..8563aa074 100644 --- a/deploy/apecloud-mysql/templates/class.yaml +++ b/deploy/apecloud-mysql/templates/class.yaml @@ -10,54 +10,50 @@ spec: groups: - resourceConstraintRef: kb-resource-constraint-general template: | - cpu: {{ printf "{{ or .cpu 1 }}" }} - memory: {{ printf "{{ or .memory 4 }}Gi" }} + cpu: {{ printf "{{ .cpu }}" }} + memory: {{ printf "{{ .memory }}Gi" }} volumes: - name: data - size: {{ printf "{{ or .dataStorageSize 10 }}Gi" }} - - name: log - size: {{ printf "{{ or .logStorageSize 1 }}Gi" }} - vars: [ cpu, memory, dataStorageSize, logStorageSize ] + size: {{ printf "{{ .dataStorageSize }}Gi" }} + vars: [ cpu, memory, dataStorageSize ] series: - namingTemplate: {{ printf "general-{{ .cpu }}c{{ .memory }}g" }} classes: - - args: [ "0.5", "0.5", "20", "1" ] - - args: [ "1", "1", "20", "1" ] - - args: [ "2", "2", "20", "1" ] - - args: [ "2", "4", "20", "1" ] - - args: [ "2", "8", "20", "1" ] - - args: [ "4", "16", "20", "1" ] - - args: [ "8", "32", "20", "1" ] - - args: [ "16", "64", "20", "1" ] - - args: [ "32", "128", "20", "1" ] - - args: [ "64", "256", "20", "1" ] - - args: [ "128", "512", "20", "1" ] + - args: [ "0.5", "0.5", "20" ] + - args: [ "1", "1", "20" ] + - args: [ "2", "2", "20" ] + - args: [ "2", "4", "20" ] + - args: [ "2", "8", "20" ] + - args: [ "4", "16", "20" ] + - args: [ "8", "32", "20" ] + - args: [ "16", "64", "20" ] + - args: [ "32", "128", "20" ] + - args: [ "64", "256", "20" ] + - args: [ "128", "512", "20" ] - resourceConstraintRef: kb-resource-constraint-memory-optimized template: | - cpu: {{ printf "{{ or .cpu 1 }}" }} - memory: {{ printf "{{ or .memory 8 }}Gi" }} + cpu: {{ printf "{{ .cpu }}" }} + memory: {{ printf "{{ .memory }}Gi" }} volumes: - name: data - size: {{ printf "{{ or .dataStorageSize 10 }}Gi" }} - - name: log - size: {{ printf "{{ or .logStorageSize 1 }}Gi" }} - vars: [ cpu, memory, dataStorageSize, logStorageSize ] + size: {{ printf "{{ .dataStorageSize }}Gi" }} + vars: [ cpu, memory, dataStorageSize ] series: - namingTemplate: {{ printf "mo-{{ .cpu }}c{{ .memory }}g" }} classes: # 1:8 - - args: [ "2", "16", "20", "1" ] - - args: [ "4", "32", "20", "1" ] - - args: [ "8", "64", "20", "1" ] - - args: [ "12", "96", "20", "1" ] - - args: [ "24", "192", "20", "1" ] - - args: [ "48", "384", "20", "1" ] + - args: [ "2", "16", "20" ] + - args: [ "4", "32", "20" ] + - args: [ "8", "64", "20" ] + - args: [ "12", "96", "20" ] + - args: [ "24", "192", "20" ] + - args: [ "48", "384", "20" ] # 1:16 - - args: [ "2", "32", "20", "1" ] - - args: [ "4", "64", "20", "1" ] - - args: [ "8", "128", "20", "1" ] - - args: [ "16", "256", "20", "1" ] - - args: [ "32", "512", "20", "1" ] - - args: [ "48", "768", "20", "1" ] - - args: [ "64", "1024", "20", "1" ] + - args: [ "2", "32", "20" ] + - args: [ "4", "64", "20" ] + - args: [ "8", "128", "20" ] + - args: [ "16", "256", "20" ] + - args: [ "32", "512", "20" ] + - args: [ "48", "768", "20" ] + - args: [ "64", "1024", "20" ] diff --git a/internal/cli/testing/testdata/class.yaml b/internal/cli/testing/testdata/class.yaml index e3b65a898..6e3592eda 100644 --- a/internal/cli/testing/testdata/class.yaml +++ b/internal/cli/testing/testdata/class.yaml @@ -12,15 +12,13 @@ spec: resourceConstraintRef: kb-resource-constraint-general # class schema template, you can set default resource values here template: | - cpu: "{{ or .cpu 1 }}" - memory: "{{ or .memory 4 }}Gi" + cpu: "{{ .cpu }}" + memory: "{{ .memory }}Gi" volumes: - name: data - size: "{{ or .dataStorageSize 10 }}Gi" - - name: log - size: "{{ or .logStorageSize 1 }}Gi" + size: "{{ .dataStorageSize }}Gi" # class schema template variables - vars: [ cpu, memory, dataStorageSize, logStorageSize ] + vars: [ cpu, memory, dataStorageSize] series: - # class name generator, you can reference variables in class schema template # it's also ok to define static class name in following class definitions @@ -30,41 +28,39 @@ spec: # 1. define arguments for class schema variables, class schema will be dynamically generated # 2. statically define complete class schema classes: - - args: [ "1", "1", "10", "1" ] - - args: [ "2", "2", "10", "1" ] - - args: [ "2", "4", "10", "1" ] - - args: [ "2", "8", "10", "1" ] - - args: [ "4", "16", "10", "1" ] - - args: [ "8", "32", "10", "1" ] - - args: [ "16", "64", "20", "1" ] - - args: [ "32", "128", "20", "1" ] - - args: [ "64", "256", "20", "1" ] - - args: [ "128", "512", "20", "1" ] + - args: [ "1", "1", "10" ] + - args: [ "2", "2", "10" ] + - args: [ "2", "4", "10" ] + - args: [ "2", "8", "10" ] + - args: [ "4", "16", "10" ] + - args: [ "8", "32", "10" ] + - args: [ "16", "64", "20" ] + - args: [ "32", "128", "20" ] + - args: [ "64", "256", "20" ] + - args: [ "128", "512", "20" ] - resourceConstraintRef: kb-resource-constraint-memory-optimized template: | - cpu: "{{ or .cpu 1 }}" - memory: "{{ or .memory 8 }}Gi" + cpu: "{{ .cpu }}" + memory: "{{ .memory }}Gi" volumes: - name: data - size: "{{ or .dataStorageSize 10 }}Gi" - - name: log - size: "{{ or .logStorageSize 1 }}Gi" - vars: [ cpu, memory, dataStorageSize, logStorageSize ] + size: "{{ .dataStorageSize }}Gi" + vars: [ cpu, memory, dataStorageSize] series: - namingTemplate: "mo-{{ .cpu }}c{{ .memory }}g" classes: - - args: [ "2", "16", "10", "1" ] - - args: [ "4", "32", "10", "1" ] - - args: [ "8", "64", "10", "1" ] - - args: [ "12", "96", "10", "1" ] - - args: [ "24", "192", "20", "1" ] - - args: [ "48", "384", "20", "1" ] - - args: [ "2", "32", "10", "1" ] - - args: [ "4", "64", "10", "1" ] - - args: [ "8", "128", "10", "1" ] - - args: [ "16", "256", "10", "1" ] - - args: [ "32", "512", "20", "1" ] - - args: [ "48", "768", "20", "1" ] - - args: [ "64", "1024", "20", "1" ] - - args: [ "128", "2048", "20", "1" ] \ No newline at end of file + - args: [ "2", "16", "10" ] + - args: [ "4", "32", "10" ] + - args: [ "8", "64", "10" ] + - args: [ "12", "96", "10" ] + - args: [ "24", "192", "20" ] + - args: [ "48", "384", "20" ] + - args: [ "2", "32", "10" ] + - args: [ "4", "64", "10" ] + - args: [ "8", "128", "10" ] + - args: [ "16", "256", "10" ] + - args: [ "32", "512", "20" ] + - args: [ "48", "768", "20" ] + - args: [ "64", "1024", "20" ] + - args: [ "128", "2048", "20" ] \ No newline at end of file From 30e6f34beac98f9a1ba993b618d3b3b8366d8b77 Mon Sep 17 00:00:00 2001 From: linghan-hub <56351212+linghan-hub@users.noreply.github.com> Date: Tue, 25 Apr 2023 10:29:36 +0800 Subject: [PATCH 167/439] chore: adjust execute yaml apply to create (#2893) --- Makefile | 6 +- deploy/apecloud-mysql-cluster/Chart.yaml | 2 +- .../apecloud-mysql-scale-cluster/Chart.yaml | 2 +- deploy/apecloud-mysql-scale/Chart.yaml | 2 +- deploy/apecloud-mysql/Chart.yaml | 2 +- deploy/chatgpt-retrieval-plugin/Chart.yaml | 2 +- deploy/clickhouse-cluster/Chart.yaml | 2 +- deploy/clickhouse/Chart.yaml | 2 +- deploy/helm/Chart.yaml | 4 +- deploy/kafka-cluster/Chart.yaml | 2 +- deploy/kafka/Chart.yaml | 2 +- deploy/milvus-cluster/Chart.yaml | 2 +- deploy/milvus/Chart.yaml | 2 +- deploy/mongodb-cluster/Chart.yaml | 2 +- deploy/mongodb/Chart.yaml | 2 +- deploy/nyancat/Chart.yaml | 4 +- deploy/postgresql-cluster/Chart.yaml | 2 +- deploy/postgresql/Chart.yaml | 2 +- deploy/qdrant-cluster/Chart.yaml | 2 +- deploy/qdrant/Chart.yaml | 2 +- deploy/redis-cluster/Chart.yaml | 2 +- deploy/redis/Chart.yaml | 2 +- deploy/weaviate-cluster/Chart.yaml | 2 +- deploy/weaviate/Chart.yaml | 2 +- .../smoketest/mongodb/00_mongodbcluster.yaml | 683 +---------------- test/e2e/testdata/smoketest/playgroundtest.go | 11 +- .../postgresql/00_postgresqlcluster.yaml | 2 +- .../testdata/smoketest/postgresql/07_cv.yaml | 3 + .../smoketest/redis/00_rediscluster.yaml | 715 +----------------- test/e2e/testdata/smoketest/smoketestrun.go | 2 +- .../smoketest/wesql/00_wesqlcluster.yaml | 2 +- .../wesql/14_backup_snapshot_restore.yaml | 6 +- 32 files changed, 43 insertions(+), 1437 deletions(-) diff --git a/Makefile b/Makefile index 84d3e7a35..b1bff4be6 100644 --- a/Makefile +++ b/Makefile @@ -752,10 +752,8 @@ endif render-smoke-testdata-manifests: ## Update E2E test dataset $(HELM) template mycluster deploy/apecloud-mysql-cluster > test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml $(HELM) template mycluster deploy/postgresql-cluster > test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml - $(HELM) template mycluster deploy/redis > test/e2e/testdata/smoketest/redis/00_rediscluster.yaml - $(HELM) template mycluster deploy/redis-cluster >> test/e2e/testdata/smoketest/redis/00_rediscluster.yaml - $(HELM) template mycluster deploy/mongodb > test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml - $(HELM) template mycluster deploy/mongodb-cluster >> test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml + $(HELM) template mycluster deploy/redis-cluster > test/e2e/testdata/smoketest/redis/00_rediscluster.yaml + $(HELM) template mycluster deploy/mongodb-cluster > test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml .PHONY: test-e2e diff --git a/deploy/apecloud-mysql-cluster/Chart.yaml b/deploy/apecloud-mysql-cluster/Chart.yaml index 8bfb608a6..95a4cfe78 100644 --- a/deploy/apecloud-mysql-cluster/Chart.yaml +++ b/deploy/apecloud-mysql-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: An ApeCloud MySQL Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "8.0.30" diff --git a/deploy/apecloud-mysql-scale-cluster/Chart.yaml b/deploy/apecloud-mysql-scale-cluster/Chart.yaml index fb76be8cf..bedab4747 100644 --- a/deploy/apecloud-mysql-scale-cluster/Chart.yaml +++ b/deploy/apecloud-mysql-scale-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: An ApeCloud MySQL-Scale Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/deploy/apecloud-mysql-scale/Chart.yaml b/deploy/apecloud-mysql-scale/Chart.yaml index 793b37ff2..721f1ece0 100644 --- a/deploy/apecloud-mysql-scale/Chart.yaml +++ b/deploy/apecloud-mysql-scale/Chart.yaml @@ -5,7 +5,7 @@ description: ApeCloud MySQL-Scale is ApeCloud MySQL proxy. ApeCloud MySQL-Scale type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/deploy/apecloud-mysql/Chart.yaml b/deploy/apecloud-mysql/Chart.yaml index ce9ec5be7..314fb48b5 100644 --- a/deploy/apecloud-mysql/Chart.yaml +++ b/deploy/apecloud-mysql/Chart.yaml @@ -9,7 +9,7 @@ description: ApeCloud MySQL is fully compatible with MySQL syntax and supports s type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "8.0.30" diff --git a/deploy/chatgpt-retrieval-plugin/Chart.yaml b/deploy/chatgpt-retrieval-plugin/Chart.yaml index 054472a09..e855405f2 100644 --- a/deploy/chatgpt-retrieval-plugin/Chart.yaml +++ b/deploy/chatgpt-retrieval-plugin/Chart.yaml @@ -5,7 +5,7 @@ description: A demo application for ChatGPT plugin. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: 0.1.0 diff --git a/deploy/clickhouse-cluster/Chart.yaml b/deploy/clickhouse-cluster/Chart.yaml index b86db763f..cb5567b1e 100644 --- a/deploy/clickhouse-cluster/Chart.yaml +++ b/deploy/clickhouse-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A ClickHouse cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: 22.9.4 diff --git a/deploy/clickhouse/Chart.yaml b/deploy/clickhouse/Chart.yaml index c67a16ef2..afa7f84e4 100644 --- a/deploy/clickhouse/Chart.yaml +++ b/deploy/clickhouse/Chart.yaml @@ -9,7 +9,7 @@ annotations: type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: 22.9.4 diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml index 751fbdb3f..d1c5b870f 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.5.0-beta.2 +appVersion: 0.5.0-beta.9 kubeVersion: '>=1.22.0-0' diff --git a/deploy/kafka-cluster/Chart.yaml b/deploy/kafka-cluster/Chart.yaml index 634fd6623..83aebacae 100644 --- a/deploy/kafka-cluster/Chart.yaml +++ b/deploy/kafka-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A Kafka server cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: 3.4.0 diff --git a/deploy/kafka/Chart.yaml b/deploy/kafka/Chart.yaml index 481ce2d98..17ddc92b2 100644 --- a/deploy/kafka/Chart.yaml +++ b/deploy/kafka/Chart.yaml @@ -11,7 +11,7 @@ annotations: type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: 3.4.0 diff --git a/deploy/milvus-cluster/Chart.yaml b/deploy/milvus-cluster/Chart.yaml index ca05710fa..267644d67 100644 --- a/deploy/milvus-cluster/Chart.yaml +++ b/deploy/milvus-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A Milvus cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "2.2.4" diff --git a/deploy/milvus/Chart.yaml b/deploy/milvus/Chart.yaml index 0e264027c..bc6264496 100644 --- a/deploy/milvus/Chart.yaml +++ b/deploy/milvus/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 # This is the version number of milvus appVersion: "2.2.4" diff --git a/deploy/mongodb-cluster/Chart.yaml b/deploy/mongodb-cluster/Chart.yaml index 5485a468e..2f908d459 100644 --- a/deploy/mongodb-cluster/Chart.yaml +++ b/deploy/mongodb-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A MongoDB cluster Helm chart for KubeBlocks type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "5.0.14" diff --git a/deploy/mongodb/Chart.yaml b/deploy/mongodb/Chart.yaml index 6eed23d60..3a887cb7e 100644 --- a/deploy/mongodb/Chart.yaml +++ b/deploy/mongodb/Chart.yaml @@ -4,7 +4,7 @@ description: MongoDB is a document database designed for ease of application dev type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "5.0.14" diff --git a/deploy/nyancat/Chart.yaml b/deploy/nyancat/Chart.yaml index a9774cec9..40baa823d 100644 --- a/deploy/nyancat/Chart.yaml +++ b/deploy/nyancat/Chart.yaml @@ -4,8 +4,8 @@ description: A demo application for showing database cluster availability. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 -appVersion: 0.5.0-beta.2 +appVersion: 0.5.0-beta.9 kubeVersion: '>=1.22.0-0' diff --git a/deploy/postgresql-cluster/Chart.yaml b/deploy/postgresql-cluster/Chart.yaml index 51f863120..6f87813f8 100644 --- a/deploy/postgresql-cluster/Chart.yaml +++ b/deploy/postgresql-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A PostgreSQL (with Patroni HA) cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "14.7.0" diff --git a/deploy/postgresql/Chart.yaml b/deploy/postgresql/Chart.yaml index 997434337..ae13d0126 100644 --- a/deploy/postgresql/Chart.yaml +++ b/deploy/postgresql/Chart.yaml @@ -4,7 +4,7 @@ description: A PostgreSQL (with Patroni HA) cluster definition Helm chart for Ku type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "14.7.0" diff --git a/deploy/qdrant-cluster/Chart.yaml b/deploy/qdrant-cluster/Chart.yaml index 75ec59a5a..82cf28579 100644 --- a/deploy/qdrant-cluster/Chart.yaml +++ b/deploy/qdrant-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A Qdrant cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "1.1.0" diff --git a/deploy/qdrant/Chart.yaml b/deploy/qdrant/Chart.yaml index 1cb33f31b..38fb257eb 100644 --- a/deploy/qdrant/Chart.yaml +++ b/deploy/qdrant/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version. -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 # This is the version number of qdrant. appVersion: "1.1.0" diff --git a/deploy/redis-cluster/Chart.yaml b/deploy/redis-cluster/Chart.yaml index 79eeb886b..90dbafa30 100644 --- a/deploy/redis-cluster/Chart.yaml +++ b/deploy/redis-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: An Redis Replication Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "7.0.6" diff --git a/deploy/redis/Chart.yaml b/deploy/redis/Chart.yaml index 438d3da4c..1eaf19bee 100644 --- a/deploy/redis/Chart.yaml +++ b/deploy/redis/Chart.yaml @@ -4,7 +4,7 @@ description: A Redis cluster definition Helm chart for Kubernetes type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "7.0.6" diff --git a/deploy/weaviate-cluster/Chart.yaml b/deploy/weaviate-cluster/Chart.yaml index 052298f7b..6ef6c4ac4 100644 --- a/deploy/weaviate-cluster/Chart.yaml +++ b/deploy/weaviate-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A weaviate cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 appVersion: "1.18.0" diff --git a/deploy/weaviate/Chart.yaml b/deploy/weaviate/Chart.yaml index 02368fba3..26790e7b9 100644 --- a/deploy/weaviate/Chart.yaml +++ b/deploy/weaviate/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version. -version: 0.5.0-beta.2 +version: 0.5.0-beta.9 # This is the version number of weaviate. appVersion: "1.18.0" diff --git a/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml b/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml index 5e62f3bbf..9a99a9c2e 100644 --- a/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml +++ b/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml @@ -1,692 +1,11 @@ --- -# Source: mongodb/templates/configtemplate.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: mongodb5.0-config-template - labels: - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -data: - mongodb.conf: |- - # mongod.conf - # for documentation of all options, see: - # http://docs.mongodb.org/manual/reference/configuration-options/ - - {{- $log_root := getVolumePathByName ( index $.podSpec.containers 0 ) "log" }} - {{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} - {{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} - {{- $phy_memory := getContainerMemory ( index $.podSpec.containers 0 ) }} - - # require port - {{- $mongodb_port := 27017 }} - {{- if $mongodb_port_info }} - {{- $mongodb_port = $mongodb_port_info.containerPort }} - {{- end }} - - # where and how to store data. - storage: - dbPath: {{ $mongodb_root }}/db - journal: - enabled: true - directoryPerDB: true - - # where to write logging data. - {{ block "logsBlock" . }} - systemLog: - destination: file - quiet: false - logAppend: true - logRotate: reopen - path: /data/mongodb/logs/mongodb.log - verbosity: 0 - {{ end }} - - # network interfaces - net: - port: {{ $mongodb_port }} - unixDomainSocket: - enabled: false - pathPrefix: {{ $mongodb_root }}/tmp - ipv6: false - bindIpAll: true - #bindIp: - - # replica set options - replication: - replSetName: replicaset - enableMajorityReadConcern: true - - # sharding options - #sharding: - #clusterRole: - - # process management options - processManagement: - fork: false - pidFilePath: {{ $mongodb_root }}/tmp/mongodb.pid - - # set parameter options - setParameter: - enableLocalhostAuthBypass: true - - # security options - security: - authorization: enabled - keyFile: /etc/mongodb/keyfile - - keyfile: |- - {{ randAscii 64 | b64enc }} ---- -# Source: mongodb/templates/metrics-configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: mongodb-metrics-config - labels: - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -data: - metrics-config.yaml: "exporters:\n prometheus:\n const_labels: []\n enable_open_metrics: false\n endpoint: 0.0.0.0:9216\n metric_expiration: 30s\n resource_to_telemetry_conversion:\n enabled: true\n send_timestamps: false\nextensions:\n health_check:\n check_collector_pipeline:\n enabled: true\n exporter_failure_threshold: 5\n interval: 2m\n endpoint: 0.0.0.0:13133\n path: /health/status\n memory_ballast:\n size_mib: 512\nprocessors:\n batch:\n timeout: 5s\n memory_limiter:\n check_interval: 10s\n limit_mib: 1024\n spike_limit_mib: 256\nreceivers:\n apecloudmongodb:\n collect-all: true\n collection_interval: 15s\n compatible-mode: true\n direct-connect: true\n global-conn-pool: false\n log-level: info\n uri: mongodb://${env:MONGODB_ROOT_USER}:${env:MONGODB_ROOT_PASSWORD}@127.0.0.1:27017/admin?ssl=false&authSource=admin\nservice:\n extensions:\n - memory_ballast\n - health_check\n pipelines:\n metrics:\n exporters:\n - prometheus\n processors:\n - memory_limiter\n receivers:\n - apecloudmongodb\n telemetry:\n logs:\n level: info\n metrics:\n address: 0.0.0.0:8888" ---- -# Source: mongodb/templates/scriptstemplate.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: mongodb-scripts - labels: - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -data: - mongos-setup.sh: |- - #!/bin/sh - - PORT=27018 - CONFIG_SVR_NAME=$KB_CLUSTER_NAME"-configsvr" - DOMAIN=$CONFIG_SVR_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" - mongos --bind_ip_all --configdb $CONFIG_SVR_NAME/$CONFIG_SVR_NAME"-0."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-1."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-2."$DOMAIN:$PORT - replicaset-setup.sh: |- - #!/bin/sh - - {{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} - {{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} - - # require port - {{- $mongodb_port := 27017 }} - {{- if $mongodb_port_info }} - {{- $mongodb_port = $mongodb_port_info.containerPort }} - {{- end }} - - PORT={{ $mongodb_port }} - MONGODB_ROOT={{ $mongodb_root }} - RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); - RPL_SET_NAME=${RPL_SET_NAME%-}; - mkdir -p $MONGODB_ROOT/db - mkdir -p $MONGODB_ROOT/logs - mkdir -p $MONGODB_ROOT/tmp - MODE=$1 - mongod $MODE --bind_ip_all --port $PORT --replSet $RPL_SET_NAME --config /etc/mongodb/mongodb.conf - - replicaset-post-start.sh: |- - #!/bin/sh - # usage: replicaset-post-start.sh type_name is_configsvr - # type_name: component.type, in uppercase - # is_configsvr: true or false, default false - {{- $mongodb_root := getVolumePathByName ( index $.podSpec.containers 0 ) "data" }} - {{- $mongodb_port_info := getPortByName ( index $.podSpec.containers 0 ) "mongodb" }} - - # require port - {{- $mongodb_port := 27017 }} - {{- if $mongodb_port_info }} - {{- $mongodb_port = $mongodb_port_info.containerPort }} - {{- end }} - - PORT={{ $mongodb_port }} - MONGODB_ROOT={{ $mongodb_root }} - INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); - INDEX=${INDEX#-}; - if [ $INDEX -ne 0 ]; then exit 0; fi - - until mongosh --quiet --port $PORT --eval "print('ready')"; do sleep 1; done - - RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); - RPL_SET_NAME=${RPL_SET_NAME%-}; - - TYPE_NAME=$1 - IS_CONFIGSVR=$2 - MEMBERS="" - i=0 - while [ $i -lt $(eval echo \$KB_"$TYPE_NAME"_N) ]; do - host=$(eval echo \$KB_"$TYPE_NAME"_"$i"_HOSTNAME) - host=$host"."$KB_NAMESPACE".svc.cluster.local" - until mongosh --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done - if [ $i -eq 0 ]; then - MEMBERS="{_id: $i, host: \"$host:$PORT\", priority:2}" - else - MEMBERS="$MEMBERS,{_id: $i, host: \"$host:$PORT\"}" - fi - i=$(( i + 1)) - done - CONFIGSVR="" - if [ ""$IS_CONFIGSVR = "true" ]; then CONFIGSVR="configsvr: true,"; fi - - until is_inited=$(mongosh --quiet --port $PORT --eval "rs.status().ok" -u root --password $MONGODB_ROOT_PASSWORD || mongosh --quiet --port $PORT --eval "try { rs.status().ok } catch (e) { 0 }") ; do sleep 1; done - if [ $is_inited -eq 1 ]; then - exit 0 - fi; - mongosh --quiet --port $PORT --eval "rs.initiate({_id: \"$RPL_SET_NAME\", $CONFIGSVR members: [$MEMBERS]})"; - - (until mongosh --quiet --port $PORT --eval "rs.isMaster().isWritablePrimary"|grep true; do sleep 1; done; - echo "create user"; - mongosh --quiet --port $PORT admin --eval "db.createUser({ user: \"$MONGODB_ROOT_USER\", pwd: \"$MONGODB_ROOT_PASSWORD\", roles: [{role: 'root', db: 'admin'}] })") /dev/null 2>&1 & - - shard-agent.sh: |- - #!/bin/sh - - INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); - INDEX=${INDEX#-}; - if [ $INDEX -ne 0 ]; then - trap : TERM INT; (while true; do sleep 1000; done) & wait - fi - - # wait main container ready - PORT=27018 - until mongosh --quiet --port $PORT --eval "rs.status().ok"; do sleep 1; done - # add shard to mongos - SHARD_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); - SHARD_NAME=${SHARD_NAME%-}; - DOMAIN=$SHARD_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" - MONGOS_HOST=$KB_CLUSTER_NAME"-mongos" - MONGOS_PORT=27017 - SHARD_CONFIG=$SHARD_NAME/$SHARD_NAME"-0."$DOMAIN:$PORT,$SHARD_NAME"-1."$DOMAIN:$PORT,$SHARD_NAME"-2."$DOMAIN:$PORT - until mongosh --quiet --host $MONGOS_HOST --port $MONGOS_PORT --eval "print('service is ready')"; do sleep 1; done - mongosh --quiet --host $MONGOS_HOST --port $MONGOS_PORT --eval "sh.addShard(\"$SHARD_CONFIG\")" - - trap : TERM INT; (while true; do sleep 1000; done) & wait ---- -# Source: mongodb/templates/backuppolicytemplate.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: BackupPolicyTemplate -metadata: - name: mongodb-backup-policy-template - labels: - clusterdefinition.kubeblocks.io/name: mongodb - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -spec: - clusterDefinitionRef: mongodb - backupPolicies: - - componentDefRef: replicaset - ttl: 7d - schedule: - baseBackup: - type: full - enable: false - cronExpression: "0 18 * * 0" - snapshot: - target: - role: primary - connectionCredentialKey: - passwordKey: password - usernameKey: username - full: - backupToolName: mongodb-physical-backup-tool - backupsHistoryLimit: 7 - target: - role: primary ---- -# Source: mongodb/templates/backuptool.yaml -apiVersion: dataprotection.kubeblocks.io/v1alpha1 -kind: BackupTool -metadata: - name: mongodb-physical-backup-tool - labels: - clusterdefinition.kubeblocks.io/name: mongodb -spec: - image: mongo:5.0.14 - deployKind: job - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 128Mi - env: - - name: DATA_DIR - value: /data/mongodb/db - physical: - restoreCommands: - - | - set -e - mkdir -p ${DATA_DIR} - res=`ls -A ${DATA_DIR}` - if [ ! -z "${res}" ]; then - echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." - exit 1 - fi - tar -xvf ${BACKUP_DIR}/${BACKUP_NAME}.tar.gz -C ${DATA_DIR}/../ - mv ${DATA_DIR}/../${BACKUP_NAME}/* ${DATA_DIR} - PORT=27017 - MONGODB_ROOT=/data/mongodb - RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); - RPL_SET_NAME=${RPL_SET_NAME%-}; - mkdir -p $MONGODB_ROOT/db - mkdir -p $MONGODB_ROOT/logs - mkdir -p $MONGODB_ROOT/tmp - MODE=$1 - mongod $MODE --bind_ip_all --port $PORT --dbpath $MONGODB_ROOT/db --directoryperdb --logpath $MONGODB_ROOT/logs/mongodb.log --logappend --pidfilepath $MONGODB_ROOT/tmp/mongodb.pid& - until mongosh --quiet --port $PORT --host $host --eval "print('peer is ready')"; do sleep 1; done - PID=`cat $MONGODB_ROOT/tmp/mongodb.pid` - - mongosh --quiet --port $PORT local --eval "db.system.replset.deleteOne({})" - mongosh --quiet --port $PORT local --eval "db.system.replset.find()" - mongosh --quiet --port $PORT admin --eval 'db.dropUser("root", {w: "majority", wtimeout: 4000})' || true - kill $PID - wait $PID - incrementalRestoreCommands: [] - logical: - restoreCommands: [] - incrementalRestoreCommands: [] - backupCommands: - - | - set -e - mkdir -p ${BACKUP_DIR}/${BACKUP_NAME} - cp -R ${DATA_DIR}/* ${BACKUP_DIR}/${BACKUP_NAME}/ - cd ${BACKUP_DIR} - tar -czvf ${BACKUP_NAME}.tar.gz ./${BACKUP_NAME} - rm -rf ${BACKUP_DIR}/${BACKUP_NAME} - incrementalBackupCommands: [] ---- -# Source: mongodb/templates/clusterdefinition.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClusterDefinition -metadata: - name: mongodb - labels: - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -spec: - type: mongodb - connectionCredential: - username: root - password: "$(RANDOM_PASSWD)" - endpoint: "$(SVC_FQDN):$(SVC_PORT_tcp-monogdb)" - host: "$(SVC_FQDN)" - port: "$(SVC_PORT_tcp-monogdb)" - headlessEndpoint: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN):$(SVC_PORT_tcp-monogdb)" - headlessHost: "$(POD_NAME_PREFIX)-0.$(HEADLESS_SVC_FQDN)" - headlessPort: "$(SVC_PORT_tcp-monogdb)" - componentDefs: - - name: replicaset - characterType: mongodb - scriptSpecs: - - name: mongodb-scripts - templateRef: mongodb-scripts - volumeName: scripts - namespace: default - defaultMode: 493 - configSpecs: - - name: mongodb-config - templateRef: mongodb5.0-config-template - namespace: default - volumeName: mongodb-config - constraintRef: mongodb-config-constraints - keys: - - mongodb.conf - defaultMode: 256 - - name: mongodb-metrics-config - templateRef: mongodb-metrics-config - namespace: default - volumeName: mongodb-metrics-config - defaultMode: 0777 - monitor: - builtIn: false - exporterConfig: - scrapePath: /metrics - scrapePort: 9216 - logConfigs: - - name: running - filePathPattern: /data/mongodb/logs/mongodb.log* - workloadType: Consensus - consensusSpec: - leader: - name: "primary" - accessMode: ReadWrite - followers: - - name: "secondary" - accessMode: Readonly - updateStrategy: Serial - probes: - roleProbe: - periodSeconds: 2 - failureThreshold: 3 - service: - ports: - - protocol: TCP - port: 27017 - volumeTypes: - - name: data - type: data - podSpec: - containers: - - name: mongodb - ports: - - name: mongodb - protocol: TCP - containerPort: 27017 - command: - - /scripts/replicaset-setup.sh - env: - - name: MONGODB_ROOT_USER - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: username - - name: MONGODB_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: password - lifecycle: - postStart: - exec: - command: - - /scripts/replicaset-post-start.sh - - REPLICASET - volumeMounts: - - mountPath: /data/mongodb - name: data - - mountPath: /etc/mongodb/mongodb.conf - name: mongodb-config - subPath: mongodb.conf - - mountPath: /etc/mongodb/keyfile - name: mongodb-config - subPath: keyfile - - name: scripts - mountPath: /scripts/replicaset-setup.sh - subPath: replicaset-setup.sh - - name: scripts - mountPath: /scripts/replicaset-post-start.sh - subPath: replicaset-post-start.sh - - name: metrics - image: registry.cn-hangzhou.aliyuncs.com/apecloud/agamotto:0.0.4 - imagePullPolicy: "IfNotPresent" - securityContext: - runAsNonRoot: true - runAsUser: 1001 - env: - - name: MONGODB_ROOT_USER - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: username - - name: MONGODB_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: password - command: - - "/bin/agamotto" - - "--config=/opt/conf/metrics-config.yaml" - ports: - - name: http-metrics - containerPort: 9216 - volumeMounts: - - name: mongodb-metrics-config - mountPath: /opt/conf ---- -# Source: mongodb/templates/sharding-clusterdefinition.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClusterDefinition -metadata: - name: mongodb-sharding - labels: - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -spec: - type: mongodb - connectionCredential: - username: root - password: "$(RANDOM_PASSWD)" - endpoint: "$(SVC_FQDN):$(SVC_PORT_tcp-monogdb)" - host: "$(SVC_FQDN)" - port: "$(SVC_PORT_tcp-monogdb)" - headlessEndpoint: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN):$(SVC_PORT_tcp-monogdb)" - headlessHost: "$(POD_NAME_PREFIX)-0.$(HEADLESS_SVC_FQDN)" - headlessPort: "$(SVC_PORT_tcp-monogdb)" - componentDefs: - - name: mongos - scriptSpecs: - - name: mongodb-scripts - templateRef: mongodb-scripts - volumeName: scripts - namespace: default - defaultMode: 493 - workloadType: Stateless - service: - ports: - - name: mongos - port: 27017 - targetPort: mongos - podSpec: - containers: - - name: mongos - ports: - - name: mongos - containerPort: 27017 - command: - - /scripts/mongos-setup.sh - volumeMounts: - - name: scripts - mountPath: /scripts/mongos-setup.sh - subPath: mongos-setup.sh - - name: configsvr - scriptSpecs: - - name: mongodb-scripts - templateRef: mongodb-scripts - volumeName: scripts - namespace: default - defaultMode: 493 - characterType: mongodb - workloadType: Consensus - consensusSpec: - leader: - name: "primary" - accessMode: ReadWrite - followers: - - name: "secondary" - accessMode: Readonly - updateStrategy: Serial - probes: - roleProbe: - periodSeconds: 2 - failureThreshold: 3 - service: - ports: - - name: configsvr - port: 27018 - targetPort: configsvr - podSpec: - containers: - - name: configsvr - ports: - - name: configsvr - containerPort: 27018 - command: - - /scripts/replicaset-setup.sh - - --configsvr - lifecycle: - postStart: - exec: - command: - - /scripts/replicaset-post-start.sh - - CONFIGSVR - - "true" - volumeMounts: - - name: scripts - mountPath: /scripts/replicaset-setup.sh - subPath: replicaset-setup.sh - - name: scripts - mountPath: /scripts/replicaset-post-start.sh - subPath: replicaset-post-start.sh - - name: shard - scriptSpecs: - - name: mongodb-scripts - templateRef: mongodb-scripts - volumeName: scripts - namespace: default - defaultMode: 493 - characterType: mongodb - workloadType: Consensus - consensusSpec: - leader: - name: "primary" - accessMode: ReadWrite - followers: - - name: "secondary" - accessMode: Readonly - updateStrategy: BestEffortParallel - probes: - roleProbe: - periodSeconds: 2 - failureThreshold: 3 - service: - ports: - - name: shard - port: 27018 - targetPort: shard - podSpec: - containers: - - name: shard - ports: - - name: shard - containerPort: 27018 - command: - - /scripts/replicaset-setup.sh - - --shardsvr - lifecycle: - postStart: - exec: - command: - - /scripts/replicaset-post-start.sh - - SHARD - - "false" - volumeMounts: - - name: scripts - mountPath: /scripts/replicaset-setup.sh - subPath: replicaset-setup.sh - - name: scripts - mountPath: /scripts/replicaset-post-start.sh - subPath: replicaset-post-start.sh - - name: agent - command: - - /scripts/shard-agent.sh - volumeMounts: - - name: scripts - mountPath: /scripts/shard-agent.sh - subPath: shard-agent.sh ---- -# Source: mongodb/templates/clusterversion.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClusterVersion -metadata: - name: mongodb-5.0.14 - labels: - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -spec: - clusterDefinitionRef: mongodb - componentVersions: - - componentDefRef: replicaset - versionsContext: - containers: - - name: mongodb - image: mongo:5.0.14 - imagePullPolicy: IfNotPresent ---- -# Source: mongodb/templates/sharding-clusterversion.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClusterVersion -metadata: - name: mongodb-sharding-5.0.14 - labels: - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -spec: - clusterDefinitionRef: mongodb-sharding - componentVersions: - - componentDefRef: mongos - versionsContext: - containers: - - name: mongos - image: mongo:5.0.14 - imagePullPolicy: IfNotPresent - - componentDefRef: configsvr - versionsContext: - containers: - - name: configsvr - image: mongo:5.0.14 - imagePullPolicy: IfNotPresent - - componentDefRef: shard - versionsContext: - containers: - - name: shard - image: mongo:5.0.14 - - name: agent - image: mongo:5.0.14 - imagePullPolicy: IfNotPresent ---- -# Source: mongodb/templates/configconstraint.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ConfigConstraint -metadata: - name: mongodb-config-constraints - labels: - helm.sh/chart: mongodb-0.5.0-beta.2 - app.kubernetes.io/name: mongodb - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "5.0.14" - app.kubernetes.io/managed-by: Helm -spec: - configurationSchema: - cue: "" - - # mysql configuration file format - formatterConfig: - format: yaml ---- # Source: mongodb-cluster/templates/replicaset.yaml apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: mongodb-cluster-0.5.0-beta.2 + helm.sh/chart: mongodb-cluster-0.5.0-beta.9 app.kubernetes.io/name: mongodb-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" diff --git a/test/e2e/testdata/smoketest/playgroundtest.go b/test/e2e/testdata/smoketest/playgroundtest.go index 01f5c8d92..94304cca5 100644 --- a/test/e2e/testdata/smoketest/playgroundtest.go +++ b/test/e2e/testdata/smoketest/playgroundtest.go @@ -138,13 +138,12 @@ func PlaygroundDestroy() { } func checkPlaygroundCluster() { - commond := "kubectl get pod -n default -l 'app.kubernetes.io/instance in (mycluster)'| grep mycluster |" + - " awk '{print $3}'" - log.Println(commond) Eventually(func(g Gomega) { - podStatus := e2eutil.ExecCommand(commond) - log.Println("podStatus is " + e2eutil.StringStrip(podStatus)) - g.Expect(e2eutil.StringStrip(podStatus)).Should(Equal("Running")) + e2eutil.WaitTime(100000) + podStatusResult := e2eutil.CheckPodStatus() + for _, result := range podStatusResult { + g.Expect(result).Should(BeTrue()) + } }, time.Second*180, time.Second*1).Should(Succeed()) cmd := "kbcli cluster list | grep mycluster | awk '{print $6}'" log.Println(cmd) diff --git a/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml b/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml index a6fe12e21..d701711f6 100644 --- a/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml +++ b/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml @@ -5,7 +5,7 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: pgcluster-0.5.0-beta.2 + helm.sh/chart: pgcluster-0.5.0-beta.9 app.kubernetes.io/name: pgcluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "14.7.0" diff --git a/test/e2e/testdata/smoketest/postgresql/07_cv.yaml b/test/e2e/testdata/smoketest/postgresql/07_cv.yaml index a60746a07..6da78cd27 100644 --- a/test/e2e/testdata/smoketest/postgresql/07_cv.yaml +++ b/test/e2e/testdata/smoketest/postgresql/07_cv.yaml @@ -10,3 +10,6 @@ spec: containers: - name: postgresql image: docker.io/apecloud/postgresql:14.7.0 + initContainers: + - image: docker.io/apecloud/postgresql:14.7.0 + name: pg-init-container \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml index 5720b38d1..fa4728d27 100644 --- a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml +++ b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml @@ -1,724 +1,11 @@ --- -# Source: redis/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: redis7-config-template - labels: - helm.sh/chart: redis-0.5.0-beta.2 - app.kubernetes.io/name: redis - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.6" - app.kubernetes.io/managed-by: Helm -data: - redis.conf: |- - bind 0.0.0.0 - port 6379 - tcp-backlog 511 - timeout 0 - tcp-keepalive 300 - daemonize no - pidfile /var/run/redis_6379.pid - {{ block "logsBlock" . }} - loglevel notice - logfile "/data/running.log" - {{ end }} - databases 16 - always-show-logo no - set-proc-title yes - proc-title-template "{title} {listen-addr} {server-mode}" - stop-writes-on-bgsave-error yes - rdbcompression yes - rdbchecksum yes - dbfilename dump.rdb - rdb-del-sync-files no - dir ./ - replica-serve-stale-data yes - replica-read-only yes - repl-diskless-sync yes - repl-diskless-sync-delay 5 - repl-diskless-sync-max-replicas 0 - repl-diskless-load disabled - repl-disable-tcp-nodelay no - replica-priority 100 - acllog-max-len 128 - lazyfree-lazy-eviction no - lazyfree-lazy-expire no - lazyfree-lazy-server-del no - replica-lazy-flush no - lazyfree-lazy-user-del no - lazyfree-lazy-user-flush no - oom-score-adj no - oom-score-adj-values 0 200 800 - disable-thp yes - appendonly yes - appendfilename "appendonly.aof" - appenddirname "appendonlydir" - appendfsync everysec - no-appendfsync-on-rewrite no - auto-aof-rewrite-percentage 100 - auto-aof-rewrite-min-size 64mb - aof-load-truncated yes - aof-use-rdb-preamble yes - aof-timestamp-enabled no - slowlog-log-slower-than 10000 - slowlog-max-len 128 - latency-monitor-threshold 0 - notify-keyspace-events "" - hash-max-listpack-entries 512 - hash-max-listpack-value 64 - list-max-listpack-size -2 - list-compress-depth 0 - set-max-intset-entries 512 - zset-max-listpack-entries 128 - zset-max-listpack-value 64 - hll-sparse-max-bytes 3000 - stream-node-max-bytes 4096 - stream-node-max-entries 100 - activerehashing yes - client-output-buffer-limit normal 0 0 0 - client-output-buffer-limit replica 256mb 64mb 60 - client-output-buffer-limit pubsub 32mb 8mb 60 - hz 10 - dynamic-hz yes - aof-rewrite-incremental-fsync yes - rdb-save-incremental-fsync yes - jemalloc-bg-thread yes - enable-debug-command yes - protected-mode no - - # maxmemory - {{- $request_memory := getContainerRequestMemory ( index $.podSpec.containers 0 ) }} - {{- if gt $request_memory 0 }} - maxmemory {{ $request_memory }} - {{- end -}} ---- -# Source: redis/templates/scripts.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: redis-scripts -data: - setup.sh: | - #!/bin/sh - set -ex - KB_PRIMARY_POD_NAME_PREFIX=${KB_PRIMARY_POD_NAME%%\.*} - if [ "$KB_PRIMARY_POD_NAME_PREFIX" = "$KB_POD_NAME" ]; then - echo "primary instance skip create a replication relationship." - exit 0 - else - until redis-cli -h $KB_PRIMARY_POD_NAME -p 6379 ping; do sleep 1; done - redis-cli -h 127.0.0.1 -p 6379 replicaof $KB_PRIMARY_POD_NAME 6379 || exit 1 - fi - redis-start.sh: | - #!/bin/sh - set -ex - echo "include /etc/conf/redis.conf" >> /etc/redis/redis.conf - echo "replica-announce-ip $KB_POD_FQDN" >> /etc/redis/redis.conf - exec redis-server /etc/redis/redis.conf \ - --loadmodule /opt/redis-stack/lib/redisearch.so ${REDISEARCH_ARGS} \ - --loadmodule /opt/redis-stack/lib/redisgraph.so ${REDISGRAPH_ARGS} \ - --loadmodule /opt/redis-stack/lib/redistimeseries.so ${REDISTIMESERIES_ARGS} \ - --loadmodule /opt/redis-stack/lib/rejson.so ${REDISJSON_ARGS} \ - --loadmodule /opt/redis-stack/lib/redisbloom.so ${REDISBLOOM_ARGS} - redis-sentinel-setup.sh: |- - #!/bin/sh - set -ex - {{- $clusterName := $.cluster.metadata.name }} - {{- $namespace := $.cluster.metadata.namespace }} - {{- /* find redis-sentinel component */}} - {{- $sentinel_component := fromJson "{}" }} - {{- $redis_component := fromJson "{}" }} - {{- $primary_index := 0 }} - {{- $primary_pod := "" }} - {{- range $i, $e := $.cluster.spec.componentSpecs }} - {{- if eq $e.componentDefRef "redis-sentinel" }} - {{- $sentinel_component = $e }} - {{- else if eq $e.componentDefRef "redis" }} - {{- $redis_component = $e }} - {{- if index $e "primaryIndex" }} - {{- if ne ($e.primaryIndex | int) 0 }} - {{- $primary_index = ($e.primaryIndex | int) }} - {{- end }} - {{- end }} - {{- end }} - {{- end }} - {{- /* build primary pod message, because currently does not support cross-component acquisition of environment variables, the service of the redis master node is assembled here through specific rules */}} - {{- $primary_pod = printf "%s-%s-%d.%s-%s-headless.%s.svc" $clusterName $redis_component.name $primary_index $clusterName $redis_component.name $namespace }} - {{- $sentinel_monitor := printf "%s-%s %s" $clusterName $redis_component.name $primary_pod }} - cat>/etc/sentinel/redis-sentinel.conf<> /etc/sentinel/redis-sentinel.conf - exec redis-server /etc/sentinel/redis-sentinel.conf --sentinel - echo "Start sentinel succeeded!" - redis-sentinel-ping.sh: |- - #!/bin/sh - set -ex - response=$( - timeout -s 3 $1 \ - redis-cli \ - -h localhost \ - -p 26379 \ - ping - ) - if [ "$?" -eq "124" ]; then - echo "Timed out" - exit 1 - fi - if [ "$response" != "PONG" ]; then - echo "$response" - exit 1 - fi ---- -# Source: redis/templates/backuppolicytemplate.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: BackupPolicyTemplate -metadata: - name: redis-backup-policy-template - labels: - clusterdefinition.kubeblocks.io/name: redis - helm.sh/chart: redis-0.5.0-beta.2 - app.kubernetes.io/name: redis - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.6" - app.kubernetes.io/managed-by: Helm -spec: - clusterDefinitionRef: redis - backupPolicies: - - componentDefRef: redis - ttl: 7d - schedule: - baseBackup: - type: snapshot - enable: false - cronExpression: "0 18 * * 0" - snapshot: - target: - connectionCredentialKey: - passwordKey: password - usernameKey: username ---- -# Source: redis/templates/clusterdefinition.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClusterDefinition -metadata: - name: redis - labels: - helm.sh/chart: redis-0.5.0-beta.2 - app.kubernetes.io/name: redis - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.6" - app.kubernetes.io/managed-by: Helm -spec: - type: redis - connectionCredential: - username: "" - password: "" - endpoint: "$(SVC_FQDN):$(SVC_PORT_redis)" - host: "$(SVC_FQDN)" - port: "$(SVC_PORT_redis)" - componentDefs: - - name: redis - workloadType: Replication - characterType: redis - probes: - roleProbe: - failureThreshold: 2 - periodSeconds: 2 - timeoutSeconds: 1 - replicationSpec: - switchPolicies: - - type: MaximumAvailability - switchStatements: - demote: - - replicaof $KB_NEW_PRIMARY_ROLE_NAME 6379 - promote: - - replicaof no one - follow: - - replicaof $KB_NEW_PRIMARY_ROLE_NAME 6379 - - type: MaximumDataProtection - switchStatements: - demote: - - replicaof $KB_NEW_PRIMARY_ROLE_NAME 6379 - promote: - - replicaof no one - follow: - - replicaof $KB_NEW_PRIMARY_ROLE_NAME 6379 - switchCmdExecutorConfig: - image: redis:7.0.5 - switchSteps: - - role: NewPrimary - command: - - /bin/sh - - -c - args: - - redis-cli -h $(KB_SWITCH_ROLE_ENDPOINT) -p 6379 $(KB_SWITCH_PROMOTE_STATEMENT) - - role: Secondaries - command: - - /bin/sh - - -c - args: - - redis-cli -h $(KB_SWITCH_ROLE_ENDPOINT) -p 6379 $(KB_SWITCH_FOLLOW_STATEMENT) - - role: OldPrimary - command: - - /bin/sh - - -c - args: - - redis-cli -h $(KB_SWITCH_ROLE_ENDPOINT) -p 6379 $(KB_SWITCH_DEMOTE_STATEMENT) - service: - ports: - - name: redis - port: 6379 - targetPort: redis - configSpecs: - - name: redis-replication-config - templateRef: redis7-config-template - constraintRef: redis7-config-constraints - namespace: default - volumeName: redis-config - scriptSpecs: - - name: redis-scripts - templateRef: redis-scripts - namespace: default - volumeName: scripts - defaultMode: 493 - monitor: - builtIn: false - exporterConfig: - scrapePort: 9121 - scrapePath: "/metrics" - logConfigs: - - name: running - filePathPattern: /data/running.log - volumeTypes: - - name: data - type: data - podSpec: - containers: - - name: redis - ports: - - name: redis - containerPort: 6379 - volumeMounts: - - name: data - mountPath: /data - - name: redis-config - mountPath: /etc/conf - - name: scripts - mountPath: /scripts - - name: redis-conf - mountPath: /etc/redis - command: ["/scripts/redis-start.sh"] - lifecycle: - postStart: - exec: - command: ["/scripts/setup.sh"] - - name: redis-exporter - image: oliver006/redis_exporter:latest - imagePullPolicy: IfNotPresent - resources: - requests: - cpu: 100m - memory: 100Mi - ports: - - name: metrics - containerPort: 9121 - livenessProbe: - httpGet: - path: / - port: metrics - readinessProbe: - httpGet: - path: / - port: metrics - systemAccounts: -# Seems redis-cli has its own mechanism to parse input tokens and there is no elegent way -# to pass $(KB_ACCOUNT_STATEMENT) to redis-cli without causing parsing error. -# Instead, using a shell script to wrap redis-cli and pass $(KB_ACCOUNT_STATEMENT) to it will do. - cmdExecutorConfig: - image: docker.io/redis:7.0.5 - command: - - sh - - -c - args: - - "redis-cli -h $(KB_ACCOUNT_ENDPOINT) $(KB_ACCOUNT_STATEMENT)" - passwordConfig: - length: 10 - numDigits: 5 - numSymbols: 0 - letterCase: MixedCases - accounts: - - name: kbadmin - provisionPolicy: - type: CreateByStmt - scope: AllPods - statements: - creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys - update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - - name: kbdataprotection - provisionPolicy: - type: CreateByStmt - scope: AllPods - statements: - creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys - update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - - name: kbmonitoring - provisionPolicy: - type: CreateByStmt - scope: AllPods - statements: - creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get - update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - - name: kbprobe - provisionPolicy: - type: CreateByStmt - scope: AllPods - statements: - creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get - update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - - name: kbreplicator - provisionPolicy: - type: CreateByStmt - scope: AllPods - statements: - creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) +psync +replconf +ping - update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - - name: redis-sentinel - workloadType: Stateful - characterType: redis - service: - ports: - - name: redis-sentinel - targetPort: redis-sentinel - port: 26379 - configSpecs: - - name: redis-replication-config - templateRef: redis7-config-template - constraintRef: redis7-config-constraints - namespace: default - volumeName: redis-config - scriptSpecs: - - name: redis-scripts - templateRef: redis-scripts - namespace: default - volumeName: scripts - defaultMode: 493 - volumeTypes: - - name: data - type: data - podSpec: - initContainers: - - name: init-redis-sentinel - imagePullPolicy: IfNotPresent - volumeMounts: - - name: data - mountPath: /data - - name: redis-config - mountPath: /etc/conf - - name: sentinel-conf - mountPath: /etc/sentinel - - name: scripts - mountPath: /scripts - command: [ "/scripts/redis-sentinel-setup.sh" ] - containers: - - name: redis-sentinel - imagePullPolicy: IfNotPresent - ports: - - containerPort: 26379 - name: redis-sentinel - volumeMounts: - - name: data - mountPath: /data - - name: redis-config - mountPath: /etc/conf - - name: sentinel-conf - mountPath: /etc/sentinel - - name: scripts - mountPath: /scripts - command: - - /bin/bash - args: - - -c - - | - set -ex - /scripts/redis-sentinel-start.sh - livenessProbe: - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 5 - exec: - command: - - sh - - -c - - /scripts/redis-sentinel-ping.sh 5 - readinessProbe: - initialDelaySeconds: 10 - periodSeconds: 5 - timeoutSeconds: 1 - successThreshold: 1 - failureThreshold: 5 - exec: - command: - - sh - - -c - - /scripts/redis-sentinel-ping.sh 1 ---- -# Source: redis/templates/clusterversion.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClusterVersion -metadata: - name: redis-7.0.6 - labels: - helm.sh/chart: redis-0.5.0-beta.2 - app.kubernetes.io/name: redis - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.6" - app.kubernetes.io/managed-by: Helm -spec: - clusterDefinitionRef: redis - componentVersions: - - componentDefRef: redis - versionsContext: - containers: - - name: redis - image: redis/redis-stack-server:7.0.6-RC8 - imagePullPolicy: IfNotPresent - - componentDefRef: redis-sentinel - versionsContext: - initContainers: - - name: init-redis-sentinel - image: redis/redis-stack-server:7.0.6-RC8 - imagePullPolicy: IfNotPresent - containers: - - name: redis-sentinel - image: redis/redis-stack-server:7.0.6-RC8 - imagePullPolicy: IfNotPresent ---- -# Source: redis/templates/configconstraint.yaml -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ConfigConstraint -metadata: - name: redis7-config-constraints - labels: - helm.sh/chart: redis-0.5.0-beta.2 - app.kubernetes.io/name: redis - app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "7.0.6" - app.kubernetes.io/managed-by: Helm -spec: - - cfgSchemaTopLevelName: RedisParameter - - # ConfigurationSchema that impose restrictions on engine parameter's rule - configurationSchema: - cue: |- - //Copyright (C) 2022-2023 ApeCloud Co., Ltd - // - //This file is part of KubeBlocks project - // - //This program is free software: you can redistribute it and/or modify - //it under the terms of the GNU Affero General Public License as published by - //the Free Software Foundation, either version 3 of the License, or - //(at your option) any later version. - // - //This program is distributed in the hope that it will be useful - //but WITHOUT ANY WARRANTY; without even the implied warranty of - //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - //GNU Affero General Public License for more details. - // - //You should have received a copy of the GNU Affero General Public License - //along with this program. If not, see . - - #RedisParameter: { - - "acllog-max-len": int & >=1 & <=10000 | *128 - - "acl-pubsub-default"?: string & "resetchannels" | "allchannels" - - activedefrag?: string & "yes" | "no" - - "active-defrag-cycle-max": int & >=1 & <=75 | *75 - - "active-defrag-cycle-min": int & >=1 & <=75 | *5 - - "active-defrag-ignore-bytes": int | *104857600 - - "active-defrag-max-scan-fields": int & >=1 & <=1000000 | *1000 - - "active-defrag-threshold-lower": int & >=1 & <=100 | *10 - - "active-defrag-threshold-upper": int & >=1 & <=100 | *100 - - "active-expire-effort": int & >=1 & <=10 | *1 - - appendfsync?: string & "always" | "everysec" | "no" - - appendonly?: string & "yes" | "no" - - "client-output-buffer-limit-normal-hard-limit": int | *0 - - "client-output-buffer-limit-normal-soft-limit": int | *0 - - "client-output-buffer-limit-normal-soft-seconds": int | *0 - - "client-output-buffer-limit-pubsub-hard-limit": int | *33554432 - - "client-output-buffer-limit-pubsub-soft-limit": int | *8388608 - - "client-output-buffer-limit-pubsub-soft-seconds": int | *60 - - "client-output-buffer-limit-replica-soft-seconds": int | *60 - - "client-query-buffer-limit": int & >=1048576 & <=1073741824 | *1073741824 - - "close-on-replica-write"?: string & "yes" | "no" - - "cluster-allow-pubsubshard-when-down"?: string & "yes" | "no" - - "cluster-allow-reads-when-down"?: string & "yes" | "no" - - "cluster-enabled"?: string & "yes" | "no" - - "cluster-preferred-endpoint-type"?: string & "tls-dynamic" | "ip" - - "cluster-require-full-coverage"?: string & "yes" | "no" - - databases: int & >=1 & <=10000 | *16 - - "hash-max-listpack-entries": int | *512 - - "hash-max-listpack-value": int | *64 - - "hll-sparse-max-bytes": int & >=1 & <=16000 | *3000 - - "latency-tracking"?: string & "yes" | "no" - - "lazyfree-lazy-eviction"?: string & "yes" | "no" - - "lazyfree-lazy-expire"?: string & "yes" | "no" - - "lazyfree-lazy-server-del"?: string & "yes" | "no" - - "lazyfree-lazy-user-del"?: string & "yes" | "no" - - "lfu-decay-time": int | *1 - - "lfu-log-factor": int | *10 - - "list-compress-depth": int | *0 - - "list-max-listpack-size": int | *-2 - - "lua-time-limit": int & 5000 | *5000 - - maxclients: int & >=1 & <=65000 | *65000 - - "maxmemory-policy"?: string & "volatile-lru" | "allkeys-lru" | "volatile-lfu" | "allkeys-lfu" | "volatile-random" | "allkeys-random" | "volatile-ttl" | "noeviction" - - "maxmemory-samples": int | *3 - - "min-replicas-max-lag": int | *10 - - "min-replicas-to-write": int | *0 - - "notify-keyspace-events"?: string - - "proto-max-bulk-len": int & >=1048576 & <=536870912 | *536870912 - - "rename-commands"?: string & "APPEND" | "BITCOUNT" | "BITFIELD" | "BITOP" | "BITPOS" | "BLPOP" | "BRPOP" | "BRPOPLPUSH" | "BZPOPMIN" | "BZPOPMAX" | "CLIENT" | "COMMAND" | "DBSIZE" | "DECR" | "DECRBY" | "DEL" | "DISCARD" | "DUMP" | "ECHO" | "EVAL" | "EVALSHA" | "EXEC" | "EXISTS" | "EXPIRE" | "EXPIREAT" | "FLUSHALL" | "FLUSHDB" | "GEOADD" | "GEOHASH" | "GEOPOS" | "GEODIST" | "GEORADIUS" | "GEORADIUSBYMEMBER" | "GET" | "GETBIT" | "GETRANGE" | "GETSET" | "HDEL" | "HEXISTS" | "HGET" | "HGETALL" | "HINCRBY" | "HINCRBYFLOAT" | "HKEYS" | "HLEN" | "HMGET" | "HMSET" | "HSET" | "HSETNX" | "HSTRLEN" | "HVALS" | "INCR" | "INCRBY" | "INCRBYFLOAT" | "INFO" | "KEYS" | "LASTSAVE" | "LINDEX" | "LINSERT" | "LLEN" | "LPOP" | "LPUSH" | "LPUSHX" | "LRANGE" | "LREM" | "LSET" | "LTRIM" | "MEMORY" | "MGET" | "MONITOR" | "MOVE" | "MSET" | "MSETNX" | "MULTI" | "OBJECT" | "PERSIST" | "PEXPIRE" | "PEXPIREAT" | "PFADD" | "PFCOUNT" | "PFMERGE" | "PING" | "PSETEX" | "PSUBSCRIBE" | "PUBSUB" | "PTTL" | "PUBLISH" | "PUNSUBSCRIBE" | "RANDOMKEY" | "READONLY" | "READWRITE" | "RENAME" | "RENAMENX" | "RESTORE" | "ROLE" | "RPOP" | "RPOPLPUSH" | "RPUSH" | "RPUSHX" | "SADD" | "SCARD" | "SCRIPT" | "SDIFF" | "SDIFFSTORE" | "SELECT" | "SET" | "SETBIT" | "SETEX" | "SETNX" | "SETRANGE" | "SINTER" | "SINTERSTORE" | "SISMEMBER" | "SLOWLOG" | "SMEMBERS" | "SMOVE" | "SORT" | "SPOP" | "SRANDMEMBER" | "SREM" | "STRLEN" | "SUBSCRIBE" | "SUNION" | "SUNIONSTORE" | "SWAPDB" | "TIME" | "TOUCH" | "TTL" | "TYPE" | "UNSUBSCRIBE" | "UNLINK" | "UNWATCH" | "WAIT" | "WATCH" | "ZADD" | "ZCARD" | "ZCOUNT" | "ZINCRBY" | "ZINTERSTORE" | "ZLEXCOUNT" | "ZPOPMAX" | "ZPOPMIN" | "ZRANGE" | "ZRANGEBYLEX" | "ZREVRANGEBYLEX" | "ZRANGEBYSCORE" | "ZRANK" | "ZREM" | "ZREMRANGEBYLEX" | "ZREMRANGEBYRANK" | "ZREMRANGEBYSCORE" | "ZREVRANGE" | "ZREVRANGEBYSCORE" | "ZREVRANK" | "ZSCORE" | "ZUNIONSTORE" | "SCAN" | "SSCAN" | "HSCAN" | "ZSCAN" | "XINFO" | "XADD" | "XTRIM" | "XDEL" | "XRANGE" | "XREVRANGE" | "XLEN" | "XREAD" | "XGROUP" | "XREADGROUP" | "XACK" | "XCLAIM" | "XPENDING" | "GEORADIUS_RO" | "GEORADIUSBYMEMBER_RO" | "LOLWUT" | "XSETID" | "SUBSTR" | "BITFIELD_RO" | "ACL" | "STRALGO" - - "repl-backlog-size": int | *1048576 - - "repl-backlog-ttl": int | *3600 - - "replica-allow-chaining"?: string & "yes" | "no" - - "replica-ignore-maxmemory"?: string & "yes" | "no" - - "replica-lazy-flush"?: string & "yes" | "no" - - "reserved-memory-percent": int & >=0 & <=100 | *25 - - "set-max-intset-entries": int & >=0 & <=500000000 | *512 - - "slowlog-log-slower-than": int | *10000 - - "slowlog-max-len": int | *128 - - "stream-node-max-bytes": int | *4096 - - "stream-node-max-entries": int | *100 - - "tcp-keepalive": int | *300 - - timeout: int | *0 - - "tracking-table-max-keys": int & >=1 & <=100000000 | *1000000 - - "zset-max-listpack-entries": int | *128 - - "zset-max-listpack-value": int | *64 - - "protected-mode"?: string & "yes" | "no" - - "enable-debug-command"?: string & "yes" | "no" | "local" - - ... - } - - configuration: #RedisParameter & { - } - - - ## require db instance restart - staticParameters: - - cluster-enabled - - databases - - maxclients - - ## reload parameters - ## dynamicParameters - dynamicParameters: - - - # redis configuration file format - formatterConfig: - format: redis ---- # Source: redis-cluster/templates/cluster.yaml apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: redis-cluster-0.5.0-beta.2 + helm.sh/chart: redis-cluster-0.5.0-beta.9 app.kubernetes.io/name: redis-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "7.0.6" diff --git a/test/e2e/testdata/smoketest/smoketestrun.go b/test/e2e/testdata/smoketest/smoketestrun.go index 3fefb65d8..cc577650b 100644 --- a/test/e2e/testdata/smoketest/smoketestrun.go +++ b/test/e2e/testdata/smoketest/smoketestrun.go @@ -125,7 +125,7 @@ func SmokeTest() { func runTestCases(files []string) { for _, file := range files { By("test " + file) - b := e2eutil.OpsYaml(file, "apply") + b := e2eutil.OpsYaml(file, "create") Expect(b).Should(BeTrue()) Eventually(func(g Gomega) { e2eutil.WaitTime(100000) diff --git a/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml b/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml index dd7f6c390..b79719842 100644 --- a/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml +++ b/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml @@ -5,7 +5,7 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: apecloud-mysql-cluster-0.5.0-beta.2 + helm.sh/chart: apecloud-mysql-cluster-0.5.0-beta.9 app.kubernetes.io/name: apecloud-mysql-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "8.0.30" diff --git a/test/e2e/testdata/smoketest/wesql/14_backup_snapshot_restore.yaml b/test/e2e/testdata/smoketest/wesql/14_backup_snapshot_restore.yaml index b467267be..995ee5570 100644 --- a/test/e2e/testdata/smoketest/wesql/14_backup_snapshot_restore.yaml +++ b/test/e2e/testdata/smoketest/wesql/14_backup_snapshot_restore.yaml @@ -6,10 +6,10 @@ metadata: kubeblocks.io/restore-from-backup: "{\"mysql\":\"backup-sbapshot-mycluster\"}" spec: clusterDefinitionRef: apecloud-mysql - clusterVersionRef: ac-mysql-8.0.30-latest + clusterVersionRef: ac-mysql-8.0.30 terminationPolicy: WipeOut componentSpecs: - - name: wesql + - name: mysql componentDefRef: mysql monitor: false replicas: 1 @@ -20,4 +20,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi \ No newline at end of file + storage: 2Gi \ No newline at end of file From 7ff43a765d0eedaf09345ce695d74248b22e802f Mon Sep 17 00:00:00 2001 From: xingran Date: Tue, 25 Apr 2023 12:45:11 +0800 Subject: [PATCH 168/439] chore: readCacheSnapshot add matchLabels filter (#2919) --- .../templates/clusterdefinition.yaml | 2 +- .../controller/lifecycle/transform_utils.go | 18 +++++++++++++++--- .../lifecycle/transformer_backup_policy_tpl.go | 3 +++ .../lifecycle/transformer_cluster_deletion.go | 3 ++- .../lifecycle/transformer_object_action.go | 3 ++- .../transformer_sts_horizontal_scaling.go | 3 ++- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index a9b52b658..f52de96fa 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -125,7 +125,7 @@ spec: - name: KUBERNETES_ROLE_LABEL value: "apps.kubeblocks.postgres.patroni/role" - name: KUBERNETES_LABELS - value: '{"app.kubernetes.io/instance":"$(KB_CLUSTER_NAME)","apps.kubeblocks.io/component-name":"$(KB_COMP_NAME)","app.kubernetes.io/managed-by":"kubeblocks"}' + value: '{"app.kubernetes.io/instance":"$(KB_CLUSTER_NAME)","apps.kubeblocks.io/component-name":"$(KB_COMP_NAME)"}' - name: RESTORE_DATA_DIR value: /home/postgres/pgdata/kb_restore - name: KB_PG_CONFIG_PATH diff --git a/internal/controller/lifecycle/transform_utils.go b/internal/controller/lifecycle/transform_utils.go index f0ceae12e..45cbdd8b6 100644 --- a/internal/controller/lifecycle/transform_utils.go +++ b/internal/controller/lifecycle/transform_utils.go @@ -162,14 +162,26 @@ func ownKinds() []client.ObjectList { } } +func getAppInstanceML(cluster appsv1alpha1.Cluster) client.MatchingLabels { + return client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.Name, + } +} + +func getAppInstanceAndManagedByML(cluster appsv1alpha1.Cluster) client.MatchingLabels { + return client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.Name, + constant.AppManagedByLabelKey: constant.AppName, + } +} + // read all objects owned by our cluster -func readCacheSnapshot(transCtx *ClusterTransformContext, cluster appsv1alpha1.Cluster, kinds ...client.ObjectList) (clusterSnapshot, error) { +func readCacheSnapshot(transCtx *ClusterTransformContext, cluster appsv1alpha1.Cluster, matchLabels client.MatchingLabels, kinds ...client.ObjectList) (clusterSnapshot, error) { // list what kinds of object cluster owns snapshot := make(clusterSnapshot) - ml := client.MatchingLabels{constant.AppInstanceLabelKey: cluster.GetName()} inNS := client.InNamespace(cluster.Namespace) for _, list := range kinds { - if err := transCtx.Client.List(transCtx.Context, list, inNS, ml); err != nil { + if err := transCtx.Client.List(transCtx.Context, list, inNS, matchLabels); err != nil { return nil, err } // reflect get list.Items diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index e73b07fe8..e49c3aeff 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -110,6 +110,7 @@ func (r *BackupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti } backupPolicy.Labels[constant.AppInstanceLabelKey] = cluster.Name backupPolicy.Labels[constant.KBAppComponentDefRefLabelKey] = policyTPL.ComponentDefRef + backupPolicy.Labels[constant.AppManagedByLabelKey] = constant.AppName // only update the role labelSelector of the backup target instance when component workload is Replication/Consensus. // because the replicas of component will change, such as 2->1. then if the target role is 'follower' and replicas is 1, @@ -169,6 +170,7 @@ func (r *BackupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba Labels: map[string]string{ constant.AppInstanceLabelKey: cluster.Name, constant.KBAppComponentDefRefLabelKey: policyTPL.ComponentDefRef, + constant.AppManagedByLabelKey: constant.AppName, }, Annotations: map[string]string{ constant.DefaultBackupPolicyAnnotationKey: "true", @@ -233,6 +235,7 @@ func (r *BackupPolicyTPLTransformer) convertBasePolicy(bp appsv1alpha1.BasePolic MatchLabels: map[string]string{ constant.AppInstanceLabelKey: clusterName, constant.KBAppComponentLabelKey: component.Name, + constant.AppManagedByLabelKey: constant.AppName, }, }, }, diff --git a/internal/controller/lifecycle/transformer_cluster_deletion.go b/internal/controller/lifecycle/transformer_cluster_deletion.go index acc050ab5..babc66800 100644 --- a/internal/controller/lifecycle/transformer_cluster_deletion.go +++ b/internal/controller/lifecycle/transformer_cluster_deletion.go @@ -62,7 +62,8 @@ func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * // there is chance that objects leak occurs because of cache stale // ignore the problem currently // TODO: GC the leaked objects - snapshot, err := readCacheSnapshot(transCtx, *cluster, kinds...) + ml := getAppInstanceML(*cluster) + snapshot, err := readCacheSnapshot(transCtx, *cluster, ml, kinds...) if err != nil { return err } diff --git a/internal/controller/lifecycle/transformer_object_action.go b/internal/controller/lifecycle/transformer_object_action.go index a468d8bf2..182c3b2d1 100644 --- a/internal/controller/lifecycle/transformer_object_action.go +++ b/internal/controller/lifecycle/transformer_object_action.go @@ -37,7 +37,8 @@ func (t *ObjectActionTransformer) Transform(ctx graph.TransformContext, dag *gra origCluster := transCtx.OrigCluster // get the old objects snapshot - oldSnapshot, err := readCacheSnapshot(transCtx, *origCluster, ownKinds()...) + ml := getAppInstanceAndManagedByML(*origCluster) + oldSnapshot, err := readCacheSnapshot(transCtx, *origCluster, ml, ownKinds()...) if err != nil { return err } diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index de9f9b6d2..97a4eade3 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -329,7 +329,8 @@ func (t *StsHorizontalScalingTransformer) Transform(ctx graph.TransformContext, // by sts: we only handle the pvc deletion which occurs in cluster deletion. // by h-scale transformer: we handle the pvc creation and deletion, the creation is handled in h-scale funcs. // so all in all, here we should only handle the pvc deletion of both types. - oldSnapshot, err := readCacheSnapshot(transCtx, *cluster, &corev1.PersistentVolumeClaimList{}) + ml := getAppInstanceML(*cluster) + oldSnapshot, err := readCacheSnapshot(transCtx, *cluster, ml, &corev1.PersistentVolumeClaimList{}) if err != nil { return err } From c23266b2264cd2ea7e10d48f1e51fd5d1a8c70f1 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 25 Apr 2023 13:55:19 +0800 Subject: [PATCH 169/439] chore: fix restart apecloud-mysql failed when using the customized comp name (#2935) --- controllers/apps/components/consensus/consensus_utils.go | 4 ++-- controllers/dataprotection/backuppolicy_controller.go | 5 ----- controllers/dataprotection/backuppolicy_controller_test.go | 3 --- .../controller/lifecycle/transformer_backup_policy_tpl.go | 6 ++++++ 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/controllers/apps/components/consensus/consensus_utils.go b/controllers/apps/components/consensus/consensus_utils.go index e06c2143b..d66578553 100644 --- a/controllers/apps/components/consensus/consensus_utils.go +++ b/controllers/apps/components/consensus/consensus_utils.go @@ -419,8 +419,8 @@ func updateConsensusRoleInfo(ctx context.Context, if len(configList.Items) > 0 { for _, config := range configList.Items { patch := client.MergeFrom(config.DeepCopy()) - config.Data["KB_"+strings.ToUpper(componentName)+"_LEADER"] = leader - config.Data["KB_"+strings.ToUpper(componentName)+"_FOLLOWERS"] = followers + config.Data["KB_"+strings.ToUpper(componentDef.Name)+"_LEADER"] = leader + config.Data["KB_"+strings.ToUpper(componentDef.Name)+"_FOLLOWERS"] = followers if err := cli.Patch(ctx, &config, patch); err != nil { return err } diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index 272d1744d..9e603dfe4 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -449,11 +449,6 @@ func (r *BackupPolicyReconciler) setGlobalPersistentVolumeClaim(backupPolicy *da backupPolicy.PersistentVolumeClaim.Name = globalPVCName } - globalStorageClass := viper.GetString(constant.CfgKeyBackupPVCStorageClass) - if pvcCfg.StorageClassName == nil && globalStorageClass != "" { - backupPolicy.PersistentVolumeClaim.StorageClassName = &globalStorageClass - } - globalInitCapacity := viper.GetString(constant.CfgKeyBackupPVCInitCapacity) if pvcCfg.InitCapacity.IsZero() && globalInitCapacity != "" { backupPolicy.PersistentVolumeClaim.InitCapacity = resource.MustParse(globalInitCapacity) diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index dd76d96ae..976a85171 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -306,10 +306,8 @@ var _ = Describe("Backup Policy Controller", func() { By("By creating a backupPolicy with empty secret") pvcName := "backup-data" pvcInitCapacity := "10Gi" - pvcStorageClass := "standard" viper.SetDefault(constant.CfgKeyBackupPVCName, pvcName) viper.SetDefault(constant.CfgKeyBackupPVCInitCapacity, pvcInitCapacity) - viper.SetDefault(constant.CfgKeyBackupPVCStorageClass, pvcStorageClass) backupPolicy := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). AddFullPolicy(). SetBackupToolName(backupToolName). @@ -321,7 +319,6 @@ var _ = Describe("Backup Policy Controller", func() { Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) g.Expect(fetched.Spec.Full.PersistentVolumeClaim.Name).To(Equal(pvcName)) - g.Expect(*fetched.Spec.Full.PersistentVolumeClaim.StorageClassName).To(Equal(pvcStorageClass)) g.Expect(fetched.Spec.Full.PersistentVolumeClaim.InitCapacity.String()).To(Equal(pvcInitCapacity)) })).Should(Succeed()) }) diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index e49c3aeff..5c41184fe 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -333,12 +333,18 @@ func (r *BackupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.Common Namespace: globalPVConfigMapNamespace, } } + globalStorageClass := viper.GetString(constant.CfgKeyBackupPVCStorageClass) + var storageClassName *string + if globalStorageClass != "" { + storageClassName = &globalStorageClass + } return &dataprotectionv1alpha1.CommonBackupPolicy{ BackupToolName: bp.BackupToolName, PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ InitCapacity: resource.MustParse(defaultInitCapacity), CreatePolicy: defaultCreatePolicy, PersistentVolumeConfigMap: persistentVolumeConfigMap, + StorageClassName: storageClassName, }, BasePolicy: r.convertBasePolicy(bp.BasePolicy, clusterName, component, workloadType), } From 6619ced313163e4dc84f3d7b2e27ea4a3b47ba9a Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Tue, 25 Apr 2023 14:07:07 +0800 Subject: [PATCH 170/439] fix: kbcli cluster with no default SC describe panic (#2923) --- internal/cli/cluster/cluster.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/cli/cluster/cluster.go b/internal/cli/cluster/cluster.go index d3ce6f341..cb17ee5e9 100644 --- a/internal/cli/cluster/cluster.go +++ b/internal/cli/cluster/cluster.go @@ -350,7 +350,11 @@ func (o *ClusterObjects) getStorageInfo(component *appsv1alpha1.ClusterComponent if labels[constant.VolumeClaimTemplateNameLabelKey] != vcTpl.Name { continue } - return *pvc.Spec.StorageClassName + if pvc.Spec.StorageClassName != nil { + return *pvc.Spec.StorageClassName + } else { + return types.None + } } return types.None From 75d92c565a4dd4104b5ce6466d97dc1ee63ff616 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Tue, 25 Apr 2023 15:15:24 +0800 Subject: [PATCH 171/439] fix: add cluster validate restore parameters. (#2928) --- deploy/csi-s3/templates/csi-s3.yaml | 1 + internal/controller/component/restore_utils.go | 18 +++++++++++++++--- .../controller/component/restore_utils_test.go | 16 ++++++++++++++++ .../lifecycle/cluster_plan_builder.go | 3 ++- .../lifecycle/transformer_cluster.go | 5 +++++ internal/controllerutil/errors.go | 1 + 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/deploy/csi-s3/templates/csi-s3.yaml b/deploy/csi-s3/templates/csi-s3.yaml index 84e507e1c..4e72b8771 100644 --- a/deploy/csi-s3/templates/csi-s3.yaml +++ b/deploy/csi-s3/templates/csi-s3.yaml @@ -58,6 +58,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet containers: - name: driver-registrar image: {{ .Values.images.registrar }} diff --git a/internal/controller/component/restore_utils.go b/internal/controller/component/restore_utils.go index 71fc9cb7c..ba34a15c2 100644 --- a/internal/controller/component/restore_utils.go +++ b/internal/controller/component/restore_utils.go @@ -24,6 +24,7 @@ import ( snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -78,7 +79,7 @@ func BuildRestoredInfo2( case dataprotectionv1alpha1.BackupTypeFull: return buildInitContainerWithFullBackup(component, backup, backupTool) case dataprotectionv1alpha1.BackupTypeSnapshot: - buildVolumeClaimTemplatesWithSnapshot(component, backup) + return buildVolumeClaimTemplatesWithSnapshot(component, backup) } return nil } @@ -153,11 +154,21 @@ func buildInitContainerWithFullBackup( // buildVolumeClaimTemplatesWithSnapshot builds the volumeClaimTemplate if it needs to restore from volumeSnapshot func buildVolumeClaimTemplatesWithSnapshot(component *SynthesizedComponent, - backup *dataprotectionv1alpha1.Backup) { + backup *dataprotectionv1alpha1.Backup) error { if len(component.VolumeClaimTemplates) == 0 { - return + return intctrlutil.NewError(intctrlutil.ErrorTypeBackupNotSupported, + "need specified volumeClaimTemplates to restore.") } vct := component.VolumeClaimTemplates[0] + backupTotalSize, err := resource.ParseQuantity(backup.Status.TotalSize) + if err != nil { + return err + } + if vct.Spec.Resources.Requests.Storage().Value() < backupTotalSize.Value() { + return intctrlutil.NewErrorf(intctrlutil.ErrorTypeStorageNotMatch, + "requests storage %s is less than source backup storage %s.", + vct.Spec.Resources.Requests.Storage(), backupTotalSize.String()) + } snapshotAPIGroup := snapshotv1.GroupName vct.Spec.DataSource = &corev1.TypedLocalObjectReference{ APIGroup: &snapshotAPIGroup, @@ -165,4 +176,5 @@ func buildVolumeClaimTemplatesWithSnapshot(component *SynthesizedComponent, Name: backup.Name, } component.VolumeClaimTemplates[0] = vct + return nil } diff --git a/internal/controller/component/restore_utils_test.go b/internal/controller/component/restore_utils_test.go index 17cb5307f..514f3c4f7 100644 --- a/internal/controller/component/restore_utils_test.go +++ b/internal/controller/component/restore_utils_test.go @@ -24,6 +24,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" @@ -72,6 +73,7 @@ var _ = Describe("probe_utils", func() { backup.Status.BackupToolName = backupToolName backup.Status.PersistentVolumeClaimName = "backup-pvc" backup.Status.Phase = expectPhase + backup.Status.TotalSize = "1Gi" })).Should(Succeed()) } @@ -115,6 +117,16 @@ var _ = Describe("probe_utils", func() { ObjectMeta: metav1.ObjectMeta{ Name: "data", }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + }, }, }, } @@ -135,6 +147,10 @@ var _ = Describe("probe_utils", func() { Name: backupName, } Expect(reflect.DeepEqual(expectDataSource, vct.Spec.DataSource)).Should(BeTrue()) + + By("error if request storage is less than backup storage") + component.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("512Mi") + Expect(BuildRestoredInfo(reqCtx, k8sClient, testCtx.DefaultNamespace, component, backupName)).Should(HaveOccurred()) }) }) diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 2921975f1..78bde7126 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -263,7 +263,8 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { } // delete secondary objects if _, ok := node.obj.(*appsv1alpha1.Cluster); !ok { - err := c.cli.Delete(c.transCtx.Context, node.obj) + err := intctrlutil.BackgroundDeleteObject(c.cli, c.transCtx.Context, node.obj) + // err := c.cli.Delete(c.transCtx.Context, node.obj) if err != nil && !apierrors.IsNotFound(err) { return err } diff --git a/internal/controller/lifecycle/transformer_cluster.go b/internal/controller/lifecycle/transformer_cluster.go index 7149977cb..9e54dad92 100644 --- a/internal/controller/lifecycle/transformer_cluster.go +++ b/internal/controller/lifecycle/transformer_cluster.go @@ -166,6 +166,11 @@ func getClusterBackupSourceMap(cluster *appsv1alpha1.Cluster) (map[string]string } compBackupMap := map[string]string{} err := json.Unmarshal([]byte(compBackupMapString), &compBackupMap) + for k := range compBackupMap { + if cluster.Spec.GetComponentByName(k) == nil { + return nil, intctrlutil.NewErrorf(intctrlutil.ErrorTypeNotFound, "restore: not found componentSpecs[*].name %s", k) + } + } return compBackupMap, err } diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index bea523b14..d1ece950c 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -51,6 +51,7 @@ const ( ErrorTypeBackupNotCompleted ErrorType = "BackupNotCompleted" // report backup not completed. ErrorTypeBackupPVCNameIsEmpty ErrorType = "BackupPVCNameIsEmpty" // pvc name for backup is empty ErrorTypeBackupJobFailed ErrorType = "BackupJobFailed" // backup job failed + ErrorTypeStorageNotMatch ErrorType = "ErrorTypeStorageNotMatch" ) var ErrFailedToAddFinalizer = errors.New("failed to add finalizer") From 58516e89ac50d8a57b00698acf5dc206b3dccfb4 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 25 Apr 2023 15:25:27 +0800 Subject: [PATCH 172/439] fix: the cluster phase always is Creating when restoring a new cluster (#2938) --- controllers/apps/cluster_controller_test.go | 15 +++++++-------- .../lifecycle/transformer_cluster_status.go | 14 ++++++++------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index b1be1c0cd..bd70e1d44 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1439,20 +1439,19 @@ var _ = Describe("Cluster Controller", func() { By("remove init container after all components are Running") Eventually(testapps.GetClusterObservedGeneration(&testCtx, client.ObjectKeyFromObject(clusterObj))).Should(BeEquivalentTo(1)) Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterObj), clusterObj)).Should(Succeed()) - Expect(testapps.ChangeObjStatus(&testCtx, clusterObj, func() { - clusterObj.Status.Components = map[string]appsv1alpha1.ClusterComponentStatus{ - replicationCompName: {Phase: appsv1alpha1.RunningClusterCompPhase}, - } - })).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(&sts), func(g Gomega, tmpSts *appsv1.StatefulSet) { g.Expect(tmpSts.Spec.Template.Spec.InitContainers).Should(BeEmpty()) })).Should(Succeed()) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, replicationCompName)).Should(Equal(appsv1alpha1.CreatingClusterCompPhase)) By("clean up annotations after cluster running") - Expect(testapps.ChangeObjStatus(&testCtx, clusterObj, func() { - clusterObj.Status.Phase = appsv1alpha1.RunningClusterPhase - })).ShouldNot(HaveOccurred()) + Expect(testapps.GetAndChangeObjStatus(&testCtx, clusterKey, func(tmpCluster *appsv1alpha1.Cluster) { + compStatus := tmpCluster.Status.Components[replicationCompName] + compStatus.Phase = appsv1alpha1.RunningClusterCompPhase + tmpCluster.Status.Components[replicationCompName] = compStatus + })()).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { + g.Expect(tmpCluster.Status.Phase).Should(Equal(appsv1alpha1.RunningClusterPhase)) g.Expect(tmpCluster.Annotations[constant.RestoreFromBackUpAnnotationKey]).Should(BeEmpty()) })).Should(Succeed()) }) diff --git a/internal/controller/lifecycle/transformer_cluster_status.go b/internal/controller/lifecycle/transformer_cluster_status.go index 3f3ce41a2..f2c9a5ac1 100644 --- a/internal/controller/lifecycle/transformer_cluster_status.go +++ b/internal/controller/lifecycle/transformer_cluster_status.go @@ -427,20 +427,22 @@ func (t *ClusterStatusTransformer) removeStsInitContainerForRestore( var doRemoveInitContainers bool for _, vertex := range vertexList { v, _ := vertex.(*lifecycleVertex) - sts, _ := v.obj.(*appsv1.StatefulSet) - initContainers := sts.Spec.Template.Spec.InitContainers + if v.oriObj == nil { + continue + } + originSts, _ := v.oriObj.(*appsv1.StatefulSet) + initContainers := originSts.Spec.Template.Spec.InitContainers restoreInitContainerName := component.GetRestoredInitContainerName(backupName) restoreInitContainerIndex, _ := intctrlutil.GetContainerByName(initContainers, restoreInitContainerName) if restoreInitContainerIndex == -1 { continue } + sts, _ := v.obj.(*appsv1.StatefulSet) doRemoveInitContainers = true initContainers = append(initContainers[:restoreInitContainerIndex], initContainers[restoreInitContainerIndex+1:]...) sts.Spec.Template.Spec.InitContainers = initContainers - if v.oriObj != nil { - v.immutable = false - v.action = actionPtr(UPDATE) - } + v.immutable = false + v.action = actionPtr(UPDATE) } if doRemoveInitContainers { // if need to remove init container, reset component to Creating. From 16e2bf7dd7c3dc545911bb4413052cbacb9ea387 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Tue, 25 Apr 2023 15:25:57 +0800 Subject: [PATCH 173/439] chore: refine kbcli for mongodb (#2927) --- deploy/mongodb-cluster/templates/replicaset.yaml | 16 ++++++++-------- deploy/mongodb-cluster/values.yaml | 16 ++++++++-------- deploy/mongodb/scripts/replicaset-post-start.tpl | 14 ++++++-------- deploy/mongodb/templates/clusterdefinition.yaml | 13 +++++++------ internal/controller/builder/builder.go | 15 ++++++++------- internal/sqlchannel/engine/mongodb.go | 2 +- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/deploy/mongodb-cluster/templates/replicaset.yaml b/deploy/mongodb-cluster/templates/replicaset.yaml index 5389319c6..f4d2d8fdf 100644 --- a/deploy/mongodb-cluster/templates/replicaset.yaml +++ b/deploy/mongodb-cluster/templates/replicaset.yaml @@ -17,14 +17,14 @@ spec: tolerations: {{ . | toYaml | nindent 4 }} {{- end }} componentSpecs: - - name: replicaset - componentDefRef: replicaset + - name: mongodb + componentDefRef: mongodb monitor: {{ $.Values.monitor.enabled }} - replicas: {{ $.Values.replicaset.replicas }} - {{- with $.Values.replicaset.tolerations }} + replicas: {{ $.Values.mongodb.replicas }} + {{- with $.Values.mongodb.tolerations }} tolerations: {{ .| toYaml | nindent 8 }} {{- end }} - {{- with $.Values.replicaset.resources }} + {{- with $.Values.mongodb.resources }} resources: limits: cpu: {{ .limits.cpu | quote }} @@ -33,15 +33,15 @@ spec: cpu: {{ .requests.cpu | quote }} memory: {{ .requests.memory | quote }} {{- end }} - {{- if $.Values.replicaset.persistence.enabled }} + {{- if $.Values.mongodb.persistence.enabled }} volumeClaimTemplates: - name: data # ref clusterdefinition components.containers.volumeMounts.name spec: - storageClassName: {{ $.Values.replicaset.persistence.data.storageClassName }} + storageClassName: {{ $.Values.mongodb.persistence.data.storageClassName }} accessModes: - ReadWriteOnce resources: requests: - storage: {{ $.Values.replicaset.persistence.data.size }} + storage: {{ $.Values.mongodb.persistence.data.size }} {{- end }} {{- end }} diff --git a/deploy/mongodb-cluster/values.yaml b/deploy/mongodb-cluster/values.yaml index 8b0007ece..1dc790be3 100644 --- a/deploy/mongodb-cluster/values.yaml +++ b/deploy/mongodb-cluster/values.yaml @@ -87,14 +87,14 @@ shard: ## tolerations: [ ] -replicaset: - ## @param replicaset.replicas Number of MongoDB replicas per replicaset to deploy +mongodb: + ## @param mongodb.replicas Number of MongoDB replicas per replicaset to deploy ## replicas: 3 ## MongoDB workload pod resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ - ## @param replicaset.resources.limits The resources limits for the init container - ## @param replicaset.resources.requests The requested resources for the init container + ## @param mongodb.resources.limits The resources limits for the init container + ## @param mongodb.resources.requests The requested resources for the init container ## resources: { } # We usually recommend not to specify default resources and to leave this as a conscious @@ -111,13 +111,13 @@ replicaset: ## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/ ## persistence: - ## @param replicaset.persistence.enabled Enable persistence using Persistent Volume Claims + ## @param mongodb.persistence.enabled Enable persistence using Persistent Volume Claims ## enabled: true ## `data` volume settings ## data: - ## @param replicaset.persistence.data.storageClassName Storage class of backing PVC + ## @param mongodb.persistence.data.storageClassName Storage class of backing PVC ## If defined, storageClassName: ## If set to "-", storageClassName: "", which disables dynamic provisioning ## If undefined (the default) or set to null, no storageClassName spec is @@ -125,10 +125,10 @@ replicaset: ## GKE, AWS & OpenStack) ## storageClassName: - ## @param replicaset.persistence.size Size of data volume + ## @param mongodb.persistence.size Size of data volume ## size: 20Gi - ## @param replicaset.tolerations Tolerations for MongoDB pods assignment + ## @param mongodb.tolerations Tolerations for MongoDB pods assignment ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## tolerations: [ ] diff --git a/deploy/mongodb/scripts/replicaset-post-start.tpl b/deploy/mongodb/scripts/replicaset-post-start.tpl index 88ef808b6..d4c79463a 100644 --- a/deploy/mongodb/scripts/replicaset-post-start.tpl +++ b/deploy/mongodb/scripts/replicaset-post-start.tpl @@ -41,14 +41,12 @@ CONFIGSVR="" if [ ""$IS_CONFIGSVR = "true" ]; then CONFIGSVR="configsvr: true,"; fi until is_inited=$(mongosh --quiet --port $PORT --eval "rs.status().ok" -u root --password $MONGODB_ROOT_PASSWORD || mongosh --quiet --port $PORT --eval "try { rs.status().ok } catch (e) { 0 }") ; do sleep 1; done -if [ $is_inited -eq 1 ]; then - exit 0 +if [ $is_inited -ne 1 ]; then + sleep 10 + set -e + mongosh --quiet --port $PORT --eval "rs.initiate({_id: \"$RPL_SET_NAME\", $CONFIGSVR members: [$MEMBERS]})"; + set +e fi; -sleep 10 -set -e -mongosh --quiet --port $PORT --eval "rs.initiate({_id: \"$RPL_SET_NAME\", $CONFIGSVR members: [$MEMBERS]})"; -set +e (until mongosh --quiet --port $PORT --eval "rs.isMaster().isWritablePrimary"|grep true; do sleep 1; done; -echo "create user"; -mongosh --quiet --port $PORT admin --eval "db.createUser({ user: \"$MONGODB_ROOT_USER\", pwd: \"$MONGODB_ROOT_PASSWORD\", roles: [{role: 'root', db: 'admin'}] })") /dev/null 2>&1 & +mongosh --quiet --port $PORT admin --eval "db.createUser({ user: '$MONGODB_ROOT_USER', pwd: '$MONGODB_ROOT_PASSWORD', roles: [{role: 'root', db: 'admin'}] })") /dev/null 2>&1 & diff --git a/deploy/mongodb/templates/clusterdefinition.yaml b/deploy/mongodb/templates/clusterdefinition.yaml index 6a5d3eb38..5e92ae88d 100644 --- a/deploy/mongodb/templates/clusterdefinition.yaml +++ b/deploy/mongodb/templates/clusterdefinition.yaml @@ -9,12 +9,12 @@ spec: connectionCredential: username: root password: {{ (include "mongodb.password" .) | quote }} - endpoint: "$(SVC_FQDN):$(SVC_PORT_tcp-monogdb)" + endpoint: "$(SVC_FQDN):$(SVC_PORT_mongodb)" host: "$(SVC_FQDN)" - port: "$(SVC_PORT_tcp-monogdb)" - headlessEndpoint: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN):$(SVC_PORT_tcp-monogdb)" - headlessHost: "$(POD_NAME_PREFIX)-0.$(HEADLESS_SVC_FQDN)" - headlessPort: "$(SVC_PORT_tcp-monogdb)" + port: "$(SVC_PORT_mongodb)" + headlessEndpoint: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN):$(SVC_PORT_mongodb)" + headlessHost: "$(KB_CLUSTER_COMP_NAME)-0.$(HEADLESS_SVC_FQDN)" + headlessPort: "$(SVC_PORT_mongodb)" componentDefs: - name: mongodb characterType: mongodb @@ -64,7 +64,8 @@ spec: timeoutSeconds: {{ .Values.roleProbe.timeoutSeconds }} service: ports: - - protocol: TCP + - name: mongodb + protocol: TCP port: 27017 volumeTypes: - name: data diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index a85a1f23f..15e3bc417 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -346,13 +346,14 @@ func BuildConnCredential(params BuilderParams) (*corev1.Secret, error) { uuidStrB64 := base64.RawStdEncoding.EncodeToString([]byte(strings.ReplaceAll(uuidStr, "-", ""))) uuidHex := hex.EncodeToString(uuidBytes) m := map[string]string{ - "$(RANDOM_PASSWD)": randomString(8), - "$(UUID)": uuidStr, - "$(UUID_B64)": uuidB64, - "$(UUID_STR_B64)": uuidStrB64, - "$(UUID_HEX)": uuidHex, - "$(SVC_FQDN)": fmt.Sprintf("%s-%s.%s.svc", params.Cluster.Name, params.Component.Name, params.Cluster.Namespace), - "$(HEADLESS_SVC_FQDN)": fmt.Sprintf("%s-%s-headless.%s.svc", params.Cluster.Name, params.Component.Name, params.Cluster.Namespace), + "$(RANDOM_PASSWD)": randomString(8), + "$(UUID)": uuidStr, + "$(UUID_B64)": uuidB64, + "$(UUID_STR_B64)": uuidStrB64, + "$(UUID_HEX)": uuidHex, + "$(SVC_FQDN)": fmt.Sprintf("%s-%s.%s.svc", params.Cluster.Name, params.Component.Name, params.Cluster.Namespace), + "$(KB_CLUSTER_COMP_NAME)": params.Cluster.Name + "-" + params.Component.Name, + "$(HEADLESS_SVC_FQDN)": fmt.Sprintf("%s-%s-headless.%s.svc", params.Cluster.Name, params.Component.Name, params.Cluster.Namespace), } if len(params.Component.Services) > 0 { for _, p := range params.Component.Services[0].Spec.Ports { diff --git a/internal/sqlchannel/engine/mongodb.go b/internal/sqlchannel/engine/mongodb.go index a140c71be..a88bb9c64 100644 --- a/internal/sqlchannel/engine/mongodb.go +++ b/internal/sqlchannel/engine/mongodb.go @@ -50,7 +50,7 @@ func (r mongodb) ConnectCommand(connectInfo *AuthInfo) []string { userPass = connectInfo.UserPasswd } - mongodbCmd := []string{fmt.Sprintf("%s -u %s -p %s", r.info.Client, userName, userPass)} + mongodbCmd := []string{fmt.Sprintf("%s mongodb://%s:%s@$KB_POD_FQDN:27017/admin?replicaSet=$KB_CLUSTER_COMP_NAME", r.info.Client, userName, userPass)} return []string{"sh", "-c", strings.Join(mongodbCmd, " ")} } From f44c49b222290b312e7876edcb8e08d3e9c457ac Mon Sep 17 00:00:00 2001 From: free6om Date: Tue, 25 Apr 2023 15:32:12 +0800 Subject: [PATCH 174/439] fix: wrong viper setting in ut (#2937) --- controllers/apps/cluster_controller_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index bd70e1d44..914dcba5d 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1138,6 +1138,7 @@ var _ = Describe("Cluster Controller", func() { testBackupError := func() { initialReplicas := int32(1) updatedReplicas := int32(3) + viper.Set("VOLUMESNAPSHOT", true) By("Creating a cluster with VolumeClaimTemplate") pvcSpec := testapps.NewPVCSpec("1Gi") @@ -1175,7 +1176,7 @@ var _ = Describe("Cluster Controller", func() { } Expect(testCtx.Create(ctx, &backup)).Should(Succeed()) - By("Checking backup status to failed, because VolumeSnapshot disabled") + By("Checking backup status to failed, because pvc not exist") Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, backup *dataprotectionv1alpha1.Backup) { g.Expect(backup.Status.Phase).Should(Equal(dataprotectionv1alpha1.BackupFailed)) })).Should(Succeed()) From fc0d16f23f1326db44b709558bac778a14d26623 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Tue, 25 Apr 2023 15:56:49 +0800 Subject: [PATCH 175/439] chore: config committer identity (#2940) --- .github/workflows/cicd-push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index 899ee428a..f359ad76a 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -35,6 +35,8 @@ jobs: - name: merge releasing to release if: ${{ startsWith(github.ref_name, 'releasing-') }} run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" bash .github/utils/merge_releasing_branch.sh pre-push: From 63ec2ad4891b84eb9c4ba7219e540e32ae8e2330 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Tue, 25 Apr 2023 16:16:21 +0800 Subject: [PATCH 176/439] chore: adjust go.mod and import order (#2941) --- go.mod | 2 +- internal/controller/lifecycle/transformer_workloads_last.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index a1cf36398..94496b182 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 github.com/onsi/ginkgo/v2 v2.7.0 github.com/onsi/gomega v1.25.0 + github.com/opencontainers/image-spec v1.1.0-rc2 github.com/pingcap/go-tpc v1.0.9 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 @@ -271,7 +272,6 @@ require ( github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opencontainers/selinux v1.10.2 // indirect diff --git a/internal/controller/lifecycle/transformer_workloads_last.go b/internal/controller/lifecycle/transformer_workloads_last.go index 6910bfb92..dea16ee1a 100644 --- a/internal/controller/lifecycle/transformer_workloads_last.go +++ b/internal/controller/lifecycle/transformer_workloads_last.go @@ -17,11 +17,11 @@ limitations under the License. package lifecycle import ( - "github.com/apecloud/kubeblocks/internal/constant" appv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/util/sets" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/graph" ) From 601f6e8afc4c92d6af901c6eb306ae521808cf18 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Tue, 25 Apr 2023 16:16:42 +0800 Subject: [PATCH 177/439] fix: cli list-instances throw error when node is not found (#2936) --- Makefile | 5 +- internal/cli/cluster/cluster.go | 16 ++++-- internal/cli/cluster/cluster_test.go | 77 +++++++++++++++++----------- internal/cli/util/util.go | 2 +- internal/cli/util/util_test.go | 5 +- 5 files changed, 65 insertions(+), 40 deletions(-) diff --git a/Makefile b/Makefile index b1bff4be6..f35e1a69f 100644 --- a/Makefile +++ b/Makefile @@ -137,8 +137,6 @@ ifeq ($(SKIP_GO_GEN), false) $(GO) generate -x ./internal/configuration/proto endif - - .PHONY: test-go-generate test-go-generate: ## Run go generate against test code. $(GO) generate -x ./internal/testutil/k8s/mocks/... @@ -269,7 +267,7 @@ kbcli: test-go-generate build-checks kbcli-fast ## Build bin/kbcli. clean-kbcli: ## Clean bin/kbcli*. rm -f bin/kbcli* -.PHONY: doc +.PHONY: kbcli-doc kbcli-doc: generate ## generate CLI command reference manual. $(GO) run ./hack/docgen/cli/main.go ./docs/user_docs/cli @@ -763,4 +761,3 @@ test-e2e: helm-package render-smoke-testdata-manifests ## Run E2E tests. # NOTE: include must be placed at the end include docker/docker.mk include cmd/cmd.mk - diff --git a/internal/cli/cluster/cluster.go b/internal/cli/cluster/cluster.go index cb17ee5e9..6695f145c 100644 --- a/internal/cli/cluster/cluster.go +++ b/internal/cli/cluster/cluster.go @@ -25,6 +25,7 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -88,6 +89,7 @@ func listResources[T any](dynamic dynamic.Interface, gvr schema.GroupVersionReso // Get all kubernetes objects belonging to the database cluster func (o *ObjectsGetter) Get() (*ClusterObjects, error) { var err error + objs := NewClusterObjects() ctx := context.TODO() corev1 := o.Client.CoreV1() @@ -184,10 +186,13 @@ func (o *ObjectsGetter) Get() (*ClusterObjects, error) { } node, err := corev1.Nodes().Get(ctx, nodeName, metav1.GetOptions{}) - if err != nil { + if err != nil && !apierrors.IsNotFound(err) { return nil, err } - objs.Nodes = append(objs.Nodes, node) + + if node != nil { + objs.Nodes = append(objs.Nodes, node) + } } } @@ -211,15 +216,16 @@ func (o *ObjectsGetter) Get() (*ClusterObjects, error) { } } } + if o.WithDataProtection { - dplistOpts := metav1.ListOptions{ + dpListOpts := metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s", constant.AppInstanceLabelKey, o.Name), } - if err := listResources(o.Dynamic, types.BackupPolicyGVR(), o.Namespace, dplistOpts, &objs.BackupPolicies); err != nil { + if err = listResources(o.Dynamic, types.BackupPolicyGVR(), o.Namespace, dpListOpts, &objs.BackupPolicies); err != nil { return nil, err } - if err := listResources(o.Dynamic, types.BackupGVR(), o.Namespace, dplistOpts, &objs.Backups); err != nil { + if err = listResources(o.Dynamic, types.BackupGVR(), o.Namespace, dpListOpts, &objs.Backups); err != nil { return nil, err } } diff --git a/internal/cli/cluster/cluster_test.go b/internal/cli/cluster/cluster_test.go index 1a588a467..43bc74fa4 100644 --- a/internal/cli/cluster/cluster_test.go +++ b/internal/cli/cluster/cluster_test.go @@ -23,52 +23,71 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" + "github.com/apecloud/kubeblocks/internal/cli/testing" ) var _ = Describe("cluster util", func() { - client := testing.FakeClientSet( + baseObjs := []runtime.Object{ testing.FakePods(3, testing.Namespace, testing.ClusterName), - testing.FakeNode(), testing.FakeSecrets(testing.Namespace, testing.ClusterName), testing.FakeServices(), - testing.FakePVCs()) + testing.FakePVCs(), + } dynamic := testing.FakeDynamicClient( testing.FakeCluster(testing.ClusterName, testing.Namespace), testing.FakeClusterDef(), testing.FakeClusterVersion()) + getOptions := GetOptions{ + WithClusterDef: true, + WithClusterVersion: true, + WithConfigMap: true, + WithService: true, + WithSecret: true, + WithPVC: true, + WithPod: true, + } + It("get cluster objects", func() { - clusterName := testing.ClusterName - getter := ObjectsGetter{ - Client: client, - Dynamic: dynamic, - Name: clusterName, - Namespace: testing.Namespace, - GetOptions: GetOptions{ - WithClusterDef: true, - WithClusterVersion: true, - WithConfigMap: true, - WithService: true, - WithSecret: true, - WithPVC: true, - WithPod: true, - }, + var ( + err error + objs *ClusterObjects + ) + + testFn := func(client kubernetes.Interface) { + clusterName := testing.ClusterName + getter := ObjectsGetter{ + Client: client, + Dynamic: dynamic, + Name: clusterName, + Namespace: testing.Namespace, + GetOptions: getOptions, + } + + objs, err = getter.Get() + Expect(err).Should(Succeed()) + Expect(objs).ShouldNot(BeNil()) + Expect(objs.Cluster.Name).Should(Equal(clusterName)) + Expect(objs.ClusterDef.Name).Should(Equal(testing.ClusterDefName)) + Expect(objs.ClusterVersion.Name).Should(Equal(testing.ClusterVersionName)) + Expect(len(objs.Pods.Items)).Should(Equal(3)) + Expect(len(objs.Secrets.Items)).Should(Equal(1)) + Expect(len(objs.Services.Items)).Should(Equal(4)) + Expect(len(objs.PVCs.Items)).Should(Equal(1)) + Expect(len(objs.GetComponentInfo())).Should(Equal(1)) } - objs, err := getter.Get() - Expect(err).Should(Succeed()) - Expect(objs).ShouldNot(BeNil()) - Expect(objs.Cluster.Name).Should(Equal(clusterName)) - Expect(objs.ClusterDef.Name).Should(Equal(testing.ClusterDefName)) - Expect(objs.ClusterVersion.Name).Should(Equal(testing.ClusterVersionName)) + By("when node is not found") + testFn(testing.FakeClientSet(baseObjs...)) + Expect(len(objs.Nodes)).Should(Equal(0)) - Expect(len(objs.Pods.Items)).Should(Equal(3)) + By("when node is available") + baseObjs = append(baseObjs, testing.FakeNode()) + testFn(testing.FakeClientSet(baseObjs...)) Expect(len(objs.Nodes)).Should(Equal(1)) - Expect(len(objs.Secrets.Items)).Should(Equal(1)) - Expect(len(objs.Services.Items)).Should(Equal(4)) - Expect(len(objs.PVCs.Items)).Should(Equal(1)) - Expect(len(objs.GetComponentInfo())).Should(Equal(1)) }) }) diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index ddf7d3712..1127e7f42 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -263,7 +263,7 @@ func GetNodeByName(nodes []*corev1.Node, name string) *corev1.Node { return node } } - return &corev1.Node{} + return nil } // ResourceIsEmpty check if resource is empty or not diff --git a/internal/cli/util/util_test.go b/internal/cli/util/util_test.go index 8588409e0..b85c554e0 100644 --- a/internal/cli/util/util_test.go +++ b/internal/cli/util/util_test.go @@ -111,7 +111,10 @@ var _ = Describe("util", func() { testFn := func(name string) bool { n := GetNodeByName(nodes, name) - return n.Name == name + if n != nil { + return n.Name == name + } + return false } Expect(testFn("test")).Should(BeTrue()) Expect(testFn("non-exists")).Should(BeFalse()) From d7bb8dcdcfeca2ed001b856572325bd1aa5a5280 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Tue, 25 Apr 2023 16:24:04 +0800 Subject: [PATCH 178/439] fix: recovery use start recoverable time failed (#2942) --- internal/controller/plan/pitr.go | 28 +++++++++++++++++++-------- internal/controller/plan/pitr_test.go | 21 +++++++++++++------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/internal/controller/plan/pitr.go b/internal/controller/plan/pitr.go index 121eed9e1..7a71d9733 100644 --- a/internal/controller/plan/pitr.go +++ b/internal/controller/plan/pitr.go @@ -162,12 +162,6 @@ func (p *PointInTimeRecoveryManager) doPrepare(component *component.SynthesizedC } else if !need { return nil } - // prepare init container - container := corev1.Container{} - container.Name = initContainerName - container.Image = viper.GetString(constant.KBToolsImage) - container.Command = []string{"sleep", "infinity"} - component.PodSpec.InitContainers = append(component.PodSpec.InitContainers, container) // prepare data pvc if len(component.VolumeClaimTemplates) == 0 { @@ -177,6 +171,24 @@ func (p *PointInTimeRecoveryManager) doPrepare(component *component.SynthesizedC if err != nil { return err } + + // recovery time start time boundary processing, this scenario is converted to back up recovery function + if latestBackup.Status.Manifests.BackupLog.StopTime.Format(time.RFC3339) == p.restoreTime.Format(time.RFC3339) { + if latestBackup.Spec.BackupType == dpv1alpha1.BackupTypeSnapshot { + delete(p.Cluster.Annotations, constant.RestoreFromSrcClusterAnnotationKey) + delete(p.Cluster.Annotations, constant.RestoreFromTimeAnnotationKey) + return p.doPrepareSnapshotBackup(component, latestBackup) + } + // TODO: support restore with full backup. + return nil + } + // prepare init container + container := corev1.Container{} + container.Name = initContainerName + container.Image = viper.GetString(constant.KBToolsImage) + container.Command = []string{"sleep", "infinity"} + component.PodSpec.InitContainers = append(component.PodSpec.InitContainers, container) + if latestBackup.Spec.BackupType == dpv1alpha1.BackupTypeSnapshot { return p.doPrepareSnapshotBackup(component, latestBackup) } @@ -250,8 +262,8 @@ func (p *PointInTimeRecoveryManager) getLatestBaseBackup() (*dpv1alpha1.Backup, // 2. get the latest backup object var latestBackup *dpv1alpha1.Backup for _, item := range backups { - if item.Status.Manifests.BackupLog.StopTime != nil && - p.restoreTime.After(item.Status.Manifests.BackupLog.StopTime.Time) { + if item.Spec.BackupType != dpv1alpha1.BackupTypeIncremental && + item.Status.Manifests.BackupLog.StopTime != nil && !p.restoreTime.Before(item.Status.Manifests.BackupLog.StopTime) { latestBackup = &item break } diff --git a/internal/controller/plan/pitr_test.go b/internal/controller/plan/pitr_test.go index 1aa46ff6a..937ebf66e 100644 --- a/internal/controller/plan/pitr_test.go +++ b/internal/controller/plan/pitr_test.go @@ -49,6 +49,10 @@ var _ = Describe("PITR Functions", func() { randomStr = testCtx.GetRandomStr() clusterName = "cluster-for-pitr-" + randomStr backupToolName string + + now = metav1.Now() + startTime = metav1.Time{Time: now.Add(-time.Hour * 2)} + stopTime = metav1.Time{Time: now.Add(time.Hour * 2)} ) cleanEnv := func() { @@ -110,7 +114,7 @@ var _ = Describe("PITR Functions", func() { clusterDef.Name, clusterVersion.Name). AddComponent(mysqlCompName, mysqlCompType). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - AddRestorePointInTime(metav1.Time{Time: metav1.Now().Time}, sourceCluster). + AddRestorePointInTime(metav1.Time{Time: stopTime.Time}, sourceCluster). Create(&testCtx).GetObject() By("By mocking a pvc") @@ -156,7 +160,6 @@ var _ = Describe("PITR Functions", func() { } By("By creating base backup: ") - now := metav1.Now() backupLabels := map[string]string{ constant.AppInstanceLabelKey: sourceCluster, constant.KBAppComponentLabelKey: mysqlCompName, @@ -167,8 +170,8 @@ var _ = Describe("PITR Functions", func() { SetBackupPolicyName("test-fake"). SetBackupType(dpv1alpha1.BackupTypeSnapshot). Create(&testCtx).GetObject() - baseStartTime := &metav1.Time{Time: now.Add(-time.Hour * 3)} - baseStopTime := &metav1.Time{Time: now.Add(-time.Hour * 2)} + baseStartTime := &startTime + baseStopTime := &now backupStatus := dpv1alpha1.BackupStatus{ Phase: dpv1alpha1.BackupCompleted, StartTimestamp: baseStartTime, @@ -180,7 +183,6 @@ var _ = Describe("PITR Functions", func() { }, }, } - backupStatus.CompletionTimestamp = &metav1.Time{Time: now.Add(-time.Hour * 2)} patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backup)) By("By creating remote pvc: ") @@ -195,8 +197,8 @@ var _ = Describe("PITR Functions", func() { constant.KBAppComponentLabelKey: mysqlCompName, constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeIncremental), } - incrStartTime := &metav1.Time{Time: now.Add(-time.Hour * 3)} - incrStopTime := &metav1.Time{Time: now.Add(time.Hour * 2)} + incrStartTime := &startTime + incrStopTime := &stopTime backupIncr := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). WithRandomName().SetLabels(incrBackupLabels). SetBackupPolicyName("test-fake"). @@ -219,8 +221,13 @@ var _ = Describe("PITR Functions", func() { }) It("Test PITR prepare", func() { + By("restore time is in range") Expect(DoPITRPrepare(ctx, testCtx.Cli, cluster, synthesizedComponent)).Should(Succeed()) Expect(synthesizedComponent.PodSpec.InitContainers).ShouldNot(BeEmpty()) + + By("restore time is at base backup stop time") + cluster.Annotations[constant.RestoreFromTimeAnnotationKey] = now.Format(time.RFC3339) + Expect(DoPITRPrepare(ctx, testCtx.Cli, cluster, synthesizedComponent)).Should(Succeed()) }) It("Test PITR job run and cleanup", func() { By("when data pvc is pending") From 7dc659f3686d083b80b0699811712838406d16f2 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Tue, 25 Apr 2023 17:01:23 +0800 Subject: [PATCH 179/439] fix: redis edit-config and configure behave inconsistently (#2906) --- internal/unstructured/redis_config.go | 4 ++-- internal/unstructured/redis_config_test.go | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/unstructured/redis_config.go b/internal/unstructured/redis_config.go index a12943e35..ff7bb5170 100644 --- a/internal/unstructured/redis_config.go +++ b/internal/unstructured/redis_config.go @@ -50,7 +50,7 @@ func (r *redisConfig) Update(key string, value any) error { } func (r *redisConfig) setString(key string, value string) error { - keys := strings.Split(key, ".") + keys := strings.Split(key, " ") v := r.GetItem(keys) lineNo := math.MaxInt32 if v != nil { @@ -107,7 +107,7 @@ func (r *redisConfig) GetItem(keys []string) *redis.Item { } func (r *redisConfig) GetString(key string) (string, error) { - keys := strings.Split(key, ".") + keys := strings.Split(key, " ") item := r.GetItem(keys) if item == nil { return "", nil diff --git a/internal/unstructured/redis_config_test.go b/internal/unstructured/redis_config_test.go index 7920726d6..38dc74b40 100644 --- a/internal/unstructured/redis_config_test.go +++ b/internal/unstructured/redis_config_test.go @@ -58,21 +58,21 @@ func TestRedisConfig(t *testing.T) { valueArgs: "256mb 64mb 60", wantErr: false, testKey: map[string]string{ - "client-output-buffer-limit.pubsub": "256mb 64mb 60", + "client-output-buffer-limit pubsub": "256mb 64mb 60", }, }, { keyArgs: []string{"client-output-buffer-limit", "normal"}, valueArgs: "128mb 32mb 0", wantErr: false, testKey: map[string]string{ - "client-output-buffer-limit.normal": "128mb 32mb 0", - "client-output-buffer-limit.pubsub": "256mb 64mb 60", + "client-output-buffer-limit normal": "128mb 32mb 0", + "client-output-buffer-limit pubsub": "256mb 64mb 60", "port": "6379", }, }} for _, tt := range tests { t.Run("config_test", func(t *testing.T) { - if err := c.Update(strings.Join(tt.keyArgs, "."), tt.valueArgs); (err != nil) != tt.wantErr { + if err := c.Update(strings.Join(tt.keyArgs, " "), tt.valueArgs); (err != nil) != tt.wantErr { t.Errorf("Update() error = %v, wantErr %v", err, tt.wantErr) } @@ -94,14 +94,14 @@ func TestRedisConfigGetAllParameters(t *testing.T) { fn mockfn want map[string]interface{} }{{ - name: "xxx", + name: "multi field update test", fn: func() ConfigObject { c, _ := LoadConfig("test", "", appsv1alpha1.RedisCfg) _ = c.Update("port", "123") - _ = c.Update("a.b", "123 234") - _ = c.Update("a.c", "345") - _ = c.Update("a.d", "1 2") - _ = c.Update("a.d.e", "1 2") + _ = c.Update("a b", "123 234") + _ = c.Update("a c", "345") + _ = c.Update("a d", "1 2") + _ = c.Update("a d e", "1 2") return c }, want: map[string]interface{}{ From bd44ecb2ed636c09725dcd1dcb3a87942129b32d Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 25 Apr 2023 21:39:22 +0800 Subject: [PATCH 180/439] chore: fix update pod annotation when pod is deleting (#2954) --- controllers/apps/components/deployment_controller.go | 5 +++++ controllers/apps/components/pod_controller.go | 5 +++++ controllers/apps/components/stateful_set_controller.go | 5 +++++ controllers/k8score/pvc_controller.go | 5 +++++ 4 files changed, 20 insertions(+) diff --git a/controllers/apps/components/deployment_controller.go b/controllers/apps/components/deployment_controller.go index 2bd4b22c4..dabdfb592 100644 --- a/controllers/apps/components/deployment_controller.go +++ b/controllers/apps/components/deployment_controller.go @@ -68,6 +68,11 @@ func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } + // skip if deploy is being deleted + if !deploy.DeletionTimestamp.IsZero() { + return intctrlutil.Reconciled() + } + return workloadCompClusterReconcile(reqCtx, r.Client, deploy, func(cluster *appsv1alpha1.Cluster, componentSpec *appsv1alpha1.ClusterComponentSpec, component types.Component) (ctrl.Result, error) { compCtx := newComponentContext(reqCtx, r.Client, r.Recorder, component, deploy, componentSpec) diff --git a/controllers/apps/components/pod_controller.go b/controllers/apps/components/pod_controller.go index 38ca74f42..d794fdffb 100644 --- a/controllers/apps/components/pod_controller.go +++ b/controllers/apps/components/pod_controller.go @@ -72,6 +72,11 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } + // skip if pod is being deleted + if !pod.DeletionTimestamp.IsZero() { + return intctrlutil.Reconciled() + } + if cluster, err = util.GetClusterByObject(reqCtx.Ctx, r.Client, pod); err != nil { return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } diff --git a/controllers/apps/components/stateful_set_controller.go b/controllers/apps/components/stateful_set_controller.go index 75821c8b3..291162136 100644 --- a/controllers/apps/components/stateful_set_controller.go +++ b/controllers/apps/components/stateful_set_controller.go @@ -68,6 +68,11 @@ func (r *StatefulSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } + // skip if sts is being deleted + if !sts.DeletionTimestamp.IsZero() { + return intctrlutil.Reconciled() + } + return workloadCompClusterReconcile(reqCtx, r.Client, sts, func(cluster *appsv1alpha1.Cluster, componentSpec *appsv1alpha1.ClusterComponentSpec, component types.Component) (ctrl.Result, error) { compCtx := newComponentContext(reqCtx, r.Client, r.Recorder, component, sts, componentSpec) diff --git a/controllers/k8score/pvc_controller.go b/controllers/k8score/pvc_controller.go index db6b8698d..96132e441 100644 --- a/controllers/k8score/pvc_controller.go +++ b/controllers/k8score/pvc_controller.go @@ -67,6 +67,11 @@ func (r *PersistentVolumeClaimReconciler) Reconcile(ctx context.Context, req ctr return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "getPVCError") } + // skip if pvc is being deleted + if !pvc.DeletionTimestamp.IsZero() { + return intctrlutil.Reconciled() + } + for _, handlePVC := range PersistentVolumeClaimHandlerMap { // ignores the not found error. if err := handlePVC(reqCtx, r.Client, pvc); err != nil && !apierrors.IsNotFound(err) { From 87ae373a6a619daba5cddf7440d7a06379a37cec Mon Sep 17 00:00:00 2001 From: free6om Date: Tue, 25 Apr 2023 23:38:26 +0800 Subject: [PATCH 181/439] fix: mysql cluster execute ops VolumeExpansion Failed (#2950) --- apis/apps/v1alpha1/cluster_types.go | 52 +++++++++ .../apps/operations/volume_expansion.go | 30 +++++- .../apps/operations/volume_expansion_test.go | 100 ++++++++++-------- 3 files changed, 136 insertions(+), 46 deletions(-) diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index b955e7d10..1f78db199 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -477,6 +477,58 @@ func init() { SchemeBuilder.Register(&Cluster{}, &ClusterList{}) } +// GetVolumeClaimNames gets all PVC names of component compName +// +// r.Spec.GetComponentByName(compName).VolumeClaimTemplates[*].Name will be used if no claimNames provided +// +// nil return if: +// 1. component compName not found or +// 2. len(VolumeClaimTemplates)==0 or +// 3. any claimNames not found +func (r *Cluster) GetVolumeClaimNames(compName string, claimNames ...string) []string { + if r == nil { + return nil + } + comp := r.Spec.GetComponentByName(compName) + if comp == nil { + return nil + } + if len(comp.VolumeClaimTemplates) == 0 { + return nil + } + if len(claimNames) == 0 { + for _, template := range comp.VolumeClaimTemplates { + claimNames = append(claimNames, template.Name) + } + } + allExist := true + for _, name := range claimNames { + found := false + for _, template := range comp.VolumeClaimTemplates { + if template.Name == name { + found = true + break + } + } + if !found { + allExist = false + break + } + } + if !allExist { + return nil + } + + pvcNames := make([]string, 0) + for _, claimName := range claimNames { + for i := 0; i < int(comp.Replicas); i++ { + pvcName := fmt.Sprintf("%s-%s-%s-%d", claimName, r.Name, compName, i) + pvcNames = append(pvcNames, pvcName) + } + } + return pvcNames +} + // GetComponentByName gets component by name. func (r ClusterSpec) GetComponentByName(componentName string) *ClusterComponentSpec { for _, v := range r.ComponentSpecs { diff --git a/controllers/apps/operations/volume_expansion.go b/controllers/apps/operations/volume_expansion.go index ddca837e3..8f1d41cc4 100644 --- a/controllers/apps/operations/volume_expansion.go +++ b/controllers/apps/operations/volume_expansion.go @@ -22,6 +22,8 @@ package operations import ( "fmt" "reflect" + "regexp" + "strconv" "time" "github.com/pkg/errors" @@ -40,6 +42,8 @@ type volumeExpansionOpsHandler struct{} var _ OpsHandler = volumeExpansionOpsHandler{} +var pvcNameRegex = regexp.MustCompile("(.*)-([0-9]+)$") + const ( // VolumeExpansionTimeOut volume expansion timeout. VolumeExpansionTimeOut = 30 * time.Minute @@ -299,11 +303,25 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti }, client.InNamespace(opsRes.Cluster.Namespace)); err != nil { return } + comp := opsRes.Cluster.Spec.GetComponentByName(componentName) + if comp == nil { + err = fmt.Errorf("comp %s of cluster %s not found", componentName, opsRes.Cluster.Name) + return + } + expectCount = int(comp.Replicas) vctKey := getComponentVCTKey(componentName, vctName) requestStorage := storageMap[vctKey] - expectCount = len(pvcList.Items) var completedCount int + var ordinal int for _, v := range pvcList.Items { + // filter PVC(s) with ordinal larger than comp.Replicas - 1, which left by scale-in + ordinal, err = getPVCOrdinal(v.Name) + if err != nil { + return + } + if ordinal > expectCount-1 { + continue + } objectKey := getPVCProgressObjectKey(v.Name) progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey, Group: vctName} // if the volume expand succeed @@ -327,7 +345,7 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti completedCount += 1 } } - isCompleted = completedCount == len(pvcList.Items) + isCompleted = completedCount == expectCount return succeedCount, expectCount, isCompleted, nil } @@ -338,3 +356,11 @@ func getComponentVCTKey(componentName, vctName string) string { func getPVCProgressObjectKey(pvcName string) string { return fmt.Sprintf("PVC/%s", pvcName) } + +func getPVCOrdinal(pvcName string) (int, error) { + subMatches := pvcNameRegex.FindStringSubmatch(pvcName) + if len(subMatches) < 3 { + return 0, fmt.Errorf("wrong pvc name: %s", pvcName) + } + return strconv.Atoi(subMatches[2]) +} diff --git a/controllers/apps/operations/volume_expansion_test.go b/controllers/apps/operations/volume_expansion_test.go index c4b154e5e..6c9fcefac 100644 --- a/controllers/apps/operations/volume_expansion_test.go +++ b/controllers/apps/operations/volume_expansion_test.go @@ -83,7 +83,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { createPVC := func(clusterName, scName, vctName, pvcName string) { // Note: in real k8s cluster, it maybe fails when pvc created by k8s controller. testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterName, - consensusCompName, "data").SetStorage("2Gi").SetStorageClass(storageClassName).Create(&testCtx) + consensusCompName, "data").SetStorage("2Gi").SetStorageClass(storageClassName).CheckedCreate(&testCtx) } mockDoOperationOnCluster := func(cluster *appsv1alpha1.Cluster, opsRequestName string, opsType appsv1alpha1.OpsType) { @@ -99,7 +99,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { })).Should(Succeed()) } - initResourcesForVolumeExpansion := func(clusterObject *appsv1alpha1.Cluster, opsRes *OpsResource, index int) (*appsv1alpha1.OpsRequest, string) { + initResourcesForVolumeExpansion := func(clusterObject *appsv1alpha1.Cluster, opsRes *OpsResource, storage string, replicas int) (*appsv1alpha1.OpsRequest, []string) { currRandomStr := testCtx.GetRandomStr() ops := testapps.NewOpsRequestObj("volumeexpansion-ops-"+currRandomStr, testCtx.DefaultNamespace, clusterObject.Name, appsv1alpha1.VolumeExpansionType) @@ -109,7 +109,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { VolumeClaimTemplates: []appsv1alpha1.OpsRequestVolumeClaimTemplate{ { Name: vctName, - Storage: resource.MustParse("3Gi"), + Storage: resource.MustParse(storage), }, }, }, @@ -123,14 +123,22 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { mockDoOperationOnCluster(clusterObject, ops.Name, appsv1alpha1.VolumeExpansionType) // create-pvc - pvcName := fmt.Sprintf("%s-%s-%s-%d", vctName, clusterObject.Name, consensusCompName, index) - createPVC(clusterObject.Name, storageClassName, vctName, pvcName) + pvcNames := opsRes.Cluster.GetVolumeClaimNames(consensusCompName) + for _, pvcName := range pvcNames { + createPVC(clusterObject.Name, storageClassName, vctName, pvcName) + // trigger pvc controller reconcile if pvc already exists + pvcKey := client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace} + Expect(testapps.GetAndChangeObj(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Labels[testCtx.GetRandomStr()] = "trigger-reconcile" + })()).ShouldNot(HaveOccurred()) + } // waiting pvc controller mark annotation to OpsRequest Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(ops), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) { g.Expect(tmpOps.Annotations).ShouldNot(BeNil()) g.Expect(tmpOps.Annotations[constant.ReconcileAnnotationKey]).ShouldNot(BeEmpty()) })).Should(Succeed()) - return ops, pvcName + + return ops, pvcNames } mockVolumeExpansionActionAndReconcile := func(reqCtx intctrlutil.RequestCtx, opsRes *OpsResource, newOps *appsv1alpha1.OpsRequest) { @@ -150,7 +158,8 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { testWarningEventOnPVC := func(reqCtx intctrlutil.RequestCtx, clusterObject *appsv1alpha1.Cluster, opsRes *OpsResource) { // init resources for volume expansion - newOps, pvcName := initResourcesForVolumeExpansion(clusterObject, opsRes, 1) + comp := opsRes.Cluster.Spec.GetComponentByName(consensusCompName) + newOps, pvcNames := initResourcesForVolumeExpansion(clusterObject, opsRes, "4Gi", int(comp.Replicas)) By("mock run volumeExpansion action and reconcileAction") mockVolumeExpansionActionAndReconcile(reqCtx, opsRes, newOps) @@ -164,7 +173,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { Message: "You've reached the maximum modification rate per volume limit. Wait at least 6 hours between modifications per EBS volume.", } stsInvolvedObject := corev1.ObjectReference{ - Name: pvcName, + Name: pvcNames[0], Kind: constant.PersistentVolumeClaimKind, Namespace: "default", } @@ -181,7 +190,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(newOps), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) { progressDetails := tmpOps.Status.Components[consensusCompName].ProgressDetails g.Expect(len(progressDetails) > 0).Should(BeTrue()) - progressDetail := findStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcName)) + progressDetail := findStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcNames[0])) g.Expect(progressDetail.Status == appsv1alpha1.FailedProgressStatus).Should(BeTrue()) })).Should(Succeed()) } @@ -193,44 +202,47 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { })).ShouldNot(HaveOccurred()) // init resources for volume expansion - newOps, pvcName := initResourcesForVolumeExpansion(clusterObject, opsRes, 0) + comp := clusterObject.Spec.GetComponentByName(consensusCompName) + newOps, pvcNames := initResourcesForVolumeExpansion(clusterObject, opsRes, "3Gi", int(comp.Replicas)) By("mock run volumeExpansion action and reconcileAction") mockVolumeExpansionActionAndReconcile(reqCtx, opsRes, newOps) By("mock pvc is resizing") - pvcKey := client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace} - Expect(testapps.GetAndChangeObjStatus(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Status.Conditions = []corev1.PersistentVolumeClaimCondition{{ - Type: corev1.PersistentVolumeClaimResizing, - Status: corev1.ConditionTrue, - LastTransitionTime: metav1.Now(), - }, - } - })()).ShouldNot(HaveOccurred()) - - Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { - conditions := tmpPVC.Status.Conditions - g.Expect(len(conditions) > 0 && conditions[0].Type == corev1.PersistentVolumeClaimResizing).Should(BeTrue()) - })).Should(Succeed()) - - // waiting OpsRequest.status.components["consensus"].vct["data"] is running - _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) - Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(newOps), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) { - progressDetails := tmpOps.Status.Components[consensusCompName].ProgressDetails - progressDetail := findStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcName)) - g.Expect(progressDetail != nil && progressDetail.Status == appsv1alpha1.ProcessingProgressStatus).Should(BeTrue()) - })).Should(Succeed()) - - By("mock pvc resizing succeed") - // mock pvc volumeExpansion succeed - Expect(testapps.GetAndChangeObjStatus(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Status.Capacity = corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("3Gi")} - })()).ShouldNot(HaveOccurred()) - - Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { - g.Expect(tmpPVC.Status.Capacity[corev1.ResourceStorage] == resource.MustParse("3Gi")).Should(BeTrue()) - })).Should(Succeed()) + for _, pvcName := range pvcNames { + pvcKey := client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace} + Expect(testapps.GetAndChangeObjStatus(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Status.Conditions = []corev1.PersistentVolumeClaimCondition{{ + Type: corev1.PersistentVolumeClaimResizing, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.Now(), + }, + } + })()).ShouldNot(HaveOccurred()) + + Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { + conditions := tmpPVC.Status.Conditions + g.Expect(len(conditions) > 0 && conditions[0].Type == corev1.PersistentVolumeClaimResizing).Should(BeTrue()) + })).Should(Succeed()) + + // waiting OpsRequest.status.components["consensus"].vct["data"] is running + _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) + Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(newOps), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) { + progressDetails := tmpOps.Status.Components[consensusCompName].ProgressDetails + progressDetail := findStatusProgressDetail(progressDetails, getPVCProgressObjectKey(pvcName)) + g.Expect(progressDetail != nil && progressDetail.Status == appsv1alpha1.ProcessingProgressStatus).Should(BeTrue()) + })).Should(Succeed()) + + By("mock pvc resizing succeed") + // mock pvc volumeExpansion succeed + Expect(testapps.GetAndChangeObjStatus(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Status.Capacity = corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("3Gi")} + })()).ShouldNot(HaveOccurred()) + + Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { + g.Expect(tmpPVC.Status.Capacity[corev1.ResourceStorage] == resource.MustParse("3Gi")).Should(BeTrue()) + })).Should(Succeed()) + } // waiting OpsRequest.status.phase is succeed _, err := GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) @@ -242,7 +254,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { testDeleteRunningVolumeExpansion := func(clusterObject *appsv1alpha1.Cluster, opsRes *OpsResource) { // init resources for volume expansion - newOps, pvcName := initResourcesForVolumeExpansion(clusterObject, opsRes, 2) + newOps, pvcNames := initResourcesForVolumeExpansion(clusterObject, opsRes, "5Gi", 1) Expect(testapps.ChangeObjStatus(&testCtx, clusterObject, func() { clusterObject.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase })).ShouldNot(HaveOccurred()) @@ -253,7 +265,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { By("test handle the invalid volumeExpansion OpsRequest") pvc := &corev1.PersistentVolumeClaim{} - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace}, pvc)).Should(Succeed()) + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: pvcNames[0], Namespace: testCtx.DefaultNamespace}, pvc)).Should(Succeed()) Expect(handleVolumeExpansionWithPVC(intctrlutil.RequestCtx{Ctx: ctx}, k8sClient, pvc)).Should(Succeed()) Eventually(testapps.GetClusterPhase(&testCtx, client.ObjectKeyFromObject(clusterObject))).Should(Equal(appsv1alpha1.RunningClusterPhase)) From 73d10d7568977d230f310a9beba02ed5873a6973 Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Wed, 26 Apr 2023 09:05:18 +0800 Subject: [PATCH 182/439] feat: add --set storageClass for create cluster and validation with UTs (#2835) --- internal/cli/cmd/cluster/cluster_test.go | 49 +++++++++++++++++- internal/cli/cmd/cluster/create.go | 65 +++++++++++++++++++++++- internal/cli/cmd/cluster/create_test.go | 34 ++++++++++--- internal/cli/testing/fake.go | 22 ++++++++ internal/cli/testing/fake_test.go | 7 +++ 5 files changed, 166 insertions(+), 11 deletions(-) diff --git a/internal/cli/cmd/cluster/cluster_test.go b/internal/cli/cmd/cluster/cluster_test.go index 063b4b03d..cbdb546ad 100644 --- a/internal/cli/cmd/cluster/cluster_test.go +++ b/internal/cli/cmd/cluster/cluster_test.go @@ -51,7 +51,8 @@ var _ = Describe("Cluster", func() { streams, _, _, _ = genericclioptions.NewTestIOStreams() tf = cmdtesting.NewTestFactory().WithNamespace("default") cd := testing.FakeClusterDef() - tf.FakeDynamicClient = testing.FakeDynamicClient(cd, testing.FakeClusterVersion()) + fakeDefaultStorageClass := testing.FakeStorageClass(testing.StorageClassName, testing.IsDefautl) + tf.FakeDynamicClient = testing.FakeDynamicClient(cd, fakeDefaultStorageClass, testing.FakeClusterVersion()) tf.Client = &clientfake.RESTClient{} }) @@ -98,6 +99,7 @@ var _ = Describe("Cluster", func() { clusterDef := testing.FakeClusterDef() tf.FakeDynamicClient = testing.FakeDynamicClient( clusterDef, + testing.FakeStorageClass(testing.StorageClassName, testing.IsDefautl), testing.FakeComponentClassDef(fmt.Sprintf("custom-%s", testing.ComponentDefName), clusterDef.Name, testing.ComponentDefName), testing.FakeComponentClassDef("custom-mysql", clusterDef.Name, "mysql"), ) @@ -265,7 +267,15 @@ var _ = Describe("Cluster", func() { Dynamic: tf.FakeDynamicClient, IOStreams: streams, }, + ComponentSpecs: make([]map[string]interface{}, 1), } + o.ComponentSpecs[0] = make(map[string]interface{}) + o.ComponentSpecs[0]["volumeClaimTemplates"] = make([]interface{}, 1) + vct := o.ComponentSpecs[0]["volumeClaimTemplates"].([]interface{}) + vct[0] = make(map[string]interface{}) + vct[0].(map[string]interface{})["spec"] = make(map[string]interface{}) + spec := vct[0].(map[string]interface{})["spec"] + spec.(map[string]interface{})["storageClassName"] = testing.StorageClassName }) It("can validate whether the ClusterDefRef is null when create a new cluster ", func() { @@ -301,6 +311,7 @@ var _ = Describe("Cluster", func() { Expect(o.Name).ShouldNot(BeEmpty()) Expect(o.Validate()).Should(Succeed()) o.Name = "" + // Expected to generate a random name Expect(o.Validate()).Should(Succeed()) }) @@ -318,6 +329,42 @@ var _ = Describe("Cluster", func() { o.Name = clusterNameMoreThan16 Expect(o.Validate()).Should(HaveOccurred()) }) + + Context("validate storageClass", func() { + It("can get all StorageClasses in K8S and check out if the cluster have a defalut StorageClasses by GetStorageClasses()", func() { + storageClasses, existedDefault, err := getStorageClasses(o.Dynamic) + Expect(err).Should(Succeed()) + Expect(storageClasses).Should(HaveKey(testing.StorageClassName)) + Expect(existedDefault).Should(BeTrue()) + fakeNotDefaultStorageClass := testing.FakeStorageClass(testing.StorageClassName, testing.IsNotDefault) + cd := testing.FakeClusterDef() + tf.FakeDynamicClient = testing.FakeDynamicClient(cd, fakeNotDefaultStorageClass, testing.FakeClusterVersion()) + storageClasses, existedDefault, err = getStorageClasses(tf.FakeDynamicClient) + Expect(err).Should(Succeed()) + Expect(storageClasses).Should(HaveKey(testing.StorageClassName)) + Expect(existedDefault).ShouldNot(BeTrue()) + }) + + It("can specify the StorageClass and the StorageClass must exist", func() { + Expect(validateStorageClass(o.Dynamic, o.ComponentSpecs)).Should(Succeed()) + fakeNotDefaultStorageClass := testing.FakeStorageClass(testing.StorageClassName+"-other", testing.IsNotDefault) + cd := testing.FakeClusterDef() + FakeDynamicClientWithNotDefaultSC := testing.FakeDynamicClient(cd, fakeNotDefaultStorageClass, testing.FakeClusterVersion()) + Expect(validateStorageClass(FakeDynamicClientWithNotDefaultSC, o.ComponentSpecs)).Should(HaveOccurred()) + }) + + It("can get valiate the default StorageClasses", func() { + vct := o.ComponentSpecs[0]["volumeClaimTemplates"].([]interface{}) + spec := vct[0].(map[string]interface{})["spec"] + delete(spec.(map[string]interface{}), "storageClassName") + Expect(validateStorageClass(o.Dynamic, o.ComponentSpecs)).Should(Succeed()) + fakeNotDefaultStorageClass := testing.FakeStorageClass(testing.StorageClassName+"-other", testing.IsNotDefault) + cd := testing.FakeClusterDef() + FakeDynamicClientWithNotDefaultSC := testing.FakeDynamicClient(cd, fakeNotDefaultStorageClass, testing.FakeClusterVersion()) + Expect(validateStorageClass(FakeDynamicClientWithNotDefaultSC, o.ComponentSpecs)).Should(HaveOccurred()) + }) + }) + }) It("delete", func() { diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 86d97cc1a..e0bbb1ba3 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -42,6 +42,7 @@ import ( "k8s.io/client-go/dynamic" cmdutil "k8s.io/kubectl/pkg/cmd/util" utilcomp "k8s.io/kubectl/pkg/util/completion" + "k8s.io/kubectl/pkg/util/storage" "k8s.io/kubectl/pkg/util/templates" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -126,6 +127,7 @@ const ( keyMemory setKey = "memory" keyReplicas setKey = "replicas" keyStorage setKey = "storage" + keyStorageClass setKey = "storageClass" keySwitchPolicy setKey = "switchPolicy" keyUnknown setKey = "unknown" ) @@ -256,6 +258,13 @@ func (o *CreateOptions) Validate() error { if len(o.Name) > 16 { return fmt.Errorf("cluster name should be less than 16 characters") } + + // validate default storageClassName + err := validateStorageClass(o.Dynamic, o.ComponentSpecs) + if err != nil { + return err + } + return nil } @@ -545,7 +554,7 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map } var comps []*appsv1alpha1.ClusterComponentSpec - for _, c := range cd.Spec.ComponentDefs { + for i, c := range cd.Spec.ComponentDefs { sets := map[setKey]string{} if setsMap != nil { sets = setsMap[c.Name] @@ -603,6 +612,10 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map }, }, }} + storageClass := getVal(&c, keyStorageClass, sets) + if len(storageClass) != 0 { + compObj.VolumeClaimTemplates[i].Spec.StorageClassName = &storageClass + } if err = buildSwitchPolicy(&c, compObj, sets); err != nil { return nil, err } @@ -615,7 +628,7 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map // specified in the set, use the cluster definition default component name. func buildCompSetsMap(values []string, cd *appsv1alpha1.ClusterDefinition) (map[string]map[setKey]string, error) { allSets := map[string]map[setKey]string{} - keys := []string{string(keyCPU), string(keyType), string(keyStorage), string(keyMemory), string(keyReplicas), string(keyClass), string(keySwitchPolicy)} + keys := []string{string(keyCPU), string(keyType), string(keyStorage), string(keyMemory), string(keyReplicas), string(keyClass), string(keyStorageClass), string(keySwitchPolicy)} parseKey := func(key string) setKey { for _, k := range keys { if strings.EqualFold(k, key) { @@ -757,3 +770,51 @@ func (f *UpdatableFlags) addFlags(cmd *cobra.Command) { }, cobra.ShellCompDirectiveNoFileComp })) } + +// validateStorageClass check whether the StorageClasses we need are exist in K8S or +// the default StorageClasses are exist +func validateStorageClass(dynamic dynamic.Interface, components []map[string]interface{}) error { + existedStorageClasses, existedDefault, err := getStorageClasses(dynamic) + if err != nil { + return err + } + for _, comp := range components { + compObj := appsv1alpha1.ClusterComponentSpec{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(comp, &compObj) + if err != nil { + return err + } + for _, vct := range compObj.VolumeClaimTemplates { + name := vct.Spec.StorageClassName + if name != nil { + // validate the specified StorageClass whether exist + if _, ok := existedStorageClasses[*name]; !ok { + return fmt.Errorf("failed to find the specified storageClass \"%s\"", *name) + } + } else if !existedDefault { + // validate the default StorageClass + return fmt.Errorf("failed to find the default storageClass, use '--set storageClass=NAME' to set it") + } + } + } + return nil +} + +// getStorageClasses return all StorageClasses in K8S and return true if the cluster have a default StorageClasses +func getStorageClasses(dynamic dynamic.Interface) (map[string]struct{}, bool, error) { + gvr := types.StorageClassGVR() + allStorageClasses := make(map[string]struct{}) + existedDefault := false + list, err := dynamic.Resource(gvr).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, false, err + } + for _, item := range list.Items { + allStorageClasses[item.GetName()] = struct{}{} + annotations := item.GetAnnotations() + if !existedDefault && annotations != nil && (annotations[storage.IsDefaultStorageClassAnnotation] == "true" || annotations[storage.BetaIsDefaultStorageClassAnnotation] == "true") { + existedDefault = true + } + } + return allStorageClasses, existedDefault, nil +} diff --git a/internal/cli/cmd/cluster/create_test.go b/internal/cli/cmd/cluster/create_test.go index b377ea004..6b81024ee 100644 --- a/internal/cli/cmd/cluster/create_test.go +++ b/internal/cli/cmd/cluster/create_test.go @@ -138,7 +138,7 @@ var _ = Describe("create", func() { }) }) - checkComponent := func(comps []*appsv1alpha1.ClusterComponentSpec, storage string, replicas int32, cpu string, memory string) { + checkComponent := func(comps []*appsv1alpha1.ClusterComponentSpec, storage string, replicas int32, cpu string, memory string, storageClassName string) { Expect(comps).ShouldNot(BeNil()) Expect(len(comps)).Should(Equal(2)) @@ -150,6 +150,13 @@ var _ = Describe("create", func() { Expect(resources).ShouldNot(BeNil()) Expect(getResource(resources, corev1.ResourceCPU)).Should(Equal(cpu)) Expect(getResource(resources, corev1.ResourceMemory)).Should(Equal(memory)) + + if storageClassName == "" { + Expect(comp.VolumeClaimTemplates[0].Spec.StorageClassName).Should(BeNil()) + } else { + Expect(*comp.VolumeClaimTemplates[0].Spec.StorageClassName).Should(Equal(storageClassName)) + } + } It("build default cluster component without environment", func() { @@ -157,7 +164,7 @@ var _ = Describe("create", func() { cd, _ := cluster.GetClusterDefByName(dynamic, testing.ClusterDefName) comps, err := buildClusterComp(cd, nil, componentClasses) Expect(err).ShouldNot(HaveOccurred()) - checkComponent(comps, "20Gi", 1, "1", "1Gi") + checkComponent(comps, "20Gi", 1, "1", "1Gi", "") }) It("build default cluster component with environment", func() { @@ -169,7 +176,7 @@ var _ = Describe("create", func() { cd, _ := cluster.GetClusterDefByName(dynamic, testing.ClusterDefName) comps, err := buildClusterComp(cd, nil, componentClasses) Expect(err).ShouldNot(HaveOccurred()) - checkComponent(comps, "5Gi", 1, "2", "2Gi") + checkComponent(comps, "5Gi", 1, "2", "2Gi", "") }) It("build cluster component with set values", func() { @@ -177,15 +184,16 @@ var _ = Describe("create", func() { cd, _ := cluster.GetClusterDefByName(dynamic, testing.ClusterDefName) setsMap := map[string]map[setKey]string{ testing.ComponentDefName: { - keyCPU: "10", - keyMemory: "2Gi", - keyStorage: "10Gi", - keyReplicas: "10", + keyCPU: "10", + keyMemory: "2Gi", + keyStorage: "10Gi", + keyReplicas: "10", + keyStorageClass: "test", }, } comps, err := buildClusterComp(cd, setsMap, componentClasses) Expect(err).Should(Succeed()) - checkComponent(comps, "10Gi", 10, "10", "2Gi") + checkComponent(comps, "10Gi", 10, "10", "2Gi", "test") setsMap[testing.ComponentDefName][keySwitchPolicy] = "invalid" cd.Spec.ComponentDefs[0].WorkloadType = appsv1alpha1.Replication @@ -326,6 +334,16 @@ var _ = Describe("create", func() { }, true, }, + { + []string{"storageClass=test"}, + []string{"my-comp"}, + map[string]map[setKey]string{ + "my-comp": { + keyStorageClass: "test", + }, + }, + true, + }, } for _, t := range testCases { diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index eabf961e0..17c3e0ec8 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -27,8 +27,10 @@ import ( "github.com/sethvargo/go-password/password" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubectl/pkg/util/storage" "k8s.io/utils/pointer" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -55,6 +57,9 @@ const ( KubeBlocksChartName = "fake-kubeblocks" KubeBlocksChartURL = "fake-kubeblocks-chart-url" BackupToolName = "fake-backup-tool" + + IsDefautl = true + IsNotDefault = false ) var ( @@ -530,3 +535,20 @@ func FakeConfigConstraint(ccName string) *appsv1alpha1.ConfigConstraint { } return cm } + +func FakeStorageClass(name string, isDefault bool) *storagev1.StorageClass { + storageClassObj := &storagev1.StorageClass{ + TypeMeta: metav1.TypeMeta{ + Kind: "StorageClass", + APIVersion: "storage.k8s.io/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + if isDefault { + storageClassObj.ObjectMeta.Annotations = make(map[string]string) + storageClassObj.ObjectMeta.Annotations[storage.IsDefaultStorageClassAnnotation] = "true" + } + return storageClassObj +} diff --git a/internal/cli/testing/fake_test.go b/internal/cli/testing/fake_test.go index 4be1ccab1..8e6f308fc 100644 --- a/internal/cli/testing/fake_test.go +++ b/internal/cli/testing/fake_test.go @@ -86,4 +86,11 @@ var _ = Describe("test fake", func() { events := FakeEvents() Expect(events).ShouldNot(BeNil()) }) + + It("fake storageClass", func() { + StorageClassDefault := FakeStorageClass(StorageClassName, IsDefautl) + Expect(StorageClassDefault).ShouldNot(BeNil()) + StorageClassNotDefault := FakeStorageClass(StorageClassName, IsDefautl) + Expect(StorageClassNotDefault).ShouldNot(BeNil()) + }) }) From 5b4d531949e7825885e17e01fc99978433e5bda8 Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Wed, 26 Apr 2023 10:18:47 +0800 Subject: [PATCH 183/439] feat: add chaos-mesh addon (#2733) Co-authored-by: huyongqii --- .../templates/addons/chaos-mesh-addon.yaml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 deploy/helm/templates/addons/chaos-mesh-addon.yaml diff --git a/deploy/helm/templates/addons/chaos-mesh-addon.yaml b/deploy/helm/templates/addons/chaos-mesh-addon.yaml new file mode 100644 index 000000000..4820bf956 --- /dev/null +++ b/deploy/helm/templates/addons/chaos-mesh-addon.yaml @@ -0,0 +1,36 @@ +apiVersion: extensions.kubeblocks.io/v1alpha1 +kind: Addon +metadata: + name: chaos-mesh + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} + "kubeblocks.io/provider": community + {{- if .Values.keepAddons }} + annotations: + helm.sh/resource-policy: keep + {{- end }} +spec: + description: 'Chaos Mesh is an open-source chaos engineering tool that facilitates testing the resiliency and reliability of distributed systems by introducing various failure scenarios in a controlled manner.' + + type: Helm + + helm: + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/chaos-mesh-0.1.0-beta.0.tgz + valuesMapping: + valueMap: + replicaCount: controllerManager.replicaCount + + jsonMap: + tolerations: controllerManager.tolerations + + resources: + cpu: + requests: controllerManager.resources.requests.cpu + memory: + requests: controllerManager.resources.requests.memory + + installable: + autoInstall: false + + defaultInstallValues: + - enabled: false \ No newline at end of file From 13b436e6261ef3e96591c5ff651a29920931f3a6 Mon Sep 17 00:00:00 2001 From: free6om Date: Wed, 26 Apr 2023 10:55:45 +0800 Subject: [PATCH 184/439] fix: cluster always updating after termination-policy updated (#2956) --- .../lifecycle/transformer_cluster_status.go | 63 ++++++++++--------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/internal/controller/lifecycle/transformer_cluster_status.go b/internal/controller/lifecycle/transformer_cluster_status.go index f2c9a5ac1..fed97784a 100644 --- a/internal/controller/lifecycle/transformer_cluster_status.go +++ b/internal/controller/lifecycle/transformer_cluster_status.go @@ -24,7 +24,6 @@ import ( "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -80,27 +79,27 @@ func (t *ClusterStatusTransformer) Transform(ctx graph.TransformContext, dag *gr } } - isStorageUpdated := func(oldSts, newSts *appsv1.StatefulSet) bool { - if oldSts == nil || newSts == nil { - return false - } - for _, oldVct := range oldSts.Spec.VolumeClaimTemplates { - var newVct *corev1.PersistentVolumeClaim - for _, v := range newSts.Spec.VolumeClaimTemplates { - if v.Name == oldVct.Name { - newVct = &v - break - } - } - if newVct == nil { - continue - } - if oldVct.Spec.Resources.Requests[corev1.ResourceStorage] != newVct.Spec.Resources.Requests[corev1.ResourceStorage] { - return true - } - } - return false - } + // isStorageUpdated := func(oldSts, newSts *appsv1.StatefulSet) bool { + // if oldSts == nil || newSts == nil { + // return false + // } + // for _, oldVct := range oldSts.Spec.VolumeClaimTemplates { + // var newVct *corev1.PersistentVolumeClaim + // for _, v := range newSts.Spec.VolumeClaimTemplates { + // if v.Name == oldVct.Name { + // newVct = &v + // break + // } + // } + // if newVct == nil { + // continue + // } + // if oldVct.Spec.Resources.Requests[corev1.ResourceStorage] != newVct.Spec.Resources.Requests[corev1.ResourceStorage] { + // return true + // } + // } + // return false + // } updateComponentsPhase := func() { vertices := findAll[*appsv1.StatefulSet](dag) @@ -143,15 +142,19 @@ func (t *ClusterStatusTransformer) Transform(ctx graph.TransformContext, dag *gr updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) continue } + // TODO(free6om): pvc expansion is not allowed by sts, but ops supports it by update the under pvc directly, + // which causes different behavior between volume expansion ops and cluster spec Update. + // should make them act same. + // // compare sts storage - if _, ok := v.obj.(*appsv1.StatefulSet); ok { - oldSts, _ := v.oriObj.(*appsv1.StatefulSet) - newSts, _ := v.obj.(*appsv1.StatefulSet) - if !isStorageUpdated(oldSts, newSts) { - continue - } - } - updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) + // if _, ok := v.obj.(*appsv1.StatefulSet); ok { + // oldSts, _ := v.oriObj.(*appsv1.StatefulSet) + // newSts, _ := v.obj.(*appsv1.StatefulSet) + // if !isStorageUpdated(oldSts, newSts) { + // continue + // } + // } + // updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) } } From fe50ae543a9c148b9c0af34f24591c99f6acd2c9 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Wed, 26 Apr 2023 11:12:41 +0800 Subject: [PATCH 185/439] chore: kbcli update account err msg (#2943) --- internal/cli/cmd/accounts/base.go | 5 ++++- internal/cli/cmd/cluster/accounts.go | 12 ++++++------ internal/sqlchannel/client.go | 5 ++--- internal/sqlchannel/client_test.go | 14 +++++++++++++- internal/sqlchannel/types.go | 28 ++++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 11 deletions(-) diff --git a/internal/cli/cmd/accounts/base.go b/internal/cli/cmd/accounts/base.go index 28e8cf207..6fce49ad0 100644 --- a/internal/cli/cmd/accounts/base.go +++ b/internal/cli/cmd/accounts/base.go @@ -147,10 +147,13 @@ func (o *AccountBaseOptions) Complete(f cmdutil.Factory) error { return nil } -func (o *AccountBaseOptions) Run(f cmdutil.Factory, streams genericclioptions.IOStreams) error { +func (o *AccountBaseOptions) Run(cmd *cobra.Command, f cmdutil.Factory, streams genericclioptions.IOStreams) error { var err error response, err := o.Do() if err != nil { + if sqlchannel.IsUnSupportedError(err) { + return fmt.Errorf("command `%s` on characterType `%s` (defined in cluster: %s, component: %s) is not supported yet", cmd.Use, o.CharType, o.ClusterName, o.ComponentName) + } return err } diff --git a/internal/cli/cmd/cluster/accounts.go b/internal/cli/cmd/cluster/accounts.go index 6f58c980b..11622b46e 100644 --- a/internal/cli/cmd/cluster/accounts.go +++ b/internal/cli/cmd/cluster/accounts.go @@ -97,7 +97,7 @@ func NewCreateAccountCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(o.Validate(args)) cmdutil.CheckErr(o.Complete(f)) - cmdutil.CheckErr(o.Run(f, streams)) + cmdutil.CheckErr(o.Run(cmd, f, streams)) }, } o.AddFlags(cmd) @@ -114,7 +114,7 @@ func NewDeleteAccountCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(o.Validate(args)) cmdutil.CheckErr(o.Complete(f)) - cmdutil.CheckErr(o.Run(f, streams)) + cmdutil.CheckErr(o.Run(cmd, f, streams)) }, } o.AddFlags(cmd) @@ -131,7 +131,7 @@ func NewDescAccountCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) * Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(o.Validate(args)) cmdutil.CheckErr(o.Complete(f)) - cmdutil.CheckErr(o.Run(f, streams)) + cmdutil.CheckErr(o.Run(cmd, f, streams)) }, } o.AddFlags(cmd) @@ -150,7 +150,7 @@ func NewListAccountsCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(o.Validate(args)) cmdutil.CheckErr(o.Complete(f)) - cmdutil.CheckErr(o.Run(f, streams)) + cmdutil.CheckErr(o.Run(cmd, f, streams)) }, } o.AddFlags(cmd) @@ -169,7 +169,7 @@ func NewGrantOptions(f cmdutil.Factory, streams genericclioptions.IOStreams) *co Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(o.Validate(args)) cmdutil.CheckErr(o.Complete(f)) - cmdutil.CheckErr(o.Run(f, streams)) + cmdutil.CheckErr(o.Run(cmd, f, streams)) }, } o.AddFlags(cmd) @@ -188,7 +188,7 @@ func NewRevokeOptions(f cmdutil.Factory, streams genericclioptions.IOStreams) *c Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(o.Validate(args)) cmdutil.CheckErr(o.Complete(f)) - cmdutil.CheckErr(o.Run(f, streams)) + cmdutil.CheckErr(o.Run(cmd, f, streams)) }, } o.AddFlags(cmd) diff --git a/internal/sqlchannel/client.go b/internal/sqlchannel/client.go index 02682e293..ba6fff5be 100644 --- a/internal/sqlchannel/client.go +++ b/internal/sqlchannel/client.go @@ -262,9 +262,8 @@ type errorResponse struct { Message string `json:"message"` } +// parseResponse parse response to errorResponse or SQLChannelResponse to capture error message if any. func parseResponse(data []byte, operation string, charType string) (SQLChannelResponse, error) { - // conver to errorResponse first, and check error code - // if error code is not empty, it means the request failed errorResponse := errorResponse{} response := SQLChannelResponse{} if err := json.Unmarshal(data, &errorResponse); err != nil { @@ -279,7 +278,7 @@ func parseResponse(data []byte, operation string, charType string) (SQLChannelRe EndTime: time.Now(), Extra: errorResponse.Message, }, - }, nil + }, SQLChannelError{Reason: UnsupportedOps} } // conver it to SQLChannelResponse diff --git a/internal/sqlchannel/client_test.go b/internal/sqlchannel/client_test.go index eef236016..f3ff1ab71 100644 --- a/internal/sqlchannel/client_test.go +++ b/internal/sqlchannel/client_test.go @@ -22,6 +22,7 @@ package sqlchannel import ( "context" "encoding/json" + "errors" "fmt" "net" "os" @@ -236,7 +237,8 @@ func TestParseSqlChannelResult(t *testing.T) { {"errorCode":"ERR_INVOKE_OUTPUT_BINDING","message":"error when invoke output binding mongodb: binding mongodb does not support operation listUsers. supported operations:checkRunning checkRole getRole"} ` sqlResposne, err := parseResponse(([]byte)(result), "listUsers", "mongodb") - assert.Nil(t, err) + assert.NotNil(t, err) + assert.True(t, IsUnSupportedError(err)) assert.Equal(t, sqlResposne.Event, RespEveFail) assert.Contains(t, sqlResposne.Message, "not supported") }) @@ -270,6 +272,16 @@ func TestParseSqlChannelResult(t *testing.T) { }) } +func TestErrMsg(t *testing.T) { + err := SQLChannelError{ + Reason: UnsupportedOps, + } + assert.True(t, strings.Contains(err.Error(), "unsupported")) + assert.False(t, IsUnSupportedError(nil)) + assert.True(t, IsUnSupportedError(err)) + assert.False(t, IsUnSupportedError(errors.New("test"))) +} + func newTCPServer(t *testing.T, daprServer pb.DaprServer, port int) (int, func()) { var l net.Listener for i := 0; i < 3; i++ { diff --git a/internal/sqlchannel/types.go b/internal/sqlchannel/types.go index 2b23b5bbd..e4bed719c 100644 --- a/internal/sqlchannel/types.go +++ b/internal/sqlchannel/types.go @@ -78,3 +78,31 @@ type SQLChannelMeta struct { EndTime time.Time `json:"endTime,omitempty"` Extra string `json:"extra,omitempty"` } + +type errorReason string + +const ( + UnsupportedOps errorReason = "unsupported operation" +) + +// SQLChannelError is the error for sqlchannel, it implements error interface +type SQLChannelError struct { + Reason errorReason +} + +var _ error = SQLChannelError{} + +func (e SQLChannelError) Error() string { + return string(e.Reason) +} + +// IsUnSupportedError checks if the error is unsupported operation error +func IsUnSupportedError(err error) bool { + if err == nil { + return false + } + if e, ok := err.(SQLChannelError); ok { + return e.Reason == UnsupportedOps + } + return false +} From c4955c08063c1b0d78d4a983f3a3300f3f18c395 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Wed, 26 Apr 2023 11:15:10 +0800 Subject: [PATCH 186/439] fix: start postgresql failed at first recovery boot. (#2957) --- deploy/postgresql/templates/backuptool-pitr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml index ae4092cb9..f79b0823a 100644 --- a/deploy/postgresql/templates/backuptool-pitr.yaml +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -49,8 +49,8 @@ spec: chmod 777 -R ${CONF_DIR}; mkdir -p ${RESTORE_SCRIPT_DIR}; echo "#!/bin/bash" > ${RESTORE_SCRIPT_DIR}/kb_restore.sh; - echo "[[ -d '${DATA_DIR}.old' ]] && mv -f ${DATA_DIR}.old ${DATA_DIR};" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; - echo "[[ -d '${DATA_DIR}.failed/data.old' ]] && mv -f ${DATA_DIR}.failed/data.old ${DATA_DIR};" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; + echo "[[ -d '${DATA_DIR}.old' ]] && mv -f ${DATA_DIR}.old/* ${DATA_DIR}/;" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; + echo "sync;" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; chmod +x ${RESTORE_SCRIPT_DIR}/kb_restore.sh; echo "restore_command='mv ${PITR_DIR}/%f %p'\nrecovery_target_time='${RECOVERY_TIME}'\nrecovery_target_action='promote'" > ${CONF_DIR}/recovery.conf; mv ${DATA_DIR} ${DATA_DIR}.old; From 7df19d4a292fd570343489a42d5df42a127224fc Mon Sep 17 00:00:00 2001 From: "yunju.lly" Date: Wed, 26 Apr 2023 11:38:18 +0800 Subject: [PATCH 187/439] fix: add mongodb replicaset restart alert rule (#2959) --- deploy/helm/values.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 3b21fe746..e6fa0d34f 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1099,6 +1099,15 @@ prometheus: summary: 'MongoDB is Down' description: 'MongoDB instance is down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' + - alert: MongodbRestarted + expr: 'time() - process_start_time_seconds < 60' + for: 0m + labels: + severity: info + annotations: + summary: 'Mongodb has just been restarted (< 60s)' + description: 'Mongodb has just been restarted {{ $value | printf "%.1f" }} seconds ago\n LABELS = {{ $labels }}' + - alert: MongodbReplicaMemberUnhealthy expr: 'max_over_time(mongodb_rs_members_health[1m]) == 0' for: 0m From 18cec939b9e91cacf6793a4f0b3ea562662ea6d8 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 26 Apr 2023 13:02:06 +0800 Subject: [PATCH 188/439] fix: cluster phase is incorrect when cluster is hscaling. (#2960) --- controllers/apps/cluster_controller_test.go | 31 +------ .../apps/opsrequest_controller_test.go | 82 ++++++++++++++++--- .../lifecycle/transformer_cluster_status.go | 44 +++++----- .../controller/lifecycle/transformer_init.go | 12 +++ .../transformer_sts_horizontal_scaling.go | 13 --- .../apps/cluster_consensus_test_util.go | 2 +- 6 files changed, 105 insertions(+), 79 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 914dcba5d..0c6cd9fa2 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -99,6 +99,7 @@ var _ = Describe("Cluster Controller", func() { testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PodSignature, true, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupSignature, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) + testapps.ClearResources(&testCtx, intctrlutil.VolumeSnapshotSignature, inNS) // non-namespaced testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, ml) testapps.ClearResources(&testCtx, intctrlutil.BackupToolSignature, ml) @@ -639,36 +640,6 @@ var _ = Describe("Cluster Controller", func() { By("Checking backup policy created from backup policy template") policyName := lifecycle.DeriveBackupPolicyName(clusterKey.Name, compDef.Name) - // REVIEW/TODO: (chantu) - // caught following error, it appears that BackupPolicy is statically setup or only work with 1st - // componentDefs? - // - // Unexpected error: - // <*errors.StatusError | 0x140023b5b80>: { - // ErrStatus: { - // TypeMeta: {Kind: "", APIVersion: ""}, - // ListMeta: { - // SelfLink: "", - // ResourceVersion: "", - // Continue: "", - // RemainingItemCount: nil, - // }, - // Status: "Failure", - // Message: "backuppolicies.dataprotection.kubeblocks.io \"test-clusterstqcba-consensus-backup-policy\" not found", - // Reason: "NotFound", - // Details: { - // Name: "test-clusterstqcba-consensus-backup-policy", - // Group: "dataprotection.kubeblocks.io", - // Kind: "backuppolicies", - // UID: "", - // Causes: nil, - // RetryAfterSeconds: 0, - // }, - // Code: 404, - // }, - // } - // backuppolicies.dataprotection.kubeblocks.io "test-clusterstqcba-consensus-backup-policy" not found - // occurred clusterDef.Spec.ComponentDefs[i].HorizontalScalePolicy = &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, BackupPolicyTemplateName: backupPolicyTPLName} diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index 9261cd07c..c8eff4ef6 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -25,6 +25,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/viper" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -62,6 +64,7 @@ var _ = Describe("OpsRequest Controller", func() { inNS := client.InNamespace(testCtx.DefaultNamespace) ml := client.HasLabels{testCtx.TestObjLabelKey} testapps.ClearResources(&testCtx, intctrlutil.OpsRequestSignature, inNS, ml) + testapps.ClearResources(&testCtx, intctrlutil.VolumeSnapshotSignature, inNS) // delete cluster(and all dependent sub-resources), clusterversion and clusterdef testapps.ClearClusterResources(&testCtx) @@ -334,9 +337,27 @@ var _ = Describe("OpsRequest Controller", func() { testVerticalScaleCPUAndMemory(testapps.ConsensusMySQLComponent, ctx) }) - It("HorizontalScaling when not support snapshot", func() { - By("init backup policy template") - viper.Set("VOLUMESNAPSHOT", false) + mockCompRunning := func(replicas int32) { + stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, clusterKey, mysqlCompName) + sts := &stsList.Items[0] + Expect(int(*sts.Spec.Replicas)).To(BeEquivalentTo(replicas)) + Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { + testk8s.MockStatefulSetReady(sts) + })).ShouldNot(HaveOccurred()) + for i := 0; i < int(replicas); i++ { + podName := fmt.Sprintf("%s-%s-%d", clusterObj.Name, mysqlCompName, i) + podRole := "follower" + accessMode := "Readonly" + if i == 0 { + podRole = "leader" + accessMode = "ReadWrite" + } + testapps.MockConsensusComponentStsPod(testCtx, sts, clusterObj.Name, mysqlCompName, podName, podRole, accessMode) + } + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + } + + mockMysqlCluster := func() { createBackupPolicyTpl(clusterDefObj) replicas := int32(3) @@ -356,14 +377,7 @@ var _ = Describe("OpsRequest Controller", func() { clusterKey = client.ObjectKeyFromObject(clusterObj) By("mock component is Running") - stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, clusterKey, mysqlCompName) - sts := &stsList.Items[0] - Expect(int(*sts.Spec.Replicas)).To(BeEquivalentTo(replicas)) - Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { - testk8s.MockStatefulSetReady(sts) - })).ShouldNot(HaveOccurred()) - testapps.MockConsensusComponentPods(testCtx, sts, clusterKey.Name, mysqlCompName) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + mockCompRunning(replicas) By("mock pvc created") for i := 0; i < int(replicas); i++ { @@ -378,7 +392,9 @@ var _ = Describe("OpsRequest Controller", func() { // wait for cluster observed generation Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) mockSetClusterStatusPhaseToRunning(clusterKey) + } + mockHscaleOps := func(replicas int32) *appsv1alpha1.OpsRequest { By("create a opsRequest to horizontal scale") opsName := "hscale-ops-" + testCtx.GetRandomStr() ops := testapps.NewOpsRequestObj(opsName, testCtx.DefaultNamespace, @@ -386,11 +402,19 @@ var _ = Describe("OpsRequest Controller", func() { ops.Spec.HorizontalScalingList = []appsv1alpha1.HorizontalScaling{ { ComponentOps: appsv1alpha1.ComponentOps{ComponentName: mysqlCompName}, - Replicas: int32(5), + Replicas: replicas, }, } - opsKey := client.ObjectKeyFromObject(ops) Expect(testCtx.CreateObj(testCtx.Ctx, ops)).Should(Succeed()) + return ops + } + + It("HorizontalScaling when not support snapshot", func() { + By("init backup policy template, mysql cluster and hscale ops") + viper.Set("VOLUMESNAPSHOT", false) + mockMysqlCluster() + ops := mockHscaleOps(int32(5)) + opsKey := client.ObjectKeyFromObject(ops) By("expect component is Running if don't support volume snapshot during doing h-scale ops") Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) @@ -410,6 +434,38 @@ var _ = Describe("OpsRequest Controller", func() { })()).ShouldNot(HaveOccurred()) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) }) + + It("HorizontalScaling when support snapshot", func() { + By("init backup policy template, mysql cluster and hscale ops") + viper.Set("VOLUMESNAPSHOT", true) + mockMysqlCluster() + replicas := int32(5) + ops := mockHscaleOps(replicas) + opsKey := client.ObjectKeyFromObject(ops) + + By("expect component is Running") + Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) + // cluster phase changes to HorizontalScalingPhase first. then, it will be ConditionsError because it does not support snapshot backup after a period of time. + Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) + // component phase should be running during snapshot backup + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + + By("mock snapshot created and ready to use, component phase should change to Updating to do horizontalScaling") + snapshotKey := types.NamespacedName{Name: fmt.Sprintf("%s-%s-scaling", + clusterKey.Name, mysqlCompName), Namespace: testCtx.DefaultNamespace} + volumeSnapshot := &snapshotv1.VolumeSnapshot{} + Expect(k8sClient.Get(testCtx.Ctx, snapshotKey, volumeSnapshot)).Should(Succeed()) + readyToUse := true + volumeSnapshot.Status = &snapshotv1.VolumeSnapshotStatus{ReadyToUse: &readyToUse} + Expect(k8sClient.Status().Update(testCtx.Ctx, volumeSnapshot)).Should(Succeed()) + Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) + + By("mock component is Running and expect cluster is Running") + mockCompRunning(replicas) + Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) + Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsSucceedPhase)) + }) }) Context("with Cluster which has redis Replication", func() { diff --git a/internal/controller/lifecycle/transformer_cluster_status.go b/internal/controller/lifecycle/transformer_cluster_status.go index fed97784a..7641e28e6 100644 --- a/internal/controller/lifecycle/transformer_cluster_status.go +++ b/internal/controller/lifecycle/transformer_cluster_status.go @@ -70,13 +70,15 @@ func (t *ClusterStatusTransformer) Transform(ctx graph.TransformContext, dag *gr cluster.Status.ClusterDefGeneration = transCtx.ClusterDef.Generation } - updateClusterPhase := func() { - clusterPhase := cluster.Status.Phase - if clusterPhase == "" { - cluster.Status.Phase = appsv1alpha1.CreatingClusterPhase - } else if clusterPhase != appsv1alpha1.CreatingClusterPhase { - cluster.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase + compareContainersAttribute := func(oldContainer, newContainer reflect.Value, attributeNames ...string) bool { + for _, name := range attributeNames { + oldValue := oldContainer.FieldByName(name).Interface() + newValue := newContainer.FieldByName(name).Interface() + if !reflect.DeepEqual(oldValue, newValue) { + return true + } } + return false } // isStorageUpdated := func(oldSts, newSts *appsv1.StatefulSet) bool { @@ -121,26 +123,24 @@ func (t *ClusterStatusTransformer) Transform(ctx graph.TransformContext, dag *gr newSpec := reflect.ValueOf(v.obj).Elem().FieldByName("Spec") // compare replicas - // oldReplicas := oldSpec.FieldByName("Replicas").Interface() - // newReplicas := newSpec.FieldByName("Replicas").Interface() - // if !reflect.DeepEqual(oldReplicas, newReplicas) { - // updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) - // continue - // } - // compare cpu & memory - oldResources := oldSpec.FieldByName("Template"). + oldReplicas := oldSpec.FieldByName("Replicas").Interface() + newReplicas := newSpec.FieldByName("Replicas").Interface() + if !reflect.DeepEqual(oldReplicas, newReplicas) && !v.immutable { + updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) + continue + } + // compare containers attributes + oldContainers := oldSpec.FieldByName("Template"). FieldByName("Spec"). FieldByName("Containers"). - Index(0). - FieldByName("Resources").Interface() - newResources := newSpec.FieldByName("Template"). + Index(0) + newContainers := newSpec.FieldByName("Template"). FieldByName("Spec"). FieldByName("Containers"). - Index(0). - FieldByName("Resources").Interface() - if !reflect.DeepEqual(oldResources, newResources) { + Index(0) + isChanged := compareContainersAttribute(oldContainers, newContainers, "Resources", "Image") + if isChanged { updateComponentPhaseWithOperation(cluster, v.obj.GetLabels()[constant.KBAppComponentLabelKey]) - continue } // TODO(free6om): pvc expansion is not allowed by sts, but ops supports it by update the under pvc directly, // which causes different behavior between volume expansion ops and cluster spec Update. @@ -171,13 +171,13 @@ func (t *ClusterStatusTransformer) Transform(ctx graph.TransformContext, dag *gr case isClusterUpdating(*origCluster): transCtx.Logger.Info("update cluster status after applying resources ") updateObservedGeneration() - updateClusterPhase() updateComponentsPhase() // update components' phase in cluster.status rootVertex.action = actionPtr(STATUS) case isClusterStatusUpdating(*origCluster): initClusterStatusParams() defer func() { rootVertex.action = actionPtr(STATUS) }() + updateComponentsPhase() // checks if the controller is handling the garbage of restore. if err := t.handleGarbageOfRestoreBeforeRunning(transCtx, cluster, dag); err != nil { return err diff --git a/internal/controller/lifecycle/transformer_init.go b/internal/controller/lifecycle/transformer_init.go index f08ca86f0..a78d19860 100644 --- a/internal/controller/lifecycle/transformer_init.go +++ b/internal/controller/lifecycle/transformer_init.go @@ -41,9 +41,21 @@ func (t *initTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) if !isClusterDeleting(*t.cluster) { t.handleLatestOpsRequestProcessingCondition() } + if isClusterUpdating(*t.cluster) { + t.handleClusterPhase() + } return nil } +func (t *initTransformer) handleClusterPhase() { + clusterPhase := t.cluster.Status.Phase + if clusterPhase == "" { + t.cluster.Status.Phase = appsv1alpha1.CreatingClusterPhase + } else if clusterPhase != appsv1alpha1.CreatingClusterPhase { + t.cluster.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase + } +} + // updateLatestOpsRequestProcessingCondition handles the latest opsRequest processing condition. func (t *initTransformer) handleLatestOpsRequestProcessingCondition() { opsRecords, _ := opsutil.GetOpsRequestSliceFromCluster(t.cluster) diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 97a4eade3..7b83c4302 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -241,19 +241,6 @@ func (t *StsHorizontalScalingTransformer) Transform(ctx graph.TransformContext, } return nil } - updateClusterPhase := func() { - if *stsObj.Spec.Replicas == *stsProto.Spec.Replicas { - return - } - clusterPhase := cluster.Status.Phase - if clusterPhase == "" { - cluster.Status.Phase = appsv1alpha1.CreatingClusterPhase - } else if clusterPhase != appsv1alpha1.CreatingClusterPhase { - cluster.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase - } - } - - updateClusterPhase() // when horizontal scaling up, sometimes db needs backup to sync data from master, // log is not reliable enough since it can be recycled var err error diff --git a/internal/testutil/apps/cluster_consensus_test_util.go b/internal/testutil/apps/cluster_consensus_test_util.go index 051874556..9b8014984 100644 --- a/internal/testutil/apps/cluster_consensus_test_util.go +++ b/internal/testutil/apps/cluster_consensus_test_util.go @@ -112,7 +112,7 @@ func MockConsensusComponentStsPod( AddConsensusSetAccessModeLabel(accessMode). AddControllerRevisionHashLabel(stsUpdateRevision). AddContainer(corev1.Container{Name: DefaultMySQLContainerName, Image: ApeCloudMySQLImage}). - Create(&testCtx).GetObject() + CheckedCreate(&testCtx).GetObject() patch := client.MergeFrom(pod.DeepCopy()) pod.Status.Conditions = []corev1.PodCondition{ { From 36e76f2b8704b9f0f023bd517b5e2ff11c3c6f16 Mon Sep 17 00:00:00 2001 From: free6om Date: Wed, 26 Apr 2023 13:55:39 +0800 Subject: [PATCH 189/439] fix: h-scale too many backup error warning events (#2964) --- .../transformer_sts_horizontal_scaling.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 7b83c4302..730ddc429 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -565,10 +565,9 @@ func deleteSnapshot(cli roclient.ReadonlyClient, dag *graph.DAG, root graph.Vertex) error { ctx := reqCtx.Ctx - if err := deleteBackup(ctx, cli, cluster.Name, component.Name, dag, root); err != nil { + if err := deleteBackup(reqCtx, cli, cluster, component.Name, snapshotKey.Name, dag, root); err != nil { return client.IgnoreNotFound(err) } - reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "BackupJobDelete", "Delete backupJob/%s", snapshotKey.Name) vs := &snapshotv1.VolumeSnapshot{} compatClient := intctrlutil.VolumeSnapshotCompatClient{ReadonlyClient: cli, Ctx: ctx} if err := compatClient.Get(snapshotKey, vs); err != nil { @@ -582,17 +581,23 @@ func deleteSnapshot(cli roclient.ReadonlyClient, } // deleteBackup will delete all backup related resources created during horizontal scaling, -func deleteBackup(ctx context.Context, cli roclient.ReadonlyClient, clusterName string, componentName string, dag *graph.DAG, root graph.Vertex) error { - ml := getBackupMatchingLabels(clusterName, componentName) +func deleteBackup(reqCtx intctrlutil.RequestCtx, cli roclient.ReadonlyClient, + cluster *appsv1alpha1.Cluster, componentName, snapshotName string, + dag *graph.DAG, root graph.Vertex) error { + ml := getBackupMatchingLabels(cluster.Name, componentName) backupList := dataprotectionv1alpha1.BackupList{} - if err := cli.List(ctx, &backupList, ml); err != nil { + if err := cli.List(reqCtx.Ctx, &backupList, ml); err != nil { return err } + if len(backupList.Items) == 0 { + return nil + } for _, backup := range backupList.Items { vertex := &lifecycleVertex{obj: &backup, oriObj: &backup, action: actionPtr(DELETE)} dag.AddVertex(vertex) dag.Connect(root, vertex) } + reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "BackupJobDelete", "Delete backupJob/%s", snapshotName) return nil } From 3b89faaf7477a35971c8db61e21329d99688ad4f Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 26 Apr 2023 14:55:03 +0800 Subject: [PATCH 190/439] chore: fix no events when builder resource failed (#2966) --- controllers/apps/cluster_controller_test.go | 46 ++++++++++++------- .../lifecycle/cluster_plan_builder.go | 8 ++++ .../controller/lifecycle/transform_types.go | 6 +++ .../transformer_sts_horizontal_scaling.go | 11 +---- internal/controllerutil/errors.go | 3 ++ 5 files changed, 49 insertions(+), 25 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 0c6cd9fa2..2cfe401a7 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -49,7 +49,8 @@ import ( "github.com/apecloud/kubeblocks/controllers/apps/components/util" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/lifecycle" - intctrlutil "github.com/apecloud/kubeblocks/internal/generics" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" + "github.com/apecloud/kubeblocks/internal/generics" testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" testk8s "github.com/apecloud/kubeblocks/internal/testutil/k8s" ) @@ -95,15 +96,15 @@ var _ = Describe("Cluster Controller", func() { inNS := client.InNamespace(testCtx.DefaultNamespace) ml := client.HasLabels{testCtx.TestObjLabelKey} // namespaced - testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PersistentVolumeClaimSignature, true, inNS, ml) - testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.PodSignature, true, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicySignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.VolumeSnapshotSignature, inNS) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PersistentVolumeClaimSignature, true, inNS, ml) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, generics.PodSignature, true, inNS, ml) + testapps.ClearResources(&testCtx, generics.BackupSignature, inNS, ml) + testapps.ClearResources(&testCtx, generics.BackupPolicySignature, inNS, ml) + testapps.ClearResources(&testCtx, generics.VolumeSnapshotSignature, inNS) // non-namespaced - testapps.ClearResources(&testCtx, intctrlutil.BackupPolicyTemplateSignature, ml) - testapps.ClearResources(&testCtx, intctrlutil.BackupToolSignature, ml) - testapps.ClearResources(&testCtx, intctrlutil.StorageClassSignature, ml) + testapps.ClearResources(&testCtx, generics.BackupPolicyTemplateSignature, ml) + testapps.ClearResources(&testCtx, generics.BackupToolSignature, ml) + testapps.ClearResources(&testCtx, generics.StorageClassSignature, ml) } BeforeEach(func() { @@ -143,7 +144,7 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely(clusterKey, replicationCompName) By("Check deployment workload has been created") - Eventually(testapps.GetListLen(&testCtx, intctrlutil.DeploymentSignature, + Eventually(testapps.GetListLen(&testCtx, generics.DeploymentSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, }, client.InNamespace(clusterKey.Namespace))).ShouldNot(BeEquivalentTo(0)) @@ -172,7 +173,7 @@ var _ = Describe("Cluster Controller", func() { } By("Check associated PDB has been created") - Eventually(testapps.GetListLen(&testCtx, intctrlutil.PodDisruptionBudgetSignature, + Eventually(testapps.GetListLen(&testCtx, generics.PodDisruptionBudgetSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, }, client.InNamespace(clusterKey.Namespace))).Should(Equal(0)) @@ -192,7 +193,7 @@ var _ = Describe("Cluster Controller", func() { Expect(podSpec.TopologySpreadConstraints).Should(BeEmpty()) By("Check should create env configmap") - Eventually(testapps.GetListLen(&testCtx, intctrlutil.ConfigMapSignature, + Eventually(testapps.GetListLen(&testCtx, generics.ConfigMapSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, constant.AppConfigTypeLabelKey: "kubeblocks-env", @@ -555,7 +556,7 @@ var _ = Describe("Cluster Controller", func() { } By("Checking Backup created") - Eventually(testapps.GetListLen(&testCtx, intctrlutil.BackupSignature, + Eventually(testapps.GetListLen(&testCtx, generics.BackupSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, constant.KBAppComponentLabelKey: comp.Name, @@ -602,7 +603,7 @@ var _ = Describe("Cluster Controller", func() { } By("Check backup job cleanup") - Eventually(testapps.GetListLen(&testCtx, intctrlutil.BackupSignature, + Eventually(testapps.GetListLen(&testCtx, generics.BackupSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, constant.KBAppComponentLabelKey: comp.Name, @@ -1167,14 +1168,27 @@ var _ = Describe("Cluster Controller", func() { Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { hasBackupError := false for _, cond := range cluster.Status.Conditions { - if strings.Contains(cond.Message, "backup error") { + if strings.Contains(cond.Message, "backup for horizontalScaling failed") { hasBackupError = true break } } g.Expect(hasBackupError).Should(BeTrue()) - })).Should(Succeed()) + + By("expect for backup error event") + Eventually(func(g Gomega) { + eventList := corev1.EventList{} + Expect(k8sClient.List(ctx, &eventList, client.InNamespace(testCtx.DefaultNamespace))).Should(Succeed()) + hasBackupErrorEvent := false + for _, v := range eventList.Items { + if v.Reason == string(intctrlutil.ErrorTypeBackupFailed) { + hasBackupErrorEvent = true + break + } + } + g.Expect(hasBackupErrorEvent).Should(BeTrue()) + }).Should(Succeed()) } updateClusterAnnotation := func(cluster *appsv1alpha1.Cluster) { diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 78bde7126..b71925679 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -142,6 +142,14 @@ func (c *clusterPlanBuilder) Build() (graph.Plan, error) { return } setApplyResourceCondition(&c.transCtx.Cluster.Status.Conditions, c.transCtx.Cluster.Generation, err) + if err != nil && !IsRequeueError(err) { + reason := ReasonApplyResourcesFailed + controllerErr := intctrlutil.ToControllerError(err) + if controllerErr != nil { + reason = string(controllerErr.Type) + } + c.transCtx.GetRecorder().Event(c.transCtx.Cluster, corev1.EventTypeWarning, reason, err.Error()) + } }() // new a DAG and apply chain on it, after that we should get the final Plan diff --git a/internal/controller/lifecycle/transform_types.go b/internal/controller/lifecycle/transform_types.go index 14e6c02ee..fb27f9c23 100644 --- a/internal/controller/lifecycle/transform_types.go +++ b/internal/controller/lifecycle/transform_types.go @@ -120,6 +120,12 @@ func (r *realRequeueError) Reason() string { return r.reason } +// IsRequeueError checks if the error is a RequeueError +func IsRequeueError(err error) bool { + _, ok := err.(RequeueError) + return ok +} + type delegateClient struct { client.Client } diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 730ddc429..53149ae3e 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -395,12 +395,8 @@ func doBackup(reqCtx intctrlutil.RequestCtx, // use volume snapshot case appsv1alpha1.HScaleDataClonePolicyFromSnapshot: if !isSnapshotAvailable(cli, ctx) { - reqCtx.Recorder.Eventf(cluster, - corev1.EventTypeWarning, - "HorizontalScaleFailed", - "volume snapshot not support") // TODO: add ut - return fmt.Errorf("volume snapshot not support") + return fmt.Errorf("HorizontalScaleFailed: volume snapshot not support") } vcts := component.VolumeClaimTemplates if len(vcts) == 0 { @@ -764,10 +760,7 @@ func createBackup(reqCtx intctrlutil.RequestCtx, if len(backupList.Items) > 0 { // check backup status, if failed return error if backupList.Items[0].Status.Phase == dataprotectionv1alpha1.BackupFailed { - reqCtx.Recorder.Eventf(cluster, corev1.EventTypeWarning, - "HorizontalScaleFailed", "backup %s status failed", backupKey.Name) - return fmt.Errorf("cluster %s h-scale failed, backup error: %s", - cluster.Name, backupList.Items[0].Status.FailureReason) + return intctrlutil.NewErrorf(intctrlutil.ErrorTypeBackupFailed, "backup for horizontalScaling failed: %s", backupList.Items[0].Status.FailureReason) } return nil } diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index d1ece950c..d39c1c5bb 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -52,6 +52,9 @@ const ( ErrorTypeBackupPVCNameIsEmpty ErrorType = "BackupPVCNameIsEmpty" // pvc name for backup is empty ErrorTypeBackupJobFailed ErrorType = "BackupJobFailed" // backup job failed ErrorTypeStorageNotMatch ErrorType = "ErrorTypeStorageNotMatch" + + // ErrorType for cluster controller + ErrorTypeBackupFailed ErrorType = "BackupFailed" ) var ErrFailedToAddFinalizer = errors.New("failed to add finalizer") From b084d25bf7b1045f3a200bd3a64eee1cb5ab4660 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 26 Apr 2023 16:12:39 +0800 Subject: [PATCH 191/439] chore: add the namespace of the oss pv template configmap (#2969) --- deploy/csi-oss/templates/pv-template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/csi-oss/templates/pv-template.yaml b/deploy/csi-oss/templates/pv-template.yaml index a58278c03..a18e29cb1 100644 --- a/deploy/csi-oss/templates/pv-template.yaml +++ b/deploy/csi-oss/templates/pv-template.yaml @@ -2,6 +2,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: oss-persistent-volume-template + namespace: {{ .Release.Namespace }} labels: kubeblocks.io/persistent-volume-template: "true" data: From 10e4ce5b0a8f115382bbfdf036a2e3179bf7f498 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 26 Apr 2023 18:11:54 +0800 Subject: [PATCH 192/439] chore: fix upload file to oss slowly (#2984) --- controllers/dataprotection/backup_controller.go | 5 ++++- deploy/mongodb/templates/backuptool.yaml | 10 +++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index d2c911a64..c91bdff72 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -58,7 +58,7 @@ import ( const ( backupPathBase = "/backupdata" - deleteBackupFilesJobNamePrefix = "delete-backup-files-" + deleteBackupFilesJobNamePrefix = "delete-" ) // BackupReconciler reconciles a Backup object @@ -992,6 +992,9 @@ func (r *BackupReconciler) deleteBackupFiles(reqCtx intctrlutil.RequestCtx, back } jobName := deleteBackupFilesJobNamePrefix + backup.Name + if len(jobName) > 60 { + jobName = jobName[:60] + } jobKey := types.NamespacedName{Namespace: backup.Namespace, Name: jobName} job := batchv1.Job{} exists, err := intctrlutil.CheckResourceExists(reqCtx.Ctx, r.Client, jobKey, &job) diff --git a/deploy/mongodb/templates/backuptool.yaml b/deploy/mongodb/templates/backuptool.yaml index 9cff5e9c1..9a8a12c60 100644 --- a/deploy/mongodb/templates/backuptool.yaml +++ b/deploy/mongodb/templates/backuptool.yaml @@ -27,8 +27,7 @@ spec: echo "${DATA_DIR} is not empty! Please make sure that the directory is empty before restoring the backup." exit 1 fi - tar -xvf ${BACKUP_DIR}/${BACKUP_NAME}.tar.gz -C ${DATA_DIR}/../ - mv ${DATA_DIR}/../${BACKUP_NAME}/* ${DATA_DIR} + tar -xvf ${BACKUP_DIR}/${BACKUP_NAME}.tar.gz -C ${DATA_DIR} PORT=27017 MONGODB_ROOT=/data/mongodb RPL_SET_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); @@ -52,9 +51,6 @@ spec: incrementalRestoreCommands: [] backupCommands: - | - mkdir -p ${BACKUP_DIR}/${BACKUP_NAME} - cp -R ${DATA_DIR}/* ${BACKUP_DIR}/${BACKUP_NAME}/ - cd ${BACKUP_DIR} - tar -czvf ${BACKUP_NAME}.tar.gz ./${BACKUP_NAME} - rm -rf ${BACKUP_DIR}/${BACKUP_NAME} + mkdir -p ${BACKUP_DIR} && cd ${DATA_DIR} + tar -czvf ${BACKUP_DIR}/${BACKUP_NAME}.tar.gz ./ incrementalBackupCommands: [] From dd28516fb758174263929208efaa67a7d364ce4e Mon Sep 17 00:00:00 2001 From: chantu Date: Wed, 26 Apr 2023 22:58:45 +0800 Subject: [PATCH 193/439] fix: cluster recreate error (#2982) --- deploy/apecloud-mysql/templates/scripts.yaml | 13 ++++-- internal/controller/builder/builder.go | 48 +------------------- internal/controller/builder/builder_test.go | 9 ++-- 3 files changed, 15 insertions(+), 55 deletions(-) diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index a2c420171..30efde94c 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -62,11 +62,16 @@ data: echo "cluster_info=$cluster_info"; mkdir -p /data/mysql/data /data/mysql/log chmod +777 -R /data/mysql; - echo "KB_MYSQL_RECREATE=$KB_MYSQL_RECREATE" - if [ "$KB_MYSQL_RECREATE" == "true" ]; then - echo "recreate from existing volumes, touch /data/mysql/data/.resetup_db" - touch /data/mysql/data/.resetup_db + echo "KB_MYSQL_CLUSTER_UID=$KB_MYSQL_CLUSTER_UID" + cluster_uid_path=/data/mysql/data/.kb_cluster_uid + if [ -f $cluster_uid_path ]; then + last_cluster_uid=`cat $cluster_uid_path` + if [ "$last_cluster_uid" != "$KB_MYSQL_CLUSTER_UID" ]; then + echo "recreate from existing volumes, touch /data/mysql/data/.resetup_db" + touch /data/mysql/data/.resetup_db + fi fi + echo "$KB_MYSQL_CLUSTER_UID" > $cluster_uid_path if [ -z $leader ] || [ ! -f "/data/mysql/data/.restore" ]; then echo "docker-entrypoint.sh mysqld --defaults-file=/opt/mysql/my.cnf --cluster-start-index=$CLUSTER_START_INDEX --cluster-info=\"$cluster_info\" --cluster-id=$CLUSTER_ID" exec docker-entrypoint.sh mysqld --defaults-file=/opt/mysql/my.cnf --cluster-start-index=$CLUSTER_START_INDEX --cluster-info="$cluster_info" --cluster-id=$CLUSTER_ID diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index 15e3bc417..e9d5ffa8c 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -36,7 +36,6 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" "sigs.k8s.io/controller-runtime/pkg/client" @@ -426,45 +425,6 @@ func BuildPVCFromSnapshot(sts *appsv1.StatefulSet, // envFrom.configMapRef with name of "$(cluster.metadata.name)-$(component.name)-env" pattern. func BuildEnvConfig(params BuilderParams, reqCtx intctrlutil.RequestCtx, cli client.Client) (*corev1.ConfigMap, error) { - isRecreateFromExistingPVC := func() (bool, error) { - ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: params.Cluster.Name, - constant.KBAppComponentLabelKey: params.Component.Name, - } - pvcList := corev1.PersistentVolumeClaimList{} - if err := cli.List(reqCtx.Ctx, &pvcList, ml); err != nil { - return false, err - } - // no pvc means it's not recreation - if len(pvcList.Items) == 0 { - return false, nil - } - // check sts existence - stsList := appsv1.StatefulSetList{} - if err := cli.List(reqCtx.Ctx, &stsList, ml); err != nil { - return false, err - } - // recreation will not have existing sts - if len(stsList.Items) > 0 { - return false, nil - } - // check pod existence - for _, pvc := range pvcList.Items { - vctName := pvc.Annotations[constant.VolumeClaimTemplateNameLabelKey] - podName := strings.TrimPrefix(pvc.Name, vctName+"-") - if err := cli.Get(reqCtx.Ctx, types.NamespacedName{Name: podName, Namespace: params.Cluster.Namespace}, &corev1.Pod{}); err != nil { - if apierrors.IsNotFound(err) { - continue - } - return false, err - } - // any pod exists means it's not a recreation - return false, nil - } - // passed all the above checks, so it's a recreation - return true, nil - } - const tplFile = "env_config_template.cue" prefix := constant.KBPrefix + "_" + strings.ToUpper(params.Component.Type) + "_" @@ -521,12 +481,8 @@ func BuildEnvConfig(params BuilderParams, reqCtx intctrlutil.RequestCtx, cli cli } } - // if created from existing pvc, set env - isRecreate, err := isRecreateFromExistingPVC() - if err != nil { - return nil, err - } - envData[prefix+"RECREATE"] = strconv.FormatBool(isRecreate) + // set cluster uid to let pod know if the cluster is recreated + envData[prefix+"CLUSTER_UID"] = string(params.Cluster.UID) config := corev1.ConfigMap{} if err := buildFromCUE(tplFile, map[string]any{ diff --git a/internal/controller/builder/builder_test.go b/internal/controller/builder/builder_test.go index 87d6b2265..4cfd320ff 100644 --- a/internal/controller/builder/builder_test.go +++ b/internal/controller/builder/builder_test.go @@ -324,15 +324,14 @@ var _ = Describe("builder", func() { reqCtx := newReqCtx() params := newParams() - By("creating pvc to make it looks like recreation") - pvcName := "test-pvc" - testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterName, - params.Component.Name, testapps.DataVolumeName).SetStorage("1Gi").CheckedCreate(&testCtx) + uuid := "12345" + By("mock a cluster uuid") + params.Cluster.UID = types.UID(uuid) cfg, err := BuildEnvConfig(*params, reqCtx, k8sClient) Expect(err).Should(BeNil()) Expect(cfg).ShouldNot(BeNil()) - Expect(cfg.Data["KB_"+strings.ToUpper(params.Component.Type)+"_RECREATE"]).Should(Equal("true")) + Expect(cfg.Data["KB_"+strings.ToUpper(params.Component.Type)+"_CLUSTER_UID"]).Should(Equal(uuid)) }) It("builds Env Config with ConsensusSet status correctly", func() { From f989472e159d27cee13f1a43bf3259464ad14a79 Mon Sep 17 00:00:00 2001 From: xingran Date: Thu, 27 Apr 2023 10:21:02 +0800 Subject: [PATCH 194/439] chore: support postgresql 12&14 multi engine version (#2987) --- .../config/pg12-config-constraint.cue | 1109 +++++++++++++++++ .../config/pg12-config-effect-scope.yaml | 62 + deploy/postgresql/config/pg12-config.tpl | 133 ++ .../postgresql/templates/clusterversion.yaml | 32 +- .../templates/configconstraint-12.yaml | 53 + ...nstraint.yaml => configconstraint-14.yaml} | 0 deploy/postgresql/templates/configmap-12.yaml | 33 + .../{configmap.yaml => configmap-14.yaml} | 0 8 files changed, 1421 insertions(+), 1 deletion(-) create mode 100644 deploy/postgresql/config/pg12-config-constraint.cue create mode 100644 deploy/postgresql/config/pg12-config-effect-scope.yaml create mode 100644 deploy/postgresql/config/pg12-config.tpl create mode 100644 deploy/postgresql/templates/configconstraint-12.yaml rename deploy/postgresql/templates/{configconstraint.yaml => configconstraint-14.yaml} (100%) create mode 100644 deploy/postgresql/templates/configmap-12.yaml rename deploy/postgresql/templates/{configmap.yaml => configmap-14.yaml} (100%) diff --git a/deploy/postgresql/config/pg12-config-constraint.cue b/deploy/postgresql/config/pg12-config-constraint.cue new file mode 100644 index 000000000..bbe702eab --- /dev/null +++ b/deploy/postgresql/config/pg12-config-constraint.cue @@ -0,0 +1,1109 @@ +//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +//This file is part of KubeBlocks project +// +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. +// +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . + +// PostgreSQL parameters: https://postgresqlco.nf/doc/en/param/ +#PGParameter: { + // Allows tablespaces directly inside pg_tblspc, for testing, pg version: 15 + allow_in_place_tablespaces?: bool + // Allows modification of the structure of system tables as well as certain other risky actions on system tables. This is otherwise not allowed even for superusers. Ill-advised use of this setting can cause irretrievable data loss or seriously corrupt the database system. + allow_system_table_mods?: bool + // Sets the application name to be reported in statistics and logs. + application_name?: string + // Sets the shell command that will be called to archive a WAL file. + archive_command?: string + // The library to use for archiving completed WAL file segments. If set to an empty string (the default), archiving via shell is enabled, and archive_command is used. Otherwise, the specified shared library is used for archiving. The WAL archiver process is restarted by the postmaster when this parameter changes. For more information, see backup-archiving-wal and archive-modules. + archive_library?: string + // When archive_mode is enabled, completed WAL segments are sent to archive storage by setting archive_command or guc-archive-library. In addition to off, to disable, there are two modes: on, and always. During normal operation, there is no difference between the two modes, but when set to always the WAL archiver is enabled also during archive recovery or standby mode. In always mode, all files restored from the archive or streamed with streaming replication will be archived (again). See continuous-archiving-in-standby for details. + archive_mode: string & "always" | "on" | "off" + // (s) Forces a switch to the next xlog file if a new file has not been started within N seconds. + archive_timeout: int & >=0 & <=2147483647 | *300 @timeDurationResource(1s) + // Enable input of NULL elements in arrays. + array_nulls?: bool + // (s) Sets the maximum allowed time to complete client authentication. + authentication_timeout?: int & >=1 & <=600 @timeDurationResource(1s) + // Use EXPLAIN ANALYZE for plan logging. + "auto_explain.log_analyze"?: bool + // Log buffers usage. + "auto_explain.log_buffers"?: bool & false | true + // EXPLAIN format to be used for plan logging. + "auto_explain.log_format"?: string & "text" | "xml" | "json" | "yaml" + + // (ms) Sets the minimum execution time above which plans will be logged. + "auto_explain.log_min_duration"?: int & >=-1 & <=2147483647 @timeDurationResource() + + // Log nested statements. + "auto_explain.log_nested_statements"?: bool & false | true + + // Collect timing data, not just row counts. + "auto_explain.log_timing"?: bool & false | true + + // Include trigger statistics in plans. + "auto_explain.log_triggers"?: bool & false | true + + // Use EXPLAIN VERBOSE for plan logging. + "auto_explain.log_verbose"?: bool & false | true + + // Fraction of queries to process. + "auto_explain.sample_rate"?: float & >=0 & <=1 + + // Starts the autovacuum subprocess. + autovacuum?: bool + + // Number of tuple inserts, updates or deletes prior to analyze as a fraction of reltuples. + autovacuum_analyze_scale_factor: float & >=0 & <=100 | *0.05 + + // Minimum number of tuple inserts, updates or deletes prior to analyze. + autovacuum_analyze_threshold?: int & >=0 & <=2147483647 + + // Age at which to autovacuum a table to prevent transaction ID wraparound. + autovacuum_freeze_max_age?: int & >=100000 & <=2000000000 + + // Sets the maximum number of simultaneously running autovacuum worker processes. + autovacuum_max_workers?: int & >=1 & <=8388607 + + // Multixact age at which to autovacuum a table to prevent multixact wraparound. + autovacuum_multixact_freeze_max_age?: int & >=10000000 & <=2000000000 + + // (s) Time to sleep between autovacuum runs. + autovacuum_naptime: int & >=1 & <=2147483 | *15 @timeDurationResource(1s) + + // (ms) Vacuum cost delay in milliseconds, for autovacuum. + autovacuum_vacuum_cost_delay?: int & >=-1 & <=100 @timeDurationResource() + + // Vacuum cost amount available before napping, for autovacuum. + autovacuum_vacuum_cost_limit?: int & >=-1 & <=10000 + + // Number of tuple inserts prior to vacuum as a fraction of reltuples. + autovacuum_vacuum_insert_scale_factor?: float & >=0 & <=100 + + // Minimum number of tuple inserts prior to vacuum, or -1 to disable insert vacuums. + autovacuum_vacuum_insert_threshold?: int & >=-1 & <=2147483647 + + // Number of tuple updates or deletes prior to vacuum as a fraction of reltuples. + autovacuum_vacuum_scale_factor: float & >=0 & <=100 | *0.1 + + // Minimum number of tuple updates or deletes prior to vacuum. + autovacuum_vacuum_threshold?: int & >=0 & <=2147483647 + + // (kB) Sets the maximum memory to be used by each autovacuum worker process. + autovacuum_work_mem?: int & >=-1 & <=2147483647 @storeResource(1KB) + + // (8Kb) Number of pages after which previously performed writes are flushed to disk. + backend_flush_after?: int & >=0 & <=256 @storeResource(8KB) + + // Sets whether "\" is allowed in string literals. + backslash_quote?: string & "safe_encoding" | "on" | "off" + + // Log backtrace for errors in these functions. + backtrace_functions?: string + + // (ms) Background writer sleep time between rounds. + bgwriter_delay?: int & >=10 & <=10000 @timeDurationResource() + + // (8Kb) Number of pages after which previously performed writes are flushed to disk. + bgwriter_flush_after?: int & >=0 & <=256 @storeResource(8KB) + + // Background writer maximum number of LRU pages to flush per round. + bgwriter_lru_maxpages?: int & >=0 & <=1000 + + // Multiple of the average buffer usage to free per round. + bgwriter_lru_multiplier?: float & >=0 & <=10 + + // Sets the output format for bytea. + bytea_output?: string & "escape" | "hex" + + // Check function bodies during CREATE FUNCTION. + check_function_bodies?: bool & false | true + + // Time spent flushing dirty buffers during checkpoint, as fraction of checkpoint interval. + checkpoint_completion_target: float & >=0 & <=1 | *0.9 + + // (8kB) Number of pages after which previously performed writes are flushed to disk. + checkpoint_flush_after?: int & >=0 & <=256 @storeResource(8KB) + + // (s) Sets the maximum time between automatic WAL checkpoints. + checkpoint_timeout?: int & >=30 & <=3600 @timeDurationResource(1s) + + // (s) Enables warnings if checkpoint segments are filled more frequently than this. + checkpoint_warning?: int & >=0 & <=2147483647 @timeDurationResource(1s) + + // time between checks for client disconnection while running queries + client_connection_check_interval?: int & >=0 & <=2147483647 @timeDurationResource() + + // Sets the clients character set encoding. + client_encoding?: string + + // Sets the message levels that are sent to the client. + client_min_messages?: string & "debug5" | "debug4" | "debug3" | "debug2" | "debug1" | "log" | "notice" | "warning" | "error" + + // Sets the delay in microseconds between transaction commit and flushing WAL to disk. + commit_delay?: int & >=0 & <=100000 + + // Sets the minimum concurrent open transactions before performing commit_delay. + commit_siblings?: int & >=0 & <=1000 + + // Enables in-core computation of a query identifier + compute_query_id?: string & "on" | "auto" + + // Sets the servers main configuration file. + config_file?: string + + // Enables the planner to use constraints to optimize queries. + constraint_exclusion?: string & "partition" | "on" | "off" + + // Sets the planners estimate of the cost of processing each index entry during an index scan. + cpu_index_tuple_cost?: float & >=0 & <=1.79769e+308 + + // Sets the planners estimate of the cost of processing each operator or function call. + cpu_operator_cost?: float & >=0 & <=1.79769e+308 + + // Sets the planners estimate of the cost of processing each tuple (row). + cpu_tuple_cost?: float & >=0 & <=1.79769e+308 + + // Sets the database to store pg_cron metadata tables + "cron.database_name"?: string + + // Log all jobs runs into the job_run_details table + "cron.log_run"?: string & "on" | "off" + + // Log all cron statements prior to execution. + "cron.log_statement"?: string & "on" | "off" + + // Maximum number of jobs that can run concurrently. + "cron.max_running_jobs": int & >=0 & <=100 | *5 + + // Enables background workers for pg_cron + "cron.use_background_workers"?: string + + // Sets the planners estimate of the fraction of a cursors rows that will be retrieved. + cursor_tuple_fraction?: float & >=0 & <=1 + + // Sets the servers data directory. + data_directory?: string + + // Sets the display format for date and time values. + datestyle?: string + + // Enables per-database user names. + db_user_namespace?: bool & false | true + + // (ms) Sets the time to wait on a lock before checking for deadlock. + deadlock_timeout?: int & >=1 & <=2147483647 @timeDurationResource() + + // Indents parse and plan tree displays. + debug_pretty_print?: bool & false | true + + // Logs each querys parse tree. + debug_print_parse?: bool & false | true + + // Logs each querys execution plan. + debug_print_plan?: bool & false | true + + // Logs each querys rewritten parse tree. + debug_print_rewritten?: bool & false | true + + // Sets the default statistics target. + default_statistics_target?: int & >=1 & <=10000 + + // Sets the default tablespace to create tables and indexes in. + default_tablespace?: string + + // Sets the default TOAST compression method for columns of newly-created tables + default_toast_compression?: string & "pglz" | "lz4" + + // Sets the default deferrable status of new transactions. + default_transaction_deferrable?: bool & false | true + + // Sets the transaction isolation level of each new transaction. + default_transaction_isolation?: string & "serializable" | "repeatable read" | "read committed" | "read uncommitted" + + // Sets the default read-only status of new transactions. + default_transaction_read_only?: bool & false | true + + // (8kB) Sets the planners assumption about the size of the disk cache. + effective_cache_size?: int & >=1 & <=2147483647 @storeResource(8KB) + + // Number of simultaneous requests that can be handled efficiently by the disk subsystem. + effective_io_concurrency?: int & >=0 & <=1000 + + // Enables or disables the query planner's use of async-aware append plan types + enable_async_append?: bool & false | true + + // Enables the planners use of bitmap-scan plans. + enable_bitmapscan?: bool & false | true + + // Enables the planner's use of gather merge plans. + enable_gathermerge?: bool & false | true + + // Enables the planners use of hashed aggregation plans. + enable_hashagg?: bool & false | true + + // Enables the planners use of hash join plans. + enable_hashjoin?: bool & false | true + + // Enables the planner's use of incremental sort steps. + enable_incremental_sort?: bool & false | true + + // Enables the planner's use of index-only-scan plans. + enable_indexonlyscan?: bool & false | true + + // Enables the planners use of index-scan plans. + enable_indexscan?: bool & false | true + + // Enables the planners use of materialization. + enable_material?: bool & false | true + + // Enables the planner's use of memoization + enable_memoize?: bool & false | true + + // Enables the planners use of merge join plans. + enable_mergejoin?: bool & false | true + + // Enables the planners use of nested-loop join plans. + enable_nestloop?: bool & false | true + + // Enables the planner's use of parallel append plans. + enable_parallel_append?: bool & false | true + + // Enables the planner's user of parallel hash plans. + enable_parallel_hash?: bool & false | true + + // Enable plan-time and run-time partition pruning. + enable_partition_pruning?: bool & false | true + + // Enables partitionwise aggregation and grouping. + enable_partitionwise_aggregate?: bool & false | true + + // Enables partitionwise join. + enable_partitionwise_join?: bool & false | true + + // Enables the planners use of sequential-scan plans. + enable_seqscan?: bool & false | true + + // Enables the planners use of explicit sort steps. + enable_sort?: bool & false | true + + // Enables the planners use of TID scan plans. + enable_tidscan?: bool & false | true + + // Warn about backslash escapes in ordinary string literals. + escape_string_warning?: bool & false | true + + // Terminate session on any error. + exit_on_error?: bool & false | true + + // Sets the number of digits displayed for floating-point values. + extra_float_digits?: int & >=-15 & <=3 + + // Forces use of parallel query facilities. + force_parallel_mode?: bool & false | true + + // Sets the FROM-list size beyond which subqueries are not collapsed. + from_collapse_limit?: int & >=1 & <=2147483647 + + // Forces synchronization of updates to disk. + fsync: bool & false | true | *true + + // Writes full pages to WAL when first modified after a checkpoint. + full_page_writes: bool & false | true | *true + + // Enables genetic query optimization. + geqo?: bool & false | true + + // GEQO: effort is used to set the default for other GEQO parameters. + geqo_effort?: int & >=1 & <=10 + + // GEQO: number of iterations of the algorithm. + geqo_generations?: int & >=0 & <=2147483647 + + // GEQO: number of individuals in the population. + geqo_pool_size?: int & >=0 & <=2147483647 @storeResource() + + // GEQO: seed for random path selection. + geqo_seed?: float & >=0 & <=1 + + // GEQO: selective pressure within the population. + geqo_selection_bias?: float & >=1.5 & <=2 + + // Sets the threshold of FROM items beyond which GEQO is used. + geqo_threshold?: int & >=2 & <=2147483647 + + // Sets the maximum allowed result for exact search by GIN. + gin_fuzzy_search_limit?: int & >=0 & <=2147483647 + + // (kB) Sets the maximum size of the pending list for GIN index. + gin_pending_list_limit?: int & >=64 & <=2147483647 @storeResource(1KB) + + // Multiple of work_mem to use for hash tables. + hash_mem_multiplier?: float & >=1 & <=1000 + + // Sets the servers hba configuration file. + hba_file?: string + + // Force group aggregation for hll + "hll.force_groupagg"?: bool & false | true + + // Allows feedback from a hot standby to the primary that will avoid query conflicts. + hot_standby_feedback?: bool & false | true + + // Use of huge pages on Linux. + huge_pages?: string & "on" | "off" | "try" + + // The size of huge page that should be requested. Controls the size of huge pages, when they are enabled with huge_pages. The default is zero (0). When set to 0, the default huge page size on the system will be used. This parameter can only be set at server start. + huge_page_size?: int & >=0 & <=2147483647 @storeResource(1KB) + + // Sets the servers ident configuration file. + ident_file?: string + + // (ms) Sets the maximum allowed duration of any idling transaction. + idle_in_transaction_session_timeout: int & >=0 & <=2147483647 | *86400000 @timeDurationResource() + + // Terminate any session that has been idle (that is, waiting for a client query), but not within an open transaction, for longer than the specified amount of time + idle_session_timeout?: int & >=0 & <=2147483647 @timeDurationResource() + + // Continues recovery after an invalid pages failure. + ignore_invalid_pages: bool & false | true | *false + + // Sets the display format for interval values. + intervalstyle?: string & "postgres" | "postgres_verbose" | "sql_standard" | "iso_8601" + + // Allow JIT compilation. + jit: bool + + // Perform JIT compilation if query is more expensive. + jit_above_cost?: float & >=-1 & <=1.79769e+308 + + // Perform JIT inlining if query is more expensive. + jit_inline_above_cost?: float & >=-1 & <=1.79769e+308 + + // Optimize JITed functions if query is more expensive. + jit_optimize_above_cost?: float & >=-1 & <=1.79769e+308 + + // Sets the FROM-list size beyond which JOIN constructs are not flattened. + join_collapse_limit?: int & >=1 & <=2147483647 + + // Sets the language in which messages are displayed. + lc_messages?: string + + // Sets the locale for formatting monetary amounts. + lc_monetary?: string + + // Sets the locale for formatting numbers. + lc_numeric?: string + + // Sets the locale for formatting date and time values. + lc_time?: string + + // Sets the host name or IP address(es) to listen to. + listen_addresses?: string + + // Enables backward compatibility mode for privilege checks on large objects. + lo_compat_privileges: bool & false | true | *false + + // (ms) Sets the minimum execution time above which autovacuum actions will be logged. + log_autovacuum_min_duration: int & >=-1 & <=2147483647 | *10000 @timeDurationResource() + + // Logs each checkpoint. + log_checkpoints: bool & false | true | *true + + // Logs each successful connection. + log_connections?: bool & false | true + + // Sets the destination for server log output. + log_destination?: string & "stderr" | "csvlog" + + // Sets the destination directory for log files. + log_directory?: string + + // Logs end of a session, including duration. + log_disconnections?: bool & false | true + + // Logs the duration of each completed SQL statement. + log_duration?: bool & false | true + + // Sets the verbosity of logged messages. + log_error_verbosity?: string & "terse" | "default" | "verbose" + + // Writes executor performance statistics to the server log. + log_executor_stats?: bool & false | true + + // Sets the file permissions for log files. + log_file_mode?: string + + // Sets the file name pattern for log files. + log_filename?: string + + // Start a subprocess to capture stderr output and/or csvlogs into log files. + logging_collector: bool & false | true | *true + + // Logs the host name in the connection logs. + log_hostname?: bool & false | true + + // (kB) Sets the maximum memory to be used for logical decoding. + logical_decoding_work_mem?: int & >=64 & <=2147483647 @storeResource(1KB) + + // Controls information prefixed to each log line. + log_line_prefix?: string + + // Logs long lock waits. + log_lock_waits?: bool & false | true + + // (ms) Sets the minimum execution time above which a sample of statements will be logged. Sampling is determined by log_statement_sample_rate. + log_min_duration_sample?: int & >=-1 & <=2147483647 @timeDurationResource() + + // (ms) Sets the minimum execution time above which statements will be logged. + log_min_duration_statement?: int & >=-1 & <=2147483647 @timeDurationResource() + + // Causes all statements generating error at or above this level to be logged. + log_min_error_statement?: string & "debug5" | "debug4" | "debug3" | "debug2" | "debug1" | "info" | "notice" | "warning" | "error" | "log" | "fatal" | "panic" + + // Sets the message levels that are logged. + log_min_messages?: string & "debug5" | "debug4" | "debug3" | "debug2" | "debug1" | "info" | "notice" | "warning" | "error" | "log" | "fatal" + + // When logging statements, limit logged parameter values to first N bytes. + log_parameter_max_length?: int & >=-1 & <=1073741823 + + // When reporting an error, limit logged parameter values to first N bytes. + log_parameter_max_length_on_error?: int & >=-1 & <=1073741823 + + // Writes parser performance statistics to the server log. + log_parser_stats?: bool & false | true + + // Writes planner performance statistics to the server log. + log_planner_stats?: bool & false | true + + // Controls whether a log message is produced when the startup process waits longer than deadlock_timeout for recovery conflicts + log_recovery_conflict_waits?: bool & false | true + + // Logs each replication command. + log_replication_commands?: bool & false | true + + // (min) Automatic log file rotation will occur after N minutes. + log_rotation_age: int & >=1 & <=1440 | *60 @timeDurationResource(1min) + + // (kB) Automatic log file rotation will occur after N kilobytes. + log_rotation_size?: int & >=0 & <=2097151 @storeResource(1KB) + + // Time between progress updates for long-running startup operations. Sets the amount of time after which the startup process will log a message about a long-running operation that is still in progress, as well as the interval between further progress messages for that operation. The default is 10 seconds. A setting of 0 disables the feature. If this value is specified without units, it is taken as milliseconds. This setting is applied separately to each operation. This parameter can only be set in the postgresql.conf file or on the server command line. + log_startup_progress_interval: int & >=0 & <=2147483647 @timeDurationResource() + + // Sets the type of statements logged. + log_statement?: string & "none" | "ddl" | "mod" | "all" + + // Fraction of statements exceeding log_min_duration_sample to be logged. + log_statement_sample_rate?: float & >=0 & <=1 + + // Writes cumulative performance statistics to the server log. + log_statement_stats?: bool + + // (kB) Log the use of temporary files larger than this number of kilobytes. + log_temp_files?: int & >=-1 & <=2147483647 @storeResource(1KB) + + // Sets the time zone to use in log messages. + log_timezone?: string + + // Set the fraction of transactions to log for new transactions. + log_transaction_sample_rate?: float & >=0 & <=1 + + // Truncate existing log files of same name during log rotation. + log_truncate_on_rotation: bool & false | true | *false + + // A variant of effective_io_concurrency that is used for maintenance work. + maintenance_io_concurrency?: int & >=0 & <=1000 + + // (kB) Sets the maximum memory to be used for maintenance operations. + maintenance_work_mem?: int & >=1024 & <=2147483647 @storeResource(1KB) + + // Sets the maximum number of concurrent connections. + max_connections?: int & >=6 & <=8388607 + + // Sets the maximum number of simultaneously open files for each server process. + max_files_per_process?: int & >=64 & <=2147483647 + + // Sets the maximum number of locks per transaction. + max_locks_per_transaction: int & >=10 & <=2147483647 | *64 + + // Maximum number of logical replication worker processes. + max_logical_replication_workers?: int & >=0 & <=262143 + + // Sets the maximum number of parallel processes per maintenance operation. + max_parallel_maintenance_workers?: int & >=0 & <=1024 + + // Sets the maximum number of parallel workers than can be active at one time. + max_parallel_workers?: int & >=0 & <=1024 + + // Sets the maximum number of parallel processes per executor node. + max_parallel_workers_per_gather?: int & >=0 & <=1024 + + // Sets the maximum number of predicate-locked tuples per page. + max_pred_locks_per_page?: int & >=0 & <=2147483647 + + // Sets the maximum number of predicate-locked pages and tuples per relation. + max_pred_locks_per_relation?: int & >=-2147483648 & <=2147483647 + + // Sets the maximum number of predicate locks per transaction. + max_pred_locks_per_transaction?: int & >=10 & <=2147483647 + + // Sets the maximum number of simultaneously prepared transactions. + max_prepared_transactions: int & >=0 & <=8388607 | *0 + + // Sets the maximum number of replication slots that the server can support. + max_replication_slots: int & >=5 & <=8388607 | *20 + + // (kB) Sets the maximum stack depth, in kilobytes. + max_stack_depth: int & >=100 & <=2147483647 | *6144 @storeResource(1KB) + + // (ms) Sets the maximum delay before canceling queries when a hot standby server is processing archived WAL data. + max_standby_archive_delay?: int & >=-1 & <=2147483647 @timeDurationResource() + + // (ms) Sets the maximum delay before canceling queries when a hot standby server is processing streamed WAL data. + max_standby_streaming_delay?: int & >=-1 & <=2147483647 @timeDurationResource() + + // Maximum number of synchronization workers per subscription + max_sync_workers_per_subscription?: int & >=0 & <=262143 + + // Sets the maximum number of simultaneously running WAL sender processes. + max_wal_senders: int & >=5 & <=8388607 | *20 + + // (MB) Sets the WAL size that triggers a checkpoint. + max_wal_size: int & >=128 & <=201326592 | *2048 @storeResource(1MB) + + // Sets the maximum number of concurrent worker processes. + max_worker_processes?: int & >=0 & <=262143 + + // Specifies the amount of memory that should be allocated at server startup for use by parallel queries + min_dynamic_shared_memory?: int & >=0 & <=715827882 @storeResource(1MB) + + // (8kB) Sets the minimum amount of index data for a parallel scan. + min_parallel_index_scan_size?: int & >=0 & <=715827882 @storeResource(8KB) + + // Sets the minimum size of relations to be considered for parallel scan. Sets the minimum size of relations to be considered for parallel scan. + min_parallel_relation_size?: int & >=0 & <=715827882 @storeResource(8KB) + + // (8kB) Sets the minimum amount of table data for a parallel scan. + min_parallel_table_scan_size?: int & >=0 & <=715827882 @storeResource(8KB) + + // (MB) Sets the minimum size to shrink the WAL to. + min_wal_size: int & >=128 & <=201326592 | *192 @storeResource(1MB) + + // (min) Time before a snapshot is too old to read pages changed after the snapshot was taken. + old_snapshot_threshold?: int & >=-1 & <=86400 @timeDurationResource(1min) + + // Emulate oracle's date output behaviour. + "orafce.nls_date_format"?: string + + // Specify timezone used for sysdate function. + "orafce.timezone"?: string + + // Controls whether Gather and Gather Merge also run subplans. + parallel_leader_participation?: bool & false | true + + // Sets the planner's estimate of the cost of starting up worker processes for parallel query. + parallel_setup_cost?: float & >=0 & <=1.79769e+308 + + // Sets the planner's estimate of the cost of passing each tuple (row) from worker to master backend. + parallel_tuple_cost?: float & >=0 & <=1.79769e+308 + + // Encrypt passwords. + password_encryption?: string & "md5" | "scram-sha-256" + + // Specifies which classes of statements will be logged by session audit logging. + "pgaudit.log"?: string & "ddl" | "function" | "misc" | "read" | "role" | "write" | "none" | "all" | "-ddl" | "-function" | "-misc" | "-read" | "-role" | "-write" + + // Specifies that session logging should be enabled in the case where all relations in a statement are in pg_catalog. + "pgaudit.log_catalog"?: bool & false | true + + // Specifies the log level that will be used for log entries. + "pgaudit.log_level"?: string & "debug5" | "debug4" | "debug3" | "debug2" | "debug1" | "info" | "notice" | "warning" | "log" + + // Specifies that audit logging should include the parameters that were passed with the statement. + "pgaudit.log_parameter"?: bool & false | true + + // Specifies whether session audit logging should create a separate log entry for each relation (TABLE, VIEW, etc.) referenced in a SELECT or DML statement. + "pgaudit.log_relation"?: bool & false | true + + // Specifies that audit logging should include the rows retrieved or affected by a statement. + "pgaudit.log_rows": bool & false | true | *false + + // Specifies whether logging will include the statement text and parameters (if enabled). + "pgaudit.log_statement": bool & false | true | *true + + // Specifies whether logging will include the statement text and parameters with the first log entry for a statement/substatement combination or with every entry. + "pgaudit.log_statement_once"?: bool & false | true + + // Specifies the master role to use for object audit logging. + "pgaudit.role"?: string + + // It specifies whether to perform Recheck which is an internal process of full text search. + "pg_bigm.enable_recheck"?: string & "on" | "off" + + // It specifies the maximum number of 2-grams of the search keyword to be used for full text search. + "pg_bigm.gin_key_limit": int & >=0 & <=2147483647 | *0 + + // It specifies the minimum threshold used by the similarity search. + "pg_bigm.similarity_limit": float & >=0 & <=1 | *0.3 + + // Logs results of hint parsing. + "pg_hint_plan.debug_print"?: string & "off" | "on" | "detailed" | "verbose" + + // Force planner to use plans specified in the hint comment preceding to the query. + "pg_hint_plan.enable_hint"?: bool & false | true + + // Force planner to not get hint by using table lookups. + "pg_hint_plan.enable_hint_table"?: bool & false | true + + // Message level of debug messages. + "pg_hint_plan.message_level"?: string & "debug5" | "debug4" | "debug3" | "debug2" | "debug1" | "log" | "info" | "notice" | "warning" | "error" + + // Message level of parse errors. + "pg_hint_plan.parse_messages"?: string & "debug5" | "debug4" | "debug3" | "debug2" | "debug1" | "log" | "info" | "notice" | "warning" | "error" + + // Batch inserts if possible + "pglogical.batch_inserts"?: bool & false | true + + // Sets log level used for logging resolved conflicts. + "pglogical.conflict_log_level"?: string & "debug5" | "debug4" | "debug3" | "debug2" | "debug1" | "info" | "notice" | "warning" | "error" | "log" | "fatal" | "panic" + + // Sets method used for conflict resolution for resolvable conflicts. + "pglogical.conflict_resolution"?: string & "error" | "apply_remote" | "keep_local" | "last_update_wins" | "first_update_wins" + + // connection options to add to all peer node connections + "pglogical.extra_connection_options"?: string + + // pglogical specific synchronous commit value + "pglogical.synchronous_commit"?: bool & false | true + + // Use SPI instead of low-level API for applying changes + "pglogical.use_spi"?: bool & false | true + + // Starts the autoprewarm worker. + "pg_prewarm.autoprewarm"?: bool & false | true + + // Sets the interval between dumps of shared buffers + "pg_prewarm.autoprewarm_interval"?: int & >=0 & <=2147483 + + // Sets if the result value is normalized or not. + "pg_similarity.block_is_normalized"?: bool & false | true + + // Sets the threshold used by the Block similarity function. + "pg_similarity.block_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Block similarity function. + "pg_similarity.block_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets if the result value is normalized or not. + "pg_similarity.cosine_is_normalized"?: bool & false | true + + // Sets the threshold used by the Cosine similarity function. + "pg_similarity.cosine_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Cosine similarity function. + "pg_similarity.cosine_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets if the result value is normalized or not. + "pg_similarity.dice_is_normalized"?: bool & false | true + + // Sets the threshold used by the Dice similarity measure. + "pg_similarity.dice_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Dice similarity measure. + "pg_similarity.dice_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets if the result value is normalized or not. + "pg_similarity.euclidean_is_normalized"?: bool & false | true + + // Sets the threshold used by the Euclidean similarity measure. + "pg_similarity.euclidean_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Euclidean similarity measure. + "pg_similarity.euclidean_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets if the result value is normalized or not. + "pg_similarity.hamming_is_normalized"?: bool & false | true + + // Sets the threshold used by the Block similarity metric. + "pg_similarity.hamming_threshold"?: float & >=0 & <=1 + + // Sets if the result value is normalized or not. + "pg_similarity.jaccard_is_normalized"?: bool & false | true + + // Sets the threshold used by the Jaccard similarity measure. + "pg_similarity.jaccard_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Jaccard similarity measure. + "pg_similarity.jaccard_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets if the result value is normalized or not. + "pg_similarity.jaro_is_normalized"?: bool & false | true + + // Sets the threshold used by the Jaro similarity measure. + "pg_similarity.jaro_threshold"?: float & >=0 & <=1 + + // Sets if the result value is normalized or not. + "pg_similarity.jarowinkler_is_normalized"?: bool & false | true + + // Sets the threshold used by the Jarowinkler similarity measure. + "pg_similarity.jarowinkler_threshold"?: float & >=0 & <=1 + + // Sets if the result value is normalized or not. + "pg_similarity.levenshtein_is_normalized"?: bool & false | true + + // Sets the threshold used by the Levenshtein similarity measure. + "pg_similarity.levenshtein_threshold"?: float & >=0 & <=1 + + // Sets if the result value is normalized or not. + "pg_similarity.matching_is_normalized"?: bool & false | true + + // Sets the threshold used by the Matching Coefficient similarity measure. + "pg_similarity.matching_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Matching Coefficient similarity measure. + "pg_similarity.matching_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets if the result value is normalized or not. + "pg_similarity.mongeelkan_is_normalized"?: bool & false | true + + // Sets the threshold used by the Monge-Elkan similarity measure. + "pg_similarity.mongeelkan_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Monge-Elkan similarity measure. + "pg_similarity.mongeelkan_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets the gap penalty used by the Needleman-Wunsch similarity measure. + "pg_similarity.nw_gap_penalty"?: float & >=-9.22337e+18 & <=9.22337e+18 + + // Sets if the result value is normalized or not. + "pg_similarity.nw_is_normalized"?: bool & false | true + + // Sets the threshold used by the Needleman-Wunsch similarity measure. + "pg_similarity.nw_threshold"?: float & >=0 & <=1 + + // Sets if the result value is normalized or not. + "pg_similarity.overlap_is_normalized"?: bool & false | true + + // Sets the threshold used by the Overlap Coefficient similarity measure. + "pg_similarity.overlap_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Overlap Coefficientsimilarity measure. + "pg_similarity.overlap_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets if the result value is normalized or not. + "pg_similarity.qgram_is_normalized"?: bool & false | true + + // Sets the threshold used by the Q-Gram similarity measure. + "pg_similarity.qgram_threshold"?: float & >=0 & <=1 + + // Sets the tokenizer for Q-Gram measure. + "pg_similarity.qgram_tokenizer"?: string & "alnum" | "gram" | "word" | "camelcase" + + // Sets if the result value is normalized or not. + "pg_similarity.swg_is_normalized"?: bool & false | true + + // Sets the threshold used by the Smith-Waterman-Gotoh similarity measure. + "pg_similarity.swg_threshold"?: float & >=0 & <=1 + + // Sets if the result value is normalized or not. + "pg_similarity.sw_is_normalized"?: bool & false | true + + // Sets the threshold used by the Smith-Waterman similarity measure. + "pg_similarity.sw_threshold"?: float & >=0 & <=1 + + // Sets the maximum number of statements tracked by pg_stat_statements. + "pg_stat_statements.max"?: int & >=100 & <=2147483647 + + // Save pg_stat_statements statistics across server shutdowns. + "pg_stat_statements.save"?: bool & false | true + + // Selects which statements are tracked by pg_stat_statements. + "pg_stat_statements.track"?: string & "none" | "top" | "all" + + // Selects whether planning duration is tracked by pg_stat_statements. + "pg_stat_statements.track_planning"?: bool & false | true + + // Selects whether utility commands are tracked by pg_stat_statements. + "pg_stat_statements.track_utility"?: bool & false | true + + // Sets the behavior for interacting with passcheck feature. + "pgtle.enable_password_check"?: string & "on" | "off" | "require" + + // Number of workers to use for a physical transport. + "pg_transport.num_workers"?: int & >=1 & <=32 + + // Specifies whether to report timing information during transport. + "pg_transport.timing"?: bool & false | true + + // (kB) Amount of memory each worker can allocate for a physical transport. + "pg_transport.work_mem"?: int & >=65536 & <=2147483647 @storeResource(1KB) + + // Controls the planner selection of custom or generic plan. + plan_cache_mode?: string & "auto" | "force_generic_plan" | "force_custom_plan" + + // Sets the TCP port the server listens on. + port?: int & >=1 & <=65535 + + // Sets the amount of time to wait after authentication on connection startup. The amount of time to delay when a new server process is started, after it conducts the authentication procedure. This is intended to give developers an opportunity to attach to the server process with a debugger. If this value is specified without units, it is taken as seconds. A value of zero (the default) disables the delay. This parameter cannot be changed after session start. + post_auth_delay?: int & >=0 & <=2147 @timeDurationResource(1s) + + // Sets the amount of time to wait before authentication on connection startup. The amount of time to delay just after a new server process is forked, before it conducts the authentication procedure. This is intended to give developers an opportunity to attach to the server process with a debugger to trace down misbehavior in authentication. If this value is specified without units, it is taken as seconds. A value of zero (the default) disables the delay. This parameter can only be set in the postgresql.conf file or on the server command line. + pre_auth_delay?: int & >=0 & <=60 @timeDurationResource(1s) + + // Enable for disable GDAL drivers used with PostGIS in Postgres 9.3.5 and above. + "postgis.gdal_enabled_drivers"?: string & "ENABLE_ALL" | "DISABLE_ALL" + + // When generating SQL fragments, quote all identifiers. + quote_all_identifiers?: bool & false | true + + // Sets the planners estimate of the cost of a nonsequentially fetched disk page. + random_page_cost?: float & >=0 & <=1.79769e+308 + + // Lower threshold of Dice similarity. Molecules with similarity lower than threshold are not similar by # operation. + "rdkit.dice_threshold"?: float & >=0 & <=1 + + // Should stereochemistry be taken into account in substructure matching. If false, no stereochemistry information is used in substructure matches. + "rdkit.do_chiral_sss"?: bool & false | true + + // Should enhanced stereochemistry be taken into account in substructure matching. + "rdkit.do_enhanced_stereo_sss"?: bool & false | true + + // Lower threshold of Tanimoto similarity. Molecules with similarity lower than threshold are not similar by % operation. + "rdkit.tanimoto_threshold"?: float & >=0 & <=1 + + // When set to fsync, PostgreSQL will recursively open and synchronize all files in the data directory before crash recovery begins + recovery_init_sync_method?: string & "fsync" | "syncfs" + + // When set to on, which is the default, PostgreSQL will automatically remove temporary files after a backend crash + remove_temp_files_after_crash: float & >=0 & <=1 | *0 + + // Reinitialize server after backend crash. + restart_after_crash?: bool & false | true + + // Enable row security. + row_security?: bool & false | true + + // Sets the schema search order for names that are not schema-qualified. + search_path?: string + + // Sets the planners estimate of the cost of a sequentially fetched disk page. + seq_page_cost?: float & >=0 & <=1.79769e+308 + + // Lists shared libraries to preload into each backend. + session_preload_libraries?: string & "auto_explain" | "orafce" | "pg_bigm" | "pg_hint_plan" | "pg_prewarm" | "pg_similarity" | "pg_stat_statements" | "pg_transport" | "plprofiler" + + // Sets the sessions behavior for triggers and rewrite rules. + session_replication_role?: string & "origin" | "replica" | "local" + + // (8kB) Sets the number of shared memory buffers used by the server. + shared_buffers?: int & >=16 & <=1073741823 @storeResource(8KB) + + // Lists shared libraries to preload into server. + // TODO support enum list, e.g. shared_preload_libraries = 'pg_stat_statements, auto_explain' + // shared_preload_libraries?: string & "auto_explain" | "orafce" | "pgaudit" | "pglogical" | "pg_bigm" | "pg_cron" | "pg_hint_plan" | "pg_prewarm" | "pg_similarity" | "pg_stat_statements" | "pg_tle" | "pg_transport" | "plprofiler" + + // Enables SSL connections. + ssl: bool & false | true | *true + + // Location of the SSL server authority file. + ssl_ca_file?: string + + // Location of the SSL server certificate file. + ssl_cert_file?: string + + // Sets the list of allowed SSL ciphers. + ssl_ciphers?: string + + // Location of the SSL server private key file + ssl_key_file?: string + + // Sets the maximum SSL/TLS protocol version to use. + ssl_max_protocol_version?: string & "TLSv1" | "TLSv1.1" | "TLSv1.2" + + // Sets the minimum SSL/TLS protocol version to use. + ssl_min_protocol_version?: string & "TLSv1" | "TLSv1.1" | "TLSv1.2" + + // Causes ... strings to treat backslashes literally. + standard_conforming_strings?: bool & false | true + + // (ms) Sets the maximum allowed duration of any statement. + statement_timeout?: int & >=0 & <=2147483647 @timeDurationResource() + + // Writes temporary statistics files to the specified directory. + stats_temp_directory?: string + + // Sets the number of connection slots reserved for superusers. + superuser_reserved_connections: int & >=0 & <=8388607 | *3 + + // Enable synchronized sequential scans. + synchronize_seqscans?: bool & false | true + + // Sets the current transactions synchronization level. + synchronous_commit?: string & "local" | "on" | "off" + + // Maximum number of TCP keepalive retransmits. + tcp_keepalives_count?: int & >=0 & <=2147483647 + + // (s) Time between issuing TCP keepalives. + tcp_keepalives_idle?: int & >=0 & <=2147483647 @timeDurationResource(1s) + + // (s) Time between TCP keepalive retransmits. + tcp_keepalives_interval?: int & >=0 & <=2147483647 @timeDurationResource(1s) + + // TCP user timeout. Specifies the amount of time that transmitted data may remain unacknowledged before the TCP connection is forcibly closed. If this value is specified without units, it is taken as milliseconds. A value of 0 (the default) selects the operating system's default. This parameter is supported only on systems that support TCP_USER_TIMEOUT; on other systems, it must be zero. In sessions connected via a Unix-domain socket, this parameter is ignored and always reads as zero. + tcp_user_timeout?: int & >=0 & <=2147483647 @timeDurationResource() + + // (8kB) Sets the maximum number of temporary buffers used by each session. + temp_buffers?: int & >=100 & <=1073741823 @storeResource(8KB) + + // (kB) Limits the total size of all temporary files used by each process. + temp_file_limit?: int & >=-1 & <=2147483647 @storeResource(1KB) + + // Sets the tablespace(s) to use for temporary tables and sort files. + temp_tablespaces?: string + + // Sets the time zone for displaying and interpreting time stamps. + timezone?: string + + // Collects information about executing commands. + track_activities?: bool & false | true + + // Sets the size reserved for pg_stat_activity.current_query, in bytes. + track_activity_query_size: int & >=100 & <=1048576 | *4096 @storeResource() + + // Collects transaction commit time. + track_commit_timestamp?: bool & false | true + + // Collects statistics on database activity. + track_counts?: bool & false | true + + // Collects function-level statistics on database activity. + track_functions?: string & "none" | "pl" | "all" + + // Collects timing statistics on database IO activity. + track_io_timing: bool & false | true | *true + + // Enables timing of WAL I/O calls. + track_wal_io_timing?: bool & false | true + + // Treats expr=NULL as expr IS NULL. + transform_null_equals?: bool & false | true + + // Sets the directory where the Unix-domain socket will be created. + unix_socket_directories?: string + + // Sets the owning group of the Unix-domain socket. + unix_socket_group?: string + + // Sets the access permissions of the Unix-domain socket. + unix_socket_permissions?: int & >=0 & <=511 + + // Updates the process title to show the active SQL command. + update_process_title: bool & false | true | *true + + // (ms) Vacuum cost delay in milliseconds. + vacuum_cost_delay?: int & >=0 & <=100 @timeDurationResource() + + // Vacuum cost amount available before napping. + vacuum_cost_limit?: int & >=1 & <=10000 + + // Vacuum cost for a page dirtied by vacuum. + vacuum_cost_page_dirty?: int & >=0 & <=10000 + + // Vacuum cost for a page found in the buffer cache. + vacuum_cost_page_hit?: int & >=0 & <=10000 + + // Vacuum cost for a page not found in the buffer cache. + vacuum_cost_page_miss: int & >=0 & <=10000 | *5 + + // Number of transactions by which VACUUM and HOT cleanup should be deferred, if any. + vacuum_defer_cleanup_age?: int & >=0 & <=1000000 + + // Specifies the maximum age (in transactions) that a table's pg_class.relfrozenxid field can attain before VACUUM takes extraordinary measures to avoid system-wide transaction ID wraparound failure + vacuum_failsafe_age: int & >=0 & <=1200000000 | *1200000000 + + // Minimum age at which VACUUM should freeze a table row. + vacuum_freeze_min_age?: int & >=0 & <=1000000000 + + // Age at which VACUUM should scan whole table to freeze tuples. + vacuum_freeze_table_age?: int & >=0 & <=2000000000 + + // Specifies the maximum age (in transactions) that a table's pg_class.relminmxid field can attain before VACUUM takes extraordinary measures to avoid system-wide multixact ID wraparound failure + vacuum_multixact_failsafe_age: int & >=0 & <=1200000000 | *1200000000 + + // Minimum age at which VACUUM should freeze a MultiXactId in a table row. + vacuum_multixact_freeze_min_age?: int & >=0 & <=1000000000 + + // Multixact age at which VACUUM should scan whole table to freeze tuples. + vacuum_multixact_freeze_table_age?: int & >=0 & <=2000000000 + + // (8kB) Sets the number of disk-page buffers in shared memory for WAL. + wal_buffers?: int & >=-1 & <=262143 @storeResource(8KB) + + // Compresses full-page writes written in WAL file. + wal_compression: bool & false | true | *true + + // Sets the WAL resource managers for which WAL consistency checks are done. + wal_consistency_checking?: string + + // Buffer size for reading ahead in the WAL during recovery. + wal_decode_buffer_size: int & >=65536 & <=1073741823 | *524288 @storeResource() + + // (MB) Sets the size of WAL files held for standby servers. + wal_keep_size: int & >=0 & <=2147483647 | *2048 @storeResource(1MB) + + // Sets whether a WAL receiver should create a temporary replication slot if no permanent slot is configured. + wal_receiver_create_temp_slot: bool & false | true | *false + + // (s) Sets the maximum interval between WAL receiver status reports to the primary. + wal_receiver_status_interval?: int & >=0 & <=2147483 @timeDurationResource(1s) + + // (ms) Sets the maximum wait time to receive data from the primary. + wal_receiver_timeout: int & >=0 & <=3600000 | *30000 @timeDurationResource() + + // Recycles WAL files by renaming them. If set to on (the default), this option causes WAL files to be recycled by renaming them, avoiding the need to create new ones. On COW file systems, it may be faster to create new ones, so the option is given to disable this behavior. + wal_recycle?: bool + + // Sets the time to wait before retrying to retrieve WAL after a failed attempt. Specifies how long the standby server should wait when WAL data is not available from any sources (streaming replication, local pg_wal or WAL archive) before trying again to retrieve WAL data. If this value is specified without units, it is taken as milliseconds. The default value is 5 seconds. This parameter can only be set in the postgresql.conf file or on the server command line. + wal_retrieve_retry_interval: int & >=1 & <=2147483647 | *5000 @timeDurationResource() + + // (ms) Sets the maximum time to wait for WAL replication. + wal_sender_timeout: int & >=0 & <=3600000 | *30000 @timeDurationResource() + + // (kB) Size of new file to fsync instead of writing WAL. + wal_skip_threshold?: int & >=0 & <=2147483647 @storeResource(1KB) + + // Selects the method used for forcing WAL updates to disk. + wal_sync_method?: string & "fsync" | "fdatasync" | "open_sync" | "open_datasync" + + // (ms) WAL writer sleep time between WAL flushes. + wal_writer_delay?: int & >=1 & <=10000 @timeDurationResource() + + // (8Kb) Amount of WAL written out by WAL writer triggering a flush. + wal_writer_flush_after?: int & >=0 & <=2147483647 @storeResource(8KB) + + // (kB) Sets the maximum memory to be used for query workspaces. + work_mem?: int & >=64 & <=2147483647 @storeResource(1KB) + + // Sets how binary values are to be encoded in XML. + xmlbinary?: string & "base64" | "hex" + + // Sets whether XML data in implicit parsing and serialization operations is to be considered as documents or content fragments. + xmloption?: string & "content" | "document" + + ... +} + +configuration: #PGParameter & { +} diff --git a/deploy/postgresql/config/pg12-config-effect-scope.yaml b/deploy/postgresql/config/pg12-config-effect-scope.yaml new file mode 100644 index 000000000..e103dfaa9 --- /dev/null +++ b/deploy/postgresql/config/pg12-config-effect-scope.yaml @@ -0,0 +1,62 @@ +# Patroni bootstrap parameters +staticParameters: + - archive_command + - shared_buffers + - logging_collector + - log_destination + - log_directory + - log_filename + - log_file_mode + - log_rotation_age + - log_truncate_on_rotation + - ssl + - ssl_ca_file + - ssl_crl_file + - ssl_cert_file + - ssl_key_file + - shared_preload_libraries + - bg_mon.listen_address + - bg_mon.history_buckets + - pg_stat_statements.track_utility + - extwlist.extensions + - extwlist.custom_path + +immutableParameters: + - archive_command + - archive_timeout + - backtrace_functions + - config_file + - cron.use_background_workers + - data_directory + - db_user_namespace + - exit_on_error + - fsync + - full_page_writes + - hba_file + - ident_file + - ignore_invalid_pages + - listen_addresses + - lo_compat_privileges + - log_directory + - log_file_mode + - logging_collector + - log_line_prefix + - log_timezone + - log_truncate_on_rotation + - port + - rds.max_tcp_buffers + - recovery_init_sync_method + - restart_after_crash + - ssl + - ssl_ca_file + - ssl_cert_file + - ssl_ciphers + - ssl_key_file + - stats_temp_directory + - superuser_reserved_connections + - unix_socket_directories + - unix_socket_group + - unix_socket_permissions + - update_process_title + - wal_receiver_create_temp_slot + - wal_sync_method diff --git a/deploy/postgresql/config/pg12-config.tpl b/deploy/postgresql/config/pg12-config.tpl new file mode 100644 index 000000000..d84146de9 --- /dev/null +++ b/deploy/postgresql/config/pg12-config.tpl @@ -0,0 +1,133 @@ +# - Connection Settings - + +{{- $buffer_unit := "B" }} +{{- $shared_buffers := 1073741824 }} +{{- $max_connections := 10000 }} +{{- $phy_memory := getContainerMemory ( index $.podSpec.containers 0 ) }} +{{- if gt $phy_memory 0 }} +{{- $shared_buffers = div $phy_memory 4 }} +{{- $max_connections = min ( div $phy_memory 9531392 ) 5000 }} +{{- end -}} + +{{- if ge $shared_buffers 1024 }} +{{- $shared_buffers = div $shared_buffers 1024 }} +{{- $buffer_unit = "KB" }} +{{- end -}} + +{{- if ge $shared_buffers 1024 }} +{{- $shared_buffers = div $shared_buffers 1024 }} +{{- $buffer_unit = "MB" }} +{{- end -}} + +{{- if ge $shared_buffers 1024 }} +{{- $shared_buffers = div $shared_buffers 1024 }} +{{ $buffer_unit = "GB" }} +{{- end -}} + +listen_addresses = '*' +port = '5432' +archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' +archive_mode = 'on' +auto_explain.log_analyze = 'True' +auto_explain.log_min_duration = '1s' +auto_explain.log_nested_statements = 'True' +auto_explain.log_timing = 'True' +auto_explain.log_verbose = 'True' +autovacuum_analyze_scale_factor = '0.05' +autovacuum_freeze_max_age = '100000000' +autovacuum_max_workers = '1' +autovacuum_naptime = '1min' +autovacuum_vacuum_cost_delay = '-1' +autovacuum_vacuum_cost_limit = '-1' +autovacuum_vacuum_scale_factor = '0.1' +bgwriter_delay = '10ms' +bgwriter_lru_maxpages = '800' +bgwriter_lru_multiplier = '5.0' +checkpoint_completion_target = '0.95' +checkpoint_timeout = '10min' +commit_delay = '20' +commit_siblings = '10' +deadlock_timeout = '50ms' +default_statistics_target = '500' +effective_cache_size = '12GB' +hot_standby = 'on' +hot_standby_feedback = 'True' +huge_pages = 'try' +idle_in_transaction_session_timeout = '1h' +listen_addresses = '0.0.0.0' +log_autovacuum_min_duration = '1s' +log_checkpoints = 'True' + +{{- block "logsBlock" . }} +{{- if hasKey $.component "enabledLogs" }} +{{- if mustHas "running" $.component.enabledLogs }} +logging_collector = 'True' +log_destination = 'csvlog' +log_directory = 'log' +log_filename = 'postgresql-%Y-%m-%d.log' +{{ end -}} +{{ end -}} +{{ end }} + +log_lock_waits = 'True' +log_min_duration_statement = '100' +log_replication_commands = 'True' +log_statement = 'ddl' +#maintenance_work_mem = '3952MB' +max_connections = '{{ $max_connections }}' +max_locks_per_transaction = '128' +max_logical_replication_workers = '8' +max_parallel_maintenance_workers = '2' +max_parallel_workers = '8' +max_parallel_workers_per_gather = '0' +max_prepared_transactions = '0' +max_replication_slots = '16' +max_standby_archive_delay = '10min' +max_standby_streaming_delay = '3min' +max_sync_workers_per_subscription = '6' +max_wal_senders = '24' +max_wal_size = '100GB' +max_worker_processes = '8' +min_wal_size = '20GB' +password_encryption = 'md5' +pg_stat_statements.max = '5000' +pg_stat_statements.track = 'all' +pg_stat_statements.track_planning = 'False' +pg_stat_statements.track_utility = 'False' +random_page_cost = '1.1' +#auto generated +shared_buffers = '{{ printf "%d%s" $shared_buffers $buffer_unit }}' +# shared_preload_libraries = 'pg_stat_statements,auto_explain,bg_mon,pgextwlist,pg_auth_mon,set_user,pg_cron,pg_stat_kcache' +superuser_reserved_connections = '10' +temp_file_limit = '100GB' +#timescaledb.max_background_workers = '6' +#timescaledb.telemetry_level = 'off' +track_activity_query_size = '8192' +track_commit_timestamp = 'True' +track_functions = 'all' +track_io_timing = 'True' +vacuum_cost_delay = '2ms' +vacuum_cost_limit = '10000' +vacuum_defer_cleanup_age = '50000' +wal_buffers = '16MB' +wal_level = 'replica' +wal_log_hints = 'on' +wal_receiver_status_interval = '1s' +wal_receiver_timeout = '60s' +wal_writer_delay = '20ms' +wal_writer_flush_after = '1MB' +work_mem = '32MB' + +{{- if $.component.tls }} +{{- $ca_file := getCAFile }} +{{- $cert_file := getCertFile }} +{{- $key_file := getKeyFile }} +# tls +ssl=ON +ssl_ca_file={{ $ca_file }} +ssl_cert_file={{ $cert_file }} +ssl_key_file={{ $key_file }} +{{- end }} + +# TODO: check the following parameters, how to set the default value +# wal_keep_segments=128 \ No newline at end of file diff --git a/deploy/postgresql/templates/clusterversion.yaml b/deploy/postgresql/templates/clusterversion.yaml index 9b611bd20..4800dac3e 100644 --- a/deploy/postgresql/templates/clusterversion.yaml +++ b/deploy/postgresql/templates/clusterversion.yaml @@ -1,7 +1,8 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterVersion metadata: - name: postgresql-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} + # major version of the component defined in values.yaml + name: postgresql-{{ .Values.image.tag }} labels: {{- include "postgresql.labels" . | nindent 4 }} spec: @@ -15,4 +16,33 @@ spec: containers: - name: postgresql image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + +--- +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterVersion +metadata: + name: postgresql-12.14.0 + labels: + {{- include "postgresql.labels" . | nindent 4 }} +spec: + clusterDefinitionRef: postgresql + componentVersions: + - componentDefRef: postgresql + configSpecs: + # name needs to consistent with the name of the configmap defined in clusterDefinition, and replace the templateRef with postgres v12.14.0 configmap + - name: postgresql-configuration + templateRef: postgresql12-configuration + constraintRef: postgresql12-cc + keys: + - postgresql.conf + namespace: {{ .Release.Namespace }} + volumeName: postgresql-config + defaultMode: 0777 + versionsContext: + initContainers: + - name: pg-init-container + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0 + containers: + - name: postgresql + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0 --- diff --git a/deploy/postgresql/templates/configconstraint-12.yaml b/deploy/postgresql/templates/configconstraint-12.yaml new file mode 100644 index 000000000..9d8ad01a0 --- /dev/null +++ b/deploy/postgresql/templates/configconstraint-12.yaml @@ -0,0 +1,53 @@ +{{- $cc := .Files.Get "config/pg14-config-effect-scope.yaml" | fromYaml }} +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ConfigConstraint +metadata: + name: postgresql12-cc + labels: + {{- include "postgresql.labels" . | nindent 4 }} +spec: + reloadOptions: + tplScriptTrigger: + sync: true + scriptConfigMapRef: patroni-reload-script + namespace: {{ .Release.Namespace }} + + # update patroni master + selector: + matchLabels: + "apps.kubeblocks.postgres.patroni/role": "master" + + # top level mysql configuration type + cfgSchemaTopLevelName: PGParameter + + # ConfigurationSchema that impose restrictions on engine parameter's rule + configurationSchema: + # schema: auto generate from mmmcue scripts + # example: ../../internal/configuration/testdata/mysql_openapi.json + cue: |- + {{- .Files.Get "config/pg12-config-constraint.cue" | nindent 6 }} + + ## require db instance restart + ## staticParameters + {{- if hasKey $cc "staticParameters" }} + staticParameters: + {{- $params := get $cc "staticParameters" }} + {{- range $params }} + - {{ . }} + {{- end }} + {{- end}} + + ## define immutable parameter list, this feature is not currently supported. + {{- if hasKey $cc "immutableParameters" }} + immutableParameters: + {{- $params := get $cc "immutableParameters" }} + {{- range $params }} + - {{ . }} + {{- end }} + {{- end}} + + + + # configuration file format + formatterConfig: + format: properties diff --git a/deploy/postgresql/templates/configconstraint.yaml b/deploy/postgresql/templates/configconstraint-14.yaml similarity index 100% rename from deploy/postgresql/templates/configconstraint.yaml rename to deploy/postgresql/templates/configconstraint-14.yaml diff --git a/deploy/postgresql/templates/configmap-12.yaml b/deploy/postgresql/templates/configmap-12.yaml new file mode 100644 index 000000000..69b3e4993 --- /dev/null +++ b/deploy/postgresql/templates/configmap-12.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgresql12-configuration + labels: + {{- include "postgresql.labels" . | nindent 4 }} +data: + postgresql.conf: |- + {{- .Files.Get "config/pg12-config.tpl" | nindent 4 }} + # TODO: check if it should trust all + pg_hba.conf: | + host all all 0.0.0.0/0 trust + host all all ::/0 trust + local all all trust + host all all 127.0.0.1/32 trust + host all all ::1/128 trust + local replication all trust + host replication all 0.0.0.0/0 md5 + host replication all ::/0 md5 + kb_restore.conf: | + method: kb_restore_from_backup + kb_restore_from_backup: + command: bash /home/postgres/pgdata/kb_restore/kb_restore.sh + keep_existing_recovery_conf: false + recovery_conf: + restore_command: cp /home/postgres/pgdata/pgroot/arch/%f %p + recovery_target_timeline: latest + kb_pitr.conf: | + method: kb_restore_from_time + kb_restore_from_time: + command: bash /home/postgres/pgdata/kb_restore/kb_restore.sh + keep_existing_recovery_conf: false + recovery_conf: {} \ No newline at end of file diff --git a/deploy/postgresql/templates/configmap.yaml b/deploy/postgresql/templates/configmap-14.yaml similarity index 100% rename from deploy/postgresql/templates/configmap.yaml rename to deploy/postgresql/templates/configmap-14.yaml From 1844742ae2b2acd2f9ebef3783c37ea879525ac3 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Thu, 27 Apr 2023 10:29:05 +0800 Subject: [PATCH 195/439] chore: improve send cluster warning events (#2988) --- .../lifecycle/cluster_plan_builder.go | 12 +++--------- .../controller/lifecycle/transform_utils.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index b71925679..c99a32320 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -139,17 +139,11 @@ func (c *clusterPlanBuilder) Build() (graph.Plan, error) { } // if pre-check failed, this is a fast return, no need to set apply resource condition if preCheckCondition.Status != metav1.ConditionTrue { + sendWaringEventWithError(c.transCtx.GetRecorder(), c.transCtx.Cluster, ReasonPreCheckFailed, err) return } setApplyResourceCondition(&c.transCtx.Cluster.Status.Conditions, c.transCtx.Cluster.Generation, err) - if err != nil && !IsRequeueError(err) { - reason := ReasonApplyResourcesFailed - controllerErr := intctrlutil.ToControllerError(err) - if controllerErr != nil { - reason = string(controllerErr.Type) - } - c.transCtx.GetRecorder().Event(c.transCtx.Cluster, corev1.EventTypeWarning, reason, err.Error()) - } + sendWaringEventWithError(c.transCtx.GetRecorder(), c.transCtx.Cluster, ReasonApplyResourcesFailed, err) }() // new a DAG and apply chain on it, after that we should get the final Plan @@ -183,7 +177,7 @@ func (p *clusterPlan) Execute() error { func (p *clusterPlan) handlePlanExecutionError(err error) error { condition := newFailedApplyResourcesCondition(err.Error()) meta.SetStatusCondition(&p.transCtx.Cluster.Status.Conditions, condition) - p.transCtx.EventRecorder.Event(p.transCtx.Cluster, corev1.EventTypeWarning, condition.Reason, condition.Message) + sendWaringEventWithError(p.transCtx.GetRecorder(), p.transCtx.Cluster, ReasonApplyResourcesFailed, err) return p.cli.Status().Patch(p.transCtx.Context, p.transCtx.Cluster, client.MergeFrom(p.transCtx.OrigCluster.DeepCopy())) } diff --git a/internal/controller/lifecycle/transform_utils.go b/internal/controller/lifecycle/transform_utils.go index 45cbdd8b6..3461c29ef 100644 --- a/internal/controller/lifecycle/transform_utils.go +++ b/internal/controller/lifecycle/transform_utils.go @@ -30,6 +30,7 @@ import ( policyv1 "k8s.io/api/policy/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" @@ -200,3 +201,19 @@ func readCacheSnapshot(transCtx *ClusterTransformContext, cluster appsv1alpha1.C return snapshot, nil } + +// sendWaringEventForCluster sends a warning event when occurs error. +func sendWaringEventWithError( + recorder record.EventRecorder, + cluster *appsv1alpha1.Cluster, + reason string, + err error) { + if err == nil { + return + } + controllerErr := intctrlutil.ToControllerError(err) + if controllerErr != nil { + reason = string(controllerErr.Type) + } + recorder.Event(cluster, corev1.EventTypeWarning, reason, err.Error()) +} From 89a4bb47db0957558b2b656b75b77a6e39a3735a Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Thu, 27 Apr 2023 10:39:36 +0800 Subject: [PATCH 196/439] chore: tidy up cluster controller ut setup (#2983) Co-authored-by: Ziang Guo Co-authored-by: free6om --- controllers/apps/cluster_controller_test.go | 818 +++++++++--------- controllers/apps/cluster_status_utils.go | 4 +- controllers/apps/cluster_status_utils_test.go | 6 +- .../apps/clusterdefinition_controller_test.go | 4 +- .../apps/clusterversion_controller_test.go | 2 +- .../replication/replication_switch_test.go | 2 +- .../replication_switch_utils_test.go | 2 +- .../replication/replication_test.go | 2 +- .../replication/replication_utils_test.go | 2 +- .../apps/configuration/config_util_test.go | 2 +- .../configconstraint_controller_test.go | 2 +- .../reconfigurerequest_controller_test.go | 2 +- controllers/apps/operations/upgrade_test.go | 6 +- .../apps/opsrequest_controller_test.go | 8 +- .../apps/systemaccount_controller_test.go | 4 +- controllers/apps/tls_utils_test.go | 2 +- internal/cli/cmd/cluster/config_ops_test.go | 2 +- internal/controller/builder/builder_test.go | 4 +- .../component/affinity_utils_test.go | 4 +- .../controller/component/component_test.go | 4 +- ...transformer_sts_horizontal_scaling_test.go | 2 +- internal/controller/plan/pitr_test.go | 4 +- internal/controller/plan/prepare.go | 4 + internal/controller/plan/prepare_test.go | 22 +- .../apps/cluster_consensus_test_util.go | 2 +- .../apps/cluster_replication_test_util.go | 6 +- internal/testutil/apps/cluster_util.go | 6 +- .../testutil/apps/clusterversion_factory.go | 2 +- test/integration/backup_mysql_test.go | 2 +- test/integration/controller_suite_test.go | 2 +- test/integration/mysql_ha_test.go | 2 +- test/integration/mysql_scale_test.go | 4 +- test/integration/redis_hscale_test.go | 2 +- 33 files changed, 491 insertions(+), 451 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 2cfe401a7..0f4443324 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -20,8 +20,8 @@ along with this program. If not, see . package apps import ( + "errors" "fmt" - "reflect" "strconv" "strings" "time" @@ -59,9 +59,13 @@ const backupPolicyTPLName = "test-backup-policy-template-mysql" var _ = Describe("Cluster Controller", func() { const ( - clusterDefName = "test-clusterdef" - clusterVersionName = "test-clusterversion" - clusterNamePrefix = "test-cluster" + clusterDefName = "test-clusterdef" + clusterVersionName = "test-clusterversion" + clusterNamePrefix = "test-cluster" + leader = "leader" + follower = "follower" + // REVIEW: + // - setup componentName and componentDefName as map entry pair statelessCompName = "stateless" statelessCompDefName = "stateless" statefulCompName = "stateful" @@ -70,17 +74,37 @@ var _ = Describe("Cluster Controller", func() { consensusCompDefName = "consensus" replicationCompName = "replication" replicationCompDefName = "replication" - leader = "leader" - follower = "follower" ) var ( - randomStr = testCtx.GetRandomStr() - clusterNameRand = "mysql-" + randomStr - clusterDefNameRand = "mysql-definition-" + randomStr - clusterVersionNameRand = "mysql-cluster-version-" + randomStr + clusterNameRand string + clusterDefNameRand string + clusterVersionNameRand string + clusterDefObj *appsv1alpha1.ClusterDefinition + clusterVersionObj *appsv1alpha1.ClusterVersion + clusterObj *appsv1alpha1.Cluster + clusterKey types.NamespacedName + allSettings map[string]interface{} ) + resetViperCfg := func() { + if allSettings != nil { + Expect(viper.MergeConfigMap(allSettings)).ShouldNot(HaveOccurred()) + allSettings = nil + } + } + + resetTestContext := func() { + clusterDefObj = nil + clusterVersionObj = nil + clusterObj = nil + randomStr := testCtx.GetRandomStr() + clusterNameRand = "mysql-" + randomStr + clusterDefNameRand = "mysql-definition-" + randomStr + clusterVersionNameRand = "mysql-cluster-version-" + randomStr + resetViperCfg() + } + // Cleanups cleanEnv := func() { // must wait until resources deleted and no longer exist before the testcases start, @@ -105,24 +129,39 @@ var _ = Describe("Cluster Controller", func() { testapps.ClearResources(&testCtx, generics.BackupPolicyTemplateSignature, ml) testapps.ClearResources(&testCtx, generics.BackupToolSignature, ml) testapps.ClearResources(&testCtx, generics.StorageClassSignature, ml) + resetTestContext() } BeforeEach(func() { cleanEnv() + allSettings = viper.AllSettings() }) AfterEach(func() { cleanEnv() }) - var ( - clusterDefObj *appsv1alpha1.ClusterDefinition - clusterVersionObj *appsv1alpha1.ClusterVersion - clusterObj *appsv1alpha1.Cluster - clusterKey types.NamespacedName - ) + // test function helpers + createAllWorkloadTypesClusterDef := func(noCreateAssociateCV ...bool) { + By("Create a clusterDefinition obj") + clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). + AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). + AddComponentDef(testapps.ConsensusMySQLComponent, consensusCompDefName). + AddComponentDef(testapps.ReplicationRedisComponent, replicationCompDefName). + AddComponentDef(testapps.StatelessNginxComponent, statelessCompDefName). + Create(&testCtx).GetObject() - // Test cases + if len(noCreateAssociateCV) > 0 && !noCreateAssociateCV[0] { + return + } + By("Create a clusterVersion obj") + clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). + AddComponentVersion(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(consensusCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(replicationCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(statelessCompDefName).AddContainerShort("nginx", testapps.NginxImage). + Create(&testCtx).GetObject() + } waitForCreatingResourceCompletely := func(clusterKey client.ObjectKey, compNames ...string) { Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) @@ -131,93 +170,14 @@ var _ = Describe("Cluster Controller", func() { } } - checkAllResourcesCreated := func() { - By("Creating a cluster") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(replicationCompName, replicationCompDefName).SetReplicas(3). - AddComponent(statelessCompName, statelessCompDefName).SetReplicas(3). - WithRandomName().Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - - By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) - - By("Check deployment workload has been created") - Eventually(testapps.GetListLen(&testCtx, generics.DeploymentSignature, - client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - }, client.InNamespace(clusterKey.Namespace))).ShouldNot(BeEquivalentTo(0)) - - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - - By("Check statefulset pod's volumes") - for _, sts := range stsList.Items { - podSpec := sts.Spec.Template - volumeNames := map[string]struct{}{} - for _, v := range podSpec.Spec.Volumes { - volumeNames[v.Name] = struct{}{} - } - - for _, cc := range [][]corev1.Container{ - podSpec.Spec.Containers, - podSpec.Spec.InitContainers, - } { - for _, c := range cc { - for _, vm := range c.VolumeMounts { - _, ok := volumeNames[vm.Name] - Expect(ok).Should(BeTrue()) - } - } - } - } - - By("Check associated PDB has been created") - Eventually(testapps.GetListLen(&testCtx, generics.PodDisruptionBudgetSignature, - client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - }, client.InNamespace(clusterKey.Namespace))).Should(Equal(0)) - - podSpec := stsList.Items[0].Spec.Template.Spec - By("Checking created sts pods template with built-in toleration") - Expect(podSpec.Tolerations).Should(HaveLen(1)) - Expect(podSpec.Tolerations[0].Key).To(Equal(constant.KubeBlocksDataNodeTolerationKey)) - - By("Checking created sts pods template with built-in Affinity") - Expect(podSpec.Affinity.PodAntiAffinity == nil && podSpec.Affinity.PodAffinity == nil).Should(BeTrue()) - Expect(podSpec.Affinity.NodeAffinity).ShouldNot(BeNil()) - Expect(podSpec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Key).To( - Equal(constant.KubeBlocksDataNodeLabelKey)) - - By("Checking created sts pods template without TopologySpreadConstraints") - Expect(podSpec.TopologySpreadConstraints).Should(BeEmpty()) - - By("Check should create env configmap") - Eventually(testapps.GetListLen(&testCtx, generics.ConfigMapSignature, - client.MatchingLabels{ - constant.AppInstanceLabelKey: clusterKey.Name, - constant.AppConfigTypeLabelKey: "kubeblocks-env", - }, client.InNamespace(clusterKey.Namespace))).Should(Equal(2)) - - By("Make sure the cluster controller has set the cluster status to Running") - for i, comp := range clusterObj.Spec.ComponentSpecs { - if comp.ComponentDefRef != replicationCompDefName || comp.Name != replicationCompName { - continue - } - stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), clusterObj.Spec.ComponentSpecs[i].Name) - for _, v := range stsList.Items { - Expect(testapps.ChangeObjStatus(&testCtx, &v, func() { - testk8s.MockStatefulSetReady(&v) - })).ShouldNot(HaveOccurred()) - } - } - } - type ExpectService struct { headless bool svcType corev1.ServiceType } + // getHeadlessSvcPorts returns the component's headless service ports by gathering all container's ports in the + // ClusterComponentDefinition.PodSpec, it's a subset of the real ports as some containers can be dynamically + // injected into the pod by the lifecycle controller, such as the probe container. getHeadlessSvcPorts := func(g Gomega, compDefName string) []corev1.ServicePort { comp, err := util.GetComponentDefByCluster(testCtx.Ctx, k8sClient, *clusterObj, compDefName) g.Expect(err).ShouldNot(HaveOccurred()) @@ -236,7 +196,7 @@ var _ = Describe("Cluster Controller", func() { return headlessSvcPorts } - validateCompSvcList := func(g Gomega, compName string, compType string, expectServices map[string]ExpectService) { + validateCompSvcList := func(g Gomega, compName string, compDefName string, expectServices map[string]ExpectService) { clusterKey = client.ObjectKeyFromObject(clusterObj) svcList := &corev1.ServiceList{} @@ -247,7 +207,11 @@ var _ = Describe("Cluster Controller", func() { for svcName, svcSpec := range expectServices { idx := slices.IndexFunc(svcList.Items, func(e corev1.Service) bool { - return strings.HasSuffix(e.Name, svcName) + parts := []string{clusterKey.Name, compName} + if svcName != "" { + parts = append(parts, svcName) + } + return strings.Join(parts, "-") == e.Name }) g.Expect(idx >= 0).To(BeTrue()) svc := svcList.Items[idx] @@ -259,24 +223,26 @@ var _ = Describe("Cluster Controller", func() { g.Expect(svc.Spec.ClusterIP).ShouldNot(Equal(corev1.ClusterIPNone)) case svc.Spec.Type == corev1.ServiceTypeClusterIP && svcSpec.headless: g.Expect(svc.Spec.ClusterIP).Should(Equal(corev1.ClusterIPNone)) - g.Expect(reflect.DeepEqual(svc.Spec.Ports, getHeadlessSvcPorts(g, compType))).Should(BeTrue()) + for _, port := range getHeadlessSvcPorts(g, compDefName) { + g.Expect(slices.Index(svc.Spec.Ports, port) >= 0).Should(BeTrue()) + } } } g.Expect(len(expectServices)).Should(Equal(len(svcList.Items))) } - testServiceAddAndDelete := func() { + testServiceAddAndDelete := func(compName, compDefName string) { By("Creating a cluster with two LoadBalancer services") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(replicationCompName, replicationCompDefName).SetReplicas(1). + AddComponent(compName, compDefName).SetReplicas(1). AddService(testapps.ServiceVPCName, corev1.ServiceTypeLoadBalancer). AddService(testapps.ServiceInternetName, corev1.ServiceTypeLoadBalancer). WithRandomName().Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) expectServices := map[string]ExpectService{ testapps.ServiceHeadlessName: {svcType: corev1.ServiceTypeClusterIP, headless: true}, @@ -284,14 +250,14 @@ var _ = Describe("Cluster Controller", func() { testapps.ServiceVPCName: {svcType: corev1.ServiceTypeLoadBalancer, headless: false}, testapps.ServiceInternetName: {svcType: corev1.ServiceTypeLoadBalancer, headless: false}, } - Eventually(func(g Gomega) { validateCompSvcList(g, replicationCompName, replicationCompDefName, expectServices) }).Should(Succeed()) + Eventually(func(g Gomega) { validateCompSvcList(g, compName, compDefName, expectServices) }).Should(Succeed()) By("Delete a LoadBalancer service") deleteService := testapps.ServiceVPCName delete(expectServices, deleteService) Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { for idx, comp := range cluster.Spec.ComponentSpecs { - if comp.ComponentDefRef != replicationCompDefName || comp.Name != replicationCompName { + if comp.ComponentDefRef != compDefName || comp.Name != compName { continue } var services []appsv1alpha1.ClusterComponentService @@ -305,13 +271,13 @@ var _ = Describe("Cluster Controller", func() { return } })()).ShouldNot(HaveOccurred()) - Eventually(func(g Gomega) { validateCompSvcList(g, replicationCompName, replicationCompDefName, expectServices) }).Should(Succeed()) + Eventually(func(g Gomega) { validateCompSvcList(g, compName, compDefName, expectServices) }).Should(Succeed()) By("Add the deleted LoadBalancer service back") expectServices[deleteService] = ExpectService{svcType: corev1.ServiceTypeLoadBalancer, headless: false} Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { for idx, comp := range cluster.Spec.ComponentSpecs { - if comp.ComponentDefRef != replicationCompDefName || comp.Name != replicationCompName { + if comp.ComponentDefRef != compDefName || comp.Name != compName { continue } comp.Services = append(comp.Services, appsv1alpha1.ClusterComponentService{ @@ -322,57 +288,24 @@ var _ = Describe("Cluster Controller", func() { return } })()).ShouldNot(HaveOccurred()) - Eventually(func(g Gomega) { validateCompSvcList(g, replicationCompName, replicationCompDefName, expectServices) }).Should(Succeed()) + Eventually(func(g Gomega) { validateCompSvcList(g, compName, compDefName, expectServices) }).Should(Succeed()) } - checkAllServicesCreate := func() { + createClusterObj := func(compName, compDefName string) { By("Creating a cluster") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(replicationCompName, replicationCompDefName).SetReplicas(1). - AddComponent(statelessCompName, statelessCompDefName).SetReplicas(3). - WithRandomName().Create(&testCtx).GetObject() + clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). + AddComponent(compName, compDefName). + Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) - By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName, statelessCompName) - - By("Checking proxy services") - nginxExpectServices := map[string]ExpectService{ - // TODO: fix me later, proxy should not have internal headless service - testapps.ServiceHeadlessName: {svcType: corev1.ServiceTypeClusterIP, headless: true}, - testapps.ServiceDefaultName: {svcType: corev1.ServiceTypeClusterIP, headless: false}, - } - Eventually(func(g Gomega) { validateCompSvcList(g, statelessCompName, statelessCompDefName, nginxExpectServices) }).Should(Succeed()) - - By("Checking mysql services") - mysqlExpectServices := map[string]ExpectService{ - testapps.ServiceHeadlessName: {svcType: corev1.ServiceTypeClusterIP, headless: true}, - testapps.ServiceDefaultName: {svcType: corev1.ServiceTypeClusterIP, headless: false}, - } - Eventually(func(g Gomega) { - validateCompSvcList(g, replicationCompName, replicationCompDefName, mysqlExpectServices) - }).Should(Succeed()) - - By("Make sure the cluster controller has set the cluster status to Running") - for i, comp := range clusterObj.Spec.ComponentSpecs { - if comp.ComponentDefRef != replicationCompDefName || comp.Name != replicationCompName { - continue - } - stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), clusterObj.Spec.ComponentSpecs[i].Name) - for _, v := range stsList.Items { - Expect(testapps.ChangeObjStatus(&testCtx, &v, func() { - testk8s.MockStatefulSetReady(&v) - })).ShouldNot(HaveOccurred()) - } - } + By("Waiting for the cluster enter running phase") + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) + Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.CreatingClusterPhase)) } - testWipeOut := func() { - By("Creating a cluster") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) + testWipeOut := func(compName, compDefName string) { + createClusterObj(compName, compDefName) By("Waiting for the cluster enter running phase") Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) @@ -387,15 +320,8 @@ var _ = Describe("Cluster Controller", func() { Eventually(testapps.CheckObjExists(&testCtx, clusterKey, &appsv1alpha1.Cluster{}, false)).Should(Succeed()) } - testDoNotTermintate := func() { - By("Creating a cluster") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - - By("Waiting for the cluster enter running phase") - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.CreatingClusterPhase)) + testDoNotTermintate := func(compName, compDefName string) { + createClusterObj(compName, compDefName) // REVIEW: this test flow @@ -443,15 +369,8 @@ var _ = Describe("Cluster Controller", func() { })()).ShouldNot(HaveOccurred()) } - testChangeReplicas := func() { - By("Creating a cluster") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - - By("Waiting for the cluster enter running phase") - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.CreatingClusterPhase)) + testChangeReplicas := func(compName, compDefName string) { + createClusterObj(compName, compDefName) replicasSeq := []int32{5, 3, 1, 0, 2, 4} expectedOG := int64(1) @@ -621,10 +540,6 @@ var _ = Describe("Cluster Controller", func() { Expect(testCtx.Cli.Get(testCtx.Ctx, clusterKey, cluster)).Should(Succeed()) initialGeneration := int(cluster.Status.ObservedGeneration) - // REVIEW/TODO: (chantu) - // ought to have HorizontalScalePolicy setup during ClusterDefinition object creation, - // following implementation is rather hack-ish. - By("Set HorizontalScalePolicy") Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(clusterDefObj), func(clusterDef *appsv1alpha1.ClusterDefinition) { @@ -647,10 +562,8 @@ var _ = Describe("Cluster Controller", func() { Eventually(testapps.CheckObjExists(&testCtx, client.ObjectKey{Name: policyName, Namespace: clusterKey.Namespace}, &dataprotectionv1alpha1.BackupPolicy{}, true)).Should(Succeed()) - } })()).ShouldNot(HaveOccurred()) - // By("Mocking all components' PVCs to bound") for _, comp := range clusterObj.Spec.ComponentSpecs { @@ -662,8 +575,6 @@ var _ = Describe("Cluster Controller", func() { createPVC(clusterKey.Name, pvcKey.Name, comp.Name) Eventually(testapps.CheckObjExists(&testCtx, pvcKey, &corev1.PersistentVolumeClaim{}, true)).Should(Succeed()) - // REVIEW/TODO: (chantu) - // why using Eventually for change object status? Eventually(testapps.GetAndChangeObjStatus(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound })).Should(Succeed()) @@ -685,18 +596,9 @@ var _ = Describe("Cluster Controller", func() { By("Checking cluster status and the number of replicas changed") Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)). Should(BeEquivalentTo(initialGeneration + len(clusterObj.Spec.ComponentSpecs))) - for i := range clusterObj.Spec.ComponentSpecs { - stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), - clusterObj.Spec.ComponentSpecs[i].Name) - for _, v := range stsList.Items { - Expect(testapps.ChangeObjStatus(&testCtx, &v, func() { - testk8s.MockStatefulSetReady(&v) - })).ShouldNot(HaveOccurred()) - } - } } - testHorizontalScale := func() { + testHorizontalScale := func(compName, compDefName string) { initialReplicas := int32(1) updatedReplicas := int32(3) @@ -704,50 +606,20 @@ var _ = Describe("Cluster Controller", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(replicationCompName, replicationCompDefName). + AddComponent(compName, compDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). SetReplicas(initialReplicas). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) // REVIEW: this test flow, wait for running phase? - horizontalScale(int(updatedReplicas)) - } - - testMultiCompHScale := func() { - initialReplicas := int32(1) - updatedReplicas := int32(3) - - By("Creating a multi components cluster with VolumeClaimTemplate") - pvcSpec := testapps.NewPVCSpec("1Gi") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(statefulCompName, statefulCompDefName). - AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - SetReplicas(initialReplicas). - AddComponent(consensusCompName, consensusCompDefName). - AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - SetReplicas(initialReplicas). - AddComponent(replicationCompName, replicationCompDefName). - AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). - SetReplicas(initialReplicas). - Create(&testCtx).GetObject() - clusterKey = client.ObjectKeyFromObject(clusterObj) - - By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, statefulCompName, consensusCompName, replicationCompName) - - // statefulCompDefName not in componentDefsWithHScalePolicy, for nil backup policy test - // REVIEW: (chantu) - // 1. this test flow, wait for running phase? - // 2. following horizontalScale only work with statefulCompDefName? - horizontalScale(int(updatedReplicas), consensusCompDefName, replicationCompDefName) + horizontalScale(int(updatedReplicas), compDefName) } - testVerticalScale := func() { + testStorageExpansion := func(compName, compDefName string) { const storageClassName = "sc-mock" const replicas = 3 @@ -769,14 +641,16 @@ var _ = Describe("Cluster Controller", func() { By("Create cluster and waiting for the cluster initialized") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(replicationCompName, replicationCompDefName). + AddComponent(compName, compDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). SetReplicas(replicas). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) + + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) By("Checking the replicas") stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) @@ -787,7 +661,7 @@ var _ = Describe("Cluster Controller", func() { for i := 0; i < replicas; i++ { pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: getPVCName(replicationCompName, i), + Name: getPVCName(compName, i), Namespace: clusterKey.Namespace, Labels: map[string]string{ constant.AppInstanceLabelKey: clusterKey.Name, @@ -799,6 +673,22 @@ var _ = Describe("Cluster Controller", func() { Expect(k8sClient.Status().Update(testCtx.Ctx, pvc)).Should(Succeed()) } + By("mock pods/sts of component are available") + switch compDefName { + case statelessCompDefName: + // ignore + case replicationCompDefName: + testapps.MockReplicationComponentPods(nil, testCtx, sts, clusterObj.Name, compDefName, nil) + case statefulCompDefName, consensusCompDefName: + testapps.MockConsensusComponentPods(testCtx, sts, clusterObj.Name, compName) + } + Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { + testk8s.MockStatefulSetReady(sts) + })).ShouldNot(HaveOccurred()) + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) + By("Updating the PVC storage size") newStorageValue := resource.MustParse("2Gi") Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { @@ -806,15 +696,9 @@ var _ = Describe("Cluster Controller", func() { comp.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = newStorageValue })()).ShouldNot(HaveOccurred()) - By("mock pods/sts of component are available") - testapps.MockConsensusComponentPods(testCtx, sts, clusterObj.Name, replicationCompName) - Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { - testk8s.MockStatefulSetReady(sts) - })).ShouldNot(HaveOccurred()) - By("Checking the resize operation finished") Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, replicationCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) By("Checking PVCs are resized") @@ -824,14 +708,14 @@ var _ = Describe("Cluster Controller", func() { pvc := &corev1.PersistentVolumeClaim{} pvcKey := types.NamespacedName{ Namespace: clusterKey.Namespace, - Name: getPVCName(replicationCompName, int(i)), + Name: getPVCName(compName, int(i)), } Expect(k8sClient.Get(testCtx.Ctx, pvcKey, pvc)).Should(Succeed()) Expect(pvc.Spec.Resources.Requests[corev1.ResourceStorage]).To(Equal(newStorageValue)) } } - testClusterAffinity := func() { + testClusterAffinity := func(compName, compDefName string) { const topologyKey = "testTopologyKey" const labelKey = "testNodeLabelKey" const labelValue = "testLabelValue" @@ -848,13 +732,13 @@ var _ = Describe("Cluster Controller", func() { clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(replicationCompName, replicationCompDefName).SetReplicas(3). + AddComponent(compName, compDefName).SetReplicas(3). WithRandomName().SetClusterAffinity(affinity). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the Affinity and TopologySpreadConstraints") Eventually(func(g Gomega) { @@ -869,7 +753,7 @@ var _ = Describe("Cluster Controller", func() { } - testComponentAffinity := func() { + testComponentAffinity := func(compName, compDefName string) { const clusterTopologyKey = "testClusterTopologyKey" const compTopologyKey = "testComponentTopologyKey" @@ -886,12 +770,12 @@ var _ = Describe("Cluster Controller", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().SetClusterAffinity(affinity). - AddComponent(replicationCompName, replicationCompDefName).SetComponentAffinity(compAffinity). + AddComponent(compName, compDefName).SetComponentAffinity(compAffinity). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the Affinity and the TopologySpreadConstraints") Eventually(func(g Gomega) { @@ -905,7 +789,7 @@ var _ = Describe("Cluster Controller", func() { }).Should(Succeed()) } - testClusterToleration := func() { + testClusterToleration := func(compName, compDefName string) { const tolerationKey = "testClusterTolerationKey" const tolerationValue = "testClusterTolerationValue" By("Creating a cluster with Toleration") @@ -917,13 +801,13 @@ var _ = Describe("Cluster Controller", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(replicationCompName, replicationCompDefName).SetReplicas(1). + AddComponent(compName, compDefName).SetReplicas(1). AddClusterToleration(toleration). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the tolerations") Eventually(func(g Gomega) { @@ -938,7 +822,7 @@ var _ = Describe("Cluster Controller", func() { }).Should(Succeed()) } - testComponentToleration := func() { + testComponentToleration := func(compName, compDefName string) { clusterTolerationKey := "testClusterTolerationKey" compTolerationKey := "testcompTolerationKey" compTolerationValue := "testcompTolerationValue" @@ -957,12 +841,12 @@ var _ = Describe("Cluster Controller", func() { } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().AddClusterToleration(toleration). - AddComponent(replicationCompName, replicationCompDefName).AddComponentToleration(compToleration). + AddComponent(compName, compDefName).AddComponentToleration(compToleration). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the tolerations") Eventually(func(g Gomega) { @@ -1014,20 +898,20 @@ var _ = Describe("Cluster Controller", func() { return names } - testThreeReplicas := func() { + testThreeReplicas := func(compName, compDefName string) { const replicas = 3 By("Mock a cluster obj") pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(replicationCompName, replicationCompDefName). + AddComponent(compName, compDefName). SetReplicas(replicas).AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) var stsList *appsv1.StatefulSetList var sts *appsv1.StatefulSet @@ -1104,26 +988,45 @@ var _ = Describe("Cluster Controller", func() { }).Should(Succeed()) By("Waiting the component be running") - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, replicationCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, compName)). + Should(Equal(appsv1alpha1.RunningClusterCompPhase)) } - testBackupError := func() { + testBackupError := func(compName, compDefName string) { initialReplicas := int32(1) updatedReplicas := int32(3) viper.Set("VOLUMESNAPSHOT", true) + By("Set HorizontalScalePolicy") + Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(clusterDefObj), + func(clusterDef *appsv1alpha1.ClusterDefinition) { + for i, def := range clusterDef.Spec.ComponentDefs { + if def.Name != compDefName { + continue + } + clusterDef.Spec.ComponentDefs[i].HorizontalScalePolicy = + &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, + BackupPolicyTemplateName: backupPolicyTPLName} + } + })()).ShouldNot(HaveOccurred()) + By("Creating a cluster with VolumeClaimTemplate") pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(replicationCompName, replicationCompDefName). + AddComponent(compName, compDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). SetReplicas(initialReplicas). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, compName) + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) + + By(fmt.Sprintf("Changing replicas to %d", updatedReplicas)) + changeCompReplicas(clusterKey, updatedReplicas, &clusterObj.Spec.ComponentSpecs[0]) + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) // REVIEW: this test flow, should wait/fake still Running phase? By("Creating backup") @@ -1137,12 +1040,12 @@ var _ = Describe("Cluster Controller", func() { Namespace: backupKey.Namespace, Labels: map[string]string{ constant.AppInstanceLabelKey: clusterKey.Name, - constant.KBAppComponentLabelKey: replicationCompName, + constant.KBAppComponentLabelKey: compName, constant.KBManagedByKey: "cluster", }, }, Spec: dataprotectionv1alpha1.BackupSpec{ - BackupPolicyName: lifecycle.DeriveBackupPolicyName(clusterKey.Name, replicationCompDefName), + BackupPolicyName: lifecycle.DeriveBackupPolicyName(clusterKey.Name, compDefName), BackupType: "snapshot", }, } @@ -1163,17 +1066,25 @@ var _ = Describe("Cluster Controller", func() { By(fmt.Sprintf("Changing replicas to %d", updatedReplicas)) changeCompReplicas(clusterKey, updatedReplicas, &clusterObj.Spec.ComponentSpecs[0]) + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) By("Checking cluster status failed with backup error") Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { - hasBackupError := false + g.Expect(viper.GetBool("VOLUMESNAPSHOT")).Should(BeTrue()) + g.Expect(cluster.Status.Conditions).ShouldNot(BeEmpty()) + var err error for _, cond := range cluster.Status.Conditions { if strings.Contains(cond.Message, "backup for horizontalScaling failed") { - hasBackupError = true + g.Expect(cond.Message).Should(ContainSubstring("backup for horizontalScaling failed")) + err = errors.New("has backup error") break } } - g.Expect(hasBackupError).Should(BeTrue()) + if err == nil { + // this expect is intended for print all cluster.Status.Conditions + g.Expect(cluster.Status.Conditions).Should(BeEmpty()) + } + g.Expect(err).Should(HaveOccurred()) })).Should(Succeed()) By("expect for backup error event") @@ -1199,49 +1110,187 @@ var _ = Describe("Cluster Controller", func() { })).ShouldNot(HaveOccurred()) } + // Test cases // Scenarios - Context("when creating cluster without clusterversion", func() { BeforeEach(func() { - By("Create a clusterDefinition obj") - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.StatefulMySQLComponent, replicationCompDefName). - Create(&testCtx).GetObject() + createAllWorkloadTypesClusterDef(true) }) It("should reconcile to create cluster with no error", func() { By("Creating a cluster") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, ""). + AddComponent(statelessCompName, statelessCompDefName).SetReplicas(3). + AddComponent(statefulCompName, statefulCompDefName).SetReplicas(3). + AddComponent(consensusCompName, consensusCompDefName).SetReplicas(3). AddComponent(replicationCompName, replicationCompDefName).SetReplicas(3). WithRandomName().Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) + waitForCreatingResourceCompletely(clusterKey, statelessCompName, statefulCompName, consensusCompName, replicationCompName) }) }) Context("when creating cluster with multiple kinds of components", func() { BeforeEach(func() { - By("Create a clusterDefinition obj") - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.StatefulMySQLComponent, statefulCompDefName). - AddComponentDef(testapps.ConsensusMySQLComponent, consensusCompDefName). - AddComponentDef(testapps.StatefulMySQLComponent, replicationCompDefName). - AddComponentDef(testapps.StatelessNginxComponent, statelessCompDefName). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(replicationCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - AddComponent(statelessCompDefName).AddContainerShort("nginx", testapps.NginxImage). - Create(&testCtx).GetObject() - - By("Creating a BackupPolicyTemplate") + createAllWorkloadTypesClusterDef() createBackupPolicyTpl(clusterDefObj) }) + createNWaitClusterObj := func(components map[string]string, + addedComponentProcessor func(compName string, factory *testapps.MockClusterFactory)) { + Expect(components).ShouldNot(BeEmpty()) + + By("Creating a cluster") + clusterBuilder := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterDefObj.Name, clusterVersionObj.Name) + + compNames := make([]string, 0, len(components)) + for compName, compDefName := range components { + clusterBuilder = clusterBuilder.AddComponent(compName, compDefName) + if addedComponentProcessor != nil { + addedComponentProcessor(compName, clusterBuilder) + } + compNames = append(compNames, compName) + } + + clusterObj = clusterBuilder.WithRandomName().Create(&testCtx).GetObject() + clusterKey = client.ObjectKeyFromObject(clusterObj) + + By("Waiting for the cluster controller to create resources completely") + waitForCreatingResourceCompletely(clusterKey, compNames...) + } + + checkAllResourcesCreated := func() { + compNameNDef := map[string]string{ + statelessCompName: statelessCompDefName, + consensusCompName: consensusCompDefName, + } + createNWaitClusterObj(compNameNDef, func(compName string, factory *testapps.MockClusterFactory) { + factory.SetReplicas(3) + }) + + By("Check deployment workload has been created") + Eventually(testapps.GetListLen(&testCtx, generics.DeploymentSignature, + client.MatchingLabels{ + constant.AppInstanceLabelKey: clusterKey.Name, + }, client.InNamespace(clusterKey.Namespace))).ShouldNot(BeEquivalentTo(0)) + + stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) + + By("Check statefulset pod's volumes") + for _, sts := range stsList.Items { + podSpec := sts.Spec.Template + volumeNames := map[string]struct{}{} + for _, v := range podSpec.Spec.Volumes { + volumeNames[v.Name] = struct{}{} + } + + for _, cc := range [][]corev1.Container{ + podSpec.Spec.Containers, + podSpec.Spec.InitContainers, + } { + for _, c := range cc { + for _, vm := range c.VolumeMounts { + _, ok := volumeNames[vm.Name] + Expect(ok).Should(BeTrue()) + } + } + } + } + + By("Check associated PDB has been created") + Eventually(testapps.GetListLen(&testCtx, generics.PodDisruptionBudgetSignature, + client.MatchingLabels{ + constant.AppInstanceLabelKey: clusterKey.Name, + }, client.InNamespace(clusterKey.Namespace))).Should(Equal(0)) + + podSpec := stsList.Items[0].Spec.Template.Spec + By("Checking created sts pods template with built-in toleration") + Expect(podSpec.Tolerations).Should(HaveLen(1)) + Expect(podSpec.Tolerations[0].Key).To(Equal(constant.KubeBlocksDataNodeTolerationKey)) + + By("Checking created sts pods template with built-in Affinity") + Expect(podSpec.Affinity.PodAntiAffinity == nil && podSpec.Affinity.PodAffinity == nil).Should(BeTrue()) + Expect(podSpec.Affinity.NodeAffinity).ShouldNot(BeNil()) + Expect(podSpec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Key).To( + Equal(constant.KubeBlocksDataNodeLabelKey)) + + By("Checking created sts pods template without TopologySpreadConstraints") + Expect(podSpec.TopologySpreadConstraints).Should(BeEmpty()) + + By("Check should create env configmap") + Eventually(testapps.GetListLen(&testCtx, generics.ConfigMapSignature, + client.MatchingLabels{ + constant.AppInstanceLabelKey: clusterKey.Name, + constant.AppConfigTypeLabelKey: "kubeblocks-env", + }, client.InNamespace(clusterKey.Namespace))).Should(Equal(2)) + } + + checkAllServicesCreate := func() { + compNameNDef := map[string]string{ + statelessCompName: statelessCompDefName, + consensusCompName: consensusCompDefName, + statefulCompName: statefulCompDefName, + replicationCompName: replicationCompDefName, + } + + createNWaitClusterObj(compNameNDef, func(compName string, factory *testapps.MockClusterFactory) { + factory.SetReplicas(3) + }) + + By("Checking stateless services") + statelessExpectServices := map[string]ExpectService{ + // TODO: fix me later, proxy should not have internal headless service + testapps.ServiceHeadlessName: {svcType: corev1.ServiceTypeClusterIP, headless: true}, + testapps.ServiceDefaultName: {svcType: corev1.ServiceTypeClusterIP, headless: false}, + } + Eventually(func(g Gomega) { + validateCompSvcList(g, statelessCompName, statelessCompDefName, statelessExpectServices) + }).Should(Succeed()) + + By("Checking stateful types services") + for compName, compNameNDef := range compNameNDef { + if compName == statelessCompName { + continue + } + consensusExpectServices := map[string]ExpectService{ + testapps.ServiceHeadlessName: {svcType: corev1.ServiceTypeClusterIP, headless: true}, + testapps.ServiceDefaultName: {svcType: corev1.ServiceTypeClusterIP, headless: false}, + } + Eventually(func(g Gomega) { + validateCompSvcList(g, compName, compNameNDef, consensusExpectServices) + }).Should(Succeed()) + } + } + + testMultiCompHScale := func() { + compNameNDef := map[string]string{ + statefulCompName: statefulCompDefName, + consensusCompName: consensusCompDefName, + replicationCompName: replicationCompDefName, + } + initialReplicas := int32(1) + updatedReplicas := int32(3) + + By("Creating a multi components cluster with VolumeClaimTemplate") + pvcSpec := testapps.NewPVCSpec("1Gi") + + createNWaitClusterObj(compNameNDef, func(compName string, factory *testapps.MockClusterFactory) { + factory.AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec).SetReplicas(initialReplicas) + }) + + By("Waiting for the cluster controller to create resources completely") + waitForCreatingResourceCompletely(clusterKey, statefulCompName, consensusCompName, replicationCompName) + + // statefulCompDefName not in componentDefsWithHScalePolicy, for nil backup policy test + // REVIEW: + // 1. this test flow, wait for running phase? + horizontalScale(int(updatedReplicas), consensusCompDefName, replicationCompDefName) + } + It("should create all sub-resources successfully", func() { checkAllResourcesCreated() }) @@ -1250,133 +1299,119 @@ var _ = Describe("Cluster Controller", func() { checkAllServicesCreate() }) - It("should add and delete service correctly", func() { - testServiceAddAndDelete() - }) - It("should successfully h-scale with multiple components", func() { testMultiCompHScale() }) }) - When("creating cluster with workloadType=stateful component", func() { - BeforeEach(func() { - By("Create a clusterDefinition obj") - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.StatefulMySQLComponent, replicationCompDefName). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(replicationCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - Create(&testCtx).GetObject() + When("creating cluster with workloadType=[Stateless|Stateful|Consensus|Replication] component", func() { + compNameNDef := map[string]string{ + statelessCompName: statelessCompDefName, + statefulCompName: statefulCompDefName, + consensusCompName: consensusCompDefName, + replicationCompName: replicationCompDefName, + } - By("Creating a BackupPolicyTemplate") + BeforeEach(func() { + createAllWorkloadTypesClusterDef() createBackupPolicyTpl(clusterDefObj) }) - It("should delete cluster resources immediately if deleting cluster with WipeOut termination policy", func() { - testWipeOut() - }) - - It("should not terminate immediately if deleting cluster with DoNotTerminate termination policy", func() { - testDoNotTermintate() - }) - - It("should create/delete pods to match the desired replica number if updating cluster's replica number to a valid value", func() { - testChangeReplicas() - }) + for compName, compDefName := range compNameNDef { + It(fmt.Sprintf("[comp: %s] should delete cluster resources immediately if deleting cluster with WipeOut termination policy", compName), func() { + testWipeOut(compName, compDefName) + }) - Context("and with cluster affinity set", func() { - It("should create pod with cluster affinity", func() { - testClusterAffinity() + It(fmt.Sprintf("[comp: %s] should not terminate immediately if deleting cluster with DoNotTerminate termination policy", compName), func() { + testDoNotTermintate(compName, compDefName) }) - }) - Context("and with both cluster affinity and component affinity set", func() { - It("Should observe the component affinity will override the cluster affinity", func() { - testComponentAffinity() + It(fmt.Sprintf("[comp: %s] should create/delete pods to match the desired replica number if updating cluster's replica number to a valid value", compName), func() { + testChangeReplicas(compName, compDefName) }) - }) - Context("and with cluster tolerations set", func() { - It("Should create pods with cluster tolerations", func() { - testClusterToleration() + It(fmt.Sprintf("[comp: %s] should add and delete service correctly", compName), func() { + testServiceAddAndDelete(compName, compDefName) }) - }) - Context("and with both cluster tolerations and component tolerations set", func() { - It("Should observe the component tolerations will override the cluster tolerations", func() { - testComponentToleration() + Context(fmt.Sprintf("[comp: %s] and with cluster affinity set", compName), func() { + It("should create pod with cluster affinity", func() { + testClusterAffinity(compName, compDefName) + }) }) - }) - Context("with pvc", func() { - It("should trigger a backup process(snapshot) and create pvcs from backup for newly created replicas when horizontal scale the cluster from 1 to 3", func() { - testHorizontalScale() + Context(fmt.Sprintf("[comp: %s] and with both cluster affinity and component affinity set", compName), func() { + It("Should observe the component affinity will override the cluster affinity", func() { + testComponentAffinity(compName, compDefName) + }) }) - }) - Context("with pvc and dynamic-provisioning storage class", func() { - It("should update PVC request storage size accordingly when vertical scale the cluster", func() { - testVerticalScale() + Context(fmt.Sprintf("[comp: %s] and with cluster tolerations set", compName), func() { + It("Should create pods with cluster tolerations", func() { + testClusterToleration(compName, compDefName) + }) }) - }) - }) - When("creating cluster with workloadType=consensus component", func() { - BeforeEach(func() { - By("Create a clusterDef obj") - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.ConsensusMySQLComponent, replicationCompDefName). - Create(&testCtx).GetObject() + Context(fmt.Sprintf("[comp: %s] and with both cluster tolerations and component tolerations set", compName), func() { + It("Should observe the component tolerations will override the cluster tolerations", func() { + testComponentToleration(compName, compDefName) + }) + }) - By("Create a clusterVersion obj") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(replicationCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - Create(&testCtx).GetObject() + Context(fmt.Sprintf("[comp: %s] with pvc", compName), func() { + It("should trigger a backup process(snapshot) and create pvcs from backup for newly created replicas when horizontal scale the cluster from 1 to 3", func() { + testHorizontalScale(compName, compDefName) + }) + }) - By("Creating a BackupPolicyTemplate") - createBackupPolicyTpl(clusterDefObj) - }) + // HACK/TODO: only Stateful and Consensus workload types passes following test, need to investigate. + // Would expect that non-stateless workload types should all pass tests. + switch compName { + case statefulCompName, consensusCompName, replicationCompName: + Context(fmt.Sprintf("[comp: %s] with pvc and dynamic-provisioning storage class", compName), func() { + It(fmt.Sprintf("[comp: %s] should update PVC request storage size accordingly", compName), func() { + testStorageExpansion(compName, compDefName) + }) + }) - It("Should success with one leader pod and two follower pods", func() { - testThreeReplicas() - }) + It(fmt.Sprintf("[comp: %s] should report error if backup error during horizontal scale", compName), func() { + testBackupError(compName, compDefName) + }) - It("should create/delete pods to match the desired replica number if updating cluster's replica number to a valid value", func() { - testChangeReplicas() - }) + Context(fmt.Sprintf("[comp: %s] with horizontal scale after storage expansion", compName), func() { + It("should succeed with horizontal scale to 5 replicas", func() { + testStorageExpansion(compName, compDefName) + horizontalScale(5, compDefName) + }) + }) + default: + } - Context("with pvc", func() { - It("should trigger a backup process(snapshot) and create pvcs from backup for newly created replicas when horizontal scale the cluster from 1 to 3", func() { - testHorizontalScale() - }) - }) + } + }) - Context("with pvc and dynamic-provisioning storage class", func() { - It("should update PVC request storage size accordingly when vertical scale the cluster", func() { - testVerticalScale() - }) - }) + When("creating cluster with workloadType=consensus component", func() { + const ( + compName = consensusCompName + compDefName = consensusCompDefName + ) - Context("with horizontalScale after verticalScale", func() { - It("should succeed", func() { - testVerticalScale() - horizontalScale(5) - }) + BeforeEach(func() { + createAllWorkloadTypesClusterDef() + createBackupPolicyTpl(clusterDefObj) }) - It("should report error if backup error during h-scale", func() { - testBackupError() + It("Should success with one leader pod and two follower pods", func() { + testThreeReplicas(compName, compDefName) }) It("test restore cluster from backup", func() { - By("mock backup") + By("mock backuptool object") backupPolicyName := "test-backup-policy" backupName := "test-backup" backupTool := testapps.CreateCustomizedObj(&testCtx, "backup/backuptool.yaml", &dataprotectionv1alpha1.BackupTool{}, testapps.RandomizedObjName()) + By("creating backup") backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). @@ -1388,53 +1423,60 @@ var _ = Describe("Cluster Controller", func() { func(g Gomega, tmpBackup *dataprotectionv1alpha1.Backup) { g.Expect(tmpBackup.Status.Phase).Should(Equal(dataprotectionv1alpha1.BackupFailed)) })).Should(Succeed()) + By("mocking backup status completed, we don't need backup reconcile here") Eventually(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(backup), func(backup *dataprotectionv1alpha1.Backup) { backup.Status.BackupToolName = backupTool.Name backup.Status.PersistentVolumeClaimName = "backup-pvc" backup.Status.Phase = dataprotectionv1alpha1.BackupCompleted })).Should(Succeed()) + By("checking backup status completed") Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(backup), func(g Gomega, tmpBackup *dataprotectionv1alpha1.Backup) { g.Expect(tmpBackup.Status.Phase).Should(Equal(dataprotectionv1alpha1.BackupCompleted)) })).Should(Succeed()) + By("creating cluster with backup") - restoreFromBackup := fmt.Sprintf(`{"%s":"%s"}`, replicationCompName, backupName) + restoreFromBackup := fmt.Sprintf(`{"%s":"%s"}`, compName, backupName) clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(replicationCompName, replicationCompDefName). + AddComponent(compName, compDefName). SetReplicas(3). AddAnnotations(constant.RestoreFromBackUpAnnotationKey, restoreFromBackup).Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, replicationCompName) - + waitForCreatingResourceCompletely(clusterKey, compName) stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) sts := stsList.Items[0] Expect(sts.Spec.Template.Spec.InitContainers).Should(HaveLen(1)) By("mock pod/sts are available and wait for component enter running phase") - testapps.MockConsensusComponentPods(testCtx, &sts, clusterObj.Name, replicationCompName) + testapps.MockConsensusComponentPods(testCtx, &sts, clusterObj.Name, compName) Expect(testapps.ChangeObjStatus(&testCtx, &sts, func() { testk8s.MockStatefulSetReady(&sts) })).ShouldNot(HaveOccurred()) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, replicationCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) By("remove init container after all components are Running") Eventually(testapps.GetClusterObservedGeneration(&testCtx, client.ObjectKeyFromObject(clusterObj))).Should(BeEquivalentTo(1)) Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterObj), clusterObj)).Should(Succeed()) + Expect(testapps.ChangeObjStatus(&testCtx, clusterObj, func() { + clusterObj.Status.Components = map[string]appsv1alpha1.ClusterComponentStatus{ + compName: {Phase: appsv1alpha1.RunningClusterCompPhase}, + } + })).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(&sts), func(g Gomega, tmpSts *appsv1.StatefulSet) { g.Expect(tmpSts.Spec.Template.Spec.InitContainers).Should(BeEmpty()) })).Should(Succeed()) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, replicationCompName)).Should(Equal(appsv1alpha1.CreatingClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, compName)).Should(Equal(appsv1alpha1.CreatingClusterCompPhase)) By("clean up annotations after cluster running") Expect(testapps.GetAndChangeObjStatus(&testCtx, clusterKey, func(tmpCluster *appsv1alpha1.Cluster) { - compStatus := tmpCluster.Status.Components[replicationCompName] + compStatus := tmpCluster.Status.Components[compName] compStatus.Phase = appsv1alpha1.RunningClusterCompPhase - tmpCluster.Status.Components[replicationCompName] = compStatus + tmpCluster.Status.Components[compName] = compStatus })()).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { g.Expect(tmpCluster.Status.Phase).Should(Equal(appsv1alpha1.RunningClusterPhase)) @@ -1444,17 +1486,13 @@ var _ = Describe("Cluster Controller", func() { }) When("creating cluster with workloadType=replication component", func() { + const ( + compName = replicationCompName + compDefName = replicationCompDefName + ) BeforeEach(func() { - By("Create a clusterDefinition obj with replication componentDefRef.") - clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). - Create(&testCtx).GetObject() - - By("Create a clusterVersion obj with replication componentDefRef.") - clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(testapps.DefaultRedisCompDefName). - AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). - Create(&testCtx).GetObject() + createAllWorkloadTypesClusterDef() + createBackupPolicyTpl(clusterDefObj) }) // REVIEW/TODO: following test always failed at cluster.phase.observerGeneration=1 @@ -1464,7 +1502,7 @@ var _ = Describe("Cluster Controller", func() { pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). - AddComponent(testapps.DefaultRedisCompName, testapps.DefaultRedisCompDefName). + AddComponent(compName, compDefName). SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). SetReplicas(testapps.DefaultReplicationReplicas). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). @@ -1472,7 +1510,7 @@ var _ = Describe("Cluster Controller", func() { clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") - waitForCreatingResourceCompletely(clusterKey, testapps.DefaultRedisCompName) + waitForCreatingResourceCompletely(clusterKey, compDefName) By("Checking statefulSet number") stsList := testk8s.ListAndCheckStatefulSetCount(&testCtx, clusterKey, 1) @@ -1483,8 +1521,8 @@ var _ = Describe("Cluster Controller", func() { })).ShouldNot(HaveOccurred()) for i := int32(0); i < *sts.Spec.Replicas; i++ { podName := fmt.Sprintf("%s-%d", sts.Name, i) - testapps.MockReplicationComponentStsPod(nil, testCtx, sts, clusterObj.Name, - testapps.DefaultRedisCompName, podName, replication.DefaultRole(i)) + testapps.MockReplicationComponentPod(nil, testCtx, sts, clusterObj.Name, + compDefName, podName, replication.DefaultRole(i)) } Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) }) diff --git a/controllers/apps/cluster_status_utils.go b/controllers/apps/cluster_status_utils.go index 4367a3592..bee51d054 100644 --- a/controllers/apps/cluster_status_utils.go +++ b/controllers/apps/cluster_status_utils.go @@ -184,9 +184,7 @@ func handleClusterPhaseWhenCompsNotReady(cluster *appsv1alpha1.Cluster, // if the component can affect and be Failed, the cluster will be Failed too. func getClusterAvailabilityEffect(componentDef *appsv1alpha1.ClusterComponentDefinition) bool { switch componentDef.WorkloadType { - case appsv1alpha1.Consensus: - return true - case appsv1alpha1.Replication: + case appsv1alpha1.Replication, appsv1alpha1.Consensus: return true default: return componentDef.MaxUnavailable != nil diff --git a/controllers/apps/cluster_status_utils_test.go b/controllers/apps/cluster_status_utils_test.go index b1de0ba9b..8e227336a 100644 --- a/controllers/apps/cluster_status_utils_test.go +++ b/controllers/apps/cluster_status_utils_test.go @@ -83,9 +83,9 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { createClusterVersion := func() { _ = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(statefulMySQLCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - AddComponent(consensusMySQLCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). - AddComponent(statelessCompDefName).AddContainerShort(testapps.DefaultNginxContainerName, testapps.NginxImage). + AddComponentVersion(statefulMySQLCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponentVersion(consensusMySQLCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponentVersion(statelessCompDefName).AddContainerShort(testapps.DefaultNginxContainerName, testapps.NginxImage). Create(&testCtx) } diff --git a/controllers/apps/clusterdefinition_controller_test.go b/controllers/apps/clusterdefinition_controller_test.go index f2d5ba227..f7ff4c310 100644 --- a/controllers/apps/clusterdefinition_controller_test.go +++ b/controllers/apps/clusterdefinition_controller_test.go @@ -88,7 +88,7 @@ var _ = Describe("ClusterDefinition Controller", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) @@ -151,7 +151,7 @@ var _ = Describe("ClusterDefinition Controller", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/clusterversion_controller_test.go b/controllers/apps/clusterversion_controller_test.go index d20c711e9..180b63a6c 100644 --- a/controllers/apps/clusterversion_controller_test.go +++ b/controllers/apps/clusterversion_controller_test.go @@ -57,7 +57,7 @@ var _ = Describe("test clusterVersion controller", func() { It("test clusterVersion controller", func() { By("create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(statefulCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() By("wait for clusterVersion phase is unavailable when clusterDef is not found") diff --git a/controllers/apps/components/replication/replication_switch_test.go b/controllers/apps/components/replication/replication_switch_test.go index 3e2f4ef9e..f73ca1da3 100644 --- a/controllers/apps/components/replication/replication_switch_test.go +++ b/controllers/apps/components/replication/replication_switch_test.go @@ -289,7 +289,7 @@ var _ = Describe("ReplicationSet Switch", func() { By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponentVersion(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/components/replication/replication_switch_utils_test.go b/controllers/apps/components/replication/replication_switch_utils_test.go index aec407797..bf034a210 100644 --- a/controllers/apps/components/replication/replication_switch_utils_test.go +++ b/controllers/apps/components/replication/replication_switch_utils_test.go @@ -207,7 +207,7 @@ var _ = Describe("ReplicationSet Switch Util", func() { By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponentVersion(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/components/replication/replication_test.go b/controllers/apps/components/replication/replication_test.go index d083700a2..f088595b8 100644 --- a/controllers/apps/components/replication/replication_test.go +++ b/controllers/apps/components/replication/replication_test.go @@ -80,7 +80,7 @@ var _ = Describe("Replication Component", func() { By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponentVersion(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() By("Creating a cluster with replication workloadType.") diff --git a/controllers/apps/components/replication/replication_utils_test.go b/controllers/apps/components/replication/replication_utils_test.go index e74143e37..b88b6f5db 100644 --- a/controllers/apps/components/replication/replication_utils_test.go +++ b/controllers/apps/components/replication/replication_utils_test.go @@ -267,7 +267,7 @@ var _ = Describe("ReplicationSet Util", func() { By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). + AddComponentVersion(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/configuration/config_util_test.go b/controllers/apps/configuration/config_util_test.go index 6535b55bc..4dc11aaf6 100644 --- a/controllers/apps/configuration/config_util_test.go +++ b/controllers/apps/configuration/config_util_test.go @@ -104,7 +104,7 @@ var _ = Describe("ConfigWrapper util test", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompDefName). + AddComponentVersion(statefulCompDefName). Create(&testCtx).GetObject() }) diff --git a/controllers/apps/configuration/configconstraint_controller_test.go b/controllers/apps/configuration/configconstraint_controller_test.go index 54d0750ac..74e6be3d8 100644 --- a/controllers/apps/configuration/configconstraint_controller_test.go +++ b/controllers/apps/configuration/configconstraint_controller_test.go @@ -89,7 +89,7 @@ var _ = Describe("ConfigConstraint Controller", func() { By("Create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompDefName). + AddComponentVersion(statefulCompDefName). AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(configSpecName), configmap.Name, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). Create(&testCtx).GetObject() diff --git a/controllers/apps/configuration/reconfigurerequest_controller_test.go b/controllers/apps/configuration/reconfigurerequest_controller_test.go index 09acbe971..cbffab746 100644 --- a/controllers/apps/configuration/reconfigurerequest_controller_test.go +++ b/controllers/apps/configuration/reconfigurerequest_controller_test.go @@ -105,7 +105,7 @@ var _ = Describe("Reconfigure Controller", func() { By("Create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompDefName). + AddComponentVersion(statefulCompDefName). AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(configSpecName), configmap.Name, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). Create(&testCtx).GetObject() diff --git a/controllers/apps/operations/upgrade_test.go b/controllers/apps/operations/upgrade_test.go index 848f2039e..f64a86eeb 100644 --- a/controllers/apps/operations/upgrade_test.go +++ b/controllers/apps/operations/upgrade_test.go @@ -70,9 +70,9 @@ var _ = Describe("Upgrade OpsRequest", func() { By("create Upgrade Ops") newClusterVersionName := "clusterversion-upgrade-" + randomStr _ = testapps.NewClusterVersionFactory(newClusterVersionName, clusterDefinitionName). - AddComponent(statelessComp).AddContainerShort(testapps.DefaultNginxContainerName, "nginx:1.14.2"). - AddComponent(consensusComp).AddContainerShort(testapps.DefaultMySQLContainerName, mysqlImageForUpdate). - AddComponent(statefulComp).AddContainerShort(testapps.DefaultMySQLContainerName, mysqlImageForUpdate). + AddComponentVersion(statelessComp).AddContainerShort(testapps.DefaultNginxContainerName, "nginx:1.14.2"). + AddComponentVersion(consensusComp).AddContainerShort(testapps.DefaultMySQLContainerName, mysqlImageForUpdate). + AddComponentVersion(statefulComp).AddContainerShort(testapps.DefaultMySQLContainerName, mysqlImageForUpdate). Create(&testCtx).GetObject() ops := testapps.NewOpsRequestObj("upgrade-ops-"+randomStr, testCtx.DefaultNamespace, clusterObject.Name, appsv1alpha1.UpgradeType) diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index c8eff4ef6..8ef56835a 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -276,7 +276,7 @@ var _ = Describe("OpsRequest Controller", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) @@ -325,7 +325,7 @@ var _ = Describe("OpsRequest Controller", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) @@ -486,7 +486,7 @@ var _ = Describe("OpsRequest Controller", func() { })).ShouldNot(HaveOccurred()) for i := int32(0); i < *sts.Spec.Replicas; i++ { podName := fmt.Sprintf("%s-%d", sts.Name, i) - pod := testapps.MockReplicationComponentStsPod(nil, testCtx, sts, clusterObj.Name, + pod := testapps.MockReplicationComponentPod(nil, testCtx, sts, clusterObj.Name, testapps.DefaultRedisCompName, podName, replication.DefaultRole(i)) podList = append(podList, pod) } @@ -502,7 +502,7 @@ var _ = Describe("OpsRequest Controller", func() { By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, + AddComponentVersion(testapps.DefaultRedisCompDefName).AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() diff --git a/controllers/apps/systemaccount_controller_test.go b/controllers/apps/systemaccount_controller_test.go index 931eb1430..8fb93236a 100644 --- a/controllers/apps/systemaccount_controller_test.go +++ b/controllers/apps/systemaccount_controller_test.go @@ -200,8 +200,8 @@ var _ = Describe("SystemAccount Controller", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - AddComponent(mysqlCompNameWOSysAcct).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompNameWOSysAcct).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() Expect(clusterDefObj).NotTo(BeNil()) diff --git a/controllers/apps/tls_utils_test.go b/controllers/apps/tls_utils_test.go index bd85778b5..291138b02 100644 --- a/controllers/apps/tls_utils_test.go +++ b/controllers/apps/tls_utils_test.go @@ -101,7 +101,7 @@ var _ = Describe("TLS self-signed cert function", func() { By("Create a clusterVersion obj") testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(statefulCompDefName).AddContainerShort(mysqlContainerName, testapps.ApeCloudMySQLImage). + AddComponentVersion(statefulCompDefName).AddContainerShort(mysqlContainerName, testapps.ApeCloudMySQLImage). CheckedCreate(&testCtx).GetObject() }) diff --git a/internal/cli/cmd/cluster/config_ops_test.go b/internal/cli/cmd/cluster/config_ops_test.go index 4c32d2dfa..efcbb8b1d 100644 --- a/internal/cli/cmd/cluster/config_ops_test.go +++ b/internal/cli/cmd/cluster/config_ops_test.go @@ -91,7 +91,7 @@ var _ = Describe("reconfigure test", func() { GetObject() By("Create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(statefulCompDefName). + AddComponentVersion(statefulCompDefName). GetObject() By("creating a cluster") clusterObj := testapps.NewClusterFactory(ns, clusterName, diff --git a/internal/controller/builder/builder_test.go b/internal/controller/builder/builder_test.go index 4cfd320ff..afc929fe0 100644 --- a/internal/controller/builder/builder_test.go +++ b/internal/controller/builder/builder_test.go @@ -81,9 +81,9 @@ var _ = Describe("builder", func() { allFieldsClusterVersionObj := func(needCreate bool) *appsv1alpha1.ClusterVersion { By("By assure an clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompDefName). + AddComponentVersion(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - AddComponent(proxyCompDefName). + AddComponentVersion(proxyCompDefName). AddInitContainerShort("nginx-init", testapps.NginxImage). AddContainerShort("nginx", testapps.NginxImage). GetObject() diff --git a/internal/controller/component/affinity_utils_test.go b/internal/controller/component/affinity_utils_test.go index 7bb76d388..340b7b55c 100644 --- a/internal/controller/component/affinity_utils_test.go +++ b/internal/controller/component/affinity_utils_test.go @@ -56,7 +56,7 @@ var _ = Describe("affinity utils", func() { GetObject() clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). GetObject() affinity := &appsv1alpha1.Affinity{ @@ -113,7 +113,7 @@ var _ = Describe("affinity utils", func() { GetObject() clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). GetObject() affinity := &appsv1alpha1.Affinity{ diff --git a/internal/controller/component/component_test.go b/internal/controller/component/component_test.go index 8e88a989a..3b9f8a4e6 100644 --- a/internal/controller/component/component_test.go +++ b/internal/controller/component/component_test.go @@ -62,9 +62,9 @@ var _ = Describe("component module", func() { AddComponentDef(testapps.StatelessNginxComponent, proxyCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompDefName). + AddComponentVersion(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - AddComponent(proxyCompDefName). + AddComponentVersion(proxyCompDefName). AddInitContainerShort("nginx-init", testapps.NginxImage). AddContainerShort("nginx", testapps.NginxImage). GetObject() diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go index 1f07a2c2a..5cec9d8d1 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go @@ -54,7 +54,7 @@ var _ = Describe("sts horizontal scaling test", func() { AddComponentDef(apps.ConsensusMySQLComponent, componentDefName). GetObject() cv := apps.NewClusterVersionFactory(clusterVerName, cd.Name). - AddComponent(componentDefName). + AddComponentVersion(componentDefName). GetObject() cluster := apps.NewClusterFactory(namespace, clusterName, cd.Name, cv.Name). AddComponent(componentName, componentDefName). diff --git a/internal/controller/plan/pitr_test.go b/internal/controller/plan/pitr_test.go index 937ebf66e..1fc60dfad 100644 --- a/internal/controller/plan/pitr_test.go +++ b/internal/controller/plan/pitr_test.go @@ -103,9 +103,9 @@ var _ = Describe("PITR Functions", func() { AddComponentDef(testapps.StatelessNginxComponent, nginxCompType). Create(&testCtx).GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompType). + AddComponentVersion(mysqlCompType). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - AddComponent(nginxCompType). + AddComponentVersion(nginxCompType). AddInitContainerShort("nginx-init", testapps.NginxImage). AddContainerShort("nginx", testapps.NginxImage). Create(&testCtx).GetObject() diff --git a/internal/controller/plan/prepare.go b/internal/controller/plan/prepare.go index 6980dc6e4..91cb38eb9 100644 --- a/internal/controller/plan/prepare.go +++ b/internal/controller/plan/prepare.go @@ -125,6 +125,9 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, return nil } + // REVIEW/TODO: + // - need higher level abstraction handling + // - or move this module to part operator controller handling switch task.Component.WorkloadType { case appsv1alpha1.Stateless: if err := workloadProcessor( @@ -195,6 +198,7 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, return err } for _, svc := range svcList { + // REVIEW/TODO: need higher level abstraction handling switch task.Component.WorkloadType { case appsv1alpha1.Consensus: addLeaderSelectorLabels(svc, task.Component) diff --git a/internal/controller/plan/prepare_test.go b/internal/controller/plan/prepare_test.go index dde991fd0..903455042 100644 --- a/internal/controller/plan/prepare_test.go +++ b/internal/controller/plan/prepare_test.go @@ -90,7 +90,7 @@ var _ = Describe("Cluster Controller", func() { AddComponentDef(testapps.StatelessNginxComponent, nginxCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(nginxCompDefName). + AddComponentVersion(nginxCompDefName). AddContainerShort("nginx", testapps.NginxImage). GetObject() cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, @@ -129,7 +129,7 @@ var _ = Describe("Cluster Controller", func() { AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompDefName). + AddComponentVersion(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). GetObject() pvcSpec := testapps.NewPVCSpec("1Gi") @@ -181,7 +181,7 @@ var _ = Describe("Cluster Controller", func() { AddConfigTemplate(cm.Name, cm.Name, cfgTpl.Name, testCtx.DefaultNamespace, "mysql-config"). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompDefName). + AddComponentVersion(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). GetObject() pvcSpec := testapps.NewPVCSpec("1Gi") @@ -230,7 +230,7 @@ var _ = Describe("Cluster Controller", func() { AddContainerVolumeMounts("mysql", []corev1.VolumeMount{{Name: "mysql-config", MountPath: "/mnt/config"}}). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompDefName). + AddComponentVersion(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). GetObject() pvcSpec := testapps.NewPVCSpec("1Gi") @@ -286,9 +286,9 @@ var _ = Describe("Cluster Controller", func() { AddComponentDef(testapps.StatelessNginxComponent, nginxCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(mysqlCompDefName). + AddComponentVersion(mysqlCompDefName). AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - AddComponent(nginxCompDefName). + AddComponentVersion(nginxCompDefName). AddContainerShort("nginx", testapps.NginxImage). GetObject() pvcSpec := testapps.NewPVCSpec("1Gi") @@ -337,9 +337,9 @@ var _ = Describe("Cluster Controller", func() { AddComponentDef(testapps.StatelessNginxComponent, nginxCompDefName). GetObject() clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(redisCompDefName). + AddComponentVersion(redisCompDefName). AddContainerShort("redis", testapps.DefaultRedisImageName). - AddComponent(nginxCompDefName). + AddComponentVersion(nginxCompDefName). AddContainerShort("nginx", testapps.NginxImage). GetObject() cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, @@ -392,15 +392,15 @@ var _ = Describe("Cluster Controller", func() { // AddComponentDef(testapps.StatelessNginxComponent, nginxCompDefName). // GetObject() // clusterVersion = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefName). - // AddComponent(redisCompDefName). + // AddComponentVersion(redisCompDefName). // AddContainerShort("redis", testapps.DefaultRedisImageName). - // AddComponent(nginxCompDefName). + // AddComponentVersion(nginxCompDefName). // AddContainerShort("nginx", testapps.NginxImage). // GetObject() // pvcSpec := testapps.NewPVCSpec("1Gi") // cluster = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, // clusterDef.Name, clusterVersion.Name). - // AddComponent(redisCompName, redisCompDefName). + // AddComponentVersion(redisCompName, redisCompDefName). // SetReplicas(2). // SetPrimaryIndex(0). // AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). diff --git a/internal/testutil/apps/cluster_consensus_test_util.go b/internal/testutil/apps/cluster_consensus_test_util.go index 9b8014984..a995f021f 100644 --- a/internal/testutil/apps/cluster_consensus_test_util.go +++ b/internal/testutil/apps/cluster_consensus_test_util.go @@ -77,7 +77,7 @@ func CreateConsensusMysqlClusterDef(testCtx testutil.TestContext, clusterDefName // CreateConsensusMysqlClusterVersion creates a mysql clusterVersion with a component of ConsensusSet type. func CreateConsensusMysqlClusterVersion(testCtx testutil.TestContext, clusterDefName, clusterVersionName, workloadType string) *appsv1alpha1.ClusterVersion { - return NewClusterVersionFactory(clusterVersionName, clusterDefName).AddComponent(workloadType).AddContainerShort("mysql", ApeCloudMySQLImage). + return NewClusterVersionFactory(clusterVersionName, clusterDefName).AddComponentVersion(workloadType).AddContainerShort("mysql", ApeCloudMySQLImage). Create(&testCtx).GetObject() } diff --git a/internal/testutil/apps/cluster_replication_test_util.go b/internal/testutil/apps/cluster_replication_test_util.go index a09c87530..296952db5 100644 --- a/internal/testutil/apps/cluster_replication_test_util.go +++ b/internal/testutil/apps/cluster_replication_test_util.go @@ -33,8 +33,8 @@ import ( "github.com/apecloud/kubeblocks/internal/testutil" ) -// MockReplicationComponentStsPod mocks to create pod of the replication StatefulSet, just using in envTest -func MockReplicationComponentStsPod( +// MockReplicationComponentPod mocks to create pod of the replication StatefulSet, just using in envTest +func MockReplicationComponentPod( g gomega.Gomega, testCtx testutil.TestContext, sts *appsv1.StatefulSet, @@ -85,7 +85,7 @@ func MockReplicationComponentPods( } else if i == 0 { role = "primary" } - pods = append(pods, MockReplicationComponentStsPod(g, testCtx, sts, clusterName, compName, podName, role)) + pods = append(pods, MockReplicationComponentPod(g, testCtx, sts, clusterName, compName, podName, role)) } return pods } diff --git a/internal/testutil/apps/cluster_util.go b/internal/testutil/apps/cluster_util.go index 938ab6ae2..ac7df2f56 100644 --- a/internal/testutil/apps/cluster_util.go +++ b/internal/testutil/apps/cluster_util.go @@ -47,9 +47,9 @@ func InitClusterWithHybridComps( AddComponentDef(StatefulMySQLComponent, statefulCompDefName). Create(&testCtx).GetObject() clusterVersion := NewClusterVersionFactory(clusterVersionName, clusterDefName). - AddComponent(statelessCompDefName).AddContainerShort(DefaultNginxContainerName, NginxImage). - AddComponent(consensusCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). - AddComponent(statefulCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). + AddComponentVersion(statelessCompDefName).AddContainerShort(DefaultNginxContainerName, NginxImage). + AddComponentVersion(consensusCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). + AddComponentVersion(statefulCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). Create(&testCtx).GetObject() pvcSpec := NewPVCSpec("1Gi") cluster := NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). diff --git a/internal/testutil/apps/clusterversion_factory.go b/internal/testutil/apps/clusterversion_factory.go index 8d78985bf..3ed7d3052 100644 --- a/internal/testutil/apps/clusterversion_factory.go +++ b/internal/testutil/apps/clusterversion_factory.go @@ -41,7 +41,7 @@ func NewClusterVersionFactory(name, cdRef string) *MockClusterVersionFactory { return f } -func (factory *MockClusterVersionFactory) AddComponent(compDefName string) *MockClusterVersionFactory { +func (factory *MockClusterVersionFactory) AddComponentVersion(compDefName string) *MockClusterVersionFactory { comp := appsv1alpha1.ClusterComponentVersion{ ComponentDefRef: compDefName, } diff --git a/test/integration/backup_mysql_test.go b/test/integration/backup_mysql_test.go index c2fbfdd14..11afeaa32 100644 --- a/test/integration/backup_mysql_test.go +++ b/test/integration/backup_mysql_test.go @@ -99,7 +99,7 @@ var _ = Describe("MySQL data protection function", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() By("Create a cluster obj") diff --git a/test/integration/controller_suite_test.go b/test/integration/controller_suite_test.go index 31a66d4fc..d7242d6b3 100644 --- a/test/integration/controller_suite_test.go +++ b/test/integration/controller_suite_test.go @@ -202,7 +202,7 @@ func CreateSimpleConsensusMySQLClusterWithConfig( By("Create a clusterVersion obj") clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompDefName). + AddComponentVersion(mysqlCompDefName). AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). AddLabels(cfgcore.GenerateTPLUniqLabelKeyWithConfig(mysqlConfigName), configmap.Name, cfgcore.GenerateConstraintsUniqLabelKeyWithConfig(constraint.Name), constraint.Name). diff --git a/test/integration/mysql_ha_test.go b/test/integration/mysql_ha_test.go index 29d94ddc1..0239ed8bc 100644 --- a/test/integration/mysql_ha_test.go +++ b/test/integration/mysql_ha_test.go @@ -201,7 +201,7 @@ var _ = Describe("MySQL High-Availability function", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) diff --git a/test/integration/mysql_scale_test.go b/test/integration/mysql_scale_test.go index b0987431a..a54c85d36 100644 --- a/test/integration/mysql_scale_test.go +++ b/test/integration/mysql_scale_test.go @@ -234,7 +234,7 @@ var _ = Describe("MySQL Scaling function", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) @@ -262,7 +262,7 @@ var _ = Describe("MySQL Scaling function", func() { By("Create a clusterVersion obj") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.GetName()). - AddComponent(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). + AddComponentVersion(mysqlCompDefName).AddContainerShort(testapps.DefaultMySQLContainerName, testapps.ApeCloudMySQLImage). Create(&testCtx).GetObject() }) diff --git a/test/integration/redis_hscale_test.go b/test/integration/redis_hscale_test.go index 34dc613de..3905c6c38 100644 --- a/test/integration/redis_hscale_test.go +++ b/test/integration/redis_hscale_test.go @@ -204,7 +204,7 @@ var _ = Describe("Redis Horizontal Scale function", func() { By("Create a clusterVersion obj with replication workloadType.") clusterVersionObj = testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponent(testapps.DefaultRedisCompDefName). + AddComponentVersion(testapps.DefaultRedisCompDefName). AddInitContainerShort(testapps.DefaultRedisInitContainerName, testapps.DefaultRedisImageName). AddContainerShort(testapps.DefaultRedisContainerName, testapps.DefaultRedisImageName). Create(&testCtx).GetObject() From 8752b25b1f0d13eba089beacc3f5f3636c87c17e Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Thu, 27 Apr 2023 12:59:07 +0800 Subject: [PATCH 197/439] feat: remove confirm when --dry-run is set for cli ops (#2946) Co-authored-by: huyongqii --- internal/cli/cmd/cluster/create.go | 2 +- internal/cli/cmd/cluster/operations.go | 4 +- internal/cli/create/create.go | 66 ++++++++++++-------------- internal/cli/create/create_test.go | 5 +- 4 files changed, 35 insertions(+), 42 deletions(-) diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index e0bbb1ba3..324fb7871 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -398,7 +398,7 @@ func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra cmd.Flags().StringVarP(&o.SetFile, "set-file", "f", "", "Use yaml file, URL, or stdin to set the cluster resource") cmd.Flags().StringArrayVar(&o.Values, "set", []string{}, "Set the cluster resource including cpu, memory, replicas and storage, or you can just specify the class, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi or --set class=general-1c1g)") cmd.Flags().StringVar(&o.Backup, "backup", "", "Set a source backup to restore data") - cmd.Flags().String("dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().StringVar(&o.DryRunStrategy, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" // add updatable flags o.UpdatableFlags.addFlags(cmd) diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 09e223a43..c69712a7a 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -104,7 +104,7 @@ func (o *OperationsOptions) buildCommonFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&o.OpsRequestName, "name", "", "OpsRequest name. if not specified, it will be randomly generated ") cmd.Flags().IntVar(&o.TTLSecondsAfterSucceed, "ttlSecondsAfterSucceed", 0, "Time to live after the OpsRequest succeed") - cmd.Flags().String("dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().StringVar(&o.DryRunStrategy, "dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" if o.HasComponentNamesFlag { cmd.Flags().StringSliceVar(&o.ComponentNames, "components", nil, " Component names to this operations") @@ -242,7 +242,7 @@ func (o *OperationsOptions) Validate() error { return err } } - if o.RequireConfirm { + if o.RequireConfirm && o.DryRunStrategy == "none" { return delete.Confirm([]string{o.Name}, o.In) } return nil diff --git a/internal/cli/create/create.go b/internal/cli/create/create.go index cbf3669e4..386caea3a 100755 --- a/internal/cli/create/create.go +++ b/internal/cli/create/create.go @@ -55,6 +55,15 @@ var ( cueTemplate embed.FS ) +type DryRunStrategy int + +const ( + // DryRunNone indicates the client will make all mutating calls + DryRunNone DryRunStrategy = iota + DryRunClient + DryRunServer +) + type Inputs struct { // Use cobra command use Use string @@ -83,9 +92,6 @@ type Inputs struct { // Group of Version, default is v1alpha1 Version string - // Command of input - Cmd *cobra.Command - // Factory Factory cmdutil.Factory @@ -123,6 +129,8 @@ type BaseOptions struct { Format printer.Format `json:"-"` + DryRunStrategy string `json:"-"` + // Quiet minimize unnecessary output Quiet bool @@ -142,7 +150,6 @@ func BuildCommand(inputs Inputs) *cobra.Command { util.CheckErr(inputs.BaseOptionsObj.Run(inputs)) }, } - inputs.Cmd = cmd if inputs.BuildFlags != nil { inputs.BuildFlags(cmd) } @@ -245,7 +252,7 @@ func (o *BaseOptions) Run(inputs Inputs) error { } previewObj := unstructuredObj - dryRunStrategy, err := GetDryRunStrategy(inputs.Cmd) + dryRunStrategy, err := o.GetDryRunStrategy() if err != nil { return err } @@ -344,6 +351,24 @@ func (o *BaseOptions) RunAsApply(inputs Inputs) error { return nil } +func (o *BaseOptions) GetDryRunStrategy() (DryRunStrategy, error) { + if o.DryRunStrategy == "" { + return DryRunNone, nil + } + switch o.DryRunStrategy { + case "client": + return DryRunClient, nil + case "server": + return DryRunServer, nil + case "unchanged": + return DryRunClient, nil + case "none": + return DryRunNone, nil + default: + return DryRunNone, fmt.Errorf(`invalid dry-run value (%v). Must be "none", "server", or "client"`, o.DryRunStrategy) + } +} + // NewCueValue convert cue template to cue Value which holds any value like Boolean,Struct,String and more cue type. func newCueValue(cueTemplateName string) (cue.Value, error) { tmplFs, _ := debme.FS(cueTemplate, "template") @@ -383,34 +408,3 @@ func convertContentToUnstructured(cueValue cue.Value) (*unstructured.Unstructure } return unstructuredObj, nil } - -type DryRunStrategy int - -const ( - // DryRunNone indicates the client will make all mutating calls - DryRunNone DryRunStrategy = iota - DryRunClient - DryRunServer -) - -func GetDryRunStrategy(cmd *cobra.Command) (DryRunStrategy, error) { - if cmd == nil { - return DryRunNone, nil - } - dryRunFlag, err := cmd.Flags().GetString("dry-run") - if err != nil { - return DryRunNone, nil - } - switch dryRunFlag { - case cmd.Flag("dry-run").NoOptDefVal: - return DryRunClient, nil - case "client": - return DryRunClient, nil - case "server": - return DryRunServer, nil - case "none": - return DryRunNone, nil - default: - return DryRunNone, fmt.Errorf(`invalid dry-run value (%v). Must be "none", "server", or "client"`, dryRunFlag) - } -} diff --git a/internal/cli/create/create_test.go b/internal/cli/create/create_test.go index ff9640ca0..b769f6b12 100755 --- a/internal/cli/create/create_test.go +++ b/internal/cli/create/create_test.go @@ -135,13 +135,12 @@ var _ = Describe("Create", func() { }, BuildFlags: func(cmd *cobra.Command) { cmd.Flags().StringVar(&baseOptions.Namespace, "clusterDefRef", "", "cluster definition") - cmd.Flags().String("dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().StringVar(&baseOptions.DryRunStrategy, "dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" printer.AddOutputFlagForCreate(cmd, &baseOptions.Format) }, } cmd := BuildCommand(inputs) - inputs.Cmd = cmd testCases := []struct { clusterName string @@ -200,7 +199,7 @@ var _ = Describe("Create", func() { Expect(baseOptions.Complete(inputs, []string{})).Should(Succeed()) Expect(baseOptions.Validate(inputs)).Should(Succeed()) - dryRunStrateg, _ := GetDryRunStrategy(cmd) + dryRunStrateg, _ := baseOptions.GetDryRunStrategy() if t.success { Expect(dryRunStrateg == t.dryRunStrateg).Should(BeTrue()) Expect(baseOptions.Run(inputs)).Should(Succeed()) From 6d044023b52c1a9f1c0c1b8c8f85b7ca33d3264a Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Thu, 27 Apr 2023 15:47:59 +0800 Subject: [PATCH 198/439] chore: rename addon milvus, qdrant and weaviate (#2974) --- deploy/helm/templates/addons/milvus-addon.yaml | 2 +- deploy/helm/templates/addons/qdrant-addon.yaml | 2 +- deploy/helm/templates/addons/weaviate-addon.yaml | 2 +- deploy/milvus-cluster/templates/cluster.yaml | 2 +- deploy/milvus/Chart.yaml | 2 +- deploy/milvus/templates/backuppolicytemplate.yaml | 6 +++--- deploy/milvus/templates/clusterdefinition.yaml | 6 +++--- deploy/milvus/templates/clusterversion.yaml | 2 +- deploy/milvus/templates/configmap.yaml | 2 +- deploy/qdrant-cluster/templates/cluster.yaml | 2 +- deploy/qdrant/Chart.yaml | 2 +- deploy/qdrant/templates/backuppolicytemplate.yaml | 6 +++--- deploy/qdrant/templates/clusterdefinition.yaml | 6 +++--- deploy/qdrant/templates/clusterversion.yaml | 2 +- deploy/qdrant/templates/configmap.yaml | 2 +- deploy/weaviate-cluster/templates/cluster.yaml | 2 +- deploy/weaviate/Chart.yaml | 2 +- deploy/weaviate/templates/backuppolicytemplate.yaml | 6 +++--- deploy/weaviate/templates/clusterdefinition.yaml | 6 +++--- deploy/weaviate/templates/clusterversion.yaml | 2 +- deploy/weaviate/templates/configmap.yaml | 2 +- docs/user_docs/kubeblocks-for-gptplugin/Installation.md | 6 +++--- 22 files changed, 36 insertions(+), 36 deletions(-) diff --git a/deploy/helm/templates/addons/milvus-addon.yaml b/deploy/helm/templates/addons/milvus-addon.yaml index 614eeb8e9..2f2250754 100644 --- a/deploy/helm/templates/addons/milvus-addon.yaml +++ b/deploy/helm/templates/addons/milvus-addon.yaml @@ -15,7 +15,7 @@ spec: type: Helm helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/milvus-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/milvus-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: autoInstall: true diff --git a/deploy/helm/templates/addons/qdrant-addon.yaml b/deploy/helm/templates/addons/qdrant-addon.yaml index 61cee3c9f..b44cb8a08 100644 --- a/deploy/helm/templates/addons/qdrant-addon.yaml +++ b/deploy/helm/templates/addons/qdrant-addon.yaml @@ -15,7 +15,7 @@ spec: type: Helm helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/qdrant-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/qdrant-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: autoInstall: true diff --git a/deploy/helm/templates/addons/weaviate-addon.yaml b/deploy/helm/templates/addons/weaviate-addon.yaml index a86a78742..06f9f9d6b 100644 --- a/deploy/helm/templates/addons/weaviate-addon.yaml +++ b/deploy/helm/templates/addons/weaviate-addon.yaml @@ -15,7 +15,7 @@ spec: type: Helm helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/weaviate-standalone-{{ default .Chart.Version .Values.versionOverride }}.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/weaviate-{{ default .Chart.Version .Values.versionOverride }}.tgz installable: autoInstall: true diff --git a/deploy/milvus-cluster/templates/cluster.yaml b/deploy/milvus-cluster/templates/cluster.yaml index fdac7bee9..5ed812892 100644 --- a/deploy/milvus-cluster/templates/cluster.yaml +++ b/deploy/milvus-cluster/templates/cluster.yaml @@ -4,7 +4,7 @@ metadata: name: {{ .Release.Name }} labels: {{ include "milvus.labels" . | nindent 4 }} spec: - clusterDefinitionRef: milvus-standalone # ref clusterdefinition.name + clusterDefinitionRef: milvus # ref clusterdefinition.name clusterVersionRef: milvus-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} # ref clusterversion.name terminationPolicy: {{ .Values.terminationPolicy }} affinity: diff --git a/deploy/milvus/Chart.yaml b/deploy/milvus/Chart.yaml index bc6264496..cde9617ef 100644 --- a/deploy/milvus/Chart.yaml +++ b/deploy/milvus/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: milvus-standalone +name: milvus description: . type: application diff --git a/deploy/milvus/templates/backuppolicytemplate.yaml b/deploy/milvus/templates/backuppolicytemplate.yaml index e6a22571a..154924259 100644 --- a/deploy/milvus/templates/backuppolicytemplate.yaml +++ b/deploy/milvus/templates/backuppolicytemplate.yaml @@ -1,12 +1,12 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: milvus-standalone-backup-policy-template + name: milvus-backup-policy-template labels: - clusterdefinition.kubeblocks.io/name: milvus-standalone + clusterdefinition.kubeblocks.io/name: milvus {{- include "milvus.labels" . | nindent 4 }} spec: - clusterDefinitionRef: milvus-standalone + clusterDefinitionRef: milvus backupPolicies: - componentDefRef: milvus ttl: 7d diff --git a/deploy/milvus/templates/clusterdefinition.yaml b/deploy/milvus/templates/clusterdefinition.yaml index 0bc3e9753..7a0553f18 100644 --- a/deploy/milvus/templates/clusterdefinition.yaml +++ b/deploy/milvus/templates/clusterdefinition.yaml @@ -2,7 +2,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterDefinition metadata: - name: milvus-standalone + name: milvus labels: {{- include "milvus.labels" . | nindent 4 }} spec: @@ -26,8 +26,8 @@ spec: scrapePort: 9187 logConfigs: configSpecs: - - name: milvus-standalone-config-template - templateRef: milvus-standalone-config-template + - name: milvus-config-template + templateRef: milvus-config-template volumeName: milvus-config namespace: {{.Release.Namespace}} service: diff --git a/deploy/milvus/templates/clusterversion.yaml b/deploy/milvus/templates/clusterversion.yaml index f13a210cd..b387ecb96 100644 --- a/deploy/milvus/templates/clusterversion.yaml +++ b/deploy/milvus/templates/clusterversion.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "milvus.labels" . | nindent 4 }} spec: - clusterDefinitionRef: milvus-standalone + clusterDefinitionRef: milvus componentVersions: - componentDefRef: minio versionsContext: diff --git a/deploy/milvus/templates/configmap.yaml b/deploy/milvus/templates/configmap.yaml index 864337864..2cda5ac6a 100644 --- a/deploy/milvus/templates/configmap.yaml +++ b/deploy/milvus/templates/configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: milvus-standalone-config-template + name: milvus-config-template namespace: {{ .Release.Namespace | quote }} labels: {{- include "milvus.labels" . | nindent 4 }} diff --git a/deploy/qdrant-cluster/templates/cluster.yaml b/deploy/qdrant-cluster/templates/cluster.yaml index e088659e1..5af9fad6a 100644 --- a/deploy/qdrant-cluster/templates/cluster.yaml +++ b/deploy/qdrant-cluster/templates/cluster.yaml @@ -4,7 +4,7 @@ metadata: name: {{ .Release.Name }} labels: {{ include "qdrant.labels" . | nindent 4 }} spec: - clusterDefinitionRef: qdrant-standalone # ref clusterdefinition.name + clusterDefinitionRef: qdrant # ref clusterdefinition.name clusterVersionRef: qdrant-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} # ref clusterversion.name terminationPolicy: {{ .Values.terminationPolicy }} affinity: diff --git a/deploy/qdrant/Chart.yaml b/deploy/qdrant/Chart.yaml index 38fb257eb..bc13d4abb 100644 --- a/deploy/qdrant/Chart.yaml +++ b/deploy/qdrant/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: qdrant-standalone +name: qdrant description: . type: application diff --git a/deploy/qdrant/templates/backuppolicytemplate.yaml b/deploy/qdrant/templates/backuppolicytemplate.yaml index d26692c41..a8f4dc89d 100644 --- a/deploy/qdrant/templates/backuppolicytemplate.yaml +++ b/deploy/qdrant/templates/backuppolicytemplate.yaml @@ -1,12 +1,12 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: qdrant-standalone-backup-policy-template + name: qdrant-backup-policy-template labels: - clusterdefinition.kubeblocks.io/name: qdrant-standalone + clusterdefinition.kubeblocks.io/name: qdrant {{- include "qdrant.labels" . | nindent 4 }} spec: - clusterDefinitionRef: qdrant-standalone + clusterDefinitionRef: qdrant backupPolicies: - componentDefRef: qdrant ttl: 7d diff --git a/deploy/qdrant/templates/clusterdefinition.yaml b/deploy/qdrant/templates/clusterdefinition.yaml index e3f9533ab..996d7ead2 100644 --- a/deploy/qdrant/templates/clusterdefinition.yaml +++ b/deploy/qdrant/templates/clusterdefinition.yaml @@ -2,7 +2,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterDefinition metadata: - name: qdrant-standalone + name: qdrant labels: {{- include "qdrant.labels" . | nindent 4 }} spec: @@ -25,8 +25,8 @@ spec: scrapePort: 9187 logConfigs: configSpecs: - - name: qdrant-standalone-config-template - templateRef: qdrant-standalone-config-template + - name: qdrant-config-template + templateRef: qdrant-config-template volumeName: qdrant-config namespace: {{ .Release.Namespace }} service: diff --git a/deploy/qdrant/templates/clusterversion.yaml b/deploy/qdrant/templates/clusterversion.yaml index c57d1e9bc..6b19fba16 100644 --- a/deploy/qdrant/templates/clusterversion.yaml +++ b/deploy/qdrant/templates/clusterversion.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "qdrant.labels" . | nindent 4 }} spec: - clusterDefinitionRef: qdrant-standalone + clusterDefinitionRef: qdrant componentVersions: - componentDefRef: qdrant versionsContext: diff --git a/deploy/qdrant/templates/configmap.yaml b/deploy/qdrant/templates/configmap.yaml index 913fdeef2..0e5ae146b 100644 --- a/deploy/qdrant/templates/configmap.yaml +++ b/deploy/qdrant/templates/configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: qdrant-standalone-config-template + name: qdrant-config-template namespace: {{ .Release.Namespace | quote }} labels: {{- include "qdrant.labels" . | nindent 4 }} diff --git a/deploy/weaviate-cluster/templates/cluster.yaml b/deploy/weaviate-cluster/templates/cluster.yaml index 0bf4a0804..c992f299e 100644 --- a/deploy/weaviate-cluster/templates/cluster.yaml +++ b/deploy/weaviate-cluster/templates/cluster.yaml @@ -4,7 +4,7 @@ metadata: name: {{ .Release.Name }} labels: {{ include "weaviate.labels" . | nindent 4 }} spec: - clusterDefinitionRef: weaviate-standalone # ref clusterdefinition.name + clusterDefinitionRef: weaviate # ref clusterdefinition.name clusterVersionRef: weaviate-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} # ref clusterversion.name terminationPolicy: {{ .Values.terminationPolicy }} affinity: diff --git a/deploy/weaviate/Chart.yaml b/deploy/weaviate/Chart.yaml index 26790e7b9..4853384ab 100644 --- a/deploy/weaviate/Chart.yaml +++ b/deploy/weaviate/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -name: weaviate-standalone +name: weaviate description: . type: application diff --git a/deploy/weaviate/templates/backuppolicytemplate.yaml b/deploy/weaviate/templates/backuppolicytemplate.yaml index 65c8d9648..37b4c2faa 100644 --- a/deploy/weaviate/templates/backuppolicytemplate.yaml +++ b/deploy/weaviate/templates/backuppolicytemplate.yaml @@ -1,12 +1,12 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: BackupPolicyTemplate metadata: - name: weaviate-standalone-backup-policy-template + name: weaviate-backup-policy-template labels: - clusterdefinition.kubeblocks.io/name: weaviate-standalone + clusterdefinition.kubeblocks.io/name: weaviate {{- include "weaviate.labels" . | nindent 4 }} spec: - clusterDefinitionRef: weaviate-standalone + clusterDefinitionRef: weaviate backupPolicies: - componentDefRef: weaviate ttl: 7d diff --git a/deploy/weaviate/templates/clusterdefinition.yaml b/deploy/weaviate/templates/clusterdefinition.yaml index 500dd3f66..af7dad577 100644 --- a/deploy/weaviate/templates/clusterdefinition.yaml +++ b/deploy/weaviate/templates/clusterdefinition.yaml @@ -2,7 +2,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterDefinition metadata: - name: weaviate-standalone + name: weaviate labels: {{- include "weaviate.labels" . | nindent 4 }} spec: @@ -25,8 +25,8 @@ spec: scrapePort: 9187 logConfigs: configSpecs: - - name: weaviate-standalone-config-template - templateRef: weaviate-standalone-config-template + - name: weaviate-config-template + templateRef: weaviate-config-template volumeName: weaviate-config namespace: {{ .Release.Namespace }} service: diff --git a/deploy/weaviate/templates/clusterversion.yaml b/deploy/weaviate/templates/clusterversion.yaml index 1b3a69b78..a74d2d4d9 100644 --- a/deploy/weaviate/templates/clusterversion.yaml +++ b/deploy/weaviate/templates/clusterversion.yaml @@ -5,7 +5,7 @@ metadata: labels: {{- include "weaviate.labels" . | nindent 4 }} spec: - clusterDefinitionRef: weaviate-standalone + clusterDefinitionRef: weaviate componentVersions: - componentDefRef: weaviate versionsContext: diff --git a/deploy/weaviate/templates/configmap.yaml b/deploy/weaviate/templates/configmap.yaml index 69e621073..176d2804c 100644 --- a/deploy/weaviate/templates/configmap.yaml +++ b/deploy/weaviate/templates/configmap.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: weaviate-standalone-config-template + name: weaviate-config-template namespace: {{ .Release.Namespace | quote }} labels: {{- include "weaviate.labels" . | nindent 4 }} diff --git a/docs/user_docs/kubeblocks-for-gptplugin/Installation.md b/docs/user_docs/kubeblocks-for-gptplugin/Installation.md index 482ec19f3..97a95b347 100644 --- a/docs/user_docs/kubeblocks-for-gptplugin/Installation.md +++ b/docs/user_docs/kubeblocks-for-gptplugin/Installation.md @@ -26,16 +26,16 @@ kbcli addon list kubectl get clusterdefintion NAME MAIN-COMPONENT-NAME STATUS AGE -qdrant-standalone qdrant Available 6m14s +qdrant qdrant Available 6m14s ``` 4. create a qdrant cluster ```shell -kbcli cluster create --cluster-definition=qdrant-standalone +kbcli cluster create --cluster-definition=qdrant Warning: cluster version is not specified, use the recently created ClusterVersion qdrant-1.1.0 Cluster lilac26 created ``` -a qdrant standalone cluster is created successfully +a qdrant cluster is created successfully #### Step 2: Start the plugin with qdrant as store with helm to install ```shell From 733832a82a64c89aa7bfb13e45af0a8c78e0a75a Mon Sep 17 00:00:00 2001 From: xingran Date: Thu, 27 Apr 2023 19:01:40 +0800 Subject: [PATCH 199/439] chore: cluster componentSpec support serviceAccountName (#2975) --- apis/apps/v1alpha1/cluster_types.go | 4 + .../bases/apps.kubeblocks.io_clusters.yaml | 4 + controllers/apps/cluster_controller_test.go | 25 +++++++ .../crds/apps.kubeblocks.io_clusters.yaml | 4 + deploy/postgresql-cluster/Chart.yaml | 10 ++- deploy/postgresql-cluster/templates/NOTES.txt | 3 +- .../postgresql-cluster/templates/_helpers.tpl | 8 +- .../postgresql-cluster/templates/cluster.yaml | 1 + deploy/postgresql-cluster/templates/role.yaml | 47 ++++++++++++ .../templates/rolebinding.yaml | 16 ++++ .../templates/serviceaccount.yaml | 8 ++ .../templates/validate.yaml | 3 + deploy/postgresql-cluster/values.yaml | 10 ++- deploy/postgresql/Chart.yaml | 9 ++- .../templates/clusterdefinition.yaml | 1 - deploy/postgresql/templates/patroni-rbac.yaml | 74 ------------------- internal/controller/component/component.go | 3 + internal/controller/component/type.go | 1 + internal/controller/plan/prepare.go | 4 - internal/testutil/apps/cluster_factory.go | 8 ++ 20 files changed, 156 insertions(+), 87 deletions(-) create mode 100644 deploy/postgresql-cluster/templates/role.yaml create mode 100644 deploy/postgresql-cluster/templates/rolebinding.yaml create mode 100644 deploy/postgresql-cluster/templates/serviceaccount.yaml create mode 100644 deploy/postgresql-cluster/templates/validate.yaml delete mode 100644 deploy/postgresql/templates/patroni-rbac.yaml diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index 1f78db199..c3f0fb604 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -181,6 +181,10 @@ type ClusterComponentSpec struct { // required when TLS enabled // +optional Issuer *Issuer `json:"issuer,omitempty"` + + // serviceAccountName is the name of the ServiceAccount that component runs depend on. + // +optional + ServiceAccountName string `json:"serviceAccountName,omitempty"` } type ComponentMessageMap map[string]string diff --git a/config/crd/bases/apps.kubeblocks.io_clusters.yaml b/config/crd/bases/apps.kubeblocks.io_clusters.yaml index 3942699b1..fba2650c2 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusters.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusters.yaml @@ -298,6 +298,10 @@ spec: type: object type: object x-kubernetes-preserve-unknown-fields: true + serviceAccountName: + description: serviceAccountName is the name of the ServiceAccount + that component runs depend on. + type: string services: description: services expose endpoints can be accessed by clients items: diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 0f4443324..7fe785e81 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -753,6 +753,27 @@ var _ = Describe("Cluster Controller", func() { } + testClusterServiceAccount := func(compName, compDefName string) { + By("Creating a cluster with target service account name") + + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterDefObj.Name, clusterVersionObj.Name). + AddComponent(compName, compDefName).SetReplicas(3). + SetServiceAccountName("test-service-account"). + Create(&testCtx).GetObject() + clusterKey = client.ObjectKeyFromObject(clusterObj) + + By("Waiting for the cluster controller to create resources completely") + waitForCreatingResourceCompletely(clusterKey, compName) + + By("Checking the podSpec.serviceAccountName") + Eventually(func(g Gomega) { + stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) + podSpec := stsList.Items[0].Spec.Template.Spec + g.Expect(podSpec.ServiceAccountName).To(Equal("test-service-account")) + }).Should(Succeed()) + } + testComponentAffinity := func(compName, compDefName string) { const clusterTopologyKey = "testClusterTopologyKey" const compTopologyKey = "testComponentTopologyKey" @@ -1334,6 +1355,10 @@ var _ = Describe("Cluster Controller", func() { testServiceAddAndDelete(compName, compDefName) }) + It(fmt.Sprintf("[comp: %s] should add serviceAccountName correctly", compName), func() { + testClusterServiceAccount(compName, compDefName) + }) + Context(fmt.Sprintf("[comp: %s] and with cluster affinity set", compName), func() { It("should create pod with cluster affinity", func() { testClusterAffinity(compName, compDefName) diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml index 3942699b1..fba2650c2 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml @@ -298,6 +298,10 @@ spec: type: object type: object x-kubernetes-preserve-unknown-fields: true + serviceAccountName: + description: serviceAccountName is the name of the ServiceAccount + that component runs depend on. + type: string services: description: services expose endpoints can be accessed by clients items: diff --git a/deploy/postgresql-cluster/Chart.yaml b/deploy/postgresql-cluster/Chart.yaml index 6f87813f8..886fa9027 100644 --- a/deploy/postgresql-cluster/Chart.yaml +++ b/deploy/postgresql-cluster/Chart.yaml @@ -6,4 +6,12 @@ type: application version: 0.5.0-beta.9 -appVersion: "14.7.0" +# appVersion specifies the version of the PostgreSQL (with Patroni HA) database to be created, +# and this value should be consistent with an existing clusterVersion. +# All supported clusterVersion versions can be viewed through `kubectl get clusterVersion`. +# The current default value is the highest version of the PostgreSQL (with Patroni HA) supported in KubeBlocks. +appVersion: "14.7.1" + +annotations: + kubeblocks.io/clusterVersions: "14.7.1,12.14.0" + kubeblocks.io/multiCV: "true" \ No newline at end of file diff --git a/deploy/postgresql-cluster/templates/NOTES.txt b/deploy/postgresql-cluster/templates/NOTES.txt index c3b3453e3..a970ee332 100644 --- a/deploy/postgresql-cluster/templates/NOTES.txt +++ b/deploy/postgresql-cluster/templates/NOTES.txt @@ -1,2 +1,3 @@ -1. Get the application URL by running these commands: +1. By default, the helm chart will create a PostgreSQL (with Patroni HA) cluster with the same version as the Chart appVersion. +2. If you need to create a different version, you need to specify the version through clusterVersionOverride value. diff --git a/deploy/postgresql-cluster/templates/_helpers.tpl b/deploy/postgresql-cluster/templates/_helpers.tpl index 05a757a7e..f38edd501 100644 --- a/deploy/postgresql-cluster/templates/_helpers.tpl +++ b/deploy/postgresql-cluster/templates/_helpers.tpl @@ -54,9 +54,9 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "postgresqlcluster.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "postgresqlcluster.fullname" .) .Values.serviceAccount.name }} +{{- if .Values.serviceAccount.enabled }} +{{- printf "kb-postgres-%s" .Release.Name | trunc 63 | trimSuffix "-" }} {{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- .Values.serviceAccount.name }} {{- end }} +{{- end }} \ No newline at end of file diff --git a/deploy/postgresql-cluster/templates/cluster.yaml b/deploy/postgresql-cluster/templates/cluster.yaml index 8740f5728..eb9d86a94 100644 --- a/deploy/postgresql-cluster/templates/cluster.yaml +++ b/deploy/postgresql-cluster/templates/cluster.yaml @@ -19,6 +19,7 @@ spec: componentDefRef: postgresql # ref clusterdefinition components.name monitor: {{ .Values.monitor.enabled | default false }} replicas: {{ .Values.replicaCount | default 2 }} + serviceAccountName: {{ include "postgresqlcluster.serviceAccountName" . }} primaryIndex: {{ .Values.primaryIndex | default 0 }} switchPolicy: type: {{ .Values.switchPolicy.type}} diff --git a/deploy/postgresql-cluster/templates/role.yaml b/deploy/postgresql-cluster/templates/role.yaml new file mode 100644 index 000000000..198ed1eff --- /dev/null +++ b/deploy/postgresql-cluster/templates/role.yaml @@ -0,0 +1,47 @@ +{{- if .Values.serviceAccount.enabled }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-{{ .Release.Namespace }}-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "postgresqlcluster.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - patch + - update + - watch + # delete is required only for 'patronictl remove' + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - patch + - update + - create + - list + - watch + # delete is required only for 'patronictl remove' + - delete + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch +{{- end }} \ No newline at end of file diff --git a/deploy/postgresql-cluster/templates/rolebinding.yaml b/deploy/postgresql-cluster/templates/rolebinding.yaml new file mode 100644 index 000000000..789a289fa --- /dev/null +++ b/deploy/postgresql-cluster/templates/rolebinding.yaml @@ -0,0 +1,16 @@ +{{- if .Values.serviceAccount.enabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-postgres-{{ .Release.Namespace }}-{{ .Release.Name }} + labels: + {{ include "postgresqlcluster.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-{{ .Release.Namespace }}-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: kb-postgres-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} +{{- end }} \ No newline at end of file diff --git a/deploy/postgresql-cluster/templates/serviceaccount.yaml b/deploy/postgresql-cluster/templates/serviceaccount.yaml new file mode 100644 index 000000000..d4d83a500 --- /dev/null +++ b/deploy/postgresql-cluster/templates/serviceaccount.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.enabled }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kb-postgres-{{ .Release.Name }} + labels: + {{ include "postgresqlcluster.labels" . | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/deploy/postgresql-cluster/templates/validate.yaml b/deploy/postgresql-cluster/templates/validate.yaml new file mode 100644 index 000000000..e5a06b71a --- /dev/null +++ b/deploy/postgresql-cluster/templates/validate.yaml @@ -0,0 +1,3 @@ +{{- if and ( not .Values.serviceAccount.enabled ) ( not .Values.serviceAccount.name ) }} + {{ fail "serviceAccount.enabled is false, the serviceAccount.name is required." }} +{{- end }} diff --git a/deploy/postgresql-cluster/values.yaml b/deploy/postgresql-cluster/values.yaml index a318813a9..cd6d32006 100644 --- a/deploy/postgresql-cluster/values.yaml +++ b/deploy/postgresql-cluster/values.yaml @@ -1,4 +1,4 @@ -# Default values for wesqlcluster. +# Default values for PostgreSQL (with Patroni HA). # This is a YAML-formatted file. # Declare variables to be passed into your templates. @@ -15,6 +15,14 @@ switchPolicy: monitor: enabled: false +# PostgreSQL (with Patroni HA) needs the corresponding RBAC permission to create a cluster(refer to role.yaml, rolebinding.yaml, serviceaccount.yaml) +# If you need to automatically create RBAC, please ensure serviceAccount.enabled=true. +# Otherwise, the user needs to create the corresponding serviceAccount, role and roleBinding permissions manually to use PostgreSQL (with Patroni HA) normally. +serviceAccount: + enabled: true + # if enabled is false, the name is required + name: "" + resources: { } # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little diff --git a/deploy/postgresql/Chart.yaml b/deploy/postgresql/Chart.yaml index ae13d0126..687630734 100644 --- a/deploy/postgresql/Chart.yaml +++ b/deploy/postgresql/Chart.yaml @@ -6,7 +6,14 @@ type: application version: 0.5.0-beta.9 -appVersion: "14.7.0" +# The helm chart contains multiple kernel versions of PostgreSQL (with Patroni HA), +# and each PostgreSQL (with Patroni HA) version corresponds to a clusterVersion object. +# appVersion should be consistent with the highest PostgreSQL (with Patroni HA) kernel version in clusterVersion. +appVersion: "14.7.1" + +annotations: + kubeblocks.io/clusterVersions: "14.7.1,12.14.0" + kubeblocks.io/multiCV: "true" home: https://kubeblocks.io/ icon: https://github.com/apecloud/kubeblocks/raw/main/img/logo.png diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index f52de96fa..75e022071 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -70,7 +70,6 @@ spec: - name: data type: data podSpec: - serviceAccountName: operator securityContext: runAsUser: 0 fsGroup: 103 diff --git a/deploy/postgresql/templates/patroni-rbac.yaml b/deploy/postgresql/templates/patroni-rbac.yaml deleted file mode 100644 index f5f22c5e5..000000000 --- a/deploy/postgresql/templates/patroni-rbac.yaml +++ /dev/null @@ -1,74 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - namespace: default - name: operator - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: operator -rules: - - apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - list - - patch - - update - - watch - # delete is required only for 'patronictl remove' - - delete - - apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - patch - - update - # the following three privileges are necessary only when using endpoints - - create - - list - - watch - # delete is required only for for 'patronictl remove' - - delete - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - patch - - update - - watch - # The following privilege is only necessary for creation of headless service - # for patronidemo-config endpoint, in order to prevent cleaning it up by the - # k8s master. You can avoid giving this privilege by explicitly creating the - # service like it is done in this manifest (lines 160..169) - - apiGroups: - - "" - resources: - - services - verbs: - - create - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: operator -subjects: - - kind: ServiceAccount - name: operator - namespace: default \ No newline at end of file diff --git a/internal/controller/component/component.go b/internal/controller/component/component.go index 421db0e62..33bc8dd22 100644 --- a/internal/controller/component/component.go +++ b/internal/controller/component/component.go @@ -63,6 +63,7 @@ func BuildComponent( VolumeTypes: clusterCompDefObj.VolumeTypes, CustomLabelSpecs: clusterCompDefObj.CustomLabelSpecs, ComponentDef: clusterCompSpec.ComponentDefRef, + ServiceAccountName: clusterCompSpec.ServiceAccountName, } // resolve component.ConfigTemplates @@ -128,6 +129,8 @@ func BuildComponent( } component.PrimaryIndex = clusterCompSpec.PrimaryIndex + // set component.PodSpec.ServiceAccountName + component.PodSpec.ServiceAccountName = component.ServiceAccountName // TODO(zhixu.zt) We need to reserve the VolumeMounts of the container for ConfigMap or Secret, // At present, it is possible to distinguish between ConfigMap volume and normal volume, diff --git a/internal/controller/component/type.go b/internal/controller/component/type.go index 698972b05..b8239b87e 100644 --- a/internal/controller/component/type.go +++ b/internal/controller/component/type.go @@ -57,6 +57,7 @@ type SynthesizedComponent struct { VolumeTypes []v1alpha1.VolumeTypeSpec `json:"VolumeTypes,omitempty"` CustomLabelSpecs []v1alpha1.CustomLabelSpec `json:"customLabelSpecs,omitempty"` ComponentDef string `json:"componentDef,omitempty"` + ServiceAccountName string `json:"serviceAccountName,omitempty"` } // GetPrimaryIndex provides PrimaryIndex value getter, if PrimaryIndex is diff --git a/internal/controller/plan/prepare.go b/internal/controller/plan/prepare.go index 91cb38eb9..430d0d2c5 100644 --- a/internal/controller/plan/prepare.go +++ b/internal/controller/plan/prepare.go @@ -173,10 +173,6 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, } } - // get the maximum value of params.component.Replicas and the number of existing statefulsets under the current component, - // then construct statefulsets for creating replicationSet or handling horizontal scaling of the replicationSet. - // REVIEW/TODO: why using Max? - // replicaCount := math.Max(float64(len(existStsList.Items)), float64(task.Component.Replicas)) if err := workloadProcessor( func(envConfig *corev1.ConfigMap) (client.Object, error) { return buildReplicationSet(reqCtx, task, envConfig.Name) diff --git a/internal/testutil/apps/cluster_factory.go b/internal/testutil/apps/cluster_factory.go index 3a090a0ab..bc2e5fa31 100644 --- a/internal/testutil/apps/cluster_factory.go +++ b/internal/testutil/apps/cluster_factory.go @@ -80,6 +80,14 @@ func (factory *MockClusterFactory) SetReplicas(replicas int32) *MockClusterFacto return factory } +func (factory *MockClusterFactory) SetServiceAccountName(serviceAccountName string) *MockClusterFactory { + comps := factory.get().Spec.ComponentSpecs + if len(comps) > 0 { + comps[len(comps)-1].ServiceAccountName = serviceAccountName + } + return factory +} + func (factory *MockClusterFactory) SetResources(resources corev1.ResourceRequirements) *MockClusterFactory { comps := factory.get().Spec.ComponentSpecs if len(comps) > 0 { From b3cf7400cfe6413ece58c4abc8fccddfadfe3fe1 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Thu, 27 Apr 2023 19:09:43 +0800 Subject: [PATCH 200/439] chore: fix apecloud-mysql hscale error (#3000) --- .../v1alpha1/backuppolicytemplate_types.go | 8 ++++ ...s.kubeblocks.io_backuppolicytemplates.yaml | 7 ++++ controllers/apps/cluster_controller_test.go | 4 +- .../templates/backuppolicytemplate.yaml | 2 + .../backuppolicytemplateforhscale.yaml | 19 +++++++++ .../templates/clusterdefinition.yaml | 2 +- .../templates/backuppolicytemplate.yaml | 2 + .../backuppolicytemplateforhscale.yaml | 19 +++++++++ .../templates/clusterdefinition.yaml | 2 +- ...s.kubeblocks.io_backuppolicytemplates.yaml | 7 ++++ internal/constant/const.go | 25 ++++++------ .../transformer_backup_policy_tpl.go | 39 ++++++++++++++----- 12 files changed, 110 insertions(+), 26 deletions(-) create mode 100644 deploy/apecloud-mysql-scale/templates/backuppolicytemplateforhscale.yaml create mode 100644 deploy/apecloud-mysql/templates/backuppolicytemplateforhscale.yaml diff --git a/apis/apps/v1alpha1/backuppolicytemplate_types.go b/apis/apps/v1alpha1/backuppolicytemplate_types.go index d92439096..cf690df17 100644 --- a/apis/apps/v1alpha1/backuppolicytemplate_types.go +++ b/apis/apps/v1alpha1/backuppolicytemplate_types.go @@ -38,6 +38,14 @@ type BackupPolicyTemplateSpec struct { // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 BackupPolicies []BackupPolicy `json:"backupPolicies"` + + // Identifier is a unique identifier for this BackupPolicyTemplate. + // this identifier will be the suffix of the automatically generated backupPolicy name. + // and must be added when multiple BackupPolicyTemplates exist, + // otherwise the generated backupPolicy override will occur. + // +optional + // +kubebuilder:validation:MaxLength=20 + Identifier string `json:"identifier,omitempty"` } type BackupPolicy struct { diff --git a/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml b/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml index e88cfd8e7..abd20cd43 100644 --- a/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml +++ b/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml @@ -397,6 +397,13 @@ spec: this is an immutable attribute. pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string + identifier: + description: Identifier is a unique identifier for this BackupPolicyTemplate. + this identifier will be the suffix of the automatically generated + backupPolicy name. and must be added when multiple BackupPolicyTemplates + exist, otherwise the generated backupPolicy override will occur. + maxLength: 20 + type: string required: - backupPolicies - clusterDefinitionRef diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 7fe785e81..d8bf615f4 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -555,7 +555,7 @@ var _ = Describe("Cluster Controller", func() { } By("Checking backup policy created from backup policy template") - policyName := lifecycle.DeriveBackupPolicyName(clusterKey.Name, compDef.Name) + policyName := lifecycle.DeriveBackupPolicyName(clusterKey.Name, compDef.Name, "") clusterDef.Spec.ComponentDefs[i].HorizontalScalePolicy = &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyFromSnapshot, BackupPolicyTemplateName: backupPolicyTPLName} @@ -1066,7 +1066,7 @@ var _ = Describe("Cluster Controller", func() { }, }, Spec: dataprotectionv1alpha1.BackupSpec{ - BackupPolicyName: lifecycle.DeriveBackupPolicyName(clusterKey.Name, compDefName), + BackupPolicyName: lifecycle.DeriveBackupPolicyName(clusterKey.Name, compDefName, ""), BackupType: "snapshot", }, } diff --git a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml index acf514eb5..c3a6bfab8 100644 --- a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml @@ -5,6 +5,8 @@ metadata: labels: clusterdefinition.kubeblocks.io/name: apecloud-mysql-scale {{- include "apecloud-mysql.labels" . | nindent 4 }} + annotations: + dataprotection.kubeblocks.io/is-default-policy-template: "true" spec: clusterDefinitionRef: apecloud-mysql-scale backupPolicies: diff --git a/deploy/apecloud-mysql-scale/templates/backuppolicytemplateforhscale.yaml b/deploy/apecloud-mysql-scale/templates/backuppolicytemplateforhscale.yaml new file mode 100644 index 000000000..c60503db1 --- /dev/null +++ b/deploy/apecloud-mysql-scale/templates/backuppolicytemplateforhscale.yaml @@ -0,0 +1,19 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: BackupPolicyTemplate +metadata: + name: apecloud-mysql-scale-backup-policy-template-for-hscale + labels: + clusterdefinition.kubeblocks.io/name: apecloud-mysql + {{- include "apecloud-mysql.labels" . | nindent 4 }} +spec: + clusterDefinitionRef: apecloud-mysql + identifier: hscale + backupPolicies: + - componentDefRef: mysql + snapshot: + hooks: + containerName: mysql + preCommands: + - "touch /data/mysql/data/.restore; sync" + target: + role: leader \ No newline at end of file diff --git a/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml b/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml index 34e70aba7..b6fb70d0c 100644 --- a/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql-scale/templates/clusterdefinition.yaml @@ -73,7 +73,7 @@ spec: targetPort: delvedebug horizontalScalePolicy: type: Snapshot - backupPolicyTemplateName: apecloud-mysql-backup-policy-template + backupPolicyTemplateName: apecloud-mysql-scale-backup-policy-template-for-hscale podSpec: containers: - name: mysql diff --git a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml index 19b61c66f..9915dab78 100644 --- a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml @@ -5,6 +5,8 @@ metadata: labels: clusterdefinition.kubeblocks.io/name: apecloud-mysql {{- include "apecloud-mysql.labels" . | nindent 4 }} + annotations: + dataprotection.kubeblocks.io/is-default-policy-template: "true" spec: clusterDefinitionRef: apecloud-mysql backupPolicies: diff --git a/deploy/apecloud-mysql/templates/backuppolicytemplateforhscale.yaml b/deploy/apecloud-mysql/templates/backuppolicytemplateforhscale.yaml new file mode 100644 index 000000000..b57c36141 --- /dev/null +++ b/deploy/apecloud-mysql/templates/backuppolicytemplateforhscale.yaml @@ -0,0 +1,19 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: BackupPolicyTemplate +metadata: + name: apecloud-mysql-backup-policy-for-hscale + labels: + clusterdefinition.kubeblocks.io/name: apecloud-mysql + {{- include "apecloud-mysql.labels" . | nindent 4 }} +spec: + clusterDefinitionRef: apecloud-mysql + identifier: hscale + backupPolicies: + - componentDefRef: mysql + snapshot: + hooks: + containerName: mysql + preCommands: + - "touch /data/mysql/data/.restore; sync" + target: + role: leader \ No newline at end of file diff --git a/deploy/apecloud-mysql/templates/clusterdefinition.yaml b/deploy/apecloud-mysql/templates/clusterdefinition.yaml index f9a726b29..c64cf222b 100644 --- a/deploy/apecloud-mysql/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql/templates/clusterdefinition.yaml @@ -60,7 +60,7 @@ spec: targetPort: mysql horizontalScalePolicy: type: Snapshot - backupPolicyTemplateName: apecloud-mysql-backup-policy-template + backupPolicyTemplateName: apecloud-mysql-backup-policy-for-hscale volumeTypes: - name: data type: data diff --git a/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml b/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml index e88cfd8e7..abd20cd43 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml @@ -397,6 +397,13 @@ spec: this is an immutable attribute. pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string + identifier: + description: Identifier is a unique identifier for this BackupPolicyTemplate. + this identifier will be the suffix of the automatically generated + backupPolicy name. and must be added when multiple BackupPolicyTemplates + exist, otherwise the generated backupPolicy override will occur. + maxLength: 20 + type: string required: - backupPolicies - clusterDefinitionRef diff --git a/internal/constant/const.go b/internal/constant/const.go index 16ba402da..7e729161c 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -86,18 +86,19 @@ const ( BackupTypeLabelKeyKey = "dataprotection.kubeblocks.io/backup-type" // kubeblocks.io annotations - OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster - ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile - RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart - SnapShotForStartAnnotationKey = "kubeblocks.io/snapshot-for-start" - RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. - ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. - LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" - DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" // DefaultBackupPolicyAnnotationKey specifies the default backup policy. - BackupDataPathPrefixAnnotationKey = "dataprotection.kubeblocks.io/path-prefix" // BackupDataPathPrefixAnnotationKey specifies the backup data path prefix. - BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" - RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. - RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. + OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster + ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile + RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart + SnapShotForStartAnnotationKey = "kubeblocks.io/snapshot-for-start" + RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. + ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. + LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" + DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" // DefaultBackupPolicyAnnotationKey specifies the default backup policy. + DefaultBackupPolicyTemplateAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy-template" // DefaultBackupPolicyTemplateAnnotationKey specifies the default backup policy template. + BackupDataPathPrefixAnnotationKey = "dataprotection.kubeblocks.io/path-prefix" // BackupDataPathPrefixAnnotationKey specifies the backup data path prefix. + BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" + RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. + RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. // ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl ConfigurationTplLabelPrefixKey = "config.kubeblocks.io/tpl" diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index 5c41184fe..f05c2de1b 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -37,7 +37,11 @@ import ( ) // BackupPolicyTPLTransformer transforms the backup policy template to the backup policy. -type BackupPolicyTPLTransformer struct{} +type BackupPolicyTPLTransformer struct { + tplCount int + tplIdentifier string + isDefaultTemplate string +} func (r *BackupPolicyTPLTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { transCtx, _ := ctx.(*ClusterTransformContext) @@ -46,7 +50,8 @@ func (r *BackupPolicyTPLTransformer) Transform(ctx graph.TransformContext, dag * if err := transCtx.Client.List(transCtx.Context, backupPolicyTPLs, client.MatchingLabels{constant.ClusterDefLabelKey: clusterDefName}); err != nil { return err } - if len(backupPolicyTPLs.Items) == 0 { + r.tplCount = len(backupPolicyTPLs.Items) + if r.tplCount == 0 { return nil } rootVertex, err := findRootVertex(dag) @@ -55,6 +60,8 @@ func (r *BackupPolicyTPLTransformer) Transform(ctx graph.TransformContext, dag * } origCluster := transCtx.OrigCluster for _, tpl := range backupPolicyTPLs.Items { + r.isDefaultTemplate = tpl.Annotations[constant.DefaultBackupPolicyTemplateAnnotationKey] + r.tplIdentifier = tpl.Spec.Identifier for _, v := range tpl.Spec.BackupPolicies { compDef := transCtx.ClusterDef.GetComponentDefByName(v.ComponentDefRef) if compDef == nil { @@ -79,14 +86,14 @@ func (r *BackupPolicyTPLTransformer) transformBackupPolicy(transCtx *ClusterTran cluster *appsv1alpha1.Cluster, workloadType appsv1alpha1.WorkloadType, tplName string) *dataprotectionv1alpha1.BackupPolicy { - backupPolicyName := DeriveBackupPolicyName(cluster.Name, policyTPL.ComponentDefRef) + backupPolicyName := DeriveBackupPolicyName(cluster.Name, policyTPL.ComponentDefRef, r.tplIdentifier) backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} if err := transCtx.Client.Get(transCtx.Context, client.ObjectKey{Namespace: cluster.Namespace, Name: backupPolicyName}, backupPolicy); err != nil && !apierrors.IsNotFound(err) { return nil } if len(backupPolicy.Name) == 0 { // build a new backup policy from the backup policy template. - return r.buildBackupPolicy(policyTPL, cluster, workloadType, tplName) + return r.buildBackupPolicy(policyTPL, cluster, workloadType, tplName, backupPolicyName) } // sync the existing backup policy with the cluster changes r.syncBackupPolicy(backupPolicy, cluster, policyTPL, workloadType, tplName) @@ -103,7 +110,7 @@ func (r *BackupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti if backupPolicy.Annotations == nil { backupPolicy.Annotations = map[string]string{} } - backupPolicy.Annotations[constant.DefaultBackupPolicyAnnotationKey] = "true" + backupPolicy.Annotations[constant.DefaultBackupPolicyAnnotationKey] = r.defaultPolicyAnnotationValue() backupPolicy.Annotations[constant.BackupPolicyTemplateAnnotationKey] = tplName if backupPolicy.Labels == nil { backupPolicy.Labels = map[string]string{} @@ -158,14 +165,16 @@ func (r *BackupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti func (r *BackupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.BackupPolicy, cluster *appsv1alpha1.Cluster, workloadType appsv1alpha1.WorkloadType, - tplName string) *dataprotectionv1alpha1.BackupPolicy { + tplName, + backupPolicyName string) *dataprotectionv1alpha1.BackupPolicy { component := r.getFirstComponent(cluster, policyTPL.ComponentDefRef) if component == nil { return nil } + backupPolicy := &dataprotectionv1alpha1.BackupPolicy{ ObjectMeta: metav1.ObjectMeta{ - Name: DeriveBackupPolicyName(cluster.Name, policyTPL.ComponentDefRef), + Name: backupPolicyName, Namespace: cluster.Namespace, Labels: map[string]string{ constant.AppInstanceLabelKey: cluster.Name, @@ -173,7 +182,7 @@ func (r *BackupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba constant.AppManagedByLabelKey: constant.AppName, }, Annotations: map[string]string{ - constant.DefaultBackupPolicyAnnotationKey: "true", + constant.DefaultBackupPolicyAnnotationKey: r.defaultPolicyAnnotationValue(), constant.BackupPolicyTemplateAnnotationKey: tplName, constant.BackupDataPathPrefixAnnotationKey: fmt.Sprintf("/%s-%s/%s", cluster.Name, cluster.UID, component.Name), }, @@ -350,9 +359,19 @@ func (r *BackupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.Common } } +func (r *BackupPolicyTPLTransformer) defaultPolicyAnnotationValue() string { + if r.tplCount > 1 && r.isDefaultTemplate != "true" { + return "false" + } + return "true" +} + // DeriveBackupPolicyName generates the backup policy name which is created from backup policy template. -func DeriveBackupPolicyName(clusterName, componentDef string) string { - return fmt.Sprintf("%s-%s-backup-policy", clusterName, componentDef) +func DeriveBackupPolicyName(clusterName, componentDef, identifier string) string { + if len(identifier) == 0 { + return fmt.Sprintf("%s-%s-backup-policy", clusterName, componentDef) + } + return fmt.Sprintf("%s-%s-backup-policy-%s", clusterName, componentDef, identifier) } var _ graph.Transformer = &BackupPolicyTPLTransformer{} From 87682121858ef864ea1568ca46e0b36819105f15 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Fri, 28 Apr 2023 10:25:17 +0800 Subject: [PATCH 201/439] fix: kbcli clean SA on uninstallation (#2998) --- .../admission/webhookconfiguration.yaml | 6 +- internal/cli/cmd/kubeblocks/install.go | 6 +- .../cli/cmd/kubeblocks/kubeblocks_objects.go | 90 ++++++++---------- .../cmd/kubeblocks/kubeblocks_objects_test.go | 92 ++++++++++++++++-- internal/cli/cmd/kubeblocks/status.go | 28 +++++- internal/cli/cmd/kubeblocks/uninstall.go | 2 +- internal/cli/cmd/kubeblocks/util.go | 18 +++- internal/cli/testing/fake.go | 94 +++++++++++++++++++ internal/cli/types/types.go | 15 +++ 9 files changed, 285 insertions(+), 66 deletions(-) diff --git a/deploy/helm/templates/admission/webhookconfiguration.yaml b/deploy/helm/templates/admission/webhookconfiguration.yaml index 25bad2588..89966ace7 100644 --- a/deploy/helm/templates/admission/webhookconfiguration.yaml +++ b/deploy/helm/templates/admission/webhookconfiguration.yaml @@ -8,7 +8,7 @@ kind: Secret metadata: name: {{ include "kubeblocks.fullname" . }}.{{ .Release.Namespace }}.svc.tls-ca labels: - {{- include "kubeblocks.selectorLabels" . | nindent 4 }} + {{- include "kubeblocks.labels" . | nindent 4 }} annotations: self-signed-cert: "true" type: kubernetes.io/tls @@ -33,6 +33,8 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: {{ include "kubeblocks.fullname" . }}-mutating-webhook-configuration + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} webhooks: - admissionReviewVersions: - v1 @@ -87,6 +89,8 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: {{ include "kubeblocks.fullname" . }}-validating-webhook-configuration + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} webhooks: - admissionReviewVersions: - v1 diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index 6f37a8459..c70515e0d 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -80,9 +80,9 @@ type InstallOptions struct { var ( installExample = templates.Examples(` - # Install KubeBlocks, the default version is same with the kbcli version, the default namespace is kb-system + # Install KubeBlocks, the default version is same with the kbcli version, the default namespace is kb-system kbcli kubeblocks install - + # Install KubeBlocks with specified version kbcli kubeblocks install --version=0.4.0 @@ -222,7 +222,7 @@ func (o *InstallOptions) waitAddonsEnabled() error { checkAddons := func() (bool, error) { allEnabled := true objects, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ - LabelSelector: buildAddonLabelSelector(), + LabelSelector: buildKubeBlocksSelectorLabels(), }) if err != nil && !apierrors.IsNotFound(err) { return false, err diff --git a/internal/cli/cmd/kubeblocks/kubeblocks_objects.go b/internal/cli/cmd/kubeblocks/kubeblocks_objects.go index f9ee5fcd3..155b1d9cd 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks_objects.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks_objects.go @@ -21,7 +21,6 @@ package kubeblocks import ( "context" - "fmt" "strings" corev1 "k8s.io/api/core/v1" @@ -41,9 +40,17 @@ import ( "github.com/apecloud/kubeblocks/internal/constant" ) +type resourceScope string + +const ( + ResourceScopeGlobal resourceScope = "global" + ResourceScopeLocal resourceScope = "namespaced" +) + type kbObjects map[schema.GroupVersionResource]*unstructured.UnstructuredList var ( + // addon resources resourceGVRs = []schema.GroupVersionResource{ types.DeployGVR(), types.StatefulSetGVR(), @@ -96,52 +103,20 @@ func getKBObjects(dynamic dynamic.Interface, namespace string, addons []*extensi } } - getWebhooks := func(gvr schema.GroupVersionResource) { - objs, err := dynamic.Resource(gvr).List(ctx, metav1.ListOptions{}) - if err != nil { - appendErr(err) - return - } - result := &unstructured.UnstructuredList{} - for _, obj := range objs.Items { - if !strings.Contains(obj.GetName(), strings.ToLower(types.KubeBlocksName)) { - continue - } - result.Items = append(result.Items, obj) - } - kbObjs[gvr] = result - } - getWebhooks(types.ValidatingWebhookConfigurationGVR()) - getWebhooks(types.MutatingWebhookConfigurationGVR()) - - // get cluster roles and cluster role bindings by label - getRBACResources := func(labelSelector string, gvr schema.GroupVersionResource, global bool) { + getObjectsByLabels := func(labelSelector string, gvr schema.GroupVersionResource, scope resourceScope) { ns := namespace - if global { + if scope == ResourceScopeGlobal { ns = metav1.NamespaceAll } - objs, err := dynamic.Resource(gvr).Namespace(ns).List(context.TODO(), metav1.ListOptions{ - LabelSelector: labelSelector, - }) - if err != nil { - appendErr(err) - return - } - if _, ok := kbObjs[gvr]; !ok { - kbObjs[gvr] = &unstructured.UnstructuredList{} + if klog.V(1).Enabled() { + klog.Infof("search objects by labels, namespace: %s, name: %s, gvr: %s", labelSelector, gvr, scope) } - target := kbObjs[gvr] - target.Items = append(target.Items, objs.Items...) - } - getRBACResources(buildAddonLabelSelector(), types.ClusterRoleGVR(), true) - getRBACResources(buildAddonLabelSelector(), types.ClusterRoleBindingGVR(), true) - // get objects by label selector - getObjects := func(labelSelector string, gvr schema.GroupVersionResource) { - objs, err := dynamic.Resource(gvr).Namespace(namespace).List(context.TODO(), metav1.ListOptions{ + objs, err := dynamic.Resource(gvr).Namespace(ns).List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, }) + if err != nil { appendErr(err) return @@ -152,10 +127,18 @@ func getKBObjects(dynamic dynamic.Interface, namespace string, addons []*extensi } target := kbObjs[gvr] target.Items = append(target.Items, objs.Items...) + if klog.V(1).Enabled() { + for _, item := range objs.Items { + klog.Infof("\tget object: %s, %s, %s", item.GetNamespace(), item.GetKind(), item.GetName()) + } + } } // get object by name - getObject := func(name string, gvr schema.GroupVersionResource) { + getObjectByName := func(name string, gvr schema.GroupVersionResource) { + if klog.V(1).Enabled() { + klog.Infof("search object by name, namespace: %s, name: %s, gvr: %s ", namespace, name, gvr) + } obj, err := dynamic.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { appendErr(err) @@ -166,23 +149,31 @@ func getKBObjects(dynamic dynamic.Interface, namespace string, addons []*extensi } target := kbObjs[gvr] target.Items = append(target.Items, *obj) + if klog.V(1).Enabled() { + klog.Infof("\tget object: %s, %s, %s", obj.GetNamespace(), obj.GetKind(), obj.GetName()) + } } + // get RBAC resources, such as ClusterRole, ClusterRoleBinding, Role, RoleBinding, ServiceAccount + getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.ClusterRoleGVR(), ResourceScopeGlobal) + getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.ClusterRoleBindingGVR(), ResourceScopeGlobal) + getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.RoleGVR(), ResourceScopeLocal) + getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.RoleBindingGVR(), ResourceScopeLocal) + getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.ServiceAccountGVR(), ResourceScopeLocal) + // get webhooks + getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.ValidatingWebhookConfigurationGVR(), ResourceScopeGlobal) + getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.MutatingWebhookConfigurationGVR(), ResourceScopeGlobal) + // get configmap for config template + getObjectsByLabels(buildConfigTypeSelectorLabels(), types.ConfigmapGVR(), ResourceScopeLocal) + getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.ConfigmapGVR(), ResourceScopeLocal) // get resources which label matches app.kubernetes.io/instance=kubeblocks or // label matches release=kubeblocks, like prometheus-server for _, selector := range buildResourceLabelSelectors(addons) { for _, gvr := range resourceGVRs { - getObjects(selector, gvr) + getObjectsByLabels(selector, gvr, ResourceScopeLocal) } } - // build label selector - configMapLabelSelector := fmt.Sprintf("%s=%s", constant.CMConfigurationTypeLabelKey, constant.ConfigTemplateType) - - // get configmap - getObjects(configMapLabelSelector, types.ConfigmapGVR()) - getObjects(buildAddonLabelSelector(), types.ConfigmapGVR()) - // get PVs by PVC if pvcs, ok := kbObjs[types.PVCGVR()]; ok { for _, obj := range pvcs.Items { @@ -191,10 +182,9 @@ func getKBObjects(dynamic dynamic.Interface, namespace string, addons []*extensi appendErr(err) continue } - getObject(pvc.Spec.VolumeName, types.PVGVR()) + getObjectByName(pvc.Spec.VolumeName, types.PVGVR()) } } - return kbObjs, utilerrors.NewAggregate(allErrs) } diff --git a/internal/cli/cmd/kubeblocks/kubeblocks_objects_test.go b/internal/cli/cmd/kubeblocks/kubeblocks_objects_test.go index 692527304..6bbc14e7f 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks_objects_test.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks_objects_test.go @@ -20,21 +20,27 @@ along with this program. If not, see . package kubeblocks import ( + "context" + + "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/dynamic" + "k8s.io/apimachinery/pkg/runtime/schema" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dpv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/constant" ) var _ = Describe("kubeblocks objects", func() { + It("delete objects", func() { dynamic := testing.FakeDynamicClient() Expect(deleteObjects(dynamic, types.DeployGVR(), nil)).Should(Succeed()) @@ -95,20 +101,65 @@ var _ = Describe("kubeblocks objects", func() { } for _, c := range testCases { - client := mockDynamicClientWithCRD(c.clusterDef, c.clusterVersion, c.backupTool) + objects := mockCRD() + objects = append(objects, testing.FakeVolumeSnapshotClass()) + objects = append(objects, c.clusterDef, c.clusterVersion, c.backupTool) + client := testing.FakeDynamicClient(objects...) objs, _ := getKBObjects(client, "", nil) Expect(removeCustomResources(client, objs)).Should(Succeed()) } }) It("delete crd", func() { - dynamic := mockDynamicClientWithCRD() + objects := mockCRD() + objects = append(objects, testing.FakeVolumeSnapshotClass()) + dynamic := testing.FakeDynamicClient(objects...) objs, _ := getKBObjects(dynamic, "", nil) Expect(deleteObjects(dynamic, types.CRDGVR(), objs[types.CRDGVR()])).Should(Succeed()) }) + + It("test getKBObjects", func() { + objects := mockCRD() + objects = append(objects, mockCRs()...) + objects = append(objects, testing.FakeVolumeSnapshotClass()) + objects = append(objects, mockRBACResources()...) + objects = append(objects, mockConfigMaps()...) + dynamic := testing.FakeDynamicClient(objects...) + objs, _ := getKBObjects(dynamic, "", nil) + + tmp, err := dynamic.Resource(types.ClusterRoleGVR()).Namespace(metav1.NamespaceAll). + List(context.TODO(), metav1.ListOptions{LabelSelector: buildKubeBlocksSelectorLabels()}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(tmp.Items).Should(HaveLen(1)) + // verify crds + Expect(objs[types.CRDGVR()].Items).Should(HaveLen(4)) + // verify crs + for _, gvr := range []schema.GroupVersionResource{types.ClusterDefGVR(), types.ClusterVersionGVR()} { + objlist, ok := objs[gvr] + Expect(ok).Should(BeTrue()) + Expect(objlist.Items).Should(HaveLen(1)) + } + + // verify rbac info + for _, gvr := range []schema.GroupVersionResource{types.RoleGVR(), types.ClusterRoleBindingGVR(), types.ServiceAccountGVR()} { + objlist, ok := objs[gvr] + Expect(ok).Should(BeTrue()) + Expect(objlist.Items).Should(HaveLen(1), gvr.String()) + } + // verify cofnig tpl + for _, gvr := range []schema.GroupVersionResource{types.ConfigmapGVR()} { + objlist, ok := objs[gvr] + Expect(ok).Should(BeTrue()) + Expect(objlist.Items).Should(HaveLen(1), gvr.String()) + } + }) }) -func mockDynamicClientWithCRD(objects ...runtime.Object) dynamic.Interface { +func mockName() string { + return uuid.NewString() +} + +func mockCRD() []runtime.Object { clusterCRD := v1.CustomResourceDefinition{ TypeMeta: metav1.TypeMeta{ Kind: "CustomResourceDefinition", @@ -162,9 +213,34 @@ func mockDynamicClientWithCRD(objects ...runtime.Object) dynamic.Interface { }, Status: v1.CustomResourceDefinitionStatus{}, } + return []runtime.Object{&clusterCRD, &clusterDefCRD, &clusterVersionCRD, &backupToolCRD} +} + +func mockCRs() []runtime.Object { + allObjects := make([]runtime.Object, 0) + allObjects = append(allObjects, testing.FakeClusterDef()) + allObjects = append(allObjects, testing.FakeClusterVersion()) + return allObjects +} + +func mockRBACResources() []runtime.Object { + sa := testing.FakeServiceAccount(mockName()) - allObjs := []runtime.Object{&clusterCRD, &clusterDefCRD, &clusterVersionCRD, &backupToolCRD, - testing.FakeVolumeSnapshotClass()} - allObjs = append(allObjs, objects...) - return testing.FakeDynamicClient(allObjs...) + cluserRole := testing.FakeClusterRole(mockName()) + cluserRoleBinding := testing.FakeClusterRoleBinding(mockName(), sa, cluserRole) + + role := testing.FakeRole(mockName()) + roleBinding := testing.FakeRoleBinding(mockName(), sa, role) + + return []runtime.Object{sa, cluserRole, cluserRoleBinding, role, roleBinding} +} + +func mockConfigMaps() []runtime.Object { + obj := testing.FakeConfigMap(mockName()) + // add a config tpl label + if obj.ObjectMeta.Labels == nil { + obj.ObjectMeta.Labels = make(map[string]string) + } + obj.ObjectMeta.Labels[constant.CMConfigurationTypeLabelKey] = constant.ConfigTemplateType + return []runtime.Object{obj} } diff --git a/internal/cli/cmd/kubeblocks/status.go b/internal/cli/cmd/kubeblocks/status.go index 028047135..503f431dd 100644 --- a/internal/cli/cmd/kubeblocks/status.go +++ b/internal/cli/cmd/kubeblocks/status.go @@ -43,6 +43,7 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" "k8s.io/metrics/pkg/apis/metrics/v1beta1" @@ -53,7 +54,7 @@ var ( infoExample = templates.Examples(` # list workloads owned by KubeBlocks kbcli kubeblocks status - + # list all resources owned by KubeBlocks, such as workloads, cluster definitions, backup template. kbcli kubeblocks status --all`) ) @@ -79,6 +80,14 @@ var ( types.ServiceGVR(), } + kubeBlocksRBAC = []schema.GroupVersionResource{ + types.ClusterRoleGVR(), + types.ClusterRoleBindingGVR(), + types.RoleGVR(), + types.RoleBindingGVR(), + types.ServiceAccountGVR(), + } + kubeBlocksStorages = []schema.GroupVersionResource{ types.PVCGVR(), } @@ -161,6 +170,7 @@ func (o *statusOptions) run() error { if o.showAll { o.showKubeBlocksResources(ctx, &allErrs) o.showKubeBlocksConfig(ctx, &allErrs) + o.showKubeBlocksRBAC(ctx, &allErrs) o.showKubeBlocksStorage(ctx, &allErrs) o.showHelmResources(ctx, &allErrs) } @@ -229,6 +239,19 @@ func (o *statusOptions) showKubeBlocksConfig(ctx context.Context, allErrs *[]err tblPrinter.Print() } +func (o *statusOptions) showKubeBlocksRBAC(ctx context.Context, allErrs *[]error) { + fmt.Fprintln(o.Out, "\nKubeBlocks RBAC:") + tblPrinter := printer.NewTablePrinter(o.Out) + tblPrinter.SetHeader("NAMESPACE", "KIND", "NAME") + unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksRBAC, selectorList, allErrs) + for _, resourceList := range unstructuredList { + for _, resource := range resourceList.Items { + tblPrinter.AddRow(resource.GetNamespace(), resource.GetKind(), resource.GetName()) + } + } + tblPrinter.Print() +} + func (o *statusOptions) showKubeBlocksStorage(ctx context.Context, allErrs *[]error) { fmt.Fprintln(o.Out, "\nKubeBlocks Storage:") tblPrinter := printer.NewTablePrinter(o.Out) @@ -374,6 +397,9 @@ func listResourceByGVR(ctx context.Context, client dynamic.Interface, namespace unstructuredList := make([]*unstructured.UnstructuredList, 0) for _, gvr := range gvrlist { for _, labelSelector := range selector { + if klog.V(1).Enabled() { + klog.Infof("listResourceByGVR: namespace=%s, gvrlist=%v, selector=%v", namespace, gvr, labelSelector) + } resource, err := client.Resource(gvr).Namespace(namespace).List(ctx, labelSelector) if err != nil { appendErrIgnoreNotFound(allErrs, err) diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index 1629ee7ba..c0c254e32 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -255,7 +255,7 @@ func (o *UninstallOptions) uninstallAddons() error { processAddons := func(processFn func(addon *extensionsv1alpha1.Addon) error) ([]*extensionsv1alpha1.Addon, error) { var addons []*extensionsv1alpha1.Addon objects, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ - LabelSelector: buildAddonLabelSelector(), + LabelSelector: buildKubeBlocksSelectorLabels(), }) if err != nil && !apierrors.IsNotFound(err) { klog.V(1).Infof("Failed to get KubeBlocks addons %s", err.Error()) diff --git a/internal/cli/cmd/kubeblocks/util.go b/internal/cli/cmd/kubeblocks/util.go index a73834256..fb320939e 100644 --- a/internal/cli/cmd/kubeblocks/util.go +++ b/internal/cli/cmd/kubeblocks/util.go @@ -141,9 +141,23 @@ func buildResourceLabelSelectors(addons []*extensionsv1alpha1.Addon) []string { return selectors } -// buildAddonLabelSelector builds labelSelector that can be used to get all build-in addons -func buildAddonLabelSelector() string { +// buildAddonLabelSelector builds labelSelector that can be used to get all kubeBlocks resources, +// including CRDs, addons (but not resources created by addons). +// and it should be consistent with the labelSelectors defined in chart. +// for example: +// {{- define "kubeblocks.selectorLabels" -}} +// app.kubernetes.io/name: {{ include "kubeblocks.name" . }} +// app.kubernetes.io/instance: {{ .Release.Name }} +// {{- end }} +func buildKubeBlocksSelectorLabels() string { return fmt.Sprintf("%s=%s,%s=%s", constant.AppInstanceLabelKey, types.KubeBlocksReleaseName, constant.AppNameLabelKey, types.KubeBlocksChartName) } + +// buildConfig builds labelSelector that can be used to get all configmaps that are used to store config templates. +// and it should be consistent with the labelSelectors defined used +// in `configuration.updateConfigMapFinalizerImpl`. +func buildConfigTypeSelectorLabels() string { + return fmt.Sprintf("%s=%s", constant.CMConfigurationTypeLabelKey, constant.ConfigTemplateType) +} diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index 17c3e0ec8..e3ea6f463 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -27,6 +27,7 @@ import ( "github.com/sethvargo/go-password/password" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -552,3 +553,96 @@ func FakeStorageClass(name string, isDefault bool) *storagev1.StorageClass { } return storageClassObj } + +func FakeServiceAccount(name string) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: Namespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + constant.AppNameLabelKey: KubeBlocksChartName}, + }, + } +} + +func FakeClusterRole(name string) *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + constant.AppNameLabelKey: KubeBlocksChartName}, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"*"}, + Resources: []string{"*"}, + Verbs: []string{"*"}, + }, + }, + } +} + +func FakeClusterRoleBinding(name string, sa *corev1.ServiceAccount, clusterRole *rbacv1.ClusterRole) *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + constant.AppNameLabelKey: KubeBlocksChartName}, + }, + RoleRef: rbacv1.RoleRef{ + Kind: clusterRole.Kind, + Name: clusterRole.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: sa.Name, + Namespace: sa.Namespace, + }, + }, + } +} + +func FakeRole(name string) *rbacv1.Role { + return &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + constant.AppNameLabelKey: KubeBlocksChartName}, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"*"}, + Resources: []string{"*"}, + Verbs: []string{"*"}, + }, + }, + } +} + +func FakeRoleBinding(name string, sa *corev1.ServiceAccount, role *rbacv1.Role) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: Namespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + constant.AppNameLabelKey: KubeBlocksChartName}, + }, + RoleRef: rbacv1.RoleRef{ + Kind: role.Kind, + Name: role.Name, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: sa.Name, + Namespace: sa.Namespace, + }, + }, + } +} diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index f27622abd..29e4a7079 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -94,6 +94,9 @@ const ( RBACAPIVersion = "v1" ClusterRoles = "clusterroles" ClusterRoleBindings = "clusterrolebindings" + Roles = "roles" + RoleBindings = "rolebindings" + ServiceAccounts = "serviceaccounts" ) // Annotations @@ -310,6 +313,18 @@ func ClusterRoleBindingGVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: RBACAPIGroup, Version: RBACAPIVersion, Resource: ClusterRoleBindings} } +func RoleGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: RBACAPIGroup, Version: RBACAPIVersion, Resource: Roles} +} + +func RoleBindingGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: RBACAPIGroup, Version: RBACAPIVersion, Resource: RoleBindings} +} + +func ServiceAccountGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: corev1.GroupName, Version: K8sCoreAPIVersion, Resource: ServiceAccounts} +} + func MigrationTaskGVR() schema.GroupVersionResource { return schema.GroupVersionResource{ Group: MigrationAPIGroup, From fbd2bcf46dfad947753eec2ab29a9c44f32f4c90 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Fri, 28 Apr 2023 10:52:06 +0800 Subject: [PATCH 202/439] chore: fix cluster not displays ConditionsError status on list cluster cmd when status of ApplyResources or ProvisioningStarted condition is "False" (#3005) --- internal/cli/cluster/cluster.go | 11 ++++++++ internal/cli/cmd/cluster/list_test.go | 14 +++++++--- .../lifecycle/cluster_plan_builder.go | 2 +- .../lifecycle/cluster_status_conditions.go | 28 +++++++++++++------ 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/internal/cli/cluster/cluster.go b/internal/cli/cluster/cluster.go index 6695f145c..161497b48 100644 --- a/internal/cli/cluster/cluster.go +++ b/internal/cli/cluster/cluster.go @@ -41,6 +41,9 @@ import ( "github.com/apecloud/kubeblocks/internal/constant" ) +// ConditionsError cluster will display this status on list cmd when the status of ApplyResources or ProvisioningStarted condition is not "True". +const ConditionsError = "ConditionsError" + type GetOptions struct { WithClusterDef bool WithClusterVersion bool @@ -119,7 +122,15 @@ func (o *ObjectsGetter) Get() (*ClusterObjects, error) { if latestOpsProcessedCondition != nil && latestOpsProcessedCondition.Status == metav1.ConditionFalse { objs.Cluster.Status.Phase = appsv1alpha1.ClusterPhase(latestOpsProcessedCondition.Reason) } + provisionCondition := meta.FindStatusCondition(objs.Cluster.Status.Conditions, appsv1alpha1.ConditionTypeProvisioningStarted) + if provisionCondition != nil && provisionCondition.Status == metav1.ConditionFalse { + objs.Cluster.Status.Phase = ConditionsError + } + applyResourcesCondition := meta.FindStatusCondition(objs.Cluster.Status.Conditions, appsv1alpha1.ConditionTypeApplyResources) + if applyResourcesCondition != nil && applyResourcesCondition.Status == metav1.ConditionFalse { + objs.Cluster.Status.Phase = ConditionsError + } // get cluster definition if o.WithClusterDef { cd := &appsv1alpha1.ClusterDefinition{} diff --git a/internal/cli/cmd/cluster/list_test.go b/internal/cli/cmd/cluster/list_test.go index 5cbd7ebfd..33fc048a7 100644 --- a/internal/cli/cmd/cluster/list_test.go +++ b/internal/cli/cmd/cluster/list_test.go @@ -37,6 +37,7 @@ import ( cmdtesting "k8s.io/kubectl/pkg/cmd/testing" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/cluster" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" ) @@ -61,8 +62,12 @@ var _ = Describe("list", func() { _ = appsv1alpha1.AddToScheme(scheme.Scheme) codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) - cluster := testing.FakeCluster(clusterName, namespace) - clusterWithCondition := testing.FakeCluster(clusterName1, namespace, metav1.Condition{ + cluster := testing.FakeCluster(clusterName, namespace, metav1.Condition{ + Type: appsv1alpha1.ConditionTypeApplyResources, + Status: metav1.ConditionFalse, + Reason: "HorizontalScaleFailed", + }) + clusterWithVerticalScaling := testing.FakeCluster(clusterName1, namespace, metav1.Condition{ Type: appsv1alpha1.ConditionTypeLatestOpsRequestProcessed, Status: metav1.ConditionFalse, Reason: verticalScalingReason, @@ -80,7 +85,7 @@ var _ = Describe("list", func() { return map[string]*http.Response{ "/namespaces/" + namespace + "/clusters": httpResp(&appsv1alpha1.ClusterList{Items: []appsv1alpha1.Cluster{*cluster}}), "/namespaces/" + namespace + "/clusters/" + clusterName: httpResp(cluster), - "/namespaces/" + namespace + "/clusters/" + clusterName1: httpResp(clusterWithCondition), + "/namespaces/" + namespace + "/clusters/" + clusterName1: httpResp(clusterWithVerticalScaling), "/namespaces/" + namespace + "/secrets": httpResp(testing.FakeSecrets(namespace, clusterName)), "/api/v1/nodes/" + testing.NodeName: httpResp(testing.FakeNode()), urlPrefix + "/services": httpResp(&corev1.ServiceList{}), @@ -92,7 +97,7 @@ var _ = Describe("list", func() { } tf.Client = tf.UnstructuredClient - tf.FakeDynamicClient = testing.FakeDynamicClient(cluster, clusterWithCondition, testing.FakeClusterDef(), testing.FakeClusterVersion()) + tf.FakeDynamicClient = testing.FakeDynamicClient(cluster, clusterWithVerticalScaling, testing.FakeClusterDef(), testing.FakeClusterVersion()) }) AfterEach(func() { @@ -108,6 +113,7 @@ var _ = Describe("list", func() { cmd.Run(cmd, []string{clusterName1}) Expect(out.String()).Should(ContainSubstring(verticalScalingReason)) + Expect(out.String()).Should(ContainSubstring(cluster.ConditionsError)) }) It("list instances", func() { diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index c99a32320..0aa6565d0 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -175,7 +175,7 @@ func (p *clusterPlan) Execute() error { } func (p *clusterPlan) handlePlanExecutionError(err error) error { - condition := newFailedApplyResourcesCondition(err.Error()) + condition := newFailedApplyResourcesCondition(err) meta.SetStatusCondition(&p.transCtx.Cluster.Status.Conditions, condition) sendWaringEventWithError(p.transCtx.GetRecorder(), p.transCtx.Cluster, ReasonApplyResourcesFailed, err) return p.cli.Status().Patch(p.transCtx.Context, p.transCtx.Cluster, client.MergeFrom(p.transCtx.OrigCluster.DeepCopy())) diff --git a/internal/controller/lifecycle/cluster_status_conditions.go b/internal/controller/lifecycle/cluster_status_conditions.go index 79060ad2a..9345ad75a 100644 --- a/internal/controller/lifecycle/cluster_status_conditions.go +++ b/internal/controller/lifecycle/cluster_status_conditions.go @@ -30,6 +30,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) const ( @@ -56,7 +57,7 @@ func conditionIsChanged(oldCondition *metav1.Condition, newCondition metav1.Cond func setProvisioningStartedCondition(conditions *[]metav1.Condition, clusterName string, clusterGeneration int64, err error) { condition := newProvisioningStartedCondition(clusterName, clusterGeneration) if err != nil { - condition = newFailedProvisioningStartedCondition(err.Error(), ReasonPreCheckFailed) + condition = newFailedProvisioningStartedCondition(err) } meta.SetStatusCondition(conditions, condition) } @@ -72,20 +73,31 @@ func newProvisioningStartedCondition(clusterName string, clusterGeneration int64 } } +func getConditionReasonWithError(defaultReason string, err error) string { + if err == nil { + return defaultReason + } + controllerErr := intctrlutil.ToControllerError(err) + if controllerErr != nil { + defaultReason = string(controllerErr.Type) + } + return defaultReason +} + // newApplyResourcesCondition creates a condition when applied resources succeed. -func newFailedProvisioningStartedCondition(message, reason string) metav1.Condition { +func newFailedProvisioningStartedCondition(err error) metav1.Condition { return metav1.Condition{ Type: appsv1alpha1.ConditionTypeProvisioningStarted, Status: metav1.ConditionFalse, - Message: message, - Reason: reason, + Message: err.Error(), + Reason: getConditionReasonWithError(ReasonPreCheckFailed, err), } } func setApplyResourceCondition(conditions *[]metav1.Condition, clusterGeneration int64, err error) { condition := newApplyResourcesCondition(clusterGeneration) if err != nil { - condition = newFailedApplyResourcesCondition(err.Error()) + condition = newFailedApplyResourcesCondition(err) } meta.SetStatusCondition(conditions, condition) } @@ -102,12 +114,12 @@ func newApplyResourcesCondition(clusterGeneration int64) metav1.Condition { } // newApplyResourcesCondition creates a condition when applied resources succeed. -func newFailedApplyResourcesCondition(message string) metav1.Condition { +func newFailedApplyResourcesCondition(err error) metav1.Condition { return metav1.Condition{ Type: appsv1alpha1.ConditionTypeApplyResources, Status: metav1.ConditionFalse, - Message: message, - Reason: ReasonApplyResourcesFailed, + Message: err.Error(), + Reason: getConditionReasonWithError(ReasonApplyResourcesFailed, err), } } From 36d38e21b4c5dc49ee17f60ecfe25801cc755aca Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Fri, 28 Apr 2023 11:04:42 +0800 Subject: [PATCH 203/439] chore: have addon's toleration sync with operator's settings (#3004) --- .../alertmanager-webhook-adaptor-addon.yaml | 3 ++ .../aws-loadbalancer-controller-addon.yaml | 3 ++ .../templates/addons/chaos-mesh-addon.yaml | 5 ++- .../templates/addons/csi-driver-addon.yaml | 9 +++++ .../helm/templates/addons/grafana-addon.yaml | 6 ++++ .../templates/addons/migration-addon.yaml | 6 ++++ .../helm/templates/addons/nyancat-addon.yaml | 5 ++- .../templates/addons/prometheus-addon.yaml | 21 +++++++++++ .../addons/snapshot-controller-addon.yaml | 35 ++++++++++++++----- 9 files changed, 83 insertions(+), 10 deletions(-) diff --git a/deploy/helm/templates/addons/alertmanager-webhook-adaptor-addon.yaml b/deploy/helm/templates/addons/alertmanager-webhook-adaptor-addon.yaml index de8ebd43f..56fd5a879 100644 --- a/deploy/helm/templates/addons/alertmanager-webhook-adaptor-addon.yaml +++ b/deploy/helm/templates/addons/alertmanager-webhook-adaptor-addon.yaml @@ -39,6 +39,9 @@ spec: defaultInstallValues: - replicas: 1 + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} installable: autoInstall: {{ .Values.prometheus.enabled }} diff --git a/deploy/helm/templates/addons/aws-loadbalancer-controller-addon.yaml b/deploy/helm/templates/addons/aws-loadbalancer-controller-addon.yaml index a0786248a..066470ebd 100644 --- a/deploy/helm/templates/addons/aws-loadbalancer-controller-addon.yaml +++ b/deploy/helm/templates/addons/aws-loadbalancer-controller-addon.yaml @@ -41,6 +41,9 @@ spec: defaultInstallValues: - replicas: 1 + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} installable: autoInstall: {{ index .Values "aws-load-balancer-controller" "enabled" }} diff --git a/deploy/helm/templates/addons/chaos-mesh-addon.yaml b/deploy/helm/templates/addons/chaos-mesh-addon.yaml index 4820bf956..7adc40b49 100644 --- a/deploy/helm/templates/addons/chaos-mesh-addon.yaml +++ b/deploy/helm/templates/addons/chaos-mesh-addon.yaml @@ -33,4 +33,7 @@ spec: autoInstall: false defaultInstallValues: - - enabled: false \ No newline at end of file + - enabled: false + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} \ No newline at end of file diff --git a/deploy/helm/templates/addons/csi-driver-addon.yaml b/deploy/helm/templates/addons/csi-driver-addon.yaml index 01705ca90..fbf2113b0 100644 --- a/deploy/helm/templates/addons/csi-driver-addon.yaml +++ b/deploy/helm/templates/addons/csi-driver-addon.yaml @@ -40,6 +40,15 @@ spec: defaultInstallValues: - enabled: false + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + extras: + - name: node + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + installable: autoInstall: {{ get ( get ( .Values | toYaml | fromYaml ) "kubeblocks-csi-driver" ) "enabled" }} diff --git a/deploy/helm/templates/addons/grafana-addon.yaml b/deploy/helm/templates/addons/grafana-addon.yaml index beced7956..2083a5692 100644 --- a/deploy/helm/templates/addons/grafana-addon.yaml +++ b/deploy/helm/templates/addons/grafana-addon.yaml @@ -45,6 +45,9 @@ spec: resources: requests: storage: 1Gi + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} - selectors: - key: KubeGitVersion @@ -56,6 +59,9 @@ spec: resources: requests: storage: 20Gi + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} installable: autoInstall: {{ .Values.grafana.enabled }} diff --git a/deploy/helm/templates/addons/migration-addon.yaml b/deploy/helm/templates/addons/migration-addon.yaml index e324816a7..71b02a00c 100644 --- a/deploy/helm/templates/addons/migration-addon.yaml +++ b/deploy/helm/templates/addons/migration-addon.yaml @@ -17,8 +17,14 @@ spec: helm: chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/dt-platform-0.1.0-test.3.tgz + # # missing values mapping!! + # valuesMapping: + installable: autoInstall: false defaultInstallValues: - enabled: true + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} diff --git a/deploy/helm/templates/addons/nyancat-addon.yaml b/deploy/helm/templates/addons/nyancat-addon.yaml index bf28d5cf1..d77f27244 100644 --- a/deploy/helm/templates/addons/nyancat-addon.yaml +++ b/deploy/helm/templates/addons/nyancat-addon.yaml @@ -33,7 +33,10 @@ spec: limits: resources.limits.memory defaultInstallValues: - - replicas: 2 + - replicas: 1 + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} installable: autoInstall: false diff --git a/deploy/helm/templates/addons/prometheus-addon.yaml b/deploy/helm/templates/addons/prometheus-addon.yaml index 56c0f2a23..294165901 100644 --- a/deploy/helm/templates/addons/prometheus-addon.yaml +++ b/deploy/helm/templates/addons/prometheus-addon.yaml @@ -65,12 +65,19 @@ spec: memory: 512Mi limits: memory: 4Gi + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + extras: - name: alertmanager replicas: 1 resources: requests: storage: 4Gi + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} # for ACK, the smallest storage size is 20Gi, the format of GitVersion is v1.24.6-aliyun.1 - selectors: @@ -86,6 +93,10 @@ spec: memory: 512Mi limits: memory: 4Gi + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + extras: - name: alertmanager replicas: 1 @@ -93,6 +104,9 @@ spec: resources: requests: storage: 20Gi + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} # for TKE, the smallest storage size is 10Gi, the format of GitVersion is v1.24.4-tke.5 - selectors: @@ -107,12 +121,19 @@ spec: memory: 512Mi limits: memory: 4Gi + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + extras: - name: alertmanager replicas: 1 resources: requests: storage: 10Gi + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} installable: autoInstall: {{ .Values.prometheus.enabled }} \ No newline at end of file diff --git a/deploy/helm/templates/addons/snapshot-controller-addon.yaml b/deploy/helm/templates/addons/snapshot-controller-addon.yaml index ba4be98ee..db3851237 100644 --- a/deploy/helm/templates/addons/snapshot-controller-addon.yaml +++ b/deploy/helm/templates/addons/snapshot-controller-addon.yaml @@ -41,30 +41,49 @@ spec: defaultInstallValues: - enabled: {{ get ( get ( .Values | toYaml | fromYaml ) "snapshot-controller" ) "enabled" }} - - storageClass: ebs.csi.aws.com - selectors: + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + + - selectors: - key: KubeGitVersion operator: Contains values: - eks - - storageClass: diskplugin.csi.alibabacloud.com - selectors: + storageClass: ebs.csi.aws.com + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + + - selectors: - key: KubeGitVersion operator: Contains values: - aliyun - - storageClass: pd.csi.storage.gke.io - selectors: + storageClass: diskplugin.csi.alibabacloud.com + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + + - selectors: - key: KubeGitVersion operator: Contains values: - gke - - storageClass: disk.csi.azure.com - selectors: + storageClass: pd.csi.storage.gke.io + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} + + - selectors: - key: KubeGitVersion operator: Contains values: - aks + storageClass: disk.csi.azure.com + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} installable: autoInstall: {{ get ( get ( .Values | toYaml | fromYaml ) "snapshot-controller" ) "enabled" }} From 7b3b8179d1a39ad268b73261a40b334691d83d6f Mon Sep 17 00:00:00 2001 From: "zheyi.cqy" Date: Fri, 28 Apr 2023 14:11:55 +0800 Subject: [PATCH 204/439] chore: migration add-on support toleration and resources (#3011) --- .../helm/templates/addons/migration-addon.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/deploy/helm/templates/addons/migration-addon.yaml b/deploy/helm/templates/addons/migration-addon.yaml index 71b02a00c..b684c3283 100644 --- a/deploy/helm/templates/addons/migration-addon.yaml +++ b/deploy/helm/templates/addons/migration-addon.yaml @@ -15,10 +15,21 @@ spec: type: Helm helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/dt-platform-0.1.0-test.3.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/dt-platform-0.1.0.tgz + valuesMapping: + valueMap: + replicaCount: replicaCount - # # missing values mapping!! - # valuesMapping: + jsonMap: + tolerations: tolerations + + resources: + cpu: + requests: resources.requests.cpu + limits: resources.limits.cpu + memory: + requests: resources.requests.memory + limits: resources.limits.memory installable: autoInstall: false From 6626c5467042c8566f8e7b115f1c76fac67a63cc Mon Sep 17 00:00:00 2001 From: wangyelei Date: Fri, 28 Apr 2023 15:28:14 +0800 Subject: [PATCH 205/439] chore: fix show incorrect cluster status on list cluster cmd (#3013) --- internal/cli/cluster/cluster.go | 10 ++++++---- internal/cli/cmd/cluster/list_test.go | 12 ++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/internal/cli/cluster/cluster.go b/internal/cli/cluster/cluster.go index 161497b48..66268990a 100644 --- a/internal/cli/cluster/cluster.go +++ b/internal/cli/cluster/cluster.go @@ -117,10 +117,12 @@ func (o *ObjectsGetter) Get() (*ClusterObjects, error) { return nil, err } - // wrap the cluster phase if the latest ops request is processing - latestOpsProcessedCondition := meta.FindStatusCondition(objs.Cluster.Status.Conditions, appsv1alpha1.ConditionTypeLatestOpsRequestProcessed) - if latestOpsProcessedCondition != nil && latestOpsProcessedCondition.Status == metav1.ConditionFalse { - objs.Cluster.Status.Phase = appsv1alpha1.ClusterPhase(latestOpsProcessedCondition.Reason) + if objs.Cluster.Status.Phase == appsv1alpha1.SpecReconcilingClusterPhase { + // wrap the cluster phase if the latest ops request is processing + latestOpsProcessedCondition := meta.FindStatusCondition(objs.Cluster.Status.Conditions, appsv1alpha1.ConditionTypeLatestOpsRequestProcessed) + if latestOpsProcessedCondition != nil && latestOpsProcessedCondition.Status == metav1.ConditionFalse { + objs.Cluster.Status.Phase = appsv1alpha1.ClusterPhase(latestOpsProcessedCondition.Reason) + } } provisionCondition := meta.FindStatusCondition(objs.Cluster.Status.Conditions, appsv1alpha1.ConditionTypeProvisioningStarted) if provisionCondition != nil && provisionCondition.Status == metav1.ConditionFalse { diff --git a/internal/cli/cmd/cluster/list_test.go b/internal/cli/cmd/cluster/list_test.go index 33fc048a7..c46310b31 100644 --- a/internal/cli/cmd/cluster/list_test.go +++ b/internal/cli/cmd/cluster/list_test.go @@ -53,6 +53,7 @@ var _ = Describe("list", func() { namespace = "test" clusterName = "test" clusterName1 = "test1" + clusterName2 = "test2" verticalScalingReason = "VerticalScaling" ) @@ -72,6 +73,9 @@ var _ = Describe("list", func() { Status: metav1.ConditionFalse, Reason: verticalScalingReason, }) + clusterWithVerticalScaling.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase + clusterWithAbnormalPhase := testing.FakeCluster(clusterName2, namespace) + clusterWithAbnormalPhase.Status.Phase = appsv1alpha1.AbnormalClusterPhase pods := testing.FakePods(3, namespace, clusterName) httpResp := func(obj runtime.Object) *http.Response { return &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, obj)} @@ -86,6 +90,7 @@ var _ = Describe("list", func() { "/namespaces/" + namespace + "/clusters": httpResp(&appsv1alpha1.ClusterList{Items: []appsv1alpha1.Cluster{*cluster}}), "/namespaces/" + namespace + "/clusters/" + clusterName: httpResp(cluster), "/namespaces/" + namespace + "/clusters/" + clusterName1: httpResp(clusterWithVerticalScaling), + "/namespaces/" + namespace + "/clusters/" + clusterName2: httpResp(clusterWithAbnormalPhase), "/namespaces/" + namespace + "/secrets": httpResp(testing.FakeSecrets(namespace, clusterName)), "/api/v1/nodes/" + testing.NodeName: httpResp(testing.FakeNode()), urlPrefix + "/services": httpResp(&corev1.ServiceList{}), @@ -97,7 +102,7 @@ var _ = Describe("list", func() { } tf.Client = tf.UnstructuredClient - tf.FakeDynamicClient = testing.FakeDynamicClient(cluster, clusterWithVerticalScaling, testing.FakeClusterDef(), testing.FakeClusterVersion()) + tf.FakeDynamicClient = testing.FakeDynamicClient(cluster, clusterWithVerticalScaling, clusterWithAbnormalPhase, testing.FakeClusterDef(), testing.FakeClusterVersion()) }) AfterEach(func() { @@ -108,12 +113,11 @@ var _ = Describe("list", func() { cmd := NewListCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) - cmd.Run(cmd, []string{clusterName}) + cmd.Run(cmd, []string{clusterName, clusterName1, clusterName2}) Expect(out.String()).Should(ContainSubstring(testing.ClusterDefName)) - - cmd.Run(cmd, []string{clusterName1}) Expect(out.String()).Should(ContainSubstring(verticalScalingReason)) Expect(out.String()).Should(ContainSubstring(cluster.ConditionsError)) + Expect(out.String()).Should(ContainSubstring(string(appsv1alpha1.AbnormalClusterPhase))) }) It("list instances", func() { From d046f7b4e20ca021b177745a1208a207936a738a Mon Sep 17 00:00:00 2001 From: xingran Date: Fri, 28 Apr 2023 16:28:40 +0800 Subject: [PATCH 206/439] chore: add postgres with patroni default readiness probe (#3018) --- deploy/postgresql/templates/clusterdefinition.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 75e022071..384b773fa 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -96,6 +96,20 @@ spec: runAsUser: 0 command: - /kb-scripts/setup.sh + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 25 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - /bin/sh + - -c + - -ee + - | + exec pg_isready -U {{ default "postgres" | quote }} -h 127.0.0.1 -p 5432 + [ -f /postgresql/tmp/.initialized ] || [ -f /postgresql/.initialized ] volumeMounts: - name: dshm mountPath: /dev/shm From 2de046de624158afa151217fd66c547e6c0c600b Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Fri, 28 Apr 2023 17:00:58 +0800 Subject: [PATCH 207/439] fix: fix 'addon cmd' toleration value err (#3014) --- docs/user_docs/cli/kbcli_addon_enable.md | 4 +-- internal/cli/cmd/addon/addon.go | 35 ++++++++++++++++++++---- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/user_docs/cli/kbcli_addon_enable.md b/docs/user_docs/cli/kbcli_addon_enable.md index 01559bc56..5d847c991 100644 --- a/docs/user_docs/cli/kbcli_addon_enable.md +++ b/docs/user_docs/cli/kbcli_addon_enable.md @@ -22,8 +22,8 @@ kbcli addon enable ADDON_NAME [flags] --memory alertmanager:16Mi/256Mi --storage: alertmanager:1Gi --replicas alertmanager:2 # Enabled "prometheus" addon with tolerations - kbcli addon enable prometheus --tolerations '[[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]]' \ - --tolerations 'alertmanager:[[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]]' + kbcli addon enable prometheus --tolerations '[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' \ + --tolerations 'alertmanager:[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' # Enabled "prometheus" addon with helm like custom settings kbcli addon enable prometheus --set prometheus.alertmanager.image.tag=v0.24.0 diff --git a/internal/cli/cmd/addon/addon.go b/internal/cli/cmd/addon/addon.go index 9a33d4878..990abd2a6 100644 --- a/internal/cli/cmd/addon/addon.go +++ b/internal/cli/cmd/addon/addon.go @@ -179,8 +179,8 @@ func newEnableCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra --memory alertmanager:16Mi/256Mi --storage: alertmanager:1Gi --replicas alertmanager:2 # Enabled "prometheus" addon with tolerations - kbcli addon enable prometheus --tolerations '[[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]]' \ - --tolerations 'alertmanager:[[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]]' + kbcli addon enable prometheus --tolerations '[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' \ + --tolerations 'alertmanager:[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' # Enabled "prometheus" addon with helm like custom settings kbcli addon enable prometheus --set prometheus.alertmanager.image.tag=v0.24.0 @@ -490,15 +490,13 @@ func (o *addonCmdOpts) buildEnablePatch(flags []*pflag.Flag, spec, install map[s return pItem, nil } - twoTuplesProcessor := func(s, flag string, + _tuplesProcessor := func(t []string, s, flag string, valueTransformer func(s, flag string) (interface{}, error), valueAssigner func(*extensionsv1alpha1.AddonInstallSpecItem, interface{}), ) error { - t := strings.SplitN(s, ":", 2) l := len(t) var name string var result interface{} - var err error switch l { case 2: name = t[0] @@ -528,6 +526,31 @@ func (o *addonCmdOpts) buildEnablePatch(flags []*pflag.Flag, spec, install map[s return nil } + twoTuplesProcessor := func(s, flag string, + valueTransformer func(s, flag string) (interface{}, error), + valueAssigner func(*extensionsv1alpha1.AddonInstallSpecItem, interface{}), + ) error { + t := strings.SplitN(s, ":", 2) + return _tuplesProcessor(t, s, flag, valueTransformer, valueAssigner) + } + + twoTuplesJSONProcessor := func(s, flag string, + valueTransformer func(s, flag string) (interface{}, error), + valueAssigner func(*extensionsv1alpha1.AddonInstallSpecItem, interface{}), + ) error { + var jsonArrary []map[string]interface{} + var t []string + + err := json.Unmarshal([]byte(s), &jsonArrary) + if err != nil { + // not a valid JSON array treat it a 2 tuples + t = strings.SplitN(s, ":", 2) + } else { + t = []string{s} + } + return _tuplesProcessor(t, s, flag, valueTransformer, valueAssigner) + } + reqLimitResTransformer := func(s, flag string) (interface{}, error) { t := strings.SplitN(s, "/", 2) if len(t) != 2 { @@ -578,7 +601,7 @@ func (o *addonCmdOpts) buildEnablePatch(flags []*pflag.Flag, spec, install map[s } for _, v := range f.TolerationsSet { - if err := twoTuplesProcessor(v, "tolerations", nil, func(item *extensionsv1alpha1.AddonInstallSpecItem, i interface{}) { + if err := twoTuplesJSONProcessor(v, "tolerations", nil, func(item *extensionsv1alpha1.AddonInstallSpecItem, i interface{}) { item.Tolerations = i.(string) }); err != nil { return err From e8e2b4de9dcd659daaeb9ac74fcecd985b3f8a33 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Fri, 28 Apr 2023 17:32:09 +0800 Subject: [PATCH 208/439] chore: add kbcli connect example support for mongodb (#3023) --- deploy/mongodb/templates/clusterdefinition.yaml | 1 + internal/cli/cmd/cluster/connect.go | 15 +++++++++++++++ internal/sqlchannel/engine/engine.go | 13 ++++++++----- internal/sqlchannel/engine/mongodb.go | 15 ++++++++++++--- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/deploy/mongodb/templates/clusterdefinition.yaml b/deploy/mongodb/templates/clusterdefinition.yaml index 5e92ae88d..cd88d6c96 100644 --- a/deploy/mongodb/templates/clusterdefinition.yaml +++ b/deploy/mongodb/templates/clusterdefinition.yaml @@ -58,6 +58,7 @@ spec: accessMode: Readonly updateStrategy: Serial probes: + roleProbeTimeoutAfterPodsReady: 300 roleProbe: failureThreshold: {{ .Values.roleProbe.failureThreshold }} periodSeconds: {{ .Values.roleProbe.periodSeconds }} diff --git a/internal/cli/cmd/cluster/connect.go b/internal/cli/cmd/cluster/connect.go index c4683524d..43f702fe2 100644 --- a/internal/cli/cmd/cluster/connect.go +++ b/internal/cli/cmd/cluster/connect.go @@ -349,6 +349,9 @@ func (o *ConnectOptions) getConnectionInfo() (*engine.ConnectionInfo, error) { return nil, err } + info.ClusterName = o.clusterName + info.ComponentName = o.componentName + info.HeadlessEndpoint = getOneHeadlessEndpoint(objs.ClusterDef, objs.Secrets) // get username and password if info.User, info.Password, err = getUserAndPassword(objs.ClusterDef, objs.Secrets); err != nil { return nil, err @@ -429,3 +432,15 @@ func getUserAndPassword(clusterDef *appsv1alpha1.ClusterDefinition, secrets *cor password, err = getSecretVal(&secret, passwordKey) return user, password, err } + +// get cluster headlessEndpoint from secrets +func getOneHeadlessEndpoint(clusterDef *appsv1alpha1.ClusterDefinition, secrets *corev1.SecretList) string { + if len(secrets.Items) == 0 { + return "" + } + val, ok := secrets.Items[0].Data["headlessEndpoint"] + if !ok { + return "" + } + return string(val) +} diff --git a/internal/sqlchannel/engine/engine.go b/internal/sqlchannel/engine/engine.go index 38685cec7..5501a7dda 100644 --- a/internal/sqlchannel/engine/engine.go +++ b/internal/sqlchannel/engine/engine.go @@ -68,11 +68,14 @@ func New(typeName string) (Interface, error) { } type ConnectionInfo struct { - Host string - User string - Password string - Database string - Port string + Host string + User string + Password string + Database string + Port string + ClusterName string + ComponentName string + HeadlessEndpoint string } type buildConnectExample func(info *ConnectionInfo) string diff --git a/internal/sqlchannel/engine/mongodb.go b/internal/sqlchannel/engine/mongodb.go index a88bb9c64..e3ba9d213 100644 --- a/internal/sqlchannel/engine/mongodb.go +++ b/internal/sqlchannel/engine/mongodb.go @@ -36,8 +36,15 @@ func newMongoDB() *mongodb { Container: "mongodb", UserEnv: "$MONGODB_ROOT_USER", PasswordEnv: "$MONGODB_ROOT_PASSWORD", + Database: "admin", + }, + examples: map[ClientType]buildConnectExample{ + CLI: func(info *ConnectionInfo) string { + return fmt.Sprintf(`# mongodb client connection example +mongosh mongodb://%s:%s@%s/%s?replicaset=%s-%s +`, info.User, info.Password, info.HeadlessEndpoint, info.Database, info.ClusterName, info.ComponentName) + }, }, - examples: map[ClientType]buildConnectExample{}, } } @@ -60,8 +67,10 @@ func (r mongodb) Container() string { } func (r mongodb) ConnectExample(info *ConnectionInfo, client string) string { - // TODO implement me - panic("implement me") + if len(info.Database) == 0 { + info.Database = r.info.Database + } + return buildExample(info, client, r.examples) } var _ Interface = &mongodb{} From 9e2a1f73d0ac069bb7207d64ace15a3071f06570 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Fri, 28 Apr 2023 19:10:59 +0800 Subject: [PATCH 209/439] chore: check if pvc exists when delete the backup and pod controller error (#3021) --- controllers/apps/cluster_controller_test.go | 2 +- controllers/dataprotection/backup_controller.go | 7 +++++++ .../dataprotection/backup_controller_test.go | 1 + controllers/dataprotection/cronjob_controller.go | 2 +- .../lifecycle/transformer_backup_policy_tpl.go | 7 +++++++ internal/controllerutil/controller_common.go | 14 ++++++++------ 6 files changed, 25 insertions(+), 8 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index d8bf615f4..0f563904e 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1600,7 +1600,7 @@ var _ = Describe("Cluster Controller", func() { })()).ShouldNot(HaveOccurred()) Eventually(testapps.CheckObj(&testCtx, clusterVersionKey, func(g Gomega, clusterVersion *appsv1alpha1.ClusterVersion) { - g.Expect(clusterVersion.Status.Phase == appsv1alpha1.UnavailablePhase).Should(BeTrue()) + g.Expect(clusterVersion.Status.Phase).Should(Equal(appsv1alpha1.UnavailablePhase)) })).Should(Succeed()) // trigger reconcile diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index c91bdff72..447127c75 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -1009,6 +1009,13 @@ func (r *BackupReconciler) deleteBackupFiles(reqCtx intctrlutil.RequestCtx, back "backup", backup.Name) return nil } + // check if pvc exists + if err = r.Client.Get(reqCtx.Ctx, types.NamespacedName{Namespace: backup.Namespace, Name: pvcName}, &corev1.PersistentVolumeClaim{}); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } backupFilePath := "" if backup.Status.Manifests != nil && backup.Status.Manifests.BackupTool != nil { diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index a2b4d8fe1..cbc0ff153 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -253,6 +253,7 @@ var _ = Describe("Backup Controller test", func() { By("checking Backup object, it should be deleted") Eventually(testapps.CheckObjExists(&testCtx, backupKey, &dataprotectionv1alpha1.Backup{}, false)).Should(Succeed()) + // TODO: add delete backup test case with the pvc not exists }) }) diff --git a/controllers/dataprotection/cronjob_controller.go b/controllers/dataprotection/cronjob_controller.go index f69c92839..7bad6b9f2 100644 --- a/controllers/dataprotection/cronjob_controller.go +++ b/controllers/dataprotection/cronjob_controller.go @@ -91,6 +91,6 @@ func (r *CronJobReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&batchv1.CronJob{}). Owns(&batchv1.Job{}). - WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). + WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.ManagedByKubeBlocksFilterPredicate)). Complete(r) } diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index f05c2de1b..aff787803 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -59,6 +59,7 @@ func (r *BackupPolicyTPLTransformer) Transform(ctx graph.TransformContext, dag * return err } origCluster := transCtx.OrigCluster + backupPolicyNames := map[string]struct{}{} for _, tpl := range backupPolicyTPLs.Items { r.isDefaultTemplate = tpl.Annotations[constant.DefaultBackupPolicyTemplateAnnotationKey] r.tplIdentifier = tpl.Spec.Identifier @@ -72,9 +73,15 @@ func (r *BackupPolicyTPLTransformer) Transform(ctx graph.TransformContext, dag * if backupPolicy == nil { continue } + // if exist multiple backup policy templates and duplicate spec.identifier, + // the backupPolicy that may be generated may have duplicate names, and it is necessary to check if it already exists. + if _, ok := backupPolicyNames[backupPolicy.Name]; ok { + continue + } vertex := &lifecycleVertex{obj: backupPolicy} dag.AddVertex(vertex) dag.Connect(rootVertex, vertex) + backupPolicyNames[backupPolicy.Name] = struct{}{} } } return nil diff --git a/internal/controllerutil/controller_common.go b/internal/controllerutil/controller_common.go index a1df716a8..6396a481a 100644 --- a/internal/controllerutil/controller_common.go +++ b/internal/controllerutil/controller_common.go @@ -210,13 +210,15 @@ func RecordCreatedEvent(r record.EventRecorder, cr client.Object) { } } -// WorkloadFilterPredicate provide filter predicate for workload objects, i.e., deployment/statefulset/pod/pvc. +// WorkloadFilterPredicate provides filter predicate for workload objects, i.e., deployment/statefulset/pod/pvc. func WorkloadFilterPredicate(object client.Object) bool { - objLabels := object.GetLabels() - if objLabels == nil { - return false - } - return objLabels[constant.AppManagedByLabelKey] == constant.AppName + _, containCompNameLabelKey := object.GetLabels()[constant.KBAppComponentLabelKey] + return ManagedByKubeBlocksFilterPredicate(object) && containCompNameLabelKey +} + +// ManagedByKubeBlocksFilterPredicate provides filter predicate for objects managed by kubeBlocks. +func ManagedByKubeBlocksFilterPredicate(object client.Object) bool { + return object.GetLabels()[constant.AppManagedByLabelKey] == constant.AppName } // IgnoreIsAlreadyExists return errors that is not AlreadyExists From 36ac995e41328bd65b70538318a96c1b578d0c48 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Sat, 29 Apr 2023 11:31:15 +0800 Subject: [PATCH 210/439] chore: cli supports to create additional resource for postgresql (#3009) Co-authored-by: xingran --- .../postgresql-cluster/templates/_helpers.tpl | 2 +- .../templates/rolebinding.yaml | 4 +- .../templates/serviceaccount.yaml | 2 +- .../templates/clusterdefinition.yaml | 5 - internal/cli/cmd/cluster/create.go | 248 ++++++++++++++---- internal/cli/cmd/cluster/delete.go | 109 +++++++- internal/cli/cmd/playground/init.go | 3 - internal/cli/create/create.go | 42 ++- internal/cli/delete/delete.go | 7 +- internal/cli/delete/delete_test.go | 2 +- 10 files changed, 346 insertions(+), 78 deletions(-) diff --git a/deploy/postgresql-cluster/templates/_helpers.tpl b/deploy/postgresql-cluster/templates/_helpers.tpl index f38edd501..1c02d0255 100644 --- a/deploy/postgresql-cluster/templates/_helpers.tpl +++ b/deploy/postgresql-cluster/templates/_helpers.tpl @@ -55,7 +55,7 @@ Create the name of the service account to use */}} {{- define "postgresqlcluster.serviceAccountName" -}} {{- if .Values.serviceAccount.enabled }} -{{- printf "kb-postgres-%s" .Release.Name | trunc 63 | trimSuffix "-" }} +{{- printf "kb-sa-%s" .Release.Name | trunc 63 | trimSuffix "-" }} {{- else }} {{- .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/postgresql-cluster/templates/rolebinding.yaml b/deploy/postgresql-cluster/templates/rolebinding.yaml index 789a289fa..ebc0a083f 100644 --- a/deploy/postgresql-cluster/templates/rolebinding.yaml +++ b/deploy/postgresql-cluster/templates/rolebinding.yaml @@ -2,7 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: kb-postgres-{{ .Release.Namespace }}-{{ .Release.Name }} + name: kb-rolebinding-{{ .Release.Namespace }}-{{ .Release.Name }} labels: {{ include "postgresqlcluster.labels" . | nindent 4 }} roleRef: @@ -11,6 +11,6 @@ roleRef: name: kb-role-{{ .Release.Namespace }}-{{ .Release.Name }} subjects: - kind: ServiceAccount - name: kb-postgres-{{ .Release.Name }} + name: kb-sa-{{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- end }} \ No newline at end of file diff --git a/deploy/postgresql-cluster/templates/serviceaccount.yaml b/deploy/postgresql-cluster/templates/serviceaccount.yaml index d4d83a500..2eb331be8 100644 --- a/deploy/postgresql-cluster/templates/serviceaccount.yaml +++ b/deploy/postgresql-cluster/templates/serviceaccount.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: kb-postgres-{{ .Release.Name }} + name: kb-sa-{{ .Release.Name }} labels: {{ include "postgresqlcluster.labels" . | nindent 4 }} {{- end }} \ No newline at end of file diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 384b773fa..923c1437c 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -82,13 +82,10 @@ spec: volumeMounts: - name: data mountPath: /home/postgres/pgdata - mode: 0777 - name: postgresql-config mountPath: /home/postgres/conf - mode: 0777 - name: scripts mountPath: /kb-scripts - mode: 0777 containers: - name: postgresql imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} @@ -115,10 +112,8 @@ spec: mountPath: /dev/shm - name: data mountPath: /home/postgres/pgdata - mode: 0777 - name: postgresql-config mountPath: /home/postgres/conf - mode: 0777 - name: scripts mountPath: /kb-scripts ports: diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 324fb7871..6c575e76b 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -39,7 +39,10 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" + corev1ac "k8s.io/client-go/applyconfigurations/core/v1" + rbacv1ac "k8s.io/client-go/applyconfigurations/rbac/v1" "k8s.io/client-go/dynamic" + "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" utilcomp "k8s.io/kubectl/pkg/util/completion" "k8s.io/kubectl/pkg/util/storage" @@ -172,6 +175,8 @@ type CreateOptions struct { SetFile string `json:"-"` Values []string `json:"-"` + shouldCreateDependencies bool `json:"-"` + // backup name to restore in creation Backup string `json:"backup,omitempty"` UpdatableFlags @@ -259,16 +264,14 @@ func (o *CreateOptions) Validate() error { return fmt.Errorf("cluster name should be less than 16 characters") } - // validate default storageClassName - err := validateStorageClass(o.Dynamic, o.ComponentSpecs) - if err != nil { - return err - } - return nil } func (o *CreateOptions) Complete() error { + if err := o.Validate(); err != nil { + return err + } + components, err := o.buildComponents() if err != nil { return err @@ -285,73 +288,182 @@ func (o *CreateOptions) Complete() error { if len(tolerations) > 0 { o.Tolerations = tolerations } - return nil + + // validate default storageClassName + return validateStorageClass(o.Dynamic, o.ComponentSpecs) +} + +func (o *CreateOptions) CleanUp() error { + if o.Client == nil { + return nil + } + + return deleteDependencies(o.Client, o.Namespace, o.Name) } // buildComponents build components from file or set values func (o *CreateOptions) buildComponents() ([]map[string]interface{}, error) { var ( - componentByte []byte - err error + err error + cd *appsv1alpha1.ClusterDefinition + compSpecs []*appsv1alpha1.ClusterComponentSpec ) - componentClasses, err := class.ListClassesByClusterDefinition(o.Dynamic, o.ClusterDefRef) + compClasses, err := class.ListClassesByClusterDefinition(o.Dynamic, o.ClusterDefRef) + if err != nil { + return nil, err + } + + cd, err = cluster.GetClusterDefByName(o.Dynamic, o.ClusterDefRef) if err != nil { return nil, err } // build components from file - components := o.ComponentSpecs if len(o.SetFile) > 0 { - if componentByte, err = MultipleSourceComponents(o.SetFile, o.IOStreams.In); err != nil { + var ( + compByte []byte + comps []map[string]interface{} + ) + if compByte, err = MultipleSourceComponents(o.SetFile, o.IOStreams.In); err != nil { return nil, err } - if componentByte, err = yaml.YAMLToJSON(componentByte); err != nil { + if compByte, err = yaml.YAMLToJSON(compByte); err != nil { return nil, err } - if err = json.Unmarshal(componentByte, &components); err != nil { + if err = json.Unmarshal(compByte, &comps); err != nil { return nil, err } - for _, item := range components { - var comp appsv1alpha1.ClusterComponentSpec - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(item, &comp); err != nil { - return nil, err - } - if _, err = class.ValidateComponentClass(&comp, componentClasses); err != nil { + for _, comp := range comps { + var compSpec appsv1alpha1.ClusterComponentSpec + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(comp, &compSpec); err != nil { return nil, err } + compSpecs = append(compSpecs, &compSpec) } - return components, nil - } - - // build components from set values or environment variables - if len(components) == 0 { - cd, err := cluster.GetClusterDefByName(o.Dynamic, o.ClusterDefRef) + } else { + // build components from set values or environment variables + compSets, err := buildCompSetsMap(o.Values, cd) if err != nil { return nil, err } - compSets, err := buildCompSetsMap(o.Values, cd) + compSpecs, err = buildClusterComp(cd, compSets, compClasses) if err != nil { return nil, err } + } - componentObjs, err := buildClusterComp(cd, compSets, componentClasses) - if err != nil { + var comps []map[string]interface{} + for _, compSpec := range compSpecs { + // validate component classes + if _, err = class.ValidateComponentClass(compSpec, compClasses); err != nil { return nil, err } - for _, compObj := range componentObjs { - if _, err = class.ValidateComponentClass(compObj, componentClasses); err != nil { - return nil, err - } - comp, err := runtime.DefaultUnstructuredConverter.ToUnstructured(compObj) - if err != nil { - return nil, err - } - components = append(components, comp) + + // create component dependencies + if err = o.buildDependenciesFn(cd, compSpec); err != nil { + return nil, err + } + + comp, err := runtime.DefaultUnstructuredConverter.ToUnstructured(compSpec) + if err != nil { + return nil, err } + comps = append(comps, comp) + } + return comps, nil +} + +const ( + saNamePrefix = "kb-sa-" + roleNamePrefix = "kb-role-" + roleBindingNamePrefix = "kb-rolebinding-" +) + +// buildDependenciesFn create dependencies function for components, e.g. postgresql depends on +// a service account, a role and a rolebinding +func (o *CreateOptions) buildDependenciesFn(cd *appsv1alpha1.ClusterDefinition, + compSpec *appsv1alpha1.ClusterComponentSpec) error { + + // HACK: now we only support postgresql cluster definition + if c, err := shouldCreateDependencies(cd, compSpec); err != nil { + return err + } else if !c { + return nil } - return components, nil + + // set component service account name + compSpec.ServiceAccountName = saNamePrefix + o.Name + o.shouldCreateDependencies = true + return nil +} + +func (o *CreateOptions) CreateDependencies(dryRun []string) error { + var ( + saName = saNamePrefix + o.Name + roleName = roleNamePrefix + o.Name + roleBindingName = roleBindingNamePrefix + o.Name + ) + + if !o.shouldCreateDependencies { + return nil + } + + klog.V(1).Infof("create dependencies for cluster %s", o.Name) + // create service account + labels := buildResourceLabels(o.Name) + applyOptions := metav1.ApplyOptions{FieldManager: "kbcli", DryRun: dryRun} + sa := corev1ac.ServiceAccount(saName, o.Namespace).WithLabels(labels) + + klog.V(1).Infof("create service account %s", saName) + if _, err := o.Client.CoreV1().ServiceAccounts(o.Namespace).Apply(context.TODO(), sa, applyOptions); err != nil { + return err + } + + // create role + klog.V(1).Infof("create role %s", roleName) + role := rbacv1ac.Role(roleName, o.Namespace).WithRules([]*rbacv1ac.PolicyRuleApplyConfiguration{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"create", "get", "list", "patch", "update", "watch", "delete"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "get", "list", "patch", "update", "watch", "delete"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + }...).WithLabels(labels) + if _, err := o.Client.RbacV1().Roles(o.Namespace).Apply(context.TODO(), role, applyOptions); err != nil { + return err + } + + // create role binding + rbacAPIGroup := "rbac.authorization.k8s.io" + rbacKind := "Role" + saKind := "ServiceAccount" + roleBinding := rbacv1ac.RoleBinding(roleBindingName, o.Namespace).WithLabels(labels). + WithSubjects([]*rbacv1ac.SubjectApplyConfiguration{ + { + Kind: &saKind, + Name: &saName, + Namespace: &o.Namespace, + }, + }...). + WithRoleRef(&rbacv1ac.RoleRefApplyConfiguration{ + APIGroup: &rbacAPIGroup, + Kind: &rbacKind, + Name: &roleName, + }) + klog.V(1).Infof("create role binding %s", roleBindingName) + _, err := o.Client.RbacV1().RoleBindings(o.Namespace).Apply(context.TODO(), roleBinding, applyOptions) + return err } // MultipleSourceComponents get component data from multiple source, such as stdin, URI and local file @@ -381,17 +493,18 @@ func MultipleSourceComponents(fileName string, in io.Reader) ([]byte, error) { func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &CreateOptions{BaseOptions: create.BaseOptions{IOStreams: streams}} inputs := create.Inputs{ - Use: "create [NAME]", - Short: "Create a cluster.", - Example: clusterCreateExample, - CueTemplateName: CueTemplateName, - ResourceName: types.ResourceClusters, - BaseOptionsObj: &o.BaseOptions, - Options: o, - Factory: f, - Validate: o.Validate, - Complete: o.Complete, - PreCreate: o.PreCreate, + Use: "create [NAME]", + Short: "Create a cluster.", + Example: clusterCreateExample, + CueTemplateName: CueTemplateName, + ResourceName: types.ResourceClusters, + BaseOptionsObj: &o.BaseOptions, + Options: o, + Factory: f, + Complete: o.Complete, + PreCreate: o.PreCreate, + CleanUpFn: o.CleanUp, + CreateDependencies: o.CreateDependencies, BuildFlags: func(cmd *cobra.Command) { cmd.Flags().StringVar(&o.ClusterDefRef, "cluster-definition", "", "Specify cluster definition, run \"kbcli cd list\" to show all available cluster definitions") cmd.Flags().StringVar(&o.ClusterVersionRef, "cluster-version", "", "Specify cluster version, run \"kbcli cv list\" to show all available cluster versions, use the latest version if not specified") @@ -485,7 +598,8 @@ func setEnableAllLogs(c *appsv1alpha1.Cluster, cd *appsv1alpha1.ClusterDefinitio } } -func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map[setKey]string, componentClasses map[string]map[string]*appsv1alpha1.ComponentClassInstance) ([]*appsv1alpha1.ClusterComponentSpec, error) { +func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map[setKey]string, + componentClasses map[string]map[string]*appsv1alpha1.ComponentClassInstance) ([]*appsv1alpha1.ClusterComponentSpec, error) { // get value from set values and environment variables, the second return value is // true if the value is from environment variables getVal := func(c *appsv1alpha1.ClusterComponentDefinition, key setKey, sets map[setKey]string) string { @@ -505,6 +619,7 @@ func buildClusterComp(cd *appsv1alpha1.ClusterDefinition, setsMap map[string]map return "2" } } + // the default replicas is 3 if not set by command flag, for Consensus workload if c.WorkloadType == appsv1alpha1.Consensus { if key == keyReplicas { @@ -818,3 +933,34 @@ func getStorageClasses(dynamic dynamic.Interface) (map[string]struct{}, bool, er } return allStorageClasses, existedDefault, nil } + +func shouldCreateDependencies(cd *appsv1alpha1.ClusterDefinition, compSpec *appsv1alpha1.ClusterComponentSpec) (bool, error) { + var compDef *appsv1alpha1.ClusterComponentDefinition + if cd.Spec.Type != "postgresql" { + return false, nil + } + + // get cluster component definition + for i, def := range cd.Spec.ComponentDefs { + if def.Name == compSpec.ComponentDefRef { + compDef = &cd.Spec.ComponentDefs[i] + } + } + + if compDef == nil { + return false, fmt.Errorf("failed to find component definition for componnet %s", compSpec.Name) + } + + // for postgresql, we need to create a service account, a role and a rolebinding + if compDef.CharacterType != "postgresql" { + return false, nil + } + return true, nil +} + +func buildResourceLabels(clusterName string) map[string]string { + return map[string]string{ + constant.AppInstanceLabelKey: clusterName, + constant.AppManagedByLabelKey: "kbcli", + } +} diff --git a/internal/cli/cmd/cluster/delete.go b/internal/cli/cmd/cluster/delete.go index dda4985b2..e05958c5c 100644 --- a/internal/cli/cmd/cluster/delete.go +++ b/internal/cli/cmd/cluster/delete.go @@ -20,17 +20,23 @@ along with this program. If not, see . package cluster import ( + "context" "fmt" "github.com/spf13/cobra" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/errors" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/cluster" "github.com/apecloud/kubeblocks/internal/cli/delete" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" @@ -44,6 +50,7 @@ var deleteExample = templates.Examples(` func NewDeleteCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := delete.NewDeleteOptions(f, streams, types.ClusterGVR()) o.PreDeleteHook = clusterPreDeleteHook + o.PostDeleteHook = clusterPostDeleteHook cmd := &cobra.Command{ Use: "delete NAME", @@ -66,14 +73,9 @@ func deleteCluster(o *delete.DeleteOptions, args []string) error { return o.Run() } -func clusterPreDeleteHook(object runtime.Object) error { - if object.GetObjectKind().GroupVersionKind().Kind != appsv1alpha1.ClusterKind { - klog.V(1).Infof("object %s is not of kind %s, skip PreDeleteHook.", object.GetObjectKind().GroupVersionKind().Kind, appsv1alpha1.ClusterKind) - return nil - } - unstructed := object.(*unstructured.Unstructured) - cluster := &appsv1alpha1.Cluster{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructed.Object, cluster); err != nil { +func clusterPreDeleteHook(o *delete.DeleteOptions, object runtime.Object) error { + cluster, err := getClusterFromObject(object) + if err != nil { return err } if cluster.Spec.TerminationPolicy == appsv1alpha1.DoNotTerminate { @@ -81,3 +83,94 @@ func clusterPreDeleteHook(object runtime.Object) error { } return nil } + +func clusterPostDeleteHook(o *delete.DeleteOptions, object runtime.Object) error { + c, err := getClusterFromObject(object) + if err != nil { + return err + } + + dynamic, err := o.Factory.DynamicClient() + if err != nil { + return err + } + + client, err := o.Factory.KubernetesClientSet() + if err != nil { + return err + } + + // HACK: for a postgresql cluster, we need to delete the sa, role and rolebinding + cd, err := cluster.GetClusterDefByName(dynamic, c.Spec.ClusterDefRef) + if err != nil { + return err + } + for _, compSpec := range c.Spec.ComponentSpecs { + if err = deleteCompDependencies(client, c.Namespace, c.Name, cd, &compSpec); err != nil { + return err + } + } + return nil +} + +func deleteCompDependencies(client kubernetes.Interface, ns string, name string, cd *appsv1alpha1.ClusterDefinition, + compSpec *appsv1alpha1.ClusterComponentSpec) error { + if d, err := shouldCreateDependencies(cd, compSpec); err != nil { + return err + } else if !d { + return nil + } + return deleteDependencies(client, ns, name) +} + +func deleteDependencies(client kubernetes.Interface, ns string, name string) error { + klog.V(1).Infof("delete dependencies for cluster %s", name) + var ( + saName = saNamePrefix + name + roleName = roleNamePrefix + name + roleBindingName = roleBindingNamePrefix + name + allErr []error + ) + + // now, delete the dependencies, for postgresql, we delete sa, role and rolebinding + ctx := context.TODO() + gracePeriod := int64(0) + deleteOptions := metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod} + checkErr := func(err error) bool { + if err != nil && !apierrors.IsNotFound(err) { + return true + } + return false + } + + // delete rolebinding + klog.V(1).Infof("delete rolebinding %s", roleBindingName) + if err := client.RbacV1().RoleBindings(ns).Delete(ctx, roleBindingName, deleteOptions); checkErr(err) { + allErr = append(allErr, err) + } + + // delete service account + klog.V(1).Infof("delete service account %s", saName) + if err := client.CoreV1().ServiceAccounts(ns).Delete(ctx, saName, deleteOptions); checkErr(err) { + allErr = append(allErr, err) + } + + // delete role + klog.V(1).Infof("delete role %s", roleName) + if err := client.RbacV1().Roles(ns).Delete(ctx, roleName, deleteOptions); checkErr(err) { + allErr = append(allErr, err) + } + return errors.NewAggregate(allErr) +} + +func getClusterFromObject(object runtime.Object) (*appsv1alpha1.Cluster, error) { + if object.GetObjectKind().GroupVersionKind().Kind != appsv1alpha1.ClusterKind { + return nil, fmt.Errorf("object %s is not of kind %s", object.GetObjectKind().GroupVersionKind().Kind, appsv1alpha1.ClusterKind) + } + unstructured := object.(*unstructured.Unstructured) + cluster := &appsv1alpha1.Cluster{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured.Object, cluster); err != nil { + return nil, err + } + return cluster, nil +} diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index bfbc499cc..de1a6fd8a 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -513,9 +513,6 @@ func (o *initOptions) newCreateOptions() (*cmdcluster.CreateOptions, error) { options.Values = append(options.Values, "replicas=3") } - if err = options.Validate(); err != nil { - return nil, err - } if err = options.Complete(); err != nil { return nil, err } diff --git a/internal/cli/create/create.go b/internal/cli/create/create.go index 386caea3a..6f02cf582 100755 --- a/internal/cli/create/create.go +++ b/internal/cli/create/create.go @@ -31,7 +31,7 @@ import ( cuejson "cuelang.org/go/encoding/json" "github.com/leaanthony/debme" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -55,6 +55,8 @@ var ( cueTemplate embed.FS ) +type CreateDependency func(dryRun []string) error + type DryRunStrategy int const ( @@ -109,6 +111,12 @@ type Inputs struct { // CustomOutPut will be executed after creating successfully. CustomOutPut func(options *BaseOptions) + // CleanUpFn will be executed after creating failed. + CleanUpFn func() error + + // CreateDependencies will be executed before creating. + CreateDependencies CreateDependency + // ResourceNameGVRForCompletion resource name for completion. ResourceNameGVRForCompletion schema.GroupVersionResource } @@ -264,11 +272,28 @@ func (o *BaseOptions) Run(inputs Inputs) error { if dryRunStrategy == DryRunServer { createOptions.DryRun = []string{metav1.DryRunAll} } - // create k8s resource + + // create dependencies + if inputs.CreateDependencies != nil { + if err = inputs.CreateDependencies(createOptions.DryRun); err != nil { + return err + } + } + + // create kubernetes resource previewObj, err = o.Dynamic.Resource(gvr).Namespace(o.Namespace).Create(context.TODO(), previewObj, createOptions) if err != nil { + if apierrors.IsAlreadyExists(err) { + return err + } + + // for other errors, clean up dependencies + if cleanErr := o.CleanUp(inputs); cleanErr != nil { + fmt.Fprintf(o.ErrOut, "clean up denpendencies failed: %v\n", cleanErr) + } return err } + if dryRunStrategy != DryRunServer { o.Name = previewObj.GetName() if o.Quiet { @@ -289,6 +314,17 @@ func (o *BaseOptions) Run(inputs Inputs) error { return printer.PrintObj(previewObj, o.Out) } +func (o *BaseOptions) CleanUp(inputs Inputs) error { + if inputs.CreateDependencies == nil { + return nil + } + + if inputs.CleanUpFn != nil { + return inputs.CleanUpFn() + } + return nil +} + // RunAsApply execute command. the options of parameter contain the command flags and args. // if the resource exists, run as "kubectl apply". func (o *BaseOptions) RunAsApply(inputs Inputs) error { @@ -339,7 +375,7 @@ func (o *BaseOptions) RunAsApply(inputs Inputs) error { objectByte, metav1.PatchOptions{}); err != nil { // create object if not found - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { if _, err = o.Dynamic.Resource(gvr).Namespace(o.Namespace).Create( context.TODO(), unstructuredObj, metav1.CreateOptions{}); err != nil { return err diff --git a/internal/cli/delete/delete.go b/internal/cli/delete/delete.go index b077d3662..13c40787d 100644 --- a/internal/cli/delete/delete.go +++ b/internal/cli/delete/delete.go @@ -38,7 +38,8 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/util/prompt" ) -type DeleteHook func(object runtime.Object) error +type DeleteHook func(options *DeleteOptions, object runtime.Object) error + type DeleteOptions struct { Factory cmdutil.Factory Namespace string @@ -218,14 +219,14 @@ func (o *DeleteOptions) preDeleteResource(info *resource.Info) error { return err } } - return o.PreDeleteHook(info.Object) + return o.PreDeleteHook(o, info.Object) } return nil } func (o *DeleteOptions) postDeleteResource(object runtime.Object) error { if o.PostDeleteHook != nil { - return o.PostDeleteHook(object) + return o.PostDeleteHook(o, object) } return nil } diff --git a/internal/cli/delete/delete_test.go b/internal/cli/delete/delete_test.go index 4ec5ef80b..30001727e 100644 --- a/internal/cli/delete/delete_test.go +++ b/internal/cli/delete/delete_test.go @@ -188,7 +188,7 @@ var _ = Describe("Delete", func() { By("set pre-delete hook") // block cluster deletion - fakePreDeleteHook := func(object runtime.Object) error { + fakePreDeleteHook := func(o *DeleteOptions, object runtime.Object) error { if object.GetObjectKind().GroupVersionKind().Kind == appsv1alpha1.ClusterKind { return fmt.Errorf("fake pre-delete hook error") } else { From 7bb72276f4311a5515d5738377e8564c552fe55c Mon Sep 17 00:00:00 2001 From: wangyelei Date: Mon, 1 May 2023 19:02:28 +0800 Subject: [PATCH 211/439] chore: fix mongodb backup failed when data being written (#3033) --- deploy/mongodb/templates/backuptool.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/mongodb/templates/backuptool.yaml b/deploy/mongodb/templates/backuptool.yaml index 9a8a12c60..303473711 100644 --- a/deploy/mongodb/templates/backuptool.yaml +++ b/deploy/mongodb/templates/backuptool.yaml @@ -53,4 +53,5 @@ spec: - | mkdir -p ${BACKUP_DIR} && cd ${DATA_DIR} tar -czvf ${BACKUP_DIR}/${BACKUP_NAME}.tar.gz ./ + echo $? incrementalBackupCommands: [] From 6f1b68dc71f86e49fdace2270a716dd6b539f07e Mon Sep 17 00:00:00 2001 From: wangyelei Date: Thu, 4 May 2023 09:56:56 +0800 Subject: [PATCH 212/439] chore: fix the .restore file always exist after hscale apecloud-mysql (#3041) --- .../templates/backuppolicytemplateforhscale.yaml | 2 ++ .../apecloud-mysql/templates/backuppolicytemplateforhscale.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/deploy/apecloud-mysql-scale/templates/backuppolicytemplateforhscale.yaml b/deploy/apecloud-mysql-scale/templates/backuppolicytemplateforhscale.yaml index c60503db1..143abe1ec 100644 --- a/deploy/apecloud-mysql-scale/templates/backuppolicytemplateforhscale.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuppolicytemplateforhscale.yaml @@ -15,5 +15,7 @@ spec: containerName: mysql preCommands: - "touch /data/mysql/data/.restore; sync" + postCommands: + - "rm -f /data/mysql/data/.restore; sync" target: role: leader \ No newline at end of file diff --git a/deploy/apecloud-mysql/templates/backuppolicytemplateforhscale.yaml b/deploy/apecloud-mysql/templates/backuppolicytemplateforhscale.yaml index b57c36141..b25685963 100644 --- a/deploy/apecloud-mysql/templates/backuppolicytemplateforhscale.yaml +++ b/deploy/apecloud-mysql/templates/backuppolicytemplateforhscale.yaml @@ -15,5 +15,7 @@ spec: containerName: mysql preCommands: - "touch /data/mysql/data/.restore; sync" + postCommands: + - "rm -f /data/mysql/data/.restore; sync" target: role: leader \ No newline at end of file From 90ac07e6a4a48a31fc6c267feea294d6f0c3b1c2 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Thu, 4 May 2023 11:21:06 +0800 Subject: [PATCH 213/439] fix: restore apecloud-mysql failed from snapshot backup (#3047) --- deploy/apecloud-mysql/templates/scripts.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index 30efde94c..df3534a13 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -64,7 +64,7 @@ data: chmod +777 -R /data/mysql; echo "KB_MYSQL_CLUSTER_UID=$KB_MYSQL_CLUSTER_UID" cluster_uid_path=/data/mysql/data/.kb_cluster_uid - if [ -f $cluster_uid_path ]; then + if [ -f $cluster_uid_path ] && [ ! -f /data/mysql/data/.restore_new_cluster ]; then last_cluster_uid=`cat $cluster_uid_path` if [ "$last_cluster_uid" != "$KB_MYSQL_CLUSTER_UID" ]; then echo "recreate from existing volumes, touch /data/mysql/data/.resetup_db" From f2ae70956e34077c6c7ff8443a6f9d1fcf385a56 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Thu, 4 May 2023 14:28:48 +0800 Subject: [PATCH 214/439] feat: support delphic (#2933) --- .../templates/cluster.yaml | 2 +- deploy/apecloud-mysql-cluster/values.yaml | 2 + .../templates/cluster.yaml | 2 +- .../apecloud-mysql-scale-cluster/values.yaml | 2 + .../clickhouse-cluster/templates/cluster.yaml | 2 +- deploy/clickhouse-cluster/values.yaml | 5 +- deploy/delphic/.helmignore | 23 ++++ deploy/delphic/Chart.lock | 9 ++ deploy/delphic/Chart.yaml | 37 ++++++ deploy/delphic/README.md | 50 ++++++++ deploy/delphic/templates/NOTES.txt | 22 ++++ deploy/delphic/templates/_helpers.tpl | 95 +++++++++++++++ deploy/delphic/templates/deployment.yaml | 115 ++++++++++++++++++ deploy/delphic/templates/ingress.yaml | 61 ++++++++++ deploy/delphic/templates/job.yaml | 53 ++++++++ deploy/delphic/templates/secret.yaml | 19 +++ deploy/delphic/templates/service.yaml | 15 +++ deploy/delphic/templates/serviceaccount.yaml | 12 ++ .../templates/tests/test-connection.yaml | 15 +++ deploy/delphic/values.yaml | 107 ++++++++++++++++ deploy/etcd-cluster/templates/cluster.yaml | 2 +- deploy/etcd-cluster/values.yaml | 2 +- deploy/kafka-cluster/templates/cluster.yaml | 2 +- deploy/kafka-cluster/values.yaml | 3 + deploy/milvus-cluster/templates/cluster.yaml | 2 +- deploy/milvus-cluster/values.yaml | 3 + deploy/mongodb-cluster/templates/cluster.yaml | 2 +- .../postgresql-cluster/templates/cluster.yaml | 2 +- deploy/postgresql-cluster/values.yaml | 3 + deploy/qdrant-cluster/templates/cluster.yaml | 2 +- deploy/qdrant-cluster/values.yaml | 2 + deploy/redis-cluster/templates/cluster.yaml | 2 +- deploy/redis-cluster/values.yaml | 3 + .../weaviate-cluster/templates/cluster.yaml | 2 +- deploy/weaviate-cluster/values.yaml | 2 + 35 files changed, 669 insertions(+), 13 deletions(-) create mode 100644 deploy/delphic/.helmignore create mode 100644 deploy/delphic/Chart.lock create mode 100644 deploy/delphic/Chart.yaml create mode 100644 deploy/delphic/README.md create mode 100644 deploy/delphic/templates/NOTES.txt create mode 100644 deploy/delphic/templates/_helpers.tpl create mode 100644 deploy/delphic/templates/deployment.yaml create mode 100644 deploy/delphic/templates/ingress.yaml create mode 100644 deploy/delphic/templates/job.yaml create mode 100644 deploy/delphic/templates/secret.yaml create mode 100644 deploy/delphic/templates/service.yaml create mode 100644 deploy/delphic/templates/serviceaccount.yaml create mode 100644 deploy/delphic/templates/tests/test-connection.yaml create mode 100644 deploy/delphic/values.yaml diff --git a/deploy/apecloud-mysql-cluster/templates/cluster.yaml b/deploy/apecloud-mysql-cluster/templates/cluster.yaml index aa6f5536a..09cb84d27 100644 --- a/deploy/apecloud-mysql-cluster/templates/cluster.yaml +++ b/deploy/apecloud-mysql-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "apecloud-mysql-cluster.fullname" . }} labels: {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: apecloud-mysql # ref clusterdefinition.name diff --git a/deploy/apecloud-mysql-cluster/values.yaml b/deploy/apecloud-mysql-cluster/values.yaml index 414ec673d..dab27ad50 100644 --- a/deploy/apecloud-mysql-cluster/values.yaml +++ b/deploy/apecloud-mysql-cluster/values.yaml @@ -7,6 +7,8 @@ replicaCount: 3 terminationPolicy: Delete clusterVersionOverride: "" +nameOverride: "" +fullnameOverride: "" monitor: enabled: false diff --git a/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml b/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml index accd02e77..bcf8d2767 100644 --- a/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml +++ b/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "apecloud-mysql-cluster.fullname" . }} labels: {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: apecloud-mysql-scale # ref clusterdefinition.name diff --git a/deploy/apecloud-mysql-scale-cluster/values.yaml b/deploy/apecloud-mysql-scale-cluster/values.yaml index 016b39146..4cfcdcbd2 100644 --- a/deploy/apecloud-mysql-scale-cluster/values.yaml +++ b/deploy/apecloud-mysql-scale-cluster/values.yaml @@ -7,6 +7,8 @@ replicaCount: 3 terminationPolicy: Delete clusterVersionOverride: "" +nameOverride: "" +fullnameOverride: "" monitor: enabled: false diff --git a/deploy/clickhouse-cluster/templates/cluster.yaml b/deploy/clickhouse-cluster/templates/cluster.yaml index 43c9dca96..b320090b8 100644 --- a/deploy/clickhouse-cluster/templates/cluster.yaml +++ b/deploy/clickhouse-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "clickhouse-cluster.fullname" . }} labels: {{ include "clickhouse-cluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: clickhouse # ref clusterdefinition.name diff --git a/deploy/clickhouse-cluster/values.yaml b/deploy/clickhouse-cluster/values.yaml index c00617590..de904667d 100644 --- a/deploy/clickhouse-cluster/values.yaml +++ b/deploy/clickhouse-cluster/values.yaml @@ -275,4 +275,7 @@ ingress: ## port: ## name: http ## - extraRules: [] \ No newline at end of file + extraRules: [] + +nameOverride: "" +fullnameOverride: "" diff --git a/deploy/delphic/.helmignore b/deploy/delphic/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/delphic/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/delphic/Chart.lock b/deploy/delphic/Chart.lock new file mode 100644 index 000000000..0eb5c6364 --- /dev/null +++ b/deploy/delphic/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: pgcluster + repository: file://../postgresql-cluster + version: 0.5.0-beta.2 +- name: redis-cluster + repository: file://../redis-cluster + version: 0.5.0-beta.2 +digest: sha256:3e5fb77b9fe9b4de74300a45a789778d2e9be4fe10a564e0af7a0b0e2f07e53c +generated: "2023-04-25T11:32:23.847825+08:00" diff --git a/deploy/delphic/Chart.yaml b/deploy/delphic/Chart.yaml new file mode 100644 index 000000000..9c221be73 --- /dev/null +++ b/deploy/delphic/Chart.yaml @@ -0,0 +1,37 @@ +apiVersion: v2 +name: delphic +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" + + +dependencies: + - condition: postgres.enabled + name: pgcluster + alias: postgres + repository: file://../postgresql-cluster + version: 0.5.0-beta.2 + - condition: redis.enabled + name: redis-cluster + alias: redis + repository: file://../redis-cluster + version: 0.5.0-beta.2 \ No newline at end of file diff --git a/deploy/delphic/README.md b/deploy/delphic/README.md new file mode 100644 index 000000000..89dc9e66e --- /dev/null +++ b/deploy/delphic/README.md @@ -0,0 +1,50 @@ +1. Enable the addon postgresql and redis. + ```shell + kbcli addon enable postgresql + kbcli addon enable redis + ``` +2. When enabled successfully, you can check it with ```kbcli addon list``` and ```kubectl get clusterdefinition```. + ```shell + kbcli addon list + kubectl get clusterdefinition + NAME MAIN-COMPONENT-NAME STATUS AGE + redis redis Available 6h43m + postgresql postgresql Available 6h42m + ``` +3. Install the delphic with helm. + ```shell + # TODO publish the delphic to public helm repository + helm install delphic ./deploy/delphic + ``` +4. Check whether the plugin is installed successfully. + ``` + kubectl get pods + NAME READY STATUS RESTARTS AGE + delphic-redis-redis-0 3/3 Running 0 42m + delphic-redis-redis-1 3/3 Running 0 42m + delphic-redis-redis-sentinel-0 1/1 Running 1 (41m ago) 42m + delphic-redis-redis-sentinel-2 1/1 Running 1 (41m ago) 42m + delphic-redis-redis-sentinel-1 1/1 Running 1 (41m ago) 42m + delphic-6f747fb8f7-4hdh5 5/5 Running 0 43m + delphic-create-django-user-lmpnq 1/1 Completed 1 43m + delphic-postgres-postgresql-0 4/4 Running 0 42m + delphic-postgres-postgresql-1 4/4 Running 0 42m + ``` +5. Find the username and password of web console. + On Mac OS X: + ``` + kubectl get secret delphic-django-secret -o jsonpath='{.data.username}' | base64 -D + kubectl get secret delphic-django-secret -o jsonpath='{.data.password}' | base64 -D + ``` + + On Linux: + ``` + kubectl get secret delphic-django-secret -o jsonpath='{.data.username}' | base64 -d + kubectl get secret delphic-django-secret -o jsonpath='{.data.password}' | base64 -d + ``` + +6. Port-forward the Plugin Portal to access it. + ```shell + kubectl port-forward port-forward deployment/delphic 3000:3000 8000:8000 + ``` +7. In your web browser, open the plugin portal with the address ```http://127.0.0.1:3000``` \ No newline at end of file diff --git a/deploy/delphic/templates/NOTES.txt b/deploy/delphic/templates/NOTES.txt new file mode 100644 index 000000000..c917929a9 --- /dev/null +++ b/deploy/delphic/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "delphic.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "delphic.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "delphic.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "delphic.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/deploy/delphic/templates/_helpers.tpl b/deploy/delphic/templates/_helpers.tpl new file mode 100644 index 000000000..f9b0fff23 --- /dev/null +++ b/deploy/delphic/templates/_helpers.tpl @@ -0,0 +1,95 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "delphic.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "delphic.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "delphic.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "delphic.labels" -}} +helm.sh/chart: {{ include "delphic.chart" . }} +{{ include "delphic.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "delphic.selectorLabels" -}} +app.kubernetes.io/name: {{ include "delphic.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "delphic.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "delphic.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "delphic.common.envs" }} +- name: REDIS_URL + value: redis://{{ .Release.Name }}-{{ .Values.redis.nameOverride }}-redis:6379 +- name: MODEL_NAME + value: text-davinci-003 +- name: MAX_TOKENS + value: "512" +- name: USE_DOCKER + value: "yes" +- name: POSTGRES_HOST + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-{{ .Values.postgres.nameOverride }}-conn-credential + key: host +- name: POSTGRES_PORT + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-{{ .Values.postgres.nameOverride }}-conn-credential + key: port +- name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-{{ .Values.postgres.nameOverride }}-conn-credential + key: username +- name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-{{ .Values.postgres.nameOverride }}-conn-credential + key: password +- name: POSTGRES_DB + value: delphic +{{- end }} diff --git a/deploy/delphic/templates/deployment.yaml b/deploy/delphic/templates/deployment.yaml new file mode 100644 index 000000000..d56afe6e4 --- /dev/null +++ b/deploy/delphic/templates/deployment.yaml @@ -0,0 +1,115 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "delphic.fullname" . }} + labels: + {{- include "delphic.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "delphic.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "delphic.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "delphic.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + - name: postgres-init + image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.postgres.repository }}:{{ .Values.image.postgres.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + PGPASSWORD=${POSTGRES_PASSWORD} psql -h${POSTGRES_HOST} -U ${POSTGRES_USER} -p ${POSTGRES_PORT} -tc "SELECT 1 FROM pg_database WHERE datname = '${POSTGRES_DB}'" + if [ $? != 0 ]; then + PGPASSWORD=${POSTGRES_PASSWORD} psql -h${POSTGRES_HOST} -U ${POSTGRES_USER} -p ${POSTGRES_PORT} -c "CREATE DATABASE ${POSTGRES_DB}" + fi + env: + {{ include "delphic.common.envs" . | nindent 12}} + containers: + - name: django + command: + - /entrypoint + - /start + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + env: + - name: OPENAI_API_KEY + value: {{ .Values.openai_api_key }} + {{ include "delphic.common.envs" . | nindent 12}} + - name: frontend + image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.frontend.repository }}:{{ .Values.image.frontend.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.frontend.resources | nindent 12 }} + - name: celeryworker + command: + - /entrypoint + - /start-celeryworker + image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.celeryWorker.resources | nindent 12 }} + env: + {{ include "delphic.common.envs" . | nindent 12}} + - name: celerybeat + command: + - /entrypoint + - /start-celerybeat + image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.celeryBeat.resources | nindent 12 }} + env: + {{ include "delphic.common.envs" . | nindent 12}} + - name: flower + command: + - /entrypoint + - /start-flower + image: "{{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + resources: + {{- toYaml .Values.flower.resources | nindent 12 }} + env: + - name: CELERY_FLOWER_USER + valueFrom: + secretKeyRef: + name: {{ include "delphic.fullname" . }}-celery-flower-secret + key: username + - name: CELERY_FLOWER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "delphic.fullname" . }}-celery-flower-secret + key: password + {{ include "delphic.common.envs" . | nindent 12}} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/delphic/templates/ingress.yaml b/deploy/delphic/templates/ingress.yaml new file mode 100644 index 000000000..118372551 --- /dev/null +++ b/deploy/delphic/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "delphic.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "delphic.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/delphic/templates/job.yaml b/deploy/delphic/templates/job.yaml new file mode 100644 index 000000000..2414ee767 --- /dev/null +++ b/deploy/delphic/templates/job.yaml @@ -0,0 +1,53 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "delphic.fullname" . }}-create-django-user + labels: + {{- include "delphic.labels" . | nindent 4 }} +spec: + ttlSecondsAfterFinished: 3600 + template: + metadata: + name: {{ include "delphic.fullname" . }}-create-django-user + labels: + {{- include "delphic.labels" . | nindent 8 }} + spec: + serviceAccountName: {{ include "delphic.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + restartPolicy: OnFailure + containers: + - name: post-install-job + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + /entrypoint python manage.py createsuperuser --noinput + env: + - name: DJANGO_SUPERUSER_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "delphic.fullname" . }}-django-secret + key: username + - name: DJANGO_SUPERUSER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "delphic.fullname" . }}-django-secret + key: password + - name: DJANGO_SUPERUSER_EMAIL + value: admin@admin.org + {{ include "delphic.common.envs" . | nindent 10}} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/deploy/delphic/templates/secret.yaml b/deploy/delphic/templates/secret.yaml new file mode 100644 index 000000000..7fa049627 --- /dev/null +++ b/deploy/delphic/templates/secret.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "delphic.fullname" . }}-celery-flower-secret +type: Opaque +data: + # generate 32 chars long random string, base64 encode it and then double-quote the result string. + username: {{ randAlphaNum 32 | b64enc | quote }} + password: {{ randAlphaNum 64 | b64enc | quote }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "delphic.fullname" . }}-django-secret +type: Opaque +data: + username: {{ randAlphaNum 8 | b64enc | quote }} + password: {{ randAlphaNum 16 | b64enc | quote }} diff --git a/deploy/delphic/templates/service.yaml b/deploy/delphic/templates/service.yaml new file mode 100644 index 000000000..555711c67 --- /dev/null +++ b/deploy/delphic/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "delphic.fullname" . }} + labels: + {{- include "delphic.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "delphic.selectorLabels" . | nindent 4 }} diff --git a/deploy/delphic/templates/serviceaccount.yaml b/deploy/delphic/templates/serviceaccount.yaml new file mode 100644 index 000000000..365325816 --- /dev/null +++ b/deploy/delphic/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "delphic.serviceAccountName" . }} + labels: + {{- include "delphic.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/delphic/templates/tests/test-connection.yaml b/deploy/delphic/templates/tests/test-connection.yaml new file mode 100644 index 000000000..c1cf8c8f0 --- /dev/null +++ b/deploy/delphic/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "delphic.fullname" . }}-test-connection" + labels: + {{- include "delphic.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "delphic.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/deploy/delphic/values.yaml b/deploy/delphic/values.yaml new file mode 100644 index 000000000..056cb6aab --- /dev/null +++ b/deploy/delphic/values.yaml @@ -0,0 +1,107 @@ +# Default values for delphic. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + registry: registry.cn-hangzhou.aliyuncs.com + repository: apecloud/delphic + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: v1.0.0 + frontend: + repository: apecloud/delphic-frontend + tag: v1.0.0 + + postgres: + repository: apecloud/spilo + tag: 14.7.1 + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +postgres: + enabled: true + nameOverride: postgres + +redis: + enabled: true + nameOverride: redis + +celeryWorker: + resources: {} + +celeryBeat: + resources: {} + +flower: + resources: {} + +frontend: + resources: {} + +openai_api_key: "" + + diff --git a/deploy/etcd-cluster/templates/cluster.yaml b/deploy/etcd-cluster/templates/cluster.yaml index c27b9b0a9..fb8fdd42f 100644 --- a/deploy/etcd-cluster/templates/cluster.yaml +++ b/deploy/etcd-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "etcd-cluster.fullname" . }} labels: {{- include "etcd-cluster.labels" . | nindent 4 }} spec: diff --git a/deploy/etcd-cluster/values.yaml b/deploy/etcd-cluster/values.yaml index 8704e3a62..1b6219123 100644 --- a/deploy/etcd-cluster/values.yaml +++ b/deploy/etcd-cluster/values.yaml @@ -173,4 +173,4 @@ ingress: ## port: ## name: http ## - extraRules: [] + extraRules: [] \ No newline at end of file diff --git a/deploy/kafka-cluster/templates/cluster.yaml b/deploy/kafka-cluster/templates/cluster.yaml index 788f7f938..a740cc4e8 100644 --- a/deploy/kafka-cluster/templates/cluster.yaml +++ b/deploy/kafka-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "kafka-cluster.fullname" . }} labels: {{ include "kafka-cluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: kafka # ref clusterdefinition.name diff --git a/deploy/kafka-cluster/values.yaml b/deploy/kafka-cluster/values.yaml index 17c1cc836..03554d0d6 100644 --- a/deploy/kafka-cluster/values.yaml +++ b/deploy/kafka-cluster/values.yaml @@ -186,3 +186,6 @@ topologyKeys: ## @param affinity is affinity setting for Kafka cluster pods assignment ## affinity: {} + +nameOverride: "" +fullnameOverride: "" diff --git a/deploy/milvus-cluster/templates/cluster.yaml b/deploy/milvus-cluster/templates/cluster.yaml index 5ed812892..f87bb2696 100644 --- a/deploy/milvus-cluster/templates/cluster.yaml +++ b/deploy/milvus-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "milvus.fullname" . }} labels: {{ include "milvus.labels" . | nindent 4 }} spec: clusterDefinitionRef: milvus # ref clusterdefinition.name diff --git a/deploy/milvus-cluster/values.yaml b/deploy/milvus-cluster/values.yaml index 3f14e3c39..3b7a70406 100644 --- a/deploy/milvus-cluster/values.yaml +++ b/deploy/milvus-cluster/values.yaml @@ -2,6 +2,9 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +nameOverride: "" +fullnameOverride: "" + replicaCount: 1 terminationPolicy: Delete diff --git a/deploy/mongodb-cluster/templates/cluster.yaml b/deploy/mongodb-cluster/templates/cluster.yaml index b2097c645..088f0f6b1 100644 --- a/deploy/mongodb-cluster/templates/cluster.yaml +++ b/deploy/mongodb-cluster/templates/cluster.yaml @@ -2,7 +2,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }}-sharding + name: {{ include "mongodb-cluster.fullname" . }}-sharding labels: {{- include "mongodb-cluster.labels" . | nindent 4}} spec: diff --git a/deploy/postgresql-cluster/templates/cluster.yaml b/deploy/postgresql-cluster/templates/cluster.yaml index eb9d86a94..5612ee13c 100644 --- a/deploy/postgresql-cluster/templates/cluster.yaml +++ b/deploy/postgresql-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "postgresqlcluster.fullname" . }} labels: {{ include "postgresqlcluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: postgresql # ref clusterdefinition.name diff --git a/deploy/postgresql-cluster/values.yaml b/deploy/postgresql-cluster/values.yaml index cd6d32006..90206a091 100644 --- a/deploy/postgresql-cluster/values.yaml +++ b/deploy/postgresql-cluster/values.yaml @@ -2,6 +2,9 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +nameOverride: "" +fullnameOverride: "" + replicaCount: 2 terminationPolicy: Delete diff --git a/deploy/qdrant-cluster/templates/cluster.yaml b/deploy/qdrant-cluster/templates/cluster.yaml index 5af9fad6a..b8419d8f2 100644 --- a/deploy/qdrant-cluster/templates/cluster.yaml +++ b/deploy/qdrant-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "qdrant.fullname" . }} labels: {{ include "qdrant.labels" . | nindent 4 }} spec: clusterDefinitionRef: qdrant # ref clusterdefinition.name diff --git a/deploy/qdrant-cluster/values.yaml b/deploy/qdrant-cluster/values.yaml index 3f14e3c39..73e11fc22 100644 --- a/deploy/qdrant-cluster/values.yaml +++ b/deploy/qdrant-cluster/values.yaml @@ -6,6 +6,8 @@ replicaCount: 1 terminationPolicy: Delete clusterVersionOverride: "" +nameOverride: "" +fullnameOverride: "" monitor: enabled: false diff --git a/deploy/redis-cluster/templates/cluster.yaml b/deploy/redis-cluster/templates/cluster.yaml index d24ee37d9..fdd3a3ff0 100644 --- a/deploy/redis-cluster/templates/cluster.yaml +++ b/deploy/redis-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "redis-cluster.fullname" . }} labels: {{ include "redis-cluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: redis # ref clusterDefinition.name diff --git a/deploy/redis-cluster/values.yaml b/deploy/redis-cluster/values.yaml index 896990c2a..5624084fc 100644 --- a/deploy/redis-cluster/values.yaml +++ b/deploy/redis-cluster/values.yaml @@ -2,6 +2,9 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +nameOverride: "" +fullnameOverride: "" + replicaCount: 2 sentinelReplicaCount: 3 diff --git a/deploy/weaviate-cluster/templates/cluster.yaml b/deploy/weaviate-cluster/templates/cluster.yaml index c992f299e..e714797e9 100644 --- a/deploy/weaviate-cluster/templates/cluster.yaml +++ b/deploy/weaviate-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ .Release.Name }} + name: {{ include "weaviate.fullname" . }} labels: {{ include "weaviate.labels" . | nindent 4 }} spec: clusterDefinitionRef: weaviate # ref clusterdefinition.name diff --git a/deploy/weaviate-cluster/values.yaml b/deploy/weaviate-cluster/values.yaml index 3f14e3c39..73e11fc22 100644 --- a/deploy/weaviate-cluster/values.yaml +++ b/deploy/weaviate-cluster/values.yaml @@ -6,6 +6,8 @@ replicaCount: 1 terminationPolicy: Delete clusterVersionOverride: "" +nameOverride: "" +fullnameOverride: "" monitor: enabled: false From 26cebc0b55b89523e774a784ff594a2553ae966c Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Thu, 4 May 2023 14:50:30 +0800 Subject: [PATCH 215/439] fix: some command examples can not be executed (#3034) --- docs/user_docs/cli/kbcli_addon_enable.md | 5 +++-- docs/user_docs/cli/kbcli_cluster_create.md | 19 ++++++++++++------- internal/cli/cmd/addon/addon.go | 5 +++-- internal/cli/cmd/cluster/create.go | 19 ++++++++++++------- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/user_docs/cli/kbcli_addon_enable.md b/docs/user_docs/cli/kbcli_addon_enable.md index 5d847c991..aefdf67c7 100644 --- a/docs/user_docs/cli/kbcli_addon_enable.md +++ b/docs/user_docs/cli/kbcli_addon_enable.md @@ -19,10 +19,11 @@ kbcli addon enable ADDON_NAME [flags] # Enabled "prometheus" addon and its extra alertmanager component with custom resources settings kbcli addon enable prometheus --memory 512Mi/4Gi --storage 8Gi --replicas 2 \ - --memory alertmanager:16Mi/256Mi --storage: alertmanager:1Gi --replicas alertmanager:2 + --memory alertmanager:16Mi/256Mi --storage alertmanager:1Gi --replicas alertmanager:2 # Enabled "prometheus" addon with tolerations - kbcli addon enable prometheus --tolerations '[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' \ + kbcli addon enable prometheus \ + --tolerations '[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' \ --tolerations 'alertmanager:[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' # Enabled "prometheus" addon with helm like custom settings diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index 9a3c488c1..0dfc90c55 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -18,9 +18,10 @@ kbcli cluster create [NAME] [flags] kbcli cluster create mycluster --cluster-definition apecloud-mysql # Output resource information in YAML format, but do not create resources. - kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run=client -o yaml + kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run -o yaml - # Output resource information in YAML format, the information will be sent to the server, but the resource will not be actually created. + # Output resource information in YAML format, the information will be sent to the server + # but the resource will not be actually created. kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run=server -o yaml # Create a cluster and set termination policy DoNotTerminate that will prevent the cluster from being deleted @@ -41,27 +42,31 @@ kbcli cluster create [NAME] [flags] # Create a cluster and set cpu to 1 core, memory to 1Gi, storage size to 20Gi and replicas to 3 kbcli cluster create mycluster --cluster-definition apecloud-mysql --set cpu=1,memory=1Gi,storage=20Gi,replicas=3 - # Create a cluster and set the class to general-1c1g, valid classes can be found by executing the command "kbcli class list --cluster-definition=" + # Create a cluster and set the class to general-1c1g + # run "kbcli class list --cluster-definition=cluster-definition-name" to get the class list kbcli cluster create mycluster --cluster-definition apecloud-mysql --set class=general-1c1g # Create a cluster with replicationSet workloadType and set switchPolicy to Noop kbcli cluster create mycluster --cluster-definition postgresql --set switchPolicy=Noop # Create a cluster and use a URL to set cluster resource - kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file https://kubeblocks.io/yamls/apecloud-mysql.yaml + kbcli cluster create mycluster --cluster-definition apecloud-mysql \ + --set-file https://kubeblocks.io/yamls/apecloud-mysql.yaml # Create a cluster and load cluster resource set from stdin cat << EOF | kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file - - name: my-test ... # Create a cluster forced to scatter by node - kbcli cluster create --cluster-definition apecloud-mysql --topology-keys kubernetes.io/hostname --pod-anti-affinity Required + kbcli cluster create --cluster-definition apecloud-mysql --topology-keys kubernetes.io/hostname \ + --pod-anti-affinity Required # Create a cluster in specific labels nodes - kbcli cluster create --cluster-definition apecloud-mysql --node-labels '"topology.kubernetes.io/zone=us-east-1a","disktype=ssd,essd"' + kbcli cluster create --cluster-definition apecloud-mysql \ + --node-labels '"topology.kubernetes.io/zone=us-east-1a","disktype=ssd,essd"' # Create a Cluster with two tolerations - kbcli cluster create --cluster-definition apecloud-mysql --tolerations '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule","key=diskType,value=ssd,operator=Equal,effect=NoSchedule"' + kbcli cluster create --cluster-definition apecloud-mysql --tolerations \ '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule","key=diskType,value=ssd,operator=Equal,effect=NoSchedule"' # Create a cluster, with each pod runs on their own dedicated node kbcli cluster create --cluster-definition apecloud-mysql --tenancy=DedicatedNode diff --git a/internal/cli/cmd/addon/addon.go b/internal/cli/cmd/addon/addon.go index 990abd2a6..a325f6355 100644 --- a/internal/cli/cmd/addon/addon.go +++ b/internal/cli/cmd/addon/addon.go @@ -176,10 +176,11 @@ func newEnableCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra # Enabled "prometheus" addon and its extra alertmanager component with custom resources settings kbcli addon enable prometheus --memory 512Mi/4Gi --storage 8Gi --replicas 2 \ - --memory alertmanager:16Mi/256Mi --storage: alertmanager:1Gi --replicas alertmanager:2 + --memory alertmanager:16Mi/256Mi --storage alertmanager:1Gi --replicas alertmanager:2 # Enabled "prometheus" addon with tolerations - kbcli addon enable prometheus --tolerations '[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' \ + kbcli addon enable prometheus \ + --tolerations '[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' \ --tolerations 'alertmanager:[{"key":"taintkey","operator":"Equal","effect":"NoSchedule","value":"true"}]' # Enabled "prometheus" addon with helm like custom settings diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 6c575e76b..7af43c5ee 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -67,9 +67,10 @@ var clusterCreateExample = templates.Examples(` kbcli cluster create mycluster --cluster-definition apecloud-mysql # Output resource information in YAML format, but do not create resources. - kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run=client -o yaml + kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run -o yaml - # Output resource information in YAML format, the information will be sent to the server, but the resource will not be actually created. + # Output resource information in YAML format, the information will be sent to the server + # but the resource will not be actually created. kbcli cluster create mycluster --cluster-definition apecloud-mysql --dry-run=server -o yaml # Create a cluster and set termination policy DoNotTerminate that will prevent the cluster from being deleted @@ -90,27 +91,31 @@ var clusterCreateExample = templates.Examples(` # Create a cluster and set cpu to 1 core, memory to 1Gi, storage size to 20Gi and replicas to 3 kbcli cluster create mycluster --cluster-definition apecloud-mysql --set cpu=1,memory=1Gi,storage=20Gi,replicas=3 - # Create a cluster and set the class to general-1c1g, valid classes can be found by executing the command "kbcli class list --cluster-definition=" + # Create a cluster and set the class to general-1c1g + # run "kbcli class list --cluster-definition=cluster-definition-name" to get the class list kbcli cluster create mycluster --cluster-definition apecloud-mysql --set class=general-1c1g # Create a cluster with replicationSet workloadType and set switchPolicy to Noop kbcli cluster create mycluster --cluster-definition postgresql --set switchPolicy=Noop # Create a cluster and use a URL to set cluster resource - kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file https://kubeblocks.io/yamls/apecloud-mysql.yaml + kbcli cluster create mycluster --cluster-definition apecloud-mysql \ + --set-file https://kubeblocks.io/yamls/apecloud-mysql.yaml # Create a cluster and load cluster resource set from stdin cat << EOF | kbcli cluster create mycluster --cluster-definition apecloud-mysql --set-file - - name: my-test ... # Create a cluster forced to scatter by node - kbcli cluster create --cluster-definition apecloud-mysql --topology-keys kubernetes.io/hostname --pod-anti-affinity Required + kbcli cluster create --cluster-definition apecloud-mysql --topology-keys kubernetes.io/hostname \ + --pod-anti-affinity Required # Create a cluster in specific labels nodes - kbcli cluster create --cluster-definition apecloud-mysql --node-labels '"topology.kubernetes.io/zone=us-east-1a","disktype=ssd,essd"' + kbcli cluster create --cluster-definition apecloud-mysql \ + --node-labels '"topology.kubernetes.io/zone=us-east-1a","disktype=ssd,essd"' # Create a Cluster with two tolerations - kbcli cluster create --cluster-definition apecloud-mysql --tolerations '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule","key=diskType,value=ssd,operator=Equal,effect=NoSchedule"' + kbcli cluster create --cluster-definition apecloud-mysql --tolerations \ '"key=engineType,value=mongo,operator=Equal,effect=NoSchedule","key=diskType,value=ssd,operator=Equal,effect=NoSchedule"' # Create a cluster, with each pod runs on their own dedicated node kbcli cluster create --cluster-definition apecloud-mysql --tenancy=DedicatedNode From 297da380a324f36d9d25c760c09025ed659b59c1 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Thu, 4 May 2023 15:47:12 +0800 Subject: [PATCH 216/439] fix: update redis persistence dir to /data (#3051) --- deploy/redis/config/redis7-config.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/redis/config/redis7-config.tpl b/deploy/redis/config/redis7-config.tpl index d7d8ebada..f9ff9a4cf 100644 --- a/deploy/redis/config/redis7-config.tpl +++ b/deploy/redis/config/redis7-config.tpl @@ -18,7 +18,7 @@ rdbcompression yes rdbchecksum yes dbfilename dump.rdb rdb-del-sync-files no -dir ./ +dir /data replica-serve-stale-data yes replica-read-only yes repl-diskless-sync yes From 6554870df67585c5878add1276cdacd9d421448f Mon Sep 17 00:00:00 2001 From: shanshanying Date: Thu, 4 May 2023 19:34:34 +0800 Subject: [PATCH 217/439] fix: clusterversion specify client image (#3030) --- apis/apps/v1alpha1/clusterversion_types.go | 5 +++++ .../apps.kubeblocks.io_clusterversions.yaml | 5 +++++ controllers/apps/systemaccount_controller.go | 17 +++++++++++++---- controllers/apps/systemaccount_util.go | 11 +++++++++++ .../apps.kubeblocks.io_clusterversions.yaml | 5 +++++ deploy/postgresql/templates/clusterversion.yaml | 2 ++ 6 files changed, 41 insertions(+), 4 deletions(-) diff --git a/apis/apps/v1alpha1/clusterversion_types.go b/apis/apps/v1alpha1/clusterversion_types.go index 810e8ed73..0af45846c 100644 --- a/apis/apps/v1alpha1/clusterversion_types.go +++ b/apis/apps/v1alpha1/clusterversion_types.go @@ -82,6 +82,11 @@ type ClusterComponentVersion struct { // +listMapKey=name ConfigSpecs []ComponentConfigSpec `json:"configSpecs,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"name"` + // clientImage define image for the component to connect database or engines. + // This value has a higher proirity over ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig.image. + // +optional + ClientImage string `json:"clientImage,omitempty"` + // versionContext defines containers images' context for component versions, // this value replaces ClusterDefinition.spec.componentDefs.podSpec.[initContainers | containers] VersionsCtx VersionsContext `json:"versionsContext"` diff --git a/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml index f023b9d1f..1083c4526 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml @@ -62,6 +62,11 @@ spec: description: ClusterComponentVersion is an application version component spec. properties: + clientImage: + description: clientImage define image for the component to connect + database or engines. This value has a higher proirity over + ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig.image. + type: string componentDefRef: description: componentDefRef reference one of the cluster component definition names in ClusterDefinition API (spec.componentDefs.name). diff --git a/controllers/apps/systemaccount_controller.go b/controllers/apps/systemaccount_controller.go index af2dd29b9..15b00ebcd 100644 --- a/controllers/apps/systemaccount_controller.go +++ b/controllers/apps/systemaccount_controller.go @@ -138,18 +138,25 @@ func (r *SystemAccountReconciler) Reconcile(ctx context.Context, req ctrl.Reques return intctrlutil.Reconciled() } + // wait till the cluster is running + if cluster.Status.Phase != appsv1alpha1.RunningClusterPhase { + reqCtx.Log.V(1).Info("Cluster is not ready yet", "cluster", req.NamespacedName) + return intctrlutil.Reconciled() + } + clusterdefinition := &appsv1alpha1.ClusterDefinition{} clusterDefNS := types.NamespacedName{Name: cluster.Spec.ClusterDefRef} if err := r.Client.Get(reqCtx.Ctx, clusterDefNS, clusterdefinition); err != nil { return intctrlutil.RequeueWithErrorAndRecordEvent(cluster, r.Recorder, err, reqCtx.Log) } - // wait till the cluster is running - if cluster.Status.Phase != appsv1alpha1.RunningClusterPhase { - reqCtx.Log.V(1).Info("Cluster is not ready yet", "cluster", req.NamespacedName) - return intctrlutil.Reconciled() + clusterVersion := &appsv1alpha1.ClusterVersion{} + if err := r.Client.Get(reqCtx.Ctx, types.NamespacedName{Name: cluster.Spec.ClusterVersionRef}, clusterVersion); err != nil { + return intctrlutil.RequeueWithErrorAndRecordEvent(cluster, r.Recorder, err, reqCtx.Log) } + componentVersions := clusterVersion.Spec.GetDefNameMappingComponents() + // process accounts per component processAccountsForComponent := func(compDef *appsv1alpha1.ClusterComponentDefinition, compDecl *appsv1alpha1.ClusterComponentSpec, svcEP *corev1.Endpoints, headlessEP *corev1.Endpoints) error { @@ -205,6 +212,8 @@ func (r *SystemAccountReconciler) Reconcile(ctx context.Context, req ctrl.Reques case appsv1alpha1.CreateByStmt: if engine == nil { execConfig := compDef.SystemAccounts.CmdExecutorConfig + // complete execConfig with settings from component version + completeExecConfig(execConfig, componentVersions[compDef.Name]) engine = newCustomizedEngine(execConfig, cluster, compDecl.Name) } if err := r.createByStmt(reqCtx, cluster, compDef, compKey, engine, account, svcEP, headlessEP, strategy); err != nil { diff --git a/controllers/apps/systemaccount_util.go b/controllers/apps/systemaccount_util.go index a18f2d303..6f1dae93c 100644 --- a/controllers/apps/systemaccount_util.go +++ b/controllers/apps/systemaccount_util.go @@ -390,3 +390,14 @@ func calibrateJobMetaAndSpec(job *batchv1.Job, cluster *appsv1alpha1.Cluster, co tolerations = componetutil.PatchBuiltInToleration(tolerations) job.Spec.Template.Spec.Tolerations = tolerations } + +// completeExecConfig override the image of execConfig if version is not nil. +func completeExecConfig(execConfig *appsv1alpha1.CmdExecutorConfig, version *appsv1alpha1.ClusterComponentVersion) { + if version == nil { + return + } + if len(version.ClientImage) == 0 { + return + } + execConfig.Image = version.ClientImage +} diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml index f023b9d1f..1083c4526 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml @@ -62,6 +62,11 @@ spec: description: ClusterComponentVersion is an application version component spec. properties: + clientImage: + description: clientImage define image for the component to connect + database or engines. This value has a higher proirity over + ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig.image. + type: string componentDefRef: description: componentDefRef reference one of the cluster component definition names in ClusterDefinition API (spec.componentDefs.name). diff --git a/deploy/postgresql/templates/clusterversion.yaml b/deploy/postgresql/templates/clusterversion.yaml index 4800dac3e..0a4538898 100644 --- a/deploy/postgresql/templates/clusterversion.yaml +++ b/deploy/postgresql/templates/clusterversion.yaml @@ -16,6 +16,7 @@ spec: containers: - name: postgresql image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + clientImage: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} --- apiVersion: apps.kubeblocks.io/v1alpha1 @@ -45,4 +46,5 @@ spec: containers: - name: postgresql image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0 + clientImage: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0 --- From c3fed87ea402cb8463de9193894373e5abb4b981 Mon Sep 17 00:00:00 2001 From: chantu Date: Thu, 4 May 2023 19:35:22 +0800 Subject: [PATCH 218/439] fix: role change error when restart (#3054) --- deploy/apecloud-mysql/templates/scripts.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index df3534a13..62669181f 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -43,6 +43,13 @@ data: fi fi /scripts/upgrade-learner.sh & + else + # if self is in topology and role is learner, upgrade it to follower + # sometimes pod crash before upgrade learner finished + is_learner=`echo $in_topology | grep "Learner"` + if [ ! -z "$is_learner" ]; then + /scripts/upgrade-learner.sh & + fi fi fi cluster_info=""; From e342e0c3bc7fb3aa25120528914b1cad104b520b Mon Sep 17 00:00:00 2001 From: runsun Date: Thu, 4 May 2023 19:50:16 +0800 Subject: [PATCH 219/439] chore: prometheus and alertmanager use emptyDir by default (#3061) --- deploy/helm/values.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index e6fa0d34f..887a7504a 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -323,7 +323,7 @@ prometheus: ## If true, alertmanager will create/use a Persistent Volume Claim ## If false, use emptyDir ## - enabled: true + enabled: false ## alertmanager data Persistent Volume size ## @@ -346,7 +346,7 @@ prometheus: ## If true, use a statefulset instead of a deployment for pod management. ## This allows to scale replicas to more than 1 pod ## - enabled: true + enabled: false ## Alertmanager headless service to use for the statefulset ## @@ -530,7 +530,7 @@ prometheus: ## If true, Prometheus server will create/use a Persistent Volume Claim ## If false, use emptyDir ## - enabled: true + enabled: false ## Prometheus server data Persistent Volume size ## @@ -553,7 +553,7 @@ prometheus: ## If true, use a statefulset instead of a deployment for pod management. ## This allows to scale replicas to more than 1 pod ## - enabled: true + enabled: false ## Prometheus server resource requests and limits ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ From e9b9b329b93925bd28fbf22009866d3094eda57c Mon Sep 17 00:00:00 2001 From: runsun Date: Thu, 4 May 2023 19:50:28 +0800 Subject: [PATCH 220/439] fix: delete the default storageclass alicloud-disk-efficiency in addon (#2996) --- deploy/helm/templates/addons/grafana-addon.yaml | 1 - deploy/helm/templates/addons/prometheus-addon.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/deploy/helm/templates/addons/grafana-addon.yaml b/deploy/helm/templates/addons/grafana-addon.yaml index 2083a5692..0cbc01839 100644 --- a/deploy/helm/templates/addons/grafana-addon.yaml +++ b/deploy/helm/templates/addons/grafana-addon.yaml @@ -55,7 +55,6 @@ spec: values: - aliyun replicas: 1 - storageClass: alicloud-disk-efficiency resources: requests: storage: 20Gi diff --git a/deploy/helm/templates/addons/prometheus-addon.yaml b/deploy/helm/templates/addons/prometheus-addon.yaml index 294165901..a66f0f39b 100644 --- a/deploy/helm/templates/addons/prometheus-addon.yaml +++ b/deploy/helm/templates/addons/prometheus-addon.yaml @@ -86,7 +86,6 @@ spec: values: - aliyun replicas: 1 - storageClass: alicloud-disk-efficiency resources: requests: storage: 20Gi @@ -100,7 +99,6 @@ spec: extras: - name: alertmanager replicas: 1 - storageClass: alicloud-disk-efficiency resources: requests: storage: 20Gi From dff408d029833c723e12194b93cb58eb7b788cd0 Mon Sep 17 00:00:00 2001 From: runsun Date: Thu, 4 May 2023 20:57:46 +0800 Subject: [PATCH 221/439] chore: fix some minor issues for dashboards (#3067) --- deploy/helm/dashboards/cadvisor-exporter.json | 78 ++++++++++++------- deploy/helm/dashboards/mysql-overview.json | 59 +++++++------- .../helm/dashboards/postgresql-overview.json | 58 +++++++------- 3 files changed, 107 insertions(+), 88 deletions(-) diff --git a/deploy/helm/dashboards/cadvisor-exporter.json b/deploy/helm/dashboards/cadvisor-exporter.json index d8793e6dd..8b8176931 100644 --- a/deploy/helm/dashboards/cadvisor-exporter.json +++ b/deploy/helm/dashboards/cadvisor-exporter.json @@ -26,7 +26,6 @@ "fiscalYearStartMonth": 0, "gnetId": 14282, "graphTooltip": 0, - "id": 5, "links": [ { "asDropdown": false, @@ -110,7 +109,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -210,7 +210,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -310,7 +311,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -410,7 +412,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -510,7 +513,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -610,7 +614,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -710,7 +715,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -810,7 +816,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -910,7 +917,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1010,7 +1018,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1110,7 +1119,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2000,7 +2010,7 @@ "refId": "A" } ], - "title": "Read Network Traffic", + "title": "Send Network Traffic", "type": "timeseries" }, { @@ -2260,7 +2270,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2276,7 +2287,7 @@ "h": 6, "w": 8, "x": 0, - "y": 27 + "y": 3 }, "id": 43, "options": { @@ -2360,7 +2371,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2376,7 +2388,7 @@ "h": 6, "w": 8, "x": 8, - "y": 27 + "y": 3 }, "id": 44, "options": { @@ -2460,7 +2472,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2476,7 +2489,7 @@ "h": 6, "w": 8, "x": 16, - "y": 27 + "y": 3 }, "id": 45, "options": { @@ -2560,7 +2573,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2576,7 +2590,7 @@ "h": 6, "w": 8, "x": 0, - "y": 33 + "y": 9 }, "id": 46, "options": { @@ -2660,7 +2674,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2676,7 +2691,7 @@ "h": 6, "w": 8, "x": 8, - "y": 33 + "y": 9 }, "id": 48, "options": { @@ -2760,7 +2775,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2776,7 +2792,7 @@ "h": 6, "w": 8, "x": 16, - "y": 33 + "y": 9 }, "id": 47, "options": { @@ -2860,7 +2876,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2876,7 +2893,7 @@ "h": 6, "w": 8, "x": 0, - "y": 39 + "y": 15 }, "id": 49, "options": { @@ -2960,7 +2977,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2976,7 +2994,7 @@ "h": 6, "w": 8, "x": 8, - "y": 39 + "y": 15 }, "id": 50, "options": { @@ -3174,4 +3192,4 @@ "uid": "pMEd7m0Mz", "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/deploy/helm/dashboards/mysql-overview.json b/deploy/helm/dashboards/mysql-overview.json index 0dc70dbdc..a0fe2fbb7 100644 --- a/deploy/helm/dashboards/mysql-overview.json +++ b/deploy/helm/dashboards/mysql-overview.json @@ -26,7 +26,6 @@ "fiscalYearStartMonth": 0, "gnetId": 11323, "graphTooltip": 1, - "id": 3, "links": [ { "asDropdown": false, @@ -150,13 +149,15 @@ "datasourceErrors": {}, "editorMode": "code", "errors": {}, + "exemplar": false, "expr": "sum(mysql_global_status_uptime{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}) by (namespace,app_kubernetes_io_instance,pod)", "format": "time_series", + "instant": true, "interval": "1m", "intervalFactor": 1, "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", "metric": "", - "range": true, + "range": false, "refId": "A", "step": 300 } @@ -166,6 +167,7 @@ }, { "datasource": { + "type": "prometheus", "uid": "$datasource" }, "description": "**Current QPS**\n\nBased on the queries reported by MySQL's ``SHOW STATUS`` command, it is the number of statements executed by the server within the last second. This variable includes statements executed within stored programs, unlike the Questions variable. It does not count \n``COM_PING`` or ``COM_STATISTICS`` commands.", @@ -237,13 +239,17 @@ "uid": "$datasource" }, "datasourceErrors": {}, + "editorMode": "code", "errors": {}, + "exemplar": false, "expr": "sum(rate(mysql_global_status_queries{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}[$__rate_interval])) by (namespace,app_kubernetes_io_instance,pod)", "format": "time_series", + "instant": true, "interval": "1m", "intervalFactor": 1, "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", "metric": "", + "range": false, "refId": "A", "step": 20 } @@ -253,6 +259,7 @@ }, { "datasource": { + "type": "prometheus", "uid": "$datasource" }, "description": "**InnoDB Buffer Pool Size**\n\nInnoDB maintains a storage area called the buffer pool for caching data and indexes in memory. Knowing how the InnoDB buffer pool works, and taking advantage of it to keep frequently accessed data in memory, is one of the most important aspects of MySQL tuning. The goal is to keep the working set in memory. In most cases, this should be between 60%-90% of available memory on a dedicated database host, but depends on many factors.", @@ -324,13 +331,17 @@ "uid": "$datasource" }, "datasourceErrors": {}, + "editorMode": "code", "errors": {}, + "exemplar": false, "expr": "sum(mysql_global_variables_innodb_buffer_pool_size{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}) by (namespace,app_kubernetes_io_instance,pod)", "format": "time_series", + "instant": true, "interval": "1m", "intervalFactor": 1, "legendFormat": "{{namespace}} | {{app_kubernetes_io_instance}} | {{pod}}", "metric": "", + "range": false, "refId": "A", "step": 300 } @@ -1069,8 +1080,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1205,8 +1215,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1401,8 +1410,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1556,8 +1564,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1688,8 +1695,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1812,8 +1818,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2347,8 +2352,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2461,8 +2465,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2570,8 +2573,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2702,8 +2704,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2807,8 +2808,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2964,8 +2964,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3137,8 +3136,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3290,8 +3288,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4133,4 +4130,4 @@ "uid": "549c2bf8936f7767ea6ac47c47b00f2a", "version": 1, "weekStart": "" -} +} \ No newline at end of file diff --git a/deploy/helm/dashboards/postgresql-overview.json b/deploy/helm/dashboards/postgresql-overview.json index bd409740a..0671209ed 100644 --- a/deploy/helm/dashboards/postgresql-overview.json +++ b/deploy/helm/dashboards/postgresql-overview.json @@ -26,7 +26,6 @@ "fiscalYearStartMonth": 0, "gnetId": 11323, "graphTooltip": 1, - "id": 13, "links": [ { "asDropdown": false, @@ -141,7 +140,7 @@ "datasourceErrors": {}, "editorMode": "code", "errors": {}, - "expr": "count(sum by(namespace)(pg_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}))", + "expr": "count(sum by(namespace)(pg_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})) or vector(0)", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -217,7 +216,7 @@ "datasourceErrors": {}, "editorMode": "code", "errors": {}, - "expr": "count(sum by(namespace, app_kubernetes_io_instance)(pg_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}))", + "expr": "count(sum by(namespace, app_kubernetes_io_instance)(pg_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})) or vector(0)", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -293,7 +292,7 @@ "datasourceErrors": {}, "editorMode": "code", "errors": {}, - "expr": "sum(rate(pg_stat_database_xact_commit{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}[$__rate_interval]) + rate(pg_stat_database_xact_rollback{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(rate(pg_stat_database_xact_commit{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}[$__rate_interval]) + rate(pg_stat_database_xact_rollback{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}[$__rate_interval])) or vector(0)", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -704,7 +703,7 @@ "datasourceErrors": {}, "editorMode": "code", "errors": {}, - "expr": "count(pg_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"} > 0)", + "expr": "count(pg_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"} > 0) or vector(0)", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -785,14 +784,14 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "count(pg_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"} <= 0)", + "expr": "count(pg_up{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"} <= 0) or vector(0)", "format": "time_series", - "instant": true, + "instant": false, "interval": "1m", "intervalFactor": 1, "legendFormat": "__auto", "metric": "", - "range": false, + "range": true, "refId": "A", "step": 20 } @@ -863,7 +862,7 @@ "datasourceErrors": {}, "editorMode": "code", "errors": {}, - "expr": "sum(rate(pg_stat_statements_calls{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}[$__rate_interval]))", + "expr": "sum(rate(pg_stat_statements_calls{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"}[$__rate_interval])) or vector(0)", "format": "time_series", "interval": "1m", "intervalFactor": 1, @@ -1495,7 +1494,8 @@ "mode": "absolute", "steps": [ { - "color": "dark-green" + "color": "dark-green", + "value": null } ] }, @@ -2483,7 +2483,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2499,7 +2500,7 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 33 }, "id": 432, "links": [], @@ -2594,7 +2595,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2610,7 +2612,7 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 33 }, "id": 434, "links": [], @@ -2705,7 +2707,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2721,7 +2724,7 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 41 }, "id": 433, "links": [], @@ -2816,7 +2819,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2832,7 +2836,7 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 41 }, "id": 435, "links": [], @@ -2927,7 +2931,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2943,7 +2948,7 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 49 }, "id": 436, "links": [], @@ -3038,7 +3043,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3054,7 +3060,7 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 49 }, "id": 437, "links": [], @@ -5415,8 +5421,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5520,8 +5525,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6092,4 +6096,4 @@ "uid": "5UxloIJVk", "version": 1, "weekStart": "" -} \ No newline at end of file +} From b17c62c8575830923d9a5393ea17e85eb55e2697 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Fri, 5 May 2023 09:35:08 +0800 Subject: [PATCH 222/439] fix: properties(dependency sdk) extensions can cause variables in the configuration file to be replaced (#3062) --- deploy/postgresql/config/pg12-config.tpl | 3 ++- deploy/postgresql/config/pg14-config.tpl | 3 ++- internal/unstructured/viper_wrap_test.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/postgresql/config/pg12-config.tpl b/deploy/postgresql/config/pg12-config.tpl index d84146de9..e173be8f4 100644 --- a/deploy/postgresql/config/pg12-config.tpl +++ b/deploy/postgresql/config/pg12-config.tpl @@ -26,7 +26,8 @@ listen_addresses = '*' port = '5432' -archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' +# archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' +archive_command = '[[ $(date +%H%M) == 1200 ]] && rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' archive_mode = 'on' auto_explain.log_analyze = 'True' auto_explain.log_min_duration = '1s' diff --git a/deploy/postgresql/config/pg14-config.tpl b/deploy/postgresql/config/pg14-config.tpl index c89467e27..5eff02923 100644 --- a/deploy/postgresql/config/pg14-config.tpl +++ b/deploy/postgresql/config/pg14-config.tpl @@ -26,7 +26,8 @@ listen_addresses = '*' port = '5432' -archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' +# archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' +archive_command = '[[ $(date +%H%M) == 1200 ]] && rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' archive_mode = 'on' auto_explain.log_analyze = 'True' auto_explain.log_min_duration = '1s' diff --git a/internal/unstructured/viper_wrap_test.go b/internal/unstructured/viper_wrap_test.go index 2ab1c9760..7043ef086 100644 --- a/internal/unstructured/viper_wrap_test.go +++ b/internal/unstructured/viper_wrap_test.go @@ -62,8 +62,8 @@ func TestPropertiesFormat(t *testing.T) { const propertiesContext = ` listen_addresses = '*' port = '5432' +archive_command = '[[ $(date +%H%M) == 1200 ]] && rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' -#archive_command = 'wal_dir=/pg/arcwal; [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); /bin/mkdir -p ${wal_dir}/$(date +%Y%m%d) && /usr/bin/lz4 -q -z %p > ${wal_dir}/$(date +%Y%m%d)/%f.lz4' #archive_mode = 'True' auto_explain.log_analyze = 'True' auto_explain.log_min_duration = '1s' @@ -81,6 +81,7 @@ autovacuum_naptime = '1min' assert.EqualValues(t, propConfigObj.Get("auto_explain.log_nested_statements"), "'True'") assert.EqualValues(t, propConfigObj.Get("auto_explain.log_min_duration"), "'1s'") assert.EqualValues(t, propConfigObj.Get("autovacuum_naptime"), "'1min'") + assert.EqualValues(t, propConfigObj.Get("archive_command"), `'[[ $(date +%H%M) == 1200 ]] && rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz'`) dumpContext, err := propConfigObj.Marshal() assert.Nil(t, err) From 0ae9e07f2c6fb40d6993a079736f05b4344102a1 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Fri, 5 May 2023 10:45:54 +0800 Subject: [PATCH 223/439] feat: have failed pod logs to Addon condition errors and events (#3058) --- .github/utils/bug_stats.sh | 20 ++ apis/extensions/v1alpha1/addon_types.go | 90 +++++- apis/extensions/v1alpha1/addon_types_test.go | 45 ++- .../apps/components/deployment_controller.go | 1 + controllers/apps/components/pod_controller.go | 2 +- .../components/stateful_set_controller.go | 1 + controllers/extensions/addon_controller.go | 36 ++- .../extensions/addon_controller_stages.go | 278 ++++++++++++------ .../extensions/addon_controller_test.go | 60 +++- controllers/extensions/const.go | 12 +- go.mod | 32 +- go.sum | 60 ++-- internal/cli/cmd/addon/addon.go | 18 -- internal/controllerutil/type.go | 6 + 14 files changed, 479 insertions(+), 182 deletions(-) create mode 100755 .github/utils/bug_stats.sh diff --git a/.github/utils/bug_stats.sh b/.github/utils/bug_stats.sh new file mode 100755 index 000000000..317bd2798 --- /dev/null +++ b/.github/utils/bug_stats.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -o errexit +set -o nounset + +# requires `git` and `gh` commands, ref. https://cli.github.com/manual/installation for installation guides. + +workdir=$(dirname $0) +. ${workdir}/gh_env +. ${workdir}/functions.bash + +bug_report_md_file=${1} + +crit_cnt=$(cat ${bug_report_md_file} | grep crit | wc -l) +major_cnt=$(cat ${bug_report_md_file} | grep major | wc -l) +minor_cnt=$(cat ${bug_report_md_file} | grep minor | wc -l) +total_cnt=$(cat ${bug_report_md_file} | wc -l) +total_cnt=$((total_cnt-2)) + +printf "bug stats\ntotal open: %s\ncritial: %s\nmajor: %s\nminor: %s\n" ${total_cnt} ${crit_cnt} ${major_cnt} ${minor_cnt} \ No newline at end of file diff --git a/apis/extensions/v1alpha1/addon_types.go b/apis/extensions/v1alpha1/addon_types.go index 9beb80be2..bdf0eed1b 100644 --- a/apis/extensions/v1alpha1/addon_types.go +++ b/apis/extensions/v1alpha1/addon_types.go @@ -233,6 +233,26 @@ type ResourceMappingItem struct { Memory *ResourceReqLimItem `json:"memory,omitempty"` } +func (r *ResourceMappingItem) HasStorageMapping() bool { + return !(r == nil || r.Storage == "") +} + +func (r *ResourceMappingItem) HasCPUReqMapping() bool { + return !(r == nil || r.CPU == nil || r.CPU.Requests == "") +} + +func (r *ResourceMappingItem) HasMemReqMapping() bool { + return !(r == nil || r.CPU == nil || r.Memory.Requests == "") +} + +func (r *ResourceMappingItem) HasCPULimMapping() bool { + return !(r == nil || r.CPU == nil || r.CPU.Limits == "") +} + +func (r *ResourceMappingItem) HasMemLimMapping() bool { + return !(r == nil || r.CPU == nil || r.Memory.Limits == "") +} + type ResourceReqLimItem struct { // Requests value mapping key. // +optional @@ -278,6 +298,26 @@ type AddonInstallSpec struct { ExtraItems []AddonInstallExtraItem `json:"extras,omitempty"` } +func (r *AddonInstallSpec) IsDisabled() bool { + return r == nil || !r.Enabled +} + +func (r *AddonInstallSpec) HasSetValues() bool { + if r == nil { + return false + } + + if !r.AddonInstallSpecItem.IsEmpty() { + return true + } + for _, i := range r.ExtraItems { + if !i.IsEmpty() { + return true + } + } + return false +} + type AddonInstallExtraItem struct { AddonInstallSpecItem `json:",inline"` @@ -308,6 +348,14 @@ type AddonInstallSpecItem struct { Resources ResourceRequirements `json:"resources,omitempty"` } +func (r AddonInstallSpecItem) IsEmpty() bool { + return r.Replicas == nil && + r.PVEnabled == nil && + r.StorageClass == "" && + r.Tolerations == "" && + len(r.Resources.Requests) == 0 +} + type ResourceRequirements struct { // Limits describes the maximum amount of compute resources allowed. // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ @@ -516,20 +564,24 @@ func (r *HelmTypeInstallSpec) BuildMergedValues(installSpec *AddonInstallSpec) H } } + if valueMapping.ResourcesMapping == nil { + return + } + for k, v := range installSpecItem.Resources.Requests { switch k { case corev1.ResourceStorage: - if valueMapping.ResourcesMapping.Storage != "" { + if valueMapping.ResourcesMapping.HasStorageMapping() { installValues.SetValues = append(installValues.SetValues, fmt.Sprintf("%s=%v", valueMapping.ResourcesMapping.Storage, v.ToUnstructured())) } case corev1.ResourceCPU: - if valueMapping.ResourcesMapping.CPU.Requests != "" { + if valueMapping.ResourcesMapping.HasCPUReqMapping() { installValues.SetValues = append(installValues.SetValues, fmt.Sprintf("%s=%v", valueMapping.ResourcesMapping.CPU.Requests, v.ToUnstructured())) } case corev1.ResourceMemory: - if valueMapping.ResourcesMapping.Memory.Requests != "" { + if valueMapping.ResourcesMapping.HasMemReqMapping() { installValues.SetValues = append(installValues.SetValues, fmt.Sprintf("%s=%v", valueMapping.ResourcesMapping.Memory.Requests, v.ToUnstructured())) } @@ -539,12 +591,12 @@ func (r *HelmTypeInstallSpec) BuildMergedValues(installSpec *AddonInstallSpec) H for k, v := range installSpecItem.Resources.Limits { switch k { case corev1.ResourceCPU: - if valueMapping.ResourcesMapping.CPU.Limits != "" { + if valueMapping.ResourcesMapping.HasCPULimMapping() { installValues.SetValues = append(installValues.SetValues, fmt.Sprintf("%s=%v", valueMapping.ResourcesMapping.CPU.Limits, v.ToUnstructured())) } case corev1.ResourceMemory: - if valueMapping.ResourcesMapping.Memory.Limits != "" { + if valueMapping.ResourcesMapping.HasMemLimMapping() { installValues.SetValues = append(installValues.SetValues, fmt.Sprintf("%s=%v", valueMapping.ResourcesMapping.Memory.Limits, v.ToUnstructured())) } @@ -564,6 +616,34 @@ func (r *HelmTypeInstallSpec) BuildMergedValues(installSpec *AddonInstallSpec) H return installValues } +// BuildContainerArgs derive helm container args +func (r *HelmTypeInstallSpec) BuildContainerArgs(helmContainer *corev1.Container, installValues HelmInstallValues) error { + // add extra helm install option flags + for k, v := range r.InstallOptions { + helmContainer.Args = append(helmContainer.Args, fmt.Sprintf("--%s", k)) + if v != "" { + helmContainer.Args = append(helmContainer.Args, v) + } + } + + // set values from URL + for _, urlValue := range installValues.URLs { + helmContainer.Args = append(helmContainer.Args, "--values", urlValue) + } + + // set key1=val1,key2=val2 value + if len(installValues.SetValues) > 0 { + helmContainer.Args = append(helmContainer.Args, "--set", + strings.Join(installValues.SetValues, ",")) + } + + // set key1=jsonval1,key2=jsonval2 JSON value, applied multiple + for _, v := range installValues.SetJSONValues { + helmContainer.Args = append(helmContainer.Args, "--set-json", v) + } + return nil +} + // GetSortedDefaultInstallValues return DefaultInstallValues items with items that has // provided selector first. func (r AddonSpec) GetSortedDefaultInstallValues() []AddonDefaultInstallSpecItem { diff --git a/apis/extensions/v1alpha1/addon_types_test.go b/apis/extensions/v1alpha1/addon_types_test.go index 17c0019fc..33078c492 100644 --- a/apis/extensions/v1alpha1/addon_types_test.go +++ b/apis/extensions/v1alpha1/addon_types_test.go @@ -345,18 +345,21 @@ func TestHelmInstallSpecBuildMergedValues(t *testing.T) { mappingName("primary", sc))).Should(BeElementOf(mergedValues.SetValues)) } -func TestAddonSpecMisc(t *testing.T) { +func TestAddonMisc(t *testing.T) { g := NewGomegaWithT(t) - addonSpec := AddonSpec{} - g.Expect(addonSpec.InstallSpec.GetEnabled()).Should(BeFalse()) - g.Expect(addonSpec.Helm.BuildMergedValues(nil)).Should(BeEquivalentTo(HelmInstallValues{})) - addonSpec.InstallSpec = &AddonInstallSpec{ + addon := Addon{} + g.Expect(addon.GetExtraNames()).Should(BeEmpty()) + g.Expect(addon.Spec.Installable.GetSelectorsStrings()).Should(BeEmpty()) + g.Expect(addon.Spec.InstallSpec.GetEnabled()).Should(BeFalse()) + g.Expect(addon.Spec.Helm.BuildMergedValues(nil)).Should(BeEquivalentTo(HelmInstallValues{})) + + addon.Spec.InstallSpec = &AddonInstallSpec{ Enabled: true, AddonInstallSpecItem: NewAddonInstallSpecItem(), } - g.Expect(addonSpec.InstallSpec.GetEnabled()).Should(BeTrue()) + g.Expect(addon.Spec.InstallSpec.GetEnabled()).Should(BeTrue()) - addonSpec.DefaultInstallValues = []AddonDefaultInstallSpecItem{ + addon.Spec.DefaultInstallValues = []AddonDefaultInstallSpecItem{ { AddonInstallSpec: AddonInstallSpec{ Enabled: true, @@ -376,6 +379,32 @@ func TestAddonSpecMisc(t *testing.T) { }, } - di := addonSpec.GetSortedDefaultInstallValues() + di := addon.Spec.GetSortedDefaultInstallValues() g.Expect(di).Should(HaveLen(2)) } + +func TestAddonInstallHasSetValues(t *testing.T) { + g := NewGomegaWithT(t) + + installSpec := &AddonInstallSpec{ + Enabled: true, + ExtraItems: []AddonInstallExtraItem{ + { + Name: "extra", + }, + }, + } + + g.Expect(installSpec.IsDisabled()).Should(BeFalse()) + g.Expect(installSpec.HasSetValues()).Should(BeFalse()) + installSpec.ExtraItems[0].AddonInstallSpecItem = AddonInstallSpecItem{ + StorageClass: "sc", + } + g.Expect(installSpec.HasSetValues()).Should(BeTrue()) + installSpec.ExtraItems = nil + g.Expect(installSpec.HasSetValues()).Should(BeFalse()) + installSpec.AddonInstallSpecItem = AddonInstallSpecItem{ + StorageClass: "sc", + } + g.Expect(installSpec.HasSetValues()).Should(BeTrue()) +} diff --git a/controllers/apps/components/deployment_controller.go b/controllers/apps/components/deployment_controller.go index dabdfb592..8545e4857 100644 --- a/controllers/apps/components/deployment_controller.go +++ b/controllers/apps/components/deployment_controller.go @@ -97,5 +97,6 @@ func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&appsv1.Deployment{}). Owns(&appsv1.ReplicaSet{}). WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). + Named("deployment-watcher"). Complete(r) } diff --git a/controllers/apps/components/pod_controller.go b/controllers/apps/components/pod_controller.go index d794fdffb..6d62e2196 100644 --- a/controllers/apps/components/pod_controller.go +++ b/controllers/apps/components/pod_controller.go @@ -111,7 +111,6 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } r.Recorder.Eventf(pod, corev1.EventTypeNormal, "AddAnnotation", "add annotation %s=%s", constant.LeaderAnnotationKey, componentStatus.ConsensusSetStatus.Leader.Pod) - return intctrlutil.Reconciled() } @@ -120,5 +119,6 @@ func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&corev1.Pod{}). WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). + Named("pod-watcher"). Complete(r) } diff --git a/controllers/apps/components/stateful_set_controller.go b/controllers/apps/components/stateful_set_controller.go index 291162136..40a246f90 100644 --- a/controllers/apps/components/stateful_set_controller.go +++ b/controllers/apps/components/stateful_set_controller.go @@ -100,5 +100,6 @@ func (r *StatefulSetReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&appsv1.StatefulSet{}). Owns(&corev1.Pod{}). WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). + Named("statefulset-watcher"). Complete(r) } diff --git a/controllers/extensions/addon_controller.go b/controllers/extensions/addon_controller.go index 8f8a7133d..504b6decd 100644 --- a/controllers/extensions/addon_controller.go +++ b/controllers/extensions/addon_controller.go @@ -28,12 +28,16 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" @@ -103,6 +107,10 @@ func (r *AddonReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return ctrlerihandler.NewTypeHandler(&autoInstallCheckStage{stageCtx: buildStageCtx(next...)}) } + enabledAutoValuesStageBuilder := func(next ...ctrlerihandler.Handler) ctrlerihandler.Handler { + return ctrlerihandler.NewTypeHandler(&enabledWithDefaultValuesStage{stageCtx: buildStageCtx(next...)}) + } + progressingStageBuilder := func(next ...ctrlerihandler.Handler) ctrlerihandler.Handler { return ctrlerihandler.NewTypeHandler(&progressingHandler{stageCtx: buildStageCtx(next...)}) } @@ -116,6 +124,7 @@ func (r *AddonReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl genIDProceedStageBuilder, installableCheckStageBuilder, autoInstallCheckStageBuilder, + enabledAutoValuesStageBuilder, progressingStageBuilder, terminalStateStageBuilder, ).Handler("") @@ -137,21 +146,30 @@ func (r *AddonReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl func (r *AddonReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&extensionsv1alpha1.Addon{}). - // TODO: replace with controller-idioms's adopt lib - // Watches(&source.Kind{Type: &batchv1.Job{}}, - // &handler.EnqueueRequestForObject{}, - // builder.WithPredicates(&jobCompletionPredicate{reconciler: r, Log: log.FromContext(context.TODO())})). + Watches(&source.Kind{Type: &batchv1.Job{}}, handler.EnqueueRequestsFromMapFunc(r.findAddonJobs)). WithOptions(controller.Options{ MaxConcurrentReconciles: viper.GetInt(maxConcurrentReconcilesKey), }). Complete(r) } -// type jobCompletionPredicate struct { -// predicate.Funcs -// reconciler *AddonReconciler -// Log logr.Logger -// } +func (r *AddonReconciler) findAddonJobs(job client.Object) []reconcile.Request { + labels := job.GetLabels() + if _, ok := labels[constant.AddonNameLabelKey]; !ok { + return []reconcile.Request{} + } + if v, ok := labels[constant.AppManagedByLabelKey]; !ok || v != constant.AppName { + return []reconcile.Request{} + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: job.GetNamespace(), + Name: job.GetName(), + }, + }, + } +} func (r *AddonReconciler) cleanupJobPods(reqCtx intctrlutil.RequestCtx) error { if err := r.DeleteAllOf(reqCtx.Ctx, &corev1.Pod{}, diff --git a/controllers/extensions/addon_controller_stages.go b/controllers/extensions/addon_controller_stages.go index 4be2e54c9..a74d05631 100644 --- a/controllers/extensions/addon_controller_stages.go +++ b/controllers/extensions/addon_controller_stages.go @@ -28,11 +28,13 @@ import ( ctrlerihandler "github.com/authzed/controller-idioms/handler" "github.com/spf13/viper" + "golang.org/x/exp/slices" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -55,6 +57,12 @@ const ( func init() { viper.SetDefault(addonSANameKey, "kubeblocks-addon-installer") + viper.SetDefault(addonHelmInstallOptKey, []string{ + "--atomic", + "--cleanup-on-fail", + "--wait", + }) + viper.SetDefault(addonHelmUninstallOptKey, []string{}) } func (r *stageCtx) setReconciled() { @@ -114,6 +122,10 @@ type autoInstallCheckStage struct { stageCtx } +type enabledWithDefaultValuesStage struct { + stageCtx +} + type progressingHandler struct { stageCtx enablingStage enablingStage @@ -270,7 +282,7 @@ func (r *installableCheckStage) Handle(ctx context.Context) { Type: extensionsv1alpha1.ConditionTypeChecked, Status: metav1.ConditionFalse, ObservedGeneration: addon.Generation, - Reason: AddonSpecInstallableReqUnmatched, + Reason: InstallableRequirementUnmatched, Message: "spec.installable.selectors has no matching requirement.", LastTransitionTime: metav1.Now(), }) @@ -295,36 +307,23 @@ func (r *autoInstallCheckStage) Handle(ctx context.Context) { return } // proceed if has specified addon.spec.installSpec - if addon.Spec.InstallSpec != nil { + if addon.Spec.InstallSpec.HasSetValues() { r.reqCtx.Log.V(1).Info("has specified addon.spec.installSpec") return } + enabledAddonWithDefaultValues(ctx, &r.stageCtx, addon, AddonAutoInstall, "Addon enabled auto-install") + }) + r.next.Handle(ctx) +} - setInstallSpec := func(di *extensionsv1alpha1.AddonDefaultInstallSpecItem) { - addon.Spec.InstallSpec = di.AddonInstallSpec.DeepCopy() - addon.Spec.InstallSpec.Enabled = true - if err := r.reconciler.Client.Update(ctx, addon); err != nil { - r.setRequeueWithErr(err, "") - return - } - r.reconciler.Event(addon, "Normal", AddonAutoInstall, - "Addon enabled auto-install") - r.setReconciled() - } - - for _, di := range addon.Spec.GetSortedDefaultInstallValues() { - if len(di.Selectors) == 0 { - setInstallSpec(&di) - return - } - for _, s := range di.Selectors { - if !s.MatchesFromConfig() { - continue - } - setInstallSpec(&di) - return - } +func (r *enabledWithDefaultValuesStage) Handle(ctx context.Context) { + r.process(func(addon *extensionsv1alpha1.Addon) { + r.reqCtx.Log.V(1).Info("enabledWithDefaultValuesStage", "phase", addon.Status.Phase) + if addon.Spec.InstallSpec.HasSetValues() || + addon.Spec.InstallSpec.IsDisabled() { + return } + enabledAddonWithDefaultValues(ctx, &r.stageCtx, addon, AddonSetDefaultValues, "Addon enabled with default values") }) r.next.Handle(ctx) } @@ -431,25 +430,16 @@ func (r *helmTypeInstallStage) Handle(ctx context.Context) { // info. from conditions. if helmInstallJob.Status.Failed > 0 { // job failed set terminal state phase - patch := client.MergeFrom(addon.DeepCopy()) - addon.Status.ObservedGeneration = addon.Generation - addon.Status.Phase = extensionsv1alpha1.AddonFailed - meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ - Type: extensionsv1alpha1.ConditionTypeFailed, - Status: metav1.ConditionFalse, - ObservedGeneration: addon.Generation, - Reason: AddonSpecInstallFailed, - Message: "installation failed", - LastTransitionTime: metav1.Now(), - }) - - if err := r.reconciler.Status().Patch(ctx, addon, patch); err != nil { - r.setRequeueWithErr(err, "") - return - } - r.reconciler.Event(addon, "Warning", InstallationFailed, + setAddonErrorConditions(ctx, &r.stageCtx, addon, true, true, InstallationFailed, fmt.Sprintf("Installation failed, do inspect error from jobs.batch %s", key.String())) - r.setReconciled() + // only allow to do pod logs if max concurrent reconciles > 1, also considered that helm + // cmd error only has limited contents + if viper.GetInt(maxConcurrentReconcilesKey) > 1 { + if err := logFailedJobPodToCondError(ctx, &r.stageCtx, addon, key.Name, InstallationFailedLogs); err != nil { + r.setRequeueWithErr(err, "") + return + } + } return } r.setRequeueAfter(time.Second, "") @@ -465,49 +455,44 @@ func (r *helmTypeInstallStage) Handle(ctx context.Context) { helmInstallJob.ObjectMeta.Namespace = key.Namespace helmJobPodSpec := &helmInstallJob.Spec.Template.Spec helmContainer := &helmInstallJob.Spec.Template.Spec.Containers[0] - helmContainer.Args = []string{ + helmContainer.Args = append([]string{ "upgrade", "--install", "$(RELEASE_NAME)", "$(CHART)", "--namespace", "$(RELEASE_NS)", - "--timeout", - "10m", "--create-namespace", - "--atomic", - "--cleanup-on-fail", - "--wait", - } - - // add extra helm install option flags - for k, v := range addon.Spec.Helm.InstallOptions { - helmContainer.Args = append(helmContainer.Args, fmt.Sprintf("--%s", k)) - if v != "" { - helmContainer.Args = append(helmContainer.Args, v) - } - } + }, viper.GetStringSlice(addonHelmInstallOptKey)...) installValues := addon.Spec.Helm.BuildMergedValues(addon.Spec.InstallSpec) - // set values from URL - for _, urlValue := range installValues.URLs { - helmContainer.Args = append(helmContainer.Args, "--values", urlValue) + if err = addon.Spec.Helm.BuildContainerArgs(helmContainer, installValues); err != nil { + r.setRequeueWithErr(err, "") + return } // set values from file for _, cmRef := range installValues.ConfigMapRefs { cm := &corev1.ConfigMap{} - if err := r.reconciler.Get(ctx, client.ObjectKey{ + key := client.ObjectKey{ Name: cmRef.Name, - Namespace: mgrNS}, cm); err != nil { + Namespace: mgrNS} + if err := r.reconciler.Get(ctx, key, cm); err != nil { if !apierrors.IsNotFound(err) { r.setRequeueWithErr(err, "") return } r.setRequeueAfter(time.Second, fmt.Sprintf("ConfigMap %s not found", cmRef.Name)) + setAddonErrorConditions(ctx, &r.stageCtx, addon, false, true, AddonRefObjError, + fmt.Sprintf("ConfigMap object %v not found", key)) + return + } + if !findDataKey(cm.Data, cmRef) { + setAddonErrorConditions(ctx, &r.stageCtx, addon, true, true, AddonRefObjError, + fmt.Sprintf("Attach ConfigMap %v volume source failed, key %s not found", key, cmRef.Key)) + r.setReconciled() return } - // TODO: validate cmRef.key exist in cm attachVolumeMount(helmJobPodSpec, cmRef, cm.Name, "cm", func() corev1.VolumeSource { return corev1.VolumeSource{ @@ -528,18 +513,25 @@ func (r *helmTypeInstallStage) Handle(ctx context.Context) { for _, secretRef := range installValues.SecretRefs { secret := &corev1.Secret{} - if err := r.reconciler.Get(ctx, client.ObjectKey{ + key := client.ObjectKey{ Name: secretRef.Name, - Namespace: mgrNS}, secret); err != nil { + Namespace: mgrNS} + if err := r.reconciler.Get(ctx, key, secret); err != nil { if !apierrors.IsNotFound(err) { r.setRequeueWithErr(err, "") return } r.setRequeueAfter(time.Second, fmt.Sprintf("Secret %s not found", secret.Name)) + setAddonErrorConditions(ctx, &r.stageCtx, addon, false, true, AddonRefObjError, + fmt.Sprintf("Secret object %v not found", key)) + return + } + if !findDataKey(secret.Data, secretRef) { + setAddonErrorConditions(ctx, &r.stageCtx, addon, true, true, AddonRefObjError, + fmt.Sprintf("Attach Secret %v volume source failed, key %s not found", key, secretRef.Key)) + r.setReconciled() return } - // TODO: validate secretRef.key exist in secret - attachVolumeMount(helmJobPodSpec, secretRef, secret.Name, "secret", func() corev1.VolumeSource { return corev1.VolumeSource{ @@ -556,17 +548,6 @@ func (r *helmTypeInstallStage) Handle(ctx context.Context) { }) } - // set key1=val1,key2=val2 value - if len(installValues.SetValues) > 0 { - helmContainer.Args = append(helmContainer.Args, "--set", - strings.Join(installValues.SetValues, ",")) - } - - // set key1=jsonval1,key2=jsonval2 JSON value, applied multiple - for _, v := range installValues.SetJSONValues { - helmContainer.Args = append(helmContainer.Args, "--set-json", v) - } - if err := r.reconciler.Create(ctx, helmInstallJob); err != nil { r.setRequeueWithErr(err, "") return @@ -615,6 +596,14 @@ func (r *helmTypeUninstallStage) Handle(ctx context.Context) { r.reconciler.Event(addon, "Warning", UninstallationFailed, fmt.Sprintf("Uninstallation failed, do inspect error from jobs.batch %s", key.String())) + // only allow to do pod logs if max concurrent reconciles > 1, also considered that helm + // cmd error only has limited contents + if viper.GetInt(maxConcurrentReconcilesKey) > 1 { + if err := logFailedJobPodToCondError(ctx, &r.stageCtx, addon, key.Name, UninstallationFailedLogs); err != nil { + r.setRequeueWithErr(err, "") + return + } + } if err := r.reconciler.Delete(ctx, helmUninstallJob); client.IgnoreNotFound(err) != nil { r.setRequeueWithErr(err, "") @@ -663,14 +652,12 @@ func (r *helmTypeUninstallStage) Handle(ctx context.Context) { } helmUninstallJob.ObjectMeta.Name = key.Name helmUninstallJob.ObjectMeta.Namespace = key.Namespace - helmUninstallJob.Spec.Template.Spec.Containers[0].Args = []string{ + helmUninstallJob.Spec.Template.Spec.Containers[0].Args = append([]string{ "delete", "$(RELEASE_NAME)", "--namespace", "$(RELEASE_NS)", - "--timeout", - "10m", - } + }, viper.GetStringSlice(addonHelmUninstallOptKey)...) r.reqCtx.Log.V(1).Info("create helm uninstall job", "job", key) if err := r.reconciler.Create(ctx, helmUninstallJob); err != nil { r.reqCtx.Log.V(1).Info("helmTypeUninstallStage", "job", key, "err", err) @@ -782,6 +769,7 @@ func createHelmJobProto(addon *extensionsv1alpha1.Addon) (*batchv1.Job, error) { } } ttlSec := int32(ttl.Seconds()) + backoffLimit := int32(3) helmProtoJob := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -790,6 +778,7 @@ func createHelmJobProto(addon *extensionsv1alpha1.Addon) (*batchv1.Job, error) { }, }, Spec: batchv1.JobSpec{ + BackoffLimit: &backoffLimit, TTLSecondsAfterFinished: &ttlSec, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -799,11 +788,11 @@ func createHelmJobProto(addon *extensionsv1alpha1.Addon) (*batchv1.Job, error) { }, }, Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyOnFailure, + RestartPolicy: corev1.RestartPolicyNever, ServiceAccountName: viper.GetString("KUBEBLOCKS_ADDON_SA_NAME"), Containers: []corev1.Container{ { - Name: strings.ToLower(string(addon.Spec.Type)), + Name: getJobMainContainerName(addon), Image: viper.GetString(constant.KBToolsImage), ImagePullPolicy: corev1.PullPolicy(viper.GetString(constant.CfgAddonJobImgPullPolicy)), // TODO: need have image that is capable of following settings, current settings @@ -883,3 +872,120 @@ func createHelmJobProto(addon *extensionsv1alpha1.Addon) (*batchv1.Job, error) { } return helmProtoJob, nil } + +func enabledAddonWithDefaultValues(ctx context.Context, stageCtx *stageCtx, + addon *extensionsv1alpha1.Addon, reason, message string) { + setInstallSpec := func(di *extensionsv1alpha1.AddonDefaultInstallSpecItem) { + addon.Spec.InstallSpec = di.AddonInstallSpec.DeepCopy() + addon.Spec.InstallSpec.Enabled = true + if err := stageCtx.reconciler.Client.Update(ctx, addon); err != nil { + stageCtx.setRequeueWithErr(err, "") + return + } + stageCtx.reconciler.Event(addon, "Normal", reason, message) + stageCtx.setReconciled() + } + + for _, di := range addon.Spec.GetSortedDefaultInstallValues() { + if len(di.Selectors) == 0 { + setInstallSpec(&di) + return + } + for _, s := range di.Selectors { + if !s.MatchesFromConfig() { + continue + } + setInstallSpec(&di) + return + } + } +} + +func setAddonErrorConditions(ctx context.Context, + stageCtx *stageCtx, + addon *extensionsv1alpha1.Addon, + setFailedStatus, recordEvent bool, + reason, message string, + eventMessage ...string) { + patch := client.MergeFrom(addon.DeepCopy()) + addon.Status.ObservedGeneration = addon.Generation + if setFailedStatus { + addon.Status.Phase = extensionsv1alpha1.AddonFailed + } + meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{ + Type: extensionsv1alpha1.ConditionTypeChecked, + Status: metav1.ConditionFalse, + ObservedGeneration: addon.Generation, + Reason: reason, + Message: message, + LastTransitionTime: metav1.Now(), + }) + + if err := stageCtx.reconciler.Status().Patch(ctx, addon, patch); err != nil { + stageCtx.setRequeueWithErr(err, "") + return + } + if !recordEvent { + return + } + if len(eventMessage) > 0 && eventMessage[0] != "" { + stageCtx.reconciler.Event(addon, "Warning", reason, eventMessage[0]) + } else { + stageCtx.reconciler.Event(addon, "Warning", reason, message) + } +} + +func getJobMainContainerName(addon *extensionsv1alpha1.Addon) string { + return strings.ToLower(string(addon.Spec.Type)) +} + +func logFailedJobPodToCondError(ctx context.Context, stageCtx *stageCtx, addon *extensionsv1alpha1.Addon, + jobName, reason string) error { + podList := &corev1.PodList{} + if err := stageCtx.reconciler.List(ctx, podList, + client.InNamespace(viper.GetString(constant.CfgKeyCtrlrMgrNS)), + client.MatchingLabels{ + constant.AddonNameLabelKey: stageCtx.reqCtx.Req.Name, + constant.AppManagedByLabelKey: constant.AppName, + "job-name": jobName, + }); err != nil { + return err + } + + // sort pod with latest creation place front + slices.SortFunc(podList.Items, func(a, b corev1.Pod) bool { + return b.CreationTimestamp.Before(&(a.CreationTimestamp)) + }) + +podsloop: + for _, pod := range podList.Items { + switch pod.Status.Phase { + case corev1.PodFailed: + clientset, err := corev1client.NewForConfig(stageCtx.reconciler.RestConfig) + if err != nil { + return err + } + currOpts := &corev1.PodLogOptions{ + Container: getJobMainContainerName(addon), + } + req := clientset.Pods(pod.Namespace).GetLogs(pod.Name, currOpts) + data, err := req.DoRaw(ctx) + if err != nil { + return err + } + setAddonErrorConditions(ctx, stageCtx, addon, false, true, reason, string(data)) + break podsloop + } + } + return nil +} + +func findDataKey[V string | []byte](data map[string]V, refObj extensionsv1alpha1.DataObjectKeySelector) bool { + for k := range data { + if k != refObj.Key { + continue + } + return true + } + return false +} diff --git a/controllers/extensions/addon_controller_test.go b/controllers/extensions/addon_controller_test.go index cd0b4ca69..655fd689c 100644 --- a/controllers/extensions/addon_controller_test.go +++ b/controllers/extensions/addon_controller_test.go @@ -229,7 +229,7 @@ var _ = Describe("Addon controller", func() { Expect(addon.Spec.DefaultInstallValues).ShouldNot(BeEmpty()) } - progressingPhaseCheck := func(genID int, expectPhase extensionsv1alpha1.AddonPhase, handler func()) { + addonStatusPhaseCheck := func(genID int, expectPhase extensionsv1alpha1.AddonPhase, handler func()) { Eventually(func(g Gomega) { _, err := doReconcile() Expect(err).To(Not(HaveOccurred())) @@ -258,7 +258,7 @@ var _ = Describe("Addon controller", func() { } enablingPhaseCheck := func(genID int) { - progressingPhaseCheck(genID, extensionsv1alpha1.AddonEnabling, func() { + addonStatusPhaseCheck(genID, extensionsv1alpha1.AddonEnabling, func() { By("By fake active install job") jobKey := client.ObjectKey{ Namespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), @@ -271,7 +271,7 @@ var _ = Describe("Addon controller", func() { } disablingPhaseCheck := func(genID int) { - progressingPhaseCheck(genID, extensionsv1alpha1.AddonDisabling, nil) + addonStatusPhaseCheck(genID, extensionsv1alpha1.AddonDisabling, nil) } checkAddonDeleted := func(g Gomega) { @@ -305,9 +305,16 @@ var _ = Describe("Addon controller", func() { Expect(testCtx.CreateObj(ctx, helmRelease)).Should(Succeed()) } - It("should successfully reconcile a custom resource for Addon", func() { + It("should successfully reconcile a custom resource for Addon with spec.type=Helm", func() { By("By create an addon") - createAddonSpecWithRequiredAttributes(nil) + createAddonSpecWithRequiredAttributes(func(newOjb *extensionsv1alpha1.Addon) { + newOjb.Spec.Type = extensionsv1alpha1.HelmType + newOjb.Spec.Helm = &extensionsv1alpha1.HelmTypeInstallSpec{ + InstallOptions: extensionsv1alpha1.HelmInstallOptions{ + "--debug": "true", + }, + } + }) By("By checking status.observedGeneration and status.phase=disabled") Eventually(func(g Gomega) { @@ -542,7 +549,7 @@ var _ = Describe("Addon controller", func() { // "extensions.kubeblocks.io/skip-installable-check" }) - It("should successfully reconcile a custom resource for Addon with CM and secret values", func() { + It("should successfully reconcile a custom resource for Addon with CM and secret ref values", func() { By("By create an addon with spec.helm.installValues.configMapRefs set") cm := testapps.CreateCustomizedObj(&testCtx, "addon/cm-values.yaml", &corev1.ConfigMap{}, func(newCM *corev1.ConfigMap) { @@ -552,6 +559,8 @@ var _ = Describe("Addon controller", func() { &corev1.Secret{}, func(newSecret *corev1.Secret) { newSecret.Namespace = viper.GetString(constant.CfgKeyCtrlrMgrNS) }) + + By("By addon enabled via auto-install") createAddonSpecWithRequiredAttributes(func(newOjb *extensionsv1alpha1.Addon) { newOjb.Spec.Installable.AutoInstall = true for k := range cm.Data { @@ -569,12 +578,47 @@ var _ = Describe("Addon controller", func() { }) } }) - - By("By addon autoInstall auto added") enablingPhaseCheck(2) By("By enabled addon with fake completed install job status") fakeInstallationCompletedJob(2) }) + + It("should failed reconcile a custom resource for Addon with missing CM ref values", func() { + By("By create an addon with spec.helm.installValues.configMapRefs set") + cm := testapps.CreateCustomizedObj(&testCtx, "addon/cm-values.yaml", + &corev1.ConfigMap{}, func(newCM *corev1.ConfigMap) { + newCM.Namespace = viper.GetString(constant.CfgKeyCtrlrMgrNS) + }) + + By("By addon enabled via auto-install") + createAddonSpecWithRequiredAttributes(func(newOjb *extensionsv1alpha1.Addon) { + newOjb.Spec.Installable.AutoInstall = true + newOjb.Spec.Helm.InstallValues.ConfigMapRefs = append(newOjb.Spec.Helm.InstallValues.ConfigMapRefs, + extensionsv1alpha1.DataObjectKeySelector{ + Name: cm.Name, + Key: "unknown", + }) + }) + addonStatusPhaseCheck(2, extensionsv1alpha1.AddonFailed, nil) + }) + + It("should failed reconcile a custom resource for Addon with missing secret ref values", func() { + By("By create an addon with spec.helm.installValues.configMapRefs set") + secret := testapps.CreateCustomizedObj(&testCtx, "addon/secret-values.yaml", + &corev1.Secret{}, func(newSecret *corev1.Secret) { + newSecret.Namespace = viper.GetString(constant.CfgKeyCtrlrMgrNS) + }) + By("By addon enabled via auto-install") + createAddonSpecWithRequiredAttributes(func(newOjb *extensionsv1alpha1.Addon) { + newOjb.Spec.Installable.AutoInstall = true + newOjb.Spec.Helm.InstallValues.SecretRefs = append(newOjb.Spec.Helm.InstallValues.SecretRefs, + extensionsv1alpha1.DataObjectKeySelector{ + Name: secret.Name, + Key: "unknown", + }) + }) + addonStatusPhaseCheck(2, extensionsv1alpha1.AddonFailed, nil) + }) }) }) diff --git a/controllers/extensions/const.go b/controllers/extensions/const.go index 22c97ebde..00d52ad63 100644 --- a/controllers/extensions/const.go +++ b/controllers/extensions/const.go @@ -29,21 +29,25 @@ const ( NoDeleteJobs = "extensions.kubeblocks.io/no-delete-jobs" // condition reasons - AddonDisabled = "AddonDisabled" - AddonEnabled = "AddonEnabled" - AddonSpecInstallFailed = "AddonSpecInstallFailed" - AddonSpecInstallableReqUnmatched = "AddonSpecInstallableRequirementUnmatched" + AddonDisabled = "AddonDisabled" + AddonEnabled = "AddonEnabled" // event reasons InstallableCheckSkipped = "InstallableCheckSkipped" InstallableRequirementUnmatched = "InstallableRequirementUnmatched" AddonAutoInstall = "AddonAutoInstall" + AddonSetDefaultValues = "AddonSetDefaultValues" DisablingAddon = "DisablingAddon" EnablingAddon = "EnablingAddon" InstallationFailed = "InstallationFailed" + InstallationFailedLogs = "InstallationFailedLogs" UninstallationFailed = "UninstallationFailed" + UninstallationFailedLogs = "UninstallationFailedLogs" + AddonRefObjError = "ReferenceObjectError" // config keys used in viper maxConcurrentReconcilesKey = "MAXCONCURRENTRECONCILES_ADDON" addonSANameKey = "KUBEBLOCKS_ADDON_SA_NAME" + addonHelmInstallOptKey = "KUBEBLOCKS_ADDON_HELM_INSTALL_OPTIONS" + addonHelmUninstallOptKey = "KUBEBLOCKS_ADDON_HELM_UNINSTALL_OPTIONS" ) diff --git a/go.mod b/go.mod index 94496b182..8a34de3a7 100644 --- a/go.mod +++ b/go.mod @@ -46,8 +46,8 @@ require ( github.com/leaanthony/debme v1.2.1 github.com/manifoldco/promptui v0.9.0 github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 - github.com/onsi/ginkgo/v2 v2.7.0 - github.com/onsi/gomega v1.25.0 + github.com/onsi/ginkgo/v2 v2.9.1 + github.com/onsi/gomega v1.27.4 github.com/opencontainers/image-spec v1.1.0-rc2 github.com/pingcap/go-tpc v1.0.9 github.com/pkg/errors v0.9.1 @@ -73,7 +73,7 @@ require ( go.uber.org/zap v1.24.0 golang.org/x/crypto v0.5.0 golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f - golang.org/x/net v0.7.0 + golang.org/x/net v0.8.0 golang.org/x/sync v0.1.0 google.golang.org/grpc v1.52.0 google.golang.org/protobuf v1.28.1 @@ -87,11 +87,11 @@ require ( k8s.io/client-go v0.26.1 k8s.io/component-base v0.26.1 k8s.io/cri-api v0.25.0 - k8s.io/klog/v2 v2.90.0 - k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 + k8s.io/klog/v2 v2.90.1 + k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a k8s.io/kubectl v0.26.0 k8s.io/metrics v0.26.0 - k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 + k8s.io/utils v0.0.0-20230209194617-a36077c30491 sigs.k8s.io/controller-runtime v0.14.4 sigs.k8s.io/kustomize/kyaml v0.13.9 sigs.k8s.io/yaml v1.3.0 @@ -172,11 +172,12 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/errors v0.20.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/strfmt v0.21.3 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-redis/redis/v7 v7.4.1 // indirect + github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/go-test/deep v1.0.8 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect @@ -184,7 +185,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/goodhosts/hostsfile v0.1.1 // indirect github.com/google/btree v1.0.1 // indirect @@ -192,6 +193,7 @@ require ( github.com/google/gnostic v0.6.9 // indirect github.com/google/go-intervals v0.0.2 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect @@ -347,13 +349,13 @@ require ( go.uber.org/multierr v1.8.0 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20230221090011-e4bae7ad2296 // indirect - golang.org/x/mod v0.7.0 // indirect + golang.org/x/mod v0.9.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.4.0 // indirect + golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/api v0.107.0 // indirect @@ -368,7 +370,7 @@ require ( k8s.io/component-helpers v0.26.0 // indirect oras.land/oras-go v1.2.2 // indirect periph.io/x/host/v3 v3.8.0 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.12.1 // indirect sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect diff --git a/go.sum b/go.sum index 88ebb8e68..2e29b28de 100644 --- a/go.sum +++ b/go.sum @@ -764,13 +764,13 @@ github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuA github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.21.3 h1:xwhj5X6CjXEZZHMWy1zKJxvW9AfHC9pkyUjLvHtKG7o= @@ -792,6 +792,7 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= @@ -875,8 +876,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -956,6 +958,7 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -1409,8 +1412,8 @@ github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1ls github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.7.0 h1:/XxtEV3I3Eif/HobnVx9YmJgk8ENdRsuUmM+fLCFNow= -github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1420,8 +1423,8 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= -github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -1968,8 +1971,9 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2048,8 +2052,8 @@ golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2227,8 +2231,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2236,8 +2240,8 @@ golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2249,8 +2253,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2349,8 +2353,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2710,13 +2714,13 @@ k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.90.0 h1:VkTxIV/FjRXn1fgNNcKGM8cfmL1Z33ZjXRTVxKCoF5M= -k8s.io/klog/v2 v2.90.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-aggregator v0.19.12 h1:OwyNUe/7/gxzEnaLd3sC9Yrpx0fZAERzvFslX5Qq5g8= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= +k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/kubectl v0.26.0 h1:xmrzoKR9CyNdzxBmXV7jW9Ln8WMrwRK6hGbbf69o4T0= k8s.io/kubectl v0.26.0/go.mod h1:eInP0b+U9XUJWSYeU9XZnTA+cVYuWyl3iYPGtru0qhQ= k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= @@ -2724,8 +2728,8 @@ k8s.io/metrics v0.26.0 h1:U/NzZHKDrIVGL93AUMRkqqXjOah3wGvjSnKmG/5NVCs= k8s.io/metrics v0.26.0/go.mod h1:cf5MlG4ZgWaEFZrR9+sOImhZ2ICMpIdNurA+D8snIs8= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f/go.mod h1:9VQ397fNXEnF84t90W4r4TRCQK+pg9f8ugVfyj+S26w= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= @@ -2741,8 +2745,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyz sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.14.4 h1:Kd/Qgx5pd2XUL08eOV2vwIq3L9GhIbJ5Nxengbd4/0M= sigs.k8s.io/controller-runtime v0.14.4/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s= sigs.k8s.io/kustomize/kustomize/v4 v4.5.7 h1:cDW6AVMl6t/SLuQaezMET8hgnadZGIAr8tUrxFVOrpg= diff --git a/internal/cli/cmd/addon/addon.go b/internal/cli/cmd/addon/addon.go index a325f6355..31231872c 100644 --- a/internal/cli/cmd/addon/addon.go +++ b/internal/cli/cmd/addon/addon.go @@ -445,24 +445,6 @@ func (o *addonCmdOpts) buildEnablePatch(flags []*pflag.Flag, spec, install map[s }() if o.addonEnableFlags.useDefault() { - if len(o.addon.Spec.DefaultInstallValues) == 0 { - installSpec.Enabled = true - return nil - } - - for _, di := range o.addon.Spec.GetSortedDefaultInstallValues() { - if len(di.Selectors) == 0 { - installSpec = di.AddonInstallSpec - break - } - for _, s := range di.Selectors { - if !s.MatchesFromConfig() { - continue - } - installSpec = di.AddonInstallSpec - break - } - } installSpec.Enabled = true return nil } diff --git a/internal/controllerutil/type.go b/internal/controllerutil/type.go index 414cf1e18..097e30497 100644 --- a/internal/controllerutil/type.go +++ b/internal/controllerutil/type.go @@ -41,3 +41,9 @@ func (r *RequestCtx) UpdateCtxValue(key, val any) context.Context { r.Ctx = context.WithValue(r.Ctx, key, val) return p } + +// WithValue returns a copy of parent in which the value associated with key is +// val. +func (r *RequestCtx) WithValue(key, val any) context.Context { + return context.WithValue(r.Ctx, key, val) +} From ac22b48c0d1eb43d596687e08782390b14667284 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Fri, 5 May 2023 16:07:21 +0800 Subject: [PATCH 224/439] support: update csi-s3 images to support OSS and arm64 arch. (#3080) --- deploy/csi-s3/values.yaml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/deploy/csi-s3/values.yaml b/deploy/csi-s3/values.yaml index 97de65b28..fca9cada4 100644 --- a/deploy/csi-s3/values.yaml +++ b/deploy/csi-s3/values.yaml @@ -1,13 +1,13 @@ --- images: # Source: quay.io/k8scsi/csi-attacher:v3.0.1 - attacher: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-attacher:v3.0.1 + attacher: registry.cn-hangzhou.aliyuncs.com/apecloud/csi-attacher:v3.4.0 # Source: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 - registrar: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-node-driver-registrar:v1.2.0 + registrar: registry.cn-hangzhou.aliyuncs.com/apecloud/csi-node-driver-registrar:v2.5.1 # Source: quay.io/k8scsi/csi-provisioner:v2.1.0 - provisioner: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-provisioner:v2.1.0 + provisioner: registry.cn-hangzhou.aliyuncs.com/apecloud/csi-provisioner:v3.1.0 # Main image - csi: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-s3-driver:0.31.3 + csi: registry.cn-hangzhou.aliyuncs.com/apecloud/csi-s3-driver:0.31.3 storageClass: # Specifies whether the storage class should be created @@ -19,6 +19,17 @@ storageClass: # mounter to use - either geesefs, s3fs or rclone (default geesefs) mounter: geesefs # GeeseFS mount options + # mounter: geesefs + # mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" + + # S3FS mount options + # mounter: s3fs + # use legacy API calling style which do not support the virtual-host request style: + # mountOptions: "-o use_path_request_style" + # NOTE: + # aliyun OSS only support s3fs, and DO NOT set "-o use_path_request_style": + # mounter: s3fs + # mountOptions: "" mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" # Volume reclaim policy reclaimPolicy: Retain From 6ae43c8e6989e2f3ffda64ca011a1a8790224218 Mon Sep 17 00:00:00 2001 From: runsun Date: Fri, 5 May 2023 16:08:35 +0800 Subject: [PATCH 225/439] fix: monitor compatible with pg12 (#3075) --- .../templates/clusterdefinition.yaml | 2 +- .../postgresql/templates/clusterversion.yaml | 5 + .../templates/metrics-configmap-12.yaml | 279 +++++++++++++++ .../templates/metrics-configmap-14.yaml | 319 +++++++++++++++++ .../templates/metrics-configmap.yaml | 8 - deploy/postgresql/values.yaml | 325 ------------------ 6 files changed, 604 insertions(+), 334 deletions(-) create mode 100644 deploy/postgresql/templates/metrics-configmap-12.yaml create mode 100644 deploy/postgresql/templates/metrics-configmap-14.yaml delete mode 100644 deploy/postgresql/templates/metrics-configmap.yaml diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 923c1437c..c77427c7b 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -51,7 +51,7 @@ spec: volumeName: postgresql-config defaultMode: 0777 - name: postgresql-custom-metrics - templateRef: postgresql-custom-metrics + templateRef: postgresql14-custom-metrics namespace: {{ .Release.Namespace }} volumeName: postgresql-custom-metrics defaultMode: 0777 diff --git a/deploy/postgresql/templates/clusterversion.yaml b/deploy/postgresql/templates/clusterversion.yaml index 0a4538898..cd0eb4ab6 100644 --- a/deploy/postgresql/templates/clusterversion.yaml +++ b/deploy/postgresql/templates/clusterversion.yaml @@ -39,6 +39,11 @@ spec: namespace: {{ .Release.Namespace }} volumeName: postgresql-config defaultMode: 0777 + - name: postgresql-custom-metrics + templateRef: postgresql12-custom-metrics + namespace: {{ .Release.Namespace }} + volumeName: postgresql-custom-metrics + defaultMode: 0777 versionsContext: initContainers: - name: pg-init-container diff --git a/deploy/postgresql/templates/metrics-configmap-12.yaml b/deploy/postgresql/templates/metrics-configmap-12.yaml new file mode 100644 index 000000000..338b865a0 --- /dev/null +++ b/deploy/postgresql/templates/metrics-configmap-12.yaml @@ -0,0 +1,279 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgresql12-custom-metrics + labels: + {{- include "postgresql.labels" . | nindent 4 }} +data: + custom-metrics.yaml: |- + pg_postmaster: + query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()" + master: true + metrics: + - start_time_seconds: + usage: "GAUGE" + description: "Time at which postmaster started" + + pg_replication: + query: | + SELECT + (case when (not pg_is_in_recovery() or pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()) then 0 else greatest (0, extract(epoch from (now() - pg_last_xact_replay_timestamp()))) end) as lag, + (case when pg_is_in_recovery() then 0 else 1 end) as is_master + master: true + metrics: + - lag: + usage: "GAUGE" + description: "Replication lag behind master in seconds" + - is_master: + usage: "GAUGE" + description: "Instance is master or slave" + + pg_stat_user_tables: + query: | + SELECT + current_database() datname, + schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_tup_ins, + n_tup_upd, + n_tup_del, + n_tup_hot_upd, + n_live_tup, + n_dead_tup, + n_mod_since_analyze, + COALESCE(last_vacuum, '1970-01-01Z') as last_vacuum, + COALESCE(last_autovacuum, '1970-01-01Z') as last_autovacuum, + COALESCE(last_analyze, '1970-01-01Z') as last_analyze, + COALESCE(last_autoanalyze, '1970-01-01Z') as last_autoanalyze, + vacuum_count, + autovacuum_count, + analyze_count, + autoanalyze_count + FROM + pg_stat_user_tables + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - schemaname: + usage: "LABEL" + description: "Name of the schema that this table is in" + - relname: + usage: "LABEL" + description: "Name of this table" + - seq_scan: + usage: "COUNTER" + description: "Number of sequential scans initiated on this table" + - seq_tup_read: + usage: "COUNTER" + description: "Number of live rows fetched by sequential scans" + - idx_scan: + usage: "COUNTER" + description: "Number of index scans initiated on this table" + - idx_tup_fetch: + usage: "COUNTER" + description: "Number of live rows fetched by index scans" + - n_tup_ins: + usage: "COUNTER" + description: "Number of rows inserted" + - n_tup_upd: + usage: "COUNTER" + description: "Number of rows updated" + - n_tup_del: + usage: "COUNTER" + description: "Number of rows deleted" + - n_tup_hot_upd: + usage: "COUNTER" + description: "Number of rows HOT updated (i.e., with no separate index update required)" + - n_live_tup: + usage: "GAUGE" + description: "Estimated number of live rows" + - n_dead_tup: + usage: "GAUGE" + description: "Estimated number of dead rows" + - n_mod_since_analyze: + usage: "GAUGE" + description: "Estimated number of rows changed since last analyze" + - last_vacuum: + usage: "GAUGE" + description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)" + - last_autovacuum: + usage: "GAUGE" + description: "Last time at which this table was vacuumed by the autovacuum daemon" + - last_analyze: + usage: "GAUGE" + description: "Last time at which this table was manually analyzed" + - last_autoanalyze: + usage: "GAUGE" + description: "Last time at which this table was analyzed by the autovacuum daemon" + - vacuum_count: + usage: "COUNTER" + description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)" + - autovacuum_count: + usage: "COUNTER" + description: "Number of times this table has been vacuumed by the autovacuum daemon" + - analyze_count: + usage: "COUNTER" + description: "Number of times this table has been manually analyzed" + - autoanalyze_count: + usage: "COUNTER" + description: "Number of times this table has been analyzed by the autovacuum daemon" + + pg_statio_user_tables: + query: | + SELECT + current_database() datname, + schemaname, + relname, + heap_blks_read, + heap_blks_hit, + idx_blks_read, + idx_blks_hit, + toast_blks_read, + toast_blks_hit, + tidx_blks_read, + tidx_blks_hit + FROM + pg_statio_user_tables + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - schemaname: + usage: "LABEL" + description: "Name of the schema that this table is in" + - relname: + usage: "LABEL" + description: "Name of this table" + - heap_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table" + - heap_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table" + - idx_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from all indexes on this table" + - idx_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in all indexes on this table" + - toast_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table's TOAST table (if any)" + - toast_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table's TOAST table (if any)" + - tidx_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table's TOAST table indexes (if any)" + - tidx_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table's TOAST table indexes (if any)" + + # WARNING: This set of metrics can be very expensive on a busy server as every unique query executed will create an additional time series + pg_stat_statements: + query: | + SELECT + t2.rolname, + t3.datname, + queryid, + calls, + total_time / 1000 as total_exec_time_seconds, + min_time / 1000 as min_exec_time_seconds, + max_time / 1000 as max_exec_time_seconds, + mean_time / 1000 as mean_exec_time_seconds, + stddev_time / 1000 as stddev_exec_time_seconds, + rows, + shared_blks_hit, + shared_blks_read, + shared_blks_dirtied, + shared_blks_written, + local_blks_hit, + local_blks_read, + local_blks_dirtied, + local_blks_written, + temp_blks_read, + temp_blks_written, + blk_read_time / 1000 as blk_read_time_seconds, + blk_write_time / 1000 as blk_write_time_seconds + FROM + pg_stat_statements t1 + JOIN + pg_roles t2 + ON (t1.userid=t2.oid) + JOIN + pg_database t3 + ON (t1.dbid=t3.oid) + WHERE t2.rolname != 'rdsadmin' + master: true + metrics: + - rolname: + usage: "LABEL" + description: "Name of user" + - datname: + usage: "LABEL" + description: "Name of database" + - queryid: + usage: "LABEL" + description: "Query ID" + - calls: + usage: "COUNTER" + description: "Number of times executed" + - total_exec_time_seconds: + usage: "COUNTER" + description: "Total time spent in the statement" + - min_exec_time_seconds: + usage: "GAUGE" + description: "Minimum time spent in the statement" + - max_exec_time_seconds: + usage: "GAUGE" + description: "Maximum time spent in the statement" + - mean_exec_time_seconds: + usage: "GAUGE" + description: "Mean time spent in the statement" + - stddev_exec_time_seconds: + usage: "GAUGE" + description: "Population standard deviation of time spent in the statement" + - rows: + usage: "COUNTER" + description: "Total number of rows retrieved or affected by the statement" + - shared_blks_hit: + usage: "COUNTER" + description: "Total number of shared block cache hits by the statement" + - shared_blks_read: + usage: "COUNTER" + description: "Total number of shared blocks read by the statement" + - shared_blks_dirtied: + usage: "COUNTER" + description: "Total number of shared blocks dirtied by the statement" + - shared_blks_written: + usage: "COUNTER" + description: "Total number of shared blocks written by the statement" + - local_blks_hit: + usage: "COUNTER" + description: "Total number of local block cache hits by the statement" + - local_blks_read: + usage: "COUNTER" + description: "Total number of local blocks read by the statement" + - local_blks_dirtied: + usage: "COUNTER" + description: "Total number of local blocks dirtied by the statement" + - local_blks_written: + usage: "COUNTER" + description: "Total number of local blocks written by the statement" + - temp_blks_read: + usage: "COUNTER" + description: "Total number of temp blocks read by the statement" + - temp_blks_written: + usage: "COUNTER" + description: "Total number of temp blocks written by the statement" + - blk_read_time_seconds: + usage: "COUNTER" + description: "Total time the statement spent reading blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" + - blk_write_time_seconds: + usage: "COUNTER" + description: "Total time the statement spent writing blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" diff --git a/deploy/postgresql/templates/metrics-configmap-14.yaml b/deploy/postgresql/templates/metrics-configmap-14.yaml new file mode 100644 index 000000000..39232565b --- /dev/null +++ b/deploy/postgresql/templates/metrics-configmap-14.yaml @@ -0,0 +1,319 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgresql14-custom-metrics + labels: + {{- include "postgresql.labels" . | nindent 4 }} +data: + custom-metrics.yaml: |- + pg_postmaster: + query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()" + master: true + metrics: + - start_time_seconds: + usage: "GAUGE" + description: "Time at which postmaster started" + + pg_replication: + query: | + SELECT + (case when (not pg_is_in_recovery() or pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()) then 0 else greatest (0, extract(epoch from (now() - pg_last_xact_replay_timestamp()))) end) as lag, + (case when pg_is_in_recovery() then 0 else 1 end) as is_master + master: true + metrics: + - lag: + usage: "GAUGE" + description: "Replication lag behind master in seconds" + - is_master: + usage: "GAUGE" + description: "Instance is master or slave" + + pg_stat_user_tables: + query: | + SELECT + current_database() datname, + schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_tup_ins, + n_tup_upd, + n_tup_del, + n_tup_hot_upd, + n_live_tup, + n_dead_tup, + n_mod_since_analyze, + n_ins_since_vacuum, + COALESCE(last_vacuum, '1970-01-01Z') as last_vacuum, + COALESCE(last_autovacuum, '1970-01-01Z') as last_autovacuum, + COALESCE(last_analyze, '1970-01-01Z') as last_analyze, + COALESCE(last_autoanalyze, '1970-01-01Z') as last_autoanalyze, + vacuum_count, + autovacuum_count, + analyze_count, + autoanalyze_count + FROM + pg_stat_user_tables + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - schemaname: + usage: "LABEL" + description: "Name of the schema that this table is in" + - relname: + usage: "LABEL" + description: "Name of this table" + - seq_scan: + usage: "COUNTER" + description: "Number of sequential scans initiated on this table" + - seq_tup_read: + usage: "COUNTER" + description: "Number of live rows fetched by sequential scans" + - idx_scan: + usage: "COUNTER" + description: "Number of index scans initiated on this table" + - idx_tup_fetch: + usage: "COUNTER" + description: "Number of live rows fetched by index scans" + - n_tup_ins: + usage: "COUNTER" + description: "Number of rows inserted" + - n_tup_upd: + usage: "COUNTER" + description: "Number of rows updated" + - n_tup_del: + usage: "COUNTER" + description: "Number of rows deleted" + - n_tup_hot_upd: + usage: "COUNTER" + description: "Number of rows HOT updated (i.e., with no separate index update required)" + - n_live_tup: + usage: "GAUGE" + description: "Estimated number of live rows" + - n_dead_tup: + usage: "GAUGE" + description: "Estimated number of dead rows" + - n_mod_since_analyze: + usage: "GAUGE" + description: "Estimated number of rows changed since last analyze" + - n_ins_since_vacuum: + usage: "GAUGE" + description: "Estimated number of rows inserted since this table was last vacuumed" + - last_vacuum: + usage: "GAUGE" + description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)" + - last_autovacuum: + usage: "GAUGE" + description: "Last time at which this table was vacuumed by the autovacuum daemon" + - last_analyze: + usage: "GAUGE" + description: "Last time at which this table was manually analyzed" + - last_autoanalyze: + usage: "GAUGE" + description: "Last time at which this table was analyzed by the autovacuum daemon" + - vacuum_count: + usage: "COUNTER" + description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)" + - autovacuum_count: + usage: "COUNTER" + description: "Number of times this table has been vacuumed by the autovacuum daemon" + - analyze_count: + usage: "COUNTER" + description: "Number of times this table has been manually analyzed" + - autoanalyze_count: + usage: "COUNTER" + description: "Number of times this table has been analyzed by the autovacuum daemon" + + pg_statio_user_tables: + query: | + SELECT + current_database() datname, + schemaname, + relname, + heap_blks_read, + heap_blks_hit, + idx_blks_read, + idx_blks_hit, + toast_blks_read, + toast_blks_hit, + tidx_blks_read, + tidx_blks_hit + FROM + pg_statio_user_tables + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - schemaname: + usage: "LABEL" + description: "Name of the schema that this table is in" + - relname: + usage: "LABEL" + description: "Name of this table" + - heap_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table" + - heap_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table" + - idx_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from all indexes on this table" + - idx_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in all indexes on this table" + - toast_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table's TOAST table (if any)" + - toast_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table's TOAST table (if any)" + - tidx_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table's TOAST table indexes (if any)" + - tidx_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table's TOAST table indexes (if any)" + + # WARNING: This set of metrics can be very expensive on a busy server as every unique query executed will create an additional time series + pg_stat_statements: + query: | + SELECT + t2.rolname, + t3.datname, + queryid, + plans, + total_plan_time / 1000 as total_plan_time_seconds, + min_plan_time / 1000 as min_plan_time_seconds, + max_plan_time / 1000 as max_plan_time_seconds, + mean_plan_time / 1000 as mean_plan_time_seconds, + stddev_plan_time / 1000 as stddev_plan_time_seconds, + calls, + total_exec_time / 1000 as total_exec_time_seconds, + min_exec_time / 1000 as min_exec_time_seconds, + max_exec_time / 1000 as max_exec_time_seconds, + mean_exec_time / 1000 as mean_exec_time_seconds, + stddev_exec_time / 1000 as stddev_exec_time_seconds, + rows, + shared_blks_hit, + shared_blks_read, + shared_blks_dirtied, + shared_blks_written, + local_blks_hit, + local_blks_read, + local_blks_dirtied, + local_blks_written, + temp_blks_read, + temp_blks_written, + blk_read_time / 1000 as blk_read_time_seconds, + blk_write_time / 1000 as blk_write_time_seconds, + wal_records, + wal_fpi, + wal_bytes + FROM + pg_stat_statements t1 + JOIN + pg_roles t2 + ON (t1.userid=t2.oid) + JOIN + pg_database t3 + ON (t1.dbid=t3.oid) + WHERE t2.rolname != 'rdsadmin' + master: true + metrics: + - rolname: + usage: "LABEL" + description: "Name of user" + - datname: + usage: "LABEL" + description: "Name of database" + - queryid: + usage: "LABEL" + description: "Query ID" + - plans: + usage: "COUNTER" + description: "Number of times the statement was planned" + - total_plan_time_seconds: + usage: "COUNTER" + description: "Total time spent planning the statement" + - min_plan_time_seconds: + usage: "GAUGE" + description: "Minimum time spent planning the statement" + - max_plan_time_seconds: + usage: "GAUGE" + description: "Maximum time spent planning the statement" + - mean_plan_time_seconds: + usage: "GAUGE" + description: "Mean time spent planning the statement" + - stddev_plan_time_seconds: + usage: "GAUGE" + description: "Population standard deviation of time spent planning the statement" + - calls: + usage: "COUNTER" + description: "Number of times executed" + - total_exec_time_seconds: + usage: "COUNTER" + description: "Total time spent in the statement" + - min_exec_time_seconds: + usage: "GAUGE" + description: "Minimum time spent in the statement" + - max_exec_time_seconds: + usage: "GAUGE" + description: "Maximum time spent in the statement" + - mean_exec_time_seconds: + usage: "GAUGE" + description: "Mean time spent in the statement" + - stddev_exec_time_seconds: + usage: "GAUGE" + description: "Population standard deviation of time spent in the statement" + - rows: + usage: "COUNTER" + description: "Total number of rows retrieved or affected by the statement" + - shared_blks_hit: + usage: "COUNTER" + description: "Total number of shared block cache hits by the statement" + - shared_blks_read: + usage: "COUNTER" + description: "Total number of shared blocks read by the statement" + - shared_blks_dirtied: + usage: "COUNTER" + description: "Total number of shared blocks dirtied by the statement" + - shared_blks_written: + usage: "COUNTER" + description: "Total number of shared blocks written by the statement" + - local_blks_hit: + usage: "COUNTER" + description: "Total number of local block cache hits by the statement" + - local_blks_read: + usage: "COUNTER" + description: "Total number of local blocks read by the statement" + - local_blks_dirtied: + usage: "COUNTER" + description: "Total number of local blocks dirtied by the statement" + - local_blks_written: + usage: "COUNTER" + description: "Total number of local blocks written by the statement" + - temp_blks_read: + usage: "COUNTER" + description: "Total number of temp blocks read by the statement" + - temp_blks_written: + usage: "COUNTER" + description: "Total number of temp blocks written by the statement" + - blk_read_time_seconds: + usage: "COUNTER" + description: "Total time the statement spent reading blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" + - blk_write_time_seconds: + usage: "COUNTER" + description: "Total time the statement spent writing blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" + - wal_records: + usage: "COUNTER" + description: "Total number of WAL records generated by the statement" + - wal_fpi: + usage: "COUNTER" + description: "Total number of WAL full page images generated by the statement" + - wal_bytes: + usage: "COUNTER" + description: "Total amount of WAL generated by the statement in bytes" diff --git a/deploy/postgresql/templates/metrics-configmap.yaml b/deploy/postgresql/templates/metrics-configmap.yaml deleted file mode 100644 index 4b26e3207..000000000 --- a/deploy/postgresql/templates/metrics-configmap.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: postgresql-custom-metrics - labels: - {{- include "postgresql.labels" . | nindent 4 }} -data: - custom-metrics.yaml: {{ toYaml .Values.metrics.customMetrics | quote }} diff --git a/deploy/postgresql/values.yaml b/deploy/postgresql/values.yaml index a15dee0c3..691cb23d0 100644 --- a/deploy/postgresql/values.yaml +++ b/deploy/postgresql/values.yaml @@ -109,330 +109,5 @@ metrics: ## pullSecrets: [ ] - ## @param metrics.customMetrics Define additional custom metrics - ## ref: https://github.com/wrouesnel/postgres_exporter#adding-new-metrics-via-a-config-file - ## customMetrics: - ## pg_database: - ## query: "SELECT d.datname AS name, CASE WHEN pg_catalog.has_database_privilege(d.datname, 'CONNECT') THEN pg_catalog.pg_database_size(d.datname) ELSE 0 END AS size_bytes FROM pg_catalog.pg_database d where datname not in ('template0', 'template1', 'postgres')" - ## metrics: - ## - name: - ## usage: "LABEL" - ## description: "Name of the database" - ## - size_bytes: - ## usage: "GAUGE" - ## description: "Size of the database in bytes" - ## - customMetrics: - pg_postmaster: - query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()" - master: true - metrics: - - start_time_seconds: - usage: "GAUGE" - description: "Time at which postmaster started" - - pg_replication: - query: | - SELECT - (case when (not pg_is_in_recovery() or pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn()) then 0 else greatest (0, extract(epoch from (now() - pg_last_xact_replay_timestamp()))) end) as lag, - (case when pg_is_in_recovery() then 0 else 1 end) as is_master - master: true - metrics: - - lag: - usage: "GAUGE" - description: "Replication lag behind master in seconds" - - is_master: - usage: "GAUGE" - description: "Instance is master or slave" - - pg_stat_user_tables: - query: | - SELECT - current_database() datname, - schemaname, - relname, - seq_scan, - seq_tup_read, - idx_scan, - idx_tup_fetch, - n_tup_ins, - n_tup_upd, - n_tup_del, - n_tup_hot_upd, - n_live_tup, - n_dead_tup, - n_mod_since_analyze, - n_ins_since_vacuum, - COALESCE(last_vacuum, '1970-01-01Z') as last_vacuum, - COALESCE(last_autovacuum, '1970-01-01Z') as last_autovacuum, - COALESCE(last_analyze, '1970-01-01Z') as last_analyze, - COALESCE(last_autoanalyze, '1970-01-01Z') as last_autoanalyze, - vacuum_count, - autovacuum_count, - analyze_count, - autoanalyze_count - FROM - pg_stat_user_tables - metrics: - - datname: - usage: "LABEL" - description: "Name of current database" - - schemaname: - usage: "LABEL" - description: "Name of the schema that this table is in" - - relname: - usage: "LABEL" - description: "Name of this table" - - seq_scan: - usage: "COUNTER" - description: "Number of sequential scans initiated on this table" - - seq_tup_read: - usage: "COUNTER" - description: "Number of live rows fetched by sequential scans" - - idx_scan: - usage: "COUNTER" - description: "Number of index scans initiated on this table" - - idx_tup_fetch: - usage: "COUNTER" - description: "Number of live rows fetched by index scans" - - n_tup_ins: - usage: "COUNTER" - description: "Number of rows inserted" - - n_tup_upd: - usage: "COUNTER" - description: "Number of rows updated" - - n_tup_del: - usage: "COUNTER" - description: "Number of rows deleted" - - n_tup_hot_upd: - usage: "COUNTER" - description: "Number of rows HOT updated (i.e., with no separate index update required)" - - n_live_tup: - usage: "GAUGE" - description: "Estimated number of live rows" - - n_dead_tup: - usage: "GAUGE" - description: "Estimated number of dead rows" - - n_mod_since_analyze: - usage: "GAUGE" - description: "Estimated number of rows changed since last analyze" - - n_ins_since_vacuum: - usage: "GAUGE" - description: "Estimated number of rows inserted since this table was last vacuumed" - - last_vacuum: - usage: "GAUGE" - description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)" - - last_autovacuum: - usage: "GAUGE" - description: "Last time at which this table was vacuumed by the autovacuum daemon" - - last_analyze: - usage: "GAUGE" - description: "Last time at which this table was manually analyzed" - - last_autoanalyze: - usage: "GAUGE" - description: "Last time at which this table was analyzed by the autovacuum daemon" - - vacuum_count: - usage: "COUNTER" - description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)" - - autovacuum_count: - usage: "COUNTER" - description: "Number of times this table has been vacuumed by the autovacuum daemon" - - analyze_count: - usage: "COUNTER" - description: "Number of times this table has been manually analyzed" - - autoanalyze_count: - usage: "COUNTER" - description: "Number of times this table has been analyzed by the autovacuum daemon" - - pg_statio_user_tables: - query: | - SELECT - current_database() datname, - schemaname, - relname, - heap_blks_read, - heap_blks_hit, - idx_blks_read, - idx_blks_hit, - toast_blks_read, - toast_blks_hit, - tidx_blks_read, - tidx_blks_hit - FROM - pg_statio_user_tables - metrics: - - datname: - usage: "LABEL" - description: "Name of current database" - - schemaname: - usage: "LABEL" - description: "Name of the schema that this table is in" - - relname: - usage: "LABEL" - description: "Name of this table" - - heap_blks_read: - usage: "COUNTER" - description: "Number of disk blocks read from this table" - - heap_blks_hit: - usage: "COUNTER" - description: "Number of buffer hits in this table" - - idx_blks_read: - usage: "COUNTER" - description: "Number of disk blocks read from all indexes on this table" - - idx_blks_hit: - usage: "COUNTER" - description: "Number of buffer hits in all indexes on this table" - - toast_blks_read: - usage: "COUNTER" - description: "Number of disk blocks read from this table's TOAST table (if any)" - - toast_blks_hit: - usage: "COUNTER" - description: "Number of buffer hits in this table's TOAST table (if any)" - - tidx_blks_read: - usage: "COUNTER" - description: "Number of disk blocks read from this table's TOAST table indexes (if any)" - - tidx_blks_hit: - usage: "COUNTER" - description: "Number of buffer hits in this table's TOAST table indexes (if any)" - - # WARNING: This set of metrics can be very expensive on a busy server as every unique query executed will create an additional time series - pg_stat_statements: - query: | - SELECT - t2.rolname, - t3.datname, - queryid, - plans, - total_plan_time / 1000 as total_plan_time_seconds, - min_plan_time / 1000 as min_plan_time_seconds, - max_plan_time / 1000 as max_plan_time_seconds, - mean_plan_time / 1000 as mean_plan_time_seconds, - stddev_plan_time / 1000 as stddev_plan_time_seconds, - calls, - total_exec_time / 1000 as total_exec_time_seconds, - min_exec_time / 1000 as min_exec_time_seconds, - max_exec_time / 1000 as max_exec_time_seconds, - mean_exec_time / 1000 as mean_exec_time_seconds, - stddev_exec_time / 1000 as stddev_exec_time_seconds, - rows, - shared_blks_hit, - shared_blks_read, - shared_blks_dirtied, - shared_blks_written, - local_blks_hit, - local_blks_read, - local_blks_dirtied, - local_blks_written, - temp_blks_read, - temp_blks_written, - blk_read_time / 1000 as blk_read_time_seconds, - blk_write_time / 1000 as blk_write_time_seconds, - wal_records, - wal_fpi, - wal_bytes - FROM - pg_stat_statements t1 - JOIN - pg_roles t2 - ON (t1.userid=t2.oid) - JOIN - pg_database t3 - ON (t1.dbid=t3.oid) - WHERE t2.rolname != 'rdsadmin' - master: true - metrics: - - rolname: - usage: "LABEL" - description: "Name of user" - - datname: - usage: "LABEL" - description: "Name of database" - - queryid: - usage: "LABEL" - description: "Query ID" - - plans: - usage: "COUNTER" - description: "Number of times the statement was planned" - - total_plan_time_seconds: - usage: "COUNTER" - description: "Total time spent planning the statement" - - min_plan_time_seconds: - usage: "GAUGE" - description: "Minimum time spent planning the statement" - - max_plan_time_seconds: - usage: "GAUGE" - description: "Maximum time spent planning the statement" - - mean_plan_time_seconds: - usage: "GAUGE" - description: "Mean time spent planning the statement" - - stddev_plan_time_seconds: - usage: "GAUGE" - description: "Population standard deviation of time spent planning the statement" - - calls: - usage: "COUNTER" - description: "Number of times executed" - - total_exec_time_seconds: - usage: "COUNTER" - description: "Total time spent in the statement" - - min_exec_time_seconds: - usage: "GAUGE" - description: "Minimum time spent in the statement" - - max_exec_time_seconds: - usage: "GAUGE" - description: "Maximum time spent in the statement" - - mean_exec_time_seconds: - usage: "GAUGE" - description: "Mean time spent in the statement" - - stddev_exec_time_seconds: - usage: "GAUGE" - description: "Population standard deviation of time spent in the statement" - - rows: - usage: "COUNTER" - description: "Total number of rows retrieved or affected by the statement" - - shared_blks_hit: - usage: "COUNTER" - description: "Total number of shared block cache hits by the statement" - - shared_blks_read: - usage: "COUNTER" - description: "Total number of shared blocks read by the statement" - - shared_blks_dirtied: - usage: "COUNTER" - description: "Total number of shared blocks dirtied by the statement" - - shared_blks_written: - usage: "COUNTER" - description: "Total number of shared blocks written by the statement" - - local_blks_hit: - usage: "COUNTER" - description: "Total number of local block cache hits by the statement" - - local_blks_read: - usage: "COUNTER" - description: "Total number of local blocks read by the statement" - - local_blks_dirtied: - usage: "COUNTER" - description: "Total number of local blocks dirtied by the statement" - - local_blks_written: - usage: "COUNTER" - description: "Total number of local blocks written by the statement" - - temp_blks_read: - usage: "COUNTER" - description: "Total number of temp blocks read by the statement" - - temp_blks_written: - usage: "COUNTER" - description: "Total number of temp blocks written by the statement" - - blk_read_time_seconds: - usage: "COUNTER" - description: "Total time the statement spent reading blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" - - blk_write_time_seconds: - usage: "COUNTER" - description: "Total time the statement spent writing blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" - - wal_records: - usage: "COUNTER" - description: "Total number of WAL records generated by the statement" - - wal_fpi: - usage: "COUNTER" - description: "Total number of WAL full page images generated by the statement" - - wal_bytes: - usage: "COUNTER" - description: "Total amount of WAL generated by the statement in bytes" logConfigs: running: /home/postgres/pgdata/pgroot/data/log/postgresql-* \ No newline at end of file From 47c51d52e67ab9652122f53a2b183cdc20a02636 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Fri, 5 May 2023 16:31:43 +0800 Subject: [PATCH 226/439] fix: csi-s3/csi-hostpath should consistent with operator's toleration settings (#3087) --- deploy/csi-oss/Chart.yaml | 8 - deploy/csi-oss/README.md | 32 ---- deploy/csi-oss/templates/oss-plugin.yaml | 137 ------------------ deploy/csi-oss/templates/pv-template.yaml | 32 ---- deploy/csi-oss/templates/rbac.yaml | 97 ------------- deploy/csi-oss/templates/secret.yaml | 8 - deploy/csi-oss/values.yaml | 42 ------ deploy/csi-s3/templates/csi-s3.yaml | 2 +- deploy/csi-s3/values.yaml | 8 - .../addons/csi-hostpath-driver-addon.yaml | 7 + .../helm/templates/addons/csi-oss-addon.yaml | 27 ---- .../helm/templates/addons/csi-oss-values.yaml | 13 -- .../helm/templates/addons/csi-s3-addon.yaml | 7 + deploy/helm/values.yaml | 17 --- 14 files changed, 15 insertions(+), 422 deletions(-) delete mode 100644 deploy/csi-oss/Chart.yaml delete mode 100644 deploy/csi-oss/README.md delete mode 100644 deploy/csi-oss/templates/oss-plugin.yaml delete mode 100644 deploy/csi-oss/templates/pv-template.yaml delete mode 100644 deploy/csi-oss/templates/rbac.yaml delete mode 100644 deploy/csi-oss/templates/secret.yaml delete mode 100644 deploy/csi-oss/values.yaml delete mode 100644 deploy/helm/templates/addons/csi-oss-addon.yaml delete mode 100644 deploy/helm/templates/addons/csi-oss-values.yaml diff --git a/deploy/csi-oss/Chart.yaml b/deploy/csi-oss/Chart.yaml deleted file mode 100644 index e8fe9c492..000000000 --- a/deploy/csi-oss/Chart.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -appVersion: 1.1.7 -description: Container Storage Interface (CSI) driver for oss volumes -home: https://github.com/kubernetes-sigs/alibaba-cloud-csi-driver -keywords: -- oss -name: csi-oss -version: 1.1.7 diff --git a/deploy/csi-oss/README.md b/deploy/csi-oss/README.md deleted file mode 100644 index e53782901..000000000 --- a/deploy/csi-oss/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Helm chart for csi-oss - -This chart adds oss volume support to your cluster. - -## Install chart - -- Helm 2.x: `helm install [--set secret.akId=... --set secret.akSecret=... ...] --namespace kube-system --name csi-oss .` -- Helm 3.x: `helm install [--set secret.akId=... --set secret.akSecret=... ...] --namespace kube-system csi-oss` - -After installation succeeds, you can get a status of Chart: `helm status csi-oss`. - -## Delete Chart - -- Helm 2.x: `helm delete --purge csi-oss` -- Helm 3.x: `helm uninstall csi-oss --namespace kube-system` - -## Configuration - -By default, this chart creates a secret and a configmap with persistentVolume. You should at least set `secret.akId`, `secret.akSecret` and `storageConfig.bucket` -to your [Alibaba OSS CSI-DRIVER](https://github.com/kubernetes-sigs/alibaba-cloud-csi-driver/blob/master/docs/oss.md) keys for it to work. - -The following table lists all configuration parameters and their default values. - -| Parameter | Description | Default | -| ---------------------------- |------------------------------------------------------------------------------------|--------------------------------------------------------| -| `storageConfig.endpoint` | Mount OSS access domain name | oss-cn-hangzhou.aliyuncs.com | -| `storageConfig.bucket` | The OSS bucket that needs to be mounted | | -| `storageConfig.path` | Indicates the directory structure of the relative bucket root file during mounting | / | -| `storageConfig.otherOpts` | Support for inputting customized parameters when mounting OSS | -o max_stat_cache_size=0 -o allow_other | -| `secret.name` | Name of the secret | csi-oss-secret | -| `secret.akId` | OSS Access Key | | -| `secret.akSecret` | OSS Secret Key | | \ No newline at end of file diff --git a/deploy/csi-oss/templates/oss-plugin.yaml b/deploy/csi-oss/templates/oss-plugin.yaml deleted file mode 100644 index baea8be37..000000000 --- a/deploy/csi-oss/templates/oss-plugin.yaml +++ /dev/null @@ -1,137 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: CSIDriver -metadata: - name: ossplugin.csi.alibabacloud.com -spec: - attachRequired: false - podInfoOnMount: true ---- -kind: DaemonSet -apiVersion: apps/v1 -metadata: - name: oss-csi-plugin - namespace: {{ .Release.Namespace }} -spec: - selector: - matchLabels: - app: oss-csi-plugin - template: - metadata: - labels: - app: oss-csi-plugin - spec: - {{- with .Values.daemonSetTolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: type - operator: NotIn - values: - - virtual-kubelet - nodeSelector: - beta.kubernetes.io/os: linux - serviceAccount: csi-admin - priorityClassName: system-node-critical - hostNetwork: true - hostPID: true - containers: - - name: oss-driver-registrar - image: {{ .Values.images.registrar }} - imagePullPolicy: Always - args: - - "--v=5" - - "--csi-address=/var/lib/kubelet/csi-plugins/ossplugin.csi.alibabacloud.com/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/csi-plugins/ossplugin.csi.alibabacloud.com/csi.sock" - volumeMounts: - - name: kubelet-dir - mountPath: /var/lib/kubelet/ - - name: registration-dir - mountPath: /registration - - name: csi-plugin - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true - image: {{ .Values.images.csi }} - imagePullPolicy: "Always" - args: - - "--endpoint=$(CSI_ENDPOINT)" - - "--v=2" - - "--driver=oss" - - "--nodeid=$(KUBE_NODE_NAME)" - env: - - name: KUBE_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/csi-plugins/driverplugin.csi.alibabacloud.com-replace/csi.sock - - name: MAX_VOLUMES_PERNODE - value: "15" - - name: SERVICE_TYPE - value: "plugin" - livenessProbe: - httpGet: - path: /healthz - port: healthz - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 30 - timeoutSeconds: 5 - failureThreshold: 5 - ports: - - name: healthz - containerPort: 11260 - protocol: TCP - volumeMounts: - - name: kubelet-dir - mountPath: /var/lib/kubelet/ - mountPropagation: "Bidirectional" - - name: etc - mountPath: /host/etc - - name: host-log - mountPath: /var/log/ - - name: ossconnectordir - mountPath: /host/usr/ - - name: container-dir - mountPath: /var/lib/container - mountPropagation: "Bidirectional" - - name: host-dev - mountPath: /dev - mountPropagation: "HostToContainer" - volumes: - - name: registration-dir - hostPath: - path: /var/lib/kubelet/plugins_registry - type: DirectoryOrCreate - - name: container-dir - hostPath: - path: /var/lib/container - type: DirectoryOrCreate - - name: kubelet-dir - hostPath: - path: /var/lib/kubelet - type: Directory - - name: host-dev - hostPath: - path: /dev - - name: host-log - hostPath: - path: /var/log/ - - name: etc - hostPath: - path: /etc - - name: ossconnectordir - hostPath: - path: /usr/ - updateStrategy: - rollingUpdate: - maxUnavailable: 10% - type: RollingUpdate \ No newline at end of file diff --git a/deploy/csi-oss/templates/pv-template.yaml b/deploy/csi-oss/templates/pv-template.yaml deleted file mode 100644 index a18e29cb1..000000000 --- a/deploy/csi-oss/templates/pv-template.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: oss-persistent-volume-template - namespace: {{ .Release.Namespace }} - labels: - kubeblocks.io/persistent-volume-template: "true" -data: - persistentVolume: | - apiVersion: v1 - kind: PersistentVolume - metadata: - name: $(GENERATE_NAME) - labels: - alicloud-pvname: $(GENERATE_NAME) - spec: - capacity: - storage: 100Gi - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - csi: - driver: ossplugin.csi.alibabacloud.com - volumeHandle: $(GENERATE_NAME) - nodePublishSecretRef: - name: {{ .Values.secret.name }} - namespace: {{ .Release.Namespace }} - volumeAttributes: - bucket: "{{ .Values.storageConfig.bucket }}" - url: "{{ .Values.storageConfig.endpoint }}" - otherOpts: "{{ .Values.storageConfig.otherOpts }}" - path: "{{ .Values.storageConfig.path }}" \ No newline at end of file diff --git a/deploy/csi-oss/templates/rbac.yaml b/deploy/csi-oss/templates/rbac.yaml deleted file mode 100644 index 9b4507db7..000000000 --- a/deploy/csi-oss/templates/rbac.yaml +++ /dev/null @@ -1,97 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: csi-admin - namespace: {{ .Release.Namespace }} ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: alicloud-csi-plugin -rules: - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "update", "create", "delete", "patch"] - - apiGroups: [""] - resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["persistentvolumeclaims/status"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["csinodes"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "watch", "list", "delete", "update", "create"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["csi.storage.k8s.io"] - resources: ["csinodeinfos"] - verbs: ["get", "list", "watch"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments"] - verbs: ["get", "list", "watch", "update", "patch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotclasses"] - verbs: ["get", "list", "watch", "create"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents"] - verbs: ["create", "get", "list", "watch", "update", "delete"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "list", "watch", "delete", "get", "update", "patch"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "create", "list", "watch", "delete", "update"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshotcontents/status"] - verbs: ["update"] - - apiGroups: ["storage.k8s.io"] - resources: ["volumeattachments/status"] - verbs: ["patch"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch"] - - apiGroups: ["snapshot.storage.k8s.io"] - resources: ["volumesnapshots/status"] - verbs: ["update"] - - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["namespaces"] - verbs: ["get", "list"] - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: alicloud-csi-plugin -subjects: - - kind: ServiceAccount - name: csi-admin - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: alicloud-csi-plugin - apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/csi-oss/templates/secret.yaml b/deploy/csi-oss/templates/secret.yaml deleted file mode 100644 index e5b9d4fb3..000000000 --- a/deploy/csi-oss/templates/secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - namespace: {{ .Release.Namespace }} - name: {{ .Values.secret.name }} -stringData: - akId: {{ required "akId required, please input it!" .Values.secret.akId }} - akSecret: {{ required "akSecret required, please input it!" .Values.secret.akSecret }} \ No newline at end of file diff --git a/deploy/csi-oss/values.yaml b/deploy/csi-oss/values.yaml deleted file mode 100644 index 871e46a92..000000000 --- a/deploy/csi-oss/values.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -images: - registrar: registry.cn-hangzhou.aliyuncs.com/acs/csi-node-driver-registrar:v1.2.0 - # Main image - csi: registry.cn-hangzhou.aliyuncs.com/acs/csi-plugin:v1.18.8.47-906bd535-aliyun - -storageConfig: - # Endpoint of the oss service, e.g. oss-cn-hangzhou.aliyuncs.com - endpoint: oss-cn-hangzhou.aliyuncs.com - # oss bucket - bucket: "" - # mount path of the oss bucket - path: "/" - # mount options - otherOpts: "-o max_stat_cache_size=0 -o allow_other" - -secret: - # Name of the secret - name: csi-oss-secret - # AccessKey ID - akId: "" - # AccessKey Secret - akSecret: "" - -daemonSetTolerations: - - operator: Exists - -affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - preference: - matchExpressions: - - key: kb-controller - operator: In - values: - - "true" - - key: kubernetes.io/arch - operator: In - values: - - "amd64" - diff --git a/deploy/csi-s3/templates/csi-s3.yaml b/deploy/csi-s3/templates/csi-s3.yaml index 4e72b8771..9b5ea7340 100644 --- a/deploy/csi-s3/templates/csi-s3.yaml +++ b/deploy/csi-s3/templates/csi-s3.yaml @@ -53,7 +53,7 @@ spec: app: csi-s3 spec: serviceAccount: csi-s3 - {{- with .Values.daemonSetTolerations }} + {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/deploy/csi-s3/values.yaml b/deploy/csi-s3/values.yaml index fca9cada4..5ccd7faf6 100644 --- a/deploy/csi-s3/values.yaml +++ b/deploy/csi-s3/values.yaml @@ -55,14 +55,6 @@ secret: region: "" tolerations: - - key: kb-controller - operator: Equal - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/master - operator: "Exists" - -daemonSetTolerations: - operator: Exists affinity: diff --git a/deploy/helm/templates/addons/csi-hostpath-driver-addon.yaml b/deploy/helm/templates/addons/csi-hostpath-driver-addon.yaml index e98ffcf73..401dc8a7d 100644 --- a/deploy/helm/templates/addons/csi-hostpath-driver-addon.yaml +++ b/deploy/helm/templates/addons/csi-hostpath-driver-addon.yaml @@ -20,8 +20,15 @@ spec: - name: csi-hostpath-driver-chart-kubeblocks-values key: values-kubeblocks-override.yaml + valuesMapping: + jsonMap: + tolerations: tolerations + defaultInstallValues: - enabled: true + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} installable: autoInstall: {{ get ( get ( .Values | toYaml | fromYaml ) "csi-hostpath-driver" ) "enabled" }} diff --git a/deploy/helm/templates/addons/csi-oss-addon.yaml b/deploy/helm/templates/addons/csi-oss-addon.yaml deleted file mode 100644 index e8e43f62c..000000000 --- a/deploy/helm/templates/addons/csi-oss-addon.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: extensions.kubeblocks.io/v1alpha1 -kind: Addon -metadata: - name: csi-oss - labels: - {{- include "kubeblocks.labels" . | nindent 4 }} - "kubeblocks.io/provider": community - {{- if .Values.keepAddons }} - annotations: - helm.sh/resource-policy: keep - {{- end }} -spec: - description: Container Storage Interface (CSI) driver for oss volumes - type: Helm - - helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/csi-oss-{{ default .Chart.Version .Values.versionOverride }}.tgz - installValues: - configMapRefs: - - name: csi-oss-chart-kubeblocks-values - key: values-kubeblocks-override.yaml - - defaultInstallValues: - - enabled: true - - installable: - autoInstall: {{ get ( get ( .Values | toYaml | fromYaml ) "csi-oss" ) "enabled" }} diff --git a/deploy/helm/templates/addons/csi-oss-values.yaml b/deploy/helm/templates/addons/csi-oss-values.yaml deleted file mode 100644 index 0174bdb4a..000000000 --- a/deploy/helm/templates/addons/csi-oss-values.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: csi-oss-chart-kubeblocks-values - labels: - {{- include "kubeblocks.labels" . | nindent 4 }} - {{- if .Values.keepAddons }} - annotations: - helm.sh/resource-policy: keep - {{- end }} -data: - values-kubeblocks-override.yaml: |- - {{- get ( .Values | toYaml | fromYaml ) "csi-oss" | toYaml | nindent 4 }} \ No newline at end of file diff --git a/deploy/helm/templates/addons/csi-s3-addon.yaml b/deploy/helm/templates/addons/csi-s3-addon.yaml index f22d125d6..fa4e32896 100644 --- a/deploy/helm/templates/addons/csi-s3-addon.yaml +++ b/deploy/helm/templates/addons/csi-s3-addon.yaml @@ -21,8 +21,15 @@ spec: - name: csi-s3-chart-kubeblocks-values key: values-kubeblocks-override.yaml + valuesMapping: + jsonMap: + tolerations: tolerations + defaultInstallValues: - enabled: true + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} installable: autoInstall: {{ get ( get ( .Values | toYaml | fromYaml ) "csi-s3" ) "enabled" }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 887a7504a..4e4fd8afd 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1680,23 +1680,6 @@ csi-s3: singleBucket: "" mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" -csi-oss: - enabled: false - ## @param csi-oss.secret.akId -- oss Access Key. - ## @param csi-oss.secret.akSecret -- oss Secret Key. - secret: - akId: "" - akSecret: "" - ## @param csi-oss.storageConfig.bucket -- oss bucket name. - ## @param csi-oss.storageConfig.endpoint -- endpoint of the oss service, e.g. oss-cn-hangzhou.aliyuncs.com. - ## @param csi-oss.storageConfig.path -- mount path of the oss bucket. - ## @param csi-oss.storageConfig.otherOpts -- mount options. - storageConfig: - bucket: "" - endpoint: "oss-cn-hangzhou.aliyuncs.com" - path: "/" - otherOpts: "-o max_stat_cache_size=0 -o allow_other" - alertmanager-webhook-adaptor: ## Linkage with prometheus.enabled ## From 8294e1cc4fc4176fbc2bde424144e2bacbe0e375 Mon Sep 17 00:00:00 2001 From: runsun Date: Fri, 5 May 2023 18:26:45 +0800 Subject: [PATCH 227/439] chore: prometheus and alertmanager use statefulSet by default (#3092) --- deploy/helm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 4e4fd8afd..98d7f9763 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -346,7 +346,7 @@ prometheus: ## If true, use a statefulset instead of a deployment for pod management. ## This allows to scale replicas to more than 1 pod ## - enabled: false + enabled: true ## Alertmanager headless service to use for the statefulset ## @@ -553,7 +553,7 @@ prometheus: ## If true, use a statefulset instead of a deployment for pod management. ## This allows to scale replicas to more than 1 pod ## - enabled: false + enabled: true ## Prometheus server resource requests and limits ## Ref: http://kubernetes.io/docs/user-guide/compute-resources/ From fdbeceb48d72f43c856f063d77b5e217786656c8 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Fri, 5 May 2023 19:04:10 +0800 Subject: [PATCH 228/439] fix: fixed 'addon enable' cmd with specified flags doesn't modified `spec.install.enabled` attribute (#3082) --- apis/extensions/v1alpha1/addon_types.go | 21 ++++++++++++++------- docs/user_docs/cli/kbcli_addon_enable.md | 4 +++- internal/cli/cmd/addon/addon.go | 19 ++++++++++++++++--- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/apis/extensions/v1alpha1/addon_types.go b/apis/extensions/v1alpha1/addon_types.go index bdf0eed1b..93d62ab52 100644 --- a/apis/extensions/v1alpha1/addon_types.go +++ b/apis/extensions/v1alpha1/addon_types.go @@ -531,6 +531,19 @@ func (r *HelmTypeInstallSpec) BuildMergedValues(installSpec *AddonInstallSpec) H } installValues := r.InstallValues processor := func(installSpecItem AddonInstallSpecItem, valueMapping HelmValuesMappingItem) { + var pvEnabled *bool + defer func() { + if v := valueMapping.HelmValueMap.PVEnabled; v != "" && pvEnabled != nil { + installValues.SetValues = append(installValues.SetValues, + fmt.Sprintf("%s=%v", v, *pvEnabled)) + } + }() + + if installSpecItem.PVEnabled != nil { + b := *installSpecItem.PVEnabled + pvEnabled = &b + } + if installSpecItem.Replicas != nil && *installSpecItem.Replicas >= 0 { if v := valueMapping.HelmValueMap.ReplicaCount; v != "" { installValues.SetValues = append(installValues.SetValues, @@ -550,13 +563,6 @@ func (r *HelmTypeInstallSpec) BuildMergedValues(installSpec *AddonInstallSpec) H } } - if installSpecItem.PVEnabled != nil { - if v := valueMapping.HelmValueMap.PVEnabled; v != "" { - installValues.SetValues = append(installValues.SetValues, - fmt.Sprintf("%s=%v", v, *installSpecItem.PVEnabled)) - } - } - if installSpecItem.Tolerations != "" { if v := valueMapping.HelmJSONMap.Tolerations; v != "" { installValues.SetJSONValues = append(installValues.SetJSONValues, @@ -602,6 +608,7 @@ func (r *HelmTypeInstallSpec) BuildMergedValues(installSpec *AddonInstallSpec) H } } } + } processor(installSpec.AddonInstallSpecItem, r.ValuesMapping.HelmValuesMappingItem) for _, ei := range installSpec.ExtraItems { diff --git a/docs/user_docs/cli/kbcli_addon_enable.md b/docs/user_docs/cli/kbcli_addon_enable.md index aefdf67c7..67443ad7f 100644 --- a/docs/user_docs/cli/kbcli_addon_enable.md +++ b/docs/user_docs/cli/kbcli_addon_enable.md @@ -47,7 +47,9 @@ kbcli addon enable ADDON_NAME [flags] --set stringArray set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2), it's only being processed if addon's type is helm. --show-managed-fields If true, keep the managedFields when printing objects in JSON or YAML format. --storage stringArray Sets addon storage size (--storage [extraName:]) (can specify multiple if has extra items)). - Additional notes for Helm type Addon, that resizing storage will fail if modified value is a storage request size + Additional notes: + 1. Specify '0' value will removed storage values settings and explicitly disabled 'persistentVolumeEnabled' attribute. + 2. For Helm type Addon, that resizing storage will fail if modified value is a storage request size that belongs to StatefulSet's volume claim template, to resolve 'Failed' Addon status possible action is disable and re-enable the addon (More info on how-to resize a PVC: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources). diff --git a/internal/cli/cmd/addon/addon.go b/internal/cli/cmd/addon/addon.go index 31231872c..e8f1a54e2 100644 --- a/internal/cli/cmd/addon/addon.go +++ b/internal/cli/cmd/addon/addon.go @@ -203,7 +203,9 @@ func newEnableCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra "Sets addon CPU resource values (--cpu [extraName:]/) (can specify multiple if has extra items))") cmd.Flags().StringArrayVar(&o.addonEnableFlags.StorageSets, "storage", []string{}, `Sets addon storage size (--storage [extraName:]) (can specify multiple if has extra items)). -Additional notes for Helm type Addon, that resizing storage will fail if modified value is a storage request size +Additional notes: +1. Specify '0' value will removed storage values settings and explicitly disabled 'persistentVolumeEnabled' attribute. +2. For Helm type Addon, that resizing storage will fail if modified value is a storage request size that belongs to StatefulSet's volume claim template, to resolve 'Failed' Addon status possible action is disable and re-enable the addon (More info on how-to resize a PVC: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources). `) @@ -427,6 +429,7 @@ func addonEnableDisableHandler(o *addonCmdOpts, cmd *cobra.Command, args []strin func (o *addonCmdOpts) buildEnablePatch(flags []*pflag.Flag, spec, install map[string]interface{}) (err error) { extraNames := o.addon.GetExtraNames() installSpec := extensionsv1alpha1.AddonInstallSpec{ + Enabled: true, AddonInstallSpecItem: extensionsv1alpha1.NewAddonInstallSpecItem(), } // only using named return value in defer function @@ -445,7 +448,6 @@ func (o *addonCmdOpts) buildEnablePatch(flags []*pflag.Flag, spec, install map[s }() if o.addonEnableFlags.useDefault() { - installSpec.Enabled = true return nil } @@ -599,7 +601,18 @@ func (o *addonCmdOpts) buildEnablePatch(flags []*pflag.Flag, spec, install map[s } return q, nil }, func(item *extensionsv1alpha1.AddonInstallSpecItem, i interface{}) { - item.Resources.Requests[corev1.ResourceStorage] = i.(resource.Quantity) + q := i.(resource.Quantity) + // for 0 storage size, remove storage request value and explicitly disabled `persistentVolumeEnabled` + if v, _ := q.AsInt64(); v == 0 { + delete(item.Resources.Requests, corev1.ResourceStorage) + b := false + item.PVEnabled = &b + return + } + item.Resources.Requests[corev1.ResourceStorage] = q + // explicitly enabled `persistentVolumeEnabled` if provided storage size settings + b := true + item.PVEnabled = &b }); err != nil { return err } From f28335342ad62fe97c6c99fb7db2c2fd8488864a Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Sat, 6 May 2023 11:00:33 +0800 Subject: [PATCH 229/439] chore: remove runner and bot feedback message (#3095) --- .github/utils/utils.sh | 190 +++++++++++++++++++++++ .github/workflows/cicd-pull-request.yml | 21 +++ .github/workflows/cicd-push.yml | 24 ++- .github/workflows/package-version.yml | 20 +++ .github/workflows/release-create.yml | 16 +- .github/workflows/release-helm-chart.yml | 19 +++ .github/workflows/release-image.yml | 19 +++ .github/workflows/release-publish.yml | 54 +++++-- .github/workflows/release-version.yml | 46 ++++++ .github/workflows/trigger-release.yml | 41 +++++ 10 files changed, 429 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/trigger-release.yml diff --git a/.github/utils/utils.sh b/.github/utils/utils.sh index 5953d730d..414a0bdc4 100644 --- a/.github/utils/utils.sh +++ b/.github/utils/utils.sh @@ -17,9 +17,20 @@ Usage: $(basename "$0") 5) update release latest 6) get the ci trigger mode 7) check package version + 8) kill apiserver and etcd + 9) remove runner + 10) trigger release + 11) release message + 12) send message -tn, --tag-name Release tag name -gr, --github-repo Github Repo -gt, --github-token Github token + -rn, --runner-name The runner name + -bn, --branch-name The branch name + -c, --content The trigger request content + -bw, --bot-webhook The bot webhook + -tt, --trigger-type The trigger type (e.g. release/package) + -ru, --run-url The run url EOF } @@ -32,6 +43,13 @@ main() { local GITHUB_REPO local GITHUB_TOKEN local TRIGGER_MODE="" + local RUNNER_NAME="" + local BRANCH_NAME="" + local CONTENT="" + local BOT_WEBHOOK="" + local TRIGGER_TYPE="release" + local RELEASE_VERSION="" + local RUN_URL="" parse_command_line "$@" @@ -57,6 +75,21 @@ main() { 7) check_package_version ;; + 8) + kill_server_etcd + ;; + 9) + remove_runner + ;; + 10) + trigger_release + ;; + 11) + release_message + ;; + 12) + send_message + ;; *) show_help break @@ -95,6 +128,42 @@ parse_command_line() { shift fi ;; + -rn|--runner-name) + if [[ -n "${2:-}" ]]; then + RUNNER_NAME="$2" + shift + fi + ;; + -bn|--branch-name) + if [[ -n "${2:-}" ]]; then + BRANCH_NAME="$2" + shift + fi + ;; + -c|--content) + if [[ -n "${2:-}" ]]; then + CONTENT="$2" + shift + fi + ;; + -bw|--bot-webhook) + if [[ -n "${2:-}" ]]; then + BOT_WEBHOOK="$2" + shift + fi + ;; + -tt|--trigger-type) + if [[ -n "${2:-}" ]]; then + TRIGGER_TYPE="$2" + shift + fi + ;; + -ru|--run-url) + if [[ -n "${2:-}" ]]; then + RUN_URL="$2" + shift + fi + ;; *) break ;; @@ -128,6 +197,127 @@ update_release_latest() { -d '{"draft":false,"prerelease":false,"make_latest":true}' } +kill_server_etcd() { + server="kube-apiserver\|etcd" + for pid in $( ps -ef | grep "$server" | grep -v "grep $server" | awk '{print $2}' ); do + kill $pid + done +} + +remove_runner() { + runners_url=$GITHUB_API/repos/$LATEST_REPO/actions/runners + runners_list=$( gh_curl -s $runners_url ) + total_count=$( echo "$runners_list" | jq '.total_count' ) + for i in $(seq 0 $total_count); do + if [[ "$i" == "$total_count" ]]; then + break + fi + runner_name=$( echo "$runners_list" | jq ".runners[$i].name" --raw-output ) + runner_status=$( echo "$runners_list" | jq ".runners[$i].status" --raw-output ) + runner_busy=$( echo "$runners_list" | jq ".runners[$i].busy" --raw-output ) + runner_id=$( echo "$runners_list" | jq ".runners[$i].id" --raw-output ) + if [[ "$runner_name" == "$RUNNER_NAME" && "$runner_status" == "online" && "$runner_busy" == "false" ]]; then + echo "runner_name:"$runner_name + gh_curl -L -X DELETE $runners_url/$runner_id + break + fi + done +} + +check_numeric() { + input=${1:-""} + if [[ $input =~ ^[0-9]+$ ]]; then + echo $(( ${input} )) + else + echo "no" + fi +} + +get_next_available_tag() { + tag_type="$1" + index="" + release_list=$( gh release list --repo $LATEST_REPO ) + for tag in $( echo "$release_list" | (grep "$tag_type" || true) ) ;do + if [[ "$tag" != "$tag_type"* ]]; then + continue + fi + tmp=${tag#*$tag_type} + numeric=$( check_numeric "$tmp" ) + if [[ "$numeric" == "no" ]]; then + continue + fi + if [[ $numeric -gt $index ]]; then + index=$numeric + fi + done + + if [[ -z "$index" ]];then + index=0 + else + index=$(( $index + 1 )) + fi + + RELEASE_VERSION="${tag_type}${index}" +} + +release_next_available_tag() { + dispatches_url=$1 + v_head="v$TAG_NAME" + alpha_type="$v_head.0-alpha." + beta_type="$v_head.0-beta." + rc_type="$v_head.0-rc." + stable_type="$v_head." + case "$CONTENT" in + *alpha*) + get_next_available_tag "$alpha_type" + ;; + *beta*) + get_next_available_tag "$beta_type" + ;; + *rc*) + get_next_available_tag "$rc_type" + ;; + *stable*) + get_next_available_tag $stable_type + ;; + esac + + if [[ ! -z "$RELEASE_VERSION" ]];then + gh_curl -X POST $dispatches_url -d '{"ref":"'$BRANCH_NAME'","inputs":{"release_version":"'$RELEASE_VERSION'"}}' + fi +} + +usage_message() { + curl -H "Content-Type: application/json" -X POST $BOT_WEBHOOK \ + -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Usage:","content":[[{"tag":"text","text":"sorry master, please enter the correct format\n"},{"tag":"text","text":"1. do release\n"},{"tag":"text","text":"2. {\"ref\":\"\",\"inputs\":{\"release_version\":\"\"}}"}]]}}}}' +} + +trigger_release() { + echo "CONTENT:$CONTENT" + dispatches_url=$GITHUB_API/repos/$LATEST_REPO/actions/workflows/$TRIGGER_TYPE-version.yml/dispatches + + if [[ "$CONTENT" == "do"*"release" ]]; then + release_next_available_tag "$dispatches_url" + else + usage_message + fi +} + +release_message() { + curl -H "Content-Type: application/json" -X POST $BOT_WEBHOOK \ + -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Release:","content":[[{"tag":"text","text":"yes master, release "},{"tag":"a","text":"['$TAG_NAME']","href":"https://github.com/'$LATEST_REPO'/releases/tag/'$TAG_NAME'"},{"tag":"text","text":" is on its way..."}]]}}}}' +} + +send_message() { + if [[ "$CONTENT" == *"success" ]]; then + curl -H "Content-Type: application/json" -X POST $BOT_WEBHOOK \ + -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Success:","content":[[{"tag":"text","text":"'$CONTENT'"}]]}}}}' + else + curl -H "Content-Type: application/json" -X POST $BOT_WEBHOOK \ + -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Error:","content":[[{"tag":"text","text":"sorry master, "},{"tag":"a","text":"['$CONTENT']","href":"'$RUN_URL'"}]]}}}}' + fi +} + add_trigger_mode() { trigger_mode=$1 if [[ "$TRIGGER_MODE" != *"$trigger_mode"* ]]; then diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index 498badfdc..2c2cabb7d 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -32,6 +32,8 @@ jobs: name: make test needs: trigger-mode if: contains(needs.trigger-mode.outputs.trigger-mode, '[test]') + outputs: + runner-name: ${{ steps.get_runner_name.outputs.runner_name }} runs-on: [ self-hosted, eks-fargate-runner ] steps: - uses: apecloud/checkout@main @@ -46,6 +48,25 @@ jobs: run: | make test + - name: kill kube-apiserver and etcd + id: get_runner_name + if: ${{ always() }} + run: | + echo runner_name=${RUNNER_NAME} >> $GITHUB_OUTPUT + bash .github/utils/utils.sh --type 8 + + remove-runner: + needs: make-test + runs-on: ubuntu-latest + if: ${{ always() }} + steps: + - uses: actions/checkout@v3 + - name: remove runner + run: | + bash .github/utils/utils.sh --type 9 \ + --github-token ${{ env.GITHUB_TOKEN }} \ + --runner-name ${{ needs.make-test.outputs.runner-name }} + check-image: name: check image needs: trigger-mode diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index f359ad76a..5085337de 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -10,9 +10,6 @@ on: env: GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - GITLAB_GO_CACHE_PROJECT_ID: 98800 - GO_CACHE: "go-cache" - GO_CACHE_DIR: "/root/.cache" GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }} @@ -114,6 +111,8 @@ jobs: needs: trigger-mode runs-on: [self-hosted, eks-fargate-runner ] if: contains(needs.trigger-mode.outputs.trigger-mode, '[test]') + outputs: + runner-name: ${{ steps.get_runner_name.outputs.runner_name }} steps: - uses: apecloud/checkout@main - name: make manifests check @@ -147,6 +146,25 @@ jobs: name: codecov-report verbose: true + - name: kill kube-apiserver and etcd + id: get_runner_name + if: ${{ always() }} + run: | + echo runner_name=${RUNNER_NAME} >> $GITHUB_OUTPUT + bash .github/utils/utils.sh --type 8 + + remove-runner: + needs: make-test + runs-on: ubuntu-latest + if: ${{ needs.make-test.result != 'skipped' }} + steps: + - uses: actions/checkout@v3 + - name: remove runner + run: | + bash .github/utils/utils.sh --type 9 \ + --github-token ${{ env.GITHUB_TOKEN }} \ + --runner-name ${{ needs.make-test.outputs.runner-name }} + check-image: needs: trigger-mode if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') && github.ref_name != 'main' }} diff --git a/.github/workflows/package-version.yml b/.github/workflows/package-version.yml index afd1b2346..7135ab104 100644 --- a/.github/workflows/package-version.yml +++ b/.github/workflows/package-version.yml @@ -12,6 +12,7 @@ run-name: ref_name:${{ github.ref_name }} release_version:${{ inputs.release_ver env: GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PACKAGE_BOT_WEBHOOK: ${{ secrets.PACKAGE_BOT_WEBHOOK }} jobs: @@ -21,6 +22,12 @@ jobs: - name: checkout branch ${{ github.ref_name }} uses: actions/checkout@v3 + - name: package message + run: | + bash .github/utils/utils.sh --type 11 \ + --tag-name "${{ inputs.release_version }}" \ + --bot-webhook ${{ env.PACKAGE_BOT_WEBHOOK }} + - name: package check run: | bash .github/utils/utils.sh --type 7 --tag-name "${{ inputs.release_version }}" @@ -31,3 +38,16 @@ jobs: custom_tag: ${{ inputs.release_version }} github_token: ${{ env.GITHUB_TOKEN }} tag_prefix: "" + + send-message: + runs-on: ubuntu-latest + needs: package-version + if: ${{ failure() || cancelled() }} + steps: + - uses: actions/checkout@v3 + - name: send message + run: | + bash .github/utils/utils.sh --type 12 \ + --content "package\u00a0error" \ + --bot-webhook ${{ env.PACKAGE_BOT_WEBHOOK }} \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" diff --git a/.github/workflows/release-create.yml b/.github/workflows/release-create.yml index 8a59dc887..6232e9372 100644 --- a/.github/workflows/release-create.yml +++ b/.github/workflows/release-create.yml @@ -7,6 +7,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + RELEASE_BOT_WEBHOOK: ${{ secrets.RELEASE_BOT_WEBHOOK }} jobs: publish: @@ -38,4 +39,17 @@ jobs: prerelease: true - name: sanitized release body if: not ${{ env.WITH_RELEASE_NOTES }} - run: ./.github/utils/sanitize_release_body.sh \ No newline at end of file + run: ./.github/utils/sanitize_release_body.sh + + send-message: + runs-on: ubuntu-latest + needs: publish + if: ${{ failure() || cancelled() }} + steps: + - uses: actions/checkout@v3 + - name: send message + run: | + bash .github/utils/utils.sh --type 12 \ + --content "release\u00a0create\u00a0error"\ + --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ No newline at end of file diff --git a/.github/workflows/release-helm-chart.yml b/.github/workflows/release-helm-chart.yml index c8c3fb38e..850201c64 100644 --- a/.github/workflows/release-helm-chart.yml +++ b/.github/workflows/release-helm-chart.yml @@ -13,6 +13,7 @@ on: env: RELEASE_VERSION: ${{ github.ref_name }} + RELEASE_BOT_WEBHOOK: ${{ secrets.RELEASE_BOT_WEBHOOK }} jobs: @@ -41,4 +42,22 @@ jobs: DEP_CHART_DIR: "deploy/helm/depend-charts" secrets: inherit + send-message: + runs-on: ubuntu-latest + needs: release-chart + if: ${{ always() && github.event.action == 'published' }} + steps: + - uses: actions/checkout@v3 + - name: send message + run: | + CONTENT="release\u00a0chart\u00a0error" + if [[ "${{ needs.make-test.result }}" == "success" ]]; then + CONTENT="release\u00a0chart\u00a0success" + fi + + bash .github/utils/utils.sh --type 12 \ + --content "${CONTENT}"\ + --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 04df61eb1..7b3a56bf7 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -15,6 +15,7 @@ on: env: RELEASE_VERSION: ${{ github.ref_name }} + RELEASE_BOT_WEBHOOK: ${{ secrets.RELEASE_BOT_WEBHOOK }} jobs: @@ -57,3 +58,21 @@ jobs: VERSION: "${{ needs.image-tag.outputs.tag-name }}" GO_VERSION: "1.20" secrets: inherit + + send-message: + runs-on: ubuntu-latest + needs: [ release-image, release-tools-image ] + if: ${{ always() && github.event.action == 'published' }} + steps: + - uses: actions/checkout@v3 + - name: send message + run: | + CONTENT="release\u00a0image\u00a0error" + if [[ "${{ needs.release-image.result }}" == "success" && "${{ needs.release-tools-image.result }}" == "success" ]]; then + CONTENT="release\u00a0image\u00a0success" + fi + + bash .github/utils/utils.sh --type 12 \ + --content "${CONTENT}"\ + --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index b4a09985a..e977e8310 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -13,6 +13,7 @@ env: CLI_REPO: 'apecloud/kbcli' GITLAB_KBCLI_PROJECT_ID: 85948 GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }} + RELEASE_BOT_WEBHOOK: ${{ secrets.RELEASE_BOT_WEBHOOK }} jobs: create-release-kbcli: @@ -101,13 +102,17 @@ jobs: echo "ASSET_NAME=${{ env.CLI_FILENAME }}.tar.gz" >> $GITHUB_ENV echo "ASSET_CONTENT_TYPE=application/gzip" >> $GITHUB_ENV - - name: upload release asset ${{ matrix.os }} - uses: actions/upload-release-asset@main - with: - upload_url: ${{ steps.get_release.outputs.upload_url }} - asset_path: ./bin/${{ env.ASSET_NAME }} - asset_name: ${{ env.ASSET_NAME }} - asset_content_type: ${{ env.ASSET_CONTENT_TYPE }} + - name: upload gitlab kbcli asset ${{ matrix.os }} + env: + CLI_BINARY: ${{ env.CLI_NAME }}-${{ matrix.os }}-${{ env.TAG_NAME }}.tar.gz + run: | + bash ${{ github.workspace }}/.github/utils/release_gitlab.sh \ + --type 2 \ + --project-id ${{ env.GITLAB_KBCLI_PROJECT_ID }} \ + --tag-name ${{ env.TAG_NAME }} \ + --asset-path ./bin/${{ env.ASSET_NAME }} \ + --asset-name ${{ env.ASSET_NAME }} \ + --access-token ${{ env.GITLAB_ACCESS_TOKEN }} - name: get release kbcli upload url run: | @@ -125,14 +130,29 @@ jobs: asset_name: ${{ env.ASSET_NAME }} asset_content_type: ${{ env.ASSET_CONTENT_TYPE }} - - name: upload gitlab kbcli asset ${{ matrix.os }} - env: - CLI_BINARY: ${{ env.CLI_NAME }}-${{ matrix.os }}-${{ env.TAG_NAME }}.tar.gz + - name: upload release asset ${{ matrix.os }} + continue-on-error: true + uses: actions/upload-release-asset@main + with: + upload_url: ${{ steps.get_release.outputs.upload_url }} + asset_path: ./bin/${{ env.ASSET_NAME }} + asset_name: ${{ env.ASSET_NAME }} + asset_content_type: ${{ env.ASSET_CONTENT_TYPE }} + + send-message: + runs-on: ubuntu-latest + needs: upload-release-assert + if: ${{ always() }} + steps: + - uses: actions/checkout@v3 + - name: send message run: | - bash ${{ github.workspace }}/.github/utils/release_gitlab.sh \ - --type 2 \ - --project-id ${{ env.GITLAB_KBCLI_PROJECT_ID }} \ - --tag-name ${{ env.TAG_NAME }} \ - --asset-path ./bin/${{ env.ASSET_NAME }} \ - --asset-name ${{ env.ASSET_NAME }} \ - --access-token ${{ env.GITLAB_ACCESS_TOKEN }} + CONTENT="release\u00a0kbcli\u00a0error" + if [[ "${{ needs.upload-release-assert.result }}" == "success" ]]; then + CONTENT="release\u00a0kbcli\u00a0success" + fi + + bash .github/utils/utils.sh --type 12 \ + --content "${CONTENT}"\ + --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml index 535bbcde5..061b6fb0b 100644 --- a/.github/workflows/release-version.yml +++ b/.github/workflows/release-version.yml @@ -12,6 +12,7 @@ run-name: ref_name:${{ github.ref_name }} release_version:${{ inputs.release_ver env: GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + RELEASE_BOT_WEBHOOK: ${{ secrets.RELEASE_BOT_WEBHOOK }} jobs: @@ -22,8 +23,21 @@ jobs: - name: Auto merge releasing PR run: ./.github/utils/merge_releasing_pr.sh + release-message: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: release message + run: | + bash .github/utils/utils.sh --type 11 \ + --tag-name "${{ inputs.release_version }}" \ + --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} + release-test: + needs: release-message runs-on: [ self-hosted, eks-fargate-runner ] + outputs: + runner-name: ${{ steps.get_runner_name.outputs.runner_name }} steps: - uses: apecloud/checkout@main - name: vendor lint test @@ -33,6 +47,25 @@ jobs: cp -r /go/bin/setup-envtest ./bin/setup-envtest make mod-vendor lint test + - name: kill kube-apiserver and etcd + id: get_runner_name + if: ${{ always() }} + run: | + echo runner_name=${RUNNER_NAME} >> $GITHUB_OUTPUT + bash .github/utils/utils.sh --type 8 + + remove-runner: + needs: make-test + runs-on: ubuntu-latest + if: ${{ always() }} + steps: + - uses: actions/checkout@v3 + - name: remove runner + run: | + bash .github/utils/utils.sh --type 9 \ + --github-token ${{ env.GITHUB_TOKEN }} \ + --runner-name ${{ needs.make-test.outputs.runner-name }} + release-version: needs: release-test runs-on: ubuntu-latest @@ -45,3 +78,16 @@ jobs: custom_tag: ${{ inputs.release_version }} github_token: ${{ env.GITHUB_TOKEN }} tag_prefix: "" + + send-message: + runs-on: ubuntu-latest + needs: release-version + if: ${{ failure() || cancelled() }} + steps: + - uses: actions/checkout@v3 + - name: send message + run: | + bash .github/utils/utils.sh --type 12 \ + --content "release\u00a0error"\ + --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ No newline at end of file diff --git a/.github/workflows/trigger-release.yml b/.github/workflows/trigger-release.yml new file mode 100644 index 000000000..52e2e432e --- /dev/null +++ b/.github/workflows/trigger-release.yml @@ -0,0 +1,41 @@ +name: TRIGGER-RELEASE + +on: + workflow_dispatch: + inputs: + trigger-content: + description: 'the trigger request content' + required: false + default: '' + trigger-type: + description: 'the trigger type (e.g. release/package)' + required: false + default: 'release' + +run-name: ${{ inputs.trigger-type }}:${{ inputs.trigger-content }} + +env: + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PACKAGE_BOT_WEBHOOK: ${{ secrets.PACKAGE_BOT_WEBHOOK }} + RELEASE_BOT_WEBHOOK: ${{ secrets.RELEASE_BOT_WEBHOOK }} + + +jobs: + trigger-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: trigger release + id: get_release_version + run: | + BOT_WEBHOOK=${{ env.RELEASE_BOT_WEBHOOK }} + if [[ "${{ inputs.trigger-type }}" == "package" ]]; then + BOT_WEBHOOK=${{ env.PACKAGE_BOT_WEBHOOK }} + fi + RELEASE_VERSION=`bash .github/utils/utils.sh --type 10 \ + --tag-name "${{ vars.CURRENT_RELEASE_VERSION }}" \ + --branch-name "${{ vars.CURRENT_RELEASE_BRANCH }}" \ + --content '${{ inputs.trigger-content }}' \ + --trigger-type "${{ inputs.trigger-type }}" \ + --bot-webhook ${BOT_WEBHOOK} \ + --github-token ${{ env.GITHUB_TOKEN }}` From d536170b69b3c12dfb7e4d9285f5a83976549f93 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Sat, 6 May 2023 11:15:22 +0800 Subject: [PATCH 230/439] chore: fix remove runner (#3105) --- .github/workflows/release-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml index 061b6fb0b..89a34960d 100644 --- a/.github/workflows/release-version.yml +++ b/.github/workflows/release-version.yml @@ -55,7 +55,7 @@ jobs: bash .github/utils/utils.sh --type 8 remove-runner: - needs: make-test + needs: release-test runs-on: ubuntu-latest if: ${{ always() }} steps: From f53b81dd56c7c8a2f9e23e12e8ecd0f0ca0829af Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Sat, 6 May 2023 11:55:26 +0800 Subject: [PATCH 231/439] chore: improve kubeblocks install and uninstall command (#3053) --- .../user_docs/cli/kbcli_kubeblocks_install.md | 3 +- .../cli/kbcli_kubeblocks_uninstall.md | 2 + .../user_docs/cli/kbcli_kubeblocks_upgrade.md | 3 +- .../user_docs/cli/kbcli_playground_destroy.md | 5 +- internal/cli/cmd/cli.go | 9 + internal/cli/cmd/cluster/dataprotection.go | 2 +- internal/cli/cmd/kubeblocks/config.go | 1 + internal/cli/cmd/kubeblocks/install.go | 161 +++++------ .../cli/cmd/kubeblocks/kubeblocks_objects.go | 13 +- internal/cli/cmd/kubeblocks/status.go | 4 +- internal/cli/cmd/kubeblocks/uninstall.go | 256 ++++++++++-------- internal/cli/cmd/kubeblocks/upgrade.go | 16 +- internal/cli/cmd/playground/destroy.go | 34 ++- internal/cli/cmd/playground/init.go | 30 +- internal/cli/cmd/playground/util.go | 8 +- internal/cli/printer/spinner.go | 77 ------ internal/cli/spinner/spinner.go | 147 ++++++++++ .../cli/{printer => spinner}/spinner_test.go | 10 +- internal/cli/types/types.go | 16 +- internal/cli/util/helm/config.go | 4 +- internal/cli/util/helm/helm.go | 5 +- internal/cli/util/log.go | 88 ++++++ internal/cli/util/util.go | 6 - 23 files changed, 560 insertions(+), 340 deletions(-) delete mode 100644 internal/cli/printer/spinner.go create mode 100644 internal/cli/spinner/spinner.go rename internal/cli/{printer => spinner}/spinner_test.go (84%) create mode 100644 internal/cli/util/log.go diff --git a/docs/user_docs/cli/kbcli_kubeblocks_install.md b/docs/user_docs/cli/kbcli_kubeblocks_install.md index 5e65b928d..1b13816d2 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks_install.md +++ b/docs/user_docs/cli/kbcli_kubeblocks_install.md @@ -35,9 +35,10 @@ kbcli kubeblocks install [flags] --set-file stringArray Set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2) --set-json stringArray Set JSON values on the command line (can specify multiple or separate values with commas: key1=jsonval1,key2=jsonval2) --set-string stringArray Set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) - --timeout duration Time to wait for installing KubeBlocks (default 30m0s) + --timeout duration Time to wait for installing KubeBlocks, such as --timeout=10m (default 5m0s) -f, --values strings Specify values in a YAML file or a URL (can specify multiple) --version string KubeBlocks version + --wait Wait for KubeBlocks to be ready, including all the auto installed add-ons. It will wait for as long as --timeout (default true) ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_kubeblocks_uninstall.md b/docs/user_docs/cli/kbcli_kubeblocks_uninstall.md index 745b7e669..5db4e1990 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks_uninstall.md +++ b/docs/user_docs/cli/kbcli_kubeblocks_uninstall.md @@ -23,6 +23,8 @@ kbcli kubeblocks uninstall [flags] --remove-namespace Remove default created "kb-system" namespace or not --remove-pvcs Remove PersistentVolumeClaim or not --remove-pvs Remove PersistentVolume or not + --timeout duration Time to wait for uninstalling KubeBlocks, such as --timeout=5m (default 5m0s) + --wait Wait for KubeBlocks to be uninstalled, including all the add-ons. It will wait for as long as --timeout (default true) ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_kubeblocks_upgrade.md b/docs/user_docs/cli/kbcli_kubeblocks_upgrade.md index f65f7adcf..04c627334 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks_upgrade.md +++ b/docs/user_docs/cli/kbcli_kubeblocks_upgrade.md @@ -27,9 +27,10 @@ kbcli kubeblocks upgrade [flags] --set-file stringArray Set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2) --set-json stringArray Set JSON values on the command line (can specify multiple or separate values with commas: key1=jsonval1,key2=jsonval2) --set-string stringArray Set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) - --timeout duration Time to wait for upgrading KubeBlocks (default 30m0s) + --timeout duration Time to wait for upgrading KubeBlocks, such as --timeout=10m (default 5m0s) -f, --values strings Specify values in a YAML file or a URL (can specify multiple) --version string Set KubeBlocks version + --wait Wait for KubeBlocks to be ready. It will wait for as long as --timeout (default true) ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_playground_destroy.md b/docs/user_docs/cli/kbcli_playground_destroy.md index 5cb02a067..8c621adac 100644 --- a/docs/user_docs/cli/kbcli_playground_destroy.md +++ b/docs/user_docs/cli/kbcli_playground_destroy.md @@ -18,8 +18,9 @@ kbcli playground destroy [flags] ### Options ``` - -h, --help help for destroy - --purge Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks. (default true) + -h, --help help for destroy + --purge Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks. (default true) + --timeout duration Time to wait for installing KubeBlocks, such as --timeout=10m (default 30m0s) ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/cli.go b/internal/cli/cmd/cli.go index 7f128585c..fa454cb57 100644 --- a/internal/cli/cmd/cli.go +++ b/internal/cli/cmd/cli.go @@ -54,6 +54,12 @@ const ( cliName = "kbcli" ) +func init() { + if _, err := util.GetCliHomeDir(); err != nil { + fmt.Println("Failed to create kbcli home dir:", err) + } +} + func NewDefaultCliCmd() *cobra.Command { cmd := NewCliCmd() @@ -124,6 +130,9 @@ A Command Line Interface for KubeBlocks`, matchVersionKubeConfigFlags := cmdutil.NewMatchVersionFlags(kubeConfigFlags) matchVersionKubeConfigFlags.AddFlags(flags) + // add klog flags + util.AddKlogFlags(flags) + f := cmdutil.NewFactory(matchVersionKubeConfigFlags) ioStreams := genericclioptions.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr} diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index eae2723ea..2c531e3c9 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -283,7 +283,7 @@ func NewListBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *c cmd := &cobra.Command{ Use: "list-backups", Short: "List backups.", - Aliases: []string{"ls-backup"}, + Aliases: []string{"ls-backups"}, Example: listBackupExample, ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { diff --git a/internal/cli/cmd/kubeblocks/config.go b/internal/cli/cmd/kubeblocks/config.go index c606e555b..e9e90b953 100644 --- a/internal/cli/cmd/kubeblocks/config.go +++ b/internal/cli/cmd/kubeblocks/config.go @@ -83,6 +83,7 @@ func NewConfigCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra o := &InstallOptions{ Options: Options{ IOStreams: streams, + Wait: true, }, } diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index c70515e0d..5d14e1585 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -27,16 +27,15 @@ import ( "strings" "time" - "github.com/briandowns/spinner" + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/spf13/viper" - "golang.org/x/exp/slices" "helm.sh/helm/v3/pkg/cli/values" "helm.sh/helm/v3/pkg/repo" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -46,6 +45,7 @@ import ( extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/spinner" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" @@ -65,6 +65,8 @@ type Options struct { Namespace string Client kubernetes.Interface Dynamic dynamic.Interface + Timeout time.Duration + Wait bool } type InstallOptions struct { @@ -75,7 +77,6 @@ type InstallOptions struct { CreateNamespace bool Check bool ValueOpts values.Options - timeout time.Duration } var ( @@ -91,6 +92,10 @@ var ( # Install KubeBlocks with other settings, for example, set replicaCount to 3 kbcli kubeblocks install --set replicaCount=3`) + + spinnerMsg = func(format string, a ...any) spinner.Option { + return spinner.WithMessage(fmt.Sprintf("%-50s", fmt.Sprintf(format, a...))) + } ) func newInstallCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { @@ -115,7 +120,8 @@ func newInstallCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr cmd.Flags().StringVar(&o.Version, "version", version.DefaultKubeBlocksVersion, "KubeBlocks version") cmd.Flags().BoolVar(&o.CreateNamespace, "create-namespace", false, "Create the namespace if not present") cmd.Flags().BoolVar(&o.Check, "check", true, "Check kubernetes environment before install") - cmd.Flags().DurationVar(&o.timeout, "timeout", 1800*time.Second, "Time to wait for installing KubeBlocks") + cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for installing KubeBlocks, such as --timeout=10m") + cmd.Flags().BoolVar(&o.Wait, "wait", true, "Wait for KubeBlocks to be ready, including all the auto installed add-ons. It will wait for as long as --timeout") helm.AddValueOptionsFlags(cmd.Flags(), &o.ValueOpts) return cmd @@ -123,6 +129,12 @@ func newInstallCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr func (o *Options) Complete(f cmdutil.Factory, cmd *cobra.Command) error { var err error + + // default write log to file + if err = util.EnableLogToFile(cmd.Flags()); err != nil { + fmt.Fprintf(o.Out, "Failed to enable the log file %s", err.Error()) + } + if o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace(); err != nil { return err } @@ -187,21 +199,21 @@ func (o *InstallOptions) Install() error { o.ValueOpts.Values = append(o.ValueOpts.Values, fmt.Sprintf(kMonitorParam, o.Monitor)) // add helm repo - spinner := printer.Spinner(o.Out, "%-50s", "Add and update repo "+types.KubeBlocksRepoName) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Add and update repo "+types.KubeBlocksRepoName)) + defer s.Fail() // Add repo, if exists, will update it if err = helm.AddRepo(&repo.Entry{Name: types.KubeBlocksRepoName, URL: util.GetHelmChartRepoURL()}); err != nil { return err } - spinner(true) + s.Success() - // install KubeBlocks chart - spinner = printer.Spinner(o.Out, "%-50s", "Install KubeBlocks "+o.Version) - defer spinner(false) + // install KubeBlocks + s = spinner.New(o.Out, spinnerMsg("Install KubeBlocks "+o.Version)) + defer s.Fail() if err = o.installChart(); err != nil { return err } - spinner(true) + s.Success() // wait for auto-install addons to be ready if err = o.waitAddonsEnabled(); err != nil { @@ -209,8 +221,14 @@ func (o *InstallOptions) Install() error { } if !o.Quiet { - fmt.Fprintf(o.Out, "\nKubeBlocks %s installed to namespace %s SUCCESSFULLY!\n", - o.Version, o.HelmCfg.Namespace()) + msg := fmt.Sprintf("\nKubeBlocks %s installed to namespace %s SUCCESSFULLY!\n", o.Version, o.HelmCfg.Namespace()) + if !o.Wait { + msg = fmt.Sprintf(` +KubeBlocks %s is installing to namespace %s. +You can check the KubeBlocks status by running "kbcli kubeblocks status" +`, o.Version, o.HelmCfg.Namespace()) + } + fmt.Fprint(o.Out, msg) o.printNotes() } return nil @@ -218,21 +236,26 @@ func (o *InstallOptions) Install() error { // waitAddonsEnabled waits for auto-install addons status to be enabled func (o *InstallOptions) waitAddonsEnabled() error { - addons := make(map[string]bool) + if !o.Wait { + return nil + } + + // addons record the addons and its status + addons := make(map[string]string) checkAddons := func() (bool, error) { allEnabled := true - objects, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ + objs, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ LabelSelector: buildKubeBlocksSelectorLabels(), }) if err != nil && !apierrors.IsNotFound(err) { return false, err } - if objects == nil || len(objects.Items) == 0 { + if objs == nil || len(objs.Items) == 0 { klog.V(1).Info("No Addons found") return true, nil } - for _, obj := range objects.Items { + for _, obj := range objs.Items { addon := extensionsv1alpha1.Addon{} if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &addon); err != nil { return false, err @@ -244,19 +267,11 @@ func (o *InstallOptions) waitAddonsEnabled() error { continue } - installable := false - if addon.Spec.InstallSpec != nil { - installable = addon.Spec.Installable.AutoInstall - } - - klog.V(1).Infof("Addon: %s, enabled: %v, status: %s, auto-install: %v", - addon.Name, addon.Spec.InstallSpec.GetEnabled(), addon.Status.Phase, installable) - // addon is enabled, then check its status + // addon should be auto installed, check its status if addon.Spec.InstallSpec.GetEnabled() { - addons[addon.Name] = true + addons[addon.Name] = string(addon.Status.Phase) if addon.Status.Phase != extensionsv1alpha1.AddonEnabled { - klog.V(1).Infof("Addon %s is not enabled yet", addon.Name) - addons[addon.Name] = false + klog.V(1).Infof("Addon %s is not enabled yet, status %s", addon.Name, addon.Status.Phase) allEnabled = false } } @@ -264,77 +279,67 @@ func (o *InstallOptions) waitAddonsEnabled() error { return allEnabled, nil } - okMsg := func(msg string) string { - return fmt.Sprintf("%-50s %s\n", msg, printer.BoldGreen("OK")) - } - failMsg := func(msg string) string { - return fmt.Sprintf("%-50s %s\n", msg, printer.BoldRed("FAIL")) - } suffixMsg := func(msg string) string { - return fmt.Sprintf(" %-50s", msg) + return fmt.Sprintf("%-50s", msg) + } + + addonMsg := func(msg, status string) string { + return fmt.Sprintf("%-48s %s", msg, status) } // create spinner - msg := "Wait for addons to be ready" - s := spinner.New(spinner.CharSets[11], 100*time.Millisecond) - s.Writer = o.Out - _ = s.Color("cyan") - s.Suffix = suffixMsg(msg) - s.Start() - - var prevUnready []string + allMsg := "" + msg := "Wait for addons to be enabled" + s := spinner.New(o.Out, spinnerMsg(msg)) + // check addon installing progress checkProgress := func() { if len(addons) == 0 { return } - unready := make([]string, 0) - ready := make([]string, 0) + all := make([]string, 0) for k, v := range addons { - if v { - ready = append(ready, k) - } else { - unready = append(unready, k) - } - } - sort.Strings(unready) - s.Suffix = suffixMsg(fmt.Sprintf("%s\n %s", msg, strings.Join(unready, "\n "))) - for _, r := range ready { - if !slices.Contains(prevUnready, r) { + if v == string(extensionsv1alpha1.AddonEnabled) { + all = append(all, addonMsg("Addon "+k, printer.BoldGreen("OK"))) continue } - s.FinalMSG = okMsg("Addon " + r) - s.Stop() - s.Suffix = suffixMsg(fmt.Sprintf("%s\n %s", msg, strings.Join(unready, "\n "))) - s.Start() + all = append(all, addonMsg("Addon "+k, v)) } - prevUnready = unready + sort.Strings(all) + allMsg = fmt.Sprintf("%s\n %s", msg, strings.Join(all, "\n ")) + s.SetMessage(suffixMsg(allMsg)) } var ( - allEnabled bool - err error + allEnabled bool + err error + spinnerDone = func(s *spinner.Spinner) { + s.SetFinalMsg(allMsg) + s.Done("") + fmt.Fprintln(o.Out) + } ) - // wait for all auto-install addons to be enabled - for i := 0; i < viper.GetInt("KB_WAIT_ADDON_TIMES"); i++ { + + // wait all addons to be enabled, or timeout + if err = wait.PollImmediate(5*time.Second, o.Timeout, func() (bool, error) { allEnabled, err = checkAddons() if err != nil { - s.FinalMSG = failMsg(msg) - s.Stop() - return err + return false, err } checkProgress() if allEnabled { - s.FinalMSG = okMsg(msg) - s.Stop() - return nil + spinnerDone(s) + return true, nil + } + return false, nil + }); err != nil { + spinnerDone(s) + if err == wait.ErrWaitTimeout { + return errors.New("timeout waiting for auto-install addons to be enabled, run \"kbcli addon list\" to check addon status") } - time.Sleep(5 * time.Second) + return err } - // timeout to wait for all auto-install addons to be enabled - s.FinalMSG = fmt.Sprintf("%-50s %s\n", msg, printer.BoldRed("TIMEOUT")) - s.Stop() return nil } @@ -410,7 +415,7 @@ func (o *InstallOptions) checkRemainedResource() error { // the addon resources. objs, err := getKBObjects(o.Dynamic, ns, nil) if err != nil { - fmt.Fprintf(o.ErrOut, "Check whether there are resources left by KubeBlocks before: %s\n", err.Error()) + fmt.Fprintf(o.ErrOut, "Failed to get resources left by KubeBlocks before: %s\n", err.Error()) } res := getRemainedResource(objs) @@ -464,13 +469,13 @@ func (o *InstallOptions) buildChart() *helm.InstallOpts { return &helm.InstallOpts{ Name: types.KubeBlocksChartName, Chart: types.KubeBlocksChartName + "/" + types.KubeBlocksChartName, - Wait: true, + Wait: o.Wait, Version: o.Version, Namespace: o.HelmCfg.Namespace(), ValueOpts: &o.ValueOpts, TryTimes: 2, CreateNamespace: o.CreateNamespace, - Timeout: o.timeout, + Timeout: o.Timeout, Atomic: true, } } diff --git a/internal/cli/cmd/kubeblocks/kubeblocks_objects.go b/internal/cli/cmd/kubeblocks/kubeblocks_objects.go index 155b1d9cd..58966a564 100644 --- a/internal/cli/cmd/kubeblocks/kubeblocks_objects.go +++ b/internal/cli/cmd/kubeblocks/kubeblocks_objects.go @@ -109,10 +109,7 @@ func getKBObjects(dynamic dynamic.Interface, namespace string, addons []*extensi ns = metav1.NamespaceAll } - if klog.V(1).Enabled() { - klog.Infof("search objects by labels, namespace: %s, name: %s, gvr: %s", labelSelector, gvr, scope) - } - + klog.V(1).Infof("search objects by labels, namespace: %s, name: %s, gvr: %s", labelSelector, gvr, scope) objs, err := dynamic.Resource(gvr).Namespace(ns).List(context.TODO(), metav1.ListOptions{ LabelSelector: labelSelector, }) @@ -136,9 +133,7 @@ func getKBObjects(dynamic dynamic.Interface, namespace string, addons []*extensi // get object by name getObjectByName := func(name string, gvr schema.GroupVersionResource) { - if klog.V(1).Enabled() { - klog.Infof("search object by name, namespace: %s, name: %s, gvr: %s ", namespace, name, gvr) - } + klog.V(1).Infof("search object by name, namespace: %s, name: %s, gvr: %s ", namespace, name, gvr) obj, err := dynamic.Resource(gvr).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { appendErr(err) @@ -149,9 +144,7 @@ func getKBObjects(dynamic dynamic.Interface, namespace string, addons []*extensi } target := kbObjs[gvr] target.Items = append(target.Items, *obj) - if klog.V(1).Enabled() { - klog.Infof("\tget object: %s, %s, %s", obj.GetNamespace(), obj.GetKind(), obj.GetName()) - } + klog.V(1).Infof("\tget object: %s, %s, %s", obj.GetNamespace(), obj.GetKind(), obj.GetName()) } // get RBAC resources, such as ClusterRole, ClusterRoleBinding, Role, RoleBinding, ServiceAccount getObjectsByLabels(buildKubeBlocksSelectorLabels(), types.ClusterRoleGVR(), ResourceScopeGlobal) diff --git a/internal/cli/cmd/kubeblocks/status.go b/internal/cli/cmd/kubeblocks/status.go index 503f431dd..a4686ffa0 100644 --- a/internal/cli/cmd/kubeblocks/status.go +++ b/internal/cli/cmd/kubeblocks/status.go @@ -397,9 +397,7 @@ func listResourceByGVR(ctx context.Context, client dynamic.Interface, namespace unstructuredList := make([]*unstructured.UnstructuredList, 0) for _, gvr := range gvrlist { for _, labelSelector := range selector { - if klog.V(1).Enabled() { - klog.Infof("listResourceByGVR: namespace=%s, gvrlist=%v, selector=%v", namespace, gvr, labelSelector) - } + klog.V(1).Infof("listResourceByGVR: namespace=%s, gvrlist=%v, selector=%v", namespace, gvr, labelSelector) resource, err := client.Resource(gvr).Namespace(namespace).List(ctx, labelSelector) if err != nil { appendErrIgnoreNotFound(allErrs, err) diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index c0c254e32..4074d40d0 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -29,7 +29,6 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" - "github.com/spf13/viper" "helm.sh/helm/v3/pkg/repo" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,18 +36,19 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" k8sapitypes "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/dynamic" "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" - "k8s.io/utils/strings/slices" extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/spinner" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" - "github.com/apecloud/kubeblocks/internal/constant" ) var ( @@ -93,55 +93,25 @@ func newUninstallCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *co cmd.Flags().BoolVar(&o.removePVs, "remove-pvs", false, "Remove PersistentVolume or not") cmd.Flags().BoolVar(&o.removePVCs, "remove-pvcs", false, "Remove PersistentVolumeClaim or not") cmd.Flags().BoolVar(&o.RemoveNamespace, "remove-namespace", false, "Remove default created \"kb-system\" namespace or not") + cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for uninstalling KubeBlocks, such as --timeout=5m") + cmd.Flags().BoolVar(&o.Wait, "wait", true, "Wait for KubeBlocks to be uninstalled, including all the add-ons. It will wait for as long as --timeout") return cmd } func (o *UninstallOptions) PreCheck() error { // wait user to confirm if !o.AutoApprove { - printer.Warning(o.Out, "uninstall will remove all KubeBlocks resources.\n") + printer.Warning(o.Out, "this action will remove all KubeBlocks resources.\n") if err := confirmUninstall(o.In); err != nil { return err } } - preCheckList := []string{ - "clusters.apps.kubeblocks.io", - } - ctx := context.Background() - - // delete crds - crs := map[string][]string{} - crdList, err := o.Dynamic.Resource(types.CRDGVR()).List(ctx, metav1.ListOptions{}) - if err != nil { + // check if there is any resource should be removed first, if so, return error + // and ask user to remove them manually + if err := checkResources(o.Dynamic); err != nil { return err } - for _, crd := range crdList.Items { - // find kubeblocks crds - if strings.Contains(crd.GetName(), constant.APIGroup) && - slices.Contains(preCheckList, crd.GetName()) { - gvr, err := getGVRByCRD(&crd) - if err != nil { - return err - } - // find custom resource - objList, err := o.Dynamic.Resource(*gvr).List(ctx, metav1.ListOptions{}) - if err != nil { - return err - } - for _, item := range objList.Items { - crs[crd.GetName()] = append(crs[crd.GetName()], item.GetName()) - } - } - } - - if len(crs) > 0 { - errMsg := bytes.NewBufferString("failed to uninstall, the following custom resources need to be removed first:\n") - for k, v := range crs { - errMsg.WriteString(fmt.Sprintf(" %s: %s\n", k, strings.Join(v, " "))) - } - return errors.Errorf(errMsg.String()) - } // verify where kubeblocks is installed kbNamespace, err := util.GetKubeBlocksNamespace(o.Client) @@ -159,21 +129,23 @@ func (o *UninstallOptions) PreCheck() error { } func (o *UninstallOptions) Uninstall() error { - printSpinner := func(spinner func(result bool), err error) { + printSpinner := func(s *spinner.Spinner, err error) { if err == nil || apierrors.IsNotFound(err) || strings.Contains(err.Error(), "release: not found") { - spinner(true) + s.Success() return } - spinner(false) + s.Fail() fmt.Fprintf(o.Out, " %s\n", err.Error()) } - newSpinner := func(msg string) func(result bool) { - return printer.Spinner(o.Out, fmt.Sprintf("%-50s", msg)) + newSpinner := func(msg string) *spinner.Spinner { + return spinner.New(o.Out, spinner.WithMessage(fmt.Sprintf("%-50s", msg))) } // uninstall all KubeBlocks addons - printSpinner(newSpinner("Uninstall KubeBlocks addons"), o.uninstallAddons()) + if err := o.uninstallAddons(); err != nil { + return err + } // uninstall helm release that will delete custom resources, but since finalizers is not empty, // custom resources will not be deleted, so we will remove finalizers later. @@ -231,95 +203,167 @@ func (o *UninstallOptions) Uninstall() error { deleteNamespace(o.Client, types.DefaultNamespace)) } - fmt.Fprintln(o.Out, "Uninstall KubeBlocks done.") + if o.Wait { + fmt.Fprintln(o.Out, "Uninstall KubeBlocks done.") + } else { + fmt.Fprintf(o.Out, "KubeBlocks is uninstalling, run \"kbcli kubeblocks status\" to check status.\n") + } return nil } // uninstallAddons uninstall all KubeBlocks addons func (o *UninstallOptions) uninstallAddons() error { + addonStatus := make(map[string]string) + var ( allErrs []error - stop bool err error - ) - uninstallAddon := func(addon *extensionsv1alpha1.Addon) error { - klog.V(1).Infof("Uninstall %s", addon.Name) - if _, err := o.Dynamic.Resource(types.AddonGVR()).Patch(context.TODO(), addon.Name, k8sapitypes.JSONPatchType, - []byte("[{\"op\": \"replace\", \"path\": \"/spec/install/enabled\", \"value\": false }]"), - metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { - return err - } - return nil - } - - processAddons := func(processFn func(addon *extensionsv1alpha1.Addon) error) ([]*extensionsv1alpha1.Addon, error) { - var addons []*extensionsv1alpha1.Addon - objects, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ - LabelSelector: buildKubeBlocksSelectorLabels(), - }) - if err != nil && !apierrors.IsNotFound(err) { - klog.V(1).Infof("Failed to get KubeBlocks addons %s", err.Error()) - allErrs = append(allErrs, err) - return nil, utilerrors.NewAggregate(allErrs) - } - if objects == nil { - return nil, nil - } - - // if all addons are disabled, then we will stop uninstalling addons - stop = true - for _, obj := range objects.Items { - addon := extensionsv1alpha1.Addon{} - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &addon); err != nil { - klog.V(1).Infof("Failed to convert KubeBlocks addon %s", err.Error()) + msg = "Wait for addons to be disabled" + + processAddons = func(uninstall bool) error { + objects, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ + LabelSelector: buildKubeBlocksSelectorLabels(), + }) + if err != nil && !apierrors.IsNotFound(err) { + klog.V(1).Infof("Failed to get KubeBlocks addons %s", err.Error()) allErrs = append(allErrs, err) - continue + return utilerrors.NewAggregate(allErrs) } - klog.V(1).Infof("Addon: %s, enabled: %v, status: %s", - addon.Name, addon.Spec.InstallSpec.GetEnabled(), addon.Status.Phase) - addons = append(addons, &addon) - if addon.Status.Phase == extensionsv1alpha1.AddonDisabled { - continue + if objects == nil { + return nil } - // if there is an enabled addon, then we will continue uninstalling addons - // and wait for a while to make sure all addons are disabled - stop = false - if processFn == nil { - continue + + for _, obj := range objects.Items { + addon := extensionsv1alpha1.Addon{} + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &addon); err != nil { + klog.V(1).Infof("Failed to convert KubeBlocks addon %s", err.Error()) + allErrs = append(allErrs, err) + continue + } + + if uninstall { + // we only need to uninstall addons that are not disabled + if addon.Status.Phase == extensionsv1alpha1.AddonDisabled { + continue + } + addonStatus[addon.Name] = string(addon.Status.Phase) + o.addons = append(o.addons, &addon) + + // uninstall addons + if err = disableAddon(o.Dynamic, &addon); err != nil { + klog.V(1).Infof("Failed to uninstall KubeBlocks addon %s %s", addon.Name, err.Error()) + allErrs = append(allErrs, err) + } + } else { + // update addons if exists + if _, ok := addonStatus[addon.Name]; ok { + addonStatus[addon.Name] = string(addon.Status.Phase) + } + } } - if err = processFn(&addon); err != nil && !apierrors.IsNotFound(err) { - klog.V(1).Infof("Failed to uninstall KubeBlocks addon %s", err.Error()) - allErrs = append(allErrs, err) + return utilerrors.NewAggregate(allErrs) + } + + buildMsg = func() (string, bool) { + var addonMsg []string + allDisabled := true + for k, v := range addonStatus { + if v == string(extensionsv1alpha1.AddonDisabled) { + v = printer.BoldGreen("OK") + } else { + allDisabled = false + } + addonMsg = append(addonMsg, fmt.Sprintf("%-48s %s", "Addon "+k, v)) } + sort.Strings(addonMsg) + return fmt.Sprintf("%-50s\n %s", msg, strings.Join(addonMsg, "\n ")), allDisabled } - return addons, utilerrors.NewAggregate(allErrs) + ) + + var s *spinner.Spinner + if !o.Wait { + s = spinner.New(o.Out, spinner.WithMessage(fmt.Sprintf("%-50s", "Uninstall KubeBlocks addons"))) + } else { + s = spinner.New(o.Out, spinner.WithMessage(fmt.Sprintf("%-50s", msg))) } // get all addons and uninstall them - if o.addons, err = processAddons(uninstallAddon); err != nil { + if err = processAddons(true); err != nil { + s.Fail() return err } - if len(o.addons) == 0 || stop { + if len(addonStatus) == 0 || !o.Wait { + s.Success() return nil } + spinnerDone := func(s *spinner.Spinner, msg string) { + s.SetFinalMsg(msg) + s.Done("") + fmt.Fprintln(o.Out) + } + // check if all addons are disabled, if so, then we will stop checking addons // status otherwise, we will wait for a while and check again - for i := 0; i < viper.GetInt("KB_WAIT_ADDON_TIMES"); i++ { - klog.V(1).Infof("Wait for %d seconds and check addons disabled again", 5) - time.Sleep(5 * time.Second) - // pass a nil processFn, we will only check addons status, do not try to - // uninstall addons again - if o.addons, err = processAddons(nil); err != nil { + if err = wait.PollImmediate(5*time.Second, o.Timeout, func() (bool, error) { + // we will only check addons status, do not try to uninstall addons again + if err = processAddons(false); err != nil { + return false, err + } + m, allDisabled := buildMsg() + s.SetMessage(m) + if allDisabled { + spinnerDone(s, m) + return true, nil + } + return false, nil + }); err != nil { + m, _ := buildMsg() + spinnerDone(s, m) + if err == wait.ErrWaitTimeout { + allErrs = append(allErrs, errors.New("timeout waiting for addons to be disabled, run \"kbcli addon list\" to check addon status")) + } else { + allErrs = append(allErrs, err) + } + } + return utilerrors.NewAggregate(allErrs) +} + +func checkResources(dynamic dynamic.Interface) error { + ctx := context.Background() + gvrList := []schema.GroupVersionResource{ + types.ClusterGVR(), + types.BackupGVR(), + } + + crs := map[string][]string{} + for _, gvr := range gvrList { + objList, err := dynamic.Resource(gvr).List(ctx, metav1.ListOptions{}) + if err != nil { return err } - if stop { - return nil + for _, item := range objList.Items { + crs[gvr.Resource] = append(crs[gvr.Resource], item.GetName()) + } + } + + if len(crs) > 0 { + errMsg := bytes.NewBufferString("failed to uninstall, the following resources need to be removed first\n") + for k, v := range crs { + errMsg.WriteString(fmt.Sprintf(" %s: %s\n", k, strings.Join(v, " "))) } + return errors.Errorf(errMsg.String()) } - if !stop { - allErrs = append(allErrs, fmt.Errorf("failed to uninstall KubeBlocks addons")) + return nil +} + +func disableAddon(dynamic dynamic.Interface, addon *extensionsv1alpha1.Addon) error { + klog.V(1).Infof("Uninstall %s, status %s", addon.Name, addon.Status.Phase) + if _, err := dynamic.Resource(types.AddonGVR()).Patch(context.TODO(), addon.Name, k8sapitypes.JSONPatchType, + []byte("[{\"op\": \"replace\", \"path\": \"/spec/install/enabled\", \"value\": false }]"), + metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err } - return utilerrors.NewAggregate(allErrs) + return nil } diff --git a/internal/cli/cmd/kubeblocks/upgrade.go b/internal/cli/cmd/kubeblocks/upgrade.go index a49607073..e322aa1f7 100644 --- a/internal/cli/cmd/kubeblocks/upgrade.go +++ b/internal/cli/cmd/kubeblocks/upgrade.go @@ -31,6 +31,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/spinner" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" @@ -65,7 +66,8 @@ func newUpgradeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr cmd.Flags().StringVar(&o.Version, "version", "", "Set KubeBlocks version") cmd.Flags().BoolVar(&o.Check, "check", true, "Check kubernetes environment before upgrade") - cmd.Flags().DurationVar(&o.timeout, "timeout", 1800*time.Second, "Time to wait for upgrading KubeBlocks") + cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for upgrading KubeBlocks, such as --timeout=10m") + cmd.Flags().BoolVar(&o.Wait, "wait", true, "Wait for KubeBlocks to be ready. It will wait for as long as --timeout") helm.AddValueOptionsFlags(cmd.Flags(), &o.ValueOpts) return cmd @@ -111,27 +113,27 @@ func (o *InstallOptions) Upgrade() error { } // add helm repo - spinner := printer.Spinner(o.Out, "%-40s", "Add and update repo "+types.KubeBlocksChartName) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Add and update repo "+types.KubeBlocksChartName)) + defer s.Fail() // Add repo, if exists, will update it if err = helm.AddRepo(&repo.Entry{Name: types.KubeBlocksChartName, URL: util.GetHelmChartRepoURL()}); err != nil { return err } - spinner(true) + s.Success() // it's time to upgrade msg := "" if o.Version != "" { msg = "to " + o.Version } - spinner = printer.Spinner(o.Out, "%-40s", "Upgrading KubeBlocks "+msg) - defer spinner(false) + s = spinner.New(o.Out, spinnerMsg("Upgrading KubeBlocks "+msg)) + defer s.Fail() // upgrade KubeBlocks chart if err = o.upgradeChart(); err != nil { return err } // successfully upgraded - spinner(true) + s.Success() if !o.Quiet { fmt.Fprintf(o.Out, "\nKubeBlocks has been upgraded %s SUCCESSFULLY!\n", msg) diff --git a/internal/cli/cmd/playground/destroy.go b/internal/cli/cmd/playground/destroy.go index 543397b99..8cb5ccea9 100644 --- a/internal/cli/cmd/playground/destroy.go +++ b/internal/cli/cmd/playground/destroy.go @@ -43,6 +43,7 @@ import ( cp "github.com/apecloud/kubeblocks/internal/cli/cloudprovider" "github.com/apecloud/kubeblocks/internal/cli/cmd/kubeblocks" "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/spinner" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" @@ -61,7 +62,8 @@ type destroyOptions struct { // purge resources, before destroy kubernetes cluster we should delete cluster and // uninstall KubeBlocks - purge bool + purge bool + timeout time.Duration } func newDestroyCmd(streams genericclioptions.IOStreams) *cobra.Command { @@ -79,6 +81,7 @@ func newDestroyCmd(streams genericclioptions.IOStreams) *cobra.Command { } cmd.Flags().BoolVar(&o.purge, "purge", true, "Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks.") + cmd.Flags().DurationVar(&o.timeout, "timeout", 1800*time.Second, "Time to wait for installing KubeBlocks, such as --timeout=10m") return cmd } @@ -108,15 +111,15 @@ func (o *destroyOptions) destroy() error { // destroyLocal destroy local k3d cluster that will destroy all resources func (o *destroyOptions) destroyLocal() error { provider := cp.NewLocalCloudProvider(o.Out, o.ErrOut) - spinner := printer.Spinner(o.Out, "%-50s", "Delete playground k3d cluster "+o.prevCluster.ClusterName) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Delete playground k3d cluster "+o.prevCluster.ClusterName)) + defer s.Fail() if err := provider.DeleteK8sCluster(o.prevCluster); err != nil { if !strings.Contains(err.Error(), "no cluster found") && !strings.Contains(err.Error(), "does not exist") { return err } } - spinner(true) + s.Success() if err := o.removeKubeConfig(); err != nil { return err @@ -267,13 +270,13 @@ func (o *destroyOptions) deleteClusters(dynamic dynamic.Interface) error { return nil } - spinner := printer.Spinner(o.Out, fmt.Sprintf("%-50s", "Delete clusters created by KubeBlocks")) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Delete clusters created by KubeBlocks")) + defer s.Fail() // get all clusters clusters, err := getClusters() if clusters == nil || len(clusters.Items) == 0 { - spinner(true) + s.Success() return nil } @@ -331,7 +334,7 @@ func (o *destroyOptions) deleteClusters(dynamic dynamic.Interface) error { return err } - spinner(true) + s.Success() return nil } @@ -342,6 +345,7 @@ func (o *destroyOptions) uninstallKubeBlocks(client kubernetes.Interface, dynami IOStreams: o.IOStreams, Client: client, Dynamic: dynamic, + Wait: true, }, AutoApprove: true, RemoveNamespace: true, @@ -359,17 +363,17 @@ func (o *destroyOptions) uninstallKubeBlocks(client kubernetes.Interface, dynami } func (o *destroyOptions) removeKubeConfig() error { - spinner := printer.Spinner(o.Out, "%-50s", "Remove kubeconfig from "+defaultKubeConfigPath) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Remove kubeconfig from "+defaultKubeConfigPath)) + defer s.Fail() if err := kubeConfigRemove(o.prevCluster.KubeConfig, defaultKubeConfigPath); err != nil { if os.IsNotExist(err) { - spinner(true) + s.Success() return nil } else { return err } } - spinner(true) + s.Success() clusterContext, err := kubeConfigCurrentContext(o.prevCluster.KubeConfig) if err != nil { @@ -391,11 +395,11 @@ func (o *destroyOptions) removeKubeConfig() error { // remove state file func (o *destroyOptions) removeStateFile() error { - spinner := printer.Spinner(o.Out, "Remove state file %s", o.stateFilePath) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Remove state file %s", o.stateFilePath)) + defer s.Fail() if err := removeStateFile(o.stateFilePath); err != nil { return err } - spinner(true) + s.Success() return nil } diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index de1a6fd8a..f1d922012 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -42,6 +42,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/cmd/kubeblocks" "github.com/apecloud/kubeblocks/internal/cli/create" "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/spinner" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" @@ -67,6 +68,10 @@ var ( kbcli playground init --cloud-provider gcp --region us-central1`) supportedCloudProviders = []string{cp.Local, cp.AWS, cp.GCP, cp.AliCloud, cp.TencentCloud} + + spinnerMsg = func(format string, a ...any) spinner.Option { + return spinner.WithMessage(fmt.Sprintf("%-50s", fmt.Sprintf(format, a...))) + } ) type initOptions struct { @@ -160,12 +165,12 @@ func (o *initOptions) local() error { } // create a local kubernetes cluster (k3d cluster) to deploy KubeBlocks - spinner := printer.Spinner(o.Out, "%-50s", "Create k3d cluster: "+clusterInfo.ClusterName) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Create k3d cluster: "+clusterInfo.ClusterName)) + defer s.Fail() if err = provider.CreateK8sCluster(clusterInfo); err != nil { return errors.Wrap(err, "failed to set up k3d cluster") } - spinner(true) + s.Success() clusterInfo, err = o.writeStateFile(provider) if err != nil { @@ -311,8 +316,8 @@ func (o *initOptions) writeStateFile(provider cp.Interface) (*cp.K8sClusterInfo, // merge created kubernetes cluster kubeconfig to ~/.kube/config and set it as default func (o *initOptions) setKubeConfig(info *cp.K8sClusterInfo) error { - spinner := printer.Spinner(o.Out, "%-50s", "Merge kubeconfig to "+defaultKubeConfigPath) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Merge kubeconfig to "+defaultKubeConfigPath)) + defer s.Fail() // check if the default kubeconfig file exists, if not, create it if _, err := os.Stat(defaultKubeConfigPath); os.IsNotExist(err) { @@ -328,15 +333,15 @@ func (o *initOptions) setKubeConfig(info *cp.K8sClusterInfo) error { writeKubeConfigOptions{UpdateExisting: true, UpdateCurrentContext: true}); err != nil { return errors.Wrapf(err, "failed to write cluster %s kubeconfig", info.ClusterName) } - spinner(true) + s.Success() currentContext, err := kubeConfigCurrentContext(info.KubeConfig) - spinner = printer.Spinner(o.Out, "%-50s", "Switch current context to "+currentContext) - defer spinner(false) + s = spinner.New(o.Out, spinnerMsg("Switch current context to "+currentContext)) + defer s.Fail() if err != nil { return err } - spinner(true) + s.Success() return nil } @@ -370,12 +375,12 @@ func (o *initOptions) installKBAndCluster(info *cp.K8sClusterInfo) error { if o.clusterVersion != "" { clusterInfo += ", ClusterVersion: " + o.clusterVersion } - spinner := printer.Spinner(o.Out, "Create cluster %s (%s)", kbClusterName, clusterInfo) - defer spinner(false) + s := spinner.New(o.Out, spinnerMsg("Create cluster %s (%s)", kbClusterName, clusterInfo)) + defer s.Fail() if err = o.createCluster(); err != nil && !apierrors.IsAlreadyExists(err) { return errors.Wrapf(err, "failed to create cluster %s", kbClusterName) } - spinner(true) + s.Success() fmt.Fprintf(os.Stdout, "\nKubeBlocks playground init SUCCESSFULLY!\n\n") fmt.Fprintf(os.Stdout, "Kubernetes cluster \"%s\" has been created.\n", info.ClusterName) @@ -407,6 +412,7 @@ func (o *initOptions) installKubeBlocks(k8sClusterName string) error { IOStreams: o.IOStreams, Client: client, Dynamic: dynamic, + Wait: true, }, Version: o.kbVersion, Monitor: true, diff --git a/internal/cli/cmd/playground/util.go b/internal/cli/cmd/playground/util.go index fe491371f..a48bd19bc 100644 --- a/internal/cli/cmd/playground/util.go +++ b/internal/cli/cmd/playground/util.go @@ -33,7 +33,7 @@ import ( "k8s.io/client-go/kubernetes" cp "github.com/apecloud/kubeblocks/internal/cli/cloudprovider" - "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/spinner" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/version" ) @@ -120,8 +120,8 @@ func readClusterInfoFromFile(path string) (*cp.K8sClusterInfo, error) { } func writeAndUseKubeConfig(kubeConfig string, kubeConfigPath string, out io.Writer) error { - spinner := printer.Spinner(out, fmt.Sprintf("%-50s", "Write kubeconfig to "+kubeConfigPath)) - defer spinner(false) + s := spinner.New(out, spinnerMsg("Write kubeconfig to "+kubeConfigPath)) + defer s.Fail() if err := kubeConfigWrite(kubeConfig, kubeConfigPath, writeKubeConfigOptions{ UpdateExisting: true, UpdateCurrentContext: true, @@ -134,7 +134,7 @@ func writeAndUseKubeConfig(kubeConfig string, kubeConfigPath string, out io.Writ return err } - spinner(true) + s.Success() return nil } diff --git a/internal/cli/printer/spinner.go b/internal/cli/printer/spinner.go deleted file mode 100644 index dacd218bd..000000000 --- a/internal/cli/printer/spinner.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package printer - -import ( - "fmt" - "io" - "os" - "os/signal" - "runtime" - "sync" - "syscall" - "time" - - "github.com/briandowns/spinner" - - "github.com/apecloud/kubeblocks/internal/cli/types" -) - -func Spinner(w io.Writer, fmtstr string, a ...any) func(result bool) { - msg := fmt.Sprintf(fmtstr, a...) - var once sync.Once - var s *spinner.Spinner - - if runtime.GOOS == types.GoosWindows { - fmt.Fprintf(w, "%s\n", msg) - return func(result bool) {} - } else { - s = spinner.New(spinner.CharSets[11], 100*time.Millisecond) - s.Writer = w - s.HideCursor = true - _ = s.Color("cyan") - s.Suffix = fmt.Sprintf(" %s", msg) - s.Start() - - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - // Capture the interrupt signal, make the `spinner` program exit gracefully, and prevent the cursor from disappearing. - go func() { - <-c - s.Stop() - // Show cursor in terminal. - fmt.Fprintf(s.Writer, "\033[?25h") - os.Exit(0) - }() - } - - return func(result bool) { - once.Do(func() { - if s != nil { - s.Stop() - } - if result { - fmt.Fprintf(w, "%s %s\n", msg, BoldGreen("OK")) - } else { - fmt.Fprintf(w, "%s %s\n", msg, BoldRed("FAIL")) - } - }) - } -} diff --git a/internal/cli/spinner/spinner.go b/internal/cli/spinner/spinner.go new file mode 100644 index 000000000..7a5531c35 --- /dev/null +++ b/internal/cli/spinner/spinner.go @@ -0,0 +1,147 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package spinner + +import ( + "fmt" + "io" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/briandowns/spinner" + + "github.com/apecloud/kubeblocks/internal/cli/printer" +) + +type Spinner struct { + s *spinner.Spinner + delay time.Duration + cancel chan struct{} +} + +type Option func(*Spinner) + +func WithMessage(msg string) Option { + return func(s *Spinner) { + s.UpdateSpinnerMessage(msg) + } +} + +func WithDelay(delay time.Duration) Option { + return func(s *Spinner) { + s.delay = delay + } +} + +func (s *Spinner) UpdateSpinnerMessage(msg string) { + s.s.Suffix = fmt.Sprintf(" %s", msg) +} + +func (s *Spinner) SetMessage(msg string) { + s.UpdateSpinnerMessage(msg) + if !s.s.Active() { + s.Start() + } +} + +func (s *Spinner) Start() { + if s.cancel != nil { + return + } + if s.delay == 0 { + s.s.Start() + return + } + s.cancel = make(chan struct{}, 1) + go func() { + select { + case <-s.cancel: + return + case <-time.After(s.delay): + s.s.Start() + s.cancel = nil + } + time.Sleep(50 * time.Millisecond) + }() +} + +func (s *Spinner) Done(status string) { + if s.cancel != nil { + close(s.cancel) + } + s.stop(status) +} + +func (s *Spinner) SetFinalMsg(msg string) { + s.s.FinalMSG = msg + if !s.s.Active() { + s.Start() + } +} + +func (s *Spinner) stop(status string) { + if s.s == nil { + return + } + + if status != "" { + s.s.FinalMSG = fmt.Sprintf("%s %s\n", strings.TrimPrefix(s.s.Suffix, " "), status) + } + s.s.Stop() + + // show cursor in terminal. + fmt.Fprintf(s.s.Writer, "\033[?25h") +} + +func (s *Spinner) Success() { + s.Done(printer.BoldGreen("OK")) +} + +func (s *Spinner) Fail() { + s.Done(printer.BoldRed("FAIL")) +} + +func New(w io.Writer, opts ...Option) *Spinner { + res := &Spinner{} + res.s = spinner.New(spinner.CharSets[11], + 100*time.Millisecond, + spinner.WithWriter(w), + spinner.WithHiddenCursor(true), + spinner.WithColor("cyan"), + ) + + for _, opt := range opts { + opt(res) + } + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + // Capture the interrupt signal, make the `spinner` program exit gracefully, and prevent the cursor from disappearing. + go func() { + <-c + res.Done("") + os.Exit(0) + }() + res.Start() + return res +} diff --git a/internal/cli/printer/spinner_test.go b/internal/cli/spinner/spinner_test.go similarity index 84% rename from internal/cli/printer/spinner_test.go rename to internal/cli/spinner/spinner_test.go index da7c1a59b..a678e43bf 100644 --- a/internal/cli/printer/spinner_test.go +++ b/internal/cli/spinner/spinner_test.go @@ -17,7 +17,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package printer +package spinner import ( "os" @@ -27,10 +27,10 @@ import ( var _ = Describe("Spinner", func() { It("Test Spinner", func() { - spinner := Spinner(os.Stdout, "spinner test ... ") - spinner(true) + s := New(os.Stdout, WithMessage("spinner test ... ")) + s.Success() - spinner = Spinner(os.Stdout, "spinner test ... ") - spinner(false) + s = New(os.Stdout, WithMessage("spinner test ... ")) + s.Fail() }) }) diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index 29e4a7079..0379bc6a2 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -37,6 +37,9 @@ const ( // CliHomeEnv defines kbcli home system env CliHomeEnv = "KBCLI_HOME" + // DefaultLogFilePrefix is the default log file prefix + DefaultLogFilePrefix = "kbcli" + // DefaultNamespace is the namespace where kubeblocks is installed if // no other namespace is specified DefaultNamespace = "kb-system" @@ -111,13 +114,12 @@ const ( // DataProtection API group const ( - DPAPIGroup = "dataprotection.kubeblocks.io" - DPAPIVersion = "v1alpha1" - ResourceBackups = "backups" - ResourceBackupTools = "backuptools" - ResourceRestoreJobs = "restorejobs" - ResourceBackupPolicies = "backuppolicies" - ResourceBackupPolicyTemplates = "backuppolicytemplates" + DPAPIGroup = "dataprotection.kubeblocks.io" + DPAPIVersion = "v1alpha1" + ResourceBackups = "backups" + ResourceBackupTools = "backuptools" + ResourceRestoreJobs = "restorejobs" + ResourceBackupPolicies = "backuppolicies" ) // Extensions API group diff --git a/internal/cli/util/helm/config.go b/internal/cli/util/helm/config.go index 8203e0719..3c9338805 100644 --- a/internal/cli/util/helm/config.go +++ b/internal/cli/util/helm/config.go @@ -20,8 +20,6 @@ along with this program. If not, see . package helm import ( - "os" - "helm.sh/helm/v3/pkg/action" ) @@ -43,7 +41,7 @@ func NewConfig(namespace string, kubeConfig string, ctx string, debug bool) *Con } if debug { - cfg.logFn = GetVerboseLog(os.Stdout) + cfg.logFn = GetVerboseLog() } else { cfg.logFn = GetQuiteLog() } diff --git a/internal/cli/util/helm/helm.go b/internal/cli/util/helm/helm.go index 7706fd254..c20bbc2df 100644 --- a/internal/cli/util/helm/helm.go +++ b/internal/cli/util/helm/helm.go @@ -51,6 +51,7 @@ import ( "helm.sh/helm/v3/pkg/storage/driver" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/rest" + "k8s.io/klog/v2" "github.com/apecloud/kubeblocks/internal/cli/types" ) @@ -553,8 +554,8 @@ func GetQuiteLog() action.DebugLog { return func(format string, v ...interface{}) {} } -func GetVerboseLog(out io.Writer) action.DebugLog { +func GetVerboseLog() action.DebugLog { return func(format string, v ...interface{}) { - fmt.Fprintf(out, format+"\n", v...) + klog.Infof(format+"\n", v...) } } diff --git a/internal/cli/util/log.go b/internal/cli/util/log.go new file mode 100644 index 000000000..f1ee506ad --- /dev/null +++ b/internal/cli/util/log.go @@ -0,0 +1,88 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package util + +import ( + "flag" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/spf13/pflag" + "k8s.io/klog/v2" + + "github.com/apecloud/kubeblocks/internal/cli/types" +) + +func EnableLogToFile(fs *pflag.FlagSet) error { + logFile, err := getCliLogFile() + if err != nil { + return err + } + + setFlag := func(kv map[string]string) { + for k, v := range kv { + _ = fs.Set(k, v) + } + } + + if klog.V(1).Enabled() { + // if log is enabled, write log to standard output and log file + setFlag(map[string]string{ + "alsologtostderr": "true", + "logtostderr": "false", + "log-file": logFile, + }) + } else { + // if log is not enabled, enable it and write log to file + setFlag(map[string]string{ + "v": "1", + "logtostderr": "false", + "alsologtostderr": "false", + "log-file": logFile, + }) + } + return nil +} + +func getCliLogFile() (string, error) { + homeDir, err := GetCliHomeDir() + if err != nil { + return "", err + } + return filepath.Join(homeDir, fmt.Sprintf("%s-%s.log", types.DefaultLogFilePrefix, time.Now().Format("2006-01-02"))), nil +} + +// AddKlogFlags adds flags from k8s.io/klog +// marks the flags as hidden to avoid showing them in help +func AddKlogFlags(fs *pflag.FlagSet) { + local := flag.NewFlagSet("klog", flag.ExitOnError) + klog.InitFlags(local) + local.VisitAll(func(f *flag.Flag) { + f.Name = strings.ReplaceAll(f.Name, "_", "-") + if fs.Lookup(f.Name) != nil { + return + } + newFlag := pflag.PFlagFromGoFlag(f) + newFlag.Hidden = true + fs.AddFlag(newFlag) + }) +} diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 1127e7f42..eef0ea1a3 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -71,12 +71,6 @@ import ( "github.com/apecloud/kubeblocks/internal/constant" ) -func init() { - if _, err := GetCliHomeDir(); err != nil { - fmt.Println("Failed to create kbcli home dir:", err) - } -} - // CloseQuietly closes `io.Closer` quietly. Very handy and helpful for code // quality too. func CloseQuietly(d io.Closer) { From 5c8f0a60168357585d0392a1012746ce1b09b34e Mon Sep 17 00:00:00 2001 From: wangyelei Date: Sat, 6 May 2023 12:07:40 +0800 Subject: [PATCH 232/439] chore: reduce the waiting time for KubeBlocks configuration to take effect (#3102) --- internal/cli/cmd/kubeblocks/config.go | 55 ++++++++++++++++++++++++++- internal/cli/types/types.go | 1 + internal/constant/const.go | 2 +- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/internal/cli/cmd/kubeblocks/config.go b/internal/cli/cmd/kubeblocks/config.go index e9e90b953..d139f22c4 100644 --- a/internal/cli/cmd/kubeblocks/config.go +++ b/internal/cli/cmd/kubeblocks/config.go @@ -23,20 +23,26 @@ import ( "context" "fmt" "sort" + "strings" + "time" "github.com/spf13/cobra" "golang.org/x/exp/maps" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/kubernetes" cmdutil "k8s.io/kubectl/pkg/cmd/util" + deploymentutil "k8s.io/kubectl/pkg/util/deployment" "k8s.io/kubectl/pkg/util/templates" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" + "github.com/apecloud/kubeblocks/internal/constant" ) const configKey = "config.yaml" @@ -95,7 +101,7 @@ func NewConfigCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra Run: func(cmd *cobra.Command, args []string) { util.CheckErr(o.Complete(f, cmd)) util.CheckErr(o.Upgrade()) - // TODO: post handle after the config updates + util.CheckErr(markKubeBlocksPodsToLoadConfigMap(o.Client)) }, } helm.AddValueOptionsFlags(cmd.Flags(), &o.ValueOpts) @@ -182,3 +188,50 @@ func getKubeBlocksConfigMap(o *InstallOptions) (*corev1.ConfigMap, error) { } return configMap, nil } + +// markKubeBlocksPodsToLoadConfigMap marks an annotation of the KubeBlocks pods to load the projected volumes of configmap. +// kubelet periodically requeues the Pod after 60-90 seconds, exactly how long it takes Secret/ConfigMaps updates to be reflected to the volumes. +// so can modify the annotation of the pod to directly enter the coordination queue and make changes of the configmap to effective in a timely. +func markKubeBlocksPodsToLoadConfigMap(client kubernetes.Interface) error { + deploy, err := util.GetKubeBlocksDeploy(client) + if err != nil { + return err + } + if deploy == nil { + return nil + } + pods, err := client.CoreV1().Pods(deploy.Namespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=" + types.KubeBlocksChartName, + }) + if err != nil { + return err + } + if len(pods.Items) == 0 { + return nil + } + condition := deploymentutil.GetDeploymentCondition(deploy.Status, appsv1.DeploymentProgressing) + if condition == nil { + return nil + } + podBelongToKubeBlocks := func(pod corev1.Pod) bool { + for _, v := range pod.OwnerReferences { + if v.Kind == constant.ReplicaSetKind && strings.Contains(condition.Message, v.Name) { + return true + } + } + return false + } + for _, pod := range pods.Items { + belongToKubeBlocks := podBelongToKubeBlocks(pod) + if !belongToKubeBlocks { + continue + } + // mark the pod to load configmap + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[types.ReloadConfigMapAnnotationKey] = time.Now().Format(time.RFC3339Nano) + _, _ = client.CoreV1().Pods(deploy.Namespace).Update(context.TODO(), &pod, metav1.UpdateOptions{}) + } + return nil +} diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index 0379bc6a2..29a7e5e02 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -110,6 +110,7 @@ const ( ClassProviderLabelKey = "class.kubeblocks.io/provider" ResourceConstraintProviderLabelKey = "resourceconstraint.kubeblocks.io/provider" + ReloadConfigMapAnnotationKey = "kubeblocks.io/reload-configmap" // mark an annotation to load configmap ) // DataProtection API group diff --git a/internal/constant/const.go b/internal/constant/const.go index 7e729161c..e59c44dbd 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -163,7 +163,7 @@ const ( PersistentVolumeClaimKind = "PersistentVolumeClaim" CronJobKind = "CronJob" JobKind = "Job" - ReplicaSetKind = "ReplicaSetKind" + ReplicaSetKind = "ReplicaSet" VolumeSnapshotKind = "VolumeSnapshot" ServiceKind = "Service" ConfigMapKind = "ConfigMap" From 6fe80bb703f07cdaaa6dd3c318fc81fa77c4c124 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Sat, 6 May 2023 12:07:59 +0800 Subject: [PATCH 233/439] fix: failed render configspec for tpltool (#3068) --- cmd/tpl/app/mock_client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/tpl/app/mock_client.go b/cmd/tpl/app/mock_client.go index 539e65ed9..d03115cb5 100644 --- a/cmd/tpl/app/mock_client.go +++ b/cmd/tpl/app/mock_client.go @@ -89,19 +89,19 @@ func (m *mockClient) List(ctx context.Context, list client.ObjectList, opts ...c } func (m mockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { - return cfgcore.MakeError("not support") + return nil } func (m mockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { - return cfgcore.MakeError("not support") + return nil } func (m mockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { - return cfgcore.MakeError("not support") + return nil } func (m mockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { - return cfgcore.MakeError("not support") + return nil } func (m mockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { From a768d7ff1a5c5566c12731ff99adc402b00178b9 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Sat, 6 May 2023 12:31:00 +0800 Subject: [PATCH 234/439] chore: fix chart message error (#3108) --- .github/workflows/release-helm-chart.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-helm-chart.yml b/.github/workflows/release-helm-chart.yml index 850201c64..b1aa57a09 100644 --- a/.github/workflows/release-helm-chart.yml +++ b/.github/workflows/release-helm-chart.yml @@ -51,7 +51,7 @@ jobs: - name: send message run: | CONTENT="release\u00a0chart\u00a0error" - if [[ "${{ needs.make-test.result }}" == "success" ]]; then + if [[ "${{ needs.release-chart.result }}" == "success" ]]; then CONTENT="release\u00a0chart\u00a0success" fi From c4e718e3c5ff9ef6b0024ea96117bee075509446 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sat, 6 May 2023 14:33:56 +0800 Subject: [PATCH 235/439] chore: tidy up controllers/apps/cluster_status_utils.handleEventForClusterStatus handling (#3111) --- controllers/apps/cluster_status_utils.go | 25 ++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/controllers/apps/cluster_status_utils.go b/controllers/apps/cluster_status_utils.go index bee51d054..8d365fc90 100644 --- a/controllers/apps/cluster_status_utils.go +++ b/controllers/apps/cluster_status_utils.go @@ -193,26 +193,32 @@ func getClusterAvailabilityEffect(componentDef *appsv1alpha1.ClusterComponentDef // getComponentRelatedInfo gets componentMap, clusterAvailabilityMap and component definition information func getComponentRelatedInfo(cluster *appsv1alpha1.Cluster, clusterDef *appsv1alpha1.ClusterDefinition, - componentName string) (map[string]string, map[string]bool, *appsv1alpha1.ClusterComponentDefinition) { + componentName string) (map[string]string, map[string]bool, *appsv1alpha1.ClusterComponentDefinition, error) { var ( compDefName string componentMap = map[string]string{} componentDef *appsv1alpha1.ClusterComponentDefinition ) for _, v := range cluster.Spec.ComponentSpecs { - if v.Name == componentName { + componentMap[v.Name] = v.ComponentDefRef + if compDefName == "" && v.Name == componentName { compDefName = v.ComponentDefRef } - componentMap[v.Name] = v.ComponentDefRef + } + if compDefName == "" { + return nil, nil, nil, fmt.Errorf("expected %s component not found", componentName) } clusterAvailabilityEffectMap := map[string]bool{} for i, v := range clusterDef.Spec.ComponentDefs { clusterAvailabilityEffectMap[v.Name] = getClusterAvailabilityEffect(&v) - if v.Name == compDefName { + if componentDef == nil && v.Name == compDefName { componentDef = &clusterDef.Spec.ComponentDefs[i] } } - return componentMap, clusterAvailabilityEffectMap, componentDef + if componentDef == nil { + return nil, nil, nil, fmt.Errorf("expected %s componentDef not found", compDefName) + } + return componentMap, clusterAvailabilityEffectMap, componentDef, nil } // handleClusterStatusByEvent handles the cluster status when warning event happened @@ -240,11 +246,14 @@ func handleClusterStatusByEvent(ctx context.Context, cli client.Client, recorder componentName := labels[constant.KBAppComponentLabelKey] // get the component phase by component name and sync to Cluster.status.components patch := client.MergeFrom(cluster.DeepCopy()) - componentMap, clusterAvailabilityEffectMap, componentDef := getComponentRelatedInfo(cluster, clusterDef, componentName) clusterComponent := cluster.Spec.GetComponentByName(componentName) if clusterComponent == nil { return nil } + componentMap, clusterAvailabilityEffectMap, componentDef, err := getComponentRelatedInfo(cluster, clusterDef, componentName) + if err != nil { + return err + } // get the component status by event and check whether the component status needs to be synchronized to the cluster component, err := components.NewComponentByType(cli, cluster, clusterComponent, *componentDef) if err != nil { @@ -269,14 +278,11 @@ func handleClusterStatusByEvent(ctx context.Context, cli client.Client, recorder // TODO: Unified cluster event processing // handleEventForClusterStatus handles event for cluster Warning and Failed phase func handleEventForClusterStatus(ctx context.Context, cli client.Client, recorder record.EventRecorder, event *corev1.Event) error { - type predicateProcessor struct { pred func() bool processor func() error } - nilReturnHandler := func() error { return nil } - pps := []predicateProcessor{ { // handle cronjob complete or fail event @@ -313,7 +319,6 @@ func handleEventForClusterStatus(ctx context.Context, cli client.Client, recorde }, }, } - for _, pp := range pps { if pp.pred() { return pp.processor() From b6669245314959f1cbdc6e5d8aeb30d5172ba286 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sat, 6 May 2023 14:38:33 +0800 Subject: [PATCH 236/439] chore: update addon controller RBAC (#3097) --- config/rbac/role.yaml | 7 ++++++ controllers/extensions/addon_controller.go | 3 ++- deploy/helm/config/rbac/role.yaml | 7 ++++++ hack/boilerplate.cue.txt | 24 +++++++++---------- internal/cli/cmd/plugin/plugin.go | 23 ++++++++++-------- internal/cli/cmd/plugin/plugin_test.go | 23 ++++++++++-------- .../lifecycle/transformer_cluster_deletion.go | 23 ++++++++++-------- .../controller/lifecycle/transformer_pitr.go | 23 ++++++++++-------- ...sformer_validate_and_load_ref_resources.go | 23 ++++++++++-------- .../transformer_validate_enable_logs.go | 23 ++++++++++-------- .../lifecycle/transformer_workloads_last.go | 23 ++++++++++-------- 11 files changed, 119 insertions(+), 83 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 3981b5213..def2d8282 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -434,6 +434,13 @@ rules: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/controllers/extensions/addon_controller.go b/controllers/extensions/addon_controller.go index 504b6decd..e41818da2 100644 --- a/controllers/extensions/addon_controller.go +++ b/controllers/extensions/addon_controller.go @@ -63,7 +63,8 @@ func init() { // +kubebuilder:rbac:groups=extensions.kubeblocks.io,resources=addons/finalizers,verbs=update // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete;deletecollection -// +kubebuilder:rbac:groups=core,resources=pods,verbs=delete;deletecollection +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;delete;deletecollection +// +kubebuilder:rbac:groups=core,resources=pods/log,verbs=get;list // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/deploy/helm/config/rbac/role.yaml b/deploy/helm/config/rbac/role.yaml index 3981b5213..def2d8282 100644 --- a/deploy/helm/config/rbac/role.yaml +++ b/deploy/helm/config/rbac/role.yaml @@ -434,6 +434,13 @@ rules: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/hack/boilerplate.cue.txt b/hack/boilerplate.cue.txt index b7a6f2184..be951210c 100644 --- a/hack/boilerplate.cue.txt +++ b/hack/boilerplate.cue.txt @@ -1,17 +1,17 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . diff --git a/internal/cli/cmd/plugin/plugin.go b/internal/cli/cmd/plugin/plugin.go index 7105b4ef8..9942e6779 100644 --- a/internal/cli/cmd/plugin/plugin.go +++ b/internal/cli/cmd/plugin/plugin.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plugin diff --git a/internal/cli/cmd/plugin/plugin_test.go b/internal/cli/cmd/plugin/plugin_test.go index 6129d04d8..b93448e28 100644 --- a/internal/cli/cmd/plugin/plugin_test.go +++ b/internal/cli/cmd/plugin/plugin_test.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package plugin diff --git a/internal/controller/lifecycle/transformer_cluster_deletion.go b/internal/controller/lifecycle/transformer_cluster_deletion.go index babc66800..64e9cc9c9 100644 --- a/internal/controller/lifecycle/transformer_cluster_deletion.go +++ b/internal/controller/lifecycle/transformer_cluster_deletion.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_pitr.go b/internal/controller/lifecycle/transformer_pitr.go index e54c48998..b8e8a527b 100644 --- a/internal/controller/lifecycle/transformer_pitr.go +++ b/internal/controller/lifecycle/transformer_pitr.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_validate_and_load_ref_resources.go b/internal/controller/lifecycle/transformer_validate_and_load_ref_resources.go index 045fda8a8..f46bd7b28 100644 --- a/internal/controller/lifecycle/transformer_validate_and_load_ref_resources.go +++ b/internal/controller/lifecycle/transformer_validate_and_load_ref_resources.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_validate_enable_logs.go b/internal/controller/lifecycle/transformer_validate_enable_logs.go index dc137d1bc..1644cf42e 100644 --- a/internal/controller/lifecycle/transformer_validate_enable_logs.go +++ b/internal/controller/lifecycle/transformer_validate_enable_logs.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle diff --git a/internal/controller/lifecycle/transformer_workloads_last.go b/internal/controller/lifecycle/transformer_workloads_last.go index dea16ee1a..5b8e93d63 100644 --- a/internal/controller/lifecycle/transformer_workloads_last.go +++ b/internal/controller/lifecycle/transformer_workloads_last.go @@ -1,17 +1,20 @@ /* -Copyright ApeCloud, Inc. +Copyright (C) 2022-2023 ApeCloud Co., Ltd -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This file is part of KubeBlocks project - http://www.apache.org/licenses/LICENSE-2.0 +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . */ package lifecycle From 6c107361624a616097c743bd34f418a55cd9a56c Mon Sep 17 00:00:00 2001 From: xingran Date: Sat, 6 May 2023 15:55:13 +0800 Subject: [PATCH 237/439] fix: adjust dependency resource deletion order when cluster deletion (#3065) --- controllers/apps/operations/stop.go | 3 +- internal/controller/graph/dag.go | 17 +++--- .../lifecycle/cluster_plan_builder.go | 55 ++++++++++++++++++- .../lifecycle/transformer_cluster_deletion.go | 20 +++++++ 4 files changed, 83 insertions(+), 12 deletions(-) diff --git a/controllers/apps/operations/stop.go b/controllers/apps/operations/stop.go index 12785e90b..a8ef81b5d 100644 --- a/controllers/apps/operations/stop.go +++ b/controllers/apps/operations/stop.go @@ -142,8 +142,7 @@ func getCompMapFromLastConfiguration(opsRequest *appsv1alpha1.OpsRequest) realAf func deleteConfigMaps(ctx context.Context, cli client.Client, cluster *appsv1alpha1.Cluster) error { inNS := client.InNamespace(cluster.Namespace) ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.GetName(), - constant.AppManagedByLabelKey: constant.AppName, + constant.AppInstanceLabelKey: cluster.GetName(), } return cli.DeleteAllOf(ctx, &corev1.ConfigMap{}, inNS, ml) } diff --git a/internal/controller/graph/dag.go b/internal/controller/graph/dag.go index 27c178f6e..0aaff9347 100644 --- a/internal/controller/graph/dag.go +++ b/internal/controller/graph/dag.go @@ -159,7 +159,7 @@ func (d *DAG) WalkReverseTopoOrder(walkFunc WalkFunc) error { func (d *DAG) Root() Vertex { roots := make([]Vertex, 0) for n := range d.vertices { - if len(d.inAdj(n)) == 0 { + if len(d.InAdj(n)) == 0 { roots = append(roots, n) } } @@ -211,7 +211,7 @@ func (d *DAG) validate() error { } marked[v] = true - adjacent := d.outAdj(v) + adjacent := d.OutAdj(v) for _, vertex := range adjacent { if err := walk(vertex); err != nil { return err @@ -246,9 +246,9 @@ func (d *DAG) topologicalOrder(reverse bool) []Vertex { } var adjacent []Vertex if reverse { - adjacent = d.outAdj(v) + adjacent = d.OutAdj(v) } else { - adjacent = d.inAdj(v) + adjacent = d.InAdj(v) } for _, vertex := range adjacent { walk(vertex) @@ -259,12 +259,11 @@ func (d *DAG) topologicalOrder(reverse bool) []Vertex { for v := range d.vertices { walk(v) } - return orders } -// outAdj returns all adjacent vertices that v points to -func (d *DAG) outAdj(v Vertex) []Vertex { +// OutAdj returns all adjacent vertices that v points to +func (d *DAG) OutAdj(v Vertex) []Vertex { vertices := make([]Vertex, 0) for e := range d.edges { if e.From() == v { @@ -274,8 +273,8 @@ func (d *DAG) outAdj(v Vertex) []Vertex { return vertices } -// inAdj returns all adjacent vertices that point to v -func (d *DAG) inAdj(v Vertex) []Vertex { +// InAdj returns all adjacent vertices that point to v +func (d *DAG) InAdj(v Vertex) []Vertex { vertices := make([]Vertex, 0) for e := range d.edges { if e.To() == v { diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 0aa6565d0..36f21837f 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -31,12 +31,14 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + componentutil "github.com/apecloud/kubeblocks/controllers/apps/components/util" opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" "github.com/apecloud/kubeblocks/internal/constant" roclient "github.com/apecloud/kubeblocks/internal/controller/client" @@ -64,6 +66,7 @@ type clusterPlanBuilder struct { cli client.Client transCtx *ClusterTransformContext transformers graph.TransformerChain + dag *graph.DAG } // clusterPlan a graph.Plan implementation for Cluster reconciliation @@ -151,6 +154,8 @@ func (c *clusterPlanBuilder) Build() (graph.Plan, error) { err = c.transformers.ApplyTo(c.transCtx, dag) // log for debug c.transCtx.Logger.Info(fmt.Sprintf("DAG: %s", dag)) + // add dag to clusterPlanBuilder + c.dag = dag // we got the execution plan plan := &clusterPlan{ @@ -265,7 +270,12 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { } // delete secondary objects if _, ok := node.obj.(*appsv1alpha1.Cluster); !ok { - err := intctrlutil.BackgroundDeleteObject(c.cli, c.transCtx.Context, node.obj) + // check dependency resources has been deleted before deleting the resource + err := c.checkDependencyResourcesDeleted(node) + if err != nil { + return err + } + err = intctrlutil.BackgroundDeleteObject(c.cli, c.transCtx.Context, node.obj) // err := c.cli.Delete(c.transCtx.Context, node.obj) if err != nil && !apierrors.IsNotFound(err) { return err @@ -388,3 +398,46 @@ func (c *clusterPlanBuilder) emitPhaseUpdatingEvent(oldPhase, newPhase appsv1alp _ = opsutil.MarkRunningOpsRequestAnnotation(c.transCtx.Context, c.cli, cluster) } } + +// checkDependencyResourcesDeleted checks if the dependency resources are deleted when cluster is deleted. +func (c *clusterPlanBuilder) checkDependencyResourcesDeleted(node *lifecycleVertex) error { + if c.dag == nil { + return nil + } + // get the dependency resources + outAdj := c.dag.OutAdj(node) + if len(outAdj) == 0 { + return nil + } + for _, out := range outAdj { + outNode, ok := out.(*lifecycleVertex) + if !ok { + return fmt.Errorf("wrong vertex type %v", outNode) + } + // if the node.obj is StatefulSet, check if the pods are deleted + sts, ok := outNode.obj.(*appsv1.StatefulSet) + if ok { + pods, err := componentutil.GetPodListByStatefulSet(c.transCtx.Context, c.cli, sts) + if err != nil { + return err + } + if len(pods) > 0 { + return &realRequeueError{reason: fmt.Sprintf("waiting dependency resource delete, %s/%s dependency resource statefulSet %s/%s still have pods", + node.obj.GetNamespace(), node.obj.GetName(), outNode.obj.GetNamespace(), outNode.obj.GetName()), requeueAfter: requeueDuration} + } + } + // check if the dependency resource is deleted + err := c.cli.Get(c.transCtx.Context, types.NamespacedName{Name: outNode.obj.GetName(), Namespace: outNode.obj.GetNamespace()}, outNode.obj) + if err != nil { + if apierrors.IsNotFound(err) { + continue + } + return err + } + if outNode.obj != nil { + return &realRequeueError{reason: fmt.Sprintf("waiting dependency resource delete, %s/%s dependency resource %s/%s is not deleted", + node.obj.GetNamespace(), node.obj.GetName(), outNode.obj.GetNamespace(), outNode.obj.GetName()), requeueAfter: requeueDuration} + } + } + return nil +} diff --git a/internal/controller/lifecycle/transformer_cluster_deletion.go b/internal/controller/lifecycle/transformer_cluster_deletion.go index 64e9cc9c9..15246e5ef 100644 --- a/internal/controller/lifecycle/transformer_cluster_deletion.go +++ b/internal/controller/lifecycle/transformer_cluster_deletion.go @@ -79,6 +79,10 @@ func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * dag.AddVertex(vertex) dag.Connect(root, vertex) } + + // adjust the dependency resource deletion order + adjustDependencyResourceDeletionOrder(root, dag) + root.action = actionPtr(DELETE) // fast return, that is stopping the plan.Build() stage and jump to plan.Execute() directly @@ -120,4 +124,20 @@ func kindsForWipeOut() []client.ObjectList { return append(kinds, kindsPlus...) } +// adjustDependencyResourceDeletionOrder adjusts the deletion order of resources by adjusting DAG topology. +// find all vertices of StatefulSets and connect them to ConfigMap, +// this is to ensure that ConfigMap is deleted after StatefulSet Workloads are deleted. +func adjustDependencyResourceDeletionOrder(root *lifecycleVertex, dag *graph.DAG) { + vertices := findAll[*appsv1.StatefulSet](dag) + cmVertices := findAll[*corev1.ConfigMap](dag) + if len(vertices) > 0 && len(cmVertices) > 0 { + for _, vertex := range vertices { + dag.RemoveEdge(graph.RealEdge(root, vertex)) + for _, cmVertex := range cmVertices { + dag.Connect(cmVertex, vertex) + } + } + } +} + var _ graph.Transformer = &ClusterDeletionTransformer{} From a5affb592d97291aa4693c55e42fccf325199d62 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Sat, 6 May 2023 18:17:36 +0800 Subject: [PATCH 238/439] fix: incorrect environment variables KB_PRIMARY_POD_NAME (#3063) --- internal/controller/builder/builder.go | 24 ++++++--------------- internal/controller/builder/builder_test.go | 11 +++------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index e9d5ffa8c..2ad043edc 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -434,25 +434,15 @@ func BuildEnvConfig(params BuilderParams, reqCtx intctrlutil.RequestCtx, cli cli for j := 0; j < int(params.Component.Replicas); j++ { hostNameTplKey := prefix + strconv.Itoa(j) + "_HOSTNAME" hostNameTplValue := params.Cluster.Name + "-" + params.Component.Name + "-" + strconv.Itoa(j) - - if params.Component.WorkloadType != appsv1alpha1.Replication { - envData[hostNameTplKey] = fmt.Sprintf("%s.%s", hostNameTplValue, svcName) - continue - } + envData[hostNameTplKey] = fmt.Sprintf("%s.%s", hostNameTplValue, svcName) // build env for replication workload - // the 1st replica's hostname should not have suffix like '-0' - if j == 0 { - envData[hostNameTplKey] = fmt.Sprintf("%s.%s", hostNameTplValue, svcName) - } else { - envData[hostNameTplKey] = fmt.Sprintf("%s.%s", hostNameTplValue+"-0", svcName) - } - // if primaryIndex is 0, the pod name have to be no suffix '-0' - primaryIndex := params.Component.GetPrimaryIndex() - if primaryIndex == 0 { - envData[constant.KBReplicationSetPrimaryPodName] = fmt.Sprintf("%s-%s-%d.%s", params.Cluster.Name, params.Component.Name, primaryIndex, svcName) - } else { - envData[constant.KBReplicationSetPrimaryPodName] = fmt.Sprintf("%s-%s-%d-%d.%s", params.Cluster.Name, params.Component.Name, primaryIndex, 0, svcName) + if params.Component.WorkloadType == appsv1alpha1.Replication { + envData[constant.KBReplicationSetPrimaryPodName] = fmt.Sprintf("%s-%s-%d.%s", + params.Cluster.Name, + params.Component.Name, + params.Component.GetPrimaryIndex(), + svcName) } } diff --git a/internal/controller/builder/builder_test.go b/internal/controller/builder/builder_test.go index afc929fe0..a9536e0f8 100644 --- a/internal/controller/builder/builder_test.go +++ b/internal/controller/builder/builder_test.go @@ -374,13 +374,8 @@ var _ = Describe("builder", func() { stsName := fmt.Sprintf("%s-%s", params.Cluster.Name, params.Component.Name) svcName := fmt.Sprintf("%s-headless", stsName) By("Checking KB_PRIMARY_POD_NAME value be right") - if int(params.Component.GetPrimaryIndex()) == 0 { - Expect(cfg.Data["KB_PRIMARY_POD_NAME"]). - Should(Equal(stsName + "-" + strconv.Itoa(int(params.Component.GetPrimaryIndex())) + "." + svcName)) - } else { - Expect(cfg.Data["KB_PRIMARY_POD_NAME"]). - Should(Equal(stsName + "-" + strconv.Itoa(int(params.Component.GetPrimaryIndex())) + "-0." + svcName)) - } + Expect(cfg.Data["KB_PRIMARY_POD_NAME"]). + Should(Equal(stsName + "-" + strconv.Itoa(int(params.Component.GetPrimaryIndex())) + "." + svcName)) for i := 0; i < int(params.Component.Replicas); i++ { if i == 0 { By("Checking the 1st replica's hostname should not have suffix '-0'") @@ -388,7 +383,7 @@ var _ = Describe("builder", func() { Should(Equal(stsName + "-" + strconv.Itoa(0) + "." + svcName)) } else { Expect(cfg.Data["KB_"+strings.ToUpper(params.Component.Type)+"_"+strconv.Itoa(i)+"_HOSTNAME"]). - Should(Equal(stsName + "-" + strconv.Itoa(int(params.Component.GetPrimaryIndex())) + "-0." + svcName)) + Should(Equal(stsName + "-" + strconv.Itoa(int(params.Component.GetPrimaryIndex())) + "." + svcName)) } } } From 81590b3781e561d2804404e9c4bb5010b81ee5f3 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Sat, 6 May 2023 19:26:48 +0800 Subject: [PATCH 239/439] fix: fixed #3109, addon controller auto-install with empty default values handling (#3118) --- controllers/extensions/addon_controller.go | 2 +- .../extensions/addon_controller_stages.go | 25 +++++++++++++++---- controllers/extensions/const.go | 1 + 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/controllers/extensions/addon_controller.go b/controllers/extensions/addon_controller.go index e41818da2..0f5d5d966 100644 --- a/controllers/extensions/addon_controller.go +++ b/controllers/extensions/addon_controller.go @@ -186,7 +186,7 @@ func (r *AddonReconciler) cleanupJobPods(reqCtx intctrlutil.RequestCtx) error { } func (r *AddonReconciler) deleteExternalResources(reqCtx intctrlutil.RequestCtx, addon *extensionsv1alpha1.Addon) (*ctrl.Result, error) { - if addon.Annotations != nil && addon.Annotations[NoDeleteJobs] == "true" { + if addon.Annotations != nil && addon.Annotations[NoDeleteJobs] == trueVal { return nil, nil } deleteJobIfExist := func(jobName string) error { diff --git a/controllers/extensions/addon_controller_stages.go b/controllers/extensions/addon_controller_stages.go index a74d05631..d9b216ac7 100644 --- a/controllers/extensions/addon_controller_stages.go +++ b/controllers/extensions/addon_controller_stages.go @@ -53,6 +53,7 @@ const ( resultValueKey = "result" errorValueKey = "err" operandValueKey = "operand" + trueVal = "true" ) func init() { @@ -71,10 +72,15 @@ func (r *stageCtx) setReconciled() { } func (r *stageCtx) setRequeueAfter(duration time.Duration, msg string) { - res, err := intctrlutil.RequeueAfter(time.Second, r.reqCtx.Log, msg) + res, err := intctrlutil.RequeueAfter(duration, r.reqCtx.Log, msg) r.updateResultNErr(&res, err) } +// func (r *stageCtx) setRequeue(msg string) { +// res, err := intctrlutil.Requeue(r.reqCtx.Log, msg) +// r.updateResultNErr(&res, err) +// } + func (r *stageCtx) setRequeueWithErr(err error, msg string) { res, err := intctrlutil.CheckedRequeueWithError(err, r.reqCtx.Log, msg) r.updateResultNErr(&res, err) @@ -262,7 +268,7 @@ func (r *installableCheckStage) Handle(ctx context.Context) { if addon.Spec.InstallSpec != nil { return } - if addon.Annotations != nil && addon.Annotations[SkipInstallableCheck] == "true" { + if addon.Annotations != nil && addon.Annotations[SkipInstallableCheck] == trueVal { r.reconciler.Event(addon, "Warning", InstallableCheckSkipped, "Installable check skipped.") return @@ -307,7 +313,7 @@ func (r *autoInstallCheckStage) Handle(ctx context.Context) { return } // proceed if has specified addon.spec.installSpec - if addon.Spec.InstallSpec.HasSetValues() { + if addon.Spec.InstallSpec != nil { r.reqCtx.Log.V(1).Info("has specified addon.spec.installSpec") return } @@ -319,8 +325,11 @@ func (r *autoInstallCheckStage) Handle(ctx context.Context) { func (r *enabledWithDefaultValuesStage) Handle(ctx context.Context) { r.process(func(addon *extensionsv1alpha1.Addon) { r.reqCtx.Log.V(1).Info("enabledWithDefaultValuesStage", "phase", addon.Status.Phase) - if addon.Spec.InstallSpec.HasSetValues() || - addon.Spec.InstallSpec.IsDisabled() { + if addon.Spec.InstallSpec.HasSetValues() || addon.Spec.InstallSpec.IsDisabled() { + r.reqCtx.Log.V(1).Info("has specified addon.spec.installSpec") + return + } + if v, ok := addon.Annotations[AddonDefaultIsEmpty]; ok && v == trueVal { return } enabledAddonWithDefaultValues(ctx, &r.stageCtx, addon, AddonSetDefaultValues, "Addon enabled with default values") @@ -878,6 +887,12 @@ func enabledAddonWithDefaultValues(ctx context.Context, stageCtx *stageCtx, setInstallSpec := func(di *extensionsv1alpha1.AddonDefaultInstallSpecItem) { addon.Spec.InstallSpec = di.AddonInstallSpec.DeepCopy() addon.Spec.InstallSpec.Enabled = true + if addon.Annotations == nil { + addon.Annotations = map[string]string{} + } + if di.AddonInstallSpec.IsEmpty() { + addon.Annotations[AddonDefaultIsEmpty] = trueVal + } if err := stageCtx.reconciler.Client.Update(ctx, addon); err != nil { stageCtx.setRequeueWithErr(err, "") return diff --git a/controllers/extensions/const.go b/controllers/extensions/const.go index 00d52ad63..f04c97629 100644 --- a/controllers/extensions/const.go +++ b/controllers/extensions/const.go @@ -27,6 +27,7 @@ const ( ControllerPaused = "controller.kubeblocks.io/controller-paused" SkipInstallableCheck = "extensions.kubeblocks.io/skip-installable-check" NoDeleteJobs = "extensions.kubeblocks.io/no-delete-jobs" + AddonDefaultIsEmpty = "addons.extensions.kubeblocks.io/default-is-empty" // condition reasons AddonDisabled = "AddonDisabled" From fb5f4c1b7e00ddbb9d57fc4422b5c9b01e4f13d7 Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Mon, 8 May 2023 12:58:30 +0800 Subject: [PATCH 240/439] fix: kbcli resources list alphabetically (#3072) --- internal/cli/list/list.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 3c3209d6c..e6da3148d 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -63,8 +63,8 @@ type ListOptions struct { // print the result or not, if true, use default printer to print, otherwise, // only return the result to caller. - Print bool - + Print bool + SortBy string genericclioptions.IOStreams } @@ -75,6 +75,7 @@ func NewListOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, IOStreams: streams, GVR: gvr, Print: true, + SortBy: ".metadata.name", } } @@ -84,6 +85,7 @@ func (o *ListOptions) AddFlags(cmd *cobra.Command, isClusterScope ...bool) { } cmd.Flags().StringVarP(&o.LabelSelector, "selector", "l", o.LabelSelector, "Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints.") cmd.Flags().BoolVar(&o.ShowLabels, "show-labels", false, "When printing, show all labels as the last column (default hide labels column)") + //Todo: --sortBy supports custom field sorting, now `list` is to sort using the `.metadata.name` field in default printer.AddOutputFlag(cmd, &o.Format) } @@ -130,6 +132,7 @@ func (o *ListOptions) Complete() error { } if o.Format.IsHumanReadable() { + p = &cmdget.SortingPrinter{Delegate: p, SortField: o.SortBy} p = &cmdget.TablePrinter{Delegate: p} } return p.PrintObj, nil @@ -178,6 +181,9 @@ func (o *ListOptions) transformRequests(req *rest.Request) { fmt.Sprintf("application/json;as=Table;v=%s;g=%s", metav1beta1.SchemeGroupVersion.Version, metav1beta1.GroupName), "application/json", }, ",")) + if len(o.SortBy) > 0 { + req.Param("includeObject", "Object") + } } func (o *ListOptions) printResult(r *resource.Result) error { From 019efe188e87c2577899c6d8de73d821c75baf39 Mon Sep 17 00:00:00 2001 From: xingran Date: Mon, 8 May 2023 13:41:29 +0800 Subject: [PATCH 241/439] chore: add pg-replication componentDef to compatible v0.4.5 standalone postgres (#3115) --- .../templates/clusterdefinition.yaml | 290 ++++++++++++++++++ internal/cli/cluster/helper.go | 2 +- 2 files changed, 291 insertions(+), 1 deletion(-) diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index c77427c7b..35a1e6009 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -310,3 +310,293 @@ spec: statements: creation: CREATE USER $(USERNAME) WITH REPLICATION PASSWORD '$(PASSWD)'; update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; + ## The following componentDef definition is for forward compatibility with the v0.4.5 single-node postgres cluster. + ## It is obsolete and should not be used in versions after 0.5.0. Subsequent versions will remove this componentDef. + - name: pg-replication + workloadType: Replication + characterType: postgresql + probes: + monitor: + builtIn: false + exporterConfig: + scrapePath: /metrics + scrapePort: 9187 + logConfigs: + {{- range $name,$pattern := .Values.logConfigs }} + - name: {{ $name }} + filePathPattern: {{ $pattern }} + {{- end }} + configSpecs: + - name: postgresql-configuration + templateRef: postgresql-configuration + constraintRef: postgresql14-cc + keys: + - postgresql.conf + namespace: {{ .Release.Namespace }} + volumeName: postgresql-config + defaultMode: 0777 + - name: postgresql-custom-metrics + templateRef: postgresql14-custom-metrics + namespace: {{ .Release.Namespace }} + volumeName: postgresql-custom-metrics + defaultMode: 0777 + scriptSpecs: + - name: postgresql-scripts + templateRef: postgresql-scripts + namespace: {{ .Release.Namespace }} + volumeName: scripts + defaultMode: 0777 + service: + ports: + - name: tcp-postgresql + protocol: TCP + port: 5432 + targetPort: tcp-postgresql + - name: http-metrics-postgresql + port: 9187 + targetPort: http-metrics + volumeTypes: + - name: data + type: data + podSpec: + securityContext: + fsGroup: 1001 + containers: + - name: postgresql + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + securityContext: + runAsUser: 0 + volumeMounts: + - name: dshm + mountPath: /dev/shm + - name: data + mountPath: /postgresql + - name: postgresql-config + mountPath: /postgresql/conf + - name: scripts + mountPath: /scripts + ports: + - name: tcp-postgresql + containerPort: 5432 + command: + - /scripts/setup.sh + livenessProbe: + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - /bin/sh + - -c + - exec pg_isready -U {{ default "postgres" | quote }} -h 127.0.0.1 -p 5432 + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - /bin/sh + - -c + - -ee + - | + exec pg_isready -U {{ default "postgres" | quote }} -h 127.0.0.1 -p 5432 + [ -f /postgresql/tmp/.initialized ] || [ -f /postgresql/.initialized ] + env: + - name: BITNAMI_DEBUG + value: "false" + - name: POSTGRESQL_PORT_NUMBER + value: "5432" + - name: POSTGRESQL_VOLUME_DIR + value: /postgresql + - name: PGDATA + value: /postgresql/data + - name: PGUSER + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: postgres-password + # Authentication + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: username + - name: POSTGRES_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: postgres-password + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: postgres-password + - name: POSTGRES_DB + value: {{ (include "postgresql.database" .) | quote }} + # Audit + - name: POSTGRESQL_LOG_HOSTNAME + value: {{ .Values.audit.logHostname | quote }} + - name: POSTGRESQL_LOG_CONNECTIONS + value: {{ .Values.audit.logConnections | quote }} + - name: POSTGRESQL_LOG_DISCONNECTIONS + value: {{ .Values.audit.logDisconnections | quote }} + {{- if .Values.audit.logLinePrefix }} + - name: POSTGRESQL_LOG_LINE_PREFIX + value: {{ .Values.audit.logLinePrefix | quote }} + {{- end }} + {{- if .Values.audit.logTimezone }} + - name: POSTGRESQL_LOG_TIMEZONE + value: {{ .Values.audit.logTimezone | quote }} + {{- end }} + {{- if .Values.audit.pgAuditLog }} + - name: POSTGRESQL_PGAUDIT_LOG + value: {{ .Values.audit.pgAuditLog | quote }} + {{- end }} + - name: POSTGRESQL_PGAUDIT_LOG_CATALOG + value: {{ .Values.audit.pgAuditLogCatalog | quote }} + # Others + - name: POSTGRESQL_CLIENT_MIN_MESSAGES + value: {{ .Values.audit.clientMinMessages | quote }} + - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES + value: {{ .Values.postgresqlSharedPreloadLibraries | quote }} + {{- if .Values.primary.extraEnvVars }} + {{- include "tplvalues.render" (dict "value" .Values.primary.extraEnvVars "context" $) | nindent 12 }} + {{- end }} + {{- if or .Values.primary.extraEnvVarsCM .Values.primary.extraEnvVarsSecret }} + envFrom: + {{- if .Values.primary.extraEnvVarsCM }} + - configMapRef: + name: {{ .Values.primary.extraEnvVarsCM }} + {{- end }} + {{- if .Values.primary.extraEnvVarsSecret }} + - secretRef: + name: {{ .Values.primary.extraEnvVarsSecret }} + {{- end }} + {{- end }} + - name: metrics + image: {{ .Values.metrics.image.registry | default "docker.io" }}/{{ .Values.metrics.image.repository }}:{{ .Values.metrics.image.tag }} + imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} + securityContext: + runAsUser: 0 + env: + {{- $database := "postgres" }} + {{- $sslmode := "disable" }} + - name: DATA_SOURCE_URI + value: {{ printf "127.0.0.1:5432/%s?sslmode=%s" $database $sslmode }} + - name: DATA_SOURCE_PASS + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: postgres-password + - name: DATA_SOURCE_USER + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: username + command: + - "/opt/bitnami/postgres-exporter/bin/postgres_exporter" + - "--auto-discover-databases" + - "--extend.query-path=/opt/conf/custom-metrics.yaml" + - "--exclude-databases=template0,template1" + - "--log.level=info" + ports: + - name: http-metrics + containerPort: 9187 + livenessProbe: + failureThreshold: 6 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + httpGet: + path: / + port: http-metrics + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + httpGet: + path: / + port: http-metrics + volumeMounts: + - name: postgresql-custom-metrics + mountPath: /opt/conf + volumes: + - name: dshm + emptyDir: + medium: Memory + {{- with .Values.shmVolume.sizeLimit }} + sizeLimit: {{ . }} + {{- end }} + systemAccounts: + cmdExecutorConfig: + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ default .Values.image.tag .Chart.AppVersion }} + command: + - psql + args: + - -h$(KB_ACCOUNT_ENDPOINT) + - -c + - $(KB_ACCOUNT_STATEMENT) + env: + - name: PGUSER + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: postgres-password + passwordConfig: + length: 10 + numDigits: 5 + numSymbols: 0 + letterCase: MixedCases + accounts: + - name: kbadmin + provisionPolicy: + type: CreateByStmt + scope: AnyPods + statements: + creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)'; + deletion: DROP USER IF EXISTS $(USERNAME); + - name: kbdataprotection + provisionPolicy: + type: CreateByStmt + scope: AnyPods + statements: + creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)'; + deletion: DROP USER IF EXISTS $(USERNAME); + - name: kbprobe + provisionPolicy: + type: CreateByStmt + scope: AnyPods + statements: + creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; GRANT pg_monitor TO $(USERNAME); + deletion: DROP USER IF EXISTS $(USERNAME); + - name: kbmonitoring + provisionPolicy: + type: CreateByStmt + scope: AnyPods + statements: + creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; GRANT pg_monitor TO $(USERNAME); + deletion: DROP USER IF EXISTS $(USERNAME); + - name: kbreplicator + provisionPolicy: + type: CreateByStmt + scope: AnyPods + statements: + creation: CREATE USER $(USERNAME) WITH REPLICATION PASSWORD '$(PASSWD)'; + deletion: DROP USER IF EXISTS $(USERNAME); diff --git a/internal/cli/cluster/helper.go b/internal/cli/cluster/helper.go index 50bebdf89..69e374805 100644 --- a/internal/cli/cluster/helper.go +++ b/internal/cli/cluster/helper.go @@ -243,7 +243,7 @@ func GetClusterDefByName(dynamic dynamic.Interface, name string) (*appsv1alpha1. } func GetDefaultCompName(cd *appsv1alpha1.ClusterDefinition) (string, error) { - if len(cd.Spec.ComponentDefs) == 1 { + if len(cd.Spec.ComponentDefs) > 0 { return cd.Spec.ComponentDefs[0].Name, nil } return "", fmt.Errorf("failed to get the default component definition name") From ff598371b636098f51e126a0e7a72d8ac6ea5925 Mon Sep 17 00:00:00 2001 From: zjx20 Date: Mon, 8 May 2023 17:18:50 +0800 Subject: [PATCH 242/439] chore: delete empty folders when all backups have been deleted (#3129) --- .../dataprotection/backup_controller.go | 35 ++++++++++++++++++- .../restorejob_controller_test.go | 34 ++++++++---------- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 447127c75..11d4e5ef8 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -746,11 +746,44 @@ func (r *BackupReconciler) createDeleteBackupFileJob( backupPVCName string, backupFilePath string) error { + // make sure the path has a leading slash + if !strings.HasPrefix(backupFilePath, "/") { + backupFilePath = "/" + backupFilePath + } + + // this script first deletes the directory where the backup is located (including files + // in the directory), and then traverses up the path level by level to clean up empty directories. + deleteScript := fmt.Sprintf(` + backupPathBase=%s; + targetPath="${backupPathBase}%s"; + + echo "removing backup files in ${targetPath}"; + rm -rf "${targetPath}"; + + absBackupPathBase=$(realpath "${backupPathBase}"); + curr=$(realpath "${targetPath}"); + while true; do + parent=$(dirname "${curr}"); + if [ "${parent}" == "${absBackupPathBase}" ]; then + echo "reach backupPathBase ${backupPathBase}, done"; + break; + fi; + if [ ! "$(ls -A "${parent}")" ]; then + echo "${parent} is empty, removing it..."; + rmdir "${parent}"; + else + echo "${parent} is not empty, done"; + break; + fi; + curr="${parent}"; + done + `, backupPathBase, backupFilePath) + // build container container := corev1.Container{} container.Name = backup.Name container.Command = []string{"sh", "-c"} - container.Args = []string{fmt.Sprintf("rm -rf %s%s", backupPathBase, backupFilePath)} + container.Args = []string{deleteScript} container.Image = viper.GetString(constant.KBToolsImage) container.ImagePullPolicy = corev1.PullPolicy(viper.GetString(constant.KBImagePullPolicy)) diff --git a/controllers/dataprotection/restorejob_controller_test.go b/controllers/dataprotection/restorejob_controller_test.go index 79a009aad..11ef660b6 100644 --- a/controllers/dataprotection/restorejob_controller_test.go +++ b/controllers/dataprotection/restorejob_controller_test.go @@ -129,28 +129,24 @@ var _ = Describe("RestoreJob Controller", func() { } patchBackupStatus := func(phase dataprotectionv1alpha1.BackupPhase, key types.NamespacedName) { - backup := dataprotectionv1alpha1.Backup{} - Eventually(func() error { - return k8sClient.Get(ctx, key, &backup) - }).Should(Succeed()) - Expect(k8sClient.Get(ctx, key, &backup)).Should(Succeed()) - - patch := client.MergeFrom(backup.DeepCopy()) - backup.Status.Phase = phase - Expect(k8sClient.Status().Patch(ctx, &backup, patch)).Should(Succeed()) + Eventually(testapps.GetAndChangeObjStatus(&testCtx, key, func(backup *dataprotectionv1alpha1.Backup) { + backup.Status.Phase = phase + })).Should(Succeed()) } patchK8sJobStatus := func(jobStatus batchv1.JobConditionType, key types.NamespacedName) { - k8sJob := batchv1.Job{} - Eventually(func() error { - return k8sClient.Get(ctx, key, &k8sJob) - }).Should(Succeed()) - Expect(k8sClient.Get(ctx, key, &k8sJob)).Should(Succeed()) - - patch := client.MergeFrom(k8sJob.DeepCopy()) - jobCondition := batchv1.JobCondition{Type: jobStatus} - k8sJob.Status.Conditions = append(k8sJob.Status.Conditions, jobCondition) - Expect(k8sClient.Status().Patch(ctx, &k8sJob, patch)).Should(Succeed()) + Eventually(testapps.GetAndChangeObjStatus(&testCtx, key, func(job *batchv1.Job) { + found := false + for _, cond := range job.Status.Conditions { + if cond.Type == jobStatus { + found = true + } + } + if !found { + jobCondition := batchv1.JobCondition{Type: jobStatus} + job.Status.Conditions = append(job.Status.Conditions, jobCondition) + } + })).Should(Succeed()) } testRestoreJob := func(withResources ...bool) { From 694439b054d9834ae73d056705ce17caf281d725 Mon Sep 17 00:00:00 2001 From: "zheyi.cqy" Date: Mon, 8 May 2023 17:23:36 +0800 Subject: [PATCH 243/439] chore: prompt optimization when migration does not install crd (#3134) --- internal/cli/cmd/migration/base.go | 18 ++++++++++++++++++ internal/cli/cmd/migration/create.go | 8 ++------ internal/cli/cmd/migration/describe.go | 8 ++------ internal/cli/cmd/migration/list.go | 2 +- internal/cli/cmd/migration/logs.go | 8 ++------ internal/cli/cmd/migration/templates.go | 2 +- internal/cli/cmd/migration/terminate.go | 2 +- 7 files changed, 27 insertions(+), 21 deletions(-) diff --git a/internal/cli/cmd/migration/base.go b/internal/cli/cmd/migration/base.go index 03b3fa5cd..d0ae51134 100644 --- a/internal/cli/cmd/migration/base.go +++ b/internal/cli/cmd/migration/base.go @@ -22,8 +22,10 @@ package migration import ( "context" "fmt" + "os" "strings" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -32,6 +34,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/types" migrationv1 "github.com/apecloud/kubeblocks/internal/cli/types/migrationapi" + "github.com/apecloud/kubeblocks/internal/cli/util" ) const ( @@ -40,6 +43,10 @@ const ( SerialJobOrderAnnotation = "common.apecloud.io/serial_job_order" ) +const ( + invalidMigrationCrdAdvice = "to use migration-related functions, please ensure that the addon of migration is enabled. you can use: 'kbcli addon enable migration' to enable the addon" +) + // Endpoint // Todo: For the source or target is cluster in KubeBlocks. A better way is to get secret from {$clustername}-conn-credential, so the username, password, addresses can be omitted @@ -202,6 +209,17 @@ func IsMigrationCrdValidWithDynamic(dynamic *dynamic.Interface) (bool, error) { return true, nil } +func PrintCrdInvalidError(err error) { + if err == nil { + return + } + if !errors.IsNotFound(err) { + util.CheckErr(err) + } + fmt.Fprintf(os.Stderr, "hint: %s\n", invalidMigrationCrdAdvice) + os.Exit(cmdutil.DefaultErrorExitCode) +} + func IsMigrationCrdValidWithFactory(factory cmdutil.Factory) (bool, error) { dynamic, err := factory.DynamicClient() if err != nil { diff --git a/internal/cli/cmd/migration/create.go b/internal/cli/cmd/migration/create.go index 81967e8f5..f898f6afd 100644 --- a/internal/cli/cmd/migration/create.go +++ b/internal/cli/cmd/migration/create.go @@ -26,7 +26,6 @@ import ( "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/cli-runtime/pkg/genericclioptions" cmdutil "k8s.io/kubectl/pkg/cmd/util" @@ -107,11 +106,8 @@ func NewMigrationCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStream func (o *CreateMigrationOptions) Validate() error { var err error - _, err = IsMigrationCrdValidWithDynamic(&o.Dynamic) - if errors.IsNotFound(err) { - return fmt.Errorf("datamigration crd is not install") - } else if err != nil { - return err + if _, err = IsMigrationCrdValidWithDynamic(&o.Dynamic); err != nil { + PrintCrdInvalidError(err) } if o.Template == "" { diff --git a/internal/cli/cmd/migration/describe.go b/internal/cli/cmd/migration/describe.go index 4ea8a5d94..c9b835d5c 100644 --- a/internal/cli/cmd/migration/describe.go +++ b/internal/cli/cmd/migration/describe.go @@ -32,7 +32,6 @@ import ( appv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" @@ -107,11 +106,8 @@ func (o *describeOptions) complete(args []string) error { return err } - _, err = IsMigrationCrdValidWithDynamic(&o.dynamic) - if errors.IsNotFound(err) { - return fmt.Errorf("datamigration crd is not install") - } else if err != nil { - return err + if _, err = IsMigrationCrdValidWithDynamic(&o.dynamic); err != nil { + PrintCrdInvalidError(err) } if len(args) == 0 { diff --git a/internal/cli/cmd/migration/list.go b/internal/cli/cmd/migration/list.go index 49b21abe2..958b31df2 100644 --- a/internal/cli/cmd/migration/list.go +++ b/internal/cli/cmd/migration/list.go @@ -39,7 +39,7 @@ func NewMigrationListCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) ValidArgsFunction: util.ResourceNameCompletionFunc(f, o.GVR), Run: func(cmd *cobra.Command, args []string) { _, validErr := IsMigrationCrdValidWithFactory(o.Factory) - util.CheckErr(validErr) + PrintCrdInvalidError(validErr) o.Names = args _, err := o.Run() util.CheckErr(err) diff --git a/internal/cli/cmd/migration/logs.go b/internal/cli/cmd/migration/logs.go index 3a342fd4e..fe5ec056b 100644 --- a/internal/cli/cmd/migration/logs.go +++ b/internal/cli/cmd/migration/logs.go @@ -28,7 +28,6 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/dynamic" @@ -109,11 +108,8 @@ func (o *LogsOptions) complete(f cmdutil.Factory, cmd *cobra.Command, args []str return err } - _, err = IsMigrationCrdValidWithDynamic(&o.Dynamic) - if errors.IsNotFound(err) { - return fmt.Errorf("datamigration crd is not install") - } else if err != nil { - return err + if _, err = IsMigrationCrdValidWithDynamic(&o.Dynamic); err != nil { + PrintCrdInvalidError(err) } taskObj, err := o.getMigrationObjects(o.taskName) diff --git a/internal/cli/cmd/migration/templates.go b/internal/cli/cmd/migration/templates.go index f1bb2ca51..89163507e 100644 --- a/internal/cli/cmd/migration/templates.go +++ b/internal/cli/cmd/migration/templates.go @@ -39,7 +39,7 @@ func NewMigrationTemplatesCmd(f cmdutil.Factory, streams genericclioptions.IOStr ValidArgsFunction: util.ResourceNameCompletionFunc(f, o.GVR), Run: func(cmd *cobra.Command, args []string) { _, validErr := IsMigrationCrdValidWithFactory(o.Factory) - util.CheckErr(validErr) + PrintCrdInvalidError(validErr) o.Names = args _, err := o.Run() util.CheckErr(err) diff --git a/internal/cli/cmd/migration/terminate.go b/internal/cli/cmd/migration/terminate.go index 9c476b681..7b599e772 100644 --- a/internal/cli/cmd/migration/terminate.go +++ b/internal/cli/cmd/migration/terminate.go @@ -40,7 +40,7 @@ func NewMigrationTerminateCmd(f cmdutil.Factory, streams genericclioptions.IOStr ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.MigrationTaskGVR()), Run: func(cmd *cobra.Command, args []string) { _, validErr := IsMigrationCrdValidWithFactory(o.Factory) - util.CheckErr(validErr) + PrintCrdInvalidError(validErr) util.CheckErr(deleteMigrationTask(o, args)) }, } From 76db197e40a4313c7260c7e984771ff0c79bcdf4 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Mon, 8 May 2023 18:55:49 +0800 Subject: [PATCH 244/439] chore: refine kb status selector and update status msg (#3135) --- internal/cli/cmd/kubeblocks/status.go | 51 +++++++++++++++++++----- internal/cli/cmd/kubeblocks/uninstall.go | 6 ++- internal/constant/const.go | 1 + 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/internal/cli/cmd/kubeblocks/status.go b/internal/cli/cmd/kubeblocks/status.go index a4686ffa0..d01927534 100644 --- a/internal/cli/cmd/kubeblocks/status.go +++ b/internal/cli/cmd/kubeblocks/status.go @@ -80,9 +80,12 @@ var ( types.ServiceGVR(), } - kubeBlocksRBAC = []schema.GroupVersionResource{ + kubeBlocksClusterRBAC = []schema.GroupVersionResource{ types.ClusterRoleGVR(), types.ClusterRoleBindingGVR(), + } + + kubeBlocksNamespacedRBAC = []schema.GroupVersionResource{ types.RoleGVR(), types.RoleBindingGVR(), types.ServiceAccountGVR(), @@ -96,6 +99,7 @@ var ( types.ConfigmapGVR(), types.SecretGVR(), } + notAvailable = "N/A" ) type statusOptions struct { @@ -205,9 +209,17 @@ func (o *statusOptions) buildSelectorList(ctx context.Context, allErrs *[]error) func (o *statusOptions) showAddons() { fmt.Fprintln(o.Out, "\nKubeBlocks Addons:") tbl := printer.NewTablePrinter(o.Out) - tbl.SetHeader("NAME", "STATUS", "TYPE") + tbl.SetHeader("NAME", "STATUS", "TYPE", "PROVIDER") + + var provider string + var ok bool for _, addon := range o.addons { - tbl.AddRow(addon.Name, addon.Status.Phase, addon.Spec.Type) + if addon.Labels == nil { + provider = notAvailable + } else if provider, ok = addon.Labels[constant.AddonProviderLableKey]; !ok { + provider = notAvailable + } + tbl.AddRow(addon.Name, addon.Status.Phase, addon.Spec.Type, provider) } tbl.Print() } @@ -240,15 +252,28 @@ func (o *statusOptions) showKubeBlocksConfig(ctx context.Context, allErrs *[]err } func (o *statusOptions) showKubeBlocksRBAC(ctx context.Context, allErrs *[]error) { - fmt.Fprintln(o.Out, "\nKubeBlocks RBAC:") + fmt.Fprintln(o.Out, "\nKubeBlocks Global RBAC:") tblPrinter := printer.NewTablePrinter(o.Out) + tblPrinter.SetHeader("KIND", "NAME") + unstructuredList := listResourceByGVR(ctx, o.dynamic, metav1.NamespaceAll, kubeBlocksClusterRBAC, selectorList, allErrs) + for _, resourceList := range unstructuredList { + for _, resource := range resourceList.Items { + tblPrinter.AddRow(resource.GetKind(), resource.GetName()) + } + } + + tblPrinter.Print() + + fmt.Fprintln(o.Out, "\nKubeBlocks Namespaced RBAC:") + tblPrinter = printer.NewTablePrinter(o.Out) tblPrinter.SetHeader("NAMESPACE", "KIND", "NAME") - unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksRBAC, selectorList, allErrs) + unstructuredList = listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksNamespacedRBAC, selectorList, allErrs) for _, resourceList := range unstructuredList { for _, resource := range resourceList.Items { tblPrinter.AddRow(resource.GetNamespace(), resource.GetKind(), resource.GetName()) } } + tblPrinter.Print() } @@ -287,13 +312,17 @@ func (o *statusOptions) showHelmResources(ctx context.Context, allErrs *[]error) tblPrinter := printer.NewTablePrinter(o.Out) tblPrinter.SetHeader("NAMESPACE", "KIND", "NAME", "STATUS") - helmLabel := func(name string) string { - return fmt.Sprintf("%s=%s,%s=%s", "name", name, "owner", "helm") + helmLabel := func(name []string) string { + return fmt.Sprintf("%s in (%s),%s=%s", "name", strings.Join(name, ","), "owner", "helm") } - selectors := []metav1.ListOptions{{LabelSelector: types.KubeBlocksHelmLabel}} + // init helm release list with 'kubeblocks' + helmReleaseList := []string{types.KubeBlocksChartName} + // add add one names name = $kubeblocks-addons$ for _, addon := range o.addons { - selectors = append(selectors, metav1.ListOptions{LabelSelector: helmLabel(util.BuildAddonReleaseName(addon.Name))}) + helmReleaseList = append(helmReleaseList, util.BuildAddonReleaseName(addon.Name)) } + // label selector 'owner=helm,name in (kubeblocks,kb-addon-mongodb,kb-addon-redis...)' + selectors := []metav1.ListOptions{{LabelSelector: helmLabel(helmReleaseList)}} unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, helmConfigurations, selectors, allErrs) for _, resourceList := range unstructuredList { for _, resource := range resourceList.Items { @@ -383,8 +412,8 @@ func computeMetricByWorkloads(ctx context.Context, ns string, workloads []*unstr for _, resource := range workload.Items { name := resource.GetName() if podsMetrics == nil { - cpuMetricMap[name] = "N/A" - memMetricMap[name] = "N/A" + cpuMetricMap[name] = notAvailable + memMetricMap[name] = notAvailable continue } computeResources(name, podsMetrics) diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index 4074d40d0..2561b5dc6 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -110,7 +110,9 @@ func (o *UninstallOptions) PreCheck() error { // check if there is any resource should be removed first, if so, return error // and ask user to remove them manually if err := checkResources(o.Dynamic); err != nil { - return err + if !apierrors.IsNotFound(err) { + return err + } } // verify where kubeblocks is installed @@ -206,7 +208,7 @@ func (o *UninstallOptions) Uninstall() error { if o.Wait { fmt.Fprintln(o.Out, "Uninstall KubeBlocks done.") } else { - fmt.Fprintf(o.Out, "KubeBlocks is uninstalling, run \"kbcli kubeblocks status\" to check status.\n") + fmt.Fprintf(o.Out, "KubeBlocks is uninstalling, run \"kbcli kubeblocks status -A\" to check kubeblocks resources.\n") } return nil } diff --git a/internal/constant/const.go b/internal/constant/const.go index e59c44dbd..d6f4254a9 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -84,6 +84,7 @@ const ( ClassProviderLabelKey = "class.kubeblocks.io/provider" BackupToolTypeLabelKey = "kubeblocks.io/backup-tool-type" BackupTypeLabelKeyKey = "dataprotection.kubeblocks.io/backup-type" + AddonProviderLableKey = "kubeblocks.io/provider" // AddonProviderLableKey marks the addon provider // kubeblocks.io annotations OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster From 5db3fd8e746c695392a204ee1f249d8e593a6888 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 8 May 2023 19:24:43 +0800 Subject: [PATCH 245/439] chore: update pg12 config template (#3069) --- .../config/pg12-config-constraint.cue | 4 +- deploy/postgresql/config/pg12-config.tpl | 337 ++++++++++++++---- deploy/postgresql/config/pg14-config.tpl | 12 +- 3 files changed, 271 insertions(+), 82 deletions(-) diff --git a/deploy/postgresql/config/pg12-config-constraint.cue b/deploy/postgresql/config/pg12-config-constraint.cue index bbe702eab..26ddd9f20 100644 --- a/deploy/postgresql/config/pg12-config-constraint.cue +++ b/deploy/postgresql/config/pg12-config-constraint.cue @@ -580,7 +580,7 @@ max_wal_senders: int & >=5 & <=8388607 | *20 // (MB) Sets the WAL size that triggers a checkpoint. - max_wal_size: int & >=128 & <=201326592 | *2048 @storeResource(1MB) + max_wal_size: int & >=2 & <=2147483647 | *2048 @storeResource(1MB) // Sets the maximum number of concurrent worker processes. max_worker_processes?: int & >=0 & <=262143 @@ -598,7 +598,7 @@ min_parallel_table_scan_size?: int & >=0 & <=715827882 @storeResource(8KB) // (MB) Sets the minimum size to shrink the WAL to. - min_wal_size: int & >=128 & <=201326592 | *192 @storeResource(1MB) + min_wal_size: int & >=2 & <=2147483647 | *192 @storeResource(1MB) // (min) Time before a snapshot is too old to read pages changed after the snapshot was taken. old_snapshot_threshold?: int & >=-1 & <=86400 @timeDurationResource(1min) diff --git a/deploy/postgresql/config/pg12-config.tpl b/deploy/postgresql/config/pg12-config.tpl index e173be8f4..70fd87862 100644 --- a/deploy/postgresql/config/pg12-config.tpl +++ b/deploy/postgresql/config/pg12-config.tpl @@ -3,61 +3,150 @@ {{- $buffer_unit := "B" }} {{- $shared_buffers := 1073741824 }} {{- $max_connections := 10000 }} +{{- $autovacuum_max_workers := 3 }} {{- $phy_memory := getContainerMemory ( index $.podSpec.containers 0 ) }} +{{- $phy_cpu := getContainerCPU ( index $.podSpec.containers 0 ) }} {{- if gt $phy_memory 0 }} {{- $shared_buffers = div $phy_memory 4 }} {{- $max_connections = min ( div $phy_memory 9531392 ) 5000 }} -{{- end -}} +{{- $autovacuum_max_workers = min ( max ( div $phy_memory 17179869184 ) 3 ) 10 }} +{{- end }} {{- if ge $shared_buffers 1024 }} {{- $shared_buffers = div $shared_buffers 1024 }} -{{- $buffer_unit = "KB" }} -{{- end -}} +{{- $buffer_unit = "kB" }} +{{- end }} {{- if ge $shared_buffers 1024 }} {{- $shared_buffers = div $shared_buffers 1024 }} {{- $buffer_unit = "MB" }} -{{- end -}} +{{- end }} {{- if ge $shared_buffers 1024 }} {{- $shared_buffers = div $shared_buffers 1024 }} {{ $buffer_unit = "GB" }} -{{- end -}} +{{- end }} listen_addresses = '*' port = '5432' # archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' archive_command = '[[ $(date +%H%M) == 1200 ]] && rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' archive_mode = 'on' -auto_explain.log_analyze = 'True' -auto_explain.log_min_duration = '1s' -auto_explain.log_nested_statements = 'True' +auto_explain.log_analyze = 'False' +auto_explain.log_buffers = 'False' +auto_explain.log_format = 'text' +auto_explain.log_min_duration = '-1' +auto_explain.log_nested_statements = 'False' auto_explain.log_timing = 'True' -auto_explain.log_verbose = 'True' -autovacuum_analyze_scale_factor = '0.05' -autovacuum_freeze_max_age = '100000000' -autovacuum_max_workers = '1' -autovacuum_naptime = '1min' -autovacuum_vacuum_cost_delay = '-1' -autovacuum_vacuum_cost_limit = '-1' -autovacuum_vacuum_scale_factor = '0.1' -bgwriter_delay = '10ms' -bgwriter_lru_maxpages = '800' -bgwriter_lru_multiplier = '5.0' -checkpoint_completion_target = '0.95' -checkpoint_timeout = '10min' -commit_delay = '20' -commit_siblings = '10' -deadlock_timeout = '50ms' -default_statistics_target = '500' -effective_cache_size = '12GB' -hot_standby = 'on' -hot_standby_feedback = 'True' +auto_explain.log_triggers = 'False' +auto_explain.log_verbose = 'False' +auto_explain.sample_rate = '1' +autovacuum_analyze_scale_factor = '0.1' +autovacuum_analyze_threshold = '50' +autovacuum_freeze_max_age = '200000000' +autovacuum_max_workers = '{{ $autovacuum_max_workers }}' +autovacuum_multixact_freeze_max_age = '400000000' +autovacuum_naptime = '15s' +autovacuum_vacuum_cost_delay = '2' +autovacuum_vacuum_cost_limit = '200' +autovacuum_vacuum_scale_factor = '0.05' +autovacuum_vacuum_threshold = '50' +{{- if gt $phy_memory 0 }} +autovacuum_work_mem = '{{ printf "%dkB" ( max ( div $phy_memory 65536 ) 131072 ) }}' +{{- end }} +backend_flush_after = '0' +backslash_quote = 'safe_encoding' +bgwriter_delay = '200ms' +bgwriter_flush_after = '64' +bgwriter_lru_maxpages = '1000' +bgwriter_lru_multiplier = '10.0' +bytea_output = 'hex' +check_function_bodies = 'True' +checkpoint_completion_target = '0.4' +checkpoint_flush_after = '32' +checkpoint_timeout = '25min' +checkpoint_warning = '30s' +client_min_messages = 'notice' +# commit_delay = '20' +commit_siblings = '5' +constraint_exclusion = 'partition' + +#extension: pg_cron +cron.database_name = 'postgres' +cron.log_statement = 'on' +cron.max_running_jobs = '32' + +cursor_tuple_fraction = '0.1' +datestyle = 'ISO,YMD' +deadlock_timeout = '1000ms' +debug_pretty_print = 'True' +debug_print_parse = 'False' +debug_print_plan = 'False' +debug_print_rewritten = 'False' +default_statistics_target = '100' +default_transaction_deferrable = 'False' +default_transaction_isolation = 'read committed' +# unit 8KB +{{- if gt $phy_memory 0 }} +effective_cache_size = '{{ printf "%dMB" ( div ( div $phy_memory 16384 ) 128 ) }}' +{{- end }} +effective_io_concurrency = '1' +enable_bitmapscan = 'True' +enable_gathermerge = 'True' +enable_hashagg = 'True' +enable_hashjoin = 'True' +enable_indexonlyscan = 'True' +enable_indexscan = 'True' +enable_material = 'True' +enable_mergejoin = 'True' +enable_nestloop = 'True' +enable_parallel_append = 'True' +enable_parallel_hash = 'True' +enable_partition_pruning = 'True' +# patroni off +enable_partitionwise_aggregate = 'True' +# patroni off +enable_partitionwise_join = 'True' +enable_seqscan = 'True' +enable_sort = 'True' +enable_tidscan = 'True' +escape_string_warning = 'True' +extra_float_digits = '1' +force_parallel_mode = '0' +from_collapse_limit = '8' +#fsync=off # patroni for Extreme Performance +#full_page_writes=off # patroni for Extreme Performance +geqo = 'True' +geqo_effort = '5' +geqo_generations = '0' +geqo_pool_size = '0' +geqo_seed = '0' +geqo_selection_bias = '2' +geqo_threshold = '12' +gin_fuzzy_search_limit = '0' +gin_pending_list_limit = '4096kB' +# patroni on +hot_standby_feedback = 'False' +# rds huge_pages=on, patroni try huge_pages = 'try' -idle_in_transaction_session_timeout = '1h' -listen_addresses = '0.0.0.0' -log_autovacuum_min_duration = '1s' +#patroni 10min +idle_in_transaction_session_timeout = '3600000ms' +index_adviser.enable_log = 'on' +index_adviser.max_aggregation_column_count = '10' +index_adviser.max_candidate_index_count = '500' +intervalstyle = 'postgres' +join_collapse_limit = '8' +lc_monetary = 'C' +lc_numeric = 'C' +lc_time = 'C' +lock_timeout = '0' +# patroni 1s +log_autovacuum_min_duration = '10000' log_checkpoints = 'True' +log_connections = 'False' +log_disconnections = 'False' +log_duration = 'False' +log_executor_stats = 'False' {{- block "logsBlock" . }} {{- if hasKey $.component "enabledLogs" }} @@ -70,65 +159,165 @@ log_filename = 'postgresql-%Y-%m-%d.log' {{ end -}} {{ end }} -log_lock_waits = 'True' -log_min_duration_statement = '100' -log_replication_commands = 'True' +# log_lock_waits = 'True' +log_min_duration_statement = '1000' +log_parser_stats = 'False' +log_planner_stats = 'False' +log_replication_commands = 'False' log_statement = 'ddl' +log_statement_stats = 'False' +log_temp_files = '128kB' +log_transaction_sample_rate = '0' #maintenance_work_mem = '3952MB' max_connections = '{{ $max_connections }}' -max_locks_per_transaction = '128' -max_logical_replication_workers = '8' -max_parallel_maintenance_workers = '2' -max_parallel_workers = '8' -max_parallel_workers_per_gather = '0' -max_prepared_transactions = '0' +max_files_per_process = '1000' +max_logical_replication_workers = '32' +max_locks_per_transaction = '64' +max_parallel_maintenance_workers = '{{ max ( div $phy_cpu 2 ) 2 }}' +max_parallel_workers = '{{ max ( div ( mul $phy_cpu 3 ) 4 ) 8 }}' +max_parallel_workers_per_gather = '{{ max ( div $phy_cpu 2 ) 2 }}' +max_pred_locks_per_page = '2' +max_pred_locks_per_relation = '-2' +max_pred_locks_per_transaction = '64' +max_prepared_transactions = '100' max_replication_slots = '16' -max_standby_archive_delay = '10min' -max_standby_streaming_delay = '3min' -max_sync_workers_per_subscription = '6' -max_wal_senders = '24' -max_wal_size = '100GB' -max_worker_processes = '8' -min_wal_size = '20GB' +max_stack_depth = '2MB' + +max_standby_archive_delay = '300000ms' +max_standby_streaming_delay = '300000ms' +max_sync_workers_per_subscription = '2' +max_wal_senders = '64' +# {LEAST(GREATEST(DBInstanceClassMemory/2097152, 2048), 16384)} +max_wal_size = '{{ printf "%dMB" ( min ( max ( div $phy_memory 2097152 ) 2048 ) 16384 ) }}' +max_worker_processes = '{{ max $phy_cpu 8 }}' +# min_parallel_index_scan_size unit is 8KB, 64 = 512KB +min_parallel_index_scan_size = '512kB' +# min_parallel_table_scan_size unit is 8KB, 1024 = 8MB +min_parallel_table_scan_size = '8MB' +{{- if gt $phy_memory 0 }} +# min_wal_size={LEAST(GREATEST(DBInstanceClassMemory/8388608, 256), 8192)} # patroni 1/20 disk size +min_wal_size = '{{ printf "%dMB" ( min ( max ( div $phy_memory 8388608 ) 256 ) 8192 ) }}' +{{- end }} + +old_snapshot_threshold = '-1' +operator_precedence_warning = 'off' +parallel_leader_participation = 'True' + password_encryption = 'md5' pg_stat_statements.max = '5000' -pg_stat_statements.track = 'all' -pg_stat_statements.track_planning = 'False' +pg_stat_statements.save = 'False' + +# patroni all +pg_stat_statements.track = 'top' +# pg_stat_statements.track_planning = 'False' pg_stat_statements.track_utility = 'False' + +#extension: pgaudit +pgaudit.log_catalog = 'True' +pgaudit.log_level = 'log' +pgaudit.log_parameter = 'False' +pgaudit.log_relation = 'False' +pgaudit.log_statement_once = 'False' +# TODO +# pgaudit.role = '' + +#extension: pglogical +pglogical.batch_inserts = 'True' +pglogical.conflict_log_level = 'log' +pglogical.conflict_resolution = 'apply_remote' +# TODO +# pglogical.extra_connection_options = '' +pglogical.synchronous_commit = 'False' +pglogical.use_spi = 'False' +plan_cache_mode = 'auto' +quote_all_identifiers = 'False' + random_page_cost = '1.1' +row_security = 'True' +session_replication_role = 'origin' + +#extension: sql_firewall +sql_firewall.firewall = 'disable' + #auto generated shared_buffers = '{{ printf "%d%s" $shared_buffers $buffer_unit }}' # shared_preload_libraries = 'pg_stat_statements,auto_explain,bg_mon,pgextwlist,pg_auth_mon,set_user,pg_cron,pg_stat_kcache' -superuser_reserved_connections = '10' -temp_file_limit = '100GB' -#timescaledb.max_background_workers = '6' -#timescaledb.telemetry_level = 'off' -track_activity_query_size = '8192' -track_commit_timestamp = 'True' -track_functions = 'all' -track_io_timing = 'True' -vacuum_cost_delay = '2ms' -vacuum_cost_limit = '10000' -vacuum_defer_cleanup_age = '50000' -wal_buffers = '16MB' -wal_level = 'replica' -wal_log_hints = 'on' -wal_receiver_status_interval = '1s' -wal_receiver_timeout = '60s' -wal_writer_delay = '20ms' -wal_writer_flush_after = '1MB' -work_mem = '32MB' {{- if $.component.tls }} {{- $ca_file := getCAFile }} {{- $cert_file := getCertFile }} {{- $key_file := getKeyFile }} # tls -ssl=ON -ssl_ca_file={{ $ca_file }} -ssl_cert_file={{ $cert_file }} -ssl_key_file={{ $key_file }} +ssl = 'True' +ssl_ca_file = '{{ $ca_file }}' +ssl_cert_file = '{{ $cert_file }}' +ssl_key_file = '{{ $key_file }}' {{- end }} -# TODO: check the following parameters, how to set the default value -# wal_keep_segments=128 \ No newline at end of file +# ssl_max_protocol_version='' +ssl_min_protocol_version = 'TLSv1' +standard_conforming_strings = 'True' +statement_timeout = '0' +#patroni 10 +superuser_reserved_connections = '20' +synchronize_seqscans = 'True' + +# rds off ，patroni off for Extreme Performance +synchronous_commit = 'off' +# synchronous_standby_names='' +tcp_keepalives_count = '10' +tcp_keepalives_idle = '45s' +tcp_keepalives_interval = '10s' +temp_buffers = '8MB' + +# {DBInstanceClassMemory/1024} +{{- if gt $phy_memory 0 }} +temp_file_limit = '{{ printf "%dkB" ( div $phy_memory 1024 ) }}' +{{- end }} + +#extension: timescaledb +#timescaledb.max_background_workers = '6' +#timescaledb.telemetry_level = 'off' +# TODO timezone +# timezone=Asia/Shanghai +track_activity_query_size = '4096' +track_commit_timestamp = 'False' +track_functions = 'pl' +track_io_timing = 'True' +transform_null_equals = 'False' + + +vacuum_cleanup_index_scale_factor = '0.1' +# patroni 20ms +vacuum_cost_delay = '0' +# patroni 2000 +vacuum_cost_limit = '10000' +vacuum_cost_page_dirty = '20' +vacuum_cost_page_hit = '1' +vacuum_cost_page_miss = '2' +# patroni 50000 +vacuum_defer_cleanup_age = '0' +vacuum_freeze_min_age = '50000000' +vacuum_freeze_table_age = '200000000' +vacuum_multixact_freeze_min_age = '5000000' +vacuum_multixact_freeze_table_age = '200000000' +# wal_buffers ={LEAST(GREATEST(DBInstanceClassMemory/2097152, 2048), 16384)} # patroni 16M +# unit 8KB +wal_buffers = '{{ printf "%dMB" ( div ( min ( max ( div $phy_memory 2097152 ) 2048) 16384 ) 128 ) }}' +wal_compression = 'True' +wal_keep_segments = '128' +# patroni minimal for Extreme Performance +wal_level = 'replica' +# patroni on , off for Extreme Performance +wal_log_hints = 'False' +wal_receiver_status_interval = '1s' +wal_receiver_timeout = '60000' +wal_sender_timeout = '60000' +# patroni 20ms +wal_writer_delay = '200ms' +# rds unit 8KB, so 1M, patroni 1M +wal_writer_flush_after = '1MB' +# {GREATEST(DBInstanceClassMemory/4194304, 4096)} +work_mem = '{{ printf "%dkB" ( max ( div $phy_memory 4194304 ) 4096 ) }}' +xmlbinary = 'base64' +xmloption = 'content' \ No newline at end of file diff --git a/deploy/postgresql/config/pg14-config.tpl b/deploy/postgresql/config/pg14-config.tpl index 5eff02923..d56c52d18 100644 --- a/deploy/postgresql/config/pg14-config.tpl +++ b/deploy/postgresql/config/pg14-config.tpl @@ -7,22 +7,22 @@ {{- if gt $phy_memory 0 }} {{- $shared_buffers = div $phy_memory 4 }} {{- $max_connections = min ( div $phy_memory 9531392 ) 5000 }} -{{- end -}} +{{- end }} {{- if ge $shared_buffers 1024 }} {{- $shared_buffers = div $shared_buffers 1024 }} -{{- $buffer_unit = "KB" }} -{{- end -}} +{{- $buffer_unit = "kB" }} +{{- end }} {{- if ge $shared_buffers 1024 }} {{- $shared_buffers = div $shared_buffers 1024 }} {{- $buffer_unit = "MB" }} -{{- end -}} +{{- end }} {{- if ge $shared_buffers 1024 }} {{- $shared_buffers = div $shared_buffers 1024 }} {{ $buffer_unit = "GB" }} -{{- end -}} +{{- end }} listen_addresses = '*' port = '5432' @@ -125,7 +125,7 @@ work_mem = '32MB' {{- $cert_file := getCertFile }} {{- $key_file := getKeyFile }} # tls -ssl=ON +ssl= 'True' ssl_ca_file={{ $ca_file }} ssl_cert_file={{ $cert_file }} ssl_key_file={{ $key_file }} From a7bec0071caf80a0866d1318543f6ba312aa94ae Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 8 May 2023 19:25:34 +0800 Subject: [PATCH 246/439] chore: update pg12 config template (#3069) From fd7742939e2528f38f8f563cbef3527b3c535325 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 8 May 2023 19:26:33 +0800 Subject: [PATCH 247/439] chore: update pg12 config template (#3069) From 8b24b6662bbd65e99e12ed0aac3e62f1831c8d8e Mon Sep 17 00:00:00 2001 From: chantu Date: Mon, 8 May 2023 22:19:53 +0800 Subject: [PATCH 248/439] fix: drop followers only when scaling in (#3130) --- cmd/manager/main.go | 9 -- controllers/apps/cluster_controller_test.go | 17 ++- controllers/apps/components/component.go | 33 +++++ .../apps/components/deployment_controller.go | 6 + controllers/apps/components/pod_controller.go | 124 ------------------ .../components/stateful_set_controller.go | 5 + .../templates/clusterdefinition.yaml | 3 + deploy/apecloud-mysql/templates/scripts.yaml | 32 ++++- internal/constant/const.go | 1 + 9 files changed, 89 insertions(+), 141 deletions(-) delete mode 100644 controllers/apps/components/pod_controller.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 43b206de9..529952614 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -393,15 +393,6 @@ func main() { os.Exit(1) } - if err = (&components.PodReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("pod-controller"), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Pod") - os.Exit(1) - } - if err = (&appscontrollers.ComponentClassReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 0f563904e..640ca2417 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -28,6 +28,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/viper" @@ -411,6 +413,7 @@ var _ = Describe("Cluster Controller", func() { Name: stsName + "-" + strconv.Itoa(i), Namespace: testCtx.DefaultNamespace, Labels: map[string]string{ + constant.AppManagedByLabelKey: constant.AppName, constant.AppInstanceLabelKey: clusterName, constant.KBAppComponentLabelKey: componentName, appsv1.ControllerRevisionHashLabelKey: "mock-version", @@ -942,9 +945,10 @@ var _ = Describe("Cluster Controller", func() { sts = &stsList.Items[0] }).Should(Succeed()) - By("Creating mock pods in StatefulSet") + By("Creating mock pods in StatefulSet, and set controller reference") pods := mockPodsForConsensusTest(clusterObj, replicas) for _, pod := range pods { + Expect(controllerutil.SetControllerReference(sts, &pod, scheme.Scheme)).Should(Succeed()) Expect(testCtx.CreateObj(testCtx.Ctx, &pod)).Should(Succeed()) // mock the status to pass the isReady(pod) check in consensus_set pod.Status.Conditions = []corev1.PodCondition{{ @@ -982,6 +986,17 @@ var _ = Describe("Cluster Controller", func() { g.Expect(followerCount).Should(Equal(2)) }).Should(Succeed()) + By("Checking pods' annotations") + Eventually(func(g Gomega) { + pods, err := util.GetPodListByStatefulSet(ctx, k8sClient, sts) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(len(pods)).Should(Equal(int(*sts.Spec.Replicas))) + for _, pod := range pods { + g.Expect(pod.Annotations).ShouldNot(BeNil()) + g.Expect(pod.Annotations[constant.ComponentReplicasAnnotationKey]).Should(Equal(strconv.Itoa(int(*sts.Spec.Replicas)))) + } + }, time.Second*100).Should(Succeed()) + By("Updating StatefulSet's status") sts.Status.UpdateRevision = "mock-version" sts.Status.Replicas = int32(replicas) diff --git a/controllers/apps/components/component.go b/controllers/apps/components/component.go index d5250240d..98f3d5a8b 100644 --- a/controllers/apps/components/component.go +++ b/controllers/apps/components/component.go @@ -21,9 +21,11 @@ package components import ( "context" + "strconv" "time" "golang.org/x/exp/slices" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -184,3 +186,34 @@ func patchWorkloadCustomLabel( } return nil } + +func updateComponentInfoToPods( + ctx context.Context, + cli client.Client, + cluster *appsv1alpha1.Cluster, + componentSpec *appsv1alpha1.ClusterComponentSpec) error { + ml := client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.GetName(), + constant.KBAppComponentLabelKey: componentSpec.Name, + } + podList := corev1.PodList{} + if err := cli.List(ctx, &podList, ml); err != nil { + return err + } + replicasStr := strconv.Itoa(int(componentSpec.Replicas)) + for _, pod := range podList.Items { + if pod.Annotations != nil && + pod.Annotations[constant.ComponentReplicasAnnotationKey] == replicasStr { + continue + } + patch := client.MergeFrom(pod.DeepCopy()) + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + pod.Annotations[constant.ComponentReplicasAnnotationKey] = replicasStr + if err := cli.Patch(ctx, &pod, patch); err != nil { + return err + } + } + return nil +} diff --git a/controllers/apps/components/deployment_controller.go b/controllers/apps/components/deployment_controller.go index 8545e4857..6a19044ab 100644 --- a/controllers/apps/components/deployment_controller.go +++ b/controllers/apps/components/deployment_controller.go @@ -76,6 +76,11 @@ func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) return workloadCompClusterReconcile(reqCtx, r.Client, deploy, func(cluster *appsv1alpha1.Cluster, componentSpec *appsv1alpha1.ClusterComponentSpec, component types.Component) (ctrl.Result, error) { compCtx := newComponentContext(reqCtx, r.Client, r.Recorder, component, deploy, componentSpec) + // update component info to pods' annotations + if err := updateComponentInfoToPods(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { + reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "StatefulSet Deploy updateComponentInfoToPods Failed", err.Error()) + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } // patch the current componentSpec workload's custom labels if err := patchWorkloadCustomLabel(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "Deployment Controller PatchWorkloadCustomLabelFailed", err.Error()) @@ -96,6 +101,7 @@ func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&appsv1.Deployment{}). Owns(&appsv1.ReplicaSet{}). + Owns(&corev1.Pod{}). WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). Named("deployment-watcher"). Complete(r) diff --git a/controllers/apps/components/pod_controller.go b/controllers/apps/components/pod_controller.go deleted file mode 100644 index 6d62e2196..000000000 --- a/controllers/apps/components/pod_controller.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package components - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/util" - "github.com/apecloud/kubeblocks/internal/constant" - intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" -) - -// PodReconciler reconciles a Pod object -type PodReconciler struct { - client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder -} - -// +kubebuilder:rbac:groups=apps,resources=pods,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps,resources=pods/status,verbs=get -// +kubebuilder:rbac:groups=apps,resources=pods/finalizers,verbs=update - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile -func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - var ( - pod = &corev1.Pod{} - err error - cluster *appsv1alpha1.Cluster - ok bool - componentName string - componentStatus appsv1alpha1.ClusterComponentStatus - ) - - reqCtx := intctrlutil.RequestCtx{ - Ctx: ctx, - Req: req, - Log: log.FromContext(ctx).WithValues("pod", req.NamespacedName), - } - - if err = r.Client.Get(reqCtx.Ctx, reqCtx.Req.NamespacedName, pod); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - - // skip if pod is being deleted - if !pod.DeletionTimestamp.IsZero() { - return intctrlutil.Reconciled() - } - - if cluster, err = util.GetClusterByObject(reqCtx.Ctx, r.Client, pod); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - if cluster == nil { - return intctrlutil.Reconciled() - } - - if componentName, ok = pod.Labels[constant.KBAppComponentLabelKey]; !ok { - return intctrlutil.Reconciled() - } - - if cluster.Status.Components == nil { - return intctrlutil.Reconciled() - } - if componentStatus, ok = cluster.Status.Components[componentName]; !ok { - return intctrlutil.Reconciled() - } - if componentStatus.ConsensusSetStatus == nil { - return intctrlutil.Reconciled() - } - if componentStatus.ConsensusSetStatus.Leader.Pod == util.ComponentStatusDefaultPodName { - return intctrlutil.Reconciled() - } - - // sync leader status from cluster.status - patch := client.MergeFrom(pod.DeepCopy()) - if pod.Annotations == nil { - pod.Annotations = make(map[string]string) - } - pod.Annotations[constant.LeaderAnnotationKey] = componentStatus.ConsensusSetStatus.Leader.Pod - if err = r.Client.Patch(reqCtx.Ctx, pod, patch); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - r.Recorder.Eventf(pod, corev1.EventTypeNormal, "AddAnnotation", "add annotation %s=%s", constant.LeaderAnnotationKey, componentStatus.ConsensusSetStatus.Leader.Pod) - return intctrlutil.Reconciled() -} - -// SetupWithManager sets up the controller with the Manager. -func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Pod{}). - WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). - Named("pod-watcher"). - Complete(r) -} diff --git a/controllers/apps/components/stateful_set_controller.go b/controllers/apps/components/stateful_set_controller.go index 40a246f90..9c8d697c9 100644 --- a/controllers/apps/components/stateful_set_controller.go +++ b/controllers/apps/components/stateful_set_controller.go @@ -76,6 +76,11 @@ func (r *StatefulSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) return workloadCompClusterReconcile(reqCtx, r.Client, sts, func(cluster *appsv1alpha1.Cluster, componentSpec *appsv1alpha1.ClusterComponentSpec, component types.Component) (ctrl.Result, error) { compCtx := newComponentContext(reqCtx, r.Client, r.Recorder, component, sts, componentSpec) + // update component info to pods' annotations + if err := updateComponentInfoToPods(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { + reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "StatefulSet Controller updateComponentInfoToPods Failed", err.Error()) + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } // patch the current componentSpec workload's custom labels if err := patchWorkloadCustomLabel(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "StatefulSet Controller PatchWorkloadCustomLabelFailed", err.Error()) diff --git a/deploy/apecloud-mysql/templates/clusterdefinition.yaml b/deploy/apecloud-mysql/templates/clusterdefinition.yaml index c64cf222b..d1843f9b5 100644 --- a/deploy/apecloud-mysql/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql/templates/clusterdefinition.yaml @@ -170,6 +170,9 @@ spec: - path: "leader" fieldRef: fieldPath: metadata.annotations['cs.apps.kubeblocks.io/leader'] + - path: "component-replicas" + fieldRef: + fieldPath: metadata.annotations['apps.kubeblocks.io/component-replicas'] systemAccounts: cmdExecutorConfig: image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index 62669181f..3019304e5 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -15,6 +15,10 @@ data: idx=${KB_POD_NAME##*-} host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) echo "host=$host" + # update replicas to persistent file + component_replicas_path=/data/mysql/.kb_component_replicas + current_component_replicas=`cat /etc/annotations/component-replicas` + echo $current_component_replicas > $component_replicas_path if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then echo "no leader or self is leader, no need to call add." else @@ -70,7 +74,7 @@ data: mkdir -p /data/mysql/data /data/mysql/log chmod +777 -R /data/mysql; echo "KB_MYSQL_CLUSTER_UID=$KB_MYSQL_CLUSTER_UID" - cluster_uid_path=/data/mysql/data/.kb_cluster_uid + cluster_uid_path=/data/mysql/.kb_cluster_uid if [ -f $cluster_uid_path ] && [ ! -f /data/mysql/data/.restore_new_cluster ]; then last_cluster_uid=`cat $cluster_uid_path` if [ "$last_cluster_uid" != "$KB_MYSQL_CLUSTER_UID" ]; then @@ -155,16 +159,17 @@ data: done pre-stop.sh: | #!/bin/bash + drop_followers() { leader=`cat /etc/annotations/leader` - echo "leader=$leader" - echo "KB_POD_NAME=$KB_POD_NAME" + echo "leader=$leader" >> /data/mysql/.kb_pre_stop.log + echo "KB_POD_NAME=$KB_POD_NAME" >> /data/mysql/.kb_pre_stop.log if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then - echo "no leader or self is leader, exit" + echo "no leader or self is leader, exit" >> /data/mysql/.kb_pre_stop.log exit 0 fi idx=${KB_POD_NAME##*-} host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) - echo "host=$host" + echo "host=$host" >> /data/mysql/.kb_pre_stop.log leader_idx=${leader##*-} leader_host=$(eval echo \$KB_MYSQL_"$leader_idx"_HOSTNAME) if [ ! -z $leader_host ]; then @@ -173,7 +178,20 @@ data: if [ ! -z $MYSQL_ROOT_PASSWORD ]; then password_flag="-p$MYSQL_ROOT_PASSWORD" fi - echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.downgrade_follower('$host:13306');\" 2>&1 " + echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.downgrade_follower('$host:13306');\" 2>&1 " >> /data/mysql/.kb_pre_stop.log mysql $host_flag -uroot $password_flag -e "call dbms_consensus.downgrade_follower('$host:13306');" 2>&1 - echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.drop_learner('$host:13306');\" 2>&1 " + echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.drop_learner('$host:13306');\" 2>&1 " >> /data/mysql/.kb_pre_stop.log mysql $host_flag -uroot $password_flag -e "call dbms_consensus.drop_learner('$host:13306');" 2>&1 + } + component_replicas_path=/data/mysql/.kb_component_replicas + current_component_replicas=`cat /etc/annotations/component-replicas` + if [ -f $component_replicas_path ]; then + last_component_replicas=`cat $component_replicas_path` + # check is scaling in but not scaling in to 0 + if [ "$last_component_replicas" -gt "$current_component_replicas" ] && [ $current_component_replicas -ne 0 ]; then + # only scaling in need to drop followers + drop_followers + else + echo "no need to drop followers" + fi + fi diff --git a/internal/constant/const.go b/internal/constant/const.go index d6f4254a9..aebdd9a67 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -94,6 +94,7 @@ const ( RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" + ComponentReplicasAnnotationKey = "apps.kubeblocks.io/component-replicas" // ComponentReplicasAnnotationKey specifies the number of pods in replicas DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" // DefaultBackupPolicyAnnotationKey specifies the default backup policy. DefaultBackupPolicyTemplateAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy-template" // DefaultBackupPolicyTemplateAnnotationKey specifies the default backup policy template. BackupDataPathPrefixAnnotationKey = "dataprotection.kubeblocks.io/path-prefix" // BackupDataPathPrefixAnnotationKey specifies the backup data path prefix. From a8a4255495e47a06cf1a842dbb0e895a8d1e6465 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Tue, 9 May 2023 09:10:29 +0800 Subject: [PATCH 249/439] chore: refactor cli create (#2917) --- docs/user_docs/cli/cli.md | 4 +- docs/user_docs/cli/kbcli_cluster.md | 4 +- docs/user_docs/cli/kbcli_cluster_backup.md | 12 +- docs/user_docs/cli/kbcli_cluster_hscale.md | 6 +- .../cli/kbcli_cluster_list-backup-policy.md | 2 +- .../cli/kbcli_cluster_reconfigure.md | 65 ++++ docs/user_docs/cli/kbcli_cluster_restart.md | 8 +- docs/user_docs/cli/kbcli_cluster_start.md | 4 +- docs/user_docs/cli/kbcli_cluster_stop.md | 4 +- docs/user_docs/cli/kbcli_cluster_upgrade.md | 4 +- .../cli/kbcli_cluster_volume-expand.md | 5 +- docs/user_docs/cli/kbcli_cluster_vscale.md | 10 +- docs/user_docs/cli/kbcli_migration_create.md | 2 +- internal/cli/cmd/cluster/cluster_test.go | 43 ++- internal/cli/cmd/cluster/config_edit.go | 40 +-- internal/cli/cmd/cluster/config_ops.go | 37 +- internal/cli/cmd/cluster/config_ops_test.go | 4 +- internal/cli/cmd/cluster/config_util.go | 6 +- internal/cli/cmd/cluster/config_util_test.go | 4 +- internal/cli/cmd/cluster/config_wrapper.go | 8 +- internal/cli/cmd/cluster/connect.go | 2 +- internal/cli/cmd/cluster/create.go | 93 ++--- internal/cli/cmd/cluster/dataprotection.go | 82 +++-- .../cli/cmd/cluster/dataprotection_test.go | 4 +- internal/cli/cmd/cluster/operations.go | 327 ++++++++++-------- internal/cli/cmd/cluster/operations_test.go | 8 +- internal/cli/cmd/migration/create.go | 65 ++-- internal/cli/cmd/playground/init.go | 75 ++-- internal/cli/create/create.go | 275 ++++----------- internal/cli/create/create_test.go | 191 +++------- internal/cli/delete/delete.go | 15 +- 31 files changed, 647 insertions(+), 762 deletions(-) create mode 100644 docs/user_docs/cli/kbcli_cluster_reconfigure.md diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index a034a0a77..978c35fe7 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -43,8 +43,7 @@ Manage classes Cluster command. -* [kbcli cluster backup](kbcli_cluster_backup.md) - Create a backup. -* [kbcli cluster configure](kbcli_cluster_configure.md) - Reconfigure parameters with the specified components in the cluster. +* [kbcli cluster backup](kbcli_cluster_backup.md) - Create a backup for the cluster. * [kbcli cluster connect](kbcli_cluster_connect.md) - Connect to a cluster or instance. * [kbcli cluster create](kbcli_cluster_create.md) - Create a cluster. * [kbcli cluster create-account](kbcli_cluster_create-account.md) - Create account for a cluster @@ -74,6 +73,7 @@ Cluster command. * [kbcli cluster list-logs](kbcli_cluster_list-logs.md) - List supported log files in cluster. * [kbcli cluster list-ops](kbcli_cluster_list-ops.md) - List all opsRequests. * [kbcli cluster logs](kbcli_cluster_logs.md) - Access cluster log file. +* [kbcli cluster reconfigure](kbcli_cluster_reconfigure.md) - Reconfigure parameters with the specified components in the cluster. * [kbcli cluster restart](kbcli_cluster_restart.md) - Restart the specified components in the cluster. * [kbcli cluster restore](kbcli_cluster_restore.md) - Restore a new cluster from backup. * [kbcli cluster revoke-role](kbcli_cluster_revoke-role.md) - Revoke role from account diff --git a/docs/user_docs/cli/kbcli_cluster.md b/docs/user_docs/cli/kbcli_cluster.md index 48a8cd37e..baa21fa03 100644 --- a/docs/user_docs/cli/kbcli_cluster.md +++ b/docs/user_docs/cli/kbcli_cluster.md @@ -37,8 +37,7 @@ Cluster command. ### SEE ALSO -* [kbcli cluster backup](kbcli_cluster_backup.md) - Create a backup. -* [kbcli cluster configure](kbcli_cluster_configure.md) - Reconfigure parameters with the specified components in the cluster. +* [kbcli cluster backup](kbcli_cluster_backup.md) - Create a backup for the cluster. * [kbcli cluster connect](kbcli_cluster_connect.md) - Connect to a cluster or instance. * [kbcli cluster create](kbcli_cluster_create.md) - Create a cluster. * [kbcli cluster create-account](kbcli_cluster_create-account.md) - Create account for a cluster @@ -68,6 +67,7 @@ Cluster command. * [kbcli cluster list-logs](kbcli_cluster_list-logs.md) - List supported log files in cluster. * [kbcli cluster list-ops](kbcli_cluster_list-ops.md) - List all opsRequests. * [kbcli cluster logs](kbcli_cluster_logs.md) - Access cluster log file. +* [kbcli cluster reconfigure](kbcli_cluster_reconfigure.md) - Reconfigure parameters with the specified components in the cluster. * [kbcli cluster restart](kbcli_cluster_restart.md) - Restart the specified components in the cluster. * [kbcli cluster restore](kbcli_cluster_restore.md) - Restore a new cluster from backup. * [kbcli cluster revoke-role](kbcli_cluster_revoke-role.md) - Revoke role from account diff --git a/docs/user_docs/cli/kbcli_cluster_backup.md b/docs/user_docs/cli/kbcli_cluster_backup.md index 4234d7bff..6b0f54864 100644 --- a/docs/user_docs/cli/kbcli_cluster_backup.md +++ b/docs/user_docs/cli/kbcli_cluster_backup.md @@ -2,26 +2,26 @@ title: kbcli cluster backup --- -Create a backup. +Create a backup for the cluster. ``` -kbcli cluster backup [flags] +kbcli cluster backup NAME [flags] ``` ### Examples ``` # create a backup - kbcli cluster backup cluster-name + kbcli cluster backup mycluster # create a snapshot backup - kbcli cluster backup cluster-name --backup-type snapshot + kbcli cluster backup mycluster --backup-type snapshot # create a full backup - kbcli cluster backup cluster-name --backup-type full + kbcli cluster backup mycluster --backup-type full # create a backup with specified backup policy - kbcli cluster backup cluster-name --backup-policy + kbcli cluster backup mycluster --backup-policy ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_hscale.md b/docs/user_docs/cli/kbcli_cluster_hscale.md index fd5f82147..22dcc0336 100644 --- a/docs/user_docs/cli/kbcli_cluster_hscale.md +++ b/docs/user_docs/cli/kbcli_cluster_hscale.md @@ -5,14 +5,14 @@ title: kbcli cluster hscale Horizontally scale the specified components in the cluster. ``` -kbcli cluster hscale [flags] +kbcli cluster hscale NAME [flags] ``` ### Examples ``` - # expand storage resources of specified components, separate with commas when more than one - kbcli cluster hscale --components= --replicas=3 + # expand storage resources of specified components, separate with commas when component name more than one + kbcli cluster hscale mycluster --components=mysql --replicas=3 ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md b/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md index 5379ea1f2..9ac4710da 100644 --- a/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md +++ b/docs/user_docs/cli/kbcli_cluster_list-backup-policy.md @@ -14,7 +14,7 @@ kbcli cluster list-backup-policy [flags] # list all backup policy kbcli cluster list-backup-policy - # using short cmd to list backup policy of specified cluster + # using short cmd to list backup policy of the specified cluster kbcli cluster list-bp mycluster ``` diff --git a/docs/user_docs/cli/kbcli_cluster_reconfigure.md b/docs/user_docs/cli/kbcli_cluster_reconfigure.md new file mode 100644 index 000000000..e706ed9e0 --- /dev/null +++ b/docs/user_docs/cli/kbcli_cluster_reconfigure.md @@ -0,0 +1,65 @@ +--- +title: kbcli cluster reconfigure +--- + +Reconfigure parameters with the specified components in the cluster. + +``` +kbcli cluster reconfigure NAME --set key=value[,key=value] [--component=component-name] [--config-spec=config-spec-name] [--config-file=config-file] [flags] +``` + +### Examples + +``` + # update component params + kbcli cluster configure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=1000,general_log=OFF + + # if only one component, and one config spec, and one config file, simplify the use of configure. e.g: + # update mysql max_connections, cluster name is mycluster + kbcli cluster configure mycluster --set max_connections=2000 +``` + +### Options + +``` + --component string Specify the name of Component to be updated. If the cluster has only one component, unset the parameter. + --config-file string Specify the name of the configuration file to be updated (e.g. for mysql: --config-file=my.cnf). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. + --config-spec string Specify the name of the configuration template to be updated (e.g. for apecloud-mysql: --config-spec=mysql-3node-tpl). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. + --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + -h, --help help for reconfigure + --name string OpsRequest name. if not specified, it will be randomly generated + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --set strings Specify updated parameter list. For details about the parameters, refer to kbcli sub command: 'kbcli cluster describe-config'. + --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli cluster](kbcli_cluster.md) - Cluster command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_cluster_restart.md b/docs/user_docs/cli/kbcli_cluster_restart.md index 76f10b971..690f3e6f9 100644 --- a/docs/user_docs/cli/kbcli_cluster_restart.md +++ b/docs/user_docs/cli/kbcli_cluster_restart.md @@ -5,17 +5,17 @@ title: kbcli cluster restart Restart the specified components in the cluster. ``` -kbcli cluster restart [flags] +kbcli cluster restart NAME [flags] ``` ### Examples ``` # restart all components - kbcli cluster restart + kbcli cluster restart mycluster - # restart specifies the component, separate with commas when more than one - kbcli cluster restart --components= + # restart specifies the component, separate with commas when component more than one + kbcli cluster restart mycluster --components=mysql ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_start.md b/docs/user_docs/cli/kbcli_cluster_start.md index 215f5dae2..e7cfef57d 100644 --- a/docs/user_docs/cli/kbcli_cluster_start.md +++ b/docs/user_docs/cli/kbcli_cluster_start.md @@ -5,14 +5,14 @@ title: kbcli cluster start Start the cluster if cluster is stopped. ``` -kbcli cluster start [flags] +kbcli cluster start NAME [flags] ``` ### Examples ``` # start the cluster when cluster is stopped - kbcli cluster start + kbcli cluster start mycluster ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_stop.md b/docs/user_docs/cli/kbcli_cluster_stop.md index 911721b01..a1accbced 100644 --- a/docs/user_docs/cli/kbcli_cluster_stop.md +++ b/docs/user_docs/cli/kbcli_cluster_stop.md @@ -5,14 +5,14 @@ title: kbcli cluster stop Stop the cluster and release all the pods of the cluster. ``` -kbcli cluster stop [flags] +kbcli cluster stop NAME [flags] ``` ### Examples ``` # stop the cluster and release all the pods of the cluster - kbcli cluster stop + kbcli cluster stop mycluster ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_upgrade.md b/docs/user_docs/cli/kbcli_cluster_upgrade.md index 46cc32be4..950d41fb7 100644 --- a/docs/user_docs/cli/kbcli_cluster_upgrade.md +++ b/docs/user_docs/cli/kbcli_cluster_upgrade.md @@ -5,14 +5,14 @@ title: kbcli cluster upgrade Upgrade the cluster version. ``` -kbcli cluster upgrade [flags] +kbcli cluster upgrade NAME [flags] ``` ### Examples ``` # upgrade the cluster to the specified version - kbcli cluster upgrade --cluster-version= + kbcli cluster upgrade mycluster --cluster-version=ac-mysql-8.0.30 ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_volume-expand.md b/docs/user_docs/cli/kbcli_cluster_volume-expand.md index 3f6f75507..e8f9684a3 100644 --- a/docs/user_docs/cli/kbcli_cluster_volume-expand.md +++ b/docs/user_docs/cli/kbcli_cluster_volume-expand.md @@ -5,15 +5,14 @@ title: kbcli cluster volume-expand Expand volume with the specified components and volumeClaimTemplates in the cluster. ``` -kbcli cluster volume-expand [flags] +kbcli cluster volume-expand NAME [flags] ``` ### Examples ``` # restart specifies the component, separate with commas when more than one - kbcli cluster volume-expand --components= \ - --volume-claim-templates=data --storage=10Gi + kbcli cluster volume-expand mycluster --components=mysql --volume-claim-templates=data --storage=10Gi ``` ### Options diff --git a/docs/user_docs/cli/kbcli_cluster_vscale.md b/docs/user_docs/cli/kbcli_cluster_vscale.md index 6bf3d593e..28eeee02a 100644 --- a/docs/user_docs/cli/kbcli_cluster_vscale.md +++ b/docs/user_docs/cli/kbcli_cluster_vscale.md @@ -5,17 +5,17 @@ title: kbcli cluster vscale Vertically scale the specified components in the cluster. ``` -kbcli cluster vscale [flags] +kbcli cluster vscale NAME [flags] ``` ### Examples ``` - # scale the computing resources of specified components, separate with commas when more than one - kbcli cluster vscale --components= --cpu=500m --memory=500Mi + # scale the computing resources of specified components, separate with commas when component more than one + kbcli cluster vscale mycluster --components=mysql --cpu=500m --memory=500Mi - # scale the computing resources of specified components by class, available classes can be get by executing the command "kbcli class list --cluster-definition " - kbcli cluster vscale --components= --class= + # scale the computing resources of specified components by class, run command 'kbcli class list --cluster-definition cluster-definition-name' to get available classes + kbcli cluster vscale mycluster --components=mysql --class=general-2c4g ``` ### Options diff --git a/docs/user_docs/cli/kbcli_migration_create.md b/docs/user_docs/cli/kbcli_migration_create.md index bf8445804..e06fc2197 100644 --- a/docs/user_docs/cli/kbcli_migration_create.md +++ b/docs/user_docs/cli/kbcli_migration_create.md @@ -5,7 +5,7 @@ title: kbcli migration create Create a migration task. ``` -kbcli migration create name [flags] +kbcli migration create NAME [flags] ``` ### Examples diff --git a/internal/cli/cmd/cluster/cluster_test.go b/internal/cli/cmd/cluster/cluster_test.go index cbdb546ad..d290a8804 100644 --- a/internal/cli/cmd/cluster/cluster_test.go +++ b/internal/cli/cmd/cluster/cluster_test.go @@ -44,12 +44,16 @@ var _ = Describe("Cluster", func() { testComponentWithInvalidResourcePath = "../../testing/testdata/component_with_invalid_resource.yaml" ) + const ( + clusterName = "test" + namespace = "default" + ) var streams genericclioptions.IOStreams var tf *cmdtesting.TestFactory BeforeEach(func() { streams, _, _, _ = genericclioptions.NewTestIOStreams() - tf = cmdtesting.NewTestFactory().WithNamespace("default") + tf = cmdtesting.NewTestFactory().WithNamespace(namespace) cd := testing.FakeClusterDef() fakeDefaultStorageClass := testing.FakeStorageClass(testing.StorageClassName, testing.IsDefautl) tf.FakeDynamicClient = testing.FakeDynamicClient(cd, fakeDefaultStorageClass, testing.FakeClusterVersion()) @@ -69,11 +73,12 @@ var _ = Describe("Cluster", func() { UpdatableFlags: UpdatableFlags{ TerminationPolicy: "Delete", }, - BaseOptions: create.BaseOptions{ - Dynamic: tf.FakeDynamicClient, + CreateOptions: create.CreateOptions{ + Factory: tf, + Dynamic: tf.FakeDynamicClient, + IOStreams: streams, }, } - o.IOStreams = streams Expect(o.Validate()).To(Succeed()) Expect(o.Name).ShouldNot(BeEmpty()) }) @@ -104,7 +109,14 @@ var _ = Describe("Cluster", func() { testing.FakeComponentClassDef("custom-mysql", clusterDef.Name, "mysql"), ) o = &CreateOptions{ - BaseOptions: create.BaseOptions{IOStreams: streams, Name: "test", Dynamic: tf.FakeDynamicClient}, + CreateOptions: create.CreateOptions{ + IOStreams: streams, + Name: clusterName, + Dynamic: tf.FakeDynamicClient, + CueTemplateName: CueTemplateName, + Factory: tf, + GVR: types.ClusterGVR(), + }, SetFile: "", ClusterDefRef: testing.ClusterDefName, ClusterVersionRef: "cluster-version", @@ -120,18 +132,12 @@ var _ = Describe("Cluster", func() { }) Run := func() { - inputs := create.Inputs{ - ResourceName: types.ResourceClusters, - CueTemplateName: CueTemplateName, - Options: o, - Factory: tf, - } - - Expect(o.BaseOptions.Complete(inputs, []string{"test"})).Should(Succeed()) - Expect(o.Namespace).To(Equal("default")) - Expect(o.Name).To(Equal("test")) - - Expect(o.Run(inputs)).Should(Succeed()) + o.CreateOptions.Options = o + o.Args = []string{clusterName} + Expect(o.CreateOptions.Complete()).Should(Succeed()) + Expect(o.Namespace).To(Equal(namespace)) + Expect(o.Name).To(Equal(clusterName)) + Expect(o.Run()).Should(Succeed()) } It("validate tolerations", func() { @@ -261,7 +267,8 @@ var _ = Describe("Cluster", func() { UpdatableFlags: UpdatableFlags{ TerminationPolicy: "Delete", }, - BaseOptions: create.BaseOptions{ + CreateOptions: create.CreateOptions{ + Factory: tf, Namespace: "default", Name: "mycluster", Dynamic: tf.FakeDynamicClient, diff --git a/internal/cli/cmd/cluster/config_edit.go b/internal/cli/cmd/cluster/config_edit.go index 151b09e11..b4c3f70f8 100644 --- a/internal/cli/cmd/cluster/config_edit.go +++ b/internal/cli/cmd/cluster/config_edit.go @@ -58,7 +58,7 @@ var ( func (o *editConfigOptions) Run(fn func(info *cfgcore.ConfigPatchInfo, cc *appsv1alpha1.ConfigConstraintSpec) error) error { wrapper := o.wrapper - cfgEditContext := newConfigContext(o.BaseOptions, o.Name, wrapper.ComponentName(), wrapper.ConfigSpecName(), wrapper.ConfigFile()) + cfgEditContext := newConfigContext(o.CreateOptions, o.Name, wrapper.ComponentName(), wrapper.ConfigSpecName(), wrapper.ConfigFile()) if err := cfgEditContext.prepare(); err != nil { return err } @@ -161,40 +161,32 @@ func (o *editConfigOptions) confirmReconfigure(promptStr string) (bool, error) { // NewEditConfigureCmd shows the difference between two configuration version. func NewEditConfigureCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - editOptions := &editConfigOptions{ + o := &editConfigOptions{ configOpsOptions: configOpsOptions{ editMode: true, - OperationsOptions: newBaseOperationsOptions(streams, appsv1alpha1.ReconfiguringType, false), + OperationsOptions: newBaseOperationsOptions(f, streams, appsv1alpha1.ReconfiguringType, false), }} - inputs := buildOperationsInputs(f, editOptions.OperationsOptions) - inputs.Use = editConfigUse - inputs.Short = "Edit the config file of the component." - inputs.Example = editConfigExample - inputs.BuildFlags = func(cmd *cobra.Command) { - editOptions.buildReconfigureCommonFlags(cmd) - cmd.Flags().BoolVar(&editOptions.replaceFile, "replace", false, "Specify whether to replace the config file. Default to false.") - } - inputs.Complete = editOptions.Complete - inputs.Validate = editOptions.Validate cmd := &cobra.Command{ - Use: inputs.Use, - Short: inputs.Short, - Example: inputs.Example, + Use: editConfigUse, + Short: "Edit the config file of the component.", + Example: editConfigExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(inputs.BaseOptionsObj.Complete(inputs, args)) - util.CheckErr(inputs.BaseOptionsObj.Validate(inputs)) - util.CheckErr(editOptions.Run(func(info *cfgcore.ConfigPatchInfo, cc *appsv1alpha1.ConfigConstraintSpec) error { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + util.CheckErr(o.Complete()) + util.CheckErr(o.Validate()) + util.CheckErr(o.Run(func(info *cfgcore.ConfigPatchInfo, cc *appsv1alpha1.ConfigConstraintSpec) error { // generate patch for config formatterConfig := cc.FormatterConfig params := cfgcore.GenerateVisualizedParamsList(info, formatterConfig, nil) - editOptions.KeyValues = fromKeyValuesToMap(params, editOptions.CfgFile) - return inputs.BaseOptionsObj.Run(inputs) + o.KeyValues = fromKeyValuesToMap(params, o.CfgFile) + return o.CreateOptions.Run() })) }, } - if inputs.BuildFlags != nil { - inputs.BuildFlags(cmd) - } + o.buildReconfigureCommonFlags(cmd) + cmd.Flags().BoolVar(&o.replaceFile, "replace", false, "Specify whether to replace the config file. Default to false.") return cmd } diff --git a/internal/cli/cmd/cluster/config_ops.go b/internal/cli/cmd/cluster/config_ops.go index 915de1b58..1352bc52b 100644 --- a/internal/cli/cmd/cluster/config_ops.go +++ b/internal/cli/cmd/cluster/config_ops.go @@ -30,7 +30,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/internal/cli/create" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" @@ -75,7 +74,7 @@ func (o *configOpsOptions) Complete() error { o.KeyValues = kvs } - wrapper, err := newConfigWrapper(o.BaseOptions, o.Name, o.ComponentName, o.CfgTemplateName, o.CfgFile, o.KeyValues) + wrapper, err := newConfigWrapper(o.CreateOptions, o.Name, o.ComponentName, o.CfgTemplateName, o.CfgFile, o.KeyValues) if err != nil { return err } @@ -193,9 +192,9 @@ func (o *configOpsOptions) printConfigureTips() { printer.NewPair("ClusterName", o.Name)) } -// buildCommonFlags build common flags for operations command +// buildReconfigureCommonFlags build common flags for reconfigure command func (o *configOpsOptions) buildReconfigureCommonFlags(cmd *cobra.Command) { - o.buildCommonFlags(cmd) + o.addCommonFlags(cmd) cmd.Flags().StringSliceVar(&o.Parameters, "set", nil, "Specify updated parameter list. For details about the parameters, refer to kbcli sub command: 'kbcli cluster describe-config'.") cmd.Flags().StringVar(&o.ComponentName, "component", "", "Specify the name of Component to be updated. If the cluster has only one component, unset the parameter.") cmd.Flags().StringVar(&o.CfgTemplateName, "config-spec", "", "Specify the name of the configuration template to be updated (e.g. for apecloud-mysql: --config-spec=mysql-3node-tpl). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'.") @@ -206,17 +205,21 @@ func (o *configOpsOptions) buildReconfigureCommonFlags(cmd *cobra.Command) { func NewReconfigureCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &configOpsOptions{ editMode: false, - OperationsOptions: newBaseOperationsOptions(streams, appsv1alpha1.ReconfiguringType, false), - } - inputs := buildOperationsInputs(f, o.OperationsOptions) - inputs.Use = "configure NAME --set key=value[,key=value] [--component=component-name] [--config-spec=config-spec-name] [--config-file=config-file]" - inputs.Short = "Reconfigure parameters with the specified components in the cluster." - inputs.Example = createReconfigureExample - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildReconfigureCommonFlags(cmd) - } - - inputs.Complete = o.Complete - inputs.Validate = o.Validate - return create.BuildCommand(inputs) + OperationsOptions: newBaseOperationsOptions(f, streams, appsv1alpha1.ReconfiguringType, false), + } + cmd := &cobra.Command{ + Use: "reconfigure NAME --set key=value[,key=value] [--component=component-name] [--config-spec=config-spec-name] [--config-file=config-file]", + Short: "Reconfigure parameters with the specified components in the cluster.", + Example: createReconfigureExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.buildReconfigureCommonFlags(cmd) + return cmd } diff --git a/internal/cli/cmd/cluster/config_ops_test.go b/internal/cli/cmd/cluster/config_ops_test.go index efcbb8b1d..10bfaf17f 100644 --- a/internal/cli/cmd/cluster/config_ops_test.go +++ b/internal/cli/cmd/cluster/config_ops_test.go @@ -103,7 +103,7 @@ var _ = Describe("reconfigure test", func() { o := &configOpsOptions{ // nil cannot be set to a map struct in CueLang, so init the map of KeyValues. OperationsOptions: &OperationsOptions{ - BaseOptions: *ops, + CreateOptions: *ops, }, } o.KeyValues = make(map[string]string) @@ -127,7 +127,7 @@ var _ = Describe("reconfigure test", func() { in := &bytes.Buffer{} in.Write([]byte("yes\n")) - o.BaseOptions.In = io.NopCloser(in) + o.CreateOptions.In = io.NopCloser(in) Expect(o.Validate()).Should(Succeed()) }) diff --git a/internal/cli/cmd/cluster/config_util.go b/internal/cli/cmd/cluster/config_util.go index 9a9d7695e..5473bb44e 100644 --- a/internal/cli/cmd/cluster/config_util.go +++ b/internal/cli/cmd/cluster/config_util.go @@ -40,7 +40,7 @@ import ( ) type configEditContext struct { - create.BaseOptions + create.CreateOptions clusterName string componentName string @@ -99,9 +99,9 @@ func (c *configEditContext) getUnifiedDiffString() (string, error) { return difflib.GetUnifiedDiffString(diff) } -func newConfigContext(baseOptions create.BaseOptions, clusterName, componentName, configSpec, file string) *configEditContext { +func newConfigContext(baseOptions create.CreateOptions, clusterName, componentName, configSpec, file string) *configEditContext { return &configEditContext{ - BaseOptions: baseOptions, + CreateOptions: baseOptions, clusterName: clusterName, componentName: componentName, configSpecName: configSpec, diff --git a/internal/cli/cmd/cluster/config_util_test.go b/internal/cli/cmd/cluster/config_util_test.go index 402d05076..b239e9069 100644 --- a/internal/cli/cmd/cluster/config_util_test.go +++ b/internal/cli/cmd/cluster/config_util_test.go @@ -32,10 +32,10 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/types" ) -func NewFakeOperationsOptions(ns, cName string, opsType appsv1alpha1.OpsType, objs ...runtime.Object) (*cmdtesting.TestFactory, *create.BaseOptions) { +func NewFakeOperationsOptions(ns, cName string, opsType appsv1alpha1.OpsType, objs ...runtime.Object) (*cmdtesting.TestFactory, *create.CreateOptions) { streams, _, _, _ := genericclioptions.NewTestIOStreams() tf := cmdtesting.NewTestFactory().WithNamespace(ns) - baseOptions := &create.BaseOptions{ + baseOptions := &create.CreateOptions{ IOStreams: streams, Name: cName, Namespace: ns, diff --git a/internal/cli/cmd/cluster/config_wrapper.go b/internal/cli/cmd/cluster/config_wrapper.go index 3d785c342..bf16726c4 100644 --- a/internal/cli/cmd/cluster/config_wrapper.go +++ b/internal/cli/cmd/cluster/config_wrapper.go @@ -33,12 +33,12 @@ import ( ) type configWrapper struct { - create.BaseOptions + create.CreateOptions clusterName string updatedParams map[string]string - // auto fill field + // autofill field componentName string configSpecName string configKey string @@ -214,7 +214,7 @@ func (w *configWrapper) filterForReconfiguring(data map[string]string) []string return keys } -func newConfigWrapper(baseOptions create.BaseOptions, clusterName, componentName, configSpec, configKey string, params map[string]string) (*configWrapper, error) { +func newConfigWrapper(baseOptions create.CreateOptions, clusterName, componentName, configSpec, configKey string, params map[string]string) (*configWrapper, error) { var ( err error clusterObj *appsv1alpha1.Cluster @@ -229,7 +229,7 @@ func newConfigWrapper(baseOptions create.BaseOptions, clusterName, componentName } w := &configWrapper{ - BaseOptions: baseOptions, + CreateOptions: baseOptions, clusterObj: clusterObj, clusterDefObj: clusterDefObj, clusterName: clusterName, diff --git a/internal/cli/cmd/cluster/connect.go b/internal/cli/cmd/cluster/connect.go index 43f702fe2..57f9e86fc 100644 --- a/internal/cli/cmd/cluster/connect.go +++ b/internal/cli/cmd/cluster/connect.go @@ -298,7 +298,7 @@ func (o *ConnectOptions) getTargetPod() error { return fmt.Errorf("component name is not set yet") } - // get instantces for given cluster name and component name + // get instances for given cluster name and component name infos := cluster.GetSimpleInstanceInfosForComponent(o.Dynamic, o.clusterName, o.componentName, o.Namespace) if len(infos) == 0 || infos[0].Name == computil.ComponentStatusDefaultPodName { return fmt.Errorf("failed to find the instance to connect, please check cluster status") diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 7af43c5ee..d5912b5be 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -185,7 +185,56 @@ type CreateOptions struct { // backup name to restore in creation Backup string `json:"backup,omitempty"` UpdatableFlags - create.BaseOptions + create.CreateOptions `json:"-"` +} + +func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewCreateOptions(f, streams) + cmd := &cobra.Command{ + Use: "create [NAME]", + Short: "Create a cluster.", + Example: clusterCreateExample, + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run()) + }, + } + + cmd.Flags().StringVar(&o.ClusterDefRef, "cluster-definition", "", "Specify cluster definition, run \"kbcli cd list\" to show all available cluster definitions") + cmd.Flags().StringVar(&o.ClusterVersionRef, "cluster-version", "", "Specify cluster version, run \"kbcli cv list\" to show all available cluster versions, use the latest version if not specified") + cmd.Flags().StringVarP(&o.SetFile, "set-file", "f", "", "Use yaml file, URL, or stdin to set the cluster resource") + cmd.Flags().StringArrayVar(&o.Values, "set", []string{}, "Set the cluster resource including cpu, memory, replicas and storage, or you can just specify the class, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi or --set class=general-1c1g)") + cmd.Flags().StringVar(&o.Backup, "backup", "", "Set a source backup to restore data") + cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" + // add updatable flags + o.UpdatableFlags.addFlags(cmd) + + // add print flags + printer.AddOutputFlagForCreate(cmd, &o.Format) + + // set required flag + util.CheckErr(cmd.MarkFlagRequired("cluster-definition")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewCreateOptions(f cmdutil.Factory, streams genericclioptions.IOStreams) *CreateOptions { + o := &CreateOptions{CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateName, + GVR: types.ClusterGVR(), + }} + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o } func setMonitor(monitor bool, components []map[string]interface{}) { @@ -283,7 +332,7 @@ func (o *CreateOptions) Complete() error { } setMonitor(o.Monitor, components) - if err := setBackup(o, components); err != nil { + if err = setBackup(o, components); err != nil { return err } o.ComponentSpecs = components @@ -495,46 +544,6 @@ func MultipleSourceComponents(fileName string, in io.Reader) ([]byte, error) { return io.ReadAll(data) } -func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := &CreateOptions{BaseOptions: create.BaseOptions{IOStreams: streams}} - inputs := create.Inputs{ - Use: "create [NAME]", - Short: "Create a cluster.", - Example: clusterCreateExample, - CueTemplateName: CueTemplateName, - ResourceName: types.ResourceClusters, - BaseOptionsObj: &o.BaseOptions, - Options: o, - Factory: f, - Complete: o.Complete, - PreCreate: o.PreCreate, - CleanUpFn: o.CleanUp, - CreateDependencies: o.CreateDependencies, - BuildFlags: func(cmd *cobra.Command) { - cmd.Flags().StringVar(&o.ClusterDefRef, "cluster-definition", "", "Specify cluster definition, run \"kbcli cd list\" to show all available cluster definitions") - cmd.Flags().StringVar(&o.ClusterVersionRef, "cluster-version", "", "Specify cluster version, run \"kbcli cv list\" to show all available cluster versions, use the latest version if not specified") - cmd.Flags().StringVarP(&o.SetFile, "set-file", "f", "", "Use yaml file, URL, or stdin to set the cluster resource") - cmd.Flags().StringArrayVar(&o.Values, "set", []string{}, "Set the cluster resource including cpu, memory, replicas and storage, or you can just specify the class, each set corresponds to a component.(e.g. --set cpu=1,memory=1Gi,replicas=3,storage=20Gi or --set class=general-1c1g)") - cmd.Flags().StringVar(&o.Backup, "backup", "", "Set a source backup to restore data") - cmd.Flags().StringVar(&o.DryRunStrategy, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) - cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" - // add updatable flags - o.UpdatableFlags.addFlags(cmd) - - // add print flags - printer.AddOutputFlagForCreate(cmd, &o.Format) - - // set required flag - util.CheckErr(cmd.MarkFlagRequired("cluster-definition")) - - // register flag completion func - registerFlagCompletionFunc(cmd, f) - }, - } - - return create.BuildCommand(inputs) -} - func registerFlagCompletionFunc(cmd *cobra.Command, f cmdutil.Factory) { util.CheckErr(cmd.RegisterFlagCompletionFunc( "cluster-definition", diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 2c531e3c9..40127c0d3 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -57,7 +57,7 @@ var ( # list all backup policy kbcli cluster list-backup-policy - # using short cmd to list backup policy of specified cluster + # using short cmd to list backup policy of the specified cluster kbcli cluster list-bp mycluster `) editExample = templates.Examples(` @@ -69,16 +69,16 @@ var ( `) createBackupExample = templates.Examples(` # create a backup - kbcli cluster backup cluster-name + kbcli cluster backup mycluster # create a snapshot backup - kbcli cluster backup cluster-name --backup-type snapshot + kbcli cluster backup mycluster --backup-type snapshot # create a full backup - kbcli cluster backup cluster-name --backup-type full + kbcli cluster backup mycluster --backup-type full # create a backup with specified backup policy - kbcli cluster backup cluster-name --backup-policy + kbcli cluster backup mycluster --backup-policy `) listBackupExample = templates.Examples(` # list all backup @@ -104,7 +104,8 @@ type CreateBackupOptions struct { BackupName string `json:"backupName"` Role string `json:"role,omitempty"` BackupPolicy string `json:"backupPolicy"` - create.BaseOptions + + create.CreateOptions `json:"-"` } type ListBackupOptions struct { @@ -117,7 +118,8 @@ func (o *CreateBackupOptions) Complete() error { if len(o.BackupName) == 0 { o.BackupName = strings.Join([]string{"backup", o.Namespace, o.Name, time.Now().Format("20060102150405")}, "-") } - return nil + + return o.CreateOptions.Complete() } func (o *CreateBackupOptions) Validate() error { @@ -180,35 +182,42 @@ func (o *CreateBackupOptions) getDefaultBackupPolicy() (string, error) { } func NewCreateBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := &CreateBackupOptions{BaseOptions: create.BaseOptions{IOStreams: streams}} - customOutPut := func(opt *create.BaseOptions) { + customOutPut := func(opt *create.CreateOptions) { output := fmt.Sprintf("Backup %s created successfully, you can view the progress:", opt.Name) printer.PrintLine(output) nextLine := fmt.Sprintf("\tkbcli cluster list-backups --name=%s -n %s", opt.Name, opt.Namespace) printer.PrintLine(nextLine) } - inputs := create.Inputs{ - Use: "backup", - Short: "Create a backup.", - Example: createBackupExample, - CueTemplateName: "backup_template.cue", - ResourceName: types.ResourceBackups, - Group: types.DPAPIGroup, - Version: types.DPAPIVersion, - BaseOptionsObj: &o.BaseOptions, - Options: o, - Factory: f, - Complete: o.Complete, - Validate: o.Validate, - CustomOutPut: customOutPut, - ResourceNameGVRForCompletion: types.ClusterGVR(), - BuildFlags: func(cmd *cobra.Command) { - cmd.Flags().StringVar(&o.BackupType, "backup-type", "snapshot", "Backup type") - cmd.Flags().StringVar(&o.BackupName, "backup-name", "", "Backup name") - cmd.Flags().StringVar(&o.BackupPolicy, "backup-policy", "", "Backup policy name, this flag will be ignored when backup-type is snapshot") + + o := &CreateBackupOptions{ + CreateOptions: create.CreateOptions{ + IOStreams: streams, + Factory: f, + GVR: types.BackupGVR(), + CueTemplateName: "backup_template.cue", + CustomOutPut: customOutPut, }, } - return create.BuildCommand(inputs) + o.CreateOptions.Options = o + + cmd := &cobra.Command{ + Use: "backup NAME", + Short: "Create a backup for the cluster.", + Example: createBackupExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + + cmd.Flags().StringVar(&o.BackupType, "backup-type", "snapshot", "Backup type") + cmd.Flags().StringVar(&o.BackupName, "backup-name", "", "Backup name") + cmd.Flags().StringVar(&o.BackupPolicy, "backup-policy", "", "Backup policy name, this flag will be ignored when backup-type is snapshot") + + return cmd } // getClusterNameMap get cluster list by namespace and convert to map. @@ -345,7 +354,7 @@ type CreateRestoreOptions struct { RestoreTimeStr string `json:"restoreTimeStr,omitempty"` SourceCluster string `json:"sourceCluster,omitempty"` - create.BaseOptions + create.CreateOptions `json:"-"` } func (o *CreateRestoreOptions) getClusterObject(backup *dataprotectionv1alpha1.Backup) (*appsv1alpha1.Cluster, error) { @@ -523,19 +532,20 @@ func (o *CreateRestoreOptions) Validate() error { func NewCreateRestoreCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := &CreateRestoreOptions{} - o.IOStreams = streams - inputs := create.Inputs{ - BaseOptionsObj: &create.BaseOptions{IOStreams: streams}, - Options: o, - Factory: f, + o.CreateOptions = create.CreateOptions{ + IOStreams: streams, + Factory: f, + Options: o, } + cmd := &cobra.Command{ Use: "restore", Short: "Restore a new cluster from backup.", Example: createRestoreExample, ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(o.Complete(inputs, args)) + o.Args = args + util.CheckErr(o.Complete()) util.CheckErr(o.Validate()) util.CheckErr(o.Run()) }, diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index f9de33a42..76359bbad 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -102,12 +102,12 @@ var _ = Describe("DataProtection", func() { It("validate create backup", func() { By("without cluster name") o := &CreateBackupOptions{ - BaseOptions: create.BaseOptions{ + CreateOptions: create.CreateOptions{ Dynamic: testing.FakeDynamicClient(), IOStreams: streams, + Factory: tf, }, } - o.IOStreams = streams Expect(o.Validate()).To(MatchError("missing cluster name")) By("test without default backupPolicy") diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index c69712a7a..8f39c2077 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -45,7 +45,7 @@ import ( ) type OperationsOptions struct { - create.BaseOptions + create.CreateOptions `json:"-"` HasComponentNamesFlag bool `json:"-"` // RequireConfirm if it is true, the second verification will be performed before creating ops. RequireConfirm bool `json:"-"` @@ -86,25 +86,43 @@ type OperationsOptions struct { Services []appsv1alpha1.ClusterComponentService `json:"services,omitempty"` } -func newBaseOperationsOptions(streams genericclioptions.IOStreams, opsType appsv1alpha1.OpsType, hasComponentNamesFlag bool) *OperationsOptions { - return &OperationsOptions{ +func newBaseOperationsOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, + opsType appsv1alpha1.OpsType, hasComponentNamesFlag bool) *OperationsOptions { + customOutPut := func(opt *create.CreateOptions) { + output := fmt.Sprintf("OpsRequest %s created successfully, you can view the progress:", opt.Name) + printer.PrintLine(output) + nextLine := fmt.Sprintf("\tkbcli cluster describe-ops %s -n %s", opt.Name, opt.Namespace) + printer.PrintLine(nextLine) + } + + o := &OperationsOptions{ // nil cannot be set to a map struct in CueLang, so init the map of KeyValues. KeyValues: map[string]string{}, - BaseOptions: create.BaseOptions{IOStreams: streams}, OpsType: opsType, HasComponentNamesFlag: hasComponentNamesFlag, RequireConfirm: true, + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: "cluster_operations_template.cue", + GVR: types.OpsGVR(), + CustomOutPut: customOutPut, + }, } + + o.OpsTypeLower = strings.ToLower(string(o.OpsType)) + o.CreateOptions.Options = o + return o } -// buildCommonFlags build common flags for operations command -func (o *OperationsOptions) buildCommonFlags(cmd *cobra.Command) { +// addCommonFlags add common flags for operations command +func (o *OperationsOptions) addCommonFlags(cmd *cobra.Command) { // add print flags printer.AddOutputFlagForCreate(cmd, &o.Format) cmd.Flags().StringVar(&o.OpsRequestName, "name", "", "OpsRequest name. if not specified, it will be randomly generated ") cmd.Flags().IntVar(&o.TTLSecondsAfterSucceed, "ttlSecondsAfterSucceed", 0, "Time to live after the OpsRequest succeed") - cmd.Flags().StringVar(&o.DryRunStrategy, "dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" if o.HasComponentNamesFlag { cmd.Flags().StringSliceVar(&o.ComponentNames, "components", nil, " Component names to this operations") @@ -210,12 +228,12 @@ func (o *OperationsOptions) Validate() error { } // check if cluster exist - unstructuredObj, err := o.Dynamic.Resource(types.ClusterGVR()).Namespace(o.Namespace).Get(context.TODO(), o.Name, metav1.GetOptions{}) + obj, err := o.Dynamic.Resource(types.ClusterGVR()).Namespace(o.Namespace).Get(context.TODO(), o.Name, metav1.GetOptions{}) if err != nil { return err } var cluster appsv1alpha1.Cluster - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj.Object, &cluster); err != nil { + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &cluster); err != nil { return err } @@ -242,35 +260,12 @@ func (o *OperationsOptions) Validate() error { return err } } - if o.RequireConfirm && o.DryRunStrategy == "none" { + if o.RequireConfirm && o.DryRun == "none" { return delete.Confirm([]string{o.Name}, o.In) } return nil } -// buildOperationsInputs builds operations inputs -func buildOperationsInputs(f cmdutil.Factory, o *OperationsOptions) create.Inputs { - o.OpsTypeLower = strings.ToLower(string(o.OpsType)) - customOutPut := func(opt *create.BaseOptions) { - output := fmt.Sprintf("OpsRequest %s created successfully, you can view the progress:", opt.Name) - printer.PrintLine(output) - nextLine := fmt.Sprintf("\tkbcli cluster describe-ops %s -n %s", opt.Name, opt.Namespace) - printer.PrintLine(nextLine) - } - return create.Inputs{ - CueTemplateName: "cluster_operations_template.cue", - ResourceName: types.ResourceOpsRequests, - BaseOptionsObj: &o.BaseOptions, - Options: o, - Factory: f, - Validate: o.Validate, - CustomOutPut: customOutPut, - Group: types.AppsAPIGroup, - Version: types.AppsAPIVersion, - ResourceNameGVRForCompletion: types.ClusterGVR(), - } -} - func (o *OperationsOptions) validateExpose() error { switch util.ExposeType(o.ExposeType) { case "", util.ExposeToVPC, util.ExposeToInternet: @@ -359,111 +354,141 @@ func (o *OperationsOptions) fillExpose() error { var restartExample = templates.Examples(` # restart all components - kbcli cluster restart + kbcli cluster restart mycluster - # restart specifies the component, separate with commas when more than one - kbcli cluster restart --components= + # restart specifies the component, separate with commas when component more than one + kbcli cluster restart mycluster --components=mysql `) // NewRestartCmd creates a restart command func NewRestartCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := newBaseOperationsOptions(streams, appsv1alpha1.RestartType, true) - inputs := buildOperationsInputs(f, o) - inputs.Use = "restart" - inputs.Short = "Restart the specified components in the cluster." - inputs.Example = restartExample - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildCommonFlags(cmd) - } - inputs.Complete = o.CompleteRestartOps - return create.BuildCommand(inputs) + o := newBaseOperationsOptions(f, streams, appsv1alpha1.RestartType, true) + cmd := &cobra.Command{ + Use: "restart NAME", + Short: "Restart the specified components in the cluster.", + Example: restartExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.CompleteRestartOps()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.addCommonFlags(cmd) + return cmd } var upgradeExample = templates.Examples(` # upgrade the cluster to the specified version - kbcli cluster upgrade --cluster-version= + kbcli cluster upgrade mycluster --cluster-version=ac-mysql-8.0.30 `) // NewUpgradeCmd creates a upgrade command func NewUpgradeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := newBaseOperationsOptions(streams, appsv1alpha1.UpgradeType, false) - inputs := buildOperationsInputs(f, o) - inputs.Use = "upgrade" - inputs.Short = "Upgrade the cluster version." - inputs.Example = upgradeExample - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildCommonFlags(cmd) - cmd.Flags().StringVar(&o.ClusterVersionRef, "cluster-version", "", "Reference cluster version (required)") - } - return create.BuildCommand(inputs) + o := newBaseOperationsOptions(f, streams, appsv1alpha1.UpgradeType, false) + cmd := &cobra.Command{ + Use: "upgrade NAME", + Short: "Upgrade the cluster version.", + Example: upgradeExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.addCommonFlags(cmd) + cmd.Flags().StringVar(&o.ClusterVersionRef, "cluster-version", "", "Reference cluster version (required)") + return cmd } var verticalScalingExample = templates.Examples(` - # scale the computing resources of specified components, separate with commas when more than one - kbcli cluster vscale --components= --cpu=500m --memory=500Mi + # scale the computing resources of specified components, separate with commas when component more than one + kbcli cluster vscale mycluster --components=mysql --cpu=500m --memory=500Mi - # scale the computing resources of specified components by class, available classes can be get by executing the command "kbcli class list --cluster-definition " - kbcli cluster vscale --components= --class= + # scale the computing resources of specified components by class, run command 'kbcli class list --cluster-definition cluster-definition-name' to get available classes + kbcli cluster vscale mycluster --components=mysql --class=general-2c4g `) // NewVerticalScalingCmd creates a vertical scaling command func NewVerticalScalingCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := newBaseOperationsOptions(streams, appsv1alpha1.VerticalScalingType, true) - inputs := buildOperationsInputs(f, o) - inputs.Use = "vscale" - inputs.Short = "Vertically scale the specified components in the cluster." - inputs.Example = verticalScalingExample - inputs.Complete = o.CompleteComponentsFlag - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildCommonFlags(cmd) - cmd.Flags().StringVar(&o.CPU, "cpu", "", "Requested and limited size of component cpu") - cmd.Flags().StringVar(&o.Memory, "memory", "", "Requested and limited size of component memory") - cmd.Flags().StringVar(&o.Class, "class", "", "Component class") - } - return create.BuildCommand(inputs) + o := newBaseOperationsOptions(f, streams, appsv1alpha1.VerticalScalingType, true) + cmd := &cobra.Command{ + Use: "vscale NAME", + Short: "Vertically scale the specified components in the cluster.", + Example: verticalScalingExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.CompleteComponentsFlag()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.addCommonFlags(cmd) + cmd.Flags().StringVar(&o.CPU, "cpu", "", "Requested and limited size of component cpu") + cmd.Flags().StringVar(&o.Memory, "memory", "", "Requested and limited size of component memory") + cmd.Flags().StringVar(&o.Class, "class", "", "Component class") + return cmd } var horizontalScalingExample = templates.Examples(` - # expand storage resources of specified components, separate with commas when more than one - kbcli cluster hscale --components= --replicas=3 + # expand storage resources of specified components, separate with commas when component name more than one + kbcli cluster hscale mycluster --components=mysql --replicas=3 `) // NewHorizontalScalingCmd creates a horizontal scaling command func NewHorizontalScalingCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := newBaseOperationsOptions(streams, appsv1alpha1.HorizontalScalingType, true) - inputs := buildOperationsInputs(f, o) - inputs.Use = "hscale" - inputs.Short = "Horizontally scale the specified components in the cluster." - inputs.Example = horizontalScalingExample - inputs.Complete = o.CompleteComponentsFlag - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildCommonFlags(cmd) - cmd.Flags().IntVar(&o.Replicas, "replicas", o.Replicas, "Replicas with the specified components") - _ = cmd.MarkFlagRequired("replicas") - } - return create.BuildCommand(inputs) + o := newBaseOperationsOptions(f, streams, appsv1alpha1.HorizontalScalingType, true) + cmd := &cobra.Command{ + Use: "hscale NAME", + Short: "Horizontally scale the specified components in the cluster.", + Example: horizontalScalingExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.CompleteComponentsFlag()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + + o.addCommonFlags(cmd) + cmd.Flags().IntVar(&o.Replicas, "replicas", o.Replicas, "Replicas with the specified components") + _ = cmd.MarkFlagRequired("replicas") + return cmd } var volumeExpansionExample = templates.Examples(` # restart specifies the component, separate with commas when more than one - kbcli cluster volume-expand --components= \ - --volume-claim-templates=data --storage=10Gi + kbcli cluster volume-expand mycluster --components=mysql --volume-claim-templates=data --storage=10Gi `) // NewVolumeExpansionCmd creates a vertical scaling command func NewVolumeExpansionCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := newBaseOperationsOptions(streams, appsv1alpha1.VolumeExpansionType, true) - inputs := buildOperationsInputs(f, o) - inputs.Use = "volume-expand" - inputs.Short = "Expand volume with the specified components and volumeClaimTemplates in the cluster." - inputs.Example = volumeExpansionExample - inputs.Complete = o.CompleteComponentsFlag - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildCommonFlags(cmd) - cmd.Flags().StringSliceVarP(&o.VCTNames, "volume-claim-templates", "t", nil, "VolumeClaimTemplate names in components (required)") - cmd.Flags().StringVar(&o.Storage, "storage", "", "Volume storage size (required)") - } - return create.BuildCommand(inputs) + o := newBaseOperationsOptions(f, streams, appsv1alpha1.VolumeExpansionType, true) + cmd := &cobra.Command{ + Use: "volume-expand NAME", + Short: "Expand volume with the specified components and volumeClaimTemplates in the cluster.", + Example: volumeExpansionExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.CompleteComponentsFlag()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.addCommonFlags(cmd) + cmd.Flags().StringSliceVarP(&o.VCTNames, "volume-claim-templates", "t", nil, "VolumeClaimTemplate names in components (required)") + cmd.Flags().StringVar(&o.Storage, "storage", "", "Volume storage size (required)") + return cmd } var ( @@ -481,60 +506,80 @@ var ( // NewExposeCmd creates an expose command func NewExposeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := newBaseOperationsOptions(streams, appsv1alpha1.ExposeType, true) - inputs := buildOperationsInputs(f, o) - inputs.Use = "expose NAME --enable=[true|false] --type=[vpc|internet]" - inputs.Short = "Expose a cluster with a new endpoint, the new endpoint can be found by executing 'kbcli cluster describe NAME'." - inputs.Example = exposeExamples - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildCommonFlags(cmd) - cmd.Flags().StringVar(&o.ExposeType, "type", "", "Expose type, currently supported types are 'vpc', 'internet'") - util.CheckErr(cmd.RegisterFlagCompletionFunc("type", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return []string{string(util.ExposeToVPC), string(util.ExposeToInternet)}, cobra.ShellCompDirectiveNoFileComp - })) - cmd.Flags().StringVar(&o.ExposeEnabled, "enable", "", "Enable or disable the expose, values can be true or false") - util.CheckErr(cmd.RegisterFlagCompletionFunc("enable", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return []string{"true", "false"}, cobra.ShellCompDirectiveNoFileComp - })) - _ = cmd.MarkFlagRequired("enable") - } - inputs.Complete = o.fillExpose - return create.BuildCommand(inputs) + o := newBaseOperationsOptions(f, streams, appsv1alpha1.ExposeType, true) + cmd := &cobra.Command{ + Use: "expose NAME --enable=[true|false] --type=[vpc|internet]", + Short: "Expose a cluster with a new endpoint, the new endpoint can be found by executing 'kbcli cluster describe NAME'.", + Example: exposeExamples, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.fillExpose()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.addCommonFlags(cmd) + cmd.Flags().StringVar(&o.ExposeType, "type", "", "Expose type, currently supported types are 'vpc', 'internet'") + cmd.Flags().StringVar(&o.ExposeEnabled, "enable", "", "Enable or disable the expose, values can be true or false") + + util.CheckErr(cmd.RegisterFlagCompletionFunc("type", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return []string{string(util.ExposeToVPC), string(util.ExposeToInternet)}, cobra.ShellCompDirectiveNoFileComp + })) + util.CheckErr(cmd.RegisterFlagCompletionFunc("enable", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return []string{"true", "false"}, cobra.ShellCompDirectiveNoFileComp + })) + + _ = cmd.MarkFlagRequired("enable") + return cmd } var stopExample = templates.Examples(` # stop the cluster and release all the pods of the cluster - kbcli cluster stop + kbcli cluster stop mycluster `) // NewStopCmd creates a stop command func NewStopCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := newBaseOperationsOptions(streams, appsv1alpha1.StopType, false) - inputs := buildOperationsInputs(f, o) - inputs.Use = "stop" - inputs.Short = "Stop the cluster and release all the pods of the cluster." - inputs.Example = stopExample - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildCommonFlags(cmd) - } - return create.BuildCommand(inputs) + o := newBaseOperationsOptions(f, streams, appsv1alpha1.StopType, false) + cmd := &cobra.Command{ + Use: "stop NAME", + Short: "Stop the cluster and release all the pods of the cluster.", + Example: stopExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.addCommonFlags(cmd) + return cmd } var startExample = templates.Examples(` # start the cluster when cluster is stopped - kbcli cluster start + kbcli cluster start mycluster `) // NewStartCmd creates a start command func NewStartCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := newBaseOperationsOptions(streams, appsv1alpha1.StartType, false) + o := newBaseOperationsOptions(f, streams, appsv1alpha1.StartType, false) o.RequireConfirm = false - inputs := buildOperationsInputs(f, o) - inputs.Use = "start" - inputs.Short = "Start the cluster if cluster is stopped." - inputs.Example = startExample - inputs.BuildFlags = func(cmd *cobra.Command) { - o.buildCommonFlags(cmd) - } - return create.BuildCommand(inputs) + cmd := &cobra.Command{ + Use: "start NAME", + Short: "Start the cluster if cluster is stopped.", + Example: startExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) + }, + } + o.addCommonFlags(cmd) + return cmd } diff --git a/internal/cli/cmd/cluster/operations_test.go b/internal/cli/cmd/cluster/operations_test.go index 7942ef593..a32413fd4 100644 --- a/internal/cli/cmd/cluster/operations_test.go +++ b/internal/cli/cmd/cluster/operations_test.go @@ -69,7 +69,7 @@ var _ = Describe("operations", func() { }) initCommonOperationOps := func(opsType appsv1alpha1.OpsType, clusterName string, hasComponentNamesFlag bool) *OperationsOptions { - o := newBaseOperationsOptions(streams, opsType, hasComponentNamesFlag) + o := newBaseOperationsOptions(tf, streams, opsType, hasComponentNamesFlag) o.Dynamic = tf.FakeDynamicClient o.Name = clusterName o.Namespace = testing.Namespace @@ -77,7 +77,7 @@ var _ = Describe("operations", func() { } It("Upgrade Ops", func() { - o := newBaseOperationsOptions(streams, appsv1alpha1.UpgradeType, false) + o := newBaseOperationsOptions(tf, streams, appsv1alpha1.UpgradeType, false) o.Dynamic = tf.FakeDynamicClient By("validate o.name is null") @@ -168,8 +168,8 @@ var _ = Describe("operations", func() { It("Restart ops", func() { o := initCommonOperationOps(appsv1alpha1.RestartType, clusterName, true) By("expect for not found error") - inputs := buildOperationsInputs(tf, o) - Expect(o.Complete(inputs, []string{clusterName + "2"})) + o.Args = []string{clusterName + "2"} + Expect(o.Complete()) Expect(o.CompleteRestartOps().Error()).Should(ContainSubstring("not found")) By("expect for complete success") diff --git a/internal/cli/cmd/migration/create.go b/internal/cli/cmd/migration/create.go index f898f6afd..2b2276895 100644 --- a/internal/cli/cmd/migration/create.go +++ b/internal/cli/cmd/migration/create.go @@ -67,40 +67,45 @@ type CreateMigrationOptions struct { Resources []string `json:"resources,omitempty"` ResourceModel map[string]interface{} `json:"resourceModel,omitempty"` ServerID uint32 `json:"serverId,omitempty"` - create.BaseOptions + create.CreateOptions `json:"-"` } func NewMigrationCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := &CreateMigrationOptions{BaseOptions: create.BaseOptions{IOStreams: streams}} - inputs := create.Inputs{ - Use: "create name", - Short: "Create a migration task.", - Example: CreateTemplate, - CueTemplateName: "migration_template.cue", - ResourceName: types.ResourceMigrationTasks, - Group: types.MigrationAPIGroup, - Version: types.MigrationAPIVersion, - BaseOptionsObj: &o.BaseOptions, - Options: o, - Factory: f, - Validate: o.Validate, - ResourceNameGVRForCompletion: types.MigrationTaskGVR(), - BuildFlags: func(cmd *cobra.Command) { - cmd.Flags().StringVar(&o.Template, "template", "", "Specify migration template, run \"kbcli migration templates\" to show all available migration templates") - cmd.Flags().StringVar(&o.Source, "source", "", "Set the source database information for migration.such as '{username}:{password}@{connection_address}:{connection_port}/[{database}]'") - cmd.Flags().StringVar(&o.Sink, "sink", "", "Set the sink database information for migration.such as '{username}:{password}@{connection_address}:{connection_port}/[{database}]") - cmd.Flags().StringSliceVar(&o.MigrationObject, "migration-object", []string{}, "Set the data objects that need to be migrated,such as '\"db1.table1\",\"db2\"'") - cmd.Flags().StringSliceVar(&o.Steps, "steps", []string{}, "Set up migration steps,such as: precheck=true,init-struct=true,init-data=true,cdc=true") - cmd.Flags().StringSliceVar(&o.Tolerations, "tolerations", []string{}, "Tolerations for migration, such as '\"key=engineType,value=pg,operator=Equal,effect=NoSchedule\"'") - cmd.Flags().StringSliceVar(&o.Resources, "resources", []string{}, "Resources limit for migration, such as '\"cpu=3000m,memory=3Gi\"'") - - util.CheckErr(cmd.MarkFlagRequired("template")) - util.CheckErr(cmd.MarkFlagRequired("source")) - util.CheckErr(cmd.MarkFlagRequired("sink")) - util.CheckErr(cmd.MarkFlagRequired("migration-object")) + o := &CreateMigrationOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: "migration_template.cue", + GVR: types.MigrationTaskGVR(), + }} + o.CreateOptions.Options = o + + cmd := &cobra.Command{ + Use: "create NAME", + Short: "Create a migration task.", + Example: CreateTemplate, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.MigrationTaskGVR()), + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Run()) }, } - return create.BuildCommand(inputs) + + cmd.Flags().StringVar(&o.Template, "template", "", "Specify migration template, run \"kbcli migration templates\" to show all available migration templates") + cmd.Flags().StringVar(&o.Source, "source", "", "Set the source database information for migration.such as '{username}:{password}@{connection_address}:{connection_port}/[{database}]'") + cmd.Flags().StringVar(&o.Sink, "sink", "", "Set the sink database information for migration.such as '{username}:{password}@{connection_address}:{connection_port}/[{database}]") + cmd.Flags().StringSliceVar(&o.MigrationObject, "migration-object", []string{}, "Set the data objects that need to be migrated,such as '\"db1.table1\",\"db2\"'") + cmd.Flags().StringSliceVar(&o.Steps, "steps", []string{}, "Set up migration steps,such as: precheck=true,init-struct=true,init-data=true,cdc=true") + cmd.Flags().StringSliceVar(&o.Tolerations, "tolerations", []string{}, "Tolerations for migration, such as '\"key=engineType,value=pg,operator=Equal,effect=NoSchedule\"'") + cmd.Flags().StringSliceVar(&o.Resources, "resources", []string{}, "Resources limit for migration, such as '\"cpu=3000m,memory=3Gi\"'") + + util.CheckErr(cmd.MarkFlagRequired("template")) + util.CheckErr(cmd.MarkFlagRequired("source")) + util.CheckErr(cmd.MarkFlagRequired("sink")) + util.CheckErr(cmd.MarkFlagRequired("migration-object")) + return cmd } func (o *CreateMigrationOptions) Validate() error { @@ -244,7 +249,7 @@ func (o *CreateMigrationOptions) BuildWithResources() error { func (o *CreateMigrationOptions) BuildWithRuntimeParams() error { template := migrationv1.MigrationTemplate{} templateGvr := types.MigrationTemplateGVR() - if err := APIResource(&o.BaseOptions.Dynamic, &templateGvr, o.Template, "", &template); err != nil { + if err := APIResource(&o.CreateOptions.Dynamic, &templateGvr, o.Template, "", &template); err != nil { return err } diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index f1d922012..f40f737d6 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -447,20 +447,42 @@ func (o *initOptions) installKubeBlocks(k8sClusterName string) error { // createCluster construct a cluster create options and run func (o *initOptions) createCluster() error { - // construct a cluster create options and run - options, err := o.newCreateOptions() - if err != nil { - return err + options := &cmdcluster.CreateOptions{ + CreateOptions: create.CreateOptions{ + Factory: util.NewFactory(), + IOStreams: genericclioptions.NewTestIOStreamsDiscard(), + Namespace: defaultNamespace, + Name: kbClusterName, + CueTemplateName: cmdcluster.CueTemplateName, + GVR: types.ClusterGVR(), + }, + UpdatableFlags: cmdcluster.UpdatableFlags{ + TerminationPolicy: "WipeOut", + Monitor: true, + PodAntiAffinity: "Preferred", + Tenancy: "SharedNode", + }, + ClusterDefRef: o.clusterDef, + ClusterVersionRef: o.clusterVersion, } + options.CreateOptions.Options = options + options.CreateOptions.PreCreate = options.PreCreate - inputs := create.Inputs{ - BaseOptionsObj: &options.BaseOptions, - Options: options, - CueTemplateName: cmdcluster.CueTemplateName, - ResourceName: types.ResourceClusters, + // if we are running on cloud, create cluster with three replicas + if o.cloudProvider != cp.Local { + options.Values = append(options.Values, "replicas=3") } - return options.Run(inputs) + if err := options.CreateOptions.Complete(); err != nil { + return err + } + if err := options.Validate(); err != nil { + return err + } + if err := options.Complete(); err != nil { + return err + } + return options.Run() } // checkExistedCluster check playground kubernetes cluster exists or not, playground @@ -491,36 +513,3 @@ func (o *initOptions) checkExistedCluster() error { } return nil } - -func (o *initOptions) newCreateOptions() (*cmdcluster.CreateOptions, error) { - dynamicClient, err := util.NewFactory().DynamicClient() - if err != nil { - return nil, err - } - options := &cmdcluster.CreateOptions{ - BaseOptions: create.BaseOptions{ - IOStreams: genericclioptions.NewTestIOStreamsDiscard(), - Namespace: defaultNamespace, - Name: kbClusterName, - Dynamic: dynamicClient, - }, - UpdatableFlags: cmdcluster.UpdatableFlags{ - TerminationPolicy: "WipeOut", - Monitor: true, - PodAntiAffinity: "Preferred", - Tenancy: "SharedNode", - }, - ClusterDefRef: o.clusterDef, - ClusterVersionRef: o.clusterVersion, - } - - // if we are running on cloud, create cluster with three replicas - if o.cloudProvider != cp.Local { - options.Values = append(options.Values, "replicas=3") - } - - if err = options.Complete(); err != nil { - return nil, err - } - return options, nil -} diff --git a/internal/cli/create/create.go b/internal/cli/create/create.go index 6f02cf582..9afe7a98f 100755 --- a/internal/cli/create/create.go +++ b/internal/cli/create/create.go @@ -30,13 +30,11 @@ import ( "cuelang.org/go/cue/cuecontext" cuejson "cuelang.org/go/encoding/json" "github.com/leaanthony/debme" - "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" - k8sapitypes "k8s.io/apimachinery/pkg/types" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/printers" "k8s.io/client-go/dynamic" @@ -45,9 +43,6 @@ import ( "k8s.io/kubectl/pkg/scheme" "github.com/apecloud/kubeblocks/internal/cli/printer" - "github.com/apecloud/kubeblocks/internal/cli/types" - - "github.com/apecloud/kubeblocks/internal/cli/util" ) var ( @@ -66,119 +61,64 @@ const ( DryRunServer ) -type Inputs struct { - // Use cobra command use - Use string - - // Short is the short description shown in the 'help' output. - Short string - - // Example is examples of how to use the command. - Example string - - // BaseOptionsObj - BaseOptionsObj *BaseOptions - - // Options a command options object which extends BaseOptions - Options interface{} +// CreateOptions the options of creation command should inherit baseOptions +type CreateOptions struct { + Factory cmdutil.Factory + Namespace string - // CueTemplateName cue template file name + // Name Resource name of the command line operation + Name string + Args []string + Dynamic dynamic.Interface + Client kubernetes.Interface + Format printer.Format + ToPrinter func(*meta.RESTMapping, bool) (printers.ResourcePrinterFunc, error) + DryRun string + + // CueTemplateName cue template file name to render the resource CueTemplateName string - // ResourceName k8s resource name - ResourceName string - - // Group of API, default is apps - Group string - - // Group of Version, default is v1alpha1 - Version string - - // Factory - Factory cmdutil.Factory - - // ValidateFunc optional, custom validate func - Validate func() error + // Options a command options object which extends CreateOptions that will be used + // to render the cue template + Options interface{} - // Complete optional, do custom complete options - Complete func() error + // GVR is the GroupVersionResource of the resource to be created + GVR schema.GroupVersionResource - BuildFlags func(*cobra.Command) + // CustomOutPut will be executed after creating successfully. + CustomOutPut func(options *CreateOptions) // PreCreate optional, make changes on yaml before create PreCreate func(*unstructured.Unstructured) error - // CustomOutPut will be executed after creating successfully. - CustomOutPut func(options *BaseOptions) - // CleanUpFn will be executed after creating failed. CleanUpFn func() error // CreateDependencies will be executed before creating. CreateDependencies CreateDependency - // ResourceNameGVRForCompletion resource name for completion. - ResourceNameGVRForCompletion schema.GroupVersionResource -} - -// BaseOptions the options of creation command should inherit baseOptions -type BaseOptions struct { - // Namespace k8s namespace - Namespace string `json:"namespace"` - - // Name Resource name of the command line operation - Name string `json:"name"` - - Dynamic dynamic.Interface `json:"-"` - - Client kubernetes.Interface `json:"-"` - - ToPrinter func(*meta.RESTMapping, bool) (printers.ResourcePrinterFunc, error) `json:"-"` - - Format printer.Format `json:"-"` - - DryRunStrategy string `json:"-"` - // Quiet minimize unnecessary output Quiet bool genericclioptions.IOStreams } -// BuildCommand build create command -func BuildCommand(inputs Inputs) *cobra.Command { - cmd := &cobra.Command{ - Use: inputs.Use, - Short: inputs.Short, - Example: inputs.Example, - ValidArgsFunction: util.ResourceNameCompletionFunc(inputs.Factory, inputs.ResourceNameGVRForCompletion), - Run: func(cmd *cobra.Command, args []string) { - util.CheckErr(inputs.BaseOptionsObj.Complete(inputs, args)) - util.CheckErr(inputs.BaseOptionsObj.Validate(inputs)) - util.CheckErr(inputs.BaseOptionsObj.Run(inputs)) - }, - } - if inputs.BuildFlags != nil { - inputs.BuildFlags(cmd) - } - return cmd -} - -func (o *BaseOptions) Complete(inputs Inputs, args []string) error { +func (o *CreateOptions) Complete() error { var err error - if o.Namespace, _, err = inputs.Factory.ToRawKubeConfigLoader().Namespace(); err != nil { + if o.Namespace, _, err = o.Factory.ToRawKubeConfigLoader().Namespace(); err != nil { return err } - if len(args) > 0 { - o.Name = args[0] + // now we use the first argument as the resource name + if len(o.Args) > 0 { + o.Name = o.Args[0] } - if o.Dynamic, err = inputs.Factory.DynamicClient(); err != nil { + if o.Dynamic, err = o.Factory.DynamicClient(); err != nil { return err } - if o.Client, err = inputs.Factory.KubernetesClientSet(); err != nil { + if o.Client, err = o.Factory.KubernetesClientSet(); err != nil { return err } @@ -200,73 +140,28 @@ func (o *BaseOptions) Complete(inputs Inputs, args []string) error { return p.PrintObj, nil } - // do custom options complete - if inputs.Complete != nil { - if err = inputs.Complete(); err != nil { - return err - } - } - return nil -} - -func (o *BaseOptions) Validate(inputs Inputs) error { - // do options validate - if inputs.Validate != nil { - if err := inputs.Validate(); err != nil { - return err - } - } return nil } // Run execute command. the options of parameter contain the command flags and args. -func (o *BaseOptions) Run(inputs Inputs) error { - var ( - cueValue cue.Value - err error - unstructuredObj *unstructured.Unstructured - optionsByte []byte - ) - - if optionsByte, err = json.Marshal(inputs.Options); err != nil { - return err - } - - if cueValue, err = newCueValue(inputs.CueTemplateName); err != nil { - return err - } - - if cueValue, err = fillOptions(cueValue, optionsByte); err != nil { - return err - } - - if unstructuredObj, err = convertContentToUnstructured(cueValue); err != nil { +func (o *CreateOptions) Run() error { + resObj, err := o.buildResourceObj() + if err != nil { return err } - if inputs.PreCreate != nil { - if err = inputs.PreCreate(unstructuredObj); err != nil { + if o.PreCreate != nil { + if err = o.PreCreate(resObj); err != nil { return err } } - group := inputs.Group - if len(group) == 0 { - group = types.AppsAPIGroup - } - - version := inputs.Version - if len(version) == 0 { - version = types.AppsAPIVersion - } - previewObj := unstructuredObj dryRunStrategy, err := o.GetDryRunStrategy() if err != nil { return err } if dryRunStrategy != DryRunClient { - gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: inputs.ResourceName} createOptions := metav1.CreateOptions{} if dryRunStrategy == DryRunServer { @@ -274,35 +169,35 @@ func (o *BaseOptions) Run(inputs Inputs) error { } // create dependencies - if inputs.CreateDependencies != nil { - if err = inputs.CreateDependencies(createOptions.DryRun); err != nil { + if o.CreateDependencies != nil { + if err = o.CreateDependencies(createOptions.DryRun); err != nil { return err } } // create kubernetes resource - previewObj, err = o.Dynamic.Resource(gvr).Namespace(o.Namespace).Create(context.TODO(), previewObj, createOptions) + resObj, err = o.Dynamic.Resource(o.GVR).Namespace(o.Namespace).Create(context.TODO(), resObj, createOptions) if err != nil { if apierrors.IsAlreadyExists(err) { return err } // for other errors, clean up dependencies - if cleanErr := o.CleanUp(inputs); cleanErr != nil { - fmt.Fprintf(o.ErrOut, "clean up denpendencies failed: %v\n", cleanErr) + if cleanErr := o.CleanUp(); cleanErr != nil { + fmt.Fprintf(o.ErrOut, "Failed to clean up denpendencies: %v\n", cleanErr) } return err } if dryRunStrategy != DryRunServer { - o.Name = previewObj.GetName() + o.Name = resObj.GetName() if o.Quiet { return nil } - if inputs.CustomOutPut != nil { - inputs.CustomOutPut(o) + if o.CustomOutPut != nil { + o.CustomOutPut(o) } else { - fmt.Fprintf(o.Out, "%s %s created\n", previewObj.GetKind(), previewObj.GetName()) + fmt.Fprintf(o.Out, "%s %s created\n", resObj.GetKind(), resObj.GetName()) } return nil } @@ -311,87 +206,57 @@ func (o *BaseOptions) Run(inputs Inputs) error { if err != nil { return err } - return printer.PrintObj(previewObj, o.Out) + return printer.PrintObj(resObj, o.Out) } -func (o *BaseOptions) CleanUp(inputs Inputs) error { - if inputs.CreateDependencies == nil { +func (o *CreateOptions) CleanUp() error { + if o.CreateDependencies == nil { return nil } - if inputs.CleanUpFn != nil { - return inputs.CleanUpFn() + if o.CleanUpFn != nil { + return o.CleanUpFn() } return nil } -// RunAsApply execute command. the options of parameter contain the command flags and args. -// if the resource exists, run as "kubectl apply". -func (o *BaseOptions) RunAsApply(inputs Inputs) error { +func (o *CreateOptions) buildResourceObj() (*unstructured.Unstructured, error) { var ( - cueValue cue.Value - err error - unstructuredObj *unstructured.Unstructured - optionsByte []byte + cueValue cue.Value + err error + optionsByte []byte ) - if optionsByte, err = json.Marshal(inputs.Options); err != nil { - return err - } - - if cueValue, err = newCueValue(inputs.CueTemplateName); err != nil { - return err + if optionsByte, err = json.Marshal(o.Options); err != nil { + return nil, err } - if cueValue, err = fillOptions(cueValue, optionsByte); err != nil { - return err + // append namespace and name to options and marshal to json + m := make(map[string]interface{}) + if err = json.Unmarshal(optionsByte, &m); err != nil { + return nil, err } - - if unstructuredObj, err = convertContentToUnstructured(cueValue); err != nil { - return err + m["namespace"] = o.Namespace + m["name"] = o.Name + if optionsByte, err = json.Marshal(m); err != nil { + return nil, err } - group := inputs.Group - if len(group) == 0 { - group = types.AppsAPIGroup + if cueValue, err = newCueValue(o.CueTemplateName); err != nil { + return nil, err } - version := inputs.Version - if len(version) == 0 { - version = types.AppsAPIVersion - } - // create k8s resource - gvr := schema.GroupVersionResource{Group: group, Version: version, Resource: inputs.ResourceName} - objectName, _, err := unstructured.NestedString(unstructuredObj.Object, "metadata", "name") - if err != nil { - return err - } - objectByte, err := json.Marshal(unstructuredObj) - if err != nil { - return err - } - if _, err := o.Dynamic.Resource(gvr).Namespace(o.Namespace).Patch( - context.TODO(), objectName, k8sapitypes.MergePatchType, - objectByte, metav1.PatchOptions{}); err != nil { - - // create object if not found - if apierrors.IsNotFound(err) { - if _, err = o.Dynamic.Resource(gvr).Namespace(o.Namespace).Create( - context.TODO(), unstructuredObj, metav1.CreateOptions{}); err != nil { - return err - } - } else { - return err - } + if cueValue, err = fillOptions(cueValue, optionsByte); err != nil { + return nil, err } - return nil + return convertContentToUnstructured(cueValue) } -func (o *BaseOptions) GetDryRunStrategy() (DryRunStrategy, error) { - if o.DryRunStrategy == "" { +func (o *CreateOptions) GetDryRunStrategy() (DryRunStrategy, error) { + if o.DryRun == "" { return DryRunNone, nil } - switch o.DryRunStrategy { + switch o.DryRun { case "client": return DryRunClient, nil case "server": @@ -401,7 +266,7 @@ func (o *BaseOptions) GetDryRunStrategy() (DryRunStrategy, error) { case "none": return DryRunNone, nil default: - return DryRunNone, fmt.Errorf(`invalid dry-run value (%v). Must be "none", "server", or "client"`, o.DryRunStrategy) + return DryRunNone, fmt.Errorf(`invalid dry-run value (%v). Must be "none", "server", or "client"`, o.DryRun) } } diff --git a/internal/cli/create/create_test.go b/internal/cli/create/create_test.go index b769f6b12..81c0d0b66 100755 --- a/internal/cli/create/create_test.go +++ b/internal/cli/create/create_test.go @@ -20,16 +20,14 @@ along with this program. If not, see . package create import ( + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/api/meta" "k8s.io/cli-runtime/pkg/genericclioptions" - "k8s.io/cli-runtime/pkg/printers" clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" - "k8s.io/kubectl/pkg/scheme" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/testing" @@ -37,19 +35,35 @@ import ( ) var _ = Describe("Create", func() { + const ( + clusterName = "test" + cueFileName = "create_template_test.cue" + ) + var ( - tf *cmdtesting.TestFactory - streams genericclioptions.IOStreams - baseOptions BaseOptions + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + options CreateOptions ) BeforeEach(func() { streams, _, _, _ = genericclioptions.NewTestIOStreams() tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) tf.Client = &clientfake.RESTClient{} - baseOptions = BaseOptions{ - Name: "test", - IOStreams: streams, + clusterOptions := map[string]interface{}{ + "clusterDefRef": "test-def", + "clusterVersionRef": "test-clusterversion-ref", + "components": []string{}, + "terminationPolicy": "Halt", + } + options = CreateOptions{ + Factory: tf, + Name: clusterName, + Namespace: testing.Namespace, + IOStreams: streams, + GVR: types.ClusterGVR(), + CueTemplateName: cueFileName, + Options: clusterOptions, } }) @@ -58,96 +72,19 @@ var _ = Describe("Create", func() { }) Context("Create Objects", func() { - It("test Create run", func() { - clusterOptions := map[string]interface{}{ - "name": "test", - "namespace": testing.Namespace, - "clusterDefRef": "test-def", - "clusterVersionRef": "test-clusterversion-ref", - "components": []string{}, - "terminationPolicy": "Halt", - } - - inputs := Inputs{ - CueTemplateName: "create_template_test.cue", - ResourceName: types.ResourceClusters, - BaseOptionsObj: &baseOptions, - Options: clusterOptions, - Factory: tf, - Validate: func() error { - return nil - }, - Complete: func() error { - return nil - }, - BuildFlags: func(cmd *cobra.Command) { - cmd.Flags().StringVar(&baseOptions.Namespace, "clusterDefRef", "", "cluster definition") - }, - } - cmd := BuildCommand(inputs) - Expect(cmd).ShouldNot(BeNil()) - Expect(cmd.Flags().Lookup("clusterDefRef")).ShouldNot(BeNil()) - - Expect(baseOptions.Complete(inputs, []string{})).Should(Succeed()) - Expect(baseOptions.Validate(inputs)).Should(Succeed()) - Expect(baseOptions.Run(inputs)).Should(Succeed()) + It("Complete", func() { + options.Args = []string{} + Expect(options.Complete()).Should(Succeed()) }) - It("test create dry-run", func() { - clusterOptions := map[string]interface{}{ - "name": "test", - "namespace": testing.Namespace, - "clusterDefRef": "test-def", - "clusterVersionRef": "test-clusterversion-ref", - "components": []string{}, - "terminationPolicy": "Halt", - } - - inputs := Inputs{ - CueTemplateName: "create_template_test.cue", - ResourceName: types.ResourceClusters, - BaseOptionsObj: &baseOptions, - Options: clusterOptions, - Factory: tf, - Validate: func() error { - return nil - }, - Complete: func() error { - baseOptions.ToPrinter = func(mapping *meta.RESTMapping, withNamespace bool) (printers.ResourcePrinterFunc, error) { - var p printers.ResourcePrinter - var err error - switch baseOptions.Format { - case printer.JSON: - p = &printers.JSONPrinter{} - case printer.YAML: - p = &printers.YAMLPrinter{} - default: - return nil, genericclioptions.NoCompatiblePrinterError{AllowedFormats: []string{"JOSN", "YAML"}} - } - - p, err = printers.NewTypeSetter(scheme.Scheme).WrapToPrinter(p, nil) - if err != nil { - return nil, err - } - return p.PrintObj, nil - } - return nil - }, - BuildFlags: func(cmd *cobra.Command) { - cmd.Flags().StringVar(&baseOptions.Namespace, "clusterDefRef", "", "cluster definition") - cmd.Flags().StringVar(&baseOptions.DryRunStrategy, "dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) - cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" - printer.AddOutputFlagForCreate(cmd, &baseOptions.Format) - }, - } - cmd := BuildCommand(inputs) - + It("test create with dry-run", func() { + options.Format = printer.YAML testCases := []struct { - clusterName string - isUseDryRun bool - mode string - dryRunStrateg DryRunStrategy - success bool + clusterName string + isUseDryRun bool + mode string + dryRunStrategy DryRunStrategy + success bool }{ { // test do not use dry-run strategy "test1", @@ -187,64 +124,22 @@ var _ = Describe("Create", func() { } for _, t := range testCases { - clusterOptions["name"] = t.clusterName - Expect(cmd).ShouldNot(BeNil()) - Expect(cmd.Flags().Lookup("clusterDefRef")).ShouldNot(BeNil()) - Expect(cmd.Flags().Lookup("dry-run")).ShouldNot(BeNil()) - Expect(cmd.Flags().Lookup("output")).ShouldNot(BeNil()) + By(fmt.Sprintf("when isDryRun %v, dryRunStrategy %v, mode %s", + t.isUseDryRun, t.dryRunStrategy, t.mode)) + options.Name = t.clusterName if t.isUseDryRun { - Expect(cmd.Flags().Set("dry-run", t.mode)).Should(Succeed()) + options.DryRun = t.mode } + Expect(options.Complete()).Should(Succeed()) - Expect(baseOptions.Complete(inputs, []string{})).Should(Succeed()) - Expect(baseOptions.Validate(inputs)).Should(Succeed()) - - dryRunStrateg, _ := baseOptions.GetDryRunStrategy() + s, _ := options.GetDryRunStrategy() if t.success { - Expect(dryRunStrateg == t.dryRunStrateg).Should(BeTrue()) - Expect(baseOptions.Run(inputs)).Should(Succeed()) + Expect(s == t.dryRunStrategy).Should(BeTrue()) + Expect(options.Run()).Should(Succeed()) } else { - Expect(dryRunStrateg == t.dryRunStrateg).Should(BeFalse()) + Expect(s).ShouldNot(Equal(t.dryRunStrategy)) } } }) - - It("test Create runAsApply", func() { - clusterOptions := map[string]interface{}{ - "name": "test-apply", - "namespace": testing.Namespace, - "clusterDefRef": "test-def", - "clusterVersionRef": "test-clusterversion-ref", - "components": []string{}, - "terminationPolicy": "Halt", - } - - inputs := Inputs{ - CueTemplateName: "create_template_test.cue", - ResourceName: types.ResourceClusters, - BaseOptionsObj: &baseOptions, - Options: clusterOptions, - Factory: tf, - Validate: func() error { - return nil - }, - Complete: func() error { - return nil - }, - BuildFlags: func(cmd *cobra.Command) { - cmd.Flags().StringVar(&baseOptions.Namespace, "clusterDefRef", "", "cluster definition") - }, - } - cmd := BuildCommand(inputs) - Expect(cmd).ShouldNot(BeNil()) - Expect(cmd.Flags().Lookup("clusterDefRef")).ShouldNot(BeNil()) - - Expect(baseOptions.Complete(inputs, []string{})).Should(Succeed()) - Expect(baseOptions.Validate(inputs)).Should(Succeed()) - // create - Expect(baseOptions.RunAsApply(inputs)).Should(Succeed()) - // apply if exists - Expect(baseOptions.RunAsApply(inputs)).Should(Succeed()) - }) }) }) diff --git a/internal/cli/delete/delete.go b/internal/cli/delete/delete.go index 13c40787d..73c01b96d 100644 --- a/internal/cli/delete/delete.go +++ b/internal/cli/delete/delete.go @@ -213,15 +213,16 @@ func (o *DeleteOptions) deleteResource(info *resource.Info, deleteOptions *metav } func (o *DeleteOptions) preDeleteResource(info *resource.Info) error { - if o.PreDeleteHook != nil { - if info.Object == nil { - if err := info.Get(); err != nil { - return err - } + if o.PreDeleteHook == nil { + return nil + } + + if info.Object == nil { + if err := info.Get(); err != nil { + return err } - return o.PreDeleteHook(o, info.Object) } - return nil + return o.PreDeleteHook(o, info.Object) } func (o *DeleteOptions) postDeleteResource(object runtime.Object) error { From 7f0aa18291027ace3558cb3782e2217d8a930059 Mon Sep 17 00:00:00 2001 From: xingran Date: Tue, 9 May 2023 10:39:55 +0800 Subject: [PATCH 250/439] chore: revert pg-replication componentDef to compatible v0.4.5 standalone postgres (#3145) --- .../templates/clusterdefinition.yaml | 290 ------------------ internal/cli/cluster/helper.go | 2 +- 2 files changed, 1 insertion(+), 291 deletions(-) diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 35a1e6009..c77427c7b 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -310,293 +310,3 @@ spec: statements: creation: CREATE USER $(USERNAME) WITH REPLICATION PASSWORD '$(PASSWD)'; update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; - ## The following componentDef definition is for forward compatibility with the v0.4.5 single-node postgres cluster. - ## It is obsolete and should not be used in versions after 0.5.0. Subsequent versions will remove this componentDef. - - name: pg-replication - workloadType: Replication - characterType: postgresql - probes: - monitor: - builtIn: false - exporterConfig: - scrapePath: /metrics - scrapePort: 9187 - logConfigs: - {{- range $name,$pattern := .Values.logConfigs }} - - name: {{ $name }} - filePathPattern: {{ $pattern }} - {{- end }} - configSpecs: - - name: postgresql-configuration - templateRef: postgresql-configuration - constraintRef: postgresql14-cc - keys: - - postgresql.conf - namespace: {{ .Release.Namespace }} - volumeName: postgresql-config - defaultMode: 0777 - - name: postgresql-custom-metrics - templateRef: postgresql14-custom-metrics - namespace: {{ .Release.Namespace }} - volumeName: postgresql-custom-metrics - defaultMode: 0777 - scriptSpecs: - - name: postgresql-scripts - templateRef: postgresql-scripts - namespace: {{ .Release.Namespace }} - volumeName: scripts - defaultMode: 0777 - service: - ports: - - name: tcp-postgresql - protocol: TCP - port: 5432 - targetPort: tcp-postgresql - - name: http-metrics-postgresql - port: 9187 - targetPort: http-metrics - volumeTypes: - - name: data - type: data - podSpec: - securityContext: - fsGroup: 1001 - containers: - - name: postgresql - imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} - securityContext: - runAsUser: 0 - volumeMounts: - - name: dshm - mountPath: /dev/shm - - name: data - mountPath: /postgresql - - name: postgresql-config - mountPath: /postgresql/conf - - name: scripts - mountPath: /scripts - ports: - - name: tcp-postgresql - containerPort: 5432 - command: - - /scripts/setup.sh - livenessProbe: - failureThreshold: 6 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - exec: - command: - - /bin/sh - - -c - - exec pg_isready -U {{ default "postgres" | quote }} -h 127.0.0.1 -p 5432 - readinessProbe: - failureThreshold: 6 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - exec: - command: - - /bin/sh - - -c - - -ee - - | - exec pg_isready -U {{ default "postgres" | quote }} -h 127.0.0.1 -p 5432 - [ -f /postgresql/tmp/.initialized ] || [ -f /postgresql/.initialized ] - env: - - name: BITNAMI_DEBUG - value: "false" - - name: POSTGRESQL_PORT_NUMBER - value: "5432" - - name: POSTGRESQL_VOLUME_DIR - value: /postgresql - - name: PGDATA - value: /postgresql/data - - name: PGUSER - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: username - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: postgres-password - # Authentication - - name: POSTGRES_USER - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: username - - name: POSTGRES_POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: postgres-password - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: postgres-password - - name: POSTGRES_DB - value: {{ (include "postgresql.database" .) | quote }} - # Audit - - name: POSTGRESQL_LOG_HOSTNAME - value: {{ .Values.audit.logHostname | quote }} - - name: POSTGRESQL_LOG_CONNECTIONS - value: {{ .Values.audit.logConnections | quote }} - - name: POSTGRESQL_LOG_DISCONNECTIONS - value: {{ .Values.audit.logDisconnections | quote }} - {{- if .Values.audit.logLinePrefix }} - - name: POSTGRESQL_LOG_LINE_PREFIX - value: {{ .Values.audit.logLinePrefix | quote }} - {{- end }} - {{- if .Values.audit.logTimezone }} - - name: POSTGRESQL_LOG_TIMEZONE - value: {{ .Values.audit.logTimezone | quote }} - {{- end }} - {{- if .Values.audit.pgAuditLog }} - - name: POSTGRESQL_PGAUDIT_LOG - value: {{ .Values.audit.pgAuditLog | quote }} - {{- end }} - - name: POSTGRESQL_PGAUDIT_LOG_CATALOG - value: {{ .Values.audit.pgAuditLogCatalog | quote }} - # Others - - name: POSTGRESQL_CLIENT_MIN_MESSAGES - value: {{ .Values.audit.clientMinMessages | quote }} - - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES - value: {{ .Values.postgresqlSharedPreloadLibraries | quote }} - {{- if .Values.primary.extraEnvVars }} - {{- include "tplvalues.render" (dict "value" .Values.primary.extraEnvVars "context" $) | nindent 12 }} - {{- end }} - {{- if or .Values.primary.extraEnvVarsCM .Values.primary.extraEnvVarsSecret }} - envFrom: - {{- if .Values.primary.extraEnvVarsCM }} - - configMapRef: - name: {{ .Values.primary.extraEnvVarsCM }} - {{- end }} - {{- if .Values.primary.extraEnvVarsSecret }} - - secretRef: - name: {{ .Values.primary.extraEnvVarsSecret }} - {{- end }} - {{- end }} - - name: metrics - image: {{ .Values.metrics.image.registry | default "docker.io" }}/{{ .Values.metrics.image.repository }}:{{ .Values.metrics.image.tag }} - imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }} - securityContext: - runAsUser: 0 - env: - {{- $database := "postgres" }} - {{- $sslmode := "disable" }} - - name: DATA_SOURCE_URI - value: {{ printf "127.0.0.1:5432/%s?sslmode=%s" $database $sslmode }} - - name: DATA_SOURCE_PASS - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: postgres-password - - name: DATA_SOURCE_USER - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: username - command: - - "/opt/bitnami/postgres-exporter/bin/postgres_exporter" - - "--auto-discover-databases" - - "--extend.query-path=/opt/conf/custom-metrics.yaml" - - "--exclude-databases=template0,template1" - - "--log.level=info" - ports: - - name: http-metrics - containerPort: 9187 - livenessProbe: - failureThreshold: 6 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - httpGet: - path: / - port: http-metrics - readinessProbe: - failureThreshold: 6 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - httpGet: - path: / - port: http-metrics - volumeMounts: - - name: postgresql-custom-metrics - mountPath: /opt/conf - volumes: - - name: dshm - emptyDir: - medium: Memory - {{- with .Values.shmVolume.sizeLimit }} - sizeLimit: {{ . }} - {{- end }} - systemAccounts: - cmdExecutorConfig: - image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ default .Values.image.tag .Chart.AppVersion }} - command: - - psql - args: - - -h$(KB_ACCOUNT_ENDPOINT) - - -c - - $(KB_ACCOUNT_STATEMENT) - env: - - name: PGUSER - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: username - - name: PGPASSWORD - valueFrom: - secretKeyRef: - name: $(CONN_CREDENTIAL_SECRET_NAME) - key: postgres-password - passwordConfig: - length: 10 - numDigits: 5 - numSymbols: 0 - letterCase: MixedCases - accounts: - - name: kbadmin - provisionPolicy: - type: CreateByStmt - scope: AnyPods - statements: - creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)'; - deletion: DROP USER IF EXISTS $(USERNAME); - - name: kbdataprotection - provisionPolicy: - type: CreateByStmt - scope: AnyPods - statements: - creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)'; - deletion: DROP USER IF EXISTS $(USERNAME); - - name: kbprobe - provisionPolicy: - type: CreateByStmt - scope: AnyPods - statements: - creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; GRANT pg_monitor TO $(USERNAME); - deletion: DROP USER IF EXISTS $(USERNAME); - - name: kbmonitoring - provisionPolicy: - type: CreateByStmt - scope: AnyPods - statements: - creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; GRANT pg_monitor TO $(USERNAME); - deletion: DROP USER IF EXISTS $(USERNAME); - - name: kbreplicator - provisionPolicy: - type: CreateByStmt - scope: AnyPods - statements: - creation: CREATE USER $(USERNAME) WITH REPLICATION PASSWORD '$(PASSWD)'; - deletion: DROP USER IF EXISTS $(USERNAME); diff --git a/internal/cli/cluster/helper.go b/internal/cli/cluster/helper.go index 69e374805..50bebdf89 100644 --- a/internal/cli/cluster/helper.go +++ b/internal/cli/cluster/helper.go @@ -243,7 +243,7 @@ func GetClusterDefByName(dynamic dynamic.Interface, name string) (*appsv1alpha1. } func GetDefaultCompName(cd *appsv1alpha1.ClusterDefinition) (string, error) { - if len(cd.Spec.ComponentDefs) > 0 { + if len(cd.Spec.ComponentDefs) == 1 { return cd.Spec.ComponentDefs[0].Name, nil } return "", fmt.Errorf("failed to get the default component definition name") From d1941332a7fd1a8d2dcef1d34d10a3fff11c0988 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Tue, 9 May 2023 14:20:43 +0800 Subject: [PATCH 251/439] Revert "fix: drop followers only when scaling in" (#3153) --- cmd/manager/main.go | 9 ++ controllers/apps/cluster_controller_test.go | 17 +-- controllers/apps/components/component.go | 33 ----- .../apps/components/deployment_controller.go | 6 - controllers/apps/components/pod_controller.go | 124 ++++++++++++++++++ .../components/stateful_set_controller.go | 5 - .../templates/clusterdefinition.yaml | 3 - deploy/apecloud-mysql/templates/scripts.yaml | 32 +---- internal/constant/const.go | 1 - 9 files changed, 141 insertions(+), 89 deletions(-) create mode 100644 controllers/apps/components/pod_controller.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 529952614..43b206de9 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -393,6 +393,15 @@ func main() { os.Exit(1) } + if err = (&components.PodReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("pod-controller"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Pod") + os.Exit(1) + } + if err = (&appscontrollers.ComponentClassReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 640ca2417..0f563904e 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -28,8 +28,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/viper" @@ -413,7 +411,6 @@ var _ = Describe("Cluster Controller", func() { Name: stsName + "-" + strconv.Itoa(i), Namespace: testCtx.DefaultNamespace, Labels: map[string]string{ - constant.AppManagedByLabelKey: constant.AppName, constant.AppInstanceLabelKey: clusterName, constant.KBAppComponentLabelKey: componentName, appsv1.ControllerRevisionHashLabelKey: "mock-version", @@ -945,10 +942,9 @@ var _ = Describe("Cluster Controller", func() { sts = &stsList.Items[0] }).Should(Succeed()) - By("Creating mock pods in StatefulSet, and set controller reference") + By("Creating mock pods in StatefulSet") pods := mockPodsForConsensusTest(clusterObj, replicas) for _, pod := range pods { - Expect(controllerutil.SetControllerReference(sts, &pod, scheme.Scheme)).Should(Succeed()) Expect(testCtx.CreateObj(testCtx.Ctx, &pod)).Should(Succeed()) // mock the status to pass the isReady(pod) check in consensus_set pod.Status.Conditions = []corev1.PodCondition{{ @@ -986,17 +982,6 @@ var _ = Describe("Cluster Controller", func() { g.Expect(followerCount).Should(Equal(2)) }).Should(Succeed()) - By("Checking pods' annotations") - Eventually(func(g Gomega) { - pods, err := util.GetPodListByStatefulSet(ctx, k8sClient, sts) - g.Expect(err).ShouldNot(HaveOccurred()) - g.Expect(len(pods)).Should(Equal(int(*sts.Spec.Replicas))) - for _, pod := range pods { - g.Expect(pod.Annotations).ShouldNot(BeNil()) - g.Expect(pod.Annotations[constant.ComponentReplicasAnnotationKey]).Should(Equal(strconv.Itoa(int(*sts.Spec.Replicas)))) - } - }, time.Second*100).Should(Succeed()) - By("Updating StatefulSet's status") sts.Status.UpdateRevision = "mock-version" sts.Status.Replicas = int32(replicas) diff --git a/controllers/apps/components/component.go b/controllers/apps/components/component.go index 98f3d5a8b..d5250240d 100644 --- a/controllers/apps/components/component.go +++ b/controllers/apps/components/component.go @@ -21,11 +21,9 @@ package components import ( "context" - "strconv" "time" "golang.org/x/exp/slices" - corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -186,34 +184,3 @@ func patchWorkloadCustomLabel( } return nil } - -func updateComponentInfoToPods( - ctx context.Context, - cli client.Client, - cluster *appsv1alpha1.Cluster, - componentSpec *appsv1alpha1.ClusterComponentSpec) error { - ml := client.MatchingLabels{ - constant.AppInstanceLabelKey: cluster.GetName(), - constant.KBAppComponentLabelKey: componentSpec.Name, - } - podList := corev1.PodList{} - if err := cli.List(ctx, &podList, ml); err != nil { - return err - } - replicasStr := strconv.Itoa(int(componentSpec.Replicas)) - for _, pod := range podList.Items { - if pod.Annotations != nil && - pod.Annotations[constant.ComponentReplicasAnnotationKey] == replicasStr { - continue - } - patch := client.MergeFrom(pod.DeepCopy()) - if pod.Annotations == nil { - pod.Annotations = make(map[string]string) - } - pod.Annotations[constant.ComponentReplicasAnnotationKey] = replicasStr - if err := cli.Patch(ctx, &pod, patch); err != nil { - return err - } - } - return nil -} diff --git a/controllers/apps/components/deployment_controller.go b/controllers/apps/components/deployment_controller.go index 6a19044ab..8545e4857 100644 --- a/controllers/apps/components/deployment_controller.go +++ b/controllers/apps/components/deployment_controller.go @@ -76,11 +76,6 @@ func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) return workloadCompClusterReconcile(reqCtx, r.Client, deploy, func(cluster *appsv1alpha1.Cluster, componentSpec *appsv1alpha1.ClusterComponentSpec, component types.Component) (ctrl.Result, error) { compCtx := newComponentContext(reqCtx, r.Client, r.Recorder, component, deploy, componentSpec) - // update component info to pods' annotations - if err := updateComponentInfoToPods(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { - reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "StatefulSet Deploy updateComponentInfoToPods Failed", err.Error()) - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } // patch the current componentSpec workload's custom labels if err := patchWorkloadCustomLabel(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "Deployment Controller PatchWorkloadCustomLabelFailed", err.Error()) @@ -101,7 +96,6 @@ func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&appsv1.Deployment{}). Owns(&appsv1.ReplicaSet{}). - Owns(&corev1.Pod{}). WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). Named("deployment-watcher"). Complete(r) diff --git a/controllers/apps/components/pod_controller.go b/controllers/apps/components/pod_controller.go new file mode 100644 index 000000000..6d62e2196 --- /dev/null +++ b/controllers/apps/components/pod_controller.go @@ -0,0 +1,124 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package components + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/controllers/apps/components/util" + "github.com/apecloud/kubeblocks/internal/constant" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" +) + +// PodReconciler reconciles a Pod object +type PodReconciler struct { + client.Client + Scheme *runtime.Scheme + Recorder record.EventRecorder +} + +// +kubebuilder:rbac:groups=apps,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=pods/status,verbs=get +// +kubebuilder:rbac:groups=apps,resources=pods/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile +func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + var ( + pod = &corev1.Pod{} + err error + cluster *appsv1alpha1.Cluster + ok bool + componentName string + componentStatus appsv1alpha1.ClusterComponentStatus + ) + + reqCtx := intctrlutil.RequestCtx{ + Ctx: ctx, + Req: req, + Log: log.FromContext(ctx).WithValues("pod", req.NamespacedName), + } + + if err = r.Client.Get(reqCtx.Ctx, reqCtx.Req.NamespacedName, pod); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } + + // skip if pod is being deleted + if !pod.DeletionTimestamp.IsZero() { + return intctrlutil.Reconciled() + } + + if cluster, err = util.GetClusterByObject(reqCtx.Ctx, r.Client, pod); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } + if cluster == nil { + return intctrlutil.Reconciled() + } + + if componentName, ok = pod.Labels[constant.KBAppComponentLabelKey]; !ok { + return intctrlutil.Reconciled() + } + + if cluster.Status.Components == nil { + return intctrlutil.Reconciled() + } + if componentStatus, ok = cluster.Status.Components[componentName]; !ok { + return intctrlutil.Reconciled() + } + if componentStatus.ConsensusSetStatus == nil { + return intctrlutil.Reconciled() + } + if componentStatus.ConsensusSetStatus.Leader.Pod == util.ComponentStatusDefaultPodName { + return intctrlutil.Reconciled() + } + + // sync leader status from cluster.status + patch := client.MergeFrom(pod.DeepCopy()) + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + pod.Annotations[constant.LeaderAnnotationKey] = componentStatus.ConsensusSetStatus.Leader.Pod + if err = r.Client.Patch(reqCtx.Ctx, pod, patch); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } + r.Recorder.Eventf(pod, corev1.EventTypeNormal, "AddAnnotation", "add annotation %s=%s", constant.LeaderAnnotationKey, componentStatus.ConsensusSetStatus.Leader.Pod) + return intctrlutil.Reconciled() +} + +// SetupWithManager sets up the controller with the Manager. +func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.Pod{}). + WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). + Named("pod-watcher"). + Complete(r) +} diff --git a/controllers/apps/components/stateful_set_controller.go b/controllers/apps/components/stateful_set_controller.go index 9c8d697c9..40a246f90 100644 --- a/controllers/apps/components/stateful_set_controller.go +++ b/controllers/apps/components/stateful_set_controller.go @@ -76,11 +76,6 @@ func (r *StatefulSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) return workloadCompClusterReconcile(reqCtx, r.Client, sts, func(cluster *appsv1alpha1.Cluster, componentSpec *appsv1alpha1.ClusterComponentSpec, component types.Component) (ctrl.Result, error) { compCtx := newComponentContext(reqCtx, r.Client, r.Recorder, component, sts, componentSpec) - // update component info to pods' annotations - if err := updateComponentInfoToPods(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { - reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "StatefulSet Controller updateComponentInfoToPods Failed", err.Error()) - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } // patch the current componentSpec workload's custom labels if err := patchWorkloadCustomLabel(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "StatefulSet Controller PatchWorkloadCustomLabelFailed", err.Error()) diff --git a/deploy/apecloud-mysql/templates/clusterdefinition.yaml b/deploy/apecloud-mysql/templates/clusterdefinition.yaml index d1843f9b5..c64cf222b 100644 --- a/deploy/apecloud-mysql/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql/templates/clusterdefinition.yaml @@ -170,9 +170,6 @@ spec: - path: "leader" fieldRef: fieldPath: metadata.annotations['cs.apps.kubeblocks.io/leader'] - - path: "component-replicas" - fieldRef: - fieldPath: metadata.annotations['apps.kubeblocks.io/component-replicas'] systemAccounts: cmdExecutorConfig: image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index 3019304e5..62669181f 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -15,10 +15,6 @@ data: idx=${KB_POD_NAME##*-} host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) echo "host=$host" - # update replicas to persistent file - component_replicas_path=/data/mysql/.kb_component_replicas - current_component_replicas=`cat /etc/annotations/component-replicas` - echo $current_component_replicas > $component_replicas_path if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then echo "no leader or self is leader, no need to call add." else @@ -74,7 +70,7 @@ data: mkdir -p /data/mysql/data /data/mysql/log chmod +777 -R /data/mysql; echo "KB_MYSQL_CLUSTER_UID=$KB_MYSQL_CLUSTER_UID" - cluster_uid_path=/data/mysql/.kb_cluster_uid + cluster_uid_path=/data/mysql/data/.kb_cluster_uid if [ -f $cluster_uid_path ] && [ ! -f /data/mysql/data/.restore_new_cluster ]; then last_cluster_uid=`cat $cluster_uid_path` if [ "$last_cluster_uid" != "$KB_MYSQL_CLUSTER_UID" ]; then @@ -159,17 +155,16 @@ data: done pre-stop.sh: | #!/bin/bash - drop_followers() { leader=`cat /etc/annotations/leader` - echo "leader=$leader" >> /data/mysql/.kb_pre_stop.log - echo "KB_POD_NAME=$KB_POD_NAME" >> /data/mysql/.kb_pre_stop.log + echo "leader=$leader" + echo "KB_POD_NAME=$KB_POD_NAME" if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then - echo "no leader or self is leader, exit" >> /data/mysql/.kb_pre_stop.log + echo "no leader or self is leader, exit" exit 0 fi idx=${KB_POD_NAME##*-} host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) - echo "host=$host" >> /data/mysql/.kb_pre_stop.log + echo "host=$host" leader_idx=${leader##*-} leader_host=$(eval echo \$KB_MYSQL_"$leader_idx"_HOSTNAME) if [ ! -z $leader_host ]; then @@ -178,20 +173,7 @@ data: if [ ! -z $MYSQL_ROOT_PASSWORD ]; then password_flag="-p$MYSQL_ROOT_PASSWORD" fi - echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.downgrade_follower('$host:13306');\" 2>&1 " >> /data/mysql/.kb_pre_stop.log + echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.downgrade_follower('$host:13306');\" 2>&1 " mysql $host_flag -uroot $password_flag -e "call dbms_consensus.downgrade_follower('$host:13306');" 2>&1 - echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.drop_learner('$host:13306');\" 2>&1 " >> /data/mysql/.kb_pre_stop.log + echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.drop_learner('$host:13306');\" 2>&1 " mysql $host_flag -uroot $password_flag -e "call dbms_consensus.drop_learner('$host:13306');" 2>&1 - } - component_replicas_path=/data/mysql/.kb_component_replicas - current_component_replicas=`cat /etc/annotations/component-replicas` - if [ -f $component_replicas_path ]; then - last_component_replicas=`cat $component_replicas_path` - # check is scaling in but not scaling in to 0 - if [ "$last_component_replicas" -gt "$current_component_replicas" ] && [ $current_component_replicas -ne 0 ]; then - # only scaling in need to drop followers - drop_followers - else - echo "no need to drop followers" - fi - fi diff --git a/internal/constant/const.go b/internal/constant/const.go index aebdd9a67..d6f4254a9 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -94,7 +94,6 @@ const ( RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" - ComponentReplicasAnnotationKey = "apps.kubeblocks.io/component-replicas" // ComponentReplicasAnnotationKey specifies the number of pods in replicas DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" // DefaultBackupPolicyAnnotationKey specifies the default backup policy. DefaultBackupPolicyTemplateAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy-template" // DefaultBackupPolicyTemplateAnnotationKey specifies the default backup policy template. BackupDataPathPrefixAnnotationKey = "dataprotection.kubeblocks.io/path-prefix" // BackupDataPathPrefixAnnotationKey specifies the backup data path prefix. From a2d83dd4f660b334ec25d72588b24fe1b63415e0 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Tue, 9 May 2023 14:44:38 +0800 Subject: [PATCH 252/439] fix: pitr bugs (#3142) --- .../dataprotection/backuppolicy_controller.go | 3 +- deploy/csi-s3/templates/_helpers.tpl | 18 ++++- deploy/csi-s3/values.yaml | 2 +- deploy/postgresql/config/pg12-config.tpl | 2 +- deploy/postgresql/config/pg14-config.tpl | 2 +- .../templates/backuppolicytemplate.yaml | 2 +- .../postgresql/templates/backuptool-pitr.yaml | 8 ++- deploy/postgresql/templates/scripts.yaml | 4 +- .../lifecycle/cluster_plan_builder.go | 5 ++ internal/controller/plan/pitr.go | 68 ++++++++++++++----- internal/controller/plan/pitr_test.go | 22 ++++++ internal/controllerutil/errors.go | 1 + internal/testutil/apps/pod_factory.go | 5 ++ 13 files changed, 114 insertions(+), 28 deletions(-) diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index 9e603dfe4..fe73e3b07 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -25,6 +25,7 @@ import ( "fmt" "reflect" "sort" + "strings" "github.com/leaanthony/debme" "github.com/spf13/viper" @@ -203,7 +204,7 @@ func (r *BackupPolicyReconciler) removeExpiredBackups(reqCtx intctrlutil.Request now := metav1.Now() for _, item := range backups.Items { // ignore retained backup. - if item.GetLabels()[constant.BackupProtectionLabelKey] == constant.BackupRetain { + if strings.EqualFold(item.GetLabels()[constant.BackupProtectionLabelKey], constant.BackupRetain) { continue } if item.Status.Expiration != nil && item.Status.Expiration.Before(&now) { diff --git a/deploy/csi-s3/templates/_helpers.tpl b/deploy/csi-s3/templates/_helpers.tpl index e84ca23ce..92c75e011 100644 --- a/deploy/csi-s3/templates/_helpers.tpl +++ b/deploy/csi-s3/templates/_helpers.tpl @@ -2,9 +2,21 @@ Expand the mountOptions of the storageClass. */}} {{- define "storageClass.mountOptions" -}} -{{- if .Values.secret.region }} -{{- printf "%s --region %s" .Values.storageClass.mountOptions .Values.secret.region }} +{{- if eq .Values.storageClass.mounter "geesefs" }} + {{- if hasSuffix ".aliyuncs.com" .Values.secret.endpoint }} + {{- printf "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 --subdomain %s" .Values.storageClass.mountOptions }} + {{- else if .Values.secret.region }} + {{- printf "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 --region %s %s" .Values.storageClass.mountOptions .Values.secret.region }} + {{- else }} + {{- printf "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 %s" .Values.storageClass.mountOptions }} + {{- end }} +{{- else if eq .Values.storageClass.mounter "s3fs" }} + {{- if hasSuffix ".aliyuncs.com" .Values.secret.endpoint }} + {{- .Values.storageClass.mountOptions }} + {{- else }} + {{- printf "-o use_path_request_style %s" .Values.storageClass.mountOptions }} + {{- end }} {{- else }} -{{- .Values.storageClass.mountOptions }} + {{- .Values.storageClass.mountOptions }} {{- end }} {{- end }} \ No newline at end of file diff --git a/deploy/csi-s3/values.yaml b/deploy/csi-s3/values.yaml index 5ccd7faf6..448776e1c 100644 --- a/deploy/csi-s3/values.yaml +++ b/deploy/csi-s3/values.yaml @@ -30,7 +30,7 @@ storageClass: # aliyun OSS only support s3fs, and DO NOT set "-o use_path_request_style": # mounter: s3fs # mountOptions: "" - mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" + mountOptions: "" # Volume reclaim policy reclaimPolicy: Retain # Annotations for the storage class diff --git a/deploy/postgresql/config/pg12-config.tpl b/deploy/postgresql/config/pg12-config.tpl index 70fd87862..f68edd8c3 100644 --- a/deploy/postgresql/config/pg12-config.tpl +++ b/deploy/postgresql/config/pg12-config.tpl @@ -30,7 +30,7 @@ listen_addresses = '*' port = '5432' # archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' -archive_command = '[[ $(date +%H%M) == 1200 ]] && rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' +archive_command = 'if [ $(date +%H%M) -eq 1200 ]; then rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); fi; mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz && sync /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' archive_mode = 'on' auto_explain.log_analyze = 'False' auto_explain.log_buffers = 'False' diff --git a/deploy/postgresql/config/pg14-config.tpl b/deploy/postgresql/config/pg14-config.tpl index d56c52d18..12b2e3c39 100644 --- a/deploy/postgresql/config/pg14-config.tpl +++ b/deploy/postgresql/config/pg14-config.tpl @@ -27,7 +27,7 @@ listen_addresses = '*' port = '5432' # archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' -archive_command = '[[ $(date +%H%M) == 1200 ]] && rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' +archive_command = 'if [ $(date +%H%M) -eq 1200 ]; then rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); fi; mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz && sync /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' archive_mode = 'on' auto_explain.log_analyze = 'True' auto_explain.log_min_duration = '1s' diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index 86a9a9d80..2ed418d3c 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -26,7 +26,7 @@ spec: hooks: containerName: postgresql preCommands: - - psql -c "SELECT txid_current();CHECKPOINT;" + - psql -c "CHECKPOINT;" backupStatusUpdates: - path: manifests.backupLog containerName: postgresql diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml index f79b0823a..22f76e4f8 100644 --- a/deploy/postgresql/templates/backuptool-pitr.yaml +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -40,7 +40,8 @@ spec: mkdir -p ${PITR_DIR}; cd ${PITR_DIR} for i in $(find ${BACKUP_DIR} -name "*.gz"); do - cp ${i} $(basename $i) + echo "copying ${i}"; + cp ${i} $(basename $i); gzip -df $(basename $i); done chmod 777 -R ${PITR_DIR}; @@ -54,6 +55,7 @@ spec: chmod +x ${RESTORE_SCRIPT_DIR}/kb_restore.sh; echo "restore_command='mv ${PITR_DIR}/%f %p'\nrecovery_target_time='${RECOVERY_TIME}'\nrecovery_target_action='promote'" > ${CONF_DIR}/recovery.conf; mv ${DATA_DIR} ${DATA_DIR}.old; + echo "done."; sync; backupCommands: - | @@ -63,12 +65,16 @@ spec: TODAY_INCR_LOG=${BACKUP_DIR}/$(date +%Y%m%d); mkdir -p ${TODAY_INCR_LOG}; for i in $(find ${ARCHIVE_LOG_DIR} -name "*.gz"); do + echo "uploading ${i}"; mv -f ${i} ${TODAY_INCR_LOG}/; done if [ -d ${LOG_DIR} ]; then cd ${LOG_DIR}; LATEST_LOG=$(ls -t . | grep '[[:digit:]]$\|.partial$'|head -n 1); + echo "uploading ${TODAY_INCR_LOG}/${LATEST_LOG}.gz"; gzip -kqc ${LATEST_LOG} > ${TODAY_INCR_LOG}/${LATEST_LOG}.gz; fi + echo "done." + sync; type: pitr \ No newline at end of file diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index a1b930a88..647644d02 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -104,8 +104,8 @@ data: set -o errexit set -o nounset SHOW_START_TIME=$1 - LOG_START_TIME=$(pg_waldump $(ls -tr $PGDATA/pg_wal/ | grep '[[:digit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') - LOG_STOP_TIME=$(pg_waldump $(ls -t $PGDATA/pg_wal/ | grep '[[:digit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + LOG_START_TIME=$(pg_waldump $(ls -Ftr $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + LOG_STOP_TIME=$(pg_waldump $(ls -Ft $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') LOG_START_TIME=$(date -d "$LOG_START_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') LOG_STOP_TIME=$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') [[ $SHOW_START_TIME == "false" ]] && printf "{\"stopTime\": \"$LOG_STOP_TIME\"}" || printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}" \ No newline at end of file diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 36f21837f..c2ea6d922 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "reflect" + "strings" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -270,6 +271,10 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { } // delete secondary objects if _, ok := node.obj.(*appsv1alpha1.Cluster); !ok { + // retain backup for data protection even if the cluster is wiped out. + if strings.EqualFold(node.obj.GetLabels()[constant.BackupProtectionLabelKey], constant.BackupRetain) { + return nil + } // check dependency resources has been deleted before deleting the resource err := c.checkDependencyResourcesDeleted(node) if err != nil { diff --git a/internal/controller/plan/pitr.go b/internal/controller/plan/pitr.go index 7a71d9733..d328f229e 100644 --- a/internal/controller/plan/pitr.go +++ b/internal/controller/plan/pitr.go @@ -133,7 +133,7 @@ func (p *PointInTimeRecoveryManager) doRecoveryJob() (shouldRequeue bool, err er // mount the data+log pvc, and run scripts job to prepare data if err = p.runRecoveryJob(); err != nil { - if err.Error() == "waiting PVC Bound" { + if intctrlutil.IsTargetError(err, intctrlutil.ErrorTypeNeedWaiting) { return true, nil } return false, err @@ -393,6 +393,48 @@ func (p *PointInTimeRecoveryManager) getIncrementalPVC(componentName string) (*c return &pvc, nil } +func (p *PointInTimeRecoveryManager) getDataPVCs(componentName string) ([]corev1.PersistentVolumeClaim, error) { + podList := corev1.PodList{} + podLabels := map[string]string{ + constant.AppInstanceLabelKey: p.Cluster.Name, + constant.KBAppComponentLabelKey: componentName, + } + if err := p.Client.List(p.Ctx, &podList, + client.InNamespace(p.namespace), + client.MatchingLabels(podLabels)); err != nil { + return nil, err + } + dataPVCs := []corev1.PersistentVolumeClaim{} + for _, targetPod := range podList.Items { + if targetPod.Spec.NodeName == "" { + return nil, intctrlutil.NewError(intctrlutil.ErrorTypeNeedWaiting, "waiting Pod scheduled") + } + for _, volume := range targetPod.Spec.Volumes { + if volume.PersistentVolumeClaim == nil { + continue + } + dataPVC := corev1.PersistentVolumeClaim{} + pvcKey := types.NamespacedName{Namespace: targetPod.Namespace, Name: volume.PersistentVolumeClaim.ClaimName} + if err := p.Client.Get(p.Ctx, pvcKey, &dataPVC); err != nil { + return nil, err + } + if dataPVC.Labels[constant.VolumeTypeLabelKey] != string(appsv1alpha1.VolumeTypeData) { + continue + } + if dataPVC.Status.Phase != corev1.ClaimBound { + return nil, intctrlutil.NewError(intctrlutil.ErrorTypeNeedWaiting, "waiting PVC Bound") + } + if dataPVC.Annotations == nil { + dataPVC.Annotations = map[string]string{} + } + dataPVC.Annotations["pod-name"] = targetPod.Name + dataPVC.Annotations["node-name"] = targetPod.Spec.NodeName + dataPVCs = append(dataPVCs, dataPVC) + } + } + return dataPVCs, nil +} + func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, err error) { objs = make([]client.Object, 0) @@ -407,18 +449,11 @@ func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, constant.KBAppComponentLabelKey: componentSpec.Name, } // get data dir pvc name - dataPVCList := corev1.PersistentVolumeClaimList{} - dataPVCLabels := map[string]string{ - constant.AppInstanceLabelKey: p.Cluster.Name, - constant.KBAppComponentLabelKey: componentSpec.Name, - constant.VolumeTypeLabelKey: string(appsv1alpha1.VolumeTypeData), - } - if err = p.Client.List(p.Ctx, &dataPVCList, - client.InNamespace(p.namespace), - client.MatchingLabels(dataPVCLabels)); err != nil { + dataPVCs, err := p.getDataPVCs(componentSpec.Name) + if err != nil { return objs, err } - if len(dataPVCList.Items) == 0 { + if len(dataPVCs) == 0 { return objs, errors.New("not found data pvc") } recoveryInfo, err := p.getRecoveryInfo(componentSpec.Name) @@ -430,10 +465,7 @@ func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, return objs, err } dataVolumeMount := getVolumeMount(recoveryInfo) - for i, dataPVC := range dataPVCList.Items { - if dataPVC.Status.Phase != corev1.ClaimBound { - return objs, errors.New("waiting PVC Bound") - } + for _, dataPVC := range dataPVCs { volumes := []corev1.Volume{ {Name: "data", VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: dataPVC.Name}}}, @@ -450,18 +482,20 @@ func (p *PointInTimeRecoveryManager) buildResourceObjs() (objs []client.Object, if image == "" { image = viper.GetString(constant.KBToolsImage) } - jobName := fmt.Sprintf("pitr-phy-%s-%s-%d", p.Cluster.Name, componentSpec.Name, i) + jobName := fmt.Sprintf("pitr-phy-%s", dataPVC.Annotations["pod-name"]) job, err := builder.BuildPITRJob(jobName, p.Cluster, image, []string{"sh", "-c"}, recoveryInfo.Physical.RestoreCommands, volumes, volumeMounts, recoveryInfo.Env) job.SetLabels(commonLabels) + job.Spec.Template.Spec.NodeName = dataPVC.Annotations["node-name"] if err != nil { return objs, err } // create logic restore job if p.Cluster.Status.Phase == appsv1alpha1.RunningClusterPhase && len(recoveryInfo.Logical.RestoreCommands) > 0 { - logicJobName := fmt.Sprintf("pitr-logic-%s-%s-%d", p.Cluster.Name, componentSpec.Name, i) + logicJobName := fmt.Sprintf("pitr-logic-%s", dataPVC.Annotations["pod-name"]) logicJob, err := builder.BuildPITRJob(logicJobName, p.Cluster, image, []string{"sh", "-c"}, recoveryInfo.Logical.RestoreCommands, volumes, volumeMounts, recoveryInfo.Env) + logicJob.Spec.Template.Spec.NodeName = dataPVC.Annotations["node-name"] if err != nil { return objs, err } diff --git a/internal/controller/plan/pitr_test.go b/internal/controller/plan/pitr_test.go index 1fc60dfad..f93722fab 100644 --- a/internal/controller/plan/pitr_test.go +++ b/internal/controller/plan/pitr_test.go @@ -66,6 +66,16 @@ var _ = Describe("PITR Functions", func() { testapps.ClearClusterResources(&testCtx) inNS := client.InNamespace(testCtx.DefaultNamespace) ml := client.HasLabels{testCtx.TestObjLabelKey} + + deletionPropagation := metav1.DeletePropagationBackground + deletionGracePeriodSeconds := int64(0) + opts := client.DeleteAllOfOptions{ + DeleteOptions: client.DeleteOptions{ + GracePeriodSeconds: &deletionGracePeriodSeconds, + PropagationPolicy: &deletionPropagation, + }, + } + testapps.ClearResources(&testCtx, generics.PodSignature, inNS, ml, &opts) testapps.ClearResources(&testCtx, generics.BackupSignature, inNS, ml) testapps.ClearResources(&testCtx, generics.BackupPolicySignature, inNS, ml) testapps.ClearResources(&testCtx, generics.JobSignature, inNS, ml) @@ -123,6 +133,18 @@ var _ = Describe("PITR Functions", func() { SetStorage("1Gi"). Create(&testCtx).GetObject() + By("By mocking a pod") + volume := corev1.Volume{Name: pvc.Name, VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc.Name}}} + _ = testapps.NewPodFactory(testCtx.DefaultNamespace, clusterName+"-"+mysqlCompName+"-0"). + AddAppInstanceLabel(clusterName). + AddAppComponentLabel(mysqlCompName). + AddAppManangedByLabel(). + AddVolume(volume). + AddContainer(corev1.Container{Name: testapps.DefaultMySQLContainerName, Image: testapps.ApeCloudMySQLImage}). + AddNodeName("fake-node-name"). + Create(&testCtx).GetObject() + By("By creating backup tool: ") backupSelfDefineObj := &dpv1alpha1.BackupTool{} backupSelfDefineObj.SetLabels(map[string]string{ diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index d39c1c5bb..a35d09f9d 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -55,6 +55,7 @@ const ( // ErrorType for cluster controller ErrorTypeBackupFailed ErrorType = "BackupFailed" + ErrorTypeNeedWaiting ErrorType = "NeedWaiting" // waiting for next reconcile ) var ErrFailedToAddFinalizer = errors.New("failed to add finalizer") diff --git a/internal/testutil/apps/pod_factory.go b/internal/testutil/apps/pod_factory.go index 0b6d277cd..2054fc445 100644 --- a/internal/testutil/apps/pod_factory.go +++ b/internal/testutil/apps/pod_factory.go @@ -52,3 +52,8 @@ func (factory *MockPodFactory) AddVolume(volume corev1.Volume) *MockPodFactory { *volumes = append(*volumes, volume) return factory } + +func (factory *MockPodFactory) AddNodeName(nodeName string) *MockPodFactory { + factory.get().Spec.NodeName = nodeName + return factory +} From e63daca674de80fedfcc22ced4f54d5194e691a6 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Tue, 9 May 2023 16:01:06 +0800 Subject: [PATCH 253/439] chore: tidy up DAG codes (#2991) --- config/rbac/role.yaml | 37 ++++++---------- controllers/apps/cluster_controller.go | 1 + controllers/apps/cluster_controller_test.go | 34 +++++++-------- controllers/apps/components/pod_controller.go | 6 +-- .../apps/opsrequest_controller_test.go | 4 +- .../backuppolicy_controller_test.go | 4 +- deploy/helm/config/rbac/role.yaml | 37 ++++++---------- internal/controller/graph/transformer.go | 17 ++++---- .../lifecycle/cluster_plan_builder.go | 7 ++-- .../lifecycle/transformer_cluster_deletion.go | 4 +- .../transformer_sts_horizontal_scaling.go | 42 +++++++++---------- internal/testutil/apps/common_util.go | 9 ++-- 12 files changed, 86 insertions(+), 116 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index def2d8282..7d70ebab0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -30,30 +30,6 @@ rules: - deployments/status verbs: - get -- apiGroups: - - apps - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - pods/finalizers - verbs: - - update -- apiGroups: - - apps - resources: - - pods/status - verbs: - - get - apiGroups: - apps resources: @@ -421,6 +397,7 @@ rules: resources: - pods verbs: + - create - delete - deletecollection - get @@ -434,6 +411,12 @@ rules: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - update - apiGroups: - "" resources: @@ -441,6 +424,12 @@ rules: verbs: - get - list +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get - apiGroups: - "" resources: diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index a2ff369d3..76fd48a83 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -88,6 +88,7 @@ import ( // read + update access // +kubebuilder:rbac:groups=core,resources=endpoints,verbs=get;list;watch;update;patch // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;update;patch +// +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=update // +kubebuilder:rbac:groups=core,resources=pods/exec,verbs=create // read only + watch access diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 0f563904e..6bf651ce8 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -475,11 +475,11 @@ var _ = Describe("Cluster Controller", func() { } By("Checking Backup created") - Eventually(testapps.GetListLen(&testCtx, generics.BackupSignature, + Eventually(testapps.List(&testCtx, generics.BackupSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, constant.KBAppComponentLabelKey: comp.Name, - }, client.InNamespace(clusterKey.Namespace))).Should(Equal(1)) + }, client.InNamespace(clusterKey.Namespace))).Should(HaveLen(1)) By("Mocking VolumeSnapshot and set it as ReadyToUse") snapshotKey := types.NamespacedName{Name: fmt.Sprintf("%s-%s-scaling", @@ -522,11 +522,11 @@ var _ = Describe("Cluster Controller", func() { } By("Check backup job cleanup") - Eventually(testapps.GetListLen(&testCtx, generics.BackupSignature, + Eventually(testapps.List(&testCtx, generics.BackupSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, constant.KBAppComponentLabelKey: comp.Name, - }, client.InNamespace(clusterKey.Namespace))).Should(Equal(0)) + }, client.InNamespace(clusterKey.Namespace))).Should(HaveLen(0)) Eventually(testapps.CheckObjExists(&testCtx, snapshotKey, &snapshotv1.VolumeSnapshot{}, false)).Should(Succeed()) checkUpdatedStsReplicas() @@ -1194,10 +1194,10 @@ var _ = Describe("Cluster Controller", func() { }) By("Check deployment workload has been created") - Eventually(testapps.GetListLen(&testCtx, generics.DeploymentSignature, + Eventually(testapps.List(&testCtx, generics.DeploymentSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, - }, client.InNamespace(clusterKey.Namespace))).ShouldNot(BeEquivalentTo(0)) + }, client.InNamespace(clusterKey.Namespace))).ShouldNot(HaveLen(0)) stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) @@ -1223,10 +1223,10 @@ var _ = Describe("Cluster Controller", func() { } By("Check associated PDB has been created") - Eventually(testapps.GetListLen(&testCtx, generics.PodDisruptionBudgetSignature, + Eventually(testapps.List(&testCtx, generics.PodDisruptionBudgetSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, - }, client.InNamespace(clusterKey.Namespace))).Should(Equal(0)) + }, client.InNamespace(clusterKey.Namespace))).Should(HaveLen(0)) podSpec := stsList.Items[0].Spec.Template.Spec By("Checking created sts pods template with built-in toleration") @@ -1243,11 +1243,11 @@ var _ = Describe("Cluster Controller", func() { Expect(podSpec.TopologySpreadConstraints).Should(BeEmpty()) By("Check should create env configmap") - Eventually(testapps.GetListLen(&testCtx, generics.ConfigMapSignature, + Eventually(testapps.List(&testCtx, generics.ConfigMapSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, constant.AppConfigTypeLabelKey: "kubeblocks-env", - }, client.InNamespace(clusterKey.Namespace))).Should(Equal(2)) + }, client.InNamespace(clusterKey.Namespace))).Should(HaveLen(2)) } checkAllServicesCreate := func() { @@ -1383,18 +1383,18 @@ var _ = Describe("Cluster Controller", func() { }) }) - Context(fmt.Sprintf("[comp: %s] with pvc", compName), func() { - It("should trigger a backup process(snapshot) and create pvcs from backup for newly created replicas when horizontal scale the cluster from 1 to 3", func() { - testHorizontalScale(compName, compDefName) - }) - }) - // HACK/TODO: only Stateful and Consensus workload types passes following test, need to investigate. // Would expect that non-stateless workload types should all pass tests. switch compName { case statefulCompName, consensusCompName, replicationCompName: + Context(fmt.Sprintf("[comp: %s] with pvc", compName), func() { + It("should trigger a backup process(snapshot) and create pvcs from backup for newly created replicas when horizontal scale the cluster from 1 to 3", func() { + testHorizontalScale(compName, compDefName) + }) + }) + Context(fmt.Sprintf("[comp: %s] with pvc and dynamic-provisioning storage class", compName), func() { - It(fmt.Sprintf("[comp: %s] should update PVC request storage size accordingly", compName), func() { + It("should update PVC request storage size accordingly", func() { testStorageExpansion(compName, compDefName) }) }) diff --git a/controllers/apps/components/pod_controller.go b/controllers/apps/components/pod_controller.go index 6d62e2196..68cd2e495 100644 --- a/controllers/apps/components/pod_controller.go +++ b/controllers/apps/components/pod_controller.go @@ -43,9 +43,9 @@ type PodReconciler struct { Recorder record.EventRecorder } -// +kubebuilder:rbac:groups=apps,resources=pods,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=apps,resources=pods/status,verbs=get -// +kubebuilder:rbac:groups=apps,resources=pods/finalizers,verbs=update +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get +// +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=update // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index 8ef56835a..6aee31bc9 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -473,10 +473,10 @@ var _ = Describe("OpsRequest Controller", func() { var stsList = &appsv1.StatefulSetList{} createStsPodAndMockStsReady := func() { - Eventually(testapps.GetListLen(&testCtx, intctrlutil.StatefulSetSignature, + Eventually(testapps.List(&testCtx, intctrlutil.StatefulSetSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterObj.Name, - }, client.InNamespace(clusterObj.Namespace))).Should(BeEquivalentTo(1)) + }, client.InNamespace(clusterObj.Namespace))).Should(HaveLen(1)) stsList = testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, client.ObjectKeyFromObject(clusterObj), testapps.DefaultRedisCompName) Expect(stsList.Items).Should(HaveLen(1)) diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index 976a85171..9ee9ef743 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -228,9 +228,9 @@ var _ = Describe("Backup Policy Controller", func() { patchCronJobStatus(getCronjobKey(dpv1alpha1.BackupTypeFull)) By("retain the latest backup") - Eventually(testapps.GetListLen(&testCtx, intctrlutil.BackupSignature, + Eventually(testapps.List(&testCtx, intctrlutil.BackupSignature, client.MatchingLabels(backupPolicy.Spec.Full.Target.LabelsSelector.MatchLabels), - client.InNamespace(backupPolicy.Namespace))).Should(Equal(1)) + client.InNamespace(backupPolicy.Namespace))).Should(HaveLen(1)) }) }) diff --git a/deploy/helm/config/rbac/role.yaml b/deploy/helm/config/rbac/role.yaml index def2d8282..7d70ebab0 100644 --- a/deploy/helm/config/rbac/role.yaml +++ b/deploy/helm/config/rbac/role.yaml @@ -30,30 +30,6 @@ rules: - deployments/status verbs: - get -- apiGroups: - - apps - resources: - - pods - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - pods/finalizers - verbs: - - update -- apiGroups: - - apps - resources: - - pods/status - verbs: - - get - apiGroups: - apps resources: @@ -421,6 +397,7 @@ rules: resources: - pods verbs: + - create - delete - deletecollection - get @@ -434,6 +411,12 @@ rules: - pods/exec verbs: - create +- apiGroups: + - "" + resources: + - pods/finalizers + verbs: + - update - apiGroups: - "" resources: @@ -441,6 +424,12 @@ rules: verbs: - get - list +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get - apiGroups: - "" resources: diff --git a/internal/controller/graph/transformer.go b/internal/controller/graph/transformer.go index eeab6b511..9ec6cb54d 100644 --- a/internal/controller/graph/transformer.go +++ b/internal/controller/graph/transformer.go @@ -45,25 +45,22 @@ type Transformer interface { // TransformerChain chains a group Transformer together type TransformerChain []Transformer -// ErrFastReturn is used to stop the Transformer chain for some purpose. +// ErrNoops is used to stop the Transformer chain for some purpose. // Use it in Transformer.Transform when all jobs have done and no need to run following transformers -var ErrFastReturn = errors.New("fast return") +var ErrNoops = errors.New("No-Ops") // ApplyTo applies TransformerChain t to dag -func (t *TransformerChain) ApplyTo(ctx TransformContext, dag *DAG) error { - if t == nil { - return nil - } - for _, transformer := range *t { +func (r TransformerChain) ApplyTo(ctx TransformContext, dag *DAG) error { + for _, transformer := range r { if err := transformer.Transform(ctx, dag); err != nil { - return fastReturnErrorToNil(err) + return ignoredIfNoops(err) } } return nil } -func fastReturnErrorToNil(err error) error { - if err == ErrFastReturn { +func ignoredIfNoops(err error) error { + if err == ErrNoops { return nil } return err diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index c2ea6d922..4ae4f7b63 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -150,15 +150,14 @@ func (c *clusterPlanBuilder) Build() (graph.Plan, error) { sendWaringEventWithError(c.transCtx.GetRecorder(), c.transCtx.Cluster, ReasonApplyResourcesFailed, err) }() - // new a DAG and apply chain on it, after that we should get the final Plan + // new a DAG and apply chain on it dag := graph.NewDAG() err = c.transformers.ApplyTo(c.transCtx, dag) - // log for debug - c.transCtx.Logger.Info(fmt.Sprintf("DAG: %s", dag)) + c.transCtx.Logger.V(1).Info(fmt.Sprintf("DAG: %s", dag)) // add dag to clusterPlanBuilder c.dag = dag - // we got the execution plan + // construct execution plan plan := &clusterPlan{ dag: dag, walkFunc: c.defaultWalkFunc, diff --git a/internal/controller/lifecycle/transformer_cluster_deletion.go b/internal/controller/lifecycle/transformer_cluster_deletion.go index 15246e5ef..2d9732439 100644 --- a/internal/controller/lifecycle/transformer_cluster_deletion.go +++ b/internal/controller/lifecycle/transformer_cluster_deletion.go @@ -49,7 +49,7 @@ func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * switch cluster.Spec.TerminationPolicy { case v1alpha1.DoNotTerminate: transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeWarning, "DoNotTerminate", "spec.terminationPolicy %s is preventing deletion.", cluster.Spec.TerminationPolicy) - return graph.ErrFastReturn + return graph.ErrNoops case v1alpha1.Halt: kinds = kindsForHalt() case v1alpha1.Delete: @@ -86,7 +86,7 @@ func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * root.action = actionPtr(DELETE) // fast return, that is stopping the plan.Build() stage and jump to plan.Execute() directly - return graph.ErrFastReturn + return graph.ErrNoops } func kindsForDoNotTerminate() []client.ObjectList { diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 53149ae3e..efc0eb51b 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -243,25 +243,24 @@ func (t *StsHorizontalScalingTransformer) Transform(ctx graph.TransformContext, } // when horizontal scaling up, sometimes db needs backup to sync data from master, // log is not reliable enough since it can be recycled - var err error switch { // scale out case *stsObj.Spec.Replicas < *stsProto.Spec.Replicas: - err = scaleOut() + if err := scaleOut(); err != nil { + return err + } case *stsObj.Spec.Replicas > *stsProto.Spec.Replicas: - err = scaleIn() - } - if err != nil { - return err + if err := scaleIn(); err != nil { + return err + } } emitHorizontalScalingEvent() - - if err = postScaleOut(); err != nil { + if err := postScaleOut(); err != nil { return err } - return nil } + findPVCsToBeDeleted := func(pvcSnapshot clusterSnapshot) []*corev1.PersistentVolumeClaim { stsToBeDeleted := make([]*appsv1.StatefulSet, 0) // list sts to be deleted @@ -509,8 +508,8 @@ func checkedCreateDeletePVCCronJob(cli roclient.ReadonlyClient, "CronJobCreate", "create cronjob to delete pvc/%s", pvcKey.Name) + return nil } - return nil } @@ -636,35 +635,32 @@ func doSnapshot(cli roclient.ReadonlyClient, backupPolicyTemplateName string, dag *graph.DAG, root graph.Vertex) error { - ctx := reqCtx.Ctx - backupPolicyTemplate := &appsv1alpha1.BackupPolicyTemplate{} - if err := cli.Get(ctx, client.ObjectKey{Name: backupPolicyTemplateName}, backupPolicyTemplate); err != nil && !apierrors.IsNotFound(err) { - return err - } - if len(backupPolicyTemplate.Name) > 0 { - // if there is backuppolicytemplate created by provider - // create backupjob CR, will ignore error if already exists - err := createBackup(reqCtx, cli, stsObj, componentDef, backupPolicyTemplateName, snapshotKey, cluster, dag, root) - if err != nil { + if err := cli.Get(ctx, client.ObjectKey{Name: backupPolicyTemplateName}, backupPolicyTemplate); err != nil { + if !apierrors.IsNotFound(err) { return err } - } else { // no backuppolicytemplate, then try native volumesnapshot pvcName := strings.Join([]string{vcts[0].Name, stsObj.Name, "0"}, "-") snapshot, err := builder.BuildVolumeSnapshot(snapshotKey, pvcName, stsObj) if err != nil { return err } - if err := controllerutil.SetControllerReference(cluster, snapshot, scheme); err != nil { + if err = controllerutil.SetControllerReference(cluster, snapshot, scheme); err != nil { return err } vertex := &lifecycleVertex{obj: snapshot, action: actionPtr(CREATE)} dag.AddVertex(vertex) dag.Connect(root, vertex) - reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "VolumeSnapshotCreate", "Create volumesnapshot/%s", snapshotKey.Name) + return nil + } + + // if there is backuppolicytemplate created by provider + // create backupjob CR, will ignore error if already exists + if err := createBackup(reqCtx, cli, stsObj, componentDef, backupPolicyTemplateName, snapshotKey, cluster, dag, root); err != nil { + return err } return nil } diff --git a/internal/testutil/apps/common_util.go b/internal/testutil/apps/common_util.go index 04c1a6560..e9a4a457f 100644 --- a/internal/testutil/apps/common_util.go +++ b/internal/testutil/apps/common_util.go @@ -152,14 +152,13 @@ func CheckObj[T intctrlutil.Object, PT intctrlutil.PObject[T]](testCtx *testutil // Helper functions to check fields of resource lists when writing unit tests. -func GetListLen[T intctrlutil.Object, PT intctrlutil.PObject[T], +func List[T intctrlutil.Object, PT intctrlutil.PObject[T], L intctrlutil.ObjList[T], PL intctrlutil.PObjList[T, L]]( - testCtx *testutil.TestContext, _ func(T, L), opt ...client.ListOption) func(gomega.Gomega) int { - return func(g gomega.Gomega) int { + testCtx *testutil.TestContext, _ func(T, L), opt ...client.ListOption) func(gomega.Gomega) []T { + return func(g gomega.Gomega) []T { var objList L g.Expect(testCtx.Cli.List(testCtx.Ctx, PL(&objList), opt...)).To(gomega.Succeed()) - items := reflect.ValueOf(&objList).Elem().FieldByName("Items").Interface().([]T) - return len(items) + return reflect.ValueOf(&objList).Elem().FieldByName("Items").Interface().([]T) } } From c9a25ae78f5b941569bd503ff00ce8bdf0b75cdc Mon Sep 17 00:00:00 2001 From: zjx20 Date: Tue, 9 May 2023 16:41:45 +0800 Subject: [PATCH 254/439] fix: use a port assigned by the OS in TestETCD() to avoid conflicts (#3131) --- cmd/probe/internal/binding/etcd/etcd_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/probe/internal/binding/etcd/etcd_test.go b/cmd/probe/internal/binding/etcd/etcd_test.go index 76e0f2d26..bb03d8b6e 100644 --- a/cmd/probe/internal/binding/etcd/etcd_test.go +++ b/cmd/probe/internal/binding/etcd/etcd_test.go @@ -21,11 +21,11 @@ package etcd import ( "context" + "fmt" "io/ioutil" - "math/rand" + "net" "net/url" "os" - "strconv" "testing" "time" @@ -41,15 +41,14 @@ const ( etcdStartTimeout = 30 ) -// randomize the port to avoid conflicting -var testEndpoint = "http://localhost:" + strconv.Itoa(52600+rand.Intn(1000)) - func TestETCD(t *testing.T) { - etcdServer, err := startEtcdServer(testEndpoint) - defer stopEtcdServer(etcdServer) + etcdServer, err := startEtcdServer("http://localhost:0") if err != nil { t.Errorf("start embedded etcd server error: %s", err) } + defer stopEtcdServer(etcdServer) + testEndpoint := fmt.Sprintf("http://%s", etcdServer.ETCD.Clients[0].Addr().(*net.TCPAddr).String()) + t.Run("Invoke GetRole", func(t *testing.T) { e := mockEtcd(etcdServer) role, err := e.GetRole(context.Background(), &bindings.InvokeRequest{}, &bindings.InvokeResponse{}) From 5709276d9662dd3fe78d6ecdaa75970aa5789cc2 Mon Sep 17 00:00:00 2001 From: chantu Date: Tue, 9 May 2023 17:00:43 +0800 Subject: [PATCH 255/439] fix: ut for drop followers only when scaling in (#3152) --- cmd/manager/main.go | 9 -- config/rbac/role.yaml | 7 - controllers/apps/cluster_controller_test.go | 29 +++- controllers/apps/components/component.go | 36 +++++ .../apps/components/deployment_controller.go | 5 + controllers/apps/components/pod_controller.go | 124 ------------------ .../apps/components/pod_controller_test.go | 110 ---------------- .../components/stateful_set_controller.go | 4 + .../templates/clusterdefinition.yaml | 3 + deploy/apecloud-mysql/templates/scripts.yaml | 32 ++++- deploy/helm/config/rbac/role.yaml | 7 - internal/constant/const.go | 1 + 12 files changed, 98 insertions(+), 269 deletions(-) delete mode 100644 controllers/apps/components/pod_controller.go delete mode 100644 controllers/apps/components/pod_controller_test.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 43b206de9..529952614 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -393,15 +393,6 @@ func main() { os.Exit(1) } - if err = (&components.PodReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Recorder: mgr.GetEventRecorderFor("pod-controller"), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "Pod") - os.Exit(1) - } - if err = (&appscontrollers.ComponentClassReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7d70ebab0..ed9b6c617 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -397,7 +397,6 @@ rules: resources: - pods verbs: - - create - delete - deletecollection - get @@ -424,12 +423,6 @@ rules: verbs: - get - list -- apiGroups: - - "" - resources: - - pods/status - verbs: - - get - apiGroups: - "" resources: diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 6bf651ce8..f256c160c 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -28,6 +28,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/viper" @@ -400,7 +402,7 @@ var _ = Describe("Cluster Controller", func() { compName, "data").SetStorage("1Gi").CheckedCreate(&testCtx) } - mockPodsForConsensusTest := func(cluster *appsv1alpha1.Cluster, number int) []corev1.Pod { + mockPodsForTest := func(cluster *appsv1alpha1.Cluster, number int) []corev1.Pod { componentName := cluster.Spec.ComponentSpecs[0].Name clusterName := cluster.Name stsName := cluster.Name + "-" + componentName @@ -411,6 +413,7 @@ var _ = Describe("Cluster Controller", func() { Name: stsName + "-" + strconv.Itoa(i), Namespace: testCtx.DefaultNamespace, Labels: map[string]string{ + constant.AppManagedByLabelKey: constant.AppName, constant.AppInstanceLabelKey: clusterName, constant.KBAppComponentLabelKey: componentName, appsv1.ControllerRevisionHashLabelKey: "mock-version", @@ -447,8 +450,12 @@ var _ = Describe("Cluster Controller", func() { Expect(int(*stsList.Items[0].Spec.Replicas)).To(BeEquivalentTo(comp.Replicas)) By("Creating mock pods in StatefulSet") - pods := mockPodsForConsensusTest(clusterObj, int(comp.Replicas)) - for _, pod := range pods { + pods := mockPodsForTest(clusterObj, int(comp.Replicas)) + for i, pod := range pods { + if comp.ComponentDefRef == replicationCompDefName && i == 0 { + By("mocking primary for replication to pass check") + pods[0].ObjectMeta.Labels[constant.RoleLabelKey] = "primary" + } Expect(testCtx.CheckedCreateObj(testCtx.Ctx, &pod)).Should(Succeed()) // mock the status to pass the isReady(pod) check in consensus_set pod.Status.Conditions = []corev1.PodCondition{{ @@ -942,9 +949,10 @@ var _ = Describe("Cluster Controller", func() { sts = &stsList.Items[0] }).Should(Succeed()) - By("Creating mock pods in StatefulSet") - pods := mockPodsForConsensusTest(clusterObj, replicas) + By("Creating mock pods in StatefulSet, and set controller reference") + pods := mockPodsForTest(clusterObj, replicas) for _, pod := range pods { + Expect(controllerutil.SetControllerReference(sts, &pod, scheme.Scheme)).Should(Succeed()) Expect(testCtx.CreateObj(testCtx.Ctx, &pod)).Should(Succeed()) // mock the status to pass the isReady(pod) check in consensus_set pod.Status.Conditions = []corev1.PodCondition{{ @@ -982,6 +990,17 @@ var _ = Describe("Cluster Controller", func() { g.Expect(followerCount).Should(Equal(2)) }).Should(Succeed()) + By("Checking pods' annotations") + Eventually(func(g Gomega) { + pods, err := util.GetPodListByStatefulSet(ctx, k8sClient, sts) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(pods).Should(HaveLen(int(*sts.Spec.Replicas))) + for _, pod := range pods { + g.Expect(pod.Annotations).ShouldNot(BeNil()) + g.Expect(pod.Annotations[constant.ComponentReplicasAnnotationKey]).Should(Equal(strconv.Itoa(int(*sts.Spec.Replicas)))) + } + }).Should(Succeed()) + By("Updating StatefulSet's status") sts.Status.UpdateRevision = "mock-version" sts.Status.Replicas = int32(replicas) diff --git a/controllers/apps/components/component.go b/controllers/apps/components/component.go index d5250240d..b3f5c4201 100644 --- a/controllers/apps/components/component.go +++ b/controllers/apps/components/component.go @@ -21,9 +21,11 @@ package components import ( "context" + "strconv" "time" "golang.org/x/exp/slices" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -184,3 +186,37 @@ func patchWorkloadCustomLabel( } return nil } + +func updateComponentInfoToPods( + ctx context.Context, + cli client.Client, + cluster *appsv1alpha1.Cluster, + componentSpec *appsv1alpha1.ClusterComponentSpec) error { + if cluster == nil || componentSpec == nil { + return nil + } + ml := client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.GetName(), + constant.KBAppComponentLabelKey: componentSpec.Name, + } + podList := corev1.PodList{} + if err := cli.List(ctx, &podList, ml); err != nil { + return err + } + replicasStr := strconv.Itoa(int(componentSpec.Replicas)) + for _, pod := range podList.Items { + if pod.Annotations != nil && + pod.Annotations[constant.ComponentReplicasAnnotationKey] == replicasStr { + continue + } + patch := client.MergeFrom(pod.DeepCopy()) + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + pod.Annotations[constant.ComponentReplicasAnnotationKey] = replicasStr + if err := cli.Patch(ctx, &pod, patch); err != nil { + return err + } + } + return nil +} diff --git a/controllers/apps/components/deployment_controller.go b/controllers/apps/components/deployment_controller.go index 8545e4857..908823ca1 100644 --- a/controllers/apps/components/deployment_controller.go +++ b/controllers/apps/components/deployment_controller.go @@ -76,6 +76,10 @@ func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) return workloadCompClusterReconcile(reqCtx, r.Client, deploy, func(cluster *appsv1alpha1.Cluster, componentSpec *appsv1alpha1.ClusterComponentSpec, component types.Component) (ctrl.Result, error) { compCtx := newComponentContext(reqCtx, r.Client, r.Recorder, component, deploy, componentSpec) + // update component info to pods' annotations + if err := updateComponentInfoToPods(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } // patch the current componentSpec workload's custom labels if err := patchWorkloadCustomLabel(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "Deployment Controller PatchWorkloadCustomLabelFailed", err.Error()) @@ -96,6 +100,7 @@ func (r *DeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&appsv1.Deployment{}). Owns(&appsv1.ReplicaSet{}). + Owns(&corev1.Pod{}). WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). Named("deployment-watcher"). Complete(r) diff --git a/controllers/apps/components/pod_controller.go b/controllers/apps/components/pod_controller.go deleted file mode 100644 index 68cd2e495..000000000 --- a/controllers/apps/components/pod_controller.go +++ /dev/null @@ -1,124 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package components - -import ( - "context" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/record" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/predicate" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/controllers/apps/components/util" - "github.com/apecloud/kubeblocks/internal/constant" - intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" -) - -// PodReconciler reconciles a Pod object -type PodReconciler struct { - client.Client - Scheme *runtime.Scheme - Recorder record.EventRecorder -} - -// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get -// +kubebuilder:rbac:groups=core,resources=pods/finalizers,verbs=update - -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile -func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - var ( - pod = &corev1.Pod{} - err error - cluster *appsv1alpha1.Cluster - ok bool - componentName string - componentStatus appsv1alpha1.ClusterComponentStatus - ) - - reqCtx := intctrlutil.RequestCtx{ - Ctx: ctx, - Req: req, - Log: log.FromContext(ctx).WithValues("pod", req.NamespacedName), - } - - if err = r.Client.Get(reqCtx.Ctx, reqCtx.Req.NamespacedName, pod); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - - // skip if pod is being deleted - if !pod.DeletionTimestamp.IsZero() { - return intctrlutil.Reconciled() - } - - if cluster, err = util.GetClusterByObject(reqCtx.Ctx, r.Client, pod); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - if cluster == nil { - return intctrlutil.Reconciled() - } - - if componentName, ok = pod.Labels[constant.KBAppComponentLabelKey]; !ok { - return intctrlutil.Reconciled() - } - - if cluster.Status.Components == nil { - return intctrlutil.Reconciled() - } - if componentStatus, ok = cluster.Status.Components[componentName]; !ok { - return intctrlutil.Reconciled() - } - if componentStatus.ConsensusSetStatus == nil { - return intctrlutil.Reconciled() - } - if componentStatus.ConsensusSetStatus.Leader.Pod == util.ComponentStatusDefaultPodName { - return intctrlutil.Reconciled() - } - - // sync leader status from cluster.status - patch := client.MergeFrom(pod.DeepCopy()) - if pod.Annotations == nil { - pod.Annotations = make(map[string]string) - } - pod.Annotations[constant.LeaderAnnotationKey] = componentStatus.ConsensusSetStatus.Leader.Pod - if err = r.Client.Patch(reqCtx.Ctx, pod, patch); err != nil { - return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") - } - r.Recorder.Eventf(pod, corev1.EventTypeNormal, "AddAnnotation", "add annotation %s=%s", constant.LeaderAnnotationKey, componentStatus.ConsensusSetStatus.Leader.Pod) - return intctrlutil.Reconciled() -} - -// SetupWithManager sets up the controller with the Manager. -func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&corev1.Pod{}). - WithEventFilter(predicate.NewPredicateFuncs(intctrlutil.WorkloadFilterPredicate)). - Named("pod-watcher"). - Complete(r) -} diff --git a/controllers/apps/components/pod_controller_test.go b/controllers/apps/components/pod_controller_test.go deleted file mode 100644 index d8214e9d0..000000000 --- a/controllers/apps/components/pod_controller_test.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Copyright (C) 2022-2023 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package components - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/internal/constant" - intctrlutil "github.com/apecloud/kubeblocks/internal/generics" - testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" -) - -var _ = Describe("Pod Controller", func() { - - var ( - randomStr = testCtx.GetRandomStr() - clusterName = "mysql-" + randomStr - clusterDefName = "cluster-definition-consensus-" + randomStr - clusterVersionName = "cluster-version-operations-" + randomStr - ) - - const ( - revisionID = "6fdd48d9cd" - consensusCompName = "consensus" - consensusCompType = "consensus" - ) - - cleanAll := func() { - // must wait until resources deleted and no longer exist before the testcases start, - // otherwise if later it needs to create some new resource objects with the same name, - // in race conditions, it will find the existence of old objects, resulting failure to - // create the new objects. - By("clean resources") - - // delete cluster(and all dependent sub-resources), clusterversion and clusterdef - testapps.ClearClusterResources(&testCtx) - - // clear rest resources - inNS := client.InNamespace(testCtx.DefaultNamespace) - ml := client.HasLabels{testCtx.TestObjLabelKey} - // namespaced resources - testapps.ClearResources(&testCtx, intctrlutil.OpsRequestSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.StatefulSetSignature, inNS, ml) - testapps.ClearResources(&testCtx, intctrlutil.PodSignature, inNS, ml, client.GracePeriodSeconds(0)) - } - - BeforeEach(cleanAll) - - AfterEach(cleanAll) - - Context("test controller", func() { - It("test pod controller", func() { - - leaderName := "test-leader-name" - podName := "test-pod-name" - - By("mock cluster object") - _, _, cluster := testapps.InitConsensusMysql(testCtx, clusterDefName, - clusterVersionName, clusterName, consensusCompType, consensusCompName) - - By("mock cluster's consensus status") - Expect(testapps.ChangeObjStatus(&testCtx, cluster, func() { - cluster.Status.Components = map[string]appsv1alpha1.ClusterComponentStatus{} - cluster.Status.Components[consensusCompName] = appsv1alpha1.ClusterComponentStatus{ - ConsensusSetStatus: &appsv1alpha1.ConsensusSetStatus{ - Leader: appsv1alpha1.ConsensusMemberStatus{ - Pod: leaderName, - AccessMode: "ReadWrite", - }, - }, - } - })).Should(Succeed()) - - By("triggering pod reconcile") - pod := testapps.NewPodFactory(cluster.Namespace, podName). - AddContainer(corev1.Container{Name: testapps.DefaultMySQLContainerName, Image: testapps.ApeCloudMySQLImage}). - AddLabels(constant.AppInstanceLabelKey, cluster.Name). - AddLabels(constant.KBAppComponentLabelKey, consensusCompName). - Create(&testCtx).GetObject() - podKey := client.ObjectKeyFromObject(pod) - - By("checking pod has leader annotation") - testapps.CheckObj(&testCtx, podKey, func(g Gomega, pod *corev1.Pod) { - g.Expect(pod.Annotations).ShouldNot(BeNil()) - g.Expect(pod.Annotations[constant.LeaderAnnotationKey]).Should(Equal(leaderName)) - }) - }) - }) -}) diff --git a/controllers/apps/components/stateful_set_controller.go b/controllers/apps/components/stateful_set_controller.go index 40a246f90..890c75dec 100644 --- a/controllers/apps/components/stateful_set_controller.go +++ b/controllers/apps/components/stateful_set_controller.go @@ -76,6 +76,10 @@ func (r *StatefulSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) return workloadCompClusterReconcile(reqCtx, r.Client, sts, func(cluster *appsv1alpha1.Cluster, componentSpec *appsv1alpha1.ClusterComponentSpec, component types.Component) (ctrl.Result, error) { compCtx := newComponentContext(reqCtx, r.Client, r.Recorder, component, sts, componentSpec) + // update component info to pods' annotations + if err := updateComponentInfoToPods(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { + return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") + } // patch the current componentSpec workload's custom labels if err := patchWorkloadCustomLabel(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "StatefulSet Controller PatchWorkloadCustomLabelFailed", err.Error()) diff --git a/deploy/apecloud-mysql/templates/clusterdefinition.yaml b/deploy/apecloud-mysql/templates/clusterdefinition.yaml index c64cf222b..d1843f9b5 100644 --- a/deploy/apecloud-mysql/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql/templates/clusterdefinition.yaml @@ -170,6 +170,9 @@ spec: - path: "leader" fieldRef: fieldPath: metadata.annotations['cs.apps.kubeblocks.io/leader'] + - path: "component-replicas" + fieldRef: + fieldPath: metadata.annotations['apps.kubeblocks.io/component-replicas'] systemAccounts: cmdExecutorConfig: image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index 62669181f..c0ccedfcd 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -15,6 +15,10 @@ data: idx=${KB_POD_NAME##*-} host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) echo "host=$host" + # update replicas to persistent file + component_replicas_path=/data/mysql/.kb_component_replicas + current_component_replicas=`cat /etc/annotations/component-replicas` + echo $current_component_replicas > $component_replicas_path if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then echo "no leader or self is leader, no need to call add." else @@ -70,7 +74,7 @@ data: mkdir -p /data/mysql/data /data/mysql/log chmod +777 -R /data/mysql; echo "KB_MYSQL_CLUSTER_UID=$KB_MYSQL_CLUSTER_UID" - cluster_uid_path=/data/mysql/data/.kb_cluster_uid + cluster_uid_path=/data/mysql/.kb_cluster_uid if [ -f $cluster_uid_path ] && [ ! -f /data/mysql/data/.restore_new_cluster ]; then last_cluster_uid=`cat $cluster_uid_path` if [ "$last_cluster_uid" != "$KB_MYSQL_CLUSTER_UID" ]; then @@ -155,16 +159,17 @@ data: done pre-stop.sh: | #!/bin/bash + drop_followers() { leader=`cat /etc/annotations/leader` - echo "leader=$leader" - echo "KB_POD_NAME=$KB_POD_NAME" + echo "leader=$leader" >> /data/mysql/.kb_pre_stop.log + echo "KB_POD_NAME=$KB_POD_NAME" >> /data/mysql/.kb_pre_stop.log if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then - echo "no leader or self is leader, exit" + echo "no leader or self is leader, exit" >> /data/mysql/.kb_pre_stop.log exit 0 fi idx=${KB_POD_NAME##*-} host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) - echo "host=$host" + echo "host=$host" >> /data/mysql/.kb_pre_stop.log leader_idx=${leader##*-} leader_host=$(eval echo \$KB_MYSQL_"$leader_idx"_HOSTNAME) if [ ! -z $leader_host ]; then @@ -173,7 +178,20 @@ data: if [ ! -z $MYSQL_ROOT_PASSWORD ]; then password_flag="-p$MYSQL_ROOT_PASSWORD" fi - echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.downgrade_follower('$host:13306');\" 2>&1 " + echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.downgrade_follower('$host:13306');\" 2>&1 " >> /data/mysql/.kb_pre_stop.log mysql $host_flag -uroot $password_flag -e "call dbms_consensus.downgrade_follower('$host:13306');" 2>&1 - echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.drop_learner('$host:13306');\" 2>&1 " + echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.drop_learner('$host:13306');\" 2>&1 " >> /data/mysql/.kb_pre_stop.log mysql $host_flag -uroot $password_flag -e "call dbms_consensus.drop_learner('$host:13306');" 2>&1 + } + component_replicas_path=/data/mysql/.kb_component_replicas + current_component_replicas=`cat /etc/annotations/component-replicas` + if [ -f $component_replicas_path ]; then + last_component_replicas=`cat $component_replicas_path` + # check is scaling in but not scaling in to 0 + if [ "$last_component_replicas" -gt "$current_component_replicas" ] && [ $current_component_replicas -ne 0 ]; then + # only scaling in need to drop followers + drop_followers + else + echo "no need to drop followers" + fi + fi diff --git a/deploy/helm/config/rbac/role.yaml b/deploy/helm/config/rbac/role.yaml index 7d70ebab0..ed9b6c617 100644 --- a/deploy/helm/config/rbac/role.yaml +++ b/deploy/helm/config/rbac/role.yaml @@ -397,7 +397,6 @@ rules: resources: - pods verbs: - - create - delete - deletecollection - get @@ -424,12 +423,6 @@ rules: verbs: - get - list -- apiGroups: - - "" - resources: - - pods/status - verbs: - - get - apiGroups: - "" resources: diff --git a/internal/constant/const.go b/internal/constant/const.go index d6f4254a9..aebdd9a67 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -94,6 +94,7 @@ const ( RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" + ComponentReplicasAnnotationKey = "apps.kubeblocks.io/component-replicas" // ComponentReplicasAnnotationKey specifies the number of pods in replicas DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" // DefaultBackupPolicyAnnotationKey specifies the default backup policy. DefaultBackupPolicyTemplateAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy-template" // DefaultBackupPolicyTemplateAnnotationKey specifies the default backup policy template. BackupDataPathPrefixAnnotationKey = "dataprotection.kubeblocks.io/path-prefix" // BackupDataPathPrefixAnnotationKey specifies the backup data path prefix. From 6af0af019d2f900ce32739bd98d4eb70e3aa9ab6 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Tue, 9 May 2023 22:00:25 +0800 Subject: [PATCH 256/439] chore: change ci image buildx platforms (#3169) --- .github/workflows/cicd-push.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index 5085337de..051c32f8d 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -175,6 +175,7 @@ jobs: IMG: "apecloud/kubeblocks" VERSION: "check" GO_VERSION: "1.20" + BUILDX_PLATFORMS: "linux/amd64" secrets: inherit check-tools-image: @@ -188,6 +189,7 @@ jobs: IMG: "apecloud/kubeblocks-tools" VERSION: "check" GO_VERSION: "1.20" + BUILDX_PLATFORMS: "linux/amd64" secrets: inherit check-helm: From a0804d28e6c17303b2828022e32c4a334173b7a6 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Tue, 9 May 2023 22:18:43 +0800 Subject: [PATCH 257/439] chore: add release version to message (#3110) --- .github/utils/utils.sh | 4 ++-- .github/workflows/package-version.yml | 2 +- .github/workflows/release-create.yml | 10 ++++++++-- .github/workflows/release-helm-chart.yml | 4 ++-- .github/workflows/release-image.yml | 4 ++-- .github/workflows/release-publish.yml | 4 ++-- .github/workflows/release-version.yml | 2 +- 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.github/utils/utils.sh b/.github/utils/utils.sh index 414a0bdc4..51e2fe2d0 100644 --- a/.github/utils/utils.sh +++ b/.github/utils/utils.sh @@ -289,7 +289,7 @@ release_next_available_tag() { usage_message() { curl -H "Content-Type: application/json" -X POST $BOT_WEBHOOK \ - -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Usage:","content":[[{"tag":"text","text":"sorry master, please enter the correct format\n"},{"tag":"text","text":"1. do release\n"},{"tag":"text","text":"2. {\"ref\":\"\",\"inputs\":{\"release_version\":\"\"}}"}]]}}}}' + -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Usage:","content":[[{"tag":"text","text":"please enter the correct format\n"},{"tag":"text","text":"1. do release\n"},{"tag":"text","text":"2. {\"ref\":\"\",\"inputs\":{\"release_version\":\"\"}}"}]]}}}}' } trigger_release() { @@ -314,7 +314,7 @@ send_message() { -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Success:","content":[[{"tag":"text","text":"'$CONTENT'"}]]}}}}' else curl -H "Content-Type: application/json" -X POST $BOT_WEBHOOK \ - -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Error:","content":[[{"tag":"text","text":"sorry master, "},{"tag":"a","text":"['$CONTENT']","href":"'$RUN_URL'"}]]}}}}' + -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Error:","content":[[{"tag":"a","text":"['$CONTENT']","href":"'$RUN_URL'"}]]}}}}' fi } diff --git a/.github/workflows/package-version.yml b/.github/workflows/package-version.yml index 7135ab104..506702d8d 100644 --- a/.github/workflows/package-version.yml +++ b/.github/workflows/package-version.yml @@ -48,6 +48,6 @@ jobs: - name: send message run: | bash .github/utils/utils.sh --type 12 \ - --content "package\u00a0error" \ + --content "package\u00a0${{ inputs.release_version }}\u00a0error" \ --bot-webhook ${{ env.PACKAGE_BOT_WEBHOOK }} \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" diff --git a/.github/workflows/release-create.yml b/.github/workflows/release-create.yml index 6232e9372..da7503da3 100644 --- a/.github/workflows/release-create.yml +++ b/.github/workflows/release-create.yml @@ -13,11 +13,17 @@ jobs: publish: name: create a release runs-on: ubuntu-latest + outputs: + rel-version: ${{ steps.get_rel_version.outputs.rel_version }} steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - name: Parse release version and set REL_VERSION - run: python ./.github/utils/is_rc_or_stable_release_version.py + id: get_rel_version + run: | + python ./.github/utils/is_rc_or_stable_release_version.py + echo rel_version=v${{ env.REL_VERSION }} >> $GITHUB_OUTPUT + - name: release pre-release without release notes uses: softprops/action-gh-release@v1 if: not ${{ env.WITH_RELEASE_NOTES }} @@ -50,6 +56,6 @@ jobs: - name: send message run: | bash .github/utils/utils.sh --type 12 \ - --content "release\u00a0create\u00a0error"\ + --content "release\u00a0${{ ${{ needs.publish.outputs.rel-version }} }}\u00a0create\u00a0error"\ --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ No newline at end of file diff --git a/.github/workflows/release-helm-chart.yml b/.github/workflows/release-helm-chart.yml index b1aa57a09..58246bf9a 100644 --- a/.github/workflows/release-helm-chart.yml +++ b/.github/workflows/release-helm-chart.yml @@ -50,9 +50,9 @@ jobs: - uses: actions/checkout@v3 - name: send message run: | - CONTENT="release\u00a0chart\u00a0error" + CONTENT="release\u00a0${{ env.RELEASE_VERSION }}\u00a0chart\u00a0error" if [[ "${{ needs.release-chart.result }}" == "success" ]]; then - CONTENT="release\u00a0chart\u00a0success" + CONTENT="release\u00a0${{ env.RELEASE_VERSION }}\u00a0chart\u00a0success" fi bash .github/utils/utils.sh --type 12 \ diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index 7b3a56bf7..ded157d27 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -67,9 +67,9 @@ jobs: - uses: actions/checkout@v3 - name: send message run: | - CONTENT="release\u00a0image\u00a0error" + CONTENT="release\u00a0${{ env.RELEASE_VERSION }}\u00a0image\u00a0error" if [[ "${{ needs.release-image.result }}" == "success" && "${{ needs.release-tools-image.result }}" == "success" ]]; then - CONTENT="release\u00a0image\u00a0success" + CONTENT="release\u00a0${{ env.RELEASE_VERSION }}\u00a0image\u00a0success" fi bash .github/utils/utils.sh --type 12 \ diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index e977e8310..ef6b1ddb4 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -147,9 +147,9 @@ jobs: - uses: actions/checkout@v3 - name: send message run: | - CONTENT="release\u00a0kbcli\u00a0error" + CONTENT="release\u00a0${{ env.TAG_NAME }}\u00a0kbcli\u00a0error" if [[ "${{ needs.upload-release-assert.result }}" == "success" ]]; then - CONTENT="release\u00a0kbcli\u00a0success" + CONTENT="release\u00a0${{ env.TAG_NAME }}\u00a0kbcli\u00a0success" fi bash .github/utils/utils.sh --type 12 \ diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml index 89a34960d..8157e8d4e 100644 --- a/.github/workflows/release-version.yml +++ b/.github/workflows/release-version.yml @@ -88,6 +88,6 @@ jobs: - name: send message run: | bash .github/utils/utils.sh --type 12 \ - --content "release\u00a0error"\ + --content "release\u00a0${{ inputs.release_version }}\u00a0error"\ --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ No newline at end of file From db88aea2fe3be1c05abcb11089dcc2a1756e6d90 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Tue, 9 May 2023 22:32:54 +0800 Subject: [PATCH 258/439] chore: 'kbcli addon list' sort results (#3168) --- internal/cli/cmd/addon/addon.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/cli/cmd/addon/addon.go b/internal/cli/cmd/addon/addon.go index e8f1a54e2..34159143d 100644 --- a/internal/cli/cmd/addon/addon.go +++ b/internal/cli/cmd/addon/addon.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "fmt" + "sort" "strconv" "strings" @@ -745,6 +746,28 @@ func addonListRun(o *list.ListOptions) error { } printRows := func(tbl *printer.TablePrinter) error { + // sort addons with .status.Phase then .metadata.name + sort.SliceStable(infos, func(i, j int) bool { + toAddon := func(idx int) *extensionsv1alpha1.Addon { + addon := &extensionsv1alpha1.Addon{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(infos[idx].Object.(*unstructured.Unstructured).Object, addon); err != nil { + return nil + } + return addon + } + iAddon := toAddon(i) + jAddon := toAddon(j) + if iAddon == nil { + return true + } + if jAddon == nil { + return false + } + if iAddon.Status.Phase == jAddon.Status.Phase { + return iAddon.GetName() < jAddon.GetName() + } + return iAddon.Status.Phase < jAddon.Status.Phase + }) for _, info := range infos { addon := &extensionsv1alpha1.Addon{} obj := info.Object.(*unstructured.Unstructured) From b457d08eade0a1d75f23ed94d6d0ad00540d8e6f Mon Sep 17 00:00:00 2001 From: zjx20 Date: Wed, 10 May 2023 10:14:25 +0800 Subject: [PATCH 259/439] fix: don't attach a finalizer to the auto created backup PVC (#3156) --- controllers/dataprotection/backup_controller.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 11d4e5ef8..00749ae7c 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -289,8 +289,6 @@ func (r *BackupReconciler) createPVCWithStorageClassName(reqCtx intctrlutil.Requ }, }, } - // add a finalizer - controllerutil.AddFinalizer(pvc, dataProtectionFinalizerName) err := r.Client.Create(reqCtx.Ctx, pvc) return client.IgnoreAlreadyExists(err) } From 2bd0e3624b68b9a623a9416c641242ce250eafac Mon Sep 17 00:00:00 2001 From: shanshanying Date: Wed, 10 May 2023 10:23:32 +0800 Subject: [PATCH 260/439] fix: kbcli status support more workloads (#3162) --- internal/cli/cmd/kubeblocks/status.go | 264 ++++++++++++++------- internal/cli/cmd/kubeblocks/status_test.go | 110 ++++++--- internal/cli/cmd/kubeblocks/uninstall.go | 6 +- internal/cli/testing/fake.go | 147 ++++++++++++ internal/cli/types/types.go | 22 ++ internal/constant/const.go | 1 + 6 files changed, 436 insertions(+), 114 deletions(-) diff --git a/internal/cli/cmd/kubeblocks/status.go b/internal/cli/cmd/kubeblocks/status.go index d01927534..afbab717e 100644 --- a/internal/cli/cmd/kubeblocks/status.go +++ b/internal/cli/cmd/kubeblocks/status.go @@ -22,6 +22,7 @@ package kubeblocks import ( "context" "fmt" + "strconv" "strings" "github.com/containerd/stargz-snapshotter/estargz/errorutil" @@ -33,7 +34,6 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/constant" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,8 +46,10 @@ import ( "k8s.io/klog/v2" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" - "k8s.io/metrics/pkg/apis/metrics/v1beta1" metrics "k8s.io/metrics/pkg/client/clientset/versioned" + + tablePrinter "github.com/jedib0t/go-pretty/v6/table" + text "github.com/jedib0t/go-pretty/v6/text" ) var ( @@ -60,11 +62,12 @@ var ( ) var ( - selectorList = []metav1.ListOptions{{LabelSelector: types.InstanceLabelSelector}, {LabelSelector: types.ReleaseLabelSelector}} - kubeBlocksWorkloads = []schema.GroupVersionResource{ types.DeployGVR(), types.StatefulSetGVR(), + types.DaemonSetGVR(), + types.JobGVR(), + types.CronJobGVR(), } kubeBlocksGlobalCustomResources = []schema.GroupVersionResource{ @@ -104,12 +107,13 @@ var ( type statusOptions struct { genericclioptions.IOStreams - client kubernetes.Interface - dynamic dynamic.Interface - mc metrics.Interface - showAll bool - ns string - addons []*extensionsv1alpha1.Addon + client kubernetes.Interface + dynamic dynamic.Interface + mc metrics.Interface + showAll bool + ns string + addons []*extensionsv1alpha1.Addon + selectorList []metav1.ListOptions } func newStatusCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { @@ -150,13 +154,6 @@ func (o *statusOptions) complete(f cmdutil.Factory) error { if err != nil { return err } - o.ns = metav1.NamespaceAll - return nil -} - -func (o *statusOptions) run() error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() o.ns, _ = util.GetKubeBlocksNamespace(o.client) if o.ns == "" { @@ -166,6 +163,17 @@ func (o *statusOptions) run() error { fmt.Fprintf(o.Out, "Kuberblocks is deployed in namespace: %s\n", o.ns) } + o.selectorList = []metav1.ListOptions{ + {LabelSelector: fmt.Sprintf("%s=%s", constant.AppManagedByLabelKey, constant.AppName)}, // app.kubernetes.io/managed-by=kubeblocks + } + + return nil +} + +func (o *statusOptions) run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + allErrs := make([]error, 0) o.buildSelectorList(ctx, &allErrs) o.showWorkloads(ctx, &allErrs) @@ -195,20 +203,45 @@ func (o *statusOptions) buildSelectorList(ctx context.Context, allErrs *[]error) addons = append(addons, addon) } } - // build addon instance selector o.addons = addons - - var selectors []metav1.ListOptions for _, selector := range buildResourceLabelSelectors(addons) { - selectors = append(selectors, metav1.ListOptions{LabelSelector: selector}) + o.selectorList = append(o.selectorList, metav1.ListOptions{LabelSelector: selector}) } - selectorList = selectors } func (o *statusOptions) showAddons() { fmt.Fprintln(o.Out, "\nKubeBlocks Addons:") tbl := printer.NewTablePrinter(o.Out) + + tbl.Tbl.SetColumnConfigs([]tablePrinter.ColumnConfig{ + { + Name: "STATUS", + Transformer: func(val interface{}) string { + var ok bool + var addonPhase extensionsv1alpha1.AddonPhase + if addonPhase, ok = val.(extensionsv1alpha1.AddonPhase); !ok { + return fmt.Sprint(val) + } + var color text.Color + switch addonPhase { + case extensionsv1alpha1.AddonEnabled: + color = text.FgGreen + case extensionsv1alpha1.AddonFailed: + color = text.FgRed + case extensionsv1alpha1.AddonDisabled: + color = text.Faint + case extensionsv1alpha1.AddonEnabling, extensionsv1alpha1.AddonDisabling: + color = text.FgCyan + default: + return fmt.Sprint(addonPhase) + } + return color.Sprint(addonPhase) + }, + }, + }, + ) + tbl.SetHeader("NAME", "STATUS", "TYPE", "PROVIDER") var provider string @@ -229,7 +262,7 @@ func (o *statusOptions) showKubeBlocksResources(ctx context.Context, allErrs *[] tblPrinter := printer.NewTablePrinter(o.Out) tblPrinter.SetHeader("KIND", "NAME") - unstructuredList := listResourceByGVR(ctx, o.dynamic, metav1.NamespaceAll, kubeBlocksGlobalCustomResources, selectorList, allErrs) + unstructuredList := listResourceByGVR(ctx, o.dynamic, metav1.NamespaceAll, kubeBlocksGlobalCustomResources, o.selectorList, allErrs) for _, resourceList := range unstructuredList { for _, resource := range resourceList.Items { tblPrinter.AddRow(resource.GetKind(), resource.GetName()) @@ -242,7 +275,7 @@ func (o *statusOptions) showKubeBlocksConfig(ctx context.Context, allErrs *[]err fmt.Fprintln(o.Out, "\nKubeBlocks Configurations:") tblPrinter := printer.NewTablePrinter(o.Out) tblPrinter.SetHeader("NAMESPACE", "KIND", "NAME") - unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksConfigurations, selectorList, allErrs) + unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksConfigurations, o.selectorList, allErrs) for _, resourceList := range unstructuredList { for _, resource := range resourceList.Items { tblPrinter.AddRow(resource.GetNamespace(), resource.GetKind(), resource.GetName()) @@ -255,7 +288,7 @@ func (o *statusOptions) showKubeBlocksRBAC(ctx context.Context, allErrs *[]error fmt.Fprintln(o.Out, "\nKubeBlocks Global RBAC:") tblPrinter := printer.NewTablePrinter(o.Out) tblPrinter.SetHeader("KIND", "NAME") - unstructuredList := listResourceByGVR(ctx, o.dynamic, metav1.NamespaceAll, kubeBlocksClusterRBAC, selectorList, allErrs) + unstructuredList := listResourceByGVR(ctx, o.dynamic, metav1.NamespaceAll, kubeBlocksClusterRBAC, o.selectorList, allErrs) for _, resourceList := range unstructuredList { for _, resource := range resourceList.Items { tblPrinter.AddRow(resource.GetKind(), resource.GetName()) @@ -267,7 +300,7 @@ func (o *statusOptions) showKubeBlocksRBAC(ctx context.Context, allErrs *[]error fmt.Fprintln(o.Out, "\nKubeBlocks Namespaced RBAC:") tblPrinter = printer.NewTablePrinter(o.Out) tblPrinter.SetHeader("NAMESPACE", "KIND", "NAME") - unstructuredList = listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksNamespacedRBAC, selectorList, allErrs) + unstructuredList = listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksNamespacedRBAC, o.selectorList, allErrs) for _, resourceList := range unstructuredList { for _, resource := range resourceList.Items { tblPrinter.AddRow(resource.GetNamespace(), resource.GetKind(), resource.GetName()) @@ -292,7 +325,7 @@ func (o *statusOptions) showKubeBlocksStorage(ctx context.Context, allErrs *[]er tblPrinter.AddRow(pvc.GetNamespace(), pvc.Kind, pvc.GetName(), pvc.Status.Capacity.Storage()) } - unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksStorages, selectorList, allErrs) + unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksStorages, o.selectorList, allErrs) for _, resourceList := range unstructuredList { for _, resource := range resourceList.Items { switch resource.GetKind() { @@ -336,90 +369,153 @@ func (o *statusOptions) showHelmResources(ctx context.Context, allErrs *[]error) func (o *statusOptions) showWorkloads(ctx context.Context, allErrs *[]error) { fmt.Fprintln(o.Out, "\nKubeBlocks Workloads:") tblPrinter := printer.NewTablePrinter(o.Out) - tblPrinter.SetHeader("NAMESPACE", "KIND", "NAME", "READY PODS", "CPU(cores)", "MEMORY(bytes)") + tblPrinter.Tbl.SetColumnConfigs([]tablePrinter.ColumnConfig{ + { + Name: "READY PODS", + Transformer: func(val interface{}) (valStr string) { + var ok bool + if valStr, ok = val.(string); !ok { + return fmt.Sprint(val) + } + if valStr == notAvailable || len(valStr) == 0 { + return valStr + } + // split string by '/' + podsInfo := strings.Split(valStr, "/") + if len(podsInfo) != 2 { + return valStr + } + readyPods, totalPods := int(0), int(0) + readyPods, _ = strconv.Atoi(podsInfo[0]) + totalPods, _ = strconv.Atoi(podsInfo[1]) + + var color text.Color + if readyPods != totalPods { + color = text.FgRed + } else { + color = text.FgGreen + } + return color.Sprint(valStr) + }, + }, + }, + ) - unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksWorkloads, selectorList, allErrs) + tblPrinter.SetHeader("NAMESPACE", "KIND", "NAME", "READY PODS", "CPU(cores)", "MEMORY(bytes)", "CREATED-AT") - cpuMap, memMap := computeMetricByWorkloads(ctx, o.ns, unstructuredList, o.mc, allErrs) + unstructuredList := listResourceByGVR(ctx, o.dynamic, o.ns, kubeBlocksWorkloads, o.selectorList, allErrs) - renderDeploy := func(raw *unstructured.Unstructured) { - deploy := &appsv1.Deployment{} - err := runtime.DefaultUnstructuredConverter.FromUnstructured(raw.Object, deploy) - if err != nil { - appendErrIgnoreNotFound(allErrs, err) - return + cpuMap, memMap, readyMap := computeMetricByWorkloads(ctx, o.ns, unstructuredList, o.mc, allErrs) + + for _, workload := range unstructuredList { + for _, resource := range workload.Items { + createdAt := resource.GetCreationTimestamp() + name := resource.GetName() + row := []interface{}{resource.GetNamespace(), resource.GetKind(), name, readyMap[name], cpuMap[name], memMap[name], util.TimeFormat(&createdAt)} + tblPrinter.AddRow(row...) } - name := deploy.GetName() - tblPrinter.AddRow(deploy.GetNamespace(), deploy.Kind, deploy.GetName(), - fmt.Sprintf("%d/%d", deploy.Status.ReadyReplicas, deploy.Status.Replicas), - cpuMap[name], memMap[name]) } + tblPrinter.Print() +} - renderStatefulSet := func(raw *unstructured.Unstructured) { - sts := &appsv1.StatefulSet{} - err := runtime.DefaultUnstructuredConverter.FromUnstructured(raw.Object, sts) - if err != nil { - appendErrIgnoreNotFound(allErrs, err) - return - } - name := sts.GetName() - tblPrinter.AddRow(sts.GetNamespace(), sts.Kind, sts.GetName(), - fmt.Sprintf("%d/%d", sts.Status.ReadyReplicas, sts.Status.Replicas), - cpuMap[name], memMap[name]) +func getNestedSelectorAsString(obj map[string]interface{}, fields ...string) (string, error) { + val, found, err := unstructured.NestedStringMap(obj, fields...) + if !found || err != nil { + return "", fmt.Errorf("failed to get selector for %v, using field %s", obj, fields) + } + // convert it to string + var pair []string + for k, v := range val { + pair = append(pair, fmt.Sprintf("%s=%s", k, v)) } + return strings.Join(pair, ","), nil +} - for _, workload := range unstructuredList { - for _, resource := range workload.Items { - switch resource.GetKind() { - case constant.DeploymentKind: - renderDeploy(&resource) - case constant.StatefulSetKind: - renderStatefulSet(&resource) - default: - err := fmt.Errorf("unsupported worklkoad type: %s", resource.GetKind()) - appendErrIgnoreNotFound(allErrs, err) - } +func getNestedInt64(obj map[string]interface{}, fields ...string) int64 { + val, found, err := unstructured.NestedInt64(obj, fields...) + if !found || err != nil { + if klog.V(1).Enabled() { + klog.Errorf("failed to get int64 for %s, using field %s", obj, fields) } } - tblPrinter.Print() + return val } -func computeMetricByWorkloads(ctx context.Context, ns string, workloads []*unstructured.UnstructuredList, mc metrics.Interface, allErrs *[]error) (cpuMetricMap, memMetricMap map[string]string) { +func computeMetricByWorkloads(ctx context.Context, ns string, workloads []*unstructured.UnstructuredList, mc metrics.Interface, allErrs *[]error) (cpuMetricMap, memMetricMap, readyMap map[string]string) { cpuMetricMap = make(map[string]string) memMetricMap = make(map[string]string) + readyMap = make(map[string]string) - podsMetrics, err := mc.MetricsV1beta1().PodMetricses(ns).List(ctx, metav1.ListOptions{}) - if err != nil { - appendErrIgnoreNotFound(allErrs, err) - return - } - - computeResources := func(name string, podsMetrics *v1beta1.PodMetricsList) { - cpuUsage, memUsage := int64(0), int64(0) - for _, pod := range podsMetrics.Items { - if strings.HasPrefix(pod.Name, name) { + computeMetrics := func(namespace, name string, matchLabels string) { + if pods, err := mc.MetricsV1beta1().PodMetricses(namespace).List(ctx, metav1.ListOptions{LabelSelector: matchLabels}); err != nil { + if klog.V(1).Enabled() { + klog.Errorf("faied to get pod metrics for %s/%s, selector: , error: %v", namespace, name, matchLabels, err) + } + } else { + cpuUsage, memUsage := int64(0), int64(0) + for _, pod := range pods.Items { for _, container := range pod.Containers { cpuUsage += container.Usage.Cpu().MilliValue() - memUsage += container.Usage.Memory().Value() / (1024 * 1024) + memUsage += container.Usage.Memory().Value() / 1024 / 1024 } } + cpuMetricMap[name] = fmt.Sprintf("%dm", cpuUsage) + memMetricMap[name] = fmt.Sprintf("%dMi", memUsage) + } + } + + computeWorkloadRunningMeta := func(resource *unstructured.Unstructured, getReadyRepilca func() []string, getTotalReplicas func() []string, getSelector func() []string) error { + name := resource.GetName() + + readyMap[name] = notAvailable + cpuMetricMap[name] = notAvailable + memMetricMap[name] = notAvailable + + if getReadyRepilca != nil && getTotalReplicas != nil { + readyReplicas := getNestedInt64(resource.Object, getReadyRepilca()...) + replicas := getNestedInt64(resource.Object, getTotalReplicas()...) + readyMap[name] = fmt.Sprintf("%d/%d", readyReplicas, replicas) + } + + if getSelector != nil { + if matchLabels, err := getNestedSelectorAsString(resource.Object, getSelector()...); err != nil { + return err + } else { + computeMetrics(resource.GetNamespace(), name, matchLabels) + } } - cpuMetricMap[name] = fmt.Sprintf("%dm", cpuUsage) - memMetricMap[name] = fmt.Sprintf("%dMi", memUsage) + return nil } + readyReplicas := func() []string { return []string{"status", "readyReplicas"} } + replicas := func() []string { return []string{"status", "replicas"} } + matchLabels := func() []string { return []string{"spec", "selector", "matchLabels"} } + daemonReady := func() []string { return []string{"status", "numberReady"} } + daemonTotal := func() []string { return []string{"status", "desiredNumberScheduled"} } + jobReady := func() []string { return []string{"status", "succeeded"} } + jobTotal := func() []string { return []string{"spec", "completions"} } + for _, workload := range workloads { for _, resource := range workload.Items { - name := resource.GetName() - if podsMetrics == nil { - cpuMetricMap[name] = notAvailable - memMetricMap[name] = notAvailable - continue + var err error + switch resource.GetKind() { + case constant.DeploymentKind, constant.StatefulSetKind: + err = computeWorkloadRunningMeta(&resource, readyReplicas, replicas, matchLabels) + case constant.DaemonSetKind: + err = computeWorkloadRunningMeta(&resource, daemonReady, daemonTotal, matchLabels) + case constant.JobKind: + err = computeWorkloadRunningMeta(&resource, jobReady, jobTotal, matchLabels) + case constant.CronJobKind: + err = computeWorkloadRunningMeta(&resource, nil, nil, nil) + default: + err = fmt.Errorf("unsupported workload kind: %s, name: %s", resource.GetKind(), resource.GetName()) + } + if err != nil { + appendErrIgnoreNotFound(allErrs, err) } - computeResources(name, podsMetrics) } } - return cpuMetricMap, memMetricMap + return cpuMetricMap, memMetricMap, readyMap } func listResourceByGVR(ctx context.Context, client dynamic.Interface, namespace string, gvrlist []schema.GroupVersionResource, selector []metav1.ListOptions, allErrs *[]error) []*unstructured.UnstructuredList { diff --git a/internal/cli/cmd/kubeblocks/status_test.go b/internal/cli/cmd/kubeblocks/status_test.go index 583531cfa..29b767981 100644 --- a/internal/cli/cmd/kubeblocks/status_test.go +++ b/internal/cli/cmd/kubeblocks/status_test.go @@ -21,30 +21,84 @@ package kubeblocks import ( "context" + "net/http" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/spf13/cobra" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes/scheme" clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/constant" ) var _ = Describe("kubeblocks status", func() { - var cmd *cobra.Command - var streams genericclioptions.IOStreams - var tf *cmdtesting.TestFactory + var ( + namespace = "test" + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + stsName = "test-sts" + deployName = "test-deploy" + ) BeforeEach(func() { - streams, _, _, _ = genericclioptions.NewTestIOStreams() - tf = cmdtesting.NewTestFactory().WithNamespace(namespace) - tf.Client = &clientfake.RESTClient{} + tf = cmdtesting.NewTestFactory().WithNamespace("test") + codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) + // add workloads + extraLabels := map[string]string{ + "appName": "JohnSnow", + "slogan": "YouknowNothing", + } + + deploy := testing.FakeDeploy(deployName, namespace, extraLabels) + deploymentList := &appsv1.DeploymentList{} + deploymentList.Items = []appsv1.Deployment{*deploy} + + sts := testing.FakeStatefulSet(stsName, namespace, extraLabels) + statefulSetList := &appsv1.StatefulSetList{} + statefulSetList.Items = []appsv1.StatefulSet{*sts} + stsPods := testing.FakePodForSts(sts) + + job := testing.FakeJob("test-job", namespace, extraLabels) + jobList := &batchv1.JobList{} + jobList.Items = []batchv1.Job{*job} + + cronjob := testing.FakeCronJob("test-cronjob", namespace, extraLabels) + cronjobList := &batchv1.CronJobList{} + cronjobList.Items = []batchv1.CronJob{*cronjob} + + httpResp := func(obj runtime.Object) *http.Response { + return &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, obj)} + } + + tf.UnstructuredClient = &clientfake.RESTClient{ + GroupVersion: schema.GroupVersion{Group: types.AppsAPIGroup, Version: types.AppsAPIVersion}, + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + Client: clientfake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { + urlPrefix := "/api/v1/namespaces/" + namespace + return map[string]*http.Response{ + urlPrefix + "/deployments": httpResp(deploymentList), + urlPrefix + "/statefulsets": httpResp(statefulSetList), + urlPrefix + "/jobs": httpResp(jobList), + urlPrefix + "/cronjobs": httpResp(cronjobList), + urlPrefix + "/pods": httpResp(stsPods), + }[req.URL.Path], nil + }), + } + + tf.Client = tf.UnstructuredClient + tf.FakeDynamicClient = testing.FakeDynamicClient(deploy, sts) + streams = genericclioptions.NewTestIOStreamsDiscard() }) AfterEach(func() { @@ -53,7 +107,7 @@ var _ = Describe("kubeblocks status", func() { It("pre-run status", func() { var cfg string - cmd = newStatusCmd(tf, streams) + cmd := newStatusCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) Expect(cmd.HasSubCommands()).Should(BeFalse()) @@ -71,37 +125,39 @@ var _ = Describe("kubeblocks status", func() { Expect(o.showAll).To(Equal(false)) }) - It("run status", func() { - ns := "demo" - - mockDeploy := func() *appsv1.Deployment { - deploy := &appsv1.Deployment{} - deploy.SetNamespace(ns) - deploy.SetLabels(map[string]string{ - "app.kubernetes.io/name": types.KubeBlocksChartName, - "app.kubernetes.io/version": "latest", - }) - return deploy - } - + It("list resources", func() { + clientSet, _ := tf.KubernetesClientSet() o := &statusOptions{ IOStreams: streams, - ns: ns, - client: testing.FakeClientSet(mockDeploy()), + ns: namespace, + client: clientSet, mc: testing.FakeMetricsClientSet(), - dynamic: testing.FakeDynamicClient(mockDeploy()), + dynamic: tf.FakeDynamicClient, showAll: true, } By("make sure mocked deploy is injected") ctx := context.Background() - deploys, err := o.dynamic.Resource(types.DeployGVR()).Namespace(ns).List(ctx, metav1.ListOptions{}) + deploys, err := o.dynamic.Resource(types.DeployGVR()).Namespace(namespace).List(ctx, metav1.ListOptions{}) Expect(err).Should(Succeed()) Expect(len(deploys.Items)).Should(BeEquivalentTo(1)) + statefulsets, err := o.dynamic.Resource(types.StatefulSetGVR()).Namespace(namespace).List(ctx, metav1.ListOptions{}) + Expect(err).Should(Succeed()) + Expect(len(statefulsets.Items)).Should(BeEquivalentTo(1)) + By("check deployment can be hit by selector") allErrs := make([]error, 0) - unstructuredList := listResourceByGVR(ctx, o.dynamic, ns, kubeBlocksWorkloads, selectorList, &allErrs) - Expect(len(unstructuredList)).Should(BeEquivalentTo(len(kubeBlocksWorkloads) * len(selectorList))) + o.buildSelectorList(ctx, &allErrs) + unstructuredList := listResourceByGVR(ctx, o.dynamic, namespace, kubeBlocksWorkloads, o.selectorList, &allErrs) + // will list update to five types of worklaods + Expect(len(unstructuredList)).Should(BeEquivalentTo(5)) + for _, list := range unstructuredList { + if list.GetKind() == constant.DeploymentKind || list.GetKind() == constant.StatefulSetKind || list.GetKind() == constant.JobKind || list.GetKind() == constant.CronJobKind { + Expect(len(list.Items)).Should(BeEquivalentTo(1)) + } else { + Expect(len(list.Items)).Should(BeEquivalentTo(0)) + } + } Expect(o.run()).To(Succeed()) }) }) diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index 2561b5dc6..dae491bf5 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -123,10 +123,10 @@ func (o *UninstallOptions) PreCheck() error { fmt.Fprintf(o.Out, "to find out the namespace where KubeBlocks is installed, please use:\n\t'kbcli kubeblocks status'\n") fmt.Fprintf(o.Out, "to uninstall KubeBlocks completely, please use:\n\t`kbcli kubeblocks uninstall -n `\n") } - } else if o.Namespace != kbNamespace { - o.Namespace = kbNamespace - fmt.Fprintf(o.Out, "Uninstall KubeBlocks in namespace \"%s\"\n", kbNamespace) } + o.Namespace = kbNamespace + fmt.Fprintf(o.Out, "Uninstall KubeBlocks in namespace \"%s\"\n", kbNamespace) + return nil } diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index e3ea6f463..1c7fd1489 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -26,6 +26,7 @@ import ( snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/sethvargo/go-password/password" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" @@ -646,3 +647,149 @@ func FakeRoleBinding(name string, sa *corev1.ServiceAccount, role *rbacv1.Role) }, } } + +func FakeDeploy(name string, namespace string, extraLabels map[string]string) *appsv1.Deployment { + labels := map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + } + // extraLabels will override the labels above if there is a conflict + for k, v := range extraLabels { + labels[k] = v + } + labels["app"] = name + + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + } +} + +func FakeStatefulSet(name string, namespace string, extraLabels map[string]string) *appsv1.StatefulSet { + labels := map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + } + // extraLabels will override the labels above if there is a conflict + for k, v := range extraLabels { + labels[k] = v + } + labels["app"] = name + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: pointer.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + Status: appsv1.StatefulSetStatus{ + Replicas: 1, + }, + } +} + +func FakePodForSts(sts *appsv1.StatefulSet) *corev1.PodList { + pods := &corev1.PodList{} + for i := 0; i < int(*sts.Spec.Replicas); i++ { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%d", sts.Name, i), + Namespace: sts.Namespace, + Labels: sts.Spec.Template.Labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: sts.Name, + Image: "fake-image", + }, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + pods.Items = append(pods.Items, *pod) + } + return pods +} + +func FakeJob(name string, namespace string, extraLabels map[string]string) *batchv1.Job { + labels := map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + } + // extraLabels will override the labels above if there is a conflict + for k, v := range extraLabels { + labels[k] = v + } + labels["app"] = name + + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: batchv1.JobSpec{ + Completions: pointer.Int32(1), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + Status: batchv1.JobStatus{ + Active: 1, + Ready: pointer.Int32(1), + }, + } +} + +func FakeCronJob(name string, namespace string, extraLabels map[string]string) *batchv1.CronJob { + labels := map[string]string{ + constant.AppInstanceLabelKey: types.KubeBlocksReleaseName, + } + // extraLabels will override the labels above if there is a conflict + for k, v := range extraLabels { + labels[k] = v + } + labels["app"] = name + + return &batchv1.CronJob{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: labels, + }, + Spec: batchv1.CronJobSpec{ + Schedule: "*/1 * * * *", + JobTemplate: batchv1.JobTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + }, + }, + } +} diff --git a/internal/cli/types/types.go b/internal/cli/types/types.go index 29a7e5e02..84ff042ad 100644 --- a/internal/cli/types/types.go +++ b/internal/cli/types/types.go @@ -23,8 +23,10 @@ import ( "fmt" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/runtime/schema" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -58,9 +60,18 @@ const ( ResourceDeployments = "deployments" ResourceConfigmaps = "configmaps" ResourceStatefulSets = "statefulsets" + ResourceDaemonSets = "daemonsets" ResourceSecrets = "secrets" ) +// K8s batch API group +const ( + K8SBatchAPIGroup = batchv1.GroupName + K8sBatchAPIVersion = "v1" + ResourceJobs = "jobs" + ResourceCronJobs = "cronjobs" +) + // K8s webhook API group const ( WebhookAPIGroup = "admissionregistration.k8s.io" @@ -257,6 +268,10 @@ func StatefulSetGVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: appsv1.GroupName, Version: K8sCoreAPIVersion, Resource: ResourceStatefulSets} } +func DaemonSetGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: appsv1.GroupName, Version: K8sCoreAPIVersion, Resource: ResourceDaemonSets} +} + func DeployGVR() schema.GroupVersionResource { return schema.GroupVersionResource{Group: appsv1.GroupName, Version: K8sCoreAPIVersion, Resource: ResourceDeployments} } @@ -351,3 +366,10 @@ func CustomResourceDefinitionGVR() schema.GroupVersionResource { Resource: ResourceCustomResourceDefinition, } } + +func JobGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: K8SBatchAPIGroup, Version: K8sBatchAPIVersion, Resource: ResourceJobs} +} +func CronJobGVR() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: K8SBatchAPIGroup, Version: K8sBatchAPIVersion, Resource: ResourceCronJobs} +} diff --git a/internal/constant/const.go b/internal/constant/const.go index aebdd9a67..caea4c3f9 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -169,6 +169,7 @@ const ( VolumeSnapshotKind = "VolumeSnapshot" ServiceKind = "Service" ConfigMapKind = "ConfigMap" + DaemonSetKind = "DaemonSet" ) const ( From f4c784508c2f6578bf6c668fb86ba90c8dad17ac Mon Sep 17 00:00:00 2001 From: shaojiang Date: Wed, 10 May 2023 10:40:53 +0800 Subject: [PATCH 261/439] fix: csi-s3 mount options bug (#3166) --- deploy/csi-s3/templates/_helpers.tpl | 2 +- deploy/helm/values.yaml | 2 +- .../templates/backuppolicytemplate.yaml | 2 ++ .../postgresql/templates/backuptool-pitr.yaml | 6 ----- deploy/postgresql/templates/scripts.yaml | 22 ++++++++++++++----- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/deploy/csi-s3/templates/_helpers.tpl b/deploy/csi-s3/templates/_helpers.tpl index 92c75e011..332e2461f 100644 --- a/deploy/csi-s3/templates/_helpers.tpl +++ b/deploy/csi-s3/templates/_helpers.tpl @@ -6,7 +6,7 @@ Expand the mountOptions of the storageClass. {{- if hasSuffix ".aliyuncs.com" .Values.secret.endpoint }} {{- printf "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 --subdomain %s" .Values.storageClass.mountOptions }} {{- else if .Values.secret.region }} - {{- printf "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 --region %s %s" .Values.storageClass.mountOptions .Values.secret.region }} + {{- printf "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 --region %s %s" .Values.secret.region .Values.storageClass.mountOptions }} {{- else }} {{- printf "--memory-limit 1000 --dir-mode 0777 --file-mode 0666 %s" .Values.storageClass.mountOptions }} {{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 98d7f9763..c3d4af4cd 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1678,7 +1678,7 @@ csi-s3: storageClass: create: true singleBucket: "" - mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" + mountOptions: "" alertmanager-webhook-adaptor: ## Linkage with prometheus.enabled diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index 2ed418d3c..77dec27e2 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -36,6 +36,8 @@ spec: backupToolName: postgres-basebackup incremental: backupToolName: postgres-pitr + target: + role: primary backupStatusUpdates: - path: manifests.backupLog containerName: postgresql diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml index 22f76e4f8..74bc9912e 100644 --- a/deploy/postgresql/templates/backuptool-pitr.yaml +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -68,12 +68,6 @@ spec: echo "uploading ${i}"; mv -f ${i} ${TODAY_INCR_LOG}/; done - if [ -d ${LOG_DIR} ]; then - cd ${LOG_DIR}; - LATEST_LOG=$(ls -t . | grep '[[:digit:]]$\|.partial$'|head -n 1); - echo "uploading ${TODAY_INCR_LOG}/${LATEST_LOG}.gz"; - gzip -kqc ${LATEST_LOG} > ${TODAY_INCR_LOG}/${LATEST_LOG}.gz; - fi echo "done." sync; diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index 647644d02..340419ef9 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -104,8 +104,20 @@ data: set -o errexit set -o nounset SHOW_START_TIME=$1 - LOG_START_TIME=$(pg_waldump $(ls -Ftr $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') - LOG_STOP_TIME=$(pg_waldump $(ls -Ft $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') - LOG_START_TIME=$(date -d "$LOG_START_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') - LOG_STOP_TIME=$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') - [[ $SHOW_START_TIME == "false" ]] && printf "{\"stopTime\": \"$LOG_STOP_TIME\"}" || printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}" \ No newline at end of file + LOG_START_TIME="" + if [ "$SHOW_START_TIME" == "false" ]; then + if [ "$(pg_waldump $(psql -Atc "select pg_walfile_name(pg_current_wal_lsn())") --rmgr=Transaction 2>/dev/null |tail -n 1)" != "" ]; then + psql -c "select pg_switch_wal()" >/dev/null 2>&1 + sleep 1 + fi + LOG_STOP_TIME=$(pg_waldump $(psql -Atc "select last_archived_wal from pg_stat_archiver") --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + [[ "${LOG_STOP_TIME}" != "" ]] && printf "{\"stopTime\": \"$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ')\"}" || printf "{}" + else + LOG_START_TIME=$(pg_waldump $(ls -Ftr $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + for i in $(ls -Ft $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'); do LOG_STOP_TIME=$(pg_waldump $i --rmgr=Transaction 2>/dev/null|tail -n 1); [[ "$LOG_STOP_TIME" != "" ]] && break; done + LOG_STOP_TIME=$(echo $LOG_STOP_TIME |awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + if [ "${LOG_START_TIME}" == "" ]; then LOG_START_TIME=${LOG_STOP_TIME}; fi + LOG_START_TIME=$(date -d "$LOG_START_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') + LOG_STOP_TIME=$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') + printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}" + fi \ No newline at end of file From f6019d3b04b7668a5d42a6d72bc0b8bff3eade2b Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Wed, 10 May 2023 10:42:07 +0800 Subject: [PATCH 262/439] fix: describe vscale should show component class and validate not working (#3160) --- .../apps/operations/vertical_scaling.go | 2 ++ internal/cli/cmd/cluster/describe_ops.go | 11 ++++++-- internal/cli/cmd/cluster/operations.go | 28 ++++++++++++------- internal/cli/cmd/cluster/operations_test.go | 1 + 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/controllers/apps/operations/vertical_scaling.go b/controllers/apps/operations/vertical_scaling.go index 181905087..fede25ce0 100644 --- a/controllers/apps/operations/vertical_scaling.go +++ b/controllers/apps/operations/vertical_scaling.go @@ -23,6 +23,7 @@ import ( "reflect" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -64,6 +65,7 @@ func (vs verticalScalingHandler) Action(reqCtx intctrlutil.RequestCtx, cli clien } if verticalScaling.Class != "" { component.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: verticalScaling.Class} + component.Resources = corev1.ResourceRequirements{} } else { // clear old class ref component.ClassDefRef = &appsv1alpha1.ClassDefRef{} diff --git a/internal/cli/cmd/cluster/describe_ops.go b/internal/cli/cmd/cluster/describe_ops.go index 8e166124e..ed5dba05b 100644 --- a/internal/cli/cmd/cluster/describe_ops.go +++ b/internal/cli/cmd/cluster/describe_ops.go @@ -270,11 +270,16 @@ func (o *describeOpsOptions) getVerticalScalingCommand(spec appsv1alpha1.OpsRequ spec.VerticalScalingList, convertObject, getCompName) commands := make([]string, len(componentNameSlice)) for i := range componentNameSlice { - resource := resourceSlice[i].(corev1.ResourceRequirements) commands[i] = fmt.Sprintf("kbcli cluster vscale %s --components=%s", spec.ClusterRef, strings.Join(componentNameSlice[i], ",")) - commands[i] += o.addResourceFlag("cpu", resource.Limits.Cpu()) - commands[i] += o.addResourceFlag("memory", resource.Limits.Memory()) + class := spec.VerticalScalingList[i].Class + if class != "" { + commands[i] += fmt.Sprintf("--class=%s", class) + } else { + resource := resourceSlice[i].(corev1.ResourceRequirements) + commands[i] += o.addResourceFlag("cpu", resource.Limits.Cpu()) + commands[i] += o.addResourceFlag("memory", resource.Limits.Memory()) + } } return commands } diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 8f39c2077..40511b8e7 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -186,6 +186,12 @@ func (o *OperationsOptions) validateVolumeExpansion() error { } func (o *OperationsOptions) validateVScale(cluster *appsv1alpha1.Cluster) error { + if o.Class != "" && (o.CPU != "" || o.Memory != "") { + return fmt.Errorf("class and cpu/memory cannot be both specified") + } + if o.Class == "" && o.CPU == "" && o.Memory == "" { + return fmt.Errorf("class or cpu/memory must be specified") + } componentClasses, err := class.ListClassesByClusterDefinition(o.Dynamic, cluster.Spec.ClusterDefRef) if err != nil { return err @@ -194,17 +200,19 @@ func (o *OperationsOptions) validateVScale(cluster *appsv1alpha1.Cluster) error fillClassParams := func(comp *appsv1alpha1.ClusterComponentSpec) { if o.Class != "" { comp.ClassDefRef = &appsv1alpha1.ClassDefRef{Class: o.Class} + comp.Resources = corev1.ResourceRequirements{} + } else { + comp.ClassDefRef = &appsv1alpha1.ClassDefRef{} + requests := make(corev1.ResourceList) + if o.CPU != "" { + requests[corev1.ResourceCPU] = resource.MustParse(o.CPU) + } + if o.Memory != "" { + requests[corev1.ResourceMemory] = resource.MustParse(o.Memory) + } + requests.DeepCopyInto(&comp.Resources.Requests) + requests.DeepCopyInto(&comp.Resources.Limits) } - - requests := make(corev1.ResourceList) - if o.CPU != "" { - requests[corev1.ResourceCPU] = resource.MustParse(o.CPU) - } - if o.Memory != "" { - requests[corev1.ResourceMemory] = resource.MustParse(o.Memory) - } - requests.DeepCopyInto(&comp.Resources.Requests) - requests.DeepCopyInto(&comp.Resources.Limits) } for _, name := range o.ComponentNames { diff --git a/internal/cli/cmd/cluster/operations_test.go b/internal/cli/cmd/cluster/operations_test.go index a32413fd4..720b64d74 100644 --- a/internal/cli/cmd/cluster/operations_test.go +++ b/internal/cli/cmd/cluster/operations_test.go @@ -56,6 +56,7 @@ var _ = Describe("operations", func() { clusterWithOneComp.Spec.ComponentSpecs = []appsv1alpha1.ClusterComponentSpec{ clusterWithOneComp.Spec.ComponentSpecs[0], } + clusterWithOneComp.Spec.ComponentSpecs[0].ClassDefRef = &appsv1alpha1.ClassDefRef{Class: testapps.Class1c1gName} classDef := testapps.NewComponentClassDefinitionFactory("custom", clusterWithOneComp.Spec.ClusterDefRef, testing.ComponentDefName). AddClasses(testapps.DefaultResourceConstraintName, []string{testapps.Class1c1gName}). GetObject() From 3dec1f6b5114308826993335e4fa31e3a8274676 Mon Sep 17 00:00:00 2001 From: "yunju.lly" Date: Wed, 10 May 2023 11:54:20 +0800 Subject: [PATCH 263/439] fix: mongodb restart alert title is incorrect (#3171) --- deploy/helm/values.yaml | 2 +- deploy/mongodb/values.yaml | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index c3d4af4cd..da3579242 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1100,7 +1100,7 @@ prometheus: description: 'MongoDB instance is down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}' - alert: MongodbRestarted - expr: 'time() - process_start_time_seconds < 60' + expr: 'mongodb_instance_uptime_seconds < 60' for: 0m labels: severity: info diff --git a/deploy/mongodb/values.yaml b/deploy/mongodb/values.yaml index ae4db120f..6024a400b 100644 --- a/deploy/mongodb/values.yaml +++ b/deploy/mongodb/values.yaml @@ -39,8 +39,6 @@ metrics: pullPolicy: IfNotPresent config: extensions: - memory_ballast: - size_mib: 512 health_check: endpoint: 0.0.0.0:13133 path: /health/status @@ -90,4 +88,4 @@ metrics: processors: [memory_limiter] exporters: [prometheus] - extensions: [memory_ballast, health_check] + extensions: [health_check] From 3c65cca0753a47bc04eb96c37fccfb10c1b6cdf4 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Wed, 10 May 2023 12:15:00 +0800 Subject: [PATCH 264/439] chore: Dockerfile remove default ARG GOPROXY settings (#3121) --- Makefile | 17 +++++--- docker/Dockerfile | 42 +++++++++++-------- docker/Dockerfile-tools | 90 +++++++++++++++++++++++++++-------------- docker/docker.mk | 46 ++++++++++++--------- 4 files changed, 123 insertions(+), 72 deletions(-) diff --git a/Makefile b/Makefile index f35e1a69f..ae0783c5e 100644 --- a/Makefile +++ b/Makefile @@ -34,10 +34,9 @@ CHART_PATH = deploy/helm WEBHOOK_CERT_DIR ?= /tmp/k8s-webhook-server/serving-certs + # Go setup export GO111MODULE = auto -# export GOPROXY = https://proxy.golang.org -export GOPROXY = https://goproxy.cn export GOSUMDB = sum.golang.org export GONOPROXY = github.com/apecloud export GONOSUMDB = github.com/apecloud @@ -52,6 +51,15 @@ GOBIN=$(shell $(GO) env GOPATH)/bin else GOBIN=$(shell $(GO) env GOBIN) endif +GOPROXY := $(shell go env GOPROXY) +ifeq ($(GOPROXY),) +GOPROXY := https://proxy.golang.org +## use following GOPROXY settings for Chinese mainland developers. +#GOPROXY := https://goproxy.cn +endif +export GOPROXY + + LD_FLAGS="-s -w -X main.version=v${VERSION} -X main.buildDate=`date -u +'%Y-%m-%dT%H:%M:%SZ'` -X main.gitCommit=`git rev-parse HEAD`" # Which architecture to build - see $(ALL_ARCH) for options. # if the 'local' rule is being run, detect the ARCH from 'go env' @@ -59,10 +67,7 @@ LD_FLAGS="-s -w -X main.version=v${VERSION} -X main.buildDate=`date -u +'%Y-%m-% local : ARCH ?= $(shell go env GOOS)-$(shell go env GOARCH) ARCH ?= linux-amd64 -# docker build setup -# BUILDX_PLATFORMS ?= $(subst -,/,$(ARCH)) -BUILDX_PLATFORMS ?= linux/amd64,linux/arm64 -BUILDX_OUTPUT_TYPE ?= docker + TAG_LATEST ?= false BUILDX_ENABLED ?= false diff --git a/docker/Dockerfile b/docker/Dockerfile index 12d7db318..4969f0632 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,11 @@ # Build the manager binary -FROM --platform=${BUILDPLATFORM} golang:1.20 as builder +ARG DIST_IMG=gcr.io/distroless/static:nonroot +# use following dist image arg if you have problem access gcr.io registry, i.e., China region. +#ARG DIST_IMG=katanomi/distroless-static:nonroot + +ARG GO_VERSION=1.20 + +FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} as builder ## docker buildx build injected build-args: #BUILDPLATFORM — matches the current machine. (e.g. linux/amd64) @@ -14,7 +20,8 @@ FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ARG TARGETOS ARG TARGETARCH -ARG GOPROXY=https://goproxy.cn +ARG GOPROXY +#ARG GOPROXY=https://goproxy.cn ARG LD_FLAGS="-s -w" ENV GONOPROXY=github.com/apecloud @@ -22,31 +29,34 @@ ENV GONOSUMDB=github.com/apecloud ENV GOPRIVATE=github.com/apecloud ENV GOPROXY=${GOPROXY} -WORKDIR /workspace +WORKDIR /src # Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum +#COPY go.mod go.mod +#COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer # RUN go mod download # Copy the go source -COPY cmd/manager/main.go cmd/manager/main.go -COPY cmd/manager/ cmd/manager/ -COPY apis/ apis/ -COPY internal/ internal/ -COPY controllers/ controllers/ -COPY test/testdata/testdata.go test/testdata/testdata.go - -## have manager as last build step due to its volatility -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o manager cmd/manager/main.go +#COPY cmd/manager/main.go cmd/manager/main.go +#COPY cmd/manager/ cmd/manager/ +#COPY apis/ apis/ +#COPY internal/ internal/ +#COPY controllers/ controllers/ +#COPY test/testdata/testdata.go test/testdata/testdata.go + +RUN --mount=type=bind,target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + go env && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -o /out/manager ./cmd/manager/main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM katanomi/distroless-static:nonroot +FROM ${DIST_IMG} as dist WORKDIR / -COPY --from=builder /workspace/manager . +COPY --from=builder /out/manager . USER 65532:65532 ENTRYPOINT ["/manager"] diff --git a/docker/Dockerfile-tools b/docker/Dockerfile-tools index f5c3354a0..16230ffaa 100644 --- a/docker/Dockerfile-tools +++ b/docker/Dockerfile-tools @@ -1,21 +1,34 @@ # Build the kubeblocks tools binaries # includes kbcli, kubectl, and manager tools. -FROM --platform=${BUILDPLATFORM} golang:1.20 as builder ## docker buildx build injected build-args: #BUILDPLATFORM — matches the current machine. (e.g. linux/amd64) #BUILDOS — os component of BUILDPLATFORM, e.g. linux #BUILDARCH — e.g. amd64, arm64, riscv64 -#BUILDVARIANT — used to set ARM variant, e.g. v7 +#BUILDVARIANT — used to set build ARM variant, e.g. v7 #TARGETPLATFORM — The value set with --platform flag on build #TARGETOS - OS component from --platform, e.g. linux #TARGETARCH - Architecture from --platform, e.g. arm64 -#TARGETVARIANT +#TARGETVARIANT - used to set target ARM variant, e.g. v7 +ARG GO_VERSION=1.20 + +FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} as bin-downloader ARG TARGETOS ARG TARGETARCH +ARG KUBECTL_VERSION=1.26.3 + +WORKDIR /workspace -ARG GOPROXY=https://goproxy.cn +# Download binaries +RUN curl -fsSL https://dl.k8s.io/v${KUBECTL_VERSION}/kubernetes-client-${TARGETOS}-${TARGETARCH}.tar.gz | tar -zxv + + +FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} as builder +ARG TARGETOS +ARG TARGETARCH +ARG GOPROXY +#ARG GOPROXY=https://goproxy.cn ARG LD_FLAGS="-s -w" ENV GONOPROXY=github.com/apecloud @@ -23,54 +36,69 @@ ENV GONOSUMDB=github.com/apecloud ENV GOPRIVATE=github.com/apecloud ENV GOPROXY=${GOPROXY} -WORKDIR /workspace +WORKDIR /src # Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum +#COPY go.mod go.mod +#COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer # RUN go mod download # Copy the go source -COPY internal/ internal/ -COPY controllers/ controllers/ -COPY cmd/reloader/ cmd/reloader/ -COPY cmd/probe/ cmd/probe/ -COPY externalapis/ externalapis/ -COPY version/ version/ -COPY cmd/cli/ cmd/cli/ -COPY apis/ apis/ -COPY test/testdata/testdata.go test/testdata/testdata.go - -# Download binaries -RUN curl -fsSL https://dl.k8s.io/v1.26.3/kubernetes-client-${TARGETOS}-${TARGETARCH}.tar.gz | tar -zxv +#COPY internal/ internal/ +#COPY controllers/ controllers/ +#COPY cmd/reloader/ cmd/reloader/ +#COPY cmd/probe/ cmd/probe/ +#COPY externalapis/ externalapis/ +#COPY version/ version/ +#COPY cmd/cli/ cmd/cli/ +#COPY apis/ apis/ +#COPY test/testdata/testdata.go test/testdata/testdata.go # Build -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o reloader cmd/reloader/main.go -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o killer cmd/reloader/container_killer/killer.go -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o probe cmd/probe/main.go -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o kbcli cmd/cli/main.go +RUN --mount=type=bind,target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + go env && \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/killer cmd/reloader/container_killer/killer.go + +RUN --mount=type=bind,target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/reloader cmd/reloader/main.go + +RUN --mount=type=bind,target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/probe cmd/probe/main.go + +RUN --mount=type=bind,target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/kbcli cmd/cli/main.go # Use alpine -FROM docker.io/alpine:3.17 +FROM docker.io/alpine:3.17 as dist +ARG APK_MIRROR +#ARG APK_MIRROR="mirrors.aliyun.com" # install tools via apk -RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +ENV APK_MIRROR=${APK_MIRROR} +RUN if [ -n "${APK_MIRROR}" ]; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories; fi RUN apk add --no-cache curl helm \ && rm -rf /var/cache/apk/* # use apk to install kubectl in the next alpine version. -COPY --from=builder /workspace/kubernetes/client/bin/kubectl /bin +COPY --from=bin-downloader /workspace/kubernetes/client/bin/kubectl /bin # copy kubeblocks tools -COPY --from=builder /workspace/killer /bin -COPY --from=builder /workspace/reloader /bin COPY config/probe config/probe -COPY --from=builder /workspace/probe /bin -COPY --from=builder /workspace/kbcli /bin +COPY --from=builder /out/killer /bin +COPY --from=builder /out/reloader /bin +COPY --from=builder /out/probe /bin +COPY --from=builder /out/kbcli /bin -WORKDIR / # mkdir kbcli config dir and helm cache dir. RUN mkdir /.kbcli && chown -R 65532:65532 /.kbcli \ && mkdir /.cache && chown -R 65532:65532 /.cache diff --git a/docker/docker.mk b/docker/docker.mk index 01bded261..d3a174eb8 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -24,11 +24,15 @@ DEBIAN_MIRROR=mirrors.aliyun.com # Docker image build and push setting -DOCKER:=docker +DOCKER:=DOCKER_BUILDKIT=1 docker DOCKERFILE_DIR?=./docker +# BUILDX_PLATFORMS ?= $(subst -,/,$(ARCH)) +BUILDX_PLATFORMS ?= linux/amd64,linux/arm64 + # Image URL to use all building/pushing image targets IMG ?= docker.io/apecloud/$(APP_NAME) +TOOL_IMG ?= docker.io/apecloud/$(APP_NAME)-tools CLI_IMG ?= docker.io/apecloud/kbcli CLI_TAG ?= v$(CLI_VERSION) @@ -46,9 +50,9 @@ BUILDX_ARGS ?= build-dev-image: DOCKER_BUILD_ARGS += --build-arg DEBIAN_MIRROR=$(DEBIAN_MIRROR) --build-arg GITHUB_PROXY=$(GITHUB_PROXY) --build-arg GOPROXY=$(GOPROXY) build-dev-image: ## Build dev container image. ifneq ($(BUILDX_ENABLED), true) - docker build $(DOCKERFILE_DIR)/. $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/${DEV_CONTAINER_DOCKERFILE} -t $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_VERSION_TAG) + $(DOCKER) $(DOCKERFILE_DIR)/. $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/${DEV_CONTAINER_DOCKERFILE} -t $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_VERSION_TAG) else - docker buildx build $(DOCKERFILE_DIR)/. $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -f $(DOCKERFILE_DIR)/$(DEV_CONTAINER_DOCKERFILE) -t $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_VERSION_TAG) $(BUILDX_ARGS) + $(DOCKER) buildx build $(DOCKERFILE_DIR)/. $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -f $(DOCKERFILE_DIR)/$(DEV_CONTAINER_DOCKERFILE) -t $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_VERSION_TAG) $(BUILDX_ARGS) endif @@ -56,65 +60,69 @@ endif push-dev-image: DOCKER_BUILD_ARGS += --build-arg DEBIAN_MIRROR=$(DEBIAN_MIRROR) --build-arg GITHUB_PROXY=$(GITHUB_PROXY) --build-arg GOPROXY=$(GOPROXY) push-dev-image: ## Push dev container image. ifneq ($(BUILDX_ENABLED), true) - docker push $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_VERSION_TAG) + $(DOCKER) push $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_VERSION_TAG) else - docker buildx build . $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -f $(DOCKERFILE_DIR)/$(DEV_CONTAINER_DOCKERFILE) -t $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_VERSION_TAG) --push $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -f $(DOCKERFILE_DIR)/$(DEV_CONTAINER_DOCKERFILE) -t $(DEV_CONTAINER_IMAGE_NAME):$(DEV_CONTAINER_VERSION_TAG) --push $(BUILDX_ARGS) endif .PHONY: build-manager-image +build-manager-image: DOCKER_BUILD_ARGS += --cache-to type=gha,scope=${GITHUB_REF_NAME}-manager-image --cache-from type=gha,scope=${GITHUB_REF_NAME}-manager-image build-manager-image: generate ## Build Operator manager container image. ifneq ($(BUILDX_ENABLED), true) - docker build . -t ${IMG}:${VERSION} -f $(DOCKERFILE_DIR)/Dockerfile -t ${IMG}:latest + $(DOCKER) build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile -t ${IMG}:${VERSION} -t ${IMG}:latest else ifeq ($(TAG_LATEST), true) - docker buildx build . -f $(DOCKERFILE_DIR)/Dockerfile $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${IMG}:latest $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile --platform $(BUILDX_PLATFORMS) -t ${IMG}:latest $(BUILDX_ARGS) else - docker buildx build . -f $(DOCKERFILE_DIR)/Dockerfile $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${IMG}:${VERSION} $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile --platform $(BUILDX_PLATFORMS) -t ${IMG}:${VERSION} $(BUILDX_ARGS) endif endif .PHONY: push-manager-image +push-manager-image: DOCKER_BUILD_ARGS += --cache-to type=gha,scope=${GITHUB_REF_NAME}-manager-image --cache-from type=gha,scope=${GITHUB_REF_NAME}-manager-image push-manager-image: generate ## Push Operator manager container image. ifneq ($(BUILDX_ENABLED), true) ifeq ($(TAG_LATEST), true) - docker push ${IMG}:latest + $(DOCKER) push ${IMG}:latest else - docker push ${IMG}:${VERSION} + $(DOCKER) push ${IMG}:${VERSION} endif else ifeq ($(TAG_LATEST), true) - docker buildx build . -f $(DOCKERFILE_DIR)/Dockerfile $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${IMG}:latest --push $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile --platform $(BUILDX_PLATFORMS) -t ${IMG}:latest --push $(BUILDX_ARGS) else - docker buildx build . -f $(DOCKERFILE_DIR)/Dockerfile $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${IMG}:${VERSION} --push $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile --platform $(BUILDX_PLATFORMS) -t ${IMG}:${VERSION} --push $(BUILDX_ARGS) endif endif .PHONY: build-tools-image +build-tools-image: DOCKER_BUILD_ARGS += --cache-to type=gha,scope=${GITHUB_REF_NAME}-tools-image --cache-from type=gha,scope=${GITHUB_REF_NAME}-tools-image build-tools-image: generate ## Build tools container image. ifneq ($(BUILDX_ENABLED), true) - docker build . -t ${IMG}:${VERSION} -f $(DOCKERFILE_DIR)/Dockerfile-tools -t ${IMG}:latest + $(DOCKER) build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile-tools -t ${TOOL_IMG}:${VERSION} -t ${TOOL_IMG}:latest else ifeq ($(TAG_LATEST), true) - docker buildx build . -f $(DOCKERFILE_DIR)/Dockerfile-tools $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${IMG}:latest $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile-tools $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${TOOL_IMG}:latest $(BUILDX_ARGS) else - docker buildx build . -f $(DOCKERFILE_DIR)/Dockerfile-tools $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${IMG}:${VERSION} $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile-tools $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${TOOL_IMG}:${VERSION} $(BUILDX_ARGS) endif endif .PHONY: push-tools-image +push-tools-image: DOCKER_BUILD_ARGS += --cache-to type=gha,scope=${GITHUB_REF_NAME}-tools-image --cache-from type=gha,scope=${GITHUB_REF_NAME}-tools-image push-tools-image: generate ## Push tools container image. ifneq ($(BUILDX_ENABLED), true) ifeq ($(TAG_LATEST), true) - docker push ${IMG}:latest + $(DOCKER) push ${TOOL_IMG}:latest else - docker push ${IMG}:${VERSION} + $(DOCKER) push ${TOOL_IMG}:${VERSION} endif else ifeq ($(TAG_LATEST), true) - docker buildx build . -f $(DOCKERFILE_DIR)/Dockerfile-tools $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${IMG}:latest --push $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile-tools --platform $(BUILDX_PLATFORMS) -t ${TOOL_IMG}:latest --push $(BUILDX_ARGS) else - docker buildx build . -f $(DOCKERFILE_DIR)/Dockerfile-tools $(DOCKER_BUILD_ARGS) --platform $(BUILDX_PLATFORMS) -t ${IMG}:${VERSION} --push $(BUILDX_ARGS) + $(DOCKER) buildx build . $(DOCKER_BUILD_ARGS) -f $(DOCKERFILE_DIR)/Dockerfile-tools --platform $(BUILDX_PLATFORMS) -t ${TOOL_IMG}:${VERSION} --push $(BUILDX_ARGS) endif endif From d7f6814f1b522701d1a7038e2c552d137144f02b Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Wed, 10 May 2023 17:00:18 +0800 Subject: [PATCH 265/439] chore: cli cluster describe does not show events (#3094) --- internal/cli/cmd/cluster/describe.go | 28 ++++------------------- internal/cli/cmd/cluster/describe_test.go | 9 ++------ 2 files changed, 6 insertions(+), 31 deletions(-) diff --git a/internal/cli/cmd/cluster/describe.go b/internal/cli/cmd/cluster/describe.go index 5b8fec2ce..74de7d65a 100644 --- a/internal/cli/cmd/cluster/describe.go +++ b/internal/cli/cmd/cluster/describe.go @@ -41,7 +41,6 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" - "github.com/apecloud/kubeblocks/internal/constant" ) var ( @@ -135,7 +134,6 @@ func (o *describeOptions) describeCluster(name string) error { WithClusterDef: true, WithService: true, WithPod: true, - WithEvent: true, WithPVC: true, WithDataProtection: true, }, @@ -166,7 +164,7 @@ func (o *describeOptions) describeCluster(name string) error { showDataProtection(o.BackupPolicies, o.Backups, o.Out) // events - showEvents(o.Events, o.Cluster.Name, o.Cluster.Namespace, o.Out) + showEvents(o.Cluster.Name, o.Cluster.Namespace, o.Out) fmt.Fprintln(o.Out) return nil @@ -206,27 +204,9 @@ func showImages(comps []*cluster.ComponentInfo, out io.Writer) { tbl.Print() } -func showEvents(events *corev1.EventList, name string, namespace string, out io.Writer) { - objs := util.SortEventsByLastTimestamp(events, corev1.EventTypeWarning) - - // print last 5 events - title := fmt.Sprintf("\nEvents(last 5 warnings, see more:kbcli cluster list-events -n %s %s):", namespace, name) - tbl := newTbl(out, title, "TIME", "TYPE", "REASON", "OBJECT", "MESSAGE") - cnt := 0 - for _, o := range *objs { - e := o.(*corev1.Event) - // do not output KubeBlocks probe events - if e.InvolvedObject.FieldPath == constant.ProbeCheckRolePath { - continue - } - - tbl.AddRow(util.GetEventTimeStr(e), e.Type, e.Reason, util.GetEventObject(e), e.Message) - cnt++ - if cnt == 5 { - break - } - } - tbl.Print() +func showEvents(name string, namespace string, out io.Writer) { + // hint user how to get events + fmt.Fprintf(out, "\nShow cluster events: kbcli cluster list-events -n %s %s", namespace, name) } func showEndpoints(c *appsv1alpha1.Cluster, svcList *corev1.ServiceList, out io.Writer) { diff --git a/internal/cli/cmd/cluster/describe_test.go b/internal/cli/cmd/cluster/describe_test.go index 7b99f538c..584d0b458 100644 --- a/internal/cli/cmd/cluster/describe_test.go +++ b/internal/cli/cmd/cluster/describe_test.go @@ -109,13 +109,8 @@ var _ = Describe("Expose", func() { It("showEvents", func() { out := &bytes.Buffer{} - showEvents(testing.FakeEvents(), "test-cluster", namespace, out) - strs := strings.Split(out.String(), "\n") - - // sorted - firstEvent := strs[3] - secondEvent := strs[4] - Expect(strings.Compare(firstEvent, secondEvent) < 0).Should(BeTrue()) + showEvents("test-cluster", namespace, out) + Expect(out.String()).ShouldNot(BeEmpty()) }) It("showDataProtections", func() { From 99c6834ebc6d8cfec5c873d3141a8228c26dd89e Mon Sep 17 00:00:00 2001 From: chantu Date: Wed, 10 May 2023 17:54:35 +0800 Subject: [PATCH 266/439] fix: mysql real time component replicas info (#3177) --- controllers/apps/cluster_controller.go | 2 + deploy/apecloud-mysql/templates/scripts.yaml | 19 ++--- .../lifecycle/transformer_sts_pods.go | 82 +++++++++++++++++++ 3 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 internal/controller/lifecycle/transformer_sts_pods.go diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index 76fd48a83..2b450bfe5 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -201,6 +201,8 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct &lifecycle.ClusterStatusTransformer{}, // handle PITR &lifecycle.PITRTransformer{Client: r.Client}, + // update the real-time component replicas info to pods + &lifecycle.StsPodsTransformer{}, // always safe to put your transformer below ). Build() diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index c0ccedfcd..9d96ff13b 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -15,10 +15,6 @@ data: idx=${KB_POD_NAME##*-} host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) echo "host=$host" - # update replicas to persistent file - component_replicas_path=/data/mysql/.kb_component_replicas - current_component_replicas=`cat /etc/annotations/component-replicas` - echo $current_component_replicas > $component_replicas_path if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then echo "no leader or self is leader, no need to call add." else @@ -167,7 +163,6 @@ data: echo "no leader or self is leader, exit" >> /data/mysql/.kb_pre_stop.log exit 0 fi - idx=${KB_POD_NAME##*-} host=$(eval echo \$KB_MYSQL_"$idx"_HOSTNAME) echo "host=$host" >> /data/mysql/.kb_pre_stop.log leader_idx=${leader##*-} @@ -183,15 +178,13 @@ data: echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.drop_learner('$host:13306');\" 2>&1 " >> /data/mysql/.kb_pre_stop.log mysql $host_flag -uroot $password_flag -e "call dbms_consensus.drop_learner('$host:13306');" 2>&1 } - component_replicas_path=/data/mysql/.kb_component_replicas + idx=${KB_POD_NAME##*-} current_component_replicas=`cat /etc/annotations/component-replicas` - if [ -f $component_replicas_path ]; then - last_component_replicas=`cat $component_replicas_path` - # check is scaling in but not scaling in to 0 - if [ "$last_component_replicas" -gt "$current_component_replicas" ] && [ $current_component_replicas -ne 0 ]; then + echo "current replicas: $current_component_replicas" >> /data/mysql/.kb_pre_stop.log + if [ ! $idx -lt $current_component_replicas ] && [ $current_component_replicas -ne 0 ]; then + # if idx greater than or equal to current_component_replicas means the cluster's scaling in # only scaling in need to drop followers drop_followers - else - echo "no need to drop followers" - fi + else + echo "no need to drop followers" >> /data/mysql/.kb_pre_stop.log fi diff --git a/internal/controller/lifecycle/transformer_sts_pods.go b/internal/controller/lifecycle/transformer_sts_pods.go new file mode 100644 index 000000000..de3ee8cdd --- /dev/null +++ b/internal/controller/lifecycle/transformer_sts_pods.go @@ -0,0 +1,82 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd +This file is part of KubeBlocks project +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package lifecycle + +import ( + "strconv" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/controller/graph" +) + +type StsPodsTransformer struct{} + +func (t *StsPodsTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + origCluster := transCtx.OrigCluster + + if isClusterDeleting(*origCluster) { + return nil + } + + handlePodsUpdate := func(vertex *lifecycleVertex) error { + stsObj, _ := vertex.oriObj.(*appsv1.StatefulSet) + stsProto, _ := vertex.obj.(*appsv1.StatefulSet) + + if stsObj.Spec.Replicas != stsProto.Spec.Replicas { + ml := client.MatchingLabels{ + constant.AppInstanceLabelKey: stsObj.Labels[constant.AppInstanceLabelKey], + constant.KBAppComponentLabelKey: stsObj.Labels[constant.KBAppComponentLabelKey], + } + podList := corev1.PodList{} + if err := transCtx.Client.List(transCtx.Context, &podList, ml); err != nil { + return err + } + for _, pod := range podList.Items { + obj := pod.DeepCopy() + if obj.Annotations == nil { + obj.Annotations = make(map[string]string) + } + obj.Annotations[constant.ComponentReplicasAnnotationKey] = strconv.Itoa(int(*stsProto.Spec.Replicas)) + v := &lifecycleVertex{ + obj: obj, + oriObj: &pod, + action: actionPtr(UPDATE), + } + dag.AddVertex(v) + dag.Connect(vertex, v) + } + } + return nil + } + + vertices := findAll[*appsv1.StatefulSet](dag) + for _, vertex := range vertices { + v, _ := vertex.(*lifecycleVertex) + if v.obj != nil && v.oriObj != nil && v.action != nil && *v.action == UPDATE { + if err := handlePodsUpdate(v); err != nil { + return err + } + } + } + return nil +} + +var _ graph.Transformer = &StsPodsTransformer{} From 498b7071e754ffd8785c6a2655cec3f2efdc3b3a Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Wed, 10 May 2023 17:55:31 +0800 Subject: [PATCH 267/439] feat: add settings to set data plane toleration and node affinity (#3116) --- cmd/manager/main.go | 36 +++-- cmd/tpl/app/workflow.go | 2 +- controllers/apps/cluster_controller_test.go | 4 +- controllers/apps/suite_test.go | 10 ++ controllers/apps/systemaccount_controller.go | 4 +- controllers/apps/systemaccount_util.go | 13 +- controllers/apps/systemaccount_util_test.go | 8 +- deploy/helm/templates/configmap.yaml | 50 ++++--- deploy/helm/values.yaml | 20 +++ internal/constant/const.go | 13 +- internal/controller/builder/builder_test.go | 3 +- .../controller/component/affinity_utils.go | 139 +++++++++++++----- .../component/affinity_utils_test.go | 122 ++++++++------- internal/controller/component/component.go | 25 ++-- .../controller/component/component_test.go | 12 +- .../lifecycle/transformer_cluster.go | 6 +- .../transformer_sts_horizontal_scaling.go | 16 +- internal/controller/plan/prepare_test.go | 18 ++- internal/sqlchannel/client_test.go | 10 +- 19 files changed, 333 insertions(+), 178 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 529952614..ab7900b18 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -122,29 +122,45 @@ func (r flagName) viperName() string { } func validateRequiredToParseConfigs() error { - if jobTTL := viper.GetString(constant.CfgKeyAddonJobTTL); jobTTL != "" { - if _, err := time.ParseDuration(jobTTL); err != nil { - return err + validateTolerations := func(val string) error { + if val == "" { + return nil } + var tolerations []corev1.Toleration + return json.Unmarshal([]byte(val), &tolerations) } - if cmTolerations := viper.GetString(constant.CfgKeyCtrlrMgrTolerations); cmTolerations != "" { - Tolerations := []corev1.Toleration{} - if err := json.Unmarshal([]byte(cmTolerations), &Tolerations); err != nil { - return err + + validateAffinity := func(val string) error { + if val == "" { + return nil } - } - if cmAffinity := viper.GetString(constant.CfgKeyCtrlrMgrAffinity); cmAffinity != "" { affinity := corev1.Affinity{} - if err := json.Unmarshal([]byte(cmAffinity), &affinity); err != nil { + return json.Unmarshal([]byte(val), &affinity) + } + + if jobTTL := viper.GetString(constant.CfgKeyAddonJobTTL); jobTTL != "" { + if _, err := time.ParseDuration(jobTTL); err != nil { return err } } + if err := validateTolerations(viper.GetString(constant.CfgKeyCtrlrMgrTolerations)); err != nil { + return err + } + if err := validateAffinity(viper.GetString(constant.CfgKeyCtrlrMgrAffinity)); err != nil { + return err + } if cmNodeSelector := viper.GetString(constant.CfgKeyCtrlrMgrNodeSelector); cmNodeSelector != "" { nodeSelector := map[string]string{} if err := json.Unmarshal([]byte(cmNodeSelector), &nodeSelector); err != nil { return err } } + if err := validateTolerations(viper.GetString(constant.CfgKeyDataPlaneTolerations)); err != nil { + return err + } + if err := validateAffinity(viper.GetString(constant.CfgKeyDataPlaneAffinity)); err != nil { + return err + } return nil } diff --git a/cmd/tpl/app/workflow.go b/cmd/tpl/app/workflow.go index b9ec44d0e..1dd12ce9b 100644 --- a/cmd/tpl/app/workflow.go +++ b/cmd/tpl/app/workflow.go @@ -108,7 +108,7 @@ func (w *templateRenderWorkflow) Prepare(ctx intctrlutil.RequestCtx, componentTy return nil, cfgcore.MakeError("component[%s] is not defined in cluster definition", componentType) } - return component.BuildComponent(ctx, *cluster, *w.clusterDefObj, *clusterCompDef, clusterCompSpec[0]), nil + return component.BuildComponent(ctx, *cluster, *w.clusterDefObj, *clusterCompDef, clusterCompSpec[0]) } func (w *templateRenderWorkflow) getRenderedConfigSpec() ([]componentedConfigSpec, error) { diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index f256c160c..0c28460be 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1250,13 +1250,13 @@ var _ = Describe("Cluster Controller", func() { podSpec := stsList.Items[0].Spec.Template.Spec By("Checking created sts pods template with built-in toleration") Expect(podSpec.Tolerations).Should(HaveLen(1)) - Expect(podSpec.Tolerations[0].Key).To(Equal(constant.KubeBlocksDataNodeTolerationKey)) + Expect(podSpec.Tolerations[0].Key).To(Equal(testDataPlaneTolerationKey)) By("Checking created sts pods template with built-in Affinity") Expect(podSpec.Affinity.PodAntiAffinity == nil && podSpec.Affinity.PodAffinity == nil).Should(BeTrue()) Expect(podSpec.Affinity.NodeAffinity).ShouldNot(BeNil()) Expect(podSpec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Key).To( - Equal(constant.KubeBlocksDataNodeLabelKey)) + Equal(testDataPlaneNodeAffinityKey)) By("Checking created sts pods template without TopologySpreadConstraints") Expect(podSpec.TopologySpreadConstraints).Should(BeEmpty()) diff --git a/controllers/apps/suite_test.go b/controllers/apps/suite_test.go index 279cb4926..950fa2ba3 100644 --- a/controllers/apps/suite_test.go +++ b/controllers/apps/suite_test.go @@ -21,6 +21,7 @@ package apps import ( "context" + "fmt" "go/build" "path/filepath" "testing" @@ -55,6 +56,11 @@ import ( // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. +const ( + testDataPlaneNodeAffinityKey = "testDataPlaneNodeAffinityKey" + testDataPlaneTolerationKey = "testDataPlaneTolerationKey" +) + var cfg *rest.Config var k8sClient client.Client var testEnv *envtest.Environment @@ -84,6 +90,10 @@ var _ = BeforeSuite(func() { } viper.SetDefault(constant.CfgKeyCtrlrReconcileRetryDurationMS, 10) + viper.Set(constant.CfgKeyDataPlaneTolerations, + fmt.Sprintf("[{\"key\":\"%s\", \"operator\": \"Exists\", \"effect\": \"NoSchedule\"}]", testDataPlaneTolerationKey)) + viper.Set(constant.CfgKeyDataPlaneAffinity, + fmt.Sprintf("{\"nodeAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"preference\":{\"matchExpressions\":[{\"key\":\"%s\",\"operator\":\"In\",\"values\":[\"true\"]}]},\"weight\":100}]}}", testDataPlaneNodeAffinityKey)) ctx, cancel = context.WithCancel(context.TODO()) logger = logf.FromContext(ctx).WithValues() logger.Info("logger start") diff --git a/controllers/apps/systemaccount_controller.go b/controllers/apps/systemaccount_controller.go index 15b00ebcd..fc0fa02cc 100644 --- a/controllers/apps/systemaccount_controller.go +++ b/controllers/apps/systemaccount_controller.go @@ -295,7 +295,9 @@ func (r *SystemAccountReconciler) createByStmt(reqCtx intctrlutil.RequestCtx, // render a job object job := renderJob(engine, compKey, stmts, ep) // before create job, we adjust job's attributes, such as labels, tolerations w.r.t cluster info. - calibrateJobMetaAndSpec(job, cluster, compKey, account.Name) + if err := calibrateJobMetaAndSpec(job, cluster, compKey, account.Name); err != nil { + return err + } // update owner reference if err := controllerutil.SetControllerReference(cluster, job, scheme); err != nil { return err diff --git a/controllers/apps/systemaccount_util.go b/controllers/apps/systemaccount_util.go index 6f1dae93c..7fda12aba 100644 --- a/controllers/apps/systemaccount_util.go +++ b/controllers/apps/systemaccount_util.go @@ -363,7 +363,7 @@ func getDebugMode(annotatedDebug string) bool { return viper.GetBool(systemAccountsDebugMode) || debugOn } -func calibrateJobMetaAndSpec(job *batchv1.Job, cluster *appsv1alpha1.Cluster, compKey componentUniqueKey, account appsv1alpha1.AccountName) { +func calibrateJobMetaAndSpec(job *batchv1.Job, cluster *appsv1alpha1.Cluster, compKey componentUniqueKey, account appsv1alpha1.AccountName) error { debugModeOn := getDebugMode(cluster.Annotations[debugClusterAnnotationKey]) // add label ml := getLabelsForSecretsAndJobs(compKey) @@ -379,16 +379,13 @@ func calibrateJobMetaAndSpec(job *batchv1.Job, cluster *appsv1alpha1.Cluster, co } // add toleration - tolerations := cluster.Spec.Tolerations clusterComp := cluster.Spec.GetComponentByName(compKey.componentName) - if clusterComp != nil { - if len(clusterComp.Tolerations) != 0 { - tolerations = clusterComp.Tolerations - } + tolerations, err := componetutil.BuildTolerations(cluster, clusterComp) + if err != nil { + return err } - // add built-in toleration - tolerations = componetutil.PatchBuiltInToleration(tolerations) job.Spec.Template.Spec.Tolerations = tolerations + return nil } // completeExecConfig override the image of execConfig if version is not nil. diff --git a/controllers/apps/systemaccount_util_test.go b/controllers/apps/systemaccount_util_test.go index b72b1e0db..0ad5ee74c 100644 --- a/controllers/apps/systemaccount_util_test.go +++ b/controllers/apps/systemaccount_util_test.go @@ -209,7 +209,7 @@ func TestRenderJob(t *testing.T) { endpoint := "10.0.0.1" job := renderJob(engine, compKey, creationStmt, endpoint) assert.NotNil(t, job) - calibrateJobMetaAndSpec(job, cluster, compKey, acc.Name) + _ = calibrateJobMetaAndSpec(job, cluster, compKey, acc.Name) assert.NotNil(t, job.Spec.TTLSecondsAfterFinished) assert.Equal(t, (int32)(0), *job.Spec.TTLSecondsAfterFinished) envList := job.Spec.Template.Spec.Containers[0].Env @@ -220,7 +220,7 @@ func TestRenderJob(t *testing.T) { assert.NotNil(t, job) // set debug mode on cluster.Annotations[debugClusterAnnotationKey] = "True" - calibrateJobMetaAndSpec(job, cluster, compKey, acc.Name) + _ = calibrateJobMetaAndSpec(job, cluster, compKey, acc.Name) assert.Nil(t, job.Spec.TTLSecondsAfterFinished) assert.NotNil(t, secrets) // set debug mode off @@ -231,7 +231,7 @@ func TestRenderJob(t *testing.T) { cluster.Spec.Tolerations = toleration job = renderJob(engine, compKey, creationStmt, endpoint) assert.NotNil(t, job) - calibrateJobMetaAndSpec(job, cluster, compKey, acc.Name) + _ = calibrateJobMetaAndSpec(job, cluster, compKey, acc.Name) jobToleration := job.Spec.Template.Spec.Tolerations assert.Equal(t, 2, len(jobToleration)) // make sure the toleration is added to job and contains our built-in toleration @@ -239,7 +239,7 @@ func TestRenderJob(t *testing.T) { for _, t := range jobToleration { tolerationKeys = append(tolerationKeys, t.Key) } - assert.Contains(t, tolerationKeys, constant.KubeBlocksDataNodeTolerationKey) + assert.Contains(t, tolerationKeys, testDataPlaneTolerationKey) assert.Contains(t, tolerationKeys, toleration[0].Key) case appsv1alpha1.ReferToExisting: assert.False(t, strings.Contains(acc.ProvisionPolicy.SecretRef.Name, constant.ConnCredentialPlaceHolder)) diff --git a/deploy/helm/templates/configmap.yaml b/deploy/helm/templates/configmap.yaml index fd8b8fb01..89cf1cf62 100644 --- a/deploy/helm/templates/configmap.yaml +++ b/deploy/helm/templates/configmap.yaml @@ -5,29 +5,37 @@ metadata: labels: {{- include "kubeblocks.labels" . | nindent 4 }} data: - config.yaml: | - # the global pvc name which persistent volume claim to store the backup data. - # will replace the pvc name when it is empty in the backup policy. - BACKUP_PVC_NAME: "{{ .Values.dataProtection.backupPVCName }}" + config.yaml: | + # the global pvc name which persistent volume claim to store the backup data. + # will replace the pvc name when it is empty in the backup policy. + BACKUP_PVC_NAME: "{{ .Values.dataProtection.backupPVCName }}" - # the init capacity of pvc for creating the pvc, e.g. 10Gi. - # will replace the init capacity when it is empty in the backup policy. - BACKUP_PVC_INIT_CAPACITY: "{{ .Values.dataProtection.backupPVCInitCapacity }}" + # the init capacity of pvc for creating the pvc, e.g. 10Gi. + # will replace the init capacity when it is empty in the backup policy. + BACKUP_PVC_INIT_CAPACITY: "{{ .Values.dataProtection.backupPVCInitCapacity }}" - # the pvc storage class name. - # will replace the storageClassName when it is nil in the backup policy. - BACKUP_PVC_STORAGE_CLASS: "{{ .Values.dataProtection.backupPVCStorageClassName }}" + # the pvc storage class name. + # will replace the storageClassName when it is nil in the backup policy. + BACKUP_PVC_STORAGE_CLASS: "{{ .Values.dataProtection.backupPVCStorageClassName }}" - # the pvc create policy. - # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. - # otherwise, using "Never" policy. - # only affect the backupPolicy automatically created by KubeBs-locks. - BACKUP_PVC_CREATE_POLICY: "{{ .Values.dataProtection.backupPVCCreatePolicy }}" + # the pvc create policy. + # if the storageClass supports dynamic provisioning, recommend "IfNotPresent" policy. + # otherwise, using "Never" policy. + # only affect the backupPolicy automatically created by KubeBs-locks. + BACKUP_PVC_CREATE_POLICY: "{{ .Values.dataProtection.backupPVCCreatePolicy }}" - # the configmap name of the pv template. if the csi-driver not support dynamic provisioning, - # you can provide a configmap which contains key "persistentVolume" and value of the persistentVolume struct. - # only effective when storageClass is empty. - BACKUP_PV_CONFIGMAP_NAME: "{{ .Values.dataProtection.backupPVConfigMapName }}" + # the configmap name of the pv template. if the csi-driver not support dynamic provisioning, + # you can provide a configmap which contains key "persistentVolume" and value of the persistentVolume struct. + # only effective when storageClass is empty. + BACKUP_PV_CONFIGMAP_NAME: "{{ .Values.dataProtection.backupPVConfigMapName }}" - # the configmap namespace of the pv template. - BACKUP_PV_CONFIGMAP_NAMESPACE: "{{ .Values.dataProtection.backupPVConfigMapNamespace }}" \ No newline at end of file + # the configmap namespace of the pv template. + BACKUP_PV_CONFIGMAP_NAMESPACE: "{{ .Values.dataProtection.backupPVConfigMapNamespace }}" + + {{- with .Values.dataPlane }} + # data plane tolerations + DATA_PLANE_TOLERATIONS: {{ toJson .tolerations | squote }} + + # data plane affinity + DATA_PLANE_AFFINITY: {{ toJson .affinity | squote }} + {{- end }} \ No newline at end of file diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index da3579242..316db5304 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -222,6 +222,26 @@ affinity: values: - "true" +## @param data plane settings +## +dataPlane: + tolerations: + - key: kb-data + operator: Equal + value: "true" + effect: NoSchedule + + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kb-data + operator: In + values: + - "true" + ## PDB settings ## ## @param podDisruptionBudget.minAvailable diff --git a/internal/constant/const.go b/internal/constant/const.go index caea4c3f9..5885f71e2 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -37,6 +37,10 @@ const ( // addon config keys CfgKeyAddonJobTTL = "ADDON_JOB_TTL" CfgAddonJobImgPullPolicy = "ADDON_JOB_IMAGE_PULL_POLICY" + + // data plane config key + CfgKeyDataPlaneTolerations = "DATA_PLANE_TOLERATIONS" + CfgKeyDataPlaneAffinity = "DATA_PLANE_AFFINITY" ) const ( @@ -195,15 +199,6 @@ const ( ProbeCheckRolePath = "spec.containers{" + RoleProbeContainerName + "}" ProbeCheckStatusPath = "spec.containers{" + StatusProbeContainerName + "}" ProbeCheckRunningPath = "spec.containers{" + RunningProbeContainerName + "}" - - // KubeBlocksDataNodeLabelKey is the node label key of the built-in data node label - KubeBlocksDataNodeLabelKey = "kb-data" - // KubeBlocksDataNodeLabelValue is the node label value of the built-in data node label - KubeBlocksDataNodeLabelValue = "true" - // KubeBlocksDataNodeTolerationKey is the taint label key of the built-in data node taint - KubeBlocksDataNodeTolerationKey = "kb-data" - // KubeBlocksDataNodeTolerationValue is the taint label value of the built-in data node taint - KubeBlocksDataNodeTolerationValue = "true" ) const ( diff --git a/internal/controller/builder/builder_test.go b/internal/controller/builder/builder_test.go index a9536e0f8..768497971 100644 --- a/internal/controller/builder/builder_test.go +++ b/internal/controller/builder/builder_test.go @@ -152,13 +152,14 @@ var _ = Describe("builder", func() { cluster, clusterDef, clusterVersion, _ := newAllFieldsClusterObj(clusterDef, clusterVersion, false) reqCtx := newReqCtx() By("assign every available fields") - component := component.BuildComponent( + component, err := component.BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0]) + Expect(err).Should(Succeed()) Expect(component).ShouldNot(BeNil()) return component } diff --git a/internal/controller/component/affinity_utils.go b/internal/controller/component/affinity_utils.go index 8dd97a4c3..412133598 100644 --- a/internal/controller/component/affinity_utils.go +++ b/internal/controller/component/affinity_utils.go @@ -20,13 +20,15 @@ along with this program. If not, see . package component import ( + "encoding/json" "strings" + "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - intctrlutil "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/constant" ) func buildPodTopologySpreadConstraints( @@ -53,8 +55,8 @@ func buildPodTopologySpreadConstraints( TopologyKey: topologyKey, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - intctrlutil.AppInstanceLabelKey: cluster.Name, - intctrlutil.KBAppComponentLabelKey: component.Name, + constant.AppInstanceLabelKey: cluster.Name, + constant.KBAppComponentLabelKey: component.Name, }, }, }) @@ -66,6 +68,23 @@ func buildPodAffinity( cluster *appsv1alpha1.Cluster, clusterOrCompAffinity *appsv1alpha1.Affinity, component *SynthesizedComponent, +) (*corev1.Affinity, error) { + affinity := buildNewAffinity(cluster, clusterOrCompAffinity, component) + + // read data plane affinity from config and merge it + dpAffinity := new(corev1.Affinity) + if val := viper.GetString(constant.CfgKeyDataPlaneAffinity); val != "" { + if err := json.Unmarshal([]byte(val), &dpAffinity); err != nil { + return nil, err + } + } + return mergeAffinity(affinity, dpAffinity) +} + +func buildNewAffinity( + cluster *appsv1alpha1.Cluster, + clusterOrCompAffinity *appsv1alpha1.Affinity, + component *SynthesizedComponent, ) *corev1.Affinity { if clusterOrCompAffinity == nil { return nil @@ -99,8 +118,8 @@ func buildPodAffinity( TopologyKey: topologyKey, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - intctrlutil.AppInstanceLabelKey: cluster.Name, - intctrlutil.KBAppComponentLabelKey: component.Name, + constant.AppInstanceLabelKey: cluster.Name, + constant.KBAppComponentLabelKey: component.Name, }, }, }) @@ -125,7 +144,7 @@ func buildPodAffinity( if clusterOrCompAffinity.Tenancy == appsv1alpha1.DedicatedNode { var labelSelectorReqs []metav1.LabelSelectorRequirement labelSelectorReqs = append(labelSelectorReqs, metav1.LabelSelectorRequirement{ - Key: intctrlutil.WorkloadTypeLabelKey, + Key: constant.WorkloadTypeLabelKey, Operator: metav1.LabelSelectorOpIn, Values: appsv1alpha1.WorkloadTypes, }) @@ -141,42 +160,86 @@ func buildPodAffinity( return affinity } -// patchBuiltInAffinity patches built-in affinity configuration -func patchBuiltInAffinity(affinity *corev1.Affinity) *corev1.Affinity { - var matchExpressions []corev1.NodeSelectorRequirement - matchExpressions = append(matchExpressions, corev1.NodeSelectorRequirement{ - Key: intctrlutil.KubeBlocksDataNodeLabelKey, - Operator: corev1.NodeSelectorOpIn, - Values: []string{intctrlutil.KubeBlocksDataNodeLabelValue}, - }) - preferredSchedulingTerm := corev1.PreferredSchedulingTerm{ - Preference: corev1.NodeSelectorTerm{ - MatchExpressions: matchExpressions, - }, - Weight: 100, +// mergeAffinity merge affinity from src to dest +func mergeAffinity(dest, src *corev1.Affinity) (*corev1.Affinity, error) { + if src == nil { + return dest, nil } - if affinity != nil && affinity.NodeAffinity != nil { - affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append( - affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution, preferredSchedulingTerm) - } else { - if affinity == nil { - affinity = new(corev1.Affinity) + + if dest == nil { + return src.DeepCopy(), nil + } + + rst := dest.DeepCopy() + skipPodAffinity := src.PodAffinity == nil + skipPodAntiAffinity := src.PodAntiAffinity == nil + skipNodeAffinity := src.NodeAffinity == nil + + if rst.PodAffinity == nil && !skipPodAffinity { + rst.PodAffinity = src.PodAffinity + skipPodAffinity = true + } + if rst.PodAntiAffinity == nil && !skipPodAntiAffinity { + rst.PodAntiAffinity = src.PodAntiAffinity + skipPodAntiAffinity = true + } + if rst.NodeAffinity == nil && !skipNodeAffinity { + rst.NodeAffinity = src.NodeAffinity + skipNodeAffinity = true + } + + // if not skip, both are not nil + if !skipPodAffinity { + rst.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append( + rst.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, + src.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution...) + + rst.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append( + rst.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, + src.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution...) + } + if !skipPodAntiAffinity { + rst.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append( + rst.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, + src.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution...) + + rst.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append( + rst.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, + src.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution...) + } + if !skipNodeAffinity { + rst.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append( + rst.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution, + src.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution...) + + skip := src.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil + if rst.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil && !skip { + rst.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = src.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + skip = true } - affinity.NodeAffinity = &corev1.NodeAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{preferredSchedulingTerm}, + if !skip { + rst.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( + rst.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, + src.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms...) } } - - return affinity + return rst, nil } -// PatchBuiltInToleration patches built-in tolerations configuration -func PatchBuiltInToleration(tolerations []corev1.Toleration) []corev1.Toleration { - tolerations = append(tolerations, corev1.Toleration{ - Key: intctrlutil.KubeBlocksDataNodeTolerationKey, - Operator: corev1.TolerationOpEqual, - Value: intctrlutil.KubeBlocksDataNodeTolerationValue, - Effect: corev1.TaintEffectNoSchedule, - }) - return tolerations +// BuildTolerations builds tolerations from config +func BuildTolerations(cluster *appsv1alpha1.Cluster, clusterCompSpec *appsv1alpha1.ClusterComponentSpec) ([]corev1.Toleration, error) { + tolerations := cluster.Spec.Tolerations + if clusterCompSpec != nil && len(clusterCompSpec.Tolerations) != 0 { + tolerations = clusterCompSpec.Tolerations + } + + // build data plane tolerations from config + var dpTolerations []corev1.Toleration + if val := viper.GetString(constant.CfgKeyDataPlaneTolerations); val != "" { + if err := json.Unmarshal([]byte(val), &dpTolerations); err != nil { + return nil, err + } + } + + return append(tolerations, dpTolerations...), nil } diff --git a/internal/controller/component/affinity_utils_test.go b/internal/controller/component/affinity_utils_test.go index 340b7b55c..c261ea6c0 100644 --- a/internal/controller/component/affinity_utils_test.go +++ b/internal/controller/component/affinity_utils_test.go @@ -20,9 +20,12 @@ along with this program. If not, see . package component import ( + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -38,19 +41,19 @@ var _ = Describe("affinity utils", func() { clusterName = "test-cluster" mysqlCompDefName = "replicasets" mysqlCompName = "mysql" + + clusterTolerationKey = "testClusterTolerationKey" + topologyKey = "testTopologyKey" + labelKey = "testNodeLabelKey" + labelValue = "testLabelValue" + nodeKey = "testNodeKey" ) var ( clusterObj *appsv1alpha1.Cluster component *SynthesizedComponent - ) - - Context("with PodAntiAffinity set to Required", func() { - const topologyKey = "testTopologyKey" - const labelKey = "testNodeLabelKey" - const labelValue = "testLabelValue" - BeforeEach(func() { + buildObjs = func(podAntiAffinity appsv1alpha1.PodAntiAffinity) { clusterDefObj := testapps.NewClusterDefFactory(clusterDefName). AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). GetObject() @@ -60,41 +63,71 @@ var _ = Describe("affinity utils", func() { GetObject() affinity := &appsv1alpha1.Affinity{ - PodAntiAffinity: appsv1alpha1.Required, + PodAntiAffinity: podAntiAffinity, TopologyKeys: []string{topologyKey}, NodeLabels: map[string]string{ labelKey: labelValue, }, } + + toleration := corev1.Toleration{ + Key: clusterTolerationKey, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoExecute, + } + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name). AddComponent(mysqlCompName, mysqlCompDefName). SetClusterAffinity(affinity). + AddClusterToleration(toleration). GetObject() reqCtx := intctrlutil.RequestCtx{ Ctx: ctx, Log: tlog, } - component = BuildComponent( + component, _ = BuildComponent( reqCtx, *clusterObj, *clusterDefObj, clusterDefObj.Spec.ComponentDefs[0], clusterObj.Spec.ComponentSpecs[0], - &clusterVersionObj.Spec.ComponentVersions[0]) + &clusterVersionObj.Spec.ComponentVersions[0], + ) + } + ) + + Context("with PodAntiAffinity set to Required", func() { + BeforeEach(func() { + buildObjs(appsv1alpha1.Required) Expect(component).ShouldNot(BeNil()) }) It("should have correct Affinity and TopologySpreadConstraints", func() { - affinity := buildPodAffinity(clusterObj, clusterObj.Spec.Affinity, component) + affinity, err := buildPodAffinity(clusterObj, clusterObj.Spec.Affinity, component) + Expect(err).Should(Succeed()) Expect(affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).Should(Equal(labelKey)) Expect(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey).Should(Equal(topologyKey)) Expect(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution).Should(BeEmpty()) + Expect(affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution).Should(BeEmpty()) - affinity = patchBuiltInAffinity(affinity) - Expect(affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Key).Should( - Equal(constant.KubeBlocksDataNodeLabelKey)) + topologySpreadConstraints := buildPodTopologySpreadConstraints(clusterObj, clusterObj.Spec.Affinity, component) + Expect(topologySpreadConstraints[0].WhenUnsatisfiable).Should(Equal(corev1.DoNotSchedule)) + Expect(topologySpreadConstraints[0].TopologyKey).Should(Equal(topologyKey)) + }) + + It("when data plane affinity is set, should have correct Affinity and TopologySpreadConstraints", func() { + viper.Set(constant.CfgKeyDataPlaneAffinity, + fmt.Sprintf("{\"nodeAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"preference\":{\"matchExpressions\":[{\"key\":\"%s\",\"operator\":\"In\",\"values\":[\"true\"]}]},\"weight\":100}]}}", nodeKey)) + defer viper.Set(constant.CfgKeyDataPlaneAffinity, "") + + affinity, err := buildPodAffinity(clusterObj, clusterObj.Spec.Affinity, component) + Expect(err).Should(Succeed()) + Expect(affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).Should(Equal(labelKey)) + Expect(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey).Should(Equal(topologyKey)) + Expect(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution).Should(BeEmpty()) + Expect(affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Key).Should(Equal(nodeKey)) topologySpreadConstraints := buildPodTopologySpreadConstraints(clusterObj, clusterObj.Spec.Affinity, component) Expect(topologySpreadConstraints[0].WhenUnsatisfiable).Should(Equal(corev1.DoNotSchedule)) @@ -102,50 +135,39 @@ var _ = Describe("affinity utils", func() { }) }) - Context("with PodAntiAffinity set to Preferred", func() { - const topologyKey = "testTopologyKey" - const labelKey = "testNodeLabelKey" - const labelValue = "testLabelValue" - + Context("with tolerations", func() { BeforeEach(func() { - clusterDefObj := testapps.NewClusterDefFactory(clusterDefName). - AddComponentDef(testapps.StatefulMySQLComponent, mysqlCompDefName). - GetObject() + buildObjs(appsv1alpha1.Required) + }) - clusterVersionObj := testapps.NewClusterVersionFactory(clusterVersionName, clusterDefObj.Name). - AddComponentVersion(mysqlCompDefName).AddContainerShort("mysql", testapps.ApeCloudMySQLImage). - GetObject() + It("should have correct tolerations", func() { + tolerations, err := BuildTolerations(clusterObj, &clusterObj.Spec.ComponentSpecs[0]) + Expect(err).Should(Succeed()) + Expect(tolerations).ShouldNot(BeEmpty()) + Expect(tolerations[0].Key).Should(Equal(clusterTolerationKey)) + }) - affinity := &appsv1alpha1.Affinity{ - PodAntiAffinity: appsv1alpha1.Preferred, - TopologyKeys: []string{topologyKey}, - NodeLabels: map[string]string{ - labelKey: labelValue, - }, - } - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, - clusterDefObj.Name, clusterVersionObj.Name). - AddComponent(mysqlCompName, mysqlCompDefName). - SetClusterAffinity(affinity). - GetObject() + It("when data plane tolerations is set, should have correct tolerations", func() { + const dpTolerationKey = "dataPlaneTolerationKey" + viper.Set(constant.CfgKeyDataPlaneTolerations, fmt.Sprintf("[{\"key\":\"%s\", \"operator\": \"Exists\", \"effect\": \"NoSchedule\"}]", dpTolerationKey)) + defer viper.Set(constant.CfgKeyDataPlaneTolerations, "") + tolerations, err := BuildTolerations(clusterObj, &clusterObj.Spec.ComponentSpecs[0]) + Expect(err).Should(Succeed()) + Expect(tolerations).Should(HaveLen(2)) + Expect(tolerations[0].Key).Should(Equal(clusterTolerationKey)) + Expect(tolerations[1].Key).Should(Equal(dpTolerationKey)) + }) + }) - reqCtx := intctrlutil.RequestCtx{ - Ctx: ctx, - Log: tlog, - } - component = BuildComponent( - reqCtx, - *clusterObj, - *clusterDefObj, - clusterDefObj.Spec.ComponentDefs[0], - clusterObj.Spec.ComponentSpecs[0], - &clusterVersionObj.Spec.ComponentVersions[0], - ) + Context("with PodAntiAffinity set to Preferred", func() { + BeforeEach(func() { + buildObjs(appsv1alpha1.Preferred) Expect(component).ShouldNot(BeNil()) }) It("should have correct Affinity and TopologySpreadConstraints", func() { - affinity := buildPodAffinity(clusterObj, clusterObj.Spec.Affinity, component) + affinity, err := buildPodAffinity(clusterObj, clusterObj.Spec.Affinity, component) + Expect(err).Should(Succeed()) Expect(affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).Should(Equal(labelKey)) Expect(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution).Should(BeEmpty()) Expect(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight).ShouldNot(BeNil()) diff --git a/internal/controller/component/component.go b/internal/controller/component/component.go index 33bc8dd22..6104eb5e0 100644 --- a/internal/controller/component/component.go +++ b/internal/controller/component/component.go @@ -41,8 +41,8 @@ func BuildComponent( clusterCompDef appsv1alpha1.ClusterComponentDefinition, clusterCompSpec appsv1alpha1.ClusterComponentSpec, clusterCompVers ...*appsv1alpha1.ClusterComponentVersion, -) *SynthesizedComponent { - +) (*SynthesizedComponent, error) { + var err error clusterCompDefObj := clusterCompDef.DeepCopy() component := &SynthesizedComponent{ ClusterDefName: clusterDef.Name, @@ -92,15 +92,15 @@ func BuildComponent( if clusterCompSpec.Affinity != nil { affinity = clusterCompSpec.Affinity } - podAffinity := buildPodAffinity(&cluster, affinity, component) - component.PodSpec.Affinity = patchBuiltInAffinity(podAffinity) + if component.PodSpec.Affinity, err = buildPodAffinity(&cluster, affinity, component); err != nil { + reqCtx.Log.Error(err, "build pod affinity failed.") + return nil, err + } component.PodSpec.TopologySpreadConstraints = buildPodTopologySpreadConstraints(&cluster, affinity, component) - - tolerations := cluster.Spec.Tolerations - if len(clusterCompSpec.Tolerations) != 0 { - tolerations = clusterCompSpec.Tolerations + if component.PodSpec.Tolerations, err = BuildTolerations(&cluster, &clusterCompSpec); err != nil { + reqCtx.Log.Error(err, "build pod tolerations failed.") + return nil, err } - component.PodSpec.Tolerations = PatchBuiltInToleration(tolerations) if clusterCompSpec.VolumeClaimTemplates != nil { component.VolumeClaimTemplates = clusterCompSpec.ToVolumeClaimTemplates() @@ -143,15 +143,14 @@ func BuildComponent( // } buildMonitorConfig(&clusterCompDef, &clusterCompSpec, component) - err := buildProbeContainers(reqCtx, component) - if err != nil { + if err = buildProbeContainers(reqCtx, component); err != nil { reqCtx.Log.Error(err, "build probe container failed.") - return nil + return nil, err } replaceContainerPlaceholderTokens(component, GetEnvReplacementMapForConnCredential(cluster.GetName())) - return component + return component, nil } // appendOrOverrideContainerAttr is used to append targetContainer to compContainers or override the attributes of compContainers with a given targetContainer, diff --git a/internal/controller/component/component_test.go b/internal/controller/component/component_test.go index 3b9f8a4e6..efbe2059c 100644 --- a/internal/controller/component/component_test.go +++ b/internal/controller/component/component_test.go @@ -82,45 +82,49 @@ var _ = Describe("component module", func() { Ctx: ctx, Log: tlog, } - component := BuildComponent( + component, err := BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0]) + Expect(err).Should(Succeed()) Expect(component).ShouldNot(BeNil()) By("leave clusterVersion.versionCtx empty initContains and containers") clusterVersion.Spec.ComponentVersions[0].VersionsCtx.Containers = nil clusterVersion.Spec.ComponentVersions[0].VersionsCtx.InitContainers = nil - component = BuildComponent( + component, err = BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0]) + Expect(err).Should(Succeed()) Expect(component).ShouldNot(BeNil()) By("new container in clusterVersion not in clusterDefinition") - component = BuildComponent( + component, err = BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[1]) + Expect(err).Should(Succeed()) Expect(len(component.PodSpec.Containers)).Should(Equal(2)) By("new init container in clusterVersion not in clusterDefinition") - component = BuildComponent( + component, err = BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[1]) + Expect(err).Should(Succeed()) Expect(len(component.PodSpec.InitContainers)).Should(Equal(1)) }) diff --git a/internal/controller/lifecycle/transformer_cluster.go b/internal/controller/lifecycle/transformer_cluster.go index 9e54dad92..f55d8304d 100644 --- a/internal/controller/lifecycle/transformer_cluster.go +++ b/internal/controller/lifecycle/transformer_cluster.go @@ -109,7 +109,11 @@ func (c *ClusterTransformer) Transform(ctx graph.TransformContext, dag *graph.DA compVer := clusterCompVerMap[compDefName] compSpecs := clusterCompSpecMap[compDefName] for _, compSpec := range compSpecs { - if err := prepareComp(component.BuildComponent(reqCtx, *cluster, *transCtx.ClusterDef, compDef, compSpec, compVer)); err != nil { + c, err := component.BuildComponent(reqCtx, *cluster, *transCtx.ClusterDef, compDef, compSpec, compVer) + if err != nil { + return err + } + if err := prepareComp(c); err != nil { return err } } diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index efc0eb51b..03f68ddb4 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -75,13 +75,18 @@ func (t *StsHorizontalScalingTransformer) Transform(ctx graph.TransformContext, Namespace: stsObj.Namespace, Name: stsObj.Name + "-scaling", } + // find component of current statefulset componentName := stsObj.Labels[constant.KBAppComponentLabelKey] - components := mergeComponentsList(reqCtx, + components, err := mergeComponentsList(reqCtx, *cluster, *transCtx.ClusterDef, transCtx.ClusterDef.Spec.ComponentDefs, cluster.Spec.ComponentSpecs) + if err != nil { + return err + } + comp := getComponent(components, componentName) if comp == nil { if *stsObj.Spec.Replicas != *stsProto.Spec.Replicas { @@ -344,18 +349,21 @@ func mergeComponentsList(reqCtx intctrlutil.RequestCtx, cluster appsv1alpha1.Cluster, clusterDef appsv1alpha1.ClusterDefinition, clusterCompDefList []appsv1alpha1.ClusterComponentDefinition, - clusterCompSpecList []appsv1alpha1.ClusterComponentSpec) []component.SynthesizedComponent { + clusterCompSpecList []appsv1alpha1.ClusterComponentSpec) ([]component.SynthesizedComponent, error) { var compList []component.SynthesizedComponent for _, compDef := range clusterCompDefList { for _, compSpec := range clusterCompSpecList { if compSpec.ComponentDefRef != compDef.Name { continue } - comp := component.BuildComponent(reqCtx, cluster, clusterDef, compDef, compSpec) + comp, err := component.BuildComponent(reqCtx, cluster, clusterDef, compDef, compSpec) + if err != nil { + return nil, err + } compList = append(compList, *comp) } } - return compList + return compList, nil } func getComponent(componentList []component.SynthesizedComponent, name string) *component.SynthesizedComponent { diff --git a/internal/controller/plan/prepare_test.go b/internal/controller/plan/prepare_test.go index 903455042..d344bcec5 100644 --- a/internal/controller/plan/prepare_test.go +++ b/internal/controller/plan/prepare_test.go @@ -104,13 +104,14 @@ var _ = Describe("Cluster Controller", func() { Ctx: ctx, Log: logger, } - component := component.BuildComponent( + component, err := component.BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0]) + Expect(err).Should(Succeed()) task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) @@ -145,7 +146,7 @@ var _ = Describe("Cluster Controller", func() { Ctx: ctx, Log: logger, } - component := component.BuildComponent( + component, err := component.BuildComponent( reqCtx, *cluster, *clusterDef, @@ -153,6 +154,7 @@ var _ = Describe("Cluster Controller", func() { cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0], ) + Expect(err).Should(Succeed()) task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) @@ -197,13 +199,14 @@ var _ = Describe("Cluster Controller", func() { Ctx: ctx, Log: logger, } - component := component.BuildComponent( + component, err := component.BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0]) + Expect(err).Should(Succeed()) task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) @@ -246,13 +249,14 @@ var _ = Describe("Cluster Controller", func() { Ctx: ctx, Log: logger, } - component := component.BuildComponent( + component, err := component.BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0]) + Expect(err).Should(Succeed()) task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) @@ -304,13 +308,14 @@ var _ = Describe("Cluster Controller", func() { Ctx: ctx, Log: logger, } - component := component.BuildComponent( + component, err := component.BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0]) + Expect(err).Should(Succeed()) task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) @@ -355,13 +360,14 @@ var _ = Describe("Cluster Controller", func() { Ctx: ctx, Log: logger, } - component := component.BuildComponent( + component, err := component.BuildComponent( reqCtx, *cluster, *clusterDef, clusterDef.Spec.ComponentDefs[0], cluster.Spec.ComponentSpecs[0], &clusterVersion.Spec.ComponentVersions[0]) + Expect(err).Should(Succeed()) task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) diff --git a/internal/sqlchannel/client_test.go b/internal/sqlchannel/client_test.go index f3ff1ab71..e0dd0b0eb 100644 --- a/internal/sqlchannel/client_test.go +++ b/internal/sqlchannel/client_test.go @@ -36,7 +36,7 @@ import ( "google.golang.org/grpc" corev1 "k8s.io/api/core/v1" - intctrlutil "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/constant" testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) @@ -80,12 +80,12 @@ func TestNewClientWithPod(t *testing.T) { GetObject() pod.Spec.Containers[0].Ports = []corev1.ContainerPort{{ ContainerPort: int32(3501), - Name: intctrlutil.ProbeHTTPPortName, + Name: constant.ProbeHTTPPortName, Protocol: "TCP", }, { ContainerPort: int32(port), - Name: intctrlutil.ProbeGRPCPortName, + Name: constant.ProbeGRPCPortName, Protocol: "TCP", }, } @@ -324,12 +324,12 @@ func initSQLChannelClient(t *testing.T) (*testDaprServer, *OperationClient, func pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ { ContainerPort: int32(3501), - Name: intctrlutil.ProbeHTTPPortName, + Name: constant.ProbeHTTPPortName, Protocol: "TCP", }, { ContainerPort: int32(port), - Name: intctrlutil.ProbeGRPCPortName, + Name: constant.ProbeGRPCPortName, Protocol: "TCP", }, } From 64c45ce0e94959ffd2efe3603efcb7a68c3afbd4 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Wed, 10 May 2023 18:07:53 +0800 Subject: [PATCH 268/439] chore: update the csi-s3 pvc default capacity to 20Gi (#3176) --- internal/constant/const.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/constant/const.go b/internal/constant/const.go index 5885f71e2..c613ef7ba 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -225,4 +225,4 @@ const ( AccountPasswdForSecret = "password" ) -const DefaultBackupPvcInitCapacity = "100Gi" +const DefaultBackupPvcInitCapacity = "20Gi" From e2a1e1d320c8461e76ec8acca55975410a2e74db Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 10 May 2023 19:31:01 +0800 Subject: [PATCH 269/439] fix: reduction storage size of mysql cluster successfully but it didn't actually take effect (#3170) --- apis/apps/v1alpha1/cluster_webhook_test.go | 1 + apis/apps/v1alpha1/opsrequest_webhook.go | 27 ++++- apis/apps/v1alpha1/opsrequest_webhook_test.go | 101 ++++++++++-------- controllers/apps/operations/restart.go | 4 +- controllers/apps/operations/restart_test.go | 29 ++++- .../apps/operations/volume_expansion.go | 13 ++- internal/constant/const.go | 15 +-- 7 files changed, 123 insertions(+), 67 deletions(-) diff --git a/apis/apps/v1alpha1/cluster_webhook_test.go b/apis/apps/v1alpha1/cluster_webhook_test.go index ecfa2e020..c77fb6a0c 100644 --- a/apis/apps/v1alpha1/cluster_webhook_test.go +++ b/apis/apps/v1alpha1/cluster_webhook_test.go @@ -299,6 +299,7 @@ spec: volumeClaimTemplates: - name: data spec: + storageClassName: standard resources: requests: storage: 1Gi diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index dc103be49..1885bea5a 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/spf13/viper" "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -38,6 +39,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/webhook" + + "github.com/apecloud/kubeblocks/internal/constant" ) // log is for logging in this package. @@ -335,6 +338,7 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client. existInSpec bool storageClassName *string allowExpansion bool + requestStorage resource.Quantity } // component name -> vct name -> entity @@ -344,20 +348,32 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client. if _, ok := vols[comp.ComponentName]; !ok { vols[comp.ComponentName] = make(map[string]Entity) } - vols[comp.ComponentName][vct.Name] = Entity{false, nil, false} + vols[comp.ComponentName][vct.Name] = Entity{false, nil, false, vct.Storage} } } - + // TODO: remove it after supporting to recover volume expansion when it fails. + recoverVolumeExpansionFailure := viper.GetBool(constant.CfgRecoverVolumeExpansionFailure) // traverse the spec to update volumes for _, comp := range cluster.Spec.ComponentSpecs { if _, ok := vols[comp.Name]; !ok { continue // ignore not-exist component } for _, vct := range comp.VolumeClaimTemplates { - if _, ok := vols[comp.Name][vct.Name]; !ok { + e, ok := vols[comp.Name][vct.Name] + if !ok { continue } - vols[comp.Name][vct.Name] = Entity{true, vct.Spec.StorageClassName, false} + // TODO: + // compare the requested storage size with the pvc.status.capacity when KubeBlocks supports to manage the pvc by self + // and supports to recover volume expansion when it is fails. + previousValue := *vct.Spec.Resources.Requests.Storage() + if e.requestStorage.Cmp(previousValue) < 0 && !recoverVolumeExpansionFailure { + return fmt.Errorf(`requested storage size of volumeClaimTemplate "%s" can not less than previous values "%s" unless both Kubernetes and KubeBlocks support RECOVER_VOLUME_EXPANSION_FAILURE`, + vct.Name, previousValue.String()) + } + e.existInSpec = true + e.storageClassName = vct.Spec.StorageClassName + vols[comp.Name][vct.Name] = e } } @@ -375,7 +391,8 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client. if err != nil { continue // ignore the error and take it as not-supported } - vols[cname][vname] = Entity{e.existInSpec, e.storageClassName, allowExpansion} + e.allowExpansion = allowExpansion + vols[cname][vname] = e } } diff --git a/apis/apps/v1alpha1/opsrequest_webhook_test.go b/apis/apps/v1alpha1/opsrequest_webhook_test.go index a885f9862..ed384b82c 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook_test.go +++ b/apis/apps/v1alpha1/opsrequest_webhook_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" "github.com/sethvargo/go-password/password" + "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -34,10 +35,15 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/kubectl/pkg/util/storage" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/internal/constant" ) var _ = Describe("OpsRequest webhook", func() { - + const ( + componentName = "replicasets" + proxyComponentName = "proxy" + ) var ( randomStr = testCtx.GetRandomStr() clusterDefinitionName = "opswebhook-mysql-definition-" + randomStr @@ -45,8 +51,6 @@ var _ = Describe("OpsRequest webhook", func() { clusterVersionNameForUpgrade = "opswebhook-mysql-upgrade-" + randomStr clusterName = "opswebhook-mysql-" + randomStr opsRequestName = "opswebhook-mysql-ops-" + randomStr - replicaSetComponentName = "replicasets" - proxyComponentName = "proxy" ) cleanupObjects := func() { // Add any setup steps that needs to be executed before each test @@ -191,7 +195,7 @@ var _ = Describe("OpsRequest webhook", func() { }, }, { - ComponentOps: ComponentOps{ComponentName: replicaSetComponentName}, + ComponentOps: ComponentOps{ComponentName: componentName}, ResourceRequirements: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ "cpu": resource.MustParse("200m"), @@ -233,7 +237,9 @@ var _ = Describe("OpsRequest webhook", func() { } testVolumeExpansion := func(cluster *Cluster) { - volumeExpansionList := []VolumeExpansion{ + By("By testing volumeExpansion - target component not exist") + opsRequest := createTestOpsRequest(clusterName, opsRequestName, VolumeExpansionType) + opsRequest.Spec.VolumeExpansionList = []VolumeExpansion{ { ComponentOps: ComponentOps{ComponentName: "ve-not-exist"}, VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ @@ -243,21 +249,31 @@ var _ = Describe("OpsRequest webhook", func() { }, }, }, - { - ComponentOps: ComponentOps{ComponentName: replicaSetComponentName}, - VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ - { - Name: "log", - Storage: resource.MustParse("2Gi"), - }, - { - Name: "data", - Storage: resource.MustParse("2Gi"), - }, + } + Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring(notFoundComponentsString("ve-not-exist"))) + + By("By testing volumeExpansion - target volume not exist") + volumeExpansionList := []VolumeExpansion{{ + ComponentOps: ComponentOps{ComponentName: componentName}, + VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ + { + Name: "log", + Storage: resource.MustParse("2Gi"), + }, + { + Name: "data", + Storage: resource.MustParse("2Gi"), }, }, + }, + } + opsRequest.Spec.VolumeExpansionList = volumeExpansionList + Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring("volumeClaimTemplates: [log] not found in component: " + componentName)) + + By("By testing volumeExpansion - storageClass do not support volume expansion") + volumeExpansionList = []VolumeExpansion{ { - ComponentOps: ComponentOps{ComponentName: replicaSetComponentName}, + ComponentOps: ComponentOps{ComponentName: componentName}, VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ { Name: "data", @@ -265,44 +281,35 @@ var _ = Describe("OpsRequest webhook", func() { }, }, }, + } + opsRequest.Spec.VolumeExpansionList = volumeExpansionList + notSupportMsg := fmt.Sprintf("volumeClaimTemplate: [data] not support volume expansion in component: %s, you can view infos by command: kubectl get sc", componentName) + Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring(notSupportMsg)) + + By("testing volumeExpansion - storageClass supports volume expansion") + storageClassName := "standard" + storageClass := createStorageClass(testCtx.Ctx, storageClassName, "true", true) + Expect(storageClass).ShouldNot(BeNil()) + + By("testing volumeExpansion with smaller storage, expect an error occurs") + opsRequest.Spec.VolumeExpansionList = []VolumeExpansion{ { - ComponentOps: ComponentOps{ComponentName: replicaSetComponentName}, + ComponentOps: ComponentOps{ComponentName: "replicasets"}, VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ - { - Name: "log", - Storage: resource.MustParse("2Gi"), - }, { Name: "data", - Storage: resource.MustParse("2Gi"), + Storage: resource.MustParse("500Mi"), }, }, }, } + Expect(testCtx.CreateObj(ctx, opsRequest)).Should(HaveOccurred()) + Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring("can not less than previous values")) - By("By testing volumeExpansion - target component not exist") - opsRequest := createTestOpsRequest(clusterName, opsRequestName, VolumeExpansionType) - opsRequest.Spec.VolumeExpansionList = []VolumeExpansion{volumeExpansionList[0]} - Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring(notFoundComponentsString("ve-not-exist"))) + By("test volumeExpansion with smaller storage and RECOVER_VOLUME_EXPANSION_FAILURE=true, expect succeed") + viper.Set(constant.CfgRecoverVolumeExpansionFailure, true) + Expect(testCtx.CreateObj(ctx, opsRequest)).Should(Succeed()) - By("By testing volumeExpansion - target volume not exist") - opsRequest.Spec.VolumeExpansionList = []VolumeExpansion{volumeExpansionList[1]} - Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring("volumeClaimTemplates: [log] not found in component: replicasets")) - - By("By testing volumeExpansion - create a new storage class") - storageClassName := "sc-test-volume-expansion" - storageClass := createStorageClass(testCtx.Ctx, storageClassName, "false", true) - Expect(storageClass != nil).Should(BeTrue()) - - By("By testing volumeExpansion - has no pvc") - for _, compSpec := range cluster.Spec.ComponentSpecs { - for _, vct := range compSpec.VolumeClaimTemplates { - Expect(vct.Spec.StorageClassName == nil).Should(BeTrue()) - } - } - opsRequest.Spec.VolumeExpansionList = []VolumeExpansion{volumeExpansionList[2]} - notSupportMsg := "volumeClaimTemplate: [data] not support volume expansion in component: replicasets, you can view infos by command: kubectl get sc" - Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring(notSupportMsg)) // TODO By("testing volumeExpansion - pvc exists") // TODO @@ -324,7 +331,7 @@ var _ = Describe("OpsRequest webhook", func() { Replicas: 2, }, { - ComponentOps: ComponentOps{ComponentName: replicaSetComponentName}, + ComponentOps: ComponentOps{ComponentName: componentName}, Replicas: 2, }, } @@ -397,7 +404,7 @@ var _ = Describe("OpsRequest webhook", func() { By("By testing restart. if api is legal, it will create successfully") Eventually(func() bool { - opsRequest.Spec.RestartList[0].ComponentName = replicaSetComponentName + opsRequest.Spec.RestartList[0].ComponentName = componentName err := testCtx.CheckedCreateObj(ctx, opsRequest) return err == nil }).Should(BeTrue()) diff --git a/controllers/apps/operations/restart.go b/controllers/apps/operations/restart.go index 46dce02d2..aeedbe5d4 100644 --- a/controllers/apps/operations/restart.go +++ b/controllers/apps/operations/restart.go @@ -41,8 +41,8 @@ func init() { restartBehaviour := OpsBehaviour{ // if cluster is Abnormal or Failed, new opsRequest may can repair it. // TODO: we should add "force" flag for these opsRequest. - FromClusterPhases: appsv1alpha1.GetClusterTerminalPhases(), - ToClusterPhase: appsv1alpha1.SpecReconcilingClusterPhase, // appsv1alpha1.RebootingPhase, + FromClusterPhases: appsv1alpha1.GetClusterUpRunningPhases(), + ToClusterPhase: appsv1alpha1.SpecReconcilingClusterPhase, OpsHandler: restartOpsHandler{}, MaintainClusterPhaseBySelf: true, ProcessingReasonInClusterCondition: ProcessingReasonRestarting, diff --git a/controllers/apps/operations/restart_test.go b/controllers/apps/operations/restart_test.go index b59e885f1..627092ea2 100644 --- a/controllers/apps/operations/restart_test.go +++ b/controllers/apps/operations/restart_test.go @@ -22,7 +22,6 @@ package operations import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -62,11 +61,18 @@ var _ = Describe("Restart OpsRequest", func() { AfterEach(cleanEnv) Context("Test OpsRequest", func() { - It("Test restart OpsRequest", func() { + var ( + opsRes *OpsResource + cluster *appsv1alpha1.Cluster + reqCtx intctrlutil.RequestCtx + ) + BeforeEach(func() { By("init operations resources ") - reqCtx := intctrlutil.RequestCtx{Ctx: testCtx.Ctx} - opsRes, _, _ := initOperationsResources(clusterDefinitionName, clusterVersionName, clusterName) + opsRes, _, cluster = initOperationsResources(clusterDefinitionName, clusterVersionName, clusterName) + reqCtx = intctrlutil.RequestCtx{Ctx: testCtx.Ctx} + }) + It("Test restart OpsRequest", func() { By("create Restart opsRequest") opsRes.OpsRequest = createRestartOpsObj(clusterName, "restart-ops-"+randomStr) mockComponentIsOperating(opsRes.Cluster, appsv1alpha1.SpecReconcilingClusterCompPhase, consensusComp, statelessComp) @@ -90,6 +96,21 @@ var _ = Describe("Restart OpsRequest", func() { Expect(len(h.GetRealAffectedComponentMap(opsRes.OpsRequest))).Should(Equal(2)) }) + It("expect failed when cluster is stopped", func() { + By("mock cluster is stopped") + Expect(testapps.ChangeObjStatus(&testCtx, cluster, func() { + cluster.Status.Phase = appsv1alpha1.StoppedClusterPhase + })).Should(Succeed()) + By("create Restart opsRequest") + opsRes.OpsRequest = createRestartOpsObj(clusterName, "restart-ops-"+randomStr) + _, err := GetOpsManager().Do(reqCtx, k8sClient, opsRes) + Expect(err).Should(HaveOccurred()) + Expect(err.Error()).Should(ContainSubstring("opsRequest kind: Restart is forbidden when Cluster.status.Phase is Stopped")) + Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(opsRes.OpsRequest), + func(g Gomega, fetched *appsv1alpha1.OpsRequest) { + g.Expect(fetched.Status.Phase).To(Equal(appsv1alpha1.OpsFailedPhase)) + })).Should(Succeed()) + }) }) }) diff --git a/controllers/apps/operations/volume_expansion.go b/controllers/apps/operations/volume_expansion.go index 8f1d41cc4..12f83250c 100644 --- a/controllers/apps/operations/volume_expansion.go +++ b/controllers/apps/operations/volume_expansion.go @@ -324,15 +324,24 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti } objectKey := getPVCProgressObjectKey(v.Name) progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey, Group: vctName} + currStorageSize := v.Status.Capacity.Storage() // if the volume expand succeed - if v.Status.Capacity.Storage().Cmp(requestStorage) >= 0 { + if currStorageSize.Cmp(requestStorage) == 0 { succeedCount += 1 completedCount += 1 - message := fmt.Sprintf("Successfully expand volume: %s in Component: %s ", objectKey, componentName) + message := fmt.Sprintf("Successfully expand volume: %s in Component: %s", objectKey, componentName) progressDetail.SetStatusAndMessage(appsv1alpha1.SucceedProgressStatus, message) setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) continue } + if currStorageSize.Cmp(requestStorage) > 0 { + completedCount += 1 + message := fmt.Sprintf("requested storage size of %s can not less than current storage size: %s", + objectKey, currStorageSize.String()) + progressDetail.SetStatusAndMessage(appsv1alpha1.FailedProgressStatus, message) + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) + continue + } if ve.pvcIsResizing(&v) { message := fmt.Sprintf("Start expanding volume: %s in Component: %s ", objectKey, componentName) progressDetail.SetStatusAndMessage(appsv1alpha1.ProcessingProgressStatus, message) diff --git a/internal/constant/const.go b/internal/constant/const.go index c613ef7ba..b7a15dd01 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -26,13 +26,14 @@ const ( CfgKeyCtrlrMgrAffinity = "CM_AFFINITY" CfgKeyCtrlrMgrNodeSelector = "CM_NODE_SELECTOR" CfgKeyCtrlrMgrTolerations = "CM_TOLERATIONS" - CfgKeyCtrlrReconcileRetryDurationMS = "CM_RECON_RETRY_DURATION_MS" // accept time - CfgKeyBackupPVCName = "BACKUP_PVC_NAME" // the global pvc name which persistent volume claim to store the backup data - CfgKeyBackupPVCInitCapacity = "BACKUP_PVC_INIT_CAPACITY" // the init capacity of pvc for creating the pvc, e.g. 10Gi. - CfgKeyBackupPVCStorageClass = "BACKUP_PVC_STORAGE_CLASS" // the pvc storage class name. - CfgKeyBackupPVCCreatePolicy = "BACKUP_PVC_CREATE_POLICY" // the pvc create policy. support "IfNotPresent" or "Never" - CfgKeyBackupPVConfigmapName = "BACKUP_PV_CONFIGMAP_NAME" // the configmap name which contains a persistentVolume template. - CfgKeyBackupPVConfigmapNamespace = "BACKUP_PV_CONFIGMAP_NAMESPACE" // the configmap namespace which contains a persistentVolume template. + CfgKeyCtrlrReconcileRetryDurationMS = "CM_RECON_RETRY_DURATION_MS" // accept time + CfgKeyBackupPVCName = "BACKUP_PVC_NAME" // the global pvc name which persistent volume claim to store the backup data + CfgKeyBackupPVCInitCapacity = "BACKUP_PVC_INIT_CAPACITY" // the init capacity of pvc for creating the pvc, e.g. 10Gi. + CfgKeyBackupPVCStorageClass = "BACKUP_PVC_STORAGE_CLASS" // the pvc storage class name. + CfgKeyBackupPVCCreatePolicy = "BACKUP_PVC_CREATE_POLICY" // the pvc create policy. support "IfNotPresent" or "Never" + CfgKeyBackupPVConfigmapName = "BACKUP_PV_CONFIGMAP_NAME" // the configmap name which contains a persistentVolume template. + CfgKeyBackupPVConfigmapNamespace = "BACKUP_PV_CONFIGMAP_NAMESPACE" // the configmap namespace which contains a persistentVolume template. + CfgRecoverVolumeExpansionFailure = "RECOVER_VOLUME_EXPANSION_FAILURE" // refer to feature gates RecoverVolumeExpansionFailure of k8s. // addon config keys CfgKeyAddonJobTTL = "ADDON_JOB_TTL" From 5254c18ed0f008c9ac8adda7da089f63a9daedfb Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Thu, 11 May 2023 10:46:13 +0800 Subject: [PATCH 270/439] fix: the parameters of mysql are case-insensitive, the specific parameters will be solved temporarily (#2767) --- deploy/apecloud-mysql/config/mysql8-config-constraint.cue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/apecloud-mysql/config/mysql8-config-constraint.cue b/deploy/apecloud-mysql/config/mysql8-config-constraint.cue index a5744fae2..528e3df1b 100644 --- a/deploy/apecloud-mysql/config/mysql8-config-constraint.cue +++ b/deploy/apecloud-mysql/config/mysql8-config-constraint.cue @@ -69,7 +69,7 @@ binlog_expire_logs_seconds: int & >=0 & <=4294967295 | *2592000 // Row-based, Statement-based or Mixed replication - binlog_format?: string & "ROW" | "STATEMENT" | "MIXED" + binlog_format?: string & "ROW" | "STATEMENT" | "MIXED" | "row" | "statement" | "mixed" // Controls how many microseconds the binary log commit waits before synchronizing the binary log file to disk. binlog_group_commit_sync_delay?: int & >=0 & <=1000000 @@ -87,10 +87,10 @@ binlog_order_commits?: string & "0" | "1" | "OFF" | "ON" // Whether the server logs full or minimal rows with row-based replication. - binlog_row_image?: string & "FULL" | "MINIMAL" | "NOBLOB" + binlog_row_image?: string & "FULL" | "MINIMAL" | "NOBLOB" | "full" | "minimal" | "noblob" // Controls whether metadata is logged using FULL or MINIMAL format. FULL causes all metadata to be logged; MINIMAL means that only metadata actually required by slave is logged. Default: MINIMAL. - binlog_row_metadata?: string & "FULL" | "MINIMAL" + binlog_row_metadata?: string & "FULL" | "MINIMAL" | "full" | "minimal" // When enabled, it causes a MySQL 5.6.2 or later server to write informational log events such as row query log events into its binary log. binlog_rows_query_log_events?: string & "0" | "1" | "OFF" | "ON" @@ -187,7 +187,7 @@ default_password_lifetime: int & >=0 & <=65535 | *0 // The default storage engine (table type). - default_storage_engine?: string & "InnoDB" | "MRG_MYISAM" | "BLACKHOLE" | "CSV" | "MEMORY" | "FEDERATED" | "ARCHIVE" | "MyISAM" | "xengine" + default_storage_engine?: string & "InnoDB" | "MRG_MYISAM" | "BLACKHOLE" | "CSV" | "MEMORY" | "FEDERATED" | "ARCHIVE" | "MyISAM" | "xengine" | "XENGINE" | "INNODB" | "innodb" // Server current time zone default_time_zone?: string From f9b358113b417592bffce89fc05758c9ff69b101 Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Thu, 11 May 2023 11:59:18 +0800 Subject: [PATCH 271/439] fix: `kbcli playground init` log info repeat frequently in windows (#3073) Co-authored-by: 1aal <1aal@users.noreply.github.com> --- deploy/helm/values.yaml | 2 +- docs/user_docs/cli/kbcli_playground_init.md | 1 + internal/cli/cloudprovider/k3d.go | 2 +- internal/cli/cmd/kubeblocks/install.go | 6 +- internal/cli/cmd/kubeblocks/uninstall.go | 8 +- internal/cli/cmd/playground/base.go | 1 + internal/cli/cmd/playground/init.go | 2 + internal/cli/spinner/spinner.go | 33 +++-- internal/cli/spinner/windows_spinner.go | 143 ++++++++++++++++++++ internal/cli/util/util.go | 5 + 10 files changed, 183 insertions(+), 20 deletions(-) create mode 100644 internal/cli/spinner/windows_spinner.go diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 316db5304..49beb2d59 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -41,7 +41,7 @@ fullnameOverride: "" updateStrategy: rollingUpdate: maxSurge: 1 - maxUnavailable: 1 + maxUnavailable: 0 type: RollingUpdate ## Change `hostNetwork` to `true` when you want the KubeBlocks's pod to share its host's network namespace. diff --git a/docs/user_docs/cli/kbcli_playground_init.md b/docs/user_docs/cli/kbcli_playground_init.md index 88958cbc8..26b7597ff 100644 --- a/docs/user_docs/cli/kbcli_playground_init.md +++ b/docs/user_docs/cli/kbcli_playground_init.md @@ -35,6 +35,7 @@ kbcli playground init [flags] --cluster-version string Cluster definition -h, --help help for init --region string The region to create kubernetes cluster + --timeout duration Time to wait for initing playground, such as --timeout=10m (default 5m0s) --version string KubeBlocks version ``` diff --git a/internal/cli/cloudprovider/k3d.go b/internal/cli/cloudprovider/k3d.go index 0a1f9a57d..c1c146156 100644 --- a/internal/cli/cloudprovider/k3d.go +++ b/internal/cli/cloudprovider/k3d.go @@ -368,7 +368,7 @@ func setUpK3d(ctx context.Context, cluster *config.ClusterConfig) error { for _, c := range l { if c.Name == cluster.Name { if c, err := k3dClient.ClusterGet(ctx, runtimes.SelectedRuntime, c); err == nil { - fmt.Printf(" Detected an existing cluster: %s", c.Name) + fmt.Printf("Detected an existing cluster: %s\n", c.Name) return nil } break diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index 5d14e1585..0be74c520 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -174,6 +174,9 @@ func (o *InstallOptions) Install() error { return err } + // Todo: KubeBlocks maybe already install but it's status could be Failed. + // For example: 'kbcli playground init' in windows will fail and try 'kbcli playground init' again immediately, + // kbcli will output SUCCESSFULLY, however the addon csi is failed and KubeBlocks do not install SUCCESSFULLY if v.KubeBlocks != "" { printer.Warning(o.Out, "KubeBlocks %s already exists, repeated installation is not supported.\n\n", v.KubeBlocks) fmt.Fprintln(o.Out, "If you want to upgrade it, please use \"kbcli kubeblocks upgrade\".") @@ -313,13 +316,12 @@ func (o *InstallOptions) waitAddonsEnabled() error { var ( allEnabled bool err error - spinnerDone = func(s *spinner.Spinner) { + spinnerDone = func(s spinner.Interface) { s.SetFinalMsg(allMsg) s.Done("") fmt.Fprintln(o.Out) } ) - // wait all addons to be enabled, or timeout if err = wait.PollImmediate(5*time.Second, o.Timeout, func() (bool, error) { allEnabled, err = checkAddons() diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index dae491bf5..6ebdd8e77 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -131,7 +131,7 @@ func (o *UninstallOptions) PreCheck() error { } func (o *UninstallOptions) Uninstall() error { - printSpinner := func(s *spinner.Spinner, err error) { + printSpinner := func(s spinner.Interface, err error) { if err == nil || apierrors.IsNotFound(err) || strings.Contains(err.Error(), "release: not found") { s.Success() @@ -140,7 +140,7 @@ func (o *UninstallOptions) Uninstall() error { s.Fail() fmt.Fprintf(o.Out, " %s\n", err.Error()) } - newSpinner := func(msg string) *spinner.Spinner { + newSpinner := func(msg string) spinner.Interface { return spinner.New(o.Out, spinner.WithMessage(fmt.Sprintf("%-50s", msg))) } @@ -282,7 +282,7 @@ func (o *UninstallOptions) uninstallAddons() error { } ) - var s *spinner.Spinner + var s spinner.Interface if !o.Wait { s = spinner.New(o.Out, spinner.WithMessage(fmt.Sprintf("%-50s", "Uninstall KubeBlocks addons"))) } else { @@ -300,7 +300,7 @@ func (o *UninstallOptions) uninstallAddons() error { return nil } - spinnerDone := func(s *spinner.Spinner, msg string) { + spinnerDone := func(s spinner.Interface, msg string) { s.SetFinalMsg(msg) s.Done("") fmt.Fprintln(o.Out) diff --git a/internal/cli/cmd/playground/base.go b/internal/cli/cmd/playground/base.go index 3742e7637..89e3873f9 100644 --- a/internal/cli/cmd/playground/base.go +++ b/internal/cli/cmd/playground/base.go @@ -30,6 +30,7 @@ import ( type baseOptions struct { startTime time.Time + Timeout time.Duration // prevCluster is the previous cluster info prevCluster *cp.K8sClusterInfo // kubeConfigPath is the tmp kubeconfig path that will be used when int and destroy diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index f40f737d6..3a71aaedd 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -106,6 +106,7 @@ func newInitCmd(streams genericclioptions.IOStreams) *cobra.Command { cmd.Flags().StringVar(&o.kbVersion, "version", version.DefaultKubeBlocksVersion, "KubeBlocks version") cmd.Flags().StringVar(&o.cloudProvider, "cloud-provider", defaultCloudProvider, fmt.Sprintf("Cloud provider type, one of %v", supportedCloudProviders)) cmd.Flags().StringVar(&o.region, "region", "", "The region to create kubernetes cluster") + cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for initing playground, such as --timeout=10m") util.CheckErr(cmd.RegisterFlagCompletionFunc( "cloud-provider", @@ -413,6 +414,7 @@ func (o *initOptions) installKubeBlocks(k8sClusterName string) error { Client: client, Dynamic: dynamic, Wait: true, + Timeout: o.Timeout, }, Version: o.kbVersion, Monitor: true, diff --git a/internal/cli/spinner/spinner.go b/internal/cli/spinner/spinner.go index 7a5531c35..c84699ec3 100644 --- a/internal/cli/spinner/spinner.go +++ b/internal/cli/spinner/spinner.go @@ -31,6 +31,7 @@ import ( "github.com/briandowns/spinner" "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/util" ) type Spinner struct { @@ -39,26 +40,30 @@ type Spinner struct { cancel chan struct{} } -type Option func(*Spinner) - -func WithMessage(msg string) Option { - return func(s *Spinner) { - s.UpdateSpinnerMessage(msg) - } +type Interface interface { + Start() + Done(status string) + Success() + Fail() + SetMessage(msg string) + SetFinalMsg(msg string) + updateSpinnerMessage(msg string) } -func WithDelay(delay time.Duration) Option { - return func(s *Spinner) { - s.delay = delay +type Option func(Interface) + +func WithMessage(msg string) Option { + return func(s Interface) { + s.updateSpinnerMessage(msg) } } -func (s *Spinner) UpdateSpinnerMessage(msg string) { +func (s *Spinner) updateSpinnerMessage(msg string) { s.s.Suffix = fmt.Sprintf(" %s", msg) } func (s *Spinner) SetMessage(msg string) { - s.UpdateSpinnerMessage(msg) + s.updateSpinnerMessage(msg) if !s.s.Active() { s.Start() } @@ -121,7 +126,11 @@ func (s *Spinner) Fail() { s.Done(printer.BoldRed("FAIL")) } -func New(w io.Writer, opts ...Option) *Spinner { +func New(w io.Writer, opts ...Option) Interface { + if util.IsWindows() { + return NewWindowsSpinner(w, opts...) + } + res := &Spinner{} res.s = spinner.New(spinner.CharSets[11], 100*time.Millisecond, diff --git a/internal/cli/spinner/windows_spinner.go b/internal/cli/spinner/windows_spinner.go new file mode 100644 index 000000000..f1c89b24f --- /dev/null +++ b/internal/cli/spinner/windows_spinner.go @@ -0,0 +1,143 @@ +package spinner + +import ( + "fmt" + "io" + "os" + "os/signal" + "strings" + "sync" + "syscall" + + "time" + + "github.com/apecloud/kubeblocks/internal/cli/printer" +) + +var char = []string{"⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"} + +type WindowsSpinner struct { // no thread/goroutine safe + msg string + lastOutplain string + FinalMSG string + active bool + chars []string + cancel chan struct{} + Writer io.Writer + delay time.Duration + mu *sync.RWMutex +} + +func NewWindowsSpinner(w io.Writer, opts ...Option) *WindowsSpinner { + res := &WindowsSpinner{ + chars: char, + active: false, + cancel: make(chan struct{}, 1), + Writer: w, + mu: &sync.RWMutex{}, + delay: 100 * time.Millisecond, + } + for _, opt := range opts { + opt(res) + } + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + res.Done("") + os.Exit(0) + }() + res.Start() + return res +} + +func (s *WindowsSpinner) updateSpinnerMessage(msg string) { + s.msg = fmt.Sprintf(" %s", msg) +} + +func (s *WindowsSpinner) Done(status string) { + if status != "" { + s.FinalMSG = fmt.Sprintf("%s %s\n", strings.TrimPrefix(s.msg, " "), status) + } + s.stop() +} + +func (s *WindowsSpinner) Success() { + if len(s.msg) == 0 { + return + } + s.Done(printer.BoldGreen("OK")) + +} + +func (s *WindowsSpinner) Fail() { + if len(s.msg) == 0 { + return + } + s.Done(printer.BoldRed("FAIL")) +} + +func (s *WindowsSpinner) Start() { + s.active = true + + go func() { + for { + for i := 0; i < len(s.chars); i++ { + select { + case <-s.cancel: + return + default: + s.mu.Lock() + if !s.active { + defer s.mu.Unlock() + return + } + outPlain := fmt.Sprintf("\r%s%s", s.chars[i], s.msg) + s.erase() + s.lastOutplain = outPlain + fmt.Print(outPlain) + s.mu.Unlock() + // fmt.Fprint(s.Writer, outPlain) + time.Sleep(s.delay) + } + } + } + }() +} + +func (s *WindowsSpinner) SetMessage(msg string) { + s.mu.Lock() + defer s.mu.Unlock() + s.msg = msg +} + +func (s *WindowsSpinner) SetFinalMsg(msg string) { + s.FinalMSG = msg +} + +// remove lastOutplain +func (s *WindowsSpinner) erase() { + split := strings.Split(s.lastOutplain, "\n") + for i := 0; i < len(split); i++ { + if i > 0 { + fmt.Print("\033[A") + } + fmt.Print("\r\033[K") + } +} + +// stop stops the indicator. +func (s *WindowsSpinner) stop() { + s.mu.Lock() + defer s.mu.Unlock() + if s.active { + s.active = false + if s.FinalMSG != "" { + s.erase() + fmt.Print(s.FinalMSG) + } + s.cancel <- struct{}{} + close(s.cancel) + } +} diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index eef0ea1a3..71afa05c9 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -759,3 +759,8 @@ func CreateResourceIfAbsent( } return nil } + +// IsWindows return true if the kbcli runtime situation is windows +func IsWindows() bool { + return runtime.GOOS == types.GoosWindows +} From 3a871e2ad294c1f551e0c97d02dee5fdbcf7b375 Mon Sep 17 00:00:00 2001 From: runsun Date: Thu, 11 May 2023 14:10:19 +0800 Subject: [PATCH 272/439] fix: slack title link is not valid (#3181) --- internal/cli/cmd/alert/add_receiver.go | 2 +- internal/cli/cmd/alert/types.go | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/cli/cmd/alert/add_receiver.go b/internal/cli/cmd/alert/add_receiver.go index 7815a940f..0cdd1d1a5 100644 --- a/internal/cli/cmd/alert/add_receiver.go +++ b/internal/cli/cmd/alert/add_receiver.go @@ -407,7 +407,7 @@ func buildSlackConfigs(slacks []string) ([]*slackConfig, error) { if len(m) == 0 { return nil, fmt.Errorf("invalid slack: %s, slack config should be in the format of api_url=my-api-url,channel=my-channel,username=my-username", slackStr) } - s := slackConfig{} + s := slackConfig{TitleLink: ""} for k, v := range m { // check slackConfig keys switch slackKey(k) { diff --git a/internal/cli/cmd/alert/types.go b/internal/cli/cmd/alert/types.go index 540f91121..5dba349f9 100644 --- a/internal/cli/cmd/alert/types.go +++ b/internal/cli/cmd/alert/types.go @@ -102,9 +102,10 @@ type slackKey string // slackConfig keys const ( - slackAPIURL slackKey = "api_url" - slackChannel slackKey = "channel" - slackUsername slackKey = "username" + slackAPIURL slackKey = "api_url" + slackChannel slackKey = "channel" + slackUsername slackKey = "username" + slackTitleLink slackKey = "title_link" ) // emailConfig is the email config of receiver @@ -119,10 +120,13 @@ type webhookConfig struct { MaxAlerts int `json:"max_alerts,omitempty"` } +// slackConfig is the alertmanager slack config of receiver +// ref: https://prometheus.io/docs/alerting/latest/configuration/#slack_config type slackConfig struct { - APIURL string `json:"api_url,omitempty"` - Channel string `json:"channel,omitempty"` - Username string `json:"username,omitempty"` + APIURL string `json:"api_url,omitempty"` + Channel string `json:"channel,omitempty"` + Username string `json:"username,omitempty"` + TitleLink string `json:"title_link"` } // receiver is the receiver of alert From 46ecd97da0bb29818869bc42b9cea409dad77ec6 Mon Sep 17 00:00:00 2001 From: huangzhangshu Date: Thu, 11 May 2023 14:14:32 +0800 Subject: [PATCH 273/439] chore: fix token --- .github/workflows/cicd-pull-request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index 2c2cabb7d..70b983bc7 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -9,6 +9,7 @@ env: GO_CACHE: "go-cache" GO_CACHE_DIR: "/root/.cache" GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }} + GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} jobs: trigger-mode: From c504f48c748e12efae71051b0154933a2c5f9e8c Mon Sep 17 00:00:00 2001 From: free6om Date: Thu, 11 May 2023 15:14:11 +0800 Subject: [PATCH 274/439] fix: cluster observed generation wrong patched (#3184) --- controllers/apps/cluster_controller_test.go | 7 ++++++- internal/controller/lifecycle/cluster_plan_builder.go | 7 ++++--- internal/testutil/apps/constant.go | 6 +++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 0c28460be..da18e9ed9 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -153,7 +153,7 @@ var _ = Describe("Cluster Controller", func() { AddComponentDef(testapps.StatelessNginxComponent, statelessCompDefName). Create(&testCtx).GetObject() - if len(noCreateAssociateCV) > 0 && !noCreateAssociateCV[0] { + if len(noCreateAssociateCV) > 0 && noCreateAssociateCV[0] { return } By("Create a clusterVersion obj") @@ -1152,6 +1152,7 @@ var _ = Describe("Cluster Controller", func() { // Test cases // Scenarios + // TODO: add case: empty image in cd, should report applyResourceFailed condition Context("when creating cluster without clusterversion", func() { BeforeEach(func() { createAllWorkloadTypesClusterDef(true) @@ -1592,6 +1593,7 @@ var _ = Describe("Cluster Controller", func() { By("test when clusterDefinition not found") Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { + g.Expect(tmpCluster.Status.ObservedGeneration).Should(BeZero()) condition := meta.FindStatusCondition(tmpCluster.Status.Conditions, appsv1alpha1.ConditionTypeProvisioningStarted) g.Expect(condition).ShouldNot(BeNil()) g.Expect(condition.Reason).Should(BeEquivalentTo(lifecycle.ReasonPreCheckFailed)) @@ -1630,6 +1632,7 @@ var _ = Describe("Cluster Controller", func() { Eventually(func(g Gomega) { updateClusterAnnotation(cluster) g.Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { + g.Expect(cluster.Status.ObservedGeneration).Should(BeZero()) condition := meta.FindStatusCondition(cluster.Status.Conditions, appsv1alpha1.ConditionTypeProvisioningStarted) g.Expect(condition).ShouldNot(BeNil()) g.Expect(condition.Reason).Should(BeEquivalentTo(lifecycle.ReasonPreCheckFailed)) @@ -1649,6 +1652,7 @@ var _ = Describe("Cluster Controller", func() { updateClusterAnnotation(cluster) By("test preCheckFailed") Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { + g.Expect(cluster.Status.ObservedGeneration).Should(BeZero()) condition := meta.FindStatusCondition(cluster.Status.Conditions, appsv1alpha1.ConditionTypeProvisioningStarted) g.Expect(condition != nil && condition.Reason == lifecycle.ReasonPreCheckFailed).Should(BeTrue()) })).Should(Succeed()) @@ -1670,6 +1674,7 @@ var _ = Describe("Cluster Controller", func() { Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(cluster), func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { + g.Expect(tmpCluster.Status.ObservedGeneration).ShouldNot(BeEquivalentTo(tmpCluster.Generation)) condition := meta.FindStatusCondition(tmpCluster.Status.Conditions, appsv1alpha1.ConditionTypeApplyResources) g.Expect(condition != nil && condition.Reason == lifecycle.ReasonApplyResourcesFailed).Should(BeTrue()) })).Should(Succeed()) diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 4ae4f7b63..360c11531 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -180,10 +180,11 @@ func (p *clusterPlan) Execute() error { } func (p *clusterPlan) handlePlanExecutionError(err error) error { + clusterCopy := p.transCtx.OrigCluster.DeepCopy() condition := newFailedApplyResourcesCondition(err) - meta.SetStatusCondition(&p.transCtx.Cluster.Status.Conditions, condition) - sendWaringEventWithError(p.transCtx.GetRecorder(), p.transCtx.Cluster, ReasonApplyResourcesFailed, err) - return p.cli.Status().Patch(p.transCtx.Context, p.transCtx.Cluster, client.MergeFrom(p.transCtx.OrigCluster.DeepCopy())) + meta.SetStatusCondition(&clusterCopy.Status.Conditions, condition) + sendWaringEventWithError(p.transCtx.GetRecorder(), clusterCopy, ReasonApplyResourcesFailed, err) + return p.cli.Status().Patch(p.transCtx.Context, clusterCopy, client.MergeFrom(p.transCtx.OrigCluster)) } // Do the real works diff --git a/internal/testutil/apps/constant.go b/internal/testutil/apps/constant.go index bc9f0ad8d..31656784e 100644 --- a/internal/testutil/apps/constant.go +++ b/internal/testutil/apps/constant.go @@ -69,7 +69,8 @@ var ( CharacterType: "stateless", PodSpec: &corev1.PodSpec{ Containers: []corev1.Container{{ - Name: DefaultNginxContainerName, + Name: DefaultNginxContainerName, + Image: NginxImage, }}, }, Service: &appsv1alpha1.ServiceSpec{ @@ -118,6 +119,7 @@ var ( defaultMySQLContainer = corev1.Container{ Name: DefaultMySQLContainerName, + Image: ApeCloudMySQLImage, ImagePullPolicy: corev1.PullIfNotPresent, Ports: []corev1.ContainerPort{ { @@ -236,6 +238,7 @@ var ( defaultRedisInitContainer = corev1.Container{ Name: DefaultRedisInitContainerName, + Image: DefaultRedisImageName, ImagePullPolicy: corev1.PullIfNotPresent, VolumeMounts: defaultReplicationRedisVolumeMounts, Command: []string{"/scripts/init.sh"}, @@ -243,6 +246,7 @@ var ( defaultRedisContainer = corev1.Container{ Name: DefaultRedisContainerName, + Image: DefaultRedisImageName, ImagePullPolicy: corev1.PullIfNotPresent, Ports: []corev1.ContainerPort{ { From 0cb9cbca0eb42f9d66e556a97266fc0989f0cdb2 Mon Sep 17 00:00:00 2001 From: chantu Date: Thu, 11 May 2023 15:34:02 +0800 Subject: [PATCH 275/439] fix: host info inconsistence (#3199) --- deploy/apecloud-mysql/templates/scripts.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index 9d96ff13b..bf0cb1837 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -57,12 +57,12 @@ data: if [ $i -ne 0 ]; then cluster_info="$cluster_info;"; fi; - host=$(eval echo \$KB_MYSQL_"$i"_HOSTNAME) + tmp_host=$(eval echo \$KB_MYSQL_"$i"_HOSTNAME) # setup pod weight, prefer pod 0 to be leader if [ $i -eq 0 ]; then - cluster_info="$cluster_info$host:13306#9N"; + cluster_info="$cluster_info$tmp_host:13306#9N"; else - cluster_info="$cluster_info$host:13306#1N"; + cluster_info="$cluster_info$tmp_host:13306#1N"; fi done; cluster_info="$cluster_info@$(($idx+1))"; From 10ea2aef6833b61b10cc557eca3172df2269be13 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Thu, 11 May 2023 16:11:37 +0800 Subject: [PATCH 276/439] chore: helm template add chart lables (#3196) --- deploy/chatgpt-retrieval-plugin/templates/clusterrole.yaml | 2 ++ .../chatgpt-retrieval-plugin/templates/clusterrolebinding.yaml | 2 ++ deploy/helm/templates/class/componentclassconstraint.yaml | 2 ++ deploy/mongodb/templates/backuptool.yaml | 1 + deploy/nyancat/templates/clusterrole.yaml | 2 ++ deploy/nyancat/templates/clusterrolebinding.yaml | 2 ++ deploy/redis/templates/scripts.yaml | 2 ++ 7 files changed, 13 insertions(+) diff --git a/deploy/chatgpt-retrieval-plugin/templates/clusterrole.yaml b/deploy/chatgpt-retrieval-plugin/templates/clusterrole.yaml index 83b941682..d51e76abf 100644 --- a/deploy/chatgpt-retrieval-plugin/templates/clusterrole.yaml +++ b/deploy/chatgpt-retrieval-plugin/templates/clusterrole.yaml @@ -2,6 +2,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ include "gptplugin.fullname" . }} + labels: + {{- include "gptplugin.labels" . | nindent 4 }} rules: - apiGroups: [""] resources: ["services", "pods", "secrets"] diff --git a/deploy/chatgpt-retrieval-plugin/templates/clusterrolebinding.yaml b/deploy/chatgpt-retrieval-plugin/templates/clusterrolebinding.yaml index e0eca0c92..ea56af650 100644 --- a/deploy/chatgpt-retrieval-plugin/templates/clusterrolebinding.yaml +++ b/deploy/chatgpt-retrieval-plugin/templates/clusterrolebinding.yaml @@ -2,6 +2,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ include "gptplugin.fullname" . }} + labels: + {{- include "gptplugin.labels" . | nindent 4 }} subjects: - kind: ServiceAccount name: {{ include "gptplugin.serviceAccountName" . }} diff --git a/deploy/helm/templates/class/componentclassconstraint.yaml b/deploy/helm/templates/class/componentclassconstraint.yaml index ddf6c38fc..aa1aad675 100644 --- a/deploy/helm/templates/class/componentclassconstraint.yaml +++ b/deploy/helm/templates/class/componentclassconstraint.yaml @@ -4,6 +4,7 @@ metadata: name: kb-resource-constraint-general labels: resourceconstraint.kubeblocks.io/provider: kubeblocks + {{- include "kubeblocks.labels" . | nindent 4 }} spec: constraints: - cpu: @@ -30,6 +31,7 @@ metadata: name: kb-resource-constraint-memory-optimized labels: resourceconstraint.kubeblocks.io/provider: kubeblocks + {{- include "kubeblocks.labels" . | nindent 4 }} spec: constraints: - cpu: diff --git a/deploy/mongodb/templates/backuptool.yaml b/deploy/mongodb/templates/backuptool.yaml index 303473711..7e26d25f5 100644 --- a/deploy/mongodb/templates/backuptool.yaml +++ b/deploy/mongodb/templates/backuptool.yaml @@ -4,6 +4,7 @@ metadata: name: mongodb-physical-backup-tool labels: clusterdefinition.kubeblocks.io/name: mongodb + {{- include "mongodb.labels" . | nindent 4 }} spec: image: mongo:5.0.14 deployKind: job diff --git a/deploy/nyancat/templates/clusterrole.yaml b/deploy/nyancat/templates/clusterrole.yaml index f1eb5de40..78c8f0495 100644 --- a/deploy/nyancat/templates/clusterrole.yaml +++ b/deploy/nyancat/templates/clusterrole.yaml @@ -2,6 +2,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ include "nyancat.fullname" . }} + labels: + {{- include "nyancat.labels" . | nindent 4 }} rules: - apiGroups: [""] resources: ["services", "pods", "secrets"] diff --git a/deploy/nyancat/templates/clusterrolebinding.yaml b/deploy/nyancat/templates/clusterrolebinding.yaml index 02e74c5a0..bf5b49d8f 100644 --- a/deploy/nyancat/templates/clusterrolebinding.yaml +++ b/deploy/nyancat/templates/clusterrolebinding.yaml @@ -2,6 +2,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: {{ include "nyancat.fullname" . }} + labels: + {{- include "nyancat.labels" . | nindent 4 }} subjects: - kind: ServiceAccount name: {{ include "nyancat.serviceAccountName" . }} diff --git a/deploy/redis/templates/scripts.yaml b/deploy/redis/templates/scripts.yaml index 3b3e8c794..bcea1b030 100644 --- a/deploy/redis/templates/scripts.yaml +++ b/deploy/redis/templates/scripts.yaml @@ -2,6 +2,8 @@ apiVersion: v1 kind: ConfigMap metadata: name: redis-scripts + labels: + {{- include "redis.labels" . | nindent 4 }} data: setup.sh: | #!/bin/sh From bb10506f2c5baf20048caa90923e26e6269cb754 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Thu, 11 May 2023 16:52:05 +0800 Subject: [PATCH 277/439] fix: udpate client image in cluster version (#3180) --- deploy/apecloud-mysql/templates/clusterversion.yaml | 1 + deploy/mongodb/templates/clusterversion.yaml | 1 + deploy/redis/templates/clusterdefinition.yaml | 2 +- deploy/redis/templates/clusterversion.yaml | 1 + internal/cli/cmd/accounts/base.go | 2 +- internal/cli/cmd/playground/destroy.go | 2 +- internal/cli/cmd/playground/init.go | 2 +- 7 files changed, 7 insertions(+), 4 deletions(-) diff --git a/deploy/apecloud-mysql/templates/clusterversion.yaml b/deploy/apecloud-mysql/templates/clusterversion.yaml index 22c92bc33..7ac7ba1e6 100644 --- a/deploy/apecloud-mysql/templates/clusterversion.yaml +++ b/deploy/apecloud-mysql/templates/clusterversion.yaml @@ -13,3 +13,4 @@ spec: - name: mysql image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + clientImage: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} \ No newline at end of file diff --git a/deploy/mongodb/templates/clusterversion.yaml b/deploy/mongodb/templates/clusterversion.yaml index 9eb4ef83a..1eb5cfc44 100644 --- a/deploy/mongodb/templates/clusterversion.yaml +++ b/deploy/mongodb/templates/clusterversion.yaml @@ -13,3 +13,4 @@ spec: - name: mongodb image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + clientImage: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} \ No newline at end of file diff --git a/deploy/redis/templates/clusterdefinition.yaml b/deploy/redis/templates/clusterdefinition.yaml index d95c88971..01adeb81b 100644 --- a/deploy/redis/templates/clusterdefinition.yaml +++ b/deploy/redis/templates/clusterdefinition.yaml @@ -133,7 +133,7 @@ spec: # to pass $(KB_ACCOUNT_STATEMENT) to redis-cli without causing parsing error. # Instead, using a shell script to wrap redis-cli and pass $(KB_ACCOUNT_STATEMENT) to it will do. cmdExecutorConfig: - image: docker.io/redis:7.0.5 + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ default .Values.image.tag }} command: - sh - -c diff --git a/deploy/redis/templates/clusterversion.yaml b/deploy/redis/templates/clusterversion.yaml index 4ee5aba1d..0adb370d2 100644 --- a/deploy/redis/templates/clusterversion.yaml +++ b/deploy/redis/templates/clusterversion.yaml @@ -13,6 +13,7 @@ spec: - name: redis image: {{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + clientImage: {{ .Values.image.repository }}:{{ .Values.image.tag }} - componentDefRef: redis-sentinel versionsContext: initContainers: diff --git a/internal/cli/cmd/accounts/base.go b/internal/cli/cmd/accounts/base.go index 6fce49ad0..291d40ca5 100644 --- a/internal/cli/cmd/accounts/base.go +++ b/internal/cli/cmd/accounts/base.go @@ -167,7 +167,7 @@ func (o *AccountBaseOptions) Run(cmd *cobra.Command, f cmdutil.Factory, streams case sqlchannel.CreateUserOp: o.printGeneralInfo(response) if response.Event == sqlchannel.RespEveSucc { - printer.Alert(o.Out, "Please do REMEMBER the password for the new user! Once forgotten, it cannot be retrieved!") + printer.Alert(o.Out, "Please do REMEMBER the password for the new user! Once forgotten, it cannot be retrieved!\n") } err = nil case sqlchannel.DescribeUserOp: diff --git a/internal/cli/cmd/playground/destroy.go b/internal/cli/cmd/playground/destroy.go index 8cb5ccea9..50c725676 100644 --- a/internal/cli/cmd/playground/destroy.go +++ b/internal/cli/cmd/playground/destroy.go @@ -134,7 +134,7 @@ func (o *destroyOptions) destroyLocal() error { func (o *destroyOptions) destroyCloud() error { var err error - printer.Warning(o.Out, `This action will destroy the kubernetes cluster, there may be residual resources, + printer.Warning(o.Out, `This action will destroy the kubernetes cluster, there may be residual resources, please confirm and manually clean up related resources after this action. `) diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index 3a71aaedd..4467a01ec 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -52,7 +52,7 @@ import ( var ( initExample = templates.Examples(` - # create a k3d cluster on local host and install KubeBlocks + # create a k3d cluster on local host and install KubeBlocks kbcli playground init # create an AWS EKS cluster and install KubeBlocks, the region is required From 3b7651cac5deb0033a93cc05b2f4a8e4f17702bd Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Thu, 11 May 2023 17:00:06 +0800 Subject: [PATCH 278/439] chore: tidy up helm values (#3189) --- deploy/helm/values.yaml | 44 +++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 49beb2d59..30e43853e 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -41,7 +41,7 @@ fullnameOverride: "" updateStrategy: rollingUpdate: maxSurge: 1 - maxUnavailable: 0 + maxUnavailable: 40% type: RollingUpdate ## Change `hostNetwork` to `true` when you want the KubeBlocks's pod to share its host's network namespace. @@ -60,6 +60,19 @@ hostNetwork: false ## dnsPolicy: ClusterFirst +## Configure podDisruptionBudget spec settings +## +## @param podDisruptionBudget.minAvailable +## @param podDisruptionBudget.maxUnavailable +podDisruptionBudget: + # Configures the minimum available pods for Kubeblocks disruptions. + # Cannot be used if `maxUnavailable` is set. + minAvailable: 1 + # Configures the maximum unavailable pods for Kubeblocks disruptions. + # Cannot be used if `minAvailable` is set. + maxUnavailable: + + ## Logger settings ## ## @param loggerSettings.developmentMode @@ -153,15 +166,16 @@ serviceMonitor: # Only used if `service.type` is `NodePort`. nodePort: -## @param topologySpreadConstraints +## KubeBlocks pods deployment topologySpreadConstraints settings ## +## @param topologySpreadConstraints topologySpreadConstraints: [] ## Resource settings ## -## @param topologySpreadConstraints.limits -## @param topologySpreadConstraints.requests +## @param resources.limits +## @param resources.requests resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -242,14 +256,6 @@ dataPlane: values: - "true" -## PDB settings -## -## @param podDisruptionBudget.minAvailable -## @param podDisruptionBudget.maxUnavailable -podDisruptionBudget: - minAvailable: 1 - maxUnavailable: - ## AdmissionWebhooks settings ## ## @param admissionWebhooks.enabled @@ -262,14 +268,14 @@ admissionWebhooks: ## Data protection settings ## +## @param dataProtection.enableVolumeSnapshot - set this to true if cluster does have snapshot.storage.k8s.io API installed +## @param dataProtection.backupPVCName - set the default pvc to store the file for backup +## @param dataProtection.backupPVCInitCapacity - set the default pvc initCapacity if the pvc need to be created by backup controller +## @param dataProtection.backupPVCStorageClassName - set the default pvc storageClassName if the pvc need to be created by backup controller +## @param dataProtection.backupPVCCreatePolicy - set the default create policy of the pvc, optional values: IfNotPresent, Never +## @param dataProtection.backupPVConfigMapName - set the default configmap name which contains key "persistentVolume" and value of the persistentVolume struct. +## @param dataProtection.backupPVConfigMapNamespace - set the default configmap namespace of pv template. dataProtection: - ## @param dataProtection.enableVolumeSnapshot - set this to true if cluster does have snapshot.storage.k8s.io API installed - ## @param dataProtection.backupPVCName - set the default pvc to store the file for backup - ## @param dataProtection.backupPVCInitCapacity - set the default pvc initCapacity if the pvc need to be created by backup controller - ## @param dataProtection.backupPVCStorageClassName - set the default pvc storageClassName if the pvc need to be created by backup controller - ## @param dataProtection.backupPVCCreatePolicy - set the default create policy of the pvc, optional values: IfNotPresent, Never - ## @param dataProtection.backupPVConfigMapName - set the default configmap name which contains key "persistentVolume" and value of the persistentVolume struct. - ## @param dataProtection.backupPVConfigMapNamespace - set the default configmap namespace of pv template. enableVolumeSnapshot: false backupPVCName: "" backupPVCInitCapacity: "" From 1f95da694b220325ad6273d5fd4e093100f48a1e Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Thu, 11 May 2023 17:57:41 +0800 Subject: [PATCH 279/439] chore: require pr ci check (#3203) --- .github/workflows/cicd-pull-request.yml | 6 +++--- .github/workflows/cicd-push.yml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index 70b983bc7..dcd55b7f5 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -71,7 +71,7 @@ jobs: check-image: name: check image needs: trigger-mode - if: needs.trigger-mode.outputs.trigger-mode == '[docker]' + if: contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 with: MAKE_OPS_PRE: "generate" @@ -84,7 +84,7 @@ jobs: check-tools-image: name: check image needs: trigger-mode - if: needs.trigger-mode.outputs.trigger-mode == '[docker]' + if: contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 with: MAKE_OPS_PRE: "generate" @@ -97,7 +97,7 @@ jobs: check-helm: name: check helm needs: trigger-mode - if: needs.trigger-mode.outputs.trigger-mode == '[deploy]' + if: contains(needs.trigger-mode.outputs.trigger-mode, '[deploy]') uses: apecloud/apecd/.github/workflows/release-charts.yml@v0.5.2 with: MAKE_OPS: "bump-chart-ver" diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index 051c32f8d..f1b0d919a 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -179,7 +179,6 @@ jobs: secrets: inherit check-tools-image: - name: check image needs: trigger-mode if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') && github.ref_name != 'main' }} uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 From ef7708d5b446b4f7d979305c248ca66f99c6250b Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Thu, 11 May 2023 18:44:01 +0800 Subject: [PATCH 280/439] fix: add chart dependencies to charts directory (#3146) --- deploy/delphic/Chart.lock | 12 ++++++------ deploy/delphic/Chart.yaml | 16 ++++++---------- .../delphic/charts/pgcluster-0.5.0-beta.23.tgz | Bin 0 -> 2804 bytes .../charts/redis-cluster-0.5.0-beta.23.tgz | Bin 0 -> 2333 bytes deploy/delphic/templates/_helpers.tpl | 10 +++++----- deploy/delphic/templates/deployment.yaml | 2 -- deploy/delphic/values.yaml | 4 ++-- 7 files changed, 19 insertions(+), 25 deletions(-) create mode 100644 deploy/delphic/charts/pgcluster-0.5.0-beta.23.tgz create mode 100644 deploy/delphic/charts/redis-cluster-0.5.0-beta.23.tgz diff --git a/deploy/delphic/Chart.lock b/deploy/delphic/Chart.lock index 0eb5c6364..86d4ab306 100644 --- a/deploy/delphic/Chart.lock +++ b/deploy/delphic/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: pgcluster - repository: file://../postgresql-cluster - version: 0.5.0-beta.2 + repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable + version: 0.5.0-beta.23 - name: redis-cluster - repository: file://../redis-cluster - version: 0.5.0-beta.2 -digest: sha256:3e5fb77b9fe9b4de74300a45a789778d2e9be4fe10a564e0af7a0b0e2f07e53c -generated: "2023-04-25T11:32:23.847825+08:00" + repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable + version: 0.5.0-beta.23 +digest: sha256:2455b2fe7ebbe34b584a623f17b9e57aaa9ef03160f4dee5ae1b8bf4efa04129 +generated: "2023-05-09T22:46:49.543381+08:00" diff --git a/deploy/delphic/Chart.yaml b/deploy/delphic/Chart.yaml index 9c221be73..128a4c5ad 100644 --- a/deploy/delphic/Chart.yaml +++ b/deploy/delphic/Chart.yaml @@ -25,13 +25,9 @@ appVersion: "1.16.0" dependencies: - - condition: postgres.enabled - name: pgcluster - alias: postgres - repository: file://../postgresql-cluster - version: 0.5.0-beta.2 - - condition: redis.enabled - name: redis-cluster - alias: redis - repository: file://../redis-cluster - version: 0.5.0-beta.2 \ No newline at end of file + - name: pgcluster + repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable + version: ~0.5.0-0 + - name: redis-cluster + repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable + version: ~0.5.0-0 \ No newline at end of file diff --git a/deploy/delphic/charts/pgcluster-0.5.0-beta.23.tgz b/deploy/delphic/charts/pgcluster-0.5.0-beta.23.tgz new file mode 100644 index 0000000000000000000000000000000000000000..cc6d056f0ead1746d2cf0dbb6a2e7026e179126e GIT binary patch literal 2804 zcmVDc zVQyr3R8em|NM&qo0PH(kbK5qvdB(5UBe#>wv1mz_>@%9_Lt=N@oaXk$opk!tfykwV z8Uz>sl%qKQe)~7L@M_v}8rR9!F&-=uSS)rIdj;rx8S_FLRC`A&s!X(@8GmrgZ8RE< z4yIH0I~t9;zoY5s@WFUGog7RjlhNV+gVA_@GCFtwquVB=uNB%*^jgYIV`c?!s8Lcd`1$E$2-LvO$TNto zwn8cu{8B9NnJ0k=n&BKOohD|J;~Z$7a~3-$$+{$96wRX1aDj$KlYR2Fx8bX`+Wyy+ z7pVUZ2e8HdC)2}e*Z!xo$-VvGMcaX6OliRl`2BY#b)6YTWCwm-F%3)uh2NjPd@)R= z%BV4zK*~5;fMbj~RS0XU7+rAGU?eOMa;h~Zz=V;ok%a<-S|KAS72TqiXY9B=}(tBSQ2UGw~{*AOPMe zypl2}B$I*}scbh2OKCEPlyZ%P>|F1S1Y)AWtPo(t|xxn4+>HN^*1-x9c7k%;5+<8r7P`NaNT_#C1L19Ztw|)DQPJY0F|81uTjPjJCFk;tiKWBNnXn)<`Lob& zJhKZ3j8SWyRlb=}N@O^>013$hDKmLgU$*ptH21Sg0`jSvXZ9jlEQqd(IajNwzs~5hPeMEnd!Yt1+Rj10phsx&f_PE@5ek%6g-K zWw!-a`Y(&OT*j=FES4+i&kuZx6On&};oef~v;RL<$gTH`Onxh+z!v-8pB|1n_CGrq zP4DghF53C|-V^fUhumc^Wv(PsAHv;<5P7n9aY4?{ht^9lfiTFuAJFf4(4EN68o;oI zbGcFEkCbdoh3K6xFD+zckDqKX7BL*`yGNFt6luynz+gClfXk*{7Z-#$I*%&X5LQSF zo20!jC}%0d#CjITK}3E-kK4f;yMpCL!vbS!^G+=@bVmnDph4~VP8sqIWFcu+T#N~HRbJ}LvVvnU<+p& zBMkm`0E0gUpKn$&%cStM>YLE&Aw97+TZr^(&(S}J^K)Co@FKa6$;gv&9i&E%v61RU zSSo60R1m<^DvhvXG|!{c%AO-8_o}15^Yfl5pWuBV4R!=_8B0b5U1Bm^Y&@b;yH0#X z+AGkLK>TNjw9D_Xml$8RUN1TCB%*$9n9#-+mTg{|rr4+P-M|p4aR^5fp_acHo6S1@ z)m8qi|E!{G@TZHRrad-)o&NVZkSkUdm!(oae(kt@TdgnuD>v)5j)J%4f3sP){+k`_ z-`9V4(wbK^G|zR^zR_8aDbH7QOir0d=5Q45RWdZRJV2>@(j|8S!T)XZSiJJ_pFl8? z*phE*5!ejNE)f8aty^J+N%0obIn;>U(NJ1~B^8)6;d*{xv@>~le%?)M8eyXJ4Odi@ z7Z>fdL(C}`?m5D?f(ke%cwz1=K{nxe+|+=)C7y{XoaA80@|F>=yw|N|Jo&1ZG~tlaYKUT zxp_m<<>_!mW3BxDptuHx#?f)v+;OlaFV2f*epf@32gDZ0Klj14RJcn>c?R7aZJta2 z?ZotUG;*iLxpy;h@~~^kiwuuAW!bM~ysxP)P1Cyeo7h+=?NI5+BYtUBw`F@y|AHHD z^mLEhTMFG^z)})*q1$2vn}(v+FJ*$dy^Q7PEz;yS#SFd{F}6l~^zG{HY|*`iT=o%r zx9eLxckIjms$0ohT4PK8H=fNV-TZHS|NYP1v>O**!{M;?J&#(@IHHAFNyR?8+#ovr zzmPMAZ1Jt+*pn)rE{0aahTW7==hW5Hmj=8si4x>;vRTfoGtQ9a{H3f@6&%A&AI;fM zN)~zL3WhKkkY498gjkA{Ei;;H*R@8qXpXv4x_Vursh&C0^-E4oysF-cJZa84YYgr{ z;=YDx-fV*ueB*AHALjmUCN@0W%08^H_<5htAc`cHOqgE&nhagbLq`xT=D#j~xlFdQ z_PKlYt)CcN<^P+XqI~HZ+wOl4XXB3lKbTI(_x}Gb+Kv7HU-13m`=6^rKv1&JjRd#h z<0VI1RN8VoxQJ`mA^?qu_C;|_7mGhJHX*)vtS2p@JGQD7G`&IyHw%bc-A|wW_k4Kv zMLM?i`;T$={`YV^ySM*4X@AN7D=Y74Zm*$Vslm5k#@~8${q|d*{jVu!HkG-p8gQHa zPY(CH_21$C?B4$Gq6Pb>B7sNl8`mwK;PLff?#}FSea;LiWgG@qAZvf7+lQ)=`$yf& z>#r@ke?+V8KUyKr*iuM^x4gzS`=9N1@Bj9v<9qwRlePnIs4=L7`_A8QKb);lz@lK> zf4)!SQ@TW5{pW`+@?0vTL9dW=SaP|5%vQThEO$X6r-rT3R>!MGKNSht0f9@GS;M1T zVah&W;?LFp=WztDZE-J!n`udc9F_a-JR;HY$sZ?1DkM7)K392nbOH%ennX)x_S|1j zK8Y3|)t>uXcC40r_D}h$*J7_mw4m{6k-Pt0(d0>_&vNo4TF_JSBr=(OmWnOOlYf&P zct;hJg@)(HKWY-?O4?kWL@YtN=fRcylSFGB%LMoCJ=A@>Z};t6ZvP1Y0RR6gr_#Lu GJ^%nqGK#AJ literal 0 HcmV?d00001 diff --git a/deploy/delphic/charts/redis-cluster-0.5.0-beta.23.tgz b/deploy/delphic/charts/redis-cluster-0.5.0-beta.23.tgz new file mode 100644 index 0000000000000000000000000000000000000000..b58eeaeb5fa87db7cf9ad5e5f3d40a1899d0aca7 GIT binary patch literal 2333 zcmV+&3F7u2iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH$#Z`(N1{mfr6kG+fC&9kN)+jRf~2V5^HdcEz-BI&hQ91aCd zjchg)sgjfvH_iR_14&7i9D9>ugBG_4KiD!k@0=M9M>d!WJK(ak4oz~rWX91o%jKhc z9K+#oIGs$w_i#Ae{vJ*b$BzytlhJfC8V!#okA{ciqv`k&4DVBjeWSFFnMcDv%xhb@ z|B=B8@fM90TFqcJBCIHypGR~^rzAzojVN50dZOUeI}Ds+Aq8iC7LKFyz~52kz`X}S zrVad3&hc+j^K(ndQs+2>rE`UyB}wo(Ds(~w_f+foxp0c*f^wZF%vy9dv5xU`8!-}* zFXGc6Ob>K!70wNSR}EW|0g)Y5P%zrABU&Goc(43QfRL z%**(sz+L;Qb@(T5YvSScOEy$_+= zaSBTM@MbA22n!5;d-C$dAk!vi&S46fkmvzUkV|F|R?G-CmuSIhm?IQEj#CiIX;|yh zfWy3y%%P=(RKa+xOXX%TBE+I{LZLkEEQ|?pXmX)K^!%!&;OiNjphSm+R8m%1>-7qa z5h>1~*CQm?N;qw11OSy6E1f~cq(wpsBXVZe&sB<-GZ+$LFNEXEiaG&^Vf#w!!V7A3 zX*gQH(1&*jr7fA1YcR-lo})@ZY3Fse1#*$CA+4<1paN5=2aw2C2+kXnd4qDTEEl@8 zfiz!g!7*%tTY5HVXtX|)R)bL7AhT$Fo>e@48)w0k3bX=LD`B+CQ8^12!Yx4x=OhN& zjVM2ZE&0;3e<_skS&YK-%6z{?jgtAR{ssBM_c*-@#JXukpI8NIHd3lV?4J?6!_=q3z)BKhw;l6!3RH6zQ+1C zwaUY||FJLr-!D=6bV*(DhfBd*;(t7uj<(`|I)1qSeVuW2l^l?#mqn;fmKbh9ptE>H zhi(8lNUpER)z!e)G(sWty5*tA=)qtE6Y3rsTM7Q~1o}2!D?DUfpuuN<>M9P?ad;H@ zS(#Akd?jw`&cq5GJ&a2UqO0G-eII2;GO5-HL);9 z&Pt15b&lbB8c*D)b0$!x7JLyK>JALmTBy^0oUemDPF|gPSf3Dmar6@<&oeM6eMJ_D za2;-xRaqA2go6tqrLTrdUkit-&kJ8kdy9EJWY@$ZYb;tXHdd`b_O9Q>>e4oJz6%=6 z{&M$i-N@M3A`+pe(NGb$ma{YYg~Am^D3?L+dpr2v_O@t6rgwL;-H#pDYwf9zmg*Mo z_cQEk>+ zih?dt=F~2eAo~oiuKcA$mgzp_Xk@<+%A!Q>w0Tjnpe+xolz!6mr2HBy3firfn|8|8 z)ee-;@Uhemw~TO;O3oBpU^i^+iQ z)bVx0TL}bAJW*iLkVIyLtiZNkn?JFK2bZD32m+9UnjLQGSyR z=}-G8sczY9YM}RV>o{^f(JEb9{o}@&8rYSVWsb*^iTq7{jBbRk zL%?Uvu2-i!yxP+0_JqdLNcg39pN^}C?XR-rW-!jt+Lt;-+gSx% zgPFL*G-lEN`-#G9CD));!1N+PfhN3Hri7lHy+3o>An8NZx_Wzj2B|QX(1mbG_>AHc zI{##n@L5kR7m5F-DIiN20&)SFuIrtCh!&_#A zE-gGid1?tQjQ)V!5h_w-Nd!0g1EDL+b&AQKtX1$Z9>&A?la2od00960NM5sn04x9i DQ$Cj3 literal 0 HcmV?d00001 diff --git a/deploy/delphic/templates/_helpers.tpl b/deploy/delphic/templates/_helpers.tpl index f9b0fff23..eff808411 100644 --- a/deploy/delphic/templates/_helpers.tpl +++ b/deploy/delphic/templates/_helpers.tpl @@ -63,7 +63,7 @@ Create the name of the service account to use {{- define "delphic.common.envs" }} - name: REDIS_URL - value: redis://{{ .Release.Name }}-{{ .Values.redis.nameOverride }}-redis:6379 + value: redis://{{ .Release.Name }}-{{ index .Values "redis-cluster" "nameOverride" }}-redis:6379 - name: MODEL_NAME value: text-davinci-003 - name: MAX_TOKENS @@ -73,22 +73,22 @@ Create the name of the service account to use - name: POSTGRES_HOST valueFrom: secretKeyRef: - name: {{ .Release.Name }}-{{ .Values.postgres.nameOverride }}-conn-credential + name: {{ .Release.Name }}-{{ .Values.pgcluster.nameOverride }}-conn-credential key: host - name: POSTGRES_PORT valueFrom: secretKeyRef: - name: {{ .Release.Name }}-{{ .Values.postgres.nameOverride }}-conn-credential + name: {{ .Release.Name }}-{{ .Values.pgcluster.nameOverride }}-conn-credential key: port - name: POSTGRES_USER valueFrom: secretKeyRef: - name: {{ .Release.Name }}-{{ .Values.postgres.nameOverride }}-conn-credential + name: {{ .Release.Name }}-{{ .Values.pgcluster.nameOverride }}-conn-credential key: username - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - name: {{ .Release.Name }}-{{ .Values.postgres.nameOverride }}-conn-credential + name: {{ .Release.Name }}-{{ .Values.pgcluster.nameOverride }}-conn-credential key: password - name: POSTGRES_DB value: delphic diff --git a/deploy/delphic/templates/deployment.yaml b/deploy/delphic/templates/deployment.yaml index d56afe6e4..d31eb704e 100644 --- a/deploy/delphic/templates/deployment.yaml +++ b/deploy/delphic/templates/deployment.yaml @@ -5,9 +5,7 @@ metadata: labels: {{- include "delphic.labels" . | nindent 4 }} spec: - {{- if not .Values.autoscaling.enabled }} replicas: {{ .Values.replicaCount }} - {{- end }} selector: matchLabels: {{- include "delphic.selectorLabels" . | nindent 6 }} diff --git a/deploy/delphic/values.yaml b/deploy/delphic/values.yaml index 056cb6aab..a79d6228d 100644 --- a/deploy/delphic/values.yaml +++ b/deploy/delphic/values.yaml @@ -82,11 +82,11 @@ tolerations: [] affinity: {} -postgres: +pgcluster: enabled: true nameOverride: postgres -redis: +redis-cluster: enabled: true nameOverride: redis From e0c2899c2bea30ac4dfcf7cfd9c058bf1252f86c Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Thu, 11 May 2023 20:56:53 +0800 Subject: [PATCH 281/439] chore: Updated pg performance parameters (#3205) (#3206) --- deploy/postgresql/config/pg12-config.tpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/postgresql/config/pg12-config.tpl b/deploy/postgresql/config/pg12-config.tpl index f68edd8c3..6557ddffe 100644 --- a/deploy/postgresql/config/pg12-config.tpl +++ b/deploy/postgresql/config/pg12-config.tpl @@ -62,9 +62,9 @@ bgwriter_lru_maxpages = '1000' bgwriter_lru_multiplier = '10.0' bytea_output = 'hex' check_function_bodies = 'True' -checkpoint_completion_target = '0.4' +checkpoint_completion_target = '0.9' checkpoint_flush_after = '32' -checkpoint_timeout = '25min' +checkpoint_timeout = '15min' checkpoint_warning = '30s' client_min_messages = 'notice' # commit_delay = '20' @@ -188,7 +188,7 @@ max_standby_streaming_delay = '300000ms' max_sync_workers_per_subscription = '2' max_wal_senders = '64' # {LEAST(GREATEST(DBInstanceClassMemory/2097152, 2048), 16384)} -max_wal_size = '{{ printf "%dMB" ( min ( max ( div $phy_memory 2097152 ) 2048 ) 16384 ) }}' +max_wal_size = '{{ printf "%dMB" ( min ( max ( div $phy_memory 2097152 ) 2048 ) 32768 ) }}' max_worker_processes = '{{ max $phy_cpu 8 }}' # min_parallel_index_scan_size unit is 8KB, 64 = 512KB min_parallel_index_scan_size = '512kB' From 828484d73c5cd33f5f9d886b0a455846d8f19304 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Thu, 11 May 2023 21:50:19 +0800 Subject: [PATCH 282/439] chore: fix kbcli delete ops failed when ops is running (#3204) --- internal/cli/cmd/cluster/delete_ops.go | 45 ++++++ internal/cli/cmd/cluster/delete_ops_test.go | 144 ++++++++++++++++++++ internal/cli/cmd/cluster/operations_test.go | 26 ---- internal/cli/testing/factory.go | 1 + 4 files changed, 190 insertions(+), 26 deletions(-) create mode 100644 internal/cli/cmd/cluster/delete_ops_test.go diff --git a/internal/cli/cmd/cluster/delete_ops.go b/internal/cli/cmd/cluster/delete_ops.go index e8264ad34..10630d049 100644 --- a/internal/cli/cmd/cluster/delete_ops.go +++ b/internal/cli/cmd/cluster/delete_ops.go @@ -20,12 +20,20 @@ along with this program. If not, see . package cluster import ( + "context" "fmt" + jsonpatch "github.com/evanphx/json-patch" "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + apitypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" "k8s.io/cli-runtime/pkg/genericclioptions" cmdutil "k8s.io/kubectl/pkg/cmd/util" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/delete" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" @@ -33,6 +41,7 @@ import ( func NewDeleteOpsCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := delete.NewDeleteOptions(f, streams, types.OpsGVR()) + o.PreDeleteHook = preDeleteOps cmd := &cobra.Command{ Use: "delete-ops", Short: "Delete an OpsRequest.", @@ -47,6 +56,42 @@ func NewDeleteOpsCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *co return cmd } +func preDeleteOps(o *delete.DeleteOptions, obj runtime.Object) error { + unstructured := obj.(*unstructured.Unstructured) + opsRequest := &appsv1alpha1.OpsRequest{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured.Object, opsRequest); err != nil { + return err + } + if opsRequest.Status.Phase != appsv1alpha1.OpsRunningPhase { + return nil + } + if !o.Force { + return fmt.Errorf(`OpsRequest "%s" is Running, you can specify "--force" to delete it`, opsRequest.Name) + } + // remove the finalizers + dynamic, err := o.Factory.DynamicClient() + if err != nil { + return err + } + oldOps := opsRequest.DeepCopy() + opsRequest.Finalizers = []string{} + oldData, err := json.Marshal(oldOps) + if err != nil { + return err + } + newData, err := json.Marshal(opsRequest) + if err != nil { + return err + } + patchBytes, err := jsonpatch.CreateMergePatch(oldData, newData) + if err != nil { + return err + } + _, err = dynamic.Resource(types.OpsGVR()).Namespace(opsRequest.Namespace).Patch(context.TODO(), + opsRequest.Name, apitypes.MergePatchType, patchBytes, metav1.PatchOptions{}) + return err +} + // completeForDeleteOps complete cmd for delete OpsRequest, if resource name // is not specified, construct a label selector based on the cluster name to // delete all OpeRequest belonging to the cluster. diff --git a/internal/cli/cmd/cluster/delete_ops_test.go b/internal/cli/cmd/cluster/delete_ops_test.go new file mode 100644 index 000000000..6d8f07947 --- /dev/null +++ b/internal/cli/cmd/cluster/delete_ops_test.go @@ -0,0 +1,144 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package cluster + +import ( + "bytes" + "fmt" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes/scheme" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/delete" + clitesting "github.com/apecloud/kubeblocks/internal/cli/testing" + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var _ = Describe("Expose", func() { + const ( + namespace = "test" + opsName = "test-ops" + ) + + var ( + streams genericclioptions.IOStreams + tf *cmdtesting.TestFactory + in *bytes.Buffer + ) + generateOpsObject := func(opsName string, phase appsv1alpha1.OpsPhase) *appsv1alpha1.OpsRequest { + return &appsv1alpha1.OpsRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: opsName, + Namespace: namespace, + }, + Spec: appsv1alpha1.OpsRequestSpec{ + ClusterRef: "test-cluster", + Type: "Restart", + }, + Status: appsv1alpha1.OpsRequestStatus{ + Phase: phase, + }, + } + } + BeforeEach(func() { + streams, in, _, _ = genericclioptions.NewTestIOStreams() + tf = clitesting.NewTestFactory(namespace) + }) + + AfterEach(func() { + tf.Cleanup() + }) + + initClient := func(opsRequest runtime.Object) { + _ = appsv1alpha1.AddToScheme(scheme.Scheme) + codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) + httpResp := func(obj runtime.Object) *http.Response { + return &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, obj)} + } + + tf.UnstructuredClient = &clientfake.RESTClient{ + GroupVersion: schema.GroupVersion{Group: types.AppsAPIGroup, Version: types.AppsAPIVersion}, + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + Client: clientfake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { + return httpResp(opsRequest), nil + }), + } + + tf.FakeDynamicClient = clitesting.FakeDynamicClient(opsRequest) + tf.Client = tf.UnstructuredClient + } + + It("test completeForDeleteOps function", func() { + clusterName := "wesql" + args := []string{clusterName} + clusterLabel := util.BuildLabelSelectorByNames("", args) + testLabel := "kubeblocks.io/test=test" + + By("test delete OpsRequest with cluster") + o := delete.NewDeleteOptions(tf, streams, types.OpsGVR()) + Expect(completeForDeleteOps(o, args)).Should(Succeed()) + Expect(o.LabelSelector == clusterLabel).Should(BeTrue()) + + By("test delete OpsRequest with cluster and custom label") + o.LabelSelector = testLabel + Expect(completeForDeleteOps(o, args)).Should(Succeed()) + Expect(o.LabelSelector == testLabel+","+clusterLabel).Should(BeTrue()) + + By("test delete OpsRequest with name") + o.Names = []string{"test1"} + Expect(completeForDeleteOps(o, nil)).Should(Succeed()) + Expect(len(o.ConfirmedNames)).Should(Equal(1)) + }) + + It("Testing the deletion of running OpsRequest", func() { + By("init opsRequests and k8s client") + runningOps := generateOpsObject(opsName, appsv1alpha1.OpsRunningPhase) + initClient(runningOps) + + By("expect error when deleting running opsRequest") + o := delete.NewDeleteOptions(tf, streams, types.OpsGVR()) + o.PreDeleteHook = preDeleteOps + o.Names = []string{runningOps.Name} + in.Write([]byte(runningOps.Name + "\n")) + err := o.Run() + Expect(err).ShouldNot(BeNil()) + Expect(err.Error()).Should(Equal(fmt.Sprintf(`OpsRequest "%s" is Running, you can specify "--force" to delete it`, runningOps.Name))) + + By("expect success when deleting running opsRequest with --force") + o.GracePeriod = 0 + o.Names = []string{runningOps.Name} + in.Write([]byte(runningOps.Name + "\n")) + o.Force = true + err = o.Run() + Expect(err).Should(BeNil()) + }) +}) diff --git a/internal/cli/cmd/cluster/operations_test.go b/internal/cli/cmd/cluster/operations_test.go index 720b64d74..5724af923 100644 --- a/internal/cli/cmd/cluster/operations_test.go +++ b/internal/cli/cmd/cluster/operations_test.go @@ -29,10 +29,7 @@ import ( cmdtesting "k8s.io/kubectl/pkg/cmd/testing" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/internal/cli/delete" "github.com/apecloud/kubeblocks/internal/cli/testing" - "github.com/apecloud/kubeblocks/internal/cli/types" - "github.com/apecloud/kubeblocks/internal/cli/util" testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) @@ -185,27 +182,4 @@ var _ = Describe("operations", func() { capturedOutput, _ := done() Expect(testing.ContainExpectStrings(capturedOutput, "kbcli cluster describe-ops")).Should(BeTrue()) }) - - It("list and delete operations", func() { - clusterName := "wesql" - args := []string{clusterName} - clusterLabel := util.BuildLabelSelectorByNames("", args) - testLabel := "kubeblocks.io/test=test" - - By("test delete OpsRequest with cluster") - o := delete.NewDeleteOptions(tf, streams, types.OpsGVR()) - Expect(completeForDeleteOps(o, args)).Should(Succeed()) - Expect(o.LabelSelector == clusterLabel).Should(BeTrue()) - - By("test delete OpsRequest with cluster and custom label") - o.LabelSelector = testLabel - Expect(completeForDeleteOps(o, args)).Should(Succeed()) - Expect(o.LabelSelector == testLabel+","+clusterLabel).Should(BeTrue()) - - By("test delete OpsRequest with name") - o.Names = []string{"test1"} - Expect(completeForDeleteOps(o, nil)).Should(Succeed()) - Expect(len(o.ConfirmedNames)).Should(Equal(1)) - }) - }) diff --git a/internal/cli/testing/factory.go b/internal/cli/testing/factory.go index d1e9edde4..98de22c17 100644 --- a/internal/cli/testing/factory.go +++ b/internal/cli/testing/factory.go @@ -116,6 +116,7 @@ func testDynamicResources() []*restmapper.APIGroupResources { VersionedResources: map[string][]metav1.APIResource{ "v1alpha1": { {Name: "clusters", Namespaced: true, Kind: "Cluster"}, + {Name: "opsrequests", Namespaced: true, Kind: "OpsRequest"}, }, }, }, From 1464e664edcdca07ccbdf708fe47e79d3c704911 Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Thu, 11 May 2023 22:15:31 +0800 Subject: [PATCH 283/439] chore: fixed controllers/apps/components/deployment_controller.go and stateful_set_controller.go nil pointer reference error (#3209) --- .../apps/components/deployment_controller.go | 9 +++++---- .../apps/components/stateful_set_controller.go | 9 +++++---- .../transformer_sts_horizontal_scaling.go | 14 +++++++------- internal/controllerutil/controller_common.go | 10 ++++------ internal/controllerutil/type.go | 17 +++++++++++++++++ 5 files changed, 38 insertions(+), 21 deletions(-) diff --git a/controllers/apps/components/deployment_controller.go b/controllers/apps/components/deployment_controller.go index 908823ca1..41e198357 100644 --- a/controllers/apps/components/deployment_controller.go +++ b/controllers/apps/components/deployment_controller.go @@ -59,9 +59,10 @@ func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) ) reqCtx := intctrlutil.RequestCtx{ - Ctx: ctx, - Req: req, - Log: log.FromContext(ctx).WithValues("deployment", req.NamespacedName), + Ctx: ctx, + Req: req, + Log: log.FromContext(ctx).WithValues("deployment", req.NamespacedName), + Recorder: r.Recorder, } if err = r.Client.Get(reqCtx.Ctx, reqCtx.Req.NamespacedName, deploy); err != nil { @@ -82,7 +83,7 @@ func (r *DeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // patch the current componentSpec workload's custom labels if err := patchWorkloadCustomLabel(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { - reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "Deployment Controller PatchWorkloadCustomLabelFailed", err.Error()) + reqCtx.Event(cluster, corev1.EventTypeWarning, "Deployment Controller PatchWorkloadCustomLabelFailed", err.Error()) return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } if requeueAfter, err := updateComponentStatusInClusterStatus(compCtx, cluster); err != nil { diff --git a/controllers/apps/components/stateful_set_controller.go b/controllers/apps/components/stateful_set_controller.go index 890c75dec..42e04e3ae 100644 --- a/controllers/apps/components/stateful_set_controller.go +++ b/controllers/apps/components/stateful_set_controller.go @@ -59,9 +59,10 @@ func (r *StatefulSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ) reqCtx := intctrlutil.RequestCtx{ - Ctx: ctx, - Req: req, - Log: log.FromContext(ctx).WithValues("statefulSet", req.NamespacedName), + Ctx: ctx, + Req: req, + Log: log.FromContext(ctx).WithValues("statefulSet", req.NamespacedName), + Recorder: r.Recorder, } if err = r.Client.Get(reqCtx.Ctx, reqCtx.Req.NamespacedName, sts); err != nil { @@ -82,7 +83,7 @@ func (r *StatefulSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) } // patch the current componentSpec workload's custom labels if err := patchWorkloadCustomLabel(reqCtx.Ctx, r.Client, cluster, componentSpec); err != nil { - reqCtx.Recorder.Event(cluster, corev1.EventTypeWarning, "StatefulSet Controller PatchWorkloadCustomLabelFailed", err.Error()) + reqCtx.Event(cluster, corev1.EventTypeWarning, "StatefulSet Controller PatchWorkloadCustomLabelFailed", err.Error()) return intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "") } reqCtx.Log.V(1).Info("before updateComponentStatusInClusterStatus", diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 03f68ddb4..4bd5d939e 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -395,7 +395,7 @@ func doBackup(reqCtx intctrlutil.RequestCtx, // use backup tool such as xtrabackup case appsv1alpha1.HScaleDataClonePolicyFromBackup: // TODO: db core not support yet, leave it empty - reqCtx.Recorder.Eventf(cluster, + reqCtx.Eventf(cluster, corev1.EventTypeWarning, "HorizontalScaleFailed", "scale with backup tool not support yet") @@ -407,7 +407,7 @@ func doBackup(reqCtx intctrlutil.RequestCtx, } vcts := component.VolumeClaimTemplates if len(vcts) == 0 { - reqCtx.Recorder.Eventf(cluster, + reqCtx.Eventf(cluster, corev1.EventTypeNormal, "HorizontalScale", "no VolumeClaimTemplates, no need to do data clone.") @@ -511,7 +511,7 @@ func checkedCreateDeletePVCCronJob(cli roclient.ReadonlyClient, vertex := &lifecycleVertex{obj: cronJob, action: actionPtr(CREATE)} dag.AddVertex(vertex) dag.Connect(root, vertex) - reqCtx.Recorder.Eventf(cluster, + reqCtx.Eventf(cluster, corev1.EventTypeNormal, "CronJobCreate", "create cronjob to delete pvc/%s", @@ -579,7 +579,7 @@ func deleteSnapshot(cli roclient.ReadonlyClient, vertex := &lifecycleVertex{obj: vs, oriObj: vs, action: actionPtr(DELETE)} dag.AddVertex(vertex) dag.Connect(root, vertex) - reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "VolumeSnapshotDelete", "Delete volumeSnapshot/%s", snapshotKey.Name) + reqCtx.Eventf(cluster, corev1.EventTypeNormal, "VolumeSnapshotDelete", "Delete volumeSnapshot/%s", snapshotKey.Name) return nil } @@ -600,7 +600,7 @@ func deleteBackup(reqCtx intctrlutil.RequestCtx, cli roclient.ReadonlyClient, dag.AddVertex(vertex) dag.Connect(root, vertex) } - reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "BackupJobDelete", "Delete backupJob/%s", snapshotName) + reqCtx.Eventf(cluster, corev1.EventTypeNormal, "BackupJobDelete", "Delete backupJob/%s", snapshotName) return nil } @@ -661,7 +661,7 @@ func doSnapshot(cli roclient.ReadonlyClient, vertex := &lifecycleVertex{obj: snapshot, action: actionPtr(CREATE)} dag.AddVertex(vertex) dag.Connect(root, vertex) - reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "VolumeSnapshotCreate", "Create volumesnapshot/%s", snapshotKey.Name) + reqCtx.Eventf(cluster, corev1.EventTypeNormal, "VolumeSnapshotCreate", "Create volumesnapshot/%s", snapshotKey.Name) return nil } @@ -791,7 +791,7 @@ func createBackup(reqCtx intctrlutil.RequestCtx, return err } - reqCtx.Recorder.Eventf(cluster, corev1.EventTypeNormal, "BackupJobCreate", "Create backupJob/%s", backupKey.Name) + reqCtx.Eventf(cluster, corev1.EventTypeNormal, "BackupJobCreate", "Create backupJob/%s", backupKey.Name) return nil } diff --git a/internal/controllerutil/controller_common.go b/internal/controllerutil/controller_common.go index 6396a481a..1e91e26ac 100644 --- a/internal/controllerutil/controller_common.go +++ b/internal/controllerutil/controller_common.go @@ -134,11 +134,11 @@ func HandleCRDeletion(reqCtx RequestCtx, cluster, ok := cr.(*v1alpha1.Cluster) // throw warning event if terminationPolicy set to DoNotTerminate if ok && cluster.Spec.TerminationPolicy == v1alpha1.DoNotTerminate { - reqCtx.Recorder.Eventf(cr, corev1.EventTypeWarning, constant.ReasonDeleteFailed, + reqCtx.Eventf(cr, corev1.EventTypeWarning, constant.ReasonDeleteFailed, "Deleting %s: %s failed due to terminationPolicy set to DoNotTerminate", strings.ToLower(cr.GetObjectKind().GroupVersionKind().Kind), cr.GetName()) } else { - reqCtx.Recorder.Eventf(cr, corev1.EventTypeNormal, constant.ReasonDeletingCR, "Deleting %s: %s", + reqCtx.Eventf(cr, corev1.EventTypeNormal, constant.ReasonDeletingCR, "Deleting %s: %s", strings.ToLower(cr.GetObjectKind().GroupVersionKind().Kind), cr.GetName()) } } @@ -162,10 +162,8 @@ func HandleCRDeletion(reqCtx RequestCtx, return ResultToP(CheckedRequeueWithError(err, reqCtx.Log, "")) } // record resources deleted event - if reqCtx.Recorder != nil { - reqCtx.Recorder.Eventf(cr, corev1.EventTypeNormal, constant.ReasonDeletedCR, "Deleted %s: %s", - strings.ToLower(cr.GetObjectKind().GroupVersionKind().Kind), cr.GetName()) - } + reqCtx.Eventf(cr, corev1.EventTypeNormal, constant.ReasonDeletedCR, "Deleted %s: %s", + strings.ToLower(cr.GetObjectKind().GroupVersionKind().Kind), cr.GetName()) } } diff --git a/internal/controllerutil/type.go b/internal/controllerutil/type.go index 097e30497..728722cdb 100644 --- a/internal/controllerutil/type.go +++ b/internal/controllerutil/type.go @@ -23,6 +23,7 @@ import ( "context" "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" ) @@ -35,6 +36,22 @@ type RequestCtx struct { Recorder record.EventRecorder } +// Event is wrapper for Recorder.Event, if Recorder is nil, then it's no-op. +func (r *RequestCtx) Event(object runtime.Object, eventtype, reason, message string) { + if r == nil || r.Recorder == nil { + return + } + r.Recorder.Event(object, eventtype, reason, message) +} + +// Eventf is wrapper for Recorder.Eventf, if Recorder is nil, then it's no-op. +func (r *RequestCtx) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) { + if r == nil || r.Recorder == nil { + return + } + r.Recorder.Eventf(object, eventtype, reason, messageFmt, args...) +} + // UpdateCtxValue update Context value, return parent Context. func (r *RequestCtx) UpdateCtxValue(key, val any) context.Context { p := r.Ctx From 5176e47d6c71eb294b8bc68cbed58fa346605618 Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Thu, 11 May 2023 22:52:32 +0800 Subject: [PATCH 284/439] feat: support cli fault network (#3100) --- docs/user_docs/cli/cli.md | 7 + docs/user_docs/cli/kbcli.md | 1 + docs/user_docs/cli/kbcli_fault.md | 43 +++ docs/user_docs/cli/kbcli_fault_network.md | 50 +++ .../cli/kbcli_fault_network_bandwidth.md | 100 ++++++ .../cli/kbcli_fault_network_corrupt.md | 97 +++++ .../cli/kbcli_fault_network_delay.md | 98 +++++ docs/user_docs/cli/kbcli_fault_network_dns.md | 44 +++ .../cli/kbcli_fault_network_dns_error.md | 64 ++++ .../cli/kbcli_fault_network_dns_random.md | 64 ++++ .../cli/kbcli_fault_network_duplicate.md | 97 +++++ .../user_docs/cli/kbcli_fault_network_http.md | 46 +++ .../cli/kbcli_fault_network_http_abort.md | 87 +++++ .../cli/kbcli_fault_network_http_delay.md | 87 +++++ .../cli/kbcli_fault_network_http_patch.md | 88 +++++ .../cli/kbcli_fault_network_http_replace.md | 89 +++++ .../user_docs/cli/kbcli_fault_network_loss.md | 97 +++++ .../cli/kbcli_fault_network_partition.md | 95 +++++ go.mod | 3 + go.sum | 7 + internal/cli/cmd/cli.go | 2 + internal/cli/cmd/fault/fault.go | 117 ++++++ internal/cli/cmd/fault/fault_constant.go | 134 +++++++ internal/cli/cmd/fault/fault_dns.go | 155 ++++++++ internal/cli/cmd/fault/fault_http.go | 257 ++++++++++++++ internal/cli/cmd/fault/fault_network.go | 335 ++++++++++++++++++ .../create/template/dns_chaos_template.cue | 56 +++ .../create/template/http_chaos_template.cue | 103 ++++++ .../template/network_chaos_template.cue | 127 +++++++ 29 files changed, 2550 insertions(+) create mode 100644 docs/user_docs/cli/kbcli_fault.md create mode 100644 docs/user_docs/cli/kbcli_fault_network.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_bandwidth.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_corrupt.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_delay.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_dns.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_dns_error.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_dns_random.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_duplicate.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_http.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_http_abort.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_http_delay.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_http_patch.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_http_replace.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_loss.md create mode 100644 docs/user_docs/cli/kbcli_fault_network_partition.md create mode 100644 internal/cli/cmd/fault/fault.go create mode 100644 internal/cli/cmd/fault/fault_constant.go create mode 100644 internal/cli/cmd/fault/fault_dns.go create mode 100644 internal/cli/cmd/fault/fault_http.go create mode 100644 internal/cli/cmd/fault/fault_network.go create mode 100644 internal/cli/create/template/dns_chaos_template.cue create mode 100644 internal/cli/create/template/http_chaos_template.cue create mode 100644 internal/cli/create/template/network_chaos_template.cue diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index 978c35fe7..e55b7d539 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -107,6 +107,13 @@ List and open the KubeBlocks dashboards. * [kbcli dashboard open](kbcli_dashboard_open.md) - Open one dashboard. +## [fault](kbcli_fault.md) + +Inject faults to pod. + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. + + ## [kubeblocks](kbcli_kubeblocks.md) KubeBlocks operation commands. diff --git a/docs/user_docs/cli/kbcli.md b/docs/user_docs/cli/kbcli.md index 981c94a1b..65bba3c67 100644 --- a/docs/user_docs/cli/kbcli.md +++ b/docs/user_docs/cli/kbcli.md @@ -62,6 +62,7 @@ kbcli [flags] * [kbcli clusterdefinition](kbcli_clusterdefinition.md) - ClusterDefinition command. * [kbcli clusterversion](kbcli_clusterversion.md) - ClusterVersion command. * [kbcli dashboard](kbcli_dashboard.md) - List and open the KubeBlocks dashboards. +* [kbcli fault](kbcli_fault.md) - Inject faults to pod. * [kbcli kubeblocks](kbcli_kubeblocks.md) - KubeBlocks operation commands. * [kbcli migration](kbcli_migration.md) - Data migration between two data sources. * [kbcli options](kbcli_options.md) - Print the list of flags inherited by all commands. diff --git a/docs/user_docs/cli/kbcli_fault.md b/docs/user_docs/cli/kbcli_fault.md new file mode 100644 index 000000000..63119b47d --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault.md @@ -0,0 +1,43 @@ +--- +title: kbcli fault +--- + +Inject faults to pod. + +### Options + +``` + -h, --help help for fault +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network.md b/docs/user_docs/cli/kbcli_fault_network.md new file mode 100644 index 000000000..414204069 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network.md @@ -0,0 +1,50 @@ +--- +title: kbcli fault network +--- + +Network chaos. + +### Options + +``` + -h, --help help for network +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault](kbcli_fault.md) - Inject faults to pod. +* [kbcli fault network bandwidth](kbcli_fault_network_bandwidth.md) - Limit the bandwidth that pods use to communicate with other objects. +* [kbcli fault network corrupt](kbcli_fault_network_corrupt.md) - Distorts the messages a pod communicates with other objects. +* [kbcli fault network delay](kbcli_fault_network_delay.md) - Make pods communicate with other objects lazily. +* [kbcli fault network dns](kbcli_fault_network_dns.md) - Inject faults into DNS server. +* [kbcli fault network duplicate](kbcli_fault_network_duplicate.md) - Make pods communicate with other objects to pick up duplicate packets. +* [kbcli fault network http](kbcli_fault_network_http.md) - Intercept HTTP requests and responses. +* [kbcli fault network loss](kbcli_fault_network_loss.md) - Cause pods to communicate with other objects to drop packets. +* [kbcli fault network partition](kbcli_fault_network_partition.md) - Make a pod network partitioned from other objects. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_bandwidth.md b/docs/user_docs/cli/kbcli_fault_network_bandwidth.md new file mode 100644 index 000000000..5777a4711 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_bandwidth.md @@ -0,0 +1,100 @@ +--- +title: kbcli fault network bandwidth +--- + +Limit the bandwidth that pods use to communicate with other objects. + +``` +kbcli fault network bandwidth [flags] +``` + +### Examples + +``` + # Isolate all pods network under the default namespace from the outside world, including the k8s internal network. + kbcli fault network partition + + # The specified pod is isolated from the k8s external network "kubeblocks.io". + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + + # Isolate the network between two pods. + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + + // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. + # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. + kbcli fault network loss --loss=50 + + # Block the specified pod communication, so that the packet loss rate is 50%. + kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + + kbcli fault network corrupt --corrupt=50 + + # Blocks specified pod communication with a 50% packet corruption rate. + kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + + kbcli fault network duplicate --duplicate=50 + + # Block specified pod communication so that the packet repetition rate is 50%. + kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + + kbcli fault network delay --latency=10s + + # Block the communication of the specified pod, causing its network delay for 10s. + kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s +``` + +### Options + +``` + --buffer uint32 the maximum number of bytes that can be sent instantaneously. (default 1) + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for bandwidth + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --limit uint32 the number of bytes waiting in the queue. (default 1) + --minburst uint32 the size of the peakrate bucket. + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --peakrate uint the maximum consumption rate of the bucket. + --rate string the rate at which the bandwidth is limited. For example : 10 bps/kbps/mbps/gbps. + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_corrupt.md b/docs/user_docs/cli/kbcli_fault_network_corrupt.md new file mode 100644 index 000000000..6559660d3 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_corrupt.md @@ -0,0 +1,97 @@ +--- +title: kbcli fault network corrupt +--- + +Distorts the messages a pod communicates with other objects. + +``` +kbcli fault network corrupt [flags] +``` + +### Examples + +``` + # Isolate all pods network under the default namespace from the outside world, including the k8s internal network. + kbcli fault network partition + + # The specified pod is isolated from the k8s external network "kubeblocks.io". + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + + # Isolate the network between two pods. + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + + // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. + # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. + kbcli fault network loss --loss=50 + + # Block the specified pod communication, so that the packet loss rate is 50%. + kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + + kbcli fault network corrupt --corrupt=50 + + # Blocks specified pod communication with a 50% packet corruption rate. + kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + + kbcli fault network duplicate --duplicate=50 + + # Block specified pod communication so that the packet repetition rate is 50%. + kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + + kbcli fault network delay --latency=10s + + # Block the communication of the specified pod, causing its network delay for 10s. + kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s +``` + +### Options + +``` + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + --corrupt string Indicates the probability of a packet error occurring. Value range: [0, 100]. + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for corrupt + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_delay.md b/docs/user_docs/cli/kbcli_fault_network_delay.md new file mode 100644 index 000000000..9e7ceb3b0 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_delay.md @@ -0,0 +1,98 @@ +--- +title: kbcli fault network delay +--- + +Make pods communicate with other objects lazily. + +``` +kbcli fault network delay [flags] +``` + +### Examples + +``` + # Isolate all pods network under the default namespace from the outside world, including the k8s internal network. + kbcli fault network partition + + # The specified pod is isolated from the k8s external network "kubeblocks.io". + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + + # Isolate the network between two pods. + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + + // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. + # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. + kbcli fault network loss --loss=50 + + # Block the specified pod communication, so that the packet loss rate is 50%. + kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + + kbcli fault network corrupt --corrupt=50 + + # Blocks specified pod communication with a 50% packet corruption rate. + kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + + kbcli fault network duplicate --duplicate=50 + + # Block specified pod communication so that the packet repetition rate is 50%. + kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + + kbcli fault network delay --latency=10s + + # Block the communication of the specified pod, causing its network delay for 10s. + kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s +``` + +### Options + +``` + -c, --correlation string Indicates the probability of a packet error occurring. Value range: [0, 100]. (default "0") + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for delay + --jitter string the variation range of the delay time. (default "0ms") + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --latency string the length of time to delay. + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_dns.md b/docs/user_docs/cli/kbcli_fault_network_dns.md new file mode 100644 index 000000000..e049edbbf --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_dns.md @@ -0,0 +1,44 @@ +--- +title: kbcli fault network dns +--- + +Inject faults into DNS server. + +### Options + +``` + -h, --help help for dns +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. +* [kbcli fault network dns error](kbcli_fault_network_dns_error.md) - Make DNS return an error when resolving external domain names. +* [kbcli fault network dns random](kbcli_fault_network_dns_random.md) - Make DNS return any IP when resolving external domain names. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_dns_error.md b/docs/user_docs/cli/kbcli_fault_network_dns_error.md new file mode 100644 index 000000000..eac118792 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_dns_error.md @@ -0,0 +1,64 @@ +--- +title: kbcli fault network dns error +--- + +Make DNS return an error when resolving external domain names. + +``` +kbcli fault network dns error [flags] +``` + +### Examples + +``` + // Inject DNS faults into all pods under the default namespace, so that any IP is returned when accessing the baidu.com domain name. + kbcli fault DNS random --patterns=baidu.com --duration=1m + + // Inject DNS faults into all pods under the default namespace, so that error is returned when accessing the baidu.com domain name. + kbcli fault DNS error --patterns=baidu.com --duration=1m +``` + +### Options + +``` + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for error + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --patterns stringArray Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network dns](kbcli_fault_network_dns.md) - Inject faults into DNS server. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_dns_random.md b/docs/user_docs/cli/kbcli_fault_network_dns_random.md new file mode 100644 index 000000000..ca57706b3 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_dns_random.md @@ -0,0 +1,64 @@ +--- +title: kbcli fault network dns random +--- + +Make DNS return any IP when resolving external domain names. + +``` +kbcli fault network dns random [flags] +``` + +### Examples + +``` + // Inject DNS faults into all pods under the default namespace, so that any IP is returned when accessing the baidu.com domain name. + kbcli fault DNS random --patterns=baidu.com --duration=1m + + // Inject DNS faults into all pods under the default namespace, so that error is returned when accessing the baidu.com domain name. + kbcli fault DNS error --patterns=baidu.com --duration=1m +``` + +### Options + +``` + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for random + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --patterns stringArray Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network dns](kbcli_fault_network_dns.md) - Inject faults into DNS server. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_duplicate.md b/docs/user_docs/cli/kbcli_fault_network_duplicate.md new file mode 100644 index 000000000..0eb3cb8b7 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_duplicate.md @@ -0,0 +1,97 @@ +--- +title: kbcli fault network duplicate +--- + +Make pods communicate with other objects to pick up duplicate packets. + +``` +kbcli fault network duplicate [flags] +``` + +### Examples + +``` + # Isolate all pods network under the default namespace from the outside world, including the k8s internal network. + kbcli fault network partition + + # The specified pod is isolated from the k8s external network "kubeblocks.io". + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + + # Isolate the network between two pods. + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + + // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. + # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. + kbcli fault network loss --loss=50 + + # Block the specified pod communication, so that the packet loss rate is 50%. + kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + + kbcli fault network corrupt --corrupt=50 + + # Blocks specified pod communication with a 50% packet corruption rate. + kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + + kbcli fault network duplicate --duplicate=50 + + # Block specified pod communication so that the packet repetition rate is 50%. + kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + + kbcli fault network delay --latency=10s + + # Block the communication of the specified pod, causing its network delay for 10s. + kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s +``` + +### Options + +``` + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duplicate string the probability of a packet being repeated. Value range: [0, 100]. + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for duplicate + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_http.md b/docs/user_docs/cli/kbcli_fault_network_http.md new file mode 100644 index 000000000..294cf3082 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_http.md @@ -0,0 +1,46 @@ +--- +title: kbcli fault network http +--- + +Intercept HTTP requests and responses. + +### Options + +``` + -h, --help help for http +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. +* [kbcli fault network http abort](kbcli_fault_network_http_abort.md) - Abort the HTTP request and response. +* [kbcli fault network http delay](kbcli_fault_network_http_delay.md) - Delay the HTTP request and response. +* [kbcli fault network http patch](kbcli_fault_network_http_patch.md) - Patch the HTTP request and response. +* [kbcli fault network http replace](kbcli_fault_network_http_replace.md) - Replace the HTTP request and response. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_http_abort.md b/docs/user_docs/cli/kbcli_fault_network_http_abort.md new file mode 100644 index 000000000..b13322843 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_http_abort.md @@ -0,0 +1,87 @@ +--- +title: kbcli fault network http abort +--- + +Abort the HTTP request and response. + +``` +kbcli fault network http abort [flags] +``` + +### Examples + +``` + # By default, the method of GET from port 80 is blocked. + kbcli fault network http abort --duration=1m + + # Block the method of GET from port 4399. + kbcli fault network http abort --port=4399 --duration=1m + + # Block the method of POST from port 4399. + kbcli fault network http abort --port=4399 --method=POST --duration=1m + + # Delays post requests from port 4399. + kbcli fault network http delay --port=4399 --method=POST --delay=15s + + # Replace the GET method sent from port 80 with the PUT method. + kbcli fault network http replace --replace-method=PUT --duration=1m + + # Replace the GET method sent from port 80 with the PUT method, and replace the request body. + kbcli fault network http replace --body="you are good luck" --replace-method=PUT --duration=2m + + # Replace the response content "you" from port 80. + kbcli fault network http replace --target=Response --body=you --duration=30s + + # AAppend content to the body of the post request sent from port 4399, in JSON format. + kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s +``` + +### Options + +``` + --abort Indicates whether to inject the fault that interrupts the connection. (default true) + --code int32 The status code responded by target. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for abort + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The URI path of the target request. Supports Matching wildcards. (default "*") + --port int32 The TCP port that the target service listens on. (default 80) + --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network http](kbcli_fault_network_http.md) - Intercept HTTP requests and responses. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_http_delay.md b/docs/user_docs/cli/kbcli_fault_network_http_delay.md new file mode 100644 index 000000000..e8c8d5bcd --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_http_delay.md @@ -0,0 +1,87 @@ +--- +title: kbcli fault network http delay +--- + +Delay the HTTP request and response. + +``` +kbcli fault network http delay [flags] +``` + +### Examples + +``` + # By default, the method of GET from port 80 is blocked. + kbcli fault network http abort --duration=1m + + # Block the method of GET from port 4399. + kbcli fault network http abort --port=4399 --duration=1m + + # Block the method of POST from port 4399. + kbcli fault network http abort --port=4399 --method=POST --duration=1m + + # Delays post requests from port 4399. + kbcli fault network http delay --port=4399 --method=POST --delay=15s + + # Replace the GET method sent from port 80 with the PUT method. + kbcli fault network http replace --replace-method=PUT --duration=1m + + # Replace the GET method sent from port 80 with the PUT method, and replace the request body. + kbcli fault network http replace --body="you are good luck" --replace-method=PUT --duration=2m + + # Replace the response content "you" from port 80. + kbcli fault network http replace --target=Response --body=you --duration=30s + + # AAppend content to the body of the post request sent from port 4399, in JSON format. + kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s +``` + +### Options + +``` + --code int32 The status code responded by target. + --delay string The time for delay. (default "10s") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for delay + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The URI path of the target request. Supports Matching wildcards. (default "*") + --port int32 The TCP port that the target service listens on. (default 80) + --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network http](kbcli_fault_network_http.md) - Intercept HTTP requests and responses. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_http_patch.md b/docs/user_docs/cli/kbcli_fault_network_http_patch.md new file mode 100644 index 000000000..93b520404 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_http_patch.md @@ -0,0 +1,88 @@ +--- +title: kbcli fault network http patch +--- + +Patch the HTTP request and response. + +``` +kbcli fault network http patch [flags] +``` + +### Examples + +``` + # By default, the method of GET from port 80 is blocked. + kbcli fault network http abort --duration=1m + + # Block the method of GET from port 4399. + kbcli fault network http abort --port=4399 --duration=1m + + # Block the method of POST from port 4399. + kbcli fault network http abort --port=4399 --method=POST --duration=1m + + # Delays post requests from port 4399. + kbcli fault network http delay --port=4399 --method=POST --delay=15s + + # Replace the GET method sent from port 80 with the PUT method. + kbcli fault network http replace --replace-method=PUT --duration=1m + + # Replace the GET method sent from port 80 with the PUT method, and replace the request body. + kbcli fault network http replace --body="you are good luck" --replace-method=PUT --duration=2m + + # Replace the response content "you" from port 80. + kbcli fault network http replace --target=Response --body=you --duration=30s + + # AAppend content to the body of the post request sent from port 4399, in JSON format. + kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s +``` + +### Options + +``` + --body string The fault of the request body or response body with patch faults. + --code int32 The status code responded by target. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for patch + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The URI path of the target request. Supports Matching wildcards. (default "*") + --port int32 The TCP port that the target service listens on. (default 80) + --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") + --type string The type of patch faults of the request body or response body. Currently, it only supports JSON. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network http](kbcli_fault_network_http.md) - Intercept HTTP requests and responses. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_http_replace.md b/docs/user_docs/cli/kbcli_fault_network_http_replace.md new file mode 100644 index 000000000..9e3a7b6fb --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_http_replace.md @@ -0,0 +1,89 @@ +--- +title: kbcli fault network http replace +--- + +Replace the HTTP request and response. + +``` +kbcli fault network http replace [flags] +``` + +### Examples + +``` + # By default, the method of GET from port 80 is blocked. + kbcli fault network http abort --duration=1m + + # Block the method of GET from port 4399. + kbcli fault network http abort --port=4399 --duration=1m + + # Block the method of POST from port 4399. + kbcli fault network http abort --port=4399 --method=POST --duration=1m + + # Delays post requests from port 4399. + kbcli fault network http delay --port=4399 --method=POST --delay=15s + + # Replace the GET method sent from port 80 with the PUT method. + kbcli fault network http replace --replace-method=PUT --duration=1m + + # Replace the GET method sent from port 80 with the PUT method, and replace the request body. + kbcli fault network http replace --body="you are good luck" --replace-method=PUT --duration=2m + + # Replace the response content "you" from port 80. + kbcli fault network http replace --target=Response --body=you --duration=30s + + # AAppend content to the body of the post request sent from port 4399, in JSON format. + kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s +``` + +### Options + +``` + --body string The content of the request body or response body to replace the failure. + --code int32 The status code responded by target. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for replace + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The URI path of the target request. Supports Matching wildcards. (default "*") + --port int32 The TCP port that the target service listens on. (default 80) + --replace-method string The replaced content of the HTTP request method. + --replace-path string The URI path used to replace content. + --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network http](kbcli_fault_network_http.md) - Intercept HTTP requests and responses. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_loss.md b/docs/user_docs/cli/kbcli_fault_network_loss.md new file mode 100644 index 000000000..dbaaf543b --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_loss.md @@ -0,0 +1,97 @@ +--- +title: kbcli fault network loss +--- + +Cause pods to communicate with other objects to drop packets. + +``` +kbcli fault network loss [flags] +``` + +### Examples + +``` + # Isolate all pods network under the default namespace from the outside world, including the k8s internal network. + kbcli fault network partition + + # The specified pod is isolated from the k8s external network "kubeblocks.io". + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + + # Isolate the network between two pods. + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + + // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. + # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. + kbcli fault network loss --loss=50 + + # Block the specified pod communication, so that the packet loss rate is 50%. + kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + + kbcli fault network corrupt --corrupt=50 + + # Blocks specified pod communication with a 50% packet corruption rate. + kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + + kbcli fault network duplicate --duplicate=50 + + # Block specified pod communication so that the packet repetition rate is 50%. + kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + + kbcli fault network delay --latency=10s + + # Block the communication of the specified pod, causing its network delay for 10s. + kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s +``` + +### Options + +``` + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for loss + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --loss string Indicates the probability of a packet error occurring. Value range: [0, 100]. + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_network_partition.md b/docs/user_docs/cli/kbcli_fault_network_partition.md new file mode 100644 index 000000000..01e30ddb0 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_network_partition.md @@ -0,0 +1,95 @@ +--- +title: kbcli fault network partition +--- + +Make a pod network partitioned from other objects. + +``` +kbcli fault network partition [flags] +``` + +### Examples + +``` + # Isolate all pods network under the default namespace from the outside world, including the k8s internal network. + kbcli fault network partition + + # The specified pod is isolated from the k8s external network "kubeblocks.io". + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + + # Isolate the network between two pods. + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + + // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. + # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. + kbcli fault network loss --loss=50 + + # Block the specified pod communication, so that the packet loss rate is 50%. + kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + + kbcli fault network corrupt --corrupt=50 + + # Blocks specified pod communication with a 50% packet corruption rate. + kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + + kbcli fault network duplicate --duplicate=50 + + # Block specified pod communication so that the packet repetition rate is 50%. + kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + + kbcli fault network delay --latency=10s + + # Block the communication of the specified pod, causing its network delay for 10s. + kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s +``` + +### Options + +``` + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for partition + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault network](kbcli_fault_network.md) - Network chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/go.mod b/go.mod index 8a34de3a7..b9679c5a7 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/authzed/controller-idioms v0.7.0 github.com/bhmj/jsonslice v1.1.2 github.com/briandowns/spinner v1.23.0 + github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230423031423-0b31a519b502 github.com/clbanning/mxj/v2 v2.5.7 github.com/containerd/stargz-snapshotter/estargz v0.13.0 github.com/containers/common v0.49.1 @@ -117,6 +118,7 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect + github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect @@ -290,6 +292,7 @@ require ( github.com/prometheus/statsd_exporter v0.22.3 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc // indirect github.com/rivo/uniseg v0.4.3 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect github.com/rubenv/sql-migrate v1.2.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d // indirect diff --git a/go.sum b/go.sum index 2e29b28de..bce9c5796 100644 --- a/go.sum +++ b/go.sum @@ -313,6 +313,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= @@ -382,6 +384,7 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0Bsq github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bxcodec/faker v2.0.1+incompatible h1:P0KUpUw5w6WJXwrPfv35oc91i4d8nf40Nwln+M/+faA= github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f h1:tRk+aBit+q3oqnj/1mF5HHhP2yxJM2lSa0afOJxQ3nE= github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f/go.mod h1:uEyr4WpAH4hio6LFriaPkL938XnrvLpNPmQHBdrmbIE= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= @@ -401,6 +404,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= +github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230423031423-0b31a519b502 h1:dlu7F5rX2PA4laECDbFXwtDKktUK31lcC09wU70L3QY= +github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230423031423-0b31a519b502/go.mod h1:5qllHIhMkPEWjIimDum42JtMj0P1Tn9x91XUceuPNjY= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= @@ -1562,6 +1567,8 @@ github.com/replicatedhq/troubleshoot v0.57.0/go.mod h1:R5VdixzaBXfWLbP9mcLuZKs/b github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/internal/cli/cmd/cli.go b/internal/cli/cmd/cli.go index fa454cb57..d19a500b4 100644 --- a/internal/cli/cmd/cli.go +++ b/internal/cli/cmd/cli.go @@ -41,6 +41,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/cmd/clusterdefinition" "github.com/apecloud/kubeblocks/internal/cli/cmd/clusterversion" "github.com/apecloud/kubeblocks/internal/cli/cmd/dashboard" + "github.com/apecloud/kubeblocks/internal/cli/cmd/fault" "github.com/apecloud/kubeblocks/internal/cli/cmd/kubeblocks" "github.com/apecloud/kubeblocks/internal/cli/cmd/migration" "github.com/apecloud/kubeblocks/internal/cli/cmd/options" @@ -151,6 +152,7 @@ A Command Line Interface for KubeBlocks`, addon.NewAddonCmd(f, ioStreams), migration.NewMigrationCmd(f, ioStreams), plugin.NewPluginCmd(ioStreams), + fault.NewFaultCmd(f, ioStreams), ) filters := []string{"options"} diff --git a/internal/cli/cmd/fault/fault.go b/internal/cli/cmd/fault/fault.go new file mode 100644 index 000000000..2ee579a84 --- /dev/null +++ b/internal/cli/cmd/fault/fault.go @@ -0,0 +1,117 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +type FaultBaseOptions struct { + Action string `json:"action"` + + Mode string `json:"mode"` + + Value string `json:"value"` + + NamespaceSelector []string `json:"namespaceSelector"` + + Label map[string]string `json:"label"` + + Duration string `json:"duration"` +} + +func NewFaultCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "fault", + Short: "Inject faults to pod.", + } + cmd.AddCommand( + NewNetworkChaosCmd(f, streams), + ) + return cmd +} + +func registerFlagCompletionFunc(cmd *cobra.Command, f cmdutil.Factory) { + var formatsWithDesc = map[string]string{ + "JSON": "Output result in JSON format", + "YAML": "Output result in YAML format", + } + util.CheckErr(cmd.RegisterFlagCompletionFunc("output", + func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + var names []string + for format, desc := range formatsWithDesc { + if strings.HasPrefix(format, toComplete) { + names = append(names, fmt.Sprintf("%s\t%s", format, desc)) + } + } + return names, cobra.ShellCompDirectiveNoFileComp + })) +} + +func (o *FaultBaseOptions) BaseValidate() error { + if ok, err := IsRegularMatch(o.Duration); !ok { + return err + } + + if o.Value == "" && (o.Mode == "fixed" || o.Mode == "fixed-percent" || o.Mode == "random-max-percent") { + return fmt.Errorf("you must use --value to specify an integer") + } + + if ok, err := IsInteger(o.Value); !ok { + return err + } + + return nil +} + +func (o *FaultBaseOptions) BaseComplete() error { + return nil +} + +func IsRegularMatch(str string) (bool, error) { + pattern := regexp.MustCompile(`^\d+(ms|s|m|h)$`) + if str != "" && !pattern.MatchString(str) { + return false, fmt.Errorf("invalid duration:%s; input format must be in the form of number + time unit, like 10s, 10m", str) + } else { + return true, nil + } +} + +func IsInteger(str string) (bool, error) { + if _, err := strconv.Atoi(str); str != "" && err != nil { + return false, fmt.Errorf("invalid value:%s; must be an integer", str) + } else { + return true, nil + } +} + +func GetGVR(group, version, resourceName string) schema.GroupVersionResource { + return schema.GroupVersionResource{Group: group, Version: version, Resource: resourceName} +} diff --git a/internal/cli/cmd/fault/fault_constant.go b/internal/cli/cmd/fault/fault_constant.go new file mode 100644 index 000000000..05359993a --- /dev/null +++ b/internal/cli/cmd/fault/fault_constant.go @@ -0,0 +1,134 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +// Unchanged DryRun flag +const ( + Unchanged = "unchanged" +) + +// GVR +const ( + Group = "chaos-mesh.org" + Version = "v1alpha1" + + ResourcePodChaos = "podchaos" + ResourceNetworkChaos = "networkchaos" + ResourceIOChaos = "iochaos" + ResourceStressChaos = "stresschaos" + ResourceDNSChaos = "dnschaos" + ResourceTimeChaos = "timechaos" + ResourceHTTPChaos = "httpchaos" + ResourceAWSChaos = "awschaos" + ResourceGCPChaos = "gcpchaos" +) + +// Cue Template Name +const ( + CueTemplatePodChaos = "pod_chaos_template.cue" + CueTemplateNetworkChaos = "network_chaos_template.cue" + CueTemplateIOChaos = "io_chaos_template.cue" + CueTemplateStressChaos = "stress_chaos_template.cue" + CueTemplateDNSChaos = "dns_chaos_template.cue" + CueTemplateTimeChaos = "time_chaos_template.cue" + CueTemplateHTTPChaos = "http_chaos_template.cue" + CueTemplateAWSChaos = "aws_chaos_template.cue" + CueTemplateGCPChaos = "gcp_chaos_template.cue" +) + +// Pod Chaos Command +const ( + Kill = "kill" + KillShort = "kill pod" + Failure = "failure" + FailureShort = "failure pod" + KillContainer = "kill-container" + KillContainerShort = "kill containers" +) + +// NetWork Chaos Command +const ( + Partition = "partition" + PartitionShort = "Make a pod network partitioned from other objects." + Loss = "loss" + LossShort = "Cause pods to communicate with other objects to drop packets." + Delay = "delay" + DelayShort = "Make pods communicate with other objects lazily." + Duplicate = "duplicate" + DuplicateShort = "Make pods communicate with other objects to pick up duplicate packets." + Corrupt = "corrupt" + CorruptShort = "Distorts the messages a pod communicates with other objects." + Bandwidth = "bandwidth" + BandwidthShort = "Limit the bandwidth that pods use to communicate with other objects." +) + +// DNS Chaos Command +const ( + Random = "random" + RandomShort = "Make DNS return any IP when resolving external domain names." + Error = "error" + ErrorShort = "Make DNS return an error when resolving external domain names." +) + +// Network Chaos Command +const ( + Latency = "latency" + LatencyShort = "Delayed IO operations." + Fault = "fault" + FaultShort = "Causes IO operations to return specific errors." + Attribute = "attribute" + AttributeShort = "Override the attributes of the file." + Mistake = "mistake" + MistakeShort = "Alters the contents of the file, distorting the contents of the file." +) + +// Stress Chaos Command +const ( + Stress = "stress" + StressShort = "Add memory pressure or CPU load to the system." +) + +// Time Chaos Command +const ( + Time = "time" + TimeShort = "Clock skew failure." +) + +// HTTP Chaos Command +const ( + Abort = "abort" + AbortShort = "Abort the HTTP request and response." + HTTPDelay = "delay" + HTTPDelayShort = "Delay the HTTP request and response." + Replace = "replace" + ReplaceShort = "Replace the HTTP request and response." + Patch = "patch" + PatchShort = "Patch the HTTP request and response." +) + +// AWS And GCP Chaos Command +const ( + Stop = "stop" + StopShort = "Stop instance" + Restart = "restart" + RestartShort = "Restart instance" + DetachVolume = "detach-volume" + DetachVolumeShort = "Detach volume" +) diff --git a/internal/cli/cmd/fault/fault_dns.go b/internal/cli/cmd/fault/fault_dns.go new file mode 100644 index 000000000..822980d2a --- /dev/null +++ b/internal/cli/cmd/fault/fault_dns.go @@ -0,0 +1,155 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var faultDNSExample = templates.Examples(` + // Inject DNS faults into all pods under the default namespace, so that any IP is returned when accessing the baidu.com domain name. + kbcli fault DNS random --patterns=baidu.com --duration=1m + + // Inject DNS faults into all pods under the default namespace, so that error is returned when accessing the baidu.com domain name. + kbcli fault DNS error --patterns=baidu.com --duration=1m +`) + +type DNSChaosOptions struct { + Patterns []string `json:"patterns"` + + FaultBaseOptions + + create.CreateOptions `json:"-"` +} + +func NewDNSChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *DNSChaosOptions { + o := &DNSChaosOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateDNSChaos, + GVR: GetGVR(Group, Version, ResourceDNSChaos), + }, + FaultBaseOptions: FaultBaseOptions{Action: action}, + } + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o +} + +func NewDNSChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "dns", + Short: "Inject faults into DNS server.", + } + cmd.AddCommand( + NewRandomCmd(f, streams), + NewErrorCmd(f, streams), + ) + return cmd +} + +func NewRandomCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewDNSChaosOptions(f, streams, string(v1alpha1.RandomAction)) + cmd := o.NewCobraCommand(Random, RandomShort) + + o.AddCommonFlag(cmd) + util.CheckErr(cmd.MarkFlagRequired("patterns")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewErrorCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewDNSChaosOptions(f, streams, string(v1alpha1.ErrorAction)) + cmd := o.NewCobraCommand(Error, ErrorShort) + + o.AddCommonFlag(cmd) + util.CheckErr(cmd.MarkFlagRequired("patterns")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func (o *DNSChaosOptions) NewCobraCommand(use, short string) *cobra.Command { + return &cobra.Command{ + Use: use, + Short: short, + Example: faultDNSExample, + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run()) + }, + } +} + +func (o *DNSChaosOptions) AddCommonFlag(cmd *cobra.Command) { + cmd.Flags().StringVar(&o.Mode, "mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) + cmd.Flags().StringVar(&o.Value, "value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) + cmd.Flags().StringVar(&o.Duration, "duration", "10s", "Supported formats of the duration are: ms / s / m / h.") + cmd.Flags().StringToStringVar(&o.Label, "label", map[string]string{}, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) + cmd.Flags().StringArrayVar(&o.NamespaceSelector, "namespace-selector", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) + + cmd.Flags().StringArrayVar(&o.Patterns, "patterns", nil, `Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *.`) + + cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged + + printer.AddOutputFlagForCreate(cmd, &o.Format) +} + +func (o *DNSChaosOptions) Validate() error { + return o.BaseValidate() +} + +func (o *DNSChaosOptions) Complete() error { + return o.BaseComplete() +} + +func (o *DNSChaosOptions) PreCreate(obj *unstructured.Unstructured) error { + c := &v1alpha1.DNSChaos{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, c); err != nil { + return err + } + + data, e := runtime.DefaultUnstructuredConverter.ToUnstructured(c) + if e != nil { + return e + } + obj.SetUnstructuredContent(data) + return nil +} diff --git a/internal/cli/cmd/fault/fault_http.go b/internal/cli/cmd/fault/fault_http.go new file mode 100644 index 000000000..9509573af --- /dev/null +++ b/internal/cli/cmd/fault/fault_http.go @@ -0,0 +1,257 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "fmt" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/printer" +) + +var faultHTTPExample = templates.Examples(` + # By default, the method of GET from port 80 is blocked. + kbcli fault network http abort --duration=1m + + # Block the method of GET from port 4399. + kbcli fault network http abort --port=4399 --duration=1m + + # Block the method of POST from port 4399. + kbcli fault network http abort --port=4399 --method=POST --duration=1m + + # Delays post requests from port 4399. + kbcli fault network http delay --port=4399 --method=POST --delay=15s + + # Replace the GET method sent from port 80 with the PUT method. + kbcli fault network http replace --replace-method=PUT --duration=1m + + # Replace the GET method sent from port 80 with the PUT method, and replace the request body. + kbcli fault network http replace --body="you are good luck" --replace-method=PUT --duration=2m + + # Replace the response content "you" from port 80. + kbcli fault network http replace --target=Response --body=you --duration=30s + + # AAppend content to the body of the post request sent from port 4399, in JSON format. + kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s +`) + +type HTTPChaosOptions struct { + Target string `json:"target"` + Port int32 `json:"port"` + Path string `json:"path"` + Method string `json:"method"` + Code int32 `json:"code,omitempty"` + + // abort command + Abort bool `json:"abort,omitempty"` + // delay command + Delay string `json:"delay,omitempty"` + + // replace command + ReplaceBody []byte `json:"replaceBody,omitempty"` + InputReplaceBody string `json:"-"` + ReplacePath string `json:"replacePath,omitempty"` + ReplaceMethod string `json:"replaceMethod,omitempty"` + + // patch command + PatchBodyValue string `json:"patchBodyValue,omitempty"` + PatchBodyType string `json:"patchBodyType,omitempty"` + + FaultBaseOptions + + create.CreateOptions `json:"-"` +} + +func NewHTTPChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *HTTPChaosOptions { + o := &HTTPChaosOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateHTTPChaos, + GVR: GetGVR(Group, Version, ResourceHTTPChaos), + }, + FaultBaseOptions: FaultBaseOptions{Action: action}, + } + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o +} + +func NewHTTPChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "http", + Short: "Intercept HTTP requests and responses.", + } + cmd.AddCommand( + NewAbortCmd(f, streams), + NewHTTPDelayCmd(f, streams), + NewReplaceCmd(f, streams), + NewPatchCmd(f, streams), + ) + return cmd +} + +func NewAbortCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewHTTPChaosOptions(f, streams, "") + + cmd := o.NewCobraCommand(Abort, AbortShort) + + o.AddCommonFlag(cmd) + cmd.Flags().BoolVar(&o.Abort, "abort", true, `Indicates whether to inject the fault that interrupts the connection.`) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewHTTPDelayCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewHTTPChaosOptions(f, streams, "") + + cmd := o.NewCobraCommand(HTTPDelay, HTTPDelayShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Delay, "delay", "10s", `The time for delay.`) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewReplaceCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewHTTPChaosOptions(f, streams, "") + + cmd := o.NewCobraCommand(Replace, ReplaceShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.InputReplaceBody, "body", "", `The content of the request body or response body to replace the failure.`) + cmd.Flags().StringVar(&o.ReplacePath, "replace-path", "", `The URI path used to replace content.`) + cmd.Flags().StringVar(&o.ReplaceMethod, "replace-method", "", `The replaced content of the HTTP request method.`) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewPatchCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewHTTPChaosOptions(f, streams, "") + + cmd := o.NewCobraCommand(Patch, PatchShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.PatchBodyValue, "body", "", `The fault of the request body or response body with patch faults.`) + cmd.Flags().StringVar(&o.PatchBodyType, "type", "", `The type of patch faults of the request body or response body. Currently, it only supports JSON.`) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func (o *HTTPChaosOptions) NewCobraCommand(use, short string) *cobra.Command { + return &cobra.Command{ + Use: use, + Short: short, + Example: faultHTTPExample, + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run()) + }, + } +} + +func (o *HTTPChaosOptions) AddCommonFlag(cmd *cobra.Command) { + + cmd.Flags().StringVar(&o.Mode, "mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) + cmd.Flags().StringVar(&o.Value, "value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) + cmd.Flags().StringVar(&o.Duration, "duration", "10s", "Supported formats of the duration are: ms / s / m / h.") + cmd.Flags().StringToStringVar(&o.Label, "label", map[string]string{}, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) + cmd.Flags().StringArrayVar(&o.NamespaceSelector, "namespace-selector", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) + + cmd.Flags().StringVar(&o.Target, "target", "Request", `Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time.`) + cmd.Flags().Int32Var(&o.Port, "port", 80, `The TCP port that the target service listens on.`) + cmd.Flags().StringVar(&o.Path, "path", "*", `The URI path of the target request. Supports Matching wildcards.`) + cmd.Flags().StringVar(&o.Method, "method", "GET", `The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH.`) + cmd.Flags().Int32Var(&o.Code, "code", 0, `The status code responded by target.`) + + cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged + + printer.AddOutputFlagForCreate(cmd, &o.Format) +} + +func (o *HTTPChaosOptions) Validate() error { + if o.PatchBodyType != "" && o.PatchBodyType != "JSON" { + return fmt.Errorf("the --type only supports JSON") + } + if o.PatchBodyValue != "" && o.PatchBodyType == "" { + return fmt.Errorf("the --type is required when --body is specified") + } + if o.PatchBodyType != "" && o.PatchBodyValue == "" { + return fmt.Errorf("the --body is required when --type is specified") + } + + var msg interface{} + if o.PatchBodyValue != "" && json.Unmarshal([]byte(o.PatchBodyValue), &msg) != nil { + return fmt.Errorf("the --body is not a valid JSON") + } + + if o.Target == "Request" && o.Code != 0 { + return fmt.Errorf("the --code is only supported when --target is Response") + } + + if ok, err := IsRegularMatch(o.Delay); !ok { + return err + } + return o.BaseValidate() +} + +func (o *HTTPChaosOptions) Complete() error { + o.ReplaceBody = []byte(o.InputReplaceBody) + return o.BaseComplete() +} + +func (o *HTTPChaosOptions) PreCreate(obj *unstructured.Unstructured) error { + c := &v1alpha1.HTTPChaos{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, c); err != nil { + return err + } + + data, e := runtime.DefaultUnstructuredConverter.ToUnstructured(c) + if e != nil { + return e + } + obj.SetUnstructuredContent(data) + return nil +} diff --git a/internal/cli/cmd/fault/fault_network.go b/internal/cli/cmd/fault/fault_network.go new file mode 100644 index 000000000..592319228 --- /dev/null +++ b/internal/cli/cmd/fault/fault_network.go @@ -0,0 +1,335 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "fmt" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var faultNetWorkExample = templates.Examples(` + # Isolate all pods network under the default namespace from the outside world, including the k8s internal network. + kbcli fault network partition + + # The specified pod is isolated from the k8s external network "kubeblocks.io". + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + + # Isolate the network between two pods. + kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + + // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. + # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. + kbcli fault network loss --loss=50 + + # Block the specified pod communication, so that the packet loss rate is 50%. + kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + + kbcli fault network corrupt --corrupt=50 + + # Blocks specified pod communication with a 50% packet corruption rate. + kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + + kbcli fault network duplicate --duplicate=50 + + # Block specified pod communication so that the packet repetition rate is 50%. + kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + + kbcli fault network delay --latency=10s + + # Block the communication of the specified pod, causing its network delay for 10s. + kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s +`) + +type NetworkChaosOptions struct { + // Specify the network direction + Direction string `json:"direction"` + // Indicates a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + // such as "www.baidu.com". Only works with direction: to. + ExternalTargets []string `json:"externalTargets,omitempty"` + // Specifies the labels that target Pods come with. + TargetLabel map[string]string `json:"targetLabel,omitempty"` + // Specifies the namespaces to which target Pods belong. + TargetNamespaceSelector string `json:"targetNamespaceSelector"` + + TargetMode string `json:"targetMode"` + TargetValue string `json:"targetValue"` + + // The percentage of packet loss + Loss string `json:"loss,omitempty"` + // The percentage of packet corruption + Corrupt string `json:"corrupt,omitempty"` + // The percentage of packet duplication + Duplicate string `json:"duplicate,omitempty"` + // The latency of delay + Latency string `json:"latency,omitempty"` + // The jitter of delay + Jitter string `json:"jitter"` + + // The correlation of loss or corruption or duplication or delay + Correlation string `json:"correlation"` + + // Bandwidth command + Rate string `json:"rate,omitempty"` + Limit uint32 `json:"limit"` + Buffer uint32 `json:"buffer"` + Peakrate uint64 `json:"peakrate"` + Minburst uint32 `json:"minburst"` + + FaultBaseOptions + + create.CreateOptions `json:"-"` +} + +func NewNetworkChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *NetworkChaosOptions { + o := &NetworkChaosOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateNetworkChaos, + GVR: GetGVR(Group, Version, ResourceNetworkChaos), + }, + FaultBaseOptions: FaultBaseOptions{Action: action}, + } + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o +} + +func NewNetworkChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "network", + Short: "Network chaos.", + } + cmd.AddCommand( + NewPartitionCmd(f, streams), + NewLossCmd(f, streams), + NewDelayCmd(f, streams), + NewDuplicateCmd(f, streams), + NewCorruptCmd(f, streams), + NewBandwidthCmd(f, streams), + NewDNSChaosCmd(f, streams), + NewHTTPChaosCmd(f, streams), + ) + return cmd +} + +func NewPartitionCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNetworkChaosOptions(f, streams, string(v1alpha1.PartitionAction)) + + cmd := o.NewCobraCommand(Partition, PartitionShort) + + o.AddCommonFlag(cmd) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewLossCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNetworkChaosOptions(f, streams, string(v1alpha1.LossAction)) + + cmd := o.NewCobraCommand(Loss, LossShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Loss, "loss", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + + util.CheckErr(cmd.MarkFlagRequired("loss")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewDelayCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNetworkChaosOptions(f, streams, string(v1alpha1.DelayAction)) + + cmd := o.NewCobraCommand(Delay, DelayShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Latency, "latency", "", `the length of time to delay.`) + cmd.Flags().StringVar(&o.Jitter, "jitter", "0ms", `the variation range of the delay time.`) + cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) + + util.CheckErr(cmd.MarkFlagRequired("latency")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewDuplicateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNetworkChaosOptions(f, streams, string(v1alpha1.DuplicateAction)) + + cmd := o.NewCobraCommand(Duplicate, DuplicateShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Duplicate, "duplicate", "", `the probability of a packet being repeated. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + + util.CheckErr(cmd.MarkFlagRequired("duplicate")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewCorruptCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNetworkChaosOptions(f, streams, string(v1alpha1.CorruptAction)) + + cmd := o.NewCobraCommand(Corrupt, CorruptShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Corrupt, "corrupt", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + + util.CheckErr(cmd.MarkFlagRequired("corrupt")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewBandwidthCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNetworkChaosOptions(f, streams, string(v1alpha1.BandwidthAction)) + + cmd := o.NewCobraCommand(Bandwidth, BandwidthShort) + + o.AddCommonFlag(cmd) + + cmd.Flags().StringVar(&o.Rate, "rate", "", `the rate at which the bandwidth is limited. For example : 10 bps/kbps/mbps/gbps.`) + cmd.Flags().Uint32Var(&o.Limit, "limit", 1, `the number of bytes waiting in the queue.`) + cmd.Flags().Uint32Var(&o.Buffer, "buffer", 1, `the maximum number of bytes that can be sent instantaneously.`) + cmd.Flags().Uint64Var(&o.Peakrate, "peakrate", 0, `the maximum consumption rate of the bucket.`) + cmd.Flags().Uint32Var(&o.Minburst, "minburst", 0, `the size of the peakrate bucket.`) + + util.CheckErr(cmd.MarkFlagRequired("rate")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func (o *NetworkChaosOptions) NewCobraCommand(use, short string) *cobra.Command { + return &cobra.Command{ + Use: use, + Short: short, + Example: faultNetWorkExample, + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run()) + }, + } +} + +func (o *NetworkChaosOptions) AddCommonFlag(cmd *cobra.Command) { + + cmd.Flags().StringVar(&o.Mode, "mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) + cmd.Flags().StringVar(&o.Value, "value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) + cmd.Flags().StringVar(&o.Duration, "duration", "10s", "Supported formats of the duration are: ms / s / m / h.") + cmd.Flags().StringToStringVar(&o.Label, "label", map[string]string{}, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) + cmd.Flags().StringArrayVar(&o.NamespaceSelector, "namespace-selector", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) + + cmd.Flags().StringVar(&o.Direction, "direction", "to", `You can select "to"" or "from"" or "both"".`) + cmd.Flags().StringArrayVarP(&o.ExternalTargets, "external-targets", "e", nil, "a network target outside of Kubernetes, which can be an IPv4 address or a domain name,\n\t such as \"www.baidu.com\". Only works with direction: to.") + cmd.Flags().StringVar(&o.TargetMode, "target-mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) + cmd.Flags().StringVar(&o.TargetValue, "target-value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) + cmd.Flags().StringToStringVar(&o.TargetLabel, "target-label", nil, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) + cmd.Flags().StringVar(&o.TargetNamespaceSelector, "target-namespace-selector", "default", `Specifies the namespace into which you want to inject faults.`) + + cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged + + printer.AddOutputFlagForCreate(cmd, &o.Format) +} + +func (o *NetworkChaosOptions) Validate() error { + if o.TargetValue == "" && (o.TargetMode == "fixed" || o.TargetMode == "fixed-percent" || o.TargetMode == "random-max-percent") { + return fmt.Errorf("you must use --value to specify an integer") + } + + if ok, err := IsInteger(o.TargetValue); !ok { + return err + } + + if ok, err := IsInteger(o.Loss); !ok { + return err + } + + if ok, err := IsInteger(o.Corrupt); !ok { + return err + } + + if ok, err := IsInteger(o.Duplicate); !ok { + return err + } + + if ok, err := IsInteger(o.Correlation); !ok { + return err + } + + if ok, err := IsRegularMatch(o.Latency); !ok { + return err + } + + if ok, err := IsRegularMatch(o.Jitter); !ok { + return err + } + + return o.BaseValidate() +} + +func (o *NetworkChaosOptions) Complete() error { + return o.BaseComplete() +} + +func (o *NetworkChaosOptions) PreCreate(obj *unstructured.Unstructured) error { + c := &v1alpha1.NetworkChaos{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, c); err != nil { + return err + } + + data, e := runtime.DefaultUnstructuredConverter.ToUnstructured(c) + if e != nil { + return e + } + obj.SetUnstructuredContent(data) + return nil +} diff --git a/internal/cli/create/template/dns_chaos_template.cue b/internal/cli/create/template/dns_chaos_template.cue new file mode 100644 index 000000000..1ba654e0c --- /dev/null +++ b/internal/cli/create/template/dns_chaos_template.cue @@ -0,0 +1,56 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// required, command line input options for parameters and flags +options: { + namespace: string + action: string + + namespaceSelector: [...] + mode: string + value: string + duration: string + label?: {} + + patterns: [...] +} + +// required, k8s api resource content +content: { + kind: "DNSChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "dns-chaos-" + namespace: options.namespace + } + spec:{ + selector:{ + namespaces: options.namespaceSelector + if options.label != _|_ { + labelSelectors:{ + options.label + } + } + } + action: options.action + mode: options.mode + value: options.value + duration: options.duration + + patterns: options.patterns + } +} diff --git a/internal/cli/create/template/http_chaos_template.cue b/internal/cli/create/template/http_chaos_template.cue new file mode 100644 index 000000000..6b412e547 --- /dev/null +++ b/internal/cli/create/template/http_chaos_template.cue @@ -0,0 +1,103 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// required, command line input options for parameters and flags +options: { + namespace: string + label?: {} + namespaceSelector: [...] + mode: string + value: string + duration: string + + target: string + port: int32 + path: string + method: string + code?: int32 + + abort?: bool + delay?: string + + repalceBody?: bytes + replacePath?: string + replaceMethod?: string + + patchBodyValue?: string + patchBodyType?: string +} + +// required, k8s api resource content +content: { + kind: "HTTPChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "http-chaos-" + namespace: options.namespace + } + spec:{ + selector:{ + namespaces: options.namespaceSelector + if options.label != _|_ { + labelSelectors:{ + options.label + } + } + } + mode: options.mode + value: options.value + duration: options.duration + + target: options.target + port: options.port + path: options.path + method: options.method + if options.code != _|_ { + code: options.code + } + + if options.abort != _|_ { + abort: options.abort + } + if options.delay != _|_ { + delay: options.delay + } + if options.replaceBody != _|_ || options.replacePath != _|_ || options.replaceMethod != _|_{ + replace:{ + if options.replaceBody != _|_ { + body: options.replaceBody + } + if options.replacePath != _|_ { + path: options.replacePath + } + if options.replaceMethod != _|_ { + method: options.replaceMethod + } + } + } + if options.patchBodyValue != _|_ && options.patchBodyType != _|_ { + patch:{ + if options.patchBodyValue != _|_ && options.patchBodyType != _|_ { + body: { + value: options.patchBodyValue + type: options.patchBodyType + } + } + } + } + } +} diff --git a/internal/cli/create/template/network_chaos_template.cue b/internal/cli/create/template/network_chaos_template.cue new file mode 100644 index 000000000..622d08220 --- /dev/null +++ b/internal/cli/create/template/network_chaos_template.cue @@ -0,0 +1,127 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// required, command line input options for parameters and flags +options: { + namespace: string + action: string + + namespaceSelector: [...] + mode: string + value: string + duration: string + label?: {} + + direction: string + externalTargets?: [...] + + targetNamespaceSelector: string + targetMode: string + targetValue: string + targetLabel?: {} + + loss?: string + corrupt?: string + duplicate?: string + + latency?: string + jitter: string + + correlation: string + + rate?: string + limit: uint32 + buffer: uint32 + peakrate: uint32 + minburst: uint32 +} + +// required, k8s api resource content +content: { + kind: "NetworkChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "network-chaos-" + namespace: options.namespace + } + spec:{ + selector:{ + namespaces: options.namespaceSelector + if options.label != _|_ { + labelSelectors:{ + options.label + } + } + } + mode: options.mode + value: options.value + action: options.action + duration: options.duration + + direction: options.direction + if options.externalTargets != _|_ { + externalTargets: options.externalTargets + } + if options.targetLabel != _|_ { + target:{ + mode: options.targetMode + value: options.targetValue + selector:{ + namespaces: [options.targetNamespaceSelector] + labelSelectors:{ + options.targetLabel + } + } + } + } + if options.loss != _|_ { + loss:{ + loss: options.loss + correlation: options.correlation + } + } + if options.corrupt != _|_ { + corrupt:{ + corrupt: options.corrupt + correlation: options.correlation + } + } + if options.duplicate != _|_ { + duplicate:{ + duplicate: options.duplicate + correlation: options.correlation + } + } + if options.latency != _|_{ + delay:{ + latency: options.latency + jitter: options.jitter + correlation: options.correlation + } + } + if options.rate != _|_{ + bandwidth:{ + rate: options.rate + limit: options.limit + buffer: options.buffer + peakrate: options.peakrate + minburst: options.minburst + correlation: options.correlation + } + } + } +} From 71bee20f9a4dbfef86aeacc6e2c6ec23d76552ee Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Fri, 12 May 2023 14:02:58 +0800 Subject: [PATCH 285/439] feat: run preflight when install kubeblocks (#2873) Co-authored-by: kubeJocker --- .../user_docs/cli/kbcli_kubeblocks_install.md | 1 + .../cli/kbcli_kubeblocks_preflight.md | 7 +- externalapis/preflight/v1beta2/type.go | 21 +- go.mod | 4 +- internal/cli/cmd/kubeblocks/config_test.go | 2 +- .../cmd/kubeblocks/data/ack_preflight.yaml | 19 +- .../cmd/kubeblocks/data/eks_preflight.yaml | 17 +- .../cmd/kubeblocks/data/gke_preflight.yaml | 17 +- .../cmd/kubeblocks/data/tke_preflight.yaml | 17 +- internal/cli/cmd/kubeblocks/install.go | 16 +- internal/cli/cmd/kubeblocks/preflight.go | 82 +++++--- internal/cli/cmd/kubeblocks/preflight_test.go | 8 - internal/controllerutil/errors.go | 4 + internal/preflight/analyze.go | 12 +- internal/preflight/analyze_test.go | 2 +- internal/preflight/analyzer/analyze_result.go | 8 + internal/preflight/analyzer/analyzer.go | 9 +- internal/preflight/analyzer/anzlyzer_test.go | 8 +- .../preflight/analyzer/kb_storage_class.go | 7 +- internal/preflight/analyzer/kb_taint.go | 189 ++++++++++++++++++ internal/preflight/analyzer/kb_taint_test.go | 151 ++++++++++++++ internal/preflight/collect.go | 20 +- internal/preflight/collect_test.go | 2 +- internal/preflight/text_results.go | 26 ++- internal/preflight/text_results_test.go | 28 ++- 25 files changed, 565 insertions(+), 112 deletions(-) create mode 100644 internal/preflight/analyzer/kb_taint.go create mode 100644 internal/preflight/analyzer/kb_taint_test.go diff --git a/docs/user_docs/cli/kbcli_kubeblocks_install.md b/docs/user_docs/cli/kbcli_kubeblocks_install.md index 1b13816d2..dc6567058 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks_install.md +++ b/docs/user_docs/cli/kbcli_kubeblocks_install.md @@ -29,6 +29,7 @@ kbcli kubeblocks install [flags] ``` --check Check kubernetes environment before install (default true) --create-namespace Create the namespace if not present + --force If present, just print fail item and continue with the following steps -h, --help help for install --monitor Auto install monitoring add-ons including prometheus, grafana and alertmanager-webhook-adaptor (default true) --set stringArray Set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) diff --git a/docs/user_docs/cli/kbcli_kubeblocks_preflight.md b/docs/user_docs/cli/kbcli_kubeblocks_preflight.md index a79271ab8..fd7b0224f 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks_preflight.md +++ b/docs/user_docs/cli/kbcli_kubeblocks_preflight.md @@ -31,14 +31,17 @@ kbcli kubeblocks preflight [flags] --collector-image string the full name of the collector image to use --collector-pullpolicy string the pull policy of the collector image --debug enable debug logging - --format string output format, one of human, json, yaml. only used when interactive is set to false, default format is yaml (default "yaml") -h, --help help for preflight - --interactive interactive preflights, default value is false -n, --namespace string If present, the namespace scope for this CLI request -o, --output string specify the output file path for the preflight checks --selector string selector (label query) to filter remote collection nodes on. + --set stringArray Set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) + --set-file stringArray Set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2) + --set-json stringArray Set JSON values on the command line (can specify multiple or separate values with commas: key1=jsonval1,key2=jsonval2) + --set-string stringArray Set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) --since string force pod logs collectors to return logs newer than a relative duration like 5s, 2m, or 3h. --since-time string force pod logs collectors to return logs after a specific date (RFC3339) + -f, --values strings Specify values in a YAML file or a URL (can specify multiple) --verbose print more verbose logs, default value is false ``` diff --git a/externalapis/preflight/v1beta2/type.go b/externalapis/preflight/v1beta2/type.go index f759c2085..f091d34c0 100644 --- a/externalapis/preflight/v1beta2/type.go +++ b/externalapis/preflight/v1beta2/type.go @@ -19,7 +19,11 @@ along with this program. If not, see . package v1beta2 -import troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" +import ( + v1 "k8s.io/api/core/v1" + + troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" +) // ExtendCollect defines extended data collector for k8s cluster type ExtendCollect struct { @@ -41,6 +45,9 @@ type ExtendAnalyze struct { // StorageClass is to determine the correctness of target storage class // +optional StorageClass *KBStorageClassAnalyze `json:"storageClass,omitempty"` + // Taint is to Determine the matching between the taint and tolerance + // +optional + Taint *KBTaintAnalyze `json:"taint,omitempty"` } type HostUtility struct { @@ -105,6 +112,18 @@ type KBStorageClassAnalyze struct { Provisioner string `json:"provisioner,omitempty"` } +// KBTaintAnalyze matches the analysis of taints with TolerationsMap +type KBTaintAnalyze struct { + // analyzeMeta is defined in troubleshoot.sh + troubleshoot.AnalyzeMeta `json:",inline"` + // outcomes are expected user defined results. + // +kubebuilder:validation:Required + Outcomes []*troubleshoot.Outcome `json:"outcomes"` + // tolerations are tolerance configuration passed by kbcli + // +optional + TolerationsMap map[string][]v1.Toleration `json:"tolerations"` +} + type ExtendHostAnalyze struct { // hostUtility is to analyze the presence of target utility. // +optional diff --git a/go.mod b/go.mod index b9679c5a7..401a56d9d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/Masterminds/semver/v3 v3.2.0 github.com/Masterminds/sprig/v3 v3.2.3 github.com/StudioSol/set v1.0.0 - github.com/ahmetalpbalkan/go-cursor v0.0.0-20131010032410-8136607ea412 github.com/authzed/controller-idioms v0.7.0 github.com/bhmj/jsonslice v1.1.2 github.com/briandowns/spinner v1.23.0 @@ -87,6 +86,7 @@ require ( k8s.io/cli-runtime v0.26.1 k8s.io/client-go v0.26.1 k8s.io/component-base v0.26.1 + k8s.io/component-helpers v0.26.0 k8s.io/cri-api v0.25.0 k8s.io/klog/v2 v2.90.1 k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a @@ -119,6 +119,7 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect + github.com/ahmetalpbalkan/go-cursor v0.0.0-20131010032410-8136607ea412 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect @@ -370,7 +371,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 // indirect k8s.io/apiserver v0.26.1 // indirect - k8s.io/component-helpers v0.26.0 // indirect oras.land/oras-go v1.2.2 // indirect periph.io/x/host/v3 v3.8.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/internal/cli/cmd/kubeblocks/config_test.go b/internal/cli/cmd/kubeblocks/config_test.go index 915d98f46..5c8a6a83c 100644 --- a/internal/cli/cmd/kubeblocks/config_test.go +++ b/internal/cli/cmd/kubeblocks/config_test.go @@ -106,7 +106,7 @@ var _ = Describe("backupconfig", func() { } cmd := NewConfigCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) - Expect(o.Install()).Should(Succeed()) + Expect(o.PrecheckBeforeInstall()).Should(Succeed()) }) It("run describe config cmd", func() { diff --git a/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml b/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml index 5a233595e..71b5cbdec 100644 --- a/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml @@ -7,7 +7,7 @@ spec: - clusterInfo: {} analyzers: - clusterVersion: - checkName: GKE-Version + checkName: ACK-Version outcomes: - fail: when: "< 1.22.0" @@ -25,6 +25,13 @@ spec: message: This application requires at least 3 nodes - pass: message: This cluster has enough nodes. + - storageClass: + checkName: Required-Default-SC + outcomes: + - fail: + message: The default storage class was not found. To learn more details, please check https://help.aliyun.com/document_detail/189288.html. + - pass: + message: default storage class is the presence, and all good on storage classes extendAnalyzers: - clusterAccess: checkName: Check-K8S-Access @@ -33,12 +40,10 @@ spec: message: k8s cluster access fail - pass: message: k8s cluster access ok - - storageClass: - checkName: Required-Cloud-SSD-SC - storageClassType: "cloud_ssd" - provisioner: "diskplugin.csi.alibabacloud.com" + - taint: + checkName: Required-Taint-Match outcomes: - fail: - message: The cloud_ssd storage class was not found + message: The taint matching failed. - pass: - message: cloud_ssd is the presence, and all good on storage classes \ No newline at end of file + message: The taint matching succeeded. \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml b/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml index 35b6324bc..f2e710d6e 100644 --- a/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml @@ -25,6 +25,13 @@ spec: message: This application requires at least 3 nodes - pass: message: This cluster has enough nodes. + - storageClass: + checkName: Required-Default-SC + outcomes: + - fail: + message: The default storage class was not found. To learn more details, please check https://docs.aws.amazon.com/zh_cn/eks/latest/userguide/storage-classes.html. + - pass: + message: default storage class is the presence, and all good on storage classes - deploymentStatus: checkName: AWS-Load-Balancer-Check name: aws-load-balancer-controller @@ -49,12 +56,10 @@ spec: message: k8s cluster access fail - pass: message: k8s cluster access ok - - storageClass: - checkName: Required-GP3-SC - storageClassType: "gp3" - provisioner: "ebs.csi.aws.com" + - taint: + checkName: Required-Taint-Match outcomes: - fail: - message: The gp3 storage class was not found + message: The taint matching failed. - pass: - message: gp3 is the presence, and all good on storage classes \ No newline at end of file + message: The taint matching succeeded. \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml b/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml index 1b47740c6..ef55b78fb 100644 --- a/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml @@ -25,6 +25,13 @@ spec: message: This application requires at least 3 nodes - pass: message: This cluster has enough nodes. + - storageClass: + checkName: Required-Default-SC + outcomes: + - fail: + message: The default storage class was not found. To learn more details, please check https://docs.aws.amazon.com/zh_cn/eks/latest/userguide/storage-classes.html. + - pass: + message: default storage class is the presence, and all good on storage classes extendAnalyzers: - clusterAccess: checkName: Check-K8S-Access @@ -33,12 +40,10 @@ spec: message: k8s cluster access fail - pass: message: k8s cluster access ok - - storageClass: - checkName: Required-PREMIUM-SC - storageClassType: "pd-ssd" - provisioner: "pd.csi.storage.gke.io" + - taint: + checkName: Required-Taint-Match outcomes: - fail: - message: The premium storage class was not found + message: The taint matching failed. - pass: - message: premium is the presence, and all good on storage classes \ No newline at end of file + message: The taint matching succeeded. \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml b/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml index 95519e753..d5dee2393 100644 --- a/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml @@ -25,6 +25,13 @@ spec: message: This application requires at least 3 nodes - pass: message: This cluster has enough nodes. + - storageClass: + checkName: Required-Default-SC + outcomes: + - fail: + message: The default storage class was not found. To learn more details, please check https://cloud.tencent.com/document/product/457/44235. + - pass: + message: default storage class is the presence, and all good on storage classes extendAnalyzers: - clusterAccess: checkName: Check-K8S-Access @@ -33,12 +40,10 @@ spec: message: k8s cluster access fail - pass: message: k8s cluster access ok - - storageClass: - checkName: Required-CBS-SC - storageClassType: "cbs" - provisioner: "com.tencent.cloud.csi.cbs" + - taint: + checkName: Required-Taint-Match outcomes: - fail: - message: The cbs storage class was not found + message: The taint matching failed. - pass: - message: cbs is the presence, and all good on storage classes \ No newline at end of file + message: The taint matching succeeded. \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index 0be74c520..171f44e9e 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -28,6 +28,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/replicatedhq/troubleshoot/pkg/preflight" "github.com/spf13/cobra" "github.com/spf13/pflag" "helm.sh/helm/v3/pkg/cli/values" @@ -105,6 +106,12 @@ func newInstallCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr }, } + p := &PreflightOptions{ + PreflightFlags: preflight.NewPreflightFlags(), + IOStreams: streams, + } + *p.Interactive = false + cmd := &cobra.Command{ Use: "install", Short: "Install KubeBlocks.", @@ -112,6 +119,8 @@ func newInstallCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr Example: installExample, Run: func(cmd *cobra.Command, args []string) { util.CheckErr(o.Complete(f, cmd)) + util.CheckErr(o.PrecheckBeforeInstall()) + util.CheckErr(p.Preflight(f, args, o.ValueOpts)) util.CheckErr(o.Install()) }, } @@ -122,6 +131,7 @@ func newInstallCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr cmd.Flags().BoolVar(&o.Check, "check", true, "Check kubernetes environment before install") cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for installing KubeBlocks, such as --timeout=10m") cmd.Flags().BoolVar(&o.Wait, "wait", true, "Wait for KubeBlocks to be ready, including all the auto installed add-ons. It will wait for as long as --timeout") + cmd.Flags().BoolVar(&p.force, flagForce, p.force, "If present, just print fail item and continue with the following steps") helm.AddValueOptionsFlags(cmd.Flags(), &o.ValueOpts) return cmd @@ -167,7 +177,7 @@ func (o *Options) Complete(f cmdutil.Factory, cmd *cobra.Command) error { return err } -func (o *InstallOptions) Install() error { +func (o *InstallOptions) PrecheckBeforeInstall() error { // check if KubeBlocks has been installed v, err := util.GetVersionInfo(o.Client) if err != nil { @@ -197,7 +207,11 @@ func (o *InstallOptions) Install() error { if err = o.preCheck(v); err != nil { return err } + return nil +} +func (o *InstallOptions) Install() error { + var err error // add monitor parameters o.ValueOpts.Values = append(o.ValueOpts.Values, fmt.Sprintf(kMonitorParam, o.Monitor)) diff --git a/internal/cli/cmd/kubeblocks/preflight.go b/internal/cli/cmd/kubeblocks/preflight.go index 88693972d..6b58af59f 100644 --- a/internal/cli/cmd/kubeblocks/preflight.go +++ b/internal/cli/cmd/kubeblocks/preflight.go @@ -27,13 +27,13 @@ import ( "os/signal" "strings" - "github.com/ahmetalpbalkan/go-cursor" "github.com/fatih/color" "github.com/pkg/errors" analyze "github.com/replicatedhq/troubleshoot/pkg/analyze" "github.com/replicatedhq/troubleshoot/pkg/preflight" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" + "helm.sh/helm/v3/pkg/cli/values" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/rest" cmdutil "k8s.io/kubectl/pkg/cmd/util" @@ -41,13 +41,12 @@ import ( preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" "github.com/apecloud/kubeblocks/internal/cli/util" + "github.com/apecloud/kubeblocks/internal/cli/util/helm" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" kbpreflight "github.com/apecloud/kubeblocks/internal/preflight" - kbinteractive "github.com/apecloud/kubeblocks/internal/preflight/interactive" ) const ( - flagInteractive = "interactive" - flagFormat = "format" flagCollectorImage = "collector-image" flagCollectorPullPolicy = "collector-pullpolicy" flagCollectWithoutPermissions = "collect-without-permissions" @@ -58,9 +57,11 @@ const ( flagDebug = "debug" flagNamespace = "namespace" flagVerbose = "verbose" + flagForce = "force" PreflightPattern = "data/%s_preflight.yaml" HostPreflightPattern = "data/%s_hostpreflight.yaml" + PreflightMessage = "Run a preflight to check that the environment meets the requirement for KubeBlocks. It takes 10~20 seconds." ) var ( @@ -89,6 +90,8 @@ type PreflightOptions struct { checkYamlData [][]byte namespace string verbose bool + force bool + ValueOpts values.Options } func NewPreflightCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { @@ -103,13 +106,10 @@ func NewPreflightCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *co Example: preflightExample, Run: func(cmd *cobra.Command, args []string) { util.CheckErr(p.complete(f, args)) - util.CheckErr(p.validate()) util.CheckErr(p.run()) }, } // add flags - cmd.Flags().BoolVar(p.Interactive, flagInteractive, false, "interactive preflights, default value is false") - cmd.Flags().StringVar(p.Format, flagFormat, "yaml", "output format, one of human, json, yaml. only used when interactive is set to false, default format is yaml") cmd.Flags().StringVar(p.CollectorImage, flagCollectorImage, *p.CollectorImage, "the full name of the collector image to use") cmd.Flags().StringVar(p.CollectorPullPolicy, flagCollectorPullPolicy, *p.CollectorPullPolicy, "the pull policy of the collector image") cmd.Flags().BoolVar(p.CollectWithoutPermissions, flagCollectWithoutPermissions, *p.CollectWithoutPermissions, "always run preflight checks even if some require permissions that preflight does not have") @@ -120,6 +120,7 @@ func NewPreflightCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *co cmd.Flags().BoolVar(p.Debug, flagDebug, *p.Debug, "enable debug logging") cmd.Flags().StringVarP(&p.namespace, flagNamespace, "n", "", "If present, the namespace scope for this CLI request") cmd.Flags().BoolVar(&p.verbose, flagVerbose, p.verbose, "print more verbose logs, default value is false") + helm.AddValueOptionsFlags(cmd.Flags(), &p.ValueOpts) return cmd } @@ -137,30 +138,56 @@ func LoadVendorCheckYaml(vendorName util.K8sProvider) ([][]byte, error) { return yamlDataList, nil } +func (p *PreflightOptions) Preflight(f cmdutil.Factory, args []string, opts values.Options) error { + // if force flag set, skip preflight + if p.force { + return nil + } + p.ValueOpts = opts + *p.Format = "yaml" + + var err error + if err = p.complete(f, args); err != nil { + if intctrlutil.IsTargetError(err, intctrlutil.ErrorTypeSkipPreflight) { + return nil + } + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, err.Error()) + } + if err = p.run(); err != nil { + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, err.Error()) + } + return nil +} + func (p *PreflightOptions) complete(f cmdutil.Factory, args []string) error { // default no args, and run default validating vendor if len(args) == 0 { clientSet, err := f.KubernetesClientSet() if err != nil { - return errors.New("init k8s client failed, and please check kubeconfig") + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, "init k8s client failed, and please check kubeconfig") } versionInfo, err := util.GetVersionInfo(clientSet) if err != nil { - return errors.New("get k8s version of server failed, and please check your k8s accessibility") + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, "get k8s version of server failed, and please check your k8s accessibility") } vendorName, err := util.GetK8sProvider(versionInfo.Kubernetes, clientSet) if err != nil { - return errors.New("get k8s cloud provider failed, and please check your k8s accessibility") + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, "get k8s cloud provider failed, and please check your k8s accessibility") } p.checkYamlData, err = LoadVendorCheckYaml(vendorName) if err != nil { - return err + return intctrlutil.NewError(intctrlutil.ErrorTypeSkipPreflight, err.Error()) } - color.New(color.FgCyan).Printf("current provider %s. collecting and analyzing data will take 10-20 seconds... \n", vendorName) + color.New(color.FgCyan).Println(PreflightMessage) } else { p.checkFileList = args - color.New(color.FgCyan).Println("collecting and analyzing data will take 10-20 seconds...") + color.New(color.FgCyan).Println(PreflightMessage) + } + if len(p.checkFileList) < 1 && len(p.checkYamlData) < 1 { + return intctrlutil.NewError(intctrlutil.ErrorTypeSkipPreflight, "must specify at least one checks yaml") } + + p.factory = f // conceal warning logs rest.SetDefaultWarningHandler(rest.NoWarnings{}) go func() { @@ -172,13 +199,6 @@ func (p *PreflightOptions) complete(f cmdutil.Factory, args []string) error { return nil } -func (p *PreflightOptions) validate() error { - if len(p.checkFileList) < 1 && len(p.checkYamlData) < 1 { - return fmt.Errorf("must specify at least one checks yaml") - } - return nil -} - func (p *PreflightOptions) run() error { var ( kbPreflight *preflightv1beta2.Preflight @@ -188,10 +208,6 @@ func (p *PreflightOptions) run() error { preflightName string err error ) - if *p.Interactive { - fmt.Print(cursor.Hide()) - defer fmt.Print(cursor.Show()) - } // set progress chan progressCh := make(chan interface{}) defer close(progressCh) @@ -202,12 +218,12 @@ func (p *PreflightOptions) run() error { progressCollections.Go(CollectProgress(ctx, progressCh, p.verbose)) // 1. load yaml if kbPreflight, kbHostPreflight, preflightName, err = kbpreflight.LoadPreflightSpec(p.checkFileList, p.checkYamlData); err != nil { - return err + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, err.Error()) } // 2. collect data - collectResults, err = kbpreflight.CollectPreflight(p.factory, ctx, kbPreflight, kbHostPreflight, progressCh) + collectResults, err = kbpreflight.CollectPreflight(p.factory, &p.ValueOpts, ctx, kbPreflight, kbHostPreflight, progressCh) if err != nil { - return err + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, err.Error()) } // 3. analyze data for _, res := range collectResults { @@ -215,17 +231,17 @@ func (p *PreflightOptions) run() error { } cancelFunc() if err := progressCollections.Wait(); err != nil { - return err + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, err.Error()) } // 4. display analyzed data if len(analyzeResults) == 0 { - return errors.New("no data has been collected") + fmt.Fprintln(p.Out, "no data has been collected") + return nil } - if *p.Interactive { - return kbinteractive.ShowInteractiveResults(preflightName, analyzeResults, *p.Output) - } else { - return kbpreflight.ShowTextResults(preflightName, analyzeResults, *p.Format, p.verbose) + if err = kbpreflight.ShowTextResults(preflightName, analyzeResults, *p.Format, p.verbose); err != nil { + return intctrlutil.NewError(intctrlutil.ErrorTypePreflightCommon, err.Error()) } + return nil } func CollectProgress(ctx context.Context, progressCh <-chan interface{}, verbose bool) func() error { diff --git a/internal/cli/cmd/kubeblocks/preflight_test.go b/internal/cli/cmd/kubeblocks/preflight_test.go index b60f3f036..cd9577194 100644 --- a/internal/cli/cmd/kubeblocks/preflight_test.go +++ b/internal/cli/cmd/kubeblocks/preflight_test.go @@ -93,10 +93,8 @@ var _ = Describe("Preflight API Test", func() { PreflightFlags: preflight.NewPreflightFlags(), } Expect(p.complete(tf, nil)).Should(HaveOccurred()) - Expect(p.validate()).Should(HaveOccurred()) Expect(p.complete(tf, []string{"file1", "file2"})).Should(Succeed()) Expect(len(p.checkFileList)).Should(Equal(2)) - Expect(p.validate()).Should(Succeed()) }) It("run test", func() { @@ -113,12 +111,6 @@ var _ = Describe("Preflight API Test", func() { err := p.run() g.Expect(err).NotTo(HaveOccurred()) }).Should(Succeed()) - By("non-interactive mode, and expect error") - p.checkFileList = []string{"../../testing/testdata/hostpreflight_nil.yaml"} - Eventually(func(g Gomega) { - err := p.run() - g.Expect(err).To(HaveOccurred()) - }).Should(Succeed()) }) It("LoadVendorCheckYaml test, and expect fail", func() { diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index a35d09f9d..2b93e14e5 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -56,6 +56,10 @@ const ( // ErrorType for cluster controller ErrorTypeBackupFailed ErrorType = "BackupFailed" ErrorTypeNeedWaiting ErrorType = "NeedWaiting" // waiting for next reconcile + + // ErrorType for preflight + ErrorTypePreflightCommon = "PreflightCommon" + ErrorTypeSkipPreflight = "SkipPreflight" ) var ErrFailedToAddFinalizer = errors.New("failed to add finalizer") diff --git a/internal/preflight/analyze.go b/internal/preflight/analyze.go index b0a461ca1..d6bab2cd8 100644 --- a/internal/preflight/analyze.go +++ b/internal/preflight/analyze.go @@ -28,6 +28,7 @@ import ( analyze "github.com/replicatedhq/troubleshoot/pkg/analyze" troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" "github.com/replicatedhq/troubleshoot/pkg/preflight" + "helm.sh/helm/v3/pkg/cli/values" preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" kbanalyzer "github.com/apecloud/kubeblocks/internal/preflight/analyzer" @@ -35,6 +36,7 @@ import ( type KBClusterCollectResult struct { preflight.ClusterCollectResult + HelmOptions *values.Options AnalyzerSpecs []*troubleshoot.Analyze KbAnalyzerSpecs []*preflightv1beta2.ExtendAnalyze } @@ -46,19 +48,19 @@ type KBHostCollectResult struct { } func (c KBClusterCollectResult) Analyze() []*analyze.AnalyzeResult { - return doAnalyze(c.Context, c.AllCollectedData, c.AnalyzerSpecs, c.KbAnalyzerSpecs, nil, nil) + return doAnalyze(c.Context, c.AllCollectedData, c.AnalyzerSpecs, c.KbAnalyzerSpecs, nil, nil, c.HelmOptions) } func (c KBHostCollectResult) Analyze() []*analyze.AnalyzeResult { - return doAnalyze(c.Context, c.AllCollectedData, nil, nil, c.AnalyzerSpecs, c.KbAnalyzerSpecs) + return doAnalyze(c.Context, c.AllCollectedData, nil, nil, c.AnalyzerSpecs, c.KbAnalyzerSpecs, nil) } -func doAnalyze(ctx context.Context, - allCollectedData map[string][]byte, +func doAnalyze(ctx context.Context, allCollectedData map[string][]byte, analyzers []*troubleshoot.Analyze, kbAnalyzers []*preflightv1beta2.ExtendAnalyze, hostAnalyzers []*troubleshoot.HostAnalyze, kbhHostAnalyzers []*preflightv1beta2.ExtendHostAnalyze, + options *values.Options, ) []*analyze.AnalyzeResult { getCollectedFileContents := func(fileName string) ([]byte, error) { contents, ok := allCollectedData[fileName] @@ -102,7 +104,7 @@ func doAnalyze(ctx context.Context, } } for _, kbAnalyzer := range kbAnalyzers { - analyzeResult := kbanalyzer.KBAnalyze(ctx, kbAnalyzer, getCollectedFileContents, getChildCollectedFileContents) + analyzeResult := kbanalyzer.KBAnalyze(ctx, kbAnalyzer, getCollectedFileContents, getChildCollectedFileContents, options) analyzeResults = append(analyzeResults, analyzeResult...) } for _, hostAnalyzer := range hostAnalyzers { diff --git a/internal/preflight/analyze_test.go b/internal/preflight/analyze_test.go index 35def024e..7ff35e53c 100644 --- a/internal/preflight/analyze_test.go +++ b/internal/preflight/analyze_test.go @@ -52,7 +52,7 @@ var _ = Describe("analyze_test", func() { It("doAnalyze test, and expect success", func() { Eventually(func(g Gomega) { - analyzeList := doAnalyze(ctx, allCollectedData, analyzers, kbAnalyzers, hostAnalyzers, kbhHostAnalyzers) + analyzeList := doAnalyze(ctx, allCollectedData, analyzers, kbAnalyzers, hostAnalyzers, kbhHostAnalyzers, nil) g.Expect(len(analyzeList)).Should(Equal(4)) g.Expect(analyzeList[0].IsPass).Should(Equal(true)) g.Expect(analyzeList[1].IsFail).Should(Equal(true)) diff --git a/internal/preflight/analyzer/analyze_result.go b/internal/preflight/analyzer/analyze_result.go index e8cde05dd..1d418c2e4 100644 --- a/internal/preflight/analyzer/analyze_result.go +++ b/internal/preflight/analyzer/analyze_result.go @@ -24,6 +24,14 @@ import ( troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" ) +const ( + MissingOutcomeMessage = "there is a missing outcome message" + IncorrectOutcomeType = "there is an incorrect outcome type" + PassType = "Pass" + WarnType = "Warn" + FailType = "Fail" +) + func newAnalyzeResult(title string, resultType string, outcomes []*troubleshoot.Outcome) *analyze.AnalyzeResult { for _, outcome := range outcomes { if outcome == nil { diff --git a/internal/preflight/analyzer/analyzer.go b/internal/preflight/analyzer/analyzer.go index 7d8a47c9c..79b95e63d 100644 --- a/internal/preflight/analyzer/analyzer.go +++ b/internal/preflight/analyzer/analyzer.go @@ -25,6 +25,7 @@ import ( "github.com/pkg/errors" analyze "github.com/replicatedhq/troubleshoot/pkg/analyze" + "helm.sh/helm/v3/pkg/cli/values" preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" ) @@ -38,19 +39,21 @@ type KBAnalyzer interface { type GetCollectedFileContents func(string) ([]byte, error) type GetChildCollectedFileContents func(string, []string) (map[string][]byte, error) -func GetAnalyzer(analyzer *preflightv1beta2.ExtendAnalyze) (KBAnalyzer, bool) { +func GetAnalyzer(analyzer *preflightv1beta2.ExtendAnalyze, options *values.Options) (KBAnalyzer, bool) { switch { case analyzer.ClusterAccess != nil: return &AnalyzeClusterAccess{analyzer: analyzer.ClusterAccess}, true case analyzer.StorageClass != nil: return &AnalyzeStorageClassByKb{analyzer: analyzer.StorageClass}, true + case analyzer.Taint != nil: + return &AnalyzeTaintClassByKb{analyzer: analyzer.Taint, HelmOpts: options}, true default: return nil, false } } -func KBAnalyze(ctx context.Context, kbAnalyzer *preflightv1beta2.ExtendAnalyze, getFile func(string) ([]byte, error), findFiles func(string, []string) (map[string][]byte, error)) []*analyze.AnalyzeResult { - analyzer, ok := GetAnalyzer(kbAnalyzer) +func KBAnalyze(ctx context.Context, kbAnalyzer *preflightv1beta2.ExtendAnalyze, getFile func(string) ([]byte, error), findFiles func(string, []string) (map[string][]byte, error), options *values.Options) []*analyze.AnalyzeResult { + analyzer, ok := GetAnalyzer(kbAnalyzer, options) if !ok { return NewAnalyzeResultError(analyzer, errors.New("invalid analyzer")) } diff --git a/internal/preflight/analyzer/anzlyzer_test.go b/internal/preflight/analyzer/anzlyzer_test.go index 51f7ab3a0..ba747d3ea 100644 --- a/internal/preflight/analyzer/anzlyzer_test.go +++ b/internal/preflight/analyzer/anzlyzer_test.go @@ -39,7 +39,7 @@ var _ = Describe("analyzer_test", func() { Context("KBAnalyze test", func() { It("KBAnalyze test, and ExtendAnalyze is nil", func() { Eventually(func(g Gomega) { - res := KBAnalyze(context.TODO(), &preflightv1beta2.ExtendAnalyze{}, nil, nil) + res := KBAnalyze(context.TODO(), &preflightv1beta2.ExtendAnalyze{}, nil, nil, nil) g.Expect(res[0].IsFail).Should(BeTrue()) }).Should(Succeed()) }) @@ -77,7 +77,7 @@ var _ = Describe("analyzer_test", func() { getCollectedFileContents := func(string) ([]byte, error) { return b, nil } - res := KBAnalyze(context.TODO(), kbAnalyzer, getCollectedFileContents, nil) + res := KBAnalyze(context.TODO(), kbAnalyzer, getCollectedFileContents, nil, nil) Expect(len(res)).Should(Equal(1)) g.Expect(res[0].IsPass).Should(BeTrue()) }).Should(Succeed()) @@ -125,10 +125,10 @@ var _ = Describe("analyzer_test", func() { It("GetAnalyzer test, and expect success", func() { Eventually(func(g Gomega) { - collector, ok := GetAnalyzer(&preflightv1beta2.ExtendAnalyze{ClusterAccess: &preflightv1beta2.ClusterAccessAnalyze{}}) + collector, ok := GetAnalyzer(&preflightv1beta2.ExtendAnalyze{ClusterAccess: &preflightv1beta2.ClusterAccessAnalyze{}}, nil) g.Expect(collector).ShouldNot(BeNil()) g.Expect(ok).Should(BeTrue()) - collector, ok = GetAnalyzer(&preflightv1beta2.ExtendAnalyze{}) + collector, ok = GetAnalyzer(&preflightv1beta2.ExtendAnalyze{}, nil) g.Expect(collector).Should(BeNil()) g.Expect(ok).Should(BeFalse()) }).Should(Succeed()) diff --git a/internal/preflight/analyzer/kb_storage_class.go b/internal/preflight/analyzer/kb_storage_class.go index af602d537..f336f3b8d 100644 --- a/internal/preflight/analyzer/kb_storage_class.go +++ b/internal/preflight/analyzer/kb_storage_class.go @@ -31,12 +31,7 @@ import ( ) const ( - StorageClassPath = "cluster-resources/storage-classes.json" - MissingOutcomeMessage = "there is a missing outcome message" - IncorrectOutcomeType = "there is an incorrect outcome type" - PassType = "Pass" - WarnType = "Warn" - FailType = "Fail" + StorageClassPath = "cluster-resources/storage-classes.json" ) type AnalyzeStorageClassByKb struct { diff --git a/internal/preflight/analyzer/kb_taint.go b/internal/preflight/analyzer/kb_taint.go new file mode 100644 index 000000000..f9755a313 --- /dev/null +++ b/internal/preflight/analyzer/kb_taint.go @@ -0,0 +1,189 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package analyzer + +import ( + "encoding/json" + "fmt" + "strings" + + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/getter" + + analyze "github.com/replicatedhq/troubleshoot/pkg/analyze" + "helm.sh/helm/v3/pkg/cli/values" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + v1helper "k8s.io/component-helpers/scheduling/corev1" + + preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" + "github.com/apecloud/kubeblocks/internal/preflight/util" +) + +const ( + NodesPath = "cluster-resources/nodes.json" + Tolerations = "tolerations" + KubeBlocks = "kubeblocks" +) + +type AnalyzeTaintClassByKb struct { + analyzer *preflightv1beta2.KBTaintAnalyze + HelmOpts *values.Options +} + +func (a *AnalyzeTaintClassByKb) Title() string { + return util.TitleOrDefault(a.analyzer.AnalyzeMeta, "Kubeblocks Taints") +} + +func (a *AnalyzeTaintClassByKb) GetAnalyzer() *preflightv1beta2.KBTaintAnalyze { + return a.analyzer +} + +func (a *AnalyzeTaintClassByKb) IsExcluded() (bool, error) { + return util.IsExcluded(a.analyzer.Exclude) +} + +func (a *AnalyzeTaintClassByKb) Analyze(getFile GetCollectedFileContents, findFiles GetChildCollectedFileContents) ([]*analyze.AnalyzeResult, error) { + result, err := a.analyzeTaint(getFile, findFiles) + if err != nil { + return []*analyze.AnalyzeResult{result}, err + } + result.Strict = a.analyzer.Strict.BoolOrDefaultFalse() + return []*analyze.AnalyzeResult{result}, nil +} + +func (a *AnalyzeTaintClassByKb) analyzeTaint(getFile GetCollectedFileContents, findFiles GetChildCollectedFileContents) (*analyze.AnalyzeResult, error) { + if a.HelmOpts == nil { + return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil + } + nodesData, err := getFile(NodesPath) + if err != nil { + return newFailedResultWithMessage(a.Title(), fmt.Sprintf("get jsonfile failed, err:%v", err)), err + } + var nodes v1.NodeList + if err = json.Unmarshal(nodesData, &nodes); err != nil { + return newFailedResultWithMessage(a.Title(), fmt.Sprintf("get jsonfile failed, err:%v", err)), err + } + err = a.generateTolerations() + if err != nil { + return newFailedResultWithMessage(a.Title(), fmt.Sprintf("get tolerations failed, err:%v", err)), err + } + return a.doAnalyzeTaint(nodes) +} + +func (a *AnalyzeTaintClassByKb) doAnalyzeTaint(nodes v1.NodeList) (*analyze.AnalyzeResult, error) { + if a.analyzer.TolerationsMap == nil { + return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil + } + taintFailResult := []string{} + for _, node := range nodes.Items { + if node.Spec.Taints == nil || len(node.Spec.Taints) == 0 { + return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil + } + } + + for k, tolerations := range a.analyzer.TolerationsMap { + count := 0 + for _, node := range nodes.Items { + count += countTolerableTaints(node.Spec.Taints, tolerations) + } + if count <= 0 { + taintFailResult = append(taintFailResult, k) + } + } + if len(taintFailResult) > 0 { + result := newAnalyzeResult(a.Title(), FailType, a.analyzer.Outcomes) + result.Message += fmt.Sprintf(" Taint check failed components: %s", strings.Join(taintFailResult, ", ")) + return result, nil + } + return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil +} + +func (a *AnalyzeTaintClassByKb) generateTolerations() error { + optsMap, err := a.getHelmValues() + if err != nil { + return err + } + + tolerations := map[string][]v1.Toleration{} + getTolerationsMap(optsMap, "", tolerations) + a.analyzer.TolerationsMap = tolerations + return nil +} + +func (a *AnalyzeTaintClassByKb) getHelmValues() (map[string]interface{}, error) { + settings := cli.New() + p := getter.All(settings) + vals, err := a.HelmOpts.MergeValues(p) + if err != nil { + return nil, err + } + return vals, nil +} + +func getTolerationsMap(tolerationData map[string]interface{}, addonName string, tolerationsMap map[string][]v1.Toleration) { + var tmpTolerationList []v1.Toleration + var tmpToleration v1.Toleration + + for k, v := range tolerationData { + if k == Tolerations { + tolerationList := v.([]interface{}) + tmpTolerationList = []v1.Toleration{} + for _, t := range tolerationList { + toleration := t.(map[string]interface{}) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(toleration, &tmpToleration); err != nil { + continue + } + tmpTolerationList = append(tmpTolerationList, tmpToleration) + } + if addonName == "" { + addonName = KubeBlocks + } + tolerationsMap[addonName] = tmpTolerationList + continue + } + + switch v := v.(type) { + case map[string]interface{}: + if addonName != "" { + addonName += "." + } + addonName += k + getTolerationsMap(v, addonName, tolerationsMap) + default: + continue + } + } +} + +func countTolerableTaints(taints []v1.Taint, tolerations []v1.Toleration) int { + tolerableTaints := 0 + for _, taint := range taints { + // check only on taints that have effect NoSchedule + if taint.Effect != v1.TaintEffectNoSchedule { + continue + } + + if v1helper.TolerationsTolerateTaint(tolerations, &taint) { + tolerableTaints++ + } + } + return tolerableTaints +} diff --git a/internal/preflight/analyzer/kb_taint_test.go b/internal/preflight/analyzer/kb_taint_test.go new file mode 100644 index 000000000..612848720 --- /dev/null +++ b/internal/preflight/analyzer/kb_taint_test.go @@ -0,0 +1,151 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package analyzer + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/pkg/errors" + troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + "helm.sh/helm/v3/pkg/cli/values" + v1 "k8s.io/api/core/v1" + + preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" +) + +var ( + nodeList1 = v1.NodeList{Items: []v1.Node{ + {Spec: v1.NodeSpec{Taints: []v1.Taint{ + {Key: "dev", Value: "true", Effect: v1.TaintEffectNoSchedule}, + }}}, + {Spec: v1.NodeSpec{Taints: []v1.Taint{ + {Key: "dev", Value: "false", Effect: v1.TaintEffectNoSchedule}, + }}}, + }} + nodeList2 = v1.NodeList{Items: []v1.Node{ + {Spec: v1.NodeSpec{Taints: []v1.Taint{ + {Key: "dev", Value: "false", Effect: v1.TaintEffectNoSchedule}, + }}}, + }} + nodeList3 = v1.NodeList{Items: []v1.Node{ + {Spec: v1.NodeSpec{}}, + }} +) + +var _ = Describe("taint_class_test", func() { + var ( + analyzer AnalyzeTaintClassByKb + ) + Context("analyze taint test", func() { + BeforeEach(func() { + JSONStr := "tolerations=[ { \"key\": \"dev\", \"operator\": \"Equal\", \"effect\": \"NoSchedule\", \"value\": \"true\" }, " + + "{ \"key\": \"large\", \"operator\": \"Equal\", \"effect\": \"NoSchedule\", \"value\": \"true\" } ]," + + "prometheus.server.tolerations=[ { \"key\": \"dev\", \"operator\": \"Equal\", \"effect\": \"NoSchedule\", \"value\": \"true\" }, " + + "{ \"key\": \"large\", \"operator\": \"Equal\", \"effect\": \"NoSchedule\", \"value\": \"true\" } ]," + + "grafana.tolerations=[ { \"key\": \"dev\", \"operator\": \"Equal\", \"effect\": \"NoSchedule\", \"value\": \"true\" }, " + + "{ \"key\": \"large\", \"operator\": \"Equal\", \"effect\": \"NoSchedule\", \"value\": \"true\" } ]," + analyzer = AnalyzeTaintClassByKb{ + analyzer: &preflightv1beta2.KBTaintAnalyze{ + Outcomes: []*troubleshoot.Outcome{ + { + Pass: &troubleshoot.SingleOutcome{ + Message: "analyze storage class success", + }, + Fail: &troubleshoot.SingleOutcome{ + Message: "analyze storage class fail", + }, + }, + }, + }, + HelmOpts: &values.Options{JSONValues: []string{JSONStr}}, + } + + }) + It("Analyze test, and get file failed", func() { + Eventually(func(g Gomega) { + getCollectedFileContents := func(filename string) ([]byte, error) { + return nil, errors.New("get file failed") + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).To(HaveOccurred()) + g.Expect(res[0].IsFail).Should(BeTrue()) + g.Expect(res[0].IsPass).Should(BeFalse()) + }).Should(Succeed()) + }) + + It("Analyze test, and return of get file is not clusterResource", func() { + Eventually(func(g Gomega) { + getCollectedFileContents := func(filename string) ([]byte, error) { + return []byte("test"), nil + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).To(HaveOccurred()) + g.Expect(res[0].IsFail).Should(BeTrue()) + g.Expect(res[0].IsPass).Should(BeFalse()) + }).Should(Succeed()) + }) + + It("Analyze test, and analyzer result is expected that pass is true", func() { + Eventually(func(g Gomega) { + g.Expect(analyzer.IsExcluded()).Should(BeFalse()) + b, err := json.Marshal(nodeList1) + g.Expect(err).NotTo(HaveOccurred()) + getCollectedFileContents := func(filename string) ([]byte, error) { + return b, nil + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res[0].IsPass).Should(BeTrue()) + g.Expect(res[0].IsFail).Should(BeFalse()) + }).Should(Succeed()) + }) + It("Analyze test, and analyzer result is expected that fail is true", func() { + Eventually(func(g Gomega) { + g.Expect(analyzer.IsExcluded()).Should(BeFalse()) + b, err := json.Marshal(nodeList2) + g.Expect(err).NotTo(HaveOccurred()) + getCollectedFileContents := func(filename string) ([]byte, error) { + return b, nil + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res[0].IsPass).Should(BeFalse()) + g.Expect(res[0].IsFail).Should(BeTrue()) + }).Should(Succeed()) + }) + It("Analyze test, the taints are nil, and analyzer result is expected that pass is true", func() { + Eventually(func(g Gomega) { + g.Expect(analyzer.IsExcluded()).Should(BeFalse()) + b, err := json.Marshal(nodeList3) + g.Expect(err).NotTo(HaveOccurred()) + getCollectedFileContents := func(filename string) ([]byte, error) { + return b, nil + } + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res[0].IsPass).Should(BeTrue()) + g.Expect(res[0].IsFail).Should(BeFalse()) + }).Should(Succeed()) + }) + }) +}) diff --git a/internal/preflight/collect.go b/internal/preflight/collect.go index 51d1d9e79..ace90c338 100644 --- a/internal/preflight/collect.go +++ b/internal/preflight/collect.go @@ -31,6 +31,7 @@ import ( "github.com/replicatedhq/troubleshoot/pkg/logger" "github.com/replicatedhq/troubleshoot/pkg/preflight" "github.com/spf13/viper" + "helm.sh/helm/v3/pkg/cli/values" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" cmdutil "k8s.io/kubectl/pkg/cmd/util" @@ -39,14 +40,14 @@ import ( kbcollector "github.com/apecloud/kubeblocks/internal/preflight/collector" ) -func CollectPreflight(f cmdutil.Factory, ctx context.Context, kbPreflight *preflightv1beta2.Preflight, kbHostPreflight *preflightv1beta2.HostPreflight, progressCh chan interface{}) ([]preflight.CollectResult, error) { +func CollectPreflight(f cmdutil.Factory, helmOpts *values.Options, ctx context.Context, kbPreflight *preflightv1beta2.Preflight, kbHostPreflight *preflightv1beta2.HostPreflight, progressCh chan interface{}) ([]preflight.CollectResult, error) { var ( collectResults []preflight.CollectResult err error ) // deal with preflight if kbPreflight != nil && (len(kbPreflight.Spec.ExtendCollectors) > 0 || len(kbPreflight.Spec.Collectors) > 0) { - res, err := CollectClusterData(ctx, kbPreflight, f, progressCh) + res, err := CollectClusterData(ctx, kbPreflight, f, helmOpts, progressCh) if err != nil { return collectResults, errors.Wrap(err, "failed to collect data in cluster") } @@ -127,7 +128,7 @@ func CollectHost(ctx context.Context, opts preflight.CollectOpts, collectors []p } // CollectClusterData transforms the specs of Preflight to Collector, and sets the collectOpts, such as restConfig, Namespace, and ProgressChan -func CollectClusterData(ctx context.Context, kbPreflight *preflightv1beta2.Preflight, f cmdutil.Factory, progressCh chan interface{}) (*preflight.CollectResult, error) { +func CollectClusterData(ctx context.Context, kbPreflight *preflightv1beta2.Preflight, f cmdutil.Factory, helmOpts *values.Options, progressCh chan interface{}) (*preflight.CollectResult, error) { var err error v := viper.GetViper() @@ -182,12 +183,18 @@ func CollectClusterData(ctx context.Context, kbPreflight *preflightv1beta2.Prefl // // todo user defined cluster collector // } - collectResults, err := CollectCluster(ctx, collectOpts, collectors, allCollectorsMap, kbPreflight) + collectResults, err := CollectCluster(ctx, collectOpts, collectors, allCollectorsMap, kbPreflight, helmOpts) return &collectResults, err } -// CollectCluster collects ciuster data against by Collector，and returns the collected data which is encapsulated in CollectResult struct -func CollectCluster(ctx context.Context, opts preflight.CollectOpts, allCollectors []pkgcollector.Collector, allCollectorsMap map[reflect.Type][]pkgcollector.Collector, kbPreflight *preflightv1beta2.Preflight) (preflight.CollectResult, error) { +// CollectCluster collects cluster data against by Collector，and returns the collected data which is encapsulated in CollectResult struct +func CollectCluster(ctx context.Context, + opts preflight.CollectOpts, + allCollectors []pkgcollector.Collector, + allCollectorsMap map[reflect.Type][]pkgcollector.Collector, + kbPreflight *preflightv1beta2.Preflight, + helmOpts *values.Options, +) (preflight.CollectResult, error) { var foundForbidden bool allCollectedData := make(map[string][]byte) collectorList := map[string]preflight.CollectorStatus{} @@ -223,6 +230,7 @@ func CollectCluster(ctx context.Context, opts preflight.CollectOpts, allCollecto }, AnalyzerSpecs: kbPreflight.Spec.Analyzers, KbAnalyzerSpecs: kbPreflight.Spec.ExtendAnalyzers, + HelmOptions: helmOpts, } if foundForbidden && !opts.IgnorePermissionErrors { diff --git a/internal/preflight/collect_test.go b/internal/preflight/collect_test.go index 73c7c8911..d277d47cd 100644 --- a/internal/preflight/collect_test.go +++ b/internal/preflight/collect_test.go @@ -100,7 +100,7 @@ var _ = Describe("collect_test", func() { g.Expect(<-progressCh).NotTo(BeNil()) } }() - results, err := CollectPreflight(tf, context.TODO(), preflight, hostPreflight, progressCh) + results, err := CollectPreflight(tf, nil, context.TODO(), preflight, hostPreflight, progressCh) g.Expect(err).NotTo(HaveOccurred()) g.Expect(len(results)).Should(BeNumerically(">=", 3)) }).WithTimeout(timeOut).Should(Succeed()) diff --git a/internal/preflight/text_results.go b/internal/preflight/text_results.go index 780e43331..6fdb3b05d 100644 --- a/internal/preflight/text_results.go +++ b/internal/preflight/text_results.go @@ -29,6 +29,11 @@ import ( "gopkg.in/yaml.v2" ) +const ( + FailMessage = "Fail items were found. Please resolve the fail items and try again." + PassMessage = "The kubernetes cluster preflight check pass, and you can enjoy KubeBlocks now." +) + type TextResultOutput struct { Title string `json:"title" yaml:"title"` Message string `json:"message" yaml:"message"` @@ -71,11 +76,19 @@ func ShowTextResults(preflightName string, analyzeResults []*analyzerunner.Analy } func showTextResultsJSON(preflightName string, analyzeResults []*analyzerunner.AnalyzeResult, verbose bool) error { - b, err := json.MarshalIndent(showStdoutResultsStructured(preflightName, analyzeResults, verbose), "", " ") + output := showStdoutResultsStructured(preflightName, analyzeResults, verbose) + b, err := json.MarshalIndent(output, "", " ") if err != nil { return errors.Wrap(err, "failed to marshal results as json") } - fmt.Printf("%s\n", b) + if len(b) <= 5 { + fmt.Println(PassMessage) + } else { + fmt.Printf("%s\n", b) + } + if len(output.Fail) > 0 { + return errors.New(FailMessage) + } return nil } @@ -87,10 +100,10 @@ func showStdoutResultsYAML(preflightName string, analyzeResults []*analyzerunner failInfo = color.New(color.FgRed) ) if len(data.Warn) == 0 && len(data.Fail) == 0 { - passInfo.Println("congratulations, your kubernetes cluster preflight check pass, and begin to enjoy KubeBlocks...") + fmt.Println(PassMessage) } if len(data.Pass) > 0 { - passInfo.Println("pass items") + passInfo.Println("Pass items") if b, err := yaml.Marshal(data.Pass); err != nil { return errors.Wrap(err, "failed to marshal results as yaml") } else { @@ -98,7 +111,7 @@ func showStdoutResultsYAML(preflightName string, analyzeResults []*analyzerunner } } if len(data.Warn) > 0 { - warnInfo.Println("warn items") + warnInfo.Println("Warn items") if b, err := yaml.Marshal(data.Warn); err != nil { return errors.Wrap(err, "failed to marshal results as yaml") } else { @@ -106,12 +119,13 @@ func showStdoutResultsYAML(preflightName string, analyzeResults []*analyzerunner } } if len(data.Fail) > 0 { - failInfo.Println("fail items") + failInfo.Println("Fail items") if b, err := yaml.Marshal(data.Fail); err != nil { return errors.Wrap(err, "failed to marshal results as yaml") } else { fmt.Printf("%s\n", b) } + return errors.New(FailMessage) } return nil } diff --git a/internal/preflight/text_results_test.go b/internal/preflight/text_results_test.go index f91cf1e82..cf9dabe37 100644 --- a/internal/preflight/text_results_test.go +++ b/internal/preflight/text_results_test.go @@ -42,12 +42,6 @@ var _ = Describe("text_results_test", func() { Message: "message for pass test", URI: "https://kubernetes.io", }, - { - IsFail: true, - Title: "fail item", - Message: "message for fail test", - URI: "https://kubernetes.io", - }, { IsWarn: true, Title: "warn item", @@ -65,6 +59,26 @@ var _ = Describe("text_results_test", func() { g.Expect(err).NotTo(HaveOccurred()) err = ShowTextResults(preflightName, analyzeResults, unknownFormat, false) g.Expect(err).To(HaveOccurred()) - }).Should(Succeed()) + }).ShouldNot(HaveOccurred()) + }) + It("ShowStdoutResults Test", func() { + analyzeResults := []*analyzerunner.AnalyzeResult{ + { + IsFail: true, + Title: "fail item", + Message: "message for fail test", + URI: "https://kubernetes.io", + }, + } + Eventually(func(g Gomega) { + err := ShowTextResults(preflightName, analyzeResults, humanFormat, false) + g.Expect(err).To(HaveOccurred()) + err = ShowTextResults(preflightName, analyzeResults, jsonFormat, true) + g.Expect(err).NotTo(HaveOccurred()) + err = ShowTextResults(preflightName, analyzeResults, yamlFormat, false) + g.Expect(err).NotTo(HaveOccurred()) + err = ShowTextResults(preflightName, analyzeResults, unknownFormat, false) + g.Expect(err).To(HaveOccurred()) + }).Should(HaveOccurred()) }) }) From c41f69b6b12218f443ce2ffdf2dce803c2ea20a0 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Fri, 12 May 2023 15:13:56 +0800 Subject: [PATCH 286/439] chore: migrate parameter archive_command from Local configuration to Dynamic configuration (#3216) --- deploy/postgresql/templates/scripts.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index 340419ef9..6f35977ab 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -54,12 +54,6 @@ data: postgresql = local_config['postgresql'] postgresql['config_dir'] = '/home/postgres/pgdata/conf' postgresql['custom_conf'] = '/home/postgres/conf/postgresql.conf' - # TODO add local postgresql.parameters - if not 'parameters' in postgresql: - postgresql['parameters'] = {} - parameters = postgresql_conf_to_dict("/home/postgres/conf/postgresql.conf") - if 'archive_command' in parameters: - postgresql['parameters']['archive_command'] = parameters['archive_command'] # add pg_hba.conf with open('/home/postgres/conf/pg_hba.conf', 'r') as f: lines = read_file_lines(f) From bb43d12c0fde3645c05284a629034318c42245fa Mon Sep 17 00:00:00 2001 From: xingran Date: Fri, 12 May 2023 17:18:08 +0800 Subject: [PATCH 287/439] chore: rename patroni scope and remove dependency resource deleted check (#3215) --- .../apps/components/util/component_utils.go | 8 +-- .../templates/clusterdefinition.yaml | 4 +- internal/constant/const.go | 9 +-- internal/controller/builder/builder.go | 7 +++ internal/controller/component/component.go | 14 +++-- internal/controller/graph/dag.go | 16 +++--- .../lifecycle/cluster_plan_builder.go | 55 +------------------ .../lifecycle/transformer_cluster_deletion.go | 20 ------- internal/controller/plan/builtin_env.go | 3 +- internal/controller/plan/builtin_env_test.go | 1 + 10 files changed, 39 insertions(+), 98 deletions(-) diff --git a/controllers/apps/components/util/component_utils.go b/controllers/apps/components/util/component_utils.go index 6f810f681..dbc94bf36 100644 --- a/controllers/apps/components/util/component_utils.go +++ b/controllers/apps/components/util/component_utils.go @@ -399,8 +399,8 @@ func PatchGVRCustomLabels(ctx context.Context, cli client.Client, cluster *appsv if err := GetObjectListByCustomLabels(ctx, cli, *cluster, objectList, client.MatchingLabels(matchLabels)); err != nil { return err } - labelKey = replaceKBEnvPlaceholderTokens(cluster.Name, componentName, labelKey) - labelValue = replaceKBEnvPlaceholderTokens(cluster.Name, componentName, labelValue) + labelKey = replaceKBEnvPlaceholderTokens(cluster, componentName, labelKey) + labelValue = replaceKBEnvPlaceholderTokens(cluster, componentName, labelValue) switch gvk.Kind { case constant.StatefulSetKind: stsList := objectList.(*appsv1.StatefulSetList) @@ -506,7 +506,7 @@ func getObjectListMapOfResourceKind() map[string]client.ObjectList { } // replaceKBEnvPlaceholderTokens replaces the placeholder tokens in the string strToReplace with builtInEnvMap and return new string. -func replaceKBEnvPlaceholderTokens(clusterName, componentName, strToReplace string) string { - builtInEnvMap := componentutil.GetReplacementMapForBuiltInEnv(clusterName, componentName) +func replaceKBEnvPlaceholderTokens(cluster *appsv1alpha1.Cluster, componentName, strToReplace string) string { + builtInEnvMap := componentutil.GetReplacementMapForBuiltInEnv(cluster, componentName) return componentutil.ReplaceNamedVars(builtInEnvMap, strToReplace, -1, true) } diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index c77427c7b..1365a185f 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -18,7 +18,7 @@ spec: characterType: postgresql customLabelSpecs: - key: apps.kubeblocks.postgres.patroni/scope - value: "$(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-patroni" + value: "$(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-patroni$(KB_CLUSTER_UID_POSTFIX_8)" resources: - gvk: "v1/Pod" selector: @@ -127,7 +127,7 @@ spec: - name: KUBERNETES_USE_CONFIGMAPS value: "true" - name: SCOPE - value: "$(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-patroni" + value: "$(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-patroni$(KB_CLUSTER_UID_POSTFIX_8)" - name: KUBERNETES_SCOPE_LABEL value: "apps.kubeblocks.postgres.patroni/scope" - name: KUBERNETES_ROLE_LABEL diff --git a/internal/constant/const.go b/internal/constant/const.go index b7a15dd01..dda59961d 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -45,10 +45,11 @@ const ( ) const ( - ConnCredentialPlaceHolder = "$(CONN_CREDENTIAL_SECRET_NAME)" - KBCompNamePlaceHolder = "$(KB_COMP_NAME)" - KBClusterNamePlaceHolder = "$(KB_CLUSTER_NAME)" - KBClusterCompNamePlaceHolder = "$(KB_CLUSTER_COMP_NAME)" + ConnCredentialPlaceHolder = "$(CONN_CREDENTIAL_SECRET_NAME)" + KBCompNamePlaceHolder = "$(KB_COMP_NAME)" + KBClusterNamePlaceHolder = "$(KB_CLUSTER_NAME)" + KBClusterCompNamePlaceHolder = "$(KB_CLUSTER_COMP_NAME)" + KBClusterUIDPostfix8PlaceHolder = "$(KB_CLUSTER_UID_POSTFIX_8)" ) const ( diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index 2ad043edc..e657c3ee1 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -163,10 +163,17 @@ func injectEnvs(params BuilderParams, envConfigName string, c *corev1.Container) }) } + var kbClusterPostfix8 string + if len(params.Cluster.UID) > 8 { + kbClusterPostfix8 = string(params.Cluster.UID)[len(params.Cluster.UID)-8:] + } else { + kbClusterPostfix8 = string(params.Cluster.UID) + } toInjectEnvs = append(toInjectEnvs, []corev1.EnvVar{ {Name: "KB_CLUSTER_NAME", Value: params.Cluster.Name}, {Name: "KB_COMP_NAME", Value: params.Component.Name}, {Name: "KB_CLUSTER_COMP_NAME", Value: params.Cluster.Name + "-" + params.Component.Name}, + {Name: "KB_CLUSTER_UID_POSTFIX_8", Value: kbClusterPostfix8}, {Name: "KB_POD_FQDN", Value: fmt.Sprintf("%s.%s-headless.%s.svc", "$(KB_POD_NAME)", "$(KB_CLUSTER_COMP_NAME)", "$(KB_NAMESPACE)")}, }...) diff --git a/internal/controller/component/component.go b/internal/controller/component/component.go index 6104eb5e0..b863ad9ab 100644 --- a/internal/controller/component/component.go +++ b/internal/controller/component/component.go @@ -243,12 +243,18 @@ func replaceContainerPlaceholderTokens(component *SynthesizedComponent, namedVal } // GetReplacementMapForBuiltInEnv gets the replacement map for KubeBlocks built-in environment variables. -func GetReplacementMapForBuiltInEnv(clusterName, componentName string) map[string]string { - return map[string]string{ - constant.KBClusterNamePlaceHolder: clusterName, +func GetReplacementMapForBuiltInEnv(cluster *appsv1alpha1.Cluster, componentName string) map[string]string { + replacementMap := map[string]string{ + constant.KBClusterNamePlaceHolder: cluster.Name, constant.KBCompNamePlaceHolder: componentName, - constant.KBClusterCompNamePlaceHolder: fmt.Sprintf("%s-%s", clusterName, componentName), + constant.KBClusterCompNamePlaceHolder: fmt.Sprintf("%s-%s", cluster.Name, componentName), + } + if len(cluster.UID) > 8 { + replacementMap[constant.KBClusterUIDPostfix8PlaceHolder] = string(cluster.UID)[len(cluster.UID)-8:] + } else { + replacementMap[constant.KBClusterUIDPostfix8PlaceHolder] = string(cluster.UID) } + return replacementMap } // ReplaceNamedVars replaces the placeholder in targetVar if it is match and returns the replaced result diff --git a/internal/controller/graph/dag.go b/internal/controller/graph/dag.go index 0aaff9347..77c40db1e 100644 --- a/internal/controller/graph/dag.go +++ b/internal/controller/graph/dag.go @@ -159,7 +159,7 @@ func (d *DAG) WalkReverseTopoOrder(walkFunc WalkFunc) error { func (d *DAG) Root() Vertex { roots := make([]Vertex, 0) for n := range d.vertices { - if len(d.InAdj(n)) == 0 { + if len(d.inAdj(n)) == 0 { roots = append(roots, n) } } @@ -211,7 +211,7 @@ func (d *DAG) validate() error { } marked[v] = true - adjacent := d.OutAdj(v) + adjacent := d.outAdj(v) for _, vertex := range adjacent { if err := walk(vertex); err != nil { return err @@ -246,9 +246,9 @@ func (d *DAG) topologicalOrder(reverse bool) []Vertex { } var adjacent []Vertex if reverse { - adjacent = d.OutAdj(v) + adjacent = d.outAdj(v) } else { - adjacent = d.InAdj(v) + adjacent = d.inAdj(v) } for _, vertex := range adjacent { walk(vertex) @@ -262,8 +262,8 @@ func (d *DAG) topologicalOrder(reverse bool) []Vertex { return orders } -// OutAdj returns all adjacent vertices that v points to -func (d *DAG) OutAdj(v Vertex) []Vertex { +// outAdj returns all adjacent vertices that v points to +func (d *DAG) outAdj(v Vertex) []Vertex { vertices := make([]Vertex, 0) for e := range d.edges { if e.From() == v { @@ -273,8 +273,8 @@ func (d *DAG) OutAdj(v Vertex) []Vertex { return vertices } -// InAdj returns all adjacent vertices that point to v -func (d *DAG) InAdj(v Vertex) []Vertex { +// inAdj returns all adjacent vertices that point to v +func (d *DAG) inAdj(v Vertex) []Vertex { vertices := make([]Vertex, 0) for e := range d.edges { if e.To() == v { diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 360c11531..cfbb444a5 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -32,14 +32,12 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - componentutil "github.com/apecloud/kubeblocks/controllers/apps/components/util" opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" "github.com/apecloud/kubeblocks/internal/constant" roclient "github.com/apecloud/kubeblocks/internal/controller/client" @@ -67,7 +65,6 @@ type clusterPlanBuilder struct { cli client.Client transCtx *ClusterTransformContext transformers graph.TransformerChain - dag *graph.DAG } // clusterPlan a graph.Plan implementation for Cluster reconciliation @@ -154,8 +151,6 @@ func (c *clusterPlanBuilder) Build() (graph.Plan, error) { dag := graph.NewDAG() err = c.transformers.ApplyTo(c.transCtx, dag) c.transCtx.Logger.V(1).Info(fmt.Sprintf("DAG: %s", dag)) - // add dag to clusterPlanBuilder - c.dag = dag // construct execution plan plan := &clusterPlan{ @@ -275,12 +270,7 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { if strings.EqualFold(node.obj.GetLabels()[constant.BackupProtectionLabelKey], constant.BackupRetain) { return nil } - // check dependency resources has been deleted before deleting the resource - err := c.checkDependencyResourcesDeleted(node) - if err != nil { - return err - } - err = intctrlutil.BackgroundDeleteObject(c.cli, c.transCtx.Context, node.obj) + err := intctrlutil.BackgroundDeleteObject(c.cli, c.transCtx.Context, node.obj) // err := c.cli.Delete(c.transCtx.Context, node.obj) if err != nil && !apierrors.IsNotFound(err) { return err @@ -403,46 +393,3 @@ func (c *clusterPlanBuilder) emitPhaseUpdatingEvent(oldPhase, newPhase appsv1alp _ = opsutil.MarkRunningOpsRequestAnnotation(c.transCtx.Context, c.cli, cluster) } } - -// checkDependencyResourcesDeleted checks if the dependency resources are deleted when cluster is deleted. -func (c *clusterPlanBuilder) checkDependencyResourcesDeleted(node *lifecycleVertex) error { - if c.dag == nil { - return nil - } - // get the dependency resources - outAdj := c.dag.OutAdj(node) - if len(outAdj) == 0 { - return nil - } - for _, out := range outAdj { - outNode, ok := out.(*lifecycleVertex) - if !ok { - return fmt.Errorf("wrong vertex type %v", outNode) - } - // if the node.obj is StatefulSet, check if the pods are deleted - sts, ok := outNode.obj.(*appsv1.StatefulSet) - if ok { - pods, err := componentutil.GetPodListByStatefulSet(c.transCtx.Context, c.cli, sts) - if err != nil { - return err - } - if len(pods) > 0 { - return &realRequeueError{reason: fmt.Sprintf("waiting dependency resource delete, %s/%s dependency resource statefulSet %s/%s still have pods", - node.obj.GetNamespace(), node.obj.GetName(), outNode.obj.GetNamespace(), outNode.obj.GetName()), requeueAfter: requeueDuration} - } - } - // check if the dependency resource is deleted - err := c.cli.Get(c.transCtx.Context, types.NamespacedName{Name: outNode.obj.GetName(), Namespace: outNode.obj.GetNamespace()}, outNode.obj) - if err != nil { - if apierrors.IsNotFound(err) { - continue - } - return err - } - if outNode.obj != nil { - return &realRequeueError{reason: fmt.Sprintf("waiting dependency resource delete, %s/%s dependency resource %s/%s is not deleted", - node.obj.GetNamespace(), node.obj.GetName(), outNode.obj.GetNamespace(), outNode.obj.GetName()), requeueAfter: requeueDuration} - } - } - return nil -} diff --git a/internal/controller/lifecycle/transformer_cluster_deletion.go b/internal/controller/lifecycle/transformer_cluster_deletion.go index 2d9732439..7bb87cc98 100644 --- a/internal/controller/lifecycle/transformer_cluster_deletion.go +++ b/internal/controller/lifecycle/transformer_cluster_deletion.go @@ -79,10 +79,6 @@ func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * dag.AddVertex(vertex) dag.Connect(root, vertex) } - - // adjust the dependency resource deletion order - adjustDependencyResourceDeletionOrder(root, dag) - root.action = actionPtr(DELETE) // fast return, that is stopping the plan.Build() stage and jump to plan.Execute() directly @@ -124,20 +120,4 @@ func kindsForWipeOut() []client.ObjectList { return append(kinds, kindsPlus...) } -// adjustDependencyResourceDeletionOrder adjusts the deletion order of resources by adjusting DAG topology. -// find all vertices of StatefulSets and connect them to ConfigMap, -// this is to ensure that ConfigMap is deleted after StatefulSet Workloads are deleted. -func adjustDependencyResourceDeletionOrder(root *lifecycleVertex, dag *graph.DAG) { - vertices := findAll[*appsv1.StatefulSet](dag) - cmVertices := findAll[*corev1.ConfigMap](dag) - if len(vertices) > 0 && len(cmVertices) > 0 { - for _, vertex := range vertices { - dag.RemoveEdge(graph.RealEdge(root, vertex)) - for _, cmVertex := range cmVertices { - dag.Connect(cmVertex, vertex) - } - } - } -} - var _ graph.Transformer = &ClusterDeletionTransformer{} diff --git a/internal/controller/plan/builtin_env.go b/internal/controller/plan/builtin_env.go index 235d96b13..354afd585 100644 --- a/internal/controller/plan/builtin_env.go +++ b/internal/controller/plan/builtin_env.go @@ -227,9 +227,8 @@ func (w *envWrapper) checkAndReplaceEnv(value string, container *corev1.Containe func (w *envWrapper) doEnvReplace(replacedVars *set.LinkedHashSetString, oldValue string, container *corev1.Container) (string, error) { var ( - clusterName = w.localObjects.Cluster.Name componentName = w.localObjects.Component.Name - builtInEnvMap = component.GetReplacementMapForBuiltInEnv(clusterName, componentName) + builtInEnvMap = component.GetReplacementMapForBuiltInEnv(w.localObjects.Cluster, componentName) ) builtInEnvMap[constant.ConnCredentialPlaceHolder] = component.GenerateConnCredential(w.localObjects.Cluster.Name) diff --git a/internal/controller/plan/builtin_env_test.go b/internal/controller/plan/builtin_env_test.go index f90dbddd2..a0cf300af 100644 --- a/internal/controller/plan/builtin_env_test.go +++ b/internal/controller/plan/builtin_env_test.go @@ -201,6 +201,7 @@ bootstrap: cluster = &appsv1alpha1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "my", + UID: "b006a20c-fb03-441c-bffa-2605cad7e297", }, } cfgTemplate = []appsv1alpha1.ComponentConfigSpec{{ From e6f9c23b8e619cfd372bde105fa192c6f5859040 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 18:57:51 +0800 Subject: [PATCH 288/439] chore(deps): bump github.com/docker/distribution from 2.8.1+incompatible to 2.8.2+incompatible (#3212) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Nash Tsai --- go.mod | 4 +- go.sum | 156 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 401a56d9d..e750f75d4 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,6 @@ require ( github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 github.com/onsi/ginkgo/v2 v2.9.1 github.com/onsi/gomega v1.27.4 - github.com/opencontainers/image-spec v1.1.0-rc2 github.com/pingcap/go-tpc v1.0.9 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 @@ -151,7 +150,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-metrics v0.0.1 // indirect @@ -277,6 +276,7 @@ require ( github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opencontainers/selinux v1.10.2 // indirect diff --git a/go.sum b/go.sum index bce9c5796..c65a8ec55 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= +bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -193,6 +194,8 @@ cuelang.org/go v0.4.3 h1:W3oBBjDTm7+IZfCKZAmC8uDG0eYfJL4Pp/xbbCMKaVo= cuelang.org/go v0.4.3/go.mod h1:7805vR9H+VoBNdWFdI7jyDR3QLUPp4+naHfbcgp55HI= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 h1:SCbEWT58NSt7d2mcFdvxC9uyrdcTfvBbPLThhkDmXzg= +github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a h1:XVdatQFSP2YhJGjqLLIfW8QBk4loz/SCe/PxkXDiW+s= github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a/go.mod h1:C0A1KeiVHs+trY6gUTPhhGammbrZ30ZfXRW/nuT7HLw= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= @@ -203,6 +206,7 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -211,11 +215,13 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4= @@ -233,10 +239,12 @@ github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQX github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -268,6 +276,7 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= @@ -278,15 +287,20 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= +github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY= github.com/Microsoft/hcsshim v0.9.6/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20220407094043-a94812496cf5/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -303,6 +317,8 @@ github.com/Shopify/toxiproxy/v2 v2.1.6-0.20210914104332-15ea381dcdae/go.mod h1:/ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StudioSol/set v1.0.0 h1:G27J71la+Da08WidabBkoRrvPLTa4cdCn0RjvyJ5WKQ= github.com/StudioSol/set v1.0.0/go.mod h1:hIUNZPo6rEGF43RlPXHq7Fjmf+HkVJBqAjtK7Z9LoIU= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/agrea/ptr v0.0.0-20180711073057-77a518d99b7b h1:WMhlIaJkDgEQSVJQM06YV+cYUl1r5OY5//ijMXJNqtA= @@ -316,6 +332,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -342,6 +359,7 @@ github.com/authzed/controller-idioms v0.7.0/go.mod h1:0B/PmqCguKv8b3azSMF+HdyKpK github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -390,6 +408,7 @@ github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f/go.mod h1:uEyr4WpAH github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -466,6 +485,7 @@ github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4S github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= +github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= @@ -482,12 +502,16 @@ github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= +github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= +github.com/containerd/containerd v1.6.3/go.mod h1:gCVGrYRYFm2E8GmuUIbj/NGD7DLZQLzSJQazjVKDOig= github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns= github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -497,6 +521,7 @@ github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cE github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= +github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= @@ -506,6 +531,9 @@ github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1S github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= +github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= +github.com/containerd/go-cni v1.1.4/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= @@ -515,10 +543,13 @@ github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= +github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= +github.com/containerd/stargz-snapshotter/estargz v0.11.4/go.mod h1:7vRJIcImfY8bpifnMjt+HTJoQxASq7T28MYbP15/Nf0= github.com/containerd/stargz-snapshotter/estargz v0.13.0 h1:fD7AwuVV+B40p0d9qVkH/Au1qhp8hn/HWJHIYjpEcfw= github.com/containerd/stargz-snapshotter/estargz v0.13.0/go.mod h1:m+9VaGJGlhCnrcEUod8mYumTmRgblwd3rC5UCEh2Yp0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= @@ -539,19 +570,31 @@ github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNR github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= +github.com/containernetworking/cni v1.1.0/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= +github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= +github.com/containers/common v0.48.1 h1:Te1lfQt1TDyGjig+Equ0j1LOFV1VuuObz/WnB7J151M= +github.com/containers/common v0.48.1/go.mod h1:zPLZCfLXfnd1jI0QRsD4By54fP4k1+ifQs+tulIe3o0= github.com/containers/common v0.49.1 h1:6y4/s2WwYxrv+Cox7fotOo316wuZI+iKKPUQweCYv50= github.com/containers/common v0.49.1/go.mod h1:ueM5hT0itKqCQvVJDs+EtjornAQtrHYxQJzP2gxeGIg= +github.com/containers/image/v5 v5.21.1/go.mod h1:zl35egpcDQa79IEXIuoUe1bW+D1pdxRxYjNlyb3YiXw= github.com/containers/image/v5 v5.24.0 h1:2Pu8ztTntqNxteVN15bORCQnM8rfnbYuyKwUiiKUBuc= github.com/containers/image/v5 v5.24.0/go.mod h1:oss5F6ssGQz8ZtC79oY+fuzYA3m3zBek9tq9gmhuvHc= +github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= +github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= github.com/containers/ocicrypt v1.1.7 h1:thhNr4fu2ltyGz8aMx8u48Ae0Pnbip3ePP9/mzkZ/3U= github.com/containers/ocicrypt v1.1.7/go.mod h1:7CAhjcj2H8AYp5YvEie7oVSK2AhBY8NscCYRawuDNtw= +github.com/containers/storage v1.40.0/go.mod h1:zUyPC3CFIGR1OhY1CKkffxgw9+LuH76PGvVcFj38dgs= github.com/containers/storage v1.45.3 h1:GbtTvTtp3GW2/tcFg5VhgHXcYMwVn2KfZKiHjf9FAOM= github.com/containers/storage v1.45.3/go.mod h1:OdRUYHrq1HP6iAo79VxqtYuJzC5j4eA2I60jKOoCT7g= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -559,6 +602,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -575,6 +619,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -589,6 +634,7 @@ github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4= +github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= github.com/dapr/components-contrib v1.9.6 h1:C12fnZhNim8AsOzTCsWZDZ0MXqsaG161fStVuVAQgN4= github.com/dapr/components-contrib v1.9.6/go.mod h1:U0cjxEEbZR7sNN9i1ZdWnkIOZP8iRSvoyF2gRhBaHfc= github.com/dapr/dapr v1.9.5 h1:oUFpy8+Z1lBS0XYKBCGtFa+GOEf0EDnFU26Rqpwalqc= @@ -613,6 +659,7 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/disiqueira/gotree/v3 v3.0.2/go.mod h1:ZuyjE4+mUQZlbpkI24AmruZKhg3VHEgPLDY8Qk+uUu8= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= @@ -622,12 +669,15 @@ github.com/docker/cli v20.10.24+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hH github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE= github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0= @@ -680,6 +730,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= @@ -700,6 +751,7 @@ github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -752,6 +804,7 @@ github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNV github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= @@ -769,11 +822,13 @@ github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuA github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= @@ -783,6 +838,7 @@ github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqb github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI= @@ -961,6 +1017,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= @@ -991,6 +1048,8 @@ github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMd github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -1000,6 +1059,7 @@ github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= @@ -1099,7 +1159,9 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= @@ -1129,6 +1191,7 @@ github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuT github.com/jhump/protoreflect v1.13.0 h1:zrrZqa7JAc2YGgPSzZZkmUXJ5G6NRPdxOg/9t7ISImA= github.com/jhump/protoreflect v1.13.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc= @@ -1182,12 +1245,15 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid/v2 v2.0.14 h1:QRqdp6bb9M9S5yyKeYteXKuoKE4p0tGlra81fKOpWH8= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 h1:BcxbplxjtczA1a6d3wYoa7a0WL3rq9DKBMGHeKyjEF0= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1251,6 +1317,7 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= @@ -1357,10 +1424,14 @@ github.com/moby/sys/mount v0.3.0/go.mod h1:U2Z3ur2rXPFrFmy4q6WMwWrBOAQGYtYTRVM8B github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1391,6 +1462,7 @@ github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5w github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.0.0-20200811152831-6cf413ae40e0/go.mod h1:wBEpHwM2OdmeNpdCvRPUlkEbBuaFmcK4Wv8Q7FuGW3c= github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d h1:x3S6kxmy49zXVVyhcnrFqxvNVCBPb2KZ9hV2RBdS840= @@ -1414,9 +1486,12 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1427,7 +1502,10 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -1438,7 +1516,10 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84/go.mod h1:Qnt1q4cjDNQI9bT832ziho5Iw2BhK8o1KwLOwW56VP4= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -1447,6 +1528,8 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1457,10 +1540,12 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.m github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.2 h1:NFy2xCsjn7+WspbfZkUd5zyVeisV7VFbPSP96+8/ha4= github.com/opencontainers/selinux v1.10.2/go.mod h1:cARutUbaUrlRClyvxOICCgKixCs6L05aUsohzA3EkHQ= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -1502,6 +1587,7 @@ github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1 h1:oL4IBbcqwhhNWh31bjOX8 github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/proglottis/gpgme v0.1.1/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0-pre1.0.20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1578,6 +1664,7 @@ github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XF github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rubenv/sql-migrate v1.2.0 h1:fOXMPLMd41sK7Tg75SXDec15k3zg5WNV6SjuDRiNfcU= github.com/rubenv/sql-migrate v1.2.0/go.mod h1:Z5uVnq7vrIrPmHbVFfR4YLHRZquxeHpckCnRq0P/K9Y= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1585,6 +1672,7 @@ github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUcc github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d h1:Q+gqLBOPkFGHyCJxXMRqtUgUbTjI8/Ze8vu8GGyNFwo= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= @@ -1594,11 +1682,14 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/securego/gosec/v2 v2.4.0/go.mod h1:0/Q4cjmlFDfDUj1+Fib61sc+U5IQb2w+Iv9/C3wPVko= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -1651,6 +1742,7 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -1699,6 +1791,7 @@ github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/sykesm/zap-logfmt v0.0.4 h1:U2WzRvmIWG1wDLCFY3sz8UeEmsdHQjHFNlIdmroVFaI= github.com/sykesm/zap-logfmt v0.0.4/go.mod h1:AuBd9xQjAe3URrWT1BBDk2v2onAZHkZkWRMiYZXiZWA= +github.com/sylabs/sif/v2 v2.7.0/go.mod h1:TiyBWsgWeh5yBeQFNuQnvROwswqK7YJT8JA1L53bsXQ= github.com/sylabs/sif/v2 v2.9.0 h1:q9K92j1QW4/QLOtKh9YZpJHrXav6x15AVhQGPVLcg+4= github.com/sylabs/sif/v2 v2.9.0/go.mod h1:bRdFzcqif0eDjwx0isG4cgTFoKTQn/vfBXVSoP2rB2Y= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1726,6 +1819,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= @@ -1750,12 +1844,15 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= +github.com/vbauerster/mpb/v7 v7.4.1/go.mod h1:Ygg2mV9Vj9sQBWqsK2m2pidcf9H3s6bNKtqd3/M4gBo= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= @@ -1795,6 +1892,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -1827,12 +1925,16 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v2 v2.305.6 h1:fIDR0p4KMjw01MJMfUIDWdQbjo06PD6CeYM5z4EHLi0= go.etcd.io/etcd/client/v2 v2.305.6/go.mod h1:BHha8XJGe8vCIBfWBpbBLVZ4QjOIlfoouvOwydu63E0= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= go.etcd.io/etcd/client/v3 v3.5.6 h1:coLs69PWCXE9G4FKquzNaSHrRyMCAXwF+IX1tAPVO8E= go.etcd.io/etcd/client/v3 v3.5.6/go.mod h1:f6GRinRMCsFVv9Ht42EyY7nfsVGwrNO0WEoS2pRKzQk= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/pkg/v3 v3.5.6 h1:k1GZrGrfMHy5/cg2bxNGsmLTFisatyhDYCFLRuaavWg= go.etcd.io/etcd/pkg/v3 v3.5.6/go.mod h1:qATwUzDb6MLyGWq2nUj+jwXqZJcxkCuabh0P7Cuff3k= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/raft/v3 v3.5.6 h1:tOmx6Ym6rn2GpZOrvTGJZciJHek6RnC3U/zNInzIN50= go.etcd.io/etcd/raft/v3 v3.5.6/go.mod h1:wL8kkRGx1Hp8FmZUuHfL3K2/OaGIDaXGr1N7i2G07J0= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.etcd.io/etcd/server/v3 v3.5.6 h1:RXuwaB8AMiV62TqcqIt4O4bG8NWjsxOkDJVT3MZI5Ds= go.etcd.io/etcd/server/v3 v3.5.6/go.mod h1:6/Gfe8XTGXQJgLYQ65oGKMfPivb2EASLUSMSWN9Sroo= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= @@ -1848,33 +1950,49 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0/go.mod h1:M1hVZHNxcbkAlcvrOMlpQ4YOO3Awf+4N2dxkZL3xm04= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0/go.mod h1:ceUgdyfNv4h4gLxHR0WNfDiiVmZFodZhZSbOLhpxqXE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.7.0 h1:pLP0MH4MAqeTEV0g/4flxw9O8Is48uAIauAnjznbW50= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.7.0/go.mod h1:aFXT9Ng2seM9eizF+LfKiyPBGy8xIZKwhusC1gIu3hA= go.opentelemetry.io/otel/exporters/zipkin v1.7.0 h1:X0FZj+kaIdLi29UiyrEGDhRTYsEXj9GdEW5Y39UQFEE= go.opentelemetry.io/otel/exporters/zipkin v1.7.0/go.mod h1:9YBXeOMFLQGwNEjsxMRiWPGoJX83usGMhbCmxUbNe5I= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= +go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU= go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= +go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.opentelemetry.io/proto/otlp v0.16.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -1889,6 +2007,7 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -1928,10 +2047,13 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1977,6 +2099,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= @@ -2037,12 +2160,16 @@ golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5o golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -2170,6 +2297,7 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2207,9 +2335,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2220,8 +2351,10 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2242,6 +2375,8 @@ golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2269,6 +2404,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2359,6 +2495,7 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= @@ -2469,6 +2606,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2576,6 +2714,7 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -2677,6 +2816,7 @@ k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= @@ -2685,11 +2825,14 @@ k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlm k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= k8s.io/apiserver v0.26.1 h1:6vmnAqCDO194SVCPU3MU8NcDgSqsUA62tBUSWrFXhsc= k8s.io/apiserver v0.26.1/go.mod h1:wr75z634Cv+sifswE9HlAo5FQ7UoUauIICRlOE+5dCg= k8s.io/cli-runtime v0.26.1 h1:f9+bRQ1V3elQsx37KmZy5fRAh56mVLbE9A7EMdlqVdI= @@ -2698,6 +2841,7 @@ k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= @@ -2705,6 +2849,7 @@ k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NI k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= k8s.io/component-helpers v0.26.0 h1:KNgwqs3EUdK0HLfW4GhnbD+q/Zl9U021VfIU7qoVYFk= @@ -2713,6 +2858,7 @@ k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= k8s.io/cri-api v0.25.0 h1:INwdXsCDSA/0hGNdPxdE2dQD6ft/5K1EaKXZixvSQxg= k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -2721,11 +2867,15 @@ k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-aggregator v0.19.12 h1:OwyNUe/7/gxzEnaLd3sC9Yrpx0fZAERzvFslX5Qq5g8= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/kubectl v0.26.0 h1:xmrzoKR9CyNdzxBmXV7jW9Ln8WMrwRK6hGbbf69o4T0= @@ -2735,6 +2885,8 @@ k8s.io/metrics v0.26.0 h1:U/NzZHKDrIVGL93AUMRkqqXjOah3wGvjSnKmG/5NVCs= k8s.io/metrics v0.26.0/go.mod h1:cf5MlG4ZgWaEFZrR9+sOImhZ2ICMpIdNurA+D8snIs8= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f/go.mod h1:9VQ397fNXEnF84t90W4r4TRCQK+pg9f8ugVfyj+S26w= @@ -2750,6 +2902,7 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.14.4 h1:Kd/Qgx5pd2XUL08eOV2vwIq3L9GhIbJ5Nxengbd4/0M= sigs.k8s.io/controller-runtime v0.14.4/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= @@ -2763,6 +2916,7 @@ sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From 70e7e1f38704e3bd3435a6a3c2d1461324d6545f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 18:58:11 +0800 Subject: [PATCH 289/439] chore(deps): bump github.com/cloudflare/circl from 1.1.0 to 1.3.3 (#3211) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Nash Tsai --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e750f75d4..9924c0312 100644 --- a/go.mod +++ b/go.mod @@ -133,7 +133,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/cloudflare/circl v1.1.0 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/cockroachdb/apd/v2 v2.0.1 // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/containerd/containerd v1.6.18 // indirect diff --git a/go.sum b/go.sum index c65a8ec55..5e19f57b6 100644 --- a/go.sum +++ b/go.sum @@ -450,8 +450,9 @@ github.com/clbanning/mxj/v2 v2.5.7/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e85keuznYcH5rqI438v41pKcBl4ZxQ= github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= From bc752eb6e33b6533912e73dbd4519cfa38528995 Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Sun, 14 May 2023 20:26:40 +0800 Subject: [PATCH 290/439] feat: `kbcli` support `cd list-components` (#3161) Co-authored-by: L.DongMing --- docs/user_docs/cli/cli.md | 1 + docs/user_docs/cli/kbcli_clusterdefinition.md | 1 + .../clusterdefinition/clusterdefinition.go | 1 + .../cmd/clusterdefinition/list_compoents.go | 97 +++++++++++++++++ .../clusterdefinition/list_component_test.go | 100 ++++++++++++++++++ internal/cli/testing/factory.go | 2 + 6 files changed, 202 insertions(+) create mode 100644 internal/cli/cmd/clusterdefinition/list_compoents.go create mode 100644 internal/cli/cmd/clusterdefinition/list_component_test.go diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index e55b7d539..673c42630 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -90,6 +90,7 @@ Cluster command. ClusterDefinition command. * [kbcli clusterdefinition list](kbcli_clusterdefinition_list.md) - List ClusterDefinitions. +* [kbcli clusterdefinition list-components](kbcli_clusterdefinition_list-components.md) - List cluster definition components. ## [clusterversion](kbcli_clusterversion.md) diff --git a/docs/user_docs/cli/kbcli_clusterdefinition.md b/docs/user_docs/cli/kbcli_clusterdefinition.md index 2fee0e6cb..02f8872f2 100644 --- a/docs/user_docs/cli/kbcli_clusterdefinition.md +++ b/docs/user_docs/cli/kbcli_clusterdefinition.md @@ -38,6 +38,7 @@ ClusterDefinition command. * [kbcli clusterdefinition list](kbcli_clusterdefinition_list.md) - List ClusterDefinitions. +* [kbcli clusterdefinition list-components](kbcli_clusterdefinition_list-components.md) - List cluster definition components. #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/internal/cli/cmd/clusterdefinition/clusterdefinition.go b/internal/cli/cmd/clusterdefinition/clusterdefinition.go index d217e14c4..cb723d739 100644 --- a/internal/cli/cmd/clusterdefinition/clusterdefinition.go +++ b/internal/cli/cmd/clusterdefinition/clusterdefinition.go @@ -42,6 +42,7 @@ func NewClusterDefinitionCmd(f cmdutil.Factory, streams genericclioptions.IOStre } cmd.AddCommand(NewListCmd(f, streams)) + cmd.AddCommand(NewListComponentsCmd(f, streams)) return cmd } diff --git a/internal/cli/cmd/clusterdefinition/list_compoents.go b/internal/cli/cmd/clusterdefinition/list_compoents.go new file mode 100644 index 000000000..51e1579c3 --- /dev/null +++ b/internal/cli/cmd/clusterdefinition/list_compoents.go @@ -0,0 +1,97 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package clusterdefinition + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/list" + "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var ( + listComponentsExample = templates.Examples(` + # List all components belong to the cluster definition. + kbcli clusterdefinition list-components apecloud-mysql`) +) + +func NewListComponentsCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := list.NewListOptions(f, streams, types.ClusterDefGVR()) + cmd := &cobra.Command{ + Use: "list-components", + Short: "List cluster definition components.", + Example: listComponentsExample, + Aliases: []string{"ls-comps"}, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, o.GVR), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(validate(args)) + o.FieldSelector = fmt.Sprintf("metadata.name=%s", args[0]) + util.CheckErr(run(o)) + }, + } + return cmd +} + +func validate(args []string) error { + if len(args) == 0 { + return fmt.Errorf("missing clusterdefinition name") + } else if len(args) > 1 { + return fmt.Errorf("only support one clusterdefinition name") + } + return nil +} + +func run(o *list.ListOptions) error { + o.Print = false + r, err := o.Run() + if err != nil { + return err + } + infos, err := r.Infos() + if err != nil { + return err + } + if len(infos) == 0 { + return fmt.Errorf("no clusterdefinition %s found", o.Names[0]) + } + p := printer.NewTablePrinter(o.Out) + p.SetHeader("NAME", "WORKLOAD-TYPE", "CHARACTER-TYPE") + for _, info := range infos { + var cd v1alpha1.ClusterDefinition + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(info.Object.(*unstructured.Unstructured).Object, &cd); err != nil { + return err + } + for _, comp := range cd.Spec.ComponentDefs { + p.AddRow(comp.Name, comp.WorkloadType, comp.CharacterType) + } + } + p.Print() + return nil +} diff --git a/internal/cli/cmd/clusterdefinition/list_component_test.go b/internal/cli/cmd/clusterdefinition/list_component_test.go new file mode 100644 index 000000000..e6de64304 --- /dev/null +++ b/internal/cli/cmd/clusterdefinition/list_component_test.go @@ -0,0 +1,100 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package clusterdefinition + +import ( + "bytes" + "fmt" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + "k8s.io/kubectl/pkg/scheme" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/testing" + "github.com/apecloud/kubeblocks/internal/cli/types" +) + +var _ = Describe("clusterdefinition list components", func() { + var ( + cmd *cobra.Command + streams genericclioptions.IOStreams + out *bytes.Buffer + tf *cmdtesting.TestFactory + ) + + const ( + namespace = testing.Namespace + clusterdefinitionName = testing.ClusterDefName + ) + + mockClient := func(data runtime.Object) *cmdtesting.TestFactory { + tf := testing.NewTestFactory(namespace) + codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) + tf.UnstructuredClient = &clientfake.RESTClient{ + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + GroupVersion: schema.GroupVersion{Group: types.AppsAPIGroup, Version: types.AppsAPIVersion}, + Resp: &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, data)}, + } + tf.Client = tf.UnstructuredClient + tf.FakeDynamicClient = testing.FakeDynamicClient(data) + return tf + } + + BeforeEach(func() { + _ = appsv1alpha1.AddToScheme(scheme.Scheme) + clusterDef := testing.FakeClusterDef() + tf = mockClient(clusterDef) + streams, _, out, _ = genericclioptions.NewTestIOStreams() + cmd = NewListComponentsCmd(tf, streams) + }) + + AfterEach(func() { + tf.Cleanup() + }) + + It("create list-components cmd", func() { + cmd := NewListComponentsCmd(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + + It("list-components requires a clusterdefinition Name", func() { + Expect(validate([]string{})).Should(HaveOccurred()) + }) + + It("list-components ", func() { + cmd.Run(cmd, []string{clusterdefinitionName}) + expected := `NAME WORKLOAD-TYPE CHARACTER-TYPE +fake-component-type mysql +fake-component-type-1 mysql +` + fmt.Println(out.String()) + Expect(expected).Should(Equal(out.String())) + }) +}) diff --git a/internal/cli/testing/factory.go b/internal/cli/testing/factory.go index 98de22c17..b5332fdb3 100644 --- a/internal/cli/testing/factory.go +++ b/internal/cli/testing/factory.go @@ -116,6 +116,8 @@ func testDynamicResources() []*restmapper.APIGroupResources { VersionedResources: map[string][]metav1.APIResource{ "v1alpha1": { {Name: "clusters", Namespaced: true, Kind: "Cluster"}, + {Name: "clusterdefinitions", Namespaced: false, Kind: "clusterdefinition"}, + {Name: "clusterversions", Namespaced: false, Kind: "clusterversion"}, {Name: "opsrequests", Namespaced: true, Kind: "OpsRequest"}, }, }, From 1b2d9c69d299b3b44cd6013b7fa825e5f59a8d0e Mon Sep 17 00:00:00 2001 From: wangyelei Date: Mon, 15 May 2023 10:43:55 +0800 Subject: [PATCH 291/439] fix: opsRequest phase will be incorrect when source phase of the cluster phase is Failed (#3228) --- .../apps/operations/ops_progress_util.go | 22 +++++++++---------- controllers/apps/operations/ops_util.go | 2 +- .../apps/opsrequest_controller_test.go | 16 ++++++++------ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/controllers/apps/operations/ops_progress_util.go b/controllers/apps/operations/ops_progress_util.go index 1986d0b99..8f5fb6cd1 100644 --- a/controllers/apps/operations/ops_progress_util.go +++ b/controllers/apps/operations/ops_progress_util.go @@ -224,17 +224,16 @@ func handleStatelessProgress(reqCtx intctrlutil.RequestCtx, for _, v := range podList.Items { objectKey := getProgressObjectKey(v.Kind, v.Name) progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey} - if podIsPendingDuringOperation(opsStartTime, &v, compStatus.Phase) { - handlePendingProgressDetail(opsRes, compStatus, progressDetail) - continue - } - if podProcessedSuccessful(currComponent, opsStartTime, &v, minReadySeconds, compStatus.Phase, pgRes.opsIsCompleted) { completedCount += 1 handleSucceedProgressDetail(opsRes, pgRes, compStatus, progressDetail) continue } + if podIsPendingDuringOperation(opsStartTime, &v) { + handlePendingProgressDetail(opsRes, compStatus, progressDetail) + continue + } completedCount += handleFailedOrProcessingProgressDetail(opsRes, pgRes, compStatus, progressDetail, &v) } compStatus.ProgressDetails = removeStatelessExpiredPod(podList, compStatus.ProgressDetails) @@ -266,16 +265,16 @@ func handleStatefulSetProgress(reqCtx intctrlutil.RequestCtx, for _, v := range podList.Items { objectKey := getProgressObjectKey(v.Kind, v.Name) progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey} - if podIsPendingDuringOperation(opsStartTime, &v, compStatus.Phase) { - handlePendingProgressDetail(opsRes, compStatus, progressDetail) - continue - } if podProcessedSuccessful(currComponent, opsStartTime, &v, minReadySeconds, compStatus.Phase, pgRes.opsIsCompleted) { completedCount += 1 handleSucceedProgressDetail(opsRes, pgRes, compStatus, progressDetail) continue } + if podIsPendingDuringOperation(opsStartTime, &v) { + handlePendingProgressDetail(opsRes, compStatus, progressDetail) + continue + } completedCount += handleFailedOrProcessingProgressDetail(opsRes, pgRes, compStatus, progressDetail, &v) } return completedCount, err @@ -330,9 +329,8 @@ func handleFailedOrProcessingProgressDetail(opsRes *OpsResource, } // podIsPendingDuringOperation checks if pod is pending during the component is doing operation. -func podIsPendingDuringOperation(opsStartTime metav1.Time, pod *corev1.Pod, componentPhase appsv1alpha1.ClusterComponentPhase) bool { - return pod.CreationTimestamp.Before(&opsStartTime) && pod.DeletionTimestamp.IsZero() && - !slices.Contains(appsv1alpha1.GetComponentTerminalPhases(), componentPhase) +func podIsPendingDuringOperation(opsStartTime metav1.Time, pod *corev1.Pod) bool { + return pod.CreationTimestamp.Before(&opsStartTime) && pod.DeletionTimestamp.IsZero() } // podIsFailedDuringOperation checks if pod is failed during operation. diff --git a/controllers/apps/operations/ops_util.go b/controllers/apps/operations/ops_util.go index 48cb63020..9609a04ed 100644 --- a/controllers/apps/operations/ops_util.go +++ b/controllers/apps/operations/ops_util.go @@ -125,7 +125,7 @@ func reconcileActionWithComponentOps(reqCtx intctrlutil.RequestCtx, } // TODO: judge whether ops is Failed according to whether progressDetail has failed pods. // now we check the ops is Failed by the component phase, it may be not accurate during h-scale replicas. - if isFailed { + if isFailed && opsRes.Cluster.Status.ObservedGeneration >= opsRes.OpsRequest.Status.ClusterGeneration { return appsv1alpha1.OpsFailedPhase, 0, nil } if completedProgressCount != expectProgressCount { diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index 6aee31bc9..a3f61050e 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -169,14 +169,16 @@ var _ = Describe("OpsRequest Controller", func() { clusterObj = clusterFactory.Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) - By("Waiting for the cluster enter running phase") - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) + By("Waiting for the cluster enters creating phase") Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.CreatingClusterPhase)) By("mock pod/sts are available and wait for cluster enter running phase") podName := fmt.Sprintf("%s-%s-0", clusterObj.Name, mysqlCompName) pod := testapps.MockConsensusComponentStsPod(testCtx, nil, clusterObj.Name, mysqlCompName, podName, "leader", "ReadWrite") + // the opsRequest will use startTime to check some condition. + // if there is no sleep for 1 second, unstable error may occur. + time.Sleep(time.Second) if workloadType == testapps.StatefulMySQLComponent { lastTransTime := metav1.NewTime(time.Now().Add(-1 * (defaultMinReadySeconds + 1) * time.Second)) Expect(testapps.ChangeObjStatus(&testCtx, pod, func() { @@ -211,16 +213,16 @@ var _ = Describe("OpsRequest Controller", func() { } Expect(testCtx.CreateObj(testCtx.Ctx, verticalScalingOpsRequest)).Should(Succeed()) - By("check VerticalScalingOpsRequest running") + By("wait for VerticalScalingOpsRequest is running") Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, mysqlCompName)).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) checkLatestOpsIsProcessing(clusterKey, verticalScalingOpsRequest.Spec.Type) - By("check Cluster and changed component phase is VerticalScaling") + By("cluster and component phase are Updating") Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { - g.Expect(cluster.Status.Phase).To(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) // VerticalScalingPhase - g.Expect(cluster.Status.Components[mysqlCompName].Phase).To(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) // VerticalScalingPhase + g.Expect(cluster.Status.Phase).To(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) + g.Expect(cluster.Status.Components[mysqlCompName].Phase).To(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) })).Should(Succeed()) By("mock bring Cluster and changed component back to running status") @@ -231,7 +233,7 @@ var _ = Describe("OpsRequest Controller", func() { Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) checkLatestOpsHasProcessed(clusterKey) - By("patch opsrequest controller to run") + By("notice opsrequest controller to run") Expect(testapps.ChangeObj(&testCtx, verticalScalingOpsRequest, func(lopsReq *appsv1alpha1.OpsRequest) { if lopsReq.Annotations == nil { lopsReq.Annotations = map[string]string{} From fd08a0bab850d522fde68505dec5e3b9b0cfd114 Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Mon, 15 May 2023 14:42:30 +0800 Subject: [PATCH 292/439] fix: fix unknown output format when run preflight (#3239) --- docs/user_docs/cli/kbcli_kubeblocks_preflight.md | 1 + internal/cli/cmd/kubeblocks/preflight.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/user_docs/cli/kbcli_kubeblocks_preflight.md b/docs/user_docs/cli/kbcli_kubeblocks_preflight.md index fd7b0224f..cf0f5e72d 100644 --- a/docs/user_docs/cli/kbcli_kubeblocks_preflight.md +++ b/docs/user_docs/cli/kbcli_kubeblocks_preflight.md @@ -31,6 +31,7 @@ kbcli kubeblocks preflight [flags] --collector-image string the full name of the collector image to use --collector-pullpolicy string the pull policy of the collector image --debug enable debug logging + --format string output format, one of json, yaml. only used when interactive is set to false, default format is yaml (default "yaml") -h, --help help for preflight -n, --namespace string If present, the namespace scope for this CLI request -o, --output string specify the output file path for the preflight checks diff --git a/internal/cli/cmd/kubeblocks/preflight.go b/internal/cli/cmd/kubeblocks/preflight.go index 6b58af59f..c89875b5d 100644 --- a/internal/cli/cmd/kubeblocks/preflight.go +++ b/internal/cli/cmd/kubeblocks/preflight.go @@ -58,6 +58,7 @@ const ( flagNamespace = "namespace" flagVerbose = "verbose" flagForce = "force" + flagFormat = "format" PreflightPattern = "data/%s_preflight.yaml" HostPreflightPattern = "data/%s_hostpreflight.yaml" @@ -110,6 +111,7 @@ func NewPreflightCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *co }, } // add flags + cmd.Flags().StringVar(p.Format, flagFormat, "yaml", "output format, one of json, yaml. only used when interactive is set to false, default format is yaml") cmd.Flags().StringVar(p.CollectorImage, flagCollectorImage, *p.CollectorImage, "the full name of the collector image to use") cmd.Flags().StringVar(p.CollectorPullPolicy, flagCollectorPullPolicy, *p.CollectorPullPolicy, "the pull policy of the collector image") cmd.Flags().BoolVar(p.CollectWithoutPermissions, flagCollectWithoutPermissions, *p.CollectWithoutPermissions, "always run preflight checks even if some require permissions that preflight does not have") From 2f3891d01260892c9d68372dff519969ff58e60c Mon Sep 17 00:00:00 2001 From: dingben Date: Mon, 15 May 2023 15:27:04 +0800 Subject: [PATCH 293/439] feat: support plugin manger as krew (#2981) --- docs/user_docs/cli/cli.md | 6 + docs/user_docs/cli/kbcli_plugin.md | 6 + docs/user_docs/cli/kbcli_plugin_describe.md | 56 +++ docs/user_docs/cli/kbcli_plugin_index.md | 50 +++ docs/user_docs/cli/kbcli_plugin_index_add.md | 53 +++ .../cli/kbcli_plugin_index_delete.md | 53 +++ docs/user_docs/cli/kbcli_plugin_index_list.md | 53 +++ .../cli/kbcli_plugin_index_update.md | 46 +++ docs/user_docs/cli/kbcli_plugin_install.md | 56 +++ docs/user_docs/cli/kbcli_plugin_search.md | 53 +++ docs/user_docs/cli/kbcli_plugin_uninstall.md | 53 +++ docs/user_docs/cli/kbcli_plugin_upgrade.md | 57 +++ go.mod | 8 +- go.sum | 154 -------- hack/license/header-check.sh | 3 +- internal/cli/cmd/plugin/describe.go | 81 ++++ internal/cli/cmd/plugin/download/download.go | 247 +++++++++++++ internal/cli/cmd/plugin/download/fetch.go | 57 +++ internal/cli/cmd/plugin/download/verifier.go | 55 +++ internal/cli/cmd/plugin/index.go | 255 +++++++++++++ internal/cli/cmd/plugin/install.go | 203 ++++++++++ internal/cli/cmd/plugin/pathutil.go | 346 ++++++++++++++++++ internal/cli/cmd/plugin/platform.go | 86 +++++ internal/cli/cmd/plugin/plugin.go | 35 +- internal/cli/cmd/plugin/search.go | 97 +++++ internal/cli/cmd/plugin/types.go | 184 ++++++++++ internal/cli/cmd/plugin/uninstall.go | 91 +++++ internal/cli/cmd/plugin/upgrade.go | 196 ++++++++++ internal/cli/cmd/plugin/utils.go | 242 ++++++++++++ internal/cli/spinner/windows_spinner.go | 19 + internal/cli/util/git.go | 72 ++++ 31 files changed, 2813 insertions(+), 160 deletions(-) create mode 100644 docs/user_docs/cli/kbcli_plugin_describe.md create mode 100644 docs/user_docs/cli/kbcli_plugin_index.md create mode 100644 docs/user_docs/cli/kbcli_plugin_index_add.md create mode 100644 docs/user_docs/cli/kbcli_plugin_index_delete.md create mode 100644 docs/user_docs/cli/kbcli_plugin_index_list.md create mode 100644 docs/user_docs/cli/kbcli_plugin_index_update.md create mode 100644 docs/user_docs/cli/kbcli_plugin_install.md create mode 100644 docs/user_docs/cli/kbcli_plugin_search.md create mode 100644 docs/user_docs/cli/kbcli_plugin_uninstall.md create mode 100644 docs/user_docs/cli/kbcli_plugin_upgrade.md create mode 100644 internal/cli/cmd/plugin/describe.go create mode 100755 internal/cli/cmd/plugin/download/download.go create mode 100755 internal/cli/cmd/plugin/download/fetch.go create mode 100755 internal/cli/cmd/plugin/download/verifier.go create mode 100644 internal/cli/cmd/plugin/index.go create mode 100755 internal/cli/cmd/plugin/install.go create mode 100644 internal/cli/cmd/plugin/pathutil.go create mode 100755 internal/cli/cmd/plugin/platform.go create mode 100644 internal/cli/cmd/plugin/search.go create mode 100644 internal/cli/cmd/plugin/types.go create mode 100644 internal/cli/cmd/plugin/uninstall.go create mode 100644 internal/cli/cmd/plugin/upgrade.go create mode 100644 internal/cli/cmd/plugin/utils.go diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index 673c42630..f904a7e7c 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -162,7 +162,13 @@ Provides utilities for interacting with plugins. Plugins provide extended functionality that is not part of the major command-line distribution. +* [kbcli plugin describe](kbcli_plugin_describe.md) - Describe a plugin +* [kbcli plugin index](kbcli_plugin_index.md) - Manage custom plugin indexes +* [kbcli plugin install](kbcli_plugin_install.md) - Install kbcli or kubectl plugins * [kbcli plugin list](kbcli_plugin_list.md) - List all visible plugin executables on a user's PATH +* [kbcli plugin search](kbcli_plugin_search.md) - Search kbcli or kubectl plugins +* [kbcli plugin uninstall](kbcli_plugin_uninstall.md) - Uninstall kbcli or kubectl plugins +* [kbcli plugin upgrade](kbcli_plugin_upgrade.md) - Upgrade kbcli or kubectl plugins ## [version](kbcli_version.md) diff --git a/docs/user_docs/cli/kbcli_plugin.md b/docs/user_docs/cli/kbcli_plugin.md index f9627d808..9d132e7e0 100644 --- a/docs/user_docs/cli/kbcli_plugin.md +++ b/docs/user_docs/cli/kbcli_plugin.md @@ -43,7 +43,13 @@ Provides utilities for interacting with plugins. ### SEE ALSO +* [kbcli plugin describe](kbcli_plugin_describe.md) - Describe a plugin +* [kbcli plugin index](kbcli_plugin_index.md) - Manage custom plugin indexes +* [kbcli plugin install](kbcli_plugin_install.md) - Install kbcli or kubectl plugins * [kbcli plugin list](kbcli_plugin_list.md) - List all visible plugin executables on a user's PATH +* [kbcli plugin search](kbcli_plugin_search.md) - Search kbcli or kubectl plugins +* [kbcli plugin uninstall](kbcli_plugin_uninstall.md) - Uninstall kbcli or kubectl plugins +* [kbcli plugin upgrade](kbcli_plugin_upgrade.md) - Upgrade kbcli or kubectl plugins #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/docs/user_docs/cli/kbcli_plugin_describe.md b/docs/user_docs/cli/kbcli_plugin_describe.md new file mode 100644 index 000000000..8d57c2fd7 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_describe.md @@ -0,0 +1,56 @@ +--- +title: kbcli plugin describe +--- + +Describe a plugin + +``` +kbcli plugin describe [flags] +``` + +### Examples + +``` + # Describe a plugin + kbcli plugin describe [PLUGIN] + + # Describe a plugin with index + kbcli plugin describe [INDEX/PLUGIN] +``` + +### Options + +``` + -h, --help help for describe +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_index.md b/docs/user_docs/cli/kbcli_plugin_index.md new file mode 100644 index 000000000..3a4048110 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_index.md @@ -0,0 +1,50 @@ +--- +title: kbcli plugin index +--- + +Manage custom plugin indexes + +### Synopsis + +Manage which repositories are used to discover plugins and install plugins from + +### Options + +``` + -h, --help help for index +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. +* [kbcli plugin index add](kbcli_plugin_index_add.md) - Add a new index +* [kbcli plugin index delete](kbcli_plugin_index_delete.md) - Remove a configured index +* [kbcli plugin index list](kbcli_plugin_index_list.md) - List configured indexes +* [kbcli plugin index update](kbcli_plugin_index_update.md) - update all configured indexes + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_index_add.md b/docs/user_docs/cli/kbcli_plugin_index_add.md new file mode 100644 index 000000000..052b81627 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_index_add.md @@ -0,0 +1,53 @@ +--- +title: kbcli plugin index add +--- + +Add a new index + +``` +kbcli plugin index add [flags] +``` + +### Examples + +``` + # Add a new plugin index + kbcli plugin index add myIndex +``` + +### Options + +``` + -h, --help help for add +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin index](kbcli_plugin_index.md) - Manage custom plugin indexes + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_index_delete.md b/docs/user_docs/cli/kbcli_plugin_index_delete.md new file mode 100644 index 000000000..47c790555 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_index_delete.md @@ -0,0 +1,53 @@ +--- +title: kbcli plugin index delete +--- + +Remove a configured index + +``` +kbcli plugin index delete [flags] +``` + +### Examples + +``` + # Delete a plugin index + kbcli plugin index delete myIndex +``` + +### Options + +``` + -h, --help help for delete +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin index](kbcli_plugin_index.md) - Manage custom plugin indexes + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_index_list.md b/docs/user_docs/cli/kbcli_plugin_index_list.md new file mode 100644 index 000000000..1931e4d0a --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_index_list.md @@ -0,0 +1,53 @@ +--- +title: kbcli plugin index list +--- + +List configured indexes + +``` +kbcli plugin index list [flags] +``` + +### Examples + +``` + # List all configured plugin indexes + kbcli plugin index list +``` + +### Options + +``` + -h, --help help for list +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin index](kbcli_plugin_index.md) - Manage custom plugin indexes + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_index_update.md b/docs/user_docs/cli/kbcli_plugin_index_update.md new file mode 100644 index 000000000..0ab0caafd --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_index_update.md @@ -0,0 +1,46 @@ +--- +title: kbcli plugin index update +--- + +update all configured indexes + +``` +kbcli plugin index update [flags] +``` + +### Options + +``` + -h, --help help for update +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin index](kbcli_plugin_index.md) - Manage custom plugin indexes + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_install.md b/docs/user_docs/cli/kbcli_plugin_install.md new file mode 100644 index 000000000..54933733f --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_install.md @@ -0,0 +1,56 @@ +--- +title: kbcli plugin install +--- + +Install kbcli or kubectl plugins + +``` +kbcli plugin install [flags] +``` + +### Examples + +``` + # install a kbcli or kubectl plugin by name + kbcli plugin install [PLUGIN] + + # install a kbcli or kubectl plugin by name and index + kbcli plugin install [INDEX/PLUGIN] +``` + +### Options + +``` + -h, --help help for install +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_search.md b/docs/user_docs/cli/kbcli_plugin_search.md new file mode 100644 index 000000000..e54626591 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_search.md @@ -0,0 +1,53 @@ +--- +title: kbcli plugin search +--- + +Search kbcli or kubectl plugins + +``` +kbcli plugin search [flags] +``` + +### Examples + +``` + # search a kbcli or kubectl plugin by name + kbcli plugin search myplugin +``` + +### Options + +``` + -h, --help help for search +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_uninstall.md b/docs/user_docs/cli/kbcli_plugin_uninstall.md new file mode 100644 index 000000000..f96801fa4 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_uninstall.md @@ -0,0 +1,53 @@ +--- +title: kbcli plugin uninstall +--- + +Uninstall kbcli or kubectl plugins + +``` +kbcli plugin uninstall [flags] +``` + +### Examples + +``` + # uninstall a kbcli or kubectl plugin by name + kbcli plugin uninstall [PLUGIN] +``` + +### Options + +``` + -h, --help help for uninstall +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_plugin_upgrade.md b/docs/user_docs/cli/kbcli_plugin_upgrade.md new file mode 100644 index 000000000..3ee5e5902 --- /dev/null +++ b/docs/user_docs/cli/kbcli_plugin_upgrade.md @@ -0,0 +1,57 @@ +--- +title: kbcli plugin upgrade +--- + +Upgrade kbcli or kubectl plugins + +``` +kbcli plugin upgrade [flags] +``` + +### Examples + +``` + # upgrade installed plugins with specified name + kbcli plugin upgrade myplugin + + # upgrade installed plugin to a newer version + kbcli plugin upgrade --all +``` + +### Options + +``` + --all Upgrade all installed plugins + -h, --help help for upgrade +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/go.mod b/go.mod index 9924c0312..1ff04d4d9 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/briandowns/spinner v1.23.0 github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230423031423-0b31a519b502 github.com/clbanning/mxj/v2 v2.5.7 + github.com/cockroachdb/errors v1.2.4 github.com/containerd/stargz-snapshotter/estargz v0.13.0 github.com/containers/common v0.49.1 github.com/dapr/components-contrib v1.9.6 @@ -48,6 +49,7 @@ require ( github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 github.com/onsi/ginkgo/v2 v2.9.1 github.com/onsi/gomega v1.27.4 + github.com/opencontainers/image-spec v1.1.0-rc2 github.com/pingcap/go-tpc v1.0.9 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 @@ -117,8 +119,8 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect - github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect github.com/ahmetalpbalkan/go-cursor v0.0.0-20131010032410-8136607ea412 // indirect + github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect @@ -130,11 +132,13 @@ require ( github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/cockroachdb/apd/v2 v2.0.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/containerd/containerd v1.6.18 // indirect github.com/containers/image/v5 v5.24.0 // indirect @@ -165,6 +169,7 @@ require ( github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fvbommel/sortorder v1.0.2 // indirect + github.com/getsentry/raven-go v0.2.0 // indirect github.com/go-errors/errors v1.4.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.0 // indirect @@ -276,7 +281,6 @@ require ( github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.5 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opencontainers/selinux v1.10.2 // indirect diff --git a/go.sum b/go.sum index 5e19f57b6..2cf78cd69 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,4 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -194,8 +193,6 @@ cuelang.org/go v0.4.3 h1:W3oBBjDTm7+IZfCKZAmC8uDG0eYfJL4Pp/xbbCMKaVo= cuelang.org/go v0.4.3/go.mod h1:7805vR9H+VoBNdWFdI7jyDR3QLUPp4+naHfbcgp55HI= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 h1:SCbEWT58NSt7d2mcFdvxC9uyrdcTfvBbPLThhkDmXzg= -github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a h1:XVdatQFSP2YhJGjqLLIfW8QBk4loz/SCe/PxkXDiW+s= github.com/AdhityaRamadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a/go.mod h1:C0A1KeiVHs+trY6gUTPhhGammbrZ30ZfXRW/nuT7HLw= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= @@ -206,7 +203,6 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 h1:jp0dGvZ7ZK0mgqnTSClMxa5 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -215,13 +211,11 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4= @@ -239,12 +233,10 @@ github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQX github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= -github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -276,7 +268,6 @@ github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugX github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= @@ -287,20 +278,15 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY= github.com/Microsoft/hcsshim v0.9.6/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= -github.com/ProtonMail/go-crypto v0.0.0-20220407094043-a94812496cf5/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I= github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -317,8 +303,6 @@ github.com/Shopify/toxiproxy/v2 v2.1.6-0.20210914104332-15ea381dcdae/go.mod h1:/ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StudioSol/set v1.0.0 h1:G27J71la+Da08WidabBkoRrvPLTa4cdCn0RjvyJ5WKQ= github.com/StudioSol/set v1.0.0/go.mod h1:hIUNZPo6rEGF43RlPXHq7Fjmf+HkVJBqAjtK7Z9LoIU= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/agrea/ptr v0.0.0-20180711073057-77a518d99b7b h1:WMhlIaJkDgEQSVJQM06YV+cYUl1r5OY5//ijMXJNqtA= @@ -332,7 +316,6 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -359,7 +342,6 @@ github.com/authzed/controller-idioms v0.7.0/go.mod h1:0B/PmqCguKv8b3azSMF+HdyKpK github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.44.122 h1:p6mw01WBaNpbdP2xrisz5tIkcNwzj/HysobNoaAHjgo= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -408,7 +390,6 @@ github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f/go.mod h1:uEyr4WpAH github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -486,7 +467,6 @@ github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4S github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= @@ -503,16 +483,12 @@ github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= -github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= -github.com/containerd/containerd v1.6.3/go.mod h1:gCVGrYRYFm2E8GmuUIbj/NGD7DLZQLzSJQazjVKDOig= github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns= github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -522,7 +498,6 @@ github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cE github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= @@ -532,9 +507,6 @@ github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1S github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= -github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-cni v1.1.4/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= @@ -544,13 +516,10 @@ github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= -github.com/containerd/imgcrypt v1.1.3/go.mod h1:/TPA1GIDXMzbj01yd8pIbQiLdQxed5ue1wb8bP7PQu4= -github.com/containerd/imgcrypt v1.1.4/go.mod h1:LorQnPtzL/T0IyCeftcsMEO7AqxUDbdO8j/tSUpgxvo= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= -github.com/containerd/stargz-snapshotter/estargz v0.11.4/go.mod h1:7vRJIcImfY8bpifnMjt+HTJoQxASq7T28MYbP15/Nf0= github.com/containerd/stargz-snapshotter/estargz v0.13.0 h1:fD7AwuVV+B40p0d9qVkH/Au1qhp8hn/HWJHIYjpEcfw= github.com/containerd/stargz-snapshotter/estargz v0.13.0/go.mod h1:m+9VaGJGlhCnrcEUod8mYumTmRgblwd3rC5UCEh2Yp0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= @@ -571,31 +540,19 @@ github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNR github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= -github.com/containernetworking/cni v1.1.0/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= -github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= -github.com/containers/common v0.48.1 h1:Te1lfQt1TDyGjig+Equ0j1LOFV1VuuObz/WnB7J151M= -github.com/containers/common v0.48.1/go.mod h1:zPLZCfLXfnd1jI0QRsD4By54fP4k1+ifQs+tulIe3o0= github.com/containers/common v0.49.1 h1:6y4/s2WwYxrv+Cox7fotOo316wuZI+iKKPUQweCYv50= github.com/containers/common v0.49.1/go.mod h1:ueM5hT0itKqCQvVJDs+EtjornAQtrHYxQJzP2gxeGIg= -github.com/containers/image/v5 v5.21.1/go.mod h1:zl35egpcDQa79IEXIuoUe1bW+D1pdxRxYjNlyb3YiXw= github.com/containers/image/v5 v5.24.0 h1:2Pu8ztTntqNxteVN15bORCQnM8rfnbYuyKwUiiKUBuc= github.com/containers/image/v5 v5.24.0/go.mod h1:oss5F6ssGQz8ZtC79oY+fuzYA3m3zBek9tq9gmhuvHc= -github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/containers/ocicrypt v1.1.3/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= -github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f/go.mod h1:xpdkbVAuaH3WzbEabUd5yDsl9SwJA5pABH85425Es2g= github.com/containers/ocicrypt v1.1.7 h1:thhNr4fu2ltyGz8aMx8u48Ae0Pnbip3ePP9/mzkZ/3U= github.com/containers/ocicrypt v1.1.7/go.mod h1:7CAhjcj2H8AYp5YvEie7oVSK2AhBY8NscCYRawuDNtw= -github.com/containers/storage v1.40.0/go.mod h1:zUyPC3CFIGR1OhY1CKkffxgw9+LuH76PGvVcFj38dgs= github.com/containers/storage v1.45.3 h1:GbtTvTtp3GW2/tcFg5VhgHXcYMwVn2KfZKiHjf9FAOM= github.com/containers/storage v1.45.3/go.mod h1:OdRUYHrq1HP6iAo79VxqtYuJzC5j4eA2I60jKOoCT7g= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -603,7 +560,6 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -620,7 +576,6 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -635,7 +590,6 @@ github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4= -github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= github.com/dapr/components-contrib v1.9.6 h1:C12fnZhNim8AsOzTCsWZDZ0MXqsaG161fStVuVAQgN4= github.com/dapr/components-contrib v1.9.6/go.mod h1:U0cjxEEbZR7sNN9i1ZdWnkIOZP8iRSvoyF2gRhBaHfc= github.com/dapr/dapr v1.9.5 h1:oUFpy8+Z1lBS0XYKBCGtFa+GOEf0EDnFU26Rqpwalqc= @@ -660,7 +614,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/disiqueira/gotree/v3 v3.0.2/go.mod h1:ZuyjE4+mUQZlbpkI24AmruZKhg3VHEgPLDY8Qk+uUu8= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= @@ -670,15 +623,12 @@ github.com/docker/cli v20.10.24+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hH github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE= github.com/docker/docker v20.10.24+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0= @@ -731,7 +681,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= @@ -752,7 +701,6 @@ github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -805,7 +753,6 @@ github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNV github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= @@ -823,13 +770,11 @@ github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuA github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= @@ -839,7 +784,6 @@ github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqb github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI= @@ -1018,7 +962,6 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= @@ -1049,8 +992,6 @@ github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMd github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -1060,7 +1001,6 @@ github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= @@ -1160,9 +1100,7 @@ github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= -github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= @@ -1192,7 +1130,6 @@ github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuT github.com/jhump/protoreflect v1.13.0 h1:zrrZqa7JAc2YGgPSzZZkmUXJ5G6NRPdxOg/9t7ISImA= github.com/jhump/protoreflect v1.13.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= -github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE= github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc= @@ -1246,15 +1183,12 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid/v2 v2.0.14 h1:QRqdp6bb9M9S5yyKeYteXKuoKE4p0tGlra81fKOpWH8= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 h1:BcxbplxjtczA1a6d3wYoa7a0WL3rq9DKBMGHeKyjEF0= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1318,7 +1252,6 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= @@ -1425,14 +1358,10 @@ github.com/moby/sys/mount v0.3.0/go.mod h1:U2Z3ur2rXPFrFmy4q6WMwWrBOAQGYtYTRVM8B github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= -github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1463,7 +1392,6 @@ github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5w github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/networkplumbing/go-nft v0.2.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.0.0-20200811152831-6cf413ae40e0/go.mod h1:wBEpHwM2OdmeNpdCvRPUlkEbBuaFmcK4Wv8Q7FuGW3c= github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d h1:x3S6kxmy49zXVVyhcnrFqxvNVCBPb2KZ9hV2RBdS840= @@ -1487,12 +1415,9 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1503,10 +1428,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -1517,10 +1439,7 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2-0.20211117181255-693428a734f5/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84/go.mod h1:Qnt1q4cjDNQI9bT832ziho5Iw2BhK8o1KwLOwW56VP4= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -1529,8 +1448,6 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -1541,12 +1458,10 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.m github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.2 h1:NFy2xCsjn7+WspbfZkUd5zyVeisV7VFbPSP96+8/ha4= github.com/opencontainers/selinux v1.10.2/go.mod h1:cARutUbaUrlRClyvxOICCgKixCs6L05aUsohzA3EkHQ= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -1588,7 +1503,6 @@ github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1 h1:oL4IBbcqwhhNWh31bjOX8 github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/proglottis/gpgme v0.1.1/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0-pre1.0.20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1665,7 +1579,6 @@ github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XF github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rubenv/sql-migrate v1.2.0 h1:fOXMPLMd41sK7Tg75SXDec15k3zg5WNV6SjuDRiNfcU= github.com/rubenv/sql-migrate v1.2.0/go.mod h1:Z5uVnq7vrIrPmHbVFfR4YLHRZquxeHpckCnRq0P/K9Y= -github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1673,7 +1586,6 @@ github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUcc github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= -github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d h1:Q+gqLBOPkFGHyCJxXMRqtUgUbTjI8/Ze8vu8GGyNFwo= github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4= @@ -1683,14 +1595,11 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sebdah/goldie v1.0.0 h1:9GNhIat69MSlz/ndaBg48vl9dF5fI+NBB6kfOxgfkMc= github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= -github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/securego/gosec/v2 v2.4.0/go.mod h1:0/Q4cjmlFDfDUj1+Fib61sc+U5IQb2w+Iv9/C3wPVko= github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -1743,7 +1652,6 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -1792,7 +1700,6 @@ github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/sykesm/zap-logfmt v0.0.4 h1:U2WzRvmIWG1wDLCFY3sz8UeEmsdHQjHFNlIdmroVFaI= github.com/sykesm/zap-logfmt v0.0.4/go.mod h1:AuBd9xQjAe3URrWT1BBDk2v2onAZHkZkWRMiYZXiZWA= -github.com/sylabs/sif/v2 v2.7.0/go.mod h1:TiyBWsgWeh5yBeQFNuQnvROwswqK7YJT8JA1L53bsXQ= github.com/sylabs/sif/v2 v2.9.0 h1:q9K92j1QW4/QLOtKh9YZpJHrXav6x15AVhQGPVLcg+4= github.com/sylabs/sif/v2 v2.9.0/go.mod h1:bRdFzcqif0eDjwx0isG4cgTFoKTQn/vfBXVSoP2rB2Y= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1820,7 +1727,6 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= @@ -1845,15 +1751,12 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= -github.com/vbauerster/mpb/v7 v7.4.1/go.mod h1:Ygg2mV9Vj9sQBWqsK2m2pidcf9H3s6bNKtqd3/M4gBo= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= @@ -1893,7 +1796,6 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -1926,16 +1828,12 @@ go.etcd.io/etcd/client/pkg/v3 v3.5.6/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v2 v2.305.6 h1:fIDR0p4KMjw01MJMfUIDWdQbjo06PD6CeYM5z4EHLi0= go.etcd.io/etcd/client/v2 v2.305.6/go.mod h1:BHha8XJGe8vCIBfWBpbBLVZ4QjOIlfoouvOwydu63E0= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= go.etcd.io/etcd/client/v3 v3.5.6 h1:coLs69PWCXE9G4FKquzNaSHrRyMCAXwF+IX1tAPVO8E= go.etcd.io/etcd/client/v3 v3.5.6/go.mod h1:f6GRinRMCsFVv9Ht42EyY7nfsVGwrNO0WEoS2pRKzQk= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/pkg/v3 v3.5.6 h1:k1GZrGrfMHy5/cg2bxNGsmLTFisatyhDYCFLRuaavWg= go.etcd.io/etcd/pkg/v3 v3.5.6/go.mod h1:qATwUzDb6MLyGWq2nUj+jwXqZJcxkCuabh0P7Cuff3k= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/raft/v3 v3.5.6 h1:tOmx6Ym6rn2GpZOrvTGJZciJHek6RnC3U/zNInzIN50= go.etcd.io/etcd/raft/v3 v3.5.6/go.mod h1:wL8kkRGx1Hp8FmZUuHfL3K2/OaGIDaXGr1N7i2G07J0= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.etcd.io/etcd/server/v3 v3.5.6 h1:RXuwaB8AMiV62TqcqIt4O4bG8NWjsxOkDJVT3MZI5Ds= go.etcd.io/etcd/server/v3 v3.5.6/go.mod h1:6/Gfe8XTGXQJgLYQ65oGKMfPivb2EASLUSMSWN9Sroo= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= @@ -1951,49 +1849,33 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 h1:xFSRQBbXF6VvYRf2lqMJXxoB72XI1K/azav8TekHHSw= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0/go.mod h1:h8TWwRAhQpOd0aM5nYsRD8+flnkj+526GEIVlarH7eY= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= go.opentelemetry.io/otel v1.10.0 h1:Y7DTJMR6zs1xkS/upamJYk0SxxN4C9AqRd77jmZnyY4= go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0/go.mod h1:M1hVZHNxcbkAlcvrOMlpQ4YOO3Awf+4N2dxkZL3xm04= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 h1:TaB+1rQhddO1sF71MpZOZAuSPW1klK2M8XxfrBMfK7Y= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0/go.mod h1:78XhIg8Ht9vR4tbLNUhXsiOnE2HOuSeKAiAcoVQEpOY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0/go.mod h1:ceUgdyfNv4h4gLxHR0WNfDiiVmZFodZhZSbOLhpxqXE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 h1:pDDYmo0QadUPal5fwXoY1pmMpFcdyhXOmL5drCrI3vU= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0/go.mod h1:Krqnjl22jUJ0HgMzw5eveuCvFDXY4nSYb4F8t5gdrag= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 h1:KtiUEhQmj/Pa874bVYKGNVdq8NPKiacPbaRRtgXi+t4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0/go.mod h1:OfUCyyIiDvNXHWpcWgbF+MWvqPZiNa3YDEnivcnYsV0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.7.0 h1:pLP0MH4MAqeTEV0g/4flxw9O8Is48uAIauAnjznbW50= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.7.0/go.mod h1:aFXT9Ng2seM9eizF+LfKiyPBGy8xIZKwhusC1gIu3hA= go.opentelemetry.io/otel/exporters/zipkin v1.7.0 h1:X0FZj+kaIdLi29UiyrEGDhRTYsEXj9GdEW5Y39UQFEE= go.opentelemetry.io/otel/exporters/zipkin v1.7.0/go.mod h1:9YBXeOMFLQGwNEjsxMRiWPGoJX83usGMhbCmxUbNe5I= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= -go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU= go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU= go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v1.10.0 h1:npQMbR8o7mum8uF95yFbOEJffhs1sbCOfDh8zAJiH5E= go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= -go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.opentelemetry.io/proto/otlp v0.16.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -2008,7 +1890,6 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -2048,13 +1929,10 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -2100,7 +1978,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= @@ -2161,16 +2038,12 @@ golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5o golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -2298,7 +2171,6 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2336,12 +2208,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2352,10 +2221,8 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2376,8 +2243,6 @@ golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2405,7 +2270,6 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2496,7 +2360,6 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= @@ -2607,7 +2470,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2715,7 +2577,6 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -2817,7 +2678,6 @@ k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.22.5/go.mod h1:mEhXyLaSD1qTOf40rRiKXkc+2iCem09rWLlFwhCEiAs= k8s.io/api v0.26.1 h1:f+SWYiPd/GsiWwVRz+NbFyCgvv75Pk9NK6dlkZgpCRQ= k8s.io/api v0.26.1/go.mod h1:xd/GBNgR0f707+ATNyPmQ1oyKSgndzXij81FzWGsejg= k8s.io/apiextensions-apiserver v0.26.1 h1:cB8h1SRk6e/+i3NOrQgSFij1B2S0Y0wDoNl66bn8RMI= @@ -2826,14 +2686,11 @@ k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlm k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.22.5/go.mod h1:xziclGKwuuJ2RM5/rSFQSYAj0zdbci3DH8kj+WvyN0U= k8s.io/apimachinery v0.26.1 h1:8EZ/eGJL+hY/MYCNwhmDzVqq2lPl3N3Bo8rvweJwXUQ= k8s.io/apimachinery v0.26.1/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= -k8s.io/apiserver v0.22.5/go.mod h1:s2WbtgZAkTKt679sYtSudEQrTGWUSQAPe6MupLnlmaQ= k8s.io/apiserver v0.26.1 h1:6vmnAqCDO194SVCPU3MU8NcDgSqsUA62tBUSWrFXhsc= k8s.io/apiserver v0.26.1/go.mod h1:wr75z634Cv+sifswE9HlAo5FQ7UoUauIICRlOE+5dCg= k8s.io/cli-runtime v0.26.1 h1:f9+bRQ1V3elQsx37KmZy5fRAh56mVLbE9A7EMdlqVdI= @@ -2842,7 +2699,6 @@ k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= -k8s.io/client-go v0.22.5/go.mod h1:cs6yf/61q2T1SdQL5Rdcjg9J1ElXSwbjSrW2vFImM4Y= k8s.io/client-go v0.26.1 h1:87CXzYJnAMGaa/IDDfRdhTzxk/wzGZ+/HUQpqgVSZXU= k8s.io/client-go v0.26.1/go.mod h1:IWNSglg+rQ3OcvDkhY6+QLeasV4OYHDjdqeWkDQZwGE= k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= @@ -2850,7 +2706,6 @@ k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NI k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/component-base v0.22.5/go.mod h1:VK3I+TjuF9eaa+Ln67dKxhGar5ynVbwnGrUiNF4MqCI= k8s.io/component-base v0.26.1 h1:4ahudpeQXHZL5kko+iDHqLj/FSGAEUnSVO0EBbgDd+4= k8s.io/component-base v0.26.1/go.mod h1:VHrLR0b58oC035w6YQiBSbtsf0ThuSwXP+p5dD/kAWU= k8s.io/component-helpers v0.26.0 h1:KNgwqs3EUdK0HLfW4GhnbD+q/Zl9U021VfIU7qoVYFk= @@ -2859,7 +2714,6 @@ k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= -k8s.io/cri-api v0.23.1/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= k8s.io/cri-api v0.25.0 h1:INwdXsCDSA/0hGNdPxdE2dQD6ft/5K1EaKXZixvSQxg= k8s.io/cri-api v0.25.0/go.mod h1:J1rAyQkSJ2Q6I+aBMOVgg2/cbbebso6FNa0UagiR0kc= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -2868,15 +2722,11 @@ k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAE k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-aggregator v0.19.12 h1:OwyNUe/7/gxzEnaLd3sC9Yrpx0fZAERzvFslX5Qq5g8= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= -k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/kubectl v0.26.0 h1:xmrzoKR9CyNdzxBmXV7jW9Ln8WMrwRK6hGbbf69o4T0= @@ -2886,8 +2736,6 @@ k8s.io/metrics v0.26.0 h1:U/NzZHKDrIVGL93AUMRkqqXjOah3wGvjSnKmG/5NVCs= k8s.io/metrics v0.26.0/go.mod h1:cf5MlG4ZgWaEFZrR9+sOImhZ2ICMpIdNurA+D8snIs8= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mvdan.cc/gofumpt v0.0.0-20200709182408-4fd085cb6d5f/go.mod h1:9VQ397fNXEnF84t90W4r4TRCQK+pg9f8ugVfyj+S26w= @@ -2903,7 +2751,6 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.14.4 h1:Kd/Qgx5pd2XUL08eOV2vwIq3L9GhIbJ5Nxengbd4/0M= sigs.k8s.io/controller-runtime v0.14.4/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= @@ -2917,7 +2764,6 @@ sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/hack/license/header-check.sh b/hack/license/header-check.sh index 90216cfe2..31c19d0fc 100755 --- a/hack/license/header-check.sh +++ b/hack/license/header-check.sh @@ -26,8 +26,9 @@ set -e -o pipefail # Initialize vars ERR=false FAIL=false +EXCLUDES_DIR="vendor\|internal/cli/cmd/plugin/download" -for file in $(git ls-files | grep '\.cue\|\.go$' | grep -v vendor/); do +for file in $(git ls-files | grep '\.cue\|\.go$' | grep -v $EXCLUDES_DIR); do echo -n "Header check: $file... " if [[ -z $(cat ${file} | grep "Copyright (C) 2022-2023 ApeCloud Co., Ltd\|Code generated by") ]]; then ERR=true diff --git a/internal/cli/cmd/plugin/describe.go b/internal/cli/cmd/plugin/describe.go new file mode 100644 index 000000000..0df572b3e --- /dev/null +++ b/internal/cli/cmd/plugin/describe.go @@ -0,0 +1,81 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "fmt" + "io" + + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" +) + +var pluginDescribeExample = templates.Examples(` + # Describe a plugin + kbcli plugin describe [PLUGIN] + + # Describe a plugin with index + kbcli plugin describe [INDEX/PLUGIN] + `) + +func NewPluginDescribeCmd(streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "describe", + Short: "Describe a plugin", + Example: pluginDescribeExample, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(printPluginInfo(streams.Out, args[0])) + }, + } + return cmd +} + +func printPluginInfo(out io.Writer, name string) error { + indexName, pluginName := CanonicalPluginName(name) + plugin, err := LoadPluginByName(paths.IndexPluginsPath(indexName), pluginName) + if err != nil { + return err + } + + fmt.Fprintf(out, "NAME: %s\n", plugin.Name) + fmt.Fprintf(out, "INDEX: %s\n", indexName) + if platform, ok, err := GetMatchingPlatform(plugin.Spec.Platforms); err == nil && ok { + if platform.URI != "" { + fmt.Fprintf(out, "URI: %s\n", platform.URI) + fmt.Fprintf(out, "SHA256: %s\n", platform.Sha256) + } + } + if plugin.Spec.Version != "" { + fmt.Fprintf(out, "VERSION: %s\n", plugin.Spec.Version) + } + if plugin.Spec.Homepage != "" { + fmt.Fprintf(out, "HOMEPAGE: %s\n", plugin.Spec.Homepage) + } + if plugin.Spec.Description != "" { + fmt.Fprintf(out, "DESCRIPTION: \n%s\n", plugin.Spec.Description) + } + if plugin.Spec.Caveats != "" { + fmt.Fprintf(out, "CAVEATS:\n%s\n", indent(plugin.Spec.Caveats)) + } + return nil +} diff --git a/internal/cli/cmd/plugin/download/download.go b/internal/cli/cmd/plugin/download/download.go new file mode 100755 index 000000000..b58e40caf --- /dev/null +++ b/internal/cli/cmd/plugin/download/download.go @@ -0,0 +1,247 @@ +// Copyright 2019 The Kubernetes Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package download + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "k8s.io/klog/v2" +) + +// download gets a file from the internet in memory and writes it content +// to a Verifier. +func download(url string, verifier Verifier, fetcher Fetcher) (io.ReaderAt, int64, error) { + body, err := fetcher.Get(url) + if err != nil { + return nil, 0, errors.Wrapf(err, "failed to obtain plugin archive") + } + defer body.Close() + + klog.V(3).Infof("Reading archive file into memory") + data, err := io.ReadAll(io.TeeReader(body, verifier)) + if err != nil { + return nil, 0, errors.Wrap(err, "could not read archive") + } + klog.V(2).Infof("Read %d bytes from archive into memory", len(data)) + + return bytes.NewReader(data), int64(len(data)), verifier.Verify() +} + +// extractZIP extracts a zip file into the target directory. +func extractZIP(targetDir string, read io.ReaderAt, size int64) error { + klog.V(4).Infof("Extracting zip archive to %q", targetDir) + zipReader, err := zip.NewReader(read, size) + if err != nil { + return err + } + + for _, f := range zipReader.File { + if err := suspiciousPath(f.Name); err != nil { + return err + } + + path := filepath.Join(targetDir, filepath.FromSlash(f.Name)) + if f.FileInfo().IsDir() { + if err := os.MkdirAll(path, f.Mode()); err != nil { + return errors.Wrap(err, "can't create directory tree") + } + continue + } + + dir := filepath.Dir(path) + klog.V(4).Infof("zip: ensuring parent dirs exist for regular file, dir=%s", dir) + if err := os.MkdirAll(dir, 0o755); err != nil { + return errors.Wrap(err, "failed to create directory for zip entry") + } + src, err := f.Open() + if err != nil { + return errors.Wrap(err, "could not open inflating zip file") + } + + dst, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, f.Mode()) + if err != nil { + src.Close() + return errors.Wrap(err, "can't create file in zip destination dir") + } + closeAll := func() { + src.Close() + dst.Close() + } + + if _, err := io.Copy(dst, src); err != nil { + closeAll() + return errors.Wrap(err, "can't copy content to zip destination file") + } + closeAll() + } + + return nil +} + +// extractTARGZ extracts a gzipped tar file into the target directory. +func extractTARGZ(targetDir string, at io.ReaderAt, size int64) error { + klog.V(4).Infof("tar: extracting to %q", targetDir) + in := io.NewSectionReader(at, 0, size) + + gzr, err := gzip.NewReader(in) + if err != nil { + return errors.Wrap(err, "failed to create gzip reader") + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return errors.Wrap(err, "tar extraction error") + } + klog.V(4).Infof("tar: processing %q (type=%d, mode=%s)", hdr.Name, hdr.Typeflag, os.FileMode(hdr.Mode)) + // see https://golang.org/cl/78355 for handling pax_global_header + if hdr.Name == "pax_global_header" { + klog.V(4).Infof("tar: skipping pax_global_header file") + continue + } + + if err := suspiciousPath(hdr.Name); err != nil { + return err + } + + path := filepath.Join(targetDir, filepath.FromSlash(hdr.Name)) + switch hdr.Typeflag { + case tar.TypeDir: + if err := os.MkdirAll(path, os.FileMode(hdr.Mode)); err != nil { + return errors.Wrap(err, "failed to create directory from tar") + } + case tar.TypeReg: + dir := filepath.Dir(path) + klog.V(4).Infof("tar: ensuring parent dirs exist for regular file, dir=%s", dir) + if err := os.MkdirAll(dir, 0o755); err != nil { + return errors.Wrap(err, "failed to create directory for tar") + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, os.FileMode(hdr.Mode)) + if err != nil { + return errors.Wrapf(err, "failed to create file %q", path) + } + + if _, err := io.Copy(f, tr); err != nil { + f.Close() + return errors.Wrapf(err, "failed to copy %q from tar into file", hdr.Name) + } + f.Close() + default: + return errors.Errorf("unable to handle file type %d for %q in tar", hdr.Typeflag, hdr.Name) + } + klog.V(4).Infof("tar: processed %q", hdr.Name) + } + klog.V(4).Infof("tar extraction to %s complete", targetDir) + return nil +} + +func suspiciousPath(path string) error { + if strings.Contains(path, "..") { + return errors.Errorf("refusing to unpack archive with suspicious entry %q", path) + } + + if strings.HasPrefix(path, `/`) || strings.HasPrefix(path, `\`) { + return errors.Errorf("refusing to unpack archive with absolute entry %q", path) + } + + return nil +} + +func detectMIMEType(at io.ReaderAt) (string, error) { + buf := make([]byte, 512) + n, err := at.ReadAt(buf, 0) + if err != nil && err != io.EOF { + return "", errors.Wrap(err, "failed to read first 512 bytes") + } + if n < 512 { + klog.V(5).Infof("Did only read %d of 512 bytes to determine the file type", n) + } + + // Cut off mime extra info beginning with ';' i.e: + // "text/plain; charset=utf-8" should result in "text/plain". + return strings.Split(http.DetectContentType(buf[:n]), ";")[0], nil +} + +type extractor func(targetDir string, read io.ReaderAt, size int64) error + +var defaultExtractors = map[string]extractor{ + "application/zip": extractZIP, + "application/x-gzip": extractTARGZ, +} + +func extractArchive(dst string, at io.ReaderAt, size int64) error { + t, err := detectMIMEType(at) + if err != nil { + return errors.Wrap(err, "failed to determine content type") + } + klog.V(4).Infof("detected %q file type", t) + exf, ok := defaultExtractors[t] + if !ok { + return errors.Errorf("mime type %q for archive file is not a supported archive format", t) + } + return errors.Wrap(exf(dst, at, size), "failed to extract file") +} + +// Downloader is responsible for fetching, verifying and extracting a binary. +type Downloader struct { + verifier Verifier + fetcher Fetcher +} + +// NewDownloader builds a new Downloader. +func NewDownloader(v Verifier, f Fetcher) Downloader { + return Downloader{ + verifier: v, + fetcher: f, + } +} + +// Get pulls the uri and verifies it. On success, the download gets extracted +// into dst. +func (d Downloader) Get(uri, dst string) error { + body, size, err := download(uri, d.verifier, d.fetcher) + if err != nil { + return err + } + return extractArchive(dst, body, size) +} + +// DownloadAndExtract downloads the specified archive uri (or uses the provided overrideFile, if a non-empty value) +// while validating its checksum with the provided sha256sum, and extracts its contents to extractDir that must be. +// created. +func DownloadAndExtract(extractDir, uri, sha256sum, overrideFile string) error { + var fetcher Fetcher = HTTPFetcher{} + if overrideFile != "" { + fetcher = NewFileFetcher(overrideFile) + } + + verifier := NewSha256Verifier(sha256sum) + err := NewDownloader(verifier, fetcher).Get(uri, extractDir) + return errors.Wrap(err, "failed to unpack the plugin archive") +} diff --git a/internal/cli/cmd/plugin/download/fetch.go b/internal/cli/cmd/plugin/download/fetch.go new file mode 100755 index 000000000..9dd922de4 --- /dev/null +++ b/internal/cli/cmd/plugin/download/fetch.go @@ -0,0 +1,57 @@ +// Copyright 2019 The Kubernetes Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package download + +import ( + "io" + "net/http" + "os" + + "github.com/pkg/errors" + "k8s.io/klog/v2" +) + +type Fetcher interface { + // Get gets the file and returns an stream to read the file. + Get(uri string) (io.ReadCloser, error) +} + +var _ Fetcher = HTTPFetcher{} + +// HTTPFetcher is used to get a file from a http:// or https:// schema path. +type HTTPFetcher struct{} + +// Get gets the file and returns an stream to read the file. +func (HTTPFetcher) Get(uri string) (io.ReadCloser, error) { + klog.V(2).Infof("Fetching %q", uri) + resp, err := http.Get(uri) + if err != nil { + return nil, errors.Wrapf(err, "failed to download %q", uri) + } + return resp.Body, nil +} + +var _ Fetcher = fileFetcher{} + +type fileFetcher struct{ f string } + +func (f fileFetcher) Get(_ string) (io.ReadCloser, error) { + klog.V(2).Infof("Reading %q", f.f) + file, err := os.Open(f.f) + return file, errors.Wrapf(err, "failed to open archive file %q for reading", f.f) +} + +// NewFileFetcher returns a local file reader. +func NewFileFetcher(path string) Fetcher { return fileFetcher{f: path} } diff --git a/internal/cli/cmd/plugin/download/verifier.go b/internal/cli/cmd/plugin/download/verifier.go new file mode 100755 index 000000000..2c2828b0e --- /dev/null +++ b/internal/cli/cmd/plugin/download/verifier.go @@ -0,0 +1,55 @@ +// Copyright 2019 The Kubernetes Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package download + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "hash" + "io" + + "github.com/pkg/errors" + "k8s.io/klog/v2" +) + +type Verifier interface { + io.Writer + Verify() error +} + +var _ Verifier = sha256Verifier{} + +type sha256Verifier struct { + hash.Hash + wantedHash []byte +} + +// NewSha256Verifier creates a Verifier that tests against the given hash. +func NewSha256Verifier(hashed string) Verifier { + raw, _ := hex.DecodeString(hashed) + return sha256Verifier{ + Hash: sha256.New(), + wantedHash: raw, + } +} + +func (v sha256Verifier) Verify() error { + klog.V(1).Infof("Compare sha256 (%s) signed version", hex.EncodeToString(v.wantedHash)) + if bytes.Equal(v.wantedHash, v.Sum(nil)) { + return nil + } + return errors.Errorf("checksum does not match, want: %x, got %x", v.wantedHash, v.Sum(nil)) +} diff --git a/internal/cli/cmd/plugin/index.go b/internal/cli/cmd/plugin/index.go new file mode 100644 index 000000000..c364bad06 --- /dev/null +++ b/internal/cli/cmd/plugin/index.go @@ -0,0 +1,255 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "fmt" + "io" + "os" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var ( + pluginListIndexExample = templates.Examples(` + # List all configured plugin indexes + kbcli plugin index list + `) + + pluginAddIndexExample = templates.Examples(` + # Add a new plugin index + kbcli plugin index add myIndex + `) + + pluginDeleteIndexExample = templates.Examples(` + # Delete a plugin index + kbcli plugin index delete myIndex + `) +) + +func NewPluginIndexCmd(streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "index", + Short: "Manage custom plugin indexes", + Long: "Manage which repositories are used to discover plugins and install plugins from", + } + + cmd.AddCommand(NewPluginIndexListCmd(streams)) + cmd.AddCommand(NewPluginIndexAddCmd(streams)) + cmd.AddCommand(NewPluginIndexDeleteCmd(streams)) + cmd.AddCommand(NewPluginIndexUpdateCmd(streams)) + return cmd +} + +type PluginIndexOptions struct { + IndexName string + URL string + + genericclioptions.IOStreams +} + +func (o *PluginIndexOptions) ListIndex() error { + indexes, err := ListIndexes(paths) + if err != nil { + return errors.Wrap(err, "failed to list indexes") + } + + p := NewPluginIndexPrinter(o.IOStreams.Out) + for _, index := range indexes { + addPluginIndexRow(index.Name, index.URL, p) + } + p.Print() + + return nil +} + +func (o *PluginIndexOptions) AddIndex() error { + err := AddIndex(paths, o.IndexName, o.URL) + if err != nil { + return err + } + return nil +} + +func (o *PluginIndexOptions) DeleteIndex() error { + err := DeleteIndex(paths, o.IndexName) + if err != nil { + return err + } + return nil +} + +func (o *PluginIndexOptions) UpdateIndex() error { + indexes, err := ListIndexes(paths) + if err != nil { + return errors.Wrap(err, "failed to list indexes") + } + + for _, idx := range indexes { + indexPath := paths.IndexPath(idx.Name) + klog.V(1).Infof("Updating the local copy of plugin index (%s)", indexPath) + if err := util.EnsureUpdated(idx.URL, indexPath); err != nil { + klog.Warningf("failed to update index %q: %v", idx.Name, err) + continue + } + + fmt.Fprintf(o.Out, "Updated the local copy of plugin index %q\n", idx.Name) + } + + return nil +} + +func NewPluginIndexListCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &PluginIndexOptions{ + IOStreams: streams, + } + + cmd := &cobra.Command{ + Use: "list", + Short: "List configured indexes", + Example: pluginListIndexExample, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.ListIndex()) + }, + } + + return cmd +} + +func NewPluginIndexAddCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &PluginIndexOptions{ + IOStreams: streams, + } + + cmd := &cobra.Command{ + Use: "add", + Short: "Add a new index", + Example: pluginAddIndexExample, + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + o.IndexName = args[0] + o.URL = args[1] + cmdutil.CheckErr(o.AddIndex()) + }, + } + + return cmd +} + +func NewPluginIndexDeleteCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &PluginIndexOptions{ + IOStreams: streams, + } + + cmd := &cobra.Command{ + Use: "delete", + Short: "Remove a configured index", + Example: pluginDeleteIndexExample, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + o.IndexName = args[0] + cmdutil.CheckErr(o.DeleteIndex()) + }, + } + + return cmd +} + +func NewPluginIndexUpdateCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &PluginIndexOptions{ + IOStreams: streams, + } + + cmd := &cobra.Command{ + Use: "update", + Short: "update all configured indexes", + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.UpdateIndex()) + }, + } + + return cmd +} + +func NewPluginIndexPrinter(out io.Writer) *printer.TablePrinter { + t := printer.NewTablePrinter(out) + t.SetHeader("INDEX", "URL") + return t +} + +func addPluginIndexRow(index, url string, p *printer.TablePrinter) { + p.AddRow(index, url) +} + +// ListIndexes returns a slice of Index objects. The path argument is used as +// the base path of the index. +func ListIndexes(paths *Paths) ([]Index, error) { + entries, err := os.ReadDir(paths.IndexBase()) + if err != nil { + return nil, err + } + + var indexes []Index + for _, e := range entries { + if !e.IsDir() { + continue + } + indexName := e.Name() + remote, err := util.GitGetRemoteURL(paths.IndexPath(indexName)) + if err != nil { + return nil, errors.Wrapf(err, "failed to list the remote URL for index %s", indexName) + } + + indexes = append(indexes, Index{ + Name: indexName, + URL: remote, + }) + } + return indexes, nil +} + +// AddIndex initializes a new index to install plugins from. +func AddIndex(paths *Paths, name, url string) error { + dir := paths.IndexPath(name) + if _, err := os.Stat(dir); os.IsNotExist(err) { + return util.EnsureCloned(url, dir) + } else if err != nil { + return err + } + return errors.New("index already exists") +} + +// DeleteIndex removes specified index name. If index does not exist, returns an error that can be tested by os.IsNotExist. +func DeleteIndex(paths *Paths, name string) error { + dir := paths.IndexPath(name) + if _, err := os.Stat(dir); err != nil { + return err + } + + return os.RemoveAll(dir) +} diff --git a/internal/cli/cmd/plugin/install.go b/internal/cli/cmd/plugin/install.go new file mode 100755 index 000000000..007d46779 --- /dev/null +++ b/internal/cli/cmd/plugin/install.go @@ -0,0 +1,203 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/cmd/plugin/download" +) + +var ( + pluginInstallExample = templates.Examples(` + # install a kbcli or kubectl plugin by name + kbcli plugin install [PLUGIN] + + # install a kbcli or kubectl plugin by name and index + kbcli plugin install [INDEX/PLUGIN] + `) +) + +type pluginInstallOption struct { + plugins []pluginEntry + + genericclioptions.IOStreams +} + +type pluginEntry struct { + index string + plugin Plugin +} + +func NewPluginInstallCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &pluginInstallOption{ + IOStreams: streams, + } + cmd := &cobra.Command{ + Use: "install", + Short: "Install kbcli or kubectl plugins", + Example: pluginInstallExample, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.complete(args)) + cmdutil.CheckErr(o.install()) + }, + } + return cmd +} + +func (o *pluginInstallOption) complete(names []string) error { + for _, name := range names { + indexName, pluginName := CanonicalPluginName(name) + plugin, err := LoadPluginByName(paths.IndexPluginsPath(indexName), pluginName) + if err != nil { + if os.IsNotExist(err) { + return errors.Errorf("plugin %q does not exist in the plugin index", name) + } + return errors.Wrapf(err, "failed to load plugin %q from the index", name) + } + o.plugins = append(o.plugins, pluginEntry{ + index: indexName, + plugin: plugin, + }) + } + return nil +} + +func (o *pluginInstallOption) install() error { + var failed []string + var returnErr error + for _, entry := range o.plugins { + plugin := entry.plugin + fmt.Fprintf(o.Out, "Installing plugin: %s\n", plugin.Name) + err := Install(paths, plugin, entry.index, InstallOpts{}) + if err == ErrIsAlreadyInstalled { + klog.Warningf("Skipping plugin %q, it is already installed", plugin.Name) + continue + } + if err != nil { + klog.Warningf("failed to install plugin %q: %v", plugin.Name, err) + if returnErr == nil { + returnErr = err + } + failed = append(failed, plugin.Name) + continue + } + fmt.Fprintf(o.Out, "Installed plugin: %s\n", plugin.Name) + output := fmt.Sprintf("Use this plugin:\n\tkubectl %s\n", plugin.Name) + if plugin.Spec.Homepage != "" { + output += fmt.Sprintf("Documentation:\n\t%s\n", plugin.Spec.Homepage) + } + if plugin.Spec.Caveats != "" { + output += fmt.Sprintf("Caveats:\n%s\n", indent(plugin.Spec.Caveats)) + } + fmt.Fprintln(o.Out, indent(output)) + } + if len(failed) > 0 { + return errors.Wrapf(returnErr, "failed to install some plugins: %+v", failed) + } + return nil +} + +// Install will download and install a plugin. The operation tries +// to not get the plugin dir in a bad state if it fails during the process. +func Install(p *Paths, plugin Plugin, indexName string, opts InstallOpts) error { + klog.V(2).Infof("Looking for installed versions") + _, err := ReadReceiptFromFile(p.PluginInstallReceiptPath(plugin.Name)) + if err == nil { + return ErrIsAlreadyInstalled + } else if !os.IsNotExist(err) { + return errors.Wrap(err, "failed to look up plugin receipt") + } + + // Find available installation candidate + candidate, ok, err := GetMatchingPlatform(plugin.Spec.Platforms) + if err != nil { + return errors.Wrap(err, "failed trying to find a matching platform in plugin spec") + } + if !ok { + return errors.Errorf("plugin %q does not offer installation for this platform", plugin.Name) + } + + // The actual install should be the last action so that a failure during receipt + // saving does not result in an installed plugin without receipt. + klog.V(3).Infof("Install plugin %s at version=%s", plugin.Name, plugin.Spec.Version) + if err := install(installOperation{ + pluginName: plugin.Name, + platform: candidate, + + binDir: p.BinPath(), + installDir: p.PluginVersionInstallPath(plugin.Name, plugin.Spec.Version), + }, opts); err != nil { + return errors.Wrap(err, "install failed") + } + + klog.V(3).Infof("Storing install receipt for plugin %s", plugin.Name) + err = StoreReceipt(NewReceipt(plugin, indexName, metav1.Now()), p.PluginInstallReceiptPath(plugin.Name)) + return errors.Wrap(err, "installation receipt could not be stored, uninstall may fail") +} + +func install(op installOperation, opts InstallOpts) error { + // Download and extract + klog.V(3).Infof("Creating download staging directory") + downloadStagingDir, err := os.MkdirTemp("", "kbcli-downloads") + if err != nil { + return errors.Wrapf(err, "could not create staging dir %q", downloadStagingDir) + } + klog.V(3).Infof("Successfully created download staging directory %q", downloadStagingDir) + defer func() { + klog.V(3).Infof("Deleting the download staging directory %s", downloadStagingDir) + if err := os.RemoveAll(downloadStagingDir); err != nil { + klog.Warningf("failed to clean up download staging directory: %s", err) + } + }() + if err := download.DownloadAndExtract(downloadStagingDir, op.platform.URI, op.platform.Sha256, opts.ArchiveFileOverride); err != nil { + return errors.Wrap(err, "failed to unpack into staging dir") + } + + applyDefaults(&op.platform) + if err := moveToInstallDir(downloadStagingDir, op.installDir, op.platform.Files); err != nil { + return errors.Wrap(err, "failed while moving files to the installation directory") + } + + subPathAbs, err := filepath.Abs(op.installDir) + if err != nil { + return errors.Wrapf(err, "failed to get the absolute fullPath of %q", op.installDir) + } + fullPath := filepath.Join(op.installDir, filepath.FromSlash(op.platform.Bin)) + pathAbs, err := filepath.Abs(fullPath) + if err != nil { + return errors.Wrapf(err, "failed to get the absolute fullPath of %q", fullPath) + } + if _, ok := IsSubPath(subPathAbs, pathAbs); !ok { + return errors.Wrapf(err, "the fullPath %q does not extend the sub-fullPath %q", fullPath, op.installDir) + } + err = createOrUpdateLink(op.binDir, fullPath, op.pluginName) + return errors.Wrap(err, "failed to link installed plugin") +} diff --git a/internal/cli/cmd/plugin/pathutil.go b/internal/cli/cmd/plugin/pathutil.go new file mode 100644 index 000000000..dc1160eb1 --- /dev/null +++ b/internal/cli/cmd/plugin/pathutil.go @@ -0,0 +1,346 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "io" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/pkg/errors" + "k8s.io/klog/v2" + + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +type move struct { + from, to string +} + +func findMoveTargets(fromDir, toDir string, fo FileOperation) ([]move, error) { + if fo.To != filepath.Clean(fo.To) { + return nil, errors.Errorf("the provided path is not clean, %q should be %q", fo.To, filepath.Clean(fo.To)) + } + fromDir, err := filepath.Abs(fromDir) + if err != nil { + return nil, errors.Wrap(err, "could not get the relative path for the move src") + } + + klog.V(4).Infof("Trying to move single file directly from=%q to=%q with file operation=%#v", fromDir, toDir, fo) + if m, ok, err := getDirectMove(fromDir, toDir, fo); err != nil { + return nil, errors.Wrap(err, "failed to detect single move operation") + } else if ok { + klog.V(3).Infof("Detected single move from file operation=%#v", fo) + return []move{m}, nil + } + + klog.V(4).Infoln("Wasn't a single file, proceeding with Glob move") + newDir, err := filepath.Abs(filepath.Join(filepath.FromSlash(toDir), filepath.FromSlash(fo.To))) + if err != nil { + return nil, errors.Wrap(err, "could not get the relative path for the move dst") + } + + gl, err := filepath.Glob(filepath.Join(filepath.FromSlash(fromDir), filepath.FromSlash(fo.From))) + if err != nil { + return nil, errors.Wrap(err, "could not get files using a glob string") + } + if len(gl) == 0 { + return nil, errors.Errorf("no files in the plugin archive matched the glob pattern=%s", fo.From) + } + + moves := make([]move, 0, len(gl)) + for _, v := range gl { + newPath := filepath.Join(newDir, filepath.Base(filepath.FromSlash(v))) + // Check secure path + m := move{from: v, to: newPath} + if !isMoveAllowed(fromDir, toDir, m) { + return nil, errors.Errorf("can't move, move target %v is not a subpath from=%q, to=%q", m, fromDir, toDir) + } + moves = append(moves, m) + } + return moves, nil +} + +func getDirectMove(fromDir, toDir string, fo FileOperation) (move, bool, error) { + var m move + fromDir, err := filepath.Abs(fromDir) + if err != nil { + return m, false, errors.Wrap(err, "could not get the relative path for the move src") + } + + toDir, err = filepath.Abs(toDir) + if err != nil { + return m, false, errors.Wrap(err, "could not get the relative path for the move src") + } + + // Check is direct file (not a Glob) + fromFilePath := filepath.Clean(filepath.Join(fromDir, fo.From)) + _, err = os.Stat(fromFilePath) + if err != nil { + return m, false, nil + } + + // If target is empty use old file name. + if filepath.Clean(fo.To) == "." { + fo.To = filepath.Base(fromFilePath) + } + + // Build new file name + toFilePath, err := filepath.Abs(filepath.Join(filepath.FromSlash(toDir), filepath.FromSlash(fo.To))) + if err != nil { + return m, false, errors.Wrap(err, "could not get the relative path for the move dst") + } + + // Check sane path + m = move{from: fromFilePath, to: toFilePath} + if !isMoveAllowed(fromDir, toDir, m) { + return move{}, false, errors.Errorf("can't move, move target %v is out of bounds from=%q, to=%q", m, fromDir, toDir) + } + + return m, true, nil +} + +func isMoveAllowed(fromBase, toBase string, m move) bool { + _, okFrom := IsSubPath(fromBase, m.from) + _, okTo := IsSubPath(toBase, m.to) + return okFrom && okTo +} + +func moveFiles(fromDir, toDir string, fo FileOperation) error { + klog.V(4).Infof("Finding move targets from %q to %q with file operation=%#v", fromDir, toDir, fo) + moves, err := findMoveTargets(fromDir, toDir, fo) + if err != nil { + return errors.Wrap(err, "could not find move targets") + } + + for _, m := range moves { + klog.V(2).Infof("Move file from %q to %q", m.from, m.to) + if err := os.MkdirAll(filepath.Dir(m.to), 0o755); err != nil { + return errors.Wrapf(err, "failed to create move path %q", filepath.Dir(m.to)) + } + + if err = renameOrCopy(m.from, m.to); err != nil { + return errors.Wrapf(err, "could not rename/copy file from %q to %q", m.from, m.to) + } + } + klog.V(4).Infoln("Move operations are complete") + return nil +} + +func moveAllFiles(fromDir, toDir string, fos []FileOperation) error { + for _, fo := range fos { + if err := moveFiles(fromDir, toDir, fo); err != nil { + return errors.Wrap(err, "failed moving files") + } + } + return nil +} + +// moveToInstallDir moves plugins from srcDir to dstDir (created in this method) with given FileOperation. +func moveToInstallDir(srcDir, installDir string, fos []FileOperation) error { + installationDir := filepath.Dir(installDir) + klog.V(4).Infof("Creating directory %q", installationDir) + if err := os.MkdirAll(installationDir, 0o755); err != nil { + return errors.Wrapf(err, "error creating directory at %q", installationDir) + } + + tmp, err := os.MkdirTemp("", "kbcli-temp-move") + klog.V(4).Infof("Creating temp plugin move operations dir %q", tmp) + if err != nil { + return errors.Wrap(err, "failed to find a temporary director") + } + defer os.RemoveAll(tmp) + + if err = moveAllFiles(srcDir, tmp, fos); err != nil { + return errors.Wrap(err, "failed to move files") + } + + klog.V(2).Infof("Move directory %q to %q", tmp, installDir) + if err = renameOrCopy(tmp, installDir); err != nil { + defer func() { + klog.V(3).Info("Cleaning up installation directory due to error during copying files") + os.Remove(installDir) + }() + return errors.Wrapf(err, "could not rename/copy directory %q to %q", tmp, installDir) + } + return nil +} + +// renameOrCopy will try to rename a dir or file. If rename is not supported, a manual copy will be performed. +// Existing files at "to" will be deleted. +func renameOrCopy(from, to string) error { + // Try atomic rename (does not work cross partition). + fi, err := os.Stat(to) + if err != nil && !os.IsNotExist(err) { + return errors.Wrapf(err, "error checking move target dir %q", to) + } + if fi != nil && fi.IsDir() { + klog.V(4).Infof("There's already a directory at move target %q. deleting.", to) + if err := os.RemoveAll(to); err != nil { + return errors.Wrapf(err, "error cleaning up dir %q", to) + } + klog.V(4).Infof("Move target directory %q cleaned up", to) + } + + err = os.Rename(from, to) + // Fallback for invalid cross-device link (errno:18). + if isCrossDeviceRenameErr(err) { + klog.V(2).Infof("Cross-device link error while copying, fallback to manual copy") + return errors.Wrap(copyTree(from, to), "failed to copy directory tree as a fallback") + } + return err +} + +// copyTree copies files or directories, recursively. +func copyTree(from, to string) (err error) { + return filepath.Walk(from, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + newPath, _ := ReplaceBase(path, from, to) + if info.IsDir() { + klog.V(4).Infof("Creating new dir %q", newPath) + err = os.MkdirAll(newPath, info.Mode()) + } else { + klog.V(4).Infof("Copying file %q", newPath) + err = copyFile(path, newPath, info.Mode()) + } + return err + }) +} + +func copyFile(source, dst string, mode os.FileMode) (err error) { + sf, err := os.Open(source) + if err != nil { + return err + } + defer sf.Close() + + df, err := os.Create(dst) + if err != nil { + return err + } + defer df.Close() + + _, err = io.Copy(df, sf) + if err != nil { + return err + } + return os.Chmod(dst, mode) +} + +// isCrossDeviceRenameErr determines if a os.Rename error is due to cross-fs/drive/volume copying. +func isCrossDeviceRenameErr(err error) bool { + le, ok := err.(*os.LinkError) + if !ok { + return false + } + errno, ok := le.Err.(syscall.Errno) + if !ok { + return false + } + return (util.IsWindows() && errno == 17) || // syscall.ERROR_NOT_SAME_DEVICE + (!util.IsWindows() && errno == 18) // syscall.EXDEV +} + +// IsSubPath checks if the extending path is an extension of the basePath, it will return the extending path +// elements. Both paths have to be absolute or have the same root directory. The remaining path elements +func IsSubPath(basePath, subPath string) (string, bool) { + extendingPath, err := filepath.Rel(basePath, subPath) + if err != nil { + return "", false + } + if strings.HasPrefix(extendingPath, "..") { + return "", false + } + return extendingPath, true +} + +// ReplaceBase will return a replacement path with replacement as a base of the path instead of the old base. a/b/c, a, d -> d/b/c +func ReplaceBase(path, old, replacement string) (string, error) { + extendingPath, ok := IsSubPath(old, path) + if !ok { + return "", errors.Errorf("can't replace %q in %q, it is not a subpath", old, path) + } + return filepath.Join(replacement, extendingPath), nil +} + +// CanonicalPluginName resolves a plugin's index and name from input string. +// If an index is not specified, the default index name is assumed. +func CanonicalPluginName(in string) (string, string) { + if strings.Count(in, "/") == 0 { + return DefaultIndexName, in + } + p := strings.SplitN(in, "/", 2) + return p[0], p[1] +} + +func createOrUpdateLink(binDir, binary, plugin string) error { + dst := filepath.Join(binDir, pluginNameToBin(plugin, util.IsWindows())) + + if err := removeLink(dst); err != nil { + return errors.Wrap(err, "failed to remove old symlink") + } + if _, err := os.Stat(binary); os.IsNotExist(err) { + return errors.Wrapf(err, "can't create symbolic link, source binary (%q) cannot be found in extracted archive", binary) + } + + // Create new + klog.V(2).Infof("Creating symlink to %q at %q", binary, dst) + if err := os.Symlink(binary, dst); err != nil { + return errors.Wrapf(err, "failed to create a symlink from %q to %q", binary, dst) + } + klog.V(2).Infof("Created symlink at %q", dst) + + return nil +} + +// removeLink removes a symlink reference if exists. +func removeLink(path string) error { + fi, err := os.Lstat(path) + if os.IsNotExist(err) { + klog.V(3).Infof("No file found at %q", path) + return nil + } else if err != nil { + return errors.Wrapf(err, "failed to read the symlink in %q", path) + } + + if fi.Mode()&os.ModeSymlink == 0 { + return errors.Errorf("file %q is not a symlink (mode=%s)", path, fi.Mode()) + } + if err := os.Remove(path); err != nil { + return errors.Wrapf(err, "failed to remove the symlink in %q", path) + } + klog.V(3).Infof("Removed symlink from %q", path) + return nil +} + +// pluginNameToBin creates the name of the symlink file for the plugin name. +// It converts dashes to underscores. +func pluginNameToBin(name string, isWindows bool) string { + name = strings.ReplaceAll(name, "-", "_") + name = "kbcli-" + name + if isWindows { + name += ".exe" + } + return name +} diff --git a/internal/cli/cmd/plugin/platform.go b/internal/cli/cmd/plugin/platform.go new file mode 100755 index 000000000..b26ff608b --- /dev/null +++ b/internal/cli/cmd/plugin/platform.go @@ -0,0 +1,86 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "fmt" + "os" + "runtime" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" +) + +// GetMatchingPlatform finds the platform spec in the specified plugin that +// matches the os/arch of the current machine (can be overridden via KREW_OS +// and/or KREW_ARCH). +func GetMatchingPlatform(platforms []Platform) (Platform, bool, error) { + return matchPlatform(platforms, OSArch()) +} + +// matchPlatform returns the first matching platform to given os/arch. +func matchPlatform(platforms []Platform, env OSArchPair) (Platform, bool, error) { + envLabels := labels.Set{ + "os": env.OS, + "arch": env.Arch, + } + klog.V(2).Infof("Matching platform for labels(%v)", envLabels) + + for i, platform := range platforms { + sel, err := metav1.LabelSelectorAsSelector(platform.Selector) + if err != nil { + return Platform{}, false, errors.Wrap(err, "failed to compile label selector") + } + if sel.Matches(envLabels) { + klog.V(2).Infof("Found matching platform with index (%d)", i) + return platform, true, nil + } + } + return Platform{}, false, nil +} + +// OSArchPair is wrapper around operating system and architecture +type OSArchPair struct { + OS, Arch string +} + +// String converts environment into a string +func (p OSArchPair) String() string { + return fmt.Sprintf("%s/%s", p.OS, p.Arch) +} + +// OSArch returns the OS/arch combination to be used on the current system. It +// can be overridden by setting KREW_OS and/or KREW_ARCH environment variables. +func OSArch() OSArchPair { + return OSArchPair{ + OS: getEnvOrDefault("KBLCI_OS", runtime.GOOS), + Arch: getEnvOrDefault("KBCLI_ARCH", runtime.GOARCH), + } +} + +func getEnvOrDefault(env, absent string) string { + v := os.Getenv(env) + if v != "" { + return v + } + return absent +} diff --git a/internal/cli/cmd/plugin/plugin.go b/internal/cli/cmd/plugin/plugin.go index 9942e6779..2cf4caf98 100644 --- a/internal/cli/cmd/plugin/plugin.go +++ b/internal/cli/cmd/plugin/plugin.go @@ -50,6 +50,7 @@ var ( `) ValidPluginFilenamePrefixes = []string{"kbcli", "kubectl"} + paths = GetKbcliPluginPath() ) func NewPluginCmd(streams genericclioptions.IOStreams) *cobra.Command { @@ -59,7 +60,35 @@ func NewPluginCmd(streams genericclioptions.IOStreams) *cobra.Command { Long: pluginLong, } - cmd.AddCommand(NewPluginListCmd(streams)) + if err := EnsureDirs(paths.BasePath(), + paths.BinPath(), + paths.InstallPath(), + paths.IndexBase(), + paths.InstallReceiptsPath()); err != nil { + klog.Fatal(err) + } + + // check if index exist, if indexes don't exist, download default index + indexes, err := ListIndexes(paths) + if err != nil { + klog.Fatal(err) + } + if len(indexes) == 0 { + klog.Info("start download default index") + if err := AddIndex(paths, DefaultIndexName, DefaultIndexURI); err != nil { + klog.Fatal("failed to download default index", err) + } + } + + cmd.AddCommand( + NewPluginListCmd(streams), + NewPluginIndexCmd(streams), + NewPluginInstallCmd(streams), + NewPluginUninstallCmd(streams), + NewPluginSearchCmd(streams), + NewPluginDescribeCmd(streams), + NewPluginUpgradeCmd(streams), + ) return cmd } @@ -117,7 +146,7 @@ func (o *PluginListOptions) Run() error { pluginWarnings++ } } - addRow(name, path, p) + addPluginRow(name, path, p) } p.Print() klog.V(1).Info(errMsg) @@ -273,6 +302,6 @@ func NewPluginPrinter(out io.Writer) *printer.TablePrinter { return t } -func addRow(name, path string, p *printer.TablePrinter) { +func addPluginRow(name, path string, p *printer.TablePrinter) { p.AddRow(name, path) } diff --git a/internal/cli/cmd/plugin/search.go b/internal/cli/cmd/plugin/search.go new file mode 100644 index 000000000..eb23c7522 --- /dev/null +++ b/internal/cli/cmd/plugin/search.go @@ -0,0 +1,97 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "io" + "os" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/printer" +) + +var ( + pluginSearchExample = templates.Examples(` + # search a kbcli or kubectl plugin by name + kbcli plugin search myplugin + `) +) + +func NewPluginSearchCmd(streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "search", + Short: "Search kbcli or kubectl plugins", + Example: pluginSearchExample, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(searchPlugin(streams, args[0])) + }, + } + + return cmd +} + +func searchPlugin(streams genericclioptions.IOStreams, name string) error { + indexes, err := ListIndexes(paths) + if err != nil { + return errors.Wrap(err, "failed to list indexes") + } + + var plugins []pluginEntry + for _, index := range indexes { + plugin, err := LoadPluginByName(paths.IndexPluginsPath(index.Name), name) + if err != nil && !os.IsNotExist(err) { + klog.V(1).Info("failed to load plugin %q from the index", name) + } else { + plugins = append(plugins, pluginEntry{ + index: index.Name, + plugin: plugin, + }) + } + } + + p := NewPluginSearchPrinter(streams.Out) + for _, plugin := range plugins { + _, err := os.Stat(paths.PluginInstallReceiptPath(name)) + addPluginSearchRow(plugin.index, plugin.plugin.Name, !os.IsNotExist(err), p) + } + p.Print() + return nil +} + +func NewPluginSearchPrinter(out io.Writer) *printer.TablePrinter { + t := printer.NewTablePrinter(out) + t.SetHeader("INDEX", "NAME", "INSTALLED") + return t +} + +func addPluginSearchRow(index, plugin string, installed bool, p *printer.TablePrinter) { + if installed { + p.AddRow(index, plugin, "yes") + } else { + p.AddRow(index, plugin, "no") + } +} diff --git a/internal/cli/cmd/plugin/types.go b/internal/cli/cmd/plugin/types.go new file mode 100644 index 000000000..c596b2437 --- /dev/null +++ b/internal/cli/cmd/plugin/types.go @@ -0,0 +1,184 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" + "sigs.k8s.io/yaml" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + DefaultIndexURI = "https://github.com/kubernetes-sigs/krew-index.git" + DefaultIndexName = "default" + ManifestExtension = ".yaml" + PluginKind = "Plugin" +) + +var SupportAPIVersion = []string{ + "krew.googlecontainertools.github.com/v1alpha2", + "kbcli.googlecontainertools.github.com/v1alpha2", +} + +type Paths struct { + base string + tmp string +} + +func (p *Paths) BasePath() string { + return p.base +} + +func (p *Paths) IndexBase() string { + return filepath.Join(p.base, "index") +} + +func (p *Paths) IndexPath(name string) string { + return filepath.Join(p.IndexBase(), name) +} + +func (p *Paths) IndexPluginsPath(name string) string { + return filepath.Join(p.IndexPath(name), "plugins") +} + +func (p *Paths) InstallReceiptsPath() string { + return filepath.Join(p.base, "receipts") +} + +func (p *Paths) BinPath() string { + return filepath.Join(p.base, "bin") +} + +func (p *Paths) InstallPath() string { + return filepath.Join(p.base, "store") +} + +func (p *Paths) PluginInstallPath(plugin string) string { + return filepath.Join(p.InstallPath(), plugin) +} + +func (p *Paths) PluginVersionInstallPath(plugin, version string) string { + return filepath.Join(p.InstallPath(), plugin, version) +} + +func (p *Paths) PluginInstallReceiptPath(plugin string) string { + return filepath.Join(p.InstallReceiptsPath(), plugin+".yaml") +} + +type Index struct { + Name string + URL string +} + +type Plugin struct { + metav1.TypeMeta `json:",inline" yaml:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty" yaml:"metadata"` + + Spec PluginSpec `json:"spec"` +} + +// PluginSpec is the plugin specification. +type PluginSpec struct { + Version string `json:"version,omitempty"` + ShortDescription string `json:"shortDescription,omitempty"` + Description string `json:"description,omitempty"` + Caveats string `json:"caveats,omitempty"` + Homepage string `json:"homepage,omitempty"` + + Platforms []Platform `json:"platforms,omitempty"` +} + +// Platform describes how to perform an installation on a specific platform +// and how to match the target platform (os, arch). +type Platform struct { + URI string `json:"uri,omitempty"` + Sha256 string `json:"sha256,omitempty"` + + Selector *metav1.LabelSelector `json:"selector,omitempty"` + Files []FileOperation `json:"files"` + + // Bin specifies the path to the plugin executable. + // The path is relative to the root of the installation folder. + // The binary will be linked after all FileOperations are executed. + Bin string `json:"bin"` +} + +// FileOperation specifies a file copying operation from plugin archive to the +// installation directory. +type FileOperation struct { + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` +} + +// Receipt describes a plugin receipt file. +type Receipt struct { + Plugin `json:",inline" yaml:",inline"` + + Status ReceiptStatus `json:"status"` +} + +// ReceiptStatus contains information about the installed plugin. +type ReceiptStatus struct { + Source SourceIndex `json:"source"` +} + +// SourceIndex contains information about the index a plugin was installed from. +type SourceIndex struct { + // Name is the configured name of an index a plugin was installed from. + Name string `json:"name"` +} + +type InstallOpts struct { + ArchiveFileOverride string +} + +type installOperation struct { + pluginName string + platform Platform + + binDir string + installDir string +} + +// NewReceipt returns a new receipt with the given plugin and index name. +func NewReceipt(plugin Plugin, indexName string, timestamp metav1.Time) Receipt { + plugin.CreationTimestamp = timestamp + return Receipt{ + Plugin: plugin, + Status: ReceiptStatus{ + Source: SourceIndex{ + Name: indexName, + }, + }, + } +} +func StoreReceipt(receipt Receipt, dest string) error { + yamlBytes, err := yaml.Marshal(receipt) + if err != nil { + return errors.Wrapf(err, "convert to yaml") + } + + err = os.WriteFile(dest, yamlBytes, 0o644) + return errors.Wrapf(err, "write plugin receipt %q", dest) +} diff --git a/internal/cli/cmd/plugin/uninstall.go b/internal/cli/cmd/plugin/uninstall.go new file mode 100644 index 000000000..3efd47235 --- /dev/null +++ b/internal/cli/cmd/plugin/uninstall.go @@ -0,0 +1,91 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var ( + pluginUninstallExample = templates.Examples(` + # uninstall a kbcli or kubectl plugin by name + kbcli plugin uninstall [PLUGIN] + `) +) + +func NewPluginUninstallCmd(_ genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "uninstall", + Short: "Uninstall kbcli or kubectl plugins", + Example: pluginUninstallExample, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(uninstallPlugins(args)) + }, + } + + return cmd +} + +func uninstallPlugins(names []string) error { + for _, name := range names { + klog.V(4).Infof("Going to uninstall plugin %s\n", name) + if err := uninstall(paths, name); err != nil { + return errors.Wrapf(err, "failed to uninstall plugin %s", name) + } + } + return nil +} + +func uninstall(p *Paths, name string) error { + if _, err := ReadReceiptFromFile(p.PluginInstallReceiptPath(name)); err != nil { + if os.IsNotExist(err) { + return ErrIsNotInstalled + } + return errors.Wrapf(err, "failed to look up install receipt for plugin %q", name) + } + + klog.V(1).Infof("Deleting plugin %s", name) + + symlinkPath := filepath.Join(p.BinPath(), pluginNameToBin(name, util.IsWindows())) + klog.V(3).Infof("Unlink %q", symlinkPath) + if err := removeLink(symlinkPath); err != nil { + return errors.Wrap(err, "could not uninstall symlink of plugin") + } + + pluginInstallPath := p.PluginInstallPath(name) + klog.V(3).Infof("Deleting path %q", pluginInstallPath) + if err := os.RemoveAll(pluginInstallPath); err != nil { + return errors.Wrapf(err, "could not remove plugin directory %q", pluginInstallPath) + } + pluginReceiptPath := p.PluginInstallReceiptPath(name) + klog.V(3).Infof("Deleting plugin receipt %q", pluginReceiptPath) + err := os.Remove(pluginReceiptPath) + return errors.Wrapf(err, "could not remove plugin receipt %q", pluginReceiptPath) +} diff --git a/internal/cli/cmd/plugin/upgrade.go b/internal/cli/cmd/plugin/upgrade.go new file mode 100644 index 000000000..8e6029887 --- /dev/null +++ b/internal/cli/cmd/plugin/upgrade.go @@ -0,0 +1,196 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "fmt" + "os" + "strings" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + k8sver "k8s.io/apimachinery/pkg/util/version" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/klog/v2" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" +) + +var ( + pluginUpgradeExample = templates.Examples(` + # upgrade installed plugins with specified name + kbcli plugin upgrade myplugin + + # upgrade installed plugin to a newer version + kbcli plugin upgrade --all + `) +) + +type upgradeOptions struct { + // common user flags + all bool + + pluginNames []string + genericclioptions.IOStreams +} + +func NewPluginUpgradeCmd(streams genericclioptions.IOStreams) *cobra.Command { + o := &upgradeOptions{ + IOStreams: streams, + } + + cmd := &cobra.Command{ + Use: "upgrade", + Short: "Upgrade kbcli or kubectl plugins", + Example: pluginUpgradeExample, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.complete(args)) + cmdutil.CheckErr(o.run()) + }, + } + + cmd.Flags().BoolVar(&o.all, "all", o.all, "Upgrade all installed plugins") + + return cmd +} + +func (o *upgradeOptions) complete(args []string) error { + if o.all { + installed, err := GetInstalledPluginReceipts(paths.InstallReceiptsPath()) + if err != nil { + return err + } + for _, receipt := range installed { + o.pluginNames = append(o.pluginNames, receipt.Status.Source.Name+"/"+receipt.Name) + } + } else { + if len(args) == 0 { + return errors.New("no plugin name specified") + } + for _, arg := range args { + receipt, err := ReadReceiptFromFile(paths.PluginInstallReceiptPath(arg)) + if err != nil { + return err + } + o.pluginNames = append(o.pluginNames, receipt.Status.Source.Name+"/"+receipt.Name) + } + } + + return nil +} + +func (o *upgradeOptions) run() error { + for _, name := range o.pluginNames { + indexName, pluginName := CanonicalPluginName(name) + + plugin, err := LoadPluginByName(paths.IndexPluginsPath(indexName), pluginName) + if err != nil { + return err + } + + fmt.Fprintf(o.Out, "Upgrading plugin: %s\n", name) + if err := Upgrade(paths, plugin, indexName); err != nil { + if err == ErrIsAlreadyUpgraded { + fmt.Fprintf(o.Out, "Plugin %q is already upgraded\n", name) + continue + } + return err + } + } + return nil +} + +// Upgrade will reinstall and delete the old plugin. The operation tries +// to not get the plugin dir in a bad state if it fails during the process. +func Upgrade(p *Paths, plugin Plugin, indexName string) error { + installReceipt, err := ReadReceiptFromFile(p.PluginInstallReceiptPath(plugin.Name)) + if err != nil { + return errors.Wrapf(err, "failed to load install receipt for plugin %q", plugin.Name) + } + + curVersion := installReceipt.Spec.Version + curv, err := parseVersion(curVersion) + if err != nil { + return errors.Wrapf(err, "failed to parse installed plugin version (%q) as a semver value", curVersion) + } + + // Find available installation candidate + candidate, ok, err := GetMatchingPlatform(plugin.Spec.Platforms) + if err != nil { + return errors.Wrap(err, "failed trying to find a matching platform in plugin spec") + } + if !ok { + return errors.Errorf("plugin %q does not offer installation for this platform (%s)", + plugin.Name, OSArch()) + } + + newVersion := plugin.Spec.Version + newv, err := parseVersion(newVersion) + if err != nil { + return errors.Wrapf(err, "failed to parse candidate version spec (%q)", newVersion) + } + klog.V(2).Infof("Comparing versions: current=%s target=%s", curv, newv) + + // See if it's a newer version + if !curv.LessThan(newv) { + klog.V(3).Infof("Plugin does not need upgrade (%s ≥ %s)", curv, newv) + return ErrIsAlreadyUpgraded + } + klog.V(1).Infof("Plugin needs upgrade (%s < %s)", curv, newv) + + // Re-Install + klog.V(1).Infof("Installing new version %s", newVersion) + if err := install(installOperation{ + pluginName: plugin.Name, + platform: candidate, + + installDir: p.PluginVersionInstallPath(plugin.Name, newVersion), + binDir: p.BinPath(), + }, InstallOpts{}); err != nil { + return errors.Wrap(err, "failed to install new version") + } + + klog.V(2).Infof("Upgrading install receipt for plugin %s", plugin.Name) + if err = StoreReceipt(NewReceipt(plugin, indexName, installReceipt.CreationTimestamp), p.PluginInstallReceiptPath(plugin.Name)); err != nil { + return errors.Wrap(err, "installation receipt could not be stored, uninstall may fail") + } + + // Clean old installations + klog.V(2).Infof("Starting old version cleanup") + return cleanupInstallation(p, plugin, curVersion) +} + +// cleanupInstallation will remove a plugin directly +func cleanupInstallation(p *Paths, plugin Plugin, oldVersion string) error { + klog.V(1).Infof("Remove old plugin installation under %q", p.PluginVersionInstallPath(plugin.Name, oldVersion)) + return os.RemoveAll(p.PluginVersionInstallPath(plugin.Name, oldVersion)) +} + +func parseVersion(s string) (*k8sver.Version, error) { + var vv *k8sver.Version + if !strings.HasPrefix(s, "v") { + return vv, errors.Errorf("version string %q does not start with 'v'", s) + } + vv, err := k8sver.ParseSemantic(s) + if err != nil { + return vv, err + } + return vv, nil +} diff --git a/internal/cli/cmd/plugin/utils.go b/internal/cli/cmd/plugin/utils.go new file mode 100644 index 000000000..ad97b8cee --- /dev/null +++ b/internal/cli/cmd/plugin/utils.go @@ -0,0 +1,242 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package plugin + +import ( + "io" + "os" + "path/filepath" + "regexp" + "strings" + "unicode" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/homedir" + "k8s.io/klog/v2" + "sigs.k8s.io/yaml" +) + +var ( + ErrIsAlreadyInstalled = errors.New("can't install, the newest version is already installed") + ErrIsNotInstalled = errors.New("plugin is not installed") + ErrIsAlreadyUpgraded = errors.New("can't upgrade, the newest version is already installed") +) + +func GetKbcliPluginPath() *Paths { + base := filepath.Join(homedir.HomeDir(), ".kbcli", "plugins") + return NewPaths(base) +} + +func EnsureDirs(paths ...string) error { + for _, p := range paths { + if err := os.MkdirAll(p, os.ModePerm); err != nil { + return err + } + } + return nil +} + +func NewPaths(base string) *Paths { + return &Paths{base: base, tmp: os.TempDir()} +} + +func LoadPluginByName(pluginsDir, pluginName string) (Plugin, error) { + klog.V(4).Infof("Reading plugin %q from %s", pluginName, pluginsDir) + return ReadPluginFromFile(filepath.Join(pluginsDir, pluginName+ManifestExtension)) +} + +func ReadPluginFromFile(path string) (Plugin, error) { + var plugin Plugin + err := readFromFile(path, &plugin) + if err != nil { + return plugin, err + } + return plugin, errors.Wrap(ValidatePlugin(plugin.Name, plugin), "plugin manifest validation error") +} + +func ReadReceiptFromFile(path string) (Receipt, error) { + var receipt Receipt + err := readFromFile(path, &receipt) + if err != nil { + return receipt, err + } + return receipt, nil +} + +func readFromFile(path string, as interface{}) error { + f, err := os.Open(path) + if err != nil { + return err + } + err = decodeFile(f, &as) + return errors.Wrapf(err, "failed to parse yaml file %q", path) +} + +func decodeFile(r io.ReadCloser, as interface{}) error { + defer r.Close() + b, err := io.ReadAll(r) + if err != nil { + return err + } + return yaml.Unmarshal(b, &as) +} + +func indent(s string) string { + out := "\\\n" + s = strings.TrimRightFunc(s, unicode.IsSpace) + out += regexp.MustCompile("(?m)^").ReplaceAllString(s, " | ") + out += "\n/" + return out +} + +func applyDefaults(platform *Platform) { + if platform.Files == nil { + platform.Files = []FileOperation{{From: "*", To: "."}} + klog.V(4).Infof("file operation not specified, assuming %v", platform.Files) + } +} + +// GetInstalledPluginReceipts returns a list of receipts. +func GetInstalledPluginReceipts(receiptsDir string) ([]Receipt, error) { + files, err := filepath.Glob(filepath.Join(receiptsDir, "*"+ManifestExtension)) + if err != nil { + return nil, errors.Wrapf(err, "failed to glob receipts directory (%s) for manifests", receiptsDir) + } + out := make([]Receipt, 0, len(files)) + for _, f := range files { + r, err := ReadReceiptFromFile(f) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse plugin install receipt %s", f) + } + out = append(out, r) + klog.V(4).Infof("parsed receipt for %s: version=%s", r.GetObjectMeta().GetName(), r.Spec.Version) + + } + return out, nil +} + +func isSupportAPIVersion(apiVersion string) bool { + for _, v := range SupportAPIVersion { + if apiVersion == v { + return true + } + } + return false +} + +func ValidatePlugin(name string, p Plugin) error { + if !isSupportAPIVersion(p.APIVersion) { + return errors.Errorf("plugin manifest has apiVersion=%q, not supported in this version of krew (try updating plugin index or install a newer version of krew)", p.APIVersion) + } + if p.Kind != PluginKind { + return errors.Errorf("plugin manifest has kind=%q, but only %q is supported", p.Kind, PluginKind) + } + if p.Name != name { + return errors.Errorf("plugin manifest has name=%q, but expected %q", p.Name, name) + } + if p.Spec.ShortDescription == "" { + return errors.New("should have a short description") + } + if len(p.Spec.Platforms) == 0 { + return errors.New("should have a platform") + } + if p.Spec.Version == "" { + return errors.New("should have a version") + } + if _, err := parseVersion(p.Spec.Version); err != nil { + return errors.Wrap(err, "failed to parse version") + } + for _, pl := range p.Spec.Platforms { + if err := validatePlatform(pl); err != nil { + return errors.Wrapf(err, "platform (%+v) is badly constructed", pl) + } + } + return nil +} + +func validatePlatform(p Platform) error { + if p.URI == "" { + return errors.New("`uri` has to be set") + } + if p.Sha256 == "" { + return errors.New("`sha256` sum has to be set") + } + if p.Bin == "" { + return errors.New("`bin` has to be set") + } + if err := validateFiles(p.Files); err != nil { + return errors.Wrap(err, "`files` is invalid") + } + if err := validateSelector(p.Selector); err != nil { + return errors.Wrap(err, "invalid platform selector") + } + return nil +} + +func validateFiles(fops []FileOperation) error { + if fops == nil { + return nil + } + if len(fops) == 0 { + return errors.New("`files` has to be unspecified or non-empty") + } + for _, op := range fops { + if op.From == "" { + return errors.New("`from` field has to be set") + } else if op.To == "" { + return errors.New("`to` field has to be set") + } + } + return nil +} + +// validateSelector checks if the platform selector uses supported keys and is not empty or nil. +func validateSelector(sel *metav1.LabelSelector) error { + if sel == nil { + return errors.New("nil selector is not supported") + } + if sel.MatchLabels == nil && len(sel.MatchExpressions) == 0 { + return errors.New("empty selector is not supported") + } + + // check for unsupported keys + keys := []string{} + for k := range sel.MatchLabels { + keys = append(keys, k) + } + for _, expr := range sel.MatchExpressions { + keys = append(keys, expr.Key) + } + for _, key := range keys { + if key != "os" && key != "arch" { + return errors.Errorf("key %q not supported", key) + } + } + + if sel.MatchLabels != nil && len(sel.MatchLabels) == 0 { + return errors.New("`matchLabels` specified but empty") + } + if sel.MatchExpressions != nil && len(sel.MatchExpressions) == 0 { + return errors.New("`matchExpressions` specified but empty") + } + + return nil +} diff --git a/internal/cli/spinner/windows_spinner.go b/internal/cli/spinner/windows_spinner.go index f1c89b24f..ac50fb871 100644 --- a/internal/cli/spinner/windows_spinner.go +++ b/internal/cli/spinner/windows_spinner.go @@ -1,3 +1,22 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + package spinner import ( diff --git a/internal/cli/util/git.go b/internal/cli/util/git.go index b511b9cdd..420af029a 100644 --- a/internal/cli/util/git.go +++ b/internal/cli/util/git.go @@ -20,10 +20,17 @@ along with this program. If not, see . package util import ( + "bytes" + "io" "os" + "os/exec" + "path/filepath" + "strings" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/pkg/errors" + "k8s.io/klog/v2" ) // CloneGitRepo clone git repo to local path @@ -70,3 +77,68 @@ func CloneGitRepo(url, branch, path string) error { }) return err } + +func GitGetRemoteURL(dir string) (string, error) { + return ExecGitCommand(dir, "config", "--get", "remote.origin.url") +} + +// EnsureCloned will clone into the destination path, otherwise will return no error. +func EnsureCloned(uri, destinationPath string) error { + if ok, err := IsGitCloned(destinationPath); err != nil { + return err + } else if !ok { + _, err = ExecGitCommand("", "clone", "-v", uri, destinationPath) + return err + } + return nil +} + +// IsGitCloned will test if the path is a git dir. +func IsGitCloned(gitPath string) (bool, error) { + f, err := os.Stat(filepath.Join(gitPath, ".git")) + if os.IsNotExist(err) { + return false, nil + } + return err == nil && f.IsDir(), err +} + +// EnsureUpdated will ensure the destination path exists and is up to date. +func EnsureUpdated(uri, destinationPath string) error { + if err := EnsureCloned(uri, destinationPath); err != nil { + return err + } + return UpdateAndCleanUntracked(destinationPath) +} + +// UpdateAndCleanUntracked will fetch origin and set HEAD to origin/HEAD +// and also will create a pristine working directory by removing +// untracked files and directories. +func UpdateAndCleanUntracked(destinationPath string) error { + if _, err := ExecGitCommand(destinationPath, "fetch", "-v"); err != nil { + return errors.Wrapf(err, "fetch index at %q failed", destinationPath) + } + + if _, err := ExecGitCommand(destinationPath, "reset", "--hard", "@{upstream}"); err != nil { + return errors.Wrapf(err, "reset index at %q failed", destinationPath) + } + + _, err := ExecGitCommand(destinationPath, "clean", "-xfd") + return errors.Wrapf(err, "clean index at %q failed", destinationPath) +} + +// ExecGitCommand executes a git command in the given directory. +func ExecGitCommand(pwd string, args ...string) (string, error) { + klog.V(4).Infof("Going to run git %s", strings.Join(args, " ")) + cmd := exec.Command("git", args...) + cmd.Dir = pwd + buf := bytes.Buffer{} + var w io.Writer = &buf + if klog.V(2).Enabled() { + w = io.MultiWriter(w, os.Stderr) + } + cmd.Stdout, cmd.Stderr = w, w + if err := cmd.Run(); err != nil { + return "", errors.Wrapf(err, "command execution failure, output=%q", buf.String()) + } + return strings.TrimSpace(buf.String()), nil +} From 8c57cb540bd4bc5ed22327eb38cfcd09c6d5b82b Mon Sep 17 00:00:00 2001 From: dingben Date: Mon, 15 May 2023 15:27:28 +0800 Subject: [PATCH 294/439] feat: support to specify default clusterVersion and cli use it (#3104) --- .../postgresql/templates/clusterversion.yaml | 2 + internal/cli/cmd/cluster/cluster_test.go | 3 +- internal/cli/cmd/cluster/create.go | 68 ++++++++++++++++--- internal/constant/const.go | 1 + 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/deploy/postgresql/templates/clusterversion.yaml b/deploy/postgresql/templates/clusterversion.yaml index cd0eb4ab6..3d3cbd5ea 100644 --- a/deploy/postgresql/templates/clusterversion.yaml +++ b/deploy/postgresql/templates/clusterversion.yaml @@ -23,6 +23,8 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterVersion metadata: name: postgresql-12.14.0 + annotations: + kubeblocks.io/is-default-cluster-version: "true" labels: {{- include "postgresql.labels" . | nindent 4 }} spec: diff --git a/internal/cli/cmd/cluster/cluster_test.go b/internal/cli/cmd/cluster/cluster_test.go index d290a8804..aa6d3c90f 100644 --- a/internal/cli/cmd/cluster/cluster_test.go +++ b/internal/cli/cmd/cluster/cluster_test.go @@ -105,6 +105,7 @@ var _ = Describe("Cluster", func() { tf.FakeDynamicClient = testing.FakeDynamicClient( clusterDef, testing.FakeStorageClass(testing.StorageClassName, testing.IsDefautl), + testing.FakeClusterVersion(), testing.FakeComponentClassDef(fmt.Sprintf("custom-%s", testing.ComponentDefName), clusterDef.Name, testing.ComponentDefName), testing.FakeComponentClassDef("custom-mysql", clusterDef.Name, "mysql"), ) @@ -119,7 +120,7 @@ var _ = Describe("Cluster", func() { }, SetFile: "", ClusterDefRef: testing.ClusterDefName, - ClusterVersionRef: "cluster-version", + ClusterVersionRef: testing.ClusterVersionName, UpdatableFlags: UpdatableFlags{ PodAntiAffinity: "Preferred", TopologyKeys: []string{"kubernetes.io/hostname"}, diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index d5912b5be..e5959bfe0 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -32,6 +32,7 @@ import ( "github.com/ghodss/yaml" "github.com/spf13/cobra" "github.com/spf13/viper" + "golang.org/x/exp/maps" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -290,13 +291,8 @@ func (o *CreateOptions) Validate() error { return fmt.Errorf("a valid termination policy is needed, use --termination-policy to specify one of: DoNotTerminate, Halt, Delete, WipeOut") } - if o.ClusterVersionRef == "" { - version, err := cluster.GetLatestVersion(o.Dynamic, o.ClusterDefRef) - if err != nil { - return err - } - o.ClusterVersionRef = version - fmt.Fprintf(o.Out, "Info: --cluster-version is not specified, ClusterVersion %s is applied by default\n", o.ClusterVersionRef) + if err := o.validateClusterVersion(); err != nil { + return err } if len(o.Values) > 0 && len(o.SetFile) > 0 { @@ -941,13 +937,69 @@ func getStorageClasses(dynamic dynamic.Interface) (map[string]struct{}, bool, er for _, item := range list.Items { allStorageClasses[item.GetName()] = struct{}{} annotations := item.GetAnnotations() - if !existedDefault && annotations != nil && (annotations[storage.IsDefaultStorageClassAnnotation] == "true" || annotations[storage.BetaIsDefaultStorageClassAnnotation] == "true") { + if !existedDefault && annotations != nil && (annotations[storage.IsDefaultStorageClassAnnotation] == annotationTrueValue || annotations[storage.BetaIsDefaultStorageClassAnnotation] == annotationTrueValue) { existedDefault = true } } return allStorageClasses, existedDefault, nil } +// validateClusterVersion check whether the cluster version we need is exist in K8S or +// the default cluster version is exist +func (o *CreateOptions) validateClusterVersion() error { + existedClusterVersions, defaultVersion, existedDefault, err := getClusterVersions(o.Dynamic, o.ClusterDefRef) + if err != nil { + return err + } + switch { + case o.ClusterVersionRef != "": + if _, ok := existedClusterVersions[o.ClusterVersionRef]; !ok { + return fmt.Errorf("failed to find the specified cluster version \"%s\"", o.ClusterVersionRef) + } + case !existedDefault: + // if default version is not set and there is only one version, use it + if len(existedClusterVersions) == 1 { + o.ClusterVersionRef = maps.Keys(existedClusterVersions)[0] + fmt.Fprintf(o.Out, "Info: --cluster-version is not specified, ClusterVersion %s is applied by default\n", o.ClusterVersionRef) + } else { + return fmt.Errorf("failed to find the default cluster version, use '--cluster-version ClusterVersion' to set it") + } + case existedDefault: + // TODO: achieve this in operator + if existedDefault { + o.ClusterVersionRef = defaultVersion + fmt.Fprintf(o.Out, "Info: --cluster-version is not specified, ClusterVersion %s is applied by default\n", o.ClusterVersionRef) + } + } + + return nil +} + +// getClusterVersions return all cluster versions in K8S and return true if the cluster have a default cluster version +func getClusterVersions(dynamic dynamic.Interface, clusterDef string) (map[string]struct{}, string, bool, error) { + allClusterVersions := make(map[string]struct{}) + existedDefault := false + defaultVersion := "" + list, err := dynamic.Resource(types.ClusterVersionGVR()).List(context.Background(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", constant.ClusterDefLabelKey, clusterDef), + }) + if err != nil { + return nil, defaultVersion, false, err + } + for _, item := range list.Items { + allClusterVersions[item.GetName()] = struct{}{} + annotations := item.GetAnnotations() + if annotations != nil && annotations[constant.DefaultClusterVersionAnnotationKey] == annotationTrueValue { + if existedDefault { + return nil, defaultVersion, existedDefault, fmt.Errorf("clusterDef %s has more than one default cluster version", clusterDef) + } + existedDefault = true + defaultVersion = item.GetName() + } + } + return allClusterVersions, defaultVersion, existedDefault, nil +} + func shouldCreateDependencies(cd *appsv1alpha1.ClusterDefinition, compSpec *appsv1alpha1.ClusterComponentSpec) (bool, error) { var compDef *appsv1alpha1.ClusterComponentDefinition if cd.Spec.Type != "postgresql" { diff --git a/internal/constant/const.go b/internal/constant/const.go index dda59961d..630e033f0 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -107,6 +107,7 @@ const ( BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. + DefaultClusterVersionAnnotationKey = "kubeblocks.io/is-default-cluster-version" // DefaultClusterVersionAnnotationKey specifies the default cluster version. // ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl ConfigurationTplLabelPrefixKey = "config.kubeblocks.io/tpl" From 3715df64cf3146dfaca0ca6a9fb2bd47426f248a Mon Sep 17 00:00:00 2001 From: linghan-hub <56351212+linghan-hub@users.noreply.github.com> Date: Mon, 15 May 2023 16:17:43 +0800 Subject: [PATCH 295/439] chore: adjust e2e test cases (#3187) --- deploy/apecloud-mysql-cluster/Chart.yaml | 2 +- .../apecloud-mysql-scale-cluster/Chart.yaml | 2 +- deploy/apecloud-mysql-scale/Chart.yaml | 2 +- deploy/apecloud-mysql/Chart.yaml | 2 +- deploy/chatgpt-retrieval-plugin/Chart.yaml | 2 +- deploy/clickhouse-cluster/Chart.yaml | 2 +- deploy/clickhouse/Chart.yaml | 2 +- deploy/helm/Chart.yaml | 4 +- deploy/kafka-cluster/Chart.yaml | 2 +- deploy/kafka/Chart.yaml | 2 +- deploy/milvus-cluster/Chart.yaml | 2 +- deploy/milvus/Chart.yaml | 2 +- deploy/mongodb-cluster/Chart.yaml | 2 +- deploy/mongodb/Chart.yaml | 2 +- deploy/nyancat/Chart.yaml | 4 +- deploy/postgresql-cluster/Chart.yaml | 2 +- deploy/postgresql/Chart.yaml | 2 +- deploy/qdrant-cluster/Chart.yaml | 2 +- deploy/qdrant/Chart.yaml | 2 +- deploy/redis-cluster/Chart.yaml | 2 +- deploy/redis/Chart.yaml | 2 +- deploy/weaviate-cluster/Chart.yaml | 2 +- deploy/weaviate/Chart.yaml | 2 +- .../smoketest/mongodb/00_mongodbcluster.yaml | 6 +- .../smoketest/mongodb/01_vexpand.yaml | 2 +- .../testdata/smoketest/mongodb/02_stop.yaml | 2 +- .../testdata/smoketest/mongodb/03_start.yaml | 2 +- .../testdata/smoketest/mongodb/04_vscale.yaml | 2 +- .../smoketest/mongodb/05_restart.yaml | 2 +- .../postgresql/00_postgresqlcluster.yaml | 94 ++++++++++++++++++- .../smoketest/postgresql/01_vscale.yaml | 2 +- .../smoketest/postgresql/02_vexpand.yaml | 2 +- .../smoketest/postgresql/03_stop.yaml | 2 +- .../smoketest/postgresql/04_start.yaml | 2 +- .../smoketest/postgresql/05_hscale_up.yaml | 2 +- .../smoketest/postgresql/06_hscale_down.yaml | 2 +- .../testdata/smoketest/postgresql/07_cv.yaml | 6 +- .../smoketest/postgresql/08_upgrade.yaml | 4 +- .../smoketest/postgresql/09_restart.yaml | 2 +- .../postgresql/10_backup_snapshot.yaml | 4 +- ...e.yaml => 11_backup_snapshot_restore.yaml} | 13 +-- .../smoketest/redis/00_rediscluster.yaml | 4 +- .../testdata/smoketest/redis/01_vscale.yaml | 2 +- .../smoketest/redis/02_hscale_up.yaml | 2 +- .../smoketest/redis/03_hscale_down.yaml | 2 +- .../e2e/testdata/smoketest/redis/04_stop.yaml | 2 +- .../testdata/smoketest/redis/05_start.yaml | 2 +- test/e2e/testdata/smoketest/redis/06_cv.yaml | 12 --- .../{08_restart.yaml => 06_restart.yaml} | 4 +- .../{09_vexpand.yaml => 07_vexpand.yaml} | 2 +- test/e2e/testdata/smoketest/redis/08_cv.yaml | 26 +++++ .../{07_upgrade.yaml => 09_upgrade.yaml} | 2 +- test/e2e/testdata/smoketest/smoketestrun.go | 10 +- .../smoketest/wesql/00_wesqlcluster.yaml | 4 +- 54 files changed, 192 insertions(+), 83 deletions(-) rename test/e2e/testdata/smoketest/postgresql/{11_backup_sbapshot_restore.yaml => 11_backup_snapshot_restore.yaml} (66%) delete mode 100644 test/e2e/testdata/smoketest/redis/06_cv.yaml rename test/e2e/testdata/smoketest/redis/{08_restart.yaml => 06_restart.yaml} (70%) rename test/e2e/testdata/smoketest/redis/{09_vexpand.yaml => 07_vexpand.yaml} (86%) create mode 100644 test/e2e/testdata/smoketest/redis/08_cv.yaml rename test/e2e/testdata/smoketest/redis/{07_upgrade.yaml => 09_upgrade.yaml} (80%) diff --git a/deploy/apecloud-mysql-cluster/Chart.yaml b/deploy/apecloud-mysql-cluster/Chart.yaml index 95a4cfe78..bf561c13a 100644 --- a/deploy/apecloud-mysql-cluster/Chart.yaml +++ b/deploy/apecloud-mysql-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: An ApeCloud MySQL Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "8.0.30" diff --git a/deploy/apecloud-mysql-scale-cluster/Chart.yaml b/deploy/apecloud-mysql-scale-cluster/Chart.yaml index bedab4747..25b21f4ec 100644 --- a/deploy/apecloud-mysql-scale-cluster/Chart.yaml +++ b/deploy/apecloud-mysql-scale-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: An ApeCloud MySQL-Scale Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/deploy/apecloud-mysql-scale/Chart.yaml b/deploy/apecloud-mysql-scale/Chart.yaml index 721f1ece0..69fee33be 100644 --- a/deploy/apecloud-mysql-scale/Chart.yaml +++ b/deploy/apecloud-mysql-scale/Chart.yaml @@ -5,7 +5,7 @@ description: ApeCloud MySQL-Scale is ApeCloud MySQL proxy. ApeCloud MySQL-Scale type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 # This is the version number of the ApeCloud MySQL being deployed, # rather than the version number of ApeCloud MySQL-Scale itself. diff --git a/deploy/apecloud-mysql/Chart.yaml b/deploy/apecloud-mysql/Chart.yaml index 314fb48b5..17da6b052 100644 --- a/deploy/apecloud-mysql/Chart.yaml +++ b/deploy/apecloud-mysql/Chart.yaml @@ -9,7 +9,7 @@ description: ApeCloud MySQL is fully compatible with MySQL syntax and supports s type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "8.0.30" diff --git a/deploy/chatgpt-retrieval-plugin/Chart.yaml b/deploy/chatgpt-retrieval-plugin/Chart.yaml index e855405f2..30c517269 100644 --- a/deploy/chatgpt-retrieval-plugin/Chart.yaml +++ b/deploy/chatgpt-retrieval-plugin/Chart.yaml @@ -5,7 +5,7 @@ description: A demo application for ChatGPT plugin. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: 0.1.0 diff --git a/deploy/clickhouse-cluster/Chart.yaml b/deploy/clickhouse-cluster/Chart.yaml index cb5567b1e..e6e46f51a 100644 --- a/deploy/clickhouse-cluster/Chart.yaml +++ b/deploy/clickhouse-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A ClickHouse cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: 22.9.4 diff --git a/deploy/clickhouse/Chart.yaml b/deploy/clickhouse/Chart.yaml index afa7f84e4..df9c80360 100644 --- a/deploy/clickhouse/Chart.yaml +++ b/deploy/clickhouse/Chart.yaml @@ -9,7 +9,7 @@ annotations: type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: 22.9.4 diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml index d1c5b870f..9c7068293 100644 --- a/deploy/helm/Chart.yaml +++ b/deploy/helm/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: 0.5.0-beta.9 +appVersion: 0.5.0-beta.24 kubeVersion: '>=1.22.0-0' diff --git a/deploy/kafka-cluster/Chart.yaml b/deploy/kafka-cluster/Chart.yaml index 83aebacae..665ff7ab1 100644 --- a/deploy/kafka-cluster/Chart.yaml +++ b/deploy/kafka-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A Kafka server cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: 3.4.0 diff --git a/deploy/kafka/Chart.yaml b/deploy/kafka/Chart.yaml index 17ddc92b2..bb2c6d19d 100644 --- a/deploy/kafka/Chart.yaml +++ b/deploy/kafka/Chart.yaml @@ -11,7 +11,7 @@ annotations: type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: 3.4.0 diff --git a/deploy/milvus-cluster/Chart.yaml b/deploy/milvus-cluster/Chart.yaml index 267644d67..4d2d6d606 100644 --- a/deploy/milvus-cluster/Chart.yaml +++ b/deploy/milvus-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A Milvus cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "2.2.4" diff --git a/deploy/milvus/Chart.yaml b/deploy/milvus/Chart.yaml index cde9617ef..4a135a813 100644 --- a/deploy/milvus/Chart.yaml +++ b/deploy/milvus/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 # This is the version number of milvus appVersion: "2.2.4" diff --git a/deploy/mongodb-cluster/Chart.yaml b/deploy/mongodb-cluster/Chart.yaml index 2f908d459..ca14ddb1f 100644 --- a/deploy/mongodb-cluster/Chart.yaml +++ b/deploy/mongodb-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A MongoDB cluster Helm chart for KubeBlocks type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "5.0.14" diff --git a/deploy/mongodb/Chart.yaml b/deploy/mongodb/Chart.yaml index 3a887cb7e..82b1b08ed 100644 --- a/deploy/mongodb/Chart.yaml +++ b/deploy/mongodb/Chart.yaml @@ -4,7 +4,7 @@ description: MongoDB is a document database designed for ease of application dev type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "5.0.14" diff --git a/deploy/nyancat/Chart.yaml b/deploy/nyancat/Chart.yaml index 40baa823d..906f71bc8 100644 --- a/deploy/nyancat/Chart.yaml +++ b/deploy/nyancat/Chart.yaml @@ -4,8 +4,8 @@ description: A demo application for showing database cluster availability. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 -appVersion: 0.5.0-beta.9 +appVersion: 0.5.0-beta.24 kubeVersion: '>=1.22.0-0' diff --git a/deploy/postgresql-cluster/Chart.yaml b/deploy/postgresql-cluster/Chart.yaml index 886fa9027..a300940c2 100644 --- a/deploy/postgresql-cluster/Chart.yaml +++ b/deploy/postgresql-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: A PostgreSQL (with Patroni HA) cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 # appVersion specifies the version of the PostgreSQL (with Patroni HA) database to be created, # and this value should be consistent with an existing clusterVersion. diff --git a/deploy/postgresql/Chart.yaml b/deploy/postgresql/Chart.yaml index 687630734..ac8bcc825 100644 --- a/deploy/postgresql/Chart.yaml +++ b/deploy/postgresql/Chart.yaml @@ -4,7 +4,7 @@ description: A PostgreSQL (with Patroni HA) cluster definition Helm chart for Ku type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 # The helm chart contains multiple kernel versions of PostgreSQL (with Patroni HA), # and each PostgreSQL (with Patroni HA) version corresponds to a clusterVersion object. diff --git a/deploy/qdrant-cluster/Chart.yaml b/deploy/qdrant-cluster/Chart.yaml index 82cf28579..793ed6189 100644 --- a/deploy/qdrant-cluster/Chart.yaml +++ b/deploy/qdrant-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A Qdrant cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "1.1.0" diff --git a/deploy/qdrant/Chart.yaml b/deploy/qdrant/Chart.yaml index bc13d4abb..227712281 100644 --- a/deploy/qdrant/Chart.yaml +++ b/deploy/qdrant/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version. -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 # This is the version number of qdrant. appVersion: "1.1.0" diff --git a/deploy/redis-cluster/Chart.yaml b/deploy/redis-cluster/Chart.yaml index 90dbafa30..fcd612190 100644 --- a/deploy/redis-cluster/Chart.yaml +++ b/deploy/redis-cluster/Chart.yaml @@ -4,7 +4,7 @@ description: An Redis Replication Cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "7.0.6" diff --git a/deploy/redis/Chart.yaml b/deploy/redis/Chart.yaml index 1eaf19bee..1ccb968de 100644 --- a/deploy/redis/Chart.yaml +++ b/deploy/redis/Chart.yaml @@ -4,7 +4,7 @@ description: A Redis cluster definition Helm chart for Kubernetes type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "7.0.6" diff --git a/deploy/weaviate-cluster/Chart.yaml b/deploy/weaviate-cluster/Chart.yaml index 6ef6c4ac4..4254b0356 100644 --- a/deploy/weaviate-cluster/Chart.yaml +++ b/deploy/weaviate-cluster/Chart.yaml @@ -4,6 +4,6 @@ description: A weaviate cluster Helm chart for KubeBlocks. type: application -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 appVersion: "1.18.0" diff --git a/deploy/weaviate/Chart.yaml b/deploy/weaviate/Chart.yaml index 4853384ab..32f36a70a 100644 --- a/deploy/weaviate/Chart.yaml +++ b/deploy/weaviate/Chart.yaml @@ -5,7 +5,7 @@ description: . type: application # This is the chart version. -version: 0.5.0-beta.9 +version: 0.5.0-beta.24 # This is the version number of weaviate. appVersion: "1.18.0" diff --git a/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml b/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml index 9a99a9c2e..eb2b08774 100644 --- a/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml +++ b/test/e2e/testdata/smoketest/mongodb/00_mongodbcluster.yaml @@ -5,7 +5,7 @@ kind: Cluster metadata: name: mycluster labels: - helm.sh/chart: mongodb-cluster-0.5.0-beta.9 + helm.sh/chart: mongodb-cluster-0.5.0-beta.24 app.kubernetes.io/name: mongodb-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "5.0.14" @@ -18,8 +18,8 @@ spec: topologyKeys: - kubernetes.io/hostname componentSpecs: - - name: replicaset - componentDefRef: replicaset + - name: mongodb + componentDefRef: mongodb monitor: false replicas: 3 volumeClaimTemplates: diff --git a/test/e2e/testdata/smoketest/mongodb/01_vexpand.yaml b/test/e2e/testdata/smoketest/mongodb/01_vexpand.yaml index 0e1633071..d4c845bcd 100644 --- a/test/e2e/testdata/smoketest/mongodb/01_vexpand.yaml +++ b/test/e2e/testdata/smoketest/mongodb/01_vexpand.yaml @@ -6,7 +6,7 @@ spec: clusterRef: mycluster type: VolumeExpansion volumeExpansion: - - componentName: replicaset + - componentName: mongodb volumeClaimTemplates: - name: data storage: "2Gi" \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/mongodb/02_stop.yaml b/test/e2e/testdata/smoketest/mongodb/02_stop.yaml index dc42f742d..063f11cf5 100644 --- a/test/e2e/testdata/smoketest/mongodb/02_stop.yaml +++ b/test/e2e/testdata/smoketest/mongodb/02_stop.yaml @@ -7,4 +7,4 @@ spec: ttlSecondsAfterSucceed: 27017 type: Stop restart: - - componentName: replicaset \ No newline at end of file + - componentName: mongodb \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/mongodb/03_start.yaml b/test/e2e/testdata/smoketest/mongodb/03_start.yaml index c3eddb51a..96195dd4b 100644 --- a/test/e2e/testdata/smoketest/mongodb/03_start.yaml +++ b/test/e2e/testdata/smoketest/mongodb/03_start.yaml @@ -7,4 +7,4 @@ spec: ttlSecondsAfterSucceed: 27017 type: Start restart: - - componentName: replicaset \ No newline at end of file + - componentName: mongodb \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/mongodb/04_vscale.yaml b/test/e2e/testdata/smoketest/mongodb/04_vscale.yaml index ae3e4c701..5a145b32b 100644 --- a/test/e2e/testdata/smoketest/mongodb/04_vscale.yaml +++ b/test/e2e/testdata/smoketest/mongodb/04_vscale.yaml @@ -6,7 +6,7 @@ spec: clusterRef: mycluster type: VerticalScaling verticalScaling: - - componentName: replicaset + - componentName: mongodb requests: cpu: "500m" memory: 500Mi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/mongodb/05_restart.yaml b/test/e2e/testdata/smoketest/mongodb/05_restart.yaml index bf4a126b9..5a87555d3 100644 --- a/test/e2e/testdata/smoketest/mongodb/05_restart.yaml +++ b/test/e2e/testdata/smoketest/mongodb/05_restart.yaml @@ -7,4 +7,4 @@ spec: ttlSecondsAfterSucceed: 27017 type: Restart restart: - - componentName: replicaset \ No newline at end of file + - componentName: mongodb \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml b/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml index d701711f6..be9a57b2e 100644 --- a/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml +++ b/test/e2e/testdata/smoketest/postgresql/00_postgresqlcluster.yaml @@ -1,18 +1,103 @@ --- +# Source: pgcluster/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kb-sa-mycluster + labels: + + helm.sh/chart: pgcluster-0.5.0-beta.24 + app.kubernetes.io/name: pgcluster + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "14.7.1" + app.kubernetes.io/managed-by: Helm +--- +# Source: pgcluster/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-default-mycluster + namespace: default + labels: + + helm.sh/chart: pgcluster-0.5.0-beta.24 + app.kubernetes.io/name: pgcluster + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "14.7.1" + app.kubernetes.io/managed-by: Helm +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - patch + - update + - watch + # delete is required only for 'patronictl remove' + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - patch + - update + - create + - list + - watch + # delete is required only for 'patronictl remove' + - delete + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch +--- +# Source: pgcluster/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-rolebinding-default-mycluster + labels: + + helm.sh/chart: pgcluster-0.5.0-beta.24 + app.kubernetes.io/name: pgcluster + app.kubernetes.io/instance: mycluster + app.kubernetes.io/version: "14.7.1" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-default-mycluster +subjects: + - kind: ServiceAccount + name: kb-sa-mycluster + namespace: default +--- # Source: pgcluster/templates/cluster.yaml apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: mycluster + name: mycluster-pgcluster labels: - helm.sh/chart: pgcluster-0.5.0-beta.9 + helm.sh/chart: pgcluster-0.5.0-beta.24 app.kubernetes.io/name: pgcluster app.kubernetes.io/instance: mycluster - app.kubernetes.io/version: "14.7.0" + app.kubernetes.io/version: "14.7.1" app.kubernetes.io/managed-by: Helm spec: clusterDefinitionRef: postgresql # ref clusterdefinition.name - clusterVersionRef: postgresql-14.7.0 # ref clusterversion.name + clusterVersionRef: postgresql-14.7.1 # ref clusterversion.name terminationPolicy: Delete affinity: componentSpecs: @@ -20,6 +105,7 @@ spec: componentDefRef: postgresql # ref clusterdefinition components.name monitor: false replicas: 2 + serviceAccountName: kb-sa-mycluster primaryIndex: 0 switchPolicy: type: Noop diff --git a/test/e2e/testdata/smoketest/postgresql/01_vscale.yaml b/test/e2e/testdata/smoketest/postgresql/01_vscale.yaml index 15085de54..488d51b6c 100644 --- a/test/e2e/testdata/smoketest/postgresql/01_vscale.yaml +++ b/test/e2e/testdata/smoketest/postgresql/01_vscale.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-vscale spec: - clusterRef: mycluster + clusterRef: mycluster-pgcluster type: VerticalScaling verticalScaling: - componentName: postgresql diff --git a/test/e2e/testdata/smoketest/postgresql/02_vexpand.yaml b/test/e2e/testdata/smoketest/postgresql/02_vexpand.yaml index ba32370ce..438a5d65e 100644 --- a/test/e2e/testdata/smoketest/postgresql/02_vexpand.yaml +++ b/test/e2e/testdata/smoketest/postgresql/02_vexpand.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-vexpand spec: - clusterRef: mycluster + clusterRef: mycluster-pgcluster type: VolumeExpansion volumeExpansion: - componentName: postgresql diff --git a/test/e2e/testdata/smoketest/postgresql/03_stop.yaml b/test/e2e/testdata/smoketest/postgresql/03_stop.yaml index c3220f818..b7432c544 100644 --- a/test/e2e/testdata/smoketest/postgresql/03_stop.yaml +++ b/test/e2e/testdata/smoketest/postgresql/03_stop.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-stop spec: - clusterRef: mycluster + clusterRef: mycluster-pgcluster ttlSecondsAfterSucceed: 5432 type: Stop restart: diff --git a/test/e2e/testdata/smoketest/postgresql/04_start.yaml b/test/e2e/testdata/smoketest/postgresql/04_start.yaml index 200178e44..13d612e05 100644 --- a/test/e2e/testdata/smoketest/postgresql/04_start.yaml +++ b/test/e2e/testdata/smoketest/postgresql/04_start.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-start spec: - clusterRef: mycluster + clusterRef: mycluster-pgcluster ttlSecondsAfterSucceed: 5432 type: Start restart: diff --git a/test/e2e/testdata/smoketest/postgresql/05_hscale_up.yaml b/test/e2e/testdata/smoketest/postgresql/05_hscale_up.yaml index d156a4069..79bc8bf2b 100644 --- a/test/e2e/testdata/smoketest/postgresql/05_hscale_up.yaml +++ b/test/e2e/testdata/smoketest/postgresql/05_hscale_up.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-hscale-up spec: - clusterRef: mycluster + clusterRef: mycluster-pgcluster type: HorizontalScaling horizontalScaling: - componentName: postgresql diff --git a/test/e2e/testdata/smoketest/postgresql/06_hscale_down.yaml b/test/e2e/testdata/smoketest/postgresql/06_hscale_down.yaml index 5cfc80700..fbb8ed33c 100644 --- a/test/e2e/testdata/smoketest/postgresql/06_hscale_down.yaml +++ b/test/e2e/testdata/smoketest/postgresql/06_hscale_down.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-hscale-down spec: - clusterRef: mycluster + clusterRef: mycluster-pgcluster type: HorizontalScaling horizontalScaling: - componentName: postgresql diff --git a/test/e2e/testdata/smoketest/postgresql/07_cv.yaml b/test/e2e/testdata/smoketest/postgresql/07_cv.yaml index 6da78cd27..47fd51ac3 100644 --- a/test/e2e/testdata/smoketest/postgresql/07_cv.yaml +++ b/test/e2e/testdata/smoketest/postgresql/07_cv.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: ClusterVersion metadata: - name: postgresql-14.7.0-latest + name: postgresql-14.7.1-latest spec: clusterDefinitionRef: postgresql componentVersions: @@ -9,7 +9,7 @@ spec: versionsContext: containers: - name: postgresql - image: docker.io/apecloud/postgresql:14.7.0 + image: registry.cn-hangzhou.aliyuncs.com/apecloud/spilo:14.7.1 initContainers: - - image: docker.io/apecloud/postgresql:14.7.0 + - image: registry.cn-hangzhou.aliyuncs.com/apecloud/spilo:14.7.1 name: pg-init-container \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml b/test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml index abf304c68..7018f4b86 100644 --- a/test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml +++ b/test/e2e/testdata/smoketest/postgresql/08_upgrade.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-upgrade spec: - clusterRef: mycluster + clusterRef: mycluster-pgcluster type: Upgrade upgrade: - clusterVersionRef: postgresql-14.7.0-latest \ No newline at end of file + clusterVersionRef: postgresql-14.7.1-latest \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/09_restart.yaml b/test/e2e/testdata/smoketest/postgresql/09_restart.yaml index 331c103b6..dfe4cb3d0 100644 --- a/test/e2e/testdata/smoketest/postgresql/09_restart.yaml +++ b/test/e2e/testdata/smoketest/postgresql/09_restart.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-restart spec: - clusterRef: mycluster + clusterRef: mycluster-pgcluster ttlSecondsAfterSucceed: 5432 type: Restart restart: diff --git a/test/e2e/testdata/smoketest/postgresql/10_backup_snapshot.yaml b/test/e2e/testdata/smoketest/postgresql/10_backup_snapshot.yaml index 74998c0fa..4afcd662e 100644 --- a/test/e2e/testdata/smoketest/postgresql/10_backup_snapshot.yaml +++ b/test/e2e/testdata/smoketest/postgresql/10_backup_snapshot.yaml @@ -4,7 +4,7 @@ metadata: labels: app.kubernetes.io/instance: mycluster dataprotection.kubeblocks.io/backup-type: snapshot - name: backup-sbapshot-mycluster + name: backup-snapshot-mycluster spec: - backupPolicyName: mycluster-postgresql-backup-policy + backupPolicyName: mycluster-pgcluster-postgresql-backup-policy backupType: snapshot \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/postgresql/11_backup_sbapshot_restore.yaml b/test/e2e/testdata/smoketest/postgresql/11_backup_snapshot_restore.yaml similarity index 66% rename from test/e2e/testdata/smoketest/postgresql/11_backup_sbapshot_restore.yaml rename to test/e2e/testdata/smoketest/postgresql/11_backup_snapshot_restore.yaml index 89aeb5629..48ae36239 100644 --- a/test/e2e/testdata/smoketest/postgresql/11_backup_sbapshot_restore.yaml +++ b/test/e2e/testdata/smoketest/postgresql/11_backup_snapshot_restore.yaml @@ -1,16 +1,17 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: mycluster-sbapshot + name: mycluster-snapshot annotations: - kubeblocks.io/restore-from-backup: "{\"postgresql\":\"backup-sbapshot-mycluster\"}" + kubeblocks.io/restore-from-backup: "{\"postgresql\":\"backup-snapshot-mycluster\"}" spec: clusterDefinitionRef: postgresql - clusterVersionRef: postgresql-14.7.0 + clusterVersionRef: postgresql-14.7.1 terminationPolicy: WipeOut componentSpecs: - - name: wesql - componentDefRef: mysql + - name: postgresql + componentDefRef: postgresql + serviceAccountName: kb-sa-mycluster monitor: false replicas: 1 volumeClaimTemplates: @@ -20,4 +21,4 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi \ No newline at end of file + storage: 20Gi \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml index fa4728d27..cd7cc4a07 100644 --- a/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml +++ b/test/e2e/testdata/smoketest/redis/00_rediscluster.yaml @@ -3,9 +3,9 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: mycluster + name: mycluster-redis-cluster labels: - helm.sh/chart: redis-cluster-0.5.0-beta.9 + helm.sh/chart: redis-cluster-0.5.0-beta.24 app.kubernetes.io/name: redis-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "7.0.6" diff --git a/test/e2e/testdata/smoketest/redis/01_vscale.yaml b/test/e2e/testdata/smoketest/redis/01_vscale.yaml index 12f474466..6997618ce 100644 --- a/test/e2e/testdata/smoketest/redis/01_vscale.yaml +++ b/test/e2e/testdata/smoketest/redis/01_vscale.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-vscale spec: - clusterRef: mycluster + clusterRef: mycluster-redis-cluster type: VerticalScaling verticalScaling: - componentName: redis diff --git a/test/e2e/testdata/smoketest/redis/02_hscale_up.yaml b/test/e2e/testdata/smoketest/redis/02_hscale_up.yaml index 3a268beb1..1e6fa4e39 100644 --- a/test/e2e/testdata/smoketest/redis/02_hscale_up.yaml +++ b/test/e2e/testdata/smoketest/redis/02_hscale_up.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-hscale-up spec: - clusterRef: mycluster + clusterRef: mycluster-redis-cluster type: HorizontalScaling horizontalScaling: - componentName: redis diff --git a/test/e2e/testdata/smoketest/redis/03_hscale_down.yaml b/test/e2e/testdata/smoketest/redis/03_hscale_down.yaml index bca2e520d..c5be5d225 100644 --- a/test/e2e/testdata/smoketest/redis/03_hscale_down.yaml +++ b/test/e2e/testdata/smoketest/redis/03_hscale_down.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-hscale-down spec: - clusterRef: mycluster + clusterRef: mycluster-redis-cluster type: HorizontalScaling horizontalScaling: - componentName: redis diff --git a/test/e2e/testdata/smoketest/redis/04_stop.yaml b/test/e2e/testdata/smoketest/redis/04_stop.yaml index c8f7bbaf2..3b80455c7 100644 --- a/test/e2e/testdata/smoketest/redis/04_stop.yaml +++ b/test/e2e/testdata/smoketest/redis/04_stop.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-stop spec: - clusterRef: mycluster + clusterRef: mycluster-redis-cluster ttlSecondsAfterSucceed: 3600 type: Stop restart: diff --git a/test/e2e/testdata/smoketest/redis/05_start.yaml b/test/e2e/testdata/smoketest/redis/05_start.yaml index 83ebbd60b..78c5539e7 100644 --- a/test/e2e/testdata/smoketest/redis/05_start.yaml +++ b/test/e2e/testdata/smoketest/redis/05_start.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-start spec: - clusterRef: mycluster + clusterRef: mycluster-redis-cluster ttlSecondsAfterSucceed: 3600 type: Start restart: diff --git a/test/e2e/testdata/smoketest/redis/06_cv.yaml b/test/e2e/testdata/smoketest/redis/06_cv.yaml deleted file mode 100644 index 8c5d177d8..000000000 --- a/test/e2e/testdata/smoketest/redis/06_cv.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: ClusterVersion -metadata: - name: redis-7.0.6-latest -spec: - clusterDefinitionRef: redis - componentVersions: - - componentDefRef: redis - versionsContext: - containers: - - name: redis - image: docker.io/apecloud/redis:latest diff --git a/test/e2e/testdata/smoketest/redis/08_restart.yaml b/test/e2e/testdata/smoketest/redis/06_restart.yaml similarity index 70% rename from test/e2e/testdata/smoketest/redis/08_restart.yaml rename to test/e2e/testdata/smoketest/redis/06_restart.yaml index e9a31e1e2..993d4135a 100644 --- a/test/e2e/testdata/smoketest/redis/08_restart.yaml +++ b/test/e2e/testdata/smoketest/redis/06_restart.yaml @@ -3,8 +3,8 @@ kind: OpsRequest metadata: name: ops-restart spec: - clusterRef: mycluster + clusterRef: mycluster-redis-cluster ttlSecondsAfterSucceed: 3600 type: Restart restart: - - componentName: redis-repl \ No newline at end of file + - componentName: redis \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/09_vexpand.yaml b/test/e2e/testdata/smoketest/redis/07_vexpand.yaml similarity index 86% rename from test/e2e/testdata/smoketest/redis/09_vexpand.yaml rename to test/e2e/testdata/smoketest/redis/07_vexpand.yaml index ba32370ce..8e97ff4f2 100644 --- a/test/e2e/testdata/smoketest/redis/09_vexpand.yaml +++ b/test/e2e/testdata/smoketest/redis/07_vexpand.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-vexpand spec: - clusterRef: mycluster + clusterRef: mycluster-redis-cluster type: VolumeExpansion volumeExpansion: - componentName: postgresql diff --git a/test/e2e/testdata/smoketest/redis/08_cv.yaml b/test/e2e/testdata/smoketest/redis/08_cv.yaml new file mode 100644 index 000000000..93848ebf0 --- /dev/null +++ b/test/e2e/testdata/smoketest/redis/08_cv.yaml @@ -0,0 +1,26 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterVersion +metadata: + name: redis-7.0.6-latest +spec: + clusterDefinitionRef: redis + componentVersions: + - componentDefRef: redis + versionsContext: + containers: + - image: redis/redis-stack-server:7.0.6-RC8 + imagePullPolicy: IfNotPresent + name: redis + resources: {} + - componentDefRef: redis-sentinel + versionsContext: + containers: + - image: redis/redis-stack-server:7.0.6-RC8 + imagePullPolicy: IfNotPresent + name: redis-sentinel + resources: {} + initContainers: + - image: redis/redis-stack-server:7.0.6-RC8 + imagePullPolicy: IfNotPresent + name: init-redis-sentinel + resources: {} \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/redis/07_upgrade.yaml b/test/e2e/testdata/smoketest/redis/09_upgrade.yaml similarity index 80% rename from test/e2e/testdata/smoketest/redis/07_upgrade.yaml rename to test/e2e/testdata/smoketest/redis/09_upgrade.yaml index 43ca5a22a..a789b1995 100644 --- a/test/e2e/testdata/smoketest/redis/07_upgrade.yaml +++ b/test/e2e/testdata/smoketest/redis/09_upgrade.yaml @@ -3,7 +3,7 @@ kind: OpsRequest metadata: name: ops-upgrade spec: - clusterRef: mycluster + clusterRef: mycluster-redis-cluster type: Upgrade upgrade: clusterVersionRef: redis-7.0.6-latest \ No newline at end of file diff --git a/test/e2e/testdata/smoketest/smoketestrun.go b/test/e2e/testdata/smoketest/smoketestrun.go index cc577650b..950640921 100644 --- a/test/e2e/testdata/smoketest/smoketestrun.go +++ b/test/e2e/testdata/smoketest/smoketestrun.go @@ -93,6 +93,13 @@ func SmokeTest() { } } }) + It("check addon", func() { + enabledSc := " kbcli addon enable csi-hostpath-driver" + log.Println(enabledSc) + csi := e2eutil.ExecCommand(enabledSc) + log.Println(csi) + }) + It("run test cases", func() { dir, err := os.Getwd() if err != nil { @@ -125,6 +132,7 @@ func SmokeTest() { func runTestCases(files []string) { for _, file := range files { By("test " + file) + b := e2eutil.OpsYaml(file, "create") Expect(b).Should(BeTrue()) Eventually(func(g Gomega) { @@ -137,7 +145,7 @@ func runTestCases(files []string) { Eventually(func(g Gomega) { clusterStatusResult := e2eutil.CheckClusterStatus() g.Expect(clusterStatusResult).Should(BeTrue()) - }, time.Second*240, time.Second*1).Should(Succeed()) + }, time.Second*300, time.Second*1).Should(Succeed()) } if len(files) > 0 { diff --git a/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml b/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml index b79719842..497b24bcf 100644 --- a/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml +++ b/test/e2e/testdata/smoketest/wesql/00_wesqlcluster.yaml @@ -3,9 +3,9 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: mycluster + name: mycluster-apecloud-mysql-cluster labels: - helm.sh/chart: apecloud-mysql-cluster-0.5.0-beta.9 + helm.sh/chart: apecloud-mysql-cluster-0.5.0-beta.24 app.kubernetes.io/name: apecloud-mysql-cluster app.kubernetes.io/instance: mycluster app.kubernetes.io/version: "8.0.30" From 54e1b564e2044b2b22017ddcf50e7ab4a1c07f65 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Mon, 15 May 2023 16:18:16 +0800 Subject: [PATCH 296/439] fix: cli playground failed (#3240) --- internal/cli/cmd/kubeblocks/config_test.go | 2 +- internal/cli/cmd/kubeblocks/install.go | 8 ++++---- internal/cli/cmd/kubeblocks/install_test.go | 8 ++++---- internal/cli/cmd/kubeblocks/upgrade.go | 2 +- internal/cli/cmd/playground/init.go | 3 +++ 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/internal/cli/cmd/kubeblocks/config_test.go b/internal/cli/cmd/kubeblocks/config_test.go index 5c8a6a83c..19358af1e 100644 --- a/internal/cli/cmd/kubeblocks/config_test.go +++ b/internal/cli/cmd/kubeblocks/config_test.go @@ -106,7 +106,7 @@ var _ = Describe("backupconfig", func() { } cmd := NewConfigCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) - Expect(o.PrecheckBeforeInstall()).Should(Succeed()) + Expect(o.PreCheck()).Should(Succeed()) }) It("run describe config cmd", func() { diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index 171f44e9e..bb1ff5ee5 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -119,7 +119,7 @@ func newInstallCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr Example: installExample, Run: func(cmd *cobra.Command, args []string) { util.CheckErr(o.Complete(f, cmd)) - util.CheckErr(o.PrecheckBeforeInstall()) + util.CheckErr(o.PreCheck()) util.CheckErr(p.Preflight(f, args, o.ValueOpts)) util.CheckErr(o.Install()) }, @@ -177,7 +177,7 @@ func (o *Options) Complete(f cmdutil.Factory, cmd *cobra.Command) error { return err } -func (o *InstallOptions) PrecheckBeforeInstall() error { +func (o *InstallOptions) PreCheck() error { // check if KubeBlocks has been installed v, err := util.GetVersionInfo(o.Client) if err != nil { @@ -204,7 +204,7 @@ func (o *InstallOptions) PrecheckBeforeInstall() error { return err } - if err = o.preCheck(v); err != nil { + if err = o.checkVersion(v); err != nil { return err } return nil @@ -359,7 +359,7 @@ func (o *InstallOptions) waitAddonsEnabled() error { return nil } -func (o *InstallOptions) preCheck(v util.Version) error { +func (o *InstallOptions) checkVersion(v util.Version) error { if !o.Check { return nil } diff --git a/internal/cli/cmd/kubeblocks/install_test.go b/internal/cli/cmd/kubeblocks/install_test.go index 4e4687310..09fa7ac02 100644 --- a/internal/cli/cmd/kubeblocks/install_test.go +++ b/internal/cli/cmd/kubeblocks/install_test.go @@ -96,7 +96,7 @@ var _ = Describe("kubeblocks install", func() { o.printNotes() }) - It("preCheck", func() { + It("checkVersion", func() { o := &InstallOptions{ Options: Options{ IOStreams: genericclioptions.NewTestIOStreamsDiscard(), @@ -106,14 +106,14 @@ var _ = Describe("kubeblocks install", func() { } By("kubernetes version is empty") v := util.Version{} - Expect(o.preCheck(v).Error()).Should(ContainSubstring("failed to get kubernetes version")) + Expect(o.checkVersion(v).Error()).Should(ContainSubstring("failed to get kubernetes version")) By("kubernetes is provided by cloud provider") v.Kubernetes = "v1.25.0-eks" - Expect(o.preCheck(v)).Should(Succeed()) + Expect(o.checkVersion(v)).Should(Succeed()) By("kubernetes is not provided by cloud provider") v.Kubernetes = "v1.25.0" - Expect(o.preCheck(v)).Should(Succeed()) + Expect(o.checkVersion(v)).Should(Succeed()) }) }) diff --git a/internal/cli/cmd/kubeblocks/upgrade.go b/internal/cli/cmd/kubeblocks/upgrade.go index e322aa1f7..ca65a7c7c 100644 --- a/internal/cli/cmd/kubeblocks/upgrade.go +++ b/internal/cli/cmd/kubeblocks/upgrade.go @@ -108,7 +108,7 @@ func (o *InstallOptions) Upgrade() error { } fmt.Fprintf(o.Out, "Current KubeBlocks version %s.\n", v) - if err = o.preCheck(v); err != nil { + if err = o.checkVersion(v); err != nil { return err } diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index 4467a01ec..f3e3d51e9 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -444,6 +444,9 @@ func (o *initOptions) installKubeBlocks(k8sClusterName string) error { ) } + if err = insOpts.PreCheck(); err != nil { + return err + } return insOpts.Install() } From 4a2cff389561760a0b99b0e3f525e818e9e9ded5 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Mon, 15 May 2023 16:23:11 +0800 Subject: [PATCH 297/439] support: modify backup type named for backup policy (#3227) --- .../v1alpha1/backuppolicytemplate_types.go | 39 +++++++------- apis/dataprotection/v1alpha1/backup_types.go | 20 +++---- .../v1alpha1/backuppolicy_types.go | 47 ++++++++-------- apis/dataprotection/v1alpha1/types.go | 15 ++---- ...s.kubeblocks.io_backuppolicytemplates.yaml | 53 +++++++++++-------- ...otection.kubeblocks.io_backuppolicies.yaml | 53 +++++++++++-------- .../dataprotection.kubeblocks.io_backups.yaml | 10 ++-- controllers/apps/cluster_controller_test.go | 2 +- .../dataprotection/backup_controller.go | 6 +-- .../dataprotection/backup_controller_test.go | 10 ++-- .../dataprotection/backuppolicy_controller.go | 40 +++++++------- .../backuppolicy_controller_test.go | 20 +++---- controllers/dataprotection/cue/cronjob.cue | 2 +- .../restorejob_controller_test.go | 2 +- .../templates/backuppolicytemplate.yaml | 13 +++-- .../templates/backuppolicytemplate.yaml | 13 +++-- ...s.kubeblocks.io_backuppolicytemplates.yaml | 53 +++++++++++-------- ...otection.kubeblocks.io_backuppolicies.yaml | 53 +++++++++++-------- .../dataprotection.kubeblocks.io_backups.yaml | 10 ++-- .../templates/backuppolicytemplate.yaml | 6 +-- .../templates/backuppolicytemplate.yaml | 13 +++-- .../templates/backuppolicytemplate.yaml | 15 +++--- .../templates/backuppolicytemplate.yaml | 6 +-- .../redis/templates/backuppolicytemplate.yaml | 6 +-- .../templates/backuppolicytemplate.yaml | 6 +-- docs/user_docs/cli/kbcli_cluster_backup.md | 8 +-- internal/cli/cmd/cluster/dataprotection.go | 17 +++--- .../cli/cmd/cluster/dataprotection_test.go | 2 +- internal/cli/cmd/cluster/describe.go | 20 ++++--- .../controller/component/restore_utils.go | 2 +- .../component/restore_utils_test.go | 2 +- .../transformer_backup_policy_tpl.go | 40 ++++++-------- internal/controller/plan/pitr.go | 4 +- internal/controller/plan/pitr_test.go | 4 +- .../testutil/apps/backuppolicy_factory.go | 47 ++++++++-------- .../apps/backuppolicytemplate_factory.go | 46 ++++++++-------- test/integration/backup_mysql_test.go | 2 +- 37 files changed, 387 insertions(+), 320 deletions(-) diff --git a/apis/apps/v1alpha1/backuppolicytemplate_types.go b/apis/apps/v1alpha1/backuppolicytemplate_types.go index cf690df17..e8759e6a3 100644 --- a/apis/apps/v1alpha1/backuppolicytemplate_types.go +++ b/apis/apps/v1alpha1/backuppolicytemplate_types.go @@ -55,11 +55,9 @@ type BackupPolicy struct { // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` ComponentDefRef string `json:"componentDefRef"` - // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long - // the Backup should be retained. if not set, will be retained forever. - // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` + // retention describe how long the Backup should be retained. if not set, will be retained forever. // +optional - TTL *string `json:"ttl,omitempty"` + Retention *RetentionSpec `json:"retention,omitempty"` // schedule policy for backup. // +optional @@ -69,30 +67,35 @@ type BackupPolicy struct { // +optional Snapshot *SnapshotPolicy `json:"snapshot,omitempty"` - // the policy for full backup. + // the policy for datafile backup. // +optional - Full *CommonBackupPolicy `json:"full,omitempty"` + Datafile *CommonBackupPolicy `json:"datafile,omitempty"` + + // the policy for logfile backup. + // +optional + Logfile *CommonBackupPolicy `json:"logfile,omitempty"` +} - // the policy for incremental backup. +type RetentionSpec struct { + // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long + // the Backup should be retained. if not set, will be retained forever. + // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` // +optional - Incremental *CommonBackupPolicy `json:"incremental,omitempty"` + TTL *string `json:"ttl,omitempty"` } type Schedule struct { - // schedule policy for base backup. + // schedule policy for snapshot backup. // +optional - BaseBackup *BaseBackupSchedulePolicy `json:"baseBackup,omitempty"` + Snapshot *SchedulePolicy `json:"snapshot,omitempty"` - // schedule policy for incremental backup. + // schedule policy for datafile backup. // +optional - Incremental *SchedulePolicy `json:"incremental,omitempty"` -} + Datafile *SchedulePolicy `json:"datafile,omitempty"` -type BaseBackupSchedulePolicy struct { - SchedulePolicy `json:",inline"` - // the type of base backup, only support full and snapshot. - // +kubebuilder:validation:Required - Type BaseBackupType `json:"type"` + // schedule policy for logfile backup. + // +optional + Logfile *SchedulePolicy `json:"logfile,omitempty"` } type SchedulePolicy struct { diff --git a/apis/dataprotection/v1alpha1/backup_types.go b/apis/dataprotection/v1alpha1/backup_types.go index 4f36ee442..8b77fb99e 100644 --- a/apis/dataprotection/v1alpha1/backup_types.go +++ b/apis/dataprotection/v1alpha1/backup_types.go @@ -33,8 +33,8 @@ type BackupSpec struct { // +kubebuilder:validation:Pattern:=`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$` BackupPolicyName string `json:"backupPolicyName"` - // Backup Type. full or incremental or snapshot. if unset, default is full. - // +kubebuilder:default=full + // Backup Type. datafile or logfile or snapshot. if unset, default is datafile. + // +kubebuilder:default=datafile BackupType BackupType `json:"backupType"` // if backupType is incremental, parentBackupName is required. @@ -195,13 +195,13 @@ func (r *BackupSpec) Validate(backupPolicy *BackupPolicy) error { if backupPolicy.Spec.Snapshot == nil { return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeSnapshot) } - case BackupTypeFull: - if backupPolicy.Spec.Full == nil { - return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeFull) + case BackupTypeDataFile: + if backupPolicy.Spec.Datafile == nil { + return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeDataFile) } - case BackupTypeIncremental: - if backupPolicy.Spec.Incremental == nil { - return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeIncremental) + case BackupTypeLogFile: + if backupPolicy.Spec.Logfile == nil { + return fmt.Errorf(notSupportedMessage, r.BackupPolicyName, BackupTypeLogFile) } } return nil @@ -217,9 +217,9 @@ func GetRecoverableTimeRange(backups []Backup) []BackupLogStatus { b.Status.Manifests.BackupLog.StopTime == nil { continue } - if b.Spec.BackupType == BackupTypeIncremental { + if b.Spec.BackupType == BackupTypeLogFile { incrementalBackup = &b - } else if b.Spec.BackupType != BackupTypeIncremental && b.Status.Phase == BackupCompleted { + } else if b.Spec.BackupType != BackupTypeLogFile && b.Status.Phase == BackupCompleted { baseBackups = append(baseBackups, b) } } diff --git a/apis/dataprotection/v1alpha1/backuppolicy_types.go b/apis/dataprotection/v1alpha1/backuppolicy_types.go index 773197781..8dc80baad 100644 --- a/apis/dataprotection/v1alpha1/backuppolicy_types.go +++ b/apis/dataprotection/v1alpha1/backuppolicy_types.go @@ -30,11 +30,9 @@ import ( // BackupPolicySpec defines the desired state of BackupPolicy type BackupPolicySpec struct { - // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long - // the Backup should be retained. if not set, will be retained forever. - // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` + // retention describe how long the Backup should be retained. if not set, will be retained forever. // +optional - TTL *string `json:"ttl,omitempty"` + Retention *RetentionSpec `json:"retention,omitempty"` // schedule policy for backup. // +optional @@ -44,30 +42,35 @@ type BackupPolicySpec struct { // +optional Snapshot *SnapshotPolicy `json:"snapshot,omitempty"` - // the policy for full backup. + // the policy for datafile backup. // +optional - Full *CommonBackupPolicy `json:"full,omitempty"` + Datafile *CommonBackupPolicy `json:"datafile,omitempty"` + + // the policy for logfile backup. + // +optional + Logfile *CommonBackupPolicy `json:"logfile,omitempty"` +} - // the policy for incremental backup. +type RetentionSpec struct { + // ttl is a time string ending with the 'd'|'D'|'h'|'H' character to describe how long + // the Backup should be retained. if not set, will be retained forever. + // +kubebuilder:validation:Pattern:=`^\d+[d|D|h|H]$` // +optional - Incremental *CommonBackupPolicy `json:"incremental,omitempty"` + TTL *string `json:"ttl,omitempty"` } type Schedule struct { - // schedule policy for base backup. + // schedule policy for snapshot backup. // +optional - BaseBackup *BaseBackupSchedulePolicy `json:"baseBackup,omitempty"` + Snapshot *SchedulePolicy `json:"snapshot,omitempty"` - // schedule policy for incremental backup. + // schedule policy for datafile backup. // +optional - Incremental *SchedulePolicy `json:"incremental,omitempty"` -} + Datafile *SchedulePolicy `json:"datafile,omitempty"` -type BaseBackupSchedulePolicy struct { - SchedulePolicy `json:",inline"` - // the type of base backup, only support full and snapshot. - // +kubebuilder:validation:Required - Type BaseBackupType `json:"type"` + // schedule policy for logfile backup. + // +optional + Logfile *SchedulePolicy `json:"logfile,omitempty"` } type SchedulePolicy struct { @@ -301,10 +304,10 @@ func init() { func (r *BackupPolicySpec) GetCommonPolicy(backupType BackupType) *CommonBackupPolicy { switch backupType { - case BackupTypeFull: - return r.Full - case BackupTypeIncremental: - return r.Incremental + case BackupTypeDataFile: + return r.Datafile + case BackupTypeLogFile: + return r.Logfile } return nil } diff --git a/apis/dataprotection/v1alpha1/types.go b/apis/dataprotection/v1alpha1/types.go index 78b50a700..f516ced24 100644 --- a/apis/dataprotection/v1alpha1/types.go +++ b/apis/dataprotection/v1alpha1/types.go @@ -31,15 +31,15 @@ const ( BackupFailed BackupPhase = "Failed" ) -// BackupType the backup type, marked backup set is full or incremental or snapshot. +// BackupType the backup type, marked backup set is datafile or logfile or snapshot. // +enum -// +kubebuilder:validation:Enum={full,incremental,snapshot} +// +kubebuilder:validation:Enum={datafile,logfile,snapshot} type BackupType string const ( - BackupTypeFull BackupType = "full" - BackupTypeIncremental BackupType = "incremental" - BackupTypeSnapshot BackupType = "snapshot" + BackupTypeDataFile BackupType = "datafile" + BackupTypeLogFile BackupType = "logfile" + BackupTypeSnapshot BackupType = "snapshot" ) // BaseBackupType the base backup type. @@ -47,11 +47,6 @@ const ( // +kubebuilder:validation:Enum={full,snapshot} type BaseBackupType string -const ( - BaseBackupTypeFull BaseBackupType = "full" - BaseBackupTypeSnapshot BaseBackupType = "snapshot" -) - // CreatePVCPolicy the policy how to create the PersistentVolumeClaim for backup. // +enum // +kubebuilder:validation:Enum={IfNotPresent,Never} diff --git a/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml b/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml index abd20cd43..51ba57b1c 100644 --- a/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml +++ b/config/crd/bases/apps.kubeblocks.io_backuppolicytemplates.yaml @@ -59,8 +59,8 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string - full: - description: the policy for full backup. + datafile: + description: the policy for datafile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -147,8 +147,8 @@ spec: type: string type: object type: object - incremental: - description: the policy for incremental backup. + logfile: + description: the policy for logfile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -235,11 +235,22 @@ spec: type: string type: object type: object + retention: + description: retention describe how long the Backup should be + retained. if not set, will be retained forever. + properties: + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + type: object schedule: description: schedule policy for backup. properties: - baseBackup: - description: schedule policy for base backup. + datafile: + description: schedule policy for datafile backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -248,20 +259,26 @@ spec: enable: description: enable or disable the schedule. type: boolean - type: - description: the type of base backup, only support full - and snapshot. - enum: - - full - - snapshot + required: + - cronExpression + - enable + type: object + logfile: + description: schedule policy for logfile backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - cronExpression - enable - - type type: object - incremental: - description: schedule policy for incremental backup. + snapshot: + description: schedule policy for snapshot backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -378,12 +395,6 @@ spec: type: string type: object type: object - ttl: - description: ttl is a time string ending with the 'd'|'D'|'h'|'H' - character to describe how long the Backup should be retained. - if not set, will be retained forever. - pattern: ^\d+[d|D|h|H]$ - type: string required: - componentDefRef type: object diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml index 46a131acb..b15056397 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -50,8 +50,8 @@ spec: spec: description: BackupPolicySpec defines the desired state of BackupPolicy properties: - full: - description: the policy for full backup. + datafile: + description: the policy for datafile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -232,8 +232,8 @@ spec: - persistentVolumeClaim - target type: object - incremental: - description: the policy for incremental backup. + logfile: + description: the policy for logfile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -414,11 +414,22 @@ spec: - persistentVolumeClaim - target type: object + retention: + description: retention describe how long the Backup should be retained. + if not set, will be retained forever. + properties: + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + type: object schedule: description: schedule policy for backup. properties: - baseBackup: - description: schedule policy for base backup. + datafile: + description: schedule policy for datafile backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -427,20 +438,26 @@ spec: enable: description: enable or disable the schedule. type: boolean - type: - description: the type of base backup, only support full and - snapshot. - enum: - - full - - snapshot + required: + - cronExpression + - enable + type: object + logfile: + description: schedule policy for logfile backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - cronExpression - enable - - type type: object - incremental: - description: schedule policy for incremental backup. + snapshot: + description: schedule policy for snapshot backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -596,12 +613,6 @@ spec: required: - target type: object - ttl: - description: ttl is a time string ending with the 'd'|'D'|'h'|'H' - character to describe how long the Backup should be retained. if - not set, will be retained forever. - pattern: ^\d+[d|D|h|H]$ - type: string type: object status: description: BackupPolicyStatus defines the observed state of BackupPolicy diff --git a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml index 19138c606..c1c96731a 100644 --- a/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml +++ b/config/crd/bases/dataprotection.kubeblocks.io_backups.yaml @@ -61,12 +61,12 @@ spec: pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string backupType: - default: full - description: Backup Type. full or incremental or snapshot. if unset, - default is full. + default: datafile + description: Backup Type. datafile or logfile or snapshot. if unset, + default is datafile. enum: - - full - - incremental + - datafile + - logfile - snapshot type: string parentBackupName: diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index da18e9ed9..8c6be115d 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -1460,7 +1460,7 @@ var _ = Describe("Cluster Controller", func() { By("creating backup") backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() By("waiting for backup failed, because no backup policy exists") diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 00749ae7c..1041edc40 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -223,9 +223,9 @@ func (r *BackupReconciler) doNewPhaseAction( // update Phase to InProgress backup.Status.Phase = dataprotectionv1alpha1.BackupInProgress backup.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now().UTC()} - if backupPolicy.Spec.TTL != nil { + if backupPolicy.Spec.Retention != nil && backupPolicy.Spec.Retention.TTL != nil { backup.Status.Expiration = &metav1.Time{ - Time: backup.Status.StartTimestamp.Add(dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.TTL)), + Time: backup.Status.StartTimestamp.Add(dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.Retention.TTL)), } } @@ -448,7 +448,7 @@ func (r *BackupReconciler) doInProgressPhaseAction( backup.Status.Phase = dataprotectionv1alpha1.BackupFailed backup.Status.FailureReason = job.Status.Conditions[0].Reason } - if backup.Spec.BackupType == dataprotectionv1alpha1.BackupTypeIncremental { + if backup.Spec.BackupType == dataprotectionv1alpha1.BackupTypeLogFile { if backup.Status.Manifests != nil && backup.Status.Manifests.BackupLog != nil && backup.Status.Manifests.BackupLog.StartTime == nil { diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index cbc0ff153..561eaf257 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -158,7 +158,7 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) }) @@ -200,7 +200,7 @@ var _ = Describe("Backup Controller test", func() { By("creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) @@ -408,7 +408,7 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) } @@ -459,7 +459,7 @@ var _ = Describe("Backup Controller test", func() { By("set persistentVolumeConfigmap") configMapName := "pv-template-configmap" Expect(testapps.ChangeObj(&testCtx, backupPolicy, func(tmpObj *dataprotectionv1alpha1.BackupPolicy) { - tmpObj.Spec.Full.PersistentVolumeClaim.PersistentVolumeConfigMap = &dataprotectionv1alpha1.PersistentVolumeConfigMap{ + tmpObj.Spec.Datafile.PersistentVolumeClaim.PersistentVolumeConfigMap = &dataprotectionv1alpha1.PersistentVolumeConfigMap{ Name: configMapName, Namespace: testCtx.DefaultNamespace, } @@ -537,7 +537,7 @@ var _ = Describe("Backup Controller test", func() { By("By creating a backup from backupPolicy: " + backupPolicyName) backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) }) diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index fe73e3b07..63ccfd57d 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -131,8 +131,8 @@ func (r *BackupPolicyReconciler) deleteExternalResources(reqCtx intctrlutil.Requ // delete cronjob resource cronjob := &batchv1.CronJob{} - for _, v := range []dataprotectionv1alpha1.BackupType{dataprotectionv1alpha1.BackupTypeFull, - dataprotectionv1alpha1.BackupTypeIncremental, dataprotectionv1alpha1.BackupTypeSnapshot} { + for _, v := range []dataprotectionv1alpha1.BackupType{dataprotectionv1alpha1.BackupTypeDataFile, + dataprotectionv1alpha1.BackupTypeLogFile, dataprotectionv1alpha1.BackupTypeSnapshot} { key := types.NamespacedName{ Namespace: viper.GetString(constant.CfgKeyCtrlrMgrNS), Name: r.getCronJobName(backupPolicy.Name, backupPolicy.Namespace, v), @@ -279,8 +279,8 @@ func (r *BackupPolicyReconciler) buildCronJob( return nil, err } var ttl metav1.Duration - if backupPolicy.Spec.TTL != nil { - ttl = metav1.Duration{Duration: dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.TTL)} + if backupPolicy.Spec.Retention != nil && backupPolicy.Spec.Retention.TTL != nil { + ttl = metav1.Duration{Duration: dataprotectionv1alpha1.ToDuration(backupPolicy.Spec.Retention.TTL)} } cueValue := intctrlutil.NewCUEBuilder(*cueTpl) options := backupPolicyOptions{ @@ -303,8 +303,8 @@ func (r *BackupPolicyReconciler) buildCronJob( return nil, err } cuePath := "cronjob" - if backType == dataprotectionv1alpha1.BackupTypeIncremental { - cuePath = "cronjob_incremental" + if backType == dataprotectionv1alpha1.BackupTypeLogFile { + cuePath = "cronjob_logfile" } cronjobByte, err := cueValue.Lookup(cuePath) if err != nil { @@ -399,47 +399,47 @@ func (r *BackupPolicyReconciler) handleSnapshotPolicy( return nil } var cronExpression string - schedule := backupPolicy.Spec.Schedule.BaseBackup - if schedule != nil && schedule.Enable && schedule.Type == dataprotectionv1alpha1.BaseBackupTypeSnapshot { + schedule := backupPolicy.Spec.Schedule.Snapshot + if schedule != nil && schedule.Enable { cronExpression = schedule.CronExpression } return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Snapshot.BasePolicy, cronExpression, dataprotectionv1alpha1.BackupTypeSnapshot) } -// handleFullPolicy handles full policy. +// handleFullPolicy handles datafile policy. func (r *BackupPolicyReconciler) handleFullPolicy( reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - if backupPolicy.Spec.Full == nil { + if backupPolicy.Spec.Datafile == nil { // TODO delete cronjob if exists return nil } var cronExpression string - schedule := backupPolicy.Spec.Schedule.BaseBackup - if schedule != nil && schedule.Enable && schedule.Type == dataprotectionv1alpha1.BaseBackupTypeFull { + schedule := backupPolicy.Spec.Schedule.Datafile + if schedule != nil && schedule.Enable { cronExpression = schedule.CronExpression } - r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Full) - return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Full.BasePolicy, - cronExpression, dataprotectionv1alpha1.BackupTypeFull) + r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Datafile) + return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Datafile.BasePolicy, + cronExpression, dataprotectionv1alpha1.BackupTypeDataFile) } // handleIncrementalPolicy handles incremental policy. func (r *BackupPolicyReconciler) handleIncrementalPolicy( reqCtx intctrlutil.RequestCtx, backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { - if backupPolicy.Spec.Incremental == nil { + if backupPolicy.Spec.Logfile == nil { return nil } var cronExpression string - schedule := backupPolicy.Spec.Schedule.Incremental + schedule := backupPolicy.Spec.Schedule.Logfile if schedule != nil && schedule.Enable { cronExpression = schedule.CronExpression } - r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Incremental) - return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Incremental.BasePolicy, - cronExpression, dataprotectionv1alpha1.BackupTypeIncremental) + r.setGlobalPersistentVolumeClaim(backupPolicy.Spec.Logfile) + return r.handlePolicy(reqCtx, backupPolicy, backupPolicy.Spec.Logfile.BasePolicy, + cronExpression, dataprotectionv1alpha1.BackupTypeLogFile) } // setGlobalPersistentVolumeClaim sets global config of pvc to common policy. diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index 9ee9ef743..344528408 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -150,7 +150,7 @@ var _ = Describe("Backup Policy Controller", func() { Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) })).Should(Succeed()) - Eventually(testapps.CheckObj(&testCtx, getCronjobKey(dpv1alpha1.BackupTypeFull), func(g Gomega, fetched *batchv1.CronJob) { + Eventually(testapps.CheckObj(&testCtx, getCronjobKey(dpv1alpha1.BackupTypeDataFile), func(g Gomega, fetched *batchv1.CronJob) { g.Expect(fetched.Spec.Schedule).To(Equal(defaultSchedule)) })).Should(Succeed()) }) @@ -166,26 +166,26 @@ var _ = Describe("Backup Policy Controller", func() { autoBackupLabel := map[string]string{ dataProtectionLabelAutoBackupKey: "true", dataProtectionLabelBackupPolicyKey: backupPolicyName, - dataProtectionLabelBackupTypeKey: string(dpv1alpha1.BaseBackupTypeFull), + dataProtectionLabelBackupTypeKey: string(dpv1alpha1.BackupTypeDataFile), } By("create a expired backup") backupExpired := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). SetBackupPolicyName(backupPolicyName). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() By("create 1st limit backup") backupOutLimit1 := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). SetBackupPolicyName(backupPolicyName). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() By("create 2nd limit backup") backupOutLimit2 := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupNamePrefix). WithRandomName().AddLabelsInMap(autoBackupLabel). SetBackupPolicyName(backupPolicyName). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() By("waiting expired backup completed") @@ -225,11 +225,11 @@ var _ = Describe("Backup Policy Controller", func() { patchBackupStatus(backupStatus, client.ObjectKeyFromObject(backupOutLimit2)) // trigger the backup policy controller through update cronjob - patchCronJobStatus(getCronjobKey(dpv1alpha1.BackupTypeFull)) + patchCronJobStatus(getCronjobKey(dpv1alpha1.BackupTypeDataFile)) By("retain the latest backup") Eventually(testapps.List(&testCtx, intctrlutil.BackupSignature, - client.MatchingLabels(backupPolicy.Spec.Full.Target.LabelsSelector.MatchLabels), + client.MatchingLabels(backupPolicy.Spec.Datafile.Target.LabelsSelector.MatchLabels), client.InNamespace(backupPolicy.Namespace))).Should(HaveLen(1)) }) }) @@ -296,7 +296,7 @@ var _ = Describe("Backup Policy Controller", func() { backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) - g.Expect(fetched.Spec.Full.Target.Secret.Name).To(Equal(randomSecretName)) + g.Expect(fetched.Spec.Datafile.Target.Secret.Name).To(Equal(randomSecretName)) })).Should(Succeed()) }) }) @@ -318,8 +318,8 @@ var _ = Describe("Backup Policy Controller", func() { backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) - g.Expect(fetched.Spec.Full.PersistentVolumeClaim.Name).To(Equal(pvcName)) - g.Expect(fetched.Spec.Full.PersistentVolumeClaim.InitCapacity.String()).To(Equal(pvcInitCapacity)) + g.Expect(fetched.Spec.Datafile.PersistentVolumeClaim.Name).To(Equal(pvcName)) + g.Expect(fetched.Spec.Datafile.PersistentVolumeClaim.InitCapacity.String()).To(Equal(pvcInitCapacity)) })).Should(Succeed()) }) }) diff --git a/controllers/dataprotection/cue/cronjob.cue b/controllers/dataprotection/cue/cronjob.cue index e20d136ea..ca98465b2 100644 --- a/controllers/dataprotection/cue/cronjob.cue +++ b/controllers/dataprotection/cue/cronjob.cue @@ -78,7 +78,7 @@ EOF } } -cronjob_incremental: { +cronjob_logfile: { apiVersion: "batch/v1" kind: "CronJob" metadata: { diff --git a/controllers/dataprotection/restorejob_controller_test.go b/controllers/dataprotection/restorejob_controller_test.go index 11ef660b6..e777d7723 100644 --- a/controllers/dataprotection/restorejob_controller_test.go +++ b/controllers/dataprotection/restorejob_controller_test.go @@ -82,7 +82,7 @@ var _ = Describe("RestoreJob Controller", func() { By("By assure an backup obj") return testapps.NewBackupFactory(testCtx.DefaultNamespace, "backup-job-"). WithRandomName().SetBackupPolicyName(backupPolicy). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() } diff --git a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml index c3a6bfab8..437548b7f 100644 --- a/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuppolicytemplate.yaml @@ -11,12 +11,15 @@ spec: clusterDefinitionRef: apecloud-mysql-scale backupPolicies: - componentDefRef: mysql - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false - cronExpression: "0 18 * * 0" + cronExpression: "0 18 * * *" + datafile: + enable: false + cronExpression: "0 18 * * *" snapshot: hooks: containerName: mysql @@ -26,5 +29,5 @@ spec: - "rm -f /data/mysql/data/.restore_new_cluster; sync" target: role: leader - full: + datafile: backupToolName: xtrabackup-for-apecloud-mysql-scale \ No newline at end of file diff --git a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml index 9915dab78..8a21fb0d8 100644 --- a/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml +++ b/deploy/apecloud-mysql/templates/backuppolicytemplate.yaml @@ -11,12 +11,15 @@ spec: clusterDefinitionRef: apecloud-mysql backupPolicies: - componentDefRef: mysql - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false - cronExpression: "0 18 * * 0" + cronExpression: "0 18 * * *" + datafile: + enable: false + cronExpression: "0 18 * * *" snapshot: hooks: containerName: mysql @@ -26,5 +29,5 @@ spec: - "rm -f /data/mysql/data/.restore_new_cluster; sync" target: role: leader - full: + datafile: backupToolName: xtrabackup-for-apecloud-mysql \ No newline at end of file diff --git a/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml b/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml index abd20cd43..51ba57b1c 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_backuppolicytemplates.yaml @@ -59,8 +59,8 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string - full: - description: the policy for full backup. + datafile: + description: the policy for datafile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -147,8 +147,8 @@ spec: type: string type: object type: object - incremental: - description: the policy for incremental backup. + logfile: + description: the policy for logfile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -235,11 +235,22 @@ spec: type: string type: object type: object + retention: + description: retention describe how long the Backup should be + retained. if not set, will be retained forever. + properties: + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + type: object schedule: description: schedule policy for backup. properties: - baseBackup: - description: schedule policy for base backup. + datafile: + description: schedule policy for datafile backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -248,20 +259,26 @@ spec: enable: description: enable or disable the schedule. type: boolean - type: - description: the type of base backup, only support full - and snapshot. - enum: - - full - - snapshot + required: + - cronExpression + - enable + type: object + logfile: + description: schedule policy for logfile backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - cronExpression - enable - - type type: object - incremental: - description: schedule policy for incremental backup. + snapshot: + description: schedule policy for snapshot backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -378,12 +395,6 @@ spec: type: string type: object type: object - ttl: - description: ttl is a time string ending with the 'd'|'D'|'h'|'H' - character to describe how long the Backup should be retained. - if not set, will be retained forever. - pattern: ^\d+[d|D|h|H]$ - type: string required: - componentDefRef type: object diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml index 46a131acb..b15056397 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backuppolicies.yaml @@ -50,8 +50,8 @@ spec: spec: description: BackupPolicySpec defines the desired state of BackupPolicy properties: - full: - description: the policy for full backup. + datafile: + description: the policy for datafile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -232,8 +232,8 @@ spec: - persistentVolumeClaim - target type: object - incremental: - description: the policy for incremental backup. + logfile: + description: the policy for logfile backup. properties: backupStatusUpdates: description: define how to update metadata for backup status. @@ -414,11 +414,22 @@ spec: - persistentVolumeClaim - target type: object + retention: + description: retention describe how long the Backup should be retained. + if not set, will be retained forever. + properties: + ttl: + description: ttl is a time string ending with the 'd'|'D'|'h'|'H' + character to describe how long the Backup should be retained. + if not set, will be retained forever. + pattern: ^\d+[d|D|h|H]$ + type: string + type: object schedule: description: schedule policy for backup. properties: - baseBackup: - description: schedule policy for base backup. + datafile: + description: schedule policy for datafile backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -427,20 +438,26 @@ spec: enable: description: enable or disable the schedule. type: boolean - type: - description: the type of base backup, only support full and - snapshot. - enum: - - full - - snapshot + required: + - cronExpression + - enable + type: object + logfile: + description: schedule policy for logfile backup. + properties: + cronExpression: + description: the cron expression for schedule, the timezone + is in UTC. see https://en.wikipedia.org/wiki/Cron. type: string + enable: + description: enable or disable the schedule. + type: boolean required: - cronExpression - enable - - type type: object - incremental: - description: schedule policy for incremental backup. + snapshot: + description: schedule policy for snapshot backup. properties: cronExpression: description: the cron expression for schedule, the timezone @@ -596,12 +613,6 @@ spec: required: - target type: object - ttl: - description: ttl is a time string ending with the 'd'|'D'|'h'|'H' - character to describe how long the Backup should be retained. if - not set, will be retained forever. - pattern: ^\d+[d|D|h|H]$ - type: string type: object status: description: BackupPolicyStatus defines the observed state of BackupPolicy diff --git a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml index 19138c606..c1c96731a 100644 --- a/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml +++ b/deploy/helm/crds/dataprotection.kubeblocks.io_backups.yaml @@ -61,12 +61,12 @@ spec: pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string backupType: - default: full - description: Backup Type. full or incremental or snapshot. if unset, - default is full. + default: datafile + description: Backup Type. datafile or logfile or snapshot. if unset, + default is datafile. enum: - - full - - incremental + - datafile + - logfile - snapshot type: string parentBackupName: diff --git a/deploy/milvus/templates/backuppolicytemplate.yaml b/deploy/milvus/templates/backuppolicytemplate.yaml index 154924259..c1be0359c 100644 --- a/deploy/milvus/templates/backuppolicytemplate.yaml +++ b/deploy/milvus/templates/backuppolicytemplate.yaml @@ -9,10 +9,10 @@ spec: clusterDefinitionRef: milvus backupPolicies: - componentDefRef: milvus - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * 0" snapshot: diff --git a/deploy/mongodb/templates/backuppolicytemplate.yaml b/deploy/mongodb/templates/backuppolicytemplate.yaml index 177dd0fbd..1002039f5 100644 --- a/deploy/mongodb/templates/backuppolicytemplate.yaml +++ b/deploy/mongodb/templates/backuppolicytemplate.yaml @@ -9,19 +9,22 @@ spec: clusterDefinitionRef: mongodb backupPolicies: - componentDefRef: mongodb - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: full + snapshot: enable: false - cronExpression: "0 18 * * 0" + cronExpression: "0 18 * * *" + datafile: + enable: false + cronExpression: "0 18 * * *" snapshot: target: role: primary connectionCredentialKey: passwordKey: password usernameKey: username - full: + datafile: backupToolName: mongodb-physical-backup-tool backupsHistoryLimit: 7 target: diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index 77dec27e2..6b896b6da 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -9,13 +9,16 @@ spec: clusterDefinitionRef: postgresql backupPolicies: - componentDefRef: postgresql - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * *" - incremental: + datafile: + enable: false + cronExpression: "0 18 * * *" + logfile: enable: false cronExpression: "*/5 * * * *" snapshot: @@ -32,9 +35,9 @@ spec: containerName: postgresql script: /kb-scripts/backup-log-collector.sh true updateStage: post - full: + datafile: backupToolName: postgres-basebackup - incremental: + logfile: backupToolName: postgres-pitr target: role: primary diff --git a/deploy/qdrant/templates/backuppolicytemplate.yaml b/deploy/qdrant/templates/backuppolicytemplate.yaml index a8f4dc89d..553f8ee16 100644 --- a/deploy/qdrant/templates/backuppolicytemplate.yaml +++ b/deploy/qdrant/templates/backuppolicytemplate.yaml @@ -9,10 +9,10 @@ spec: clusterDefinitionRef: qdrant backupPolicies: - componentDefRef: qdrant - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * 0" snapshot: diff --git a/deploy/redis/templates/backuppolicytemplate.yaml b/deploy/redis/templates/backuppolicytemplate.yaml index f707cabf2..dff9af636 100644 --- a/deploy/redis/templates/backuppolicytemplate.yaml +++ b/deploy/redis/templates/backuppolicytemplate.yaml @@ -9,10 +9,10 @@ spec: clusterDefinitionRef: redis backupPolicies: - componentDefRef: redis - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * 0" snapshot: diff --git a/deploy/weaviate/templates/backuppolicytemplate.yaml b/deploy/weaviate/templates/backuppolicytemplate.yaml index 37b4c2faa..579493cea 100644 --- a/deploy/weaviate/templates/backuppolicytemplate.yaml +++ b/deploy/weaviate/templates/backuppolicytemplate.yaml @@ -9,10 +9,10 @@ spec: clusterDefinitionRef: weaviate backupPolicies: - componentDefRef: weaviate - ttl: 7d + retention: + ttl: 7d schedule: - baseBackup: - type: snapshot + snapshot: enable: false cronExpression: "0 18 * * 0" snapshot: diff --git a/docs/user_docs/cli/kbcli_cluster_backup.md b/docs/user_docs/cli/kbcli_cluster_backup.md index 6b0f54864..c9c5f06dc 100644 --- a/docs/user_docs/cli/kbcli_cluster_backup.md +++ b/docs/user_docs/cli/kbcli_cluster_backup.md @@ -15,10 +15,10 @@ kbcli cluster backup NAME [flags] kbcli cluster backup mycluster # create a snapshot backup - kbcli cluster backup mycluster --backup-type snapshot + kbcli cluster backup mycluster --type snapshot - # create a full backup - kbcli cluster backup mycluster --backup-type full + # create a datafile backup + kbcli cluster backup mycluster --type datafile # create a backup with specified backup policy kbcli cluster backup mycluster --backup-policy @@ -29,8 +29,8 @@ kbcli cluster backup NAME [flags] ``` --backup-name string Backup name --backup-policy string Backup policy name, this flag will be ignored when backup-type is snapshot - --backup-type string Backup type (default "snapshot") -h, --help help for backup + --type string Backup type (default "snapshot") ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 40127c0d3..671a30347 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -72,10 +72,10 @@ var ( kbcli cluster backup mycluster # create a snapshot backup - kbcli cluster backup mycluster --backup-type snapshot + kbcli cluster backup mycluster --type snapshot - # create a full backup - kbcli cluster backup mycluster --backup-type full + # create a datafile backup + kbcli cluster backup mycluster --type datafile # create a backup with specified backup policy kbcli cluster backup mycluster --backup-policy @@ -213,7 +213,7 @@ func NewCreateBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) }, } - cmd.Flags().StringVar(&o.BackupType, "backup-type", "snapshot", "Backup type") + cmd.Flags().StringVar(&o.BackupType, "type", "snapshot", "Backup type") cmd.Flags().StringVar(&o.BackupName, "backup-name", "", "Backup name") cmd.Flags().StringVar(&o.BackupPolicy, "backup-policy", "", "Backup policy name, this flag will be ignored when backup-type is snapshot") @@ -595,14 +595,19 @@ func printBackupPolicyList(o list.ListOptions) error { } tbl := printer.NewTablePrinter(o.Out) - tbl.SetHeader("NAME", "DEFAULT", "CLUSTER", "CREATE-TIME") + tbl.SetHeader("NAME", "DEFAULT", "CLUSTER", "CREATE-TIME", "STATUS") for _, obj := range backupPolicyList.Items { defaultPolicy, ok := obj.GetAnnotations()[constant.DefaultBackupPolicyAnnotationKey] + backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, backupPolicy); err != nil { + return err + } if !ok { defaultPolicy = "false" } createTime := obj.GetCreationTimestamp() - tbl.AddRow(obj.GetName(), defaultPolicy, obj.GetLabels()[constant.AppInstanceLabelKey], util.TimeFormat(&createTime)) + tbl.AddRow(obj.GetName(), defaultPolicy, obj.GetLabels()[constant.AppInstanceLabelKey], + util.TimeFormat(&createTime), backupPolicy.Status.Phase) } tbl.Print() return nil diff --git a/internal/cli/cmd/cluster/dataprotection_test.go b/internal/cli/cmd/cluster/dataprotection_test.go index 76359bbad..1ef634202 100644 --- a/internal/cli/cmd/cluster/dataprotection_test.go +++ b/internal/cli/cmd/cluster/dataprotection_test.go @@ -258,7 +258,7 @@ var _ = Describe("DataProtection", func() { baseBackup.TypeMeta = backupTypeMeta baseBackup.Status.Phase = dataprotectionv1alpha1.BackupCompleted incrBackup := testapps.NewBackupFactory(testing.Namespace, backupName). - SetBackupType(dataprotectionv1alpha1.BackupTypeIncremental). + SetBackupType(dataprotectionv1alpha1.BackupTypeLogFile). SetBackLog(now.Add(-time.Minute), now.Add(time.Minute)). SetLabels(backupLabels).GetObject() incrBackup.TypeMeta = backupTypeMeta diff --git a/internal/cli/cmd/cluster/describe.go b/internal/cli/cmd/cluster/describe.go index 74de7d65a..b62ace4e8 100644 --- a/internal/cli/cmd/cluster/describe.go +++ b/internal/cli/cmd/cluster/describe.go @@ -239,16 +239,22 @@ func showDataProtection(backupPolicies []dpv1alpha1.BackupPolicy, backups []dpv1 backupSchedule := printer.NoneString backupType := printer.NoneString scheduleEnable := "Disabled" - if policy.Spec.Schedule.BaseBackup != nil { - if policy.Spec.Schedule.BaseBackup.Enable { + if policy.Spec.Schedule.Snapshot != nil { + if policy.Spec.Schedule.Snapshot.Enable { scheduleEnable = "Enabled" + backupSchedule = policy.Spec.Schedule.Snapshot.CronExpression + backupType = string(dpv1alpha1.BackupTypeSnapshot) + } + } + if policy.Spec.Schedule.Datafile != nil { + if policy.Spec.Schedule.Datafile.Enable { + scheduleEnable = "Enabled" + backupSchedule = policy.Spec.Schedule.Datafile.CronExpression + backupType = string(dpv1alpha1.BackupTypeDataFile) } - backupSchedule = policy.Spec.Schedule.BaseBackup.CronExpression - backupType = string(policy.Spec.Schedule.BaseBackup.Type) - } - if policy.Spec.TTL != nil { - ttlString = *policy.Spec.TTL + if policy.Spec.Retention != nil && policy.Spec.Retention.TTL != nil { + ttlString = *policy.Spec.Retention.TTL } lastScheduleTime := printer.NoneString if policy.Status.LastScheduleTime != nil { diff --git a/internal/controller/component/restore_utils.go b/internal/controller/component/restore_utils.go index ba34a15c2..717546e55 100644 --- a/internal/controller/component/restore_utils.go +++ b/internal/controller/component/restore_utils.go @@ -76,7 +76,7 @@ func BuildRestoredInfo2( return intctrlutil.NewErrorf(intctrlutil.ErrorTypeBackupNotCompleted, "backup %s is not completed", backup.Name) } switch backup.Spec.BackupType { - case dataprotectionv1alpha1.BackupTypeFull: + case dataprotectionv1alpha1.BackupTypeDataFile: return buildInitContainerWithFullBackup(component, backup, backupTool) case dataprotectionv1alpha1.BackupTypeSnapshot: return buildVolumeClaimTemplatesWithSnapshot(component, backup) diff --git a/internal/controller/component/restore_utils_test.go b/internal/controller/component/restore_utils_test.go index 514f3c4f7..9a15aa900 100644 --- a/internal/controller/component/restore_utils_test.go +++ b/internal/controller/component/restore_utils_test.go @@ -84,7 +84,7 @@ var _ = Describe("probe_utils", func() { } backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). SetBackupPolicyName(backupPolicyName). - SetBackupType(dataprotectionv1alpha1.BackupTypeFull). + SetBackupType(dataprotectionv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() updateBackupStatus(backup, backupToolName, dataprotectionv1alpha1.BackupCompleted) component := &SynthesizedComponent{ diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index aff787803..449eb6c74 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -158,13 +158,13 @@ func (r *BackupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti backupPolicy.Spec.Snapshot.Target = syncTheRoleLabel(backupPolicy.Spec.Snapshot.Target, policyTPL.Snapshot.BasePolicy) } - if backupPolicy.Spec.Full != nil && policyTPL.Full != nil { - backupPolicy.Spec.Full.Target = syncTheRoleLabel(backupPolicy.Spec.Full.Target, - policyTPL.Full.BasePolicy) + if backupPolicy.Spec.Datafile != nil && policyTPL.Datafile != nil { + backupPolicy.Spec.Datafile.Target = syncTheRoleLabel(backupPolicy.Spec.Datafile.Target, + policyTPL.Datafile.BasePolicy) } - if backupPolicy.Spec.Incremental != nil && policyTPL.Incremental != nil { - backupPolicy.Spec.Incremental.Target = syncTheRoleLabel(backupPolicy.Spec.Incremental.Target, - policyTPL.Incremental.BasePolicy) + if backupPolicy.Spec.Logfile != nil && policyTPL.Logfile != nil { + backupPolicy.Spec.Logfile.Target = syncTheRoleLabel(backupPolicy.Spec.Logfile.Target, + policyTPL.Logfile.BasePolicy) } } @@ -196,11 +196,17 @@ func (r *BackupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba }, } bpSpec := backupPolicy.Spec - bpSpec.TTL = policyTPL.TTL - bpSpec.Schedule.BaseBackup = r.convertBaseBackupSchedulePolicy(policyTPL.Schedule.BaseBackup) - bpSpec.Schedule.Incremental = r.convertSchedulePolicy(policyTPL.Schedule.Incremental) - bpSpec.Full = r.convertCommonPolicy(policyTPL.Full, cluster.Name, *component, workloadType) - bpSpec.Incremental = r.convertCommonPolicy(policyTPL.Incremental, cluster.Name, *component, workloadType) + if policyTPL.Retention != nil { + bpSpec.Retention = &dataprotectionv1alpha1.RetentionSpec{ + TTL: policyTPL.Retention.TTL, + } + } + + bpSpec.Schedule.Snapshot = r.convertSchedulePolicy(policyTPL.Schedule.Snapshot) + bpSpec.Schedule.Datafile = r.convertSchedulePolicy(policyTPL.Schedule.Datafile) + bpSpec.Schedule.Logfile = r.convertSchedulePolicy(policyTPL.Schedule.Logfile) + bpSpec.Datafile = r.convertCommonPolicy(policyTPL.Datafile, cluster.Name, *component, workloadType) + bpSpec.Logfile = r.convertCommonPolicy(policyTPL.Logfile, cluster.Name, *component, workloadType) bpSpec.Snapshot = r.convertSnapshotPolicy(policyTPL.Snapshot, cluster.Name, *component, workloadType) backupPolicy.Spec = bpSpec return backupPolicy @@ -228,18 +234,6 @@ func (r *BackupPolicyTPLTransformer) convertSchedulePolicy(sp *appsv1alpha1.Sche } } -// convertBaseBackupSchedulePolicy converts the baseBackupSchedulePolicy from backupPolicyTemplate. -func (r *BackupPolicyTPLTransformer) convertBaseBackupSchedulePolicy(sp *appsv1alpha1.BaseBackupSchedulePolicy) *dataprotectionv1alpha1.BaseBackupSchedulePolicy { - if sp == nil { - return nil - } - schedulePolicy := r.convertSchedulePolicy(&sp.SchedulePolicy) - return &dataprotectionv1alpha1.BaseBackupSchedulePolicy{ - Type: dataprotectionv1alpha1.BaseBackupType(sp.Type), - SchedulePolicy: *schedulePolicy, - } -} - // convertBasePolicy converts the basePolicy from backupPolicyTemplate. func (r *BackupPolicyTPLTransformer) convertBasePolicy(bp appsv1alpha1.BasePolicy, clusterName string, diff --git a/internal/controller/plan/pitr.go b/internal/controller/plan/pitr.go index d328f229e..51408351e 100644 --- a/internal/controller/plan/pitr.go +++ b/internal/controller/plan/pitr.go @@ -262,7 +262,7 @@ func (p *PointInTimeRecoveryManager) getLatestBaseBackup() (*dpv1alpha1.Backup, // 2. get the latest backup object var latestBackup *dpv1alpha1.Backup for _, item := range backups { - if item.Spec.BackupType != dpv1alpha1.BackupTypeIncremental && + if item.Spec.BackupType != dpv1alpha1.BackupTypeLogFile && item.Status.Manifests.BackupLog.StopTime != nil && !p.restoreTime.Before(item.Status.Manifests.BackupLog.StopTime) { latestBackup = &item break @@ -367,7 +367,7 @@ func (p *PointInTimeRecoveryManager) getIncrementalBackup(componentName string) client.MatchingLabels{ constant.AppInstanceLabelKey: p.sourceCluster, constant.KBAppComponentLabelKey: componentName, - constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeIncremental), + constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeLogFile), }); err != nil { return nil, err } diff --git a/internal/controller/plan/pitr_test.go b/internal/controller/plan/pitr_test.go index f93722fab..f294fac07 100644 --- a/internal/controller/plan/pitr_test.go +++ b/internal/controller/plan/pitr_test.go @@ -217,14 +217,14 @@ var _ = Describe("PITR Functions", func() { incrBackupLabels := map[string]string{ constant.AppInstanceLabelKey: sourceCluster, constant.KBAppComponentLabelKey: mysqlCompName, - constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeIncremental), + constant.BackupTypeLabelKeyKey: string(dpv1alpha1.BackupTypeLogFile), } incrStartTime := &startTime incrStopTime := &stopTime backupIncr := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). WithRandomName().SetLabels(incrBackupLabels). SetBackupPolicyName("test-fake"). - SetBackupType(dpv1alpha1.BackupTypeIncremental). + SetBackupType(dpv1alpha1.BackupTypeLogFile). Create(&testCtx).GetObject() backupStatus = dpv1alpha1.BackupStatus{ Phase: dpv1alpha1.BackupCompleted, diff --git a/internal/testutil/apps/backuppolicy_factory.go b/internal/testutil/apps/backuppolicy_factory.go index 1ffc081f0..175a302bf 100644 --- a/internal/testutil/apps/backuppolicy_factory.go +++ b/internal/testutil/apps/backuppolicy_factory.go @@ -42,10 +42,10 @@ func NewBackupPolicyFactory(namespace, name string) *MockBackupPolicyFactory { func (factory *MockBackupPolicyFactory) setBasePolicyField(setField func(basePolicy *dataprotectionv1alpha1.BasePolicy)) { var basePolicy *dataprotectionv1alpha1.BasePolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull: - basePolicy = &factory.get().Spec.Full.BasePolicy - case dataprotectionv1alpha1.BackupTypeIncremental: - basePolicy = &factory.get().Spec.Incremental.BasePolicy + case dataprotectionv1alpha1.BackupTypeDataFile: + basePolicy = &factory.get().Spec.Datafile.BasePolicy + case dataprotectionv1alpha1.BackupTypeLogFile: + basePolicy = &factory.get().Spec.Logfile.BasePolicy case dataprotectionv1alpha1.BackupTypeSnapshot: basePolicy = &factory.get().Spec.Snapshot.BasePolicy } @@ -59,10 +59,10 @@ func (factory *MockBackupPolicyFactory) setBasePolicyField(setField func(basePol func (factory *MockBackupPolicyFactory) setCommonPolicyField(setField func(commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy)) { var commonPolicy *dataprotectionv1alpha1.CommonBackupPolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull: - commonPolicy = factory.get().Spec.Full - case dataprotectionv1alpha1.BackupTypeIncremental: - commonPolicy = factory.get().Spec.Incremental + case dataprotectionv1alpha1.BackupTypeDataFile: + commonPolicy = factory.get().Spec.Datafile + case dataprotectionv1alpha1.BackupTypeLogFile: + commonPolicy = factory.get().Spec.Logfile } if commonPolicy == nil { // ignore @@ -74,15 +74,16 @@ func (factory *MockBackupPolicyFactory) setCommonPolicyField(setField func(commo func (factory *MockBackupPolicyFactory) setScheduleField(setField func(schedulePolicy *dataprotectionv1alpha1.SchedulePolicy)) { var schedulePolicy *dataprotectionv1alpha1.SchedulePolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull, dataprotectionv1alpha1.BackupTypeSnapshot: - factory.get().Spec.Schedule.BaseBackup = &dataprotectionv1alpha1.BaseBackupSchedulePolicy{ - SchedulePolicy: dataprotectionv1alpha1.SchedulePolicy{}, - Type: dataprotectionv1alpha1.BaseBackupType(factory.backupType), - } - schedulePolicy = &factory.get().Spec.Schedule.BaseBackup.SchedulePolicy - case dataprotectionv1alpha1.BackupTypeIncremental: - schedulePolicy = &dataprotectionv1alpha1.SchedulePolicy{} - factory.get().Spec.Schedule.Incremental = schedulePolicy + case dataprotectionv1alpha1.BackupTypeDataFile: + factory.get().Spec.Schedule.Datafile = &dataprotectionv1alpha1.SchedulePolicy{} + schedulePolicy = factory.get().Spec.Schedule.Datafile + case dataprotectionv1alpha1.BackupTypeSnapshot: + factory.get().Spec.Schedule.Snapshot = &dataprotectionv1alpha1.SchedulePolicy{} + schedulePolicy = factory.get().Spec.Schedule.Snapshot + // todo: set logfile schedule + case dataprotectionv1alpha1.BackupTypeLogFile: + factory.get().Spec.Schedule.Logfile = &dataprotectionv1alpha1.SchedulePolicy{} + schedulePolicy = factory.get().Spec.Schedule.Snapshot } if schedulePolicy == nil { // ignore @@ -100,22 +101,22 @@ func (factory *MockBackupPolicyFactory) AddSnapshotPolicy() *MockBackupPolicyFac } func (factory *MockBackupPolicyFactory) AddFullPolicy() *MockBackupPolicyFactory { - factory.get().Spec.Full = &dataprotectionv1alpha1.CommonBackupPolicy{ + factory.get().Spec.Datafile = &dataprotectionv1alpha1.CommonBackupPolicy{ PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ CreatePolicy: dataprotectionv1alpha1.CreatePVCPolicyIfNotPresent, }, } - factory.backupType = dataprotectionv1alpha1.BackupTypeFull + factory.backupType = dataprotectionv1alpha1.BackupTypeDataFile return factory } func (factory *MockBackupPolicyFactory) AddIncrementalPolicy() *MockBackupPolicyFactory { - factory.get().Spec.Incremental = &dataprotectionv1alpha1.CommonBackupPolicy{ + factory.get().Spec.Logfile = &dataprotectionv1alpha1.CommonBackupPolicy{ PersistentVolumeClaim: dataprotectionv1alpha1.PersistentVolumeClaim{ CreatePolicy: dataprotectionv1alpha1.CreatePVCPolicyIfNotPresent, }, } - factory.backupType = dataprotectionv1alpha1.BackupTypeIncremental + factory.backupType = dataprotectionv1alpha1.BackupTypeLogFile return factory } @@ -135,7 +136,9 @@ func (factory *MockBackupPolicyFactory) SetSchedule(schedule string, enable bool } func (factory *MockBackupPolicyFactory) SetTTL(duration string) *MockBackupPolicyFactory { - factory.get().Spec.TTL = &duration + factory.get().Spec.Retention = &dataprotectionv1alpha1.RetentionSpec{ + TTL: &duration, + } return factory } diff --git a/internal/testutil/apps/backuppolicytemplate_factory.go b/internal/testutil/apps/backuppolicytemplate_factory.go index d6acdaf1d..efaaa2982 100644 --- a/internal/testutil/apps/backuppolicytemplate_factory.go +++ b/internal/testutil/apps/backuppolicytemplate_factory.go @@ -59,7 +59,9 @@ func (factory *MockBackupPolicyTemplateFactory) AddBackupPolicy(componentDef str } func (factory *MockBackupPolicyTemplateFactory) SetTTL(duration string) *MockBackupPolicyTemplateFactory { - factory.getLastBackupPolicy().TTL = &duration + factory.getLastBackupPolicy().Retention = &appsv1alpha1.RetentionSpec{ + TTL: &duration, + } return factory } @@ -67,10 +69,10 @@ func (factory *MockBackupPolicyTemplateFactory) setBasePolicyField(setField func backupPolicy := factory.getLastBackupPolicy() var basePolicy *appsv1alpha1.BasePolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull: - basePolicy = &backupPolicy.Full.BasePolicy - case dataprotectionv1alpha1.BackupTypeIncremental: - basePolicy = &backupPolicy.Incremental.BasePolicy + case dataprotectionv1alpha1.BackupTypeDataFile: + basePolicy = &backupPolicy.Datafile.BasePolicy + case dataprotectionv1alpha1.BackupTypeLogFile: + basePolicy = &backupPolicy.Logfile.BasePolicy case dataprotectionv1alpha1.BackupTypeSnapshot: basePolicy = &backupPolicy.Snapshot.BasePolicy } @@ -85,10 +87,10 @@ func (factory *MockBackupPolicyTemplateFactory) setCommonPolicyField(setField fu backupPolicy := factory.getLastBackupPolicy() var commonPolicy *appsv1alpha1.CommonBackupPolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull: - commonPolicy = backupPolicy.Full - case dataprotectionv1alpha1.BackupTypeIncremental: - commonPolicy = backupPolicy.Incremental + case dataprotectionv1alpha1.BackupTypeDataFile: + commonPolicy = backupPolicy.Datafile + case dataprotectionv1alpha1.BackupTypeLogFile: + commonPolicy = backupPolicy.Logfile } if commonPolicy == nil { // ignore @@ -101,15 +103,15 @@ func (factory *MockBackupPolicyTemplateFactory) setScheduleField(setField func(s backupPolicy := factory.getLastBackupPolicy() var schedulePolicy *appsv1alpha1.SchedulePolicy switch factory.backupType { - case dataprotectionv1alpha1.BackupTypeFull, dataprotectionv1alpha1.BackupTypeSnapshot: - backupPolicy.Schedule.BaseBackup = &appsv1alpha1.BaseBackupSchedulePolicy{ - SchedulePolicy: appsv1alpha1.SchedulePolicy{}, - Type: appsv1alpha1.BaseBackupType(factory.backupType), - } - schedulePolicy = &backupPolicy.Schedule.BaseBackup.SchedulePolicy - case dataprotectionv1alpha1.BackupTypeIncremental: - schedulePolicy = &appsv1alpha1.SchedulePolicy{} - backupPolicy.Schedule.Incremental = schedulePolicy + case dataprotectionv1alpha1.BackupTypeSnapshot: + backupPolicy.Schedule.Snapshot = &appsv1alpha1.SchedulePolicy{} + schedulePolicy = backupPolicy.Schedule.Snapshot + case dataprotectionv1alpha1.BackupTypeDataFile: + backupPolicy.Schedule.Datafile = &appsv1alpha1.SchedulePolicy{} + schedulePolicy = backupPolicy.Schedule.Datafile + case dataprotectionv1alpha1.BackupTypeLogFile: + backupPolicy.Schedule.Logfile = &appsv1alpha1.SchedulePolicy{} + schedulePolicy = backupPolicy.Schedule.Logfile } if schedulePolicy == nil { // ignore @@ -129,15 +131,15 @@ func (factory *MockBackupPolicyTemplateFactory) AddSnapshotPolicy() *MockBackupP func (factory *MockBackupPolicyTemplateFactory) AddFullPolicy() *MockBackupPolicyTemplateFactory { backupPolicy := factory.getLastBackupPolicy() - backupPolicy.Full = &appsv1alpha1.CommonBackupPolicy{} - factory.backupType = dataprotectionv1alpha1.BackupTypeFull + backupPolicy.Datafile = &appsv1alpha1.CommonBackupPolicy{} + factory.backupType = dataprotectionv1alpha1.BackupTypeDataFile return factory } func (factory *MockBackupPolicyTemplateFactory) AddIncrementalPolicy() *MockBackupPolicyTemplateFactory { backupPolicy := factory.getLastBackupPolicy() - backupPolicy.Incremental = &appsv1alpha1.CommonBackupPolicy{} - factory.backupType = dataprotectionv1alpha1.BackupTypeIncremental + backupPolicy.Logfile = &appsv1alpha1.CommonBackupPolicy{} + factory.backupType = dataprotectionv1alpha1.BackupTypeLogFile return factory } diff --git a/test/integration/backup_mysql_test.go b/test/integration/backup_mysql_test.go index 11afeaa32..c94f7863a 100644 --- a/test/integration/backup_mysql_test.go +++ b/test/integration/backup_mysql_test.go @@ -152,7 +152,7 @@ var _ = Describe("MySQL data protection function", func() { backup := testapps.NewBackupFactory(testCtx.DefaultNamespace, backupName). WithRandomName(). SetBackupPolicyName(backupPolicyKey.Name). - SetBackupType(dpv1alpha1.BackupTypeFull). + SetBackupType(dpv1alpha1.BackupTypeDataFile). Create(&testCtx).GetObject() backupKey = client.ObjectKeyFromObject(backup) } From 0f4e557ae79147c8ef2d5cdec75120ded0cdd2f9 Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Mon, 15 May 2023 19:54:46 +0800 Subject: [PATCH 298/439] fix: fixed default storage class and taint check (#3249) --- .../cmd/kubeblocks/data/ack_preflight.yaml | 24 ++++++++--------- .../cmd/kubeblocks/data/eks_preflight.yaml | 26 +++++++++---------- .../cmd/kubeblocks/data/gke_preflight.yaml | 24 ++++++++--------- .../cmd/kubeblocks/data/tke_preflight.yaml | 24 ++++++++--------- .../preflight/analyzer/kb_storage_class.go | 12 ++++++++- .../analyzer/kb_storage_class_test.go | 26 +++++++++++++++++++ internal/preflight/analyzer/kb_taint.go | 20 +++++++------- internal/preflight/analyzer/kb_taint_test.go | 19 ++++++++++++-- 8 files changed, 114 insertions(+), 61 deletions(-) diff --git a/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml b/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml index 71b5cbdec..ede7deafd 100644 --- a/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/ack_preflight.yaml @@ -11,11 +11,11 @@ spec: outcomes: - fail: when: "< 1.22.0" - message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0. + message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0 uri: https://www.kubernetes.io - pass: when: ">= 1.22.0" - message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes. + message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes uri: https://www.kubernetes.io - nodeResources: checkName: At-Least-3-Nodes @@ -24,14 +24,7 @@ spec: when: "count() < 3" message: This application requires at least 3 nodes - pass: - message: This cluster has enough nodes. - - storageClass: - checkName: Required-Default-SC - outcomes: - - fail: - message: The default storage class was not found. To learn more details, please check https://help.aliyun.com/document_detail/189288.html. - - pass: - message: default storage class is the presence, and all good on storage classes + message: This cluster has enough nodes extendAnalyzers: - clusterAccess: checkName: Check-K8S-Access @@ -44,6 +37,13 @@ spec: checkName: Required-Taint-Match outcomes: - fail: - message: The taint matching failed. + message: all nodes had taints that the pod didn't tolerate + - pass: + message: The taint matching succeeded + - storageClass: + checkName: Required-Default-SC + outcomes: + - warn: + message: The default storage class was not found. To learn more details, please check https://help.aliyun.com/document_detail/189288.html; Alternatively use option --set storageClass= when creating cluster - pass: - message: The taint matching succeeded. \ No newline at end of file + message: default storage class is the presence, and all good on storage classes \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml b/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml index f2e710d6e..c5311e07c 100644 --- a/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/eks_preflight.yaml @@ -25,13 +25,6 @@ spec: message: This application requires at least 3 nodes - pass: message: This cluster has enough nodes. - - storageClass: - checkName: Required-Default-SC - outcomes: - - fail: - message: The default storage class was not found. To learn more details, please check https://docs.aws.amazon.com/zh_cn/eks/latest/userguide/storage-classes.html. - - pass: - message: default storage class is the presence, and all good on storage classes - deploymentStatus: checkName: AWS-Load-Balancer-Check name: aws-load-balancer-controller @@ -39,15 +32,15 @@ spec: outcomes: - warn: when: "absent" # note that the "absent" failure state must be listed first if used. - message: The aws-load-balancer-controller deployment is not present. + message: The aws-load-balancer-controller deployment is not present - warn: when: "< 1" - message: The aws-load-balancer-controller deployment does not have any ready replicas. + message: The aws-load-balancer-controller deployment does not have any ready replicas - warn: when: "= 1" - message: The aws-load-balancer-controller deployment has only a single ready replica. + message: The aws-load-balancer-controller deployment has only a single ready replica - pass: - message: There are multiple replicas of the aws-load-balancer-controller deployment ready. + message: There are multiple replicas of the aws-load-balancer-controller deployment ready extendAnalyzers: - clusterAccess: checkName: Check-K8S-Access @@ -60,6 +53,13 @@ spec: checkName: Required-Taint-Match outcomes: - fail: - message: The taint matching failed. + message: all nodes had taints that the pod didn't tolerate + - pass: + message: The taint matching succeeded + - storageClass: + checkName: Required-Default-SC + outcomes: + - warn: + message: The default storage class was not found. To learn more details, please check https://docs.aws.amazon.com/zh_cn/eks/latest/userguide/storage-classes.html; Alternatively use option --set storageClass= when creating cluster - pass: - message: The taint matching succeeded. \ No newline at end of file + message: default storage class is the presence, and all good on storage classes \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml b/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml index ef55b78fb..8010b1907 100644 --- a/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/gke_preflight.yaml @@ -11,11 +11,11 @@ spec: outcomes: - fail: when: "< 1.22.0" - message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0. + message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0 uri: https://www.kubernetes.io - pass: when: ">= 1.22.0" - message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes. + message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes uri: https://www.kubernetes.io - nodeResources: checkName: At-Least-3-Nodes @@ -24,14 +24,7 @@ spec: when: "count() < 3" message: This application requires at least 3 nodes - pass: - message: This cluster has enough nodes. - - storageClass: - checkName: Required-Default-SC - outcomes: - - fail: - message: The default storage class was not found. To learn more details, please check https://docs.aws.amazon.com/zh_cn/eks/latest/userguide/storage-classes.html. - - pass: - message: default storage class is the presence, and all good on storage classes + message: This cluster has enough nodes extendAnalyzers: - clusterAccess: checkName: Check-K8S-Access @@ -44,6 +37,13 @@ spec: checkName: Required-Taint-Match outcomes: - fail: - message: The taint matching failed. + message: all nodes had taints that the pod didn't tolerate + - pass: + message: The taint matching succeeded + - storageClass: + checkName: Required-Default-SC + outcomes: + - warn: + message: The default storage class was not found. To learn more details, please check https://cloud.google.com/anthos/clusters/docs/on-prem/latest/how-to/default-storage-class; Alternatively use option --set storageClass= when creating cluster - pass: - message: The taint matching succeeded. \ No newline at end of file + message: default storage class is the presence, and all good on storage classes \ No newline at end of file diff --git a/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml b/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml index d5dee2393..f88096638 100644 --- a/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml +++ b/internal/cli/cmd/kubeblocks/data/tke_preflight.yaml @@ -11,11 +11,11 @@ spec: outcomes: - fail: when: "< 1.22.0" - message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0. + message: This application requires at least Kubernetes 1.20.0 or later, and recommends 1.22.0 uri: https://www.kubernetes.io - pass: when: ">= 1.22.0" - message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes. + message: Your cluster meets the recommended and required versions（>= 1.22.0） of Kubernetes uri: https://www.kubernetes.io - nodeResources: checkName: At-Least-3-Nodes @@ -24,14 +24,7 @@ spec: when: "count() < 3" message: This application requires at least 3 nodes - pass: - message: This cluster has enough nodes. - - storageClass: - checkName: Required-Default-SC - outcomes: - - fail: - message: The default storage class was not found. To learn more details, please check https://cloud.tencent.com/document/product/457/44235. - - pass: - message: default storage class is the presence, and all good on storage classes + message: This cluster has enough nodes extendAnalyzers: - clusterAccess: checkName: Check-K8S-Access @@ -44,6 +37,13 @@ spec: checkName: Required-Taint-Match outcomes: - fail: - message: The taint matching failed. + message: all nodes had taints that the pod didn't tolerate + - pass: + message: The taint matching succeeded + - storageClass: + checkName: Required-Default-SC + outcomes: + - warn: + message: The default storage class was not found. To learn more details, please check https://cloud.tencent.com/document/product/457/44235; Alternatively use option --set storageClass= when creating cluster - pass: - message: The taint matching succeeded. \ No newline at end of file + message: default storage class is the presence, and all good on storage classes \ No newline at end of file diff --git a/internal/preflight/analyzer/kb_storage_class.go b/internal/preflight/analyzer/kb_storage_class.go index f336f3b8d..3f57ab788 100644 --- a/internal/preflight/analyzer/kb_storage_class.go +++ b/internal/preflight/analyzer/kb_storage_class.go @@ -25,6 +25,7 @@ import ( analyze "github.com/replicatedhq/troubleshoot/pkg/analyze" storagev1beta1 "k8s.io/api/storage/v1beta1" + "k8s.io/kubectl/pkg/util/storage" preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" "github.com/apecloud/kubeblocks/internal/preflight/util" @@ -70,6 +71,15 @@ func (a *AnalyzeStorageClassByKb) analyzeStorageClass(analyzer *preflightv1beta2 } for _, storageClass := range storageClasses.Items { + // if storageClassType not set, check if default storageClass exists + if analyzer.StorageClassType == "" { + val := storageClass.Annotations[storage.IsDefaultStorageClassAnnotation] + if val == "true" { + return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil + } + continue + } + if storageClass.Parameters["type"] != analyzer.StorageClassType { continue } @@ -77,7 +87,7 @@ func (a *AnalyzeStorageClassByKb) analyzeStorageClass(analyzer *preflightv1beta2 return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil } } - return newAnalyzeResult(a.Title(), FailType, a.analyzer.Outcomes), nil + return newAnalyzeResult(a.Title(), WarnType, a.analyzer.Outcomes), nil } var _ KBAnalyzer = &AnalyzeStorageClassByKb{} diff --git a/internal/preflight/analyzer/kb_storage_class_test.go b/internal/preflight/analyzer/kb_storage_class_test.go index 57fa18011..c8e449097 100644 --- a/internal/preflight/analyzer/kb_storage_class_test.go +++ b/internal/preflight/analyzer/kb_storage_class_test.go @@ -25,6 +25,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubectl/pkg/util/storage" troubleshoot "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" storagev1beta1 "k8s.io/api/storage/v1beta1" @@ -41,6 +43,15 @@ var ( }, }, } + clusterResources2 = storagev1beta1.StorageClassList{ + Items: []storagev1beta1.StorageClass{ + { + Provisioner: "ebs.csi.aws.com", + Parameters: map[string]string{"type": "gp3"}, + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{storage.IsDefaultStorageClassAnnotation: "true"}}, + }, + }, + } ) var _ = Describe("kb_storage_class_test", func() { @@ -118,5 +129,20 @@ var _ = Describe("kb_storage_class_test", func() { g.Expect(res[0].IsFail).Should(BeTrue()) }).Should(Succeed()) }) + It("Analyze test, and analyzer result is expected that fail is true", func() { + Eventually(func(g Gomega) { + g.Expect(analyzer.IsExcluded()).Should(BeFalse()) + b, err := json.Marshal(clusterResources2) + g.Expect(err).NotTo(HaveOccurred()) + getCollectedFileContents := func(filename string) ([]byte, error) { + return b, nil + } + analyzer.analyzer.StorageClassType = "" + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res[0].IsPass).Should(BeTrue()) + g.Expect(res[0].IsWarn).Should(BeFalse()) + }).Should(Succeed()) + }) }) }) diff --git a/internal/preflight/analyzer/kb_taint.go b/internal/preflight/analyzer/kb_taint.go index f9755a313..94c4e960d 100644 --- a/internal/preflight/analyzer/kb_taint.go +++ b/internal/preflight/analyzer/kb_taint.go @@ -70,9 +70,6 @@ func (a *AnalyzeTaintClassByKb) Analyze(getFile GetCollectedFileContents, findFi } func (a *AnalyzeTaintClassByKb) analyzeTaint(getFile GetCollectedFileContents, findFiles GetChildCollectedFileContents) (*analyze.AnalyzeResult, error) { - if a.HelmOpts == nil { - return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil - } nodesData, err := getFile(NodesPath) if err != nil { return newFailedResultWithMessage(a.Title(), fmt.Sprintf("get jsonfile failed, err:%v", err)), err @@ -99,6 +96,10 @@ func (a *AnalyzeTaintClassByKb) doAnalyzeTaint(nodes v1.NodeList) (*analyze.Anal } } + if a.analyzer.TolerationsMap == nil || len(a.analyzer.TolerationsMap) == 0 { + return newAnalyzeResult(a.Title(), FailType, a.analyzer.Outcomes), nil + } + for k, tolerations := range a.analyzer.TolerationsMap { count := 0 for _, node := range nodes.Items { @@ -117,13 +118,14 @@ func (a *AnalyzeTaintClassByKb) doAnalyzeTaint(nodes v1.NodeList) (*analyze.Anal } func (a *AnalyzeTaintClassByKb) generateTolerations() error { - optsMap, err := a.getHelmValues() - if err != nil { - return err - } - tolerations := map[string][]v1.Toleration{} - getTolerationsMap(optsMap, "", tolerations) + if a.HelmOpts != nil { + optsMap, err := a.getHelmValues() + if err != nil { + return err + } + getTolerationsMap(optsMap, "", tolerations) + } a.analyzer.TolerationsMap = tolerations return nil } diff --git a/internal/preflight/analyzer/kb_taint_test.go b/internal/preflight/analyzer/kb_taint_test.go index 612848720..4668a3654 100644 --- a/internal/preflight/analyzer/kb_taint_test.go +++ b/internal/preflight/analyzer/kb_taint_test.go @@ -69,10 +69,10 @@ var _ = Describe("taint_class_test", func() { Outcomes: []*troubleshoot.Outcome{ { Pass: &troubleshoot.SingleOutcome{ - Message: "analyze storage class success", + Message: "analyze taint success", }, Fail: &troubleshoot.SingleOutcome{ - Message: "analyze storage class fail", + Message: "analyze taint fail", }, }, }, @@ -147,5 +147,20 @@ var _ = Describe("taint_class_test", func() { g.Expect(res[0].IsFail).Should(BeFalse()) }).Should(Succeed()) }) + It("Analyze test, the tolerations are nil, and analyzer result is expected that fail is true", func() { + Eventually(func(g Gomega) { + g.Expect(analyzer.IsExcluded()).Should(BeFalse()) + b, err := json.Marshal(nodeList2) + g.Expect(err).NotTo(HaveOccurred()) + getCollectedFileContents := func(filename string) ([]byte, error) { + return b, nil + } + analyzer.HelmOpts = nil + res, err := analyzer.Analyze(getCollectedFileContents, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(res[0].IsPass).Should(BeFalse()) + g.Expect(res[0].IsFail).Should(BeTrue()) + }).Should(Succeed()) + }) }) }) From c48d9b8f36004c2a94122ae2b524ff596e3e71f3 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Mon, 15 May 2023 20:01:08 +0800 Subject: [PATCH 299/439] chore: fix remove runner skipped (#3252) --- .github/workflows/cicd-pull-request.yml | 4 ++-- .github/workflows/cicd-push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index dcd55b7f5..9b793f97e 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -57,9 +57,9 @@ jobs: bash .github/utils/utils.sh --type 8 remove-runner: - needs: make-test + needs: [ trigger-mode, make-test ] runs-on: ubuntu-latest - if: ${{ always() }} + if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[test]') && always() }} steps: - uses: actions/checkout@v3 - name: remove runner diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index f1b0d919a..8faaecbc8 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -154,9 +154,9 @@ jobs: bash .github/utils/utils.sh --type 8 remove-runner: - needs: make-test + needs: [ trigger-mode, make-test ] runs-on: ubuntu-latest - if: ${{ needs.make-test.result != 'skipped' }} + if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[test]') && always() }} steps: - uses: actions/checkout@v3 - name: remove runner From 1fcee6cba2632dec41d15a01178adea9ee304b53 Mon Sep 17 00:00:00 2001 From: zhangtao <111836083+sophon-zt@users.noreply.github.com> Date: Mon, 15 May 2023 21:42:14 +0800 Subject: [PATCH 300/439] chore: adjust prompt for verification failed (#3222) --- internal/configuration/cue_visitor.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/configuration/cue_visitor.go b/internal/configuration/cue_visitor.go index 2f4758a8b..932e61831 100644 --- a/internal/configuration/cue_visitor.go +++ b/internal/configuration/cue_visitor.go @@ -223,6 +223,10 @@ func processCfgNotStringParam(data interface{}, context *cue.Context, tpl cue.Va if !exist { return nil } - return transNumberOrBoolType(typeTransformer.fieldTypes[fieldPath], obj, fn, typeTransformer.fieldUnits[fieldPath], trimString) + err := transNumberOrBoolType(typeTransformer.fieldTypes[fieldPath], obj, fn, typeTransformer.fieldUnits[fieldPath], trimString) + if err != nil { + return WrapError(err, "failed to parse field %s", fieldPath) + } + return nil }, false) } From edf3ac9ef0666514731830f0b61832ecf895bea1 Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Tue, 16 May 2023 09:05:01 +0800 Subject: [PATCH 301/439] chore: fix the field name and use the `io.Writer` to output (#3251) --- internal/cli/spinner/windows_spinner.go | 33 ++++++++++++------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/internal/cli/spinner/windows_spinner.go b/internal/cli/spinner/windows_spinner.go index ac50fb871..94136cf52 100644 --- a/internal/cli/spinner/windows_spinner.go +++ b/internal/cli/spinner/windows_spinner.go @@ -36,15 +36,15 @@ import ( var char = []string{"⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"} type WindowsSpinner struct { // no thread/goroutine safe - msg string - lastOutplain string - FinalMSG string - active bool - chars []string - cancel chan struct{} - Writer io.Writer - delay time.Duration - mu *sync.RWMutex + msg string + lastOutput string + FinalMSG string + active bool + chars []string + cancel chan struct{} + Writer io.Writer + delay time.Duration + mu *sync.RWMutex } func NewWindowsSpinner(w io.Writer, opts ...Option) *WindowsSpinner { @@ -114,10 +114,9 @@ func (s *WindowsSpinner) Start() { } outPlain := fmt.Sprintf("\r%s%s", s.chars[i], s.msg) s.erase() - s.lastOutplain = outPlain - fmt.Print(outPlain) + s.lastOutput = outPlain + fmt.Fprint(s.Writer, outPlain) s.mu.Unlock() - // fmt.Fprint(s.Writer, outPlain) time.Sleep(s.delay) } } @@ -135,14 +134,14 @@ func (s *WindowsSpinner) SetFinalMsg(msg string) { s.FinalMSG = msg } -// remove lastOutplain +// remove lastOutput func (s *WindowsSpinner) erase() { - split := strings.Split(s.lastOutplain, "\n") + split := strings.Split(s.lastOutput, "\n") for i := 0; i < len(split); i++ { if i > 0 { - fmt.Print("\033[A") + fmt.Fprint(s.Writer, "\033[A") } - fmt.Print("\r\033[K") + fmt.Fprint(s.Writer, "\r\033[K") } } @@ -154,7 +153,7 @@ func (s *WindowsSpinner) stop() { s.active = false if s.FinalMSG != "" { s.erase() - fmt.Print(s.FinalMSG) + fmt.Fprint(s.Writer, s.FinalMSG) } s.cancel <- struct{}{} close(s.cancel) From 883620996f1fce10c73b7d469c2619bfb7336b11 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Tue, 16 May 2023 11:58:44 +0800 Subject: [PATCH 302/439] chore: local playground with one replica (#3255) --- docs/user_docs/cli/kbcli_playground_init.md | 2 +- internal/cli/cloudprovider/interface.go | 4 ++-- internal/cli/cloudprovider/k3d.go | 2 +- internal/cli/cloudprovider/k3d_test.go | 2 +- internal/cli/cloudprovider/provider.go | 2 +- internal/cli/cmd/playground/destroy.go | 2 +- internal/cli/cmd/playground/init.go | 9 ++++++--- 7 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/user_docs/cli/kbcli_playground_init.md b/docs/user_docs/cli/kbcli_playground_init.md index 26b7597ff..1877e636b 100644 --- a/docs/user_docs/cli/kbcli_playground_init.md +++ b/docs/user_docs/cli/kbcli_playground_init.md @@ -35,7 +35,7 @@ kbcli playground init [flags] --cluster-version string Cluster definition -h, --help help for init --region string The region to create kubernetes cluster - --timeout duration Time to wait for initing playground, such as --timeout=10m (default 5m0s) + --timeout duration Time to wait for init playground, such as --timeout=10m (default 5m0s) --version string KubeBlocks version ``` diff --git a/internal/cli/cloudprovider/interface.go b/internal/cli/cloudprovider/interface.go index 74325abde..bf1c5e70c 100644 --- a/internal/cli/cloudprovider/interface.go +++ b/internal/cli/cloudprovider/interface.go @@ -43,9 +43,9 @@ type Interface interface { func New(provider, tfRootPath string, stdout, stderr io.Writer) (Interface, error) { switch provider { case AWS, TencentCloud, AliCloud, GCP: - return NewCloudProvider(provider, tfRootPath, stdout, stderr) + return newCloudProvider(provider, tfRootPath, stdout, stderr) case Local: - return NewLocalCloudProvider(stdout, stderr), nil + return newLocalCloudProvider(stdout, stderr), nil default: return nil, errors.New(fmt.Sprintf("Unknown cloud provider %s", provider)) } diff --git a/internal/cli/cloudprovider/k3d.go b/internal/cli/cloudprovider/k3d.go index c1c146156..bf4dc9d78 100644 --- a/internal/cli/cloudprovider/k3d.go +++ b/internal/cli/cloudprovider/k3d.go @@ -80,7 +80,7 @@ func init() { } } -func NewLocalCloudProvider(stdout, stderr io.Writer) *localCloudProvider { +func newLocalCloudProvider(stdout, stderr io.Writer) Interface { return &localCloudProvider{ stdout: stdout, stderr: stderr, diff --git a/internal/cli/cloudprovider/k3d_test.go b/internal/cli/cloudprovider/k3d_test.go index 9ab1dbc91..e3afa3c65 100644 --- a/internal/cli/cloudprovider/k3d_test.go +++ b/internal/cli/cloudprovider/k3d_test.go @@ -29,7 +29,7 @@ import ( var _ = Describe("playground", func() { var ( - provider = NewLocalCloudProvider(os.Stdout, os.Stderr) + provider = newLocalCloudProvider(os.Stdout, os.Stderr) clusterName = "k3d-tb-est" ) diff --git a/internal/cli/cloudprovider/provider.go b/internal/cli/cloudprovider/provider.go index 75fad4179..430973553 100644 --- a/internal/cli/cloudprovider/provider.go +++ b/internal/cli/cloudprovider/provider.go @@ -36,7 +36,7 @@ type cloudProvider struct { var _ Interface = &cloudProvider{} -func NewCloudProvider(provider, rootPath string, stdout, stderr io.Writer) (Interface, error) { +func newCloudProvider(provider, rootPath string, stdout, stderr io.Writer) (Interface, error) { k8sSvc := K8sService(provider) if k8sSvc == "" { return nil, fmt.Errorf("unknown cloud provider %s", provider) diff --git a/internal/cli/cmd/playground/destroy.go b/internal/cli/cmd/playground/destroy.go index 50c725676..c81497cae 100644 --- a/internal/cli/cmd/playground/destroy.go +++ b/internal/cli/cmd/playground/destroy.go @@ -110,7 +110,7 @@ func (o *destroyOptions) destroy() error { // destroyLocal destroy local k3d cluster that will destroy all resources func (o *destroyOptions) destroyLocal() error { - provider := cp.NewLocalCloudProvider(o.Out, o.ErrOut) + provider, _ := cp.New(cp.Local, "", o.Out, o.ErrOut) s := spinner.New(o.Out, spinnerMsg("Delete playground k3d cluster "+o.prevCluster.ClusterName)) defer s.Fail() if err := provider.DeleteK8sCluster(o.prevCluster); err != nil { diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index f3e3d51e9..cc08ac5dd 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -106,7 +106,7 @@ func newInitCmd(streams genericclioptions.IOStreams) *cobra.Command { cmd.Flags().StringVar(&o.kbVersion, "version", version.DefaultKubeBlocksVersion, "KubeBlocks version") cmd.Flags().StringVar(&o.cloudProvider, "cloud-provider", defaultCloudProvider, fmt.Sprintf("Cloud provider type, one of %v", supportedCloudProviders)) cmd.Flags().StringVar(&o.region, "region", "", "The region to create kubernetes cluster") - cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for initing playground, such as --timeout=10m") + cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for init playground, such as --timeout=10m") util.CheckErr(cmd.RegisterFlagCompletionFunc( "cloud-provider", @@ -473,8 +473,11 @@ func (o *initOptions) createCluster() error { options.CreateOptions.Options = options options.CreateOptions.PreCreate = options.PreCreate - // if we are running on cloud, create cluster with three replicas - if o.cloudProvider != cp.Local { + // if we are running on local, create cluster with one replica + if o.cloudProvider == cp.Local { + options.Values = append(options.Values, "replicas=1") + } else { + // if we are running on cloud, create cluster with three replicas options.Values = append(options.Values, "replicas=3") } From 7a8095a204a268b9d4b0db24f579c1c3777203f4 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Tue, 16 May 2023 15:00:06 +0800 Subject: [PATCH 303/439] chore: update mongo helm (#3247) --- .../templates/sharding-clusterdefinition.yaml | 6 +-- .../templates/sharding-scriptstemplate.yaml | 41 +++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 deploy/mongodb/templates/sharding-scriptstemplate.yaml diff --git a/deploy/mongodb/templates/sharding-clusterdefinition.yaml b/deploy/mongodb/templates/sharding-clusterdefinition.yaml index 96d4dc01e..072c23f67 100644 --- a/deploy/mongodb/templates/sharding-clusterdefinition.yaml +++ b/deploy/mongodb/templates/sharding-clusterdefinition.yaml @@ -19,7 +19,7 @@ spec: - name: mongos scriptSpecs: - name: mongodb-scripts - templateRef: mongodb-scripts + templateRef: mongodb-sharding-scripts volumeName: scripts namespace: {{ .Release.Namespace }} defaultMode: 493 @@ -44,7 +44,7 @@ spec: - name: configsvr scriptSpecs: - name: mongodb-scripts - templateRef: mongodb-scripts + templateRef: mongodb-sharding-scripts volumeName: scripts namespace: {{ .Release.Namespace }} defaultMode: 493 @@ -94,7 +94,7 @@ spec: - name: shard scriptSpecs: - name: mongodb-scripts - templateRef: mongodb-scripts + templateRef: mongodb-sharding-scripts volumeName: scripts namespace: {{ .Release.Namespace }} defaultMode: 493 diff --git a/deploy/mongodb/templates/sharding-scriptstemplate.yaml b/deploy/mongodb/templates/sharding-scriptstemplate.yaml new file mode 100644 index 000000000..d791696fd --- /dev/null +++ b/deploy/mongodb/templates/sharding-scriptstemplate.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-sharding-scripts + labels: + {{- include "mongodb.labels" . | nindent 4 }} +data: + mongos-setup.sh: |- + #!/bin/sh + + PORT=27018 + CONFIG_SVR_NAME=$KB_CLUSTER_NAME"-configsvr" + DOMAIN=$CONFIG_SVR_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" + mongos --bind_ip_all --configdb $CONFIG_SVR_NAME/$CONFIG_SVR_NAME"-0."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-1."$DOMAIN:$PORT,$CONFIG_SVR_NAME"-2."$DOMAIN:$PORT + replicaset-setup.sh: |- + {{- .Files.Get "scripts/replicaset-setup.tpl" | nindent 4 }} + replicaset-post-start.sh: |- + {{- .Files.Get "scripts/replicaset-post-start.tpl" | nindent 4 }} + shard-agent.sh: |- + #!/bin/sh + + INDEX=$(echo $KB_POD_NAME | grep -o "\-[0-9]\+\$"); + INDEX=${INDEX#-}; + if [ $INDEX -ne 0 ]; then + trap : TERM INT; (while true; do sleep 1000; done) & wait + fi + + # wait main container ready + PORT=27018 + until mongosh --quiet --port $PORT --eval "rs.status().ok"; do sleep 1; done + # add shard to mongos + SHARD_NAME=$(echo $KB_POD_NAME | grep -o ".*-"); + SHARD_NAME=${SHARD_NAME%-}; + DOMAIN=$SHARD_NAME"-headless."$KB_NAMESPACE".svc.cluster.local" + MONGOS_HOST=$KB_CLUSTER_NAME"-mongos" + MONGOS_PORT=27017 + SHARD_CONFIG=$SHARD_NAME/$SHARD_NAME"-0."$DOMAIN:$PORT,$SHARD_NAME"-1."$DOMAIN:$PORT,$SHARD_NAME"-2."$DOMAIN:$PORT + until mongosh --quiet --host $MONGOS_HOST --port $MONGOS_PORT --eval "print('service is ready')"; do sleep 1; done + mongosh --quiet --host $MONGOS_HOST --port $MONGOS_PORT --eval "sh.addShard(\"$SHARD_CONFIG\")" + + trap : TERM INT; (while true; do sleep 1000; done) & wait From 8bd61d5c53432e9c8fbf0d0a658c5657a302919d Mon Sep 17 00:00:00 2001 From: kubeJocker <102039539+kubeJocker@users.noreply.github.com> Date: Tue, 16 May 2023 16:08:32 +0800 Subject: [PATCH 304/439] fix: preflight-incorrect-when-taints-match-partly (#3270) --- internal/preflight/analyzer/kb_taint.go | 22 ++++++++++---------- internal/preflight/analyzer/kb_taint_test.go | 2 ++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/internal/preflight/analyzer/kb_taint.go b/internal/preflight/analyzer/kb_taint.go index 94c4e960d..08dcf39ce 100644 --- a/internal/preflight/analyzer/kb_taint.go +++ b/internal/preflight/analyzer/kb_taint.go @@ -31,7 +31,6 @@ import ( "helm.sh/helm/v3/pkg/cli/values" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" - v1helper "k8s.io/component-helpers/scheduling/corev1" preflightv1beta2 "github.com/apecloud/kubeblocks/externalapis/preflight/v1beta2" "github.com/apecloud/kubeblocks/internal/preflight/util" @@ -86,9 +85,6 @@ func (a *AnalyzeTaintClassByKb) analyzeTaint(getFile GetCollectedFileContents, f } func (a *AnalyzeTaintClassByKb) doAnalyzeTaint(nodes v1.NodeList) (*analyze.AnalyzeResult, error) { - if a.analyzer.TolerationsMap == nil { - return newAnalyzeResult(a.Title(), PassType, a.analyzer.Outcomes), nil - } taintFailResult := []string{} for _, node := range nodes.Items { if node.Spec.Taints == nil || len(node.Spec.Taints) == 0 { @@ -103,7 +99,9 @@ func (a *AnalyzeTaintClassByKb) doAnalyzeTaint(nodes v1.NodeList) (*analyze.Anal for k, tolerations := range a.analyzer.TolerationsMap { count := 0 for _, node := range nodes.Items { - count += countTolerableTaints(node.Spec.Taints, tolerations) + if isTolerableTaints(node.Spec.Taints, tolerations) { + count++ + } } if count <= 0 { taintFailResult = append(taintFailResult, k) @@ -175,17 +173,19 @@ func getTolerationsMap(tolerationData map[string]interface{}, addonName string, } } -func countTolerableTaints(taints []v1.Taint, tolerations []v1.Toleration) int { - tolerableTaints := 0 +func isTolerableTaints(taints []v1.Taint, tolerations []v1.Toleration) bool { + tolerableCount := 0 for _, taint := range taints { // check only on taints that have effect NoSchedule if taint.Effect != v1.TaintEffectNoSchedule { continue } - - if v1helper.TolerationsTolerateTaint(tolerations, &taint) { - tolerableTaints++ + for _, toleration := range tolerations { + if toleration.ToleratesTaint(&taint) { + tolerableCount++ + break + } } } - return tolerableTaints + return tolerableCount >= len(taints) } diff --git a/internal/preflight/analyzer/kb_taint_test.go b/internal/preflight/analyzer/kb_taint_test.go index 4668a3654..31f21624b 100644 --- a/internal/preflight/analyzer/kb_taint_test.go +++ b/internal/preflight/analyzer/kb_taint_test.go @@ -37,6 +37,7 @@ var ( nodeList1 = v1.NodeList{Items: []v1.Node{ {Spec: v1.NodeSpec{Taints: []v1.Taint{ {Key: "dev", Value: "true", Effect: v1.TaintEffectNoSchedule}, + {Key: "large", Value: "true", Effect: v1.TaintEffectNoSchedule}, }}}, {Spec: v1.NodeSpec{Taints: []v1.Taint{ {Key: "dev", Value: "false", Effect: v1.TaintEffectNoSchedule}, @@ -45,6 +46,7 @@ var ( nodeList2 = v1.NodeList{Items: []v1.Node{ {Spec: v1.NodeSpec{Taints: []v1.Taint{ {Key: "dev", Value: "false", Effect: v1.TaintEffectNoSchedule}, + {Key: "large", Value: "true", Effect: v1.TaintEffectNoSchedule}, }}}, }} nodeList3 = v1.NodeList{Items: []v1.Node{ From 03df97e8b623c2a906ad735f712f15de770f5974 Mon Sep 17 00:00:00 2001 From: dingben Date: Tue, 16 May 2023 16:30:59 +0800 Subject: [PATCH 305/439] feat: support auto complete cv based on cd when cli create cluster (#3025) --- internal/cli/cmd/cluster/create.go | 10 +++- internal/cli/util/completion.go | 69 +++++++++++++++++++++++ internal/cli/util/completion_test.go | 82 ++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 internal/cli/util/completion_test.go diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index e5959bfe0..1b7f44739 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -549,7 +549,15 @@ func registerFlagCompletionFunc(cmd *cobra.Command, f cmdutil.Factory) { util.CheckErr(cmd.RegisterFlagCompletionFunc( "cluster-version", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return utilcomp.CompGetResource(f, cmd, util.GVRToString(types.ClusterVersionGVR()), toComplete), cobra.ShellCompDirectiveNoFileComp + var clusterVersion []string + clusterDefinition, err := cmd.Flags().GetString("cluster-definition") + if clusterDefinition == "" || err != nil { + clusterVersion = utilcomp.CompGetResource(f, cmd, util.GVRToString(types.ClusterVersionGVR()), toComplete) + } else { + label := fmt.Sprintf("%s=%s", constant.ClusterDefLabelKey, clusterDefinition) + clusterVersion = util.CompGetResourceWithLabels(f, cmd, util.GVRToString(types.ClusterVersionGVR()), []string{label}, toComplete) + } + return clusterVersion, cobra.ShellCompDirectiveNoFileComp })) var formatsWithDesc = map[string]string{ diff --git a/internal/cli/util/completion.go b/internal/cli/util/completion.go index deac3fa87..7d2f10c5e 100644 --- a/internal/cli/util/completion.go +++ b/internal/cli/util/completion.go @@ -20,8 +20,17 @@ along with this program. If not, see . package util import ( + "bytes" + "io/ioutil" + "os" + "strings" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubectl/pkg/cmd/get" cmdutil "k8s.io/kubectl/pkg/cmd/util" utilcomp "k8s.io/kubectl/pkg/util/completion" ) @@ -43,3 +52,63 @@ func ResourceNameCompletionFunc(f cmdutil.Factory, gvr schema.GroupVersionResour return availableComps, cobra.ShellCompDirectiveNoFileComp } } + +// CompGetResourceWithLabels gets the list of the resource specified which begin with `toComplete` and have the specified labels. +// example: CompGetResourceWithLabels(f, cmd, "pods", []string{"app=nginx"}, toComplete) +// gets the name of the pods which have the label `app=nginx` and begin with `toComplete` +func CompGetResourceWithLabels(f cmdutil.Factory, cmd *cobra.Command, resourceName string, labels []string, toComplete string) []string { + template := "{{ range .items }}{{ .metadata.name }} {{ end }}" + return CompGetFromTemplateWithLabels(&template, f, "", cmd, []string{resourceName}, labels, toComplete) +} + +// CompGetFromTemplateWithLabels executes a Get operation using the specified template and args and returns the results +// which begin with `toComplete` and have the specified labels. +// example: CompGetFromTemplateWithLabels(&template, f, "", cmd, []string{"pods"}, []string{"app=nginx"}, toComplete) +// will get the output of `kubectl get pods --template=template -l app=nginx`, and split the output by space and return +func CompGetFromTemplateWithLabels(template *string, f cmdutil.Factory, namespace string, cmd *cobra.Command, args []string, labels []string, toComplete string) []string { + buf := new(bytes.Buffer) + streams := genericclioptions.IOStreams{In: os.Stdin, Out: buf, ErrOut: ioutil.Discard} + o := get.NewGetOptions("kubectl", streams) + + // Get the list of names of the specified resource + o.PrintFlags.TemplateFlags.GoTemplatePrintFlags.TemplateArgument = template + format := "go-template" + o.PrintFlags.OutputFormat = &format + + // Do the steps Complete() would have done. + // We cannot actually call Complete() or Validate() as these function check for + // the presence of flags, which, in our case won't be there + if namespace != "" { + o.Namespace = namespace + o.ExplicitNamespace = true + } else { + var err error + o.Namespace, o.ExplicitNamespace, err = f.ToRawKubeConfigLoader().Namespace() + if err != nil { + return nil + } + } + + o.ToPrinter = func(mapping *meta.RESTMapping, outputObjects *bool, withNamespace bool, withKind bool) (printers.ResourcePrinterFunc, error) { + printer, err := o.PrintFlags.ToPrinter() + if err != nil { + return nil, err + } + return printer.PrintObj, nil + } + + if len(labels) > 0 { + o.LabelSelector = strings.Join(labels, ",") + } + + _ = o.Run(f, cmd, args) + + var comps []string + resources := strings.Split(buf.String(), " ") + for _, res := range resources { + if res != "" && strings.HasPrefix(res, toComplete) { + comps = append(comps, res) + } + } + return comps +} diff --git a/internal/cli/util/completion_test.go b/internal/cli/util/completion_test.go new file mode 100644 index 000000000..80f604c3e --- /dev/null +++ b/internal/cli/util/completion_test.go @@ -0,0 +1,82 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package util + +import ( + "fmt" + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes/scheme" + clientfake "k8s.io/client-go/rest/fake" + "k8s.io/kubectl/pkg/cmd/get" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" + "github.com/apecloud/kubeblocks/internal/constant" +) + +var _ = Describe("completion", func() { + const ( + namespace = testing.Namespace + clusterName = testing.ClusterName + ) + + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + pods = testing.FakePods(3, namespace, clusterName) + ) + + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + }) + + AfterEach(func() { + tf.Cleanup() + }) + + It("test completion pods", func() { + cmd := get.NewCmdGet("kbcli", tf, streams) + codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) + tf.UnstructuredClient = &clientfake.RESTClient{ + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + Client: clientfake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { + switch req.URL.Query().Get(metav1.LabelSelectorQueryParam("v1")) { + case fmt.Sprintf("%s=%s", constant.RoleLabelKey, "leader"): + return &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, &pods.Items[0])}, nil + case "": + return &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, &pods.Items[0])}, nil + default: + return nil, fmt.Errorf("unexpected request: %v", req.URL) + } + }), + } + + Expect(len(CompGetResourceWithLabels(tf, cmd, "pods", []string{}, ""))).Should(Equal(1)) + Expect(len(CompGetResourceWithLabels(tf, cmd, "pods", []string{fmt.Sprintf("%s=%s", constant.RoleLabelKey, "leader")}, ""))).Should(Equal(1)) + }) +}) From ad613283b70745981b8390d8312fb393e9322ebf Mon Sep 17 00:00:00 2001 From: ZhuuChaang <104178632+ZhuuChaang@users.noreply.github.com> Date: Tue, 16 May 2023 18:35:03 +0800 Subject: [PATCH 306/439] feat: add grafana dashboard for node exporter (#3278) --- deploy/helm/dashboards/node-exporter.json | 22413 ++++++++++++++++++++ 1 file changed, 22413 insertions(+) create mode 100644 deploy/helm/dashboards/node-exporter.json diff --git a/deploy/helm/dashboards/node-exporter.json b/deploy/helm/dashboards/node-exporter.json new file mode 100644 index 000000000..1f820987a --- /dev/null +++ b/deploy/helm/dashboards/node-exporter.json @@ -0,0 +1,22413 @@ +{ + "annotations": { + "list": [ + { + "$$hashKey": "object:1058", + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 1860, + "graphTooltip": 1, + "id": 23, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "KubeBlocks", + "tooltip": "An open-source and cloud-neutral DBaaS with Kubernetes.", + "type": "link", + "url": "https://github.com/apecloud/kubeblocks" + }, + { + "asDropdown": false, + "icon": "cloud", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "ApeCloud", + "tooltip": "Improved productivity, cost-efficiency and business continuity.", + "type": "link", + "url": "https://kubeblocks.io/" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 322, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total number of CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\"}) by (cpu))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 240 + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total RAM", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 75, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\"}", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "RAM Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total SWAP", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\"}", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total Disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 323, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "topk(1,node_filesystem_size_bytes{instance=\"$node\",device!~'rootfs'})", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk Total", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Basic CPU info", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 12, + "y": 1 + }, + "id": 382, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy System", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Busy User", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\",mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy IOWait", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=~\".*irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy IRQs", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Busy Other", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "Idle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "System uptime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 1 + }, + "hideTimeOverride": true, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_time_seconds{instance=\"$node\"} - node_boot_time_seconds{instance=\"$node\"}", + "intervalFactor": 1, + "legendFormat": "The node has been on for how many time", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "System Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Busy state of all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 60 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 3 + }, + "id": 20, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode!=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))) * 100", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Busy", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Non available RAM memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 60 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 3 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "((node_memory_MemTotal_bytes{instance=\"$node\"} - node_memory_MemFree_bytes{instance=\"$node\"}) / (node_memory_MemTotal_bytes{instance=\"$node\"} )) * 100", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 - ((node_memory_MemAvailable_bytes{instance=\"$node\"} * 100) / node_memory_MemTotal_bytes{instance=\"$node\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "B", + "step": 240 + } + ], + "title": "RAM Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Used Swap", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 60 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 3 + }, + "id": 21, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "((node_memory_SwapTotal_bytes{instance=\"$node\"} - node_memory_SwapFree_bytes{instance=\"$node\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\"} )) * 100", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "SWAP Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Used Disk", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 60 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 3 + }, + "id": 324, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "((topk(1,node_filesystem_size_bytes{instance=\"$node\",device!~'rootfs'})- topk(1,node_filesystem_free_bytes{instance=\"$node\",device!~'rootfs'})) / topk(1,node_filesystem_size_bytes{instance=\"$node\",device!~'rootfs'})) * 100", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk Used", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "system load", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 4 + }, + "hideTimeOverride": true, + "id": 325, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "node_load5{instance=\"$node\"} ", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "jobs waiting and running", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "System Load", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Basic memory usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SWAP Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Available" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#DEDAF7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 78, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Total", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\"} - node_memory_MemFree_bytes{instance=\"$node\"} - (node_memory_Cached_bytes{instance=\"$node\"} + node_memory_Buffers_bytes{instance=\"$node\"} + node_memory_SReclaimable_bytes{instance=\"$node\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "RAM Used", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Cached_bytes{instance=\"$node\"} + node_memory_Buffers_bytes{instance=\"$node\"} + node_memory_SReclaimable_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Cache + Buffer", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemFree_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "RAM Free", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\"} - node_memory_SwapFree_bytes{instance=\"$node\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SWAP Used", + "refId": "E", + "step": 240 + } + ], + "title": "Memory ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of bytes read from or written to the device per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 431, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Disk space used of all filesystems mounted", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 152, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"$node\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=\"$node\",device!~'rootfs'})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Disk Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Basic network info per interface", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_eth2" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CCA300", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "recv_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_bytes_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_drop_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#967302", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "trans_errs_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 74, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_receive_bytes_total{instance=\"$node\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "recv {{device}}", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_transmit_bytes_total{instance=\"$node\"}[$__rate_interval])*8", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "trans {{device}} ", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 327, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Iowait - Waiting for I/O to complete" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Irq - Servicing interrupts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nice - Niced processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Softirq - Servicing softirqs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Steal - Time spent in other operating systems when running in a virtualized environment" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System - Processes executing in kernel mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User - Normal processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 3, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"system\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "System - Processes executing in kernel mode", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"user\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "User - Normal processes executing in user mode", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"nice\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Nice - Niced processes executing in user mode", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"iowait\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Iowait - Waiting for I/O to complete", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"irq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Irq - Servicing interrupts", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"softirq\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Softirq - Servicing softirqs", + "range": true, + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"steal\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "range": true, + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by(instance) (irate(node_cpu_seconds_total{instance=\"$node\", mode=\"idle\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Idle - Waiting for something to happen", + "range": true, + "refId": "J", + "step": 240 + } + ], + "title": "CPU Modes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time each CPU spend in system mode in every default interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 341, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_cpu_seconds_total{instance=\"$node\",mode=\"system\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU System Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time each CPU spend in IDLE mode in every default interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 334, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_cpu_seconds_total{instance=\"$node\",mode=\"idle\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Idle Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time each CPU spend in system mode in every default interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 21 + }, + "id": 368, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_cpu_seconds_total{instance=\"$node\",mode=\"user\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU User Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time each CPU spend in iowait mode in every default interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 21 + }, + "id": 335, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_cpu_seconds_total{instance=\"$node\",mode=\"iowait\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU IOWait time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time each CPU spend in iqr mode in every default interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 21 + }, + "id": 338, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_cpu_seconds_total{instance=\"$node\",mode=\"irq\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Irq Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time each CPU spend in nice mode in every default interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 27 + }, + "id": 337, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_cpu_seconds_total{instance=\"$node\",mode=\"nice\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Nice time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time each CPU spend in softirq mode in every default interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 336, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_cpu_seconds_total{instance=\"$node\",mode=\"softirq\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Softirq time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "the time each CPU spend in steal mode in every default interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 342, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_cpu_seconds_total{instance=\"$node\",mode=\"steal\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Steal time", + "type": "timeseries" + } + ], + "title": "CPU per Core", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 266, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap - Swap memory usage" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused - Free memory unassigned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Hardware Corrupted - *./" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 24, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\"} - node_memory_MemFree_bytes{instance=\"$node\"} - node_memory_Buffers_bytes{instance=\"$node\"} - node_memory_Cached_bytes{instance=\"$node\"} - node_memory_Slab_bytes{instance=\"$node\"} - node_memory_PageTables_bytes{instance=\"$node\"} - node_memory_SwapCached_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Apps - Memory used by user-space applications", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_PageTables_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_SwapCached_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Slab_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Cached_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Cache - Parked file data (file content) cache", + "refId": "E", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Buffers_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "refId": "F", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemFree_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Unused - Free memory unassigned", + "refId": "G", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\"} - node_memory_SwapFree_bytes{instance=\"$node\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Swap - Swap space used", + "refId": "H", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "refId": "I", + "step": 240 + } + ], + "title": "Memory Stack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 136, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Inactive_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Active_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Active / Inactive", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 430, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Active_file_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Active_anon_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory Active / Inactive Detail", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 128, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "DirectMap1G - Amount of pages mapped as this size", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap2M - Amount of pages mapped as this size", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "DirectMap4K - Amount of pages mapped as this size", + "refId": "C", + "step": 240 + } + ], + "title": "Memory DirectMap", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 135, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Committed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 138, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Mapped_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Shmem_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages", + "refId": "D", + "step": 240 + } + ], + "title": "Memory Shared and Mapped", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 137, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Unevictable_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Mlocked_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Unevictable and MLocked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "KernelStack - Kernel memory stack. This is not reclaimable" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 160, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_KernelStack_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Percpu_bytes{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PerCPU - Per CPU memory allocated dynamically by loadable modules", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Kernel per CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 130, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Writeback_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Writeback - Memory which is actively being written back to disk", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Dirty_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "NFS Unstable - NFS memory blocks waiting to be written into storage", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory Writeback and Dirty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 21 + }, + "id": 131, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Slab_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Slab Total - total size of slab memory ", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Memory Slab", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 21 + }, + "id": 70, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocChunk - Largest contiguous block of vmalloc area which is free", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocTotal - Total size of vmalloc memory area", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Vmalloc", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "=", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Inactive *./" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 21 + }, + "id": 129, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonHugePages - Memory in anonymous huge pages", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_AnonPages_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "AnonPages - Memory in user pages not backed by files", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Anonymous", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 27 + }, + "id": 71, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_HugePages_Total{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages - Total size of the pool of huge pages", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_Hugepagesize_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Hugepagesize - Huge Page size", + "refId": "B", + "step": 240 + } + ], + "title": "Memory HugePages Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 428, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_HugePages_Free{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_HugePages_Surp{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Memory HugePages Counter", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 426, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_memory_Bounce_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Bounce - Memory used for block device bounce buffers", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Memory Bounce", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A" + } + ], + "title": "Memory Meminfo", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 370, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 176, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_vmstat_pgpgin{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesin - Page in operations", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_vmstat_pgpgout{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pagesout - Page out operations", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 22, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_vmstat_pswpin{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpin - Pages swapped in", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_vmstat_pswpout{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pswpout - Pages swapped out", + "refId": "B", + "step": 240 + } + ], + "title": "Memory Pages Swap In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pgfault - Page major and minor fault operations" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 175, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_vmstat_pgfault{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgfault - Page major and minor fault operations", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_vmstat_pgmajfault{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgmajfault - Major page fault operations", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_vmstat_pgfault{instance=\"$node\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Pgminfault - Minor page fault operations", + "refId": "C", + "step": 240 + } + ], + "title": "Memory Page Faults", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 47 + }, + "id": 307, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_vmstat_oom_kill{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "oom killer invocations ", + "refId": "A", + "step": 240 + } + ], + "title": "OOM Killer", + "type": "timeseries" + } + ], + "title": "Memory Vmstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 386, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 29 + }, + "id": 260, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_timex_estimated_error_seconds{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Estimated error in seconds", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_timex_offset_seconds{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Time offset in between local system and reference clock", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_timex_maxerror_seconds{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum error in seconds", + "refId": "C", + "step": 240 + } + ], + "title": "Time Synchronized Drift", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 29 + }, + "id": 291, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_timex_loop_time_constant{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Phase-locked loop time adjust", + "refId": "A", + "step": 240 + } + ], + "title": "Time PLL Adjust", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Variation*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 29 + }, + "id": 168, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_timex_sync_status{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Local clock frequency adjustment", + "refId": "B", + "step": 240 + } + ], + "title": "Time Synchronized Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 294, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_timex_tick_seconds{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Seconds between clock ticks", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_timex_tai_offset_seconds{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "International Atomic Time (TAI) offset", + "refId": "B", + "step": 240 + } + ], + "title": "Time Misc", + "type": "timeseries" + } + ], + "title": "System Time Synchronize", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 376, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 30 + }, + "id": 62, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_procs_running{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Processes in runnable state", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_procs_blocked{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Processes blocked waiting for I/O to complete", + "refId": "C", + "step": 240 + } + ], + "title": "Processes Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 30 + }, + "id": 345, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "process_virtual_memory_bytes{instance=\"$node\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Processes virtual memory size", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "process_resident_memory_max_bytes{instance=\"$node\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum amount of virtual memory available", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "process_virtual_memory_max_bytes{instance=\"$node\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum amount of virtual memory available", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Processes Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 30 + }, + "id": 148, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_forks_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Processes forks second", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Processes Forks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 36 + }, + "id": 379, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_processes_state{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Processes in {{state}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Processes States", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 36 + }, + "id": 378, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_processes_pids{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Processes Pids", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Processes Pid Number", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 36 + }, + "id": 377, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_processes_max_threads{instance=\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum number of threads", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_processes_threads{instance=\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Allocated threads in system", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Threads Number and Limit", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 42 + }, + "id": 380, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_processes_threads_state{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Threads in {{thread_state}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Threads States", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*waiting.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 42 + }, + "id": 388, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_schedstat_running_seconds_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - seconds spent running a process", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_schedstat_waiting_seconds_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }} - seconds spent by processing waiting for this CPU", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Process Schedule Stats Running / Waiting", + "type": "timeseries" + } + ], + "title": "System Processes", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 333, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 7, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_load1{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 1m", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_load5{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 5m", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_load15{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 15m", + "refId": "C", + "step": 240 + } + ], + "title": "System Load", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max*./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 64, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_max_fds{instance=\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Maximum open file descriptors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_open_fds{instance=\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Open file descriptors", + "refId": "B", + "step": 240 + } + ], + "title": "Process File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 8, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_context_switches_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Context switches", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_intr_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Interrupts", + "refId": "B", + "step": 240 + } + ], + "title": "Context Switches / Interrupts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 37 + }, + "id": 306, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_schedstat_timeslices_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{ cpu }}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Schedule timeslices executed by each cpu", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 37 + }, + "id": 390, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(process_cpu_seconds_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Time spend", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Process CPU Time Spend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 37 + }, + "id": 151, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_entropy_available_bits{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Entropy available to random number generators", + "refId": "A", + "step": 240 + } + ], + "title": "Entropy", + "type": "timeseries" + } + ], + "title": "System Misc", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 329, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number (after merges) of I/O requests completed per second for the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_reads_completed_total{instance=\"$node\"}[$__rate_interval])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_writes_completed_total{instance=\"$node\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Completed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of bytes read from or written to the device per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 433, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Written bytes", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of bytes read from or written to the device per IO", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 33, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_read_bytes_total{instance=\"$node\"}[$__rate_interval])/ irate(node_disk_reads_completed_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_written_bytes_total{instance=\"$node\"}[$__rate_interval])/ irate(node_disk_writes_completed_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Written bytes", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Average R/W Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 37, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_read_time_seconds_total{instance=\"$node\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=\"$node\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read wait time avg", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_disk_write_time_seconds_total{instance=\"$node\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=\"$node\"}[$__rate_interval])", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write wait time avg", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Average Wait Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of read and write requests merged per second that were queued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 133, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_reads_merged_total{instance=\"$node\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Read merged", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_writes_merged_total{instance=\"$node\"}[$__rate_interval])", + "intervalFactor": 1, + "legendFormat": "{{device}} - Write merged", + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Merged", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 301, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_discards_completed_total{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - Discards completed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_discards_merged_total{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Discards merged", + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Discards completed / merged", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Percentage of elapsed time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100% for devices serving requests serially. But for devices serving requests in parallel, such as RAID arrays and modern SSDs, this number does not reflect their performance limits.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 36, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_io_time_seconds_total{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_discard_time_seconds_total{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - discard", + "refId": "B", + "step": 240 + } + ], + "title": "Time Spent Doing I/Os", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The average queue length of the requests that were issued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 35, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}", + "refId": "A", + "step": 240 + } + ], + "title": "Average Queue Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 34, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_disk_io_now{instance=\"$node\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO now", + "refId": "A", + "step": 240 + } + ], + "title": "Instantaneous Queue Size", + "type": "timeseries" + } + ], + "title": "Disk", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 372, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 156, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filesystem_size_bytes{instance=\"$node\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "title": "Filesystem Space Used", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 43, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filesystem_avail_bytes{instance=\"$node\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Available", + "metric": "", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filesystem_free_bytes{instance=\"$node\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filesystem_size_bytes{instance=\"$node\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Size", + "refId": "C", + "step": 240 + } + ], + "title": "Filesystem Space Available", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 28, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filefd_maximum{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Max open files", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filefd_allocated{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Open files", + "refId": "B", + "step": 240 + } + ], + "title": "File Descriptor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 219, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_filesystem_files{instance=\"$node\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - File nodes total", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Total Inodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 41, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_filesystem_files_free{instance=\"$node\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Free file nodes", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Free Inodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "/ ReadOnly" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 44, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filesystem_readonly{instance=\"$node\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - ReadOnly", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filesystem_device_error{instance=\"$node\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{mountpoint}} - Device error", + "refId": "B", + "step": 240 + } + ], + "title": "Filesystem in ReadOnly / Error", + "type": "timeseries" + } + ], + "title": "File System", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 272, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 88 + }, + "id": 60, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_receive_packets_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_transmit_packets_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic by Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 88 + }, + "id": 142, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_receive_errs_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive errors", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_network_transmit_errs_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit errors", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 88 + }, + "id": 290, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_softnet_processed_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Processed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_softnet_dropped_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Dropped", + "refId": "B", + "step": 240 + } + ], + "title": "Softnet Packets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 94 + }, + "id": 144, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_receive_fifo_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive fifo", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_transmit_fifo_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit fifo", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Fifo", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 94 + }, + "id": 141, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_receive_compressed_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive compressed", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_transmit_compressed_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit compressed", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Compressed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 94 + }, + "id": 143, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_receive_drop_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive drop", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_transmit_drop_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit drop", + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Drop", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 100 + }, + "id": 232, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_transmit_colls_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Transmit colls", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Colls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 100 + }, + "id": 145, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_receive_frame_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive frame", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Frame", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 100 + }, + "id": 146, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_receive_multicast_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Receive multicast", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Multicast", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 106 + }, + "id": 280, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_network_speed_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Speed", + "refId": "A", + "step": 240 + } + ], + "title": "Speed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "NF conntrack limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 106 + }, + "id": 61, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_nf_conntrack_entries{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack entries - Number of currently allocated flow entries for connection tracking", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_nf_conntrack_entries_limit{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "NF conntrack limit - Maximum size of connection tracking table", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "NF Contrack", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 106 + }, + "id": 231, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_network_transmit_carrier_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}} - Statistic transmit_carrier", + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Carrier", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 112 + }, + "id": 310, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_softnet_times_squeezed_total{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "CPU {{cpu}} - Squeezed", + "refId": "A", + "step": 240 + } + ], + "title": "Softnet Out of Quota", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 112 + }, + "id": 289, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_network_transmit_queue_length{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Interface transmit queue length", + "refId": "A", + "step": 240 + } + ], + "title": "Queue Length", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 112 + }, + "id": 424, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_arp_entries{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - ARP entries", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "ARP Entries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 118 + }, + "id": 309, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_network_up{operstate=\"up\",instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{interface}} - Operational state UP", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_network_carrier{instance=\"$node\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link state", + "refId": "B" + } + ], + "title": "Network Operational Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 118 + }, + "id": 288, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_network_mtu_bytes{instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ device }} - Bytes", + "refId": "A", + "step": 240 + } + ], + "title": "MTU", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A" + } + ], + "title": "Network Traffic", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 374, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 125 + }, + "id": 394, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_sockstat_TCP_alloc{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_alloc - Allocated sockets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_TCP_inuse{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_TCP_mem{instance=\"$node\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_mem - Used memory for tcp", + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_TCP_orphan{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_orphan - Orphan sockets", + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_TCP_tw{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_tw - Sockets waiting close", + "refId": "E", + "step": 240 + } + ], + "title": "Sockstat TCP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 125 + }, + "id": 396, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_UDP_inuse{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_inuse - Udp sockets currently in use", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_UDP_mem{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_mem - Used memory for udp", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat UDP", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 125 + }, + "id": 392, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_inuse - Frag sockets currently in use", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_RAW_inuse{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RAW_inuse - Raw sockets currently in use", + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat FRAG / RAW", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 131 + }, + "id": 220, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - TCP sockets in that state", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - UDP sockets in that state", + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_FRAG_memory{instance=\"$node\"}", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_memory - Used memory for frag", + "refId": "C" + } + ], + "title": "Sockstat Memory Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 131 + }, + "id": 126, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_sockstat_sockets_used{instance=\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Sockets_used - Sockets currently in use", + "refId": "A", + "step": 240 + } + ], + "title": "Sockstat Used", + "type": "timeseries" + } + ], + "title": "Network Sockstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 398, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 138 + }, + "id": 406, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Ip_Forwarding{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Forwarding - IP forwarding", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Netstat IP Forwarding", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 138 + }, + "id": 412, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP Direct Transition", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*MaxConn *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 138 + }, + "id": 410, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dynamic is \"-1\")", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 144 + }, + "id": 416, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_InSegs{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_OutSegs{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 144 + }, + "id": 402, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_InMsgs{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "refId": "B", + "step": 240 + } + ], + "title": "ICMP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 144 + }, + "id": 422, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Udp_InDatagrams{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InDatagrams - Datagrams received", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutDatagrams - Datagrams sent", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "UDP In / Out", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "octets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 150 + }, + "id": 408, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_IpExt_InOctets{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InOctets - Received octets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_IpExt_OutOctets{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "OutOctets - Sent octets", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Netstat IP In / Out Octets", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 150 + }, + "id": 414, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_InErrs{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "legendFormat": "OutRsts - Segments sent with RST flag", + "range": true, + "refId": "F" + } + ], + "title": "TCP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 150 + }, + "id": 400, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Icmp_InErrors{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "ICMP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 156 + }, + "id": 420, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Udp_InErrors{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Udp_NoPorts{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_UdpLite_InErrors{instance=\"$node\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RcvbufErrors - UDP buffer errors received", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SndbufErrors - UDP buffer errors send", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "UDP Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Sent.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 156 + }, + "id": 418, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesRecv - SYN cookies received", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesSent - SYN cookies sent", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "TCP SynCookie", + "type": "timeseries" + } + ], + "title": "Network Netstat", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 363, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": -8, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*error.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 139 + }, + "id": 365, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_scrape_collector_success{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape success", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_textfile_scrape_error{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape textfile error (1 = true)", + "refId": "B", + "step": 240 + } + ], + "title": "Node Exporter Scrape", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 139 + }, + "id": 367, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{collector}} - Scrape duration", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Node Exporter Scrape Time", + "type": "timeseries" + } + ], + "title": "Node Exporter", + "type": "row" + } + ], + "refresh": false, + "revision": 1, + "schemaVersion": 37, + "style": "dark", + "tags": [ + "linux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "192.168.49.2:9100", + "value": "192.168.49.2:9100" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(node_uname_info{nodename=\"$Node\"}, instance)", + "hide": 2, + "includeAll": false, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(node_uname_info{nodename=\"$Node\"}, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "diskdevices", + "options": [ + { + "selected": true, + "text": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "value": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+" + } + ], + "query": "[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DataSource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "minikube", + "value": "minikube" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(node_uname_info{}, nodename)", + "hide": 0, + "includeAll": false, + "label": "node", + "multi": false, + "name": "Node", + "options": [], + "query": { + "query": "label_values(node_uname_info{}, nodename)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Node Exporter", + "uid": "nodeexporter1af4132", + "version": 1, + "weekStart": "" +} \ No newline at end of file From 7ca48c3db78ec969e9cbd206bad648f6be7710f3 Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Tue, 16 May 2023 19:14:46 +0800 Subject: [PATCH 307/439] chore: update chaos-mesh (#3260) --- deploy/helm/templates/addons/chaos-mesh-addon.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/deploy/helm/templates/addons/chaos-mesh-addon.yaml b/deploy/helm/templates/addons/chaos-mesh-addon.yaml index 7adc40b49..44e5d3517 100644 --- a/deploy/helm/templates/addons/chaos-mesh-addon.yaml +++ b/deploy/helm/templates/addons/chaos-mesh-addon.yaml @@ -15,7 +15,16 @@ spec: type: Helm helm: - chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/chaos-mesh-0.1.0-beta.0.tgz + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/chaos-mesh-2.5.2.tgz + + installValues: + setValues: + - "version=2.5.2" + - "chaosDaemon.privileged=true" + - "dnsServer.create=true" + - "chaosDaemon.runtime=containerd" + - "chaosDaemon.socketPath=/run/containerd/containerd.sock" + valuesMapping: valueMap: replicaCount: controllerManager.replicaCount From 00a196b3452bafd259c5e5638b190d205c4694ef Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Tue, 16 May 2023 20:04:22 +0800 Subject: [PATCH 308/439] feat: support cli fault pod (#3101) Co-authored-by: huyongqii --- docs/user_docs/cli/cli.md | 1 + ...kbcli_clusterdefinition_list-components.md | 53 +++++ docs/user_docs/cli/kbcli_fault.md | 1 + .../cli/kbcli_fault_network_bandwidth.md | 59 +++--- .../cli/kbcli_fault_network_corrupt.md | 53 ++--- .../cli/kbcli_fault_network_delay.md | 55 +++--- .../cli/kbcli_fault_network_dns_error.md | 26 +-- .../cli/kbcli_fault_network_dns_random.md | 26 +-- .../cli/kbcli_fault_network_duplicate.md | 53 ++--- .../cli/kbcli_fault_network_http_abort.md | 34 ++-- .../cli/kbcli_fault_network_http_delay.md | 34 ++-- .../cli/kbcli_fault_network_http_patch.md | 36 ++-- .../cli/kbcli_fault_network_http_replace.md | 38 ++-- .../user_docs/cli/kbcli_fault_network_loss.md | 53 ++--- .../cli/kbcli_fault_network_partition.md | 49 +++-- docs/user_docs/cli/kbcli_fault_pod.md | 45 +++++ docs/user_docs/cli/kbcli_fault_pod_failure.md | 94 +++++++++ .../cli/kbcli_fault_pod_kill-container.md | 95 +++++++++ docs/user_docs/cli/kbcli_fault_pod_kill.md | 95 +++++++++ internal/cli/cmd/fault/fault.go | 48 ++++- internal/cli/cmd/fault/fault_constant.go | 4 +- internal/cli/cmd/fault/fault_dns.go | 32 ++- internal/cli/cmd/fault/fault_dns_test.go | 93 +++++++++ internal/cli/cmd/fault/fault_http.go | 31 +-- internal/cli/cmd/fault/fault_http_test.go | 139 +++++++++++++ internal/cli/cmd/fault/fault_network.go | 57 +++--- internal/cli/cmd/fault/fault_network_test.go | 184 +++++++++++++++++ internal/cli/cmd/fault/fault_pod.go | 186 ++++++++++++++++++ internal/cli/cmd/fault/fault_pod_test.go | 96 +++++++++ internal/cli/cmd/fault/suite_test.go | 32 +++ .../create/template/dns_chaos_template.cue | 15 +- .../create/template/http_chaos_template.cue | 12 +- .../template/network_chaos_template.cue | 27 +-- .../create/template/pod_chaos_template.cue | 49 +++++ 34 files changed, 1562 insertions(+), 343 deletions(-) create mode 100644 docs/user_docs/cli/kbcli_clusterdefinition_list-components.md create mode 100644 docs/user_docs/cli/kbcli_fault_pod.md create mode 100644 docs/user_docs/cli/kbcli_fault_pod_failure.md create mode 100644 docs/user_docs/cli/kbcli_fault_pod_kill-container.md create mode 100644 docs/user_docs/cli/kbcli_fault_pod_kill.md create mode 100644 internal/cli/cmd/fault/fault_dns_test.go create mode 100644 internal/cli/cmd/fault/fault_http_test.go create mode 100644 internal/cli/cmd/fault/fault_network_test.go create mode 100644 internal/cli/cmd/fault/fault_pod.go create mode 100644 internal/cli/cmd/fault/fault_pod_test.go create mode 100644 internal/cli/cmd/fault/suite_test.go create mode 100644 internal/cli/create/template/pod_chaos_template.cue diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index f904a7e7c..c3eb3de88 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -113,6 +113,7 @@ List and open the KubeBlocks dashboards. Inject faults to pod. * [kbcli fault network](kbcli_fault_network.md) - Network chaos. +* [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. ## [kubeblocks](kbcli_kubeblocks.md) diff --git a/docs/user_docs/cli/kbcli_clusterdefinition_list-components.md b/docs/user_docs/cli/kbcli_clusterdefinition_list-components.md new file mode 100644 index 000000000..49123f3ae --- /dev/null +++ b/docs/user_docs/cli/kbcli_clusterdefinition_list-components.md @@ -0,0 +1,53 @@ +--- +title: kbcli clusterdefinition list-components +--- + +List cluster definition components. + +``` +kbcli clusterdefinition list-components [flags] +``` + +### Examples + +``` + # List all components belong to the cluster definition. + kbcli clusterdefinition list-components apecloud-mysql +``` + +### Options + +``` + -h, --help help for list-components +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli clusterdefinition](kbcli_clusterdefinition.md) - ClusterDefinition command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault.md b/docs/user_docs/cli/kbcli_fault.md index 63119b47d..895fde1c2 100644 --- a/docs/user_docs/cli/kbcli_fault.md +++ b/docs/user_docs/cli/kbcli_fault.md @@ -38,6 +38,7 @@ Inject faults to pod. * [kbcli fault network](kbcli_fault_network.md) - Network chaos. +* [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/docs/user_docs/cli/kbcli_fault_network_bandwidth.md b/docs/user_docs/cli/kbcli_fault_network_bandwidth.md index 5777a4711..d4e73f28f 100644 --- a/docs/user_docs/cli/kbcli_fault_network_bandwidth.md +++ b/docs/user_docs/cli/kbcli_fault_network_bandwidth.md @@ -15,57 +15,64 @@ kbcli fault network bandwidth [flags] kbcli fault network partition # The specified pod is isolated from the k8s external network "kubeblocks.io". - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + kbcli fault network partition mycluster-mysql-1 --external-targets=kubeblocks.io # Isolate the network between two pods. - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + kbcli fault network partition mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. kbcli fault network loss --loss=50 # Block the specified pod communication, so that the packet loss rate is 50%. - kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + kbcli fault network loss mysql-cluster-mysql-2 --loss=50 kbcli fault network corrupt --corrupt=50 # Blocks specified pod communication with a 50% packet corruption rate. - kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + kbcli fault network corrupt mysql-cluster-mysql-2 --corrupt=50 kbcli fault network duplicate --duplicate=50 # Block specified pod communication so that the packet repetition rate is 50%. - kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + kbcli fault network duplicate mysql-cluster-mysql-2 --duplicate=50 kbcli fault network delay --latency=10s # Block the communication of the specified pod, causing its network delay for 10s. - kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s + kbcli fault network delay mysql-cluster-mysql-2 --latency=10s + + # Limit the communication bandwidth between mysql-cluster-mysql-2 and the outside. + kbcli fault network bandwidth mysql-cluster-mysql-2 --rate=1kbps --duration=1m ``` ### Options ``` - --buffer uint32 the maximum number of bytes that can be sent instantaneously. (default 1) - --direction string You can select "to"" or "from"" or "both"". (default "to") - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, - such as "www.baidu.com". Only works with direction: to. - -h, --help help for bandwidth - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --limit uint32 the number of bytes waiting in the queue. (default 1) - --minburst uint32 the size of the peakrate bucket. - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --peakrate uint the maximum consumption rate of the bucket. - --rate string the rate at which the bandwidth is limited. For example : 10 bps/kbps/mbps/gbps. - --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") - --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --buffer uint32 the maximum number of bytes that can be sent instantaneously. (default 1) + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-target stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for bandwidth + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --limit uint32 the number of bytes waiting in the queue. (default 1) + --minburst uint32 the size of the peakrate bucket. + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --peakrate uint the maximum consumption rate of the bucket. + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --rate string the rate at which the bandwidth is limited. For example : 10 bps/kbps/mbps/gbps. + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_corrupt.md b/docs/user_docs/cli/kbcli_fault_network_corrupt.md index 6559660d3..968e0cd97 100644 --- a/docs/user_docs/cli/kbcli_fault_network_corrupt.md +++ b/docs/user_docs/cli/kbcli_fault_network_corrupt.md @@ -15,54 +15,61 @@ kbcli fault network corrupt [flags] kbcli fault network partition # The specified pod is isolated from the k8s external network "kubeblocks.io". - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + kbcli fault network partition mycluster-mysql-1 --external-targets=kubeblocks.io # Isolate the network between two pods. - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + kbcli fault network partition mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. kbcli fault network loss --loss=50 # Block the specified pod communication, so that the packet loss rate is 50%. - kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + kbcli fault network loss mysql-cluster-mysql-2 --loss=50 kbcli fault network corrupt --corrupt=50 # Blocks specified pod communication with a 50% packet corruption rate. - kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + kbcli fault network corrupt mysql-cluster-mysql-2 --corrupt=50 kbcli fault network duplicate --duplicate=50 # Block specified pod communication so that the packet repetition rate is 50%. - kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + kbcli fault network duplicate mysql-cluster-mysql-2 --duplicate=50 kbcli fault network delay --latency=10s # Block the communication of the specified pod, causing its network delay for 10s. - kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s + kbcli fault network delay mysql-cluster-mysql-2 --latency=10s + + # Limit the communication bandwidth between mysql-cluster-mysql-2 and the outside. + kbcli fault network bandwidth mysql-cluster-mysql-2 --rate=1kbps --duration=1m ``` ### Options ``` - -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") - --corrupt string Indicates the probability of a packet error occurring. Value range: [0, 100]. - --direction string You can select "to"" or "from"" or "both"". (default "to") - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, - such as "www.baidu.com". Only works with direction: to. - -h, --help help for corrupt - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") - --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + --corrupt string Indicates the probability of a packet error occurring. Value range: [0, 100]. + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-target stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for corrupt + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_delay.md b/docs/user_docs/cli/kbcli_fault_network_delay.md index 9e7ceb3b0..21ec9fd4a 100644 --- a/docs/user_docs/cli/kbcli_fault_network_delay.md +++ b/docs/user_docs/cli/kbcli_fault_network_delay.md @@ -15,55 +15,62 @@ kbcli fault network delay [flags] kbcli fault network partition # The specified pod is isolated from the k8s external network "kubeblocks.io". - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + kbcli fault network partition mycluster-mysql-1 --external-targets=kubeblocks.io # Isolate the network between two pods. - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + kbcli fault network partition mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. kbcli fault network loss --loss=50 # Block the specified pod communication, so that the packet loss rate is 50%. - kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + kbcli fault network loss mysql-cluster-mysql-2 --loss=50 kbcli fault network corrupt --corrupt=50 # Blocks specified pod communication with a 50% packet corruption rate. - kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + kbcli fault network corrupt mysql-cluster-mysql-2 --corrupt=50 kbcli fault network duplicate --duplicate=50 # Block specified pod communication so that the packet repetition rate is 50%. - kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + kbcli fault network duplicate mysql-cluster-mysql-2 --duplicate=50 kbcli fault network delay --latency=10s # Block the communication of the specified pod, causing its network delay for 10s. - kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s + kbcli fault network delay mysql-cluster-mysql-2 --latency=10s + + # Limit the communication bandwidth between mysql-cluster-mysql-2 and the outside. + kbcli fault network bandwidth mysql-cluster-mysql-2 --rate=1kbps --duration=1m ``` ### Options ``` - -c, --correlation string Indicates the probability of a packet error occurring. Value range: [0, 100]. (default "0") - --direction string You can select "to"" or "from"" or "both"". (default "to") - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, - such as "www.baidu.com". Only works with direction: to. - -h, --help help for delay - --jitter string the variation range of the delay time. (default "0ms") - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --latency string the length of time to delay. - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") - --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --correlation string Indicates the probability of a packet error occurring. Value range: [0, 100]. (default "0") + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-target stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for delay + --jitter string the variation range of the delay time. (default "0ms") + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --latency string the length of time to delay. + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_dns_error.md b/docs/user_docs/cli/kbcli_fault_network_dns_error.md index eac118792..4c040b2c8 100644 --- a/docs/user_docs/cli/kbcli_fault_network_dns_error.md +++ b/docs/user_docs/cli/kbcli_fault_network_dns_error.md @@ -12,24 +12,28 @@ kbcli fault network dns error [flags] ``` // Inject DNS faults into all pods under the default namespace, so that any IP is returned when accessing the baidu.com domain name. - kbcli fault DNS random --patterns=baidu.com --duration=1m + kbcli fault dns random --patterns=baidu.com --duration=1m // Inject DNS faults into all pods under the default namespace, so that error is returned when accessing the baidu.com domain name. - kbcli fault DNS error --patterns=baidu.com --duration=1m + kbcli fault dns error --patterns=baidu.com --duration=1m ``` ### Options ``` - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -h, --help help for error - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --patterns stringArray Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for error + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --patterns stringArray Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *. + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_dns_random.md b/docs/user_docs/cli/kbcli_fault_network_dns_random.md index ca57706b3..d091ab4b7 100644 --- a/docs/user_docs/cli/kbcli_fault_network_dns_random.md +++ b/docs/user_docs/cli/kbcli_fault_network_dns_random.md @@ -12,24 +12,28 @@ kbcli fault network dns random [flags] ``` // Inject DNS faults into all pods under the default namespace, so that any IP is returned when accessing the baidu.com domain name. - kbcli fault DNS random --patterns=baidu.com --duration=1m + kbcli fault dns random --patterns=baidu.com --duration=1m // Inject DNS faults into all pods under the default namespace, so that error is returned when accessing the baidu.com domain name. - kbcli fault DNS error --patterns=baidu.com --duration=1m + kbcli fault dns error --patterns=baidu.com --duration=1m ``` ### Options ``` - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -h, --help help for random - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --patterns stringArray Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for random + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --patterns stringArray Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *. + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_duplicate.md b/docs/user_docs/cli/kbcli_fault_network_duplicate.md index 0eb3cb8b7..41e96e07e 100644 --- a/docs/user_docs/cli/kbcli_fault_network_duplicate.md +++ b/docs/user_docs/cli/kbcli_fault_network_duplicate.md @@ -15,54 +15,61 @@ kbcli fault network duplicate [flags] kbcli fault network partition # The specified pod is isolated from the k8s external network "kubeblocks.io". - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + kbcli fault network partition mycluster-mysql-1 --external-targets=kubeblocks.io # Isolate the network between two pods. - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + kbcli fault network partition mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. kbcli fault network loss --loss=50 # Block the specified pod communication, so that the packet loss rate is 50%. - kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + kbcli fault network loss mysql-cluster-mysql-2 --loss=50 kbcli fault network corrupt --corrupt=50 # Blocks specified pod communication with a 50% packet corruption rate. - kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + kbcli fault network corrupt mysql-cluster-mysql-2 --corrupt=50 kbcli fault network duplicate --duplicate=50 # Block specified pod communication so that the packet repetition rate is 50%. - kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + kbcli fault network duplicate mysql-cluster-mysql-2 --duplicate=50 kbcli fault network delay --latency=10s # Block the communication of the specified pod, causing its network delay for 10s. - kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s + kbcli fault network delay mysql-cluster-mysql-2 --latency=10s + + # Limit the communication bandwidth between mysql-cluster-mysql-2 and the outside. + kbcli fault network bandwidth mysql-cluster-mysql-2 --rate=1kbps --duration=1m ``` ### Options ``` - -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") - --direction string You can select "to"" or "from"" or "both"". (default "to") - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duplicate string the probability of a packet being repeated. Value range: [0, 100]. - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, - such as "www.baidu.com". Only works with direction: to. - -h, --help help for duplicate - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") - --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duplicate string the probability of a packet being repeated. Value range: [0, 100]. + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-target stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for duplicate + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_http_abort.md b/docs/user_docs/cli/kbcli_fault_network_http_abort.md index b13322843..1a19abf27 100644 --- a/docs/user_docs/cli/kbcli_fault_network_http_abort.md +++ b/docs/user_docs/cli/kbcli_fault_network_http_abort.md @@ -32,27 +32,31 @@ kbcli fault network http abort [flags] # Replace the response content "you" from port 80. kbcli fault network http replace --target=Response --body=you --duration=30s - # AAppend content to the body of the post request sent from port 4399, in JSON format. + # Append content to the body of the post request sent from port 4399, in JSON format. kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s ``` ### Options ``` - --abort Indicates whether to inject the fault that interrupts the connection. (default true) - --code int32 The status code responded by target. - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -h, --help help for abort - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --path string The URI path of the target request. Supports Matching wildcards. (default "*") - --port int32 The TCP port that the target service listens on. (default 80) - --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --abort Indicates whether to inject the fault that interrupts the connection. (default true) + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --code int32 The status code responded by target. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for abort + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The URI path of the target request. Supports Matching wildcards. (default "*") + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --port int32 The TCP port that the target service listens on. (default 80) + --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_http_delay.md b/docs/user_docs/cli/kbcli_fault_network_http_delay.md index e8c8d5bcd..baf38f4ff 100644 --- a/docs/user_docs/cli/kbcli_fault_network_http_delay.md +++ b/docs/user_docs/cli/kbcli_fault_network_http_delay.md @@ -32,27 +32,31 @@ kbcli fault network http delay [flags] # Replace the response content "you" from port 80. kbcli fault network http replace --target=Response --body=you --duration=30s - # AAppend content to the body of the post request sent from port 4399, in JSON format. + # Append content to the body of the post request sent from port 4399, in JSON format. kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s ``` ### Options ``` - --code int32 The status code responded by target. - --delay string The time for delay. (default "10s") - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -h, --help help for delay - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --path string The URI path of the target request. Supports Matching wildcards. (default "*") - --port int32 The TCP port that the target service listens on. (default 80) - --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --code int32 The status code responded by target. + --delay string The time for delay. (default "10s") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for delay + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The URI path of the target request. Supports Matching wildcards. (default "*") + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --port int32 The TCP port that the target service listens on. (default 80) + --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_http_patch.md b/docs/user_docs/cli/kbcli_fault_network_http_patch.md index 93b520404..73feead72 100644 --- a/docs/user_docs/cli/kbcli_fault_network_http_patch.md +++ b/docs/user_docs/cli/kbcli_fault_network_http_patch.md @@ -32,28 +32,32 @@ kbcli fault network http patch [flags] # Replace the response content "you" from port 80. kbcli fault network http replace --target=Response --body=you --duration=30s - # AAppend content to the body of the post request sent from port 4399, in JSON format. + # Append content to the body of the post request sent from port 4399, in JSON format. kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s ``` ### Options ``` - --body string The fault of the request body or response body with patch faults. - --code int32 The status code responded by target. - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -h, --help help for patch - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --path string The URI path of the target request. Supports Matching wildcards. (default "*") - --port int32 The TCP port that the target service listens on. (default 80) - --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") - --type string The type of patch faults of the request body or response body. Currently, it only supports JSON. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --body string The fault of the request body or response body with patch faults. + --code int32 The status code responded by target. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for patch + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The URI path of the target request. Supports Matching wildcards. (default "*") + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --port int32 The TCP port that the target service listens on. (default 80) + --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") + --type string The type of patch faults of the request body or response body. Currently, it only supports JSON. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_http_replace.md b/docs/user_docs/cli/kbcli_fault_network_http_replace.md index 9e3a7b6fb..9b2580dbd 100644 --- a/docs/user_docs/cli/kbcli_fault_network_http_replace.md +++ b/docs/user_docs/cli/kbcli_fault_network_http_replace.md @@ -32,29 +32,33 @@ kbcli fault network http replace [flags] # Replace the response content "you" from port 80. kbcli fault network http replace --target=Response --body=you --duration=30s - # AAppend content to the body of the post request sent from port 4399, in JSON format. + # Append content to the body of the post request sent from port 4399, in JSON format. kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s ``` ### Options ``` - --body string The content of the request body or response body to replace the failure. - --code int32 The status code responded by target. - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -h, --help help for replace - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --path string The URI path of the target request. Supports Matching wildcards. (default "*") - --port int32 The TCP port that the target service listens on. (default 80) - --replace-method string The replaced content of the HTTP request method. - --replace-path string The URI path used to replace content. - --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --body string The content of the request body or response body to replace the failure. + --code int32 The status code responded by target. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for replace + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --method string The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH. (default "GET") + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The URI path of the target request. Supports Matching wildcards. (default "*") + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --port int32 The TCP port that the target service listens on. (default 80) + --replace-method string The replaced content of the HTTP request method. + --replace-path string The URI path used to replace content. + --target string Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time. (default "Request") + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_loss.md b/docs/user_docs/cli/kbcli_fault_network_loss.md index dbaaf543b..2bcd55085 100644 --- a/docs/user_docs/cli/kbcli_fault_network_loss.md +++ b/docs/user_docs/cli/kbcli_fault_network_loss.md @@ -15,54 +15,61 @@ kbcli fault network loss [flags] kbcli fault network partition # The specified pod is isolated from the k8s external network "kubeblocks.io". - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + kbcli fault network partition mycluster-mysql-1 --external-targets=kubeblocks.io # Isolate the network between two pods. - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + kbcli fault network partition mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. kbcli fault network loss --loss=50 # Block the specified pod communication, so that the packet loss rate is 50%. - kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + kbcli fault network loss mysql-cluster-mysql-2 --loss=50 kbcli fault network corrupt --corrupt=50 # Blocks specified pod communication with a 50% packet corruption rate. - kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + kbcli fault network corrupt mysql-cluster-mysql-2 --corrupt=50 kbcli fault network duplicate --duplicate=50 # Block specified pod communication so that the packet repetition rate is 50%. - kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + kbcli fault network duplicate mysql-cluster-mysql-2 --duplicate=50 kbcli fault network delay --latency=10s # Block the communication of the specified pod, causing its network delay for 10s. - kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s + kbcli fault network delay mysql-cluster-mysql-2 --latency=10s + + # Limit the communication bandwidth between mysql-cluster-mysql-2 and the outside. + kbcli fault network bandwidth mysql-cluster-mysql-2 --rate=1kbps --duration=1m ``` ### Options ``` - -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") - --direction string You can select "to"" or "from"" or "both"". (default "to") - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, - such as "www.baidu.com". Only works with direction: to. - -h, --help help for loss - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --loss string Indicates the probability of a packet error occurring. Value range: [0, 100]. - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") - --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-target stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for loss + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --loss string Indicates the probability of a packet error occurring. Value range: [0, 100]. + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_network_partition.md b/docs/user_docs/cli/kbcli_fault_network_partition.md index 01e30ddb0..b655dda15 100644 --- a/docs/user_docs/cli/kbcli_fault_network_partition.md +++ b/docs/user_docs/cli/kbcli_fault_network_partition.md @@ -15,52 +15,59 @@ kbcli fault network partition [flags] kbcli fault network partition # The specified pod is isolated from the k8s external network "kubeblocks.io". - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + kbcli fault network partition mycluster-mysql-1 --external-targets=kubeblocks.io # Isolate the network between two pods. - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + kbcli fault network partition mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. kbcli fault network loss --loss=50 # Block the specified pod communication, so that the packet loss rate is 50%. - kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + kbcli fault network loss mysql-cluster-mysql-2 --loss=50 kbcli fault network corrupt --corrupt=50 # Blocks specified pod communication with a 50% packet corruption rate. - kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + kbcli fault network corrupt mysql-cluster-mysql-2 --corrupt=50 kbcli fault network duplicate --duplicate=50 # Block specified pod communication so that the packet repetition rate is 50%. - kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + kbcli fault network duplicate mysql-cluster-mysql-2 --duplicate=50 kbcli fault network delay --latency=10s # Block the communication of the specified pod, causing its network delay for 10s. - kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s + kbcli fault network delay mysql-cluster-mysql-2 --latency=10s + + # Limit the communication bandwidth between mysql-cluster-mysql-2 and the outside. + kbcli fault network bandwidth mysql-cluster-mysql-2 --rate=1kbps --duration=1m ``` ### Options ``` - --direction string You can select "to"" or "from"" or "both"". (default "to") - --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") - -e, --external-targets stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, - such as "www.baidu.com". Only works with direction: to. - -h, --help help for partition - --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --namespace-selector stringArray Specifies the namespace into which you want to inject faults. (default [default]) - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) - --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") - --target-namespace-selector string Specifies the namespace into which you want to inject faults. (default "default") - --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. - --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --direction string You can select "to"" or "from"" or "both"". (default "to") + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -e, --external-target stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, + such as "www.baidu.com". Only works with direction: to. + -h, --help help for partition + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) + --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_pod.md b/docs/user_docs/cli/kbcli_fault_pod.md new file mode 100644 index 000000000..144789ad5 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_pod.md @@ -0,0 +1,45 @@ +--- +title: kbcli fault pod +--- + +Pod chaos. + +### Options + +``` + -h, --help help for pod +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault](kbcli_fault.md) - Inject faults to pod. +* [kbcli fault pod failure](kbcli_fault_pod_failure.md) - failure pod +* [kbcli fault pod kill](kbcli_fault_pod_kill.md) - kill pod +* [kbcli fault pod kill-container](kbcli_fault_pod_kill-container.md) - kill containers + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_pod_failure.md b/docs/user_docs/cli/kbcli_fault_pod_failure.md new file mode 100644 index 000000000..9ce43a8c7 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_pod_failure.md @@ -0,0 +1,94 @@ +--- +title: kbcli fault pod failure +--- + +failure pod + +``` +kbcli fault pod failure [flags] +``` + +### Examples + +``` + # kill all pods in default namespace + kbcli fault pod kill + + # kill any pod in default namespace + kbcli fault pod kill --mode=one + + # kill two pods in default namespace + kbcli fault pod kill --mode=fixed --value=2 + + # kill 50% pods in default namespace + kbcli fault pod kill --mode=percentage --value=50 + + # kill mysql-cluster-mysql-0 pod in default namespace + kbcli fault pod kill mysql-cluster-mysql-0 + + # kill all pods in default namespace + kbcli fault pod kill --ns-fault="default" + + # --label is required to specify the pods that need to be killed. + kbcli fault pod kill --label statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 + + # kill pod under the specified node. + kbcli fault pod kill --node=minikube-m02 + + # kill pod under the specified node-label. + kbcli fault pod kill --node-label=kubernetes.io/arch=arm64 + + # Allow the experiment to last for one minute. + kbcli fault pod failure --duration=1m + + # kill container in pod + kbcli fault pod kill-container mysql-cluster-mysql-0 --container=mysql +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for failure + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_pod_kill-container.md b/docs/user_docs/cli/kbcli_fault_pod_kill-container.md new file mode 100644 index 000000000..200d61151 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_pod_kill-container.md @@ -0,0 +1,95 @@ +--- +title: kbcli fault pod kill-container +--- + +kill containers + +``` +kbcli fault pod kill-container [flags] +``` + +### Examples + +``` + # kill all pods in default namespace + kbcli fault pod kill + + # kill any pod in default namespace + kbcli fault pod kill --mode=one + + # kill two pods in default namespace + kbcli fault pod kill --mode=fixed --value=2 + + # kill 50% pods in default namespace + kbcli fault pod kill --mode=percentage --value=50 + + # kill mysql-cluster-mysql-0 pod in default namespace + kbcli fault pod kill mysql-cluster-mysql-0 + + # kill all pods in default namespace + kbcli fault pod kill --ns-fault="default" + + # --label is required to specify the pods that need to be killed. + kbcli fault pod kill --label statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 + + # kill pod under the specified node. + kbcli fault pod kill --node=minikube-m02 + + # kill pod under the specified node-label. + kbcli fault pod kill --node-label=kubernetes.io/arch=arm64 + + # Allow the experiment to last for one minute. + kbcli fault pod failure --duration=1m + + # kill container in pod + kbcli fault pod kill-container mysql-cluster-mysql-0 --container=mysql +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --container stringArray the name of the container you want to kill, such as mysql, prometheus. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for kill-container + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_pod_kill.md b/docs/user_docs/cli/kbcli_fault_pod_kill.md new file mode 100644 index 000000000..210cda8d2 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_pod_kill.md @@ -0,0 +1,95 @@ +--- +title: kbcli fault pod kill +--- + +kill pod + +``` +kbcli fault pod kill [flags] +``` + +### Examples + +``` + # kill all pods in default namespace + kbcli fault pod kill + + # kill any pod in default namespace + kbcli fault pod kill --mode=one + + # kill two pods in default namespace + kbcli fault pod kill --mode=fixed --value=2 + + # kill 50% pods in default namespace + kbcli fault pod kill --mode=percentage --value=50 + + # kill mysql-cluster-mysql-0 pod in default namespace + kbcli fault pod kill mysql-cluster-mysql-0 + + # kill all pods in default namespace + kbcli fault pod kill --ns-fault="default" + + # --label is required to specify the pods that need to be killed. + kbcli fault pod kill --label statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 + + # kill pod under the specified node. + kbcli fault pod kill --node=minikube-m02 + + # kill pod under the specified node-label. + kbcli fault pod kill --node-label=kubernetes.io/arch=arm64 + + # Allow the experiment to last for one minute. + kbcli fault pod failure --duration=1m + + # kill container in pod + kbcli fault pod kill-container mysql-cluster-mysql-0 --container=mysql +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -g, --grace-period int Grace period represents the duration in seconds before the pod should be killed + -h, --help help for kill + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/internal/cli/cmd/fault/fault.go b/internal/cli/cmd/fault/fault.go index 2ee579a84..e889f9adc 100644 --- a/internal/cli/cmd/fault/fault.go +++ b/internal/cli/cmd/fault/fault.go @@ -30,9 +30,27 @@ import ( "k8s.io/cli-runtime/pkg/genericclioptions" cmdutil "k8s.io/kubectl/pkg/cmd/util" + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/util" ) +type Selector struct { + PodNameSelectors map[string][]string `json:"pods"` + + NamespaceSelectors []string `json:"namespaces"` + + LabelSelectors map[string]string `json:"labelSelectors"` + + PodPhaseSelectors []string `json:"podPhaseSelectors"` + + NodeLabelSelectors map[string]string `json:"nodeSelectors"` + + AnnotationSelectors map[string]string `json:"annotationSelectors"` + + NodeNameSelectors []string `json:"nodes"` +} + type FaultBaseOptions struct { Action string `json:"action"` @@ -40,11 +58,11 @@ type FaultBaseOptions struct { Value string `json:"value"` - NamespaceSelector []string `json:"namespaceSelector"` + Duration string `json:"duration"` - Label map[string]string `json:"label"` + Selector `json:"selector"` - Duration string `json:"duration"` + create.CreateOptions `json:"-"` } func NewFaultCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { @@ -53,6 +71,7 @@ func NewFaultCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra. Short: "Inject faults to pod.", } cmd.AddCommand( + NewPodChaosCmd(f, streams), NewNetworkChaosCmd(f, streams), ) return cmd @@ -75,6 +94,23 @@ func registerFlagCompletionFunc(cmd *cobra.Command, f cmdutil.Factory) { })) } +func (o *FaultBaseOptions) AddCommonFlag(cmd *cobra.Command) { + cmd.Flags().StringVar(&o.Mode, "mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) + cmd.Flags().StringVar(&o.Value, "value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) + cmd.Flags().StringVar(&o.Duration, "duration", "10s", "Supported formats of the duration are: ms / s / m / h.") + cmd.Flags().StringToStringVar(&o.LabelSelectors, "label", map[string]string{}, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0.`) + cmd.Flags().StringArrayVar(&o.NamespaceSelectors, "ns-fault", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) + cmd.Flags().StringArrayVar(&o.PodPhaseSelectors, "phase", []string{}, `Specify the pod that injects the fault by the state of the pod.`) + cmd.Flags().StringToStringVar(&o.NodeLabelSelectors, "node-label", map[string]string{}, `label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux.`) + cmd.Flags().StringArrayVar(&o.NodeNameSelectors, "node", []string{}, `Inject faults into pods in the specified node.`) + cmd.Flags().StringToStringVar(&o.AnnotationSelectors, "annotation", map[string]string{}, `Select the pod to inject the fault according to Annotation.`) + + cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged + + printer.AddOutputFlagForCreate(cmd, &o.Format) +} + func (o *FaultBaseOptions) BaseValidate() error { if ok, err := IsRegularMatch(o.Duration); !ok { return err @@ -92,6 +128,12 @@ func (o *FaultBaseOptions) BaseValidate() error { } func (o *FaultBaseOptions) BaseComplete() error { + if len(o.Args) > 0 { + o.PodNameSelectors = make(map[string][]string, len(o.NamespaceSelectors)) + for _, ns := range o.NamespaceSelectors { + o.PodNameSelectors[ns] = o.Args + } + } return nil } diff --git a/internal/cli/cmd/fault/fault_constant.go b/internal/cli/cmd/fault/fault_constant.go index 05359993a..a80f5cb44 100644 --- a/internal/cli/cmd/fault/fault_constant.go +++ b/internal/cli/cmd/fault/fault_constant.go @@ -91,8 +91,8 @@ const ( const ( Latency = "latency" LatencyShort = "Delayed IO operations." - Fault = "fault" - FaultShort = "Causes IO operations to return specific errors." + Errno = "errno" + ErrnoShort = "Causes IO operations to return specific errors." Attribute = "attribute" AttributeShort = "Override the attributes of the file." Mistake = "mistake" diff --git a/internal/cli/cmd/fault/fault_dns.go b/internal/cli/cmd/fault/fault_dns.go index 822980d2a..4fa267b79 100644 --- a/internal/cli/cmd/fault/fault_dns.go +++ b/internal/cli/cmd/fault/fault_dns.go @@ -29,35 +29,34 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/apecloud/kubeblocks/internal/cli/create" - "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/util" ) var faultDNSExample = templates.Examples(` // Inject DNS faults into all pods under the default namespace, so that any IP is returned when accessing the baidu.com domain name. - kbcli fault DNS random --patterns=baidu.com --duration=1m + kbcli fault dns random --patterns=baidu.com --duration=1m // Inject DNS faults into all pods under the default namespace, so that error is returned when accessing the baidu.com domain name. - kbcli fault DNS error --patterns=baidu.com --duration=1m + kbcli fault dns error --patterns=baidu.com --duration=1m `) type DNSChaosOptions struct { Patterns []string `json:"patterns"` FaultBaseOptions - - create.CreateOptions `json:"-"` } func NewDNSChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *DNSChaosOptions { o := &DNSChaosOptions{ - CreateOptions: create.CreateOptions{ - Factory: f, - IOStreams: streams, - CueTemplateName: CueTemplateDNSChaos, - GVR: GetGVR(Group, Version, ResourceDNSChaos), + FaultBaseOptions: FaultBaseOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateDNSChaos, + GVR: GetGVR(Group, Version, ResourceDNSChaos), + }, + Action: action, }, - FaultBaseOptions: FaultBaseOptions{Action: action}, } o.CreateOptions.PreCreate = o.PreCreate o.CreateOptions.Options = o @@ -118,18 +117,9 @@ func (o *DNSChaosOptions) NewCobraCommand(use, short string) *cobra.Command { } func (o *DNSChaosOptions) AddCommonFlag(cmd *cobra.Command) { - cmd.Flags().StringVar(&o.Mode, "mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) - cmd.Flags().StringVar(&o.Value, "value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) - cmd.Flags().StringVar(&o.Duration, "duration", "10s", "Supported formats of the duration are: ms / s / m / h.") - cmd.Flags().StringToStringVar(&o.Label, "label", map[string]string{}, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) - cmd.Flags().StringArrayVar(&o.NamespaceSelector, "namespace-selector", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) + o.FaultBaseOptions.AddCommonFlag(cmd) cmd.Flags().StringArrayVar(&o.Patterns, "patterns", nil, `Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *.`) - - cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) - cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged - - printer.AddOutputFlagForCreate(cmd, &o.Format) } func (o *DNSChaosOptions) Validate() error { diff --git a/internal/cli/cmd/fault/fault_dns_test.go b/internal/cli/cmd/fault/fault_dns_test.go new file mode 100644 index 000000000..d3b0029d9 --- /dev/null +++ b/internal/cli/cmd/fault/fault_dns_test.go @@ -0,0 +1,93 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "k8s.io/cli-runtime/pkg/genericclioptions" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" +) + +var _ = Describe("Fault Network DNS", func() { + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + }) + + AfterEach(func() { + tf.Cleanup() + }) + Context("test fault network dns", func() { + + It("fault network dns random", func() { + inputs := [][]string{ + {"--dry-run=client", "--patterns=kubeblocks.io"}, + {"--mode=one", "--patterns=kubeblocks.io", "--dry-run=client"}, + {"--mode=fixed", "--value=2", "--patterns=kubeblocks.io", "--dry-run=client"}, + {"--mode=fixed-percent", "--value=50", "--patterns=kubeblocks.io", "--dry-run=client"}, + {"--mode=random-max-percent", "--value=50", "--patterns=kubeblocks.io", "--dry-run=client"}, + {"--ns-fault=kb-system", "--patterns=kubeblocks.io", "--dry-run=client"}, + {"--node=minikube-m02", "--patterns=kubeblocks.io", "--dry-run=client"}, + {"--label=app.kubernetes.io/component=mysql", "--patterns=kubeblocks.io", "--dry-run=client"}, + {"--node-label=kubernetes.io/arch=arm64", "--patterns=kubeblocks.io", "--dry-run=client"}, + {"--annotation=example-annotation=group-a", "--patterns=kubeblocks.io", "--dry-run=client"}, + } + o := NewDNSChaosOptions(tf, streams, string(v1alpha1.RandomAction)) + cmd := o.NewCobraCommand(Random, RandomShort) + o.AddCommonFlag(cmd) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network dns error", func() { + inputs := [][]string{ + {"--patterns=kubeblocks.io", "--dry-run=client"}, + } + o := NewDNSChaosOptions(tf, streams, string(v1alpha1.ErrorAction)) + cmd := o.NewCobraCommand(Error, ErrorShort) + o.AddCommonFlag(cmd) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + }) +}) diff --git a/internal/cli/cmd/fault/fault_http.go b/internal/cli/cmd/fault/fault_http.go index 9509573af..bdd6f78b6 100644 --- a/internal/cli/cmd/fault/fault_http.go +++ b/internal/cli/cmd/fault/fault_http.go @@ -32,7 +32,6 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/apecloud/kubeblocks/internal/cli/create" - "github.com/apecloud/kubeblocks/internal/cli/printer" ) var faultHTTPExample = templates.Examples(` @@ -57,7 +56,7 @@ var faultHTTPExample = templates.Examples(` # Replace the response content "you" from port 80. kbcli fault network http replace --target=Response --body=you --duration=30s - # AAppend content to the body of the post request sent from port 4399, in JSON format. + # Append content to the body of the post request sent from port 4399, in JSON format. kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s `) @@ -84,19 +83,19 @@ type HTTPChaosOptions struct { PatchBodyType string `json:"patchBodyType,omitempty"` FaultBaseOptions - - create.CreateOptions `json:"-"` } func NewHTTPChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *HTTPChaosOptions { o := &HTTPChaosOptions{ - CreateOptions: create.CreateOptions{ - Factory: f, - IOStreams: streams, - CueTemplateName: CueTemplateHTTPChaos, - GVR: GetGVR(Group, Version, ResourceHTTPChaos), + FaultBaseOptions: FaultBaseOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateHTTPChaos, + GVR: GetGVR(Group, Version, ResourceHTTPChaos), + }, + Action: action, }, - FaultBaseOptions: FaultBaseOptions{Action: action}, } o.CreateOptions.PreCreate = o.PreCreate o.CreateOptions.Options = o @@ -192,23 +191,13 @@ func (o *HTTPChaosOptions) NewCobraCommand(use, short string) *cobra.Command { } func (o *HTTPChaosOptions) AddCommonFlag(cmd *cobra.Command) { - - cmd.Flags().StringVar(&o.Mode, "mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) - cmd.Flags().StringVar(&o.Value, "value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) - cmd.Flags().StringVar(&o.Duration, "duration", "10s", "Supported formats of the duration are: ms / s / m / h.") - cmd.Flags().StringToStringVar(&o.Label, "label", map[string]string{}, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) - cmd.Flags().StringArrayVar(&o.NamespaceSelector, "namespace-selector", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) + o.FaultBaseOptions.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Target, "target", "Request", `Specifies whether the target of fault injuection is Request or Response. The target-related fields should be configured at the same time.`) cmd.Flags().Int32Var(&o.Port, "port", 80, `The TCP port that the target service listens on.`) cmd.Flags().StringVar(&o.Path, "path", "*", `The URI path of the target request. Supports Matching wildcards.`) cmd.Flags().StringVar(&o.Method, "method", "GET", `The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH.`) cmd.Flags().Int32Var(&o.Code, "code", 0, `The status code responded by target.`) - - cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) - cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged - - printer.AddOutputFlagForCreate(cmd, &o.Format) } func (o *HTTPChaosOptions) Validate() error { diff --git a/internal/cli/cmd/fault/fault_http_test.go b/internal/cli/cmd/fault/fault_http_test.go new file mode 100644 index 000000000..3a7b44b7d --- /dev/null +++ b/internal/cli/cmd/fault/fault_http_test.go @@ -0,0 +1,139 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" +) + +var _ = Describe("Fault Network HTPP", func() { + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + }) + + AfterEach(func() { + tf.Cleanup() + }) + + Context("test fault network http", func() { + It("fault network http abort", func() { + inputs := [][]string{ + {"--dry-run=client"}, + {"--mode=one", "--dry-run=client"}, + {"--mode=fixed", "--value=2", "--dry-run=client"}, + {"--mode=fixed-percent", "--value=50", "--dry-run=client"}, + {"--mode=random-max-percent", "--value=50", "--dry-run=client"}, + {"--ns-fault=kb-system", "--dry-run=client"}, + {"--node=minikube-m02", "--dry-run=client"}, + {"--label=app.kubernetes.io/component=mysql", "--dry-run=client"}, + {"--node-label=kubernetes.io/arch=arm64", "--dry-run=client"}, + {"--annotation=example-annotation=group-a", "--dry-run=client"}, + {"--abort=true", "--dry-run=client"}, + } + o := NewHTTPChaosOptions(tf, streams, "") + cmd := o.NewCobraCommand(Abort, AbortShort) + o.AddCommonFlag(cmd) + cmd.Flags().BoolVar(&o.Abort, "abort", true, `Indicates whether to inject the fault that interrupts the connection.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network http delay", func() { + inputs := [][]string{ + {""}, + {"--delay=50s", "--dry-run=client"}, + } + o := NewHTTPChaosOptions(tf, streams, "") + cmd := o.NewCobraCommand(Delay, DelayShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Delay, "delay", "10s", `The time for delay.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network http replace", func() { + inputs := [][]string{ + {"--dry-run=client"}, + {"--replace-method=PUT", "--body=\"you are good luck\"", "--replace-path=/local/", "--duration=1m", "--dry-run=client"}, + {"--target=Response", "--replace-method=PUT", "--body=you", "--replace-path=/local/", "--duration=1m", "--dry-run=client"}, + } + o := NewHTTPChaosOptions(tf, streams, "") + cmd := o.NewCobraCommand(Replace, ReplaceShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.InputReplaceBody, "body", "", `The content of the request body or response body to replace the failure.`) + cmd.Flags().StringVar(&o.ReplacePath, "replace-path", "", `The URI path used to replace content.`) + cmd.Flags().StringVar(&o.ReplaceMethod, "replace-method", "", `The replaced content of the HTTP request method.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network http patch", func() { + inputs := [][]string{ + {"--dry-run=client"}, + {"--body=\"you are good luck\"", "--type=JSON", "--duration=1m", "--dry-run=client"}, + {"--target=Response", "--body=\"you are good luck\"", "--type=JSON", "--duration=1m", "--dry-run=client"}, + } + o := NewHTTPChaosOptions(tf, streams, "") + cmd := o.NewCobraCommand(Patch, PatchShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.PatchBodyValue, "body", "", `The fault of the request body or response body with patch faults.`) + cmd.Flags().StringVar(&o.PatchBodyType, "type", "", `The type of patch faults of the request body or response body. Currently, it only supports JSON.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + }) +}) diff --git a/internal/cli/cmd/fault/fault_network.go b/internal/cli/cmd/fault/fault_network.go index 592319228..b554d3c50 100644 --- a/internal/cli/cmd/fault/fault_network.go +++ b/internal/cli/cmd/fault/fault_network.go @@ -31,7 +31,6 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/apecloud/kubeblocks/internal/cli/create" - "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/util" ) @@ -40,32 +39,35 @@ var faultNetWorkExample = templates.Examples(` kbcli fault network partition # The specified pod is isolated from the k8s external network "kubeblocks.io". - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --external-targets=kubeblocks.io + kbcli fault network partition mycluster-mysql-1 --external-targets=kubeblocks.io # Isolate the network between two pods. - kbcli fault network partition --label=statefulset.kubernetes.io/pod-name=mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 + kbcli fault network partition mycluster-mysql-1 --target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2 // Like the partition command, the target can be specified through --target-label or --external-targets. The pod only has obstacles in communicating with this target. If the target is not specified, all communication will be blocked. # Block all pod communication under the default namespace, resulting in a 50% packet loss rate. kbcli fault network loss --loss=50 # Block the specified pod communication, so that the packet loss rate is 50%. - kbcli fault network loss --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --loss=50 + kbcli fault network loss mysql-cluster-mysql-2 --loss=50 kbcli fault network corrupt --corrupt=50 # Blocks specified pod communication with a 50% packet corruption rate. - kbcli fault network corrupt --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --corrupt=50 + kbcli fault network corrupt mysql-cluster-mysql-2 --corrupt=50 kbcli fault network duplicate --duplicate=50 # Block specified pod communication so that the packet repetition rate is 50%. - kbcli fault network duplicate --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --duplicate=50 + kbcli fault network duplicate mysql-cluster-mysql-2 --duplicate=50 kbcli fault network delay --latency=10s # Block the communication of the specified pod, causing its network delay for 10s. - kbcli fault network delay --label=statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 --latency=10s + kbcli fault network delay mysql-cluster-mysql-2 --latency=10s + + # Limit the communication bandwidth between mysql-cluster-mysql-2 and the outside. + kbcli fault network bandwidth mysql-cluster-mysql-2 --rate=1kbps --duration=1m `) type NetworkChaosOptions struct { @@ -74,13 +76,13 @@ type NetworkChaosOptions struct { // Indicates a network target outside of Kubernetes, which can be an IPv4 address or a domain name, // such as "www.baidu.com". Only works with direction: to. ExternalTargets []string `json:"externalTargets,omitempty"` - // Specifies the labels that target Pods come with. - TargetLabel map[string]string `json:"targetLabel,omitempty"` - // Specifies the namespaces to which target Pods belong. - TargetNamespaceSelector string `json:"targetNamespaceSelector"` - TargetMode string `json:"targetMode"` + TargetMode string `json:"targetMode,omitempty"` TargetValue string `json:"targetValue"` + // Specifies the labels that target Pods come with. + TargetLabelSelectors map[string]string `json:"targetLabelSelectors,omitempty"` + // Specifies the namespaces to which target Pods belong. + TargetNamespaceSelectors []string `json:"targetNamespaceSelectors"` // The percentage of packet loss Loss string `json:"loss,omitempty"` @@ -104,19 +106,18 @@ type NetworkChaosOptions struct { Minburst uint32 `json:"minburst"` FaultBaseOptions - - create.CreateOptions `json:"-"` } func NewNetworkChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *NetworkChaosOptions { o := &NetworkChaosOptions{ - CreateOptions: create.CreateOptions{ + FaultBaseOptions: FaultBaseOptions{CreateOptions: create.CreateOptions{ Factory: f, IOStreams: streams, CueTemplateName: CueTemplateNetworkChaos, GVR: GetGVR(Group, Version, ResourceNetworkChaos), }, - FaultBaseOptions: FaultBaseOptions{Action: action}, + Action: action, + }, } o.CreateOptions.PreCreate = o.PreCreate o.CreateOptions.Options = o @@ -260,24 +261,14 @@ func (o *NetworkChaosOptions) NewCobraCommand(use, short string) *cobra.Command } func (o *NetworkChaosOptions) AddCommonFlag(cmd *cobra.Command) { - - cmd.Flags().StringVar(&o.Mode, "mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) - cmd.Flags().StringVar(&o.Value, "value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) - cmd.Flags().StringVar(&o.Duration, "duration", "10s", "Supported formats of the duration are: ms / s / m / h.") - cmd.Flags().StringToStringVar(&o.Label, "label", map[string]string{}, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) - cmd.Flags().StringArrayVar(&o.NamespaceSelector, "namespace-selector", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) + o.FaultBaseOptions.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Direction, "direction", "to", `You can select "to"" or "from"" or "both"".`) - cmd.Flags().StringArrayVarP(&o.ExternalTargets, "external-targets", "e", nil, "a network target outside of Kubernetes, which can be an IPv4 address or a domain name,\n\t such as \"www.baidu.com\". Only works with direction: to.") - cmd.Flags().StringVar(&o.TargetMode, "target-mode", "all", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) + cmd.Flags().StringArrayVarP(&o.ExternalTargets, "external-target", "e", nil, "a network target outside of Kubernetes, which can be an IPv4 address or a domain name,\n\t such as \"www.baidu.com\". Only works with direction: to.") + cmd.Flags().StringVar(&o.TargetMode, "target-mode", "", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) cmd.Flags().StringVar(&o.TargetValue, "target-value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) - cmd.Flags().StringToStringVar(&o.TargetLabel, "target-label", nil, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) - cmd.Flags().StringVar(&o.TargetNamespaceSelector, "target-namespace-selector", "default", `Specifies the namespace into which you want to inject faults.`) - - cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) - cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged - - printer.AddOutputFlagForCreate(cmd, &o.Format) + cmd.Flags().StringToStringVar(&o.TargetLabelSelectors, "target-label", nil, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) + cmd.Flags().StringArrayVar(&o.TargetNamespaceSelectors, "target-ns-fault", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) } func (o *NetworkChaosOptions) Validate() error { @@ -285,6 +276,10 @@ func (o *NetworkChaosOptions) Validate() error { return fmt.Errorf("you must use --value to specify an integer") } + if (o.TargetLabelSelectors != nil || o.TargetValue != "") && o.TargetMode == "" { + return fmt.Errorf("you must use --mode to specify an experiment mode") + } + if ok, err := IsInteger(o.TargetValue); !ok { return err } diff --git a/internal/cli/cmd/fault/fault_network_test.go b/internal/cli/cmd/fault/fault_network_test.go new file mode 100644 index 000000000..4827ae206 --- /dev/null +++ b/internal/cli/cmd/fault/fault_network_test.go @@ -0,0 +1,184 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "k8s.io/cli-runtime/pkg/genericclioptions" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" +) + +var _ = Describe("Fault Network", func() { + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + }) + + AfterEach(func() { + tf.Cleanup() + }) + + Context("test fault network", func() { + It("fault network partition", func() { + inputs := [][]string{ + {"--dry-run=client"}, + {"--mode=one", "--dry-run=client"}, + {"--mode=fixed", "--value=2", "--dry-run=client"}, + {"--mode=fixed-percent", "--value=50", "--dry-run=client"}, + {"--mode=random-max-percent", "--value=50", "--dry-run=client"}, + {"--ns-fault=kb-system", "--dry-run=client"}, + {"--node=minikube-m02", "--dry-run=client"}, + {"--label=app.kubernetes.io/component=mysql", "--dry-run=client"}, + {"--node-label=kubernetes.io/arch=arm64", "--dry-run=client"}, + {"--annotation=example-annotation=group-a", "--dry-run=client"}, + {"--external-target=kubeblocks.io", "--dry-run=client"}, + {"--target-mode=one", "--target-label=statefulset.kubernetes.io/pod-name=mycluster-mysql-2", "--target-ns-fault=default", "--dry-run=client"}, + } + o := NewNetworkChaosOptions(tf, streams, string(v1alpha1.PartitionAction)) + cmd := o.NewCobraCommand(Partition, PartitionShort) + o.AddCommonFlag(cmd) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network loss", func() { + inputs := [][]string{ + {"--loss=50", "--dry-run=client"}, + {"--loss=50", "--correlation=100", "--dry-run=client"}, + } + o := NewNetworkChaosOptions(tf, streams, string(v1alpha1.LossAction)) + cmd := o.NewCobraCommand(Loss, LossShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Loss, "loss", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network delay", func() { + inputs := [][]string{ + {"--latency=50s", "--dry-run=client"}, + {"--latency=50s", "--jitter=10s", "--dry-run=client"}, + {"--latency=50s", "--correlation=100", "--dry-run=client"}, + {"--latency=50s", "--jitter=10s", "--correlation=100", "--dry-run=client"}, + } + o := NewNetworkChaosOptions(tf, streams, string(v1alpha1.DelayAction)) + cmd := o.NewCobraCommand(Delay, DelayShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Latency, "latency", "", `the length of time to delay.`) + cmd.Flags().StringVar(&o.Jitter, "jitter", "0ms", `the variation range of the delay time.`) + cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network duplicate", func() { + inputs := [][]string{ + {"--duplicate=50", "--dry-run=client"}, + {"--duplicate=50", "--correlation=100", "--dry-run=client"}, + } + o := NewNetworkChaosOptions(tf, streams, string(v1alpha1.DuplicateAction)) + cmd := o.NewCobraCommand(Duplicate, DuplicateShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Duplicate, "duplicate", "", `the probability of a packet being repeated. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network corrupt", func() { + inputs := [][]string{ + {"--corrupt=50", "--dry-run=client"}, + {"--corrupt=50", "--correlation=100", "--dry-run=client"}, + } + o := NewNetworkChaosOptions(tf, streams, string(v1alpha1.CorruptAction)) + cmd := o.NewCobraCommand(Corrupt, CorruptShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Corrupt, "corrupt", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault network bandwidth", func() { + inputs := [][]string{ + {"--rate=10kbps", "--dry-run=client"}, + {"--rate=10kbps", "--limit=1000", "--buffer=100", "--peakrate=10", "--minburst=5", "--dry-run=client"}, + } + o := NewNetworkChaosOptions(tf, streams, string(v1alpha1.BandwidthAction)) + cmd := o.NewCobraCommand(Bandwidth, BandwidthShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringVar(&o.Rate, "rate", "", `the rate at which the bandwidth is limited. For example : 10 bps/kbps/mbps/gbps.`) + cmd.Flags().Uint32Var(&o.Limit, "limit", 1, `the number of bytes waiting in the queue.`) + cmd.Flags().Uint32Var(&o.Buffer, "buffer", 1, `the maximum number of bytes that can be sent instantaneously.`) + cmd.Flags().Uint64Var(&o.Peakrate, "peakrate", 0, `the maximum consumption rate of the bucket.`) + cmd.Flags().Uint32Var(&o.Minburst, "minburst", 0, `the size of the peakrate bucket.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + }) +}) diff --git a/internal/cli/cmd/fault/fault_pod.go b/internal/cli/cmd/fault/fault_pod.go new file mode 100644 index 000000000..3a64aa580 --- /dev/null +++ b/internal/cli/cmd/fault/fault_pod.go @@ -0,0 +1,186 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var faultPodExample = templates.Examples(` + # kill all pods in default namespace + kbcli fault pod kill + + # kill any pod in default namespace + kbcli fault pod kill --mode=one + + # kill two pods in default namespace + kbcli fault pod kill --mode=fixed --value=2 + + # kill 50% pods in default namespace + kbcli fault pod kill --mode=percentage --value=50 + + # kill mysql-cluster-mysql-0 pod in default namespace + kbcli fault pod kill mysql-cluster-mysql-0 + + # kill all pods in default namespace + kbcli fault pod kill --ns-fault="default" + + # --label is required to specify the pods that need to be killed. + kbcli fault pod kill --label statefulset.kubernetes.io/pod-name=mysql-cluster-mysql-2 + + # kill pod under the specified node. + kbcli fault pod kill --node=minikube-m02 + + # kill pod under the specified node-label. + kbcli fault pod kill --node-label=kubernetes.io/arch=arm64 + + # Allow the experiment to last for one minute. + kbcli fault pod failure --duration=1m + + # kill container in pod + kbcli fault pod kill-container mysql-cluster-mysql-0 --container=mysql +`) + +type PodChaosOptions struct { + // GracePeriod waiting time, after which fault injection is performed + GracePeriod int64 `json:"gracePeriod"` + ContainerNames []string `json:"containerNames,omitempty"` + + FaultBaseOptions +} + +func NewPodChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *PodChaosOptions { + o := &PodChaosOptions{ + FaultBaseOptions: FaultBaseOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplatePodChaos, + GVR: GetGVR(Group, Version, ResourcePodChaos), + }, + Action: action, + }, + } + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o +} + +func NewPodChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "pod", + Short: "Pod chaos.", + } + cmd.AddCommand( + NewPodKillCmd(f, streams), + NewPodFailureCmd(f, streams), + NewContainerKillCmd(f, streams), + ) + return cmd +} + +func NewPodKillCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewPodChaosOptions(f, streams, string(v1alpha1.PodKillAction)) + cmd := o.NewCobraCommand(Kill, KillShort) + + o.AddCommonFlag(cmd) + cmd.Flags().Int64VarP(&o.GracePeriod, "grace-period", "g", 0, "Grace period represents the duration in seconds before the pod should be killed") + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewPodFailureCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewPodChaosOptions(f, streams, string(v1alpha1.PodFailureAction)) + cmd := o.NewCobraCommand(Failure, FailureShort) + + o.AddCommonFlag(cmd) + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func NewContainerKillCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewPodChaosOptions(f, streams, string(v1alpha1.ContainerKillAction)) + cmd := o.NewCobraCommand(KillContainer, KillContainerShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringArrayVarP(&o.ContainerNames, "container", "c", nil, "the name of the container you want to kill, such as mysql, prometheus.") + + util.CheckErr(cmd.MarkFlagRequired("container")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) + + return cmd +} + +func (o *PodChaosOptions) NewCobraCommand(use, short string) *cobra.Command { + return &cobra.Command{ + Use: use, + Short: short, + Example: faultPodExample, + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run()) + }, + } +} + +func (o *PodChaosOptions) AddCommonFlag(cmd *cobra.Command) { + o.FaultBaseOptions.AddCommonFlag(cmd) +} + +func (o *PodChaosOptions) Validate() error { + return o.BaseValidate() +} + +func (o *PodChaosOptions) Complete() error { + return o.BaseComplete() +} + +func (o *PodChaosOptions) PreCreate(obj *unstructured.Unstructured) error { + c := &v1alpha1.PodChaos{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, c); err != nil { + return err + } + + data, e := runtime.DefaultUnstructuredConverter.ToUnstructured(c) + if e != nil { + return e + } + obj.SetUnstructuredContent(data) + return nil +} diff --git a/internal/cli/cmd/fault/fault_pod_test.go b/internal/cli/cmd/fault/fault_pod_test.go new file mode 100644 index 000000000..b1e55d3b8 --- /dev/null +++ b/internal/cli/cmd/fault/fault_pod_test.go @@ -0,0 +1,96 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "k8s.io/cli-runtime/pkg/genericclioptions" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" +) + +var _ = Describe("Fault POD", func() { + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + }) + + AfterEach(func() { + tf.Cleanup() + }) + + Context("test fault pod", func() { + It("fault pod kill", func() { + inputs := [][]string{ + {"--dry-run=client"}, + {"--mode=one", "--dry-run=client"}, + {"--mode=fixed", "--value=2", "--dry-run=client"}, + {"--mode=fixed-percent", "--value=50", "--dry-run=client"}, + {"--mode=random-max-percent", "--value=50", "--dry-run=client"}, + {"--grace-period=5", "--dry-run=client"}, + {"--ns-fault=kb-system", "--dry-run=client"}, + {"--node=minikube-m02", "--dry-run=client"}, + {"--label=app.kubernetes.io/component=mysql", "--dry-run=client"}, + {"--node-label=kubernetes.io/arch=arm64", "--dry-run=client"}, + {"--annotation=example-annotation=group-a", "--dry-run=client"}, + } + o := NewPodChaosOptions(tf, streams, string(v1alpha1.PodKillAction)) + cmd := o.NewCobraCommand(Kill, KillShort) + o.AddCommonFlag(cmd) + cmd.Flags().Int64VarP(&o.GracePeriod, "grace-period", "g", 0, "Grace period represents the duration in seconds before the pod should be killed") + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault pod kill-container", func() { + inputs := [][]string{ + {"--container=mysql", "--container=config-manager", "--dry-run=client"}, + } + o := NewPodChaosOptions(tf, streams, string(v1alpha1.ContainerKillAction)) + cmd := o.NewCobraCommand(KillContainer, KillContainerShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringArrayVarP(&o.ContainerNames, "container", "c", nil, "the name of the container you want to kill, such as mysql, prometheus.") + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + }) +}) diff --git a/internal/cli/cmd/fault/suite_test.go b/internal/cli/cmd/fault/suite_test.go new file mode 100644 index 000000000..7f1e8aba4 --- /dev/null +++ b/internal/cli/cmd/fault/suite_test.go @@ -0,0 +1,32 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestDashboard(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Fault Suite") +} diff --git a/internal/cli/create/template/dns_chaos_template.cue b/internal/cli/create/template/dns_chaos_template.cue index 1ba654e0c..92be304be 100644 --- a/internal/cli/create/template/dns_chaos_template.cue +++ b/internal/cli/create/template/dns_chaos_template.cue @@ -19,13 +19,11 @@ options: { namespace: string action: string - - namespaceSelector: [...] + selector: {} mode: string value: string duration: string - label?: {} - + patterns: [...] } @@ -38,14 +36,7 @@ content: { namespace: options.namespace } spec:{ - selector:{ - namespaces: options.namespaceSelector - if options.label != _|_ { - labelSelectors:{ - options.label - } - } - } + selector: options.selector action: options.action mode: options.mode value: options.value diff --git a/internal/cli/create/template/http_chaos_template.cue b/internal/cli/create/template/http_chaos_template.cue index 6b412e547..5299d2a31 100644 --- a/internal/cli/create/template/http_chaos_template.cue +++ b/internal/cli/create/template/http_chaos_template.cue @@ -18,8 +18,7 @@ // required, command line input options for parameters and flags options: { namespace: string - label?: {} - namespaceSelector: [...] + selector: {} mode: string value: string duration: string @@ -50,14 +49,7 @@ content: { namespace: options.namespace } spec:{ - selector:{ - namespaces: options.namespaceSelector - if options.label != _|_ { - labelSelectors:{ - options.label - } - } - } + selector: options.selector mode: options.mode value: options.value duration: options.duration diff --git a/internal/cli/create/template/network_chaos_template.cue b/internal/cli/create/template/network_chaos_template.cue index 622d08220..1db5fc6b0 100644 --- a/internal/cli/create/template/network_chaos_template.cue +++ b/internal/cli/create/template/network_chaos_template.cue @@ -19,20 +19,18 @@ options: { namespace: string action: string - - namespaceSelector: [...] + selector: {} mode: string value: string duration: string - label?: {} direction: string externalTargets?: [...] - targetNamespaceSelector: string - targetMode: string - targetValue: string - targetLabel?: {} + targetMode?: string + targetValue: string + targetNamespaceSelectors: [...string] + targetLabelSelectors: {} loss?: string corrupt?: string @@ -59,14 +57,7 @@ content: { namespace: options.namespace } spec:{ - selector:{ - namespaces: options.namespaceSelector - if options.label != _|_ { - labelSelectors:{ - options.label - } - } - } + selector: options.selector mode: options.mode value: options.value action: options.action @@ -76,14 +67,14 @@ content: { if options.externalTargets != _|_ { externalTargets: options.externalTargets } - if options.targetLabel != _|_ { + if options.targetMode != _|_ { target:{ mode: options.targetMode value: options.targetValue selector:{ - namespaces: [options.targetNamespaceSelector] + namespaces: options.targetNamespaceSelectors labelSelectors:{ - options.targetLabel + options.targetLabelSelectors } } } diff --git a/internal/cli/create/template/pod_chaos_template.cue b/internal/cli/create/template/pod_chaos_template.cue new file mode 100644 index 000000000..e653dc464 --- /dev/null +++ b/internal/cli/create/template/pod_chaos_template.cue @@ -0,0 +1,49 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// required, command line input options for parameters and flags +options: { + namespace: string + action: string + selector: {} + mode: string + value: string + duration: string + + gracePeriod: int64 + containerNames: [...] +} + +// required, k8s api resource content +content: { + kind: "PodChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "pod-chaos-" + namespace: options.namespace + } + spec:{ + selector: options.selector + mode: options.mode + value: options.value + action: options.action + duration: options.duration + + gracePeriod: options.gracePeriod + containerNames: options.containerNames + } +} From 7fc48d98d9ea7c33506ac11c6aa9cfc94c3df77f Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Tue, 16 May 2023 20:24:13 +0800 Subject: [PATCH 309/439] fix: the kbcli cd list-components panic when the clusterdefinition not found (#3237) --- .../cmd/clusterdefinition/list_compoents.go | 13 ++++------ .../clusterdefinition/list_component_test.go | 24 +++++++++++++++---- internal/cli/testing/fake.go | 15 ++++++++++++ 3 files changed, 39 insertions(+), 13 deletions(-) diff --git a/internal/cli/cmd/clusterdefinition/list_compoents.go b/internal/cli/cmd/clusterdefinition/list_compoents.go index 51e1579c3..e16a5d23e 100644 --- a/internal/cli/cmd/clusterdefinition/list_compoents.go +++ b/internal/cli/cmd/clusterdefinition/list_compoents.go @@ -44,6 +44,7 @@ var ( func NewListComponentsCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := list.NewListOptions(f, streams, types.ClusterDefGVR()) + o.AllNamespaces = true cmd := &cobra.Command{ Use: "list-components", Short: "List cluster definition components.", @@ -52,7 +53,7 @@ func NewListComponentsCmd(f cmdutil.Factory, streams genericclioptions.IOStreams ValidArgsFunction: util.ResourceNameCompletionFunc(f, o.GVR), Run: func(cmd *cobra.Command, args []string) { util.CheckErr(validate(args)) - o.FieldSelector = fmt.Sprintf("metadata.name=%s", args[0]) + o.Names = args util.CheckErr(run(o)) }, } @@ -62,14 +63,13 @@ func NewListComponentsCmd(f cmdutil.Factory, streams genericclioptions.IOStreams func validate(args []string) error { if len(args) == 0 { return fmt.Errorf("missing clusterdefinition name") - } else if len(args) > 1 { - return fmt.Errorf("only support one clusterdefinition name") } return nil } func run(o *list.ListOptions) error { o.Print = false + r, err := o.Run() if err != nil { return err @@ -78,18 +78,15 @@ func run(o *list.ListOptions) error { if err != nil { return err } - if len(infos) == 0 { - return fmt.Errorf("no clusterdefinition %s found", o.Names[0]) - } p := printer.NewTablePrinter(o.Out) - p.SetHeader("NAME", "WORKLOAD-TYPE", "CHARACTER-TYPE") + p.SetHeader("NAME", "WORKLOAD-TYPE", "CHARACTER-TYPE", "CLUSTER-DEFINITION") for _, info := range infos { var cd v1alpha1.ClusterDefinition if err = runtime.DefaultUnstructuredConverter.FromUnstructured(info.Object.(*unstructured.Unstructured).Object, &cd); err != nil { return err } for _, comp := range cd.Spec.ComponentDefs { - p.AddRow(comp.Name, comp.WorkloadType, comp.CharacterType) + p.AddRow(comp.Name, comp.WorkloadType, comp.CharacterType, cd.Name) } } p.Print() diff --git a/internal/cli/cmd/clusterdefinition/list_component_test.go b/internal/cli/cmd/clusterdefinition/list_component_test.go index e6de64304..2664ded8a 100644 --- a/internal/cli/cmd/clusterdefinition/list_component_test.go +++ b/internal/cli/cmd/clusterdefinition/list_component_test.go @@ -37,6 +37,7 @@ import ( "k8s.io/kubectl/pkg/scheme" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/list" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" ) @@ -88,13 +89,26 @@ var _ = Describe("clusterdefinition list components", func() { Expect(validate([]string{})).Should(HaveOccurred()) }) - It("list-components ", func() { + It("cd list-components when the cd do not exist", func() { + o := list.NewListOptions(tf, streams, types.ClusterDefGVR()) + o.AllNamespaces = true + codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) + tf.UnstructuredClient = &clientfake.RESTClient{ + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + GroupVersion: schema.GroupVersion{Group: types.AppsAPIGroup, Version: types.AppsAPIVersion}, + Resp: &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, testing.FakeResourceNotFound(types.ClusterDefGVR(), clusterdefinitionName+"-no-exist"))}, + } + Expect(run(o)).Should(HaveOccurred()) + + }) + + It("list-components", func() { cmd.Run(cmd, []string{clusterdefinitionName}) - expected := `NAME WORKLOAD-TYPE CHARACTER-TYPE -fake-component-type mysql -fake-component-type-1 mysql + expected := `NAME WORKLOAD-TYPE CHARACTER-TYPE CLUSTER-DEFINITION +fake-component-type mysql fake-cluster-definition +fake-component-type-1 mysql fake-cluster-definition ` - fmt.Println(out.String()) Expect(expected).Should(Equal(out.String())) + fmt.Println(out.String()) }) }) diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index 1c7fd1489..84b00a5ad 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -32,6 +32,7 @@ import ( storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kubectl/pkg/util/storage" "k8s.io/utils/pointer" @@ -793,3 +794,17 @@ func FakeCronJob(name string, namespace string, extraLabels map[string]string) * }, } } + +func FakeResourceNotFound(versionResource schema.GroupVersionResource, name string) *metav1.Status { + return &metav1.Status{ + TypeMeta: metav1.TypeMeta{ + Kind: "Status", + APIVersion: "v1", + }, + Status: "Failure", + Message: fmt.Sprintf("%s.%s \"%s\" not found", versionResource.Resource, versionResource.Group, name), + Reason: "NotFound", + Details: nil, + Code: 404, + } +} From d6723464eec1caa5d1ba196c5e612e0001b374fb Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Wed, 17 May 2023 09:11:04 +0800 Subject: [PATCH 310/439] feat: cli `cv list` support to specify cluster definition (#2961) Co-authored-by: 1aal <1aal@users.noreply.github.com> --- docs/user_docs/cli/kbcli_cluster_expose.md | 2 +- docs/user_docs/cli/kbcli_cluster_hscale.md | 2 +- docs/user_docs/cli/kbcli_cluster_restart.md | 2 +- .../cli/kbcli_cluster_volume-expand.md | 2 +- docs/user_docs/cli/kbcli_cluster_vscale.md | 2 +- .../cli/kbcli_clusterversion_list.md | 9 ++- internal/cli/cmd/class/list.go | 3 +- internal/cli/cmd/cluster/operations.go | 2 +- .../cli/cmd/clusterversion/clusterversion.go | 14 +++- .../cmd/clusterversion/clusterversion_test.go | 81 ++++++++++++++++++- internal/cli/testing/fake.go | 3 + internal/cli/util/flags/flags.go | 20 +++++ internal/cli/util/util.go | 4 + 13 files changed, 132 insertions(+), 14 deletions(-) create mode 100644 internal/cli/util/flags/flags.go diff --git a/docs/user_docs/cli/kbcli_cluster_expose.md b/docs/user_docs/cli/kbcli_cluster_expose.md index 281bb5867..3e7433225 100644 --- a/docs/user_docs/cli/kbcli_cluster_expose.md +++ b/docs/user_docs/cli/kbcli_cluster_expose.md @@ -24,7 +24,7 @@ kbcli cluster expose NAME --enable=[true|false] --type=[vpc|internet] [flags] ### Options ``` - --components strings Component names to this operations + --components strings Component names to this operations --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") --enable string Enable or disable the expose, values can be true or false -h, --help help for expose diff --git a/docs/user_docs/cli/kbcli_cluster_hscale.md b/docs/user_docs/cli/kbcli_cluster_hscale.md index 22dcc0336..a16d25ed3 100644 --- a/docs/user_docs/cli/kbcli_cluster_hscale.md +++ b/docs/user_docs/cli/kbcli_cluster_hscale.md @@ -18,7 +18,7 @@ kbcli cluster hscale NAME [flags] ### Options ``` - --components strings Component names to this operations + --components strings Component names to this operations --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for hscale --name string OpsRequest name. if not specified, it will be randomly generated diff --git a/docs/user_docs/cli/kbcli_cluster_restart.md b/docs/user_docs/cli/kbcli_cluster_restart.md index 690f3e6f9..226376b04 100644 --- a/docs/user_docs/cli/kbcli_cluster_restart.md +++ b/docs/user_docs/cli/kbcli_cluster_restart.md @@ -21,7 +21,7 @@ kbcli cluster restart NAME [flags] ### Options ``` - --components strings Component names to this operations + --components strings Component names to this operations --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for restart --name string OpsRequest name. if not specified, it will be randomly generated diff --git a/docs/user_docs/cli/kbcli_cluster_volume-expand.md b/docs/user_docs/cli/kbcli_cluster_volume-expand.md index e8f9684a3..eba0c516c 100644 --- a/docs/user_docs/cli/kbcli_cluster_volume-expand.md +++ b/docs/user_docs/cli/kbcli_cluster_volume-expand.md @@ -18,7 +18,7 @@ kbcli cluster volume-expand NAME [flags] ### Options ``` - --components strings Component names to this operations + --components strings Component names to this operations --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for volume-expand --name string OpsRequest name. if not specified, it will be randomly generated diff --git a/docs/user_docs/cli/kbcli_cluster_vscale.md b/docs/user_docs/cli/kbcli_cluster_vscale.md index 28eeee02a..34734b8a5 100644 --- a/docs/user_docs/cli/kbcli_cluster_vscale.md +++ b/docs/user_docs/cli/kbcli_cluster_vscale.md @@ -22,7 +22,7 @@ kbcli cluster vscale NAME [flags] ``` --class string Component class - --components strings Component names to this operations + --components strings Component names to this operations --cpu string Requested and limited size of component cpu --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for vscale diff --git a/docs/user_docs/cli/kbcli_clusterversion_list.md b/docs/user_docs/cli/kbcli_clusterversion_list.md index 64affc368..bf0a7d320 100644 --- a/docs/user_docs/cli/kbcli_clusterversion_list.md +++ b/docs/user_docs/cli/kbcli_clusterversion_list.md @@ -18,10 +18,11 @@ kbcli clusterversion list [flags] ### Options ``` - -h, --help help for list - -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) - -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. - --show-labels When printing, show all labels as the last column (default hide labels column) + --cluster-definition string Specify cluster definition, run "kbcli clusterdefinition list" to show all available cluster definition + -h, --help help for list + -o, --output format prints the output in the specified format. Allowed values: table, json, yaml, wide (default table) + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2). Matching objects must satisfy all of the specified label constraints. + --show-labels When printing, show all labels as the last column (default hide labels column) ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/class/list.go b/internal/cli/cmd/class/list.go index dccff40a5..d9b6354e3 100644 --- a/internal/cli/cmd/class/list.go +++ b/internal/cli/cmd/class/list.go @@ -35,6 +35,7 @@ import ( "github.com/apecloud/kubeblocks/internal/class" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/util" + "github.com/apecloud/kubeblocks/internal/cli/util/flags" ) type ListOptions struct { @@ -60,7 +61,7 @@ func NewListCommand(f cmdutil.Factory, streams genericclioptions.IOStreams) *cob util.CheckErr(o.run()) }, } - cmd.Flags().StringVar(&o.ClusterDefRef, "cluster-definition", "", "Specify cluster definition, run \"kbcli clusterdefinition list\" to show all available cluster definition") + flags.AddClusterDefinitionFlag(f, cmd, &o.ClusterDefRef) util.CheckErr(cmd.MarkFlagRequired("cluster-definition")) return cmd } diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index 40511b8e7..fbc6724be 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -125,7 +125,7 @@ func (o *OperationsOptions) addCommonFlags(cmd *cobra.Command) { cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) cmd.Flags().Lookup("dry-run").NoOptDefVal = "unchanged" if o.HasComponentNamesFlag { - cmd.Flags().StringSliceVar(&o.ComponentNames, "components", nil, " Component names to this operations") + cmd.Flags().StringSliceVar(&o.ComponentNames, "components", nil, "Component names to this operations") } } diff --git a/internal/cli/cmd/clusterversion/clusterversion.go b/internal/cli/cmd/clusterversion/clusterversion.go index 9be9d8480..eb80f0602 100644 --- a/internal/cli/cmd/clusterversion/clusterversion.go +++ b/internal/cli/cmd/clusterversion/clusterversion.go @@ -28,12 +28,18 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/list" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" + "github.com/apecloud/kubeblocks/internal/cli/util/flags" ) var listExample = templates.Examples(` # list all ClusterVersion kbcli clusterversion list`) +type ListClusterVersionOptions struct { + *list.ListOptions + clusterDefinitionRef string +} + func NewClusterVersionCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { cmd := &cobra.Command{ Use: "clusterversion", @@ -46,7 +52,9 @@ func NewClusterVersionCmd(f cmdutil.Factory, streams genericclioptions.IOStreams } func NewListCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { - o := list.NewListOptions(f, streams, types.ClusterVersionGVR()) + o := &ListClusterVersionOptions{ + ListOptions: list.NewListOptions(f, streams, types.ClusterVersionGVR()), + } cmd := &cobra.Command{ Use: "list", Short: "List ClusterVersions.", @@ -54,11 +62,15 @@ func NewListCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C Aliases: []string{"ls"}, ValidArgsFunction: util.ResourceNameCompletionFunc(f, o.GVR), Run: func(cmd *cobra.Command, args []string) { + if len(o.clusterDefinitionRef) != 0 { + o.LabelSelector = util.BuildClusterDefinitionRefLable(o.LabelSelector, []string{o.clusterDefinitionRef}) + } o.Names = args _, err := o.Run() util.CheckErr(err) }, } o.AddFlags(cmd, true) + flags.AddClusterDefinitionFlag(f, cmd, &o.clusterDefinitionRef) return cmd } diff --git a/internal/cli/cmd/clusterversion/clusterversion_test.go b/internal/cli/cmd/clusterversion/clusterversion_test.go index 7c9ba84ec..536813d0c 100644 --- a/internal/cli/cmd/clusterversion/clusterversion_test.go +++ b/internal/cli/cmd/clusterversion/clusterversion_test.go @@ -20,19 +20,87 @@ along with this program. If not, see . package clusterversion import ( + "bytes" + "net/http" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/cli-runtime/pkg/resource" + "k8s.io/client-go/kubernetes/scheme" + clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/testing" + "github.com/apecloud/kubeblocks/internal/cli/types" ) var _ = Describe("clusterversion", func() { var streams genericclioptions.IOStreams var tf *cmdtesting.TestFactory + out := new(bytes.Buffer) + + mockRestTable := func() *metav1.Table { + var Type = "string" + table := &metav1.Table{ + TypeMeta: metav1.TypeMeta{ + Kind: "Table", + APIVersion: "meta.k8s.io/v1", + }, + ColumnDefinitions: []metav1.TableColumnDefinition{ + { + Name: "NAME", + Type: Type, + }, { + Name: "CLUSTER-DEFINITION", + Type: Type, + }, { + Name: "STATUS", + Type: Type, + }, + { + Name: "AGE", + Type: Type, + }, + }, + Rows: []metav1.TableRow{ + { + Cells: []interface{}{ + testing.ClusterVersionName, + testing.ClusterDefName, + "Available", + "0s", + }, + }, + }, + } + return table + } + + mockClient := func(data runtime.Object) *cmdtesting.TestFactory { + tf := testing.NewTestFactory(testing.Namespace) + codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) + tf.UnstructuredClient = &clientfake.RESTClient{ + NegotiatedSerializer: resource.UnstructuredPlusDefaultContentConfig().NegotiatedSerializer, + GroupVersion: schema.GroupVersion{Group: types.AppsAPIGroup, Version: types.AppsAPIVersion}, + Resp: &http.Response{StatusCode: http.StatusOK, Header: cmdtesting.DefaultHeader(), Body: cmdtesting.ObjBody(codec, data)}, + } + tf.Client = tf.UnstructuredClient + tf.FakeDynamicClient = testing.FakeDynamicClient(data) + return tf + } BeforeEach(func() { - streams, _, _, _ = genericclioptions.NewTestIOStreams() - tf = cmdtesting.NewTestFactory() + _ = appsv1alpha1.AddToScheme(scheme.Scheme) + _ = metav1.AddMetaToScheme(scheme.Scheme) + streams, _, out, _ = genericclioptions.NewTestIOStreams() + table := mockRestTable() + tf = mockClient(table) }) AfterEach(func() { @@ -49,4 +117,13 @@ var _ = Describe("clusterversion", func() { cmd := NewListCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) }) + + It("list --cluster-definition", func() { + cmd := NewListCmd(tf, streams) + cmd.Run(cmd, []string{"--cluster-definition=" + testing.ClusterDefName}) + expected := `NAME CLUSTER-DEFINITION STATUS AGE +fake-cluster-version fake-cluster-definition Available 0s +` + Expect(expected).Should(Equal(out.String())) + }) }) diff --git a/internal/cli/testing/fake.go b/internal/cli/testing/fake.go index 84b00a5ad..c1ca8712f 100644 --- a/internal/cli/testing/fake.go +++ b/internal/cli/testing/fake.go @@ -304,6 +304,9 @@ func FakeComponentClassDef(name string, clusterDefRef string, componentDefRef st func FakeClusterVersion() *appsv1alpha1.ClusterVersion { cv := &appsv1alpha1.ClusterVersion{} + gvr := types.ClusterVersionGVR() + cv.TypeMeta.APIVersion = gvr.GroupVersion().String() + cv.TypeMeta.Kind = types.KindClusterVersion cv.Name = ClusterVersionName cv.SetLabels(map[string]string{ constant.ClusterDefLabelKey: ClusterDefName, diff --git a/internal/cli/util/flags/flags.go b/internal/cli/util/flags/flags.go new file mode 100644 index 000000000..8c75ba914 --- /dev/null +++ b/internal/cli/util/flags/flags.go @@ -0,0 +1,20 @@ +package flags + +import ( + "github.com/spf13/cobra" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + utilcomp "k8s.io/kubectl/pkg/util/completion" + + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +// AddClusterDefinitionFlag add a flag "cluster-definition" for the cmd and store the value of the flag +// in string p +func AddClusterDefinitionFlag(f cmdutil.Factory, cmd *cobra.Command, p *string) { + cmd.Flags().StringVar(p, "cluster-definition", *p, "Specify cluster definition, run \"kbcli clusterdefinition list\" to show all available cluster definition") + util.CheckErr(cmd.RegisterFlagCompletionFunc("cluster-definition", + func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return utilcomp.CompGetResource(f, cmd, util.GVRToString(types.ClusterDefGVR()), toComplete), cobra.ShellCompDirectiveNoFileComp + })) +} diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 71afa05c9..5d3f0af3f 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -760,6 +760,10 @@ func CreateResourceIfAbsent( return nil } +func BuildClusterDefinitionRefLable(prefix string, clusterDef []string) string { + return buildLabelSelectors(prefix, constant.AppNameLabelKey, clusterDef) +} + // IsWindows return true if the kbcli runtime situation is windows func IsWindows() bool { return runtime.GOOS == types.GoosWindows From 29b56cda734973eebe469a90513efcf43afbaf90 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Wed, 17 May 2023 09:17:52 +0800 Subject: [PATCH 311/439] chore: support vector databases llama/chroma/azuresearch/supabase/postgres for chatgpt retrival plugin (#3274) --- .../templates/deployment.yaml | 48 ++++++++++++++++ deploy/chatgpt-retrieval-plugin/values.yaml | 55 ++++++++++++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/deploy/chatgpt-retrieval-plugin/templates/deployment.yaml b/deploy/chatgpt-retrieval-plugin/templates/deployment.yaml index 845e40034..62aef4c49 100644 --- a/deploy/chatgpt-retrieval-plugin/templates/deployment.yaml +++ b/deploy/chatgpt-retrieval-plugin/templates/deployment.yaml @@ -121,6 +121,54 @@ spec: value: {{.Values.datastore.REDIS_DISTANCE_METRIC | default "COSINE" | quote}} - name: REDIS_INDEX_TYPE value: {{.Values.datastore.REDIS_INDEX_TYPE | default "FLAT" | quote}} + - name: LLAMA_INDEX_TYPE + value: {{.Values.datastore.LLAMA_INDEX_TYPE | default "simple_dict" | quote}} + - name: LLAMA_INDEX_JSON_PATH + value: {{.Values.datastore.LLAMA_INDEX_JSON_PATH | default | quote}} + - name: LLAMA_QUERY_KWARGS_JSON_PATH + value: {{.Values.datastore.LLAMA_QUERY_KWARGS_JSON_PATH | default | quote}} + - name: LLAMA_RESPONSE_MODE + value: {{.Values.datastore.LLAMA_RESPONSE_MODE | default "no_text" | quote}} + - name: CHROMA_COLLECTION + value: {{.Values.datastore.CHROMA_COLLECTION | default "openaiembeddings" | quote}} + - name: CHROMA_IN_MEMORY + value: {{.Values.datastore.CHROMA_IN_MEMORY | default "True" | quote}} + - name: CHROMA_PERSISTENCE_DIR + value: {{.Values.datastore.CHROMA_PERSISTENCE_DIR | default "openai" | quote}} + - name: CHROMA_HOST + value: {{.Values.datastore.CHROMA_HOST | default "http://127.0.0.1" | quote}} + - name: CHROMA_PORT + value: {{.Values.datastore.CHROMA_PORT | default "8080" | quote}} + - name: AZURESEARCH_SERVICE + value: {{.Values.datastore.AZURESEARCH_SERVICE | default | quote}} + - name: AZURESEARCH_INDEX + value: {{.Values.datastore.AZURESEARCH_INDEX | default | quote}} + - name: AZURESEARCH_API_KEY + value: {{.Values.datastore.AZURESEARCH_API_KEY | default | quote}} + - name: AZURESEARCH_DISABLE_HYBRID + value: {{.Values.datastore.AZURESEARCH_DISABLE_HYBRID | default | quote}} + - name: AZURESEARCH_SEMANTIC_CONFIG + value: {{.Values.datastore.AZURESEARCH_SEMANTIC_CONFIG | default | quote}} + - name: AZURESEARCH_LANGUAGE + value: {{.Values.datastore.AZURESEARCH_LANGUAGE | default "en-us" | quote}} + - name: AZURESEARCH_DIMENSIONS + value: {{.Values.datastore.AZURESEARCH_DIMENSIONS | default "1536" | quote}} + - name: SUPABASE_URL + value: {{.Values.datastore.SUPABASE_URL | default | quote}} + - name: SUPABASE_ANON_KEY + value: {{.Values.datastore.SUPABASE_ANON_KEY | default | quote}} + - name: SUPABASE_SERVICE_ROLE_KEY + value: {{.Values.datastore.SUPABASE_SERVICE_ROLE_KEY | default | quote}} + - name: PG_HOST + value: {{.Values.datastore.PG_HOST | default "localhost" | quote}} + - name: PG_PORT + value: {{.Values.datastore.PG_PORT | default "5432" | quote}} + - name: PG_PASSWORD + value: {{.Values.datastore.PG_PASSWORD | default "postgres" | quote}} + - name: PG_USER + value: {{.Values.datastore.PG_USER | default "postgres" | quote}} + - name: PG_DB + value: {{.Values.datastore.PG_DB | default "postgres" | quote}} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deploy/chatgpt-retrieval-plugin/values.yaml b/deploy/chatgpt-retrieval-plugin/values.yaml index 6f16f2248..00d27e12f 100644 --- a/deploy/chatgpt-retrieval-plugin/values.yaml +++ b/deploy/chatgpt-retrieval-plugin/values.yaml @@ -51,7 +51,7 @@ website: legal_info_url: hello@legal.com datastore: - # in list of (pinecone, weaviate, zilliz, milvus, qdrant, redis) + # in list of (pinecone, weaviate, zilliz, milvus, qdrant, redis, llama, chroma, azuresearch, supabase, postgres) DATASTORE: # Yes Your secret token to protect the local plugin API BEARER_TOKEN: @@ -132,6 +132,59 @@ datastore: # Optional Vector index algorithm type default:FLAT REDIS_INDEX_TYPE: FLAT + # Optional Index type (see below for details) + LLAMA_INDEX_TYPE: simple_dict + # Optional Path to saved Index json file + LLAMA_INDEX_JSON_PATH: + # Optional Path to saved query kwargs json file + LLAMA_QUERY_KWARGS_JSON_PATH: + # Optional Response mode for query + LLAMA_RESPONSE_MODE: no_text + + # Optional Your chosen Chroma collection name to store your embeddings + CHROMA_COLLECTION: openaiembeddings + # Optional If set to True, ignore CHROMA_HOST and CHROMA_PORT and just use an in-memory Chroma instance + CHROMA_IN_MEMORY: True + # Optional If set, and CHROMA_IN_MEMORY is set, persist to and load from this directory. + CHROMA_PERSISTENCE_DIR: openai + # Optional Your Chroma instance host address (see notes below) + CHROMA_HOST: http://127.0.0.1 + # Optional Your Chroma port number + CHROMA_PORT: 8000 + + # Required Name of your search service + AZURESEARCH_SERVICE: + # Required Name of your search index + AZURESEARCH_INDEX: + # Optional Your API key, if using key-based auth instead of Azure managed identity + AZURESEARCH_API_KEY: + # Optional Disable hybrid search and only use vector similarity + AZURESEARCH_DISABLE_HYBRID: + # Optional Enable L2 re-ranking with this configuration name see re-ranking below + AZURESEARCH_SEMANTIC_CONFIG: + # Optional If using L2 re-ranking, language for queries/documents (valid values listed here) + AZURESEARCH_LANGUAGE: en-us + # Optional Vector size for embeddings + AZURESEARCH_DIMENSIONS: 1536 + + # Required Supabase Project URL + SUPABASE_URL: + # Optional Supabase Project API anon key + SUPABASE_ANON_KEY: + # Optional Supabase Project API service key, will be used if provided instead of anon key + SUPABASE_SERVICE_ROLE_KEY: + + # Optional Postgres host + PG_HOST: localhost + # Optional Postgres port + PG_PORT: 5432 + # Optional Postgres password + PG_PASSWORD: postgres + # Optional Postgres username + PG_USER: postgres + # Optional Postgres database + PG_DB: postgres + resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little From 291b4b2485b1c3057a48208d6c3182f22d66800f Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Wed, 17 May 2023 09:18:11 +0800 Subject: [PATCH 312/439] feat: support opensearch (#3081) --- .../templates/addons/opensearch-addon.yaml | 22 ++ deploy/opensearch-cluster/.helmignore | 23 ++ deploy/opensearch-cluster/Chart.yaml | 24 +++ deploy/opensearch-cluster/templates/NOTES.txt | 22 ++ .../opensearch-cluster/templates/_helpers.tpl | 62 ++++++ .../opensearch-cluster/templates/cluster.yaml | 53 +++++ deploy/opensearch-cluster/values.yaml | 64 ++++++ deploy/opensearch/.helmignore | 23 ++ deploy/opensearch/Chart.yaml | 24 +++ deploy/opensearch/TODO | 6 + deploy/opensearch/configs/opensearch.yaml.tpl | 52 +++++ deploy/opensearch/templates/NOTES.txt | 22 ++ deploy/opensearch/templates/_helpers.tpl | 62 ++++++ .../templates/clusterdefinition.yaml | 204 ++++++++++++++++++ .../opensearch/templates/clusterversion.yaml | 31 +++ .../templates/configconstraint.yaml | 9 + deploy/opensearch/templates/configmap.yaml | 10 + deploy/opensearch/values.yaml | 60 ++++++ 18 files changed, 773 insertions(+) create mode 100644 deploy/helm/templates/addons/opensearch-addon.yaml create mode 100644 deploy/opensearch-cluster/.helmignore create mode 100644 deploy/opensearch-cluster/Chart.yaml create mode 100644 deploy/opensearch-cluster/templates/NOTES.txt create mode 100644 deploy/opensearch-cluster/templates/_helpers.tpl create mode 100644 deploy/opensearch-cluster/templates/cluster.yaml create mode 100644 deploy/opensearch-cluster/values.yaml create mode 100644 deploy/opensearch/.helmignore create mode 100644 deploy/opensearch/Chart.yaml create mode 100644 deploy/opensearch/TODO create mode 100644 deploy/opensearch/configs/opensearch.yaml.tpl create mode 100644 deploy/opensearch/templates/NOTES.txt create mode 100644 deploy/opensearch/templates/_helpers.tpl create mode 100644 deploy/opensearch/templates/clusterdefinition.yaml create mode 100644 deploy/opensearch/templates/clusterversion.yaml create mode 100644 deploy/opensearch/templates/configconstraint.yaml create mode 100644 deploy/opensearch/templates/configmap.yaml create mode 100644 deploy/opensearch/values.yaml diff --git a/deploy/helm/templates/addons/opensearch-addon.yaml b/deploy/helm/templates/addons/opensearch-addon.yaml new file mode 100644 index 000000000..e22c601a6 --- /dev/null +++ b/deploy/helm/templates/addons/opensearch-addon.yaml @@ -0,0 +1,22 @@ +apiVersion: extensions.kubeblocks.io/v1alpha1 +kind: Addon +metadata: + name: opensearch + labels: + {{- include "kubeblocks.labels" . | nindent 4 }} + "kubeblocks.io/provider": community + {{- if .Values.keepAddons }} + annotations: + helm.sh/resource-policy: keep + {{- end }} +spec: + description: 'OpenSearch is a scalable, flexible, and extensible open-source software suite for search, analytics, and observability applications licensed under Apache 2.0.' + + type: Helm + + helm: + chartLocationURL: https://jihulab.com/api/v4/projects/85949/packages/helm/stable/charts/opensearch-{{ default .Chart.Version .Values.versionOverride }}.tgz + + installable: + autoInstall: false + diff --git a/deploy/opensearch-cluster/.helmignore b/deploy/opensearch-cluster/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/opensearch-cluster/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/opensearch-cluster/Chart.yaml b/deploy/opensearch-cluster/Chart.yaml new file mode 100644 index 000000000..42ebeb36e --- /dev/null +++ b/deploy/opensearch-cluster/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: opensearch-cluster +description: A Helm chart for OpenSearch Cluster + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "2.7.0" diff --git a/deploy/opensearch-cluster/templates/NOTES.txt b/deploy/opensearch-cluster/templates/NOTES.txt new file mode 100644 index 000000000..253569192 --- /dev/null +++ b/deploy/opensearch-cluster/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "opensearch-cluster.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "opensearch-cluster.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "opensearch-cluster.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "opensearch-cluster.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/deploy/opensearch-cluster/templates/_helpers.tpl b/deploy/opensearch-cluster/templates/_helpers.tpl new file mode 100644 index 000000000..bb7108842 --- /dev/null +++ b/deploy/opensearch-cluster/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "opensearch-cluster.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "opensearch-cluster.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "opensearch-cluster.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "opensearch-cluster.labels" -}} +helm.sh/chart: {{ include "opensearch-cluster.chart" . }} +{{ include "opensearch-cluster.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "opensearch-cluster.selectorLabels" -}} +app.kubernetes.io/name: {{ include "opensearch-cluster.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "opensearch-cluster.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "opensearch-cluster.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/opensearch-cluster/templates/cluster.yaml b/deploy/opensearch-cluster/templates/cluster.yaml new file mode 100644 index 000000000..120c21db6 --- /dev/null +++ b/deploy/opensearch-cluster/templates/cluster.yaml @@ -0,0 +1,53 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + name: {{ include "opensearch-cluster.fullname" . }} + labels: {{ include "opensearch-cluster.labels" . | nindent 4 }} +spec: + clusterDefinitionRef: opensearch # ref clusterdefinition.name + clusterVersionRef: opensearch-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} # ref clusterversion.name + terminationPolicy: {{ .Values.terminationPolicy }} + affinity: + {{- with .Values.topologyKeys }} + topologyKeys: {{ . | toYaml | nindent 6 }} + {{- end }} + {{- with $.Values.tolerations }} + tolerations: {{ . | toYaml | nindent 4 }} + {{- end }} + componentSpecs: + - name: opensearch # user-defined + componentDefRef: opensearch # ref clusterdefinition componentDefs.name + monitor: {{ .Values.monitor.enabled | default false }} + replicas: {{ .Values.replicaCount | default 3 }} + {{- with .Values.resources }} + resources: + limits: + cpu: {{ .limits.cpu | quote }} + memory: {{ .limits.memory | quote }} + requests: + cpu: {{ .requests.cpu | quote }} + memory: {{ .requests.memory | quote }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - name: data # ref clusterdefinition components.containers.volumeMounts.name + spec: + storageClassName: {{ .Values.persistence.data.storageClassName }} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.persistence.data.size }} + {{- end }} + - name: dashboard # user-defined + componentDefRef: dashboard # ref clusterdefinition componentDefs.name + replicas: {{ .Values.dashboard.replicaCount | default 1 }} + {{- with .Values.dashboard.resources }} + resources: + limits: + cpu: {{ .limits.cpu | quote }} + memory: {{ .limits.memory | quote }} + requests: + cpu: {{ .requests.cpu | quote }} + memory: {{ .requests.memory | quote }} + {{- end }} diff --git a/deploy/opensearch-cluster/values.yaml b/deploy/opensearch-cluster/values.yaml new file mode 100644 index 000000000..dad58c269 --- /dev/null +++ b/deploy/opensearch-cluster/values.yaml @@ -0,0 +1,64 @@ +# Default values for opensearch-cluster. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +terminationPolicy: Delete + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +ingress: + enabled: false + +service: + type: ClusterIP + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +topologyKeys: +- kubernetes.io/hostname + +monitor: + enabled: false + +persistence: + enabled: true + data: + storageClassName: + size: 1Gi + +dashboard: + resources: {} + replicaCount: 1 \ No newline at end of file diff --git a/deploy/opensearch/.helmignore b/deploy/opensearch/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/deploy/opensearch/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/deploy/opensearch/Chart.yaml b/deploy/opensearch/Chart.yaml new file mode 100644 index 000000000..ce3b85797 --- /dev/null +++ b/deploy/opensearch/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: opensearch +description: A Helm chart for OpenSearch + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "2.7.0" diff --git a/deploy/opensearch/TODO b/deploy/opensearch/TODO new file mode 100644 index 000000000..b6c208dc5 --- /dev/null +++ b/deploy/opensearch/TODO @@ -0,0 +1,6 @@ +* support plugin +* support account +* enhance security +* splitting different roles +* remove OPENSEARCH_JAVA_OPTS +* optimize NOTES.txt \ No newline at end of file diff --git a/deploy/opensearch/configs/opensearch.yaml.tpl b/deploy/opensearch/configs/opensearch.yaml.tpl new file mode 100644 index 000000000..5aab99d8c --- /dev/null +++ b/deploy/opensearch/configs/opensearch.yaml.tpl @@ -0,0 +1,52 @@ +{{- $clusterName := $.cluster.metadata.name }} + +cluster.name: {{$clusterName}} + +# Bind to all interfaces because we don't know what IP address Docker will assign to us. +network.host: 0.0.0.0 + +# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again. +# Implicitly done if ".singleNode" is set to "true". +# discovery.type: single-node + +# Start OpenSearch Security Demo Configuration +# WARNING: revise all the lines below before you go into production +plugins: + security: + ssl: + transport: + pemcert_filepath: esnode.pem + pemkey_filepath: esnode-key.pem + pemtrustedcas_filepath: root-ca.pem + enforce_hostname_verification: false + http: + enabled: true + pemcert_filepath: esnode.pem + pemkey_filepath: esnode-key.pem + pemtrustedcas_filepath: root-ca.pem + allow_unsafe_democertificates: true + allow_default_init_securityindex: true + authcz: + admin_dn: + - CN=kirk,OU=client,O=client,L=test,C=de + audit.type: internal_opensearch + enable_snapshot_restore_privilege: true + check_snapshot_restore_write_privileges: true + restapi: + roles_enabled: ["all_access", "security_rest_api_access"] + system_indices: + enabled: true + indices: + [ + ".opendistro-alerting-config", + ".opendistro-alerting-alert*", + ".opendistro-anomaly-results*", + ".opendistro-anomaly-detector*", + ".opendistro-anomaly-checkpoints", + ".opendistro-anomaly-detection-state", + ".opendistro-reports-*", + ".opendistro-notifications-*", + ".opendistro-notebooks", + ".opendistro-asynchronous-search-response*", + ] +######## End OpenSearch Security Demo Configuration ######## diff --git a/deploy/opensearch/templates/NOTES.txt b/deploy/opensearch/templates/NOTES.txt new file mode 100644 index 000000000..1f5c1cf2f --- /dev/null +++ b/deploy/opensearch/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "opensearch.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "opensearch.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "opensearch.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "opensearch.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/deploy/opensearch/templates/_helpers.tpl b/deploy/opensearch/templates/_helpers.tpl new file mode 100644 index 000000000..ffd081f0d --- /dev/null +++ b/deploy/opensearch/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "opensearch.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "opensearch.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "opensearch.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "opensearch.labels" -}} +helm.sh/chart: {{ include "opensearch.chart" . }} +{{ include "opensearch.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "opensearch.selectorLabels" -}} +app.kubernetes.io/name: {{ include "opensearch.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "opensearch.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "opensearch.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/deploy/opensearch/templates/clusterdefinition.yaml b/deploy/opensearch/templates/clusterdefinition.yaml new file mode 100644 index 000000000..62a5b0fbc --- /dev/null +++ b/deploy/opensearch/templates/clusterdefinition.yaml @@ -0,0 +1,204 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterDefinition +metadata: + name: opensearch + labels: + {{- include "opensearch.labels" . | nindent 4 }} +spec: + type: opensearch + connectionCredential: + username: root + password: "$(RANDOM_PASSWD)" + endpoint: "https://$(SVC_FQDN):$(SVC_PORT_http)" + host: "$(SVC_FQDN)" + port: "$(SVC_PORT_http)" + componentDefs: + - name: opensearch + characterType: opensearch + monitor: + builtIn: false + exporterConfig: + scrapePath: /metrics + scrapePort: 9600 + configSpecs: + - name: opensearch-config-template + templateRef: opensearch-config-template + volumeName: opensearch-config + namespace: {{.Release.Namespace}} + workloadType: Stateful + service: + ports: + - name: http + port: 9200 + targetPort: http + - name: transport + port: 9300 + targetPort: transport + volumeTypes: + - name: data + type: data + podSpec: + initContainers: + - name: fsgroup-volume + imagePullPolicy: IfNotPresent + command: ['sh', '-c'] + args: + - 'chown -R 1000:1000 /usr/share/opensearch/data' + securityContext: + runAsUser: 0 + volumeMounts: + - name: data + mountPath: /usr/share/opensearch/data + - name: sysctl + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - | + set -xe + DESIRED="262144" + CURRENT=$(sysctl -n vm.max_map_count) + if [ "$DESIRED" -gt "$CURRENT" ]; then + sysctl -w vm.max_map_count=$DESIRED + fi + securityContext: + runAsUser: 0 + privileged: true + containers: + - name: opensearch + imagePullPolicy: IfNotPresent + readinessProbe: + tcpSocket: + port: 9200 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + startupProbe: + tcpSocket: + port: 9200 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 30 + ports: + - name: http + containerPort: 9200 + - name: transport + containerPort: 9300 + - name: metrics + containerPort: 9600 + env: + - name: node.name + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: cluster.initial_master_nodes + value: "$(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-0" + - name: discovery.seed_hosts + value: "$(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-headless" + - name: cluster.name + value: "$(KB_CLUSTER_NAME)" + - name: network.host + value: "0.0.0.0" + - name: OPENSEARCH_JAVA_OPTS + value: "-Xmx512M -Xms512M" + - name: node.roles + value: "master,ingest,data,remote_cluster_client" + volumeMounts: + - mountPath: /usr/share/opensearch/data + name: data + - mountPath: /usr/share/opensearch/config/opensearch.yaml + subPath: opensearch.yaml + name: opensearch-config + - name: opensearch-master-graceful-termination-handler + imagePullPolicy: IfNotPresent + command: + - "sh" + - -c + - | + #!/usr/bin/env bash + set -eo pipefail + + http () { + local path="${1}" + if [ -n "${USERNAME}" ] && [ -n "${PASSWORD}" ]; then + BASIC_AUTH="-u ${USERNAME}:${PASSWORD}" + else + BASIC_AUTH='' + fi + curl -XGET -s -k --fail ${BASIC_AUTH} https://$(KB_CLUSTER_NAME)-$(KB_COMP_NAME)-headless:9200:${path} + } + + cleanup () { + while true ; do + local master="$(http "/_cat/master?h=node" || echo "")" + if [[ $master == "$(KB_CLUSTER_NAME)-$(KB_COMP_NAME)"* && $master != "${NODE_NAME}" ]]; then + echo "This node is not master." + break + fi + echo "This node is still master, waiting gracefully for it to step down" + sleep 1 + done + + exit 0 + } + + trap cleanup SIGTERM + + sleep infinity & + wait $! + - name: dashboard + characterType: opensearch-dashboard + workloadType: Stateless + service: + ports: + - name: http + port: 5601 + targetPort: http + podSpec: + containers: + - name: dashboard + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + command: + - sh + - -c + - | + #!/usr/bin/bash + set -e + bash opensearch-dashboards-docker-entrypoint.sh opensearch-dashboards + env: + - name: OPENSEARCH_HOSTS + valueFrom: + secretKeyRef: + name: $(CONN_CREDENTIAL_SECRET_NAME) + key: endpoint + - name: SERVER_HOST + value: "0.0.0.0" + startupProbe: + tcpSocket: + port: 5601 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 20 + successThreshold: 1 + initialDelaySeconds: 10 + livenessProbe: + tcpSocket: + port: 5601 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 10 + successThreshold: 1 + initialDelaySeconds: 10 + readinessProbe: + tcpSocket: + port: 5601 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 10 + successThreshold: 1 + initialDelaySeconds: 10 + ports: + - containerPort: 5601 + name: http + protocol: TCP \ No newline at end of file diff --git a/deploy/opensearch/templates/clusterversion.yaml b/deploy/opensearch/templates/clusterversion.yaml new file mode 100644 index 000000000..b2bc2c18c --- /dev/null +++ b/deploy/opensearch/templates/clusterversion.yaml @@ -0,0 +1,31 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ClusterVersion +metadata: + name: opensearch-{{ default .Chart.AppVersion .Values.clusterVersionOverride }} + labels: + {{- include "opensearch.labels" . | nindent 4 }} +spec: + clusterDefinitionRef: opensearch + componentVersions: + - componentDefRef: opensearch + versionsContext: + initContainers: + - name: fsgroup-volume + image: {{ .Values.image.registry | default "docker.io" }}/busybox:latest + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + - name: sysctl + image: {{ .Values.image.registry | default "docker.io" }}/busybox:latest + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + containers: + - name: opensearch + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + - name: opensearch-master-graceful-termination-handler + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} + - componentDefRef: dashboard + versionsContext: + containers: + - name: dashboard + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.dashboard.repository }}:{{ default .Chart.AppVersion .Values.image.dashboard.tag }} + imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} diff --git a/deploy/opensearch/templates/configconstraint.yaml b/deploy/opensearch/templates/configconstraint.yaml new file mode 100644 index 000000000..1e90fa824 --- /dev/null +++ b/deploy/opensearch/templates/configconstraint.yaml @@ -0,0 +1,9 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ConfigConstraint +metadata: + name: opensearch-config-constraint + labels: + {{- include "opensearch.labels" . | nindent 4 }} +spec: + formatterConfig: + format: yaml \ No newline at end of file diff --git a/deploy/opensearch/templates/configmap.yaml b/deploy/opensearch/templates/configmap.yaml new file mode 100644 index 000000000..d4d752aac --- /dev/null +++ b/deploy/opensearch/templates/configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: opensearch-config-template + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "opensearch.labels" . | nindent 4 }} +data: + opensearch.yaml: |- + {{- .Files.Get "configs/opensearch.yaml.tpl" | nindent 4 }} \ No newline at end of file diff --git a/deploy/opensearch/values.yaml b/deploy/opensearch/values.yaml new file mode 100644 index 000000000..6e31c2dcb --- /dev/null +++ b/deploy/opensearch/values.yaml @@ -0,0 +1,60 @@ +# Default values for opensearch. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + registry: registry.cn-hangzhou.aliyuncs.com + repository: opensearchproject/opensearch + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + dashboard: + repository: opensearchproject/opensearch-dashboards + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" +clusterVersionOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} From 8cc782634eae714ff4cdb2c4fd3f0e57b974d874 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Wed, 17 May 2023 11:33:53 +0800 Subject: [PATCH 313/439] fix: missing defaultInstallValues field (#3291) --- deploy/helm/templates/addons/opensearch-addon.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deploy/helm/templates/addons/opensearch-addon.yaml b/deploy/helm/templates/addons/opensearch-addon.yaml index e22c601a6..4d24c35dc 100644 --- a/deploy/helm/templates/addons/opensearch-addon.yaml +++ b/deploy/helm/templates/addons/opensearch-addon.yaml @@ -20,3 +20,8 @@ spec: installable: autoInstall: false + defaultInstallValues: + - enabled: false + {{- with .Values.tolerations }} + tolerations: {{ toJson . | quote }} + {{- end }} From 980817f543ab079ab63b3a36278e072c2af77dce Mon Sep 17 00:00:00 2001 From: runsun Date: Wed, 17 May 2023 11:52:59 +0800 Subject: [PATCH 314/439] chore: fix some minor issues for grafana dashboards (#3290) --- deploy/helm/dashboards/node-exporter.json | 98 +++++----- .../helm/dashboards/postgresql-overview.json | 167 +++++++++++------- 2 files changed, 156 insertions(+), 109 deletions(-) diff --git a/deploy/helm/dashboards/node-exporter.json b/deploy/helm/dashboards/node-exporter.json index 1f820987a..81eac5297 100644 --- a/deploy/helm/dashboards/node-exporter.json +++ b/deploy/helm/dashboards/node-exporter.json @@ -26,7 +26,7 @@ "fiscalYearStartMonth": 0, "gnetId": 1860, "graphTooltip": 1, - "id": 23, + "id": 16, "links": [ { "asDropdown": false, @@ -13281,7 +13281,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -13610,7 +13611,7 @@ "h": 6, "w": 8, "x": 0, - "y": 8 + "y": 21 }, "id": 9, "links": [], @@ -13708,7 +13709,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -14037,7 +14039,7 @@ "h": 6, "w": 8, "x": 8, - "y": 8 + "y": 21 }, "id": 433, "links": [], @@ -14137,7 +14139,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -14466,7 +14469,7 @@ "h": 6, "w": 8, "x": 16, - "y": 8 + "y": 21 }, "id": 33, "links": [], @@ -14568,7 +14571,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -14897,7 +14901,7 @@ "h": 6, "w": 8, "x": 0, - "y": 14 + "y": 27 }, "id": 37, "links": [], @@ -15001,7 +15005,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -15330,7 +15335,7 @@ "h": 6, "w": 8, "x": 8, - "y": 14 + "y": 27 }, "id": 133, "links": [], @@ -15426,7 +15431,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -15743,7 +15749,7 @@ "h": 6, "w": 8, "x": 16, - "y": 14 + "y": 27 }, "id": 301, "links": [], @@ -15842,7 +15848,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -16159,7 +16166,7 @@ "h": 6, "w": 8, "x": 0, - "y": 20 + "y": 33 }, "id": 36, "links": [], @@ -16258,7 +16265,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -16575,7 +16583,7 @@ "h": 6, "w": 8, "x": 8, - "y": 20 + "y": 33 }, "id": 35, "links": [], @@ -16662,7 +16670,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -16979,7 +16988,7 @@ "h": 6, "w": 8, "x": 16, - "y": 20 + "y": 33 }, "id": 34, "links": [], @@ -17080,7 +17089,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -17096,7 +17106,7 @@ "h": 6, "w": 8, "x": 0, - "y": 33 + "y": 22 }, "id": 156, "links": [], @@ -17183,7 +17193,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -17199,7 +17210,7 @@ "h": 6, "w": 8, "x": 8, - "y": 33 + "y": 22 }, "id": 43, "links": [], @@ -17314,7 +17325,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -17330,7 +17342,7 @@ "h": 6, "w": 8, "x": 16, - "y": 33 + "y": 22 }, "id": 28, "links": [], @@ -17429,7 +17441,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -17445,7 +17458,7 @@ "h": 6, "w": 8, "x": 0, - "y": 39 + "y": 28 }, "id": 219, "links": [], @@ -17535,7 +17548,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -17551,7 +17565,7 @@ "h": 6, "w": 8, "x": 8, - "y": 39 + "y": 28 }, "id": 41, "links": [], @@ -17642,7 +17656,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -17674,7 +17689,7 @@ "h": 6, "w": 8, "x": 16, - "y": 39 + "y": 28 }, "id": 44, "links": [], @@ -22273,21 +22288,22 @@ "schemaVersion": 37, "style": "dark", "tags": [ - "linux" + "linux", + "node" ], "templating": { "list": [ { "current": { "selected": false, - "text": "192.168.49.2:9100", - "value": "192.168.49.2:9100" + "text": "172.21.0.3:9100", + "value": "172.21.0.3:9100" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(node_uname_info{nodename=\"$Node\"}, instance)", + "definition": "label_values(node_uname_info{node=\"$Node\"}, instance)", "hide": 2, "includeAll": false, "label": "Host:", @@ -22295,7 +22311,7 @@ "name": "node", "options": [], "query": { - "query": "label_values(node_uname_info{nodename=\"$Node\"}, instance)", + "query": "label_values(node_uname_info{node=\"$Node\"}, instance)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -22350,14 +22366,14 @@ { "current": { "selected": false, - "text": "minikube", - "value": "minikube" + "text": "k3d-k3s-default-server-0", + "value": "k3d-k3s-default-server-0" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(node_uname_info{}, nodename)", + "definition": "label_values(node_uname_info{}, node)", "hide": 0, "includeAll": false, "label": "node", @@ -22365,7 +22381,7 @@ "name": "Node", "options": [], "query": { - "query": "label_values(node_uname_info{}, nodename)", + "query": "label_values(node_uname_info{}, node)", "refId": "StandardVariableQuery" }, "refresh": 1, @@ -22410,4 +22426,4 @@ "uid": "nodeexporter1af4132", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/deploy/helm/dashboards/postgresql-overview.json b/deploy/helm/dashboards/postgresql-overview.json index 0671209ed..608b50a13 100644 --- a/deploy/helm/dashboards/postgresql-overview.json +++ b/deploy/helm/dashboards/postgresql-overview.json @@ -26,6 +26,7 @@ "fiscalYearStartMonth": 0, "gnetId": 11323, "graphTooltip": 1, + "id": 14, "links": [ { "asDropdown": false, @@ -1416,7 +1417,7 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "topk(5, pg_stat_statements_calls{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})", + "expr": "topk(5, pg_stat_statements_calls{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"})", "format": "table", "instant": true, "interval": "1m", @@ -1578,7 +1579,7 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "topk(5, pg_stat_statements_mean_exec_time_seconds{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\"})", + "expr": "topk(5, pg_stat_statements_mean_exec_time_seconds{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"})", "format": "table", "instant": true, "interval": "1m", @@ -1692,7 +1693,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1708,7 +1710,7 @@ "h": 8, "w": 12, "x": 0, - "y": 2 + "y": 31 }, "id": 413, "links": [], @@ -1801,7 +1803,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1817,7 +1820,7 @@ "h": 8, "w": 12, "x": 12, - "y": 2 + "y": 31 }, "id": 414, "links": [], @@ -1924,7 +1927,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1940,7 +1944,7 @@ "h": 8, "w": 12, "x": 0, - "y": 3 + "y": 32 }, "id": 394, "links": [], @@ -2033,7 +2037,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2049,7 +2054,7 @@ "h": 8, "w": 12, "x": 12, - "y": 3 + "y": 32 }, "id": 395, "links": [], @@ -2142,7 +2147,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2158,7 +2164,7 @@ "h": 8, "w": 12, "x": 0, - "y": 11 + "y": 40 }, "id": 396, "links": [], @@ -2251,7 +2257,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2267,7 +2274,7 @@ "h": 8, "w": 12, "x": 12, - "y": 11 + "y": 40 }, "id": 397, "links": [], @@ -2360,7 +2367,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -2376,7 +2384,7 @@ "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 48 }, "id": 398, "links": [], @@ -2532,7 +2540,7 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "rate(pg_stat_user_tables_idx_scan{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval]) > 0", + "expr": "rate(pg_stat_user_tables_idx_scan{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval])", "format": "time_series", "instant": false, "interval": "1m", @@ -2644,7 +2652,7 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "rate(pg_stat_user_tables_idx_tup_fetch{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval]) / rate(pg_stat_user_tables_idx_scan{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval]) > 0", + "expr": "rate(pg_stat_user_tables_idx_tup_fetch{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval]) / rate(pg_stat_user_tables_idx_scan{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval])", "format": "time_series", "instant": false, "interval": "1m", @@ -2756,7 +2764,7 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "rate(pg_stat_user_tables_seq_scan{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval]) > 0", + "expr": "rate(pg_stat_user_tables_seq_scan{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval])", "format": "time_series", "instant": false, "interval": "1m", @@ -2868,7 +2876,7 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "rate(pg_stat_user_tables_seq_tup_read{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval]) / rate(pg_stat_user_tables_seq_scan{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval]) > 0", + "expr": "rate(pg_stat_user_tables_seq_tup_read{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval]) / rate(pg_stat_user_tables_seq_scan{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}[$__rate_interval])", "format": "time_series", "instant": false, "interval": "1m", @@ -2980,7 +2988,7 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "pg_stat_user_tables_n_live_tup{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"} > 0", + "expr": "pg_stat_user_tables_n_live_tup{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}", "format": "time_series", "instant": false, "interval": "1m", @@ -3092,7 +3100,7 @@ "editorMode": "code", "errors": {}, "exemplar": false, - "expr": "pg_stat_user_tables_n_dead_tup{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"} > 0", + "expr": "pg_stat_user_tables_n_dead_tup{namespace=~\"$namespace\", app_kubernetes_io_instance=~\"$cluster\",pod=~\"$instance\",datname=~\"$database\"}", "format": "time_series", "instant": false, "interval": "1m", @@ -3169,7 +3177,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3185,7 +3194,7 @@ "h": 8, "w": 12, "x": 0, - "y": 5 + "y": 34 }, "id": 389, "links": [], @@ -3278,7 +3287,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3294,7 +3304,7 @@ "h": 8, "w": 12, "x": 12, - "y": 5 + "y": 34 }, "id": 390, "links": [], @@ -3387,7 +3397,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3403,7 +3414,7 @@ "h": 8, "w": 12, "x": 0, - "y": 13 + "y": 42 }, "id": 391, "links": [], @@ -3496,7 +3507,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3512,7 +3524,7 @@ "h": 8, "w": 12, "x": 12, - "y": 13 + "y": 42 }, "id": 423, "links": [], @@ -3641,7 +3653,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3657,7 +3670,7 @@ "h": 8, "w": 12, "x": 0, - "y": 6 + "y": 35 }, "id": 92, "links": [], @@ -3750,7 +3763,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3766,7 +3780,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 35 }, "id": 384, "links": [], @@ -3859,7 +3873,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3875,7 +3890,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 43 }, "id": 385, "links": [], @@ -3968,7 +3983,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -3984,7 +4000,7 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 43 }, "id": 386, "links": [], @@ -4175,7 +4191,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4191,7 +4208,7 @@ "h": 8, "w": 12, "x": 0, - "y": 7 + "y": 36 }, "id": 408, "links": [], @@ -4284,7 +4301,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4300,7 +4318,7 @@ "h": 8, "w": 12, "x": 12, - "y": 7 + "y": 36 }, "id": 409, "links": [], @@ -4393,7 +4411,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4409,7 +4428,7 @@ "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 44 }, "id": 401, "links": [], @@ -4502,7 +4521,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4518,7 +4538,7 @@ "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 44 }, "id": 402, "links": [], @@ -4611,7 +4631,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4627,7 +4648,7 @@ "h": 8, "w": 12, "x": 0, - "y": 23 + "y": 52 }, "id": 403, "links": [], @@ -4796,7 +4817,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4812,7 +4834,7 @@ "h": 8, "w": 12, "x": 12, - "y": 23 + "y": 52 }, "id": 410, "links": [], @@ -4924,7 +4946,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -4940,7 +4963,7 @@ "h": 8, "w": 12, "x": 0, - "y": 31 + "y": 60 }, "id": 415, "links": [], @@ -5066,7 +5089,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5082,7 +5106,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 37 }, "id": 406, "links": [], @@ -5175,7 +5199,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5191,7 +5216,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 37 }, "id": 407, "links": [], @@ -5298,7 +5323,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5314,7 +5340,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 38 }, "id": 418, "links": [], @@ -5421,7 +5447,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5437,7 +5464,7 @@ "h": 8, "w": 12, "x": 0, - "y": 10 + "y": 39 }, "id": 486, "links": [], @@ -5525,7 +5552,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5541,7 +5569,7 @@ "h": 8, "w": 12, "x": 12, - "y": 10 + "y": 39 }, "id": 484, "links": [], @@ -5629,7 +5657,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5645,7 +5674,7 @@ "h": 8, "w": 12, "x": 0, - "y": 18 + "y": 47 }, "id": 483, "links": [], @@ -5730,7 +5759,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5746,7 +5776,7 @@ "h": 8, "w": 12, "x": 12, - "y": 18 + "y": 47 }, "id": 463, "options": { @@ -5825,7 +5855,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -5841,7 +5872,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 55 }, "id": 485, "links": [], From 86472a1c57050536ee8149dce4f3e1364d2633d6 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Wed, 17 May 2023 14:19:21 +0800 Subject: [PATCH 315/439] chore: improve ApeCloud mysql description (#3292) --- deploy/apecloud-mysql/Chart.yaml | 8 ++------ deploy/helm/templates/addons/apecloud-mysql-addon.yaml | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/deploy/apecloud-mysql/Chart.yaml b/deploy/apecloud-mysql/Chart.yaml index 17da6b052..930fe8132 100644 --- a/deploy/apecloud-mysql/Chart.yaml +++ b/deploy/apecloud-mysql/Chart.yaml @@ -1,11 +1,7 @@ apiVersion: v2 name: apecloud-mysql -description: ApeCloud MySQL is fully compatible with MySQL syntax and supports single-availability - zone deployment, double-availability zone deployment, and multiple-availability zone deployment. - Based on the Paxos consensus protocol, ApeCloud MySQL realizes automatic leader election, log - synchronization, and strict consistency. ApeCloud MySQL is the optimum choice for the production - environment since it can automatically perform a high-availability switch to maintain business continuity - when container exceptions, server exceptions, or availability zone exceptions occur. +description: ApeCloud MySQL is a database that is compatible with MySQL syntax and achieves high availability + through the utilization of the RAFT consensus protocol. type: application diff --git a/deploy/helm/templates/addons/apecloud-mysql-addon.yaml b/deploy/helm/templates/addons/apecloud-mysql-addon.yaml index 133523602..e2a812881 100644 --- a/deploy/helm/templates/addons/apecloud-mysql-addon.yaml +++ b/deploy/helm/templates/addons/apecloud-mysql-addon.yaml @@ -10,12 +10,8 @@ metadata: helm.sh/resource-policy: keep {{- end }} spec: - description: 'ApeCloud MySQL is fully compatible with MySQL syntax and supports single-availability - zone deployment, double-availability zone deployment, and multiple-availability zone deployment. - Based on the Paxos consensus protocol, ApeCloud MySQL realizes automatic leader election, log - synchronization, and strict consistency. ApeCloud MySQL is the optimum choice for the production - environment since it can automatically perform a high-availability switch to maintain business continuity - when container exceptions, server exceptions, or availability zone exceptions occur.' + description: 'ApeCloud MySQL is a database that is compatible with MySQL syntax and achieves high availability + through the utilization of the RAFT consensus protocol.' type: Helm From 55280a167b9125a0e6be426e3fb20f78934ca050 Mon Sep 17 00:00:00 2001 From: xuriwuyun Date: Wed, 17 May 2023 14:29:06 +0800 Subject: [PATCH 316/439] fix: probe event lost (#3172) Co-authored-by: L.DongMing --- cmd/probe/internal/binding/base.go | 29 ++--- .../internal/binding/mysql/mysql_test.go | 2 +- cmd/probe/internal/binding/redis/redis.go | 4 +- cmd/probe/internal/binding/utils.go | 96 ++++++++++++++++ controllers/apps/cluster_controller.go | 5 +- controllers/apps/cluster_controller_test.go | 7 +- controllers/k8score/event_controller.go | 13 +-- controllers/k8score/event_controller_test.go | 9 +- .../templates/_helpers.tpl | 6 +- .../templates/cluster.yaml | 1 + .../templates/role.yaml | 14 +++ .../templates/rolebinding.yaml | 14 +++ .../templates/serviceaccount.yaml | 6 + deploy/apecloud-mysql-cluster/values.yaml | 6 +- .../templates/_helpers.tpl | 6 +- .../templates/cluster.yaml | 1 + .../templates/role.yaml | 14 +++ .../templates/rolebinding.yaml | 14 +++ .../templates/serviceaccount.yaml | 6 + .../apecloud-mysql-scale-cluster/values.yaml | 6 +- .../clickhouse-cluster/templates/_helpers.tpl | 6 +- .../clickhouse-cluster/templates/cluster.yaml | 1 + deploy/clickhouse-cluster/templates/role.yaml | 14 +++ .../templates/rolebinding.yaml | 14 +++ .../templates/serviceaccount.yaml | 6 + deploy/clickhouse-cluster/values.yaml | 4 + deploy/etcd-cluster/templates/_helpers.tpl | 6 +- deploy/etcd-cluster/templates/cluster.yaml | 3 +- deploy/etcd-cluster/templates/role.yaml | 14 +++ .../etcd-cluster/templates/rolebinding.yaml | 14 +++ .../templates/serviceaccount.yaml | 6 + deploy/etcd-cluster/values.yaml | 6 +- deploy/kafka-cluster/templates/_helpers.tpl | 6 +- deploy/kafka-cluster/templates/role.yaml | 14 +++ .../kafka-cluster/templates/rolebinding.yaml | 14 +++ .../templates/serviceaccount.yaml | 6 + deploy/kafka-cluster/values.yaml | 4 + deploy/mongodb-cluster/Chart.yaml | 2 +- deploy/mongodb-cluster/templates/NOTES.txt | 40 +++---- deploy/mongodb-cluster/templates/_helpers.tpl | 6 +- .../mongodb-cluster/templates/replicaset.yaml | 1 + deploy/mongodb-cluster/templates/role.yaml | 14 +++ .../templates/rolebinding.yaml | 14 +++ .../templates/serviceaccount.yaml | 6 + deploy/mongodb-cluster/values.yaml | 4 + deploy/mongodb/Chart.yaml | 2 +- deploy/postgresql-cluster/templates/role.yaml | 10 +- .../templates/rolebinding.yaml | 8 +- .../templates/serviceaccount.yaml | 4 +- deploy/redis-cluster/templates/_helpers.tpl | 6 +- deploy/redis-cluster/templates/cluster.yaml | 1 + deploy/redis-cluster/templates/role.yaml | 14 +++ .../redis-cluster/templates/rolebinding.yaml | 14 +++ .../templates/serviceaccount.yaml | 6 + deploy/redis-cluster/values.yaml | 4 + docs/release_notes/v0.6.0/_category_.yml | 4 + docs/release_notes/v0.6.0/v0.6.0.md | 24 ++++ internal/cli/cmd/cluster/cluster_test.go | 17 +-- internal/cli/cmd/cluster/create.go | 107 +++++++++++------- internal/cli/cmd/cluster/delete.go | 10 +- internal/controller/builder/builder.go | 1 + 61 files changed, 548 insertions(+), 168 deletions(-) create mode 100644 deploy/apecloud-mysql-cluster/templates/role.yaml create mode 100644 deploy/apecloud-mysql-cluster/templates/rolebinding.yaml create mode 100644 deploy/apecloud-mysql-cluster/templates/serviceaccount.yaml create mode 100644 deploy/apecloud-mysql-scale-cluster/templates/role.yaml create mode 100644 deploy/apecloud-mysql-scale-cluster/templates/rolebinding.yaml create mode 100644 deploy/apecloud-mysql-scale-cluster/templates/serviceaccount.yaml create mode 100644 deploy/clickhouse-cluster/templates/role.yaml create mode 100644 deploy/clickhouse-cluster/templates/rolebinding.yaml create mode 100644 deploy/clickhouse-cluster/templates/serviceaccount.yaml create mode 100644 deploy/etcd-cluster/templates/role.yaml create mode 100644 deploy/etcd-cluster/templates/rolebinding.yaml create mode 100644 deploy/etcd-cluster/templates/serviceaccount.yaml create mode 100644 deploy/kafka-cluster/templates/role.yaml create mode 100644 deploy/kafka-cluster/templates/rolebinding.yaml create mode 100644 deploy/kafka-cluster/templates/serviceaccount.yaml create mode 100644 deploy/mongodb-cluster/templates/role.yaml create mode 100644 deploy/mongodb-cluster/templates/rolebinding.yaml create mode 100644 deploy/mongodb-cluster/templates/serviceaccount.yaml create mode 100644 deploy/redis-cluster/templates/role.yaml create mode 100644 deploy/redis-cluster/templates/rolebinding.yaml create mode 100644 deploy/redis-cluster/templates/serviceaccount.yaml create mode 100644 docs/release_notes/v0.6.0/_category_.yml create mode 100644 docs/release_notes/v0.6.0/v0.6.0.md diff --git a/cmd/probe/internal/binding/base.go b/cmd/probe/internal/binding/base.go index 4c2c592e9..a8347b69e 100644 --- a/cmd/probe/internal/binding/base.go +++ b/cmd/probe/internal/binding/base.go @@ -188,9 +188,10 @@ func (ops *BaseOperations) Invoke(ctx context.Context, req *bindings.InvokeReque func (ops *BaseOperations) CheckRoleOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { opsRes := OpsResult{} + opsRes["operation"] = CheckRoleOperation opsRes["originalRole"] = ops.OriRole if ops.GetRole == nil { - message := fmt.Sprintf("roleCheck operation is not implemented for %v", ops.DBType) + message := fmt.Sprintf("checkRole operation is not implemented for %v", ops.DBType) ops.Logger.Errorf(message) opsRes["event"] = OperationNotImplemented opsRes["message"] = message @@ -200,12 +201,12 @@ func (ops *BaseOperations) CheckRoleOps(ctx context.Context, req *bindings.Invok role, err := ops.GetRole(ctx, req, resp) if err != nil { - ops.Logger.Infof("error executing roleCheck: %v", err) + ops.Logger.Infof("error executing checkRole: %v", err) opsRes["event"] = OperationFailed opsRes["message"] = err.Error() if ops.CheckRoleFailedCount%ops.FailedEventReportFrequency == 0 { ops.Logger.Infof("role checks failed %v times continuously", ops.CheckRoleFailedCount) - resp.Metadata[StatusCode] = OperationFailedHTTPCode + SentProbeEvent(ctx, opsRes, ops.Logger) } ops.CheckRoleFailedCount++ return opsRes, nil @@ -222,9 +223,7 @@ func (ops *BaseOperations) CheckRoleOps(ctx context.Context, req *bindings.Invok opsRes["role"] = role if ops.OriRole != role { ops.OriRole = role - ops.RoleUnchangedCount = 0 - } else { - ops.RoleUnchangedCount++ + SentProbeEvent(ctx, opsRes, ops.Logger) } // RoleUnchangedCount is the count of consecutive role unchanged checks. @@ -232,16 +231,16 @@ func (ops *BaseOperations) CheckRoleOps(ctx context.Context, req *bindings.Invok // then the roleCheck event will be reported at roleEventReportFrequency so that the event controller // can always get relevant roleCheck events in order to maintain the pod label accurately, even in cases // of roleChanged events being lost or the pod role label being deleted or updated incorrectly. - if ops.RoleUnchangedCount < ops.RoleDetectionThreshold && ops.RoleUnchangedCount%roleEventReportFrequency == 0 { - resp.Metadata[StatusCode] = OperationFailedHTTPCode - } + // if ops.RoleUnchangedCount < ops.RoleDetectionThreshold && ops.RoleUnchangedCount%roleEventReportFrequency == 0 { + // resp.Metadata[StatusCode] = OperationFailedHTTPCode + // } return opsRes, nil } func (ops *BaseOperations) GetRoleOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { opsRes := OpsResult{} if ops.GetRole == nil { - message := fmt.Sprintf("roleCheck operation is not implemented for %v", ops.DBType) + message := fmt.Sprintf("getRole operation is not implemented for %v", ops.DBType) ops.Logger.Errorf(message) opsRes["event"] = OperationNotImplemented opsRes["message"] = message @@ -251,12 +250,12 @@ func (ops *BaseOperations) GetRoleOps(ctx context.Context, req *bindings.InvokeR role, err := ops.GetRole(ctx, req, resp) if err != nil { - ops.Logger.Infof("error executing roleCheck: %v", err) + ops.Logger.Infof("error executing getRole: %v", err) opsRes["event"] = OperationFailed opsRes["message"] = err.Error() if ops.CheckRoleFailedCount%ops.FailedEventReportFrequency == 0 { - ops.Logger.Infof("role checks failed %v times continuously", ops.CheckRoleFailedCount) - resp.Metadata[StatusCode] = OperationFailedHTTPCode + ops.Logger.Infof("getRole failed %v times continuously", ops.CheckRoleFailedCount) + // resp.Metadata[StatusCode] = OperationFailedHTTPCode } ops.CheckRoleFailedCount++ return opsRes, nil @@ -295,6 +294,7 @@ func (ops *BaseOperations) roleValidate(role string) (bool, string) { func (ops *BaseOperations) CheckRunningOps(ctx context.Context, req *bindings.InvokeRequest, resp *bindings.InvokeResponse) (OpsResult, error) { var message string opsRes := OpsResult{} + opsRes["operation"] = CheckRunningOperation host := net.JoinHostPort(ops.DBAddress, strconv.Itoa(ops.DBPort)) // sql exec timeout need to be less than httpget's timeout which default is 1s. @@ -306,7 +306,8 @@ func (ops *BaseOperations) CheckRunningOps(ctx context.Context, req *bindings.In opsRes["message"] = message if ops.CheckRunningFailedCount%ops.FailedEventReportFrequency == 0 { ops.Logger.Infof("running checks failed %v times continuously", ops.CheckRunningFailedCount) - resp.Metadata[StatusCode] = OperationFailedHTTPCode + // resp.Metadata[StatusCode] = OperationFailedHTTPCode + SentProbeEvent(ctx, opsRes, ops.Logger) } ops.CheckRunningFailedCount++ return opsRes, nil diff --git a/cmd/probe/internal/binding/mysql/mysql_test.go b/cmd/probe/internal/binding/mysql/mysql_test.go index c7ce71952..8ebb202a4 100644 --- a/cmd/probe/internal/binding/mysql/mysql_test.go +++ b/cmd/probe/internal/binding/mysql/mysql_test.go @@ -84,7 +84,7 @@ func TestInit(t *testing.T) { func TestInitDelay(t *testing.T) { // Initialize a new instance of MysqlOperations. mysqlOps, _, _ := mockDatabase(t) - mysqlOps.initIfNeed() + // mysqlOps.initIfNeed() t.Run("Invalid url", func(t *testing.T) { mysqlOps.db = nil mysqlOps.initIfNeed() diff --git a/cmd/probe/internal/binding/redis/redis.go b/cmd/probe/internal/binding/redis/redis.go index 4f8d5489c..bbb5b3afa 100644 --- a/cmd/probe/internal/binding/redis/redis.go +++ b/cmd/probe/internal/binding/redis/redis.go @@ -550,8 +550,10 @@ func (r *Redis) priv2Role(commands string) RoleType { } func (r *Redis) Close() error { + if r.cancel == nil { + return nil + } r.cancel() - return r.client.Close() } diff --git a/cmd/probe/internal/binding/utils.go b/cmd/probe/internal/binding/utils.go index 6d0441baf..35eb7ab7e 100644 --- a/cmd/probe/internal/binding/utils.go +++ b/cmd/probe/internal/binding/utils.go @@ -20,12 +20,23 @@ along with this program. If not, see . package binding import ( + "bytes" "context" "encoding/json" "fmt" + "os" + "text/template" "time" "github.com/dapr/components-contrib/bindings" + "github.com/dapr/kit/logger" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" ) type UserInfo struct { @@ -212,3 +223,88 @@ func String2RoleType(roleName string) RoleType { } return CustomizedRole } + +func SentProbeEvent(ctx context.Context, opsResult OpsResult, log logger.Logger) { + log.Infof("send event: %v", opsResult) + event, err := createProbeEvent(opsResult) + if err != nil { + log.Infof("generate event failed: %v", err) + return + } + + config, err := rest.InClusterConfig() + if err != nil { + log.Infof("get k8s client config failed: %v", err) + return + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Infof("k8s client create failed: %v", err) + return + } + namespace := os.Getenv("KB_NAMESPACE") + for i := 0; i < 3; i++ { + _, err = clientset.CoreV1().Events(namespace).Create(ctx, event, metav1.CreateOptions{}) + if err == nil { + break + } + log.Infof("send event failed: %v", err) + } +} + +func createProbeEvent(opsResult OpsResult) (*corev1.Event, error) { + eventTmpl := ` +apiVersion: v1 +kind: Event +metadata: + name: {{ .PodName }}.{{ .EventSeq }} + namespace: {{ .Namespace }} +involvedObject: + apiVersion: v1 + fieldPath: spec.containers{sqlchannel} + kind: Pod + name: {{ .PodName }} + namespace: {{ .Namespace }} +reason: RoleChanged +type: Normal +source: + component: sqlchannel +` + + // get pod object + podName := os.Getenv("KB_POD_NAME") + podUID := os.Getenv("KB_POD_UID") + nodeName := os.Getenv("KB_NODENAME") + namespace := os.Getenv("KB_NAMESPACE") + msg, _ := json.Marshal(opsResult) + seq := rand.String(16) + roleValue := map[string]string{ + "PodName": podName, + "Namespace": namespace, + "EventSeq": seq, + } + tmpl, err := template.New("event-tmpl").Parse(eventTmpl) + if err != nil { + return nil, err + } + buf := new(bytes.Buffer) + err = tmpl.Execute(buf, roleValue) + if err != nil { + return nil, err + } + + event := &corev1.Event{} + _, _, err = scheme.Codecs.UniversalDeserializer().Decode(buf.Bytes(), nil, event) + if err != nil { + return nil, err + } + event.Message = string(msg) + event.InvolvedObject.UID = types.UID(podUID) + event.Source.Host = nodeName + event.Reason = string(opsResult["operation"].(bindings.OperationKind)) + event.FirstTimestamp = metav1.Now() + event.LastTimestamp = metav1.Now() + + return event, nil +} diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index 2b450bfe5..ec8f1d2eb 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -38,6 +38,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + probeutil "github.com/apecloud/kubeblocks/cmd/probe/util" "github.com/apecloud/kubeblocks/controllers/k8score" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/lifecycle" @@ -247,7 +248,7 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { // Handle is the event handler for the cluster status event. func (r *ClusterStatusEventHandler) Handle(cli client.Client, reqCtx intctrlutil.RequestCtx, recorder record.EventRecorder, event *corev1.Event) error { - if event.InvolvedObject.FieldPath != constant.ProbeCheckRolePath { + if event.Reason != string(probeutil.CheckRoleOperation) { return handleEventForClusterStatus(reqCtx.Ctx, cli, recorder, event) } @@ -259,7 +260,7 @@ func (r *ClusterStatusEventHandler) Handle(cli client.Client, reqCtx intctrlutil } // if probe message event is checkRoleFailed, it means the cluster is abnormal, need to handle the cluster status - if message.Event == k8score.ProbeEventCheckRoleFailed { + if message.Event == probeutil.OperationFailed { return handleEventForClusterStatus(reqCtx.Ctx, cli, recorder, event) } return nil diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 8c6be115d..b88471a27 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -47,6 +47,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" + probeutil "github.com/apecloud/kubeblocks/cmd/probe/util" "github.com/apecloud/kubeblocks/controllers/apps/components/replication" "github.com/apecloud/kubeblocks/controllers/apps/components/util" "github.com/apecloud/kubeblocks/internal/constant" @@ -900,8 +901,8 @@ var _ = Describe("Cluster Controller", func() { Name: pod.Name + "-event", Namespace: testCtx.DefaultNamespace, }, - Reason: "Unhealthy", - Message: `Readiness probe failed: {"event":"Success","originalRole":"Leader","role":"Follower"}`, + Reason: string(probeutil.CheckRoleOperation), + Message: `{"event":"Success","originalRole":"Leader","role":"Follower"}`, InvolvedObject: corev1.ObjectReference{ Name: pod.Name, Namespace: testCtx.DefaultNamespace, @@ -911,7 +912,7 @@ var _ = Describe("Cluster Controller", func() { } events = append(events, event) } - events[0].Message = `Readiness probe failed: {"event":"Success","originalRole":"Leader","role":"Leader"}` + events[0].Message = `{"event":"Success","originalRole":"Leader","role":"Leader"}` return events } diff --git a/controllers/k8score/event_controller.go b/controllers/k8score/event_controller.go index a0c661cf7..a4c5e8d37 100644 --- a/controllers/k8score/event_controller.go +++ b/controllers/k8score/event_controller.go @@ -22,7 +22,6 @@ package k8score import ( "context" "encoding/json" - "regexp" "strings" corev1 "k8s.io/api/core/v1" @@ -35,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + probeutil "github.com/apecloud/kubeblocks/cmd/probe/util" "github.com/apecloud/kubeblocks/controllers/apps/components/consensus" "github.com/apecloud/kubeblocks/controllers/apps/components/replication" componentutil "github.com/apecloud/kubeblocks/controllers/apps/components/util" @@ -115,7 +115,7 @@ func (r *EventReconciler) SetupWithManager(mgr ctrl.Manager) error { // Handle handles role changed event. func (r *RoleChangeEventHandler) Handle(cli client.Client, reqCtx intctrlutil.RequestCtx, recorder record.EventRecorder, event *corev1.Event) error { - if event.InvolvedObject.FieldPath != constant.ProbeCheckRolePath { + if event.Reason != string(probeutil.CheckRoleOperation) { return nil } var ( @@ -195,14 +195,7 @@ func handleRoleChangedEvent(cli client.Client, reqCtx intctrlutil.RequestCtx, re // ParseProbeEventMessage parses probe event message. func ParseProbeEventMessage(reqCtx intctrlutil.RequestCtx, event *corev1.Event) *ProbeMessage { message := &ProbeMessage{} - re := regexp.MustCompile(`Readiness probe failed: ({.*})`) - matches := re.FindStringSubmatch(event.Message) - if len(matches) != 2 { - reqCtx.Log.Info("parser Readiness probe event message failed", "message", event.Message) - return nil - } - msg := matches[1] - err := json.Unmarshal([]byte(msg), message) + err := json.Unmarshal([]byte(event.Message), message) if err != nil { // not role related message, ignore it reqCtx.Log.Info("not role message", "message", event.Message, "error", err) diff --git a/controllers/k8score/event_controller_test.go b/controllers/k8score/event_controller_test.go index 89e3a3c29..d96a7db1d 100644 --- a/controllers/k8score/event_controller_test.go +++ b/controllers/k8score/event_controller_test.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + probeutil "github.com/apecloud/kubeblocks/cmd/probe/util" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" "github.com/apecloud/kubeblocks/internal/generics" testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" @@ -125,7 +126,7 @@ involvedObject: kind: Pod name: {{ .PodName }} namespace: default -message: "Readiness probe failed: {\"event\":\"roleChanged\",\"originalRole\":\"secondary\",\"role\":\"{{ .Role }}\"}" +message: "{\"event\":\"roleChanged\",\"originalRole\":\"secondary\",\"role\":\"{{ .Role }}\"}" reason: RoleChanged type: Normal ` @@ -149,12 +150,14 @@ type: Normal return nil, err } - event, _, err := scheme.Codecs.UniversalDeserializer().Decode(buf.Bytes(), nil, nil) + event := &corev1.Event{} + _, _, err = scheme.Codecs.UniversalDeserializer().Decode(buf.Bytes(), nil, event) if err != nil { return nil, err } + event.Reason = string(probeutil.CheckRoleOperation) - return event.(*corev1.Event), nil + return event, nil } func createInvolvedPod(name string) corev1.Pod { diff --git a/deploy/apecloud-mysql-cluster/templates/_helpers.tpl b/deploy/apecloud-mysql-cluster/templates/_helpers.tpl index 90137c9f9..6b1d05a2c 100644 --- a/deploy/apecloud-mysql-cluster/templates/_helpers.tpl +++ b/deploy/apecloud-mysql-cluster/templates/_helpers.tpl @@ -54,9 +54,5 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "apecloud-mysql-cluster.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "apecloud-mysql-cluster.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- default ((printf "kb-sa-%s" .Release.Name) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/apecloud-mysql-cluster/templates/cluster.yaml b/deploy/apecloud-mysql-cluster/templates/cluster.yaml index 09cb84d27..75675a0cd 100644 --- a/deploy/apecloud-mysql-cluster/templates/cluster.yaml +++ b/deploy/apecloud-mysql-cluster/templates/cluster.yaml @@ -19,6 +19,7 @@ spec: componentDefRef: mysql # ref clusterdefinition componentDefs.name monitor: {{ .Values.monitor.enabled | default false }} replicas: {{ .Values.replicaCount | default 3 }} + serviceAccountName: {{ include "apecloud-mysql-cluster.serviceAccountName" . }} enabledLogs: {{ .Values.enabledLogs | toJson | indent 4 }} {{- with .Values.resources }} resources: diff --git a/deploy/apecloud-mysql-cluster/templates/role.yaml b/deploy/apecloud-mysql-cluster/templates/role.yaml new file mode 100644 index 000000000..00aacf4cf --- /dev/null +++ b/deploy/apecloud-mysql-cluster/templates/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/deploy/apecloud-mysql-cluster/templates/rolebinding.yaml b/deploy/apecloud-mysql-cluster/templates/rolebinding.yaml new file mode 100644 index 000000000..9ac4d7588 --- /dev/null +++ b/deploy/apecloud-mysql-cluster/templates/rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-rolebinding-{{ .Release.Name }} + labels: + {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ include "apecloud-mysql-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/apecloud-mysql-cluster/templates/serviceaccount.yaml b/deploy/apecloud-mysql-cluster/templates/serviceaccount.yaml new file mode 100644 index 000000000..b29b17731 --- /dev/null +++ b/deploy/apecloud-mysql-cluster/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "apecloud-mysql-cluster.serviceAccountName" . }} + labels: + {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} diff --git a/deploy/apecloud-mysql-cluster/values.yaml b/deploy/apecloud-mysql-cluster/values.yaml index dab27ad50..e863abc89 100644 --- a/deploy/apecloud-mysql-cluster/values.yaml +++ b/deploy/apecloud-mysql-cluster/values.yaml @@ -60,4 +60,8 @@ topologyKeys: ## @param tolerations ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## -tolerations: [ ] \ No newline at end of file +tolerations: [ ] + +# The RABC permission used by cluster component pod, now include event.create +serviceAccount: + name: "" diff --git a/deploy/apecloud-mysql-scale-cluster/templates/_helpers.tpl b/deploy/apecloud-mysql-scale-cluster/templates/_helpers.tpl index 90137c9f9..6b1d05a2c 100644 --- a/deploy/apecloud-mysql-scale-cluster/templates/_helpers.tpl +++ b/deploy/apecloud-mysql-scale-cluster/templates/_helpers.tpl @@ -54,9 +54,5 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "apecloud-mysql-cluster.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "apecloud-mysql-cluster.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- default ((printf "kb-sa-%s" .Release.Name) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml b/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml index bcf8d2767..5062afe70 100644 --- a/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml +++ b/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml @@ -20,6 +20,7 @@ spec: monitor: {{ .Values.monitor.enabled | default false }} replicas: {{ .Values.replicaCount | default 3 }} enabledLogs: {{ .Values.enabledLogs | toJson | indent 4 }} + serviceAccountName: {{ include "apecloud-mysql-cluster.serviceAccountName" . }} {{- with .Values.resources }} resources: limits: diff --git a/deploy/apecloud-mysql-scale-cluster/templates/role.yaml b/deploy/apecloud-mysql-scale-cluster/templates/role.yaml new file mode 100644 index 000000000..00aacf4cf --- /dev/null +++ b/deploy/apecloud-mysql-scale-cluster/templates/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/deploy/apecloud-mysql-scale-cluster/templates/rolebinding.yaml b/deploy/apecloud-mysql-scale-cluster/templates/rolebinding.yaml new file mode 100644 index 000000000..9ac4d7588 --- /dev/null +++ b/deploy/apecloud-mysql-scale-cluster/templates/rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-rolebinding-{{ .Release.Name }} + labels: + {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ include "apecloud-mysql-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/apecloud-mysql-scale-cluster/templates/serviceaccount.yaml b/deploy/apecloud-mysql-scale-cluster/templates/serviceaccount.yaml new file mode 100644 index 000000000..b29b17731 --- /dev/null +++ b/deploy/apecloud-mysql-scale-cluster/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "apecloud-mysql-cluster.serviceAccountName" . }} + labels: + {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} diff --git a/deploy/apecloud-mysql-scale-cluster/values.yaml b/deploy/apecloud-mysql-scale-cluster/values.yaml index 4cfcdcbd2..378238ed2 100644 --- a/deploy/apecloud-mysql-scale-cluster/values.yaml +++ b/deploy/apecloud-mysql-scale-cluster/values.yaml @@ -60,4 +60,8 @@ topologyKeys: ## @param tolerations ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ ## -tolerations: [ ] \ No newline at end of file +tolerations: [ ] + +# The RABC permission used by cluster component pod, now include event.create +serviceAccount: + name: "" diff --git a/deploy/clickhouse-cluster/templates/_helpers.tpl b/deploy/clickhouse-cluster/templates/_helpers.tpl index ff4117fb0..1c942287e 100644 --- a/deploy/clickhouse-cluster/templates/_helpers.tpl +++ b/deploy/clickhouse-cluster/templates/_helpers.tpl @@ -54,9 +54,5 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "clickhouse-cluster.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "clickhouse-cluster.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- default (( printf "kb-sa-%s" .Release.Name ) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/clickhouse-cluster/templates/cluster.yaml b/deploy/clickhouse-cluster/templates/cluster.yaml index b320090b8..d8354be02 100644 --- a/deploy/clickhouse-cluster/templates/cluster.yaml +++ b/deploy/clickhouse-cluster/templates/cluster.yaml @@ -22,6 +22,7 @@ spec: monitor: {{ $.Values.monitor.enabled }} serviceType: {{ $.Values.service.type | default "ClusterIP" }} replicas: {{ .replicaCount | default 2 }} + serviceAccountName: {{ include "clickhouse-cluster.serviceAccountName" $ }} {{- with .tolerations }} tolerations: {{ .| toYaml | nindent 8 }} {{- end }} diff --git a/deploy/clickhouse-cluster/templates/role.yaml b/deploy/clickhouse-cluster/templates/role.yaml new file mode 100644 index 000000000..7dae7749b --- /dev/null +++ b/deploy/clickhouse-cluster/templates/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "clickhouse-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/deploy/clickhouse-cluster/templates/rolebinding.yaml b/deploy/clickhouse-cluster/templates/rolebinding.yaml new file mode 100644 index 000000000..314981c2c --- /dev/null +++ b/deploy/clickhouse-cluster/templates/rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-rolebinding-{{ .Release.Name }} + labels: + {{ include "clickhouse-cluster.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ include "clickhouse-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/clickhouse-cluster/templates/serviceaccount.yaml b/deploy/clickhouse-cluster/templates/serviceaccount.yaml new file mode 100644 index 000000000..fd0953eb3 --- /dev/null +++ b/deploy/clickhouse-cluster/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "clickhouse-cluster.serviceAccountName" . }} + labels: + {{ include "clickhouse-cluster.labels" . | nindent 4 }} diff --git a/deploy/clickhouse-cluster/values.yaml b/deploy/clickhouse-cluster/values.yaml index de904667d..4c8c57057 100644 --- a/deploy/clickhouse-cluster/values.yaml +++ b/deploy/clickhouse-cluster/values.yaml @@ -279,3 +279,7 @@ ingress: nameOverride: "" fullnameOverride: "" + +# The RABC permission used by cluster component pod, now include event.create +serviceAccount: + name: "" diff --git a/deploy/etcd-cluster/templates/_helpers.tpl b/deploy/etcd-cluster/templates/_helpers.tpl index b7c399f57..beb17576a 100644 --- a/deploy/etcd-cluster/templates/_helpers.tpl +++ b/deploy/etcd-cluster/templates/_helpers.tpl @@ -54,9 +54,5 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "etcd-cluster.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "etcd-cluster.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- default ((printf "kb-sa-%s" .Release.Name) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/etcd-cluster/templates/cluster.yaml b/deploy/etcd-cluster/templates/cluster.yaml index fb8fdd42f..edf1491e4 100644 --- a/deploy/etcd-cluster/templates/cluster.yaml +++ b/deploy/etcd-cluster/templates/cluster.yaml @@ -21,6 +21,7 @@ spec: monitor: {{ .Values.monitor.enabled }} serviceType: {{ $.Values.service.type | default "ClusterIP" }} replicas: {{ .Values.replicaCount | default "3" }} + serviceAccountName: {{ include "etcd-cluster.serviceAccountName" . }} {{- with .Values.resources }} resources: limits: @@ -40,4 +41,4 @@ spec: resources: requests: storage: {{ .Values.persistence.data.size }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/deploy/etcd-cluster/templates/role.yaml b/deploy/etcd-cluster/templates/role.yaml new file mode 100644 index 000000000..28d66f90f --- /dev/null +++ b/deploy/etcd-cluster/templates/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "etcd-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/deploy/etcd-cluster/templates/rolebinding.yaml b/deploy/etcd-cluster/templates/rolebinding.yaml new file mode 100644 index 000000000..2bfc497e9 --- /dev/null +++ b/deploy/etcd-cluster/templates/rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-rolebinding-{{ .Release.Name }} + labels: + {{ include "etcd-cluster.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ include "etcd-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/etcd-cluster/templates/serviceaccount.yaml b/deploy/etcd-cluster/templates/serviceaccount.yaml new file mode 100644 index 000000000..2d9e1cf10 --- /dev/null +++ b/deploy/etcd-cluster/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "etcd-cluster.serviceAccountName" . }} + labels: + {{ include "etcd-cluster.labels" . | nindent 4 }} diff --git a/deploy/etcd-cluster/values.yaml b/deploy/etcd-cluster/values.yaml index 1b6219123..7fe5da0b7 100644 --- a/deploy/etcd-cluster/values.yaml +++ b/deploy/etcd-cluster/values.yaml @@ -173,4 +173,8 @@ ingress: ## port: ## name: http ## - extraRules: [] \ No newline at end of file + extraRules: [] + +# The RABC permission used by cluster component pod, now include event.create +serviceAccount: + name: "" diff --git a/deploy/kafka-cluster/templates/_helpers.tpl b/deploy/kafka-cluster/templates/_helpers.tpl index 91a40da16..dceca86d3 100644 --- a/deploy/kafka-cluster/templates/_helpers.tpl +++ b/deploy/kafka-cluster/templates/_helpers.tpl @@ -54,9 +54,5 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "kafka-cluster.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "kafka-cluster.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- default ((printf "kb-sa-%s" .Release.Name) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/kafka-cluster/templates/role.yaml b/deploy/kafka-cluster/templates/role.yaml new file mode 100644 index 000000000..171090bf4 --- /dev/null +++ b/deploy/kafka-cluster/templates/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "kafka-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/deploy/kafka-cluster/templates/rolebinding.yaml b/deploy/kafka-cluster/templates/rolebinding.yaml new file mode 100644 index 000000000..59e20b218 --- /dev/null +++ b/deploy/kafka-cluster/templates/rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-rolebinding-{{ .Release.Name }} + labels: + {{ include "kafka-cluster.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ include "kafka-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/kafka-cluster/templates/serviceaccount.yaml b/deploy/kafka-cluster/templates/serviceaccount.yaml new file mode 100644 index 000000000..b24c1376d --- /dev/null +++ b/deploy/kafka-cluster/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "kafka-cluster.serviceAccountName" . }} + labels: + {{ include "kafka-cluster.labels" . | nindent 4 }} diff --git a/deploy/kafka-cluster/values.yaml b/deploy/kafka-cluster/values.yaml index 03554d0d6..9106e496e 100644 --- a/deploy/kafka-cluster/values.yaml +++ b/deploy/kafka-cluster/values.yaml @@ -189,3 +189,7 @@ affinity: {} nameOverride: "" fullnameOverride: "" + +# The RABC permission used by cluster component pod, now include event.create +serviceAccount: + name: "" diff --git a/deploy/mongodb-cluster/Chart.yaml b/deploy/mongodb-cluster/Chart.yaml index ca14ddb1f..ac8a54862 100644 --- a/deploy/mongodb-cluster/Chart.yaml +++ b/deploy/mongodb-cluster/Chart.yaml @@ -19,5 +19,5 @@ keywords: - replication maintainers: - - name: free6om + - name: xuriwuyun url: https://github.com/apecloud/kubeblocks/deploy diff --git a/deploy/mongodb-cluster/templates/NOTES.txt b/deploy/mongodb-cluster/templates/NOTES.txt index 5466e6632..55cb7d091 100644 --- a/deploy/mongodb-cluster/templates/NOTES.txt +++ b/deploy/mongodb-cluster/templates/NOTES.txt @@ -1,22 +1,18 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mongodb-cluster.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "mongodb-cluster.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "mongodb-cluster.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "mongodb-cluster.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} +** Please be patient while the chart is being deployed ** + +To get MongoDB connection address accessed from within your cluster: + + export MONGODB_ADDRESS=$(kubectl get secret --namespace {{ .Release.Namespace }} -l app.kubernetes.io/managed-by=kubeblocks,app.kubernetes.io/instance={{ .Release.Name }} -o jsonpath="{.items[0].data.headlessEndpoint}" | base64 -d) + +To get the root password run: + + export MONGODB_ROOT_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} -l app.kubernetes.io/managed-by=kubeblocks,app.kubernetes.io/instance={{ .Release.Name}} -o jsonpath="{.items[0].data.password}" | base64 -d) + +To connect to your database, create a MongoDB client container: + + kubectl run --namespace {{ .Release.Namespace }} {{ .Release.Name }}-client --rm --tty -i --restart='Never' --env="MONGODB_ROOT_PASSWORD=$MONGODB_ROOT_PASSWORD" --env="MONGODB_ADDRESS=$MONGODB_ADDRESS" --image mongo:5.0.14 --command -- bash + +Then, run the following command: + + mongosh admin --host $MONGODB_ADDRESS --authenticationDatabase admin -u root -p $MONGODB_ROOT_PASSWORD + diff --git a/deploy/mongodb-cluster/templates/_helpers.tpl b/deploy/mongodb-cluster/templates/_helpers.tpl index 6132a557c..70bcd1221 100644 --- a/deploy/mongodb-cluster/templates/_helpers.tpl +++ b/deploy/mongodb-cluster/templates/_helpers.tpl @@ -54,9 +54,5 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "mongodb-cluster.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "mongodb-cluster.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- default ((printf "kb-sa-%s" .Release.Name) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/mongodb-cluster/templates/replicaset.yaml b/deploy/mongodb-cluster/templates/replicaset.yaml index f4d2d8fdf..197472d16 100644 --- a/deploy/mongodb-cluster/templates/replicaset.yaml +++ b/deploy/mongodb-cluster/templates/replicaset.yaml @@ -21,6 +21,7 @@ spec: componentDefRef: mongodb monitor: {{ $.Values.monitor.enabled }} replicas: {{ $.Values.mongodb.replicas }} + serviceAccountName: {{ include "mongodb-cluster.serviceAccountName" . }} {{- with $.Values.mongodb.tolerations }} tolerations: {{ .| toYaml | nindent 8 }} {{- end }} diff --git a/deploy/mongodb-cluster/templates/role.yaml b/deploy/mongodb-cluster/templates/role.yaml new file mode 100644 index 000000000..64bf78eb6 --- /dev/null +++ b/deploy/mongodb-cluster/templates/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "mongodb-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/deploy/mongodb-cluster/templates/rolebinding.yaml b/deploy/mongodb-cluster/templates/rolebinding.yaml new file mode 100644 index 000000000..883684566 --- /dev/null +++ b/deploy/mongodb-cluster/templates/rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-rolebinding-{{ .Release.Name }} + labels: + {{ include "mongodb-cluster.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ include "mongodb-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/mongodb-cluster/templates/serviceaccount.yaml b/deploy/mongodb-cluster/templates/serviceaccount.yaml new file mode 100644 index 000000000..b9dde5342 --- /dev/null +++ b/deploy/mongodb-cluster/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mongodb-cluster.serviceAccountName" . }} + labels: + {{ include "mongodb-cluster.labels" . | nindent 4 }} diff --git a/deploy/mongodb-cluster/values.yaml b/deploy/mongodb-cluster/values.yaml index 1dc790be3..2912ec979 100644 --- a/deploy/mongodb-cluster/values.yaml +++ b/deploy/mongodb-cluster/values.yaml @@ -311,3 +311,7 @@ ingress: enabledLogs: - running + +# The RABC permission used by cluster component pod, now include event.create +serviceAccount: + name: "" diff --git a/deploy/mongodb/Chart.yaml b/deploy/mongodb/Chart.yaml index 82b1b08ed..c65641c51 100644 --- a/deploy/mongodb/Chart.yaml +++ b/deploy/mongodb/Chart.yaml @@ -19,5 +19,5 @@ keywords: - replication maintainers: - - name: free6om + - name: xuriwuyun url: https://github.com/apecloud/kubeblocks/deploy diff --git a/deploy/postgresql-cluster/templates/role.yaml b/deploy/postgresql-cluster/templates/role.yaml index 198ed1eff..6b0875755 100644 --- a/deploy/postgresql-cluster/templates/role.yaml +++ b/deploy/postgresql-cluster/templates/role.yaml @@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: kb-role-{{ .Release.Namespace }}-{{ .Release.Name }} + name: kb-role-{{ .Release.Name }} namespace: {{ .Release.Namespace }} labels: {{ include "postgresqlcluster.labels" . | nindent 4 }} @@ -44,4 +44,10 @@ rules: - patch - update - watch -{{- end }} \ No newline at end of file + - apiGroups: + - "" + resources: + - events + verbs: + - create +{{- end }} diff --git a/deploy/postgresql-cluster/templates/rolebinding.yaml b/deploy/postgresql-cluster/templates/rolebinding.yaml index ebc0a083f..eebdc620b 100644 --- a/deploy/postgresql-cluster/templates/rolebinding.yaml +++ b/deploy/postgresql-cluster/templates/rolebinding.yaml @@ -2,15 +2,15 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: kb-rolebinding-{{ .Release.Namespace }}-{{ .Release.Name }} + name: kb-rolebinding-{{ .Release.Name }} labels: {{ include "postgresqlcluster.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: kb-role-{{ .Release.Namespace }}-{{ .Release.Name }} + name: kb-role-{{ .Release.Name }} subjects: - kind: ServiceAccount - name: kb-sa-{{ .Release.Name }} + name: {{ include "postgresqlcluster.serviceAccountName" . }} namespace: {{ .Release.Namespace }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/deploy/postgresql-cluster/templates/serviceaccount.yaml b/deploy/postgresql-cluster/templates/serviceaccount.yaml index 2eb331be8..b3482824c 100644 --- a/deploy/postgresql-cluster/templates/serviceaccount.yaml +++ b/deploy/postgresql-cluster/templates/serviceaccount.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: kb-sa-{{ .Release.Name }} + name: {{ include "postgresqlcluster.serviceAccountName" . }} labels: {{ include "postgresqlcluster.labels" . | nindent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/deploy/redis-cluster/templates/_helpers.tpl b/deploy/redis-cluster/templates/_helpers.tpl index 637303914..63f120115 100644 --- a/deploy/redis-cluster/templates/_helpers.tpl +++ b/deploy/redis-cluster/templates/_helpers.tpl @@ -54,9 +54,5 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "redis-cluster.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "redis-cluster.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} +{{- default ((printf "kb-sa-%s" .Release.Name) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/redis-cluster/templates/cluster.yaml b/deploy/redis-cluster/templates/cluster.yaml index fdd3a3ff0..f60b87f88 100644 --- a/deploy/redis-cluster/templates/cluster.yaml +++ b/deploy/redis-cluster/templates/cluster.yaml @@ -20,6 +20,7 @@ spec: monitor: {{ .Values.monitor.enabled | default false }} enabledLogs: {{ .Values.enabledLogs | toJson | indent 4 }} replicas: {{ .Values.replicaCount | default 2 }} + serviceAccountName: {{ include "redis-cluster.serviceAccountName" . }} primaryIndex: {{ .Values.primaryIndex | default 0 }} switchPolicy: type: {{ .Values.switchPolicy.type}} diff --git a/deploy/redis-cluster/templates/role.yaml b/deploy/redis-cluster/templates/role.yaml new file mode 100644 index 000000000..2ab399e72 --- /dev/null +++ b/deploy/redis-cluster/templates/role.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kb-role-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{ include "redis-cluster.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create diff --git a/deploy/redis-cluster/templates/rolebinding.yaml b/deploy/redis-cluster/templates/rolebinding.yaml new file mode 100644 index 000000000..7ba67d5f8 --- /dev/null +++ b/deploy/redis-cluster/templates/rolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kb-rolebinding-{{ .Release.Name }} + labels: + {{ include "redis-cluster.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kb-role-{{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ include "redis-cluster.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/deploy/redis-cluster/templates/serviceaccount.yaml b/deploy/redis-cluster/templates/serviceaccount.yaml new file mode 100644 index 000000000..c76c9743e --- /dev/null +++ b/deploy/redis-cluster/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "redis-cluster.serviceAccountName" . }} + labels: + {{ include "redis-cluster.labels" . | nindent 4 }} diff --git a/deploy/redis-cluster/values.yaml b/deploy/redis-cluster/values.yaml index 5624084fc..b7c6b8d6f 100644 --- a/deploy/redis-cluster/values.yaml +++ b/deploy/redis-cluster/values.yaml @@ -49,3 +49,7 @@ tolerations: [ ] enabledLogs: - running + +# The RABC permission used by cluster component pod, now include event.create +serviceAccount: + name: "" diff --git a/docs/release_notes/v0.6.0/_category_.yml b/docs/release_notes/v0.6.0/_category_.yml new file mode 100644 index 000000000..a916d53d5 --- /dev/null +++ b/docs/release_notes/v0.6.0/_category_.yml @@ -0,0 +1,4 @@ +position: 1 +label: v0.6.0 +collapsible: true +collapsed: true diff --git a/docs/release_notes/v0.6.0/v0.6.0.md b/docs/release_notes/v0.6.0/v0.6.0.md new file mode 100644 index 000000000..d00981c84 --- /dev/null +++ b/docs/release_notes/v0.6.0/v0.6.0.md @@ -0,0 +1,24 @@ +# KubeBlocks 0.6.0 (TBD) + +We are happy to announce the release of KubeBlocks 0.6.0 with some exciting new features and improvements. + +## Highlights + +## Acknowledgements + +Thanks to everyone who made this release possible! + + +## What's Changed + + +### New Features + + + +#### Compatibility +- Pass the AWS EKS v1.22 / v1.23 / v1.24 / v1.25 compatibility test. + +#### Maintainability +* Automatic pod container environment variables: + * KB_POD_UID - Pod UID diff --git a/internal/cli/cmd/cluster/cluster_test.go b/internal/cli/cmd/cluster/cluster_test.go index aa6d3c90f..e0f298f9d 100644 --- a/internal/cli/cmd/cluster/cluster_test.go +++ b/internal/cli/cmd/cluster/cluster_test.go @@ -79,22 +79,11 @@ var _ = Describe("Cluster", func() { IOStreams: streams, }, } - Expect(o.Validate()).To(Succeed()) + o.Options = o + Expect(o.Complete()).To(Succeed()) Expect(o.Name).ShouldNot(BeEmpty()) + Expect(o.Run()).Should(HaveOccurred()) }) - - It("new command", func() { - cmd := NewCreateCmd(tf, streams) - Expect(cmd).ShouldNot(BeNil()) - Expect(cmd.Flags().Set("cluster-definition", testing.ClusterDefName)).Should(Succeed()) - Expect(cmd.Flags().Set("cluster-version", testing.ClusterVersionName)).Should(Succeed()) - Expect(cmd.Flags().Set("set-file", testComponentPath)).Should(Succeed()) - Expect(cmd.Flags().Set("termination-policy", "Delete")).Should(Succeed()) - - // must succeed otherwise exit 1 and make test fails - cmd.Run(nil, []string{"test1"}) - }) - }) Context("run", func() { diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 1b7f44739..b28fee5bd 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -233,8 +233,9 @@ func NewCreateOptions(f cmdutil.Factory, streams genericclioptions.IOStreams) *C CueTemplateName: CueTemplateName, GVR: types.ClusterGVR(), }} - o.CreateOptions.PreCreate = o.PreCreate o.CreateOptions.Options = o + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.CreateDependencies = o.CreateDependencies return o } @@ -436,13 +437,6 @@ const ( func (o *CreateOptions) buildDependenciesFn(cd *appsv1alpha1.ClusterDefinition, compSpec *appsv1alpha1.ClusterComponentSpec) error { - // HACK: now we only support postgresql cluster definition - if c, err := shouldCreateDependencies(cd, compSpec); err != nil { - return err - } else if !c { - return nil - } - // set component service account name compSpec.ServiceAccountName = saNamePrefix + o.Name o.shouldCreateDependencies = true @@ -476,20 +470,35 @@ func (o *CreateOptions) CreateDependencies(dryRun []string) error { role := rbacv1ac.Role(roleName, o.Namespace).WithRules([]*rbacv1ac.PolicyRuleApplyConfiguration{ { APIGroups: []string{""}, - Resources: []string{"configmaps"}, - Verbs: []string{"create", "get", "list", "patch", "update", "watch", "delete"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"endpoints"}, - Verbs: []string{"create", "get", "list", "patch", "update", "watch", "delete"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", "list", "patch", "update", "watch"}, + Resources: []string{"events"}, + Verbs: []string{"create"}, }, }...).WithLabels(labels) + + // postgresql need more rules for patronic + if ok, err := o.isPostgresqlCluster(); err != nil { + return err + } else if ok { + rules := []rbacv1ac.PolicyRuleApplyConfiguration{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"create", "get", "list", "patch", "update", "watch", "delete"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"endpoints"}, + Verbs: []string{"create", "get", "list", "patch", "update", "watch", "delete"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{"get", "list", "patch", "update", "watch"}, + }, + } + role.Rules = append(role.Rules, rules...) + } + if _, err := o.Client.RbacV1().Roles(o.Namespace).Apply(context.TODO(), role, applyOptions); err != nil { return err } @@ -600,6 +609,40 @@ func (o *CreateOptions) PreCreate(obj *unstructured.Unstructured) error { return nil } +func (o *CreateOptions) isPostgresqlCluster() (bool, error) { + cd, err := cluster.GetClusterDefByName(o.Dynamic, o.ClusterDefRef) + if err != nil { + return false, err + } + + var compDef *appsv1alpha1.ClusterComponentDefinition + if cd.Spec.Type != "postgresql" { + return false, nil + } + + // get cluster component definition + if len(o.ComponentSpecs) == 0 { + return false, fmt.Errorf("find no cluster componnet") + } + compSpec := o.ComponentSpecs[0] + for i, def := range cd.Spec.ComponentDefs { + compDefRef := compSpec["componentDefRef"] + if compDefRef != nil && def.Name == compDefRef.(string) { + compDef = &cd.Spec.ComponentDefs[i] + } + } + + if compDef == nil { + return false, fmt.Errorf("failed to find component definition for componnet %v", compSpec["Name"]) + } + + // for postgresql, we need to create a service account, a role and a rolebinding + if compDef.CharacterType != "postgresql" { + return false, nil + } + return true, nil +} + // setEnableAllLog set enable all logs, and ignore enabledLogs of component level. func setEnableAllLogs(c *appsv1alpha1.Cluster, cd *appsv1alpha1.ClusterDefinition) { for idx, comCluster := range c.Spec.ComponentSpecs { @@ -1008,30 +1051,6 @@ func getClusterVersions(dynamic dynamic.Interface, clusterDef string) (map[strin return allClusterVersions, defaultVersion, existedDefault, nil } -func shouldCreateDependencies(cd *appsv1alpha1.ClusterDefinition, compSpec *appsv1alpha1.ClusterComponentSpec) (bool, error) { - var compDef *appsv1alpha1.ClusterComponentDefinition - if cd.Spec.Type != "postgresql" { - return false, nil - } - - // get cluster component definition - for i, def := range cd.Spec.ComponentDefs { - if def.Name == compSpec.ComponentDefRef { - compDef = &cd.Spec.ComponentDefs[i] - } - } - - if compDef == nil { - return false, fmt.Errorf("failed to find component definition for componnet %s", compSpec.Name) - } - - // for postgresql, we need to create a service account, a role and a rolebinding - if compDef.CharacterType != "postgresql" { - return false, nil - } - return true, nil -} - func buildResourceLabels(clusterName string) map[string]string { return map[string]string{ constant.AppInstanceLabelKey: clusterName, diff --git a/internal/cli/cmd/cluster/delete.go b/internal/cli/cmd/cluster/delete.go index e05958c5c..0eda28874 100644 --- a/internal/cli/cmd/cluster/delete.go +++ b/internal/cli/cmd/cluster/delete.go @@ -115,11 +115,11 @@ func clusterPostDeleteHook(o *delete.DeleteOptions, object runtime.Object) error func deleteCompDependencies(client kubernetes.Interface, ns string, name string, cd *appsv1alpha1.ClusterDefinition, compSpec *appsv1alpha1.ClusterComponentSpec) error { - if d, err := shouldCreateDependencies(cd, compSpec); err != nil { - return err - } else if !d { - return nil - } + // if d, err := shouldCreateDependencies(cd, compSpec); err != nil { + // return err + // } else if !d { + // return nil + // } return deleteDependencies(client, ns, name) } diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index e657c3ee1..1b5a47c32 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -139,6 +139,7 @@ func injectEnvs(params BuilderParams, envConfigName string, c *corev1.Container) fieldPath string }{ {name: "KB_POD_NAME", fieldPath: "metadata.name"}, + {name: "KB_POD_UID", fieldPath: "metadata.uid"}, {name: "KB_NAMESPACE", fieldPath: "metadata.namespace"}, {name: "KB_SA_NAME", fieldPath: "spec.serviceAccountName"}, {name: "KB_NODENAME", fieldPath: "spec.nodeName"}, From aeef5dd70c93b462bae96e1487c1d9c499162344 Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Wed, 17 May 2023 20:14:23 +0800 Subject: [PATCH 317/439] feat: support kbcli fault io time and stress (#3263) Co-authored-by: huyongqii --- docs/user_docs/cli/cli.md | 3 + docs/user_docs/cli/kbcli_fault.md | 3 + docs/user_docs/cli/kbcli_fault_io.md | 46 ++++ .../user_docs/cli/kbcli_fault_io_attribute.md | 94 +++++++ docs/user_docs/cli/kbcli_fault_io_errno.md | 88 +++++++ docs/user_docs/cli/kbcli_fault_io_latency.md | 88 +++++++ docs/user_docs/cli/kbcli_fault_io_mistake.md | 90 +++++++ docs/user_docs/cli/kbcli_fault_stress.md | 75 ++++++ docs/user_docs/cli/kbcli_fault_time.md | 79 ++++++ internal/cli/cmd/fault/fault.go | 3 + internal/cli/cmd/fault/fault_io.go | 231 ++++++++++++++++++ internal/cli/cmd/fault/fault_io_test.go | 146 +++++++++++ internal/cli/cmd/fault/fault_stress.go | 146 +++++++++++ internal/cli/cmd/fault/fault_stress_test.go | 78 ++++++ internal/cli/cmd/fault/fault_time.go | 135 ++++++++++ internal/cli/cmd/fault/fault_time_test.go | 76 ++++++ .../cli/create/template/io_chaos_template.cue | 69 ++++++ .../create/template/stress_chaos_template.cue | 46 ++++ .../create/template/time_chaos_template.cue | 49 ++++ 19 files changed, 1545 insertions(+) create mode 100644 docs/user_docs/cli/kbcli_fault_io.md create mode 100644 docs/user_docs/cli/kbcli_fault_io_attribute.md create mode 100644 docs/user_docs/cli/kbcli_fault_io_errno.md create mode 100644 docs/user_docs/cli/kbcli_fault_io_latency.md create mode 100644 docs/user_docs/cli/kbcli_fault_io_mistake.md create mode 100644 docs/user_docs/cli/kbcli_fault_stress.md create mode 100644 docs/user_docs/cli/kbcli_fault_time.md create mode 100644 internal/cli/cmd/fault/fault_io.go create mode 100644 internal/cli/cmd/fault/fault_io_test.go create mode 100644 internal/cli/cmd/fault/fault_stress.go create mode 100644 internal/cli/cmd/fault/fault_stress_test.go create mode 100644 internal/cli/cmd/fault/fault_time.go create mode 100644 internal/cli/cmd/fault/fault_time_test.go create mode 100644 internal/cli/create/template/io_chaos_template.cue create mode 100644 internal/cli/create/template/stress_chaos_template.cue create mode 100644 internal/cli/create/template/time_chaos_template.cue diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index c3eb3de88..ca7c6c969 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -112,8 +112,11 @@ List and open the KubeBlocks dashboards. Inject faults to pod. +* [kbcli fault io](kbcli_fault_io.md) - IO chaos. * [kbcli fault network](kbcli_fault_network.md) - Network chaos. * [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. +* [kbcli fault stress](kbcli_fault_stress.md) - Add memory pressure or CPU load to the system. +* [kbcli fault time](kbcli_fault_time.md) - Clock skew failure. ## [kubeblocks](kbcli_kubeblocks.md) diff --git a/docs/user_docs/cli/kbcli_fault.md b/docs/user_docs/cli/kbcli_fault.md index 895fde1c2..cdd11e7ac 100644 --- a/docs/user_docs/cli/kbcli_fault.md +++ b/docs/user_docs/cli/kbcli_fault.md @@ -37,8 +37,11 @@ Inject faults to pod. ### SEE ALSO +* [kbcli fault io](kbcli_fault_io.md) - IO chaos. * [kbcli fault network](kbcli_fault_network.md) - Network chaos. * [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. +* [kbcli fault stress](kbcli_fault_stress.md) - Add memory pressure or CPU load to the system. +* [kbcli fault time](kbcli_fault_time.md) - Clock skew failure. #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/docs/user_docs/cli/kbcli_fault_io.md b/docs/user_docs/cli/kbcli_fault_io.md new file mode 100644 index 000000000..80f1f2ced --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_io.md @@ -0,0 +1,46 @@ +--- +title: kbcli fault io +--- + +IO chaos. + +### Options + +``` + -h, --help help for io +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault](kbcli_fault.md) - Inject faults to pod. +* [kbcli fault io attribute](kbcli_fault_io_attribute.md) - Override the attributes of the file. +* [kbcli fault io errno](kbcli_fault_io_errno.md) - Causes IO operations to return specific errors. +* [kbcli fault io latency](kbcli_fault_io_latency.md) - Delayed IO operations. +* [kbcli fault io mistake](kbcli_fault_io_mistake.md) - Alters the contents of the file, distorting the contents of the file. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_io_attribute.md b/docs/user_docs/cli/kbcli_fault_io_attribute.md new file mode 100644 index 000000000..3e2bb2b9d --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_io_attribute.md @@ -0,0 +1,94 @@ +--- +title: kbcli fault io attribute +--- + +Override the attributes of the file. + +``` +kbcli fault io attribute [flags] +``` + +### Examples + +``` + # Affects the first container in default namespace's all pods. Delay all IO operations under the /data path by 10s. + kbcli fault io latency --delay=10s --volume-path=/data + + # Affects the first container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data + + # Affects the mysql container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data -c=mysql + + # There is a 50% probability of affecting the read IO operation of the test.txt file under the /data path. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data --path=test.txt --percent=50 --method=READ -c=mysql + + # Same as above.Make all IO operations under the /data path return the specified error number 22 (Invalid argument). + kbcli fault io errno --volume-path=/data --errno=22 + + # Same as above.Modify the IO operation permission attribute of the files under the /data path to 72.(110 in octal). + kbcli fault io attribute --volume-path=/data --perm=72 + + # Modify all files so that random positions of 1's with a maximum length of 10 bytes will be replaced with 0's. + kbcli fault io mistake --volume-path=/data --filling=zero --max-occurrences=10 --max-length=1 +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --blocks uint The number of blocks the file occupies. + -c, --container stringArray The name of the container, such as mysql, prometheus.If it's empty, the first container will be injected. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + --gid uint32 The owner's group ID. + -h, --help help for attribute + --ino uint ino number. + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --method stringArray The file system calls that need to inject faults. For example: WRITE READ + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --nlink uint32 The number of hard links. + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The effective scope of the injection error can be a wildcard or a single file. + --percent int Probability of failure per operation, in %. (default 100) + --perm uint16 Decimal representation of file permissions. + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --size uint File size. + --uid uint32 Owner's user ID. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --volume-path string The mount point of the volume in the target container must be the root directory of the mount. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault io](kbcli_fault_io.md) - IO chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_io_errno.md b/docs/user_docs/cli/kbcli_fault_io_errno.md new file mode 100644 index 000000000..3218e6e58 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_io_errno.md @@ -0,0 +1,88 @@ +--- +title: kbcli fault io errno +--- + +Causes IO operations to return specific errors. + +``` +kbcli fault io errno [flags] +``` + +### Examples + +``` + # Affects the first container in default namespace's all pods. Delay all IO operations under the /data path by 10s. + kbcli fault io latency --delay=10s --volume-path=/data + + # Affects the first container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data + + # Affects the mysql container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data -c=mysql + + # There is a 50% probability of affecting the read IO operation of the test.txt file under the /data path. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data --path=test.txt --percent=50 --method=READ -c=mysql + + # Same as above.Make all IO operations under the /data path return the specified error number 22 (Invalid argument). + kbcli fault io errno --volume-path=/data --errno=22 + + # Same as above.Modify the IO operation permission attribute of the files under the /data path to 72.(110 in octal). + kbcli fault io attribute --volume-path=/data --perm=72 + + # Modify all files so that random positions of 1's with a maximum length of 10 bytes will be replaced with 0's. + kbcli fault io mistake --volume-path=/data --filling=zero --max-occurrences=10 --max-length=1 +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --container stringArray The name of the container, such as mysql, prometheus.If it's empty, the first container will be injected. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + --errno int The returned error number. + -h, --help help for errno + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --method stringArray The file system calls that need to inject faults. For example: WRITE READ + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The effective scope of the injection error can be a wildcard or a single file. + --percent int Probability of failure per operation, in %. (default 100) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --volume-path string The mount point of the volume in the target container must be the root directory of the mount. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault io](kbcli_fault_io.md) - IO chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_io_latency.md b/docs/user_docs/cli/kbcli_fault_io_latency.md new file mode 100644 index 000000000..83819a919 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_io_latency.md @@ -0,0 +1,88 @@ +--- +title: kbcli fault io latency +--- + +Delayed IO operations. + +``` +kbcli fault io latency [flags] +``` + +### Examples + +``` + # Affects the first container in default namespace's all pods. Delay all IO operations under the /data path by 10s. + kbcli fault io latency --delay=10s --volume-path=/data + + # Affects the first container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data + + # Affects the mysql container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data -c=mysql + + # There is a 50% probability of affecting the read IO operation of the test.txt file under the /data path. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data --path=test.txt --percent=50 --method=READ -c=mysql + + # Same as above.Make all IO operations under the /data path return the specified error number 22 (Invalid argument). + kbcli fault io errno --volume-path=/data --errno=22 + + # Same as above.Modify the IO operation permission attribute of the files under the /data path to 72.(110 in octal). + kbcli fault io attribute --volume-path=/data --perm=72 + + # Modify all files so that random positions of 1's with a maximum length of 10 bytes will be replaced with 0's. + kbcli fault io mistake --volume-path=/data --filling=zero --max-occurrences=10 --max-length=1 +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --container stringArray The name of the container, such as mysql, prometheus.If it's empty, the first container will be injected. + --delay string Specific delay time. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for latency + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --method stringArray The file system calls that need to inject faults. For example: WRITE READ + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The effective scope of the injection error can be a wildcard or a single file. + --percent int Probability of failure per operation, in %. (default 100) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --volume-path string The mount point of the volume in the target container must be the root directory of the mount. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault io](kbcli_fault_io.md) - IO chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_io_mistake.md b/docs/user_docs/cli/kbcli_fault_io_mistake.md new file mode 100644 index 000000000..d13e638d4 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_io_mistake.md @@ -0,0 +1,90 @@ +--- +title: kbcli fault io mistake +--- + +Alters the contents of the file, distorting the contents of the file. + +``` +kbcli fault io mistake [flags] +``` + +### Examples + +``` + # Affects the first container in default namespace's all pods. Delay all IO operations under the /data path by 10s. + kbcli fault io latency --delay=10s --volume-path=/data + + # Affects the first container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data + + # Affects the mysql container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data -c=mysql + + # There is a 50% probability of affecting the read IO operation of the test.txt file under the /data path. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data --path=test.txt --percent=50 --method=READ -c=mysql + + # Same as above.Make all IO operations under the /data path return the specified error number 22 (Invalid argument). + kbcli fault io errno --volume-path=/data --errno=22 + + # Same as above.Modify the IO operation permission attribute of the files under the /data path to 72.(110 in octal). + kbcli fault io attribute --volume-path=/data --perm=72 + + # Modify all files so that random positions of 1's with a maximum length of 10 bytes will be replaced with 0's. + kbcli fault io mistake --volume-path=/data --filling=zero --max-occurrences=10 --max-length=1 +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --container stringArray The name of the container, such as mysql, prometheus.If it's empty, the first container will be injected. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + --filling string The filling content of the error data can only be zero (filling with 0) or random (filling with random bytes). + -h, --help help for mistake + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --max-length int The maximum length (in bytes) of each error. (default 1) + --max-occurrences int The maximum number of times an error can occur per operation. (default 1) + --method stringArray The file system calls that need to inject faults. For example: WRITE READ + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --path string The effective scope of the injection error can be a wildcard or a single file. + --percent int Probability of failure per operation, in %. (default 100) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. + --volume-path string The mount point of the volume in the target container must be the root directory of the mount. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault io](kbcli_fault_io.md) - IO chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_stress.md b/docs/user_docs/cli/kbcli_fault_stress.md new file mode 100644 index 000000000..fb8bb76aa --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_stress.md @@ -0,0 +1,75 @@ +--- +title: kbcli fault stress +--- + +Add memory pressure or CPU load to the system. + +``` +kbcli fault stress [flags] +``` + +### Examples + +``` + # Affects the first container in default namespace's all pods.Making CPU load up to 50%, and the memory up to 100MB. + kbcli fault stress --cpu-worker=2 --cpu-load=50 --memory-worker=1 --memory-size=100Mi + + # Affects the first container in mycluster-mysql-0 pod. Making the CPU load up to 50%, and the memory up to 500MB. + kbcli fault stress mycluster-mysql-0 --cpu-worker=2 --cpu-load=50 + + # Affects the mysql container in mycluster-mysql-0 pod. Making the memory up to 500MB. + kbcli fault stress mycluster-mysql-0 --memory-worker=2 --memory-size=500Mi -c=mysql +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + -c, --container stringArray The name of the container, such as mysql, prometheus.If it's empty, the first container will be injected. + --cpu-load int Specifies the percentage of CPU occupied. 0 means no extra load added, 100 means full load. The total load is workers * load. + --cpu-worker int Specifies the number of threads that exert CPU pressure. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for stress + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --memory-size string Specify the size of the allocated memory or the percentage of the total memory, and the sum of the allocated memory is size. For example:256MB or 25% + --memory-worker int Specifies the number of threads that apply memory pressure. + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault](kbcli_fault.md) - Inject faults to pod. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_time.md b/docs/user_docs/cli/kbcli_fault_time.md new file mode 100644 index 000000000..f0f19803d --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_time.md @@ -0,0 +1,79 @@ +--- +title: kbcli fault time +--- + +Clock skew failure. + +``` +kbcli fault time [flags] +``` + +### Examples + +``` + # Affects the first container in default namespace's all pods.Shifts the clock back five seconds. + kbcli fault time --time-offset=-5s + + # Affects the first container in default namespace's all pods. + kbcli fault time --time-offset=-5m5s + + # Affects the first container in mycluster-mysql-0 pod. Shifts the clock forward five seconds. + kbcli fault time mycluster-mysql-0 --time-offset=+5s50ms + + # Affects the mysql container in mycluster-mysql-0 pod. Shifts the clock forward five seconds. + kbcli fault time mycluster-mysql-0 --time-offset=+5s -c=mysql + + # The clock that specifies the effect of time offset is CLOCK_REALTIME. + kbcli fault time mycluster-mysql-0 --time-offset=+5s --clock-id=CLOCK_REALTIME -c=mysql +``` + +### Options + +``` + --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) + --clock-id stringArray Specifies the clock on which the time offset acts.If it's empty, it will be set to ['CLOCK_REALTIME'].See clock_gettime [https://man7.org/linux/man-pages/man2/clock_gettime.2.html] document for details. + -c, --container stringArray Specifies the injected container name. For example: mysql. If it's empty, the first container will be injected. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") + -h, --help help for time + --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) + --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") + --node stringArray Inject faults into pods in the specified node. + --node-label stringToString label for node, such as '"kubernetes.io/arch=arm64,kubernetes.io/hostname=minikube-m03,kubernetes.io/os=linux. (default []) + --ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --phase stringArray Specify the pod that injects the fault by the state of the pod. + --time-offset string Specifies the length of the time offset. For example: -5s, -10m100ns. + --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault](kbcli_fault.md) - Inject faults to pod. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/internal/cli/cmd/fault/fault.go b/internal/cli/cmd/fault/fault.go index e889f9adc..9ed07b2db 100644 --- a/internal/cli/cmd/fault/fault.go +++ b/internal/cli/cmd/fault/fault.go @@ -73,6 +73,9 @@ func NewFaultCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra. cmd.AddCommand( NewPodChaosCmd(f, streams), NewNetworkChaosCmd(f, streams), + NewTimeChaosCmd(f, streams), + NewIOChaosCmd(f, streams), + NewStressChaosCmd(f, streams), ) return cmd } diff --git a/internal/cli/cmd/fault/fault_io.go b/internal/cli/cmd/fault/fault_io.go new file mode 100644 index 000000000..5431e1780 --- /dev/null +++ b/internal/cli/cmd/fault/fault_io.go @@ -0,0 +1,231 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var faultIOExample = templates.Examples(` + # Affects the first container in default namespace's all pods. Delay all IO operations under the /data path by 10s. + kbcli fault io latency --delay=10s --volume-path=/data + + # Affects the first container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data + + # Affects the mysql container in mycluster-mysql-0 pod. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data -c=mysql + + # There is a 50% probability of affecting the read IO operation of the test.txt file under the /data path. + kbcli fault io latency mycluster-mysql-0 --delay=10s --volume-path=/data --path=test.txt --percent=50 --method=READ -c=mysql + + # Same as above.Make all IO operations under the /data path return the specified error number 22 (Invalid argument). + kbcli fault io errno --volume-path=/data --errno=22 + + # Same as above.Modify the IO operation permission attribute of the files under the /data path to 72.(110 in octal). + kbcli fault io attribute --volume-path=/data --perm=72 + + # Modify all files so that random positions of 1's with a maximum length of 10 bytes will be replaced with 0's. + kbcli fault io mistake --volume-path=/data --filling=zero --max-occurrences=10 --max-length=1 +`) + +type IOAttribute struct { + Ino uint64 `json:"ino,omitempty"` + Size uint64 `json:"size,omitempty"` + Blocks uint64 `json:"blocks,omitempty"` + Perm uint16 `json:"perm,omitempty"` + Nlink uint32 `json:"nlink,omitempty"` + UID uint32 `json:"uid,omitempty"` + GID uint32 `json:"gid,omitempty"` +} + +type IOMistake struct { + Filling string `json:"filling,omitempty"` + MaxOccurrences int `json:"maxOccurrences,omitempty"` + MaxLength int `json:"maxLength,omitempty"` +} + +type IOChaosOptions struct { + // Parameters required by the `latency` command. + Delay string `json:"delay"` + + // Parameters required by the `fault` command. + Errno int `json:"errno"` + + // Parameters required by the `attribute` command. + IOAttribute `json:"attr,omitempty"` + + // Parameters required by the `mistake` command. + IOMistake `json:"mistake,omitempty"` + + VolumePath string `json:"volumePath"` + Path string `json:"path"` + Percent int `json:"percent"` + Methods []string `json:"methods,omitempty"` + ContainerNames []string `json:"containerNames,omitempty"` + + FaultBaseOptions +} + +func NewIOChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "io", + Short: "IO chaos.", + } + cmd.AddCommand( + NewIOLatencyCmd(f, streams), + NewIOFaultCmd(f, streams), + NewIOAttributeOverrideCmd(f, streams), + NewIOMistakeCmd(f, streams), + ) + return cmd +} + +func NewIOChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *IOChaosOptions { + o := &IOChaosOptions{ + FaultBaseOptions: FaultBaseOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateIOChaos, + GVR: GetGVR(Group, Version, ResourceIOChaos), + }, + Action: action, + }, + } + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o +} + +func NewIOLatencyCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewIOChaosOptions(f, streams, string(v1alpha1.IoLatency)) + cmd := o.NewCobraCommand(Latency, LatencyShort) + + o.AddCommonFlag(cmd, f) + cmd.Flags().StringVar(&o.Delay, "delay", "", `Specific delay time.`) + + util.CheckErr(cmd.MarkFlagRequired("delay")) + return cmd +} + +func NewIOFaultCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewIOChaosOptions(f, streams, string(v1alpha1.IoFaults)) + cmd := o.NewCobraCommand(Errno, ErrnoShort) + + o.AddCommonFlag(cmd, f) + cmd.Flags().IntVar(&o.Errno, "errno", 0, `The returned error number.`) + + util.CheckErr(cmd.MarkFlagRequired("errno")) + return cmd +} + +func NewIOAttributeOverrideCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewIOChaosOptions(f, streams, string(v1alpha1.IoAttrOverride)) + cmd := o.NewCobraCommand(Attribute, AttributeShort) + + o.AddCommonFlag(cmd, f) + cmd.Flags().Uint64Var(&o.Ino, "ino", 0, `ino number.`) + cmd.Flags().Uint64Var(&o.Size, "size", 0, `File size.`) + cmd.Flags().Uint64Var(&o.Blocks, "blocks", 0, `The number of blocks the file occupies.`) + cmd.Flags().Uint16Var(&o.Perm, "perm", 0, `Decimal representation of file permissions.`) + cmd.Flags().Uint32Var(&o.Nlink, "nlink", 0, `The number of hard links.`) + cmd.Flags().Uint32Var(&o.UID, "uid", 0, `Owner's user ID.`) + cmd.Flags().Uint32Var(&o.GID, "gid", 0, `The owner's group ID.`) + + return cmd +} + +func NewIOMistakeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewIOChaosOptions(f, streams, string(v1alpha1.IoMistake)) + cmd := o.NewCobraCommand(Mistake, MistakeShort) + + o.AddCommonFlag(cmd, f) + cmd.Flags().StringVar(&o.Filling, "filling", "", `The filling content of the error data can only be zero (filling with 0) or random (filling with random bytes).`) + cmd.Flags().IntVar(&o.MaxOccurrences, "max-occurrences", 1, `The maximum number of times an error can occur per operation.`) + cmd.Flags().IntVar(&o.MaxLength, "max-length", 1, `The maximum length (in bytes) of each error.`) + + util.CheckErr(cmd.MarkFlagRequired("filling")) + util.CheckErr(cmd.MarkFlagRequired("max-occurrences")) + util.CheckErr(cmd.MarkFlagRequired("max-length")) + + return cmd +} + +func (o *IOChaosOptions) NewCobraCommand(use, short string) *cobra.Command { + return &cobra.Command{ + Use: use, + Short: short, + Example: faultIOExample, + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run()) + }, + } +} + +func (o *IOChaosOptions) AddCommonFlag(cmd *cobra.Command, f cmdutil.Factory) { + o.FaultBaseOptions.AddCommonFlag(cmd) + + cmd.Flags().StringVar(&o.VolumePath, "volume-path", "", `The mount point of the volume in the target container must be the root directory of the mount.`) + cmd.Flags().StringVar(&o.Path, "path", "", `The effective scope of the injection error can be a wildcard or a single file.`) + cmd.Flags().IntVar(&o.Percent, "percent", 100, `Probability of failure per operation, in %.`) + cmd.Flags().StringArrayVar(&o.Methods, "method", nil, "The file system calls that need to inject faults. For example: WRITE READ") + cmd.Flags().StringArrayVarP(&o.ContainerNames, "container", "c", nil, "The name of the container, such as mysql, prometheus.If it's empty, the first container will be injected.") + + util.CheckErr(cmd.MarkFlagRequired("volume-path")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) +} + +func (o *IOChaosOptions) Validate() error { + return o.BaseValidate() +} + +func (o *IOChaosOptions) Complete() error { + return o.BaseComplete() +} + +func (o *IOChaosOptions) PreCreate(obj *unstructured.Unstructured) error { + c := &v1alpha1.IOChaos{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, c); err != nil { + return err + } + + data, e := runtime.DefaultUnstructuredConverter.ToUnstructured(c) + if e != nil { + return e + } + obj.SetUnstructuredContent(data) + return nil +} diff --git a/internal/cli/cmd/fault/fault_io_test.go b/internal/cli/cmd/fault/fault_io_test.go new file mode 100644 index 000000000..6d4328694 --- /dev/null +++ b/internal/cli/cmd/fault/fault_io_test.go @@ -0,0 +1,146 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "k8s.io/cli-runtime/pkg/genericclioptions" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" +) + +var _ = Describe("Fault IO", func() { + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + }) + + AfterEach(func() { + tf.Cleanup() + }) + + Context("test fault io", func() { + + It("fault io latency", func() { + inputs := [][]string{ + {"--mode=one", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--mode=fixed", "--value=2", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--mode=fixed-percent", "--value=50", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--mode=random-max-percent", "--value=50", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--ns-fault=kb-system", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--node=minikube-m02", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--label=app.kubernetes.io/component=mysql", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--node-label=kubernetes.io/arch=arm64", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--annotation=example-annotation=group-a", "--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--delay=10s", "--volume-path=/data", "--dry-run=client"}, + {"--delay=10s", "--volume-path=/data", "--path=test.txt", "--percent=50", "--method=READ", "-c=mysql", "--dry-run=client"}, + } + o := NewIOChaosOptions(tf, streams, string(v1alpha1.IoLatency)) + cmd := o.NewCobraCommand(Latency, LatencyShort) + o.AddCommonFlag(cmd, tf) + cmd.Flags().StringVar(&o.Delay, "delay", "", `Specific delay time.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault io errno", func() { + inputs := [][]string{ + {"--errno=22", "--volume-path=/data", "--dry-run=client"}, + {"--errno=22", "--volume-path=/data", "--path=test.txt", "--percent=50", "--method=READ", "--dry-run=client"}, + } + o := NewIOChaosOptions(tf, streams, string(v1alpha1.IoFaults)) + cmd := o.NewCobraCommand(Errno, ErrnoShort) + o.AddCommonFlag(cmd, tf) + cmd.Flags().IntVar(&o.Errno, "errno", 0, `The returned error number.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault io attribute", func() { + inputs := [][]string{ + {"--perm=72", "--size=72", "--blocks=72", "--nlink=72", "--ino=72", + "--uid=72", "--gid=72", "--volume-path=/data", "--dry-run=client"}, + } + o := NewIOChaosOptions(tf, streams, string(v1alpha1.IoAttrOverride)) + cmd := o.NewCobraCommand(Attribute, AttributeShort) + o.AddCommonFlag(cmd, tf) + cmd.Flags().Uint64Var(&o.Ino, "ino", 0, `ino number.`) + cmd.Flags().Uint64Var(&o.Size, "size", 0, `File size.`) + cmd.Flags().Uint64Var(&o.Blocks, "blocks", 0, `The number of blocks the file occupies.`) + cmd.Flags().Uint16Var(&o.Perm, "perm", 0, `Decimal representation of file permissions.`) + cmd.Flags().Uint32Var(&o.Nlink, "nlink", 0, `The number of hard links.`) + cmd.Flags().Uint32Var(&o.UID, "uid", 0, `Owner's user ID.`) + cmd.Flags().Uint32Var(&o.GID, "gid", 0, `The owner's group ID.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + It("fault io mistake", func() { + inputs := [][]string{ + {"--volume-path=/data", "--filling=zero", "--max-occurrences=10", "--max-length=1", "--dry-run=client"}, + {"--volume-path=/data", "--filling=random", "--max-occurrences=10", "--max-length=1", "--dry-run=client"}, + {"--volume-path=/data", "--filling=zero", "--max-occurrences=10", "--max-length=1", "--path=test.txt", "--percent=50", "--method=READ", "--dry-run=client"}, + } + o := NewIOChaosOptions(tf, streams, string(v1alpha1.IoMistake)) + cmd := o.NewCobraCommand(Mistake, MistakeShort) + o.AddCommonFlag(cmd, tf) + cmd.Flags().StringVar(&o.Filling, "filling", "", `The filling content of the error data can only be zero (filling with 0) or random (filling with random bytes).`) + cmd.Flags().IntVar(&o.MaxOccurrences, "max-occurrences", 1, `The maximum number of times an error can occur per operation.`) + cmd.Flags().IntVar(&o.MaxLength, "max-length", 1, `The maximum length (in bytes) of each error.`) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + + }) +}) diff --git a/internal/cli/cmd/fault/fault_stress.go b/internal/cli/cmd/fault/fault_stress.go new file mode 100644 index 000000000..6e1cc932b --- /dev/null +++ b/internal/cli/cmd/fault/fault_stress.go @@ -0,0 +1,146 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "fmt" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/create" +) + +var faultStressExample = templates.Examples(` + # Affects the first container in default namespace's all pods.Making CPU load up to 50%, and the memory up to 100MB. + kbcli fault stress --cpu-worker=2 --cpu-load=50 --memory-worker=1 --memory-size=100Mi + + # Affects the first container in mycluster-mysql-0 pod. Making the CPU load up to 50%, and the memory up to 500MB. + kbcli fault stress mycluster-mysql-0 --cpu-worker=2 --cpu-load=50 + + # Affects the mysql container in mycluster-mysql-0 pod. Making the memory up to 500MB. + kbcli fault stress mycluster-mysql-0 --memory-worker=2 --memory-size=500Mi -c=mysql +`) + +type CPU struct { + Workers int `json:"workers"` + Load int `json:"load"` +} + +type Memory struct { + Workers int `json:"workers"` + Size string `json:"size"` +} + +type Stressors struct { + CPU `json:"cpu"` + Memory `json:"memory"` +} + +type StressChaosOptions struct { + Stressors `json:"stressors"` + ContainerNames []string `json:"containerNames,omitempty"` + + FaultBaseOptions +} + +func NewStressChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *StressChaosOptions { + o := &StressChaosOptions{ + FaultBaseOptions: FaultBaseOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateStressChaos, + GVR: GetGVR(Group, Version, ResourceStressChaos), + }, + Action: action, + }, + } + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o +} + +func NewStressChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewStressChaosOptions(f, streams, "") + cmd := o.NewCobraCommand(Stress, StressShort) + + o.AddCommonFlag(cmd, f) + return cmd +} + +func (o *StressChaosOptions) NewCobraCommand(use, short string) *cobra.Command { + return &cobra.Command{ + Use: use, + Short: short, + Example: faultStressExample, + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run()) + }, + } +} + +func (o *StressChaosOptions) AddCommonFlag(cmd *cobra.Command, f cmdutil.Factory) { + o.FaultBaseOptions.AddCommonFlag(cmd) + + cmd.Flags().IntVar(&o.CPU.Workers, "cpu-worker", 0, `Specifies the number of threads that exert CPU pressure.`) + cmd.Flags().IntVar(&o.CPU.Load, "cpu-load", 0, `Specifies the percentage of CPU occupied. 0 means no extra load added, 100 means full load. The total load is workers * load.`) + cmd.Flags().IntVar(&o.Memory.Workers, "memory-worker", 0, `Specifies the number of threads that apply memory pressure.`) + cmd.Flags().StringVar(&o.Memory.Size, "memory-size", "", `Specify the size of the allocated memory or the percentage of the total memory, and the sum of the allocated memory is size. For example:256MB or 25%`) + cmd.Flags().StringArrayVarP(&o.ContainerNames, "container", "c", nil, "The name of the container, such as mysql, prometheus.If it's empty, the first container will be injected.") + + // register flag completion func + registerFlagCompletionFunc(cmd, f) +} + +func (o *StressChaosOptions) Validate() error { + if o.Memory.Workers == 0 && o.CPU.Workers == 0 { + return fmt.Errorf("the CPU or Memory workers must have at least one greater than 0, Use --cpu-workers or --memory-workers to specify") + } + + return o.BaseValidate() +} + +func (o *StressChaosOptions) Complete() error { + return o.BaseComplete() +} + +func (o *StressChaosOptions) PreCreate(obj *unstructured.Unstructured) error { + c := &v1alpha1.StressChaos{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, c); err != nil { + return err + } + + data, e := runtime.DefaultUnstructuredConverter.ToUnstructured(c) + if e != nil { + return e + } + obj.SetUnstructuredContent(data) + return nil +} diff --git a/internal/cli/cmd/fault/fault_stress_test.go b/internal/cli/cmd/fault/fault_stress_test.go new file mode 100644 index 000000000..533527f74 --- /dev/null +++ b/internal/cli/cmd/fault/fault_stress_test.go @@ -0,0 +1,78 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" +) + +var _ = Describe("Fault Stress", func() { + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + }) + + AfterEach(func() { + tf.Cleanup() + }) + Context("test fault stress", func() { + + It("fault stress", func() { + inputs := [][]string{ + {"--mode=one", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--mode=fixed", "--value=2", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--mode=fixed-percent", "--value=50", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--mode=random-max-percent", "--value=50", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--ns-fault=kb-system", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--node=minikube-m02", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--label=app.kubernetes.io/component=mysql", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--node-label=kubernetes.io/arch=arm64", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--annotation=example-annotation=group-a", "--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--cpu-worker=2", "--cpu-load=50", "--dry-run=client"}, + {"--memory-worker=2", "--memory-size=500Mi", "-c=mysql", "--dry-run=client"}, + {"--cpu-worker=2", "--cpu-load=50", "--memory-worker=1", "--memory-size=100Mi", "--dry-run=client"}, + {"--cpu-worker=2", "--cpu-load=50", "--memory-worker=1", "--memory-size=100Mi", "--dry-run=client", "--container=mysql"}, + } + o := NewStressChaosOptions(tf, streams, "") + cmd := o.NewCobraCommand(Stress, StressShort) + o.AddCommonFlag(cmd, tf) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + }) +}) diff --git a/internal/cli/cmd/fault/fault_time.go b/internal/cli/cmd/fault/fault_time.go new file mode 100644 index 000000000..42950aa41 --- /dev/null +++ b/internal/cli/cmd/fault/fault_time.go @@ -0,0 +1,135 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var faultTimeExample = templates.Examples(` + # Affects the first container in default namespace's all pods.Shifts the clock back five seconds. + kbcli fault time --time-offset=-5s + + # Affects the first container in default namespace's all pods. + kbcli fault time --time-offset=-5m5s + + # Affects the first container in mycluster-mysql-0 pod. Shifts the clock forward five seconds. + kbcli fault time mycluster-mysql-0 --time-offset=+5s50ms + + # Affects the mysql container in mycluster-mysql-0 pod. Shifts the clock forward five seconds. + kbcli fault time mycluster-mysql-0 --time-offset=+5s -c=mysql + + # The clock that specifies the effect of time offset is CLOCK_REALTIME. + kbcli fault time mycluster-mysql-0 --time-offset=+5s --clock-id=CLOCK_REALTIME -c=mysql +`) + +type TimeChaosOptions struct { + TimeOffset string `json:"timeOffset"` + + ClockIds []string `json:"clockIds,omitempty"` + + ContainerNames []string `json:"containerNames,omitempty"` + + FaultBaseOptions +} + +func NewTimeChaosOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, action string) *TimeChaosOptions { + o := &TimeChaosOptions{ + FaultBaseOptions: FaultBaseOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateTimeChaos, + GVR: GetGVR(Group, Version, ResourceTimeChaos), + }, + Action: action, + }, + } + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o +} + +func NewTimeChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewTimeChaosOptions(f, streams, "") + cmd := o.NewCobraCommand(Time, TimeShort) + + o.AddCommonFlag(cmd, f) + return cmd +} + +func (o *TimeChaosOptions) NewCobraCommand(use, short string) *cobra.Command { + return &cobra.Command{ + Use: use, + Short: short, + Example: faultTimeExample, + Run: func(cmd *cobra.Command, args []string) { + o.Args = args + cmdutil.CheckErr(o.CreateOptions.Complete()) + cmdutil.CheckErr(o.Validate()) + cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Run()) + }, + } +} + +func (o *TimeChaosOptions) AddCommonFlag(cmd *cobra.Command, f cmdutil.Factory) { + o.FaultBaseOptions.AddCommonFlag(cmd) + + cmd.Flags().StringVar(&o.TimeOffset, "time-offset", "", "Specifies the length of the time offset. For example: -5s, -10m100ns.") + cmd.Flags().StringArrayVar(&o.ClockIds, "clock-id", nil, `Specifies the clock on which the time offset acts.If it's empty, it will be set to ['CLOCK_REALTIME'].See clock_gettime [https://man7.org/linux/man-pages/man2/clock_gettime.2.html] document for details.`) + cmd.Flags().StringArrayVarP(&o.ContainerNames, "container", "c", nil, `Specifies the injected container name. For example: mysql. If it's empty, the first container will be injected.`) + + util.CheckErr(cmd.MarkFlagRequired("time-offset")) + + // register flag completion func + registerFlagCompletionFunc(cmd, f) +} + +func (o *TimeChaosOptions) Validate() error { + return o.BaseValidate() +} + +func (o *TimeChaosOptions) Complete() error { + return o.BaseComplete() +} + +func (o *TimeChaosOptions) PreCreate(obj *unstructured.Unstructured) error { + c := &v1alpha1.TimeChaos{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, c); err != nil { + return err + } + + data, e := runtime.DefaultUnstructuredConverter.ToUnstructured(c) + if e != nil { + return e + } + obj.SetUnstructuredContent(data) + return nil +} diff --git a/internal/cli/cmd/fault/fault_time_test.go b/internal/cli/cmd/fault/fault_time_test.go new file mode 100644 index 000000000..ef6b108f7 --- /dev/null +++ b/internal/cli/cmd/fault/fault_time_test.go @@ -0,0 +1,76 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" +) + +var _ = Describe("Fault Time", func() { + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + }) + + AfterEach(func() { + tf.Cleanup() + }) + Context("test fault time", func() { + + It("fault time", func() { + inputs := [][]string{ + {"--mode=one", "--time-offset=-5s", "--dry-run=client"}, + {"--mode=fixed", "--value=2", "--time-offset=-5s", "--dry-run=client"}, + {"--mode=fixed-percent", "--value=50", "--time-offset=-5s", "--dry-run=client"}, + {"--mode=random-max-percent", "--value=50", "--time-offset=-5s", "--dry-run=client"}, + {"--ns-fault=kb-system", "--time-offset=-5s", "--dry-run=client"}, + {"--node=minikube-m02", "--time-offset=-5s", "--dry-run=client"}, + {"--label=app.kubernetes.io/component=mysql", "--time-offset=-5s", "--dry-run=client"}, + {"--node-label=kubernetes.io/arch=arm64", "--time-offset=-5s", "--dry-run=client"}, + {"--annotation=example-annotation=group-a", "--time-offset=-5s", "--dry-run=client"}, + {"--time-offset=-5s", "--dry-run=client"}, + {"--time-offset=+5s", "--clock-id=CLOCK_REALTIME", "-c=mysql", "--dry-run=client"}, + } + o := NewTimeChaosOptions(tf, streams, "") + cmd := o.NewCobraCommand(Time, TimeShort) + o.AddCommonFlag(cmd, tf) + + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.CreateOptions.Complete()) + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + Expect(o.Run()).Should(Succeed()) + } + }) + }) +}) diff --git a/internal/cli/create/template/io_chaos_template.cue b/internal/cli/create/template/io_chaos_template.cue new file mode 100644 index 000000000..cb0a798dd --- /dev/null +++ b/internal/cli/create/template/io_chaos_template.cue @@ -0,0 +1,69 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// required, command line input options for parameters and flags +options: { + namespace: string + action: string + selector: {} + mode: string + value: string + duration: string + + delay: string + errno: int + attr?: {} + mistake?: {} + + volumePath: string + path: string + percent: int + methods: [...] + containerNames: [...] +} + +// required, k8s api resource content +content: { + kind: "IOChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "io-chaos-" + namespace: options.namespace + } + spec:{ + selector: options.selector + mode: options.mode + value: options.value + action: options.action + duration: options.duration + + delay: options.delay + errno: options.errno + if len(options.attr) != 0 { + attr: options.attr + } + if len(options.mistake) != 0 { + mistake: options.mistake + } + + volumePath: options.volumePath + path: options.path + percent: options.percent + methods: options.methods + containerNames: options.containerNames + } +} diff --git a/internal/cli/create/template/stress_chaos_template.cue b/internal/cli/create/template/stress_chaos_template.cue new file mode 100644 index 000000000..95d490a85 --- /dev/null +++ b/internal/cli/create/template/stress_chaos_template.cue @@ -0,0 +1,46 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// required, command line input options for parameters and flags +options: { + namespace: string + selector: {} + mode: string + value: string + duration: string + + stressors: {} + containerNames: [...] +} + +// required, k8s api resource content +content: { + kind: "StressChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "stress-chaos-" + namespace: options.namespace + } + spec:{ + selector: options.selector + mode: options.mode + value: options.value + duration: options.duration + stressors: options.stressors + containerNames: options.containerNames + } +} diff --git a/internal/cli/create/template/time_chaos_template.cue b/internal/cli/create/template/time_chaos_template.cue new file mode 100644 index 000000000..debd3585c --- /dev/null +++ b/internal/cli/create/template/time_chaos_template.cue @@ -0,0 +1,49 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// required, command line input options for parameters and flags +options: { + namespace: string + selector: {} + mode: string + value: string + duration: string + + timeOffset: string + clockIds: [...] + containerNames: [...] +} + +// required, k8s api resource content +content: { + kind: "TimeChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "time-chaos-" + namespace: options.namespace + } + spec:{ + selector: options.selector + mode: options.mode + value: options.value + duration: options.duration + + timeOffset: options.timeOffset + clockIds: options.clockIds + containerNames: options.containerNames + } +} From fbaad03652cfdb4a38f7c1d15e6b80af77e9c06c Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Wed, 17 May 2023 20:32:40 +0800 Subject: [PATCH 318/439] chore: have Stateful workloadType using StatefulSet's default spec.podManagementPolicy=OrderReady and spec.minReadySeconds=0 settings (#3122) Co-authored-by: wangyelei --- apis/apps/v1alpha1/cluster_types.go | 23 ++ apis/apps/v1alpha1/clusterdefinition_types.go | 295 +++++++++++++++-- apis/apps/v1alpha1/type.go | 16 + ...apps.kubeblocks.io_clusterdefinitions.yaml | 297 ++++++++++++++++-- .../bases/apps.kubeblocks.io_clusters.yaml | 6 + controllers/apps/cluster_controller_test.go | 253 ++++++++------- controllers/apps/cluster_status_utils.go | 13 +- controllers/apps/cluster_status_utils_test.go | 16 +- .../apps/components/consensus/consensus.go | 115 +++---- .../components/consensus/consensus_test.go | 6 +- .../components/consensus/consensus_utils.go | 87 +---- .../consensus/consensus_utils_test.go | 2 +- .../components/deployment_controller_test.go | 10 +- .../components/replication/replication.go | 8 +- .../replication/replication_switch_test.go | 2 +- .../replication/replication_switch_utils.go | 2 +- .../replication_switch_utils_test.go | 2 +- .../apps/components/stateful/stateful.go | 134 +++++++- .../apps/components/stateful/stateful_test.go | 6 +- .../stateful_set_controller_test.go | 13 +- .../components/stateless/stateless_test.go | 4 +- .../apps/components/util/component_utils.go | 18 ++ .../components/util/component_utils_test.go | 8 +- controllers/apps/components/util/plan.go | 10 +- controllers/apps/configuration/policy_util.go | 2 +- .../apps/configuration/reconfigure_policy.go | 8 +- .../rolling_upgrade_policy_test.go | 12 +- .../operations/horizontal_scaling_test.go | 2 +- .../apps/operations/ops_progress_util_test.go | 6 +- controllers/apps/operations/restart_test.go | 4 +- controllers/apps/operations/suite_test.go | 4 +- .../apps/operations/util/common_util_test.go | 2 +- .../apps/operations/volume_expansion_test.go | 4 +- .../apps/opsrequest_controller_test.go | 88 +++--- controllers/apps/tls_utils_test.go | 9 +- .../restorejob_controller_test.go | 2 +- ...apps.kubeblocks.io_clusterdefinitions.yaml | 297 ++++++++++++++++-- .../crds/apps.kubeblocks.io_clusters.yaml | 6 + .../config/pg12-config-constraint.cue | 2 +- .../config/pg14-config-constraint.cue | 2 +- internal/controller/builder/builder.go | 35 +-- internal/controller/builder/builder_test.go | 2 +- .../builder/cue/backup_job_template.cue | 24 +- .../builder/cue/backup_manifests_template.cue | 24 +- .../builder/cue/config_manager_sidecar.cue | 24 +- .../builder/cue/config_template.cue | 24 +- .../builder/cue/conn_credential_template.cue | 24 +- .../cue/delete_pvc_cron_job_template.cue | 24 +- .../builder/cue/deployment_template.cue | 24 +- .../builder/cue/env_config_template.cue | 26 +- .../builder/cue/headless_service_template.cue | 24 +- .../controller/builder/cue/pdb_template.cue | 33 +- .../builder/cue/pitr_job_template.cue | 24 +- .../controller/builder/cue/pvc_template.cue | 24 +- .../builder/cue/service_template.cue | 24 +- .../builder/cue/snapshot_template.cue | 24 +- .../builder/cue/statefulset_template.cue | 30 +- .../builder/cue/tls_certs_secret_template.cue | 24 +- internal/controller/component/component.go | 31 +- internal/controller/component/type.go | 6 +- .../lifecycle/cluster_plan_builder.go | 15 +- .../lifecycle/transformer_cluster_deletion.go | 7 +- .../lifecycle/transformer_cluster_status.go | 2 +- internal/controller/plan/prepare.go | 114 +++---- internal/controller/plan/prepare_test.go | 109 ++++--- .../apps/cluster_consensus_test_util.go | 24 +- .../apps/cluster_stateless_test_util.go | 8 +- internal/testutil/apps/cluster_util.go | 31 +- internal/testutil/apps/clusterdef_factory.go | 2 +- internal/testutil/apps/common_util.go | 4 +- internal/testutil/apps/constant.go | 4 +- internal/testutil/apps/native_object_util.go | 2 +- internal/testutil/apps/opsrequest_util.go | 2 +- internal/testutil/k8s/deployment_util.go | 18 ++ internal/testutil/k8s/statefulset_util.go | 34 +- 75 files changed, 1780 insertions(+), 867 deletions(-) diff --git a/apis/apps/v1alpha1/cluster_types.go b/apis/apps/v1alpha1/cluster_types.go index c3f0fb604..9c30bebf2 100644 --- a/apis/apps/v1alpha1/cluster_types.go +++ b/apis/apps/v1alpha1/cluster_types.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // ClusterSpec defines the desired state of Cluster @@ -185,6 +186,28 @@ type ClusterComponentSpec struct { // serviceAccountName is the name of the ServiceAccount that component runs depend on. // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` + + // noCreatePDB defines PodDistruptionBudget creation behavior, set to true if creation of PodDistruptionBudget + // for this component is not needed. Defaults to false. + // +kubebuilder:default=false + // +optional + NoCreatePDB bool `json:"noCreatePDB,omitempty"` +} + +// GetMinAvailable wraps the 'prefer' value return, as for component replicaCount <= 1 will return 0 value, +// and for replicaCount=2 will return 1. +func (r *ClusterComponentSpec) GetMinAvailable(prefer *intstr.IntOrString) *intstr.IntOrString { + if r == nil || r.NoCreatePDB || prefer == nil { + return nil + } + if r.Replicas <= 1 { + m := intstr.FromInt(0) + return &m + } else if r.Replicas == 2 { + m := intstr.FromInt(1) + return &m + } + return prefer } type ComponentMessageMap map[string]string diff --git a/apis/apps/v1alpha1/clusterdefinition_types.go b/apis/apps/v1alpha1/clusterdefinition_types.go index 8d0a4ffd8..012b4f8b1 100644 --- a/apis/apps/v1alpha1/clusterdefinition_types.go +++ b/apis/apps/v1alpha1/clusterdefinition_types.go @@ -22,6 +22,7 @@ package v1alpha1 import ( "strings" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -84,8 +85,7 @@ type SystemAccountSpec struct { // CmdExecutorConfig specifies how to perform creation and deletion statements. type CmdExecutorConfig struct { CommandExecutorEnvItem `json:",inline"` - - CommandExecutorItem `json:",inline"` + CommandExecutorItem `json:",inline"` } // PasswordConfig helps provide to customize complexity of password generation pattern. @@ -243,6 +243,7 @@ type VolumeTypeSpec struct { // ClusterComponentDefinition provides a workload component specification template, // with attributes that strongly work with stateful workloads and day-2 operations // behaviors. +// +kubebuilder:validation:XValidation:rule="has(self.workloadType) && self.workloadType == 'Consensus' ? has(self.consensusSpec) : !has(self.consensusSpec)",message="componentDefs.consensusSpec is required when componentDefs.workloadType is Consensus, and forbidden otherwise" type ClusterComponentDefinition struct { // name of the component, it can be any valid string. // +kubebuilder:validation:Required @@ -267,14 +268,6 @@ type ClusterComponentDefinition struct { // +optional CharacterType string `json:"characterType,omitempty"` - // The maximum number of pods that can be unavailable during scaling. - // Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). - // Absolute number is calculated from percentage by rounding down. This value is ignored - // if workloadType is Consensus. - // +kubebuilder:validation:XIntOrString - // +optional - MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` - // The configSpec field provided by provider, and // finally this configTemplateRefs will be rendered into the user's own configuration file according to the user's cluster. // +optional @@ -320,13 +313,21 @@ type ClusterComponentDefinition struct { // +optional Service *ServiceSpec `json:"service,omitempty"` + // statelessSpec defines stateless related spec if workloadType is Stateless. + // +optional + StatelessSpec *StatelessSetSpec `json:"statelessSpec,omitempty"` + + // statefulSpec defines stateful related spec if workloadType is Stateful. + // +optional + StatefulSpec *StatefulSetSpec `json:"statefulSpec,omitempty"` + // consensusSpec defines consensus related spec if workloadType is Consensus, required if workloadType is Consensus. // +optional ConsensusSpec *ConsensusSetSpec `json:"consensusSpec,omitempty"` - // replicationSpec defines replication related spec if workloadType is Replication, required if workloadType is Replication. + // replicationSpec defines replication related spec if workloadType is Replication. // +optional - ReplicationSpec *ReplicationSpec `json:"replicationSpec,omitempty"` + ReplicationSpec *ReplicationSetSpec `json:"replicationSpec,omitempty"` // horizontalScalePolicy controls the behavior of horizontal scale. // +optional @@ -361,6 +362,118 @@ type ClusterComponentDefinition struct { CustomLabelSpecs []CustomLabelSpec `json:"customLabelSpecs,omitempty"` } +func (r *ClusterComponentDefinition) GetStatefulSetWorkload() StatefulSetWorkload { + switch r.WorkloadType { + case Stateless: + return nil + case Stateful: + return r.StatefulSpec + case Consensus: + return r.ConsensusSpec + case Replication: + return r.ReplicationSpec + } + panic("unreachable") +} + +// GetMinAvailable get workload's minAvailable settings, return 51% for workloadType=Consensus, +// value 1 pod for workloadType=[Stateless|Stateful|Replication]. +func (r *ClusterComponentDefinition) GetMinAvailable() *intstr.IntOrString { + if r == nil { + return nil + } + switch r.WorkloadType { + case Consensus: + // Consensus workload have min pods of >50%. + v := intstr.FromString("51%") + return &v + case Replication, Stateful, Stateless: + // Stateful & Replication workload have min. pod being 1. + v := intstr.FromInt(1) + return &v + } + return nil +} + +// GetMaxUnavailable get workload's maxUnavailable settings, this value is not suitable for PDB.spec.maxUnavailable +// usage, as a PDB with maxUnavailable=49% and if workload's replicaCount=3 and allowed disruption pod count is 2, +// check following setup: +// +// #cmd: kubectl get sts,po,pdb -l app.kubernetes.io/instance=consul +// NAME READY AGE +// statefulset.apps/consul 3/3 3h23m +// +// NAME READY STATUS RESTARTS AGE +// pod/consul-0 1/1 Running 0 3h +// pod/consul-2 1/1 Running 0 16s +// pod/consul-1 1/1 Running 0 16s +// +// NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE +// poddisruptionbudget.policy/consul N/A 49% 2 3h23m +// +// VS. using minAvailable=51% will result allowed disruption pod count is 1 +// +// NAME READY AGE +// statefulset.apps/consul 3/3 3h26m +// +// NAME READY STATUS RESTARTS AGE +// pod/consul-0 1/1 Running 0 3h3m +// pod/consul-2 1/1 Running 0 3m35s +// pod/consul-1 1/1 Running 0 3m35s +// +// NAME MIN AVAILABLE MAX UNAVAILABLE ALLOWED DISRUPTIONS AGE +// poddisruptionbudget.policy/consul 51% N/A 1 3h26m +func (r *ClusterComponentDefinition) GetMaxUnavailable() *intstr.IntOrString { + if r == nil { + return nil + } + + getMaxUnavailable := func(ssus appsv1.StatefulSetUpdateStrategy) *intstr.IntOrString { + if ssus.RollingUpdate == nil { + return nil + } + return ssus.RollingUpdate.MaxUnavailable + } + + switch r.WorkloadType { + case Stateless: + if r.StatelessSpec == nil || r.StatelessSpec.UpdateStrategy.RollingUpdate == nil { + return nil + } + return r.StatelessSpec.UpdateStrategy.RollingUpdate.MaxUnavailable + case Stateful, Consensus, Replication: + _, s := r.GetStatefulSetWorkload().FinalStsUpdateStrategy() + return getMaxUnavailable(s) + } + panic("unreachable") +} + +func (r *ClusterComponentDefinition) IsStatelessWorkload() bool { + return r.WorkloadType == Stateless +} + +func (r *ClusterComponentDefinition) GetCommonStatefulSpec() (*StatefulSetSpec, error) { + if r.IsStatelessWorkload() { + return nil, ErrWorkloadTypeIsStateless + } + switch r.WorkloadType { + case Stateful: + return r.StatefulSpec, nil + case Consensus: + if r.ConsensusSpec != nil { + return &r.ConsensusSpec.StatefulSetSpec, nil + } + case Replication: + if r.ReplicationSpec != nil { + return &r.ReplicationSpec.StatefulSetSpec, nil + } + default: + panic("unreachable") + // return nil, ErrWorkloadTypeIsUnknown + } + return nil, nil +} + type ServiceSpec struct { // The list of ports that are exposed by this service. // More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies @@ -516,7 +629,100 @@ type ClusterDefinitionProbes struct { RoleProbeTimeoutAfterPodsReady int32 `json:"roleProbeTimeoutAfterPodsReady,omitempty"` } +type StatelessSetSpec struct { + // updateStrategy defines the underlying deployment strategy to use to replace existing pods with new ones. + // +optional + // +patchStrategy=retainKeys + UpdateStrategy appsv1.DeploymentStrategy `json:"updateStrategy,omitempty"` +} + +type StatefulSetSpec struct { + // updateStrategy, Pods update strategy. + // In case of workloadType=Consensus the update strategy will be following: + // + // serial: update Pods one by one that guarantee minimum component unavailable time. + // Learner -> Follower(with AccessMode=none) -> Follower(with AccessMode=readonly) -> Follower(with AccessMode=readWrite) -> Leader + // bestEffortParallel: update Pods in parallel that guarantee minimum component un-writable time. + // Learner, Follower(minority) in parallel -> Follower(majority) -> Leader, keep majority online all the time. + // parallel: force parallel + // +kubebuilder:default=Serial + // +optional + UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"` + + // llPodManagementPolicy is the low-level controls how pods are created during initial scale up, + // when replacing pods on nodes, or when scaling down. + // `OrderedReady` policy specify where pods are created in increasing order (pod-0, then + // pod-1, etc) and the controller will wait until each pod is ready before + // continuing. When scaling down, the pods are removed in the opposite order. + // `Parallel` policy specify create pods in parallel + // to match the desired scale without waiting, and on scale down will delete + // all pods at once. + // +optional + LLPodManagementPolicy appsv1.PodManagementPolicyType `json:"llPodManagementPolicy,omitempty"` + + // llUpdateStrategy indicates the low-level StatefulSetUpdateStrategy that will be + // employed to update Pods in the StatefulSet when a revision is made to + // Template. Will ignore `updateStrategy` attribute if provided. + // +optional + LLUpdateStrategy *appsv1.StatefulSetUpdateStrategy `json:"llUpdateStrategy,omitempty"` +} + +var _ StatefulSetWorkload = &StatefulSetSpec{} + +func (r *StatefulSetSpec) GetUpdateStrategy() UpdateStrategy { + if r == nil { + return SerialStrategy + } + return r.UpdateStrategy +} + +func (r *StatefulSetSpec) FinalStsUpdateStrategy() (appsv1.PodManagementPolicyType, appsv1.StatefulSetUpdateStrategy) { + if r == nil { + r = &StatefulSetSpec{ + UpdateStrategy: SerialStrategy, + } + } + return r.finalStsUpdateStrategy() +} + +func (r *StatefulSetSpec) finalStsUpdateStrategy() (appsv1.PodManagementPolicyType, appsv1.StatefulSetUpdateStrategy) { + if r.LLUpdateStrategy != nil { + return r.LLPodManagementPolicy, *r.LLUpdateStrategy + } + + switch r.UpdateStrategy { + case BestEffortParallelStrategy: + m := intstr.FromString("49%") + return appsv1.ParallelPodManagement, appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ + // alpha feature since v1.24 + // ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#maximum-unavailable-pods + MaxUnavailable: &m, + }, + } + case ParallelStrategy: + return appsv1.ParallelPodManagement, appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + } + case SerialStrategy: + fallthrough + default: + m := intstr.FromInt(1) + return appsv1.OrderedReadyPodManagement, appsv1.StatefulSetUpdateStrategy{ + Type: appsv1.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &appsv1.RollingUpdateStatefulSetStrategy{ + // alpha feature since v1.24 + // ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#maximum-unavailable-pods + MaxUnavailable: &m, + }, + } + } +} + type ConsensusSetSpec struct { + StatefulSetSpec `json:",inline"` + // leader, one single leader. // +kubebuilder:validation:Required Leader ConsensusMember `json:"leader"` @@ -528,16 +734,40 @@ type ConsensusSetSpec struct { // learner, no voting right. // +optional Learner *ConsensusMember `json:"learner,omitempty"` +} - // updateStrategy, Pods update strategy. - // serial: update Pods one by one that guarantee minimum component unavailable time. - // Learner -> Follower(with AccessMode=none) -> Follower(with AccessMode=readonly) -> Follower(with AccessMode=readWrite) -> Leader - // bestEffortParallel: update Pods in parallel that guarantee minimum component un-writable time. - // Learner, Follower(minority) in parallel -> Follower(majority) -> Leader, keep majority online all the time. - // parallel: force parallel - // +kubebuilder:default=Serial - // +optional - UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"` +var _ StatefulSetWorkload = &ConsensusSetSpec{} + +func (r *ConsensusSetSpec) GetUpdateStrategy() UpdateStrategy { + if r == nil { + return SerialStrategy + } + return r.UpdateStrategy +} + +func (r *ConsensusSetSpec) FinalStsUpdateStrategy() (appsv1.PodManagementPolicyType, appsv1.StatefulSetUpdateStrategy) { + if r == nil { + r = NewConsensusSetSpec() + } + if r.LLUpdateStrategy != nil { + return r.LLPodManagementPolicy, *r.LLUpdateStrategy + } + _, s := r.StatefulSetSpec.finalStsUpdateStrategy() + // switch r.UpdateStrategy { + // case SerialStrategy, BestEffortParallelStrategy: + s.Type = appsv1.OnDeleteStatefulSetStrategyType + s.RollingUpdate = nil + // } + return appsv1.ParallelPodManagement, s +} + +func NewConsensusSetSpec() *ConsensusSetSpec { + return &ConsensusSetSpec{ + Leader: DefaultLeader, + StatefulSetSpec: StatefulSetSpec{ + UpdateStrategy: SerialStrategy, + }, + } } type ConsensusMember struct { @@ -561,7 +791,9 @@ type ConsensusMember struct { Replicas *int32 `json:"replicas,omitempty"` } -type ReplicationSpec struct { +type ReplicationSetSpec struct { + StatefulSetSpec `json:",inline"` + // switchPolicies defines a collection of different types of switchPolicy, and each type of switchPolicy is limited to one. // +kubebuilder:validation:Required // +kubebuilder:validation:MinItems=1 @@ -572,6 +804,25 @@ type ReplicationSpec struct { SwitchCmdExecutorConfig *SwitchCmdExecutorConfig `json:"switchCmdExecutorConfig"` } +var _ StatefulSetWorkload = &ReplicationSetSpec{} + +func (r *ReplicationSetSpec) GetUpdateStrategy() UpdateStrategy { + if r == nil { + return SerialStrategy + } + return r.UpdateStrategy +} + +func (r *ReplicationSetSpec) FinalStsUpdateStrategy() (appsv1.PodManagementPolicyType, appsv1.StatefulSetUpdateStrategy) { + if r == nil { + r = &ReplicationSetSpec{} + } + return r.StatefulSetSpec.finalStsUpdateStrategy() + // _, s := r.StatefulSetSpec.finalStsUpdateStrategy() + // s.Type = appsv1.OnDeleteStatefulSetStrategyType + // return appsv1.ParallelPodManagement, s +} + type SwitchPolicy struct { // switchPolicyType defines type of the switchPolicy. // MaximumAvailability: when the primary is active, do switch if the synchronization delay = 0 in the user-defined lagProbe data delay detection logic, otherwise do not switch. The primary is down, switch immediately. diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index 599774e48..7e67763dd 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -21,6 +21,9 @@ along with this program. If not, see . package v1alpha1 import ( + "errors" + + appsv1 "k8s.io/api/apps/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" ) @@ -524,3 +527,16 @@ func RegisterWebhookManager(mgr manager.Manager) { } type ComponentNameSet map[string]struct{} + +var ( + ErrWorkloadTypeIsUnknown = errors.New("workloadType is unknown") + ErrWorkloadTypeIsStateless = errors.New("workloadType should not be stateless") + ErrNotMatchingCompDef = errors.New("not matching componentDefRef") +) + +// StatefulSetWorkload interface +// +kubebuilder:object:generate=false +type StatefulSetWorkload interface { + FinalStsUpdateStrategy() (appsv1.PodManagementPolicyType, appsv1.StatefulSetUpdateStrategy) + GetUpdateStrategy() UpdateStrategy +} diff --git a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml index d4ad71843..868747153 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterdefinitions.yaml @@ -220,17 +220,72 @@ spec: - accessMode - name type: object + llPodManagementPolicy: + description: llPodManagementPolicy is the low-level controls + how pods are created during initial scale up, when replacing + pods on nodes, or when scaling down. `OrderedReady` policy + specify where pods are created in increasing order (pod-0, + then pod-1, etc) and the controller will wait until each + pod is ready before continuing. When scaling down, the + pods are removed in the opposite order. `Parallel` policy + specify create pods in parallel to match the desired scale + without waiting, and on scale down will delete all pods + at once. + type: string + llUpdateStrategy: + description: llUpdateStrategy indicates the low-level StatefulSetUpdateStrategy + that will be employed to update Pods in the StatefulSet + when a revision is made to Template. Will ignore `updateStrategy` + attribute if provided. + properties: + rollingUpdate: + description: RollingUpdate is used to communicate parameters + when Type is RollingUpdateStatefulSetStrategyType. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be unavailable during the update. Value can be + an absolute number (ex: 5) or a percentage of + desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding up. This can not be + 0. Defaults to 1. This field is alpha-level and + is only honored by servers that enable the MaxUnavailableStatefulSet + feature. The field applies to all pods in the + range 0 to Replicas-1. That means if there is + any unavailable pod in the range 0 to Replicas-1, + it will be counted towards MaxUnavailable.' + x-kubernetes-int-or-string: true + partition: + description: Partition indicates the ordinal at + which the StatefulSet should be partitioned for + updates. During a rolling update, all pods from + ordinal Replicas-1 to Partition are updated. All + pods from ordinal Partition-1 to 0 remain untouched. + This is helpful in being able to do a canary based + deployment. The default value is 0. + format: int32 + type: integer + type: object + type: + description: Type indicates the type of the StatefulSetUpdateStrategy. + Default is RollingUpdate. + type: string + type: object updateStrategy: default: Serial - description: 'updateStrategy, Pods update strategy. serial: - update Pods one by one that guarantee minimum component - unavailable time. Learner -> Follower(with AccessMode=none) - -> Follower(with AccessMode=readonly) -> Follower(with - AccessMode=readWrite) -> Leader bestEffortParallel: update - Pods in parallel that guarantee minimum component un-writable - time. Learner, Follower(minority) in parallel -> Follower(majority) - -> Leader, keep majority online all the time. parallel: - force parallel' + description: "updateStrategy, Pods update strategy. In case + of workloadType=Consensus the update strategy will be + following: \n serial: update Pods one by one that guarantee + minimum component unavailable time. Learner -> Follower(with + AccessMode=none) -> Follower(with AccessMode=readonly) + -> Follower(with AccessMode=readWrite) -> Leader bestEffortParallel: + update Pods in parallel that guarantee minimum component + un-writable time. Learner, Follower(minority) in parallel + -> Follower(majority) -> Leader, keep majority online + all the time. parallel: force parallel" enum: - Serial - BestEffortParallel @@ -339,16 +394,6 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - maxUnavailable: - anyOf: - - type: integer - - type: string - description: 'The maximum number of pods that can be unavailable - during scaling. Value can be an absolute number (ex: 5) or - a percentage of desired pods (ex: 10%). Absolute number is - calculated from percentage by rounding down. This value is - ignored if workloadType is Consensus.' - x-kubernetes-int-or-string: true monitor: description: monitor is monitoring config which provided by provider. @@ -8050,9 +8095,62 @@ spec: type: object replicationSpec: description: replicationSpec defines replication related spec - if workloadType is Replication, required if workloadType is - Replication. + if workloadType is Replication. properties: + llPodManagementPolicy: + description: llPodManagementPolicy is the low-level controls + how pods are created during initial scale up, when replacing + pods on nodes, or when scaling down. `OrderedReady` policy + specify where pods are created in increasing order (pod-0, + then pod-1, etc) and the controller will wait until each + pod is ready before continuing. When scaling down, the + pods are removed in the opposite order. `Parallel` policy + specify create pods in parallel to match the desired scale + without waiting, and on scale down will delete all pods + at once. + type: string + llUpdateStrategy: + description: llUpdateStrategy indicates the low-level StatefulSetUpdateStrategy + that will be employed to update Pods in the StatefulSet + when a revision is made to Template. Will ignore `updateStrategy` + attribute if provided. + properties: + rollingUpdate: + description: RollingUpdate is used to communicate parameters + when Type is RollingUpdateStatefulSetStrategyType. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be unavailable during the update. Value can be + an absolute number (ex: 5) or a percentage of + desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding up. This can not be + 0. Defaults to 1. This field is alpha-level and + is only honored by servers that enable the MaxUnavailableStatefulSet + feature. The field applies to all pods in the + range 0 to Replicas-1. That means if there is + any unavailable pod in the range 0 to Replicas-1, + it will be counted towards MaxUnavailable.' + x-kubernetes-int-or-string: true + partition: + description: Partition indicates the ordinal at + which the StatefulSet should be partitioned for + updates. During a rolling update, all pods from + ordinal Replicas-1 to Partition are updated. All + pods from ordinal Partition-1 to 0 remain untouched. + This is helpful in being able to do a canary based + deployment. The default value is 0. + format: int32 + type: integer + type: object + type: + description: Type indicates the type of the StatefulSetUpdateStrategy. + Default is RollingUpdate. + type: string + type: object switchCmdExecutorConfig: description: switchCmdExecutorConfig configs how to get client SDK and perform switch statements. @@ -8284,6 +8382,23 @@ spec: type: object minItems: 1 type: array + updateStrategy: + default: Serial + description: "updateStrategy, Pods update strategy. In case + of workloadType=Consensus the update strategy will be + following: \n serial: update Pods one by one that guarantee + minimum component unavailable time. Learner -> Follower(with + AccessMode=none) -> Follower(with AccessMode=readonly) + -> Follower(with AccessMode=readWrite) -> Leader bestEffortParallel: + update Pods in parallel that guarantee minimum component + un-writable time. Learner, Follower(minority) in parallel + -> Follower(majority) -> Leader, keep majority online + all the time. parallel: force parallel" + enum: + - Serial + - BestEffortParallel + - Parallel + type: string required: - switchCmdExecutorConfig type: object @@ -8401,6 +8516,141 @@ spec: - protocol x-kubernetes-list-type: map type: object + statefulSpec: + description: statefulSpec defines stateful related spec if workloadType + is Stateful. + properties: + llPodManagementPolicy: + description: llPodManagementPolicy is the low-level controls + how pods are created during initial scale up, when replacing + pods on nodes, or when scaling down. `OrderedReady` policy + specify where pods are created in increasing order (pod-0, + then pod-1, etc) and the controller will wait until each + pod is ready before continuing. When scaling down, the + pods are removed in the opposite order. `Parallel` policy + specify create pods in parallel to match the desired scale + without waiting, and on scale down will delete all pods + at once. + type: string + llUpdateStrategy: + description: llUpdateStrategy indicates the low-level StatefulSetUpdateStrategy + that will be employed to update Pods in the StatefulSet + when a revision is made to Template. Will ignore `updateStrategy` + attribute if provided. + properties: + rollingUpdate: + description: RollingUpdate is used to communicate parameters + when Type is RollingUpdateStatefulSetStrategyType. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be unavailable during the update. Value can be + an absolute number (ex: 5) or a percentage of + desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding up. This can not be + 0. Defaults to 1. This field is alpha-level and + is only honored by servers that enable the MaxUnavailableStatefulSet + feature. The field applies to all pods in the + range 0 to Replicas-1. That means if there is + any unavailable pod in the range 0 to Replicas-1, + it will be counted towards MaxUnavailable.' + x-kubernetes-int-or-string: true + partition: + description: Partition indicates the ordinal at + which the StatefulSet should be partitioned for + updates. During a rolling update, all pods from + ordinal Replicas-1 to Partition are updated. All + pods from ordinal Partition-1 to 0 remain untouched. + This is helpful in being able to do a canary based + deployment. The default value is 0. + format: int32 + type: integer + type: object + type: + description: Type indicates the type of the StatefulSetUpdateStrategy. + Default is RollingUpdate. + type: string + type: object + updateStrategy: + default: Serial + description: "updateStrategy, Pods update strategy. In case + of workloadType=Consensus the update strategy will be + following: \n serial: update Pods one by one that guarantee + minimum component unavailable time. Learner -> Follower(with + AccessMode=none) -> Follower(with AccessMode=readonly) + -> Follower(with AccessMode=readWrite) -> Leader bestEffortParallel: + update Pods in parallel that guarantee minimum component + un-writable time. Learner, Follower(minority) in parallel + -> Follower(majority) -> Leader, keep majority online + all the time. parallel: force parallel" + enum: + - Serial + - BestEffortParallel + - Parallel + type: string + type: object + statelessSpec: + description: statelessSpec defines stateless related spec if + workloadType is Stateless. + properties: + updateStrategy: + description: updateStrategy defines the underlying deployment + strategy to use to replace existing pods with new ones. + properties: + rollingUpdate: + description: 'Rolling update config params. Present + only if DeploymentStrategyType = RollingUpdate. --- + TODO: Update this to follow our convention for oneOf, + whatever we decide it to be.' + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be scheduled above the desired number of pods. + Value can be an absolute number (ex: 5) or a percentage + of desired pods (ex: 10%). This can not be 0 if + MaxUnavailable is 0. Absolute number is calculated + from percentage by rounding up. Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet + can be scaled up immediately when the rolling + update starts, such that the total number of old + and new pods do not exceed 130% of desired pods. + Once old pods have been killed, new ReplicaSet + can be scaled up further, ensuring that total + number of pods running at any time during the + update is at most 130% of desired pods.' + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be unavailable during the update. Value can be + an absolute number (ex: 5) or a percentage of + desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding down. This can not + be 0 if MaxSurge is 0. Defaults to 25%. Example: + when this is set to 30%, the old ReplicaSet can + be scaled down to 70% of desired pods immediately + when the rolling update starts. Once new pods + are ready, old ReplicaSet can be scaled down further, + followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all + times during the update is at least 70% of desired + pods.' + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + type: object systemAccounts: description: Statement to create system account. properties: @@ -8718,6 +8968,11 @@ spec: - name - workloadType type: object + x-kubernetes-validations: + - message: componentDefs.consensusSpec is required when componentDefs.workloadType + is Consensus, and forbidden otherwise + rule: 'has(self.workloadType) && self.workloadType == ''Consensus'' + ? has(self.consensusSpec) : !has(self.consensusSpec)' minItems: 1 type: array x-kubernetes-list-map-keys: diff --git a/config/crd/bases/apps.kubeblocks.io_clusters.yaml b/config/crd/bases/apps.kubeblocks.io_clusters.yaml index fba2650c2..78973ba06 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusters.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusters.yaml @@ -235,6 +235,12 @@ spec: maxLength: 15 pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string + noCreatePDB: + default: false + description: noCreatePDB defines PodDistruptionBudget creation + behavior, set to true if creation of PodDistruptionBudget + for this component is not needed. Defaults to false. + type: boolean primaryIndex: description: primaryIndex determines which index is primary when workloadType is Replication, index number starts from diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index b88471a27..a9d64631b 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -26,12 +26,9 @@ import ( "strings" "time" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - - snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "github.com/spf13/viper" "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" @@ -42,8 +39,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/scheme" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" @@ -169,7 +168,7 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely := func(clusterKey client.ObjectKey, compNames ...string) { Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) for _, compName := range compNames { - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, compName)).Should(Equal(appsv1alpha1.CreatingClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, compName)).Should(Equal(appsv1alpha1.CreatingClusterCompPhase)) } } @@ -357,40 +356,68 @@ var _ = Describe("Cluster Controller", func() { })()).ShouldNot(HaveOccurred()) } - changeStatefulSetReplicas := func(clusterName types.NamespacedName, replicas int32) { + changeComponentReplicas := func(clusterName types.NamespacedName, replicas int32) { Expect(testapps.GetAndChangeObj(&testCtx, clusterName, func(cluster *appsv1alpha1.Cluster) { - if len(cluster.Spec.ComponentSpecs) == 0 { - cluster.Spec.ComponentSpecs = []appsv1alpha1.ClusterComponentSpec{ - { - Name: replicationCompName, - ComponentDefRef: replicationCompDefName, - Replicas: replicas, - }} - } else { - cluster.Spec.ComponentSpecs[0].Replicas = replicas - } + Expect(cluster.Spec.ComponentSpecs).Should(HaveLen(1)) + cluster.Spec.ComponentSpecs[0].Replicas = replicas })()).ShouldNot(HaveOccurred()) } + getPodSpec := func(sts *appsv1.StatefulSet, deploy *appsv1.Deployment) *corev1.PodSpec { + if sts != nil { + return &sts.Spec.Template.Spec + } else if deploy != nil { + return &deploy.Spec.Template.Spec + } + panic("unreachable") + } + + checkSingleWorkload := func(compDefName string, expects func(g Gomega, sts *appsv1.StatefulSet, deploy *appsv1.Deployment)) { + isStsWorkload := true + switch compDefName { + case statelessCompDefName: + isStsWorkload = false + case statefulCompDefName, replicationCompDefName, consensusCompDefName: + break + default: + panic("unreachable") + } + + if isStsWorkload { + Eventually(func(g Gomega) { + l := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) + expects(g, &l.Items[0], nil) + }).Should(Succeed()) + } else { + Eventually(func(g Gomega) { + l := testk8s.ListAndCheckDeployment(&testCtx, clusterKey) + expects(g, nil, &l.Items[0]) + }).Should(Succeed()) + } + } + testChangeReplicas := func(compName, compDefName string) { + Expect(compDefName).Should(BeElementOf(statelessCompDefName, statefulCompDefName, replicationCompDefName, consensusCompDefName)) createClusterObj(compName, compDefName) - replicasSeq := []int32{5, 3, 1, 0, 2, 4} expectedOG := int64(1) for _, replicas := range replicasSeq { By(fmt.Sprintf("Change replicas to %d", replicas)) - changeStatefulSetReplicas(clusterKey, replicas) + changeComponentReplicas(clusterKey, replicas) expectedOG++ - By("Checking cluster status and the number of replicas changed") Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, fetched *appsv1alpha1.Cluster) { g.Expect(fetched.Status.ObservedGeneration).To(BeEquivalentTo(expectedOG)) g.Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.CreatingClusterPhase)) })).Should(Succeed()) - Eventually(func(g Gomega) { - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - g.Expect(int(*stsList.Items[0].Spec.Replicas)).To(BeEquivalentTo(replicas)) - }).Should(Succeed()) + + checkSingleWorkload(compDefName, func(g Gomega, sts *appsv1.StatefulSet, deploy *appsv1.Deployment) { + if sts != nil { + g.Expect(int(*sts.Spec.Replicas)).To(BeEquivalentTo(replicas)) + } else { + g.Expect(int(*deploy.Spec.Replicas)).To(BeEquivalentTo(replicas)) + } + }) } } @@ -688,13 +715,13 @@ var _ = Describe("Cluster Controller", func() { case replicationCompDefName: testapps.MockReplicationComponentPods(nil, testCtx, sts, clusterObj.Name, compDefName, nil) case statefulCompDefName, consensusCompDefName: - testapps.MockConsensusComponentPods(testCtx, sts, clusterObj.Name, compName) + testapps.MockConsensusComponentPods(&testCtx, sts, clusterObj.Name, compName) } Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { testk8s.MockStatefulSetReady(sts) })).ShouldNot(HaveOccurred()) Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) By("Updating the PVC storage size") @@ -706,7 +733,7 @@ var _ = Describe("Cluster Controller", func() { By("Checking the resize operation finished") Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) By("Checking PVCs are resized") @@ -729,6 +756,8 @@ var _ = Describe("Cluster Controller", func() { const labelValue = "testLabelValue" By("Creating a cluster with Affinity") + Expect(compDefName).Should(BeElementOf(statelessCompDefName, statefulCompDefName, replicationCompDefName, consensusCompDefName)) + affinity := &appsv1alpha1.Affinity{ PodAntiAffinity: appsv1alpha1.Required, TopologyKeys: []string{topologyKey}, @@ -749,20 +778,19 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the Affinity and TopologySpreadConstraints") - Eventually(func(g Gomega) { - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - podSpec := stsList.Items[0].Spec.Template.Spec + checkSingleWorkload(compDefName, func(g Gomega, sts *appsv1.StatefulSet, deploy *appsv1.Deployment) { + podSpec := getPodSpec(sts, deploy) g.Expect(podSpec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key).To(Equal(labelKey)) g.Expect(podSpec.TopologySpreadConstraints[0].WhenUnsatisfiable).To(Equal(corev1.DoNotSchedule)) g.Expect(podSpec.TopologySpreadConstraints[0].TopologyKey).To(Equal(topologyKey)) g.Expect(podSpec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution).Should(HaveLen(1)) g.Expect(podSpec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey).To(Equal(topologyKey)) - }).Should(Succeed()) - + }) } testClusterServiceAccount := func(compName, compDefName string) { By("Creating a cluster with target service account name") + Expect(compDefName).Should(BeElementOf(statelessCompDefName, statefulCompDefName, replicationCompDefName, consensusCompDefName)) clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name). @@ -775,11 +803,10 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the podSpec.serviceAccountName") - Eventually(func(g Gomega) { - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - podSpec := stsList.Items[0].Spec.Template.Spec + checkSingleWorkload(compDefName, func(g Gomega, sts *appsv1.StatefulSet, deploy *appsv1.Deployment) { + podSpec := getPodSpec(sts, deploy) g.Expect(podSpec.ServiceAccountName).To(Equal("test-service-account")) - }).Should(Succeed()) + }) } testComponentAffinity := func(compName, compDefName string) { @@ -787,6 +814,7 @@ var _ = Describe("Cluster Controller", func() { const compTopologyKey = "testComponentTopologyKey" By("Creating a cluster with Affinity") + Expect(compDefName).Should(BeElementOf(statelessCompDefName, statefulCompDefName, replicationCompDefName, consensusCompDefName)) affinity := &appsv1alpha1.Affinity{ PodAntiAffinity: appsv1alpha1.Required, TopologyKeys: []string{clusterTopologyKey}, @@ -807,31 +835,30 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the Affinity and the TopologySpreadConstraints") - Eventually(func(g Gomega) { - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - podSpec := stsList.Items[0].Spec.Template.Spec + checkSingleWorkload(compDefName, func(g Gomega, sts *appsv1.StatefulSet, deploy *appsv1.Deployment) { + podSpec := getPodSpec(sts, deploy) g.Expect(podSpec.TopologySpreadConstraints[0].WhenUnsatisfiable).To(Equal(corev1.ScheduleAnyway)) g.Expect(podSpec.TopologySpreadConstraints[0].TopologyKey).To(Equal(compTopologyKey)) g.Expect(podSpec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight).ShouldNot(BeNil()) g.Expect(podSpec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution).Should(HaveLen(1)) g.Expect(podSpec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey).To(Equal(corev1.LabelHostname)) - }).Should(Succeed()) + }) } testClusterToleration := func(compName, compDefName string) { const tolerationKey = "testClusterTolerationKey" const tolerationValue = "testClusterTolerationValue" By("Creating a cluster with Toleration") - toleration := corev1.Toleration{ - Key: tolerationKey, - Value: tolerationValue, - Operator: corev1.TolerationOpEqual, - Effect: corev1.TaintEffectNoSchedule, - } + Expect(compDefName).Should(BeElementOf(statelessCompDefName, statefulCompDefName, replicationCompDefName, consensusCompDefName)) clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName).SetReplicas(1). - AddClusterToleration(toleration). + AddClusterToleration(corev1.Toleration{ + Key: tolerationKey, + Value: tolerationValue, + Operator: corev1.TolerationOpEqual, + Effect: corev1.TaintEffectNoSchedule, + }). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) @@ -839,29 +866,24 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the tolerations") - Eventually(func(g Gomega) { - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - podSpec := stsList.Items[0].Spec.Template.Spec + checkSingleWorkload(compDefName, func(g Gomega, sts *appsv1.StatefulSet, deploy *appsv1.Deployment) { + podSpec := getPodSpec(sts, deploy) g.Expect(podSpec.Tolerations).Should(HaveLen(2)) - toleration = podSpec.Tolerations[0] - g.Expect(toleration.Key).Should(BeEquivalentTo(tolerationKey)) - g.Expect(toleration.Value).Should(BeEquivalentTo(tolerationValue)) - g.Expect(toleration.Operator).Should(BeEquivalentTo(corev1.TolerationOpEqual)) - g.Expect(toleration.Effect).Should(BeEquivalentTo(corev1.TaintEffectNoSchedule)) - }).Should(Succeed()) + t := podSpec.Tolerations[0] + g.Expect(t.Key).Should(BeEquivalentTo(tolerationKey)) + g.Expect(t.Value).Should(BeEquivalentTo(tolerationValue)) + g.Expect(t.Operator).Should(BeEquivalentTo(corev1.TolerationOpEqual)) + g.Expect(t.Effect).Should(BeEquivalentTo(corev1.TaintEffectNoSchedule)) + }) } - testComponentToleration := func(compName, compDefName string) { + testStsWorkloadComponentToleration := func(compName, compDefName string) { clusterTolerationKey := "testClusterTolerationKey" compTolerationKey := "testcompTolerationKey" compTolerationValue := "testcompTolerationValue" By("Creating a cluster with Toleration") - toleration := corev1.Toleration{ - Key: clusterTolerationKey, - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoExecute, - } + Expect(compDefName).Should(BeElementOf(statelessCompDefName, statefulCompDefName, replicationCompDefName, consensusCompDefName)) compToleration := corev1.Toleration{ Key: compTolerationKey, Value: compTolerationValue, @@ -869,7 +891,12 @@ var _ = Describe("Cluster Controller", func() { Effect: corev1.TaintEffectNoSchedule, } clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, - clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().AddClusterToleration(toleration). + clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). + AddClusterToleration(corev1.Toleration{ + Key: clusterTolerationKey, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoExecute, + }). AddComponent(compName, compDefName).AddComponentToleration(compToleration). Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) @@ -878,16 +905,15 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely(clusterKey, compName) By("Checking the tolerations") - Eventually(func(g Gomega) { - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) - podSpec := stsList.Items[0].Spec.Template.Spec + checkSingleWorkload(compDefName, func(g Gomega, sts *appsv1.StatefulSet, deploy *appsv1.Deployment) { + podSpec := getPodSpec(sts, deploy) Expect(podSpec.Tolerations).Should(HaveLen(2)) - toleration = podSpec.Tolerations[0] - g.Expect(toleration.Key).Should(BeEquivalentTo(compTolerationKey)) - g.Expect(toleration.Value).Should(BeEquivalentTo(compTolerationValue)) - g.Expect(toleration.Operator).Should(BeEquivalentTo(corev1.TolerationOpEqual)) - g.Expect(toleration.Effect).Should(BeEquivalentTo(corev1.TaintEffectNoSchedule)) - }).Should(Succeed()) + t := podSpec.Tolerations[0] + g.Expect(t.Key).Should(BeEquivalentTo(compTolerationKey)) + g.Expect(t.Value).Should(BeEquivalentTo(compTolerationValue)) + g.Expect(t.Operator).Should(BeEquivalentTo(corev1.TolerationOpEqual)) + g.Expect(t.Effect).Should(BeEquivalentTo(corev1.TaintEffectNoSchedule)) + }) } mockRoleChangedEvent := func(key types.NamespacedName, sts *appsv1.StatefulSet) []corev1.Event { @@ -1029,7 +1055,7 @@ var _ = Describe("Cluster Controller", func() { }).Should(Succeed()) By("Waiting the component be running") - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, compName)). + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, compName)). Should(Equal(appsv1alpha1.RunningClusterCompPhase)) } @@ -1247,7 +1273,7 @@ var _ = Describe("Cluster Controller", func() { Eventually(testapps.List(&testCtx, generics.PodDisruptionBudgetSignature, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, - }, client.InNamespace(clusterKey.Namespace))).Should(HaveLen(0)) + }, client.InNamespace(clusterKey.Namespace))).ShouldNot(BeEmpty()) podSpec := stsList.Items[0].Spec.Template.Spec By("Checking created sts pods template with built-in toleration") @@ -1346,7 +1372,7 @@ var _ = Describe("Cluster Controller", func() { }) }) - When("creating cluster with workloadType=[Stateless|Stateful|Consensus|Replication] component", func() { + When("creating cluster with all workloadTypes (being Stateless|Stateful|Consensus|Replication) component", func() { compNameNDef := map[string]string{ statelessCompName: statelessCompDefName, statefulCompName: statefulCompDefName, @@ -1356,7 +1382,6 @@ var _ = Describe("Cluster Controller", func() { BeforeEach(func() { createAllWorkloadTypesClusterDef() - createBackupPolicyTpl(clusterDefObj) }) for compName, compDefName := range compNameNDef { @@ -1368,14 +1393,14 @@ var _ = Describe("Cluster Controller", func() { testDoNotTermintate(compName, compDefName) }) - It(fmt.Sprintf("[comp: %s] should create/delete pods to match the desired replica number if updating cluster's replica number to a valid value", compName), func() { - testChangeReplicas(compName, compDefName) - }) - It(fmt.Sprintf("[comp: %s] should add and delete service correctly", compName), func() { testServiceAddAndDelete(compName, compDefName) }) + It(fmt.Sprintf("[comp: %s] should create/delete pods to match the desired replica number if updating cluster's replica number to a valid value", compName), func() { + testChangeReplicas(compName, compDefName) + }) + It(fmt.Sprintf("[comp: %s] should add serviceAccountName correctly", compName), func() { testClusterServiceAccount(compName, compDefName) }) @@ -1400,39 +1425,47 @@ var _ = Describe("Cluster Controller", func() { Context(fmt.Sprintf("[comp: %s] and with both cluster tolerations and component tolerations set", compName), func() { It("Should observe the component tolerations will override the cluster tolerations", func() { - testComponentToleration(compName, compDefName) + testStsWorkloadComponentToleration(compName, compDefName) }) }) + } + }) - // HACK/TODO: only Stateful and Consensus workload types passes following test, need to investigate. - // Would expect that non-stateless workload types should all pass tests. - switch compName { - case statefulCompName, consensusCompName, replicationCompName: - Context(fmt.Sprintf("[comp: %s] with pvc", compName), func() { - It("should trigger a backup process(snapshot) and create pvcs from backup for newly created replicas when horizontal scale the cluster from 1 to 3", func() { - testHorizontalScale(compName, compDefName) - }) - }) + When("creating cluster with stateful workloadTypes (being Stateful|Consensus|Replication) component", func() { + compNameNDef := map[string]string{ + statefulCompName: statefulCompDefName, + consensusCompName: consensusCompDefName, + replicationCompName: replicationCompDefName, + } - Context(fmt.Sprintf("[comp: %s] with pvc and dynamic-provisioning storage class", compName), func() { - It("should update PVC request storage size accordingly", func() { - testStorageExpansion(compName, compDefName) - }) - }) + BeforeEach(func() { + createAllWorkloadTypesClusterDef() + createBackupPolicyTpl(clusterDefObj) + }) - It(fmt.Sprintf("[comp: %s] should report error if backup error during horizontal scale", compName), func() { - testBackupError(compName, compDefName) + for compName, compDefName := range compNameNDef { + Context(fmt.Sprintf("[comp: %s] with pvc", compName), func() { + It("should trigger a backup process(snapshot) and create pvcs from backup for newly created replicas when horizontal scale the cluster from 1 to 3", func() { + testHorizontalScale(compName, compDefName) }) + }) - Context(fmt.Sprintf("[comp: %s] with horizontal scale after storage expansion", compName), func() { - It("should succeed with horizontal scale to 5 replicas", func() { - testStorageExpansion(compName, compDefName) - horizontalScale(5, compDefName) - }) + Context(fmt.Sprintf("[comp: %s] with pvc and dynamic-provisioning storage class", compName), func() { + It("should update PVC request storage size accordingly", func() { + testStorageExpansion(compName, compDefName) }) - default: - } + }) + + It(fmt.Sprintf("[comp: %s] should report error if backup error during horizontal scale", compName), func() { + testBackupError(compName, compDefName) + }) + Context(fmt.Sprintf("[comp: %s] with horizontal scale after storage expansion", compName), func() { + It("should succeed with horizontal scale to 5 replicas", func() { + testStorageExpansion(compName, compDefName) + horizontalScale(5, compDefName) + }) + }) } }) @@ -1499,11 +1532,11 @@ var _ = Describe("Cluster Controller", func() { Expect(sts.Spec.Template.Spec.InitContainers).Should(HaveLen(1)) By("mock pod/sts are available and wait for component enter running phase") - testapps.MockConsensusComponentPods(testCtx, &sts, clusterObj.Name, compName) + testapps.MockConsensusComponentPods(&testCtx, &sts, clusterObj.Name, compName) Expect(testapps.ChangeObjStatus(&testCtx, &sts, func() { testk8s.MockStatefulSetReady(&sts) })).ShouldNot(HaveOccurred()) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, compName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) By("remove init container after all components are Running") Eventually(testapps.GetClusterObservedGeneration(&testCtx, client.ObjectKeyFromObject(clusterObj))).Should(BeEquivalentTo(1)) @@ -1516,7 +1549,7 @@ var _ = Describe("Cluster Controller", func() { Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(&sts), func(g Gomega, tmpSts *appsv1.StatefulSet) { g.Expect(tmpSts.Spec.Template.Spec.InitContainers).Should(BeEmpty()) })).Should(Succeed()) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, compName)).Should(Equal(appsv1alpha1.CreatingClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, compName)).Should(Equal(appsv1alpha1.CreatingClusterCompPhase)) By("clean up annotations after cluster running") Expect(testapps.GetAndChangeObjStatus(&testCtx, clusterKey, func(tmpCluster *appsv1alpha1.Cluster) { @@ -1559,7 +1592,7 @@ var _ = Describe("Cluster Controller", func() { waitForCreatingResourceCompletely(clusterKey, compDefName) By("Checking statefulSet number") - stsList := testk8s.ListAndCheckStatefulSetCount(&testCtx, clusterKey, 1) + stsList := testk8s.ListAndCheckStatefulSetItemsCount(&testCtx, clusterKey, 1) sts := &stsList.Items[0] Expect(testapps.ChangeObjStatus(&testCtx, sts, func() { @@ -1577,7 +1610,7 @@ var _ = Describe("Cluster Controller", func() { Context("test cluster Failed/Abnormal phase", func() { It("test cluster conditions", func() { By("init cluster") - cluster := testapps.CreateConsensusMysqlCluster(testCtx, clusterDefNameRand, + cluster := testapps.CreateConsensusMysqlCluster(&testCtx, clusterDefNameRand, clusterVersionNameRand, clusterNameRand, consensusCompDefName, consensusCompName) clusterKey := client.ObjectKeyFromObject(cluster) @@ -1613,8 +1646,8 @@ var _ = Describe("Cluster Controller", func() { // })).Should(Succeed()) By("test when clusterVersion not Available") - _ = testapps.CreateConsensusMysqlClusterDef(testCtx, clusterDefNameRand, consensusCompDefName) - clusterVersion := testapps.CreateConsensusMysqlClusterVersion(testCtx, clusterDefNameRand, clusterVersionNameRand, consensusCompDefName) + _ = testapps.CreateConsensusMysqlClusterDef(&testCtx, clusterDefNameRand, consensusCompDefName) + clusterVersion := testapps.CreateConsensusMysqlClusterVersion(&testCtx, clusterDefNameRand, clusterVersionNameRand, consensusCompDefName) clusterVersionKey := client.ObjectKeyFromObject(clusterVersion) // mock clusterVersion unavailable Expect(testapps.GetAndChangeObj(&testCtx, clusterVersionKey, func(clusterVersion *appsv1alpha1.ClusterVersion) { diff --git a/controllers/apps/cluster_status_utils.go b/controllers/apps/cluster_status_utils.go index 8d365fc90..cc04db9e2 100644 --- a/controllers/apps/cluster_status_utils.go +++ b/controllers/apps/cluster_status_utils.go @@ -180,17 +180,6 @@ func handleClusterPhaseWhenCompsNotReady(cluster *appsv1alpha1.Cluster, } } -// getClusterAvailabilityEffect whether the component will affect the cluster availability. -// if the component can affect and be Failed, the cluster will be Failed too. -func getClusterAvailabilityEffect(componentDef *appsv1alpha1.ClusterComponentDefinition) bool { - switch componentDef.WorkloadType { - case appsv1alpha1.Replication, appsv1alpha1.Consensus: - return true - default: - return componentDef.MaxUnavailable != nil - } -} - // getComponentRelatedInfo gets componentMap, clusterAvailabilityMap and component definition information func getComponentRelatedInfo(cluster *appsv1alpha1.Cluster, clusterDef *appsv1alpha1.ClusterDefinition, componentName string) (map[string]string, map[string]bool, *appsv1alpha1.ClusterComponentDefinition, error) { @@ -210,7 +199,7 @@ func getComponentRelatedInfo(cluster *appsv1alpha1.Cluster, clusterDef *appsv1al } clusterAvailabilityEffectMap := map[string]bool{} for i, v := range clusterDef.Spec.ComponentDefs { - clusterAvailabilityEffectMap[v.Name] = getClusterAvailabilityEffect(&v) + clusterAvailabilityEffectMap[v.Name] = componentDef.GetMaxUnavailable() != nil if componentDef == nil && v.Name == compDefName { componentDef = &clusterDef.Spec.ComponentDefs[i] } diff --git a/controllers/apps/cluster_status_utils_test.go b/controllers/apps/cluster_status_utils_test.go index 8e227336a..93d0d5575 100644 --- a/controllers/apps/cluster_status_utils_test.go +++ b/controllers/apps/cluster_status_utils_test.go @@ -65,13 +65,13 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { AfterEach(cleanEnv) const statefulMySQLCompDefName = "stateful" - const statefulMySQLCompName = "mysql1" + const statefulMySQLCompName = "stateful" const consensusMySQLCompDefName = "consensus" - const consensusMySQLCompName = "mysql2" + const consensusMySQLCompName = "consensus" const statelessCompDefName = "stateless" - const nginxCompName = "nginx" + const statelessCompName = "nginx" createClusterDef := func() { _ = testapps.NewClusterDefFactory(clusterDefName). @@ -93,7 +93,7 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { return testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). AddComponent(statefulMySQLCompName, statefulMySQLCompDefName).SetReplicas(3). AddComponent(consensusMySQLCompName, consensusMySQLCompDefName).SetReplicas(3). - AddComponent(nginxCompName, statelessCompDefName).SetReplicas(3). + AddComponent(statelessCompName, statelessCompDefName).SetReplicas(3). Create(&testCtx).GetObject() } @@ -250,9 +250,9 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { false) By("watch warning event from Deployment and component workload type is Stateless") - deploy := getDeployment(nginxCompName) + deploy := getDeployment(statelessCompName) setInvolvedObject(event, constant.DeploymentKind, deploy.Name) - handleAndCheckComponentStatus(nginxCompName, event, + handleAndCheckComponentStatus(statelessCompName, event, appsv1alpha1.FailedClusterPhase, appsv1alpha1.FailedClusterCompPhase, false) @@ -268,9 +268,9 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { By("test the cluster phase when stateless component is Failed and other components are Running") // set nginx component phase to Failed Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(cluster), func(tmpCluster *appsv1alpha1.Cluster) { - compStatus := tmpCluster.Status.Components[nginxCompName] + compStatus := tmpCluster.Status.Components[statelessCompName] compStatus.Phase = appsv1alpha1.FailedClusterCompPhase - tmpCluster.Status.SetComponentStatus(nginxCompName, compStatus) + tmpCluster.Status.SetComponentStatus(statelessCompName, compStatus) })()).ShouldNot(HaveOccurred()) // expect cluster phase is Abnormal by cluster controller. diff --git a/controllers/apps/components/consensus/consensus.go b/controllers/apps/components/consensus/consensus.go index 11804922b..6178fe849 100644 --- a/controllers/apps/components/consensus/consensus.go +++ b/controllers/apps/components/consensus/consensus.go @@ -179,85 +179,44 @@ func (r *ConsensusComponent) HandleUpdate(ctx context.Context, obj client.Object if r == nil { return nil } - - stsObj := util.ConvertToStatefulSet(obj) - // get compDefName from stsObj.name - compDefName := r.Cluster.Spec.GetComponentDefRefName(stsObj.Labels[constant.KBAppComponentLabelKey]) - - // get component from ClusterDefinition by compDefName - component, err := util.GetComponentDefByCluster(ctx, r.Cli, *r.Cluster, compDefName) - if err != nil { - return err - } - - if component == nil || component.WorkloadType != appsv1alpha1.Consensus { - return nil - } - pods, err := util.GetPodListByStatefulSet(ctx, r.Cli, stsObj) - if err != nil { - return err - } - - // update cluster.status.component.consensusSetStatus based on all pods currently exist - componentName := stsObj.Labels[constant.KBAppComponentLabelKey] - - // first, get the old status - var oldConsensusSetStatus *appsv1alpha1.ConsensusSetStatus - if v, ok := r.Cluster.Status.Components[componentName]; ok { - oldConsensusSetStatus = v.ConsensusSetStatus - } - // create the initial status - newConsensusSetStatus := &appsv1alpha1.ConsensusSetStatus{ - Leader: appsv1alpha1.ConsensusMemberStatus{ - Name: "", - Pod: util.ComponentStatusDefaultPodName, - AccessMode: appsv1alpha1.None, - }, - } - // then, calculate the new status - setConsensusSetStatusRoles(newConsensusSetStatus, component, pods) - // if status changed, do update - if !cmp.Equal(newConsensusSetStatus, oldConsensusSetStatus) { - patch := client.MergeFrom((*r.Cluster).DeepCopy()) - if err = util.InitClusterComponentStatusIfNeed(r.Cluster, componentName, *component); err != nil { - return err - } - componentStatus := r.Cluster.Status.Components[componentName] - componentStatus.ConsensusSetStatus = newConsensusSetStatus - r.Cluster.Status.SetComponentStatus(componentName, componentStatus) - if err = r.Cli.Status().Patch(ctx, r.Cluster, patch); err != nil { - return err - } - // add consensus role info to pod env - if err := updateConsensusRoleInfo(ctx, r.Cli, r.Cluster, component, componentName, pods); err != nil { - return err - } - } - - // prepare to do pods Deletion, that's the only thing we should do, - // the statefulset reconciler will do the others. - // to simplify the process, we do pods Deletion after statefulset reconcile done, - // that is stsObj.Generation == stsObj.Status.ObservedGeneration - if stsObj.Generation != stsObj.Status.ObservedGeneration { - return nil - } - - // then we wait all pods' presence, that is len(pods) == stsObj.Spec.Replicas - // only then, we have enough info about the previous pods before delete the current one - if len(pods) != int(*stsObj.Spec.Replicas) { - return nil - } - - // we don't check whether pod role label present: prefer stateful set's Update done than role probing ready - - // generate the pods Deletion plan - plan := generateConsensusUpdatePlan(ctx, r.Cli, stsObj, pods, *component) - // execute plan - if _, err := plan.WalkOneStep(); err != nil { - return err - } - return nil + return r.StatefulComponent.HandleUpdateWithProcessors(ctx, obj, + func(componentDef *appsv1alpha1.ClusterComponentDefinition, pods []corev1.Pod, componentName string) error { + // first, get the old status + var oldConsensusSetStatus *appsv1alpha1.ConsensusSetStatus + if v, ok := r.Cluster.Status.Components[componentName]; ok { + oldConsensusSetStatus = v.ConsensusSetStatus + } + // create the initial status + newConsensusSetStatus := &appsv1alpha1.ConsensusSetStatus{ + Leader: appsv1alpha1.ConsensusMemberStatus{ + Name: "", + Pod: util.ComponentStatusDefaultPodName, + AccessMode: appsv1alpha1.None, + }, + } + // then, calculate the new status + setConsensusSetStatusRoles(newConsensusSetStatus, componentDef, pods) + // if status changed, do update + if !cmp.Equal(newConsensusSetStatus, oldConsensusSetStatus) { + patch := client.MergeFrom((*r.Cluster).DeepCopy()) + if err := util.InitClusterComponentStatusIfNeed(r.Cluster, componentName, *componentDef); err != nil { + return err + } + componentStatus := r.Cluster.Status.Components[componentName] + componentStatus.ConsensusSetStatus = newConsensusSetStatus + r.Cluster.Status.SetComponentStatus(componentName, componentStatus) + if err := r.Cli.Status().Patch(ctx, r.Cluster, patch); err != nil { + return err + } + // add consensus role info to pod env + if err := updateConsensusRoleInfo(ctx, r.Cli, r.Cluster, componentDef, componentName, pods); err != nil { + return err + } + } + return nil + }, ComposeRolePriorityMap, generateConsensusSerialPlan, generateConsensusBestEffortParallelPlan, generateConsensusParallelPlan) } + func NewConsensusComponent( cli client.Client, cluster *appsv1alpha1.Cluster, diff --git a/controllers/apps/components/consensus/consensus_test.go b/controllers/apps/components/consensus/consensus_test.go index 43bc89de3..b41c3e407 100644 --- a/controllers/apps/components/consensus/consensus_test.go +++ b/controllers/apps/components/consensus/consensus_test.go @@ -97,10 +97,10 @@ var _ = Describe("Consensus Component", func() { Context("Consensus Component test", func() { It("Consensus Component test", func() { By(" init cluster, statefulSet, pods") - clusterDef, _, cluster := testapps.InitConsensusMysql(testCtx, clusterDefName, + clusterDef, _, cluster := testapps.InitConsensusMysql(&testCtx, clusterDefName, clusterVersionName, clusterName, "consensus", consensusCompName) - sts := testapps.MockConsensusComponentStatefulSet(testCtx, clusterName, consensusCompName) + sts := testapps.MockConsensusComponentStatefulSet(&testCtx, clusterName, consensusCompName) componentName := consensusCompName compDefName := cluster.Spec.GetComponentDefRefName(componentName) componentDef := clusterDef.GetComponentDefByName(compDefName) @@ -130,7 +130,7 @@ var _ = Describe("Consensus Component", func() { Expect(isRunning == false).Should(BeTrue()) podName := sts.Name + "-0" - podList := testapps.MockConsensusComponentPods(testCtx, sts, clusterName, consensusCompName) + podList := testapps.MockConsensusComponentPods(&testCtx, sts, clusterName, consensusCompName) By("expect for pod is available") Expect(consensusComponent.PodIsAvailable(podList[0], defaultMinReadySeconds)).Should(BeTrue()) diff --git a/controllers/apps/components/consensus/consensus_utils.go b/controllers/apps/components/consensus/consensus_utils.go index d66578553..a35de7a5a 100644 --- a/controllers/apps/components/consensus/consensus_utils.go +++ b/controllers/apps/components/consensus/consensus_utils.go @@ -24,10 +24,7 @@ import ( "sort" "strings" - "github.com/pkg/errors" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -63,70 +60,6 @@ const ( // unknownPriority = 0 ) -// SortPods sorts pods by their role priority -func SortPods(pods []corev1.Pod, rolePriorityMap map[string]int) { - // make a Serial pod list, - // e.g.: unknown -> empty -> learner -> follower1 -> follower2 -> leader, with follower1.Name < follower2.Name - sort.SliceStable(pods, func(i, j int) bool { - roleI := pods[i].Labels[constant.RoleLabelKey] - roleJ := pods[j].Labels[constant.RoleLabelKey] - - if rolePriorityMap[roleI] == rolePriorityMap[roleJ] { - _, ordinal1 := intctrlutil.GetParentNameAndOrdinal(&pods[i]) - _, ordinal2 := intctrlutil.GetParentNameAndOrdinal(&pods[j]) - return ordinal1 < ordinal2 - } - - return rolePriorityMap[roleI] < rolePriorityMap[roleJ] - }) -} - -// generateConsensusUpdatePlan generates Update plan based on UpdateStrategy -func generateConsensusUpdatePlan(ctx context.Context, cli client.Client, stsObj *appsv1.StatefulSet, pods []corev1.Pod, - component appsv1alpha1.ClusterComponentDefinition) *util.Plan { - plan := &util.Plan{} - plan.Start = &util.Step{} - plan.WalkFunc = func(obj interface{}) (bool, error) { - pod, ok := obj.(corev1.Pod) - if !ok { - return false, errors.New("wrong type: obj not Pod") - } - - // if DeletionTimestamp is not nil, it is terminating. - if pod.DeletionTimestamp != nil { - return true, nil - } - - // if pod is the latest version, we do nothing - if intctrlutil.GetPodRevision(&pod) == stsObj.Status.UpdateRevision { - // wait until ready - return !intctrlutil.PodIsReadyWithLabel(pod), nil - } - - // delete the pod to trigger associate StatefulSet to re-create it - if err := cli.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { - return false, err - } - - return true, nil - } - - rolePriorityMap := ComposeRolePriorityMap(component) - SortPods(pods, rolePriorityMap) - - // generate plan by UpdateStrategy - switch component.ConsensusSpec.UpdateStrategy { - case appsv1alpha1.SerialStrategy: - generateConsensusSerialPlan(plan, pods) - case appsv1alpha1.ParallelStrategy: - generateConsensusParallelPlan(plan, pods) - case appsv1alpha1.BestEffortParallelStrategy: - generateConsensusBestEffortParallelPlan(plan, pods, rolePriorityMap) - } - - return plan -} - // unknown & empty & learner & 1/2 followers -> 1/2 followers -> leader func generateConsensusBestEffortParallelPlan(plan *util.Plan, pods []corev1.Pod, rolePriorityMap map[string]int) { start := plan.Start @@ -184,7 +117,7 @@ func generateConsensusBestEffortParallelPlan(plan *util.Plan, pods []corev1.Pod, } // unknown & empty & leader & followers & learner -func generateConsensusParallelPlan(plan *util.Plan, pods []corev1.Pod) { +func generateConsensusParallelPlan(plan *util.Plan, pods []corev1.Pod, rolePriorityMap map[string]int) { start := plan.Start for _, pod := range pods { nextStep := &util.Step{} @@ -194,7 +127,7 @@ func generateConsensusParallelPlan(plan *util.Plan, pods []corev1.Pod) { } // unknown -> empty -> learner -> followers(none->readonly->readwrite) -> leader -func generateConsensusSerialPlan(plan *util.Plan, pods []corev1.Pod) { +func generateConsensusSerialPlan(plan *util.Plan, pods []corev1.Pod, rolePriorityMap map[string]int) { start := plan.Start for _, pod := range pods { nextStep := &util.Step{} @@ -205,18 +138,17 @@ func generateConsensusSerialPlan(plan *util.Plan, pods []corev1.Pod) { } // ComposeRolePriorityMap generates a priority map based on roles. -func ComposeRolePriorityMap(component appsv1alpha1.ClusterComponentDefinition) map[string]int { - if component.ConsensusSpec == nil { - component.ConsensusSpec = &appsv1alpha1.ConsensusSetSpec{Leader: appsv1alpha1.DefaultLeader} +func ComposeRolePriorityMap(componentDef *appsv1alpha1.ClusterComponentDefinition) map[string]int { + if componentDef.ConsensusSpec == nil { + componentDef.ConsensusSpec = appsv1alpha1.NewConsensusSetSpec() } - rolePriorityMap := make(map[string]int, 0) rolePriorityMap[""] = emptyPriority - rolePriorityMap[component.ConsensusSpec.Leader.Name] = leaderPriority - if component.ConsensusSpec.Learner != nil { - rolePriorityMap[component.ConsensusSpec.Learner.Name] = learnerPriority + rolePriorityMap[componentDef.ConsensusSpec.Leader.Name] = leaderPriority + if componentDef.ConsensusSpec.Learner != nil { + rolePriorityMap[componentDef.ConsensusSpec.Learner.Name] = learnerPriority } - for _, follower := range component.ConsensusSpec.Followers { + for _, follower := range componentDef.ConsensusSpec.Followers { switch follower.AccessMode { case appsv1alpha1.None: rolePriorityMap[follower.Name] = followerNonePriority @@ -226,7 +158,6 @@ func ComposeRolePriorityMap(component appsv1alpha1.ClusterComponentDefinition) m rolePriorityMap[follower.Name] = followerReadWritePriority } } - return rolePriorityMap } diff --git a/controllers/apps/components/consensus/consensus_utils_test.go b/controllers/apps/components/consensus/consensus_utils_test.go index c9c93175c..81777d969 100644 --- a/controllers/apps/components/consensus/consensus_utils_test.go +++ b/controllers/apps/components/consensus/consensus_utils_test.go @@ -160,7 +160,7 @@ func TestSortPods(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.args.pods = randSort(tt.want) - SortPods(tt.args.pods, tt.args.rolePriorityMap) + util.SortPods(tt.args.pods, tt.args.rolePriorityMap, constant.RoleLabelKey) if !tt.wantErr { assert.Equal(t, tt.args.pods, tt.want) } diff --git a/controllers/apps/components/deployment_controller_test.go b/controllers/apps/components/deployment_controller_test.go index 89861ba31..c1d28d52c 100644 --- a/controllers/apps/components/deployment_controller_test.go +++ b/controllers/apps/components/deployment_controller_test.go @@ -83,24 +83,26 @@ var _ = Describe("Deployment Controller", func() { cluster := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). AddComponent(statelessCompName, statelessCompDefName).SetReplicas(2).Create(&testCtx).GetObject() + clusterKey := client.ObjectKeyFromObject(cluster) + By("patch cluster to Running") Expect(testapps.ChangeObjStatus(&testCtx, cluster, func() { cluster.Status.Phase = appsv1alpha1.RunningClusterPhase })) By("create the deployment of the stateless component") - deploy := testapps.MockStatelessComponentDeploy(testCtx, clusterName, statelessCompName) + deploy := testapps.MockStatelessComponentDeploy(&testCtx, clusterName, statelessCompName) newDeploymentKey := client.ObjectKey{Name: deploy.Name, Namespace: namespace} Eventually(testapps.CheckObj(&testCtx, newDeploymentKey, func(g Gomega, deploy *appsv1.Deployment) { g.Expect(deploy.Generation == 1).Should(BeTrue()) })).Should(Succeed()) By("check stateless component phase is Failed") - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterName, statelessCompName)).Should(Equal(appsv1alpha1.FailedClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, statelessCompName)).Should(Equal(appsv1alpha1.FailedClusterCompPhase)) By("mock error message and PodCondition about some pod's failure") podName := fmt.Sprintf("%s-%s-%s", clusterName, statelessCompName, testCtx.GetRandomStr()) - pod := testapps.MockStatelessPod(testCtx, deploy, clusterName, statelessCompName, podName) + pod := testapps.MockStatelessPod(&testCtx, deploy, clusterName, statelessCompName, podName) // mock pod container is failed errMessage := "Back-off pulling image nginx:latest" Expect(testapps.ChangeObjStatus(&testCtx, pod, func() { @@ -144,7 +146,7 @@ var _ = Describe("Deployment Controller", func() { })).Should(Succeed()) By("waiting for the component to be running") - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterName, statelessCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, statelessCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) }) }) }) diff --git a/controllers/apps/components/replication/replication.go b/controllers/apps/components/replication/replication.go index 64ab198e0..28b937075 100644 --- a/controllers/apps/components/replication/replication.go +++ b/controllers/apps/components/replication/replication.go @@ -163,10 +163,12 @@ func (r *ReplicationComponent) HandleUpdate(ctx context.Context, obj client.Obje return err } } - if err := util.DeleteStsPods(ctx, r.Cli, sts); err != nil { - return err - } } + // // REVIEW/TODO: (Y-Rookie) + // // 1. should employ rolling deletion as default strategy instead of delete them all. + // if err := util.DeleteStsPods(ctx, r.Cli, sts); err != nil { + // return err + // } // sync cluster.spec.componentSpecs.[x].primaryIndex when failover occurs and switchPolicy is Noop. if err := syncPrimaryIndex(ctx, r.Cli, r.Cluster, sts.Labels[constant.KBAppComponentLabelKey]); err != nil { return err diff --git a/controllers/apps/components/replication/replication_switch_test.go b/controllers/apps/components/replication/replication_switch_test.go index f73ca1da3..54752942f 100644 --- a/controllers/apps/components/replication/replication_switch_test.go +++ b/controllers/apps/components/replication/replication_switch_test.go @@ -233,7 +233,7 @@ var _ = Describe("ReplicationSet Switch", func() { BeforeEach(func() { By("Mock a replicationSpec with SwitchPolicy and SwitchCmdExecutorConfig.") - mockReplicationSpec := &appsv1alpha1.ReplicationSpec{ + mockReplicationSpec := &appsv1alpha1.ReplicationSetSpec{ SwitchPolicies: []appsv1alpha1.SwitchPolicy{ { Type: appsv1alpha1.MaximumAvailability, diff --git a/controllers/apps/components/replication/replication_switch_utils.go b/controllers/apps/components/replication/replication_switch_utils.go index 6821a398b..a11051b5f 100644 --- a/controllers/apps/components/replication/replication_switch_utils.go +++ b/controllers/apps/components/replication/replication_switch_utils.go @@ -308,7 +308,7 @@ func (pdm *ProbeDetectManager) lagDetect(pod *corev1.Pod) (*LagDetectResult, err // getSwitchStatementsBySwitchPolicyType gets the SwitchStatements corresponding to switchPolicyType func getSwitchStatementsBySwitchPolicyType(switchPolicyType appsv1alpha1.SwitchPolicyType, - replicationSpec *appsv1alpha1.ReplicationSpec) (*appsv1alpha1.SwitchStatements, error) { + replicationSpec *appsv1alpha1.ReplicationSetSpec) (*appsv1alpha1.SwitchStatements, error) { if replicationSpec == nil || len(replicationSpec.SwitchPolicies) == 0 { return nil, fmt.Errorf("replicationSpec and replicationSpec.SwitchPolicies can not be nil") } diff --git a/controllers/apps/components/replication/replication_switch_utils_test.go b/controllers/apps/components/replication/replication_switch_utils_test.go index bf034a210..b13bf3fa7 100644 --- a/controllers/apps/components/replication/replication_switch_utils_test.go +++ b/controllers/apps/components/replication/replication_switch_utils_test.go @@ -151,7 +151,7 @@ var _ = Describe("ReplicationSet Switch Util", func() { BeforeEach(func() { By("Mock a replicationSpec with SwitchPolicy and SwitchCmdExecutorConfig.") - replicationSpec := &appsv1alpha1.ReplicationSpec{ + replicationSpec := &appsv1alpha1.ReplicationSetSpec{ SwitchPolicies: []appsv1alpha1.SwitchPolicy{ { Type: appsv1alpha1.MaximumAvailability, diff --git a/controllers/apps/components/stateful/stateful.go b/controllers/apps/components/stateful/stateful.go index 0a006f26e..92ec32323 100644 --- a/controllers/apps/components/stateful/stateful.go +++ b/controllers/apps/components/stateful/stateful.go @@ -21,10 +21,12 @@ package stateful import ( "context" + "errors" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "k8s.io/kubectl/pkg/util/podutils" @@ -33,6 +35,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/controllers/apps/components/types" "github.com/apecloud/kubeblocks/controllers/apps/components/util" + "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -89,10 +92,139 @@ func (r *StatefulComponent) GetPhaseWhenPodsNotReady(ctx context.Context, compon stsObj.Status.AvailableReplicas, checkExistFailedPodOfLatestRevision), nil } -func (r *StatefulComponent) HandleUpdate(ctx context.Context, obj client.Object) error { +// HandleUpdateWithProcessors extended HandleUpdate() with custom processors +// REVIEW/TODO: (nashtsai) +// 1. too many args +func (r *StatefulComponent) HandleUpdateWithProcessors(ctx context.Context, obj client.Object, + compStatusProcessor func(compDef *appsv1alpha1.ClusterComponentDefinition, pods []corev1.Pod, componentName string) error, + priorityMapper func(component *appsv1alpha1.ClusterComponentDefinition) map[string]int, + serialStrategyHandler, bestEffortParallelStrategyHandler, parallelStrategyHandler func(plan *util.Plan, pods []corev1.Pod, rolePriorityMap map[string]int)) error { + if r == nil { + return nil + } + + stsObj := util.ConvertToStatefulSet(obj) + // get compDefName from stsObj.name + compDefName := r.Cluster.Spec.GetComponentDefRefName(stsObj.Labels[constant.KBAppComponentLabelKey]) + + // get componentDef from ClusterDefinition by compDefName + componentDef, err := util.GetComponentDefByCluster(ctx, r.Cli, *r.Cluster, compDefName) + if err != nil { + return err + } + + if componentDef == nil || componentDef.IsStatelessWorkload() { + return nil + } + pods, err := util.GetPodListByStatefulSet(ctx, r.Cli, stsObj) + if err != nil { + return err + } + + // update cluster.status.component.consensusSetStatus based on all pods currently exist + if compStatusProcessor != nil { + componentName := stsObj.Labels[constant.KBAppComponentLabelKey] + if err = compStatusProcessor(componentDef, pods, componentName); err != nil { + return err + } + } + + // prepare to do pods Deletion, that's the only thing we should do, + // the statefulset reconciler will do the others. + // to simplify the process, we do pods Deletion after statefulset reconcile done, + // that is stsObj.Generation == stsObj.Status.ObservedGeneration + if stsObj.Generation != stsObj.Status.ObservedGeneration { + return nil + } + + // then we wait all pods' presence, that is len(pods) == stsObj.Spec.Replicas + // only then, we have enough info about the previous pods before delete the current one + if len(pods) != int(*stsObj.Spec.Replicas) { + return nil + } + + // generate the pods Deletion plan + plan := generateUpdatePlan(ctx, r.Cli, stsObj, pods, componentDef, priorityMapper, + serialStrategyHandler, bestEffortParallelStrategyHandler, parallelStrategyHandler) + // execute plan + if _, err := plan.WalkOneStep(); err != nil { + return err + } return nil } +// generateConsensusUpdatePlan generates Update plan based on UpdateStrategy +func generateUpdatePlan(ctx context.Context, cli client.Client, stsObj *appsv1.StatefulSet, pods []corev1.Pod, + componentDef *appsv1alpha1.ClusterComponentDefinition, + priorityMapper func(component *appsv1alpha1.ClusterComponentDefinition) map[string]int, + serialStrategyHandler, bestEffortParallelStrategyHandler, parallelStrategyHandler func(plan *util.Plan, pods []corev1.Pod, rolePriorityMap map[string]int)) *util.Plan { + stsWorkload := componentDef.GetStatefulSetWorkload() + _, s := stsWorkload.FinalStsUpdateStrategy() + switch s.Type { + case appsv1.RollingUpdateStatefulSetStrategyType, "": + return nil + } + + plan := &util.Plan{} + plan.Start = &util.Step{} + plan.WalkFunc = func(obj interface{}) (bool, error) { + pod, ok := obj.(corev1.Pod) + if !ok { + return false, errors.New("wrong type: obj not Pod") + } + + // if DeletionTimestamp is not nil, it is terminating. + if pod.DeletionTimestamp != nil { + return true, nil + } + + // if pod is the latest version, we do nothing + if intctrlutil.GetPodRevision(&pod) == stsObj.Status.UpdateRevision { + // wait until ready + return !intctrlutil.PodIsReadyWithLabel(pod), nil + } + + // delete the pod to trigger associate StatefulSet to re-create it + if err := cli.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { + return false, err + } + + return true, nil + } + + var rolePriorityMap map[string]int + if priorityMapper != nil { + rolePriorityMap = priorityMapper(componentDef) + util.SortPods(pods, rolePriorityMap, constant.RoleLabelKey) + } + + // generate plan by UpdateStrategy + switch stsWorkload.GetUpdateStrategy() { + case appsv1alpha1.ParallelStrategy: + if parallelStrategyHandler != nil { + parallelStrategyHandler(plan, pods, rolePriorityMap) + } + case appsv1alpha1.BestEffortParallelStrategy: + if bestEffortParallelStrategyHandler != nil { + bestEffortParallelStrategyHandler(plan, pods, rolePriorityMap) + } + case appsv1alpha1.SerialStrategy: + fallthrough + default: + if serialStrategyHandler != nil { + serialStrategyHandler(plan, pods, rolePriorityMap) + } + } + return plan +} + +func (r *StatefulComponent) HandleUpdate(ctx context.Context, obj client.Object) error { + if r == nil { + return nil + } + return r.HandleUpdateWithProcessors(ctx, obj, nil, nil, nil, nil, nil) +} + func NewStatefulComponent( cli client.Client, cluster *appsv1alpha1.Cluster, diff --git a/controllers/apps/components/stateful/stateful_test.go b/controllers/apps/components/stateful/stateful_test.go index 4bf237ddb..fa2c521d6 100644 --- a/controllers/apps/components/stateful/stateful_test.go +++ b/controllers/apps/components/stateful/stateful_test.go @@ -74,9 +74,9 @@ var _ = Describe("Stateful Component", func() { Context("Stateful Component test", func() { It("Stateful Component test", func() { By(" init cluster, statefulSet, pods") - clusterDef, _, cluster := testapps.InitConsensusMysql(testCtx, clusterDefName, + clusterDef, _, cluster := testapps.InitConsensusMysql(&testCtx, clusterDefName, clusterVersionName, clusterName, statefulCompDefRef, statefulCompName) - _ = testapps.MockConsensusComponentStatefulSet(testCtx, clusterName, statefulCompName) + _ = testapps.MockConsensusComponentStatefulSet(&testCtx, clusterName, statefulCompName) stsList := &appsv1.StatefulSetList{} Eventually(func() bool { _ = k8sClient.List(ctx, stsList, client.InNamespace(testCtx.DefaultNamespace), client.MatchingLabels{ @@ -109,7 +109,7 @@ var _ = Describe("Stateful Component", func() { Expect(podsReady == false).Should(BeTrue()) By("create pods of sts") - podList := testapps.MockConsensusComponentPods(testCtx, sts, clusterName, statefulCompName) + podList := testapps.MockConsensusComponentPods(&testCtx, sts, clusterName, statefulCompName) By("test stateful component is abnormal") // mock pod is not ready diff --git a/controllers/apps/components/stateful_set_controller_test.go b/controllers/apps/components/stateful_set_controller_test.go index 45a7a850d..3fbce66b3 100644 --- a/controllers/apps/components/stateful_set_controller_test.go +++ b/controllers/apps/components/stateful_set_controller_test.go @@ -100,7 +100,7 @@ var _ = Describe("StatefulSet Controller", func() { })).Should(Succeed()) By("create pods of statefulset") - pods := testapps.MockConsensusComponentPods(testCtx, sts, clusterName, consensusCompName) + pods := testapps.MockConsensusComponentPods(&testCtx, sts, clusterName, consensusCompName) By("Mock a pod without role label and it will wait for HandleProbeTimeoutWhenPodsReady") leaderPod := pods[0] @@ -143,9 +143,10 @@ var _ = Describe("StatefulSet Controller", func() { Context("test controller", func() { It("test statefulSet controller", func() { By("mock cluster object") - _, _, cluster := testapps.InitConsensusMysql(testCtx, clusterDefName, + _, _, cluster := testapps.InitConsensusMysql(&testCtx, clusterDefName, clusterVersionName, clusterName, consensusCompType, consensusCompName) + clusterKey := client.ObjectKeyFromObject(cluster) // REVIEW/TODO: "Rebooting" got refactored By("mock cluster phase is 'Rebooting' and restart operation is running on cluster") Expect(testapps.ChangeObjStatus(&testCtx, cluster, func() { @@ -157,7 +158,7 @@ var _ = Describe("StatefulSet Controller", func() { }, } })).Should(Succeed()) - _ = testapps.CreateRestartOpsRequest(testCtx, clusterName, opsRequestName, []string{consensusCompName}) + _ = testapps.CreateRestartOpsRequest(&testCtx, clusterName, opsRequestName, []string{consensusCompName}) Expect(testapps.ChangeObj(&testCtx, cluster, func(lcluster *appsv1alpha1.Cluster) { lcluster.Annotations = map[string]string{ constant.OpsRequestAnnotationKey: fmt.Sprintf(`[{"name":"%s","clusterPhase":"Updating"}]`, opsRequestName), @@ -165,14 +166,14 @@ var _ = Describe("StatefulSet Controller", func() { })).Should(Succeed()) // trigger statefulset controller Reconcile - sts := testapps.MockConsensusComponentStatefulSet(testCtx, clusterName, consensusCompName) + sts := testapps.MockConsensusComponentStatefulSet(&testCtx, clusterName, consensusCompName) By("mock the StatefulSet and pods are ready") // mock statefulSet available and consensusSet component is running pods := testUsingEnvTest(sts) By("check the component phase becomes Running") - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterName, consensusCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, consensusCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) By("mock component of cluster is stopping") Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(cluster), func(tmpCluster *appsv1alpha1.Cluster) { @@ -199,7 +200,7 @@ var _ = Describe("StatefulSet Controller", func() { } By("check the component phase becomes Stopped") - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterName, consensusCompName)).Should(Equal(appsv1alpha1.StoppedClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, consensusCompName)).Should(Equal(appsv1alpha1.StoppedClusterCompPhase)) By("test updateStrategy with Serial") testUpdateStrategy(appsv1alpha1.SerialStrategy, consensusCompName, 1) diff --git a/controllers/apps/components/stateless/stateless_test.go b/controllers/apps/components/stateless/stateless_test.go index 2ad9b1519..a25bf1af1 100644 --- a/controllers/apps/components/stateless/stateless_test.go +++ b/controllers/apps/components/stateless/stateless_test.go @@ -78,7 +78,7 @@ var _ = Describe("Stateful Component", func() { Create(&testCtx).GetObject() cluster := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). AddComponent(statelessCompName, statelessCompDefName).SetReplicas(2).Create(&testCtx).GetObject() - deploy := testapps.MockStatelessComponentDeploy(testCtx, clusterName, statelessCompName) + deploy := testapps.MockStatelessComponentDeploy(&testCtx, clusterName, statelessCompName) clusterComponent := cluster.Spec.GetComponentByName(statelessCompName) componentDef := clusterDef.GetComponentDefByName(clusterComponent.ComponentDefRef) statelessComponent, err := NewStatelessComponent(k8sClient, cluster, clusterComponent, *componentDef) @@ -89,7 +89,7 @@ var _ = Describe("Stateful Component", func() { By("test pod is ready") rsName := deploy.Name + "-5847cb795c" - pod := testapps.MockStatelessPod(testCtx, deploy, clusterName, statelessCompName, rsName+randomStr) + pod := testapps.MockStatelessPod(&testCtx, deploy, clusterName, statelessCompName, rsName+randomStr) lastTransTime := metav1.NewTime(time.Now().Add(-1 * (defaultMinReadySeconds + 1) * time.Second)) testk8s.MockPodAvailable(pod, lastTransTime) Expect(statelessComponent.PodIsAvailable(pod, defaultMinReadySeconds)).Should(BeTrue()) diff --git a/controllers/apps/components/util/component_utils.go b/controllers/apps/components/util/component_utils.go index dbc94bf36..f8aa9b856 100644 --- a/controllers/apps/components/util/component_utils.go +++ b/controllers/apps/components/util/component_utils.go @@ -23,6 +23,7 @@ import ( "context" "errors" "fmt" + "sort" "strings" "time" @@ -38,6 +39,7 @@ import ( "github.com/apecloud/kubeblocks/internal/constant" client2 "github.com/apecloud/kubeblocks/internal/controller/client" componentutil "github.com/apecloud/kubeblocks/internal/controller/component" + intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" "github.com/apecloud/kubeblocks/internal/generics" ) @@ -492,6 +494,22 @@ func GetCustomLabelWorkloadKind() []string { } } +// SortPods sorts pods by their role priority +func SortPods(pods []corev1.Pod, priorityMap map[string]int, idLabelKey string) { + // make a Serial pod list, + // e.g.: unknown -> empty -> learner -> follower1 -> follower2 -> leader, with follower1.Name < follower2.Name + sort.SliceStable(pods, func(i, j int) bool { + roleI := pods[i].Labels[idLabelKey] + roleJ := pods[j].Labels[idLabelKey] + if priorityMap[roleI] == priorityMap[roleJ] { + _, ordinal1 := intctrlutil.GetParentNameAndOrdinal(&pods[i]) + _, ordinal2 := intctrlutil.GetParentNameAndOrdinal(&pods[j]) + return ordinal1 < ordinal2 + } + return priorityMap[roleI] < priorityMap[roleJ] + }) +} + // getObjectListMapOfResourceKind returns the mapping of resource kind and its object list. func getObjectListMapOfResourceKind() map[string]client.ObjectList { return map[string]client.ObjectList{ diff --git a/controllers/apps/components/util/component_utils_test.go b/controllers/apps/components/util/component_utils_test.go index 28dcd4d7a..76ad363b4 100644 --- a/controllers/apps/components/util/component_utils_test.go +++ b/controllers/apps/components/util/component_utils_test.go @@ -162,11 +162,11 @@ var _ = Describe("Consensus Component", func() { Context("Consensus Component test", func() { It("Consensus Component test", func() { By(" init cluster, statefulSet, pods") - _, _, cluster := testapps.InitClusterWithHybridComps(testCtx, clusterDefName, + _, _, cluster := testapps.InitClusterWithHybridComps(&testCtx, clusterDefName, clusterVersionName, clusterName, statelessCompName, "stateful", consensusCompName) - sts := testapps.MockConsensusComponentStatefulSet(testCtx, clusterName, consensusCompName) - testapps.MockStatelessComponentDeploy(testCtx, clusterName, statelessCompName) - _ = testapps.MockConsensusComponentPods(testCtx, sts, clusterName, consensusCompName) + sts := testapps.MockConsensusComponentStatefulSet(&testCtx, clusterName, consensusCompName) + testapps.MockStatelessComponentDeploy(&testCtx, clusterName, statelessCompName) + _ = testapps.MockConsensusComponentPods(&testCtx, sts, clusterName, consensusCompName) By("test GetComponentDefByCluster function") componentDef, _ := GetComponentDefByCluster(ctx, k8sClient, *cluster, consensusCompDefRef) diff --git a/controllers/apps/components/util/plan.go b/controllers/apps/components/util/plan.go index e0c5b8b27..e186adfeb 100644 --- a/controllers/apps/components/util/plan.go +++ b/controllers/apps/components/util/plan.go @@ -31,7 +31,14 @@ type Step struct { type WalkFunc func(obj interface{}) (bool, error) +// WalkOneStep process plan stepping +// @return isCompleted +// @return err func (p *Plan) WalkOneStep() (bool, error) { + if p == nil { + return true, nil + } + if len(p.Start.NextSteps) == 0 { return true, nil } @@ -46,7 +53,6 @@ func (p *Plan) WalkOneStep() (bool, error) { shouldStop = true } } - if shouldStop { return false, nil } @@ -63,7 +69,6 @@ func (p *Plan) WalkOneStep() (bool, error) { } } } - return plan.WalkOneStep() } @@ -73,6 +78,5 @@ func containStep(steps []*Step, step *Step) bool { return true } } - return false } diff --git a/controllers/apps/configuration/policy_util.go b/controllers/apps/configuration/policy_util.go index 8a3e305d4..606813289 100644 --- a/controllers/apps/configuration/policy_util.go +++ b/controllers/apps/configuration/policy_util.go @@ -161,7 +161,7 @@ func getConsensusPods(params reconfigureParams) ([]corev1.Pod, error) { } // sort pods - consensus.SortPods(pods, consensus.ComposeRolePriorityMap(*params.Component)) + util.SortPods(pods, consensus.ComposeRolePriorityMap(params.Component), constant.RoleLabelKey) r := make([]corev1.Pod, 0, len(pods)) for i := len(pods); i > 0; i-- { r = append(r, pods[i-1:i]...) diff --git a/controllers/apps/configuration/reconfigure_policy.go b/controllers/apps/configuration/reconfigure_policy.go index ebf7f51e2..a0e656d28 100644 --- a/controllers/apps/configuration/reconfigure_policy.go +++ b/controllers/apps/configuration/reconfigure_policy.go @@ -154,17 +154,17 @@ func (param *reconfigureParams) maxRollingReplicas() int32 { replicas = param.getTargetReplicas() ) - if param.Component.MaxUnavailable == nil { + if param.Component.GetMaxUnavailable() == nil { return defaultRolling } - v, isPercent, err := intctrlutil.GetIntOrPercentValue(param.Component.MaxUnavailable) + v, isPercentage, err := intctrlutil.GetIntOrPercentValue(param.Component.GetMaxUnavailable()) if err != nil { - param.Ctx.Log.Error(err, "failed to get MaxUnavailable!") + param.Ctx.Log.Error(err, "failed to get maxUnavailable!") return defaultRolling } - if isPercent { + if isPercentage { r = int32(math.Floor(float64(v) * float64(replicas) / 100)) } else { r = int32(util.Min(v, param.getTargetReplicas())) diff --git a/controllers/apps/configuration/rolling_upgrade_policy_test.go b/controllers/apps/configuration/rolling_upgrade_policy_test.go index 65064cf2f..df7195c85 100644 --- a/controllers/apps/configuration/rolling_upgrade_policy_test.go +++ b/controllers/apps/configuration/rolling_upgrade_policy_test.go @@ -20,10 +20,10 @@ along with this program. If not, see . package configuration import ( + "github.com/golang/mock/gomock" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "github.com/golang/mock/gomock" + apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" metautil "k8s.io/apimachinery/pkg/util/intstr" @@ -175,7 +175,13 @@ var _ = Describe("Reconfigure RollingPolicy", func() { var pods []corev1.Pod { mockParam.Component.WorkloadType = appsv1alpha1.Stateful - mockParam.Component.MaxUnavailable = func() *metautil.IntOrString { v := metautil.FromString("100%"); return &v }() + mockParam.Component.StatefulSpec = &appsv1alpha1.StatefulSetSpec{ + LLUpdateStrategy: &apps.StatefulSetUpdateStrategy{ + RollingUpdate: &apps.RollingUpdateStatefulSetStrategy{ + MaxUnavailable: func() *metautil.IntOrString { v := metautil.FromString("100%"); return &v }(), + }, + }, + } pods = newMockPodsWithStatefulSet(&mockParam.ComponentUnits[0], defaultReplica) } diff --git a/controllers/apps/operations/horizontal_scaling_test.go b/controllers/apps/operations/horizontal_scaling_test.go index 8c61d7d53..a2303059c 100644 --- a/controllers/apps/operations/horizontal_scaling_test.go +++ b/controllers/apps/operations/horizontal_scaling_test.go @@ -121,7 +121,7 @@ var _ = Describe("HorizontalScaling OpsRequest", func() { // mock pod created according to horizontalScaling replicas for _, v := range []int{1, 2} { podName := fmt.Sprintf("%s-%s-%d", clusterName, consensusComp, v) - testapps.MockConsensusComponentStsPod(testCtx, nil, clusterName, consensusComp, podName, "follower", "ReadOnly") + testapps.MockConsensusComponentStsPod(&testCtx, nil, clusterName, consensusComp, podName, "follower", "ReadOnly") } opsRes.OpsRequest = createHorizontalScaling(clusterName, 3) diff --git a/controllers/apps/operations/ops_progress_util_test.go b/controllers/apps/operations/ops_progress_util_test.go index 4b8554c9f..f850ac916 100644 --- a/controllers/apps/operations/ops_progress_util_test.go +++ b/controllers/apps/operations/ops_progress_util_test.go @@ -84,7 +84,7 @@ var _ = Describe("Ops ProgressDetails", func() { By("mock one pod of StatefulSet to update successfully") testk8s.RemovePodFinalizer(ctx, testCtx, pod) - testapps.MockConsensusComponentStsPod(testCtx, nil, clusterName, consensusComp, + testapps.MockConsensusComponentStsPod(&testCtx, nil, clusterName, consensusComp, pod.Name, "leader", "ReadWrite") _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) @@ -95,7 +95,7 @@ var _ = Describe("Ops ProgressDetails", func() { testProgressDetailsWithStatelessPodUpdating := func(reqCtx intctrlutil.RequestCtx, opsRes *OpsResource) { By("create a new pod") newPodName := "busybox-" + testCtx.GetRandomStr() - testapps.MockStatelessPod(testCtx, nil, clusterName, statelessComp, newPodName) + testapps.MockStatelessPod(&testCtx, nil, clusterName, statelessComp, newPodName) newPod := &corev1.Pod{} Expect(k8sClient.Get(ctx, client.ObjectKey{Name: newPodName, Namespace: testCtx.DefaultNamespace}, newPod)).Should(Succeed()) _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) @@ -183,7 +183,7 @@ var _ = Describe("Ops ProgressDetails", func() { By("test the progressDetails when scaling up replicas") podName := fmt.Sprintf("%s-%s-%d", clusterName, consensusComp, 0) - testapps.MockConsensusComponentStsPod(testCtx, nil, clusterName, consensusComp, + testapps.MockConsensusComponentStsPod(&testCtx, nil, clusterName, consensusComp, podName, "leader", "ReadWrite") pod = &corev1.Pod{} Expect(k8sClient.Get(ctx, client.ObjectKey{Name: podName, Namespace: testCtx.DefaultNamespace}, pod)).Should(Succeed()) diff --git a/controllers/apps/operations/restart_test.go b/controllers/apps/operations/restart_test.go index 627092ea2..b8cecf39f 100644 --- a/controllers/apps/operations/restart_test.go +++ b/controllers/apps/operations/restart_test.go @@ -83,8 +83,8 @@ var _ = Describe("Restart OpsRequest", func() { Eventually(testapps.GetOpsRequestPhase(&testCtx, client.ObjectKeyFromObject(opsRes.OpsRequest))).Should(Equal(appsv1alpha1.OpsCreatingPhase)) By("test restart action and reconcile function") - testapps.MockConsensusComponentStatefulSet(testCtx, clusterName, consensusComp) - testapps.MockStatelessComponentDeploy(testCtx, clusterName, statelessComp) + testapps.MockConsensusComponentStatefulSet(&testCtx, clusterName, consensusComp) + testapps.MockStatelessComponentDeploy(&testCtx, clusterName, statelessComp) rHandler := restartOpsHandler{} _ = rHandler.Action(reqCtx, k8sClient, opsRes) diff --git a/controllers/apps/operations/suite_test.go b/controllers/apps/operations/suite_test.go index 7637d9bdf..54d9cc1f9 100644 --- a/controllers/apps/operations/suite_test.go +++ b/controllers/apps/operations/suite_test.go @@ -144,7 +144,7 @@ var _ = AfterSuite(func() { func initOperationsResources(clusterDefinitionName, clusterVersionName, clusterName string) (*OpsResource, *appsv1alpha1.ClusterDefinition, *appsv1alpha1.Cluster) { - clusterDef, _, clusterObject := testapps.InitClusterWithHybridComps(testCtx, clusterDefinitionName, + clusterDef, _, clusterObject := testapps.InitClusterWithHybridComps(&testCtx, clusterDefinitionName, clusterVersionName, clusterName, statelessComp, statefulComp, consensusComp) opsRes := &OpsResource{ Cluster: clusterObject, @@ -171,7 +171,7 @@ func initOperationsResources(clusterDefinitionName, func initConsensusPods(ctx context.Context, cli client.Client, opsRes *OpsResource, clusterName string) []corev1.Pod { // mock the pods of consensusSet component - testapps.MockConsensusComponentPods(testCtx, nil, clusterName, consensusComp) + testapps.MockConsensusComponentPods(&testCtx, nil, clusterName, consensusComp) podList, err := util.GetComponentPodList(ctx, cli, *opsRes.Cluster, consensusComp) Expect(err).Should(Succeed()) // the opsRequest will use startTime to check some condition. diff --git a/controllers/apps/operations/util/common_util_test.go b/controllers/apps/operations/util/common_util_test.go index 1c602bff9..a93c80227 100644 --- a/controllers/apps/operations/util/common_util_test.go +++ b/controllers/apps/operations/util/common_util_test.go @@ -58,7 +58,7 @@ var _ = Describe("OpsRequest Controller", func() { Context("Test OpsRequest", func() { It("Should Test all OpsRequest", func() { - cluster := testapps.CreateConsensusMysqlCluster(testCtx, clusterDefinitionName, + cluster := testapps.CreateConsensusMysqlCluster(&testCtx, clusterDefinitionName, clusterVersionName, clusterName, "consensus", consensusCompName) By("init restart OpsRequest") testOpsName := "restart-" + randomStr diff --git a/controllers/apps/operations/volume_expansion_test.go b/controllers/apps/operations/volume_expansion_test.go index 6c9fcefac..ee8913d33 100644 --- a/controllers/apps/operations/volume_expansion_test.go +++ b/controllers/apps/operations/volume_expansion_test.go @@ -274,10 +274,10 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { Context("Test VolumeExpansion", func() { It("VolumeExpansion should work", func() { reqCtx := intctrlutil.RequestCtx{Ctx: ctx} - _, _, clusterObject := testapps.InitConsensusMysql(testCtx, clusterDefinitionName, + _, _, clusterObject := testapps.InitConsensusMysql(&testCtx, clusterDefinitionName, clusterVersionName, clusterName, "consensus", consensusCompName) // init storageClass - sc := testapps.CreateStorageClass(testCtx, storageClassName, true) + sc := testapps.CreateStorageClass(&testCtx, storageClassName, true) Expect(testapps.ChangeObj(&testCtx, sc, func(lsc *storagev1.StorageClass) { lsc.Annotations = map[string]string{storage.IsDefaultStorageClassAnnotation: "true"} })).ShouldNot(HaveOccurred()) diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index a3f61050e..473dd75eb 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -50,7 +50,7 @@ var _ = Describe("OpsRequest Controller", func() { const clusterDefName = "test-clusterdef" const clusterVersionName = "test-clusterversion" const clusterNamePrefix = "test-cluster" - const mysqlCompDefName = "consensus" + const mysqlCompDefName = "mysql" const mysqlCompName = "mysql" const defaultMinReadySeconds = 10 @@ -63,7 +63,7 @@ var _ = Describe("OpsRequest Controller", func() { inNS := client.InNamespace(testCtx.DefaultNamespace) ml := client.HasLabels{testCtx.TestObjLabelKey} - testapps.ClearResources(&testCtx, intctrlutil.OpsRequestSignature, inNS, ml) + testapps.ClearResourcesWithRemoveFinalizerOption(&testCtx, intctrlutil.OpsRequestSignature, true, inNS, ml) testapps.ClearResources(&testCtx, intctrlutil.VolumeSnapshotSignature, inNS) // delete cluster(and all dependent sub-resources), clusterversion and clusterdef @@ -78,7 +78,6 @@ var _ = Describe("OpsRequest Controller", func() { BeforeEach(func() { cleanEnv() - }) AfterEach(func() { @@ -174,7 +173,7 @@ var _ = Describe("OpsRequest Controller", func() { By("mock pod/sts are available and wait for cluster enter running phase") podName := fmt.Sprintf("%s-%s-0", clusterObj.Name, mysqlCompName) - pod := testapps.MockConsensusComponentStsPod(testCtx, nil, clusterObj.Name, mysqlCompName, + pod := testapps.MockConsensusComponentStsPod(&testCtx, nil, clusterObj.Name, mysqlCompName, podName, "leader", "ReadWrite") // the opsRequest will use startTime to check some condition. // if there is no sleep for 1 second, unstable error may occur. @@ -216,7 +215,7 @@ var _ = Describe("OpsRequest Controller", func() { By("wait for VerticalScalingOpsRequest is running") Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, mysqlCompName)).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) checkLatestOpsIsProcessing(clusterKey, verticalScalingOpsRequest.Spec.Type) By("cluster and component phase are Updating") @@ -229,7 +228,7 @@ var _ = Describe("OpsRequest Controller", func() { Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(&mysqlSts), func(tmpSts *appsv1.StatefulSet) { testk8s.MockStatefulSetReady(tmpSts) })()).ShouldNot(HaveOccurred()) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterObj.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) checkLatestOpsHasProcessed(clusterKey) @@ -331,14 +330,6 @@ var _ = Describe("OpsRequest Controller", func() { Create(&testCtx).GetObject() }) - It("issue an VerticalScalingOpsRequest should change Cluster's resource requirements successfully", func() { - ctx := verticalScalingContext{ - source: resourceContext{class: &testapps.Class1c1g}, - target: resourceContext{resource: testapps.Class2c4g.ToResourceRequirements()}, - } - testVerticalScaleCPUAndMemory(testapps.ConsensusMySQLComponent, ctx) - }) - mockCompRunning := func(replicas int32) { stsList := testk8s.ListAndCheckStatefulSetWithComponent(&testCtx, clusterKey, mysqlCompName) sts := &stsList.Items[0] @@ -354,14 +345,13 @@ var _ = Describe("OpsRequest Controller", func() { podRole = "leader" accessMode = "ReadWrite" } - testapps.MockConsensusComponentStsPod(testCtx, sts, clusterObj.Name, mysqlCompName, podName, podRole, accessMode) + testapps.MockConsensusComponentStsPod(&testCtx, sts, clusterObj.Name, mysqlCompName, podName, podRole, accessMode) } - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) } - mockMysqlCluster := func() { + createMysqlCluster := func(replicas int32) { createBackupPolicyTpl(clusterDefObj) - replicas := int32(3) By("set component to horizontal with snapshot policy and create a cluster") Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(clusterDefObj), @@ -394,9 +384,12 @@ var _ = Describe("OpsRequest Controller", func() { // wait for cluster observed generation Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) mockSetClusterStatusPhaseToRunning(clusterKey) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) } - mockHscaleOps := func(replicas int32) *appsv1alpha1.OpsRequest { + createClusterHscaleOps := func(replicas int32) *appsv1alpha1.OpsRequest { By("create a opsRequest to horizontal scale") opsName := "hscale-ops-" + testCtx.GetRandomStr() ops := testapps.NewOpsRequestObj(opsName, testCtx.DefaultNamespace, @@ -411,48 +404,63 @@ var _ = Describe("OpsRequest Controller", func() { return ops } + It("issue an VerticalScalingOpsRequest should change Cluster's resource requirements successfully", func() { + ctx := verticalScalingContext{ + source: resourceContext{class: &testapps.Class1c1g}, + target: resourceContext{resource: testapps.Class2c4g.ToResourceRequirements()}, + } + testVerticalScaleCPUAndMemory(testapps.ConsensusMySQLComponent, ctx) + }) + It("HorizontalScaling when not support snapshot", func() { By("init backup policy template, mysql cluster and hscale ops") viper.Set("VOLUMESNAPSHOT", false) - mockMysqlCluster() - ops := mockHscaleOps(int32(5)) + createMysqlCluster(3) + ops := createClusterHscaleOps(5) opsKey := client.ObjectKeyFromObject(ops) By("expect component is Running if don't support volume snapshot during doing h-scale ops") Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) - // cluster phase changes to HorizontalScalingPhase first. then, it will be ConditionsError because it does not support snapshot backup after a period of time. - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) // HorizontalScalingPhase - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) - - By("delete h-scale ops") - testapps.DeleteObject(&testCtx, opsKey, ops) - Expect(testapps.ChangeObj(&testCtx, ops, func(lopsReq *appsv1alpha1.OpsRequest) { - lopsReq.SetFinalizers([]string{}) - })).ShouldNot(HaveOccurred()) - - By("reset replicas to 1 and cluster should reconcile to Running") + Eventually(testapps.GetClusterGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) + Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, fetched *appsv1alpha1.Cluster) { + // expect cluster phase is Updating during Hscale. + g.Expect(fetched.Status.Phase).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) + // when snapshot is not supported, the expected component phase is running. + g.Expect(fetched.Status.Components[mysqlCompName].Phase).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + // expect preCheckFailed condition to occur. + condition := meta.FindStatusCondition(fetched.Status.Conditions, appsv1alpha1.ConditionTypeProvisioningStarted) + g.Expect(condition).ShouldNot(BeNil()) + g.Expect(condition.Status).Should(BeFalse()) + g.Expect(condition.Reason).Should(Equal(lifecycle.ReasonPreCheckFailed)) + g.Expect(condition.Message).Should(Equal("HorizontalScaleFailed: volume snapshot not support")) + })) + + By("reset replicas to 3 and cluster phase should be reconciled to Running") Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { cluster.Spec.ComponentSpecs[0].Replicas = int32(3) })()).ShouldNot(HaveOccurred()) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) }) - It("HorizontalScaling when support snapshot", func() { + It("HorizontalScaling via volume snapshot backup", func() { By("init backup policy template, mysql cluster and hscale ops") viper.Set("VOLUMESNAPSHOT", true) - mockMysqlCluster() + createMysqlCluster(3) + replicas := int32(5) - ops := mockHscaleOps(replicas) + ops := createClusterHscaleOps(replicas) opsKey := client.ObjectKeyFromObject(ops) By("expect component is Running") Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) - // cluster phase changes to HorizontalScalingPhase first. then, it will be ConditionsError because it does not support snapshot backup after a period of time. + Eventually(testapps.GetClusterGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) + // the expected cluster phase is Updating during Hscale. Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) // component phase should be running during snapshot backup - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) - By("mock snapshot created and ready to use, component phase should change to Updating to do horizontalScaling") + By("mock VolumeSnapshot status is ready, component phase should change to Updating when component is horizontally scaling.") snapshotKey := types.NamespacedName{Name: fmt.Sprintf("%s-%s-scaling", clusterKey.Name, mysqlCompName), Namespace: testCtx.DefaultNamespace} volumeSnapshot := &snapshotv1.VolumeSnapshot{} @@ -461,7 +469,7 @@ var _ = Describe("OpsRequest Controller", func() { volumeSnapshot.Status = &snapshotv1.VolumeSnapshotStatus{ReadyToUse: &readyToUse} Expect(k8sClient.Status().Update(testCtx.Ctx, volumeSnapshot)).Should(Succeed()) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) - Eventually(testapps.GetClusterComponentPhase(testCtx, clusterKey.Name, mysqlCompName)).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) + Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) By("mock component is Running and expect cluster is Running") mockCompRunning(replicas) @@ -497,7 +505,7 @@ var _ = Describe("OpsRequest Controller", func() { By("init replication cluster") // init storageClass storageClassName := "standard" - testapps.CreateStorageClass(testCtx, storageClassName, true) + testapps.CreateStorageClass(&testCtx, storageClassName, true) clusterDefObj = testapps.NewClusterDefFactory(clusterDefName). AddComponentDef(testapps.ReplicationRedisComponent, testapps.DefaultRedisCompDefName). Create(&testCtx).GetObject() diff --git a/controllers/apps/tls_utils_test.go b/controllers/apps/tls_utils_test.go index 291138b02..177c1ea32 100644 --- a/controllers/apps/tls_utils_test.go +++ b/controllers/apps/tls_utils_test.go @@ -45,7 +45,7 @@ var _ = Describe("TLS self-signed cert function", func() { clusterDefName = "test-clusterdef-tls" clusterVersionName = "test-clusterversion-tls" clusterNamePrefix = "test-cluster" - statefulCompDefName = "replicasets" + statefulCompDefName = "mysql" statefulCompName = "mysql" mysqlContainerName = "mysql" configSpecName = "mysql-config-tpl" @@ -291,8 +291,11 @@ var _ = Describe("TLS self-signed cert function", func() { SetTLS(false). Create(&testCtx). GetObject() - Eventually(testapps.GetClusterObservedGeneration(&testCtx, client.ObjectKeyFromObject(clusterObj))).Should(BeEquivalentTo(1)) - stsList := testk8s.ListAndCheckStatefulSet(&testCtx, client.ObjectKeyFromObject(clusterObj)) + clusterKey := client.ObjectKeyFromObject(clusterObj) + Eventually(k8sClient.Get(ctx, clusterKey, clusterObj)).Should(Succeed()) + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) + Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.CreatingClusterPhase)) + stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) sts := stsList.Items[0] cd := &appsv1alpha1.ClusterDefinition{} Expect(k8sClient.Get(ctx, types.NamespacedName{Name: clusterDefName, Namespace: testCtx.DefaultNamespace}, cd)).Should(Succeed()) diff --git a/controllers/dataprotection/restorejob_controller_test.go b/controllers/dataprotection/restorejob_controller_test.go index e777d7723..e5017f8f4 100644 --- a/controllers/dataprotection/restorejob_controller_test.go +++ b/controllers/dataprotection/restorejob_controller_test.go @@ -152,7 +152,7 @@ var _ = Describe("RestoreJob Controller", func() { testRestoreJob := func(withResources ...bool) { By("By creating a statefulset and pod") sts := assureStatefulSetObj() - testapps.MockConsensusComponentPods(testCtx, sts, clusterName, compName) + testapps.MockConsensusComponentPods(&testCtx, sts, clusterName, compName) By("By creating a backupTool") backupTool := assureBackupToolObj(withResources...) diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml index d4ad71843..868747153 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterdefinitions.yaml @@ -220,17 +220,72 @@ spec: - accessMode - name type: object + llPodManagementPolicy: + description: llPodManagementPolicy is the low-level controls + how pods are created during initial scale up, when replacing + pods on nodes, or when scaling down. `OrderedReady` policy + specify where pods are created in increasing order (pod-0, + then pod-1, etc) and the controller will wait until each + pod is ready before continuing. When scaling down, the + pods are removed in the opposite order. `Parallel` policy + specify create pods in parallel to match the desired scale + without waiting, and on scale down will delete all pods + at once. + type: string + llUpdateStrategy: + description: llUpdateStrategy indicates the low-level StatefulSetUpdateStrategy + that will be employed to update Pods in the StatefulSet + when a revision is made to Template. Will ignore `updateStrategy` + attribute if provided. + properties: + rollingUpdate: + description: RollingUpdate is used to communicate parameters + when Type is RollingUpdateStatefulSetStrategyType. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be unavailable during the update. Value can be + an absolute number (ex: 5) or a percentage of + desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding up. This can not be + 0. Defaults to 1. This field is alpha-level and + is only honored by servers that enable the MaxUnavailableStatefulSet + feature. The field applies to all pods in the + range 0 to Replicas-1. That means if there is + any unavailable pod in the range 0 to Replicas-1, + it will be counted towards MaxUnavailable.' + x-kubernetes-int-or-string: true + partition: + description: Partition indicates the ordinal at + which the StatefulSet should be partitioned for + updates. During a rolling update, all pods from + ordinal Replicas-1 to Partition are updated. All + pods from ordinal Partition-1 to 0 remain untouched. + This is helpful in being able to do a canary based + deployment. The default value is 0. + format: int32 + type: integer + type: object + type: + description: Type indicates the type of the StatefulSetUpdateStrategy. + Default is RollingUpdate. + type: string + type: object updateStrategy: default: Serial - description: 'updateStrategy, Pods update strategy. serial: - update Pods one by one that guarantee minimum component - unavailable time. Learner -> Follower(with AccessMode=none) - -> Follower(with AccessMode=readonly) -> Follower(with - AccessMode=readWrite) -> Leader bestEffortParallel: update - Pods in parallel that guarantee minimum component un-writable - time. Learner, Follower(minority) in parallel -> Follower(majority) - -> Leader, keep majority online all the time. parallel: - force parallel' + description: "updateStrategy, Pods update strategy. In case + of workloadType=Consensus the update strategy will be + following: \n serial: update Pods one by one that guarantee + minimum component unavailable time. Learner -> Follower(with + AccessMode=none) -> Follower(with AccessMode=readonly) + -> Follower(with AccessMode=readWrite) -> Leader bestEffortParallel: + update Pods in parallel that guarantee minimum component + un-writable time. Learner, Follower(minority) in parallel + -> Follower(majority) -> Leader, keep majority online + all the time. parallel: force parallel" enum: - Serial - BestEffortParallel @@ -339,16 +394,6 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map - maxUnavailable: - anyOf: - - type: integer - - type: string - description: 'The maximum number of pods that can be unavailable - during scaling. Value can be an absolute number (ex: 5) or - a percentage of desired pods (ex: 10%). Absolute number is - calculated from percentage by rounding down. This value is - ignored if workloadType is Consensus.' - x-kubernetes-int-or-string: true monitor: description: monitor is monitoring config which provided by provider. @@ -8050,9 +8095,62 @@ spec: type: object replicationSpec: description: replicationSpec defines replication related spec - if workloadType is Replication, required if workloadType is - Replication. + if workloadType is Replication. properties: + llPodManagementPolicy: + description: llPodManagementPolicy is the low-level controls + how pods are created during initial scale up, when replacing + pods on nodes, or when scaling down. `OrderedReady` policy + specify where pods are created in increasing order (pod-0, + then pod-1, etc) and the controller will wait until each + pod is ready before continuing. When scaling down, the + pods are removed in the opposite order. `Parallel` policy + specify create pods in parallel to match the desired scale + without waiting, and on scale down will delete all pods + at once. + type: string + llUpdateStrategy: + description: llUpdateStrategy indicates the low-level StatefulSetUpdateStrategy + that will be employed to update Pods in the StatefulSet + when a revision is made to Template. Will ignore `updateStrategy` + attribute if provided. + properties: + rollingUpdate: + description: RollingUpdate is used to communicate parameters + when Type is RollingUpdateStatefulSetStrategyType. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be unavailable during the update. Value can be + an absolute number (ex: 5) or a percentage of + desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding up. This can not be + 0. Defaults to 1. This field is alpha-level and + is only honored by servers that enable the MaxUnavailableStatefulSet + feature. The field applies to all pods in the + range 0 to Replicas-1. That means if there is + any unavailable pod in the range 0 to Replicas-1, + it will be counted towards MaxUnavailable.' + x-kubernetes-int-or-string: true + partition: + description: Partition indicates the ordinal at + which the StatefulSet should be partitioned for + updates. During a rolling update, all pods from + ordinal Replicas-1 to Partition are updated. All + pods from ordinal Partition-1 to 0 remain untouched. + This is helpful in being able to do a canary based + deployment. The default value is 0. + format: int32 + type: integer + type: object + type: + description: Type indicates the type of the StatefulSetUpdateStrategy. + Default is RollingUpdate. + type: string + type: object switchCmdExecutorConfig: description: switchCmdExecutorConfig configs how to get client SDK and perform switch statements. @@ -8284,6 +8382,23 @@ spec: type: object minItems: 1 type: array + updateStrategy: + default: Serial + description: "updateStrategy, Pods update strategy. In case + of workloadType=Consensus the update strategy will be + following: \n serial: update Pods one by one that guarantee + minimum component unavailable time. Learner -> Follower(with + AccessMode=none) -> Follower(with AccessMode=readonly) + -> Follower(with AccessMode=readWrite) -> Leader bestEffortParallel: + update Pods in parallel that guarantee minimum component + un-writable time. Learner, Follower(minority) in parallel + -> Follower(majority) -> Leader, keep majority online + all the time. parallel: force parallel" + enum: + - Serial + - BestEffortParallel + - Parallel + type: string required: - switchCmdExecutorConfig type: object @@ -8401,6 +8516,141 @@ spec: - protocol x-kubernetes-list-type: map type: object + statefulSpec: + description: statefulSpec defines stateful related spec if workloadType + is Stateful. + properties: + llPodManagementPolicy: + description: llPodManagementPolicy is the low-level controls + how pods are created during initial scale up, when replacing + pods on nodes, or when scaling down. `OrderedReady` policy + specify where pods are created in increasing order (pod-0, + then pod-1, etc) and the controller will wait until each + pod is ready before continuing. When scaling down, the + pods are removed in the opposite order. `Parallel` policy + specify create pods in parallel to match the desired scale + without waiting, and on scale down will delete all pods + at once. + type: string + llUpdateStrategy: + description: llUpdateStrategy indicates the low-level StatefulSetUpdateStrategy + that will be employed to update Pods in the StatefulSet + when a revision is made to Template. Will ignore `updateStrategy` + attribute if provided. + properties: + rollingUpdate: + description: RollingUpdate is used to communicate parameters + when Type is RollingUpdateStatefulSetStrategyType. + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be unavailable during the update. Value can be + an absolute number (ex: 5) or a percentage of + desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding up. This can not be + 0. Defaults to 1. This field is alpha-level and + is only honored by servers that enable the MaxUnavailableStatefulSet + feature. The field applies to all pods in the + range 0 to Replicas-1. That means if there is + any unavailable pod in the range 0 to Replicas-1, + it will be counted towards MaxUnavailable.' + x-kubernetes-int-or-string: true + partition: + description: Partition indicates the ordinal at + which the StatefulSet should be partitioned for + updates. During a rolling update, all pods from + ordinal Replicas-1 to Partition are updated. All + pods from ordinal Partition-1 to 0 remain untouched. + This is helpful in being able to do a canary based + deployment. The default value is 0. + format: int32 + type: integer + type: object + type: + description: Type indicates the type of the StatefulSetUpdateStrategy. + Default is RollingUpdate. + type: string + type: object + updateStrategy: + default: Serial + description: "updateStrategy, Pods update strategy. In case + of workloadType=Consensus the update strategy will be + following: \n serial: update Pods one by one that guarantee + minimum component unavailable time. Learner -> Follower(with + AccessMode=none) -> Follower(with AccessMode=readonly) + -> Follower(with AccessMode=readWrite) -> Leader bestEffortParallel: + update Pods in parallel that guarantee minimum component + un-writable time. Learner, Follower(minority) in parallel + -> Follower(majority) -> Leader, keep majority online + all the time. parallel: force parallel" + enum: + - Serial + - BestEffortParallel + - Parallel + type: string + type: object + statelessSpec: + description: statelessSpec defines stateless related spec if + workloadType is Stateless. + properties: + updateStrategy: + description: updateStrategy defines the underlying deployment + strategy to use to replace existing pods with new ones. + properties: + rollingUpdate: + description: 'Rolling update config params. Present + only if DeploymentStrategyType = RollingUpdate. --- + TODO: Update this to follow our convention for oneOf, + whatever we decide it to be.' + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be scheduled above the desired number of pods. + Value can be an absolute number (ex: 5) or a percentage + of desired pods (ex: 10%). This can not be 0 if + MaxUnavailable is 0. Absolute number is calculated + from percentage by rounding up. Defaults to 25%. + Example: when this is set to 30%, the new ReplicaSet + can be scaled up immediately when the rolling + update starts, such that the total number of old + and new pods do not exceed 130% of desired pods. + Once old pods have been killed, new ReplicaSet + can be scaled up further, ensuring that total + number of pods running at any time during the + update is at most 130% of desired pods.' + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can + be unavailable during the update. Value can be + an absolute number (ex: 5) or a percentage of + desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding down. This can not + be 0 if MaxSurge is 0. Defaults to 25%. Example: + when this is set to 30%, the old ReplicaSet can + be scaled down to 70% of desired pods immediately + when the rolling update starts. Once new pods + are ready, old ReplicaSet can be scaled down further, + followed by scaling up the new ReplicaSet, ensuring + that the total number of pods available at all + times during the update is at least 70% of desired + pods.' + x-kubernetes-int-or-string: true + type: object + type: + description: Type of deployment. Can be "Recreate" or + "RollingUpdate". Default is RollingUpdate. + type: string + type: object + type: object systemAccounts: description: Statement to create system account. properties: @@ -8718,6 +8968,11 @@ spec: - name - workloadType type: object + x-kubernetes-validations: + - message: componentDefs.consensusSpec is required when componentDefs.workloadType + is Consensus, and forbidden otherwise + rule: 'has(self.workloadType) && self.workloadType == ''Consensus'' + ? has(self.consensusSpec) : !has(self.consensusSpec)' minItems: 1 type: array x-kubernetes-list-map-keys: diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml index fba2650c2..78973ba06 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusters.yaml @@ -235,6 +235,12 @@ spec: maxLength: 15 pattern: ^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$ type: string + noCreatePDB: + default: false + description: noCreatePDB defines PodDistruptionBudget creation + behavior, set to true if creation of PodDistruptionBudget + for this component is not needed. Defaults to false. + type: boolean primaryIndex: description: primaryIndex determines which index is primary when workloadType is Replication, index number starts from diff --git a/deploy/postgresql/config/pg12-config-constraint.cue b/deploy/postgresql/config/pg12-config-constraint.cue index 26ddd9f20..773e87cd7 100644 --- a/deploy/postgresql/config/pg12-config-constraint.cue +++ b/deploy/postgresql/config/pg12-config-constraint.cue @@ -910,7 +910,7 @@ shared_buffers?: int & >=16 & <=1073741823 @storeResource(8KB) // Lists shared libraries to preload into server. - // TODO support enum list, e.g. shared_preload_libraries = 'pg_stat_statements, auto_explain' + // TODO: support enum list, e.g. shared_preload_libraries = 'pg_stat_statements, auto_explain' // shared_preload_libraries?: string & "auto_explain" | "orafce" | "pgaudit" | "pglogical" | "pg_bigm" | "pg_cron" | "pg_hint_plan" | "pg_prewarm" | "pg_similarity" | "pg_stat_statements" | "pg_tle" | "pg_transport" | "plprofiler" // Enables SSL connections. diff --git a/deploy/postgresql/config/pg14-config-constraint.cue b/deploy/postgresql/config/pg14-config-constraint.cue index 775749618..e4408fe58 100644 --- a/deploy/postgresql/config/pg14-config-constraint.cue +++ b/deploy/postgresql/config/pg14-config-constraint.cue @@ -913,7 +913,7 @@ shared_buffers?: int & >=16 & <=1073741823 @storeResource(8KB) // Lists shared libraries to preload into server. - // TODO support enum list, e.g. shared_preload_libraries = 'pg_stat_statements, auto_explain' + // TODO: support enum list, e.g. shared_preload_libraries = 'pg_stat_statements, auto_explain' // shared_preload_libraries?: string & "auto_explain" | "orafce" | "pgaudit" | "pglogical" | "pg_bigm" | "pg_cron" | "pg_hint_plan" | "pg_prewarm" | "pg_similarity" | "pg_stat_statements" | "pg_tle" | "pg_transport" | "plprofiler" // Enables SSL connections. diff --git a/internal/controller/builder/builder.go b/internal/controller/builder/builder.go index 1b5a47c32..1602907e3 100644 --- a/internal/controller/builder/builder.go +++ b/internal/controller/builder/builder.go @@ -233,10 +233,9 @@ func BuildPersistentVolumeClaimLabels(sts *appsv1.StatefulSet, pvc *corev1.Persi } } -func BuildSvcList(params BuilderParams) ([]*corev1.Service, error) { +func BuildSvcListWithCustomAttributes(params BuilderParams, customAttributeSetter func(*corev1.Service)) ([]*corev1.Service, error) { const tplFile = "service_template.cue" - - var result []*corev1.Service + var result = make([]*corev1.Service, 0, len(params.Component.Services)) for _, item := range params.Component.Services { if len(item.Spec.Ports) == 0 { continue @@ -249,15 +248,16 @@ func BuildSvcList(params BuilderParams) ([]*corev1.Service, error) { }, "svc", &svc); err != nil { return nil, err } + if customAttributeSetter != nil { + customAttributeSetter(&svc) + } result = append(result, &svc) } - return result, nil } func BuildHeadlessSvc(params BuilderParams) (*corev1.Service, error) { const tplFile = "headless_service_template.cue" - service := corev1.Service{} if err := buildFromCUE(tplFile, map[string]any{ "cluster": params.Cluster, @@ -279,6 +279,10 @@ func BuildSts(reqCtx intctrlutil.RequestCtx, params BuilderParams, envConfigName return nil, err } + if params.Component.StatefulSetWorkload != nil { + sts.Spec.PodManagementPolicy, sts.Spec.UpdateStrategy = params.Component.StatefulSetWorkload.FinalStsUpdateStrategy() + } + // update sts.spec.volumeClaimTemplates[].metadata.labels if len(sts.Spec.VolumeClaimTemplates) > 0 && len(sts.GetLabels()) > 0 { for index, vct := range sts.Spec.VolumeClaimTemplates { @@ -371,11 +375,9 @@ func BuildConnCredential(params BuilderParams) (*corev1.Secret, error) { // 2nd pass replace $(CONN_CREDENTIAL) variables m = map[string]string{} - for k, v := range connCredential.StringData { m[fmt.Sprintf("$(CONN_CREDENTIAL).%s", k)] = v } - replaceData(m) return &connCredential, nil } @@ -389,13 +391,11 @@ func BuildPDB(params BuilderParams) (*policyv1.PodDisruptionBudget, error) { }, "pdb", &pdb); err != nil { return nil, err } - return &pdb, nil } func BuildDeploy(reqCtx intctrlutil.RequestCtx, params BuilderParams) (*appsv1.Deployment, error) { const tplFile = "deployment_template.cue" - deploy := appsv1.Deployment{} if err := buildFromCUE(tplFile, map[string]any{ "cluster": params.Cluster, @@ -403,7 +403,9 @@ func BuildDeploy(reqCtx intctrlutil.RequestCtx, params BuilderParams) (*appsv1.D }, "deployment", &deploy); err != nil { return nil, err } - + if params.Component.StatelessSpec != nil { + deploy.Spec.Strategy = params.Component.StatelessSpec.UpdateStrategy + } if err := processContainersInjection(reqCtx, params, "", &deploy.Spec.Template.Spec); err != nil { return nil, err } @@ -415,7 +417,6 @@ func BuildPVCFromSnapshot(sts *appsv1.StatefulSet, pvcKey types.NamespacedName, snapshotName string, component *component.SynthesizedComponent) (*corev1.PersistentVolumeClaim, error) { - pvc := corev1.PersistentVolumeClaim{} if err := buildFromCUE("pvc_template.cue", map[string]any{ "sts": sts, @@ -432,9 +433,7 @@ func BuildPVCFromSnapshot(sts *appsv1.StatefulSet, // BuildEnvConfig build cluster component context ConfigMap object, which is to be used in workload container's // envFrom.configMapRef with name of "$(cluster.metadata.name)-$(component.name)-env" pattern. func BuildEnvConfig(params BuilderParams, reqCtx intctrlutil.RequestCtx, cli client.Client) (*corev1.ConfigMap, error) { - const tplFile = "env_config_template.cue" - prefix := constant.KBPrefix + "_" + strings.ToUpper(params.Component.Type) + "_" svcName := strings.Join([]string{params.Cluster.Name, params.Component.Name, "headless"}, "-") envData := map[string]string{} @@ -481,7 +480,6 @@ func BuildEnvConfig(params BuilderParams, reqCtx intctrlutil.RequestCtx, cli cli // set cluster uid to let pod know if the cluster is recreated envData[prefix+"CLUSTER_UID"] = string(params.Cluster.UID) - config := corev1.ConfigMap{} if err := buildFromCUE(tplFile, map[string]any{ "cluster": params.Cluster, @@ -490,7 +488,6 @@ func BuildEnvConfig(params BuilderParams, reqCtx intctrlutil.RequestCtx, cli cli }, "config", &config); err != nil { return nil, err } - return &config, nil } @@ -505,7 +502,6 @@ func BuildBackup(sts *appsv1.StatefulSet, }, "backup_job", &backup); err != nil { return nil, err } - return &backup, nil } @@ -520,16 +516,13 @@ func BuildVolumeSnapshot(snapshotKey types.NamespacedName, }, "snapshot", &snapshot); err != nil { return nil, err } - return &snapshot, nil } func BuildCronJob(pvcKey types.NamespacedName, schedule string, sts *appsv1.StatefulSet) (*batchv1.CronJob, error) { - serviceAccount := viper.GetString("KUBEBLOCKS_SERVICEACCOUNT_NAME") - cronJob := batchv1.CronJob{} if err := buildFromCUE("delete_pvc_cron_job_template.cue", map[string]any{ "pvc": pvcKey, @@ -539,7 +532,6 @@ func BuildCronJob(pvcKey types.NamespacedName, }, "cronjob", &cronJob); err != nil { return nil, err } - return &cronJob, nil } @@ -636,7 +628,6 @@ func BuildCfgManagerContainer(sidecarRenderedParam *cfgcm.CfgManagerBuildParams) func BuildTLSSecret(namespace, clusterName, componentName string) (*corev1.Secret, error) { const tplFile = "tls_certs_secret_template.cue" - secret := &corev1.Secret{} pathedName := componentPathedName{ Namespace: namespace, @@ -651,7 +642,6 @@ func BuildTLSSecret(namespace, clusterName, componentName string) (*corev1.Secre func BuildBackupManifestsJob(key types.NamespacedName, backup *dataprotectionv1alpha1.Backup, podSpec *corev1.PodSpec) (*batchv1.Job, error) { const tplFile = "backup_manifests_template.cue" - job := &batchv1.Job{} if err := buildFromCUE(tplFile, map[string]any{ @@ -682,6 +672,5 @@ func BuildPITRJob(name string, cluster *appsv1alpha1.Cluster, image string, comm }, "job", job); err != nil { return nil, err } - return job, nil } diff --git a/internal/controller/builder/builder_test.go b/internal/controller/builder/builder_test.go index 768497971..98fcced74 100644 --- a/internal/controller/builder/builder_test.go +++ b/internal/controller/builder/builder_test.go @@ -204,7 +204,7 @@ var _ = Describe("builder", func() { It("builds Service correctly", func() { params := newParams() - svcList, err := BuildSvcList(*params) + svcList, err := BuildSvcListWithCustomAttributes(*params, nil) Expect(err).Should(BeNil()) Expect(svcList).ShouldNot(BeEmpty()) }) diff --git a/internal/controller/builder/cue/backup_job_template.cue b/internal/controller/builder/cue/backup_job_template.cue index c91cf30ea..5369a660c 100644 --- a/internal/controller/builder/cue/backup_job_template.cue +++ b/internal/controller/builder/cue/backup_job_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . sts: { metadata: { diff --git a/internal/controller/builder/cue/backup_manifests_template.cue b/internal/controller/builder/cue/backup_manifests_template.cue index 88d7a7e65..cd41196c0 100644 --- a/internal/controller/builder/cue/backup_manifests_template.cue +++ b/internal/controller/builder/cue/backup_manifests_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . backup: { metadata: { diff --git a/internal/controller/builder/cue/config_manager_sidecar.cue b/internal/controller/builder/cue/config_manager_sidecar.cue index c5f851151..949b79a34 100644 --- a/internal/controller/builder/cue/config_manager_sidecar.cue +++ b/internal/controller/builder/cue/config_manager_sidecar.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . template: { name: parameter.name diff --git a/internal/controller/builder/cue/config_template.cue b/internal/controller/builder/cue/config_template.cue index c3763c0e0..56278929b 100644 --- a/internal/controller/builder/cue/config_template.cue +++ b/internal/controller/builder/cue/config_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . meta: { clusterDefinition: { diff --git a/internal/controller/builder/cue/conn_credential_template.cue b/internal/controller/builder/cue/conn_credential_template.cue index 8f6a2f2b4..237012a49 100644 --- a/internal/controller/builder/cue/conn_credential_template.cue +++ b/internal/controller/builder/cue/conn_credential_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . clusterdefinition: { metadata: { diff --git a/internal/controller/builder/cue/delete_pvc_cron_job_template.cue b/internal/controller/builder/cue/delete_pvc_cron_job_template.cue index 762e3f368..84ed91891 100644 --- a/internal/controller/builder/cue/delete_pvc_cron_job_template.cue +++ b/internal/controller/builder/cue/delete_pvc_cron_job_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . pvc: { Name: string diff --git a/internal/controller/builder/cue/deployment_template.cue b/internal/controller/builder/cue/deployment_template.cue index b2bb58cad..6c57ca356 100644 --- a/internal/controller/builder/cue/deployment_template.cue +++ b/internal/controller/builder/cue/deployment_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/env_config_template.cue b/internal/controller/builder/cue/env_config_template.cue index dd2ef2cdc..76412f5a8 100644 --- a/internal/controller/builder/cue/env_config_template.cue +++ b/internal/controller/builder/cue/env_config_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . cluster: { metadata: { @@ -38,7 +38,7 @@ config: { "app.kubernetes.io/managed-by": "kubeblocks" // configmap selector for env update "apps.kubeblocks.io/config-type": "kubeblocks-env" - "apps.kubeblocks.io/component-name": component.name + "apps.kubeblocks.io/component-name": "\(component.name)" } } data: [string]: string diff --git a/internal/controller/builder/cue/headless_service_template.cue b/internal/controller/builder/cue/headless_service_template.cue index d81fcd8c7..2d62170d1 100644 --- a/internal/controller/builder/cue/headless_service_template.cue +++ b/internal/controller/builder/cue/headless_service_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/pdb_template.cue b/internal/controller/builder/cue/pdb_template.cue index 4bb702dbf..1e5fdc6c4 100644 --- a/internal/controller/builder/cue/pdb_template.cue +++ b/internal/controller/builder/cue/pdb_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . cluster: { metadata: { @@ -24,10 +24,7 @@ cluster: { component: { clusterDefName: string name: string - // FIXME not defined in apis - maxUnavailable: string - podSpec: containers: [...] - volumeClaimTemplates: [...] + minAvailable: string | int } pdb: { @@ -45,8 +42,8 @@ pdb: { } } "spec": { - if component.maxUnavailable != _|_ { - maxUnavailable: component.maxUnavailable + if component.minAvailable != _|_ { + minAvailable: component.minAvailable } selector: { matchLabels: { diff --git a/internal/controller/builder/cue/pitr_job_template.cue b/internal/controller/builder/cue/pitr_job_template.cue index 1ed22b8fd..3eef8a853 100644 --- a/internal/controller/builder/cue/pitr_job_template.cue +++ b/internal/controller/builder/cue/pitr_job_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . container: { name: "pitr" diff --git a/internal/controller/builder/cue/pvc_template.cue b/internal/controller/builder/cue/pvc_template.cue index 4b34a43e8..aeb355edf 100644 --- a/internal/controller/builder/cue/pvc_template.cue +++ b/internal/controller/builder/cue/pvc_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . sts: { metadata: { diff --git a/internal/controller/builder/cue/service_template.cue b/internal/controller/builder/cue/service_template.cue index 27193553e..ad423824e 100644 --- a/internal/controller/builder/cue/service_template.cue +++ b/internal/controller/builder/cue/service_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . cluster: { metadata: { diff --git a/internal/controller/builder/cue/snapshot_template.cue b/internal/controller/builder/cue/snapshot_template.cue index 264b5c437..270a01bf9 100644 --- a/internal/controller/builder/cue/snapshot_template.cue +++ b/internal/controller/builder/cue/snapshot_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . snapshot_key: { Name: string diff --git a/internal/controller/builder/cue/statefulset_template.cue b/internal/controller/builder/cue/statefulset_template.cue index 57ca9925a..9a4d7dd26 100644 --- a/internal/controller/builder/cue/statefulset_template.cue +++ b/internal/controller/builder/cue/statefulset_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . cluster: { metadata: { @@ -55,10 +55,8 @@ statefulset: { "app.kubernetes.io/managed-by": "kubeblocks" "apps.kubeblocks.io/component-name": "\(component.name)" } - serviceName: "\(cluster.metadata.name)-\(component.name)-headless" - replicas: component.replicas - minReadySeconds: 10 - podManagementPolicy: "Parallel" + serviceName: "\(cluster.metadata.name)-\(component.name)-headless" + replicas: component.replicas template: { metadata: { labels: { diff --git a/internal/controller/builder/cue/tls_certs_secret_template.cue b/internal/controller/builder/cue/tls_certs_secret_template.cue index 521fe978f..e5a4df682 100644 --- a/internal/controller/builder/cue/tls_certs_secret_template.cue +++ b/internal/controller/builder/cue/tls_certs_secret_template.cue @@ -1,19 +1,19 @@ -//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// Copyright (C) 2022-2023 ApeCloud Co., Ltd // -//This file is part of KubeBlocks project +// This file is part of KubeBlocks project // -//This program is free software: you can redistribute it and/or modify -//it under the terms of the GNU Affero General Public License as published by -//the Free Software Foundation, either version 3 of the License, or -//(at your option) any later version. +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // -//This program is distributed in the hope that it will be useful -//but WITHOUT ANY WARRANTY; without even the implied warranty of -//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//GNU Affero General Public License for more details. +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. // -//You should have received a copy of the GNU Affero General Public License -//along with this program. If not, see . +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . pathedName: { namespace: string diff --git a/internal/controller/component/component.go b/internal/controller/component/component.go index b863ad9ab..3163032ef 100644 --- a/internal/controller/component/component.go +++ b/internal/controller/component/component.go @@ -49,31 +49,29 @@ func BuildComponent( Name: clusterCompSpec.Name, Type: clusterCompDefObj.Name, CharacterType: clusterCompDefObj.CharacterType, - MaxUnavailable: clusterCompDefObj.MaxUnavailable, WorkloadType: clusterCompDefObj.WorkloadType, + StatelessSpec: clusterCompDefObj.StatelessSpec, + StatefulSpec: clusterCompDefObj.StatefulSpec, ConsensusSpec: clusterCompDefObj.ConsensusSpec, + ReplicationSpec: clusterCompDefObj.ReplicationSpec, PodSpec: clusterCompDefObj.PodSpec, Probes: clusterCompDefObj.Probes, LogConfigs: clusterCompDefObj.LogConfigs, HorizontalScalePolicy: clusterCompDefObj.HorizontalScalePolicy, + ConfigTemplates: clusterCompDefObj.ConfigSpecs, + ScriptTemplates: clusterCompDefObj.ScriptSpecs, + VolumeTypes: clusterCompDefObj.VolumeTypes, + CustomLabelSpecs: clusterCompDefObj.CustomLabelSpecs, + StatefulSetWorkload: clusterCompDefObj.GetStatefulSetWorkload(), + MinAvailable: clusterCompSpec.GetMinAvailable(clusterCompDefObj.GetMinAvailable()), Replicas: clusterCompSpec.Replicas, EnabledLogs: clusterCompSpec.EnabledLogs, TLS: clusterCompSpec.TLS, Issuer: clusterCompSpec.Issuer, - VolumeTypes: clusterCompDefObj.VolumeTypes, - CustomLabelSpecs: clusterCompDefObj.CustomLabelSpecs, ComponentDef: clusterCompSpec.ComponentDefRef, ServiceAccountName: clusterCompSpec.ServiceAccountName, } - // resolve component.ConfigTemplates - if clusterCompDefObj.ConfigSpecs != nil { - component.ConfigTemplates = clusterCompDefObj.ConfigSpecs - } - if clusterCompDefObj.ScriptSpecs != nil { - component.ScriptTemplates = clusterCompDefObj.ScriptSpecs - } - if len(clusterCompVers) > 0 && clusterCompVers[0] != nil { // only accept 1st ClusterVersion override context clusterCompVer := clusterCompVers[0] @@ -87,6 +85,7 @@ func BuildComponent( } } + // handle component.PodSpec extra settings // set affinity and tolerations affinity := cluster.Spec.Affinity if clusterCompSpec.Affinity != nil { @@ -105,16 +104,13 @@ func BuildComponent( if clusterCompSpec.VolumeClaimTemplates != nil { component.VolumeClaimTemplates = clusterCompSpec.ToVolumeClaimTemplates() } - if clusterCompSpec.Resources.Requests != nil || clusterCompSpec.Resources.Limits != nil { component.PodSpec.Containers[0].Resources = clusterCompSpec.Resources } - if clusterCompDefObj.Service != nil { service := corev1.Service{Spec: clusterCompDefObj.Service.ToSVCSpec()} service.Spec.Type = corev1.ServiceTypeClusterIP component.Services = append(component.Services, service) - for _, item := range clusterCompSpec.Services { service = corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -127,13 +123,12 @@ func BuildComponent( component.Services = append(component.Services, service) } } - component.PrimaryIndex = clusterCompSpec.PrimaryIndex // set component.PodSpec.ServiceAccountName component.PodSpec.ServiceAccountName = component.ServiceAccountName - // TODO(zhixu.zt) We need to reserve the VolumeMounts of the container for ConfigMap or Secret, - // At present, it is possible to distinguish between ConfigMap volume and normal volume, + // TODO: (zhixu.zt) We need to reserve the VolumeMounts of the container for ConfigMap or Secret, + // At present, it is not possible to distinguish between ConfigMap volume and normal volume, // Compare the VolumeName of configTemplateRef and Name of VolumeMounts // // if component.VolumeClaimTemplates == nil { @@ -147,9 +142,7 @@ func BuildComponent( reqCtx.Log.Error(err, "build probe container failed.") return nil, err } - replaceContainerPlaceholderTokens(component, GetEnvReplacementMapForConnCredential(cluster.GetName())) - return component, nil } diff --git a/internal/controller/component/type.go b/internal/controller/component/type.go index b8239b87e..cef1a2ed2 100644 --- a/internal/controller/component/type.go +++ b/internal/controller/component/type.go @@ -37,10 +37,13 @@ type SynthesizedComponent struct { Name string `json:"name,omitempty"` Type string `json:"type,omitempty"` CharacterType string `json:"characterType,omitempty"` - MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + MinAvailable *intstr.IntOrString `json:"minAvailable,omitempty"` Replicas int32 `json:"replicas"` WorkloadType v1alpha1.WorkloadType `json:"workloadType,omitempty"` + StatelessSpec *v1alpha1.StatelessSetSpec `json:"statelessSpec,omitempty"` + StatefulSpec *v1alpha1.StatefulSetSpec `json:"statefulSpec,omitempty"` ConsensusSpec *v1alpha1.ConsensusSetSpec `json:"consensusSpec,omitempty"` + ReplicationSpec *v1alpha1.ReplicationSetSpec `json:"replicationSpec,omitempty"` PrimaryIndex *int32 `json:"primaryIndex,omitempty"` PodSpec *corev1.PodSpec `json:"podSpec,omitempty"` Services []corev1.Service `json:"services,omitempty"` @@ -58,6 +61,7 @@ type SynthesizedComponent struct { CustomLabelSpecs []v1alpha1.CustomLabelSpec `json:"customLabelSpecs,omitempty"` ComponentDef string `json:"componentDef,omitempty"` ServiceAccountName string `json:"serviceAccountName,omitempty"` + StatefulSetWorkload v1alpha1.StatefulSetWorkload } // GetPrimaryIndex provides PrimaryIndex value getter, if PrimaryIndex is diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index cfbb444a5..c89bb732f 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -29,6 +29,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -331,13 +332,16 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec handlePVC := func(origObj, pvcProto *corev1.PersistentVolumeClaim) (client.Object, error) { pvcObj := origObj.DeepCopy() - if pvcObj.Spec.Resources.Requests[corev1.ResourceStorage] == pvcProto.Spec.Resources.Requests[corev1.ResourceStorage] { - return pvcObj, nil - } pvcObj.Spec.Resources.Requests[corev1.ResourceStorage] = pvcProto.Spec.Resources.Requests[corev1.ResourceStorage] return pvcObj, nil } + handlePDB := func(origObj, pdbProto *policyv1.PodDisruptionBudget) (client.Object, error) { + pdbObj := origObj.DeepCopy() + pdbObj.Spec = pdbProto.Spec + return pdbObj, nil + } + switch v := node.obj.(type) { case *appsv1.StatefulSet: return handleSts(node.oriObj.(*appsv1.StatefulSet), v) @@ -347,10 +351,9 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec return handleSvc(node.oriObj.(*corev1.Service), v) case *corev1.PersistentVolumeClaim: return handlePVC(node.oriObj.(*corev1.PersistentVolumeClaim), v) - case *corev1.Secret, *corev1.ConfigMap: - return v, nil + case *policyv1.PodDisruptionBudget: + return handlePDB(node.oriObj.(*policyv1.PodDisruptionBudget), v) } - return node.obj, nil } diff --git a/internal/controller/lifecycle/transformer_cluster_deletion.go b/internal/controller/lifecycle/transformer_cluster_deletion.go index 7bb87cc98..ef3096a92 100644 --- a/internal/controller/lifecycle/transformer_cluster_deletion.go +++ b/internal/controller/lifecycle/transformer_cluster_deletion.go @@ -48,7 +48,8 @@ func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * kinds := make([]client.ObjectList, 0) switch cluster.Spec.TerminationPolicy { case v1alpha1.DoNotTerminate: - transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeWarning, "DoNotTerminate", "spec.terminationPolicy %s is preventing deletion.", cluster.Spec.TerminationPolicy) + transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeWarning, "DoNotTerminate", + "spec.terminationPolicy %s is preventing deletion.", cluster.Spec.TerminationPolicy) return graph.ErrNoops case v1alpha1.Halt: kinds = kindsForHalt() @@ -92,12 +93,12 @@ func kindsForDoNotTerminate() []client.ObjectList { func kindsForHalt() []client.ObjectList { kinds := kindsForDoNotTerminate() kindsPlus := []client.ObjectList{ + &policyv1.PodDisruptionBudgetList{}, + &corev1.ServiceList{}, &appsv1.StatefulSetList{}, &appsv1.DeploymentList{}, - &corev1.ServiceList{}, &corev1.SecretList{}, &corev1.ConfigMapList{}, - &policyv1.PodDisruptionBudgetList{}, } return append(kinds, kindsPlus...) } diff --git a/internal/controller/lifecycle/transformer_cluster_status.go b/internal/controller/lifecycle/transformer_cluster_status.go index 7641e28e6..a4778baae 100644 --- a/internal/controller/lifecycle/transformer_cluster_status.go +++ b/internal/controller/lifecycle/transformer_cluster_status.go @@ -504,7 +504,7 @@ func getClusterAvailabilityEffect(componentDef *appsv1alpha1.ClusterComponentDef case appsv1alpha1.Consensus, appsv1alpha1.Replication: return true default: - return componentDef.MaxUnavailable != nil + return componentDef.GetMaxUnavailable() != nil } } diff --git a/internal/controller/plan/prepare.go b/internal/controller/plan/prepare.go index 430d0d2c5..357c8f81e 100644 --- a/internal/controller/plan/prepare.go +++ b/internal/controller/plan/prepare.go @@ -109,7 +109,7 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, }() // render config template - configs, err := buildCfg(task, workload, podSpec, reqCtx.Ctx, cli) + configs, err := renderConfigNScriptFiles(task, workload, podSpec, reqCtx.Ctx, cli) if err != nil { return err } @@ -125,32 +125,8 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, return nil } - // REVIEW/TODO: - // - need higher level abstraction handling - // - or move this module to part operator controller handling - switch task.Component.WorkloadType { - case appsv1alpha1.Stateless: - if err := workloadProcessor( - func(envConfig *corev1.ConfigMap) (client.Object, error) { - return builder.BuildDeploy(reqCtx, task.GetBuilderParams()) - }); err != nil { - return err - } - case appsv1alpha1.Stateful: - if err := workloadProcessor( - func(envConfig *corev1.ConfigMap) (client.Object, error) { - return builder.BuildSts(reqCtx, task.GetBuilderParams(), envConfig.Name) - }); err != nil { - return err - } - case appsv1alpha1.Consensus: - if err := workloadProcessor( - func(envConfig *corev1.ConfigMap) (client.Object, error) { - return buildConsensusSet(reqCtx, task, envConfig.Name) - }); err != nil { - return err - } - case appsv1alpha1.Replication: + // pre-condition check + if task.Component.WorkloadType == appsv1alpha1.Replication { // get the number of existing pods under the current component var existPodList = &corev1.PodList{} if err := componentutil.GetObjectListByComponentName(reqCtx.Ctx, cli, *task.Cluster, existPodList, task.Component.Name); err != nil { @@ -158,7 +134,7 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, } // If the Pods already exists, check whether there is an HA switching and the HA process is prioritized to handle. - // TODO(xingran) After refactoring, HA switching will be handled in the replicationSet controller. + // TODO: (xingran) After refactoring, HA switching will be handled in the replicationSet controller. if len(existPodList.Items) > 0 { primaryIndexChanged, _, err := replication.CheckPrimaryIndexChanged(reqCtx.Ctx, cli, task.Cluster, task.Component.Name, task.Component.GetPrimaryIndex()) @@ -173,44 +149,57 @@ func PrepareComponentResources(reqCtx intctrlutil.RequestCtx, cli client.Client, } } - if err := workloadProcessor( - func(envConfig *corev1.ConfigMap) (client.Object, error) { - return buildReplicationSet(reqCtx, task, envConfig.Name) - }); err != nil { - return err - } } - if needBuildPDB(task) { + // TODO: may add a PDB transform to Create/Update/Delete. + // if no these handle, the cluster controller will occur an error during reconciling. + // conditional build PodDisruptionBudget + if task.Component.MinAvailable != nil { pdb, err := builder.BuildPDB(task.GetBuilderParams()) if err != nil { return err } task.AppendResource(pdb) + } else { + panic("this shouldn't happen") } - svcList, err := builder.BuildSvcList(task.GetBuilderParams()) - if err != nil { - return err - } - for _, svc := range svcList { - // REVIEW/TODO: need higher level abstraction handling + svcList, err := builder.BuildSvcListWithCustomAttributes(task.GetBuilderParams(), func(svc *corev1.Service) { switch task.Component.WorkloadType { case appsv1alpha1.Consensus: addLeaderSelectorLabels(svc, task.Component) case appsv1alpha1.Replication: svc.Spec.Selector[constant.RoleLabelKey] = string(replication.Primary) } + }) + if err != nil { + return err + } + for _, svc := range svcList { task.AppendResource(svc) } - return nil -} -// needBuildPDB check whether the PodDisruptionBudget needs to be built -func needBuildPDB(task *intctrltypes.ReconcileTask) bool { - // TODO: add ut - comp := task.Component - return comp.WorkloadType == appsv1alpha1.Consensus && comp.MaxUnavailable != nil + // REVIEW/TODO: + // - need higher level abstraction handling + // - or move this module to part operator controller handling + switch task.Component.WorkloadType { + case appsv1alpha1.Stateless: + if err := workloadProcessor( + func(envConfig *corev1.ConfigMap) (client.Object, error) { + return builder.BuildDeploy(reqCtx, task.GetBuilderParams()) + }); err != nil { + return err + } + case appsv1alpha1.Stateful, appsv1alpha1.Consensus, appsv1alpha1.Replication: + if err := workloadProcessor( + func(envConfig *corev1.ConfigMap) (client.Object, error) { + return builder.BuildSts(reqCtx, task.GetBuilderParams(), envConfig.Name) + }); err != nil { + return err + } + } + + return nil } // TODO multi roles with same accessMode support @@ -221,35 +210,10 @@ func addLeaderSelectorLabels(service *corev1.Service, component *component.Synth } } -// buildConsensusSet build on a stateful set -func buildConsensusSet(reqCtx intctrlutil.RequestCtx, - task *intctrltypes.ReconcileTask, - envConfigName string) (*appsv1.StatefulSet, error) { - sts, err := builder.BuildSts(reqCtx, task.GetBuilderParams(), envConfigName) - if err != nil { - return sts, err - } - - sts.Spec.UpdateStrategy.Type = appsv1.OnDeleteStatefulSetStrategyType - return sts, err -} - -// buildReplicationSet builds a replication component on statefulSet. -func buildReplicationSet(reqCtx intctrlutil.RequestCtx, - task *intctrltypes.ReconcileTask, - envConfigName string) (*appsv1.StatefulSet, error) { - sts, err := builder.BuildSts(reqCtx, task.GetBuilderParams(), envConfigName) - if err != nil { - return nil, err - } - sts.Spec.UpdateStrategy.Type = appsv1.OnDeleteStatefulSetStrategyType - return sts, nil -} - -// buildCfg generate volumes for PodTemplate, volumeMount for container, rendered configTemplate and scriptTemplate, +// renderConfigNScriptFiles generate volumes for PodTemplate, volumeMount for container, rendered configTemplate and scriptTemplate, // and generate configManager sidecar for the reconfigure operation. // TODO rename this function, this function name is not very reasonable, but there is no suitable name. -func buildCfg(task *intctrltypes.ReconcileTask, +func renderConfigNScriptFiles(task *intctrltypes.ReconcileTask, obj client.Object, podSpec *corev1.PodSpec, ctx context.Context, diff --git a/internal/controller/plan/prepare_test.go b/internal/controller/plan/prepare_test.go index d344bcec5..1c7ff9644 100644 --- a/internal/controller/plan/prepare_test.go +++ b/internal/controller/plan/prepare_test.go @@ -20,6 +20,7 @@ along with this program. If not, see . package plan import ( + "fmt" "reflect" . "github.com/onsi/ginkgo/v2" @@ -116,11 +117,17 @@ var _ = Describe("Cluster Controller", func() { Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) resources := *task.Resources - Expect(len(resources)).Should(Equal(4)) - Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("Deployment")) - Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("Service")) + expects := []string{ + "PodDisruptionBudget", + "Service", + "ConfigMap", + "Service", + "Deployment", + } + Expect(resources).Should(HaveLen(len(expects))) + for i, v := range expects { + Expect(reflect.TypeOf(resources[i]).String()).Should(ContainSubstring(v), fmt.Sprintf("failed at idx %d", i)) + } }) }) @@ -159,14 +166,22 @@ var _ = Describe("Cluster Controller", func() { Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) resources := *task.Resources - Expect(len(resources)).Should(Equal(4)) - Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("StatefulSet")) - - container := clusterDef.Spec.ComponentDefs[0].PodSpec.Containers[0] - sts := resources[2].(*appsv1.StatefulSet) - Expect(len(sts.Spec.Template.Spec.Volumes)).Should(Equal(len(container.VolumeMounts))) + expects := []string{ + "PodDisruptionBudget", + "Service", + "ConfigMap", + "Service", + "StatefulSet", + } + Expect(resources).Should(HaveLen(len(expects))) + for i, v := range expects { + Expect(reflect.TypeOf(resources[i]).String()).Should(ContainSubstring(v), fmt.Sprintf("failed at idx %d", i)) + if v == "StatefulSet" { + container := clusterDef.Spec.ComponentDefs[0].PodSpec.Containers[0] + sts := resources[i].(*appsv1.StatefulSet) + Expect(len(sts.Spec.Template.Spec.Volumes)).Should(Equal(len(container.VolumeMounts))) + } + } }) }) @@ -211,11 +226,18 @@ var _ = Describe("Cluster Controller", func() { Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) resources := *task.Resources - Expect(len(resources)).Should(Equal(5)) - Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("StatefulSet")) + expects := []string{ + "PodDisruptionBudget", + "Service", + "ConfigMap", + "Service", + "ConfigMap", + "StatefulSet", + } + Expect(resources).Should(HaveLen(len(expects))) + for i, v := range expects { + Expect(reflect.TypeOf(resources[i]).String()).Should(ContainSubstring(v), fmt.Sprintf("failed at idx %d", i)) + } }) }) @@ -261,18 +283,25 @@ var _ = Describe("Cluster Controller", func() { Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) resources := *task.Resources - Expect(len(resources)).Should(Equal(5)) - Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("StatefulSet")) - + expects := []string{ + "PodDisruptionBudget", + "Service", + "ConfigMap", + "Service", + "ConfigMap", + "StatefulSet", + } + Expect(resources).Should(HaveLen(len(expects))) + for i, v := range expects { + Expect(reflect.TypeOf(resources[i]).String()).Should(ContainSubstring(v), fmt.Sprintf("failed at idx %d", i)) + if v == "StatefulSet" { + sts := resources[i].(*appsv1.StatefulSet) + podSpec := sts.Spec.Template.Spec + Expect(len(podSpec.Containers)).Should(Equal(2)) + } + } originPodSpec := clusterDef.Spec.ComponentDefs[0].PodSpec Expect(len(originPodSpec.Containers)).Should(Equal(1)) - - sts := resources[3].(*appsv1.StatefulSet) - podSpec := sts.Spec.Template.Spec - Expect(len(podSpec.Containers)).Should(Equal(2)) }) }) @@ -318,13 +347,18 @@ var _ = Describe("Cluster Controller", func() { Expect(err).Should(Succeed()) task := types.InitReconcileTask(clusterDef, clusterVersion, cluster, component) Expect(PrepareComponentResources(reqCtx, testCtx.Cli, task)).Should(Succeed()) - resources := *task.Resources - Expect(len(resources)).Should(Equal(4)) - Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) - Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("StatefulSet")) - Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("Service")) + expects := []string{ + "PodDisruptionBudget", + "Service", + "ConfigMap", + "Service", + "StatefulSet", + } + Expect(resources).Should(HaveLen(len(expects))) + for i, v := range expects { + Expect(reflect.TypeOf(resources[i]).String()).Should(ContainSubstring(v), fmt.Sprintf("failed at idx %d", i)) + } }) }) @@ -374,11 +408,12 @@ var _ = Describe("Cluster Controller", func() { resources := *task.Resources // REVIEW: (free6om) // missing connection credential, TLS secret objs check? - Expect(resources).Should(HaveLen(4)) - Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("ConfigMap")) + Expect(resources).Should(HaveLen(5)) + Expect(reflect.TypeOf(resources[0]).String()).Should(ContainSubstring("PodDisruptionBudget")) Expect(reflect.TypeOf(resources[1]).String()).Should(ContainSubstring("Service")) - Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("StatefulSet")) + Expect(reflect.TypeOf(resources[2]).String()).Should(ContainSubstring("ConfigMap")) Expect(reflect.TypeOf(resources[3]).String()).Should(ContainSubstring("Service")) + Expect(reflect.TypeOf(resources[4]).String()).Should(ContainSubstring("StatefulSet")) }) }) diff --git a/internal/testutil/apps/cluster_consensus_test_util.go b/internal/testutil/apps/cluster_consensus_test_util.go index a995f021f..1f935e33a 100644 --- a/internal/testutil/apps/cluster_consensus_test_util.go +++ b/internal/testutil/apps/cluster_consensus_test_util.go @@ -42,7 +42,7 @@ const ( // InitConsensusMysql initializes a cluster environment which only contains a component of ConsensusSet type for testing, // includes ClusterDefinition/ClusterVersion/Cluster resources. -func InitConsensusMysql(testCtx testutil.TestContext, +func InitConsensusMysql(testCtx *testutil.TestContext, clusterDefName, clusterVersionName, clusterName, @@ -56,7 +56,7 @@ func InitConsensusMysql(testCtx testutil.TestContext, // CreateConsensusMysqlCluster creates a mysql cluster with a component of ConsensusSet type. func CreateConsensusMysqlCluster( - testCtx testutil.TestContext, + testCtx *testutil.TestContext, clusterDefName, clusterVersionName, clusterName, @@ -65,35 +65,35 @@ func CreateConsensusMysqlCluster( pvcSpec := NewPVCSpec("2Gi") return NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). AddComponent(consensusCompName, workloadType).SetReplicas(3).SetEnabledLogs(errorLogName). - AddVolumeClaimTemplate("data", pvcSpec).Create(&testCtx).GetObject() + AddVolumeClaimTemplate("data", pvcSpec).Create(testCtx).GetObject() } // CreateConsensusMysqlClusterDef creates a mysql clusterDefinition with a component of ConsensusSet type. -func CreateConsensusMysqlClusterDef(testCtx testutil.TestContext, clusterDefName, componentDefName string) *appsv1alpha1.ClusterDefinition { +func CreateConsensusMysqlClusterDef(testCtx *testutil.TestContext, clusterDefName, componentDefName string) *appsv1alpha1.ClusterDefinition { filePathPattern := "/data/mysql/log/mysqld.err" return NewClusterDefFactory(clusterDefName).AddComponentDef(ConsensusMySQLComponent, componentDefName). - AddLogConfig(errorLogName, filePathPattern).Create(&testCtx).GetObject() + AddLogConfig(errorLogName, filePathPattern).Create(testCtx).GetObject() } // CreateConsensusMysqlClusterVersion creates a mysql clusterVersion with a component of ConsensusSet type. -func CreateConsensusMysqlClusterVersion(testCtx testutil.TestContext, clusterDefName, clusterVersionName, workloadType string) *appsv1alpha1.ClusterVersion { +func CreateConsensusMysqlClusterVersion(testCtx *testutil.TestContext, clusterDefName, clusterVersionName, workloadType string) *appsv1alpha1.ClusterVersion { return NewClusterVersionFactory(clusterVersionName, clusterDefName).AddComponentVersion(workloadType).AddContainerShort("mysql", ApeCloudMySQLImage). - Create(&testCtx).GetObject() + Create(testCtx).GetObject() } // MockConsensusComponentStatefulSet mocks the component statefulSet, just using in envTest func MockConsensusComponentStatefulSet( - testCtx testutil.TestContext, + testCtx *testutil.TestContext, clusterName, consensusCompName string) *appsv1.StatefulSet { stsName := clusterName + "-" + consensusCompName return NewStatefulSetFactory(testCtx.DefaultNamespace, stsName, clusterName, consensusCompName).SetReplicas(int32(3)). - AddContainer(corev1.Container{Name: DefaultMySQLContainerName, Image: ApeCloudMySQLImage}).Create(&testCtx).GetObject() + AddContainer(corev1.Container{Name: DefaultMySQLContainerName, Image: ApeCloudMySQLImage}).Create(testCtx).GetObject() } // MockConsensusComponentStsPod mocks to create the pod of the consensus StatefulSet, just using in envTest func MockConsensusComponentStsPod( - testCtx testutil.TestContext, + testCtx *testutil.TestContext, sts *appsv1.StatefulSet, clusterName, consensusCompName, @@ -112,7 +112,7 @@ func MockConsensusComponentStsPod( AddConsensusSetAccessModeLabel(accessMode). AddControllerRevisionHashLabel(stsUpdateRevision). AddContainer(corev1.Container{Name: DefaultMySQLContainerName, Image: ApeCloudMySQLImage}). - CheckedCreate(&testCtx).GetObject() + CheckedCreate(testCtx).GetObject() patch := client.MergeFrom(pod.DeepCopy()) pod.Status.Conditions = []corev1.PodCondition{ { @@ -126,7 +126,7 @@ func MockConsensusComponentStsPod( // MockConsensusComponentPods mocks the component pods, just using in envTest func MockConsensusComponentPods( - testCtx testutil.TestContext, + testCtx *testutil.TestContext, sts *appsv1.StatefulSet, clusterName, consensusCompName string) []*corev1.Pod { diff --git a/internal/testutil/apps/cluster_stateless_test_util.go b/internal/testutil/apps/cluster_stateless_test_util.go index 480497952..45f8c0fce 100644 --- a/internal/testutil/apps/cluster_stateless_test_util.go +++ b/internal/testutil/apps/cluster_stateless_test_util.go @@ -29,14 +29,14 @@ import ( ) // MockStatelessComponentDeploy mocks a deployment workload of the stateless component. -func MockStatelessComponentDeploy(testCtx testutil.TestContext, clusterName, componentName string) *appsv1.Deployment { +func MockStatelessComponentDeploy(testCtx *testutil.TestContext, clusterName, componentName string) *appsv1.Deployment { deployName := clusterName + "-" + componentName return NewDeploymentFactory(testCtx.DefaultNamespace, deployName, clusterName, componentName).SetMinReadySeconds(int32(10)).SetReplicas(int32(2)). - AddContainer(corev1.Container{Name: DefaultNginxContainerName, Image: NginxImage}).Create(&testCtx).GetObject() + AddContainer(corev1.Container{Name: DefaultNginxContainerName, Image: NginxImage}).Create(testCtx).GetObject() } // MockStatelessPod mocks the pods of the deployment workload. -func MockStatelessPod(testCtx testutil.TestContext, deploy *appsv1.Deployment, clusterName, componentName, podName string) *corev1.Pod { +func MockStatelessPod(testCtx *testutil.TestContext, deploy *appsv1.Deployment, clusterName, componentName, podName string) *corev1.Pod { var newRs *appsv1.ReplicaSet if deploy != nil { newRs = &appsv1.ReplicaSet{ @@ -52,5 +52,5 @@ func MockStatelessPod(testCtx testutil.TestContext, deploy *appsv1.Deployment, c AddAppComponentLabel(componentName). AddAppManangedByLabel(). AddContainer(corev1.Container{Name: DefaultNginxContainerName, Image: NginxImage}). - Create(&testCtx).GetObject() + Create(testCtx).GetObject() } diff --git a/internal/testutil/apps/cluster_util.go b/internal/testutil/apps/cluster_util.go index ac7df2f56..5a4d42c91 100644 --- a/internal/testutil/apps/cluster_util.go +++ b/internal/testutil/apps/cluster_util.go @@ -34,7 +34,7 @@ import ( // InitClusterWithHybridComps initializes a cluster environment for testing, includes ClusterDefinition/ClusterVersion/Cluster resources. func InitClusterWithHybridComps( - testCtx testutil.TestContext, + testCtx *testutil.TestContext, clusterDefName, clusterVersionName, clusterName, @@ -45,44 +45,44 @@ func InitClusterWithHybridComps( AddComponentDef(StatelessNginxComponent, statelessCompDefName). AddComponentDef(ConsensusMySQLComponent, consensusCompDefName). AddComponentDef(StatefulMySQLComponent, statefulCompDefName). - Create(&testCtx).GetObject() + Create(testCtx).GetObject() clusterVersion := NewClusterVersionFactory(clusterVersionName, clusterDefName). AddComponentVersion(statelessCompDefName).AddContainerShort(DefaultNginxContainerName, NginxImage). AddComponentVersion(consensusCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). AddComponentVersion(statefulCompDefName).AddContainerShort(DefaultMySQLContainerName, NginxImage). - Create(&testCtx).GetObject() + Create(testCtx).GetObject() pvcSpec := NewPVCSpec("1Gi") cluster := NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). AddComponent(statelessCompDefName, statelessCompDefName).SetReplicas(1). AddComponent(consensusCompDefName, consensusCompDefName).SetReplicas(3). AddComponent(statefulCompDefName, statefulCompDefName).SetReplicas(3). AddVolumeClaimTemplate(DataVolumeName, pvcSpec). - Create(&testCtx).GetObject() + Create(testCtx).GetObject() return clusterDef, clusterVersion, cluster } -func CreateK8sResource(testCtx testutil.TestContext, obj client.Object) client.Object { +func CreateK8sResource(testCtx *testutil.TestContext, obj client.Object) client.Object { gomega.Expect(testCtx.CreateObj(testCtx.Ctx, obj)).Should(gomega.Succeed()) // wait until cluster created - gomega.Eventually(CheckObjExists(&testCtx, client.ObjectKeyFromObject(obj), + gomega.Eventually(CheckObjExists(testCtx, client.ObjectKeyFromObject(obj), obj, true)).Should(gomega.Succeed()) return obj } -func CheckedCreateK8sResource(testCtx testutil.TestContext, obj client.Object) client.Object { +func CheckedCreateK8sResource(testCtx *testutil.TestContext, obj client.Object) client.Object { gomega.Expect(testCtx.CheckedCreateObj(testCtx.Ctx, obj)).Should(gomega.Succeed()) // wait until cluster created - gomega.Eventually(CheckObjExists(&testCtx, client.ObjectKeyFromObject(obj), + gomega.Eventually(CheckObjExists(testCtx, client.ObjectKeyFromObject(obj), obj, true)).Should(gomega.Succeed()) return obj } // GetClusterComponentPhase gets the component phase of testing cluster for verification. -func GetClusterComponentPhase(testCtx testutil.TestContext, clusterName, componentName string) func(g gomega.Gomega) appsv1alpha1.ClusterComponentPhase { +func GetClusterComponentPhase(testCtx *testutil.TestContext, clusterKey types.NamespacedName, componentName string) func(g gomega.Gomega) appsv1alpha1.ClusterComponentPhase { return func(g gomega.Gomega) appsv1alpha1.ClusterComponentPhase { tmpCluster := &appsv1alpha1.Cluster{} - g.Expect(testCtx.Cli.Get(context.Background(), client.ObjectKey{Name: clusterName, - Namespace: testCtx.DefaultNamespace}, tmpCluster)).Should(gomega.Succeed()) + g.Expect(testCtx.Cli.Get(context.Background(), client.ObjectKey{Name: clusterKey.Name, + Namespace: clusterKey.Namespace}, tmpCluster)).Should(gomega.Succeed()) return tmpCluster.Status.Components[componentName].Phase } } @@ -96,6 +96,15 @@ func GetClusterPhase(testCtx *testutil.TestContext, clusterKey types.NamespacedN } } +// GetClusterGeneration gets the testing cluster's metadata.generation. +func GetClusterGeneration(testCtx *testutil.TestContext, clusterKey types.NamespacedName) func(gomega.Gomega) int64 { + return func(g gomega.Gomega) int64 { + cluster := &appsv1alpha1.Cluster{} + g.Expect(testCtx.Cli.Get(testCtx.Ctx, clusterKey, cluster)).Should(gomega.Succeed()) + return cluster.GetGeneration() + } +} + // GetClusterObservedGeneration gets the testing cluster's ObservedGeneration in status for verification. func GetClusterObservedGeneration(testCtx *testutil.TestContext, clusterKey types.NamespacedName) func(gomega.Gomega) int64 { return func(g gomega.Gomega) int64 { diff --git a/internal/testutil/apps/clusterdef_factory.go b/internal/testutil/apps/clusterdef_factory.go index 0b47530ff..7ee0764b6 100644 --- a/internal/testutil/apps/clusterdef_factory.go +++ b/internal/testutil/apps/clusterdef_factory.go @@ -221,7 +221,7 @@ func (factory *MockClusterDefFactory) AddContainerVolumeMounts(containerName str return factory } -func (factory *MockClusterDefFactory) AddReplicationSpec(replicationSpec *appsv1alpha1.ReplicationSpec) *MockClusterDefFactory { +func (factory *MockClusterDefFactory) AddReplicationSpec(replicationSpec *appsv1alpha1.ReplicationSetSpec) *MockClusterDefFactory { comp := factory.getLastCompDef() if comp == nil { return factory diff --git a/internal/testutil/apps/common_util.go b/internal/testutil/apps/common_util.go index e9a4a457f..416d2004c 100644 --- a/internal/testutil/apps/common_util.go +++ b/internal/testutil/apps/common_util.go @@ -264,13 +264,13 @@ func NewCustomizedObj[T intctrlutil.Object, PT intctrlutil.PObject[T]]( func CreateCustomizedObj[T intctrlutil.Object, PT intctrlutil.PObject[T]](testCtx *testutil.TestContext, filePath string, pobj PT, actions ...any) PT { pobj = NewCustomizedObj(filePath, pobj, actions...) - return CreateK8sResource(*testCtx, pobj).(PT) + return CreateK8sResource(testCtx, pobj).(PT) } func CheckedCreateCustomizedObj[T intctrlutil.Object, PT intctrlutil.PObject[T]](testCtx *testutil.TestContext, filePath string, pobj PT, actions ...any) PT { pobj = NewCustomizedObj(filePath, pobj, actions...) - return CheckedCreateK8sResource(*testCtx, pobj).(PT) + return CheckedCreateK8sResource(testCtx, pobj).(PT) } // Helper functions to delete object. diff --git a/internal/testutil/apps/constant.go b/internal/testutil/apps/constant.go index 31656784e..25ad95760 100644 --- a/internal/testutil/apps/constant.go +++ b/internal/testutil/apps/constant.go @@ -183,7 +183,9 @@ var ( Name: "follower", AccessMode: appsv1alpha1.Readonly, }}, - UpdateStrategy: appsv1alpha1.BestEffortParallelStrategy, + StatefulSetSpec: appsv1alpha1.StatefulSetSpec{ + UpdateStrategy: appsv1alpha1.BestEffortParallelStrategy, + }, } defaultMySQLService = appsv1alpha1.ServiceSpec{ diff --git a/internal/testutil/apps/native_object_util.go b/internal/testutil/apps/native_object_util.go index 6224402ff..6afc84476 100644 --- a/internal/testutil/apps/native_object_util.go +++ b/internal/testutil/apps/native_object_util.go @@ -68,7 +68,7 @@ func NewPVC(size string) corev1.PersistentVolumeClaimSpec { } } -func CreateStorageClass(testCtx testutil.TestContext, storageClassName string, +func CreateStorageClass(testCtx *testutil.TestContext, storageClassName string, allowVolumeExpansion bool) *storagev1.StorageClass { storageClass := &storagev1.StorageClass{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/testutil/apps/opsrequest_util.go b/internal/testutil/apps/opsrequest_util.go index a5ae3cd44..29a460baa 100644 --- a/internal/testutil/apps/opsrequest_util.go +++ b/internal/testutil/apps/opsrequest_util.go @@ -32,7 +32,7 @@ import ( ) // CreateRestartOpsRequest creates a OpsRequest of restart type for testing. -func CreateRestartOpsRequest(testCtx testutil.TestContext, clusterName, opsRequestName string, componentNames []string) *appsv1alpha1.OpsRequest { +func CreateRestartOpsRequest(testCtx *testutil.TestContext, clusterName, opsRequestName string, componentNames []string) *appsv1alpha1.OpsRequest { opsRequest := NewOpsRequestObj(opsRequestName, testCtx.DefaultNamespace, clusterName, appsv1alpha1.RestartType) componentList := make([]appsv1alpha1.ComponentOps, len(componentNames)) for i := range componentNames { diff --git a/internal/testutil/k8s/deployment_util.go b/internal/testutil/k8s/deployment_util.go index 6041bd7fb..cadabf438 100644 --- a/internal/testutil/k8s/deployment_util.go +++ b/internal/testutil/k8s/deployment_util.go @@ -22,9 +22,15 @@ package testutil import ( "fmt" + "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/testutil" ) // MockDeploymentReady mocks deployment is ready @@ -54,3 +60,15 @@ func MockPodAvailable(pod *corev1.Pod, lastTransitionTime metav1.Time) { }, } } + +func ListAndCheckDeployment(testCtx *testutil.TestContext, key types.NamespacedName) *appsv1.DeploymentList { + deployList := &appsv1.DeploymentList{} + gomega.Eventually(func(g gomega.Gomega) { + g.Expect(testCtx.Cli.List(testCtx.Ctx, deployList, client.MatchingLabels{ + constant.AppInstanceLabelKey: key.Name, + }, client.InNamespace(key.Namespace))).Should(gomega.Succeed()) + g.Expect(deployList.Items).ShouldNot(gomega.BeNil()) + g.Expect(deployList.Items).ShouldNot(gomega.BeEmpty()) + }).Should(gomega.Succeed()) + return deployList +} diff --git a/internal/testutil/k8s/statefulset_util.go b/internal/testutil/k8s/statefulset_util.go index 6cdb97057..d853781e4 100644 --- a/internal/testutil/k8s/statefulset_util.go +++ b/internal/testutil/k8s/statefulset_util.go @@ -25,7 +25,7 @@ import ( "reflect" "github.com/onsi/gomega" - apps "k8s.io/api/apps/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,7 +42,7 @@ const ( ) // NewFakeStatefulSet creates a fake StatefulSet workload object for testing. -func NewFakeStatefulSet(name string, replicas int) *apps.StatefulSet { +func NewFakeStatefulSet(name string, replicas int) *appsv1.StatefulSet { template := corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -57,12 +57,12 @@ func NewFakeStatefulSet(name string, replicas int) *apps.StatefulSet { template.Labels = map[string]string{"foo": "bar"} statefulSetReplicas := int32(replicas) Revision := name + "-d5df5b8d6" - return &apps.StatefulSet{ + return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: corev1.NamespaceDefault, }, - Spec: apps.StatefulSetSpec{ + Spec: appsv1.StatefulSetSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{"foo": "bar"}, }, @@ -70,7 +70,7 @@ func NewFakeStatefulSet(name string, replicas int) *apps.StatefulSet { Template: template, ServiceName: "governingsvc", }, - Status: apps.StatefulSetStatus{ + Status: appsv1.StatefulSetStatus{ AvailableReplicas: statefulSetReplicas, ObservedGeneration: 0, ReadyReplicas: statefulSetReplicas, @@ -82,14 +82,14 @@ func NewFakeStatefulSet(name string, replicas int) *apps.StatefulSet { } // NewFakeStatefulSetPod creates a fake pod of the StatefulSet workload for testing. -func NewFakeStatefulSetPod(set *apps.StatefulSet, ordinal int) *corev1.Pod { +func NewFakeStatefulSetPod(set *appsv1.StatefulSet, ordinal int) *corev1.Pod { pod := &corev1.Pod{} pod.Name = fmt.Sprintf("%s-%d", set.Name, ordinal) return pod } // MockStatefulSetReady mocks the StatefulSet workload is ready. -func MockStatefulSetReady(sts *apps.StatefulSet) { +func MockStatefulSetReady(sts *appsv1.StatefulSet) { sts.Status.AvailableReplicas = *sts.Spec.Replicas sts.Status.ObservedGeneration = sts.Generation sts.Status.Replicas = *sts.Spec.Replicas @@ -154,8 +154,8 @@ func RemovePodFinalizer(ctx context.Context, testCtx testutil.TestContext, pod * }).Should(gomega.Satisfy(apierrors.IsNotFound)) } -func ListAndCheckStatefulSet(testCtx *testutil.TestContext, key types.NamespacedName) *apps.StatefulSetList { - stsList := &apps.StatefulSetList{} +func ListAndCheckStatefulSet(testCtx *testutil.TestContext, key types.NamespacedName) *appsv1.StatefulSetList { + stsList := &appsv1.StatefulSetList{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(testCtx.Cli.List(testCtx.Ctx, stsList, client.MatchingLabels{ constant.AppInstanceLabelKey: key.Name, @@ -166,8 +166,8 @@ func ListAndCheckStatefulSet(testCtx *testutil.TestContext, key types.Namespaced return stsList } -func ListAndCheckStatefulSetCount(testCtx *testutil.TestContext, key types.NamespacedName, cnt int) *apps.StatefulSetList { - stsList := &apps.StatefulSetList{} +func ListAndCheckStatefulSetItemsCount(testCtx *testutil.TestContext, key types.NamespacedName, cnt int) *appsv1.StatefulSetList { + stsList := &appsv1.StatefulSetList{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(testCtx.Cli.List(testCtx.Ctx, stsList, client.MatchingLabels{ constant.AppInstanceLabelKey: key.Name, @@ -177,8 +177,8 @@ func ListAndCheckStatefulSetCount(testCtx *testutil.TestContext, key types.Names return stsList } -func ListAndCheckStatefulSetWithComponent(testCtx *testutil.TestContext, key types.NamespacedName, componentName string) *apps.StatefulSetList { - stsList := &apps.StatefulSetList{} +func ListAndCheckStatefulSetWithComponent(testCtx *testutil.TestContext, key types.NamespacedName, componentName string) *appsv1.StatefulSetList { + stsList := &appsv1.StatefulSetList{} gomega.Eventually(func(g gomega.Gomega) { g.Expect(testCtx.Cli.List(testCtx.Ctx, stsList, client.MatchingLabels{ constant.AppInstanceLabelKey: key.Name, @@ -202,17 +202,17 @@ func ListAndCheckPodCountWithComponent(testCtx *testutil.TestContext, key types. return podList } -func PatchStatefulSetStatus(testCtx *testutil.TestContext, stsName string, status apps.StatefulSetStatus) { +func PatchStatefulSetStatus(testCtx *testutil.TestContext, stsName string, status appsv1.StatefulSetStatus) { objectKey := client.ObjectKey{Name: stsName, Namespace: testCtx.DefaultNamespace} - gomega.Expect(testapps.GetAndChangeObjStatus(testCtx, objectKey, func(newSts *apps.StatefulSet) { + gomega.Expect(testapps.GetAndChangeObjStatus(testCtx, objectKey, func(newSts *appsv1.StatefulSet) { newSts.Status = status })()).Should(gomega.Succeed()) - gomega.Eventually(testapps.CheckObj(testCtx, objectKey, func(g gomega.Gomega, newSts *apps.StatefulSet) { + gomega.Eventually(testapps.CheckObj(testCtx, objectKey, func(g gomega.Gomega, newSts *appsv1.StatefulSet) { g.Expect(reflect.DeepEqual(newSts.Status, status)).Should(gomega.BeTrue()) })).Should(gomega.Succeed()) } -func InitStatefulSetStatus(testCtx testutil.TestContext, statefulset *apps.StatefulSet, controllerRevision string) { +func InitStatefulSetStatus(testCtx testutil.TestContext, statefulset *appsv1.StatefulSet, controllerRevision string) { gomega.Expect(testapps.ChangeObjStatus(&testCtx, statefulset, func() { statefulset.Status.UpdateRevision = controllerRevision statefulset.Status.CurrentRevision = controllerRevision From eaaaf6b36c7059b1e1823b315b47c6f25ecb0f2b Mon Sep 17 00:00:00 2001 From: yijing <107018199+ahjing99@users.noreply.github.com> Date: Thu, 18 May 2023 11:45:30 +0800 Subject: [PATCH 319/439] Update milestoneclose.yml --- .github/workflows/milestoneclose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/milestoneclose.yml b/.github/workflows/milestoneclose.yml index 627d5b8df..67bff0a77 100644 --- a/.github/workflows/milestoneclose.yml +++ b/.github/workflows/milestoneclose.yml @@ -7,7 +7,7 @@ on: env: GITHUB_TOKEN: ${{ secrets.KUBEBLOCKS_TOKEN }} REPO: kubeblocks - milestone: 4 + milestone: 5 ORGANIZATION: apecloud PROJECT_NUMBER: 2 From 6e35253584d13e575a2310e6a2741e23c7b194a9 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Thu, 18 May 2023 13:52:33 +0800 Subject: [PATCH 320/439] fix: helm resource name conflict (#3299) --- .../templates/_helpers.tpl | 6 ++++- .../templates/cluster.yaml | 2 +- .../templates/role.yaml | 2 +- .../templates/rolebinding.yaml | 4 +-- .../templates/_helpers.tpl | 6 ++++- .../templates/cluster.yaml | 2 +- .../templates/role.yaml | 2 +- .../templates/rolebinding.yaml | 4 +-- .../clickhouse-cluster/templates/_helpers.tpl | 6 ++++- .../clickhouse-cluster/templates/cluster.yaml | 2 +- deploy/clickhouse-cluster/templates/role.yaml | 2 +- .../templates/rolebinding.yaml | 4 +-- .../templates/tests/test-connection.yaml | 4 +-- deploy/delphic/Chart.lock | 12 ++++----- deploy/delphic/Chart.yaml | 23 ++++-------------- deploy/delphic/README.md | 2 +- .../charts/pgcluster-0.5.0-beta.23.tgz | Bin 2804 -> 0 bytes .../charts/pgcluster-0.5.0-beta.24.tgz | Bin 0 -> 2826 bytes .../charts/redis-cluster-0.5.0-beta.23.tgz | Bin 2333 -> 0 bytes .../charts/redis-cluster-0.5.0-beta.24.tgz | Bin 0 -> 2669 bytes deploy/etcd-cluster/templates/_helpers.tpl | 6 ++++- deploy/etcd-cluster/templates/cluster.yaml | 2 +- deploy/etcd-cluster/templates/role.yaml | 2 +- .../etcd-cluster/templates/rolebinding.yaml | 4 +-- .../templates/tests/test-connection.yaml | 4 +-- deploy/kafka-cluster/templates/_helpers.tpl | 6 ++++- deploy/kafka-cluster/templates/cluster.yaml | 2 +- deploy/kafka-cluster/templates/role.yaml | 2 +- .../kafka-cluster/templates/rolebinding.yaml | 4 +-- .../templates/tests/test-connection.yaml | 4 +-- deploy/milvus-cluster/templates/_helpers.tpl | 10 ++++---- deploy/milvus-cluster/templates/cluster.yaml | 2 +- deploy/milvus-cluster/values.yaml | 3 +++ deploy/mongodb-cluster/templates/_helpers.tpl | 6 ++++- deploy/mongodb-cluster/templates/cluster.yaml | 2 +- deploy/mongodb-cluster/templates/role.yaml | 2 +- .../templates/rolebinding.yaml | 4 +-- .../opensearch-cluster/templates/_helpers.tpl | 10 ++++---- .../opensearch-cluster/templates/cluster.yaml | 2 +- deploy/opensearch-cluster/values.yaml | 5 +++- .../postgresql-cluster/templates/_helpers.tpl | 10 ++++---- .../postgresql-cluster/templates/cluster.yaml | 2 +- deploy/postgresql-cluster/templates/role.yaml | 2 +- .../templates/rolebinding.yaml | 4 +-- deploy/qdrant-cluster/templates/_helpers.tpl | 10 ++++---- deploy/qdrant-cluster/templates/cluster.yaml | 2 +- deploy/qdrant-cluster/values.yaml | 3 +++ deploy/redis-cluster/templates/_helpers.tpl | 6 ++++- deploy/redis-cluster/templates/cluster.yaml | 2 +- deploy/redis-cluster/templates/role.yaml | 2 +- .../redis-cluster/templates/rolebinding.yaml | 4 +-- .../weaviate-cluster/templates/_helpers.tpl | 10 ++++---- .../weaviate-cluster/templates/cluster.yaml | 2 +- deploy/weaviate-cluster/values.yaml | 4 +++ 54 files changed, 128 insertions(+), 100 deletions(-) delete mode 100644 deploy/delphic/charts/pgcluster-0.5.0-beta.23.tgz create mode 100644 deploy/delphic/charts/pgcluster-0.5.0-beta.24.tgz delete mode 100644 deploy/delphic/charts/redis-cluster-0.5.0-beta.23.tgz create mode 100644 deploy/delphic/charts/redis-cluster-0.5.0-beta.24.tgz diff --git a/deploy/apecloud-mysql-cluster/templates/_helpers.tpl b/deploy/apecloud-mysql-cluster/templates/_helpers.tpl index 6b1d05a2c..0a1088741 100644 --- a/deploy/apecloud-mysql-cluster/templates/_helpers.tpl +++ b/deploy/apecloud-mysql-cluster/templates/_helpers.tpl @@ -50,9 +50,13 @@ app.kubernetes.io/name: {{ include "apecloud-mysql-cluster.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "clustername" -}} +{{ include "apecloud-mysql-cluster.fullname" .}} +{{- end}} + {{/* Create the name of the service account to use */}} {{- define "apecloud-mysql-cluster.serviceAccountName" -}} -{{- default ((printf "kb-sa-%s" .Release.Name) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} +{{- default (printf "kb-%s" (include "clustername" .)) .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/apecloud-mysql-cluster/templates/cluster.yaml b/deploy/apecloud-mysql-cluster/templates/cluster.yaml index 75675a0cd..d02f155af 100644 --- a/deploy/apecloud-mysql-cluster/templates/cluster.yaml +++ b/deploy/apecloud-mysql-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ include "apecloud-mysql-cluster.fullname" . }} + name: {{ include "clustername" . }} labels: {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: apecloud-mysql # ref clusterdefinition.name diff --git a/deploy/apecloud-mysql-cluster/templates/role.yaml b/deploy/apecloud-mysql-cluster/templates/role.yaml index 00aacf4cf..39f74a5f5 100644 --- a/deploy/apecloud-mysql-cluster/templates/role.yaml +++ b/deploy/apecloud-mysql-cluster/templates/role.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: kb-role-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} namespace: {{ .Release.Namespace }} labels: {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} diff --git a/deploy/apecloud-mysql-cluster/templates/rolebinding.yaml b/deploy/apecloud-mysql-cluster/templates/rolebinding.yaml index 9ac4d7588..ec145c939 100644 --- a/deploy/apecloud-mysql-cluster/templates/rolebinding.yaml +++ b/deploy/apecloud-mysql-cluster/templates/rolebinding.yaml @@ -1,13 +1,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: kb-rolebinding-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} labels: {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: kb-role-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} subjects: - kind: ServiceAccount name: {{ include "apecloud-mysql-cluster.serviceAccountName" . }} diff --git a/deploy/apecloud-mysql-scale-cluster/templates/_helpers.tpl b/deploy/apecloud-mysql-scale-cluster/templates/_helpers.tpl index 6b1d05a2c..0a1088741 100644 --- a/deploy/apecloud-mysql-scale-cluster/templates/_helpers.tpl +++ b/deploy/apecloud-mysql-scale-cluster/templates/_helpers.tpl @@ -50,9 +50,13 @@ app.kubernetes.io/name: {{ include "apecloud-mysql-cluster.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "clustername" -}} +{{ include "apecloud-mysql-cluster.fullname" .}} +{{- end}} + {{/* Create the name of the service account to use */}} {{- define "apecloud-mysql-cluster.serviceAccountName" -}} -{{- default ((printf "kb-sa-%s" .Release.Name) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} +{{- default (printf "kb-%s" (include "clustername" .)) .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml b/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml index 5062afe70..57cd9d841 100644 --- a/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml +++ b/deploy/apecloud-mysql-scale-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ include "apecloud-mysql-cluster.fullname" . }} + name: {{ include "clustername" . }} labels: {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: apecloud-mysql-scale # ref clusterdefinition.name diff --git a/deploy/apecloud-mysql-scale-cluster/templates/role.yaml b/deploy/apecloud-mysql-scale-cluster/templates/role.yaml index 00aacf4cf..39f74a5f5 100644 --- a/deploy/apecloud-mysql-scale-cluster/templates/role.yaml +++ b/deploy/apecloud-mysql-scale-cluster/templates/role.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: kb-role-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} namespace: {{ .Release.Namespace }} labels: {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} diff --git a/deploy/apecloud-mysql-scale-cluster/templates/rolebinding.yaml b/deploy/apecloud-mysql-scale-cluster/templates/rolebinding.yaml index 9ac4d7588..ec145c939 100644 --- a/deploy/apecloud-mysql-scale-cluster/templates/rolebinding.yaml +++ b/deploy/apecloud-mysql-scale-cluster/templates/rolebinding.yaml @@ -1,13 +1,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: kb-rolebinding-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} labels: {{ include "apecloud-mysql-cluster.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: kb-role-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} subjects: - kind: ServiceAccount name: {{ include "apecloud-mysql-cluster.serviceAccountName" . }} diff --git a/deploy/clickhouse-cluster/templates/_helpers.tpl b/deploy/clickhouse-cluster/templates/_helpers.tpl index 1c942287e..dacc31da8 100644 --- a/deploy/clickhouse-cluster/templates/_helpers.tpl +++ b/deploy/clickhouse-cluster/templates/_helpers.tpl @@ -50,9 +50,13 @@ app.kubernetes.io/name: {{ include "clickhouse-cluster.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "clustername" -}} +{{ include "clickhouse-cluster.fullname" .}} +{{- end}} + {{/* Create the name of the service account to use */}} {{- define "clickhouse-cluster.serviceAccountName" -}} -{{- default (( printf "kb-sa-%s" .Release.Name ) | trunc 63 | trimSuffix "-") .Values.serviceAccount.name }} +{{- default (printf "kb-%s" (include "clustername" .)) .Values.serviceAccount.name }} {{- end }} diff --git a/deploy/clickhouse-cluster/templates/cluster.yaml b/deploy/clickhouse-cluster/templates/cluster.yaml index d8354be02..8b74a0ca6 100644 --- a/deploy/clickhouse-cluster/templates/cluster.yaml +++ b/deploy/clickhouse-cluster/templates/cluster.yaml @@ -1,7 +1,7 @@ apiVersion: apps.kubeblocks.io/v1alpha1 kind: Cluster metadata: - name: {{ include "clickhouse-cluster.fullname" . }} + name: {{ include "clustername" . }} labels: {{ include "clickhouse-cluster.labels" . | nindent 4 }} spec: clusterDefinitionRef: clickhouse # ref clusterdefinition.name diff --git a/deploy/clickhouse-cluster/templates/role.yaml b/deploy/clickhouse-cluster/templates/role.yaml index 7dae7749b..a8e7080b8 100644 --- a/deploy/clickhouse-cluster/templates/role.yaml +++ b/deploy/clickhouse-cluster/templates/role.yaml @@ -1,7 +1,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: kb-role-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} namespace: {{ .Release.Namespace }} labels: {{ include "clickhouse-cluster.labels" . | nindent 4 }} diff --git a/deploy/clickhouse-cluster/templates/rolebinding.yaml b/deploy/clickhouse-cluster/templates/rolebinding.yaml index 314981c2c..86036ac66 100644 --- a/deploy/clickhouse-cluster/templates/rolebinding.yaml +++ b/deploy/clickhouse-cluster/templates/rolebinding.yaml @@ -1,13 +1,13 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: kb-rolebinding-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} labels: {{ include "clickhouse-cluster.labels" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: kb-role-{{ .Release.Name }} + name: kb-{{ include "clustername" . }} subjects: - kind: ServiceAccount name: {{ include "clickhouse-cluster.serviceAccountName" . }} diff --git a/deploy/clickhouse-cluster/templates/tests/test-connection.yaml b/deploy/clickhouse-cluster/templates/tests/test-connection.yaml index c437bd44f..c08d4e47e 100644 --- a/deploy/clickhouse-cluster/templates/tests/test-connection.yaml +++ b/deploy/clickhouse-cluster/templates/tests/test-connection.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Pod metadata: - name: "{{ include "clickhouse-cluster.fullname" . }}-test-connection" + name: "{{ include "clustername" . }}-test-connection" labels: {{- include "clickhouse-cluster.labels" . | nindent 4 }} annotations: @@ -11,5 +11,5 @@ spec: - name: wget image: busybox command: ['wget'] - args: ['{{ include "clickhouse-cluster.fullname" . }}:{{ .Values.service.port }}'] + args: ['{{ include "clustername" . }}:{{ .Values.service.port }}'] restartPolicy: Never diff --git a/deploy/delphic/Chart.lock b/deploy/delphic/Chart.lock index 86d4ab306..da225dbcc 100644 --- a/deploy/delphic/Chart.lock +++ b/deploy/delphic/Chart.lock @@ -1,9 +1,9 @@ dependencies: - name: pgcluster - repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable - version: 0.5.0-beta.23 + repository: file://../postgresql-cluster + version: 0.5.0-beta.24 - name: redis-cluster - repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable - version: 0.5.0-beta.23 -digest: sha256:2455b2fe7ebbe34b584a623f17b9e57aaa9ef03160f4dee5ae1b8bf4efa04129 -generated: "2023-05-09T22:46:49.543381+08:00" + repository: file://../redis-cluster + version: 0.5.0-beta.24 +digest: sha256:d0aefa69ab29206d5ba039b8cac7729b518023e66cd5965933ad20ced95f8477 +generated: "2023-05-17T16:57:07.087916+08:00" diff --git a/deploy/delphic/Chart.yaml b/deploy/delphic/Chart.yaml index 128a4c5ad..4c2b53f88 100644 --- a/deploy/delphic/Chart.yaml +++ b/deploy/delphic/Chart.yaml @@ -1,33 +1,20 @@ apiVersion: v2 name: delphic -description: A Helm chart for Kubernetes +description: A simple framework to use LlamaIndex to build and deploy LLM agents that can be used to analyze and manipulate text data from documents. -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. type: application -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) version: 0.1.0 -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. appVersion: "1.16.0" dependencies: - name: pgcluster - repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable +# repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable + repository: file://../postgresql-cluster version: ~0.5.0-0 - name: redis-cluster - repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable +# repository: https://jihulab.com/api/v4/projects/85949/packages/helm/stable + repository: file://../redis-cluster version: ~0.5.0-0 \ No newline at end of file diff --git a/deploy/delphic/README.md b/deploy/delphic/README.md index 89dc9e66e..b8784bd55 100644 --- a/deploy/delphic/README.md +++ b/deploy/delphic/README.md @@ -45,6 +45,6 @@ 6. Port-forward the Plugin Portal to access it. ```shell - kubectl port-forward port-forward deployment/delphic 3000:3000 8000:8000 + kubectl port-forward deployment/delphic 3000:3000 8000:8000 ``` 7. In your web browser, open the plugin portal with the address ```http://127.0.0.1:3000``` \ No newline at end of file diff --git a/deploy/delphic/charts/pgcluster-0.5.0-beta.23.tgz b/deploy/delphic/charts/pgcluster-0.5.0-beta.23.tgz deleted file mode 100644 index cc6d056f0ead1746d2cf0dbb6a2e7026e179126e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2804 zcmVDc zVQyr3R8em|NM&qo0PH(kbK5qvdB(5UBe#>wv1mz_>@%9_Lt=N@oaXk$opk!tfykwV z8Uz>sl%qKQe)~7L@M_v}8rR9!F&-=uSS)rIdj;rx8S_FLRC`A&s!X(@8GmrgZ8RE< z4yIH0I~t9;zoY5s@WFUGog7RjlhNV+gVA_@GCFtwquVB=uNB%*^jgYIV`c?!s8Lcd`1$E$2-LvO$TNto zwn8cu{8B9NnJ0k=n&BKOohD|J;~Z$7a~3-$$+{$96wRX1aDj$KlYR2Fx8bX`+Wyy+ z7pVUZ2e8HdC)2}e*Z!xo$-VvGMcaX6OliRl`2BY#b)6YTWCwm-F%3)uh2NjPd@)R= z%BV4zK*~5;fMbj~RS0XU7+rAGU?eOMa;h~Zz=V;ok%a<-S|KAS72TqiXY9B=}(tBSQ2UGw~{*AOPMe zypl2}B$I*}scbh2OKCEPlyZ%P>|F1S1Y)AWtPo(t|xxn4+>HN^*1-x9c7k%;5+<8r7P`NaNT_#C1L19Ztw|)DQPJY0F|81uTjPjJCFk;tiKWBNnXn)<`Lob& zJhKZ3j8SWyRlb=}N@O^>013$hDKmLgU$*ptH21Sg0`jSvXZ9jlEQqd(IajNwzs~5hPeMEnd!Yt1+Rj10phsx&f_PE@5ek%6g-K zWw!-a`Y(&OT*j=FES4+i&kuZx6On&};oef~v;RL<$gTH`Onxh+z!v-8pB|1n_CGrq zP4DghF53C|-V^fUhumc^Wv(PsAHv;<5P7n9aY4?{ht^9lfiTFuAJFf4(4EN68o;oI zbGcFEkCbdoh3K6xFD+zckDqKX7BL*`yGNFt6luynz+gClfXk*{7Z-#$I*%&X5LQSF zo20!jC}%0d#CjITK}3E-kK4f;yMpCL!vbS!^G+=@bVmnDph4~VP8sqIWFcu+T#N~HRbJ}LvVvnU<+p& zBMkm`0E0gUpKn$&%cStM>YLE&Aw97+TZr^(&(S}J^K)Co@FKa6$;gv&9i&E%v61RU zSSo60R1m<^DvhvXG|!{c%AO-8_o}15^Yfl5pWuBV4R!=_8B0b5U1Bm^Y&@b;yH0#X z+AGkLK>TNjw9D_Xml$8RUN1TCB%*$9n9#-+mTg{|rr4+P-M|p4aR^5fp_acHo6S1@ z)m8qi|E!{G@TZHRrad-)o&NVZkSkUdm!(oae(kt@TdgnuD>v)5j)J%4f3sP){+k`_ z-`9V4(wbK^G|zR^zR_8aDbH7QOir0d=5Q45RWdZRJV2>@(j|8S!T)XZSiJJ_pFl8? z*phE*5!ejNE)f8aty^J+N%0obIn;>U(NJ1~B^8)6;d*{xv@>~le%?)M8eyXJ4Odi@ z7Z>fdL(C}`?m5D?f(ke%cwz1=K{nxe+|+=)C7y{XoaA80@|F>=yw|N|Jo&1ZG~tlaYKUT zxp_m<<>_!mW3BxDptuHx#?f)v+;OlaFV2f*epf@32gDZ0Klj14RJcn>c?R7aZJta2 z?ZotUG;*iLxpy;h@~~^kiwuuAW!bM~ysxP)P1Cyeo7h+=?NI5+BYtUBw`F@y|AHHD z^mLEhTMFG^z)})*q1$2vn}(v+FJ*$dy^Q7PEz;yS#SFd{F}6l~^zG{HY|*`iT=o%r zx9eLxckIjms$0ohT4PK8H=fNV-TZHS|NYP1v>O**!{M;?J&#(@IHHAFNyR?8+#ovr zzmPMAZ1Jt+*pn)rE{0aahTW7==hW5Hmj=8si4x>;vRTfoGtQ9a{H3f@6&%A&AI;fM zN)~zL3WhKkkY498gjkA{Ei;;H*R@8qXpXv4x_Vursh&C0^-E4oysF-cJZa84YYgr{ z;=YDx-fV*ueB*AHALjmUCN@0W%08^H_<5htAc`cHOqgE&nhagbLq`xT=D#j~xlFdQ z_PKlYt)CcN<^P+XqI~HZ+wOl4XXB3lKbTI(_x}Gb+Kv7HU-13m`=6^rKv1&JjRd#h z<0VI1RN8VoxQJ`mA^?qu_C;|_7mGhJHX*)vtS2p@JGQD7G`&IyHw%bc-A|wW_k4Kv zMLM?i`;T$={`YV^ySM*4X@AN7D=Y74Zm*$Vslm5k#@~8${q|d*{jVu!HkG-p8gQHa zPY(CH_21$C?B4$Gq6Pb>B7sNl8`mwK;PLff?#}FSea;LiWgG@qAZvf7+lQ)=`$yf& z>#r@ke?+V8KUyKr*iuM^x4gzS`=9N1@Bj9v<9qwRlePnIs4=L7`_A8QKb);lz@lK> zf4)!SQ@TW5{pW`+@?0vTL9dW=SaP|5%vQThEO$X6r-rT3R>!MGKNSht0f9@GS;M1T zVah&W;?LFp=WztDZE-J!n`udc9F_a-JR;HY$sZ?1DkM7)K392nbOH%ennX)x_S|1j zK8Y3|)t>uXcC40r_D}h$*J7_mw4m{6k-Pt0(d0>_&vNo4TF_JSBr=(OmWnOOlYf&P zct;hJg@)(HKWY-?O4?kWL@YtN=fRcylSFGB%LMoCJ=A@>Z};t6ZvP1Y0RR6gr_#Lu GJ^%nqGK#AJ diff --git a/deploy/delphic/charts/pgcluster-0.5.0-beta.24.tgz b/deploy/delphic/charts/pgcluster-0.5.0-beta.24.tgz new file mode 100644 index 0000000000000000000000000000000000000000..3449325ed9dcee89ed86cf7e3a31fd8e4bc5816f GIT binary patch literal 2826 zcmV+l3-$CLiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH(kbK5qvdFHR!Be#<{$ATqWw$Es$4~gArbDG;1chc!o2O^ge zY7k%mP>$mG`|aQ0!mDXZP93LT$9S+v5LoOk_6|6o$Gp%6)$Z|vDnnK@zh1(Vb7mIsmKr4mgP)&mL%;@pMxH@z zr4>@C;Fn^C&pir}pczh~lIh5-a-0IqbIxK%WV9?17?A@q8qUyAvNwso^)`I7R_p(g z@&fhOFaT@xfA3(j-_`%gWb~l__tCcC1XEgY1AhOVO5LP}B-(;s7fb`wK;idiuU-yQ zsWNH|CXh0YcEAb7oGOGRRgBI!YA_OJ2szan6JWwfSjj?x!7S(0peB)1-Zx8BiX}LO z!5~Tt&aZSv3Y|6|%R-nb>_rAu#)Q-Ex017XHH8!8XfTS}_NxshS1hAy^+F{0aSEd- z(&x;?ivS`5@V4W%lz9|oQZOTx?M7h)ejE;6-XV z?AEs+XklbV4T~w~t8yue#17;fsYXDd3xy@PX8~Lw_Xlgx?asb23smP!cKxO9Miv*dd$6~;2lP$VFvu{i4-W0tNU3CdMX zcT^h*9O=?>%4(5Sq*w|aGg)W{I$lT?qtnHb%fxFPN%_{H|jrmqAVn#aA#>)bS>@{{3;y>&GMtP%~y+fU*Su zab8T}U^L3Ydxn`*t10aL!~%sM3e=tP>Eh3{pb&_RmA%$IWR79QHo5SA_9+Qi@3YX4uE;@#J8e*N*3n2+X;)>s$+?;jp? z{Qu-=Gykd+rdR0U!OD1YrnKnbnZS_Fxr~a5N=x=VEbH-33VPoqCL{} zRMm0`OH)+V%mgSqt#PGcvT(~~%tFaxzJPwm??W6C{k=8vep{dZ|FJ-B%_1@RoumS5 z^nZVH)Q$fS4o461|Gl(}i`}Quj~{avmXtw|OucV*6`|W|)?;crpTBIra2!r7O0xTO-U0z0! z!}GW@4PgbFA*|p-K{-nqCf2ez3P|)Ddf0Bfu`5_=G|VujHlWlpLsvRb0u5?Ucgm0_ z8f@m`A`Wk8T#)u;<}J<1>r*?{1+_WG+a!9Cf3I9B}dK*&4KEI+rzv(}Lhu_ba%Wj>&Mg z@_D7GCm)@nRDJqkbY{I(8gwrZCaYDSQApC8$dVB&rKNA zri#=+yIRzBHX(I~P{#;~|7NUfCjGB_UD1iw4VUb?DI?57(o6`Gv46oVL7~E@f!<#SFgZP`O`r^q|tFb^hph_ zf*s2&mjZ=3%`w)Vpka`m*7g+|)zBOJ#EojqJ;8Lt{jTTeatCZH>-bIHhrY7X4WH}m zf`%y5PYwqUx;49b8qt`#S5Ld@p4MqWpF<-x!1l>S1DEE91}>upE}b(D-~35O15k4q zv0p0y%_3)WMx)U&#b@dl^g7 zJEX~PiWz((Vr+fwk+blE5D-LAh+1+p*xtBxD*XpJ@T-}vxg+>QT65BGoW zr*%w5|LLz4+2L^5y3eC#G$yn#3#r&A7ZH%N{|hl=h!)>Tjy<8`*=)EjTC^`Zr+s;< z>u$xAA&~i^xKb4y!*LzW*-uIqd8O<_7!0Cb^Du;1ij>VWnrqj!L^W%Ebt!K3I!9AI zbEfN;oSJx1y%l-VoORy#xCM#3($GAf1}XT;9T7jy{W(l*csN>pTtV^kZciYJB$rH> zUiz8@-AF@+5iR1sEq%F6){=Jny}B)vmv{|_1LGPl<16X^b@sn`$K9F*d=8cnGI0 zE;3`3SvcDu_Ywv$d4m>j`j{8^#49u(0?itc;YUvuCWB$HxF~C zWZQLh8B)qP46XszzJJ>Xs*w+n?&0;fX5Bxc)%s5s$TKzFz)Z=)A)?eQCI)@p^H41%4pCF%NUo#}{UgE>q`Q8&l7B{IsbiVo c?t_JTXbDc zVQyr3R8em|NM&qo0PH$#Z`(N1{mfr6kG+fC&9kN)+jRf~2V5^HdcEz-BI&hQ91aCd zjchg)sgjfvH_iR_14&7i9D9>ugBG_4KiD!k@0=M9M>d!WJK(ak4oz~rWX91o%jKhc z9K+#oIGs$w_i#Ae{vJ*b$BzytlhJfC8V!#okA{ciqv`k&4DVBjeWSFFnMcDv%xhb@ z|B=B8@fM90TFqcJBCIHypGR~^rzAzojVN50dZOUeI}Ds+Aq8iC7LKFyz~52kz`X}S zrVad3&hc+j^K(ndQs+2>rE`UyB}wo(Ds(~w_f+foxp0c*f^wZF%vy9dv5xU`8!-}* zFXGc6Ob>K!70wNSR}EW|0g)Y5P%zrABU&Goc(43QfRL z%**(sz+L;Qb@(T5YvSScOEy$_+= zaSBTM@MbA22n!5;d-C$dAk!vi&S46fkmvzUkV|F|R?G-CmuSIhm?IQEj#CiIX;|yh zfWy3y%%P=(RKa+xOXX%TBE+I{LZLkEEQ|?pXmX)K^!%!&;OiNjphSm+R8m%1>-7qa z5h>1~*CQm?N;qw11OSy6E1f~cq(wpsBXVZe&sB<-GZ+$LFNEXEiaG&^Vf#w!!V7A3 zX*gQH(1&*jr7fA1YcR-lo})@ZY3Fse1#*$CA+4<1paN5=2aw2C2+kXnd4qDTEEl@8 zfiz!g!7*%tTY5HVXtX|)R)bL7AhT$Fo>e@48)w0k3bX=LD`B+CQ8^12!Yx4x=OhN& zjVM2ZE&0;3e<_skS&YK-%6z{?jgtAR{ssBM_c*-@#JXukpI8NIHd3lV?4J?6!_=q3z)BKhw;l6!3RH6zQ+1C zwaUY||FJLr-!D=6bV*(DhfBd*;(t7uj<(`|I)1qSeVuW2l^l?#mqn;fmKbh9ptE>H zhi(8lNUpER)z!e)G(sWty5*tA=)qtE6Y3rsTM7Q~1o}2!D?DUfpuuN<>M9P?ad;H@ zS(#Akd?jw`&cq5GJ&a2UqO0G-eII2;GO5-HL);9 z&Pt15b&lbB8c*D)b0$!x7JLyK>JALmTBy^0oUemDPF|gPSf3Dmar6@<&oeM6eMJ_D za2;-xRaqA2go6tqrLTrdUkit-&kJ8kdy9EJWY@$ZYb;tXHdd`b_O9Q>>e4oJz6%=6 z{&M$i-N@M3A`+pe(NGb$ma{YYg~Am^D3?L+dpr2v_O@t6rgwL;-H#pDYwf9zmg*Mo z_cQEk>+ zih?dt=F~2eAo~oiuKcA$mgzp_Xk@<+%A!Q>w0Tjnpe+xolz!6mr2HBy3firfn|8|8 z)ee-;@Uhemw~TO;O3oBpU^i^+iQ z)bVx0TL}bAJW*iLkVIyLtiZNkn?JFK2bZD32m+9UnjLQGSyR z=}-G8sczY9YM}RV>o{^f(JEb9{o}@&8rYSVWsb*^iTq7{jBbRk zL%?Uvu2-i!yxP+0_JqdLNcg39pN^}C?XR-rW-!jt+Lt;-+gSx% zgPFL*G-lEN`-#G9CD));!1N+PfhN3Hri7lHy+3o>An8NZx_Wzj2B|QX(1mbG_>AHc zI{##n@L5kR7m5F-DIiN20&)SFuIrtCh!&_#A zE-gGid1?tQjQ)V!5h_w-Nd!0g1EDL+b&AQKtX1$Z9>&A?la2od00960NM5sn04x9i DQ$Cj3 diff --git a/deploy/delphic/charts/redis-cluster-0.5.0-beta.24.tgz b/deploy/delphic/charts/redis-cluster-0.5.0-beta.24.tgz new file mode 100644 index 0000000000000000000000000000000000000000..df2c0c6f01bf7e858b30d017844a8334c1288ed8 GIT binary patch literal 2669 zcmV-z3X=67iwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PI`)ZyUMs&(HiT<}2Ufl6)iTtrP+ZG-n^u^zt~^*BlN-ap3Nb zB*t8FLvrOPPWrbOB=;#xmUD6K_8|Tst@xN34*40*3|ATxZo09^jYZu%UNUXznx*2w zEtY=2-=B`h{=47rZ+`cugVBS*csv@6hNHn`@Sr~!J)MjmK>s#{*i~|4nSRj!YF?X@ z`yVMR<8M(LuH+0>L&CDG`uUXh>69dBV$Cz_tDZ?XbsGjwF%vvyt`?5N=7GPWNFjDR z2ZmG+o1Xid*p7N46i zqQ}!kc^j34u&idh zDmC1v6WZ@aXc--j$-S(98~-aNax}MX0QTU2IP6b1@IM|71_%7#MLB{KoU>e52y5@5 z+u9VA9Kp{^Zh#wL@awZzFS~Q4Q)VqDFy{hYhZBqi(+Df3Ig11uunHoC%;7izF0F#K z$~9O_Gr=qxN<3QLtWay7;0!t)GS7v$Qb`KT9;;m184L+AC@q&LPFoElLM-Z(OAoT2 zl;H7t1}7-cA|b_O3ggyhkSfWo(lY{p(n+7pV9ta=LNd)$rq?fIf*)qkC&XNE8!rp$ z1R$XMwNjZA)Tms?Xk5b){DP31oC&c8jj>8olnF>>ooY;mF`us?DU@Fg1tt^MK_FWp zSf_X9^d2i|Vy<%INykgYWAtlfmyQi;8WxEOqkzj;BQt26IhJww*4BW|B`66fS6nNZ zqO=AsxLtza)(Z5rD^Z?+$>XJC|B6fQj2(p+bGM9;D7avn*M%kTa^u@m$3S9?#xT8p;^(h1RetseajwN0BF(&R=I)&NuGaH+lmG~xa_e;t z5NG)eCjEZuf2Eizy`I77IVS)b-{)wq?>TtR3CY|hF&1Tvtvz9N?o{NCl|aK-rP%_H z1vBQgGyS2Kf5d{4RhbfMvHlU)o^2PL8nr|R0IqsVW!#a6gd82gUoxgy3RVf!eh(X0 zt8reRLhaFsicK$8GRDmMhdE#5VQ9}CY_`i}(yi(B$k7pLke|Un;GcvP^Lwclg^W6v z(yf;VKEmmhiE?&s(`lq=_dWu1Gk-~9Z;8MPnmE%4aq|M#a)H_rdbc-lXl|94Ra z6rN)+)D0IE{(kz>LAWKi?qd--e-dEEQ_nOt6WowLd%LZ=-bxDg%b)_uG~ zOG7Hjc|}0qhOn~f+QTR3PRSUiBmcDEBC+xrg~Hzsb=<-Q7s5qD-gENqyc52Vc8_|? zn5b4UpH~{)446vI?QFk^)`h9)eB);@`}570wU)8cMIb^?gQCLNQqET87YdhYF6|sT z-N-3Z;VXRd zi7XZ)bpF2so!>fN4kwkSN(MR2J~>0LAm|cBO3kw8WuL+2W##N|pBxSBcl9zTFt$p+ zELhOS4k{-7tjbBcHkM_y9R^kDw9CsaLq5U#Tv^;u!c{6MlWc)WH(Ccmq{&?QRt$1Vy_w7K{PiJ_|05brFAi4o%3_6R5>3NH@*QF7CBCqBX%yq--7} zMdSx8gnrJj<_7nxs~Io8-g?YDEHQH)dCWYl#BR-rK6+G{t2SA$&JEq%_nCZ`Wtaa~ zF2Zf3g7^4;lm4K8#s8a54*uU=l;-^p%Q8djOAT*Z2TWv3HX!F*CbROYhoop(nFhOR zuiDUah=RgGUT4=tbpm_>$z_7l!q^RVzLsNDbUbm!g1ad76z9cdOUPT!uXge9uE!Qr ztp|25FE@Rg#@SJtjP1!|`24xewrDSTZplVFqg>Fu(i-%{=3ex5NF_R>Cpd4c@Qwv;bvLEzeM+~;>Nk~M)X^g437_g{f)$=+7~IT>HVHK9 zy&C>zyo=G+f-A+PMSFDfCZoZ7SP#2;=3NJwZ6_qHiM1~;8geun8gld-a%^04gc2@0 z8ibnNsB5hVR2!=?`SOaRp`^xzNhr-^ez702Jn#%nG|uv-Q&)E^bNG$Ep=w+A_GQ2Z zr?T;8ysF;2xV2DGRy3lx#(hJzGpoKUCGr%H1>@<@<>_1tUE;|Z<2FQ+Ef6uZ2+B|! z8VJ8~JEw&kzx;*CtqY4|H0G5`(6k$z45!GFUo^M)M#k7`-Lk^F^n&H-ZCh4!F5j+}yVxdMNi(cEx{M3B0ZM zfcD-0jmKB+|E81aA^y9I(v1If#A3>FyHuKg^d3Jw|5FhL76IU?61XM$JCC~i0zmh} zWNce%%2&6-#3E4n;!siN0*jXzSjL}gm1mWu>_VqQw%XQicu&FjpDWZ+qosIv;l7vY zKUS{8f5c_N<>IT&z&`v>N1OM5Plx@d2mIef`5pNGK5UEMX9m6&;7Xu#(+N}nzea8X z&@fv!meS~g!~bwH+{FLW z;rM|6yC|*q|4Wo_Z1(H3y*J+UUkUyD*3=)fwDC`uC{n(VO5-i^*oXhgXtMeK|6qFf z|IeM2Blw9~cl`M`_QTS{#S$e%IT!Anhm6JNY=NfwBY?@XOlfPtEKvwp2o*ufY`o-h z@dPvqX88&sV|Ll7XEGs2AaN1iFnpM4obwNugml;cJ)-bNiZv+dM>>%pL+xJ}r-YuI z{dQ)RMsftjs|#YkP63-b71@ b9Zc4t9Lk~mH Date: Thu, 18 May 2023 15:14:54 +0800 Subject: [PATCH 321/439] chore: archive_command support online update (#3306) (#3309) --- deploy/postgresql/config/pg12-config-effect-scope.yaml | 2 -- deploy/postgresql/config/pg14-config-effect-scope.yaml | 2 -- deploy/postgresql/templates/clusterversion.yaml | 6 +++--- deploy/postgresql/templates/configconstraint-12.yaml | 2 +- deploy/postgresql/values.yaml | 2 +- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/deploy/postgresql/config/pg12-config-effect-scope.yaml b/deploy/postgresql/config/pg12-config-effect-scope.yaml index e103dfaa9..f01623b21 100644 --- a/deploy/postgresql/config/pg12-config-effect-scope.yaml +++ b/deploy/postgresql/config/pg12-config-effect-scope.yaml @@ -1,6 +1,5 @@ # Patroni bootstrap parameters staticParameters: - - archive_command - shared_buffers - logging_collector - log_destination @@ -22,7 +21,6 @@ staticParameters: - extwlist.custom_path immutableParameters: - - archive_command - archive_timeout - backtrace_functions - config_file diff --git a/deploy/postgresql/config/pg14-config-effect-scope.yaml b/deploy/postgresql/config/pg14-config-effect-scope.yaml index e103dfaa9..f01623b21 100644 --- a/deploy/postgresql/config/pg14-config-effect-scope.yaml +++ b/deploy/postgresql/config/pg14-config-effect-scope.yaml @@ -1,6 +1,5 @@ # Patroni bootstrap parameters staticParameters: - - archive_command - shared_buffers - logging_collector - log_destination @@ -22,7 +21,6 @@ staticParameters: - extwlist.custom_path immutableParameters: - - archive_command - archive_timeout - backtrace_functions - config_file diff --git a/deploy/postgresql/templates/clusterversion.yaml b/deploy/postgresql/templates/clusterversion.yaml index 3d3cbd5ea..ba88f5197 100644 --- a/deploy/postgresql/templates/clusterversion.yaml +++ b/deploy/postgresql/templates/clusterversion.yaml @@ -49,9 +49,9 @@ spec: versionsContext: initContainers: - name: pg-init-container - image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0 + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1 containers: - name: postgresql - image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0 - clientImage: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.0 + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1 + clientImage: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1 --- diff --git a/deploy/postgresql/templates/configconstraint-12.yaml b/deploy/postgresql/templates/configconstraint-12.yaml index 9d8ad01a0..13bf07a13 100644 --- a/deploy/postgresql/templates/configconstraint-12.yaml +++ b/deploy/postgresql/templates/configconstraint-12.yaml @@ -1,4 +1,4 @@ -{{- $cc := .Files.Get "config/pg14-config-effect-scope.yaml" | fromYaml }} +{{- $cc := .Files.Get "config/pg12-config-effect-scope.yaml" | fromYaml }} apiVersion: apps.kubeblocks.io/v1alpha1 kind: ConfigConstraint metadata: diff --git a/deploy/postgresql/values.yaml b/deploy/postgresql/values.yaml index 691cb23d0..c2fd0500e 100644 --- a/deploy/postgresql/values.yaml +++ b/deploy/postgresql/values.yaml @@ -11,7 +11,7 @@ image: registry: registry.cn-hangzhou.aliyuncs.com repository: apecloud/spilo - tag: 14.7.1 + tag: 14.7.2 digest: "" ## Specify a imagePullPolicy ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' From 9cff6ca272fd33e1aefadd536b2682b62ed4b42e Mon Sep 17 00:00:00 2001 From: shanshanying Date: Thu, 18 May 2023 17:55:51 +0800 Subject: [PATCH 322/439] fix: refactor client-image field in ClusterVersion API (#3322) --- apis/apps/v1alpha1/clusterversion_types.go | 14 +- .../apps.kubeblocks.io_clusterversions.yaml | 145 ++++++++++++++++- controllers/apps/systemaccount_util.go | 21 ++- controllers/apps/systemaccount_util_test.go | 88 ++++++++++ .../templates/clusterversion.yaml | 4 +- .../apps.kubeblocks.io_clusterversions.yaml | 145 ++++++++++++++++- deploy/mongodb/templates/clusterversion.yaml | 4 +- .../postgresql/templates/clusterversion.yaml | 9 +- deploy/redis/templates/clusterversion.yaml | 4 +- .../create/template/dns_chaos_template.cue | 42 ++--- .../create/template/http_chaos_template.cue | 86 +++++----- .../cli/create/template/io_chaos_template.cue | 68 ++++---- .../template/network_chaos_template.cue | 154 +++++++++--------- .../create/template/pod_chaos_template.cue | 44 ++--- .../create/template/stress_chaos_template.cue | 42 ++--- .../create/template/time_chaos_template.cue | 48 +++--- internal/cli/util/flags/flags.go | 19 +++ .../controller/builder/cue/pdb_template.cue | 2 +- 18 files changed, 673 insertions(+), 266 deletions(-) diff --git a/apis/apps/v1alpha1/clusterversion_types.go b/apis/apps/v1alpha1/clusterversion_types.go index 0af45846c..3814edcf7 100644 --- a/apis/apps/v1alpha1/clusterversion_types.go +++ b/apis/apps/v1alpha1/clusterversion_types.go @@ -82,16 +82,24 @@ type ClusterComponentVersion struct { // +listMapKey=name ConfigSpecs []ComponentConfigSpec `json:"configSpecs,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"name"` - // clientImage define image for the component to connect database or engines. - // This value has a higher proirity over ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig.image. + // systemAccountSpec define image for the component to connect database or engines. + // It overrides `image` and `env` attributes defined in ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig. + // To clean default envs settings, set `SystemAccountSpec.CmdExecutorConfig.Env` to empty list. // +optional - ClientImage string `json:"clientImage,omitempty"` + SystemAccountSpec *SystemAccountShortSpec `json:"systemAccountSpec,omitempty"` // versionContext defines containers images' context for component versions, // this value replaces ClusterDefinition.spec.componentDefs.podSpec.[initContainers | containers] VersionsCtx VersionsContext `json:"versionsContext"` } +// SystemAccountShortSpec is a short version of SystemAccountSpec, with only CmdExecutorConfig field. +type SystemAccountShortSpec struct { + // cmdExecutorConfig configs how to get client SDK and perform statements. + // +kubebuilder:validation:Required + CmdExecutorConfig *CommandExecutorEnvItem `json:"cmdExecutorConfig"` +} + type VersionsContext struct { // Provide ClusterDefinition.spec.componentDefs.podSpec.initContainers override // values, typical scenarios are application container image updates. diff --git a/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml b/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml index 1083c4526..8d04ae87c 100644 --- a/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_clusterversions.yaml @@ -62,11 +62,6 @@ spec: description: ClusterComponentVersion is an application version component spec. properties: - clientImage: - description: clientImage define image for the component to connect - database or engines. This value has a higher proirity over - ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig.image. - type: string componentDefRef: description: componentDefRef reference one of the cluster component definition names in ClusterDefinition API (spec.componentDefs.name). @@ -140,6 +135,146 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + systemAccountSpec: + description: systemAccountSpec define image for the component + to connect database or engines. It overrides `image` and `env` + attributes defined in ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig. + To clean default envs settings, set `SystemAccountSpec.CmdExecutorConfig.Env` + to empty list. + properties: + cmdExecutorConfig: + description: cmdExecutorConfig configs how to get client + SDK and perform statements. + properties: + env: + description: envs is a list of environment variables. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults + to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: + supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in + the pod's namespace + properties: + key: + description: The key of the secret to + select from. Must be a valid secret + key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + image: + description: image for Connector when executing the + command. + type: string + required: + - image + type: object + required: + - cmdExecutorConfig + type: object versionsContext: description: versionContext defines containers images' context for component versions, this value replaces ClusterDefinition.spec.componentDefs.podSpec.[initContainers diff --git a/controllers/apps/systemaccount_util.go b/controllers/apps/systemaccount_util.go index 7fda12aba..403ccf73f 100644 --- a/controllers/apps/systemaccount_util.go +++ b/controllers/apps/systemaccount_util.go @@ -186,7 +186,9 @@ func renderJob(engine *customizedEngine, key componentUniqueKey, statement []str // place statements and endpoints before user defined envs. envs := make([]corev1.EnvVar, 0, 2+len(engine.getEnvs())) envs = append(envs, statementEnv, endpointEnv) - envs = append(envs, engine.getEnvs()...) + if len(engine.getEnvs()) > 0 { + envs = append(envs, engine.getEnvs()...) + } job := &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ @@ -390,11 +392,22 @@ func calibrateJobMetaAndSpec(job *batchv1.Job, cluster *appsv1alpha1.Cluster, co // completeExecConfig override the image of execConfig if version is not nil. func completeExecConfig(execConfig *appsv1alpha1.CmdExecutorConfig, version *appsv1alpha1.ClusterComponentVersion) { - if version == nil { + if version == nil || version.SystemAccountSpec == nil || version.SystemAccountSpec.CmdExecutorConfig == nil { return } - if len(version.ClientImage) == 0 { + sysAccountSpec := version.SystemAccountSpec + if len(sysAccountSpec.CmdExecutorConfig.Image) > 0 { + execConfig.Image = sysAccountSpec.CmdExecutorConfig.Image + } + + // envs from sysAccountSpec will override the envs from execConfig + if sysAccountSpec.CmdExecutorConfig.Env == nil { return } - execConfig.Image = version.ClientImage + if len(sysAccountSpec.CmdExecutorConfig.Env) == 0 { + // clean up envs + execConfig.Env = nil + } else { + execConfig.Env = sysAccountSpec.CmdExecutorConfig.Env + } } diff --git a/controllers/apps/systemaccount_util_test.go b/controllers/apps/systemaccount_util_test.go index 0ad5ee74c..853e27c55 100644 --- a/controllers/apps/systemaccount_util_test.go +++ b/controllers/apps/systemaccount_util_test.go @@ -21,6 +21,7 @@ package apps import ( "math/rand" + "reflect" "strings" "testing" @@ -57,6 +58,7 @@ func mockSystemAccountsSpec() *appsv1alpha1.SystemAccountSpec { PasswordConfig: pwdConfig, Accounts: []appsv1alpha1.SystemAccountConfig{}, } + var account appsv1alpha1.SystemAccountConfig var scope appsv1alpha1.ProvisionScope for _, name := range getAllSysAccounts() { @@ -358,3 +360,89 @@ func TestRenderCreationStmt(t *testing.T) { } } } + +func TestMergeSystemAccountConfig(t *testing.T) { + systemAccount := mockSystemAccountsSpec() + // Make sure env is not empty + if systemAccount.CmdExecutorConfig.Env == nil { + systemAccount.CmdExecutorConfig.Env = []corev1.EnvVar{} + } + + if len(systemAccount.CmdExecutorConfig.Env) == 0 { + systemAccount.CmdExecutorConfig.Env = append(systemAccount.CmdExecutorConfig.Env, corev1.EnvVar{ + Name: "cluster-def-env", + Value: "cluster-def-env-value", + }) + } + // nil spec + componentVersion := &appsv1alpha1.ClusterComponentVersion{ + SystemAccountSpec: nil, + } + accountConfig := systemAccount.CmdExecutorConfig.DeepCopy() + completeExecConfig(accountConfig, componentVersion) + assert.Equal(t, systemAccount.CmdExecutorConfig.Image, accountConfig.Image) + assert.Len(t, accountConfig.Env, len(systemAccount.CmdExecutorConfig.Env)) + if len(systemAccount.CmdExecutorConfig.Env) > 0 { + assert.True(t, reflect.DeepEqual(accountConfig.Env, systemAccount.CmdExecutorConfig.Env)) + } + + // empty spec + accountConfig = systemAccount.CmdExecutorConfig.DeepCopy() + componentVersion.SystemAccountSpec = &appsv1alpha1.SystemAccountShortSpec{ + CmdExecutorConfig: &appsv1alpha1.CommandExecutorEnvItem{}, + } + + completeExecConfig(accountConfig, componentVersion) + assert.Equal(t, systemAccount.CmdExecutorConfig.Image, accountConfig.Image) + assert.Len(t, accountConfig.Env, len(systemAccount.CmdExecutorConfig.Env)) + if len(systemAccount.CmdExecutorConfig.Env) > 0 { + assert.True(t, reflect.DeepEqual(accountConfig.Env, systemAccount.CmdExecutorConfig.Env)) + } + + // spec with image + mockImageName := "test-image" + accountConfig = systemAccount.CmdExecutorConfig.DeepCopy() + componentVersion.SystemAccountSpec = &appsv1alpha1.SystemAccountShortSpec{ + CmdExecutorConfig: &appsv1alpha1.CommandExecutorEnvItem{ + Image: mockImageName, + Env: nil, + }, + } + completeExecConfig(accountConfig, componentVersion) + assert.NotEqual(t, systemAccount.CmdExecutorConfig.Image, accountConfig.Image) + assert.Equal(t, mockImageName, accountConfig.Image) + assert.Len(t, accountConfig.Env, len(systemAccount.CmdExecutorConfig.Env)) + if len(systemAccount.CmdExecutorConfig.Env) > 0 { + assert.True(t, reflect.DeepEqual(accountConfig.Env, systemAccount.CmdExecutorConfig.Env)) + } + // sepc with empty envs + accountConfig = systemAccount.CmdExecutorConfig.DeepCopy() + componentVersion.SystemAccountSpec = &appsv1alpha1.SystemAccountShortSpec{ + CmdExecutorConfig: &appsv1alpha1.CommandExecutorEnvItem{ + Image: mockImageName, + Env: []corev1.EnvVar{}, + }, + } + completeExecConfig(accountConfig, componentVersion) + assert.NotEqual(t, systemAccount.CmdExecutorConfig.Image, accountConfig.Image) + assert.Equal(t, mockImageName, accountConfig.Image) + assert.Len(t, accountConfig.Env, 0) + + // sepc with envs + testEnv := corev1.EnvVar{ + Name: "test-env", + Value: "test-value", + } + accountConfig = systemAccount.CmdExecutorConfig.DeepCopy() + componentVersion.SystemAccountSpec = &appsv1alpha1.SystemAccountShortSpec{ + CmdExecutorConfig: &appsv1alpha1.CommandExecutorEnvItem{ + Image: mockImageName, + Env: []corev1.EnvVar{testEnv}, + }, + } + completeExecConfig(accountConfig, componentVersion) + assert.NotEqual(t, systemAccount.CmdExecutorConfig.Image, accountConfig.Image) + assert.Equal(t, mockImageName, accountConfig.Image) + assert.Len(t, accountConfig.Env, 1) + assert.Contains(t, accountConfig.Env, testEnv) +} diff --git a/deploy/apecloud-mysql/templates/clusterversion.yaml b/deploy/apecloud-mysql/templates/clusterversion.yaml index 7ac7ba1e6..22b8b7f5c 100644 --- a/deploy/apecloud-mysql/templates/clusterversion.yaml +++ b/deploy/apecloud-mysql/templates/clusterversion.yaml @@ -13,4 +13,6 @@ spec: - name: mysql image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} - clientImage: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} \ No newline at end of file + systemAccountSpec: + cmdExecutorConfig: + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} \ No newline at end of file diff --git a/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml b/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml index 1083c4526..8d04ae87c 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_clusterversions.yaml @@ -62,11 +62,6 @@ spec: description: ClusterComponentVersion is an application version component spec. properties: - clientImage: - description: clientImage define image for the component to connect - database or engines. This value has a higher proirity over - ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig.image. - type: string componentDefRef: description: componentDefRef reference one of the cluster component definition names in ClusterDefinition API (spec.componentDefs.name). @@ -140,6 +135,146 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + systemAccountSpec: + description: systemAccountSpec define image for the component + to connect database or engines. It overrides `image` and `env` + attributes defined in ClusterDefinition.spec.componentDefs.systemAccountSpec.cmdExecutorConfig. + To clean default envs settings, set `SystemAccountSpec.CmdExecutorConfig.Env` + to empty list. + properties: + cmdExecutorConfig: + description: cmdExecutorConfig configs how to get client + SDK and perform statements. + properties: + env: + description: envs is a list of environment variables. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. + Must be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) + are expanded using the previously defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. + Double $$ are reduced to a single $, which allows + for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" + will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults + to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: + supports metadata.name, metadata.namespace, + `metadata.labels['''']`, `metadata.annotations['''']`, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the + FieldPath is written in terms of, defaults + to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, + requests.cpu, requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in + the pod's namespace + properties: + key: + description: The key of the secret to + select from. Must be a valid secret + key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + x-kubernetes-preserve-unknown-fields: true + image: + description: image for Connector when executing the + command. + type: string + required: + - image + type: object + required: + - cmdExecutorConfig + type: object versionsContext: description: versionContext defines containers images' context for component versions, this value replaces ClusterDefinition.spec.componentDefs.podSpec.[initContainers diff --git a/deploy/mongodb/templates/clusterversion.yaml b/deploy/mongodb/templates/clusterversion.yaml index 1eb5cfc44..fbdea5a50 100644 --- a/deploy/mongodb/templates/clusterversion.yaml +++ b/deploy/mongodb/templates/clusterversion.yaml @@ -13,4 +13,6 @@ spec: - name: mongodb image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} - clientImage: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} \ No newline at end of file + systemAccountSpec: + cmdExecutorConfig: + image: {{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }} \ No newline at end of file diff --git a/deploy/postgresql/templates/clusterversion.yaml b/deploy/postgresql/templates/clusterversion.yaml index ba88f5197..39fa233d2 100644 --- a/deploy/postgresql/templates/clusterversion.yaml +++ b/deploy/postgresql/templates/clusterversion.yaml @@ -16,7 +16,9 @@ spec: containers: - name: postgresql image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} - clientImage: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + systemAccountSpec: + cmdExecutorConfig: + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} --- apiVersion: apps.kubeblocks.io/v1alpha1 @@ -53,5 +55,6 @@ spec: containers: - name: postgresql image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1 - clientImage: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1 ---- + systemAccountSpec: + cmdExecutorConfig: + image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:12.14.1 diff --git a/deploy/redis/templates/clusterversion.yaml b/deploy/redis/templates/clusterversion.yaml index 0adb370d2..54608aa13 100644 --- a/deploy/redis/templates/clusterversion.yaml +++ b/deploy/redis/templates/clusterversion.yaml @@ -13,7 +13,9 @@ spec: - name: redis image: {{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ default .Values.image.pullPolicy "IfNotPresent" }} - clientImage: {{ .Values.image.repository }}:{{ .Values.image.tag }} + systemAccountSpec: + cmdExecutorConfig: + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} - componentDefRef: redis-sentinel versionsContext: initContainers: diff --git a/internal/cli/create/template/dns_chaos_template.cue b/internal/cli/create/template/dns_chaos_template.cue index 92be304be..96a63787d 100644 --- a/internal/cli/create/template/dns_chaos_template.cue +++ b/internal/cli/create/template/dns_chaos_template.cue @@ -17,31 +17,31 @@ // required, command line input options for parameters and flags options: { - namespace: string - action: string - selector: {} - mode: string - value: string - duration: string - - patterns: [...] + namespace: string + action: string + selector: {} + mode: string + value: string + duration: string + + patterns: [...] } // required, k8s api resource content content: { - kind: "DNSChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "dns-chaos-" - namespace: options.namespace - } - spec:{ - selector: options.selector - action: options.action - mode: options.mode - value: options.value - duration: options.duration + kind: "DNSChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "dns-chaos-" + namespace: options.namespace + } + spec: { + selector: options.selector + action: options.action + mode: options.mode + value: options.value + duration: options.duration patterns: options.patterns - } + } } diff --git a/internal/cli/create/template/http_chaos_template.cue b/internal/cli/create/template/http_chaos_template.cue index 5299d2a31..0163d6362 100644 --- a/internal/cli/create/template/http_chaos_template.cue +++ b/internal/cli/create/template/http_chaos_template.cue @@ -17,59 +17,59 @@ // required, command line input options for parameters and flags options: { - namespace: string - selector: {} - mode: string - value: string - duration: string + namespace: string + selector: {} + mode: string + value: string + duration: string - target: string - port: int32 - path: string - method: string - code?: int32 + target: string + port: int32 + path: string + method: string + code?: int32 - abort?: bool - delay?: string + abort?: bool + delay?: string - repalceBody?: bytes - replacePath?: string - replaceMethod?: string + repalceBody?: bytes + replacePath?: string + replaceMethod?: string - patchBodyValue?: string - patchBodyType?: string + patchBodyValue?: string + patchBodyType?: string } // required, k8s api resource content content: { - kind: "HTTPChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "http-chaos-" - namespace: options.namespace - } - spec:{ - selector: options.selector - mode: options.mode - value: options.value - duration: options.duration + kind: "HTTPChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "http-chaos-" + namespace: options.namespace + } + spec: { + selector: options.selector + mode: options.mode + value: options.value + duration: options.duration target: options.target - port: options.port - path: options.path - method: options.method - if options.code != _|_ { - code: options.code - } + port: options.port + path: options.path + method: options.method + if options.code != _|_ { + code: options.code + } - if options.abort != _|_ { - abort: options.abort - } - if options.delay != _|_ { + if options.abort != _|_ { + abort: options.abort + } + if options.delay != _|_ { delay: options.delay } - if options.replaceBody != _|_ || options.replacePath != _|_ || options.replaceMethod != _|_{ - replace:{ + if options.replaceBody != _|_ || options.replacePath != _|_ || options.replaceMethod != _|_ { + replace: { if options.replaceBody != _|_ { body: options.replaceBody } @@ -82,14 +82,14 @@ content: { } } if options.patchBodyValue != _|_ && options.patchBodyType != _|_ { - patch:{ + patch: { if options.patchBodyValue != _|_ && options.patchBodyType != _|_ { body: { value: options.patchBodyValue - type: options.patchBodyType + type: options.patchBodyType } } } } - } + } } diff --git a/internal/cli/create/template/io_chaos_template.cue b/internal/cli/create/template/io_chaos_template.cue index cb0a798dd..0d6f4a917 100644 --- a/internal/cli/create/template/io_chaos_template.cue +++ b/internal/cli/create/template/io_chaos_template.cue @@ -17,41 +17,41 @@ // required, command line input options for parameters and flags options: { - namespace: string - action: string - selector: {} - mode: string - value: string - duration: string + namespace: string + action: string + selector: {} + mode: string + value: string + duration: string - delay: string - errno: int - attr?: {} - mistake?: {} + delay: string + errno: int + attr?: {} + mistake?: {} - volumePath: string - path: string - percent: int - methods: [...] - containerNames: [...] + volumePath: string + path: string + percent: int + methods: [...] + containerNames: [...] } // required, k8s api resource content content: { - kind: "IOChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "io-chaos-" - namespace: options.namespace - } - spec:{ - selector: options.selector - mode: options.mode - value: options.value - action: options.action - duration: options.duration + kind: "IOChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "io-chaos-" + namespace: options.namespace + } + spec: { + selector: options.selector + mode: options.mode + value: options.value + action: options.action + duration: options.duration - delay: options.delay + delay: options.delay errno: options.errno if len(options.attr) != 0 { attr: options.attr @@ -60,10 +60,10 @@ content: { mistake: options.mistake } - volumePath: options.volumePath - path: options.path - percent: options.percent - methods: options.methods - containerNames: options.containerNames - } + volumePath: options.volumePath + path: options.path + percent: options.percent + methods: options.methods + containerNames: options.containerNames + } } diff --git a/internal/cli/create/template/network_chaos_template.cue b/internal/cli/create/template/network_chaos_template.cue index 1db5fc6b0..e28029198 100644 --- a/internal/cli/create/template/network_chaos_template.cue +++ b/internal/cli/create/template/network_chaos_template.cue @@ -17,102 +17,102 @@ // required, command line input options for parameters and flags options: { - namespace: string - action: string - selector: {} - mode: string - value: string - duration: string + namespace: string + action: string + selector: {} + mode: string + value: string + duration: string - direction: string - externalTargets?: [...] + direction: string + externalTargets?: [...] - targetMode?: string - targetValue: string + targetMode?: string + targetValue: string targetNamespaceSelectors: [...string] - targetLabelSelectors: {} + targetLabelSelectors: {} - loss?: string - corrupt?: string - duplicate?: string + loss?: string + corrupt?: string + duplicate?: string - latency?: string - jitter: string + latency?: string + jitter: string - correlation: string + correlation: string - rate?: string - limit: uint32 - buffer: uint32 - peakrate: uint32 - minburst: uint32 + rate?: string + limit: uint32 + buffer: uint32 + peakrate: uint32 + minburst: uint32 } // required, k8s api resource content content: { - kind: "NetworkChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "network-chaos-" - namespace: options.namespace - } - spec:{ - selector: options.selector - mode: options.mode - value: options.value - action: options.action - duration: options.duration + kind: "NetworkChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "network-chaos-" + namespace: options.namespace + } + spec: { + selector: options.selector + mode: options.mode + value: options.value + action: options.action + duration: options.duration - direction: options.direction - if options.externalTargets != _|_ { - externalTargets: options.externalTargets - } - if options.targetMode != _|_ { - target:{ - mode: options.targetMode - value: options.targetValue - selector:{ - namespaces: options.targetNamespaceSelectors - labelSelectors:{ - options.targetLabelSelectors - } + direction: options.direction + if options.externalTargets != _|_ { + externalTargets: options.externalTargets + } + if options.targetMode != _|_ { + target: { + mode: options.targetMode + value: options.targetValue + selector: { + namespaces: options.targetNamespaceSelectors + labelSelectors: { + options.targetLabelSelectors } } - } - if options.loss != _|_ { - loss:{ - loss: options.loss + } + } + if options.loss != _|_ { + loss: { + loss: options.loss correlation: options.correlation } - } + } if options.corrupt != _|_ { - corrupt:{ - corrupt: options.corrupt - correlation: options.correlation - } + corrupt: { + corrupt: options.corrupt + correlation: options.correlation + } } if options.duplicate != _|_ { - duplicate:{ - duplicate: options.duplicate - correlation: options.correlation - } + duplicate: { + duplicate: options.duplicate + correlation: options.correlation + } } - if options.latency != _|_{ - delay:{ - latency: options.latency - jitter: options.jitter - correlation: options.correlation - } - } - if options.rate != _|_{ - bandwidth:{ - rate: options.rate - limit: options.limit - buffer: options.buffer - peakrate: options.peakrate - minburst: options.minburst - correlation: options.correlation - } + if options.latency != _|_ { + delay: { + latency: options.latency + jitter: options.jitter + correlation: options.correlation + } + } + if options.rate != _|_ { + bandwidth: { + rate: options.rate + limit: options.limit + buffer: options.buffer + peakrate: options.peakrate + minburst: options.minburst + correlation: options.correlation + } } - } + } } diff --git a/internal/cli/create/template/pod_chaos_template.cue b/internal/cli/create/template/pod_chaos_template.cue index e653dc464..654479624 100644 --- a/internal/cli/create/template/pod_chaos_template.cue +++ b/internal/cli/create/template/pod_chaos_template.cue @@ -17,33 +17,33 @@ // required, command line input options for parameters and flags options: { - namespace: string - action: string - selector: {} - mode: string - value: string - duration: string + namespace: string + action: string + selector: {} + mode: string + value: string + duration: string - gracePeriod: int64 - containerNames: [...] + gracePeriod: int64 + containerNames: [...] } // required, k8s api resource content content: { - kind: "PodChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "pod-chaos-" - namespace: options.namespace - } - spec:{ + kind: "PodChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "pod-chaos-" + namespace: options.namespace + } + spec: { selector: options.selector - mode: options.mode - value: options.value - action: options.action - duration: options.duration + mode: options.mode + value: options.value + action: options.action + duration: options.duration - gracePeriod: options.gracePeriod - containerNames: options.containerNames - } + gracePeriod: options.gracePeriod + containerNames: options.containerNames + } } diff --git a/internal/cli/create/template/stress_chaos_template.cue b/internal/cli/create/template/stress_chaos_template.cue index 95d490a85..94a74885a 100644 --- a/internal/cli/create/template/stress_chaos_template.cue +++ b/internal/cli/create/template/stress_chaos_template.cue @@ -17,30 +17,30 @@ // required, command line input options for parameters and flags options: { - namespace: string - selector: {} - mode: string - value: string - duration: string + namespace: string + selector: {} + mode: string + value: string + duration: string - stressors: {} - containerNames: [...] + stressors: {} + containerNames: [...] } // required, k8s api resource content content: { - kind: "StressChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "stress-chaos-" - namespace: options.namespace - } - spec:{ - selector: options.selector - mode: options.mode - value: options.value - duration: options.duration - stressors: options.stressors - containerNames: options.containerNames - } + kind: "StressChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "stress-chaos-" + namespace: options.namespace + } + spec: { + selector: options.selector + mode: options.mode + value: options.value + duration: options.duration + stressors: options.stressors + containerNames: options.containerNames + } } diff --git a/internal/cli/create/template/time_chaos_template.cue b/internal/cli/create/template/time_chaos_template.cue index debd3585c..68512adf2 100644 --- a/internal/cli/create/template/time_chaos_template.cue +++ b/internal/cli/create/template/time_chaos_template.cue @@ -17,33 +17,33 @@ // required, command line input options for parameters and flags options: { - namespace: string - selector: {} - mode: string - value: string - duration: string - - timeOffset: string - clockIds: [...] - containerNames: [...] + namespace: string + selector: {} + mode: string + value: string + duration: string + + timeOffset: string + clockIds: [...] + containerNames: [...] } // required, k8s api resource content content: { - kind: "TimeChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "time-chaos-" - namespace: options.namespace - } - spec:{ - selector: options.selector - mode: options.mode - value: options.value - duration: options.duration + kind: "TimeChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "time-chaos-" + namespace: options.namespace + } + spec: { + selector: options.selector + mode: options.mode + value: options.value + duration: options.duration - timeOffset: options.timeOffset - clockIds: options.clockIds - containerNames: options.containerNames - } + timeOffset: options.timeOffset + clockIds: options.clockIds + containerNames: options.containerNames + } } diff --git a/internal/cli/util/flags/flags.go b/internal/cli/util/flags/flags.go index 8c75ba914..544abcdc8 100644 --- a/internal/cli/util/flags/flags.go +++ b/internal/cli/util/flags/flags.go @@ -1,3 +1,22 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + package flags import ( diff --git a/internal/controller/builder/cue/pdb_template.cue b/internal/controller/builder/cue/pdb_template.cue index 1e5fdc6c4..476ab0b5f 100644 --- a/internal/controller/builder/cue/pdb_template.cue +++ b/internal/controller/builder/cue/pdb_template.cue @@ -24,7 +24,7 @@ cluster: { component: { clusterDefName: string name: string - minAvailable: string | int + minAvailable: string | int } pdb: { From b5abe41caa9e6ef95f7e7c891d606c834d1fa353 Mon Sep 17 00:00:00 2001 From: chantu Date: Thu, 18 May 2023 17:57:14 +0800 Subject: [PATCH 323/439] fix: leader should switchover first before scaling in (#3301) --- deploy/apecloud-mysql/templates/scripts.yaml | 31 +++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index bf0cb1837..d1ba7eb32 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -58,12 +58,7 @@ data: cluster_info="$cluster_info;"; fi; tmp_host=$(eval echo \$KB_MYSQL_"$i"_HOSTNAME) - # setup pod weight, prefer pod 0 to be leader - if [ $i -eq 0 ]; then - cluster_info="$cluster_info$tmp_host:13306#9N"; - else - cluster_info="$cluster_info$tmp_host:13306#1N"; - fi + cluster_info="$cluster_info$tmp_host:13306"; done; cluster_info="$cluster_info@$(($idx+1))"; echo "cluster_info=$cluster_info"; @@ -156,7 +151,6 @@ data: pre-stop.sh: | #!/bin/bash drop_followers() { - leader=`cat /etc/annotations/leader` echo "leader=$leader" >> /data/mysql/.kb_pre_stop.log echo "KB_POD_NAME=$KB_POD_NAME" >> /data/mysql/.kb_pre_stop.log if [ -z "$leader" -o "$KB_POD_NAME" = "$leader" ]; then @@ -178,11 +172,34 @@ data: echo "mysql $host_flag -uroot $password_flag -e \"call dbms_consensus.drop_learner('$host:13306');\" 2>&1 " >> /data/mysql/.kb_pre_stop.log mysql $host_flag -uroot $password_flag -e "call dbms_consensus.drop_learner('$host:13306');" 2>&1 } + switchover() { + if [ ! -z $MYSQL_ROOT_PASSWORD ]; then + password_flag="-p$MYSQL_ROOT_PASSWORD" + fi + new_leader_host=$KB_MYSQL_0_HOSTNAME + if [ "$KB_POD_NAME" = "$leader" ]; then + echo "self is leader, need to switchover" >> /data/mysql/.kb_pre_stop.log + echo "mysql -uroot $password_flag -e \"call dbms_consensus.change_leader('$new_leader_host:13306');\" 2>&1" >> /data/mysql/.kb_pre_stop.log + mysql -uroot $password_flag -e "call dbms_consensus.change_leader('$new_leader_host:13306');" 2>&1 + sleep 1 + role_info=`mysql -uroot $password_flag -e "select * from information_schema.wesql_cluster_local;" 2>&1` + echo "role_info=$role_info" >> /data/mysql/.kb_pre_stop.log + is_follower=`echo $role_info | grep "Follower"` + if [ ! -z "$is_follower" ]; then + echo "new_leader=$new_leader_host" >> /data/mysql/.kb_pre_stop.log + leader=`echo "$new_leader_host" | cut -d "." -f 1` + idx=${KB_POD_NAME##*-} + fi + fi + } + leader=`cat /etc/annotations/leader` idx=${KB_POD_NAME##*-} current_component_replicas=`cat /etc/annotations/component-replicas` echo "current replicas: $current_component_replicas" >> /data/mysql/.kb_pre_stop.log if [ ! $idx -lt $current_component_replicas ] && [ $current_component_replicas -ne 0 ]; then # if idx greater than or equal to current_component_replicas means the cluster's scaling in + # switch leader before leader scaling in itself + switchover # only scaling in need to drop followers drop_followers else From 414601baad49fff61f55e2ebe88187e1633bd923 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Thu, 18 May 2023 19:34:46 +0800 Subject: [PATCH 324/439] fix: hscale is always running if fail pods are exist before doing hscale (#3305) --- .../apps/operations/ops_progress_util.go | 81 +++++++++++-------- .../apps/operations/ops_progress_util_test.go | 47 ++++++----- 2 files changed, 76 insertions(+), 52 deletions(-) diff --git a/controllers/apps/operations/ops_progress_util.go b/controllers/apps/operations/ops_progress_util.go index 8f5fb6cd1..ac5fd5ffc 100644 --- a/controllers/apps/operations/ops_progress_util.go +++ b/controllers/apps/operations/ops_progress_util.go @@ -248,14 +248,7 @@ func handleStatefulSetProgress(reqCtx intctrlutil.RequestCtx, podList *corev1.PodList, pgRes progressResource, compStatus *appsv1alpha1.OpsRequestComponentStatus) (int32, error) { - currComponent, err := components.NewComponentByType(cli, - opsRes.Cluster, pgRes.clusterComponent, *pgRes.clusterComponentDef) - if err != nil { - return 0, err - } - var componentName = pgRes.clusterComponent.Name - minReadySeconds, err := util.GetComponentStsMinReadySeconds(reqCtx.Ctx, - cli, *opsRes.Cluster, componentName) + currComponent, minReadySeconds, err := getCompImplAndMinReadySeconds(reqCtx, cli, opsRes, pgRes) if err != nil { return 0, err } @@ -280,6 +273,23 @@ func handleStatefulSetProgress(reqCtx intctrlutil.RequestCtx, return completedCount, err } +func getCompImplAndMinReadySeconds(reqCtx intctrlutil.RequestCtx, + cli client.Client, + opsRes *OpsResource, + pgRes progressResource) (types.Component, int32, error) { + currComponent, err := components.NewComponentByType(cli, + opsRes.Cluster, pgRes.clusterComponent, *pgRes.clusterComponentDef) + if err != nil { + return nil, 0, err + } + minReadySeconds, err := util.GetComponentStsMinReadySeconds(reqCtx.Ctx, + cli, *opsRes.Cluster, pgRes.clusterComponent.Name) + if err != nil { + return nil, 0, err + } + return currComponent, minReadySeconds, nil +} + // handlePendingProgressDetail handles the pending progressDetail and sets it to progressDetails. func handlePendingProgressDetail(opsRes *OpsResource, compStatus *appsv1alpha1.OpsRequestComponentStatus, @@ -435,7 +445,7 @@ func handleComponentProgressForScalingReplicas(reqCtx intctrlutil.RequestCtx, expectProgressCount = dValue * -1 } if !isScaleOut { - completedCount, err = handleScaleDownProgress(opsRes, pgRes, podList, compStatus) + completedCount, err = handleScaleDownProgress(reqCtx, cli, opsRes, pgRes, podList, compStatus) expectProgressCount = getFinalExpectCount(compStatus, expectProgressCount) return expectProgressCount, completedCount, err } @@ -455,14 +465,7 @@ func handleScaleOutProgress(reqCtx intctrlutil.RequestCtx, pgRes progressResource, podList *corev1.PodList, compStatus *appsv1alpha1.OpsRequestComponentStatus) (int32, error) { - var componentName = pgRes.clusterComponent.Name - currComponent, err := components.NewComponentByType(cli, - opsRes.Cluster, pgRes.clusterComponent, *pgRes.clusterComponentDef) - if err != nil { - return 0, err - } - minReadySeconds, err := util.GetComponentWorkloadMinReadySeconds(reqCtx.Ctx, - cli, *opsRes.Cluster, pgRes.clusterComponentDef.WorkloadType, componentName) + currComponent, minReadySeconds, err := getCompImplAndMinReadySeconds(reqCtx, cli, opsRes, pgRes) if err != nil { return 0, err } @@ -476,30 +479,19 @@ func handleScaleOutProgress(reqCtx intctrlutil.RequestCtx, progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey} if currComponent.PodIsAvailable(&v, minReadySeconds) { completedCount += 1 - message := fmt.Sprintf("Successfully created pod: %s in Component: %s", objectKey, componentName) - progressDetail.SetStatusAndMessage(appsv1alpha1.SucceedProgressStatus, message) - setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, - &compStatus.ProgressDetails, progressDetail) + pgRes.opsMessageKey = "created" + handleSucceedProgressDetail(opsRes, pgRes, compStatus, progressDetail) continue } - - if util.IsFailedOrAbnormal(compStatus.Phase) { - // means the pod is failed. - podMessage := getFailedPodMessage(opsRes.Cluster, componentName, &v) - message := fmt.Sprintf("Failed to create pod: %s in Component: %s, message: %s", objectKey, componentName, podMessage) - progressDetail.SetStatusAndMessage(appsv1alpha1.FailedProgressStatus, message) - completedCount += 1 - } else { - progressDetail.SetStatusAndMessage(appsv1alpha1.ProcessingProgressStatus, "Start to create pod: "+objectKey) - } - setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, - &compStatus.ProgressDetails, progressDetail) + completedCount += handleFailedOrProcessingProgressDetail(opsRes, pgRes, compStatus, progressDetail, &v) } return completedCount, nil } // handleScaleDownProgress handles the progressDetails of scaled down replicas. func handleScaleDownProgress( + reqCtx intctrlutil.RequestCtx, + cli client.Client, opsRes *OpsResource, pgRes progressResource, podList *corev1.PodList, @@ -537,6 +529,29 @@ func handleScaleDownProgress( setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) } + // handle the re-created pods if these pods are failed before doing horizontal scaling. + currComponent, minReadySeconds, err := getCompImplAndMinReadySeconds(reqCtx, cli, opsRes, pgRes) + if err != nil { + return 0, err + } + for _, v := range podList.Items { + objectKey := getProgressObjectKey(constant.PodKind, v.Name) + progressDetail := findStatusProgressDetail(compStatus.ProgressDetails, objectKey) + if progressDetail == nil { + continue + } + if isCompletedProgressStatus(progressDetail.Status) { + completedCount += 1 + continue + } + pgRes.opsMessageKey = "re-create" + if currComponent.PodIsAvailable(&v, minReadySeconds) { + completedCount += 1 + handleSucceedProgressDetail(opsRes, pgRes, compStatus, *progressDetail) + continue + } + completedCount += handleFailedOrProcessingProgressDetail(opsRes, pgRes, compStatus, *progressDetail, &v) + } return completedCount, nil } diff --git a/controllers/apps/operations/ops_progress_util_test.go b/controllers/apps/operations/ops_progress_util_test.go index f850ac916..8bf72fb58 100644 --- a/controllers/apps/operations/ops_progress_util_test.go +++ b/controllers/apps/operations/ops_progress_util_test.go @@ -20,7 +20,6 @@ along with this program. If not, see . package operations import ( - "fmt" "time" . "github.com/onsi/ginkgo/v2" @@ -122,8 +121,7 @@ var _ = Describe("Ops ProgressDetails", func() { By("create restart ops and pods of consensus component") opsRes.OpsRequest = createRestartOpsObj(clusterName, "restart-"+randomStr) - mockComponentIsOperating(opsRes.Cluster, appsv1alpha1.SpecReconcilingClusterCompPhase, consensusComp, statelessComp) // appsv1alpha1.RebootingPhase - // TODO: add RebootingPhase status condition + mockComponentIsOperating(opsRes.Cluster, appsv1alpha1.SpecReconcilingClusterCompPhase, consensusComp, statelessComp) podList := initConsensusPods(ctx, k8sClient, opsRes, clusterName) By("mock restart OpsRequest is Running") @@ -136,13 +134,11 @@ var _ = Describe("Ops ProgressDetails", func() { By("test the progressDetails when stateless pod updates during restart operation") Expect(opsRes.OpsRequest.Status.Components[statelessComp].Phase).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) // appsv1alpha1.RebootingPhase - // TODO: check RebootingPhase status condition testProgressDetailsWithStatelessPodUpdating(reqCtx, opsRes) By("create horizontalScaling operation to test the progressDetails when scaling down the replicas") - opsRes.OpsRequest = createHorizontalScaling(clusterName, 1) + opsRes.OpsRequest = createHorizontalScaling(clusterName, 2) mockComponentIsOperating(opsRes.Cluster, appsv1alpha1.SpecReconcilingClusterCompPhase, consensusComp) // appsv1alpha1.HorizontalScalingPhase - // TODO: add HorizontalScalingPhase status condition initClusterForOps(opsRes) By("mock HorizontalScaling OpsRequest phase is running") @@ -153,18 +149,31 @@ var _ = Describe("Ops ProgressDetails", func() { _, err = GetOpsManager().Do(reqCtx, k8sClient, opsRes) Expect(err).ShouldNot(HaveOccurred()) - By("mock the pod is terminating") - pod := &podList[0] - pod.Kind = constant.PodKind - testk8s.MockPodIsTerminating(ctx, testCtx, pod) + By("mock the pod is terminating, pod[0] is target pod to delete. and mock pod[1] is failed and deleted by stateful controller") + for i := 0; i < 2; i++ { + pod := &podList[i] + pod.Kind = constant.PodKind + testk8s.MockPodIsTerminating(ctx, testCtx, pod) + _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) + Expect(getProgressDetailStatus(opsRes, consensusComp, pod)).Should(Equal(appsv1alpha1.ProcessingProgressStatus)) + + } + By("mock the target pod is deleted and progressDetail status should be succeed") + targetPod := &podList[0] + testk8s.RemovePodFinalizer(ctx, testCtx, targetPod) _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) - Expect(getProgressDetailStatus(opsRes, consensusComp, pod)).Should(Equal(appsv1alpha1.ProcessingProgressStatus)) + Expect(getProgressDetailStatus(opsRes, consensusComp, targetPod)).Should(Equal(appsv1alpha1.SucceedProgressStatus)) + Expect(opsRes.OpsRequest.Status.Progress).Should(Equal("1/2")) - By("mock the pod is deleted and progressDetail status should be succeed") + By("mock the pod[1] to re-create") + pod := &podList[1] testk8s.RemovePodFinalizer(ctx, testCtx, pod) + testapps.MockConsensusComponentStsPod(&testCtx, nil, clusterName, consensusComp, + pod.Name, "Follower", "ReadWrite") + // expect the progress is 2/2 _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) - Expect(getProgressDetailStatus(opsRes, consensusComp, pod)).Should(Equal(appsv1alpha1.SucceedProgressStatus)) - Expect(opsRes.OpsRequest.Status.Progress).Should(Equal("1/2")) + Expect(getProgressDetailStatus(opsRes, consensusComp, targetPod)).Should(Equal(appsv1alpha1.SucceedProgressStatus)) + Expect(opsRes.OpsRequest.Status.Progress).Should(Equal("2/2")) By("create horizontalScaling operation to test the progressDetails when scaling up the replicas ") initClusterForOps(opsRes) @@ -172,6 +181,8 @@ var _ = Describe("Ops ProgressDetails", func() { Expect(testapps.ChangeObj(&testCtx, opsRes.Cluster, func(lcluster *appsv1alpha1.Cluster) { lcluster.Spec.ComponentSpecs[1].Replicas = expectClusterComponentReplicas })).ShouldNot(HaveOccurred()) + // ops will use the startTimestamp to make decision, start time should not equal the pod createTime during testing. + time.Sleep(time.Second) opsRes.OpsRequest = createHorizontalScaling(clusterName, 3) // update ops phase to Running first _, err = GetOpsManager().Do(reqCtx, k8sClient, opsRes) @@ -182,13 +193,11 @@ var _ = Describe("Ops ProgressDetails", func() { Expect(err).ShouldNot(HaveOccurred()) By("test the progressDetails when scaling up replicas") - podName := fmt.Sprintf("%s-%s-%d", clusterName, consensusComp, 0) testapps.MockConsensusComponentStsPod(&testCtx, nil, clusterName, consensusComp, - podName, "leader", "ReadWrite") - pod = &corev1.Pod{} - Expect(k8sClient.Get(ctx, client.ObjectKey{Name: podName, Namespace: testCtx.DefaultNamespace}, pod)).Should(Succeed()) + targetPod.Name, "leader", "ReadWrite") + Expect(k8sClient.Get(ctx, client.ObjectKey{Name: targetPod.Name, Namespace: testCtx.DefaultNamespace}, targetPod)).Should(Succeed()) _, _ = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) - Expect(getProgressDetailStatus(opsRes, consensusComp, pod)).Should(Equal(appsv1alpha1.SucceedProgressStatus)) + Expect(getProgressDetailStatus(opsRes, consensusComp, targetPod)).Should(Equal(appsv1alpha1.SucceedProgressStatus)) Expect(opsRes.OpsRequest.Status.Progress).Should(Equal("1/1")) }) From 4a74d52ae20b9f7e6451345304fc87409ecd771c Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Fri, 19 May 2023 10:54:16 +0800 Subject: [PATCH 325/439] feat: add `--auto-approve` for some cmd (#3241) Co-authored-by: 1aal <1aal@users.noreply.github.com> --- .../cli/kbcli_cluster_delete-account.md | 1 + docs/user_docs/cli/kbcli_cluster_expose.md | 1 + docs/user_docs/cli/kbcli_cluster_hscale.md | 1 + .../cli/kbcli_cluster_reconfigure.md | 5 +++-- docs/user_docs/cli/kbcli_cluster_restart.md | 1 + docs/user_docs/cli/kbcli_cluster_stop.md | 1 + docs/user_docs/cli/kbcli_cluster_upgrade.md | 1 + .../cli/kbcli_cluster_volume-expand.md | 1 + docs/user_docs/cli/kbcli_cluster_vscale.md | 1 + .../user_docs/cli/kbcli_playground_destroy.md | 1 + docs/user_docs/cli/kbcli_playground_init.md | 3 ++- internal/cli/cmd/accounts/delete.go | 6 +++++- internal/cli/cmd/cluster/accounts.go | 1 + internal/cli/cmd/cluster/config_ops.go | 8 ++++++-- internal/cli/cmd/cluster/operations.go | 19 +++++++++++++------ internal/cli/cmd/playground/destroy.go | 17 ++++++++++------- internal/cli/cmd/playground/init.go | 19 +++++++++++++------ 17 files changed, 62 insertions(+), 25 deletions(-) diff --git a/docs/user_docs/cli/kbcli_cluster_delete-account.md b/docs/user_docs/cli/kbcli_cluster_delete-account.md index ea8bcd8f1..b0266f63a 100644 --- a/docs/user_docs/cli/kbcli_cluster_delete-account.md +++ b/docs/user_docs/cli/kbcli_cluster_delete-account.md @@ -22,6 +22,7 @@ kbcli cluster delete-account [flags] ### Options ``` + --auto-approve Skip interactive approval before deleting account --component string Specify the name of component to be connected. If not specified, the first component will be used. -h, --help help for delete-account -i, --instance string Specify the name of instance to be connected. diff --git a/docs/user_docs/cli/kbcli_cluster_expose.md b/docs/user_docs/cli/kbcli_cluster_expose.md index 3e7433225..fb560c03f 100644 --- a/docs/user_docs/cli/kbcli_cluster_expose.md +++ b/docs/user_docs/cli/kbcli_cluster_expose.md @@ -24,6 +24,7 @@ kbcli cluster expose NAME --enable=[true|false] --type=[vpc|internet] [flags] ### Options ``` + --auto-approve Skip interactive approval before exposing the cluster --components strings Component names to this operations --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") --enable string Enable or disable the expose, values can be true or false diff --git a/docs/user_docs/cli/kbcli_cluster_hscale.md b/docs/user_docs/cli/kbcli_cluster_hscale.md index a16d25ed3..ef0d6af45 100644 --- a/docs/user_docs/cli/kbcli_cluster_hscale.md +++ b/docs/user_docs/cli/kbcli_cluster_hscale.md @@ -18,6 +18,7 @@ kbcli cluster hscale NAME [flags] ### Options ``` + --auto-approve Skip interactive approval before horizontally scaling the cluster --components strings Component names to this operations --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for hscale diff --git a/docs/user_docs/cli/kbcli_cluster_reconfigure.md b/docs/user_docs/cli/kbcli_cluster_reconfigure.md index e706ed9e0..13fe15348 100644 --- a/docs/user_docs/cli/kbcli_cluster_reconfigure.md +++ b/docs/user_docs/cli/kbcli_cluster_reconfigure.md @@ -12,16 +12,17 @@ kbcli cluster reconfigure NAME --set key=value[,key=value] [--component=componen ``` # update component params - kbcli cluster configure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=1000,general_log=OFF + kbcli cluster reconfigure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=1000,general_log=OFF # if only one component, and one config spec, and one config file, simplify the use of configure. e.g: # update mysql max_connections, cluster name is mycluster - kbcli cluster configure mycluster --set max_connections=2000 + kbcli cluster reconfigure mycluster --set max_connections=2000 ``` ### Options ``` + --auto-approve Skip interactive approval before reconfigure the cluster --component string Specify the name of Component to be updated. If the cluster has only one component, unset the parameter. --config-file string Specify the name of the configuration file to be updated (e.g. for mysql: --config-file=my.cnf). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. --config-spec string Specify the name of the configuration template to be updated (e.g. for apecloud-mysql: --config-spec=mysql-3node-tpl). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. diff --git a/docs/user_docs/cli/kbcli_cluster_restart.md b/docs/user_docs/cli/kbcli_cluster_restart.md index 226376b04..b93a61c61 100644 --- a/docs/user_docs/cli/kbcli_cluster_restart.md +++ b/docs/user_docs/cli/kbcli_cluster_restart.md @@ -21,6 +21,7 @@ kbcli cluster restart NAME [flags] ### Options ``` + --auto-approve Skip interactive approval before restarting the cluster --components strings Component names to this operations --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for restart diff --git a/docs/user_docs/cli/kbcli_cluster_stop.md b/docs/user_docs/cli/kbcli_cluster_stop.md index a1accbced..1d0a4de7b 100644 --- a/docs/user_docs/cli/kbcli_cluster_stop.md +++ b/docs/user_docs/cli/kbcli_cluster_stop.md @@ -18,6 +18,7 @@ kbcli cluster stop NAME [flags] ### Options ``` + --auto-approve Skip interactive approval before stopping the cluster --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for stop --name string OpsRequest name. if not specified, it will be randomly generated diff --git a/docs/user_docs/cli/kbcli_cluster_upgrade.md b/docs/user_docs/cli/kbcli_cluster_upgrade.md index 950d41fb7..d005e6a0d 100644 --- a/docs/user_docs/cli/kbcli_cluster_upgrade.md +++ b/docs/user_docs/cli/kbcli_cluster_upgrade.md @@ -18,6 +18,7 @@ kbcli cluster upgrade NAME [flags] ### Options ``` + --auto-approve Skip interactive approval before upgrading the cluster --cluster-version string Reference cluster version (required) --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for upgrade diff --git a/docs/user_docs/cli/kbcli_cluster_volume-expand.md b/docs/user_docs/cli/kbcli_cluster_volume-expand.md index eba0c516c..fc7639208 100644 --- a/docs/user_docs/cli/kbcli_cluster_volume-expand.md +++ b/docs/user_docs/cli/kbcli_cluster_volume-expand.md @@ -18,6 +18,7 @@ kbcli cluster volume-expand NAME [flags] ### Options ``` + --auto-approve Skip interactive approval before expanding the cluster volume --components strings Component names to this operations --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") -h, --help help for volume-expand diff --git a/docs/user_docs/cli/kbcli_cluster_vscale.md b/docs/user_docs/cli/kbcli_cluster_vscale.md index 34734b8a5..fdb0a3562 100644 --- a/docs/user_docs/cli/kbcli_cluster_vscale.md +++ b/docs/user_docs/cli/kbcli_cluster_vscale.md @@ -21,6 +21,7 @@ kbcli cluster vscale NAME [flags] ### Options ``` + --auto-approve Skip interactive approval before vertically scaling the cluster --class string Component class --components strings Component names to this operations --cpu string Requested and limited size of component cpu diff --git a/docs/user_docs/cli/kbcli_playground_destroy.md b/docs/user_docs/cli/kbcli_playground_destroy.md index 8c621adac..abaf7751b 100644 --- a/docs/user_docs/cli/kbcli_playground_destroy.md +++ b/docs/user_docs/cli/kbcli_playground_destroy.md @@ -18,6 +18,7 @@ kbcli playground destroy [flags] ### Options ``` + --auto-approve Skip interactive approval before destroying the playground -h, --help help for destroy --purge Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks. (default true) --timeout duration Time to wait for installing KubeBlocks, such as --timeout=10m (default 30m0s) diff --git a/docs/user_docs/cli/kbcli_playground_init.md b/docs/user_docs/cli/kbcli_playground_init.md index 1877e636b..2c79a715b 100644 --- a/docs/user_docs/cli/kbcli_playground_init.md +++ b/docs/user_docs/cli/kbcli_playground_init.md @@ -30,12 +30,13 @@ kbcli playground init [flags] ### Options ``` + --auto-approve Skip interactive approval during the initialization of playground --cloud-provider string Cloud provider type, one of [local aws gcp alicloud tencentcloud] (default "local") --cluster-definition string Cluster definition (default "apecloud-mysql") --cluster-version string Cluster definition -h, --help help for init --region string The region to create kubernetes cluster - --timeout duration Time to wait for init playground, such as --timeout=10m (default 5m0s) + --timeout duration Time to wait for initing playground, such as --timeout=10m (default 5m0s) --version string KubeBlocks version ``` diff --git a/internal/cli/cmd/accounts/delete.go b/internal/cli/cmd/accounts/delete.go index abb4d9e41..de4220ae4 100644 --- a/internal/cli/cmd/accounts/delete.go +++ b/internal/cli/cmd/accounts/delete.go @@ -30,7 +30,8 @@ import ( type DeleteUserOptions struct { *AccountBaseOptions - info sqlchannel.UserInfo + info sqlchannel.UserInfo + AutoApprove bool } func NewDeleteUserOptions(f cmdutil.Factory, streams genericclioptions.IOStreams) *DeleteUserOptions { @@ -51,6 +52,9 @@ func (o *DeleteUserOptions) Validate(args []string) error { if len(o.info.UserName) == 0 { return errMissingUserName } + if o.AutoApprove { + return nil + } if err := delete.Confirm([]string{o.info.UserName}, o.In); err != nil { return err } diff --git a/internal/cli/cmd/cluster/accounts.go b/internal/cli/cmd/cluster/accounts.go index 11622b46e..40ec734df 100644 --- a/internal/cli/cmd/cluster/accounts.go +++ b/internal/cli/cmd/cluster/accounts.go @@ -118,6 +118,7 @@ func NewDeleteAccountCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) }, } o.AddFlags(cmd) + cmd.Flags().BoolVar(&o.AutoApprove, "auto-approve", false, "Skip interactive approval before deleting account") return cmd } diff --git a/internal/cli/cmd/cluster/config_ops.go b/internal/cli/cmd/cluster/config_ops.go index 1352bc52b..4f7008dd4 100644 --- a/internal/cli/cmd/cluster/config_ops.go +++ b/internal/cli/cmd/cluster/config_ops.go @@ -53,11 +53,11 @@ type configOpsOptions struct { var ( createReconfigureExample = templates.Examples(` # update component params - kbcli cluster configure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=1000,general_log=OFF + kbcli cluster reconfigure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=1000,general_log=OFF # if only one component, and one config spec, and one config file, simplify the use of configure. e.g: # update mysql max_connections, cluster name is mycluster - kbcli cluster configure mycluster --set max_connections=2000 + kbcli cluster reconfigure mycluster --set max_connections=2000 `) ) @@ -152,6 +152,9 @@ func (o *configOpsOptions) checkChangedParamsAndDoubleConfirm(cc *appsv1alpha1.C } func (o *configOpsOptions) confirmReconfigureWithRestart() error { + if o.autoApprove { + return nil + } const confirmStr = "yes" printer.Warning(o.Out, restartConfirmPrompt) _, err := prompt.NewPrompt(fmt.Sprintf("Please type \"%s\" to confirm:", confirmStr), @@ -221,5 +224,6 @@ func NewReconfigureCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) * }, } o.buildReconfigureCommonFlags(cmd) + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before reconfigure the cluster") return cmd } diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index fbc6724be..f077075a7 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -47,8 +47,8 @@ import ( type OperationsOptions struct { create.CreateOptions `json:"-"` HasComponentNamesFlag bool `json:"-"` - // RequireConfirm if it is true, the second verification will be performed before creating ops. - RequireConfirm bool `json:"-"` + // autoApprove if it is true, will skip the double check. + autoApprove bool `json:"-"` ComponentNames []string `json:"componentNames,omitempty"` OpsRequestName string `json:"opsRequestName"` TTLSecondsAfterSucceed int `json:"ttlSecondsAfterSucceed"` @@ -100,7 +100,7 @@ func newBaseOperationsOptions(f cmdutil.Factory, streams genericclioptions.IOStr KeyValues: map[string]string{}, OpsType: opsType, HasComponentNamesFlag: hasComponentNamesFlag, - RequireConfirm: true, + autoApprove: false, CreateOptions: create.CreateOptions{ Factory: f, IOStreams: streams, @@ -268,7 +268,7 @@ func (o *OperationsOptions) Validate() error { return err } } - if o.RequireConfirm && o.DryRun == "none" { + if !o.autoApprove && o.DryRun == "none" { return delete.Confirm([]string{o.Name}, o.In) } return nil @@ -385,6 +385,7 @@ func NewRestartCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr }, } o.addCommonFlags(cmd) + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before restarting the cluster") return cmd } @@ -410,6 +411,7 @@ func NewUpgradeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr } o.addCommonFlags(cmd) cmd.Flags().StringVar(&o.ClusterVersionRef, "cluster-version", "", "Reference cluster version (required)") + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before upgrading the cluster") return cmd } @@ -441,6 +443,7 @@ func NewVerticalScalingCmd(f cmdutil.Factory, streams genericclioptions.IOStream cmd.Flags().StringVar(&o.CPU, "cpu", "", "Requested and limited size of component cpu") cmd.Flags().StringVar(&o.Memory, "memory", "", "Requested and limited size of component memory") cmd.Flags().StringVar(&o.Class, "class", "", "Component class") + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before vertically scaling the cluster") return cmd } @@ -468,6 +471,7 @@ func NewHorizontalScalingCmd(f cmdutil.Factory, streams genericclioptions.IOStre o.addCommonFlags(cmd) cmd.Flags().IntVar(&o.Replicas, "replicas", o.Replicas, "Replicas with the specified components") + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before horizontally scaling the cluster") _ = cmd.MarkFlagRequired("replicas") return cmd } @@ -477,7 +481,7 @@ var volumeExpansionExample = templates.Examples(` kbcli cluster volume-expand mycluster --components=mysql --volume-claim-templates=data --storage=10Gi `) -// NewVolumeExpansionCmd creates a vertical scaling command +// NewVolumeExpansionCmd creates a volume expanding command func NewVolumeExpansionCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := newBaseOperationsOptions(f, streams, appsv1alpha1.VolumeExpansionType, true) cmd := &cobra.Command{ @@ -496,6 +500,7 @@ func NewVolumeExpansionCmd(f cmdutil.Factory, streams genericclioptions.IOStream o.addCommonFlags(cmd) cmd.Flags().StringSliceVarP(&o.VCTNames, "volume-claim-templates", "t", nil, "VolumeClaimTemplate names in components (required)") cmd.Flags().StringVar(&o.Storage, "storage", "", "Volume storage size (required)") + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before expanding the cluster volume") return cmd } @@ -531,6 +536,7 @@ func NewExposeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra o.addCommonFlags(cmd) cmd.Flags().StringVar(&o.ExposeType, "type", "", "Expose type, currently supported types are 'vpc', 'internet'") cmd.Flags().StringVar(&o.ExposeEnabled, "enable", "", "Enable or disable the expose, values can be true or false") + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before exposing the cluster") util.CheckErr(cmd.RegisterFlagCompletionFunc("type", func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { return []string{string(util.ExposeToVPC), string(util.ExposeToInternet)}, cobra.ShellCompDirectiveNoFileComp @@ -564,6 +570,7 @@ func NewStopCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C }, } o.addCommonFlags(cmd) + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before stopping the cluster") return cmd } @@ -575,7 +582,7 @@ var startExample = templates.Examples(` // NewStartCmd creates a start command func NewStartCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := newBaseOperationsOptions(f, streams, appsv1alpha1.StartType, false) - o.RequireConfirm = false + o.autoApprove = true cmd := &cobra.Command{ Use: "start NAME", Short: "Start the cluster if cluster is stopped.", diff --git a/internal/cli/cmd/playground/destroy.go b/internal/cli/cmd/playground/destroy.go index c81497cae..fb69727e9 100644 --- a/internal/cli/cmd/playground/destroy.go +++ b/internal/cli/cmd/playground/destroy.go @@ -62,8 +62,9 @@ type destroyOptions struct { // purge resources, before destroy kubernetes cluster we should delete cluster and // uninstall KubeBlocks - purge bool - timeout time.Duration + autoApprove bool + purge bool + timeout time.Duration } func newDestroyCmd(streams genericclioptions.IOStreams) *cobra.Command { @@ -82,7 +83,7 @@ func newDestroyCmd(streams genericclioptions.IOStreams) *cobra.Command { cmd.Flags().BoolVar(&o.purge, "purge", true, "Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks.") cmd.Flags().DurationVar(&o.timeout, "timeout", 1800*time.Second, "Time to wait for installing KubeBlocks, such as --timeout=10m") - + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before destroying the playground") return cmd } @@ -143,10 +144,12 @@ func (o *destroyOptions) destroyCloud() error { o.prevCluster.ClusterName, o.prevCluster.String()) // confirm to destroy - entered, _ := prompt.NewPrompt("Enter a value:", nil, o.In).Run() - if entered != yesStr { - fmt.Fprintf(o.Out, "\nPlayground destroy cancelled.\n") - return cmdutil.ErrExit + if !o.autoApprove { + entered, _ := prompt.NewPrompt("Enter a value:", nil, o.In).Run() + if entered != yesStr { + fmt.Fprintf(o.Out, "\nPlayground destroy cancelled.\n") + return cmdutil.ErrExit + } } o.startTime = time.Now() diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index cc08ac5dd..a853f5c2e 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -82,6 +82,7 @@ type initOptions struct { clusterVersion string cloudProvider string region string + autoApprove bool baseOptions } @@ -106,7 +107,8 @@ func newInitCmd(streams genericclioptions.IOStreams) *cobra.Command { cmd.Flags().StringVar(&o.kbVersion, "version", version.DefaultKubeBlocksVersion, "KubeBlocks version") cmd.Flags().StringVar(&o.cloudProvider, "cloud-provider", defaultCloudProvider, fmt.Sprintf("Cloud provider type, one of %v", supportedCloudProviders)) cmd.Flags().StringVar(&o.region, "region", "", "The region to create kubernetes cluster") - cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for init playground, such as --timeout=10m") + cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for initing playground, such as --timeout=10m") + cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval during the initialization of playground") util.CheckErr(cmd.RegisterFlagCompletionFunc( "cloud-provider", @@ -263,11 +265,13 @@ func (o *initOptions) cloud() error { // confirmToContinue confirms to continue init or not if there is an existed kubernetes cluster func (o *initOptions) confirmToContinue() error { clusterName := o.prevCluster.ClusterName - printer.Warning(o.Out, "Found an existed cluster %s, do you want to continue to initialize this cluster?\n Only 'yes' will be accepted to confirm.\n\n", clusterName) - entered, _ := prompt.NewPrompt("Enter a value:", nil, o.In).Run() - if entered != yesStr { - fmt.Fprintf(o.Out, "\nPlayground init cancelled, please destroy the old cluster first.\n") - return cmdutil.ErrExit + if !o.autoApprove { + printer.Warning(o.Out, "Found an existed cluster %s, do you want to continue to initialize this cluster?\n Only 'yes' will be accepted to confirm.\n\n", clusterName) + entered, _ := prompt.NewPrompt("Enter a value:", nil, o.In).Run() + if entered != yesStr { + fmt.Fprintf(o.Out, "\nPlayground init cancelled, please destroy the old cluster first.\n") + return cmdutil.ErrExit + } } fmt.Fprintf(o.Out, "Continue to initialize %s %s cluster %s... \n", o.cloudProvider, cp.K8sService(o.cloudProvider), clusterName) @@ -285,6 +289,9 @@ The whole process will take about %s, please wait patiently, if it takes a long time, please check the network environment and try again. `, printer.BoldRed("20 minutes")) + if o.autoApprove { + return nil + } // confirm to run fmt.Fprintf(o.Out, "\nDo you want to perform this action?\n Only 'yes' will be accepted to approve.\n\n") entered, _ := prompt.NewPrompt("Enter a value:", nil, o.In).Run() From 5ff67f98439c5b267db188ee8c139ad14f38401d Mon Sep 17 00:00:00 2001 From: dingben Date: Fri, 19 May 2023 15:33:00 +0800 Subject: [PATCH 326/439] feat: addon association cli plugin (#3269) --- apis/extensions/v1alpha1/addon_types.go | 18 +++++ .../extensions.kubeblocks.io_addons.yaml | 18 +++++ .../crds/extensions.kubeblocks.io_addons.yaml | 18 +++++ internal/cli/cmd/addon/addon.go | 66 +++++++++++++++++++ internal/cli/cmd/plugin/index.go | 5 +- internal/cli/cmd/plugin/install.go | 13 ++-- internal/cli/cmd/plugin/plugin.go | 50 +++++++------- internal/cli/cmd/plugin/upgrade.go | 12 ++-- 8 files changed, 164 insertions(+), 36 deletions(-) diff --git a/apis/extensions/v1alpha1/addon_types.go b/apis/extensions/v1alpha1/addon_types.go index 93d62ab52..30d556f3f 100644 --- a/apis/extensions/v1alpha1/addon_types.go +++ b/apis/extensions/v1alpha1/addon_types.go @@ -60,6 +60,10 @@ type AddonSpec struct { // Addon installable spec., provide selector and auto-install settings. // +optional Installable *InstallableSpec `json:"installable,omitempty"` + + // Plugin installation spec. + // +optional + CliPlugins []CliPlugin `json:"cliPlugins,omitempty"` } // AddonStatus defines the observed state of Addon @@ -233,6 +237,20 @@ type ResourceMappingItem struct { Memory *ResourceReqLimItem `json:"memory,omitempty"` } +type CliPlugin struct { + // Name of the plugin. + // +kubebuilder:validation:Required + Name string `json:"name"` + + // The index repository of the plugin. + // +kubebuilder:validation:Required + IndexRepository string `json:"indexRepository"` + + // The description of the plugin. + // +optional + Description string `json:"description,omitempty"` +} + func (r *ResourceMappingItem) HasStorageMapping() bool { return !(r == nil || r.Storage == "") } diff --git a/config/crd/bases/extensions.kubeblocks.io_addons.yaml b/config/crd/bases/extensions.kubeblocks.io_addons.yaml index a17aa6fcd..b33c58417 100644 --- a/config/crd/bases/extensions.kubeblocks.io_addons.yaml +++ b/config/crd/bases/extensions.kubeblocks.io_addons.yaml @@ -49,6 +49,24 @@ spec: spec: description: AddonSpec defines the desired state of Addon properties: + cliPlugins: + description: Plugin installation spec. + items: + properties: + description: + description: The description of the plugin. + type: string + indexRepository: + description: The index repository of the plugin. + type: string + name: + description: Name of the plugin. + type: string + required: + - indexRepository + - name + type: object + type: array defaultInstallValues: description: Default installation parameters. items: diff --git a/deploy/helm/crds/extensions.kubeblocks.io_addons.yaml b/deploy/helm/crds/extensions.kubeblocks.io_addons.yaml index a17aa6fcd..b33c58417 100644 --- a/deploy/helm/crds/extensions.kubeblocks.io_addons.yaml +++ b/deploy/helm/crds/extensions.kubeblocks.io_addons.yaml @@ -49,6 +49,24 @@ spec: spec: description: AddonSpec defines the desired state of Addon properties: + cliPlugins: + description: Plugin installation spec. + items: + properties: + description: + description: The description of the plugin. + type: string + indexRepository: + description: The index repository of the plugin. + type: string + name: + description: Name of the plugin. + type: string + required: + - indexRepository + - name + type: object + type: array defaultInstallValues: description: Default installation parameters. items: diff --git a/internal/cli/cmd/addon/addon.go b/internal/cli/cmd/addon/addon.go index 34159143d..ee29d21ce 100644 --- a/internal/cli/cmd/addon/addon.go +++ b/internal/cli/cmd/addon/addon.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "fmt" + "path" "sort" "strconv" "strings" @@ -44,6 +45,7 @@ import ( "k8s.io/utils/strings/slices" extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/cmd/plugin" "github.com/apecloud/kubeblocks/internal/cli/list" "github.com/apecloud/kubeblocks/internal/cli/patch" "github.com/apecloud/kubeblocks/internal/cli/printer" @@ -305,6 +307,11 @@ func (o *addonCmdOpts) validate() error { return fmt.Errorf("addon %s INSTALLABLE-SELECTOR has no matching requirement", o.Names) } } + + if err := o.installAndUpgradePlugins(); err != nil { + fmt.Fprintf(o.Out, "failed to install/upgrade plugins: %v\n", err) + } + return nil } @@ -798,3 +805,62 @@ func addonListRun(o *list.ListOptions) error { } return nil } + +func (o *addonCmdOpts) installAndUpgradePlugins() error { + if len(o.addon.Spec.CliPlugins) == 0 { + return nil + } + + plugin.InitPlugin() + + paths := plugin.GetKbcliPluginPath() + indexes, err := plugin.ListIndexes(paths) + if err != nil { + return err + } + + indexRepositoryToNme := make(map[string]string) + for _, index := range indexes { + indexRepositoryToNme[index.URL] = index.Name + } + + var plugins []string + var names []string + for _, p := range o.addon.Spec.CliPlugins { + names = append(names, p.Name) + indexName, ok := indexRepositoryToNme[p.IndexRepository] + if !ok { + // index not found, add it + _, indexName = path.Split(p.IndexRepository) + if err := plugin.AddIndex(paths, indexName, p.IndexRepository); err != nil { + return err + } + } + plugins = append(plugins, fmt.Sprintf("%s/%s", indexName, p.Name)) + } + + installOption := &plugin.PluginInstallOption{ + IOStreams: o.IOStreams, + } + upgradeOption := &plugin.UpgradeOptions{ + IOStreams: o.IOStreams, + } + + // install plugins + if err := installOption.Complete(plugins); err != nil { + return err + } + if err := installOption.Install(); err != nil { + return err + } + + // upgrade existed plugins + if err := upgradeOption.Complete(names); err != nil { + return err + } + if err := upgradeOption.Run(); err != nil { + return err + } + + return nil +} diff --git a/internal/cli/cmd/plugin/index.go b/internal/cli/cmd/plugin/index.go index c364bad06..8e8def42d 100644 --- a/internal/cli/cmd/plugin/index.go +++ b/internal/cli/cmd/plugin/index.go @@ -235,13 +235,16 @@ func ListIndexes(paths *Paths) ([]Index, error) { // AddIndex initializes a new index to install plugins from. func AddIndex(paths *Paths, name, url string) error { + if name == "" { + return errors.New("index name must be specified") + } dir := paths.IndexPath(name) if _, err := os.Stat(dir); os.IsNotExist(err) { return util.EnsureCloned(url, dir) } else if err != nil { return err } - return errors.New("index already exists") + return fmt.Errorf("index %q already exists", name) } // DeleteIndex removes specified index name. If index does not exist, returns an error that can be tested by os.IsNotExist. diff --git a/internal/cli/cmd/plugin/install.go b/internal/cli/cmd/plugin/install.go index 007d46779..068044fdf 100755 --- a/internal/cli/cmd/plugin/install.go +++ b/internal/cli/cmd/plugin/install.go @@ -45,7 +45,7 @@ var ( `) ) -type pluginInstallOption struct { +type PluginInstallOption struct { plugins []pluginEntry genericclioptions.IOStreams @@ -57,7 +57,7 @@ type pluginEntry struct { } func NewPluginInstallCmd(streams genericclioptions.IOStreams) *cobra.Command { - o := &pluginInstallOption{ + o := &PluginInstallOption{ IOStreams: streams, } cmd := &cobra.Command{ @@ -65,14 +65,14 @@ func NewPluginInstallCmd(streams genericclioptions.IOStreams) *cobra.Command { Short: "Install kbcli or kubectl plugins", Example: pluginInstallExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(o.complete(args)) - cmdutil.CheckErr(o.install()) + cmdutil.CheckErr(o.Complete(args)) + cmdutil.CheckErr(o.Install()) }, } return cmd } -func (o *pluginInstallOption) complete(names []string) error { +func (o *PluginInstallOption) Complete(names []string) error { for _, name := range names { indexName, pluginName := CanonicalPluginName(name) plugin, err := LoadPluginByName(paths.IndexPluginsPath(indexName), pluginName) @@ -90,7 +90,7 @@ func (o *pluginInstallOption) complete(names []string) error { return nil } -func (o *pluginInstallOption) install() error { +func (o *PluginInstallOption) Install() error { var failed []string var returnErr error for _, entry := range o.plugins { @@ -98,7 +98,6 @@ func (o *pluginInstallOption) install() error { fmt.Fprintf(o.Out, "Installing plugin: %s\n", plugin.Name) err := Install(paths, plugin, entry.index, InstallOpts{}) if err == ErrIsAlreadyInstalled { - klog.Warningf("Skipping plugin %q, it is already installed", plugin.Name) continue } if err != nil { diff --git a/internal/cli/cmd/plugin/plugin.go b/internal/cli/cmd/plugin/plugin.go index 2cf4caf98..b575ccb14 100644 --- a/internal/cli/cmd/plugin/plugin.go +++ b/internal/cli/cmd/plugin/plugin.go @@ -25,7 +25,6 @@ import ( "io" "os" "path/filepath" - "runtime" "strings" "github.com/spf13/cobra" @@ -35,6 +34,7 @@ import ( "k8s.io/kubectl/pkg/util/templates" "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/util" ) var ( @@ -58,26 +58,9 @@ func NewPluginCmd(streams genericclioptions.IOStreams) *cobra.Command { Use: "plugin", Short: "Provides utilities for interacting with plugins.", Long: pluginLong, - } - - if err := EnsureDirs(paths.BasePath(), - paths.BinPath(), - paths.InstallPath(), - paths.IndexBase(), - paths.InstallReceiptsPath()); err != nil { - klog.Fatal(err) - } - - // check if index exist, if indexes don't exist, download default index - indexes, err := ListIndexes(paths) - if err != nil { - klog.Fatal(err) - } - if len(indexes) == 0 { - klog.Info("start download default index") - if err := AddIndex(paths, DefaultIndexName, DefaultIndexURI); err != nil { - klog.Fatal("failed to download default index", err) - } + PersistentPreRun: func(cmd *cobra.Command, args []string) { + InitPlugin() + }, } cmd.AddCommand( @@ -257,7 +240,7 @@ func isExecutable(fullPath string) (bool, error) { return false, err } - if runtime.GOOS == "windows" { + if util.IsWindows() { fileExt := strings.ToLower(filepath.Ext(fullPath)) switch fileExt { @@ -305,3 +288,26 @@ func NewPluginPrinter(out io.Writer) *printer.TablePrinter { func addPluginRow(name, path string, p *printer.TablePrinter) { p.AddRow(name, path) } + +func InitPlugin() { + // Ensure that the base directories exist + if err := EnsureDirs(paths.BasePath(), + paths.BinPath(), + paths.InstallPath(), + paths.IndexBase(), + paths.InstallReceiptsPath()); err != nil { + klog.Fatal(err) + } + + // check if index exist, if indexes don't exist, download default index + indexes, err := ListIndexes(paths) + if err != nil { + klog.Fatal(err) + } + if len(indexes) == 0 { + klog.V(1).Info("no indexes found, downloading default index") + if err := AddIndex(paths, DefaultIndexName, DefaultIndexURI); err != nil { + klog.Fatal("failed to download default index", err) + } + } +} diff --git a/internal/cli/cmd/plugin/upgrade.go b/internal/cli/cmd/plugin/upgrade.go index 8e6029887..8b224c8c9 100644 --- a/internal/cli/cmd/plugin/upgrade.go +++ b/internal/cli/cmd/plugin/upgrade.go @@ -43,7 +43,7 @@ var ( `) ) -type upgradeOptions struct { +type UpgradeOptions struct { // common user flags all bool @@ -52,7 +52,7 @@ type upgradeOptions struct { } func NewPluginUpgradeCmd(streams genericclioptions.IOStreams) *cobra.Command { - o := &upgradeOptions{ + o := &UpgradeOptions{ IOStreams: streams, } @@ -61,8 +61,8 @@ func NewPluginUpgradeCmd(streams genericclioptions.IOStreams) *cobra.Command { Short: "Upgrade kbcli or kubectl plugins", Example: pluginUpgradeExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(o.complete(args)) - cmdutil.CheckErr(o.run()) + cmdutil.CheckErr(o.Complete(args)) + cmdutil.CheckErr(o.Run()) }, } @@ -71,7 +71,7 @@ func NewPluginUpgradeCmd(streams genericclioptions.IOStreams) *cobra.Command { return cmd } -func (o *upgradeOptions) complete(args []string) error { +func (o *UpgradeOptions) Complete(args []string) error { if o.all { installed, err := GetInstalledPluginReceipts(paths.InstallReceiptsPath()) if err != nil { @@ -96,7 +96,7 @@ func (o *upgradeOptions) complete(args []string) error { return nil } -func (o *upgradeOptions) run() error { +func (o *UpgradeOptions) Run() error { for _, name := range o.pluginNames { indexName, pluginName := CanonicalPluginName(name) From 7606e14c997629562152bdc2e94f0032fa12b790 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Fri, 19 May 2023 15:40:29 +0800 Subject: [PATCH 327/439] perf: add reconfigure if backup policy schedule enable updated (#3298) --- .../v1alpha1/backuppolicy_types.go | 12 ++ .../dataprotection/backuppolicy_controller.go | 124 ++++++++++++++++++ .../backuppolicy_controller_test.go | 48 ++++++- deploy/postgresql/config/pg12-config.tpl | 3 +- deploy/postgresql/config/pg14-config.tpl | 3 +- .../templates/backuppolicytemplate.yaml | 12 ++ .../postgresql/templates/backuptool-pitr.yaml | 13 +- deploy/postgresql/templates/scripts.yaml | 7 +- internal/constant/const.go | 1 + .../transformer_backup_policy_tpl.go | 22 ++-- internal/controllerutil/errors.go | 3 + 11 files changed, 227 insertions(+), 21 deletions(-) diff --git a/apis/dataprotection/v1alpha1/backuppolicy_types.go b/apis/dataprotection/v1alpha1/backuppolicy_types.go index 8dc80baad..5a8ee56e9 100644 --- a/apis/dataprotection/v1alpha1/backuppolicy_types.go +++ b/apis/dataprotection/v1alpha1/backuppolicy_types.go @@ -312,6 +312,18 @@ func (r *BackupPolicySpec) GetCommonPolicy(backupType BackupType) *CommonBackupP return nil } +func (r *BackupPolicySpec) GetCommonSchedulePolicy(backupType BackupType) *SchedulePolicy { + switch backupType { + case BackupTypeSnapshot: + return r.Schedule.Snapshot + case BackupTypeDataFile: + return r.Schedule.Datafile + case BackupTypeLogFile: + return r.Schedule.Logfile + } + return nil +} + // ToDuration converts the ttl string to time.Duration. func ToDuration(ttl *string) time.Duration { if ttl == nil { diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index 63ccfd57d..630003666 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" @@ -183,6 +184,9 @@ func (r *BackupPolicyReconciler) patchStatusFailed(reqCtx intctrlutil.RequestCtx backupPolicy *dataprotectionv1alpha1.BackupPolicy, reason string, err error) (ctrl.Result, error) { + if intctrlutil.IsTargetError(err, intctrlutil.ErrorTypeRequeue) { + return intctrlutil.RequeueAfter(reconcileInterval, reqCtx.Log, "") + } backupPolicyDeepCopy := backupPolicy.DeepCopy() backupPolicy.Status.Phase = dataprotectionv1alpha1.PolicyFailed backupPolicy.Status.FailureReason = err.Error() @@ -382,6 +386,10 @@ func (r *BackupPolicyReconciler) handlePolicy(reqCtx intctrlutil.RequestCtx, basePolicy dataprotectionv1alpha1.BasePolicy, cronExpression string, backType dataprotectionv1alpha1.BackupType) error { + + if err := r.reconfigure(reqCtx, backupPolicy, basePolicy, backType); err != nil { + return err + } // create/delete/patch cronjob workload if err := r.reconcileCronJob(reqCtx, backupPolicy, basePolicy, cronExpression, backType); err != nil { @@ -455,3 +463,119 @@ func (r *BackupPolicyReconciler) setGlobalPersistentVolumeClaim(backupPolicy *da backupPolicy.PersistentVolumeClaim.InitCapacity = resource.MustParse(globalInitCapacity) } } + +type backupReconfigureRef struct { + Name string `json:"name"` + Key string `json:"key"` + Enable parameterPairs `json:"enable,omitempty"` + Disable parameterPairs `json:"disable,omitempty"` +} + +type parameterPairs map[string][]appsv1alpha1.ParameterPair + +func (r *BackupPolicyReconciler) reconfigure(reqCtx intctrlutil.RequestCtx, + backupPolicy *dataprotectionv1alpha1.BackupPolicy, + basePolicy dataprotectionv1alpha1.BasePolicy, + backType dataprotectionv1alpha1.BackupType) error { + + reconfigRef := backupPolicy.Annotations[constant.ReconfigureRefAnnotationKey] + if reconfigRef == "" { + return nil + } + configRef := backupReconfigureRef{} + if err := json.Unmarshal([]byte(reconfigRef), &configRef); err != nil { + return err + } + + enable := false + commonSchedule := backupPolicy.Spec.GetCommonSchedulePolicy(backType) + if commonSchedule != nil { + enable = commonSchedule.Enable + } + if backupPolicy.Annotations[constant.LastAppliedConfigAnnotation] == "" && !enable { + // disable in the first policy created, no need reconfigure because default configs had been set. + return nil + } + configParameters := configRef.Disable + if enable { + configParameters = configRef.Enable + } + if configParameters == nil { + return nil + } + parameters := configParameters[string(backType)] + if len(parameters) == 0 { + // skip reconfigure if not found parameters. + return nil + } + updateParameterPairsBytes, _ := json.Marshal(parameters) + updateParameterPairs := string(updateParameterPairsBytes) + if updateParameterPairs == backupPolicy.Annotations[constant.LastAppliedConfigAnnotation] { + // reconcile the config job if finished + return r.reconcileReconfigure(reqCtx, backupPolicy) + } + + ops := appsv1alpha1.OpsRequest{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: backupPolicy.Name + "-", + Namespace: backupPolicy.Namespace, + Labels: map[string]string{ + dataProtectionLabelBackupPolicyKey: backupPolicy.Name, + }, + }, + Spec: appsv1alpha1.OpsRequestSpec{ + Type: appsv1alpha1.ReconfiguringType, + ClusterRef: basePolicy.Target.LabelsSelector.MatchLabels[constant.AppInstanceLabelKey], + Reconfigure: &appsv1alpha1.Reconfigure{ + ComponentOps: appsv1alpha1.ComponentOps{ + ComponentName: basePolicy.Target.LabelsSelector.MatchLabels[constant.KBAppComponentLabelKey], + }, + Configurations: []appsv1alpha1.Configuration{ + { + Name: configRef.Name, + Keys: []appsv1alpha1.ParameterConfig{ + { + Key: configRef.Key, + Parameters: parameters, + }, + }, + }, + }, + }, + }, + } + if err := r.Client.Create(reqCtx.Ctx, &ops); err != nil { + return err + } + + r.Recorder.Eventf(backupPolicy, corev1.EventTypeNormal, "Reconfiguring", "update config %s", updateParameterPairs) + patch := client.MergeFrom(backupPolicy.DeepCopy()) + if backupPolicy.Annotations == nil { + backupPolicy.Annotations = map[string]string{} + } + backupPolicy.Annotations[constant.LastAppliedConfigAnnotation] = updateParameterPairs + return r.Client.Patch(reqCtx.Ctx, backupPolicy, patch) +} + +func (r *BackupPolicyReconciler) reconcileReconfigure(reqCtx intctrlutil.RequestCtx, + backupPolicy *dataprotectionv1alpha1.BackupPolicy) error { + + opsList := appsv1alpha1.OpsRequestList{} + if err := r.Client.List(reqCtx.Ctx, &opsList, + client.InNamespace(backupPolicy.Namespace), + client.MatchingLabels{dataProtectionLabelBackupPolicyKey: backupPolicy.Name}); err != nil { + return err + } + if len(opsList.Items) > 0 { + sort.Slice(opsList.Items, func(i, j int) bool { + return opsList.Items[j].CreationTimestamp.Before(&opsList.Items[i].CreationTimestamp) + }) + latestOps := opsList.Items[0] + if latestOps.Status.Phase == appsv1alpha1.OpsFailedPhase { + return intctrlutil.NewErrorf(intctrlutil.ErrorTypeReconfigureFailed, "ops failed %s", latestOps.Name) + } else if latestOps.Status.Phase != appsv1alpha1.OpsSucceedPhase { + return intctrlutil.NewErrorf(intctrlutil.ErrorTypeRequeue, "requeue to waiting ops %s finished.", latestOps.Name) + } + } + return nil +} diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index 344528408..b0e95eca1 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -302,7 +302,7 @@ var _ = Describe("Backup Policy Controller", func() { }) Context("creating a backupPolicy with global backup config", func() { - It("ccreating a backupPolicy with global backup config", func() { + It("creating a backupPolicy with global backup config", func() { By("By creating a backupPolicy with empty secret") pvcName := "backup-data" pvcInitCapacity := "10Gi" @@ -323,6 +323,52 @@ var _ = Describe("Backup Policy Controller", func() { })).Should(Succeed()) }) }) + Context("creating a logfile backupPolicy", func() { + It("with reconfigure config", func() { + By("creating a backupPolicy") + pvcName := "backup-data" + pvcInitCapacity := "10Gi" + viper.SetDefault(constant.CfgKeyBackupPVCName, pvcName) + viper.SetDefault(constant.CfgKeyBackupPVCInitCapacity, pvcInitCapacity) + reconfigureRef := `{ + "name": "postgresql-configuration", + "key": "postgresql.conf", + "enable": { + "logfile": [{"key":"archive_command","value":"''"}] + }, + "disable": { + "logfile": [{"key": "archive_command","value":"'/bin/true'"}] + } + }` + backupPolicy := testapps.NewBackupPolicyFactory(testCtx.DefaultNamespace, backupPolicyName). + AddAnnotations(constant.ReconfigureRefAnnotationKey, reconfigureRef). + SetBackupToolName(backupToolName). + AddIncrementalPolicy(). + AddMatchLabels(constant.AppInstanceLabelKey, clusterName). + AddSnapshotPolicy(). + AddMatchLabels(constant.AppInstanceLabelKey, clusterName). + Create(&testCtx).GetObject() + backupPolicyKey := client.ObjectKeyFromObject(backupPolicy) + Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { + g.Expect(fetched.Status.Phase).To(Equal(dpv1alpha1.PolicyAvailable)) + })).Should(Succeed()) + By("enable schedule for reconfigure") + Eventually(testapps.GetAndChangeObj(&testCtx, backupPolicyKey, func(fetched *dpv1alpha1.BackupPolicy) { + fetched.Spec.Schedule.Logfile = &dpv1alpha1.SchedulePolicy{Enable: true, CronExpression: "* * * * *"} + })).Should(Succeed()) + Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { + g.Expect(fetched.Annotations[constant.LastAppliedConfigAnnotation]).To(Equal(`[{"key":"archive_command","value":"''"}]`)) + })).Should(Succeed()) + + By("disable schedule for reconfigure") + Eventually(testapps.GetAndChangeObj(&testCtx, backupPolicyKey, func(fetched *dpv1alpha1.BackupPolicy) { + fetched.Spec.Schedule.Logfile.Enable = false + })).Should(Succeed()) + Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { + g.Expect(fetched.Annotations[constant.LastAppliedConfigAnnotation]).To(Equal(`[{"key":"archive_command","value":"'/bin/true'"}]`)) + })).Should(Succeed()) + }) + }) }) }) diff --git a/deploy/postgresql/config/pg12-config.tpl b/deploy/postgresql/config/pg12-config.tpl index 6557ddffe..62fbc36d6 100644 --- a/deploy/postgresql/config/pg12-config.tpl +++ b/deploy/postgresql/config/pg12-config.tpl @@ -29,8 +29,7 @@ listen_addresses = '*' port = '5432' -# archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' -archive_command = 'if [ $(date +%H%M) -eq 1200 ]; then rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); fi; mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz && sync /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' +archive_command = '/bin/true' archive_mode = 'on' auto_explain.log_analyze = 'False' auto_explain.log_buffers = 'False' diff --git a/deploy/postgresql/config/pg14-config.tpl b/deploy/postgresql/config/pg14-config.tpl index 12b2e3c39..c268037af 100644 --- a/deploy/postgresql/config/pg14-config.tpl +++ b/deploy/postgresql/config/pg14-config.tpl @@ -26,8 +26,7 @@ listen_addresses = '*' port = '5432' -# archive_command = 'wal_dir=/home/postgres/pgdata/pgroot/arcwal; wal_dir_today=${wal_dir}/$(date +%Y%m%d); [[ $(date +%H%M) == 1200 ]] && rm -rf ${wal_dir}/$(date -d"yesterday" +%Y%m%d); mkdir -p ${wal_dir_today} && gzip -kqc %p > ${wal_dir_today}/%f.gz' -archive_command = 'if [ $(date +%H%M) -eq 1200 ]; then rm -rf /home/postgres/pgdata/pgroot/arcwal/$(date -d"yesterday" +%Y%m%d); fi; mkdir -p /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d) && gzip -kqc %p > /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz && sync /home/postgres/pgdata/pgroot/arcwal/$(date +%Y%m%d)/%f.gz' +archive_command = '/bin/true' archive_mode = 'on' auto_explain.log_analyze = 'True' auto_explain.log_min_duration = '1s' diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index 6b896b6da..da7b9054a 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -5,6 +5,18 @@ metadata: labels: clusterdefinition.kubeblocks.io/name: postgresql {{- include "postgresql.labels" . | nindent 4 }} + annotations: + dataprotection.kubeblocks.io/reconfigure-ref: | + { + "name": "postgresql-configuration", + "key": "postgresql.conf", + "enable": { + "logfile": [{"key": "archive_command","value": "''"}] + }, + "disable": { + "logfile": [{ "key": "archive_command","value": "'/bin/true'"}] + } + } spec: clusterDefinitionRef: postgresql backupPolicies: diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml index 74bc9912e..b9d6abd05 100644 --- a/deploy/postgresql/templates/backuptool-pitr.yaml +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -23,8 +23,6 @@ spec: value: $KB_RECOVERY_TIME - name: TIME_FORMAT value: 2006-01-02 15:04:05 MST - - name: ARCHIVE_LOG_DIR - value: $(VOLUME_DATA_DIR)/pgroot/arcwal - name: LOG_DIR value: $(VOLUME_DATA_DIR)/pgroot/data/pg_wal image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} @@ -37,6 +35,7 @@ spec: restoreCommands: - | set -e; + if [ -d ${DATA_DIR}.old ]; then echo "${DATA_DIR}.old directory already exists, skip restore."; exit 0; fi mkdir -p ${PITR_DIR}; cd ${PITR_DIR} for i in $(find ${BACKUP_DIR} -name "*.gz"); do @@ -64,9 +63,13 @@ spec: if [ -d ${EXPIRED_INCR_LOG} ]; then rm -rf ${EXPIRED_INCR_LOG}; fi TODAY_INCR_LOG=${BACKUP_DIR}/$(date +%Y%m%d); mkdir -p ${TODAY_INCR_LOG}; - for i in $(find ${ARCHIVE_LOG_DIR} -name "*.gz"); do - echo "uploading ${i}"; - mv -f ${i} ${TODAY_INCR_LOG}/; + cd ${LOG_DIR} + for i in $(find ./archive_status/ -name "*.ready"); do + wal_ready_name="${i##*/}" + wal_name=${wal_ready_name%.*} + echo "uploading ${wal_name}"; + gzip -kqc ${wal_name} > ${TODAY_INCR_LOG}/${wal_name}.gz; + mv -f ${i} ./archive_status/${wal_name}.done; done echo "done." sync; diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index 6f35977ab..23b962ece 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -100,11 +100,12 @@ data: SHOW_START_TIME=$1 LOG_START_TIME="" if [ "$SHOW_START_TIME" == "false" ]; then - if [ "$(pg_waldump $(psql -Atc "select pg_walfile_name(pg_current_wal_lsn())") --rmgr=Transaction 2>/dev/null |tail -n 1)" != "" ]; then + latest_transaction=$(pg_waldump $(psql -Atc "select pg_walfile_name(pg_current_wal_lsn())") --rmgr=Transaction 2>/dev/null |tail -n 1) + if [ "${latest_transaction}" != "" ]; then psql -c "select pg_switch_wal()" >/dev/null 2>&1 sleep 1 fi - LOG_STOP_TIME=$(pg_waldump $(psql -Atc "select last_archived_wal from pg_stat_archiver") --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') + LOG_STOP_TIME=$(echo ${latest_transaction}|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') [[ "${LOG_STOP_TIME}" != "" ]] && printf "{\"stopTime\": \"$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ')\"}" || printf "{}" else LOG_START_TIME=$(pg_waldump $(ls -Ftr $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') @@ -114,4 +115,4 @@ data: LOG_START_TIME=$(date -d "$LOG_START_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') LOG_STOP_TIME=$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}" - fi \ No newline at end of file + fi diff --git a/internal/constant/const.go b/internal/constant/const.go index 630e033f0..713f102e5 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -108,6 +108,7 @@ const ( RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. DefaultClusterVersionAnnotationKey = "kubeblocks.io/is-default-cluster-version" // DefaultClusterVersionAnnotationKey specifies the default cluster version. + ReconfigureRefAnnotationKey = "dataprotection.kubeblocks.io/reconfigure-ref" // ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl ConfigurationTplLabelPrefixKey = "config.kubeblocks.io/tpl" diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index 449eb6c74..94a12301a 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -69,7 +69,7 @@ func (r *BackupPolicyTPLTransformer) Transform(ctx graph.TransformContext, dag * return intctrlutil.NewNotFound("componentDef %s not found in ClusterDefinition: %s ", v.ComponentDefRef, clusterDefName) } // build the backup policy from the template. - backupPolicy := r.transformBackupPolicy(transCtx, v, origCluster, compDef.WorkloadType, tpl.Name) + backupPolicy := r.transformBackupPolicy(transCtx, v, origCluster, compDef.WorkloadType, &tpl) if backupPolicy == nil { continue } @@ -92,7 +92,7 @@ func (r *BackupPolicyTPLTransformer) transformBackupPolicy(transCtx *ClusterTran policyTPL appsv1alpha1.BackupPolicy, cluster *appsv1alpha1.Cluster, workloadType appsv1alpha1.WorkloadType, - tplName string) *dataprotectionv1alpha1.BackupPolicy { + tpl *appsv1alpha1.BackupPolicyTemplate) *dataprotectionv1alpha1.BackupPolicy { backupPolicyName := DeriveBackupPolicyName(cluster.Name, policyTPL.ComponentDefRef, r.tplIdentifier) backupPolicy := &dataprotectionv1alpha1.BackupPolicy{} if err := transCtx.Client.Get(transCtx.Context, client.ObjectKey{Namespace: cluster.Namespace, Name: backupPolicyName}, backupPolicy); err != nil && !apierrors.IsNotFound(err) { @@ -100,10 +100,10 @@ func (r *BackupPolicyTPLTransformer) transformBackupPolicy(transCtx *ClusterTran } if len(backupPolicy.Name) == 0 { // build a new backup policy from the backup policy template. - return r.buildBackupPolicy(policyTPL, cluster, workloadType, tplName, backupPolicyName) + return r.buildBackupPolicy(policyTPL, cluster, workloadType, tpl, backupPolicyName) } // sync the existing backup policy with the cluster changes - r.syncBackupPolicy(backupPolicy, cluster, policyTPL, workloadType, tplName) + r.syncBackupPolicy(backupPolicy, cluster, policyTPL, workloadType, tpl) return backupPolicy } @@ -112,13 +112,16 @@ func (r *BackupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti cluster *appsv1alpha1.Cluster, policyTPL appsv1alpha1.BackupPolicy, workloadType appsv1alpha1.WorkloadType, - tplName string) { + tpl *appsv1alpha1.BackupPolicyTemplate) { // update labels and annotations of the backup policy. if backupPolicy.Annotations == nil { backupPolicy.Annotations = map[string]string{} } backupPolicy.Annotations[constant.DefaultBackupPolicyAnnotationKey] = r.defaultPolicyAnnotationValue() - backupPolicy.Annotations[constant.BackupPolicyTemplateAnnotationKey] = tplName + backupPolicy.Annotations[constant.BackupPolicyTemplateAnnotationKey] = tpl.Name + if tpl.Annotations[constant.ReconfigureRefAnnotationKey] != "" { + backupPolicy.Annotations[constant.ReconfigureRefAnnotationKey] = tpl.Annotations[constant.ReconfigureRefAnnotationKey] + } if backupPolicy.Labels == nil { backupPolicy.Labels = map[string]string{} } @@ -172,7 +175,7 @@ func (r *BackupPolicyTPLTransformer) syncBackupPolicy(backupPolicy *dataprotecti func (r *BackupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.BackupPolicy, cluster *appsv1alpha1.Cluster, workloadType appsv1alpha1.WorkloadType, - tplName, + tpl *appsv1alpha1.BackupPolicyTemplate, backupPolicyName string) *dataprotectionv1alpha1.BackupPolicy { component := r.getFirstComponent(cluster, policyTPL.ComponentDefRef) if component == nil { @@ -190,11 +193,14 @@ func (r *BackupPolicyTPLTransformer) buildBackupPolicy(policyTPL appsv1alpha1.Ba }, Annotations: map[string]string{ constant.DefaultBackupPolicyAnnotationKey: r.defaultPolicyAnnotationValue(), - constant.BackupPolicyTemplateAnnotationKey: tplName, + constant.BackupPolicyTemplateAnnotationKey: tpl.Name, constant.BackupDataPathPrefixAnnotationKey: fmt.Sprintf("/%s-%s/%s", cluster.Name, cluster.UID, component.Name), }, }, } + if tpl.Annotations[constant.ReconfigureRefAnnotationKey] != "" { + backupPolicy.Annotations[constant.ReconfigureRefAnnotationKey] = tpl.Annotations[constant.ReconfigureRefAnnotationKey] + } bpSpec := backupPolicy.Spec if policyTPL.Retention != nil { bpSpec.Retention = &dataprotectionv1alpha1.RetentionSpec{ diff --git a/internal/controllerutil/errors.go b/internal/controllerutil/errors.go index 2b93e14e5..996ca1927 100644 --- a/internal/controllerutil/errors.go +++ b/internal/controllerutil/errors.go @@ -45,6 +45,8 @@ const ( // ErrorTypeNotFound not found any resource. ErrorTypeNotFound ErrorType = "NotFound" + ErrorTypeRequeue ErrorType = "Requeue" // requeue for reconcile. + // ErrorType for backup ErrorTypeBackupNotSupported ErrorType = "BackupNotSupported" // this backup type not supported ErrorTypeBackupPVTemplateNotFound ErrorType = "BackupPVTemplateNotFound" // this pv template not found @@ -52,6 +54,7 @@ const ( ErrorTypeBackupPVCNameIsEmpty ErrorType = "BackupPVCNameIsEmpty" // pvc name for backup is empty ErrorTypeBackupJobFailed ErrorType = "BackupJobFailed" // backup job failed ErrorTypeStorageNotMatch ErrorType = "ErrorTypeStorageNotMatch" + ErrorTypeReconfigureFailed ErrorType = "ErrorTypeReconfigureFailed" // ErrorType for cluster controller ErrorTypeBackupFailed ErrorType = "BackupFailed" From d2d701e37c0f82d6710223fc27cada46eb10a996 Mon Sep 17 00:00:00 2001 From: chantu Date: Fri, 19 May 2023 17:12:25 +0800 Subject: [PATCH 328/439] fix: volume size rollback when expansion fail (#3280) --- apis/apps/v1alpha1/opsrequest_webhook.go | 6 +- controllers/apps/cluster_controller_test.go | 142 +++++++++++ deploy/helm/templates/deployment.yaml | 4 + deploy/helm/values.yaml | 6 + go.mod | 6 +- internal/constant/const.go | 4 +- .../controller/builder/cue/pvc_template.cue | 2 +- .../lifecycle/cluster_plan_builder.go | 12 +- .../controller/lifecycle/transform_types.go | 1 + .../lifecycle/transformer_sts_pvc.go | 221 ++++++++++++++++-- 10 files changed, 380 insertions(+), 24 deletions(-) diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index 1885bea5a..57abd8055 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -454,9 +454,9 @@ func (r *OpsRequest) getSCNameByPvc(ctx context.Context, vctName string) *string { pvcList := &corev1.PersistentVolumeClaimList{} if err := cli.List(ctx, pvcList, client.InNamespace(r.Namespace), client.MatchingLabels{ - "app.kubernetes.io/instance": r.Spec.ClusterRef, - "apps.kubeblocks.io/component-name": compName, - "vct.kubeblocks.io/name": vctName, + constant.AppInstanceLabelKey: r.Spec.ClusterRef, + constant.KBAppComponentLabelKey: compName, + constant.PVCNameLabelKey: vctName, }, client.Limit(1)); err != nil { return nil } diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index a9d64631b..95b12163e 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -750,6 +750,144 @@ var _ = Describe("Cluster Controller", func() { } } + testVolumeExpansionFailedAndRecover := func(compName, compDefName string) { + + const storageClassName = "test-sc" + const replicas = 3 + + By("Mock a StorageClass which allows resize") + allowVolumeExpansion := true + storageClass := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: storageClassName, + }, + Provisioner: "kubernetes.io/no-provisioner", + AllowVolumeExpansion: &allowVolumeExpansion, + } + Expect(testCtx.CreateObj(testCtx.Ctx, storageClass)).Should(Succeed()) + + By("Creating a cluster with VolumeClaimTemplate") + pvcSpec := testapps.NewPVCSpec("1Gi") + pvcSpec.StorageClassName = &storageClass.Name + + By("Create cluster and waiting for the cluster initialized") + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). + AddComponent(compName, compDefName). + AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). + SetReplicas(replicas). + Create(&testCtx).GetObject() + clusterKey = client.ObjectKeyFromObject(clusterObj) + + By("Waiting for the cluster controller to create resources completely") + waitForCreatingResourceCompletely(clusterKey, compName) + + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) + + By("Checking the replicas") + stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) + sts := &stsList.Items[0] + Expect(*sts.Spec.Replicas).Should(BeEquivalentTo(replicas)) + + By("Mock PVCs in Bound Status") + for i := 0; i < replicas; i++ { + tmpSpec := pvcSpec.ToV1PersistentVolumeClaimSpec() + tmpSpec.VolumeName = getPVCName(compName, i) + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: getPVCName(compName, i), + Namespace: clusterKey.Namespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: clusterKey.Name, + }}, + Spec: tmpSpec, + } + Expect(testCtx.CreateObj(testCtx.Ctx, pvc)).Should(Succeed()) + pvc.Status.Phase = corev1.ClaimBound // only bound pvc allows resize + Expect(k8sClient.Status().Update(testCtx.Ctx, pvc)).Should(Succeed()) + } + + By("mocking PVs") + for i := 0; i < replicas; i++ { + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: getPVCName(compName, i), // use same name as pvc + Namespace: clusterKey.Namespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: clusterKey.Name, + }}, + Spec: corev1.PersistentVolumeSpec{ + Capacity: corev1.ResourceList{ + "storage": resource.MustParse("1Gi"), + }, + AccessModes: []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, + StorageClassName: storageClassName, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/opt/volume/nginx", + Type: nil, + }, + }, + ClaimRef: &corev1.ObjectReference{ + Name: getPVCName(compName, i), + }, + }, + } + Expect(testCtx.CreateObj(testCtx.Ctx, pv)).Should(Succeed()) + } + + By("Updating the PVC storage size") + newStorageValue := resource.MustParse("2Gi") + Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { + comp := &cluster.Spec.ComponentSpecs[0] + comp.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = newStorageValue + })()).ShouldNot(HaveOccurred()) + + By("Checking the resize operation finished") + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(2)) + + By("Checking PVCs are resized") + stsList = testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) + sts = &stsList.Items[0] + for i := *sts.Spec.Replicas - 1; i >= 0; i-- { + pvc := &corev1.PersistentVolumeClaim{} + pvcKey := types.NamespacedName{ + Namespace: clusterKey.Namespace, + Name: getPVCName(compName, int(i)), + } + Expect(k8sClient.Get(testCtx.Ctx, pvcKey, pvc)).Should(Succeed()) + Expect(pvc.Spec.Resources.Requests[corev1.ResourceStorage]).To(Equal(newStorageValue)) + } + + By("Updating the PVC storage size back") + originStorageValue := resource.MustParse("1Gi") + Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { + comp := &cluster.Spec.ComponentSpecs[0] + comp.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = originStorageValue + })()).ShouldNot(HaveOccurred()) + + By("Checking the resize operation finished") + Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(3)) + + By("Checking PVCs are resized") + Eventually(func(g Gomega) { + stsList = testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) + sts = &stsList.Items[0] + for i := *sts.Spec.Replicas - 1; i >= 0; i-- { + pvc := &corev1.PersistentVolumeClaim{} + pvcKey := types.NamespacedName{ + Namespace: clusterKey.Namespace, + Name: getPVCName(compName, int(i)), + } + g.Expect(k8sClient.Get(testCtx.Ctx, pvcKey, pvc)).Should(Succeed()) + g.Expect(pvc.Spec.Resources.Requests[corev1.ResourceStorage]).To(Equal(originStorageValue)) + } + }).Should(Succeed()) + } + testClusterAffinity := func(compName, compDefName string) { const topologyKey = "testTopologyKey" const labelKey = "testNodeLabelKey" @@ -1456,6 +1594,10 @@ var _ = Describe("Cluster Controller", func() { }) }) + It(fmt.Sprintf("[comp: %s] should be able to recover if volume expansion fails", compName), func() { + testVolumeExpansionFailedAndRecover(compName, compDefName) + }) + It(fmt.Sprintf("[comp: %s] should report error if backup error during horizontal scale", compName), func() { testBackupError(compName, compDefName) }) diff --git a/deploy/helm/templates/deployment.yaml b/deploy/helm/templates/deployment.yaml index 5d8dbccf5..43011732c 100644 --- a/deploy/helm/templates/deployment.yaml +++ b/deploy/helm/templates/deployment.yaml @@ -96,6 +96,10 @@ spec: - name: KUBEBLOCKS_ADDON_SA_NAME value: {{ include "kubeblocks.addonSAName" . }} {{- end }} + {{- if .Values.enabledAlphaFeatureGates.recoverVolumeExpansionFailure }} + - name: RECOVER_VOLUME_EXPANSION_FAILURE + value: "true" + {{- end }} {{- with .Values.securityContext }} securityContext: {{- toYaml . | nindent 12 }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 30e43853e..7c6b9cb2a 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -1792,3 +1792,9 @@ aws-load-balancer-controller: operator: In values: - "true" + +## k8s cluster feature gates, ref: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ +enabledAlphaFeatureGates: + ## @param enabledAlphaFeatureGates.recoverVolumeExpansionFailure -- Specifies whether feature gates RecoverVolumeExpansionFailure is enabled in k8s cluster. + ## + recoverVolumeExpansionFailure: false \ No newline at end of file diff --git a/go.mod b/go.mod index 1ff04d4d9..8fe27aa3a 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/briandowns/spinner v1.23.0 github.com/chaos-mesh/chaos-mesh/api v0.0.0-20230423031423-0b31a519b502 github.com/clbanning/mxj/v2 v2.5.7 - github.com/cockroachdb/errors v1.2.4 github.com/containerd/stargz-snapshotter/estargz v0.13.0 github.com/containers/common v0.49.1 github.com/dapr/components-contrib v1.9.6 @@ -87,7 +86,6 @@ require ( k8s.io/cli-runtime v0.26.1 k8s.io/client-go v0.26.1 k8s.io/component-base v0.26.1 - k8s.io/component-helpers v0.26.0 k8s.io/cri-api v0.25.0 k8s.io/klog/v2 v2.90.1 k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a @@ -132,13 +130,11 @@ require ( github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect - github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/cockroachdb/apd/v2 v2.0.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/containerd/containerd v1.6.18 // indirect github.com/containers/image/v5 v5.24.0 // indirect @@ -169,7 +165,6 @@ require ( github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fvbommel/sortorder v1.0.2 // indirect - github.com/getsentry/raven-go v0.2.0 // indirect github.com/go-errors/errors v1.4.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.0 // indirect @@ -375,6 +370,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 // indirect k8s.io/apiserver v0.26.1 // indirect + k8s.io/component-helpers v0.26.0 // indirect oras.land/oras-go v1.2.2 // indirect periph.io/x/host/v3 v3.8.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/internal/constant/const.go b/internal/constant/const.go index 713f102e5..5843ca58a 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -80,7 +80,8 @@ const ( ConsensusSetAccessModeLabelKey = "cs.apps.kubeblocks.io/access-mode" AppConfigTypeLabelKey = "apps.kubeblocks.io/config-type" WorkloadTypeLabelKey = "apps.kubeblocks.io/workload-type" - VolumeClaimTemplateNameLabelKey = "vct.kubeblocks.io/name" + VolumeClaimTemplateNameLabelKey = "apps.kubeblocks.io/vct-name" + PVCNameLabelKey = "apps.kubeblocks.io/pvc-name" RoleLabelKey = "kubeblocks.io/role" // RoleLabelKey consensusSet and replicationSet role label key BackupProtectionLabelKey = "kubeblocks.io/backup-protection" // BackupProtectionLabelKey Backup delete protection policy label AddonNameLabelKey = "extensions.kubeblocks.io/addon-name" @@ -108,6 +109,7 @@ const ( RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. DefaultClusterVersionAnnotationKey = "kubeblocks.io/is-default-cluster-version" // DefaultClusterVersionAnnotationKey specifies the default cluster version. + PVLastClaimPolicyAnnotationKey = "apps.kubeblocks.io/pv-last-claim-policy" ReconfigureRefAnnotationKey = "dataprotection.kubeblocks.io/reconfigure-ref" // ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl diff --git a/internal/controller/builder/cue/pvc_template.cue b/internal/controller/builder/cue/pvc_template.cue index aeb355edf..f81745b0c 100644 --- a/internal/controller/builder/cue/pvc_template.cue +++ b/internal/controller/builder/cue/pvc_template.cue @@ -41,7 +41,7 @@ pvc: { name: pvc_key.Name namespace: pvc_key.Namespace labels: { - "vct.kubeblocks.io/name": volumeClaimTemplate.metadata.name + "apps.kubeblocks.io/vct-name": volumeClaimTemplate.metadata.name for k, v in sts.metadata.labels { "\(k)": "\(v)" } diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index c89bb732f..00a092dde 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -257,6 +257,12 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { c.transCtx.Logger.Error(err, fmt.Sprintf("update %T error: %s", o, node.oriObj.GetName())) return err } + case PATCH: + patch := client.MergeFrom(node.oriObj) + if err := c.cli.Patch(c.transCtx.Context, node.obj, patch); !apierrors.IsNotFound(err) { + c.transCtx.Logger.Error(err, fmt.Sprintf("patch %T error", node.oriObj)) + return err + } case DELETE: if controllerutil.RemoveFinalizer(node.obj, dbClusterFinalizerName) { err := c.cli.Update(c.transCtx.Context, node.obj) @@ -332,7 +338,11 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec handlePVC := func(origObj, pvcProto *corev1.PersistentVolumeClaim) (client.Object, error) { pvcObj := origObj.DeepCopy() - pvcObj.Spec.Resources.Requests[corev1.ResourceStorage] = pvcProto.Spec.Resources.Requests[corev1.ResourceStorage] + if pvcObj.Spec.Resources.Requests == nil { + pvcObj.Spec.Resources.Requests = pvcProto.Spec.Resources.Requests + } else { + pvcObj.Spec.Resources.Requests[corev1.ResourceStorage] = pvcProto.Spec.Resources.Requests[corev1.ResourceStorage] + } return pvcObj, nil } diff --git a/internal/controller/lifecycle/transform_types.go b/internal/controller/lifecycle/transform_types.go index fb27f9c23..5a27536d5 100644 --- a/internal/controller/lifecycle/transform_types.go +++ b/internal/controller/lifecycle/transform_types.go @@ -61,6 +61,7 @@ type Action string const ( CREATE = Action("CREATE") UPDATE = Action("UPDATE") + PATCH = Action("PATCH") DELETE = Action("DELETE") STATUS = Action("STATUS") ) diff --git a/internal/controller/lifecycle/transformer_sts_pvc.go b/internal/controller/lifecycle/transformer_sts_pvc.go index 1df2bd3fe..783cf6184 100644 --- a/internal/controller/lifecycle/transformer_sts_pvc.go +++ b/internal/controller/lifecycle/transformer_sts_pvc.go @@ -21,11 +21,16 @@ package lifecycle import ( "fmt" + "reflect" + "github.com/spf13/viper" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/graph" ) @@ -39,6 +44,200 @@ func (t *StsPVCTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG return nil } + // reference: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#recovering-from-failure-when-expanding-volumes + // 1. Mark the PersistentVolume(PV) that is bound to the PersistentVolumeClaim(PVC) with Retain reclaim policy. + // 2. Delete the PVC. Since PV has Retain reclaim policy - we will not lose any data when we recreate the PVC. + // 3. Delete the claimRef entry from PV specs, so as new PVC can bind to it. This should make the PV Available. + // 4. Re-create the PVC with smaller size than PV and set volumeName field of the PVC to the name of the PV. This should bind new PVC to existing PV. + // 5. Don't forget to restore the reclaim policy of the PV. + updatePVCSize := func(vertex *lifecycleVertex, pvcKey types.NamespacedName, pvc *corev1.PersistentVolumeClaim, pvcNotFound bool, vctProto *corev1.PersistentVolumeClaim) error { + + newPVC := pvc.DeepCopy() + if pvcNotFound { + newPVC.Name = pvcKey.Name + newPVC.Namespace = pvcKey.Namespace + newPVC.SetLabels(vctProto.Labels) + newPVC.Spec = vctProto.Spec + ml := client.MatchingLabels{ + constant.PVCNameLabelKey: pvcKey.Name, + } + pvList := corev1.PersistentVolumeList{} + if err := transCtx.Client.List(transCtx.Context, &pvList, ml); err != nil { + return err + } + for _, pv := range pvList.Items { + // find pv referenced this pvc + if pv.Spec.ClaimRef == nil { + continue + } + if pv.Spec.ClaimRef.Name == pvcKey.Name { + newPVC.Spec.VolumeName = pv.Name + break + } + } + } else { + newPVC.Spec.Resources.Requests[corev1.ResourceStorage] = vctProto.Spec.Resources.Requests[corev1.ResourceStorage] + // delete annotation to make it re-bind + delete(newPVC.Annotations, "pv.kubernetes.io/bind-completed") + } + + // for simple update + simpleUpdateVertex := &lifecycleVertex{ + obj: newPVC, + oriObj: pvc, + action: actionPtr(UPDATE), + } + + pvNotFound := false + + // step 1: update pv to retain + pv := &corev1.PersistentVolume{} + pvKey := types.NamespacedName{ + Namespace: pvcKey.Namespace, + Name: newPVC.Spec.VolumeName, + } + if err := transCtx.Client.Get(transCtx.Context, pvKey, pv); err != nil { + if errors.IsNotFound(err) { + pvNotFound = true + } else { + return err + } + } + + type pvcRecreateStep int + const ( + pvPolicyRetainStep pvcRecreateStep = iota + deletePVCStep + removePVClaimRefStep + createPVCStep + pvRestorePolicyStep + ) + + addStepMap := map[pvcRecreateStep]func(fromVertex *lifecycleVertex, step pvcRecreateStep) *lifecycleVertex{ + pvPolicyRetainStep: func(fromVertex *lifecycleVertex, step pvcRecreateStep) *lifecycleVertex { + // step 1: update pv to retain + retainPV := pv.DeepCopy() + if retainPV.Labels == nil { + retainPV.Labels = make(map[string]string) + } + // add label to pv, in case pvc get deleted, and we can't find pv + retainPV.Labels[constant.PVCNameLabelKey] = pvcKey.Name + if retainPV.Annotations == nil { + retainPV.Annotations = make(map[string]string) + } + retainPV.Annotations[constant.PVLastClaimPolicyAnnotationKey] = string(pv.Spec.PersistentVolumeReclaimPolicy) + retainPV.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain + retainPVVertex := &lifecycleVertex{ + obj: retainPV, + oriObj: pv, + action: actionPtr(PATCH), + } + dag.AddVertex(retainPVVertex) + dag.Connect(fromVertex, retainPVVertex) + return retainPVVertex + }, + deletePVCStep: func(fromVertex *lifecycleVertex, step pvcRecreateStep) *lifecycleVertex { + // step 2: delete pvc, this will not delete pv because policy is 'retain' + deletePVCVertex := &lifecycleVertex{obj: pvc, action: actionPtr(DELETE)} + removeFinalizerPVC := pvc.DeepCopy() + removeFinalizerPVC.SetFinalizers([]string{}) + removeFinalizerPVCVertex := &lifecycleVertex{ + obj: removeFinalizerPVC, + oriObj: pvc, + action: actionPtr(PATCH), + } + dag.AddVertex(deletePVCVertex) + dag.AddVertex(removeFinalizerPVCVertex) + dag.Connect(removeFinalizerPVCVertex, deletePVCVertex) + dag.Connect(fromVertex, removeFinalizerPVCVertex) + return deletePVCVertex + }, + removePVClaimRefStep: func(fromVertex *lifecycleVertex, step pvcRecreateStep) *lifecycleVertex { + // step 3: remove claimRef in pv + removeClaimRefPV := pv.DeepCopy() + if removeClaimRefPV.Spec.ClaimRef != nil { + removeClaimRefPV.Spec.ClaimRef.UID = "" + removeClaimRefPV.Spec.ClaimRef.ResourceVersion = "" + } + removeClaimRefVertex := &lifecycleVertex{ + obj: removeClaimRefPV, + oriObj: pv, + action: actionPtr(PATCH), + } + dag.AddVertex(removeClaimRefVertex) + dag.Connect(fromVertex, removeClaimRefVertex) + return removeClaimRefVertex + }, + createPVCStep: func(fromVertex *lifecycleVertex, step pvcRecreateStep) *lifecycleVertex { + // step 4: create new pvc + newPVC.SetResourceVersion("") + createNewPVCVertex := &lifecycleVertex{ + obj: newPVC, + action: actionPtr(CREATE), + } + dag.AddVertex(createNewPVCVertex) + dag.Connect(fromVertex, createNewPVCVertex) + return createNewPVCVertex + }, + pvRestorePolicyStep: func(fromVertex *lifecycleVertex, step pvcRecreateStep) *lifecycleVertex { + // step 5: restore to previous pv policy + restorePV := pv.DeepCopy() + policy := corev1.PersistentVolumeReclaimPolicy(restorePV.Annotations[constant.PVLastClaimPolicyAnnotationKey]) + if len(policy) == 0 { + policy = corev1.PersistentVolumeReclaimDelete + } + restorePV.Spec.PersistentVolumeReclaimPolicy = policy + restorePVVertex := &lifecycleVertex{ + obj: restorePV, + oriObj: pv, + action: actionPtr(PATCH), + } + dag.AddVertex(restorePVVertex) + dag.Connect(fromVertex, restorePVVertex) + return restorePVVertex + }, + } + + updatePVCByRecreateFromStep := func(fromStep pvcRecreateStep) { + lastVertex := vertex + for step := pvRestorePolicyStep; step >= fromStep && step >= pvPolicyRetainStep; step-- { + lastVertex = addStepMap[step](lastVertex, step) + } + } + + targetQuantity := vctProto.Spec.Resources.Requests[corev1.ResourceStorage] + if pvcNotFound && !pvNotFound { + // this could happen if create pvc step failed when recreating pvc + updatePVCByRecreateFromStep(removePVClaimRefStep) + return nil + } + if pvcNotFound && pvNotFound { + // if both pvc and pv not found, do nothing + return nil + } + if reflect.DeepEqual(pvc.Spec.Resources, newPVC.Spec.Resources) && pv.Spec.PersistentVolumeReclaimPolicy == corev1.PersistentVolumeReclaimRetain { + // this could happen if create pvc succeeded but last step failed + updatePVCByRecreateFromStep(pvRestorePolicyStep) + return nil + } + if pvcQuantity := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; !viper.GetBool(constant.CfgRecoverVolumeExpansionFailure) && + pvcQuantity.Cmp(targetQuantity) == 1 && // check if it's compressing volume + targetQuantity.Cmp(*pvc.Status.Capacity.Storage()) >= 0 { // check if target size is greater than or equal to actual size + // this branch means we can update pvc size by recreate it + updatePVCByRecreateFromStep(pvPolicyRetainStep) + return nil + } + if pvcQuantity := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; pvcQuantity.Cmp(vctProto.Spec.Resources.Requests[corev1.ResourceStorage]) != 0 { + // use pvc's update without anything extra + dag.AddVertex(simpleUpdateVertex) + dag.Connect(vertex, simpleUpdateVertex) + return nil + } + // all the else means no need to update + + return nil + } + handlePVCUpdate := func(vertex *lifecycleVertex) error { stsObj, _ := vertex.oriObj.(*appsv1.StatefulSet) stsProto, _ := vertex.obj.(*appsv1.StatefulSet) @@ -59,10 +258,7 @@ func (t *StsPVCTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG continue } - if vct.Spec.Resources.Requests[corev1.ResourceStorage] == vctProto.Spec.Resources.Requests[corev1.ResourceStorage] { - continue - } - + pvcNotFound := false for i := *stsObj.Spec.Replicas - 1; i >= 0; i-- { pvc := &corev1.PersistentVolumeClaim{} pvcKey := types.NamespacedName{ @@ -70,17 +266,16 @@ func (t *StsPVCTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG Name: fmt.Sprintf("%s-%s-%d", vct.Name, stsObj.Name, i), } if err := transCtx.Client.Get(transCtx.Context, pvcKey, pvc); err != nil { - return err + if errors.IsNotFound(err) { + pvcNotFound = true + } else { + return err + } } - obj := pvc.DeepCopy() - obj.Spec.Resources.Requests[corev1.ResourceStorage] = vctProto.Spec.Resources.Requests[corev1.ResourceStorage] - v := &lifecycleVertex{ - obj: obj, - oriObj: pvc, - action: actionPtr(UPDATE), + + if err := updatePVCSize(vertex, pvcKey, pvc, pvcNotFound, vctProto); err != nil { + return err } - dag.AddVertex(v) - dag.Connect(vertex, v) } } return nil From e33f3bbf0eec7218253fc51cecb0aa6e67dada71 Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Mon, 22 May 2023 14:29:06 +0800 Subject: [PATCH 329/439] feat: support cli fault node aws gcp (#3265) Co-authored-by: huyongqii --- docs/user_docs/cli/cli.md | 1 + docs/user_docs/cli/kbcli_fault.md | 1 + docs/user_docs/cli/kbcli_fault_node.md | 45 +++ .../cli/kbcli_fault_node_detach-volume.md | 80 +++++ .../user_docs/cli/kbcli_fault_node_restart.md | 78 +++++ docs/user_docs/cli/kbcli_fault_node_stop.md | 78 +++++ internal/cli/cmd/fault/fault.go | 1 + internal/cli/cmd/fault/fault_constant.go | 46 +-- internal/cli/cmd/fault/fault_node.go | 283 ++++++++++++++++++ internal/cli/cmd/fault/fault_node_test.go | 101 +++++++ .../create/template/node_chaos_template.cue | 65 ++++ 11 files changed, 759 insertions(+), 20 deletions(-) create mode 100644 docs/user_docs/cli/kbcli_fault_node.md create mode 100644 docs/user_docs/cli/kbcli_fault_node_detach-volume.md create mode 100644 docs/user_docs/cli/kbcli_fault_node_restart.md create mode 100644 docs/user_docs/cli/kbcli_fault_node_stop.md create mode 100644 internal/cli/cmd/fault/fault_node.go create mode 100644 internal/cli/cmd/fault/fault_node_test.go create mode 100644 internal/cli/create/template/node_chaos_template.cue diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index ca7c6c969..19fbb204e 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -114,6 +114,7 @@ Inject faults to pod. * [kbcli fault io](kbcli_fault_io.md) - IO chaos. * [kbcli fault network](kbcli_fault_network.md) - Network chaos. +* [kbcli fault node](kbcli_fault_node.md) - Node chaos. * [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. * [kbcli fault stress](kbcli_fault_stress.md) - Add memory pressure or CPU load to the system. * [kbcli fault time](kbcli_fault_time.md) - Clock skew failure. diff --git a/docs/user_docs/cli/kbcli_fault.md b/docs/user_docs/cli/kbcli_fault.md index cdd11e7ac..600b04e7e 100644 --- a/docs/user_docs/cli/kbcli_fault.md +++ b/docs/user_docs/cli/kbcli_fault.md @@ -39,6 +39,7 @@ Inject faults to pod. * [kbcli fault io](kbcli_fault_io.md) - IO chaos. * [kbcli fault network](kbcli_fault_network.md) - Network chaos. +* [kbcli fault node](kbcli_fault_node.md) - Node chaos. * [kbcli fault pod](kbcli_fault_pod.md) - Pod chaos. * [kbcli fault stress](kbcli_fault_stress.md) - Add memory pressure or CPU load to the system. * [kbcli fault time](kbcli_fault_time.md) - Clock skew failure. diff --git a/docs/user_docs/cli/kbcli_fault_node.md b/docs/user_docs/cli/kbcli_fault_node.md new file mode 100644 index 000000000..e89be737b --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_node.md @@ -0,0 +1,45 @@ +--- +title: kbcli fault node +--- + +Node chaos. + +### Options + +``` + -h, --help help for node +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault](kbcli_fault.md) - Inject faults to pod. +* [kbcli fault node detach-volume](kbcli_fault_node_detach-volume.md) - Detach volume +* [kbcli fault node restart](kbcli_fault_node_restart.md) - Restart instance +* [kbcli fault node stop](kbcli_fault_node_stop.md) - Stop instance + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_node_detach-volume.md b/docs/user_docs/cli/kbcli_fault_node_detach-volume.md new file mode 100644 index 000000000..72271d001 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_node_detach-volume.md @@ -0,0 +1,80 @@ +--- +title: kbcli fault node detach-volume +--- + +Detach volume + +``` +kbcli fault node detach-volume [flags] +``` + +### Examples + +``` + # Stop a specified EC2 instance. + kbcli fault node stop node1 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Stop two specified EC2 instances. + kbcli fault node stop node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Restart two specified EC2 instances. + kbcli fault node restart node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Detach two specified volume from two specified EC2 instances. + kbcli fault node detach-volume node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 + + # Stop two specified GCK instances. + kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + + # Restart two specified GCK instances. + kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + + # Detach two specified volume from two specified GCK instances. + kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret --device-name=/d1,/d2 +``` + +### Options + +``` + -c, --cloud-provider string Cloud provider type, one of [aws gcp] + --device-name strings The device name of the volume. + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "30s") + -h, --help help for detach-volume + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --project string The name of the GCP project. Only available when cloud-provider=gcp. + --region string The region of the node. + --secret-name string The name of the Kubernetes Secret that stores the kubernetes cluster authentication information. + --volume-id strings The volume ids of the ec2. Only available when cloud-provider=aws. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault node](kbcli_fault_node.md) - Node chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_node_restart.md b/docs/user_docs/cli/kbcli_fault_node_restart.md new file mode 100644 index 000000000..625abf1bf --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_node_restart.md @@ -0,0 +1,78 @@ +--- +title: kbcli fault node restart +--- + +Restart instance + +``` +kbcli fault node restart [flags] +``` + +### Examples + +``` + # Stop a specified EC2 instance. + kbcli fault node stop node1 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Stop two specified EC2 instances. + kbcli fault node stop node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Restart two specified EC2 instances. + kbcli fault node restart node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Detach two specified volume from two specified EC2 instances. + kbcli fault node detach-volume node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 + + # Stop two specified GCK instances. + kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + + # Restart two specified GCK instances. + kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + + # Detach two specified volume from two specified GCK instances. + kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret --device-name=/d1,/d2 +``` + +### Options + +``` + -c, --cloud-provider string Cloud provider type, one of [aws gcp] + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "30s") + -h, --help help for restart + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --project string The name of the GCP project. Only available when cloud-provider=gcp. + --region string The region of the node. + --secret-name string The name of the Kubernetes Secret that stores the kubernetes cluster authentication information. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault node](kbcli_fault_node.md) - Node chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_fault_node_stop.md b/docs/user_docs/cli/kbcli_fault_node_stop.md new file mode 100644 index 000000000..f9e67e475 --- /dev/null +++ b/docs/user_docs/cli/kbcli_fault_node_stop.md @@ -0,0 +1,78 @@ +--- +title: kbcli fault node stop +--- + +Stop instance + +``` +kbcli fault node stop [flags] +``` + +### Examples + +``` + # Stop a specified EC2 instance. + kbcli fault node stop node1 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Stop two specified EC2 instances. + kbcli fault node stop node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Restart two specified EC2 instances. + kbcli fault node restart node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Detach two specified volume from two specified EC2 instances. + kbcli fault node detach-volume node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 + + # Stop two specified GCK instances. + kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + + # Restart two specified GCK instances. + kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + + # Detach two specified volume from two specified GCK instances. + kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret --device-name=/d1,/d2 +``` + +### Options + +``` + -c, --cloud-provider string Cloud provider type, one of [aws gcp] + --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") + --duration string Supported formats of the duration are: ms / s / m / h. (default "30s") + -h, --help help for stop + -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) + --project string The name of the GCP project. Only available when cloud-provider=gcp. + --region string The region of the node. + --secret-name string The name of the Kubernetes Secret that stores the kubernetes cluster authentication information. +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli fault node](kbcli_fault_node.md) - Node chaos. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/internal/cli/cmd/fault/fault.go b/internal/cli/cmd/fault/fault.go index 9ed07b2db..95ab712e9 100644 --- a/internal/cli/cmd/fault/fault.go +++ b/internal/cli/cmd/fault/fault.go @@ -76,6 +76,7 @@ func NewFaultCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra. NewTimeChaosCmd(f, streams), NewIOChaosCmd(f, streams), NewStressChaosCmd(f, streams), + NewNodeChaosCmd(f, streams), ) return cmd } diff --git a/internal/cli/cmd/fault/fault_constant.go b/internal/cli/cmd/fault/fault_constant.go index a80f5cb44..59acab34b 100644 --- a/internal/cli/cmd/fault/fault_constant.go +++ b/internal/cli/cmd/fault/fault_constant.go @@ -19,6 +19,8 @@ along with this program. If not, see . package fault +import cp "github.com/apecloud/kubeblocks/internal/cli/cloudprovider" + // Unchanged DryRun flag const ( Unchanged = "unchanged" @@ -31,26 +33,28 @@ const ( ResourcePodChaos = "podchaos" ResourceNetworkChaos = "networkchaos" + ResourceDNSChaos = "dnschaos" + ResourceHTTPChaos = "httpchaos" ResourceIOChaos = "iochaos" ResourceStressChaos = "stresschaos" - ResourceDNSChaos = "dnschaos" ResourceTimeChaos = "timechaos" - ResourceHTTPChaos = "httpchaos" ResourceAWSChaos = "awschaos" ResourceGCPChaos = "gcpchaos" + + KindAWSChaos = "AWSChaos" + KindGCPChaos = "GCPChaos" ) // Cue Template Name const ( CueTemplatePodChaos = "pod_chaos_template.cue" CueTemplateNetworkChaos = "network_chaos_template.cue" + CueTemplateDNSChaos = "dns_chaos_template.cue" + CueTemplateHTTPChaos = "http_chaos_template.cue" CueTemplateIOChaos = "io_chaos_template.cue" CueTemplateStressChaos = "stress_chaos_template.cue" - CueTemplateDNSChaos = "dns_chaos_template.cue" CueTemplateTimeChaos = "time_chaos_template.cue" - CueTemplateHTTPChaos = "http_chaos_template.cue" - CueTemplateAWSChaos = "aws_chaos_template.cue" - CueTemplateGCPChaos = "gcp_chaos_template.cue" + CueTemplateNodeChaos = "node_chaos_template.cue" ) // Pod Chaos Command @@ -87,7 +91,19 @@ const ( ErrorShort = "Make DNS return an error when resolving external domain names." ) -// Network Chaos Command +// HTTP Chaos Command +const ( + Abort = "abort" + AbortShort = "Abort the HTTP request and response." + HTTPDelay = "delay" + HTTPDelayShort = "Delay the HTTP request and response." + Replace = "replace" + ReplaceShort = "Replace the HTTP request and response." + Patch = "patch" + PatchShort = "Patch the HTTP request and response." +) + +// IO Chaos Command const ( Latency = "latency" LatencyShort = "Delayed IO operations." @@ -111,19 +127,7 @@ const ( TimeShort = "Clock skew failure." ) -// HTTP Chaos Command -const ( - Abort = "abort" - AbortShort = "Abort the HTTP request and response." - HTTPDelay = "delay" - HTTPDelayShort = "Delay the HTTP request and response." - Replace = "replace" - ReplaceShort = "Replace the HTTP request and response." - Patch = "patch" - PatchShort = "Patch the HTTP request and response." -) - -// AWS And GCP Chaos Command +// Node Chaos Command const ( Stop = "stop" StopShort = "Stop instance" @@ -132,3 +136,5 @@ const ( DetachVolume = "detach-volume" DetachVolumeShort = "Detach volume" ) + +var supportedCloudProviders = []string{cp.AWS, cp.GCP} diff --git a/internal/cli/cmd/fault/fault_node.go b/internal/cli/cmd/fault/fault_node.go new file mode 100644 index 000000000..91af3f160 --- /dev/null +++ b/internal/cli/cmd/fault/fault_node.go @@ -0,0 +1,283 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + "fmt" + + "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + cp "github.com/apecloud/kubeblocks/internal/cli/cloudprovider" + "github.com/apecloud/kubeblocks/internal/cli/create" + "github.com/apecloud/kubeblocks/internal/cli/printer" + "github.com/apecloud/kubeblocks/internal/cli/util" +) + +var faultNodeExample = templates.Examples(` + # Stop a specified EC2 instance. + kbcli fault node stop node1 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Stop two specified EC2 instances. + kbcli fault node stop node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Restart two specified EC2 instances. + kbcli fault node restart node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + + # Detach two specified volume from two specified EC2 instances. + kbcli fault node detach-volume node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 + + # Stop two specified GCK instances. + kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + + # Restart two specified GCK instances. + kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + + # Detach two specified volume from two specified GCK instances. + kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret --device-name=/d1,/d2 +`) + +type NodeChaoOptions struct { + Kind string `json:"kind"` + + Action string `json:"action"` + + CloudProvider string `json:"-"` + + SecretName string `json:"secretName"` + + Region string `json:"region"` + + Instance string `json:"instance"` + + VolumeID string `json:"volumeID"` + VolumeIDs []string `json:"-"` + + DeviceName string `json:"deviceName,omitempty"` + DeviceNames []string `json:"-"` + + Project string `json:"project"` + + Duration string `json:"duration"` + + create.CreateOptions `json:"-"` +} + +func NewNodeOptions(f cmdutil.Factory, streams genericclioptions.IOStreams) *NodeChaoOptions { + o := &NodeChaoOptions{ + CreateOptions: create.CreateOptions{ + Factory: f, + IOStreams: streams, + CueTemplateName: CueTemplateNodeChaos, + }, + } + o.CreateOptions.PreCreate = o.PreCreate + o.CreateOptions.Options = o + return o +} + +func NewNodeChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "node", + Short: "Node chaos.", + } + + cmd.AddCommand( + NewStopCmd(f, streams), + NewRestartCmd(f, streams), + NewDetachVolumeCmd(f, streams), + ) + return cmd +} + +func NewStopCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNodeOptions(f, streams) + cmd := o.NewCobraCommand(Stop, StopShort) + + o.AddCommonFlag(cmd) + return cmd +} + +func NewRestartCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNodeOptions(f, streams) + cmd := o.NewCobraCommand(Restart, RestartShort) + + o.AddCommonFlag(cmd) + return cmd +} + +func NewDetachVolumeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := NewNodeOptions(f, streams) + cmd := o.NewCobraCommand(DetachVolume, DetachVolumeShort) + + o.AddCommonFlag(cmd) + cmd.Flags().StringSliceVar(&o.VolumeIDs, "volume-id", nil, "The volume ids of the ec2. Only available when cloud-provider=aws.") + cmd.Flags().StringSliceVar(&o.DeviceNames, "device-name", nil, "The device name of the volume.") + + util.CheckErr(cmd.MarkFlagRequired("device-name")) + return cmd +} + +func (o *NodeChaoOptions) NewCobraCommand(use, short string) *cobra.Command { + return &cobra.Command{ + Use: use, + Short: short, + Example: faultNodeExample, + Run: func(cmd *cobra.Command, args []string) { + cmdutil.CheckErr(o.Execute(use, args)) + }, + } +} + +func (o *NodeChaoOptions) Execute(action string, args []string) error { + o.Args = args + if err := o.CreateOptions.Complete(); err != nil { + return err + } + if err := o.Complete(action); err != nil { + return err + } + if err := o.Validate(); err != nil { + return err + } + + for idx, arg := range o.Args { + o.Instance = arg + if o.DeviceNames != nil { + o.DeviceName = o.DeviceNames[idx] + } + if o.VolumeIDs != nil { + o.VolumeID = o.VolumeIDs[idx] + } + if err := o.Run(); err != nil { + return err + } + } + return nil +} + +func (o *NodeChaoOptions) AddCommonFlag(cmd *cobra.Command) { + cmd.Flags().StringVarP(&o.CloudProvider, "cloud-provider", "c", "", fmt.Sprintf("Cloud provider type, one of %v", supportedCloudProviders)) + cmd.Flags().StringVar(&o.SecretName, "secret-name", "", "The name of the Kubernetes Secret that stores the kubernetes cluster authentication information.") + cmd.Flags().StringVar(&o.Region, "region", "", "The region of the node.") + cmd.Flags().StringVar(&o.Project, "project", "", "The name of the GCP project. Only available when cloud-provider=gcp.") + cmd.Flags().StringVar(&o.Duration, "duration", "30s", "Supported formats of the duration are: ms / s / m / h.") + + cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) + cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged + printer.AddOutputFlagForCreate(cmd, &o.Format) + + util.CheckErr(cmd.MarkFlagRequired("cloud-provider")) + util.CheckErr(cmd.MarkFlagRequired("secret-name")) + util.CheckErr(cmd.MarkFlagRequired("region")) + + // register flag completion func + registerFlagCompletionFunc(cmd, o.Factory) +} + +func (o *NodeChaoOptions) Validate() error { + if ok, err := IsRegularMatch(o.Duration); !ok { + return err + } + + if len(o.Args) == 0 { + return fmt.Errorf("node instance is required") + } + + switch o.CloudProvider { + case cp.AWS: + if o.Project != "" { + return fmt.Errorf("--project is not supported when cloud provider is aws") + } + if o.Action == DetachVolume && o.VolumeIDs == nil { + return fmt.Errorf("--volume-id is required when cloud provider is aws") + } + if o.Action == DetachVolume && len(o.DeviceNames) != len(o.VolumeIDs) { + return fmt.Errorf("the number of volume-id must be equal to the number of device-name") + } + case cp.GCP: + if o.Project == "" { + return fmt.Errorf("--project is required when cloud provider is gcp") + } + if o.VolumeIDs != nil { + return fmt.Errorf(" --volume-id is not supported when cloud provider is gcp") + } + default: + return fmt.Errorf("cloud provider type, one of %v", supportedCloudProviders) + } + + if o.DeviceNames != nil && len(o.Args) != len(o.DeviceNames) { + return fmt.Errorf("the number of device-name must be equal to the number of node") + } + return nil +} + +func (o *NodeChaoOptions) Complete(action string) error { + if o.CloudProvider == cp.AWS { + o.GVR = GetGVR(Group, Version, ResourceAWSChaos) + o.Kind = KindAWSChaos + switch action { + case Stop: + o.Action = string(v1alpha1.Ec2Stop) + case Restart: + o.Action = string(v1alpha1.Ec2Restart) + case DetachVolume: + o.Action = string(v1alpha1.DetachVolume) + } + } else if o.CloudProvider == cp.GCP { + o.GVR = GetGVR(Group, Version, ResourceGCPChaos) + o.Kind = KindGCPChaos + switch action { + case Stop: + o.Action = string(v1alpha1.NodeStop) + case Restart: + o.Action = string(v1alpha1.NodeReset) + case DetachVolume: + o.Action = string(v1alpha1.DiskLoss) + } + } + return nil +} + +func (o *NodeChaoOptions) PreCreate(obj *unstructured.Unstructured) error { + var c v1alpha1.InnerObject + + if o.CloudProvider == cp.AWS { + c = &v1alpha1.AWSChaos{} + } else if o.CloudProvider == cp.GCP { + c = &v1alpha1.GCPChaos{} + } + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, c); err != nil { + return err + } + + data, e := runtime.DefaultUnstructuredConverter.ToUnstructured(c) + if e != nil { + return e + } + obj.SetUnstructuredContent(data) + return nil +} diff --git a/internal/cli/cmd/fault/fault_node_test.go b/internal/cli/cmd/fault/fault_node_test.go new file mode 100644 index 000000000..2b28f854d --- /dev/null +++ b/internal/cli/cmd/fault/fault_node_test.go @@ -0,0 +1,101 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package fault + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/cli-runtime/pkg/genericclioptions" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + "github.com/apecloud/kubeblocks/internal/cli/testing" +) + +var _ = Describe("Fault Node", func() { + var ( + tf *cmdtesting.TestFactory + streams genericclioptions.IOStreams + ) + BeforeEach(func() { + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = cmdtesting.NewTestFactory().WithNamespace(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + }) + + AfterEach(func() { + tf.Cleanup() + }) + + Context("test fault node", func() { + It("fault node stop", func() { + inputs := [][]string{ + {"-c=aws", "--region=cn-northwest-1", "--secret-name=cloud-key-secret", "--dry-run=client"}, + {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--secret-name=cloud-key-secret", "--dry-run=client"}, + } + o := NewNodeOptions(tf, streams) + cmd := o.NewCobraCommand(Stop, StopShort) + o.AddCommonFlag(cmd) + + o.Args = []string{"node1", "node2"} + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.Execute(Stop, o.Args)).Should(Succeed()) + } + }) + + It("fault node restart", func() { + inputs := [][]string{ + {"-c=aws", "--region=cn-northwest-1", "--secret-name=cloud-key-secret", "--dry-run=client"}, + {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--secret-name=cloud-key-secret", "--dry-run=client"}, + } + o := NewNodeOptions(tf, streams) + cmd := o.NewCobraCommand(Restart, RestartShort) + o.AddCommonFlag(cmd) + + o.Args = []string{"node1", "node2"} + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.Execute(Restart, o.Args)).Should(Succeed()) + } + }) + + It("fault node detach-volume", func() { + inputs := [][]string{ + {"-c=aws", "--region=cn-northwest-1", "--secret-name=cloud-key-secret", "--volume-id=v1,v2", "--device-name=/d1,/d2", "--dry-run=client"}, + {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--secret-name=cloud-key-secret", "--device-name=/d1,/d2", "--dry-run=client"}, + } + o := NewNodeOptions(tf, streams) + cmd := o.NewCobraCommand(DetachVolume, DetachVolumeShort) + o.AddCommonFlag(cmd) + cmd.Flags().StringSliceVar(&o.VolumeIDs, "volume-id", nil, "The volume id of the ec2.") + cmd.Flags().StringSliceVar(&o.DeviceNames, "device-name", nil, "The device name of the volume.") + + o.Args = []string{"node1", "node2"} + for _, input := range inputs { + Expect(cmd.Flags().Parse(input)).Should(Succeed()) + Expect(o.Execute(DetachVolume, o.Args)).Should(Succeed()) + o.VolumeIDs = nil + o.DeviceNames = nil + } + }) + }) +}) diff --git a/internal/cli/create/template/node_chaos_template.cue b/internal/cli/create/template/node_chaos_template.cue new file mode 100644 index 000000000..cc074bf23 --- /dev/null +++ b/internal/cli/create/template/node_chaos_template.cue @@ -0,0 +1,65 @@ +// Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// required, command line input options for parameters and flags +options: { + kind: string + namespace: string + action: string + secretName: string + region: string + instance: string + volumeID: string + deviceName?: string + duration: string + + project: string +} + +// required, k8s api resource content +content: { + kind: options.kind + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "node-chaos-" + namespace: options.namespace + } + spec:{ + action: options.action + secretName: options.secretName + duration: options.duration + + if options.kind == "AWSChaos" { + awsRegion: options.region + ec2Instance: options.instance + if options.deviceName != _|_ { + volumeID: options.volumeID + deviceName: options.deviceName + } + } + + if options.kind == "GCPChaos" { + project: options.project + zone: options.region + instance: options.instance + if options.deviceName != _|_ { + deviceNames:[options.deviceName] + } + + } + } +} From 25877b7e5c578038d2b9a00b1bb24b3cac0bea84 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Mon, 22 May 2023 15:43:28 +0800 Subject: [PATCH 330/439] feat: improve addon description and show addon failed message when install/uninstall kubeblocks (#3300) --- internal/cli/cmd/addon/addon.go | 43 +++++-- internal/cli/cmd/kubeblocks/config_test.go | 2 +- internal/cli/cmd/kubeblocks/install.go | 93 +++++++-------- internal/cli/cmd/kubeblocks/uninstall.go | 126 ++++++++++----------- internal/cli/cmd/kubeblocks/util.go | 108 ++++++++++++++++++ internal/cli/cmd/kubeblocks/util_test.go | 67 +++++++++++ internal/cli/printer/printer.go | 3 + 7 files changed, 314 insertions(+), 128 deletions(-) diff --git a/internal/cli/cmd/addon/addon.go b/internal/cli/cmd/addon/addon.go index ee29d21ce..5f12e67c2 100644 --- a/internal/cli/cmd/addon/addon.go +++ b/internal/cli/cmd/addon/addon.go @@ -29,6 +29,7 @@ import ( "strings" "github.com/docker/cli/cli" + "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -130,6 +131,7 @@ func newDescribeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cob Use: "describe ADDON_NAME", Short: "Describe an addon specification.", Args: cli.ExactArgs(1), + Aliases: []string{"desc"}, ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.AddonGVR()), Run: func(cmd *cobra.Command, args []string) { util.CheckErr(o.init(args)) @@ -366,7 +368,7 @@ func addonDescribeHandler(o *addonCmdOpts, cmd *cobra.Command, args []string) er return nil } - labels := []string{} + var labels []string for k, v := range o.addon.Labels { if strings.Contains(k, constant.APIGroup) { labels = append(labels, fmt.Sprintf("%s=%s", k, v)) @@ -376,14 +378,18 @@ func addonDescribeHandler(o *addonCmdOpts, cmd *cobra.Command, args []string) er printer.PrintPairStringToLine("Description", o.addon.Spec.Description, 0) printer.PrintPairStringToLine("Labels", strings.Join(labels, ","), 0) printer.PrintPairStringToLine("Type", string(o.addon.Spec.Type), 0) - printer.PrintPairStringToLine("Extras", strings.Join(o.addon.GetExtraNames(), ","), 0) + if len(o.addon.GetExtraNames()) > 0 { + printer.PrintPairStringToLine("Extras", strings.Join(o.addon.GetExtraNames(), ","), 0) + } printer.PrintPairStringToLine("Status", string(o.addon.Status.Phase), 0) var autoInstall bool if o.addon.Spec.Installable != nil { autoInstall = o.addon.Spec.Installable.AutoInstall } printer.PrintPairStringToLine("Auto-install", strconv.FormatBool(autoInstall), 0) - printer.PrintPairStringToLine("Auto-install selector", strings.Join(o.addon.Spec.Installable.GetSelectorsStrings(), ","), 0) + if len(o.addon.Spec.Installable.GetSelectorsStrings()) > 0 { + printer.PrintPairStringToLine("Auto-install selector", strings.Join(o.addon.Spec.Installable.GetSelectorsStrings(), ","), 0) + } switch o.addon.Status.Phase { case extensionsv1alpha1.AddonEnabled: @@ -391,7 +397,7 @@ func addonDescribeHandler(o *addonCmdOpts, cmd *cobra.Command, args []string) er printer.PrintLineWithTabSeparator() if err := printer.PrintTable(o.Out, nil, printInstalled, "NAME", "REPLICAS", "STORAGE", "CPU (REQ/LIMIT)", "MEMORY (REQ/LIMIT)", "STORAGE-CLASS", - "TOLERATIONS", "PV Enabled"); err != nil { + "TOLERATIONS", "PV-ENABLED"); err != nil { return err } default: @@ -416,12 +422,35 @@ func addonDescribeHandler(o *addonCmdOpts, cmd *cobra.Command, args []string) er } if err := printer.PrintTable(o.Out, nil, printInstallable, "NAME", "REPLICAS", "STORAGE", "CPU (REQ/LIMIT)", "MEMORY (REQ/LIMIT)", "STORAGE-CLASS", - "TOLERATIONS", "PV Enabled"); err != nil { + "TOLERATIONS", "PV-ENABLED"); err != nil { return err } printer.PrintLineWithTabSeparator() } } + + // print failed message + if o.addon.Status.Phase == extensionsv1alpha1.AddonFailed { + var tbl *printer.TablePrinter + printHeader := true + for _, c := range o.addon.Status.Conditions { + if c.Status == metav1.ConditionTrue { + continue + } + if printHeader { + fmt.Fprintln(o.Out, "Failed Message") + tbl = printer.NewTablePrinter(o.Out) + tbl.Tbl.SetColumnConfigs([]table.ColumnConfig{ + {Number: 3, WidthMax: 120}, + }) + tbl.SetHeader("TIME", "REASON", "MESSAGE") + printHeader = false + } + tbl.AddRow(util.TimeFormat(&c.LastTransitionTime), c.Reason, c.Message) + } + tbl.Print() + } + return nil } @@ -531,10 +560,10 @@ func (o *addonCmdOpts) buildEnablePatch(flags []*pflag.Flag, spec, install map[s valueTransformer func(s, flag string) (interface{}, error), valueAssigner func(*extensionsv1alpha1.AddonInstallSpecItem, interface{}), ) error { - var jsonArrary []map[string]interface{} + var jsonArray []map[string]interface{} var t []string - err := json.Unmarshal([]byte(s), &jsonArrary) + err := json.Unmarshal([]byte(s), &jsonArray) if err != nil { // not a valid JSON array treat it a 2 tuples t = strings.SplitN(s, ":", 2) diff --git a/internal/cli/cmd/kubeblocks/config_test.go b/internal/cli/cmd/kubeblocks/config_test.go index 19358af1e..a8fd85d09 100644 --- a/internal/cli/cmd/kubeblocks/config_test.go +++ b/internal/cli/cmd/kubeblocks/config_test.go @@ -106,7 +106,7 @@ var _ = Describe("backupconfig", func() { } cmd := NewConfigCmd(tf, streams) Expect(cmd).ShouldNot(BeNil()) - Expect(o.PreCheck()).Should(Succeed()) + Expect(o.PreCheck()).Should(HaveOccurred()) }) It("run describe config cmd", func() { diff --git a/internal/cli/cmd/kubeblocks/install.go b/internal/cli/cmd/kubeblocks/install.go index bb1ff5ee5..12aa843f2 100644 --- a/internal/cli/cmd/kubeblocks/install.go +++ b/internal/cli/cmd/kubeblocks/install.go @@ -31,6 +31,7 @@ import ( "github.com/replicatedhq/troubleshoot/pkg/preflight" "github.com/spf13/cobra" "github.com/spf13/pflag" + "golang.org/x/exp/maps" "helm.sh/helm/v3/pkg/cli/values" "helm.sh/helm/v3/pkg/repo" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -80,6 +81,13 @@ type InstallOptions struct { ValueOpts values.Options } +type addonStatus struct { + allEnabled bool + allDisabled bool + hasFailed bool + outputMsg string +} + var ( installExample = templates.Examples(` # Install KubeBlocks, the default version is same with the kbcli version, the default namespace is kb-system @@ -190,7 +198,7 @@ func (o *InstallOptions) PreCheck() error { if v.KubeBlocks != "" { printer.Warning(o.Out, "KubeBlocks %s already exists, repeated installation is not supported.\n\n", v.KubeBlocks) fmt.Fprintln(o.Out, "If you want to upgrade it, please use \"kbcli kubeblocks upgrade\".") - return nil + return cmdutil.ErrExit } // check whether the namespace exists @@ -234,6 +242,7 @@ func (o *InstallOptions) Install() error { // wait for auto-install addons to be ready if err = o.waitAddonsEnabled(); err != nil { + fmt.Fprintf(o.Out, "Failed to wait for auto-install addons to be enabled, run \"kbcli kubeblocks status\" to check the status\n") return err } @@ -257,102 +266,82 @@ func (o *InstallOptions) waitAddonsEnabled() error { return nil } - // addons record the addons and its status - addons := make(map[string]string) - checkAddons := func() (bool, error) { - allEnabled := true + addons := make(map[string]*extensionsv1alpha1.Addon) + fetchAddons := func() error { objs, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ LabelSelector: buildKubeBlocksSelectorLabels(), }) if err != nil && !apierrors.IsNotFound(err) { - return false, err + return err } if objs == nil || len(objs.Items) == 0 { klog.V(1).Info("No Addons found") - return true, nil + return nil } for _, obj := range objs.Items { - addon := extensionsv1alpha1.Addon{} - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &addon); err != nil { - return false, err + addon := &extensionsv1alpha1.Addon{} + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, addon); err != nil { + return err } if addon.Status.ObservedGeneration == 0 { klog.V(1).Infof("Addon %s is not observed yet", addon.Name) - allEnabled = false continue } // addon should be auto installed, check its status if addon.Spec.InstallSpec.GetEnabled() { - addons[addon.Name] = string(addon.Status.Phase) + addons[addon.Name] = addon if addon.Status.Phase != extensionsv1alpha1.AddonEnabled { klog.V(1).Infof("Addon %s is not enabled yet, status %s", addon.Name, addon.Status.Phase) - allEnabled = false + } + if addon.Status.Phase == extensionsv1alpha1.AddonFailed { + klog.V(1).Infof("Addon %s failed:", addon.Name) + for _, c := range addon.Status.Conditions { + klog.V(1).Infof(" %s: %s", c.Reason, c.Message) + } } } } - return allEnabled, nil + return nil } suffixMsg := func(msg string) string { return fmt.Sprintf("%-50s", msg) } - addonMsg := func(msg, status string) string { - return fmt.Sprintf("%-48s %s", msg, status) - } - // create spinner - allMsg := "" - msg := "Wait for addons to be enabled" - s := spinner.New(o.Out, spinnerMsg(msg)) - - // check addon installing progress - checkProgress := func() { - if len(addons) == 0 { - return - } - all := make([]string, 0) - for k, v := range addons { - if v == string(extensionsv1alpha1.AddonEnabled) { - all = append(all, addonMsg("Addon "+k, printer.BoldGreen("OK"))) - continue - } - all = append(all, addonMsg("Addon "+k, v)) - } - sort.Strings(all) - allMsg = fmt.Sprintf("%s\n %s", msg, strings.Join(all, "\n ")) - s.SetMessage(suffixMsg(allMsg)) - } - + msg := "" + header := "Wait for addons to be enabled" + failedErr := errors.New("there are some addons failed to be enabled") + s := spinner.New(o.Out, spinnerMsg(header)) var ( - allEnabled bool err error - spinnerDone = func(s spinner.Interface) { - s.SetFinalMsg(allMsg) + spinnerDone = func() { + s.SetFinalMsg(msg) s.Done("") fmt.Fprintln(o.Out) } ) // wait all addons to be enabled, or timeout if err = wait.PollImmediate(5*time.Second, o.Timeout, func() (bool, error) { - allEnabled, err = checkAddons() - if err != nil { + if err = fetchAddons(); err != nil || len(addons) == 0 { return false, err } - checkProgress() - if allEnabled { - spinnerDone(s) + status := checkAddons(maps.Values(addons), true) + msg = suffixMsg(fmt.Sprintf("%s\n %s", header, status.outputMsg)) + s.SetMessage(msg) + if status.allEnabled { + spinnerDone() return true, nil + } else if status.hasFailed { + return false, failedErr } return false, nil }); err != nil { - spinnerDone(s) - if err == wait.ErrWaitTimeout { - return errors.New("timeout waiting for auto-install addons to be enabled, run \"kbcli addon list\" to check addon status") - } + spinnerDone() + printAddonMsg(o.Out, maps.Values(addons), true) return err } diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index 6ebdd8e77..b193f12d5 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -29,6 +29,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" + "golang.org/x/exp/maps" "helm.sh/helm/v3/pkg/repo" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -146,6 +147,7 @@ func (o *UninstallOptions) Uninstall() error { // uninstall all KubeBlocks addons if err := o.uninstallAddons(); err != nil { + fmt.Fprintf(o.Out, "Failed to uninstall addons, run \"kbcli kubeblocks uninstall\" to retry.\n") return err } @@ -215,78 +217,67 @@ func (o *UninstallOptions) Uninstall() error { // uninstallAddons uninstall all KubeBlocks addons func (o *UninstallOptions) uninstallAddons() error { - addonStatus := make(map[string]string) - var ( allErrs []error err error - msg = "Wait for addons to be disabled" - - processAddons = func(uninstall bool) error { - objects, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ - LabelSelector: buildKubeBlocksSelectorLabels(), - }) - if err != nil && !apierrors.IsNotFound(err) { - klog.V(1).Infof("Failed to get KubeBlocks addons %s", err.Error()) + header = "Wait for addons to be disabled" + s spinner.Interface + msg string + ) + + addons := make(map[string]*extensionsv1alpha1.Addon) + processAddons := func(uninstall bool) error { + objects, err := o.Dynamic.Resource(types.AddonGVR()).List(context.TODO(), metav1.ListOptions{ + LabelSelector: buildKubeBlocksSelectorLabels(), + }) + if err != nil && !apierrors.IsNotFound(err) { + klog.V(1).Infof("Failed to get KubeBlocks addons %s", err.Error()) + allErrs = append(allErrs, err) + return utilerrors.NewAggregate(allErrs) + } + if objects == nil { + return nil + } + + for _, obj := range objects.Items { + addon := &extensionsv1alpha1.Addon{} + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, addon); err != nil { + klog.V(1).Infof("Failed to convert KubeBlocks addon %s", err.Error()) allErrs = append(allErrs, err) - return utilerrors.NewAggregate(allErrs) - } - if objects == nil { - return nil + continue } - for _, obj := range objects.Items { - addon := extensionsv1alpha1.Addon{} - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &addon); err != nil { - klog.V(1).Infof("Failed to convert KubeBlocks addon %s", err.Error()) - allErrs = append(allErrs, err) + if uninstall { + // we only need to uninstall addons that are not disabled + if addon.Status.Phase == extensionsv1alpha1.AddonDisabled { continue } + addons[addon.Name] = addon + o.addons = append(o.addons, addon) - if uninstall { - // we only need to uninstall addons that are not disabled - if addon.Status.Phase == extensionsv1alpha1.AddonDisabled { - continue - } - addonStatus[addon.Name] = string(addon.Status.Phase) - o.addons = append(o.addons, &addon) - - // uninstall addons - if err = disableAddon(o.Dynamic, &addon); err != nil { - klog.V(1).Infof("Failed to uninstall KubeBlocks addon %s %s", addon.Name, err.Error()) - allErrs = append(allErrs, err) - } - } else { - // update addons if exists - if _, ok := addonStatus[addon.Name]; ok { - addonStatus[addon.Name] = string(addon.Status.Phase) - } + // uninstall addons + if err = disableAddon(o.Dynamic, addon); err != nil { + klog.V(1).Infof("Failed to uninstall KubeBlocks addon %s %s", addon.Name, err.Error()) + allErrs = append(allErrs, err) } - } - return utilerrors.NewAggregate(allErrs) - } - - buildMsg = func() (string, bool) { - var addonMsg []string - allDisabled := true - for k, v := range addonStatus { - if v == string(extensionsv1alpha1.AddonDisabled) { - v = printer.BoldGreen("OK") - } else { - allDisabled = false + } else { + // update cached addon if exists + if _, ok := addons[addon.Name]; ok { + addons[addon.Name] = addon } - addonMsg = append(addonMsg, fmt.Sprintf("%-48s %s", "Addon "+k, v)) } - sort.Strings(addonMsg) - return fmt.Sprintf("%-50s\n %s", msg, strings.Join(addonMsg, "\n ")), allDisabled } - ) + return utilerrors.NewAggregate(allErrs) + } + + suffixMsg := func(msg string) string { + return fmt.Sprintf("%-50s", msg) + } - var s spinner.Interface if !o.Wait { s = spinner.New(o.Out, spinner.WithMessage(fmt.Sprintf("%-50s", "Uninstall KubeBlocks addons"))) } else { - s = spinner.New(o.Out, spinner.WithMessage(fmt.Sprintf("%-50s", msg))) + s = spinner.New(o.Out, spinner.WithMessage(fmt.Sprintf("%-50s", header))) } // get all addons and uninstall them @@ -295,12 +286,12 @@ func (o *UninstallOptions) uninstallAddons() error { return err } - if len(addonStatus) == 0 || !o.Wait { + if len(addons) == 0 || !o.Wait { s.Success() return nil } - spinnerDone := func(s spinner.Interface, msg string) { + spinnerDone := func() { s.SetFinalMsg(msg) s.Done("") fmt.Fprintln(o.Out) @@ -313,21 +304,20 @@ func (o *UninstallOptions) uninstallAddons() error { if err = processAddons(false); err != nil { return false, err } - m, allDisabled := buildMsg() - s.SetMessage(m) - if allDisabled { - spinnerDone(s, m) + status := checkAddons(maps.Values(addons), false) + msg = suffixMsg(fmt.Sprintf("%s\n %s", header, status.outputMsg)) + s.SetMessage(msg) + if status.allDisabled { + spinnerDone() return true, nil + } else if status.hasFailed { + return false, errors.New("there are some addons failed to disabled") } return false, nil }); err != nil { - m, _ := buildMsg() - spinnerDone(s, m) - if err == wait.ErrWaitTimeout { - allErrs = append(allErrs, errors.New("timeout waiting for addons to be disabled, run \"kbcli addon list\" to check addon status")) - } else { - allErrs = append(allErrs, err) - } + spinnerDone() + printAddonMsg(o.Out, maps.Values(addons), false) + allErrs = append(allErrs, err) } return utilerrors.NewAggregate(allErrs) } diff --git a/internal/cli/cmd/kubeblocks/util.go b/internal/cli/cmd/kubeblocks/util.go index fb320939e..520b4a1bb 100644 --- a/internal/cli/cmd/kubeblocks/util.go +++ b/internal/cli/cmd/kubeblocks/util.go @@ -23,9 +23,11 @@ import ( "context" "fmt" "io" + "sort" "strings" "github.com/Masterminds/semver/v3" + "github.com/jedib0t/go-pretty/v6/table" "github.com/pkg/errors" "golang.org/x/exp/slices" "helm.sh/helm/v3/pkg/repo" @@ -35,6 +37,7 @@ import ( "k8s.io/client-go/kubernetes" extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/helm" @@ -161,3 +164,108 @@ func buildKubeBlocksSelectorLabels() string { func buildConfigTypeSelectorLabels() string { return fmt.Sprintf("%s=%s", constant.CMConfigurationTypeLabelKey, constant.ConfigTemplateType) } + +// printAddonMsg print addon message when has failed addon or timeout +func printAddonMsg(out io.Writer, addons []*extensionsv1alpha1.Addon, install bool) { + var ( + enablingAddons []string + disablingAddons []string + failedAddons []*extensionsv1alpha1.Addon + ) + + for _, addon := range addons { + switch addon.Status.Phase { + case extensionsv1alpha1.AddonEnabling: + enablingAddons = append(enablingAddons, addon.Name) + case extensionsv1alpha1.AddonDisabling: + disablingAddons = append(disablingAddons, addon.Name) + case extensionsv1alpha1.AddonFailed: + for _, c := range addon.Status.Conditions { + if c.Status == metav1.ConditionFalse { + failedAddons = append(failedAddons, addon) + break + } + } + } + } + + // print failed addon messages + if len(failedAddons) > 0 { + printFailedAddonMsg(out, failedAddons) + } + + // print enabling addon messages + if install && len(enablingAddons) > 0 { + fmt.Fprintf(out, "\nEnabling addons: %s\n", strings.Join(enablingAddons, ", ")) + fmt.Fprintf(out, "Please wait for a while and try to run \"kbcli addon list\" to check addons status.\n") + } + + if !install && len(disablingAddons) > 0 { + fmt.Fprintf(out, "\nDisabling addons: %s\n", strings.Join(disablingAddons, ", ")) + fmt.Fprintf(out, "Please wait for a while and try to run \"kbcli addon list\" to check addons status.\n") + } +} + +func printFailedAddonMsg(out io.Writer, addons []*extensionsv1alpha1.Addon) { + fmt.Fprintf(out, "\nFailed addons:\n") + tbl := printer.NewTablePrinter(out) + tbl.Tbl.SetColumnConfigs([]table.ColumnConfig{ + {Number: 4, WidthMax: 120}, + }) + tbl.SetHeader("NAME", "TIME", "REASON", "MESSAGE") + for _, addon := range addons { + var times, reasons, messages []string + for _, c := range addon.Status.Conditions { + if c.Status != metav1.ConditionFalse { + continue + } + times = append(times, util.TimeFormat(&c.LastTransitionTime)) + reasons = append(reasons, c.Reason) + messages = append(messages, c.Message) + } + tbl.AddRow(addon.Name, strings.Join(times, "\n"), strings.Join(reasons, "\n"), strings.Join(messages, "\n")) + } + tbl.Print() +} + +func checkAddons(addons []*extensionsv1alpha1.Addon, install bool) *addonStatus { + status := &addonStatus{ + allEnabled: true, + allDisabled: true, + hasFailed: false, + outputMsg: "", + } + + if len(addons) == 0 { + return status + } + + all := make([]string, 0) + for _, addon := range addons { + s := string(addon.Status.Phase) + switch addon.Status.Phase { + case extensionsv1alpha1.AddonEnabled: + if install { + s = printer.BoldGreen("OK") + } + status.allDisabled = false + case extensionsv1alpha1.AddonDisabled: + if !install { + s = printer.BoldGreen("OK") + } + status.allEnabled = false + case extensionsv1alpha1.AddonFailed: + status.hasFailed = true + status.allEnabled = false + status.allDisabled = false + case extensionsv1alpha1.AddonDisabling: + status.allDisabled = false + case extensionsv1alpha1.AddonEnabling: + status.allEnabled = false + } + all = append(all, fmt.Sprintf("%-48s %s", addon.Name, s)) + } + sort.Strings(all) + status.outputMsg = strings.Join(all, "\n ") + return status +} diff --git a/internal/cli/cmd/kubeblocks/util_test.go b/internal/cli/cmd/kubeblocks/util_test.go index 86fddcf7d..4a8d74792 100644 --- a/internal/cli/cmd/kubeblocks/util_test.go +++ b/internal/cli/cmd/kubeblocks/util_test.go @@ -24,8 +24,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + extensionsv1alpha1 "github.com/apecloud/kubeblocks/apis/extensions/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" ) @@ -75,4 +78,68 @@ var _ = Describe("kubeblocks", func() { _, _ = in.Write([]byte("uninstall-kubeblocks\n")) Expect(confirmUninstall(in)).Should(Succeed()) }) + + It("printAddonMsg", func() { + const ( + reason = "test-failed-reason" + ) + + fakeAddOn := func(name string, conditionTrue bool, msg string) *extensionsv1alpha1.Addon { + addon := &extensionsv1alpha1.Addon{} + addon.Name = name + addon.Status = extensionsv1alpha1.AddonStatus{} + if conditionTrue { + addon.Status.Phase = extensionsv1alpha1.AddonEnabled + } else { + addon.Status.Phase = extensionsv1alpha1.AddonFailed + addon.Status.Conditions = []metav1.Condition{ + { + Message: msg, + Reason: reason, + Status: metav1.ConditionFalse, + }, + { + Message: msg, + Reason: reason, + Status: metav1.ConditionFalse, + }, + } + } + return addon + } + + testCases := []struct { + desc string + addons []*extensionsv1alpha1.Addon + expected string + }{ + { + desc: "addons is nil", + addons: nil, + expected: "", + }, + { + desc: "addons without false condition", + addons: []*extensionsv1alpha1.Addon{ + fakeAddOn("addon", true, ""), + }, + expected: "", + }, + { + desc: "addons with false condition", + addons: []*extensionsv1alpha1.Addon{ + fakeAddOn("addon1", true, ""), + fakeAddOn("addon2", false, "failed to enable addon2"), + }, + expected: "failed to enable addon2", + }, + } + + for _, c := range testCases { + By(c.desc) + out := &bytes.Buffer{} + printAddonMsg(out, c.addons, true) + Expect(out.String()).To(ContainSubstring(c.expected)) + } + }) }) diff --git a/internal/cli/printer/printer.go b/internal/cli/printer/printer.go index 0fe436b9e..acbbe2881 100644 --- a/internal/cli/printer/printer.go +++ b/internal/cli/printer/printer.go @@ -111,6 +111,9 @@ func (t *TablePrinter) AddRow(row ...interface{}) { } func (t *TablePrinter) Print() { + if t == nil || t.Tbl == nil { + return + } t.Tbl.Render() } From 249ddc626305768b699863f5a557522aaf3ed2d0 Mon Sep 17 00:00:00 2001 From: huangzhangshu <109708205+JashBook@users.noreply.github.com> Date: Mon, 22 May 2023 17:08:05 +0800 Subject: [PATCH 331/439] chore: adjust release workflows (#3329) --- .github/utils/utils.sh | 19 +++++++++--------- .github/workflows/cicd-pull-request.yml | 11 +++++++--- .github/workflows/cicd-push.yml | 11 ++++++---- .github/workflows/release-create.yml | 2 +- .github/workflows/release-helm-chart.yml | 3 ++- .github/workflows/release-image.yml | 4 ++-- deploy/delphic/Chart.lock | 9 --------- .../charts/pgcluster-0.5.0-beta.24.tgz | Bin 2826 -> 0 bytes .../charts/redis-cluster-0.5.0-beta.24.tgz | Bin 2669 -> 0 bytes 9 files changed, 29 insertions(+), 30 deletions(-) delete mode 100644 deploy/delphic/Chart.lock delete mode 100644 deploy/delphic/charts/pgcluster-0.5.0-beta.24.tgz delete mode 100644 deploy/delphic/charts/redis-cluster-0.5.0-beta.24.tgz diff --git a/.github/utils/utils.sh b/.github/utils/utils.sh index 51e2fe2d0..11525a89d 100644 --- a/.github/utils/utils.sh +++ b/.github/utils/utils.sh @@ -236,7 +236,7 @@ check_numeric() { get_next_available_tag() { tag_type="$1" index="" - release_list=$( gh release list --repo $LATEST_REPO ) + release_list=$( gh release list --repo $LATEST_REPO --limit 100 ) for tag in $( echo "$release_list" | (grep "$tag_type" || true) ) ;do if [[ "$tag" != "$tag_type"* ]]; then continue @@ -246,7 +246,7 @@ get_next_available_tag() { if [[ "$numeric" == "no" ]]; then continue fi - if [[ $numeric -gt $index ]]; then + if [[ $numeric -gt $index || -z "$index" ]]; then index=$numeric fi done @@ -262,11 +262,13 @@ get_next_available_tag() { release_next_available_tag() { dispatches_url=$1 - v_head="v$TAG_NAME" - alpha_type="$v_head.0-alpha." - beta_type="$v_head.0-beta." - rc_type="$v_head.0-rc." - stable_type="$v_head." + v_major_minor="v$TAG_NAME" + stable_type="$v_major_minor." + get_next_available_tag $stable_type + v_number=$RELEASE_VERSION + alpha_type="$v_number-alpha." + beta_type="$v_number-beta." + rc_type="$v_number-rc." case "$CONTENT" in *alpha*) get_next_available_tag "$alpha_type" @@ -277,9 +279,6 @@ release_next_available_tag() { *rc*) get_next_available_tag "$rc_type" ;; - *stable*) - get_next_available_tag $stable_type - ;; esac if [[ ! -z "$RELEASE_VERSION" ]];then diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index 9b793f97e..306a3a254 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -72,37 +72,42 @@ jobs: name: check image needs: trigger-mode if: contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') - uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 + uses: apecloud/apecd/.github/workflows/release-image.yml@v0.6.1 with: MAKE_OPS_PRE: "generate" MAKE_OPS: "build-manager-image" IMG: "apecloud/kubeblocks" VERSION: "check" GO_VERSION: "1.20" + BUILDX_PLATFORMS: "linux/amd64" + SYNC_ENABLE: false secrets: inherit check-tools-image: name: check image needs: trigger-mode if: contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') - uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 + uses: apecloud/apecd/.github/workflows/release-image.yml@v0.6.1 with: MAKE_OPS_PRE: "generate" MAKE_OPS: "build-tools-image" IMG: "apecloud/kubeblocks-tools" VERSION: "check" GO_VERSION: "1.20" + BUILDX_PLATFORMS: "linux/amd64" + SYNC_ENABLE: false secrets: inherit check-helm: name: check helm needs: trigger-mode if: contains(needs.trigger-mode.outputs.trigger-mode, '[deploy]') - uses: apecloud/apecd/.github/workflows/release-charts.yml@v0.5.2 + uses: apecloud/apecd/.github/workflows/release-charts.yml@v0.6.1 with: MAKE_OPS: "bump-chart-ver" VERSION: "v0.4.0-check" CHART_NAME: "kubeblocks" CHART_DIR: "deploy/helm" PUSH_ENABLE: false + DEP_REPO: "helm dep update deploy/delphic" secrets: inherit diff --git a/.github/workflows/cicd-push.yml b/.github/workflows/cicd-push.yml index 8faaecbc8..e8776c2d4 100644 --- a/.github/workflows/cicd-push.yml +++ b/.github/workflows/cicd-push.yml @@ -57,7 +57,7 @@ jobs: - name: pcregrep Chinese run: | FILE_PATH=`git diff --name-only HEAD HEAD^` - + python ${{ github.workspace }}/.github/utils/pcregrep.py \ --source="${{ github.workspace }}/pcregrep.out" \ --filter="$FILE_PATH" @@ -168,7 +168,7 @@ jobs: check-image: needs: trigger-mode if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') && github.ref_name != 'main' }} - uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 + uses: apecloud/apecd/.github/workflows/release-image.yml@v0.6.1 with: MAKE_OPS_PRE: "generate" MAKE_OPS: "build-manager-image" @@ -176,12 +176,13 @@ jobs: VERSION: "check" GO_VERSION: "1.20" BUILDX_PLATFORMS: "linux/amd64" + SYNC_ENABLE: false secrets: inherit check-tools-image: needs: trigger-mode if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') && github.ref_name != 'main' }} - uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 + uses: apecloud/apecd/.github/workflows/release-image.yml@v0.6.1 with: MAKE_OPS_PRE: "generate" MAKE_OPS: "build-tools-image" @@ -189,18 +190,20 @@ jobs: VERSION: "check" GO_VERSION: "1.20" BUILDX_PLATFORMS: "linux/amd64" + SYNC_ENABLE: false secrets: inherit check-helm: needs: trigger-mode if: ${{ contains(needs.trigger-mode.outputs.trigger-mode, '[deploy]') && github.ref_name != 'main' }} - uses: apecloud/apecd/.github/workflows/release-charts.yml@v0.5.2 + uses: apecloud/apecd/.github/workflows/release-charts.yml@v0.6.1 with: MAKE_OPS: "bump-chart-ver" VERSION: "v0.4.0-check" CHART_NAME: "kubeblocks" CHART_DIR: "deploy/helm" PUSH_ENABLE: false + DEP_REPO: "helm dep update deploy/delphic" secrets: inherit deploy-kubeblocks-io: diff --git a/.github/workflows/release-create.yml b/.github/workflows/release-create.yml index da7503da3..7a9389b80 100644 --- a/.github/workflows/release-create.yml +++ b/.github/workflows/release-create.yml @@ -56,6 +56,6 @@ jobs: - name: send message run: | bash .github/utils/utils.sh --type 12 \ - --content "release\u00a0${{ ${{ needs.publish.outputs.rel-version }} }}\u00a0create\u00a0error"\ + --content "release\u00a0${{ needs.publish.outputs.rel-version }}\u00a0create\u00a0error"\ --bot-webhook ${{ env.RELEASE_BOT_WEBHOOK }} \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ No newline at end of file diff --git a/.github/workflows/release-helm-chart.yml b/.github/workflows/release-helm-chart.yml index 58246bf9a..cf9a4a738 100644 --- a/.github/workflows/release-helm-chart.yml +++ b/.github/workflows/release-helm-chart.yml @@ -33,13 +33,14 @@ jobs: release-chart: needs: chart-version - uses: apecloud/apecd/.github/workflows/release-charts.yml@v0.5.2 + uses: apecloud/apecd/.github/workflows/release-charts.yml@v0.6.1 with: MAKE_OPS: "bump-chart-ver" VERSION: "${{ needs.chart-version.outputs.chart-version }}" CHART_NAME: "kubeblocks" CHART_DIR: "deploy/helm" DEP_CHART_DIR: "deploy/helm/depend-charts" + DEP_REPO: "helm dep update deploy/delphic" secrets: inherit send-message: diff --git a/.github/workflows/release-image.yml b/.github/workflows/release-image.yml index ded157d27..8e3eb522a 100644 --- a/.github/workflows/release-image.yml +++ b/.github/workflows/release-image.yml @@ -39,7 +39,7 @@ jobs: release-image: needs: image-tag - uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 + uses: apecloud/apecd/.github/workflows/release-image.yml@v0.6.1 with: MAKE_OPS_PRE: "generate" MAKE_OPS: "push-manager-image" @@ -50,7 +50,7 @@ jobs: release-tools-image: needs: image-tag - uses: apecloud/apecd/.github/workflows/release-image.yml@v0.2.0 + uses: apecloud/apecd/.github/workflows/release-image.yml@v0.6.1 with: MAKE_OPS_PRE: "generate" MAKE_OPS: "push-tools-image" diff --git a/deploy/delphic/Chart.lock b/deploy/delphic/Chart.lock deleted file mode 100644 index da225dbcc..000000000 --- a/deploy/delphic/Chart.lock +++ /dev/null @@ -1,9 +0,0 @@ -dependencies: -- name: pgcluster - repository: file://../postgresql-cluster - version: 0.5.0-beta.24 -- name: redis-cluster - repository: file://../redis-cluster - version: 0.5.0-beta.24 -digest: sha256:d0aefa69ab29206d5ba039b8cac7729b518023e66cd5965933ad20ced95f8477 -generated: "2023-05-17T16:57:07.087916+08:00" diff --git a/deploy/delphic/charts/pgcluster-0.5.0-beta.24.tgz b/deploy/delphic/charts/pgcluster-0.5.0-beta.24.tgz deleted file mode 100644 index 3449325ed9dcee89ed86cf7e3a31fd8e4bc5816f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2826 zcmV+l3-$CLiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PH(kbK5qvdFHR!Be#<{$ATqWw$Es$4~gArbDG;1chc!o2O^ge zY7k%mP>$mG`|aQ0!mDXZP93LT$9S+v5LoOk_6|6o$Gp%6)$Z|vDnnK@zh1(Vb7mIsmKr4mgP)&mL%;@pMxH@z zr4>@C;Fn^C&pir}pczh~lIh5-a-0IqbIxK%WV9?17?A@q8qUyAvNwso^)`I7R_p(g z@&fhOFaT@xfA3(j-_`%gWb~l__tCcC1XEgY1AhOVO5LP}B-(;s7fb`wK;idiuU-yQ zsWNH|CXh0YcEAb7oGOGRRgBI!YA_OJ2szan6JWwfSjj?x!7S(0peB)1-Zx8BiX}LO z!5~Tt&aZSv3Y|6|%R-nb>_rAu#)Q-Ex017XHH8!8XfTS}_NxshS1hAy^+F{0aSEd- z(&x;?ivS`5@V4W%lz9|oQZOTx?M7h)ejE;6-XV z?AEs+XklbV4T~w~t8yue#17;fsYXDd3xy@PX8~Lw_Xlgx?asb23smP!cKxO9Miv*dd$6~;2lP$VFvu{i4-W0tNU3CdMX zcT^h*9O=?>%4(5Sq*w|aGg)W{I$lT?qtnHb%fxFPN%_{H|jrmqAVn#aA#>)bS>@{{3;y>&GMtP%~y+fU*Su zab8T}U^L3Ydxn`*t10aL!~%sM3e=tP>Eh3{pb&_RmA%$IWR79QHo5SA_9+Qi@3YX4uE;@#J8e*N*3n2+X;)>s$+?;jp? z{Qu-=Gykd+rdR0U!OD1YrnKnbnZS_Fxr~a5N=x=VEbH-33VPoqCL{} zRMm0`OH)+V%mgSqt#PGcvT(~~%tFaxzJPwm??W6C{k=8vep{dZ|FJ-B%_1@RoumS5 z^nZVH)Q$fS4o461|Gl(}i`}Quj~{avmXtw|OucV*6`|W|)?;crpTBIra2!r7O0xTO-U0z0! z!}GW@4PgbFA*|p-K{-nqCf2ez3P|)Ddf0Bfu`5_=G|VujHlWlpLsvRb0u5?Ucgm0_ z8f@m`A`Wk8T#)u;<}J<1>r*?{1+_WG+a!9Cf3I9B}dK*&4KEI+rzv(}Lhu_ba%Wj>&Mg z@_D7GCm)@nRDJqkbY{I(8gwrZCaYDSQApC8$dVB&rKNA zri#=+yIRzBHX(I~P{#;~|7NUfCjGB_UD1iw4VUb?DI?57(o6`Gv46oVL7~E@f!<#SFgZP`O`r^q|tFb^hph_ zf*s2&mjZ=3%`w)Vpka`m*7g+|)zBOJ#EojqJ;8Lt{jTTeatCZH>-bIHhrY7X4WH}m zf`%y5PYwqUx;49b8qt`#S5Ld@p4MqWpF<-x!1l>S1DEE91}>upE}b(D-~35O15k4q zv0p0y%_3)WMx)U&#b@dl^g7 zJEX~PiWz((Vr+fwk+blE5D-LAh+1+p*xtBxD*XpJ@T-}vxg+>QT65BGoW zr*%w5|LLz4+2L^5y3eC#G$yn#3#r&A7ZH%N{|hl=h!)>Tjy<8`*=)EjTC^`Zr+s;< z>u$xAA&~i^xKb4y!*LzW*-uIqd8O<_7!0Cb^Du;1ij>VWnrqj!L^W%Ebt!K3I!9AI zbEfN;oSJx1y%l-VoORy#xCM#3($GAf1}XT;9T7jy{W(l*csN>pTtV^kZciYJB$rH> zUiz8@-AF@+5iR1sEq%F6){=Jny}B)vmv{|_1LGPl<16X^b@sn`$K9F*d=8cnGI0 zE;3`3SvcDu_Ywv$d4m>j`j{8^#49u(0?itc;YUvuCWB$HxF~C zWZQLh8B)qP46XszzJJ>Xs*w+n?&0;fX5Bxc)%s5s$TKzFz)Z=)A)?eQCI)@p^H41%4pCF%NUo#}{UgE>q`Q8&l7B{IsbiVo c?t_JTXbDc zVQyr3R8em|NM&qo0PI`)ZyUMs&(HiT<}2Ufl6)iTtrP+ZG-n^u^zt~^*BlN-ap3Nb zB*t8FLvrOPPWrbOB=;#xmUD6K_8|Tst@xN34*40*3|ATxZo09^jYZu%UNUXznx*2w zEtY=2-=B`h{=47rZ+`cugVBS*csv@6hNHn`@Sr~!J)MjmK>s#{*i~|4nSRj!YF?X@ z`yVMR<8M(LuH+0>L&CDG`uUXh>69dBV$Cz_tDZ?XbsGjwF%vvyt`?5N=7GPWNFjDR z2ZmG+o1Xid*p7N46i zqQ}!kc^j34u&idh zDmC1v6WZ@aXc--j$-S(98~-aNax}MX0QTU2IP6b1@IM|71_%7#MLB{KoU>e52y5@5 z+u9VA9Kp{^Zh#wL@awZzFS~Q4Q)VqDFy{hYhZBqi(+Df3Ig11uunHoC%;7izF0F#K z$~9O_Gr=qxN<3QLtWay7;0!t)GS7v$Qb`KT9;;m184L+AC@q&LPFoElLM-Z(OAoT2 zl;H7t1}7-cA|b_O3ggyhkSfWo(lY{p(n+7pV9ta=LNd)$rq?fIf*)qkC&XNE8!rp$ z1R$XMwNjZA)Tms?Xk5b){DP31oC&c8jj>8olnF>>ooY;mF`us?DU@Fg1tt^MK_FWp zSf_X9^d2i|Vy<%INykgYWAtlfmyQi;8WxEOqkzj;BQt26IhJww*4BW|B`66fS6nNZ zqO=AsxLtza)(Z5rD^Z?+$>XJC|B6fQj2(p+bGM9;D7avn*M%kTa^u@m$3S9?#xT8p;^(h1RetseajwN0BF(&R=I)&NuGaH+lmG~xa_e;t z5NG)eCjEZuf2Eizy`I77IVS)b-{)wq?>TtR3CY|hF&1Tvtvz9N?o{NCl|aK-rP%_H z1vBQgGyS2Kf5d{4RhbfMvHlU)o^2PL8nr|R0IqsVW!#a6gd82gUoxgy3RVf!eh(X0 zt8reRLhaFsicK$8GRDmMhdE#5VQ9}CY_`i}(yi(B$k7pLke|Un;GcvP^Lwclg^W6v z(yf;VKEmmhiE?&s(`lq=_dWu1Gk-~9Z;8MPnmE%4aq|M#a)H_rdbc-lXl|94Ra z6rN)+)D0IE{(kz>LAWKi?qd--e-dEEQ_nOt6WowLd%LZ=-bxDg%b)_uG~ zOG7Hjc|}0qhOn~f+QTR3PRSUiBmcDEBC+xrg~Hzsb=<-Q7s5qD-gENqyc52Vc8_|? zn5b4UpH~{)446vI?QFk^)`h9)eB);@`}570wU)8cMIb^?gQCLNQqET87YdhYF6|sT z-N-3Z;VXRd zi7XZ)bpF2so!>fN4kwkSN(MR2J~>0LAm|cBO3kw8WuL+2W##N|pBxSBcl9zTFt$p+ zELhOS4k{-7tjbBcHkM_y9R^kDw9CsaLq5U#Tv^;u!c{6MlWc)WH(Ccmq{&?QRt$1Vy_w7K{PiJ_|05brFAi4o%3_6R5>3NH@*QF7CBCqBX%yq--7} zMdSx8gnrJj<_7nxs~Io8-g?YDEHQH)dCWYl#BR-rK6+G{t2SA$&JEq%_nCZ`Wtaa~ zF2Zf3g7^4;lm4K8#s8a54*uU=l;-^p%Q8djOAT*Z2TWv3HX!F*CbROYhoop(nFhOR zuiDUah=RgGUT4=tbpm_>$z_7l!q^RVzLsNDbUbm!g1ad76z9cdOUPT!uXge9uE!Qr ztp|25FE@Rg#@SJtjP1!|`24xewrDSTZplVFqg>Fu(i-%{=3ex5NF_R>Cpd4c@Qwv;bvLEzeM+~;>Nk~M)X^g437_g{f)$=+7~IT>HVHK9 zy&C>zyo=G+f-A+PMSFDfCZoZ7SP#2;=3NJwZ6_qHiM1~;8geun8gld-a%^04gc2@0 z8ibnNsB5hVR2!=?`SOaRp`^xzNhr-^ez702Jn#%nG|uv-Q&)E^bNG$Ep=w+A_GQ2Z zr?T;8ysF;2xV2DGRy3lx#(hJzGpoKUCGr%H1>@<@<>_1tUE;|Z<2FQ+Ef6uZ2+B|! z8VJ8~JEw&kzx;*CtqY4|H0G5`(6k$z45!GFUo^M)M#k7`-Lk^F^n&H-ZCh4!F5j+}yVxdMNi(cEx{M3B0ZM zfcD-0jmKB+|E81aA^y9I(v1If#A3>FyHuKg^d3Jw|5FhL76IU?61XM$JCC~i0zmh} zWNce%%2&6-#3E4n;!siN0*jXzSjL}gm1mWu>_VqQw%XQicu&FjpDWZ+qosIv;l7vY zKUS{8f5c_N<>IT&z&`v>N1OM5Plx@d2mIef`5pNGK5UEMX9m6&;7Xu#(+N}nzea8X z&@fv!meS~g!~bwH+{FLW z;rM|6yC|*q|4Wo_Z1(H3y*J+UUkUyD*3=)fwDC`uC{n(VO5-i^*oXhgXtMeK|6qFf z|IeM2Blw9~cl`M`_QTS{#S$e%IT!Anhm6JNY=NfwBY?@XOlfPtEKvwp2o*ufY`o-h z@dPvqX88&sV|Ll7XEGs2AaN1iFnpM4obwNugml;cJ)-bNiZv+dM>>%pL+xJ}r-YuI z{dQ)RMsftjs|#YkP63-b71@ b9Zc4t9Lk~mH Date: Tue, 23 May 2023 11:55:36 +0800 Subject: [PATCH 332/439] feat: optimize cue template of fault network (#3372) --- .../cli/kbcli_fault_network_bandwidth.md | 2 +- .../cli/kbcli_fault_network_corrupt.md | 4 +- .../cli/kbcli_fault_network_delay.md | 6 +- .../cli/kbcli_fault_network_duplicate.md | 4 +- .../user_docs/cli/kbcli_fault_network_loss.md | 4 +- .../cli/kbcli_fault_network_partition.md | 2 +- internal/cli/cmd/fault/fault_dns.go | 9 +- internal/cli/cmd/fault/fault_http.go | 55 ++++--- internal/cli/cmd/fault/fault_network.go | 145 ++++++++++-------- internal/cli/cmd/fault/fault_network_test.go | 8 +- .../create/template/http_chaos_template.cue | 103 +++++-------- .../template/network_chaos_template.cue | 133 ++++++---------- 12 files changed, 220 insertions(+), 255 deletions(-) diff --git a/docs/user_docs/cli/kbcli_fault_network_bandwidth.md b/docs/user_docs/cli/kbcli_fault_network_bandwidth.md index d4e73f28f..c1e8cee75 100644 --- a/docs/user_docs/cli/kbcli_fault_network_bandwidth.md +++ b/docs/user_docs/cli/kbcli_fault_network_bandwidth.md @@ -70,7 +70,7 @@ kbcli fault network bandwidth [flags] --rate string the rate at which the bandwidth is limited. For example : 10 bps/kbps/mbps/gbps. --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. - --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` diff --git a/docs/user_docs/cli/kbcli_fault_network_corrupt.md b/docs/user_docs/cli/kbcli_fault_network_corrupt.md index 968e0cd97..a48db6922 100644 --- a/docs/user_docs/cli/kbcli_fault_network_corrupt.md +++ b/docs/user_docs/cli/kbcli_fault_network_corrupt.md @@ -50,7 +50,7 @@ kbcli fault network corrupt [flags] ``` --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) - -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. --corrupt string Indicates the probability of a packet error occurring. Value range: [0, 100]. --direction string You can select "to"" or "from"" or "both"". (default "to") --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") @@ -67,7 +67,7 @@ kbcli fault network corrupt [flags] --phase stringArray Specify the pod that injects the fault by the state of the pod. --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. - --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` diff --git a/docs/user_docs/cli/kbcli_fault_network_delay.md b/docs/user_docs/cli/kbcli_fault_network_delay.md index 21ec9fd4a..05d73f5fe 100644 --- a/docs/user_docs/cli/kbcli_fault_network_delay.md +++ b/docs/user_docs/cli/kbcli_fault_network_delay.md @@ -50,14 +50,14 @@ kbcli fault network delay [flags] ``` --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) - -c, --correlation string Indicates the probability of a packet error occurring. Value range: [0, 100]. (default "0") + -c, --correlation string Indicates the probability of a packet error occurring. Value range: [0, 100]. --direction string You can select "to"" or "from"" or "both"". (default "to") --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") -e, --external-target stringArray a network target outside of Kubernetes, which can be an IPv4 address or a domain name, such as "www.baidu.com". Only works with direction: to. -h, --help help for delay - --jitter string the variation range of the delay time. (default "0ms") + --jitter string the variation range of the delay time. --label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0. (default []) --latency string the length of time to delay. --mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. (default "all") @@ -68,7 +68,7 @@ kbcli fault network delay [flags] --phase stringArray Specify the pod that injects the fault by the state of the pod. --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. - --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` diff --git a/docs/user_docs/cli/kbcli_fault_network_duplicate.md b/docs/user_docs/cli/kbcli_fault_network_duplicate.md index 41e96e07e..ef980077b 100644 --- a/docs/user_docs/cli/kbcli_fault_network_duplicate.md +++ b/docs/user_docs/cli/kbcli_fault_network_duplicate.md @@ -50,7 +50,7 @@ kbcli fault network duplicate [flags] ``` --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) - -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. --direction string You can select "to"" or "from"" or "both"". (default "to") --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") --duplicate string the probability of a packet being repeated. Value range: [0, 100]. @@ -67,7 +67,7 @@ kbcli fault network duplicate [flags] --phase stringArray Specify the pod that injects the fault by the state of the pod. --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. - --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` diff --git a/docs/user_docs/cli/kbcli_fault_network_loss.md b/docs/user_docs/cli/kbcli_fault_network_loss.md index 2bcd55085..5ce423542 100644 --- a/docs/user_docs/cli/kbcli_fault_network_loss.md +++ b/docs/user_docs/cli/kbcli_fault_network_loss.md @@ -50,7 +50,7 @@ kbcli fault network loss [flags] ``` --annotation stringToString Select the pod to inject the fault according to Annotation. (default []) - -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. (default "0") + -c, --correlation string Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100]. --direction string You can select "to"" or "from"" or "both"". (default "to") --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") --duration string Supported formats of the duration are: ms / s / m / h. (default "10s") @@ -67,7 +67,7 @@ kbcli fault network loss [flags] --phase stringArray Specify the pod that injects the fault by the state of the pod. --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. - --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` diff --git a/docs/user_docs/cli/kbcli_fault_network_partition.md b/docs/user_docs/cli/kbcli_fault_network_partition.md index b655dda15..c9e1d595e 100644 --- a/docs/user_docs/cli/kbcli_fault_network_partition.md +++ b/docs/user_docs/cli/kbcli_fault_network_partition.md @@ -65,7 +65,7 @@ kbcli fault network partition [flags] --phase stringArray Specify the pod that injects the fault by the state of the pod. --target-label stringToString label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"' (default []) --target-mode string You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with. - --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. (default [default]) + --target-ns-fault stringArray Specifies the namespace into which you want to inject faults. --target-value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. --value string If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject. ``` diff --git a/internal/cli/cmd/fault/fault_dns.go b/internal/cli/cmd/fault/fault_dns.go index 4fa267b79..ed00fff13 100644 --- a/internal/cli/cmd/fault/fault_dns.go +++ b/internal/cli/cmd/fault/fault_dns.go @@ -82,9 +82,6 @@ func NewRandomCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra o.AddCommonFlag(cmd) util.CheckErr(cmd.MarkFlagRequired("patterns")) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } @@ -95,9 +92,6 @@ func NewErrorCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra. o.AddCommonFlag(cmd) util.CheckErr(cmd.MarkFlagRequired("patterns")) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } @@ -120,6 +114,9 @@ func (o *DNSChaosOptions) AddCommonFlag(cmd *cobra.Command) { o.FaultBaseOptions.AddCommonFlag(cmd) cmd.Flags().StringArrayVar(&o.Patterns, "patterns", nil, `Select the domain name template that matches the failure behavior, and support placeholders ? and wildcards *.`) + + // register flag completion func + registerFlagCompletionFunc(cmd, o.Factory) } func (o *DNSChaosOptions) Validate() error { diff --git a/internal/cli/cmd/fault/fault_http.go b/internal/cli/cmd/fault/fault_http.go index bdd6f78b6..e58b85219 100644 --- a/internal/cli/cmd/fault/fault_http.go +++ b/internal/cli/cmd/fault/fault_http.go @@ -60,6 +60,22 @@ var faultHTTPExample = templates.Examples(` kbcli fault network http patch --method=POST --port=4399 --body="you are good luck" --type=JSON --duration=30s `) +type HTTPReplace struct { + ReplaceBody []byte `json:"body,omitempty"` + InputReplaceBody string `json:"-"` + ReplacePath string `json:"path,omitempty"` + ReplaceMethod string `json:"method,omitempty"` +} + +type HTTPPatch struct { + HTTPPatchBody `json:"body,omitempty"` +} + +type HTTPPatchBody struct { + PatchBodyValue string `json:"value,omitempty"` + PatchBodyType string `json:"type,omitempty"` +} + type HTTPChaosOptions struct { Target string `json:"target"` Port int32 `json:"port"` @@ -71,16 +87,10 @@ type HTTPChaosOptions struct { Abort bool `json:"abort,omitempty"` // delay command Delay string `json:"delay,omitempty"` - // replace command - ReplaceBody []byte `json:"replaceBody,omitempty"` - InputReplaceBody string `json:"-"` - ReplacePath string `json:"replacePath,omitempty"` - ReplaceMethod string `json:"replaceMethod,omitempty"` - + HTTPReplace `json:"replace,omitempty"` // patch command - PatchBodyValue string `json:"patchBodyValue,omitempty"` - PatchBodyType string `json:"patchBodyType,omitempty"` + HTTPPatch `json:"patch,omitempty"` FaultBaseOptions } @@ -118,35 +128,26 @@ func NewHTTPChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *co func NewAbortCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewHTTPChaosOptions(f, streams, "") - cmd := o.NewCobraCommand(Abort, AbortShort) o.AddCommonFlag(cmd) cmd.Flags().BoolVar(&o.Abort, "abort", true, `Indicates whether to inject the fault that interrupts the connection.`) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } func NewHTTPDelayCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewHTTPChaosOptions(f, streams, "") - cmd := o.NewCobraCommand(HTTPDelay, HTTPDelayShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Delay, "delay", "10s", `The time for delay.`) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } func NewReplaceCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewHTTPChaosOptions(f, streams, "") - cmd := o.NewCobraCommand(Replace, ReplaceShort) o.AddCommonFlag(cmd) @@ -154,24 +155,17 @@ func NewReplaceCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr cmd.Flags().StringVar(&o.ReplacePath, "replace-path", "", `The URI path used to replace content.`) cmd.Flags().StringVar(&o.ReplaceMethod, "replace-method", "", `The replaced content of the HTTP request method.`) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } func NewPatchCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewHTTPChaosOptions(f, streams, "") - cmd := o.NewCobraCommand(Patch, PatchShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.PatchBodyValue, "body", "", `The fault of the request body or response body with patch faults.`) cmd.Flags().StringVar(&o.PatchBodyType, "type", "", `The type of patch faults of the request body or response body. Currently, it only supports JSON.`) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } @@ -198,26 +192,29 @@ func (o *HTTPChaosOptions) AddCommonFlag(cmd *cobra.Command) { cmd.Flags().StringVar(&o.Path, "path", "*", `The URI path of the target request. Supports Matching wildcards.`) cmd.Flags().StringVar(&o.Method, "method", "GET", `The HTTP method of the target request method.For example: GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH.`) cmd.Flags().Int32Var(&o.Code, "code", 0, `The status code responded by target.`) + + // register flag completion func + registerFlagCompletionFunc(cmd, o.Factory) } func (o *HTTPChaosOptions) Validate() error { if o.PatchBodyType != "" && o.PatchBodyType != "JSON" { - return fmt.Errorf("the --type only supports JSON") + return fmt.Errorf("--type only supports JSON") } if o.PatchBodyValue != "" && o.PatchBodyType == "" { - return fmt.Errorf("the --type is required when --body is specified") + return fmt.Errorf("--type is required when --body is specified") } if o.PatchBodyType != "" && o.PatchBodyValue == "" { - return fmt.Errorf("the --body is required when --type is specified") + return fmt.Errorf("--body is required when --type is specified") } var msg interface{} if o.PatchBodyValue != "" && json.Unmarshal([]byte(o.PatchBodyValue), &msg) != nil { - return fmt.Errorf("the --body is not a valid JSON") + return fmt.Errorf("--body is not a valid JSON") } if o.Target == "Request" && o.Code != 0 { - return fmt.Errorf("the --code is only supported when --target is Response") + return fmt.Errorf("--code is only supported when --target=Response") } if ok, err := IsRegularMatch(o.Delay); !ok { diff --git a/internal/cli/cmd/fault/fault_network.go b/internal/cli/cmd/fault/fault_network.go index b554d3c50..194902bd2 100644 --- a/internal/cli/cmd/fault/fault_network.go +++ b/internal/cli/cmd/fault/fault_network.go @@ -70,40 +70,87 @@ var faultNetWorkExample = templates.Examples(` kbcli fault network bandwidth mysql-cluster-mysql-2 --rate=1kbps --duration=1m `) -type NetworkChaosOptions struct { - // Specify the network direction - Direction string `json:"direction"` - // Indicates a network target outside of Kubernetes, which can be an IPv4 address or a domain name, - // such as "www.baidu.com". Only works with direction: to. - ExternalTargets []string `json:"externalTargets,omitempty"` +type Target struct { + TargetMode string `json:"mode,omitempty"` + TargetValue string `json:"value,omitempty"` + TargetSelector `json:"selector,omitempty"` +} - TargetMode string `json:"targetMode,omitempty"` - TargetValue string `json:"targetValue"` +type TargetSelector struct { // Specifies the labels that target Pods come with. - TargetLabelSelectors map[string]string `json:"targetLabelSelectors,omitempty"` + TargetLabelSelectors map[string]string `json:"labelSelectors,omitempty"` // Specifies the namespaces to which target Pods belong. - TargetNamespaceSelectors []string `json:"targetNamespaceSelectors"` + TargetNamespaceSelectors []string `json:"namespaces,omitempty"` +} +// NetworkLoss Loss command +type NetworkLoss struct { // The percentage of packet loss Loss string `json:"loss,omitempty"` - // The percentage of packet corruption - Corrupt string `json:"corrupt,omitempty"` - // The percentage of packet duplication - Duplicate string `json:"duplicate,omitempty"` + // The correlation of loss or corruption or duplication or delay + Correlation string `json:"correlation,omitempty"` +} + +// NetworkDelay Delay command +type NetworkDelay struct { // The latency of delay Latency string `json:"latency,omitempty"` // The jitter of delay - Jitter string `json:"jitter"` + Jitter string `json:"jitter,omitempty"` + // The correlation of loss or corruption or duplication or delay + Correlation string `json:"correlation,omitempty"` +} +// NetworkDuplicate Duplicate command +type NetworkDuplicate struct { + // The percentage of packet duplication + Duplicate string `json:"duplicate,omitempty"` + // The correlation of loss or corruption or duplication or delay + Correlation string `json:"correlation,omitempty"` +} + +// NetworkCorrupt Corrupt command +type NetworkCorrupt struct { + // The percentage of packet corruption + Corrupt string `json:"corrupt,omitempty"` // The correlation of loss or corruption or duplication or delay - Correlation string `json:"correlation"` + Correlation string `json:"correlation,omitempty"` +} + +// NetworkBandwidth Bandwidth command +type NetworkBandwidth struct { + // the rate at which the bandwidth is limited. + Rate string `json:"rate,omitempty"` + // the number of bytes waiting in the queue. + Limit uint32 `json:"limit,omitempty"` + // the maximum number of bytes that can be sent instantaneously. + Buffer uint32 `json:"buffer,omitempty"` + // the bucket's maximum consumption rate. Reference: https://man7.org/linux/man-pages/man8/tc-tbf.8.html. + Peakrate uint64 `json:"peakrate,omitempty"` + // the size of the peakrate bucket. Reference: https://man7.org/linux/man-pages/man8/tc-tbf.8.html. + Minburst uint32 `json:"minburst,omitempty"` +} + +type NetworkChaosOptions struct { + // Specify the network direction + Direction string `json:"direction"` + + // A network target outside of Kubernetes, which can be an IPv4 address or a domain name, + // such as "kubeblocks.io". Only works with direction: to. + ExternalTargets []string `json:"externalTargets,omitempty"` - // Bandwidth command - Rate string `json:"rate,omitempty"` - Limit uint32 `json:"limit"` - Buffer uint32 `json:"buffer"` - Peakrate uint64 `json:"peakrate"` - Minburst uint32 `json:"minburst"` + // A collection of target pods. Pods can be selected by namespace and label. + Target `json:"target,omitempty"` + + NetworkLoss `json:"loss,omitempty"` + + NetworkDelay `json:"delay,omitempty"` + + NetworkDuplicate `json:"duplicate,omitempty"` + + NetworkCorrupt `json:"corrupt,omitempty"` + + NetworkBandwidth `json:"bandwidth,omitempty"` FaultBaseOptions } @@ -144,93 +191,71 @@ func NewNetworkChaosCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) func NewPartitionCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewNetworkChaosOptions(f, streams, string(v1alpha1.PartitionAction)) - cmd := o.NewCobraCommand(Partition, PartitionShort) o.AddCommonFlag(cmd) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } func NewLossCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewNetworkChaosOptions(f, streams, string(v1alpha1.LossAction)) - cmd := o.NewCobraCommand(Loss, LossShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Loss, "loss", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) - cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.NetworkLoss.Correlation, "correlation", "c", "", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) util.CheckErr(cmd.MarkFlagRequired("loss")) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } func NewDelayCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewNetworkChaosOptions(f, streams, string(v1alpha1.DelayAction)) - cmd := o.NewCobraCommand(Delay, DelayShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Latency, "latency", "", `the length of time to delay.`) - cmd.Flags().StringVar(&o.Jitter, "jitter", "0ms", `the variation range of the delay time.`) - cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) + cmd.Flags().StringVar(&o.Jitter, "jitter", "", `the variation range of the delay time.`) + cmd.Flags().StringVarP(&o.NetworkDelay.Correlation, "correlation", "c", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) util.CheckErr(cmd.MarkFlagRequired("latency")) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } func NewDuplicateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewNetworkChaosOptions(f, streams, string(v1alpha1.DuplicateAction)) - cmd := o.NewCobraCommand(Duplicate, DuplicateShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Duplicate, "duplicate", "", `the probability of a packet being repeated. Value range: [0, 100].`) - cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.NetworkDuplicate.Correlation, "correlation", "c", "", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) util.CheckErr(cmd.MarkFlagRequired("duplicate")) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } func NewCorruptCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewNetworkChaosOptions(f, streams, string(v1alpha1.CorruptAction)) - cmd := o.NewCobraCommand(Corrupt, CorruptShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Corrupt, "corrupt", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) - cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.NetworkCorrupt.Correlation, "correlation", "c", "", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) util.CheckErr(cmd.MarkFlagRequired("corrupt")) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } func NewBandwidthCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { o := NewNetworkChaosOptions(f, streams, string(v1alpha1.BandwidthAction)) - cmd := o.NewCobraCommand(Bandwidth, BandwidthShort) o.AddCommonFlag(cmd) - cmd.Flags().StringVar(&o.Rate, "rate", "", `the rate at which the bandwidth is limited. For example : 10 bps/kbps/mbps/gbps.`) cmd.Flags().Uint32Var(&o.Limit, "limit", 1, `the number of bytes waiting in the queue.`) cmd.Flags().Uint32Var(&o.Buffer, "buffer", 1, `the maximum number of bytes that can be sent instantaneously.`) @@ -239,9 +264,6 @@ func NewBandwidthCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *co util.CheckErr(cmd.MarkFlagRequired("rate")) - // register flag completion func - registerFlagCompletionFunc(cmd, f) - return cmd } @@ -268,16 +290,23 @@ func (o *NetworkChaosOptions) AddCommonFlag(cmd *cobra.Command) { cmd.Flags().StringVar(&o.TargetMode, "target-mode", "", `You can select "one", "all", "fixed", "fixed-percent", "random-max-percent", Specify the experimental mode, that is, which Pods to experiment with.`) cmd.Flags().StringVar(&o.TargetValue, "target-value", "", `If you choose mode=fixed or fixed-percent or random-max-percent, you can enter a value to specify the number or percentage of pods you want to inject.`) cmd.Flags().StringToStringVar(&o.TargetLabelSelectors, "target-label", nil, `label for pod, such as '"app.kubernetes.io/component=mysql, statefulset.kubernetes.io/pod-name=mycluster-mysql-0"'`) - cmd.Flags().StringArrayVar(&o.TargetNamespaceSelectors, "target-ns-fault", []string{"default"}, `Specifies the namespace into which you want to inject faults.`) + cmd.Flags().StringArrayVar(&o.TargetNamespaceSelectors, "target-ns-fault", nil, `Specifies the namespace into which you want to inject faults.`) + + // register flag completion func + registerFlagCompletionFunc(cmd, o.Factory) } func (o *NetworkChaosOptions) Validate() error { if o.TargetValue == "" && (o.TargetMode == "fixed" || o.TargetMode == "fixed-percent" || o.TargetMode == "random-max-percent") { - return fmt.Errorf("you must use --value to specify an integer") + return fmt.Errorf("--value is required to specify pod nums or percentage") + } + + if (o.TargetNamespaceSelectors != nil || o.TargetLabelSelectors != nil) && o.TargetMode == "" { + return fmt.Errorf("--target-mode is required to specify a target mode") } - if (o.TargetLabelSelectors != nil || o.TargetValue != "") && o.TargetMode == "" { - return fmt.Errorf("you must use --mode to specify an experiment mode") + if o.ExternalTargets != nil && o.Direction != "to" { + return fmt.Errorf("--direction=to is required when specifying external targets") } if ok, err := IsInteger(o.TargetValue); !ok { @@ -296,10 +325,6 @@ func (o *NetworkChaosOptions) Validate() error { return err } - if ok, err := IsInteger(o.Correlation); !ok { - return err - } - if ok, err := IsRegularMatch(o.Latency); !ok { return err } diff --git a/internal/cli/cmd/fault/fault_network_test.go b/internal/cli/cmd/fault/fault_network_test.go index 4827ae206..9b20c708d 100644 --- a/internal/cli/cmd/fault/fault_network_test.go +++ b/internal/cli/cmd/fault/fault_network_test.go @@ -84,7 +84,7 @@ var _ = Describe("Fault Network", func() { cmd := o.NewCobraCommand(Loss, LossShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Loss, "loss", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) - cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.NetworkLoss.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) for _, input := range inputs { Expect(cmd.Flags().Parse(input)).Should(Succeed()) @@ -107,7 +107,7 @@ var _ = Describe("Fault Network", func() { o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Latency, "latency", "", `the length of time to delay.`) cmd.Flags().StringVar(&o.Jitter, "jitter", "0ms", `the variation range of the delay time.`) - cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.NetworkDelay.Correlation, "correlation", "c", "0", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) for _, input := range inputs { Expect(cmd.Flags().Parse(input)).Should(Succeed()) @@ -127,7 +127,7 @@ var _ = Describe("Fault Network", func() { cmd := o.NewCobraCommand(Duplicate, DuplicateShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Duplicate, "duplicate", "", `the probability of a packet being repeated. Value range: [0, 100].`) - cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.NetworkDuplicate.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) for _, input := range inputs { Expect(cmd.Flags().Parse(input)).Should(Succeed()) @@ -147,7 +147,7 @@ var _ = Describe("Fault Network", func() { cmd := o.NewCobraCommand(Corrupt, CorruptShort) o.AddCommonFlag(cmd) cmd.Flags().StringVar(&o.Corrupt, "corrupt", "", `Indicates the probability of a packet error occurring. Value range: [0, 100].`) - cmd.Flags().StringVarP(&o.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) + cmd.Flags().StringVarP(&o.NetworkCorrupt.Correlation, "correlation", "c", "0", `Indicates the correlation between the probability of a packet error occurring and whether it occurred the previous time. Value range: [0, 100].`) for _, input := range inputs { Expect(cmd.Flags().Parse(input)).Should(Succeed()) diff --git a/internal/cli/create/template/http_chaos_template.cue b/internal/cli/create/template/http_chaos_template.cue index 0163d6362..62f82181d 100644 --- a/internal/cli/create/template/http_chaos_template.cue +++ b/internal/cli/create/template/http_chaos_template.cue @@ -17,79 +17,60 @@ // required, command line input options for parameters and flags options: { - namespace: string - selector: {} - mode: string - value: string - duration: string + namespace: string + selector: {} + mode: string + value: string + duration: string - target: string - port: int32 - path: string - method: string - code?: int32 + target: string + port: int32 + path: string + method: string + code?: int32 - abort?: bool - delay?: string - - repalceBody?: bytes - replacePath?: string - replaceMethod?: string - - patchBodyValue?: string - patchBodyType?: string + abort?: bool + delay?: string + replace?: {} + patch?: {} } // required, k8s api resource content content: { - kind: "HTTPChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata: { - generateName: "http-chaos-" - namespace: options.namespace - } - spec: { - selector: options.selector - mode: options.mode - value: options.value - duration: options.duration + kind: "HTTPChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "http-chaos-" + namespace: options.namespace + } + spec:{ + selector: options.selector + mode: options.mode + value: options.value + duration: options.duration target: options.target - port: options.port - path: options.path - method: options.method - if options.code != _|_ { - code: options.code - } + port: options.port + path: options.path + method: options.method + if options.code != _|_ { + code: options.code + } - if options.abort != _|_ { - abort: options.abort - } - if options.delay != _|_ { + if options.abort != _|_ { + abort: options.abort + } + + if options.delay != _|_ { delay: options.delay } - if options.replaceBody != _|_ || options.replacePath != _|_ || options.replaceMethod != _|_ { - replace: { - if options.replaceBody != _|_ { - body: options.replaceBody - } - if options.replacePath != _|_ { - path: options.replacePath - } - if options.replaceMethod != _|_ { - method: options.replaceMethod - } - } + + if len(options.replace) != 0 { + replace: options.replace } - if options.patchBodyValue != _|_ && options.patchBodyType != _|_ { - patch: { - if options.patchBodyValue != _|_ && options.patchBodyType != _|_ { - body: { - value: options.patchBodyValue - type: options.patchBodyType - } - } - } + + if len(options.patch["body"]) != 0 { + patch: options.patch } } } diff --git a/internal/cli/create/template/network_chaos_template.cue b/internal/cli/create/template/network_chaos_template.cue index e28029198..b2c0ea99e 100644 --- a/internal/cli/create/template/network_chaos_template.cue +++ b/internal/cli/create/template/network_chaos_template.cue @@ -17,102 +17,67 @@ // required, command line input options for parameters and flags options: { - namespace: string - action: string - selector: {} - mode: string - value: string - duration: string + namespace: string + action: string + selector: {} + mode: string + value: string + duration: string - direction: string - externalTargets?: [...] + direction: string + externalTargets?: [...] + target?: {} - targetMode?: string - targetValue: string - targetNamespaceSelectors: [...string] - targetLabelSelectors: {} - - loss?: string - corrupt?: string - duplicate?: string - - latency?: string - jitter: string - - correlation: string - - rate?: string - limit: uint32 - buffer: uint32 - peakrate: uint32 - minburst: uint32 + loss?: {} + delay?: {} + duplicate?: {} + corrupt?: {} + bandwidth?: {} } // required, k8s api resource content content: { - kind: "NetworkChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata: { - generateName: "network-chaos-" - namespace: options.namespace - } - spec: { - selector: options.selector - mode: options.mode - value: options.value - action: options.action - duration: options.duration + kind: "NetworkChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata:{ + generateName: "network-chaos-" + namespace: options.namespace + } + spec:{ + selector: options.selector + mode: options.mode + value: options.value + action: options.action + duration: options.duration - direction: options.direction - if options.externalTargets != _|_ { - externalTargets: options.externalTargets - } - if options.targetMode != _|_ { - target: { - mode: options.targetMode - value: options.targetValue - selector: { - namespaces: options.targetNamespaceSelectors - labelSelectors: { - options.targetLabelSelectors - } - } - } + direction: options.direction + + if options.externalTargets != _|_ { + externalTargets: options.externalTargets + } + + if options.target["mode"] != _|_ || len(options.target["selector"]) !=0 { + target: options.target } - if options.loss != _|_ { - loss: { - loss: options.loss - correlation: options.correlation - } + + if options.loss["loss"] != _|_ { + loss: options.loss } - if options.corrupt != _|_ { - corrupt: { - corrupt: options.corrupt - correlation: options.correlation - } + + if options.delay["latency"] != _|_ { + delay: options.delay } - if options.duplicate != _|_ { - duplicate: { - duplicate: options.duplicate - correlation: options.correlation - } + + if options.corrupt["corrupt"] != _|_ { + corrupt: options.corrupt } - if options.latency != _|_ { - delay: { - latency: options.latency - jitter: options.jitter - correlation: options.correlation - } + + if options.duplicate["duplicate"] != _|_ { + duplicate: options.duplicate } - if options.rate != _|_ { - bandwidth: { - rate: options.rate - limit: options.limit - buffer: options.buffer - peakrate: options.peakrate - minburst: options.minburst - correlation: options.correlation - } + + if options.bandwidth["rate"] != _|_ { + bandwidth: options.bandwidth } } } From 47fdd684bd02c590f99919a742d606801bf1b2c8 Mon Sep 17 00:00:00 2001 From: huyongqii <129354195+huyongqii@users.noreply.github.com> Date: Tue, 23 May 2023 13:18:56 +0800 Subject: [PATCH 333/439] feat: auto build secret for fault node (#3369) --- .../cli/kbcli_fault_node_detach-volume.md | 17 +- .../user_docs/cli/kbcli_fault_node_restart.md | 17 +- docs/user_docs/cli/kbcli_fault_node_stop.md | 17 +- internal/cli/cmd/fault/fault_constant.go | 3 + internal/cli/cmd/fault/fault_node.go | 210 +++++++++++++++++- internal/cli/cmd/fault/fault_node_test.go | 21 +- 6 files changed, 241 insertions(+), 44 deletions(-) diff --git a/docs/user_docs/cli/kbcli_fault_node_detach-volume.md b/docs/user_docs/cli/kbcli_fault_node_detach-volume.md index 72271d001..54c3603a4 100644 --- a/docs/user_docs/cli/kbcli_fault_node_detach-volume.md +++ b/docs/user_docs/cli/kbcli_fault_node_detach-volume.md @@ -12,30 +12,31 @@ kbcli fault node detach-volume [flags] ``` # Stop a specified EC2 instance. - kbcli fault node stop node1 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node stop node1 -c=aws --region=cn-northwest-1 --duration=3m # Stop two specified EC2 instances. - kbcli fault node stop node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node stop node1 node2 -c=aws --region=cn-northwest-1 --duration=3m # Restart two specified EC2 instances. - kbcli fault node restart node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node restart node1 node2 -c=aws --region=cn-northwest-1 --duration=3m # Detach two specified volume from two specified EC2 instances. - kbcli fault node detach-volume node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 + kbcli fault node detach-volume node1 node2 -c=aws --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 # Stop two specified GCK instances. - kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering # Restart two specified GCK instances. - kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering # Detach two specified volume from two specified GCK instances. - kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret --device-name=/d1,/d2 + kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --device-name=/d1,/d2 ``` ### Options ``` + --auto-approve Skip interactive approval before create secret. -c, --cloud-provider string Cloud provider type, one of [aws gcp] --device-name strings The device name of the volume. --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") @@ -44,7 +45,7 @@ kbcli fault node detach-volume [flags] -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) --project string The name of the GCP project. Only available when cloud-provider=gcp. --region string The region of the node. - --secret-name string The name of the Kubernetes Secret that stores the kubernetes cluster authentication information. + --secret string The name of the secret containing cloud provider specific credentials. --volume-id strings The volume ids of the ec2. Only available when cloud-provider=aws. ``` diff --git a/docs/user_docs/cli/kbcli_fault_node_restart.md b/docs/user_docs/cli/kbcli_fault_node_restart.md index 625abf1bf..9f8c27990 100644 --- a/docs/user_docs/cli/kbcli_fault_node_restart.md +++ b/docs/user_docs/cli/kbcli_fault_node_restart.md @@ -12,30 +12,31 @@ kbcli fault node restart [flags] ``` # Stop a specified EC2 instance. - kbcli fault node stop node1 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node stop node1 -c=aws --region=cn-northwest-1 --duration=3m # Stop two specified EC2 instances. - kbcli fault node stop node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node stop node1 node2 -c=aws --region=cn-northwest-1 --duration=3m # Restart two specified EC2 instances. - kbcli fault node restart node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node restart node1 node2 -c=aws --region=cn-northwest-1 --duration=3m # Detach two specified volume from two specified EC2 instances. - kbcli fault node detach-volume node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 + kbcli fault node detach-volume node1 node2 -c=aws --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 # Stop two specified GCK instances. - kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering # Restart two specified GCK instances. - kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering # Detach two specified volume from two specified GCK instances. - kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret --device-name=/d1,/d2 + kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --device-name=/d1,/d2 ``` ### Options ``` + --auto-approve Skip interactive approval before create secret. -c, --cloud-provider string Cloud provider type, one of [aws gcp] --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") --duration string Supported formats of the duration are: ms / s / m / h. (default "30s") @@ -43,7 +44,7 @@ kbcli fault node restart [flags] -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) --project string The name of the GCP project. Only available when cloud-provider=gcp. --region string The region of the node. - --secret-name string The name of the Kubernetes Secret that stores the kubernetes cluster authentication information. + --secret string The name of the secret containing cloud provider specific credentials. ``` ### Options inherited from parent commands diff --git a/docs/user_docs/cli/kbcli_fault_node_stop.md b/docs/user_docs/cli/kbcli_fault_node_stop.md index f9e67e475..20cd46202 100644 --- a/docs/user_docs/cli/kbcli_fault_node_stop.md +++ b/docs/user_docs/cli/kbcli_fault_node_stop.md @@ -12,30 +12,31 @@ kbcli fault node stop [flags] ``` # Stop a specified EC2 instance. - kbcli fault node stop node1 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node stop node1 -c=aws --region=cn-northwest-1 --duration=3m # Stop two specified EC2 instances. - kbcli fault node stop node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node stop node1 node2 -c=aws --region=cn-northwest-1 --duration=3m # Restart two specified EC2 instances. - kbcli fault node restart node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node restart node1 node2 -c=aws --region=cn-northwest-1 --duration=3m # Detach two specified volume from two specified EC2 instances. - kbcli fault node detach-volume node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 + kbcli fault node detach-volume node1 node2 -c=aws --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 # Stop two specified GCK instances. - kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering # Restart two specified GCK instances. - kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering # Detach two specified volume from two specified GCK instances. - kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret --device-name=/d1,/d2 + kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --device-name=/d1,/d2 ``` ### Options ``` + --auto-approve Skip interactive approval before create secret. -c, --cloud-provider string Cloud provider type, one of [aws gcp] --dry-run string[="unchanged"] Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") --duration string Supported formats of the duration are: ms / s / m / h. (default "30s") @@ -43,7 +44,7 @@ kbcli fault node stop [flags] -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) --project string The name of the GCP project. Only available when cloud-provider=gcp. --region string The region of the node. - --secret-name string The name of the Kubernetes Secret that stores the kubernetes cluster authentication information. + --secret string The name of the secret containing cloud provider specific credentials. ``` ### Options inherited from parent commands diff --git a/internal/cli/cmd/fault/fault_constant.go b/internal/cli/cmd/fault/fault_constant.go index 59acab34b..77fb04158 100644 --- a/internal/cli/cmd/fault/fault_constant.go +++ b/internal/cli/cmd/fault/fault_constant.go @@ -135,6 +135,9 @@ const ( RestartShort = "Restart instance" DetachVolume = "detach-volume" DetachVolumeShort = "Detach volume" + + AWSSecretName = "cloud-key-secret-aws" + GCPSecretName = "cloud-key-secret-gcp" ) var supportedCloudProviders = []string{cp.AWS, cp.GCP} diff --git a/internal/cli/cmd/fault/fault_node.go b/internal/cli/cmd/fault/fault_node.go index 91af3f160..916ea3d74 100644 --- a/internal/cli/cmd/fault/fault_node.go +++ b/internal/cli/cmd/fault/fault_node.go @@ -20,13 +20,24 @@ along with this program. If not, see . package fault import ( + "bufio" + "context" + "encoding/base64" "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" "github.com/chaos-mesh/chaos-mesh/api/v1alpha1" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/kubernetes" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" @@ -34,29 +45,30 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/create" "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/util" + "github.com/apecloud/kubeblocks/internal/cli/util/prompt" ) var faultNodeExample = templates.Examples(` # Stop a specified EC2 instance. - kbcli fault node stop node1 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node stop node1 -c=aws --region=cn-northwest-1 --duration=3m # Stop two specified EC2 instances. - kbcli fault node stop node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node stop node1 node2 -c=aws --region=cn-northwest-1 --duration=3m # Restart two specified EC2 instances. - kbcli fault node restart node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=3m + kbcli fault node restart node1 node2 -c=aws --region=cn-northwest-1 --duration=3m # Detach two specified volume from two specified EC2 instances. - kbcli fault node detach-volume node1 node2 -c=aws --secret-name=cloud-key-secret --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 + kbcli fault node detach-volume node1 node2 -c=aws --region=cn-northwest-1 --duration=1m --volume-id=v1,v2 --device-name=/d1,/d2 # Stop two specified GCK instances. - kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + kbcli fault node stop node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering # Restart two specified GCK instances. - kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret + kbcli fault node restart node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering # Detach two specified volume from two specified GCK instances. - kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --secret-name=cloud-key-secret --device-name=/d1,/d2 + kbcli fault node detach-volume node1 node2 -c=gcp --region=us-central1-c --project=apecloud-platform-engineering --device-name=/d1,/d2 `) type NodeChaoOptions struct { @@ -82,6 +94,8 @@ type NodeChaoOptions struct { Duration string `json:"duration"` + AutoApprove bool `json:"-"` + create.CreateOptions `json:"-"` } @@ -146,12 +160,12 @@ func (o *NodeChaoOptions) NewCobraCommand(use, short string) *cobra.Command { Short: short, Example: faultNodeExample, Run: func(cmd *cobra.Command, args []string) { - cmdutil.CheckErr(o.Execute(use, args)) + cmdutil.CheckErr(o.Execute(use, args, false)) }, } } -func (o *NodeChaoOptions) Execute(action string, args []string) error { +func (o *NodeChaoOptions) Execute(action string, args []string, testEnv bool) error { o.Args = args if err := o.CreateOptions.Complete(); err != nil { return err @@ -171,6 +185,9 @@ func (o *NodeChaoOptions) Execute(action string, args []string) error { if o.VolumeIDs != nil { o.VolumeID = o.VolumeIDs[idx] } + if err := o.CreateSecret(testEnv); err != nil { + return err + } if err := o.Run(); err != nil { return err } @@ -180,17 +197,17 @@ func (o *NodeChaoOptions) Execute(action string, args []string) error { func (o *NodeChaoOptions) AddCommonFlag(cmd *cobra.Command) { cmd.Flags().StringVarP(&o.CloudProvider, "cloud-provider", "c", "", fmt.Sprintf("Cloud provider type, one of %v", supportedCloudProviders)) - cmd.Flags().StringVar(&o.SecretName, "secret-name", "", "The name of the Kubernetes Secret that stores the kubernetes cluster authentication information.") cmd.Flags().StringVar(&o.Region, "region", "", "The region of the node.") cmd.Flags().StringVar(&o.Project, "project", "", "The name of the GCP project. Only available when cloud-provider=gcp.") + cmd.Flags().StringVar(&o.SecretName, "secret", "", "The name of the secret containing cloud provider specific credentials.") cmd.Flags().StringVar(&o.Duration, "duration", "30s", "Supported formats of the duration are: ms / s / m / h.") + cmd.Flags().BoolVar(&o.AutoApprove, "auto-approve", false, "Skip interactive approval before create secret.") cmd.Flags().StringVar(&o.DryRun, "dry-run", "none", `Must be "client", or "server". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource.`) cmd.Flags().Lookup("dry-run").NoOptDefVal = Unchanged printer.AddOutputFlagForCreate(cmd, &o.Format) util.CheckErr(cmd.MarkFlagRequired("cloud-provider")) - util.CheckErr(cmd.MarkFlagRequired("secret-name")) util.CheckErr(cmd.MarkFlagRequired("region")) // register flag completion func @@ -238,6 +255,9 @@ func (o *NodeChaoOptions) Complete(action string) error { if o.CloudProvider == cp.AWS { o.GVR = GetGVR(Group, Version, ResourceAWSChaos) o.Kind = KindAWSChaos + if o.SecretName == "" { + o.SecretName = AWSSecretName + } switch action { case Stop: o.Action = string(v1alpha1.Ec2Stop) @@ -249,6 +269,9 @@ func (o *NodeChaoOptions) Complete(action string) error { } else if o.CloudProvider == cp.GCP { o.GVR = GetGVR(Group, Version, ResourceGCPChaos) o.Kind = KindGCPChaos + if o.SecretName == "" { + o.SecretName = GCPSecretName + } switch action { case Stop: o.Action = string(v1alpha1.NodeStop) @@ -281,3 +304,168 @@ func (o *NodeChaoOptions) PreCreate(obj *unstructured.Unstructured) error { obj.SetUnstructuredContent(data) return nil } + +func (o *NodeChaoOptions) CreateSecret(testEnv bool) error { + if testEnv { + return nil + } + + if o.DryRun != "none" { + return nil + } + + config, err := o.Factory.ToRESTConfig() + if err != nil { + return err + } + + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + return err + } + + // Check if Secret already exists + secretClient := clientSet.CoreV1().Secrets(o.Namespace) + _, err = secretClient.Get(context.TODO(), o.SecretName, metav1.GetOptions{}) + if err == nil { + fmt.Printf("Secret %s exists under %s namespace.\n", o.SecretName, o.Namespace) + return nil + } else if !k8serrors.IsNotFound(err) { + return err + } + + if err := o.confirmToContinue(); err != nil { + return err + } + + switch o.CloudProvider { + case "aws": + if err := handleAWS(clientSet, o.Namespace, o.SecretName); err != nil { + return err + } + case "gcp": + if err := handleGCP(clientSet, o.Namespace, o.SecretName); err != nil { + return err + } + default: + return fmt.Errorf("unknown cloud provider:%s", o.CloudProvider) + } + return nil +} + +func (o *NodeChaoOptions) confirmToContinue() error { + if !o.AutoApprove { + printer.Warning(o.Out, "A secret will be created for the cloud account to access %s, do you want to continue to create this secret: %s ?\n Only 'yes' will be accepted to confirm.\n\n", o.CloudProvider, o.SecretName) + entered, _ := prompt.NewPrompt("Enter a value:", nil, o.In).Run() + if entered != "yes" { + fmt.Fprintf(o.Out, "\nCancel automatic secert creation. You will not be able to access the nodes on the cluster.\n") + return cmdutil.ErrExit + } + } + fmt.Fprintf(o.Out, "Continue to create secret: %s\n", o.SecretName) + return nil +} + +func handleAWS(clientSet *kubernetes.Clientset, namespace, secretName string) error { + accessKeyID, secretAccessKey, err := readAWSCredentials() + if err != nil { + return err + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{ + "aws_access_key_id": accessKeyID, + "aws_secret_access_key": secretAccessKey, + }, + } + + createdSecret, err := clientSet.CoreV1().Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + return err + } + + fmt.Printf("Secret %s created successfully\n", createdSecret.Name) + return nil +} + +func handleGCP(clientSet *kubernetes.Clientset, namespace, secretName string) error { + home, err := os.UserHomeDir() + if err != nil { + return err + } + + filePath := filepath.Join(home, ".config", "gcloud", "application_default_credentials.json") + data, err := ioutil.ReadFile(filePath) + jsonData := string(data) + fmt.Println(jsonData) + if err != nil { + return err + } + encodedData := base64.StdEncoding.EncodeToString([]byte(jsonData)) + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + StringData: map[string]string{ + "service_account": encodedData, + }, + } + + createdSecret, err := clientSet.CoreV1().Secrets(namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + return err + } + + fmt.Printf("Secret %s created successfully\n", createdSecret.Name) + return nil +} + +func readAWSCredentials() (string, string, error) { + home, err := os.UserHomeDir() + if err != nil { + return "", "", err + } + filePath := filepath.Join(home, ".aws", "credentials") + file, err := os.Open(filePath) + if err != nil { + return "", "", err + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + fmt.Printf("unable to close file: %s", err) + } + }(file) + + // Read file content line by line using bufio.Scanner + scanner := bufio.NewScanner(file) + accessKeyID := "" + secretAccessKey := "" + + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "aws_access_key_id") { + accessKeyID = strings.TrimSpace(strings.SplitN(line, "=", 2)[1]) + } else if strings.HasPrefix(line, "aws_secret_access_key") { + secretAccessKey = strings.TrimSpace(strings.SplitN(line, "=", 2)[1]) + } + } + + if scanner.Err() != nil { + return "", "", scanner.Err() + } + + if accessKeyID == "" || secretAccessKey == "" { + return "", "", fmt.Errorf("unable to find valid AWS access key information") + } + + return accessKeyID, secretAccessKey, nil +} diff --git a/internal/cli/cmd/fault/fault_node_test.go b/internal/cli/cmd/fault/fault_node_test.go index 2b28f854d..c9b6b978f 100644 --- a/internal/cli/cmd/fault/fault_node_test.go +++ b/internal/cli/cmd/fault/fault_node_test.go @@ -48,8 +48,9 @@ var _ = Describe("Fault Node", func() { Context("test fault node", func() { It("fault node stop", func() { inputs := [][]string{ - {"-c=aws", "--region=cn-northwest-1", "--secret-name=cloud-key-secret", "--dry-run=client"}, - {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--secret-name=cloud-key-secret", "--dry-run=client"}, + {"-c=aws", "--region=cn-northwest-1", "--dry-run=client"}, + {"-c=aws", "--region=cn-northwest-1", "--secret=test-secret", "--dry-run=client"}, + {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--dry-run=client"}, } o := NewNodeOptions(tf, streams) cmd := o.NewCobraCommand(Stop, StopShort) @@ -58,14 +59,15 @@ var _ = Describe("Fault Node", func() { o.Args = []string{"node1", "node2"} for _, input := range inputs { Expect(cmd.Flags().Parse(input)).Should(Succeed()) - Expect(o.Execute(Stop, o.Args)).Should(Succeed()) + Expect(o.Execute(Stop, o.Args, true)).Should(Succeed()) } }) It("fault node restart", func() { inputs := [][]string{ - {"-c=aws", "--region=cn-northwest-1", "--secret-name=cloud-key-secret", "--dry-run=client"}, - {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--secret-name=cloud-key-secret", "--dry-run=client"}, + {"-c=aws", "--region=cn-northwest-1", "--dry-run=client"}, + {"-c=aws", "--region=cn-northwest-1", "--secret=test-secret", "--dry-run=client"}, + {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--dry-run=client"}, } o := NewNodeOptions(tf, streams) cmd := o.NewCobraCommand(Restart, RestartShort) @@ -74,14 +76,15 @@ var _ = Describe("Fault Node", func() { o.Args = []string{"node1", "node2"} for _, input := range inputs { Expect(cmd.Flags().Parse(input)).Should(Succeed()) - Expect(o.Execute(Restart, o.Args)).Should(Succeed()) + Expect(o.Execute(Restart, o.Args, true)).Should(Succeed()) } }) It("fault node detach-volume", func() { inputs := [][]string{ - {"-c=aws", "--region=cn-northwest-1", "--secret-name=cloud-key-secret", "--volume-id=v1,v2", "--device-name=/d1,/d2", "--dry-run=client"}, - {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--secret-name=cloud-key-secret", "--device-name=/d1,/d2", "--dry-run=client"}, + {"-c=aws", "--region=cn-northwest-1", "--volume-id=v1,v2", "--device-name=/d1,/d2", "--dry-run=client"}, + {"-c=aws", "--region=cn-northwest-1", "--volume-id=v1,v2", "--device-name=/d1,/d2", "--secret=test-secret", "--dry-run=client"}, + {"-c=gcp", "--region=us-central1-c", "--project=apecloud-platform-engineering", "--device-name=/d1,/d2", "--dry-run=client"}, } o := NewNodeOptions(tf, streams) cmd := o.NewCobraCommand(DetachVolume, DetachVolumeShort) @@ -92,7 +95,7 @@ var _ = Describe("Fault Node", func() { o.Args = []string{"node1", "node2"} for _, input := range inputs { Expect(cmd.Flags().Parse(input)).Should(Succeed()) - Expect(o.Execute(DetachVolume, o.Args)).Should(Succeed()) + Expect(o.Execute(DetachVolume, o.Args, true)).Should(Succeed()) o.VolumeIDs = nil o.DeviceNames = nil } From c62392b54f3a876ea0d63689a74888aa3d3c7fc9 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 23 May 2023 16:01:27 +0800 Subject: [PATCH 334/439] chore: change nodeName to nodeSelector (#3389) --- controllers/apps/operations/volume_expansion_test.go | 4 ++-- controllers/dataprotection/backup_controller.go | 12 +++++++++--- controllers/dataprotection/backup_controller_test.go | 2 +- controllers/dataprotection/type.go | 2 ++ internal/cli/cmd/cluster/dataprotection.go | 7 +++++-- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/controllers/apps/operations/volume_expansion_test.go b/controllers/apps/operations/volume_expansion_test.go index ee8913d33..2004e2a04 100644 --- a/controllers/apps/operations/volume_expansion_test.go +++ b/controllers/apps/operations/volume_expansion_test.go @@ -142,11 +142,11 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { } mockVolumeExpansionActionAndReconcile := func(reqCtx intctrlutil.RequestCtx, opsRes *OpsResource, newOps *appsv1alpha1.OpsRequest) { - Expect(testapps.ChangeObjStatus(&testCtx, newOps, func() { + Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(newOps), func(newOps *appsv1alpha1.OpsRequest) { _, _ = GetOpsManager().Do(reqCtx, k8sClient, opsRes) newOps.Status.Phase = appsv1alpha1.OpsRunningPhase newOps.Status.StartTimestamp = metav1.Time{Time: time.Now()} - })).ShouldNot(HaveOccurred()) + })()).ShouldNot(HaveOccurred()) // do volume-expand action _, _ = GetOpsManager().Do(reqCtx, k8sClient, opsRes) diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 1041edc40..76cd5fcac 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -1273,9 +1273,15 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, r.appendBackupVolumeMount(pvcName, &podSpec, &podSpec.Containers[0]) // the pod of job needs to be scheduled on the same node as the workload pod, because it needs to share one pvc - // see: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename - podSpec.NodeName = clusterPod.Spec.NodeName - + podSpec.NodeSelector = map[string]string{ + hostNameLabelKey: clusterPod.Spec.NodeName, + } + // ignore taints + podSpec.Tolerations = []corev1.Toleration{ + { + Operator: corev1.TolerationOpExists, + }, + } return podSpec, nil } diff --git a/controllers/dataprotection/backup_controller_test.go b/controllers/dataprotection/backup_controller_test.go index 561eaf257..95e6fac4f 100644 --- a/controllers/dataprotection/backup_controller_test.go +++ b/controllers/dataprotection/backup_controller_test.go @@ -166,7 +166,7 @@ var _ = Describe("Backup Controller test", func() { It("should succeed after job completes", func() { By("Check backup job's nodeName equals pod's nodeName") Eventually(testapps.CheckObj(&testCtx, backupKey, func(g Gomega, fetched *batchv1.Job) { - g.Expect(fetched.Spec.Template.Spec.NodeName).To(Equal(nodeName)) + g.Expect(fetched.Spec.Template.Spec.NodeSelector[hostNameLabelKey]).To(Equal(nodeName)) })).Should(Succeed()) patchK8sJobStatus(backupKey, batchv1.JobComplete) diff --git a/controllers/dataprotection/type.go b/controllers/dataprotection/type.go index 07c0b14f8..26d4ed014 100644 --- a/controllers/dataprotection/type.go +++ b/controllers/dataprotection/type.go @@ -46,6 +46,8 @@ const ( // the key of persistentVolumeTemplate in the configmap. persistentVolumeTemplateKey = "persistentVolume" + + hostNameLabelKey = "kubernetes.io/hostname" ) var reconcileInterval = time.Second diff --git a/internal/cli/cmd/cluster/dataprotection.go b/internal/cli/cmd/cluster/dataprotection.go index 671a30347..5653b5544 100644 --- a/internal/cli/cmd/cluster/dataprotection.go +++ b/internal/cli/cmd/cluster/dataprotection.go @@ -113,7 +113,10 @@ type ListBackupOptions struct { BackupName string } -func (o *CreateBackupOptions) Complete() error { +func (o *CreateBackupOptions) CompleteBackup() error { + if err := o.Complete(); err != nil { + return err + } // generate backupName if len(o.BackupName) == 0 { o.BackupName = strings.Join([]string{"backup", o.Namespace, o.Name, time.Now().Format("20060102150405")}, "-") @@ -207,7 +210,7 @@ func NewCreateBackupCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args - cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.CompleteBackup()) cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.Run()) }, From 0611bc483a14819a0c64d6a9d1bad986d7a5b9c2 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Tue, 23 May 2023 16:12:34 +0800 Subject: [PATCH 335/439] fix: remove backup job requested resource (#3388) --- deploy/apecloud-mysql-scale/templates/backuptool.yaml | 7 ------- deploy/apecloud-mysql/templates/backuptool.yaml | 7 ------- deploy/mongodb/templates/backuptool.yaml | 7 ------- deploy/postgresql/templates/backuptool.yaml | 7 ------- test/testdata/backup/backuptool.yaml | 4 ---- 5 files changed, 32 deletions(-) diff --git a/deploy/apecloud-mysql-scale/templates/backuptool.yaml b/deploy/apecloud-mysql-scale/templates/backuptool.yaml index 341844543..0260a3a93 100644 --- a/deploy/apecloud-mysql-scale/templates/backuptool.yaml +++ b/deploy/apecloud-mysql-scale/templates/backuptool.yaml @@ -8,13 +8,6 @@ metadata: spec: image: registry.cn-hangzhou.aliyuncs.com/apecloud/percona-xtrabackup deployKind: job - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "1" - memory: 128Mi env: - name: DATA_DIR value: /data/mysql/data diff --git a/deploy/apecloud-mysql/templates/backuptool.yaml b/deploy/apecloud-mysql/templates/backuptool.yaml index 0a7d5396c..8729c836a 100644 --- a/deploy/apecloud-mysql/templates/backuptool.yaml +++ b/deploy/apecloud-mysql/templates/backuptool.yaml @@ -8,13 +8,6 @@ metadata: spec: image: registry.cn-hangzhou.aliyuncs.com/apecloud/percona-xtrabackup deployKind: job - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "500m" - memory: 128Mi env: - name: DATA_DIR value: /data/mysql/data diff --git a/deploy/mongodb/templates/backuptool.yaml b/deploy/mongodb/templates/backuptool.yaml index 7e26d25f5..3c8b267ca 100644 --- a/deploy/mongodb/templates/backuptool.yaml +++ b/deploy/mongodb/templates/backuptool.yaml @@ -8,13 +8,6 @@ metadata: spec: image: mongo:5.0.14 deployKind: job - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "500m" - memory: 128Mi env: - name: DATA_DIR value: /data/mongodb/db diff --git a/deploy/postgresql/templates/backuptool.yaml b/deploy/postgresql/templates/backuptool.yaml index 1028bf0bf..97eb1f540 100644 --- a/deploy/postgresql/templates/backuptool.yaml +++ b/deploy/postgresql/templates/backuptool.yaml @@ -8,13 +8,6 @@ metadata: spec: image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} deployKind: job - resources: - limits: - cpu: "1" - memory: 2Gi - requests: - cpu: "500m" - memory: 128Mi env: - name: RESTORE_DATA_DIR value: /home/postgres/pgdata/kb_restore diff --git a/test/testdata/backup/backuptool.yaml b/test/testdata/backup/backuptool.yaml index 354c72d41..e08dd26d9 100644 --- a/test/testdata/backup/backuptool.yaml +++ b/test/testdata/backup/backuptool.yaml @@ -5,10 +5,6 @@ metadata: spec: image: registry.cn-hangzhou.aliyuncs.com/apecloud/percona-xtrabackup deployKind: job - resources: - limits: - cpu: "1" - memory: 2Gi env: - name: DATA_DIR value: /var/lib/mysql From 377b42336a21ec9b4553725b67bee4a39d3a8e78 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Tue, 23 May 2023 17:46:13 +0800 Subject: [PATCH 336/439] chore: optimize PITR for more accurate recovery time (#3378) --- controllers/dataprotection/backup_controller.go | 4 ++++ .../dataprotection/backuppolicy_controller.go | 5 ++++- .../templates/backuppolicytemplate.yaml | 2 +- deploy/postgresql/templates/backuptool-pitr.yaml | 16 ++++++++++++++-- deploy/postgresql/templates/scripts.yaml | 9 ++++----- 5 files changed, 27 insertions(+), 9 deletions(-) diff --git a/controllers/dataprotection/backup_controller.go b/controllers/dataprotection/backup_controller.go index 76cd5fcac..4449a03aa 100644 --- a/controllers/dataprotection/backup_controller.go +++ b/controllers/dataprotection/backup_controller.go @@ -1212,6 +1212,10 @@ func (r *BackupReconciler) buildBackupToolPodSpec(reqCtx intctrlutil.RequestCtx, container.Command = []string{"sh", "-c"} container.Args = backupTool.Spec.BackupCommands container.Image = backupTool.Spec.Image + if container.Image == "" { + // TODO(dsj): need determine container name to get, temporary use first container + container.Image = clusterPod.Spec.Containers[0].Image + } if backupTool.Spec.Resources != nil { container.Resources = *backupTool.Spec.Resources } diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index 630003666..a3ab55445 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -554,7 +554,10 @@ func (r *BackupPolicyReconciler) reconfigure(reqCtx intctrlutil.RequestCtx, backupPolicy.Annotations = map[string]string{} } backupPolicy.Annotations[constant.LastAppliedConfigAnnotation] = updateParameterPairs - return r.Client.Patch(reqCtx.Ctx, backupPolicy, patch) + if err := r.Client.Patch(reqCtx.Ctx, backupPolicy, patch); err != nil { + return err + } + return intctrlutil.NewErrorf(intctrlutil.ErrorTypeRequeue, "requeue to waiting ops %s finished.", ops.Name) } func (r *BackupPolicyReconciler) reconcileReconfigure(reqCtx intctrlutil.RequestCtx, diff --git a/deploy/postgresql/templates/backuppolicytemplate.yaml b/deploy/postgresql/templates/backuppolicytemplate.yaml index da7b9054a..5692e79a3 100644 --- a/deploy/postgresql/templates/backuppolicytemplate.yaml +++ b/deploy/postgresql/templates/backuppolicytemplate.yaml @@ -57,4 +57,4 @@ spec: - path: manifests.backupLog containerName: postgresql script: /kb-scripts/backup-log-collector.sh false - updateStage: pre + updateStage: post diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml index b9d6abd05..776787ca8 100644 --- a/deploy/postgresql/templates/backuptool-pitr.yaml +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -25,7 +25,7 @@ spec: value: 2006-01-02 15:04:05 MST - name: LOG_DIR value: $(VOLUME_DATA_DIR)/pgroot/data/pg_wal - image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} + image: "" logical: restoreCommands: - | @@ -61,10 +61,22 @@ spec: set -e; EXPIRED_INCR_LOG=${BACKUP_DIR}/$(date -d"7 day ago" +%Y%m%d); if [ -d ${EXPIRED_INCR_LOG} ]; then rm -rf ${EXPIRED_INCR_LOG}; fi + export PGPASSWORD=${DB_PASSWORD} + PSQL="psql -h ${DB_HOST} -U ${DB_USER}" + LAST_TRANS=$(pg_waldump $(${PSQL} -Atc "select pg_walfile_name(pg_current_wal_lsn())") --rmgr=Transaction |tail -n 1) + if [ "${LAST_TRANS}" != "" ] && [ "$(find ${LOG_DIR}/archive_status/ -name '*.ready')" = "" ]; then + echo "switch wal file" + ${PSQL} -c "select pg_switch_wal()" + for i in $(seq 1 60); do + echo "waiting wal ready ..." + if [ "$(find ${LOG_DIR}/archive_status/ -name '*.ready')" != "" ]; then break; fi + sleep 1 + done + fi TODAY_INCR_LOG=${BACKUP_DIR}/$(date +%Y%m%d); mkdir -p ${TODAY_INCR_LOG}; cd ${LOG_DIR} - for i in $(find ./archive_status/ -name "*.ready"); do + for i in $(ls -tr ./archive_status/*.ready); do wal_ready_name="${i##*/}" wal_name=${wal_ready_name%.*} echo "uploading ${wal_name}"; diff --git a/deploy/postgresql/templates/scripts.yaml b/deploy/postgresql/templates/scripts.yaml index 23b962ece..5188a3aae 100644 --- a/deploy/postgresql/templates/scripts.yaml +++ b/deploy/postgresql/templates/scripts.yaml @@ -99,13 +99,12 @@ data: set -o nounset SHOW_START_TIME=$1 LOG_START_TIME="" + LOG_STOP_TIME="" if [ "$SHOW_START_TIME" == "false" ]; then - latest_transaction=$(pg_waldump $(psql -Atc "select pg_walfile_name(pg_current_wal_lsn())") --rmgr=Transaction 2>/dev/null |tail -n 1) - if [ "${latest_transaction}" != "" ]; then - psql -c "select pg_switch_wal()" >/dev/null 2>&1 - sleep 1 + latest_done_wal=$(ls -t ${PGDATA}/pg_wal/archive_status/|grep ".done"|head -n 1) + if [ "${latest_done_wal}" != "" ]; then + LOG_STOP_TIME=$(pg_waldump ${latest_done_wal%.*} --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') fi - LOG_STOP_TIME=$(echo ${latest_transaction}|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') [[ "${LOG_STOP_TIME}" != "" ]] && printf "{\"stopTime\": \"$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ')\"}" || printf "{}" else LOG_START_TIME=$(pg_waldump $(ls -Ftr $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') From 39ba5f472ca346fd41a8bcc9079a8dc7929ebbd2 Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Tue, 23 May 2023 18:56:58 +0800 Subject: [PATCH 337/439] feat: support `kbcli cv set-default cv-name` and `kbcli cv unset-default cv-name` (#3317) Co-authored-by: 1aal <1aal@users.noreply.github.com> --- docs/user_docs/cli/cli.md | 2 + docs/user_docs/cli/kbcli_clusterversion.md | 2 + .../cli/kbcli_clusterversion_set-default.md | 53 +++++ .../cli/kbcli_clusterversion_unset-default.md | 53 +++++ .../cmd/clusterdefinition/list_compoents.go | 1 + .../cli/cmd/clusterversion/clusterversion.go | 49 ++++- .../cmd/clusterversion/clusterversion_test.go | 50 +---- .../cli/cmd/clusterversion/set_default.go | 196 ++++++++++++++++++ .../cmd/clusterversion/set_default_test.go | 194 +++++++++++++++++ internal/cli/printer/printer.go | 18 ++ internal/cli/printer/printer_test.go | 22 ++ 11 files changed, 595 insertions(+), 45 deletions(-) create mode 100644 docs/user_docs/cli/kbcli_clusterversion_set-default.md create mode 100644 docs/user_docs/cli/kbcli_clusterversion_unset-default.md create mode 100644 internal/cli/cmd/clusterversion/set_default.go create mode 100644 internal/cli/cmd/clusterversion/set_default_test.go diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index 19fbb204e..d4212b508 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -98,6 +98,8 @@ ClusterDefinition command. ClusterVersion command. * [kbcli clusterversion list](kbcli_clusterversion_list.md) - List ClusterVersions. +* [kbcli clusterversion set-default](kbcli_clusterversion_set-default.md) - Set the clusterversion to the default clusterversion for its clusterdefinition. +* [kbcli clusterversion unset-default](kbcli_clusterversion_unset-default.md) - Unset the clusterversion if it's default. ## [dashboard](kbcli_dashboard.md) diff --git a/docs/user_docs/cli/kbcli_clusterversion.md b/docs/user_docs/cli/kbcli_clusterversion.md index 134243281..3a11304dd 100644 --- a/docs/user_docs/cli/kbcli_clusterversion.md +++ b/docs/user_docs/cli/kbcli_clusterversion.md @@ -38,6 +38,8 @@ ClusterVersion command. * [kbcli clusterversion list](kbcli_clusterversion_list.md) - List ClusterVersions. +* [kbcli clusterversion set-default](kbcli_clusterversion_set-default.md) - Set the clusterversion to the default clusterversion for its clusterdefinition. +* [kbcli clusterversion unset-default](kbcli_clusterversion_unset-default.md) - Unset the clusterversion if it's default. #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/docs/user_docs/cli/kbcli_clusterversion_set-default.md b/docs/user_docs/cli/kbcli_clusterversion_set-default.md new file mode 100644 index 000000000..05b0702a2 --- /dev/null +++ b/docs/user_docs/cli/kbcli_clusterversion_set-default.md @@ -0,0 +1,53 @@ +--- +title: kbcli clusterversion set-default +--- + +Set the clusterversion to the default clusterversion for its clusterdefinition. + +``` +kbcli clusterversion set-default NAME [flags] +``` + +### Examples + +``` + # set ac-mysql-8.0.30 as the default clusterversion + kbcli clusterversion set-default ac-mysql-8.0.30 +``` + +### Options + +``` + -h, --help help for set-default +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli clusterversion](kbcli_clusterversion.md) - ClusterVersion command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/docs/user_docs/cli/kbcli_clusterversion_unset-default.md b/docs/user_docs/cli/kbcli_clusterversion_unset-default.md new file mode 100644 index 000000000..5543dc71a --- /dev/null +++ b/docs/user_docs/cli/kbcli_clusterversion_unset-default.md @@ -0,0 +1,53 @@ +--- +title: kbcli clusterversion unset-default +--- + +Unset the clusterversion if it's default. + +``` +kbcli clusterversion unset-default NAME [flags] +``` + +### Examples + +``` + # unset ac-mysql-8.0.30 to default clusterversion if it's default + kbcli clusterversion unset-default ac-mysql-8.0.30 +``` + +### Options + +``` + -h, --help help for unset-default +``` + +### Options inherited from parent commands + +``` + --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. + --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. + --as-uid string UID to impersonate for the operation. + --cache-dir string Default cache directory (default "$HOME/.kube/cache") + --certificate-authority string Path to a cert file for the certificate authority + --client-certificate string Path to a client certificate file for TLS + --client-key string Path to a client key file for TLS + --cluster string The name of the kubeconfig cluster to use + --context string The name of the kubeconfig context to use + --disable-compression If true, opt-out of response compression for all requests to the server + --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure + --kubeconfig string Path to the kubeconfig file to use for CLI requests. + --match-server-version Require server version to match client version + -n, --namespace string If present, the namespace scope for this CLI request + --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") + -s, --server string The address and port of the Kubernetes API server + --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used + --token string Bearer token for authentication to the API server + --user string The name of the kubeconfig user to use +``` + +### SEE ALSO + +* [kbcli clusterversion](kbcli_clusterversion.md) - ClusterVersion command. + +#### Go Back to [CLI Overview](cli.md) Homepage. + diff --git a/internal/cli/cmd/clusterdefinition/list_compoents.go b/internal/cli/cmd/clusterdefinition/list_compoents.go index e16a5d23e..402c3ca25 100644 --- a/internal/cli/cmd/clusterdefinition/list_compoents.go +++ b/internal/cli/cmd/clusterdefinition/list_compoents.go @@ -80,6 +80,7 @@ func run(o *list.ListOptions) error { } p := printer.NewTablePrinter(o.Out) p.SetHeader("NAME", "WORKLOAD-TYPE", "CHARACTER-TYPE", "CLUSTER-DEFINITION") + p.SortBy(1) for _, info := range infos { var cd v1alpha1.ClusterDefinition if err = runtime.DefaultUnstructuredConverter.FromUnstructured(info.Object.(*unstructured.Unstructured).Object, &cd); err != nil { diff --git a/internal/cli/cmd/clusterversion/clusterversion.go b/internal/cli/cmd/clusterversion/clusterversion.go index eb80f0602..778bd6ddd 100644 --- a/internal/cli/cmd/clusterversion/clusterversion.go +++ b/internal/cli/cmd/clusterversion/clusterversion.go @@ -21,14 +21,19 @@ package clusterversion import ( "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" cmdutil "k8s.io/kubectl/pkg/cmd/util" "k8s.io/kubectl/pkg/util/templates" + "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/list" + "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" "github.com/apecloud/kubeblocks/internal/cli/util/flags" + "github.com/apecloud/kubeblocks/internal/constant" ) var listExample = templates.Examples(` @@ -48,6 +53,8 @@ func NewClusterVersionCmd(f cmdutil.Factory, streams genericclioptions.IOStreams } cmd.AddCommand(NewListCmd(f, streams)) + cmd.AddCommand(newSetDefaultCMD(f, streams)) + cmd.AddCommand(newUnSetDefaultCMD(f, streams)) return cmd } @@ -66,11 +73,49 @@ func NewListCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C o.LabelSelector = util.BuildClusterDefinitionRefLable(o.LabelSelector, []string{o.clusterDefinitionRef}) } o.Names = args - _, err := o.Run() - util.CheckErr(err) + util.CheckErr(run(o)) }, } o.AddFlags(cmd, true) flags.AddClusterDefinitionFlag(f, cmd, &o.clusterDefinitionRef) return cmd } + +func run(o *ListClusterVersionOptions) error { + if !o.Format.IsHumanReadable() { + _, err := o.Run() + return err + } + o.Print = false + r, err := o.Run() + if err != nil { + return err + } + infos, err := r.Infos() + if err != nil { + return err + } + p := printer.NewTablePrinter(o.Out) + p.SetHeader("NAME", "CLUSTER-DEFINITION", "STATUS", "IS-DEFAULT", "CREATED-TIME") + p.SortBy(1) + for _, info := range infos { + var cv v1alpha1.ClusterVersion + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(info.Object.(*unstructured.Unstructured).Object, &cv); err != nil { + return err + } + isDefaultValue := isDefault(&cv) + p.AddRow(cv.Name, cv.Labels[constant.ClusterDefLabelKey], cv.Status.Phase, isDefaultValue, util.TimeFormat(&cv.CreationTimestamp)) + } + p.Print() + return nil +} + +func isDefault(cv *v1alpha1.ClusterVersion) string { + if cv.Annotations == nil { + return "false" + } + if _, ok := cv.Annotations[constant.DefaultClusterVersionAnnotationKey]; !ok { + return "false" + } + return cv.Annotations[constant.DefaultClusterVersionAnnotationKey] +} diff --git a/internal/cli/cmd/clusterversion/clusterversion_test.go b/internal/cli/cmd/clusterversion/clusterversion_test.go index 536813d0c..11880920b 100644 --- a/internal/cli/cmd/clusterversion/clusterversion_test.go +++ b/internal/cli/cmd/clusterversion/clusterversion_test.go @@ -21,6 +21,7 @@ package clusterversion import ( "bytes" + "fmt" "net/http" . "github.com/onsi/ginkgo/v2" @@ -38,50 +39,14 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/testing" "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" ) var _ = Describe("clusterversion", func() { var streams genericclioptions.IOStreams var tf *cmdtesting.TestFactory out := new(bytes.Buffer) - - mockRestTable := func() *metav1.Table { - var Type = "string" - table := &metav1.Table{ - TypeMeta: metav1.TypeMeta{ - Kind: "Table", - APIVersion: "meta.k8s.io/v1", - }, - ColumnDefinitions: []metav1.TableColumnDefinition{ - { - Name: "NAME", - Type: Type, - }, { - Name: "CLUSTER-DEFINITION", - Type: Type, - }, { - Name: "STATUS", - Type: Type, - }, - { - Name: "AGE", - Type: Type, - }, - }, - Rows: []metav1.TableRow{ - { - Cells: []interface{}{ - testing.ClusterVersionName, - testing.ClusterDefName, - "Available", - "0s", - }, - }, - }, - } - return table - } - + var CreateTime string mockClient := func(data runtime.Object) *cmdtesting.TestFactory { tf := testing.NewTestFactory(testing.Namespace) codec := scheme.Codecs.LegacyCodec(scheme.Scheme.PrioritizedVersionsAllGroups()...) @@ -99,8 +64,9 @@ var _ = Describe("clusterversion", func() { _ = appsv1alpha1.AddToScheme(scheme.Scheme) _ = metav1.AddMetaToScheme(scheme.Scheme) streams, _, out, _ = genericclioptions.NewTestIOStreams() - table := mockRestTable() - tf = mockClient(table) + fakeCV := testing.FakeClusterVersion() + CreateTime = util.TimeFormat(&fakeCV.CreationTimestamp) + tf = mockClient(fakeCV) }) AfterEach(func() { @@ -121,9 +87,7 @@ var _ = Describe("clusterversion", func() { It("list --cluster-definition", func() { cmd := NewListCmd(tf, streams) cmd.Run(cmd, []string{"--cluster-definition=" + testing.ClusterDefName}) - expected := `NAME CLUSTER-DEFINITION STATUS AGE -fake-cluster-version fake-cluster-definition Available 0s -` + expected := fmt.Sprintf("NAME CLUSTER-DEFINITION STATUS IS-DEFAULT CREATED-TIME \nfake-cluster-version fake-cluster-definition false %s \n", CreateTime) Expect(expected).Should(Equal(out.String())) }) }) diff --git a/internal/cli/cmd/clusterversion/set_default.go b/internal/cli/cmd/clusterversion/set_default.go new file mode 100644 index 000000000..b7439459d --- /dev/null +++ b/internal/cli/cmd/clusterversion/set_default.go @@ -0,0 +1,196 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package clusterversion + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apitypes "k8s.io/apimachinery/pkg/types" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/dynamic" + cmdutil "k8s.io/kubectl/pkg/cmd/util" + "k8s.io/kubectl/pkg/util/templates" + + "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/cli/util" + "github.com/apecloud/kubeblocks/internal/constant" +) + +var ( + setDefaultExample = templates.Examples(` + # set ac-mysql-8.0.30 as the default clusterversion + kbcli clusterversion set-default ac-mysql-8.0.30`, + ) + + unsetDefaultExample = templates.Examples(` + # unset ac-mysql-8.0.30 to default clusterversion if it's default + kbcli clusterversion unset-default ac-mysql-8.0.30`) + + clusterVersionGVR = types.ClusterVersionGVR() +) + +const ( + annotationTrueValue = "true" + annotationFalseValue = "false" +) + +type SetOrUnsetDefaultOption struct { + Factory cmdutil.Factory + IOStreams genericclioptions.IOStreams + // `set-default` cmd will set the setDefault to be true, while `unset-default` cmd set it false + setDefault bool +} + +func newSetOrUnsetDefaultOptions(f cmdutil.Factory, streams genericclioptions.IOStreams, toSet bool) *SetOrUnsetDefaultOption { + return &SetOrUnsetDefaultOption{ + Factory: f, + IOStreams: streams, + setDefault: toSet, + } +} + +func newSetDefaultCMD(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := newSetOrUnsetDefaultOptions(f, streams, true) + cmd := &cobra.Command{ + Use: "set-default NAME", + Short: "Set the clusterversion to the default clusterversion for its clusterdefinition.", + Example: setDefaultExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, clusterVersionGVR), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(o.validate(args)) + util.CheckErr(o.run(args)) + }, + } + return cmd +} + +func newUnSetDefaultCMD(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.Command { + o := newSetOrUnsetDefaultOptions(f, streams, false) + cmd := &cobra.Command{ + Use: "unset-default NAME", + Short: "Unset the clusterversion if it's default.", + Example: unsetDefaultExample, + ValidArgsFunction: util.ResourceNameCompletionFunc(f, clusterVersionGVR), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(o.validate(args)) + util.CheckErr(o.run(args)) + }, + } + return cmd +} + +func (o *SetOrUnsetDefaultOption) run(args []string) error { + client, err := o.Factory.DynamicClient() + if err != nil { + return err + } + var allErrs []error + // unset-default logic + if !o.setDefault { + for _, cv := range args { + if err := patchDefaultClusterVersionAnnotations(client, cv, annotationFalseValue); err != nil { + allErrs = append(allErrs, err) + } + } + return utilerrors.NewAggregate(allErrs) + } + // set-default logic + cv2Cd, cd2DefaultCv, err := getMapsBetweenCvAndCd(client) + if err != nil { + return err + } + // alreadySet is to marks if two input args have the same clusterdefintion + alreadySet := make(map[string]string) + for _, cv := range args { + cd, ok := cv2Cd[cv] + if !ok { + allErrs = append(allErrs, fmt.Errorf("cluterversion \"%s\" not found", cv)) + continue + } + if _, ok := cd2DefaultCv[cd]; ok && cv != cd2DefaultCv[cd] { + allErrs = append(allErrs, fmt.Errorf("clusterdefinition \"%s\" already has a default cluster version \"%s\"", cv2Cd[cv], cd2DefaultCv[cd])) + continue + } + if _, ok := alreadySet[cd]; ok { + allErrs = append(allErrs, fmt.Errorf("\"%s\" has the same clusterdefinition with \"%s\"", cv, alreadySet[cd])) + continue + } + if err := patchDefaultClusterVersionAnnotations(client, cv, annotationTrueValue); err != nil { + allErrs = append(allErrs, err) + continue + } + alreadySet[cd] = cv + } + return utilerrors.NewAggregate(allErrs) +} + +func (o *SetOrUnsetDefaultOption) validate(args []string) error { + if len(args) == 0 { + return fmt.Errorf("clusterversion name should be specified, run \"kbcli clusterversion list\" to list the clusterversions") + } + return nil +} + +// patchDefaultClusterVersionAnnotations will patch the Annotations for the clusterversion in K8S +func patchDefaultClusterVersionAnnotations(client dynamic.Interface, cvName string, value string) error { + patchData := map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": map[string]interface{}{ + constant.DefaultClusterVersionAnnotationKey: value, + }, + }, + } + patchBytes, _ := json.Marshal(patchData) + _, err := client.Resource(clusterVersionGVR).Patch(context.Background(), cvName, apitypes.MergePatchType, patchBytes, metav1.PatchOptions{}) + return err +} + +func getMapsBetweenCvAndCd(client dynamic.Interface) (map[string]string, map[string]string, error) { + lists, err := client.Resource(clusterVersionGVR).List(context.Background(), metav1.ListOptions{}) + if err != nil { + return nil, nil, err + } + cvToCd := make(map[string]string) + cdToDefaultCv := make(map[string]string) + for _, item := range lists.Items { + name := item.GetName() + annotations := item.GetAnnotations() + labels := item.GetLabels() + if labels == nil { + continue + } + if _, ok := labels[constant.ClusterDefLabelKey]; !ok { + continue + } + cvToCd[name] = labels[constant.ClusterDefLabelKey] + if annotations == nil { + continue + } + if annotations[constant.DefaultClusterVersionAnnotationKey] == annotationTrueValue { + cdToDefaultCv[labels[constant.ClusterDefLabelKey]] = name + } + } + return cvToCd, cdToDefaultCv, nil +} diff --git a/internal/cli/cmd/clusterversion/set_default_test.go b/internal/cli/cmd/clusterversion/set_default_test.go new file mode 100644 index 000000000..6b78dadb5 --- /dev/null +++ b/internal/cli/cmd/clusterversion/set_default_test.go @@ -0,0 +1,194 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package clusterversion + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/cli-runtime/pkg/genericclioptions" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes/scheme" + clientfake "k8s.io/client-go/rest/fake" + cmdtesting "k8s.io/kubectl/pkg/cmd/testing" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/cli/testing" + "github.com/apecloud/kubeblocks/internal/constant" +) + +var _ = Describe("set-default", func() { + var streams genericclioptions.IOStreams + var tf *cmdtesting.TestFactory + + const ( + cluterversion = testing.ClusterVersionName + clusterversionInSameCD = testing.ClusterVersionName + "-sameCD" + ClusterversionOtherCD = testing.ClusterVersionName + "-other" + errorClusterversion = "08jfa2" + ) + + beginWithMultipleClusterversion := func() { + tf.FakeDynamicClient = testing.FakeDynamicClient([]runtime.Object{ + &appsv1alpha1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: cluterversion, + Labels: map[string]string{ + constant.ClusterDefLabelKey: testing.ClusterDefName, + }, + }, + }, + &appsv1alpha1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterversionInSameCD, + Labels: map[string]string{ + constant.ClusterDefLabelKey: testing.ClusterDefName, + }, + }, + }, + &appsv1alpha1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: ClusterversionOtherCD, + Labels: map[string]string{ + constant.ClusterDefLabelKey: testing.ClusterDefName + "-other", + }, + }, + }, + }...) + } + + getFakeClusterVersion := func(dynamic dynamic.Interface, clusterVersionName string) (*appsv1alpha1.ClusterVersion, error) { + var cv appsv1alpha1.ClusterVersion + u, err := dynamic.Resource(clusterVersionGVR).Get(context.Background(), clusterVersionName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, &cv) + if err != nil { + return nil, err + } + return &cv, nil + } + + var validateSetOrUnsetResult func(needToChecks []string, value []string) + validateSetOrUnsetResult = func(needToChecks []string, value []string) { + if len(needToChecks) == 0 { + return + } + cv, err := getFakeClusterVersion(tf.FakeDynamicClient, needToChecks[0]) + Expect(err).Should(Succeed()) + Expect(isDefault(cv)).Should(Equal(value[0])) + validateSetOrUnsetResult(needToChecks[1:], value[1:]) + } + + BeforeEach(func() { + _ = appsv1alpha1.AddToScheme(scheme.Scheme) + _ = metav1.AddMetaToScheme(scheme.Scheme) + streams, _, _, _ = genericclioptions.NewTestIOStreams() + tf = testing.NewTestFactory(testing.Namespace) + tf.Client = &clientfake.RESTClient{} + beginWithMultipleClusterversion() + }) + + It("test isDefault Func", func() { + cv := testing.FakeClusterVersion() + Expect(isDefault(cv)).Should(Equal(annotationFalseValue)) + cv.SetAnnotations(map[string]string{ + constant.DefaultClusterVersionAnnotationKey: annotationFalseValue, + }) + Expect(isDefault(cv)).Should(Equal(annotationFalseValue)) + cv.Annotations[constant.DefaultClusterVersionAnnotationKey] = annotationTrueValue + Expect(isDefault(cv)).Should(Equal(annotationTrueValue)) + }) + + It("set-default cmd", func() { + cmd := newSetDefaultCMD(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + + It("unset-default cmd", func() { + cmd := newUnSetDefaultCMD(tf, streams) + Expect(cmd).ShouldNot(BeNil()) + }) + + It("set-default empty args", func() { + o := newSetOrUnsetDefaultOptions(tf, streams, true) + Expect(o.validate([]string{})).Should(HaveOccurred()) + }) + + It("set-default error args", func() { + o := newSetOrUnsetDefaultOptions(tf, streams, true) + Expect(o.run([]string{errorClusterversion})).Should(HaveOccurred()) + }) + + It("unset-default empty args", func() { + o := newSetOrUnsetDefaultOptions(tf, streams, false) + Expect(o.validate([]string{})).Should(HaveOccurred()) + }) + + It("unset-default error args", func() { + o := newSetOrUnsetDefaultOptions(tf, streams, false) + Expect(o.run([]string{errorClusterversion})).Should(HaveOccurred()) + }) + + It("set-default and unset-default", func() { + // before set-default + validateSetOrUnsetResult([]string{cluterversion}, []string{annotationFalseValue}) + // set-default + cmd := newSetDefaultCMD(tf, streams) + cmd.Run(cmd, []string{cluterversion}) + validateSetOrUnsetResult([]string{cluterversion}, []string{annotationTrueValue}) + // unset-default + cmd = newUnSetDefaultCMD(tf, streams) + cmd.Run(cmd, []string{cluterversion}) + validateSetOrUnsetResult([]string{cluterversion}, []string{annotationFalseValue}) + }) + + It("the clusterDef already have a default cv when set-default", func() { + cmd := newSetDefaultCMD(tf, streams) + cmd.Run(cmd, []string{cluterversion}) + validateSetOrUnsetResult([]string{cluterversion, clusterversionInSameCD}, []string{annotationTrueValue, annotationFalseValue}) + o := newSetOrUnsetDefaultOptions(tf, streams, true) + err := o.run([]string{clusterversionInSameCD}) + Expect(err).Should(HaveOccurred()) + }) + + It("set-default args belong the same cd", func() { + o := newSetOrUnsetDefaultOptions(tf, streams, true) + err := o.run([]string{cluterversion, cluterversion}) + Expect(err).Should(HaveOccurred()) + }) + + It("set-default and unset-default more than one args", func() { + cmd := newSetDefaultCMD(tf, streams) + validateSetOrUnsetResult([]string{cluterversion, ClusterversionOtherCD}, []string{annotationFalseValue, annotationFalseValue}) + // set-default + cmd.Run(cmd, []string{cluterversion, ClusterversionOtherCD}) + validateSetOrUnsetResult([]string{cluterversion, ClusterversionOtherCD}, []string{annotationTrueValue, annotationTrueValue}) + // unset-defautl + cmd = newUnSetDefaultCMD(tf, streams) + cmd.Run(cmd, []string{cluterversion, ClusterversionOtherCD}) + validateSetOrUnsetResult([]string{cluterversion, ClusterversionOtherCD}, []string{annotationFalseValue, annotationFalseValue}) + }) +}) diff --git a/internal/cli/printer/printer.go b/internal/cli/printer/printer.go index acbbe2881..d7a3c4104 100644 --- a/internal/cli/printer/printer.go +++ b/internal/cli/printer/printer.go @@ -117,6 +117,24 @@ func (t *TablePrinter) Print() { t.Tbl.Render() } +// SortBy will sort the table alphabetically by the column you specify, it will be sorted by the first table column in default. +// The columnNumber index start from 1 +func (t *TablePrinter) SortBy(columnNumber ...int) { + if len(columnNumber) == 0 { + t.Tbl.SortBy([]table.SortBy{ + { + Number: 1, + }, + }) + return + } + res := make([]table.SortBy, len(columnNumber)) + for i := range columnNumber { + res[i].Number = columnNumber[i] + } + t.Tbl.SortBy(res) +} + // PrintPairStringToLine print pair string for a line , the format is as follows "*:\t". // spaceCount is the space character count which is placed in the offset of field string. // the default values of tabCount is 2. diff --git a/internal/cli/printer/printer_test.go b/internal/cli/printer/printer_test.go index 0de5b490b..a7d845a3a 100644 --- a/internal/cli/printer/printer_test.go +++ b/internal/cli/printer/printer_test.go @@ -112,3 +112,25 @@ func checkOutPut(t *testing.T, captureFunc func() (string, error), expect string } assert.Equal(t, expect, capturedOutput) } + +func TestSort(t *testing.T) { + printer := NewTablePrinter(os.Stdout) + headerRow := make([]interface{}, len(header)) + for i, h := range header { + headerRow[i] = h + } + printer.SetHeader(headerRow...) + printer.SortBy(1) + for _, r := range [][]string{ + {"cedar51", "default", "apecloud-mysql", "ac-mysql-8.0.30", "Delete", "Feb 20,2023 16:39 UTC+0800"}, + {"brier63", "default", "apecloud-mysql", "ac-mysql-8.0.30", "Delete", "Feb 20,2023 16:39 UTC+0800"}, + {"alpha19", "default", "apecloud-mysql", "ac-mysql-8.0.30", "Delete", "Feb 20,2023 16:39 UTC+0800"}, + } { + row := make([]interface{}, len(r)) + for i, rr := range r { + row[i] = rr + } + printer.AddRow(row...) + } + printer.Print() +} From 17822649ae2971bdaac47b1067833c53a2ea32b6 Mon Sep 17 00:00:00 2001 From: Ziang Guo Date: Tue, 23 May 2023 21:24:50 +0800 Subject: [PATCH 338/439] chore: weaviate enhancement (#3358) --- deploy/weaviate/Chart.yaml | 2 +- .../config/weaviate-config-constraint.cue | 63 +++++++++++++++++++ .../weaviate/templates/clusterdefinition.yaml | 27 ++++---- .../weaviate/templates/configconstraint.yaml | 41 ++++++++++++ deploy/weaviate/templates/configmap.yaml | 22 ++++++- 5 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 deploy/weaviate/config/weaviate-config-constraint.cue create mode 100644 deploy/weaviate/templates/configconstraint.yaml diff --git a/deploy/weaviate/Chart.yaml b/deploy/weaviate/Chart.yaml index 32f36a70a..c4aff5c33 100644 --- a/deploy/weaviate/Chart.yaml +++ b/deploy/weaviate/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: weaviate -description: . +description: Weaviate is an open-source vector database. It allows you to store data objects and vector embeddings from your favorite ML-models, and scale seamlessly into billions of data objects. type: application diff --git a/deploy/weaviate/config/weaviate-config-constraint.cue b/deploy/weaviate/config/weaviate-config-constraint.cue new file mode 100644 index 000000000..d73cd76e1 --- /dev/null +++ b/deploy/weaviate/config/weaviate-config-constraint.cue @@ -0,0 +1,63 @@ +//Copyright (C) 2022-2023 ApeCloud Co., Ltd +// +//This file is part of KubeBlocks project +// +//This program is free software: you can redistribute it and/or modify +//it under the terms of the GNU Affero General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. +// +//This program is distributed in the hope that it will be useful +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU Affero General Public License for more details. +// +//You should have received a copy of the GNU Affero General Public License +//along with this program. If not, see . + +#WeaviateEnvs: { + + // Which modules to enable in the setup? + ENABLE_MODULES ?: string + + // The endpoint where to reach the transformers module if enabled + TRANSFORMERS_INFERENCE_API ?: string + + // The endpoint where to reach the clip module if enabled + CLIP_INFERENCE_API ?: string + + QNA_INFERENCE_API ?: string + + // The endpoint where to reach the img2vec-neural module if enabled + IMAGE_INFERENCE_API ?: string + + // The id of the AWS access key for the desired account. + AWS_ACCESS_KEY_ID ?: string + + // The secret AWS access key for the desired account. + AWS_SECRET_ACCESS_KEY ?: string + + // The path to the secret GCP service account or workload identity file. + GOOGLE_APPLICATION_CREDENTIALS ?: string + + // The name of your Azure Storage account. + AZURE_STORAGE_ACCOUNT ?: string + + // An access key for your Azure Storage account. + AZURE_STORAGE_KEY ?: string + + // A string that includes the authorization information required. + AZURE_STORAGE_CONNECTION_STRING ?: string + + SPELLCHECK_INFERENCE_API ?: string + NER_INFERENCE_API ?: string + SUM_INFERENCE_API ?: string + OPENAI_APIKEY ?: string + HUGGINGFACE_APIKEY ?: string + COHERE_APIKEY ?: string + PALM_APIKEY ?: string + +} + +// SectionName is section name +[SectionName=_]: #WeaviateEnvs diff --git a/deploy/weaviate/templates/clusterdefinition.yaml b/deploy/weaviate/templates/clusterdefinition.yaml index af7dad577..6bd9bc3b1 100644 --- a/deploy/weaviate/templates/clusterdefinition.yaml +++ b/deploy/weaviate/templates/clusterdefinition.yaml @@ -22,11 +22,12 @@ spec: builtIn: false exporterConfig: scrapePath: /metrics - scrapePort: 9187 + scrapePort: 2112 logConfigs: configSpecs: - name: weaviate-config-template templateRef: weaviate-config-template + constraintRef: weaviate-config-constraints volumeName: weaviate-config namespace: {{ .Release.Namespace }} service: @@ -44,18 +45,11 @@ spec: - name: weaviate imagePullPolicy: {{default .Values.images.pullPolicy "IfNotPresent"}} command: - - /bin/weaviate - args: - - --host - - 0.0.0.0 - - --port - - "8080" - - --scheme - - http - - --config-file - - /weaviate-config/conf.yaml - - --read-timeout=60s - - --write-timeout=60s + - /bin/sh + - -c + - | + export $(cat /weaviate-config/envs | xargs) + /bin/weaviate --host 0.0.0.0 --port "8080" --scheme http --config-file /weaviate-config/conf.yaml --read-timeout=60s --write-timeout=60s securityContext: runAsUser: 0 livenessProbe: @@ -100,7 +94,7 @@ spec: - name: tcp-weaviate containerPort: 8080 - name: tcp-metrics - containerPort: 9091 + containerPort: 2112 env: - name: CLUSTER_DATA_BIND_PORT value: "7001" @@ -109,7 +103,9 @@ spec: - name: GOGC value: "100" - name: PROMETHEUS_MONITORING_ENABLED - value: "false" + value: "true" + - name: PROMETHEUS_MONITORING_PORT + value: "2112" - name: QUERY_MAXIMUM_RESULTS value: "100000" - name: REINDEX_VECTOR_DIMENSIONS_AT_STARTUP @@ -126,3 +122,4 @@ spec: value: "7000" - name: CLUSTER_DATA_BIND_PORT value: "7001" + diff --git a/deploy/weaviate/templates/configconstraint.yaml b/deploy/weaviate/templates/configconstraint.yaml new file mode 100644 index 000000000..569ca890c --- /dev/null +++ b/deploy/weaviate/templates/configconstraint.yaml @@ -0,0 +1,41 @@ +{{- $cc := .Files.Get "config/mysql8-config-effect-scope.yaml" | fromYaml }} +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: ConfigConstraint +metadata: + name: weaviate-config-constraints + labels: + {{- include "weaviate.labels" . | nindent 4 }} +spec: + # top level mysql configuration type + cfgSchemaTopLevelName: WeaviateEnvs + + # ConfigurationSchema that impose restrictions on engine parameter's rule + configurationSchema: + # schema: auto generate from mmmcue scripts + # example: ../../internal/configuration/testdata/mysql_openapi.json + cue: |- + {{- .Files.Get "config/weavaite-config-constraint.cue" | nindent 6 }} + + ## define static parameter list + staticParameters: + - ENABLE_MODULES + - TRANSFORMERS_INFERENCE_API + - CLIP_INFERENCE_API + - QNA_INFERENCE_API + - IMAGE_INFERENCE_API + - SPELLCHECK_INFERENCE_API + - NER_INFERENCE_API + - SUM_INFERENCE_API + - OPENAI_APIKEY + - HUGGINGFACE_APIKEY + - COHERE_APIKEY + - PALM_APIKEY + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - GOOGLE_APPLICATION_CREDENTIALS + - AZURE_STORAGE_ACCOUNT + - AZURE_STORAGE_KEY + - AZURE_STORAGE_CONNECTION_STRING + + formatterConfig: + format: ini \ No newline at end of file diff --git a/deploy/weaviate/templates/configmap.yaml b/deploy/weaviate/templates/configmap.yaml index 176d2804c..4f79422d5 100644 --- a/deploy/weaviate/templates/configmap.yaml +++ b/deploy/weaviate/templates/configmap.yaml @@ -17,4 +17,24 @@ data: enabled: false query_defaults: limit: 100 - debug: false \ No newline at end of file + debug: false + + envs: |- + ENABLE_MODULES="" + TRANSFORMERS_INFERENCE_API="" + CLIP_INFERENCE_API="" + QNA_INFERENCE_API="" + IMAGE_INFERENCE_API="" + SPELLCHECK_INFERENCE_API="" + NER_INFERENCE_API="" + SUM_INFERENCE_API="" + OPENAI_APIKEY="" + HUGGINGFACE_APIKEY="" + COHERE_APIKEY="" + PALM_APIKEY="" + AWS_ACCESS_KEY_ID="" + AWS_SECRET_ACCESS_KEY="" + GOOGLE_APPLICATION_CREDENTIALS="" + AZURE_STORAGE_ACCOUNT="" + AZURE_STORAGE_KEY="" + AZURE_STORAGE_CONNECTION_STRING="" \ No newline at end of file From aa679cb8313325e7a0acfac7b6dab02de8098f06 Mon Sep 17 00:00:00 2001 From: dingben Date: Wed, 24 May 2023 09:53:47 +0800 Subject: [PATCH 339/439] feat: cli cluster create --set-file support use complete config file (#3288) --- internal/cli/cmd/cluster/cluster_test.go | 8 ++ internal/cli/cmd/cluster/create.go | 125 ++++++++++++++---- internal/cli/testing/testdata/cluster.yaml | 30 +++++ internal/cli/testing/testdata/component.yaml | 2 +- .../testdata/component_with_class_1c1g.yaml | 2 +- .../component_with_invalid_class.yaml | 2 +- .../component_with_invalid_resource.yaml | 16 +-- .../component_with_resource_1c1g.yaml | 16 +-- 8 files changed, 154 insertions(+), 47 deletions(-) create mode 100644 internal/cli/testing/testdata/cluster.yaml diff --git a/internal/cli/cmd/cluster/cluster_test.go b/internal/cli/cmd/cluster/cluster_test.go index e0f298f9d..d660accea 100644 --- a/internal/cli/cmd/cluster/cluster_test.go +++ b/internal/cli/cmd/cluster/cluster_test.go @@ -42,6 +42,7 @@ var _ = Describe("Cluster", func() { testComponentWithInvalidClassPath = "../../testing/testdata/component_with_invalid_class.yaml" testComponentWithResourcePath = "../../testing/testdata/component_with_resource_1c1g.yaml" testComponentWithInvalidResourcePath = "../../testing/testdata/component_with_invalid_resource.yaml" + testClusterPath = "../../testing/testdata/cluster.yaml" ) const ( @@ -81,6 +82,7 @@ var _ = Describe("Cluster", func() { } o.Options = o Expect(o.Complete()).To(Succeed()) + Expect(o.Validate()).To(Succeed()) Expect(o.Name).ShouldNot(BeEmpty()) Expect(o.Run()).Should(HaveOccurred()) }) @@ -245,6 +247,12 @@ var _ = Describe("Cluster", func() { o.SetFile = testComponentWithInvalidResourcePath Expect(o.Complete()).Should(HaveOccurred()) }) + + It("should succeed if create cluster with a complete config file", func() { + o.SetFile = testClusterPath + Expect(o.Complete()).Should(Succeed()) + Expect(o.Validate()).Should(Succeed()) + }) }) Context("create validate", func() { diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index b28fee5bd..63dbf24e0 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -198,8 +198,8 @@ func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra Run: func(cmd *cobra.Command, args []string) { o.Args = args cmdutil.CheckErr(o.CreateOptions.Complete()) - cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.Complete()) + cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.Run()) }, } @@ -217,9 +217,6 @@ func NewCreateCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra // add print flags printer.AddOutputFlagForCreate(cmd, &o.Format) - // set required flag - util.CheckErr(cmd.MarkFlagRequired("cluster-definition")) - // register flag completion func registerFlagCompletionFunc(cmd, f) @@ -319,11 +316,43 @@ func (o *CreateOptions) Validate() error { } func (o *CreateOptions) Complete() error { - if err := o.Validate(); err != nil { + var ( + compByte []byte + cls *appsv1alpha1.Cluster + clusterCompSpecs []appsv1alpha1.ClusterComponentSpec + err error + ) + if len(o.SetFile) > 0 { + if compByte, err = MultipleSourceComponents(o.SetFile, o.IOStreams.In); err != nil { + return err + } + if compByte, err = yaml.YAMLToJSON(compByte); err != nil { + return err + } + + // compatible with old file format that only specify the components + if err = json.Unmarshal(compByte, &cls); err != nil { + if clusterCompSpecs, err = parseClusterComponentSpec(compByte); err != nil { + return err + } + } else { + clusterCompSpecs = cls.Spec.ComponentSpecs + } + } + + // build annotation + o.buildAnnotation(cls) + + // build cluster definition + if err := o.buildClusterDef(cls); err != nil { return err } - components, err := o.buildComponents() + // build cluster version + o.buildClusterVersion(cls) + + // build components + components, err := o.buildComponents(clusterCompSpecs) if err != nil { return err } @@ -353,7 +382,7 @@ func (o *CreateOptions) CleanUp() error { } // buildComponents build components from file or set values -func (o *CreateOptions) buildComponents() ([]map[string]interface{}, error) { +func (o *CreateOptions) buildComponents(clusterCompSpecs []appsv1alpha1.ClusterComponentSpec) ([]map[string]interface{}, error) { var ( err error cd *appsv1alpha1.ClusterDefinition @@ -370,27 +399,9 @@ func (o *CreateOptions) buildComponents() ([]map[string]interface{}, error) { return nil, err } - // build components from file - if len(o.SetFile) > 0 { - var ( - compByte []byte - comps []map[string]interface{} - ) - if compByte, err = MultipleSourceComponents(o.SetFile, o.IOStreams.In); err != nil { - return nil, err - } - if compByte, err = yaml.YAMLToJSON(compByte); err != nil { - return nil, err - } - if err = json.Unmarshal(compByte, &comps); err != nil { - return nil, err - } - for _, comp := range comps { - var compSpec appsv1alpha1.ClusterComponentSpec - if err = runtime.DefaultUnstructuredConverter.FromUnstructured(comp, &compSpec); err != nil { - return nil, err - } - compSpecs = append(compSpecs, &compSpec) + if clusterCompSpecs != nil { + for _, comp := range clusterCompSpecs { + compSpecs = append(compSpecs, &comp) } } else { // build components from set values or environment variables @@ -1057,3 +1068,61 @@ func buildResourceLabels(clusterName string) map[string]string { constant.AppManagedByLabelKey: "kbcli", } } + +// build the cluster definition +// if the cluster definition is not specified, we will use the cluster definition in the cluster component +// if both of them are not specified, we will return an error +func (o *CreateOptions) buildClusterDef(cls *appsv1alpha1.Cluster) error { + if o.ClusterDefRef != "" { + return nil + } + + if cls != nil && cls.Spec.ClusterDefRef != "" { + o.ClusterDefRef = cls.Spec.ClusterDefRef + return nil + } + + return fmt.Errorf("a valid cluster definition is needed, use --cluster-definition to specify one, run \"kbcli clusterdefinition list\" to show all cluster definition") +} + +// build the cluster version +// if the cluster version is not specified, we will use the cluster version in the cluster component +// if both of them are not specified, we use default cluster version +func (o *CreateOptions) buildClusterVersion(cls *appsv1alpha1.Cluster) { + if o.ClusterVersionRef != "" { + return + } + + if cls != nil && cls.Spec.ClusterVersionRef != "" { + o.ClusterVersionRef = cls.Spec.ClusterVersionRef + } +} + +func (o *CreateOptions) buildAnnotation(cls *appsv1alpha1.Cluster) { + if cls == nil { + return + } + + if o.Annotations == nil { + o.Annotations = cls.Annotations + } +} + +// parse the cluster component spec +// compatible with old file format that only specify the components +func parseClusterComponentSpec(compByte []byte) ([]appsv1alpha1.ClusterComponentSpec, error) { + var compSpecs []appsv1alpha1.ClusterComponentSpec + var comps []map[string]interface{} + if err := json.Unmarshal(compByte, &comps); err != nil { + return nil, err + } + for _, comp := range comps { + var compSpec appsv1alpha1.ClusterComponentSpec + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(comp, &compSpec); err != nil { + return nil, err + } + compSpecs = append(compSpecs, compSpec) + } + + return compSpecs, nil +} diff --git a/internal/cli/testing/testdata/cluster.yaml b/internal/cli/testing/testdata/cluster.yaml new file mode 100644 index 000000000..832caf7af --- /dev/null +++ b/internal/cli/testing/testdata/cluster.yaml @@ -0,0 +1,30 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + annotations: {} + name: test-mycluster + namespace: default +spec: + affinity: + nodeLabels: {} + podAntiAffinity: Preferred + tenancy: SharedNode + topologyKeys: [] + clusterDefinitionRef: apecloud-mysql + clusterVersionRef: ac-mysql-8.0.30 + componentSpecs: + - componentDefRef: mysql + monitor: true + name: mysql + replicas: 3 + resources: {} + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + terminationPolicy: Delete + tolerations: [] \ No newline at end of file diff --git a/internal/cli/testing/testdata/component.yaml b/internal/cli/testing/testdata/component.yaml index 644530ad8..a4984b2ff 100644 --- a/internal/cli/testing/testdata/component.yaml +++ b/internal/cli/testing/testdata/component.yaml @@ -11,4 +11,4 @@ resources: requests: storage: 1Gi - volumeMode: Filesystem + volumeMode: Filesystem \ No newline at end of file diff --git a/internal/cli/testing/testdata/component_with_class_1c1g.yaml b/internal/cli/testing/testdata/component_with_class_1c1g.yaml index a4d035abf..a5adfde07 100644 --- a/internal/cli/testing/testdata/component_with_class_1c1g.yaml +++ b/internal/cli/testing/testdata/component_with_class_1c1g.yaml @@ -13,4 +13,4 @@ resources: requests: storage: 1Gi - volumeMode: Filesystem + volumeMode: Filesystem \ No newline at end of file diff --git a/internal/cli/testing/testdata/component_with_invalid_class.yaml b/internal/cli/testing/testdata/component_with_invalid_class.yaml index 23a158ecb..d9f359f45 100644 --- a/internal/cli/testing/testdata/component_with_invalid_class.yaml +++ b/internal/cli/testing/testdata/component_with_invalid_class.yaml @@ -13,4 +13,4 @@ resources: requests: storage: 1Gi - volumeMode: Filesystem + volumeMode: Filesystem \ No newline at end of file diff --git a/internal/cli/testing/testdata/component_with_invalid_resource.yaml b/internal/cli/testing/testdata/component_with_invalid_resource.yaml index 8141020ba..efdc476b1 100644 --- a/internal/cli/testing/testdata/component_with_invalid_resource.yaml +++ b/internal/cli/testing/testdata/component_with_invalid_resource.yaml @@ -8,11 +8,11 @@ cpu: 3 memory: 7Gi volumeClaimTemplates: - - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - volumeMode: Filesystem + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumeMode: Filesystem \ No newline at end of file diff --git a/internal/cli/testing/testdata/component_with_resource_1c1g.yaml b/internal/cli/testing/testdata/component_with_resource_1c1g.yaml index 479d00ec3..cfdf53ae8 100644 --- a/internal/cli/testing/testdata/component_with_resource_1c1g.yaml +++ b/internal/cli/testing/testdata/component_with_resource_1c1g.yaml @@ -8,11 +8,11 @@ cpu: 1 memory: 1Gi volumeClaimTemplates: - - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - volumeMode: Filesystem + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + volumeMode: Filesystem \ No newline at end of file From e2afbfcefed865cc7f649137b718a19eefdbba05 Mon Sep 17 00:00:00 2001 From: dingben Date: Wed, 24 May 2023 09:57:11 +0800 Subject: [PATCH 340/439] fix: return error when uninstall an unsuccessfully installed kubeblocks (#3355) --- internal/cli/cmd/kubeblocks/uninstall.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index b193f12d5..a1eb948da 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -249,7 +249,7 @@ func (o *UninstallOptions) uninstallAddons() error { if uninstall { // we only need to uninstall addons that are not disabled - if addon.Status.Phase == extensionsv1alpha1.AddonDisabled { + if addon.Spec.InstallSpec.IsDisabled() { continue } addons[addon.Name] = addon From 8a1e8665b13ccd83d3a82f2476bc0af52201cabf Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Wed, 24 May 2023 10:19:01 +0800 Subject: [PATCH 341/439] chore: improve cluster delete output info when cluster status is empty (#3377) --- internal/cli/cmd/cluster/create.go | 2 +- internal/cli/cmd/cluster/delete.go | 34 +++++++----------------- internal/cli/cmd/kubeblocks/uninstall.go | 11 ++++---- internal/cli/delete/delete.go | 5 ++-- 4 files changed, 19 insertions(+), 33 deletions(-) diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 63dbf24e0..837cd7e8b 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -486,7 +486,7 @@ func (o *CreateOptions) CreateDependencies(dryRun []string) error { }, }...).WithLabels(labels) - // postgresql need more rules for patronic + // postgresql need more rules for patroni if ok, err := o.isPostgresqlCluster(); err != nil { return err } else if ok { diff --git a/internal/cli/cmd/cluster/delete.go b/internal/cli/cmd/cluster/delete.go index 0eda28874..77347dd33 100644 --- a/internal/cli/cmd/cluster/delete.go +++ b/internal/cli/cmd/cluster/delete.go @@ -36,7 +36,6 @@ import ( "k8s.io/kubectl/pkg/util/templates" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - "github.com/apecloud/kubeblocks/internal/cli/cluster" "github.com/apecloud/kubeblocks/internal/cli/delete" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" @@ -74,6 +73,10 @@ func deleteCluster(o *delete.DeleteOptions, args []string) error { } func clusterPreDeleteHook(o *delete.DeleteOptions, object runtime.Object) error { + if object == nil { + return nil + } + cluster, err := getClusterFromObject(object) if err != nil { return err @@ -85,12 +88,11 @@ func clusterPreDeleteHook(o *delete.DeleteOptions, object runtime.Object) error } func clusterPostDeleteHook(o *delete.DeleteOptions, object runtime.Object) error { - c, err := getClusterFromObject(object) - if err != nil { - return err + if object == nil { + return nil } - dynamic, err := o.Factory.DynamicClient() + c, err := getClusterFromObject(object) if err != nil { return err } @@ -101,28 +103,12 @@ func clusterPostDeleteHook(o *delete.DeleteOptions, object runtime.Object) error } // HACK: for a postgresql cluster, we need to delete the sa, role and rolebinding - cd, err := cluster.GetClusterDefByName(dynamic, c.Spec.ClusterDefRef) - if err != nil { + if err = deleteDependencies(client, c.Namespace, c.Name); err != nil { return err } - for _, compSpec := range c.Spec.ComponentSpecs { - if err = deleteCompDependencies(client, c.Namespace, c.Name, cd, &compSpec); err != nil { - return err - } - } return nil } -func deleteCompDependencies(client kubernetes.Interface, ns string, name string, cd *appsv1alpha1.ClusterDefinition, - compSpec *appsv1alpha1.ClusterComponentSpec) error { - // if d, err := shouldCreateDependencies(cd, compSpec); err != nil { - // return err - // } else if !d { - // return nil - // } - return deleteDependencies(client, ns, name) -} - func deleteDependencies(client kubernetes.Interface, ns string, name string) error { klog.V(1).Infof("delete dependencies for cluster %s", name) var ( @@ -167,9 +153,9 @@ func getClusterFromObject(object runtime.Object) (*appsv1alpha1.Cluster, error) if object.GetObjectKind().GroupVersionKind().Kind != appsv1alpha1.ClusterKind { return nil, fmt.Errorf("object %s is not of kind %s", object.GetObjectKind().GroupVersionKind().Kind, appsv1alpha1.ClusterKind) } - unstructured := object.(*unstructured.Unstructured) + u := object.(*unstructured.Unstructured) cluster := &appsv1alpha1.Cluster{} - if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured.Object, cluster); err != nil { + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, cluster); err != nil { return nil, err } return cluster, nil diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index a1eb948da..97f659a20 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -111,9 +111,7 @@ func (o *UninstallOptions) PreCheck() error { // check if there is any resource should be removed first, if so, return error // and ask user to remove them manually if err := checkResources(o.Dynamic); err != nil { - if !apierrors.IsNotFound(err) { - return err - } + return err } // verify where kubeblocks is installed @@ -125,8 +123,11 @@ func (o *UninstallOptions) PreCheck() error { fmt.Fprintf(o.Out, "to uninstall KubeBlocks completely, please use:\n\t`kbcli kubeblocks uninstall -n `\n") } } + o.Namespace = kbNamespace - fmt.Fprintf(o.Out, "Uninstall KubeBlocks in namespace \"%s\"\n", kbNamespace) + if kbNamespace != "" { + fmt.Fprintf(o.Out, "Uninstall KubeBlocks in namespace \"%s\"\n", kbNamespace) + } return nil } @@ -332,7 +333,7 @@ func checkResources(dynamic dynamic.Interface) error { crs := map[string][]string{} for _, gvr := range gvrList { objList, err := dynamic.Resource(gvr).List(ctx, metav1.ListOptions{}) - if err != nil { + if err != nil && !apierrors.IsNotFound(err) { return err } for _, item := range objList.Items { diff --git a/internal/cli/delete/delete.go b/internal/cli/delete/delete.go index 73c01b96d..93c1a248d 100644 --- a/internal/cli/delete/delete.go +++ b/internal/cli/delete/delete.go @@ -171,7 +171,6 @@ func (o *DeleteOptions) deleteResult(r *resource.Result) error { if err != nil { return err } - var runtimeObj runtime.Object deleteInfos = append(deleteInfos, info) found++ @@ -182,10 +181,10 @@ func (o *DeleteOptions) deleteResult(r *resource.Result) error { if err = o.preDeleteResource(info); err != nil { return err } - if runtimeObj, err = o.deleteResource(info, options); err != nil { + if _, err = o.deleteResource(info, options); err != nil { return err } - if err = o.postDeleteResource(runtimeObj); err != nil { + if err = o.postDeleteResource(info.Object); err != nil { return err } fmt.Fprintf(o.Out, "%s %s deleted\n", info.Mapping.GroupVersionKind.Kind, info.Name) From c415ec5548ac53c6507f15c153493c92d6585697 Mon Sep 17 00:00:00 2001 From: wangyelei Date: Wed, 24 May 2023 10:56:54 +0800 Subject: [PATCH 342/439] chore: support volume expansion can run concurrently with other type opsRequest and recover from failure (#3386) --- apis/apps/v1alpha1/opsrequest_webhook.go | 73 +++++--- apis/apps/v1alpha1/opsrequest_webhook_test.go | 126 ++++++++------ apis/apps/v1alpha1/type.go | 9 - apis/apps/v1alpha1/webhook_suite_test.go | 4 + controllers/apps/cluster_status_utils.go | 42 +---- controllers/apps/cluster_status_utils_test.go | 3 +- controllers/apps/operations/ops_util.go | 11 -- controllers/apps/operations/type.go | 2 - .../apps/operations/volume_expansion.go | 156 +++++++----------- .../apps/operations/volume_expansion_test.go | 94 +++++------ .../operations/volume_expansion_updater.go | 77 ++++----- controllers/apps/opsrequest_controller.go | 4 +- .../apps/opsrequest_controller_test.go | 17 +- internal/cli/cmd/cluster/config_ops.go | 1 + internal/cli/cmd/cluster/operations.go | 45 ++++- internal/cli/cmd/cluster/operations_test.go | 50 +++++- internal/cli/printer/format.go | 17 ++ internal/constant/const.go | 1 + .../lifecycle/transformer_cluster_status.go | 22 +-- internal/testutil/apps/opsrequest_util.go | 5 + 20 files changed, 386 insertions(+), 373 deletions(-) diff --git a/apis/apps/v1alpha1/opsrequest_webhook.go b/apis/apps/v1alpha1/opsrequest_webhook.go index 57abd8055..25f503a14 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook.go +++ b/apis/apps/v1alpha1/opsrequest_webhook.go @@ -27,7 +27,6 @@ import ( "strings" "github.com/pkg/errors" - "github.com/spf13/viper" "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -308,7 +307,13 @@ func (r *OpsRequest) validateVolumeExpansion(ctx context.Context, cli client.Cli if err := r.checkComponentExistence(cluster, componentNames); err != nil { return err } - + runningOpsList, err := GetRunningOpsByOpsType(ctx, cli, r.Spec.ClusterRef, r.Namespace, string(VolumeExpansionType)) + if err != nil { + return err + } + if len(runningOpsList) > 0 && runningOpsList[0].Name != r.Name { + return fmt.Errorf("existing other VolumeExpansion OpsRequest: %s is running in Cluster: %s, handle this OpsRequest first", runningOpsList[0].Name, cluster.Name) + } return r.checkVolumesAllowExpansion(ctx, cli, cluster) } @@ -351,8 +356,6 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client. vols[comp.ComponentName][vct.Name] = Entity{false, nil, false, vct.Storage} } } - // TODO: remove it after supporting to recover volume expansion when it fails. - recoverVolumeExpansionFailure := viper.GetBool(constant.CfgRecoverVolumeExpansionFailure) // traverse the spec to update volumes for _, comp := range cluster.Spec.ComponentSpecs { if _, ok := vols[comp.Name]; !ok { @@ -363,14 +366,6 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client. if !ok { continue } - // TODO: - // compare the requested storage size with the pvc.status.capacity when KubeBlocks supports to manage the pvc by self - // and supports to recover volume expansion when it is fails. - previousValue := *vct.Spec.Resources.Requests.Storage() - if e.requestStorage.Cmp(previousValue) < 0 && !recoverVolumeExpansionFailure { - return fmt.Errorf(`requested storage size of volumeClaimTemplate "%s" can not less than previous values "%s" unless both Kubernetes and KubeBlocks support RECOVER_VOLUME_EXPANSION_FAILURE`, - vct.Name, previousValue.String()) - } e.existInSpec = true e.storageClassName = vct.Spec.StorageClassName vols[comp.Name][vct.Name] = e @@ -378,14 +373,16 @@ func (r *OpsRequest) checkVolumesAllowExpansion(ctx context.Context, cli client. } // check all used storage classes + var err error for cname, compVols := range vols { for vname := range compVols { e := vols[cname][vname] if !e.existInSpec { continue } - if e.storageClassName == nil { - e.storageClassName = r.getSCNameByPvc(ctx, cli, cname, vname) + e.storageClassName, err = r.getSCNameByPvcAndCheckStorageSize(ctx, cli, cname, vname, e.requestStorage) + if err != nil { + return err } allowExpansion, err := r.checkStorageClassAllowExpansion(ctx, cli, e.storageClassName) if err != nil { @@ -447,23 +444,30 @@ func (r *OpsRequest) checkStorageClassAllowExpansion(ctx context.Context, return *storageClass.AllowVolumeExpansion, nil } -// getSCNameByPvc gets the storageClassName by pvc. -func (r *OpsRequest) getSCNameByPvc(ctx context.Context, +// getSCNameByPvcAndCheckStorageSize gets the storageClassName by pvc and checks if the storage size is valid. +func (r *OpsRequest) getSCNameByPvcAndCheckStorageSize(ctx context.Context, cli client.Client, compName, - vctName string) *string { + vctName string, + requestStorage resource.Quantity) (*string, error) { pvcList := &corev1.PersistentVolumeClaimList{} if err := cli.List(ctx, pvcList, client.InNamespace(r.Namespace), client.MatchingLabels{ - constant.AppInstanceLabelKey: r.Spec.ClusterRef, - constant.KBAppComponentLabelKey: compName, - constant.PVCNameLabelKey: vctName, + constant.AppInstanceLabelKey: r.Spec.ClusterRef, + constant.KBAppComponentLabelKey: compName, + constant.VolumeClaimTemplateNameLabelKey: vctName, }, client.Limit(1)); err != nil { - return nil + return nil, err } if len(pvcList.Items) == 0 { - return nil + return nil, nil + } + pvc := pvcList.Items[0] + previousValue := *pvc.Status.Capacity.Storage() + if requestStorage.Cmp(previousValue) < 0 { + return nil, fmt.Errorf(`requested storage size of volumeClaimTemplate "%s" can not less than status.capacity.storage "%s" `, + vctName, previousValue.String()) } - return pvcList.Items[0].Spec.StorageClassName + return pvc.Spec.StorageClassName, nil } // validateVerticalResourceList checks if k8s resourceList is legal @@ -484,3 +488,26 @@ func notEmptyError(target string) error { func invalidValueError(target string, value string) error { return fmt.Errorf(`invalid value for "%s": %s`, target, value) } + +// GetRunningOpsByOpsType gets the running opsRequests by type. +func GetRunningOpsByOpsType(ctx context.Context, cli client.Client, + clusterName, namespace, opsType string) ([]OpsRequest, error) { + opsRequestList := &OpsRequestList{} + if err := cli.List(ctx, opsRequestList, client.MatchingLabels{ + constant.AppInstanceLabelKey: clusterName, + constant.OpsRequestTypeLabelKey: opsType, + }, client.InNamespace(namespace)); err != nil { + return nil, err + } + if len(opsRequestList.Items) == 0 { + return nil, nil + } + var runningOpsList []OpsRequest + for _, v := range opsRequestList.Items { + if v.Status.Phase == OpsRunningPhase { + runningOpsList = append(runningOpsList, v) + break + } + } + return runningOpsList, nil +} diff --git a/apis/apps/v1alpha1/opsrequest_webhook_test.go b/apis/apps/v1alpha1/opsrequest_webhook_test.go index ed384b82c..9bd8edb73 100644 --- a/apis/apps/v1alpha1/opsrequest_webhook_test.go +++ b/apis/apps/v1alpha1/opsrequest_webhook_test.go @@ -27,7 +27,6 @@ import ( . "github.com/onsi/gomega" "github.com/sethvargo/go-password/password" - "github.com/spf13/viper" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -95,13 +94,43 @@ var _ = Describe("OpsRequest webhook", func() { AllowVolumeExpansion: &allowVolumeExpansion, } err := testCtx.CheckedCreateObj(ctx, storageClass) - if err != nil { - fmt.Printf("create storage class error: %s\n", err.Error()) - } Expect(err).Should(BeNil()) return storageClass } + createPVC := func(clusterName, compName, storageClassName, vctName string, index int) *corev1.PersistentVolumeClaim { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%s-%d", vctName, clusterName, compName, index), + Namespace: testCtx.DefaultNamespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: clusterName, + constant.VolumeClaimTemplateNameLabelKey: vctName, + constant.KBAppComponentLabelKey: compName, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "storage": resource.MustParse("1Gi"), + }, + }, + StorageClassName: &storageClassName, + }, + } + Expect(testCtx.CheckedCreateObj(ctx, pvc)).ShouldNot(HaveOccurred()) + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pvc), pvc)).ShouldNot(HaveOccurred()) + patch := client.MergeFrom(pvc.DeepCopy()) + pvc.Status.Capacity = corev1.ResourceList{ + "storage": resource.MustParse("1Gi"), + } + Expect(k8sClient.Status().Patch(ctx, pvc, patch)).ShouldNot(HaveOccurred()) + return pvc + } + notFoundComponentsString := func(notFoundComponents string) string { return fmt.Sprintf("components: [%s] not found", notFoundComponents) } @@ -237,19 +266,24 @@ var _ = Describe("OpsRequest webhook", func() { } testVolumeExpansion := func(cluster *Cluster) { - By("By testing volumeExpansion - target component not exist") - opsRequest := createTestOpsRequest(clusterName, opsRequestName, VolumeExpansionType) - opsRequest.Spec.VolumeExpansionList = []VolumeExpansion{ - { - ComponentOps: ComponentOps{ComponentName: "ve-not-exist"}, - VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ - { - Name: "data", - Storage: resource.MustParse("2Gi"), + getSingleVolumeExpansionList := func(compName, vctName, storage string) []VolumeExpansion { + return []VolumeExpansion{ + { + ComponentOps: ComponentOps{ComponentName: compName}, + VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ + { + Name: vctName, + Storage: resource.MustParse(storage), + }, }, }, - }, + } } + defaultVCTName := "data" + targetStorage := "2Gi" + By("By testing volumeExpansion - target component not exist") + opsRequest := createTestOpsRequest(clusterName, opsRequestName, VolumeExpansionType) + opsRequest.Spec.VolumeExpansionList = getSingleVolumeExpansionList("ve-not-exist", defaultVCTName, targetStorage) Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring(notFoundComponentsString("ve-not-exist"))) By("By testing volumeExpansion - target volume not exist") @@ -258,11 +292,11 @@ var _ = Describe("OpsRequest webhook", func() { VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ { Name: "log", - Storage: resource.MustParse("2Gi"), + Storage: resource.MustParse(targetStorage), }, { - Name: "data", - Storage: resource.MustParse("2Gi"), + Name: defaultVCTName, + Storage: resource.MustParse(targetStorage), }, }, }, @@ -271,17 +305,7 @@ var _ = Describe("OpsRequest webhook", func() { Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring("volumeClaimTemplates: [log] not found in component: " + componentName)) By("By testing volumeExpansion - storageClass do not support volume expansion") - volumeExpansionList = []VolumeExpansion{ - { - ComponentOps: ComponentOps{ComponentName: componentName}, - VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ - { - Name: "data", - Storage: resource.MustParse("2Gi"), - }, - }, - }, - } + volumeExpansionList = getSingleVolumeExpansionList(componentName, defaultVCTName, targetStorage) opsRequest.Spec.VolumeExpansionList = volumeExpansionList notSupportMsg := fmt.Sprintf("volumeClaimTemplate: [data] not support volume expansion in component: %s, you can view infos by command: kubectl get sc", componentName) Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring(notSupportMsg)) @@ -290,34 +314,25 @@ var _ = Describe("OpsRequest webhook", func() { storageClassName := "standard" storageClass := createStorageClass(testCtx.Ctx, storageClassName, "true", true) Expect(storageClass).ShouldNot(BeNil()) + // mock to create pvc + createPVC(clusterName, componentName, storageClassName, defaultVCTName, 0) By("testing volumeExpansion with smaller storage, expect an error occurs") - opsRequest.Spec.VolumeExpansionList = []VolumeExpansion{ - { - ComponentOps: ComponentOps{ComponentName: "replicasets"}, - VolumeClaimTemplates: []OpsRequestVolumeClaimTemplate{ - { - Name: "data", - Storage: resource.MustParse("500Mi"), - }, - }, - }, - } + opsRequest.Spec.VolumeExpansionList = getSingleVolumeExpansionList(componentName, defaultVCTName, "500Mi") Expect(testCtx.CreateObj(ctx, opsRequest)).Should(HaveOccurred()) - Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring("can not less than previous values")) - - By("test volumeExpansion with smaller storage and RECOVER_VOLUME_EXPANSION_FAILURE=true, expect succeed") - viper.Set(constant.CfgRecoverVolumeExpansionFailure, true) - Expect(testCtx.CreateObj(ctx, opsRequest)).Should(Succeed()) - - // TODO - By("testing volumeExpansion - pvc exists") - // TODO - By("By testing volumeExpansion - (TODO)use specified storage class") - // Eventually(func() bool { - // opsRequest.Spec.VolumeExpansionList = []VolumeExpansion{volumeExpansionList[3]} - // Expect(testCtx.CheckedCreateObj(ctx, opsRequest)).Should(BeNil()) - // }).Should(BeTrue()) + Expect(testCtx.CreateObj(ctx, opsRequest).Error()).To(ContainSubstring(`requested storage size of volumeClaimTemplate "data" can not less than status.capacity.storage "1Gi"`)) + + By("testing other volumeExpansion opsRequest exists") + opsRequest.Spec.VolumeExpansionList = getSingleVolumeExpansionList(componentName, defaultVCTName, targetStorage) + Expect(testCtx.CreateObj(ctx, opsRequest)).ShouldNot(HaveOccurred()) + // mock ops to running + patch := client.MergeFrom(opsRequest.DeepCopy()) + opsRequest.Status.Phase = OpsRunningPhase + Expect(k8sClient.Status().Patch(ctx, opsRequest, patch)).ShouldNot(HaveOccurred()) + // create another ops + opsRequest1 := createTestOpsRequest(clusterName, opsRequestName+"1", VolumeExpansionType) + opsRequest1.Spec.VolumeExpansionList = getSingleVolumeExpansionList(componentName, defaultVCTName, "3Gi") + Expect(testCtx.CreateObj(ctx, opsRequest1).Error()).Should(ContainSubstring("existing other VolumeExpansion OpsRequest")) } testHorizontalScaling := func(clusterDef *ClusterDefinition, cluster *Cluster) { @@ -461,10 +476,13 @@ kind: OpsRequest metadata: name: %s namespace: default + labels: + app.kubernetes.io/instance: %s + ops.kubeblocks.io/ops-type: %s spec: clusterRef: %s type: %s -`, opsRequestName+randomStr, clusterName, opsType) +`, opsRequestName+randomStr, clusterName, opsType, clusterName, opsType) opsRequest := &OpsRequest{} _ = yaml.Unmarshal([]byte(opsRequestYaml), opsRequest) return opsRequest diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index 7e67763dd..6cff1222a 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -118,15 +118,6 @@ const ( SpecReconcilingClusterCompPhase ClusterComponentPhase = "Updating" CreatingClusterCompPhase ClusterComponentPhase = "Creating" // DeletingClusterCompPhase ClusterComponentPhase = "Deleting" // DO REVIEW: may merged with Stopping - - // REVIEW: following are variant of "Updating", why not have "Updating" phase with detail Status.Conditions - // VolumeExpandingClusterCompPhase ClusterComponentPhase = "VolumeExpanding" - // HorizontalScalingClusterCompPhase ClusterComponentPhase = "HorizontalScaling" - // VerticalScalingClusterCompPhase ClusterComponentPhase = "VerticalScaling" - // VersionUpgradingClusterCompPhase ClusterComponentPhase = "Upgrading" - // ReconfiguringClusterCompPhase ClusterComponentPhase = "Reconfiguring" - // ExposingClusterCompPhase ClusterComponentPhase = "Exposing" - // RollingClusterCompPhase ClusterComponentPhase = "Rolling" // REVIEW: original value is Rebooting, and why not having stopping -> stopped -> starting -> running ) const ( diff --git a/apis/apps/v1alpha1/webhook_suite_test.go b/apis/apps/v1alpha1/webhook_suite_test.go index 4a4e4b232..2f4ab63df 100644 --- a/apis/apps/v1alpha1/webhook_suite_test.go +++ b/apis/apps/v1alpha1/webhook_suite_test.go @@ -30,6 +30,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" admissionv1 "k8s.io/api/admission/v1" admissionregv1 "k8s.io/api/admissionregistration/v1" @@ -112,6 +113,9 @@ var _ = BeforeSuite(func() { err = storagev1.AddToScheme(scheme) Expect(err).NotTo(HaveOccurred()) + err = corev1.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred()) + // +kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme}) diff --git a/controllers/apps/cluster_status_utils.go b/controllers/apps/cluster_status_utils.go index cc04db9e2..1c2bca0b5 100644 --- a/controllers/apps/cluster_status_utils.go +++ b/controllers/apps/cluster_status_utils.go @@ -40,6 +40,7 @@ import ( opsutil "github.com/apecloud/kubeblocks/controllers/apps/operations/util" "github.com/apecloud/kubeblocks/controllers/k8score" "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/controller/lifecycle" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -141,45 +142,6 @@ func getEventInvolvedObject(ctx context.Context, cli client.Client, event *corev return nil, err } -// handleClusterPhaseWhenCompsNotReady handles the Cluster.status.phase when some components are Abnormal or Failed. -// TODO: Clear definitions need to be added to determine whether components will affect cluster availability in ClusterDefinition. -func handleClusterPhaseWhenCompsNotReady(cluster *appsv1alpha1.Cluster, - componentMap map[string]string, - clusterAvailabilityEffectMap map[string]bool) { - var ( - clusterIsFailed bool - failedCompCount int - isVolumeExpanding bool - ) - opsRecords, _ := opsutil.GetOpsRequestSliceFromCluster(cluster) - if len(opsRecords) != 0 && opsRecords[0].Type == appsv1alpha1.VolumeExpansionType { - isVolumeExpanding = true - } - for k, v := range cluster.Status.Components { - // determine whether other components are still doing operation, i.e., create/restart/scaling. - // waiting for operation to complete except for volumeExpansion operation. - // because this operation will not affect cluster availability. - if !slices.Contains(appsv1alpha1.GetComponentTerminalPhases(), v.Phase) && !isVolumeExpanding { - return - } - if v.Phase == appsv1alpha1.FailedClusterCompPhase { - failedCompCount += 1 - componentDefName := componentMap[k] - // if the component can affect cluster availability, set Cluster.status.phase to Failed - if clusterAvailabilityEffectMap[componentDefName] { - clusterIsFailed = true - break - } - } - } - // If all components fail or there are failed components that affect the availability of the cluster, set phase to Failed - if failedCompCount == len(cluster.Status.Components) || clusterIsFailed { - cluster.Status.Phase = appsv1alpha1.FailedClusterPhase - } else { - cluster.Status.Phase = appsv1alpha1.AbnormalClusterPhase - } -} - // getComponentRelatedInfo gets componentMap, clusterAvailabilityMap and component definition information func getComponentRelatedInfo(cluster *appsv1alpha1.Cluster, clusterDef *appsv1alpha1.ClusterDefinition, componentName string) (map[string]string, map[string]bool, *appsv1alpha1.ClusterComponentDefinition, error) { @@ -256,7 +218,7 @@ func handleClusterStatusByEvent(ctx context.Context, cli client.Client, recorder return nil } // handle Cluster.status.phase when some components are not ready. - handleClusterPhaseWhenCompsNotReady(cluster, componentMap, clusterAvailabilityEffectMap) + lifecycle.HandleClusterPhaseWhenCompsNotReady(cluster, componentMap, clusterAvailabilityEffectMap) if err = cli.Status().Patch(ctx, cluster, patch); err != nil { return err } diff --git a/controllers/apps/cluster_status_utils_test.go b/controllers/apps/cluster_status_utils_test.go index 93d0d5575..3a7b0fee6 100644 --- a/controllers/apps/cluster_status_utils_test.go +++ b/controllers/apps/cluster_status_utils_test.go @@ -34,6 +34,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/controller/lifecycle" intctrlutil "github.com/apecloud/kubeblocks/internal/generics" testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" testk8s "github.com/apecloud/kubeblocks/internal/testutil/k8s" @@ -151,7 +152,7 @@ var _ = Describe("test cluster Failed/Abnormal phase", func() { Phase: compPhase, }, } - handleClusterPhaseWhenCompsNotReady(clusterObj, nil, nil) + lifecycle.HandleClusterPhaseWhenCompsNotReady(clusterObj, nil, nil) Expect(clusterObj.Status.Phase).Should(Equal(expectClusterPhase)) } diff --git a/controllers/apps/operations/ops_util.go b/controllers/apps/operations/ops_util.go index 9609a04ed..e00a94876 100644 --- a/controllers/apps/operations/ops_util.go +++ b/controllers/apps/operations/ops_util.go @@ -218,17 +218,6 @@ func patchValidateErrorCondition(ctx context.Context, cli client.Client, opsRes return PatchOpsStatus(ctx, cli, opsRes, appsv1alpha1.OpsFailedPhase, condition) } -// getOpsRequestNameFromAnnotation gets OpsRequest.name from cluster.annotations -func getOpsRequestNameFromAnnotation(cluster *appsv1alpha1.Cluster, opsType appsv1alpha1.OpsType) *string { - opsRequestSlice, _ := opsutil.GetOpsRequestSliceFromCluster(cluster) - for _, v := range opsRequestSlice { - if v.Type == opsType { - return &v.Name - } - } - return nil -} - // GetOpsRecorderFromSlice gets OpsRequest recorder from slice by target cluster phase func GetOpsRecorderFromSlice(opsRequestSlice []appsv1alpha1.OpsRecorder, opsRequestName string) (int, appsv1alpha1.OpsRecorder) { diff --git a/controllers/apps/operations/type.go b/controllers/apps/operations/type.go index 9c70088de..c09505388 100644 --- a/controllers/apps/operations/type.go +++ b/controllers/apps/operations/type.go @@ -101,8 +101,6 @@ const ( ProcessingReasonHorizontalScaling = "HorizontalScaling" // ProcessingReasonVerticalScaling is the reason of the "OpsRequestProcessed" condition for the vertical scaling opsRequest processing in cluster. ProcessingReasonVerticalScaling = "VerticalScaling" - // ProcessingReasonVolumeExpanding is the reason of the "OpsRequestProcessed" condition for the volume expansion opsRequest processing in cluster. - ProcessingReasonVolumeExpanding = "VolumeExpanding" // ProcessingReasonStarting is the reason of the "OpsRequestProcessed" condition for the start opsRequest processing in cluster. ProcessingReasonStarting = "Starting" // ProcessingReasonStopping is the reason of the "OpsRequestProcessed" condition for the stop opsRequest processing in cluster. diff --git a/controllers/apps/operations/volume_expansion.go b/controllers/apps/operations/volume_expansion.go index 12f83250c..c27bd5067 100644 --- a/controllers/apps/operations/volume_expansion.go +++ b/controllers/apps/operations/volume_expansion.go @@ -27,7 +27,6 @@ import ( "time" "github.com/pkg/errors" - "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,11 +51,7 @@ const ( func init() { // the volume expansion operation only support online expanding now, so this operation not affect the cluster availability. volumeExpansionBehaviour := OpsBehaviour{ - FromClusterPhases: appsv1alpha1.GetClusterUpRunningPhases(), - ToClusterPhase: appsv1alpha1.SpecReconcilingClusterPhase, - MaintainClusterPhaseBySelf: true, - OpsHandler: volumeExpansionOpsHandler{}, - ProcessingReasonInClusterCondition: ProcessingReasonVolumeExpanding, + OpsHandler: volumeExpansionOpsHandler{}, } opsMgr := GetOpsManager() @@ -97,16 +92,15 @@ func (ve volumeExpansionOpsHandler) Action(reqCtx intctrlutil.RequestCtx, cli cl // the Reconcile function for volume expansion opsRequest. func (ve volumeExpansionOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCtx, cli client.Client, opsRes *OpsResource) (appsv1alpha1.OpsPhase, time.Duration, error) { var ( - opsRequest = opsRes.OpsRequest - // decide whether all pvcs of volumeClaimTemplate are Failed or Succeed - allVCTCompleted = true - requeueAfter time.Duration - err error - opsRequestPhase = appsv1alpha1.OpsRunningPhase - oldOpsRequestStatus = opsRequest.Status.DeepCopy() - oldClusterStatus = opsRes.Cluster.Status.DeepCopy() - expectProgressCount int - succeedProgressCount int + opsRequest = opsRes.OpsRequest + requeueAfter time.Duration + err error + opsRequestPhase = appsv1alpha1.OpsRunningPhase + oldOpsRequestStatus = opsRequest.Status.DeepCopy() + oldClusterStatus = opsRes.Cluster.Status.DeepCopy() + expectProgressCount int + succeedProgressCount int + completedProgressCount int ) patch := client.MergeFrom(opsRequest.DeepCopy()) @@ -119,27 +113,22 @@ func (ve volumeExpansionOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCt // sync the volumeClaimTemplate status and component phase On the OpsRequest and Cluster. for _, v := range opsRequest.Spec.VolumeExpansionList { compStatus := opsRequest.Status.Components[v.ComponentName] - completedOnComponent := true for _, vct := range v.VolumeClaimTemplates { - succeedCount, expectCount, isCompleted, err := ve.handleVCTExpansionProgress(reqCtx, cli, opsRes, + succeedCount, expectCount, completedCount, err := ve.handleVCTExpansionProgress(reqCtx, cli, opsRes, &compStatus, storageMap, v.ComponentName, vct.Name) if err != nil { return "", requeueAfter, err } expectProgressCount += expectCount succeedProgressCount += succeedCount - if !isCompleted { - requeueAfter = time.Minute - allVCTCompleted = false - completedOnComponent = false - } + completedProgressCount += completedCount } - // when component expand volume completed, do it. - ve.setComponentPhaseForClusterAndOpsRequest(&compStatus, opsRes.Cluster, v.ComponentName, completedOnComponent) opsRequest.Status.Components[v.ComponentName] = compStatus } - opsRequest.Status.Progress = fmt.Sprintf("%d/%d", succeedProgressCount, expectProgressCount) - + if completedProgressCount != expectProgressCount { + requeueAfter = time.Minute + } + opsRequest.Status.Progress = fmt.Sprintf("%d/%d", completedProgressCount, expectProgressCount) // patch OpsRequest.status.components if !reflect.DeepEqual(oldOpsRequestStatus, opsRequest.Status) { if err = cli.Status().Patch(reqCtx.Ctx, opsRequest, patch); err != nil { @@ -148,26 +137,24 @@ func (ve volumeExpansionOpsHandler) ReconcileAction(reqCtx intctrlutil.RequestCt } // check all pvcs of volumeClaimTemplate are successful - allVCTSucceed := expectProgressCount == succeedProgressCount - if allVCTSucceed { - opsRequestPhase = appsv1alpha1.OpsSucceedPhase - } else if allVCTCompleted { - // all volume claim template volume expansion completed, but allVCTSucceed is false. - // decide the OpsRequest is failed. - opsRequestPhase = appsv1alpha1.OpsFailedPhase - } - - if ve.checkIsTimeOut(opsRequest, allVCTSucceed) { - // if volume expansion timed out, do it - opsRequestPhase = appsv1alpha1.OpsFailedPhase - err = errors.New(fmt.Sprintf("Timed out waiting for volume expansion completed, the timeout is %g minutes", VolumeExpansionTimeOut.Minutes())) + if expectProgressCount == completedProgressCount { + if expectProgressCount == succeedProgressCount { + opsRequestPhase = appsv1alpha1.OpsSucceedPhase + } else { + opsRequestPhase = appsv1alpha1.OpsFailedPhase + } + } else { + // check whether the volume expansion operation has timed out + if time.Now().After(opsRequest.Status.StartTimestamp.Add(VolumeExpansionTimeOut)) { + // if volume expansion timed out, do it + opsRequestPhase = appsv1alpha1.OpsFailedPhase + err = errors.New(fmt.Sprintf("Timed out waiting for volume expansion completed, the timeout is %g minutes", VolumeExpansionTimeOut.Minutes())) + } } - // when opsRequest completed or cluster status is changed, do it if patchErr := ve.patchClusterStatus(reqCtx, cli, opsRes, opsRequestPhase, oldClusterStatus, clusterPatch); patchErr != nil { return "", requeueAfter, patchErr } - return opsRequestPhase, requeueAfter, err } @@ -205,38 +192,6 @@ func (ve volumeExpansionOpsHandler) SaveLastConfiguration(reqCtx intctrlutil.Req return nil } -// checkIsTimeOut check whether the volume expansion operation has timed out -func (ve volumeExpansionOpsHandler) checkIsTimeOut(opsRequest *appsv1alpha1.OpsRequest, allVCTSucceed bool) bool { - return !allVCTSucceed && time.Now().After(opsRequest.Status.StartTimestamp.Add(VolumeExpansionTimeOut)) -} - -// setClusterComponentPhaseToRunning when component expand volume completed, check whether change the component status. -func (ve volumeExpansionOpsHandler) setComponentPhaseForClusterAndOpsRequest(component *appsv1alpha1.OpsRequestComponentStatus, - cluster *appsv1alpha1.Cluster, - componentName string, - completedOnComponent bool) { - if !completedOnComponent { - return - } - c, ok := cluster.Status.Components[componentName] - if !ok { - return - } - p := c.Phase - if p == appsv1alpha1.SpecReconcilingClusterCompPhase { - p = appsv1alpha1.RunningClusterCompPhase - } - c.Phase = p - cluster.Status.SetComponentStatus(componentName, c) - component.Phase = p -} - -// isExpansionCompleted check the expansion is completed -func (ve volumeExpansionOpsHandler) isExpansionCompleted(phase appsv1alpha1.ProgressStatus) bool { - return slices.Contains([]appsv1alpha1.ProgressStatus{appsv1alpha1.FailedProgressStatus, - appsv1alpha1.SucceedProgressStatus}, phase) -} - // patchClusterStatus patch cluster status func (ve volumeExpansionOpsHandler) patchClusterStatus(reqCtx intctrlutil.RequestCtx, cli client.Client, @@ -282,9 +237,7 @@ func (ve volumeExpansionOpsHandler) getRequestStorageMap(opsRequest *appsv1alpha func (ve volumeExpansionOpsHandler) initComponentStatus(opsRequest *appsv1alpha1.OpsRequest) { opsRequest.Status.Components = map[string]appsv1alpha1.OpsRequestComponentStatus{} for _, v := range opsRequest.Spec.VolumeExpansionList { - opsRequest.Status.Components[v.ComponentName] = appsv1alpha1.OpsRequestComponentStatus{ - Phase: appsv1alpha1.SpecReconcilingClusterCompPhase, - } + opsRequest.Status.Components[v.ComponentName] = appsv1alpha1.OpsRequestComponentStatus{} } } @@ -294,52 +247,59 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti opsRes *OpsResource, compStatus *appsv1alpha1.OpsRequestComponentStatus, storageMap map[string]resource.Quantity, - componentName, vctName string) (succeedCount int, expectCount int, isCompleted bool, err error) { + componentName, vctName string) (int, int, int, error) { + var ( + succeedCount int + expectCount int + completedCount int + err error + ) pvcList := &corev1.PersistentVolumeClaimList{} if err = cli.List(reqCtx.Ctx, pvcList, client.MatchingLabels{ constant.AppInstanceLabelKey: opsRes.Cluster.Name, constant.KBAppComponentLabelKey: componentName, constant.VolumeClaimTemplateNameLabelKey: vctName, }, client.InNamespace(opsRes.Cluster.Namespace)); err != nil { - return + return 0, 0, 0, err } comp := opsRes.Cluster.Spec.GetComponentByName(componentName) if comp == nil { err = fmt.Errorf("comp %s of cluster %s not found", componentName, opsRes.Cluster.Name) - return + return 0, 0, 0, err } expectCount = int(comp.Replicas) vctKey := getComponentVCTKey(componentName, vctName) requestStorage := storageMap[vctKey] - var completedCount int var ordinal int for _, v := range pvcList.Items { // filter PVC(s) with ordinal larger than comp.Replicas - 1, which left by scale-in ordinal, err = getPVCOrdinal(v.Name) if err != nil { - return + return 0, 0, 0, err } if ordinal > expectCount-1 { continue } objectKey := getPVCProgressObjectKey(v.Name) - progressDetail := appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey, Group: vctName} + progressDetail := findStatusProgressDetail(compStatus.ProgressDetails, objectKey) + if progressDetail == nil { + progressDetail = &appsv1alpha1.ProgressStatusDetail{ObjectKey: objectKey, Group: vctName} + } + if progressDetail.Status == appsv1alpha1.FailedProgressStatus { + completedCount += 1 + continue + } currStorageSize := v.Status.Capacity.Storage() - // if the volume expand succeed - if currStorageSize.Cmp(requestStorage) == 0 { + // should check if the spec.resources.requests.storage equals to the requested storage + // and pvc is bound if the pvc is re-created for recovery. + if currStorageSize.Cmp(requestStorage) == 0 && + v.Spec.Resources.Requests.Storage().Cmp(requestStorage) == 0 && + v.Status.Phase == corev1.ClaimBound { succeedCount += 1 completedCount += 1 message := fmt.Sprintf("Successfully expand volume: %s in Component: %s", objectKey, componentName) progressDetail.SetStatusAndMessage(appsv1alpha1.SucceedProgressStatus, message) - setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) - continue - } - if currStorageSize.Cmp(requestStorage) > 0 { - completedCount += 1 - message := fmt.Sprintf("requested storage size of %s can not less than current storage size: %s", - objectKey, currStorageSize.String()) - progressDetail.SetStatusAndMessage(appsv1alpha1.FailedProgressStatus, message) - setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, *progressDetail) continue } if ve.pvcIsResizing(&v) { @@ -349,13 +309,9 @@ func (ve volumeExpansionOpsHandler) handleVCTExpansionProgress(reqCtx intctrluti message := fmt.Sprintf("Waiting for an external controller to process the pvc: %s in Component: %s ", objectKey, componentName) progressDetail.SetStatusAndMessage(appsv1alpha1.PendingProgressStatus, message) } - setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, progressDetail) - if ve.isExpansionCompleted(progressDetail.Status) { - completedCount += 1 - } + setComponentStatusProgressDetail(opsRes.Recorder, opsRes.OpsRequest, &compStatus.ProgressDetails, *progressDetail) } - isCompleted = completedCount == expectCount - return succeedCount, expectCount, isCompleted, nil + return succeedCount, expectCount, completedCount, nil } func getComponentVCTKey(componentName, vctName string) string { diff --git a/controllers/apps/operations/volume_expansion_test.go b/controllers/apps/operations/volume_expansion_test.go index 2004e2a04..43ae83c3c 100644 --- a/controllers/apps/operations/volume_expansion_test.go +++ b/controllers/apps/operations/volume_expansion_test.go @@ -20,7 +20,6 @@ along with this program. If not, see . package operations import ( - "fmt" "time" . "github.com/onsi/ginkgo/v2" @@ -83,23 +82,21 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { createPVC := func(clusterName, scName, vctName, pvcName string) { // Note: in real k8s cluster, it maybe fails when pvc created by k8s controller. testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterName, - consensusCompName, "data").SetStorage("2Gi").SetStorageClass(storageClassName).CheckedCreate(&testCtx) - } - - mockDoOperationOnCluster := func(cluster *appsv1alpha1.Cluster, opsRequestName string, opsType appsv1alpha1.OpsType) { - Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(cluster), func(tmpCluster *appsv1alpha1.Cluster) { - if tmpCluster.Annotations == nil { - tmpCluster.Annotations = map[string]string{} - } - tmpCluster.Annotations[constant.OpsRequestAnnotationKey] = fmt.Sprintf(`[{"type": "%s", "name":"%s"}]`, opsType, opsRequestName) - })()).ShouldNot(HaveOccurred()) - - Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(cluster), func(g Gomega, myCluster *appsv1alpha1.Cluster) { - g.Expect(getOpsRequestNameFromAnnotation(myCluster, appsv1alpha1.VolumeExpansionType)).ShouldNot(BeNil()) - })).Should(Succeed()) + consensusCompName, testapps.DataVolumeName).AddLabels(constant.AppInstanceLabelKey, clusterName, + constant.VolumeClaimTemplateNameLabelKey, testapps.DataVolumeName, + constant.KBAppComponentLabelKey, consensusCompName).SetStorage("2Gi").SetStorageClass(storageClassName).CheckedCreate(&testCtx) } initResourcesForVolumeExpansion := func(clusterObject *appsv1alpha1.Cluster, opsRes *OpsResource, storage string, replicas int) (*appsv1alpha1.OpsRequest, []string) { + pvcNames := opsRes.Cluster.GetVolumeClaimNames(consensusCompName) + for _, pvcName := range pvcNames { + createPVC(clusterObject.Name, storageClassName, vctName, pvcName) + // mock pvc is Bound + Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace}, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Status.Phase = corev1.ClaimBound + })()).ShouldNot(HaveOccurred()) + + } currRandomStr := testCtx.GetRandomStr() ops := testapps.NewOpsRequestObj("volumeexpansion-ops-"+currRandomStr, testCtx.DefaultNamespace, clusterObject.Name, appsv1alpha1.VolumeExpansionType) @@ -118,42 +115,35 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { // create opsRequest ops = testapps.CreateOpsRequest(ctx, testCtx, ops) + return ops, pvcNames + } - By("mock do operation on cluster") - mockDoOperationOnCluster(clusterObject, ops.Name, appsv1alpha1.VolumeExpansionType) + mockVolumeExpansionActionAndReconcile := func(reqCtx intctrlutil.RequestCtx, opsRes *OpsResource, newOps *appsv1alpha1.OpsRequest, pvcNames []string) { + // first step, validate ops and update phase to Creating + _, err := GetOpsManager().Do(reqCtx, k8sClient, opsRes) + Expect(err).Should(BeNil()) - // create-pvc - pvcNames := opsRes.Cluster.GetVolumeClaimNames(consensusCompName) + // next step, do volume-expand action + _, err = GetOpsManager().Do(reqCtx, k8sClient, opsRes) + Expect(err).Should(BeNil()) + + By("mock pvc.spec.resources.request.storage has applied by cluster controller") for _, pvcName := range pvcNames { - createPVC(clusterObject.Name, storageClassName, vctName, pvcName) - // trigger pvc controller reconcile if pvc already exists - pvcKey := client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace} - Expect(testapps.GetAndChangeObj(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { - pvc.Labels[testCtx.GetRandomStr()] = "trigger-reconcile" + Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKey{Name: pvcName, Namespace: testCtx.DefaultNamespace}, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = newOps.Spec.VolumeExpansionList[0].VolumeClaimTemplates[0].Storage })()).ShouldNot(HaveOccurred()) } - // waiting pvc controller mark annotation to OpsRequest - Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(ops), func(g Gomega, tmpOps *appsv1alpha1.OpsRequest) { - g.Expect(tmpOps.Annotations).ShouldNot(BeNil()) - g.Expect(tmpOps.Annotations[constant.ReconcileAnnotationKey]).ShouldNot(BeEmpty()) - })).Should(Succeed()) - - return ops, pvcNames - } - mockVolumeExpansionActionAndReconcile := func(reqCtx intctrlutil.RequestCtx, opsRes *OpsResource, newOps *appsv1alpha1.OpsRequest) { - Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(newOps), func(newOps *appsv1alpha1.OpsRequest) { - _, _ = GetOpsManager().Do(reqCtx, k8sClient, opsRes) + By("mock opsRequest is Running") + Expect(testapps.ChangeObjStatus(&testCtx, newOps, func() { newOps.Status.Phase = appsv1alpha1.OpsRunningPhase newOps.Status.StartTimestamp = metav1.Time{Time: time.Now()} - })()).ShouldNot(HaveOccurred()) + })).ShouldNot(HaveOccurred()) - // do volume-expand action - _, _ = GetOpsManager().Do(reqCtx, k8sClient, opsRes) + // reconcile ops status opsRes.OpsRequest = newOps - _, err := GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) - Expect(err == nil).Should(BeTrue()) - Eventually(testapps.GetOpsRequestCompPhase(ctx, testCtx, newOps.Name, consensusCompName)).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) + _, err = GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) + Expect(err).Should(BeNil()) } testWarningEventOnPVC := func(reqCtx intctrlutil.RequestCtx, clusterObject *appsv1alpha1.Cluster, opsRes *OpsResource) { @@ -162,7 +152,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { newOps, pvcNames := initResourcesForVolumeExpansion(clusterObject, opsRes, "4Gi", int(comp.Replicas)) By("mock run volumeExpansion action and reconcileAction") - mockVolumeExpansionActionAndReconcile(reqCtx, opsRes, newOps) + mockVolumeExpansionActionAndReconcile(reqCtx, opsRes, newOps, pvcNames) By("test warning event and volumeExpansion failed") // test when the event does not reach the conditions @@ -180,9 +170,8 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { event.InvolvedObject = stsInvolvedObject pvcEventHandler := PersistentVolumeClaimEventHandler{} Expect(pvcEventHandler.Handle(k8sClient, reqCtx, eventRecorder, event)).Should(Succeed()) - Eventually(testapps.GetOpsRequestCompPhase(ctx, testCtx, newOps.Name, consensusCompName)).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) - // test when the event reach the conditions + // test when the event reaches the conditions event.Count = 5 event.FirstTimestamp = metav1.Time{Time: time.Now()} event.LastTimestamp = metav1.Time{Time: time.Now().Add(61 * time.Second)} @@ -206,7 +195,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { newOps, pvcNames := initResourcesForVolumeExpansion(clusterObject, opsRes, "3Gi", int(comp.Replicas)) By("mock run volumeExpansion action and reconcileAction") - mockVolumeExpansionActionAndReconcile(reqCtx, opsRes, newOps) + mockVolumeExpansionActionAndReconcile(reqCtx, opsRes, newOps, pvcNames) By("mock pvc is resizing") for _, pvcName := range pvcNames { @@ -218,6 +207,7 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { LastTransitionTime: metav1.Now(), }, } + pvc.Status.Phase = corev1.ClaimBound })()).ShouldNot(HaveOccurred()) Eventually(testapps.CheckObj(&testCtx, pvcKey, func(g Gomega, tmpPVC *corev1.PersistentVolumeClaim) { @@ -244,20 +234,15 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { })).Should(Succeed()) } - // waiting OpsRequest.status.phase is succeed + // waiting for OpsRequest.status.phase is succeed _, err := GetOpsManager().Reconcile(reqCtx, k8sClient, opsRes) - Expect(err == nil).Should(BeTrue()) - Expect(opsRes.OpsRequest.Status.Phase == appsv1alpha1.OpsSucceedPhase).Should(BeTrue()) - - testWarningEventOnPVC(reqCtx, clusterObject, opsRes) + Expect(err).Should(BeNil()) + Expect(opsRes.OpsRequest.Status.Phase).Should(Equal(appsv1alpha1.OpsSucceedPhase)) } testDeleteRunningVolumeExpansion := func(clusterObject *appsv1alpha1.Cluster, opsRes *OpsResource) { // init resources for volume expansion newOps, pvcNames := initResourcesForVolumeExpansion(clusterObject, opsRes, "5Gi", 1) - Expect(testapps.ChangeObjStatus(&testCtx, clusterObject, func() { - clusterObject.Status.Phase = appsv1alpha1.SpecReconcilingClusterPhase - })).ShouldNot(HaveOccurred()) Expect(k8sClient.Delete(ctx, newOps)).Should(Succeed()) Eventually(func() error { return k8sClient.Get(ctx, client.ObjectKey{Name: newOps.Name, Namespace: testCtx.DefaultNamespace}, &appsv1alpha1.OpsRequest{}) @@ -300,6 +285,9 @@ var _ = Describe("OpsRequest Controller Volume Expansion Handler", func() { By("Test VolumeExpansion") testVolumeExpansion(reqCtx, clusterObject, opsRes, randomStr) + By("Test Warning Event occurs during volume expanding") + testWarningEventOnPVC(reqCtx, clusterObject, opsRes) + By("Test delete the Running VolumeExpansion OpsRequest") testDeleteRunningVolumeExpansion(clusterObject, opsRes) }) diff --git a/controllers/apps/operations/volume_expansion_updater.go b/controllers/apps/operations/volume_expansion_updater.go index 3c3a3a066..dcff7e48a 100644 --- a/controllers/apps/operations/volume_expansion_updater.go +++ b/controllers/apps/operations/volume_expansion_updater.go @@ -20,12 +20,10 @@ along with this program. If not, see . package operations import ( - "context" "time" "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" @@ -57,51 +55,21 @@ func init() { // handleVolumeExpansionOperation handles the pvc for the volume expansion OpsRequest. // it will be triggered when the PersistentVolumeClaim has changed. func handleVolumeExpansionWithPVC(reqCtx intctrlutil.RequestCtx, cli client.Client, pvc *corev1.PersistentVolumeClaim) error { - clusterName := pvc.Labels[constant.AppInstanceLabelKey] - cluster := &appsv1alpha1.Cluster{} - if err := cli.Get(reqCtx.Ctx, client.ObjectKey{Name: clusterName, Namespace: pvc.Namespace}, cluster); err != nil { + opsRequestList, err := appsv1alpha1.GetRunningOpsByOpsType(reqCtx.Ctx, cli, + pvc.Labels[constant.AppInstanceLabelKey], pvc.Namespace, string(appsv1alpha1.VolumeExpansionType)) + if err != nil { return err } - // check whether the cluster is expanding volume - opsRequestName := getOpsRequestNameFromAnnotation(cluster, appsv1alpha1.VolumeExpansionType) - if opsRequestName == nil { + if len(opsRequestList) == 0 { return nil } // notice the OpsRequest to reconcile - err := opsutil.PatchOpsRequestReconcileAnnotation(reqCtx.Ctx, cli, cluster.Namespace, *opsRequestName) - // if the OpsRequest is not found, means it is deleted by user. - // we should delete the invalid OpsRequest annotation in the cluster and reconcile the cluster phase. - if apierrors.IsNotFound(err) { - opsRequestSlice, _ := opsutil.GetOpsRequestSliceFromCluster(cluster) - notExistOps := map[string]struct{}{ - *opsRequestName: {}, - } - if err = opsutil.RemoveClusterInvalidOpsRequestAnnotation(reqCtx.Ctx, cli, cluster, - opsRequestSlice, notExistOps); err != nil { + for _, ops := range opsRequestList { + if err = opsutil.PatchOpsRequestReconcileAnnotation(reqCtx.Ctx, cli, pvc.Namespace, ops.Name); err != nil { return err } - return handleClusterVolumeExpandingPhase(reqCtx.Ctx, cli, cluster) - } - return err -} - -// handleClusterVolumeExpandingPhase this function will reconcile the cluster status phase when the OpsRequest is deleted. -func handleClusterVolumeExpandingPhase(ctx context.Context, - cli client.Client, - cluster *appsv1alpha1.Cluster) error { - if cluster.Status.Phase != appsv1alpha1.SpecReconcilingClusterPhase { - return nil - } - patch := client.MergeFrom(cluster.DeepCopy()) - for k, v := range cluster.Status.Components { - if v.Phase == appsv1alpha1.SpecReconcilingClusterCompPhase { - v.Phase = appsv1alpha1.RunningClusterCompPhase - cluster.Status.SetComponentStatus(k, v) - } } - // REVIEW: a single component status affect cluster level status? - cluster.Status.Phase = appsv1alpha1.RunningClusterPhase - return cli.Status().Patch(ctx, cluster, patch) + return nil } // Handle the warning events on pvcs. if the events are resize failed events, update the OpsRequest.status. @@ -133,7 +101,7 @@ func (pvcEventHandler PersistentVolumeClaimEventHandler) Handle(cli client.Clien } // here, if the volume expansion ops is running. we will change the pvc status to Failed on the OpsRequest. - return pvcEventHandler.handlePVCFailedStatusOnOpsRequest(cli, reqCtx, recorder, event, pvc) + return pvcEventHandler.handlePVCFailedStatusOnRunningOpsRequests(cli, reqCtx, recorder, event, pvc) } // isTargetResizeFailedEvents checks the event is the resize failed events. @@ -144,7 +112,7 @@ func (pvcEventHandler PersistentVolumeClaimEventHandler) isTargetResizeFailedEve } // handlePVCFailedStatusOnOpsRequest if the volume expansion ops is running. we will change the pvc status to Failed on the OpsRequest, -func (pvcEventHandler PersistentVolumeClaimEventHandler) handlePVCFailedStatusOnOpsRequest(cli client.Client, +func (pvcEventHandler PersistentVolumeClaimEventHandler) handlePVCFailedStatusOnRunningOpsRequests(cli client.Client, reqCtx intctrlutil.RequestCtx, recorder record.EventRecorder, event *corev1.Event, @@ -160,15 +128,28 @@ func (pvcEventHandler PersistentVolumeClaimEventHandler) handlePVCFailedStatusOn }, cluster); err != nil { return err } - // get the volume expansion ops which is running on cluster. - opsRequestName := getOpsRequestNameFromAnnotation(cluster, appsv1alpha1.VolumeExpansionType) - if opsRequestName == nil { + opsRequestList, err := appsv1alpha1.GetRunningOpsByOpsType(reqCtx.Ctx, cli, + pvc.Labels[constant.AppInstanceLabelKey], pvc.Namespace, string(appsv1alpha1.VolumeExpansionType)) + if err != nil { + return err + } + if len(opsRequestList) == 0 { return nil } - opsRequest := &appsv1alpha1.OpsRequest{} - if err = cli.Get(reqCtx.Ctx, client.ObjectKey{Name: *opsRequestName, Namespace: pvc.Namespace}, opsRequest); err != nil { - return err + for _, ops := range opsRequestList { + if err = pvcEventHandler.handlePVCFailedStatus(cli, reqCtx, recorder, event, pvc, &ops); err != nil { + return err + } } + return nil +} + +func (pvcEventHandler PersistentVolumeClaimEventHandler) handlePVCFailedStatus(cli client.Client, + reqCtx intctrlutil.RequestCtx, + recorder record.EventRecorder, + event *corev1.Event, + pvc *corev1.PersistentVolumeClaim, + opsRequest *appsv1alpha1.OpsRequest) error { compsStatus := opsRequest.Status.Components if compsStatus == nil { return nil @@ -202,7 +183,7 @@ func (pvcEventHandler PersistentVolumeClaimEventHandler) handlePVCFailedStatusOn if !isChanged { return nil } - if err = cli.Status().Patch(reqCtx.Ctx, opsRequest, patch); err != nil { + if err := cli.Status().Patch(reqCtx.Ctx, opsRequest, patch); err != nil { return err } recorder.Event(opsRequest, corev1.EventTypeWarning, event.Reason, event.Message) diff --git a/controllers/apps/opsrequest_controller.go b/controllers/apps/opsrequest_controller.go index bb1879d6e..fd263b248 100644 --- a/controllers/apps/opsrequest_controller.go +++ b/controllers/apps/opsrequest_controller.go @@ -182,7 +182,8 @@ func (r *OpsRequestReconciler) addClusterLabelAndSetOwnerReference(reqCtx intctr // add label of clusterRef opsRequest := opsRes.OpsRequest clusterName := opsRequest.Labels[constant.AppInstanceLabelKey] - if clusterName == opsRequest.Spec.ClusterRef { + opsType := opsRequest.Labels[constant.OpsRequestTypeLabelKey] + if clusterName == opsRequest.Spec.ClusterRef && opsType == string(opsRequest.Spec.Type) { return nil, nil } patch := client.MergeFrom(opsRequest.DeepCopy()) @@ -190,6 +191,7 @@ func (r *OpsRequestReconciler) addClusterLabelAndSetOwnerReference(reqCtx intctr opsRequest.Labels = map[string]string{} } opsRequest.Labels[constant.AppInstanceLabelKey] = opsRequest.Spec.ClusterRef + opsRequest.Labels[constant.OpsRequestTypeLabelKey] = string(opsRequest.Spec.Type) scheme, _ := appsv1alpha1.SchemeBuilder.Build() if err := controllerutil.SetOwnerReference(opsRes.Cluster, opsRequest, scheme); err != nil { return intctrlutil.ResultToP(intctrlutil.CheckedRequeueWithError(err, reqCtx.Log, "")) diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index 473dd75eb..6ee07d59e 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -375,7 +375,7 @@ var _ = Describe("OpsRequest Controller", func() { for i := 0; i < int(replicas); i++ { pvcName := fmt.Sprintf("%s-%s-%s-%d", testapps.DataVolumeName, clusterKey.Name, mysqlCompName, i) pvc := testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterKey.Name, - mysqlCompName, "data").SetStorage("1Gi").Create(&testCtx).GetObject() + mysqlCompName, testapps.DataVolumeName).SetStorage("1Gi").Create(&testCtx).GetObject() // mock pvc bound Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(pvc), func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimBound @@ -527,6 +527,15 @@ var _ = Describe("OpsRequest Controller", func() { Create(&testCtx).GetObject() // mock sts ready and create pod createStsPodAndMockStsReady() + // mock pvc creation + for i := 0; i < testapps.DefaultReplicationReplicas; i++ { + pvcName := fmt.Sprintf("%s-%s-%s-%d", testapps.DataVolumeName, clusterObj.Name, mysqlCompName, i) + testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterObj.Name, + mysqlCompName, testapps.DataVolumeName).AddLabels(constant.AppInstanceLabelKey, clusterObj.Name, + constant.VolumeClaimTemplateNameLabelKey, testapps.DataVolumeName, + constant.KBAppComponentLabelKey, testapps.DefaultRedisCompName). + SetStorage("1Gi").SetStorageClass(storageClassName).Create(&testCtx).GetObject() + } // wait for cluster to running Eventually(testapps.GetClusterPhase(&testCtx, client.ObjectKeyFromObject(clusterObj))). Should(Equal(appsv1alpha1.RunningClusterPhase)) @@ -552,12 +561,6 @@ var _ = Describe("OpsRequest Controller", func() { clusterKey = client.ObjectKeyFromObject(clusterObj) opsKey := client.ObjectKeyFromObject(volumeExpandOps) Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) - Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, tmlCluster *appsv1alpha1.Cluster) { - opsSlice, _ := opsutil.GetOpsRequestSliceFromCluster(tmlCluster) - g.Expect(opsSlice).Should(HaveLen(1)) - g.Expect(tmlCluster.Status.Components[testapps.DefaultRedisCompName].Phase).Should(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) // VolumeExpandingPhase - // TODO: status conditions for VolumeExpandingPhase - })).Should(Succeed()) By("delete the Running ops") testapps.DeleteObject(&testCtx, opsKey, volumeExpandOps) diff --git a/internal/cli/cmd/cluster/config_ops.go b/internal/cli/cmd/cluster/config_ops.go index 4f7008dd4..2ef46b89c 100644 --- a/internal/cli/cmd/cluster/config_ops.go +++ b/internal/cli/cmd/cluster/config_ops.go @@ -217,6 +217,7 @@ func NewReconfigureCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) * ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.CreateOptions.Complete()) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.Validate()) diff --git a/internal/cli/cmd/cluster/operations.go b/internal/cli/cmd/cluster/operations.go index f077075a7..f96249592 100755 --- a/internal/cli/cmd/cluster/operations.go +++ b/internal/cli/cmd/cluster/operations.go @@ -42,6 +42,7 @@ import ( "github.com/apecloud/kubeblocks/internal/cli/printer" "github.com/apecloud/kubeblocks/internal/cli/types" "github.com/apecloud/kubeblocks/internal/cli/util" + "github.com/apecloud/kubeblocks/internal/constant" ) type OperationsOptions struct { @@ -182,6 +183,34 @@ func (o *OperationsOptions) validateVolumeExpansion() error { if len(o.Storage) == 0 { return fmt.Errorf("missing storage") } + for _, cName := range o.ComponentNames { + for _, vctName := range o.VCTNames { + labels := fmt.Sprintf("%s=%s,%s=%s,%s=%s", + constant.AppInstanceLabelKey, o.Name, + constant.KBAppComponentLabelKey, cName, + constant.VolumeClaimTemplateNameLabelKey, vctName, + ) + pvcs, err := o.Client.CoreV1().PersistentVolumeClaims(o.Namespace).List(context.Background(), + metav1.ListOptions{LabelSelector: labels, Limit: 1}) + if err != nil { + return err + } + if len(pvcs.Items) == 0 { + continue + } + pvc := pvcs.Items[0] + specStorage := pvc.Spec.Resources.Requests.Storage() + statusStorage := pvc.Status.Capacity.Storage() + targetStorage := resource.MustParse(o.Storage) + // determine whether the opsRequest is a recovery action for volume expansion failure + if specStorage.Cmp(targetStorage) > 0 && + statusStorage.Cmp(targetStorage) <= 0 { + o.autoApprove = false + fmt.Fprintln(o.Out, printer.BoldYellow("Warning: this opsRequest is a recovery action for volume expansion failure and will re-create the PersistentVolumeClaims when RECOVER_VOLUME_EXPANSION_FAILURE=false")) + break + } + } + } return nil } @@ -252,19 +281,19 @@ func (o *OperationsOptions) Validate() error { switch o.OpsType { case appsv1alpha1.VolumeExpansionType: - if err := o.validateVolumeExpansion(); err != nil { + if err = o.validateVolumeExpansion(); err != nil { return err } case appsv1alpha1.UpgradeType: - if err := o.validateUpgrade(); err != nil { + if err = o.validateUpgrade(); err != nil { return err } case appsv1alpha1.VerticalScalingType: - if err := o.validateVScale(&cluster); err != nil { + if err = o.validateVScale(&cluster); err != nil { return err } case appsv1alpha1.ExposeType: - if err := o.validateExpose(); err != nil { + if err = o.validateExpose(); err != nil { return err } } @@ -378,6 +407,7 @@ func NewRestartCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.CompleteRestartOps()) cmdutil.CheckErr(o.Validate()) @@ -404,6 +434,7 @@ func NewUpgradeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobr ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.Run()) @@ -433,6 +464,7 @@ func NewVerticalScalingCmd(f cmdutil.Factory, streams genericclioptions.IOStream ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.CompleteComponentsFlag()) cmdutil.CheckErr(o.Validate()) @@ -462,6 +494,7 @@ func NewHorizontalScalingCmd(f cmdutil.Factory, streams genericclioptions.IOStre ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.CompleteComponentsFlag()) cmdutil.CheckErr(o.Validate()) @@ -491,6 +524,7 @@ func NewVolumeExpansionCmd(f cmdutil.Factory, streams genericclioptions.IOStream ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.CompleteComponentsFlag()) cmdutil.CheckErr(o.Validate()) @@ -527,6 +561,7 @@ func NewExposeCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.fillExpose()) cmdutil.CheckErr(o.Validate()) @@ -564,6 +599,7 @@ func NewStopCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.C ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.Run()) @@ -590,6 +626,7 @@ func NewStartCmd(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra. ValidArgsFunction: util.ResourceNameCompletionFunc(f, types.ClusterGVR()), Run: func(cmd *cobra.Command, args []string) { o.Args = args + cmdutil.BehaviorOnFatal(printer.FatalWithRedColor) cmdutil.CheckErr(o.Complete()) cmdutil.CheckErr(o.Validate()) cmdutil.CheckErr(o.Run()) diff --git a/internal/cli/cmd/cluster/operations_test.go b/internal/cli/cmd/cluster/operations_test.go index 5724af923..0307acb40 100644 --- a/internal/cli/cmd/cluster/operations_test.go +++ b/internal/cli/cmd/cluster/operations_test.go @@ -21,15 +21,21 @@ package cluster import ( "bytes" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" clientfake "k8s.io/client-go/rest/fake" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/cli/testing" + "github.com/apecloud/kubeblocks/internal/constant" testapps "github.com/apecloud/kubeblocks/internal/testutil/apps" ) @@ -66,9 +72,10 @@ var _ = Describe("operations", func() { tf.Cleanup() }) - initCommonOperationOps := func(opsType appsv1alpha1.OpsType, clusterName string, hasComponentNamesFlag bool) *OperationsOptions { + initCommonOperationOps := func(opsType appsv1alpha1.OpsType, clusterName string, hasComponentNamesFlag bool, objs ...runtime.Object) *OperationsOptions { o := newBaseOperationsOptions(tf, streams, opsType, hasComponentNamesFlag) o.Dynamic = tf.FakeDynamicClient + o.Client = testing.FakeClientSet(objs...) o.Name = clusterName o.Namespace = testing.Namespace return o @@ -94,18 +101,53 @@ var _ = Describe("operations", func() { }) It("VolumeExpand Ops", func() { - o := initCommonOperationOps(appsv1alpha1.VolumeExpansionType, clusterName, true) + compName := "replicasets" + vctName := "data" + persistentVolumeClaim := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-%s-%d", vctName, clusterName, compName, 0), + Namespace: testing.Namespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: clusterName, + constant.VolumeClaimTemplateNameLabelKey: vctName, + constant.KBAppComponentLabelKey: compName, + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + "storage": resource.MustParse("3Gi"), + }, + }, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Capacity: map[corev1.ResourceName]resource.Quantity{ + "storage": resource.MustParse("1Gi"), + }, + }, + } + o := initCommonOperationOps(appsv1alpha1.VolumeExpansionType, clusterName, true, persistentVolumeClaim) By("validate volumeExpansion when components is null") Expect(o.Validate()).To(MatchError(`missing components, please specify the "--components" flag for multi-components cluster`)) By("validate volumeExpansion when vct-names is null") - o.ComponentNames = []string{"replicasets"} + o.ComponentNames = []string{compName} Expect(o.Validate()).To(MatchError("missing volume-claim-templates")) By("validate volumeExpansion when storage is null") - o.VCTNames = []string{"data"} + o.VCTNames = []string{vctName} Expect(o.Validate()).To(MatchError("missing storage")) + + By("validate recovery from volume expansion failure") o.Storage = "2Gi" + Expect(o.Validate()).Should(Succeed()) + Expect(o.Out.(*bytes.Buffer).String()).To(ContainSubstring("Warning: this opsRequest is a recovery action for volume expansion failure and will re-create the PersistentVolumeClaims when RECOVER_VOLUME_EXPANSION_FAILURE=false")) + + By("validate passed") + o.Storage = "4Gi" in.Write([]byte(o.Name + "\n")) Expect(o.Validate()).Should(Succeed()) }) diff --git a/internal/cli/printer/format.go b/internal/cli/printer/format.go index fbb56db4b..791fdf6a8 100755 --- a/internal/cli/printer/format.go +++ b/internal/cli/printer/format.go @@ -21,9 +21,11 @@ package printer import ( "fmt" + "os" "strings" "github.com/spf13/cobra" + "k8s.io/klog/v2" "k8s.io/kubectl/pkg/cmd/util" ) @@ -118,3 +120,18 @@ func (o *outputValue) Set(s string) error { *o = outputValue(outfmt) return nil } + +// FatalWithRedColor when an error occurs, set the red color to print it. +func FatalWithRedColor(msg string, code int) { + if klog.V(99).Enabled() { + klog.FatalDepth(2, msg) + } + if len(msg) > 0 { + // add newline if needed + if !strings.HasSuffix(msg, "\n") { + msg += "\n" + } + fmt.Fprint(os.Stderr, BoldRed(msg)) + } + os.Exit(code) +} diff --git a/internal/constant/const.go b/internal/constant/const.go index 5843ca58a..5f72f61f3 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -92,6 +92,7 @@ const ( BackupToolTypeLabelKey = "kubeblocks.io/backup-tool-type" BackupTypeLabelKeyKey = "dataprotection.kubeblocks.io/backup-type" AddonProviderLableKey = "kubeblocks.io/provider" // AddonProviderLableKey marks the addon provider + OpsRequestTypeLabelKey = "ops.kubeblocks.io/ops-type" // kubeblocks.io annotations OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster diff --git a/internal/controller/lifecycle/transformer_cluster_status.go b/internal/controller/lifecycle/transformer_cluster_status.go index a4778baae..7ac9421e6 100644 --- a/internal/controller/lifecycle/transformer_cluster_status.go +++ b/internal/controller/lifecycle/transformer_cluster_status.go @@ -354,7 +354,7 @@ func (t *ClusterStatusTransformer) handleExistAbnormalOrFailed(transCtx *Cluster componentMap, clusterAvailabilityEffectMap, _ := getComponentRelatedInfo(cluster, *transCtx.ClusterDef, "") // handle the cluster status when some components are not ready. - handleClusterPhaseWhenCompsNotReady(cluster, componentMap, clusterAvailabilityEffectMap) + HandleClusterPhaseWhenCompsNotReady(cluster, componentMap, clusterAvailabilityEffectMap) } // cleanupAnnotationsAfterRunning cleans up the cluster annotations after cluster is Running. @@ -456,27 +456,17 @@ func (t *ClusterStatusTransformer) removeStsInitContainerForRestore( return doRemoveInitContainers, nil } -// handleClusterPhaseWhenCompsNotReady handles the Cluster.status.phase when some components are Abnormal or Failed. -// REVIEW: seem duplicated handling -// Deprecated: -func handleClusterPhaseWhenCompsNotReady(cluster *appsv1alpha1.Cluster, +// HandleClusterPhaseWhenCompsNotReady handles the Cluster.status.phase when some components are Abnormal or Failed. +func HandleClusterPhaseWhenCompsNotReady(cluster *appsv1alpha1.Cluster, componentMap map[string]string, clusterAvailabilityEffectMap map[string]bool) { var ( - clusterIsFailed bool - failedCompCount int - isVolumeExpanding bool + clusterIsFailed bool + failedCompCount int ) - opsRecords, _ := opsutil.GetOpsRequestSliceFromCluster(cluster) - if len(opsRecords) != 0 && opsRecords[0].Type == appsv1alpha1.VolumeExpansionType { - isVolumeExpanding = true - } for k, v := range cluster.Status.Components { - // determine whether other components are still doing operation, i.e., create/restart/scaling. - // waiting for operation to complete except for volumeExpansion operation. - // because this operation will not affect cluster availability. - if !slices.Contains(appsv1alpha1.GetComponentTerminalPhases(), v.Phase) && !isVolumeExpanding { + if !slices.Contains(appsv1alpha1.GetComponentTerminalPhases(), v.Phase) { return } if v.Phase == appsv1alpha1.FailedClusterCompPhase { diff --git a/internal/testutil/apps/opsrequest_util.go b/internal/testutil/apps/opsrequest_util.go index 29a460baa..c32391e02 100644 --- a/internal/testutil/apps/opsrequest_util.go +++ b/internal/testutil/apps/opsrequest_util.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/testutil" ) @@ -48,6 +49,10 @@ func NewOpsRequestObj(opsRequestName, namespace, clusterName string, opsType app ObjectMeta: metav1.ObjectMeta{ Name: opsRequestName, Namespace: namespace, + Labels: map[string]string{ + constant.AppInstanceLabelKey: clusterName, + constant.OpsRequestTypeLabelKey: string(opsType), + }, }, Spec: appsv1alpha1.OpsRequestSpec{ ClusterRef: clusterName, From fed16c6621f5cf1309ff67048e697c273289cfca Mon Sep 17 00:00:00 2001 From: demian0110 Date: Wed, 24 May 2023 11:37:57 +0800 Subject: [PATCH 343/439] chore: change apecloud-mysql cv tag (#3409) --- deploy/apecloud-mysql/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/apecloud-mysql/values.yaml b/deploy/apecloud-mysql/values.yaml index 735d0987d..22efaf8f3 100644 --- a/deploy/apecloud-mysql/values.yaml +++ b/deploy/apecloud-mysql/values.yaml @@ -7,7 +7,7 @@ image: repository: apecloud/apecloud-mysql-server pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: 8.0.30-5.alpha5.20230319.g28f261a.5 + tag: 8.0.30-5.alpha8.20230523.g3e93ae7.5 ## MySQL Cluster parameters cluster: From e567c392121399ce89ce5672832ccf3ab7881784 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Wed, 24 May 2023 13:13:34 +0800 Subject: [PATCH 344/439] chore: cli remove unused comamnd and improve example (#3396) --- docs/user_docs/cli/cli.md | 5 +- docs/user_docs/cli/kbcli.md | 2 +- docs/user_docs/cli/kbcli_backup-config.md | 61 ----------------- docs/user_docs/cli/kbcli_cluster_configure.md | 65 ------------------- docs/user_docs/cli/kbcli_cluster_create.md | 13 +++- docs/user_docs/cli/kbcli_playground.md | 5 +- .../user_docs/cli/kbcli_playground_destroy.md | 6 +- docs/user_docs/cli/kbcli_playground_guide.md | 46 ------------- docs/user_docs/cli/kbcli_playground_init.md | 32 +++++++-- .../try-kubeblocks-in-5m-on-local-host.md | 2 +- internal/cli/cloudprovider/k3d.go | 2 +- internal/cli/cmd/cluster/create.go | 31 +++++++-- internal/cli/cmd/playground/destroy.go | 19 ++---- internal/cli/cmd/playground/init.go | 37 ++++++++--- internal/cli/cmd/playground/init_test.go | 5 -- internal/cli/cmd/playground/palyground.go | 5 +- 16 files changed, 109 insertions(+), 227 deletions(-) delete mode 100644 docs/user_docs/cli/kbcli_backup-config.md delete mode 100644 docs/user_docs/cli/kbcli_cluster_configure.md delete mode 100644 docs/user_docs/cli/kbcli_playground_guide.md diff --git a/docs/user_docs/cli/cli.md b/docs/user_docs/cli/cli.md index d4212b508..57cc14268 100644 --- a/docs/user_docs/cli/cli.md +++ b/docs/user_docs/cli/cli.md @@ -156,10 +156,9 @@ Print the list of flags inherited by all commands. ## [playground](kbcli_playground.md) -Bootstrap a playground KubeBlocks in local host or cloud. +Bootstrap or destroy a playground KubeBlocks in local host or cloud. -* [kbcli playground destroy](kbcli_playground_destroy.md) - Destroy the playground kubernetes cluster. -* [kbcli playground guide](kbcli_playground_guide.md) - Display playground cluster user guide. +* [kbcli playground destroy](kbcli_playground_destroy.md) - Destroy the playground KubeBlocks and kubernetes cluster. * [kbcli playground init](kbcli_playground_init.md) - Bootstrap a kubernetes cluster and install KubeBlocks for playground. diff --git a/docs/user_docs/cli/kbcli.md b/docs/user_docs/cli/kbcli.md index 65bba3c67..c5d70ea89 100644 --- a/docs/user_docs/cli/kbcli.md +++ b/docs/user_docs/cli/kbcli.md @@ -66,7 +66,7 @@ kbcli [flags] * [kbcli kubeblocks](kbcli_kubeblocks.md) - KubeBlocks operation commands. * [kbcli migration](kbcli_migration.md) - Data migration between two data sources. * [kbcli options](kbcli_options.md) - Print the list of flags inherited by all commands. -* [kbcli playground](kbcli_playground.md) - Bootstrap a playground KubeBlocks in local host or cloud. +* [kbcli playground](kbcli_playground.md) - Bootstrap or destroy a playground KubeBlocks in local host or cloud. * [kbcli plugin](kbcli_plugin.md) - Provides utilities for interacting with plugins. * [kbcli version](kbcli_version.md) - Print the version information, include kubernetes, KubeBlocks and kbcli version. diff --git a/docs/user_docs/cli/kbcli_backup-config.md b/docs/user_docs/cli/kbcli_backup-config.md deleted file mode 100644 index 9461f0c94..000000000 --- a/docs/user_docs/cli/kbcli_backup-config.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: kbcli backup-config ---- - -KubeBlocks backup config. - -``` -kbcli backup-config [flags] -``` - -### Examples - -``` - # Enable the snapshot-controller and volume snapshot, to support snapshot backup. - kbcli backup-config --set snapshot-controller.enabled=true - - # If you have already installed a snapshot-controller, only enable the snapshot backup feature - kbcli backup-config --set dataProtection.enableVolumeSnapshot=true -``` - -### Options - -``` - -h, --help help for backup-config - --set stringArray Set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) - --set-file stringArray Set values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2) - --set-json stringArray Set JSON values on the command line (can specify multiple or separate values with commas: key1=jsonval1,key2=jsonval2) - --set-string stringArray Set STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2) - -f, --values strings Specify values in a YAML file or a URL (can specify multiple) -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - - - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_cluster_configure.md b/docs/user_docs/cli/kbcli_cluster_configure.md deleted file mode 100644 index 9ef2591b8..000000000 --- a/docs/user_docs/cli/kbcli_cluster_configure.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -title: kbcli cluster configure ---- - -Reconfigure parameters with the specified components in the cluster. - -``` -kbcli cluster configure NAME --set key=value[,key=value] [--component=component-name] [--config-spec=config-spec-name] [--config-file=config-file] [flags] -``` - -### Examples - -``` - # update component params - kbcli cluster configure mycluster --component=mysql --config-spec=mysql-3node-tpl --config-file=my.cnf --set max_connections=1000,general_log=OFF - - # if only one component, and one config spec, and one config file, simplify the use of configure. e.g: - # update mysql max_connections, cluster name is mycluster - kbcli cluster configure mycluster --set max_connections=2000 -``` - -### Options - -``` - --component string Specify the name of Component to be updated. If the cluster has only one component, unset the parameter. - --config-file string Specify the name of the configuration file to be updated (e.g. for mysql: --config-file=my.cnf). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. - --config-spec string Specify the name of the configuration template to be updated (e.g. for apecloud-mysql: --config-spec=mysql-3node-tpl). What templates or configure files are available for this cluster can refer to kbcli sub command: 'kbcli cluster describe-config'. - --dry-run string[="unchanged"] Must be "server", or "client". If client strategy, only print the object that would be sent, without sending it. If server strategy, submit server-side request without persisting the resource. (default "none") - -h, --help help for configure - --name string OpsRequest name. if not specified, it will be randomly generated - -o, --output format prints the output in the specified format. Allowed values: JSON and YAML (default yaml) - --set strings Specify updated parameter list. For details about the parameters, refer to kbcli sub command: 'kbcli cluster describe-config'. - --ttlSecondsAfterSucceed int Time to live after the OpsRequest succeed -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli cluster](kbcli_cluster.md) - Cluster command. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_cluster_create.md b/docs/user_docs/cli/kbcli_cluster_create.md index 0dfc90c55..0ddb1a797 100644 --- a/docs/user_docs/cli/kbcli_cluster_create.md +++ b/docs/user_docs/cli/kbcli_cluster_create.md @@ -27,11 +27,11 @@ kbcli cluster create [NAME] [flags] # Create a cluster and set termination policy DoNotTerminate that will prevent the cluster from being deleted kbcli cluster create mycluster --cluster-definition apecloud-mysql --termination-policy DoNotTerminate - # In scenarios where you want to delete resources such as statements, deployments, services, pdb, but keep PVCs + # In scenarios where you want to delete resources such as statefulsets, deployments, services, pdb, but keep PVCs # when deleting the cluster, use termination policy Halt kbcli cluster create mycluster --cluster-definition apecloud-mysql --termination-policy Halt - # In scenarios where you want to delete resource such as statements, deployments, services, pdb, and including + # In scenarios where you want to delete resource such as statefulsets, deployments, services, pdb, and including # PVCs when deleting the cluster, use termination policy Delete kbcli cluster create mycluster --cluster-definition apecloud-mysql --termination-policy Delete @@ -42,6 +42,10 @@ kbcli cluster create [NAME] [flags] # Create a cluster and set cpu to 1 core, memory to 1Gi, storage size to 20Gi and replicas to 3 kbcli cluster create mycluster --cluster-definition apecloud-mysql --set cpu=1,memory=1Gi,storage=20Gi,replicas=3 + # Create a cluster and set storageClass to csi-hostpath-sc, if storageClass is not specified, + # the default storage class will be used + kbcli cluster create mycluster --cluster-definition apecloud-mysql --set storageClass=csi-hostpath-sc + # Create a cluster and set the class to general-1c1g # run "kbcli class list --cluster-definition=cluster-definition-name" to get the class list kbcli cluster create mycluster --cluster-definition apecloud-mysql --set class=general-1c1g @@ -49,6 +53,11 @@ kbcli cluster create [NAME] [flags] # Create a cluster with replicationSet workloadType and set switchPolicy to Noop kbcli cluster create mycluster --cluster-definition postgresql --set switchPolicy=Noop + # Create a cluster with more than one component, use "--set type=component-name" to specify the component, + # if not specified, the main component will be used, run "kbcli cd list-components CLUSTER-DEFINITION-NAME" + # to show the components in the cluster definition + kbcli cluster create mycluster --cluster-definition redis --set type=redis,cpu=1 --set type=redis-sentinel,cpu=200m + # Create a cluster and use a URL to set cluster resource kbcli cluster create mycluster --cluster-definition apecloud-mysql \ --set-file https://kubeblocks.io/yamls/apecloud-mysql.yaml diff --git a/docs/user_docs/cli/kbcli_playground.md b/docs/user_docs/cli/kbcli_playground.md index 3d4823619..c6414a799 100644 --- a/docs/user_docs/cli/kbcli_playground.md +++ b/docs/user_docs/cli/kbcli_playground.md @@ -2,7 +2,7 @@ title: kbcli playground --- -Bootstrap a playground KubeBlocks in local host or cloud. +Bootstrap or destroy a playground KubeBlocks in local host or cloud. ### Options @@ -37,8 +37,7 @@ Bootstrap a playground KubeBlocks in local host or cloud. ### SEE ALSO -* [kbcli playground destroy](kbcli_playground_destroy.md) - Destroy the playground kubernetes cluster. -* [kbcli playground guide](kbcli_playground_guide.md) - Display playground cluster user guide. +* [kbcli playground destroy](kbcli_playground_destroy.md) - Destroy the playground KubeBlocks and kubernetes cluster. * [kbcli playground init](kbcli_playground_init.md) - Bootstrap a kubernetes cluster and install KubeBlocks for playground. #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/docs/user_docs/cli/kbcli_playground_destroy.md b/docs/user_docs/cli/kbcli_playground_destroy.md index abaf7751b..5468a0270 100644 --- a/docs/user_docs/cli/kbcli_playground_destroy.md +++ b/docs/user_docs/cli/kbcli_playground_destroy.md @@ -2,7 +2,7 @@ title: kbcli playground destroy --- -Destroy the playground kubernetes cluster. +Destroy the playground KubeBlocks and kubernetes cluster. ``` kbcli playground destroy [flags] @@ -21,7 +21,7 @@ kbcli playground destroy [flags] --auto-approve Skip interactive approval before destroying the playground -h, --help help for destroy --purge Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks. (default true) - --timeout duration Time to wait for installing KubeBlocks, such as --timeout=10m (default 30m0s) + --timeout duration Time to wait for installing KubeBlocks, such as --timeout=10m (default 5m0s) ``` ### Options inherited from parent commands @@ -50,7 +50,7 @@ kbcli playground destroy [flags] ### SEE ALSO -* [kbcli playground](kbcli_playground.md) - Bootstrap a playground KubeBlocks in local host or cloud. +* [kbcli playground](kbcli_playground.md) - Bootstrap or destroy a playground KubeBlocks in local host or cloud. #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/docs/user_docs/cli/kbcli_playground_guide.md b/docs/user_docs/cli/kbcli_playground_guide.md deleted file mode 100644 index 2377fec1f..000000000 --- a/docs/user_docs/cli/kbcli_playground_guide.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: kbcli playground guide ---- - -Display playground cluster user guide. - -``` -kbcli playground guide [flags] -``` - -### Options - -``` - -h, --help help for guide -``` - -### Options inherited from parent commands - -``` - --as string Username to impersonate for the operation. User could be a regular user or a service account in a namespace. - --as-group stringArray Group to impersonate for the operation, this flag can be repeated to specify multiple groups. - --as-uid string UID to impersonate for the operation. - --cache-dir string Default cache directory (default "$HOME/.kube/cache") - --certificate-authority string Path to a cert file for the certificate authority - --client-certificate string Path to a client certificate file for TLS - --client-key string Path to a client key file for TLS - --cluster string The name of the kubeconfig cluster to use - --context string The name of the kubeconfig context to use - --disable-compression If true, opt-out of response compression for all requests to the server - --insecure-skip-tls-verify If true, the server's certificate will not be checked for validity. This will make your HTTPS connections insecure - --kubeconfig string Path to the kubeconfig file to use for CLI requests. - --match-server-version Require server version to match client version - -n, --namespace string If present, the namespace scope for this CLI request - --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (default "0") - -s, --server string The address and port of the Kubernetes API server - --tls-server-name string Server name to use for server certificate validation. If it is not provided, the hostname used to contact the server is used - --token string Bearer token for authentication to the API server - --user string The name of the kubeconfig user to use -``` - -### SEE ALSO - -* [kbcli playground](kbcli_playground.md) - Bootstrap a playground KubeBlocks in local host or cloud. - -#### Go Back to [CLI Overview](cli.md) Homepage. - diff --git a/docs/user_docs/cli/kbcli_playground_init.md b/docs/user_docs/cli/kbcli_playground_init.md index 2c79a715b..bab776f2b 100644 --- a/docs/user_docs/cli/kbcli_playground_init.md +++ b/docs/user_docs/cli/kbcli_playground_init.md @@ -4,6 +4,12 @@ title: kbcli playground init Bootstrap a kubernetes cluster and install KubeBlocks for playground. +### Synopsis + +Bootstrap a kubernetes cluster and install KubeBlocks for playground. + + If no any cloud provider be specified, a k3d cluster named kb-playground will be created on local host, otherwise a kubernetes cluster will be created on the specified cloud. Then KubeBlocks will be installed on the created kubernetes cluster, and an apecloud-mysql cluster named mycluster will be created. + ``` kbcli playground init [flags] ``` @@ -24,7 +30,23 @@ kbcli playground init [flags] kbcli playground init --cloud-provider tencentcloud --region ap-chengdu # create a Google cloud GKE cluster and install KubeBlocks, the region is required - kbcli playground init --cloud-provider gcp --region us-central1 + kbcli playground init --cloud-provider gcp --region us-east1 + + # after init, run the following commands to experience KubeBlocks quickly + # list database cluster and check its status + kbcli cluster list + + # get cluster information + kbcli cluster describe mycluster + + # connect to database + kbcli cluster connect mycluster + + # view the Grafana + kbcli dashboard open kubeblocks-grafana + + # destroy playground + kbcli playground destroy ``` ### Options @@ -32,11 +54,11 @@ kbcli playground init [flags] ``` --auto-approve Skip interactive approval during the initialization of playground --cloud-provider string Cloud provider type, one of [local aws gcp alicloud tencentcloud] (default "local") - --cluster-definition string Cluster definition (default "apecloud-mysql") - --cluster-version string Cluster definition + --cluster-definition string Specify the cluster definition, run "kbcli cd list" to get the available cluster definitions (default "apecloud-mysql") + --cluster-version string Specify the cluster version, run "kbcli cv list" to get the available cluster versions -h, --help help for init --region string The region to create kubernetes cluster - --timeout duration Time to wait for initing playground, such as --timeout=10m (default 5m0s) + --timeout duration Time to wait for init playground, such as --timeout=10m (default 5m0s) --version string KubeBlocks version ``` @@ -66,7 +88,7 @@ kbcli playground init [flags] ### SEE ALSO -* [kbcli playground](kbcli_playground.md) - Bootstrap a playground KubeBlocks in local host or cloud. +* [kbcli playground](kbcli_playground.md) - Bootstrap or destroy a playground KubeBlocks in local host or cloud. #### Go Back to [CLI Overview](cli.md) Homepage. diff --git a/docs/user_docs/quick-start/try-kubeblocks-in-5m-on-local-host.md b/docs/user_docs/quick-start/try-kubeblocks-in-5m-on-local-host.md index b4d2e32ef..2b50293b8 100644 --- a/docs/user_docs/quick-start/try-kubeblocks-in-5m-on-local-host.md +++ b/docs/user_docs/quick-start/try-kubeblocks-in-5m-on-local-host.md @@ -57,7 +57,7 @@ Meet the following requirements for smooth operation of Playground and other fun You just created a cluster named `mycluster` in the default namespace. - You can find the Playground user guide under the installation success tip. View this guide again by running `kbcli playground guide`. + You can find the Playground user guide under the installation success tip. View this guide again by running `kbcli playground init -h`. diff --git a/internal/cli/cloudprovider/k3d.go b/internal/cli/cloudprovider/k3d.go index bf4dc9d78..4607505b3 100644 --- a/internal/cli/cloudprovider/k3d.go +++ b/internal/cli/cloudprovider/k3d.go @@ -368,7 +368,7 @@ func setUpK3d(ctx context.Context, cluster *config.ClusterConfig) error { for _, c := range l { if c.Name == cluster.Name { if c, err := k3dClient.ClusterGet(ctx, runtimes.SelectedRuntime, c); err == nil { - fmt.Printf("Detected an existing cluster: %s\n", c.Name) + klog.V(1).Info("Detected an existing cluster: %s\n", c.Name) return nil } break diff --git a/internal/cli/cmd/cluster/create.go b/internal/cli/cmd/cluster/create.go index 837cd7e8b..ea17e91f4 100755 --- a/internal/cli/cmd/cluster/create.go +++ b/internal/cli/cmd/cluster/create.go @@ -77,11 +77,11 @@ var clusterCreateExample = templates.Examples(` # Create a cluster and set termination policy DoNotTerminate that will prevent the cluster from being deleted kbcli cluster create mycluster --cluster-definition apecloud-mysql --termination-policy DoNotTerminate - # In scenarios where you want to delete resources such as statements, deployments, services, pdb, but keep PVCs + # In scenarios where you want to delete resources such as statefulsets, deployments, services, pdb, but keep PVCs # when deleting the cluster, use termination policy Halt kbcli cluster create mycluster --cluster-definition apecloud-mysql --termination-policy Halt - # In scenarios where you want to delete resource such as statements, deployments, services, pdb, and including + # In scenarios where you want to delete resource such as statefulsets, deployments, services, pdb, and including # PVCs when deleting the cluster, use termination policy Delete kbcli cluster create mycluster --cluster-definition apecloud-mysql --termination-policy Delete @@ -92,6 +92,10 @@ var clusterCreateExample = templates.Examples(` # Create a cluster and set cpu to 1 core, memory to 1Gi, storage size to 20Gi and replicas to 3 kbcli cluster create mycluster --cluster-definition apecloud-mysql --set cpu=1,memory=1Gi,storage=20Gi,replicas=3 + # Create a cluster and set storageClass to csi-hostpath-sc, if storageClass is not specified, + # the default storage class will be used + kbcli cluster create mycluster --cluster-definition apecloud-mysql --set storageClass=csi-hostpath-sc + # Create a cluster and set the class to general-1c1g # run "kbcli class list --cluster-definition=cluster-definition-name" to get the class list kbcli cluster create mycluster --cluster-definition apecloud-mysql --set class=general-1c1g @@ -99,6 +103,11 @@ var clusterCreateExample = templates.Examples(` # Create a cluster with replicationSet workloadType and set switchPolicy to Noop kbcli cluster create mycluster --cluster-definition postgresql --set switchPolicy=Noop + # Create a cluster with more than one component, use "--set type=component-name" to specify the component, + # if not specified, the main component will be used, run "kbcli cd list-components CLUSTER-DEFINITION-NAME" + # to show the components in the cluster definition + kbcli cluster create mycluster --cluster-definition redis --set type=redis,cpu=1 --set type=redis-sentinel,cpu=200m + # Create a cluster and use a URL to set cluster resource kbcli cluster create mycluster --cluster-definition apecloud-mysql \ --set-file https://kubeblocks.io/yamls/apecloud-mysql.yaml @@ -1013,6 +1022,20 @@ func (o *CreateOptions) validateClusterVersion() error { if err != nil { return err } + + dryRun, err := o.GetDryRunStrategy() + if err != nil { + return err + } + + printCvInfo := func(cv string) { + // if dryRun is not None, we don't need to print the info, avoid the output yaml file including the info + if dryRun != create.DryRunNone { + return + } + fmt.Fprintf(o.Out, "Info: --cluster-version is not specified, ClusterVersion %s is applied by default\n", cv) + } + switch { case o.ClusterVersionRef != "": if _, ok := existedClusterVersions[o.ClusterVersionRef]; !ok { @@ -1022,7 +1045,7 @@ func (o *CreateOptions) validateClusterVersion() error { // if default version is not set and there is only one version, use it if len(existedClusterVersions) == 1 { o.ClusterVersionRef = maps.Keys(existedClusterVersions)[0] - fmt.Fprintf(o.Out, "Info: --cluster-version is not specified, ClusterVersion %s is applied by default\n", o.ClusterVersionRef) + printCvInfo(o.ClusterVersionRef) } else { return fmt.Errorf("failed to find the default cluster version, use '--cluster-version ClusterVersion' to set it") } @@ -1030,7 +1053,7 @@ func (o *CreateOptions) validateClusterVersion() error { // TODO: achieve this in operator if existedDefault { o.ClusterVersionRef = defaultVersion - fmt.Fprintf(o.Out, "Info: --cluster-version is not specified, ClusterVersion %s is applied by default\n", o.ClusterVersionRef) + printCvInfo(o.ClusterVersionRef) } } diff --git a/internal/cli/cmd/playground/destroy.go b/internal/cli/cmd/playground/destroy.go index fb69727e9..e9087a7e9 100644 --- a/internal/cli/cmd/playground/destroy.go +++ b/internal/cli/cmd/playground/destroy.go @@ -73,7 +73,7 @@ func newDestroyCmd(streams genericclioptions.IOStreams) *cobra.Command { } cmd := &cobra.Command{ Use: "destroy", - Short: "Destroy the playground kubernetes cluster.", + Short: "Destroy the playground KubeBlocks and kubernetes cluster.", Example: destroyExample, Run: func(cmd *cobra.Command, args []string) { util.CheckErr(o.validate()) @@ -82,22 +82,11 @@ func newDestroyCmd(streams genericclioptions.IOStreams) *cobra.Command { } cmd.Flags().BoolVar(&o.purge, "purge", true, "Purge all resources before destroy kubernetes cluster, delete all clusters created by KubeBlocks and uninstall KubeBlocks.") - cmd.Flags().DurationVar(&o.timeout, "timeout", 1800*time.Second, "Time to wait for installing KubeBlocks, such as --timeout=10m") + cmd.Flags().DurationVar(&o.timeout, "timeout", 300*time.Second, "Time to wait for installing KubeBlocks, such as --timeout=10m") cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval before destroying the playground") return cmd } -func newGuideCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "guide", - Short: "Display playground cluster user guide.", - Run: func(cmd *cobra.Command, args []string) { - printGuide() - }, - } - return cmd -} - func (o *destroyOptions) destroy() error { if o.prevCluster == nil { return fmt.Errorf("no playground cluster found") @@ -309,7 +298,7 @@ func (o *destroyOptions) deleteClusters(dynamic dynamic.Interface) error { // check all clusters termination policy is WipeOut if checkWipeOut { - if err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { + if err = wait.PollImmediate(5*time.Second, o.timeout, func() (bool, error) { return checkClusters(func(cluster *appsv1alpha1.Cluster) bool { if cluster.Spec.TerminationPolicy != appsv1alpha1.WipeOut { klog.V(1).Infof("Cluster %s termination policy is %s", cluster.Name, cluster.Spec.TerminationPolicy) @@ -327,7 +316,7 @@ func (o *destroyOptions) deleteClusters(dynamic dynamic.Interface) error { } // check and wait all clusters are deleted - if err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { + if err = wait.PollImmediate(5*time.Second, o.timeout, func() (bool, error) { return checkClusters(func(cluster *appsv1alpha1.Cluster) bool { // always return false if any cluster is not deleted klog.V(1).Infof("Cluster %s is not deleted", cluster.Name) diff --git a/internal/cli/cmd/playground/init.go b/internal/cli/cmd/playground/init.go index a853f5c2e..0965b158e 100644 --- a/internal/cli/cmd/playground/init.go +++ b/internal/cli/cmd/playground/init.go @@ -51,6 +51,12 @@ import ( ) var ( + initLong = templates.LongDesc(`Bootstrap a kubernetes cluster and install KubeBlocks for playground. + +If no any cloud provider be specified, a k3d cluster named kb-playground will be created on local host, +otherwise a kubernetes cluster will be created on the specified cloud. Then KubeBlocks will be installed +on the created kubernetes cluster, and an apecloud-mysql cluster named mycluster will be created.`) + initExample = templates.Examples(` # create a k3d cluster on local host and install KubeBlocks kbcli playground init @@ -65,7 +71,23 @@ var ( kbcli playground init --cloud-provider tencentcloud --region ap-chengdu # create a Google cloud GKE cluster and install KubeBlocks, the region is required - kbcli playground init --cloud-provider gcp --region us-central1`) + kbcli playground init --cloud-provider gcp --region us-east1 + + # after init, run the following commands to experience KubeBlocks quickly + # list database cluster and check its status + kbcli cluster list + + # get cluster information + kbcli cluster describe mycluster + + # connect to database + kbcli cluster connect mycluster + + # view the Grafana + kbcli dashboard open kubeblocks-grafana + + # destroy playground + kbcli playground destroy`) supportedCloudProviders = []string{cp.Local, cp.AWS, cp.GCP, cp.AliCloud, cp.TencentCloud} @@ -95,6 +117,7 @@ func newInitCmd(streams genericclioptions.IOStreams) *cobra.Command { cmd := &cobra.Command{ Use: "init", Short: "Bootstrap a kubernetes cluster and install KubeBlocks for playground.", + Long: initLong, Example: initExample, Run: func(cmd *cobra.Command, args []string) { util.CheckErr(o.validate()) @@ -102,12 +125,12 @@ func newInitCmd(streams genericclioptions.IOStreams) *cobra.Command { }, } - cmd.Flags().StringVar(&o.clusterDef, "cluster-definition", defaultClusterDef, "Cluster definition") - cmd.Flags().StringVar(&o.clusterVersion, "cluster-version", "", "Cluster definition") + cmd.Flags().StringVar(&o.clusterDef, "cluster-definition", defaultClusterDef, "Specify the cluster definition, run \"kbcli cd list\" to get the available cluster definitions") + cmd.Flags().StringVar(&o.clusterVersion, "cluster-version", "", "Specify the cluster version, run \"kbcli cv list\" to get the available cluster versions") cmd.Flags().StringVar(&o.kbVersion, "version", version.DefaultKubeBlocksVersion, "KubeBlocks version") cmd.Flags().StringVar(&o.cloudProvider, "cloud-provider", defaultCloudProvider, fmt.Sprintf("Cloud provider type, one of %v", supportedCloudProviders)) cmd.Flags().StringVar(&o.region, "region", "", "The region to create kubernetes cluster") - cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for initing playground, such as --timeout=10m") + cmd.Flags().DurationVar(&o.Timeout, "timeout", 300*time.Second, "Time to wait for init playground, such as --timeout=10m") cmd.Flags().BoolVar(&o.autoApprove, "auto-approve", false, "Skip interactive approval during the initialization of playground") util.CheckErr(cmd.RegisterFlagCompletionFunc( @@ -302,10 +325,6 @@ if it takes a long time, please check the network environment and try again. return nil } -func printGuide() { - fmt.Fprintf(os.Stdout, guideStr, kbClusterName) -} - // writeStateFile writes cluster info to state file and return the new cluster info with kubeconfig func (o *initOptions) writeStateFile(provider cp.Interface) (*cp.K8sClusterInfo, error) { clusterInfo, err := provider.GetClusterInfo() @@ -399,7 +418,7 @@ func (o *initOptions) installKBAndCluster(info *cp.K8sClusterInfo) error { fmt.Fprintf(o.Out, "Elapsed time: %s\n", time.Since(o.startTime).Truncate(time.Second)) } - printGuide() + fmt.Fprintf(o.Out, guideStr, kbClusterName) return nil } diff --git a/internal/cli/cmd/playground/init_test.go b/internal/cli/cmd/playground/init_test.go index 5fb84a097..d9544e6e4 100644 --- a/internal/cli/cmd/playground/init_test.go +++ b/internal/cli/cmd/playground/init_test.go @@ -64,9 +64,4 @@ var _ = Describe("playground", func() { } Expect(o.validate()).Should(HaveOccurred()) }) - - It("guide", func() { - cmd := newGuideCmd() - Expect(cmd).ShouldNot(BeNil()) - }) }) diff --git a/internal/cli/cmd/playground/palyground.go b/internal/cli/cmd/playground/palyground.go index 68f5543f6..29d1906b8 100644 --- a/internal/cli/cmd/playground/palyground.go +++ b/internal/cli/cmd/playground/palyground.go @@ -27,15 +27,14 @@ import ( // NewPlaygroundCmd creates the playground command func NewPlaygroundCmd(streams genericclioptions.IOStreams) *cobra.Command { cmd := &cobra.Command{ - Use: "playground [init | destroy | guide]", - Short: "Bootstrap a playground KubeBlocks in local host or cloud.", + Use: "playground [init | destroy]", + Short: "Bootstrap or destroy a playground KubeBlocks in local host or cloud.", } // add subcommands cmd.AddCommand( newInitCmd(streams), newDestroyCmd(streams), - newGuideCmd(), ) return cmd From ed5b7a859647e596af3a64d058f8a052e9d8ea61 Mon Sep 17 00:00:00 2001 From: "L.DongMing" Date: Wed, 24 May 2023 16:56:43 +0800 Subject: [PATCH 345/439] fix: cli failed to uninstall kubeblocks when crd has been deleted (#3421) --- internal/cli/cmd/kubeblocks/uninstall.go | 5 +++++ internal/cli/cmd/kubeblocks/uninstall_test.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/internal/cli/cmd/kubeblocks/uninstall.go b/internal/cli/cmd/kubeblocks/uninstall.go index 97f659a20..d8fe7021c 100644 --- a/internal/cli/cmd/kubeblocks/uninstall.go +++ b/internal/cli/cmd/kubeblocks/uninstall.go @@ -336,6 +336,11 @@ func checkResources(dynamic dynamic.Interface) error { if err != nil && !apierrors.IsNotFound(err) { return err } + + if objList == nil { + continue + } + for _, item := range objList.Items { crs[gvr.Resource] = append(crs[gvr.Resource], item.GetName()) } diff --git a/internal/cli/cmd/kubeblocks/uninstall_test.go b/internal/cli/cmd/kubeblocks/uninstall_test.go index bb4f94880..c8afd7213 100644 --- a/internal/cli/cmd/kubeblocks/uninstall_test.go +++ b/internal/cli/cmd/kubeblocks/uninstall_test.go @@ -77,4 +77,9 @@ var _ = Describe("kubeblocks uninstall", func() { } Expect(o.Uninstall()).Should(Succeed()) }) + + It("checkResources", func() { + fakeDynamic := testing.FakeDynamicClient() + Expect(checkResources(fakeDynamic)).Should(Succeed()) + }) }) From 24a73f00b6afcfb33d167013c9f2784e5a524e5f Mon Sep 17 00:00:00 2001 From: chantu Date: Wed, 24 May 2023 17:25:11 +0800 Subject: [PATCH 346/439] fix: touch restore file when scaling in (#3340) --- deploy/apecloud-mysql/templates/scripts.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/apecloud-mysql/templates/scripts.yaml b/deploy/apecloud-mysql/templates/scripts.yaml index d1ba7eb32..e25fc80e8 100644 --- a/deploy/apecloud-mysql/templates/scripts.yaml +++ b/deploy/apecloud-mysql/templates/scripts.yaml @@ -198,6 +198,8 @@ data: echo "current replicas: $current_component_replicas" >> /data/mysql/.kb_pre_stop.log if [ ! $idx -lt $current_component_replicas ] && [ $current_component_replicas -ne 0 ]; then # if idx greater than or equal to current_component_replicas means the cluster's scaling in + # put .restore on pvc for next scaling out, if pvc not deleted + touch /data/mysql/data/.restore; sync # switch leader before leader scaling in itself switchover # only scaling in need to drop followers From 916029efb86fa9fef811bf5d018ad46678d347ba Mon Sep 17 00:00:00 2001 From: a le <101848970+1aal@users.noreply.github.com> Date: Wed, 24 May 2023 18:57:08 +0800 Subject: [PATCH 347/439] fix: add IS-MAIN column and fix the output for multiple cd input (#3410) --- .../cli/cmd/clusterdefinition/list_compoents.go | 13 +++++++++---- .../cmd/clusterdefinition/list_component_test.go | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/internal/cli/cmd/clusterdefinition/list_compoents.go b/internal/cli/cmd/clusterdefinition/list_compoents.go index 402c3ca25..e7bee4769 100644 --- a/internal/cli/cmd/clusterdefinition/list_compoents.go +++ b/internal/cli/cmd/clusterdefinition/list_compoents.go @@ -79,15 +79,20 @@ func run(o *list.ListOptions) error { return err } p := printer.NewTablePrinter(o.Out) - p.SetHeader("NAME", "WORKLOAD-TYPE", "CHARACTER-TYPE", "CLUSTER-DEFINITION") - p.SortBy(1) + p.SetHeader("NAME", "WORKLOAD-TYPE", "CHARACTER-TYPE", "CLUSTER-DEFINITION", "IS-MAIN") + p.SortBy(4, 1) for _, info := range infos { var cd v1alpha1.ClusterDefinition if err = runtime.DefaultUnstructuredConverter.FromUnstructured(info.Object.(*unstructured.Unstructured).Object, &cd); err != nil { return err } - for _, comp := range cd.Spec.ComponentDefs { - p.AddRow(comp.Name, comp.WorkloadType, comp.CharacterType, cd.Name) + for i, comp := range cd.Spec.ComponentDefs { + if i == 0 { + p.AddRow(printer.BoldGreen(comp.Name), comp.WorkloadType, comp.CharacterType, cd.Name, "true") + } else { + p.AddRow(comp.Name, comp.WorkloadType, comp.CharacterType, cd.Name, "false") + } + } } p.Print() diff --git a/internal/cli/cmd/clusterdefinition/list_component_test.go b/internal/cli/cmd/clusterdefinition/list_component_test.go index 2664ded8a..4eb6a8efe 100644 --- a/internal/cli/cmd/clusterdefinition/list_component_test.go +++ b/internal/cli/cmd/clusterdefinition/list_component_test.go @@ -104,9 +104,9 @@ var _ = Describe("clusterdefinition list components", func() { It("list-components", func() { cmd.Run(cmd, []string{clusterdefinitionName}) - expected := `NAME WORKLOAD-TYPE CHARACTER-TYPE CLUSTER-DEFINITION -fake-component-type mysql fake-cluster-definition -fake-component-type-1 mysql fake-cluster-definition + expected := `NAME WORKLOAD-TYPE CHARACTER-TYPE CLUSTER-DEFINITION IS-MAIN +fake-component-type mysql fake-cluster-definition true +fake-component-type-1 mysql fake-cluster-definition false ` Expect(expected).Should(Equal(out.String())) fmt.Println(out.String()) From 49fdcb7cca51784c46d9e4f2a510b4b59476b8bd Mon Sep 17 00:00:00 2001 From: Nash Tsai Date: Wed, 24 May 2023 23:51:38 +0800 Subject: [PATCH 348/439] chore: preserve data objects (PVC, Secret, CM) for cluster object deletion with terminationPolicy=Halt (#3352) Co-authored-by: wangyelei --- apis/apps/v1alpha1/type.go | 1 + controllers/apps/class_controller.go | 3 +- controllers/apps/cluster_controller.go | 11 +- controllers/apps/cluster_controller_test.go | 247 +++++++++++++----- .../replication/replication_utils.go | 5 +- .../replication/replication_utils_test.go | 2 +- .../apps/configuration/config_annotation.go | 6 +- .../reconfigurerequest_controller.go | 2 +- controllers/apps/const.go | 1 - .../apps/operations/reconfigure_util.go | 2 +- .../apps/systemaccount_controller_test.go | 2 +- controllers/apps/systemaccount_util.go | 2 +- .../dataprotection/backuppolicy_controller.go | 6 +- .../backuppolicy_controller_test.go | 4 +- internal/constant/const.go | 114 ++++---- internal/controller/graph/transformer.go | 10 +- .../lifecycle/cluster_plan_builder.go | 25 +- .../lifecycle/cluster_plan_utils.go | 38 +-- .../lifecycle/cluster_plan_utils_test.go | 13 +- .../controller/lifecycle/transform_types.go | 9 +- .../controller/lifecycle/transform_utils.go | 24 +- ...fix_meta.go => transformer_assure_meta.go} | 11 +- .../transformer_backup_policy_tpl.go | 8 +- .../lifecycle/transformer_cluster_deletion.go | 100 +++++-- .../lifecycle/transformer_halt_recovering.go | 213 +++++++++++++++ .../lifecycle/transformer_object_action.go | 4 +- .../lifecycle/transformer_ownership.go | 5 +- .../transformer_sts_horizontal_scaling.go | 6 +- ...transformer_sts_horizontal_scaling_test.go | 5 +- internal/controllerutil/controller_common.go | 15 +- .../apps/cluster_consensus_test_util.go | 25 +- internal/testutil/apps/common_util.go | 2 +- 32 files changed, 673 insertions(+), 248 deletions(-) rename internal/controller/lifecycle/{transformer_fix_meta.go => transformer_assure_meta.go} (81%) create mode 100644 internal/controller/lifecycle/transformer_halt_recovering.go diff --git a/apis/apps/v1alpha1/type.go b/apis/apps/v1alpha1/type.go index 6cff1222a..1c003029c 100644 --- a/apis/apps/v1alpha1/type.go +++ b/apis/apps/v1alpha1/type.go @@ -123,6 +123,7 @@ const ( const ( // define the cluster condition type ConditionTypeLatestOpsRequestProcessed = "LatestOpsRequestProcessed" // ConditionTypeLatestOpsRequestProcessed describes whether the latest OpsRequest that affect the cluster lifecycle has been processed. + ConditionTypeHaltRecovery = "HaltRecovery" // ConditionTypeHaltRecovery describe Halt recovery processing stage ConditionTypeProvisioningStarted = "ProvisioningStarted" // ConditionTypeProvisioningStarted the operator starts resource provisioning to create or change the cluster ConditionTypeApplyResources = "ApplyResources" // ConditionTypeApplyResources the operator start to apply resources to create or change the cluster ConditionTypeReplicasReady = "ReplicasReady" // ConditionTypeReplicasReady all pods of components are ready diff --git a/controllers/apps/class_controller.go b/controllers/apps/class_controller.go index b0e5be064..3494a14a7 100644 --- a/controllers/apps/class_controller.go +++ b/controllers/apps/class_controller.go @@ -33,6 +33,7 @@ import ( appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" "github.com/apecloud/kubeblocks/internal/class" "github.com/apecloud/kubeblocks/internal/cli/types" + "github.com/apecloud/kubeblocks/internal/constant" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -75,7 +76,7 @@ func (r *ComponentClassReconciler) Reconcile(ctx context.Context, req reconcile. constraintsMap[cf.GetName()] = cf } - res, err := intctrlutil.HandleCRDeletion(reqCtx, r, classDefinition, dbClusterFinalizerName, func() (*ctrl.Result, error) { + res, err := intctrlutil.HandleCRDeletion(reqCtx, r, classDefinition, constant.DBClusterFinalizerName, func() (*ctrl.Result, error) { // TODO validate if existing cluster reference classes being deleted return nil, nil }) diff --git a/controllers/apps/cluster_controller.go b/controllers/apps/cluster_controller.go index ec8f1d2eb..923963711 100644 --- a/controllers/apps/cluster_controller.go +++ b/controllers/apps/cluster_controller.go @@ -165,10 +165,12 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // handle deletion // handle cluster deletion first &lifecycle.ClusterDeletionTransformer{}, - // fix meta - // fix finalizer and cd&cv labels - &lifecycle.FixMetaTransformer{}, - // validate + // check is recovering from halted cluster + &lifecycle.HaltRecoveryTransformer{}, + // assure meta-data info + // update finalizer and cd&cv labels + &lifecycle.AssureMetaTransformer{}, + // validate ref objects // validate cd & cv's existence and availability &lifecycle.ValidateAndLoadRefResourcesTransformer{}, // validate config @@ -217,7 +219,6 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if errBuild != nil { return requeueError(errBuild) } - return intctrlutil.Reconciled() } diff --git a/controllers/apps/cluster_controller_test.go b/controllers/apps/cluster_controller_test.go index 95b12163e..bbbf6e2b4 100644 --- a/controllers/apps/cluster_controller_test.go +++ b/controllers/apps/cluster_controller_test.go @@ -20,6 +20,7 @@ along with this program. If not, see . package apps import ( + "encoding/json" "errors" "fmt" "strconv" @@ -63,7 +64,7 @@ var _ = Describe("Cluster Controller", func() { const ( clusterDefName = "test-clusterdef" clusterVersionName = "test-clusterversion" - clusterNamePrefix = "test-cluster" + clusterName = "test-cluster" // this become cluster prefix name if used with testapps.NewClusterFactory().WithRandomName() leader = "leader" follower = "follower" // REVIEW: @@ -235,7 +236,7 @@ var _ = Describe("Cluster Controller", func() { testServiceAddAndDelete := func(compName, compDefName string) { By("Creating a cluster with two LoadBalancer services") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name). AddComponent(compName, compDefName).SetReplicas(1). AddService(testapps.ServiceVPCName, corev1.ServiceTypeLoadBalancer). @@ -295,7 +296,7 @@ var _ = Describe("Cluster Controller", func() { createClusterObj := func(compName, compDefName string) { By("Creating a cluster") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName). Create(&testCtx).GetObject() @@ -639,7 +640,7 @@ var _ = Describe("Cluster Controller", func() { By("Creating a single component cluster with VolumeClaimTemplate") pvcSpec := testapps.NewPVCSpec("1Gi") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). @@ -674,7 +675,7 @@ var _ = Describe("Cluster Controller", func() { pvcSpec.StorageClassName = &storageClass.Name By("Create cluster and waiting for the cluster initialized") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). @@ -771,7 +772,7 @@ var _ = Describe("Cluster Controller", func() { pvcSpec.StorageClassName = &storageClass.Name By("Create cluster and waiting for the cluster initialized") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). @@ -905,7 +906,7 @@ var _ = Describe("Cluster Controller", func() { Tenancy: appsv1alpha1.SharedNode, } - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name). AddComponent(compName, compDefName).SetReplicas(3). WithRandomName().SetClusterAffinity(affinity). @@ -930,7 +931,7 @@ var _ = Describe("Cluster Controller", func() { By("Creating a cluster with target service account name") Expect(compDefName).Should(BeElementOf(statelessCompDefName, statefulCompDefName, replicationCompDefName, consensusCompDefName)) - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name). AddComponent(compName, compDefName).SetReplicas(3). SetServiceAccountName("test-service-account"). @@ -963,7 +964,7 @@ var _ = Describe("Cluster Controller", func() { TopologyKeys: []string{compTopologyKey}, Tenancy: appsv1alpha1.DedicatedNode, } - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName().SetClusterAffinity(affinity). AddComponent(compName, compDefName).SetComponentAffinity(compAffinity). Create(&testCtx).GetObject() @@ -988,7 +989,7 @@ var _ = Describe("Cluster Controller", func() { const tolerationValue = "testClusterTolerationValue" By("Creating a cluster with Toleration") Expect(compDefName).Should(BeElementOf(statelessCompDefName, statefulCompDefName, replicationCompDefName, consensusCompDefName)) - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName).SetReplicas(1). AddClusterToleration(corev1.Toleration{ @@ -1028,7 +1029,7 @@ var _ = Describe("Cluster Controller", func() { Operator: corev1.TolerationOpEqual, Effect: corev1.TaintEffectNoSchedule, } - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddClusterToleration(corev1.Toleration{ Key: clusterTolerationKey, @@ -1096,7 +1097,7 @@ var _ = Describe("Cluster Controller", func() { By("Mock a cluster obj") pvcSpec := testapps.NewPVCSpec("1Gi") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName). SetReplicas(replicas).AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). @@ -1217,7 +1218,7 @@ var _ = Describe("Cluster Controller", func() { By("Creating a cluster with VolumeClaimTemplate") pvcSpec := testapps.NewPVCSpec("1Gi") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName). AddVolumeClaimTemplate(testapps.DataVolumeName, pvcSpec). @@ -1325,7 +1326,7 @@ var _ = Describe("Cluster Controller", func() { It("should reconcile to create cluster with no error", func() { By("Creating a cluster") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, ""). AddComponent(statelessCompName, statelessCompDefName).SetReplicas(3). AddComponent(statefulCompName, statefulCompDefName).SetReplicas(3). @@ -1346,11 +1347,12 @@ var _ = Describe("Cluster Controller", func() { }) createNWaitClusterObj := func(components map[string]string, - addedComponentProcessor func(compName string, factory *testapps.MockClusterFactory)) { + addedComponentProcessor func(compName string, factory *testapps.MockClusterFactory), + withFixedName ...bool) { Expect(components).ShouldNot(BeEmpty()) By("Creating a cluster") - clusterBuilder := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterBuilder := testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name) compNames := make([]string, 0, len(components)) @@ -1361,22 +1363,20 @@ var _ = Describe("Cluster Controller", func() { } compNames = append(compNames, compName) } - - clusterObj = clusterBuilder.WithRandomName().Create(&testCtx).GetObject() + if len(withFixedName) == 0 || !withFixedName[0] { + clusterBuilder.WithRandomName() + } + clusterObj = clusterBuilder.Create(&testCtx).GetObject() clusterKey = client.ObjectKeyFromObject(clusterObj) By("Waiting for the cluster controller to create resources completely") waitForCreatingResourceCompletely(clusterKey, compNames...) } - checkAllResourcesCreated := func() { - compNameNDef := map[string]string{ - statelessCompName: statelessCompDefName, - consensusCompName: consensusCompDefName, - } + checkAllResourcesCreated := func(compNameNDef map[string]string) { createNWaitClusterObj(compNameNDef, func(compName string, factory *testapps.MockClusterFactory) { factory.SetReplicas(3) - }) + }, true) By("Check deployment workload has been created") Eventually(testapps.List(&testCtx, generics.DeploymentSignature, @@ -1428,24 +1428,15 @@ var _ = Describe("Cluster Controller", func() { Expect(podSpec.TopologySpreadConstraints).Should(BeEmpty()) By("Check should create env configmap") - Eventually(testapps.List(&testCtx, generics.ConfigMapSignature, - client.MatchingLabels{ + Eventually(func(g Gomega) { + cmList := &corev1.ConfigMapList{} + Expect(k8sClient.List(testCtx.Ctx, cmList, client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, constant.AppConfigTypeLabelKey: "kubeblocks-env", - }, client.InNamespace(clusterKey.Namespace))).Should(HaveLen(2)) - } - - checkAllServicesCreate := func() { - compNameNDef := map[string]string{ - statelessCompName: statelessCompDefName, - consensusCompName: consensusCompDefName, - statefulCompName: statefulCompDefName, - replicationCompName: replicationCompDefName, - } - - createNWaitClusterObj(compNameNDef, func(compName string, factory *testapps.MockClusterFactory) { - factory.SetReplicas(3) - }) + }, client.InNamespace(clusterKey.Namespace))).Should(Succeed()) + Expect(cmList.Items).ShouldNot(BeEmpty()) + Expect(cmList.Items).Should(HaveLen(len(compNameNDef))) + }).Should(Succeed()) By("Checking stateless services") statelessExpectServices := map[string]ExpectService{ @@ -1497,12 +1488,111 @@ var _ = Describe("Cluster Controller", func() { horizontalScale(int(updatedReplicas), consensusCompDefName, replicationCompDefName) } - It("should create all sub-resources successfully", func() { - checkAllResourcesCreated() - }) + It("should create all sub-resources successfully, with terminationPolicy=Halt lifecycle", func() { + compNameNDef := map[string]string{ + statelessCompName: statelessCompDefName, + consensusCompName: consensusCompDefName, + statefulCompName: statefulCompDefName, + replicationCompName: replicationCompDefName, + } + checkAllResourcesCreated(compNameNDef) + + By("Mocking components' PVCs to bound") + stsList := testk8s.ListAndCheckStatefulSet(&testCtx, clusterKey) + for _, sts := range stsList.Items { + compName, ok := sts.Labels[constant.KBAppComponentLabelKey] + Expect(ok).Should(BeTrue()) + for i := int(*sts.Spec.Replicas); i >= 0; i-- { + pvcKey := types.NamespacedName{ + Namespace: clusterKey.Namespace, + Name: getPVCName(compName, i), + } + createPVC(clusterKey.Name, pvcKey.Name, compName) + Eventually(testapps.CheckObjExists(&testCtx, pvcKey, &corev1.PersistentVolumeClaim{}, true)).Should(Succeed()) + Expect(testapps.GetAndChangeObjStatus(&testCtx, pvcKey, func(pvc *corev1.PersistentVolumeClaim) { + pvc.Status.Phase = corev1.ClaimBound + })()).ShouldNot(HaveOccurred()) + } + } + + By("delete the cluster and should preserved PVC,Secret,CM resources") + deleteCluster := func(termPolicy appsv1alpha1.TerminationPolicyType) { + // TODO: would be better that cluster is created with terminationPolicy=Halt instead of + // reassign the value after created + Expect(testapps.GetAndChangeObj(&testCtx, clusterKey, func(cluster *appsv1alpha1.Cluster) { + cluster.Spec.TerminationPolicy = termPolicy + })()).ShouldNot(HaveOccurred()) + testapps.DeleteObject(&testCtx, clusterKey, &appsv1alpha1.Cluster{}) + Eventually(testapps.CheckObjExists(&testCtx, clusterKey, &appsv1alpha1.Cluster{}, false)).Should(Succeed()) + } + deleteCluster(appsv1alpha1.Halt) + + By("check should preserved PVC,Secret,CM resources") + + checkPreservedObjects := func(uid types.UID) (*corev1.PersistentVolumeClaimList, *corev1.SecretList, *corev1.ConfigMapList) { + checkObject := func(obj client.Object) { + clusterJSON, ok := obj.GetAnnotations()[constant.LastAppliedClusterAnnotationKey] + Expect(ok).Should(BeTrue()) + Expect(clusterJSON).ShouldNot(BeEmpty()) + lastAppliedCluster := &appsv1alpha1.Cluster{} + Expect(json.Unmarshal([]byte(clusterJSON), lastAppliedCluster)).ShouldNot(HaveOccurred()) + Expect(lastAppliedCluster.UID).Should(BeEquivalentTo(uid)) + } + listOptions := []client.ListOption{ + client.InNamespace(clusterKey.Namespace), + client.MatchingLabels{ + constant.AppInstanceLabelKey: clusterKey.Name, + }, + } + pvcList := &corev1.PersistentVolumeClaimList{} + Expect(k8sClient.List(testCtx.Ctx, pvcList, listOptions...)).Should(Succeed()) + + cmList := &corev1.ConfigMapList{} + Expect(k8sClient.List(testCtx.Ctx, cmList, listOptions...)).Should(Succeed()) - It("should create corresponding services correctly", func() { - checkAllServicesCreate() + secretList := &corev1.SecretList{} + Expect(k8sClient.List(testCtx.Ctx, secretList, listOptions...)).Should(Succeed()) + if uid != "" { + By("check pvc resources preserved") + Expect(pvcList.Items).ShouldNot(BeEmpty()) + + for _, pvc := range pvcList.Items { + checkObject(&pvc) + } + By("check secret resources preserved") + Expect(cmList.Items).ShouldNot(BeEmpty()) + for _, secret := range secretList.Items { + checkObject(&secret) + } + By("check configmap resources preserved") + Expect(secretList.Items).ShouldNot(BeEmpty()) + for _, cm := range cmList.Items { + checkObject(&cm) + } + } + return pvcList, secretList, cmList + } + initPVCList, initSecretList, initCMList := checkPreservedObjects(clusterObj.UID) + + By("create recovering cluster") + lastClusterUID := clusterObj.UID + checkAllResourcesCreated(compNameNDef) + Expect(clusterObj.UID).ShouldNot(Equal(lastClusterUID)) + lastPVCList, lastSecretList, lastCMList := checkPreservedObjects("") + + Expect(outOfOrderEqualFunc(initPVCList.Items, lastPVCList.Items, func(i corev1.PersistentVolumeClaim, j corev1.PersistentVolumeClaim) bool { + return i.UID == j.UID + })).Should(BeTrue()) + Expect(outOfOrderEqualFunc(initSecretList.Items, lastSecretList.Items, func(i corev1.Secret, j corev1.Secret) bool { + return i.UID == j.UID + })).Should(BeTrue()) + Expect(outOfOrderEqualFunc(initCMList.Items, lastCMList.Items, func(i corev1.ConfigMap, j corev1.ConfigMap) bool { + return i.UID == j.UID + })).Should(BeTrue()) + + By("delete the cluster and should preserved PVC,Secret,CM resources but result updated the new last applied cluster UID") + deleteCluster(appsv1alpha1.Halt) + checkPreservedObjects(clusterObj.UID) }) It("should successfully h-scale with multiple components", func() { @@ -1523,11 +1613,11 @@ var _ = Describe("Cluster Controller", func() { }) for compName, compDefName := range compNameNDef { - It(fmt.Sprintf("[comp: %s] should delete cluster resources immediately if deleting cluster with WipeOut termination policy", compName), func() { + It(fmt.Sprintf("[comp: %s] should delete cluster resources immediately if deleting cluster with terminationPolicy=WipeOut", compName), func() { testWipeOut(compName, compDefName) }) - It(fmt.Sprintf("[comp: %s] should not terminate immediately if deleting cluster with DoNotTerminate termination policy", compName), func() { + It(fmt.Sprintf("[comp: %s] should not terminate immediately if deleting cluster with terminationPolicy=DoNotTerminate", compName), func() { testDoNotTermintate(compName, compDefName) }) @@ -1660,7 +1750,7 @@ var _ = Describe("Cluster Controller", func() { By("creating cluster with backup") restoreFromBackup := fmt.Sprintf(`{"%s":"%s"}`, compName, backupName) - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName). SetReplicas(3). @@ -1721,7 +1811,7 @@ var _ = Describe("Cluster Controller", func() { It("Should success with primary pod and secondary pod", func() { By("Mock a cluster obj with replication componentDefRef.") pvcSpec := testapps.NewPVCSpec("1Gi") - clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, + clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefObj.Name, clusterVersionObj.Name).WithRandomName(). AddComponent(compName, compDefName). SetPrimaryIndex(testapps.DefaultReplicationPrimaryIndex). @@ -1753,20 +1843,10 @@ var _ = Describe("Cluster Controller", func() { It("test cluster conditions", func() { By("init cluster") cluster := testapps.CreateConsensusMysqlCluster(&testCtx, clusterDefNameRand, - clusterVersionNameRand, clusterNameRand, consensusCompDefName, consensusCompName) + clusterVersionNameRand, clusterNameRand, consensusCompDefName, consensusCompName, + "2Gi") clusterKey := client.ObjectKeyFromObject(cluster) - By("mock pvc created") - for i := 0; i < 3; i++ { - pvcName := fmt.Sprintf("%s-%s-%s-%d", testapps.DataVolumeName, clusterKey.Name, consensusCompName, i) - pvc := testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterKey.Name, - consensusCompName, "data").SetStorage("2Gi").Create(&testCtx).GetObject() - // mock pvc bound - Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(pvc), func(pvc *corev1.PersistentVolumeClaim) { - pvc.Status.Phase = corev1.ClaimBound - })()).ShouldNot(HaveOccurred()) - } - By("test when clusterDefinition not found") Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { g.Expect(tmpCluster.Status.ObservedGeneration).Should(BeZero()) @@ -1821,7 +1901,7 @@ var _ = Describe("Cluster Controller", func() { })()).ShouldNot(HaveOccurred()) Eventually(testapps.CheckObj(&testCtx, clusterVersionKey, func(g Gomega, clusterVersion *appsv1alpha1.ClusterVersion) { - g.Expect(clusterVersion.Status.Phase == appsv1alpha1.AvailablePhase).Should(BeTrue()) + g.Expect(clusterVersion.Status.Phase).Should(Equal(appsv1alpha1.AvailablePhase)) })).Should(Succeed()) // trigger reconcile @@ -1830,7 +1910,8 @@ var _ = Describe("Cluster Controller", func() { Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { g.Expect(cluster.Status.ObservedGeneration).Should(BeZero()) condition := meta.FindStatusCondition(cluster.Status.Conditions, appsv1alpha1.ConditionTypeProvisioningStarted) - g.Expect(condition != nil && condition.Reason == lifecycle.ReasonPreCheckFailed).Should(BeTrue()) + g.Expect(condition).ShouldNot(BeNil()) + g.Expect(condition.Reason).Should(Equal(lifecycle.ReasonPreCheckFailed)) })).Should(Succeed()) By("reset and waiting cluster to Creating") @@ -1843,16 +1924,31 @@ var _ = Describe("Cluster Controller", func() { g.Expect(tmpCluster.Status.ObservedGeneration).ShouldNot(BeZero()) })).Should(Succeed()) - By("test apply resources failed") + By("mock pvc of component to create") + for i := 0; i < testapps.ConsensusReplicas; i++ { + pvcName := fmt.Sprintf("%s-%s-%s-%d", testapps.DataVolumeName, clusterKey.Name, consensusCompName, i) + pvc := testapps.NewPersistentVolumeClaimFactory(testCtx.DefaultNamespace, pvcName, clusterKey.Name, + consensusCompName, "data").SetStorage("2Gi").Create(&testCtx).GetObject() + // mock pvc bound + Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(pvc), func(pvc *corev1.PersistentVolumeClaim) { + pvc.Status.Phase = corev1.ClaimBound + pvc.Status.Capacity = corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("2Gi"), + } + })()).ShouldNot(HaveOccurred()) + } + + By("apply smaller PVC size will should failed") Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(cluster), func(tmpCluster *appsv1alpha1.Cluster) { tmpCluster.Spec.ComponentSpecs[0].VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("1Gi") })()).ShouldNot(HaveOccurred()) Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(cluster), func(g Gomega, tmpCluster *appsv1alpha1.Cluster) { - g.Expect(tmpCluster.Status.ObservedGeneration).ShouldNot(BeEquivalentTo(tmpCluster.Generation)) + // REVIEW/TODO: (wangyelei) following expects causing inconsistent behavior condition := meta.FindStatusCondition(tmpCluster.Status.Conditions, appsv1alpha1.ConditionTypeApplyResources) - g.Expect(condition != nil && condition.Reason == lifecycle.ReasonApplyResourcesFailed).Should(BeTrue()) + g.Expect(condition).ShouldNot(BeNil()) + g.Expect(condition.Reason).Should(Equal(lifecycle.ReasonApplyResourcesFailed)) })).Should(Succeed()) }) }) @@ -1874,3 +1970,22 @@ func createBackupPolicyTpl(clusterDefObj *appsv1alpha1.ClusterDefinition) { } bpt.Create(&testCtx) } + +func outOfOrderEqualFunc[E1, E2 any](s1 []E1, s2 []E2, eq func(E1, E2) bool) bool { + if l := len(s1); l != len(s2) { + return false + } + + for _, v1 := range s1 { + isEq := false + for _, v2 := range s2 { + if isEq = eq(v1, v2); isEq { + break + } + } + if !isEq { + return false + } + } + return true +} diff --git a/controllers/apps/components/replication/replication_utils.go b/controllers/apps/components/replication/replication_utils.go index 21a15a444..3cc43a8e7 100644 --- a/controllers/apps/components/replication/replication_utils.go +++ b/controllers/apps/components/replication/replication_utils.go @@ -40,9 +40,8 @@ import ( type ReplicationRole string const ( - Primary ReplicationRole = "primary" - Secondary ReplicationRole = "secondary" - DBClusterFinalizerName = "cluster.kubeblocks.io/finalizer" + Primary ReplicationRole = "primary" + Secondary ReplicationRole = "secondary" ) // syncReplicationSetClusterStatus syncs replicationSet pod status to cluster.status.component[componentName].ReplicationStatus. diff --git a/controllers/apps/components/replication/replication_utils_test.go b/controllers/apps/components/replication/replication_utils_test.go index b88b6f5db..eac72f041 100644 --- a/controllers/apps/components/replication/replication_utils_test.go +++ b/controllers/apps/components/replication/replication_utils_test.go @@ -91,7 +91,7 @@ var _ = Describe("ReplicationSet Util", func() { } sts := testapps.NewStatefulSetFactory(testCtx.DefaultNamespace, clusterObj.Name+"-"+testapps.DefaultRedisCompName, clusterObj.Name, testapps.DefaultRedisCompName). - AddFinalizers([]string{DBClusterFinalizerName}). + AddFinalizers([]string{constant.DBClusterFinalizerName}). AddContainer(container). AddAppInstanceLabel(clusterObj.Name). AddAppComponentLabel(testapps.DefaultRedisCompName). diff --git a/controllers/apps/configuration/config_annotation.go b/controllers/apps/configuration/config_annotation.go index 8e4bd28e8..063a46758 100644 --- a/controllers/apps/configuration/config_annotation.go +++ b/controllers/apps/configuration/config_annotation.go @@ -73,7 +73,7 @@ func checkAndApplyConfigsChanged(client client.Client, ctx intctrlutil.RequestCt return false, err } - lastConfig, ok := annotations[constant.LastAppliedConfigAnnotation] + lastConfig, ok := annotations[constant.LastAppliedConfigAnnotationKey] if !ok { return updateAppliedConfigs(client, ctx, cm, configData, cfgcore.ReconfigureCreatedPhase) } @@ -89,7 +89,7 @@ func updateAppliedConfigs(cli client.Client, ctx intctrlutil.RequestCtx, config config.ObjectMeta.Annotations = map[string]string{} } - config.ObjectMeta.Annotations[constant.LastAppliedConfigAnnotation] = string(configData) + config.ObjectMeta.Annotations[constant.LastAppliedConfigAnnotationKey] = string(configData) hash, err := util.ComputeHash(config.Data) if err != nil { return false, err @@ -116,7 +116,7 @@ func updateAppliedConfigs(cli client.Client, ctx intctrlutil.RequestCtx, config func getLastVersionConfig(cm *corev1.ConfigMap) (map[string]string, error) { data := make(map[string]string, 0) - cfgContent, ok := cm.GetAnnotations()[constant.LastAppliedConfigAnnotation] + cfgContent, ok := cm.GetAnnotations()[constant.LastAppliedConfigAnnotationKey] if !ok { return data, nil } diff --git a/controllers/apps/configuration/reconfigurerequest_controller.go b/controllers/apps/configuration/reconfigurerequest_controller.go index fe4b3d492..9080a2d39 100644 --- a/controllers/apps/configuration/reconfigurerequest_controller.go +++ b/controllers/apps/configuration/reconfigurerequest_controller.go @@ -297,7 +297,7 @@ func (r *ReconfigureRequestReconciler) handleConfigEvent(params reconfigureParam ) if len(cm.Annotations) != 0 { - lastOpsRequest = cm.Annotations[constant.LastAppliedOpsCRAnnotation] + lastOpsRequest = cm.Annotations[constant.LastAppliedOpsCRAnnotationKey] } eventContext := cfgcore.ConfigEventContext{ diff --git a/controllers/apps/const.go b/controllers/apps/const.go index 9ad4e9680..b07225e18 100644 --- a/controllers/apps/const.go +++ b/controllers/apps/const.go @@ -24,7 +24,6 @@ const ( maxConcurReconClusterDefKey = "MAXCONCURRENTRECONCILES_CLUSTERDEF" // name of our custom finalizer - dbClusterFinalizerName = "cluster.kubeblocks.io/finalizer" dbClusterDefFinalizerName = "clusterdefinition.kubeblocks.io/finalizer" clusterVersionFinalizerName = "clusterversion.kubeblocks.io/finalizer" opsRequestFinalizerName = "opsrequest.kubeblocks.io/finalizer" diff --git a/controllers/apps/operations/reconfigure_util.go b/controllers/apps/operations/reconfigure_util.go index aac795e5a..94360ee36 100644 --- a/controllers/apps/operations/reconfigure_util.go +++ b/controllers/apps/operations/reconfigure_util.go @@ -95,7 +95,7 @@ func persistCfgCM(cmObj *corev1.ConfigMap, newCfg map[string]string, cli client. if cmObj.Annotations == nil { cmObj.Annotations = make(map[string]string) } - cmObj.Annotations[constant.LastAppliedOpsCRAnnotation] = opsCrName + cmObj.Annotations[constant.LastAppliedOpsCRAnnotationKey] = opsCrName cfgcore.SetParametersUpdateSource(cmObj, constant.ReconfigureUserSource) return cli.Patch(ctx, cmObj, patch) } diff --git a/controllers/apps/systemaccount_controller_test.go b/controllers/apps/systemaccount_controller_test.go index 8fb93236a..a21473cb3 100644 --- a/controllers/apps/systemaccount_controller_test.go +++ b/controllers/apps/systemaccount_controller_test.go @@ -461,7 +461,7 @@ var _ = Describe("SystemAccount Controller", func() { g.Expect(k8sClient.List(ctx, secretsForAcct, ml)).To(Succeed()) for _, secret := range secretsForAcct.Items { // each secret has finalizer - g.Expect(controllerutil.ContainsFinalizer(&secret, dbClusterFinalizerName)).To(BeTrue()) + g.Expect(controllerutil.ContainsFinalizer(&secret, constant.DBClusterFinalizerName)).To(BeTrue()) g.Expect(len(secret.ObjectMeta.OwnerReferences)).To(BeEquivalentTo(1)) g.Expect(checkOwnerReferenceToObj(secret.OwnerReferences[0], cluster)).To(BeTrue()) } diff --git a/controllers/apps/systemaccount_util.go b/controllers/apps/systemaccount_util.go index 403ccf73f..4deee471f 100644 --- a/controllers/apps/systemaccount_util.go +++ b/controllers/apps/systemaccount_util.go @@ -243,7 +243,7 @@ func renderSecret(key componentUniqueKey, username string, labels client.Matchin Namespace: key.namespace, Name: strings.Join([]string{key.clusterName, key.componentName, username}, "-"), Labels: labels, - Finalizers: []string{dbClusterFinalizerName}, + Finalizers: []string{constant.DBClusterFinalizerName}, }, Data: data, } diff --git a/controllers/dataprotection/backuppolicy_controller.go b/controllers/dataprotection/backuppolicy_controller.go index a3ab55445..1e1f41b0e 100644 --- a/controllers/dataprotection/backuppolicy_controller.go +++ b/controllers/dataprotection/backuppolicy_controller.go @@ -492,7 +492,7 @@ func (r *BackupPolicyReconciler) reconfigure(reqCtx intctrlutil.RequestCtx, if commonSchedule != nil { enable = commonSchedule.Enable } - if backupPolicy.Annotations[constant.LastAppliedConfigAnnotation] == "" && !enable { + if backupPolicy.Annotations[constant.LastAppliedConfigAnnotationKey] == "" && !enable { // disable in the first policy created, no need reconfigure because default configs had been set. return nil } @@ -510,7 +510,7 @@ func (r *BackupPolicyReconciler) reconfigure(reqCtx intctrlutil.RequestCtx, } updateParameterPairsBytes, _ := json.Marshal(parameters) updateParameterPairs := string(updateParameterPairsBytes) - if updateParameterPairs == backupPolicy.Annotations[constant.LastAppliedConfigAnnotation] { + if updateParameterPairs == backupPolicy.Annotations[constant.LastAppliedConfigAnnotationKey] { // reconcile the config job if finished return r.reconcileReconfigure(reqCtx, backupPolicy) } @@ -553,7 +553,7 @@ func (r *BackupPolicyReconciler) reconfigure(reqCtx intctrlutil.RequestCtx, if backupPolicy.Annotations == nil { backupPolicy.Annotations = map[string]string{} } - backupPolicy.Annotations[constant.LastAppliedConfigAnnotation] = updateParameterPairs + backupPolicy.Annotations[constant.LastAppliedConfigAnnotationKey] = updateParameterPairs if err := r.Client.Patch(reqCtx.Ctx, backupPolicy, patch); err != nil { return err } diff --git a/controllers/dataprotection/backuppolicy_controller_test.go b/controllers/dataprotection/backuppolicy_controller_test.go index b0e95eca1..35d8ba138 100644 --- a/controllers/dataprotection/backuppolicy_controller_test.go +++ b/controllers/dataprotection/backuppolicy_controller_test.go @@ -357,7 +357,7 @@ var _ = Describe("Backup Policy Controller", func() { fetched.Spec.Schedule.Logfile = &dpv1alpha1.SchedulePolicy{Enable: true, CronExpression: "* * * * *"} })).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Annotations[constant.LastAppliedConfigAnnotation]).To(Equal(`[{"key":"archive_command","value":"''"}]`)) + g.Expect(fetched.Annotations[constant.LastAppliedConfigAnnotationKey]).To(Equal(`[{"key":"archive_command","value":"''"}]`)) })).Should(Succeed()) By("disable schedule for reconfigure") @@ -365,7 +365,7 @@ var _ = Describe("Backup Policy Controller", func() { fetched.Spec.Schedule.Logfile.Enable = false })).Should(Succeed()) Eventually(testapps.CheckObj(&testCtx, backupPolicyKey, func(g Gomega, fetched *dpv1alpha1.BackupPolicy) { - g.Expect(fetched.Annotations[constant.LastAppliedConfigAnnotation]).To(Equal(`[{"key":"archive_command","value":"'/bin/true'"}]`)) + g.Expect(fetched.Annotations[constant.LastAppliedConfigAnnotationKey]).To(Equal(`[{"key":"archive_command","value":"'/bin/true'"}]`)) })).Should(Succeed()) }) }) diff --git a/internal/constant/const.go b/internal/constant/const.go index 5f72f61f3..cec54cab7 100644 --- a/internal/constant/const.go +++ b/internal/constant/const.go @@ -74,70 +74,67 @@ const ( ZoneLabelKey = "topology.kubernetes.io/zone" // kubeblocks.io labels - ClusterDefLabelKey = "clusterdefinition.kubeblocks.io/name" - KBAppComponentLabelKey = "apps.kubeblocks.io/component-name" - KBAppComponentDefRefLabelKey = "apps.kubeblocks.io/component-def-ref" - ConsensusSetAccessModeLabelKey = "cs.apps.kubeblocks.io/access-mode" - AppConfigTypeLabelKey = "apps.kubeblocks.io/config-type" - WorkloadTypeLabelKey = "apps.kubeblocks.io/workload-type" - VolumeClaimTemplateNameLabelKey = "apps.kubeblocks.io/vct-name" - PVCNameLabelKey = "apps.kubeblocks.io/pvc-name" - RoleLabelKey = "kubeblocks.io/role" // RoleLabelKey consensusSet and replicationSet role label key - BackupProtectionLabelKey = "kubeblocks.io/backup-protection" // BackupProtectionLabelKey Backup delete protection policy label - AddonNameLabelKey = "extensions.kubeblocks.io/addon-name" - ClusterAccountLabelKey = "account.kubeblocks.io/name" - VolumeTypeLabelKey = "kubeblocks.io/volume-type" - KBManagedByKey = "apps.kubeblocks.io/managed-by" // KBManagedByKey marks resources that auto created during operation - ClassProviderLabelKey = "class.kubeblocks.io/provider" - BackupToolTypeLabelKey = "kubeblocks.io/backup-tool-type" - BackupTypeLabelKeyKey = "dataprotection.kubeblocks.io/backup-type" - AddonProviderLableKey = "kubeblocks.io/provider" // AddonProviderLableKey marks the addon provider - OpsRequestTypeLabelKey = "ops.kubeblocks.io/ops-type" + BackupProtectionLabelKey = "kubeblocks.io/backup-protection" // BackupProtectionLabelKey Backup delete protection policy label + BackupToolTypeLabelKey = "kubeblocks.io/backup-tool-type" + AddonProviderLableKey = "kubeblocks.io/provider" // AddonProviderLableKey marks the addon provider + RoleLabelKey = "kubeblocks.io/role" // RoleLabelKey consensusSet and replicationSet role label key + VolumeTypeLabelKey = "kubeblocks.io/volume-type" + ClusterAccountLabelKey = "account.kubeblocks.io/name" + KBAppComponentLabelKey = "apps.kubeblocks.io/component-name" + KBAppComponentDefRefLabelKey = "apps.kubeblocks.io/component-def-ref" + AppConfigTypeLabelKey = "apps.kubeblocks.io/config-type" + KBManagedByKey = "apps.kubeblocks.io/managed-by" // KBManagedByKey marks resources that auto created during operation + PVCNameLabelKey = "apps.kubeblocks.io/pvc-name" + VolumeClaimTemplateNameLabelKey = "apps.kubeblocks.io/vct-name" + WorkloadTypeLabelKey = "apps.kubeblocks.io/workload-type" + ClassProviderLabelKey = "class.kubeblocks.io/provider" + ClusterDefLabelKey = "clusterdefinition.kubeblocks.io/name" + CMConfigurationSpecProviderLabelKey = "config.kubeblocks.io/config-spec" // CMConfigurationSpecProviderLabelKey is ComponentConfigSpec name + CMConfigurationCMKeysLabelKey = "config.kubeblocks.io/configmap-keys" // CMConfigurationCMKeysLabelKey Specify keys + CMConfigurationTemplateNameLabelKey = "config.kubeblocks.io/config-template-name" + CMConfigurationTypeLabelKey = "config.kubeblocks.io/config-type" + CMInsConfigurationHashLabelKey = "config.kubeblocks.io/config-hash" + CMConfigurationConstraintsNameLabelKey = "config.kubeblocks.io/config-constraints-name" + ConsensusSetAccessModeLabelKey = "cs.apps.kubeblocks.io/access-mode" + BackupTypeLabelKeyKey = "dataprotection.kubeblocks.io/backup-type" + AddonNameLabelKey = "extensions.kubeblocks.io/addon-name" + OpsRequestTypeLabelKey = "ops.kubeblocks.io/ops-type" // kubeblocks.io annotations - OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster - ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile - RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart - SnapShotForStartAnnotationKey = "kubeblocks.io/snapshot-for-start" - RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. - ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. - LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" - ComponentReplicasAnnotationKey = "apps.kubeblocks.io/component-replicas" // ComponentReplicasAnnotationKey specifies the number of pods in replicas - DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" // DefaultBackupPolicyAnnotationKey specifies the default backup policy. - DefaultBackupPolicyTemplateAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy-template" // DefaultBackupPolicyTemplateAnnotationKey specifies the default backup policy template. - BackupDataPathPrefixAnnotationKey = "dataprotection.kubeblocks.io/path-prefix" // BackupDataPathPrefixAnnotationKey specifies the backup data path prefix. - BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" - RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. - RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. - DefaultClusterVersionAnnotationKey = "kubeblocks.io/is-default-cluster-version" // DefaultClusterVersionAnnotationKey specifies the default cluster version. - PVLastClaimPolicyAnnotationKey = "apps.kubeblocks.io/pv-last-claim-policy" - ReconfigureRefAnnotationKey = "dataprotection.kubeblocks.io/reconfigure-ref" - - // ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl - ConfigurationTplLabelPrefixKey = "config.kubeblocks.io/tpl" - ConfigurationConstraintsLabelPrefixKey = "config.kubeblocks.io/constraints" - - LastAppliedOpsCRAnnotation = "config.kubeblocks.io/last-applied-ops-name" - LastAppliedConfigAnnotation = "config.kubeblocks.io/last-applied-configuration" + ClusterSnapshotAnnotationKey = "kubeblocks.io/cluster-snapshot" // ClusterSnapshotAnnotationKey saves the snapshot of cluster. + DefaultClusterVersionAnnotationKey = "kubeblocks.io/is-default-cluster-version" // DefaultClusterVersionAnnotationKey specifies the default cluster version. + OpsRequestAnnotationKey = "kubeblocks.io/ops-request" // OpsRequestAnnotationKey OpsRequest annotation key in Cluster + ReconcileAnnotationKey = "kubeblocks.io/reconcile" // ReconcileAnnotationKey Notify k8s object to reconcile + RestartAnnotationKey = "kubeblocks.io/restart" // RestartAnnotationKey the annotation which notices the StatefulSet/DeploySet to restart + RestoreFromTimeAnnotationKey = "kubeblocks.io/restore-from-time" // RestoreFromTimeAnnotationKey specifies the time to recover from the backup. + RestoreFromSrcClusterAnnotationKey = "kubeblocks.io/restore-from-source-cluster" // RestoreFromSrcClusterAnnotationKey specifies the source cluster to recover from the backup. + RestoreFromBackUpAnnotationKey = "kubeblocks.io/restore-from-backup" // RestoreFromBackUpAnnotationKey specifies the component to recover from the backup. + SnapShotForStartAnnotationKey = "kubeblocks.io/snapshot-for-start" + ComponentReplicasAnnotationKey = "apps.kubeblocks.io/component-replicas" // ComponentReplicasAnnotationKey specifies the number of pods in replicas + BackupPolicyTemplateAnnotationKey = "apps.kubeblocks.io/backup-policy-template" + LastAppliedClusterAnnotationKey = "apps.kubeblocks.io/last-applied-cluster" + PVLastClaimPolicyAnnotationKey = "apps.kubeblocks.io/pv-last-claim-policy" + HaltRecoveryAllowInconsistentCVAnnotKey = "clusters.apps.kubeblocks.io/allow-inconsistent-cv" + HaltRecoveryAllowInconsistentResAnnotKey = "clusters.apps.kubeblocks.io/allow-inconsistent-resource" + LeaderAnnotationKey = "cs.apps.kubeblocks.io/leader" + DefaultBackupPolicyAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy" // DefaultBackupPolicyAnnotationKey specifies the default backup policy. + DefaultBackupPolicyTemplateAnnotationKey = "dataprotection.kubeblocks.io/is-default-policy-template" // DefaultBackupPolicyTemplateAnnotationKey specifies the default backup policy template. + BackupDataPathPrefixAnnotationKey = "dataprotection.kubeblocks.io/path-prefix" // BackupDataPathPrefixAnnotationKey specifies the backup data path prefix. + ReconfigureRefAnnotationKey = "dataprotection.kubeblocks.io/reconfigure-ref" DisableUpgradeInsConfigurationAnnotationKey = "config.kubeblocks.io/disable-reconfigure" + LastAppliedConfigAnnotationKey = "config.kubeblocks.io/last-applied-configuration" + LastAppliedOpsCRAnnotationKey = "config.kubeblocks.io/last-applied-ops-name" UpgradePolicyAnnotationKey = "config.kubeblocks.io/reconfigure-policy" - UpgradeRestartAnnotationKey = "config.kubeblocks.io/restart" KBParameterUpdateSourceAnnotationKey = "config.kubeblocks.io/reconfigure-source" + UpgradeRestartAnnotationKey = "config.kubeblocks.io/restart" - // CMConfigurationTypeLabelKey configmap is config template type, e.g: "tpl", "instance" - CMConfigurationTypeLabelKey = "config.kubeblocks.io/config-type" - CMConfigurationTemplateNameLabelKey = "config.kubeblocks.io/config-template-name" - CMConfigurationConstraintsNameLabelKey = "config.kubeblocks.io/config-constraints-name" - CMInsConfigurationHashLabelKey = "config.kubeblocks.io/config-hash" - - // CMConfigurationSpecProviderLabelKey is ComponentConfigSpec name - CMConfigurationSpecProviderLabelKey = "config.kubeblocks.io/config-spec" - - // CMConfigurationCMKeysLabelKey Specify keys - CMConfigurationCMKeysLabelKey = "config.kubeblocks.io/configmap-keys" + // kubeblocks.io well-known finalizers + DBClusterFinalizerName = "cluster.kubeblocks.io/finalizer" + ConfigurationTemplateFinalizerName = "config.kubeblocks.io/finalizer" - // CMInsConfigurationLabelKey configmap is configuration file for component - // CMInsConfigurationLabelKey = "config.kubeblocks.io/ins-configure" + // ConfigurationTplLabelPrefixKey clusterVersion or clusterdefinition using tpl + ConfigurationTplLabelPrefixKey = "config.kubeblocks.io/tpl" + ConfigurationConstraintsLabelPrefixKey = "config.kubeblocks.io/constraints" // CMInsLastReconfigurePhaseKey defines the current phase CMInsLastReconfigurePhaseKey = "config.kubeblocks.io/last-applied-reconfigure-phase" @@ -145,9 +142,6 @@ const ( // CMInsEnableRerenderTemplateKey is used to enable rerender template CMInsEnableRerenderTemplateKey = "config.kubeblocks.io/enable-rerender" - // configuration finalizer - ConfigurationTemplateFinalizerName = "config.kubeblocks.io/finalizer" - // ClassAnnotationKey is used to specify the class of components ClassAnnotationKey = "cluster.kubeblocks.io/component-class" ) diff --git a/internal/controller/graph/transformer.go b/internal/controller/graph/transformer.go index 9ec6cb54d..9c56c6976 100644 --- a/internal/controller/graph/transformer.go +++ b/internal/controller/graph/transformer.go @@ -45,22 +45,22 @@ type Transformer interface { // TransformerChain chains a group Transformer together type TransformerChain []Transformer -// ErrNoops is used to stop the Transformer chain for some purpose. +// ErrPrematureStop is used to stop the Transformer chain for some purpose. // Use it in Transformer.Transform when all jobs have done and no need to run following transformers -var ErrNoops = errors.New("No-Ops") +var ErrPrematureStop = errors.New("Premature-Stop") // ApplyTo applies TransformerChain t to dag func (r TransformerChain) ApplyTo(ctx TransformContext, dag *DAG) error { for _, transformer := range r { if err := transformer.Transform(ctx, dag); err != nil { - return ignoredIfNoops(err) + return ignoredIfPrematureStop(err) } } return nil } -func ignoredIfNoops(err error) error { - if err == ErrNoops { +func ignoredIfPrematureStop(err error) error { + if err == ErrPrematureStop { return nil } return err diff --git a/internal/controller/lifecycle/cluster_plan_builder.go b/internal/controller/lifecycle/cluster_plan_builder.go index 00a092dde..c36510e61 100644 --- a/internal/controller/lifecycle/cluster_plan_builder.go +++ b/internal/controller/lifecycle/cluster_plan_builder.go @@ -264,7 +264,7 @@ func (c *clusterPlanBuilder) defaultWalkFunc(vertex graph.Vertex) error { return err } case DELETE: - if controllerutil.RemoveFinalizer(node.obj, dbClusterFinalizerName) { + if controllerutil.RemoveFinalizer(node.obj, constant.DBClusterFinalizerName) { err := c.cli.Update(c.transCtx.Context, node.obj) if err != nil && !apierrors.IsNotFound(err) { c.transCtx.Logger.Error(err, fmt.Sprintf("delete %T error: %s", node.obj, node.obj.GetName())) @@ -311,10 +311,9 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec *stsObj.Spec.Replicas, *stsProto.Spec.Replicas) } - // keep the original template annotations. - // if annotations exist and are replaced, the statefulSet will be updated. - mergeAnnotations(stsObj.Spec.Template.Annotations, - &stsProto.Spec.Template.Annotations) + // merge stsObj.Spec.Template.Annotations to stsProto.Spec.Template.Annotations + // then reassign it with stsObj.Spec.Template = stsProto.Spec.Template + mergeAnnotations(stsObj.Spec.Template.Annotations, &stsProto.Spec.Template.Annotations) stsObj.Spec.Template = stsProto.Spec.Template stsObj.Spec.Replicas = stsProto.Spec.Replicas stsObj.Spec.UpdateStrategy = stsProto.Spec.UpdateStrategy @@ -323,8 +322,9 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec handleDeploy := func(origObj, deployProto *appsv1.Deployment) (client.Object, error) { deployObj := origObj.DeepCopy() - mergeAnnotations(deployObj.Spec.Template.Annotations, - &deployProto.Spec.Template.Annotations) + // merge deployObj.Spec.Template.Annotations to deployProto.Spec.Template.Annotations + // then reassign it with deployObj.Spec = deployProto.Spec + mergeAnnotations(deployObj.Spec.Template.Annotations, &deployProto.Spec.Template.Annotations) deployObj.Spec = deployProto.Spec return deployObj, nil } @@ -332,7 +332,7 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec handleSvc := func(origObj, svcProto *corev1.Service) (client.Object, error) { svcObj := origObj.DeepCopy() svcObj.Spec = svcProto.Spec - svcObj.Annotations = mergeServiceAnnotations(svcObj.Annotations, svcProto.Annotations) + mergeServiceAnnotations(svcProto.Annotations, &svcObj.Annotations) return svcObj, nil } @@ -343,6 +343,15 @@ func (c *clusterPlanBuilder) buildUpdateObj(node *lifecycleVertex) (client.Objec } else { pvcObj.Spec.Resources.Requests[corev1.ResourceStorage] = pvcProto.Spec.Resources.Requests[corev1.ResourceStorage] } + // if proto object is a real object resources, simply override annotations, this could happen + // due to ClusterDeletionTransformer may result preserved PVC objects when cluster termination + // policy is equal to Halt. + if pvcProto.UID != "" { + pvcObj.Annotations = pvcProto.Annotations + pvcObj.OwnerReferences = pvcProto.OwnerReferences + } else { + mergeAnnotations(pvcProto.Annotations, &pvcObj.Annotations) + } return pvcObj, nil } diff --git a/internal/controller/lifecycle/cluster_plan_utils.go b/internal/controller/lifecycle/cluster_plan_utils.go index b24e14c71..595b9e9fb 100644 --- a/internal/controller/lifecycle/cluster_plan_utils.go +++ b/internal/controller/lifecycle/cluster_plan_utils.go @@ -21,39 +21,45 @@ package lifecycle import ( "strings" - - "golang.org/x/exp/maps" ) // mergeAnnotations keeps the original annotations. -// if annotations exist and are replaced, the Deployment/StatefulSet will be updated. -func mergeAnnotations(originalAnnotations map[string]string, targetAnnotations *map[string]string) { +// if annotations exist and are replaced. +func mergeAnnotations(originalAnnotations map[string]string, targetAnnotations *map[string]string, filters ...func(k, v string) bool) { if targetAnnotations == nil { return } + if len(originalAnnotations) == 0 { + return + } if *targetAnnotations == nil { *targetAnnotations = map[string]string{} } for k, v := range originalAnnotations { + filtered := false + for _, filter := range filters { + if filter != nil && filter(k, v) { + filtered = true + break + } + } + if filtered { + continue + } + // if the annotation not exist in targetAnnotations, copy it from original. if _, ok := (*targetAnnotations)[k]; !ok { (*targetAnnotations)[k] = v + continue } } } // mergeServiceAnnotations keeps the original annotations except prometheus scrape annotations. // if annotations exist and are replaced, the Service will be updated. -func mergeServiceAnnotations(originalAnnotations, targetAnnotations map[string]string) map[string]string { - if len(originalAnnotations) == 0 { - return targetAnnotations - } - tmpAnnotations := make(map[string]string, len(originalAnnotations)+len(targetAnnotations)) - for k, v := range originalAnnotations { - if !strings.HasPrefix(k, "prometheus.io") { - tmpAnnotations[k] = v - } - } - maps.Copy(tmpAnnotations, targetAnnotations) - return tmpAnnotations +func mergeServiceAnnotations(originalAnnotations map[string]string, targetAnnotations *map[string]string) { + mergeAnnotations(originalAnnotations, targetAnnotations, + func(k, v string) bool { + return strings.HasPrefix(k, "prometheus.io") + }) } diff --git a/internal/controller/lifecycle/cluster_plan_utils_test.go b/internal/controller/lifecycle/cluster_plan_utils_test.go index 01f16c688..f48504073 100644 --- a/internal/controller/lifecycle/cluster_plan_utils_test.go +++ b/internal/controller/lifecycle/cluster_plan_utils_test.go @@ -27,27 +27,30 @@ import ( var _ = Describe("cluster plan utils test", func() { Context("test mergeServiceAnnotations", func() { It("original and target annotations are nil", func() { - Expect(mergeServiceAnnotations(nil, nil)).Should(BeNil()) + mergeServiceAnnotations(nil, nil) }) It("target annotations is nil", func() { originalAnnotations := map[string]string{"k1": "v1"} - Expect(mergeServiceAnnotations(originalAnnotations, nil)).To(Equal(originalAnnotations)) + mergeServiceAnnotations(originalAnnotations, nil) }) It("original annotations is nil", func() { targetAnnotations := map[string]string{"k1": "v1"} - Expect(mergeServiceAnnotations(nil, targetAnnotations)).To(Equal(targetAnnotations)) + mergeServiceAnnotations(nil, &targetAnnotations) + Expect(targetAnnotations).To(Equal(targetAnnotations)) }) It("original annotations have prometheus annotations which should be removed", func() { originalAnnotations := map[string]string{"k1": "v1", "prometheus.io/path": "/metrics"} targetAnnotations := map[string]string{"k2": "v2"} expectAnnotations := map[string]string{"k1": "v1", "k2": "v2"} - Expect(mergeServiceAnnotations(originalAnnotations, targetAnnotations)).To(Equal(expectAnnotations)) + mergeServiceAnnotations(originalAnnotations, &targetAnnotations) + Expect(targetAnnotations).To(Equal(expectAnnotations)) }) It("target annotations should override original annotations", func() { originalAnnotations := map[string]string{"k1": "v1", "prometheus.io/path": "/metrics"} targetAnnotations := map[string]string{"k1": "v11"} expectAnnotations := map[string]string{"k1": "v11"} - Expect(mergeServiceAnnotations(originalAnnotations, targetAnnotations)).To(Equal(expectAnnotations)) + mergeServiceAnnotations(originalAnnotations, &targetAnnotations) + Expect(targetAnnotations).To(Equal(expectAnnotations)) }) It("should merge annotations from original that not exist in target to final result", func() { diff --git a/internal/controller/lifecycle/transform_types.go b/internal/controller/lifecycle/transform_types.go index 5a27536d5..163cbe680 100644 --- a/internal/controller/lifecycle/transform_types.go +++ b/internal/controller/lifecycle/transform_types.go @@ -51,7 +51,6 @@ func init() { const ( // TODO: deduplicate - dbClusterFinalizerName = "cluster.kubeblocks.io/finalizer" clusterDefLabelKey = "clusterdefinition.kubeblocks.io/name" clusterVersionLabelKey = "clusterversion.kubeblocks.io/name" ) @@ -69,9 +68,9 @@ const ( // default reconcile requeue after duration var requeueDuration = time.Millisecond * 100 -type gvkName struct { - gvk schema.GroupVersionKind - ns, name string +type gvkNObjKey struct { + schema.GroupVersionKind + client.ObjectKey } // lifecycleVertex describes expected object spec and how to reach it @@ -97,7 +96,7 @@ func (v lifecycleVertex) String() string { return fmt.Sprintf("{obj:%T, immutable: %v, action: %v}", v.obj, v.immutable, *v.action) } -type clusterSnapshot map[gvkName]client.Object +type clusterOwningObjects map[gvkNObjKey]client.Object type RequeueError interface { RequeueAfter() time.Duration diff --git a/internal/controller/lifecycle/transform_utils.go b/internal/controller/lifecycle/transform_utils.go index 3461c29ef..61aa2283f 100644 --- a/internal/controller/lifecycle/transform_utils.go +++ b/internal/controller/lifecycle/transform_utils.go @@ -73,15 +73,17 @@ func findRootVertex(dag *graph.DAG) (*lifecycleVertex, error) { return rootVertex, nil } -func getGVKName(object client.Object, scheme *runtime.Scheme) (*gvkName, error) { +func getGVKName(object client.Object, scheme *runtime.Scheme) (*gvkNObjKey, error) { gvk, err := apiutil.GVKForObject(object, scheme) if err != nil { return nil, err } - return &gvkName{ - gvk: gvk, - ns: object.GetNamespace(), - name: object.GetName(), + return &gvkNObjKey{ + GroupVersionKind: gvk, + ObjectKey: client.ObjectKey{ + Namespace: object.GetNamespace(), + Name: object.GetName(), + }, }, nil } @@ -176,10 +178,11 @@ func getAppInstanceAndManagedByML(cluster appsv1alpha1.Cluster) client.MatchingL } } -// read all objects owned by our cluster -func readCacheSnapshot(transCtx *ClusterTransformContext, cluster appsv1alpha1.Cluster, matchLabels client.MatchingLabels, kinds ...client.ObjectList) (clusterSnapshot, error) { +// getClusterOwningObjects read objects owned by our cluster with kinds and label matching specifier. +func getClusterOwningObjects(transCtx *ClusterTransformContext, cluster appsv1alpha1.Cluster, + matchLabels client.MatchingLabels, kinds ...client.ObjectList) (clusterOwningObjects, error) { // list what kinds of object cluster owns - snapshot := make(clusterSnapshot) + objs := make(clusterOwningObjects) inNS := client.InNamespace(cluster.Namespace) for _, list := range kinds { if err := transCtx.Client.List(transCtx.Context, list, inNS, matchLabels); err != nil { @@ -195,11 +198,10 @@ func readCacheSnapshot(transCtx *ClusterTransformContext, cluster appsv1alpha1.C if err != nil { return nil, err } - snapshot[*name] = object + objs[*name] = object } } - - return snapshot, nil + return objs, nil } // sendWaringEventForCluster sends a warning event when occurs error. diff --git a/internal/controller/lifecycle/transformer_fix_meta.go b/internal/controller/lifecycle/transformer_assure_meta.go similarity index 81% rename from internal/controller/lifecycle/transformer_fix_meta.go rename to internal/controller/lifecycle/transformer_assure_meta.go index a89d7311d..81e29df59 100644 --- a/internal/controller/lifecycle/transformer_fix_meta.go +++ b/internal/controller/lifecycle/transformer_assure_meta.go @@ -22,20 +22,21 @@ package lifecycle import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/graph" ) -type FixMetaTransformer struct{} +type AssureMetaTransformer struct{} -func (t *FixMetaTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { +func (t *AssureMetaTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { transCtx, _ := ctx.(*ClusterTransformContext) cluster := transCtx.Cluster // The object is not being deleted, so if it does not have our finalizer, // then lets add the finalizer and update the object. This is equivalent // registering our finalizer. - if !controllerutil.ContainsFinalizer(cluster, dbClusterFinalizerName) { - controllerutil.AddFinalizer(cluster, dbClusterFinalizerName) + if !controllerutil.ContainsFinalizer(cluster, constant.DBClusterFinalizerName) { + controllerutil.AddFinalizer(cluster, constant.DBClusterFinalizerName) } // patch the label to prevent the label from being modified by the user. @@ -56,4 +57,4 @@ func (t *FixMetaTransformer) Transform(ctx graph.TransformContext, dag *graph.DA return nil } -var _ graph.Transformer = &FixMetaTransformer{} +var _ graph.Transformer = &AssureMetaTransformer{} diff --git a/internal/controller/lifecycle/transformer_backup_policy_tpl.go b/internal/controller/lifecycle/transformer_backup_policy_tpl.go index 94a12301a..291b079f3 100644 --- a/internal/controller/lifecycle/transformer_backup_policy_tpl.go +++ b/internal/controller/lifecycle/transformer_backup_policy_tpl.go @@ -43,6 +43,10 @@ type BackupPolicyTPLTransformer struct { isDefaultTemplate string } +const ( + trueVal = "true" +) + func (r *BackupPolicyTPLTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { transCtx, _ := ctx.(*ClusterTransformContext) clusterDefName := transCtx.ClusterDef.Name @@ -367,10 +371,10 @@ func (r *BackupPolicyTPLTransformer) convertCommonPolicy(bp *appsv1alpha1.Common } func (r *BackupPolicyTPLTransformer) defaultPolicyAnnotationValue() string { - if r.tplCount > 1 && r.isDefaultTemplate != "true" { + if r.tplCount > 1 && r.isDefaultTemplate != trueVal { return "false" } - return "true" + return trueVal } // DeriveBackupPolicyName generates the backup policy name which is created from backup policy template. diff --git a/internal/controller/lifecycle/transformer_cluster_deletion.go b/internal/controller/lifecycle/transformer_cluster_deletion.go index ef3096a92..31414163f 100644 --- a/internal/controller/lifecycle/transformer_cluster_deletion.go +++ b/internal/controller/lifecycle/transformer_cluster_deletion.go @@ -20,15 +20,18 @@ along with this program. If not, see . package lifecycle import ( + "encoding/json" "strings" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" dataprotectionv1alpha1 "github.com/apecloud/kubeblocks/apis/dataprotection/v1alpha1" "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/graph" @@ -43,20 +46,29 @@ func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * if !isClusterDeleting(*cluster) { return nil } + root, err := findRootVertex(dag) + if err != nil { + return err + } // list all kinds to be deleted based on v1alpha1.TerminationPolicyType - kinds := make([]client.ObjectList, 0) + var toDeleteKinds, toPreserveKinds []client.ObjectList switch cluster.Spec.TerminationPolicy { - case v1alpha1.DoNotTerminate: + case appsv1alpha1.DoNotTerminate: transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeWarning, "DoNotTerminate", "spec.terminationPolicy %s is preventing deletion.", cluster.Spec.TerminationPolicy) - return graph.ErrNoops - case v1alpha1.Halt: - kinds = kindsForHalt() - case v1alpha1.Delete: - kinds = kindsForDelete() - case v1alpha1.WipeOut: - kinds = kindsForWipeOut() + return graph.ErrPrematureStop + case appsv1alpha1.Halt: + toDeleteKinds = kindsForHalt() + toPreserveKinds = []client.ObjectList{ + &corev1.PersistentVolumeClaimList{}, + &corev1.SecretList{}, + &corev1.ConfigMapList{}, + } + case appsv1alpha1.Delete: + toDeleteKinds = kindsForDelete() + case appsv1alpha1.WipeOut: + toDeleteKinds = kindsForWipeOut() } transCtx.EventRecorder.Eventf(cluster, corev1.EventTypeNormal, constant.ReasonDeletingCR, "Deleting %s: %s", @@ -67,23 +79,73 @@ func (t *ClusterDeletionTransformer) Transform(ctx graph.TransformContext, dag * // ignore the problem currently // TODO: GC the leaked objects ml := getAppInstanceML(*cluster) - snapshot, err := readCacheSnapshot(transCtx, *cluster, ml, kinds...) - if err != nil { + + preserveObjects := func() error { + if len(toPreserveKinds) == 0 { + return nil + } + + objs, err := getClusterOwningObjects(transCtx, *cluster, ml, toPreserveKinds...) + if err != nil { + return err + } + // construct cluster spec JSON string + clusterSpec := cluster.DeepCopy() + clusterSpec.ObjectMeta = metav1.ObjectMeta{ + Name: cluster.GetName(), + UID: cluster.GetUID(), + } + clusterSpec.Status = appsv1alpha1.ClusterStatus{} + b, err := json.Marshal(*clusterSpec) + if err != nil { + return err + } + clusterJSON := string(b) + for _, o := range objs { + origObj := o.DeepCopyObject().(client.Object) + controllerutil.RemoveFinalizer(o, constant.DBClusterFinalizerName) + ownerRefs := o.GetOwnerReferences() + for i, ref := range ownerRefs { + if ref.Kind != appsv1alpha1.ClusterKind || + !strings.Contains(ref.APIVersion, appsv1alpha1.GroupVersion.Group) { + continue + } + ownerRefs = append(ownerRefs[:i], ownerRefs[i+1:]...) + break + } + o.SetOwnerReferences(ownerRefs) + annot := o.GetAnnotations() + if annot == nil { + annot = map[string]string{} + } + // annotated last-applied Cluster spec + annot[constant.LastAppliedClusterAnnotationKey] = clusterJSON + o.SetAnnotations(annot) + vertex := &lifecycleVertex{obj: o, oriObj: origObj, action: actionPtr(UPDATE)} + dag.AddVertex(vertex) + dag.Connect(root, vertex) + } + return nil + } + // handle preserved objects update vertex + if err := preserveObjects(); err != nil { return err } - root, err := findRootVertex(dag) + + // add objects deletion vertex + objs, err := getClusterOwningObjects(transCtx, *cluster, ml, toDeleteKinds...) if err != nil { return err } - for _, object := range snapshot { - vertex := &lifecycleVertex{obj: object, action: actionPtr(DELETE)} + for _, o := range objs { + vertex := &lifecycleVertex{obj: o, action: actionPtr(DELETE)} dag.AddVertex(vertex) dag.Connect(root, vertex) } root.action = actionPtr(DELETE) // fast return, that is stopping the plan.Build() stage and jump to plan.Execute() directly - return graph.ErrNoops + return graph.ErrPrematureStop } func kindsForDoNotTerminate() []client.ObjectList { @@ -97,8 +159,8 @@ func kindsForHalt() []client.ObjectList { &corev1.ServiceList{}, &appsv1.StatefulSetList{}, &appsv1.DeploymentList{}, - &corev1.SecretList{}, - &corev1.ConfigMapList{}, + &corev1.ServiceList{}, + &policyv1.PodDisruptionBudgetList{}, } return append(kinds, kindsPlus...) } @@ -106,6 +168,8 @@ func kindsForHalt() []client.ObjectList { func kindsForDelete() []client.ObjectList { kinds := kindsForHalt() kindsPlus := []client.ObjectList{ + &corev1.SecretList{}, + &corev1.ConfigMapList{}, &corev1.PersistentVolumeClaimList{}, &dataprotectionv1alpha1.BackupPolicyList{}, &batchv1.JobList{}, diff --git a/internal/controller/lifecycle/transformer_halt_recovering.go b/internal/controller/lifecycle/transformer_halt_recovering.go new file mode 100644 index 000000000..6b930e691 --- /dev/null +++ b/internal/controller/lifecycle/transformer_halt_recovering.go @@ -0,0 +1,213 @@ +/* +Copyright (C) 2022-2023 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package lifecycle + +import ( + "encoding/json" + "fmt" + + "github.com/authzed/controller-idioms/hash" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" + "github.com/apecloud/kubeblocks/internal/controller/graph" +) + +type HaltRecoveryTransformer struct{} + +func (t *HaltRecoveryTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { + transCtx, _ := ctx.(*ClusterTransformContext) + cluster := transCtx.Cluster + + if cluster.Status.ObservedGeneration != 0 { + // skip handling for cluster.status.observedGeneration > 0 + return nil + } + + listOptions := []client.ListOption{ + client.InNamespace(cluster.Namespace), + client.MatchingLabels{ + constant.AppInstanceLabelKey: cluster.Name, + }, + } + + pvcList := &corev1.PersistentVolumeClaimList{} + if err := transCtx.Client.List(transCtx.Context, pvcList, listOptions...); err != nil { + return newRequeueError(requeueDuration, err.Error()) + } + + if len(pvcList.Items) == 0 { + return nil + } + + emitError := func(newCondition metav1.Condition) error { + if newCondition.LastTransitionTime.IsZero() { + newCondition.LastTransitionTime = metav1.Now() + } + newCondition.Status = metav1.ConditionFalse + oldCondition := meta.FindStatusCondition(cluster.Status.Conditions, newCondition.Type) + if oldCondition == nil { + cluster.Status.Conditions = append(cluster.Status.Conditions, newCondition) + } else { + *oldCondition = newCondition + } + transCtx.EventRecorder.Event(transCtx.Cluster, corev1.EventTypeWarning, newCondition.Reason, newCondition.Message) + return graph.ErrPrematureStop + } + + // halt recovering from last applied record stored in pvc's annotation + l, ok := pvcList.Items[0].Annotations[constant.LastAppliedClusterAnnotationKey] + if !ok || l == "" { + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "UncleanedResources", + Message: fmt.Sprintf("found uncleaned resources, requires manual deletion, check with `kubectl -n %s get pvc,secret,cm -l %s=%s`", + cluster.Namespace, constant.AppInstanceLabelKey, cluster.Name), + }) + } + + lc := &appsv1alpha1.Cluster{} + if err := json.Unmarshal([]byte(l), lc); err != nil { + return newRequeueError(requeueDuration, err.Error()) + } + + // skip if same cluster UID + if lc.UID == cluster.UID { + return nil + } + + // check clusterDefRef equality + if cluster.Spec.ClusterDefRef != lc.Spec.ClusterDefRef { + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("not equal to last applied cluster.spec.clusterDefRef %s", lc.Spec.ClusterDefRef), + }) + } + + // check clusterVersionRef equality but allow clusters.apps.kubeblocks.io/allow-inconsistent-cv=true annotation override + if cluster.Spec.ClusterVersionRef != lc.Spec.ClusterVersionRef && + cluster.Annotations[constant.HaltRecoveryAllowInconsistentCVAnnotKey] != trueVal { + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("not equal to last applied cluster.spec.clusterVersionRef %s; add '%s=true' annotation if void this check", + lc.Spec.ClusterVersionRef, constant.HaltRecoveryAllowInconsistentCVAnnotKey), + }) + } + + // check component len equality + if l := len(lc.Spec.ComponentSpecs); l != len(cluster.Spec.ComponentSpecs) { + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("inconsistent spec.componentSpecs counts to last applied cluster.spec.componentSpecs (len=%d)", l), + }) + } + + // check every components' equality + for _, comp := range cluster.Spec.ComponentSpecs { + found := false + for _, lastUsedComp := range lc.Spec.ComponentSpecs { + // only need to verify [name, componentDefRef, replicas] for equality + if comp.Name != lastUsedComp.Name { + continue + } + if comp.ComponentDefRef != lastUsedComp.ComponentDefRef { + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("not equal to last applied cluster.spec.componetSpecs[%s].componentDefRef=%s", + comp.Name, lastUsedComp.ComponentDefRef), + }) + } + if comp.Replicas != lastUsedComp.Replicas { + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("not equal to last applied cluster.spec.componetSpecs[%s].replicas=%d", + comp.Name, lastUsedComp.Replicas), + }) + } + + // following only check resource related spec., will skip check if HaltRecoveryAllowInconsistentResAnnotKey + // annotation is specified + if cluster.Annotations[constant.HaltRecoveryAllowInconsistentResAnnotKey] == trueVal { + found = true + break + } + if hash.Object(comp.VolumeClaimTemplates) != hash.Object(lastUsedComp.VolumeClaimTemplates) { + objJSON, _ := json.Marshal(&lastUsedComp.VolumeClaimTemplates) + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("not equal to last applied cluster.spec.componetSpecs[%s].volumeClaimTemplates=%s; add '%s=true' annotation to void this check", + comp.Name, objJSON, constant.HaltRecoveryAllowInconsistentResAnnotKey), + }) + } + + if lastUsedComp.ClassDefRef != nil { + if comp.ClassDefRef == nil || hash.Object(*comp.ClassDefRef) != hash.Object(*lastUsedComp.ClassDefRef) { + objJSON, _ := json.Marshal(lastUsedComp.ClassDefRef) + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("not equal to last applied cluster.spec.componetSpecs[%s].classDefRef=%s; add '%s=true' annotation to void this check", + comp.Name, objJSON, constant.HaltRecoveryAllowInconsistentResAnnotKey), + }) + } + } else if comp.ClassDefRef != nil { + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("not equal to last applied cluster.spec.componetSpecs[%s].classDefRef=null; add '%s=true' annotation to void this check", + comp.Name, constant.HaltRecoveryAllowInconsistentResAnnotKey), + }) + } + + if hash.Object(comp.Resources) != hash.Object(lastUsedComp.Resources) { + objJSON, _ := json.Marshal(&lastUsedComp.Resources) + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("not equal to last applied cluster.spec.componetSpecs[%s].resources=%s; add '%s=true' annotation to void this check", + comp.Name, objJSON, constant.HaltRecoveryAllowInconsistentResAnnotKey), + }) + } + found = true + break + } + if !found { + return emitError(metav1.Condition{ + Type: appsv1alpha1.ConditionTypeHaltRecovery, + Reason: "HaltRecoveryFailed", + Message: fmt.Sprintf("cluster.spec.componetSpecs[%s] not found in last applied cluster", + comp.Name), + }) + } + } + return nil +} + +var _ graph.Transformer = &HaltRecoveryTransformer{} diff --git a/internal/controller/lifecycle/transformer_object_action.go b/internal/controller/lifecycle/transformer_object_action.go index 182c3b2d1..2f2c0cec2 100644 --- a/internal/controller/lifecycle/transformer_object_action.go +++ b/internal/controller/lifecycle/transformer_object_action.go @@ -38,7 +38,7 @@ func (t *ObjectActionTransformer) Transform(ctx graph.TransformContext, dag *gra // get the old objects snapshot ml := getAppInstanceAndManagedByML(*origCluster) - oldSnapshot, err := readCacheSnapshot(transCtx, *origCluster, ml, ownKinds()...) + oldSnapshot, err := getClusterOwningObjects(transCtx, *origCluster, ml, ownKinds()...) if err != nil { return err } @@ -49,7 +49,7 @@ func (t *ObjectActionTransformer) Transform(ctx graph.TransformContext, dag *gra } // we have the target objects snapshot in dag - newNameVertices := make(map[gvkName]graph.Vertex) + newNameVertices := make(map[gvkNObjKey]graph.Vertex) for _, vertex := range dag.Vertices() { v, _ := vertex.(*lifecycleVertex) if v == rootVertex { diff --git a/internal/controller/lifecycle/transformer_ownership.go b/internal/controller/lifecycle/transformer_ownership.go index 7e7706eb0..030647674 100644 --- a/internal/controller/lifecycle/transformer_ownership.go +++ b/internal/controller/lifecycle/transformer_ownership.go @@ -23,6 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/graph" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" ) @@ -37,10 +38,10 @@ func (f *OwnershipTransformer) Transform(ctx graph.TransformContext, dag *graph. } vertices := findAllNot[*appsv1alpha1.Cluster](dag) - controllerutil.AddFinalizer(rootVertex.obj, dbClusterFinalizerName) + controllerutil.AddFinalizer(rootVertex.obj, constant.DBClusterFinalizerName) for _, vertex := range vertices { v, _ := vertex.(*lifecycleVertex) - if err := intctrlutil.SetOwnership(rootVertex.obj, v.obj, scheme, dbClusterFinalizerName); err != nil { + if err := intctrlutil.SetOwnership(rootVertex.obj, v.obj, scheme, constant.DBClusterFinalizerName); err != nil { return err } } diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go index 4bd5d939e..092d971e9 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling.go @@ -266,7 +266,7 @@ func (t *StsHorizontalScalingTransformer) Transform(ctx graph.TransformContext, return nil } - findPVCsToBeDeleted := func(pvcSnapshot clusterSnapshot) []*corev1.PersistentVolumeClaim { + findPVCsToBeDeleted := func(pvcSnapshot clusterOwningObjects) []*corev1.PersistentVolumeClaim { stsToBeDeleted := make([]*appsv1.StatefulSet, 0) // list sts to be deleted for _, vertex := range dag.Vertices() { @@ -321,7 +321,7 @@ func (t *StsHorizontalScalingTransformer) Transform(ctx graph.TransformContext, // by h-scale transformer: we handle the pvc creation and deletion, the creation is handled in h-scale funcs. // so all in all, here we should only handle the pvc deletion of both types. ml := getAppInstanceML(*cluster) - oldSnapshot, err := readCacheSnapshot(transCtx, *cluster, ml, &corev1.PersistentVolumeClaimList{}) + oldSnapshot, err := getClusterOwningObjects(transCtx, *cluster, ml, &corev1.PersistentVolumeClaimList{}) if err != nil { return err } @@ -808,7 +808,7 @@ func createPVCFromSnapshot(vct corev1.PersistentVolumeClaimTemplate, } rootVertex, _ := root.(*lifecycleVertex) cluster, _ := rootVertex.obj.(*appsv1alpha1.Cluster) - if err = intctrlutil.SetOwnership(cluster, pvc, scheme, dbClusterFinalizerName); err != nil { + if err = intctrlutil.SetOwnership(cluster, pvc, scheme, constant.DBClusterFinalizerName); err != nil { return err } vertex := &lifecycleVertex{obj: pvc, action: actionPtr(CREATE)} diff --git a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go index 5cec9d8d1..f553d42c8 100644 --- a/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go +++ b/internal/controller/lifecycle/transformer_sts_horizontal_scaling_test.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/apecloud/kubeblocks/internal/constant" "github.com/apecloud/kubeblocks/internal/controller/graph" intctrlutil "github.com/apecloud/kubeblocks/internal/controllerutil" "github.com/apecloud/kubeblocks/internal/testutil/apps" @@ -70,10 +71,10 @@ var _ = Describe("sts horizontal scaling test", func() { pvc1 := apps.NewPersistentVolumeClaimFactory(namespace, pvcNameBase+"1", cluster.Name, componentName, volumeName). AddAppInstanceLabel(clusterName). GetObject() - Expect(intctrlutil.SetOwnership(cluster, pvc1, scheme, dbClusterFinalizerName)).Should(Succeed()) + Expect(intctrlutil.SetOwnership(cluster, pvc1, scheme, constant.DBClusterFinalizerName)).Should(Succeed()) pvc2 := pvc1.DeepCopy() pvc2.Name = pvcNameBase + "2" - Expect(intctrlutil.SetOwnership(cluster, pvc2, scheme, dbClusterFinalizerName)).Should(Succeed()) + Expect(intctrlutil.SetOwnership(cluster, pvc2, scheme, constant.DBClusterFinalizerName)).Should(Succeed()) By("prepare params for transformer") ctrl, k8sMock := testutil.SetupK8sMock() diff --git a/internal/controllerutil/controller_common.go b/internal/controllerutil/controller_common.go index 1e91e26ac..4f9319b66 100644 --- a/internal/controllerutil/controller_common.go +++ b/internal/controllerutil/controller_common.go @@ -240,10 +240,17 @@ func BackgroundDeleteObject(cli client.Client, ctx context.Context, obj client.O return nil } -// SetOwnership set owner reference and add finalizer if not exists -func SetOwnership(owner, obj client.Object, scheme *runtime.Scheme, finalizer string) error { - if err := controllerutil.SetControllerReference(owner, obj, scheme); err != nil { - return err +// SetOwnership provides helper function controllerutil.SetControllerReference/controllerutil.SetOwnerReference +// and controllerutil.AddFinalizer if not exists. +func SetOwnership(owner, obj client.Object, scheme *runtime.Scheme, finalizer string, useOwnerReference ...bool) error { + if len(useOwnerReference) > 0 && useOwnerReference[0] { + if err := controllerutil.SetOwnerReference(owner, obj, scheme); err != nil { + return err + } + } else { + if err := controllerutil.SetControllerReference(owner, obj, scheme); err != nil { + return err + } } if !controllerutil.ContainsFinalizer(obj, finalizer) { // pvc objects do not need to add finalizer diff --git a/internal/testutil/apps/cluster_consensus_test_util.go b/internal/testutil/apps/cluster_consensus_test_util.go index 1f935e33a..5a4a048a6 100644 --- a/internal/testutil/apps/cluster_consensus_test_util.go +++ b/internal/testutil/apps/cluster_consensus_test_util.go @@ -34,10 +34,11 @@ import ( ) const ( - errorLogName = "error" - leader = "leader" - follower = "follower" - learner = "learner" + errorLogName = "error" + leader = "leader" + follower = "follower" + learner = "learner" + ConsensusReplicas = 3 ) // InitConsensusMysql initializes a cluster environment which only contains a component of ConsensusSet type for testing, @@ -61,10 +62,14 @@ func CreateConsensusMysqlCluster( clusterVersionName, clusterName, workloadType, - consensusCompName string) *appsv1alpha1.Cluster { - pvcSpec := NewPVCSpec("2Gi") + consensusCompName string, pvcSize ...string) *appsv1alpha1.Cluster { + size := "2Gi" + if len(pvcSize) > 0 { + size = pvcSize[0] + } + pvcSpec := NewPVCSpec(size) return NewClusterFactory(testCtx.DefaultNamespace, clusterName, clusterDefName, clusterVersionName). - AddComponent(consensusCompName, workloadType).SetReplicas(3).SetEnabledLogs(errorLogName). + AddComponent(consensusCompName, workloadType).SetReplicas(ConsensusReplicas).SetEnabledLogs(errorLogName). AddVolumeClaimTemplate("data", pvcSpec).Create(testCtx).GetObject() } @@ -87,7 +92,7 @@ func MockConsensusComponentStatefulSet( clusterName, consensusCompName string) *appsv1.StatefulSet { stsName := clusterName + "-" + consensusCompName - return NewStatefulSetFactory(testCtx.DefaultNamespace, stsName, clusterName, consensusCompName).SetReplicas(int32(3)). + return NewStatefulSetFactory(testCtx.DefaultNamespace, stsName, clusterName, consensusCompName).SetReplicas(ConsensusReplicas). AddContainer(corev1.Container{Name: DefaultMySQLContainerName, Image: ApeCloudMySQLImage}).Create(testCtx).GetObject() } @@ -130,8 +135,8 @@ func MockConsensusComponentPods( sts *appsv1.StatefulSet, clusterName, consensusCompName string) []*corev1.Pod { - podList := make([]*corev1.Pod, 3) - for i := 0; i < 3; i++ { + podList := make([]*corev1.Pod, ConsensusReplicas) + for i := 0; i < ConsensusReplicas; i++ { podName := fmt.Sprintf("%s-%s-%d", clusterName, consensusCompName, i) podRole := "follower" accessMode := "Readonly" diff --git a/internal/testutil/apps/common_util.go b/internal/testutil/apps/common_util.go index 416d2004c..1e56df7ab 100644 --- a/internal/testutil/apps/common_util.go +++ b/internal/testutil/apps/common_util.go @@ -52,7 +52,7 @@ func ResetToIgnoreFinalizers() { // REVIEW: adding following is a hack, if tests are running as // controller-runtime manager setup. constant.ConfigurationTemplateFinalizerName, - "cluster.kubeblocks.io/finalizer", + constant.DBClusterFinalizerName, } } From 4468b16f5bc7133be79732de0182cabc1b3b0f48 Mon Sep 17 00:00:00 2001 From: shanshanying Date: Thu, 25 May 2023 10:13:36 +0800 Subject: [PATCH 349/439] chore: use YALM field reference to reduce redundancy (#3427) --- .../templates/clusterdefinition.yaml | 9 +-- .../templates/clusterdefinition.yaml | 18 ++--- deploy/redis/templates/clusterdefinition.yaml | 18 ++--- .../config/weaviate-config-constraint.cue | 55 +++++++-------- .../create/template/http_chaos_template.cue | 70 +++++++++---------- .../template/network_chaos_template.cue | 62 ++++++++-------- .../create/template/node_chaos_template.cue | 52 +++++++------- 7 files changed, 129 insertions(+), 155 deletions(-) diff --git a/deploy/apecloud-mysql/templates/clusterdefinition.yaml b/deploy/apecloud-mysql/templates/clusterdefinition.yaml index d1843f9b5..56b872a0e 100644 --- a/deploy/apecloud-mysql/templates/clusterdefinition.yaml +++ b/deploy/apecloud-mysql/templates/clusterdefinition.yaml @@ -216,19 +216,14 @@ spec: creation: CREATE USER $(USERNAME) IDENTIFIED BY '$(PASSWD)';GRANT RELOAD, LOCK TABLES, PROCESS, REPLICATION CLIENT ON *.* TO $(USERNAME); GRANT LOCK TABLES,RELOAD,PROCESS,REPLICATION CLIENT, SUPER,SELECT,EVENT,TRIGGER,SHOW VIEW ON *.* TO $(USERNAME); update: ALTER USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; - name: kbprobe - provisionPolicy: + provisionPolicy: &kbReadonlyAcctRef type: CreateByStmt scope: AnyPods statements: creation: CREATE USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT REPLICATION CLIENT, PROCESS ON *.* TO $(USERNAME); GRANT SELECT ON performance_schema.* TO $(USERNAME); update: ALTER USER (USERNAME) IDENTIFIED BY '$(PASSWD)'; - name: kbmonitoring - provisionPolicy: - type: CreateByStmt - scope: AnyPods - statements: - creation: CREATE USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; GRANT REPLICATION CLIENT, PROCESS ON *.* TO $(USERNAME); GRANT SELECT ON performance_schema.* TO $(USERNAME); - update: ALTER USER $(USERNAME) IDENTIFIED BY '$(PASSWD)'; + provisionPolicy: *kbReadonlyAcctRef - name: kbreplicator provisionPolicy: type: CreateByStmt diff --git a/deploy/postgresql/templates/clusterdefinition.yaml b/deploy/postgresql/templates/clusterdefinition.yaml index 1365a185f..d2c283206 100644 --- a/deploy/postgresql/templates/clusterdefinition.yaml +++ b/deploy/postgresql/templates/clusterdefinition.yaml @@ -276,33 +276,23 @@ spec: letterCase: MixedCases accounts: - name: kbadmin - provisionPolicy: + provisionPolicy: &kbAdminAcctRef type: CreateByStmt scope: AnyPods statements: creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)'; update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; - name: kbdataprotection - provisionPolicy: - type: CreateByStmt - scope: AnyPods - statements: - creation: CREATE USER $(USERNAME) SUPERUSER PASSWORD '$(PASSWD)'; - update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; + provisionPolicy: *kbAdminAcctRef - name: kbprobe - provisionPolicy: + provisionPolicy: &kbReadonlyAcctRef type: CreateByStmt scope: AnyPods statements: creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; GRANT pg_monitor TO $(USERNAME); update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; - name: kbmonitoring - provisionPolicy: - type: CreateByStmt - scope: AnyPods - statements: - creation: CREATE USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; GRANT pg_monitor TO $(USERNAME); - update: ALTER USER $(USERNAME) WITH PASSWORD '$(PASSWD)'; + provisionPolicy: *kbReadonlyAcctRef - name: kbreplicator provisionPolicy: type: CreateByStmt diff --git a/deploy/redis/templates/clusterdefinition.yaml b/deploy/redis/templates/clusterdefinition.yaml index 01adeb81b..5cc7ca894 100644 --- a/deploy/redis/templates/clusterdefinition.yaml +++ b/deploy/redis/templates/clusterdefinition.yaml @@ -146,33 +146,23 @@ spec: letterCase: MixedCases accounts: - name: kbadmin - provisionPolicy: + provisionPolicy: &kbadminAcctRef type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbdataprotection - provisionPolicy: - type: CreateByStmt - scope: AllPods - statements: - creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allcommands allkeys - update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) + provisionPolicy: *kbadminAcctRef - name: kbmonitoring - provisionPolicy: + provisionPolicy: &kbReadOnlyAcctRef type: CreateByStmt scope: AllPods statements: creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) - name: kbprobe - provisionPolicy: - type: CreateByStmt - scope: AllPods - statements: - creation: ACL SETUSER $(USERNAME) ON \>$(PASSWD) allkeys +get - update: ACL SETUSER $(USERNAME) ON \>$(PASSWD) + provisionPolicy: *kbReadOnlyAcctRef - name: kbreplicator provisionPolicy: type: CreateByStmt diff --git a/deploy/weaviate/config/weaviate-config-constraint.cue b/deploy/weaviate/config/weaviate-config-constraint.cue index d73cd76e1..1d0c7cdbe 100644 --- a/deploy/weaviate/config/weaviate-config-constraint.cue +++ b/deploy/weaviate/config/weaviate-config-constraint.cue @@ -18,45 +18,44 @@ #WeaviateEnvs: { // Which modules to enable in the setup? - ENABLE_MODULES ?: string + ENABLE_MODULES?: string - // The endpoint where to reach the transformers module if enabled - TRANSFORMERS_INFERENCE_API ?: string + // The endpoint where to reach the transformers module if enabled + TRANSFORMERS_INFERENCE_API?: string - // The endpoint where to reach the clip module if enabled - CLIP_INFERENCE_API ?: string + // The endpoint where to reach the clip module if enabled + CLIP_INFERENCE_API?: string - QNA_INFERENCE_API ?: string + QNA_INFERENCE_API?: string - // The endpoint where to reach the img2vec-neural module if enabled - IMAGE_INFERENCE_API ?: string + // The endpoint where to reach the img2vec-neural module if enabled + IMAGE_INFERENCE_API?: string - // The id of the AWS access key for the desired account. - AWS_ACCESS_KEY_ID ?: string + // The id of the AWS access key for the desired account. + AWS_ACCESS_KEY_ID?: string - // The secret AWS access key for the desired account. - AWS_SECRET_ACCESS_KEY ?: string + // The secret AWS access key for the desired account. + AWS_SECRET_ACCESS_KEY?: string - // The path to the secret GCP service account or workload identity file. - GOOGLE_APPLICATION_CREDENTIALS ?: string + // The path to the secret GCP service account or workload identity file. + GOOGLE_APPLICATION_CREDENTIALS?: string - // The name of your Azure Storage account. - AZURE_STORAGE_ACCOUNT ?: string + // The name of your Azure Storage account. + AZURE_STORAGE_ACCOUNT?: string - // An access key for your Azure Storage account. - AZURE_STORAGE_KEY ?: string + // An access key for your Azure Storage account. + AZURE_STORAGE_KEY?: string - // A string that includes the authorization information required. - AZURE_STORAGE_CONNECTION_STRING ?: string - - SPELLCHECK_INFERENCE_API ?: string - NER_INFERENCE_API ?: string - SUM_INFERENCE_API ?: string - OPENAI_APIKEY ?: string - HUGGINGFACE_APIKEY ?: string - COHERE_APIKEY ?: string - PALM_APIKEY ?: string + // A string that includes the authorization information required. + AZURE_STORAGE_CONNECTION_STRING?: string + SPELLCHECK_INFERENCE_API?: string + NER_INFERENCE_API?: string + SUM_INFERENCE_API?: string + OPENAI_APIKEY?: string + HUGGINGFACE_APIKEY?: string + COHERE_APIKEY?: string + PALM_APIKEY?: string } // SectionName is section name diff --git a/internal/cli/create/template/http_chaos_template.cue b/internal/cli/create/template/http_chaos_template.cue index 62f82181d..87700b44f 100644 --- a/internal/cli/create/template/http_chaos_template.cue +++ b/internal/cli/create/template/http_chaos_template.cue @@ -17,51 +17,51 @@ // required, command line input options for parameters and flags options: { - namespace: string - selector: {} - mode: string - value: string - duration: string + namespace: string + selector: {} + mode: string + value: string + duration: string - target: string - port: int32 - path: string - method: string - code?: int32 + target: string + port: int32 + path: string + method: string + code?: int32 - abort?: bool - delay?: string - replace?: {} - patch?: {} + abort?: bool + delay?: string + replace?: {} + patch?: {} } // required, k8s api resource content content: { - kind: "HTTPChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "http-chaos-" - namespace: options.namespace - } - spec:{ - selector: options.selector - mode: options.mode - value: options.value - duration: options.duration + kind: "HTTPChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "http-chaos-" + namespace: options.namespace + } + spec: { + selector: options.selector + mode: options.mode + value: options.value + duration: options.duration target: options.target - port: options.port - path: options.path - method: options.method - if options.code != _|_ { - code: options.code - } + port: options.port + path: options.path + method: options.method + if options.code != _|_ { + code: options.code + } - if options.abort != _|_ { - abort: options.abort - } + if options.abort != _|_ { + abort: options.abort + } - if options.delay != _|_ { + if options.delay != _|_ { delay: options.delay } diff --git a/internal/cli/create/template/network_chaos_template.cue b/internal/cli/create/template/network_chaos_template.cue index b2c0ea99e..494a193e5 100644 --- a/internal/cli/create/template/network_chaos_template.cue +++ b/internal/cli/create/template/network_chaos_template.cue @@ -17,46 +17,46 @@ // required, command line input options for parameters and flags options: { - namespace: string - action: string - selector: {} - mode: string - value: string - duration: string + namespace: string + action: string + selector: {} + mode: string + value: string + duration: string - direction: string - externalTargets?: [...] - target?: {} + direction: string + externalTargets?: [...] + target?: {} - loss?: {} - delay?: {} - duplicate?: {} - corrupt?: {} - bandwidth?: {} + loss?: {} + delay?: {} + duplicate?: {} + corrupt?: {} + bandwidth?: {} } // required, k8s api resource content content: { - kind: "NetworkChaos" - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "network-chaos-" - namespace: options.namespace - } - spec:{ - selector: options.selector - mode: options.mode - value: options.value - action: options.action - duration: options.duration + kind: "NetworkChaos" + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "network-chaos-" + namespace: options.namespace + } + spec: { + selector: options.selector + mode: options.mode + value: options.value + action: options.action + duration: options.duration - direction: options.direction + direction: options.direction - if options.externalTargets != _|_ { - externalTargets: options.externalTargets - } + if options.externalTargets != _|_ { + externalTargets: options.externalTargets + } - if options.target["mode"] != _|_ || len(options.target["selector"]) !=0 { + if options.target["mode"] != _|_ || len(options.target["selector"]) != 0 { target: options.target } diff --git a/internal/cli/create/template/node_chaos_template.cue b/internal/cli/create/template/node_chaos_template.cue index cc074bf23..f07862072 100644 --- a/internal/cli/create/template/node_chaos_template.cue +++ b/internal/cli/create/template/node_chaos_template.cue @@ -17,47 +17,47 @@ // required, command line input options for parameters and flags options: { - kind: string - namespace: string - action: string - secretName: string - region: string - instance: string - volumeID: string - deviceName?: string - duration: string + kind: string + namespace: string + action: string + secretName: string + region: string + instance: string + volumeID: string + deviceName?: string + duration: string - project: string + project: string } // required, k8s api resource content content: { - kind: options.kind - apiVersion: "chaos-mesh.org/v1alpha1" - metadata:{ - generateName: "node-chaos-" - namespace: options.namespace - } - spec:{ - action: options.action + kind: options.kind + apiVersion: "chaos-mesh.org/v1alpha1" + metadata: { + generateName: "node-chaos-" + namespace: options.namespace + } + spec: { + action: options.action secretName: options.secretName - duration: options.duration + duration: options.duration - if options.kind == "AWSChaos" { - awsRegion: options.region + if options.kind == "AWSChaos" { + awsRegion: options.region ec2Instance: options.instance if options.deviceName != _|_ { - volumeID: options.volumeID + volumeID: options.volumeID deviceName: options.deviceName } } - if options.kind == "GCPChaos" { - project: options.project - zone: options.region + if options.kind == "GCPChaos" { + project: options.project + zone: options.region instance: options.instance if options.deviceName != _|_ { - deviceNames:[options.deviceName] + deviceNames: [options.deviceName] } } From 746d815169795844e4b9b18f471c576760798b12 Mon Sep 17 00:00:00 2001 From: shaojiang Date: Thu, 25 May 2023 11:44:41 +0800 Subject: [PATCH 350/439] chore: support PostgreSQL PITR if exists multi timeline (#3438) --- deploy/postgresql/templates/backuptool-pitr.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deploy/postgresql/templates/backuptool-pitr.yaml b/deploy/postgresql/templates/backuptool-pitr.yaml index 776787ca8..e204db17a 100644 --- a/deploy/postgresql/templates/backuptool-pitr.yaml +++ b/deploy/postgresql/templates/backuptool-pitr.yaml @@ -31,6 +31,7 @@ spec: - | set -e; rm -f ${CONF_DIR}/recovery.conf; + rm -rf ${PITR_DIR}; physical: restoreCommands: - | @@ -52,7 +53,10 @@ spec: echo "[[ -d '${DATA_DIR}.old' ]] && mv -f ${DATA_DIR}.old/* ${DATA_DIR}/;" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; echo "sync;" >> ${RESTORE_SCRIPT_DIR}/kb_restore.sh; chmod +x ${RESTORE_SCRIPT_DIR}/kb_restore.sh; - echo "restore_command='mv ${PITR_DIR}/%f %p'\nrecovery_target_time='${RECOVERY_TIME}'\nrecovery_target_action='promote'" > ${CONF_DIR}/recovery.conf; + echo "restore_command='case "%f" in *history) cp ${PITR_DIR}/%f %p ;; *) mv ${PITR_DIR}/%f %p ;; esac'" > ${CONF_DIR}/recovery.conf; + echo "recovery_target_time='${RECOVERY_TIME}'" >> ${CONF_DIR}/recovery.conf; + echo "recovery_target_action='promote'" >> ${CONF_DIR}/recovery.conf; + echo "recovery_target_timeline='latest'" >> ${CONF_DIR}/recovery.conf; mv ${DATA_DIR} ${DATA_DIR}.old; echo "done."; sync; From cdda41f72c50d10ebb8006f8e09cfb113c44dfed Mon Sep 17 00:00:00 2001 From: CrystalL <110531738+TalktoCrystal@users.noreply.github.com> Date: Thu, 25 May 2023 13:07:59 +0800 Subject: [PATCH 351/439] docs: docs for 0.5.0 (#2488) --- README.md | 72 +-- docs/README.md | 3 - docs/img/banner-readme.png | Bin 0 -> 1337295 bytes docs/img/kubeblocks-structure.png | Bin 0 -> 503152 bytes docs/img/pgsql-ha-after.png | Bin 0 -> 168784 bytes docs/img/pgsql-ha-before.png | Bin 0 -> 132261 bytes docs/img/pgsql-ha-pg_stat_replication.png | Bin 0 -> 49675 bytes docs/img/pgsql-migration-describe-task.png | Bin 0 -> 235162 bytes docs/img/pgsql-migration-sink.png | Bin 0 -> 109344 bytes docs/img/pgsql-migration-timestamp.png | Bin 0 -> 195968 bytes docs/img/redis-ha-after.png | Bin 0 -> 196086 bytes docs/img/redis-ha-before.png | Bin 0 -> 195278 bytes docs/img/redis-ha-info-replication.png | Bin 0 -> 121592 bytes docs/user_docs/api/_category_.yml | 4 - .../api/lifecycle-management/_category_.yml | 4 - .../lifecycle-management-api.md | 307 ---------- .../lifecycle-management/ops-request-api.md | 263 -------- .../api/observability/access-logs.md | 264 -------- docs/user_docs/cli/_category_.yml | 2 +- .../user_docs/connect_database/_category_.yml | 2 +- ...nect-database-in-production-environment.md | 12 +- ...connect-database-in-testing-environment.md | 8 +- .../overview-of-database-connection.md | 12 +- ...hatgpt-retrieval-plugin-with-kubeblocks.md | 109 ++++ .../{enable-add-ons.md => enable-addons.md} | 40 +- ...tion-and-uninstall-kbcli-and-kubeblocks.md | 114 ++++ ...tall-and-uninstall-kbcli-and-kubeblocks.md | 180 ------ ...nd-kubeblocks-on-new-kubernetes-cluster.md | 205 +++++++ ...ocks-on-the-existed-kubernetes-clusters.md | 101 +++ docs/user_docs/installation/introduction.md | 17 + docs/user_docs/introduction/introduction.md | 20 +- .../kubeblocks-for-gptplugin/Installation.md | 63 -- .../kubeblocks-for-gptplugin/Introduction.md | 13 - .../kubeblocks-for-mongodb/_category_.yml | 4 + .../backup-and-restore}/_category_.yml | 2 +- .../data-file-backup-and-restore.md | 182 ++++++ ...snapshot-backup-and-restore-for-mongodb.md | 124 ++++ .../cluster-management}/_category_.yml | 2 +- ...create-and-connect-to-a-mongodb-cluster.md | 174 ++++++ .../delete-mongodb-cluster.md} | 41 +- .../cluster-management/expand-volume.md | 62 ++ .../restart-mongodb-cluster.md | 28 + .../scale-for-apecloud-mongodb.md | 114 ++++ .../start-stop-a-cluster.md | 35 ++ .../configuration/configuration.md | 78 +++ .../apecloud-mysql-intro.md | 18 +- .../apecloud-mysql-intro/zengine-intro.md | 19 + ...ackup-and-restore-for-mysql-paxos-group.md | 114 ---- .../data-file-backup-and-restore.md | 182 ++++++ ... snapshot-backup-and-restore-for-mysql.md} | 112 ++-- .../create-and-connect-a-mysql-cluster.md | 255 +++++--- .../delete-mysql-cluster.md | 47 +- .../cluster-management/expand-volume.md | 140 +++-- .../cluster-management/handle-an-exception.md | 19 +- .../restart-mysql-cluster.md | 42 +- .../scale-for-apecloud-mysql.md | 128 ++-- ...p-a-cluster.md => stop-start-a-cluster.md} | 23 +- .../cluster-type/cluster-types.md | 47 ++ .../cluster-type/customize-class-type.md | 70 +++ .../configuration/configuration.md | 228 ++++--- .../high-availability/high-availability.md | 71 ++- .../migration/migration.md | 174 ++++-- .../observability/access-logs.md | 136 ----- .../observability/monitor-database.md | 102 ---- .../storage-management/.gitkeep | 0 .../_category_.yml | 0 .../use-zengine.md | 60 ++ .../zengine-intro.md | 19 + .../backup-and-restore/_category_.yml | 2 +- ...p-and-restore-for-postgresql-standalone.md | 136 ----- .../data-file-backup-and-restore.md | 182 ++++++ .../backup-and-restore/pitr-for-postgresql.md | 268 ++++++++ .../snapshot-backup-and-restore-for-pgsql.md | 125 ++++ .../cluster-management/_category_.yml | 2 +- ...create-and-connect-a-postgresql-cluster.md | 225 ++++--- .../delete-a-postgresql-cluster.md | 53 ++ .../cluster-management/expand-volume.md | 145 +++-- .../cluster-management/handle-an-exception.md | 11 +- ...ter.md => restart-a-postgresql-cluster.md} | 29 +- ...l.md => scale-for-a-postgresql-cluster.md} | 37 +- .../start-stop-a-cluster.md | 11 +- .../configuration/_category_.yml | 2 +- .../configuration/configuration.md | 305 ++++++++++ .../high-availability/_category_.yml | 4 + .../high-availability/high-availability.md | 115 ++++ .../introduction/_category_.yml | 4 + .../introduction/introduction.md | 21 + .../migration}/_category_.yml | 2 +- .../migration/feature-and-limit-list.md | 57 ++ .../migration/migration.md | 312 ++++++++++ .../observability/access-logs.md | 132 ---- .../observability/alert.md | 139 ----- .../observability/monitor-database.md | 103 ---- .../kubeblocks-for-redis/_category_.yml | 2 +- .../backup-and-restore/_category_.yml | 4 + .../snapshot-backup-and-restore-for-redis.md | 125 ++++ .../cluster-management/_category_.yml | 4 + .../create-and-connect-a-redis-cluster.md | 205 +++++++ .../delete-a-redis-cluster.md | 53 ++ .../cluster-management/expand-volume.md | 120 ++++ .../cluster-management/handle-an-exception.md | 42 ++ .../restart-a-redis-cluster.md | 73 +++ .../scale-for-a-redis-cluster.md | 301 +++++++++ .../stop-start-a-redis-cluster.md | 160 +++++ .../configuration/_category_.yml | 4 + .../configuration/configuration.md | 304 ++++++++++ .../high-availability/_category_.yml | 4 + .../high-availability/high-availability.md | 105 ++++ .../observability/_category_.yml | 2 +- docs/user_docs/observability/access-logs.md | 157 +++++ .../observability/alert.md | 39 +- .../observability/monitor-database.md | 165 +++++ .../try-kubeblocks-functions-on-cloud.md | 345 ----------- .../quick-start/try-kubeblocks-on-cloud.md | 573 ++++++++++++++++++ ...ost.md => try-kubeblocks-on-local-host.md} | 151 +++-- .../resource-scheduling/_category_.yml | 2 +- .../resource-scheduling.md | 16 +- docs/user_docs/user-management/_category_.yml | 4 + .../user-management/manage_user_accounts.md | 73 +++ 119 files changed, 6903 insertions(+), 3219 deletions(-) create mode 100644 docs/img/banner-readme.png create mode 100644 docs/img/kubeblocks-structure.png create mode 100644 docs/img/pgsql-ha-after.png create mode 100644 docs/img/pgsql-ha-before.png create mode 100644 docs/img/pgsql-ha-pg_stat_replication.png create mode 100644 docs/img/pgsql-migration-describe-task.png create mode 100644 docs/img/pgsql-migration-sink.png create mode 100644 docs/img/pgsql-migration-timestamp.png create mode 100644 docs/img/redis-ha-after.png create mode 100644 docs/img/redis-ha-before.png create mode 100644 docs/img/redis-ha-info-replication.png delete mode 100644 docs/user_docs/api/_category_.yml delete mode 100644 docs/user_docs/api/lifecycle-management/_category_.yml delete mode 100644 docs/user_docs/api/lifecycle-management/lifecycle-management-api.md delete mode 100644 docs/user_docs/api/lifecycle-management/ops-request-api.md delete mode 100644 docs/user_docs/api/observability/access-logs.md create mode 100644 docs/user_docs/installation/chatgpt-retrieval-plugin-with-kubeblocks.md rename docs/user_docs/installation/{enable-add-ons.md => enable-addons.md} (64%) create mode 100644 docs/user_docs/installation/handle-exception-and-uninstall-kbcli-and-kubeblocks.md delete mode 100644 docs/user_docs/installation/install-and-uninstall-kbcli-and-kubeblocks.md create mode 100644 docs/user_docs/installation/install-kbcli-and-kubeblocks-on-new-kubernetes-cluster.md create mode 100644 docs/user_docs/installation/install-kbcli-and-kubeblocks-on-the-existed-kubernetes-clusters.md create mode 100644 docs/user_docs/installation/introduction.md delete mode 100644 docs/user_docs/kubeblocks-for-gptplugin/Installation.md delete mode 100644 docs/user_docs/kubeblocks-for-gptplugin/Introduction.md create mode 100644 docs/user_docs/kubeblocks-for-mongodb/_category_.yml rename docs/user_docs/{kubeblocks-for-postgresql/observability => kubeblocks-for-mongodb/backup-and-restore}/_category_.yml (63%) create mode 100644 docs/user_docs/kubeblocks-for-mongodb/backup-and-restore/data-file-backup-and-restore.md create mode 100644 docs/user_docs/kubeblocks-for-mongodb/backup-and-restore/snapshot-backup-and-restore-for-mongodb.md rename docs/user_docs/{api/observability => kubeblocks-for-mongodb/cluster-management}/_category_.yml (63%) create mode 100644 docs/user_docs/kubeblocks-for-mongodb/cluster-management/create-and-connect-to-a-mongodb-cluster.md rename docs/user_docs/{kubeblocks-for-postgresql/cluster-management/delete-postgresql-cluster.md => kubeblocks-for-mongodb/cluster-management/delete-mongodb-cluster.md} (55%) create mode 100644 docs/user_docs/kubeblocks-for-mongodb/cluster-management/expand-volume.md create mode 100644 docs/user_docs/kubeblocks-for-mongodb/cluster-management/restart-mongodb-cluster.md create mode 100644 docs/user_docs/kubeblocks-for-mongodb/cluster-management/scale-for-apecloud-mongodb.md create mode 100644 docs/user_docs/kubeblocks-for-mongodb/cluster-management/start-stop-a-cluster.md create mode 100644 docs/user_docs/kubeblocks-for-mongodb/configuration/configuration.md create mode 100644 docs/user_docs/kubeblocks-for-mysql/apecloud-mysql-intro/zengine-intro.md delete mode 100644 docs/user_docs/kubeblocks-for-mysql/backup-and-restore/backup-and-restore-for-mysql-paxos-group.md create mode 100644 docs/user_docs/kubeblocks-for-mysql/backup-and-restore/data-file-backup-and-restore.md rename docs/user_docs/kubeblocks-for-mysql/backup-and-restore/{backup-and-restore-for-mysql-standalone.md => snapshot-backup-and-restore-for-mysql.md} (55%) rename docs/user_docs/kubeblocks-for-mysql/cluster-management/{start-stop-a-cluster.md => stop-start-a-cluster.md} (92%) create mode 100644 docs/user_docs/kubeblocks-for-mysql/cluster-type/cluster-types.md create mode 100644 docs/user_docs/kubeblocks-for-mysql/cluster-type/customize-class-type.md delete mode 100644 docs/user_docs/kubeblocks-for-mysql/observability/access-logs.md delete mode 100644 docs/user_docs/kubeblocks-for-mysql/observability/monitor-database.md delete mode 100644 docs/user_docs/kubeblocks-for-mysql/storage-management/.gitkeep rename docs/user_docs/kubeblocks-for-mysql/{storage-management => use-zengine-with-apecloud-for-mysql}/_category_.yml (100%) create mode 100644 docs/user_docs/kubeblocks-for-mysql/use-zengine-with-apecloud-for-mysql/use-zengine.md create mode 100644 docs/user_docs/kubeblocks-for-mysql/use-zengine-with-apecloud-for-mysql/zengine-intro.md delete mode 100644 docs/user_docs/kubeblocks-for-postgresql/backup-and-restore/backup-and-restore-for-postgresql-standalone.md create mode 100644 docs/user_docs/kubeblocks-for-postgresql/backup-and-restore/data-file-backup-and-restore.md create mode 100644 docs/user_docs/kubeblocks-for-postgresql/backup-and-restore/pitr-for-postgresql.md create mode 100644 docs/user_docs/kubeblocks-for-postgresql/backup-and-restore/snapshot-backup-and-restore-for-pgsql.md create mode 100644 docs/user_docs/kubeblocks-for-postgresql/cluster-management/delete-a-postgresql-cluster.md rename docs/user_docs/kubeblocks-for-postgresql/cluster-management/{restart-postgresql-cluster.md => restart-a-postgresql-cluster.md} (69%) rename docs/user_docs/kubeblocks-for-postgresql/cluster-management/{scale-for-postgresql.md => scale-for-a-postgresql-cluster.md} (82%) create mode 100644 docs/user_docs/kubeblocks-for-postgresql/configuration/configuration.md create mode 100644 docs/user_docs/kubeblocks-for-postgresql/high-availability/_category_.yml create mode 100644 docs/user_docs/kubeblocks-for-postgresql/high-availability/high-availability.md create mode 100644 docs/user_docs/kubeblocks-for-postgresql/introduction/_category_.yml create mode 100644 docs/user_docs/kubeblocks-for-postgresql/introduction/introduction.md rename docs/user_docs/{kubeblocks-for-kafka => kubeblocks-for-postgresql/migration}/_category_.yml (62%) create mode 100644 docs/user_docs/kubeblocks-for-postgresql/migration/feature-and-limit-list.md create mode 100644 docs/user_docs/kubeblocks-for-postgresql/migration/migration.md delete mode 100644 docs/user_docs/kubeblocks-for-postgresql/observability/access-logs.md delete mode 100644 docs/user_docs/kubeblocks-for-postgresql/observability/alert.md delete mode 100644 docs/user_docs/kubeblocks-for-postgresql/observability/monitor-database.md create mode 100644 docs/user_docs/kubeblocks-for-redis/backup-and-restore/_category_.yml create mode 100644 docs/user_docs/kubeblocks-for-redis/backup-and-restore/snapshot-backup-and-restore-for-redis.md create mode 100644 docs/user_docs/kubeblocks-for-redis/cluster-management/_category_.yml create mode 100644 docs/user_docs/kubeblocks-for-redis/cluster-management/create-and-connect-a-redis-cluster.md create mode 100644 docs/user_docs/kubeblocks-for-redis/cluster-management/delete-a-redis-cluster.md create mode 100644 docs/user_docs/kubeblocks-for-redis/cluster-management/expand-volume.md create mode 100644 docs/user_docs/kubeblocks-for-redis/cluster-management/handle-an-exception.md create mode 100644 docs/user_docs/kubeblocks-for-redis/cluster-management/restart-a-redis-cluster.md create mode 100644 docs/user_docs/kubeblocks-for-redis/cluster-management/scale-for-a-redis-cluster.md create mode 100644 docs/user_docs/kubeblocks-for-redis/cluster-management/stop-start-a-redis-cluster.md create mode 100644 docs/user_docs/kubeblocks-for-redis/configuration/_category_.yml create mode 100644 docs/user_docs/kubeblocks-for-redis/configuration/configuration.md create mode 100644 docs/user_docs/kubeblocks-for-redis/high-availability/_category_.yml create mode 100644 docs/user_docs/kubeblocks-for-redis/high-availability/high-availability.md rename docs/user_docs/{kubeblocks-for-mysql => }/observability/_category_.yml (80%) create mode 100644 docs/user_docs/observability/access-logs.md rename docs/user_docs/{kubeblocks-for-mysql => }/observability/alert.md (75%) create mode 100644 docs/user_docs/observability/monitor-database.md delete mode 100644 docs/user_docs/quick-start/try-kubeblocks-functions-on-cloud.md create mode 100644 docs/user_docs/quick-start/try-kubeblocks-on-cloud.md rename docs/user_docs/quick-start/{try-kubeblocks-in-5m-on-local-host.md => try-kubeblocks-on-local-host.md} (71%) create mode 100644 docs/user_docs/user-management/_category_.yml create mode 100644 docs/user_docs/user-management/manage_user_accounts.md diff --git a/README.md b/README.md index a1302bb0a..7653c46e4 100644 --- a/README.md +++ b/README.md @@ -9,30 +9,23 @@ [![TODOs](https://img.shields.io/endpoint?url=https://api.tickgit.com/badge?repo=github.com/apecloud/kubeblocks)](https://www.tickgit.com/browse?repo=github.com/apecloud/kubeblocks) [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/apecloud)](https://artifacthub.io/packages/search?repo=apecloud) - -![image](./docs/img/banner_website_version.png) - - +![image](./docs/img/banner-readme.png) - [KubeBlocks](#kubeblocks) - [What is KubeBlocks](#what-is-kubeblocks) + - [Why you need KubeBlocks](#why-you-need-kubeblocks) - [Goals](#goals) - - [Key Features](#key-features) - - [Documents](#documents) - - [Quick start with KubeBlocks](#quick-start-with-kubeblocks) - - [Introduction](#introduction) - - [Installation](#installation) - - [User documents](#user-documents) - - [Design proposal](#design-proposal) + - [Key features](#key-features) + - [Get started with KubeBlocks](#get-started-with-kubeblocks) - [Community](#community) - [Contributing to KubeBlocks](#contributing-to-kubeblocks) - [License](#license) - ## What is KubeBlocks -KubeBlocks is an open-source tool designed to help developers and platform engineers build and manage stateful workloads, such as databases and analytics, on Kubernetes. It is cloud-neutral and supports multiple public cloud providers, providing a unified and declarative approach to increase productivity in DevOps practices. -The name KubeBlocks is derived from Kubernetes and building blocks, which indicates that standardizing databases and analytics on Kubernetes can be both productive and enjoyable, like playing with construction toys. KubeBlocks combines the large-scale production experiences of top public cloud providers with enhanced usability and stability. +KubeBlocks is an open-source, cloud-native data infrastructure designed to help application developers and platform engineers manage database and analytical workloads on Kubernetes. It is cloud-neutral and supports multiple cloud service providers, offering a unified and declarative approach to increase productivity in DevOps practices. + +The name KubeBlocks is derived from Kubernetes and LEGO blocks, which indicates that building database and analytical workloads on Kubernetes can be both productive and enjoyable, like playing with construction toys. KubeBlocks combines the large-scale production experiences of top cloud service providers with enhanced usability and stability. ### Why you need KubeBlocks @@ -41,50 +34,43 @@ Kubernetes has become the de facto standard for container orchestration. It mana To address these challenges, and solve the problem of complexity, KubeBlocks introduces ReplicationSet and ConsensusSet, with the following capabilities: - Role-based update order reduces downtime caused by upgrading versions, scaling, and rebooting. -- Latency-based election weight reduces the possibility of related workloads or components being located in different available zones. - Maintains the status of data replication and automatically repairs replication errors or delays. ### Goals -- Enhance stateful applications control plane manageability on Kubernetes clusters, being open sourced and cloud neutral -- Manage data platforms without a high cognitive load of cloud computing, Kubernetes, and database knowledge -- Be community-driven, embracing extensibility, and providing domain functions without vendor lock-in -- Reduce costs by only paying for the infrastructure and increasing the utilization of resources with flexible scheduling -- Support the most popular databases, analytical software, and their bundled tools -- Provide the most advanced user experience based on the concepts of IaC and GitOps + +- Enhance stateful workloads on Kubernetes, being open-source and cloud-neutral. +- Manage data infrastructure without a high cognitive load of cloud computing, Kubernetes, and database knowledge. +- Be community-driven, embracing extensibility, and providing domain functions without vendor lock-in. +- Reduce costs by only paying for the infrastructure and increasing the utilization of resources with flexible scheduling. +- Support the most popular RDBMS, NoSQL, streaming and analytical systems, and their bundled tools. +- Provide the most advanced user experience based on the concepts of IaC and GitOps. ### Key features -- Kubernetes-native and multi-cloud supported. -- Supports multiple database engines, including MySQL, PostgreSQL, Redis, MongoDB, and more. + +- Be compatible with AWS, GCP, Azure, and Alibaba Cloud. +- Supports MySQL, PostgreSQL, Redis, MongoDB, Kafka, and more. - Provides production-level performance, resilience, scalability, and observability. - Simplifies day-2 operations, such as upgrading, scaling, monitoring, backup, and restore. -- Declarative configuration is made simple, and imperative commands are made powerful. -- The learning curve is flat, and you are welcome to submit new issues on GitHub. +- Contains a powerful and intuitive command line tool. +- Sets up a full-stack, production-ready data infrastructure in minutes. +## Get started with KubeBlocks -For detailed feature information, see [Feature list](https://github.com/apecloud/kubeblocks/blob/support/rewrite_kb_introduction/docs/user_docs/Introduction/feature_list.md) - -## Documents -### Quick start with KubeBlocks -[Quick Start](docs/user_docs/quick_start_guide.md) shows you the quickest way to get started with KubeBlocks. -### Introduction -[Introduction](docs/user_docs/introduction/introduction.md) is a detailed information on KubeBlocks. -### Installation -[Installation](docs/user_docs/installation) document for install KubeBlocks, playground, kbctl, and create database clusters. -### User documents -[User documents](docs/user_docs) for instruction to use KubeBlocks. -### Design proposal -[Design proposal](docs/design_docs) for design motivation and methodology. +[Quick Start](./docs/user_docs/quick-start/) shows you the quickest way to get started with KubeBlocks. ## Community + - KubeBlocks [Slack Channel](https://kubeblocks.slack.com/ssb/redirect) - KubeBlocks Github [Discussions](https://github.com/apecloud/kubeblocks/discussions) -- Questions tagged [#KubeBlocks](https://stackoverflow.com/questions/tagged/KubeBlocks) on StackOverflow -- Follow us on Twitter [@KubeBlocks](https://twitter.com/KubeBlocks) + ## Contributing to KubeBlocks + Your contributions and suggestions are welcomed and appreciated. + - See the [Contributing Guide](docs/CONTRIBUTING.md) for details on typical contribution workflows. -- See the [Development Guide](docs/DEVELOPING.md) to get started with building and developing. -- See the [Docs Contributing Guide](docs/CONTRIBUTING_DOCS.md) to get started with contributing to the KubeBlocks docs. +- See the [Developer Guide](docs/DEVELOPING.md) to get started with building and developing. ## License -KubeBlocks is under the Apache 2.0 license. See the [LICENSE](./LICENSE) file for details. + +KubeBlocks is under the GNU Affero General Public License v3.0. +See the [LICENSE](./LICENSE) file for details. diff --git a/docs/README.md b/docs/README.md index 2c5009d95..c46b82099 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,9 +2,6 @@ Here store design documents, release notes and user documents of KubeBlocks. -## What is KubeBlocks -KubeBlocks, running on K8S, offers a universal view through multi-clouds and on-premises, provides a consistent experience for fully managing multiple databases, and relieves the burden of maintaining miscellaneous operators. -![image](https://user-images.githubusercontent.com/110531738/202367695-babd2ebc-8b7f-4a3d-b1d7-2d7b1b69f2bc.png#width="60%") ## Find documents * Refer to [design docs](./design_docs) for our design motivation and methodology. * Refer to [user docs](./user_docs) for instructions to use KubeBlocks. diff --git a/docs/img/banner-readme.png b/docs/img/banner-readme.png new file mode 100644 index 0000000000000000000000000000000000000000..729367956286a1ab515e51e99c6bd2f426142cec GIT binary patch literal 1337295 zcmagFWmsE5*Df3gMN05u#i2lpyE{dSwm8KJ?he7NNb%wjq%Bfhi@SS};;t#~Zin_c z-;eWNeco@bWGC4>nau22bFX{dYbN}?k_^W4m(KwJ0LDAnw<-VtmIwfV6oHETbVlFt zM)T`xq!#+l+>+&5?Ead<6@jTq$zn#8qfdkQURFFHr~2YVBqknP8I>sHkde^EI-cqqAqdtrTfXz${N*LhfRfq5#bwb zGqeqAXmUCYo-7=wFaiPqUS5y2Kk$jh!nZb7x6I%+hiBd29;}b-sYR%cmmiuU5s?7c zO0dH@gU!t;phGW0xofL+fev6!17NXyt;ac!mi_VhsONPAgzrv_JoD@OJ|QyF=jiDB zw8meX+14XO9ywR-S0a2bo~}O3dmaHC)5KCs+P(+fI~v)!m8wL;2azP$?JnUo551rK z?xWv~t#MW8GU0zd)(WqCPp#MssaJ`6OVyvahP&;@aZHhPj~4Ha0VobpZd0m3`8jCQ z=6isE7=fuZ?gkL^J!Ou3i5>^QHs|7iTmk^`t=BPPj0k|?G>ypvmSd{O)l(T^DTc&{S4ra=b%-XDvG~ z%bcT{BQlO{300umjkWDx+CZ)JFQ!b6tBu8qRsM3@-?>ryLZCZrO#$uOE0vrCTsdx2 z(M_b>EDnj(?7|Pj1)0B43&RAmKEAWfc6-mN8d`wI<(fu2oK7G)n8W;iax~4--15?b zb=#!@Z83;kDtR<{-1;}lHheqi%pHUgf)Ug!Yy6tnn%IjtpQx6YHB}++*+g<`pBmdV z2T7`V3Ra4}x)#F)gExNM0B)i{qDG=`BD_#d-C1395>kk+R-}$qq_5$VC#_!fe!T$g z7eUcv0hxMZ0Z-|Oin~h8d)Nnq?=1%Rfj3}`%8*ae*&(vu)xfnq#8FfPJshUTS`t%E z8o*s9>l8&}oyzMy1-ECnByOx;3aXOMg=WQOab_7?j>8C;JryfQ{IgKG5vmc45ie#S zvnX@LlvRmYiEjz-oabE2obkTXKEeL(+@Y&OhFAtu#<_s4z@;l%15d+;hEEOiuCN`E z3Ggmt8|{4H9CLedOy=D7?E5+4&f(PI7zHmI7b;IIu?;&P_-H5={M$OkvTp1sc=ZR) zphZJmm&Nz7ci-vj=ng{7*SS3iMa+urNsWuSmP{9{g^Y|XIp%Y^a=JHK-V7&Lr#Qg06zjZB26$^RLbsr`&{j z^09-s$!^Jns5jMRU*#?UA zh#+QpW`(xowNNh$x6t@b`KJ1!+-2Sh-VENyT(z!3P6&>bo*5#vBWxlx2fPfZ0$Kwp zkW!IyQMjK)J!?c}1&QEGfY30sDV*Z%+XzocuECrO=1gg0_yedJ_#a7b*oNp7$ft0k zLtv=O*uM68*9zN1HK<`LGB#m#6qdxPgx5YV-Y{?!vg~}QR&{Z7xn!Cu-q5}^Z;n2W zDTu5WoVTiRax*WsvQe=C@1@uI*A>1QA)gLB~sqZw^Jm;I|)wzq%76) zE$Z8cirP9kp)Xq*8i^Nti-JCqHx?nv(yl$6Xv!*#kT~Km^)#o78gHkTiea;kv&{Sn z&;;mr6%5OyYQ|}{+Sj$pW{iyS7#ol++X0gJ(1FVhnrr9Nd3>8s3S-}^RZ8Du5-70# zVD!)-(($O1ZI=D`i*woRVdt6#J&?e=IkxDxo_IyPise ziHGIUxX+-)U}e(%`!y{U^%E<|!b!AB(0FdHBx9f^S7Jdz|Davz*~fY?zEPry`k__w zq>hFk17V^;u|x5X(yMamx~(-$C4))tbBOKS+>%`F!2Yi6ZqBY<@uhZanOw7#_~dBF z5k?8ganvS>fvwz<%U4gU^XTE~;elZww-NWg&QMM9{K;3I?-!mKetLr%_a#NOEwglU zQ>}~cUZG&8d^UJD;z zZv@7#=bPkbW+;gY|4LneTq@k9TI+q#Yc`jJWr`BI94OQpT=t(X!DIA}njFrkFOBDE zYvoIXONC{89qw0U%XI8_O)m%cHHAuD&J=eA#||g_ZmilSou{v+S@g0X&4)dEB(-*Q zi==h54d+*G7yE?n&kqz9^gLrc%I;d`?srgbuy*{aA1X}SY*(%f+YDiy+E|B}XrY-8 z36B$@tK%|5GLtmYG*K}b6ci*FFPl$HM&eiVEA#PS1HKxMkX`2@`11YQtvpQYZsrL6 zIs(>O3Mn{sIxH$E6X>PsrDz&~2p{L&m5j{$9M&B+EcZ53q`tlrdw733b_4FsA632- zfqOsPv0etB-ZIPZ!JbGaC0JHP5diR{0|5Mk0f3vQQ~tjJ09Q5uV8;*u5J&+42pyme zAB3JRpgG8DI|BeLG=H859=W1!0Biu@-CJ=r59r?gb`_~+=GL{3AH2@%?E*dt!oiS< z-A`BbbAiQ|=~HKs{U&srsY7RklYZZ8JsEd-`Wws}CgpPri=0g9;?YrY=erld$D7bB zebN{&Rxy91cd>vJ7`@<$w1-C>#ihfg*w4-NnEFN<>i?~rkbZ&-191M=@+?d0?f+6X z>Q#@l|Cj4;-)0K_|5j0yc?tM;9htRK>2;9S&8>TUSJS(ro=DzOr61pg55I;tX$b+* zPz{eEG17m3+G75~o8VtPi4(PYBY2w+q*&5BxrGsMxn_UvnPPIXZS-C0#DlcIT0j|M zpaeAV`-lB0LW0jlBL2SnABR4QTxbR)z20Z^K?)tdCo^ z)UacyqbYev8a8rQr*i*B#nw$UoB1nJ!6{d3R=$;EQ4K*S#vJc37DGa zW#`|;IUOymLl6kqe|6IRtrnIP2>WOKC(X%={MvK}q;B30?){kS?TFpPVlJctmcY#o$0*25o)TL(^=b$Z4=nHdRZp&SnS-^Rcd9~lV$X<&!BBK%lNo^}T%S4Rh*=TR7=;o(&L z1d)c=fpNWV=A=#`2z~vIa<|S9B7Q-XUg6fAtvxc=KZASNBCviwh*y(m6_qH8h=T|YZRQYJ>1F1D(vsj2r}M?Sn;AzSjjwJp`H zKWPg6JNPFuitXnAY9Y?G{<~Rz{Z}G_7WMVDYg9%JyhGktfX*}67WqC-kb&XMEoRuyD8`U}6YvC;c#HMHsWyZd#&vTw({ee3Yn z*4F9bU(ksYRoOaVePB-d_uOxJ>!#%ve%mWdv25@(&a{`&0ypjKK#KU0Oc>2DzaW3I zhWTqnn8SlW?nw&}>z=i&nThl5`i{~tmi^FG|1L?B{Z};jdLd52m)`#8Jt)0oDQzP$3=sT25XJ`za&Ay4ZlKLC3TTdxrqItinPOBewdLhXd?Iaw?-I)A?%l0x_foh86*1!O!T;s^?=8s*KoFn!tfVZVsig4T>cN- z4Qq0#6g=f|zARsVvRh^H>OHc{94I?uz}Cs-9p|nk05W79!Z)(Mg3MpR0_j1deX%b+ z68)^x8zdddq0IK9@X0np^MUz;{FgBT2W01WS3il%^fJqZBf7Ey)lGY(FkwqA!(vp3 z|5a13%gExaUpdTv`Ig!JaOqxpumN&P7vCRTvd{c*!ut4S+vO%w`pw_=8od24a}82Z z?X&gp4mRA3|75t1k2(4Kax477uAQ8G(T!jtH!6gT#eng%rp&!{zut-mej>xz%1)x} z=yudx_;s<=9LN~7vA3a{XCB&!LKaF7bjK9%y}OBYw;%F2SLw{bT;43jh4hdLt#&@Q z3HOTmFp^IwK40T~F`Mu0xl1s(c|yW^xHX)9*SY)%oGvqfXSgpxehd5uyq7Wmi!a{D zWqSp0w}j>2GS^;wJRo<67URwm1H&I=2dhf(hT#bC^QJ8<#Z$)?BHR0Zc1k^oQSv>e z-;vta3QlY^NHoZ@sbA2^L~D0AYY)m&YQt}3g#Tb$i8WrS&E&5S2yzhZP4=S&gw9j) zA<3=${Do$((D?Z8PMLxUI+Q}QHwxYhLbmfKt0)vl?Gz7fiHw>vv$K#b@L!J}7y9PE zcsEuqcP*E2*oBTrUF+J!ebV=dfJm?zZOUDrc@OCsIKm9+y9o#hx3B@%V z(U%w{XyY_NTv}aWEeb6#qz*hxlQBmFs+-Jh}nKf0e zOM-iRPhOeKy~)d`2dhlE{)wNIbXLtMQZEZ!MnYK3KhsklcL{=oP|Q*V1mfvNYewl< z;hgLI^&-8C?`Dtp%-hW`6+qIYuxnRTPa;q9y+!!OZwDJ6KlU2+KXAQqEB8w=u5KJtEDG%#i^dG;ihlY>M3VMcFyjB@*k_L*JO7 zf~{KuIA!=1xrn`}ZO7hgVzWS)1@-M+L@ur_mmn6sXo)K}ATSYDoV+fiE3=3(vVWg6G1(RrpIKX~=&`$# zU_E~h4w|E8CFm87b4Ka>5_?OyGcdu$o$N+$W3N%HA^p;iBGhPc&t_dYTlG_0kF=s5 zN&{7eE^UaK(i8*H*nsopkdyC#x38RBc%ceN^6$Nv&+&8o7iskFRdH(&nTU?@b)R%(dLF^aq=06G=qQKEiHgT$6M3lEMn_9Z_Hrog&@R89;-_hHYHj#% z9`ykciQ-E2Fwh~-+Kwkhg?@AUFnT`ONv4?*u5B!ZEypmMI72FOe%Bf zNKC-SXge-*+EuCN#CA9=bE2 zcB7bU%#}t0YFwiUg>Pk;L-U0t2XyDGJm5V9_AQm&7t-DR-{2Fs&LZ`q<}E~h{`1#w zmVYuAZGQfGi2on1ex0;!u<$Qlx%>^9kzgd7YUe;u)Yb2_;xXpjM8u~J8cGgdhYYS- z4tkHV*Tp%R%+?eqF=MJ#bV$6a#|v_F&EJCgl@E*F&$5Vmy9PCI2qhyggz^{bq&o+$ zvREFYkKZM4T(FFYDWL(X{I#V}@d@&^xI2s6J3vJ><#keF=~@dNYdjkJhK_hu{WF+k z?Q8BWF$S11zO2hu+KcAzHkC_HM$Vae^5digkNOu?A1<12?+&)u zi&%`zHhZYWL`@o~;!Yy)&;BCpwaEX)tpCBpHgrZYQm+{tD~b4o_WAj?`O_{*p>CuZ zm{1m!C;L@SEma&Mbl!@fHIrUcKxH4l6qcM$jMYZ%DnyK6s&e$6b8;AWY?3|mGrBt) zCU!Mt6O$v9upp58J&U*G8_4<)&Gq=Kpjpz5UQJb^xbMwsJw?zcwv&9H{DO~7kexp& zm@Tc29r>#}>NyJmD^8(a@f=J0Z??@(?>c<)TP)kI=iMB=mmedYDWa!2J~@PE~alGy4`5@6O;Go2;_?;P?D(4K2nDNPXhb&*}Leb z-SW1}JCUDnzus@g>G22n@CUrz5!QbECI0$z0_WFPWXI6KP%xzj69O2jCB>xhV<*iJ z;b^>gh{$9FNcGk@OpSfqM(x*PTYu9!{E)doVr$%OSg_1iWjdh7xyOXNzEM%&o}bB2 zoE+1UT1m3}c~zKp8)6-CbbKhe11amFHj+2U*$mX%1L-HrR2reKo~6_JDOJoW*`#2h zPo<`bc5cdK7o=D|>fO7X;H|0Rwq0+snav~1Ggp-U0E#LefyOCjnojeqG|w22+}^=N-29wK|S5rB_#|aD06`=HhNwgiOgok^I>HU1&Af0Z>vR0vO(t>w{3`^%$R2w~vwUY< zp49(ABu}N)CcQ<7yJ?T(Pu#g=aD04^@6{`Os!nZ=!fIsdZcena)!g;q2z%8!%yI z-c^qG@#Uht-$;V9W@+RZfJ6k}@cU$GmmoK&KY>YStYahtav@QRqpZ$`&N`YCH9T0m zrx|#db@@Z2cMbV`k9)j_EaV|YtOM^==w86yre-lTYjX@MwG+}iW+}65ZtTxE&n@cd zoSxO>8a1dh0dLm+i-CSG{w?Z3)eKiY(`B_JlyUs$vWrU49XIpI%! z?P;VW^;-p2m7gd?PO3zp(K$Qh0o}~e^hoO_ zaawL@?!AGDEl#ZFZzC`2H8#23Uso8>q_NvZHWa2z1w{3?9dlGz;O)QvIQZS?^a9T6 zPVN?R!i$k=s%PqrC{@fqUiklq zK=AxKL_2wQvjJs7zK9kOc@j>y{|Haf->=0)#lm`eijBM&-%H6|5qAoM!Ra}T6PLpG zY@?UqA{ewU(lS~KwnMwvUNw-1B*U0CXBgtg&iT9*f}8>+w4-pX8IZpS4+G{YcZfTlt5_)$Wd-t}VY z6~3$g22(|*6SE9*OOJK#X<`kq#8sD7`wxq*Nlw_R>x?9P6Goem3&*9<$y=*Zy` z*I5Y27MmS3mQ4+UY;6VM*IwuxP`Hwg+>ZYOX|eVBk^^{Xx}3m(PU6Z5jMS$Pn7u5IwvevzowTK`P`~P+M-rX`4=T^Y((nAnY^$;2`+Ek-*r;4JsK}HSgBM$E1WdNucqJ$utn5 zy<)`!pe(JldC6lcn^|AIcw{cx6j<*J$zSbBBu=ie+FA9#IPg-_Q?9$?ajfZ#sA>KW z%OLzOEJJ!=#5nQF#lBm{J$YeH$I&BgoAa0Y%D_QnUidF{${x0m*FGRavr56O*u~$f zW<*vYzS$HZRS$;WgR>fa4JX~IZI$N}S6(A=1e7YD&B7!`i<7f(-z2U$4}^c+ojFXt z+v@zB)#MXZYj0vx7jaqJu8D1IxvU5@la5qJ3)sZ`9-O&I=K_gf(fg*TI=8g#WD-}( z2IJMrQEU*Ye3|D0k;cbUN2+};1g`Iu4PH^BQVoqB5J1JJddsthHa{tBrSrj+O*^v^ z=a9qFSGnq#VDgOFVu?~Mxfl62oR~p%@U(Lz=Hu($mF-Hnd##S|{WcUO9}Sk;Hlsz_ z*3^V*bO#kD1I9DF?-p}1pFDp>K@*Rh3De2wE#`V3IOK{SKGv3Xx6cP|td?Z^9v)!U znZn&89nYuxN=Y-Eg}*=JH}dK%nI+HCRU1>h^S52kIi9i7?A=JN4aZx#h@X9fJ;JN? z^!X=VCa%ifAA(>SMvn1Xhp`iHl1S0FnXaJ2ZKSXOXhtE&6J>l7u6e@Se|*C!(N&#O zmltS$2c^K(z_QT4MjPJ3V0!)KZ*JkFc-vt1U%^xiUmDiJlWW-4A8hW-oEUlc)y#bw zl^t6WjhROsO^BYg{WZFO=L&Mae2_^?reb3%fX}h}eMHbVYZ^h;3OMiswur1I-5MRo@X2wJO zg%e1N8^@z(#BIY#QNy=W<~Y+jV%aJ+Z{G0r+qt5^^0z?@OMfVEwqvGP2QiO5A!G*{ zAcm&y#c;f@wPY^=EA|X+7*M0)M5DKv0p=YCmq1_#d%zl=1Gdy=IqoNSs80+WT~0jY z*t@eW-4#Toxbe@A4lk=)aB+!p9GA#?-Rs5{COZac+7<+wh2PqN8L3l?o z(Jm{P`TfmV%MfZShKVU2eXlPzolpXun2=MZr15E31;-U)}#2c-p|e}fSgnWv$#Cq+&^^K9>}Wv zdCQ_+b>GCR_E%!?lKQvYGFGX$AbL>m>erponF-mgld7p@lwdm%m=Sz=It=Q6#}BR# zei;h#IdqW>aZQ;(RHE|&u(od#0o7lh*HLNEg0_|BFMO8+%CdYKf}aSX((<>s+?ZzYNyJZ(oM) z<12+KD&3Mr-;&K41q$Za)LzDWAyOn&p5~H5P**^T5W}|jniM2uWEqD}(TRU!O|qQ#D2ESYIJLq?7pa&L&{REo5u`d0*Bi^G@i7Vrnw5_gcw1Z${*yBUY3KRQ<`*Jso>~DR!B81z z%;@O;H&NnT`MX|oJkE#hRPL7=d-|z(;JoMIhN%|ZK6ZR|yfZp_=4Cz+%?FB@b0kBd zWXLflyiP0#H5Lecx%IOdd6-;@2w$r&5rfBjta%I*0IBO;4A_CI{r&aTXUOPcl(^vp zva?!S%mkJZd?rdEBzxz=wOi<=T(`Q*OX~wlgEs5jJ6jyC>VER=4Ts;huca*FeUk)X z4w2Za<3p($<(!{Ndg^$2mF96r-ASfDp4{g$3yX1mkB(#GEM(v)p*dBDHSKF-geMel zWcF^EefRDO6n+un%dlP_%1=c!(EwE?`B}l;PdbE`Z<|M=GdB&t#Jfxii24f0^-uyy zemnHkDC)A>JOgt!Xj2B~Qxtnip{5dX9O>0$LtQFBQARQyq6 zbRyvB5%f#fT+2to>hj$^B20-Kv7UF_in0+*&c-Rk7cq01{qZPUyR!W2LQ@xt(tJAU z9@){(3CpybU(dyb;IDYS#UK9>4@v$ly*&Il4|0Bd3c&91FO@m;p0kB;#s^`gCdkng zH{uPUnt>0TKmzGTMEtizFL??LwbWM;n508COn^lULx}xQUp5KUV6DKezx@3t5CKt* z3b#lM%009esZH)c@qNis zNgw{bbNc%13?sTh^Y?6r zzu-JB?(ADOt-Id!aPeNY5p5um=I@O$!*sPyBb8lmB&y5w{`!(0SUFWi)=a(;G9 z!tfv|v!$)7qf-IE{ix>@1`PSCZ4|3|U#wBCx@6)~A7-lwFkxs6S0_%kW)zmi)E%f0o- z{~^Bp`S|d1TT3J4BNIULck=;n-mJe`iLZV>}WJt_cy;!%1XX{D7~VaQ~mP`2Kq*0_A3MhMFH<7 zRX;_y`H$g9(n33R;$@mdX_8}i0SlGexiz-S$v<&fl<(*w$*+z*7M}rcF8IsVSZV(Y@+)OQJEgXk#@d?KGuKV?D6&a7-ha&S0STvz zm*d{jCWe?(=zQixsFmgDWKNpeSOMLOy+mffjlv3E;2`_>TQj&i@Jag1!vxM7b$977 z(GNjo>CAsObqxP*GTv78Oc%50Gm@j={cS5pf;iDncN*0fiRP;1Xz$lwC~PVsy6WO6 z$7^%CbPV%2T9=%BiM}DtQU#1SV(}4Gi$qAc-fM z`4Od!ffCX6ZWqKKTfyE3m0zjXs}(Yz!*s%2p~crj95-;)jm9JL4!mX0kHCUB+z?OK zJhHpG1{ubeHDypQWA;9Aysx@hoOQH)okZlm{aS4{ZafD>ykR`5BVP4Hc8}iboaWX!4k~t1 zoKALisA?}idv%Ub0-%n*Fa^zw)_8|@aLT$e<@!oTXL}*!Z=rTYb zfhEY^Up+Rk1OuoU}ovw*=y!R0UWLSp!DW@bkC+@3RjOActP}&+iNO! zzah7TLsAK@yozsDh>A}>B2@5KMetFp1(tVWiHKk+C$=2$dZa65a^&JwIWGmDE?qUb&hi!YS^vbn2>61Y5MP=}z-~QU>Q8fO-OVPN=luV2^W+B-qMHr_L*Pj zCXjK`^o(nzb8hT`*~TjjwIW}Y{Vqll&Ij3S;L$BoDK^5)F8zle5UT953I&Y?plrjm zd%BxRzlFy*+i%B@6zH@pg;^$3G5Ha3P;+TH)UXj#-4C zf^Vh7n`A^E>zb{>K~)*TtVX|2GNUm1s^-GJA|(@Mh=++;`jS_TV3lwK$FeKirmDXc zL{dqT$@k4HPCKsmMxI&+XT9ka)R@c4cg>E*CyA=yCa!ZHb$&Eiaf#dZqBrk!=0o3} z=N#aiBSJY0!KaUCr3o=`;(GLQ<9#r9<9(Fj$$t!E*b_=m4EI(%NaX@;2qF7d#XE<> z&zeFn6ihgRXjK4}INnWiGdEVwzNeqq9`sGC9DU1IT-qFO{kKMzjrZROmffq;CHsYS z-#~1>{K+5m$;Fiq1DtaV> z6D&jc;~&BX?}gpw=WWXDs>YT~Yv#an6da2A;D}TIlK70C6oP0mOSK*h37~$W_)UZ7 z)KSViMYELBzVpvE@}342Y1P#v%>1Peo}PoCVSnC$MZVRZ1AMbpKV!dZ*6Z`dEQi|_ zaFea2@m7Ljc}PP``<7W=fllF0LZN`^I8zE2W15+io_lb4hs~#1F;dBqH{w}xTHGP7 zuLV|_D$1E|VI!S60b#S4vt{fX)BV>dhN2x;XsKC%GO%s{8;Rs3vl&@z&t|gNeghd_ z<}(HPt8<4;S>hXz{nlss^a`Y$O~DTz+gOpZ-z*b45QK%`zQ;7PQ8rQ`vBd;|#cqxV zUB_>#(RzvtdQC-ct38P1(}iQ&AJ`WSK7E$262s%{?G2xFG8AD|Jah$tk6O7}M0BDX z;}$J%mHD)NJHFimPcH~{=k9p-Z0_Xq&gZ+z&m*T9Pc}(=oyq5Cf$ham^jAl#5Uj&g z*?QAHvMnVSn)P*8yT!An2e60#+b7Ady}!S}sfYB-v#mv(h1+K$F1iD$TRSl?eE$lj zkmvu@-TE%mfHDz7B|!o6D8fY}@;~?^sT5gDN`g$CPE?v~XuW?|R5Pb12cPALnt@%j z_PgnIKY9;sPWIZA9dIc2azVru$dW=Ur(9_(yV1}45QGxCe!!0*Kr;{0K6cJuj5n4t zg0|X21Wde1F7eOaoH41wEG@Qoqjk^glJn-44D35R|&RlHjZ?o`su~>lP=Liz4q8be=QEi0g&Dhq`cs zN--~s0f7~02CnzTc*qW6R*X9VDl!!kb*|CLNrP=cDn`bUS@VbnfG`lTPID z!B0_*pQo+4BrzV@3;z6M03K%C$N!ODRc~5d{;{}v!)uT%Iw@88!s~#XPUD_P|D>XM z%+`ik|FO4s3ocgkdC7{&oaa`uhM>02`vyUiz;9*zMAhsbn6{c=l>*MBd|Up~SXILY zr&bG5N&hBh7?|S-YNteYny9iNU@BBin&2Od;vby-Y}#zB?am~jY!7@(-#D?U@G_{a z-B3kwreat3SYEe6&#g0Nw)i2EtG+`=GqMpyr}|p2$=;IAxnqpIw--G+hBHD+a`EjE zCX87fn4TV&r_Ek~nW98d-t5z7V5B3Azwnxl@JmIxYin5138Sz#&lFQau#ouik#PhYKR4 zZIoUcb5lgoBD4ya(7sbX^%8jI6CwmuaXxlPV5OQgl-K*MTD1#B)6`hMP z)2pV+;aGF=ZV#JWFq-wkbW9?2oJp9Dba-m@#_0uImz*|7Pi1adT7Qh)DdZ_XAa%n9 zCdvb-;oznwXfn+$k78Y>mihLN^t7&>Og(m7oMpY&EEBpF4;M;O6^QLk|B%D}Ba8E3 z8-jH}GPs?f?jB?6C+22;pXIz+FX~BEexegEc^&m~!GUpFXi&)}Irx4Vcl|{K_xYxK zOO-9|AMFQ1{O{z+viyNV<$!mCsKepwF-+QWwT>!q-I&~b6#`DV;YOj+kGwkbyhSv@ z2%RRU8^Kc2*4~S485`G2ixXpzz|-0mY{w!hh1Y1K><7DGPLq@Iy**@?vkCmck|evS z$`q)R8ilh8Oj}Vi$5Nie>m$tLb<{{FfG5Gw(X9_H5T&MQut}x0`+UdN%{xAXS?Ga( z@Qj9O#KY8jr6+gDQFHwbwtHlTt@%2oql-;o&IE7YiCx=~$KfyllP1xjp#)e9w{33Erhp2f0I!}vype?4fZXT4)Q3d4 z9To%5(Ulms(|DOBda1=rJU^4CFLJ9+vu%)@ep7Vhh`nogpL*QQW^zLn&uOl>20PIi zs%ILr$u1C-;9jl1m094wYW;FATETuV?5c(HEq-Bp!Y7)*tGEG6+pTqdeCshTW46@< ze8O4iaZMbpu;IvYP#=88iNqNwIU<~ix4e{kv+oS6UKCcE7c3rkYO?IagKv%Py9}H& zlVA`0WDOR3DaUyB&7kseWm-!Z+_*$(w;l5AFBB;Me|rV~-?anYtjrBV`<(&vp};{q zGEbK6o;FB!|B!q(Pts9k=x2di8)kdw>#JH_!MkVEN`AdtJLE|XC;jimE#$s)CS;## zC%TYlkt9jH`|a6Nl=*dgJCB}pkKDoKI(B0A4LuY8&uAVxiD0?}Hj|gxm_5HDgD2>F zF&Kd880l^qB`TmAaaZ>eI@#4iL*uR#uc}Q-0dm?^jiwQzg4Cf!%N2ZR{3I z=18$6BA%gw=Q;HyHrwS4`C|f17tmIPK>NPv>#9S30oQ)CSFP<=yunBL8%^O}NFU3C zqux!Hvy77`+am9;EC}j<=tUC+rIuK(6iL~%+O*I&J=XWbtb7b6;}Jt+N_)v4lYCaY zLX*r1eiVsK{Bn_`^CGu@ZD=lk^aKpK#GcKss{F5ODPrylafTvJ%g{TWLyR0$d~1M= zx9~eH%&~$(a*6Kk9oFlsr6MScU(e4Bzroz$Rxr3h8AL`Pcl)&cP|n>c`FbZAYIDb& z%=7rP&foz*{>IjL#v>#MpC5j(ksv!OKfp`GPa<;0wX~ke=EYC*ywjm?dreYl4h3`o zK7#n{hB)u)c)fs63)(bI5ySD%>xb5e-Vo(?;!HLhyyu>`X}V`cjo-KTY5Wr`zSWmL zwtv`qdO)xw{+{@M4;$Ja&qo%&ca@-oOI);1+R0? z1HHzUP0oX_DNNua8#ye(HY5C}aaHN%I# z`;7zox;v7c5f63;rde{E6&w8uZL!_vsgZ7U)SyDY_L(#&U_V^7+x6-?{e;!~`War< zT64bkA@6KR{&lg>>YxmLic}%0I@5J8RcGvoOuEnF73oL^ED~ckI^)wf3cZu=t_H2#lO_8Js6-*K#fZ#FpTP8M6&RYm^s+_hv;Mk+Ig#WG;s%oDD=dsQA=Px#DFP zk%l4@zEUtW_@+zU@|;d)jb7LzhBF-Cf#oB?(B%vfvdSG1v$2N|yDQLT08wqh+QBqM zP9L7f!S8v&jR%BwxlgyT90%OzXUOW#+qobSc$0DcRV<23>stAIzH`2N9~f}sxQC|6 zAegR}C*vM?d(HhrnD&fuXPb6ykApzJ<(~Er#+=yxd!ts%(^8*#wVqLB7nh!Yfz7p5*X!KIdH1ga}T{hD7YPsv69y8&%*ljZdKX}&I*Rd|L@?x?FQ zy$Tu1;m<8S`J@5Ib>Xh6K$zpZh22CrQi2RoQhboSBWFe0^HLT8>BdMy9A>^PP;>+< z&hKO#dQLp~7$oOnsNnba6)isquh5LlyE+A8 z1I4)qj0P97OEI$OA7B~104aKSW=$ik@HTAni2dFH%bap1lHtJny#2F~_z3g;4hv+4 zzQ;+A0=G_+t4i|+U2?^H(L*C&)5RU0(EQdl2jBQydI3tu3&uyS7<7@W`-F`9mXflD zU3jjfz&ZNMP&3a@<&M|$v#dQeXqyN+Ji@XU6Fg>BwOhFb))BwD&wEOSFL;(+{#^@4MEjk-*TyXCNq6d zbvc*6(0Kgjl6BGCZ#~#_GTuRgqr))xMY#=OM{Z9nr70yJaW6)mV^8jt0B;AZj^Ih} z@puMAHdv`Pl#aF^)%25lJ}u9e@->(*Xq|ZOg@x)XL8QSHVs&-SRE3!ZixedWV^py= zvXdFPUH(KNr!X~j%8<9^NklqBxcoA654eLD%t~9L8!pFmJ`JC>74hdQzVgI07In2w ziK|kyX5;GW*FGk>8CFG)Kir9w$IOD(1!m~9qO?6viwpgqHoXz!528*MP;#@2N;Tg&h(^F~i4AY$PN< z$k2ta+G6vVZrxLB{W=O;1M)u`G8Q(Nb8Hvvj{A$`{2#{NIx5P4jUE+2LK>uz6h^wF zVL(cdQV{{^2I=nZMnD=yltvoq?id&vhVG#U7&?YK=X-wVu66G^=dL?zP5=8opXb^8 z*?Ye%F~_FlM?S7ST%Qb55L~uIJt=?yIFmYnkmGh+Km7P-sApSUi|O~$CK<`5_mRmk zNpR&W{DEw6Ko4@r2Rs!#n-h;1M^G4+%$@FBltrwFb$rDP3|!wLQZ(E6Hug%TxW2;l z;#|l6|4chbBB9sWTJDupS8?!yH%;4s2 zETdhGcH8jYnZ6nYoF6Z(q%=s$PY!RHk^xcsGyTp>#C3VampqUz%kxxYN$M$(2IlqB zmIhn;DFV^ID`39Lz0muo1e#$XgSTl(2PDZo#Ib{@-!CPnUc5nrS&{|&XBr*Au4Xhw zALmaHHwTnVwX9WwPW$W|(jX&6p>Qlayr-!Mz?}dd(9rWiGX#>rU1oNgLo_^5V~cfs z=piBER)e*nAp```FPbx#4X3LxOcc-D79o2)1>wHGGX<`@X^tdnO>SVxW_ z+I0?-Za4_ZOmC&-6g!0NR_o6rDPw5c0P0{lR1Q`x*&wi(Nh37a!kMhf*Iy~nm0)1O zGJK>{DBa<|*m}-5mV3nu`Z~rnazig7azhqTc2{?NE`pvomw+qm=()5#Ikut?YH|0c zc-A>9ETm&-^XFc-yIItUGV;O#M=Ijl&aUwPlln3kgo=K(0HWYf>ltG0FazN@Nd!P$ zI8^ao4{Nadwd0P>1!REbB(RfnO{h=SZI;CLIJ-}Hbl8xP5CXr0Kw`@<`m|9b-h?N~4j+Vxp&|Ko-|Wl@D>vJ{l3j-ez6t8 zsU7?s*GG8ws|t|}L^STLn$O>i`2OgJ0ooi4?=PEcmY6hWsziruRRuKvP_|kOs*P=n zH}I_?WiUWT2t=h}=!dZ!^Rz|}5l6?&A-dsno_Rl(lh&1463<*U4k01-!l`KOFz2|g z!JBn`s|4Gh&zLcpP~&g>n$G5^yzT2u=G|`_M7ZUuH(sa0+eX(IRRsA7tE&j+ol)cc}b&X^CdyV zz+=wCHrH46S0X-tWFgRzIK@L8k5ARLkN){LH{WeCL@{VxBB40Y)8iC&@zGiQ4YLTI z*2aIp(>?LW|0?Zmr%26lxieb3{Mg+VnoB=QB&zTnpU%djhp&?U{?QY5*|E9bB}Ftv z(*1%EpJ+@YiNM02QQtZgLrCr<^Nn?eVH2cn&cXTWuewNMo%$jkWzYhL$@9d#=O6k| zSX%jKbd5-G`M^QJ5!C2gS|j<&w!%GITB%lvt;|_G_z5G)(?s3u%%%I3&yP5whfk?m z0WP_mT#S}Q^dXRu_TI)a?cg8BYVO5ZnQ?J!wK}SJiQz%*JV6j`tyZxcbi{l5{8YvR?%L4Sl0uMrUyha5n~Lz?2K-n< z-1d1z3x`H4>DNjs*-lH^lYS{-H18C!Y#{c08w}xf=UHqH@v>#WT3BiFm=CZ%@p|cL z5Hn%L8U+1gV*1;Hza)|`#T0tDuD?t8*rrWs^Cgw+DTXq}!BDRC9T)J%sBaZ{XLu4} zxNPR1S&|tYH`>`>IQ3?`A5(VU>duF=MXGAw;rk?-8q<6VWkVH!j)N(|5l|qye8#Qs zb3M*7BEvA-GChUTY#^HOPs=E8{nXS0Mv;5)j3P30SP-oCq*eflJbBgyi_rqT1LYX{ z1dB7uzLr94Q*c~-)>odg{b<)7teI@tk@pY!=3)Hr3)q6R8ka|r)6dr_%YghP|>wX)1Uv;4Av`*i2Z2{W+Cw%^M z1*!EdeI^$iUmmdXePyumHCt7xGI@a9a!<0e)%LVJthELl-F>Mog0$^4)GI#Cpp4)@ z4cUuCV{UwCv+tw5Jfz`3;Lj@(Z}WijUcXVOq zMpHlK!6=JO>Y3TUfO5ruq42GQ=>I+$e{zYCPiwmQ30qO3LvKw`h{!e*6ctiIBxxle zbLp3E8uY~O1yj9S7kW+g7Xl$N$_MWiI5?^Uy~6OxLcwQ~XoM@sbRP%b?yh4rJ4qsI zY}v4aq+UGJq|?^%na%+t`#=#P7F|bv7CfRH-w|Jec(6clfoe|U>o-$8l~TpDrx6^x;Ut}Z%6RQ-a{jVL5S3(QHdkQp zWS`c53absIqG)es{6VyMj8`aVbed2;JHe=#VP@E2qgoEwu8&!#{tHTxpP+AJ))-z5AD`$=i zV*GLq%UKPjlb9pOw@ud!HOi**?Ji$^5GH^JfcJ(vbJRJ2$k)zr_k8-NG+9AM`speo zqI|C`6*+E^8sramC@zrdjIWEj)bp9oZYz=3wAi(8LY`z-TQ=wRRbQl2jJJ1ODb#BP z{Xynfa^wb!7KDuM+ZSrH^v)j3^Prc7Y>rad{;~8oA@_P|x6|DRTaTH)akF_D#0@hA zf7p)o>R){3*7ml|>Bo30ZQQ@bysL%#UWC&BPnzeSZ(CWE&O1iAo{nE-LQnI?U2qI7 zV(^?~R3R71hKSj=07yZIp=n9a0oM}y`?zmngi_*b2abitCEcIG22zukCFxpk_y`T! zMty5unKKgzlHpVRw(N8XAm8N-lA(VzEb-G9mdp|j5&xlS+ef3fG0;llKj_{Xs$+H% zpN%=#gu&u8hz=Dm=3c+h^>ia+eu8u5Dr8|~Ys_su%-X$;5Bo9NnfTO?PGC8uele*Z zoR3i*ZEI`qwA`kc z1+|9*@XKf4A6ldQB(_f668Yxm-vd_$J0Y?ojMAD^qp$dL6j)XBOYvgzvf)rj_rw#% z+)Bd!)C(s`(b%bh2pE|6)h#z4p4xkJ|1u^<%Dmvc>8Kycn6NF8)5QQ@`v8*blZU~( z4*dIzj>!8hbrP)#d=+7mqv5@URP#H=vSad*V?wZ2#};|{^^)bph zyLvV3Ad5*}J0`T<2rzHTi7n1&tf#P`54F7Y(v?MVM-cKzCx+0V6y}(1H&jgC8fjs& zU*RaQ>r98&cG;6AIOmz0vaSQP!vd`q?~v4%z#adJ9!=Fcle=wtGkusT9FGx&-MOa3 zf&?u${e6+we2hDgwyVau1R-%LyZM`}&_GcAkXN?7dM2!_UTV>F@@sq8Qoe|zf-nabBOFr zOyFhWvn8>0rn4tQ>)dN0{SqU2>(7_lrr^#D3+!V{-XqK0V`*>6;_wxe^#I)r@#ug9 z)U#6mEa@~)^*}E;(Dn|=Sqyh%HW9Gfow;QCHnsQ;%Yy@1l)`@Io0_a@ErQy(=HlA? z9}ndH_YeR5N~^o=68&m+sh)FKz;>;NGqLl*xNF|d0kN|Zu(98@O4>py0d_2dOh)Cd}A zQ`7=9=tEzq3nh*0EmhMBY#W)wLt*$RHsEC9=#d6$d4KZv5s=pK7%yf)JK(2RApqnF zsyPGkY1@W}5!80D+`2h8z8`yhZpp7OH&(9eOQdePTSt!$WnCB$o+C*c}OJp{k+#M z$-~-~#0qa0_i<5-@lga#0inOW{J(_M+B6T)KJuroY1&)TT zwGt=U05PDcOORvpy=Y6;W2Q3dS-xjT+;b_gI&hZc%mb4NW(EXr9snzg*3_Nqd4mkXABlCC{#%5^I8ikX{$tySD zXu>`BgS+8RJkX@(3OKb#Fl+0-Y4>j}h0cH9pYoKOgS;FwbzE8aspy92noulL)pxZ< zr*p#Ankz%=R39T!iP$_#2urN(sZ3_o{ol{B6-ufD6Bm%iyB+%Zj`P4=ImzkiX*pb$ z&niHhg1Edot~7IV3GwuVxXQdbvEU_VjXLT!C{4tF`MG&GeBbD9n~1ppto9IF?=R@IIPBWL z*uA!J`YKg<<+opTK82CfVU;@&mdw+BC6Bg+i@0undNU!7*t{IjI4t4;qlw9Kufq^k z$1zjizE&`P$(>-Itw6WE~VbIp@-QWr)7 zQyZ$MyPyh8zTlU-eBYC32%V%)EV`ucpcv&&#>lkh40f&B!6 zB>XP*SQd0p$?DF{aw}A;Cu$cEQQraibr*TF|1V7r2nxNXPZyM836HbIk8T zM<7BLT)H3n{t;T1W*k?%OGgL!8EEE>YRw6#GS2zNQQ{xZ@_1 z#0{HL+Tp#={X6xQlV!?W=`&_+|9nO_a#Hjg<65s1!o{6a8^<9bYOEddxodtt^K?H> z|CZ6k(m_cIwbzej*|j_H+couo_c5G9ubC~>Uha6Rfzv9_hL#@%EV$s|@yq2`B_=pA zm06(Chd*2%v8l+3-#C9nW>}M)U{4&3S$h zA)%C=DMIZ0J|7}6ig-uy_#$t&XspNYBlc~#k3GdgCjA`pzOh=0y4`>B=V7-`&>@)` zZO>j0Y3kK7`@me^$JE;CHrMtqnm_HC#Ip*vMZchCeC>t;DKF@H_e-k7Zq;YJ%yB`m z8UE^TSmFN5XSSx}^gxe4v2k^Ir6mA?VEG!rPs7Tvh>ixZ$?^RTeQXxJ9P-3q3-nOd zY4{k^n-C!6cr)BNn>yvDwu;ZWxLUgp+;Y9J;VwtEWBjyy%r5l)L1ba*LO3#V%*kr| zWZeZ$Lv85_So7;cs`=8J+}J$;Fw?J&k}v}zUBK9=*psx=^PRoGk&Sz2 z>3;$Af%|q&aQNh7YuXo&W#*zR0b2b`;l_+a!NiK`adwj!v5HmF*yt9Le~)T51IFSX z*)%$W*Ue}gX$G9~G+o;CdX!sS=b1F680VyzK4+QZ8OY3%m(kQ{=gBfk&jvpyM_iPjWyCASpdDXeGLn; z7pO{fB&=IDPhIBtoM-Oh9Y{WrxQwe1A-!UyrN<5 zrx!P?Ko&Z1oV;%Qgj>qXSz|AsrAaz}dv$+5zO#|vQ}hVx;lIrBU6zIkA(PijES3}I zNmNrhqpn{+YC1_2-8;#HVWu?@c>jgyHi2K74kzYLOs@M6Doj*1%K*;{1|k!)-Biv5 zIGOPdH?QYciO(>s>dwg2d4g0$E%N`h^#?7p!n(Gm9Ss$9;>c%$3-XU?wgo=lC zp%!+v*GiA1K%GZA=pMbS@7;OR?`L0m8uSJ`*Ty<2L*2FCwff!b{^=Pz=&Y4|b7U>) z%A9!GfscJ`u5=t~FK;S1(=WKOGvm;B@o$z3@IP2CRC-E4W|L2fwkxt}{$oF7DNb_T zT3inZaohGdXy#ATjggRyizY)8QVHXvNwopTZEyQsuHGd6o;#ipM%VmS+{zmMB#1Av zmEc_AN%#D8v%N$=J40+943E)xg?B{CZ1Is;qk@z(kxO6E)M825YC#6)bBm=!B?E>G zg4Oo^neiDpquhzym@JM-H>C^#Y<;1&aErgaumqc2+jY#z2CperG`4(L2s7DfOlInU ze4((yDl@QY%00^82+uqLgln#=6Oj#R|9(+3*JTNy3{4y~m~v0sy&ajJ2`-Pzn!X^R ztr@`0(`>2I6UVLZCkjkWxg2cAzo&$eY?De-nAS9XR&+71wOHVd$D%vF-NJ!9bb~61 zQ-MCW<~hGsUtQZ8o}5iEoK5*2E1hlv+C1pSBnek<=PT$swK!YUMeO0<&oE4Ji+O1Q zhE{UO%MeH>)QP^|*n)Dgf8?$@c<&W&4hD7*W7^J1^@oR&bSHOaw7@(FCq>kaHN;tT z+~XkClCJq<=O)D$WQvRAN3DKW(-0+rKFR8@VoINh%;wZSlW&kcL9km;- zKdir*x{#3hBjXzZ3){qH+i0?@XO30vdRbXR+^E^#M9`}}|RC@mc#6k4%1(m(5A+#`dhjY#r$B#4fbppqr>n8%KC%J+;Nza)A7jpw2Xm}mznTdD ziY*)GmA@#l-jzjvx>gR^wRp)W7%Cxzk-#+!i>vrPd5zb_4=adg!JtL&O{^X_{`>|h zrn_fpyETV{vBYf~s!?jeC-VoV8WWZCx5e_sj>IxRX6ag>n0wQU0Y-OHT_Tztmd3~2 zs~G#;h~?ULe*V_>d`QGPPc_^JC!o2K<`2(S(A?sC98qD_zPkAQ6rDeqw|R2?GL#_; z2U`tylnvbC^J7Ux2e2R_+FCjfZ8u@X_C8x@(C8l5eklNk_=%9so3CX(Ah{kB{qBon zCes*nryhu-;DIDt^osQH_-*XNpB+}$;Xi+E{d7uV^5kB=3s&@Uz|<$4Ug|)0Xq@Lc zzhSlo1ybS&hR>PWu_6oXi@@4CCRp)Ppd;A5^Ql`#vE`2fe$6qpIC_{4j_kIv^tYZG zK?4;R9GD=U71-K@7nq@fP^;hQ{+c$VIk?=~$i_sDV?Zq}n=$R@7PReouF-NXL86Pr zU+Q!a0T_l}rfH$*F9ed54{mtVkVf3Urot+?D4sUI(AIBOuNo=Hc9GxSyqJIVou0Z5aDsf^S93Re5l8j=-Ey1501QSM%Y-<+i1zXXXt zN`6vxPq)|0D{XHx-LUqux2AwQdEeTkE(6iW(OY)@ld-KOjR=%i-fiP>lETpJANs64;=g8#d`}x|bvvfTr{zKPczt5fm5&yr z*^8Vb^dBqpdCX{*?A&}oIMai+`Pa?ZJYG9GUuk6B22^vk0{}OWVo|jmiBHZ%Gj3fG&>!7gI@Uej^#>y>h&|PP$J`;ip2bKz zc&uPpNSh(u*i^#~57q@PjaJEw%&7h9&iO?W%GiH_eErZyD5$pRbq8-oN!}!CRjO3R zbkBRpme6%;)zeksMyaT{px0y>TeTK8uZ<_ON8SuX8mnPvi9BN84^y^Y*1&BasOHnB z568alQ?ivW?eSstGeaIG_S7>yBp)?|6s2yPDvu_JE={|gL0RXavpX?UxnlRND_t(1 zJhJM1LB^+a)~CSsdu#`({+89Xo>k+Uj>LyFyu(c(W2nAKHpxNrQkGF0m%lOg!C&7W zYqf9ngvA2v>7*QdcGede_3#w5&@BE|i}Dq^WdtayfsX#%-?^Y!Z61oP9iRJI0O|Wv zVZFSu!orl0%Az5`3W$wBG5hZdMvS6T_ClJLLj%9K<+4it9J_Vs-WObFY7wb=sSd18 zDsNbqBrIf=2DFSvOaCK_!q&FCzfeQdzdj(`b+G0RJ?8kzwZ{@=YpRmRTcn^~0mYf8 z-5kCh-~r|$5LXE|J(kMkcC3~n26#*V3R`3W$3kE6mxP?9;%3mhow zP!vgz>spH>tsei-7j1;DK&@+F6FkFmuR>TSR}Wn)vMmHnUb9q%3M5KOy-4C>9Okn+ zjQ3$$3;m{$-Yz?TjT<>PKxD=m+XW$diW=aEQqu}yRRqjpe6hwB|0D(NVQxC&l=Fp6C~$IeW?I-HWdWqifGY<>48QK*%fE&3;H>*`r(9d@-Z-tp0Oh|!AJG&e z^0(o8YLB_br)Vpy1E^pQAI{21FL`uuF!3QbdO$+N>g1au70Z@4$u_;EOxmxCLSYgF zS28b74=_MIrrnSP%v}6^qu!I7jNq<(exZR&n3AirdozYC=D_<%F-LL>Ov+oZ^T)5b z6fvk(FXPC?&BJsMtQuA4C;-GrO%zp~m#bn`sIhO+ZEG5Fq&x?0_RZPb5o(-jQ2z8y zlu%k~0jYN>^RMZ%yF38}cI4v|nD{X3-D8jJiYf#|#o6#(d+5rJO;eXqnjgC*ACvjQ zXmP{#w}vKzW2hjI)ul{?2YvR*L6*xgy{m}%W9qDX(E&eJ5jqwk$UH5wYu<`U^YiV^ z;+kmn*W;-Q0?P2--(8lxrQXSV{pO_>m!I)>R#T^?jNHx$|B%!3Z{>wvvb}|k<Neb zeo_1m0S-x)8>U!waxOmkz(*7Bb#ww-@;q5BZYIUXr`zXW?8~0j=^pB@hef-?(y`#a zy+dVKo}|SZvfIQm7-a*Uh~`xihWIWcK%(-8NoET&#_4oX&j{j-6CvF_Nct8Hh71lc z0S!9&M=$5cL1v$m4iyAF-E@Z4>6Y8+($aIjOR<;o$$$$tV`+7iSm=j^SL?PfowDt_ zu&XgJTF_8Lro3>y#(*O^H|GeAt~I_1E%OYp-bYANMI6*6@nkJh z97L_(Ftl??FcpW18OndSeW&s-zd?Wsucwaks(o%JdBK0NSzLkoN z>6flosn)qUK-rzvHmFOXzBxG}=*HJ*hV|3egsVLnh$u){V!&wHjp55St$RO$@pI_6 z;5fU>>S(Y~J#?XMc=lKjbxco>hGq-Mb}$v?I&+J_&~_0qdE#L899{aJWp#AI<7eau z>17O|24hpw{X#JCT=~ng-Is?5*B$`g&6A!;;}A-r4$sZpHBY{i(}z^Um25;zg4)f= zWs*jOS+EXx1Cny!hvb^C(Ueu9Cqo$aWrH@CT96{!I%6aV!tKQ+s+O(GR9>GSOVlt-e zz$_?>!-LZKljF2dMqs0AhmFzWIcW-U|p#F3Pbs;-|C^k$e%-t+ad@8gChwag5Gubr zOfH0tr%joaJdpE!S@}dXHPLabDVDvTFU4B>H9lA~(BF}Oe5?kmU+ADZ)@w2Oh0JT!w$zdKr%M4cm!MQ4uc(gdcomCmotMy@XZxO~ubYWfE(cV6 zCRZZ}6av5!6h%@MZ*M2u{9aPb+8EYlzx$dqN+Ig#Zah-4v*O&cIlZalg?~3^M4^+OHpQn@@%W}E#d>I!E@=z$VmxJRc8I{;7n*SXlOwN9sT9YkjrEZe)fqwZooX-l%{#c^of`WrsfRq z@{%!Z7~j3Jb=dZRpHXgdJo(nXwNMZ5&Tz?vz$~61J3%Tiiyo8Ul9^4OmDIF;PbJft z2Ro3FOS_Jq*YF|Ly;1&>tc$g`O1^3*M%aBCAEoe*UtRiovij)QUFpD$%4UUBND3V{ zAY^CgnhXClgF}AD_uNRQ@vCZfL;fy+Kva;ZzI(LgGs-{r?mwO&y-u1+Ex{IPR;-JV zl2hnj8hx$A%`-cnv=sr4h}oRkoC0wU4@@3m_SYcKYp3rY?dOR%oBNh?J)@9f>oDh| zn(8Ac&T(%_NeyW&X+(VC%CCiHwRC@kklVkwtnX=I|3e3X+WH*R1IU+q_Lv(wiD)ug zw`M{f$1WZ~Q?7SJHWEUF!V;CB1kr*-<(9LJeRP7A`;?mkX6*0Ziu+?r*B=8ekqF(S zS8Wgs&>VPKRzE|mw#N1Eu>$TB{I`j+&f%8s5NPe=_E_$=ulA|2`_OtKh7!@MC-r~G zX9~({YCh?|99|w>@sLVK-Ek#LSlLy$^IK;w@S|{dK@(dgu!!!?Evj$D3nRj2-@`>`+6upZIHX1@+&!eDO~@TCf=~DJ1d;8ct?6Bs$L22 zs-jn^n(>o(uU8^wo_tdnbfnYX;$A%!9;;{B^$idCeE@P*{dyH?`0jqc=*km5YIwS}9+8e&?{NuTSuUV{ z5%}{XpK|x{dp#uHMyUJ?KGROIyXZeQnHG3NP5T8CkFdsZgYfkr?Na|;C_~#I` z8OKh?n!Yil>Se`>aot}mX$=Ur-RM{Ul|M~xZ%yA25U#D+7G#lbdsPivVCH zxyr6G=fOd77gWO0alx6Nkh1%yb1X-|UYh`jFfIss9#}7`it8QubLT_yk0(cZLq?6G zMpZdG8yy<4^R!jz$k^?B)nAxjLo(O}1gL)=C4rbzz1xFK=2={Bf0odv(iHYZi zhM$%yg&zO5uKt_O#Zb|WZ`hU49-pF>6sbZ|kVD!E-`de6aR!|&0VIoFS-@^FEZ$8F zf!~H%GlLghmElP*+DPzUmLaA)cei?%{pj}1NCC6(*8=^(qTg?Il*S>At2^8`f0)h%v^B2)QIizgS`BJ$ z_2e-F<3O23tZ`s>H!-6xc<;XQ`s6xbed6A*+G^F54(>v-y7{VgKo9=AkBUE=X@(XO z`00w0z=IfZfAY=+dC?)Z29uKdWq?^&ge7skD@o^n4g|c5FzTPZ6xtjTYCl>2g&v*m zHhEtZ1F?4zx3Si4exkM4?`c?T@Y6BNvg*6<82k0#llW8V^c16yUd~HrC$B#UueG^q zUP1FMHDzDr2-AwOZU(Ox|9mY`&%V2(bZ~Z60%xUzarlaM-bDYRT^o50S5`>YufKAr zYV(NevZ4XVSlP(5nB_#af8SYN#VAf2?zwY(WL`%`#Me%xv;v{z?(W9CiUVQeAB3?u zvph%G1miEG(@y z;A6*p6&0}))t6ha@&u%Week|2IZAAzjSITDPrNxNwiA#g`}|Cx%l1L$NMkbQSN`kL z4viaVU14t9kjW(`W43$KcK2kg=(&S_%JkG+Gh!Q&=OOda+XBFK13g3RTA#=Lfj+K`C7Nd0aYXalmtSa_5WX?SOq)r#?g+Y zG_I#n$M3Q~w`&yhOJ0l!=<__~Q;cddvBR=w1zvq}dMzE>ha@iYps!xaT6gq}R!dc_ z$vy~OPPtuR=@3IZjlAZ@7cE>;K8++*`TCqS_Ch&!bHwxI@9E;d`rn*_u9IHH_PMRg z5v94lewz%|J=_>D97t%7a8r-;D^TrBNg(&JVDnhG(3#zegpzH(qvc=x>~% z7fZ1U5n83f`StG;_cl5+8fbf_vOOBjtvT#9Z1jGp}GJpQb z=*%J*Vez#J{ki(w+*J3om7#BOp%)k4+fI@ho>=nh*j=8!B%Y4Hh)J5S?;ZbH&4M$Y<@dWXXgXp?W3$I2%m*%>E8#q?tS(BVKuU?H@m}z0 zDVLi|f)|2g{{-!urM}o5_s6i!jMLeWRQe8|VIE-m=~ms|0{>{W-&z}-_HeZK7!ftM zEU~?dEQEiIoATPSqE7eNo6uaR{BqooHh)Km5yv&Phh{lIU#&++YRXXhZpOHNc!l*s zmc=TjkG0J(8f}a$0IT4Q%tPQd^kDj803${&>2T@EU#htB3x7c$g)x_IuP%ys?}}DL zvMh43<1X?lzzHQ4c0QUFVX3;}V7U*+4yKGO@ZXzx9rCHvZNo)XQ{{EN{o25t=!NKi z>W46x6KMJGrt{x>%s>C#{JpKGeWWkB9(&$I_s3qT*m*t$o%v}Sib93C~I|dhyDBO^DtRW9ZvL-g%8RyZ?dc* z76;M??V`7SG)&f|?^yPU;$&NOYXtm3tCc_~KE1PW%0rFFFsN=q- zB_Qmrpf=fzu}uk7D7u3UIP%rE9(ko8b*}c>FDYGP)gwpRH!oFdBj8i;C?5P1^*ui& zVFELu`U}n1_eIK(g}8^B7yYIXvrqg`_7cqH`|*TDZm9lJ;qDSZ$pK8i3=GS#tsTlWT>)F~I0ljj zo-v9jkQ&#Rf7ZXP%^-49nDSR7^?$qU-}z@{XJRkX(0BJ{U0jeJC}uj*tEQdm672H6 zMjerGQg@!cZ@SrHYp*_M^V`XDp~foZ<2LHX82;Z{%w~%fET-Pk()$j&dMn1F3C7s5$hG2edPLP;PR0yk2M-j`T1so}n7!9EFw`cJD^+WcQ|hecz&qzI zFpxhf#*OsCQ0HeSGsy}yf*G^C8M*~Atv%15mydqg`$HRV+0Tqc!XVuBV`DIS1F#~E z&Bwa&oR>Xrm;)iUJl}+LntN);!*H3D=l--X^RaF*b2@{^rl`VI))xhOOVNz%lB@Uyxx8uMql9W#x-)r6`~3(lYQEQ{%AX^juO!An|$mE>DGCym&@Ru$yo9x9nb z!^73-3M)6K#8TG?pH@UIfOCgy4CiW9a{YIMS|hSMy=$IyadlEQ(rCGZq&QG4?;?JU zmdMyWG8fpHN%aX2c;WmX_yUHD>G+QQr+WTpt3ikdQB9Xm^dm}>qJM6jP*csRQ_c~d zR->J&D}%Cr{YKj#uPI!SEQDtM25y8n0#EC9>jqwN;vRg~-1a?lP3nFjAb*nL9rxa0 z(+)!$aD4MhBbW39qqlrpQgE7S^4-YrlW2WM(lOj_NNTR|`GSv`wgrD27eM;I)5?jfh?i9z|@ zCm2&2wDGg4Z@j^i1UO$bc#U3ZQ8m-NupczI;4cL%>C&EFwqUiMHp3i{wVNFebDeEh zsev$M2-%@68Jlxc*%tw56&sCQ=)^40XD%1EgD~8=buqL!7*-`v+qplaaBy^V>|7Ra zX!1f(`-q!H|B{xol9kiki_qqIeO1AC9yPXiXTj^OB1_1#CI%(fE@%UUNhvk4ftMUJ z`@iSftd8*M^N6iUru;A!Im3&{Ej_y)vo}xzrw$j^iFg@*j7G;_7ic)g&x4nL)8#k$%dJQ-N#eqGun)ACw z(ue{-`=1+1(zhOYUTNjRAF9tr;TXs5Pjtn}%7FvxQ2Mjo^T*zabHd^*N$}TfIq-M# zpNf6I)c4#g;1pT>E~U6=y-jME>Lv3}wHro7J45FMcM)0~?Kd0b9+miP(VV~lrUzQf z@Yy1;8l8&}h@@P%vm5C15cZ@NO7;8HwDg;_j@@JV3(@+BH!)XM3jjIJI8%niPq{P< z;|XKuv+cI29tMsp)e^X^h$&M46F3|(d!Vijw!!Rq=}-UuxF`@4Z8r$rsmZ!9CO_W@2v8m=lItU8vOdex*{kXfm_l^ZS^gq(eICB}y$l`K5ei z;RjBqc38pgC_Ia#?t5sZ()K(x^|r)J^>1}O&3yGomLCw6FLwntG@8HJC3KS+#gn6_ zbQr7xa@4Xdn5+K^(}LK5A(2cKEc=nkP#fADl-%H&ZUsV2mH)=xDIG)lqf5H{^w(H& zIL3F4=Z`|)28s;^P8!v0g?~mV(BMwLu*|XY*Fkv#Bd20kg|7@*<*qs|l1y(84Bd~^ zO%Nh^Hu<=BK22V-p$0QI3iSNVh#(-#^3zg5RUeS`9V@w1K5zN}Dq>#N;uJ0DSpkhp zf%h{wZP=2L#Jn-EW#mEl__X7pOf=w>KGpZ!KVL_#KP0-c)jj8N57gE>!gKVN7h7pi za7Hs~cs&#~5j7$v=b8{_4UO)1EDFhz)3MFDF5ce#Chcn{yP+@whqHS`Lz9zTg0bdD z@@I_UuO8gbY7FIpMei=oH)?Ga$Cb$AzZDBS&r1%!YhNsSS9H1@GgFiFbMI1O8FaS; zm*4qC6fQg9sK-y~neB1lR5u}b%h=NNCGaZPg+LR#nxm4s^nL7myoIRA8g|_m2mQsr z_*kl2xLL)*$>WH%Qt$@Em;APM4-EryX*@c>?;IXO@8A!yQ$1L{<4?G3Zf-VyJ^S7R z_}+2t@!b$vx2?NMiaaztJQ;-t&|p3WCnO;G08?q!3usLE7L22kYM;9QR#5Qlh6EMR zkgJ*fEw8k3nv8-Rgpq+URt4P`*%{^Rk5hc%GcVgOH^>)%GadZO!WBt(C+@+o6AYy` z!Q~q*GImmt%pnP3ejhZK3SnN##IY*_7e?TPhfLG!L%H(TXK{VbC0rO|)f*$YQSy%| z&ST=g3S3-97Kr&K!FGfmSb zTTkDB;Si4^Tev1({e4sW-N86n0&?|Dut!~3Q8aP$Ijmnkx{r9~9H;(HPtET?9qPUf z@c)ZZ0DK5xrF;vaQDCLxRTxD}zGXB|M-O(}E8!4Yh5*}YBI_1$GjmwJH0N_=uAgw^ zedS`1Bbrm{FnYnj@?lMm7q|GYH2Q|)ix>4MPWRI=cd)E=vP#bToGk&TNS>sVKa4*{ zy?QCq`(8~ zGWt-{BCvH*w{1q=?0SH>YCwvyg3c(z75SGfM_OoAJBI(Zb(n0Z6Gx@rs3{dCSHEN@ zh%gb6qgAtbam@Tx^l+g91BZbaP3BD|Lvc7a`(-kd1n9?(uTB!eVi>`~bWy-Iu6Op3 zJg?|g-@cTlG)}?o2dyx;?KiHgJ3_Y@0Xg$OR;o})1$ss^OnVRW=T7d^>M8$uPF_%B z6!X5G4byZXM1SQ0N`_nvVE078lxbz#k=teWd-TiOAmrxUHIiNu;xbpC_r-)Xig7#~ z6Nby`13B$3TV1;xF@*KIRPsaH6h5Y;Q;~mt4(s%h!IlseF#+m2dir)Riwk=lU*cEj zL(N5AN4>@2BbpKF-Ev#5=-rOdTwxI}D`+ZIk3C*K>GpU~O_s|r8?!eKHwv{{-oWrP zaasU8m1o0?U|aX%JxTfTPEIJ^M}b4RgQzn+jgaU?-218X?%HyC)9sXs0S7~J4OKt= zBrr};arCXC&KFy_p=IOW5#$A|(f>*l=zV1=aeIQVmg}pM1bX&b2X{P?-!0kk|6=T` z+TvWJB|~rx?h@SHJvfa^2qCz;yEaba!L@PM;O-XO-625G#$ASUvGNraB>@^s71(rupaaBCJ z9=XK2!eMR?*BKc7x2eag^vxMGNg;_LUzjbJ3^ zQ{A>+s@tI>-bkb`x7CkBO-C6_;WLeWPsdPbU0x8Vxwn*r(HBT>xqO{eei+vG8G8p> z?>o>m&>+5EvHaBN{K7g;ZC21{0&)PO^1*w(DRJrQ@ zS7~_NxcYSe)?@nrWP|^_bq~4}RhUY=O>*&J8U%~61iY~rh*a7&kqGlHOAIf|{;4o5 zBwSpA$h}mbCw7hF3Biegm5==t$0Zhht&#Zb0PW^^{d`!8UVpi2l_iZ}a@inD%xqY( zesVjKgY=og!qo$e1F^z{TzQ?APXm6KJxtw&j|E&S` zs(<4rW2lh(`mVXGpF(NpQ=4B+g+#7Am`qMe^#cr+^|noJ8Ll32^Lus^O|04HOT|W# z-8_62Z_&gse!Q!-9w?Y>NZ7Hml1$+ziXY}XqlJ&dRzXKEl7|loCKeqgqC(8_br%`P zt2`~KM{D+VH~QKHuPMY3mfw>fcc8+w6r*BGGu-*J-ptVsfIaP=kvu{G4z`cD4jZXa zJ$Su&;cGyb-Z8Y2TWdJ)srugGT-pU3*7vXC-P;!I>x1YUCh#^^Z~K^4KV)V)bN>5m zUQOXS?CO)d$Ce4@xnpP9tyk!M}KQ**@x zsXsaiC^P%w0T3rmsZnIIrl-_3=u3$K6p&*6go(-lbB%o4=%Sc8*8$f z2Lm&J4Ud!td*piyrKRHsj2^B$a`e^ET0)ig=+2r~K;?S*$)-nEtbKd;5TtWrd-qc+ zSV*klP$14+ShV*w`rw^VHhJ!8za=uVK4Y0}b01lT9A@?|8nDmD*-#BayY$w%13s*O zwE-&|)#a5Z+CpF#QEz%_OWW*{9)zZNMsC*QA?>Ywysx={N!<9qYhyKDNT(=q0v$o= z^qC}mCotA22+U&>Ti4iBpEaPyKPo9tJB{tEXlk9UOVc|Lwz_W~qz%F1$PgL&orCiH z`b*M^7!#q-rS1e0u6@}g;Z~On!uZw(q}ycqdvT!YLS@;%_%skUmm#n0dDo9ie1)El zl8$ca7SC<)b5o4O{!8s#CQGQF8ydTrc2W0s`F<^)FHf%#B@{Nq89y+cg63Xai{+!T zNIJW)U2|3brU`1ia|wc9Lo+B>31I>Qtn!i2y7&2KSz6!yi2Xs)zrz_`BJj%_E6YE4 z`)2|54*~rbpkojYv1l56cDoVz!0qz|HXAeouCUMPGig%s)1xhY3E21vrE_T3N0kJI zQw7)-jOZAWBTA0n&|}kF4oA1>*lSpNTHxe_aP&eYITHyha(?-?N2{khXV2IGIxsb< zS_mY#bv66l4~x^!Wa0@1V&g6u?klf9+^!~Nc^}RQ;-~bQY<-8J?5FZdr)IDCo1mc; z2_F3M`Lv0jHcEzCJ0bq4peQ^vtF3B22)B$9)x9fb&xi$Zv;FRmIF$k zNUHQ~_q+eFk52)k$Q~}C#z;tt!14Flko8q#nW7Sgl*Gvw?iN?fjs7uXv1y|d*2ga4 z?|R|5cu2eGnyjiUZsuCo(xSa_!|ZBEW%6$%uABI)y{YcDR# zq1*n`?{SrKzEMr=?jD^1EIV08^C0s)n*aeH%~{W%ue_fp->x(t1|jR@5$Drw?<=JH zVDKqs!t(CN7A$xP9^4-Q7{_KUjuk)Z?=uFX%>!^K2&$d zc^J)a!TS>F1K$07Qi>oXZu-NP!VKRwSfR%|oXl)nFcw$u?kBwVr+jtNc0t;?L5r{3 zK7@aeK(4GU92yVHmgB>_>3dviXzLGDxBA5dd(Zkz;ZwGUrN&|y(Y+tXFMk^PHvY(> z{0EB{C58tXK3)Fz&)SKM0qlw5-kT0`ME6yUxs{sDnTPB48| zp{S%9gXFU=X6d%tBZux~9?Td_SRY_|xNQY7HgtNpiH}XjkP>ONkMl$=^iVCcQqZ0G zH_}YfwBZUhzC6TJK&K`ITN8u#WaL#FH@gk!L5qT9tWp4@aT+{Q(nzlVLlff5?4~Wa zTdwTHTpn2IIP9P@HMT-U>wfz4&)K9M1SulHKdsoVian!CVfFQAEnoK-6lvVYCb|8n z)It}vumUP|!NFl)Vi~%pR2%?H3ROeM&w}?$zjN6?_n#|#&6LTSR@{fVd9=6b!ygGU z1?}=Hl(nkCBUTwtd(R@+^8}-}q*r^f{Aq90#W%{9y~blJ8fI>COB!&{`E7ROCkk{?Z&v0dc0Ng{VH>IP+ulg_c2Q%*6b!UX_AqWf8{gWCgk5DdRykG;K21eGi8yL^Q_V~0E17U-M<7f9;xi7_YM>w_ z8+Y+dh1bV4YOIONLQ{<+<1Z1LZ_%a@E_5R+`%R~`)PFitwl zQmQRDSOHS3rbg8R#H3ul3MN(koEKr(3UB~7y3(&@^a%2=tM^vo>@KN71(sx(3YG!V zQ3mLWtx~dTYIR1}_8W?s&Hy738;^UpzLOf`Eg1zGTd&wJDL=C&a44DU(G$%G`2Cvy zTpNy_RYn;7+@Q_oa@GGKLo_5fgY?07rp{gZJgZg3r&N~cVw7@c2SmX`|03`&8omE3 z8iUZc^WZJP^C!lNIFocP%YYwtBo=LlUW2bqIi>~RvG6|Jf)CGf9(uE*UZj6#Vy0PO zo4(MoVN(u|(}XU4m_*xf^Z%HShYIWH2iA+E2CQPY4~4rBx{$q;0B~I-91^OcfP&Y< zwGy)0Y&}n72_JQ`ofLB-9O#PJ(2F}nQeM)wi5o@zGMl#z|PWPMil_+=@K$W)+tf>vTO~>ObC! z?mV#ihFvVM*siKf7F!?~gx-PqtQUsIUWl9Y1`VAzPWPM+G3&L4?*gyD{p*B^>$N(P z`^c2RfiK`d`}gCtZPhO?iCK!)zhH};EtG5apc;jGcj%7@DVU9*`nRP?FLt@G8UjER zL74Q&3SU+ctnPcUz{f z;A`nf$k$q&cq6iS+nAQMP;aYjd7c)^duFpM|N7|14U!|%7mwsv`6%h&aL#NOX+ zrz;{S-cRNLq!fOG4JHw93;w@mK+=SPga6;-X2PNGF{XB8E=2niiH0;xb$~!4b*EA) zv#%hb@G-XbQl${9)M$Q2Y2RX=0vlkLe04EJ|A~)at|QnS7ivIttKbW>L465bgxW7D z!zaGM0+b=t8Vwk7>Mo;JIUtTaTp{RqB*$RAV;y}hhjHj2D&aX3TMs`u)SX_jp-CXY zpd}gS>BIIs%_=sVy3w)4cQ#TSY(0x!>Y4L{%N$hNfE=9JN1oArLNkf>$P$`d%-O_N zMr!E~k@NxT@jeHC&}qZ2<(k5k)kxV3uhX;IHfO~z2m3^?pslw(IR_qg*zu$e$0J(q zUk+QpJF5mJ7k6d}E?^r=qZBixH@)NPe~yqCF1N>OMW@c(X+aY6Fv~Cly+}o!+Mkkl zhk&Q|!=JX+n-X{P{mYzoTiMg`C!i;Gk#}9bU6p6A?M}Y!YCmTH-*81eUi|ZyalM;5 zd)BLcz%N9+YG4`a!Q@8W8JbRSh(aLezjEh6L4#gY8` zGvb<(M@I)kL2oE*fq5H}=%dF68euqH6Fet+(N5C%SdoyZU}J_&dfO<3j1_pL;xSi& z#V4BPI^vyvEY6lLz@|^9NMb=K?<_?tdBPh-jN3prqVkUPKuGUJ zZeMBdNOG>Vs4@k_H`4QSl@%C`*LmrUGHC2pt!M82M!12)bWmKcSjTX@A2b?T$P6~^ zq)&3+WSI+-Am&Tq2;+XY<<@?uiwsd?C6OIIGx_bVx4th}jndqCEUJF$bD^rBtzfNC z{GD&`8heKxS4ph+`|?g1)o?it`C{93C#F1!22=ESqD|IgKBs(MRiOYkC183l*o=#c z%U~3P2{!t$T=SqAim}9wkL;v+F9+^=UN{;UfK3ssTFyqLM#A z?2Ouz!(Gy*Y%Kj5Fj@Yli|_N22bHU82H#uKG4OihOBy=F#~#tUtjDDqC>x>zt zsPECSY z0LT+zJ$Ow%PNnidL(aY2(xLrvOARq$UzNo@TQyD+Y4cLcWlFf$!Oh~K>X`v^u1zf4 z5p}T+wfI{N=d}#lcqX9BVEZBEF*+_sZWn$rk*!ajIO!AY;me<&t=bbZL*x8pR8nYN z;4>XbavHey{I&MVe0`95wdLG5Urkgu$<#Eb`p8+`@2u0^)ebjYz;wBNz~upYB3aZ( zuS=qw_Lwo^I9TJ@MBUJ9-}6m(G8Xvves~if@+`;C|4rODm2jdEdz0#eAil=fq}A(X zSLe{%De#5-Z3yU-o$&B;c)t2-M6+vXJ*8_zUZ03_LmJ?=QX|Z%OTZv<2k8@ zX{S!@^F7AwVdm~Er=Hg~(l>RE7UxN{;M1W``0Ub0KT7f91~_IK*C}AG>3?T8g~m&l zlA{|dP|#W`ohI05;AjlAfUIidKyc?1`8%y%Ax>-t?H%JAH`*t99TNszJ>istUvwv@ zQq{6p8TTIsFlW+XO0!r6`KTx4IXu`SxfBgr5SUU%YH^LOOElJ2csrk{_u8sG?M~lb zer74+S@21(LNk#{2*C-Z4 zyCf?+e||9)q^9nc^e+YM7Miu3Fvd5!Z6clgeYc2z38N3ix+kkIX4dT;+=LD)>)fYH zzG{S93Eb)IaF$k=5l$fH_oS+}NDRY}f5yzxW7uuye8E<>J&w_&m6Qie(rhU>mZ4MD zh)5mRQB?!a9zOre@76#67u~<}9G^Zw<8;SOtfJ6rlCDpPWb02m%3SuY?9)elxe}}1 zoSw|>dimayIn5sO`y>P!uyCdBS)t1tq8Vl`IW-qN3?07uuSl2LYt9|_Sq(6>^YQw@ z2s4O?V!$pwTS_hBViU8Y{k>f}GSR0z+3L&Bq{tVh1I5-zi-U}m5Ob~y{KOb3u+{A| z?4#EMmt(oIKPo=8Yu!*q=S1zgS$QhC3;lpmNr=Xeq7)>DW|O6DlZfB+fbc7SXjK#$ z6-r61V*BcmJHX7PX`r+HMXe*M$oBL2Dc^Gf3A zwz=FltRm6A&s!q%mkzGadDTHmwV>avvB#6H>Wm|g_W5qM z=$~por;&GWh~E9koyXZcLi|!pHu(3;G*aDnc~?->!68HulE3#3aF@muL}HFTF^$UM0EpEIyu#D z*Z*Xwg2c->dA77fMnslP7kj{F8;qX&v%b08tGT%T@0>_-|Av4XD}|zQf9i}?g2&4- zVoc+ao*lO|yM*ycR=)OFH;U_VBZ$xuVWm)lGV5Xi*L%_4;bz}Z#eQxkha%_mW z2C2$g$4w59=_fNPX;XmkVm^e0W^s+EbFQOC7^Nw479rn;Qc+u4y?Y_Ry`Q2FE9ECj za9!U>d{x;hYdqv^VyA9D^pcyIgBCzs{+O%DC333Rmbo$YM-sHxMf=g|xW64JUewTZ z>xu_mHz+p>eKBh+QV~=A89EU?J{A(}2r)w^N6iL)z-?1?_XM5b=U12f8QqlwIr_c>UbgmS?gyXZ4eRPycLmx zR^@6x{`joob!Is1#^^d#_p##ekrc#oB3O5cetSwg7r&>?j~$^po+Sboo_ntLp$G}j z7v{Rq(CYc;F=85Q_#SI0Uj+lM7MZeLT$cygBOeJ}+xkqdbFi{!HIhKzi;Lx#hmSi< zgrxYSz8g`|3*anHm(95CTybWflUW%n1U*bk=)rhN46EIKwD z?%S0aTj4E$&d1N<@gf8J_WB0@ULay|L?skE2pH7BnIkp}Ofw>E z3m{aZINBoA)PKZ^z|-HKXZ20W;Js_E)4ysE{PeJo9Hz7U!zdk7PZT-B))@S&O-s2< zZZ5Szaz-akEeaNb%*9Z*oLP66I0P>vR@tvwC%?YgEmj;Xqye1<1ruhz6>jw+~}@(5-gqpt_+Oy{Y3Z7qBI z1y?o9qv`2bV*$v@WYpS!jbfXPek>FGh(V}Cr_+s7I&wUH=UA*nHk zav;nn^HoA8e#7|vmY3=@Iyp_;vP{S~M@gtYj_!Ub zY_USBN2HqH&HN_bx8%DIz9rgRMK0#_zH;SusHk65zi1fkOwmc=R51OjrFeuolHd6y z*#zq&FNv7mez=gMaSZeKD%-gbd&*+YxlqJqjku%x9Y6ehe1A4NK3T4&*D{@(mG>XW zZz7tVoSt5Q?R~?leNN?x$3(VrNuL)sCSFK72=$V-LQFNbP(}>690BtaB}_CA!vDTS z-m?FV&S^cQ5uwG$=_qeTPW}jjKCWC7v1z$WgbM$7G{t}!iu!|477<0Z?217!o=Uh) z>0AyeX-iB;4wvWj&k6-%6AGP8O7E=$j{z9|>G*qO2MHexFC0BjzV_QUHhFPpjqYFQ zNbss{>EFXA7ly-YA5pyGd zS!iAD0u8@_2#&|*t0OR?1&#H%sTq|Pev3W|-)}u=VMH|^k1G5t%6a{heDoT35y zC#Do>3!Kw-*MmQcxLJo)_V^TXS~v=Bx2f5u>a0ceql68qSrT9R4Oyslv{T?nm@((^ z(HOJ38iq8xZW(G_qw7>=q$vkMvRzw!b;yqu<{k_$1X|&fgp{tcwW;t&rUjPK*hVK9 zqTR#6JzM^Qns4s!Faib&zFTZ#mq}l-8bg6&zkCqmE*GoYyO0&wb#qR!>V1PlWcv3J zYxsLA6kn!3IZPDus8SEU7A8vqxeI@OY;EO?vH<@*A)&EU!s;gY9`!d*`WS8zjtVAI zDwAQR@le;Cd1kH||1+G=wayjlY4eVX9{I=z&RF!gGjM&)^T!B@hHkA?Ytb?)Ff^X-p^M(0c zF529Eydh!|;p-h2u1)vMvO2j|PS8+$E_Old@0PScQg%U0lEp|e-NUfNZ{P$$`P_JV z>;0puUkQr+7py%iIcz-W(yj;~W4Cq#Mx{ZAp0Qtfl%Oj>!DWL!kwd%m0idFGxnL(( z39Re_C7APTuWArv1ZPH%U~9RTy+_q;X1+QBYB~Jd4mtrj$HxohjLj}n0r!MnL{me6 z&%B8x|6w`TOLYpi=7C*7^DSlp0(OY8if21sg~o)|TsDzZKigHkVOZn3*cc^RCZFVp z1}CF5RY5H+4vt~KZ|t~%N5eLAcA@FV#tsD`zvMADv8dOIo$F$fyYN8ouIgCIPC8M0 zVj34;06U{apCaxlFF`UHC-EAz>fG~>PuK{RTbVU=`96C@gWeF2ef7!fx$_B?(~z|f z3kJCKqfkN9Fq+utl;XnK6y1r|$^kzu zQGZvi8T%h^nCF4004hvj);G=56Rmkh%wl z=7iax6Tx?$I2CJ5Z&S{jd#u*(I|$ofjWBpC6fVe_XtZ^#vvTu23qS4uOV?$TiEXqvVGU4~?^gJQavmu1Ot_ zoxfd;4~pYSX4-gNRhtDlX=0^#`h*R=+TUQOu3-9Jnk?2n>N`hu)y4nsnR3wg8PE= zn%|yAmGB*dDSt3eAb*t|^TSID*~QvC?55vdA{d+;ln-U^$G7jIdfq!9+HvrsU+ss- zSAm{ivF3Peiia~)rM1=-#qr-!+2nhjqa48s8^w4-sWa&vkoXl;NJuLhF)5j|0QmCH zc`$q|+QVat-d157Z<<0Vg33MBgA8|G(cDF^Yxr0NxAf`E^mw6nxj2`EFgDt)Oz*;% zC6YkD3L#HSK7RQ;x9P%ZfhN1KNr9CRme@{fnRgdfk9Ut}O1wX6gi-%((t*RkPba?Z zKf^L(%Adk*6;V2L&m1ReYzUKGM;l&9I$w{ReXd3)9gVB=swi2cX*N9gLUs#P;XySS zoNN0wj|3x~US^b`vK=U6c+8`PGd#L?r!f};2qxlfndGmNEuA;4p05k~>dDN-=D&1E z*|lF$ulCx;l10tkFJqTU-Y9AM>{W1@M2*8=aclfTIgPGTmwe_5g~$wB|Mq(}$x$nn zp3?7WaaH+6BJ8xr-ZkgR4sY9biwE)ch51_(7FfpN|16Tm<(_Fo|5$4g@1ye}XrQg%Y@J^$e|?y7qt58mLtcy0IFFBImqoMv^@qozS=uBFHH!zA*) z7ia4S_h;^1qt?}(5qG5Arb2}3`rSQD z(Z|ZQe{CNRy>b5qj9m%W6b0Y=ErUt|M}9ac^`s7ij)Y3Eqb$GYT3DuD(UU9@W+r+J zBA`+GH|Vyq^C6AFP$C-91g2H6*wn8tek^HT6Mq>~PNw3zco#g>Dw{+Z*YBGAZ<=uE zbZ0v9-4GM)z&q_}4Aw-46G~n;sTy)ccfAf<^zyL@_s0WEcAt$>#pe)hm&7V_cvt6t zcwMwaPAIM-`cu?X9ky)OdVQrA=n*#kQs+m=f)~dd^FOKIfh!ha;1%oSy)R<+5;w{r zTyI{>`0_JRR9Gwl+GhVSCU=({qx5I#+|uaE)8Cq%1o!(HI79HTv1Cl_h>03K1}-6| z<*ZJ76N?_(^NY=oYabkof9a?QOs3dnUsU2nub<*xouA2Iy^c=0d%Of8YUlsnxW_+G z_bv;xqbg6gTJ+N;vZ4)Wwd;Z6VJ?ePNtMOmeZcZQS{Cvv@zlIO*&P%rN@uqF8SS&5d5i#Zr+{r7m1vglg{N@^i*!)xaZ8W9O14+qIxzNWt8j_Fj$-xraaHNFxE^2xg?mzoWc~K(R5+ zMt}S7G`vDA+4O=l7hE30KFQTXe6P2j=PR7P_bE$ToAy`=2A7lTN$S_Z4cUkLubzA% zmdr3X#9%@1VG0C1vMS|a9?lh>U#U%F<^ipT>H{P(iF_xK5l(!DZmnDImMJVU<~Pl? zX`mq$v%4O8nV1~CJt64Ae|I^X^WDd3a{+zZ^vds@0PHRCLWMMn8dtsp6?7QCB=ng@ zv_RJn)ToS3G4VhpV1N6f=b%LEZcGlCmh-yfgV0BOG#A%liK`>v(BG1fT!CxS`Roc^ zWVv?d#92Xm&%mA7fHeKH%wIJC4gRk~lpUn|mPa5J-6!BvKu3Pq{UP~SQtOf;;#2}- zeIMcx%)!S5qY1lklkKO%l7ZQ2&VjR+9d*g z_;(f?9Oz=GYGx(uOapL~2TM%DQ+i%N!gWPqU%`jdD+>58;pB8sH~_6lI6djlF~0*B z_VJg7K^67rhqB_XH{~%s+0~x zEP)XiYV+Yd1~=D?1Zrcx3}1z&=Um5DCwb+WAb4L{5qK86duV&`s`vZ~z(214Q|vA# zNtGjD)Oydc`(QkB$-V)MGDx)>IgObjwD_}MnGEsUbc6_R|ILw%KwPvF9;(Fa>fMN5 zq-)AH_x1)cDyayIL8HiEH0>!=%8p1EphbJqEx`uQfTvm41O zT`j{f$>)Zv5J5G zXiENWKISpX@4%Bp;Swezbf_}mTFxazA+U$A3w!l1PfJY1kaA>asqk{!&N>`CX#3n{ zR688pa<5o$DM~6_pq2GD*DyyRFO??3tM3|+Gij55_DI=}lP9Z`cfgHL?D$?kW`=qr zj@ZLST%~UT)$SFfZD{lm*mg z5-A$bo%)Iknd!X+)(z;_@jeuv#JpkT_h_Q8nNmMh8aqnckYXm2<=n`;@7Q*TXF=gI z1rUjuoAtyWSu}D@WvtGYe)Du7!kLAs2}7;jV$SDfY^^!I=)?Ex?WI@|U+7bvhh;T$ zuD51g9}~@qK7q8ge{qt>&<0QASB9#f{iNvF4Rus%;p=UR#jwUdChy7mRbf}R)GCc0 zHqf2(K9PT^KR$f%^a`zq3dbN@O}uQ+%)Meiwa9R*m8kPg`sE9qp6n!2m;MYDk8KsoIp7-v0oxxrQ0=f*YyVP1M%K8Ha9=e5loR9$id$( z_>3u%E93KM(DeH2O!%<9?~_{0_cSESB#wiX}0y&5RP&*Y~i!b-ic#81=>KtKP=xSwo zn5yEmlhgxp4t#JP(#@OqqLU}c`u-^Q<5&fvo8XW?gQyWyY`2!Rj3M5fIjO~wjZWi& zLFIHp8-kJx^vNA=_YWFpF4A-4ic1vzNXIK;2^`A}NiGyFtrg2y^yMDV)XK+bZjdb>8S z+|ogxmI<>}egpHhtYBtMXPY`TTHX* zH=aZbYf~y8hTuQBEK|LD(|!ijHz~Qe?eNr^VX@P>me-3VtgeXuK9Fqtu8l#5lv_b; zdZ;#XHF)o`ZiQi%>}0`BoiBKSJ>dVofR7p!RB)?i-{|b^i{0EizmT|fgZDWuXT7{( zT{q<*egC!ZCxcl5nsh=ZC?R6xHmFH2Ee>o!yM^rbunKg<%}Qsw)I-&3SxI{w9N7&_ zDG|~$_^ur&Rrf|;xHEIuFQ>JEz_e|4WtWdQt z^{+M}f|#hinUWMvv(cu~)v&Q+-ZU0q@9UrJI__tl_t90q`cf3SNWR`vp#*}Fat9iR zW>RRoQ+ou#(9c zWYg8Z*YL$Zk!uk|fXh96XBuv&JT;8#XpTn|`^(C|H895#jV@1($GI$s_I}qG1eLjM z0-#1Kqvogxj~}7mnKDwS)BF<5A=y6unIRZd55>y8G45bqe1C!6)FC6hoKIj^mwBS1 z-w~6|N;j!=VHgn&ZK#rG6#QGx%ML&(P@4sqoJ@T|2xg$;RZ0=M`vdCG`ivWkALCXs zr1{-Ww|Kc#PH@Hec%L+(L)WcJ-~6q|m{iyUhO7O0gwf|vzPhI4#l@;QA+T4I?*Vn= zrJQ~foi(wrxQsXi!Exhqt0xM+OK5AG6)kN&yG9+MZ~9a7%_SpB;~iaSg8tja|Kj=Q zf5LMwrBe%4YW)JJ7;0xW%L39=m8T_GJkR*Ug`)!1ArZfWc%e92AebXu$A9Xh%I}g{ zu=d=3D))EyV3p5A;-~hvQ{emOjNf0oX!PRi|H4)zNQ}o@tc>QBM%BtO!G3ddc~h69 z^N|~IjV~Bh|&qU@k(Ab!H==>tj&!>Jto7@?$~jYhfv!Z*5M11_ff;Fct6_1&&%nYYSOf6}3cKkZU~Ni=Wa+ z5`zYZ4=~)G=kIRW5rRs-%HJa&uf;nQm4=a@mkx6_$@sb|z38Y^!eGdEnHa=1Jg`&< zYcFJu*|sce9fl8Gh5-?tcaY>J<{-8=-b;0A9v$Tk^LbmXcC-by$xkVTuWV3bth0$L z%(!dt*(n&P-J6H*2_8wyXP`qQ&X#xehT)JbWPEwAVQq|Mju04!jTT2vl<3XpJ z^$gj+fU=*9fy08L5AOA&obfae&r3F^b6L+P)X8c;z4NfQ-&fw7sB6CLZ0U4Tmh}zO zjp|AVz&@LY_u8Jt<&5V~CyKA|@1wBVJ!4k~j0Lg<9}SCWo^GeS&W21MqU8W8065_K zLe3qE>FC>A=M1{r@|COlgn`J+p}nP4)O?5{s^9Aq&Xh5iS{Nk$OT=uIs!&*)Ry9&y z+=*mniJpXqZmP$hZ#i(?*TtNO1VxqDD73_oys#9D=y@1xt;hKLg!;NWiHH@z7$*P| zLK339+oUmNB-`eWqa1k^nmJ)$nEEl~14dWob_#OYEHvCMN%E>@>0Q;V9RxEBYo$_G z$2&N|I1lD)^oROn5r9`?!?n+_zuLih*$E4wUDnvL7MCmvcCJ|)zp2x0>g_#c@~$uL z>SCJxW)8ltv6F4sIv7#2>NJRJt=TasPo-!UE3^+$O0{AiAA;1*c;WQW_BO-KVDmyt zEm&N$*SDmupExMi08%3RfP8Lg~` z^1m)^lK;rPLBHhu8j_0Wz@rri7#O;3*&V8@Z)f9lICR*|*sc+X9z zN(l%Df2pKe|7gLu_Yh%Y^aD~tkLlX`1jT}AAeswm`v(cxs68rICErqu-mn+K`*g$@ zT!E)~lg|SP?`hi|*ggJXr;wnLVv%IIum1JAM^{BQ?E=T*rL0KfllLzbwg;Fw+gEe4 znzY}+o6u4Ow=A)>u_$_^7CM`6$yF*)CexaZkJXeg+i-^A-1 zNA_eIE41P36=TrV~Tb|y(Lu{CO4qfDx#u)r@lKNEw>2pdXha==+~0YLKSTueW+)8 z=;${#6C47!BUFnLrwWy%Uhu&@XMuW9o7asBao4O>E}ut$T_5Fucw#`~?MfC)6kpX=mR3atBNP9$KyQC(dX~|{G_nxgWfRN8`fkK zP}J$TrP~7YLO#C=cAlR0>TRsuFKG5bzqad!bt&VSb17qtWZLIK#NS zru9K9EVCB_IYve7h+qU)7~1}epUq7{XJ?qS+Rei-nn}$i{Q^b&>xc|!T1*!*@b|Mr zvYZfsrXcj1zZMMz3m>_AGRWUOSr_GScFj`FPFUjoxbn}} zQC42!&Np!56B@~;;om|_G3G23O5PiMVDbotZ<@|fTl^qJ3a4D$F-P6eyZleHgk(_q z1)lxb#oR%jiY-D$KDX?v?u7#n(4>FeY=QF1*XkB$=K=rb!TPY`PWuFic+#9%^+*<2G%Xy z-xus3h&~Z`H|d!?JvyVf-2(GPGXIgQed2!qmn1%`P;I;;KP?xn;WP$wVb}Zq!Y&rS z{IfD_>j!=3!vr;l1eA=q_FBPaF*YIC7^9>Sg2EmU_C7|PQdj)E%sGha;qXv$b5f3V z5J@&oM3=+j&cADrTXgfZFr`Mo@LT1qJb26NqNz@DpfW^cnGLTH4@6zqj~mj*?`bx2 zYMzoU7(NHX9j2ANu$pnITgDLhzJtPLN9ryX*0s9cM@2rMdyzIM!Zv^fRIj$*g}CX4 zPi1cUSz29j1|K+cT-Q~(6J)pnSw;2Tlv*3F#k2eq2yrHBTLyYLhJR6sJz=*Pvhr!oNedV(5p0|k;pC(Mq(95XvuV-b=YR>{D@kkni3n3Fsc zscAXC!2I#EFJa}jkCV)-LER0u6G1YJR-KEEDnd|zF&_&78cFB5Zp-Yp6uEBRRjYiV z08I4pS1X7KwVHMm*x_txmEgQkaJAiMTs?8H77$o@-2X@9f*+L;KyZ{Usb{b9NF@@> zV5mmmZ9cf0=EU{Bg&Hv;ElG*Pw*|W#^N@)EjwVw0^c}n$Vm|>N@Y?U{-lhPPA6~^n z3gLBG67M!c`)GTg>;*SG)-RrbJNGNRl#BMTp_HaIY)7W@>moVP@Jy?p(wOz%1#R<_ zsJB$dnR$ewNy=v6~t9 zw@-LK2>Z}Sy{}(vFWz<|{I8``-FzZ-h5-iK z*0YUlG@bO(ff;}w|4>M%lb>{pH%NE2IE@cmO3GrcWZYs5WCmkgC{T^}`EXOXRBr@- zIh(n~Q^;HgUP0%FheWsRBom<{q2qWW2NOP141Sj1n8z_(B={iJe%(yG^0)C~r!9nFqNxLHeOKQd%J?2m_StqIHDbf9OHSxZ43wTbucT(>*zEz%T*Ndm(T%D|EAJyJumYbHrou3bvrb-i?@)|-WUpoNX zKLT;r9ZMHwT$`8JNPT|KzhpZ=dT^45Sf>~aFl6}L1)=2ZQATACNz4EWZh0l$(-m9A z5M>aJB&Xlw8K_S^))-7^oowQ^SsdmarrtzOFO$xrvdH`)L3N5zilzgpB6=kwri)N=?{(ge1 zI!dm_FY=F*TgWpu^NVJGP9*OSKSg;%yf?h^x7s${eGWd6Om<=8N2~YOJBMnGw!c!3 z-uk}hWK!P&4Lk;#a6ID%?eI{{&+fG?DLBeiK(KHj=(v#F(*$Lk*`(7`lH|M|4DJH7 z=EILbp&=4wM_Ol{St_xO?_{68tA3%EU3UHTO>1C*s5H)qbxtsk5tM+~+7+g|Dm8>v zpI}^Am#FbsTG8_=WGHQCQi*_AIJe@UG(uyh;ptV!bInASpFv&gheaTU_XSzs%9ot( z(6*%p7Q=^g#P?kn$+}|ueuPGL68iZmA}q5XQL0UT@&dCZDxrKOS&qDCCoya$fnXg! zZ*0-YgjFWt5zppiVE9lw1W1lcpH84X9yNV}I64cD(qzZE-ZE-*Ompgy5rTL%tN#9X zqkqi5_@Lg`cBR%S+jK6rxH$kwxy?F~&9BXD7*n!FBW3Y$u^$}f`+f#9cwS)JnflwxV_+@Em&MpS z>tuW1ENHp05si4eH+B@Ugy0R~D3`qx*zpFG4!;Gz?CZr8%!&|=C%qE-)q6Cenk+nO zvo?q4eeNu3;Ws#vQpnW-kBhST!a)lKX7_x(VXw<8&d3hC-9ePt4OF0;nr8}@ZJXBBXQ7#QYPduKbAZx+gqZ znAWfm!Zh(TMX1_C^y4~u)RSMSnWQ4-J?+S(?jpK*qG>#Mh*E~z0jvXv&ujZ?=U4%u z-ttEL0dv|`)$K*#{|6l50-RqW5wwHgouI%T@TKgNau3g!g$fALYE^f-#}^ZRXB%M{ zUnGng5USOoiCYoWO1h(0M{0&K5GqDWo$r3Ya^jq?{##`_>q#P)MFc~wHi*~PxV!<{guBVK;(*Pk61qGrJ3 z>>BQ8mJLXeCnO$j=9Ud{@{0HWF!okaZ8qH2Xem&j6nA%0oD>V*;uLp>;#S;?6?Z67 zplFM`ySuv<3lJo@LvT*s^Y3@>GyXI77~j=1o{L;1D|4;6=9)A78c{waeY~;6U%Sv9 z#x%K5`C5f<$W_)!sFx~hQxV$4OYcM?XFX(q-(GJZPKa4d4(d)rRY>^`9}hhu@7JpB}fy90>KEyB++cD zk)Pb@_WHzUq-F6tG6L}RxWIWnphV;&cfmJD1E`8BJwGNJg|Rx$o2>nIXG0d1Mn(_hJ95KwPRO*o^UkX9q7~_O1b&( zduNXS%r!1V8Yz;U|1TXAP6JUyAD+Q42?d@!?I=GM6fJG!pfp6c$jAV7PZCHUB2M=f zG5pM6nH^(!i-e(YOqbRz0m!5*1`q+9@-O*6rcK!Rn|L?_L|+UdCl3?oo2T8FCKVST z3nQilaxn~O@vm9BZD+l?^h7l()63`U#=ZUS)s_c-s;LTk3BmGE3Z(wp$61DKUt8 zG~(>{uhyyPM``!gG>@Dwox-`Zzn5HZJ%X+K#>(-}izv@|^aQD#U?)Z{o;8C5=n#*HC1PiLXNaQB?ik?e^gpKpjP6)t##l0 zg!{r7Y(A{Fjk0j8e(~@gbL7sO50vI=^0&xXn0n9sC&vJ@`2%^4fBv?&AVcNq(r%{r zLz~xWYEZH1W+m`%`}o37mL!c3g9HFeW|z(P$U@$3QW|YDRno1Q_r9cDQVAUxhyb!J z#)#3wsDcu?K+lv^gSYd)zi?l?dy`e8bbEYQpUpVovFvi4o$ynwv`81%Ku}NpzM2;9 z@X%auIaQa@ z^8uJ6{+@0qr>FZx#KK{j6c<=hc;st#Xd*jhpyxrsi9&{(c`@UMBr@84o_m8R;&9N=q__Nr7@YX{>5slYi(uP^S) zj_!5aXXk-~aDSs`tC8+O7e&mJu}M;R7zRT?+k1)*ZMoSQ;cVe)3$_za91?d}r_<-Gd^%7k(E>Pp4D& zvosHaTJbT{bce})#4R1-(z#AfYu@NJgAt>RI!1)A@7-TbGLKhq^po z?&BGImf*K#IecsP6~OnuuZYYxMXq%s=|zobyR-ZumQT;DE9~*v#}J@!dJnkZ(Z~2j z4R$bFHR%Aek2nZ6_YK>s70qK+{QK+i)hSdv=_h%K1$Q_0FXzi$4lnbD(cvorDq)|W zEtE9mDj?)akb`H+N&VX%LIeVN0%*5j(c4~i95oYEi5N0-Ip-EB2Ae)^NFN@X>LP{) zf|2(=ssBZ$me+N@U{WtzAZab5W5d^G53^NKJ!A;31_(D5^Jmje6Ay_KRH}QT2(tSV zh^4goY?5NSCgu~e3~8YBam$xTIV>!fjBvJPzOgUG^f_e->zJP;<0hr@}%gYjlErNi-!?MDl3kR{$gZuXK^(#fR zO%sxL(08Xvny2S`W)r=+!Z(cmU8Rlo)xsxzP){jsaBk*PV3iM6RPDTPmgHxi6N%;D zO|=Zqp0T2!-t4g-=21oNm84-_i<0gzj5&j$*1A_g`P=4ocz;@xEJ=fA{;ZfTw^J;$ zJoK$m5M!J}sF89GnDqnx&!teyo*ftH z1kX(@z?#jb3cXu#8$EtD7^()RnE->{?{`(0rg_xmB1Dw~V_?BWnR^)`PkQ|I%kUhO zh4SL0cf|J;UoEq(|C)n*<(ol&tZe&Un~0!Im6BgYxo~7s8@Lu<2GsmgT~I5z!^yov z=*=`OSm6NHvndh|7}uIDXW=SLYg#48FX@b4HSHc*4G(uGYyr??u?RCL*?(p6a*xiz zbu->Rn`a%Zo#ih#`RF!CvavpRG1j@{?6kQ-c=nG6ZWZlk3~wIzoFwnYMDBE~a)FgG z0Ly@0>yJnkr%C!NH_@NN9!@VS8UNjJ`G5S6Cg@Y7e4^0&4Fg;vgfsMEhrf@#kY*}{FRki>bQ)#4ip8c*laj{GvtxzmP`3s?-QAymd=WV1Z zrNY(;=1=z@+7BJD;^}!S#DT5QV?0YOTszz}wF}#n%C8Nd!%y;|uI20Qs%D=(37v|X zf6lQ$TCV%59Gu*2k(W123j`JFh+_ufn3oFw6jl|Tl4;riz@7_Kw9wI{mCmh>aZHvcR(H^^8n8PhbfMkj>YpV$nlvszn5o4Z8e^ zQNAB4QhuXoo#S{~XYwj~?fc1TgRg3!}#%f^IY3P{WO7g^$bGI7ZrVoc%QcuUZg=V?_8{hkJq=bLQqo*B^5Y&uQa zMmw;Wyb&hUY8u8Yp%7c{apbih4;%R{o4-UjB+ec{jsT(_79iKeNEi1bn^I9w|N|hgl1w>Tta3GVAJ>`OzE=5B|)`4 zQ(WE&P}mfv^rKdXv{bUX!-#LX5En$7Ma{&oGZzko2O6!@2#l%zbq>d2LMn-LJQh&U zqlW397rVD%h>|IV}_9yb)&s}stWwwc;v_|6bY|hf`voEeGM#8!m zxiHGJjp5cl;weq-iPy8Up?5mtCd)PYNXt?YK;eMP*zTc1btrnonG#V>%0`Hl?05UQ zBEh@6FNC)nL0C->CmetMP6(tH{o3cCMFVY`n3f}nvqQa!p}p2YCP!2RLjHDL74pL|zGAxt(ObNJEa8F58+kpl zec{@bs(P)#t! zk6>vT)x*0y=+$cbZ4?l-2N?$Gpp|&+ZKv zaGTUW>1YcJURUny)*u7PGxjfYaATGIE7;iyKD02p1b$6}N{XtBSEgVIGIfCBRJK26 z^9!$RW_rA&4i1`9Qp7bo*Kc|rc=;UYuwv9n=H{@kl`FaN)+**H?GJ(KxVgDh_bywYfoe6`3FJ1Ea-7!WQ#^cEbXFH6<@Hi@mCGyd&NlgtzG&m4P<#b z6qms#>;ATK;)h7GpPcbsKW!8Rc?P ziTNl*0A+KS`1$MQR6r&F5aYy{pwS(xpjIr5U)~t9;mI)fr zviro*wke?{R>bj#;nTI(Zxma$_y6u!THu@a&!=q3XSNe7LVu^Mg zMH3l*=y;Isp?WdhAzMdVZCQh$n>KTbr6W0<&aQ4yd@r;@g1m{v`8m#6gp=Cf{deY!=An#mKT&wD5OOb+)hB zT6f6Xx48ia(@z`xggk0L>fqjZSUDKb=`V?+5EPlv6o6YJvETG*GM%L6e~8?GxsY!J{)CYs?w!X5jy^o-sDe~%+?ze z_D`bO(RlJs+a%&T2knPN^~bdhF37_@{FAyGtvA20hQtTuD%3QZox}NI$l%*e)v$2s%qaz_ zy4tk85GyIusW03XOBjXgp1p=y5xt=R(}AOvXf(RMMef=(_qsuwQ})YV=_3OXi*M(_ z=Lh5w6egHHLw41IUZIiOLEn4zU#axb=FX!tTU%xMcU^YMKHeD#?FPk((B=no(MXCX z#)qa}{b3VR^!HH2=2(RBc_F8_>%E%`&XY=MIRdlfwdiquiGil^QqLYj-i$XDP*HG?%?2F<*bY!qu3myWM_x~<~@ z_cQJ9kp+JDPsLF44|Q4iO4WNxlzFJKsN4KthUpnStMJfAbV5{cKT1(6aEPv(tQu_1 z4vjXV)m7d<>^d#+KRaQxn$kpK>wn>Wi?kR&6SHjv%s6yhm~!_3b>16@fdJ$cXq!L_ zjEFaM2x8GXRa-ilC(sNOqU{N zrhG%61`%KP6@-#yfG}By64C-W)%e;L)B5wSt-O*U^!(e1&;0p9z{R*J=$A$O8bTD| zgjSCRBEWQOf*XjzuOpD(*G4e0f(9L@jM{5HO!C!n`k<>i%J3JhK+VSyM9#!;mDtxO z;>%lHrAkvQ%~N;~sWy;+EOQ(p2@M(Us}{wW!;?1fZX5N@{pK6%t3&J+yYLgf?OPQ^ zm6!T(WKg9;*Vx_e%Uop_6z)M{w#STWzzz0ZIOn5s6>1$4^WM4KMOn0^4Ij<%USv(p z_V0o~#On+o8S6gKj7iAlSy_^@-PD|l22bujjfY{Y9PtzB*3;?PvffKBMg5X1+F!%(?KXdgj?E{GyJ>80??_*K6mR!%{agBY>_n+3b`9(mcJi%RdjmeDuF-#j zUGkD7;Cg>gM=B{K$4mm zyl?N2siy|fp^9^lt^B>;fClh^uG%L80Z5vB50lBJJkihUHISz-uG*(LnLzCMChO)E zY1LDb=I$UbSgbM}ck*2-Mk>bB4*mO@!m9@9OwR&dQzcjtcAvl)zba}OEY^C^o4Zz! z6?VcU`0`_aUQMbLot`(`B^4_*STA|quicHZ*L2rOLcj5MOPkq~XMex6OT`4)81+?p zXt;-t>y3`9d+YE-ufNFtPiI6jq_Y^*EPo{np{Fy0te3}1kmsd$^l95o@k!%Jbd)({ z-+Vi0HZ+c%)Z{gX7vdXe@UpjnENfcdj!pm02d8TC^Rptzgx|&&NYUe9dmOxe|4+qE zO5tRq|NHX47PAkVBdxs6pzB=WYgo^Z&(!z812Ko4HRbR()$*?_QxU~gqT2U}YRtl< zSzpbpGa`h_d?fz!{&Sf2Q7x4)V{*Oqp;}HS_ubu%*^&`~_{YmLr?2Mao2P%0(oF?b zdc@!FyICLM9n_}l`;-a`uj*i{WVnJ_3<14W`d0LNIC zKQ{>st6IV?eAiw-vK`7NMb+;Nic2B|QQ`}UFh+R|T-Ze!yvQ4$UaWhUx@?#*^I6VaF!N7HW$2LTIxv~Ix!=8~K0 z;WEW{8P~BzRaO)RssbTT)l=8jlOm|IBei^&M+(9fN`B)o|7*nUWZ+FT)70X`=o9s8 zl&!AJ_O7}K7eL9iNn`vwpO?D4AF3jht3QYpvYXOJAqFS8h0yp;zs{}RSqb%A%nSGc zMH`NUP17K^Yf0sN_ef!)uNv}p(cw${Bw3E5i%w|LR#J3@9DS6hk&}-{pPs|16Nv2m z9?|pgIC)FZYqPWM``?{l52$lbZTN5NIJZlI!b|^={eCJZo)-neal~cyfHM9pgKDT~ zj*V#FsCQlk%`bnqt;lT&Gw6@hGpKC|ASJjkWoVuh*z1gF&$ml#5_Or=!ZZ3HpjTrz zQ;Qd_Bb3Y@h&KZcK6yqP?`_@dW|eCI?phv4|DiDucHo{uzh;y<3n$q;h&;G@*Qk4) zT%+G;tHxgV&Slm+uf(xC7d4Hwvdcc*Q0Pj7YNlOhrvF_E?OcGXt9-2?bt%uhjcnSS zYi(M?MW5le#;9Jb#9rh3CEgXzg8fcBr#btPKFngf$kbkWC)s#>6|^F(GT6yfG?w=G zNwvp$YWpSZm%cEAs~mlOI5Q(*19;R=5+?`U^5vR-3r5Bg|ASJD2p*P0`EG8OO_g-pPG_Mq->S zz!Q5#lh#wo;1iK?wc;aWKaJ=zc2QAENT3Xq<;7T!9r?=O|3%+F%D1@hA^&3W|BAkE zLn@JJmBUy?1yc-IU%6y$e>*2Z@R0M#F{S$$B_QKF!x&lEqXtVzpxG@tdzbc;I$hl= z@p}`=AEhUCP0;Ix_y9iUvlQhmEe1sY>%Z9HtNwXJ%e=B9KW2fI`EQ^CQ5Zc})N(5~ zhS>>ZUVVHW2vGb}9)MNk@AkT1cv<2dU{#LX!e>LeF-+x5bA%)b2F~(*HFwd9j}=P3 zC*N;L#_H3dYd4K+7f*6dxklf=QiNiImMgs+)qzH2hwfxiU>G@OZ!3>%A?xD$_VALZ z<`QMgk-5`sgWYhHWzwwLP3*45EL6{ z?41JI;qf=%DAVMVQglqT$XDhnzD!R3CUL!N>~Rbgkb5dL{ErnVyG8E&QJwUsT2j}2xNICce z($Vr@gfa1SbcI?DnnGQMJ_UJL7&ZfE2|54VHyN+zBVnKPcfRdr%?NdIWt3X6xe-uD z2pWPahY}`Q9hyu6gFO%3ugxLr`k#{)w#i*Iw;4)4yh@L2iLtP3=`_p|X1Pg@EnrYZ zvx(m`jn?G4WqaF@-HKeAMXv=ezlNuW(rYg1&jdit+uujy;LVstzmXM(7fZv|7d#8^ zNt~pLpo)+r`q9b?EFiDaY{$U-1q{@Fi>O%X7q@f3qgoPSLCg}%=+fZ<^983p1^U7Q z=NNMQ#AaNE25R)`Z=qW1O6=zxgfe)S7G%bpX%ABF?KDDP7ibdfboN6oxKRh;$L-}Q zSrq$bal6;vu2`|t!yte_Va8B}jb1>95)IY8AB!&Z{&olV1%mJ46;dpQdWSrP5#?{x zZA=ucTP1+rcW~LNEh(c?+jxw8T04{}`ONw^?V5jq76e4F#QHHxa29RvFHf!| z@tcd`mtA@Q#rZE|^Hl>;42K>=Qm&s&5a}PP$z5RtiUgK8c>O+K>itvPeWnUU{ky>Z z-tthTbAzDpPpE(8NT7mo_+4HJNOhG}@6tV1ld?KrwLXMd89`IYW4Oc$%YvpD%?U!> zT=j+gd*&bf$HBdg#A{wp3lHU_Tm7`>-PWThmbRKP*Vc1Sy3;E=%)fzuDBDnfdLRLi zpfVQv6}UG!TQ>O_V?rKSuRU&rX7BIY*#2po|0z`w7D*rWe_#F^ivMr)J*%ey4eysC znAoP0qA6YBC6JlVmUz?~zQss3H06c=>@@!**Gz>?GDYTzOmlBIy$?;`j3AEOaK5ko zKAn?nXCvLELIV+*SbjQseDa*s+vJBH+pF?=#D5AWd-mHW;IJp3^g~?zx149ysv?b{Xf7@UWGyNn!32nY}MRM=B%GT&4XAxCXXj8`mFMk`Hl; zgRn#&DGju?@&cG|UYBC?;HdQpW=jrGOncFZ+h%8h0dGqKY4T2*~HaY}gbT+_zLTLkpS)r-UmwCTPU#^N^9FkF} z)iZB0lDmZ?{Xkbhr$iwn`!latX`$Vm^__8ld)<Q%t05d&#E$!8nK=}A7Q(~EhrvSPBBHB>K^tf(nE|p zUY5t)baImupQ4*C(R-q}%;tAnmUx%E61m%%OgX-4QI>RlX^6VM?NVYFEbLqC$%uzu4FrJ1o>cY6?+Y*GxtuAxm zOmn{g|L*bediALn@OF(QxJ1m_J;K^+VZRL8Ce7SJ``+NhHQS^=w>^j*vRHkO@V5Ty zvs4+bu*Ozs+{o0pQ^Q4A{sFD0M00|J-RpTmiT`c@R;*;WGBoz;Z~DxOD z&W8A+4^N3)3l&dsbCzhq&ldd=hPNafKYQE+sdlN&t82(FTR0W&iBp$2T5qmc3T9WMIuu9+Y;^L2x;%S%Jtk< z|L7PSdL~7T{aLT_^=P`67v65ZLMzxB3nnnPC1cAdV{>=ZFTfp}4NIG2#lz}A!DCk6 z0xlYMLal8M29Bjl2Jt@ZL)tk7o~GAiB=F zdBrkVz<8C!gFRWu$4`o9A4FHR+$wj$6Z==xXT{4TQi)AW%s^5Q3|LTGNW{~u#lxHc zo4a4*Xt;#*z47VlgSmmLo93&MW|yz?6MQv>Ebs!&rXgbH;hx1eIL7K| zv&&A+su?6rAK!Vqj(rr?`F4UwiSkR{`FmA34P*G*0x5ck1los@h;h#blBS9j2ITH7 z?|J}K36p8k_ia(RY=cz!LxQ~IXu^_IKIz=JQ8}vJlPSuy6Cb}bDpUL2L)XR)LeUaq z$kl93`c$hhq~`WWH&AaQZ}^FUlH{A*y`dAPvIMPE$UcrNR+XW>hec8r(7y_ss64 znF-L5jz(z%y*QlN$NT_+U)0)^wC4;T!ibUkQstpOP`#I8Eod-+$~TJG_IGy##DMVA zN`vJ5X8UQ#_4ED{hy?d{X0rl@7*CJVP4OV!jkKesjr`#;e%xD27Efn}vKG6c6T9&w zg1N_j8QR(@$6r!)gjU5(CfUC0A1ZLSzUViMEVUWW*Xgs{o>0W_FHv!kJsexSqMD_s zSutpTA(dDvOZpq&@5Sm;DJa8?zdb4xgc`J0 z)=Xv2weP*(taPMcLmcX%%pX>A4*785p+uTxNI&=criMgm_z3!bvJbc9SB!G1H- zfJVCjkU`X(cEti^zoZfoy#OXE{nB^tdDLw)^tKG7F;#EK6^R}Po{TPXr8_sepGhpH z1|rQ);BoTVYKi&#sFS?rau5dtH9c*Uk>H z2qPq>x=2*t&GL|B_n}T5Aub#3SFfKG6H_<2KOqB=a=ztA>T>DUX56%;U$C|Y zj4o6@@wfXfFrFAz@VB+l>$7nxoSk{IVx5?ZP$xSJ+H1D7)qhWg)QKz4X_OTL>T-n_ zY9Gimo)eyCjXoHP^xBRMZ?Wt9FI`{es4?toPCg4Abts8qu4JT;0Z8i7_!#%*v|UG* z_L+!IpV(#{LguLQNH zv3{aCkjpWP*rLua?mf>Z&G1H66pR~#msaW@Nc->P(so!Qit5AF`R9o~u}%#-LFB7M z6trK(Nf&Vn{our9X2Hc^#$dz&cH&Hm2=lolh4s7~sAYT97VxQ_DOO8V$XNx=u`a%w zd=_bULchBkQ^7Tm>^i6`9Uwbs+trS~2$wW>~*lF#3JK z$wTRXrO>jlr7w-TAm_^eOH<=Mc=^#oBpJG6uUh_!U427(?kCjhU6^T+8H;9xF1)86 zbrT`{mi4-P_;=mEJ2y-eI==Xvu}JyMBOMlp z<`+WZieoZTJOXCwMg+BZrj*WO@ps+|Pq6!s`P>;S*DXe^uw>M^zEwdCOGNHG-$;4f zn_fS6e0j$uo!-NeXSttm`6-!geha+>YtpGuj-DA5cGpeZnXlz6bTP%h!o8HUM>2zrhrq6L-sR!Ft zj>1MGW^zJ*ZR4McxM{yZXinLxoL?BW@WI%Jd`PNHIhUkmWMjnXQx1mjilq8@Pk(Jv zI3LqdTSO-~*IDuFx~mUUv;XzKnUEqiA{V9&`5uKc-R3l0Q1(Vl5ePtI4#1ZhgBj%( z-{>ePf)Y6YV1-`9j+{3((OX$w*5k}3@C<8|>eF*y}vKWm*Ry||# zU|zII(QnaD$3}@Qtr_|7!2@@c(U>vXR_P2A@Vmg(z^o&d+km#fFE_!jHqlQGCz`dE z&wzeFhCrwuCNC9$9SUcEI-;tqenx2wMwQ0KVZQzcrKvQ@<;nk`PW_9*O27Z3&5@=j z%4~dZt3)AM?Z)S%7wquY%;gi%9Aaoslser$LvNuR!Ij9le2hZNe z%Y@G7gBVs?({J*doIOyH+X_+N8h&>k81|Ep;f2MIySuyQ}u{g=NBx zdZhcRvPg`UK^cjQM(UID4Q%qR-rhp(s38K6&wmjB;$DEX9^p(&Qjtizk8J=VULVX7w$QKjZSBMtbe(Ms<`s;pn z!cHc>1rX24>Bc*yoN9B6^Mt;-q})%kgS2g%felNN)9K8b__irtXY{gZ1rtGUMy?Bw2w1(s^p-*>U6(^px87VIt0u?v7M?vCs^wap?p#el}aL zPiT6+3?S`#9cSb|5@w&?lkzxiBMDP zkoBO>evFxXsu^vMWU1^mu6VRHA*?FnZALIQBA7YBQ=NRu{2VRpMs@=8>_gAizKp~$ zr1lj!y38uy$K|{ozub5EvI6x%=LZFi{h;GL!3Uw`G2Yj|joWWC)QtH`POt^q5_9-L zuBlgI8FnEHN;;uymkpa4cLN&Qz_%=I9^p-&f({wnU-j$!9jYCEJ%Z+#G+K;96>ZIy zP3JWe5PdNy;QZpRkMWfS->`Iiob<-OGwJ@Zkx{9ku5wt?qv{&l#~ z-IUwwWY7CYwzPpwdgB6}l3Ecr1H8bEwn6?zf+NWWXV>fkfg8@!t_bt#Df41y@NOKb zuMMZpP;SbHhnAy;BMYGjRoR}z><}(e3~t@rWL`_}l-U|KF`}P$C8ldq(QmO*|HcIG zYx%l=179%HGJ2|5hyW8}Wt~}7`)M`mbmJZ0V*=6>PFJ^hF$6{jE#L$1;B2JM@#W>* zw?AHjjeL^H#x96U^w*4Cf`@qUyeoPUPw)kJHn~(oEMueX_gh()bVZh(ld_%qARdoG zjCzS(Gyz|JlE3n@vLvrv4yV%+w}{%V7dY;2_+A3}3?Jlf3cL(XY5P(S0Xa)t?t#_{ zE5Zoq)RwW{} zYEV`T7NzrX4A3jPY%94eKwsQlu*5y;4Ltt-@3)_$_5!c6GeSE(j_WB{7vSkR=PDkqwKX`TyvAyZOnsFp<9yTd*&Cc z^ynRWwXjDDPH`tzOsaUFkjC(@3rL2BcynjV5PjO_E}s{-wEOSZ+o@Z#*)<+_2pY8? zRngVyxf1MKc1HhhZJ7?AnsDODzroS5!sCv3F05{8>Fd0G<$sRUgT_cOVeZAxXu2e& zc^K=qh%?_%d`Jaz96lU4>u^ZKUBnn2;3^^C8li(HL!Qyzj+p+WW7u-eFKZ2*^@4yu zZEPBESnd8&x2GE=`j|7 zGImM>be;Dpr)KV<7%jh7=HuRMMkPy;Nuj z6`e1!n$KIS?5vuv#dv5+zCb`i#IyZFCTpAyQmm$X=CDI^7yFQ3xzgq;@Bb!WJt z*^N%FNQ-qNcrvMbBNb1r6RwFTp`W~>aS7Ll=V$1$vI8k6oBdCdj9yNRE{`3ag1tRN z@+y+)832-nGzPqd@`6rv=j@mLah^_h(isNNm{B{8`!lQ#W}C^N#hX%)#C2Vqd;wKU%{$gv^k+v&TP+t-Z;7asif-vkK_Tl{X1D$tMQ*PcBc zCO)g4#hy$c9H+y=J+OdIGV(GUfb=zyy$cJ1tI_XY<75qIpPVJ#zQK)vocYCuU~*}6 zQ3lEBmx3;qE+KhSOzK&FDBoE}W-1Q_PaZ8gF%Q@A7}HNfow7>p!d+8c-(|U#N?A%R zf2FHo5jvHbd~F!BB;ah>Vc{@O>nkSsoga(RXR+!9(k}Gyjxek;UO%KZb$x57z-O^B zT{(|0S0lJVWc$%fo=X7Yc;ym1`5ZfZK$1r||9FbsI$dsr+?zp;yOp_PjK||(^gGz*IYokGvVz&tK!$F%swm97R%M8NYgxs6utlvUuetpGoY@3) z&!dSqjBBT!#ty6ya-$%=PS?TyEhj~Ur^KPhp3FAMjtIkIrX<(QI6`K;uDn_2{}*@t z|Ndvd`xYQ(fc9Xzc}JeW#GJKoI(si{g>r|#Ck`T+K|%pC@#|lCV$hRx#1cnNZ(@@Z zU@GMye^f!jS}<;SL)%Wc1fT}|?zRe8N>QHeyXM|B%EUKom;dQcs@Rkq&>~Cv1hV#Y zyqmk{UgSs%`gQyXAa#ZwX!K*8J1p^0G(tlT6_e)sEo=D5J4+8>d)%^N@^ar_GC{`r zRpuDKlA0mZRU}op?4?$<39WfJjy5t1uG4ajjC=bgUw|1uTGpLzoe#NqRt?UU0>_)H z?SGfpyO&eH2XiQAR@tv_>@U4gNO-y~x-|VW*6#WZn8)*qX^;R0EqzW~{Xa!#;roYS zQFTf!zRA!ETolRVUtF8xlnpd4^Db9E8YRoB;(^D85 z2ObD|2k>7@U~mJKABOmpvd(g)MWNS0MB&7T+Lpivi^8T)-*_LFA`ddTdbZG4OZwX; z*L_#PZ+fe3VP=+GVVNBBKk`zEW`-$`$vh%pm{l8=6h_mbISNq>%9*`dhMFmOcDev%l?sylccqc-v zx=5zungu))8)*d`Q+I}B_PPjEtNw1UMU%AcJz2=v%*qytZ|h&)qBxT^8{O? zgMSYa>IL(3JXlwf8|3Rs4^_;y=?@(EP-+ zFjb;RVq^+0EXTC!p|L?Iq_`C!b<@J7dr3m=z@eAbgh+V5FcrQ*53~&42rr9p-6mtG zBFq)PuYDuQ0>G@S9&gT8E24PYtkQWja9*kSmh`i5_@vE(zdN$*y(`$xY4yijfmKt~ z5bkhUSy^uy2fe7ZW$=LV>?V2rC)8u&d6$x^k^L+(FYTHuR)i(s8iiW0bn2RP{QxT@ z!->-=n%U#BI{QyG^AF%qi;^#x(SMy87`4WE8(Q5w?ch=2-7j20v@vkn=xd*q=08lc z7fOd-zO_$To0~Sh6zEk;%h2!bAs#xKQ|`girQKM0IlcdaSo>Xf!|{d)X+1$*HVtFz zh=BCLo5<&-^@RkA3w|P`^0^IfATzBH=ef2^#ETq9>-O&PloIAQtMMQ6JPn>c!lsgP z&HP~0?TqvZiD%0O{J{FL5UGUUn3r)cC*DNNls?}ttB_SB z{c|Vn>cYihzG<`j(JpkC=SkX7gfA-BOGq#&!nRnv_pkY#*B?{TQWc@k9Mm7^KU)Tn zf6@2GEF`E^A^ezl99v|^W>y&8TU(!^7Q*U9<_J`3gVk*g>&U71@()88s`z+MPMInx z=wlg~3WWeP`E5F%6yqB*o-3NE)P-rQ8XF<-nX}{a*y9cidrsq{Y?srJTqayj%08BG zVt4QfP|)VZYSMmWFNFH{-NECPz8dzbhBWcelr1rXyLh1T zk&6*HGTA9y!M)ppA5x;)AAEW_Pd`zYEvj1*HQjyi2W)=|u#35hVZ#Wj{wk)Zb~2ks zkgXNxH}Sdi%5I%v#o11*B5cJjmW7r=W{j!-P5BNRz3ao?nn~4z5#oTswr7V$%Spq7 z%&NB7co=KB2L=iUxLdzV|4TjWJHCznS3YmKHk*RickB2iMU~o0Cug+s`v<ML%P+aP?PZ76H@CvIvaN{tD|wqFYW{a*{9Gk@a9TmApiOa1SUP1`V+o;vqv z7H#_e){lBr)g>p>K;tCc40`p~izbS-=MidQwiK=rW-RhuQR4EB1>0B9iq4^rxSvO6 zup)B@R06o>2va4;$OovMdWaIT6>8;eBVv+nWD{0Wx?{Nn>0e_RS`en!^jq)_{b~Vb zW|Gn8XS4`rq`Rk0z!o z!Hf?skcbW~L9u-LOQB1n6sZ6?mUC)uoXQ3`i&wVo;4aKzVA(uAXF2Bc z_Kgf@n6U+CXH!j`4oCCJcvg5tpQ+wuvm}_D)F$0`V8(Cd@~q$8v_9^N-f^^@-#^F$ z{sca}u}pl2d#IKl3Kw zXZ*lqtaDiX=xx6&g3+vWvUY^)DA&wa zS#2obu5-(qow%klE;~otj=!c=-nnRTXKbmD=vDgzvo`n|a=9G65FLE2_vt8haP#HE zr=L%5gXcNMa}4tIQImWtJze^*@WfgnXLlSxB4w@WG_a&E;pk3-h<7m)H~JK&iSX{@ z_6oQ0z`l#L+PI3rZ$_Pom&l~GHDiGDT9syqn>t}8=l*XB=K`t~_Vigw~T}tNcVsEs3$PD6&_@ z*h;Y@uV)KLKRV?}&J&2D5;o#4| zXGv5Fj^b>=e(=cGQ=cMu2m_F*yOUKEJ<>()RoV&VQLL;0WP3vtw}%!W3J9}K zp2abD@bC8Myu@o$iUzQCKJGOB;wXRz#Fs})@ytF6j>VpEt%?4jRKPxs$iy|JcjqtX zFM*Q|`s}qEtG{xi#B<7UOxg1hc0)8Q^~(t`bUJKl`%vwI-4I84SoQJ`o$t$BW(vhd z>oxu6+d3-vm+Rv=k0`4yuK_jf;7~xSxI{7iW`3?ao2jk#-&6uTU8(n&^{iS-HiO+w zCioXV%48O^?)XL5N}OKSBI1UY?Jjk|^IOT$|A(=+3~IaG!o8nTTA*m4xO;Jzpv4`E zySqCCr$BLs5+p!zEfjYsUMv)McbDLLvghpm<()b6J~RJhGWnJdchi%N44SAS} zCcD%vU43;f7CBj7$_Fqr-*#M9RzSH}>i7w9_fMjp1=?D>%kFMp(i_Qn^z4YIXZCA8 zHpFhoTee7I-lqKkr@HL0?F9X%@H48Yc#F)G9CYm2penwpQ^j9_Rw|i)F4|3lW;SR3 zE*;M+aiQ$i-jU5}lL);6hY}``aE}fERw0EOW#nDE(Tvd6_M~8PcOJjKgKk39S3a8u z_LyyBSLF*w=MOxQ9~?PbEoi7qY`jSuJAW97nt$B$xF+bM6g(>zK=!`1H-ba(yV-zS z3`VO0AvUvNgu2X~ThefQXi~rVO+0yxw~J!s4bV_>wgOiI^SaG0c>C(k6aOH2f^M++ z10sFtp_Mbn86uDQD!~yx%ihSJA*1zfxK4GrKnz^Q_c#2XQq^hn@9w!6Vg%apx+fRO z?CSZ%HI)gvS~AB)^dPw7LJ; z)NUqr5tCOvYxv)DKh}e7`P7cI$0=0HPHgI)*Q_$tNyc1|R~2%@vQGF;I%=4;KFGxp zd{4bXEnEL5S^~mRz8hPv4$-$DR7RLnFN{c?TV5E#h&u4hhkoG-;flBNd0JL%e>z5P z{@{R6_BQjxH5RO%+hpt*ltetZF@+3TiZP-UUTg1EMg1tU7he8DD@&yB)!G>h9uK7(RY5uFdoG$lst!Q9%d^wC)Iidx!8s@ zt@@7gtv)2K-F2^B#5DyDdDP}B{LcNR^(RDarHMn6TCn!7a<(`Rrx4d_1CU?C`xG_XKDb0Kb zxp~r^?`T=ML(z3mj#jp7MT+piOpr?fV0*GpIiE9VO~&=F#td_8?4dsOlSBdh+5G6= za>xAbtqTJe1bhDk0kFK4q{)<98pnE>gtniM-%-iS#7rVcD0>^{kL}TtME1OtZ`rJV z@jhs*Gm`%8|1Dj$x{BZb#+$}Cm&EX zgfW6R-g-mO8f~aW+SJm}N=v-_6rr+y@dZRv@7KCfAUJsr3ZIxyTtqgV#)D35`X-KB zjglYYRKiO#ff_hfz`fsRu%B)5zHH}road)?ef~2OMMgnugoswsVC*tgGd0FDWGEnH z39W6>@3W2BX20g(j9y0d_co-jf_^y#QRW6<#Qp!4)7OH3$2JN|ntUYjGasCn!ON6& z>fihKjg89ogHYSnDP~y;rA1lg-Fh_A1RGW=fXryt~Ye}jN|LlcPFvwpK z;VM2XP0_|v(f+YC7J#S}4*Mla3|oAbm{r=ms6%a?FUee^5)GxC+*}DszwvRsV3V~? zD12NNetG4n%6$v8TyQBBppw^Y@Q0n`)^>SAv)k;+a!Kpz$tgC_#bqNaA85{&-z!sq zD5bZ|zL&ONf35Fn<_uBs$o}gouK2#JmpKHUG*dOL+TTa@sS5wAxyB;vL_P40udIwC zHNWv!_w?i0rAEq@tlV&%qkjFw6%juD>@Ilxgopt z@Z<>>5$P|!a+Fo)T}xJ29Lgei{lWjJ=q6A-6b|Wl=T`anFT(#g1l*Q^=R!iR%fc>- z<(1`L9Q55Ie?RXAe;7^S#`YNny^3CBpS{pdJ$G&Buid?qK|)QGoZ9hl?0)ck_j{^e z(10Z{3m^OnnTjE=(+LVq9YAJTt^PcD&D+Hxkb+`(IW2NImO}ru-rcsGZDfYb@eSOH zvfppjZTJ&D0iyQ@#aFgHk==7o!m4P5>ib9Ha^!quP+&Qr+u)mP!NLyL)9Gut{Q|{VP>x3&=F8kNW!d-k;algr+lx2#!l{1X5Q`( zotVnS?u~@`bxEU{`x8wcsYIB7l=x#J9Upfq{MGk_xx(vDOojiP@LZ{=>39z?5*#r) zzXxQZihF-k{0w17&xwfOab6>a7gP?JI!Mpc(Ds!Be1;UCB0Sz3fny}dp?pJfCTtA5 z^BAjD$_m1a+zyi29wRncq0m6e4mwkxPfyt{YxG2;!N(^&vX!)aB6jJ}Kvo0gz3T6nbsAJnzHh^jf5xPXPXxv|8QnUFMI7%R{7iG! ziHZyMrmqSAM1Oqa2_mFa{(>#7ymI9fyGVIXnk-DE`?=7RH>9JcO=|lYK^EN!q4B*v zZ_(=PdJ(Fx&k3$pz5nv{B}dmyO4RPov+)Cmo&u!7J`4Y= z&5-nu3B6t7fnKcihVw4;twa zSMfN$qWS6i2Kq=M=k%r0ip?r!Q<;W%i`TZh_A0oYH4LJgRa~)l0wC*sH8$91qK2?j z8Kz7vVETKqYz-lP43ZPE9keM)`LvWq!W^ESdMGjNyiDY)LirFmY<+tiw4cjCmdlk@ z#qs0lbILjdATBC?4bK?mgYy-_%4|WoBmLD=UV|nf%{d(_3b_GA4C~0>7p^M&Ea^^4LCk)h#L{$}O}La? zU_QA)qq0^SfE&*7OMfts472L;XZ8DTN%lQ}`jeR)zwvr&dbPKcHS*#OS??2B{%^j7 zhHQ|_0Ur*a>cpc)|N5x6!8qkHmq0c`UWS|Yrq+E4^=}y))TxG7RLaFpG4szxt@)(I z#=Kji=(S3NY|Sz141D<|2#6c*Bf_UM)yGNZ+V4o1l9q4xw|}d$@wk zuV5;opBHTT$jkT~u`LuW6Mu zk9f4M3C4LxM7*EhS4>Ps5q^ExRlfN|q5WL+mF?<#M@1S_`DV8-KnFp$;!p)0U zxUei!I3YhPp47Xgx~oa5G%bh#jA#iPZUQXp1cm0*me+*c8)d#5vK(f3zLmW?$vS2p zze|IDtQvh)k^Fl1uIkmpB5$E~e`vjjkYPtd<)ojH!zJLM4hM{*smXyx-;ut^*U;>r zQ-19>1*UZJ3vtef%~6dQF+#&1rIHhhPdqXS!J(B^s&K}-UZN2epE;kb;d6^+ib!Eo zPj{oBc7s@r_B>PpK^3ooT+hBTur`K|PE~7~Z=6#Skq+*8FM!6@o2q3|ywr9xOo13| zi+-KbJEYzTCEFL-pa(NC@W%~aQ~$)=xkyQ~#VxZJ{k~?ESfpjqd-D0L7j#N)S+(R3 z3_uM^#2u4YhPs6=)1aGJ*Xvl!oaZJ8Tc-;Ct!pP`Gp@24Tkj6{l%=^W3rN*07HF#< zW)q+Kk(rXJ%YUVPu!sfHg2{Dn@kO|3$wtb53q(>*>SU++Sx!SNriR_r6hC z#8FMi=7MNM(y#R0e2hXXDj)ZEp-27<}(KN)^r=J zRC2UM?KGvDmTX$hCTL0-&;V>X^iI~?X5nEwhXPs1A`Fie@g159-!GSjL@H8PMc{;2 z=0ZPCd_`_@IS1@nCLQy~3a{Kl%IVDGIuvCS@zKtBj!iW8$zM|z!7zl5FpkKP8;{hW zlG<;CDSid__Kb$CQ)H{5AUbpipI{EzD!$78z%2hmj_pZ$&NMuD0`8(Chw%D?X5p%k zPQy|-;cADKH2(M0+NWhnOA;Z^os3w7)qADnhRDAj=U1FHHmUcE%}r;(OrRC@ZCTI! z|2*!-Z2wL=8fnJ98O&jnOqLo=(2XZ|(_FKz_KBb%rP8$`lN5EQSjBP=E2iR>u3*ym zXc}^4X<6raCLeJeK5-Mogpq*tMJss2GUP`(Z7MCDp|g1a8F5Kdv!dTuUPI9r+GNZ6 ztORgpr^VL^W)c#Zc-_K#F1s>1~4L}Pk5rt7=`rhaK>Ck zwP0yRFJqxBF+U{g^t1|s9WHCnuI9_`_T zR7ryj-tXQP&?D-3KtK&^+S^VcbtPLcP zE``dHK6WeNj!?O8bEi|wG+nPASUhtWz*CCzBKkr9Z=edZLnwyMa=cPLCg7PekwbLtL=GbN1jM3tW>ZDw_3O5B}|er5>KqacX>)Bs}8 zhw96Hadr~s7G@3Z^l|B*<>n#I-R8nBnRqR5YAv_h9gRof^O ze<-z~HAA!Qw54A5wF^PS*8aR??2KHS^7OzE2kG?J4L|ujMQ?C^4;bPp07r*0tJt!6 zSoSxnxTlE)q^uUe;odER|!Ky{JOS}KBN?9dyzaw9A)EaH00 zWR%IQ-zeRW(Sn11E$sv?Qkh_rn0(IFcs9jvnkkDA7TB?+rE7SRyGA8GmN1Jsgy*e zKfn#SQI0A5v9{3aG>AbQ^5RI-_uJA6S*NhO4|^-|pPQHt(ODpS;%YnotF=xz;E~@e zp_#q9CI*&3y_vrj9ccG*AuyKKJu4Q_waof}XDETF!K(8g6J0N_Dsh9)?Q7G6%xzW5 zZW+0`P3^B9QcH{pr;4Dh)%30B==In_;0~~)@2xSBXo*Cr3%D?p7%M{ z+F%&obR{P8)2cz>6<)k^4AtH)9kl?Lcv1abrw6J@m{NF63R<$+D$d#Ql(3L^36ELk zH=Fkd{3SjyXj|TiaYbiNOB}||K51_crgENVu+{p+=aV2i55i7Me5N4iZnFvW>vkN$ zu@z7R&>Kl}^u;aOFJ?k7u;LJU;vmyNwwecxRmBHFstFnZOZ8(V0WWcr&A}#NR^8nP zZ2O0V9E$tWT+{l>JApGhG z^5cASnM+K}2#+}9%|!0jZ0Gx>yB$;Em@0MuIL;OSKwBF7TP$fnKby^2B|N1%w#v_b zI3jeMXZBv(z9?#7Mz5vq!}0>$N=8!f<@kU8)*!ThC)485vl9zq(8tGE%9KRSssPD5 zpj6A2U4Dx(yg#veJPby1!^A%eNW_EK(zu3Ne+MOvE{{O`4N+X zGvJrY{_Br@GpB-^LW@EzA)Pt+<>z88O9~{|ut;=dKao~*_ zqhVI<$LAJ9Cd!P#wm;1xmIeV~-U@EYwUlb=8?Kx^s!29t2y$mBS4%Vj$aMU2B#J_H zNv{O9x2Uyp@YsoZ06~*aS%@ypLA3H-y{%GxWU{Qy8d3m)M^LB4ZyyLS1k}ErD<%!% z8@T6)FAwUAPoxtZ=XtCBHS3MJhR`Z)`_)j_FQv;y$A>xsxR<7TD0aH!v6DY*YC!Uw zJoo9?^35kl@1BExQThRD_n{`JTvf)uL<7N? z#!q|@qvuFZKX5voh6w84t4)vkVEo>9$k?iJI`ZjK8tC+)rE;e@XnV2i!jVX4?n|=Y z+~3T_JXW7>IVLb-rMNY~sk2%GZ^-c+$=SOb`W%7J=i*eT{WrQP<6rf^?I#G2J3g9z zD!Go@EVr=(>;Zndbe>NA6owL(M~z60)Yb7Y5GI@|`+&UrHkkjBSCp5}q%{~wk9!ah zM!LoEzr(ycKPxNm%D-g8ngh$bKgMu{@WBniR?J#!yJi>q@S3R3iYIC|(;i+N+{`0V znV6u^b~bfb*UQ9zhO9|(Pq;ZiqO$7`+ZQ&<4fvcG&pJb_^t9-BS%vXuC-b};__9Ef zEPn1!dr`wMGIn7jf4iPuFWRl>!qHT5g3)QJcFK>o-Z$tAp<9o5bgYv*`d>*JpS<*Z_N z@GMW{Ljb!fcwySg3COmzq1WbxQ(hO8V6cXnw zP>?5vdp8gx4o_a=u>tu#JHs}ueLIY<&TAL@{9l4Hvhe17f5c2RqiT+pif#OLSTJ{9MzB zMG^;zBHw|NmK+KLpOX4WsHqT*W>ZAuaEUibaOPf|H>^ZJGe;pFjZ=box|fH5H6fdB5v_dI zHz3zEHbcJ%imxouI|F6^n@v~fZ5L8Iw??O_1xrKL<}$$oawlI(&f8|f&63DgkrUts zspLCcm#ZyS|>&6^RIs17%p(l`~0dO(inwhL@T;yzRChPkhLA|-)ypNYpYz6 zSr)w4COLh!4(a;sCo%oSy>R=BW{7sYRZws9LuAJR+x>Ze(Ad;e(18E7k{Qv~^p-(P zvutgc9tD4dH)-&;Z0JPg<=#Z1tmz}~rgSMZS2Z#n( z8kDp<{#BKkeKU+M*43M!aV|nRFB?PAu3v$jD|g3*;?BuGPI$Y|TM+}4f>$n9b{1o>ROZ7mkCa|{4E&?H{?FRK3Rn>Iau z4R2J>BSE!b`1-dSXlvW-o>^N!fO~_#)O{hiBxMt958p=EH_f*FJaYDZ_87Xg3zYBB{Pdue99KHmp(N1z6S^v_{`Utme}BaL!GGF5TY~-nf>AZf&TJ!Kaa{Vq=db=w~9vuqU{!;@WX8 zp=4dSTd=sn8vAlFkY%S;g?ao)E>LD{4qXLGJyN{s!pri%aHSqmgnxHsR66;#l>v7- zHap;`3RgUcfK!}*={$uAKVv0D0(cQpTA0~e_s>xyDnDlPM81^+8vfN)rtqMm!M0*C zW1tsRh?_7?n=j@>wF8(cCo7Cn%}oYA@*DR0sy5FtY1aK>>ppx1_NrR!oDcZMM+p5ic^Ksp~BduJ77~ zeP0;;>N9W+U*C31IHw&+AU>bkHbu)AJq7q~|D{8MxJzY{uU6WQ^wFF@9dvk(8)J+8 zd0<`d?>mVP)w(h%Rn5`Xm754hOZu9&Tgk=xll?q$g51|;8NvMBaum(cjIpIX>B9*q z%Cv;&74r+MbK`4zjK}4S%Cm`Gp>0cK$aC|`tLbL4a{>%Bj5Kd*4%|)isj6oD==XD zt;eJBXdQrD%efnKzf%Q^QCRK^FOuO8VgAKA{P)&7)$9T`7Br7wZoQ$%ja|7 zA0{59CR6eMHW!fQ$$xmW`9e+kq@poHj??oPslQ84Xi`PK)xrj#EH=lh2zLcriN zT1D@Z<(-aSsQv^4!ofs~3CoP@VnspjBYes?suKcOXXJ^e-ma;6a=Nhlq zQd?AlR&Vy1j&~exw`N&h?7kT{MH23M<5c>+@#PO9_%lnd*p@BxsW+bB{$czig^?~w zvXHB&f%0IX(RO^`I?7w(0mDgBGw$yYE%RJtF|$vQ>Vgg;>-~e}o7*j$JtdQ(`70iVsjk zJby?26rQTk_+W~Au7FfVeINFPa&Ebr`5z`0vR)3lKWM=g;1)Cu-O~Da8*M~90>f9s zYI*rI(y@wd4^X{RC_*!|*wN2Hs6;`piKfzRyGlBCu)Uu(yv!@XC}0bWsT z3ZM?qW5u9n9T^YoE6wCAyDjS_JABK)5{Z`qAiS4=OgjijF}nCssQ*)$AbiF>;@T+Z zxRONG?-Ia}virm2O?mky^n%RjH`2z%6lz*USjPIZqhpqJqHM%1O7cX%AjIee0Ka<9 zJg}VRJgJaCUr@WPeS{fo*o+0O5!YP!pJ}%alSAQ^*5+(bNchznG9L7KDf=JwcH^xT zfu=cWB|ON!a;uw2+IN$tsm_4K>$vUhvFfHj(0efRqce6s%;GcdbgU(gt-09(KOWQg z&}f~gXKV#YX9Y?!2KDU&OsdiTw^P${y>oR%H5=+usG>0c&sx6_0&M+A)cTGeut(YE z-WP{~QVhi0zo}n5O>j(N?Whj8*4dC};AY#G^e@jmtHLE~q5tqCyfKPh=XTEgfrEE+ z=H`0s2e-_KfMeE77M5iiqzk)Sc-&Y>pC9yU7S8k7x?pfzv#q>GFk375J?=nI z2;%?wWUk&;o+#{GDRd_u|7}nwl~8AoaV4y-GrCZRxK zGhy0R4y)*?<)AF=6M^mCh%D+`?Ln`)=#7-U4jjr14(B+S)QDY+q-e?=8kKxv+OYQz z7Dn5Y)+ZkqDn*%`rZeAD(&_d9@6qv`O3-Vi6i)0>rt1meyRQ4Dh^x6MF7Bkc*+$D1 z>0Kq6DcUFP#7U$70=RdN1^>$ZeG?nv!hKDmh<)Tkh230v<}MJJ)pnG-vBi*|ITh;N z>w@b~Ou#Z&+4@=E^AjMKhp9GshLT?hV=imzzM7~&`heg{hT5__BXZ&flaOp+mTDCc z-Y9-fL329bDE)&tpWSa`)JUG~nHddb9G=z-w{D7Yh|<|ASAQI`ozp!eqFMa8XlbPr zv{E(g(BUytALW_y*CMyA(h7r1<($+n1v4Xl?VZ*>9U1q?HdP*2inz622B%BHdQHqR zzXEl+Ts?GPi2?jd8z41y{(Y&WE0#(nk#9(9tf%bQoO%dBw; z5|88MX?7YaH~j0tbTdGjGC_G4t*MuccHyWm8q?jq&MLd)Z|(9dF91$JI0x~!cq-7#*124qTt`eI^UqwfzL%@%tr2jt#X z^~){Gr#$;FUWZH7rnk8U8>^blx9m5%_f=)nTa+zOXSHw}(=GDW7zoA_k6+}N^J0g5 zKG$W`_M1@;-wR4MZOzV27UUMJY#f^GPzAkljdaJtS*)u0Cr_Y|$frFt6>SEu zeU7!TVk^PPI{t|2a%SO(#HkKLW#%&tENt(AaUb_H!<%xT9I4#Sj{c+nD2rZC3R4g4 z9ZkP@ZnS@0S(i<4C1YuR9pKHP!KXmfAfc(nYyKP*>XAUb7RInWlHj9Pm8j>;L zIb0scoE2|5HQ}{1rWt}00`B6wZ~q?#zoYiA5>)&o;qk!ynF7k-BxON7IPrYZl>QhI z+ag*2)^2pCPydgo{}Ca}!1P!u_vvp!n|~@E+_9B~?t{ECzF=RG)00*9Fe^Z*L=>9w zR1Lp+-BT6=T{TIloJyA6l&Em%v-T{untSUP^!)fbB|P!&?Hbb?Sgv&< zCoR679DW&wrc)7Z(!Y@ERVEb~P(8ZeVPF=6*rvN--BSus);XG!oxP>pY0>K5HsEk1 zTz!Sry~fWxI96eF4XzXfjdfqKp_nevmob)$PYSVigdW_XRL{Lq%S`27KCO@Y_}(D3 zH$ORQ9r4L(h5Cfi(Idw=4Razc=&^lrZWrLiJsX*`pUk3b&beW(%@T2#J7g53s>fO4 zuUCz$lQLUigf(n|#$CAq#SI$}O1o!*({IB{^4CzA_di|z9Aq)bsZ{HlS>fyCsh|&W z*j`XezV#**5xlv*VEfk&`dj}vJMnA7U$5WSU%RHS;ccoQzwVCv*w>wkY<0;?0fOtW z`%5H<;S;4bY;Di(f~B=NA!D-_(q7g}@x01u=1{Vd-Rsw-1iBH4d9X@nP-U7XLArkD zJi5Adx()26HT&?qd^opADYt?;N&8gy?xg_eglo(q9HfdSiOwj~DoDM$BqNixg7`~6 z#hq{9BCbnxLzT6fu*w!C1~+hKw`SXE;aqI{!92XHf!v;N#T@4WJnwZmU*gRCJziFZ zTS7_pvvs1Ake%O%X{BBldX#k`*_qwq0_D*&jxb#1t9aWk z*UFe2RdYDjYF&{bSBvV*$ z65Ip=Mlht;fMwbNix-RwSSb;G(FtlG^A2V-95mJCbTv0Rhigx7aq^~$jPkOb$=H9sOf z84f%+o3sb9f&6a~dEW7ctRiD`-CoAa@MI1W$sc5zW49@~{Q zCPc0TK575c8t}*MniPJeB#y^?;VVw*g6ACeN6LNiBnTfoui9{V;G!!iAvt@dWn&G> zvfk?&S(w4p<#nFyf7>V3t`FCS5*rb&zMMEp7-WfuUimA5R+HfK$eI;*zByS@m4}-K zk~@VZ%OU3(fnA)z3p9*D_Nv(OwcyW4s=7{*<4r?*bECS=x@XD+WG7RXH-{dbH?x5$ zKej2?XwyX#YxNSo{nM6$MJIIRM7y3Lq_F+{;md!flEUfRtO|3Ln;XZDd_if}Xj!U} z798FU17l#>x$Cpm&g|+aD4&yAqdB>|q1|dWcW#+OuRmX^>}pTm8gJp+Uor3T)QyWq zpLWx%`n@C{Aq-L_9r9&Uw~N@nKLJxlv#*=!{C)4row~d41uILu56T%h6h900VGlQs zWrq&89ogRYj=;En`Q4oQKH6sb)g|>>8SUOUs=v^TQ0^7VHbGJ{XG++>!R}GRd>XC8 zWhMoUkN9Jk2I#(az@&0veTjCe0j~g z?h5z6DGD#D{|bVA{dA6jQ5;{8u~i;L^VQO`XX?NksQZ#S@qkV9CXi_m6gmJ%`+CT^ zu43mfxkICD?+0KqDHKVXxn4-gfg}*A3SC8GAlunvqf%*4Pf~K`kB^;@x0>Txhq9Lh zIiYoI)Q8!=okd`Xfg-muJqKW%{b5E;TNN~+e{Ow+Dt`Ii$*X&e?Yi)7ikW_h_pSxM z#{CgfF5R%CayE$-BV^K=jUKriSJMVwA4e>nL^g@o=d5q6GSpcpCpQWQj#;JyC8wQn zW;P6ZJ!nYAgf#Co2gN=BuYDxt<6%=w7}7Y07wN^eZz68Wpw!mO=<$M``Dq`fGA(fB zDdZ`z35cgXMqA8i!5)=SM7o{|(kmgYnE?MdJu}Csn(0`ll$-?s8hR>a-U6>t!AbO) z;G61R9=SN)-7qK|%aedl5T1N)%;cu%T+(oQobB;xMrv#O+Tzm(xKHeby7;YD+K0@Z zS>R_;IKofV)F40>x01XmF+UM8gzM0yd+6m|-Gv{?SYS`g2)Uc@y|HV}-J6jc0E1GL zZL~M35mBFKGbzf0`pssEz-#8m09f-lOoF8(gLN*dkxPrEvKY>!&l3|i@cFY*lu&-Y z`M&(JDIDWWKu!TL9yMAXL^4c`Lp;Fq@2E@`8XUhwXwA#=`oZ^6M zP^@WPL-+mUE`j%g%vnwQo5*`a?Zyt@thbBm=T9=zETx-L&X2`&y}H~_T>?21C`Wrj zWTtLgxO}4%Beia+&Y|v*w`cI3c0mGz*{n$xI(N~$Qv3Kaq%$7gw^4&!%%H*}r8_o` z{8@^a6^;#j`(?N6+zls(_!|3}Q=)=K*b106L0XZDE;FmInNHYwIB2WsjaX?}8O!=K zJHE{=hVI##sgP*DC;BFfbM(cx;Z^7Djknl1Ti3t{>n;;vk;!;f6`Y|ioGUruB ze(^tQFQSOyeS36-F;JaGc`jFO$3eH6ANp(}p^$8x+z$J)=gfU7<;gupZE)4$4-~(p)M-=R$h=Du$^uH3$mgKa~a=t5Ig-`6pJh@rWB zMVxwzk%9~&iMO!FMz@k51P&9iMd$|8Dg>pQrakzfPhM5=W}tKT{uY4LgNBlq6B7>~ zrle2+lGr1MIy+5C0-(~FuNT=Vj5A9Ml>!0 zwKBm?Dj{HrlAVrp>Qj}G`C;A{xg^z|r|-^L>ae{_X~`AQB`?Z|MET%5P*OheCfuBx z)O}6xkdeKLF-NFt({2HvbaAQ~vdF%R)6Mo}?uWpx+x^?LlPm+)7Zh)dR~GmSAHIT7 z;as78WJ~UAp}5*eRK-R}TSQP#_c2?G_g$B}*Y#3+^8+aqf6cnZHzl>WG%om`s@^;b z6#BNG?NgRni@V%gC*E@j&TKXIZg*p6;g7yU71C*<-z#{ySw^*8ghC6-IDW>iqn%w6 znPkqDiJ7I@bHbbs-Hl4Zu`Cy!onQekw)qwZEnwJK9xt6;OkwWU&N%XAb!l(Vj}K>C zh7!sY<1jTz*JLbfWFA8ytC|Oq5^&I;gB?-J&L>_+KB>aqM^s1*pZ?Z&{=M_ntwW-< z5+TfamKfO*+-}e1T%p|;xDnGBn`axH=;4}0VKD@VG~~jTrt5DsXu95IkjTdP4`vw@ zniqky+40sxEd_sW$>~X#GtgORxc16E_y{|&rsrqd3tUI?u4{I7`o+>aHot6|cLzd! z{f-A`Y& zGx6v6Py(ZPkUPaZa{{o7piDg}v6?^X@2lYY)YFm#u2#U}b6rK>0#(gv zW7+C`z;ssxT0Y$@M~#m6=p3oi5?}_t0}y)QszRbQ{6GKXTrQQ!>+8Z}9jjb2cMV2+ zZe(dWOXoo5qgFlMa?b9O#;+?8j_YJ*;OwEy=LXkOaka`M}q z+tZ-Rk9YD{rL&v!3r`h%&)C5OeWdicW)!moz(=DLjBZN!cMB0ilIaT}RXSOcU7|Rj z+;J#*mdIM^-S`3AEs--nsH*7p zpVdS;Cu^Qhx(nb}k%Jgty>-Kx^PeBj*7mXmnVi--2YxQiWsGA0_p1tk8?d(&d3y0F zoRRQQAN$ql^Bn1fbQ`O(Jqtq$sOo*IthZWH;h#RaC7F2XPnQNR^4OFKGTy0}5x22M zZe=hy23AYoU6H8f!!nItGMKaIN2jzNn!v{2ww#D$dNLFU*({WSB-iI93<~Kyy~(uAuO;Zd&49VH z+YQF)*seumgAMhJJPP+cs0tL2A1`P(xamLjyH7nEw2TEeFFnHd^{}kJzvrUc@ke*W z9BKSBK8Rzf$IuPt65h-H&3sI=|6GmzWp+bL9s1+v6Z7MyT~bF(Dd``gKQzfXc@J+H z?=cn2gJft$d)gj%+WQKK-hb7J31*d_G*3I?^KrS4fZsj=_%YQrQG9Wuk-nmg4v}g;-M{0(}T5Acf_unr;X1zaC%Sp6|#;$f*O@muI^y|YC zz;`gE`-!bn^saJY;GZ5JEgWZ({*8_k^l}`QAjemb9*u_| zuh2I>n)vyVmE?o^bRtBr^5hspaO~JX3@MTx3{zgj^=qFGHyHY?#ayw=#1;BKpsX7Z z4IGIyw6`C8ue01-wcLH-tg-u^?u}&SA1FvsK6#V0Q*6B;y~sMbr-niv)OFwHpG+KX zN&tV1S?NUn`|4e*si4Y^{_)0nmkeR(y3b8cDq(m#&``Y2tDI+T%`-s z1Sm8>ngSTXN-i04p1JVnew(z4G$40mhZqjRLqm7?bR3M#r&ef+fk0(|9RI-uD^(TJ ztOo9}WsGcxRV9+uK306_5Z{ zgyPI*%{A);fRVtj!ASUP<~2S^5W7B4#4d3QdZC292``gYrpIV^{xet2Z`I|O9ZO5* z30OHJD{8WbuF1f%@n+$G+zjL7nHpGhlQqNP$2qB|7221V4-aVF0lwn5upiMDQrXnm z&`3UvzjO|k`)po@sfNcS$se1XE{^cPZb_}T&VdO5ipLd=3A%&T?7t1Jp49|O{EIrG z@N(*@60F4~B)%0WE{Z9yGd>JK#x^{Q_q@*oBKC76pP8t4yn~my4A-`{UsPNV+lKahp+fC@gL^fl$@Pl6m&?-=h#R$V5t7u4IZgB?ZMj1n1}WNNTy%W7Bt&6>+rF8N&VL%c49J)k1` zF5`<*=;?k8$xmT9KSVxRInpSFUjbTno#u&u^RM0EYhEVoiQ8LT15PFP?IETKB_T={Rjm;i$&?U!ZE9P)A@c?M1dj{u97!-L2C zQ2Z9T-1SG{b`^_Hqgsktw?L665zXs2y3`DUBp9+w9D;z>D}EW9ZwJ^|88EM?{iG)T7RG+aMI z%BgLOEv3%15REyqmbXD(q4&oEBBQnqW;KCGhOg-BbD(^*7s-i;k<)6}~M3DZ@~e%g@}opE>3dUfzO_ z_<+WNp-3^+wK`qhtpPdd7(*>|mI*eWX}&7(W6NgPm@KPHMOr;RJI!xh^LG~dVHciC z*<8zu5e+}RU#(L)vtVaysZ4zl{NqHOLjhEWGYb;&O5tSPbx?PVZHd>5-0Glinl*--6CAJmV_LPo8azV*^QaNg4chHKuBJ4r&x&qbb3xX{R$ z3K*J_{UiiY$W5pvam83ObOXJ(&rX>yKBQa1NV1^^z~Br!^s6z`mK)6n)xJ zyyg;|{T#N;y*oi<`@cnWz?u60x43>ohKKNu=V}S*SVNa`(iq6+sxLRLvPNgUGrVqH zI;mpN%*r->Roq32fVq=o#)*d*Nkpm%+iA@Gh^*p9B#eXT@gIt04gf=;4&b3y%AvQl z`dA+CH&rKozwsMrcCPH#cbM zAq6@<&ljX_U3UPZx?7<^A47@dCr;!|Am)s*cAwf;N5h$x+DfG?ou7VYB*W87(&rLI zdzULtB*8Xx71Z{>6?R>26qMsTRWuD#UCW6T?LJw#(Cre!jaynPXA>pr6kN`!`Z75M z2U{O{jahHW>nnMG8F)SQ=e|VFnDru^t!>=C>@ty%kQR)^E?4ch;ZAzmfGJRt znIxGkm^PZihTVU5_B;6gt{3V1#*k&y?0)9l@_6xo2z#rjwioDGxU^_n97=JA;>Ck& zin|wUaVZ*HTcCsj#a)8C6?Z5uL5dfb-~ocW-2Cr3_vKsPz2_xl{a#?Pva|Qho|)ab zO_Px~5P1Lhq?m`oeEv`sKLfM5UFj$wL>s&EQFsG_;o%Aq?+O?pr2_)6XF}q&)Rw0y zM(o6!Ktt_k{Klp&`O!&PRU5|9Q=(x*T5rmnYHUrBf>z@woX4Q49toK9rJgc#XK!nh zm>hJqQvPe4%OrPt+jyN2?ELtyd@-GU*2%g0mMj@s^-I5VxUdzdjA2m`){MNCO{ihk zTxm(LKl!K97S$P7Jh#) zSg z!(^fSwcz&?11ho6btAE6cEP{!m{7jsRMsu)aawF5q?`oUF)CSLY_I<@qtd`5d;5IWjPO;4WxoAMf zoZ#!Win(a$^eNDW+hB?Li*Ap=-SYEG0a*0` zf@Ha!=ABItg-dl`y_kO*Xx)Tn|Fupu=vzyv%|01IG#0{~UoX*hJ$C4y;+h#bzO{oR z`gA_?qQj-uN|)p2F*B*P?7DZ0Z41v`!N5JWfs-(w&*V65qT~C$+eR%;fftv3+OMzb zFr#dH-iR|?{JDvND8Y_X{{5j0ps9-|))kk}tZ}oNgXh_CB>uOzgz4fnHom&X`G*AD zmLbO2x4gBUB7Fbg+JFb{lllJ_#*JJB1d%vWpqc1fE{IU3i@WYsoju-rHdWs;PCZg+ zvX|)W{6--y-3l3+kK9sGtEAf(WdPeT*mI)k|IDV6t0hh*{VpE9YbaY4^)A^WDfk|K zxm4CJTo%Ps@XQhKtQ>PT{XWX_utwjvr1e!GYa+NjM}v^O9t>eA*5^}VJxcjmK7f1p6~3S{Qy~NoY>TnB(vrdWPozq%}6q z<$Y(3I2m2_ju7lTK_74TOYecbV8Q^kZMu%c=d#*(Fob;M(Go;MV0It_!COa`TRpvZ z+Otj`Tc48tKB1XQD!Qwug=}40tw7rx_&PtgVb$?sJnRW z76)1LlmJpVnm`SkW)o9I%9vEl(I4v$Ld^C~#Oa#pcCh?KbC7w|>6gTu$*b$mk9ri; z_d?)WYF^s_DPm5jHHfZ^xqzqZ8;BHNVKbZo6NG*nSFvqcQ<)euiypuj^Fp7vBZ($(<2Ac)!8Mzg7D7{eRV~{Ps`S zrNSC4Vi33(XVlX^WWOYI&BS~N<~x)F7M@}^MGAMhf7|ZLxHXfOFpQOEFHj(+p|30c zKrtotChA>*jX^$4!&jRpc91a^ln{b`T!H1Bj2q}KQNiH)M#hteMNlY*kYn&; z5IzZQ)mlrD4*J9!R1oEx*9TOso-t0$vA6DpjpCpnrohH}YCb!mT=BlL_B-w}t)9RCn%?*^!`B>dL&9DDw~TeZuq)B7 z0K*l|wVw0L8L-k%(Nl}qjGdtF3NrMi|0iCG0CBDD=zu3lhOy(W857D17B2J_mwcWM zA3*}xQFjP`)_zR|9VJKdP0;l&QxM<@6jdnkiRUI}LE^Vkbcpt*%f)&1isH91hZMCH zEAbn~fm*~=pW&AU5`#+94c9Z)vP;H=%Qwb>!QzxLuEBORMPI||;xu@oP~*aF*oXBE zR^DX0d-5kguXgPFG-$|uwO(=6q3sL`RUx`P&`|>#R7dJlt&57}=eJ zk@oq?0nZzE=aB&M#MSl`eSM5WXN(V-a>HZN!iN|?uh_d?L=FK+r9Cdd^d2SRLT~q$ zMb(Ej&?Ll~l5Gvw5~LH6ElT9pe$|ABsq>wjZM?@{ElbmjYN+|3&OqN;bNTZ&*H`7j z1qfGm4}ZLiLw8n%%TBs$*gY?6*G|qQh3JXb2jSEl+Jy9d-n@)IaXbOb>jDm>U{N!2 zi7Cf`-gJqR$s!Kt7{pLdR`82}E#}rax$5Q&p{TD#g>-mS+*d(#X&NVPwdM;qSn<%i zrfJ;MGZ{h=(UHWddZXf8dY>Gfgnf5Bb)k~U}It0nQ5>;mMRW>$MRUV)Desse};Gf!#-O1$tUq^?`C zJ%dL~#KOHMTSOmLFhF?Rl)1?qJ{b$@{~^O4%jej~I1@tfBV%gMIF1ubI}D)xLFyxz zCqLVuk$~pe1|C*6{seZ~Ty}L>vT*o#^ZienU;yiwv4TqJK27Xl{>~x1CmG~;{pF-n zle32LwRCOd#dO(zYLtNz%1*dr^U67U67r1XVx!|KQ)^(4dec@;bt%0TFP6P3Zs+*K zJqDnW!7_RSjL#x1Tc0~u^-JL?qBf`b#+D?>_WzO^*W8HMWjdmiHNc5yom!1i0?OefIEGxj@w~ zJ5!70<=jKZi3{d@QJH#lB*+Zny0sb$i!@$5k>!dz*VQ3vST*I42_(=94KIz;)_0dC ze*rZC*oMH!taSQY+r+Y2gTgWPKQ~*4-Jy&60m48$QhX3*kaE4!u)sgG7;dcW2Y!1&?++ z2Fm>_)2)~(cjMj)L!-ImA#QQ^?O-s;d43@L$c^gZvB=0qBi(&5jH+Pga|>S0LBu%B zng~q5r;a;rvR-;nmat!oy!-t^;7n_KG{I+vUMPptXnA?7OVn?CMo69yc1j~^&(s>a zb?$wX;RlXZ?HrizBCosP4k{YPmt)u(Vfr9|%v*7HvrYDXWS2AeI}-yOE@4R!k&lz) zBBsO{oX!ndH+X8=lkJ}bJ-1_t>cnKf8)3C_9n1HSiBe+`&(rS6!%F@wW@oCULT=T# zva$BDXUMH@eVzNWtG;dmA+@{&OhMec6rY;zRhI0HN(`t4Vu7>4VPhK(~Ps|9}}e^q3? z45mn#l{b0gtvTExt0?6j$rwFJBxCHc%FN!^=hEDI5pAxHHVVuhBaOo$-p-j7t(3V+ zYORpPg}Oc$Vu89_R38noAQ;{^e>`jdcuDCyVYyf<0Km7pkjtRDN2}c?0Uqf#a>$f1 zgy77O>`ewrlXuFEfMOuzp!KaOY;gQumGi8qp>|@!cyE5WA)s>?E2rRQ4dtv+Vq@2N zczq&rzAM3ML0yly;}lg+Xn^nAhueVBt$h^uCDjFk>zPn-sjCTQmWeeLnh+2}%Hb__ z_Cid(bc?J>X6&X@>`l5n;9;b)lr>-b5oe4yRrzG@#b~~jxKopF;LrnXfB`6x;`Mj!54zO_>iLdn36ZY>g;zcgTvJMY0^(pXIYLRY1kRZ z;)3Bqp3_o0Hfl{EWIN6ll$Q|g$YC6W?_8Khm;t)mU$<`j=49(6u*ZP z0d@WGqcY_LumU^#jO*le>s-gXCD3Q+0D2HN9>%P(KWpfBPo(9z9<%H;!Po5ExTklu z)e4q>RVOmb@A>RmCu`#d&n%G(APeb%K09c~Ki0L)g-y|^~ZW01KU z%v+vem5BM?!BV9!3xkDR-A#5knd=ddz%>J7tADAgRqX$ysv-yhV#iD3l}(*X$DfvK zORf9>NNyRYvyTs=;V|~~*;t-9{XBPb3b(Jbyr-YkYPtM^OeuP*vif285__bzRoP9oNPbB_OV0+vs=GyG>!y!zwCmxZPZ$)29vGnl%fn5W|L z|A_fQk=QDyk_xYuo^db@tmr0N5_YrbcGAgIpf^ZXQaUo)^LyfpX74~}W5PphO@X%~x{qc{jJNF6O2SM$-BMFm@2GMBmLGtW!|r}MU~Kiz>g@(^H`o) z_sY*Q<6CiJqvd(J#*q=*U(`m6EEBRo&kzSuxR#aGHN3! z`djtqLV65|hqBL2l0tmC>c)2S>9$^&3FoQ}xIM5qc>d%x$%Cuy!dU!83)R5+eDqr^ zY4!;(`|>_kE-SMOvs%PgfnAzRJ{G$2D2DRmkN6vgc)|MiIX`aLl36{KOq9w z@b(W%w4L{z4I0!~vM6!n;}ld=ZdmHIdo-!)ch|k)FCOx>vzU`Toe#s+bZT_KaE`p3 z&swY7U0vi$Vz*45xSjoGmjwPPbno2vW2W)V2XZ6W7kgWx>oiUhWipEF$oz`Mdjf74 z^cMHl27B$s`i98sFi5l?Bl{EC-iuk0E{4_PSUPZ+nheVlz{MvTILH=@$m97ic|Q%8 z-l0P;qg_tX!74v-v6!V*@%eB&eKGxqzqCNMkkUSS2h*4123-*}N9(If9RTG3f&Wb% zU5|GC*MOU68{`ac+VCSaXpGef7cF)?TUqxMT5cQtc|1Yz5FsveY5a^uJ#D)2QZ2%#76L?HEbu*gwi&$q13VOZ>*e9v1)i}^MdS~ zGKhH6$0EWduxi3l@CTtm(t3NulDzpCNx)bm7dW2 zAE|~ZnC%^@EFcn>A;(vZm-hfoT~`caJ=WhVEIN2;{S(vDIhAbiNlFtm@I1KiXP?)z_`L=s#boPo3|n(-dxm2{ZG>v&_NM zZoYJ%tcSE-A!SSrR?T~122#8$#1j`2kN1y5sV`Pi_Qh zeipxB!IpD4=8v(ju>T1!hjMK>ZMadn8AA(P%3K(lVx8|M zO|7=Op@5%#M}O-ks|@aSlT;+EQ5}>a6)vL=flD#yl6C~zw4y`XrmLXDaIiOlrbl^R z(NApnFlc#oXS!X6Fd!5Z_$|H@#s60;^LqFJ3#kiftfXR5`CQf6ryJw3H-W#+9+-?h zL+qHYfhBfM zbv*=w_c~U60w(*h)|>&Xqsp~;3OKpUn#b)e2L}Fl6fvy%jn|$vbIIXy*Zdgj!w7_%PsGCFv z6(6a}MUvW+_EdWDGxA;%tMD>v>j;fg`>Yt_r%|`y^xIrD@!uKl^5xBh^i7nptIiGH z(=&aRVr*9abKC^uWjD}KXi4v9u(X_FUht6FU!aQ$9B!WM8z~jrG`lfA;xO5AJrk>; zTBitW*c5fli_zY@9dwIinM6<0!9T|TZ$s+;etzE%625Bl zYDY>cYMO24n}j?IP1k#ZS3kEjL;cQZ<07<6F$pN+F));oMJWC{zYrrCzW{TXCr92R z27|?$DC!jtW`uMcKc#1w(T30m=U~*7`InhBTqc7$lBjsxrJuzm&aYJ^+SjGuZI3;?GEqXz3oZou9mc&Io#`FPnetgnqeIvKn1f zPP65vW84;oE5cPZtv|o_4IN+xihL;z_T+h&w z6_;QJAG33><;v9zlmxR6u6#usRF&t9DIVH@{NiL8w@}H)+qIIbo1!nBd9id{t|dWn z_Ij(C6_a!*Hqy1Kr7t@Ak@iY8fO>n*Wd+~$11NekEHYSHl0p?#j>$gu(cSB>u6ytm zaZ7viAuhM z%#Bdth}G1DCpjOGVLDpOF9LhX5OBl++4*u}TDzzA;|32wCoOT$Nmu;Ne}sJzg8ZbG zM)W2$bP@`AFZB}tTW}{{%6|Ve!vD+m1SUv@SnL11RshF_dW-@UzjR|!Q#^1E9QrD^ z++EkPoYr22N!CN&w+0UH_mjeJ;|}P=tzy37ssGHTG@uX5m7XA|4^!jG7bm3TxC#@^ z-WF0!<-_LrgdgHgKDtz8E6ttDyP1>`?uTt|+1yq%IAAFx6kT=ra-3dxGrjTDb!DnH zU)$>r(>BZ?TJv=vmC4tzzn zH>I9R+!rhPoI5Z=;Bn`C4_?gbzT4za)Z}dh^Y5D?^$vXUYAUagRBRzO1ih(YG7I&| z9T8=$gZ~C+9^daxp8bkicbd^WSP9ig%;9`3;v$wn?Iyqx4E$v3CXZI`rHjjwvrdTW_a1kH6Xt%vjO@rDB~|qQ2NSh z)K|F_7vBK4+oLC214jogKbuw=2${K6p&gE+6sIxlIsz%2 zFrpa`*G|g9C7+xis(AR6DGu|U_@%)4!Dx||+t~YIzFae6_rV!*&FvM!>TBc?L%Hc1 zP!^R4YIddE`P{X!q!qCr#}guHI&Ql&pv*;P)QWo1>XL-KBkwh96`~Y4C~_+5c}E|m zOUX<-*`ztD%iOd1e67t|zyY(1cMrKFv>@aavewk?P4t}tlnnHU?kTHVvXw;#bXZBE zs!)dOlI=TK*CMm~e4h-@2e{73U;1<*MTR9)@69!#rHh1E2&I;~r z9DC2fjC!$&uM>FSDf;zdW$xpp5&54t5}03&_k6>TR%Dq-HdQ<&yiEAFg)N5sR}0g9 zUqQ)bT-Ch9F!=!4WLnB<>u!}>wd@GUvQL(fb_MIk=bpPo4c*Yz`p*4THf$L}M>Dz% zV+aY`*YD@x0mbHkH(;N#ORWfYWQFMA=mSJ=hg&pR;~3wSEC$)(2e}$(AJICgJ9vKY zt6q5xXw-AIVirTJ=)Iiiqh+I;*C*nH#Z<^=^7M8EJ%s$?n@wwcBFbQc>pRg!SgOr! zWgg_Wa3BR~Sx_&eu7-er1&EZpr6E`EeRp@g+7_nx9Aol$Cldin+2dogYssTm?lX^bWN8VJ+JnOJ^+G(t-JV3&Zrfv;k6k#B|kU zM<`+E70wI;|B$SablsXy9N2;Nm{YzSHgLO(yglzC#uw4yU%n6dOQBv5?a~kE-pS9X zGM=k$Ym@X3`lz6iU1=|k*;A|W+nWqwv!A9gReL0ylnxOPneJkc-15A$6Rcx)>H$dn zzTTy7-#ZaHQV+Vc(U}@`ienlm#{DC)anrz|g?KDUu=b%?G4cJp!4ub#quQRQ_Uo_3 z5f_0c7a^{0s(7LTNf1|e9XXTf50)wSA37uk9zWdY2tnSecWNA>(BqM}1H$)F?;np& z*o**oos0i+m7Oao*I)j9|NlD)_B)}`IrD(p+$sv;BH~6Lbd|lFZr!)#sCgEbEGL)y zwy1P)_lw`~Pl+&VGhgkJF?uuhVdt)C;6;a#Tl8S&S6CMaKSclmI^0zQ zqy*?YP?)ox<(?-(eD%xvuAKtiJsY?tFsvcTsug; z?$@hu-0-vD`+Fkj%=-O;2t`i~FXvY#!Go5Gx>+e!;xK%#uf=pK(q;71{z*^l-i=nv zB3x$ZMZjXGv^2QR+9&b#bGNbio48sGd(Jln&nV5wSj&O}!+E~wc9rDDoy57ssnZq9 zOyk4wS*RxJk<>z9@EXbVdSu~a3i5ltu%joxMag(M=Ou3OJpn>MM#1~f#fN$CYt#An zq>6na^UW!n!hBlLfVc3y#c(c}cF(8K-(JWQz4pX|7O0~^nyih|WMhOEl=iIX{1;K*|J__r`#9vkA2JQd}ib~e+);ljY!VgpNYZOalC0_90=zwEV z??5=+Vi}HuO;W>L-TApT%1ywCR-0H(2Y9zAazSq-pHz z6%M#f(WgQ004gzOM}H;i)Je(qql}o2cM~am1sp79Ww)o&Y>n6bB@2`#Y3QbPeqKRZ zN_T7XCiu&O!);+yEXx6fKc2ajn*@703jUBEE3S%}$JZuwYo~xb-7WI2V2830=3hSw zvH7!h;(C2kB&alv1DR#iAkE^)y7A;B+$ob?gde}2I~56Hqw^j6p^~NV=<^0HPr~c3 z%SY0@{Nv7TZ~i;ax>D314w_1-=VXOz_8!s?J47j?zW>q~KY#xhjbVSR+T?FEaXFj| zx0&nqZW?XHScO^KaE@O+K}t>O9K%~)QnOj%s-`T96pA%5sJ7D44MgH}|0)XWE+{wJ zZvuwQN&(RGnfLvD+SLl?@Sf9E5B8;@D`-|;n*zh#Xz*unM(qdj8NRHRMSh=aS9AEj zUVU$50Y$#7^g%MTQN?sf?J5?NJ3H|v@xH1v-Zq=4cBC)OyROq>a&L_)n>=fZ zdt2nyx$gC-XC!eOtqk|=0fbbemH^BijTg=(VlE5;Sl55fHtK&zF#lMCvn`$;3kMjJ zqFFPyemcD57aX?Paz{iw<2ao!uR49C`&oGZHD!f^Id>gwIPsc?#VIp{h(cfhd!`GHi6 zWejs+1$_4;Z?ivf43?MBP8q6ycXtndCU$_o+zQD09U#`;)_yONOs+Q1N#a5< zwD9L*yw<35XzNr28AjD#>I)H94S#zm5SIV>c~BW4;b7B2&TqCcSlht&msrFb56-Ka zHNu*2TmzEkJ_^|f!Bego>kX4ZdaSag6{w|Y&g~nV0!^0eXH24HPi7{>L!=ci_G}b| zGjgS3QX~%FKkmQgrI805sVI-YAvgJjr7u>z zW&KkB^!nQEq=pr3f1Md(QQ-xisjE~$T$g&Nc_MSjqO+1#^@6SdJ2Y}Zk||Vc`&gwB z%OJ0%sXSDi#k(KR6`*>1ZfqrpY{U4wD4lcwLiB1(6D5?el0j2X#ZQZ%ua{ug*jd_78L1YaU&l`kH13p#)83=qB>Wljw z+;w84TxU1pxMx%5J4{ye{#L1m6D~E0a{vcZz5_m(&LdAKNjx1x*m*AOra&Dwd*&OZ zSNV(X0X1H!N~F5v+Aq$wt(Qb^1S!S>!a*@pO{YepB1pgaQ`CqA3eIsQEO9DH&=E7< zr^R@6RZqJmwVAFhlOcTCB9M$8Z5))u8xZQveEoCs){FYD=XImW`ARGvFjTyp zy0j6qSU2A-Q|F2<0%?~*viMuKI=293``nkWv|)eJIlr%Ec9!~1mL&Lo$CZD#Wr&4B zsdOeI)Df2GXjmWcCX#Wl#FY2qm8cjUPNHTskfO&3t4RQ#Q`m@HN%V4Fwpi(qlyRK( z^aM5Y^^@(RHOu{DX?}E<@d#z;Y5G+wlsv)0K zm`+K+%q!hU2C0fB$a4Qnp_UMS@LY;Gn#N$aHcp_xv2BmAR^ijfIldWvA+9_()naDe zD5s8LP*p%YIihV~zCAk3-;n8*^W(7Ct^y{I=CY@@xm0wju<#{{pt9W`Win~6I-Q4O zK57`lJA)jxVI~%MiUAU`aME5Kxr(PAYk2~ zk$bf=R73({Ou*IxPNF+d)09QWUdU-b2>l#aE}$QZL>nkiy-!m!F#BgV^TYqoF^y7A zYaBYq#yJ~%5sYn|a=dbacpqWSw-*E(6#>i^SH&KriUF)HPq?8yuO_9U%Or*6#uS-Q z%qMEXzN$)M(7v+<$7zTa!RX_RHY{=WH4ZW0ZbVqTee56^F_&LSuHapCKF6OHVDjNn z9_?6nLmLWHwi*Zg%)Z0njKwRB{G|k52QD{_!AR&^k6DPkn5JEuG&NyzL{eW3BV(VC zV0o*2$0A+95m(t@Q0cdXBJCNpxu~ZM^0~*5!e+VQn6qE6Y2Yfi_pA745*Q|u3KZiZ zb562RRv0ug->wndNR+9VrQ%1$nl|kG*Fw*0C3zC8&f>k~l%+wkSQRWXx3)~7K6@gQ zA(35D1zU0}b|%r$wZOYT0n?QLTke`DhAyy#Mkdq-L${g`Mk__+*1CW4H;sn#RJ1ha zzV!r?&dpIKL-}zzb&&!Sg9?!afusix;V)B7Wy4X!^Rem~<1!aFpC>diJ_{rB$f6@& z>a>sqCI6{536#50US^fvgFjHqwOSc({ZEDr zh8R@g08_FGJ;FoZjWaKfL@_iAK0UV+p*Lk!E*-aTB4*&{Cn+hEbQ~rds=rnwP#Sz~ zQ_3Y3T^4V?zTzk7R&TXz8c*DC8mnA_ozL5Zn`<;ow1~hfVlk+_pQOzgLT(ebrdV34m9g_@G+tQK3bPWgm2URmC&XN z+%-;szk^xe-Rs~N0mf$smsQ{La4K$rj#}w-*isL~4mV>QU1j~fLzNx~dp-NOm8IVs z$#uWqL&X_YOiu_PG`RTTO)Qi$tGpl1stwo~6#52;s_g6)3_wUQmcRxA9}rDvkBfE2 z5!C7c&EFq|^*&~4K{@wij z5YYbs4ZpM3FLI2zo)^EPOwAYb?=+jK(61CpasZC#%&+=(%7`ANBy@_xzA#N1)4)Zp zXE2&BBWYsy+`7B`hVyN0g3UR+y)FZz&aM$xRVe>}oqp^86j{NYaVjajOgcC0{tJk=h*+6I0A?V!XzzII{zB;8(`x+-MpefqV}Lg6JoX7G|>ZwVb6fX^$vDI11#{JvobPu zmI>1bTAa2OfxZbjSDmuqHq@f1RXsQ7CG-o1+g=&Jo3)rO?EkdMbvA!mZdcl}9!YD& z9Fb?i$y(xy$JIHVrZ=KXb8e>B1H~!-RgBR~o5^&!-z*bm4+JJZuq!gTl|0YovO?(t7xV@!$Js1vRXP(HoEC0&Kjvcm%>BfQ^z3H@9*|pcBopC zEQ`fj9e&;z{9;RgrAhz_92?1^KR0Cd6Ezvb>Ok|Hr-uVpHX)-;=o)-;-~9L($KgD) z_l$p&dsL*gW5w57^!sZwiPuRhwA_mY;PGYSuseq2Ved$JMM;{QQX?e=FgLDeaQiLD zB|EE)Bbpsp7Kd-fUzAC~d&Xahh#+eBeUAjl&sBLbP9hhVMcD7Llswh#wp)<@kY+i7 zb^>!VNsMdj+& zcZ?^LtfV!HT>djCVU$D%(ARJ5bXV#mpeAAC&_vsf{rpI}6v(5k;nSSGV@~FeRRnG| z;<}@LyFcDDNd)ART6yLl+prd{-^Rv~JPZm4W{?5P{MQv(g5Ek3)uW9@rO*~=;=Exw zJ$q-6JU%oFeS`0+%ZUswkHGe)hQ4jEpRc%6O!P2x%8UWYP>vy&y39r*_gAQ$3|q^r zyip0ADp|;l?kyF|9N*_YrMb1lDS6_wBGJjf#!24a>+7_QPOG`45C%KMe#704rt{jv zk+us}>C1hxQZm1K3vUtz+0jXr}U zfWJ!xAn(&DB~Zyl&k7E4zGj>=HI`#D3%m(P_+2)vH3zwI>mHg&KRAH>6UV&etVb#+_ zEEqOjaGZfput684&7h$mc|)2ncH&*$72SYPFD3?;l6zD!?}lQ697J=+F@hR-m)OFEjOQAJiR7ABhoYTnn31T$J>4z9CN(Ady5G_-q*us~N9z?0f`K0P z&#AH`QJKT%WM#HL3B%T!t7y0aqDR-u80;v5!Ul3I2>c`69@5-BKP;L~_Cdt?i_zO& z6P6Z>`ef1-C-{cGlQ*JG$M|)N*Z7!%$5rw|xe!!%CbMs)NGvK{A{zLk+U*~wD{Sr= zVHWoP%k_F)vGr-gYtLu25PE+aFc2hkwOw^~SsysUZsO{2A{f@NMyZqG>h;7FA%d%~ zL*8N+EHkX9p!-~!Y`DZ5k2bv^eCtMI`x8EQ{V4i=7COozaNyA4)Kh%${Nm<_$0~}&!%K}PjJU^S3)SPAjqU24 zEqK&TFM_gA`3 z%i~{CzrAi!Lpcf9dmb|rTN7AVzvE9N#W2ytp9sfu2_D&RxMEbnGIk#<<$8X~TPl`} z!t&AkbUj40N`qN`#58Q}siZ7PQ#L8V#b0)VMqC1VxLheJqiXp1@dV!-xIiTrqCRHT zxTn4|$g75fQ?v1hZ9J6&20G?FR0ZXTD4zzAFJn;f zTl=YZempWx|bz|qn`&E1)=6SrzgR;0Ep0CN%7qN#j9zgeCkX=9AiwUZtp zuIO|vvv(lh?}eoRfbjNaeHG}wLZZ9<)OWWFWZ8{QL>uQWLz2wCh+Q_CmNMH*p+OeI zvVu)Wb8r-Thfy}2x%}Q%=_vM+!XPljFdz(0rQSMXA#N!wV!@N?$TpkCTWae`-F_;{ z{jY;9{XZRScB(rNJtu)LK8&0nTbu_mm{%=VvNZZ$Ma?3f+y6-$aveJvXfzn*+O4Wz zBg_qWSGjM}X0Qi}W$42WB9NYr`Wc`+U`*EhlRmqbt~6Ih;e_@?4waU^<%3&r+>p6{ zYOXYsijAubS|jFwupys-_H>fx*okMIVFVug-U~jBBDJ%3o;NpK0Ue1mq4KsT4wR^-c4Gn8 zReqV|Cb$L1MsteA;m;^pZJ4&8Kc%#oy-4u9?*o}csbFeh5z;J9g^^0cwYToQub`u9 z244cg0ONJul(8>Iu^M*PX9=j(WksZYF(!j3w)36k<8s39(0CxLtaq+W<^t}@%->)= zSj+XjEr$l|hDl|}A$P!PDsGCb4hUk`%lt~napxCPqNL;AgkEJ6I2-`?BFfrYM}QUi zf=8M%8KkbNDl*mErZVLAKt+0pN2;sS6~X6p z;J~i&fNSdJv#!OubbbqrfVY~QjUNypG}4*{j&@eHq$|EPRvE~UA)ppo_uAX&xy_!d z?*!Yb0Hg%^jM3zJ76H0T4;(z`YdoHss_Ep9tuIisb~_P=AH_AOoWn%fd89zWnq7(Z z5BR+9^3Q%YJrWrBN3u)&cE7?}m{3Ok26LvVo61jOs!KeoVXl_vS-54ue^OQMtv>&k z7Jdfs4GuDq8u4hGBAbTt9z;JzaCzED}FuJ>5Kwa;*j=6Gfb7pr4S2%WQ+Is;wxHyu8U+URd&AWuEdd>Qi`XmI_zBbv=tq(;d)w|!9a-+J6P9P<97XOMHR)*oN(SP%@ z;o`DpdYKO?XXN(P^w5jCt^Jfy_{k_bS(cH*psgWO^(Mh|-+0 zx)vf@HiSV~ce;gsX_=nPO>a!$NaM!>5+2l7e5%8G-HYE7!ujF8wC67D9Bb}F-ODXD zUj`AQ>C-RH*x^qz{el^rLet1)k|saP25*c1*b()+zW97Ph3o;C1{3j8siC9I!2EWQ z??Z-?kQJ;5z#}Dc?~G7-nVx^1F2Be;>u^5!19a=B^rV#j3ThQt(k zkLAXG<)JUS`D$IHF5NN}z<|n!+V;EmmBHs(ZRIpgzcB;oLwRX&aLf!XxFeNbC~h)bJZow|#O*;$zy(M`Xh2DaO-HB+ z$^M8t&`Z1MFDSo>6hATKoBqD#thvQ(e0-aw$Ct?UeP^B)sTcGz6;J2&Mp!)nAq{!w z|C&zjZ~xPDa-nU4+(K0!@i^=Q?7!CzMYQ=JQu9Pxw>e)Hg#@6G+00@ydpcGN>;Dn! zECj`7x1!f-TD(L5_BNcEnJATIswN3H6_l(PH6LJOk^amoCQGaof0R5-X;5rTXUm$X z1h8XN$GPnPlK|E*``UtXbF^#HtunG={11?E55m^;N7`!R$c#>|M@_KoRd@>8nm{i(5TnGQ<;&SS3Un%_LZ`Qe7-WnURLC#h2;iv}Vl0VajB&}r!|#0O%4uq=rd%0R zR4H~wnqPkYfKpS6WG3tYiBai*-ZitnZF_-{na~i*>vF-W(Q$ChesqH4`ao^;} zZT#Ks#N_G`fnQU*c*o;0hMOLAnQX>a3GdH(klV(v!Zuq*mE`@ON-MS3lt<)u3VWYP zZHvXq#E}_qotk3hZ1QTbS=7Bo;C_vhSWr)e$i1**G_;m1#R3SIgs2VHmL8O!J9}P{ z@=$F^`6cCN6+Z?{L;~I4hL3xXOiP$~ee5SJ0=drM%_{ACeET90^47%|yy&G4mIa^x z$p2=9=Jqa1V(;0JST5BANM{=Db_k^!GEqJjIwX6EP>cTifw%@Rjh5CA>J8dKHj-+) z8%dM3o*{B0-MQU-rnQr}<23bQRC>6Ht=2%H+Q{u-TS=-FO2Wo-G)Yr%DSv73{nLGY`D%an>pUEiX;=>h;6L;#9tmTh!B*gOvM)3z(;z81~ zDI++|E^zOa+kV99DB@AZ^kegeR_)v+MHGzFH4>*?sRJ2R8_35rZkh_bDZ%S|OycS-c=m7kFegr^g*bq?Fn9~(EI zSY~ck9;DJ9lGK%Qi|JE*o%+8B`^u=cA7$GXrxds1?kyB|cXueoTHGzgwLo!qPjM~o zT8e9MiaWsr1h>5W&pr38_13!Yo)5{&O1|X7{ATvd?7b(&Qp1;vtgSB$JF}J8WqH%3 z%pyu+^h-vpM>d-qz>cN{_cWTaNPoOPoE0P+L8MQ=z$yJpFnLI-=*;4P9d6~%oQo={ zf?}?vJNRzW!i+78# zeb|uc(_RmFCw{%~nC3>>_%DRDPZcqlMaR8d3}ofv^;Ujpv^Rf7MG3dvwnW5p$G7uh z1{RWC3PVm6_y40{fwMHUU3)?$3yTNOuLU6@FzeoK>vX(f9IhL+bJbWLEFguTx!j=LPg()+KQz2ori3Fr4e4VX1qV!Hv8jA!Ae;#&6CRXW7q93(%k9ffIQY7 zdRmJOqpeJ;= z;N*}qEFr+VxoaUyNgxBMMD@3@KH~?He()xFfHU8ZqPHNwp{)^j6W?QDKWGOmfxidt z?7!iY*Pi*^!YoiRhHCbjsv#0%Su82JfJ@e0X)>Jo`s3-3m9s>xu@Z6)bZW|U{6^y8 z2Re)U*VV#>w9_hN@R>)+EQzizUA4lB_g&Yn?Z>@;+Z6n#vB$?|ay4=e<4T)Hl|bh{ z2JQ7se?-B6AbTno`wKeitra>fhzZ!nBfI%CKK8@+X?MYdG|Fsry=Yq+n1Uo@Gu_k;Dt(TTgZ0J&ACYIwy0=c@fU* zxTwig6Q{L|sOxM3)l8Mb^4wt?c(ab3$6kj#(B)WkxHrK&lB31ShqrKRyrxAH@cB7x zQos9>DDXbL2ZSB?WZn6Erhb1Ag54VE0jwfnD&8h%&4?KQmviO*e)@9j{_Wd%)2zPN z;Sb2gg8mChP3O6K1|NsW{HJOnzSNxb{%^o4jQ$B1w2DZA{0b|TV}2ahst#boC)leA zGOOeoyeHmo_8bsPg%&n9X%}8v&H@3}*YfmjEai3z1I6z(_3r+hwrE~$aF7zWi3DY9 z_`0Ni%w?`1WMMQ@D*8Kno7heD5=X%zr=`?SpzJ7GYUW&-6n?x|S1Fkui8R^-qE?K9 z;>M5}9i6o6V%~O&4wLhqe6(83Y02MUxx_4iP z4Pdvy&f>k!UqGLLUFs`M5zMN4u4q}!L0xCSq>jNe21+a;`WME!afDP!W}mSStlG|h?H8P03s>s zkXeAUE@-N|*7?! zf<4oKfYnLBA~ZYe8lDzkzDFvBdF9ZM_?fFUYnIx6pH>V^cz}W{E88rknaDcENASJg zW~?GJhbv<9`BNNwa0NgM=?IAvos-RgFrL=w(|Us6hu|_g@c4bf-ie>=N^yP6S8}1{ zG{a-VaG0=G7G08{*c}bW>eGgR*b{EWXFJfF$d$%taqG2bfLTCn`;qsa&u*+C?+`Z( zSvbZtnEsBys3#yK@G1U%o#^%~O>X3Oz+L z5!e8?jOh+^%?euP(k*k)xkhZ_P7cl1FfEe)(fB%gJ{AvBXpkLav5Hz^1})1?|GP1L zr6K}F&LnMGTUxLVQI*hcmHXd}x~lQ{5ll)~qvp}N7%CF&abIB>?t+k4B2II9LL5Il z`swn~K-&mWbjNq=@(M@3F*ATaxB%+?AhB3Mcs`o&+Wu*8x@Nr_`i-PTZOU2wfFm!z zsw$O^if7lLE7hY%C)CYkK&N|(1CmzqS!s4hQy%q$0IV{9-6wvhJZaBYZv2UVlgnRh-~qWZl5+1TV*ClN)zT0yzAC zJW>TjV+TAC`<>7B6#1UN4)DOV#Tw-`I@9WtL5&(D8sN;P;1cehJk{?W)ei*6!=JOj z-xJv_uI$M(i0wByV*WB@;y)qm2i$u?R>8w5Rrng}brakPng|uIEfaEc*^W>3YrTUE z{bQ=c{|qSGK{Ck6xpJwvGRM(+>6iu6Phlq+=l`bE68r~=N8jH*!B!6(uT#Cg66gDo zwLIm(SRfS)H`eK{;)M-Mk7^6BeRmmpR!F?0vLGVR6|E5VzIB^48M-;DctNTBv+b!c z71d$YS7lGut{Pj-!i5i5HdxuiFW>0zS)4KtiX4-G}Bm%Ai z`lQj0<5zW^i^Ab#40cM$%a2;utBsQ#M6oy_1kT)EW~2%jV(~dnfA^vz68UBXH&KQ( zMlBp`%YX*y`=;&O>uw`w!NQP(9BtZSnNitLcG#p1<&OidOi7lXSomabFE6SiK7RwR z$skyYr;Z?Vbx>Y;twB5{Uv#%eRgfcUa7_CLxQbu*$%4f$%GdhQ+u3B56UDVJugYl; z)Aji&`~E(X?bG8Lg#!|eYm8_ZRvGCW<&p||Me%VzxwZA~3nhSgh0rZ|@0B}mWAc2o zjHnXt8WApugX~yTOGAsUG8cq%M#`X;mx%h^^p>G-;S6G&(lS8tva#O76&uMSuLxSh zZNHV;#(2hiih@J#57sRK*jy|>+;!C+u?1DITi1H}e6O>g%x*B(D^0*$IY~x8D*CW* z@zmQ&+EXHr^Hj%GR(|_^=Co%$dbREyr>2%kMd|w72Di_chg*iMQMtN&Q^@O5V8WE7 z%hVWMsxGOv+i##V@jE^4yqVYl&({Q`uW)n^`UONN$SYynbimz9hrV6pWGr)`V-QLq zFDEq0@w!}-u% zzCq>c1|zCGOww&A$o>#%;pgFnx~{1C_J@}E>Ow}o^$2SZXmzb{8)-Udroci5ez~iL z{s;%pCc1=>MK%Xx<@SJDAwAeJktG)Zcij)EWIlNkAt~aTP@?)e%f`u~7WK+Ns;Q&8 z0wASlS?A#G(rq<=69E{emcf3oN^AizB1+N1<+DsY*P>43Fe;J2oj`M;`v{(t+G!F_ z7*pc~OmRFgLZJ^@O1AD(z5O6beFvfycf_Ap@7;($f=1kaKjitCJL&y$PrT%y*LWJH z+ED;8bUO<_YQ~Y0)m630BAFd|y{-@R?b2W6Ve$(6Q{+wZD?qWOd#K`(K{tc5(nPw5 zaW5`K!lvvwdbrVQ_BnEgopi3U>4L*a|F`R^GY_Oy`%qN2}J+=EY9NNU70u zTSys63HDXn0nO;SObTKT(P1yHeNxuf2`>;*8dXxlNp?aOUt<%i#Nw)jMqq3r2#CKg zJb`tIUFp3~yI5X&1L~{OqOnp3B5?YCQ*V25!?YG73$0}U?aGx_3`eDJs0*|R#;{$rl&OP?v@sw219K&(^j!O`l*BU&Cn1ARZBFzdXeZ}EouFSY6zh)b+qp}N z_pJe$0M9>zx;o!NS9chan?wTn;itee`Q#RiK~-4eN3>l0s?x?AR$QW+S8e(d zmdnDMd2|CZ9k$D|)22~7x@KPOXnR&(hpg@-Q@i~J>~@=DNE!15|qsB`WifPgzu z`h1{0UXcIAgL2mCpMTp&`NsbTVbr^LYOk#Y#eqd_M*A^g`1nN@NcURC73HBV=8hxXw++v{Xg2k1o2Wgm&&w%G$M2**Yn>gSp z$=|Eq5~6UTAm)}$xE5<#;rROkL+l!Ycd1oMlo&Il*@$5p9FAQU!su?4~z*?mik-+mQd)gCy~ z4wFX%?bE83u|GKk zNV)JPpL0p6GU94uaOGE#**kXYOi7HazYm;02#}RL)p>z=FFlT!7tNyF$c*Pu;p@l7AN6dTad~Us~E> ze^+BmYY3cuW!CDU`OPq}8Yfm|mSMK_OMP2Nk5~3o=g8KyA>9AWY76UN4!3bA8Le6b z0MOHe2YRGxy3O%;ry`F#wU<1%ZCNQtBB7R2qhPW^(Z~F5#?BXtJ>5`4j-G)g!x|0U z@7?0&P2KWT8w`WD5k7LGnJG*b`x3Eu8g}(+QJW%Xe&h;2M*qQ9;osO+AH=G}~C%peE#P;EF<-$EA3W~5Td?OB=ihI zdyY!q25#>}^+5$&o|UVP39b3XsKa9_dg|Zj>*yz^X;q=o8`RC*W?bTFmwePy;qChr zVEI0Zbr-XMEoYZj)P$30jH}DDxG_3`HyK*ZY`Ck(KRm9am#?fumYP9`y_*|>>d|f0W=C`Uafa6IklNL$ua32!2l0DyAMpGV0Ap*ZhM`K$6CRSt?{~nn{XoUD z_0?epS@?$yuilF%MOAHolM6>vn@1kpV~GCsJtUZO^Nqvv!#iOf-;(*tG9kV-VI!x2 zm~oNVKwM4nh$~USH_fa<}{75fVGBoRcSGB?gu4~cqUQ_Cn7Bq+|!rtt6 z#Fm{TIZV5AiuJ_e7i3V!8WWrsPQnu1e-=0(c6p}^Xr|TPijb|f3|x}+NOQH$?{nAl zSS{njKGo?zMLN7Fl`1Eq+jKFsZQpF+-E+PCe)@`_A3d9WrZkUFdNj(@$t!#FW*6-G z%Z3ukp*hfL@nc%v(B~%;cik}YNAd%0S<*90Q>#<&D*H;JizP)+zbRf3@+I}pw5--+ zLIVbRv6W@+AU#UavMH-O(0LF1_Bfg!U4_D^!ZJsNxcT4+Z`d|8l>{xXz~>j5fynf{ zK>OG6HV1s6xk_Y;Ni3`&XnD+SMBCnW>JmCX0x*U$x%y11?aCK z?ioUoirWXgLD{Y@&kR^=Hb(;g;$VA5~l~2aI3+Ne7Vr`}{-? z<$r3yFOagRJN*hJ3AcR{vH5u+GPkN)8qXm>AX9SmgmXvVwal-9F z_R&$HE#8Ll^T;S#ZxhE;lDilk=>(8Sl2#~x4 z3??`BD#oW@$xQ24`}H_~bvE8+t2tfZw1c}v##3}S+$JlunWmuQ-yomJzqMnlPq%|= z+eaFPH@z)W&*PiFx`*tG^HcdN>C)`j{K?umV%H9gw4uiz4bYH%h-oagsR2nLEYQwu zBjuu8c1p78n>^M_w-ox36CP@_aP;j=Mpy@rkwm(dsY3EtfKjZO5>!?KwBJ3v-myf{ zRWd0vZ$o!ZcT3IK#2zK8towb!+M!%l;5HwSW)6KPB+Segi3C`oP8uGBRF9T!ybBU< z-8Jn2430dBZTO#3n0Ox(llR=jw|SluvNtjoN6g0HQ{?q(DV-S@G>SR`Ke>xSOUz94 z=X9AtpQgEb(QuBUHmxNJAdsnM2;L6INN31Cs`T_d!8}PY;&oYA{`raKwQp&ae7VBs z#7CrOh5*2K!wu$%{jFQL|DLc8-k8S!szNA zqyAM((4v ze%;4P2LWpa2sfC4Q6!x1=S<-+HAyV3ixdUEXa1-0!fQL&-$gxbT`!%AoAq^RLNeBm`Df5+#YBgd-Mx>`g+cgjI+UiOP?^yG&2F8_v!_J4 zC?e}z_1?!E?T)ni8m@w1dRd`?Yq$Vfw2(|e?>tWA*KqQiYs!&8)NQDT$R6nj;e#<* zx547jv4>%{-3x+VK|KI{NRe{R3Uz@D<2N^nH~8HG0MSA%+)DJiyyUK-QZIMIu;oSo zE+p#iWOdd_cj?w3)V=!di}uCh?Lue=agM3cc`Pcpv(_fFd*1bQ(2zc}{)F6kJR=~aTMq6S!GvqnqgKYtYSwlbpA0{Rr zC!vp3P3yO7eVL{2+@i=c6r)nA@{8&<%-r=kF+DvAE8gZS=^mhbE63PC596p}Q~pSO zPxbM4XpMy_&)caF2$n5tdPJLyD@mhS%t_k`B!aSo;Lkpld|cjhsi`wf%9pnrY(pWYzS;wp@V!pyX0Qqk!_EQ>NNu?rS#a&WR)@JN4uKvbA);|t9pNS$?zu`ZOUP*QTtx57_l zeG|zy{C2sSANWRX(Yg%r*SrU0y z`=|C6nV{Igi7V?F$4A2={Q^(JF}{OwfjMu})8MowiQiDAVlgAN27d=vafsmFva@QF z9uk+W&9WEnB363`o~%HRWZ+_Z-HU#=PRe4+6g6e@6TLbXsWkc3)%K@=o}skuh+TmR z7anXKuH#I&omn>3Y63Q&C@^3=@-}d{OF7B8ZmPF{)94lvd9rXzRnkx<46iq#4Z-qE8Jbn+(Da>=ABjmF4y1~zeps&=%#K2 zvotga5tBgY<5XT}H{&?v=+zG5%Egjt7;&Az#qVWN#LS)MpMAT1e|8FXgHoTx_m}zv zZ4=C_JPQnvu(!uOLK{ftGsmqh%_`!%DcH4v~GlzXK8(*NbgU@*`^! zuP8qvy8aF-^Vf;?pw(bVEJPxZ_w{%@^9{G_sQpgu@XO+)`dX33THI&*+G^97- zJ;qvooo#$Vn8JKKvzVuJX{!f9r?|-zH$L5PXob&Lx&zP^R{bmeF z(T`KMId0?2DYG?XpJ9I>r!RL!r@PlScrJ{~wGo}^HSS{b;nC}%2O?4S?KPA1hG^sH z2K$gGMQR3~2?g8WA`~!Je^X2JaHdFlM1mP@)Afu1IZJVMd7ERoLqTDj)GV`mvWxK- zC7W!|WYz|Qf#+vj`5}BWB_t{rwGNGMnDjU8#{j%?O zjlJyOr`wUNUN~*Wo7}mGbhhwOM6~w2_71(h)F7$YDLdc^T+NFFS?`@kz zfulAp&?UFu!GOrs^j)X*rVF72CoxAntx`U;PJvo+r?oTnOr{$E4-`UvRVs&rOuVvS zskukd0mb({y!k;taMPqMWJ_U6xdXBODCI^{T%~#Mai&4eYS6;-TmG%6r#j6tN?$U_ z#)pj_*aGaxnmYSyCmw~ru4@s1MeAisr?w1Qyh#L@Yi+w_iw?Sf=6clku^&Z$n0Jb69+d=$~f-MlLL~CRk zuVrMlW&GcQ@qkk^Hh1G>tv_w?)~=zl*u2=eY_b2u?f8Zakssgx6u|#{=M4;PzP~Te z4bEVFIXN8;Xq%|4?2(B8biV^vQEYrU{vwepF1Le+n(Z0Eo|K4VMAn#evn`{*sv~60 z+LC!p;nNCd{-C585luM!L*o z8EKLAN^k?PfQ#DZ`8HcTd3o-eTX_{m?i>sY8hIJTa?D>srPL(u-1Ll==GUkqt`{3_ zwzulTP(gP@mec8e;>;8nM#GUauHR;KI$OZLQt$G)NLHTeaWCIFpZGF;j{|qw9FDmh z(EALfolYtzN<7tDCNGWD2+hH!s_tO~XbH{DF05(;l zgFoYtHr-eQxKB_Jx*m9UKTBT_!Kz4MnXEl1`|5g@a!9zw-nvJIk>Lk$C&Jn(^>d<7 z?uY%CR`)Hl4+uiTQ=5|G%b`i}M5Y;$-*1qe)Gg2igp~m7sr}4D`K6Z3v7x~nS1WW9PhFTNh&pw8e z$1$F4^bF;aEqZN_HW+zbdM|j^Bs{FBm``@Z@3mn{wB&-@07a%h1&tETk@4K z27{c-&KKA_HAt1YuE0ar(|8uqMAWC%^g$I8?T5iQoD`FBSn* z-FX);9!rrAD;^4w{hxMnN{+fdZyt9(cqQ- z#N?!ccC<$hxrg%|X&DK>#kee&IvFcRw5+1Ls_@o`TapSXy1aK$9udaBeJ^yW(CVTj z=r@_qQ@mKkK9639cyI@{lSf9B#*kQ!M;@v3)&>Zzf4Ev@KlzDdi>5_GKV#<0>UiEZ zqwZ7|4R*&mwY1?l_5gdIfn*N^yKare(JOFP7Qh}%eyydrS}SE$)-}%pf&~537NS{I z*pn?QXWumInAhhm0v1q!_nRLE6=_Otjz{c6pjdutDv@L@XRfy|G|tAp+19{-WHyk0 z;^|Oc;inE2JqIm2HNsRz2J)zMkChT6-9m|?^m#8$84;Vfl1@yaN{ESOdm5O0zV;n5 zQ=J*=o#jVT5l%^b{1Nx$uZt#KHlAV0=YX0;b9;LXh^VE3mM-DCZ|CiSya1D$)l+;c zEH%DW<;TB@wYScv|D|#x;(hsql-Gr(G!odI)|$({wE(jxA0@t8d0XPy=4G9{%v3R@ z@e3|I8IYCjn9%|+*VZAD=5p?#0NUgSQD|S&*1OMkW z0m+1f6xlVv-=a2zEvlJ^UvgGrwlTILOU=>%Sw~bey7}IKG<2USZf8w?s61@RbNwb6 zucK5t2#yun(fHY<2)Xy&C6f`rgmWeiFutG~?59=s`k&pHUumPthIBWT91$CV{c@k* z(6|W(idaDg?abBi4{*l0g>${{`Kw5(!gP2 zlRGY7{cah)&i;N(mSiUP@LYUZ^47Tkr|4FXUR2C}9(p@x!3d9a5sCm4B;c0HV?)@P z?`s5`76B!tqB(Y>IwN~o3lg6=&g~p?fB#$@gq@3w`uB<~TU{_0;e+1OE8P8DdB_3x zzj7bd4`krKO7Q=LAH|Ro`JSFukrh5)mM*5OOyJ+e*3g#kV_XJ*cXa z)RvOP!G#|QF3ZLn5zJzqLr;36CXry7rk=wETqLJd%~Vm+LVy?(W2ugLEL4nUq<;?x z2p{q=-5=e9VUtxl2TROWP4&Z~4DMlHxE@(6*hB)$x50UnAeyTuC(2XCb@egD&KgM; z#S|HlBa9pjiR?FOoRTqD*=(6?w$m{+y3Wb(8LE1)I^o$8%DYS~m zeD6OaBgxKf@!(t%KKy3-%`LcqCJp9AR^_^QW$JcP~PO| z{fx9vaEOxb7LLWmbKDH7XM7Ub=afup?ggs@9`If#-iSou+LF?{xZwvA=c9M6{qZrX1H&hu`GQ#{2@K#MqVKe zZF*~^pR*wNm2MVbocEz#IpMV$!jwzNvmRX$dK}66YjzxS+QGT7td4|+p=2}KlF(yP z!Q3EEM}bNQJj=DPKbO=3hkwGDE&nOJJLhGNz8=*SRI~z%VJc&+A2|S*F6m>w8~f!$ zFQSIW;!-(Ui~ghgJrit7SzDBmQ_u|u>{XHe@#aNHoKZ!Wqa>kST&BR`849DDlj3H zZ&)dbtpyBTgA>Ydka-gB5m>l7$+mg633+rA8hxLZE;?^%o)3Q9Au&`~uM>=$OU}vK zAK%9TNPNVu0mxbwtCipIa6lSx$`wLT+haY!6w4a|Bw6|!IX#TCyc;mz=X zSNxv4zxBFP*<6aT5d6-V9crb+HIk()aXdHZ7@$l8pBxrrP*y8KoV6p+dVS9+HR9yB zts3|R#9Gyb+6U72J(t=qXRq{uyMM7W7vknVyh?N_nnMA0WpA#&EoLk(aP-t$Y~mO9 zq6G3tid)EtU-*kM4@TITef(r;ZJqsi3jw&eNX+tB7~FIN!C!Y=c&;*L%;jX)Km17$ z)4;eoEb;nRR}5XU>b3s=^v8o4@e)XSJ|rAQcsk&x0oAP%lipu2{s>mnzc6K#WL#qB zlSF#AW3wW1HQaP+Z!O}V09tz8l0!;)x7h9u3H9dow4oBmQXN!!1?>p!DHKdhvzS#YxjAgu&}p+}OJ>2w83syzKK9*P&)&@F$_*F+SM+7@i*(zU_~D#ursLz#;;+%?tO0}w51&jYY} z@*_sXTi40_qxPGH@M4UO5234Jn>hmg;my15OQTuTx<$5%S(J^ZGZiv~=g0xAYD5b- z=}o_yS!h4E#I~T#7Owfp`FA z-&gsv5`g%qM*Ono-oJoM5K-@Oqe4WWkGY0z{HMseu7g4fzcksXb#Jo;y08L62N~PG z-_-P-H6wm2cjRW+n#PmpYS|vWiIzX_-XoLF-+bJChF~%UL$M$~S-`8_$E<8Z`CvMA zKS68HQ?3@q@6X<9BUXm?g@OB#NDF$nR{po{s(Wg%f0a-(s}-$W~v)Xa{V+XJFie zfz19|=zZ zOR$vEavq(kq8;PhjxaT)8GWmu_jCdwiGi~MvrmLjk{pkRHXyeaB%I@oUj%s*$vf(IO`OlD675AcMCs9 znJE1G%p_#Y)o&D-fmP?yaB0eX&Hh$ zW5DSy=Ka2~d@(R_G>;c4N=)W-X$kdK$Ak)q9PXo~p+$`4q@kOG;nflg;zJ8aZtIX+TKyS! zOEQFR&?-yGo^WrRV=9kU#>m4vwt4QWY20Y&+4~vE6-)K?#kWOSg{Z--xp4(J$%YT- z)L8Y1w@7NLLOq%ARNFn@wtvS@j`>Z!NgRGVCo78_j~w-d>%gVjpIAtM$>DZ9x8MZd z*(WAY)gfmzGWsn$=={*<<4u=MDWjw@3lwig!v#wVi5PitiqGv)z2LO%F{ugOmlXMb z6?NaajB~q@1b%vF(B8+~E4=~|XQ*gxm(vpI;p|kGB6Eq$KL`d<($xQTK0;dL6~l-d zPxt!e+Wlo=W6Nt#kzOv!Cx(Z+-+K;Or+vYbnlUXFL>1~O29 zntfJ@4d}C?fIh90T8Ed6n$nWnTLnAmb=I=58;=RiW!DOy8CtZpb*PTfbI1XaJi*=3T!whlx zzIE=*nK?nCYTA2GDICI)jGCViI@^v}5=CcAD~F8m1Md9zx5S|ujt;D~;9Jl*#rxUa zEriKy-1_RR7R-inMzbnh+~5_frWRRKpqWY6Kxg}4l}~a-n{=LW(J(vyDvH`e1V#i3 zfvh_&YrvNRXqx#}D<)t?f_~cj&-#L@%m*I%#2JMA>tegVT|eZn{xIUN$(g-YP@ZYD2Us?`;%I55P{>L(-gcI$ z#D(1vR4a0&mIX}be=h}Ms=FabZFf*mb-2Y2=BXW*4nWY3DGNrN5c!NWhsaWJiiP}y z=m^60$16e>J__k^Y=3Ox9{u00%SfPMYi%)Nq>tP}3x zMJtfhh(XBw{czNh2Te6FdA_ja0Ccv!L$1gxg|IQmDdy$uW`B`sDr>nAfi_@Lw)eJ2lmV@9(X@yDps2Ke^*LC@vc?qiHRDK-OJT- zP|F>ct9E%3xUCRU>z=Yzgb7_;?izUyq+#EZb=fZCuhF`b@Oa1Da3G^PlC>}zn^mon& z%V!)Z-Zf%ABG7B)>}1LGzy!W(xC%UL$1#di+w}Q(1;3Y=$x^~sEOMk@wtcR^}{LMo%O}C7Vfh{i9qmE7}G+yCc+SqprA$A-UOtQ1!j|1$% zAof8%YIe7yzX5ZND{K!jjK_(73+$)`L1=2UnZQ{LK!FuGQ^o|wn+XFJa@S{;(O#^b z)FA^wpOE=!!?#wq7ZY!jXph8PE)Be_&&BiS6x+lrypNA)Zq4bBYwA{%-{v6VCOtG4 zug?oa(u-8&`^&i(r6e{|+kW>rolL;7<%rAj$a}kXcDZ)(xP?WF`_1E| z#s6RdJkUJCLc(P>8Wcu|M=9?EYEzOq@x=NEab$qd|G^$&>65}&Z(XjBbWQv&WFN!t z;VE0p15Dv#@(Hf>zk?SBu?@-}Q{4X!^!Pq9=g!`io@dAI>tD_nmt!1&>!W$kE{OBf zpz}GL9G_}>pK4kMGWl;%vros&1=3!Ta)}hbJ~o;KBBpCv2&%8aFq&`3bU5kmUo~X8p6N1jn7QiB{R}er||8!lIVdulAgl)wc}yVMyt;lJ_9)! zU-bMF;i!eP2Ifn-5%8VEh$wG)9^RC%`}T_Dd=umsWhqPLvtVWJ4=YOid*$+?sP~Xq zcnEY8c*gB>82^mK6lbeNimcZ3PTn!$VA!~~_LKE5&+o1j@8FdF#hTe7|A+hJtIN{! z0DPXGEX@<6%oVr{v`4N^gy6+bXeKGe=bqehBGu6vlX%}IO01OVcvIoA} zD%|zwvIa8Ysl{onLjk;_Mhj&inPn@vxy8f6V+kPa=(b>s6wGB8g<-VdHzx~o(k151Knne8i(6X8IGFdQ18&q_@1i_pO-C|dwiQWdI`rMxW32$&9I_d+ zjg%USqoMF-5eZ3fvtu;Pl{ug2O=%mxVZ8YN=S7H4vjqBe9dxC|z@f}^$ zA3UZ|eiENf)!Kx=2*3?{mzTE>4kDHu(@(AQkquuo#0 z40!!tg+QfYvw?FT0M6cc+HVWbBd#{-8il-^K`yUr0-rm>&EQNHz;-PEx);3^2m@|$ zvV95j;(~XZ_g6Kr>mIS&DQNIB#TY!--@BLu+*<}bwghH7`yU%Z9y$Lh8vndrnLZ!s z-=o4dsm?uLCO5u*eCMat_g~zDf1b0J%S|u8v8{~D^w(#Uu=g8K36-j za{zlV(8SUmOY`fqJW{2zG7r5x2lHtbtGbCudKDG~?xV z@)SVp&o81C`}%E-7v!K5U6E#*s#{+opO?4zcm;7I4ob9iyP%H;(%OOPkF$$guP&`} z?J5}~=pv*A(4QG%iv;?PIV=16D|#e2mn>mkKm&K60PMA@P(iw=B2n{&6Cs0h$S7$C zxf^{6-!6DHh_*v2(YPcIHDzK&=i2eeO9CGo{e>@cgAuwSV9=*Xm52ao(Ec@3{>YDo z{URJLJ9|aj=+o|PK;6bp{;O!WoM%WePQ&BtY4wi2P2a%GREewc%5c@|qT7T((~DaC z-7dAP+3%;xcUK+HW3iz*jZ%o5f^gttFk7ezQ{WZlc#k+W=E8aEq*10^wjcSIJ{WZ) zs9)XfV-T?xsL+IF-o{S4G+20~c5GYV4t?v|^2>Fh6IA|-*{PVaQH(;2_r2`GzDgwwOXhN`lI+ksT(k1I<(m@d+zT~qu{We* zd~B1;T`$=uV)1X?Y`|v@{x}}05?Z2Y!@k>_XiW)LyYVKoic?s8S*IxKuindK4-IrP z6ecdWI>izQj?2IxzT>bZ_4nN;oqvJw{f6;?Z0Yu+`Y=@Sw|$Yo%hrt}7&V6OEZcLSA> zfT37smZxZUsz`vzcn3aUZZ2+)lA4!L3JOWV+Zl-|oQ@HalR+4LMF<+*L`lExo@h{i zL1jPjr=lVqB|UmDu$tf4@p=%SfvUTD7j9jK)pqgKPC>1M-or!)`%h*XX^kTsxOQCJ zVmS2vcAaZiRaJ9mb&lU|JkVS}iFJHU9%5f0FqyKN9y}y^u29IrZH*rbdmB9z8B#jj zCpLTJUb07~b4`IVpnV0kK6f&||7&UN%3qd(M~jshv*~euPx}1y2Ai zcRz6TNS=A32c&N<;Etq0h+ZeXKMn)&F##QE^bRkBqBR=dl$9z9G9XL+QC?y0viVHp zPHm9Rq9OsP2wcjIF`s7luV}w@K)cUn(&$j_NYi(x*VSlo@G-erW+(eeyft)_RusoY zq;f3gKEBZgp~7Xt++AC|!Ni-_9FP#yKVsFgh>G7-BlZZy`#)^Gbx@lP)5hBthvE|4 zy)BgD?nR1gDc0g{!L_)%YjBqW#jUuz6Wl#Oa6NhYp65H?eD9fI!b~QA?!EW5*Zy{Q z3nfa8N5u?0ATw1j5H7A`wkM09E%;fB3s`im^X=5%#={cuGbx&9KiK$InC_%0M|yfa zjaUZd>Qt`iK2Kah!6ugLHseaE;cK}l7EWh!z{hUY+Dr;%%qY97fNs96X93e0JW^Aj za~M%s;KU`X68=$7BtytEbN~EvL>fxk?ur2%q>T<<1U$ZxAH$o297PZB=aMhE-jt3B z0RD^*0n#a0;*_KV-h|n*5;qU$hPGyUBn5{%0|zhmANP?|k$p!;(d%JMphD4z33~?6 z6(X!w@)>-evCn2akP;`vxp;B!0_3rFh3XB?$@YC)(>|*GC%2E7ru6}0xRdcSK`t%$iK3%f;Brr29L zgPjNu+{aG-Y+LgQLub-Y@3PfvgSfv;Y8^)}dpE>z^?Prz^b3R8&g_u(#7o;T2ELSCk+ZWyyA}y1uU~)_$mojP*5RotFr|MjYW$EO`q%Sj*0yG<9D>is-%%RWwB{2{g_@&p3)VZPD%iCHl~=$)c>Xyi$M2(ffpGmkB$`77#t>}mxw zIoCO+E<`=9;x5jiUBvi9^`U!$wnuySMOD|16-4*CgEzqN!i$hn$JhH*dWbw7vN0dh z>?Fu;%r{@l6SG@KVY%Tlyo2u$-JL(TfrA7U!*jSvqcc~dzh=?Eov3fsn9hCOZ0W6W zaLSO6SA0N|i{~8`zkr|*NaqqCTv96F3PIlVY-w|vG{(jk zj{QdnMZDp|OsNY8_syRU!GSy1X@vE(J5LR?tqK!zp~ zNWdwz4R^xqG0BJW0tmc-FT(t(%RbJ$fwIfvPZh<$v@t|Y>!}vcB#N0o>s(QJv^4U(>18o;1>uvs1^un|cz5eq>vT*hMUs3eWhmhj# zm6OfM3yID06L-&nbJx9JDcn~X()EROo**cv21IWoj$pfW(Dj66@?GeFj?Jgo zd7^@_x^+zl?2vcs|jc!?vxw7eD8VQNGNp< z**0o70-k)x@;(L&c%Q!v+qq`mdHPSf`X4w3A6%O9W<65p*(>O_hjQHzt^nw_hH2Ha zT7OHPoif-GLI>pZ(}hwhH!L^qza3L&cCKy3b7Uj!g?m=LAkiG6Nb;ZS5 zX;$#}r%+f6xr&rMEQ{ae^keE+h|D>x67!fptHM326!c3go|_VTpW zaP~Lh|4U$aB7=5)EIZtCN#wMN#hSqWLqPM8)=YL}4~T&oO%&ZfQ`epI_M*kA+%wwy z88vPa>1z+2hVBv7fx&vugio91epWPp?2}A230%_s|c~#|+ z;@nISPA|43mU2(s>p{H0h09{y9@A<|FK6A`U+D4-@G~w=U8J<{wNutU%(%Mi3%GW$ zTaPbs3Lce!TBNd`^^^=8M$*Wq3l-zL;II!A*AB$i<00cwGR+?g_YGbLSH`yT2OYW@ zq~SF2wE~TDG2N1)KS?VP`>~7qXUJ;0;96>f!V4GB8&BjQLnX@czcXixv@;cnN0NnE z4?~h67wlMmr>j*xAv2sfErBT8v>7k>eK3+P|EI zl$M8dTaY`kq%Kvh>OT^Qpv^`Dv|v%#44O9^5WlZA`hg@MrD9H~qXPWP6! zy>FmhxhHPt(*~8djo0c~d&kFCKu=Ezt)w18;Y%QsSQmCN(MOZMn4pN?I%ew|rma9f zph1Vv-*oQT2ctbQ_|XK@18rdr_ko1;bzr{rbTM3RSDSxNrq_-ui<)|G=d)K|Khg{og*IBcr^v8$ z@aK{33#oeBkX5~;dc_0>oWc8v?VqTzsMx(5*bK|UX=i!dr)Q6<9#-X7gML2q)`nx# zC1xjs4aG^W-2K7C<~U;vYBWeLHb9=V35s$o6QptlNVQ8zKQvl+aA?jLBMsXnszb7h zuvYLEcGW%MkBMp1T-zG@_9cqz~&>*RMbj#4?i--m1F;L?#Z26b`u(JgTE7{ zF4*NpKvt)9HvZ$$#r;Hw$|!+ISI=idu1z6DBTct9cVF)Cy(W;Z5f1mldqk_Mo7)nQ*3KXU`ChoK z{#K%9<;7uH6%cwYIMghrmfZi-d0KL&@_?5~-Y?(bP+$V-(;!BHb#(Hnb1X7WOT3gq zAQYc2o48r=rI$q8)93z)PiX9^zP){Hb(LHS)?ociV-Vr`vf&Ld^B)f-2L_R{7xia! zHjNwqr&^6S?KA~ep02i8=9mKexZBUsvpi0Dy&*T=kiX8%zcpwC8wh3k;8N%1Z`cnU zRdtLdzSD#?pZ*2&Qlzw|;>_>FKi5KHr$`Ckyw1m*>W;<6!xGWpI8193{&}LEi(8JxX)X;B-!je;&$BG`*VB z?{%oDf=Dm!bVyAz=0=RSu+mun@mlMc-NaT&1=aY;_1yu(oa=5Exy{j@c_I5X!`jB~ zhQ1vK#rgIa>&3!Q%)tzMn=lQ%u@ir6Rn~7idUX{m=}0UcggF zZI&l}P(g@XVVgm`N@fTwRE2=sq(UCBk%`JS4~gez_5|Z@BwuCF+-Gdp-r&bY;}jPJ zCzC7ST3(UOrNPx7YQwI4Z+?TIEh{LpO6ZzO|k-R9}SqhiD@WGlnN@u8|xRI8{ZH9V){6%_uPu~;)83c~0YYRjP zcnIn03KG3*@>VN@WR8M0g+UD0&v}l}l!SaSvK*U)1j;^XS@hf(kEJ~V=@Lab>tEv! z(#Y|~DRD@-=@8Ggm^kN(*4^0IKzl%o41yXWt%iA;wALDZlcBr6w2xBEDQ5zd_h~j< z;34syz)gWu*fZaQR4BdBaQ0VIwP(}QC?AozGQ!>QKl?FYTZ-wPuy>N(HULi>;E=3b z6f-J*sDOlfY7ug@@~z;ZYcOnFy7nH?>CO;lo3|aaUHl4;SC>HjLxUXp+seke`%gI5 zp$yMA3G92Lc11hD1Xif|DwilXO{uH&N`%=V5u18G2OWF8t)u}L3;E7i?bVn3wq znTZK(MiY8R%_$}>i?UCZ))3hlZnCSo7nQ*$zj_sctp%ImI#{n|Ny{ETR(qAUojM0+ z;sjbK@EvcVa>BMlOoDbFyx-0(*J#1}o#n2S=Sy!Qk$`16fJzG;xyWEcZ&wIluBRKK zVB2b2U|_=K0I1=~W5Z@&`uL;wVd>k~y0io>_krcMy|`BI`qp6=_MqP1Z>}^?3z8Gq zeA9&ME;`VvM7?5F%>z;FgSC|z%%c;6Xg{fuiCIw4g(F~#@t3et-@5a?dKx^sd4LtD zY{Zd`J}*K{1(+-Mt!6GP#e|;<^T!i5#I-?^*#o3nu6AIef9%ARapM2tT0B2L4g1`& zx*ZfGk9iR4w7Dm)J&vwzAL+l`yJ7bJTdca}^9&+;<3lulKHapw9C-Kr^rOvBWX1B!9TW?A+nm9VUmx5vJ%GErI{zPxFci_(;vjg zwzr51n;I*91w&~RTt9}Oh9M!scgP5L{G6t->$f_=D4E7Lh_|!kA93jzIX{gj9X&mt zuX_C1ZPK%gD?HjawAP!rX828ar|+C|^NH8C!%M_MW3ee=;e@ygVBbnFO4M>{=sV(% z+c`;dJ6wK?9(IfZli0?<;K|1a*BeC1Mv*Li`Wt>8_iQoQ@SkfkST^>{rD9=AE5{0B zNM!2+1COYp^)i+8K|v|925Hkhyz&`XhB6WOv1uv9aY;Q3F$M+-xmFfoN?QejrZ^}$ zRE1%bUSePA3J`mpw7#5-(d!0f13E`sz}nV5aJ(;CH$g1?iAeNE`HMTs# zx^eDHPdb!M@T~^T)c!D*(!o(5;imfC96E_E#}^B${75x@ZJifFWxm<(=Nju!@OZCN3H!8)AzRtn z+Pk_A7=8L71ZoJcbG`A!#5^%riR$@@0=!qdG(!$AU|E{K?7EiiOI#t>ez*9*bon1q z9sTjYgBWq8%_AAMLhNuWZYnz%Yu4?%L=QQe3d&RigED`wg}>*+ zLcpF6#Mtqrj1$0e6cVgKS~K*pR%u>OGPLv>IGqba014BK1$4t3;ACKEje_#mG1Bym zjcfY@i`qxF8SPyid$i;IGA|M69}s*lO?^g;yfaztLm9FjP%XO!YdittAP3-!lU|#1 zy!SJnx?{iu94Q;8gnB9CNcuJ5P)3Z3m@ci;xz8TgK_FGC6;>CE6EK;cT!m}Cc+(oj zSws1G7C#%|#(KNR&FM}90cx}HRn};Z7W!WR6@$EHt9|fvR5<<*W5d#uG`X2a2)$f*<&%!%_$ViS@;4ADksO<>`}fPMD3@#Z<$A*ASU@84&Kgs+1! zSM7Cn+&Q;L@0f+ND$kdGZ`(Y<8QEpttr~BxlJB@HaQ=+c8{2hX-#q+T7wJUa;+a}j z)zqgjuu39kCtlsISP>o7p#K6Cd1bzCLdE z6uD$yx(hlxJM#z$?OvaNdkF1MyX#U}H%EEt#6OQ{{TN}!R%D4C{jt9JdCX87A(;y} zcTW)iHx%#6YuEPwx19LL-1C3F6Mor)<=?B;VEaDXmup+-aL42DTSR{-dDy=Z3=P2w zf^A;=c4x~=s&A~fk{@blfD;$QWwp2hS?Q8C6^jb@7|t!wQihS5d{Qd&D-CFT<^b4C z4ZUJr*P%EG<=W9ohJXaVii#STemdn3Z~qox{&tETu&|ir4Hmmw{KrBb@sVG~*}jLzv`g$VPAK3$CgFCKskJD%Q2W%m(YhYb0x2Id=tWRL9??YWW4?uNnV1cFKQ@KOqsZfiq8wghPC1xW@un(z8P z2)~RE-{14Kv{>J|aizzq+{bL}IgbG)+(!r<c zJD4C{Hf}dRAt#LoMOdhL>H?FFj_xyEp~Ra{kcAb}i*Vs5TH%|5ek~%rJw6Sno+A-g zqH0B>^}*@J7x8Nd{Ws_euGF~6oEBK|3*V;lc5e^*=8O^LHJRr|x zHm5HsZFZQ#2i2U*#`LjIi!P1Am0A>!?_t?L27hRqFc|~#J>_We4%b!Hn2a}N3r zZe^sPcd?Q$P0;+9*o=O`_f8#+9ltwG_i!TJZ8R(&i<<{9_xN=_a5s@4aj5*9;i+Ki zzu{@T;tO_JUEe=NKV5SVLQ@p#4eWPbXphM|3Z1bn7Q{R9W%b%I&uYKmaanEeyebp^ z?g?ckM!XEqh($FPFfX(n&vRzMo~-21u~*5RsuP3@lj6)?BKI&&?-urI)9aj@BY1c~ zQd5_l=Gb<7Ud8kb>CMIVO~WUuZm)qMhQx}emy25Hc%A3iUsnI)PSf=K*YGol?PUYj zLWh!fwcMYqJzM`T31P}}e}{Wq`GlDOeVzX@FuBc@cMwuqSxnROSq4Sr!fgJQX>Vc5 zwRw?1Kr!G(Y|BCUieAWh1BGrSw>=0X(tF8xEG*0_%7?aDL@5*M2~p z^isVjD@l$jxGxRIOeGCj42($X3C1YL#*#YI1wNGY?Vjppp@w8V9sEWjS*`=yhVS7i z$MNmTcfxR`0?%p};db#Iara+8Jg|Q&hN^a&3PR@qUj5Wdh<==GbMn_$sFtVl8jsit zOKwwc(U>{b=c56O*I1%jZTyAYrFP27^9W9BH`xV(tF5{1Crd&oOqSrKR%D?sEo*>{ zJubX9GwdW|ekYPJjqHQfOiTan@WFciF#=Z1_O=tpMz`ut2?dtD)@Vum63cw^q{xNn z_p{d=E?L{`YRw_n;#@h&&KM~SWVA z**cZ5k4u8aNPo+kmSI;pj8_qzdOO1k(4s`K-oJCKeZ14s)!pd|6oj$eQc_YBIaf`( zI&j$xbo~Bl@4&wrt~UH;{$r+1{?`y!%6N*;IqP$;8|H>Cj96Ox2Xk5gxcKz`+IoHr+Hq|j^4>_mb?SDc=N$>wZpHI64qd2)#Go*V3GJz?V#Ev zk?(*Y{zTJncw+6{Ek;=iNI2>M^gpsH=K3v`M)E6UnaHU(c!8#MpxUN6Q&Yn~ffZDM z`e2Qei~8_c8ID9RS6BQvElkx^fI-?xJ2}3&L6BTgW5M8(Kc2Wx46kf6_?0b z3bRUtrW7r$XP_|94 zWC;b7LtbCRbKmWN{C4YACyaj7L(Rc%V?WU5rXa_P*tB47dR!XYkZ#6X6dQDi2KIf( zH5QYdkhsccH9Bl_|8@ICfmQ*@ZMFAqsgjBHy0>aEO%|jIZ&V~aZ(QZ05_FPHth;MX zwX+mK*$@H)-2i=XKSO@_W_`3eerXDO1yVw_h&pVrk-fF{%FTY|!ejNWQXO1ue{-rN zA5rFX+WSst#5=?YxFm2F$q%KbtY8qvX)!T7@tdkdM?5kh#mFf7vJP^OLNc8UKK{b6muZS1qSD2QUTL#* z@MM&(VFCP^!2a~EdxCz~Gh!b=qDOV6$XoXpYGvHl25h>|AV zm}&mZ#0;=cikrz)iWIVD@^X!Sy5btqUv=yN8|DSDGLPPe{0PC*7Qfb*UtI7Ca!;Bx z>m1{;KXM&n+Uzju34J=)YnH8(2)P6*}Omhoo1<~4-)=qf$RxJfcDp(!9K%lFAqL1 zt-(M36}$qUIGEVzKP?(t{u=xS89sgsVM|-vHgm{)+R`SC!sN}8PaPwc4NO=%14c{* zWyNEuOEX)OtF_QiZnsR-#POi8x-`j4fU$m%{5kFucUhrcE?KIpa`wtwsvqQg4GGXA z*x&J;6&>5Ph(3vY4d6mLw!v6j+OOprV5<3fgCQ!Dc1Gai-7{G8@mh`sQbRd|>m|Af zuVJy^EJp06lBB#FTLdDsgVb3jNC|J>?PVDXbx0o|-o|X{{4CBd6&ngVe@vkuYwxpP z+#0JRbG|_oIM?}p)#YT+7>H*3X$bdE(X z8JmV5yQ=4AzV%t;(pukZc(Y%t(ZBtS&(d4@_D1kcx9(Fz*l*DTW4JqmfTC|Hnc_XQ z%b+w7RX~XIa7$jvJ8@@9-*`$77lDwhrc)!=a0o%wNs{#pL&bPFSa|2>(dLz^SXxNg z5!dZ8MdH9y_iwKQ=BR!4-)KS}IC@j59_}(ThmGHCE|RB%(xD-kQQjf&GWmr@4vx|1 z8`kuB4g1m16RdA{!QSp4{|1J+9vu!)ZX|z&YD$J|5f99jgEod zNbx=>TvQPfX(T3Z8aaEKT&mm=7qafVL}oTF4&}*&5E)Tjl49}6Z6!}Y#0(c>$y8tp zz<01xMjaeO?~d~e7dr#jy;F6YsEzv9)bLjl0;_V^-}QpdQFZl_q4k2{0%lRynDskH zTznA0h9Fd{`mL}F?2P7mD~(JiZ0yM0Xg1;_NbI=wd4XK`3Mjssl&;fuMcmzK%oSuA zVffv$tluc9p;LGdJ5&uC*WDeqX{Qi7zueBhvFaVY=ku)PT=`b!g1`w%JblPiIJ-u@ zACGs({FFuWB?bfXvE><6%4su%%D7#jy{2>MkXSpN#OO(VsJY(}Ln@Egh>pLFe9(v_s^Wi5cbcEk_|cHQiUqb$}BNLRGMIuIM(T>4o%RN`XCj3*BWps(AT+q~cLkPG2D z8F#-vKN+v~8cFTm*p^UdI2m-0f>ylA4YRb+(WOpPidCIF0U{f{(1ypf0cn$Cd*D!5 zIXgLBi#TLL4x0tHQwk!qP?2NBZ~|{Qb|Ugx<{Lep)cb-PLY-vf!kr<#?Wdn0rNy5Q z9b{EC<@%*aS6T*F8d&X~uFwRY9{I)v7zCUTQX*$t;M*{ESVllb|4cypmS)8sN>6Wd zn1@($*;PuM>&CR8JBHWmj)#fQIz^6$b9=Wxbb5@#q&G~$z?y=krS}6Hdo`s^K6X8a zOG1E)46OIWeP_Ua2g%Qmm3Pm`}I5=XwwyDD>nRADP%+lEw?Q=rIy9 z5g=jDQXrH!k=q5*L*ouwf9OdL$ue0Mm{Wev2^9+yQ;|}UmTCSoErSdj5%!?=)b4 zJflybC>GT5k}%WnlVAC1KAH4Im@9vr(DhWE4hMP?4c7Gsnv!pY|A@ZlZBPETT_X9CAEKC z1-4c*!9AXR1+#(+3*C3d{K$wX>!g13;6qQYb=r+Z)pLS$#BK9u0|7`* ztD>KKw7X(~uS~^=VJI39)k%#!4hj!p^qabgos#rJCdS>$^H^?sMtx{l%P!d+9#`iD zzICjkovQO|fLf}O0Udot932NX+Pto9^*7h&=!03|vpfrTyH3gGg186nZLxM0KauEX zv+@>H#>6r7Wu$1lTGOMYX5RlqP=hSs(g7zRcK_s;nQ4uIiP_|F!@|}V{rJ%3DzI_d z_E26~QCnYs&lWlJ_mL1l{b#c6UkmSVA0~RmEja6S`+218E6>xe@wJE5n=ub$$nZ*w z%i_w4;QxXIqv7{R-6r!uOS|KR<`W#jm4Qvx@2)N{sGbZ4`oiK=@@<%h3XSxVMuvE6 zYYa4C|I@V8Rn=kZ-qhrCZE%>x)T9q+v*+t`_%eX-5aiW(1jSeccS#c0QveaHg3V5z z9EW?-`y5->X$B#y3|FC7Por0M$q|&0E}R$c6}1tiLA%t3wup#`gu&dBdnKL z*V1VdcrvBgyn0;cKjKLXk`sG4awp*qcYii#sm{_lrQvR+EX`5Z=i%=G;_^1 zF307DBbp&7bBN*wq7L}S+`L)*_2&k!i_jiB7Lm3-8lanm_{E`^C&OJDLq4a?Y>u+&x!kF;BRsxVXp3xK9qOD{NF8BXZjzI&cH+dUuc&Z)?6W;(jwAT|{ zB8!2(y(kKwU^@`+YD8*x33swmkTfv%@TO5I@Bwv}7v7X2?|%Yjm}CLpcWEP9tsMFq z=3Vq#NPHz9AO^2deJ;EJL3+R88v1|!ZYcBn9AkIkSwx(#*vC)|JZT)CrpC*nt}Kly zG?!{FkM)_!AhqT3UFUHmd2G}v;^W!e{Ly>=iyDf5h|gdh@pWm)29MDcH|oZgZK*@D z{EH_~UDc|;^h0WCX^%)U2a%!9ky-ZpF%?5W*PXn3VB*}!`v~1ZAY<#y*>0# zmzxa#zts}Eco0GV=T~^AD=qz=LPd-Y?rnM`CUT+q*6(V?&&IZ(pnRaQv3687`Ns7z z6fOT&|D#azCWyxHAj|UFK*Xf|`<>8Vco&kiH|a z)FH%>=|(F?BONDRZFdXrJVkf@x@Y$yv$dTY!eL=gQr~ zPlYK?j9C}Vm(;^k5XzorqM$03)|@R91~@T@ahSR*v;KK=j6h7P6{}o+vu_Q!okdNS zB+Z^etjXgtlhth`fRA23XIXERvdiCBM$cFK2|anh$uWPEc%9PkSNy!3o<)%aoJFd% zYM345;)gC&K7CW-V3Gj`;t>JA`%B7a`3JS`=k2ET6{Mzs^6U;-Or7Y=xX zLXIdX3z3%0cHuDoSUNZyLW?kes!4FSC+R&ijTnnh&}mFp_hD=au~5%~1c8qII+$w@&%ixv$H-Q;N2rVuAq@>5}pyZxPx@C`-sJqRlwDCRI z%rIznR-KERTX<~augppT8vTFq{vW~QS^hb15@r-djLRx5{Ezkm`HXqg2$=Gv+dthq zIlDL$R+h)?@s^kur^9;J1}e05C%FpCgT4F#zpTitfyP4@KBWUCSnqy+_+4YhtQ(-6 z{95J!^&=P*&&4^{?&H2b5qi8mGLr0rC6=AleSJeebKa&3WCCB3 znQe4^NXUFCbc}dPLX5xD0GuxdzP;84u%3-1`}jhhi+uW@R~p=oDjM!4SIS!N{KX&7 zw7BY>nT9=|^R(F}HsLDq% zUXkx?eAv8Qxu)_`}q4Dth^8EbvCbmtfp<_oOE#ENx6%otzxcF^?@*4;H$FL*EyvRwV6 zX;`xwv$6h3Dsjus8$_*fsd~6u5)ej1hmhQiK!+EKt;6qAD3JFpaGR_|Q(2BVOh&$# zn6@5T%t*^xoV;_2-%OyZ3oi@K+T;55zPQk4^=kNNsfm+Hm;bi~7o(|KPR?D*WJ80~2E-MN0p;87LnMS=dZ zF?33Ca^#d9{6ZyVs~9TBXc%}Y2v+2~*86Jw&Gzg%I4QZKWS1Zat0}_2MduMn3B!K>@?sO1^_ge6Z`J{%= zRL$~)T9s;~eo?Yur-F0fFZ)1~dNE$%{Ng?%`OvALNlsBY-F-efgNQ~v(Wg%?q#i`I zU8+~?hn@;-l>Fcx+%b&Hd0ZZJ;&D46JOuyW9TQO5&^Ac9G`A5mR5D8xWE z8zywzJ0E@>BU+2kRHDoA2fL8l`vKe01Y34)h6~NHRV>739M`zzp`CEwcOUopeROmR z(xllwbdNI+wKnzTxaPb1>MeB9iO%}c;3(i%ZP@kH+ zi@H7a`wW5}`-L|mDSSZZZ4Gye9aqNWYYvY)Rzn-lDGppkHt-_$QD41Sob61*4zg*YSm7$S*dp_+@l+3n^XnkY4iUW;izt zM(TA>T2|tWGNLan3qL37_rs8@Lzi0`#$l(y^*<%Q?)U{Y?wTX;BHKEu8O?O=ZQUe= znVkq)9p4+v-g3WMBG{C1>v_JV4~_>XT$Nj^2`?U=B+qy|)qVUxrsr^f7gN{CMbaJq zHcTDStAAA?a43QnhD`X;Hvv(EK^Ppu91>Mab=;ukz02|DuU?{8a&oEw5_Lfo#XMa4 z_>Rw)ocG-*;HN63%F`Szg*1d^D+vqOD#0c%x+Hxa@P)jEzW8N+D9?_sNn7?Y0wPq} zRwFjjQrf&pKhfrNT6H|!`NN8Q%LF_Ic65m4o`(HTm^pJ3f68~j)n!2PTlA3Y<4RDj zVh}6iff}XGSh+@y?mJF}=K4b*wO@Yy`JyI%WoUj(&Pl4HUfw-QbFYXFWzz(?cS+|1 z>KZ$Ukq@-jY)5rF&9s<^3P=cB4@u}{2L*q;= zF+gKNCDYOe|541AR`7!~(}ovGf=&1GuJUi>e%9-Pqjn*zCPDmzfP9#GiAN8*tL2dy zNlzyYsfqOp%sO>?-NF(8!kKbKx#VG6QjhdttlnC2I#RYSi(cJoqlr#d^USM#pB zCBu)P7sZaQt!UvfknYzlskT3ko;{r_+U1pR_ir?{Mmul%Oobj^71&Y$pYn{1(qpr4frR3=W;%zt`Hkx7CL)0OQp~-EG zhrINWbS4?m9^4*WiL>9niy`&kEa^-O3yadl3^(>X3+8%F*4m{ttL|~Is#*XcH&52? z9qq_=8*ySd+q&kp_1%FnL)OyqMkvp+H8cUGuq&d-CG&zp7mHY z$`Y&S8`W@*#$RUcHsaZD%p%Mpr2V!@tM?e&7_LdZ_mSB)?lA^Qrvw!^efxGiTi$+T zB~IlOwMhE06VK=%LhG2b&$8WyEXs;A;eO#g&LQ;WruCKXD-Ue!d6vgk!eB-Q@P%sr zlns?Rv5ppX^+A-cRD;vk{jMz>rSlc?2p(lmrJ9YO|nGQ^@QE#7+VM?4Z|)OdOW64GorCG^Q*0v!4417b3){J9{M&e@ad zf*1}Ob5Omg4wg)a1ZDP~J*B4zb=TZ3{niS0{CUcOqpa)!O(1BvW_x}8=hduw>l)?o zMYMg--x%JV{{Lz2##DJ9hIt?RmsH zobsr<-8QYqRp19q+MJwNhvoJ^Jv`*+>$E=-`fr@c{o3#5;$VFPUVVtZ6xfY-abJ@w zV&9F7t2ws1&>9nXCE5H-5Wb^V783q3I$`v~XD9`=pj51G5C2-AH~wsWj!tfWwRjf8 z92e>?lxg-6V^h$hWr+288F+khe9;TY+OmB)(Jvezd7c~B+dZt)giQv{L{3pu{^Hg7 zVGE@$aSzwNskDC?OL5g~@_CncDy~JgwM*A>bYxYrN{2{Ttw@&8$NPB;rAm+D+_Z|! zfz;Xc{mI4U7%pBnCsr`CD-vzWe5~8-$JDx$S^v&g$$PY#*!IOdJ#Kaw8CXY%6}#J1 z8_fiU8RA|FnfDU+#nAy1N}A}#g2n>E`Q=3+JoiGTbgmjYlnmY2(uXCp$_&N|y>g~% ze0h^;T7vYEB(GYB3GvF_K7^h`qD*TkXz>p*3zYIs$5URm;J;$M=(vw*Vak3@mluZb z1m*MgP9CYk?IHjYO~k8fJh2cVz?3mTxad#dsnOUM>tBLR@aVS*$Ya=8xxg8!KVdPv zbU@i582TlU)vf_2V@}$sF+iM+e_v>iYlh0#i5n$yrcPgX+ z-a1!~#esP=yBi?$Xwto}u{Xa`n#yyb#&Cw&KJuTeKFFjJ3?j!!um@uUrTT>odE&rQKKG&T2 zXY_|(j?I}%0{3Z!Ii<{=1BBaIErycn&A$$E#Wj*rdF`hVZ`EW5ahD0+J+W?;{(-Hb z`ii$liJ5F?l;3Dppsr#OB2OD2lx{hmqdwZ!;$oi0t+hservEgA?_IrQVtMq5pT7lJ z>aok`oV2RriQea0urrXn*Dn3J*%Zi1!%5fH@E|Q%Bur7ev)9x5U4a&UTrbD!l76{d z)@B@oiqCuIl8{2;`F;u8=Wgz`)mx^afr3(cE~lGST)pp~$j|$}`V(>U3(IM3P0?`Ij@^JFgEA?ROmSYF>%8j|q@x~jyqZ^;<8=;$J zZU5mN2lVUvgv!%44;_z_x;de}xw~>!YPK&(7&VUjlv=xl1Yxogc#?#wVb3fer3k5f zbbomqwYO0r8$H!?YhrygtqmM;3FnP+QZ9$*A{GIO3l2OADYfVkg*>2jj+`@;)UQFS z)fM9P#*sP4>l`%i`m8XkCghrFmHjys%xI(TR#Q$y{be);j~xVCZm-$-6K19^ID-Kz zD<)~Ey2jMwEM=;oy8B>8>TmAt1|MYM{R$GE6Jb~wCYwDfND zLfHGH_QbE)LcnNSKt(6z1sd2kk}yusxK8@?T6dOu^P`C$(7Kgt=mQ609sAz@L)Tk| z#kDruq6q|&V4=~*-67CGa1D)10t9y&cXvo|YqW7qaCdiicXvr}w*a}l_FikBwa>ZV zkNH3YlLwjgj;c|kYOKvGtkYD(uvaj^0XFr+zKE}Z7{Ty~qSc_8eGpM9(FiN^#38TT z2j1|aptqU`z9Qt9Z(f~n_}vk|dG+_Jf4#9IJFkrLKC;2zw7zdU{oU5A6YyU*{Bs}b z8qsS>07P~%A8V>4)J}3H(!O8qC8SPw_Y zi9T-t^}UUJg8(we0r@>vX^wpMTeTp_^VSe8tCh{=o|e{?^^jOU2D@ubxU>VZzjF%* zxbpMPP~h<4V>zpLyzp<~{Y7nmoAQ2NDoOXB-Jd44ght|z(>_wg@^or;?()JmG3Uv? zzt3VoZ!u>F?rAr8MJngBV%Jj%J#41LhU$1L4+}lA=;E_ z5@2d@ET-&7u>`5weq{VbgZQ`J?A#`C*9_XCg!y1q^AE%kJrJluujwp5)7|J$v@^op z_1x@xTb9-NVXs9j%iTt<@52To>Lt|b(#2OhI6Ygl*?iQFh+I|NlMSNt%-D@@>}Drr z&euc8RScjE(<$j6Le%6=0E(FYJfdkIXMN;zqTReuz{b8%;DnN&YH98tonMW0uv#V6 z!|UDp13s1E0JB;4Rsy6_<(%8OO-?K#U-h!cM}_|qH)!u??4qg#DLID^0E%1C=fupIwAr)RyFZvj&GUh z@M`-F$5z#|0TiA8u;tu8RvSxH1Ud4^DHOzFf6% z|NZoGncrJ;G_LVJ@K>eZi>sonXdU41sma&{yPN|>)liI>b=$2cseWOq!}-Ox@)oyHTO9_Tr@T&P)+uNpr6N~| z#xOhAXvj?r?Qj%tQ}8-9cOvgs>483LJp6YsPQ1kxxrQnWb)8`QIT5<9F(Ol7Ljcvg zx7+7HTS_b9>@^VK5!=cN_40~=fb`D=B=$H*)3vSH#OE;Ob@#v7oC~OaLbLEdZ2XX}B zPy&^~#YQ?Z8u$#?ingX$1pv8|O5PB{85AK!r=#c|Ds)i@11wFRfeBSfX1LZZGE-On z*HcQ-SC7xaunTQH%f?cvfGXiiwc9x;V}ZE%q||56+f&zwK}V|uUq}6u*?qa>v?H3X zoq3XGm4+)I<>rCtNDX z?4Cbv>v9cGG3xmg1A?}Oj}}bD#l@9$cx_gvY1_WNOit;m>|MQ&tzQZZX zU?yTe%A{F$j~R%N23Pd*$zf>&{53ixXr;zal})f&g1agDBt}NZrw>LH8|oJ|WC0Zq zNPuv5leBheC1^-2ZKVz)woJNVHj>9^6sU?N2sZ!gQyco0SznaCHIdmyI9?kNAt>w2 zIE;=&b1>N4E=18MWpanpf6#KrCZfeHsd0_+)1Qbaw)N!wt6%j%-cJGf1L$M@XzZJh z5`FihX(wJ$b7>SZTnh_?js^{)!=uwukgYp%>81dtpc>Dv&e!<7P7Z-aeFp=G9CMW(X#LBM{9n5jz1m?yb31ToH`{;-yD*oMI(fm7aH1>YjI>iE;dtRK0&Qm!QL4rrgGjU zXMAJ4dftXL9%1--#=n1y&X0hRR@KbUmPbm&&>xg1V}6r+#;oM?YqrZW z8AIobMNzs;Wzn^dA`MtAw#Id~wB#iJL1qH)3yxm+xa+caS!}v|45aF<5)LNKH1$u< zP)$TtGzCgbv1RMaiJvrhPlQbO?g}4-yY8<5b;k&e<{RD$JE+iMkVJ*F{}_Dz`@zJK zlQfZhY>jyJDkOUrvVB1i^yJ(!tazHY-G3B;^^(e%)a|68&zu75(@sCL|8t1{cLMa^uU1SS z7owhknZH-|Ph3D{gZsjN+%@Yde*Ldt=onSR;q`wO?^|A{CH+HV;ua_%>{|=4ZuXs} z%?h?JlZ`q3jDwv;-@4q3v4VV)FTlS)aG55p%ErAe@r@db7U{dDNBle%L_E(VT|Y!F zl{zK+jHPOeef#LcmI?MS@fdyxH#kq0-#u-hj=-e6lEWva>-HB{W`_lSrt6tTWLUSS z*1d_JM>r_FW(zq~u!CB-pXY0MX(zW-ER8CLK*v;J*_K@H&KFjLqTJ$-I?aoJ)p{{l6 zf?vZ_i{+cHFe}3w?KTDW*S&CiFM8JR+J9a*wOq{W8G0VyhQ=nf-`5Lw`mUi!6IRzI zvy zjbQ7s(OonYrc)#kTA#JP_~Ma$l| zuORUbMxZeRI&7<$?MY8H#!{^5R^SX+AFBhTR&w#fAT#$7!IaSULgJF)J@q>_P7)m1 z90IrCEMRB|p>>nJzeKTYF~4N-@{dJ$Hv7J+(jzGS8WkFxTHvTQE50h0{&6$Di)mY% zU^jz@!7MsP>4kG*VSp>;7e|@5d|j4kZz#j0cq8enyUvb!@J=x-1GDV@8e`oZVk~t@ zwXr>^BW_-`sjjK%f$QchEe}s+yn-&=8>Y!B+UgXrG*XVozD%r(J^gN7Qbqjl=n(37 zb5K=>zV+7R0Ad`-5~xSz2duJ;Hx0UT0!JcDn-{=XBYH!ZGcL8(v|h3>OM>P1;- zzgA1-H5+R=N!KnS?INEe3w_H=E&^+t%2mRN13|g$Mz?xa3Om#p%0fA8uxwqus~IN| zC*kgVPiN2lP)uC??3&Tu>udd=Q|1C6DQwwB)_RtZ)35Ba3c`!J`J*D5IJYI)XpNJ+ zgB4u}$G?>c$xIV>L@2Xiw}gLY!M1AkbL*5!>~X1J|73)hgtJe-!UrLlup-`?n9L^T zaeB4Cj3X}5Pguw0_^dQ`aXmlI!S{!6)u_W)jD+-I0x`{PhXO7+dH98TbcMUmya--w zXc-NhCMwZ0W9M}W0lS{{rrsH*_SBfsI&u;MN>7p+yZI-){X0MN)Ubb278+U7XmB`Z z>3e!vY@2R;3|C6yYOZpHzY`eh>L%-!9huuWw6ZS4?nU3jOXP9dApiQ*E-CP*i~Afn zS7vo_b?bxw`V;YY@1KfQgD8BObY$oT0-dIQYNFyroO(Yhcp>~9;$j}Zh>u-@15YTH zMrpu?LH_7mb8WqI53)uu+Im@=7C`mb2Iqo$ISmgYA5le-C$U~~S$oq+?L@iC`|_Fl z7A$})Hr3l`N=`-TXvR%dnm(*;K{xp%GgyRIq=1(Gn6}AUn|L?cXC(oU?ueBrTT#d* z@DzZy_Qv9sGm^etp4?l!*K4RaVR-t6;)cE8um~JO@(UF#$7EHOg^QGytOLc-qGFa_ zh0c>aiQ;VLq-EUnNtv*Wrz{1NHRmai)mPM#Y(SzIm;DNJne64<5A`C={GY0) zA=!fCHxEb>{D(%{Bw>`2m_146L~6LuZy)67UWP!Y*do)L>+i~8qYV2$6&=x;4~3ac`#t;PJ~!0@ z_d77xX5hc!sij=;dPgJE=MN?9I8nOqtMB-jgF9z`Vd?1}e`=JY3na4TYeu+zU+8X} zLUyNiQzS_`rJiu_%?`@;Sg@HhMl5w{v$wRbYlqKUD^JhV^yhE>!5G*CKm{3ToJME# zxqF0t7IUWnlh_IlpHO(4Zmy?>C<R(t_JyP=Xbsj=ER}8Y; zDxs6Bqc0>>>{BQhfgF-d$FY6E1g7}}d$tv|mxT6C$r28JoO~bi8~p^&*{Iejg-+C) zD!S(mikZbt-FBk4C$Z_&wRr{)4w5D-^gs4J{>YX?wpJZ)9dmttDCSII@E`3DUB%UX z+zYW<*c7A7*x7i5p!KEh@b4$-h5Z}|AmRVkGa-d`ubnE9EgBQ0A?S*tV!YWQijjUt z{Z8N+Cs;wKZ|o*ytn1~{syNr&`?I;jb;`kdyZfZk2qC?Pug`I{!$fHhW2s)_>km9p zUUvq2cpl)@Y%{uk6Np>_#DT^EFCU=5qKX6gfy)ZZLD!UiZK*^HgC38DYxDitFDZQ{ zW6#XK&{=tVelKzKPGNQS%^~#;3WHc0?%8}LORc6}*bkM@RWb|_hM0yWHKLm8wssPq zv0A7LB0tZi=#!Y?M1q|il zP|@CmUn|L3jafV;^k9@iO7mm{Yb=IdBdyS(yf&WF2k~@=((eEu5=)*bE=ZYz@U-(w zP@yY1UM=#6gyz&U9SVds$ORg_WVfhQe|3BFd(Hd+bH^M~I@KdpZv^@s`kzv{;J*UM>!vz$7gg`BYxOSdL zWsHqD-H`K+C;h(PtR5e&reSAi==NJ3M^BC&Xgb~(RD>W#Snx@H$pqR}uk znD!8~w)q9~*QA_~92S|OQkrpE>)_Z=q`pbs79nWFJBOVCgDf$ov_~+}w$Naeci`?F z`*DN+1I<*!_|1zc_x_bl*bWS^ctbPRa)Yq`e93w_M^qjiHK)<)75N&2%*6eKtpcI+$P)Vq3T!XYmjP=kL64S zv%m*fG-XN*49qn zR?S=iwb1^!uIk(BB_sSI%nP z62!T_MiZs%?ZKFSCGCBWtI_kuu!!0W6bCqLviz>6(G~ES%{?+%4s-vEtF0ICt2HMhr1J&d%$IE#tGDYbDaOqekY0;8rzX`bd^{L5 zmPcl~EEJNh$a}=w$Nxpp5Y4_?o>ku5$-~LR5nizk!mDwZq*XG_+UH z893Uly{1YkSTm+{%X;R2c_g=HwBQqVvSP1U8rbYz=<8iD*R@>c1!qft-Uc(5Gfz_Y zBKq4W6I+564N_itR1c}LWH%vX@#HRAQSr2VGV7zmv`3>|&l7&;k_d2l0NT&=HKJXU z&L#~oPut|EFT?4IF>=p{7262~vTOiZ|3r^_a8W9+ZMvH!+Z1ITdWku9G7@dC)=n?u za<}c~wLH7`w}GCkEYj8+N;;3=pVDA1i`Y+^CCuFJQ+dQ>9$6@38Po4JGQ^T0xSyk#h!^Q0rk1G@Oo@$E9zoZ$ z^91PYIg3AA)w>wtPjEDWg^;+?Ir_in+=i=NA`p=ZD*g z>w!w5T>5BpJ?b!h8i^cVkUW^kP2R4D92AXd5JJ!_yJ%cS4hMkd|M`$Towyed1d=b9CT2*25CtcE`14Txpd19Q9c42Wa*RikfyqA zsaI5zhx!g=t71YhE2U;tzdUi`YR*zkt6N1`_*O6-6tSGydqWOp_M53KRJk=T5creh zoYmiNmgOrd)7U^nD0`?w2zz>ry%88c0z_Z#s1DH3a{J8#8ucVskx{!Zfq#&Rekofm zxptPo|Rfva0KbTwF6$Z9_M2kQen4xz|2c3!!u5 z0Yv$cBlt^F{IxG+(P|Xq+scH+dl2zHy(08a*b=h%oG$H_;P5+HZ;0wBHCY6+M8pWm zGN&mb%#4?dUz*$kn|ohsI>`n*lGMuX=LUSwYrwf#z%4@`(B`#0F}zG0xw*l}o(##N z$BfYuYP-#dxOQHWma{bwDPDaUWAT`>c;+GR>hIf^l=>-Q86V3sg8o+IaYe3irF*0K(R? zIW>c&rp8v=u0(oE)cFS4Ma({=_C(P7zjfO$^saE-Hkmc3c1KoR%B^<@G5n1p4(d!o z9=)OtVIR@`A?!}jGp!&DCU*X;(lV}C3$Byozh|pFhJTKVRh8f;x4loegco#xnKEZr z=wP9CR6v)bhY0FqPAxd~6Lb5K8McL%ZI!yYqf&-8Pq3Gtbj8OkPLQsOup}a7#$^~u zEXT7UplgW2yh=xwi@nQP(k%f%TsPT`Io;ZkY!6@af{nSs{LiCReL}hQS_d+1T4jo7 z4Fm-P1Rv?8>%-;d%=0qV5M`N1xq)b?b+5ha#@asmScRHwQ^Zo!@Kqs=Zd7Kj` zU#g61k|U#3G4HRW;`83_iMO36vQ{>5_Fr_0WAdaLd4fJaJ~Yb5#CHmR2H=%XXs8_Y17NOCJrqP8 zLQVfxCIE|qkOKVKn*iG&LNx8=C_!V=<@3v+nq)J7T*pW*%kPjlU`)V3l9K40(wM5s zgi;i4ywVCD3mpoZ%0%5akRQJPH2c_+r+76x(xO)M08Blkgff)8Qs1z%2am`%G~j@` zAWo^VK&e*%k^GKbG*t$z-mh=3tZ{6478hD##QT4iTXI;~&Y&t!V zoq-66y9-~8wrSKQ7SghXW~3+@NyQNz&f5NkA?;f#`J($?csOSjN6pk_3Ef{%*`=Pq zz$xnF=gfOR+^1=^@?8)gJ0E>Y0%npgTUFcQu?Nl(iZH z&DE&Ud_J8_0HC0eibeJ(SnCZR&K!8`(uzAWq|%Ks1+psdp<{UYc8jdgT3`Qk+5hR> z45pQ&^X_|+ubWD0ymy!Arhxa7n3-4FLY4W_F=S_#>P;`leu0Cm8!vj(#@pf zW0I*F--xmw6WYb^R;~4f|C=n?KvclYM0AaJY4)xU$831rm~QLyH>L7_=*NwxG1w;| zxC(JF^Q`cb_d!|OpGDKmUARTwS7)^`l7A!U)0nfds7jdO$M*Iiw|#!_wlLkOZmw{c zJLC|gyXNM{?Lul~IVJY3#{{gTFJ2Td#pwM-P`4IqH%`%{v7)x4H=T5!8_)gVr+9{d zS18rxI;BfKfkMXj^}F|cmr)Zotoqhobj90sn`znfD73J*@S`Y;zM?NWhrv7rOB^t_ zw4CtyUU$1SUlrpa2RzFt^ke#*b+N#N3DIp}l`4xtHEz+4vZyFWy$Rz8C2lCzeUKmfDv)0IJE<;Z0(sy7hM&nh1CNiQ_zCstM15yaC#_dY3o2Svy7ch z7v;{R?Fm7|mC*4ly{IEk0cu#AsygxoSKKaP&iY9sLykE3f1x(+SEoe zB#I{SYLP9T@25sYTWSe?61w$p?7hjjQ4 zTNuPVR%ic#gpyP-N=fi_g=g%K$<`+WkGNz4b|{1G@Xdm~0abTCT)e`iz^0fw_Q*br zdP$rTo@@#10XHx+()p;jcG&TJpZ;t8Btw|rPtA|#ESTa6_xJAbrWpZ7FtmT$l6IsW z8@?p1p{h=ze)`AiF}rmv?!3U2Q?jn5!9To>AZ^IFZ}X7TNMjOiSe97*7n=fu}Of$U`*6$+Lzt38%PF^DapU5XaYtxq)N$vIQ*RSX^ z+s8p`6OC;hp+0o<*?&`RdfSe%?ddO&HvX=#o;P>A~mcD5L9!vM-9- z!Jd@U;$WU!>jUF1AGw^Flojx>i^v_Dr(Z#Gq5Z`EOF*Id_QmL2S95H{3H6%i-28l_ z(=V*zMAquc#z8xt9$rZ!mIxD=LcUn}JjBK(a(CAl4-d~KwdC5SF(GX)&D7G9C_h1U zo_&mbScfKR4r}E|w85FRc&XeDA=YpTPZ9LdJ6WHpPMXRG2pA^!On%-mj(fY>hYdD) zt9H<5{QD9Qf@elcMp@+uz2r{b!5>^j9^K)AgLzSCBx+a=uqTADo@rn`H)FuY|oxnGFgNj0CRme3#XjdlO1$A36fTshywATuA7#PRRpzn0fGf-hbtCeonzG2v$v^E+0d{L*Nl%y zja64wTQGPRcQ>PXMY<9kNaP-sU~`gfg;N~#ZS>t42C(+g^RD3(Jp4MJ_3YW*&{0V# z!jeZb6h$pLoF;6Q(6*YA8Lvq8igW%Acp#?D4>g9KySwY*K*2esq^R~X)w2esj{eVk zwArF$iQI#qQHXpicYMD74k~(4(C3!hLYqG5r@A=(^@}CCX7RG)wLb;B-)#l&!2OMT ze=cYCuO+916y_0UjbzJ7OBzlpmVP2YR!*l369rME`_mEc!!n5~2C<2wqyO%1yjH{f zeBQKqbyd|!UmvYT!e%QgT}VUkJk`As`~!I5sQmDpDlKPrL8#0`PE(@-2Ur-1@OsdB&r~4>Ac~-Z26!k z$DJozNdNjrI@3l@?Twi^Iyd7>(AaepXkZz5p5yY-${|3Lx+Jh-HA(r@K*=pl8QcMs zG?ab$`NQ6nOYpTQtiH~prLnF9Ci(ti?k0C!goAaQyg* z+|*IT*_xxVb9}YwxWmbc#(V9Cfu(q+qpROW8p%lvls)!x*^}Q_Z8+3&rAdW-pxAVz z`o_k)WwavmTc;&6ABZDErf5-7FQLR%CvjM75n;SU&tBb^mE3{#60fugh-K5>>xaGN zTSDl#_Z#3er=hE)%@cMHRlMOVB1rGsrk*sIcvVii?}&y6;p5zbfn72NoCx;!fyTD6 z&GLljiiCr2;eI@@jaISQ(YVP(_AEpxv%X+;i}L86;Es`LnKh9oyzED;pImbpG@8^`5R3VSeE48bwzz@;8q6)yO~{Z!W9R&yA{@R^NUo2Qh~ZM!S_0-= zRU#=wdVYqeL&~$1htI@H@~iTG4_WI;DD{cLyj-7sWI=t|!19`%k3j!Q)9~WRJF@s; zo_$0O2Z94(9ORvo@xn`C`LU=Jn-gb#n6t|-&BAz^f1{9p=v)pSfU5JP!!{WgKi@vS zH=!la9O}`Q1fe*NG{0PmdW6{vyct1MK~ZZ^>alJ+rd1JN0uE8PPXz%O|Mm! zLoROc`i4)V6z+C5=LdkZhXQ>nA2~O2tn!@auMpPW95^$&YQ2Jf&nrVsN10gjPtawc zy#>ZpWooHOkNbV_ix4T%r}U2)vhLx{k_w>=FGQ?4JkMbtznz=I*>}jrYrCb}?fY>cXJA%PG zV$@z-!h0=F*rgRUnenjDP!duaR|Q{ZLyMeAT>`!JR48_=Nw`F@{8vc&^9?P$SS_&w z>eZok>Q+zT`rm@qJ{7fno;MeCNfS*@f;796|?C>D- znI_7EX5K8-$0<4YE8p%CYpd#NpX8cVxG(y&6<6hrrtw!(AS5X@6=G~0I%zhb&qiYA7A6!T$NyK}{omkz>i2MZ{O6AB4+KtgU1y~4 zCw{$8yvP5urPpHG|L%W8VZI_8Porf=#+0b^;xQmXU4xoy1d}^8>c!$CDAn462Q-nJ zbpPtc-^$pr0HrV;GudyIS+eoq=Xah{cy^M-2J{_B@WctE3nLj-SNtG{GQ4bprZ76WUPZ-v?Nw!FG`H=l8kOvf{J|FUK( zZ+T=dR#PR2=x^xKsXaF1H1zarj-!a6h=gE}3Jk#4a(4C$J=;f;3VLV-lxi8M?h!`y zLuq-BZ4xJ6J&=0QOscD2nDBh-gFNZLoY9PY+zls(@-+%$OKGtax4|B zQqyuD4gKE9uz-OPfOyoT4Krjxzu-`=N`>~`z3LIGGMz3mA@rDCqRB0j919X+mrXJd zD1`zAxxr$!C=POot+f)~F)dOv<$mP&ejGG1PDVYK^co|VxrSvlo-0b&C4ebWV9a@8 z5;;a4=hN3qc35q7&4oOcJ=AtV*k$x`st^YXC}HU}g~-0uG~sF|s=BOA)U)h;zM4@o zF3tT}NU8Nl$Puam_@mh$$k(~3kNK+l(?iQzMlCgZ5#BzUF1iBxm?#HULetyV2^eha zLLb+7!7F^CU$=4lmTk8#?t%G8g#AD!ux>?LTw!mVQc3xaY4Y2$8rH_94!w?5m;Lf<;RYqLYbxuhr=Ayj;X#JiyVexSY$TI-K(wrJpg;&aJb`ufcKHbo2!*G#>WfBtJ-0hRkH$&$=qtIwgyL z2Z+&qApS{6st)B5GEK6SYHT0#IzB;P^LJIZMj^l&+Im-&qKcMvy3z{zX7@w2PxPnYm( z{n)Staw2|yf8I~F-~aWo|I`KaudMuUp@SX1+rzJXxDQlY*ZAYe_*pNj&v_l`?_|@A zP5iIt_@~3|`~Tz*f8A`y`orRLB|Y!*aw71WyK=i6hS5H#&%WvPhrZlOps#QgrcJ;K z`wgVZ=rK}vtew!J`X*^V7Gp}l6Abzy-G_0Viy7f~_ol>`6IuBUOknGpDO{VN#vNkq zOZQH1>%3cyhvzmiM0k|^YntL=cFmr`lm!bp50t)W(wTbTHY1Ck$8itsFE2LtO|A8; z_0#RAS|yj;tfF`v-IMm(+Fm=J9b2#Q+p`YDcbMX?d)u}1GVU2h1k3(!6_+TKfu^J} zt22}J%EdQkHdVIb*Q-FdK`RT+^8xibU+p|Y^m|N#aJ2W7HcFu-VGkiaq}^P~$oH;5 zV^jK|uhQlBuSRzaO1Mmo)Ow3MqcNAeol;1q?%1-A591{GLca7M-8AHyS=?8!3vdz zQdJtSdJu=b#*WDY&Gqrvp7}`Cn`>yQ=OxydNh|D7SW^pJ{RGVgUzXVB^f(S<_NrTk zJ_clvjW8QYTgM=8 z<#XVH+jkS%WA$MI6vo{6ER@yw6{1_8NHBcf@7Ons$u4_=6S{qHhNR4)Bgb$!_iOU^68q zZP{qECex`DQaSkqUAW37*3}`Vo;^lN8)?gGV=@{2S>Rme$Wp$w6C*(L;&#xJTs_-I zI^4(+4FzT}@rj0q{0{>rDqU|Ho_V}y=nI|kFZ$mGrm!ed;aZkHaM#S@;|;V{}6aH@pl4WpZXsf z#4Y3M8XB8_Ey24F&Afa2eBJ&$FYxE_^7O#%uSW5{&&mGXB-xXqX6c+Yqw)XL_3#sg zOpfvpG)GP=L1!CdotBl}5^93a(=Iq}nfmGMU=Gsqfv1c7@Ed*TV2HfWYTW+QOqQw@ zi~SdyLv5BVf;6>04N*cZ!{P;XJKe!F3VMyZ$Q_l7QziU87v&o%Q}!|K5%q4XzM zi%7JKsd6hceK3xlUcQJsdUL0^f%jB7Jf||gl}%m{oMH^|$*!zeed;a=)Sf1=)w^i? z!yOy{F5AtOZ_+|9TA>ekZ%M3XXt=DW=sT0xc^4Yr>>H(g(eaUIgF8U#%fVXMtW1$C zuwoU$5hc|^_|dIqI>CTrRoHg08qdrK!MHq+<1@?q5?j0N$79c^r;7`l%*o=p&KdJH zc?J{L`=e9O*o<4=xaKeH)dx`Anve`43s7*hFW~m4hYf|z2Ogn>Zwqk zss*D)Y)CI+uAF%`ZQV4r&B*i-DP+Ynq_fWw&21`dXHZA|X1h7|*|v?C*;+5L!0;vo5zD*Vit*YR(3Q06_4z0(t5 zGLZQPSHMtpijJ1vAzh^(@QYy@AMg}2Vm^8N{j?S%58FJX`$nCF3kV&y3;_e23+Vd@ zt){+{^ixORmAVAGStGV`iYy=qcP72ni^$hwjbxqB* zJRwsJ-N@y7H#-$$M@yIq)}&3nd7Z`i{jsiE{JQ^(zgcg)eR%5r@6h|FD)0Cmm!IjA z^5?1j5BGn0q@Tjyy$oda+9Wytv*I{@9`!sayamgAG{VgIH{srZUcjea8PNn7BUo`u z_xk-|Zp+V+KkDpaX*+GrJ$iId;6)(`B%;}Tvr(c4QxbsZf4QOE_|@o@B{`D-WS_}( zR39C89Qzey0! z(K6s9Jcs+No*gIZE>Q`tGagzVdL5>&XDymx^@D;jNIX83o8BxuJ-_$>Qmh_SUEkEE zVQmYZU4(v5>>@`@)G3F4$3TIQ>o5!UpY!2p?6&M~u*7KOEZ@;r9#4|BciCMK>h{-6 zOG;;nsi$V?4m&BqDJ)D7_01b;$IzOF0eVk=dmmrlukkCTu)cn&EPvp%IA_@_bv>Fq zReHB;5mo6rJSJK2AQt*4!wO!q77Y^@{e?A}7P!h+F?-imNTZ94AyTi5SINK6$x7lxE*rc(SkjU(& zq8H1yhs*B9(=~f)k*;2ub}4MAZGF=G!wJeiXrc7;e3ivNk>>$9qp)nH+%Xx40+3dAXnYIP0 z><;nT%(Eotq%C6Utftm%es03JvBRk}xdBXhRfENur7ZRKHuXyK@h?inc{}bs_Fg9! z9yR%ylsxm@CoNgs&Nh4rTP&cY?V@%BO9%I@80=w?QSnZ|2U&Pp7k7@90**HWvQk2O zLYRzg?12#J6p}4Y8jCBoNsL7{N(Ch_3RDi$e%e#>2Df_o9p_qUb@I)&_hNeqKr_^! z&6<~s%-)M)OMiSyy6Et{c3?!;mO zZx!y&#J9Y>3}277zEHgiXU+%CxBKsV=!81O-U;Fu{jb?88>%(_zHC{vI^AA*uF5x% z`HkyQZ0#iC?TBMoNfaIfCuOYP01hxYrppGQ)Q~JUOmozsU*BjTovsSrfw@EqyaWPf z{BKo?=)wRMeXuaYh*W$=5fM?#^_-TKm6bN+srGBelaA>lSLw4(i(5?j*QfHIPJEn%=y0K@?cud{vCrHb zcXUH&<=brl4w4YV zTf;TU8Gq_8{%G0Cn)|DVwxkB4K#GZRt;w17CO-~atYwsio%Y8fzi761$k-G%)MX5` z=7qa@hjG7wg~)9Xz)JI|(Rh#CpBDe!w&us0Q9l0aeE%)6>n(z_$^Dl;)iP{|XS`o) zyVK{~Z{ew{xfXA9hu^-+3pXVa{ZJuC6dxD5Fqbe_cPqz{k6YuTkj!qZRfzo^36!@q z=I>a5kLkbAAaG)}U7GXI^yr*bW0r1ajmb27QT52j;yGJzz&VTzBUMVOx4NYamt8y2mYn3QoABJNuxQW zrjBq2ul){K_Gur7&-aU)qqMW+zIUm0DU5H(FLK*&xSK<2JD>k5-@Lj-MfS8{oeWgC zUC#OD7%5w9DVJtYIR9NZz)DGf^h3^6GG#l}_#yH|4TAZrOL=3ZH;v0GYh}(T*PnS1 z>%>-Ypt?a`>TeC>AFWcTZ-9BphS~37N6h@&?X-@?HPii4+ucBEdm%)DqIhfhfDevI z1%WYKF*=mP>xQv<)J${ITe$E!WGqi0MAJ==R`L`D>bz=3BQmc>A%c>lZcd?yPvrLX85wZ`1d>t{d2D za{6#9E_@3qq*W~#X5o-z&6M>-Zh3e~H*01igZ{CkG&T$OyzEf}r!;rQV(&Ew*;zA} zWBwW>sO5~pKzGp*$1vT-r?)yF4riIxUWQdNC|bry+|ri@zBXe?%c(shk2q1Bx?wjo z{Eowwz!xonj<+1-XbBlBF@}44LmfKb%o@>QM7_qIG?VBV#L~F3SI`4!@)KQ2%I>;H z_^G?}ceLI%Wq>;f4a&@UXTo{vfR1kO4cm;RVMR7+rsa16;a~dK*JE}Ghi{y7ywG42 zVDt)5y}xR4=g<(!^0F4RA3h&^D&s>M)mKtlI&IBu?(7*7w>?z$h-N$Qcq=o(z2dO9 zPcHh8+>u$-3*%p$?*CsB3a5qLqCkAqyOcsre!oPhG)-g0_zpPE(G3|E(wN!K8L;O& z<_gE}T*-wKKgPNumD$nD7zGZXxNj0bQ6yk~0}|6YQv2QvOcKb-gF*+xMq2}JhKF{# zNjQLgZcpungNp}?D7vT{_l<8;mNpY%K{)*RK+e}mfpnonuh+U$n2ocUhX<;TnhZ@E zNtzk@iW&$&6$+xNNc*^_GLts!$#(EvDCS&5PR)V}LKkn$Q`!#M$A>h!CxKS}jfk(| za1wu=HpSoW_%W{B=uQ=(6&Xqby&$l6{u;C~X+Gq+QFMxOZM5)GA4CoEL*_AR&AFIHG?jJoCw%A0;ad8{gx!`Z)$bj3G{ zLr!+r&CXUKq12)wLrq^(FWz=}aTpdRPFeSIVxgimP8Sk;Gr0~dy;|1ptHp~kLDb60 z0cwtcM3M{W)N(O%Me-39SVU5rzMUd@QZ|-DQV&O_;5OC!R1+!$=ktOgbLZy=i!gSU zxM36VeUIfu|H1@zht?Hf4-iv1B0c`-sK~JaVa7MKLqfl(`FNQ9pbMrm|JG{FEm3F>zkw*^AO%SPecY^)U%eSedYfM83h#9Q3_qz~wP? zp7m2h1Md%EMRt4$xKdN!D+nMe*R?>lxgrs@V5Js%iwr?2N~l>rEucL+(~p?2)=X#y zz>z1CrH}ZhMDAIMX6huz?H?B|DWt?iY{<@N)mm@MlZ~x^h=Gq!MsF|8!_!OuPE0`O zk`@ts&47!`gVnX<6*8h``ujd9-ji@uh7GlG9*65IhHA^T;>f>?va?QVxDew15j_0A z{St<-kbVh}`4V}ggXSn;2NAeiCB~I{h_SZFDWDitfdtILIg%I8L?ZoZn&ozp#WK`HBLP{yBkYA$gGoASx@^r&BRTB>RgXd}3CoJi)Uk75dyJjEKdv^)Ej=+~93^Hp8y7NkjEG*(V#WbIk zf$Q~IKqI>FRt*&smpQFvO)RX(1rUP+P-ZcOFGAheWv~z0Lne*H;F`wKm-*cz^_VcXtU+aEIXT7Cg8^ zu;A|Q5Zv8@2X_eWIt=c5H|KrNcklP(R#C1h1vCma9W_VSnM5|ace3vyF`&><#IVdnhI3sC`qr8wgq)1{dkIkXKU*k z{gyh19n5k(#Zh&Uds~+AdCK+{ZvL~nd`aViH1qtSy&1VqtYteovN9!$0$R7X^Aat1 zS;f0EsiaKQtOhe9X_^%V1!*lgt`5#9zpPUAs8NlAdm)ZvEwQInj#*K9MT^{qsDR`3 z9k@{98v~!{62EN5ILW}8&guvHdNQ68)7+2dOcf3{!=JEV5D+gpOIc!RsI{eks*n72 z#j(Po<2J$<*HO(k8f`+in=Wzf}g;ehN&Grk~4IE*>V5?08QKvdaGi)ln!-AH2nDhX= zh4o>BG3FlbXoE3KzJHS%Y`L8Rb-Q^bK^@=vmktM0$3IK+F?rw1bv1XFnL6HzI!!ID zyw#Hxr$sZ<3fcwgWU;f8na= zrlRv*Us;YcR!a3-Wx`abPL{vJEOlWlz_!TS;lkRU+s4VYJQEP=pm~hGWt(Pf8!U&C zYxzi1Y^RW#On=88Pqq>v;mLM#zkaE9_NhN#EvD~BWVKpo1j#skWM0lTpSvFOO;oJn zK%Q**R{2H*u__fBw4#`qIe)WesQ{bY*UF`g2kD!_&Mq^{+fpVvE!pe3A0e4IA!d=s z-Xlffm4J|yX>(}=oA=j(V=U4Q9SU4^*x2VS`^DrU$iyCY=++bMPT9!87Nv-y)_E;| zOy$NxQdpWI$VzPOLhiQKjolXSZrg1RQN{G;2FZNK5b;y&u^?EBaRM@%D&#`PzxB#; z?PuemWSeL0xra=X+orAO@5fL0h6hZ7M*B6kLuCCnhDEJf|tTQ^m?tsT%nsY>XrNxr1H%BUJcBD0Y)j>wA@@Q`T@+ zR7@J()_*@TN~?s$(JI^Ho^OC74z;)Dj3W$9-BZo#M!vDXv%Yq2Qw-FaZ;$fT9*buLM*+S#AhL@&~n&MTfh znvO4;KGGe)810R5RxMtXB>cR6alUpADH*WW7x;Q~xnMD_Y_+CPCBa{a4-S{UU>qc5 zF>8(X5R_|IdtaXdKd)u-YoyH6oQ0AyY)N#C(B3%fSB+%8_YbpEY>QiFi+y6L$9Ia; zokQ?$p5^E4+s=EqO!lK4;=PrQ7I#o(`67s6oM`5ueTH&=9J>vk48^$il^LSMk%9L= zH@`4#{xuqn4GQdUq==%^Y|r|zp>YOlnzY1Vd`B*y{!z&7MB?i;Ks%n0K>j` zsheef;{U%V5+XEVbOgB+xeygQb&Y8-pHzjg@uQ$Zvsf$gVBz6XNI2$c=`Z} zy1O|7QL}}f*SXlkvp`kn1MT-~Z-!Je` zYcfYHCtLmDI?dnxfOG!lByzlH23RhvKX$Bd?+nF1{YHBFo#l*xfWUV>tNhMu3oZK{ zoU!gkB{Tjf_wC5fMK(Ao0@+qajfiJJTOX|1)Wy)WY(waE)_vQ#0Z?tCE?la>c2}dG z0EOp=Lj>3Njdy??w`=8w-|xqe=ZWJte%!;=A^-C1X-Ag(9uA@J(>Xs#Sijx1Wbp5T zo@ZocW~NnV(+|A$^~D8~Xn_anOnIvIt~W6Io7e1>?NF=g^HCQ_ALOp`s@t^;zW75H z9k_u1FZ})eET1V{R*CK$A!Zt-vmAnWkjk(Wd5LdbQjsKH2lW`yB=%5bzjkCk6ob(6+7+CO zvtbwo!j{o|*m;eUC}!~?od`O_HTS+`;iOWb)87)xVu5O9Vr?_s*a4SWT&P+fN4!4o zZWzUr(b;@r+5fzV-W?vZ z%MG%!vbME#xch@r?(J#cAUn@kNPWt`Ju0+9a>BXzm*!0Mg7f&=v})ISm4;!7P86c! zC#${i=E&7Po~e>F1tBKwuS^L!*)7f*AO8}uuNdeGo2;Vrq5R_0T9kB*%o!O*8M;Ps zRWrb*3?&lEB1MGMnE=Gmj!Ceivx7~r_zDT&($UO8d9NLD86HvB^dnrfQ3SOjMijPY zS}0UT6ElF=4WQBH{!L6CAxn-&QIX0z@}`h^td)DDEM+;Wlm1gFy3JbJgxX=6_D2sr ziYvl@@Fhs4TfZYA-Y}`1Z+@e?b;(fGE z9W(26>gZWLBbGW=m2*}Tcgc~YQv2leisaT2NONho5=v-J%g+D)P%{n*Wr$VJ>zQ&1 zqf3z5!@-Qo<)Oq3``OJwkE`9z;-(xrEMD3gOKa4Y6jVZqeU|t`Z*V{%(i>_OqXd*$ zWGu%ZtnOd$zlD;>pMIJpYi2jdQGR9h#M-9oT~{1B<5R?E+FJQ7@#&aAYv%_kW!muM z(Q3pK;!FaFwp;56q?kE8v$ktWfP=u920xK!+S9=DfzpzcvwuK$ns$*JOh5t@lVE&O z8wQlmm@J%~C6wp#iu%m74&wcsv8#KM=JEGVnR*XUSPw|G+^JNz_87It$mqppxc;ls zh_L&>mNO;3zBKn_WJrD~!yRZP{SXnR4wLWkvl&nRpW_QFSz>#J5-rfP>d5RqwNM_^ zDA?V5(z1c;yFqGZCPA(B$J1vy{_Lv3?_3+hOSr2p6aoq~*~YsHZ3{HRPVKF0SCDqu zFJ2Y=|4!!W{awBAJMzt^ z#p_T;e`~fx`E$o>iE^>6$KzYTlOS{t1XQo$y^f8E5Fjiz8kG=~uP+}m8%;Ngvd^y_ zA493K;B2^&o00C1)VqeufZ!LUzpM{Rb**^L_B`Q!i!1`?p$bqeOvufj%F4>T?nm_Z z3X}D=e4nCU&!g6S`2l8ux=mLX(|G48y!ZLI{lk^TTQoeImSH*v?hRbDPBr0B@#x9_ zcC9z1#pvQJ589P~MkL3ZSBEnhJkDW)ua|;=_|?qmo!!Ifj{DWyq!W;5j6m`}5ZSbQ z{GX`gElo4vCW?q1p2CP8%5R%L7u@@8Z zF}lOWD7s6!ceChkn`u9C)A0e`%5yhCjyo+>ji#0r%7({?;`ANj0|6u2eC}7QL;`L> zB94%__I_a`jwoUzg$Kon2D>9pGq*|`sTizykuYF21UxYlclk1$%fF%AS#EBAkO*>V zp@lOMQbyV^K5B=ZQs|CPU&3LLTvI`m_?oseR)Z-##HvunvNS!k5#q*_@#o87str3T z$rIcXzA9fOrD3>2zYA3LU8WM;K7KtFyVf0{>KLF54GN;%A)JQ+Bs(0QNu@BTkN9N2 z<;E6TJVT>vK z$jY3ib60uU*BxU21{zLVl+PZ0U{hQ$M%>MN;0HG|fP9Q1uRb_5ErpsPd+e0tI%P9- zxbrtp#ocno!Kak)g%&Qi6q)Vb;~fT8yRs4-;#mJ6%nS|Ni?eP{SPz!5zJ!U^MUzwh z(Ld*&7^_F-kTWmtkTegpp-q@2Rn9!npZ$h%WWD5<7P9R(0e$rcK5Rtn=)n-U`6WEr zp6}jae8!Ge;bRrtX<$yN=`Vf9{PV}Vd5KRt`UfYJZ56LtItr}5O#1KsTY;~1{Wlyb z{_G{u2l#(?+k!8~oo7Kz?fd%xY`vQ2!R08b`C`gx{nRX+ioL)TT=0=z1YG-Ldx!g#6Xr z@e);M2i?wl+Qv{oE4+nG8Ar7~?g2lfxcw(DdS4p(hcUHqb~&7@U~I_nj=*>cPx^_^ z)DF=oG|!w9#_UOjE1q1eM!P zdqnYaesamxxctDm71B*?MJ$Duhc-qva(tKWR9!SkE+~_vZ^G;zBxRqHXSlHR)3xdaebH$~2WRur#8F-!`cedLvK3Ax9)QqdO$d{&pj zCmoJY4ES8Z=h6^;1JlgIc)Gvj@;i#Nt^|mc;zE0&s&QOL>&e zKDA2}IqSo>DN&_|)1W03v(_dp*Yc+Jv4`T&%oHh})!H4GFcnq8t;!)+&ZK~x_7_W) z6mSwwzMh(|EP4n|*uvWPA-nCY&23DPAxIt(Jx(*Cs}_ zx-)%-ZM6*z&s3im(Qop*CI|+J;Ml>0reAa}5Pq$$=~VSv6KU4d$H%;Ig%?{##wqEr zbd^zQb;K($%5`|wJiWcw@w71O9iT8#3uvkV!%ZxvDEslm_@oum{Hc_NF`L#*OoS1d z0gd{!W+R>v3Egb1p@|uRcQc_dW>K2amg>cV`U;V2U9SHbGW~}A4`uN;|G?1^ZP$%e zm&>3aenYlq?onO#Gkx~URk4V}l#PM2bK>iR{ws(enSVWM@$q?-ZQ`v)I=_8cTSwT^ z*=W3&tMW;Fed9Ii60MtYJ0GC~LO57Os7rMd0wW49I|>(o)F&%>E^OsUWijWJQk1Ja7m$vpNWP+YEDJWgo|KP2kkC-U-04fz0@6C4$7XIY?C z#gXx@`}8h%_0z-aluWNJpN@sMn5UfmFw1}033g%ev^!yUJ-VKFJmgrjd!DyDKUDC2 zXv7D(oGniHB)w5gnwR^VZMruLq^f}Tg+sQjp67wp|8 z(aYx9?eo7)p$&l?d2z9D+K%ZBi*0L2L(xIoi?0r;g?li;+F0&3m*jTVf<1}n@My_O zF~rm#*>dr!9Dd!xIKr+OnPc03>l8y55WOsBitr~73(Z3(7ZL?zDBQle#}`NG zZ0>Z+eH#DxjRLwQ{11FURi8+zoXzzYp~DEw1e%aj6jo!9xT0v;V!@(yo;CA0K5uUZ zJ>86`90gw)6STa3Xoml^r6`|Eo`j;ik2)t+yMtM{W3mgj+{Nmwxt};0@RY57{guYB zkl5hNnr)!P&sZ*h*6RA)5e3I?T$s)UgJU}Ui{&%oVa9_kA*Dcf;RmTWWL{?5kZIXSl#Xv%siGO{!VE}RcEe>0(Mf; zHer)b8}Teb+U80+MsXE!SlZ}|pGZ;EmNaI1C6T5mOFtmzCMMWMzE6~1VfdY`yayW2R3-zZ3Y1OT%v`yR0R!MP|fxSggJkbkvRTj^f6;s!r`2h;?UKB9_ zA&z`%TEBU?SZJaTVcADN2ZYo7Sy(K}B=TKjoKEbFmRw3S}8`*88J1(Ws?(k@8vsLJ|{{_ccsg&w{ zzk*VOoo+P42}CSg`nY;bnU z3!4UvQ(PVSR7euRWaA)C8_cR(pD^pDfmLkFPZg5C>I$6SFLea7?~+iyfLhr$;8U)E znoZnGTPa&8B|E!nXP*E7Ipf^r4NXq$*TaBm2nY-V48!UdjkG?`94%^}c(Bt_RkjSs zqOAOKsY-tNuW~l>FO-fob|;_#Q`C(0N__tocgngR$^gMS*GNyWi2Y|z!M9@WWd^_> zu3*TN&%Hot*4y>m^vX%z-yw36h_tj0Kf!;CFL;h7h!Q6R(Im_8_!|6rMs(7KGzRFB zWOq^ZS7YgW#R6r}ku8=n5>a3idhce8;LEn)Ub4?&@<}LCa2J}u>-~hUfxw2>=feH8 zMXrsOv6ftNkI z7s|IQzC_)$=s#?|>Ak)z3O<_OCZc;b-Mt09)2H#%l^Ng9Ehx7mkD%MKD=(%qy{;E` z!Hxl5?+11y{;Nqa&8*E1K+Zl5KY!4BXzLk@r+V8Oaaf-Ng)?>j0iqAl+VySrX~LVH z_=MLU+do?3{Lfh&|4M4Q`R3mpqb|AM^*_VaeV8nUC~R`r0p5>MNZyD2my~2YJolw7DASX zQ{S0r4#)sE{`&a(?Av~`L~|AuKj`t4c`7i9hBOx6mz4JvUY+tniE5N~w?P|QfzO=r zquXXAhn}}%{0Ip0=QhMX-x0qQ>Lq|(eb0SgGrM}Hj0bPL^r~OFE*4F-pHZxqxz}7$ ze#J!BS=^PB?8uXi($f`k7@Bp^_*N<>S<%$LEMgZu{uzB~+ov8;Dn9Btxv3M>2jxd! zvml&mo|?Be93JkY3=DXVJChwLAY((;&`3UG+D|c2NE9bJwa{UN%_=$jEeNEx{ify> z*;7M7#XScWPy&Fh^pLL=Ga=4A>8e z$pJ|O8NA$3k8sCN`v~o3fG0s0Z?r4SDdPAXsa@t%xw2uCJny{p3;k)hP+o%D3D+{( zNK@Cfy4QYC*VZVgs@h`{e;II&U&Q{7ov(o~FN{tn1tA3it~9kM9O>>2))08Xz6=(W zHsAI9<|0p8q)D;iVH>mvcrCo>ylg!CoNE>@gPj@7cTRbrn%BmY(U9*vJX05=R$p?I z1^+6TF8mRvZ=&*gI+NjRyd%JM@xLT+Snsh2y!A&^IHltG#DOK$oAbx?{z z{d7L|77%`-ADx&2xrF4UPoT|g^4MNad@u9$yK0JA@l)<|@4HtDnOAaQ{CMI(&>0u* zP0)te7ZWv!x!&nr4=;d^0(V?UwKl|2Y0bQ-{>!KqRFf((Sx?h|`_( z_db~pOAm(eEV!sBJRv~x5INbXW|n3`^`TaUW(qqX?8o1WwFOd#2H7KVJwfb*628cW zxi88DP4Z-w?_tX#fz+;}*t^HhOcc|?R5dPGux>sFODWv?!r z61Kf8S}m@tbYmHi7Vsm1O+)aS^voZ#I;^a$+Fu^8!!*ri`y%$`^GG4yETiF*F+!wfQVczrp zt*NWKyWj&}0HP<`gi_P)?W2YdljOv2r?_}eRi93*Ky2!G1;OhQg$^L*Kiaha0@g_O zbvDeJ)d&8j?*i5aNOz;=0*?CaNM+eGZ@_FQ`sE;cEq)@ZZRnY_dL8k$_Yp+Q-__Kf zT;;oFS9R|5Q0)6~gK@-pA^5ug>NSXFzY_lRhO|~0dlHn3<npMnGeW**?N|HjI{x#8U@}>Bkhi=UIQv;)e zeYO7_$;dfQ(l$c<*89fHc;mxI65dn^sPZXR`(*4nICzE6xImx$)B5XQ(=4vmejO`e z>%^W9?uH`?-==WR%afL=O*@z7?B{BE*sq2X%@bv4LbEZ>IY+UzSw_oxNK z;IHT+@J&It#JD2QViA%Nl28eu3z*?7{^AA*;jK!)UDlF??I`X`9xRU7emFww@mJIj zUF4GC>vIhQ;u&XCr#@*TSmXBRD>#)p9^v2$XP9RO%I){#rXke!^aYVAwoPMO!w|4Y zL`6+rh-rg=Og=E?YMNG-;HRKPP0&erKuf7KVstFC<|t8emXYeA2s-A9Wj^URA0I-Y zBqH-;Y}0ftAGN9IH{!!dpS!z`VjwlMYA)sdf};(aU^CLpyWhz#7!>$q?>_>Ie*Z2x zn)-r-@pQZP!gZh0EjxKV{JmIXX!_N;n4cv#jh_*%dqWnzaJL%G)Ig3{;tGA3hBo~4 z_4UGZT}rJFHCp9)Cr@L7Qk~=1oehEk<(qV#4|p^n1sOvvK5UE#*zJ#;x1s}+A$aw; zRNn}m@%q9CSvZCI3}ho%K9hCnDX`CZgVx$^irPe=#YqFPW6^=EA|&pe9ff{U>d{KU~~0Rmr0M9wdmtEvx;cd)$l`mbLb5#xGi=5=2CF|mqEyoG|7Yn z@Sbi?X$V}$8a^=PIzB$G26U)y35{*Re@lmN#(&U(K=sQEdA7BW5m_Bv9S#Eeb;Mq^2r=vc7A$(3G}3Icihf-yDoWM+u)Re z3a>x=EfGW55r|D(5c2TEPlkk2`=Yn#=&4!qa*p|kANGxGNTsR>u>w(qdI?)o3~ zyxsucL)GiUD&qiQ=UqS3#aku2FX6KzapIy_Ir0cT)h(9KOzXUXgK+wn%=tb0Yat!H zsdVuI<~TdEeR|U3mS}yy*ETe#1dUj%;R-SXeoHiUdy-$K8*+*j4H)SJZ@=DzCB9f8TYSRmA#SbQ)|54*M{B8-4<$4=` z2Q?s4F&?=eCieZ-rIJs19}miz0PJr`VWVi&Mbqg6;_YKOp!+?rhyxFJ3|62ln*yu0 z4K}k>)M2;%D*lYnJ3J^_LU0Ee^=KN0@WJ~Gu>15QHo3;q1g}wJ z<=@$6567Hq>yi&0X!ukPcifPI%F+hr97^=^SsSuCa+YitSgK6FG3&*a6VnFHj!&Hk zyt7CxC_6eA0G4D=2bzr{CI;O;cefy_BT^68#UkmE;OUSOVWE!)i2jl~lT0<>Co~f0 z#Rh~5nK6TNv;^Qp$@&)TS3Ao;0h@ZiOLN@XdCmv}#F!Iz^%7`)uNgC&uBu*oYd|Xs zm6}V@@QtYWG>UQKi0r_UA3|^4A+p$vK9PKDymb!p{MDfYkiPXP?D0?$mAW49?gbYirn+d=hwaUmD@a}jhpkWaHflR z!Tb2rbfh18;lw|^ncW28Cm!9q5GMV8XYK$!BGlKXY#>Yok>3C-dVX)s=iPrwIEzE_ zv+kXHfOw!ib?lR=VV(4b9Nt>&{|Gm~>B{C&1{d%54gUpsx)%K`>&FRNXT zuA9$d#9zIn+S5J1=dA^&-%7}2CzJqdm`#!jL^E(&*r{Mhc z8uz?X7?UHU(D(Ilv3Ua_X{OA466k}oxN(-f5}(O=Sje?dHN_I#BGfFOE?4)tn(D-L znOIq=TP#hoKHFrQ4jMofl7}mPi5qekcKY+#9nP#f024;in#aay$#Hyg61mLX6-y5c@5|wTt&z#a4SrJZzp)j+GySJp}M^KdYvEKKxwDbuHzmM)OX)Zju zOt!UaaS7LRVYJ%iz(b&v@#PpWk`w*$4#g&Uwb4MOy-@NWdbqcdA$2%B>VXs zrivvA^SsH>GNE59d{=uwoS+!bio6D#|OD+&)fXIghcXc!bGA(3K8O z{K%xi2St~NuQp^cxZ7f)9)4V@WK#$S&C&&B9v^$hxsDmkTKHAf7DvWH5LWfW#cVt006eW(J zc@@46=GL&`f(0j*&=%*4rliH58tWo1tdmf_vBkZ^h149Y^k?9P6+%RaIm(yXf35jN9QFNrhMIrK zE5}e$hRdd4Lg3Lxp#LQwpkyIM%MOa6#>i8FL*!Tj(hN`L5+`(x2=}>HzpX57=Alq z6*la6AE-pF+M~`<9uXtGO(#djI%`xNO3Xq`p~He}Kci|6*IO6ZS%Cjdm`Uy=qJexU zLwM?tVNq@Pn;BA*&$#I~5oN?x`X1(;Vcj;j%XC-An`2EgGc(D0L$loOQLjE7IW~6e zS}Tblzp(5g0Z_Tiht9YJ8G(QRQ&ogTO^Kkj)H>UH*y! zS*HPr$eqC~0$QJ9{=lnQ4qN(d_DRgW&n1ypy609w4um@_7G~!JAtKgHv14LMm0E=H zO8=;alcANuV$`M+bzGb7CKc6l0Ub0rTf*SYg&c*aq$;~923$;H7y20n>TAsyGSttW zdqASi)dh<`Jbfi5|9hqLfdoa3MDz#oI}lSk`HJ3t?TusOUv!_7zY~jIEn?R?0Xhd9TAa| zN)SKg@*!E(d3qKG>z$u1m)ng-3n$;UT$%sp>5od(TrTE*dtDw1oZ5r9xUBFM=LKD0 zNdEMG&az>l(3)0RS@~(+?itf{)p@~`j32AzcFpsCZ2d{DsHn(8H_SM`M6HbP=C831 z@P_1!s|MT)(Tw$nighCokJg1GJ>c2lKH^^GI&{c`NH zzRR>@Y}-j|IFj#O@5yD#b}bO=_>WiHnifpyN4f)Gn~uGC`1x0pmF@w1)(7$W3g9Mz z&wTsrf6&1jNKgxCFyQcK_nr)a*Rp~2h4ryWkS^?* z%rP?g%BB%|&L7E%YJ*Yuu5yL&e2*YJ0%;D8PcA{yvL=`lf~1w4U!=_{C48KvYWdgux^VK{LK3HQe={*|Ywbe!c*ezZ&|p^~1-MuG&G#sJ!e@XXZFL(im|%^25} z=5XI@tFFFI-)qoQ+xlHo`{RO7P)*Hw(N#vVGz756I#QKo)Q!keCDSg@@dckMy>&g^ zkHz8}gYbr@|77r&fjd!2r*tG=0f&|eT*>+zLy;quUn3}S-_eIfBo#=?Ve;$x%mQjY z42WWro$t5|zkEded1^rH#Y^#u8+aCSMT3baXxMopf_XHdtf1HPmpibYdNVL2qPiAdNO%lA*OL~p;AcAH5taQd<_H1B~|UxuY%gscd$WCAIS z9pe1psO&SY_qpPn@4aiLB{feOR;?VKv0UKL^e6*yLh#qpL>PJ~h*04iYfg)RQi;3p zk55{VOwwoNzvBtBF3z9cts|Tl1lJQch*|heqgp7PYkpDow-vA@FZs>TEg35h9|6H2 zM8~YhQ|WgNHGj#h_azSkH#FDd2E9RqqTLu;0K5Uw^iC-Dzp)iku0pN5I^Rz;y>AG>ga47A4AOb zi#jdR$H3-N48Mns+g82mQpug)509nb@`s+&Gdq(VS5rBk?+8 zlLI*XM(JGKZtv`npZeUkaXaitR>>?fU#v5nblk4#B~N4F^EidPT>+ohrF@}~3!i^? zCi}&yN=75rM@~+F=nVvT=nAX*JU02ZEl|9x5Sn7V?%J&S$$Qe=`*Gku7$5k$)+FT( z8u{Ip1yF~HG2P6x`29>sY|O~4DpEnJwZ zmPpd~VUi+zF!*9^x#g&(QKDjGwB@np4eHv^?=mOyyg4>Bx=p`p@V#+)>96Vtz_V+P z%H1>!jL{^uOA@#Gm{qG&Gq%6CYP@nWj^{p@1xUdRuqu9?qqj+p=8FEsoD)w}CEE_t7={Gn1} zLpgfQAopB(4{z|-02At=`s59}+9;W5L=16(FkBFNt#d*e?5I(>5GYV|kX(wV-(DCW zXGRRPAS>{_LD%-SDb`Gdkm6%-?vO z=l!R0ILKN0P((j|o1LMtEKtS|kax(Qr{}IL6DZ3xa)5Q$a$Gh%)W{khI~2E0f?tz8 zv$55$Z!nbXjdtF9@7cqOuT_cr6VCl<7#5+k(`&JAXFfZm{s(pw+J^TS> z|4})8kbd0ID*P{(G?N((@fmaByKSj@RBR?s{p6uxYt#yh7I|3{f)ET=xtM5DH^TEmF$l!lvR%nFV=M}*M~ zei2l$0;v^F&l8O7`ydeDr8bR!m+MU)GOj9S*d_zpcn}hIQnfbupQqZm_iVf??CmHs z{$w*B|9-pbs?X_Q-I@JHp98`If2EmmN>X2)%A7|ZFZ(FcYAD_RtJ<7`1?D(A3b!3Tep0g4#t7Q3{(S`6z<~$QF9KUiY8VT zmZ_$$D(@4=?CqOnyHjTztkL{vfh7=XQnl;pTIT`iEr~Y^aq)0D9U#ADe}J@blrC!f zFCg!=7qAwRRDr9CIO9uTf(y)1!Af`n)`~i%O;sx>1`3owwNV-;^C_~`ECscWM#}NA zPN|agvkX1blm51TQ+Og}XUvH1m+6(eKpnmFGZd7{{mjz~8D;}1K%qJqjb@!6#?g2$ zU?rr{`(qXn3KNtmGLPY6_DLJ3`vC(Hz93kIL^0#+Vu)LQ4mL*8vH@2kAkKo}(5HB6 zK{!N>vX~=8_DO4qMjO=WYBbR{aIqsH0sj?(?}Id;Ea+h!TYysGkFD^oM`F9sh#%_& zPm8rg{gJwHQ~f8L38Z3XKy#L!8RwV~1}WAbFm_xYkHRE8(QRNrf9yH;pS<~LP<#{hpUfEmY6t)*GXd;{uT>-8%l!f2ECj~g zd%hLV(~h^w7J#_dt?0q|`@jp`o9tJ{eQ(z*r2Et4&}bCpWoWG1g#BTTtHt`3r9%(Bzcjz%V&U_ap>SUk?m%)DXt zr1-34LkEXAAQqg&-6Q@F%GDv%sQy_LOAm4O^IS$G0uiTj%_ojdLN6axrTYix;ev#^ ziyvz$lZ?A~y!3=3rQmWow@BKPPSatROC5Pe`wx|uSS2OKpyRcUbl}oanZ0o~Yls+T zF-1`e=10ukTo1l}CHX=TfD{Gd&@}(BE*r z+uq>YTUV>q7xR3EB(h05f|n^M2uCDeMFh`ig%7z%O2x3s+>SwshW3*UP>Gw?o+%kR zn_m1-w5qK)-kz!^;b4>OB@#o5Ab}A_&Wf(aER##xD4FTgj5ml7W({O=RlKn?AZ`wo zKI0sINk5-s9^m{nNR?7WWDKC(Jv_pr`D%7nLO@AW&^|JOUp5ghW6tl)0WU{*VG24}I z(U)=FgsM$JPT(v^)U8|0#%>Nh-m^k3f4EHb_=%66Q5c_uEp=v+p~IzNEJP(M2{nLN z9x+l&)Y~w2V2e-6<~_mZYQSXFpkul&*>s!y8wt6zn;YpD6tlx|QW=j=wZ6RZ3A{&I z_UjXlI--N1E$$Ophm*7u9k;rJ=Dl>8i$_vzTRA{GqQ3cI+#K5BgZuaIF$BCW%v@YC zt3+1ojVvD=!&S}9_Ct?20uv7P(BoIY4&DxfqkHDjd$2*3rw4q>GH5x;5S3*uyCptgK#E6JnkLIZfz1n3>T zzO28t$qa)jjmO`Ma#OL1xAnA(FBlra8;QQ74 z!Du3L1A~p)ge*q@b#it-uAbT+?mIMS?Vhxqp2*x+61^ovF8eL0+jU>xDj9n&&dl4+ z+l~%KL%~2~EE4?-#i4HWEPfm5q_8XUNx8G7!xYPelQ8m)# zV=8=p{UWqOXJ2K1>5-L8kd_pUj8jp$@^H3XWtdcvG@z3yq3}@3`EnWocBDENLK}1bmSJe2TF#2sIyz?$*{~65$uS^NJ$Zy6Ai5`;J`@8IN4gq zu_;2d4)`>KA=o`jr~}f-m<<_m8F-ye$!QOv?TtEGa?|gu+K-WCNMcwOB;;h9BT@V--K+OWxh)DSx-Y*8;bcW$Wx^(#m-132#6HgxEAKfNm z`xZqpR8EZr>W1LftkdwRWv$S~e)s;?wt7<@pYe6rLucaJFZwfDwslri$yweOf7LJ% zlMM#n9pn0eAva2{9>Sc=N9a!agUZ`;3#L0^(2Rq`O?v9Yh-K~jg~u`>wrhKKhS?z# zf@EP~CO;mJS{>9?<+T5t@<|Fw?!n7)osh8#cRWd7xkT2GyP z2p&r7)E9}5e@x8Z|7DB=57fMLG^3-2N<93qp?Nt}79)uPCAjDzU$hPXUd__)zpZep z*~{#-A;{x<-#YiLtd7bvhTQqrgvi(TU0!~VGX(3mi#5g(j{D<^Wpg3h*pjdknt)1E zUn-k94)y&pbz+_4ln;Fx5ZIguYmX~9Cn6@4bK9yKA@=qElShKLm3~tQ@B)2R<6bTx zlTgI_JWPrpaUb48q-13501ug?RBrM+_q?n})Po~#02;Zjd%FUu!-<)LqaUcyzy0Q%V%u&U>?4L?^C?!0gwj{p5zLjLS9v&uQP>mY0;!tO4~B5AXrp zZf6@5mrWSK+rrpHMEiiB^szI}$i*&`Z>nX(5{Ol83|N|&(@U-n1Xfb3%>xEnx?FX+ znpR!XJqfx0=>!8OSsTTQB{kxpB+x-4AAdFm8`1yrD5a&JdPYhEIaDq_`_e8;98;0) z>&QpV>4KS$ax3g57ODFP+K`lf1@9ZU^XjUU)49dS!a|LL=25LsC+%rRaX(VZfycn~ zc!+rtF3RgJkdfc<%b@cbLv*;pGrT`?Jfar%Z$}Jt8DKFvL_X29wPWF^Rw0LQS4*W4 z+AtKQ?nt+S$1iwi?A3Q`LL2UFjSi#Bl4Hdv`{5mu)upEp>o8JR=>z-GK6Hkhg~q5E z*XEox&}t@Kv2z?%YbeiRlawC!A)fr0Ms^dT6@_1vU4Th16iR2`oS`P~g<4C;MqD3~ zyjRDmd}2*J8>7}Fop+?4FHR}rpU?sSAEv%KD(dfxR#H;wM!Fl3knRShyHiS18c9LA zOS(k51?f(Ql9m((q#LB(9sT~^dw;mrh~tM@v+lX)oPGA*he9{pD7P}UU|*iKj`E8f zet~Z=R6pslf^H!W^JsD2h~SrNw7$_J16oE9vka9`;J$*^!(hziv=aD>Kda zUVsuFerb&K#zkKKy6StA{JL8*g6GG)Uht3%A@((br(vP>cTsBTra~*K6y0UIyJj|2 zp$A=p;+P!l(#~g^c{OQ6dxRgjcC6zA1K~XOh|E1bQS8|CGT9Mw%Ab%AjD8<;cJz<; zuJ~AEHS`{N=iH&C^_yqbr2qUJnDW%K860@koGtIP(i@!A{SBTN#itDaLPruDWlB|{ zW2veUo#+2mN2{w{!ENl8BWu5)6|o?A7Mhcg{lDT3s(<=qQRlHl_=!A~6!baexUA$c zA{*^DpfORD4}i4X9DHj^vVY93d1?lpnoAnq@2q=re+e2MI@S@;)&NzG68JUvupS8j zBKz*dirYV~-2)Ew1VOU;1$N#_fbje`DFJO69JlFjc-p2&Cy14bO7@8_l{{qr&saS) z$hlNbnoqK232(WMYB_x@-}2N`C!GMBuNrF9eZL-Ho_#}+Mi`qgnnhKtMB5bqN+Hp= zF3G4K*&J)8aHzgEh}|%d$yl1(jZNN`%v?B}rxu&$H>O6L0+A;1I66iqapNho_F#@{ zk#{`#vSQMvJ#&WsM3P!+vU=vdI{l;UL$z&}VO}+d;U*R&L^Or$PVM}i7sGoQoaqQo zaCDu78Ly;5xVuMi(#ArsKQV=#S(m9lW#>)sUVp8qFx`jpnm>$~L>42Os+*N+ny4Dt z9JfA#OU69v5h-;aN;Vn8tJzl<^h78t{jYZRF_J17&O9iKBo>iKj(1sSj4j%a%?U^b zbi(}=*zN`5MZ~F>JHwyrI6ukz!pPBe$>vXq3ZFxXf&!nziGkYlo)ZP-76<~?9>Xb= zzEFJU@Im&M+Mm#lrc;BtC)h7kLmbC;v6|3F%7>7(IBg%yFnUSq8&FvD>g_pn={-8L z4TE+)P&_asnB*4a+0|kW(QmoUZqCD5mf;Y-b$VmThSRFNb3|?Y!Bx(*DImz9M(IZ( zigDC)W^9Rjwv!s+fBSAa`_7I;^osuYn8NgnD~n@YTBH924HQ6eX zQO2(&X@5trBb*`zxCsSOs)r;r$hZ;tFaG{{@i+p2Ki|Lz{IJ7JnUgXd%}%~FDilnZSUCpHyVmIxVE>#fh`fU)e8?%<&(NKu zfJQD>mHjY;&$4sa$^P*VIw-)bnni*g(~U`JjNrlQ)$s;d!-ZjBFq8N<7mXZd!9+#Ggv3?v&Y^AB0wy`8P!a4`iH8jM>2VgU@(5iVJs zoX@sj-yc1Fh=XyEKon|zg_!S3-hVs)ixGV6sj=Oguls=T_{rPt@jL+WB?F89Nd5I} zUts+2W7v&zakv72dmr95!^_LcNa(W9;bw)C-)_p+3xHPs3;y=L1qr=lD_)jynY@bO zglq^+A6^gZi#Ur1wKN8_#X$t7KzI2f1BsNqPYgw9Nc3IoDm|q+J!g=bl2~(s?`E=< zvQdZPNC-MI6UUy+#0-%pW+4qaOx4t#gWNBdB2Ya7$$C9S=!v?p#Oym|a)k@IaxuFZ z%aK&4kEElZFHkHH8NY9HqXw4=w(BD@QJrxc3tfIiWSpODlA>>7GCdCMVi32nA);`= zr`N{gm|h#1ffOl`7p3MU!9@_RmT7z?XqQ+mC=@s4l8=fYG2!7=4WP%TtH3gOjQWex z-vre?OKJ>vN0(Yy6^9KcQ@Yxn=S8`$5<;IthEc{J8=Uf|Gd=5p$vP}kZt9B(Aya;G zr+&C^lROdGpp=T<1NK8A%}z<`hAsu?#Gk zv}0_(5}Up6dWu~z8>lUgVGx*=9pYNmxkMaZcc#bm=!Y{W$9svlJQCDJQeJ4AUX01; zNAu@dv}vvkTe-6ywewk;U}+d8bd$e(;>poj}jR@uJs$Uv?7)Z z!C_Rz<}+L8@3IMQXiR41K;*?u$90gyiI&GH=hex2KViF)MTe}Pxx7Ldm;K=kuY3C} zZTsvV^3l}pYyh4~C<&z|M5py!kb*R3`zrNpZ9?;YEEH-+-2$(a@rR5pNE?3BSVQ`{ zL6bd|n3UAq$|`KC1QJJ!+TL;Dz%dGWuKtAR*|YrhidJ%T6gx=3As^x5Rx)$>pon=y zqo!7SLep@=!M`ip*_8iZhmzSnAbRf%bFF|i0H4lU{ePXDxti2wx*@YT{SHW zZ|{ZkvfaTCqPM1e)nG=o(G+mgbfe`95Cl%p`uO>&YJri~?Iy=%OB5ZDPY7o&7vK2I z{#irb_TM}E`=IVhS{Do2<#$cla#u{P}AztD@WtJ$C{z&3lpqf zHpwtxfsYamr|Nx^S^VlvFy}LyM0#oKAz+!xND#kHTf;h4-7|`7=J4umtbIyt7P;DM zjb14hu~YaR-^E(d#S5gyYTqIUNpF&dA)KrtxlQxL?8mBsC2I1ot>LyLP?8WMsOqtG zl%y9U=n`)7=%Sm9p(tM=wB)=Q3^^p*2V@-I2Yk1)k2^u_|0`)*{!n}$u zoyZsPc8`x3H%gt|Y{vb`9p2Cue^hxxJ478{nZJR&AVXwc?G5L^N{@R|!KFh>sV_od z!lhFXk&lQihjpBBHydeMw8&+g@}Q5hN4Juw5-^dle9^zKbvqv1)ya*c=Yi3Q-@|Eu z!^qK=k`TR3Fe8`U`ZN7yD}6Vd)LZf>10+Gm_d7W~q@ux799`Y>({p?+hW@O2QHGo2 z>@O5UlhDAjk0eUW%oi0Q^KJuTOq|Gm2bgC|wr8Xq`IYSvtAC99zx~<&>>${MUM(i6 z_~v!EwfIL3R>xUZLG*UclaRMmhzgeHETJE%*BPng-8`Pq&Y;xS&@SX|5EZ!3K#Cs^ z>m}Oe?Ht$6Pz#6hy`mx`k+^==P0}j%; zD>#C*Ws*K(<6a4X_};5qlQut+;Tb|xS!V`XLKsTyh+|IEkF*D=`G>~%}oc^a(L z>C|A@np1%_G6)Ghjq6|~dzYE@INl+6Ccn|&ZpXjgot+#+#osHuk^V1@rK*XUKu_W zlSD9iu20IB)Kv06aO=QP%bb`S4WdmJb5AIGUPb5ymwraZW~O?NQJmeEeaoQ3|5&f2 zHhy%&I#ELgUnE2j>q`ixj9k8SlnNHJR0s_g29*Y}9jlBi(N5kiTsgBjrd-8Tp)rC@ z@C)AL&gaBRXegh#*W1J;F>x}*Nur8|igc=}6@?ulzKt-wsjzIuqiA)rO~&auwCMka+YGmtI*b9494V5|Np@F>KJAR>rj4W*^#zXQyw}4weAXM)O$U zJ%5kk*IFyX9N9}rg6Zb|j6}0qo?fg3`uyBS-If@k+gnb-6NB0^%^YUTyF&4 z6xDA|E6lJbr3S9cwWng`Gq!h}LV)(vO*f67g#}kzYc$M~W zVL!4hm7th93wENfk-7p$3+d1grMqDDUu)K*2aC)1f13?t2tL( zT?g+vfB4vFRGPG8RC9`ss7#rN1&xpOXY!RM!j+DLbWw8np0*+VPUxcacPEd_1|pMy^_xwspg!lI9irAjQ@m z;yM2~v2*S`SAv%vvuPjEpx5K&P1yR>I(5;0F}nVo%Ik6bbk8LLH5sEYwo%9|+ng>#|YJ~qU{H*@lrJUl!Q;MzFhpSj0!!UECrBpG#nAm{Z z4UIC;^*hh6;^^~b2S0v#0=T1jFnoD;`0(NW@LQl0cwUN`i%V@)z<|r$?|xYP-tR%~ z|ANq&!+=`hZP;CVW}e{^4`a6PxuYrqw%23chBiRB0w96|77RqV@k#v;1L>(j4q?qx z_;&okA1UA{^>q;+jRnOMm)YwvHA&`|P*lkE%jU$Qz(H!9z@rfq4ucX27h49q@|yR4 z`$(SG#Cvv-o~3yn7b3lxVlLb6FB7F&w2~-I;+!8j;l4bh0h!S@rPJ0Z;8;dvVNc*-;4L&=O3Sw66 zTG_X(mb@IsvV}X4b_HYf$$AG>d3aSU=AQS2&wmYQ+#=E`rQj_k6wXS#rXMKwH(z@H z`H3@igmi@56Ra|EGpDp(ioT=)lec6}77Sh6D&6GTSTXy(q-1a_vZTs5v*Dzg>)yeI zmGZxX3fIi@;p~N?5@ zL_vrP{&S=d3#lR(4c_Dii}Xmm(&UJ+@VkB-w2`a#n}4`QqBm0PWKDgsnA5FEGH3SZ zpL27)Vv;5b^URjAQ%)#^ztu}L{zi?MBx?diGt&=cEec}7NkI2i!YIK@4gM0fQUCN{ zse!eKSj_ulE*@4G9A$Vx8vn;}=cpGXHW8B*h6!ULMkw-}Je}h&w7J~QlS2w<5G8}J zdL*wlv52#R6Gg*jjo*i=o48vG_{lG15o7Fd>2y?7dgwOgr1}IjnF_sxT1vQzsQ!X) zvo}tG5K`}t7QA_B`0Ek#jGmH5>{Cq48uI}n1Ox;k^KrrnE;{Gl(NTe|t*wXx#zoBu zCG*BCXeFoI`!U%)oroC8>=My;%SS9((?Zk8_{ z_4HvtIsn5nXO=hnK`woOLWZo(wr@5kD=NI-tOqfO2($qxQ)#xnTItu?e; z2&32i*OGh?pTEp?azNqPiRfy_?bAU%|EM%kBzke?r<-0n9AahMGK}Tlamo`BvF)Tj zeH@qCu`myg#-xt;oJK6;84Bl2pNMP8vUM%=N!5FK^wa8AI)3T)9!q?YKnYWI^DV}X zCDkKUGz(s@d0OcVEON$?+$!P=bL6|Ugh{#dFQ$EB$B|T34eBA491;$2BsH@6GMot7 zCLgL~VvHh1FX(T{XV+tmLizY(w2AvDh`K0<`YFh$iHLeBh&nl*<__aqlwugY$3}C& z(_@K3t!@eVfnXAbWrf&B^f8O^8?L3tx+`7#S9m(vM^#mmv5`;`J%6K&qzr{Uui;8w z0&}(F#&vxemXxv&nBSW80sf%5m`jFIAE;yeX|7$Q{(zZ@F1uu04@Ltu^4Zt?J~qme zXv`NfiDhuj+q-zrwoZO1x%8FL!{DjW*YfdsGIa@Np%n%cgrM;Ob@+%tR%fAP7VU_* zT(!XYX~GDjYo=%lfAqVyDmDMLDk9%aC$8rVrF@-*d_q?uauMOw(|ZD~^L2wVY~-&b zuIJu9dj1vjxlkA}aSw(=Ps-!xHKc_qSn>(Q67R-w8j;k7rrpO9vZvx~C+G{BS}uiF z$TgS>g&MPlubvbhO40-|syr?nrx%WUjc;Y7QbMb@KcP1Ux6chBD1PDC(7duUh(j=y zP1gFQ1+8>+tkgW&ad)#SlBMY`Epo&z{pGH0j?^L}!F-a`+@IT)PfZ4mmqM{MHT(7} z?c|Wt8ZGOHL%X?RuIbb5+LNAK3HD*AHNw1-97j< zaK2ut`d{D~t)I6%g1t5{f>DKH{m%*jyihV*#QJv}q<4mbjXbdC4vb5G!w`<1 z8Y_|OAyt?oA(*t%!~O{bF91JMg@D@q>B^s(rq3FGI57v;bgY?^(I{?LAK!RXcf(Sl z1*kj$TN?ErWvLqn=3QpLu!2|y#={R+13y<#c+&t!`WVchAgEja#p#oQ3@j8UB049r zKZ0>x$=eIGEza;|gg%A_?_~Q6*XuY7Q6VpJ^P;64zrjCbG;MV37+!5SiHFhbzLFB} zR`@+PmH75na8BDlMdoJVLNg;&KXE>@l$XAMWPM5Ts_LHRo>WvEEJE(HhCkcM!8L4t zTxa*x<1NNcYK>ey3a6HddrE71WSA~mX`D$BVo{g6j%vO}7M0gW=91EnEbm>`Ob9L+ zDlnP!jP!U}-gUTW%GbIPek*<3Ok~5R3|Dq~R5+3%y3ia&XVXTZRw{6WsP9r_e(^6u`mt|=e2*OtD9xXV=Ig_qJ%=CXJuGs zSOPi{Pok)#nJv*xGf8V^%viUjaA)60{XoiJfR~@{+@{UMSSU36!j(QZuk^Lys#faN zGa0$6ry;BGa5U0xe1tQPL8n&MBXCG2xw$GP@IWsH;IQ z+Vy!ISrxg6F@r71(Cyr1C& zLwy)ksfVVrIJl|FtF_F)^_Ejy>-q5&aXxX5MU8j?{kXQAlqF(Z__cJaI`qgmAlPIJ zubX!~SdvHP9gf4IxEisT^?=HoXu=<;gak)W$SUMJ<+_Gmg92My^Xb{5RnF9jdFG|^ zvG9zEa`rO1e~}WuPzYL3{l*@ijw_^2Rq8$8K<*BGa2dg0?5N~2*HwwYWUeTI|yy_ zCEvdZGygD1^I+%ozgRwR z7LYNC?v!y@N@l!YZ~xRFproR5@sseBhC5 z5kT%|7j*2n^We)3H6KZMnzjmxUHx=_5|eVhik6iMz6j9coxm&z1rJ;6V3eiI?hhho zdF}W)Pl2~;z%|E1(V;)XAE4--g)x!R5uP-6VB0@?x8(d7HaxyO_zA9xe;>AIZ7uvr|q39P4%XmN|f#1CP3Apdc)y`r@o_%;VYw+Qr_#M{XQZPv? z$y)ME^Rr;#J9N(-P^^ju>or6+C4Q9XrVlSfI~ORAQD?f@b8gl47khg$@VRdar9fS9 zh{(c^0Go~ScO6f^Bki+hlIFJ=Ce9?-nl&=P4JbcMDVX3EDC_&_N$RC+Bs|w|JVMQ*Y@>ubkQ{@+w*xDeKvE^}KO-kvNgD(=m_3H2E{-5l&$;qbg-h>6smm z*SLF&@`SJPavTS1VCpJMFa_zG2x@K|2g~QeElgrYv_B0LGi8+`XFAl-3Qw?y2Zqb< zgF@Jm6Cu{@#d4r4KhcEThOeL zSEyQZFTVD{o8mpAB#qWD5_lNVC5bD`qI#^|dWBesd9e+2_TZMtb@yPaNc_x*Z-PCO z8`U2v)))4fSFtvC!qxp(42jx}=TzOGw(5G8aP(QO^AFAG%5U-4ELD~@2Kmf9@ezO| z`Mwhc8HebL7BIY(-uZnJr=ZF(_f|UQX@c%Xomim$BJ8?lT%7^#MhB$9ebMz*gUoSx>M~& zu9@=+KVLfQr!mkfzi!#lS1ccWiA!$2i@utSmDjEJdJc0@^ThB#(&YWURL6_{h-Un*FcSl6_e3{mb#Ue|dLSwD$>Sy{U4$BxQgj_W;;; zzFmH(Zn!WaFW}$pjL?3r_zv4t{ZB;0t_V}&K(l>5O&`uB+|K9R5&s*1fh^@b0oydT z9l28T06_ooJIA&NAwWcGy=*(&7o4A5n?vaz7!Uv!!C>XyZ+9lcYhYQ!0?3Ac^Ct}Q zyc7T&z_!eba+d&_IbR?-pyC@Z`d@eG91kLn**NxSAg+sFexER{pC!$LzqTAZNA`k^oBeM3h`#|GP=i3P&P>gpqs)?>*%7Ko&KJpK=3yI1GBtM%5!UHb&Yq?^75QGfr;qBOE zDL^XZgSAj-;&_hId#s;~abk^Y#@Q))8gLvlHWWntK%sf1W_U)xW)dZ>A317a6HOO` zlTau_^e3qsOV!KPtb~hF>A;guJx6^_xiDh98%H6U0V9uF16SUVg}Q)KtA3JIP`bvQ zGBtgQqSmiIH*se7`?<8K9~Gj6=Bs41Y?~FU2?hg+nXnoAmaA4~`;m%~0f9$HbRoms ztE7#orqEo=0W6K6GXwhc!{yj;)(k^xU7a*&?+C}~Fbl(Hc2;gq{`XKXJG^JJ;W3@G zrKNN8m@HAY zcl^nG;?5&%qvI|8(w8X;brQvw`CK{(FF%e)GycrL*?IoO^H48S0b=xM%_M`Q)v>Jc>sx3v3U2hWO>NDdz8`1ZMQ}bS2o8Tmo}ivV=VT1Y zWy|w=fk5yJf%MmI%7~@M-dtC+YaI^>ww_&*n%!^os0m=y=i;%&TBIiCWrAX^Z(VCh z)HLR1m9aZcRX{jLA zQwNV1lk1A#C1~D$f)r#2Ae(+bR$qjrBazc|$XQE>Vho^K9DLn~0Z@uA%*-3cZ#`?l z_^=W579tK}hROM8wJZC<-+gb6iy_^C8|cqq9=Wg*bD;QmBG$hiHgOKPaeAnjrK>+4 zbMmhPKUf24@T|Cg<6dT*6lM<&0(9d4Yv~jp1)eqLF+0%^`Sab6SFI;kN`4;R;>n2| zyS8&!?1Qo%>xM~Q&9(+lj&;cI?03k&-Q;kJJHwcUm>55ZwBE*6SI5aqu-FRY5VPTA z|3Tf%dEn?Kd?4&1*exgdF+CjA#=ztuolA;b+J3d z!F2R(E^6`+4(HV6x!%~Ydj4rZv;PIaJBLA zIq=!fbb0i*S(o;AllUSPiM_522}$NQBBD-~9ho$SCLroCb*VG596^6yRb`2{iA3w@ zAePm%%L={xzW(EzF440!1M{#cYx?vouk^l9^=SfC^(@i8r9GBUB7#G9=nS51tZD5B zGT(o`#f^4YoVuIx)*KpAQNyYoV7II2Kf%udk03T zYmb2V!w7`c3^?2|nsvJCp&%XD6Rg}{H2EyF-`wT%odusAfS!{+m-#Yvso^ODLf=wCBa*TYj`gA2o^om`> z6B8xCoP_o21<@j2b|^ave}we!M*;RhN5z+~yl-w$s_vnllCzY*kbnH(e_WE?(|#to zH$TM|>_HbImWEj69kx13dp~cS`x~le*Rbm&oxzSe_%*D{*;5Jepzp;7?+@_uA`@e#+-J=h?EP7_ z=Cnja#F`YBCz%F{hQ{Q#ijXJ4K&8A~|iIvZ{BMuI6c z!A|j=VXC*i$8D;x00A9fZ!jezC?HsNE1=>dQ)&WE;E-kixd5EjdrkOO4fd34Oa??b zr0frDp=oNw!IYI8lb^GJ@t)Yydnp<>czG!e;en-Q+|kjte*4clt$)UNJodV;;3RhI zAqM#Q5^wcLxl$5rVB#|hY zg=%gyF<9{C7|vw4(%A9|Zt))$hfMAVhGO zW860ZS}1IH18_svoFVUUQjJk(feA?P>{-vsOE{pC)CGl`kD(0d!;a`319Nh;Dg%&a z4=h1%pTVYLv%nP@#M}Q33pJD%AV0AVqEQm64gmlPy2pSUqc@WI=xH=zaitV-#m@(XPiZ^N^dcUe4HB{o|g(F zrX(Y(7E#yBo4q-O^56YIHM;K;<+@&ncwTR&rf2Glp-DFz6yt>`u#Lh0{aL+*h>w%^+>3X=F@+WVZ$F{MGH6B2^J$IsSZ)gh$D~ zY42IX9$760h^@&asGJk#T63bW*n1;Xh~ds}VSRfK2s5fUyO_2D0GaU-!&%{4k{Q3n zz}GX>1g4_(4)-WTXcCx0k4P9$-Gku*LKG~?M2RHis+0{Hf~HTL3T-D^c*G2dWqj^Q z?C>2P;-2nN(#lu#-|m*^c%IWnKYuRedMZ^{*ZTvxh5Lzi2YTfa!shFlhTt|6c8iXi z?LT{kY!05yrPVQm&q$;LGnA~pPqymIL;7aPZO3IKHI(UmGSP+?3U|i5hFw*-_M0<* zWpWToa0&^Hf_of5*WuCR({+L+!P)unk<>r^p zS2LY_P-}EU@aJm5l^<5}IL~kV>lMzTWGVdWOZLBUbe}&4_Ok(#yEyEEu!;wqSRZhq zk4+TFdI8-7c%1@;@vi}*2GHx*tKedX9bR47In@Grzy(xH=b!=t76R|Q&xz~6&=#l_ z?7hFm5P)R@sXter$Bcvl&2Mr<1W?~jB!8gr|H^?gmjnE>VJuYOQF-4J)&y#zqv)FcluFKTvJfYVtbUJ=^17QM zn{w%{imB~<1>5MyY&1J{Pddm3mmcKb2D4X~Ocz(-Kd?6ky;<)u?PVRVxug>arRYDy>s(*?PN zquuhrk?@|;S4)^BiDlN!EkuSK??#2udBn6vI7ohD45hg;LM?(%LSqaK#=tzVuz36F z6>^$P7Jp1Ce`cAT5aG9}Gh-^tQ{z&h>hZKp=#pZU%SGSDr<-fM=b|}g_x8b_N8Wpt z#zA>3HX?JpQK#pVS)~wCjgbUyGzWLTBfK%)iFTv*uWXntSCqMacgYE#QkDG=I+Nw%j^Y}_gJqJG9E3riZipD0NUQ5^Iy`$NNq^pDIj`iXp3tN{nkensRX)Ju`} zH+mPzNcSy#-D%|Xn;&la5j`QURw_jzZ>_OfqJWDVS@?B`yIu!h&$FbLslO7w0D9KucKS(gBpL$DIn z|L4T4@4lV!?l(js*oo%wrux?G-*5?ls}mTJ0GxAkbs)j>yW?wf3b+9RI2%8c$uQ~8*4Wzc86&a<%u&cib!XN>ZCOw8%!H2X&?f3!PhJz-ADDKCk zP4A0|H{VF7JkoCodOKUC-hSYhRCUL9MKep&deYa_gwyf_N?*hAAbimqIUbJgz6GDy zwT(zVu+1;Qz$L?=W3I;3!gOum`7v)yogmX;Q~yospx#j5JWJ*PcBa-+{@^`ZAusQF z0sE8rzR617qSADU!l_Wl5-&s~My8|WPWAkByr`n|{9UhS3MgMr>sooPpZi~3uB`d2 z2n@#>d&W$asG4$xn95;=Fc!5O)P?e9YNImxgsoAU`Y>6Qul6be+%iQHt{j=EmDi&JYnwwdN%q=vW1*7g{Bbj@Lco zQ7cD+j;|_K$g||Y_mg789U2mh6v3TotPcKJhe2m48oku{*`LH9cX0~m{Mw!(vlzxb zl@qA_vdg1$<1eBxjYpCfE3Id@F{4td#D3UKL)v8ucw-6Z(Qkx}a}0=9^WCu~f8yX! zX)ue7Xnbr~z+z)p%M$6i-DS}AG|vcoVtV7)yZVIJ$lC}r^tCjvo`1W^2~_+3WsV`g zs^YI--Mf~u=i9y=27m5Vj+@i2mbi>fu;nqQbSNV>>{oFYlaiyk#tJRtQKtm1xhvc= zZ|&JIo;;~FA9#ZN_%XIbeTA#34p=wknAhK&PMkn1qoC=@xtW7xL&pUTF5*dT6k(`0 zeh5uPif?JCFl*bN)xLR??$-7OyT!QjHEaL^4$!|DWWj(hRB&ZS{u|5%a*43+1_hK7 zUNA!jGGXs}FwB=L3P^nqc+d??D=-_l19#LXvf2jBr9Eq3r-2LQb*8cd7Wgn4o;ooa-#MZ52N3I$=u-@N)0KyX&eky8 zKf4(g6|YG7&LOA7wj&?`{6M~{;P8Q2Fc-<5^wuM5ZZ$=XmypJf-4!o#F}3vV=}9Ah z%$Ut;2YufqfZ}6n(+8B$p1+uK2*PS?7(G1-ezc;V_O)v#vFdcnTy&kH{nF4MU{#%f zp!ytaZ@|<+X0~MBD8h+)wCEDm_Twt;kX^`_og&`4kD$o)H7rafVBT>PzP;7j>0~pO^abj>t_^Oo7yczwJS)`?`o|9?)O@Lpv zn4j;lq;13?wQ>Bdxm=^sqGLU>rEOwq=0MwxNaXq=`l4~V`<>K(>O$w&o^9-Z({w5y zun1yoTJgUrZWMU&;>Wuo2EyUxHlOC5;>?uMl`qCefAGhBg(pi;_`|0SQdI&UMak;)DRQXs5)I3;dt(96{fA)I+|JO_k32>0HUdF z+_Sp+NE|DHmZTt2LuVJoO8Kcz*-IwYyjsNJSFeM@CZr+?i|H_FA{s^uttZ@OPdB{s z2Da41!c*+6A2}e+NKmp#=9%(yVtA*lC0N?O;v_2mDOPx+IDjXSoAg)?r$XL7=)-(1 zbV?!coX!&-yXQ@8<5;4h1M?qrDiq(q1d(ix2yYnfU^(F47NQbSg{QXbbo~sM%-sFdezOjaR ziTnO&f_&4P?jM94)Jx}S;HD@iM6^Di(J0XiDW-os=qD;v3iUJ}O+eiK#B)Bc23W-u z6xODufs7wBx{t4WcO0c&2VYC~Ep@}2mP%oT6z03O^X0qSfZWA?mpf&Uy5*fU`_?ow zUw~y%>G&n?n6yB?rc&e1=|1#}O8&oVdLZmSDE=&|=fhL=3awHN=FFKb$*02gwhk49 zHd|=!!gA-J1_L3wWk4Tpo(!+&pCcFpzu{37sdwbh7HM;6OC0r@z*fz8?eHD zpYWmAQ$VuUZ$24jX+CJ#ZSj`@Fb$cg|0lj>&tTZ~58JM_td=9#0rZdnz4x^kJM1{_ zC!MV!8F2Z!(JIpVaCdm9cuc$s{6MRGs6|v&RY9n(DLcKR1A@zTkSc&<7)U^FJ&U@k zfVXnB`dD<(XZiG5Wyk#={0AA03N0^~+aQqJKw#Q4wz{gPLPziadX)VA+G57Uu0F3a z7xG1&Rf)|qOq{239`Eo7d+m_g5Zde2S#k!6deU?a3LZUP2|^pd=AH}58Vo0>W(bH9 zzDMBaL+Iy;eiqqB5~<9+Bj5-|=PKN$`r`CIQ45~SLkOaBPvhjj&&*l56mpSdGRX1v zRVfYDQ75~Eipd2gmSVT-SV)$oh==zp9$oFs)x}F|Q=C>m5-|MC(EJA*{*csC5 zq7@~TQvJYKVdZs%i)T|FmfznuAtHj*YggH6<4ai; zJ&Hs*j@0X@8mk@ldc!ZJob)wU!}%5&R|Z@`gK{Jhg{Fv@6>>@$I`&vdYf+Ycr;7M? zQ`1(7MTk!kQ{u-twHl)|SrOno)hI1fl9U6s^90lslGG^iPfsI9bNr&k-7~UixRTW} zek(LNRt8(Syr~*es8$GKz+Mnj`zofbZjJLD{?5`xbW z@7Xblh*-Ei?hlS%LWPIF_xc@XoQnfHnUm#< zdbi!Ov;8CbYdRz~!%*q{k+7ePi=Mb*k9cPn(A`(>H&#n}7+th83WZCREPaC1?blW` zT=yhw$q{>TJ9FiIb?9)A^>NE#fU}!~`WW=+guWJkX%$&><^QD{*pbAAso8S-x^_Nl zL2W0fz%4u@A|^Jqw~q=9MS9bq*BJd{_)nj*-tY79bl20Qp16H;)0vhr!V>q*8>`xu z5QiA5NiAzvgZMMCcQFqZ`PVsF%mnUF}Ou)YG@2R40K+Ih6Z@!b5>j)6#TY8HC zGtWq{WKj7u=K?8n1U*Gd2!pKg@vlVHi#sS4tlxMF6D7Oc>^C_n=mYh#=%E+H=L+Ug zewh{C@b1tE1e}Qf-E4s0kKiYo%=j03^F2UOGuPxq0ed%0y#}-PZJ`s%6uvu~BBT>p z0SOVd3lt{3-qlGi7zCakoBj+246y$Sv&%YJ_PtvBa1E?O8g>FU{#-P5w3Y5>#YRM+ zf?YNK=AfYgPUdxM4&{VCmj^J59v;x3>7V-m4ouUM3p(r%Y&#&o%P*81@YX*E&5J{T zl?D=RbFLrA-~WMQc2aU7B>gPflZZ9}WQIB2JVt%AP`xrZHE+2c(~nNtR7RU5w|SW; zVVA4PRX>-KtRIn^LR7k0M1yv;#WF88`-?$djfyF-T5x^7gB_S$`ZbCwxsW}{;8`uQ z9S>1fBg>;qr)rbit;X6UN?Mu1h@Ng*BBq+|5oSBpXGSEB^M;gq3H_|zlm{Nu3rmD4 ze_l(V6sjrfDm_w@&j(I01zn@*NLE(gT`oq6i&b;X*8iX`xBBpXt6b#zJi_*s3r5t5o@re`U@Z4$QL5j#MA07)G)ma2o0FU+MAyxLSG1) zp{&M0PB{*ABYDDxkWF{irM}~=?AGGnL0Qy$DpZ)%>)e~Z^St$Jvh0)K9rtWZj%od; zTx9qt?4JYg25a`eJUi2y9gz`?vzWByb&AfcowJevn|l=$I?WOgv+C7yW)ln+=hN^$ ziiHvtjmz*!dzlJHja~GiAd0A6$-Rab2uVO=*{U8Z`QDck$b;I%2`||gAW^*jyum90 zSG4#wu|(1b6ps$v5Hs^lD7)G5$6fczMW&R>qC3l&eT8p7Q|=9PYEFg6ezLG{*Q!Ae z+$-BHf*nXX>{YCA8gkEH5t_W{rX4JdX@sW{E{sE=l1m8TkEu)Y3~eAV=aEm~!Oa!4 z+&X(6LWLpO=(O^EZ;v%vhP+~Gdb?psu%F1v8cOu6oRcSE^+tr396wa6-PPYjkLJ_d zdJXH~L?1(5gEXAC0D8+?jMM0M0_j;y|2ehBm z$pSbUz_h4`%RiN<;IBZH`5m?`Wg}L!oz61mFyP@P^P&jSaTVDym*l~oa(8nHW_!L{ zK_O5_4Q6~GS5R37M9UMfwyClTH2{<>MGlKe+pN*=I0Wos<7* zratqJ(m{l$LY*+SC*SAL0uI;)0>D>c&wza*m{oA=-M$kvjMVXw3G{oGa^4~6$R4$V z%nbGd`+p`t|w}9WV*NFs7WF9MY zGFaU34phiT?*cC0!TLr&uI(AU!rJ6P%Q{%?0Gr)hfNSgMLtrf4XfI zhut9uPS18nVMfN6`_@?uIleAa$a#Z1P3Fg4PyOHa1Q7ZHxbUXM_tBhLEB~NK(AVUtMzBsYode(?^zzj$ z4cVR438+ElxH!N(cXGqF_=v36`F1Rui4;A^F(F-pswLYI0D(j*psGucD{$aAn zrr*-a?r5wgH#VwH1Zl!0D%Fn9B~bv-Uu{x*zgWAFX7P)e4mYjbOz2FX}g@4s;^C1*2^~ScW+9yueV^SR(a8Eb88BU6-%K^y0_K2u8v^|wSS1iqsJ?CcDDGs|L&?(n ziWTEkynY|(-9eSok{aD9Y*bBf19_?2h<^2rK;Q!ze0@vf6oSt?k#$IOD`Fc(w61h^ z&FxXm5z0}pdpm_pzR&i46rA7r{;~Y9VeCG0YY9c8 z5*N|xgjE9r4Y}w8Z_m`{pHsbv(1s>A;vmo8OPCzn2l1kQ&P~4{lg0+)7DPA+l=>N| z*C8_`kP%!!WHG`Tv+lr3$jcM&@9zhDlgu3*O~*Tc{e+s$YmKm8AF z(|Pt6F_EQ62#~)7j+2HLBc@(x*oDTE?ROW8*L&8YBk?q{S0jMF*?yD1r~nJ%g(Qzg z1%+*}wdguh!RB}W)fGQ1gkZ)$Ft6Gf_r=|a8}6n4YR#!^3g07fz&Lwqc$9hpNG@Xs zKfg7D2d4v9E@05D206HEI0!Bf)Gq?dsv{5Wk^^9Gf;n-qCI8mwEfNv@6Iwwn<+dS# zqZrvi@d(z7-d;Z3ufVPv&yyjB8*ePK59TzmND{p(dFTZfk_foa+V;G!YlZeNX(+Dc zAfl?3$)%;G_?PZ9gO7jy-aPg`n~>$(a~%K`!PQJtdml{Kae2OC0J@9j$on>LKvVU- z{I;^$M`+yodH-zj5gF{p>&6%r<-n{j{+9Nsf)Zh$ zk4OqT$cYUiDwXp|In39m;)2_DOroa7_dh}sAhR`VVRnaouQd(d3J@G|DPItF` zBknx6U5$*n8@=#=bH-VDVsRjp<$@rThsAv!iO=b+c#T%r)s#tSR1>Y98?c088qnDT4h(4CcPFLBOt+zN;tu8_HuCCLh z8G*I3L})_ByoaUu5M_D{5<4!BQrfTA7JPBADOItiU8GHMS0%Z*{mcvb?r~YP@O(t_ zU5$fYBdHE5EIHa8VYWi41@0iU1#c6?8AevBS29?|2)L^>af};O4qSd05}C$S5sj3! zR&cW@>BCpDOP3*=fj{YTcMwmBD#?SrnqdUQ6J6(4tuR-tGxax-;wH!APEXZlY64DhMA4&?MpJk9@us? z|HDgvr!mOJ?ddPrX0#)qcJv*g6;oAFacsH5sMt6MJ#!9>L_1#(S%C#B3s6ml9l92& zt-(qOKxFk>O!YtXSpl%_&GtI5TF-3o449Go_T- zivtpkQqZM84rt>$tvn6!^)tP>$-Q{EIr`6Di(7s7XSJ`)98CZCHB@R$1=Y6M&*1Up zXQJOs*q`l<^*>Mwsw&GW7OGkJw?4ANkWi?fAu>k^de4bbYCE)n1%3VxQ(plTh4+Q4fJlf^f`U@g-Q6X-bVzqM(hY*3v~-8G zbc3|gy_7T}-CdIV?)v+`H*ba+XIR(SUH6`Q&Ue0GmM^@NMak~Cv`cmI%Dn4amoTwy z7F|-Mff4NK#$1p>1MBw*R_7X?e%NHlqfySJu+KXwoZA10rW-Pp^0b~zx#KPR$DcVt zpYch?KN$)I(HbeeAT~aJ_z~MIFPe6QBE~MX&PEs~gRXwqk58Z!ns-+R^;Yw}Mi~?F zea7X+bl)W+5riB}#2r@NbKGRaB%jjiE%17u#W5qi*RPSN8GZcf!|Oq`@owk1igC}j zR0qAmD+ekOOh(~U-eTn%th}?dOYc{`!@B~cACM($Nc*I=jEP#Ni0G)0k)I=2dbhVo z92^`r3cje{cy`nX2{n07QBOBV*x9*9evGnaVuY^p-lZ#pp&6NrlTHSvo|olrd#pKu zn`!4rmfXQNKxJ+uwJhiY?0TKY(&Hi@7;{`T-kiGg^|za>VfThbaE0 z={s#7Lmc_2!6%n4F5pk$WxTAvCp$mhH}N2$#9Fy+&ri zz^o{AKKZGkMja%?r|2iW?>`1ZK*2OsX&4xNK}aH<@Xai&{3gkyx3|}GE#~dt>*@Bk z^7(yJW8=UI|GQnm`@gGT?f#%1>U7oBg|c*a;NA$ozu_?@`#kT^o`DD_1buwPO+8N_ zfTn?e6ENXi{vH9aH(!QASCcncRE^Odw||c_sr30@rF8?W+!JBp+4oksni;kV!0yw( zt6xt4e0)|e;YticIBsOtw_CtSNI=R3;^kd{mLuD4AKb(UbT5tG%YV;=bax(R?0?GR zKe3Aooc!03N&kv2-QS%O1Daz&-8t}p@|>8{#sXQ{?XBQC|7CFZ3tZ=HwzX&J9KymQ zh^*`LecwO#Q@j6bqu!srSbhWORP^3QpYHj79#tq4-6h5M-eWEIHG44#nHVkOx7?d} znBc|+aYK5{#j6!qhx05lATTodz!Tr01c!XDVWy$I+s*XFW7xN6x+(m@>RE+XASN_r z&(hz@0QBy+u&MX(os-wglKerEs=C%x@l0u0%TSW5JD zCz(dwp3|0hgQ@jQ{aS|A#R*`!1GbW^qfZ{A=Nfffs4S#QcP7Uuao}5`0-?%FB(vju zseZM*5eYukM-@VLo=kB}x6P(GH>NWA`(1g$^M;1AT&Br0_TP=nNC~r++gQH8CcBK4SOus}n9_vq~))er7L1hB3Y+{fi0( z3JDx7-8JSXlx? zRTwv9FWDgZ#KNI1Gn&>YhF_eANYf-5%7wR@OWW}xeAAP5-Kfm&XAGIOw{HFbyN=CZLj9R&zvI&^xFCo{O_!~jB?GIaYAW0) zaKjA(46KfM+k=4(eCajXP7>Po2L-MWvjEtMevg=i8q_WU#O_~#5RnCK_wbu9f*ruu zwkdm`_f;zep%ITUSt>Bh@s}0!_dm-eJ#RuGyIVZfS-K_wYp|_PKB$Q2BMc6>O)FCp z0AIidWXtajwl{#WJ^r72D4;zOB|w9p`5gg2t6YZvmaO(30sz9ZHMnbQ0rP_VmB1!~ zC~Vg`ugQ@0w~K~b3RJFEZyw-@)dl8wa>3i9&NL^||A(I4 zRtfbDFBV=+X$O*Zo!(+Xi-QwIZJp5PFiPcB851i14k%r4B8zp%VAlvm%qoY?C3mDE@tU$7_x?J_q6dL>vwoNXCqjWvqMns*wFuXDtavA4e zwY}$oauN0MP^zAudJaLs2yElwl^W5}5H5=in@NI_pni%@HRW^XNpo1`cit~t1nXBa z`Kicyd0%D%A7m&_jZ*Wt5zIm%=Y{-C=fJI1vB6B{}x2A{+s5%%vx5FCuQAafFxr_P!i52 z6dkrMSU$l(!f_hC(54T#0_Ts`%Ic25-FXSv`u~tNP$61?rL{R#?fz_yh2G;}UVs$> z9hgA_r`TKf`as_1S*wo;P_w~44WMLkLW;_@uuTwq=j7?YfhT(i+)J>CvAdSHo~My_ z{Pu_6`KDc@oyKL|=1o&^g7qF^bVszzJ}Y=xnhpElR&JmfcOC)Y0a$X0D$WU$0kK-75QvQknXsm4_3eYcyxPi06zz}FZsV_e13~(wz&^P<@^W4?;ijot$*(d#9kw! zy1c>j|4lYvtbQO$VTztTpjB=BYz*D)z|~{_MY=DOt1(7kJX3)X0t;i6HsJ{ECw|e> zwynpg3-@>Q+S5ozUy_ut%OyE5^qWV)oG*WaPA?;O!(G;`Jt5~Y1qq5G-s0`pq5Kt> z&;_i|xSzWxh|Bzv$bMSi zjoZ%n+;e(mwNx6iqgHi#!N0U9C>(5(pd+p_ppN0o9NZRH~>1L}u6DdNC%{q70~vIO8NCk4DcToPNkT3b&I{bds4Y2AyRaH~P&kE0R;?x@iM!^)B9SbhMs}<9+ z5gZE8HOmfl^OgB3G^Aig3cdBPd};WOTDZhmq{Z(%|FX~$BT5C@@*`Ibo$j4@{Gkbs zrAcMUcNr84SsKN6KB1!M!T2O9f6=&9M=OpyE_g)-Ry0<0N``s)Vq#yZrA@h2V0YZgM%c))H)VApfaD|)@?K0_C zy(!-qFzy1xpaBDqIs3l^a-osGBq0GCHi3K;Y;7t`A|F>ZkHPbrNbOf9JFI zhIAV;5AIP#|LX}MD{m@5isp0M(?qCAw>Hm+XMDE#EQDg;gJyXUc&X}v<~c^b0`6qu z<(2v`wJPE{fY`QDJe=WomlY+FwaWhpR<7t#gYgKS>NHh)l0H>p$MrL`mG@_{!!1eI zT>CtmfC3B5gwX-}?Sc*fu4s#4nHI}^*&9~(q`OY0waKx!03?*ZJtVz?YtEw^3)6`m3Hq8?2ZUiNcV)IQ2%Ay&1-${^XD$Hj4{8ap!8_r`KtB0VVuwB;u4j#2L=kZN88MKMPwY9FLI$j+5{m|PZ^KmqFIYuh(*_C}@;*(Si&$(2&x&7A!$@)Rl=3I4NL8FdM zq65sa+jCY}QA*fdH8wv_pBvGARYO1Scp#XdciJ64V^$yj17nn0ERB*$k2dcp#^VD; zb4g8qpv(z^t-(|a!<1Vc880mRwSQi}VwH68Io=RAcG_P-@cdOl z%5_Vf<&6(7vrzX{y4B#+61@hD*M{3Mb^G6`er`j z+<|L1<92W6L{3ibsGG&lapEA~PvK^Gwr{D1<&%*21)?D^P-{O06{VF0m9!Wzcs*aY z9Ccu}GXGqjaS^^A1Eyg?<-oAqL*iz2SO#c80I>H{-jm>Rmn;c1^_GfRig(w#I~K}| z|GS5)W*KRm?WH7ORpu^%LIT0Qzz2mlX~gE!tBueh)LaueTqBo2E+QjiNu`cGug9kv za7+ce8;ha7*DrEo)ZdhfY0B!^t=g_L4cWGb?(`)j|3Jnq4P}GK;c|uSa;|x`!7M=V+*+ao5&NV#o7u1=h z(0&>OwLx>U!V5;$2jlkz9LFR+NceT4$MISZiNUCLy+gm-A-19y8@A zsW7@ghnev}Y}}HGU9L@emDJBKF{@2b!Y(^%61`(9igKY&bs=c#6Qu%c6YFiDylP+q zeh^jGfT2v0u6VGpxaTmByX*%A3GW&;Pa9V5IN9UzNeg`AN6*RS{XSFHPZpz}o?;ba zx~_Odj9KA2Xf%ZkU7h|+Rg-+K75F~WFinBL)<}dMGBuPz?Rjpo71t~Fw$z>C^pgs9 zMiTQ^&&7tC;$?dNDW(y6{iLhT5{`FGtJnK-ZU+}$vuc~n{iAwI4D(YsXyT9BDz_-9 z!zd!SWQwYP;$pllVSNx-$NgF@yR!L8klfJFFdEQJXp<(|T?wc4n64g>VV(!ogku#e z$Ik7Fi@qX{qwlQ#$z35<{T^6h{{A`^qLlerO@Frr)f4JGb!i5l>kt~>qe=1^t$aN# zfAittpg*_PoCdds&D3PoLiu}ZD+06?w}q}^v-wun@y(cg}d-YFZ%h9fAUh{!oJ zUS^K#b7#X5^YB!LD(_4C|03Q%#9ayq(iX7*jC|Gky~~F^Aj=o9oWu6!8@>S6+(Dqo z?rVU7t3HkuApIW$8W-d@{_E1W5iTvmJf^=5X zo4KS}vcPB!L9zaC)fLrd#P^U!_Oq^Xgnp#Ts9kFrxw#n+PH81-~$O^4|g2K3(RoaU@s^5 zRX6p_&qIC}f?vMkb5Hm&H7j=TgFFDguZ|TO1>FG zV>B-!ZgQ<^bjdt_sBKb<`a_ijO0Oc8y54-DJDJqI%cl68d7(Y+{SRCdJ}%@zkLnMrbJyS}E_e*6y4Vy%-))B0Zwrb0P{H7J3(uKoS}Q_H5l(M9v(iLx2OI zR~kdfKmXa7{=MVl`O7*wE5fX+E6>NJkP2cGxf%yCw4kUl3j=bmt4Bi}=J#-=5gO78 z1tt3n8ur$L@M*8>sdpG74BU0;4>AfFH|J@gV;N3P%f2otb0`~AA1Gwk%(Y|!NBk3m zyDZ4XXo0X^oQ!eG5^TL8>12|rsSB2p;1eFT@obFg~dI*}PgrMwa=-WC`O42qUJ2J*Lz7_-!rL zd3n>^zvuqY_{WCtFvro^TMvq_f^Ad)tbF}udp?{(wS~1F>2}Ka~p_yHP84@(~SFnx86Dg%?}Tp;1~o+f{-AgE5(^< ze+^4Gt(s>La8?jY%Ff{-0i)cHWX(@ivJZVYzi^NR4dr=}E?qtp zi?ysBQTfPYXfQJLR?y$))8Xx3%B8z)P7yz=Pc@qaKe}WL#J7xm6XuR=q-rG%O&A^H zl`5n;t#Oov?i$)=zUqgEm}KO4huxNWK$S4?*PTR*aB&kQTF6j}w0rp!-JVN@vR8P% zRBp_bF(Ff6bj%rZj0iQ!@#$-G61TUv)C=sg1);EN1qf!-o8a1`E=sgcp8?lFjurmF$D{o8>dzkJb_mBA5}uywW+1PB#xU>{!TcYFaMF9SJrz z$J-dtFjx3wN!Fy8X3YUa`zv9)UVl1(+P1gkxpdCV3fe>D?6qP$lID6(wLqT-Sl@cB zaL+|ez56PBHjIGME!dnIm8&;FTnduIyYcs+;hS?CJfd%lu9FvNr$7dOeckn0uH=W zW5lLs$_MlDJWIvp8{y0(=meBra6Ny!s$Oqz$Gr(MUXn`IoZ76IL}Eo$6^~n_J^g_c z#iQ-B8avEZ7@qjV(u8fcNd@WyJPiZ)nxLwE)2d$^2ndK$`=<&2{#GownkQ}Tx%4{$ zCQxi|bBw;?sxV*4P*tiJB0nRhEiJ6{j4i`2X$fE&rwqvpu$m0Dj#Tq1@7?OkyQ#oZ z#xFc^t`p0}$;FH8$1{Q4$SR7xpzUxH)uh9ThsrEHCa0NbrEmub!aWMaPgCK&fMUM-Lu?v%8lEf zlyjui_)F7toEE2Zeh29^^NF6Bd24j>4@#w=V$oP`(a(Io#?w>s6dKpoxrN@^+NEAI z=9`+D291p=2a+lJwpChu=|iOr!Rxf;GB#(UP>`eE*~GQ;io|_e$=e%}^nN%*5u(Az z;~N`%#TS2abo+n-r674Ny}g~OuCb2PF6(s-jJ$hOZj;OCbhl*phs2`DXSaC;=k^gA z{qT(e8{z~rJT%{*BRJg$j$dr*y(6&b#SQX&!G+kgoKS<_TZpHh21zbS!F&}hiR~dU z%l#fx;O+0;zen(Hnf0XOU)&EOA~nCkgC=Seoy_kKzrC%f%=ah#w;VRX`j17=ouVJ! z+kU^>{(Is=1LQz^EjG+nbS&H~A+FR>X%Roj4G5tCCtY@;Vp{mj??H(D0^p>}j=x7R zXAr^{ME_6nZvDaE-*C=(Sq0O3|5F@rt?`xb|-sm+oF0z^|v%PF4^Lbx;Ma4&2-4-K}7JhhEdofRl@> zTV1f~1hB9FMe3C!usPjJJHPW11HDVTa-dR22V02-I=l1L!-Rh>k2{CRcYoo?e-T#X zV6XnacHg(t>BPQpKr8xN7|v(HMYrGT{?V8`o+<~i>pFUF%(zU(uv;X7OKo~|cQj`C zWxh1lq8Zzd7?D*CtRfdrHy*p0n7*`%g9onj(QI6&f4)44p#?d?c9j5mS112aTA%3h+sr==x6n%H@u`KRMM2pk zlW|EKEsP_2GgE9+72|7t%oq3rM3XC4`ghx>K9`679q#^aQAHd$=e)$GCK8R`^%+Qc ze<7`vbMu3m`-*QX4=Kix!eSX_Mpz+Nti%75j~8HIPW`;!-EsUG(OEn|>)#LO#3`^s$ zZu8;|yymiyE%w!AhVZ?lr1{jZ%`=eX9xd@y5lae6K?T+Svd=|zv)SEoU(v{I_(Pk~ zHbDEZjn(iaYbj6oBLbwe4%CZZ#*elV2sBMv1O1OM56M+;YAlr#8N)V@G64mzxv6|8 zA_N}ZH#AUOp6OlS)#nkRqM@;VJUrCb_o68aq-AJ)+jBH!EUo1%Ow_$T9WWOz>Nr=9 zJTS69A)Mtog`3!)Bz=>*e8c{$}^N z`{xhktEG1D1C)xk>z(hiGl>qsMb8B8{xe}_`!1wcb)&&=x1mQ5384}AyTQ}Wcfc;J z0_LDwRK#yIH8sEIub!wDDvVjzp$dBggLQjt^-~jK@V1VCzh22Ivk_>T58Oc-c#E0I z4`98%|7~qmfGYiP67t$_)7{Tvf$9mXhzJZgAMp9z{{1dX$LqQ?`Cax1@!9mQ9xteT zl1$!RvEJ3ISTp=Q&^UhK9z{R9?ecdProRdp!)6K@{PK0$h@f8J;pE@3-1&@PFm(TX zqWQ}Oy{;D_cRi+5rJ+1Iy)l=1a2MWXb=zhQr@JriYhV>eTUwFr9RD`LxC$xAWjukE zD8>!PYEL~5H(T5^=I9k*!x<&Y>`77zO=zGB_As zJ7*UXNAXS-s^zrJ-Ydkj^6)H&il+<|sbyG6=$r%}DUgn=#Z!Xvb zICMf_P&)g(FY;KWwmsL-y(qd&5E&Q$yuN{&8y>M~bpeiMT4D4vf{gW93h6T$4Y~P|c6KU0 z?MkUnF1~`vY9{$&QaDDrPdDCiY&`2DriA zE9UB6jm7vL*;uDf%p@SMQF_xv5pW6uLL zht~5cL>;N{S!Y;fUM}8S^XppepGA)ge+XiR4PLkd(7pfePn}Z5bcH9lX1@1zbRdD^ zLr+Xh-0hF~J;rYj>?;v;#qJw0`5GZ* zdi4ef?oU|Yu?Y6jKGW;3i!{d8VJWF}0vg7 zNuZ%u;CpsSR5tcdx-@0j^i+QF+? zh2r!kT33)W4+4$VF3!fgC|_4RKq1SIW3S_udCn@(9Io9uvan!JT}U0%0fF@-%lWjq z)mCu|q=uL1DX`S@x;75yClI8SG52T||9U`I-Q|WwXDGK5yP6Or8c1TqV{ooZ6co() zt-5L#XR72f$9zP^z$a<$S}X8?h(e)~z2h6UWM=t-NEU?Ur)4+-c*?n9s8RTjnrfn(~@MI3Cx%411(`D9AD!zgWH7 zT7KO6BP2sJ^L3UOF4}ObaWLkw=WqSCFAeg+>*r4i37$XOvwQ!(ZDdbU@~d0ogT<^P z<61N!_H0i^Ta$eUEmUag!RY2yg~WKxgXp^|1AkQfhVPY=^tkd+4V#E5O@wZcQ$92& zglx-(qsMZbQ`QH@pQvj+vV<(D4B{G|?CuhGIr>QWe!{l*Lr%J0Fe%325PA2uuSGry z_R)L9Sd5Y`B)&+7{nxkZ+4;4F?RVyz8!5@@KZs;gsXyMmcgsQ}Pk*4-_ghEaMSwLp ztb6W$jp6EVlSmAN4#!JPfY@!uu70Ghw%=~yIwJ&o^qb61jA@Q_vOOe`hs(xJf`+@R zXnkBejGMp?X`>K-*>wPh1mS8*VolD)Jt)K_`8w+vXYUimt7lqt)u5^3rYQWDOYj5s z^fZYO;?ixL|6j2n2i6rsH85d;1NILYDF@!oN7X%xgR(5GU5EDvhf&QD&50Rjk1x&q zZW{c(7pQb`ScDE4e|xM(BI*Wk6gbUi@wHko(*@e4sP|`EW3Lk)B(C1L!!nEup*R9k zlPyFDg9}I8tX_^Gg%sUV{ffiB8#7l|*9EEtg1LQehqfzfAYFP0Y7;I?+us$Y3`Vd& z)u-=K>V(TL1Jxe_l|e}T5V}ExvxZdGJqWH{tR}L51fKUZ>grI51eUQe?SD7zp7-~5 z)Bk?6LCy!*H@5rA);lu*H`O-B-kt^rFw(uA{jxb#qe@f%uPZ~8O498g7#*54J2h;8491L4gBNam+(6q@G0o8{5{4iEiI)8iYozcohdO%!(%-`;|j2q zDL^HGa{w}jM83*P=ktaSQ@D&-K~p<2RWS4I_MR6Jjw<>@3gel+$s3u4p)?nK0W9tx z*@K23Rd#qeQKUoAn@iNxh4UPbY2grxdeHQRx*F#G*b^>y7;kWt_7)|=$aOd~&-Fmt-VU~u@RmVq8QM5Q5b ze-PGfA1sO4^MO*G6wFOSsI9ASHL*i&X;LIfu8kf`$rSa7OkP#Rij`nG3OV!_Qasa|DZ?X~fXWnd zz8yg=%qhblip-hbjQ|$I!z;$cglEaTa8)8g{lf8<5E<3H2XCI6itCb>>Wo65ENOk& zDqqlY9%wb8Y9^5p(*{H!+2TiI1lM=G{#d7l?T~rq6HZ&mrxRXu3NczxiUDfGpKTF z^sB(G?Ogmf?YEG35xE2C{m7SbeQ}GQFDPvj&jdYIZnDl{H(zx!-qq*7VScNaG?CSi zSrTKkJnwOeWWSZZq!9F>zZmap;cje#{IKx^`lzF^maTQfBPu=US>(`sj%j5798U4F z-%f%=X@f!jtYWo94&=H7MkM@JoOH1uA?WT@?bw z41TTyT*1t$O4WHkW^@n1=>g#D$8&%yeR9(dznR=UcT948hj2UCmpPs-6!bj67mybn zn%%~uiN88YYHu-t<@O@)(H5O<7iL{t4%(Rlv2l`tUkdP+YJa(lVgY!SOBD960nHf^ zp?K35wm|}01pXjz!E{9Ewh$Bo{=z0gfVd6kb}FIq`L2bqGq{V|w<)~mVS&|EegE#E zA&0d7UpKKzSCgTxesA9lYtV6+WY348s@wurn9^r5Jn2t$4GjD6k@4O0zu={li~RjL z*aF&(mb;5)-6oe5QS^;X>W4Dp@w^Ir-WN|~We3xReLra!_+|j$sOENsg=En&?m&-l z%8*dekfjaF(PB$r-k`^_RDsP3N>7;)>TNmV8+$N>oLu>-vMJsv1yg|{&f$l7Y&6tc za+pI-_ZIy;c)ExOhD%NSja5{Ar;kjs7(1I%TqX%ahceN8yN)}ll8aluN;_9RY>at( zb_In^tBil!^YTft@*|@d3Mv&nq_JEJte#@VNHu%~ZIQxic=?{gvy43P2UH?wyVg_F zRpM5Jch2k{Y3n-9?3zD-%%n~`BW3ob+u`hB_o+pr&o=Jvu1%MtZXUNR9k%#IK07;( zY7;Do4b5ipL`f#OXU5p+L0o90f?reB+NSOTmB1$*ssq1QN@5>WFl?ScV!zfMgzu-Q zxpS;-k2PYmNctto)Q-s{Py{#bs8SKGML4oV2PsRo>cjFZ!tyqE$+;4P>qrU-1k&{_ zYrapJNV>d4KLBHTGVPu&(fR4oylKul;!eM#3+&cR?Q#fxnx3> zFc@@i>_$!0E7-E9HRZkXs>P>YFjt&go~t*uOsxjo#R*n6{A^3X_;sXSDB@t^uD`0? z)6_PWS{AxXpt{pIC{j`H$Yz>DK>VsIGWNUR(J?pPc>FW>!R%*3x%KgBva|Vj1epD0 zyKm5t1FAl`#R3$`v7*=U2_-NyR6&OptvtG`-K^G~Ps&0zH!rZ_PIKYct5s-V^yWH| zYIf|fuIXGbjignhjUnv~an;3{z1x~QIa9+5GBj~c_!a3&U9f2>CudknCBGtW++Z4q zzc#HN@!g_~`0eZ-|0T=3C3%y-{>ib%_TRyzUe_*UgYyR8=jdK*DO;EO-G>5_f$F#g z*0E690|UhAm6Ofor!JQL@IMV~HNm3<@3H<{*I@oe3e z@(C8-ex+8fb`1YAgy`loGd>b#f&OLl=`lhZ0cM&H8OhSFC5=8;Cjd`aKtXV$zsr8N znT7b@YHt|a5Bl?g+T#AQ0{H$Qun`6C0*7AM-2yDKIVc7=4lTf|YBuX@;(|k1z~lmE znbg@L0Er+o3;EK3$Q&Qz1uCDlKC{h3K*AKbZa0X>%Ml`hjVvQ5Bkt*cpByd0Z~WV> zj<38y5*AbO#se<+8C@KfCqXTiOq877DFjErR=TO+e= z_QSTzhrPf3J|jFJ5Og#%Uz3kT{d5QIa6h8+5Z1~(NYaLRoOz9W^XrLiH_&$RTpj&I z|LwK=g4E9=(BQT^7dR+%g1!iORk(DwOWYePF{<-~uTn=!B1wH)MMyr?&;^P+G@9xg zF|uHi{4KzxSR+QCF^uaG85_H(rhI;boZUL|PLjX0$ud{*zATCgnN3_sWuO|ptcTl* zz5#8$6c)FvGdDk8hU-p{ZCS>98Y=eUuSBka{gBB8PU{@hfDViRe1PjJtI8V-MY09R zx-Nd&wQm#1Yt^xV#CD+@8TAbfG1hr~oCQ^4cz@&e%9)Xgd34TpY`2@13M~+uELyRxZTu4Xn*zof>-K8g&jT!-R!lv9q2pO83%403=fqS-N8bEo^? z9VCSX#EruFm&XiUs4qUzpK{b}2G+gEFswrPg&~$3rK!{FV*G= z(OJ1+tawGGC*LqPV|0g0%Z7@vqW9I@^q)Tm^fi&uwBLUrot?*t)SuXJHxWoI3Aa9< z-rQloA9nEexaoBL5}V5r&aY?>14%7`qRki4UFhQU9juaQ0aJ8VsX2AtBPanAmM5XC z@hUo|P9_VK&%o9WXu?y6<+-`1*n05C&&~}?^MA=|{@xn>CM`Efe`X>5GO^k2c*;B@ zZOirHpchB&6 zniS-aD7UdoOxuPNE@k~)3sc_Fu(aplHv5s0jf+N)7bY#mu9YCC&b8Qu`)DO`veY!v zYyBbpvhdE>9cT05*I{aL+ERIkpCtc|+-#%A@hm7C2mdtE0PuJ~`kAvBNQ(^@s66bE z;Gb7{M!LfZ!2>b}1wpc~5yL`i*)&4QquFwBHO31!BWE}4@6U@?0Z^(a$#l>N5#LH<9spD_86cb?`*#H zLjqIi0z>&Zi)YAWZH8i%cZ>T?zl#seL+tS&qs&m}yi+wDAqY6em zz9!0mHq9^XkMRlwNhw{V*{l>~XexO~^cW>5l(WuCP>n)t%}1K(>TjgKuMyC&5j?h0 zWn~km0fXL0qT_?Nta#osG#BiiJBLLqE#VRKhFI#sDYtTo>buu}I2CH@pT(dpo=Rbi zGJ9(MiOrPao|QbVrijbZb8Wy)L*JFrl370(b)9m!obU{O5~eey{Sl3l9Ro+~1G$^8 zCG)0^Ol^Kn+MWX8w-oL&moQ1|JT*z958c((V@5S`PM)p3>*uJ?4$7+h3kFQ(vrTcV zyk);!X=8Wj5yi+Z=P>3qWz};sH*Nvd1}DvgH1p{qHuzl8XkRe9qSW} zO<81QR@3Q{3vIqARIg~lR0?9`$xkZMV zUr0%T-QvGS1&^yD7%{KRJ9b3(A`2*T`hmqu4*+dp;Vf-^{(qmre_rx|PAKuX1CwR1 zPk7xQeh42%U^c%;n*`|guK;x7?&8y2S^2*AV!cra3?TjR2E-By(MP==tb=y0ufI)Y zO*v~U##-%-y65JFp0o{Ye?pG!OrT7~IT?{R>(PIh7VV}`)A$#orf&6ifQ8fJ zJv_I?%{H+n$;b~bA$Zeh;cko2N?Sc~6apM>Y@@XxUOL)vZEm>LkZ4W~UWRJv)PdQP z9-^S=Z?qJp^4M~6WW=tfbzx}8r7q@G&yfO9FyH+$X4f?0JNTNi*Kit}pB_F;aE){p zy^v1xi=bDRpHMs`;FO(F$v_peJxa&0dG~<=$#f%c0lsp6$8X=q$*HMEm$xL$nIgU2 zn44O9TI5kCdH7n_1+o{B9oTdzRy&f;ageBSnLskKFp5A{N;D4gnpY6d$Te}haRn{i z?{YRZ{gZ1?yq~=?baIMieFCYIF5K?b@f(TKfkZnQ`ePyPyW*8l(DA<+9`x6j~RWd0f+dv%Auh_*R zT1YC!36hj^kdUbhv##t8DQ?Wb=i~CR+1_7GlGai~Tg8L(&OPTI3kRc@B=1nxg*)Kb zGn3l}Vo@aBYrks$p(&A?=TbAFQN&-BCKo}VPNuAxy%k>P+5Uc)8ow&v+uP9Vqwf29 zBsJzDq9C+t3QBSGAV&H3+4vM_3r({2 zk6CKD4q|E9w@bwzjc(?1nq2ONJl10K?i~%{LpFO-VHyJ+|MqTFl%*=q3Yto7Da6Sa z@cyzkjqxY`_POU$VwR4{6LA1_V{mXxaHbKLmX$3=|78sR@j{|z!o;(t)3kL!Ef-1U zx$oc)(eFMavZ`3o`Cl3GzdZ_s&Y2X==Doa3PS1GrSz#dI)WXn@;?MKZ)iE=fybv-F;(iCidNBr=82Brn%ase?Ek%C*TMAXo8aN@s%?N zt=8LquWgdc|GI?m{|!M$KyCV^Ifx0Nx(2gL{ps~?#g}H3FBXiY<>gV}0U#ckDQxyC z4+l%xe-jU&!~qPKP{8pC(!U*0;O3hV1v?V~`N z{%L{P5BN94A;_nz%qEx!9}9a;pZgK_Hs|n?M#p=g7~6^xhBNCmx%^jYT09&gr21!q#+Dm2ttqb&FKj%#49Q+#+oI!hiiMc=U8FUI%Fujn0YMUkqk z!pxob8%_yxlSU+dXArRD)(;e|s;Uxeoig-o?GDDoWMnF%kO|3oV8#hWr++S$N;Hxw zG`u}w;Q?s#u%4wdJ5WTAv9Z;PQR+~|$^uH&)8!t|=qgoT)h}j>$0(_$O@72u;Xszm*M>gf z=`VTDkkq;{<g0&MMSY%cAY(?wt%`Qf&uWn;T;G3>JV z>S3&5VQcGxzOyo+Di6iJp$C2-Q<`Q<#)_xQ@R_!%`|h%gB>NMoY>L|X^mH#ZEthQ> z#WJHnDt1!=x5%ERk+T4p%tR=ujBmSsf4?kPsF=C7&{zY$%P!4&wNqdPnhmAf;R)}f zU0K+4h0MbJCYhZzg!~`Au!G=A6qA?QE=5bA|N0SHmoZa0`+t~v>!2tb_l;K& z6%df_c4;J}yOCvK=?>}cl$MkfSh_<%mXcahLZrJpq&p-fg!AD0`<`?Da9|i_kYS$t zx#PM%*Hos#krPmW3a3&L9+D-P^62^E#_<&ol^)_4wPrznxp4q?os>kTI5!CQc^P>_6RuHkEY1gCQl4=#wfyoxXpTntj|sEsL+#Fl2W)<;iFERJ+jrnZ<)P^He3rn!;F&W#R>F$!Gps$kQP5eb z=DSm1G3U_6Ifghd{1el#JEwb~lOI%+B*^Ez=#byF&k6Av%8rB`FZTSU9flq4H<{>z znQxtp*Ir*;nm^$Lkx>zOJwAS}^}o6-ACEmxAFbd($Dx;+nhJaX+6La=fK2tVL_D5X z9y^srx<@=ekyE(*d~DAIBt*^kT0!fMdn+@%h$!bb&v9 zH9O$mtMzv2-?1nRs5HM{1f0Au{BZd}xIV_aWtgw^H|C?AF19%l$I^_< z7|wVDYA>NZfiu0f3q4P_0K@c)|5Xi#XA6Hf{}SNd)*1Lirst>Y_3Xoiu`1YUUJa0c zsiUiyscW3)Tx2uo2(tdtNzdG`hVf>C%U$#@yv4U;F}Rq#XuRPQN7)nQa?Ls4T;La? zTmv-xTj#r6Zh~*rjp8J+LrWO>pxjFi9e{-wP`GSR-LEp--Hkku1G>S7%-8Qke`JfC z0b2V7Cw;zfXjK+Z%8jt3O+TJc%vk7$GIMd=1w~=^SM=$Jm>6a6Km)&NjDp4%BQ`gn{Htl>;T2u9zWKPgGeF zVWA&+X;H7z)Wr}*iJer3#CP7IyG1N}G3s3v9`5@pA(}Hx06AJQf`L_3s zoE8hVj+n%zoZa~`G~hR23vdT>Q{3%&Gmj4X#2L&1m1*mV2*Nn*ksR`JBH!QP?sHLtNoh`u{ zrP48ieX@8I3_1^)kWAzV`f>X`?}YTX#Wump1*jN=QLxBrB}NpxT0CAMrh%cs_DSk74?~O?MFsxDxsHX1+FJf*^VqI0lIiVHZEIH||FIH)$LR ze*bguxC;UHvYAK&ANs9J4*9?)X4omvK5TKb-4 ztdM&$0oBd4{gI(V!65qdA5qFZY9!z~CBQ4Aq|yI=L+QTr!Dr&XjSC>j>*XlJON>r7 z-ZpO+qr|0>wMGl&S4rk}sZz&cW0&8@EO?kxfYs@xkjc@I!;tI#2!5Gv>C(pJeV);< z;_9r3o^KP%XeU1Tgaorpvs5!J-G*bV7&nY zKKS0mx}_1Ps;HGZ_uMev{yvD-JD}pc|8Q0$wM~23H5pj(qj@ z#YJH}yo7^gkr7}f5(qdVAAms4o0gDvSs&7pfx{ou8J`mzE#z4 z^^Fq=n}-{g&-wzRk<0FbAVUaNxJcQ7e zpZ$GW#fOc9mfQ5HE%l$vwP$(?nF@@!8cA)d1R)NT?iFY1mM=AX6)qLyn^F|Zup5{a z(hJ2trQWbLy~q|f5l78W;zG^hpKCf7P9Bf=VGX0jkb7x7szgvXZB*Xf#A~G6lF~IQgVqrlWtK47D+O5 zUrH?&x%#YFiyDglw#t2Uf?wK}hd}MLt>_ng)aV&dY~Qwdb1^ z&DF@4H_2R$k^xBGS55GCx{nV^*KUYLYnhyEf*fjW4Mkm>k3l)uRqxfalpSN%4VmX=mHWiw%e z;ONG;9-9fmu4s!i9lEi|D7+v{;waEXlSn@LYvK|(-lCk`44HX1doO@d6~Zk0^5@n( zuI7lUqXM7^>=vGj7{tvQ7VgTuB*+NygM&O*v8|;L_XjQ4CT5lJqNDS z{a2ex&k0T>j(E}LXG5UA>6=?C6N+6R<-Es>uZW@$NWcY&|^kdb47sZeHU<*u`k=X`2!rMeS5C1 z0oL!*Z@+&My~m>C$@et*G^ve+HRyEPe6_gNNJrAbYE06u@i|k4os=Du8pb?^$xd@V z;{p+1UGGRCX+3cs#v8uxhZ*hZ;}$hdWk@)|2SGSoXn2-?;E-t^b z9#^i;H#-#qINvkTwHt(>zIC#u?m;|Ex*B7vD8;>f^>YY3$$Yz5?AY^j z8o)hMHjqfj9%jc!fEi=Hf&^@R=Xm%=y3V#6-F^#~ma=p@zhVMO5L*P<0<7HIwv=(Y zZy9Ty=moG4V^qnw0w$ILNEs`8O?B;HV~r*MdHd5v&mAO>!+FI*T-+Wo%|;~4C6ni? z#*?zd{Ds46r|Y+0M>5wI)424TKEx+&nhR!O1;V70(3v4#trTlD?Itmho4RgA&A1{o zU9u2m0Pav_**bha_?%hWndy*M=^I7oQZk|Gsc28%XCoD4+Rd%Pox;jxJXQyELfxPX z2T37bD|@s=ar46w-LDaZ2#f7p_Uxv62{{>T*^nesNmRjd`j?hszy9bw^a_2u`-q65 zs;;8CoT2RYJ;J^k=ChiV3PE|@?xt$D*J+m0J#lX}vBw#tLnm3VcYgyOkU*MyVL62D&5sVIc zZtJ~c*AdNu8uiNniIGVhy^O%kM9Q)J-Pf-=)^z%Y-<($TlSC$PF63g^cIcrHQEOSs z)-1@qtmQ~mY5&M|a-_9HwbHlglo;YT%y4-AgU2sL>zwPK4j(){{@l{QbukWojILJ? z6aASk#w1Zol9*$fEX?hL4ar}JQiE|lp@ppU(@(CxJMWGrW>5WX?Ti>jE<^@sAG3>g zgDBdU84K0YqVxj(2g5G)#Z zZ_DOVQ}C_|er6R&J@tQwYl%M6!=q2NK0RqAVqMY~&iO`H4Fl(o(~YIw|2D>kZ-EKP zGcqgH2~CLWdU?Ik5E!Jcol{QYR{;SW>7xdgVjIxB1E28?0u=mhI+@sQQhx9AozHmt z9_WW284u?<_Y~sreCbz z8W&$C@VD+(e32x??>@Y0& z24oN;yh^i9GAn|$kyVfc4u?Z`RbQqbWN-=JwS!6_%~tj4`rp5!Y40IOFT7(Sx>11P zz}qhVYfNzVH26&iW*L;@`(*pJQ_mkF86}Gmioa zx%Au#ocFrz_z^(LguuKsUg{o3_>%}3)MwiHW=XQ!+%@}-T!#)goH^lcMD6)v;~o#Yu9bS2O;h1 z7tDW@SJ1DT_LMnEp8jj!J?JH$6sO8l6bW&JZlNl?s&Q`P&P%JOk<0SK&YBFR_a7Xr zMW0tO-T21Dup%_@nzV!M;b%g4-D{;OLRAyD{_bwU?UAhQpRbnJ&vz&6cr$}I0lQFSs`E+V#y0L?Un9sKDAT&p+pC@H98xg$bQ5&o%m6)xl z+7x0*M}w#w3oWeM7^i2grz@Y0D3BMYJAkCqMwz6{WeNGVWbvn=ji@}!LmqD!=q>01_aVXlR z;5Ga)*WYu`vjTWP6w84=;^8m~9bEgz#G| zUU8Vq21#?;gmOW~c2EHQx*^qO)kM2GnP=SL@Bkr~x7BrN=18E=WXr4kIB!Hh^S-tz zDlaog=%p=@F)!`9j8S#6$TR zM?aMt5Mi7_29%JHluBW$cOE7gIqB*Dip#q&z+dzMXoo6-0dvpqo`AcaM;1TDe-l(c zpa~v$v=S}0aXlQ2+#jTOZ9j@s>KoH-w{73w&i{Y@9YC_zSx0ypF7qkj$-9Uix-SZ- z43@@R@dA8g@94y^AS}osq*;HoKzmzJmL1AEPLxiD{jhh9t$9DGI97k{ZEtDL6L>9E z;pH@Po!9tl>9hH?u2u;?fdr&j+J(q)6G0nZC!UG*Z?Ya%Ig2ZwhHjtTJ00v75d z*_RUOq`dKV;Uuu!z(iWeKE8~}SAaXIZ^nZ4)lNEP3$NEaPuR&XF<5Z);B50#+wAAV z$FY1=z$S0|IJUPl=voCL(`xj67{>dI-67v+gGEp4QjbZl# z>4U-1ID)c;_61{IH5$6XpswcxFoP?a1YTOjxFI?YQuX%?y$M$3oFH(dB5p0G&b$-b zS<{B8-Z?j)Gdsr0yY6HcW2SD3>ABX%i~Wk2L^`Ai{1K!laQwIERs8T}!kG~S5hQx9 zp)*Xh+obu&QOj*wnXI%VO?9V!Igg}#L9NUuW^8C!cV9K3)ddCvQ8TM!(Ls)4Lx5+f zkB5xH)Fh7rsnf+RbB^mz3d7}BSoFBtdh_@47!k#DN;9o=Gt;XnR6Ach=;dWFVkV6C zK|s9EzC-YWByg@OBQAt3+T$)Z0wfL7Tlx6lxIsf4R_ThltYavzvq8^9a zl)TzVh%MD0OIfDBjRjIfrbK!+dHL$^3nh(*?jL=ClPwMUwQCp1@NYTbK&2% z8Ni@;Og-FBIn-Z2S^_?mG`$5FNso~zzzgty4#W?_d|ev%vn#N)B#fpvUg>77G5X?_ zemGV}ROS-KQ5&!TK30cW3HT2gJ*;wDNYI7Tp%?CQ96y(QG@GdDPR*Usr`TQDJ~|4_ zqGxcVg&+NN;(;;obwh)09Sl?%rVd-qM{|5X)s)CLV(Lenf4h8Yy!QEUvpaIKp_a4u z>qoF0O{3d2ZO^iIg2PN@b#>o{bjML6DcD3?+s=9I&x=eU&(B(SCC#E=MUnl)QKP;18s5e5mN0env3fEqdN5s!W3{g=p@b?6(~( zqx>?~6h4XuMLu{*B_nA}AEaZNd5}rh({#l&%}+8P5r<9?hLXfAQ5c49P*RTl))f(E zU;5qu=1?>$4FT_#kBCplmtYUXw4W6pt>96XNGb*i1KZLwbS@+zyxctHz3ODH*~_lc z7EP>o>>7DlW6E@_Olyb= z=F!^Vb!8qXEQ)=Skm_a5&iSQ@z(M3#$VCIym)L2vpdgNLR9l`9)!Jz-{(!qT%K?Yx zKvQMPmgS!jvk9T&W1YJY5sCt?iw|@MyYrkJPz#{yQdHH__@~|OhY=SjlCaTGI z01P^e=B@1nZ?hIA^Z;MibcWsOzU5(vWlIE0^d2Kt^b%Bab{h^bI&l*^m)6pS=6sYd z2N{4353C@0N>jf1tInK)wstIk1B=PALTI0^zr(u=gUd+E>Ya5+bwIY<;qb^WM8mS^ zfdu#v6=O)ru#Wcx=VvSJ48q3=>F6fZ)QT%1U&&sF zFcnyoV}fzcaCoO`UN51=EYEuAUqF8z0dvtR>{;b&T3e&_+9|DS^6E#_DhkP@- zXu048*fgK+$q&*<>1~s*n*~kd+o6@hv=N>k(o`-I&HEO<92z`*Zg+soY>#30F-aSP zFa{f27Iu5I)Rn!&C4AHCzx?6$?J=bzuIAP6PPLr%gdE>UU@vb}z6qR*P}O_)`UQ!G z-<#CTfiR4{OC-w)RlkMm>&|Wfp|D@S-yZoqI!_->4nSChHDIh%IcmAeUht`i-2_8oC3RTDg z5odv?1cy-Q7&_75s2#v!8yN6t)puY75O9YUC?zKb=`}SEF?V{R2QuXOFdybVZ($vS zvDQ#Nh><34T<|zfU%53O>Y-MCp*hfBpRC|foy`4tEX_!-bP7&h{8mKd=Rt*meuL9z z2oYOlZG$-ZRroq3eS(mq`Ki-&6ND*(nIKX1Ts_61{%0XjT6A^SfU8)hkZbxoO20&@ zPgEvXfH0AMA5=yE*|c_A$6DxJ5i@RfVJGoW_YM8P07SpZDL#sby}Gtm&bB53J4muft-9eZJtE%7Rg?QyMatr8)8_?=A{(O`iCs4)J*pp5oMFxLG40nX$WqRL! z{r)ijivRYWjP<=>6eqaO1hx2Jkc9e~z7TwhA#Kr9%zFkeGvVEb7C*6aQt~*igNR`+ zhGe*VMlYxfe}NGl`KF`Br_hL7wMP!`wc?(C5n2!^BNilHKxl3vjcsgkX$<29SUC=4 zAtb)jk#~cr3jY2Lp@+GdgL>KrM;e2rCOp%#_Xy6kMb>IjA)joc$!S|aN{)doPjJYp zRKuIbK?-`qJDjz;G~rn7c4UygowGV>nzP!Tr^La@rG(%N#V3sRcw072O(%&({|3So zr*n}tJ8Ez^c2K~|hj+9zxhO18<4Z3dwa?9$l5%F&08W zwJZQ8e$%)1s75^d-TtrI{O`2YxULVWPU|^S>0p@j2HGsVM}OsKi00$E$Z_vN5J2Gu zbhh1kJ~%8EVw?aR+AOiVpJLzh74BL90wtEFn>lbq`xf1dGGZ_YQ15#Fm9NLD))VJn zGXIf-le}CL8pGn#`&7(6z>DPuS4Jm}88_uq|19*rlI}-ICc1tZS8bJj!k2HLrrGXpGu<|H`5Hs6Z?EU!f66+Thm77A1j7TN+)t@ z6Jx9aG+g5y;nXE^2XWavvxuJU+uw?&h+(sBlmx-wbLj_3eD4cUfood11hxjS3bWrQ z{p|Fsyz)6WU>2@5Draa+m4hNMds0++FX(T*A?gfA<`ynN;*qR}I*EpKNZ!Bvvfs!* zB=_EXHV@Fr)*hFJnj1Q%ZrRg7!WCdBn;%Z?E6RbUG@z(5X1p`an#a-wvdEw zJ+Fk;GB@u)$S~C>xxDWo2#DBRuGeG$rB?6!Jada-bRtY15edzTwuRt5na#6b!pd8a zq^W>%(-&*l0!we>!XSJkIAK~Gaj@Z!5=>>Nc&(-xvK76MtsX3E>9X*NE#^{ZHQ`T- z5^)rDd^xvdLt)>M^TO6ULG-k*LGMtEL#CJ@DN4?#vr1mamH|BxrVs5vHA>|ks)M~8 zxmGEfd&!(cI^lXI{AWjv{CaSt>S6SCaX*OVx#4@Q%!)xLrP`Di9WNJCvQ6k)qQ{x& zAqDIk?Ft@jhJvzt#KyQGeS2{<#Rn)lF_qJ3W7e;%@xu2~=bnZ)#K^G+L2t*-y997P zO6ONHhQu9Zo`3rl^Zr(KMKAOCK8f+>gKcJ9Bv0C;_1uj@y`Puh%ICyzahf27UwqwY z1BDw~a!mMP!#C!`73`HRmj=Tv56rJ9uaQ5A{?@r%en6dQ_V~g~5XVV>h^d=lVvAm@ zIy`E3h4FQrav*~#g-`Nm#b7gg4xKCJqcUcl5Dg=IQ;XlRi%Z?8y5QV4z1H-l9z*9Y zgBYaZaFxSk@AKUcSS;F^^SZ|^#SzQEaXq;v+z%sK>yz}#X+^)Hyg2@I_53(!vy?(K zVszI{Zi1MJK$SU292dp!H%X>hMIvK1N%!npi)ths4xT?GbY5hs7%LX#YSVOhkB?W= za41nTWnpX{437}RGoB^NxFsY0ZO4>@vD*E9A%2BQ5UD&3i+}*Ymb{!~uuQe$=8$Z* zlw`_kWED@z!IbfKsT&lhy%lf9M?Jlzdnm4b!%`{LqYL#qf{fR#JS5f!JRSMEdh20H zW2H_>8;6O^Gt|EYsr6wPE&2((ceKe&#}sCAWc%hDqW|^0eGyScK0zWe`4ZS-T-(0A>h=8rNx~N5Gw+KJGTDE_o*};RPXuxMF4j{<;fcCT{Y0K_YKkU6U2jd0uePoGab7`EEBq1F675Un z>}OR#wu|~RjvBQRH9dp#5IX41W#LfKyR;omI+g|T z_Nhi^g+sZ~!u{v1tG?-L^S~1_5#0swglxqi=lE~X%3&BmaOHSPwuX2c#rugaGG9l1 zL(duQPx?frDKl-K;U$StvxW#N<>TG#ZZDjxTQI;nRs!2@KlLPy+(NQG(wdTvD zNbjnw5KyH!><8K#Jk75?zo_tG>3wGQk`WH5o%3|dIeA2e);i3CsC&l+5DC-YwA7hB z1Fm1xv0s)@J4v>1%6PkCqsTCS1>*@WVOb7mTG;yquOGMdNW}{G$1`Maw7#IsH7v#M zfq@|8E24i-FM%gtQf{2v*z}Q{`HYQS-@?y4NwYbo4<;YT`ep`Z!E>b)J28BE*!LZ& zncY#bs!^H&#TZ{j16szT8BZLgD3odzQ|lVYFA7mMD0pifS0ZbJ48yMtQ`cMTgxToz zW=sdwA=`>q{|IT4(<(Uc#0WzjmBZEd-s1&~izrFqj3vg==BAstmMZ~So`%R_K z040Zg?@~P~!-M4XCkzm^o*8{vl(=!iY~OEMHJ?IhrfQ!>3W%*=L_*JfV5lpV%hunV z-|cug+b4KA=EpzY6)O$pcRVS@bA#F4-Z3_}&1>@trW@L5FDW$ElDe)OF~9K8NG*9b z%t?f#s+%|o#aeP1UNj$@M>v zq-zjwS1Q~4sY@-^h^pWfOR{?G5Skd9BuqeAc8Jr1=1gu{o~Mf$k&jK|;;cg$F;AOO z%n-nmWKiDe@`ptMH1r*0Df>Z>q8{>Lic4<@IhRJNxfE%%OUEn{l*lY(bFe$>{ z9J)d(f5o6;(8X^h$I%JBu1R4utUFJ0ZDy4uHp|5cq`}~(?VXTZ%JK1NHXtM<5^%i~ zP9Y+cgHa~QjqMk*zCp|_M$aKi`zO^yy+us?>vIj2tPv(qaIK(4wLh@YVo!mw z_AV;J%FaPlbyCYK(SLH`{0IkaeI*@7`c7~P)HnX!EB2F}pP!1Min#?L3XCjYg;TD$ zY*rSg&Q&^fA?YAuxC(QwD!kJ3ShT3xzX-&JRAs$3j3kU65|5ea<&uy`cYPYun_Pn0gneZIeK`B4jy)wslO^|ESP)3 zM5p`J&cUIwscCR+jka`PfSWI)zIh;X)pYo zfe<~qYR4_1wm#%2Q|f9s&nKmFm$aHe#U#0sS0eNIIj&3z9h~qSSFSc;Umv%Yv9+{M zXqSx5A^%>t4yKa zg!=<2XL}?HUegVt3nCdFhs!^Iyr5dm>&aPLWf1K;dinxBNsq0HbA?-=2xo*42cqWM zA`>3q-%heMsRV;tLPmLW-UQ#DCABhI@pM?Vo0$K~W^4L1E>ZwC(CdysbXXO9jwfco zoTfK&A?ljc-d9W8;#9p%yNNkUZ(8Hm^|G8d=x>3YeW^FAC4>D%h)ick#Dl+kOP+d`Omx z6&qTe#(lQ`&u}p5%>B zW0qQ|;^@aK5I7IMKLV;*AjRucXUZ?Wik(t2Rqm=3k@4|~NaXr#Te@+K-eW)C` zP&ZHITYLdiH!-j-+~+0>k`Jkz(~i!k8f4h&ZSnSOA&}(9F+i0l*EBUTG08n{yU8<) z^hz$s?u>x=85&qttC(8YbWmt!TZjqId>mrX#{Nk*oXn+Xes+qjMJCs*pg5B%Syf%t zFaP!DWeRiRsF&vU~~gOpsurZ0Trii^B+4W^UvMVv40A13UuBX&3m)rw=J~Nc=jyj z*CVRy_Mm2+?Wb$X#G{1saf|hD416N=#{;6g5~m;>DVfhm*CPS8V}tb5%{9p7vGBtD zG>B=XZb@jgQ_L%n;B-Q%E{TV|Pg%dtMvPxAknh)JOACk`Plw|Z>VSk6U?^Du)9=|5 zqRXfid3Lb+D_+BCu*2J9N)whz!Hxu_dW>n*U>r%jFhVA}mh7$5O{}E~qZ$3+w6y0F zT*=Vpo_?E*DFmS-mCq;OH#qx3_4g?N2x#jhYWp()G+sDcTOKNK^@_s=Se*;6EEI;AiM?7o=c&|GjsMGr%U^%*!i&OBQ(0uqAP2ak`1GB2CA+yNC_Q9hWcUX+}f& zQnEZv<(V?-4d2!Pq#AJ}WeXCrHkonX*aA!sNmd*XNt0qz;2u}EeG9B?tLYi13jyZQ`&TRJ%3?;vP zx9qq1;Bq$`WQSuK{621^P@;fmmRa5H(M^78j z65xd5>P0Be17=^jz_D~?s{#<7ziT|-KIYlEdB{t9f=8i32=G?4R#TssC^9JEK~12Z zpzP?B`=V0xi6f#pg-$s~-0f?nHHM(MlO*2i4Js$A$on3@)TR1OYF#2h+Z(ADNW_Pr z316hso~kGC+{7KbfrP5rgWBz9@+t>%vgmNLfUn)yXgj3*h;)}K5~6n-6_!sxJ0OeI zgIiVu{=P!BW2gJh406%aXGZ~d69+Q<9>ZaM7`;uxd2aM({?W{K?K_B68CRO{LOOt#?5U{2kR_nz*reJ}iiA$5A4qhD86>j5i&b^-us-r}3+m*e+Qzrv-G|?U-g!PE~ z)TPhlIkSNV_>SLT&lTx~7wq)a%k=j+$`}o{PbnR%u6*ePd@=~A4;!@>J@DjqP*|RY zAfXVT+JxsKGb582dMXB{n>78#P@Pt(9+th;z-{T4-Nq~^qcQI^yFiuv@a7*Ugi7%a zZ}@*}FrZd>X1<{O@aX#g*JL$@-jxPieEWV5?C3p&0b$hY^>)s7Gr%xyfROxZ`&<9e zopZmNgXQb7HwmXu+kVOZ{qYJ(g(Tbe_rE4hI9sSM9OZk5EHO+6y-L96J!n0-!1Ay3Za$@SSq=dOz^Wll$_POl3vB|on4Kg^-^GLUFoeiPTDzZ zlzN{rnxI&~5!Y_(yM{Q?F{!V%#GrA|Y;7QRP&;p<=eJ*HIgKykMT5YC-vjPgoyng@ z%JYCwnFj}lt9=}ZCe$zs-WGr}BKkBE@yp8UX6<-^JaIl?H@#&k<`eUHF9dF#uD)Ab zIY21wUgSGodGF=fS?WUs{0X>TD(<5_nK~jc^VJe(1xeZ1Y`akhdf$eG_iYYMnjdxt zDN;rVvxFvrG!8GP_MsV-EoQeBgUCL90)5(_%%%|KFF$N%PCEoG_9(~qFbu8|}mds;WRyP7$PB&FZCUKL2;Vm z)REKPpx1Dw$Vl+jF@v2txC}3&0K4&hhF2u5P)aaV?ScH2AKP<{&DG;?@F_#%JX@8; zumldr7t?~*gNbH6WEMNQWzU$hdXw9tIYV6VKxOV+f+1_-T; zy{zMb@pNQe$rolRdMiNb`-jE`!K~V=P|_@NT9Ytp(C=#Nnvj%ao}8Rw?NEl-BT|Nt z>8Kg{mgIDk&Pkt;lT}QHLeouL%bChe2W*ey(#To9Y4$4`2|?G=!9S{E!7j zblAl3rbB%a_>nZgV!CG^Pql2vt0u(ltrvcY$6G&f~G3 zF%Def=z35>KB|UMm*7fka4-!JDLJprDt)+RV0Sq^%&D$+N^V#fX2xyI%**L zgsRoY*;3@!lQqJZaLd_%6Mvfoyb$A$(@e<9s??()fzs0~KuXoM9&T}*Tw#bLEhgKc z0H1MaCpyD>dDF_GPuytZRw*vukm6~}Jw{aE;tajDQIUM{^d3t2tTej6IxTU+lyllW z_&BqY@uKaQsV-$?n4~YO4&T=wEM3mEseVh39PF_lW(xyO)xlVit^N`>XR+%X&9*k~v*O#+K8<{mVf2IrA$e zb}lOG6lwb#1To8~F8}X2Gd#~(RuY4#gEZn`Fjt<-{sl2-WD}O1M>*GZ7TL~==O~MJ ztLb9J&t>My6Bal2=V8%eZut2igk|5azrUA9FW*)*8kUYn?eCN+y$X85RKsg;&^)P2 z1RvRq5M4a{g*$1`cjS)|1Za}GH#R`dH!_Aip8tUEA@iy0FkS38bdVj2gcUPY-wtW^KY)kh|APSN`h{jw*Wu;EMce%h#p`Xrgg7$w=Mi;6F=Od>yRfqL>fz+o zi~Fo|J|Gg_w9G1Q;Q(u|J^XtiD3rlmV9{V2Pf59qwnkF7+=?^p!S;#RJ_2; zI6DplZP6-v%ehgy%RSx4=|c#T!;ls3@4&qz`1_MJ~cV5%|alr?e%u+=SC zRw*z&o7at*vaclk(u}I(z89~UniIO0aK$0??Dh(J)=$>sb(^J@DeO)Xwb6Qubuuay zd)glSibI44+`P1Qy6LLz95iGR%MS(9hH@rH0l1O-yJwoo zn+a~xnDQQ8Nk)LKRaWkxjEQVEE}@MXA>(Lchff`hO?H7jL8G824nt|)RTPnbdqtdI zm7#3KTm#o`9WX3dRd6>UU&Ej!s;`js$|{u2A-~QlpMr zB|V3f%R>~VkUcb>P2}Y2ZwG>1XjhNMXWF9=JL)E{hr1hm zF_*%{l(Z^P9D~c)1`ii7A{7X3b`%a|3U~%O)JNeb4EIUmSHwenDDkUq>m!KIU8M8K z?BlZyGRVrC^_h0Pg!(54QHkZ$2hA-_S&fOY!;EQ04Lw!Y5R)!ss)fco8;|Yc8Ah0Bi-p;OUVtLWb&o3r)Amzh(Rgc%s*t>>@(O`934ezDp za-4=wy?8RsOKdM+rZl?k;x0j7m~sQhH|n=(ljqG6`{Szs3yJ~FI-M4~SUfN22`q#- zYHS`{rSG%QDDZYw--cO3roaNNq`@YKNrl<7d*ovD@|$mut#y^vppevw=v@UV47p$aNIIUUo>VdIld2M!l5nF{pKMf zlEcU|D1@=8Ob}J=(g}LUESJ$lGHWA86A(Y0ZvdT1OO;5FMe*{!Y$QByQk>R;s>C#L z(_hl6->;-7?rJ+cYdp2tdH-N*|3YeD6K`YV)xcqxiAO^UTMLzdDEUn*tI{hv%9n*! zlQsqOz;%POn--Q7rPK zn_%e>Y%6;}{!vxCY?iO`y5zZDY!x6HS;!lyI3FY--Phf3;!42PpuJR=95ygCXG3%%ewWCF<>?%P1d)2s-osvvtS#=eNVx`igQ1^!+cxXPHxb@dh#RZ3Q zrj=jC_?M@G3sr~SzhGHf!$w50KgXHud~j^%<2)*@!clnZMp4Q_im_BEgqq2kx3R0e zUsZd&$`>7(?*1;KD$QLIOD!XvORuwZhm-XO!?v7C=vqfn8S|C?+2wzIsowuB7DZt| z1_l6f7*TXV%ms^h5&=fZv+9)X3d?`o+{f8l`@?Qw%D+*3?g!wFjQo$z@K}B;(ttR@ z&tHHPfnv_#-|IcsKx9cLuny4jy(9|wO=MbD{BIvA=A5)$^c%kVf2G$mvJ{PNhOvE? z2F-*zqMVg7suf?vWuovn>S|NhZueHzFD+5sH<4j%Zru9|DE-v`#fG`hlv%wV#OuLj zXZz$T?N0c_CqQWU>xec)Nhlvu3kdXVq0YBoQ1BERvM}V@L#0) zk*0nY{P3%F4_)qNj>(k~(JeH|2)7AI%a%PivBNcm)nezk(+iVjiIWQb){#wxi|ha( zx2piydHYumuB$4SJ^r5Qm951t(z@|n6^%iSH?9OfL%317mpXGX)sbZ)3rHZECS5K{ z)#GZcZYVE@5@VpqxXc6+_<@Oy-&5LdW|ZZSbZeIFTvkt!AuOc_b0X{GRxxjahzH8I z9K_`#c*8tOWttJ-LRE7c?{phPq^gZLRwZB;21sh%R@2j(K>DZH=6N3;aQY5X5S;3GBE|PPj8)Y5#bq!FWxl#rB4W+~JohYJ3T03C5=qA(;4*?l?<= z*5AXx^QmS}Y4jM=)nzEir2PbjNkhWXq+c*Kp(2H0XV7u82%^+UDYA0u?2e+7xdHZv zs@@K`sqdfe0pF9GR|0?UM`pEk2KQ9YWwe7KAE3QC2Ciqf{D0bi3SUlI{^_(7{h9sr zMLe_gS2RhtV(IC_75O0LJoWHIoFG)Yk65hM%;L<^CA1SL=IZn-bAe~;otyZ^YYM)@ z*%^{F3PH6@nqh340`@|`NdPpT1{=k!Aey!|rd3Ehu-%<{o) zS0XLBB%2&9LBpS-SeWVMV?0`ORg@WuYgK`*^oCKAu#-tc|)t!y-L{tVSh)B9+S0X5o=CA{G45t|M~0w9iyS zrD1&ogsOoBr6Al&K>=X=)N8XCb`6c=l(EYcHUr!!7&%PYs?h0aMUngb@!{ySe@=In z%5wL#0&e9W7X$u813Vh6u-(s(fRJG2(2`NER9D{;Hjkv%R^(nL#`lg2$5pB%C_8;f zYH^6Gt*xQ|Av!w3Gj`AiEyaNFokM=LG*>BRrIj@rskBNHC-=MStd6R`S%sY&*7ZRr)$_ zH8F5FX=`g+*VZPMIpn*WM@$_>(P7}{Gb5i`o`JG{=sLYyJb$fBO)ZXTgiZU`%8sL5 zFF%MZ>6b!j^S1p@owfsGawu2|>9KgSs-SYp0eVx9%;L_VIITX{cv}i-qQJuA{ZJqC z2nBc^I$JLoWX7BjCduL`*`)VrGrglHt^R$TFZTg_PG_UGRZN9$iOVUpLKbdV7NT!d zOGdiygvTuWFX)v0FK*66UE|B~jho3MeM0agyYYdyV-pP+NXS8&e>R(8z;J=c>=FfL z5s8>XFd27wB`yx3Qrf{>>^vztJMq-wV5|``g|{j0`Y=sSM#xJj?DIN_imo=9B=LdYn5j@Enbvu_3MIpS@7 zO>WG%G;ONX*7W0*xYy8mh~#|E;PiXUITRVT1H!>+ApU}kYGY@nHx@;50a-t!{oJ@W!aCS%r zhDnv<5j8DYENuQGg46ll%z3p)L19~2J4bxX@fdwlX=kO9^Vi>Fk%`m;3wG!3bGlX> z!TULefZpvRczhz&P1$z0hp(cD<_}rx}Xa zCje*nW#6@r9E{HbK2Vu%LrS?sJN395h!u+vPM|Vd_Wo9OCen}#*Z_dsko{F1cnc+g@{LA#VU1oO~EAd^Pc!^qT zLY|$wowPgZ6Fyk})EwLOxyKh*XTmj?FlW{$Iq%My`Dwoic2XY+j9O(Nmiw3fcQEPT zpEq?GQue44Hm!qV6pZy*2Q>2+6{;xgNqAL>b?wmAW-%gf&fg4m=Sj{8!S&xNis+$X z@Qv#z);Im6$lAWsv3r7Db`<4|oh1*^8hTxBZ|3_&m;c#vf1b$#bqRtO*cTx8LQ{k* zpuFW`Z9_;fDJT8UbG3tC^vgBCZEl-vvv)U9yZz^U9bks+4FioX##8{R9grMI_5R@q z-FXCfY?*Cg{tJBuvK)fRukrj#|GUnAm53UE#X7ySxKl+ zhWb{22mg+xF-H!Ycy@0c(0%AjiL3C9f&I%BJm8W*X7{&OPcFgMF*bh zob+@fd;7PY<`(6vHgweU74mVZYM6Ai#wfekB}r|Ar?uTP>x~dT|3|pm?%x8hIz-P^ zs0Cn$BK=NvG962U1N6OSFGTym5gdH>&5Atb!GVNOMNn*LIzIk-*K>D`pf3`TIV&VE zn$YeCN0PriKo@GeRNv@Rw_&mf*Z56(6W0cNRfLgyAtRWolazM*25G?n!5%0~2<(?s^>~zDM8()I~nS4TZO6g7=ge{x)Z0Q!njw}wf zV$vjSqh7pck^^NI+DABfJwZ6CoP8KMiqYtBhk6oU#Yx5;E#9nwt0-pc)b?lRHf!(J zx}wE0aJ8p&3a-$-a(LvAz;rCy%BtqNaBukbjSoO2$Ap8n%0!j@sXatT zcJ`rpCQ%S6NMb>OlU<9OKpsd)Ih}M^uQkdHP1fab) zqF(!DqWe>VqNXPUP95jdRfHeC191%h($9v!_l}{>dm#QIdTWI){D-ac_nAy6GT58S zDF_pu{td%tvd?VEvhT>GkBhlZX~rFu*+b_Om)bfcD^%2u&^_Fp8(!+kT%!~OSrUYN zuw{$3O))i#5_uTT$=rgtkT{lQvw{?J5Od^a2h1b;Rs>G;LFF23Dab<^T^2c}+Cq)C zJyC6OsYVEYVdxPKR~KA#O((Szkg{irnWnoJlzrSQkM0~(>6lC8Lso=CDiVtmxl#yh*w?Ax3Zf#1us=HCmuSf6Rf;&}rFci5cL9ls`jN&|2Q%1~1cig|c9^Ppv zJm!~6VF9*U4V#6km!t)1c3QchSJC$5O>pGVAyT}| z*7U|_Tn)x3hPo|%aFM#PwfkGOG24dEwj8l0*@?|Di5fy|nygQPCZ{(lxG=6ieXKq* z+OE7+MV!-a2sx>WEBLKiGkETB#dH{hz4TYDrJ&O>@ka%z@oW!#<=ckan^R+PkS+6& z40K_Zi9{q&A*5zP$!m-pDdVm##X5{#w7MfHMJOMX>Gjxg2oY1{SiC0&DPcDomM#h^ zHvUCVaU^Y?ARmfTHvbnox$$3ZqCx28@#Yz$?$QKE7k~7&N1HZxd0mxTL)<$RyZ!xM z3Y@mihC+6PhNoOCfKEU_-X*n3O-lnvacGU|3-GV}{K9Sfo8?-6w=J{R%^6L#cJ%?M z>3MBq2FgA-`d+8@)oV%fH+HwD-wrQ8t>Klg)LL)A%l~Yi567GfgCyj_gDOipzYVZT zQspFhe0rq`fgCJLfCJ|{GoRgdQiZhT6cokwUmEyI)y(7bepYR3cz!xASD1Du>Vi#v zozv}v!zr#TPNA1B)5VR#$mpq6V+~-3L|HdWRI{V?6B{?H zYAm0)zAnZ*t0(Sy1pB6VHq>f}*30o^@rcm6dV5khqx6i7laJpK0%kiednQk7d&f@} z%RGOtTV&+qln{mycU=0fXsrT#3W^k0i71vFbZ`S)j{-3+3^~>06hm<*@-NZg~ z$ug-kP_oLCWjB-EIX2>rL!WJFA46g*&P&Jg1*HU6jsL{T&o0Hhar^-Vvfk~3fpFRR zMcC&~t#TRQQBujta`wBQUAv5w!2vI|^eOrZ<+&N`EOC3=<#(I0~1OE2F0bQXop|o`X9$Z_MGq7ZtxuSi0ShKX67@e=RVsmOU9+TeHuV(RM6l zk<}_l&Q|okb=^tVhQ4z3Vo^LYlS+JqJq469arSilgpssljwNeuezbPGWU=n?T?BpW zjtl`^WDMc5d$@g(R!2w2xDv{MWyFB<&L?V~o)HD24Vd{$I4_+p23PqX$62qw)8-oA zb$?OI$S$M615g|?-5|-ck+3K#g@SImh3&p$7CDC$u53@ z%ZCJsiaJmC_F^ZG?1xJwQ}?IBl@822X`UIt+qq=Ty~ihGV-PX>WcJ6|?o-r7Eo%tS zGNrk)lAe)iYVMur-TXssA@)>o%X15Nbb*vCZ6Ut|N_|RGC2`b_hh+uT>q2@e8OK0QPEECk*$J>XVv*H zU1R|vE%9=oN2al=^xiXMKO?!@RU9Pih18RYYe}5hqA|iXth)@j2@k;|ClId1{3S{j zBV$hpTS00=9bh=9QT`lNQ^^AqF!G8>5-2R5b%nAcMH-e_tB17p*!{!TY*hcZ(8yDy zVQP9BVCT-FT|eFc%56wzCQyxb*(NpH$$z-O`TjjX^?pU#CDAH>`*?DA?(Q>si$MC8 zhld9M;^6}i;0p_JX5c`pxnH^)U^ z;yO|LsDJ&Ds}yRD5<>g0)4`>a&ejCJBnfMQ(^j#IF_pe-OCq5>O&;5;P=6OMh9e?e zZS(pTA+h46)K_Q#686jqVNP!!S)<&(wM%NSR?E`)O%(P6ulU^aCkU6P8 zpePIX7u`*<&Z=+n0x(j0U8kpybQB43M(a9pM5nPM7p zxNXC!y=y`uqAUPyZ~g<@lXLzAd0?&Fq2}m3=8s{_z1ikds?v(PcSd3qT~wXSqP+)@ z=f*Bm$~_o(Xe9Q($GYfz!2}SEA;uBO4VZ%?3{WVmqQw=Zt1ed4R*7Ve9IfNhesnbj zi3>=x5@byymcjM#2_=jWq)7=mn4&t6alNH}kUc?Ih;edr-%pM?Nt8gf2@)-L1HhJF zETTsx{y|l$T9Xph1!>MGw)=2Z^?NQwtR}lY|!?1>_E(_jcXS0c|JPT ziJHNS)PTN?{4D%2SZ~dqNv9$;{1%CkaizVwuX4ZF+mnKjVJ|U$*l^AD;IpjSfV906s3%qKYyy5s`;j7^03e7P;!W5DUIIhS?^f8YZ$-dSnBKU;trGlbkmZ zK@z&q{KUEWJzb4Q0BTzaU}fo%1`)~i^VlD48&^zotBdC?=T6PMAMYzW$Ir`sLkH6 zb}m+3qDm~?Bm}}12qJV6X{BsqU%FomD!vucAAlS8ND)f{XH&cJHW^xyFaIKkXR{`? z@Rl=mA<;f#b@p(zofw~NnC=VMVcNwia<$+~d>v+8& zK)`qpte=#?o&Qo?azpJ90Q>#-%&g#XJ@0A#pKUTX%f#P5_cc%C;CBl68cMzc&RE~u z%1(&EFc^p4ieRa zV%DE_XzD|y5?+`h=d|QECSQyKCL1JsF_ttozKp>gCgSQ-&Ej!5_UN8$;T}R9;LPd^ z*FL2#2#_)17IKIBx>54^DVX=@`ujFeNlo^|@g~RHMbA>TP+{JtU`FF{;LPe%w2sXr zO^5Vo*w}6ij}U1yBD%)lVMZwazKHXDOn}W6^{YWf^`iD~|E<&Ei{(yX(QKMVq~RNG!1b(Zl5I0gE-po!d$`Ttf3VX1FY3-pF94sl26P%=qQ!igBCkI5wa z=o3-oBUj{ARb|%TOmh*b&@e(xhg`v?W*q9OSam6YM<1;QqUAjSJY?D>$o=#mq7fL;^e6P{5VGeo~1~HIaA{9VIUw3m10-{U=;5psS5C~IXqc5Seg9c$>B62lM zvHSR#nj9QCVE<;pa$31H*K$HAHh|#Co@`W2mX{-Dj|R?YC8{4uc&$XRu%xT@04rImomWJ#1IAalEgm6uNPF2~iGswtDY-CSvZc zSfnvb-A5Fx76w|jlSHJw2@9k~y!H_q+sKl)c1yJUj0w$?1gWk^q*9|x){s?QfhN&J zTopy8U69M__@%`(P+VRd4Dz{%c!P+#U|l?&nCc`N%6*YxA&)eNf7h8jMkXeMRSMiu zH$1FS2<9>&b<|QDKIP&|v^}pgRIRYA^PfhCLffFy>5R>2B8I5sbGM`uQ2X>);NWP3 zqimQR znWzvo{ZX1Ht6OW;&@FW;CgSdc>Cw2$*RQ zbzOnsi9YbF^(62HZ%f8ViR z7u^iCBg92o1Je7)NGXbWvD05hZO9AfrX)+d_6awt(Ss_Aj&&x$!igj zX~8y+oesYchP0K<2SZENOVp7ZTog*-cGcY8`~~;AYb3`o&hs2QyC`}3B4O-eG4$B> zaM9-*|LVG=z-T6auZ+UVXo+(qsC_C3mq3yyF5EbsDhSH4W?KAvXE9bhpLh~=BHIY& zDAtH}F@Og$1MG~dxH!^IZ@*{TlUQyHmo&q7+|5o+sn%)|ydyR;awI#!urcuMF;)Zd z8N6)dGVkCEk5)0H1<#h`t6^ZSu(h(Gtf*BiZ);mB(!wO@S!icX=RL4#-3lk{x88gb zJQh}(OF|3=T)5S?DLl@$OhCK z%{E%SlG^I5T~`vll5J$&K;P99y0Ib}I{I(HkjWYVR_h;L2h!4e>Bi^g=5|{SoF#H2 zrUxz);CA(09G#eN?vm~Gy2o~>uO2mC+f>=7+ehF2Qy_t{Xv>@2twZC39@2I6*71{oKOki-A%@E4#ZH$DH-?2?*2M={f zUeUGB**^OjHA+c2)rP44`6o#{Atrp$=8zqmZqXq!Vmy!Oq-jQ;&!v5GRD}kNp|4d` z`#L88#$K>(saS%4_b$uyG_@KP4b3>#?9UV`@Bv}u?w)#hhzAO!SYbBg`z6HMea*qz z^NYjbJ^z5uWm}G(eS3<*szZ@J>4#9o7`~jVQ}i>a=wu)WIY8u)Gost(1fjA2#t!`Tfk}*4w4Orf^K) zSTZ*VZf{PL1%`RQKaUNo(>~RY6$UsNEI7#^E+RCgJc03r1qzGOSx4_XB=?Jr!P08I z@?~pGxDaD0+)xtU(Mi-dWmoACFW}d}$t5(fuz($#g2#lvgK;HSHd%xJo<<&brwv?XL)<8&w*%v4B&FQyPeR;*g`+I>osp zNG*|X1;~=c2?rH-XR}OmBcZAdSx!tzZ|dYrNn&u~6Bkisc^1Z|U?MNvQsGG-|9At3lv>}O7h(Av^qgADrU{>_uJb<1B6 zsF^hU-Qz(j=x#?&r?*j!f4+h&(+P*G21>@xIH{|LTHp`cH?%l6>(|AoazVyFfecV! zq5EoL(+?A3_$IB;0mvx$Bo^?tPM<#ZF5Z*}Zd8vrd%-9eQ&*j>5C9BA2!s@5pd%X{HJRpW! zry_|)?hnw6*3_>?C1e1vTicDAI9q1D}iMa-iU|Y)jn>!6N4vj_n%QAVX3T|5V6jZ zLbVzdfRSs(6}is`ovb+Uv>6BM2>5mh0g1~U%Cwb7c2q5kUlc)H3^O?bJiTu2<3S3S|WZP;#SZ~MdSp4cHSXofi3PT)wV>Rf0xZo7qEF=sfr z*`uX0iD{(NE~y5=1+zu)c*~DTKXX{r9Z{+K0{fw=B99k#;q=#u=}*UO3H zt+lM0Q$-jN6fstCYCza8TC79W__yJn?Nmf*?*1&VlT#w}Pc@A1zfV`IT5Q*W5w^1Q z3i+_rrd^CJj71hQWZo$Met$D7IvXzXtH3{8^_W4mS@+xn(dLKX&I3BYd0hQNGU9a# zh_DWSNd_{#Y5ClQfhKlfsHXVe50U~?m9aE-WAynafc*KIg@RI74_w#g#NqkK;Sb4x z^a*ghXAYUkIa{j_X4Y$|^SU`|Ocez>dT5ohp|(8vI5`{1A@{&AHR!tNIAsC&K;KNJ z7mG$o1AGUmYd6c|`hm15AL-wi_3=N0RNAli|zXv(xg2YAhzrLT7dV>st>a6#kAvAwsiLJFpYs8;q3=)3+U`HdWy(3lBs`HO(-WYvg#{SaF3$+{GUc27 zyhoI~Ek9mT&lgsw!5Kx{_tk2(arxai1ULo7;6_KxnDzk;r!VS!EK)Sk>&@5kBuXll3Mm&ShOR#EAX2@L>->ozm9uW~U;d zY};73I(1@Ns49vb_k1jB6g(Gc&GA%~Hy-t}kjEPClxnvcpORr9G?k!ZT3h`T?otMitS@EeUWi*fNA zKP%SABL-gnHTdp=%v*M@!lFe$Kw#wR8dR`LK5A!XU*aJ670$oKl6}fV2s9ZpUzZe9 zUbJ+`gdEIDp@M896oY38#vIa4%F=6aT&Q0mo1-(qWVbUgqu&_NuS?p^3W)3ObtMUZ zet^0u+o3|&OJ=|e8>MOgx$X1taQ8yd`tsIbzH3S26ufs2ft+1@R3rq^QBfGWSVC(~HNySj~0F8CTDGq)!2%Vqh$yOKzO=^pJl4 zD{lYjamODAj$uy*nH4J>)x&wGsvTfk5;U#Io~W{<0Zb0QxDGOF0K=fPt6x$KyA~>2 zvvGTxB9o-P$cc)9&Zb64lcBu;YO50r5V=!xO`EiUYrro=ghSQBO$Z%s*8w#{2a)wr z)hrKe%)ASk5??z}>w#ok5HdS|c6{~;ciVXvf~QmKh(wZji5-6m+l_>Ky%XTfc;&lL zA|*4rL{x@_;?Os-@Mk(XS4~dkgz#GsD?2Q;0<)M5y9xi0HSC1TO|o8gCntd3;^q+v zr0)*3yp+(8;d_CAg$=^EKmNN?E+OE3clqaOhq+q~DchYo(QUYQXbqa3#_`Lmb>Sm| z58jX3S4A1^q{|JkFjSflJ8%&a;ZJ2`XPYDocn%WrHKJ&TKrmvrgiqUXA2pm7Bs z2!W*mGxUvBQS2ZvL4WiT#=+FFlVy2My;Z)n^~DJQp|AUK%;0X4Col_a7zJp97%Y{j^6b{H-1F!Iw2!MxYh zipDiS+DrfyR8e1T|8=jN2D({LP*8{ky_0#rt&J=4?hMoj`w+Z)546=$kaNVJIZrfq z7VB5k`Hi~8#KqNH-QdTO2oH?!j{?zQ3Z;CH65O6k3l*YxF+1!meOd`JY`R6u;A^2r zD!CADbX(y>5 zR2%lIkM#Q@*;W?IrYoJ~yW8N1C?M#>O!iNxu;ts9x#l4pjma=wug(25AT!NIrU zUKm36$ER@eNR=(3Z#J)BA*EzO1BbX`x^k4tRwi`Ba@AZSpBgzP1on@gfd!zN#*WnY zv*n>Ha2IZ}J6&4>y1B3ogi)CYvM90)=!&wk)bHnEB024KgWyVAM*?T#Qn=QJa`cP) zEWG3E@W#|(0Zix-5RMJn@;u4bZ8%tBINyLU&yT~y^!UvBF`gjH6 zRw?@KAk!KL9>6h5lQl?rLxYe~(__xtdmw)GZOvBsK^|HGa@jcU$H3c%X?xlL|s*b)caJ*mCJaT7n9p%~;v(HDw>9ANu~4P`o& z_wUYrv5)VW;dd^JMDOst{^sWJUUydbFMUk5ERsqO6gcBk<_E1);FF=QrA&E!XkO<4 z|3;3FpJ0oc>v*e{t3PmB>}t!+J@sgOgLn0fGrPOw%(&SBNqN#u889fAr$cqDQ9c-- z{AwRfp-PYx@JE7eGC44r?0!o|L@q_hNmZW<#rp0hR{Z$juVhJ#P zl8uZP^uH~;j_0e*6^8#6EU?Yiysp!PD>9+ay#Vy(>hk5Occ)INCzL1GzW3=jphJ{C zPX;`&Y7KwM%=7!b4*Ww6Po~S-;X;La84v3Hqmr`^JdZ->oi*@~g#HOx=nqEt`;YX% zlMm$0@_^mjb`h?}H1#f4~G`)F_`jTn2i`2Xlpe z0HV_m4>}?V456#Ou%{WHN#<5tzN?4-U-V`E{S11&cd*gUAw8t6gW?&bjhK1f%IgZY zQ)@V#4W)C-li#(rvE5@jjzC9X5TIRNHnQT~r!5Xv302@B=lHO7kuHc(0{JvtHdi6& z!qQBlr8%ksF+O~6WQ-Tn|Mdw5rJH31VlA~8m_hRWEI(}NaN}slq6}24N`)FRys<Z?Vi}Tr<-qg(W0y?)bfjTk_{Ex&slE%6vfOtVUH8XQPLKN=xbcJ=X z(V8VJX(#iD|FJfbJ*-Hc^KlX7Lwc@3{~hkMyDVeWyRT_ikA33F#zc6Y>7W#xXy)hD z-mdYTCsEd`LDHufbqoB2A8eyvNcN`(YhCWPFZXkSgLk4GH_np$Wbv=`ea=dl+TOB< z1sd5Mx1qVWXpF`Xk7n(@N*w{M~;tc>~URQ z=~)i`Z&%L&yZ#kfAR6K(xoXn}+Kn2+n;D^53ro**>ynA9E4KapePU7f7)}eN#;zs> zVrfsLD||R;T^lK6OZiBMmnFE5p!fvpuA;M%ox_RT16&&1Hgo9f{gCo|YROpe4G_>C zyb}`IX-BZBjx0n3qvk@``S`}x)(F7+I59CXYBkCAE8CmVp}99(nVWFfp-mzSAY63( zMGYOE%?O2@Fk-M+b}_@-tB5P95L#*wQWIWrRf%Gs$`_tQ#OVBCOqAaO=I4r2>jqV8 z)0RIp7^qv$35|?A)9aZ9b#ncZPFu9RRy$i8y*By8zb{&cHf6F;1gpt?_rflrBEzv) zG>T25Dv>?nqT=91cvo4sGOCNP%pZM->#3CQnH76PMx~p=#sP%`jip9cbpHw`3CNyC!Tr)w%ff z<4b5fdew^RXg!}9pHsd^3;ei5Yfei1G<2Pu-)SsK4t3bze)F43?!x*$l`Ye3tT?nS z1$plf!L@&(n)FS>{ohtJh4bhIM19We_n=mN88b$v?0)z*lQwn!oSYU8Tp? z_*f(r!@7a^i~mx|1I?fnA_U4ky%1b$X;f<02Y=VezhEiny)$NI&2cRuLnI(1E?%~+ zU$AVepTA?5S+wRS*to&Xg}}1>3}#1HQc5x}Mxu*JU5sCEy|YH4C9}=xl!G!>2xq>d8qhreMMR+Ss9y$y^&l|@tQ zfWxK6P*-Ca@x-dO!1^s91ugRSA1zrz?7z=!w3n^jBs)Iq$0`Mc*09(uk(MbbD*gm| zN%3oDLud4OcYIx5m1M0+zGnbB*tkPpRRJVP+IGnG*`JO-h;FcrEA>2`JKO;RNX;u< zLb4PukuUxA0T_T1ZXbH!0^3SemN1nHG8gJJ=J>54rNjJdv^%| zdNpSMkjNCEm0LGYKtzB>cdvs4|L_L#U+c5}ExF?d0U<`|*qN3B)_kOgS z?b?4oc7V4u*Ib3(zm6vDo|I&>EAmdLfO~@=CT?CAXOvwaZf--INL#-9O!X)AinS>W zlD18}B^-Q)1YCw5*gC6pJ9WU1A3t)*SPc=AA@Id?)Db-p3z-p@jD5VBM{-(?)48wd znqzM0=5a{;BY6o(OtOALv7-}B0p6~pCWYe7CxKMwoEC-hbJV`6t#U!>)sRXK-t(nH zegsQ0mpn*6ZshbJT?S~(JB-TNFWtwPoX+l+)Ra_ecB-bOmtNi}g zeBl^76J`FllE**E&A;@HS8#H6x!!ZVT>n5iO4CI}TjKL|j(m$gn{U50B+D#jtw@I} zpKy3Q@AbSpr`q$V$inx^*Uj9ZCBwEt)8 zw>^W)o|J9gt)V04tn%ekpi+n+bVzl}NxN$(wx}6)JXy`NXLjg-DmXYeCQi|D?_M>I zRB&W#>YZ-2CQN(#GnT1QWAKtKu5p#io`NwiIoF6-l|e&z(-X%x*I>HP9Ca1eZWge@ zUQe~^1c1(G7ZRGdKY#;+_m`}jht{{cn2-72Mh5dGAjZf)l1ZS7d@rbis$iGha8Dl7 zu!_QhpQ8V20>8r(Kd>h!$)py)nr}%N37t2GUN=Q^v^E&i9VzWYg5`#puaMN%9CYOe zA2!y(DW9=4Fv7ZwpB!Ib!d6F+5Ox|l;nQpP<3QnRo~fSO_H z5B)Go-^S?#vZ<5RWY|o9_br->eEA3|*`8bqbzzVbkgW$W(tgTIh8$u^1VB{^#Uq2| zYYduQ-d@=La-Ha6bbR&~*t9qi1b72j* z!|A+j(_+dei3(Lw1l363So_g6e9I!FJsxC9TWU=~wv8J*XKLI8m>v;)05mlvF1cS%ra-sE z`oQ_euw$(-NSuPJ;E94hBI>{d5WB+8i^S9kK%XuCG05Wk&M$!pzOvj9hY^f1dupCf zQT2XGWEtIUEt%v4Ba8bPeKv2DBM1`{uxM*`@%9HRSUknRmip~{0A(a|kN?@+^O_;= z&}Uh+54X&DZ<8=Cw5y+o!+V*1xTCkY%#L+#YFv;KOr|JY zJX`2x74H&-(1xqKUFzLJb7S5boZ4qq{gt-xvOfXUG(-g`C(*wPwqoj=AWpxLBQr`W zSPDwirWe;mkZc9B;h)mmZVzhshP2GBs9Z!Q2HXYm$7^oPu~$lzkB#YJYF^A3yd^%-YrMTe}=pKb%;z+f6*6_jem~)nxLCi zlPG}g820h-(=+w5b@NUEE^rvgutY2RFb?B8;)k&_w&MnLD2}{d&;Kfz6!&FC?{(Jy zBT63^y0KU+HlA~in5sx2#6ekcrxVz`2A$&AoI<$k8y}a+)kfi*B9@KSR>buWONWwH zp2j{=sRY)O*}({w%|)GuF>$oL%MLbEV+y5IGtw>oto5Tj)vyd^H#ok0sboQ0qnLf5 zT@jfYb3)P^x*S{PLHn>E1wy=-{G~r|*^AaHqfNfdQLgMqkj)YcobgY4`iaD4@%E3!OfI~T{O2GYnxoK?A#*ly;2k#^GNx!;Fw zlOu1YqIQ3Xox6%A$N!zyStT_R@I&8YgN1~YjbfV@O$~HD}4gQ*90t?z3 zE#B(!H_&U|6Ma`QoS zza)>u!rcBHm`Q@K>?1w36YkFqo7(Fs3=kj@v$1gv#hq^^ojPA+u=pnllXg4j+ruo3 zE{I>px22bfWPbmIDsk5wZIq)?~z`li*qW! zkxG#2%oBg8miqB*Py=DKX?;uVSY@4BuBpide^^>6T__47PSCIVfx=(NhngX8d4Gan zhr~&$LF4h+Ui2U`iVpssyn{QlmP>ND@?6``|x^3j-!!38PyYQk5bePA`u|rc}vwnQX~d z)NMsrFlp+i!uTSFZ7r`ygv!lx-jg#e_?yk|mqpj(;}_4{!@P#3IEI+!he zbwUD*;og*0%?qQ&j4Z1MMup^3B+L9_866K1yTz6WgS2{l^0Tk*9(SxGIP5U;lPlV6 zSynQwSf@F*Eztd$cO`Eh0GNDb9C20dEpOV@VZ*@WLkFkIio1k?gM?z@xBH&izCI{b z+(qkF%T~RiabKzuvN*TZ4LuNWhoy6kXu}NhT2KGly!nYCg)VJUN;QOCKwcp^ZZMuH zB~%QQz$Z~=S2Dj5+Sb-KRB>V@^C2wbN+uz9;%IxOxi2$1*Ra}f@7J+D%l5<5xbC8q z?K=`9^NR7uEv8_}LD`b6yTr%4_t7rqGNw0O%@l9lS=yM$RW0%uTQv>HI_y2r{b^dw zXAQi+J-ib;O(#^eWX_|#SKwJ`+cYEIVuE7j|McDX2XxDsq&#I+KWKL=KUP}m#7n}q z4YlLVx4ws*&VQ#}$H>Rev;R)L%$0B5KCh>Pf4;o@6vY<}V|3MV1E{g~vSW5H&rgpR zm(Lert|h=wKI-{2GW%cuO7@PWT@C?_LO%$?W zs-Y;MF{)tp#gl`SgN93u-T7rap(k#x9-(?Qob*%%Fkq4)v?@(!k;61J=`E+62vsE> z1#bsrbx;w|33EDb`DG`a)mj&LhEu=am>eedu#C9(iJ$5L7Hf9QsLg;xu$;)IoeF+`|2I8J9OU8>mZoftOnfn*}scuJ9T# z_dUU7lYu6Vp6S)Am%9db!Jml4lf$p%S+7^-!CXX&0KjipdveNoO7@9O0Nsk4QfY?i zW@MhT%j*m^dUE>Uy&!=SNV;0HM1!HRsbOk7sm$}|9Qnm#yWo0@n+<#ofn5YRSOh_c zF+>Uk@z3g)r7!our;|lIF;c0BpK!7N;S|HO0X-*UR3(ELJPf^i6FoUKHQnfT!*`f? z8MDy&+qr5K_Dwl=b`4ZUh^+ArySeT`~r zR7s#op?~TZ*J#}P-P_f&HIKhOYcBhWF)?wN_af(PR8fT=RTfGT<+3}fo9SkYG;7sa zQV8C^Gd3}K?IUyEJ3vase~Y+jCp3HdZ6iu0OqM=!kAO1d)L~_hKJ?aaA_cibZG4;6 zrkzTC;LQYDBf5g}<;UOI1PDPCWQY@}=xFRBBK`MkaInF!_kq#EZhXx;Ffl0VcV^LW zDvLuc;&g;k?vb)gbDERdn#Lek+uY#e|bAErofejhx zHW}46;%(fT=-O5cRz-z2QVEgeI?xY^ZC>MZUZtVZl^<}ArTc9aVh3Mo_q8OVZ}mPm z`cu}$gRtLBy;TrKa5F^)Y~n*KT>8{|%V^nAo`mN-K}{%Kc&3l73+Q1wHTD*^yV4{0 z=5l7{ef>2~}4 z=t&mS-nYWvZ0zDy%Tp9Ek3>!B3uJ*6VPZ>9lOIo&#H*IqACE>D6Q=DR{8MPE#OOn= zIj&Tx_7mcmRdI_3;XLDTK6rr>|H5gu!7*s^5g7cT4oP043*nP_2$VlFD%eK13TIbC zoOos{613m%9@J}l4)buiU(UPU0E~F{{_hbR`96&BxNJ{f8biL_)jy5oS-76Y5@S?y zDKXc#3-ql_a1Hiz^eiD_Ew;QDa5-8TPi<)kHZGa-lnaQXt2d?Le|LUl+J$1{p%OX5 z_S-gY7NRsVx4H6G{X-8v?^{}%lIE58H2qV9e($_axn9jm;8t)~L6$&fd<}vvMORX; z!Qh!Yktkh^*|B|KbkJi3q#*2Rz{<`kFnbQ;_!F2jBw3~Br$i+9BD#l1yT zaSN|it98ML#M_Nn(YDWLvZm;BIy3N6GD4k0q9>+&84bwWFp1xF}D(r|kY7Q|_74FO~ztb81hEWO(Dg`<@dI8&iEF+xgF z<0GY)U8Vavk;gj!@ESRz*S`xHnVCQa#m}Qo?>tVdJYDFnmXldaZUqB>B!0jhQ9x|vr&hNyG_;1^6%GBE;|8@u{eEMX! z2xI+b41aj5m+%jFS{LxVAP1b(L>{Y7gI_->YyWqzcLg2#e*9pIs)BVV#q}dZo zN29|eLxp`aOOEO3&lhK-7$Sgz?hL98a>Sgd&xxv=BfVd$d{OB_b|9r~iiOoZ|5csY zeFAdfhM}5=&AWEX2sM@ywjG>u@W?-?^t7kNqhABdu@fW{WLNN&N@WKo`2;pVTZF7sO;f6Crjr_VJ97`CsscDTA=`EbR))R zTMsnIj{Di_5YTkHVg(JGRxhIbIqp`&ldo8bn5WbZVkat*3zVGsY}^fQJ?3$_9&c0O zF!gtcJ^UjC(uAQazCjEohaUN$gn69a)5Fk7Tc$a3()uDWAT7PvZkhaeS9|0Rbwz%M$%%H(u4%fx$%R;J<0 za-dGEUxw|xj`*7Zx>wYAUiZ^UHY?!F77esP0Gy=&$7Pu*qflwaTfF^a2FGEkSOOrM>_Qx34Vq?-o}mdf?206UBN>O*M2w7pHAfXo ziC(=V80=Nwi=oS0*e^;1VBNme$_!~2ey@px{;B0+yQeHkPWl3j({Hq6KMu%D7S3{ru^8`>(5ypw zli0zUz-nqoUm*U%$mFOoqQ&4OXp^DpOyKA2CiKB(-1atAkZ?%Lloo;2L}gT;>Vq%ktTdz; zMO>^E3v8NnT4`$5RX76LShrIxcOaViTYnGju9eCL4cxdSK5*-G?S{x6$;LLxsv>WB z!9YVN4-Q_0hZ}4lQMm%Lx>UO8DV5Oa$i8}o9zFOBBGNW<*xcT`4;7cnx!nZlRPQ+& z5g19M$p#PjDD9ded*gAp25SFapjE!hH8(cL7T?L@=eXst{H}yW+_RXPv+DZD>U>Sw z)&GZhqp$k{CIPd8OH2_Yle!Ff4T9v+vP`FARlTG1C4?3*V=>@tPN!ziosuz`x@THK zoH#hmM)7cVwP36RnS4FgFlA0S23`OC=}MY7;NG$YMoy7rh_h*stFA}k;=tfHp#5mM zE|auY>@X?z!;S@p#I*{P#VZsiS0YAojPm_VA$}LaCh5I5CLJ|quju>$!M1i?b#;FY zHtiA@zif$c{HkRg8h91i3%pJ{Wp8QJDx%;R&*E)=SIhb8<*iB~=++<}pIDOsiIFHj;H`98KG|qq?FV$B-t#`7Nu}>j>JyfWY6R3+9?l|x;i2f31~44LC@tV&HIBD z9pW|fx>j7iY?5ggBvuOylSr(Pn?{x0Ya3BSOG>rcM)VP6cih23E&Q{a{{RHa|C{3V zJXdjzFTiS@&FXh3|BSX%@pt{MbFiuF`IJSvYLLt)x+>uL*!OAY-MN{0*hGon zP`3t5gFJLHp=P8?znf|4hxw*Xjhj2+<6NJ!!?!8VX!L*tcyyFDD6$Nnd)_^X0?3Ni zKNO>F)hOt#GujVUzw2@B+(CwJQ_gCk(f|+-8rbjdB0?yAxW$1-BRTLO03LQZa5=*9 zd?!w%lOpk34xdvYAoLbMe^YR;x-9^&DVlECLs2dLx9<80Uc*C#%Q=?R7N3#=vA{$&nZi_!6hD=fL*c5H^ z$Vn$}^YGY+0+pHwUufJ#{1*^$%(3oU@Db$5L&r*Zl1|{#`iUOrms934!i#o^E?m!5 zY+DW@&zvdC7{gG#%|_m3K~3n8Qr_{Kqh$;TC=1AK4Z(zBpvl3eXz0Y46=Qn-jAXUJ z@Cjv>+=aEmfeQmac;<@HvqBrFF2ZWV{&W?S^GyLkZ(Z4M3!i@<5?B%IG&u{FAc8UP%+fjYrB4K^wa)gG*i___`<)Cb;gx zSwv-F=xG8!`Mg_7Zn$+H$tjUgpne7D-%J5mp`d_5t|SmRt*foCX@n-JV$DJWgMtKj z*Ic2^%*^c8+cT!8Rs3%zMYhIy6JIdl@%hQzz%n^ zO*hisUyQUS8pqqAYCw6?x%GU{I^Wkb&y?rmqL~JEv0|Q>-fwg>DK8v{! z0t;ky?+L`#%>68Pu?QO&;RrYM%uNC|j{Pm(=I0ORTVJ!*e=XhlV32tJ@bdm~qhVn18Ls6Ak~NU`cn^iXS`{mdanYFWR&Ip5%~m%hPge zfR{o-wMe0|t509xqreA9v`>xJ^^L?3TmvoQAwFje7INYt>a+u8_L==vn(QDRm1dbm zD~vD`I3bTS-zy=amm#ZoxU5wL7D{fBw)Br&ZtBY&PzeNrO6*k)sV*hX+D0A>>u|K`Yl2|x zF$XA@+2@nc+y^M@Vc*@YC;ATd`K2MECNm$?l>SR`hOGxNbc6o^q+Q34-}Ki22J?(& zmO|!tqaysLCeEJjp0g?h9^Lu%`L61(FR+k5o_FN|9A;n@;4~-ZdGTlkTx`vE>pr|p z^>0LEYS-&y*SQBlpo{qPtiV(0w36QAJz3A zuwCvu@GcHR{H$~EBKJL`nm``RhX2x$zo0j;@IO3(MK*!cAG#n1 zE5|*~ya$v?k=LiMA;`eeANS#pc2^%AKy|{+Nrn1|+?FZb3c>`UKCYE3HvxtnM-Bd3 zywEQ+pJ0!D?GyG64&`^9_Ye~m?A zVLXZ6CJinv{Cc)iDl#OEQ>S<;ZL+I2A6GwD7b|zduDPF$3%F01X7p)82{x^0+s|O{ zLzLb4;VJylLE3aj+7CJNqMBcwe%4w4z}5D7>ADj?cq9nQ%a|XU?rPJeGgfGNN!+dq zDo?GM3EuopY-5=FCrrJn*k&4z+oX$mmK`*61hb|*QG!JYCKh3&)=$;4dBhRir+f)z zhN@{P(voVeb;$a<-w#~65^$*&WI3{| zCOh!=tS?(%*V@YF7}}lg6)?W3mYqOM!Q3;YjoK~MM%py=PEA1+QHcYtfhW$Y_A4S} zp$)ahy_ZM#7qZUw4eBrL}=HCP3Y2-F3E8=j1`C_fZFqhNk+SNLj>`G%) z*;vE3a(K^^ggwOgWY$OeZ&hR}lKZ8qtLw)i)hw+Vb#rppkolr2MUL~ntnq@ePjoii z=|`&^W8q|jSc@fgH%%yDq#|OI+|f|5Yd(6B?Ku-t0IhIN`(s?=pOb)Gg^Q!|D#uZf zPs_u1JpvLkDf*Nab3+y#+d3EE3X(Evw_cmlQ(!VcOT+JeLPr}%YUb~dN{5s%(s#YO z|0vA7Q95VS@?wP|=Pg4q$#2wdZ_(Z%U%PdtH>kiBmYXA9mj5qd3J?HPMP{=$8bXtk zm0kFs1mEqog`cj8RSCmVX#XdznM)u>nKod+Y@vU z+QMOKLQZH1rQ%*R6}+GQ5MtM06=q+^7q(|{m+X6_kDo{OpcdsjD%oZV-*1j-57QY?|6^cqXFh zIJx*bWg5Iqr54xeEkiBcRKH^xsx<`{_S=zN(Pq!pKa*ZWJ7G|f2M1nVVB9iimBq&G zXfXaDhfo4?1sgFM#6RIsnhQ+2V?k=$S38A9P(>dF{uGl5g|J6eA;6R*lHb<-!ldeb zmT^aVLuMN@hi8u>k@@{f**_+Mw29rRj}-$Ynv<ml)z4mEQf;yk9@PS<<|B^p3Of%5Z>)l+P{@W?Y@;^Ry-e zGY(CdV&=ywt}#@J+@1mxe>>w;86dNP3q6?r=N!Q3f3_tw8^VI2E2*vFH8?L;mD}=S zkgs$)x*?R3lKP=BA~QJ(N*Z5|`+QEaek+WA|3~p}$KwHgljaHyJDn&bJybU!ddZ(1XDCXydspz+?VdN#vR9?b`@w9sdjG ziYw6Of4bmr-}iF=(j5ftyFp zL8y?w@=&)o=-%GPd+CUbgrwcht;^sU{Q8vl$gys3es|7T0w^Ryi1G9RJZQVz=$YYP zn~_}F%PG*la;}|r@%E&C%XOCMf*xll9t1$NKI-E++VgBGvmW3oVmRIC%6q+d_tN)O zUP#qy=Kx?7roO(QdAGQpcicHdoHu}r{&RP^of1dCl<&HcCpL9Bn)3jzA$674r2tP1 z1f&9iQUM81|I^Y~p}Mt=SE8GTp5mQqCTW_5xtB-QvS-1|dFK9#(#;o#=ksBYKp;{; zSjjg7AMD!=U|ki!BjJtYNbGl84yi~sPhNMsR8d!qpBJgopevRhu^XJdOm`gw%0m!_ z+RoFmt4%S0W@r3j43u5bZ&Khvzio#q1&cxjtJ;?HOG&Zr5h+BWX`oD1@3mz_K__a& zbfSWvjFT)rl@IsoOHgqku;!~XsFi7!soMe&ePl8iy9t)VMg7bHS$@5Se?y9LCGV_G z)_UXa!Vg}DUm3>~+6f_HKJw9MIhbC1%?I6k61WWcKLZ$!y`{}Cp@oD>Gs)tII7Qgc@U7d>yAQc&#W((6x zR*jBEQDa!^-|U7Ee^VBqIv-5Q1BO#TRd}MGH;OAIEe+&$7tGn10W$-{ZgsjzPA@B5 zC!r#P^Gki?j7PfWJL%Vn%Lgc(pJ)(%1>rQY)3O1Hs-_a~-A0Di>>t`b))5+Ne|(+J=Uv4-M3cWFiyORMLEZ z_VrfX2TAQVYDhQGIR>{!%f@PS1vf#4?~3&+Ik>pS0KFn@EZN>{nHm!_GiKuoof1ES zB7HEqqf2;l@`%L~x$y`5GkHq+D}4#HpIifw2MB3_;)0txa%sNCR#xdXmrejX3rLju z(V=9{q~U%TX)eLACBgiRuJV;Dl+tUo`i0ut9M<2g3%9a58@fwCD+C zhg+ePsZMpl+=;Jths$XHOKxQbg*%~yTWmX>MOEp#*$vUjBd?4lN~FjcBfg9N7} zYHA+V$IF_OF`ta0oaY^G!L2OvUo+m1zr&ic1pU* zmoOw_iAc$|kl&Xqu{<8mrWV0r)rxJ**37J!O=^%HRCxo#~+1)>h2|)VN z+yoQ1pwTp;?cQW~yQ=)h>rAH#VGj%#6qMP>!U)ZBNrJCS+DDHiV;y|o3Q=4Ydo0cD z5^=;$K+-s2{Z1%>$-p+0JhvpPZC+PQF~5xBsF;$fHqtW>*cO4r^>3F6CU9TjM1Puh zuq1UgX6a0*bz5qje;5LNtv!bU>rznL2H1Z@8nMW8F6lpPvDoJO4BSZhzSV9D1ah`W zd~|Tj3eP)ycvXzB{y++EmEikjNYh2|4|u|eoJhDc$sp0F{f&` zq@UaBRO|v*Q}%9--aWASR^LaKFi;B;SAUWJKU=M>_W$hiUiRwnGL9D)Ks-vrRyNmL zh0FCz5I9Lmc-RBzA??L9hdzRScmJ_VoUa@Z9%7!gUn{=p);oV)yfwfF>j&rhUdg_F zzbPHB{{i@Go^WTRzTQI$zg)&$18T`Xq!F(-r6L^#Ccglw;$7{_y^pc+X6@Xw`|V)B z)1dEFQeJzZ-)>oHE&~M9=7iumU{SK1dwYa%ktY+tbBcZ|h?9T73jja@O+3q)c16*7 zlfsX7UE+)J%k5tOXfT1FPre=shVgv=$M(d#Zs6eeFluz>fdB>p_j@wW%^=jQ&4asu z*E?SQ!cic4E3c`@ zd?hfD9Qd*xSQ*{@@4KEn0)MWgvGpEi`L-y=lN!EFX2!PiWGy&G9Be_63Bwj`n`Y#QMWWM zdqqpgi(UB3;pn3OChd@GmX4T^q87Y%7|SNc*P>AlIFkkj;Ah=(!nOP23~ zTxFk4;Skm!g&Zsjih7x5m8H3BXh&0Ll$5;M&**CV$6LFNPSTA#LLxG=&-*A&B$$J( zQ6fq?7^8a}3kF>wat&jETpSyF2grM*h$Z_laB@O2XOr^x;|EBqG_1`ZJ7OzY@!jv) znxWUo0cFNFPTI+ad$elaogS~V``NUynT)yyw> z^DESx5zoWgvEor<#}iIhexq_~N){t8cHvG9R35n})4BqtxDRymm6^2&9&ZWMqn zDq=IWc1jQz@yk`CRV)OVI9de`50AfPBQZk95t{*dD`-;JLQ+!FfM^D=Cb4@z0S!H* z7JqN7Gskg(96DLq7S$fzC@^3!LRDo*aa1lnB_d`C0~|&uP8E^82s!n25Yd3PGXcsM zIywf3`DY)Vx?n4*b*Wi!@*NHYyBHYoa4dy-0sojVK}KTq@08N4!WDF#Liij^DijmB zwJ|VNa|T8vLo-E3?H4^62cm0*M0fPnPUvx=qr00({# zLwr9RpD`x7)GT2}g*+gvP?!Jx#YfpB>Qpu<;|6r8>6H4=)$|-;V@;@shw3J71Fs;_ zL%c)`D(K7-MBN^`WkxlJNfnz(as}e@pDa$Wcm+b{{2vLT5Y6giYvx;~J@&gD2vp5w zQ4;xg1@Un1tpB>UPSpHV$YXNlH4WAAKpa?2!ez z8=I(vYndUlwZBX=^7}Lw)G$O*eJ?TpQiV%}+lNvh#f_gADDHUSYAX(Yby^Gh}2 z%zW<^IGo5@a$bU?qXPmACC*o~^aF!Z^jf3$dr1A(VtLoiHyXAt$U5}d=YEP&EmG+O zmF!)spB>y&gL*_~U`Wdzojd(0?7&keiC9+r@QzUvWD@I;MAAQul)aK z{po6pi@0Ee(9M)MKwpv)4@mAz6^MhM#hcZ=q~k^Y7Kt1&4(a|V()-ep^15ZximQwkPlBwBQoV@1~%ZE{*OSQA-LJ4S6{rWuZ z@zdzB!Kkg`cEhm`HLc;LsR$^P?0mVcGGe#fu*~(^4+dY^YH0NlJZDV$e*fZPVkp`5 z7O_$gwV>5`l>!KL4xnz{-Wz!MgNncXw8EFMy7;Gn3drZIrF6qYKVJC02i;-uy!)T~ zh|nd>zur|O2!L(gczxdJ+T8iJ8}Ki`0o-cJEuse`>_4^1&gzObX-$Vm((LJ*72 zx})_DKnw)&shs$aVoG=*fB-kWSkT9#dI4J%->h&$CFc#?26~$N2V9 zR+-E*vAZsx)Q%t{eO@l4k=yg9ER*Rc?flYl~s5q7>yMC z>h*^k?Ojd@sg(K%q#jMU=beJy(%sqdVh!+lq<5_B9RS>|H<^j$HN7qTmOcttTHf;$ zjNvh^%k%p$?`)@Qq_QyRKMY8HB_JmJrQFMis#=C>fwl-p!x13s>NxBB+_#$#$;X9E zuwWpwc)xsCt>h3E9=N~fgGYa%V(qv=0$iTzShSNsbO}bV`$~)SC`|3?DGxpNxnf#t z%(6zumlP&`I&SLZlQ5T$)){nx$&G&F<&ZQ<*jhz*#!o)%R;TUuH* z=xTRge>nQ7*kP!oFG||%9@%0!bGep23oGnpqc*5g0x0JRFNB0RKDC_W(q=X5RxP|v zXgwqhx_1npEA^=Hp1J*vgf?aFZOQ+}(Rp*}mf^mbPg<3)Z~$R-Oc5#?fw7hIaTvBD zQ|iEe(TZ^Z#iSyf4eGrol8uGr68aBV3VBd`in|0sn{Yaucfr~as#&o60Vijbwhau5 zy4Y76id#8}P3yGDes4tmlL7TCEMGBbng^yeR+sC#;XiGUSy28<60y}p*2@qQhAEnC zVh8BP)iT(W>PtB*+xo`x7NM#?+>xhkOnM2*VKsl4Prc_2FK!yGgp~Sz_o;%DWpni` ziP7=G#kq@%1Z>8>pY;Dl3qPH|`#(jCbWnMC)bPb{y$H><#)%xH2= zeWC^lx<#KsjPpS60iHC1B#2p#(UQaoT3y@E;C;{IMRut{DFVh6m$YP~$9)#M(N-}_ zgY47%N4HM>Mn;e5>a|CSJI)0c)jJU-c2!o+y87PjULoiZafcWyg-V7Mewt^rqzdo> zX=&>|Q5LA{H*TZCe2(>E&VgO+i?eiV<7tHngSgq^FWJy5%=*cIBSmDnNM`!zy!XNE zc;)X0lC&A)PYE!sKP`Fm-N+G}My(12{!j>HBFX#Dp01w8Nv-&=@UY&*wZ){osXQ*= zX@^AEw|T?F@&3Z(ZNoOYxdiypO~A_u)VDsM0Lqv&wpua#@{PZsjia|k8ko>`YaSc8 z=dWLys>D44HV`Jy8!EVheo4*$RuUO66?wKMiW$Haxx-cxdegak|C;nEXkCSP1j@54 zK(D7q!?bz30B4%^ECj>`yvZ}$PQWdhH({*41sfFE8qb-?*WQ4p{+s_ljwbvvV7`Cu zqtQKXvT4wLlhD7pn1G|>A9_}^L3i(R@EyPuXnpFs)$4Mb69wfx?E>1KUY%PYw)SdP z4Z!YEAUglfRdq7UJQakKZBxB_7{YxJ{-1{4m3j_#4^%{FS0f}cTGnI;o;_xr7W%`m z{$vt7pf|fbNKkJ{z|5>xcg^O5;p_tM?KFZ$U-gm2kz_rRJ<3wG0WV{p z_Exq<5swoIMM|-5IUAdDDUNH~@8Pkfna@*ksz`fYq-mecl;zkt1jgFd-LQc`0oJQ6 zqZe{O&LjvN4E4+Q+k7+#zQN=&t9g=*Zjwxrv2N$|zhO};iExH+3!(q$Oys!^|C)Ar z8PmZFs!3&w*4D9}K`TmbGHS}bRZ3hTFAo5jc33R@{H|qdB3~}|-mL5x9qq+=A*stj zYn~`1d4l@)HNXq>-T47*Db#42zeRn>2MZ2ihDj;}C?KoW80Qf8#V&|hT7meD{5*=IWT(x6Q_MMP&t@JT#sWMZG2{@IUn^xk;wDbou` z`P~)}R;2fWOZ#=bFOe6`3FZ=MkuIwBkx0OAoFHnBJ(STCB-!p5rVTrWdcL*TXtX`H ze^)5)=5a>XOMtAJpXfBU$VHE!jj|_~ajB4%zS@|yrLdqvnp><++IU>_$z{n^IOyB(J>|lO}v$U5S~Rqm_kzNQp+`4o5#D)D~;A) zzv_DChU7L}I$k%G5VEpw2oI}r$!9}cTM}=z={cRsN>p5X-`3zWEG+zHH=zRbk}BmI zN&O^f60z9$#(>-)ZPYV2Z7lLoI7s#9XvvZ;cs6??)O?PP7lA?$g(_q>4Adsn;7~^Q6R!qWIJj@<{8wM{KWnK+1kc~NEgq-~jt?Kk z&tG58jj%j60nH;PKmS`&?HoQrwX$^H-`P?C%I0?~d3-Gp4{tv2d}PVM{`2zuxYS^a zzi~gn`|ZV_@b+Ocpc9zGYV=RO)Yu~W%v0BZ{Kau;&5o9ahQ;uoalg zUk%#a4I2MeY5D=dTyNlry2wH>?9E$tVz~~m^Qjt5iU3b!DSVyLIZ&P8*0K=rMLOV$ zHsCGU(BAQ_X~7*3T$;8LRk;CC0stYLzhRQCf&Sq}d1DJqo9h`&o9BCFzo7R16^Ir1 zmrjT}f~ARJ=h95g;CIP1lSZray+VE3r$0KywyWi|+eT^FqjKEVIGNU}rbRPTde@iB zA7h~FT&+k3V~DSQ2Kh}A9ie?5OyAsg)>$`O_cPWIq7~dR5|GOnAewdJwTAte$zJ95 z={z-#7q(#`N=?97=G(;W6tZ?HV4|}B4Qo-3_9Ma4B<7dRX*Sggg_T0fuF9Ve;7$4U z!f`|tq&nBSIo zq<=E`qLy9ja-mZ>n=^O6&{pCkH(JKf2jyL|dCF#K7xEP^+0589AoStU#MV*)(-dh% z+qM?0U^^~XBw94xO%QVKm627@@Kv>>e-9O{)nff|eWh%{77Z$q!^I0v zY~Ppy{z>5Dkoexm1o8{IG+%hq95emu57Yt?1>>8$$UBCb3B$z`!65hk-ffQY&x|W> z+_LW%FvILMQ&Y9|^??NFd~AeIyCZvpdK54tcb8A(A$b;1y@EgZ&!aOw8qKsYrpa53 zh!PrkeS}wUy>h;rB@oB?TCc}k(SU-2;=Fdd;4%tox;*Q6Tx_xB74M@$G$s+en;6|y z%+Wm77tJyI+@NY<>WsZ`zWS?j&1bRE4?aX(T!tQ-O<;^flGQ@rz`ua!jzouTz@A5o z4K`PeZdDhalGqlbcbhTKM!D3GtU#0OJIU;Y`6h(bPEkxawbTznb+#~ujzIkO(5pDg zU8H%j3O~~wNHS^>;~%=FcKN+l9lal^$uCH?tt#9cPhee{DVXGRlS8!y!C+2g4aMVb0rs>-LlPQI4qOPom0=dkaVdQ>Z%{1$5#I!?mIja$O~>r+q&|~?=iPidtBBok0y7e{q9i| z(@-)#a)L^|)ie2t z0XjwHXb;nOÙ&u=3AZ;`LhUJ^w1B&U;CNphiKm0FoPhFCMV;_iMs7k{{1w}CCQ zuAn6t=Y)44MM*?{aD$2B)0sAfej-MRx_dh(Gf%&h1j(SbZ4Q-JEsG78UN)cCw}%H< z4z?Eghg$;&BdwO(*h8JJN`dbl#|Dr4gMzD|S-53;8Xg5UZDyyxv$FL(G9`)Z#hkRo z=hxT%nVOR1nlV7&gR8qx&fi3e-pYgvN_1iPIii@qp9{B+^YyoaEttjhB<$h~r0Pv2 zID_3!jIHQvI>oF)5V&Wwek3rD9XpvG>HC%U{iAwOKe#FbRNrsZ=aitBR$ttf>gHaC zWp2whSlIWT4bUo1t~kGH%L9WACN{Nsr-U7FV~xsLf3<08@Z#1&0)rl&&{J&<$~?IJ+%H9+w!MbIUC%yGTTE z0*=Z`kv!c<*MQ#LXSA($*OL8SbpX5oW@zC*P;mygBMv-=?o~#L{7iv2;-8NA>7p!xdf4D#TEccfskrby3 zY`bmq($Jvj)n|^@>&o}5AcMrWf5_qp(y7zOeXjd;{H1+3F5pVd|7sPv{be9)Y@8Ki z&}k+bD~KEcis#F6zt6cZI>3IMeo~ArN`C*%bX(gDBq|3aQ{8vwHPg%aPSVP(m#z=q z%0+GkF$95g7MkNPJ?S@deaXWl5%2Ew3FHQ#sqbnR9+tMd!5J-*si~Tp>=GRV5e9tI z45_WxMp`HjrXL{lh}$L&q-X3o|A=t2r1ZA9%gYMd)z!9tDxVXx&C%idzQBl0zIN@W z4&_`@?ea&r?8IM=IzbM=b4pYr$q_Epk9u>3zJQzIwN%U6EO8{sVfy;0Rvr6dR{aNk zT5cQt?7Gebj-a7ss971CEU^ttOO<;@J$t(HgVQkAtuZL(<)T#x$}xJ2^pOZEH3;5SaOjpRGd_4miu)lg8$`^2 z)sRF<;Kx|!`1u8gYBGpQ@uBdDQ{C;VHOb?L;Sj(`%o5QU>=6X;!li)}=H#vmJhn&3C;k-@0@w|<%wu%Z*P%%e%73#*^@j`htPDYw!g{qRX$3r%RhqCYCOiU%9 z&UBBD@aaR-$BE0O(G%8qqo(ml38NtiVLXF;8tt#1zWaBk6JRz`IGB}CRz%m;d%dgU z4M;w`dkRSJxD;L%7mT0B2$diDA*-jSxTIouy)hn@k_VP~SYz@wb1H}`pv*Bej-TfLQ8&P_ z1tYM|Cq^QZq+k-F9s>yE>;;^ud(Ql_^Cs9ko96zLqIClVK*S;-qGCg-yVSCU&nd00 z1A779igS1!4tx*c1c+G&p?FmbhiM1q*`V(s`7j6*V`HxRc?EUf$0z@;MJ3;CJ0r*G zw`d&iHVAjcE4VM1L4pX0D%g3n&D`IJr5$?KHh=Q*;?0-i3`V;~<04z5G;~*_JTSRs zRxGGN>Qo*=_az_N6{-jQa{ z6ac^=Eb_JAMoR$rT%Y~o1|T!rDZyTXt+S5DTYcePfRm}}+q>sGd99UuCjgZ(`qQup ztoHXq(!$Pj@09=vqjEvrAwYX>;fh!mO>h3rd%4l5BRekTaW>5aB$%&>G&X|30hRYK zX8Cp>czZ~oV@~LPGeqciRt=WiGv(+llpKIlEEeW`W3=Rh>)$-AZ{-wPCmWLJ?}Wzy zz~@kY`=&j2W;Z~xcJ$x#@W+&tMirIzd8fNsM%dxbr+5At^V4a{$X_8g6(V+VDm|5ZnWXX{=XmW%fX$$R0Js zzsOdfI{tCuz8h2nBXve$c&%I?*8k85J&rcZXt%^QwtG)r(rxBn(<5I+3J!R>u*mhf zq&uD3_Cg9O2uYg`Do;vR#mOAA)_7m@y+i@tTQP2uJH{thV%r}Sxy=LG2Qt0AQqw+b z8)golj;RGN9 zq@wBr#NNsvO~e(L3TM=xe}}rReI>WoMRI$7uE7)0*0T6-;-a=FTD2zE~ooMaq7arIjC%CXVP*&{Kj5ZD(II~T4*hQc?% zumu@f9>*Rp*4hg;c$1oOnJm(Lqx$5mLg~boGSi~vo$Df@HS8mTKnDxyj{$@viYhy^ zwK=kBmFa(1r{964-(*?kZ0SjE^^tOlW+xK4Ak0eN8~}8jjVtGyZ)*6>DP_$w)lmX4 zxh_N65|B;caG{>)q=C5KvQ7P3az_WObE93p2!a|1D`2PrOP2BuW{3-}_{Xg>zj)mt`w zH^ZdVd^u{>>JS}|0R!aD>r_!OT7)J}G%WT<^b;;pgZbj5s_FIYBO3;Y*6*s4&v!i!;ly;Hi77r2mx-Ite*B7xk}BnvsFvj)%j@xKyWW22KcWBX-&?bW zq26NyScRa7Z75Os*&|3Fn?(7H%;HpOiYAbX}JoG8X1XEGtF zTeV~Fs_tEV;y}J;1m?U|gJ~g~R0Itb)$i46uq80(Z9ge~Qez31p=XoAvJK+FqJWZH zhiI5#YcP&a)RL5A_4>Z5A29K-W5?TkC&0H+wl79o@)WweY1wqkdK;z+bQj8X7|QQ+ zr9e)3(Sqr|ad?2arb98-j+1@Gl5qn6_=&))udEUQvB{Ch^=U04U~IsUZOx;XotyjF z-5t#~T95Vavz89B%^7UbFaD8RfA?u}E)3iF#DYlnqm;mhB|;$Of*R5G$_0_IBAa5o zk~In?YQpkh*wCY3%7Cq5DF0x$Z}nldT?d3p&a?K(_QpW@-G@g3=7s*s6&e*UEHz}H zFNm1#Zc`7r#w92<1_KxRmXyttvnWZBC12|we2xNx1h9P8I_EPzT)~}D>&DuGP4u?488zZ_uJl!paFTg zZxi)orRlu(WwTD}jc4?^JKKvgdNSQ0IjEHMJ5tL7@OAv+oAB;5Bv(Gb7>X`@PyPnA zYwR)Iw@muQ+BA4eP5@TJuE%fL;8TV$y{`LTtcEvn8|b70{cmV{r9m6g!I98S54wLf zkQx@xvY_h-h~3|C6hJoV4DA2%x*otOwocdzNbP~4&fxcaY}8e_!CqQgihS?@#l^!j ze~vNn6F3IqN&)eX{ak`k8X$e_vfX&Y_QlaZ^cy|>GHM68;2g!h>2^^0&kPvevV8wF zOzjl9aZ~fXWhMZS(t?FT=#&)~fGc%6{pzIpF5B^cDAbrAIFvGT4h?|RV0@a_u*m6& zTkbRo1z0w8bxqj>Vm`bO{aM8MGbVeoisJ5sNqo$m9qqz0Xs~%MP&rt>n(i|S7Tvs! zHR>bu!e#1MXRigPKUvzQj1JanMkmf)RyKjQ_I+6fkGXe-#n&EMe=Sw%Rk-C?XWceW zfSijIH#dD(_DO$gk*0Z6hu|DLanv3;3o635+$TQjjVb@92I8{#ewuqFn#hiQnfXR1?XpkyQct4T5M2HHN z<2xhoNdQA*!UHo9%!5Z79(oElJ|M&>Z>g=zH-5EE*!u@*-fDjzkMCxt{AowzH8fAi zBLW=o7fZzVPu~3(0L8fgOc1$SOCciL{gJ;XLe~)^qqH zz=sln0C(?6^NY7b$Emx+k`J+7)0a~HGK|4Y_i{;+ui9o6}|d#H&_+uDGY0uUK>DpZt&kZ?S|S)LhW;2f(9 z8UVBti?wcyK(a1IGx+^Tn~;&0qtLarLV-5&KqlIKyO2!1dTEC$rCl3U@yrm+eM81$ z9dVk_o(iH3N^*Evn)HukxJz|H6Tj$7_JO`{1QcMEb(8iP3Ux<>y?0VWVg20PERLDg_dP09iiqk&<66=M}%D8D#7uvtnu&|V!GZ%@?3 zesmA_5msl2&_&fLr-0hH#Q}*9<~W0kQ2gC1%SN=4ZE#*61Oz41X;K{(91mmT;DpAMv_l1Sc*2F6o{WU7UK&Jqy%?UgfciMcvB4*@@{1nO(}0IlB_H9ODtJ zOA9W3dxzujZ?p{H?MJ$3`3Gw5D7pdMJAfbcdOipm{%t?@1oF@)0kp{{&)qf`TsN(13~QmX8>7R zj#d4zODp2_Yr?G$l~=#U#Q^EU!*0^aas#6al#uJeRO|E3#a17QkSE(QI&i`;uX?k7 z)S{|SG(z@2_u25#i{wHh6B@$rx|gJ(rH%TBWVKa%qkI8b{1Gny!W$R`r1!dLR1W_w zDK~lX19a;w_Zx<2z^;GSdG~?L_oC-oH|-gy-*CFjxnP(}>w4M)fT5+Ef0=`J`~SlF zb|0NtF2}^;UQsK4A3dt07n5ezhl95PLb0(`d7|dzGSI6rZ+}&E? z&W>x-PjG>7sv5m!pxxspNzj69Kjvov@5n-I7&d!?KzEs%JwqL_pNXKmDi^P4UuGk zPLr`Um9KyR8}f|qyB3qX&%#$denJ;j0l7vmT93`Q`^P^tM#sjhF18uJT&^O2VdHEH zzJUsL%(oJZ*0Bx+OoANm7$fJc#8RZt}6v3b6!IRbHvd%!c4tXjW9!N|x6#C_Dy1IjbSY`)>y*<&D&H(RqjOm*UrZGM1a zl^=XEH&Ox9COnd#Sx?WWH2ZTDregwI4)V@3`anbLNBIcDFioa6r%rF4#%u`$oc-%&#R^tSUo+K@S7M z@kg7u6GET7BVom9yS|Sc1tKMKf=uxPxR38bnyPRn;tC_^P(%@cilY)${uN7Bh{6x= zN^W6AKep(L%7(>qJK-2Ohg(AOhu=2jl|1e38T7_95SUGu?bJ1-YZvI@Ztsa|P9*E3 z^Khk7*7G1(C(L}_QAhesw4)#4hpWsy_ja13h0{N{CEq|s){qT&2TBbQs$VcLc#>&& z6CUPB4B=@#RHVSju`44IfVAGeDqxa*=UULrn&b6oA*h+;py+f>x_h0w#}WrK zEJnqC&wRtO)FPy?h>k})!J(8wK)r!FMkHsR_XyDB>5tNiZ&g@%XXl=61pvB#_1b2c% zDaD=Q8r-!w#VJmaQi{7fMT$d#;_hxm3(w*Azn=TdWQHLb$ctp3{oQ-5&!T90C*d9K zS~1$K9_LdbgX6XT@Rn6LAo%`PWT@Bo8Cy^A3D3+LZ$f5LZ(x90F{oY<>2UBKyL#+F-T+X(*0-={aC^`QOXt{x~+Tus2!7CD%U6g3*AS zLpiq>97}HTv98F*gN+59m!*X_j4o+^8P;a{FR$L#FlyopawX~nRAN#)71mfC0~_w9 z*Yg9KQVe@$T;%r(-1<^2A`qEq&&_;SPn=-;_#FET0=)_WqRgUDIy&`*Rz|J3LWPJ_ z1)qWtld$ufm>Mv0h)3V0sNji+8}rAcn2f~Bot#H#*SZxmJ%H4~rbX#^a&v1d#xe1Avobxs#aCm#^DNgY}dko0VQ;v*`WQ({*XDkT0 zk+&cZlR>9>n3+^Qm+|JXkr$m#dLjoUQh|SUgLp^frw1e`8=g{1WsOj7(`_Z*M>LW}@pgIs@>hE;-{! zp>>`D=A~CTX6uiY6oHU{ThG8IP6*j*B}Wlw*Ug+hAOO1i(LQb@`h2>gLKMb7fd@ zH4HTT9UB9cb{csvPkU90>09u~SQakg0IAXsfKYDlW;uPl+8t9?Rju?H0BAV4Hi(j#4z!EnJ-~a@i-39>M3!mYmJb=rF{k)DH&3!{tukol{ zH67>+E4a4mz4sd1R9H5;xk)o2dP_bbay5z=*aVnrH`~_!adcgfb-4_iakyE!0nb(g z49>FCwRhvM)3Ngn>_9-p)b8+qy9?v@TUUTm>9fcs%IgDwL%;xsH1Su?4x)dasM|gc zK;==pTk86^gn4CnIVJyQ=QXnA)%wdl_kR|)@XAPgxTzZ!TYl>*w#jU9_zGF1A3>1` zCq#chy$k};!rK@$p+T}|*iVT$Zyv0~rzg5j{qw-j(+g|*#(M2XSLD`{3yF`}r&rrY z&U{w4rX7|%#so`-1Ai%LeO9d)b^blri~pS%OzzQuOFbi7QX#Ok1$r@MYH?c_X@>tLTrKA#SzYb$*7lp4*Qphc5 zu2H`wApbdMyP}XW(mXB?Muhpit{v~<(5`C6n$MeaK;hezgSeUN2Ls1WnFT`LHw>g%`s=HM(2o zm+uBadv=!}K5oU}@PwvSgmcA46}+VQ zaNw*4-GxagNr828(iWiOJluw)r4vOJFhN@gr1HZgyeND1VsUT=b(l2(T2>NXIX!S| zNm&*wHJ~7QO2z)$n*#R-WrN&?GZY^BYpdf}lMN=8A9gq62FGeRjp@KXXG=#E zY8DQb|LM**l62Yckv3R1`rdQ0GLBZ{jqxDPAWB74IDBq#P*S~u>6F|g4<&U73WUwS zXwMRBtUyq5;C}ffSKlg@I+lM-J4>6lVAgTQ_Geh4+~Dxd5Rr6fo&Bm`($tiYZG%8( zRgKiwhHmQ*HA(bvRm$t{QQPryukjSx;d4D7lrzJ+4H81{!5>n6J6btvPIDU+jTG2& zGI{^ZiYB%LD%@TI1Xw_i#+Svucw!lZAxA_Ls6yjX=9x7DZ!5^&Cl8J<>#>fR6ki_tg~?1M4#m0=*lk^@>huhwTMY8$Tf|P;z9C zgR-$pm6$!RC!HzVlR5rm-^1RS?E7a2%K_1-RI2`-LA$8vG<~BVJ+gSg-N(tbJjoGe zhG;4p=LDpSo{)>cY(3iGeR_uny^*Y}(rUhWm^#kZRcV?3V9T!YKdFp^%>Nr<$uqlm zzBcj?6u%H(cMq=>8F=tKvbl>fhY%SQapOYV~;q`v9CSJbe{)WTy#5!qDhWts zgIT}~Dji4=rT5KYQ3<=AR^oKnw@|&`4qSWM6|7p^w)p?G`*CF=B55xho`B4~cNv4l zmdhypu2piy7;Yt~1<{^d8cV@FABx7Zx6yC({exTla9@vMx4l{m2iIHqvE3dka2uHO za&I`NtR&Jd(%@eR!jK`n;*gl)_gtfN@r02=wF54I$3A*H@APJ`B|W~neR}gou*Cbh*nHWr!nJQ4LX@6%e{tpH{PbHS zPA;p@Nd>z3N-|sMJwB!pe=7O#_WDvEZ~`^hW>jR!FVsPKeLWm3N4{}~P*Ft}BEU69 zZPh9R-e8(>Ff;>~R%4+g4=d3sX0KSzNC14xKY1RE1Cxq(!LMvRm4af`(6~ubu zf@7{;pk2lzBC?-*Q$JiRQLH+NdR*8rJ`frnju>#c75H_*deXX*OHg<)?PQ*jBW4&8 z6hruJ%OEGAUC zT{Zz7{>Sl2>vAA*IQR10?@ciQ4`Lpqo07U46vhoy>hM%e>k}#;YWiZ=49(Kt@M(r$ zG?86)m{Uq{v~lg?hFcts6b&Ks10&uXi9tRm+A#~fynLCMwhg3!4oRW7dI5;*3b``p zpR2KOj8za6)c%6-TRNyF^b7t;)byY-yZ26^6+2|dc^%6ssUFxE5e`AYkDf2K!NKCgiJ!!+F%p&7kuBx*v* zhG>?bUW7PiNK;}zdA6((vorA5kGK6a#yLM}%i^=$s~1F|r|q>F=a=2qDwKMi~#i zIBf3wgzmd>%oa~_=kW<7^Q*1p1{jUq4~C+4%_B_IGP@7tpEz)p4yVf-##^|Sw*51& zd)z8Qr&~tpu5y)NlG1o}r-{5&4rnTjlU0-64l&6{#i0R$2dGYl~O&%ML%9BO}-{+fygb3*)u{p+ip^j z8?t3F3_b25RgnxNI1WJ`;M4X60_XGlCzsv_k^mR)q{QJI`<2!P^!ERQ&7%qZzv4W2 zVC&v#sPy6J^MlXLcCy-K&Y54t|Kp;0zMU_xEba{Cw*#O$$FH6IzZvnJwWq8==-AWb zo#G+(%Q<#c#>JW598fLih`V13-~6+l5=W~jwWCK>cwJiGyd$G9QG~4pv^-zd09M&{ zW!q`%I`~^j$vx0E553aZXcqeEC&zPyfztd*W8=Z^<}ex1(eopV1^#a@k**BXc-B`+ zLu-<(Q3cp}T>?qKcx=rMoBVAzpk2D}CGPB=@BG^k5b>XEA1Jr2pHd=X7Eg!-(3o*G z9of|dt|pCw!et{Dn~n3(ksTGQkd-~Mh5{L6JTpXsmJ!8n;ig5a<`>gDtT3wu0)s{4 zr&fsbd~*WxB659U*Nldm2BTVz^scDhyXL@5yNMA~l&E;xuhR1eb{u%m!*qNIMxybw z?o_#)rvW!QE;Q`j-e!lc{sYA0wo9v*_XP!3h3~W_(hEH(M^%j&1HLRAkVjG{`3w9j zqGhd-C}-D1aa``m@NY62OvJBK1{&pAK->fJ3bz%4X-oZB7gs_MyjMdmsB7Tp`tf6; z`Jhz>*e~hL(*!Y4U2$Rj)VS5FoLAx@j%#f(d4UsS$L)^jsfs};fDbqE?SsZ8z90X%skH9BWn3SvET4B*sTk;S-BYl%NH!m8Zy%n+lb1fX%)v7To> zFSL7yEIIlUD-OSJT7+F}962G8u=J8k*v+kTHgZCSRgyM3`YUZ^Fb9@9 z0u_sdJ+=0b@SUu|E{;OV##QqdqS(pJ30W!A?Ae6D@$fF>^5uNm zZ=r{L@@rtlx&@P^zYf2VkuF3yVcnqd;|U$A__WsM+ygDw-`49O12}ewq?iEha9W3? zUw>J-&9GRLMWd1p=T4<4GEqlRb-^N5Wmz*E$Lp6gL^dC8Q6odUUJ4{Q#n21N5D6h= z-mC6GlNvMmjlr<`X%=7Sovc1~xg=T~BjQah-MI*h_=30Qgs#(RrI2Fe99k#XlG>vR*b%?r8SivU&A`(H3eiSyO;FNp? z4&+Jg>FK1Mp~7Oq8ut@wOV=t6DH>gIg>F(Md%hEbOn|1Aiqfu#Bx==N$M05rpj$(-0Wo)YuV^>wNN5I{k#lc8e%lMOe<07*$u zTTQ79r}AIU_U+gc$`?|3`kK;&=I3$6puh}<66L3q87m6Pb8v9D9R6nh^eF;FLFo~d zm;K&r(K^cYX!U7%{))fg5j1D4pXc!9S5;SQZbdG@j$kU%egi`{!wbd0{0u-*BMblB z*np#OFC*LoC$UMwTVj72!AUVOiERX}@3b^|J;;mioaO!_Ik~fp&vx9nhkb2E ze|=NQLc9ClCV=ozBaCSv^cGX-D(SAN3RtuWs5$H#v<$a_BE zzwNuSg8&QT4g9=28RK#P$CIYpo%ZlbEJBFH>quz-a(f z5(hf*c>~Rm*M5B-;m)t(08n@8c1i!=ya~-PG`bLlOdyhI0;irBr}5^+>TJ!NPUuwF z_!=KKLw6#;b0S8%c6is9Nt3UbH>td4o^tr;NR)MP^CI`g5Dn)Tk$4VHz~rB#df*U` z35JD^9xOTvT=+jFk=}>X)N#eO=C)eVOG>3i`lTc`4FcIjax9^V+}r@qc~)N9&1c+< z_qwyQeHj}GE4=u|G(zt)5^=kL+_p&DZN|<7`;d6*Q14<1np{+Nept&6*^A1_w{M{dW5);Y<5xm-KDNT7R5Mc`PaBmUbi2*OV$^Q2Qg2_J9P%l)lwz+_@7{1>~)MF zx`u1p$1O40LQVaNw_>Bx5g$p)KGZJdghqO2le}egeal@{LhdJMsGw?r6~3sWiRof1 zNm8UKlL);>oE%WENNaiN^!0j~=*rTv>%2KRJI8*xlOCVobUwTm>U{aP_a!&mIWn3s z{9}%^dyBN$*}^#N*2lWxiGaT}C8{}_sUOr_&R6pasRs<}+~a;szLq2$@)B zS&*GN;x2|EA3y$e-o0ktoyfC-n&;3()d)zG=1<&D>l|e>7Y_B*>lMaH)-PFCKsNsf znHc^2m!L_LDpkTvOiZL>Z1~Z_EfyZx$F~1j%GPdpnJ$V24P+JQCR;ndB-`Qa z2y)=55{jRQ3w6ZE{xQt7TrH4 zQxsJO2GjoVLW75gEq|qADF$<+InWIR;h@pe7F0_ye}B8Ki=?||@5|Bby_9tCHz0&6 z|6{<~=F`EUBm*llilvQ0*1_DSz~HK)p-Rioy)i>4CnPY}S^zp-SiW9G!Z&6Y@?<(x zn@l(?#Wc;tZ=}>dP(ozZrApTNA<~MhCF|JXyd839?UitfY)$DexQjL96=P-(H%O<~*Wd+|CLPP6#@#+E~VC7kr+Q{I?~@ zs*%D91T>LNTrA^o2LOW?uorWeY{cmt+AaOLMsiPoQ^}YoozycS)$E(J7SoP(`MeC8 zalbkp7fvk@5tA1FF`b(mK7hD=sB5uBQ`>-+!>|uB#Xz8LH2A4Kh;zd{`bTp5l7M#u zaUifh{%(+%+oc!SjnWpyg5Kn9e+az!0DALg4Nja`-1xw2Yv=WH?-&W4FnYZNi(g0m zS3fxern)BEN3LI!rb?unH*Nq8&s}4e(lu}dIE^GbjVt&Za9#Km0bs{O+MAVb0Gq%u zXp|e6L?zP7`-$)IYX#uqaB^~13047`Ot<;ai56hJ*As=$DB{-E2X@_#>_X0X{f+VS zj`1b1Z#Z#dGWzfx+kanL5XgG1dY_{=aK}_B(nlM*HZBqkU~- zyM&Mfmv*Rnl{Azv6SI6)l<7SSvI-{LxW1V5oBdiX8om;QhncQ-Psz;KGV*0nkj7z+X~#}wdW!l9nimpo%3zzw3;ju}V};Mx z7%$P8lq+a}8qF;x*h-tQ?YJs*Xcy|STLl#fml`m;9QD8hr?PYjW&y#hdOZgL0fGC4 z5k{}Oua74QfXLTV>?Kwsp8E^;~0Qa8aIFjtTAXgDjAhmJtC)v z_^5ckXbbajc3I7t)ejt|RL<~A%wzhjPjkxu9ITlX^BueYm0xa#-|?+=R0KIMb0dmv zu?SR%3KxzBH!wI*ngZlbsd4d(j@{Q;>FDSHZpDOenN%pPSq(%{^eXQ2u;6H#0!>tE z2}V@NvBB4!_Y&eXzLi_eebbvXtRsMiNab6}EmwyXcA>d>3TrBdMMd0rYg-mNSJHw` zCIfd?)<}63XKi~sU|aW|o73hZ3ZXL7u^Tc>XoGRvz04LCvkQfW&7b;W$?Ka#>VKjH zCr}kjDui<50Ipryz=M z(?n6HCiaJ|J~lW7;TXHPwZR%Ok=rnGXM=~_v8cSr^90NoJ50_nZbeM)Mw@-6KBl%Zc zRh}82jNv)R;V?~a{fZ7FZMMFnDS@*fAS^%p1i$oeIzmhj8ReTz^fb6l4%x9J>T9x_ z0+_hpG3 z<_5;R*#CDSgy#J>ry$Y**-+0%SKHtU+VGRdzXL5D8V2cw za5wpP_zR7Am$TY)f^uLm)E<{e7>1QtE1TEpK1ZGHoxM2opCotc4SJ`a-I7X;XbD9E z0#S`=J3H_D``_-khBwc+%wznaNyr(?^Nkt4&LvD7yNx#YxxGJF5HZGk%hBpHFDHjv zXzr%RYyTU2dRd2FIqSE5L?lAg&&Z%}k$;Mmb;v9&{Gtzsm~K_phRw^^Rr5qg07&rK zv}1|inXmZvi1G7_dfBXy&vl+gg?3Xv4=5QqzoQ+v9ErTo7l2jq$k8_~03D$Aa;@e) z;k#Nl^VQ1|V3n1KAUp^hpc~)80ou!T)d-iFqo%;eCQoqKksd&zdi6+Q`>n$ZQ$)`P zy+XEMpIVqm2hh-Qp$3rx<8=`5>#t-r57>9?cNZ|Oa6g(k4K^EqHz&R65*-}Tp(tUe+am?*sAqD26=}TCw*fc6@YZ`#|}*w|$hUfU^9zuy~m?y1#1DN&&KQ`5Tz@t5&{mo&kX3 zi_HoBTK7A$eUF6Cc`m^;`jW7`g#$IGJca@EO|za`J;&QNk0nLiXb{pE$^&MlFSlNz z1IFi;h~J7nmBZmjoPL7ZIE-m}4WrVxYoF$lxG=n)O90S{QQ?#|ZK`WR0e=SchwR7O zUIN3jFe(h`AG3IKI;3M*Wk;9+=Y{F0DHcWAJ3Yv!z*Aud$P%qX(M+ck5_kwm!9U+R zb4G+!z?Q*E;~*%X~!NX7X#hf1>!z7PTNkmgCEZ}Zr)kFh5i*`H1f~6 zxls`iBukN=b&zV&FR+tJ)2kQ-{BA;C=S0Bp3sXtNZYf6oLo6SUxJ! z=H_Y1t2UrVQTje9nN>Pxo!nHod?Vx}j%UZOokjWN_a1NM0s4ks>SGp8hV&grD!YAp zY5N{5D>G>a87LQ+9T5BBCN zjg`R?kf>;uFcvyRrTQcWmuVV)ZaK2BYLu_Iu9);>IxQiSBVVpz1Xe=1-RrTqd#S?Y zn7{u>;L=Jd43ZNro`qNsF2b(r#u=bybB4?QD9^1gNMu=zg8zC%uhd~t8kQ!Zm_kKn z+@OKOQBZ%}+E-&d#q&n{C>9gIAaXO__1$a=@XE4C<8R;QU@4B;RDry6R8e zx)sKaNHrg#is2$mJ;$8$+iVMq#{InIV0_l6W$z6F4hxl?bfwV>!MNV-!k6BFM3+T4 zf;6Tfbw}7mv`wwr45Z3{1{!Y14P4LD>NG!_DEollIgTVsNeuliQf zLs2hDrn;4M@_;GjN1c-&_Tv$9M)^r`(P8+af1UG%1;H$?Oz@=?n!F#T2rCxUoM%B;io90+0|KgFwvdvm&HDd8jI5D{9+mXOZmKqvB zb@cYmc;zXv5-O(J&e;$IMqsr4D61&zX<@tObhAL7HYzxNs^*r6n7VL*E7dX3MMTgb zNc_e;_r>pDX0TapYnUHg)=Kimq%-$r#g>0yj_wf==Oh@qjzu$Ydu(NP2J898{g7xH zH^0bCIb0bDYyQ_tCu=I&neUmeuC9?Kv~EoNsqfaNN&o4_I4}!{a5%gneGT3~>2uEV zx%}^HZQ1Uz0{TLjpuRfK>;oj1RYa=@_w6BEKw;QE zV`^{<_PJ(K6T2_+{JbQanU%Hm`Q`4jCm7>DFmQ?V`Qp#lJ>cKEyq#B~mnZPofPZnM zdFC}J>;XooI#Q0nq5v9OkJSGVl0cI7)u(VZCyRftnSk!tV&Z^WQkbGw{~M5h4M-O) zyC!!y5{}UlafJZ*%|e9y^gKB>;AdG z?xLH!sV;$IjUJgPn|j?budn(K!H~m$S%zYYhX&aqid9{ zBVV0Sh%U?62}=lbAazjtdtE_H7F)9{n&f9NrAM4}&}O@8>jE|$tL$Dg+&T|RMT}W1 zTW95nehsDNR}Q2_$+a6olMBbir&gfs>WAgVI8M|a!Kom9xfRJ%*$^Y(uBH6W4c6S9 zig2>BC<5}Qw}sby6YtSkb!f@O959KLw9o)Sh^b$>9Ed5FpAI6$jP@mia_g`DZGO@E z?+nE8v8w9M)%BJ=V@q4JSiOz`Lr+Q>7Dy&o(1MyTX<4QAgXg`wo{y@VA}9d~z<+`1 zNCEZ>7ccKv@IT-C1trGnMo8h*Hc)=OFRU2fRtLV3LIK*+@3TKv=1k&b?cQ-66KS!& z|9i4B_{S@zv=o9x;e(7V{-pZR%1IvO=l#{5_w)VU>rQwSIKu#+vTrj6<_Dun_H22D z1{9_i_R)z^q6oz-y^)poU&mlZ&FTcD%z2Q_GhpS?Ij3HX0|Ne5C5i6aJd!s)y58ta z&wg~?`SV4|&`qB0!w>SeUaaQk_{wJsg9|eyU&I6BZ*JaCYta_VLXai)uqKI=zR00a zv81B(!tfyXm&w(INgzdYMOV+&dbtF@tLhpUPOfqt%z-X@&l>{ik)K^#Vo{_usawyl z;!4^B5zI9aP~*nvwLeVYF_IP8glMHH@)u}nYyHTKi^GB}-58caQ0>3mAh%pu!}>Jl zJ7Bx2x_4$&9@RAhDWj_VHc27Mb&sqypfG3u=mt0kIK2wgq$0%<(f{TcwQ>afiJDgE zfi1j+=}dp~B` zg|gGy&_$2EElOlDqcwc${kSzdJN(m@*JNcf3SW<6?>w6h9D2_Vo;A^znPNyz9$9uB z5!o`g3p0h2N;sU1+moXTqft+UdSS7INivhM@~jJaY7kjW$O;(gja<6+k1!swRcU|8 z$o}{+DLF9W%Ut@sO4i{1z0s-A(wKP_!_V34rk^_O9mV=|PN|P|DS&h7*Z#is?sX1K z$Azf5d7V&D45O!uhk5kaY4M>{jSyEwN)tHJm$WAlGKfIRoACSs2Kq(_z&!nw{NI{SpN*3k(E$`cQzLxTVEnIWQmxw4a_tn}76z z1)mGp{+mA-sZ#p=eXY=~r8gI!1G8TvI95?DBHU@)#*Vf9uM6MoOgtY?w9_799lY)q zf$o<~F7RnEa1;o;Q(60bhUX6y-xWemfr;_)RUAiqBc~i73yFJ8?zO|N)SR`Q<)qnl z|9R=~%{cH{K5+X5%a!7Af2weFY|L=JOB`^cVgtRgQ|XgbdYe1|3;+(CHlJ=8mGRY(*h_F8f;^W36lF3MOGm{*mCQ?|PcLDQ(4_og+8zHM(11 z*47rhU&S^yds$`vJFd(npeK(OQ*~$A;(CYFZW~6d?rROvSC^esg-76&YX1_fFd= zN^$6gUHwcsGUnSFt8l zuse){VkS$?RrXgf8#)kK`8i&rQj6rx8!MlWpX#hsp%=Y~0s?{xGbvQiDD7YDML3>j z2sjATEGU&IePk?IYw&w`oJ9Z$r?$4X_tz!Q-gvI*-+1AS4)geo4&PEO#iCN`sIAJr zElgTiO{VF3*|G+OWuu}QJ=!QTP|hZd7~)(UqK2+Ien_*$q}!iAIJ8#AeuTGKMRyOx1J_rQN%@fbfgHW1H`@G%ztUi}UGF0fAz1>GVwFvfeShckPDiP;25Ue%vzg%$+>CFRGU@RVbseGx{WzpRy=ds`UX$;}r(mJ8_ptLVmD|gvmKo*6yU->(;a{>E zG95;RD~`!@kSc%B45HDM%Nl6L5T^*aw-t9Kc%_C<(YcL5?;Yrf3(yUmGaFPB8|m#Z!dTiYbpmQi@@ zp9W6B1T=M|xEDeH`pQRiM2ZZM0FE_Hsc2P?}CZb@bPOSn%oREg)A`Gxy(fnJ;zm{I}Fh~-M7{i=h@)%sU9jI#0B}ZEvxt^VZhy1p7X<~(A>P>-G!Qwg{6Dk za1<&q2G-HjlVpZ~J|UO})qh*c1tvOf_9?Vo=U3!^x<1o!lQ-`OyyEW%1_o+f58c2H zv3b zGR>wDXz4>>ruN_hZ(P+!iz)J)2zLBCRo@{CAj3+6fN8L_gGE?W6l7c_cVoGQ7YLu2 zwrS`Ag%KdI6dABkhf$$PZ_^uES4rr;OVb1>1jIphL)xQxjJnvOU@X=8Wo;g{1T+F%i;Q!ZDSnE^J~lY#oua=_FZ(P|={# zEV?ACPSea%#7^m$CiGHmGYk<08w~Cf%J|?UpY|N$#-jGHgiR`ZEtQF8N=T8CEiJTm zBgZIfkkm)!HA019Z|h6z){$hb>0cC*Xu?+#P12%Kf}o|i^`(F1O2Jgt#|R=s!I<{I zv96D_MtIj>peU`vQ}OLfyqfxFVib4uScxGV6!Z*Tgp590$?==t4p^srC>1qw<`qh3 zL5`s+_zL*t381fBm(xt4qbzd}x`JbXSbn8x*Q|J;pMq0)K;||L1a@}xY1&do;_vr*&B8yv!)3%lu$YMI z(ZT%A+PO-m_S5^QAtDc;@TV$f$eAbe%~r0>Wb(}hjdky=dL_0Y+?pLpHVY{nZ9lNu z8Bz`)WYpCoMXNXH?^BD<3n?qz3YSf-AGbgOFNk`nc3sP0+AuhD;<`bD`|RYGcc^dT zZPq}L0trQy1llmrn@Gjt%Z}gc+F@q&L02WW9mj;4m&eLbgs}ZMf1o52pBcI=3l6vi z?@=~8uAa97jS>o+W7~-~&|(_b*3=9GXLhUhzl%pevz_+xVEjV1wY3G%_g&A|+0!d& z|JO#683+y3Q?d=?23+4agz@E5)1S@uTckkAb#hYhLms5a2EY54t!|R-PxJ}g9 z_f>}sYoso z;j-}^+m3)Y;rNm(8&oExpzHY58(zIpT{<2!x1>?+143hjgR4%bcL2$@KEhx`!0PD( zr@O;_TDaHn5$kcX?a&3a<+UO2(q7CTpMXjl3FFx0o~xg8Q0C<}Ny~d3YMuE3A%k?ruHPx=n%) zVEvXPGWtm6ZpAmA$CkbhHEonQtTYC6xEgNun|fpWW2_b-U?npSHbnD(Lch9t zI!9jHt9m(n8~AwAwOdSVY~pT-_+JRPoW`zVfWt;hlXk~nCx;NoJY?~ze$Kd{!y7KE zlB!tt2N(QSzqZ&|h`A0TB(u7`vA^`H3c9;|#Kyr{z3_()GA#yGF}HUb$R>1;{?bVG zvCK~VMK?Rm^p%Acg^NLAX)={@UVWqN`!w|X^wrQgfs{iKzfSV4Hy%BGgKNJ`$LMc` z@EScCAx(wEwwA?_GqKirG{q1+ICF$C5D}TP=}hhKYWsmoN?cvFY)mKbt8qbdhsLP# zKmmmrxo_-va72Bf?b`lfAM7GBs#eu#vpvv_z7Uwc^St6s-!}5yZ%BMHDk-G~Hh={+ zhA6ytfkJ;O-`QuH)qAY_A{9gn!f?uNB*RFVMy1dv5zzznoflQdR(@1aaa53YZVSc% zF}OZ#+JNxdfX$dpML24~zG=a}e!HCy=fJKo|WL4>v}ymR{8knn@jgS#9;Vh zVq#)olOa*IP*An5!cd`c)1@gv1?%brsB0NUR)&2QH`KN zSxG|E~-m8UEj-;j&jK_%6NCQ|{H! zr?MeXr~fK;IDZv70JS#mb(MCue?xTeny2IabTQ}&j?K~@{QPDu)&IUk?4~patH&<= z2+%cu&x7{?nq|Y{r!$l)Rt;c<=NYi)=>d*4EMC4@IC{`*(CgqJUiAU~h7$|&Dv-!sguzK>}(r;wMP8hQ@a>Yi#nk}agx?A#l&RE(pJA9m_h@#wu>U#pL)Q{6S_~g)D z?LQ?*x5K>xUiMsFrjH@PBGLM|MU-sjZBQQ1jUJH)Gj?^Qbe5*0AP00CC{+_Ec^_9@ z>UxhGB{HT5{D&i;Rs{=E)|j>;Dh4Bf5i+-w+JqQH38zLpavHL7UD?)__gw44{`=p% zu<2s9YOw5`S@qM?(`rK$Y<>pZ`EqD36g3*_*W9j2!9HbbFHJE$1hcBMu^T&AxMy>k zbe)PfDCz=yyXua9yW)>d+rTsH+JEfe+j4fg^hcOiwJpbav*GG#MZCQm%9%|S_3j&k z90^(3*OY$JzRO=od*hrl_KYC%eyc>1NQeEFd58?s6gnLo^49gDF>DwIC@8{osMFuq zYt47{XV(*PSMOBd-+F)G+-CvLob6m38q#7 z7eQjDnCA`!k-u1sy@NzCHTDVScOiKiqxyI-W3pbJNLXdlmdn}gWgAq?z4bbBAotlf zUA-93+lp@y0=nnKX(64(O8k!IZ`Cfur6c6LeOxk#uqf6Inb|__(SdiTYZvKjcW<6< zULF`YyY6^1F8x_vmM?iX2nr@xgq_6bwL!9z9L`6V05FSMy|{2BlDljt57Kc7O5e{3 zLoPq7PoWR3i$r4Kn5aZ@k3)np2xxaAhke3_6VVE1BQ>ccx6dz9#jSq2t7y{xIqo*= zt#Fb!MH5Xwm5rFts{LNDbWUs){DIWZU&{0`-4PsUI6gx3D$AR6&Z z7~+}!reMxvq~44<bkzDx0SmyI<*7}h2T$X74lL&xgHk1rDL^3 zO%2`2D%~^_-nQp8>m;lpLZ%yto6$6&RPTCk7=~xg=aM;j+;1-+44k8K-qNeEyVyg_ zV;U%a6}$5;Ch>m>dMhj)|7AP1 zm|p`%xsjLqz-IvS7b;IeL4gRYS914&@#@P608$bJ@D9ULEgn#+|Erg>J!Fg*|6872 zysyx{pHA=W2NEt}0Toxx%*Ha`Yh4pXXhpw%#X$j!t*p+IF3eZl((8;?8&D{4`fZ>; zokxgYJKQ_W0fxnloSc;du1|-6ikRz_|LvD`zNqcJb^$g_fFMGWiWMXQfKb9|kYOpL9Kpf(S7l#i0w&dB?!Ih&3vu%+9 zlLuwC>3XacPF#Ka;Hb)k)f$iKo6$89B_!-=-MT6E{;eE#=7_QN8?TMdqd)erW$2bs z$C;Pyyy}cExe-!EoY$2R!90x;Xx@~fcO{-|Rg4;UP3i$yZ}-+5aMPL*-fBFVU0v-+ zWcmeCs|ufYaEb>*<;QQasj&T>!%xIN5KIi4G4H;!#-|Y;l$>BNuaZULBR;ZGz*ul! za5T!muW*C8_*sl;mU1d%ph9lZ%)ClO~)UChtsP`(vVS zb9HsEXV>k!v~x8SmL%e<*jUw0O@sG$E76I)M!u!48q68|7K37sprlHyCYTb zw+Pp1{^}`_Bi127?rvOk$gIGUVn8sB#%Xz^1||qVb~Qlme)NR5ThaxR&wS^7`?KxL zk{T7}1WX-J%a+_SkG@}miXZsG2seKxxq(r_d^ZGe6y7@nL+~B)} z!JhnG^P7oGc37_&!_$b`HE0U<7etd9h&(P*q_2F(T&2!Zf&a;&NVyh~LJR3!ulq(+ zQz0cPzsR*6-}Z{$sjr4WTCXgDqx#ji>>2U zcnTSwv|J?^#7YCE$)~-oCehZZD7nw@yq?I^yNOyO4cZa|MDFu{<_u<~}@BPymX3mT=XP^D- zwbo}jifadzxPCZs4R#-Ib}m+esHW%j7RlJl6wB2Lny!FD$>%TJyrM2~RaI6$|DY)6 zlhP|eLU{>T2?H_EgE8{6WC;@iuC#x$TDn zM-+uDaU?F%&2}A0Mm8mebo3`_82(K0*7AZ&6+IJEL{0?1YSsJ&)E>V1hwH=VKPO$A zi$-^p26vKJj#QfXF;wCmjUC_KrWPbBGp_$B&$M{=E-5KVD~9@!9pnK4lNI->^fY8wed8nKdApj>_)!zw%@+^xMvRMGZ@&Pfqf1730!nf<9-1jGGGkpK* zZTx$KC|3uQ8Cdaa?rtg`Z@?e8Lv1`flAkiz+ji zuCA6pto_ygsbX>yZ$fkYZnkDZ^6>HRe^-Ouxx($JMW`A%_?=fyxc9&pOg628pEnP6 za~y+9In<-RS`Bq`{Uw4Li&^=5({D3=^L+9GYl-=0jJSJnWYhP6-Tyq z$$3BArA68WeuvZ8Ut33_Gxfou?{H$|Cy>n!DNvrfc@oK(hSgy9dH)-R7L>UKHMNM`65PL<&?$Ncv!(Iz|MVlWyEppVzp^XK4PrxkcGW45p46nZD4?@|1 zK~plUXzPT~KI}abZN=@xsyH@q2%i0Eg^(W+1P9>ll@KY&>*o5YrsC!a3R)0o(3( zZT51HvXTuN^;tWC;a!3;2Ci9OlIKjSoW~xhvCr>}5RjAzG}w|B8=Rz2ao5MM&(94T zdH#v-l~yL410Dz81A{U_s2X#NP*{G{IS03qrKLlB8!C+D%dyNS0vUD0+$mcGP&m0i zH<6!sDT3Tsg~>Iv-hQmsW{3T;wS0TF!ML0s_K*%gDFgD9x1^NMpYRnogDfQ6#^6Cn zm*1=Vam?JU;njs8MMUZ;Zg*^T9T@rGP}|s8**U`9(OXi{-aR34<9O0^Y24pG5IM}4 zsLfD?xLP^}Un@jXWG||d`U;eR!UtErBUCLl?(IF#T8I@?V2z|jps)7({nJ7VLW`hK z%sFCjU;57Bh2V?foe!@ysbrqf)bxqfI*+DSJx6-S$84i)gO4|0#TJ?6cuSeO@vAsg z20geA!2&@PDr&2rGgAa%UHGo86PcrmkQ6YvW1)y6ZNNXysRxO->ZYMnX9Up;aH|#I zomb|-FcXk;zW*2eS3Pgwml_p!1dMgSvPUNM^Bld=+CDEuyuRSk4J`$AyNqrm`)}X4rE-Z&_3OIX8Dh!OSR&L1*9W;P zJA7KP3DI)IWQ)4*QK5-+2;y?6Qeg=|g^)!aP!QG@_=0 zW`&vG)AH}0G41}M`(Xh~0rxTOHEHE&U>A!7AA|IQLDNdvEQ#g)k9}% zz~4#;$I08DatrzZfh-0zW%mzJef>(e4^8Pif zQ%(LEU3n`6I=Rgq9I(m+zz=?!0pVI;c+11GLtyqX@kA_yxYJq&4~aCl<(AU$T{9*G z&S>B*swyUsScN@LC;${L*O*!cc?p3_ZgZ_P>)~|GQisR6Y9R~%)4O(EL|Gm|$-u;* zD3Y(FncsjDv-V`p`SPt6yx@s7FPt#RWmMpWLPQgqDv8<8E+15ZSyi&;s7FmUBG5Q; zSzTY>1wcT+u(bi?=kqeBdu;G=R^V;e6TT4*DJBp8H)XIe_J6XM7m?e={u_`RuDB*? z*g&}^+4A@bO82hMzIKd$TtCl%;_Qaaf}q>@D-d^u;`}IxHR;_xo~g1JD92BP zknU<`%sv=p``rSUZ?o;l(}wIJ8bX<1a&&q;^Yp2Js#*p9a_sX6NA8rQ{0XH%$fGjiI)4O!|-ET69ip6GE+ z+~nSP#n}@F-ccCNxa^1Vo38lQ$tw)(-8}^z8DXLEnHvjS)pw|e?U*lMS?=20p(`6v zMV_an?_yx(V;n6Z01PGKSQPwU}{rdc@e>1YB1qTu*D?IR4 zWZ&|ESPARCk$pGIED8djWGCRNMP+5{j8CKpUVHGC3vgGgvj5=RB_f$AX4M_SmntYL zktWBofs{ohu!u$f5!jo6{^(`L1%RyZ#>4en;G%dXAJ3pPDLS+&xZy~Ytr1rZL%k>MLrdveaBUjJ8XM|&$DP|=y_S%7?H9Hj2Md!%=|w> zgx$wWgV;SWCuB^c5QoUC@^d-INbT@v>68c;*Bi~(NaK0}jILH~VB%%iwgfHSvG|c_ zda*weM+u)9?gvdcl3C&Ro7XyTHVfL&Pm#UeV}QzK;&`j!3-*zz7Ax}?I$Bws!D~B; zye{EK)~8p59@bR)YLK(OD16INgwL-rq|a)=(m4nIsxCLnA~K zJ|xjPMKc>uhI33S;;$tfTspdX(Keo%e~+4PPNhmvCo$L?pg3*rWP*h)>8|AknXReS zT<#eMe$TE!WnO-wL2Le;-ISweEX0jgt4zQ)Rb*6U33|CRq2y$iUKZdWNxXZh%c?-A z5J00kyVO@yN*vNK?d~#DBVmwydv(FL1(Oeb7HXT zZ_%(OQC!gvxUTl&Fu|D_jk`1MM?P(BZO}%22ZSw4SJ#BcYpKU{@C{^Q9%urbXNilW zr=`(dW9WYYG-E{XqpAU6k0a8|t6pXg(L{3XC%dgbz9Y$#rnUA}rmYAW`JIT@CO=w6?=e{G7 zDATYFR*){kN50>5rTs!j!;UgdtTtiCdTR{lal5`jz0I&DVS+_5LO~dshiaVKt%O_% z>K~}D3~4?@;S2_I4abzTCUSnX;!_X*s<|b3u`9;0R_T9!^-0=|SCzM2_W3;Vi91dL z3H%ihM8zDVyl@*mNxBZY>KESGounttxsRvobtBwfs#ZCjG6Z5w_QRWrz!|mTc!n~K zKw=qEljcxhB-(mL_U2{IYGwUAyxOpAIZu~?n*X;VsJ7KV>+nLeDhP}5*A z@+HvU9VNMf^*4bjReH!V0Vgq!5G7GoeL8l>JwkdofPX3-YO+V&en|%`U(_znwnPCC z*6rRQzSfIjTdG#4m~fo6F;3J1mL$3OT2FSvBus?-#l_W~zONu&{*Q^dvvU(>daAHl ztud^;Nw0v3S=jZ)ZLb%juYPa!nSj!^DHZDVpu@AHHL)J$g zBbb><07O0Ls>jN4qB1@}*TOBI*tc9UReK&pXsSMxF?}3pnd`a$cF1G0ZW#@iBm~nkc;{dr9Jm zeaM?1 zWqtEf#9S?B(vH_l6g1;xnN=w7u#|ER@{}$!^7La1Frapos>4&F3C}QJt8%KBl&Qy% zq1jI9$J<3#5wH?vv*9BXxAIW7OFyOV*$nc# z1ct5EYge#FK&UAhvE*F#J?-y#@?y?SM*;eYoCo{V9@fdf{TVBWCkF{Ws1kmB#A$z7w>?@j@L?kbpCV(Hocx4bVcflCQXP-TVHsG)8@8%Cc?=1`* z2d3STfN0r2`iNnIyME&aO(7tgUhmAX8APxse0pL`JI(*H9sh(A7TJl;Za`VK{> z0Oa$y3kLu1NCs}R0Z)l5A7BczwzdYE8s!4$lDXSJTbI4=@cnNE_SzjbS)bRhX?BZi zUdroc;z;l6kN!Nbj?=z|mw z#c`&DJ>?nSb9pFi&R5SLFKqm01@0cjUzfHuvLBNZUQ5C5Jh~2Q@AnQcf4gO$m|YVN zCz>m!=A0ciApUY)*G8c%M?Gy||HtxATl@UZb!KaSXw>1V_j_(!3I2gV--KCvD^z`~ zfQfp%3gkR@JcTm-?UN=0-3=QTKfky?f3S0SHvg{tJ&Qp-G4*yyWZ+%8#(Cb0QR=Kd z)-Lgb!t0c_|CK!p(b+w9N4IruG$fv1iDwy7Hk*=4TBuo0XBm;0DjqAXq(3HDk(mm6 zTaj*UW{VBlut_HmfJ4GR4wEEV*wZmEf0tn%|W9 ze(%UDo`La`Y^6HaSCRLo+@9m>{rfg;aN#6#^sN-qiK3{lz2B>ohJB8X2HkwPe0(#2 zERM**RSp1ds5nP&A0K?>-}H8OD}dA+Pk&|8Qr48sl-+J^{YR?LyrpFvl}%MBof!~h z@)*0r4?nB_08RP*RG?P5(9FlKFl?q>wRERPb?9jBaiO|3;-PD zta2#DEZHzhoFP4-3ep}zcd1!4BR&RaPPbP39|%PD`{Z(UdsU>K6Qn}$IpuRXB(21V z2m}RYeBIXzk>9$?f2%5@1>-TfM(OlANAs7(_6mqPM=QH0x!6Lb`xH~m%SO|^`QB=A zAiy3dkQotDx+ZL+)TU{aR!B>#=4tX|BADoGa(6_v%S_`9q)(Cg!g%4w8JB7D^C{4C zq#-znsnEnR4_S7#siK4Juj7mlE){cK9rL1!jk&Y}$@O8Mg6wQqSyAcikKs&S(7c!< zgW9>J5-_8`P}hZ=U5BYANM=px18cFf)vjaRe84D888^W8boG?Cd;Sl>*(yEqs1pDBmdMT0||#rMQ(b2y1buS%7nl_)$N z+~f7tVRrtsic3yE*J}bDq1KC&o1#B?~%8*NB*b3fS zt1DO4*7nZMYT~{e&k&iqY~eAm8(ds>O~~I6HI9;X_SlDQx3{Ltgxlz39H>YM<3|4Y z#`3B0{_l#{QI&t!_O`;qx&H%Sa&q#A>)p)Xa}|C`O)b=$5~sIb-9Jew4GLKgZ8(&r z#lrq1g;Xn(7KJ3L;QyC{SZEQy`gb+nt&0}F^Enm}5*h%P-U@cz6YE6Omle<$O0oW_ z+*kj)5!yLTJ{o5Kn3;7s4iFVCCzDE|+KPrGicJ))I zqZwq1jZjme(A}lCnY&M=<7(m6_B%$I7614GfU>2dqod9Qxzt5A?y(zM%gYBRB&X=7 zoALiiY87dQ!Cc9<1Qu!RnnDxm%G$J!rU}A`8IOluj8e5j5x`|HaRFp*MYDHUN9g8K>2E&Ji7 zakuND&y{qL?~Ci=B^?KCy-3IRZRFtxU*D=f^;}_Armk&W?A*}ZIUSP5%EqA{+=<_t zzNjhb-eJP-*G}%9hZcLrc6G3TX9U;LY4APAu$$LV#WAy)g{(+KQE!l z#;AXs2Nx{*enLh&Oj@b`vdrl+~ z+LY-TubKMREFrra)-_Emld5Gx3zrnM@N_A1JS zh?y5qkZte^RkjL+*gngrI67QT8J%wNo0V~nhv+h4$tttuB-QRSs6y7d^rOc*t!ahu zt3zW|2HmKcBqEe9&A~B&NdIs&Ne2>%^yWK@Li|gWDI4bANG8hnNbPs_?)E;4m?$#L zp$v6y5EYTiknR9A9?^sD01>ozweYkcW~-{N%O0m}sW#9*Msmg)3>Im3M`Fl3B?#;$ z-7c`671To%_lL%{=w+sFbB2Q_k9H7>lw{w_HxW~3`HKB^~pAzl}rksQ1Z_Ra_P?j?>x`gq)^i( ztCe@S&k`9TKfWbWs*XX)!Yx1V`7I-@GHN9!k8v(}Y%Bmw%KTk660UjO6&Cyf3{Nrd zPS^EEhc=i*<23PKqj&eM_hb}pZOt~!PmX;x7V<^CL z#Xq-1M>U=#;%Rz9J>%e+fq31~<`OmDtx%~Qu`00K$639>sn94YWn!-y&j2`5owKtd zphy$+^{g27Z93#!`KEcVmjgY-^N2v*`SQHpmirC3LyGf?MDm6uB@y)x46F&4xzi>Z z-{S=Bdd*+AuR9s7xbITAeV|&^;mG|~uIu;v0XuCW&sG@^i9*yZ0@weFmza3`oW%b( zNojfc*-Z+j&m(ii3y(v&*QeBlNz%FbEmiFB^Q6PHN93w0fOs@p(cyXbv{tsZemdLy zH_PyKfpG_Ht%Vfce^00v!tX}1oQwxzC=dVA8jCS+Bv(K|p%kkmpspK#o#Yc1+IS*d zJ~a(K(G<8Aj*d#5*q`RIPBypify$GKVRa5JYvE)H)EAmP9f-Yj+p`}3Zaky`iUJ?# zO9cG*2h#=7b!rip-i9LhU4Qy3&&QLE>)$`g_JJ<<)KM!&8TRjT{dEDy_Sddsz}_;! zn^t9;d9*PM{)MV#kfh5hMU(K%Hs~V$-n7ruitH0cC2bfIa;|~lMWocZ`nwU0nIwB| z8}q_Il7z;RH`b{_8zlx8fzV4t&qsR(R|B`R)u_E*H#VDEk_w3z<>*l!-;qm{2-omS zmQ-e|U-N;(Rx5=EOR`BwKoswitOQox3c;<%)m!+)edwtAlDIGJF)@$%dlURUQb=a- zLLKEK?+1MkyDWxXb#u4jGyUMq_77xH=5{_IzpL0F*6Ptj&&@4r3^IbB;e0O9rnAOX ztMar9S;2Yz{&dw?`u$^DgGPy#qJYt&{*DYtDhOP=3J%M`ExJ4gG|U5Xip+Q!D(?#qV*O46{^ zkoA#01XPsIla*0snShw%`+ur7*pigNJB&|4LaEJ|2`#K^1OYY2AT^D>)L49AaQ0oq zFj|qI;&anb`Ngh!QI1RsH+WWxeXl|I9ZXWF=}}kY-Mu%3$i8^Ybv`#p2zo^y5aJ) zBookuBX9!B4316ROQnPt&~db+VT`0^Djqc!Cu}xvRaf9?003t`8d<0O=;0*lH0?^rJVSA=!{8OpY3x+(_GLnmu2l3 zs=^A0`Qrnu>6B_aO(|zN%e!}%{sQkkV(qiCNDb9gJBWp1Ev+h&eu=|tM~xy5K0%j^ zswpqKIW6T~q!+Gc8vJ2uen+wwAuPD<{9V_*o?+l^ z4PFq4-fzbY@Ge%kHwWZ zr2|LUL|CSO#sJmF(T3fU*=U3_YB0Ske2?w_XDz7y@njK$7k8+Q2Av!`B&v%qU&T;t zt!zB}{WrEyhY4(DExv`Bu3l|%|GN>0bC^s5#ya}LamRhbC};qF!h~f#ae`Ot{Q2&B z0O;WfU{l;EoqFnDZY(ssf&Sm%Z!I_OBdIYMLm`iXEopqpZVEAKmnPZM`Xss7glkF_ zkbz41^|n3APxynb_gRk)x+Dqhh9qo4rqk2Ze-B|ER-XPgrHif2gi|@z;4aczxJbAqq>{f*p(etruzD9RUHr4}vIByS=bFY2 zo4=392x9(OZyT2p-iD~T1;cQAB?y-XXNF|oC`1@toP7M9URtJy6_}{00#82|peDv> zJHWwNR?|KuYu~(Cv*vOvfwyAanLx-Rshw>sHrOsgSRW3IRN9sDAU4>T zQHSD}-AcyZyM(|Iz)Cr2$o6X0`PT2G!!udpBU7a8lzTzl^I!JS(u$}bQDi3FyEwn9 ztoj{FI(EF18nX`9F_!2%;5#=o)ZcB&Yz6<%Sf1l+V^>#5Me}Z=ycMlCWKtimkq*hz z(6%;l7#TlW7c`!(w}5w!0eVn?aLjFq_^pe`o&D4nPs}p_wF#g>BQzuZldr`FqP?jF zHqg!G`S)rMR}OEobh9TU5_m;-sa2GG9h7zp>BKIO+L`Udoy69)QORAyuXg zQ5Hqr2Zn)XGcg*FwP5=E{mgJ9<+TGfl#G(T@Qsb&`5w8%GRCFN8* z4sh#JB8cX~@)S%%u|ZVYV|iDqFzE0Gm4j5SCxnBy>(jT+Wzjid+MoPJBnF=*Wg>{i zvo+!LrNxM9OLHt3qBSR{rkhF;18EQ!D-5q~>`P_r<>JXiZI#pW@+8wx=VD4)R-XA49*;SP$<+6peaW#ILE!Bd_B`;xC;j@>UIo$&V^tym?`2>H5T~c7pS2Uh zAY4f+Je7JM0Q`L|zg*)P^d6zi_`YvkvO`Of$f_OBruq{xg|bT z!Esc)b9u~FIZFE&@{!7itM)YHiWfp%l$tc z{rDxor11Us1Q)83{9EP-LDYs{c8_#3=jAr~PmKd9($z znPyqw_4)-;w%`hDzz;42AAzR~ies^0=ixE`7`>@FDEA-M(O9Sg#bi*hKem=kek|(s z0LkWdNu_sJn>-h#IDMXM1l?UJ#v}3N+gr=}-GQjgzD~cQbiNByDuc6i$BoV3U9I)- z^{5|yP-$r@ayN`r+$#He#E`MjPjKfxbHL#xzg(&w`-3&VRVm7vY<3{K5PX}B%Iw5i zqD$hkssAb9Lw(2XKYl(uj#Db@BvFe@6SR}nizln*zPO21yC@z8ULF$3?T3Wp0*&0N z)FB)8h_WbXdI-{hMJP)zO%*Xgl)En7P2ZH)m z-gfCO`LoES6g>1*+U~RxFD^T*Yx-2UJI2|W)vD_21M5&&({DEx6{Ev(#&|tpGC|U` zp|r960=XPq7M#lH{R2N|?TKEaY|RxDQTc3gQ9Vv_pFciGJ&2t8Pta2=3aPSUq;ymc zs#LV9)yy%WX)Y|OE6bGWTe;P$;UR&*t8)J6{pH%2@8u|9+<3-J_fG~t)TFWnNPHc* zkHo@o|D{+sa}dPsl}W5X`{xF2*98QpVkf_(5HUh)oGgMH*Lul<>&Wb}JqAUB%Nj24 z&{5k=Sg*vtwN7HSacP9U>WF%U9yl0a&1$t#5Y+h&LJ)@krU?-8=+NXqW*Ci9IJ3n` zyF?58W-7CB8S8GM0*{Li0KXAcwB8ZON|T1^_-$|>=5S=;6D)srt``}IoVyiib7odK zZgXIwXJ8tHz%tsLzHmr{U4 zJ636>1xG9$V`>Cc683}TW@7vz4*X|RuNk9tXB_$f&QcUgT81sF$PvrH>YQ%=x_wV+ z9+x~ytt>br8i0#P6}zQ#63K%#=xyv2D3~YpgZD&j3ssrbZ0epZR{`(bGARV{ShI?X zJ$dERTzw8y8Ht#2&13gV%&~TISJE?PR=_``j{&-yv$-NcghjC0GugWS#eLhqg^DxIPocO1*{>=#7huG?`>%`V*_h0$pAjTvXot0h-bFlYbO-8DEE6{`%sg~<` zt6c^pK;W+eog;N*O^3fW=hId!8Hx5-`c8W`by=-I*e-DH{+FAiq!KImE#|?hxrl$6 zJCwo)ntb(_Q$3BF5Dyu#D{{NUoQW=*oTiFO-fXSJ9MrsBb^NcaxE+Q~F%%-*;B478bM9XLq@w?hcKR@R@WKt~IFk1V z|7etH14X`q_URJ&Bo~$Yu=)zwIXadfCi>y^UgAU|QUT|M+Y_Gi?3f{1keX_1&ST1`gT!{iiOsyYxrwoyV8& z-9`QG8pAXGgBdzPu-maRnb4tsqC)Ho)yHZEKM;|7!Yv_Zc;%`Rb)eo4gt+}WnRK-> zBd>Q#Q^mR-ue_I$+Fy}KZj0sCN-p>n zd`BxH%$I(Hjhxg;P$X_ZQg|H2>iV+=!NSlN)66@I62XFKxw$$dHUcZLUfa5}xl=c? z3WJ?NR=1RwgRAUU0*Uc1V1J|0QlukmvRhd@tqla@jsk-KmMO13;=BSg7DBLKDi7LX zhtsFFkIZO2S%;}QtY3vIow&oP8lx^dE}^ZrQ*y6WKTq2T>KSa%C_(VD>Iw|Ge#fAm z!gM5mMYUf3$^{C&94U)+0khXY@TKcY87f{-c})?G=l3xqtlO?KbaIH#Rw&$Z`*A(|Vzc@G95VW9lqL71wbo z^_Vc9v=%AHvCuPua3W1lw z12_R7*J8a76F#YIZSB43C!5#L^RYkG#kEWBoz;8o2&?NyOkYm7sFP4NS8n>ku|UO0 zm6I+uma(DPqH2FhK`u?M5>EyIkx}d=qnT0zcTF0l+!=kfIuY`(vsNX%*mH=x-GnR{ zp^g3hd&b(@xYhL@0oor|Wr|4iW)-Ip$3DGJO$0a0Y-1{`JW(YuBEQ?AWmmp5`;1_Z z=R!%;uBLv&%XY&nb0rFKY`l97a#wL$N-}jjaQj7Z7|;F?B(X?iG?0cK;#U09ceaR) zM|d6*@jWd1rJ`0+J#p=`XnX>qFkKF3ikV0Yh=y)fPjjg7XFhi>hjG;#W#TVI zimmbdakWn78Wfa*rDyEWxeQBx{VYN^_Ij79@s?Rnzi1|gUTDuE)|?!VAWd}!NkM37 z9ofVT*$5PCwO*f{H%+^u>}~?CI?cXN`3KmI`0${j1PDr%KjXXjA|vw6ptVB*wgnzF zVmfZOdOBeW5mbawxe1v`gH1pSBK|jf+PSx9y|vs>)Gic~(14f~4s|W6?eWvVx)uNC zXRinHBz%0T#R6s8BO=BcKK1D?Xk;x!;#~88ct}pne*LBqh9aKyS*&zT1Ya8*_7C37 z%Gf>AKr)jj7{~*ZU+>Kf-4%Q4NWoG8Q4T?~I!^FiSR^R&9tR9&YCH_u9{5zpUO68Xo1NTyvJ(~_da!8r*$;V&>E7kJw$gNP7jm3I*+3L zPh4N~VQY_oc1X8XwZF7K;7{72E6wQV%IK+I!GWXP6BI||zYOe0wrN5u_b}V&x_5y? zqS|vvZ#f?hoLCyU3AXyv#&7bOd8Fb47h=pzdGfwY>r%-Fx^RmqwIRJTuGa<}9V}BV z8Im;O;Thpw{lwv4h|czd(~hMd(o0IdS z{={prDyjF`7O#<1SUZHfK9s>>Yqjg+Ymg8dfC#^3A=I^@Cp;gm7^%a4o`{wsD#IV7 zX@x+fj=_*5tsakOigwU7T7E$V#~Gl#QrTLd_f9%TMk(qCw-K2=a#7$cS}oDiwU=Q^ z6eU5Nh%Yy1#jhC>lOOpyYE@NJG+2bnXgtXhd_O?9HIAc1W4N+N&P)tg6}u1kNc4crSGNae{u;#grRi3e}s` z0xdXT#Yt@NcEj_V$$gT^c5sJJ$YsZ={|_UnOPV&AtL&T%x6ch3;;+b6W8UiAO4T4P`4Q3{Jx`cmRLgD!P-`&rP&7<#gQ}tO7RZ1K8j!O?8 zN>_7GZONt_qe1v0b8gN6xWZnw@C6B0r>R-O)uypOchLT-jFQ;HR(D29zw2B)#S+L7 zj6KEFO9t--x)NS^BWr=dd7)wX9!H+sLbjGG+Z|4u0O^xAqj|NlwRi2Pu0Re2Ti*K< z=nV$^KG$7oh$|Pj7J5mpI++YA(PxhAG6p4VrFc{PL5BykGHLrsU@LOuc+uZ=*uPWN zC=ytg<*q#K_^g4fHWeG(JAhVaBWS=-bSNhfAA@as7|`2Dd_Wvtsc2S#;dLCnliW+H zuLK-aHq4Rso<$6*vhljHOz)JZ5I(+Dd;cq$WU1ql!6KK0HE9_xPd~d(77rg;pm>V( z&$MuoO$bXwtgJBU{7)`L)({0&RB>kXQl~^=F{%ENiiuy#DbIzemKkzsl+2+ARf?Sqc3-`~@(X*o9;Z1n zOoS+O>Sb$AqP|us!Z_)<{swCoN>hD0Q-)%mljer+_)*p;1jYu_SWc|B0X8SaK%#Pf zHrH~`$qrB9@p1pzh-$-{?<&{3c!u7WGEM!~8qXRG@@$)thfj#G2m&Dl9JSMl*K;Uu zby7>;)pE5s{Ff+QWB5N)kx{k_+^n4I%Hh*?2QX8=$6p9#J-Ox8GQ@}edt@8s0>e=| zxIB6U@>Cqkfi7IqrwqP3cj0O6lN}VyM~|6{Eum+3K0AYPlKan1-Q3(@2+iaCfO_?{ z{USfDppDQSyyYbuvyCfGP5Hqae<1-(V-df*E7j{2Ge^g9ZZh}KvtKncAj~uZocZ-n z{`=6UMqO1qz^G!k0SI#Ezj$^dTl6anQ z&BxtM!2_QvW&mf@-xxn{4Iz#sdlRd6KRKbNfrqet5doKE^E@ZvY_q9naa*Qecjx!F zE!%ow;izu+o#%u_9=A#krEj3HnNRw9n`Xw+ z%IMOg9e&61;R=th4vMdqpT^6HTaq~PGT+<8bZ}%!&v0@+a|plgON>sIDJ0y^|40>4 zK2tOHQyp!^y&8Eav`@fT)HyY@PitGai!2zN!WqpSJ*03Yc?K30v2&jRkMPy)XU=f9_N7%iLA(!(+wg+nX>MxR%9rEL~`m1a5FB z7cDzp4yurQ4b}W;Qx1a|8!I%eaWuP(GUn~Ky!SSysd=?zs;OfrO7S!y?>9kT?MOo< zrYL*RsE7tKMjYodjr_^_n(G60Pb>{q97I%IAGveIh&fEUSN;XEp0yFRX|^vjiC9f3 z)2(fuMIO`9UwNlc%v#29SDCc+W$I;VF#D#V1egcMtfM?iR7=+AVk}B|ar!mkUpBjZcE=oM zF>6W{p!N369V7le=@y@;9Y3kef2=F!`Sw|(XdU{S%5Ix`!jFINJ{vfC0dC^Nn=%0& z?RW03RlKP`-( zY~6J;97dD%qbSGB89enN37N=xm5HZ-uOtmPQ7eUKi#S9b_=)_mcL6 zVv-2^-e8fF2bWI-FBs|G?SFDFq9*8hadgXDH3kz>&vqQiC$O-v0D=5%VI~d^#cBZA z@WJE$ZHf~NMTJ7cDalubINtev5ruvm(;l@(Tt3EO605(jfnK8Nkztmpje4sj;5mHc z{O=aoqW6Eal4fNIa8P_&@qV97AV1ZjgA3v`VwOS+Xsk<%x^ zB>U;Bukof5F0SrNK@N57OYF7#=zw}pBUX}p*ck-kPW%xf_r}QKDtcoPG`Xk%3H+D;>CXde1B$W#aMmRz!CVvbq}$PqK}icmb>ltD z$PTGN?_<8nDydhail}&{q&4@#y2{WFiPN*A25z&~X}vHNCPMXy&IV_&JBNdDrmE4< zqGXp8t^^QMU164Q;OOK}*hc3sFBNFR?xgW2IlG7I(a2QrWIWy>3+oU|?{jXt;t*oq zP#n6T?$PaYV%Roc>)(_QS@WWc^>!It{G{;=8$Ta9T{5-0=oa+;8ZnSIg6 zP%T_-UwPd&YQZXdQr$rzzeVO?_AK-l#T|zhdbVsL1`E3qV?c3rVx9~-;=J$z1gYt* zD~_ed_P4%#RbyY@=*;%&u9CtPj zL7N9_ih*Ow-sHXKd+<_b`KrtiX}z_L;eYu>YO4dZZ0(kqJo;Zz`~Rj?%MgtcOsPpv zT;=^3`*>`G>#|aI3ljNz+#F!4o`1Q!HhIAIzZcqoy^`{a+?{jZ4|*PaIo35|MhfaN z_%GM_2B4nnz%KOCULu`)C^cosP@w?`KU5LU(%%k5ANrhM$1~(PE2lHe9s=&*qwz`e z?aAn$rp*tMq%mWrL>4?R#YkHeAB@=2DPDi23%26^rv=|=WS2sXRy`gxywMz*c;H{B zhvy{HVbOo)9sl#u0(hfa2F_G5w(3WZMX=;h<5^)R>2mfEQPO5qWy}!5adc6{)vzu# zrwT)A&QN&00!Ez5Z>uA+Ftzy^2{$+FcXM@ZMxlQgIG!)QJ}Ex#gbDN3f~bSWj61gA ze0)K&wL#5##FVI2t9hd@otYz>^`nh!nI&5O@n7!0WX~qY;g_=#QIMP=R<5qNB_$2j z&EB!ruJQbwabgN8dPnSM$mFYZ&&){QaDU;^&&V?-W5zgC+*(a9bAE@kBpp`x;u1zp z+mnFSghI*#Kd<sHk~)SZ{)oQ zlz-r9t`aSg+Sej3)^#o(8SiQYu2u<>i=F!|lPYQ)Z+c`)hb6KR+0oO>Cy9m-m15OA z{9>cbI8Wm?gpylCXsGh3+0;H;rc{k5nRooVEZ}*CXV4!$NQ~r1m|=t!wSl)4BvLbC~;xmjx%yMD>9D4 zQ{oGhaB%tr1|vNKgC2bqJWI@Ytj+v}0XNmej0&sL z%xrYr*XzM%7g}Bv3r=twLtjm2j9_a^eU4!GpNIWe`+r3%BZsOs*4AN9)hVZbmn}m1 zQCj79jEzswX{zF8CCxHzukA=?P*>E*y7jmnmAS5PaHG>6+c&nqgE5YN)v|GaqPmtL z1yrK7N}fZQ)T7tal*RH@@&$FcVt7-3DgfSZ{a4h+PK;yEp8NVPn0$xIPw zW*wluZF&E`>(5tnZ{A-{Q@o2fLr%IW3cuW?5pFiO+b4qgrSU5{JBQZ9Sq$aiMfY8O z+mU!2R8dq(%1d;kjMgcshz{;rBuz-oWST9pZu zSqCTO6OE@EZ0{6ljT_laIv#$0P7uCFh}M=2Xlnbio4rQ9=|9@j3fDjx5JH}3q~NAB(vbi@!Az7n2<)@9YvstU-W zDd!X9L-=Q}->WCnslWK(_OaNSXF+B%cjkl!i9-AXW=%~4XndJ2TX^ts#VeDdPLrVa zAisgQJsyU#Sxr2@_Wn9rK0pc+EY8azL}p?1#8yw}k(a@H!7I|&j@ZB>$_r1t-6$8| z3yKY`_i!d8vS!}N?wuS}$IioBDzhOWZ$m z!%}YOa=Q?wJB z$P&Io^f88PXdXimtrC(Fhmv{J6n4%gg~l^(myw6eN~cr%*PkmO zZ27u~N^H=P8V(JNSKpHcijw-j%WEM#x70|%(&zY$YLO5gqzd7m=Qz7j<+c;a+Lpu3Vy)3cuwqD8af%4Yt)OTi8(1`5AYB47C^2+{bT>#zr-CmjW9F=++TkG`{n(t=f#-dnwd48>zwQC{n;gZv}|}KsjIfgq6&7czb)N9Izy!+=p zd|yYub#z%dzH@ozbVYb0baqAqh&lT-u&p^}jqeX5Ibr8dLZD{;?f*riuvb|A0Xj&) zDDNE(+)G~m^o{lW0zgJ!_X>8rRSwsWH?T~I#o8la?%B_G>TR3g0Y+`0bsU9{iTX~p zE(rk(>8DYexBc|ZqoEL{sjv@%5AbLL+-L{Ir){+>l#crKyy4vn-Yb42=7#6;| zBpzm`*tNzZCWI^FUO+ee3fGS@>p$+?O%`%BXw+rYEYgoG@;$pyrr-GXE<{<8-G zm?`mIi0g9U=B&%p-Re_fhdaQZy0?AtzYB*{jb4H#l7kY-vhcu2$G?vBm6pT$uf$(V ztnIuwT!3|@wfq};IM+t|fp7IHKloWchQKHE`cETTOFq47SLO^I)vx;XQN5p85%`ns zd`*L)YAsFFJF}6W?f8&8>ZKTFG<7>pxbp{ROI7F+78clu@W-eORJ>JV`_aC99|ULB zs%Z06Hi^DHYH1hC9rPy)WO#i*3;Ef}6nD>Q<44SLZ-V)qBY&gaPpH0Q>Jl5aMZqfF zpW7Tcfr_mtC&r3BEnX-aLNh>`%oLw|!bwMaR2!0dU{zjL>16-AH_utX-v6t=Qd)Im zZQsxk6PO+;iwcDnM2JNEfppjyx`wk!Ep)!N3|MkSh5%+B+sai2vfFtqAfX9xt?@m< zm^F`3Qjz~4%+u+8wFK?dGUcG#UP8POy*(XH^Ex8!BaFvJu}uD*vF062u`_T-zMaux zTBIgFqNlMHl-b@Ds;*o!Z*Iz#z=06qBQgS%m)tW+ZvcT(%*SAfIBqeN;p=Ncz>yiW zD$zZ>fAQ<1)|dCeYxU7(o=S<5a`TUcdIi{@@<;<)Y7s6uq_3u35?_b0>Oju$O6Hig4lD0M5U!gB?{fKSXr=*#9{JQul(Y^4m~%T!QPy~ zzL1w?bmFM0f(qX`q-IF@^TcZyf!d*+li^D|w=W!XL=6OlUNuXl$<}_8W&Gmz9s^12 zeOeBSG28g8--K*&?An@Ssvx~OA79nrO3{az3C0jQ&T4;&Q?}J@_hCbvz8ULm9zd0e zM_(Hk$Tvd+j?JeONI((zIz;fBRJ1drh~O{R{fA#Tup%`r4J3Q)xD+f#s_aU69^xD| z@*HrnDjHSpv+zD#?ToxihHZROU@lL62Pf!vyEwNGKTXye@-7 zvejMfpJyMD3tNhjD184UJ2)*n@Uq4Tfe^er4S##I?WfgGSb`K}F=De9U}L=PZDL@f z`vnULu#thvX5WdMVpWS5QAsMSx>YaxGX*#(K@H}%oBp-~)TbaJ0HH#qPbT$F4x0-| z(+LVfe+BruzuJyx0r5D&92hPCYCy%Fr5e25JFJV(cW+*Z2i*PiZl7-Nbco;t++vI| z1H-!vp6Hk+j#k!bTHMDg^hcw-(4PP1R!f5Y*Vj{_|6A=GPXG&+PRB7}SJh|ZfS}yS zz`!pWu-ZJ(^05OgT~9;%1uL+I@#%gseiNSgw8^mrt1qRBJsHl|^=trAKVVpn0#G4c zSL?4vp-`ZG^a}dl63AV3D7{H&{9*P0@V)bnES<>XUpB5(%mAQJ*jH`74e6U@J0O_q zY3^nXkG$reO`VIx7=B8SVqszV2OvzJw32gAc&8_9w8jlEvjd<%ilwSETghb%U+RH` z$K``3rvHDdq^SR^MPDqXl8l2?T1OZdx%+|cs7hNgnc*nuRAJCTof!9L{wvUJP5h<6 z?QN@9pWLzB$dQtQpHD1n1e9@{<6rH(kkksyX~?B8<)lM1+3v7=&4^NM^n9SQ0aa>F ztntp)G(CCF7E=jj>EbUb6I+iW5GF!#YeeQ$oT!-=GLkrfvBitx$Yp2KzAabtkwYVC z`G6qE3*P#Alk{21-F~f|?b-Ju@|S@x2hn~{!s3^M>d^7yr5FUG>ok0q1z3CjBB#IY zfV!{DdVi#&R7|0<58ys}Uz$aeMUE;h9|vY&U{-j42B-bFct)j|GHJnUZdG2>)IPM+ z$818}^Fh$Hj0Vm`?Re))Sy4ug*2hjhhwd(ydIP*c*Mz?NacgC>pO;UgdzpEr*_Ra?t zVIvu+L1$}pET|kGYQ5Q}7ki&MrB|@x`+Z`XaCUFir9}$NqC$It4*>r9^d51>yl%#B z_8$>~zCVhvDYF53c^7P+YYF-3bs$8A?WHeL zv87~K1)Nh=RQ~PTMCFA)KEI%)RbgOEqRDcI>Y4r9ZnbRqrktJ1vHo*i)`Yxyood9O z829*m7tuxa`fO-m(_Zo-z2Rjv-RG}7u?}lx_5Q8NmN`q$1*A|UQW7n=3p90F5@u!s zxKmVV@O%dqql0I{>K_v7>j|J{f>ooaPlg$r@RjZ7^? zjX>!+G{l+|9yi{9IA52&0UxlX71rhB(ANr}9`R8E3n3<1RRD75mxWu-#p#ivbqN`; zZg_}q^HfqTvod^_E6}0QKo}iI%QV3NtbFPwQB*XbKWvcI2}-`8%;8u5RFbKx{JL4k z#g)eA{9_7sRoSddE%!3_8LbX@@zn%+d3#f|WmdS|R7M8GQUrQFBQ;x|7x$IvIdZ;2 zD?AEK)c1ridWjS27y4npnN+ttHZ3$ zytHIw9ycpHo9mxYsKjJ}g*=daJVfPWhCaRG`mL+(r4ehov^(*NpkfwPX1bz7^;FP9wZrS7+)ty7eTg~#o8$LW|^$}KlOhg;!p0YFBd(O8Bs zAe-a?h|EVX#(9a6Qo0=g;eJ7;&q8;muGW8&_KPxGK&u&eTu&f+1)LEjKWdiw2%7Pk z+Lu~W`-XXQfD!y(U4{4P%dw=aC$JM>-~hT_K*YHn<|HiOcvRnXH&e)WCFpYaw|qZ` zT}Hj8vLA3~u5J{vl`SJk4y0#nh)-PMw*AYq1xGt(+fOCEr%MhX{IjfH_wm->r&s=( zt6yB$oY|Ll)M0&SZ%y=7owpvli=2EYmxf$uLF$Z59-$`)s54w}%|rFn7HMwrq=$DB zfK9o~tp4A0*47mtJUu(8Uh$lOiOB6G5Pmw&?NuVhJe`csBv#-S2qCS zr~jEt;iCO!p7#(zsc%<7g~q=Bw$+0D4>H5s9hpn)NBwsGYtr$TRpHK`PV^i;)nZBT z=C`I%z1OiDV-uj1mD}28NF}l4=AGNCT_k+thZZV`$__ab z@V$UM_}Ls*Km;mmn`>*-3*gV}Tx7Z&(x8cXkHc`*SErG+k&N0mo zHT7DqCwOgHRWE58Ly#EXRnvxL-B=u|d>+R!QXYS6#C+XBAj-5xFTy~0MzMzMei(8i z_tquwh_K(bYxAEnt}ixYXm;ZZnvlHbPB{~c%MAqo0EF;~*J_GNr5^jd4UpG4Vg2vRc zVt5sDO(%M{o!aVt!z6k=$bQ5A2swdwC$Slo<;)buO1^`xxZfTItJ??HyiTK5GK?Y< zg#ydbPWp^{jA+qQ-2SNaPX{3eYPq7DssfudlycbYJnA&&j7VTcs+*Ni$7CkI#+`oGY(CaAa0XcFoEobq3s==@*{zL!Oh8wek(CVU(| z&fLZ~LiC_vbIitVM{(CU-SG!Z>qjsLZh(9)=cIC3j(Qxt*kFjWm~dtZ@|3aU3X=Fg zG4`)!m)lBYQG(;^V>D^^D#aRFx>i;s4v*6F=d`HCa02ZZ5~%Zt>QXZqIye6Hlk5db zPa2lD%hr5U&Q6uZE>+cl-CPP+}|2zVb{x>5qcl?28!wc2%ssY z(!ty^CH1mg7OLU*=``qLg8m)S zT5!VMq(L<~$zQsoucHRXiXF@1Ll@*N6Bkj((#qf@(KFYFH-YfsFqeCC&a;R1lK9rS z+vNuvcbJTq>y?(0J~ykVWm$5n<%CUxiaee}O_Ei~uMQ1CB~urhvzo=MIyUSG?^_S& zxa8`H>cjnLEPo;1rB?S}2Xhr%gcwZyH@b^kg=0(=C7^Rp*6(yS`)0E@M48BWz~_q$ z{>z(;bVPPO10m?1m@8C+-w>N=7CI#}IJOR#V6~UPl~F^mkNM zRDjsfj!$%(PuR;iS+3O(ei1z*cZ-M1GPhOX0;CsypWd&bZaBXG_xxr`^nU7YAH8Gs zcD4bNb_7v;3S8ihMd>rdXXK%IB8_)(=ms?L>rGBMsfX$7Ay?Z}Cdy@2I z6n!H6?L_jfpj2zGMKM6In%mo*Y36Ied_4!oB;gGi&s*q z0H#{+zmV?}XC9HSPYXF+xAj-Eb8|N}qIWgOpphHE-6ZhCKgg)!k(S|=cr#nC%BQNZ?eQp|} z@9u5;|DC)$Nk6mQf!wt~T24+H0GdUaZRDhf?UmI9kapk`SVs)EZ|Uy}$ZWn1_Pq|K z*!R970Zu1x_=yrl|HG9opYH=g_#nskAF{8V^5tDg&(mcO#p5Z3==h`bGath!;CYK0 zd3&c50I$UF*7H3U3h+*M1k2n=orppD%b0B|59{U5bpC2zB?hHnrxy6Gunyit#bwMK zTCXTfIx>Dq%8YI&R!19{*`8tVR#s^Hrh-o=RZ--eOPJ*PHSCnZ!X12;f!+R`;T2;j zL5ft~p~Y`0H6M9-zb0JIVUwR#e}&Sa+CUsYOtnnEZmf67k&1Oi=tqWV-@Z$nyYk~l zOzyFXk8y)=%_3>W;G{c|uUlX56bEV&s_9+5B-8WnLRhggk+iG3TX}N39H)`;M zuC=XypFSv@@{?fhUNt)jAfJ1X{oH#7!vLWL+T{jN&Q+Abb12$IT-)IY= zx5OWei(P~p8x>XyL(!xWvn-pCkDk4k6+9<S1T9H6Ww+8U$|4r~Pp8i7AbHCz*~?;h+>3}QY=$OcrsyZ1@G55PG*7VhcnlzY7= z)F+9;BEKVR(XS<#Nriz)i&s~u#APv6%sMH%(9<0CzH<_oNz z{sB2c%sbcX5TGUQbqdO(LulFA~=7s#3)@-E`Y7)nws71itVbuNCp$up&?@<#?U*d-L+Zp z9kI~uEaFSKUZnbtSNNd`!z&qK#`BQj{hyr|AQ#e%gOY^vJY;8iCw_N0!-OZ-2(DFZrx6gF0L zPZW)wBz$*wjaw2va5Q7BqcP3G3|9DQ2V z*pWwlUo``DBtR)=f)G{o5$N>hJi$q}-UnJw;VkI} zD1)OwhkUYBBOt^8%PHLizVPuj;}C+_xCP8(CxZ{7Oiy+8>G3cFxmEXiyjz8hJtD?# zcprCGmkz!!CqAVIAmF|aV>cIij~91pEGSElPt<`(Tdx3HuT7FCj^Uc%>J8@Ox+ns9 z#S1>_sZ;kGu}<+osdqD^e!W2eanl>l0?3LK%dq2bT8WeJ`z0mUUqID79)|%!_)y&&+vZ#je7q*WQMIcz$cC zcSj5m5Pa?Rgs&3rcb;4gBIAE68ynpSES+IO48BI*q)ikLN1I|di<}70p1aK_nefvi z8VFsy9(%kIg=anhAtz5Bv9|JAq^I*NbrnJPbR2m08|}93R#3n^St2q=pU$BN01g&? zHCi*z0$eb;KhX~%1tM^<#<|BnWQxZJ*8)Vi?YS;zJ$TDYOc{5{M@g%sP zHOh^jd~@lB*W{v_-}Ps9myT#h9d`r+7r^#%J<?W}H8GKd}A&tQVP5O0hN!aP; zs>y$;m@%KWBqeofJnzG^DYv%YD?1Rn*y-!I zY;9O7+A5s`-Hg^2T11^EY;+v8HACAc^e>N6y)o6rN=MT@IA11##`{ z3Kuj<0Xk~Bg?sZA2-LpT&dKd<&TFzqyGD9~QBgEc{2y^~=J7+<`q(nWWq5OQt~Y~Y zt6O!v+ZCrITL9bFq@xv8{ms*>KA}voUtJ;+ytMH<55LF#r>|}vNJ|mHc-#6Ld(h$%BxHhV? zFFT&ic&OS|QOE7-nM%ZgZc6cVFHy()+)py-3{4?UF9+&XCg=hl*jzZCrK-OiX(wY4j@)hm-;y#oS^l1PUsO@(Gr zYR}@la&b0T3}P+_O@&KG!z7J^u^7cy#f!aj7rF|k{vv3o^gi)L*1dpX%o34CNitLA z$x?OeCr?nA8cU&-I%4vlrfpTCq=ih>;g`u8)=ImmzHZYwbIM#gA4K+i@A3XSpUO*h zdIwIBeD(mPlTxlg&)G9t5LR5808q}bC$ea5CXIfxlJ_d)_Bos>(-?C7$(7yp)7SC0*mLIiMTN zWp5K?C%YaNDz48F@psv}vc0Bb)+SY{p_2o?*%TMCts7BRvRAVLf>8|}~X#M+%hatkPKcBZbT>xI2hmG_6J%d>Ls{Pq=^uZb(n5fe1Ph8nCuk3s)mLu)@_ z%#$9Y?jv+@acQ*Q6oA9w+^H1jf_@F2(9D@x!~F|x8SjsrZ6ntEoN~BQPtvl{QN{Q( z!)rUfbK1l?8+&i1NAd)1e&wmfrKmkfZ-FTDFwmngz8Xe%9(i~^3hr9@6bsYjq(n0A z8&~AI{Vh5+YU&?@ZT11|#GmM-nUM)E8t?P^^8g5TfVaNwQzM79%zHca-3t%xcCVX^ zS;~^6?{=lfib1TKMVNF#GDSdvf4LNBvU4y-hdz#P{(IaliT~{Po~O?(aY|`IRK^juzi20J6P)K>HpL76yvYeN8jys zb6@s5MMlBLOWVgw9RZ`L=5%O9HSvN$+%dT9zsS7x>^xyz()wiXf z>mHd%LUjN1O#PG`JtDXJZhcklsgA=vAy=M4cZW4shuTXezMFU7j(q(-CERVVqR9XE z_XzkH955*}YG(q6FVMo%75c^PxvfXpIxnZW4U0+w3_sxG!3k@T_djvy}55fwcP&aBlE!~ly$b` zSXqgUebHWgjrCxc=^$bI>wX|%=1bV8cn7D87%7z#i>$__X#F$+bs8l>37jpOK|>U>&wQT%r3$vOq?htHvJ%W?a{U6|(VRV^lG98VY) zny*I^&Rn-0lMg9^nPnNJ1ucm%FL#}133EMlnHDvvTh6iXY}@-*pdyRXzedaK_G`xM z_d@~0=$&;st1)gryI%XqRr?)|uIIfqW-}u?x;2nR6pnx=4{&F^25c> zqmlRT){mg(HLU%~vM?&psjTOb@Q8T4S-t5UuHcdl&QU9xGt9VKD!BFq((Edtop!xBfrOt>MJ@07W$H!Y=_*7;)(u?+gL9lC>-dzEZwUk*fQTY6G$gh*D@Jf*U z{)GVWAqLuu2a?Bw%m;w!Fq~u$eQ_Z__?oOiL%`XuDWPW9gQUnPyHOfc+{yufE%{v&Y`@Yq* zKK6J^fJl0L;B7s-HAxEqvp$Cnz|&YrfZ#RQSjJ9xrf1yhZP%30=|8#L7$;M@UG0l}tRyN}rnk%q#kAv3gP7UX>$V4S_m4EgtbP8@ zK0+L7`m7KADl~%<0)YV;d5c$P#V>#UrXEJ{9R9;$CsdU+;meIr6Pew)dR)&z^wpS! z)b`-V%}rP%gZSeAQJsyWTq4JfmYdd;@XOL_vc+^2To1J@oJhu|v4q17L$gKd7DOU% zF@Yb!1woR`;hFZY5P#{zwUTJLh%9F5M?G@$qwDwqsAS)&bb8l3KAq^#LGmu+evbSg zN3;WnwBcB(%qpls6-G^(ZVDISj`1vh!zcDvTDS$o4cB#>qMJ)gJV@`_O2v3*FkVYc zc0TWRB1t{o82M9J`{JF7Qtn3D#TAN0E0%5{H0o@=AKUSn?8jvsZ5KU*MPYv`nYhig zUQOO+cJKi$Y)RvTCFktAF;E<&@i7oJL!!;s<@zk zr3dCmzifjTxVIQjHBf??K5sY`&Yy7qT}U^xEEv@-=+!6OWI@4b@4tn`oT=ET86HZe z*h=}Z(GhARiXF2fskOhGSs_qKvbsh-x+ITZInU^udy(tNZ|6Rex3Yr7=< zJ^8eAKlEhF37gstpX8G^P#Tb>|M2EFY+-1u%zZK*}@&9*K;wdNVg^6^G z9VxwO1n-@U##tX0a#h-ePzVG2iYElmQ%>(r5P`0~kx@XBu1Dycu4g2Hp4X{_^3NNf z1&2P#ZJwebfKj6R0{dE)5oh^H*gbx5aeXu93(ZqVx(2d<`+;~jJuh=q{wE^(&7qm- zu&T1Kvoa*arvYGMdjCQC6HTowde03!oOXvH;$KfzeG&b>BS4VLh_dgkYWr2&i8sU5 z@mcsjKpfiD`K4A8Xx0(^bMYdzMOY1RV8=c^SD{A)-0OBK^Wh8BzVQ*Uj}6QwP}HlP zw?I4%jNkV`Nbhc8PNW)E>hJ;BLc8yj%NiQABFq^c+r8F<2pS!>(b(IssUV1L_M1}x z+MN{^JSO@m1bH}sK!I=!c-FGx>f0*?;TL{-QuzSu;%s|32lk+@>%J|Kp0-}1ch3|p zbT;74OZVRliPiX+Oe_3Wpk=yRn6L5qomLQn5k1shV7ckGIb~dkd0A7btpdT8)B?&# zr{q@umGw5XFX1BX-`W1I_x2ANP#K|Tdd}M+YX+pX4QfpvwQ{}V5+MEm zd^xRq^`}Q{xo4u{>)M#|Ih!G1mO*MlCNW}Ws%;Zn^>aVen^FGC*QsX7Ua}c)(Z!?; zCp1dXvc}f~C1w0sT+f?@U$?Q{dES*)Slb^h2mo&+L$t7qUtZp;S`eX@tmeuwvR53f z60SUr?wUg)i|dvK7Je{G4-X_VUN=ooZ*gm8Ze`;v!P2SOdxMWyPOv|k-SOcN)sDZ- zhM~Sa7c>36CsbLl)i!xz_Qe=|AP) zWoKxJ4#y9y9PBCsh%no@zo#p&2peM6%?^(?+2b@I6^dl8;eie+sz8WVRc%Tw!RPXy z6a_c(v)&2rV0W7;*Gzpo{~5Q05_oGkp(JW%A9AwK8Du1l{kQu#|@_6WuYAPFg6XT;g2E}i;8I}ER2r1d*47Q7&@9{ zpN~1+dd-F*PiUA|*q;K9pHk%Vu;}C9B~%@NWydpy<2uGt2hxUdE7AM`^YJLtx!TX; zZ?)*(t$_k#P&nHk(!XdJBD~8#JFsO2&?FXOW#1#Q%xYpmS0BS%IG_flw8L+j>b9VG z7W2fGJWvf9o&5{z9hl9e&2lb~*Btg**X<*k(S0BrSlav>Gj_(oE>)2CbU)KsVLUz~ z%Pea3=gA{ltCRZqrF65aKZ+H4-Gr!JSzmRDhj1*8Ps--}mG#-9$8O}FkxI%}a~JTp z74kQf^0(w_SjZV}50GI*5@eP{*HT2h=kvL~hM_ak9=U|uG$=GI(kf2HMt~Dc6xMIr z-Umt!*+XW*n`zX;KM9SbvCI0ut)Q3k0Ld1Ymmhg;SBwEEj!TPoQq=u}3GhpyvH8_i zGkg1YjI^4Gks=it(K#6gB~+&>pxdLk9KTSSNKhOWUOG8$|jEGTH{TOZaPS#H=u%W{QAMSeY-M8s%UZ{)zIh;r|a;1@LvL?k#9Q&f6?>XZ%E2oyjgPPCXSpSn1KLSL(jd| z+95zz08rdsb`*%-@5Kn=UCTD==p8!=Xg52dqIlnIO5gNe{zp_B%tQ<_tiypXWBj)Q zb#))Zgxzi`*R6K&y*mds5o>E}KlBhA1c!%*ODdW(KJY)Z8?S*UneIUE%XN5i0^bN{ zU$OuFw0=Hv5k!^lTeQOVanIyxF+?O4I`iyXyL7zSNpo_5qFY!c)-{A8_ z6NSii^4Apz;JI|$ik0bFjr!k^utKn%lv-A2oUZfz2Ggqj<^8>3khoRClpHkglTiA# z5Ucd6v{$*$$FP*N#B?*+ahy1|&qI^jo_!|Ij11gZ@<4d0(u3_&ChA)nD(>F0)hs5v z`?tl_ttp?h@j8bbiZ4p)6*Q4m1_pFrr#QK7f8{sPQP`7KD`rkMXIHmw_}*qf#o8R7 zAk(jo*164JMjn?8E?D@Su!(O|&04Hin`!=NVQ|ARpFDq3@^P7ArWx$xoGPDbpNAMR z)=O)G5DmXVA&zmlJbZzx72BR!`0G*Jb@o{?q65qN@d#8ZCfIGQk{VHCzmFSdtVGY zlhL#IPolZUeMAzw4s0@kMAu_(s9+e(1=cGL$t~y5gx)T^UVYF_vTUf=e2f$pxe5&@ z75%83Lylw7ZI?`7=u?&Cxwek7eiG@zM>F;zesZL4kvruLeo_(;3A=q;XK7VhF|kLA zdr+tX&Q}W)OX<@8q!c?|$RihrC1W4ke#b_(-7QeR=)>>@6m@5k>#ZX*So=Dy^O-m; z_d-F@7;EhGB7=9hkL@bGYJml9f>W!yXl5Xsirj`qB;a3m^~ErPI-rA4CQ&f{{9ddz z`awTD*Wx+Do;htI*!Xve=8k2`Xp~_?1<{uOlAqHHMiI9cPVOzh%})F)dQRzylTj|x zSt5)b^N%itGgl#-~1hv0)+3UGEP#NA9*hIeVct%sAEflH#)l?mVV+?U97^N_8a-~@=Lhan#LO(%;m%NcAwh+V zyLdANN+H3}nm=1Hh4m7`O`V7(=LJWS9yx*^OXd+5W0fEVs&CX2iiSGyx#H;IzJ$Gz zS0KK?S`?P11g93BOF_kYmqn|S(W$)|&OT9g3u}hV=fnSYEL5rx!q^xE$9OZ_x*h~M zTc%~N9ILi8wCvY4MtZ*uR`JV~BHZ_3hD&p971Z(Z7RtG)&M9`oM9;1_(dG&pm{J@U1d6%XoxPxP_m)$NYKown*{8a+Ict zhGo_4WGQ`GNiT~_u+{NN<)WO$T4oaRJtoPb?Z4URNBY&f%*V+4QIh0)SH7y|o~KZ> zEln0Z2|?%jvdtq^xrMDpJ!*OB)t*Fd?$Y0 zI=<7af!(u^!FbqW*qCZIT-Gk5u9S&eJ=a1nYsd`vgTjdFOV!1Q=whlSC116$`forh z_fLtXL>@Y`ny_77eVi4zCr`rP%J7plDl4v@WvF`j8oqZQe-JsDI(+{PyRxcRgvok0 zKf1uGrdYd(5PcD!U04t+ukekLU(4rLo2jj*D9x{t{dP#JICgpm@e zn{$3vsOk&~_6)(dn{;_^9s9=ZyFKsEb0wgLu-xpg5m8#^Ts;E83dA9^R4O%!NBWrt zDbxco?qNJcOrUtpWL%GA(a1cOk1GXOLCLhFAUA&L_v4`INA56*GMOl z<=ko$AQF8Y;Im3E#4MFY!%)`tOqh^E7Fok`LOy3J*sEOTjD}AoDnNwNdns;)^Gn1} zt1{f!S*fn(#>l0w1KB^~IY*lSdtCUrr{JN@YG4~HVZO1Z2)KoG@mV~}!p!Z*SC!>h zT4W6LVfO*7^kA&_L7xV)t1sJhPNS%vFWjEi=rnL|MBJnp)ELblGP$VAB`TESn7b!U zW`;YegtgDgow4+7g&S+8-&QLAch?;}>z#YPSyD!0nq-uQmAxyWNJYjqvY)Gz>@ zf(t`|6_jW^!Db8WXs;VTTkUFhd|GC@IGqXr#O#)09%WodzCKGeg zK0L!}2GtBa6bam;sp5*M49{S-O**+r#w~llI&2uti%2yd!z0xU>V)g|pd_VXxavlp zE0CBX%hQsMpQ?p!b;3G+#5?%mh@QCfHdG=loiH2W?Zr}bdsmv;=-oJk$2sxM?!i6F zZMsA%ZkvBJ2ye9aTW@H6Hu;@d#L}(8nzu)#YsYzw`ps=lX6gE|{kfZ(&gSJASA))G z+e{WqKNS+Umh%q5&aM+CN8e{!VF<)Kagi~PqYr-@BqYk1=D6h^by`Cffu#3H${e~$ zXyovbmlya%yk=$|W?^G%Uk&ip9|t>b^q`jOL87qy)k=ExIgf- z4PMP@4rQ`3EKs@qSp7QMD(IZ(SPDyskv4x#&tvTIX4*E8ABoD${HRCnzNG?q`{2D? zwW&>+|Ep{2{@0{Wm)8IN?*IK01$90oP{-#tpitO5u^u<25U{L-AcOGeGSd)6VMBS- z%(m40;bF<1TEGq$x_&`tJUQg?B(X4UDLSL8FGJzQF-dx4o+Y9sN#gWU72lDKNk}*< zF^OQ}S}FMQd0-*0RGZbQtF~}pj=HnCc@-pIp3j>)BvP`SuxFg-#`ithUProL15GHQ z7B+E01j{bf`-bZ}^N}IEve!QTUR-VA_fQp{bMd9R`h6NuH@yS*Tv(&as^u>WI23x`c`kb6Go(|BcZt1NXS<61gY4;a7_zrcrk?iQzObU3`V&4ODODU0XCWYaix z*&W+iX%c0{710f8RkFX1{lPsMJ{O58EoA}7>I6kI*elHl%nHaQKG2IfdA@8`x7~=D zK6nEZqn=Z|&l&4hfS{BCZ?m*xzFIUg+C6Pjvpz?1D8!~=c$=Bm?LLy(%El_EmJaj| z$`2lW9WCmR0fa-p4D*4=WJn~MaFZx*@a2<80gU>@H-qp~pOQ(b-=M;}$1uyBm9^`t zZ#%F>NwBZ}8wL5$SzVW?oDgu?PYa?bGOg{cK^*ONp?)ZIr3K9tC+A1T6L_N1h0f6?SjO9>gli~znF_eYW zipvG5InY=(8)8TPKGMy_(0VR!u>lT@GTg`&GqU-8-=^r|TEBbY>2r*J5cbtfmaCt` zk#G}@h_HG|Ars`WH!r}^SMduLr7LQboO}-DX591%Efd?jWq3-ljU-b`rIo#(do?)kA5q0DzZ5%y1ubg?pEr6*HXubhj_ zkg%S1_M(<#xOUqZD>h06gtHeS`_}74Es_4`*B>NM0>0jvb4nHqN33)h7Yx*V`@GV@ zcT~a5B94|DLlb)<0rm$edYJn8v@7Yn3GKmRriz$CF;JN_%a7$1nQKmtl>2(^G0})A_k#M8XI^tmQwef0+vU;%+_ed8weq{m1wPI~s4?8j`c&%dkMdkfOFm_urfoAUTtO8FbI`FJWcN+ohz|D_SF zMxU&>xfReVQjFhIiLJ_|AGg-6wAJQhP_8m(|6Q=9=;wsP9S=`N4I*gA!b7Qgsram$13zE)u&T!n{nDj2piyWt_ogdFx!)ib_Oo*QZ}5st2Tv1O?rM-jizS ztj~@fAnvb3yH_8M-tJzpie7gr17YXa3QCzGy-kj=utA*>vY8g}<9iuopuVl?zWcu| z-T%ATJr%<1WRQ=XJIC&?Bir#rG7ZS<`DeN0!#(022Cg)W>NK#}b8SUm{9w^cS7$KD7IepTNHEC- zQOHu^fwF;m2g*sL(e;Tk`eb)`s7da5sIua5@TPe=p1TZfJy^+&v;LILXJ-AVE>4!w ztkm-{J3-R2cL(}BseDP_+9-8I*K&zvUrVT*2#aNe)Oq4iCUd^KdUr1A@PsX(sC# zyITYSa?`4MmtO(*-e*EGXkxfhy0}v*9&Nhk^H-T`?t-~7TeGqGj27RknxBK9OBDM);ON<6a1{k8I;c6W&-XmGWWOpgSJhf3jz*al z6zsyVjb5q6%XP@qbzDcSg`Jhaog?!3v3}Uek}7GW{UT)EdkfCh6i=x)=O*UfZ=sPR zVC0JD_`4ypm&OptyLrM1!X73DuH^8z25xiR=Y}DD?*qj`DCKScOh0d0c;${pue5gf zqK{Uewu@7qIEtFRni+q7h=O(>HE~00dJC%;iywP_VC`ovto1y`a-b9+9QCt z?-oXrcV_65m?OBJMFZ0`*(Zj?vF?Dnlk3Spr%6 zK4d<$HZ=6tS7yHtRqmjNp+E30a54g-Cs5v8DgSW$AysDqZ$>}nc~ z)FQNd3tUKkcCYU`=Hc3t!VhaQwcf_d-z;NKJ16!L? zU8wY9|4(!v?F|R?Ry%FwlK-1D|Gzsr5x%KC)o1s*1Pu@|fHH77`<@I}ZmjxRr#<7L zh=ri|`Zw5Y_U=3ATt##y-a@mzLiVpo^Ldu>-q3-{3!a2A2A{1%YiVppMr`$-b%~3N zxMCR-yhYUfS;mG6HukGdH!`M+Wb+C`Q4p?m8PTu=?sj8#d9j9C*eu>DA_hYnuTY`0 zCFP6-_=aecL`hw8PfeI+w&T2yNgLBPxjeGyh8Y!Eio8zLt$2GOA|)6patP$&0&N=lIfloqRj&}4-c@wSDfa-d;?VJm7_p^hG}}P!a!j(N`-R=04HaI zPa053>c%fFE^Y&K9Ro+l)K2{YjlZr8coWH5?`>HAWDAH}#Lvo-QRYbxIh=*41b6Uf zLd_y5NlaPTYp*9Ogh!1Clvd}8#-RKTM(9KK z{S9wE9NcKZ~(hg|Xtshe$GXU2&8hEz~g)mx%DUtR8a1 zD`fUd;e)#MO0_veTHq?`iHCNZ(vH940;ULa&K!QM5%SGy|MNWWj`0G7RJvR zF(z-STm78T7|B4yft3kaxl90qy#OpN$s|3DLb)Yd?kbfYI1*bOJb5QOBN0LhqrsvK z@oEb%mSc=zGT6oE<6u5N!=FX19Z?CYw4TtwEsu#So8l|z)Qn%2Y_j$pRnMT8q!hOF z;^*ga8DT6pi>``I&(G{9Of?8MBY{z+{gO41%_iGPQPwFz0R+PAIBIi$ZSe+6*5sa_ zp`QJ$_^q&RBvfHrG|YHq0`687>8Ovsu8Ar}9h;^-E#&-RXPzr1SRD^uXNdg_c{WtD zB_$NgpHHrkH$_tAocuWQB>$T-yKjS8m1ie%Sv24I@v;JrS7kj`&LeYC3}~DJWB2No zaP#CHjvVB8*R$h@fnLMDoYZls)(rACj|ME01Ea9)&qSpp!~m7naT6;p95v~MPMiGz z@O&9C;J;ZM4bgH-$yLGr$==+zDC~@A@QKo7cJzW-(@Iy@3qTZRIcH*xQ*Sufnc|NmBxkVC-(h_#k=wXI?h9AEYG$(;XC z&9`yE^1O-ZmHHez1V!gwo76|xaz4=cTY~EFcJ>Cr*I5z@yE@wZ`pgS=DFm4}cY1^s z(cnP2;c{d)nL<-R9RAUg#%)obh|)lB9)@ZxUC;Shar{S#Qacyzf@(!+?1sX_hSC<+ zDEIG>8JazbuF6!F0;`5%Ns&0>g5lWz!`NGewHbBm+Et1bFU5)#cP$QuBEccJ76|ST zT#LKA7l-2RP~07YL(u?1ihKLN{nq}^_F8KnbAG$F=pw2orO|ziXd{zITKULPxj7Ba1etd8j{xY^(2#I*-mM zRjVDczcNRQp`GbWhf+3rQJ(>LD{m>yCNqDfRCP#KsKbW`S!c}W*jp-8YdoSJQe>nH z!l&bCq9L_R5H^C?BJbedM8QevH!t%*7r>s0-&Q#fT*ZnxlnOW^Jl`&HUk{00m(dWc zV^F-R1$9K#)65i-ANF?@7E+UX>~ocsm2nZhb#WC4Nx)}((ScQn;rifn$gmKKCoFv? zD&v(L#A_9~J8)0BV0X+xLX_AS5nVFPyZunIYu_vCehi?5P80*APWw(s)^^9|*8)v$ zTU&&}XMI}P9+wmfl@qfd%3yZ|{;^^Cv}epTAty1&a1t2ag5OLDsIIwPEPvj}!aWU` z0^4tASpoID{S@D`OvTZ8%X56EChVT#8wKh47qKJ70*t%;&eXf0aF*!9fa4YSvk+})))ouQIzs*GQgtG+1uZ|db8wgoiQj)C8`DXvUb=#}F-fwui?l!?}7O6H?U0;s>Qb%6Yo zeGExcpsyL79v$o)Y7!0;F!l&w5{lw3GtdEHZ)b|4MkE^OMP&;qY}E4(X=skv)e=-p z$G$O-GYaZM-bg9<4yt9DRCF)TA0VfHa| z7TIjtF~$@Ub-n8RkM4>0P9)mGA^;L*eskgs6w_~61rW&8@6=wmEdK^;CtjX=^G7bO z*wxn54Z+r5Wa%du7C9)82Y#WKrI$~M;Vt|ubniX)WD3iquZ%yp{vb`InWd7c2hP+{ z&or0S=HAF25VL($*_Gn#TpQM3zslVg-h4vGGCFL^xZOFCYp}oUeByc?uiY`whNSOAn};Zk5b90HPwUE^7rSCZTzBTRlM+ zk0+--L5v#fdw}lC3Mt^Z@0G^JcAON8Bhc3dNL8=A_4#|X7zg|&*{*+s;LpE$$wMdq zr*<9i-f#B2M-E50LC)q0bMlt3b%kiMd=b}1+^sGRTEy8WwHhrs>4`*yeAtBA(p~KD z(ai;1=|L5-liPriPt?&hasmw}ef)LCAwR5jD!yFRfrGQDcNIn{Y!5XVchw{gr|;); zw8ms?i_U!*`rWC%D2@)l+gy|{jtLL@+9{{CD!}0 z%AYM%ixb}Fvrm^d7IZ^KYE%L0lMP?Cl!8d;0>7^u)O6O2Ur#)bi=L9Ht!*Eo&34Zu zZfXSR9()1MH`cZ%=ddIz0%&+QxruAnmz%3mYui)Dt7U#gVIM!>Ec=-jqQJo;qKLEL zFcAxPk9(#$Vyu5+#f{)PEKbd_yO=sqLFXLxYI}=ZYwhO5six0JALk&4Cn5ZrALi5+ z-~D~^@lg=i&O=43K)Ma=B+s9>qzp9R9pmNYH83-KLrona8!PK zaHoXDkt&o_!r?>YK)rxCaseJ$C9-DSAx@z1#V82(PP0()OrLdQ@Mr5$zOFp+jX0sHJzR zUaPpB!}udF!1d}l6dhAG<;-BOQx=S&3ZDI>-(iyuoWe7+GUBz4h3kK-#3Y_Wv{5;{uK*W}q1`Q^ zO$je8MbSb-aS~_DKO~F@_m&S%p;xoKF~g#iICw7R5_kShaHV^Xw%Qa0a>)ChLYI9r z_}d)U;)LS#IDCHxjY-0)en%LaFK(<_<~y?{t%Df(_-&e`krL&X`0cWuGV*d=b)h@ za+rTm>z1*OqmKmCRJnE$iw>nF$dAx9^UeM>Rx=u1srh6R-@em<%Y(6&zWbkBgAP$- zKbQyA=Z9eYGd!Y$*l4(5}(S=BWFkLnuJ~`Mego_ zpSdKML=hzqd&fklNS8LHzhYk9Ia3YVL~(D$3;rQ?wtp0@L7b$v1mEvqzUI-}_>rnF zlxj)*vNGO9uTqBwO|f2wzANR*MXtWB>?Hq3YU}VWF1*q@!Lf@o`2%%H%mjDHXt`vd zLS|QOb9gol!>%HUU6K~gPd5Q8Xs|_yQ?+7t;m7V*BrHF^=t}V_^^$TRKFS9loFtss z&MxUf;P(PY>+RjuE26v+pTFm)_RqhY&};uWGs5B7_Mq#2fojzR8?GgJW2~j{Y3!LO ziIjNIJ^?ifF+u=fIk;OpgqPTWCe|D=x5v}-4%?hfw)_R+>YA4ED}+_KjrHtlv9J(- zWp0Bc{&4mm(5s0s%7%GEbkI9?o`s@$ONzkXSHm2^UsyxMwd?BJ`+$-%G|ya;*dOZ! zw@i^5GDRs5Q6?WWSc=R&a5u@M9;Ruvw7R+lXMOS~d;E3U;&kxVBa2j-M7e4qtX^15 z@q$(l*^e%8%JJ*`&hGK?2Y_=-dAuDI*&9i6-M_m3D2+!V)6WIplh%$XY4%ekD&+=C zfesLQ9Wbw+ihiO1?%+)}h`V7ltM`cosj#P!sr>2tOsMgI?67j2Qb%LMB&N?bQkT~S zzgG6coQB$m1#$LEwLyb0#(~Od6Iwh4%SXOAfQ4FR}DcP|xH(+%fvw z$TTnlz{yP5p)utxPcV5~J|7gxix15ed;|2;VMcE;>JWmn|2MEjv$`|Im958%Zjcp%qSEeiGmy| z-+25*cU-u8ce8GlK)9zu;~Zjz&Vo%HQjWw0 z7VlF|ken$Z9gpink&(%)mN>j^R-xNeno;~|5Lv(jT3pxJ85nS|wIzW7_+9c~=US4) z$ic8N-JT@@Z7wQ>BrX}riHI!|6aG`bwqI|=_;KsbSl-rfLb{oP#RM~bdf3kwX=gF} zWUnY*Im9d3afRjWv&cq-NTOmWqHmQ@NB0W$9<5A4LUApj2-HwW+=1v9N+aJWm``0DCvRU{tqAp0|b zss_xliD2A?`*IgI9_g0!AG&(3+UdfUWUjS}a1w2+2kPBUQmO(g2to=rN+Q`hF$+tjf}F7^9PDz z7;EkGQP2Rmz&Sy>t@Fg>sSMxWm{Ln?apass%blOyJu{kDfV&0HkujdtrJYTWSy4p+K&K3nSJSJg-BnnbyC`{9Fo?wW?q$WUCt)oZHhK7akA zl68bXF=xE?OVNtEwnC|o{t0}IDPb7Gh-*-1m}AZRGrir>29?Bs2(5WG=3}PBcFD1= z!5hL)YqSXG*ivd~=pJRPbI>brp*hvas9t3(q<{+SuZcEfxJdSU0Fkn^w7v}-Be#9C zF&Dr4>&z<-t{w;=*u3Y0#S6z2LBGzVCUNQeS--jmbo^t2AP~;{S%ky(9)DHW*{2-0 zdz5UiElOI&%6KYvFu(Ol_$*cRfMNMhZIMcl7WTP}v>ZtMZleu^15mb}eth8;fbvr@ zMPW7(n5HJ10TPIaoX1NEeeYehTCUx3p7pIPtA)Fn6rU_9vX0OhqQF%1ztdWb1$}~( zgcu=1VnALZluI5M!qcR%baY=K77kq6f-ZHjRqStDKJ?LH-$F*3rScJsoEw=HZHqT5sFItxYK!5zi56i~=ZKVd_^+ z^6-!N?^E_~+PH`+l)Ksx4=nM72~>Fp-xhN%QFAFXvHHbvI145)33*zZK$c-!wC4JH*Dd z)I_T}IsxVMO|^lhciO}W`2D1GFrXIO6IV`=e4;sSv8q383eEQ};NYcY8=y6m#l}7$ zimQwh|I=I$E797R<%S{2o5&Pjmi1jEbcxTruA}MaNw*+SOEO*H8iNcYAeWPr^NyKW z@x(FBULzO!rV~15HNcE$Y#!2G!`FXjGMx6h+fhX+<`tMjd$P4{)>lNg$VyX~cKpYk zlmN_&#Gdz6{Pik{EI_AudV6#TA7Uh&n1xlT6(pKhMAVxFz#tZ8lr)h)HlO8NZd6cL%U=QZ~9%mDfv^)O?tPo39{J&w*x^>c#Ea*B3e zIhX}CM$57n&iT$u9LpQ%3f=b7iUAjU%EoTP0?Qe|-qv(>cK?0M+1LY;4|{W^3BZc9}kDVH%Y&o=8HYfyn3E-s5 zwk#;L5StJtqtvX(Q%fj@!qnni?iY)q?X)Md>7X82hu1ESTHNPYG_E2HJ~G0jg(Raj zS8AvWVrr;#CO1F0gqX23VjzcZW%Si`J6mxTn%J3Zb9Y&>^>zybu06;1Ys&m%Im^9D{bX=NA?F5uAv{t5qA29r{7=Rj((cts81Apn#Omr@w`X|4bO_8B0C$$?v`;hV!aZNzCY{eWEMS7zOgIEiO;|0&Ndm z2yPdbo4d^zAOGImhONyA%>0j0!)Bw|l~_t|EDnp2*^m$<4{zSIU%^a#?nX6$7%bA0 zkDV!MF&Q&zn)2A;mnJFcFfeBFZF!<`EXLB}GQ`f6EW-@B5UVz`FPB6L@B5<{(kIq| zK}&(C$xeIQ#8=194Bv_&C(hAPSJMopMvhn5d}_LOLu7ILxE~N<<_L9vge%(r$%SO{ zwTvA)X0%9()v-8CaE!gxGG&9^U;g^>%y*7?i{)45$4QQIcDjnvg+nS@OY}Ml*zK!d z9NibTMv)RA2~Qw7y@{p0r8+Q04=!U zh2T?LdPVk*0XgBG(PgTt>R7~7;kNF&c3BL$7)fUtZwNE&hNH5d(q18Mms^guN6?^hTSHxVFE9Tt4d+OAL4~XOojBxa5qYuqL?Y7!H=|kb{NYzIpQoNV1UwyG)=vXQWB{f!@Hd0dlP*@Y+bpHxjG+)F~w5aFBL%1zBvzc^Q`+|Y$b z@oa43P6|%c<+w!xNZPb@Jr-D$p0riw>klOwN;CHfApEjW_0`P6BF)1a{LP0a)0xv4 zdrJFaA2+71wzg-!X=ATq;VXPmEF!7!b+WoT8ojG7rr9y*s2avWf>G07yQ&F{(B0t6 z&Xv0E8EaV|B8R`352T6o#0s208vs0~-{%6Tg*|hX3@UcN3w+Nh;x&si$B_3YuTM#n z>ZxOjuZ-AsM%qNRsPg!mOVasxMQcw-uf)+~5!Dba+BK<@Qsr>9%Yp}hftprt^ei1CM%EEpK6rwzo^-DSI39**3|yo z@(Nhk(1#D)3wQV&CS=Rgb3KS7rk#j-T$#<7^O}Qc;_27c1apMv2|r6q;b^Hh6sBqJ zSqeY7w-p~?j$XA#RuoLyG9y%&?YvyiP`Y9AGAl6ky*J$y5*Wyt3Fsm6wdDikHej+( z7F;eR{Xh~nb_W0EL(VgyIDUrLHFBY*Z7ryqCkReTdEprsgE^LRvy`5RXrc{ALH8q4U>HS#aCTn<0Dqr-dM zZ5x;rxt6|5SONip#84{ff8%sZ=R7(wD`#L|HRcOo?6fx!MNC3M!p6hn3c?d+=(wXR zb*D7+38wRI+u7Cni^NuTbgmEd`RRc*92ZPF57Y{)DP)Hpw935kF>JkfThCbBX3@R%W_KaJM$0E)QPfW zTF3&mHYT+K1_UFffcK6z^h7a-8cbzA`kh6ZW;^rhJBAD^DG#I_l|%da9A(Frd2fWQF&ssEv<4KAW@)_Z-x;2E(sgG`#Z48rSMg^TwH(KfTMy=jIL{$7| z%e{NCx2{xY?!@nAw|@LgO|)M8IEmyFrj&oQ9cB!rw^v4w!smj=HPFad6)<_k;O+Ak z)i_2bNo5=CZb7J^7ogIEV^^&<$Dree4D>hNul$1w-PJWxN_i}eTMXGi!@*xXI-YCU z#){!d#{~m5I?%sAAIit9Yi*UQTlxi{8<~PNsA9Dp9bbx9E(RgY2W${C*Bk^7tl7aT zARqukOy;nGU;9Ap$9GWMCr%$?;7OIj`gJo&Lx20%_b(Od#rjnr==*e8*D%z+I{Yjsyf{R7#D!8 zw+O@XFIHjd;ea?U9z-2y#F5p+PvRR|!F%||RmNSh2^Mcg=$@^})V1bfTj0cB&(=5c zy^tOlRHJ7QMbp|U%ISpwWcMpsj1=~>&Sh9)+g0h1jEwepAVk(iJ!KDMGz^q3v3uxH zUy@_fG0vEB$3Jr~e(ZngWh!1FW&^22I)m6Dlzsi51_DkZr zX}fcmQ}l(Ih@%xXCTtnat~ZR>l(b+9=% zCw}84tltT9pu1mk*N%AFR`oC-x*1bhlmBYR!ms5~kEM#nTBC`NVZN=k>I=$cyXOXQ zW&*dm>bADt-BFFjr6m(PJ2Hw_@QJM5+cU@<;!0U0)tTv(hVRYYe}n^MG!$q5WghtN zSv1mT(f4`KsuhYg9W%Z|Q8Uc@mjhr2HMusut&7C(BXj&dAxV=AJ*BDl;I35dPaXOe zno%%D%-E4n`T2saq8Q%8%O5I7%P%(%&2TwV%O0~Ar|0+^YZWLSLn32_m(GH5=gW&671g|rXeGQeO{i=?5=L}1&iKr4yCiw1&@awkg`dB2 zb8adrOP!xk8B`aFRVJ3JQqkkmgDJx=rh+*~vROh(Z|aiAi{w<3O*r(O^>}=a?F~MN z`OBwQCz2DPRlA{fbbUECz~tLk!6jNQ>l-Aa)Yp8GTcb|yC(f6?JgtIH{hC$--|cpI z5Om1B-s%+vs0AXfqL^z958tU4Ru8a~U|@r1(yj8H#gk3ws5P~%XUl1i$Vh(Q7dB~b zEDoTAh(jt&VlfV`nifwln;oE8Wec#|Eojc>nG~6!ZDm+|ZLIOQcm`b|mYAdc)TWt; zXIRsEYYz|({*RIYaBFr0y9Fke>dXM%Q>xMGoeq^=Hov>S6q$D*G0#GaGc;;#oEsmJ zzAM?+J%Pki)iM6%J@CnfaZg#yw)$<7c5DVd4r$wiF$r_=VH{EHAo^iI!}8DVXeD2W zITouV<8UIhMY1!%F;@FP-f*FCwq5TGhzT7L_F|iXrgOr(Px<^V?{K&#@D>8-c68Mu z(_Rd9?}j>z*_apwfsdfb&t?4ZaTq%^(=j<4->g{MoSt7d0MW0q{>iUxBL}#$nYwxg z&ZXmF^AZZAUw0PI1&`IUJaFDqw<^!LW>^TGk!5J<2rE6bjiWj=VzIfJh^})u6Q#-<%Di z6rG|AAf0<9QiK(K)!2-m_`?HKQ6*LEv7_%7dCofi`AlxUW7;?O#UejU| zJxAR51yw;W~xV>lo?0(+DwB*e>cjr3@ho$I5OYOzkL-xYSbS;iIZH$^+8 zR<~`Kdiwfhb-d>9p!tHk`OqjT6X2_WJw`NRR+yol(ERlHdK)f)_)S@N)nM7@50URq zli=+j*RqjLcVO?)`U{<#9d5>uAj2}E(nJLkdLuB{Cm!4NtzX|qYapWlQC^VvQTi+& z?lP%GW)|%c-Eo|N@a);Fngx_@!u0I&*|T^k~>i8?H%Ua`I6Qx$Kzi z{tH1VWD!$dudkdfgz{Tk2&yMu>Up}WrN4c1mimo)`ue6oz91;beT%mfZP5NSH6sI% z{Qx;=EnQu4dWmF{p9O&5_0p>tUd13Xy0zqyEG~q2Ql@myhMQMt>FaOZ!MpkxtJ(*y z5In&C8NiDYCi3^9>leUq)UO<;fv+zAMDs_-{`pUw3NwLR_}|^{e?W16z69d-TcTYt ze2WHrgg*!5s~bUwLX;LQaz>S?RVF>PWmIG_;FQrx1Exv!nPkkc@yO8<)Rx&=1cj)U zT+qIP4bPc`C11n(<)>K$KKL9o=DZ!a@#U=!J=C~Mt)dA`l#L(w+TgPsTEB|R1;~hH z<@tb&oV%`CSo(-dEmrKB8?mwlX7`R+2vw9L*=%0*k0XdSQWYt5ng%ym%H4+{Y$WA}dbBQtd+DzSt&nQ&qo_dKC8ZKRa8rQ7NwYFU)2Qrd`}Ms^rC!(Y z#EI~eQ)q)y**u^3Js$8#$z8I>KS%xceu(Ib$l*xkDAJjZx|FA5A4Z0LfQi)Zy!+uk zAaQqBBD$i|sG@KMk=szDUxqEgK9Cxx${Yy%CK?+XZ+BV0?iQx_bOUOco{urQJDfRp zc)*sFI>A4;z*W&b9`NiJ2(C)Y+|A*8`M2mr z8>!!f_W2Wu{auvoh16|;%+Fh$|sk1>Khx}LUTc+R|$+yqwPee?N!QAS7h zCmj)!{&9gbWX76Sn6&fjgOSFcd4vX3?_2SH&Cy9adI&ml{9tNq29vwaJA8>A6i_~y zrOZqr3yP&Xc&(aw$x+IAbbMUX=K3=ur7FHJ-KDI17^wV3p#Gf2|0uf`vd8!r(6{MP zfga2>(OeEB$}R%%b}lRCd^|aw@F|JqBh_a|x5DiG9WK7s6bWeL@krE6+60tVg!~ew zRB2`Fn5HT;(k}@h;jb)ZRfp2Z0-2h22FMEyOmZd4iZm@3G}Gxmf;%#PH z6YJ~Gt*mtBtl01DLB*VkW%GIKB`UI9+S05KRx8pMhGK%Z_HN1dW zI(edK!KmbQxQKzV9ngadj~}32YpZ#7Ji@4eO!+P!H%gs7?l8erh+~$viEo0iU??j!odJ#~VfgJSK544#I~~opL;C`*pYZEB2fZ8{Q@oeZwUv zt5NV%5@)#XfzFZ7V9V@p)j+{V?|SY6;YPZk5cUqAtm7vM;DWb^n20OFp6~-B(GO07 zZ9QJ#5MV0K#*pB`JC6SP=vGfej6G`rRV%&Ev_zX=K~@;=)!kCGw|&~aEw@lFqBjvM zeB>@n?Aaj%;GJ?iYx}D##A<5;k{Aci!Yi+@rKBpqwu98e@9y~U-@^eC5-H$PY2+pg ztgxrNo~Qwe-7-SrEWR&!Wy1BRB7eY0bq$uQL8tF%! zxA`MDaFyRVr-JQFWk%G_7NQm$W}WE~kb#^e?-8|yD$*)m-2rpg`BpWmtLfU2Yh0Ws zCJGN&=Z~l2`@2i_u#Y~9DZX3^lF0Ag&*)hQhUEng|3P5gh9NyeJotxFP7!TVz1UT8 zMx$57J8wyxn(}uk-yc9Vq-y28Y4C`Mh?(H*Y90%Uz`R&4tGXD_(P(aI3}R3Irv?LD zTInII+4=faaq!AmHIf1CbHPCZ9ps#7URWldcYM$t-={_wAiof0$ls?Wm2L`<&@L|5 zfbJs$&?>uMZoS@HVFsS0*y1udiwr;yR<^2XQpZkjZ*N=&SUZGx9N+ND`Io7h^tt8) z99ol(C#zZjg-sb%E$A#kjPEanH>m4&E7*0H(KR9K%evU?7)C761pEMrE=K3lp}b}- z9(Nofe>P)o&nESapq?oiR>b<%x>~tHA_|ST{e2kj*$KOMHi0qcy3v1eIUi=ks|YX& zcX0;kOm4fea-InC0GSA-QvsI5VU?mS%P9&DEgo`&J;tVES7~hj{jWdxYPyEqq^BYvw5l~2ZB&g5nC>Sg*92u_kU6^>rhG3nuM_ z>+zgFjxUa@Pk{pqn`2=|L0(}*sj&UJykEPQ4G4$adxNIcDxot%>(5(6wDr<-Rsb&EmHlMtf{U4xs+g|P|jsBXC%(< z=y-|UxpBC~w4*>Cl&=P*Ql9YAWBZz7O=5nnBAqU3I>KH)B$VQ4bdT+KN;{xkkl@pf z7m1VS(67q4yZzn=9S2N~t;tAO;2i8Omku-oum5>S7WhHjOxiz3hn2z|$%916L3ybl zE6S``Q+7b_#~F}!N?bq?KWCeFB@F%$TSz-xCy{A^FUI|g1Sh0#v>9wVx8D-`EY2Xxs3wTxj@|CTjj|aF7;d|XB#Jlwwfe@7sa-Jl z2#rxDqyy{e3~5!yKIDfnj362kf6`X2X+1UbzL`Lgb6_!gUGYj`Ty}y5RiPj$!4D?8e$zCzzo4ofR1LUN)bpRAME&-4RM%$#^j>d zQsMJQc!a-wyse>U+PNgOqF-~`oJ&(%+t|((pr2eJ;hOQcYVRS9VgF!_7HblNsGP#g1>#N1C!>vorQ`2(L3R1 zO$Z|pR{uP)b!L-}hjaAtRD5~NCwBF)OkM~7GL0%nd;8CAYXbx?`LtW!g*8_s`5w<~*#L-BxVe11zfTznP#_mEIQX-}5%W9oOr1U6&(D z&n@$iO>NtiaqD$6GyEUJ6T$yJnr)6ryD~Wdh=mh7jzM8`7lT+1dVYzo>0DCZgCJQb1up59S$ly;BF}|NA%rgQ-9*zzG@2_uA5k@H}+Od zYhrY@WCiEv6Dh0qrjri5yAO}mXJZ*T7uHC-jZERM=#8pmuZq>uk(u0X1#Oy(v*jSS zhKcR?i}MFI@Y05E1{1*uch&)hJZy=?@Q zzZA7aCwJe!looy}H6uXh08B_$kG}_5Dd`ea4b5C*a@ub>Xn0ne8~$U%-Z_Kx#YxHp zz`{K`6x!B`rp>30Nn_!JZ{83BI1pmN&i-JAxczoZR`6~ISMX{M7kEwh_nIl}JreOg zA5Zlw1i=L4l7V%lnJ)5Bi_-CVo(=J9kIc^B$b)43qszNQ4||H{l**!B!$_0VU9uq*mX$vN;o`WiZ#F}N=zZYm6{hAL z&uk)Uz@ZKk73{%+vct7wvi2}Mi#k`uRwpI7PJ=1Hb~HGL8s0|0ndDmbVW!U<%SSI%+(o2kM%zYP6 zfME4y`@V6yUEUjZ=q*EDNTyNFO#s=yd25kg+vWT7tn;Q<&cCIbegn<<7HOzHUQ z`uaXVNPXqek-9p__H=Wve14uq#gI}}EU|ycU-xRCLCw8IFvn~n^Pk?@M&IWy1?W`H zp5EM^xONTF3ZHQS?oVVu^TdhUQs#K=ir_E%;f}O8TZLb=G@P zGl1#JKP^^b>!SYFYwqEOQaHs?N5U~X*3j&UP2YD*NChFsDeD4w;}EjzQlOfUaQh~A zp+frb@Xvx2!|KsBl<+9TC+2YlGOMeGtYupdvUj2jEv37=R=r1;rU;W>n;VHgf2L{| z5{v^UffWHUjmy#S0O-GRc z;|uLPDqbr9D^7q@N%xh`7HbN*M2e1T5(-<9Wnf=6;YS$Q8Cd{ zXQrvE`D-U^n~TU9$jbqp3^4=&gvs;a4N&PCbj=#Nx~9N|A1o$>o#N?c`W$+IMhm8p zQ#dgG1-=dd{_X!`$VOoQbpICkYqzWUh1<5Shv&eeW!{{pqhfb}J#xRCju5<@vZ#ld zR}7z{L(pYVw8v>MMXp~#FVHdtUE{h9K6_fAY`+qD+*G_hi18g;zNe9qnb6~%G!wbu z6FwVOtgdc$doZ4K<(mY{>^vNO1fx<`D9H#b^@J*hzUy-k$LkN}h$2aC4$h*-SEDig zdEcqK{)OZl1IvXYiSqVCAaV5Dt=h+Fp%JwKB1sPJL`p{re3T%Z8QB1do(d?w702~W z@X_(fW~KzPUo`Gv6b+y&z=x;Tr?X#FAjzU#Ffli#S;0ycxPU!275E8#>6%vYnkurC zK-|{K4+}>PBsrnw>n-5cwLKi-_ufJjIRqcSf=OGO(rQ>&TryB%3rllb-4OXWHi^B> z(>Da-10Wq3oW-pAeS7yVnw1-5#Cp9uplEO3^n0dJBN0QQAx7GsOy9;9k6NIPYLBub1NUk%oC z?JLbt2IkB`VrhX4M>3dTUQ zH&F!1hxQ)NC&qfu_(VURXYAt3?VcUGXh^mwR&YD)RHb455OxeWdX!`q0Dg0Qh%Nf!a)Y_Jyqvp%FpQ*cO!-LY z>J&5Ylg1tlD0~d2iWd;T6%~+Vya0k0AbLAb-J6d9%K|X=YilMJ7AO?2+_Sq7F7pm% z_rmwKlow8fu9RdN3-YsxlKI_rlRojRK4%y0_lF38P7P>U_lN>2t<&>6xbR?I&f_Z> z9|ne3YZEvg2*lP+lZRiHj0s5l{#W?R_J6mRG-IayIfvf$@3yze&K~z(za6ThRI9^F zdY9mTk$B4_>wbqE)^hJtFSHH#qjJnTZjvgQijDHr3pWW*ixZM)jV<^Z^rf#NE~1BD z?XL{i=ABy^Ps)dT;$ATY?OR$S%3EJlribhCAw3Vt|2)dUKzR>h$gHd?XDSPw2-!jW z2>U4G2{;-Yy*1=}@gfs)Mz<@#$y!NOWr(nFdm%m3CHy2v2FSOP6n-qeliz9ll_pO9 ztB9?Pe>RI}a=8!f`EzuRQ3W1|H!B3f+G`q~FdmvzL=|q7Sefhewv>yPjfQqqg#g3P zIiei{*-!w&x&$}Ad+|Xc(`_G}mW)53E;)n8?!3wO;ib=IFBudu@HP)r z*IL_Cm-ECeeACbRd+JMZ*cJiwjpGQD47)k_+&+Od=2C=2=H1o>9m-wz_x9v^7?ndL zm?)&5GQ2?D_d^VqcP?t_3`KaKcHs3*v6264DTj@7&u6a`7^<;fsI-WWB9xWG?*T(G z`5+(<3bE%)pcVDEN8ZE|@5D2!nec1iHFLO&o$$+@J*Z_trBJhQ()}2Q8&8#oZ$)&K zDw!|Yfv=)yq&?1$c1#19Sq>XQL9+E&=DVk}&s!afp6xy+32V4OQq%kxroU`&( z1D`bn^CFZf$85?D;Cxc8PG-zWu%t>;0s)<4U0D(3Lk;tc#7K(_s4lR(8c3V9W7#o_ z74@BQQ+5n0k$uxG7-x}R|E}I9iFegrV2d|k`3fkRjn>G+h^Sg)nH_kKC^ofzVa@!V zh}!$UVr2CnGUmF?CfMB{ib2opNF~^|Me1_9wB%ZP_N0n8X)r52<&*X^UCemkE7|do zgQdQ`Ew(~KZqlu%@yW>CslX|%OsTR*opbcln%;WLJw{b0a0IrW(PR)41KNG6^qUE*@YY)PjIiL+0wF`6N0+x$4yh=dw~Xs! z2S`1qR(+qw#zeyp1Gj%zM$BG)rvHoH)X)IzYav7jXS;2o0L@2l^4%ri<_T2VPd+)4 z`RpD=uW5WHaoS>J?6OHuH<%CZ*1!d2h@=FJigKol$BV9RK<$qDcjd235))_pE*@K_UkN?YUAP4BrgR^Gq5*}mz2zZt-A${-c7=zMNt0{09d=`93?GLz)YsqppMd08W?hSuqX1n{?P zt9=WUwn*9!A97Mi9`nMLgyQshsYTn)eP_^ub{~{m86u6lenDjd|5ur^8@76tX__*uIHytaR zsGqGE57cyu>>NstL}ewe=_~Lw^ns16#YlA2iBTE~f}#(DZjXj4MBhFjPeW4qGe^jT zH)U;#Bt;PS0Ih?h@>prYARz1=*a_C}`bvf7;Ww~@1UU*sUpv0i+F0zy9wyFxnyN!3Dq(YQ=bzhK<%psySqsjsy2QBHO%9_l*p#554`Cvr0Ys>FuEcy zkzohNmo%SO61Dq!kEZHz%f#nT0+~Z~haXPz?)>)SGO$83h}FjrXwTqC5JARr@!jm? zift!wl7Dm!o&vOeVY~6=vs-eLcNJG#R=}RYc}uK8>;S`X3CaQ_Gc-T_dtvtY?}l8; zftj7_Kc3d22MGxQ%^MDsy{LIa93@ZJ*T^W?h>5=cFt4s{E1No;%94Ug!Lk5~Y~nES ztpWKBj4+aYN5O!-`OnmXyoZGxJu5dOy|Om;K`31*_8@m2C^zuSA_+Z{_MqU^Ennv& zagK54{Ab26=ira+1)fsw<7n0HGO&YMcSzJ#g}3oc5iFBd;ZWScKN0nZ*!^08TugnX zlB=IskMn2WSO;iX@1(xYwiN5e*37((%eBfm<#Vl=KA~zDZQ_ZuioOptq2ZXNN`GmK zQ;`&|67ymEvZh5W{yGH8vkTu+s>Z;2LH{A@m4aojuC-P8&SSCf7Ud|Pk2 zi2$RvHj@0(Jrks(qbYm_At{O_OG@ldJw2^);hO$N&VvKNLHZ&uV#CJ6Yf#xB)#38F zKqa$U6ET|;vHT#+K=MMm9FjOE&QCE8n4w~r$bKDR1&aP)kv{$q*y$JAI6OupcIhx7 zJINark5W^KE%nwY>wBUY{zzRfJF^hu!E8p<^buzI5h+u3OI@G7S+tLKFHuk|2L0O? z0;I}CIn-uZ>V7GG5ufl?X=>;uG93O4OGwKy`X>eN?qqJy2{Q3eUx9v!$DLc79c_Q2 zD-7L6rl4B0O>tds89-Eh-KuC}sq1W1C|;qMbPSBAD@3zUl|x=8Q?$L2b7ehLI@dpR zoP+*6;$H9AaU{YPCFy5k8iN84cQyi}I`l#siRE+ta7y`n<$SFwpl1Hz77d*c;k2mu zk32ROY*bD`yoZhiU{i(>?DMh{><%Oj1ir zq*}?MyUVq($iw~FLht*F^>xw|)-H*=^48Y-U&E8yRtlNC-GDnvNk+MB9+)&AbZr*n zrfhCk=tcGoN;f^_&%Y1KvtlG~f>HnjZ|Ce0*zn9d%o*OJ8 zVi#ZXYKEM2Dth3*JF`mO7ktV-M1TcZ{y)aPGOEeHkNYncVc?QQo2Er+z1)n zNZ062B^0E`*yt%K-7!*HT0j_$v~+hp*Z;m>KhOQV*g5Q+z2aQg@Av(F(g=uj478w`BWnZibg|tQJ00g_IAo zQuz1-0^>l1?qrk7Eeq1pYf^ax#7w{CsF%iLAO=1C4p_s`-Rq;Jmtk((zNF{8+xmtN26amhx<>wAF zbsd0Y&_+~-Pym|UTYd`=SI>7wrMT4@x@Dc-zxe~mAGIMD$taK&=LQYUza>yhjyEfg zwLV8|%!4{chRgBTbN=RQ8sml5w$*S3Lqcyl4ZnaJua8<$pA+tf@PbQ?O3G?0+o%0f z-0Vme4dNadC2gAW<7Bd%yVsBOaW_9$i~RiZ*luu~o%py!;dDY1gXm=13b}ZNrKi2d ze`1ZAo@Cn_vRN$L49;S2ph#M;fInS*mthwjVh7)QM0E-oCS=PyR)$7~`>ZP;a+Z!8 za#rGvaUV)!D@pHg%26vhstQf;^%p3PC3vp%dUibzNSwGfS$XkdZ^6PMPE%H^%0H~@ zq*c|`)s=5h+HaGjC!>^gZcz7Isiv|018P6sN@niq069V|O00!&ZG?x!KIw_<6ER=; z29KSGP{wN}u%}vfJq)kSQ2}0EWfAv2{qXtm3l|Y;va0jV%@;i4eo^z+2)(UGa1pi1 zhx*tPSzHs87$Zt83o_&ATIEw2u)18xBnvp3hr}2wC33@1A^aX&Kl$6)F*+RYv|=%z za+ySl{?FaVqW2xx&795uz{9H9W_dTw-@b<`;q4pe3EOH)UpInd9GnQsrs*BdB%CIJ-&*|0Dz zC}W66O!LfaOp_8vVU10L)iaW8kLHBRLo%vrjvBE~Q@4BHs}3vQh1Q|0ij23SvTWm3 z1f$-unc@tb{QO#3uzs!gn=d}d$6%e@1zN3x;Yi_VQs}qeSO5t)m6-6u-dsfwZ5)kH zJ?Ev4g7lgq5CS}k$;@u`Mm{R0qy;Bw{O&szP}wE#mW~cEB_$>`4eRV2&~46ZQDy)~ z--Qz*`Lpp`<$Tlc0|T+aZT`m|o@r{y-1i9Xs4PmYNq0jd>F3tz8F*N7)_HPs7gi@O#W^H9dny$ACNC_l zWrKrEIE8C`l;pf|U`N;PzofU||Bg`(WUvn?!<{p@CUGkC@EAx2cesH+3&>iOoiR!- zJ96hba1~e>lwISl``x>}r&dR03vc}6PaDvi(+17-(WC$VALHGhTBtvm^(f3C^$UJr zNaB~UWG~p$T{rTMs7qt5OF91`JFW&oDb!o4R*VRmNY>FoXcPHYMa{&fBi>fe3Tta> z4Jny8K*wp0;0aaq+{IWtKUx!y^B$y0(5kwxPZS^Jq2=gWL;#GJQ;LO#8@bLSf(g6! zNt^;Ki>aS;VJ}iGB3Hnj7YT|?>r>UNjK$?)x^d;~rYsT{b_vp?@cE+(oMdCo%`R!O@#O^R4KA|*n^l-O}{c~4Hjn3CSP5LJ^ zcvgpNUBlAMKsU=NsFG~|4zycT?Hz>0FG`M)YjAMl`J>Z{ljb(DorbodBF0&-gNHAq z0$N$<0$Z;x9(!K{;H87b zo1nlOp}^kzbbgUz$v}Xf?@nOMPQVQiDoN*m$ODBudi?0GM?Gt{1L;>O!*}_2VH5Tj z_$k;?eX-#N8I}=ydij(GYe-ghea+_a$(NRP*0f27*O+6-LNSX?w|25SMzP5-qMy;{ z+Y7TF*2W@3w%*_+Z8Lbn%Ng>EP&dP5_XTF@_o31{#x^(X#oR2 z_dDH1()q6n242rTacq_}&z)n?0(^!-^2((UhA>3Kk>5A6R$1SSEu=T0 zcR8~6mdVU@d%R{>7hEtE1@!UWF>n?_BDJgy#n<`T$@z}s(1W92TK;8*`X$Oo59$}) zO?yi%YG;C{Bzlc#&hwa*neM%LU;W~m8a>KHsg?!IVC3LcN{qPQZWs`*S>ty(B%g}K zihVR>;|{03!!_y)uDT&|qrI`m&8Hpl?bQ2k70{O1Rw$;H^3j%x(ITxxywmaV3>w1+ zU+2C}*<_;j&a1gsu459F&5~PJO5MkeI1h2Z<#p6bLDZXmk7^fIicW(PH{Pq>!~Qr z8kSpdO*k}gWYO9%Vt!*xmcUM8)w{alfp4E$;mL1K5pZhr(9h~W+Dh)KwPdNc$k@X{ z#VN+Fs6LXd_tnMPIUiXSEEXf3@jZ4l`v-dd0$<)aYR^0tlu2KwQtnRtv^z3h)1*pK z=wRS_;Fa*tRNxc8J)@FYH*hw{;syg|2&L{r( zqLfv{Z`j*Y!{=t|3$ulo6_FCWu9lurB=L5$&hC!DiqD(l^ zn!d%=JcN-?k9hJI#_Nm(+wYb^KT|+6Vp*Sl%L25j$aa-ail)pX5Y=??`K_9{rLir2 zw>rSg=TYmw5$4!%MG=D)UtWAHwGs?`=G(fIn8~vPdOe$O`~yxi=@_z?=B^!Y*kkA(YDlcdJ3)3(QUzBl77eLF@(o1shHb8AV;3o`zoZlX_dsyb5IZf+gi^ zc3iwpltpKr*EPO?6Bm|A>Yl|!>DX;VbOZr`)#>8ieFwR!wGFlAf=OSiU|;cid=Tzm zvT||CtQ3qMkBl?L#&c)3;+XbUwaZ2QsnNa2A^z2&y|BkG(hm;Z`pnVsl=p&V)ri~* z0*s+T-$GJ8wERAhRR}a-f1sz&ToE)UF1)&F+tQ`Vb=KnPaM0-ZVt;bzgHf0Am`TwF zUP?9f^Uk>jc*`whN! zdDO}YU(7cDtM|{C!ajqns%d%Q+nSNwx%#Y`;yJzgR-Ws*h z|8PslD?k$aEyaBfL;iC#9u9S-rl!8a4&Gv^WXpav_Eq3zfwQ}7Ygizcz3=@Cb(5)! zzXk6aeFdcg#~AC(+(e&0dUTwql56DWF<2R&UVm7}DU}!DYnjQKG2=j=y|lR*8F-#M zw?BftiZs65*B19X4I8oslsqU4MP0P@R;L%5;k5OuGF{ zx~o`#R~b>=j*DCMP8;V)kd}S2x=P7`c}8%eGTv~VFBJ!WAGvaQnR;=u1JIn_2mM0f zi-V4SXS}4+PEJlg0AGVH7pC3W;3O@jyEi2dXx!-!@Tf4psN`R@^-s8-G0^6;J6_h@ zuXy^Tr1@fXb10LaN7e|wbl0RftwPjMmztRbR!D9kxiY(TsMq68n)WY7x$FJEwLPa~ zCi>DNre!9F3q-fYf-?U2l)Hk>8=1Cn5H>t^xijG5vudOl{?5t@j=D?#Y2pUbagnc3 za;me*ZbQbbVg-}aC|=+B&_%|7R(YbPOkaJ#kkdRt!yvq~A0gObHX$sVxjA;a=(VTK zdc#PaThRU||1xFOA^rY(){32l1GJF_uYWT^{$Xr+a+D|8#l5PzIT}h(xL(EOdSIY= zW^Oju&)w)aTLSf)6;2atnW5+il5QwV+$zONyE#Xhg|5%YGYx)Z!i91(ek z*Y26}DJRR_BUM7%UWCkmbPldn0WJr|` z_p*=S2_IwV>-~%HL2lUIMaK8shoq$JZ$Rw##Qe9Xn*aj>@bsx;5?b@&UV71 zRXqoX-_BqqM~^N$uyv$blwOTH9lN%_i$F1bniJ%;PVg&>MXy#C38C|W^UH@H>xbrW z;Ngf|YP+=Q@vTSWQsaw%|2}HF(yy$$ohrsXRzD>L8g;e3sxsFd@Dg9>*mF&n@b6rk zAj!$(@lL-*D$k>bArF8l`n~xc~S`1@fufYzK zoi03j`@3gPfss%=fM;c5BIJyL+E8I<7EwkAMhSuvBXG)uUS7%FOJwa>+@3-APY=;tp`Dr+(u3} z*S94vcNt$u__dFYKJBDN!Q0E|xI?Y`Na&xW>}&-RKkvp203ECw+K;(?Nh}a)U^tIh z+w)IiVoJGu}>)98;%njU784hhvw;M<#+IDasooIKCyW{4Ljs z0_%UTTiEmKcC#sMOPsfVQ?hvH!gBw>azYUE-+UTf!v87sxg;{Jv~+Q9oslf%f7Ppl zNl78<*IaM~PiQd|=zv9y$x>-tCadVQ0)@blc0uZ9QwY#O^;R|4jX!$EtwA~AtF8k# zxWrlFc`4}FRz%w#TGvB2Y|kN72uBHR6nQu0yt3DB#+$E4?H6?q_1n#mVr!thDJNq) z>V_DEgBf*KUQdbEXWe?svPp+61O9dU15u+KAO5=s2z;6D{n0MWmaFr!xa(MnjZ~Lq7Y+n?o z5R+S6bsTEWktkw_dD^`qjS%<~UG%|WH`(@VW83!iiy|$k+ENyfrFMjKcWX~^Y;39D zX=>A-^NM&Ja6+%Gtqp_)%?92Kp0+#=_;G#Rq&np8BAc2;OZ$B9c*=2T+YJXSe>eLj z0j>YuwkoQs-selt;`c%va$(JLtG4_L|CWZriLVcU82j<5IeEykxurFsPl|hOtKLT* ziuW9NgMB-iX7d?sh3i+HRToD3L(rFs3?7~*2Bu%DQq$Aa$+IyZ0q&Pm>k2x9DQ4Cr zO*s0!X@x(Jw$#TL@bn3e5rrlY7pe-CY!2HtYHU}ycMp#0*p!Sm8X{O24FavohL1jv z)iLd^pOF&`YemdE#boMxiqyKd4>(cMo1uGyIJmnn0)b#vh-!jGi<{{3!a~r<2$_lz zt*dvyn{+D&f?!rTny5;&f~Fiv4~R33iZ%2# z8{SNFp!^>ed6>f&RKsuOxt^munR$c9!PAf zmu_`1NRR;WnplAm!{0C>xasl7-V&F{-U)m2%@FR!bZ1yKF{3}9Tr*I!pM){=2l1~_ zCaQ$Xt%4^0F_I37wXWL;2Fr?Dy%s5ML*M4?Mr>jf7w)QQWT0B~!g(BgSzg;^#R9RT z9f%W+DP&<{eU|BbwM~x$l#0cWB&zI7M|p>r;2*!r-VQI!L_>t&6A+Bn<~9EIH^gB_ z3WITxM)T5&#!OH(2*D{$J6dCU4~&wd8Ro3QO1>>z3?>Ykr5ZQ3 z<=FIWUvtH;p@1o{K4Ew-Z$Q)3H94nJ4-(&v3cNhyKE>Xw-h?)vO*JpC#aoew#>Y+- zy`TskA_#_QXg^VS4}rnd3a=sWi)5if4lyib0Y@9g2fIUz+fucI2aBth_`s{?#iE+o ztPz_E9S&0ISF{f<#}}7Y#km{Wn$v?G{6+-g4E8V2T!D89c-mPMk%Qm1n+yOoCh*?$ zS@Ho3*gDga@>y-S<+|&^Ne)wkvX_HHAel0xj%~!eI?LqevT!bIT5Qu+>Jjc17#zye zR~}@cRz5}NB9QhAYPulIYV+?BeY!xHsq{Z&j+aj+(4FQ*7J_vqLZhWrZ&tFco)u*+ zr-0J&V~&@WX3tXsBQ^qsq$`EEp?Uw=> zv5IU3_sW+ACF25`zoYERHJMD*l@Qa9H%>|xs5V^eVS2@Y<6^|8e&t9Yrano@5K-f- zrloN&krI>QQoX-|dJneKUdz+86=o+{n7o=H#}y!`FLXBs%Es#Bg=C11&Nynxm=G;NC?tt?cf`wza*jn|bu|mmXt?$<1yU-Sra7 z=JV-=wGk`8sJ9nP=Z6I(Tb%&WIWpa7KJYg|07H%jhH5e86yPz7s$z4$PeS(gpFxKc#VTkqIiQd*o@{FJz z+|BUucGByh(oQL<-&12V6}yTD34e{UWdTewIJ|MPUy zf#}73`7E0+WwEPZ*}Sx0)UXVNv2L_6bHXl1$X5;+TDQ|&eBp5EZWp)Fs~1s_2-<8W zMdUj)cuWs!7z9KX5J<;zq6@N&pVB*BO>M5D{Cr;bATaw|^&CRD*4`f3nJy|boj&3L zd3DK#z*a}P)ZIh#cI#v{TZ|j6Y=ODknpvQ-&Cos5`SZkKJ)tKbo&3W{2{taO^$tM< zV6c|nmthl9*1J$eVg&BSzJ2d3_3;aX-C#$$z15wWl{DY_L*MX}*aechkDaha_-mGG zCb8uAQ!bJf4T%Z^t4Is_S6a0-|J<!=I+5P1?S~tbR?sc=LVhJBZG)BQ@I)qGX>|}g=l!+-$LxwGfb*7?95?+t80rM07 zj0ov296Yne9t^#R_r^-g6WYbW88^D0gwSEfz|D48nuP-odQ!ac1^RciNuu7i+e6Qb z(z%ZgY+~l#h9_2bB3l1EDe@2NkYgGMu^>oe5e^`+G8k zsF=7C42i?{v@1o#A8s%d8f}Eevh=sAV9gA`>+F& zsd5b5_V>Ts9N+8*t~8!BDHbH~V!7ksynjz1`$6K|wG9m} zgCFTIg@%R}N7J@N&w$Oq{~QIFdjOX6=*!V*ZZK((nCCXIF`0EP@>~TpQ+%n(7MNoa z*0$FVjNfhGJo7$Cm9ABc`jTP7s~=nQh}C@$YG&ZBhj^f?hJ2dbN61_ec@;qY-&PO5 zZy{>T|8FVyoofaAi(%d$E`r|>Rx*7iJhww&C50m?jXIO4cs$dxGX1%F$ckN{_Jxx_ zjrRCaGd@JC?7RYEAe50j0uD|YcOdY|Aj zjY2)FLK!5?5tBxVo7986to?=pIdg_`BkFh^0R@C_3zEjhVE(oh>7tg-Dp*P5C4=x{?tg9 z7inDHK1r^UoQWJ&u(Bigt`^(3!hhKYU2>BL1S~G(*%o!I3Z|i?1Vxw!;0mq+C{emz|CK^WaK2+Yn%)XIHZpQ282*QOpu-a&{1affPf-PNH%_eN3< zrtfdg-}X2@UVH2b*`H?N5~F8SR`+S-^4y-5vs2H#HI`TM@I8|XDB58rXKrpj&9muE zD%0lnqq5h{bBnvXAD9%?p!=OEC$;WgKg@ZTf%BA>uK7>n3B~m|QCk|ly!>Y%m&Dm` z+Ej9|c9uCU-Mq*yNM@l=C*eOD<>{W;&Up6Ze8JbcWsaaFd(TzqBe@UE4Wl~SkK%7I9yD(zQ> zlc)Q((s#4TP1b#Ls^p003*=sg>VeOwl{xss+2SZ%IDcuKQBy{V{bt3GD-d~W?JoEM z9KV&gk4JarDkjyiU&owFKs=lYj`OvMw4;F&hM;tod+efQeHAqn-q7NXKUFypKVD;@ z{QgO-e)c1TyO0`aTX zeIS)-9aiNG|6OVJwi%2|g%z(Lb-AWJ3Sdr0`V#W|$qgmk5oGD1%>gXj+E*A~BxwPW zLgdpv45XLCv`i_{qwY|Y%qNY=rtWnQftd#9>#4d|1pxzNgXOP@gW=Tl92U=bbPsE5 zCznkdy37Sl;b=_dDbjQ59s2F*f}>99yfprF8T5PG+L5a1 zZJS5+uM`Z4YoD4F7R=d|h;Y+K&$PezXJjoBNbx$@oqxM6<@IBPWsoXH&e{y%L;c`~bC5VbBMOXg!&u z$R-E8Uu$libJw4n7jLfSZcZ+*=B`kCUXduCB7Wx^@v?0Reie?i;v!Yy@iJOk-Hh>a zgAZJs70(RPNSMtMm48r$l0wGND2o<=H17*MnY;4Tu{#UAoV(Va^<@mVSs=NQh;jX)^76}K0$JyhRa zTR326mTdc!IEhM0Dc@M3Oe`?7H=0>Jl3nfD#jSZM^NjIRgmb1K>K}EV^9Y6*Y(hU3 zuQx-UiIG_OH&=0I35E9bZ+ssUke!P%GZd3^)m3LGjxNI$vpKpZH8nMkUjMdStR3Cc zGG3Vrc$jiNRlRXqd6aT;esERp_l=WsBxn^Q5;KSqZ~Xa;JGC%7%WG|-(3zlu3sy*p zjoMrIN@~3b$~qYSnza|6inh>l5=aIvKSf<7k-xgLbu-H;C695Q{3edMZ;@FF39&hx zs4rzYJ>@TZ(I$6gA*FHz$q6!#O-QSxHLJ`%idEFdP~5FX&T3b&Wy>gpYEV3t=}}UyGZC zRTPj3eQHCDn=a|!=@@vRUl@SlSDu)f$xg`0 zvXM^C0ic@e>lcpAM-+e=f_X4qg1L8K$migbcKFn>Uh!$c%WS5+_SHGF8z~AEhDsTz zQ|-utQ{uj9rFqiRLs5vQ!}B31ndp`vw+ir*uq?>}Udf)`gL35j7W9j|bG*yw;BT{j zHf4%$rR95@Bx@JL(DCWNW+8VV23U)uBj?ALS&v7TsB&fp3B!3z5Nqasc44Tjk&TM_ z`oV#LM*v_2u;P_^hNPu^F8gqGAS*4uwtjA9>g1l^qRMBZUhwA)OQADalQ#-N&L3hy zhUmUkzT)CyVJWf2%S+Un`4_=SqGlVYF2$1UP~zkEogMYn1b4*@0@7hTu9T2Ij!iC~ zkp3m;9{s~s;>cGhIrzAy=y#ZxhrgGaE%S(oYIMoz?hw)A^fcE15wrBCIwWyOBL$C> z@>eQG38r8TSwMQ(L4lWt(TAC_AZ?4(pmmBw|4I&=x3t6ZIksnHpdar<##xh9{CuFg z>WV%iIre~ks{b@~U*1g&F`1W@qM#S2T6mqR{Hw?OT;w}cVoXoWZ7@u9f~j}AEA^V;$K zt(sViTC28_QH$09Xvy_q!R`AWQ1`$FzC&`n#?{Z1J!+|TTt%pP2SusapIDn}iatR~ z8L*WsNg$@j?KotJKZVbin$=DTO|P}6kRX317}AWT4`QNE#_A4tSs9`k?*w(Yv~;gM zdrT+35U;v@030N8ivVRo%EH6B%)CYVr(LxIYM20jU*s(1e#^j@Vqtmb#mB%V>#eyr zSt+G&?Uc!J-SYbxe=yDGEZ1E(gE#G}f#-a{ZMNm{n4WRC9mMjfymCo|Dg5h*JK4O+ zx2Xy@AGE{Z5q|8on$DpzKL_lZM!+Rxei<& zn3)2&9ktqSHII9@<*s;oE#%TE|0#rHwKc9z0Js!53^^!Bd>UslM$Op!0N}XfJ>Vv4 z@rvqYb{0pk&z0bicYOcRRCTjrYLg^l_U=<^$m)5q(i}`k^cRbljqKj(o;* z(VaUvwbKS3%AYK&`uugIjNVYUv4a%|(=dWipMZ zFQ)p<98zm!ksPTSy^o<7YY{(TzJ;bW-AD}ht?!LJ$I<{_^u%u<|$@gti za7P~OGS!*!iEsurjYsC{-ZYf|{lR7{;#|>yU$20?7uvYgTw3e*cQEy*L2w)>xr(!$ zo_e(`MntY<$mr7*4#3(O?b7_pllD1e(v+shfA?x8!|i}suG_2itm7`XiVBWe^;XNQ zO3KOH?5g}6uH#~dc5ia)QKJV&*oNMJ-B6d}!w;&6vgBdv!UQ{Hqa8X&i?Yx?D0%m! zsaUOk)`OEP6i*!f?g^X9J2s;uWxJ}sOcBD0Ysa5CNMzHkDt$Q@g!e9LD_2>+sHtuK z7PCp><}(rx?2Nyl;~Sqb^K!ZaEdnnHSky6RKpRTNGzqYjxhWS+imhThe!

SK06nnx+R^O*dwP1hc;InZpzom@>hQ(I#f8$$lhSwQ&`R5& z=41jPnm%%AJdN zoHu{oV>J8XK0=+upQ%pVMv_=k_vmHvM_m zqmCzcs)oj8gS90NgND7r^VEgQ_xWlrhc-J%W7;AqSJeAijZ4#ZMA4TYsIJAOK~r_R zo3?|Ywi97$+|MD9s$W&}zT;hY8c&)!drUcUeq;5UxkXlfnXLbY+uK^BZ>9Z(kUwx6 zS`X+<(mC{O=J8YVxaN4ba8E1{o7@ z*$bMmQAx`YBLq(2#H58O5bVEUPddDLuZ>n7QL_yuGcjFTUKPxDT5oY1u{x0Vq6md^E22xjh9sba!cS(WkPt9 z?K&cUMw^Dx`IrigDH_A$yo928U55cikGl z+pzXn(O5=kw*q|ZL{#*%+&7YTgIR`_; zMTMKjPZ-v1?MMB@ZCoE9$JT(CIgpRTx{ZOoYHBZeH)t^BMujd!%k$nu6F?gqWGVGtO>@>`R&i z7Nzq^gESO_!<55y>I3pW$gE5BzghHe2|aYmG+NC72VP8iloLNzYAqMl+_}T$$c1Mm zWIE<`=F@s+?90u#1i&1>PCF&_oSQmm@49V8CLc9>6(ADmR?&y!T}TdiQh1}p#|#q` zf9?T7RIIq&Sl!m5G7ZDmDif33!jd9e6C*D!h%Ue>4QoRR{VT7RHCbo^TAnc@U(+u< zD8LZa$?yp1QBz{*1J{wU*J@XfZlYV9dGCpi-1(#1CyY-<%{Knx_?_}fUeW!Q6{o$W z#iYhkU?*jwu?2xOk867{ zq-7oFd|m^yw&9=;yJ<2FNI72q3+n$Kk$LANN(PTzTiN=Av$GH&tK(Di(8EdE33}ac z@#k&CipXJ}o+vDZI`UB=h4P2zhmNuQ@4HOgLv;DL!=apTqmXrIzREdNxghl=SZCCx z$AP!NvF^F6M~>Foimgs8ZD09;ANbcE-eG=%-&u__fEQOdmX{){| z)b$*l!+H_!6IxYiO3WgXBAdDQXSt_6(ex*mBUU+*{=IQj+`{bukCZNqB2i}4IvDx8 zlcj#-n(|pJaOEF^@7N6E#jAl2VjZVFtR3hzsf~$tdY+ua0@$EPT>aCg)YskY-zEU5 zrVtzZw|IgtiAAdtB0Gn`l{V0-j?-SJS$4V15eaz&g~p#iJHgUvAU|VYe7>lL7rJe->2?1DGM=^+t|tkPW#Z>^+nnz)+JgBhHCQlujuZYr7f|H3~E zsKM9KHQ^zq3@d`WN5T=Bb{^u`DCPw7%??ZuDgDBW^?#O6ndO88DgDUJ*f2>Awvjp+ zAHEn%4OF456hc3bcE|Aq;+DF*Bs8np={vVZH^eD`FM( z!j4J#DoX2VInJ4SxuB>Kub(~lj48)-_M(I-uS;oD$9V^s&CF`Pj{BbsBo`CCn#d90 z+}p_N1;v7rzRFoOFoMJ8vTSJNwRm3P$G&8m)R}40z}2=Cu5igOfxIPLkLz0ZO?mxe zBOnmo&Fwp6)c5}KJJ^W@1;Oo=BgnxnA?YoEDX;>zW6VvFry~)ffMrM^BsmkbL_CHq%=~l-xVq576>SgK7<>3=|L7p@r9s`b$rh z(Dg6CNp;6iJtHK-*@1_Y-Asx?>|f9a=lpN-;t(7f$r_Q&af8LD1Ad840mE0jS@lDc zGO_8@lKa=M0J*O@GeLuhrR!~LW`9R|6?{}{{jqjF6f~xj+z+tTnG^Y!NrbNYzfRs{%)_nHb7g|DyRfHO z#8!%x0>9Kxl3a*LUU8Yh2S(UsXe(TayI6GHZ1a~U$#0kjLW=x?wA{ODIA1AliyL7- zzRq|U`|jKnyxCUEpHSfNJhhQLhD88>4PR!h??}k${SmqebSroOZw$-qe+m+t*7&-? z2#RaMfY>t0MbY^&sYORzC)n{oGhM~0gr>k#Ym;h-x|K4tMYf`!bOtqF!LSa%w=d;e z8v=qcT7Itwt7H`6^VQCPfVX~kXCT5!yccL? zQi?z%$hR}R#JPH-0!mS>m73X4;FouRC~rKUBdltuyfN;k3os=E?1IXz;+ zvCQ+v2dWX(ZNNp8@$)n~S5Om^{5#ssXCQoLWo?1JUrY_c9|q_eC*6MQWi&g@{k>>I z;l@A3V`fs`vvZJZzvuY}-SqA)yQu%04tQbRmV|$f7C93kA(0;{94#Uyv0Fdcy^-F1 z9Tq)_;5@+K))O|5<~J-f{uZ3zYZ~A5WEU-vO`W~RbU}D{GJ!*XI*ktP??iWx?QGyW zC-eM~*1^f!r3BxU%YAmcbDFg<#*>WxF)i*)~@)RZePoqMj0%gQ>5|zaKiqV zdWPK~aPVRALjn8ihd7GewtRb;ajnWraa#Wq{=DHg;yDdvyb1*gJ#3MuIsYSHN_V7P zv2gzS=*PJYQL@M4Q6t7Ii?}16GS+UI`UE|Haizbzi--`H-w)&S)m8?<_8twZ@FNpL zjd@|1Cx2OGXvh>#$3LvHqqUeF@^hZ#juDxgt5|9$BomGuju0hS4ek=Y25!Vra3^K& z_Fcm3=I$oHWjst5l!}Z7=dvgA5y?!>W4Qz>fiI%CbuwoM$X%~pat2jm+0KC#0ktcU z^_=a6M6Ck)_X<*SA5cv5xZKYt^!&`x80Y0^e&LQqNTzvX<$Z{?#M9=)Lh~qXASzy4 zd9-e4AdYIa`vEIA6iG0Q^}-8Ag`;L2imh)d3|Ia_FAIQp-T_0cGU=I7)XIW((fCk* zgTBbNXff-KPY-3KJ=9Q+d|SiGZGZbl^6H=Q8~O*{6j`-AG6&9b@A)qUE|4OLcgR@f zUl;35FSE@y$^{VVDi?$FkRp+Z*2;z}(~3rCf6xrfcsKrE4xu?w*z(_B{`c!zh+=6O zKJrI?X@%{@R+#r1ScF)vNxkErO`MqVeaS~{{%=FRhH(bGc4az<_|1Kd$#e91q6&H8 zt;VpVcMEee^l<7#x(RcX>q^!G|r zB4_7cJ$e` zi0P`n8OfFy@2k(iK~U&(s95FIb_M(0?Y(KhV|mi4{-5QwjB{I62ZUenw2aKV({*>} z;%gL!i9wD3q^pMvj!N^z^XvbQu(yh8D{!}cD^RRxX^~P~3lw)N?(SaP-Mz)#rN!Od zB|w4V5Zoa+fg%Bd>skGud&fEZ?mNbQ$Ri^UWPR(CIe)W%avgn_R|q!_A2oP|&U!@z zVxPlsYFr@b(JMm36*6v@lVU_$g!l2RZDmKvy8X=8`$ZU<0p2oxEC%kja__Ef|G`|O z`qIVLdvd8dhmiHseHp9svt&vpT1{Y|FRKwn0O3hwuwaK!ki@V^MI z3svGR68q}*EYq2X-^-v7(%iA@aB&;zTf5K%yoFh2J*;Zf#Hev&6|bnDB4=U>z0hGV zwC>6F?@{aiM)^hR57Fkw$|QD$Z2?*h(|nPk#6J^S&c6dmzj76NGKKlH7J1;YlN%ij zQdSOYHQTKFKDO_Cor0i+Ab`GUmX{sAq5Gs`16*EzDqR0O`Q+&68uob4#N6#22(SIA z4-ky!L#5?1UF@dWzV~&YT2AsBv9CebV7@t+LQmK*zZ-$8xRZE1ip;R~vO4*U6MkM2Ie*CIm3InU1E`RuF!iY!|enm^-SDis-7JqoMFUG!T)WFr2Nhjo2nnQ<+ z58HO-*ELrO5vPZ?V?-?I$kc22Df=lgs`b=al4%uQhJV}UlV_A#ubopxewSF|7i5##+As8NVZ!lH za({{>Pcnye99xEpOn;f9%Jk9M>LaHr#l@p$3lJG{Z|3*+MA!Z%WD`9eXa+=NB3xpD zLebUw{Jr}*EMR9L$8TV6mug-oc^Y6K>2&u{D%7Z+AHG#ptZHI+3$kGV0K?mymt5`Sr45-U@4a&UX)qVL4S^&g;%zdrlh0Dx#?$Bl+`& zoFkojEY+AlO8~BY21O~kX~oqoY`YQA$JH4Rn1rzncWqm9-jAprapK}!%$61^^5)@P z-1RiYjOA9U*`d0U~g?ALsg)d0oUHD^V~N1wH)JA+j-EyKtl_iHso8D)H8TpZ!wnz<#56@L+fezTd#2+Q~NR@KGy9m`d^te~6) zM}(yw;^C_s51@f9-rqm@m}=JvNPK$R7zX{_^sI5gO2GfGGwb&HA1Blg4fna`Ek_Sq zuharXr?iB%Ig$_Em^R(OYkQkF3P)>FV;M9yx=w8YHSZ#H5{$n|+!##mqs_qbR<4>Q z7T+7+3+qk!YlQb$`$HeoCWm>te3<7$kUSe0dMHuE6WBgGo5Km2FvKw8Jq zLtqCD4G0bv09sb5hRXGV{kSs@Vt$Bwn;~4ZrjP5A^rYqB$9f*&jos%3t{#WMGjF~U z?I(fDBa6lJmk>tbQVq{k-D!MUkpcs<45rf)qAA}Wid|uff;|=QKuUIlM#?J8V1zQ1 zgN{Mf{sMUVAQT->HtLd?;!hLOi33AP*np4ToDx@RQEIfxD6OJYqj7;$!<>no;(V=k z$@IXSd{ErX&Lz4VAnYGNu7f=%h4o zfv`*>-7(LPY*|XeWtQ1+Omt-DkV6x<4rE_#q-)|@i@I}}?xNWxotk%WpOnWUM=6`A zmhU|_1^lA(bit0M);(%DY*Qt2vvh4g$xtGe=N+c`M8}goVN?D+Qpxyrq)uq87}|FN zGR3on$o!~rL{d3f%&-?UxWJIiApQ8a39L&K>sqfzlkj$sM*rO4pL3c=P%s}q?$;3@ zMJIN2<=)*a7~~IlN*;x)+IAwW2gAt`Ti*m*umgLx`}N&E*yBr>--TZ+tW%aH9pQG8 zhtOul&ko(VixY%-P%b}{n{3Sc>7SBhEs5fUh@_T_j#+|242wGqrPA*J$fHo9J6B7x zK#`!Yyi8Ufx7H8A)m3CV<%U^;EcEBp09uhsu#N$0=Y5YCm1>BE>(|3HF(UN(6q6%bRvC`IWC&)x4 zW0RZ=Al-Onu)UhEzJo~+n^)h`C(sItKc9v1jX;URnV;cQWW2#ER5H2N zfFZeLuB5sEoIX#e zmDbkoI@vj*rfdw>I+=V8jh@GKGg+VW`UTATU%PfY{_cM|inc`S=h^&JaB=b)p0QxU zm&Y6$B(Sis(6VWfuY#EG8yso%H;QDe`~Pfu$J;(}6dACoD(V!>sck`_;_wHd8{XD) zruVLJ9bu{j?Q-R6jmJfbZc4^DGMQ2tI4vtP0M7d{JTyQs2bf>=7=Ddg059$^J`cs8 z3RhDD7rV|7K-p`DGnnR4(-j%!u&#F!E@pt|kj6fmqdMU)(QrrPyCySX-Jp<0MUb5x z#yLm8(3>5U5hNhauIMy8==u24faE0#c;TyP{)S^AeoCRk-7#$oU;vq^wN*Yl>6=rAMNvaA0yh9_vE|xwMBPF$gZ~sX*mpGCrl{#LOAZ`U$}t z)9v;6SMQS~=cR`(X`JA^7bAOui%%!b+nM?9!h2FXkr7>{<%YEm21=!}>^2~{7Ly|_ zQOAOAa}$^hFy3R&;tTcN2;dG@E)J?}Fdv68;4I*QIvN%LVqm3qe7!_-``+mOwBJ4M z%GGy1?ExQ>KOb!R{IqN$`FDdnVgb1mBmnwzf@F1fp}T7WdM&iQloX~nH&J%T;VG#s z$Iai?*FoZ&hAH^;8sha#LG2S}Srv4;%C2RDbbK-TH?3vnlj52dq0N*Ep!@{{PhLQ#geVe7o7$sU5bKt1g2q1oPMgCra?>M?W@5S!7llZwYjm+!<#vyTQ}M!ShC8F{^#|m5 z)J--}%~|!IlpK=`wo+mfNg?4qA+cRpuG5?(=q)y5T7vqya$T|NN_3G3Cqg>Yju*tRsSAbkk|k)7&QY0&H*O@Y@6lwgeEd-d!G{8@P4-3&D$xr2LxEG z^@J+vsBlr)-2d!Mg3e|tSAEPpl(4H|^HSG*6LyUbuChw!ncVP`*|nA^2%~Kp%x92F zyl#1lv9)3IzrV33mzn%$H$5hcrT!npXd><;y&w?CR9`o8Sa617+_!G)?-Cq>zH~|7 zJOg?&YIwe2zRndfy&2F~*=`qOOnP!=9=F#W)9siPy>FF7mnA9#h?yi20Z#i4!7aZD zr*5rYwH8BKWtSEZSn-dw?Ezcc-q?s&Rt7xY&2Ofvow07)!!bVv*qos^gA@hs>C?{z z$%>lZXFD{-d564@dzIJ26~4!_=wb@T&+!#16guSNW)TU;J=85zMUACNx60$rwkFP2 z=jV55)zX^h92$G2biRjlz_(suGibK0r8`*O9Gwmj&7~Seq(gB1NzowHeCy!RL;I4U z<7&R6{Fb=QVmg}TXUw-<<7EMz&G7O$!}b)7eOEyu*>wEa7~>k|9k%|1O5;xZQo=B| zkA`RRODPP6$t_*apY`Ykr09llhPji76gg;oLY?xaR5pC$vogLkd2PFqKA<<{zN3Gn z-qj9w*YEawwwbHvMAzODz^|iLb-@^;3mayWI8PL-j%R0A_@=y=p*}Ru^L2CCh3tWF zn?Bf(>{o*GP^9Sm*7tNG8eGUIsZfKov^%Yg1Ox5VEG?mD#0@|0H(omP#Ve1RjO*<7 z>X*ziPEf}&2qpC9+nA3>oagfz-@^l5_fFJ=|M1cQ8&;$T@XEfR804Xo*5y=7D6iz`Eu-KFx9GgMXAn!>?;yqtFK+{T?{AM6<6TcOusHB>5= z{I9~qjg`pzmHW(VTI%V^JUE0WmXSs)?ik8vBS zos?8P16mR?_qqat5!a*QI!{19BIqSlPfEuZPH&mn`@PeIA!1G-$eOXSMf z+2rlyUPJv}pC$I&=SM@Az=T~+9;FIOp>_P;t1(fAXQ{ad3iuol?TpyeEjCg@sk8Nn z1C;@J_^AM-v)jD{l|(#5ps9!+G)Y#|jkt6eI94^a4Nzb5#a5Ovu@jxsi;p%`>VR@P z^t7z^L6A`vnU}&0w^wmshs>X%rL-8Kg~KH~OJt*S!Q|`fGOfh^3lY!tSpViVf&Yo2 zcFSl&E1+vbSCM{OUVN5lZJ?rAEx&Q$_xaa8aRQzwid7OdUSU{FC2nErxVN1kalUMy zO;wG;-d~})(TNqATHK(^2e&K}isVLVk7HILDVCI=diV3mN$?VA8S+$}{|UiTM_VEu zKV}|4YZeT-08tRF>5N(_RB4);m7|GpINAxMbF|avdQpg*6fm|$&k}?hPR%r|R4e;Z za@G0?I2aJ$%1>32s4_i;wu9nYWP_Pq880N>M78LYSSIn;h3~QS>Tew0x9+?H-M)l? zfL|kHi=Zy`CRmJ3fr=(6bTADa6Kn(ifG+=TxyV~OQi}%WhEim9T)LL7&PY&{?nG!R zM#h`xT-vXnvuFhOUSvYyTwebA;!+=zfioaVetp5gPPP>S8Fw5t5~iDvN*{@Bx9pM=M3b%Yx_CU_LnbRlVLZr8tm?60Pk>r*2(zU(yet80U! zP`*;kgC1o-kA6YrE_p%>NPUCcUL8_`F%Q=diAwfIsr;n>(zDaQ)lAl_Pn4|e7mOKl zG_bGk#EfWoPe(t1%Mh<+K-~eVXYjjlkb)8$XNh`-DJOPF0&mrURlX%CcuFt_!nb+5 zL_j>A<{F`iTBAn)x`FNH ztsN7?6}!O(=<7|7S8h^a2XR14&Pp`6-F5FI^X z(9^98HC{P{yLU>oFwr3v&AB+fo(s2zi1YXRCY&%!;RaYC>S6mvtS3wqASw|+rMou5t&a#P{ z$;SzkVc^v!yI`xl;YqpwsBviM?B2FFJmb*{dcfX3LpvdwP%amfS%4f+{A+mZ-l-n> zo18om#|;3dxfyoaqUM3gJ|DjugINgFeLAPZtfMdC_;OhQbACa`#oq~InCvWhVhnX-WUTsn6>Q8XGt`uQFR^R9vfebO;Nkp0AarC4)+IYe%=e0P5FDUcf9V#S zMgU$wwr&jU)Kj4hm9Ac-r&Roct5{?G1uwSA#gb}LX7#gFB; zNxB++;UeWN!~oMFH9zc8ujN9grY=RQ*{jBIg6(H~zHX!0(9~rw-`f81jIe7-POY7g zRx{mwu0yx`t1#zp9uvs*Nm$up^m1gXmGFICl{Uj1uJ~+S0@W=7 zi^zhqO>p+6W985csecHiPJq1kN$}9-#OW#z=V7P`X0;16*W-lk`(@&YRj|j7bq{Ul zHzA%~QZKNSL9SQ~{97k!^3}0Qw9hTUkKcI@k-?}3AEdY>M+t1azK$c%v%V;Fco8OE z_kl-NNaB`IG{$E1y!j?eSTt?+Z>FS-Vm=48GSNw#m7sTyYx`h7sL(0il)x%6^pDxf zw~s3lv;J(?LB&?JF`{(CPp)HtWBU4p*a>5OXbq#9kh1SEvdP<1~U9r_^aU9I`zNv`|qow z4?w@YB5=P%dxgN_int0}Fx)1?qKrI1JPRKaWSTo%Jz=$#XO(BiOOQ|FZ)xC5T?kyW zFiw#!SqwZ@_K@O)N}GObMlJQ&_G(9YY5nyQ9mD!eAtT?@*Ts@Txj#W+oQM$Z%nGdL zlM>@aH25jnCd^Cm9BRxHUr~$VMxAZ+kFSlM*&Pke8 zr!Kwkfq(mNNM2^mX_uIERD7y)=g!eXH09JyfoLSil1d=T<-?)-%5K^r%)MKY&;;Q3 zu@G7qu`vCbNM*0493{F%kUNPles%kb$vP$-QlBsz5;S{^1?5z(e?@cTSv8 z=(cE3;-UNbcVMuRuV3kKyS}1iAtE9M!t5nd%^q)TT2|h!zg$)h_=#BPFu3NyzsLZ4 zg4jAV^iElF!8(}zk8o5Ep7mDBwLmr8i} zNza&W_Bg2L*GESIjjFLo5+i?{e9y ztM?c*WB~?Y1z-a-2K&}+{L?-V6RY4B^jzNRle_{nSyAmrD{jZ7){=9nRe7+i74c?d`OvfHS z-K|_H`NzZc==Dn?$Yl|>^%4oB>n*iX(bbPFZCg22`~?OlBYui}O^~^A9v;r#gBbL> z7AmWJ>nv#oP+qTpcx`Znb@TNk9l8y%;ci%pp5T$mVPQ$b zS0)Fc(@h+=uh8WS)SAkTBnC%n zrjysGe&|j(>?@>rwM5uj;nv^Y%rKv-zCC#xUIw=DgyFqzxoJ=A3`C zse&p2K=eCNJYYmFz!nGzc#$v&OGjK^9ENV&(&WBO#=)If!)Cph|9Yz-*@$6vWE9+D49p>6TQpMqDmPa;A=1<$Onxg`>W$!q zZ@{96(IJQNFQ@BbG1hTUk`O4goYN^3(ZtvWbLoMR zU{@@SAJ}Iw{xYq{{)KB*mMvRvD|o9NQC^Q!yzzskS^YaBay&XRGlJ+h3)wKx6-eBu+VSN)lS*yx-~1WAA;+Fz9e~3&)jEaCTFdhn%o-8=D71p@O~A<)sZ3H zFS*8lU&*O~hkbA9cXt1`qrIla|FVMq{^kKHG_n4)5|Gh-+4A&3=QmjrDdN|tvPUnd`&yGJ-;^foFU7O zV`cp!wp15exHsxcL@%^U)JvL4Dmd(wra!w%1nG@D7(ZP)7)ZAlRh&FwKs^b$?FPg zRW0lakrj$n6nb+gGrB4>9qZ6vWiaJWLbXG#VtM>Mnf@fvnY;Y;∋{y5l1IgoGj?>C<8|rE);Mf&-xskbnm8|7${>PwwqR2Mq zlZ%|8qLPZLuK3)@6n)hqpZaxKk?#S0W6hA>k5ZJ-cQQ?qWlYBHF{A2CB2M-q?3E8xJGxustk_(O}~^58@}PUTpuca6spBGLbd=nv_NCp-$N-}0o%;6IzJP#TgYwB>7P~AsaWr~J9FWlO(xx<{286~ zSA(s0@3PV^Qdv$>6_Tz{r@WQD{qKAR9swY1)N|~qhip(%Qf`34R-Z$9Qbc_zzR{kM zIL>BdSAg3_m`DaOf*8uX9t6o+Hd21}l+UKh!^~w!%p_Z4DGmq*OVH{EdV=sW(JnKj z+50xXGr)}W!LGbpN49`TtcX^I`d%_KGZwh>0nbmh)8ie7yNAb6!&{TL;I@#5N9IPT zUKw+363ZJCc1WKqFR+siURZDKx2fqKI9L)4vKtJ`Su;LdCSF-!m2(~gJ-tji^WYZf z7+v!T)ik`Pt?4>fK7p+5?Wp}6x8T-fjz`c9gB235Jos?g{FS1dHrqX%4ZQ_2T@Eix z54|}$+w=`}CfYtNO7q4#U&G|;>poc`-D_VEq^s=@?_YmkJFZSjL{0S4t$M&7?0oY*1Ra3Tt>*nWMT4Gl7y&DaR!h-RkmJ z{3~2uLRkG!OwyTyS2)v z5;Na@<|<&I_f2{=zq7?4STpL_^?ro6Ca=b_zpKO5!{6eUJweB)o1yb@q)Yv04yMsR zBcrF&lupfP9^wZsLn;WX47>VjTk@6Jb+z~34m_!7sQ@s_($tQ&OwIcCk?S8zesfOx z((>29Xl!9o!hf}BqSx)8)$?tU_(6w`IHd7oNYGG6SQVJIxJG5-`mDNmSYIqp*p)d) z=git~S3L4qCZs2FdS7=_b#n?CqfXVahT`OHnZssM+(uH|);I^ql-BU&2gYop;-SP~ z;p)Xho0H7>RlzduY@Lyj0uIVM`A{m$NYxcmeQhYMwX6`wMME$<3W_NvytE2))X^aO zivIGV-SqD3bXm7z!aU-5%Csxjrlx;KcofmquJ`ES0SaBwaxlW`pTY~)eoGY>_A-Kr zPuZl#UGFw0$b{d&5g2dE3O?*WNo!#bl4DeHqep8_$kx|1I2i)3{Sf)EP}C4@(sm+} zZ(IEXkj09&J4{3`$zv>yVzwrKVrI9nvUy-FX03JOD$F*Gh+1hYDq<*4APUl zlqBHM^!!R%S%CZXht^wLB%Rt4d!*c0LrqOxKG8C?GZTp4;ku0RuqUIAz$q^`X9xT? z^0ZOB{6Zrp$ux6hn>`4J+y;&FVxZhTlSp4ZALXwV*V$Y59a<_6;`Mp=#C+ry>8RCe2DCx9mbfl zE_-P!1r2M(l`*Fb!7eJ={h-+~)Sq}t3ttMyqc9ok7+M$*4z$W7Dw^LnOv#tu(!tN< zL(>9+t7L^l98M98>y-#~4IPpANweEm=4B_k;TiD8LdX=J0sa>hxK2 z2hscnkK?bhJ)lE>uEgRecSymB3*b>C`^C>(!&@O@jB9PU>hY&k_LEKQVtiY z>pB|b(~D_~YUS5l*Nr(3s&s4s$vQR40A*fR7T#*7&mWtbwH+XB>fZO-XB~QRM7+{* zdZwtqshZ=H9_MQtc07ux(A8&o2aT4jf#1pZVH(_GaXAR~AM>U0EKdBA?}p`V&hlXqeY%drODy zwZ-Cs^S83*908_*w?OE0e=1w13vEgWEK*7}Os>uz!7_SDG~(~W9BpHaOc8h5#%gpq z@1OmbQjy`*vNZ*>FKTBmS++vFoG=Z$LoWjm7ngja+12ltFI=E*ed~Mw%Gfys8z_+Y zaVp+#TAjAvOb9*uCjW0lhZnMe_d!w23{sG=ts+H&M|hfncf|8;z*CObW(Xd!n=;sG zdjQ?Qa})93$A1&g|A*mSk>OSwG}~vk{~zw-zs}P`_n9ZMDS&vkzJ9+_(cvDo_tY)3bnxJ_d7p&awXHLceEw4 zp@8waxPf4Butcs{l{$%T<{ZsHB;Qp@^)73sy@P zsZAD=N0oA|K@fec{H=epvt)iK!~C61dMQ{f7<@tTE&KPOzLG#?3o@0C7tnWA$<=X9 z9$j&?D1644u$BlZC>4D)Zp`Jn8LfI)EniVTF11nA){Y#Ro_NPvAQf$&$Urq@k>J9v zU@$`-l9XJM&VYSaMlSn)a64Uk^JiN2NXVb%u=lxHuMY)uzQxnI-=!aR&@FhN^Su@= z@8MN66PW*TS)l@N-vD8WEQnd=VP_uzMACBqn+i`Xl12| z!CW+yqrhw2)j)YEzJB$~y}@S}uooaIe^K?muP44UOP{#IKjcvJOI;HO>BMyW*CT=X zxlx~d9tOt1sYr&YoiZ|cd8v>$j|?{I@U;}T3e$n4(vJQaINE#FP*4B86*dn<4L3uUnYfVq_} zv_II#Pa(<+n-GtGYJE%@x^Bmi>y3UBmBINUf@Yk7L4gr+#Je3UiTaGBU4b``*agZ`-vvL$;42nq zW@iHgw-J*#{q2#I(YIl*r>S>A&6jP3UPZVA_|xp+U>&r1-bk;WH6NtacDIa=Ox+_M zPY?IL&jZ&yf@>uU9==COnzt1~w+uP=b(ruANuLXEk6)SrJqJtZCu><}%ZGfcH$9S` zk1=u1zRtgH@zKhfKsHc{nC#MYXPB?$d04j;U2{#cdVHV&Q#((=J>j&UNbHBVj zE*27{|Jrcl*9p;o9u2$3fYOV3fof{qC2pznwFyI_=Ogqr zr$tPD5qO5aXUcfHKRki%?_UR4@ZuA2&Hew>ub4`jaV;AFkcpT@PW`VxuCMomrK0fN za10yMVajHJ7RFMlUrjmD_j2+ke1*!o``<0YM%fRp%obBXppMaT%*TkqHtJteQaF9!s}EkOQ=@DO3X*cg??&qP0Ex(f zDUfqZrzm)dt65`Ew0_93aM9_XaTV4B(i5_H5`>e(II>X>73$oBU8qKA0OyW{6whRg zX$aDk=WZHZrj7Taa{=CZOT|Q`c|yAOy67r?nf+En+*X6)_qNzHqPRCT%G}ZJN0%Ir zYEEXdF4v8Fmwl0sPU6ZumFU$f8Dc*6&-Dxu*GjY5(;8!+Wrft=)(7)*SQaISJ1=Ks z^_vMa2(B64ycNf zVx?o4_p+Vo0{wciCX+}ml8wC5ws%*(S=X5?D=J#tvq^VIz1cPcVoQKeN$W@ym-hy|`G*%q_xWQmRC~CwlCG+x*)D%wrF!Y%ZB_H5 zLmKC=)@BUheG57tpA~-hV^qx=J#!0-D1+3zM$}hvRd~DS465i&%BBPHG-SI=KjT&O z=VJ3Kf2F-|<{4mMkq>N;h)~i<_0ncxk{RJejsYWKcVugsHw3b?P*PK-_`v9@3d%p1 zFK<d91K4$Wy|yWne+SyS0ibp51507kmuXT&6Hyd`Hg5mB0m~ z!Rz;qj-q|8Pn@p7A2W$p&3Sp>XcjaHXAsT(!L7OI_AQ0?xzNdvG`uM|%TMo^ua4B5 z_RB7G!Mv3Sw%;u`b8x*SLPOQslS74|fk(;vwWB2eRLx4Rk)fF?KNa2R`NY6KSm|~v z2zfQNfZPKS138b*6qI@UL27`&S=qA^y~qGByF;^dhR-lr7Atvp(qLOpo`o+rlSD<~ zWJ|9HJV(-J+Oa`I+`?bpGMV)ryRs>I;N2A_1`n^G5pNc-e#^&=n3s=i6Bo-)KF0%BTJ7K|ppGiwD6^0Z)8tTgUlm12{7GM#ZcqAQDlF88QFaLnr-=kQIxq zzuCD_C1?TmdO`DZw!y_tfwT4kgI?ji)#iZg;J+W7fB;+FtvHZ{Rz09mE zmMl3Qe%`&iOM3pNMYn3bF1hZs0yilcH?Z4uR+4$UgR)^7(`L@IO5qj+aOHze4(aL-M>$k(RA%zuMZ9)^$PU59bPi zE8|i92M}_N2+Y#|Jo#vbwK z=^xISo}Gyu*b(&QTg-(J0;s^E^~S5FYZ0qYXR9PzDm=_Cp?4$;n3v z|Mazyp=rFq(CTiD!f9(T=ggHx27gZhi*Eo;BMp$k;b z<-T?6o0{r;3`6Mz`e?1jo&?TCcE0;8SUEYCzk2-zxQ690a*WTd3YS|5kU& zc>0#pMJJ5gr0$^J=cDO#%;CndC4sN`J;V5rEt8OSPQ+TpnXY%{`h`_(6+V%v{%Bz- z{)A1jczS0uePB%TqH1_F+J{?J89QT%%Hup=`g&MuRX$a?|KZ%_y zg?6am{CPYg6&a%-unjcdH$#cg@WeE;qZ>)DFQ=oAa=|Fz=;Y7Kp}{W@MYUIE$c@MD_BPPGkDJQPu%71|DBu!yod%gnljT8iB8)a+l%a;c6SpsdG?*Ht# z#`ra15nd`WX~yzZwUp*-gh+%)vPcH=Ph|2aO3xQigc@OQ$1wz z5C7(4?BW2q>72YK^p&g6{7QH~UC@jm+@T#V_5R`kKHw#zxS1egxKn5iyUry62D(-H z4D1qsa{wiK51v?1KrxVn9VR)*@BT;_XV?uck_Eg78?U2XJ^MX{PInb8n<;cidEITPNiG;*KEROXK^8q(#8RsFuZPm7Qp|> z_uHH4v3Jn+_)xXu=-VXCC~IYte0P3IhUSXAR%DmmUzd!lBi*F-TY`3^t^?Uhe~)+S zN^|$zKH(lH24f;(SJj2}NZf{`iGd?$T?4l`HiO)T-(2PObClFQUWBN;KE;4qjFaTB zySc^XgMtS0C|?uJkN2p3*T4O^#FN*?!-6M0uI&CVmrL^h&@3U;K*Jd@)O~~ZypPtoEyj78^E4B%l?EK5+kla|;LFI$-l*XdYxlKJ_n>Ci`TMTZ5GMY= zTmP-&l-dy}zJIkxUNq*TpfE8ht-!ll3{_E>81OVKXlJr;Z>~$XTli9s`*Gf^f!VDL zy@nd`_C$o9bHO+YjjkgX5QQZVC!zR4xxf`aT&0x%P>x`z= z_m)*VwJN#MCm1?0q`MoY>}_v26U#)@-Hr8mFY~Wv+2u8UYspX>h9Ek5G(DEx?+j+{ zpoLP-OKr7MFU~Hp6RwA51Jx#4om&E;R+B<}(N4@ufm)J&8pZ7~s@%9<@-7kjL5=GD zqFx2VhJ}Sy-s)DaasqkAhR@-xWf;cW#U#aDp`QqA|(C zyL>X&ulp5eZm%fi3><7U)C}A0dS%c-XSE%Njja9ku_r|$LMRI~Y`sg3=IqyUvT;qq ze`nE_uqx3OY${6HKxVC76>ZH~wR1C;fX27|YBnnws#& zZ=8N`xOJz{S!oxyA0y5|v5l2>ADXL{5$iMM<&BFxN`L%2#nK~?xxbvO=!DHJx3S+c z=0gn4P#?6Ja`V>dtGcea`nah9OSa}%dHQlY>Gx@j*j~}hF!&>lCDjNLRsmvV!)%i{ zy^)X2P|9dptN-!mg)fHY-1|w?#n@kh8bOKal?YYA?8+C@5}urdn7|H@$!1SnUO6P$ zq6+BCFMJ5R%#~h=tFtkE-V+zsR+oRy$2Q>GwZ)ukBNgi?9xdumqLqrFnfumgScuhf ze&fBKzmo2rvoY}6AY8U;Y*oQ3&CvZ?1Cf0FXT^YJ^0XG__HcAxfri?zNL`(X8%{~DvEN!%Q}1T!1~J5 z>^*JmkCnr1DHq0b8nNa21I-yWPs1XW^EzSPxfG{vqrO6%m&17QOwo$q%5@q74Jm_- zF_cmsjG)$NNBCOzx%I1ZY;-FPlWLuFVflcHoeNS6*JQSg!dBj%aARg>N@*Ej4>e!{ zEED0tWX*StpB2N+%fA^hSOeclifd?T33G|%^MIWOr5WxOr9EY)Jsqup?nWv=PH@f^ z`}H?Ok9m88&wmGR9u%Kx1GeE4XF!HlctxC5WykpNjUqSiBPWt1h5dSG@3=0>=1FIq zv-2Id_|oe#=?3z*_FqNpdiY{osLHCnSetxXxi%wYRkO$Tc2d5bwTgYsKx#z=j9uee z4igE}y#QkN6i%Dh%nTkpQHb?(5TV+<@OkRy)&em$OIey~lHLVjVA@=J;_Sa^X$&^Mq zEpEO^Fr#Vq(mZ&P`mFcx@Ef817^klfA)z!QrAVpXP#rjOFw(A zs3hR&Q#~)i9TLGy@q#mi8qQCh8a&sVHkI1$LzFp^ap)ECd!nqC5Pbs-!%CnYEw%5T1iNM}bkEg25 zOU)TJ#>(iIB!;FFET4BA>Tgb9D*Ro+by`Y0gKAlFbcJqG{7_IX z^-P6GPx0`Gs@H@saxHt)(wGO|r#Pg{6jk&ZzYMr<-oN0W>gJf1y z$h2pLI#Nv(bN-tMOeZGX3tUH$V2A5WkLw03CH45E^76f#onT!qhqHn445H(8b3wr# zKoVFYk2mK}k15S&{ndxK7e2T5najBrD;PTOt8aiiKerC=tgOab-Qm9{-D5(UIv85j z_p&i-$7F>YV@|gnbo9C0fYYrL%o(=6+G1tm5jxST`UFG-i*2`S;1-$d^n#FI^KS^q zJje(+s~Wbv&k;zEKR=qMJZ`dy_>M0IrA>Kdoshelvdi^r;`&`!2W=5E?<}32)c%Nj zJ4$AjEYTa{(?x0kY2saH?*p@{!o#`n{sloLj&pc3JLJQ|*TGlVN2Y)hGN*Qhd|1N$ z*HK=1+;0j}x8{OFj0?+iM^z>+61dH!9%p>4itKS89`+wta+2Yh@w8#~{+;VE$qVGK z0-JRBQsxG`T3W?JsOyS({AQTo>7=OFYrJQ=QfS`H(>jmIR7IS|MQwh zWFh!N99-z(k7S!SV_x}*^YuOx_oqiXi){eybKTbPT(H^&Cz6!Zq>~MsXPcfiu0r*Q zXWe+GQbHirZ_l}%PIxwD+B=AZxDblLQA4l}`ZM;~H8HQ5T}n-zJi!A;``*_m%cJLQ z5pLh+2jEHzpwy-aZA4+$#(6CV(A#FW!{(&5oa0>L(YEGSAum(NRlfQCxTYGRqV0Si z`s7Gu@Y$MQE6&XXbZwckwVK(^TU;$kn?(lR)JJcf3~+Z5!@+4l``o`YZ0z_I#txxN zR+(+`n@Rex+WT2z?KDd2+q0ux4CY%00njR zF%8Zw>FKmA(^&|U?`9bJ?k#!O!!8s25f+dW@L2QAF~JWXEBK=DPpkM}cZ(2l4e_}D z#nxL!MHRSvqXvQqQqn2i-7V7H-QC@-bazNMNDLq;3?bd!F(5hg&>eUCKlh%q-gDp2 zd)C_X>DkZo%RhDoEV-H~3R|U=_1@oIbJE$Zg`1@Pve-Tf+B=-HO?kURN?%3qz|!Gk zVcQbS+X?1LL^+_ruUd!0l=6u4$F_peP~Wx*tRBQEF%IkYVC;q9t8V?-9q<<<+&@5I|VoCVM-y z-Zn}yfc3#!o1GDe5$bP!K*g9Gx1A}C4N~LBAezo?JfXmP*x`fb9n9H#^Uc{oa~+e9 zdQdxE{Fk4#Z}2|2H+{7gwI>4HP^)0icwIp4(a!Ja>ZAP7@UBFwn2iaeoZr8-spcR(slK-AFh_ID=h(9I$2w%PkYX&-=An}; zKgG6b<(FsH2hfUT7#F1;TFI0_Q85u&zN}J;x@*y#0xNT*wQKm2Xs6v6?kZ)_iP`W>tQ2lJHjiV4!iJfTVz?!Cj4 z`ncB!cAjC!#m9n{ATr-vHp|upj2Sf>c_p~|F#`o!O5^WfAEl>FCkG)CwwH3Ipsnr8 zl&4^SAg1x|`KEKg@UhPm@U!JJ$bJs-5`KuJzI}YTwGP;|6dD2gu<8ZQw$V~^oYR0P}Te;$xm9K1h9pqcAgLY!ClmT-Y1`x62D1*gOCKjPyIw9z&xWptZqGBftLjl2N?3poKs#{5wz5J5Xge zaW6AwC{{ts)hs@usrUgz310gbe>beMTT(ZyBxSy1@Zia%J=L@X466=c$T6I-9jc=- z^n~oH(sM9)G&6BK()Cbncb729+^$fdGRl4!f489*32b{31k)U(V;BkwqW2eFU9ZKJ zl`!8a3O{kAGaE|Ky63qFAq$UAyKX=KsztCrR2MN4zNEQhjaXny5_Kg%jyThMT z2zq|Z?!NobCf@cMhW~o}-D{oDVlfdAa>D098^M_lr*;4L6SM(u_I8tdRzCH{{mIQg z5S6+n_5M&+vuXY?{&DMQz5UhvV&WI za17OXmKuh3z=2XK-LCfAQ}Ef_{UIy0N#4CsnEUwg6(+>|sbxptkn{4ur7{ovu%9gn zJHtp)xll{Qle7pic#|xXh~Hz%4~#>6_?EAcfR5WkiT|`3GT!}5h?@I>v!w;7>Z+@y z+xKu3*&lUmX8xc8+AGVPeV^Pipi@xCTTf@KW=5#!yDCU<)_eWqaD_drMO2z6ZfvP4 zKB`e;lAlh~IH#nR6!JUW%#3PvWuA9eeXN+~2R7D49m;~K;GEa7U#G4YtlVBoLqYCq z8kC|xUwjI7vxx67HR5Xp!f|PE-FQ|5O7o0j5-1V93#CmLNkMTwfRlRQ+x+VGRO$7{Jgk7*(P2YL<1Ixr{~=8X_SHn zq|W)`SW~=UdAo})1=XwyPy=QM`|e-zrR>+RW7oso!_|&Ez5z)DwLVOkU?hqOM@4E} zdeH1a?&H^Q*k7ze+2B(trAoVI-5l}JJJozv#P;I05B-z^dS0ZHXFE~~R?NHkP#}W? zv72qvBS*W^2pcU#VLyFc(i^`@nWN~QM?bg1=WY2#T~JDNvocUHhSZ_wLSs+$Ry=$+ z452Ek=}mDO8$Y$@Ek?GiusZ!`wWbSau2ZlyzopH!oYh=-C%L=Ao3Ou=%RRImZT zWMOhP2T_~lL#)`^N2Ut!_#7L~OF6mRhdDYKM1HVujHIRfvTJi*lifx4xl1+Rcp&Z= za9^-Z$AJsg@Nbi?^;q?Tji99}FAVT-BS4=&qNQ|RW$`;Pi1 zFMA>0%A81zv>dYNN2?xdX2SZ3V)LZ}gg3gd$2RVDS+A+fF++6!1oJgN!{eqkLHzdO z)Vx%nr}G#@N;Os7@8k!h$z-{>xP1NB*D0;Ppx5FON+G@0S0_#e6Cw*UxJUNBM-JZp zM5y?(PN?`yf3csZs_CetQD)|l9wTeD(!(FEj-rfGLPUlyj+U20gGomb6X|mib<%AK z#;U|rdW%vXuk7&QN~ohXbRsu&n|6|)2ctJwsi>EA6id5sk-&-xRd$W1y2Vq6Jy*A4 zSyfibKwXAwPgh?=E+|Adq+z8{1G%as9UsG9iH@-Ha2Nvf9UmD+Z64>#8aJ{8^#yht zeGZn}qs(2lbxS&1Y%(rGDEU~J57~C${=I9c+2xTV9*o=4+FDB|+oNK6SPfn*HGF<6 z^+`UHrk_39HDAmaWU}oeHxg519si^pUsh%*(7|rX%{+(^t`X!J)IO2n3q$lU^{NvX z=eOG_w~!)HxNnjqBPD4SD+DqCyM;CgsTp-Z{B7SVvvV8KH?m@WdNFG> zV`{^GnWXYCsgV>&qFzyp$pFx?e9d;4Sy zZCg6-6*b((U}Gfmbm?ujen_#4XZaIi?wfe=Nf8f2i#lej&UA02`40-;*>bBarR##8 zJ^Gx=Hc#U={x)qM^{T3WfU!+e6jY;l-|di`QUl&Dz&a3OPq%SK?QwS*FVv&>qL{Z6 z`1lxqPM+5?K3<#ew!V9^l;;9Fg_j%{NZCmdU>;&W6)I~I9R85a$m;VobBsnmu0oaR zkT*T!?6=%)E#EHQbxUDC?hb96@xiCnuPHbzC~Xp&h=ij~TaN6)d<8;Yq1)w($pY8t zvree2`#tM~sNd?b)Ojt01kLn{WL&?0S+Hf$Z{}@f7?esIh3W#)Ff%)*%ifS_&6;q?NZxsOJIy z3^9%sh+x9(NB&j)2Tb-TD2*XjM#CzvWoB{L}oKZx^`Q{2gE-KJfc&_?-q?Q#Mu4>vVGUt-PxIBC_XrjR5L8jv1^R9=ydFKs8f$s(^m`pq-@BHX%1i< zPp4*Br|P4ny!7?Q$Vs3;9H7pe@&atiJ68*aWeh-4&*pNY?Y8&xH}iXA%crnYf>Oz~ zn)kJH4;NB+9s2Th>#jY{PE-snNB3sw~YC|5$8{Z`<3#oYcflD@36GmCFGeD5bWfi3-Q zLAu$|^~akg8wX;|RbEg=D$0slPPYvPbePxromIXIF?Ta&W09r~ytcfs{8x;MpAUPT zj;R+mYvLCgE9vUx=Z`*z_ESFmJBbQU5Kx)9d5*j!xEh^V_#tjE?cj8?QdLB}(oV|D zgCJyC{_8oFNK1|dc!XnSnPc^&iAa`o8Dt+_is*{cz-m^G_@=Q@%g>aJjxbYss;uf$ z9Q5r;%*5^K3F1QY?j?J8grQ74ER^;UErLN;a?O{>e-$5knwliSM2u*=5hf!Duc1iD*uE?sw7kySoAtz=e zhLo%FMRUL1WzfF4Zw2kQP|th9t@K@53^nmn!R>3~rC`_`^6(P@4nCqauO~o?<{iAh zgxCmT(IJo27C@sfj{s#a$G$%qU061Xc|ApwnnWQjRp4Q~_$I#Fw^0h$K)t5B#l1uN zJrxBpKeN{Qbr85wgFKB}IPuzVuyA21*VI}i%Rd=oiD=biXOt`SL%bcm;k&1$dwj*n zqz{zFlkv;=Q8KLb`84?zg5%8-?K%2(U?n-8#H9!LA1n%SQ3gEXS31lpvR17c+p#qZHNu){%PNEklae2a>5h}mfkq|%hP9y=qW-k zoF^PT0y7V)d}k^8#!cv_wCjLVnZ@tQ7GG7KHhQ^wwF*Y&OZ#I4G^UC;r*Smc>jJaJwg%vOe?As!Wv;_jP^chu zN1vD82{MTMIMA?hVWJHatt-QrW-xbj4Cnax=5lBV=tb)SR8Hl#0N@9Wq|PvT{L;o;flAy5*56g62I?XO@k8x z-q(=yl#47D)u1_Fs*uVe_vrmvSoh z35H+>;HMG<#X|`GzS)L`C+v=mLV9`I+@q-OKdVMpQm9(3g)5I!tExzv80mS3vkopw zP^m4ePx96V zQQTHvb7hj>7}@U;P)1Ts$dU7EQmhsIoXCj_=M_D!J2bJ;6~CX8%n4mxk(*gIYJdka zJX{3j87!>Ma81!l(ESqHRuGajetekN+r%@-7L`u|lD(r?;Scjc zH#mJNZ>p6U_)T9osIY~ESVDtY2)8$zCRQkfA9l45O4&RW2G0o_t*4AFnkl-^1;UOW z6l7fJrJ6_H2fabGk1zaE{(&&1(U@;xRECCL@pKHoz(&08Do4>+DbL))y%`4j!m&vd z+fOq5H~q>ri+!ycEF~?0RwDIDJnfsth_6X|fUWr6HCXJX)Q4SU8Hr);rSbXqdl|TO zfjk*R8QQiBuM*aN2HhA56%DdWA3>idq4t103RW#$Q~lUR6w4mhRvpcpK!@ey$oIir z-*R$*d{h!;o28#sv)v=sU!#A)e%*Pe zi0*^tzTIhgNycY|Kb2;G?0iGCUs>esdgJAspM|idb99T#oo57~5uO2G&bO(hbMG;S zuHy6WbqP+g5sH|oN2w9y20U@&9oPiHfCRIk!eX)Yk8gBk>Fiv{(k+*zBscDf(3lg{ z`d&9a(ZX@(K@%{1xK^WF5LgI&KE8+sYU|C|@R$+6x{0~`fiGwU51ybCBSwhuwiLAN zhV+bg1YS;oNx4PH;?)|?H_;(;IC*V=1AZUh1ID1F9Y#izuJwqy;hY~wV|@=_vMfSd zs>S_UShU<=kLov`Oo*7__j}s&E_aM%CipVnvXUK|ufP&H=qvg^m+`lMu9Cp(iRYyv zCiqtX?<+h4$cYG5jmU)?#P)ds)%Gcpz|E>_d@Jt%+CKl0Gg)qiAp*R-;&O2_Kp!-< zTI$F1?zf!`ONsGM@Y=UQJK((W=$?_gES2Sx=4g$%wT88)VK;+>m4<%r{acg$yTbGK zdWMm+-3n!TL)sFS{@O-PF+BMs+&Be=40-5ki*I-ZUHM@qb)H|;@iFLj({4Nb4Z3)) zfOu)mv~2Fyq`<(`wy^5xGqP?+|L9r~v*yqvR>KUBBf6vLv@5xF^i)-vPEjl!G$nHBVf$9*@pKxUoTS)+GP~1g zW^PngzL2x0DwW}tv3Q<=l&9@9TaLBunq@-4LMijSXO6vxhW21&j(ez@^Bzd9fXaBA2`{BB*21b?*Q=&4u0~LqbAR8q=*#Z z9s-RcEGVaZyP#*oqJMaQ(=bmZH4!O75S;WmYek%|p@+WtXE(QJ@^Y6OemQsVq{sGN zt?681F%Iqc7+ykO*$&rA-m!ozTIK78-=8(Nl!*z)Uf~9`e1wTAG)71wM&Jyq{1^+Z zP)Wg3n`iE&r388SKi@9DM#KJk5$56O?N$JufLm6J_X z)WM$dWkd{JEeS4}SMYZBu<1k8%!j4$DEx`3p@^b-TI<3V4{NF1ov}0qulOWl86PV= zBeQswYL$CkwUbwefFIx+4VR-~! z?tX340c2%hRNiw|-p*ED$JauIA8IeLTSWkb_ekdn^Ez2=AJ6ffsOx+|kBhsZ*^Pr7 zuV^i-(i=>+maY>Ne2V=UVgS0eV%z0)MK&L5!zg9y;+?x3d8`xmK#!h!jaisQFs#0q z%>Vn0^=gW-;%Q+}=+ZigS}1PGmvH~1cLIO^QdAvJ(GMb4B8Z8PNAmtH@x>CUIy#(! zGJFYU!W&{;#f&u5sD09zGyVQD$J)jV%-|gA{g2g27W^V^@nX+OlR|~T8wt-|Uz9Ov zN|G*hU9djQp$WHKb@~wy`ow9Bh0?##~RbWAeRQj&4zT=G+58u-?d6l{AW#k z`5&6OxAK0b`sGppU#59=yxVYe8fjP}3pzUcm*t!rT&HMsmZx6Xa|*%(iocD*jz6aLzraJ^A-S`QY$t8w$yyzMZM}u#-ODe=iqW0)<{Dk(4gJV&)6rvHhc%Ocgm%R;8~w*8H}!)9xA|{>scVa2HN3^C-o$T;zS_1Hed^_KxG)SM{D^}``YknUhtW$;C<)b})}yI1i& zq=jgys>{Sh*5cY+w3Bb-e_R*3>$Y$~@2fWGm|#`rS$j_|UkWFwtNA4hv8^wUze*r# zlWpUf^(R`S_5L;|^LP;O940@$+%SiW+NqzVQ!Y<}*tg~tc>*}}Di)S`if|SQt z_jf5G-}mFzF_2HVPIKrZ+f1(M0ydDeoud=`WGHV0!$HI~Cey*a1xv8N#M0VQi7;WI zn^*3N2+_F2r{~VVYy9;N=PZy&zoqaUMc(ZxXH&TLZ<2RGuRfCDl?NafT}wgKKeWhi zIG93&#wWh<6_XZWFF2-_W~1m)-r|dirtwz`zI$SfFHXN590*LRjJH1G=IA+uTb}zFW$v1^`OwT zaDxAK{m*AEf-~YnnTorJ{adc&dhQ?D9bI;QS#`J@*&lDxeom?%Wh=f9X2|DQ)7_o% zk6I$#O4h`)6C8JfiP{|ti3WOoOh(D%qr&hG+{Ag=_>TV~Zy(rPt&kid5;*w$tPdaj zdU`fJTBa6nuaBDu&YIx6Ib{o9?U0L$6?t1(HDkLU0&O51Kl9*4koRkM!M>IclQhz3 zQ&xhn@~f`TI=)mgCa{p4c^RrK;mZf*E7QGWrb#7FDu|_?iZ43I%g^%%Fq{^L8b9wh z;A3^}ef&bSf0jHLw+W2lzw|TP^0)U3o#Q}ML#(8J43>f4nfUXW@bkc9ui|5G5r8P3 z$DRGh1y~e#8Bgrax`y-E@AxPo=lhSn{y%X{)XxFI87k_W17+kESikY9>VsOVyKP_S zqaRXVhKEOY4l6p)JG7)`p1acxO7y=|A=Y3K7lWUwSymZ)T8o?Hj&04`V$U6uRa#3` zn44P@K57`=7c}K=dDk@q&FU2+6jCRVohS~P^A~w41jKUA z8q=C9c_=Wdh^#OEA9nPI@0x$@fBO0O3D@;wY!@e;fdl6Lm#K-6oZQ{v+TC&MK1hA0 z^Tr=veZCiJLR@}c)nHJm1wlqi?m6f$;oabITtt7%s z!I+ltA--0LQ3i@G+zi<1>4w=TM0)D?; zYu?f(%JHzfwMvVnYnw1WlMIvg)$eYl%(mPGyuo(F?`a}7?IC4Y#gUUE`RP7My#?*> z5GQ6#oFcCMYU%jv7d3CowAvAvt7(}IRt)5HCGkxMr&B28rDr&ht@7PxBtk4#dBbV? zPOiE5`T6z0J{iY3i~>d={ma6v8dfAcCVADSCeOvSWm0)|ohj6iDYyPSzZ>kA0q(#* zo(Ao*!ip`fPW?H-pM;){Oa-16&FeU<<9fN{zki((o#kw>OCENYX%XLdP-!90&I{)G z!mW7{M7_uivv13fv8)B%URqY4my>vBQ&iR-d_0Nm=<_<4kBdX~eu^Dg_1dX%zL>8k zBr!DMQ!lm_XU<4AcWrAfww}rkc0%Qa);5(gFdFbyX;po*60i7U6|5kj4xA-e)zvub zlIh1vH+MS2lV2nw(##(s2Gm_Eg0nEGCDqJBms=##P_^R;Ybt3k_JwBZv*gANEcsf+ zS^4`l*sB<`zGK0Oh2EPv8!|rlcF<~AWeJY!4sV9EX0qX}ZyfEJH+cy40;3nXiI{q< z8Qr`78MxF>Y7OYC8w_wf_nF0*w0b`s^mG~hfq*w01$AuoAjp{;PldS&9u&y4GL}{L zE9oO{HbE9&;Gl%kCWE8u3K9!Lvz31sZG2qM;wyEM|y-3Z&`)Cmn5r<4GB z`P-2H>J9%lh8?>*AJbpXm2r&8UryGaz%)XV0alCa>rj%%;K0+L(PteSfJy)g?85(h z<^q241UsJ%XTB^`)@|K%+8<6Xd^&A}txQ!_9dv6Ia5y6pknQH`H75m=Lpj5oR;H=@ zz9x;4I_o@_>JVcj`}1=fyw;9OAH;E3*;0_(E~82aSEVy_WC7A6&YbTtcX+YVX$5M& z2&$^b1|)Q{iz_PxLvSuURI3KxDP1**oCDjv?CGq+n*GH;Y5qQ!_Ypte=+ zUU>=ohsEtL{NCs2=a(P!ZU*h*h2m#!7SMqX28o7uQ=~-PZC5B(%RIu? z@Pel1d22vCiI<|e;*TW)!i9(5N91-x>!vG5q)FkXT?q2h`FV6~MN31R z>0y~UB7Mg*naRJ*b+qY_BC#8ECeG|+n!Q5a9U5!!4lgUK#fcp6nP&>H;@W>vBNwht zdlHGeHXa}CyLuLPITv5C!fInF$$DM|bv^nY?lT>?j;z1p;M5wc;<{Y9Sd~d2;999( zkeQqu1lb-o`W_CC(|5aiW)Ywiq)HJ1M`oE>hOq!6!5N;C2(Ft@anHR3b{{`lERN`Y zGf$<9Ocr}H?~F~ahmh%yacOri_@L-IJ5(kNy*0x4h9HeQIJRVHO$FyszqKIX%*wek+v{ zG_0hh%8JAc@xd&6vq%xz0vcPY`lS7~!a5Wf*ZSNz!jwv}^lL>9X%iL+Q@2_(dIWbO z4X$qnoskP%ZE^{nFcX~YfgOYrOc~k+#F~_e=%At^)04H1vE!9y9-c)@uAI+aG4nK_ z*Im5p6xPiAK2Zr!+Z-h8)-gf6YxH7E`)+6366~jO2So%ua{mt+`U9o>UiGNYT$>8$ zhTphjUqPaZ>;+e-KFE?6gt_U&|IKC)tI(LUdmpZnb6r{%U_#QjADJtk>k##IsK=Gqwmb|(LBH2VCuVTB}(iY1{xr%LR{a4Ub&z z+$51wn7(7@=;*w2PI=9A%u0BPjx{Zt<63EcvGL5&&$76T;x3u?tJV}fs%{g^yuK0m zvWt4#MSzfE9_yJ4>u`w&La@OI2!6X;@fTBN)vn%-I7RGe;hxy1G{`bdQx)-vj2JC) zq@F)liRgk}F@B2>v4ZVfIocApb+=&J;loo6%Cxl%HF!nN3^nrz`lmiUg>bLk@^A;7 z64tCgzPGd#ZObc|jcBd-AxAjwQ`gq}1EVCU;Ybfx>`Sz0aiIrR3HNO5MP*-+yzj@5 z3XF=cs+!pA0uLLwW;?#Qut#oz@Veg!%hWwB_`0dHTW_~dsT_Q;mpe-xEh`c0r4y1Q z2Wr@|R*UEqYv(vF(Z$K-`TQRUYvwyf0z^H3Gb-ET35<_tD3+yMs9J3Kk1H6M+TPH5 zT;~>aUQk%K_UVMUQJm*GRS&=xEIMM38ygg50ngigy9L^JccuenhR&6SiGT-BCT*$7 zkxnFDD2w#4#xpC`7T%CilGj`XOU##n=Hs;f{L<&tbRxRH!*O^J7eYg^~82rE4 zh-visXf-YaR+qcUxNNKmIk3oIg`U{Rch(M(NE)s-}VW zdWm3j+wEqw-&wTbc7|oH_pErN*cw;oZFJ{gECKg_@3HVe2891?-~SCk$oIFdtFYa@ zz>ph1+Yec4>*X9uo5cDKuCTwTxE_8e#cA_f_NP_n8)00FvN!K2@tVgfG2fB}*@l7* z@wlqiWhSD@n1-7)E){-p+nW!y&Vv1%oA@pD>pG<1Pb_piC*EaG^zl!4_kl;mr6;q1 zcA!4s`od$bAm6%FC(OvTbqOpJr2bcD7S?iSs)>oz()$pe4#<1n%I< z?^tyA`vo6Rg-2$Wg^oB9cQnRznSVs|BvqZMxTEuSH; z@dS>QP}h)n6nA7KuJq$ZU`|?(ua4uS?gL{%9UnJk>4&+fsyIV+W zyODj}s=wx4>u_v$x8otMv2X)tIZ`8sO1tBomS71Fq8$8{+vjeF(Eh%&*}4++SmI=+ zmgHpdkkXe>E@<4$XZ~YUx>2&yW2u?mAp}zw`zbYgrCsv7YqhircEhxp_W1ZUQ#zBr z1u$J1MyB{8M#nk#t79jVs4@+5S&q!*yIs+!N0!-gut#?(Up+AxPs$F}nX%qV*%O(o zt1e0SDSXs6shwxdNClo>46Ce+-PB&>Jg6gi7Y?cfhaRz3KI2unGuyCX;>UlhC|}eNoY6<@LWNBZ+^R9P$4|r{7~qhb(b~?I$a&U zhp6o|y|H~a=p@|iZkO;9%W7rM($6q_qO-BLYJG9LD|Bf?V42%j5+A(Y2j*6Ke!pY+ zMgI5An9cNdp#bP|arVmI_t&1&!)~@z@(E$-eyR&YpJlt2C9Po1wP@D5pK3E=v?T#W zaj0cf!LU+>`C9Yn1Vdl1@*J_FXm|5xK$X>5_cZ3-IoXb8pOMuLzhoeq>En|@K0z3X z0`28p*HaX^3kNbGI+*6|vC?%vp*PGU?YFGJ^AW(4)dE#H+L>&t_`o?9FYBAWxVA(# z_Vq)3JvQI8t5;Ze%e>4CiQTXEcG=vUOvLi_1@n`mv}bG|tNRQ&ZAaGV9q{E--dWxK@i)mm=DeIT=kT+6LsE^G=$r zTW@seJv=>l`S^;lt(VUb?odR^*f9>y<=;Xgzg?g>Pr_gu)Op4N=RZfM4*kv}+0itBH^d46k9ZoqKOQxt*qx%WwBJ;h{3&D@7neU}r=wF}bCG*Q1<^`#9CG~I z_S18%(m;j`>JFMK(VQr#lH>2}3{T8jK$=VJDoOm?_O^sY!qwUn*G#H}hnEOQ_47Jz zT=@A9i*4~Z)*%;e-r>nJx$fE}`^D6e@z^6yZ#qYMtbx&Bc3D4>l!?kZ8jwA6N5Yc% zTthhFn6M(+^cX$}As)7%sR{l;hH^$Xn6XB>_IVR0$mBsFI$KfJlC{M9Itx?o6^yWi zh`p|vw~m{q($L&w;1U8CfBeI(@^@*j#+%6R`u2b-{TP4+(soa*$AE=(ro3qA0w`|_%8t9P4OZjhag#%>qxUrWrWg0oYQ zK-)t`8AkeCY7s=!sQ~c!IYdLlEtM2%eo)?>tdy*D zi);t5hAf~sydSJj4P;ZWkV(?N>S!Ap(|siGDj%-pP3YH>EofI(z(*%~7PhwOBAwyV z7-)!yJ!9f?GR7Qc<;IgEiwEH;mgvSBDRLHAGv3nL^E4%kcos(vsC(uhC7hG-S>HM} zQxMXEMo2}STr|~AOOqB>QbC45%%QK`b;lp{H0E`r!& z_#*AZqZ@K|3n6talG8Yk6kB4xq1kcjz!T`EZwsLqOn`%2zw2R#$c&8~ZSN*KiZjLA zNYd-vZ4gY%n704Trji0-+tM>|Mx}`4PL9w;`2k&&mV99*N6zZMEipyp4CgZ-9TjfZ zr5eM;RIR5BZ~JNLLOu(W#q~BIE;7JN*)(s>H@oNYiCdjVrbAZyF2^_r>Yg3K+q4Xq zeZz6qDWZa>{z?70$#Q#^S$GmINd5hIaGk6j!|64dfN}eubVjFYX~P?bomH>QOG{-y z1L)!5xqlNpxhmn#Z|Qw}&H4g#ihmQnJCV0A7th6yL44o`r;Prm_Qc?io)KE@;=dNvPE4iYtBh)ExyM(;BnT@(o z-NXZR;_3BGd0+Uo{dXIbP-geCju*_IUov4}upD&bRWiTa?hyFX-v=NRkjk=@ z5Uk%(eHYU#zQ-+h+#EsOfUrkIZMtr^TR!RjN*|B8%D}mO%EH#I$y>oT|J~Tvwei|q zdm46Ae&~}Jb_6C0FU{3MCY%#8r(bt0)=oDcz>Bv%I1PBgSPP8PUXskxCtUw}Nt<-E zkzG`+Sbl76HS!A7EUFEGgOwVoq$rdh!)ZrLd8Of*~89>jJgsB)Oa|oAXoacI|9y<7O_;3-) z+m5!pO4i)ARHWXGd#uG-0(&K~5vK!ZgwdpQA>iKJdm+B5(u5&hi&#uETT80Hu4&$e zlq; z{6=v6hyXKSd6o%Z8~LYG*V%2t^s3C5ccE0Q3BBskM`Zq|DE=}|Nz;+y_vN*b*cko_ z!Uk#0?jHnG8hf-AjfvW2rbYH!!bYZtVPS&4DcYVGXIxC|O7 z5lnDhLAua996M>dvbVp7eERA^G_0F;qFk$q^}4FHkFh8{UTbM=+?hIhsnf@LI;Ey^M);}c)4g|h02 zpO3UHeB^VS%ONRyW6+a;hQjdO&891mQ=-jJts#HzQ=x= zLX^}?&q^~#5|d&%0g4!$TaZ^`RxUy>4kT$@Bu$x(n;9r;V9C&4Hz6D$KU`j5g3=eb z^g0uws!|J^;`Xb|I2eJH;`$L+NxL&1oRid1nsc>8OHx`s>96=T&I$VoCwj9%S4O?2 zVoqcGniv#wO)rzF%9?p0bM*83TuiOR$9}I<<|(KKivjiA*%YfMY<2TGXC<33M+wvQ zluK%=5DoDuZRcWI=AIx|iDcNW!NJUOj^c_dG-*yqA5E3NG2N0hC;U4Z!!15cwd(TM zJ)+Wzm-{<=N8?rd-W>GeqmuV6M~Fk`yp+h~f3az2MxK}uO0;svcG@%+ue`M9{nu)Q zsO_V`QeFW$ngwx5N`}6MoV4drVwQVw4H!}~ayA*}UwOZq?EW&B8kp5#guk%zy@0fP z`1*}AlOHsm7e^*oPRbH#Z)T!d&oMw-C4aBVlsWL<%-*MxM00}I^m$OB>AHReOTVX_ zgb}A#=|50zxboLZqvhIWBklL-_Pq85^Xs+v{e#ba+7oU&5^URhdTfwR2sYM4=MR6i z*AL_ii`fzpZ(D5WK2SfBtUtb9f4t!4AsXA?38T>7{YGCh*`MDiewZZ9y0HsUwed_p zJHl2pd|_F*7V14Yb#55Sd}vNQRiYV^mSSy1k5l6p^y#a-88F_ZX>}OjW!woIheq^#m@TS=9PUfeSM~#2l+FEJlA~#h=!PQt zyQbW~`mf*oQ!Jngl~u&mUZ?b$+1&_97YfJZJma?9IbVC^_U?r&FOL&zKY{fhZhku4 zwL3m5T>bT5KY11LAIZYaf0V6v1wba>`C;etAt4D#C-h-$Wo1h<@Je(4<^gus0d$ms z2i*Al*UKgYwg}dZ0t1Dkk`hrF{)N8*dBH%KQU62GzlikCo15pgf6F#Tu%Yv#z%71)*q~#VijoJ*-4`$kFDXVc$B(4-1#oF83c)~7)5r!1H}c4UD_r) zNb2lhpXI-cuHazkeli_Af6c$be{^DCZ0j3NF`_hfrcIy{mik`0a7aZ|SaJUf!zV*T znw|ki&cd~{KB%y6c7C&+dV(WZpd_!1#@)aMwwYt8p^+_bF*|M?voF<7IyDPcLM@g6 zK2$&GeKI6y<#$309n8GWwk2p7wQx|{-RW&nH;JS#{Hla09ZR?=zfIv=PrHIZKmyq+ z7(9`q`yUXOFXvT)1)>gHcJT5%k+szfQR7iK(>pPIYtC84_efJyp2TsDf3{QYJ#KKf zp%{M2kSY%+Ahcby7=EiPBJX?g*s=)I`s@wLX6a!>8tC)>eB;QtKqt*yA?I1pRe`@! z#gc_(?!^b_JJQvxpw~m+@egf#kxV`gm*JWkDHW8&>q?pR<(^P#FSKiF(62z?Z z6ysO=dRrxyh&wo)aGqU4TtpP4#1Bas3h&~}be$vK3Z7ko=^fX4_%C~8sYf=-do_(0 zrA)$Z05`?vbcrAn?3j%Zg;gRPNI2+kBu97a2_~>pEi2aUpT9=w0WmhPZmg*lPzvz~ zZFRmZRX*Mb`=%BpBYN9NeX5rd{Man-Wt1!i=spDGS*3@pe0mRo1?!k#v~q#AGQJ*s zMOtTuTEg~c(;p&TpddA;o?@3cXin!IJH1|VO+6d%ReQ~Z>&1Xvu(R7t$JOVWB0L-z zqQN9Px7(tl7QAGf-a>uzl;-<1NDIi-=n8h3USV$)(>|E^c7(a)|6%K|g4zJMbzvBad!e|~7IzPB1&X`7 zySq~;QnWb5-QC^YH53a@aCiOE_w2LJ{^#H8hLA~SGP!uNtrIqLw#(4)~Wv@jq4aWS+ozy^i~deYiq3 z&w22{Zd6+p-*vzDc*RZ(`JA7YGv`(77qdpM;kUJ$sd#YnQioyNO%)uo%{3nLPmkvA zEVu?$1=G{Pn~=Ay*D+^~Up!p`psJVk5fT45{T3e11#bA|F(-$SEuQYXSIP@FPVKY$ zlULX4JuH8_F58T*x{1#eiWgHRmzIpEVXxwCyR(G+LKe0Wgfdq@*~QHFKTK^s+hlS^Q4;CJ^XulM6dO5cpih6P7Ck zl53E7sgt`R%GyNmtT5~MYi;}0RAk^Es~9;exFE`=j~cj1A@|9>skz1Z5i5oHoRt_| zN;W9mA?uKZd~BQUY0((~cA6AM?yt;b2^39ZIwB0Lzl;)kx;uW!4tQa^kCFJQ9ar4; zD7}|1*d|FEixn2{7<0WAUlhT?YAIl(4{=+WCBnY*ksSxO)Lj$6KPKQOF*;lgjw(ag zyEnflgyaX2=AYVShu-NWEeuS!0)CaHvi;H9K6p?eCiV+WNU%tqa(9$ZD02B`ozu4% z(+1Jj!gZmuRku*VwO6j90*NYY#O)R0J!SJ!antft7pz?TI%;sOyE=;O@}*ktl@lNn zA$90-LjgYCf_P_1P2ctSwyyg*`j36S{S`l0IdQ9gfii5ZtltHp5R$houLVaM8>s?K z#OF1Hscp;ZQida0*hC~Z7i)4E5(ovRIsQVorO4r6se4J{b8O&X=lD_c+aOrZppe@( zzkNhJvT1t6(I#@#R8%SXpb^_m>v@@Ra%_|4bfX71AI>cW{j`O~sw;t$F~4=<%rVU9 z6^S@ZGa%e;SoIFyLP0EhU>}j-k8NDh(#*0-!4sWST7DXrNukGFsmiyXk@0_~U+vD< zSOtk$k$559CWs{Of%w-q!HW*E0t>_1_kRLbA^x>i7qAlE){lpM?(R{SX@UEHQZ0Ut z_q&a^RY|>{_h_uA|2*VDIv|WW_8t`G4ycd+N0Hw_&2N3!x(9*g*sK3i&SG$+7l0xD zW3EAC&5rpcdOZvR z^rjM$rcbdlT`T05xw_3({c2rHe?w@NOZv698;F^pM@fe9*+B$wtReFm;Ce-NuLjly8{9dJCbRM!XcIJAnmMwm#4jc3sSqo&mHS&HVTrmSU zT&{f1yVU;zSrqqK;K0r{o!J=F4U3%aN3Ao;zuea`Cn~m9~!se&^;rF~~Ig+o@F}mwaU+mKVl6HN_+@b8B&3A`!9RB-+ZzqXgOEb%% zl)Oe}g}yym0B-ab_UYyf_4f!&K`^F>eP(ft3gei>w?D0I>>PHqNqIB;4%yCDg7Kae zrQ@j5jI!!DEtS(76&I5)D@=LmQzvD{>Wb<_@&>AnD+tj7>)T#RVoo+XY?)MnjGpR8{qoN>MK@RR0-)*Wur{zdfAO+6n{KbGdgQWvW^{3MA)1N5f(toy+{3eJ^CaknEVa5~`tv-qC4 z-2SS6vq7t$p9_O@^6<_o$f_MpA%T?S3}@-(tTSgIKS}4#FI}+e!?#xRRT2JXkO~hZ zul4sXTt_w0p^73Re8>TI=FO#6Hb=G8RnTKWQ5v0XVNTrNT*eepEAeDuA(!3KFPtJ{ z5bCZD#^%g6F88=`PL_CgXQx8SdmH1%{)z|taZrYXL%RG23mU?-b%BQ1KU=-)P4*PLNN4+WdC~L?Kr%t`|cyRL`^Vbj|(X zu;Kk@={)1}XVX7OeI#8x3gh0wv|%)1khmb#T`##T-a6u|EOE+D=I#@32dQ(NeT7@p z%V&P^3|?4qzNf9Tk#@zq*k`zIPGOh33IuZd6GPeC4>yuLe9trq%VrsJtZky@#;gEg zwNO)A2vyFKD2BjEv;2=dd&j-41Ky1{PFdsc%W%zxsv32rL_!->eTMJpbD*rr>x~tD z!3=xMhB?EZqG~NG+hvZ`-1#sJ=k3ujc?{>>w?0y^j$5sGm<@=$ocQeIlT>jGV%J@*L`aINO@tPeaX8`#03v*ZO@n5uisv-=k!iQ zx*J`|hWW=wd;}mYe+UIn8=a+oNcrgW27&O=lc#WXR-2*j*&amh@KKYfv&`q%ZI{YM{h^#lCR1(dAyX6|Zc%N#sA&g|^@5 zdM%0k`0Hin>YtO|b|mwEE_M%N8eLcj^8Z@o```D$kazc?Fkg5;IA_r?%aA9(@9KeH z$pxDE-(RkY3{wA#7V(%f)Ad(#kyBjDL~ZQykU(7P%OuJ&(Fc26=8`6%wxEkwgQT1C zY$nf9w`=d?*D5!=+;L?=8zBdOr@P4~XRyATz;K!Dz4)zrQ7JV6Dtv6)LflBu3a|NfL|GHXESKNkC9Kty zIr@s~kE}GK2b9yl4~q|}h5D?ntmGj->w*El+S+;{^08@?U0sn@;#f4KDiC+yAF&Qm zTCr!sE{D%PmF0gPVdP&h0M`x-1>g{tCF#1Vfn_9+Mn{tOfKN@dX(^oduiy4IP0oXb zjQytx5jw+tqYG@+#Mfm~0gH2LNXZsb$()Ta5rozjv^J7TT2FrpQ}Cmd!eQgampr zyZ5T4$;Z=@^o5ln?f`ppAV(z;|sR_bW9z*S>*H)5Epf#=3ywH1pEZueMnqz{-^HY}L&6xb_kAR+ zkrk}nBI3MsTb@R}z#bCPLS=BdBS?w+UOVqAVlItv7_`t=$*Y?)qZyaQU3-|`zgmh0IUwEn_k~VPNJd54kxwkr7#jdZDc)yaiNjX>BMLS5AdnSAUfIk)<+#}eY zpF#2&)36<(XCV|l4}Zt7_i1hi$DLa-s4IqE%f;_@3`fy>=5QNdBl(yu_eQSL`0q`v z@S#McL?%h(!5+08LATAz$M2k1Co1GpVB+yHJ!Via-yl$>Xx;+A^GGYNAS9t>Zdp3B zjDOstEv{#ip<=l+3m=vAyJ*E$M(Sf=xpqE%BTjIaD_2sD5;ie|UH}whi&$Dgm+ujo)}X-SA%!_uRO6 zoDRr>7zm*JA9d?hO2mES4WwuNe`ML+V6pVwxh_|&WJH~sFYFMl10lZ=AYRZID@O*zN)YF{g-zJYMQvMADVbmsc_{3Me_UpgB3b$3F1OPUT&Um0+s(1wH!^5%ph*59y|P_ z*7}96*bUXvdSvP2R5ePSKW4velruSKt50h^%m&>$9Ui^r&&S==g{r>_S1+X6?d)?` zHNUmK`gLa6qdPVFb(BoPVAXnB?>7!w4*m;rj%;LCsSFUCmUHUR~sgf0Usp93xZ{M>4$S@CPC zX;=TM{ZU&!TU$R`vX%IBxgHjdKdI|-N@ZhlR^j}j!nc%>{|obr1jj~dgu#m(4Vh7J zaQ)5ZzW&^$%Nj_i8g5#wP2Jl>{AZ>QdD6{Pv{*cA=%H<&XGcMWEg!Iv$rGCN*G7J) z5Q#Vd>ht}q{mwr z|D?W{aW$ZLdQ#sRka@Z;V9K8-Qg4l2uI;$}Lp7fqMYilyZ$w@#rsXXPc?r~Fo^R;R zPD>&gsuys@FleHZMk_$ z(C;Y86KfE)i9W-WxBh*tFVL&!h8d)9prce^gSR=(`aw(H_ZfLyR2K^= z)SIC*weGZ*n5u0v^qHt&F1T{_O*W)fl0$;voI~b>76&;@DP%fjip7?~7}JA{nqIbu zz)k@_Iv1Z0mpW~4E*EwPPIfp(iOF=xV55DGzQAs0z5OS0fcOF-K1hII!vhTu`f{(u zi~lIaho8y+{7D_DDg%_HIsEkLP6soiF&t5w9ults@qC6zE0W5eeJ0FRqPPG~s&1!v zY=)YAsv|Esgr7`o{^?B`>mk=06vxZ}u4uc5p;eE^?tpu_uFE@tv?3!e0;A9T6$|8Y zm2j21)%@7uqC^z*3}Y=U{xquQEW8M~4C&1Py-fBo5BzAsU}Nkt+71ueMH>nK4(4T{w*oxa`K5d|OJ!Z%xViwVcM7td2c?pl>?CB*nUBBbwvndWJua81dFqx?d z&9MQnLH0CQY@9tWjs=?K0cHP$`__iZ1Zp_GRmGaSe>ILm33cFQt+vZpjFn`fqn_BQXY5 z?f5||*d`3#+w2t9%O!&e3xzYpZN*H`#tv%i4#q0f?pray z;ue+#&ujO7C->^JGi>fA(vPm`nz6=hTVL^*EE^yeF_14J5|VNqjGj|+cb4H_3-zv0 zj@VPaB_u3`5kIs9Ly267rfor^BFq&s%hkl6qfe4^LO|Fq!QyUKnY1aY)5x)2;ew{F!toh_*1*!hAvr0~j<>vCBqeECVcjc#T!@ zGkTo0e8RwHqt0qdoLm5YjBBSqgf=|s)u~bJ6F}I6KhO*$A_rwn8O(IaNgfrAw^;4G zgG>rmV?5E7r5}$L3C6F#B!id!g4%mMn<5wCXokN_3Ky62K>@vnEzlO!YpsQM^g=QL z8yIx27e*>xcv}?M;_7b{9N%Z^v{+IJvY4RcM}S6`Y;Vheu6eS-Q+`sIM;7mB^S85x)y_ z8pI-V2EHgv4f1;`E@mGYZ8qKO1jny*NpOhjI`#;$;*EV@xjOtVd;Wn$k$(0RAInDo z$Iu;y#;oQu$JAy^F)*)1b0IEl43-8B6Md?zo>M2 z@+Pa&Y*g)sTpf6y9_UxB0szS{Bm=xO)J#p)sbL_AZqMM&j6y5AqqZS!q8yumARE1* zkl7ML2am9FjcP};fY@bV6hCSZP%Gn;?t5SCZ#RFVkUV_IYG~ZmhN!Y0w}z=alzXFd z+Trbt$oKTPSu2TiM@ho`Gd_`xv<4}8>O>qknW5fPtG}d#Rsx5r{xk?LPY?eZP1-Cc zS(9gQjP>(m4$1RdJG~e+E_D~973Hp0G^6x4>u(HW=7Ji%R=n&H6F+%rVkRtgvt}1W zP2ehR>iXE|2-!?tRbrQ%32ivF`OI}>s?NwRef4Yo-o*hMclRWChJSOmw$%f#-H>K! z(m+;v-B?6j$RU(Kqjg$(FAT=!kwrNJyd>yCKfgF8445!mh3|PEpain8cg2Sa94n@# zYnvpRQB2%W_)EB`khP9GUBcqMJolufpAcQG&Vd`bOm;EBA9G*#@}e!grEZ>%-F4CY z{IbBd4(BzxOf*;g10HW`#*URNcj!Ro!`GrYg94;O34P-z;_2Dfjo$@)v;)wrR7-SG z)tzLP1Gk&oy*4qfPF5{jX96ZADBS_2XYx;v-uMIjFLKnhkUGp){E(dx|KBC3{`ws7 zexB`yE6D2odbgJ*$MF~5ul;t z9lfbZ0TBoUl{LR&hI;h7z}L$xIe(tpzczLj`!1^21PNq=2TjKX{XUat;ybT9#cU`7 zV|ij8ou(w{{%uFUtTd3SeD(A7-9D72<@CH-hxI>qH_h~D%7~IEdDTk{f`J2H!y!n9 zAtIZgGIw0`d62)nRLBS&T3KrWJ3Q`F9lCsap6?ZukkJ+P*Sb9VPM@iLx8|3;&Q`dz zFsOVR+VsC$Uc`*U=SxIg8Na^o+r$}Tb%IR{V336SsR znf#IXM#E!9aj(_(g*Yf?DjC|lX=-WWa5qwS<9S?n`N>G)5%)8WLjIKZZ&__c_e6T< z1Xf0lXO;e>gtS#^&MTqOm&_2w)eO)05@sul)sQgpucPc0(?a40{wWnud@ z*<7&7v=xx5s^Q_1&-3*Q+IVz%P6e=1h@ZF1Xu~&%Tjne<@2ll1r z>ND|{MJ1n$%;D+M7CgdggE<>1E1??0I3ZkR`+d)&7h06h_ugBs$8wsz%mB5f5aXyR zb4_-!BN$z`2s>O$!t<W3&N8Iv=GJF2tkpcEktYPBdpVxyC4A-6q zwp>))L)q27S>r2Cyv#hjJdnJ`pGEv(-ak}PbQ^n?c>$TcbUI2Y{ze&UR)6d#2_~EV zOf;oNOX8zVX%o^fpj*p0dWT9;$vTH{1)}{12Yx(S*eGa}NQVSlKtd4xz59Cqb*r7j zvs;*viyxeVSb1+?W(R@#;i4vGr)doGAHf|u4k{+Wi_WmwNyq{Tp2I^-Q>Q8Kgr+UD z=2{*B{)o26$J~~ORru4t>}GDaM()CmqRLerjNlGyO3>Gl?W7+UUbYEp4LkTeG`!b2 zw{huL`b)=*MW5ZVst?EIjb^NWLept%?V_HWYAM!4<2d4GlqnT`&bGRer9cr`6hsrb zjc|DNGgi?#+ph6{EJc$Ps-XV6Acip&O|27VN0?D>R|afr>fHC5x&tP@z6w<7Ekj}j z!e(29a~-FQGHup{$d);8$@##yE<#j;W~HFh%Wnj--NG#|xdX7DNIWAZ*GvK>%Pq}K$7y6duNSiDtsa8pIPkIBG%0c8@yd_SG%(YE2OG|0);XrpL+ki+oEvRGGhy$u=@E= zQ9yi?l@NR`y@qHv?&xE@u&YVNau1lwXBX5C6SA4&##JLt?=KqnVj8-?Ep|A)dDq-Cril||yg`m3P}O| ztH?y8+|pvebQ1E`>E|gmZGIEPrx-Y$;bK{cA1iTkehzmh#hN`0(S+<)Hotf=cfVlm zT>d;~cpwl&)l9h=onJ>N?zry7ECy4yURE`NBO`?TpGlj~NnE-#ukwX12&zt|_-%Ky z8e)ciouR1{W{k~P4>UZ?%U)?L)A|SQyet88g_u%cI0n-QwSR&YU6xQ0S7A z(j+yz&@u1>VzJwnKPF|Y zhSSP*2s{+5L45XrG{|hKq$2_&Q67DLf#Zsv|G^9<+O5KRxT0o4BGf6%DC$ht)k@N%N?K<(2;#j2L?Gq>Wu9@QlT!9G`K?DsOtm@?2Q(Js_LvZEU|;6 zv#PNS>Kyg7fe6T0s7#dci}Ev+>{+ypV6?I<(wyQ<8j#T^Il<%cYRig1TnoLH4Qe8U zFBN)=Xu}pL6*%r?p@%g6rtpb*Rh?gMW@lN$5bm`yK*LEErYmt4qK*_B9CD}Gxih@{ zkNVM)Z&Rog_5WcA);y>CXdy>`Ih-IZq! z4<4m{voO^Iv4U*JaPDWn1az|<#TG=to$O3?0ffWkA6OMM83!|X_~%hbq~xD4N}pVa z!$sm?a72G85lBpbaMd>|gss&q6%i>yNmM75bVL5&TLn*gtM|6Efqr@xJgpZ7Ojha! z02_@~i%3lZ^}^m%M1^KJx?=E6Qss+J&QG0hR|I{0_+%wa8s;>QYo=o7^F-J2yyc7M zf%W&4o2Q{NnUPje_n&c&_qqT9qL`fK6qgj2;4N^bez0=#tq#jlfw5Oz%*CTin?4pB z_B+$RUg|3bYs1yXu3+<3be1>iH9`>{|11*}Uu$;UhwaBu1KR_5;4J$}Z`F~0;51i$5re(2J zd(%4Y2YCQN(b`*}j>9EU_a)IZFHtt4-)*_U9wejls(wmB`G2XoG}C_6p`D#iglyL3 zf0*$!Yf%OEU%nppmcEY85#?I`Y6r4^ybB#I7_02HAfigBmYab^O8D-RV2R+ur03v& z|I})ez`n>!LviZ$I@bI`M166Z#n-++%ttk$BN%hEV2HEdb>#87dzBv1WjqFAI!08h5=LR=Leg*xRf zXIZ*iK=765ENg){gL^^FHKJ<`y7u&ZFnDpCuuMV|*asY8V~h%uj*gMUfa*39`Wwxi z1%2dj9%CQ`-tNOoA7&1y3hqrdB4xUnWn1$wf>jX>6>A-W$)-aw!9E3iGP0m14Z^kT z-D+V*7rP`py>qNUz<-@em0T*vi$IS*LrW*colQF|Fyvy?WRk%!r|q^31bQP1A(Skh zqr87O!F)diff0jI*Abeb7%AQkV|R|9b;Hvo z@{#H@%WY5fEO^%`nW;df`1odNot2mxP1L)8JapZ2H(_(j7%d{>ALsb^QyK@gxjB&c zx52||YFaU;L4#$+i53uNRyZ{^b|IeahhDp>*mD)eX~!l}di>a7*+7pceX_m8&giW;15dUGxCmpC~2s8~;4 z(H`k%%|jx@mDqFx7Sb}zWJU%hly-iV)HOP7k@~BtM-vcb7lcQ zk!eIVM*UNTvn3-rT3Qf1209=3hdMY-U+8GH8lj$(2E$(r1FO?fBlx?&qwIM7eN4EX z%&|O%4J}5b!iw>oQ3Ie!tN< zsrWT8KJsnI|2iiTRWoA%!|f;ynksu=EOW~>T)NE+6WR^0jeAUZIxt|Khel&HKz;+z z>6sQ>3~<$XVC9r(&TlFNehZ~p))_}N#uPb))BixWBa9FoxrP@dB8bL~qC!VDz^J4b z=7{|e5O%I!=68^%8o2m;{@cb14(w7$GDV9!o-r8ANVOz+a+1Na4fQS8isiwHiL1IQ zc1`h8_1&&UtiBxtQG254__n{afDjViOb_O`dvAZdve~_r8ved3x?!%Uq{3^)n26}# zU|^4;{!~_H3vGY{uY~lY2BlV>DEs^QXS=7e=P#XHeFe;$??7sRW@HOhuT z`w(}4KsJqdfUmjC10)J{wc!1=Pl zNNOM>M8UeIb8-*kc*05820$5Y<~-j ztpbX9g6}5an3zEwhZ;;dY$2l#$%g;7$piRUC4h0L@g+20{-w*`=d>Vc-a39TYnOhC+34~z0Rjf;#pauUyu3Wk zI2IVzj{h68E35sg^*~4qXM;kaPR&SLXpudQ2K`Z4UuzNEOd}Uax;6R-2Xb@?6m?~i z?a$IqBKLEopYmmRq|f;#{yi}B6hPd((Z_MOGlx98V!ZF0 zf6WtJ1@OO7n{M>%IPt!?U>PbXG1?ZwXdZo|@DyH4X}o6TCC68UzEMXmbjJRYxQoB{ zk&c+4Cd62xQaYpJqej@x*9+~-sy;Wz&NQ;jvM`g{ZxM2Cey-hQL^O#$)ho7%D;LW= zGAi5vFn+A0ZhO=8BE81nja3sL8wq`EM9&m`8>g7959q<(^Wzu~ZIZ*atu2CGU4%?DqIW=|r-oPQ4L?+R2- zbU$sb6Y?>(G+%?$*3%dtr47rUK}!*WL-o-d1C!@wtRvM`eLa+S8%on5ia+_aOcga- zJ%MT;q9sswTYK?0rpi{9%)qGQg+*4#*eXMx#-opmb-UdMOr7^vlTn15e<*WUlxrAX z66lwBh$KTZEIc%iNunXGs|)yarzgcRv9LZ|V*bX}e?rk%iLwR=rO@_BgL zb5Kt+tORj%`h(e0>kafEv4fo!^sR|Iq+QWBl1Bvvu@bMf=d~L#K$t&`lj-PWLjYGh z%}@)ecd@^Xo%Q3FTQC)kcqAzq>uJuId;NSK0NmN*tUX zXTl)65|{Kb-UauG^f2LZl^ey|tu0VucHg77&EVNNrfj(C|3lZgPxL>g|Ljg}_W!=`|K~3eHZ^4g zB0^Z$fvrSVGyuRWK`$pb_G_lzB^hW1;7yPU=xcy2q5D8a^0Xi?p|+l=csR+WW|L60 zw3RvOee?BnYxhjD*pqcVEKZTkJl>N`+)GnitYvQ-+wC=%VppNOX4L+une z??9ZuLm}n&6i)W?7}IIax<8BA;Hphdym4G?+<9%M?(hMJUyC1-w0sC$)|Amw4|j=j z?k^eP54=|`GY!b|iEiYSH?!n4*6tVrJTuh4L=s2)NAEmVvm5wlw!Klwc%UNh-$6It z60n-bo*v>Yuf8E`y{Qm02K;u;`oTf7x7C}Zq8e?i)Tf1{VP3=j!wLS++2(RFoUqR2 zxQS&YzF9;$d;iy^lW8_cSGp#s%nxwTRG<0j-vf)ci9)$(f}BC4uCBymw5VAh5D zs%q3F1Znfc5iPcOKFpTXjO&LuMmuO{Z*Y4!ytb-VS!FrT<+Pw(=p49GNJt{S))zZmUO$v^ReMXHWjK*Shp&SLG1 z8eRcIH-F(q{`^upN3KADmEbuyF}2Lxl$Ca&Y4a#*lPeY~8#~7-VEG&~QYbz+TPHVH$Gvh;E3@)5~td0ERLs`m6qL1?{ z_ijwD_$wyq83_V(ovzUNXUBGQTH20&C)a?rf=E`!wJOK$>hh!pMd$a9GiO%qb6gzD z+-=7U6wcB9zoft28=3~YvJtCRb71UGp>?7~N{I)C8@n1!5ec+qh0{_13~9EERKP$a z9iR&dg#%}ckQId$nBqug?9^!Qp5^flEnjIWiN5{h^z>qFlIY~jmYq%laYi%JEx9JE zr)T}y=v=7km8@3UNX#M&=gZIzt+*1DV2nLpE0##ZXZE!9Q>6A)6?^^ltq`L2mn|Z) zi{Xv7;ph8UO*MMcLKrny3~73gP>YY63x z;L*gn6Vo1-6RR6+zb(hI2)trH-*M?Y#nJ*0ScQKm!z10^813}tAAfiJCOJ~j_Z>}b zvNNB6EEUh|{RDnkqgm^8V`uOZv;BxTpG5j*hUo20%AEA9h}$zL(f z%^DwO*#F>>g5Li-p?inF9T~iqXIox8R=?kNzg5)#PQ3Z+eO%crID2Z9@XyEP!^OV@ zLdeVx?da&(Qw<4=_tiP1Y6chlFQt&o#ZwCzi{oOOo!Sf=%kX1|dM8zz5k)ef2s5(< zP#G&#Qm`-t`E2@|HjRPl*gTFsC{Q0!do8qt(8Z5OfniciP`&H06K@0aOt0ua59V4F zc{@#>!$u8(hb-NGPzA9JuA8bI!DlU9p@`QXKWKD5%IbFCUY||1Vps{d?RBZguVOg^Vu3|hvQv^9qrCi?*Om7e zJOKR}ju@p`Iq{0)v_0ovSdwuTySFOYG4qU3== zXXhMzakAm)ewR+1d~TTTLbzn8I%Iic)l@9hH`)1);G-in)UmQ-YW7jOr$20krXdg~ zRYQ@>QNQmEyGqu`=VU!Q`l{zNN9K*|tLNW&xE42cOR83Ln~J}g)@PYqYdlr}fCG(9 zu9Ij{zRDrxUh#`2<-@ZjF~f1FR#mDT>bz{ipT8b-u;;2}Oi`?Q*=x7K{Sa-2^-gR0 zSkiY02yG+@q%kgljnY{Ez^otBheVhWKHoDwk!Gf8o8d;W>>J&J zekYjJHlh6?C)xp2bQY~4Q*9OlH4e8OX!rG2{^Q&86kfslJB7-{U{Bee6!Cog;188X z{D0#2*vk$Dudl_X6v)UIt!S7Joru$=JHhz!jt4klIZBU-qjzXk`aW6fnF3eFKXWWI zP@HDrTR0M{f$ISKN(h0!t=4}wR)%nyTz1 zup!4t2!wRM%8~roVbrM2nk5TK$&wxV6g5gNh*8;tBbPom#?FYCtKXfwFt? zLktHT6&2|+YB8p$svYLKHh6f(1lsF%^ucMERf-VlI>z2Pc2(C0*%N8mHfCi4^-%GM z^Zef0zEAcmu5jbx80lGMX0^AiD=VE|Cqrf4nE*miz;14mi1MQpdYI9lNQC#_obaA2 z5UnFU=bDgMie{@$&s~}Aiuq;jMBDL{0-=d%j`i}zjZ{MuP%9hWqlTn$#+dqfYUVZv zKIhe7#SFt)99{?p9h2A2q)S5awJAj+?|sOX_n_o`C}`_q(R_Y|1t9C&A*zHQTI*Ij^Gl4&p&ze6w!rX?HV?lCUautb4 zLy`@5Vn6x7zQMj#2)=u`iClVkPvd(Wxnl zc0LD}<5q6Zw^oPRPQR!*&c6@#nl-li`@N5`SG~&$7lv=IrhAAcZ`AIv%$yFj$MQ3( zL*cPsGz`TfROTf^%+ml)qo6ln!l{b-S#5i zeshPXF0kUy`n(g#`V3la&cFr1{0~ahe>0w*Of5;;e(?hMMZrAzQoYM-3Puc9{5@n1316eX^UZdp_YnGFelh$S z?gFsD1ec@_1vE9eL1V{FiBxAgE~jHE1bpy#Ar-O>3f!J0sVVsNN%~9g%rLRGEIkwf z3GqAv`jh0HG%IPV`|Xz3Wia7yDVqc_oz&gB(S1zw!!IOyOiwy_DA@>xp|m&5)I`^v zkI&R1^04{X38Sgy8h4mK`t!e()oo8@@>y1nu=OvNFR={$tQSPOcN%3^c#27c@*vH) z1fGf|mzTk@_1=3y)RoDUJEC#m&_{6ZHx zN*UX0$}_PZd2eZg(gu5;exm9A&)JD`)G$68P%Fw4th$$BzyRON zn_EFRHpa%p7NvA>;XBr5R9jX`bnaS4YJB-pQp6oaAex=WVQe$D$aDKhQGP8}wuj`_ zh#J_$T#Ya0W|`gKg^8Kr#^ENWqN6?@e`z{l%Vok#VARn2YjYqEYDtx03ca+P$D%mw zM8CE7ZP5l2!WMJvp>V>u_E?$re3GcJSp!Gnx_{pd1 zKqs9WWYYkUR8dRZix_m4YSo@w;M~?N@xc_1VAb&0=whVJs6U14P^hy%?hoWfHQQzM z6C++aE>R5^@axJ%5W*=*&0w)iaUzQO*nGD_GD}j|Jup4^R&<)32#Y8a=1zLX!+)OI zB@s?hx3M3ry`k_d&XyQIbNAt^APu{>S46ro2{kF#8a%R0943KARGpKudbUVuYaKxqG6m{MTvNz3bG^{MD zVo$`1at9RGaJ7^kqy30IvK|Zx(e>WjKW7C9M#53jfV1n7&!xMj9=%oCQ7a{)B>FsE z2V9@fpM5qq$KLpC9fS;3CssM(280(oMnY#we*@%~Hk>JfLVy+o9?G?(BWHIFQj&&u?BkPd26Fn1=VPY`J*J7)#% zt>CqjHb!XugWM(Wr2^SVSE`0p>^$9r(6N3_C%obFPCJ~=(|r@V|GlL22VVi*o>=rT z_5S*2vsv7dyd@8eY8<1B|Ma>T#sBZJCGf4f;|jUD`|XK(uJ8^LjSBw1H!R2k_McZl zmK!c<*}s**3q*2-(dz&D^d&Y^zS#G1++ZK_Qnmo_`3s5vtwunS!9$3FCj-=z!%KAc zB}ZNQ|3aNgyun^XxPvf7@#UKS$hwdc`#?<{L?UVe*rA}1*I^=_625d^r)0mJW+(z8 zBgq@=EGq`JkGCL3_?G8iPY)E});St|OD27QMY>g!`6RL%9bY@C|KvQTLR`B7$@RTY zTl}>#rX@2Py3UYXmlwNLC_ih0XSCTfWn0>q0Tw)hf@ZC{&i7|}5&XmW{Yms1^RTN0 zvEPcp^q{xR@cZ*jMQ&UZ-_=F!c@8u)E#)Pg1mkb!vM3Az__T#JK;kc1xi$}#16Kq2l+Y)iRZfWdziON){0oK!@l|*^+H0xD zv|Zv&Mj_ak5cad(c2*)Tpr_3#E!J2Yaw-3RbiH*@n|=4~4MmH)6Kc3q+@;XoIK|zH z26uP&7Afx1;uhQ~ZpDH-G!zlx{OuEn) zt;b4&UPTHPPPtt9UXj_~K(8F;{I!5oBYtPsZ@^E}JcoL9JCbVcO;{N((~jxMVYaPv z-{EnPZNP_y%pC(Aq}NF7@`SG>(0EWOQ)iH$U)q;fGoz@k^70MyCy$U+;V$1XuN0Di z`86Q)u>;=*mM9@Z_=4RYp=+#DUsXIPtPrED>roe`?7QTa6B7jafWJBx`~HIm zSI}Nm4XdH&nrp8EV&$c{49==KV-B{%VP_Z9db!~I)Ee5V_r8>`8ZE`aqfjqUXRUc1}6kOsVD?5pqR8Re9Ifbg5H9 z&;_D=VL@fo9bz~N?`|vzXPOF`S!GUe26Vo*-f+VpUN#3}282lO2lL~lSZl#(6s@;x z2zE2iJ#6ib*P28mhwlj}n^e(q_>$1uC=5-Vq-a`@ZT%Hur04ABH9Ru%65;wTwNaIS zpP{(8o?F-6-VXw5dvE>Wb68u2l;~^U^*eTr(fMDBt#=1XF0m1^H2$1kR&UWvEdsrL zuY~LL~{$XXZpyCVyUmeqbm=vTK&(M85JXjh%;>!HeMO$S??sdmnT%k*%5Nd7GtR4 zAC^g{N}KC~$XH0CkNv#$+Z639qb7#2Lx76E9{2Fhq4P#+^0(Y?fT!8vP1RRyKah*g z)77{KbvG97z23xWC;C1~ZGz)t3w_2?8b4e`G)*Ur)QVT)I-K;4DMP>Wf&$J;*~%5V zC8E)Ty#5te)z$wGMhYnSOYK7#1GX7+-T688|E@7W0Hxo)<$i+y6(Q zw>4j!=VZ(sv)z9D&LkA);+D(a(ZwFXT*gH`I^!w6G&tohK? z=9)fK{5e-M+}qmO|Il73hU(uHGRtg`(|bc9r;B6~{c3}F?Zz@X(u#K7+wQZqT&tz< z=VK+F)0}yPDA5x5x#36WPBGI?Cv%$@PSlH&V6i4zbstIkPAj&~ z?vqp8oiAazZz(9(Cd?n$hspUL&Uz&BrkTnd*xWS>P|hmdAQz%;z5!7?u8*r2|Df;9 zwFlG)J@_Va6MoT^<6p7h$>qs8b#i$Kx}1g0?TNNDE}tF<1>vMRepRaGUTTvPUza*q z<@9&Xu~_dH9{BCdcG*&j?e zzOLcutSxy(`l-0E|8@wY(2X1hh z47?;_l62wcSFn0MLoj*{X0;9aijGJV-x;dY`hc*Jv5&jC>X7-s-0p^7Ehtx3R#rQ_L&YS9CZ56IIsQ1{c=7l0BhaLrdF9ng>JM8IEKD=aH%cDMU_KD_&*h#EPX) zI>gK6Zh0%RF+Nf<2EQbMQWkjOvV>#Sb%wu=xcaKA| zYA(y4$h41<(MSPxQ)hzmm0Z7TMCGX9Q+!b~*$Fkyo=2MNOR)DH&A0RkgHzLjPL~+G zWw$!CbakRt`9v=EE3!I4U;|KRq;jaoycx4!C%sKX4qPwwi!@Bi| zJH!-G@ZsNr{;}vkdSmIonT6uc)l-Q6SorTXqOO1Y2GGBSZUnLk>{=E48(|n3kqgUI zE_+;^dZLlIF_#$pms^m|J4eJ7@U@El{|wpxLdpL95SCE2YFUeaP;hEDJ*K*xo*#T<7lmr^G*BF1$TPm!c)(B{{=^Jao#$ zIMA&8Dj~W-AGYph@M-rVx_|l~2f>176t#zi)6`(U83IV;m{25l#FE=!s(-9K4P#ju z+aFhlZn(I28e$#Z>_J@b@j;@J4cVDzk(R}TelvcHKm1pF;%$&3;rZsJOL$amgP3Wc zXY2^mM_@RwiWbJ(yEx1;#-Y`A-$*mwKLg*5whynf>?WGHbkXCDCC!Mlnda_1#NW)e zv$O5#yq;!0fGjgT}jJ|TwQeDS+F-$DPh8?l^UZtAfF{n^R~H+D2D z?XMc|cu`XQygQe`REhqQ_c4oX^`W6aeFH^~%XPPn5QIHoFHWSC;yuk5)d z11=SFBao|{-lVY2}E|5-z7XdQyk@H8fw@1h(3v+W7@5=`W=O3edyyexu zM>T|EAke@etmzlAgnDt##$mp^dLQqIsDMAyNR>3zG)YBP=vP)m;^fSEl0iKJ@}@{D zylmvyc&8htn$?0?6G^4JSNj0QkC7Uh@koOo?uM^_l=kMxQT;s0qgB-=jlCCwlrgfA zDL|XnM0CvtSG+(KGWIw>c=doOf@aRw)QPT>p{>G#l_CqJWgv%MKk|#UfpkFO4Ogwi zq%QRFc=_|G%Mm$I(q<#Ap_6&+TtVWE_*jes=?i+XIW?a#flnRU_+l`7+O2n6zSvIq zhefvJ^q;v)aZDQfhaNUnm&3bOJQ)_J@gJm_`F`H>Y~bkkuB#FR`Je^ALcIJ}tY(}M=)RvoT*m)v z0K>qS*WYc6dw=)tlrQ{Ocdn0c3^5=vQDs<;xU_*A zU)i?uI=LJpS&}mgwm2z{JIfP!$-ZK5PbC(#2)Yy@%>Yy2hP<9K)gaNaa&GEf&MarM zp@O2@XD#>hIL|PX*DU?1nt(eI{g9ZIS$HM`mjR9VtF#R7yv}F*r3(W;Xh%JbRCFqe zyYWCDer-&&rfv=PmR@7@JINjIeOg!7%L`Bn4kt(Um@;;5(!~mKp@1vX7_{wDaAiNsHXJKe#8F@c30c>AwX_84h zaGbi<3Dy~Ve2m-aN69STJ-*5YE;!bDS|pBmNvB!p0PDbSCUF_N6iy6rQhWQ;(;S=E zhVHMaY9%cSjC#s1UR|YPd6aQYyng>|+vxyX`L?-49_h(F*=S~E#^f^aJnnQ|xpkw~ z-#Hu6kXmBvho5H?-Wrw~eH=ubHx(Yb{1UrjY1EXJoPm-A9qK@JRyow2r$n(*l%NgV z;6zlPeNze3LRPBp4K54>5h6zmhLdMj*jE7n^qRI4DK(<>~gYFmM%S9#&JKX75Yh% zq1?F$nd3N!hsn9iC1xe%p=4(IX^A;ou+mkK+GG;fd45NspwUx@Y{6vg{ow%j-eZ)l zcL+PVAa%BNDqi4ukNr`b&p&Z>9y#ENT6)o^?zVF4!w476dIPql^>x!GkSsULuxlvZ zW{DO6lY_iW1EDt$ZdR65r!BV0+008%t;_nVi~GgT_txos_)+og-eqLlZ_+Ga(zZ3f z4p)vkZbVhS8C&}U3k!$SPIBA{Ek2cT{?9f&u#oVOI{n=Frf?N6(NVX2ijNoxMO0Pm$*rHbO#3+u|hbJ}e8x)fora(0mF)Vrif4x~{)KA>rZqJNNLGTgP$ z+)Y#(>1C~RtUWVjaTx&=lXj)yqU40UC`pwBOC!QE!o-;^C(hf}FW&7dQ^58DQK$P=?!NU;7y4oOE3e9gxetY6-@cx4BU3y#bT$ zOsDvI(w%KQ!yL~8pUdFG@VjN`#bz&UN`~u6#0!rfK%Gg}KbQ1Y5Gxv*T%@zXtc~5H zjk~u2Q24FLV>A3=<9_3L0iH^CKTWr@r_wo|x@PEbDltS2?V4Ee+}L(|Mz(Md1M|sC z8SQ?_NV0O@-w9oLdIhBxZVa%9+o`xXGc0|3S9!MhF#*%sHIDz5ImVMRal>3=^>;_XIt9{6=t@gZS;PxK^G#9{WQbTh>-`ZmE*+*_x+`sA4>oezrrZBYW#K85)_?E+_H`c{D4!gl`?llg>5Kt_j|iJ~~Be&MU#r7%2sTZloq2f#|iu>nf405LZR;!Am9j>GVt z39Dy1jsYQ#7%m_tF>TfQW%b|#jW_S+LX|lbdsDWo3Z_0ZdpqFUk5x)A2c#$1nr1kC zVe7t0dc8|{IX@L`i=TYcfHR5r@n=%#$H|Y{owk9gzyrZ{Q)#dB(homD6jr9haU*wt*JAh8S zK`aP!)A~)Awo%Ju)lqv8MzVNi%I9>%hH9R2mw}OU!m+)W*<|zTjgAq}kpN=vROzs5 z<7Yr7ji4mVhjZpi{pt5TrR5~GkZq3;6Hu_0t7g*;P1_0(#8Hn#Ua_zB;Kl*XR^6Hf zSY%G8S$?+m7^2i}wiOqwZVj}|&|WXl{Ghk6sWel7LY7}2Gs=vxR?Ks;nutXHgD&Tp z1nB*`Kt6Of_;98c_=W0flsda@0WLSyGOU1;ctSQ|{gGvFNFyF2PrKTL8Ex`_OZdvr z*B36mth7jQxnB$;-w6h(m4xP&2;4ZfH0)a*qPn$wH0PDgo!)GwB&O99tE?sk_-Sr6 z#3&y%+SkVGY7~EJE883^j>Q+Y>ReA>6A^Etj9qk^P54L6pbmW7zTb$zyniWPPpS|{07ScXm&>gDjtdF0Eeqg5-?^*W=*LH@LRv)*#H3{J9RHqXVSHFM#dd{Cb? z6sIElGRQ--zL8^&Jba{Rtf8&g%Rih~SU4C<$s7SFXXoTP1$YeFf{K53j&SkkltTmiB9WRJZB)sl7k$*Pht_WcP>VFP z3~DGuzm)r2O9%`2LM}~zi|qb*6o1XeMm&`~`BmtKb-JPzig)r(REE@`M$?a|_o=xP zzqgUw`j~oW!UJ!fD8FuhpTmp*YeV7(L(nGq&QSnO6=@VYd9KACjs@CSs+s1N`QUG3 zZ>ud1kJH=u6Iru%-w#Ca7I(zXp$-;4vXr5mrKj`x1K7K#X9H#zS9UMv8P z4|jaNbcBmIY~1@-uUX$c-gfoffGGx2+D3KQsFV}F)6nIdZEhmpo&ua){3cy=T@liA zZ>Y?=*N)Qjej+^GG2n>y`4;}fd0zH>mA8GF_dwEIeRA;#6Y21`8tgq-sXT;Lj-uU< zezWKjhj>H%)%sxHu1?i`P8@$j+(=ifU(Zg<&WDK~(by~h{=+YOj^`afZm+eOU+(A6 zYdPrVf}Q;I>UJt1M1)b|7c5dhCoN_?`)d|f5sZ80ESa>xxrtDOD7v5bd=xvs&U4u9^Zt%n(mD1r>|8Vf7!a6iK_a`g zLs+FY+O%*OK>7LP_Il+IzCMH4vO8G4c5-{c#Iwjzx1*7--|Tvjs`=du+pl>OV|B`)9pRW?@Nw+DQliJXVRpP>>*YzDuNYPu&8843S`5C7X`|KCrEQuX{c z!pi)j5_#qqgoU_csge1jSEQRLq!9jchc`CqUnpNgj0$t~_0z5HYndlPM{($MPCZO*jdnla|4N?kxDQ?u&r4T0I`*YJKIal*qs zW8ux%!q%;ucYnPhNHyv!Xl>BMu$VYQj7IM(e-FhJkNJe$iZL@O(W<^ z;tCfj9}$m;ZqBP@=yJDN?HMdCy!|kdWzgvpUsc5($ITRuFym*J-jEcZS@th=VTDS0hM2C6`BQFxR zjhV#!TANRgii&%Eiwxi6A1)O;+8ke1y*vr$YLqMiWQLQRVW)0v`1VflB#57flD(x8 zcNr7CmqPDM_vmgxCP{c#;8dNZR^#^|JA2O}_6H8pquw3>wXUct!OEsxCS&P1unZahet$}$25L*{8MmFj|4q0w#-xu>KL%hH7U=Gs=g2r6TM zP)TTrKQ%BPrHv=#jJAb_>D5MHld&cyxD5k(6pZbx>9zdgM=jR}iZR+Q6=4yHBDU{r zWUfuko`@V*OI|=x|MojPsc1Pzq@A`71g;(4q9OhC8=bMwR@249rSIUJ5CSP%EY-h~ zVhsgTy%wed>Uo+vclrO~!Aoc(Y$RtCw0LrzVQDPcXQ^CnLJA=YA9?hi3>ThhV}zyA zbM!P7ytv)6rn9O5BMY^`n$AHtshavydYeoA!_L7py0O2C-n!aV?BBI+*tVNvSZvf7 zy$+5V0n%eqK}be}!PVsRaulq;Hd#D~3;|+xRjTYv>-xvR`|MT0IN`YVEG;i$vE0Lx z$YW74MZi*4v=&vmnwj+)bD;ic>>*(Hr=ZJ(ReMH)SPq;u|Br`Cmf3A9jogLWMHdd}dCzl&!XvfpW-JcaW`TZT9vEK<*LlyJRIcgPF zrcX@O`^pAhcO~7laq==60L{gE`>pp*yl)5W6ou@P)CvN5JcT}Uc%H8ARW)zy)jZF4 zk0w5G)-;H%?D$GNb&7gkV>Qzm>^#97op=K=wCPlvevv`aD&du$h&c2Wf$I7gx33=C839@eV+ zFv1+q`gG~pwXs-y_oFho$xy{b=Btgzpp)o`pbp8Zp`B}sPl9KK%DE5>y=J+lp~}&f zC-dxdlC=W__xG!XQI*X?e!An$Q{G&O&rhS2HxOr*_X(jbhxt^YZ~l=y+WN5ABHo9C zr`8$htS;fq_Xs#j1;>o1D~v(v-r<^kkJ7*QJ0z4R&Bs=C<}M#nG`@D+O+I({di8q# z2t%Orpn0e#;^7%}!Fpp_`ZLhxdP5`98*|4s>Z@C6#zd12+oUhlrF^2}6F%E1S|4G7 z160lZ$uN$2`Mfb+eaCA_{|DorKqtpvll3sAie>_18p0jEZAUK_qgw29z;Y(|_Meu? zou}My%Qk;Y_N>PLnK|Z4c0Z1Fk9Yds-orGL4bmTJtY`bv_WV3U*V z`KHt_BD&h&Iv@YVl~@_7D)KjiH>wa7jq+Dh?-A*8O*TKWzO}LaERQKKw=N{+_1_dM z{k06)N?8#PT9p=m@;~}QUM^s20Al`~wYY2s^e69f@L*#v<{jQP`D1(QT-&W85w2T> zjC6QPe@sgv1${=RpJ}D>=b4+~+sb{IvQ1Zk(C9JC$xc46<2BjTD#_?-uC`tx7pe7r z;W6{|6=)1i+!hxAMa6nQB^RhHez78?u?Sk`X=L%55e3KZ-8>Pm2f&l=>uqUm%)6I) zaVzFy>MIvB6TJ2YfE^cod`+e1f5zs+(U=g;HrP7(^_#^~(>lSja;@Y1x`yJDD%;5XLGFF{={ufRp2gq1kEVf(#L#_98%2J2~6(5t&Tc}u6gXM)^aUyRx3Xs#^17BX{ee96l zDDma`7IETVDFH@yM)otpM*$a9ueX%OzgCwnHN;N9q0L<|$X%JLdv;GR#wwxCv0Z5? z&Rg(0ve(1Q9uSVzi4}}(LAqw?wGQXU+QC^q#QWv{Nn)ed4}U=zsTgIkJf+Vjyn&;o zb?9z;dr98cYj!Eq(pkDexIj1eccrrPJdsrqab1i@hy6NdpHwI8bGd%4Y(D;{=8xAV z6HQJs&mgk6=`J?f6bo5Hx(0*F4wS1&yoRMXQBE#~kIb>y2cXmI&cK;5ZA%=zdwToP z`t9T6xSgH1lgbIqK?PqqE%8`8CFh~t$xDpVcr7R6%wDUwu{*79*nqZ`I8 zSRJT(n@Mu2fWBd#YpB2@Pc-3Sg5Vd=p8PIq1PfkvL{|92F z1H7MJFt!q=yj-0pU&^&}^}i+aOz0X2O_nfI3yiH7exYfObH|74!+jnh)s629!^cZ8 z9+#v>7D9F(thm{T)~H1g5beVIY1XOjt?Wh+bqOGQ?T^IZiNpm!_(;qkD1B%Q<<)CFkh?maSsB zy8a-F?;$b7vLc;H-YWQD#}*PtjaTEVL2?|tvECwTwfjbya?=}PE1vehMf=4 zLn=jjSrKG_D5qp1=g4qB{b!l5+*EWi*#mBq$6Nd=l||UV$HME|HRz*nz%f@~=iAN? zM~;i3haF@DL+O70(Lts!Z#mgI&&SG|L7fvWb@?`aqSK|!rZ`{E(DV1v{5rYCevyWS zM?M$Sku2HW_wbVp|A|Zhl6`C!Wj?-q!VHSqUuUKG|r_y@=2twJ( z{*Bz&m}ZoX3N;uqMYuotGNzNP;!5%*-!{7G4w@m{!1c6~IVX}~ zV^+Hk%U$APw>*aYzMX;3XyyBa3q}j(Dq_zG>S5#zJKVm$?YyA9{oRxSvtA6(Q?6Xa zySXvSh9xrm*J$6mSm~p5KCq%b+_TG-u5| zMlqRqReEhm`zRir9WYtWM#FBR*4q)n`hJnJ|K07-YyX8&M`LAFtIm}#mZEo}lP(^N z5x^DP=0xgKz(V$SvZMSa0Nb2T=zB+sEJdW(RAtuLh4nF-<~k%XCof7{i+&RsaFm4a z=NcsX&a@FhjHxq`VR}($Uh^=Sm9xh`ox&ve*Q`JDm_sMj&AM?80wb)~&dHRAi7C^x9vR@m(KYrV|{l5=UWnep`KPGDqtjKqfOv5!Ix7qnf;lA8_`|YH0#gVHpTg1OpDkX1G z#WyTBEA6SWv}7vCrA;laP24>~n$qK{xoD^E!1_=ZKR8F$UH*hWIM~YEgECQk!0twD+;U^g>ru2w6x-h?TFuq6D8h!@%(rln5<6n_f9u$ZbqsaOElW@-qzl>+@UTk$l zY&02V^gg@ImHD!g-sUI%gTe@EJN}+L*2r9KFM%C5WS*jhIJa6h zES1FSg1X(Kjq_!yPNHJZ?)VGAV0V3JAb9AU0?Ey!-gxn&=YnQq(Q-K58bH$@;62X8 z`IDakK{WVx{Ktfi`0(#8$G`0G9U{NM@vo4;N_T%vbd{ZQb#?t$P(a-0Pr6ob|4Wzy z^TAs%!jsolO_;jXXxi!|MpB4zRn7f6)F#21$jY&=ZT(l$Ck$4%&0#Jj{4ms{e;Uv+ z&{R2%-ae{#`r1e_myBG}nDa0sGbe3la=l6tB+!j!Dx1$8- zw>xvqqV-_Bj~r)h*H09jFhklCD!{DjocGP+b%MRt_7}p-C7W6=b^6 zhMhC@k%swU#GZri<^3ec{6nZ3?lz5B}PH~{6I$5(Fo4(0C zL6ZNpN*@}!1Jdgf@4yX7TeEg2A){oy-gz0;m;DUEw6 zDxC#4Lu)sKVHR0Nm%59ixE)#eAOoo=UPYm?PdFB|P3}QxJoF3Pa*<-`9A=pFTLb$S zxU<5h@U4@;%?&_f>bxP`Dg`U43Y);its4(R_TqDKpqZ}>pat*D#mymt%s}qmH)9Ht zQbCF!VXKz-^;cFDOUGP`2jB1)wkFQD$0W5f}6X$QMQ^PmKE!dOrtG%&WBDio7xzM!14DBz-XdlFw}ztcM}_og>XgiqS?wA!wjzr<_}N z7^5Yr1>62}temOpK7nUs#B5nUx_h;@x=aTZ42iK#Os?5uh`XJo1uwFjg=V(>e82K^ zxS)UBdo-4$Oe``+o(XrWr6X(7fzT$Y0qS>|#BQB_sk@XN&Y?dMJm#i6u@ z26M#b?PoC<%=g&N@!Wso*5A6SZjy%RTmWnq(^ zUHhnwRo`TYk}$aTaY^V61aqvnlU2-Aqoc>|LAI&c$XwJXnr%fINT3TUIY6aywkFwG z<(gz1A4FT(2`Q2tIjH}l$p^3_ON1CHp1LI|8#C1>Dm+thJ5Y-0GZsk|yCJOc0Y?Fw^g2J$ zEAB!)bAi#(Gg+xUekN4Rv? zRbsXn!48?n6-h{k25d#W?6{5$pNXlbY(V0H&uR_vbYE8ochCnI`cDXZS|y{eCZo$x z@0KVgZf@cR`RBmS0RfZeBQK%dfm!EX;|K*;%;~u{VFwGGns# zqDV6+k5gNh9}$}yh8Vz^Rb9RU7;fk-@KwxXLD*pJ?~R`3l#IO3sSv}d#XoPmBvg@q zj3KRPq{P0*OHlqfgzJg{Qy-Y57-SG+YfmH$z=O{Qziq^u)CPnYRhl@W!_*h4qzElt z`M+mO2Vh1A(X-_in0#Z+U71+SvaO!ramf3-XhJAkKP|P$!-RyQvlaKYSvYJ|dWW}S zu(q6CB%EuE)O=?cBoo{CTzvG~m99-EGB;A}X)nCa=7SD2-fZC~$h_EA%dDWgm1$LVe!yAw3TfT9jaD#+M}$_eG6_Eh0V;Jpe=71+*?84rG5J? z^*g5l)NKp*rOQ52w#TLknRV4JE79gZN4{&RLvfQ0YpZ^GLsao?&t(#PL)NgwUt|_G zUFxo&wfst1hmD+uPJDC!ENOODJuOzKsai%f9qqS{d+PhNVnIx7?Zzc(Jxpr~qR11% zbRy9!Yzli(jZUy`v}?Y*{yQRg?50;(v6)56R%-}if_igvb2nZlF@d;sxEXG-g&Nbk zk?-hb#z}3BK2%DNy(pG8m9%L5t=Xcyk#B&uzgQqGmejO1gytvuc#!iIM3mxUN?3@L zw_UJ{r{O@Y)BfqhxJ)|}0ihb;#`t8Z97bktE!7MW6eiGPwjo!p*HQ;ZXjU;Pd^E%c z>*Bi%@NZy?wiiZSi>~|LI#NvEcG=lhT=ma^BwLK1cATFe>$%ep(fAKT;wg-87Lnj@ zu&Qt?>i?YWb70_i4U`M#54)yhsS4=xUN4G>Mz+x*-!#SLU~$z)<{{$b8ciAzLfCh0 z@oQ$gqG?PF3=Fn3)`Ud5Gx*aryvJ3b#&u^9*U0Z&R#kd-jv~44y??G+SO4j;h4S~n zUmb|#s8e7RtNxu3nbtx(E(R*};Y9L}V6&uCRBWvF9zvT{EXy+oB3OThnj{F!8s|iw zT7G903#j7ka7zBzbLXtY%-}7TYtuA{RyG!rjHuX!+m)3z+5IzB@Y?ohDzM}6Lcwq9 z)-RXS|9}sHdU0MLmhGC3XC^m%7XPE=JW^kbUxOP=Wam={r?|MVyDqLlTe|~v86)NH zJUeVcIeiFWW{**$UZPtDSvM_sa(H!aBGzHtM(-YTiOCtFcT`2$uKevEi+#*LP}Gtd zR_3>LRQn1K~E zdd-K}5Bw3nPj?Q1KjuN82-a*Y%dzWzZYJPw4EYfJj_z`gGw}XrNk&S_+yS=nUElvx z#APEau%#LB`qk?kj!l+W&yQQbrv>h6xG}umiuS_g7;gsrdClO5EPn`Tz@Y0P=;(T< z^fFoTiN+_ek(;Wjv%%3QZup~lOnqUc)?5X>lKPHaC^nvBxy0;cf%eMP{rRMqUzM-7 zm;=j^!(<>2{k9X{#cFLsgOct7LSr!ed!|Y}aO-muwy?7L9=Whs4CSy%RJI9?!Z3S8 zy}iESPl-C){LyGH~GgE$^vE`4`&vG7FqLO*+}jmm)- z5|~U2PG_g8=Ja)hb^e1Chdqpfbn+tXzVX#g zN;1|aPmc`cvM;T9r-rHY2p2M9p5ufF$-Qh)&n!Kkdi5Ug$stE0Co%yus2;RVtjqkH zrDyAP;__YK>z4{G-Z&^f*Ka zMsE8x4I-48hqLqU4NTi}I-{U&y^7W>t1h2J%dqz+r^%Ac3jS}GPk;VUXucVJ^##yq zN4YxL&{knAMxlET6{8^yH*vETIsb$liDz##oB{+Ue+KwL7ns~l3-V*9Pd5dvVPn{* zQ`=H@Y9Ubtwm22?4NRn(xJ`tjZ%)WRT+A0hg^y-uc{_1rhmK6)pmq6N4HcgNjDhLQ z+L~^EL2?BdHGM(jY=nUZZ||^6rPqGPF4$gfagg)4Fvaoh^iq3!{^7ihlh!n^G(9d> zx^k1(o_e!n6#><;elYGFvl2N5h%OfB2ARx3z`58yr>zixzv_RP zVtt-PZ-#gsCGrDMxVdioF5u{2KKDmz%_y7Bi9zq}L&imc*lb-w+^AIS`7;7$E&4 zZ}{hObm>gHvK4Kkc(VIG9!c=mNFgp5yXk|qBe)!k8o;mJ$WEEptK;(nkiegPD(*%W z--8uuPLTqYlRmLw)oKImx&)6M0*Qq@qq8q3ZHwMmIa+0vmkSDuTDBTEA?(SP4eT*y zzq~}B`5p4}3V8o{O0H~T0cllJ$|tJ5@uI64(*&n^LTS#|*NFdAWRCBX`Jh-+joE?< z4Kzu@x88FaLS{J)%ZA&3M$_4!U{YiX5=71m#V^T`i+o`ys&nEYCu4;+`dMK4s;8s( z=m4jf0_-klQ$01nUY|TPW#)Sd!!DVRr_N62yfJ>=v~M_SEBEL^4}o!Jfvo1jJjcqo z+E%!?%G9UXbJ;aC@x+x!LA7H#!E6Nm@u8QawT-n1v9)MvlpV1o?F$XoZ+5|b=y9R> zCEBi8|MP1cHeYa_$CpoUY z=Q=(M1}Yfgq&jobYnbjFir5rDI@Wz7DL3T_J7gX9|td$$F zY`y;Ec1a&_%j^qTSa+ToNU_gjoJLtn476_)cxABQK%pvMltNvJTJX?odgl;#Zc+w9 zbXJGu1?AM`rDi-CCPxwdp%|;(zTXiYYI!+okM)URK|Hcw85u3H=8ty(Bu~K-)#U=3 zXF+6c%0aKQC99G9JE1R7>1;|fASB!PH*z>UfBCZ0*EJ}6ITiI?ZT9+;P?J~{U-Eev zt9H%5(N%vp|3UHsbJ@mft4nRQTEM~lV9i8#5dk^w0)AhnYV%`ztXn`UO@StE+c|_O zKI(L#yjo>>t~_(l z?uh)e3MpP-kVT4T8gS8c2?)q5_oFLGvmR_==x<$ZcV=Da+m>dMP_x$~GIx%)I5phP zndMcA2lv?l#}zmwrphaIIH)HlClTwT9=GV8Ri00}pPD=VmVhM*m_LcEvaZ&gQnV(n z9yM0O(E1fDgN#$Py_uT@V}$U~sr!6QzRj1M+~9{lY|gqdg9p&vXZJv*-vG7X2?!on zBw*WS*?9gf*j&u!*pLp#gTzW$c6CT;!fT8Fv1yo3G)3opY+-QpX z?g8@FB{mOlguGU<= zI(pEL7(#(@D~1xg^?9thOe&!&_K2UHkmmp?ZkE5%5qq}N^OrZxuh*W$8_3qK$rq{G zXgrZSt^#JyU)#ssZl|qZ+Q~%M#zhmkZODqFFVDclsz706a}Plh6jZ)D4Q6Leh3Jfa z^@rMjaf3Y6{~*Metq}o|2qd@R@k#~G+kLIyJrMYq2>)vQP-5&;>H0TQ@_E$wp7n88 z<@w#e?z!!321GK^!)7S{ztgUNf#3hW*X)6Ugi%D?%cW%H9^>sw#QwcRW4*tUmN3T z?jvLOW-t+ky!sKPLZ?E8p=pZY>xBqK{cPePlpO`T44!GWKb=`uCE}^cVZWVK z<>iaFUVZSI$K8p#JY79WS{$=c6v2CY-mcZ3tE1t>7(b3itN6pm=H@Oj_Fn5%;g4(+ z){hpZ%RP3*6cFKGn5G>^)6u5C?1tRy_`rJjb3!%>`=)zBJtBP=j9msu7mC~??=Qxa z5P1$FK@T&B!n>T2?QN?0V!sGGypVE?LCsvlbPcTQg}33&P_xOAoN1b zDBQcB7o@x)3Gy_C{#f*cW`JmXDUBml#qm%qkU>Xsqur}*dSNP-|BtSB3Xii5+eRBT zY;48lJ?WD1-HGRMJujAYJ*BrAklY5fM^K4u^u|a^9r40R^ zkAQg^r$=Af0ESWyq(3}juiUx2OB`6C0byRs6|f8V&+uu>CCiaWaw_U->xtuXdv~d? z4MRf`v*4X|wL+|r03p9xD%J%)fV&0E44uXnE#@SGrHtLlAD)iFPEP38bhM$=wS0Rg z3uT}lwfO1ITwo%CseSn30&(!Eri76W*IJ~zy!GKXOl}dxOlZ1qYY;0hXwWg(Ay48u zybnpiu?fa!O_y@FD&6))smgn>^|EJ055IJ^?)@Ff$RUS2q?bp3lU8^*U|GK5*tI5K zSO-h{_169VhcLt4DTl&BdIGSoRE<5k(OYcjBz5cqFB(AG8W&>Sm?h^bM}&Ib0sc+N zU85_H@fnq9$2|u%$8wDQvG%kdqCpO(x}$!oPfLSwN`sydBoY|eNc?pTM3y9xYhscj zZ4LU(+jQ`Rm=8k$xiNPqcO2C~4P9U~7=<_G=&q!s%5ik`<6_IF+x~6P{-zJ`W(pFY z#dvshwZ#1~mzWEc`R$yzn0YE8t&I3vXs;9+TmhbJVaL$dBic-e3gBRYda7JkNJYh^ zDW7*3-yr`BWrpa(hY|+=LbJO|eR_TM64ekf-~U98KzY8kyc9AZ zy;}Z_*s2w{S20(#)GGWqc1Uv5)$j+TW?|*=ubQ6S```ElMUXf}mFEE+=&8R$^j_f8 zPxRJ9bTx=)@8II%va<|w!2p6d>j@wt+`-!H4jPXxgdbRmwa{9o)5^@-Up}DV%9)-2Y&6`p zsJh}E-&f*05h;!FKVWf1FN36A3p*-N9?DZ%T`Xb#Eip zHjoUk%Y9NawKUk*+YmHY)tQik9 zoJXz=akxg~MPtc2{Kkyz?W@2Wr381~q|j943?DQtFE^aQc7YpE`;Cn!kw*Y`+5cRB zaTNC$AqyCiFY=rnb#6lrctuR(;nTcZW#YRcze zgJ+Rh;0V&eAXks{7x{HWi*1DK7+R-M6Iy|S_w%;dx$S#mn*uQ$HS2^1UhO#Z2&AnO zo`f#qs*avois?|0

Vxw~DNd97*Xvc-j)>4Pb{00z1Wsr<$iQ1Mq?ct)#S!9l#
zy8g&-3AA|O@NC#v8&rR117DxBJ143qVYUW4@*c>KP<{Qv4i-BDrsuNzrxbOKJ(G~o
z(cN8GcOg=o>~gl8uY9lHFf(90*9nu9B*fKcQ6L(hkL)_)F>v8mX5AA7rf_B{jNF11
zms)+(Hm?+w009b8sJO4xGyDUb3i!OGIUkiHdunDs>uCU7ni?hgmF?gZ<{l%BgW-NeKO|Z!wQj697ATgQN-yI
zYE`pC02&g9b+AEQF31DEH23Pd3-ZrqQ0L6h9;;^zjJd>>#*fG7X^ea(Yg;&#&LGv~
z+S2MqaXhi)v@1e3QGV^}1dZq&@1^ePUr`TddO1He+q~yBRxWwI`+HgQwt6$y%b=9^
zLy<$8TW-{64AyF8DOXN9M!9~9i#LDakN6`LPU`Q>>H9%AdC4)c|W;5xjdHt|vf4kQ8zV^0_^g#@`g78V^n*!-_G@JMXF~YwV{Y-p7u2rZR
zLE~%X*H+(2Bx3@RY&_GmDu|4D+>&Pg;%ea`iT{w&G3(_tr%%~3>U<)@jHpshCfe3H
zKFt|b;DHZnbgOpWIZ&iwmS$n7#w+9UKx5XH
zYJ{Nm`iQ+0OV?wDlg#p%q8y!!w%KCM)vR>WS2gZt2jBv%NgReo;hdxd0ac<*+xho?
zY(1hjbyiP3zEAsqUbkIdd3@dzd|W+l?N&5x9_*iesV-Loc3wwq9f2@}2?@$-Wf&l3
zhAOr!xt!|xc%G7mu9$+Tjvq?$0n9EH&4desd-)gz*DzREi8HuM%PxK`FqD9|myT;r
z&$hEjus~0eU%>)*GQ7V-Y~K$%y)Raw)&?}Odq2sbPNerjr}q{88JAcc#sQmZLxQC6mPB&UL`uAH|c
z5aO>Nu@f5IYxVU2TWdS_u3HCe(x`}w!*l->Ov>HSkH&5(nZhPxhl)+ZYg}Z?=8#Lx
z?1TfTf|)?0dLvDG4Mwc_12`(o!lPua%%T>3G#zCpF4Um4EnFEDo
z%)MYAh2_Y0->Qv1rx#AQB-R~ytj4R``OB=&G7;3VD~l`=4)+bIv8#DTZTX**;03FJ
z4PMukl~ps=G-g71?8GHw-Mv{{kCDzo`DDWP$jKCn%XRLQW7>EF6kFw#^6$s
zs^-tz6U3a)J7Bwm@h4g4HUqVz1ujxtiNBV-W}4Thy#3rC1OlC0)i%_?5a9+`y9P|T
zN`+u)7%
zd}jZizIlHe{2xnka}lqMTb-NdZPU;(&VBPLnH*$lj1Y>%NtN!*-8%fCNQf6cG?@Umltzzkr^eRKtE4M;
zR&ka!CMd|i5`0+xWEe}YEw`{b^~(awoZ}Dr+ql~Bsv(}$xBUV`^}cL6k@fxKc-?fb
z25WDjvlW|3kV~r|=^?r))Kz%$J)hEoKsXBV41d`Sx*3gNqYEVc9%5n3w@u@_{Do)e
zl3#sL;0xY*=DQ=}sXFh~d%~hcR>S@Dk&srXlJ&&HB;ypnYl6DKJC4BZ@zm8D{1h8*
z6l0>8hb&Gb6G7uKavLwkjcCv;t|Cq^hgDP59xYDO2xun3-47aHu9+k6j)F`;2KPxt
zp!Y9YN;&6MP&*^62u9m*7qc!!4aEu|x9n89*y3ESi+<^i=bQkzx=py}vTX5#>3NN7
zG-l=->~CA~ia9z~DQB8oZ}CD%1?(-Wxc_=BmAXFK!&LK+R2S|7qqaOISR1nzeX}NQ
z1-z>=f`4)pm(tp>^|qi$`f)8BG7!?|5z-b-Q$n1RybW?Ay(4|u@_v4L9Q^G<&Av9r
zP^s6_dwk4&dqASvuxqXV>eczsP<1`Pz{d1b#2oBu2RVsc(QoGhEUYUh;E_0Tw$elh*lKnn$_z;A_712Mqc
z`)?|gJXq5^MN!9!y~bO3`27Nuf|baT3xR?x{_$V%MXUfbj&F`AFbS3*!LY<~wQt1|
zKszD^mhbpEU8OBGRm`~6eUuoLqsT+;PuZ+*`#Av($%=Wuo(B=9d`>{#+S&XdPMx>k
z*1J8^t^Vij_ot1mXX8zuPk!7s$nD9j$ET&w7k%Yq3b*Xr0>BFvi0woqMC1Ma$F~o@
z=dnlA%iI`u*LREg7}iA}ShwZtlymK_F$v3C$Ba><<=cuJsyq=jtvDYGUfU-6Y#amtn8@)B6Dyj~1B8bOtt#4^hIt?rOWzsQ~92-&=a{{x*J`d=z)|7OJh*GFeT1FSpaSo~_Qjcd*TtuN8(9HqxC`>R5b5`jtXuE`JB
zTk0fSnf1gkGSnCw*Q9*gOt~1t2AGX^tPYOxT~<{Pp>T{b8z$rM#KJPH$k6Yv(@e}xr3
zelmV;w0Mmdy0+i#w14cc?x^QM+YY_Q%=S^oz$e&w%*U=)zZb;bY}YLYgIp{&Cpkg3
zTb@6ApPzAv)?3DGtRs$cx(;@Vkt&>3{w9=!uWj5Ip{cO|#)eKD(
z6{@EX?I8SbH*m@2x!2sz(bWEwY^(+~1QULL-hEK|vUc==W`a>IsvV+O13h^xyGl)P
z+m{v|`Y7Cu#@AR?+o~AH6l7GJd5RhS#I>K-gZNcjjnF?k7a*zn~?1wJWhKO>z)Dsj=dqo^HNqU`8{BPi3nhRzAK
zSB2e9HUBp+SJNuDwj_+oPlqS0~s8HA`=469o^YXM)4(g8j|e
z*fRa=5q?>%=n`B3hg&w=DxqQw-sIGzX1N*{R%SG8^l#BFb0mUYmqgGlxmu{+73C~9>?_aAS7XO9gSJq6W9q0slR_=dhN?XCRBMb*`I#ad-=yif
zo*+62P_v4Ygu{drdxL@q6K4ro3s3jF+Atgt?J)@;rWN$kT%R$~C$)&WtO&jsOp{PL
z750)
zXcW1v{1F!+*^zXBXdA?IX~d-n`{jA5vt)5uCIs8Xl|U|7J-oA|&?)`;K)m#Q{;Xck
zj1L+LxB+2yo6je*C_Acj^{t2cr_aL&`(pfF=j9+kbz#H2NkiGJDTfAF{~z;{fFeNUqGiU4CI9s+*UDJ
zIFNZ!7I)gJAv7O`hQczPnJ@+iF_>KH?1D~_^9$I8z)gagCVZ55Kv9ucWx0KI=L%Ic
z+NQuOr+ueey1dVe^19Eb7o?D2d?K1%(*4otiWdFqRu?MZ9Cnq5>aWWkHmij}?~h?(
zamlDgFF;y^@G!D9Tl~HkC+7!M$45>=p%>&=0Fi%}#M`aCDMB;hM&#ISgKIc&buc%P
z+{)F0fqphqo6=@V2-zRXn03i@7tE&Q?L8;%8+7E-5{~YsK|MEV8eSg9MQ6uJpwhxo
zLPgEMU8|DhEIZ|e{_mNK+2~~MpMNHHe|Fa(OldaPn@uZ_FMupP;cFqH=u=s9
zNAXw_E$sH1O5AZH->AMEmEfjs{(hl1psEzQ_oNL<0~v68xRk4wsZ}6_TV1uNRcJL>
zEtLLyuhpbl1yYBw!^->LKm7kr8>}`8(t|T^_J0~96maG7e;=au>mha_9!e&c?;^^<
z*V8@x1RT3lP$u+5uv@jr-gH$%X&BZJCesq&z0X7xDm?kFmqsl`iIeNlYB3CRG3vLY|IHM+gBa;;?9$6C#utkS3$1{}Ci;Ea%H%){Rp=HtT9A&h}kmJsAYY?-4>pTkZ*|UM
zyG1=GbNdnGyvf}q7jFf+w+RuP^R3P!R+c63Em?C#fP9O?o`g>!y&BkkZ)s)7A1&PNV}L&%VG@oFh)9
z#joi?Hj=Cn@sqGzuKAnp0*$`jY;=Dq!W|b)*v2$MSuy((8)aaMgkYiDW&PD%z_nK|
zWXcRSAIzzl)sSICBq$J^1jQvWRH(n0N?*0C9;o{rv}?*h%s&6p^>xQY`YD6qQ!1aL
zer!}mEk-z}K{6xEGD$z;o^xL)PmFN8K?%P>X-GVSThGFF`QWLMR=#zEpX9F~MM@DW
zEK~?7F1atu*?5dy^rn|v5y1p*87eAlAUCoolKNw%zb}JYkF~OBqHTjDC;N`@GvrBg7#zI%~{ZnB`TAS
zZGc-cuidzUu7CqK02n;EVI3qAAC4z1sVOLJz@qk=(HmQ)8>=jn#N~r4DNg
zB}^(TgxzStPP%%InUTTI>EX3|jh)3^tws|T3x4x7sTQ5!6Q^#piAZ;^)zjmdk*SmX
zsx5zQik6PiD)&UEesOxz&_K92MMXZWn{oR}HKyd0Y`}WjV-A&|p0_LzE_`@+q^YAp
z9!?sFY@#{(i`~c=m=JZ(5!^@eJN4|#%_=e{03rYmGD|LDV1S#Gv$fpbrn=K5?K10`
z@Us1uF$9T#906kYmyVm<#9AU$eS~>6F`l5xd?W=_E($&cYRHUH)_muW_tf2;N(7l>6h~?V}Lem*=H@LOne~4y5
zsm%An9Z*FKQ=P>{D9A(wf9R
zv^ZSL?#>`-Xt{@yMIg;ZVRC!>B)pFiTmU$SYLFPNLOxCXKA!=b`|Gb*V9h@d@IS8s
zALb92fZ@?mxEO(#ID$VTTIx#^ji=|N@x0}yzxJ~uxoOL6+#XeM@NPd6N1jkTb9kSF
zO{@S(oy&)Ntr1c@8;luvrxq0vuJ!|QHXn;+^g1ZUs^GFLbyo*cFaSafG-nv~XsNEh
z)-*ZjQ>_A?Usd!wea21=-9MFW3sd{ePST9?@mAf1boHA%>VW4jI{$+2lG-Y;
z|ApLBh@#>gZ}+AMg@332D+>R=!Mee&PJAzQavlG)Ba)mUfpy?4QzAD$ChbU*^>{lI
zLZL2RHsYN2wUky~I7^`O8ZkjBc`GUGMTZURyZ+Mhh?i_L604>+?Ut-h&DtKyPjk>#iZ7x`RzNObvC
zTu6+AfhowJ$ga&pC&D)_(_EQ@xre0IfiM6%+g6QyHO2C6DH)xDt31loIB}MU3Sp3D
zkKVXuBbVSSbx9`KHGzw-+N`0@AkWc}N*{HQ5O{=bsy+9Y))Q1)yVhT*fdPY%t3sy2
z;G?qX)9mcRLdAI+C$x;^=9z258hC}|tNBr45Ypjkd*#TvZy2*`%(L@%$yvZu0s5{waItwj5Au7;i#7z-RJlF0%3P|^
zn)l>orUrwYA6;rNCe$e62}L5}0VI7<9O4}uksRgplfb8LQutfj)u>Q$WP=JJYvo+x
zxl-^AX<=e0Luw1icm*l&B$3pFn(2xW!vxGDY5e$EV#->PK*HD}<5_k`#_@dcMcjaz
z!2x<{6lx)4NJt@RRLNR)vbkV#*{}xk=pTdCZt_B1LxXCnq(D0
z*&SWevc#SKq}vR6-{6I0R#m^~#ryX%YQOc9pXI$PQR4-BZcfDq?QvU+-7l?o0m>wIPtN5{DLSH!EU
zJ|y7TP6)T>K68cIDXpzhuxAVW(02Q3TC4dRIr8l7PA@JbtbcB^Yqc+nhUJfL+O9R5
z17BrVsTrcMk;U!w7_so5Kf#vllO$C!5Y&2ATy#6;sA{&QLS9D>z;`Blv;xrszlnXf
zKxSu%a_Kez4(AfFMT21clbKqy(1P}LQ#WlTf4nn3JgFpJj+OSOfdVsr*9kG$)-05+
z11`QDoQTrjwMYWfLxm!@{FLHbTHGEg&I;DfNejCz0%}kc(euEU1nOBLB}D3H5`s`B
zp>(aAE)}F&voPUrb>mGWy8veAqhgaCpAdt4NOQ+d-M>F}#`TR2tJMOgJ1@`=zJ-Mp
z>SiQT7uu5AK-fU^;0w3MBx5n#7}=bSqA=7|Uy+|L_C<=D<3w#y%59R#ey=HnERDi|
zW^wOHd3Oif&WZ!MJ+lS+=Az7;xmZpz&7^9RS~w$pdP@*IQoYS$qdlxPZBo6L~;M8N&xNhp7`%mT(*?qyf4&HtVQru>Q|G%g4W*N$)c
zjiI^SO|p%V7#+7DfNxEokJ_G}<8JPFWQ@A*|MBl2B%<-Rw|fHUnV8OPNUXhxop@iD
z@isTZ^V_T%waTmXNhVl(iI>eF>3&<%u)7Hup?YT%oXMTlLzb3}QRZJ9ccf$-U2gx@
z*Hi`Q-EOQc1}&zP=`?Pv30hVES7h2Lay9LE?Ee&?{}cEs=UnWj9OFs*FyG@S
z#$>>l6WW*}P#vjfyMWXymlA3~YPy!lx>$p
zh!E&@0=*SU0b0Fq&^;Kh)G;*YCg2ty9udH308J74xn0U16c$`|90ZU4J(je|x8J6%
z=FUx%6vI@RYU_wuKtpeDm!AbCxPQPY61taVbgXIPoH?tNh>caZ3&AP5uRKoG(3m4r
zsd3OAHiTs`JVTt7U(3lqv?5Ua&FV&a-}v^pD1RuwybByn<(PYWM}t&sbd7!HMmm~q
z1MO2Te%D?^&&u3hxE(#L6Y+FY#!{%5M;h`);~)+hFgy}z9`*->Sm?!g97PTD{(&Q@
znL*uC3-F5+bk@}msWeFV&IBz8Ux?hFMu`)NU4-+lI4a#V;yLD5pu~gaVSL6wcs)2q
zRah3SxJ&$3NC$j+<10IOQ+6Pplg|TLYDjhsi?F_MDvKI}(E;CJ;ADtWw=iDow-Ha}
z+GRzZ^;n%V({dzs$3w(3;^Fg4b{hKGHbEq1F&pv%8Djr>X#+%TgOW?){i~hH^L6G
zH>-Q|v1nP-Gh+{y+dptuvrff9jL}cH>foi!V#thFN{|TBT0qW%
zP5B8PI4WP%kJ`8to=|2k^|;WF-MNNGO7^p3=Z_<2447C7|BeFUW%@!~6X*AH2k@vH
z(~u~j7-Gl%#&L{~C#TV}jwh!WLr`ix2d9|nH^n%=kdN;S?4m}Xsw7EITOO~vrHG^Y+6g4-g~EWYLgwDt
z$GbaeS*=}!cK#oUQzX`%8=G5ep63qcXB`O4vL1^tj@mD66_LY*M-}sOqpJ2RyJH!RS0B
zDHMC~Ck9EhDni2cZV^JZOu1ed2let$2);gx*xX$jXL5aaJCgC{-P^uN8^w-_owjiO
zTUkD>r9omdK@tEHFo3JOSa)e$h>V>TAN{o-PaWH~5+vV&m@7_PC``)IG}f{D-GK&U
z(25hfto&hW?y{?|=>8IDv9*Vv^I~w3~-C0lWzl@!ae3jQG4CGJh(o
zEAxD8D}MI*==
z_U&NZWmTP{X>8_|t>+-oU5YP+?te)xx<0)HKCWYc+udLfATjl|Fzh^6TI?WD#
zO`GoUz93!5Z@yzGOt&D@PMsxzZ0{FKLa#fJJ?cYE3=04-MHa&++_ZLZJw(*S4^^dl
zN;u=dwSV5Ub@=vto5|}M?Yij}QfEE|zO)o`H2Mm3c-&uK$ET9Z{PTLJ)gVBurK@}J
zy7ehw-*LYv3ix?4Ut(>S5wrK>$B#@txA?vgB;eoaJa+`pALem-xwGti?6|z%^tikR
z;U9F^f(5+axN<)3iX&dn<0J@oc0!0Mhpa|Z|9Wh_5_agteBbg+XlUTrJ~Ya2kG@U)
zyIf;jzjnT+_MxS&{;^5g)%pE){rMl7Gbst9J4*Kp(dX09u@ev9-Lyzk(K=}05i$C@
zX77EQA-4W>Ztv69%w%rKqgOsurPW}kS1g~?S>!6{T(11z>_ozRD*IqOjSY0om7x)p
z_j(MM2>{SNa=iPiD3{5l*d?ElchV)Y`M1lsZ_&KxWlO+nUkW+k%0{&+R^DUso|!*P&Tn9zZw{;LaTOiNgZQNtWaVemI*CfMSlqfzrO^M@-d*@!3B37iJz=Y0<@YJ|vLBN5Gm_>d{e=^E9e$;`HeGb1QrBAx9LP9;%0
z9}1RWKKUZ8fQTHg{7`!bw#?$K%CJQpSFa->R6gr$RMb;h&9uS9#P9o)&y|9I5eEuM)i0Dk4
z7bYqlfHI1J;x3a+X_55|`!x`P1I+r{VCt>`III#oyMf?^^i=u7x@8QLu)OwvW@#1t0;I+&_|9x<7Jn6(Yi1TFeYK8s
zU{n!?VWWj_GBnuHa;nltX>o
zChRIQ%(>yl2~beLd-bNN>p1Ysj=7`*@eTU2YD-AHKBFU=gi3(5Ax(F+AWgoDZ#emc
zi=;hSe$$FMJ8nS(7*a}dg^^P4*81G!1!8g)dl;TVgMP-v_ffzo{8==^I=?;8)f{k$
zX{#P<8C27mFv|NoKkx!ZNjiOka>5LPBI#7S-X4Cj!PS~U2z&b22^%S-UrHu9U}hnq
zf)AcOKh&ixNC`z6ihLxsfi49H;pI$QvsV@|cB^V3ngtx!07AO!rh>zaKhX|0RuVlz
za!x$V%S1qQL2J9EWmPf|2QzYQ8XGKJ&9D@>PUkT8#v$JdIkc^0XJcrLMcQzGjj1?lwaDNRp%&yk@bH0~`&FP(`hma$8_PE;PS&+m3jGz
z&K#td_x%aPvNrZUa`irHMe8=VZ1brHfYJqsw}nT36F=SV-Z>l6!;G_w%#cJ4a>g?
z@hvGe0CFzYfWs%p#W{!n(og}>yG0g)cG;qCWmY6~6=$^1@elo7W1?4V5bYbFM`gEI
z8WiyT^KZ;YD2U^;)m0e=_1pPyJT2|f`s3|BZOSvaINSXso2WRXj|600m-{Gun
zoNuXWoFC)!w&&dClHt@=X@CFhyy!eRRF35H)W>(-k9O}7sntNs_4(R0Mf9;l
zv^n3Fd%imWnk1!nfq3YMIWP0YPyJK^HE3d>rDnF$$NB@7&o$QDKYC-;%g3F&bA!gw
zzw40jxs%{_yD=d+od*r27CG4Q5ug9vPY^-qBk=R}^P|iDB+ax<;Q5FC3#vTeo&BHT
z%7fqQWA(y
zG8j7LWI^kB{n>BR%9{4Y
ziV-12VHkg66Yt8?+$wT2K0?RqUE14Q_e
z(R-;g7LhE$W@JXDj?n`xBR&}{gALz}%Q_Y((1*aNh0vLo!UPM+k5d(Gu#1w%HEP%E
zZ!uBFZT$uOrJ2_ipKwMU`>-Whp#eiBKwmwJ
z9wT_IIB2yNtqS%6Y5rwTm$S8j7^}-_T
z<`%*J2bH^TW`iyFi4A^ciP-fcJ$0zrImtH${*C_q(zT+<=CbaZ!@`NR=uwtGWQo~F
zc{6c6$_TgYh=es(X$vY)gL%js{p?{}j4~5tS~-~9*!bVCG0VazYV
z>xkMg$GJS3{v~KL$-~B^V#r*QtO|(Ia_W)IMWRwd_*8|O;>p~^(_qsJ
z-P6#vgDrbnpKE)(gu5&5rd10{wiLtkh6JAz$+8=cQP3O=3641&1~X79(WsJ!ET!&-
zKdBXeNvQ3u%7V?{9?hm-@vhPBni>%ic^*~9qfwVPgwh8%e(e?9@3AL)>+C>d4=*JHF5oF8s;{LC
z=H@l#H~`@?L|Z*Y)E0ls-@eSxQeqadol{ByGZc;{yxi1fip$Bm9OJ
z51b+_u+2z{b9?kEft_=$Uy)9)JIODEcN&BIc^=
zbD!S4^ONCUYj8CkfP@^llSS=Wq4i}TkpFuV%bLr)G#n^FkNnc{=AL*zyjH-Ve_)xs
ztEAfWn)s!L{NXcqxm)_$_K}c!-D{sg5Sr#Qe!&!BPlc%2HTK^;K5HKOK&kS>(68+cOS@v=6?5b%ID*j?{OW-0qK=elHOdSJmUPl00hQ1C1u&slZxaAMdN^;
z?c@i?=bHlp+KI`(S+1KqcgJ&WKhHdzT|W3m?#;kTvJ?aS6ILov5VpF4p4%lD41CaTJVd(axJ9m6Yj=w!_?
z%XK$>J<@Ifj47tX=Y7e?YlsptUG3v72@vqU2jT}kkDCkh6$9Rn*M7EdOoQ^|^X2C|
z2rb|B+SXJ-SXHFSGV`xB^aiEFTJOaUu>+P%`_Bja%hmg==mx;&^VDsLyjOV6+eps3
z^7Vxszx{@s*QLkST7&TxO~_oE^V>@R;0fa68RR71#Nbj6KI2NNR=%aKljCxlW?!7`
zabb22De6j>o?U2AB~w|a?e+J3(JuP9={3#sd)3OupNt
z%7uauydt`5f(Rpm2v89W;0vpLVt~UM?F|e5nME7MdhWvKD5|&%??zfUAw=3$g`ZrS
z{H>Jv)@0W+TY`y}%CMrFS@i+#&vq0lRpU?<8=6F8qohuj+v;7(km9KV}u*kfg|K
zuU>Y4r-WKe!*6-$PPil+!3Ow22)ltonXGbAAq6_cu47-Pd;MZicYH@=fJFeK*uuLZ
znjagKz=}{wZtNp7f(m;vR_F|dNF`_?BsA>7>S;eai1Zy7?|ASS`G8G96Z5#^wLfS4
zqAobqh|L@j)2EVNP1`QS`E9ci>N3+be
zn(-_n^PHG~^epWYNjf7mP_JankcOmm=pX13iwcvFf^U|seL#O(V>2i{2!}o$n;tE_
zB8ZmpB>j0`2qifN
z9P2RM(8RgC!ADyG8;xYPl(E`qFbUD02cHg;SZke=PnugFP|MVJ029P-JT8y$-YRBTgA
zACOQ)@S;AmZ&v$+W$SA=kQN`eqGuJ=JBjeE5%WmFntkz5t`AcYEDtET-m+(=QLs+V
zOS)ABoN+myBhpXi4}vJ5SVkRmDLy0~SwGi=`F<^e3^_hAQRDOJ9Zdk}0d-bf?pt2t
zQxmpx{+0S-o{Xc-D_YadMBEs`=qx7pn^Q|HnDGcG#i8B8WIrA2U7~b{-%9;QP)rO_
zdSy7>Z&DW;oM|Fscju0&-M7oH&qr
zFpU;VuVbqy&`g8{hFmk51vf@2q0s3!Ro4(C@^0=??ksba{NyJ$L>>w?BncQavI}7m
zQtS%Q;#FW#Z!nZaNwQp3!&>2I)WGv4$$n_mrlR;YtJe2vN(aVAY$V-F8GPeN>(>gk
z2r`N2Swf0(BxCo2MDnpO3cZgz-ia;#?sZ09f6#Onk@D%L#~^-?oD
zec;k9*y4ED(O(gQw~7E<^2t%5y^Kl?mO%{3DQ$f3O@s$zF$4!beAkpd*N0+8?}U9%
zaJM@>ygrN#cEU{`p1de@a?l=2?(LO-T8|fGGK$;U^M6!%r*s`vC%l$*KBh1KD#c&a
z0c12>|Y>|7Pf
z%F5o>d|vG@sn>tY>W%4*%$=`3S
z9S6jEs*YJUy;fFvom;=%fOK0c34EF+Kh^f4)j^cjM(~vH1?^i;pe$Brjs^p2kM!Sm
zu^v@AHf&dqJzTz5jeG}j4e)|PK_{OVTkpL8?9lI{Md(`ojV=GF&s-}5?v*_r^xD#0
zjC+H%6$Mzv^)A*P;xB%uKK^Tah~5Hxs!x&cBA7aVX@f-|DmM
z#P_ZC9aoBf_hroInaQ=Gg8%()?rpT{Z8TbLyC&mtYU8fcs43^!j`FY-%c-e7Vk2tA
z`Q)f}tzhlh+q9`_B@27aRl1E}?c62Dqp8Du4gPZJKWOtmGVfzQ?-MV-;p4=KUZzIn
zVn^OFdkP;*^%9Yza_#E553qR-J}no+ngb>l#Qd$ul%Knd$s2n)J`%tfTSGmx$u
zb#zPH7s|&I%))pY1aZQjzI?>~7GhB_q6Kipo$F6H%!y9XQ*^pqT+kH`>F@Erg3AbZrlL2Ptj~x_RsItW
zk~Mc>ubT^t{zi_x9)C6{hmcR5Z>;H{hWRqYU_H&#$u6mhSmdT0pB^o=x72zI#K
zgw3`#uCYcfH_$bQp|YYPvyOU~QA}JUG8CI+0y`n9hG#%y=@*ng{g0Vd*-4T@;#dMZr1U5(MSjbJXdN;2uu3oUq_B
zVaAUqFX2sA@`VUv81x&`;(auwWA2rOF>K(?W?AJX!P^rujk=bb5Cf=$)#+@w<+eF+
zI8TmgM8u+EK$D&FZUFfuq@Yx2hmoQ{8?%&{YkhHCiH#9vH|ah3+pJ
z*Up59EOb)u=hZ)YkK|3HYhejyws8P$I->dCm}B^EV37Eph=D+0An(3SX@eOvE^D*|
zjE_DK);20({OmoN0=xdw#L7B>o?k5FwTI|w725q2*Y_(sXgtx2>|}71=FS-{W~k7M
zZZikw`UX{Vz2g^2utDiaXU0zHr;n0ZwiP2dxKoll)@=G$D#bf8(<-wu{Q*e9q^IxD
zYPN)rTHQ|{%D)C>jDjErq8j`65+|Yr@J{g}zlyMCaffb1!uGi(Bfc}!J@U%}FQ^yE
zH4>6CQyk6>@r!M25U>%4G>mn$)1iJUrqx^SYEy3Wx_+|EsJD#Btn(+~^ikK?^v6dv
z_rxNWsH8rjuC&-2EpCy-k_d~#lz8gf&hU^agbx%^Af9c>w{3`9`vA0DnXD_k012o=
zEFGJar`FbYul%+2_opU>meT2Fg_gpoF;crIcpcv+^_z!_eyD3`Xl&bmo=@F?W=);L
zdochI!pC9F7?~7u1`-8RLFc}uM$2_Uz^m2=%bS(XFC&?PIA^t=Sv+FQWEWp6yKk6|
z*y!=iN{D#GDWb=M2=BeX<^hQd
z>!u@XePMy}xn==MSmsUl!uY0R_U<+B9Hf7kkLo-Hb#MRPuO}H_9zJi`wEtHfjL3V(|0@ZmD8i232hrsDdd8T#+K&b=
zHst0z)$0=;&+POQfJc)X(=Y#It>pQxA*c8s*DEQnIzjez|Gx9zywi1|Zt34n6RYzI
zDw=?|kt)ZsX^yU{y|->k;79S?4(Mx5oSmJUBI>E`UrwvGoLMfdt2zcj_r(VYl>}
z)Zs8;_qg1tyn0>(>(Vr}0dG8ce*&5W@O{4BdhEWh!ZWoivt(h`QH14Jf$g5)Wd?S4)Wcc2wIyVK@!(3ra&r>M&C-dj%Xq
z9e+FAG^u~~BxkOq5hQ{;suKTElq(6jk)ESvKt-tQ0C3B*fwv
z^Mf0RxkpwvEg2v!Dnt?c*G4oqkFT%u87^PN2HYReN`^^GnG;whpG6+
zS^jOeB(i^{z9KSUy=Sp}wM8g!>~^5ONRj&+~}a5-1Tw51kiKOXrRcSv&BL~s8@
z`f3b)92(Yuk&CjzJ^!Tr=jog117m8OK7|til9-ej(h$YWcX(4`z$Y5_lS#!BH?1Rr
zgTKC}kf$3J6)OHfuDizT8zEqITO6d@Fsw6PHD$
zAPZsfKv=x?Fo7X3_v9b8Pit+OEw)(`)Ta@mFsW3zF%}maf=SqnRxn^AeuclG?%9_3
zpkk$oa0OWhkItWSrB3>niVnWb)C983E2jF*7vVyILE}}!HK_(jlH~MXoqX27vDW2U?``eSC#ERJhn+pQTYwL_pJLkOpbkT6-I-KO4^|J
zN&dzL-;~z#O&m(^#hJuYBZ0EI_RC#RkSG>C-=gtmCm8|(XZQ6x`bs7Ov_z6QW2RN6
zVOWW=h8y{EQK8LxXYFj#UFqAOi7Te?IBWHAB@NcWxbS>P%pv>*nUGjhJwrGpTx#4(
zZ3h!4F16%AoY`;WW3_qS#x8b^>@DqGBR=Hr9O*j9nwx>wsIcOE3<0ajcc(2HGLNT$
zUB%Kh+&T}f-Vv)CDewjcGz_}2jWW8?NU70C$I&Q;gn~z4s`z>{&>T54&Eq?3Y6umt
z5MuW};a*ZUGgcLOhGDG8sn)vEsvNr-@pp?PasC6&0ex^<7dmQt)B4HEy1zYCjxv3c
zH-a9pD|Lt?4ZiTD2DIM%%bv+mET5cKKypT`uKfUjBOckN232X(Su@(3?%e3BrM~R#
zt6iTED1^7KoE2WW0_h3D(O4-mkhiAePVDXc}#*W!bkfuM$++ihI3G3e0MgcVoO-2Tz3$5dyluy{mjtr9cbWH3H|G>u}80RgWy;2Bl{
z8-f#3wD(UCf#!D!_z+4>&469$zkdZ>(IKj55e0uwtiKn_2bqmd<@st$CU=j#p$+}<
zpVZ$$YwG?Dapswr%`LA};_F;I);r61a}&;tLtCJL5S`8
zqRpxnzL@N)4xutyagnF=mv*F)4Z#J+AVy)@x+x|5bk>p;u<%e@sHkdn0O@>boLi1>
zYio8E=UNpHqJZ=~Q|i=<(F!()?NVo!*@)g&_7B`T#{YSC=FD>{^eu~lhuG~_yS}Fe)D34AO1#5~0WT51Q>zI-tlIso~
zOR#jhzJ8cSQL78Tq~KAc$S#^_X!>K`_^ygBs_-)}$vXzgyMwTEk3e+Zb`lsWSPYWt
zO!p{;F7h)hBTqoY2Oi5i-=9Zj60VSHQbsVl(rM)xYf1##Ks-LAtEE_SLe9Ti{{+R0
zi8JT&gT|Lq>7ll^MD<{TM0K%E!Ws&Cg5<*=g7N7hxNd8I_aD%eV5r$qWG&4{)J%%N
zf?9p^qaTN$qD>^OF{PePqP2Ct2vNf)EF(yr4^_PGaxit!FG!;-=mwdZbi%Ztmyll5
zF}Rt1DewGH(QIuV)lJY3qTE_38&RJgj4^_=qLHUmOO0lC{?jK$sJ>+|lmL^!9AOs5Yt<&&r|DNO7T5o+WDrNA*&idm
z2RiiU1zuu8pq_X>HgfRf03gubsF}w(goMMOf{^6U#qrDb?a6~H-pAW=id94la^#|I
zU1zN@&bMt`RL@VyTRpFCZsu=VB@4?VIT)XXuW%YcYQxwrD|38skk=omsJ5jXFVT`P
z+8=GGQ$@tXOd3qI_*uUDG0En1cXXk1dRnAfvb5Cj{ECBtR8-SRu19kenU$@xw^XzHI*s@N&36x6nAyA^L)^CfUd)(rRIHMl^2}dTvqQ_9xh)iBufY;u
zDsg_3BIOD)3&xs?d1NOg=e(iEO~Vr`d8hIyb6kjwmYkPxEPOZ&bHI#3YKqhM6QO34`6%mJbTy
z!o<$b4`2u9-#o|Gt?Xp~DG9ozl!r?)^B|{&pSZ^#np3_E;}H#kO?9Z++F|{B&lxaZ
zE41M$BQc}3FzI2cg))nG!J`=*<^iB
zEoEfz`#0tPjstu{ZS&tE1iP}FPxlQ5Sb7d#SLA#u1^*^?E)w=FF(e^_M&?PG5`JT(
z2)#uOd`6yz8m@Iyz}FKItaFP^9B5*%z0|;4c_1Hy+bmvh_eY@`{9YeLPfodVH!-Sy
z_&3v>twn?ymZO<|Xr_=(t0c5a({Q)fZLouP!B5~ak`^7cEg7r
z!rB9==TWYl!|T6jTiYjOgevaQKC$eb83UhmF@i^CvHK4ejFk}RUx6_=B?<}kN5b|-
zv|E`jtL-k%gr}H@H7yvmjg2F&c$>>NFWu%8*>W})Z~Po5vR-zXVj0>UoPbTI*fNxd
zI=hrXdPN@UEcGkQ=jV01OE8EW8#0k*A$lbkpLb57$1!Hn2CiE#6W6qICFi
zi|s)y|DruByH?lcB?EL{XIfO=e6D}jdm}tvADNs^VSwVqT#iiau=_^acDNIN)F2Ph
zUf&Ucd-E%*{o3z~Jws~`h=`VCC$7QNrcG~;#i1&TfttSU7+_$PeGWU1){p;|9}~Wp
z<@7MrhuOEf8dGm5F*?NMlo9YIqE(&pPv(p2VG41b3y(5S4{&QUd2jpF2fy~$!w*b
zXCB9e$@gdAa19qSi7=ZQ^QVY2&?sM55~!jD*-d1Rt)k(8Yw1X>Am<@`SOS4R2En&`
zt~yaPx2Q=F_2z6d-DXg*y@A9|Ji$HR@p=OtHArrL1Iy!`p@_TEZa?ZK%2912Q@pU<
zroJD=wQH@KmlAe0OqH4=Z@wrOk1Lwg3s3
z*mRoB-772VnVv5o3PoRioV+7lCadDV;n@Ek?c*m82@w^lrP=Ki8sm1~p_E5Mp-8)%
zAimU5J-mIGyxHt}@_OI$@lNJ@u4{5TTtlyCMOa$fX4JrwHY`82(EwtPt-}*r5Z09g
zl}ddhofT#&p5XY9&KlajUE(;ns)+kZp*Q@tMCW*C>OW#;ZS!~7|2oVNfC8Rkr!1%^
za^!B>5f;%EiStIN^lCDE+ZLk*o~~O?G>5Qo%8Z6s9?0!pFp?jV&fVVC)AQsjdNg!c
z)-$C&+s0||HJ@h7er;#%Pv4V#x|ChGH-=t$wu0fo9(KHw;+KR@zF)JmO4wG3$P!TM
zfFIRU^T%NK=%Y$ReCG=6((D*x8>`fYFj0Xpboj*Jw+#`80J8<*nlox!{2fGv)Pp1?~u1EYK*3|q@p&)?`Sn;ui3})O!kx)l8cUzka
z4u52qb}*j5n4GjH44aOCXw`_$l@s56i6=t868LxUr@F05DF^Uo0y+Xc
zERw>E-!MVrT1iGQU_vuCEsx?Ph6s`FrZZSj%G4sHux3rTj%`+)=Sk68YpnuU)z$P<
zfYDDkoea=V96N2c>5
z77I+Bt9tdEZ?mdR=eoI}`*$F}_=5;6P23I=R);rWX$bWD-psxU?qChAn=QrhB4XXF`2%egT
z6`n{5A`l)!Ptyz8m2zEG_{vg#FzD6)ZbWhGIRXuF3AUDwALD#8T78
zV5Akt?U}BbpNOl=5lK>G@0}Y-bR_l5xCF!YB@;0+2GoFsOC^vk;6*Kk$KmNK1ocoe
zeU9>I$HexNu2MvMv9|o?*aAX>!rgYCCY$0~jNa#=`QvF)WJfbHq(s*#$F*s5t3xk@
zWt|LJra%E)lIRUJ3R->m09M$clW&reLK}s^7UP@Q=->h5n>V5XyySO7i0X+Kc#F)k
z7Fk6$HGj&?#+$=hJnK5u7)auERF%J`^n84Bbu+dvBNV39D=ozu=g;%7vrm$7
zJ&YM0dqb?Ig2S1WGqqPpn8w%-#izipu3xg
z8t-jycZ-)$F{_;#V|qMBoJjhH(;KUw?+3f)wkH)AK8$f~D|(ernzo55u&9?7*O&bZ
zmzCa*8ee(_o_Pskr3XhBhG>hfC#im<1D?0v4sT1iqLHIq-`OakySl#TYTYJ_%vjn%
zUq4QLDD4)~I{hlCCyAPxD#N-=S!!%8w%~<(wxSrUEJ3&2Z#Q;PbI#*ES)}~5YBe~t
zp7tDW+YSTEN53B!)J3Ra37~ulG8l@w0+ymnXZZ^=0BF2eRy%P~H<`@X2Ikg2P@Eum
zUaM>&MPCWocy4EbWfdoaO?wz!
zM*+{Pe9H6dvfs|Mn6S0ItL7inD%xZ?Zk?*4LkRm-ivjiMLlIChWCUzVZj0Rcou4oG
ziOZd=znq+W*LgcpxqR{odA&~VYNw#kt-R}z&%#M#NbSOWxorBTU$&BX%3u0hq_Q6*oX9LMKUT0~D?SEh@{!{qOmDYlY1_4S$LpYKAX2?L?ZH87eVRsEkcM=TenaRQ|9!x0V-<3n+3
z_W$Z8+=L-hcXpZwJo@Mk0YGM!oD^TTod4HC2Z7ote@vsKSy=@C5~xB3IQ5u1r9~6t
zg|!Q9$^{Sp>-S&237}<)aeC43iQ$?Y9|)M6KFMRrmF&fi;0ju+``CDzNk)@;b=Xt{
zoJ-rGA(*bOxo!mPOU9d?!ld?#t5|%j7_zmH@r8yXDa-`1WCo5&k3C9BRvnC|Qr;RZ
zSHJXw_+exe;Z2vF(%kVA?_ya6bn!f?(7*o#7x0YTWjYdalt?%
zuudwJs(^xq@U{p2=4U@-FFVaKEd}wIC@e4BcD|O)RZ--Sd_NSgfWkroxcODpw2kfb
z{P?M6naLy+58o|&l)nsy5?SWuwzyB?O0gI_ZMGTZ;-4HI4eG{qj;S#6_8t{#l(XIL=3KVKa}C46Dee!a@xvUy*MzKV_Z3@j`p
zQ>k(hHnOq!hl-?hQSQl@wZmKPs3GVZM^q7fh4jqRZyGU8(2L1_#4)!&4!NR(SPm6F
zW}h&3qna>1j3+v|p2L%p@u6jT{6eO(5vB<6S&6
zbjLcQjtUMUfYqAa&Cz*iC!1tU=4~=&AK&M{pflBt6bL!ZoGzn$UomaU3U{=82Lovt
zMh~-Km?(ke{5+&<8CCduw0O(b%-g3Nw
zi!Q=8*Dovh5ooIExrW_>5A-3*vi7$7q~i^Utr8SxSVDx%
zCiQmEnd79E&{z@kz+43GDiU*C9-R*b-#ZjopzG%MUB;L^OUe?PIgd92ac^$jm2!5>
zdC+n_->oAgJ%2_XH%43BsP=+>hlcL`vqpF6;_$e1Y5Kwj{Z1=$$CKn^vn$w8^09we
z=eb>kb$CuzN>~)|Jc{6{qoc*#;SsxNW)!jRE}Q6{F3*dpzwTvK_o>Hs)^K5Y^aQT(
zpQUIUEF*Vpd52jZQNZC0N2tnHzFLy36e_B0m`NMQ0v)R>63N(JfI(xW{oS<>m2XT
z)c#u;_AQ+|+z(PsoWFWc`YoAcd);T9v{e=YWER>~+XG{BvfQjQumH7PZ#1jz$GiOP
zR;lPHn3$MI-TZ95aM^b_2F!Bik1Y#Y`G~N$zkU&%e&T=Sn;(E8-a^honUDn!!3~Hi
zbpwhpoO6YnKDV*YfI7Ry&HP^!XPp}4i!HzQQRx>^_2yS-BSlGfl;%uXM(XdP)Z<4|
zQQc1bLk(v_UzbKj=jbNgv{@5hhf}{?q3IHnX?zK)SejcXm)h7Y+l#W^0+A=$CnVA0w00&ug*Uk*yO`YBjw&&^
zSQmYswCVS40e^D0aU+Np8qFaS!2$SqfqZ%(xj}2^dVkDt`5Z#U;{5CXVjCJJw#U6m
zqYQv#kOg(lbI5h)_-)GF+f=yMXucO-pWDIJ*DAKn?KZyVes!DQHAs-70d`G<<|C`M
zb_>lANNQ5PKM{7Myua*rb-N#7gFbZB{KP`imiwUK4D(c(?VVaO*ZEM(mSW^+m3ap7q#p}(WrK0g;7N@C6vLr_?
z+r_v@)4DksMUU~nzN^VhcWERSgFvgD(O=K{50BoIl6PFfoWmU1wih8lnXVYm
zTXITsjx1W{KOt7;!jujal-!?OmqQ|T@LdT=Re(*116SbmvA39Bi7`r}nNvN@3{6~(
zzI|}R9%E182r;&=7M6hG4Tq(dL69_%mqZRGjtmCN;W}qsmkkjm1>Ih|07Iu|TQo8=
zivK5Ln?e
z?!7F3U4OPv228vNhtDhs%}*xg!)EIpDcvsP9TzVX+{=NL-V;4yZG$
z!h@uSX##P21|RXJo2--+*F%Fr@yfl-YTH1!;vXrC
z>(_nB8OrVu#L#&7QXbIA%j=N?DPJ^QZD9{Pzem<7Cx>ooIOj(
zNMNu=56EdDFn7Cw&kVxO@bMQE6P%_NF#a+Jt(Vn4PpfzvAGG&BKFHwKAAa}peo+@Q
zMb#@$`?HFh*aJ50`;S3=1H(pHzfDc7ao9cNaJ&H@U7zbFrBEov>p?FW(SQpMN!Yss
zEpy=8am;uD^!A{hm8fZ5sL>I5h5KN1kjfDnV@PO4wa;H_ty^oPFM@ZL
zWXoB6@;BsYXU-tj%AaHl4-B`;9dOeBGx51Mc`fs~tNPE6v^vmo>I9tJpOu+cV2O^r
zQabO~$xlE$z0mtI-;2qoC$~*2L!FSm9oaTm-oLa6lNy!`12YcB*suX$Jl*$Pj5;-#Fg^$~8_`H{b7*E<31|bj
ze0V^mAGK}(Kw6ux6PxXa*(OzmpUceW-{#x;=DtJr%bGQ&iI@{G&%7@Vy)P;Qs9Y2P
zSppuQp6?o~`gpFY^Z9sRr;`OH*M{qEB<<$@^v(Kz;t60X#D3gwzOPzi7AjX*-2vOt
z&1cP$i&KUxb?JW-<W^!Diu?*XBeN%r0uKa?V?&#Z+hi%HCzvm}Xi<#X`z4
z9zSQLce*I1M2jqnjst{VppjDP)+4?DD#bkMGRZ;r8m`7^Djru!4ll%6+FJ2Zl9b%M
zs5LuXB4J}-?*yjQSouvU+2-K^xEko@En5O?s}VikEMdgblIme^P&f?@O>K3xt&~Vr
zLlZg$_NAuk?#@1eT^w>yDdUwt(HJA!wlH3PLBX809#V#vPoxyC5M9+n=x&0N2fci0
z9$%#_w;jbr)|yOBQZh6M+_D>1=HQ@74>=JjV$?54`i!!)ily#E*{-=aQ@YE7aFG-z
zan{&pMp2!?zz045KIjJj8W{!J7*O@tJySz9b2vbx?r=&~mX2-F7a_l+NnB&Jno4$z
zuUzBLt_b7<4Wk9n>&|?m31pDD+{i!-nnTi|J4CTEEsciA<`pGbkOV?mDv?a2ZjBLHr^n3vAVhQ
z4n~{QN-R$aQ+P{XFp-)7gMQ(slq=!Kf|H3;K0-gpkuP}}Cb|9dbW55{fkc!U3yDD#
z?P8!%kx&&L9aGsydvJ~gL*941S3SZBsghykgEi`xB4|vag?_A{uF*c{BQL*2Br_;>
z7>a&*Gcv5)(7{#?QXadU1(B=*km6(qua3(nA>O^SLy{>QGBF{;%F2orX#hosA&~c>
z1+LxF)<{dI>D`VuD2n!1KNNPqztZtyYO-RnXM$tTvO+*oB-C_g<>v)_#d<$v?^K{c
zMOd(U?w)+aMN2rL=9lXTxd_Rq
zgE+sWDI4gq0ka7G6aR?Y7w>_{12Zua*?3Ra7GGc3_bZxk@A^-1%sCuS7fR3dR@@OS
zS$WVIW!)gE%dhP(H)t6_>o}(emE&afzc|GY7nya8DH{795))n7`8pvq*wMmkr)Z?&
zY_`1X0Scah|1g~LrPhi<_s@l*CYW%;1&#y}Q=Gu}W=5rQ1Tea2utD-K#Lx-+qZO1c
z1aOC{jnEo@7GQlV?=!)5-AWz07^+_6YX`nRKV6jFc@ea*tDW
zW!7z%O;zS!dx^>gk&vnhOAh7FDB=%i?(sw`_4kpenEb1!1nG;`RSmoyvm$%EZXrXk
zqLN0W^SiIf-1BJHO-smqbnWZgN4%^tANIX!Sd2)m>*>$Q&el*GS)km*x^k7q8&X}Z*d9tx
zvf?d_36N~B!#0r|g#prcB#Y!wxiranKrbzf$?DhHPo@iN(5^*Yxwn>5Rh^&(NpztLqW-BQF@+V!l<@>n42@@Q@My
zcz@5irG09Ci?s2~&#Et2s5Ixad(igH0{K*Buh-NxXGNHmWqIAJ{io87exKoMrO?lN
zUanJmTT`>BIrrQ6e*iP|)JFBA1I7iJf5l|~fLYRi#gIu2}ky6j+}
zzeTK5TKQgWU9LvfNI?5whu!&vJ#mxq(3bM13nHp3m9qmSeg#w!Cht$r+AE0y)pN(!
z!TekYN*Z7P6Q>PWf2FC)^16{zJ&J{jQH;}Zar_PA^iZbeD5r~@vm`{WCJT!i$bz_f
zE&yWU{3Zd&Klf$}e
zC3ffCXvAL3&W>)kqako!X5v5<)x@`k0K?cgIYY?Vx*R3KiC-U02puoG$WGc1a65
zIlaX5Li2!c3B6z;C!t{ZHxMO}X7(9}#7y)%@Atrf2#`71;hi0MCllNUvY=YvlH91E
z+d!#52Wn48S7w1L2stUCWRgNdz3QBccUHka^JT#mOWhw$1Q%p&4|RiOrDwLYG3ajf
zsg<5A#smUfpu5xSh247xobY{E%=KSxh^m?zF?=6(#o?ql>>&a?!s~cFdCUg3W+|v-
z$m@nJkHQtVZY19nLgNiFV}q}CHW^mC5R#5)Dwh7&@1P`sqtzwg38xXKUm+?Lc5BBC(NLjoTL_EfCe6Q5$w;AFA%^|I;MQohDTLwYqmoJ{MU~
zaRXn(ta>4?h)Ni?+p^*?Q}
zp1fYq%)rI`@Z!1uIeq3+dV2`?RM+VSwTb~^LZ$N>UG_im*RsQC?z8*BoqCf7pz`A6
z1L{oQ{Lb2{3V~bor!{xQuLD4qraM42wW~hacHos1`=%Q1BQ6R({##WphwU!Vr!Sn3
zJ_XHkAGdOs=0tyF9kVN*Tk00Hr<=Oo&UbD)W-<8yJJ1~u01%!0PbJ+&q2HgW#FP39
z0L1Qru-;Qz-m6f#HKu8%?yC2bs*Nj?Q6L}b8dwpoz)r8*GlBOz0KA%&_i2-N8NrMB
zdGmF_9lHc{+E-SnU}kEN91<9N&XS*&@k{Ihv`CgR<>i&xLTD}WQGTF#%YvZ4Yq|YL
z20#%}U%W=K57-kkiDD3yBcDntOs{#USZ?+p%6`gwK=|-twK}iubtRz7pQs4p&ME77{+8CDKGeQ~!EX1fCs#2Hf3W+GF
zLd~fO7=L?Hia8>GqcJ3wRtwW_6%eF!1cApP*Na^XV2Vp?p)bk~GDYPGSq`mh?@Vjj
zc$0nX1{!e8xOM-o=5*DF`qT}%GLw%nD~lJvn#kHJ@aXJPlzee=O!?KXaQyonOf%j%}qmg
zETka@HG~_<&~Cr=mhkW5G=pH{uHf~S9tyCa#0;G+oa~AQ6#=sjhlXV6(0FcnbL=aU!s3Vrzq~zCwCWkx;v#4LuPn4q%X-yqZnedR-MpE=xb(vqzue#~{
z9jUM@FVV)*Na6y8$BUxJ5K(z>p>by`OQ^H1HBuw80fM_y2V=um}TfpN7knXfC&FeKGQ
zw(bk+$*5{D-1O(qD1jo9Qb|BDmof9-{@oO|%~yw*V3*A&>i1`z5B8yGzJ36Mja2S-
zwG2g-DO$fvoaKqwulBS{-(88W_;R{%5bx2#APapu1S`L6Tv}Q7($`|30WWB%N{t*S
zt}ebRoCZ;#DY|-NYUd|iNvf31A7#oqB)kn-ae#Ba{XIRP?-mLwG#;WpiXR+0mZY$A
zcghHq9%WFE9}hcl{`Ytu{6!jW)8=obR$`g1j}q@6
zzM}c^{b~Y8t=`7|C|MPrXBpm4NkSv>)Q4WW`^$#76norR00y-FELxJGXZj*&4N`7W
z&UlsV!En4#xvBc!F)e#i36g!8*x#=)(-GrnOYw^WS2No_q(SF>UuWYy?{6#AFRl$O
z4X0}hAOyJzSUkIXj2>WZ-AG5b4?pvAj6dGr{zHC#X7YGoY-P1Tb$wpUV_%K|^=i4_
zlEu*{X0kI7?qZ~H+W`){MmIl|+2E}B!NJ}F>iTPe3Pn)F$cN*nUNH)%$sgYo
z6S5@PzoUCP*6EDfb+EdJLv&#pny?Q>NZ|Et4!n9!@iVr+8&+(7gcYn@E5Sv#U#H`J
zD^VU!nwr~Go-%}E^adM6H7L%FJ@(p!lrcWi-gj-vQniQx8{j-=1@Qk}c47$>I<40J
z>uW%Ydjp|++cQ&9pWY@ksH(j9tl2(GD$DE_hwh@V_z)$U-wVS44T>{C9w{1ll^Lfn
z3idn32pdSy$BG)B4b-*)sc=wt0K`WD6n^NiXIQfiOOJTUhytvXnn(_C>%Frw7^|Ev
zs$n2Auv~rr+%RB4pds=gOaqUJr4et*0y_i0UiN{D37ikDuu2>Zj#5Ii*AuB(Qe$w2
z6@DEUh>W22nxLOCgolm@;O~CsBXF!iH^So;APl2DI
zI>uw_)96Pn$Px#uAi!A;g@J>oYFW*Ui3~9-(6)lM->U8TPaOrcoWZapNSYLD?$)H-
z5JLO_7DAB9i>&G>l##WvolYi(K(_^81aYnyD&I8{Gq*pbdXC-T5!Bp^PK(!iHHdaSM
z?UqA!ifg+~+H@6XtHRtSvb3mh0bzqSh*g+9ab)+rrHtlzri|u$XMf;(B+8UE5nGw4
zC~aR9w$pygN{=Cf7rZA_)!Q6p1WfQm&;sI6#rov>`YJzGhB{%9FZxQ|W)Jjc7d(cy
z5ut|;q2pPElZYZg;Fq9rpP4u?S0V$s&$bNvlaxjYtD|Rq>JF(pr~XAwkw3n+??mpdd?Tj#Os#?N
z@B)2uVf18Fg_m<{UV5*6*{nsb2on@0^aC!CWCFP2s8Cxbse0Ddp~j?zi<^7V??*ni
z2Njps0RBua$8(29_+r7=Kl2hjXX+gLoiqv+N)uO6+MV8lGrDZse!(5nx@7`ZC9ylw
zv*t8JUVK&bdY$2EriT;FnQ
zOD9(#<+RPXiD>a+?WVnlgEu-KDZJlYinI=VrCCD`0%=_DFsa6Ydg&gI5)s!zhT)I$
z!w=2PjLqy7@mafoZC$_Z176WSve|-278EKLMseicNB2Y{LYuOwT6RoRxIwJAzmdcf
z#y4rZJ{1-cwMMLMT7S#gAm*wX8{4`}ME=e%)v3x!-;dRK#9$lM#f#BV_@%CylMH6!
zh=sqeUs6L01=_}si&N33((aVE|FibA+f25x5A^XN3$r1Y(osXy;H^?RCDySBZo4Um
zrJFR+zluOF?yTK2cv)ck*{DAFs`24HeSm6h$s_oAz|ObAtfOx^yde%XSSvO3okdc~
zNtVY?Tbvh|cOO3g;%Zqab@FltQrlpg&jL6HvpF1s0dyz3txQYaVtu{Ub{q8L7N~Fa
zI3Q?Wx1RbGo&traKyJWEBf+7zd={D)+ki03t>;^5HkZYD5#XP@#17d5V$WJ3#mvvC
z<_)&*{!c#o89(snau;|{=dBx_H!^_gXW!!h|2@+g@G-vaJ}bR7MZ2GsIhB=?gGCR*
zL@I~B4@-ZuZZ&+DG_OgKHp-A@t)M|ALsH6->-CevO5UOj1f(^&hcG6zW8gyP-MEL<
zDMIjq2_+fia!Q!-PpJb}#f(%UokTiANtKgNo4=%Kp;F6?XnMl!KL#;cN(;dgagW*zg
z@#eT6Zo#wBQ7!9}(ELvQ51~LoKY~2U04OY1x9Fw+7KQ^(uz;q>MA>iGYs+4+Nyr#A
zBz~DeS^WUMChuF>3lblX(C-1&PDuAlt96?3NLk&+7;`9^xSS%?+
zsVu5tKpLN;p1zE5{MR`Je9Bit&i)g)-~T|#*@Iya^!7*unZuVw(unc2;ML5b&MuAH
zo09D9m<$a5U>9Oo(Z-#xXWz`S3&}|3ktNH-YvV8w4$`pcANT`FoALyfpsxd+K)K^Lo7F5l
zt|y!;NG>LG6q{m#!RUOr{)v6H@sU+%wMg~3%s9(z`-3zo(l{d?x*NzICBfvv2ch>l
zj{*FBzPtvTR-K)oC6!I@z@}9*E`HW
z<^2!bTy(9>!l(z}|9GYo6nU~c9O@CzB^k3D8@;3WNn??JT@cfvLpW?<&oJ>?BU^~&
z@(EM+NfgybT0jbv@eg{zve70v)*&7ioiyS#e_k1uG7EQYYPJci`jE`9svsLZ+RWpM6f3%Zm>DTbe=v=tu^C$2EO+Ro!U~V{T27NG>eYW|)}*
zXZpI~Q+a=AFSc}F3C$x?LB!OT5j0y=@ItdN2FSujAm7qjY4OF=j_3SZogseTd??-3
zvTHI34UjqKCk*si7PAWH3S`7cugFO_*2^D~VKdWo)f?pJ|D+O}G!R^chU
zH6`)w3;9*)%$6^Sl@0Bm5+Ppkm76VxuRM#b^7B>)2TByViz`&~FjZ{DINp`NPM>cR
zfrB1wv#b4jl)AjU{2y-D6>ghWf;u7p+w6}wh3yQtrc>uOo<)Y4iDPT~rmCu|$TIJX
z@&*leAmj5QRf5aB=}_H6;#2*ZYLeAra#6)qi1VckaAIgrU%pOW{=2L4@oEaMA#IKS
zZ>#`YfKFNNA+8e7{rrkV&N>PVYUPh7_m&q~1PK}bVf||?x2R%0$mE2obRneo8{w6F9k!6%@S6=$REb+f
z^q)?#lgS!gk!Y57%~ba_EYVJlO>118>kp6(b`#)WVVJtHL&Sc3
z4?kfXVBI8y6}(qDJ&1LA{b~DQz=Fq$(sxXnv1Tz74iiP=>^AwaR>}4U@89T)Dj8X>
zzi|%qU+vBZui{FmmbUCbarS!8x{#p%(KMAqo6I-xgd-nU!;o~
z*cnkRBj5iRBs29+OwJH)0O@^M!JkIuY;wR&#!ddp1s}Jpdud(Qlg@RNvCJZWm
zD}xUjxZm)Mq{9&;b+n?KhP+{F@Pm$h5y5{ExX~INRne1VV`gVzc9mk)_6N<+z`+&%
z)`ut}gIav$jFf=n3GQ%$Oizc01dF>$QNj(hMd)4J!HTFMh^?ihGG+NDwf90<^t(yCPH32Gl^xx95Da=`q>Q`?)ydVR3
zcdwwWZu-8L7(sQ8z;%
zoSb90RT+u4G_D%~X7_JMepJf@U9}!Vd>^kZ9@w@N81v;NXEIM`O#|z3WLj}?fr6!4
z4y-tGDh0tm2>}^nD`2Mrhm!bv-QmMvHEVvXIu{_a7NgJnl88mEcSZz#>QY5a5evc-
zkS`DChugapfRkE`!)^y0Xzn03XLhbnJp27UQraC>N-|V9rk@Je;xt0cU@}|7$jm6T
zF9e~wwp|)fav?A*se0TZCN%eA7Xc#oy#b#j**2>hm%;cx26g)~u_W!2eGCW_u!NF9l4CKq1)PLzw`s
zr>>o;jYkJrpBEaR`$3(5p=`W8(LUEhZabkAOF-Rcg>$+cH#xx&Rz7g2!Rw{_A#fLQ
zGqrysznm;nnGE(IQiy4l7kE_e66$hJ34j%#s?=cqfEY*L7Cn^Tb{f~x;FBFsrv>f^^-V$=z==bJ;
zadJ#)sOee=@=Hx%XBj=91FJi0T#%~ZDJ^qMP-#;vqDXJnULQA5na6Tpc&RB34E!@g
zgl$9zdCz5gwcMZf{_!jMWrT?;OU78nG69iyK$oj!)Bd9lRssgm<{YCqs!OKDxMRow
zCo;>MJ>|*HV*9_MmIWs>xqvB_ZuL{qUrnhRb3iynHxTMHHT60^b>=K@zFt_m9j&4G
zm2a`s$!R{Sr0^5b2I7+$yn!rZ@1}5tQ?Mcr2>0OC(|gS!KsMRdv)h
zCf`|Hef*U;-P-Pp6(6`z^UE!t8(ISE(h@sIr6JVzW1-s1i}=I}*=w*)Y}jJw3LY)*E>G%{+2zZMVGA)b$&aV_$JF9Ag_ll3`4Ip7Qc(qSo=;>
zR4(3GLEs6A6zhPFeg;NGR-wGg#m)JsX2EmQeiN{Nr0}$feds-x#u=JAR+*H*`4w|b
zpP_NWHS2DFl4499@ib6?zj&@_?l{IwsB7~OkYD2QI{p^TzXEX}TtT>Mc}X-*d!V!a
z_kKdYpnl1F^vr2N3m`27(H+#8Ky~D-braCGQG~6Q6#ETkwe4r@}`ns&)P9YiC%$?SQKG0^z>K^49V|hH6rF2KX9Y_h9n~>C?kUNzrmVR+)~+OZHk!7iN&Jp`Z~UGpknEO7%^WQ>6bpBR{{Jy`
zmO*iL!L|+r_rZg^LvVL@cXtgWxI=LF!7aGEYjAf94grE|a0}<wDa0OZ241;vsgFZl3QzcU
zNZy&zl4h#*396vr-bE=4J;<6R-VYrs$_y^R^0_2ITHJEW@{ctn3oyZM+>&IDevM
z7DV|L#@yVwRhT8J_%}^L7VPpo-uRJ+P@i$cQ{ky9
zi}n#p7Su=)!%QNhHUwPVWoC3j!4Og7^ezYc*6vF-`rQNjLomQD1v>{yBO)YN-2%#_
zvI!9?Qys)
z)~H)`hCf(6Zt%#zf9b&&fM6}|02%w+xaavI_W;&v+Ueg61d!q(f(CUv{=xqbZCY~VN3}ZqU`)*HO)GLVKof?4S|O@PMQl%CvZfvdKytIT=WZ?aZ|{I
zSY5!t82S{4#M@yGYV@SH@-DUBNG|?gJ{jfp2C|FnaX~Hh3x1Q9b*IPfN=99TY~!s=VoucusS+AWJn+1{h2R)WWXxi
zPS3yBPS^eCECKJ>Uq)}aS^uKueGWeV?B9CQ%iY1qeSiWadwKV2xgU^;6jU-<&ov{2
z@*(R^u`N=I2dPMMp~7Tv@Du2vJcT+UnHG(_am}eL^>Oj=bcZZkgmbp~d28K0Punzr
zE*~u^XWkycNves9ih00j@wD9X?*$z9I;YE32N5p=9rL47iVe|Y==N%kEEodOX3m&D
z!))28cx!9eqxJhJ8CQDfM7-)p6m|EjXh_nHZUxkOMCN~jtO9a0OE!|uP%*){B9*B#
zgAcZ~RD&4{Vk`5~7FBBn_+@J+M!z(Lz#D!m?KSlkXJq>)^Gp9ccI@;#?>`QT`$(wN
zI5F#e;w;CwSuuTU4#&Wcfzjdbv&PA6z2bk411+|x8!6NC~qI2n>q=D
zHUPh!gPCBR5P_x&5#72LE9m_9PUyEvm1EWWd1v=i(`Z+3=G_jH`L$ao&IH&9`|;leUV`JMldYWkCvb7s
ztkV1S-_d{TF8O}+d$A+pjVGP_;^I7FU_hrvnah;Uz4uqo+XR+vvgAVKzOX0DOGmh~
zGLaaltVlt}lfuF~p^5DAF_2bt;`+Uz1iBgFT}*_5Xuf9YxSo!V+eNSW`^ob&>qEC8
z)XMFBkNsFdZEWdpD3PrmXEtUwZi`aJn8XxirrykWJgD`hTR)sY7_u@9RGi3FaSQo}
zLqm4XmIGu8qJ`@61A@gVA>8S*8ynd(`Fo(?R2M0nSROYNAtD!FHzW@oHLJEDioyt>kW0SExfmv?tK|G4**6=c15!f^5F(z`%t=j9L4NhWzuTlp9P?gT3U
z)pI2Hh!(2y`zDJb0ybL;xa`fbOW1vc++SrxInO!dbi7U@X>HcSI7+9>PC58=b6dt2t@M13fqllIIxb8!PxYfvYQJ^Thl81U8OIsIi|2SN)Ki~|u~2Tn+1oRWCDPUti38n3&F?GdGM
z3SaPOF(Q?lD1x(FpNixj*53V~@w+-RF+UBhPb6{Z4Vwl@7Qs509SGcDoUCa
zR(_1c++qvi8HW#vbIIfWg%0Nzr%WR$v&Xk?r~)V_$S-b2y20RTe{4t-Ib8mW4
zp4od>scb>m?7&Az$lF&C=NCLI^rS+>>lPgnA`$rGn7`{@{cMYE7fCPX_C6OAO4T&*
zLxCETQTN2)^?}F5hP&tGG7FKQU$j^lD(f$M44Q~3e(PK=qqi&)X24V#7?0Z)o@^jr
zON@75azkW$BiQ?ZFh-GvgoaL!IRv^1vFq$LExNnv{?_C0*cyo0{H>ZPtHa*!Ui%_G
zGqoaX&!%to*EAhis%L0FNwt*~%-EBPNOhha@%pEwvNvFwhgQo)wxI_4;Y7G?)g?|v
z!?jJqINlmR-ZSjzWnIi-`4`a)3M-0>mp;LSvm^7cU70$@Z4bQFVvQ^@z>Mo
zwR1xn3L?@X_?&D~(XN(ED%1u-pKU+J(9bMcD4S+{S_RbY{JPnV9?(I7HXaP?$PE(H?{aiuq7@W>$}8zISet_tNdW
zGq1$GYk9pb88D{XQDL8p9NB7iGM3bJQ#cSEvex~n^lnY9PPZ<(Y3*+1v9A2p`|5Ro
zW&L%=x*7M*+nkqvGZ~(^U*~kd>SU|o&+~M4-m9jCz~04Tx$oveTk1vq3txF>ydM(%
z=9+!Q7-H?qQchhrqh_7MUknw-gzPqjf7x}5?&1F#G*2`ese2Cq|9TgUw%ta&?V+dD
zEx=3qCF9=Ypl$RxrgdDHOE~vV>H|1=57cjax$k`guuwdF>nwEd{89pF{l^jLz|cTs
zgQ?|!7vyF2-^##F)aSqBKVQZH0SNwm4d5MD*kKUeVfgLPilyc-z^$e=$g;&y${(p5^;6EZ>nu6sLyb?
z63Wy_99b4Yd`(?w56VdR(B;}&`P1ES>U9c;qsVrPa%&j1W;f-=10qW#0Dq+rbM+ohH|w%>+6yM+GQsnZ|3dTij1zq-F*rYhIcFH(MA0T
zM+z5A0*+Lsg``~%##$#`CHlU8zh7cy`cnJ_g+s=6G}@gTt6?qZhl|c4_Ks-JDv}Nz)^346X#V7w
zF1nGv&Y-YV{8K&a8GC4UNcV)sx>zM3w!As#;Epb3V;h+b6gRP4LGyjL;zEm7qJ&Qh
zQ-VFyKBEi!RV3GgU-ASgSp4xiQTam-&oXrp6lytd*usm8tj7?9z>vro*nX*Z?w?QP
zAVNSgI_X9wdr@X%Ze{WE(pR%}r+{z5jVv^Xq9NK?Bt-zNO%szf(kyg0m|{`73{KAg
zyFdKLiiwG5()dYc<=2~tU0hmdky2&WTvBTH?rQbs>~G}*qqb0x_Q*CF
z=pi6j6r%kERR8c{VxiFpRDr(UAlQifL$A-T)#KVDJ{_M4dg*&q(WLQIq<>HZ{ri5u
z<((T7Lj+j30M=0$;Ln8R_An;ScKc{o6{M?nR!yD!uxjZL2Bl3_xr%CwvLFa|PmXtC
zJbZw*t02w~g_J+rw^C#G#gbHpG(Ox?nRL--frRFR4J-p9jq%d@t&aAz8Yt|Lx^nB2
zK3xiITDJWR_(>WIrbwMHDB6Tm&Nke`mg|%(4v^TCNT$caOSXjx!79sHl#xxO0s^f^O?RR-5~$D=nUHulJtM
zr)yb)z8L^sL+Cva>Ro!vU$kW*hz;LeR)G2QypXMNOQ#q(y%=Q`EU}%nBSQM+qs2X`
zJ*<5S%M+QTfiP@pU2_9Copj)zr*H~~ekHaniOJtG_XCYh
zM7Jls!UZA^PHRGDqG^`H8s@_bGk$Ffg*EMp_`ZNyQOj;3sBpA{8>GHyv{BFSX|kjS
zs5`P4WAP6Bxg(|-2QC|N2+=@wfq2rosqgV5;c(KmHqp&vThF8;GsGX)kr2^I$w%_6
ze-FW#Bww^rKysn%`EMakQ@cksce0@ArDFNZyQHjLT+?kFo|-8V|3F|@>hey%{cE|?
zH687wP50Wr@SOT`BDo+84I#r%hR|P6^O#w3{Al!NJ?eI_@|f-?&f}=Prv#bhTTb3_
z*@RaOhuy|ms)O%R`eViwl}2;|@%MYC_j=*VamI7`2!u|)+yKT(2b_W>gTi-9dE_B7?=Mw
z*W+XEyAxElO4mKcp=8Zj05mc6AC6Wn_X#r>*gA9Tp!x9raRNKZ4)1XKLjW)YOqGp(
zbNTIZ0lx&g53KTy9J@2<#hL4=ikz3~|N0+8_p!4!zojitU*J`N-k{o5jCCm9W_<21
zSq1(O&jEa${a=$DM4A!|dU~BOy}1B-&sfzR{eluJm;6w^BI+2$7z3nkb)Qy(hVPc*!qvw&21RZZ5He#Kjl^Q0LI=`KaJ&#I~
zScjt9GxQH6+7f)AZq?0s=D4$;9WrGABzYQ%Ld*bgX=14hQ>vQpV|M)QxrHi%)q|(P
zht+^wSq5|tL8Qsuo5|e7^b63M0x!W2+7#u+7YKQHW-WqTiN#kP&kw6lEOH)NEuX&4
za!d`?l=F+5qftqIL$TpR^FprpY})r+d(0fl18@*Obb}e1k+Y>)wv;PfunxFXnHE*c?KG*TMsJrI=z=!nU;-(0#Cu?{ikL`s>K*fCi2{qA7&@iDvQJv&V|
z1U>=dSGb_>k~B83DZfXU<&k!wf{_jxnC8=sjQ#3{kBjIl4OWbq3F0Z>b*0tO$TYrV
zHgg-v5pg{d$9e0TiKIXzafA+V|KLLUsF{-Lxt$lr{WweQae6fj=ua%VRZFf$1zluz
z*`Mmt;Upo%e|HlC7ZOF-B233D)?`q8zKSU9m&1WH}D@{IraiLd|cp)
zZhN+yy<=^xRym0#N-F-bPi)yZr?L)pb~#M_f9?GJEA7&Ync-=w<_nJ%wW8lYA?U#_<0{jh*HA@4ULvth^0$Nt>%<8$Y1zA&x`
zOxk19h;GRU3#%$_{K1i^Z3%+qW#f-)BJs#ymSMh7xEYVYaW;c!uxmGoO-WcNxW9wE
z6G-elcRyqf2#rtS(w<}}P8ir*(Pqf_@m2V1_cPQnu;h@l>gn^&rkPfoC
z)q#~Gfm`+IYu|1~^mUHvXHX{d@5myYB}_Y7qfD_vE8GgzIjiQ4Ru)T4$RNznD_e0!
zY21p?XefiJ7W9-pn2o|f7FOycRImejEzIZAxn*B^zb28deu0b5)3cdEQAf?C;vf+Y
zVQmppb{X`f0p2G1P?|{V7CiuhNRnqrT*Tvx@9pqxGlcSn*|lnDNv@z~Q+zKCz{d~~
z5hp$~)UMjjpA9{9PX}b)v$6;dO>%7v^d30u7hw#iXLpUr4f{{*^m!Wc-w}NEJz4X3
zUh_}sIQJqfDAxnJX4Mm$xU=B?f?@6FH1;-PDk5wveOQFUW?3~$o8!8*`>Q+}2aO2G6
zLnz>-8)e5^Qh#~Ymt?IDP7rt8Tw#-<2fHk(*-7AX9yL>2H@VFp$%8@^m{}<$houu*
zpGfQal(5nv=7Y>pKsln~)R#9pbmaZJch3?ol)hZ?^g^MwY%&%7STBcE%hHK)HO`D7i0t)Z;y6=z5YR
z^Rz){&G7GzJAXh3^VxVOas0cS{OZ@B-nmBga$NI|4#-5N3GxS?ppI7XX*TY^MyIHX
zM#VI0nW^dEePMwSqxTEO=6(15)n1a41HOrG#i0wY&qF`)<}7dL>g5P|+@&2Lq7_Bj
zN1&$)m)+^_cnQmRZ3@2cPIsOdT+j`BMxay5>QhyRtU9OPxO#tkd2sv>3x5E_Oj=#YnsbF3DN@gKTYtF%{jXyh1}8XU
zFw@dvRJ65>kasJ2jauCzmHnm#XKiIaBs{>5*#*^CRrU!9;eNBGqfocv4^+FGT?oZ!
z;G{(Lx`2nKVRmm63j2a4StP7jz*+lqXpLi4ordbFMz`(LIxDM6*L>0Z+3cB%w!Z$4
z#SBYR)4v%@N2??aC`-4m9IvI)M6CcaXE2$GdYZ9*{g|{!Uc+g_chFI^WS%m1_a%!a
zl8|Q_mtv+=D-G|f(_V_ey7N8h=C=b8hgH5_>xNKOiR8YDc}?t+CmRO#^2CoAf~tKJ
zPJM$|3ZLsK=0MU>GK|f-oUNXHT8YB6wmD(4i(Y4VeH%?;oCkvuRCU-!vXDwVkS?+?
z`fGB~+A(z?RrOEsn*q3X^@jpOPWK)9(T4dp`?v?p$ZiE+piL-63vJs!n!oiaNk)%T
zd55rupn<+kM^VgD+`F%3kkAH$3;tRgIH`!w!p<4C8rB|6_7C2_v
zjF;!Y+HFchq-R02SLm>kj_$x1rJ^g>1~(c*!xPCR2@MXabP*U${%5zU)n-6?3M;lY}VzPy=n+TaLw|UrRwDmBEnsoC5yIo
z_3fg|C&e<#SvX{>KN%Mx<&$%h8|Ic!78tK5>kUtQgnmCY9}*{F6mGzS)Lw
za9EfLX=SI_cxoDV^^`@+{Q(bwtQoY$By{mKB*Y#Y$@*=rkElYW@u3p;4=D=BcPr0z
zO)U2zajXuGLP4wAmCVvdBkN@2S!!uPt`fY=Kk`seXM*7wX=z*YeKJOGghqjQl
zbMC$XJqOb-jS)Cda8&@
z<1Rl+39_%K)xpu$%8E+LPd+)yJ~giBN*2=192p?k)#7T6&16}!VO>Y-X^u#PW5PwE
zNYJ4D*I%vBmRj3OZVi5Qjb>2VJ#aY8%p{*?s6B_3;74Ow?~;bl5+u=am*5wE+p|)_
z8!883eNDHd{B2q>`o$_(Q=PiLJ2Qo*6h~~;4STC*TY*qmL$bC!;3~rDkZ6oRzHUi>`a8hs)nL1`#8Rxoq{$MtLi@&l6L4yE)YH)a9c~UmS8r?r-g#9UUfCKl4@h
ztg;!;e(yzysYMV$wzwz1fg0J+{t8oIIb#Xhy;Ww-*UT(X)ydPzH7RPyO!q`|HqouJ4e)I5$Mk-;9XxMT{IGclqM&OcvgsNi&Bh_*`T8z-C
zdU!w$EX)iIr)fl^urvpX-)>aeCq
z6EKRDx_B4k8NB3x3N|kOnf@Cx+|{SoccfntS!?Yj)$86DPE6TRmW}CR^kG7J`^OU^
z(lF~})!OZAgpG!thpEe#X&x=}%f!yk&i;)sz37dHeq;aEC23RG08s6*r#JeNYB?2z
z>j!tAHSjp;Y&2l3R`aqy-VzG>X79Y0t^5J(Z$RA=x^J6K{?s6+kNrNFDM4yplk|C-
z&)wJZ`=iC}b#thwRW#rl`3@_82UzeK#AtE+S3=%>R3#NE`Qi~L1VDDMzdD^|3QZmK
zm(2Y_%#1DK9f
znmTydrUP`o7*>;`yInEjry19T!y<8@csXF|BQ_npr-NC{{jwh@h!Y$GazWInjF}*A
zMrkF?dzUvHQl$~qw;R^aVL9(4WF)HlGe8+p6#H2pz=@XE*9gP6i#4j~3bK{6%lzT1
z%j}k|BWCD(P?ofah1FmelKCHh1A3HaQ7IS`!40pU5~$w<2QtUt`0du1yGr^3B(RR<
zDOi5G+D8sH$3}dF^|)
zsIkK>i`Bk_dUe(^AsqFk*0;c!lis_vGaEr=qisgpj!_k%Jo&q1mn1#hVF)QAVid!~
zXJ%g3$iLMUmknAU^d7)&mYK6?PuYA0^c#oPO=5d
z9AYH(qDnA3s4~_aJ&3<{UX#>xxDtmsYz|gOs90fy8COZzT}LifN7kLo_GydFRU#Z?
z1=K*X$-$qEpxIevF2)mXz^&+GdPFAJ4${J9EG$)}mhIq)A#iQE?gkIKg@2&J!;-Yq
z;HLy>mk{WitL{lNYi^gxNS5FK_6{q9{%YLzY~a=PHXgF4W4muLG(eiMLj}I=!L4L=
za3Z7QA&4hJexGC|>Jr7I>m%-0bn@fa
zPIAWXz#~wvv_`$RyrTo@{)=j#>hMfdjGBLQke?`stSS|XJ#fa0qAtHepnbLAoD$CN
zQa;W>Yu>=1xQSiKdy$r(y6m_nekXB2)hv6AtyNGTnZgV?PGA?yO$jQR%jY;ils00J
zmt`q%%=u3Iub0&s|BWx9AtHW4dgQ1gYZw_h`&a1nNg&9Pfr%l0fOK*nZc!I``np9u
z6f@DbD)!(}y`QAjZ9&mO{dM3
z61bSsqZS<-qgJt$ma{rI7$UKcjcw!3Ys}bvLC&h7RrGf_b@8W?mrHqVj*-RmX4ksk
z^BIAF7kKTWHE6^84y_O&nv0GClJZFgOMKR5imzj2bXpW4kmTDQy*ia6P8A$fF1uOiP>Zcj8`233TyyR*QxpFauRG5jk&}jwg
zV?Ss1Nu>lPHuO>;Gj;_bJa5<4f)#PQj
zMbnR>iVFF~+=^}!UAIzi(C#9Nb2f;{O-cv;#F}0uQ2*1K;{k(q3^-EOm%w#i-+7QbTiqL+NRvpSIEAA;
zupmltwro{ioL271KJ@W82Cu+OVgCR}M^~3DnheY;ugIQ8@H1HtMc#S;bP|Xma>P#c
zA6RjA_POua`tzO_1S|9aap7|??KmwU)UNFh^A_*j{FrFWg1_v0aa
zTO_KgwQ>mCq+bXv=(O2joDd3_j3*>2TY}XMz{kop5>S4rlCBBTpP6flOSSa05lsr2
zB}pV~cR16%>V0J(O=xPV4AkhY?820sgAbrolTVnx%$&>5=x#`~2uLjBOJw%gP|5|N
z!HXWcC`+ReVWXOu73PE;V%FcIvkEGZq?9fqPfBKcbtx)W?LxZ-mj`IzW*RFbqNn!8
ze2eLML|nqn7teFa&M+EG!cKS+UJBO6CpE_+t(XekOZssTD~I2|nihQ-t6)X*dm)#R
zE*lm)wlg~+lkn5(qyBjV7P@kE_hVgkWfrDygrJrwhPhkAo1K`Fhh2_V?@%xAlzlG>
zcD2Tf;Hj7c`}t}7D^a(4bFW#3z|b+l!NrUQV0
z7>Vk${c%mx4?axE|7jkGH-71ht-6(4$2|;XQS&abx&N`oQ3#py`duU;WVwXyTkKXn
z4CK?mA6L|)uOGnXU-T(dY9hWd5~M=AS8Hu?;GgcZWDOcG{eHq8
zU06z_AHs^56cr^UYBExSS<^!?pkK>L=8~^AyzpC!nO0boEdnptHf6DI>KahW=$Y4
z*qvZSE;5;$c!Zl|5rh)MAHW${Y6<8JJ4oU`co<4)9t_(P5TX=R{E?J7~m%S$l1-Z40y
zXG9~)amNnJcC$(y54IH6y#f8@qg>a;9E!ha7D0iTrA>Wm`*INV@pAMBMuORu{cD-k
z>N&`Ip*rPJz1{q68>J*
z22)=Z2^SyWXSLU0BAEGznk6j#;Fr+Q=;89{Lr0&DGR>DJ1xqz)mT1<@;}V*D@c|D$
zLchacT)o8;dew65`a2B~NCg^!d4^h*`f`IeSIed!pF6R@m_dyF328XY+43PT6-8u%39ZPC`_iuC@e!enGIq>!Ud-b+npnLEb
zP!YxCYjQN%kRXJnhsfMkSI2r#`;l>I42dsq62w+I#UtJr4d=Z~0rAPWaKHI*`9t{y2I;+13fYxU0+MMHCJN)&DFvfkR8lshFtJIIWT~Q;
zkZykrN>dhN59oi9s=TokACkhK7H9y!*vA0Eyr@c9oU(na3VwDmZ*%JT4s_o+E0wL8
zt2p{ycs^Rt4^_Jv+{}DIP6abl)Wv-$Ss9mgh=MB%**A{bWegWGZFm!{f-&<&Gh40X
z1j(eLS*q(OL|{(>Jw%`|apAb&w4)5g39=I&;1_uU*?@yU=G}Az#%IUVWkf)nn*Mv&
z@-v&O^Y^X1l9F1PI$S?9r7-n(5m^r73a7CjhZS7-Nil>i)4mtPS`>6B)n)4k3@p)L
zgz0N|NaE@1cvWuE%4B1KXhLe4Vz^0%9c!5EEOJd}>#ta4qWu2ppc>>#WfcsUuOJs)
z(i}_%M#Of37B@|zha3x!u^GHXaWv|4B3q5|bAwRwFm|&7Y&&R~o!I0)k|IgCpr$~Y
z^7)gPd0T~fhqL@OLa`Isr+>oa7JJh7q90;Uqm|)I#aZK`r^)(L7qh#c3w3`$VeCJ=ch4JJ~Ek{F!ay
zPlk6%EtaUgqd6+^&&^D5{^Yb)znkyYr**eKo^3TP`;!xw&usOZ)G#O@d>b{
z!#4vBM;1Ph{cSu~J6{c?7cCzZPHERo@D0P$nOP!GT!e_ENsryCj?d=ABkSC4GL
ztpNZMca(K|8}Ii!tSV?mt}<#G7(5m2Y;g2h30{BlJ4*E{Q-6IgTg4AMI6qr=ZA$@-
zZ~9#{++}ypn_q}uvWPsacW<*5KY7Z`}i$;mC)a&+Qr-g0qNy0|2xP0bR&HWONUQpjKE*u+NdX}nlyIr#hcIhRzl@^u7VUGN
zu*XAggNA|DL**9lE5J7JkqcECxlo+_H|7*G9moC(#m2NS83NT8<;yndxwT5dhAT~o
z6TUtZC+viYfbMV^htO@yO9x0E$x2`JDrHsH+1hALEHl?Z4+EA40=6w#?K3=dF~NIe
zG%ADl}3rT&>7ot8=BXyn$2u`JTwt;F;)tB+rZ8}W8p`|T(K3`ad9Bihj<
zyNq~r*Up|vzF_MbNFptxmMV#+ei`&#wo0;OqqwUb5Ok>`^BaZk;XQp3BA<1VE-kkI
zty7z^^OEx-rYAwMJ?DU+8?FJXiyiY4S%|o70Oq+=(Ud{4?MWge96E?QzZsq-LUK3p
zft`Q66bBR)OsQ9rtT#|-X2|1miTxO_M0pt|(Q6^7$d(vc#V@X-$&
zNI+B3{vFhUUoj+2=RLEyF>+^P{uRXq79@d7-m4}JJ*TviB?BoPPvk+@k`P;&`MnmG
zN3_xb8Xd`u#v5ir0(?Xwtzl>{D+=~CO&P%&R{Luw`DHTkv-x?n%xI!)oBfb*+V^Od
zRix-lpKisoM@hTN9mgQM;@+V3_yp^d+h2%nTgl1UI*j*UFizO(k}u;~DTTE-x?B3R
zDb1S=th(2XIm$tL#F>=5Xh*yww(@ggBne|?=EqbSNzZjuy%Zqc^SoWOTIoN#&>VrX
z&_|xt+0jE3EQ9havfAcjLQ0k9-H6$^n&puOO?0~&aU0&*PuDv;=+4fF3=Azc?P
z^73cD8NYtAy4ZaEFBTSvhMkx)+sJtQ1hR(Y-fT^;UJaK}7R`hs-f
zU}m3THFJ1`Q+4RUHexmbyQYKD*k|Ejzbg?(si*CewjtI)oto3#OZQJpB%a?Z*9=L!
zDPpkf?gTR2K2av7>yK7L+!o(R2@Tj3>2AEtDyz?n7grTp*aDU`J`~EvQI>nZ!R+pZpI;#&UFuL;?3@#^K?rq
z`Zy@3&-AV*
zp1WWfs?o@#QifhFs3~2|h@qaja2T?^cUbRWU?7qW)~%fmsFD&>NJV#xAt{~s3m
zcU56{&h_(gG7-)NxlT$&@diw5w(>h}h1+2%k0$V~x|
zNoCtveEA#IKkWkO^8jJ?iBt{_Qm-SwKeg0kNZXK0_mq{z(
zgOxxMpo;AHI>%gY7^gr*(jMpUB=IT3#@`Aj`u<2H)bAQclaJESBJ@7z3&R}4^2;7U
z)>yDCJ9BrlC-;D8qv*&X`?q-pzy0gIczuvhVy!R4v^%dFIh%yzYI12csy)+VuHflbA7s#!B2<5wCU(9RwVaQa&@h8jbKbT
zT5Et*Fsy=i%Htz>#T5dhU2{Xi>}~c1c{5gy(=xf4a*Wob5@tU
zNFECk+A(Y9}%XGOxfMjA(VrCgEDEb+bw)opZdDwb111
zJVITs2+~=borpl>&*-fT+wdZTNi;fncr?6uO7zi(cj%^7Y7nyCgV=v$l^b(ac>~Mp
zW>C!XDe|x+qwmY**~{501iMu|G)am^OP^_xQsB&xA&Hfu*;IY`urAPUw;3yMy5%aK
zrzgaxl@Hi4{&HrVN~!eXeeAo5_)KO|1K%+f_bsU;`G{cCYK#Y^*XuOSRU5m^ppmw0
zD8EM%EmvX)K;N(r{e(Ut{BHbNQt|thqikAWS?I7epkS&yy0HKX<__Fd0u>9k31NaC
zIZD7l0u35u5E*BpG~eH3RT8GW4-O2TPT(Glg^I}0;^CZ8gpOfINjswMRFESh&w=N>
zDpxV4y<7E&)$(mijknJ}Uttk|xBcWsgFu|^ZX7*01oGk_MG&*S<-L_OB6OxlYZa&I
zK2WRBv}w_6)YLk<DVErAs5>%Fumk6eZ|&^NLE4O$f{?`%udX
zT5X8E-4&cn97E=DuyqXWc{p~M$xdBW#;o2Ho;_aiH+z&eoKai09(k!sis1r{<<7kO
zdqndY>zcI_V&yW0?y`iF1;l8x18e28O-Z5ptgXTziOnFRn{L_#EI;d;dpbt2l^E_X
z>7hE`1v5e!nu?10neNq_r%OS)qT!bBjo({4j$lmcxMncotS@gtsj
zihE-v51@URb6*#&>J3Edk%gdB*fEbG|54<&hHm_aqOKgBEo8|Geu4*_E|y`KG27qo
zMKq(u17;
z`-|LSh=kkM#DPL6r7t<(DL#{SrTVo~xu>9)MYdlmljZH(L1L+*q7T^#V7I$scerX6
z(B+}bnd9r!#b(h}io->yqpL%|e9)`fg}dQY2qpR3&I9SQkGQYN{KgP6*lL-dDZRtl
zXx1yM!K{H5y@cgWPtJ~~rq0CR=w_MoXHlau7B29+SeS!}lT+gR6{HXGGvJf{%*@3#
zF7$RS1oVC^|6aHJd!}F4`P%L`Tb#xj!>rkX#K+GX{!49dOVlj&z3zXZxK<9nc<*1p
z&l(BQ6#>sETLigOziP|I$2Z*92U$@u&IAepFPtC`Ne-uNHbs?<+5k=zjk;Vzc1h|C
zD%mNsu20cIQUi3d5)1gDOE2z)jw$_v1IRIYNFp^Xt>=oDd9
z+nSTunqb30r9&C1H;`AHN|RAHZQUgzV7)zxX2+{t0h*J7OTHV1l7}SgVYzU9nnBuo
zx_b32t=Z}bNR6NVZ4-n_A?6GZyAEk&)?kik{uyQc^+0w;R7E`=+iq;*(e=!2+je8o
zOUb%n-9BhEZOLD1)(AE{P=Jts-7@{~Z{k2tiQtjW9z6Rxe`eP_hR0a{y>E1#O=5Wy
zS85lV(?w_`o75Inn;2VeQm4?p9mhfq0>|`AKuoQUY+fbyNpmr-psn$4F)m5;+CSf9
zV00%QF+LoNk^A#H`vMFxK*@yopP+d&E9KsfS3?Uf97mUM;U)a93oNO)%IGf~Vxxz=)vlvZ{((N7N*K)wn`$CBMAqlZ2m
z*&XKybTOL69Jx~p-fYD(x3-TSFIU5W@Y=0LCGg+7L9PX==ASJRcNkQG8#H2|l^naz
zpoEz@cZon3;iJNdBM3ht=+i}?-p;e@!)SQ8&%z-88&r50rjfz_q2Z^%b2q2F^I6lR
z@?fM9TeW(zd`m)~A`yY@FPKu4RK~+pGsJ`a-l7oah7X!5;pY<+`I|b8(qGN%OyBR}66yubkwxvx`
zR2_CQ`p}c@w18Khnyyf*e@08^KQy7&r~c00u^&&j*kTh;Dw^HAKvq`Rp8p0sbNKvX
zag=iYoIdhMO5uyw6>lK7-S#+0W^#b3XIFp3w4Hj#)0U7$GaRI{%OW`B+~AhU^21M-
z7aAr+|6)9wnK=j6=JW9N`j*jbqzdI4z3(K@vt`%>rD;oPrO2uuBAat(kjcI3sARKr
zE9*XGZ?E3M57T1P79pG3YoCh;*A){E0TnQMksfM{`Vu;4Y(GoTv1b+64x{o?rLSa%
zxJS#o^x3_3Z)?)&UeBdJH=bg8p7X#uGP5+Q>w&#zS^ah87D5k~o(#Y?B&GaNY7ZFy
zS|OSa#l2(xFY?u7_%_rJt*cnNn0O=L`-xP#q^dT5sJaI4
zR|(TfRBiSkLegPLeRpWl3+^I{P{h&>k&ZYv6$VX%xEeZK2UgZ$(OK^kvXIq}JYme_
ze;&vjwf^fP(U9#=Jr|4_g{wU`rq@@6Y?OqH8UQIIbA(ffuRui&Ja90xU~QQ-qAQm@
zxOjuAnDcO`#8pYliEd|E92}x&a0#t|8pvjasZt_Qpt;-A8`SwH$D8{plHcX6KM2wnd2e;6so@CVmk)y-ItC+l>Gjjn
zf1kEpXZgMwtlzfty{7^f#K0uW3KI+;(zpd5fO}37@hC`Q$YBsrNu_ER@u<>-)!j_G
zoJppHzhZy7>j~8^$U9YnLMmS!`ypCA*>Z@9Fe=EK3
z4l9XgsU3<3R5Q`Dqsr$qLC5B+hMXKcRnllwiOI~ym8uqFK|Vc0XTDuuWgax~~ssZr(`Vop1Cn
zFXIBr&B?7adRORWCwBi@*elM7bd>7OGX%hXv}gGI(k#JSr&y_wujCngRZXKoH%w{C7q7gn}=*zLbD&%Pznr1yF>X!}E)xR-j6
z?vf|&hfn9&jP_i^RHPZ9S(B&XvJtfEIIo#Vd2DtJv(C9VJDYO!S%aIb
z&*sl#=N**hZO`)N)5(zVwRcH_CChMW(OGl*B0orZz?iItKR~kb2jFxCr)CxHG?j8;
z#YiUU7RV@<{U~(uD`3$!6Dgz{@0&hDtw*qhQifY|rbp80$3snWsq@0J!&-WFqmj^cw81IV-Pw_jwo^EUWA|m(2fuH-9K<<_#c+uB_+?UZ+o=0&QT@dt=D@4szQD
zMA0pvaidb9B6+v46ul!lGvfSTc7K7M8CX+R?3cGxd93yypH+9xQ5cxg)w)ehp1p>+h
zF9T4&$gNN^#i05>^!osk=-Rtd8>+2z#6w;htRk+A-+NCn&d66QL`_g465JpB+vN9P
zce6rw@1TU`UfP#kWoFCje>SvPLb#XohZV{eP;hh34KpJg;ub30l9-&xP-d#i3g^ER
zXWj&7973+6h7ZbT2i?qSyTsdt?7~i4UovdkhL{UsHa=5mA&sA;CG^kP>WqXKMBTP9
z@p~(jHZV44l79lGK)9|QT4x9{1T4#}VlapTWqh99QU!)3
zi|4fqdX^RzAwZ@S6&3Y*LwlZbwD|d2G#N_?@ed<)=8ihhApGZCz3uf=R+ZCmbkVlw
zIo`arGvGT_yTrU&dELylHT8D?ujSieF)d!xSi6N=e{kENMr>}zpL86>>j}gX9yYdV
z+X{)6H46vF~zC>!r-)7OSiTFRU#!>m<4|2N0xk-%6_H^fybvUfpG(
z-O%K6!uPfM52meFUHHX#t`0Xt$FMHXxv5b?JZF_lnY1vq|ZQHKZ^0IB~ea`oHy?^YV
zt(|ptuB-d`+#Ka(A-|4#`I6E1WKPT-QT#9L;y%}u!1hbk_xeH$HF7XlD#uz!52kzF
zI2=T>h`Dt_MQnAHqHZUIINeO0GSR-30mjNOT{1%8gR0<=Zj<}+>&AoNC=&JP_LJ~8VVZ`5|zx&58@
zZcI#m7&8o7aYnMYU?$M4K
zXlL6flfMud=PR0?pV1Uj7BxQ~9JkyylCh^wR0y`>V<|N|&W5p{Ktu#4~6EP`W
z%{N{qRzlO0&~kn4S7ueUzlH0IxjUl$`uHcB+jEbeiNH6dEqbjd!|lsvVTY#$yVU~-
zb$}(5E#DF%tDMUsa3Ud4AP{dz4bOrD`i)U+A0yZEGdhDXnsKBMH4%vsc}c7ip&HMe
zd0-FxLNq-p=_#K?1j-+wF*q-dNGkcw0ql!$+Bn!W8qV$g9oB^|x%ajn8krT9B^{}6
zhsl@Z`K7f+89|csN54A588ocX=P#b{xQ=d0uZ^x
zwS^U6>W!k8Dkb}xPD(6Y=OA29D_D4W2mki-?*sknK0dMXRlykkB_%Wxsd9Jc>qBx}
zr~8XF#_4=+NkzUl>{A^V5I~+4{p`8L?c%PfBNFWA-`+<5d+d@+O&LYDU~ZFyA@q69
z&2}cW^NG!U26hVJ+l;w`WeRq5S2tkG>*?Zu1h0^Uk55j{SXCHQT(3B<+4zTs(tQ5X
z916x*fJ$R%zpc2xddjOZw{qmlYgjqBas3m{XmE=d+8LrYMqL%G#6m{q{|6ONq{GO9
zDTef~xZOc$1OBg`9Byl|-!CXd>o%W+jcii6hYTV~BU0=6l!0`>A3Fu~pgfK3I0R}Y
zF%wz8tXc{n9V)JZqs?SPiFthvl;8)b*+dLeO%
z1J4sy|7fg2&kDl^C2~YmwA>~f8p-;zxX`=g!#iG?eR0wJ+~&-iWHwN=1Z$xN2|=N#
zv>iupQnmwSjrNqkMcVR%%p`E(&Fi=Mo92FdP80cS)-CMae4+7Bt1C&}+>%WEmOF%E
z`S~yM1ANa_j^-J!Ak)x>^=CClGQ=78)1>7(jcN_2B#r&ba%dV%AC&3i`WjZUPGqb!
zg7C%Upa1S_!`7ebMC5%K+nrn6^=jvl87aq?ngxQV3}2tAvGj|z8}C|bJb{yl0(3&}
zn(*)fblnvoX3{!nQjb9Mg##%K?S%+~Z(C3OCnq+z4|M_5=qP|u(}?Z?kZpCr{z}De
z*J6RM*W+wpi)JTqt}XIb793rpEAZExE>Qm&0#BJG^zZ*avY`RJwkjzP#xVa;rT|<%
zEuQr=gJTL!6(?xFC3xNwskgQUGzahh1u6{)DIQB9xXapo4Qwo)Ng0?lBf0X{3X}>s
zoSiy+$8}iFXL;BBV2ahx{RJ&WnKW{Eis-Y>c%!Kc-PNYa`xWFqGEDg>{X9FxmTCUE
zQHZ7MlqP>0OOUncW#7^tSE-@-JD030M_a5^`cM9>*R7hL0^P&%4^BFr#K(=6f<7#i
zPvZJ5#|(yHM`IBtwubg%Q9le{kFC>`5{Q>OQtxomyG*3S3xiMdnD?g_s_!t^T+9P!i7n
z`huJ3efeddak8$tS+)(x4j4ypYLxMCu?=m_&<^Pf4(T=#Z4~MG(n;EX?f&}%y}(4!
zt0}IEcsos+gn4=92v>fBDlpXay?IDX8LimII}zJl(8X~cbV}DF?byHp~Jd=`=8SI(m?X#T%ddct*<_(A8Cbc>K74^@WAhgzx;&~
z-gk1JPVg>EKFRBH1Ba3uC{~+1d*@on!R!n@4Bj~g$-G{qtWNE`MSl%1YV>~ihI|@z6w6@#
zXbtXR!@CW!3uaJ1yg5v|RY6&nGOrxmg_2=}@Jhad4`aP~mH~sZKCP2o`e>b823;P<
zvQ>V)e9b~xZ1IOJ#ser4OklSfYF(N7VF7B{%od
zkgLt71QWXQHrE$m89xS%qd~UFRT5!;sz~T9sgXH+@j1*4sGd<2jD+4wSS_LutO@Av}BrnREU$qY*0gnqb~)J
z3;2_gupi&#lABw}0Ywf^erlsqy;AXEokQm&Xk{Ov~XfG1#q>o<5HJ
z4cxsW^A|&qkVK5%{$?LapalL`u%28vp~9yB6+=RUbo#
z{3HIU9DZYS
z_tqGo1j?&#azwL!6bAkVV^EdPZP|Y;CdNv^-3xe8r}Vd`(J@#wR1kdJ;eh+~1{g4_
z{|Al0rsrihBrmk9{A4}k=EaEkg){B=U
z@^)SsWvb!ux~t1#*za0^|O|OK`x&d`3#9&cl_(;GE%M}g}2K->4>J;F7mo)4Z*B`5DJuVP;9JXc|*PS
z7`bA7QVhi}>jq3jt`~U&>J4Ctf=p40)=!~BFHZ?L2kwAaoDnb`JTfAk(5fh{u~aM%
z2Sp=Hy5f;HIFAfWKCso%rKf6|6s(GZQlcHK85A*4gHR1|YCIhinmP|*CC{5Jb!V=(
zK*90*SU=wSp`$vyV$qA3Se+a~<%>kW1e5bRh|RGJ=~++AfWqfoe?JbW$^*vPpx=|%
zPkhMVhc;hNGQad(^P^QLW6sHle#peIFg^_YRTrKlZ?QD1EXzqw#u2JdEM)E~RoK|8
z9#Al_ytl%dW@bBoPjES@&a8BMnMf{=2nxO7ZZjrZT>UKs3`EOAvmA-*)F;g%#F?p*
zTogZUj3gwR{+i1|Xa8i$9|&7K?EP?SY4v8P=AT!7^X6xFzTTYt5~J$muV`05W7pf%
zaKnR-LOX$KkRDeQbCZ}p3nw)E_I-38sV0hMa=0|=A}Kzq+##%Zo`R-Yt*|rb`@%sK-a
zUd5=+Si;ghO(TRQNu;+hKzY@Y9e?t*T#nkb{Ssyf&cO+U(o?)sAc0UMiWY=po#8ze
z!vD=84#Ghaej37B?nucb%%Ad}seZo2qm()!HgvbpEr^A)2~r7^k6H3i&s+&enlK=8263Q^Q1W%@*^w60X5D_h+BC;D7&A3=U
zsYC3nrz0^x-pgjJJWjJVV3z1rH#`hVg8bys;>aD~=spx+WbWrg_TY&&t?d3gWCOg3
z)!bd(T<&$h@c&>l&O`EArwWbt5@)HVMEMlaV-!OjqgwtiAW<$NRRU&Ts@1$C
z5c}IDYc9TDl(2O94f8S@f-orpd3QoHh3!@)B}NGR;7R~0S-&IvS3yyftbwNM;pTv8
zR1lN~IXUNTk@8j@2My}D|DQ^!ve^?ER$C^P?r8c-+c3_5i*Fssl~}d5r0X=b(i|L8
zgC01?E&FL`S-8y-zB!5LSq@fzl{64P`AHbYEN2HPH{Ase#>34K58fLc35&3WXq??@
zoXi6ug#JNMDL-1Zr`i)ki5WH%+NMu_($Z7Qz{}ahU|<jVq@k4wYzV(2Pjy~I
z*S8HdKV}cbk~M8T=RI@=6X6h(BZ?;dULM!6a5x)hHJ4{#<5~dfNC8iLa#q&%?r#O4
zcZ2ET%iVbK1ehM7?TO0z_pq9SeNEvd&
zdg3rVXAkA_#`|fj^THK;Nh(YsVsZaCIM|Fkd}z|{Tog_T%DRvT)Alu%_5)K0EU;g5
zg7#J-9@GHZfH_Pzn8`9ClD3TkSnpIEfN^iAxmKQr#U)lDkO|_3lbRXmjOnaH&@zxi
zL3!vBYzcq^l$`hMc`*sBUm8REt%~{YK3$d}eAJ?2%4*yB3b2aP!p4yF9Za
z8nz8)ZPVgp8IB1ooY0)r(q&iNn_o`SCyqLtn$ga!I*Pb7EX-`!p6e#vA
zIQ8uK)h<*?HV#ZUaoV4an~zpne*Ag?6E}g#@4KQv%GOTj(th<@Jix5wTTRaGL2HLQ
zfzhQ9$JLa-j9|e_!OC(J{nhHR^OcJp6Ov$_x`YoSSgKo~bv?0llFp=aXrTroj@w#j
zh#(zb>Ud-wqK9#FL1nZo8E^=k(ECQw%-meYq+t6QF)Q8QKoB#i?}=P
z&laA)capxv5!PQEfv-x=Gi;Qi9kDR}!xmEbp2pCEd&gOB*maW!Fsw+F4`|7)RK}4@
z*Nh&HXd|%^k_PD_=+t&FqVjk44QB==gjrR9I`qg8$(Bz!!^36jS8OFDhg_SJs-SRD
zs~=ud?_z9+toU|z9>w>PmiuccVV`mW9KOt+x`uAy&``9wMPB~mL$0+p%J-Gc)wB)P
z(XqIkN#V)#w*2*5eu}m0Z{fnJVY%zajd^Zy?I+i9WqHF{(qw%bmZraZTV4n>6whu7
z)5VX}1;tSEs!JN-^Jb8F>e+m-7YW@sG8o#7l@`1?hAoFG-a}S#R>T*HLL)BWqX_h+
zB`u@auG*4XnB1m+O3#&V>3JI(7ev3JLL{I_k@VmhRcM}AOfqOj1YOcbsN`V%1nu(~
zvp#3VaoIv~4g%z)5JPi#m>a^x#svN2JCn?+eGMJ~*~}2EX0GA=H&e8!`PmUR)wV?M
zIJ}L!kp$=8(HuP+<-OC8-=WCT2ui=``9KtXR7(=SD^|Y
z7>#L8lKG@Xjf9_Os)`8YA6ZzVxY2JlLcNV>aW?H^_5yTn*~y@v2-&R((UG!rjRgVp
z0^;ADj&4L0-ZOs_(FuN;-*7}W^d#NxE#2B?;H3IF5sp$e;N?=;hUA9FK64IEAG=Cz
zB{QG=v0(o%n~=yr_2*lrcW?RZ
zCvj`yZNE{7C8?$=n$(HIbi$R)9oK!9J`W
zrHEk(bXcjshmocnkmdqd8Sa(C>gM|KfhC^1lT0&!2>?=jnM5`v6Lw-4&3fKkO!w*t
za1nu0n9*!OYWvkP>>wI3if)`}?l4EkBn&iT_Cc)|PHbA2%6k(5P;H6CN{#zv7T7-H!n|xNT
zY!%f;OovleZ8=Cm!RD)%588v+JZLa5iV;-fWERFO8?0GlEEkN#&JaB^CC?$_Bdc!h
zQbHl6IJQENh6+s^Vg964Y58Yzsn^@|+HQVtY05*4hjWoCNO^*To5Lg}O43^O9q%z<
zD#X=NW|1BD6P975vU$^|P+{1Gy-(ss7$DjNDT@aEdS=VVOEK)zFCO#;|e1
z(RwG-FJlZ$G&ANq7?}djyhNr!)Ix9zL=B1M5^;9)8b>{qsRpf;I4Q-`7E#PV)cX+u
zyZMJh)VvjfEn=iuJfB@x=~$L`N%M
z;^S=dd{r2PMUC%I(i?KTFo;HgQRD7>CZ*Ej9U6b+(W)RcT-*jW9>8h;x;ku&2DOKl
ztY`PZAu5zyi6*k1zp)+=$8!FyQZ{CxCJ4Kh39Qs4)yosu^HN62MJxD4b8BNDYo^sR
za%m-Kx)@f8kp4)%51V|oY-l$A%m~E?h_t-FunDM`*;HObJUS#gYQnYhgf~1j(mk#v
zcz%2l{wl07cfL8l!P;dJ7`dwB7u?8!CLt|G5FnJynM??aoR6$!EXQ=ZFDRmhQk#ia
z&p0^PA4(a9v`SrwP8spnTcR{9l_}k^V|f0Kkt71=t_+@Iz0qa&r|#MTZy#ZwnG4P5
zP9Mi`;b=2&kf)U#^
ztHtMAX=J;rA|+9?Ah;wX)8mNdtmA&|f3dYn@O_M4aO4!azz^iK)wuwu`eREqEC~GW
zikfR^K^zw=p+L2Cw`v
z#{1|FAp3=fQ6K`$-qR+SK;x8%`@?p|G^|E7)ak$DVZ=a{G%<)KtsOWasdR9K
z_d19w3~}+14Ue}bg#YZWJ!wd(7#0ODfpz4GWoIE&CB#gwYWI+cxN|zF!q2em;XQZ5
zQfdQEwQ{5#zpX0v4nEHVOgXQSvdTV
z(W=!ff@o4wQgW7lwpy9JX3h0e-=6p097?g5L(V{`6GDAkELr!)Nh!-Oy(-DFIm$5f
z!vE#AVeS<^PWsR{d8B7*Zap0G=<%{mrKE^ssV(1lpekbIRVbIUN#wcsH^?7*ciTaV
z(9+KR)M^;*6o~&5eM-Og!fx4F0s^eY}lH#w*Rb6Wy!DR35N~C^zh;?PvGYeCk)l_(G4C*7r#{J~H_tZ8Q_~0vFMPsB<$dDN@_kOdRN|g?_-P9Qp<-ZiaCI&h
z{Gr;g`ElqaIAs&vfK`DG2>yzTc`K*i?Aa~ZzofXAi8)MNTSi4FmdqXgJqE5GU>zjt
z#+Hgk@HO5^2FT;E-<_&WdBLDr(@~-D~e!PH?6}nTahAoQEY*1hv!z
z4dO&53W_|^e4V_tjE%c*_=c174JR-vqqGDzCNp@Sf_gnSA|G#{UQ~wfmtIp}N}ar-
z%Ty)J-TBH;f2HNogqcTH6=^;J<$cQaX>N5-iIp<3%bldZ!|1XSaUj{e8T?erRAGY{
ze!w}5MJ#uph7dJKbJ9x;q~H_A)$!&5kun`I?be-W|t^0J7k?VtWi&N
zSdYb=pJ&F3$7&!){;uqMO5)rfE$H9h(h|$>x8*|84s?KHA^#*D_r?i}F-VOFPa%*$
z{TDK_$|&`XHN1lnp#!f>F_GE
zJs&u`#rAVZ24dsUG?3ju#!w=o2tou*S0r4#Ko#s5B
z4LvFRxkXGfh&(peG#_3@#~1`TlTP9FGNae_J#eF
z<ZM9)_Pwkqoi1rw(+jrxH;1;@yBYADOI-yhd)
zyUnVuCx<%HMlmw#GH2fzO|6JzkBn!g2#IIkk}2pk^G^8dtKQ>m^NJc=tqb$%KSkl5
zn2Tc@MqV{A5Pqk4b@J9ZTPXpX^QmhDT>~F<;`Tv0jCMPM*FDHS-zOUJHA&85B?&i?
z-!G75Fh>?}>~?k_?^oT3+*Vx{x*N^2yzhRgoj>cpANwYPr7@6A7*&>6Ef}sQ0JtLT
zFZ)CFw9@seEqE0hZKM4NW)2$@J9KgMd?%--t`G8}&%W7KBJO@s+iHkXI&JBf-vYKH
zqg`9!DU|phHUhj}wu!n7*N4Eq%>KE4SIS>4vF~>uUmM_=S{7uNJ)}$sL}bp+f14`Pw#k
z$#dwxE1US;kuG42!i{{RW@Dm#uhx^=`7T>Z#@^P-Pr2&?qdiJSsJ#
z*@DYs&iq{!5e4Q7q{%0RrZ6@ly<~K8Q66E$W8Gf`KZLY4WWZzep!GV29Mgm_ho=%e
zc_%I1pw)*H!17DJEU1xJm0P#S(LItAL2niG?N-ep)aiPa%k02fLgf!ST7*SdUZnCjY}qxAsPdI$*u8mQp1F7rQNTR{C|P+!`6-%av&D7L2U0i
zi&{t}$vp44HO@cvC2MoBgvV@8O+)K&<}8q$cOikEP9x4OKKPZ`wQKQ$+BHv|%$S-MRddp>_V#0pqS
zE9Rx2W|*{@)||G_|8^pl&1^#&@M!2Btio!Ree;iq3iaIC#xm?@4+RE-4sfmRFpR(C
z0*qo&loXXR9sRIM2AerLW+<#Utql>6%k=Wc+PdfR=lJ*<4Mg3xet(}7Xz$cSlcavD
zQY8{o5`H!AIIGRS2~d==XtaS5GkKtBMuFvs>AYV;Ij9FW{6X1jv9eK5Cxq8Zy>$u$
zzunQ$ZU411HBF1o=iwpQEolv%O#R4JfEe|?J#6(RReq{CKtY^4u6=fUTAW9*?86*zs
z2kollL>e~C<~SW;7#rucKgzi|9VDdz6FE4d>1K*Cxk9u9x2ZY=gr^vZK_XJf43yIi
zpMJ!DTL}-H+`6CKx-~;BkBiCg>~OxPg_a5*KXG1@F)O0Yd?be;gQg5sBKGuhOVm%#Ltw8`~c
zq$fXo`2JuT$3PJbX^}da&3Z1Z4amkdsLhj0!GSI3(
zvJS28FOnuaU41TX>}Td-=TlZ2w>ny$DwwQxDyoW!#OTHp(?j=yij$3EB(rZTX
zyUux22K*H8EQ^TZOi9JrKN&}Kx(;{d)6GYbJzo~D`(rY7wwliU$U3CYKRK|3oB3l4
z4jctX&hklpK6~Z?p6kuIe_Rlgk;&y!=dPRr{EHxAz6v!(XgsG3-4KuGnZBlmN2VUr
z;1QhHoLI=u>V0%^LM?3+JimUN=}-`69Rmn9HIn5sVYyOOv~pGCV`jEZMEg=__)Pq>cj!Bt!6&6pU`^=hH~;l0wk3)Xb6T%DDaD&T?Pn-q-PO*Z|8P~=cw5(aHjk9qc*a}su0-whT)Ret(6;bpDx0H@J1srs
zbyHo+J<;|uLs`P^tP2GlP=VDiB7aqCl`G^-zOEWg=+xS!nO%>ix`_|uU2}n6@1+K6
zXZtxcKXMXalXe|>m?F*F&U%F9Kn_q2VU6M0w8}-*Y0H0B0oX*eDl8qRt?S4)m>o0<
z&fZuu$p@zQ-L4Cu4L`BN^WO31;15!?_rqo|K;5TvJNOg5Y{0zS>c8J2VMe@xRLSkT
zUhm2@($J1Khhxc3);gX`6WxHC`Lnk3&f`p+YoKoOxBly#{_fq1{rScx&SK0908^hf0F76-0^QqA*g>IHU$Rm_s6gA
z=4(K{K<|oYo%(nR8IXX%`2-C&eI6#;uWQe1_u#nT_5UbOyB#Q|DoQUvCEoFaA>5eG
z5&c;=v2fh}YSj~fEZ97&3dUuM>WolByZ-*Mtf#+4KNi~72JnI676u9$LEu#V`m%`(
zZfg0y`W{Ku2Y&a7K9|-gt9HmQyvq(-A8^zN@j@;!ywN7Uf8X;EF4D9)?{?Gi@|O65
z5|n{^$%8G1{`zyOqQAtfC#jR)jaBo&-C}LZs3^E}hTSl;7Nvd4)E8xWbfOK^YJf6t`Lg)`${R
z&TRzcbH`8;`i3SBp>l>1J=f`0_JGE>a{TA`7SQ-i04pb8kw~aw9T-vn2#}%aoz?wY
zjxc!)nF~}O1{KAsu#GBAjmk%~&jkDb@Y$I;wDeuFIq|;&{jl8bu}cG-4qJaO@-co;
zO9tjBWOJ6J(r|;~I6wGHRE(bKZvS(&-~!6U%+{RuKc01-nxWbLI9k`=fthOWkM?nl
zK&SGO&k&p%GiSOeY_s3K>*K1ipTZH|AQS7Auw?mnn{IohBwCDQfmG2y9cs{G49hR+
zLHOd(zhGw*iq=-}^LmS;${bcm`U10GD81Wpp+u;OEV;Pv2NE9Ug-RB=Z3H!WYseo;
zVOV4u2&=bKmA?#sUl(e2x}pjuzfMPGQvCgzW6L;=@@x?z02M)OTY5o>7=>X!q5^9R
z5*w$w{+M4UR(ivKVJs_nh+B5J2#qx^y>~0k>k-k>gb|u(fgtZ&
zFZYv@IA10pnT3}osO3mTLel$A_R~pbF)NTYQp?c#ZbS^S$vHWOtV(dJe?5p7kNk7v
z0Vn+hU0fiyTDoH`TdQKzmZjuZT$mOpxl*MRaOEzaVi_({NE_A5S7{OV!7ouwnwj7&
z4A2kaNuCIcNHmYXe2KMn`p)NeBn
zQkdb~(2G4WJvS20h<&kigfymCg1_j-JB~xMj7*qp`m1#xI;KQiK?dUqQY=%OUgn)?
z<;0m)5HqpqTBnZPU&kR2g+rP5m#Y@p9=`fGzbyTZ$J3bnIb&n5a_``5X4j4>3I|Ds~xJEh~JG(xdZLWeC$GG
zj3pYpoDovy4G^t68hQXJUwg89upq1k7F5eKeV$f}qe#<|`N=34;9Rbqf~MJIXGt;7
z(FSD;O;B;*LQWTJQb3zhN;8;-X;;dLLx8{88>BforDI}f=%OxU|ktkvmKv_Vm)h+W5JR+={
zyS{O+Rt`{l)|k=X2C+-0$E7!XAgNI^Y!JtnF-k8LkRYpu#bxGc3FbS4{Ra9NcfHRP
zN6rFBGe9M;ubGKS;N>OU+Fhe($s8RJ;d0K6osnGpVCV`IhgnM;NR-jLmX24Q6kX#i#lAg+vY^MMo@%X)A~crJo6X7}JVF*ePGSf(2BdBu-;KdrdB&
z{j2Vplxz(lXh0G_phggm1JQ;4BNnoyadi!TUN0nv|D5WAotPybA8mBC^bx))YQIMO
zhu&NB4<*d=aTa}6TsipC%Q=P0vk`M%c$h)wbmr%kZZHk~szVpgiH2vV7yG1FCaKW_MnFbmH(n
zi5t@aqZ|HqDteZE|5xJQi|+gv-H>3&i(^Z+Ix%QVW+aKVvr<7fd_
zRn?YgLV;1BSy`a-uA%BOWy))>m9+1l+qyRxXEw}z@64H=GmDCfTHfED!N0)y^9LaC
z)@T0pc?qBNVZWp(lUDVA!W}qqK6a-N_5=8u4|cHYJ03&8Kce=@+Ri^BAK@aYsom6I
z9l$V1H8nLb@9;zO_$N>>_Guqo;Pqzqzf=Px5SdeDdQaTwd_LPoe%n9>+Pc?$mhOEH
zrb&QdsDAL=CU8VLWO`kV;P+B_{7?Nekw(2~Mm>(F{&SK>
z+WW41-~T5b&bpVS|8B$v=iDE`PAWhhL<%62jQS32TsQ$=#KOwjYumJ%bZ*_C1!nMP
zixjS3&wbx(VZ#fSokBDEvAZ4yGAeuNBse*PRhd2lkCPK4@l;i9ZezHdsq1VjI&oP|
zFvx-r24}|LLhQtCz*NHoR5KvMeEzZlJV_KS+3Y==nVE6>zVQQ*{piQ1g|5zB7paoF
ze?wU>BUu+&Z?nFwCtU$1)^BcI&#U}^#r36!=%vTyYZd?1Fxkr{(c7l#E1!C3#*oLT
zw6}jaNii2}WbP^na#X_Mo6pYzT_-zz;btP(tTO(heUs6V7BmOH6zzP<76Ts{Mb|;I
z-2$(lemMQTZY*MgdY*utRjVAmlm`VpH7=B3iN5WgAF)nIiL;R?Jr#NvS-FDz%vILo
zqo2@l&&6xGE_u`oiISj};;#F{n$wok=8I8B*mM6h&wz1`0xQ$J=?76HN_c#Msk}cH
z1~WgOt5@(%jCY1A7mkiM@x+3=Mj@uXfj=vrwQ@Uf6$QWm=$`>J4uEoUZS_z`N{cVG
zLlg_$+r17?Z>g3|IPgoWnWw3^>x~IJcQ81|09458f1lmo3T9tc>WX;2Hi#)SO?>r{9=kW+3gSlts+>a^_cv8ndoqi0g3H!~ZfBHNaqOha=z
z`u4*U{>IMA+Qe2g-&qZ`G-#};1$PS}&B|bS2UTQOQB%*b*d4gEm4GaR&71fLfc}}S%l+kz{)Xw
zg5yWVj8N21BOAdAClwdD*x?kpM<{
z0bW}foFScN=DyJxeuQ}RlT}}IQaR@ybM&%
zYEqk)_v&plVo{M*rD(H?i@p3wMrJH4C+BeX;y#(dH?=i`=76SLrP^{kubrHlYOuc6
z@x}L$0*vqoQ(yQ0X8Asx|GSTR{xQV2)&rP7CxHmzoE3wdO5Wk!UNiSSy4ij->MxRH
zmU6pXmzcE{X>{i$9hOx*^Sssa!#7+qDYO7i7A0xC3Cdt(A_?nImbjD@-1;cX2DQgv
z-v=1G&PTJt!oue3Nj_KzIleRhNToUFplU(VC8QC%E%_kka-#}ycT`Zbb(<-ZJsd!D)#4DMiLER^Kv-kwY5ho2J0-9BZ~229rY$DW
z0#6Pbi4rU0n(eYC6IA-KQ^qFSxE$%)$F#-(QAEv;_`XlbNH#qR5=Du$d{9d+9Tv-2
zw5&|C!jdxeCIeUt@ya
z$FD|xfiRFvx;Qq&NIC;Dx()C0){FOe(~Nqan>O&crKJa13Cr2S3N;36v+ln|l)9b0
z6Y0X+c!4JPnb}8vhMAU;xQS5ccgt+npd5-P&aNL+$BiSOBuy3?O}44q-bJ)A>(KO0
z6W>1d(fZyL7yI5mJzyRX<1b|$FXRB=suX)ymh3sr>
zeO+(IT~}_t_ihWe(>Ia+;7!1bKICigR>FTZYnK}*;3tXox6}1bD_`fV%*;LT<+uPG
zxQ{zjU)^724@|<}y)Ws;l=r411%}GoqU=Q;MxdGtf*RC6xE@$1Co2tMI2O1Od
z18_g}0zrx;&N@&wBLj}htB1&vTmw_7hEtgW$IIh@t5whAkfIZKP`O?`_qlJ){LR%1
zK5K!;lwWZst{mUuG8cw`b9ERf0%g*1WTd;>eBXb6^!;%)LiGqP-j;)-TPe9ij1o$H
z6a(6{ySlb_QX($A8lE&e^!DGL&-YL5efhvsdk+VaUp-VuR~cVs0WzlXp%d7dXg)I4
zLo(Ud~{~uZdA$&R-1dF5}jP85`_WYmXsW4rR#X>cDM;E
zA>Zcfx5jUCe%Dt9nrT5Dh4+I02BGa_^d#=ae@P){?*FlgGH{OI0Crqv8gVLn%$n1G
z-j4}w;@&V_9>Li)43Z}1s+B2~saZUSQHN;)hr5Tzld8Cv4@dUgXC8M~zwqm_NkU9m
zZ|OLonII%DS&JiEugY<63!W{U>EfzIoN>Hrk2bl*ySLDlPPHq-IG34R8m(x#*ed}g
zB9guNHL0Wh(go(nYzCLpanGu36xXEyc5U;_+0}Ts`H!~;p5=;|ZscjLI5B~}z26#|
zwoXSUh)iF6Q<^$%i<)om+ThwLJE)hgEJjZp?K}#|aPSZ$M9Yy@hY7Y#T|FR0ddohF
z$z_p_HSRqm3F{=fYGF>PS2uU7IfT}e}BRF6oym$sQPWOwSB#YI3!V2!1
zEL<{80LyZy3py6erwg^sEXpc5TsM=)G|k
ztO**C+0k=ksgYll3QId8ecLiG?ysay~#>1u*FHKXxrAE}p%^~4z^1uLKUyS<(-ygC(e
z#dr3%;HBcA@>YOG=%$r|weRrLi6GOlK-BenTwGj0=uoo%Ygth_W7!m%TdgoEJ(5VQ
z)LS&{Th2i?9Kn3Txf1
z_TA0331-lh_}fapu}1S{GUqQ8yS4AfGNBls3^~w(Fh0eXiqILx=!qy>c`@ajX(xHI
zjO9o&-GxR<&OfDKPL@|Sq8Vd^s;bL+O>Us6weZQ7n^0UGT16)UHQs1!s<$bNnA0|#
ziyRz1DNbzT<2nYGmT=Zc0M{M$&m$!C(&0|_d~a8o$;oGr|La4O@AmlN>U*GWIGNCX_-VOh
zn|x+JoQOtgA|4F#KiRnEer_kHI*W=Fz*)Pp{>BN|GmY-gEF~l)E?xtME1BNg0B5*nEDG
zaV$D=CgiUfDu(azUmwwZuDN`jRv@m|-{0=9qukm90FFSv-R|^u9mt6!_M`yVMM>y&
zA?yQ?-rh1$Er*Aeg#aA@PzsO3l7hW#Q~Y(toadm+ZKC&w(u8XuSki9fd7Xpoeb8#v
z_8%GoSpQ#B!6hgT#x(HX^LbYmxMBP_YJ`5{Nh{_acsX|fJs%f`1P6lw3Yvgq!p2;`
za^cu(hm2@*lJC5o5lz815WJzTy=-7+wpaT55d^H%#Foj+PlUXj15c{|m6}^GM{|+6
z+y1g)xB^`Qcl8_QZwHP`@WTvbY0m;0@HomSJ(acrV7u2ITUJZ%8@yfT51_w3gYKk)
z%ltpy+?eQjajw)6SdoeZf9rg@L1Tk-mvq62Om0>cO{%2Y5{%&v|pUeB&Z;ITtO0eH}zx54Y=NI^SY!Cd{qsnwI
z@qj-JJUZwn_3tmhqE)~dNB$Pd4IZv4SG9IsSs*2AFb!*}s|49PD+qE=;_Dbn7qNpN*
z$|{1er<3>s-PJl`jcAQba)1YbJ^!@jxi}rtIqCwicmi>?T7{%I1+hjUP2|6;>TdP9l@iwQ+i<0
zqvrK}=IjkW<6)@WF$%RMtMH#pUm9ulrc{WAkgG_%t)rm@h^
zXfk)EA}jK6mI3#4fnUgn!fv`g#;ppK#iA3T;9x+6*TQ8=#rTQQhKgrLk_GO+4bUC2
z<4-hOTQ@zLh9yKrf~X1I+<^vKv%zuuC0Vg+rCpQ#(Q$cAQ6|LD?PO|rIM1jLx@B`0
z0R}Q+=rk0GAk6i9;2kx{x2@(boB}=sO4Bl|ISkaD*HJ_jtr$ghHh;xSpSowa-aYwC
zb(7E8u@;Ar@Hq*}8Jx={ZmEG{Pq0-Y6?CWyPwfvcRGM-aSLsAQr3iWk;WA2IQj
z5>Yv;{}$xq(j_@}gG@9v^m1koYipN{94uo6X4`PH(zCVf9rH9>B4Di}>vVy_b*;Rf
zQAeSs4w3YJ>chbzvKC&vq6WeocxI9gNA*W80Q~Hhzd%@D~vE)jIj_qWJA&j8-FGDbypV!YrW>2
zt(V$Nf~B~Q>Jl_A;;XXemY&ZVH$80}CSK!jxVSx&vHw;-upcIi!eV?P!QhlDH2f|N
zS6@wiOcF$S!EQXGbcx9E)vUl;ht;pY4=0gw)X{rH%a1ea!sdr*{yrkaH<}S$1YryE
zYXND#B>Ld6!`7i66b4hBym@(fvI_FwpT>F27pL))Bs)1cAb_)8FpN{RP+!@#zRq;(eDq@z$OY8SdSKM&>{&FxwnZN
zfDP!~pp#=`uBRh(m~tACo@x09AH**3e;i$9P#X&v1PT-@PK#6At+*F=DDGBVg1fg!
zi@RHKcQ5Yl?ry~iuJ7`GWF}-nWae^vyWf7ht@oxs*P|{xComSZO7z9rl=7ztv$d@^
zHGZ-``1j$)(aoA68%#efmkr#8+(Px!g@2fQh>}^zWV#GO@29M(@<;hfqZ^7GQO~;U
zpZ)XO2QJA)*dGeV#(2=!>%hbm#x^!1gWKyDhh<<~E4g`v)-9JxtZTgB0ez-E_PzfY1;CKw4;v
zoxAIJdnwJ~^x~LZ0(1<02RscdK-+bfhaJaJTLd)m34N^CUe_XiyFk%jsCy;53%TXK
zfJ&W`k+J5Q;{{Bu;l2Nx31RM6mUA;V&jcJLZ4B=+S6f1s_Zp+p)SMKpJ7rX)_KB;*aaPd|Y{3X6V2!}9xC}3u7N1NppLNsu4a>C)&=*dw5>3CLfWg#_d1jb
z0iFKs5^!g#J@BncG-15LI|6aDRG*^LL0C&T!BQ$(OF{lq&))FcRJAY1QxRfJYv~)(Z^=LW
zC0XrX7}ix#GPuIYs>e{_)Xm(eg7m!5j(kwQl_P)CPab{s
zK|09eV!&Z%JuDO<
zSn;t4ds7i8Iao&J;xas6cns%VL
z&eMA1tfhsGuz_qLg;O4d^zYQ1q#BY5bd(~SO>k%_
z3?IEqwgoglQx-0zD_Q@xtR1BNcskPCa9Z?LByV?i)UWDmL3>(P40LS>#qq-T!$O0A
z_;5yS!zuM>3fd{#I1`$rU-B~5;#XOE2NqM6rVft8bW0y!&BqS5Fc*lc@#afks?MwM
zf-Px|=alncIgAp(_^IYMYq-uKli_)0VF7&Iq^^`oljkBE!O&+2Li
ziGHeY?TWLYsT&_&<>ItRFFGXwP43Soahftmsx$A?(@FJ9y
zvASjJ)TI5cu8*`Zg!_U~Uc)}B-
zy6$C>9DMpJiSOYA0sTHw3lGQT352ZtwZK(s(2LhyRcoC{;|e88i{tZ$GfPQiUEi-N3_EJqb910=FLvt20$R8q~teqf38;Irr+w?;CLctcQ+icc=mi
zwCXrDl_UqDitrvfpCU4obUr;Y_Gs8-%2sCiPiB#1MplwcwJo}Z7;OOLijA8DodU)l@Co4~mT8YC6Ywy3k_
z_`}T13?u;j05YX*D18+KOuq}ddN&Zw_zB>TE179H+a8f;{Mt91_H^p2o8b|5PIXD7cm{-8v%OOSItH0@4
zVUXocz`d3K(-?pph$sx9RiL9VfP9`ez(X6LLNd+4kf@4EpuXN4l4ag-d=jr14<~96
z0{-kxV3{bRxV;Va#099$#FmbmCki#Dl{1H0OhRt!6f>mytJ4Zlt?l5=1m
zTOxOaEf5wMn7W`xBWsrrz!84qskc7flfts@Nn&$!nz!|6oA!zhGOl$UT%lEKCfIKJ
z!8kj2ecZql^Y`0%c)V8eyhS7h&}v#)x~di0d6$sxDCV8N-^d;(mgO6Q)Lg?93z
z+8pHTT{r_I<6(+pIK=*mH}tfMU)b>V&+>gCUBCVtaIztw(59}{sGyX7v2y^7)q!xy
z(FeFXRvJa4`YxmpbR^Ob>M~aKNN(${(Jg1(gBIkSNC>x&qm3w^G*D!TM3cYzt+NEs
z8CSBJ+|DHdEyJzx_F8gg)x;*e%}AtDb(Bn
z&B5wfhLq)>3kHzPwHgW$wH%zn6UN{FG3+aM37u_VAz3V|u<5*A%+ss6+$r&B7X6kj5AmlY8WW1Y@6d#3@C
zfjW6gY(9^FNdI!9HIwGJ%X##?F#8et>-TRWn;KTC)NWJhi@*zt6@bn21w%inBBt&P
zCg7a>qvN$P-a(zWkc=mnzzv|Ll%G((%szT3yM-MQkY`qCVrlFgwn{4o)F>uh{tn3<+|I<4|pK4YuTRwulp
zc%|sy2)Ln+rk@bD;)-C+%jp;d_6wEn^#~gu1aO|+4l7k`wBOr@aE6x}X7QWr<%D!;
z88QaqE;E1G&LA*tmLCe4V4@9GnIEgmgM&$-zcvG-z{ET6n$v!`0gmZxd`)$))H!13
z?@^0VC?y-W1amHjf;IKV^VCdmKT5;C8lpSbNiN3KZTs)hnAMQs5rLb^r%ZqyTl4YE
zgX<$U-B$Y_u7wfj^tY!gV}nuw#Y40TSv(^TDPV}qnm5NYq5Q;R=e{#GYuy4QUs%IR
zc;R9hhRgZD1XO<+(vZt@GZGGF)BOwovPkqEj=+-R;-8N(GlL|ytZm;
zTsLn4s;>F<)kqFTDzD7Q=4X8*0~5NTi?&fzs!rt(U++_q@BY$>mBUSaAZ)_g+Ia6G#}4lMnaF;?l=cVI9?AF%zDwZ(v1KT7wfPRrzln2o~aPL2Hb}rcnA>qWXA)k+|ii9>766
zzqSuf8PpeKUGx*@#M5f@MYj(7=U2zFN)p_nmPt*~)-_tWV$*EjJaK9j{u{DE$@|J!sxpk^$9L$>{2sAQAyyBx%8$Vnw~1E4QDUaa({93vlv04{>yysb4&UC(fO500glG
z6iW<+bna@#A!tA)bY2^u=;3zQhOoU}Yr+(g7j4qbg<}`U?u|WQLl(U6LemFcaGLrr
zFn4Y2n`prY9`UW3QFEXBOfIwmIJ7j%Myn`*??SKvJv6r)~C1E+97#ilI2SCSK
z8k&973k1@uOB6l`sJpq;
z&;CmkBhO8QHMQaJtZw3Dw7~vk(1=j61!AH(qpYZ^6!B*HZGTu#_W7Oyi^d*yixZ=D
zv8Joi0L3WPtK2NQD(IVRL~v$FOMium`T7k07qIVOg7u<#R*A}vpj!PaQOn(u^`Q3`
z4OGPn&({aHe<=ym
zbJxnGJ@^}f<8gtgV*86u!S#p107AdbGm#9Z2FhmZuARj$|6iyNgqa+H-6g8XLSl~?
zBI|g0Pe0=~f7>dfKN6o#=Hf_sH$PQNAzkEWgw1>0NGj{L(gTjDcPw{SL
zb8?$RStHA`rei>L@6A=*gqK9_5N=Huz8wotO1SmghZ4HmVi23A9Gy%gbbfcLCHXzI
z2NoAv(g@E&IunBT99txA&5F0Hij0$ZvA*y+-#uKMRHRx*9#Y{6V_>I+d@9xTlfeA^
zyry2;VE-ds?v#DnTQgQ5I?!f+azglYN3z*DExi8;C>Z|BGgLpi^x(fkEId}&^*N6+
zNt(9&sw&=HnQRKdQN*P}fzc7?yAVQ(7I$a&me*3Fb70C+^Uv#ag3wQ5b)QhH>@S!S
zQ8#*4e~{m^A>$qyIR|56;y6zV6VSqfVpvI}QAa22?Hv+d*3i+$)GH@az@OY7=Hk4c
z_L6v<{>eY;q^;0
!vq-k{FxAt3IV$K
zg6q=q^C=K6XVe1O)h7xu1J&@Ik=t`2p)UH*zw|q6EGF4&JxWUDWT)Cw{=!sS+lqDQMd3+~V
zQQ)Fx0i(!_Xw1He0S^tdd%L!%MO$j9s{=(EP;6{$Go0X(>fW_Ystb}E0*#-xT)H{*&S141G8#+G}72Gr#h_&?ENc>Q`d?GoZE~g
zk$Kx2w{?IJWn_nV$@VW_w8urh
zfG5fVP2DgDrxVj(IT=)grM>x{eYSE-f{IB8{fIhAvf$WFGRsM=Ff4lGm}O@;wR0{CD~FT?h!?S+;J$DN8&#MM12oMt)x_JNhiRDMo6Z
z*IVB_F$+DjOns&q$Rj14dG&bLn99}ps6XqY!Z0Ho!IBDNW4M~(T{qyOzVDTmV-U@g
ziPqk-47z}UnJ{Pi7AP&k<*%P!%`<)d`0lX>L#K`Sm>`W%(B0qXDd}*0jC|MDrn%i-
z(2jM-1JW9z=TC1(Dj
zhLb*7xrH{dRIl#Aw1A^b3HM%O9lx7x-l{-T8`+e;$fvmQ`IgEaqGh17H=AZ>|8I<|
z`TX!@)$4qXfr$w-2m@X;)+AXjJ6w3sQMB-qxS?C~`x}wUVKpL)J
zj=P6z@A)fSZgnpyK;hFpp#4~;~9-4Sm8@xvjn
zy)`@K=NJ_@!Z<#|i(P7KOZLY>mmI-zwh@)|QD3Nj?6e{bCeEKaG!YvEYCTEKtJR+)xi*=qA$P(5%;@U(jZcD+QyNH2Px=uerM1hp!umCrV#{i7x%&NC
z;G{WbQst~$SR)-?YL`RYylHSjKF%3-C2=Of5k@jX>UAQOx2U`Q&23(OW6nLSXD7Nw
zjU8ed%f6?SH5>o&be2(3wr#i;M7p~qq`MnIx~?w0QE8bV5>g`ox9
z&%EE>`^Wr|@NWO!*`d~<0^php82d28NUb-_qyJD%ULQjGK%`q1HS
z!}Ikd+s<$1T4KMC{}FZn(cA=po3O%R!Xb{Xz&jK=AX}xWVOn>12I9W$m96)H1R4^1
z*)i{ZT=pf$1lYWwRSdkkiQc=d=B->`a{?QowdIyA$NNeoO2E9hrrzHV>~WxTKVrrA
ze*G2TIO;xN#729^D0ZmTzp#J}EhKn=qH)tUTg-*Op{m3JVhArEU!9#V@HU<0UL{v2
zRdv@_NkOFbCoO=fY4o8FLBx4$TnF$gwNiiLZg;x^2y(tlZsN_9waLmDKhpp{q$H6i
zdgwmqVM4oSphEQJ*zyecuKOJljGxMk0Um)5&hKic8oF}=;^7<)T(;goDFSftweD_q
zxcxV`8~CMFXieLQJgqF2t=CN{tr0r;o)Hy>Zj2_pj|#)9)`;h8B}{XjjDNn>2sa
zN_bu1UX5m^oWJvP-~`{L7}3A3USN_F__`Z1r)1&*%_bmv0&1fKr1e_S1A((PBFZZ6MyxYF$;1
zNur#yc^-(}e~cm!xZ3vmGuF_BDa}DTq{Z{ZIka8r+x*gM&bJ#emiILsmG|>iuzS7o
zLYJ)Hl~M6}n0JjX@Tgagt7FlBIzqSwv$59k?LG3{tgp9tJ;z-2EuM?z(Hj-5Biyhf
zIl%(z8yeG}izGCtd+FTj5!X$)*X3vuwVA*LCMs1{zp^smDdQV(J%|B_U2ZGXdqGFSRjtc~xyhoX-mfkOYJ5d~wiZq02u5Z_c
zx}-A{8}oHsiNKQimYQU&FYmotFAn4VaQ8C}1Q*qG9~V}YE4PRr?aWRd@576=1HPQR
z9O4>X5zuXFA6T8KQrffM@Lw$+ycTh{QqsBrrPmqFCp3j)3|!^ytz)hvB;$!LYVx5#
z3r)VMqhq3SMckC!fa=3em770sPF-eI>k+JMV}}*3M6Hv6h7fTxDSLU5;PScKV_R&
z@tIVKngRUYAi}DE^MfNkJ^=ty^2NMiVI)_7cy0b7G}U;ET1CE{kmPQpsD&(j<&H4$
zpZ~{1q(Yq5dMl{@QSaBu3iCc@2FyS$k*mkC!MM-gr-oN>dSoWt-_2eQJ&G&D?qv4l
z@Sd-D3vXK4G{^3Uc0L(8eD{H@%xXMZ^K$
zRR%P-T3`=Uez(*GEZFBfq70%T>DV#m*YDxWxb+xHn**@=2oqK&3Kq-c+pFvqO!$H|
z18Qf5R&Y|pKm5ph$iu+9(?vj>7mq{Fz8iO!wWJ==(R8BZ4Cja*Ff*ieM2YyM<|}OR
z9aVYDFqufqvQ%@4rS>yo8j}S|eeP-Umnu?zd!lOMW~>4lE4z#;!!BkAFn5L*>Ar_P
zDr{{MQP*xV9UP*ZQ|TJ91@la}whU=btRGK6ucLMryOHu42VRi(^T)Z`#_!!$mA?+Q
zgW3=1o1ryDb$wm1(
zSk`#TbFCr_oshf8*1tb$RG{%$t(q2mH+6>0RyN19io(i$8y@<}oQ#&D}fehFWU1G9m`T$k$uxq_fqAESWV
zWjpzM0D*hQob3v98URKnYZ)_lfGt>S0IlX(0r&zVR3$u9C<Lo!5<|niQ
z4}ijfCzBtfqJepUh3Q-tRxt4Sf#+})pdL<>eiXMp1HF*@ehc^SQQ`ek`GJ4i0G-HH
zOnLs6WcSU2LihCqQ=WiZB-D9SU0uC|i&zJ*eS)(EyyI&6&h5x5(3NnSvu&ga-bU;B
z6TyaizN_^a(`n8m=V2XC`2u9a*M5yZ%XNVs|E-RmXI_2n)fS<|h2Q(p#P?ked_o&>
zm*>!Lk(SVgz04l2}caRP<{7P(n^mC(qC8vIwMl-n_e9^p$f86fZk7Yal)+>J^
z1yeagH+{k}-TW6P41u-*&AY>p%=p3z#8lQ=e2?pR)>o_ETlDZ?+rIB{Zl1hd^$s$e
z&SGEQVdC&VuqnCL7S*|_@&QknDFcGx5Cb|FqRvtV$#{+uk5?a
ze5`-?c6`(7Ue8Z~odPM7T`p>rMDx5X!<)JlnQ#(n(rlVbZatFbd2cn&)pOUKU8$ZU
zLyYU!QSx7NueuA=Qg$E50BKBGIaJzaqPQ&s0AgS1c~hLh7~K%;8WTS0;*SFy0UEbLs7SadvPij7Xk)Rovi$9Cg&1n$O5+$t5w8dM%&WFq4vwZh_WJP|
zA!oFx)}UKx6mh
zvcy5WJ19Iqu3=Vnbp^e(R&yp2pVOGj8l0+8`{C3l^c!9y?uC_Q`Q-BeEWsDHM%Swk
zF91-%Jq)4lIimcCv4hdqN1vEdKoeQaVyJ4@WvVKcjVC~)ug=J>og+VA8-MOLl0pQT
z78jnSLxffAoxP4|(=qNCTj#3ae*&Vl|9%2RZX!cdiUd0t9P&k)mg8#tZxuOiHUb-d
zU}&x~C2D_ni`sN}&`9N+8eQJNGNgAK3(8!6WQUjv)Jm;X>o)mT0Ryz(;K3f-oUn1d}Hf_|_HWOC@#_J)f8OzN9
zi39Yd+-n~R)#i{*vEwlbIifNZ%%_qqiHH-hh{V^d*bxZ>?vy{va+}5~=6z7U
zu^OpAZ7_L`NDva?Tk{nzO)Gn=!Gbw@&MFb8&SWc5Z2
z$)Hrg8A%X%a{_9&I>TB6(UQ?V?lDninw!UB?zNNpgQYfV7>y`N!@dM-fkF%7aMyU1
z)9n7G<^8o^+<#;P_NMb51Trfc1Ps|2pO;i#mRPW|f&dG9LEzJ|9lDv5^C9$_1GNTC
z0AWsNL&O=O{uJKP@m%2Ze1A0XY}6a**MaJaQ1DWqrK*%7eC<>KYWe^ygA{sR&Yd6?
zK~EKY_N4y6hrK7Nx9`(&1u&D_9`xJ!YXl9V0&s!TjOdUNOwZ%A6Tm(I!>(B9mS)v(
z`r7uY;J({xgI00qKT11=r+xR~e`L3#J~JB|1i-PcwS#W80Z7d_J67ce;M}e+0G0*7
zCLOv@4KD#%YYbq|eU4mf`0@w%q7j5vXi+)?E&xhDoKytH0`|cl(C(mKn(Y{1=t$)v
zl;#WY9;8r8T@f4}^Dt#4vN)vzok+4vkBj?qdqGWIMevBD?iNcC=
zsEG4Z7-$|3lawOMb&!V_qxre4aXEf!sgFsm=_&@q?KcBJc*hH#V)M1@?QV|Nj0
zE=E0QX*W-tq2{9B&HC~gh-jDSH_y5+d#zRA{NEq;F!rj
z54kRCh;@95A8#QUdRl>W6!pCmT0TJG9sF)cCHoDM++vAd7%4;;%*1zzdRUbt$pYuTr;wKB%DjJ|0pScQ{o
zw|6HLbwQW{MZe_j2Fo24BRSueBK|R#thiA@uIzcKLELce&oj&+^OeFO;VftB}cE@=)+88s@E
zzbHk#xLz1FfISJCs%vK8R6tz#Xv3s#CUnI*^l
zcOo_4hU<4M)^^Of&q@{vX{Z-a-eBy8#BaF6M(m_E>*r+mCr^thI&qs}(xFCC@$&NN
z0_RmonZ(o#+nLTUI~l1jn<=m142aaFI+u)6$tBo&E)FW>ukFcd>ocJ4Z(t
zh@P23ZE|x1m^k2?oFV2E$DarUVRsE-5hqx6Uwe*gobl1~-jFmG!
z2%IC>O%a~vCKVP6B1yhL9q+BJ#Uk#4^hKNh4UV)zyi`P0R{TCU=X!B)tij?yY2>N%Nz3y(<%=uf#R}LnH^#&
zD^GR^dOTfD%cKkZ4XqRpk(&k)d5!WaW4mZ-I)&m=6VSMawNQOXnjqdjNi@C+&AjFV
zUpXE)`_kr+-xYmXKvK6%jjvHv6h$Yr#QKg-p`FyDx?R@y^YC!x6D^hboR}}XM5NXu
z5lBDQ&)$V}@$x<^JG*Gvma?q-Ad*}93+_p%i4l>E(z&j#cYzG9RJbYPXM3}^2R)<$
z9Qt7lh3Qz19vB7O-fC14;>o6%=9ZE1@0trFvN0X7*3?
z3RSng2yJd9Dy*gDjOr0N=bowgFsGt
zZAMADNjfoaB#}Oq6rs0w!4#ho=B>Mv4pTSr8I8W@NNqsLc!HVsrBtwk!}QCc-e`L-
z7e|`mK}(I2N)|@!U)mSj;WiO>>&0*`~1133K}0xI}A
z95j>|A+90~A5X@Xid_Rb=0BBfjsHMbNx!y%kapfG0!u^_~e(e>&RLBW+&_^FMIVU#I7&e}Bl!w>^ymZZ$;!
z2klBCv{4NTU(4ewjXXk*NH{gaP9vcnd1z_Oe@J%$;eDkf7Cj&Jn!;KfBOV&@rXutV
z0DKDXpp~QlJA*=j_^I!#{U_(Hp9kY<|Fw^yZ9N~#vn#)PoXoWeK^Hy0GwDRM{~OK4
z0ZX9aOK4&>32?Xj@7xC%m%wlROdJ9ZR28NF4z);aBOd_YQyc=;ME%g45^yW>5_Vio
zn7+I2)#Ym!CSG`|!6?Ih5ksxPw`I
z&j%$QHBQkYx>AAN#`8zb>zc|njfXXy#V~_mN3$o3F*g=NFZ*XQ^~%dn-xu8E2EXG(
z1cydDPVT4AoxEdUU7vCJ%I?d&gCtzBmE|SHp3KDVNra9-E$2`4Ve!zlN9|-a|DAT5
zDIxpf`NAc}`sSvYon2ZS0UIYDAJl+*JFT8Hw+=5T5JsFYNk@Ri*%x;)Y9|_w`USHw
zVER$VtcF3P{vI}xuQf6+&@TV5g-nk`xDJ&}1-hG&%qgFmcK80bKl<4F4CH`P5H1Jq
zy=ceK_tWD5TfyyyzW=^Z#Uof7NGCiqzVu_3;Hv&C!uvLr+Rjc)S<~5gQ>s~iBhgLq
zCS21+D>hp>?ga%R>^=M8GVsBO%;z_I6vh-zLZhL`Op37^lPGP?l=);K2|o;bY9LBjP!w+-k?*Uwp_Ch+oZ_*8JzFSa@Gbu^1Ux
zt-Gz}P9-OVdscY+=c5>1dUs29gFyYkkz%%)ea656sG%<{Rhb$oAc_9wcaj=>Nw{fF
zNP!|}r(RYbDjnQ|IHosQk&~wCyA^2%RExx7bM!OhL&+kgcwQv&hlg$95Wd1R6!mzy
zH4(_Uhhl9QM<;d{!)z&2rFwPyQpA_~Vr9AYfbFgc+f?IvlrGjIZ~|T3IG!C{xz?PG
zE%nCvk(YK9jqb=RUL+P*lbgHX)})RdWm)Og1rWlj`6ku7Q?YZ&5i3KOfmit_*=ggA
zbp|J)`OsDLuC***wne#+1fg`-w#Lx?+voM*`sEL?AK>*x2K$4lmF9GjCsN^4SW1_|
z0S6M1L-za?YHmX@pBgIsTV=GUYA46WhK6Av7zq^6mV(bebZ=Fv8oxi&VfUQU7@?Pe
zt8&g)JS3hRU9vXCRQu_Zy50e#p_4lvb+xp9b$IWx@UE=N?!)C;5#Oh3#Q~IabB~d=OkAamwB;?1G@yi0-+|g?5s&c}4bk*QcC66EKU|)~)8M`+ABrZ}Io>X42bo
zq&X3q;uUg%MLn|PF>Afr@hX-XS6$)pHWQNdfrqAa2kuA>JGJ*I*@z%%LU=^nfzsfu
zxe?i3PHEXQR=W$3+^o3lC(~we8xYd0-q#kYN$peKQFObl;tHgcDr2zk=aH}6kH6$$
zCsWgfN>$4bHVUXhH+FapNqxZW!sj~6Bn$vL4apDg8?4m0^w@-@HNtY7vx7Uj)E@6f
z7y8kba{yWwpeK)-x`POH+!SA5g%72{01O@jWW_*n4(xC=LDsa+Td6`H-8B>MFhV`?
z7da@{AAZu97k_`Iw0p}E#_UU`wsAJz<=jwW_)*N`dBZ*0e^1#go1ga}z|(#pRa!4}
z>wlK{!8Y*cTXNvRyN##k`H&3)EsETb=G=~3Ln)1N$rJK&ghJ874c*;S&<2k`(2_5|
zf*~OriBRB*qe$R4u5#io2)tD_Qn9`B=q|MCqFJe(r5^YTz5nk26+rtwwjd(&WlKd)
zbZ2D(zobo4iEp6WeCTn&YnzL9fyfOv3Gc;uw6NqTE@7;MJKx2-P7Oa7Pa$bUJUP>-^-{duhn<`3
zn_!e%8#c%siQe1h3~q0(Yp)zL#)U~$ZcvnG4B5lGmKxU`pVX`1({ykyeZw%(7%M;4
zi%AnR$f&H1>o&sF^9?~AMY-mkj+QKOR_gE1OWI}3QtwuKPK(NUDnno-AM;wVVq(>j
zXgE{|@8qNtjaR=0qqVn5MmttW
zw1G|eNmHqjn3$N(z-^aLScOWKNV8Wqjq`Hu{>b??VRA5Y<5KRO@v9<iwr
z7YojngSGzh-wi%`U&4K;$lg1PK%41qQFbB>!-_~yKJVBe2U6yv7otP=Jx8Ww0lytf
z^9x^ocxctk1|(}Z=+{uZQ@4M2b3I_jLx52`h#q;~w5Q4@UQFrb;PHVfOi}6w7P@rf?
zOAAjU^W-RM3KMwTe`ezCZdrKVTmdTY_@0Q1Q^d-%RPnGMq7^#jiQ2JEMJEo7*wkq=^y0FchUtKwqQ{ZdP{3#<$%JHxt$yjzk
zAj~xLO5Qp{6w+nc4~W8YzkT2enonj9kSl&Y9a!wjA|Vr+=54pAwm??XB{V~CJ4-Y$
z@+D1h2Hw{!khSe}Exwec?O86m{1exB?af!VKKjKNQ=QW_>E`8Oko(W8y=n!$N<_Mh
zf7!3SP76;fH^szuKmR?kw1@*|qN5(zo##Z*j9==ILk|Dq#2%
zTP&xg5`N_4-pM6d!+WkARa2uB873l^_tkCqalsg8xE&~NBOK-^=?~7%763N%ri2&1
zM#!OonZ`6Y@eT;+pjspdeW@;~8~M{JAw_qC3p5V89Vkm493Lc>Qs|Sc8bwapJ|@Va__d
zIk#2vFaB>&AzoJD)7Uw^?F7
zKAYRV@^=I=mLbp!s{s04al{3FOJKW0@Xs=h%WBs7tOx}7|H18-uYN!c!64w}!7Qbh
z%uM@M);GDz(piso9iYnGk-nhp)zgVipJ|wpsA_oH?nZmxRdmxZDd>8yYgYb)aYvE%
zzTrgCwv8j?JA-4Vdlg83omcXQLv$(y8`Hhk;ymk?aM{K$jOEi&nU#r8rfJSY>;6hK
z-Ot%4-7iS-bbliH?u9M&IL_HYw8V`fV>(3UHw*akeuUkI34LAUWuq4h#JlgZOhV#c
zsE--N%Vg^;@VX!ly`{MH7JN7cy1HVB{m;+BY7-MOX%>;xazyCxr8auC4YudQUlUr%
zYkHFrugvdL1`3HJ}A`lLh~Wv^-*S#iVD~k
zpXEWaShT~HpR}KP|D=pPtJ=nEU5p>zR(`7R=~Xosl(Gq1;MDeN(~i|saObLIX@?|?
z@O^CZFd3iOAXNC;nq&dCH)xef>3vH-o~`kulj7BL=k>ALy1|G1J#Efy;6%}5v=$;*
zZw`+S5WNd*e{lLmY}0w#5ED47oS3p6Xg%s)RY&IOteGFAX@{UuFhXP07w`1Ccb(4a
z#1~1-w-W2h99b-?b%as_!*JzKZs9dlyscabdbrbSg8VYO4Nc#FZb$f7Q2WilbgncS^gQ}}&Gyd~l5_|hAxO~bZ$JlNAVKX2eCliN
zLs2VMLdZB>p}l_`ycemXm+S9@5nzlq3y5$ox3ne(7W*b9COlN!fC1Eh7m3utTDp00La=I*x;zL3s)jENixti6N=_kKl10u*5B+Ik@3Lm_Yc_j8BS2
z)T;;4%d2!78LQ$$u*KpWZphz(bONog+=OwwlF=~{agFE^>7@w^cB#jz{_I4PU5&oj
z;3dxE)G74_Q~kU>oq7iwJKbEh%%d7#Gr1M|3GPgLp1fruH9rxYJ;o}rJzNVCh4hA}
z!gMDwNzM${beM0j$>P{8V9Ao*vY6T1C=Scy?*TknsM>8c^yZ%BgFr5_YSjub0%T<1
z-CgbkIb6O^5VKt7#htr-1if(4~(qyWUl@tL@VY@W-KF2kiJ2XxMTD58I`zqKI@Udk(-6r
z(LIH19OL!eKKb-OM&8Zj>09_Xq|MW}TJUx}Iden<{y7zRm;zJ^?ErkWRECIoBDefx
zzofJM_ZaVY^D(xjU%Xfv*;mIk3wW{@y3hw$&!(5(3gx2%D}=`gAM~@G*=()~fqyIS
zT79*|cu-+O`(`K45bZu_2gAqH>`}jHY6wb}QH3KS9$ktfvSGOP_T=xR!Iqhp)Cufc
zHN*CY=weV(-0b#M&nhZ31&x5*zO557J~fzT>3AG6)6IMmnHe{bYQ+o19I}!AsSr-Z
zXi)aJ5sU!0@LkyNK~3Y=>}dt#kN5B(PS0{#0|dn!@7gzq)m#^4_9``k5;5;<*8>B5
z$_RzHkQ}9%1Cr!Qlb;N-cU_C9^Z!nvmWw!}O=pNz#s%rrNix7K$;)r2ee4o5Rvp8
zq+eLWwgpRGX|Jp`lCP;@!;}8t*3b`Nzz!_cXCtm4Acd(8oajwB<2H464EMjSA{F#b
zehfhPGq_S~-+?o6X
z**P^k+w{D`Nib6g;wMb~j+*~nUKy5wIc4n^TP!a#BG5XMLUJiiN(fdL>dNp2F!AaCI#G}GcL
zWr3D%Rl!q+Xcahv``%=#ZI<9|OV0sg*GY4(t09N7aN9{MeEwfFsYkb+hK(Kh>t=>k
z%M4PKpt{b!(#F;jGnq1GR^zr2=FzG~(&C)YMv1OCEUV=soKt+i-{gzh)e*LxnTBA&
zQqia(`iol3OjSZk5F{QyCICJ#Y0yWK#ZRQb>f~
zoJ%v(4%XO5G?Snbz6Z(93yP6sJMn|~7`PsMdKju
z$L<}CfX_l38ocb`4#=|>x?9~xyml_Lkn5T~HM?J@kIKcAvV4FZwC-mlcA;_L;~`1*
zv*!K}psWbEZkYK$43axW&#uBuPEJC(VJ()gUBC#4k6>2xpD^HzsmSGkdSzDQF@hcj
z7kN!dpcN1Q!M~1-cYE+~ZOFuhzn}J`Q__GrR?uA{g-FtVQ{jt!gP5vBsH3yc9&{P=
zAL^_Q{Q3dIEF5UL#DB9VV56~r^|JHtWs!F`^8e9YJbS=g3n<8P->~FvM+RoS%>-Do
zBx`gGZ!g}D?1`Y*S7d2ixD16Nxp=U)x+1_Vaz^uGyt(W4{Ybj?7)nQ3`F^+HdOTR=nWVg%b;WuR>^EA!_S+?5Hc+0I!#n5{=n7{Wd
z3+L{@)Wqc9s~d1VNsMD+ikGI^Tf=S(5FLV)cCBMqmO71UWfZ5v*aT%TA7cq-mV;^y
zWDre9=N0>hl_J!#n
zu9YnonZc6Jb9cm7eUtntbu8_nAnJ%#R~_S~*QNcV99I{MTtbqIg!5zT4}G&lh1nJq
zbs=9LYxR5m%D}zaXt6H}qW^mG{&E@}4UeFU=Z41D319J>(DImoXc@@pAJSVLWTY=e8$Mjh~1sxt>d{g6PQ?c@A
z;-^=4487ZdTZy0{P-Pq|9;qtnWS2JY96fRNki=o_?~b`!k_`dr6&2xmtP#WCL~(@8
zN58ULx;2gd`bCR?akmx(aN$i&-u=td9VUfQoD}X6lJe_D)ShneM_!;$1m=UjdMzz7
z!zu{u9mF+9a{k7gLW^a~v997iQ5hL=4}otGo41vD-CkYAJckp0lz$Fn;qNS0PTR;t
z3{K8KlRxw7T9iiYcb=>0Z5eLsy>-~`tRFZAlK#&Q7%tDdAW~uXXsxA@pCFCMc_R62
z;$s26y7E{9hoXHFKFvZY0L*%s(ap3x^Y##NIjx_ZCf)amZZN(D9je!0AI0*dG*m8V
zTFZD2x01Kfay{m;5!x}e;7ZlxEu<)(+m2(aLhP6y4ZmU`*31u71}RViKr*IC4LjwH
zx@B>A4Zj5_y*@+oCJnyuTldM5lE
zD$>o|+|sBMZM>>Gg+HYZ0W$U{ef?4!Eqlz;mJ#c;Dxp)o75VR*YzauA$t4(@vs+#}
z5l24Ma*CPj)5_s<8A=nb15MSvG?;{4^GTv%_=ajNkt-L0XTNyEGwT&I|N3Ex%f>5n
z2)5RIioBx|j#u>zdt2rj+88Ak3PZ_4x!IZ_-;uRhzV%S7RC{6XdV9aOch$od8(s-Y
z`eFXi8
zF5zXlx5`0DzRrM7TVQ-iW1K0atRr2YF^90}%`2e-(4O$4y_PKD+<07zFEoc_H?KJ*pX8xKMU{n89=
z{VYk)r+t8^6$|^mTV?!m0<~L0OI`op6e$TX&bV*TZnEbIde}mTS<+|A)B3~)B6E)X
zu_-lU$P6h6OocdP`}#%PXSAvbrdBy;pk-K{z2?VoOA
zHx#94JBTR?NA(DbrZtN{8D6D2%$JF$w`%oV_bO#;`8YGbY1S)0x4`?X8pZmyv??4{
zwIiva*Ra!KuRPEnRa0EXW+@znX*H=r`-iaxqB8ipE3|#Sm?d#IQG1r8@BXcxYOd+h
z1m5^9E35`ne|v2=4w~cV2&#y8rhSTS3caExc1$1;DDCjj0-t~Y*wb|>Dw^J4X5>kX
zIu%RPsb0Nf=P!1_iPZiSLfH!p!^EMZmkFs|=?9`&dAFXpzr%XCIDs(*nw1kR0-
zF5`G!VcR6&!J{>D#|-40+Hai}EV0@0=N=I1BOFU2csVHKl>*D@@s2|!_!2laIuR7g
zvBvn_xzs?bE(fyidkT!rCdmp5xjj7+-;!{*NfQ6Rw(pc`@;~jb949mLf+%``PM6fhyt1}CKl78M|Gz%M8uNe4H}kK-WR
zpCU5!-NU(9^7p1t!fjTW?!*t>Pe|iFhK}9qoaq$1S96yPVOm8}FH=t>n2?$=*11}TZhgu6YKt9sSd^pAW4YQ&O##sf3
z)(ItqiBoBjN_K;ly&{1a$hYN@wC&nw5`fEAj$%iS(Q_CGOcD#KTI=q|l9dvfMtZAf
z2W0)^NV!N+k7uR;=|W8LHwrc*bimY|um5%V0^9S~+NKV0jVRg^$->fmus)_*N|sHq
zMhRIxoU<3uU_Mbi#!l&4aYPUDO6{i<~I`I+>b-rQve)+G;9+97?x
z`%$p(w}GKvy2ILmq>_v;^O(sRZ*fAa`AsBSQV1#g{MCZ=;s_q}8M1JsDnH+RxT
zSwW}GCTL&GO}v8S9^8kH@=!0be2TYsN0$Xg-Q0aN@SSg4vi{=h&^R~!5dG5KD*y4w
zHLq4hn?Mmff~hJ}8Ooe5am2Lj%H0~Eas;YlH=2LLMa3?@rw~Kuzo9CQTRJ64i9+ylNdbGv%`~4#(liTd#ZOVgXw|=_
zTk)lF;0^c){9gL?flp}l$6G%|(BDtxlMUaWZepk3T;o&qRfdwN<}n@BbO+!Y(%uVR
zw_5Q1{`LTSvE5K+UtE3SCaUF5)w{4W`dq}M6Svc`c2R$}CgkNh&H81Krt>fQg?`KT
z6|RodqLL@yvK8tRko)xOi94-i$9L(7X5qjN=j?1(`<7|&(#Wo!=8;>4VWA_N*;aX@
z%GiWIQ*1-GYYPdO>Co%s_QPe+a$|18PFMfKrjcLa=Lf#xyYEJ#3<;B_bj3Hfw6?~z
z=)_XLfyILXlOsLg7=w4+iR}_27xDK7KHyUcQQ@CQ|2>dHE5CUi)PZvZ1m%8T+y0#g
z?L!LxJvS>s?J5qDKL`nfo*%CMBWjhTn#2|_t`-#7u9`&uHu-txZ<<&=Z`8dIlRu8h
z0J5=yV6fsDFq3+INczW5{_9^g^*dZSd_RzwW1Wghg?2BG~!=o;$X}LNL4a*sgONSLmLcha(Fs$Qe~+9Lmf0E~!q%RLmnzpPMs5H-+k`{+5K
zlfSxH6aEc<>g@>{m8g|Lx)&KCnKV|7Rupxjs*M5Mj|&~S{TL&J+W!1EGACESCI9{W
zvCcAp3_ul?fFTU9h)-6EkSt5k9jRXBew)wzee*#YgI#bO>`>d?N+tQ|fUVdmUhezN
zov6+OI%E?7eB#2E(gLu-&aUqNSs(?nihD0Rx0o{MGmr$QLbCw7
zJ`l#vV3-UB*WxuH8mL)KK;Re~?wTpQNmV}P@^3TPA#u}#0I;S5Rw0+WGhNnhkC5pa
zrBv#E2)8MrnWAzs-=ViD(iI=wCw=6u6zRPF_1k!AE>!iTGJzyz9Xe)Ym=KH*w&QSt
zB1J!ej;YkQEgZ5)F1Gx=MbTW-^jzN8ocpYfwVdAKbIAUks?qa5k#ys`+gOi%KAe7|
z;8_#UQPg;g|Apw8ZqbNTs`S~AhOguFOh=1OAa~-2Srk&`2uwqI(*_wWtH&;_00>Yl
zYc5um87}C3QU79Oio)aWCO@Sg
zXuJhh4PA?-mX?`XT?tw_fxR-lZBnF3?0VYRjUo|S@suEb4QiVSYyMJje$m$XLad%=
zScD;99t2MBA1%vp}fndgmc3YtAfa?>i>KH^L$4sCfuGr)w;EVu=sh!xbMcsnRVG>X(taCZ
z*SSHF&`@eZY2Bg^r}`H#7Lme|M9VY#~D9uk4zwK71lv$6CokX5*)MLbw5zcma
zKPFpZB~9XoAaCQ7WmQ68MCk|rHF1M(_oM*-DSyx%vlFT42zc8Q(0KU=uXhM^Qw&Ap
zW-#c>5O*2JjIVMpqz5n?BgpFNLfW5-8%LZWUOmnnj`ovBDP}L?9Y!wH)(QHgnm_(F
z?j+RNNY0~Q-&{xO(AX&KYB(y;Y!@dT5aE8*Tp3jfr)ifrpT|GrFLy8+5P2)3$i%Dk
z^(<)EP<*2|FM%~EZ8do=c!V&9my9@qjYb5}b^e|RpHmNijrL4#z-s$VgVlgMKLYM4
z=piPIV8?#O+|v7y8AX`5k}ODth^AMS=cPlpjl>KiI9dInk#{38#7C)90PmhTLChj^
zj+BmraZ+|6-Kb6lx77zc=(PVnNqFMyjJae!np!R{fgzk8(mZ4pB5uM+k|byPM>3RK
z-vk>IYXC)T^lOwg&JkSD-?R7?!nA`~C4KScH)TB87bPu{RQ4ZHV9W{CH9fVY*3U@w
zY$P-?OGj`^t!m!Ga2L+Me>7g@(_)&z^U1(iKS)@u(I1VOaCBTEMiq1_Zj=$FL5G*R}@Uy{$Z-B5meU)6kqH0l+TyKlDRuCWS7_F6jzqvyy6-Rp;nXqs~_EMbpEK|IM9k>F#Zd)3aSbG{!Faq4%p8;
z!-SNt9<^Sg{(^nJ+>%5@>Hz1HJ781!|1o+Ma5`|@ZI<(UJKuNZ?J!SGO#$dEkZ@$o
zLCaxH>Nn}Dd!DGnjqN~l1YCcFCg?PYV#
ztI%b8@!lbjp#SY&%e?lVahce_xEp9$Er2E|x!aXKGt!9%@t1P(O+LM8TsoMT6hdf1
z%0~F7O4%EX_PLz8a!K|_Mlj@qhiZp?yc?B4eW~+=JZrunO!4QK^4IfJuaK{usa_u=
z>rmgftb(W9#oTKVo)3$X!4v!iUU*XM-pl2&?^}ZBII6_{;intr_l=4D;q^uvius*GFVtaqa7O&NtPYH<_Ym5+?LF5w8pbQ
zNpH4=ouuUcTlyzSbIbf9-?W;8x^>xqk&=b2wy7bdI8x!eu+l@-%t5()tgi%?;Cnao
zs{n7#Zu8EOzH8eg^`lk7VVj~kt68E{eoJoRU7yq_VEqmny_A+8C$U=feq7?*8OJO5
zvicV_GKMBj4yHiz@W8wV^5&0Xkw)Z7CVvJ)pS5KtB?7vq&z}MsGQS5h=xh7+r{RqZ
zN6;-_-2%ywdzVA87Rf$R`e%V-x&3cM?|6=h_X^^E`qsFaHE9MFo
zK~qbB7%*%X5T%D-{)Gbsi^a;w=e<{dgg;5-5CIGKp=`@!Li8!`sTLa5`X6S@k1C3U
zXXh>XbM*pCTFbs)JPF?-OZ=3>?_}|3AR1!$-U#=Geq6SVypnAAV#4dY&8a*`w!8b`
z{g+bg+!g+^@7-Dyv3ETmXXW7XwpJ$YNWI*&y_+b*<@mmat!W9
zXFEn|bx9B!`6@G<$jj^1@QM*)OS3$k6GgtTEWFa9qGqQMVkO7CQIYkZ9Xv(!|85qz*PTkBhFgWLKr28$$u^l+t)&VADYBlkq
zS~4pSsC)6yll2$=Tlj?oJP{q5zRnMF!)?NfLS2XkRjOyd8Ow(!L#KpW<^p2=)q{=zd)u?U6gV?`{lDf|R
zxc>u00}6qfNrK>c`r2-ldLLqSN)d(GyUC5PMQ>`^l*3sA)A(ZfoF!@9fiinaNjL!D
zMv8tT5zQN^Q9+;aRSDi_OYiM<$cRp^;vd5*2wvAgsy4fAdYLYt
z89_-C8ggp2m>GsOgm7xVNJtR7*2B{S1NA;!Kcn`%fC(NJW=@yy9cs~e@OGAHRqf6A
zUoWck0>FW}=c5VrfibQ~HWYLQAY2>H;N~BQYLw9>SzXMJ!F*GQ$tZfWfzWm70)*GQiwRLak*p2UP!hm@g{GyBmX&3i@
zXqU?CxTwvFOB+n!K3`xM(C=x#VR`8lg#^ioUXz_Ee!t45a0j<
zl#_s~g%NsOP$dZi!ju0(mx-gm=o0vnBRvqReEjboq}8a88P4a3~s2aCRNf$bbOl3X;cohH>0Mpcw@=(r%@bn-O)fdGapfB
z*?8GRS(oJ$?n6kIm^S%kxK-DALCm+`#6$$nA{?iCK3#ETxIk#5YMEBaOryM&gnh=y
z+Ie|y1EBUR7R$G6I0(^W(!&3@F*l|XVR|P|LtVn7cuia2i?`5Z-4a1H_MtWVn
ze&?qU@_7dydvCU&3EoP&V$fD>pQ#U!uT3c&)e#x~VP<8P>~b9nLQlVx0|+eJmKr_R
z+I9n}C0i_MEY&KdrArRN8-UJgWUiuL$+Y!{kKPA#Y{^4Z8~IrI!1<#^GJEf(1o$3-
zb|B!V!*uR)@zRNd7;U_9Y9_5CjKz(|w2b@5YcPN{%KtKaUHK%U*Zz-wI?M);Ux
zF&_vWaM-pHg~Ldg%0^HROFUwY&k0pPTf72tjdn`j$B^QYbD5xhc8{>C2`v>hC3Bnq0i!AGp+oA%!}?A?1xQoyeR
zSac|AO)DKamFj-1ei{wi&UbR`nIXApHjuk2mWb9$EhW=1hZe2%`P4dgEe|~uBx{B4
zRO~-5^AQ&3D`&_Ja}1N58Sm=q8{~xB_F}kk*>*8(GSdAOowyhI8
zY3ww%-Pm@c#rw2?tE>%+U}+ZS()
z2fWQNpu$|ZUu@*enzP^{4%O0rXHG(&H(kB2kaGK(j1ss?KS-$HrnEJTL}XZhY$Ow$
zbbJ3(`$)GDX&qk44HGM;PB!tgHCUTaP-CK3(Ex(RJN=(Ja}5`hn#;9V=w
z>94f3vU}z-Bh%sNS^B|u^jvm{$>2FU@0PPT>-aYdP+8cg=+xhj(L2gBp{hb2Jg4#!
z>9B06?>*$#_!xTJZT%)ahkd94-*fP)=fYGbV;|jqOUDs$5)LkYd6s3bJ_s4T$?)|4b}2W@hiLE6fuS30iS5{d*wo+Q8Mk@
zrNvuoP4|2rj+~{Vm2>J)RZ2^}v+m5b2ho*rh*?fVqD?QlGA~b3QML(WTp={PB(rdV
zixJ}_P1sD^gHO$0Pki61(#KP6
zNO+$Q^LQWD9HX*5VcLOa=k?Fm*H7{SZ><0A`=gMtPc?x5G$2X^*fHtDH(mcI_{Ts4
zl9QYe{C^HDXyaW~>}Ga1gpMV;z
zA}`Raz^STzfmQ}*!oXStJJ7K+#
zMbw%2OR3_ooq1TQpH7K>ag3JBJihYAqeC&WQK(dStaJCmAjF14A&dFa9zgi}?>S10
zIJ_!Bo-C@2d%*98fd{_ae;nATK4ZQBW^H3fSDckb^>??WKA?7
zJ;`%bFAIt=VNLhoKRf>~Gsfr@lbcO)(|;d0Gy7e4g5VZ2z6q9oKPuWVBzdC_I5k1>
z{wzj7glFUy=Sdk_Q%1GP!M0^_To|0gm(O6dDK?V2u9IFRb3|=
z?eQu^?4bRa9jmZZw{}x6qng*At-Sqn1;T_juzrvlW|pSrC_L3DP|_2RCk4Ehe-%=i
zP#OCXR5Pw_cY7jUKIT2#Jm!H~KF$NiChK9qaH5-Q8;2t{yaJRD=pBsB{c;EQy{B$S
zM;?Y}z%xw<6Uor9SQZYd@H8!JX+@+YF@te=oBK}JXkQ*KcGjMnn(em0($lq!HNw##
zH=6Qm%|mrzK6j!~BINa5CM#J*!GBu~_h!Mru^yI@o`T)0v?!LEXhFd=id8~^;O|Tn
z^%3+t))<0)d#
zx=-Es6g!(E9aDNNaLVbqFoEIdN)$(3ilyC?mFdpEbK`4$lLF(~Qa_LcC0kGqrYv3V
zL^X0d!komPiq!LLAind&572OcLNU`Ng%Osn`-jp~U^VQBkXhJJ0RG!Mh9vH{oe=w@
z11_2ZEJRSBEnV$%ZF?$&E=WXidR}Ggg;>h!a|5mS~!E2gC
zihof_X2gcKegX&qHOs~Kn2RST(;qN-7!?z=ZxzlS0iU^5%@VdvuoX5N{>+L}6U9^2
z%^0y*)v(Xq1_QJ}?5yczO>N^u&fDmk^P&O9SKR!+M{;9Lgy?(i=Y|Mz2_S?C26nEr
z@ZA2kXIJ?@yLVfmYK?eFyVQ9VzNcAfiG}+y`{p{4Z
z!^sR3%I3gh#=LEi_D|3|2D~85I#EplXylnfuEU3xW-q^nc{aJud+foD
zd-nCab61Dg#{j@UY}YZf#<7hF!f$q2n?!BoRU@_ZtYA1=MmG^p@lq^^-565(bz*il
z*9#x;XkOiJRIE0e+GF$B-fetKB~eKm2`0}~U;Al77!kje&LVgwPKXjkJGj*oUE4md
zeWSRrZcT}il6+{zQXgpgKxc=h8z6|CRE0-}yn+g*C!uK|zFFkdn`%=GFv;aH-0*eK
z%Cogy!k7q}Bmwl@oluuAWu$rhz3=3C29e*Kp#uQ7^N5Dd8)POIM_iCvefn-<-~p#3
z?G+czd=^vy5wPlW?RaT>0)kW8aJd`;S!J=J#5o%dX1>;+jm$?Ekb3TRfDH}%%ltA
z7kkpt(8cC-f`NYEocHf>$9Zk8jw(v|vjLC_ky#dX)Ss#t?Tl&US=wmSyox(8j%Tt;
z&u68mXm}qOLf!a;3@uMA_7gC$RgQ4*uUI)tOk?_8|F^t5E^`>Yc=SDKBf-{*C_~T0
zse)hB-0-Q19X;vc*`mjqe$y+Y)Z2yuzb!4WVp{iTKigz#L(}n~zgj+#vyC=QU
z?rdWEOk_=`sX%WE?8aoMe{e|<)ydnsax|Tir(<#9ul-SPA<-IaRJn~8@oX&
zK$Gu%^?EaT!y<6QB6|ts3MlvJGj4wO+57%@YAV3~0j-zMHib{_a^w+vA7HWETcTX_*bhi;%+qU&u7Vi6{QepJd!0LG?@;(&Q>3@l
zbZ3dw*@FPnm#_c7lD63$@V^aI6B7NmvBE@QlGIEXjrHQcsZcSC0kbk^C@e0y*b`du
zKB_!|TA=%K$M#Q={QoM9tD!c5-0&1^K!Kkx;r!(&a;6s>tyF=kdEuBo0m%FBf4^85`afWVbXUYltKjjyyShaa1N}adBIuuHvNFSK{U!rU=bFC+_Qkg=AltbGyULsI?>!3;!=BC&I9P(
zAY{NT<#eNI=;>xuTf1dwVISKbg~T5`&pF4;jU@z<=+A?o20x0H$T%ar6+9bDoN*;t
znzb}{&Tn+sT)}P6)exDcBXMW^H9Fd`!Q_nxQXMK7FG-r)qbL+eN%4#yl-A)_z6^$-
zt^(tNbN-4~Pi+MES8OI17asfNCqy!xiXVG!Gr3vVsMl3?3>LGx&W)$eduf+;M*U?ot$iw?FoB>pwa=Ps@u-hOFA
z_YGNv8t@n4>LF~`-Xr`p36RWf;6UP70~X>7Jd`&;BT|2-cRCKy=a?CuK`USe*5c72
z^=&#pl9u8eJ;y+cN~iGV!PB#lV3ElU-mPb(xQNQSYk*fBQP}4=zmd=oDr#woozH+3
zW&J9iF;WlxP=c^T4`a_)VPNTMH)YCr$b@~bAOa!f58sXHl?OcpogeeSClF6
zlJP3jlGE?j;Mq0byFy^WFt$R25*uaoso2Q7Urt^_yN|GV&2e<$^B~eG~modGi~Dp@YqJA
znr>!WTchxL9!|N5Q=?{3-T?jo%38b4GiF_Jb#|+Yy?f9;n;d5
zm3N8dv&}d$YLYmrDikErNy&MsiRU$xB4s-!nO00oArCSEE2C+%zV|mMx5LL?uT9Uc
z7^6XaQOkZiPEjgVMPKKgPC@7><#n}P(pze2)F@VxNKH@#;2+&1f*BBtFANTirSB9x
z;$aNPEJe+O7Bu*6nw`bhyKkPl`O*y68{k~mmU5yf+2!n5983IZ9JIp4m2Ruz1+4|>
ztEh99<5J7gTU8fd#8yX&IGcR6<9tjaiyta#D-gFM0Tm4;Q@w~qV{$QP+-iIJvq{2N
z-8{!;t?zp-VkXr%6eF#(t2QOt-&ggIa2F
z$-Q}~uvp)daFkU;&HYDGyK%o<^*d!q!06i6VoWj&t!cY*4VOQ1adM_d1U1yxgD&-&jW(GJIeH9Rco54cc}
zu~E3-1~DOjS0{F$IvjX;1}!I*k=+Q>&Q&$}S4P)e@DO1;TUuWYE
zIBMfi3vEyPmHvrESS^r2atUR(s#WQt^iv+E%`q&)ylqMaW}7LmpmY2C;4G$ml|?|Y
zk36MbRJovz`5{VqkoA)qLnTD@dF+Lp9UT*p{&9www}n&ycuaB~{qDXCUVtQmP@F4V
zO*3J^Wx_Bbn1s`gOY9`lN_QK(p)k`e
zuP_4`?h-f5JFnsabe1o}cWmn_S()6FBa@BXly(V*7xpk+$a>ZEUJ=YP4T=u%u7)rI
z>l~IKHIzIEB?&J9(j)~5k+YwGt<|m8Nx9W64Y;k=f%z|B!68NEPe?dl-O1K>L
z7A14JyQFPVb|(}BdkGzn+}wG8-C6aR)+FL3$Rsl>VANc!?7(P#X;>f)6PE%YRY2MS
zP-LmuTKe|xevSunK6>r@MaP$?^euSS6@|J$%#5v*@-wTj;HGqb3aVYC
z5RF(Vt9hYhlK4@EAbGy34tg!+Pm-o*6tJ&gMX_N*hv4X6C5Z?aT$#SX2w{o`3W=7X
zBa0LHbjkDL#{Ve68+E8s#+b+?k~ExQH9#9A%}fP6Vbt2hHlc!)whQXe>vKaX;lniP
zdMFK36SC`A>x@OMB@ptZa-#@mWADU=q7rR*a%RKKn>4{YL;5y7{r{wh8vG;xBFQ2O
zkbv$hmooQn_~D&Mu#X@vMKP^dF__gL!AtC2*xT9_{&%wL*
z*5rbh?P{>ox(zA4Z)_oH$kEz2f*rZNli=D8Pv-21z&#VLep#wO>mpT9hwz^UZv~9f
z1ZYMsEbLDkO@sBO2V-bn1#}uz(9mD9eX2mdmI*1kUU4i%@{Z4AT%8{mKrX8L?pdHg
z=(I}!Qi``wd|A2ln!Ppk2}WZTP3Z{*M+n+ZMHz!~74QURH+Q0(?>K{0W8`0K_PEa<
z_Jtd0QQ9-(bRYY+8XcaY@|}yjF?pr)y<`ctWyH~km!e*y-%c^#$YHz5O$sFWZ@BOdf4iAK@#
z_AUf##h6e1Zm;~2CU0)ujkD`#R-UD#;!IqWb7V{!|6&J>C~a?jirk;-2Hd>Q@Y+w>
z@DJW!&PAX_supS33+&S?=E5eyq`mH^+%UK1vvT?f#URNch>#u|$({WPd!Y$3&6$&O
zYg1(k%t)*!F(x=t2*FNWAb(vW_|d||h}o!Ju`M7*L)cHL)ZOg_(vs=0G?guGYb4LqXcHP6)Wk95X$Q&qzaOr
zCi`6MSnw{?3If}c(nEOi+Nnh;q@rPFY45Lz>VuBO)HGy!0%Bqo=Eaql3~aeG9p~$q
z71N3&EUc^kSFweMkIsOSeKWOMZVAam3d@Z}nVmKNDd
zldKYNTOO8tRxcXG3!Z||VTs{KP6KYtL?;M`U%xM}VaAj58OJwwjAx@p)!5k15muCJ
z;62OGxaWQgSL1{V2=%B6A^dKXP;woXnXmFt3OjE-yOyW!#c*_E)kRY`9kV1aQeVNK
z1Gf@-_v`VAFvObg+q#&hKTEs^!^{eECgy_6bMs?854e}Hj#EzA|AW|BXh3b`0yd(E3uLjDENG8Ysc>;`tkM8m#-O(&=*+$
zCWn9wbk3DwuL2QI4tcRk@8F*e8qYN-1QCzh>Ci6Qz}x$+UW`+@+2@V>
zUoP8tgXN-}RwCj963`9!zE6H|2^M{Tjmv8SMnNJ?sN(ILyLzKk8rSOluPzn*UcAh*~+qFfIo*b>hqllLuX9?Kr0{<
zooDJ~{O%;6c^N2gaLd?~H1pARx!UdPHs=>9
zpr!DASxt>@=&bDF@~Q|`FKjF&4Ia|asvOi`dBpc<#_s6KGhbf(injpIHKID?Ug;{s
z)5u)ba_u#WVXu7gFJanbii4m`_5lY=&ps`VubjB!p9l&Rp+{fzDPs7iB8Ewbj+PufLH?9
zZLHeAe+Tm`#~-?TlZhjb7dp;uiav<`qwa>W5waq;KyQ<%yI=iFv<-Bph|1N|#8buR
z%JRH@xS4ym97{|tvcwLP*Ycuzt)`W2wo8_rJg0`=o|oDd39i{7A=ZkdTFK0rQqn+E
z1@-suYkiLHRYNeo`UYz4dHQCcAhch6PeQ#uKLeJmqxg=0UfK_(KQhu%C<;`R5X9zg
zx^X?Zq{cAtD5DmJ8pE``!peyuGNvpZZ|K1^mcaNzC4OD?FH-E?b1|t2BPKIe6lN4?
z%v5;~#ZjC*j<2+`-{m+7jFriGq>>`afqH;yZ%mEgN1-&OXUksK2c`p1|MukID
z=j&Egkab^Xj#a~o`pM8?d^57>t!=EUy$-rC{VC+a2z3Rr54?(ldi|XWn`L_f?uij@
zKs^c#D|S#^xM-oVxm+piv&MwfWGxFlSL$6_G7uCU)N$w*2uUwkcst?i52ukm{LD3Q
z(LQGN!QSr_Id)LYDl!Ca$;!=HR6~F5$Um7TS0v~2S&XS@=zvtRr(iV2{kUZEDy=AM
zd%J)1rwMNstHfmG-(;lh3YVK~a2h2&lATbGo>&P#6`CYvurW*-KYBP;h!qPC44LD5
zFxDmFcJn{Ex&1w#V2EJr8z4_~NY&v6nrrZgr&dDwG*MYkI>_;}&zoXSaX;8{c|pDl
z{z;rDUT2gJNOu+{@C&7*M3`$=J~S>#{H4{
zfE#^(E`;~!SsDh}AK0UP(CxKVjgS3X9!qphMQH^0*AJV?dhU|5Nm`d*n9y^h83_Fs
z+rkAn6S~EFh&(Fu*-;M#&UxUO+?=sB&|O)Q)e4`~zuoC#HZkuCD<4yNR1VkwE)p8S
zg)!DRc4F(Y8I%j?apOEnwbNEPcyPGaOa49;29&MK=>cz$*ZJj@FvtTaeooWKUckFk
zIe?Qj9(U%Uj|j-e=i%`w`@`MmZP7RLpG$QS-T%rHXMnnv7SF$ubvU)vIRK0)9ao5f
zLI~3C^m?inJKVIVFk-D>qdv>4FiZ{!DIQ|<+qD!kKE=m6qhmxn2BQw>;5H0lIpCpE
zk_~v&){yns7PMm3#Nr5sHayP`Cos|yFYNv)fQcC1TCUGZDO$(PG+b)#rj8>H3Fau{
zW5MBWC8!}5bU!&~GFPAi>BCg>^RfE}T9f|mO6abLj<411rv?lw#r4#_PK6kWz<~yn
zg@!Gk2=tCit5|{KGS0F>jY`WrU|tUYJx!K8QA{$#^Yb?Wi6xZ&Py^d+sEG^0$+KF1
z%o^_0gdvo$Kd^fUj~gKK^`(VvNlbT1eg?`2FqyDK$Rt5U37a#`bvMvW4~>z}<1aB}V^LPKKyk&KNcxapE>%PiLBhd+taC5ws2|XL|%@zt(`BP7}V~{3(!C3#Moq
z=4oe78AVtBv987pLwk4Kwe>VttCWV@2cjYp$G%~yg}H5|n?VqMURjHq^Rmsgix
zs9DHGyM5uqZ*~Z3m`8Vsac~)NF8s-k{nRPG{}O@}opEqabnsI|YpTFRmwU&VN+*!M
z06fg$sKHN(=gY^#C?3p9w&?Fb{UvG1B>gFA&R-ae!)fR%V!z&Fp{HyvLPPNyovSeT
zTXmUO)*J~4D}K0V_I7Ib^Vvpju-kuOP-U4EG%0Qf6LiyjPyEoa{^@rpND4~lMo}gg
z-in);Yc|r>r(2ErITB4Ky!eec)lHgfrK&dcuw`xC10>^F
zNxt#fOn9`n7A<3G!O~D60CvV|ZWOFJRq*`D)C<_8WK=AW*Re@s4(4=wHKQlI&zAtJ
zsbxe2R*ys~IWi(8oP39c7!vt_T9`*NVnK&QNNUa`l$Ig10=iuoh76t`4$Qj>43He2
zHXF0zH~lya9c}acCp#&((IuDmEWf3@As)n51`Uvj8sB%7eLlb#bvys5cVm5a{#z(I
zywv2F7HL7uH2Q81X}4i!w8HV86e&yFL?Jgz7J<0x1B4VR6&<*Th_S%%6^uyRY1!=A
z@QsTcski5;*hO^HM}kT+zONyPi84W-MfwL_N3JnMUqbmwfTWSV4n7hbV4K)L(xI|&%zA7~7@)yK*
zFRZGoOULDC0;&}rs#rvEICWQP;R|&{b#8CrcNsx-5>8b!{jtzTlD_<=mO)x1fe=J*
zoNSn+Bb4>3W=0am!qkqC8bZG?gO3uUG$!6@cg)s}Gz-nj?DEd(U5qWO3k6B}YVT-<
zNX70KH!He`yPl}?rs?%1QNF=+&K0a@o#x-Jlgq|Y(QpxL(H0gCt0UQNm
z9L(!t7$I(QFA*{GNzzwazxL`8j!3R;vgY$|JDMp6pkY(7f}Ooc=$HSAZ8dv}5LlZRS9vu?7=UmCHYcKdfWbj#k*3S@FT0c4duFnM%wQNj*o%ReNOC@=0eKd~Orv8`C$N{gKsR)k~dmD>qXNAF+%
zhBDM|FjiSK;!gmpJ!(a|@@g&6IAX!Ekigc!Ub
zz@N}#m-6=lYCA3aDO)E3afr~kyIzyreB<~JCo<8$sFSm;{E7t2hb@mnp}XeJwqA6t
z-(x~9k=s13C+gd?P+hnq3TRnP5(5%dJe?YAdqxJr+oC;vn1`1)S|AFIIS+wwdJQZ9
z&w(%S&jRG@DsqEz;+{WfixLN
z1w8w|j%;d+zrP3wubai#lKD--w~-CKd+<*-yzx4|!E0^LkMj7u*fnu?viOv_@q`!L
zyOWprzsmRWRUU}{HjntfD(J+y)9iHQHuF8CE@;?Tidh}v%L3FSLh$uOvtIq##qU;-
z2|Ol_dpM^e<8cK1@oF*ryXY(><8StR9(=IlP(@m%22|%v;#Du(SFF7n8Fnlz0r6{<
zvwG3}pzRp*-jSsh9EO3$s=ltbN&U^{NXoT|2Tn|F?{i~KulSaZKUlJcd`HB`JM1ev
z+j7^L;1MK7_qD)&KlaH>ToEnv^DY#a{9!Gjm=Lje{`JqSH8EqTatfp8;XqcZj?X^f
zEslkQDGRlq777X}rNW7eC}GR#sb@rWC8Ej0(^oH_wa6BIR4xN7S?=#0Cv1&<5V2*r
z1`e0Q_OKlsI{Nb;aS%tljJ018T3VtC1b-tD^9cRelPC=ngBO<|V$@`$*~i5_Q`%3r
zD);@B^d2y{U*pNS1-dwOV9jFNk7NqY9U8vD(zJYjiv)&h@uaHYY4iQ%-0z-`D$}UkwP$k!wA
z=(=4$YZ538w@@_39r(vf&U4J^DOPy(A#abNA%%}Kno?C{51K6P%RlvCoSxx
zL`Tbvr61T+j8A-2Lky=DX)@mvu7sNyp4V<{X_|dyHa0z-dXq8$&%sa=Uoj=ka@`Z3
z0x`f-L5TgMwnGjitA;zgBaNR)vq2ct^!Ok
zQvJYH5oxqGpH@0wNu5@TDxOs61V#Cn%y{H5B~fB0uvEZ)6Hm)mQ
zEQLEru|FeZYv(>6{LR5e=q6E#RSrdxZu4DHDOeLQPQeny+S8yKESp1Y=}%p3R(6|z2jA~lJ|W_+v62xEZn$jiHK^SG%>#?N^>U<
zJERe#N9`mh9R!)!Hv}(=62%mOLKe=I3bT@&sP<&G0c(V+J!cd2yHB%;0cH!hE=zAi
zCuCl?MWBkHQf65uNs8S>V-~NvG6PLZ$zt**rywnS@+0`FHqNgL!CcQRh96S&SlUs#
z70@$VlaJ1w9X~{J41ARw80lJ?e@M;=!*3uG*yBFvMlCpdj=j0)&ifc^F7oAY@zN;5
zFD}5WfD|>W3%RoL*g85ERlaG7@cTT7FTD_(f^YpzTxE&Ou4-`5rYUp
z*yGtBt3G9Y5)uzRpUUY4LE&As^ny?}-tU{9{t*+i7c<$dtu%6E4fL~u4yIyT2Sg|k3Hx)|`lm_u$S>*priyEr^2~Pm5GM$4!r;5yxsDPPqIgL)>^~!9
z0t2q{30uFQuezmO8SjJ?4-Yn$ih1GK1+qxfl{J+)Iis>Ak+SnUWyb{>882aiTEi0R
z&+ZL%&90=vBZwma0|ivtmvp6X}t7w;Ki68Kq;pR3{-q`#tRiAean$6}&`y
zsP#TuYrQq9J9n{Vm(9?tdkg210?MDG@&k$51sBMK3#^8t+I-QW44L!U1g+aS#1dXn
zh33Sbf>xuc>;$O@?xi?X_|~c2n~dBcrdDuPCG~p6=*#vzx1Hq8s3x|iMmngFU%Bk1CY^NIhwSKQoaMIiQXf31I_wzemN4ew8E21>sZboDHpo!vg1r23w2YLn*3Z=;TMm_j!e^Yt-ukd<$j^
zk_uccpHMr$k6V0US!4;Po2Mfb{sEa@>03s1bQg#-KH%gFSxz=rt~rA&W?}Li2(9Ts
zpNnr|$@{97uW{Y4GY}o&de>9u-KbUxFGPbjA3D@r{?3I4J9}L3D(SMU0p`1&b>;_?
zaS1sxd2(Tk;($?!Pmfan5UR~MnsEFNEE-FZ^69Z6WvPlX%EGHoY*glzS=VJJwJ=gu
z6l?tp{ecr(g3I2fhn5Dt_|eZtE9A>7+;8i@!|6?F?j=I)o4nFOVG0I%IKM^04&Tez
zJC-l~vcMl@`??P*movQ$R(|ev29jzF`~=EJAKmIA$#(pBE;=+#JyJcY_k`$7wD6K
zADbpO>@52RGdGdjqHC%PcBNGK|Z8SY$jBb|~aLSMP7n&6Jgzo}|b
z-ByY-lY%GE3B=rARlfH3iCR*H$1gjH$8$Trv&zOQ#UA+gfSEZ#`%-;A5n&A=;fM39
zuB5A85Nk?Ai7Vuv$&HBD-Q|)(kB-g0ikba@Cgmmi4tD?q2iFyIelQT6l!^}jfX-B=Ne?)bEaFDjmDz4XpR
zj2h$t3a_xgj(6@{NZxexuuk5^HR>eoktmE3^<@EU>LiTfIR(UVWYkm|-ys56P5DNb
z(-N(s5E5!bYDPfFE~xKr!*r=&5kh3%YV-m^wr|X3DcPHLC)|0N98?q`Xmv~pdETd9
z;Y`6!u`?%CQ(DW$d#@r^yth>B1g~fv4br<(4wn2Rv%%;nYM9fe!MArzS*bV+A(#B0
zEg8X(sKiYO^CR5oX_eIThq~paxQiH*wYBQw)QMtennPFAjLvW^e@G|=%B(&A?AJUC
zsIXjo9MHe*L%n7lXbqjqLEWLOk?Zp~O=FD433;aUcU0-J$Z!WMG%pgWe_wmn2IW=^
zN;ky7q<|VaMQ_F{;ua3}^1{DpY7MZAuNNEFR|0;F`J~tsSCkrsr2WrdDlX0}t#$S~
z^fjJ}{Py|g`c$h=N@A^2jwU?r||1Zi%UMmt2@7(hB>qqa$R9qI%
z|66|e-lqkgLA)QYH$`;fA<6^ayPj45tQ+hMa9IlVBUFu5ig%$Sw&-xNGVcnMC2P^s)1pRHY_
zPfU9^`Jds;u%T8~nkNN}a30bJJiFC6mR`~>%Q*^Blcslbt!kJ&C~4FK)drH2x^tqy
zqgEq^q-rp<^(~i;M0}8oN>9FHReJ;sAW2rqS|BdO5swsY6T)I!H^JxdqfZiBv<2%=
zPlnsse1YMSqW-m$TZ?x$u)f5Nt*06pwHrCgCmyUeOLyA~Muh(@iT32~=mB!9aKg@a
z+N!1Zuh1qmmLs#XAi3+|ur-Q%APNik3P%|QNBdWjOZ|2n=t*}Cr7`JncfVY2YyfSGxu%e3eh^TPl^F>R(_n~B7j*JY?AWLf;%4eaY8=M8fMt5X!hwKLKh
zqrX#D!jUf5N2@q)+9rG1`QibQYNOnuA}0|YZ;aML&afJJzu`&}W8vX3<42`V9$;FYd*BYNHX{Ab&rhMe
z7QR_HW2$Oz?6u?&#|4hed%OWrj<>;Oq~_a0+2B2U+rLP@RkAvRy5{tM_WYSpVIF~i
zVj13Q-<*!}cYKaD5j^Cu~%o7szL`@
zCh`n4!90Vu0<%yFc$)DTg}04t%#!yJ`QrM@)`i!-fzS4AOl?a|I2&0AR0uBc8>Y)n
zn<%1wD|?nMk3~m@EfXR?bUGYrV>YOu#Uyo9d&YWU8Q*f7aN^!us7RL~H=4-gncRPp
zhF4>kc;VYG&`-zP}kU~{}
zpSEh8y7gLT!#OcHeXZj!|JYrzF!o?lMN(>Dk(7AuskpoA?#JFr&iE7_UM-KcbFV{$
zE2IRSSQXjetYNMgOB195OQhq=Ilm}wsURa4E#W-vPImY_(3rQW>05!XUHN8}WV`fD
zv}0`!fuQo`d09bz!xl^nS+JYC2bV
z-BOd)0^?j6cbMO>3S=#EpCuV}ZNG;*RY#8m-?9@J@1SP)jt!#Cijar!$P7YZD#s(J
zX=rpw*q(tck8pX=q~~`KsSD-6MJA7Fm=64;qA6!K{1ci*AsFxnDOW(tcVb>W$^SUu
zw56d3JJ6QE(qV)dq$OgmES0`3UeTQjcDfd$6~zcC9hPHvW+ClXhDDAPAZ=AJ@e%(|qToihQ>AH@seh0@8UI
zjGz*Uta(!QMyZsPWZ{Mh!Lu;xUyGtQaxW{xk4O`%1lMu=3se|W9vsup
z9*eo`ZWYqmHy+wkcbewCH%#vQ6UA|QFAB&+hf@PTmHDh>JsKyTLi#+4NZbxEe3Wmi
zv`ZgcraL!>Mlk|}cJ@P{HMJWSUW&(O;PwYwukR`kN{c%+x}az*KsZSfxg&)uchpdy
zSWPNc$vJm_E*&X)-G}{UAF%GU`%x#`(N^+RDFyYc_pB1eyF+Ht`Ze^xs@_#fvMquJ
z9BaGg``MHDOF2YRBOQxJnZ_OBH0D&RF*jHKm?;3|ss(?kWhIkMR~b+b9`TzI&PpXen9k|-juLoE5ZL1Iz2W$;jQ!tsNx5I(
z#q^`x5qvhM=O5q>kc2w#s~`7Wdo2f9Jl6ITyq!Qc-St1>!t2aZkH=c^zc2PkGjuqu
z5U(|Ha&vS4f2*f|QZk@;iG-kDjj>Du+y>q`*n*~%_41Oo$$=TnRzuLr&rZo!u8DT`
z#;;M5XbY$a*d}6F1G@+MDwyy%Mq0L}uZJl8$O+U&?phx~*W0FIDhq}k0UADxoRK7+
zbgZ(n#nC6-*DmlJC+#OgSxmd}?B+1;w8>cK18+q7-PI@?K?fmW4fYZj2$z5pfr}|JVxKv99w0Q{
z1G0C9L$U)6X@UM97Z*2Eo2D(z&&s5wzE;sqF41l;_n&7qmM@Doi9!2g2V8`_M=c=>
zzkv0U@isB9e4%o9uCaB&5qjx%SalO6&L9b<^
z%lpaM$L`F!(dFy(N_;+*6l^`aPp)P4e&4H{e-t%kpeQK{5xNrD;f&ZlyC#q{Dc0^w
zVgfgV9Sl4uh|>~b3MldZ2$t@o1MB96ZN&0Epmu$3DhSx3k|b@P)W
z#G|E>edM|Bqxg2?{{7lLK8Le#i-u1^Vy1lX_r*{LaQB(t4WKvZ?MOzP>bZatq-1!31ZyrWniOeWmD$W_%tTCQ)LfiPfJL
zEPl@Qx;8y^^R(ecSJ`{Ij+x?j^#nzsY00I@Q;gm8AUm}y
zVPCTm7b0qwOHIK~m-L5&8_=ck5s#AAE|TE+6
z2ezc4gH#OUIEey3m5`bzD`8M6QROF69u2SS*4ssf>U>gd<#qPd#2p^Tlt19z=iOZy
zR|~B9!Me6Hq@gN5rv$x)hx6zZYJm@x?3whP&^U}cdnZQVPEtJ37tb;#K>^h6+9fw*
z_p4sx*L7&C_+9JiV3u=bU|=;HOO%^8oUu9RE7|}T2BR8DEGaWl99gffq`+*s3E4|W
z#+!Le9)y>$NbIh0{!#QkmJHWYWNeHC4%_^1zvf)u{RQm
zC48Y1V%Qo#855-g8X1*iFg&Ij_+cbi&;SbbR1S->1f)b#i?bMxlu>F5#E?I&v!Yf*
ztl;)gmYR{ZUrj-F2#PVRL?qScsA@l}GtYwXc5M)zwN2_BEXH#=Ikna=CcU5w5iR8W
z3Zx`%s~v$=Vj7T9zT83Bbd(OQ%=)3{4m=&kyR&<6@uUsB=BB=8Ef2MuzN4U$HQ~4y
zKj)gY&tQi+r>FY!IPpcU?yD=Dp%fa@ny+cnxXY#1=`X2I&
z(FXgb^G(5SWGnfIoT%9*CN|}cVR*$BYvm2y2P%YZix&67`)aimBjss#R&=hj@u_a+
zb8_@|#T4{Xf`y@#mfy}N6#Dg*>s8R7!~AvQt84AEM4y;U^dB~VULF3N-}*T}uNvGd
zl&4-1HOXQzZvqRN#~Anc-YQdvJ(^BFQb)q;dT7UPzk!@>yWaj37ll79h7#~`-1S`5
z3ayF9ok3iw*EKQu@j98^i20Au`){376b^9d0terGufF{&N2c&8^g7$Fz7MWXE!i(z
zfH=#ifTwN!v);$^qWf+f=6pYY`awu72
zGHWGczo<^u!%K-g4X#`)??innyT2dEwceh!_jVh6O&LZPA?fG%_&Syvp0>cHo4Wp+VCn@#IKj91Y>F`xoZ&A<@)(+dj@
zBiu}xwGy3ne1f~baZv0QSK{-7ky8YFh3C(@4sUIQ!fwO-DPs
zAvfM1y!MLYP#uXt%^55r@cqf~wwvH}`^>1XDbG6LXJNoW(uY4MnAAWuHSjn_B3)kR)MH
zwun}C$OYy8RSoqR=c^1f>9W`sIdurOkcV$oemjiUpvETPUsjSWB4!7>_3*Bx1tfg<
z3B#ewRv5^b(G=*=TN0CD@wjrZ@6XTp46k<>wr3Nuy$BNmmxMIw2eH)sm}kR1qbr7r
zWTDPT_G6q4oOn#i$)f)3{%0lmtS}ft#Bl5;lKfnT00)vjj;mo*E&R&_ed|F^B0g}w
z!5KJ;B)kK&)IKln97?!ve3J>y5MV@n_A`kxgGPuT2#z8w-#VJdXe3K2q*m(P3RL%_f#w=^mX_%Oy(AMc(SZ0GmImiQ
zeHb!Pa->`Z%>-t^%_Gs(293Bpn7u(RMvs(zF+kX(Fvb2Wq)$ai@p}?jqMaf|kewnz
z&|=!jVPvlJf*p8;C>8sp^~{NBxwTcT_Y3=`N|IQ&>`vTwtZUzn6QzJLCT4?CvjF>D4gqO@beKd~q=(&!J#-m_1kz=h9+x+;NLyx6S@`FKb
zlb$&XT*0Y&J{`6N53vk0ZWN>bA;HD(;3kuiUPthx$FE*TaiBUoRh^`XAbOe!(Qwxm
z<)r<6pG@$_x3D8$Bhx)fr+8dz6(s(%kzBSM-2M}Nuvb$3fD|Ik#_td+_s~t!@S}@>@-(s@a((FA{Xi@A9D3EVdV++
z?%1N{)wNo-H_~!AgmZ7#N7ntBKS;g3nnKekr58n@)8{jGl8WNALz=zlnp1u;v3p+V
z*7X^8(Yd2FkweXKwaT2o5z|9dovzZe$BRPxmVd1w%DP>a%lhudT}kpEt77Hfj@!IQFn~}eTYuOE6jvyKLimTCT>Mte|BoyJ?qDNTmUzr8t<20y
zmU}vKPWG~|On6jcV|O}BtTeEB#o&T%BV0%mAU+#!CQV%RPiI$USzkMH+-$d7*>;dH)nV~
zt1-~DCXts;Sd=@%%8|+NkMydwX#sj=lJJ=HiH{&)M4YH+J9V-Pv?W&s6d|
z(XqCtgyaN2++^Gk*%i1DNjAHlt`KCGwN(q7px7pSI%5(di{bN}TmSeb03EZ^|Loy0
z;K(QEAb0VhrMu5Y>=N^NO;xxTP*3VV12?fN0rW)at)mmg$!w*|SJVIywpF`PpjhGW8!Hx6RbE*R^R`N~&naI;qZ
z?X#5)TS2;Q{Q!qNubgjRFYs(jZ%Y%9b*-N%^y`q!*@?%g|M33RkbsGusgv}ceQ{Lv
zu@g-|lj;3%{5MxE7x!c9RUZDKYkO>p~+d#OQZ&DtvNN|-$M>%_Fv6yR8+W)QY!sBZm79XS9@32fFXnP%ql!Z;Qda=GdgfZ%
z?n7NUGTk{?M$QEcTU^|HTmgGFwAFPVrT_B0`0@3*h8ljBhQrC+NI`o0-rqF|sbY_C
z?l4Kdp^rR)aNo6&bjuC|uwal&?)WO1zqg-CJd->TkXVq9(6H(B4Gn?5r24w<$)j79
zW|u$KE-neTj^S)P`h~#ds)R{V5ky5Mmw
z@27Lq@L=zd;$dXjpQj_GJ?jN}X$RtgI?p%Xd2+C%eM%32i3@c{wh_TfU9k>B4oZv*
zk>|IEjEeDuMh@vo<*SQo)EQ>4`gqOR_-iS%(BenUV6)&FFl%Hc(jve{WxkSIYwBq9
zud!O>*w|(_(g`8rSH+CR@?w26#3YTPn{sCpGm1EWwY|H(CAI>#Ge>fG1CXhcW|4wv+|GBIZ2
z^~!jk7UFro`Cn@k#5I(gBHM59%j&>7FfHxXKRb1_HLtMZ&Jmf?Ii3!>sPo>Mvkt=i
ziw=}LV!!+(49de!?dQ0T6xXU+J)wH^6Pd{{#fM(>&wq!U*)zatO)9`oN+?t@@LrHB
zUH(J~Zs>0(#-|7~XfMC0D+)Og<+#DBH`llec6z9f{W9h0S$`>rbH&^9EsTbzIdGjm
z)bE|X03XWrk1*!h`x4+lmBl#Dfwe7Qju};(AH65=Yt)wY7F=JgU~j?hRHAkl{8=^*pfOiak#~CvUj={6~pxh{_QJTwJ`khtGjG
z-8K&XSK3incsx(t6~#3*H37(9_mIL&-kZicTKxq5Bd1YmC)_5D
z?~l%Pb}-_Bb>i)^^0-Y~PE=Yv3A~Z%#Zt7>1}C1Co}={k$)D_!ndl)>uVYBX7p|M}
zbn5KaZ80P7tc@d@jm&czhtzMr1P5HyCy_$)%R^aVNu^9O>T|M{WWuB>y+rGi3WJea
zM>EMQ#Rp%dW0SUD>5*=1iloy@Z~Z1D*znCGNRinntS5Q{p$wC77+h-|n)oBif9>~=
zqEs0=it`Pv;`0FhVeqe}O6_E1#c%T{F(tJ^<6}WelWaw2c%_309M-#N^#W0(n@EI;
z^zs!PvfNpuwH?p+{4Hx{L5*^wg6aHE6vAG&6eoKn`Ma}sss-cBez&jPo0*iOc)Lp4
zpkcd*)o+}g$++?+)irdsT^*ccffSw*F8
zt4H8t{9~OT$L2hY-)Cdcq4Pwy=A6H#s~>Vi1>Tg`!xHF<9<5f2g~Yp7q%eHFw^i|W
z$_iqbL(Njg4h11)7I?-pj@jodijVuITSnIAFq-SDDBfvst7UchCWQFPIs>%i4wreA
zj2?lC7L)#A3on72TJ%RD4??eYicLi8M^&au0n?HDv
zBv`NPXbem%4XN>5dy_s!D)KJ`KelZ*g@mS#pH+-=DZf9jmnO)nVGb&N!V|k^5CM+pWtj4N0CxR-j%K8P3ml}Y32MZovAEAogJEL1%J{Z
zVX-RRZrSP(H<&`FNLS^Y_ZEJoWTX0!%q&})&i^P1wdw+g6~Eenj9pr&3aJcY)>T+h
zC6VeLtC8{}0^=^QFmeCuTuP26jSfc@UGyVqe5rt3guwZ2?1<$!cvqAJ%Fb!DB8|a)
z#${!Ki)*zHnYE9Fyy+DM5pFMoUlcuorv^FEhG?-?2BSbv-
zAqB+%rJ=9k8dzePIH`At_fOS5ewQhkx`>Afe_#HN;XAcJzDJ)R`L|}<$J;_$ndHRZ
z7NcLaG4!zHEPhlCt7|;y@%MUvhmEc
z8JSf(9x|Nv5jZib&yxYZcxD-P1^MDa>aUs!m4E#QmNVeVT(JJaG!1IlE7$N=S*2XU5tCdpX$vU
zLk5or5ip1VnSv|+kCzk61Sr(Il2hl{u3Rj^&xv5&bPheen86EqX4bRX@0y@y#5|zd
zbKpzv&%72lgB}BYOhU{_^>5-bndA%j=JB_lGc8i{`1Sb3S@(ge_=S1H8li3T_E?6n
zCe+k-cGWAo7G%OX9hGtP8r{K1>dAj)it2J6VyemRG-Yr1uky4<4BwA9Xc!%P1-d
zkmA@t-IVU_OXbM<3%v24;z0tdhA?P;P+rh2`?Hmt59~$k6}ewQfT@W=fdPE(i#WWd
zn%sPDIC8yy3EO>O--G;pYn8htXBy7cU$5a
ziCo=&`CP$wKRR>6+*%pjGIeC93O|yph#4514c$fv%Hjv0%7&3s-DzmA>j=(_KKnGZ
z2|~>}mqZG-XpbZax0eIrN&WtaR*S
zE+R0fN{2K;4zb?~870Jf(m!7&
zDLo97QEZfndZ$i3FBS=({R$Jh8r-lQ%|3QKPY-&e>Ap^{Ih%d=)7|0zt46wf$#4qH
z#Q`1Z8}83T<3^h}&JvpMdi&H`%=$80cLkV*sMJO&9I1+-AU$n>E@V@RWvgs-{?l=f
zM}w(RTzip1m4pCRq_@(*%0EUZW?0%OF&AXVf(!3T3H%|i6^UWHK2IF*O!e>HlZMP?
z`j`^m*`HAM-=~q54kc4fHsWOo7EkHG53i}TBR$%U@>*5m9rAbRkg4`AzgX(a)+|^o
z!?^QeGp0&YWRM9{K(tYoCbmDj0)|n8x+dasi5{EK*+d^O`{PvPUj0T0yL3EKjb0+u9A`Qs^6dtjeJEal70vxQOkh^f
zQ_vy0xZm!lY8c5Fu^jN{qge?5les-XGk7$9I3Z%i@+^Mr)d&>E#g9!9hcqMT&{a7)
zQrG?~@b&Y{-*#yxy25#a6lGgRSvvm-gjbU>An1ctJJU2vesT;>3RIQ&s=QiOOPE=Z
zSj=oyl0K`17NT4Wx?Gx~OK!8au}Oc~8IqBIKAp9K@J}V9L*(z6e3?d2LwMs^U8D|`
zi=pflE>%TCmbPM=NTf98GI&jOD7$-6AHiP=u_@7QRXNbktEjCF&>*xWPUdV9e-{BD#8HGBoI}b4(9+x^AVgPn(qP0N7(yD;ih>FVQ#TiRQElSmUpiZ%r
z$u3Q2YZi<7^q%ZnkN!NCI!;0V;}@1DfS>VU(7kNy`@=2Tcc@Ur;zes^hZSQmvI6Zb
zr8QF_<7}HO>C&!pCcNrwzQY{d)F~bxEvu)ky|hL8*!gFb3Wdp
z?{)&qBrfJ6wtDy;apNFC90_VfiRssrnj)JdFT;!y-W6<8s6VENVdH*PN)+LnZ#N@H
zqCj~Sg`+Ti?2_-{m*Ukf0t$`=r3A;e5psza#KgMgaZjL^u+`~+&<&vI0R$3f`O2n;Qtf3}apHLsN;P>*#iEpHKRpN%zdAQ_9>B|~l@Ev9T
zrUdJoFNuy4O(^n(Z;ZS16H7;w08g@F6o(6`
zL?yn2xn_Iz86O6{brfnj))mu~|y;i#<>rR|kj+%iJUEx?fRCGTxBQaq195_Gu9JiJYKpP^vy1HIC
zyN&^%&RLEPaL%Tj`t@A<<*KXwZ=!S$y`&_)Pz<7lgcuPo5fLx1&WE$HhvVn_=e1mS
z8ejwX00$IPiH(&grN4wskO{7q0fOu0J{3L{^aKEbeQ?(EL;0z8+zYPWlN=diwv1e{f;|`R
z_lg`~9BWnjHWaEP$?wI*b97rHUrGC-^O{DQo!|NRw2!6c9LoNH&i?dQu%+26bHcqPUO`=^8AaNYU}8z2
zf4bcxDJ^c$HcmaBCzw
zNOzP9!{$>gN)t!6eT~zOX!kj$hYib0Ng5vts`>Pq-7L-=-X77d#CRx*S!ciW6xnUJ%}CL%Ej3dQr`sX}8ybF41jz5#Dx{T?AS9#_Ck
z0iQr1p~ugz*%W_BzkiXMG;ZrNdlc4TtYRZrqL3JXG)<>yvou{y`jqqDweTW_OzuT;
zrOp02jM|RqK)w%<5<{Q^T(B{!%8-&}=
zzi}%xIzg=y1?jGrz1rAJOx12!)FP)#JBG>3ua!Y+^_!n{evtQ=P}1N;JEGHYOLLmj
zF^$|Y1yp~?!6%I?Q$7zqJY0#5#}H`ojgQeKH``i8K-~W-U4n~{(q0wG`CZkbuXy-)Luf^Lh=#y(#XxIQep~^RNE<&aj04zO!7z5c7+mZUBSQy^B!x{H
zIU$ik9<>zwA}Xa-AB-&qdZ`uLz51C6dK489?9B(A6b5#i>h=iI
zfG72kj)uJI$*)}FCSqnR!6K)(@4ddghKq_KG?U0e0HrS|gv4V_3e0C#az-4Kmz{dJ
zZ;i>_G&i~DPeZ8{7XyDLdCxsvihMi0#AlPLN|7}lgM2Ty{rooF+R;#yc
zGM!dc((%K&D$P2q+p*14Cq|HDo7*?}rC{$*qWGD!pm<2nunN&BcMDFA8CTw1t42aXtyc(DrtDdkkzF=GRUhk1v~TAjq2C3EIyS%K0hSixS9C|gawpML1M7V!S?W^fu#8hcF(M*-^
z{J~mSK1}s{S642P7s(t`l3JRKPSAD^PIJdNDf!VqQl6P6@ulcOpeo^&4$^&SIgD`Mf&T6f&uWHGK>9NhD$K)
zepdF@_p#Ujh^?=LhNb7q^K+@o=~PD4YWpperP=0booQDetVpWqT^CE3&tVop$5Bv|
zi4;*l(gL}%qxP#|=g>Z|WA@!EhR)*#ZNFP^TsI%3B+oy+voGsD{KsPdKor-9uwe|c
zw7+@S`BS+<-s}xQsyF4t>RrHO1Ljr5#&zKF^@|V!Sa*-fxBo*n$c!4HfEVism|`cK
zzW{K3@a4t+e8&D?eFY>R2NGD}dS5*+FLZnT2WsDt`gJ}f7PMQ~+4FP<`_E9kyuAF4
zZ_WfrEy1?{4zkMNb2+d3GWPzby1W4bo&lJ*FDO0!@(e4+)w{*k#&E?45t7)y_HV|_
z-q1_eR-qNsxt!p~?nV}w&tJ2I+#vl#TH%1iIx)Y=q$V^5(_$>u*2cu0W`&13-8y97
z4HIIPtCSGL{pl~FlAsg_6}Bj99bT?wW3NY!SBbaSI*?7~#rC-uo?$D)uW~Z#NYQ(L
z&`E4m#n>}V&71n+2y?*_ok*HHLGLF5*CmLDff$M6TFpq-qbM4KUR+V3W`SXXYqw$F
zk4yB)hjMzCCn_g#ROHL&2<4;i1B7(ep*P`I0kVTX0p7iKSuKTbu#2|7JXKqA5_u9&@%3vpm|YzR}(c!HOzF72fJKL5)0lXV%G54~`(^y&=~cCe$?e9YDj5&}QC#zdq{O08fvRD#DPLRB>E<7LbxtxBUV4BaH5
zA{YgHu;aN#1AiV@Ub^yG@HtBW(8%ye>t^u_k$R2DuRDLJpr^>nt9OmTo?-FZeI7zc5tDn
ztpnBZDYtbBynFxw<4b8saOnOwm<5UAyaNE{m!h^vaUf^UM9sTu1Ge5tW
z{w@(~;K@)QRLZ2N@TDtOnyH@0{J%iV>*aK&l9&gQVZbgTQ%W6shgVFk$u}fwVu~T?
zF5+rms7hA-yBZbIF!lHh!9u0UX36$fdxIpd(u(RCc$J>K{jxkh7p$ia)lW{66q{?}
zgiu#`lVkR+%?iVKuC=iy=wK^URS~@3D^+gXEhtxo0^=H=x+(X#)JNux-oYQ<)#}N8
z!w<|E1=GI63dt=mowbTpC!|r=03peMQmQIJJTqvsocF(T`t`xB@*^tIGSsTE4-XQQ
z|7sBBOtowsdLh!*RH4^H=IVVueYsVn*q8=`Uwd2i*&Gi~C>x8Wbaf6{e;{~#PwRUx
z1%_o*DDIUp*%kiqa!Jl%+5AkGR6bhXje}32E`qZdeVXH8N2+cR{tzmy?{LQzgLTHZ*fD<8&$li@<8jYGZr
z@Q1*3?PG^#`gz&<0jz*6IDF|7m`etodG2S}cUxf^1C;m`&CWFD09*hz{t=?#O$dV@Lj)cOt
zW8w)YX$ak!CLYx&P|_WuJ>)ckB{7QkKOM0R`j3K_nORt*X6FQa+~fZ?uH&RDw_1Q1
zr2Y8J#p3Bz8{g^rDO6SyjM_4$GT?kCaS+#IiBGBlEXY_d$Fa7lfFl#@=e(Dt;ZKvU
zFUy}gSynrJ&w<=PJZ1Xic-N)x{Y+b?kCj25FIh_epFoi#`1JSP(_bJ{3CIB?Mfl?2
z5F>V1B(`b#5EIn7SlTEy8Ly)_
zxn)GZJ~6e-Xno1VR#}t6)QAGRVS;~+C9$!*hE{T3<>(!AVRv#<%W5qhwQs~Gg{zGt
zlJjD2w$Zm{!`=Lqy1h)&2Ja{$=&=G6C<4YQLn+6&>Z>9Yu}ABLosvGj{+XVd0UCL8
zXP>Th2#OA%<%S#VD_k>W`gZa!E^bUxa~nbCsPJ4e>g;r1Vfe^xHJC9y8(=I*%Q;t9HbXM2O|-;Z&jZ^
z1VM-L|0cbi%3mbFU;IM>>?55_d~W4BPtF`3{?4}m8F&3D0#WeaQy6&crtwoG6$##J
zl!$;DkEulrLwM;*FVyv`OIVfG+S(>6Yq};%P)Tyujig|H1Z9h|;UkhrFRm~7R5L{&
z%bTeV{OPcRMqEfLsLpoLKY=n-9GQUAoo9-v(OQt52%VcRJOFaXq@Ps~S}reeO?MPN
zwW&``HCkqi<$26Na%S}9SA+@C$G^=w6l6
z^L6-+@DK@2;>qfR!cTEm<94}DC0fyZb+!-n--c16Qh-RnuDx1*K+%e#A4q)O&n)fG+AA80B|zgjwJa;V$iGdez=L#H)~ERl9F7a6fPn%9PFD3?
zmQL=uxhpIGoL$^JT!E7nlaf64PLXKMjc+Qzpkz+$;>uuvDd8FE6o80P?9fCuxF+=Y
z?{ErbZ66oqJIn8pW?U=Xhx*{ug;FskeW)9q617&egZOR5u;qVIAEC_@PYOJN>^=;(
zTY#~A+1SuKHEGt}OdB^L)oG%A1AM!Z1x>}c
z;h~|@ws&?C>`eOm6Z+X{|5P;NcZoD}H4X+2apBUYL&U`APUB$Sh!f$MA2u6`6x!>w
zlZcGb$-YhABpnFJZ_HTs3+JZz53WR1YtX2#d>vau4ErARk|F&trLtFyvVd
zz}S~llIBX$b%rY`xWL)8Ed4d2^IB^ZoBL$G(9ppFiUR1hNq#h9j32w!n0;66e1_33
z73EJKA!8}I+SFu_c(k{|+9UEoh^V{g-ig~U5jxR)ZKDpaLmD~%rMeEKbD_f_-VY=
zvHs|6Gq(l+6klC}?tv{{ADoHrPAc%xQvJ!QqQ1-2tb^Cx*CO2)CG5E`Hz`YCuuSfc
zh=75pBj6_=!5=ELE&zC^RwF)C#)}EA66HTXK!LxwCm)hYsG^Qdqa&yh{#E=-=5Hh#
zHVci3AxY;vQB3%ZrTjVvJ5DpbcIZRrYctZ+VyaJ(A9eI2m%`O8BI5}hCqGGjoEar>
zc|}ShD)rmaH#65TS=fdIJ~~ko@0Jz&x0>8h<(YjQ`jPEL^8gp8bN=FV-KLLpKjq~rRT4ex=-sl6#n7+NoL2c
z5wo$Gc--M{WazzX2ibmq9-i??Eu5ycQ>zYff4ntwW-_ypXDd|(jrpxXtP)2?(AXieY^zR>;KaT|ds5Fv~4
zg|&T=+qam~DiArBO5$kJbrl;&ySM{DKMe&*I`k*3wf4r;7;Oy&{R7H0oWH!;_H!5|
zNzkUEWgbaXqyoWf;@Te4i2K1oI&z^+ZXV}ZC$YC5?$+;3{4M~8I9(T;5c?HwN-(yO4=;n(TQGx-)4zH(ka
z^x@(H#2!RTSK#c%+5UN{*h%xK9+*;fIUYHge-+r!UB9yRn_eAC8lWosiTSOAtZxfjqtR8#l5sm0%c=_9g>tgNSz5`^O6fG(3~%siNpM;Q<9h%V~l};@u8)oKlLrS<$0eB
zVhG+B=8|bp=d|;BefZzx{#+tQsiATYXszi@Ao9K3rh=k5GKRXwRv?mg@awCuOMy}dCq5lrh(vnMNr9R+N}14_PQMVCW&#;_|5UgzYyzxB@7(n)j}DRT+-eXO?i2}AIFQUM
zIrXli+(Wm)1_4J%Tct<*`uhxZCVzxGVh;Z00l0*Eu>zc=&JJQlmWEu>5}RyL^E`K9
zJB3_uv(hZW-4|hNC#mD85wFw8%90dpY&s%KMmt09CAIfQrl4QSei
zntJ7In;9?$24Nzn!u$K7x9RI#mwUyHLv*-Gvt6c4G}@(^*VsGC()p-cr8_y)Uam8=
za1o26O&I!zrr*(4J})k$y;pvd^QP=7gKo5DQpWeOo6YnF!SoBCcc+;4rQ`ON(wRe*
zAy^9jx}EG(97olqTXr;Mf$M#2^;MSj{op0iF%m4GA+KgWlL=IB|D5KG-GPPrQ9CV;
zMY?Is{P46sYVZ4vvgKB&G)6x@I2ItX)hGP09w
z^f%&JUl+9Q7qDZ8q}ZDA6`O{+m|iFxu+j^8=)
z?Ac>`G!!SViLquji`LxlKUzj$sH|#ko)dJRg!yGY`hkJ;B~$-xG`pVbuH)Qhq_W`k
zyXtkowY$dqv?2Bo;eQ%q3RtYjD|UbFjHM^U>3s%75qOL96+pAYy_ynx78Vi`@;}dB
ze{c;wI{CJFa&porbMxy28uSEx8LEwgKDsYEJD=T58=t%VGT)Qlx4u9(7dfs}z5d-#>@6glPO+}rs
z(l&+GZ!uozj~8nh1(!+C?@4?W9IYQ*bJ}ogredwtjcwVj@SzE4~8?
zDn319KIhE{<}DS}W-R@ZRQC=Tjir79D~vPy<1tNtUwcT(AFAI$VLbdZvf}O0@^?r{
zTHqhw<~}d9wV-_k65SYi&Tj$lmIQ&kof>wu#%EVbUk9bnk4|IR{Aep3S&vd75P-_%
z!SdwEwNUc(j!naLXH@DplW{-#hWt*8)M4sh4rAn=bwF5pItenD|e_=qFIl~tPad^-b*LTAv!J{pE33sl?eS_ThYL~kr==C_+Wq}}D~yp^SX%}bt`!4VBUF1PoOU1lQ~$Px
z{T-=g`)q9oFNCA;Q&W-VrVf>k;4WO+;H=(Km|Sxl3S?aYCvbg3V{;9pr(JNy^Zg}O
zjT02NL;@M}$C7t&R-%1H?mnwjl_~hL+GCb6R}ZAhr5)(W1<9npqVl}QYtzi5ecn5&NA!(u3
z;%rEjca^OLR)8Z}84ll-SF6u0S2n;k?p=<|rYs3FOrgq6;+s{8>#4aJlGLOtseVo~
zQ^ref{*i$4TVsv^U7}$k`26(-tWbJyec4XquMR2+gt6r(gJONVpe4t!-in^Z`6~TN
zOUvS)B`dC6v(X%BCJ;}_IxLwHASp7(cY(%%(%(@Gm;xI08j120oD1(54*|9iJ5LX0
znU)mGHu@8%(;$jf`_A=a`SMWBkms>1yq%aEi4Eb52QC((<=pbXyBKozKJ
z)5ms4Z7qo}BE?8wLh0s4b>8x+UwUSXH;uPV!!Z%VirgLG_DNu>
z>v%EaNuDz0Up=7)TQ+&Mv-`e~JKJB&Souk|-=`so!|+TfI(J#DoM@n63Hzn|)Y$Ws
zS?1bkNMje&mDVQ&7ePbNMj>AB-7?`qI2d#I?}kWd(4y2`l(wmf44`o|te9h#0hg`}vIVs|$VH$cB-KoVvn46_7@%6Gh?T?}!wh{nDx&VPjItMbA
z+l=g3ivZ$B%a!B(ehKytY-QCqJ)kE!{KDkzScLHK(bp2cPsKDj1(MyB!X&q|vof|n
z?ZA+(Bc${|nOMj~ZTR5l<=X3;+KHq7YJK`)JoezebSv1kLN8WqT_HW&ins!XUuHQ{
znGTBNJkru4aABTIp$;Tb3+T*HGwCjua3H^D)Y~dL2+UP
z1`abNgEyNEtVQc%9EfLaJGTCj?-{D9F^Dkk5SCfECZ9uBx
zbpy`>f6^U3{}*_1728sMzEA}OXh+}6+X2Tw&B-mmu;lGazdQ*5942@4=R!U1rNfE9L~rH)pkz7p@=vo@6`B;-vfBcl
zAm|MZ4Q+I~F~~%Jd~legH}O9Of*}D39o=wnvlT{Q@_}1W9hmG0fLDtri^Is+jc+xH
z%Exi=4+meo8qwp@$#`L#BL}VrI27LSYT0@QaqDA&6=t(|L6f@gbgi31b}~PjaQcW*
ziQ|*=t|XltV@rGY0Fv8M!|PH{5qivgGciAEBlkA^$d!WkE*w79ce|}#b*`W|<5dc$
z%;Qpw<3S@q!j*97a%t??Hq!oYTDmc#Ib#R(>$jaFb-9TdhK!FGI>(xOyp
z@~KqC(ntg!M?QW0H;{Gm4^7-N@+5}X!Y(G}H|
zj8mzYO(!F#8cDvCU6M3RBj-&go_!
ztAULq0t5tgmn|Y|zFxQXyLKx5&Dnn;LEe7j?Q?OadLD)P{2Hn&7J@5^UEfqbK21W4
zE31J|*YJR3(IWp3L(n&o7RE`4wz#NQBu2;+s}onZx-7B@cnC1$xT_H8$VND%{qj_x
z@>H!J<{_$LPn7SB*Nm2Vz;)kd~@lQ6Ld|Rym9ar7ZBtc$Il8H|%IUCFedR?ZF(02RhhE~QwIMra2
zUA}E;rXP|ru#r+{H&B{dS))=7x$fB}_dLCE)5NDX?~ZhoUr2JFPH|I4iIgL&Z7Pal
z7!*18u8Gjz(pKAEdcUz?75ie7o^Bcd*rb78cH;gxbS;Rx)eYkxZT^_{c0E-etcmYy
zKOJO5qhM{y0+E>M3*8Vln&A{AwLT5|-d>}#I|<2B$}Y9qr8iCp%Y?h<0)%n30!8~d
z{O|DAgZw5PRs_!bIEJP1Hi+<%CwcdORuF(aF&jOGArJO2fA0`kKiP$H-SvuX)u0#7e@_mGW(DexUJjD*C|
z$q8`Y{qh$qnsWz9C(%atj-uE>XvQ>BJibJx#6RqlD$T>^t679FxXEmd>c#LD{%}uJ
z4wcd;B&`oa(c^3KE*#vNYJr?|U4Zpk%J6MXx4qcA0=`V)cQZTn`2T;76^c
z`0}~XRXNasR);W_!#2AhR8m0&WD9wl#X5t=HP$=l%A-3znlj7CdZq`}rl752&{RZL
zBu-(hYc8%utTs-uaXg3LTNV}^Lq-~a{-s2C&`bAJ(l?Yl98dX_NOv(?Jla7W#z$$e
zJ_tH){Hg20UTzE;klw!|YH+RYwn>`c^|fpBneCxeDafa%jrx?A9scH+#at8l>Wp!1NqIQB0Sy1-RS9QN;kG{n2@2a}F#Pj?={7M{V$621#N3gC2
z8H36%Bf++~p0}xan0J(p7JST0d^;E|DsxBYM%-g$0!9iWqhLhE;5-LsmqTg}M`!2h
zVgfg}qH@`NB-PF4h+WECmL_R?(W%uJpd<51l5#S28zWG9+96XnIc^wi1
zgDq$w{BKzpa60P|cEb9IV3R9+2e{Zy9D1%C9?Afh+85P0;JLEPAZ{M8n~gopvOi+y
z->*WC^nJ+$2X4Up-EKgRaAeTUyJxEB%b-Wo^Ah%e{k&8xd2v;dOI#odi}ls>cK$&Nn>Nf@n
zOt)VY<~s0|qB>Y@roa^Q`>~#{?GYM*agA+yJE}XmN^~`^YbwHR@ig`HZ|X&ylG(T^
ze}%=I9F@Fy4M-nAoN^HR4Hq?4tuq)`pamYY7Tg6>!w+$zti54L(@7N>hYl86xmVY%f>b31W}
z$F5M$32^-`;+r(VJ^NVmj^uXz^SkO7y32<@<7vzcf!Mwb)el8gi+IR{O+Qd1D}d6E%GIShm-P
zg)w#`wrTZE`Da&7gDWj`EBE0eTAFFoQ{qY)Q~m?qzv&re!yh3sX3{L9l9XZAKF*W7
zQgXs4BX8Km1V3FGeZx(oCP)dZ3J=$Fouf2Lv7k!QDD)h${QT?aNu~LE?GumSuPo7Y
zot|L}-N5k5^rzS9e1XxdOSZez@Q)Jj92Sz8%|+Qc2hp^vKpC9&wX*eIdGo2F{TsiM
zlBmab5nNVRx2}$ms^c8I-#NTL_noxlv$0y-YETsgdM8AF6{Pc>aezPbWS&DN0!+SK
zBGzLjm_99s^iis7Nkn22fnLyK0>NcW0ojKBlx1lHH@1+S#x&7z^0C$<>0dV
zcdxd9)jGxMsCxHLp?NI#?&Cm6`VH{u_00$WDvMJrB@-DH^zHwTrgIFd^zpv_AU9c)
zZDX<~+Y?TjtjU@@lWo_jCfl}c+qRA8?)&>cZ+a8vuDPzW&t7}2&w?wNSfu-^n&Q{$
znI0&l151o7>8%1+fB~X*(D+&7WI%V!aklm5w@iy`L=B+?TmH$Ct!g1r=)~ZF71GKiXusnr)b03w;SWte#}qpD#gURRLB*0BAZe^D8}egU)o^Tx*C968dA9rA@M8)q
zuD*b(1y%3fxLb7oe3LlZ|08B(n&cfRL8l}IRgley2Bp!-B>gq4FVueIb(KFW3+3sl
zNvrYs!ycoKA(J`~e2Av%X{H(%d!p*s>9b13P8PvOqkjBOSy?+f;xiR2wzgoyZa&LY
zZg5Bz29PM<02tSl>9YP#-SeeD76|=ZIg97^j;YixOmZ^e`-XsYW?q<@ZvVvxts5N5JUk
zw*;5*Lsye>1hfz_bgz|h)9C&l?Kw|AV!5^6G(nixZ^BrDhzZN6hOmLn}JB678;cy7{eCJ=Wi6}L-)p)YJRjy1{L
z+k7^E5NZ^Wg3=HYkR1T(jy!zb1lq({M{e~-h4&t>-4r7j-f(LAFI-j7q$mBdwxGGa
z`N@{QJ8L_6Ey&Hw&F(6)&owjU{M@d_H!NV6#xv{3MVkh98Sw0u=SiEiOlNzRKSvnKh;+QzQpAsAEJ<-!NnjzuR
z!?6=FkP`!2x<3}mY%C24SZEsXtv>l!=@kUse`>xe+>esd38}^5L;Fgrpv0sl5P_-y
z8o3>1Fv|omsW)d&Y`ux*e>hexf-==TyncM!T8X7ayn5Wa4hh1k&C&(^g9&K10nGNAkUdCKciz6pj5
zpMMr+qIZnuzahiwfAW8>mSy|Yoj^l*-V2ETs-geu56g7mgaQyHSi0^}gl`vwZ-V&7
z2m!+Hr_2jrrUt#&k7>1ef~`G{Gy(6+1>@*8PolHWna7#i9r*^ub%)H3+H}TgMXJ
zJo7s>OHGfHg=K-XL#RsZHFRN*B5U|j+2XTDF?(E~qllJ`3{fI#1t<6(_Md`e`DDZ$
zBz_c(y|Go;jj2(F!YY&E`RK&xLW#*L#Gn2G%n-UuZ+9KQ6E2n@!6xL*96iCNXCbh4
zauTY^L+uv81U3tvjHDvWlVY5wQA3s>n@rNxJlL&tLgP8XNNr--yqw`<;B)GK?BO!g
zH3%c+#ibA5(P+F8ddOdk|DQt3(_~#$=au3TRWDAy7hs(pFEMN#?0KTirB^lHgEt#(PmLy?JLrYzno)U{7QiGC2WNqi7Np?`UceR;*FbL{7wq5inMM
zQ#Z@j^<7+gWPJu0if5&3pNmAG9g*St*(WmLNZ9aacbtiI(oJCz-qIk0kgGDYYl1Xu
z%ftJya)wMf@#)bC|F1BB?RicN%YU=)oR#6sLgSdwCB`w@)7neoCKT8
z27cpeQq`>9e>!L{VuYBD58=clnFf8boIp)bkHPFDz(=!$%rIPWVwMzwsmnzOpe-&d
zt7vWQ%~%tszA3#MY@Tku7-cAHYv`HyE$8Lo^STJda`NM4hT>J_G01Y3I+8dXcemmy
z37Nb!YI>G1EZzzl-jYH96F_{W-x)`9Ad=%IZU6fb2IrdX2QFA&4Br6@3!6bE6~kuR
zx>PhymFmi52JHmjaeVHVNarp6r2nu(^R{)(`!_|oSKTmqQd5%>_mp*EZ6_>6oY&4M
zENtcrIpiXbh}raC<0XceBGd-vUNSKnf!7yz_{J)==eR$5jKtDp76m&$Rm%ma6Sw_a
zRy@SAnB91mv|8=_uCN%ZSk!Y8t`if;T@*8G!eQPRGrTWCsp!}Fc%@m!p
zNZ2Zyr42uaP7Q`3Upo592lwk-i)fR_@$;NTYJMXEget@W*{6^WWj#oKxJpO&pLqi+
z_%+9&$|dCA(^MBO7>|x<3_Wf`0%C{O5b~K4Izc!NhfI@rXl;eDZ3L}N{Ud?gpt>)P
z5wk;$mRb-Ot}SRVYm}IOQBZ@vYlcL_k+MmF{Q1QHoaV*9R1rtF*MLs8Y0jEyE8?%Ux*JBtI^dE<^+onP
z#h;tI`QtEWW??uaj#Z396UYH3d2F_%N6m5P)-XyO+zhF
zca@FvJ7$Y0$3B_#&uX~7H8(n&Y6T#Zb2mlxJ;PWN(Oyf24xPKuyT-@n!k1QxuGOzc
zoSE@6EvxodOQ4g!OuMbcJz8>!GO_@^DyM7Y*&zkOJu#8_#R7$r^r2uC%TSvq<$lt_
zl5@r;eLm#2v$->(lpB~Q(ZJv^^lyqB=3DFe73Ni@V+be_M>SlBa%~aa+!nl0oiqOY
z1UVeF1#KD@7Wl!B+$E1cXccZgP36E^sJ@i`n3j`v*Tbxf4|vYb54xI3gpVo>TJ5TZ
z@~T(kGDj8JHdCa8qm4VKEkaf&|Ni$|aI~*GH#GCVgd8%@Fr+hKnom_WZkahrd!M)(
zQR{L(HbbLcf+fb!pzvWEyZTV>QtzMdURT$6V#Mm+x4-)x-PavAoPs-__SxClq|nw2
zz4hYRr#Y{eIRTdJ2gE#|_wt&xmz6a>OXgLMQS<2=VA1pTdMSImN%}VV|0l3}^LZEH
zD{xg+50<0ke!7nlctHCPfyq+*cBA{|`1*d}Y^BUzEmAAHoGJ5+d-o*0&tHLiAA$RR
zF>n5&y-2q(jx~3{zzReRg?luDDw}I+dN|r1My4>gg!@bco)+F?7w<4<;QD&QTC@gy
zKR#zu_eBJW=Y1EU6QVpn#CwB*2}$O0UE9k^M7HPR#+J|jWf+y7i@a{iM!WF<0b9;g
z1cd5cNhdkUZ$`1e)4moGTF@Nz4*}}BVgj$mHl@m@NEX6!$e-=&0=-y1+6`;i5q*PC
zhxS&`JQd^Dt{EqP>EpRV7DGIph1mgp5(ljSf5~&^-XN04RR*5;sTo}rM@+>BkrOn-
zz$*mFLML6NwqR3aqlRXU1aS&Y3G#5a*0?s)T4kP*igH_$j}=jRTLdFe)+Tn|$UwTv
z$RidaSzX}+lv!W)ExXTkpu5$>#u30(Bv)g&bvZ_(O2Z<$sPoxUPC-2yLe{3ObI_uYPPlo8sGX8g7n0^6O3%Gu^S;U#oA
znRCj9d=fJ__N=}I8Eeu5EH&3jnd8gL{F^13PoWg7*$ybRe#l=901L?_FQqI5k}enU
z-+hC;Hw&guWsb*P3TS@ZW%qhnq}Oq?2uJ}Oq`6!_6_x&=em$|U)^5YL5R~$DRE)01
zC`M!{<#$LM<9}e%eSAjab-96i3Q&GMYjY3t=|ZU*fY9Dn*RKC21pvK)epFg44wvUB
z=z4cO`-5gop^R8T>Y$qV0iYq4UI7j&iA0txhdGuTxsboT!L%e|mkP*BDFHNJA&mXX
z_E<<&TtXw>;E{)qf#W)vtMCBXz0(yC!?Yi={FP{@NQwHvJ&@M}kg
z+VpPd&Nq+=W5q$4OC)|aQR$~2*^lUvn;(xa1Qtg{$5vM5Ej2x>rvxzvq59n7U-%$JZVh0vVgqR8==
z!}_6ZS)JYELmb|YuX-NI4x$!SHh_W9r_nGFQZCx@5LjqTDe^2G3L7nBKkYHq`Eiv=
zfvkF>J=Yx1!9UhxQZ?@Mmv{F_DgB9T<`6${Y9bkH?8?<5{1$iVzTLMX{NvU&BR#s3
zdMp!Jklqh&O3@Yw#9Z<$Ehbv>UJ2w%+gT6m@|$FejOt!6Y%xXFbyR`TdbrUq)&Fz(iYT9;{KoTDO;C
z;7n~(C8)RDJh1RGNa|}gbrKDg^a*%@t0AP*j)`FbA45y)gn6GF!kNZVR`i;5)o;nt
zjw36BBEPPz&C4Abc+ps138xNJnVEfVOi|UShNo
zm#I7)*5(Cu5S|2;XK&pj9AC=UeuR637nIx2{?zt0O-4X23yISarx!oABX%NE8h}7wl}oOs+w(_9)R2}jw&F37|d{OUws!M(O#~raydxOni-e(Zvp8?
zy1&Q28lXf-l8xVzdLW4<2!IapbI>)l%%hC>}s
z8&5U+rKp!8tx4~#m6cy&(=LVg<-)UWVPh-QP0sywd5_<@EabO*Js=YQw^h0u7Qe?X
zfc;SpvLo=gc!Mr7P51YPpECU$-6kClt{g=S>zVq7-0z(@H9&dI;e>}s2ZO=N^lkqq
z`krrW&~aZLN{pG{GPXUlysdcMt1qPz!
z>$~-6m>W~m|Nrd11?$DF&ljFb8Am6YdkZT^6(~c_mJn%Ugywxudg?p
z&xd8NH18D9hj*lClGk&@+lBb+%9@7Y-u+frhy55|-`Vr%S?XirOP9cN*ME=6=y?$W
zIEs~c@0RWVSEM&}fL`=yn8aZk9yS)vd>Xr|K*z3QMasSvTgl!Sa%Ck(Sun*oMCQpZ
zLXXyodnOLkoA?)bQ(~0Ck|%n|)-GiB4$_A=pF|RxbckC?T?J_)Y*N2V@lEizoQMmZ
za6ctxou211OSo2Xm|LdhaN`$BC_=NE@9qy)vKeQAXeZ%oy0GEdi{5A!!HgQAgvw1f1~JiS
zn17K<#K^Hm9fo%OryV@M&RPMU43gPns^$Y;gx2#DuhWi+%g-z6FIBW>jWv*wR6eSZ
zAYYrSAzRM03m{42gh&y)7@1QEk{MKDO_E6(vuDtqU+)umfr;RAKp|eY*-mqNm`Q2A
zT%=!Z@Mw2PBmJWj|1Hs08y&Fo;ASe?
zhjv`y2fKPjGx)WW_=!rvl6+Lw--DjlhuVP|gdgWYI>6~C7}c5<|ISUM&f)Jy-MQ&*sRkP}H#}nFYGD-9nowD%D=32mR?WXI0)eEY17T5AJ3Fx{=
zUHuuiI>@bL6h3bJtBZO_X(3^{iln~oR5LziDMC1vjp2^$rD!Ew>JLV|8S#7)d8oAd
z5Sx%X;-0srK~KGdVvvFNEf{no*GS}S+XlXsQ%_WtX021VM4`!PYZSBQ{Au^QJ*)&}
z>FKB#fi*T7J?Hi3F4mvOa2tcwINQ#!x34_|$J=$%MRaNx|48Xl7noG+-mC?_)Kkod(Sma8Q
zUIZxo(N^k`N6nwV)54H}M(GX@&hEs2?NoP)W|!`T-YD{{fR5(wC?fI~%KFk2Gt)au
zNUcO2NOk5PW$N~qbFIdcC$~w5p0vU*<~ES5Z^i^gtSJN}84Fglk#o#>ZbAGeCA1e`
z^T@*mi`*fOsLo8?8>}>VqcjXfx687=@xUebWTKR=mGr>ux#wNG2B1df#3SSkC`o&n
zb0uO^aK&OCN@Ser*FJs6hq-OtjqO!1?P{*3lKPd2!kH9$KI&8aJ%e_$6^TTr<|Gr{
zF)ukoUj2I^dY6urfGpSwGK#jExo2;D39mIut=oKvKB%3OP%kcdl_KgQQ
z7?Cj*YNa?LrnGy!-6A89X)Pi=!TdCEuXg-;>!d9Zsr(`iLpi5L=f0tB+3##V!gCOC
z1~v0UK7n4&H`tY!UtdhDoy`dU5soQS=Sf&cY1WL!MD*U)g3pY-7^|N~sby&hs3e5O
zoq6>g>N{|`(a|pn>Hg&&t3%z%L*JJBEmcn)d6tX$;z;&HC3m4y&dRyOwM-)Bm-Y_M
zc4(?2+qHM$$Wb%yiD%~rPB|<~tvW^QD4COL1)e`zs&X<+(94)2Rk~fqE3+QLeEQg*
z7I@_9eraGRE(*p*#Ppo0bp2+jF!}1W(js2@gWr)kenE?M3{ps)osnOH(a)WKuXiNn
z3-o7zi8fF<+=4aHI2faPy*HR7n=jX;Ja|#XypMfmyi*J=u69fT()f(a_Wx(=)$Vrd
z^?F--yE93?=S`CN57P-OdMY+Yvn9Zu_fcNph3);0>JG&4)xE_DT(*Jbvi+#6ZS;SI
zbIpvYqFqmGZ!c>Cj#;Mbt)Wme|1;t)mvM%W4gYsNmDi?B21qubR}FxXgV}bY+16wI
zx@(=5cnVYlZ6$afB=E;x{Z|_FUK}LwlJ^$-J^_pR=S;BgEYRDl>SqC#TiM0Snb(`j
zUDbf$Wt2MB86=JqPM_TmCiMmGJsKqw{=g{l@;^
zzgjiM?=~3iQEiL4CeW%22-><_F>)sF-XZ0%(wEC2#`FQF0~@c6nDl3$Exa84-Lm}N
zJ3Ds$Ru%pq77lPIUG(BwpTDEK2_N3DX7#uhqlok&vUVnZ%XrnCP1c2#HvhA|;wsGl
zYx%h>(!}WA#DcVGFk?7`jQj_Bd?Gq}=-Ot#I{PDjDI}>V8g58Wtr@>86E332-r{*1
zyF^(iiHE1{G>|$UXS#0EB<+otv8{qaGt0cB?UH2VsgSj6O18yas6SR!8TLDi3fgBD
zHq_Wo7vrOo9x^qVW3NVVF<83G!b?_t89~pS7#WF{+ef1&Z_$ZhJNmcF8v1}tlAVT1
zhiYaYyONtOi>$ud%C-B@RzKkUdMpd&
z9PSHGCWZiPI6~Q~>bm%s3&7U`;&R$NUJm>6YKra4Y@o$iXng~RDD3b3PPlY;iOchz
z*qXm0)UXty7Uk~e-P#tPe<%_C@Qy?+KrN<0uFas5s@wi*qQWT0WZr6^;aty{*|swV
zPZm$Wbz~DFxe{yzckF~TQ
zekWh;j%Zt1;wzY{Yip3>B0juT-R&{Q$wz8LrxD``Qo#SGoLwthzg&isS71`+PCNkD+yJz|Hw0s(ED5vumUS$JBV;6!D
z)-1Cn7IIA%a(2lAjbjqGv&Fh9BjfHTQWh4{tJK2P!p6dZD{{o6)ZzX>-QqS~!?)H8
zv@{WNSkO^wd*Z8neH*?yK^Wh49GGLw{5E|JKf2X^)u+lO*%J6KV0TKSAQj99nVDb
z-iC+~^QLEjvRIhf7YAvoq-94(Ip*t3{&W4QzT)XzwU^oK6Z|}h-Q9w
zjdrc+{Ar0q+T{}(7GZn(OlOZ}tN~JIgAO8-q==FJ5^d%edO|KSzVIrU?Nm2;`ccsg
z!kE6Q!b~EYLaAJdV$(4*b#JmGSG3iZ=5)+q_Mww`ccmpBh5YO$blo_84xQ9C7olV-
z$lzAn38Q<1I){wQ8F81cNvr$WxYladPlKjb?g?<61rUYgnjy7Jn(-K-q^vyGd{U9x
zmbM?A90*lMe5h)0wZY}Erj#}uc0&;n^Rv$^jTYap6^r~3%jOh`&R-MEHBI>iO*JLB
zCOi`Z7L|OmwIyFMa^Yhx;sLP$2#q{gg4&>p=IPidt5=*ZY$MoygWinZ3
zAUM~rNCj^Ko?Dmv-g9@(`yF-9UQpsY<-~BQle$*KJgZ8_9X|QYk8}i^2`el=fhOFP
zd28HkhNJr9fyz}UPpdSy2dDwASN5>2I$$ElfugZpxeDu&#uHKWCyQyxaOF_*8n
zGC3B}9#|hvJFsd`xu3GC<*pKp@ulFu8JcgP=fWEmsQ|sr^RKlfc25*-*2U7`Bc9?^
zRTrMI#a*<>>|=|+89$-&%vd}^C#F!pz%G}tI_<>~e%q+k%{92R5~j64H^91-d`c0adl
zCG)I>^MiUaQ}McB0{huTO07&QhqM8l2bzkwiR`ESEzsuSnxGa0;0<0pB`N(X9zn2k
zke9w{#?0QW)_NJIec=6(sXc_TwSJ1^+7|X5>f;RZE$X$h
z7vXWgKQ9e*W#Zv!0zL<{x_i8Tm_u%|l)sINa5of(g~z0nOPx1EaY6vzGrzxexm$5w
z6O|mp^tu-Jsvc3oVCLc)FyX(#X3(l{e=ixd0Nuu|&sP{M>GW=N1=O{K4s~2#J9w2%
z^wsLsiQT;bs<;n6{o8w7$hrC%+}njD&^$%*EAU;3Uok_0GiAeZKU7Yq(Mm~6+x-_n
zu(=UH*h=AgG!ndW3#2_m-_8kNyTaqWN>0}JANTzR1|(10-d@3&S(N;D4Pz(OFjzfU
zZTF&9O(*O?c&XbUg0J_Tg#>n99e7{mukTo_iR(ud(g-vCvS%vpl=iBYeVE^3xS`;P
zgAr}F5lwnsM#yRv>q56us5p-frYD$hwC(vxUeLiTD-o3V(cdZ5PMIjB#;(0-BFo!f
zl$>aW@kGG6Hky5?7{ybb|75d|GR)wp#W$!)LvZw;I-@F{mmJY+FxWk|>VtB_)BUFt
zc-Z6d>Fqm0D(iiJEg@?BSm8hUk)%VA+LAUvqw28BAP`>zY`S%NpVpA_RcP*8(`PNG
zWJ;CQ=hi9s;Z=MPkHUNjTWg1}acQd;nM0)MTQ8hVI4`~0hTC4JupaK-E+E#PI#5gG
zguY6+CuL+&*jo5G*@NYp!c<{}s8Gq1G)pfB2T*i)nLCS2pc=|m{>?BWj-=&X@PS2}
zjv=W9DUoYB?jYQ({ZmeZVXdvk@ytjmNqdnPSZwz(nk(9jet3n{%$jYg`xIuN5$3oW
zF`Q>doCQc6OdPN!>GF=?=IQF~P7kG~^Z7{m*PCq$H{Z?5U#xbeZ`QuNL6hEesfoKn
znb=qzn=T%Tr1G|P53q(faN8d8mW&TZ#x0((E21-VtL%-3xfRS5HlO=UJK@`VKSjZ_|r=p~GD4TIDgMfxYuB;W6~_l+i-
zdI32e!nofc+-|x3YK?nvsq%oPP7_o3a^`ns3UZ_uLrwxdAILmtmHwAydgmAecqQkn
zGHTGw%1Wgpt&+5=1FTP{WDyvb+sG0^uYJEC#_dk7!g@`ox39Ng?nqO}6-pgsW^;K=
zVsS+8ogtLNABg2^*5au8Hr#7#9*;tr>RER{Tqlr9tU!jj@U%VWrtzs$Ql)lbFlY#3
zHkDWZFvc|K{Ak+07v_-E0mh6X)8>yHaa*)ZW*?vHG(FazP6eBQXpdP`k;eWywU1G7
zWG{cp`2z`ki@%YK(zB^+M|`Oe4G!JcX1LX-09qg87U_5h@)MOLs$$b))j_EBC37=qlF>}ytH%$aDEuOKeSo7>B;cJIG
zyZCDQNnr4mb1Y=#=p`7(E$J~}*~6-}=y
zvU6nO_0tzTM>b`aUi8G`tBwkJ;zd=R#9F6Dt8TfRH_9P%KQ%tmuMzG(5iNOgvc%0r
zIczJC4T3Z#Va9*&mdObKuoSoaYcyKPVqTJ+EauzmpEaHL3Vs(IksHb4Jse4tgCgjs
z$!Ck9KjLtdNwmLYk^RHyVcSYnO-1g3i?d22#}VMMXkJw9B32H
zM5qur=QWS}WBzqh5-CBYc_S~f&9ds$p@Dp2Yawhh=ZeOQ0T!@*_MgT{-O`l>%Q-FH1#Lu3t&Vx}x}WgzzK)KHuaa|SdUN?&(7GV|z+$bds`|D8
zHMa7$YSJ=~VbqkBZ}eVy#&d!uuo#VBZ_-kk_DZ*uV0g`fh8s?4=5aa~(?l2GS%P`b
zigycSe?ASgRzXxjpWg77!;iLf@(75RGH9IKu$>L-)Usz}9UaR$pD8~J#mm7(Ekao>
zEj5}NxMd%ee`EWWfuC7jboWa;8(BhHZuHg%5f6=)@T6g6Tkq+(W-Of8|P#7V~@iy!Jyx1uH8n@9n$+4}EX=
z#`ZuA54{l9Y13IN81v(wHlP*>7@?2#5bu%TWiymjhx-yBQ2hS6cmEggwf`IN9@n;E
z<+ze`5xkfEav>j@xey;0;NCCH?)$=on?T;UfiC(vid5Xm&|Txx_4_-AqrkQ~oF?XE
z(EA{F()<1Ucb@2eP?mVaX<{ijDamC>%&m5w{*2pgqd$uD{U`Czi}}Ax4oeDS0suB?
z*<}yR`>-9;bh(CI3Vke=@-z|%aCXi6diMc1(;8*NU4_Mexhmipml{~Z>sGv|C>-Vf=u!&4#9Vc2)XKtvMj2Xnaq2hCoAyz;6Vvm
zh6E{JLVr7GZ`9+qRW^p}aUc^d16__Es7Gy3R9hxvcMWkXIUYT_x5FHwnHkv4ad1wR
z{iv~BAEpq0qW_xP;!e%+Xc>gbzGk;-h41#Ts~h&E31u?IR}Vv=oNVss`{7UkQKvzi
zAl+67qCGsmj;mu*&W>s_NydU4r>B0@+x?o(-jLTV%iCF=?rs}C-_Wr>w7@y+r*Rqz
zNs{Wx-%9IG-cH;0uC1dN%08)SB@?DDFdy|Y^6%l~ce_KwGQNr6vyNKv(mh-t-7mk$
z7s?ReWOG*r1P$Oe9PHxOKa6wwWF`0PTn!I5_X4h-z%7WeyQzK=Y79Xb-Z9d1gbLzt
z9=ft&yT@lXO1YL{+`aHAQwCN}g}wDxxB9EyH%jN#Ye~e=7qS@}{UOkczLlGO$Wbb9
zQ~&ZN4p)*4OsnZvuN)=ea$nj%}vigW-s2OGtmpJKJq>CeR9(Gb3
zehqn-$727+et%|8@5a$oN098u|CQwVy2=j%W#y^}1)+$;H-;jw;)blM=}Y@$RnM?x
zH+>&*%l2?djSzKQ6dELj>!ER@bKSo@dyfsdyB|`{T6|^I6p9psc3=I!AR$$mj1k

z_K7*44C{<-JH_Op(c)By_<#1jA@&6t_xdxjQe2mIwok%C=(W3mKw3nGY96Qvg*>?-
z2NRywGE-zjd&lr-q;3*7cLJt5*7P7RigX~b9OkcX0Sl+Pi5<~;dt~3VO&o|kd*YYf
z*x6hA=i4L3_5XRV5U`>!rig=H#k;KXNW&JcxwRiV|ZoOHh
zM@)a8T5VnXg8aU_yy3tv8V`lfVD$;}K8MV&YjoMGz+BT&tp8inticg<-NToyWW}4?1(1?)+npOq4ymTZv2ovZ#t-
zYMa$-KuOszn2Y&$P0=gH9x2+e$5HtLcQxLyp37J^)zW&`_;ar6vZ=L`J7Jez)U`gK
zdS?$}8zHDegM4z>S1iUbXz0cPX&E_(OW(owp>$FT%P~EQa*0vmPKQDM)=CN8u>kp?
zcGyGrpjtMb0j*Yjq*8~W&V+G(uhInHWo4Ie_-42z*P2N+pbiCS*jN=3pYVY;n
zBX5lIZU0;peB%&uc7PUKp2t+9Tjp<58HzTK)moH{SpAZy^6Kae-@aB{YePeK@0lA+
zld-g{sRM-q6*xEIldojlFJS!A9rnNWn}R@kEG)&7QB1$q-Y83l0snADoY4B?FP;nR
z_0NR&?KM=p!AyL7EwrQ(@AC4kTu4P$Vp7rquoC_TmQrWY>b;~S>~I3U#wmSZi@m=R
zc)J2)ZWNZ|;v=p-t=^qh!+dyWKrI3Ta6c;o*bTEH>X?8vu=aiYy}Pn_=sUypjg9@~
z-8a#e-OgInn#G^CDFr06Cg$}A7!aJ7ZK2;gjn^)>_g4IY+R3*6e$-a7K_1DU)|s`I
zPt9*HXKQ9C>^*(_w-xwPF$`%Qw;BK}z91ym4FJSHWtY)*1L5H?Fs7dLMX7U(MV{z9
zve!d`m&?e|FJP@bHXv-;-MO9U6Pq7-Gkz(M_b^x6Z`HX69-`sk<>0^!xCNki9_D$P
ztKsWR_RV&L{miM!ojF~H)Y+w0gOP}Ptw97~U#M+6P|Qsr?s45U+Tcy`Ziv{+p(W7661+lEGiXOqTv40ljO@hJJ82JotFQ@1Zo
z$b0ruIdvMQ?Bn?(LU+YZ2OdTC5Lb!d6!nf@QVp%LcoR}Vq8CrbD8_JR%gyCKNN
zUY{fK{oYEthFr{MVh)E#hdzmhG26$MSf3;IufD{3>eQ$LYivx8sEt8msQc)v%q{W!6f2C@+AvV3CTc{RK?{#
zqDLB6arZkr+T?8t^i?EN0I{~JU7e<*Ki_}GuxhpZ)6qd}{)+4~qbLS-S7klM!^Px?
z<#R)=s*1_!_WDhA4s#qtniC*M;u{dlujERFQMIbR{e1j|HBU!>_nLN?ahQ1XPt@VZ
zTlwXI*8*o2Zf?K5yUb>7qa}Fh;C-jHBN15ZRPFWa2gu|SOvPznqLSIl;5nKWm*sSu
zNFWcVYHUB8wT%{0nwZs)y03Lz8PRw2e-G
zyr{d97P#W~yf9hodl9&0zm*NYjphBqYt>>F487`wC2)~-W@x7m@s}tZp@AkCq2q9T
zEedt1;{ZHRw6mrLa+>Yd@+`*(Su4eX~(}#2~H^&OKh;YC#9pQY9MA^^>z^OFAx^
zwp>W+kB;!Y@`cDXHs5B&iV>b9g7C}rK&0tM*Xg3qsJA+Di;RmPBkkn83fHAOkHZ`8
zt*x!&rMd`@r(0%jZYg9PBLKp-@1V~xT*lH=E|&T2dr9z@B0(I*5mYs;l+G8!Ta7wP
zJxQz6D>G8`3Q1>^ELuypAzHenqS6+%%?b2h`0_Bx_&v+!GFe#?lM)4ns-vPJv6*dE8w
zlT=h0rW~mrVt_<*qzD8NZ{gpuc|pIHtk1{TgnAARfl}||^!)Lt>}rs(mDOoK
zHjr>M6*oAMGq?bPz%7r0^Y=c*t&H6I13wM(gq?PN^+qknOmH$??czB$xK*oxotwxuSVgs<
z0C{O69x�H_JQUy^CFMq`?W@c8=t0qu@H#BB6ny8;J)yvElmr9}<
z11AE1#=N{w^9>fHJXvUluJa*-+iZ@Cpl!ttN6!&FZq?A()NBaTwX1wJA#***HP)+o
zJnBa!Ql7gh>@w?G1mM=sP8=gshLBGp!PIqm20~bCVg0W;r+kchB+h@dgI6QrJI;fNs05;^4SwUe~u!C$2uC(pQ;kr`N|
z7+dDB#u=^QYy%+{obY8%|DO6}FgY9h@+
z)p|Fc8Hl=SQ(rMGG|SgoSbM5KeoZNu`6~u($SJ6Rn~7!-ucm5NxAZ$LKxW|#yQkq?
zKknE~z6rz=mKMeV*LHQ(XkodJ@o`bs>zmC;&@VXF!>nn)28Sr|(@~UzZCJ%$it#M_D+7LjRl4z=tjR
zP{8{I2)Dzdqpe1F#bIJT?-naOfNfuxHH#1Ig6}~oP11Rc_4N)P_XBC&#frf|x+bqZj{qAsbK`q$wC?3!-FFdw)$MEtE&aYbTr}xy>(GWeTvm!2CoI=i
ziSfb7XwLL$_U80gzx#Khm6
zj(gV<*71*7bq*^@n^U%^#-EapcdSo7OQ3G)XqY|~7|BD;PLD>Tqu_rl=2etyYIxEJTAhWsM0!;#TT=2(=l+E))`3P_gjWbnSn~!hk$v&z))FPDO5EBf#62z
z{}4pj0`|ISTNx7fI@R$!^f+73eL=hTJnJuwmtn0j@kLsFBk(-YeP$E5TlZMDyW;0P
zh@uiL!Mr?evf2|yTJDNyIl})kUG$+lXm?pZ=1iUu7}C1ygu39p?PHvx5nmC=cA{vS
zv{dpht>1HQt@Ec}8O>{~nY7Z4ei0d$bBhzIW2${yM}hj-dZuCR
zP@9_->{!R0jVW>aMal{XYniDKyOfGoahh7~(@{H^s>i&lYgsDh#>r5%gXKFsvJy#-
zb-#cV_GR*xx{j8g5%($?sAvC{#Px>Eb?006qM3*Ztiz{HXqTIbbXJzHhKMI(^sv6@
z8q=nW?;C{Iei5TG?Vd+$}pX1vyOn
zu%w|0cTV5z=inu~Qmbid$qdaQ%xoGIa4J6vgGRf1*$<1*2zi_Jc2C6PwZ8dzs<^>h
z{Ktyw+0Er;7Ft>a-)k5P$7UiKi_`gUX@;=*_0+xjVZ>
zf?Es5Uu>wZ_T1ed%boQ@2QE;cj(&^>1N{K2oG!r$x}HgBWwKLR8rg>(
zrY*gKCBbJ;FvRxWvSw@E!o`C-D2RV)S4a(?Q2JCUbwB-RbgqPEpWmNh2e}$o!Gj6M
zv)P0wVM<_lB}+g#paK$KGD!#5-(t4IFyKYY%6hNAsSw7!Klx1%{!LauuP_&bzsc|f
zaMW8g&S2ErXYfj#yowWv;P7$qoe9Ad>0Ub3cIzmu7+kWM`Lh1IJPRMSxlPh2B{@kc
zCY7%0I?Q4#&M}kMF4SukGU_6oT8r<`8~^HI{I7pTTCREIgmt3EW}o*A0Rg?~Xi%*;
zRmK?Blwo2PIUQp%G8i7^A9+wb8R!E+Dl=SqUGhc5hn;^w`U~AD3^P`YCY0N@(za_*ws7XMmdty#chYw{l^U71(Aps4?KA-X
zv*$KV^YPo7xS$k}tNT0@mIyzDk7FsCb|i*#TM|}@q1H;t-e`+5Uqd>SSGS;69aGNT
zeeIP=BwtI)TJFNM3qfhh#HinwinJ#2ekf>lPdHHpALj
zuHRjQC2S%UB*DYt@DYDEJ{Ep<1ZvQ=K+I>|HDXd4(M$YnwtF=c}
zFKI`Y`d-@y)Y)G4Emg-^5qNXH_wE1@N-IRa6b^~+dc4#|%oV89bOCnQ_qi-JkASH6
zK84*k-~pMrwA;#Y{e|OgcMTY^9>191(O5V5H~+U}Z_)eO?~MXeQ9FO`h==~uI|&Q8
zF5`Cq&affiR6GJ4U~7$B_yNiK-7u{gzqYga7n)!`PBDFMcDA-9w4$QIl5xw9nU8Pe
zJrV^RawGUlx%i(J#&oJG0fJV;Ym*xE|AgvtY9NPzwoIi(sd^863mKL|0W#R)aTTw6x-1YQon
zm7K56J^ePv^A)s}SmAhFozv6kz_Hum%_fh7I97=bbi&3hJ+Z+U0=Jnx
zhEL+SI~`Bz{JVPe_pbTH1B1S8ayvb(bLyDalEpgbYX>>kpE&$4B@q?Xh(U5Itk|h0
z!L31v8-dH;$H}P^6wi+swdaRh=1WYRb6qGP)4PF`2UD@onRRJcqj|YyUW3`j5j7{;9GS?=;5BtF^nhf7f
zdCmA!&NDeg0CWZ?XJ*4jx$!SX(-Kn-18#cBCXKunRZ??{&FmmTm|n5Gc^(&dJ#`}&
zqxhTNN@6~@kBZaAdIxlQW3B&>skaP@vjMhslR$8H3GVK0!8H&hxP;&y9D)UcySoJ^
zFt|GecXxLfU~ma`-udp{_naR?6%+%^R8if%`dO=o!)C-bcZ0CY%PUl;ce{WAqT)$w8+XN}hPst~6SLCT%
z^gT4=DraVz*&C-(M^DNteQ-al)2q`()g8l=|B%5Wy$?_5kv&RDW&=8M5dx8Lo;@|^%8Bl;4Im8-z#A2kLg72*Am)k`rsckuK(1?
z)#*;najI7>XAXOhT(x4`$l&1N2}wwP>w@y8q*RsSsoM_1E3*RU
z7BHinLp^Z)-w~$9G)dg4rKA~?%qhy(Wz1s{C+a*TX$FrPgi9RD43EmWhhK6lP_^n@
z);{u$Ae8_0vJSBYQQ4(P*ShNH^}MwmR6gq1<)lR6P_U8HkidYAjESmthu~mi@63XE
zgdM;KF@3uEf3J!0`JL_GMip?fOKB6cQ&-eUj<9G_54q$=1nfOEB^C-&rL2~p=&Le{Y)T{@xMeTzru6sY3Sc(QCg?ow+D&>*%-qbZQ
z@zRc(yQVvBpQaJ!5(fu$v6$qTkDAqtbK_-yj9-CY=uv*RG^w>}^w-&*N?FYZ@K3pX
zlV;L%IiXXPS_-*7$8~<=Z`UeS+LL+WOBX+Z?tGXzDw=(DQ7B+Z`IvQvE9^Cv0LT!(
z=8$j|ediyvZR&8H%P%JYsWgB>t$m8Au&+^JU~H
zdgDfC;wMnm8V99KAe7;&kpyzh^OfYKMFD_Gfy+k2jc8tFNzSFa<84Lk*Rc7-nI;=I
z#+c!p(G?jXyV&;6GY^kq8AY-nUa*YT#^SB3)JP3;3=pv6f;hf(Gs{}-R-*YpP;C=a
z9}|>Y7W1CG02^~TyZgpN~|oW`T2CW!n`9WRHsj_x~0;$Gd;Q
zyB9o(9(mqvA5P^1ag0yH(Zf)GQlUyy+c+>2WBTs!`Ti0*YceU8J~BZD2&)l*tgZ&M
z>;Fqx0a`h*u5FpL^K5@V%{F!6Uiuh9}zv0e67{HkOkGZt)u~}
z7~vk^1$U9&lEr)Ukct
zg2|bXnu8A^GL616)1<0V$f%R3D@tixUt_%1Djw4%qy)FthXRL4$g`eWk4+zKVF>x~
zL?9U&dCY$+co!o}cj}Cn-d?L2e;_KWOaBy4-5g2h&UeW%M;;3%=XQ=DQO+^2a1~Kd
zx0_Xaw{eHyZADpRzCOJaYBJ;R$pO@Ky&viR6l3R2{pd>9S=4Ap+d8(uvkoX77K_$wVxy^G_VrIs?)DSpPU__$2AVEqOC~RpBYf&qch`)>Tw2
zCUq025l`JRWb(U@j|Imxk^Rjrb9?;TP1+u_;F}NDLYl8`iBmt2bx1Y4;`uPh4bJZC|mRUs9^%++N{%aLnc1g{;VAs?1V^h^VHZ$l2Af&m2?(V4Vd7O2$n!!&@
z?W=JcfnoIIG8h*9}A{i1P5hHRp`
zdwxtL=xb6lMa4?!_58z&n%86$qmL)puo6pZfo;KX%4H;ZiOftWKengcm~yJ$F;IaL
z4jztxl=Pd2WtG&Q63zmcrCM5oY^)H?Pdk?q8qOV_bu+U&d
zs>?o~fN#2dsr25stBS5vQ^FdkW?2PJ7-|?^%Y5|VsE1|aQKS&?agCsxZHMI!nQJqN1rLcYcI>8t#K_wOAI#p48o1%P_
zwGaE=MCfqaW6buDnM>%L*?+hpEskz%coZ3N`HaxDgERvnok;bo)(U-{AQ;|x?5S?|
zJ`=}6SCYS{8Iz)QKd83iBV*3WbHj7cxOCOnYP&S&M7S`_3cM#*;Xhq)1w}G&kzU#T
zz~wi_fqZL+9AB}E|LCIzzEhQ*S~kADVB^|Js=K|Kj6aH%Ui|^(t`s^A&6<2nceo0@
zNIbQ1{oHRt!e?eo^K1lF%Y5b*dQOo|S?v4eNW{zPSm`M@-eNPK()YPa!(C888jkU;
zKV3EaN)u5dAM9CnU9J_fTMjIDmeiExfD2aAxF_8(KWAA+k^#SG`Y$6C0i9y_WCY?N
z%~XU&9qz-PfLPx^1tw6f8)cMSP86ZB^)8|~&vhxB=qbJYDHT@{RiTc(%xHznB9}e@
zu$lgA&=SJILm%FVNbx#3_A1!0HS7F<>S(u_ZS&dC1c_JT!7xTI8;MeMP6I~g70^coSFvbThQ&X;?c*~@
znYZqOlVg09I;?Wb)SkKqqmo58W43w*woF+SH_N9OdVa;fk)J0j`Dq+i1U^<=DHT{N5-4w6AZ48vBl*OMZdhmJWFrs|
z<`k^yAM^z20If5?!;-@?`ar%euPC8WlvglnBRzC<`?nj}s%KPrMPF0OthfWEafq~1
z=rv1E@9_|(g5tun@s)}*b><8h7sY{T23Eg-(-B^#Saer4^TMLAI
zJ$3VVrd&X6OhT8ec`<7t03CICQafSD)14b5I?NA6t=DJl`MQf$ew;-C+_EU3P1~+)
zk?IB6$opoerEO{X-DnB?xIRqYdssAyxd>wOHr9&ma66t{h=5{MPa}R0Y(Q1_a??#L
z>%aLr;j@jMvp6sL6S#LiH()>DbGOzAw#x^m;#jDAOQ?v?+}zv)xcJam5OvboDF#gJ
z-uqlROZ6=!F;Q_s^84eBR3;{MR96Nc_di>boY>7$4CZ?v8t0YZ%B
z3EAK+hX>S_xeh{(&=0Ov*m{&%C1OHGlJE+>@`Q8*QpU~*KKO$8STa?8+0%q6AkR|2ZRa!9lF!E%U6vxKhAgs?SjPk
zN6%3a*I%vzlR|Y1`_Cs~%fQIx6)btPj)GZu5oNXIpvvE_J-K3|8Xnk>OpTnFAHH@G
z_u_MaKJvaw>;XCUlhYFZ;8I^s;(a);)nDu9#=
z)x;J2=Yv1MPvr?J42jc;i}cJESAZy?w{Lh&H%XGazCR%zsei
zCw+`l;ZGla?vq38kos)`h}6#f3<^d#a(4UO7go!Z$=^DBq81?|M61lL8)9h<*utF9
zj(lRspWvonX|vtqyNoQVkn&v#D#Qss;y{dzZ$$L9pNg3))+O^@sTeOXD8IGiI&O~&
zjvSZ^@ZmU*L_P936@rw{pAz2UM%E8tX0dm&KCH{#kwFml6D6D?Vx&rssgl?B37R}aT@4U3@Hx(IDKoke>zTFmQ$b6lrr{V6oD~*Z5p0(!2!xw0->DsQ=0ZdQ#
zagx}pP42N>DWOuBbS-)visdVNU1LVbLJM}!Q5@MoOxp4lLR)Mcd4Vo!DW7h6IiHS0
zgC#d{rm$yxVV`snE!|S11Dj;N3Cb53!H$BputVxF!VSh4g;e@7CbtG6vUfS{02yTHGGfDB50=30WW15G40MEofqs
z%+$18`WH;Q$}mYfDPv_x$GJ2*m~T)j^~we94O0byI@g%O^&R-1O~c2~ZKe+sZ8K$y
z8v+d*QC)N$(@0N#pKcKdE?|CM#a^~x9;(X?%4`p%p*naRuys|od!JOu1rDsob*jly
z#-~mbO|#~l$>uGrGF?1-#*r}f`TEF`Qh%q!ij^C&OB>b7h7(QO%5SSs_O~#RcG-)E
zT>lQ!#NWuoe^6X-4xscx=Tl@86Fx1ZkyB8J(&!8j$bQSDX3eE!MgPhl9xOKp=Ip(My8ml
z(nM-=GKqLSbt&!k{lR0hrk?4ZnPkDge0ZN(gUA))T|^Z$%CAUrF}Q`ookvLQ2~-IF
zyjLck*?H`@n|*vVUu0gpU1DA_3>LCtqW~QtyORp70d}*d>ErDesp5Qv{o5kNp7&h#
zsEA9X{D)YHpy=TRV_uMM!kR9poRM-teW_iat@$>NwOplP
z?W*DX(Ej`B+Oh@d4pQ?GRdk`oTx_`o{?p_nMh}WuHB%{<6)iKNQM9FP($R9x_Y#d`
z$(Pzk4IHjNFoLAWMQlQJqO?y&u20~xX+Cg14s@`MO{<1)mcvdbH7&GZ&y&g{ZZ5C2
z9y5PyBtf3WmLs!mYJs8E*7Kzm6EGyb
zR+*TI0bd3*Vbg(O$p27eqHb`F8@`Xq5tBh$Mn0|s!!OV*5CfISWh8DQ)NJ2>R!}V2
z#_eZ@5B|lf^MpadrG6W5uYI9ybnWLFepCI0Wnx&N^zRP<6g1qPbR!F1k^N*K@1L*Ol#>8t|V|gQV
zu_tzbj9$5XV#DQIbD4=w^5KBuWh=?<=TKsLh%RRrNIvN_hulK|ca)9Vcv4UF0H}cv
zFmVA3YPx^Wj-f)f-SA+C?Dbwb!ckTDe)AXf9a(5wH5rJuBRpE_1KufqmqQGwZXW%d
zVMCT-!o`{D8vR9qB2cy+56C?1OdZcW{K3Vepd38kTBlVdJ*A^CpQh~ik@peNsZxCo{K;K<~@$vP^0|ZRh`+vNR4eYP-xg2NPqVUx(K;%
z?=!TljJsaDjEor^@?O7!&mz?J0oh57lsF2&g_>OI59_Q*+Ps8|yu5kG9xllXWwk+A
z!MLUh!c)Jo8OA5-)AVc$Rr^^h=NihS>)`AVxq2{gzv$=Bq>>xv
zhhtr)tE#1saFaN%KvD(Zh{A4}0
z>Em(-Wuh6(sbzG_@n`hq5w_ef$Zvb27DDGcLZVZSq{7t?@H|hWkjwdqeo{sY^*V-7
ztMWRwBK{a)Cqdh%XiX%+i=`1pE;8o{4W6{eQ$PT{W
zKIzMl2xM~~1{l=u2?u^@*cQG~
zwO6miHc{@mt|C$YiqBzUw5|5>Qvg%p&eM+jOE!MTBZ*;#Z&K18O(4#!HBXPidxpK_
zX!QlvH7kZ%<(=~U?P+-lPgd!Jf7yG*y_^)VbJj^0)vC||^X2(5HI<-PvnsQ=rjX~#?HkV6ClspIbp!Xm{~M|(c<^yk{HAorN}jL*s--vfzCz;u2ndkvI8q*
zeR7lgt!hYiVw!h0QS-yG1vSD~|IDd_xWjqBB8Zlnc;#{2j?tjT+J7ozQ134HPuKvH
zz9(h?MG3AEASx*0>?f?C7RB#zPz4TB`q>nQ?u5FHNR>t957tS`)GTIyX`2fZaC_ng
zNKnIlh@nST?Fi<#ubId9h5`N*1rSM@f&z<#-M|J38<%IdF#HNscZRFVbMKfZyN>s1
zCwC`)4_A5T9Bx)Z7yO-g-`zwCFdu@ssd*VeiGD}!4>RC(x4h3XqN~Vzy78rnq)Gl1
zaCCQSgexNwbh|+d$@lwrFPf%5Yz~erf=8f86pdm}a8e2vI7_r@vX%9BB{NQts
zBr-Z%$!8EcJEvgJZZ;zAof13$XFBC?`*?UB&ru0an;`>fo~JzX$#@7N-REDkx^+>-
z5aeTf8$O-1<==I{8Q)PqwA=PgKDbl3C=k?7Bo0p8z8x`Fs5DFA3_t~5hd9h$J4}zA
zMO=)J#f-@*Z&^NyPD$PRlE`l^E0$9Jjl0AyKf-%OOFP&
zF)vXwyBh-3UBSVlDrQc{e~vfF#NCS|_V%9dR0-Y~{Mev?L$Y#l^N+L@doN?Cn%yR1
zE0aOyR(Y;h2P{R{2iQTDf7kC+j{$RK7p+VdlBR_Ej2SZ6u~4!S*gn9p;(91
zv`8*=({(1&O!IZyftrbv$JE>_h+}S37nANM1+E(csBOo-zDBg%D
zG~_5=WE+I2)Z
zd+pZro}Q$%e>DU;jj`^x98%g7mzN`EyNZe_w%Odh8h6!B8Md}tM3*2%-TINLDV6<*
zG#gS~uiPZ1YpTt0j<++op!R3@Ad7Ct-}DzolSh-E|}1I53
zW>P)wWS@JL&NE8=dcuAJ3C_+UyQjgG;I8RSo_A32@iB{mLWVU#T&Le`=~N>9r!uSg
z>uQnwldQ~-9uyW6S;bS97{L@Bkd*5Z8{#Gz?NWyjaO06aFd7B>d&+-JPq0CK%5J=63&
z^ZsR!v_5{x{)ACq8Ochh_V{c#2;CM!x`*?~_5%QNacyrdD@1H#faOg9RA3ou`IH
zYtDEYUs6Gr&;e4fNa#dfTFOA`dv6SM3r8#r`7vuTYsx*f(Bb@H8^5!O#&e118I;>u
z0k<&XU-%WRxDC?r*bqei+Q;S_2cYWiD>tvuBp0aiZvwfvg>3kZjkIYfsi<6)0i?R#
zJS$X#7>7KeLEE7iu^0ZkbuTs;7#QKF^S>*;=cm9;jB&d$KPG^%9|O{48p>HtgupLb
zaX$KN_lL4#Ei;t*SSQ_N3&(F(fn|8Q5Twmn?`CT!Mr<6|;rooCQ9w*=rjDdswh7+T
zI4HvyI0@DQtHi;kW$FqjWS`g{b*t}j_>D9!gPJL&HqDz@WCrZZ20!n_ouO2rmV@_!v@+Mbu|jK?(>=;N
z14b-YvkdcIys;SKiX1gbBD^GnHDleUpXyLsHM_*~C9!98KvR8E7d{E00KNx8{@1mw
zDhwmN347Y=OQ=a$zf{20^P3IyL+En3awcq9?rk>uZ!NRl4Ab5W%3O)^l{!~s=>Vxs
zd2|Mv?BWtl!C7k(4N3t&d-Q>hJGYmu6|o-V4dSg3E(=eS7l%~47aFDuqgv7l44Scu
z+Zp);%jXbo{FmhMV~$CrPB&!VQ`*?v9WIx2I({*s5gWP#0lrqL+9j#7-dNmxG(c-t
zrjxLWN{sKiyAT%tFi>8}r){WfY-}91^dJpP`zWXy4{<9FyE(7BYu>w(kR89*E-#Ne
zAqR7hT(_m?&u*~z?}Sybpj=oS{n)f_h}OFH{zJ<6)v1>GL-!EPzNR2GV^;zda7}=M
zQW)~(>6{Ib(TJwxX(MopuSCm&kSwsB3Rq{uB*nTNkRRZQ4vsF#@_U};GgmypItmBV
zEu6PY7ub>q_S{h?P12pys22AvIa_!f6`FY83HeP_<9ALRsnc0;&tpnp0H8kx<=$E5&DFe1m#rl`Y$AgOEjV8+bmp5f%HR&mtsWJH(-NI%1Z?fUw6&vohYiw)66$
zxHS`08-q%xtH$UVD48WoImbx#$E-JKpsnq<+JY`{;e#PYN)hW(%Zpyv
zMd6JqngA2kB_j3(cbJJFNo{nU5Xm+_r79y0=V0WA_Bp`FW|$^A^7`)<{b7JC)0pLg
zAai-*d{#8TdJO|TpFgIy{A_<`HO-+
z8@i2FJdw2%G7!TB{3AEd6kAn*)Z*Cn%Q&^mu;4_e69^d;P5LV_yH27CW(k{1$RkY3
zyn4Kux0qD-^M7y$Q`qluDH>C<(-l6)9T2Li$!PSPbfav^@!Q48G~x6NGIAqe`|BQ2
zGW@A3D%;L}r@H!6TkXsG2We|lV_yqo2}hW1()e9czc_w73$h^~@~@zg_0#9nMd`d)
zn%<clmCuH-#N}7`RBTCiK`EVv1wwux`8wKj5ukYA
zuy-klK3ZgL=h;p$SGaDdh$tL&>eM8
zFSa0rf|@~|T@$0PAMK=fIJPuD(0l-+^t70erd&3RLXD3~Q;&mb42>y@cLb>DV7p|Z
z)69GQRP|Wu^gWr;Ro##qOOnr^7}!l7j1@I6c1d<9)W3-xdnxCk~@V06I4djPp_3A9_oZutcD(tj{
zy|2G{tvm*-$2940XAcGHRjOx-}i_brByul-MAiO&fm
zU>d$o7la3dQ=ran@wI>A23GELq8Er~?RV=WbnnDo?sxvPHv@C=#%+z8P2z-2HJO0R
z;bcRKBTv=tK^eu}*!cM6Zi0$AyhRdJKsblicgx)=(Fzhb=fo>cQm+#Z=!-ZG@9*^S
zyFCAk4hiRZ{_znQqk$t)i&h-0gl;}<(;VA?9Tv)kI|ErMSAddl7Z;cUr}(dB9{w|6
zWr3t8yEj7*GB-V*wN@(djR8w@;Fogp@bm#eC3Cfr&^ARcKg5VF-2%&5aR6u*A575+*j&bK3U+UZ~P{lAo@o0
zQ#o|OsId0_jyh@qlardDE$jO|>(f7qr)3r(?L^lzD1pvsJ8A}B+dwaW%{E?&L=x>2
zmhKoFND~Hx+X3;{!=$N)Q=L%47YL>Uk(cdRp9d?YKPJEs&C)NghAk9(L
zasG-nKL!ZrU1c^9PUSBCJ-BvU{q^j4m}cG(s~!w0#8X8%d;J|85WjEK0Ir!U
zoW4q8yx{k2;{4@FB?@lOXBR~Xc#Eg$o4GLYjKpQ(n{|aumbcW5X&MOuQJ7_H@2X@y
z7LucQ3#&|9^YXhMQ*Zcfkv)uW_+@Ar3ihuCUG=I49(dk&#H7fSuPf8DaB}|9VRI!C
z2Ooc3nC-m)X(^Z$m~UJWkP?|fW~uo;EKbbMQvdn|r{pTbcV-KS%QsVM$RP_9B
zM>N1CrfDQwXO8(NiML($tivb}W4|Wu%bsewI&ZKXVsjAvn2ub%_g+J9z&q8Rhss^?
z^_!s&^E_76k8K?az!_ko(iFjY;#a@c=%jyt%6C^sY;ECg!u`^~hffNIHnPe}M1`LM
zRT#KCl+Mip=&*$+yzzTJ(_NLg&>9uusNzi7H(I$aOQ~kc7{0Fq)(!H2ndlni8laN6
z9YQyLwczKMsbwgnvi*f{&FiQC1!CYqK9*##G?=-|+I#~sWH48e?8A28;nbSEK)KPuG$m=JGerq_Y5X$xqLSFyN~BG1D+POPEEXOfQ`o+5k0NDh
zZ#`nE>?aJ9!0fg0FMv2jJV0&YgT@#Vu)0`JL#
zmXH?*({+ghV${s->b=6-P}eN$rJv(4bLzXW-x(nZZ`Ea!@0g~$&Mw~Z=9%NIsUN`0
z5~dd!KrSs?ZoikV4)RHvBpn5{P1FuXz2hDxG!aByWXT-*ddI7ynBKO=k#=SPIvaFR
zw2Gt$O$|5GONsVlATZSaRloejH!Z5PH!SCU)c(|ks(ztF>a+wL?Uf2t3Ua>tMcY&Q
zeST>D--qx=Nq14(qRQb1eI>SQN_&Uo5IbxJ*c*keuRPpw>sggG+y%ukm$3dXFLv)^jvy|3ncZ1Pta{9G?{7$Q^2xYBU>M=Y70WPo7WIbSlLB%i&&
z!e42FtY8ljJurkYPT)ajafpdVFzSq@Kq@?u-YyIxk?swu4jMd*xF(FlA};)HOAEi$
zie=VcQmu34TFu^2UxdqGt|d5HDPvCB3hPY~-sxpM0FsD#Y8V3!7~~t;|50lpl^KxX
z9S(lj8!-@9+2#F}Z{pGsymA$T1X)vX`-11$s8P)t;2iUhA;i`iJX1KLMzSpwR|4{0
zUe-F|!A4xv_-J|h;X8deWKYT-`zOu)zFBKSqN2Xr#*I#i#)h1+DT;6yq)<=?Z$)wW
z%W`!PXRCAqW+5_!@x8gxA%SvQyZVV#@tAYQ$7NpE_+__*24A%TsXcq9g?+2pqUJN1
z9}NXA>P6%K^h&wyjLG&Z-j6p;dx}{&{qYv|nEuf84Xu;@*qX{j*_O{NqrF}Z3}gX!FD%V}^wmHzp_4ZrXg+0iPK``{M>vnj_=|f5
z6R^Tsf-XeZPN!|fX(9>?8mKaDSdrY`_SKoUxyc%>(^m#CH*vb8c$TVWJx=93Hff$$
zni)gw*moO#udaLFd4X==e|fS2u@^_*mjl0S0Il1pxk>}p=9RDiDg~T&2VK9sX7xbr
zTwqEVXoLtfeq_h!xy-K~vMI&w^IJNK$P=uw%B8#b-n(S_Z$d0nSi>+`9%D;fULF-^YOKm
zZ~zc+g0$aV0#5Zy;?4_Vt!vV6T+rs`iq9{>BHPnk@oJ4xKjL|C?kuh1J##bUjF
zoAb;JjO2IHE5LR?7~jytAL?ySS?z=dvh_l7I}k>5j6tNHL%VCSbQ3f5oq!4ue?!o(
zfR4o(c&g)yT_XoO`{zl&WcsT$h;It;hn%(Cg)Jca{wwzT0X)mL$z&JU4DbWKeE24C
zMJsDdhtbeu!)d^CV+fUO^f{g1C7oUc#LKvm?CEWh!CBjtGO5qCDv&$t8wPYWki6#~
zCyN{%k86-ie7_6)l=9b?OSac|qlYA;C8W64?;<3;Z=sMRfZE586$7`feV`s+pv@b-
z1@hkj#3G=+o+p}2Y~9HD(L;R`8faTl=J<5K6LST90H?3BV|=gkwVsu8fmRF3P5giI
z%(LDgR3?Ew0KVS}x_CKV_RIKDmX}hih9gZsF_BrwAn;qKShVI*$3D%u;V1LAI#=bo
zqw>G*FB(%E(@CF-(J}>eO0=`%rUaFRCd@Xj4p6qVah{N)o=R(tj!%*mztESLd4nwx
zN@ogLX))JgTr8+AJ6keY+l|e71;@bO>;>jNT5N!*%+Aj3+c4|L7LODgn??`X@LgWC
zyO$dDQ!0?$@sndRG$@OegD%7g#?6uE*c~s
zn-9+Zw%t`oerPj}-|9X#Yj`ZY@uE}H*S2=(R0b?BJhN}OxW!kER}}XnjrFO
zwRk3sk8*+dz@L%kLH#k48ek!(t?P9RwM}n(Jnh?#@NUnWPs`{*gL2`MqYDB&tZ$e`n93_ErmP#ob_f{w!mq&|
zAe$gfjyf%Jdx|_BDQ>tVM}%PU;8I`tkJL0WvQx&oO+LZkNCTG+-ibYUeA5NR@hx61
zipZ;$xv=;7)VMTEw641oa_a;Vx$v-9f{7<--&70DYZ`*jjB5{jatrVpwE~y3dG>*kM-+Zc4h`1Z`~-EBXvkZ(kEO60({ugA+#EXYTd8mJ
z9tBchXSpU&Tl(sSTw}cTB?2E~Gh*Bbn22JuAL-u}OwRTaQN8Hg!d2E*a*lj%S6*oi
zx|;GG=L<>IL)SSC%;oWqypY<)`8oi{|Chkakt$kTOE1~?&@hbGpT)2CF~iJiSiOY0
z|IP<)fKDzx6`9V&P^eu(v#y<|TXW`gar7HmClzXYeC_f3T{gtwt{tufVgD$G0WtlhMOa^EMBCtglot6+!1
z=w4ri6ssOUuf-pt_`ko-8kVgN!&jpmqQ)0u$(VuAB)MX9!TGKfQW)bdz@CQ!gF#DqxaZ*fL+tB{27@+)
znwdwiywL>9PCkT_W_$^kCTGo)d)kFGZ2Rr|r9l^?mdy|6$!VW^+5#u@X5se@dEL9v
z7Oz`GaE6l@o|y*sOQNDhI_I}~6hdH3co;iCt*yFAE2XpMM@=UpoP3Y%*i9&eNC^!C
z?(Uug6*ED4DH?fY3rd~4ye3-NM2RE@6=?M%b)PENc>X4vpqsPzOC&dPk!2OG>pm@A
zF=qXu6|DGfyVh27R&w+O(v!`QWqS4~`$<%E?RM&?%XKnSB{|&|(pGGi0ab3DEuwKW
zZPjhEfnI$L=Wa{aL;Q%8DA-ld0<}>)mPPViVEzWO^>5i7Q^osB?vdRZyu^l=xbmcJODQ*M5yuC-3b7AeeWNW>mRg8QJzart$(P@$bq
z0wh4^hj&_0s!XQ+AjPV0zOg8H>`l;A;1(1nV+RJtw$h}$84mQhV8&UEpjtYWDckx*
zE)HdaU@28o(;*_U90Bu4q*a({7prebcP-9yQM9MWj#nanMdbs?ad{*nXZp-a%ZfgC
zo<5yarzVX%=~y{$q1Lxc00=Myf2$aT@c|aSl)$5-qj#li
zGB^ap#3rVufdCuP6&b`t2+zxSsmXszBI@m-72Ws00O5*+Z+2GJ-Uwjnix%p*{a@cg
zD@^OncM`>5@7pDd#vpPoKxP3TNr<5q+NRd81UeNUh=;f*{xguu>I<#E7cY06uYW#t
z6Gy>8Z?*ugdOEj45c&eI#5S;E`C$<0tat`f~R!>P!_)-|wW
zyWvJXuMg1
z?2v;;7s?i49Il%0l?NyI3#Cdpl@`7xHL{}f45$O{=-?A`))wFJ29yO)6|0>$S!Pxw
z6hkDOQ=YxitP2JQvink&dy{K-eAFw~ZB=ijwkCq*^*4kv>cRX>>K5%aWW}Hz8POJ>
z>)-=V4Qn38F-Aw*(4vJ#^_(7z98`vixo9h16W~bg)07P?1vx_Q($9p-o?cXTVH!fh
z1r0b&@eUc+x`1=6f&~t6=z0
zRF@*yXo#fQzcclaNw9SU3Oa{-*r9?B2tsX8-b;EY(O*fNU^cr+=a)H)c3_)rD1-HL{prdDG;%FT9@E~B{v?T+C
zt8lRM%MoaPW~7M5PE@EUt*y`-F8mWh{fS=2
zL5hNCv+gyuXMtZ*vgIB{K@nCRA%foQAG<%|QUsR5DrqmD)1~AS;~9>cC1k%?y1yg(
zB)+N&5RXr9l^%al$LyOg^F^BgEYO_w`$AlC3@li2yt!+Is;249R<^9&#j;|n-t_8c
zY?Nq~&G+Imy>6-vSDeJBly-|{oxJsbSuyeJZ4^C`U5LzPsis7cusPZp=LzC!PcGKK
z#_;`Z@=I>fF3y7^Kusnj*8b{n3O7D~=j7}2v#-dseFvLHG+XIh{axK*ja#J(!cVEV
z*iCs!-{@t#{RtO-U#GCR-?4H2u@_rN{&b<+is0^=Q?#|mUrzrFrx^JXm-f@_H3Kzc
zWwN5;5HQIl#SIX8$a8fsN*5T*L3~!+>A*cT|GgyKn-
z*Kd9EA~>y1$E9Gt+!v_gD^B4Ld7eHvFRRh3cTc5#a6?Z%b$2mT;cIS19DV3!7+S%s
z|6IqVtv{Xcg)f*riEML^+gSH@|CfA0phiX~CR=etvTQR(dxhZkapvSA*f4pbNX<;;
z+MCy}BSWeIcalh0MGe{K&u^7lJG3`dG!djoT~dO4OrUpCM18_GEU1LtoIKdV3>m-t^Ty7aGJL(7@jhm=s)ox5nU^jZw
zVAt5xHdK>6+8?g4VKww5u-C@Mv|@HmA9CrSI@_IMou715({)JY{8PrArreNyFP%yt
zF!_P?qqa}p)rOmoifgKYTft$4ircM3!+TZj_~w=1cX93Nfws*MHSQ&0DhSzKyJ?&=
zBhs@mM80LEwme|idP(LGZ`w*0->v!6UNyBz<7=W5IAeuQhHTG{dhI>yr%dTp&2b^E
z15bu->~}1YsHTY1K{sRORx69c@otsG)p~+ErulpFKze{2_O;sNZLwakmlo5fT4R?9
zRK5)aqIpd%9X&k@OG|dDk7d_BlFGWgZs8*3lx&^e+{w<1a&A0i--DWuSlSsW&Hj#&|z(x0g*Qu
z8&9-3kE;h2LI>e%fosC%yno%wsr2^h?x6O)<@$es)o*ng;SCV(`t<1c3VDq7`7enb
z`T^gq47)BW@NU5GY2azO@_9KKgEyK)sQf?rBL+8KaGH4)d8|&aNs28wOM+aJKD|B@9-ajf^mtE|!`K3(&)&RY1
z2eMXVu`4dIS|(RrvR0VR7jS0>_-3o{YwP@w&0EFm=VPtT7P~=VE|fS{o9%Tdkd;
zY&WZN#G!@ZJC+2AN-=xup5~1t5PSZ0MS%rl&~eh~eb2lYnHXMeW^2!o&F{WpU=p;)
z&7_h(`)L1MkEI4L7$tEksI<@hf!Hp;)dh^;K!Ra#@V7>+$aUZOd4ctP!)y;#
zPkV;Z$m+TaY3F#<@Xo#o$x-=CCM)8e;G1NHVX!*hz$c?yZL!dC3DAh=~1(UPyYhb-V=!j;vniuwq{LQ(^3C9Kv?B&
zJaq|96qQ2E6f5(54oS~ghodsVnwaqrC)Tlg_b$mJ7bfuhPzn`@z_H|B_^eKBWDVp-VBPW2%DR!%#P)ks1F86zji_^x0Q$W6NUC|^_sK7~TP#O&exNIo%`>o1PWQKjqO8l1R!JWqta<&Z)n1+IX@@xBkd`=6Pqg+T2CEKkY|Kp|U*`YK`!h
z=c+VhQ+`NcM`nED6{4^05qq<5M=jW!U?=QH$Y2|IfQpc+3xlY8UouSHQ%3pMSPJ~E
zF6H8I(-SL-`nXQO?+k^BNpaMmsSA~FtrwMVy|=fQ6krH9?&Pr~FPl<*+td=Nu*2)j
z)2@W@b39eQ4ce*y3TQmNXfl@CM*NiMG9dwSrKq{;`j};1-_y1EX5_ydzDaEd47zY?
zdCp5`{H-ssw;=AgKY6s&c2afWCCuWk=z%8WHZs2zQ6&mhCx2iMtStd8pYMZrm)~l4$)>XYztfOtm
z#m;LlH<84sODAVgaB{OL$3K4CK63bJof4GuaEc{eJe4Q`Rlf)3I0-_kjXb#210
z9fZ@W!5U1cS<8e`O}|)E>SZ&Uj#}T%8tij6>RbjqYg@~%<=56L$koZf;Rg599dc(d
zPh=m2I52Yv@O}gxt0M!^i3{2csV^C;E1#7QM)o34opj@f3Um?jf3Y(KLlA?HasL?0
zfppCZv1&RRla2p|YMW@C
z;O@cQ-5K0n6Wrb1-4fj0T|)@&!Gi>M861MUTkx~_>inn91yj@%v-ez3yWj3!>si^|
z_?V}dwy?bDv!PYNu?05ARRJJ5D;}IhcgDhcc_EcDC69Ns{B>PrGP_@>36W
z=1yq$G${5o`H>dva*IiU)++1ytzm(iz(M>UbqB4vfFrM;K
zCO3%2H7%ZKkX{f))bhA}d@vyZTF%}rLfnGP6Ftkk?fdeB*?y+I^kF07M8jFm$URXJ
ztJ0Xles%JhNUx?^SSIIjGCH<9Ua(GrJlC=o$+%{2{|
zdvkpTPJtsXrJ+MR!;kH1fmK|}b~B+(q8K%OtIrU9;=yw^O$uj~M+xYu4csM}ie>>?
z#0?&NOBmlriIVPXi-S+acJy2awY|p-Y*G#ke$BZ7JoVOJd;R+WR_5LF4(uvis-n-f
z`2dW}9N;Jfo_kFUv2F8Kod1AkcoL}ZZ@I!Rv%;NcgyS9*fc&s=miz?YE_HU@f#&18
z(f&WP{p)r9{yP}goWJMYSKo`saslLv-?lkD{!g0-7!&^U_H)Ue0EqN$M8kg&fSP~s
z0SGgF7>)bj|2~4hjCH)Q47{^!Z_8VW42DHL(+Torv4g#|OyE$8N-TMZ`rv=I4v^*R}2y?EmO_
z8^Uv3e>|l8Uw^|2O$F{R0{Jl?s~3Ju5^Rpw8VdN_(7sC%^WUELpTap#F|;gqJ<5Vf
z)KtW9Tl-S8(~2-NkAECKUh!RqjJ;J+kb&!d9dIHiFYa&L)ZEbPCqD1=P5~m`LbL``sU2Z>EE;yQ%b0}3p96dIk1?2Grw#Z4#5TnJldXWWueP2EE
zrUyu0f5!7%6BW&G_ow1v9UpiQv{vI0-Fn?qsyW(TY`G1MB9hTSS2AD@j$no<6r!p5
z#b=|$Tz9pjxJ|50G2BWS2*YjJZ4u%>?pde)oGaWjkpqeKp%?gYc&1u4
zI26vdQwUVzZme-PFbO?y>^YLn`(!$>pU4C`^0LOM0osMRyL;$Et2WSJQ{NFSP7V$6
z7*k2mDH2N3`X644Y-98?zZ5Eb{%)HJg-Q0Mba}>^u;oj}24ta83H=Y~Afj(NA~SL2BZbk-YMzm
zDG3aXl4FDAG(5pSLo~`E2E9g#TfP|GR92i^$|(gY?5^{iSP(L5K}v&dA^?Cv84LXn
zn=+Pi#XMZ!MSTuYL7pU)6>IEdGZ7^vMaa
z>>^ddf9mA=+`RIhjl
z4w^6Ubn!Li3T`l9u2Jaw=-Y-S5F7h{-HdeBIj$IE;!?SCP0gl;y$~=idMl;Wwq0rU
zcDPx4xk-B18XBg+EB~|Z9H~%BGxkUaFV^z~4NZdqY0jpyx~ieRS*c=vXZ7J4piHrI
z^AC3U`(|cj6c|*Y{fm++6Q_xtb10HeN+edcfJ`Pb`3hq^^d%Fi^rI~WqA$B4|A~-P
zNq8@9Xe1|7ly}O#n|DZG&ey735B6yecb~-|rQwMkGpl`4nA{|8*P?_o{A`Z_IXYJi
z%kMnyU~b+_#*K5U7!f5GmD%YbhrtxALyw7aH|o#K$!p2qp%Wa)iDXszxl$#ER7|lY
zX|_oc&+)FCX_raX5^+yWz=BEoL=M9)a^+jkPDckxWK{|p_wXrrBuXh;L3&EL^%jR@
z@k|v)h$W{4UP(PRB3w6C(&%!C9dlNmkuw$>HJ6l`g0$}}t
ze91D
z6sMTWhA|8IVJ4k+BuO;1j&qe8_hDcW4auEPcwB%LQ~LO1lQGB_$6GK}`=e};wzZU=
zmUV+;r5xHL+-mEJ?fm%ep%DAu8T;tyVq32DqK;}pL3i8uO5Y{X-JM-skM%6BKC`41
zZaxas_lXp=k|x&u2ihoi*GRD3J}d~KJq~Mmqg-=Y
zT6^wL4TgZ1&u~y?qhjw6<0@M$>v&0rX$I6#Vp(>o;B8I7LwW5tmgTb&%y~NTCq~y2
z*##@PD*DL)bZ%Nkl#49Ux>lds|8tEq_GEAA{LNm4@iFk
zq)rDsN~gNMwQ|NgMzT{V!hHVW6&w#CxuNt!ase7RKf*N3Ca&Y7*4@sR;!v
zRQ@FXc45HC0OsFCxbPToI+&*My{jVwifT>t(BIu3y6+X7gXXlbknG(8b-6u=cxV{#
zuzS3IXvAGaB3T?k1CVHcMoumu!vBtyx_Q?M0BHH>%}wJE4&5&vz^;qO_m1%c&F4Hb
z3ybgPt4sfrMC!3uQ^8-${ZC5y|E&=aK-0eK*Te??x9Y&-^IMnV9O8Q*?dnF^DMrV>
z;CZWs`5FKdSg5Q#{`m4^(lKBa5FEUR$Fvi^+~2js0w`TV;Q&9~h{<ann<+tHZ0^DZdv
zMi0web2Trna@V&_SBP72{rfJ@M7AlW@JimUo~%XN)68z)Ez>GxLlB&jxM9_kYg#v2
zNGd|p@`qdxiIm2M;{_T8Q+m=kE@wN6$}^-&RJ)>rwJ?p9vFUR7;W?|&mGwO~A&f#b
z6F7kpc?uMpkP9>w3~6Akh@-7tW;|`Cfm}CfxKmn4a5?CY64$7uz1$aF
zhUx-FT?`zBKEw6T-K2`&Iq^Rp5DWry(5q+^oen)(T>z8fRZR7Oikx&=e
zTNZI$D_Cl4a}+@p({$B&<>R{N0$IRGwJxvMcHZho&*(&2En7wo?l+iy7S0TPI4yhV
zui^XN)IT+0IJ{DbDdSMdirarc)8G>D^eq}m{DEvFpAO<-qfC)=iW4k;gWS-yG(T0s
zuL_nJ#*O^>HxhfsmLf9RERNY~3SP&H(tv1J{US)zjVH)%oPrP0OBRzPbS8lkeROn`
zGg~n&JuNdofBW}0g3RUi_?}JUURH@h*{uvpSsy8Z`r5PZ$S|Id8>6{@S;Fe0IYcV4
zl#7s-0x=g|SlsQ^bwo`fAXVgkrXq-}O
z4uP`r;>8c8(O)wRsTiao3Pi(A+9dTqO0%1c%1ibASq=)l|4^Ev
z0|uS_YnD=3unIydW&b>Y8|%M*i9TwxVAz^pu;dNc5;&D%;{?gsTS3wJW27dn{rR)<
z2PIiv8|<^Bw(H%rbSwYOqYJM$l=Gzsk`p2xB<_;ve9oD?4u9w6$htF*If5c5k3p1R
z6(0k~0)-&Tui>_(zMMphS+5iVN{Yju6{QfDXZD!1aVs-i#|mogypURp-l+lmh9jsA;4SLc(E}
z%*eAr;kY~{Njg&c8@N1n@1$<%-bsovH>u9;Zj|bL@tLu1WzBFeWM?ME!wgfbB;3R*
zd^JYgpq`#m_^+I0s0t-?JJnKWWkWYDYjk6;RB`jYJ}^4i;mqXfNcOJ5n|0khQnEZQ
znYmb1krUV$#0=tTGCwdLwd}mSzy27_0nbKk`Lq9~S-K29FFmQEk`Wav*Oge~@^~grCv>
zVKWEY7?rZtOd)%QEBESb;<$ia7ZI!rzn`{Dm?ek1{&;P+(ddLab&z#;<~04V%Pz5y
zZqjD(bAsxzO?)Gv?H4=*+h>)wn3?sh|
zo??P}7Wg}5>xZEL^0K
ztpV(Mg|-U_{}ss_)Y#)~6M%yIKKI{xALv^vKmJCWZh`b%b6;rQ@$EEeFA^dV@TzsR
zyqO+!4+0d7!9cNWd^<;MM*>pSyMcVxj?(E-=hkT(bfh?*vlSQ+dyKm;y~H=?YTf;A
zMip2|%g&(S?1?~KSWdF52+C?hs~)z=N2}JjqPV`CeYrd!LZU;jFg8Py#vvD+Ud^{R
z&3gsMMVa#wB>7;6V98!gNKN2bVT3Y7l&r-G!l>FUwn#r_*?^Bq{
zFq^XD7Y~-fsXR|#9)0ZE@h-2tebpLNKsFw^jUxDc+#UWHDbReuW#|q{+
z>yqr3H~n8QffzN)0Bg}9B|O4$j5U#fs&*rJBt;VC*@z+BHe;FE?}t6)j%h!f>+zO`
zIe2BA&0*ZnlyDd(R9|{&E-RoQlZKsAjo%=+e_A`lm+>Aju`*_RMVipsuwsC`R(1fW
zHH?)ez2E*^!0TrJ$C4<#Y{Oy@`1`V!$Yh>G)wjG0(Ycs=hm48n2VB84)>`46(-!Jg
ziQtt9ZXR41zm_d$4RyRO>m<+aC#8U^!S#vU@Ow7SFH1pb8@S3v){Ppnd^H>x%`#F0
zS~sv-$8!-FB&D&^by)ah#f|r^YIf4d3FQf*^;gHHc
zv2&&qpsckP{(UJ<+gh@+nF3>N#TgHYOwhT=(ty4Zr0k!{s?2$O%a>{s`t2M3txs<1
zt~nTtFX3!fBoRf7Ucf=cG#&DNEg6LpF~TZ0&cK)nm&PnZrzvXR*9%Dd-jUEB+IP(I
z#Exv$qfqo1m++tH9EnbZw|}Mgofsyc&kkd>qwtZphU>0AJD%TKTNtVQ(86f@kdj=|
zYT331_oI8BSD7V0GtJw%WtEOsX#@SnvO=zx`Yf5eN#fUtB@uY!ANJt6y98+ucq(cQ?rIAb<7r~%TKTj`;_Sd_!xYo9c
zm!lmBz-<^~0-+g)9!;?tl*$3C-O7z9CY9PMA0&(WneX&+?Nx2x6ia-um1!*-tSb^J
zy=d*hF!2#tS{4d<#x=ry#Ba!9JB)=^K^F%HHatJ_SR<=RUNsxeGLy+a978EP&6#@XI439kZfH8jn?EyF@&aJ
zpsW2rjjwO(=pA|FbIj4*9Vi!t`6+HaXPJ9_@0J(2?uG9_n@zjcz39i1Y{;w;r!*Yq
zLGl$WGg&NT1YXuDl`F*uo_Pgp4jkOwzqe-CuMrUmd7Qou57Pmgwt!NxN&$(o;5dvV
zt>T%*T4H=Um8|%QNWrD6gr?+)-Pu-!GXpsuU-lHTr#9Q7cGRKb@T~JX7u1B2?eQSe
z;fTIv32AA5bmDPw`j>>Hc-PiOOg3J_bVmKAg;#o-54Q&kG3urE(|?5OxjyZEfoqS`
za&e?i7PlJEp44;s0upF)6-ej!u-1Wyo--@ZP`8tkUFt+{5urN)Z9mj2n*_LxfGm+Y
z8+Dj)n9Esl1+}(2+fiz_6~Kst7Dj6i*Y};dHMg^;t3gvKn^CSAS!}Y{03QAsq`WFh
zhq=V-2zf26t#z(4zR$YDxknoB!)5l%ppZB$Y^sK_b?vlRo@*+qmYoV;CpV<@7Nw#p
zLdz6I9i2Q?(Wtlz=f2AVW6`-_HO~DgJND8IV!5@d!BB~R86>x&#_K+gsf{<|{Z~fS
zD_mU2;kquXR02+ve+F>jXBb9j;FqKQ1HT?Enc&#YFev!eS=Lb3RHYY<4zwa&GXE?-
z)j2S50ZTVDM+xvbdrU?S*s_$#8$77XeHOFR;9Xc}64-slI#XY*=sdEtki{Ma@vl~}
zf@ja=_@}?kx-bqeB-x9aS-a#WzPZN>)P_@8`1IEClG0$}+{AdSNRm!#LmgBKJz#wf
zW$DCp6kX>oKgBcKWX|c8q^lZ-2?mpsmGq14mazS>TzxLbVzL;Z}uD-%&CEH%8xM
zP00OTERC|t-QhR`(mZEqb;4_m|AW94=bhu+v>O^6ixTjsKvzIezq4Z&n0xj=+RpmbukASgcU+!Q
z^ze^{JK9saQSMI=hySMR{-P_9vtttL-7Kc>4Db>=EPUL7GE49
zlMF&TIPuwA%;tNOy|U5;kry6+w4z9a+lL$wRVo*#J041VL;v$sbP`}r8oK#ukJ3JO
zw?^fhn@)0%FoU-)kxyNB5F{NJG-+}K=TIb_#CvI})DJG7@vh#X$!HNTJf&+iuL~$j
z31^M_3+*Vz&>{t=pRl%<&U@K<%RH>>q140;x=8+C(vtvHLWOvc7w`7)F@4qI
zx})hVseDH`v&ol5>|R4ePPR#>*nor-POO@Gd1->2fNTJM4$~`dXqZ|v#z4@=nG&H$
z3ylU1CX0aj)44b7^PDf$eb?xIbMVK$VCtwMmm)Y1@vjNLD3KA{iQUw4UB?M{n*tI_
zCvdGqX(=GkDcpjj(BSi%;ie)@1TuHe4ycGDy8j3#Z|vu;Yr{xJYx#0+eb|+~JF$0N
zD0E%b%ikW$L=YC>m0PsAHXTMf7Ys~#sj0@u13+j6q3n4LvV4w^@LgZi#Kt5e+(QZP
z@m3kUS7iFb??B!=Uy=>GF@*vqdAk24hkKpJuQr~RSh5~Fmmpa?7cB36zchEq*7^Ir
zjw-A|Q2?F45+0gPQmP!jXaeBtU$OAGZYJjlyuBF(>~>0A7DUeNe50x2a1-lvNh+_a
zaUJbq0e*zdZJxW8D{`QqoY_T+=&erMoE$$?DPozV7F@_YvlT5f^#?nOBNIx5W5Thp
zCPE5woE87*J2`=ld%J-3dka0k3QjlNT3zC3T(K9J?7M$qt}mMj6Q-w%0UK?=#UOt@
z-!Iu2)y6JKu_~C3%P`cpwCoFB@{<2SEE;miLVQMj|!KDhuCP$*T~d|p!TuClC0zgYwPQmPTOHf
zDu2#h03zLa1SiX{5&Q9bzmYvZlFG?op7JVdw=a5(!XYXhr|1aLu-mcbBY-kJ@(O)R
zX%|fd?FRG2%+jgqP$Jx+V+)wm7SFrKR{FX(rRm?IgEu|#u9dcD@pl-3un_PCPOEelH0=%IS}poeO>I
zZpYJ-YD-Yn*xhPaoNv?~tqjHGEo7(Rkm5*4sKw|xx%N&z&7uvM6s5oAVa=8_*09Qv
zmocYqh@zwJPvxkK~4ft1)^7N`~&?
zKo}-PoXSZUX{itPci!}Cn%ZkWRO`!!)vV3WO3`;YdYhU~Q->lXMDR~F$wL+FRdSC$
zrY0AK=(kHYbB~{Nsga1Zy-k}viXOV{&hc#&FAK>L{62F8{&MTa@q7HNfBq?)rZnQ!
zi;`4;2eo+O^B$w!1g;p|ER0G}LMT@svkSF+C>H&S^Dw8})zKAuP3I#$5Gzi_oFfQG
zMf>jsAvUCC1}0aTsQs*E^x5TcUzoe@6Ap5l7cbJvW=8`Z3D0oni`8bsn<`wT=Cl-$zI
z;dQx!#p`$LS%F6E2|~B~C-SB$N>UEqF}30Qvj_aWj!7Bm7OWszWkxO&X2vEhYpH=d
zv?X1vHly($)xPPbD}3HKQLSpBPN%hrN^}W|Yp^3MZ1s0XfH-Qu;M&u=pyBPeH|(CD
zU5`0?ITZp9LPrEyDiM@%p!^jVyWno`QC4p|b301Oal^cJXc5QKH5bliocTgmnbCcA
z6RY5&j0_ZpY*(0kBFVay@Y&Ol01JQktovkBan8A5xQWl~(dQ@TnOJN)wW7h2z!
zN9b-h7}7*Cx;;I=owaH(0{2hJ=P+gSOgi`rD=Vj51yNl`ebQmCT}{DhA<^AhJwMEq
zv*YpZ1nf%H2p&gk?$0Q1m9gUX@sn4+roMX4vm#0GW?Ra+)PLJ><$*Z#e4zixe>t)A
z{P|C)8d5KQ+Tv+Z`=&S*3lM25zvsGCjBKX^;ixS#S%8uK{}%sa9RJ>4T}Z$S*IfrR
z!@G5u;o5*X?7V%?{LJEgLCuK|q^kn$y~JBVj}SBTJI@|CNwf$AyetIz>s|*Q6oxl*
zy&WTK`hMM!`j2eG^zg1+*Nq2&yIqg-HD>@TzvREfRp9#OPUT8ca&*0N;|afJ(1v^Y
z_z(uC%X7M4w)j3+2q}XN!=9JC#g
z!5gIKo&sjW4=^9&B?-
z$JSpz(cjlZgZBPf7e9f=#7)TBM9)xOb#8hwm-ABj8Sxm*d`81AHb{?jsj4L6lf(
z$Eqbfu^#zG!hQPEvij@Znp&z$lu95TOr|U#R|zxi<;J^;T_8$RfMl#WjNr^8TIPYN
z{DS6uD&@u#wGR!20Wm2sYgOs`dcte)vWe=oyBMv|{V2Y-XA!xbK?(=QkQ(#BJ~#T~
z5!ne*Q1*b)1}l=<7(NaSHThm?4GOiK1%sT`A2zw;QC(^lOwgk%M)pvgC>pTf=-=5P
z2DUe&d6CWl>pj866vB2pKBlwto_F-W#Xq#A(xYDZPm|x+(qJ`wk7BX>AM9*6eDS4NGFh8y(afhTp8OwAU
zSh~4|ln)C61&f`X-2`r$j&7Wbwz3lzK;r{cqT5?8@WO(1i>|)%6pkj~zT_G_*`g1?
zRgo_iWB?BRRD5=vzw`;%Tey+x$dscb=*UYWlAbXK4-HFY8m;|H!iR*|DHa@I29ezC
znBp_&6RW=^%pK9e}<{`2e*d?s*!H!$iImQ=5V3<~+G6;6A~i-aC)TllNaxrdn4
zEW#`Ihhjyi5|%Wa3Cu3U)ARN!c0jbJ#%oqzYM{|bejzzn_QL9zFR{8H5l?g=$wH3W
zOGElTJz<7W?r&!Ej0^^H-ApKGVwvxS60feBXM&r9%%^J6V`8*c&fY1bK*zil-Pyt+
zx3mf6Lsz{m@(m_H+W@lp?n9GBg
zk`BKVc#re7oXFXn-yY-*c;?xk$Rj&*!>v@u!y`2MVL*lChxR>w)oS#iOOij|?|gPN
zuH`mz
zSMtQ!Vse_9Uk%YIaO#UdppT0jHjWm5zxAM6^_6F8bMpXW%Xj@%@h&%TKtxJT$^@ou
z_DGCCt}UBqtU?O8tb&j4MHCe%4MzQuEk&E|cREC>1i)JcGG~#5pEpr6yzR3dmb8m#
z2T&n;rxw}Ua?~FsBcRAqtY_K#M!gSArgMBHV;M1j+XaHf^50X<1;s(0J9QE)-^Ac@
z$5q8wxwcWDZdrj<^?*@nPzo_TImNTVOqoK6hmet5gv!(jU@_ybj7CSy;};bfoZ!Q8r5z@?e*L=5e!R@l>ak9D+okUX
zESCp~C9v|pA-h5Bx)mM!RM+@)>(Y;IVQ8p>9EjLv%}2WUuS0l0D|}a8M4$L3v;^v#
z5LNLv8j(%mO&`lz;uEf*U7*R4-?jUw+j**_o5+wp^E7h?
zYRDR~x6M36m)2Y3fDyF^UtJI=DYe~g>$X`M6
zKukUz26f6=+RD&&Kdv!W4aq(G6zbXA%NtXXqFjx@1>T#%x*nB%pR%1Tc{x^|^H_j1Hfu>ep&U-@AJ2=?)21Ho)d?EaBJWK2iC8A4LGv
z4AYCCTfmL?3`o(8>rDGC52WJ{OkHm%9B!Rk-$&Aqv-#`RY#yJg!Pe*Ra-i`ce`z!ygh68pr147^1@AuCT
zzu)RT56+tHQ6^EkYbqx0^HdyJ+}cwsruLbX`c;^cG77P-2gubw$1cwR{~NDsiYFno
z{(k2eS>4IA3|mpu(WbOH5f;b(a~^Ub2ZSsO4Ijf`X@TeQj<f+%H&D3ToF!&|{Sp-7?(Xz$?8s19{P@qCq(GpL@9-o9XBcf4F>)>T
zy!G){xr|yzvshgKPRGMGxL!yF!(cB^IP2aToWdZ}8Uluo1uKp*2@REBcaIETclVz3
z!fHDbOH|P3an8rj9(Z-N^e;}eEOqZswt8=9{xZIr#3OnH8MDTNKROF26dhURVv72T
ze~S}w|DuCF%9lHWmAfoj^eCEiWY8%5^~R6>TL}g(#?jFT0-#vffo19LXJ8@Pj?Kga
zR>m?(P?PuYK_52mV>!Hj8rghQK2_J%nx2=NLyfGuk@AKwg=cqw(3Tn#kU4KyJ#|&nq@@^8@+`0LLr!
zPf4RLqQYPjP*6LayK<3nk+gOlF|>fB`=Yb?03`M1)?KnuQxD+b;XDCUdA7nf=tETW{vx_zpugT8W1VPKwSy`F?Z~hmp{Cm%u?UO=Ri?EFye=i
zD&;Ln8rq+p^gB7E>j5h7H!S8MUtA;~5fI>_Ohc}e{!x>(oFrY+lqf__P|x@g
z(n#b+D77oD9u{oj7JllQgUTz|8a=`Es&|t4N+qj1!?4o^uJW&`25aje&rnn@l+VFN
zVgUdE*7iAaC9J%KHH&mx2)mR3`N|wc`@1cksl#Q;(gK4Dc0Rtbyhye?1qIf}H_P)$
zQzrhaC#4Ha=jQgVmrlJQ%H*-11Q5o>qi9B7r)qV!@e=fBX9eqfj`BniA%EheuFx+rx9X8cOVxEW;feKSw>!wYQ#4-u?`4aCb!`_AB>;!Pqt3clb3OSM-iu57Y}a
z`qJr94i)2KE%NR2VnajpYme?TbJPT||9(!C+9q@>_$&QltJMidTRq1}!`S*kZ6o;+
zeH3$6O+=1^$8iQOMzf5Q3x25P_cJvj3_W|@8k+cI&X$;@c{Z$SsYVUvv1FX{p{6aj
zpW$GYZxHXe8ikdssM%37-&*sFxAFHl-U`MZqy;5HWVeB>8O-2l0ZUukWZQ*m4q{Yv
zh$KoXSrxM#!hxt?J964uI#pO%L$}kHX8onn>0YPO!dt27hI^+i8y1$n;VP*^G8nSj
z=reY4Tr)GFeT-x4A;;L&-XUKZV-L;Y%hw{QFQ6~PDH1Eym6^UR{-l$~o0e6@Ua@bh
zije^mKtffV(5G|1aaKZdpW`y&Dq&>@!>l{<8>xpybl80CFXMK`WEUe!CD#Sv3G?qZ#!9xI!y
zfSYSDua$*|O%ICofunZiZ^59YZXXRA3#BMEBv5>UA%pt%)9XMQ$d=^r1RGu)a<6~V
ze6VfiM*U2F?|pG?8zi*6W
zsI*}>!NBi$HHdeQ2WwEyFRaz4pO71fOTP|${iT?(jTzSJqKrubvN}ImYPc53=iSRN
zjRRh`;|76L^!`@Y{kWAFULlw<_566$zloXhbntr~&7gfRQT+Qc
z;{BIc*1y9HDj#V}WoI(#-OR#BFZhufue*;)#YqGl;$>L`%+5bZk6wzJlEZN#2LJV&
z(!z~kUgPlpoyVSM$Mz}f6I6R>k4;0w0+(p92G7+;r`Kgx&R^)nOnM)>+U3-G28x_>
z;qyMhY~t}0G_n)@fE`C`WWA|3md33dj@gI01rqAqc>lUr9Bq@KKvimd3|3W+-sC7d
z5lAR2=gXXMvM<)$vcq>T$O&dZK~(Ad1O&8rl*zZOP{?j&Z#e1pfE&pQ=h8|RPbnGi
zRO%H}M$t*AGCHfj75DS&xPwH??M~fRqBv*<{5Dq;gzs9(P*+uJ+kQlrsCzNXt!%s`
zJ3!mm9t+ikO5l#mdB?(rzoiK`JqcgZ+`Vd^5xpcdFH1*;=$X-C7Rm=>1i5fY=2|N^
zQyz7)9i0Xlzx#*?K2W=KLC4
zYC2~0gegf>pKLxACEHElA^yPj)HmsTxIB3c{$seaHX2{odab(ihaK+Q{{nfX`@&MMsPubAoZ{;^{Fo%-+jT1
zdIWn#8m?%POrwgOqy6RHam{rf&Rasdp-*J=dFQAV>+hee)Ec4F=&-5H>I@|elJ=_}qAa3Dhfq7pnUkmhzRtNgo*8At
z7Ve=Z0h~uXJa?1zKf&)pFlh-O;#j?RMuV(-I}Q%+6q`U(QW&
zL;2+H3OLAI+9I$L%aloO^%i^9of}*ICegE_cMWw3bzpfd2WdA7$x4?SFA=uo
zVq9ceYSR&w!dQ2nP93}5aFwUpv`3smg&?V?#KwGK%JVYQ1GbcEa?ps>6a#
z=dj`j6k(L9Uq145IAA~fNJh|w8V9a(?HwZLJa-a16HXsCYU^v8_}hn;?n<_;p?uKQ
z)$MpZiubyNWw`>3FRZvo5g{eX%?D;oNjx9P%JP`;k4r8xZ`=MjKoze3Q%@fF5vjd?
z?joxzmP{IzlVt1^Up;5riBYp*74+C%y^r&hE61h>>l)QP5x?KnsRNO~q^8JOS!(fi4%RK`ZaJ)g}z
zAeNx8Ot&G1Qkjx2DkvI<<>H8;=HgD*v96(Edv7;_HY&2ju9B(89)(>f-J<4eka?egzNG=m|
z-~YK7Wc%~F`h1B^s4!Q{{dlG+S?-T+MnUKNjyImRlOgQJhS~1231ueeay!hqhakufXwOfV<0i_4}&xzfi+}p{Blf
zfGT|Q3iWMMSfF`-!@=vcY2lyc+e!W<0LK0A9uho(uL6dTGwJ{CTDtvLzmGX*Z!ZMC
zEIj0HJmfyKFU6w+2{e>|d+0?g?-roHO%8th5CA!>TgCo33%Ka61Gj0#a0UEvsK=%=Uo<(7Ccu4Q94P3loIVQkcMwP`2lf(CluK|^T?#Zv>F;#
zh^Xo2%PnuP=)z_F8Q-)_nRgj-<$X^ANJg+@iZg|O?53gO1Gj3X;h^qALT8rVj;S|+
zhT22LWW)nQET|3g@e;)_f{)n^e@-B6X=gB6c&DN7yiUj`TuYHvOhtlDjyf$O!&aV9
zD;E9UVhZGB9OGpfLrActmO!l`+RTYgm00{sfy)M2jF()I@+iWp?2!?KIA(%WrZOf4
zL4n+kA1smz{jd%SiZ1cZ&S38jHJ=F~2}Qt6uE3f)fJ;G7Q7MWo_!O^wq;OE8%V=g=
zeRlr~bWkrmL8};ohCqiUMUMni>U)CJ{hLR4eEmW+4X3N`s<-?5#=}S1j%c+WHb`zO28S1kVEd?OyAX!`Vs9bCFWIAtgelFDON41LVRQOS`KMGU5d
zpG;dW(OSp}Sa5|?Ix0KR3pIw8tc3NQ?zyp;G`!A7SfisAW>qn?QD5Yg)2MWoK3q};
z{Xr>#zu{P14L?PXL138BFGuW-=d0yqO({J^8|jz9qoDXkFV7vmmciPx2BZ7wkGJ|agQ4-MXRnuwMm=xphs{*qTVyTJsm+liFby2z`|JD*eFr9Kp~4!
z@f(S@>Nk&yyT})NB@||vRdiRs`N!ds`qHRd6gd1Ly0<=zpN*gt(Thj2Qdh55=tJl<
zHR-aNo1bx2X1yk!3hlXM#l$0K^Q(^An_L-i3O~rL(U2|wC{r4q1hs
z^_`LUba1YU5$<;%kT{KH-eYEh`OC*Y|DM^N9ak};A*OLHl-BvEhWg^tW~182g7ven
zIp9PK_8T)39W+zGfv0NQX^ci%5;dMIJ{iTg6bEdM
zUJTwwV-+z2{DD?utrR4hs9^~X<&jtt)u<2-U$Il`V9^0K?G|zH1TN}*Ya5rD=a<^4
z;RF>eF^8!yiV0tWBRFS=uc7k$BJjA6p2AHv@{R~YHOGEfTK-rIAtR5Xv`(9`ZdqI*
zVP7O+C5-*$$7?B&SKHb&{J6|_e0=QIre9BKuBSj7MOgf)R~E%rEXl2ZMyDB5
z6NMD>1J3{H36iEX1kvd)ZlB%s(kVCFnvycf1D1i_7+(azkT2
z1gk*vXKUh}p4Yduk6w)7Ddh=B)~t|6LLX$IIS}G4BpK@KIz!dhuK(}Ds%ma(qER%#ZfAO|O5sUL8?(qV=!4Y<
zd2wlfl`UK(-c7sFqu-xzzXHwMtj$~gTZz>b#S7g|c9g&M_Fctnskm*S_++~snXCno
zGnMAV*kR)4T2i&IuewlHKH2IR!qng(Zs5G8Hu&?N>URmN-b%KUx96A%H?`t+d5@<@
zd6)3&!Q+Z+mA%DF#-)(So&Z-L)sNu;*nF-R2g+453Q7TjmUPkEYP*rCDcvf8QEee(
zuKYRI=*+h74yH{Xt7&xe0yDl%rG6T3*1kLl<~3+z>E%rwP_$@TA#s44`?VIAeCGAz
zuU_o-K-_Mu3*tn00Aml^$N6@7Updke_yOn^=i>Xmy6vXGmd%Yo@OClF*S5+E_M8#S
z!gIV7|9?vL1|$zq60-s5DZ#mIm8hgyHuLi!L26o9L*{Rg%3Q|hrfT-HO4RPYFouYg0q^XWiBjrljN;|!r-4hvbT$uH{Mustu-S~(Gw(C}qVn%A<3_YWNb6;gg
zs!wvb^Kw6nh!>xR4EA~ZWknDYXA1>3c%n&n-o2FY4(5iN(g;Bc+%1~E!){zn)(AP+
zXmH0a1Evn$ysn060M_e6>NaS%xu>Q3#ooa0QgR|+`0d+SzM}~1|2sQ;1mJS%`^QQA
zP~CU7-5?sFz_ItwY8x5~H}6j_2*6%8)%eH0tNq;ga<@U|aq)10=jShi?~I$hU*j3u
zAnCH%P2G)0?01KG{7MQx#QsDskcM`N16<%`b@F9(@UHSRg3t*}QUmbC_TC17GfO_J
zz||0i-))6qu49}VlGo#5*}iSU3YvGG?v^WB<)^S25JaB?+V+uYnFYF&U~j=Vy^$lm
z)%1pG5Ggwx1m&7E%!2yxL_|75@H4${BeI^|A}6Fzm_0Y*@DV+jSpKSZ5
zOX0QoZB&C_BP-{pb=
z?^hUM!%&j6s``o0GV;1Eom}uSM@&o1pT6sY(&1Chw!co
z(G#I&OC;rnDenEfg@?!bJ*^PI91M;2jh5?5=i8xlZH^HP3`a%*O_T}&7J6cgwM0~)
z#1xzyQSx!WByO;1QH;x=>9VsDK2GH=Ok+b(fNgJ6CklTtN*{UY90nPdb0ccZ|KaMJ
zqAGvDc4wMw+uU)oZDVKGWY=WRCm3!cI6>nP2gFGY(4
z*+qj3w8MfxZ1L}rs8nhd#XOE$+wxXrKVYd?T@v$6L&fi9X0gMU$#2iL<(bewzMgy9$LZe9zVsg<&tKIB{fhRCgmQ|0?)=mdE#8znhaw{4;
zA7^?A{Je8j(5eVqmb*F_i0UzC{+pwe0lc!S?(%@~e!YkMEf|XTELpu46HR9pqit$=
z?8rcUn6~eU^oeeQJav?TNCshRQ11sbEbWyps)uOi*CJq>SGWDmrAiNO86_T^SXENp
zv~p9iGGjF>N>vt=X-FhnymMY?BzGnwp)%ylTva+
z*mf#V2HuYAQF<@c&m~L!is|I)q+6P>N*Z-QyVf3K__qikP0QneZR`HhkiK=TUF
zhJOMuuu6u}`d@i=hVqj6b*ns&T~1$*&oguu6g~toe^%s?yX?G|xYko4BSc=7PTZy1
zNuhD2z?Z%96Gr1k_1j+9)_|js{m;B3^dl~9ZZomm(L_%z8qT*jc`WwOXl|qPi}%cf
zGoKf`^m|I+R2K}DkGam2bTYp9{TbU_F(8zvuM1)bACfzh-h?o&
zSM_ex2rB&xA;LgGZgrq;=Ms3!^>QYmbzHZ3@;Ca_Q>B(|bO|3bQVywy)*j{psZ?Ph
zh+>wf*13V=hqsRegT=A0#G_#ECTG@-`Lb7yU7woopu?-E_;0GT;n-Gv1tFVOgEEuo
zK~0d87Hvzr?ZtuKBEGYEgIM}{2*Mp7DvYKe%!W46{Idm-Q0)9{cW=MU$2z_IWP+jt
zccUUtlph7M5Eln0GHL2UfC&Q#vi7f$^t(jyk!x`RiU=LPksqLPcJ?0$)fW%&R;l=mbzaM4f60zu(lg2?V{Qt
z8jEiTEA4}74t%pTcbyN+5$tkKxMVWyH}GFb{QXlcx_EwEZb&2)6Y{i)V%gIoC!6f8
z;8s!VJ6P|}*v)!g%7299OAU1(f-jiMS|A(eplsu
zzS^?GX%fAH`X5}{nM6MXgF9YwgzPcw`CfOxO>jc9Z~xj=?W4&4XDR=Oa>`&_2lv93
z{QK6J&)|{t-?`fXyMCg;n2cRl-1qa1w>B^tZwFQN%s)_sg91J(`)?2bkN*GsZ!4bw
z*R(lhI7w~VfZu$e6>Sh4P%;E}z=5IF7ya_g!9U|)Lu=RG(fCIcSVIu>e4I0UQwKM}
zPwjB-MECsbcOBu%N!^(z{R)3A%EJ+F9%NiW4TN#cC^?{Cwi7
zT!#{D6Q4=vjK5*WywZIlepyn<cu&VJ+M9K=jhDOTzOxIMZ85EK{2lkv9`e2>r1uNh3Y>R!4tvVCB~D6
zxVIjgmAU2kW*322Mb|DAqt+-Sn5+?pF5WI}zT1>7P_%x8)f~2AQw6)hkS5&TnkhmL
zbr09>@5={I88L6%LJt<_TkqaDL({NweoVznt#y+O>UO!a8dR(b>^2A(g3i`CUo+SC
z1}@ePDjQhNlJjQ+oM>s2ikH_10vvr<)3p){YN(H0a=U^~?@48*bd)2pBuXaf(Vdha
z?0&W|llfH@Pym;I*=;KV=^5}Du8gtU)#AQKt&cEj$YT+B(%;KA1C20?lY(k#0Mt;{
zz{SlifT%cH1H-(On!IFgeZeeRuwrU4HpX23BwTPbZVK&-MCe=lh?YgY>fa-f{<&e-
zct=@Ax%Wv+fd#EtCKyLj1aDZXvB2jcL~>%*<4V{d
z&ag#JvtbTF(^*p#P)1W(p;P?aGWz3v(K$jdY_<_y*?ewENiW+rLAi0!gk1O7H4e28-%hnC4jRWFhkE`!Nq#6ufWriSiKl-1qzT}POmqK6QN(N{o
z)?w;z%fCW^S^NN)I{)#sM~UYt4pik->|m=r6y=?cIt=cl&I$z{iS@Fw2GKD!-Z4=W
z-!W#aFNu`9g$l;3RDi#*=;_RvM5F
zyYsXV-fa4oWqz#62s(f87AY}SS5H!p0fQo$7q5!c0%{X6Wr3E){`7GDbi7T#p>KR!
zeWJhGH2GHRmUni){W9RAM(-(k^Qv-dB(YJ#7bZ_gtY3r1s}W*o>Etu!r=|C2C!9i|
zC?>vb=`B*ao3P4ZmFBZ;3hkDEvwSWlFK){Cy&Nk7&wGYFB8O03RZ-p8Dqpog6-^HQ
z-Sd3ATkt#rrFiU&aV@Sd_gjjko7UHIYNyBbp#cZK*uXd>W1`B55JC-y{9GG{Dj9@Y
z9+TMY{}e=Uv!->jptl^|cHi7+QeC+qKgR~CF}03|xx(Cy6+jL08r?BWEGKqFFF
zbntXgp4pl4H=!u*40)v&4TsCUxw>KGt~<=w7;d&sugI}3Hg;C+;CH977m{F9n)Res
zydF*XVF8O1
z&c6OV%$h$OedQMY^3DcgoOx55?Bd|OZRgqE9<0l2HX8cxDKPArjaK=hb5B<@$A7wX
zkYJWq^~N~dh#YZU%r(9RUFa6RB%;BZED|K?EoWA<0q9A_6#Zx3Mi<7S{QR{D&oBJ-
zQoW#nzG-PQm!gJB>_S{(t`%iCe96%8gZ8U&eqYcRpb8H_M=eV$H0D09ckk_(V`dEU
zL^yXf&jPsw6p%p|i#`e2>nA-2N?gme4km@hRsNL|U1?167_UhEUTqEXm*YyavMuUM
zkFRQh2?Rj-8*M+q%lE492L;u~-BW6QZ!bzh3u%8kC6_#B$~IaTJ^WT+IbSa5`>U`f^
zpS0Z?=KGFRv&{Z`N>N^7Fr{|Mz8kIS&4=W+D5Q%^UsL2sK`E5nBgA20T`Csh8Ui!6
z@g)pkX5k)F%W$F~fn0B;IeuPHC1n^Lb%KA^uXgj9ZCSN>DH$$$>3w^Ki%n=IHS2!N
zY4&8l+1YUgH3wYCFfyYgO{~(PY$N-Rh-^mJ;4WXufRav*u&L%g#=#E|DyPO
zn8XNhqJ(At=Li6QIQMv+>w4Ba2nyzdj1jnB|CWfWu=o{0(u`p#3j1*4yI@1h2h4FfiSo
z?|P2!Mx)6TY{8#>MJ6qFbdaG#LPV_d1dNsQy1F!2%VusCI0F;{33`9b^mnME=7`9?
zT>Ugj9#;#^=L|G`t7$DS8eu~0E3b6QY4uBQ9E&KefM6k*3cVk7VLo>uJ#=|`x)mkF
zp5&vaPGTnMc0f~GH5RXkL7(~QE0aEbc6aE~Dy|ZgojlmaNG424TJBZAnwwZSqwu&L
z!+3-9{`=)=&+rxEkIG_k8wm1r3}1LrL1)
zW7-y^Zgrr5xUy8r!*ru-mM-W$q##N7vw$)OGmx%HgVXVZiu|mi)dOohgVq*hyCD-9{HItLL~aDZCM!D
zFj+d>y7fm^eD?d)$1Ck?r`KtcL8=PVuqx<7EI>tOUNiqkZ6=3zsdm?^s+)08SN~m>
zkb*%u_Z+|`#tkxoa^0fkL;rmJ(ckHQ_9JQL7Irk5M
zU3t=&u!2pR|Faj8=`U$fLOgOf9a``|XOD-oCeOXluUEgp*{W^t3ouZ}0l}U>6+&aU
zoqJksh%$Zdx^F^d0@B~47;YofXLqVtH1<68EM@Hu&CqfLrnZ%Q8z+?f3n!Kuu&o#>
z$UPI$Ef~?2ek8})X@=I63Vq@qS}fI;1B=9EW%QYXuJIT&^V(X67y~RiHf4gIgSdxt
zIiPyT4c46MI?pUQoUe{g*HriSZh(>YR_xtTLDSW~ogp~rD9&NDGd<7r^juJb+pg8#
z@#>wY86ZiWypH9S?QUrU22WxZi^p|w1%Y!@NDjVMBT^n`vi27>6q+#y@0Tnt6WbQ+
zU>czX$bx5MprzD%S>P-Ie373YXKYHkj7olk2;n-%rFiC%*NkkO()arZ7`@KU-aQ*{
zL`fc%_}E6?K|syDdc?|D%km3}4uNci48@{WWU~Xs*7{xQU2vG99JYG_@}5
zXM&C79Yooszr&^^^tR=neaD;pQ*N^SyVL!`ryAT_uE*Qt8!PM9V|YGVTss}#
zDJ?Ga&8$3sQjV3Atx$!9n2ZAXEFSc6uTV
zRN!K|_{A^#VJMG$a2P;s)et_N=v2A5OvHx!au;?JKy-A;+;vlZZ#(o8My~DJ+v@qY
zZn|^ok=eHcHF@%@g$paj?wjy4Qw?|YQN8Xzr|376sH7ySS`A4ROudcOT`3{2C>+Dr
z&kpTR&@V49;^g^B4OSL*)9h5y-wIrI>4hkN^(T}bXqsARm&wmV<{ItZ__JGr7>Aak
z+WauSfYw6rWLK`{JykBv+
z=}qw)loRp@=i2Z|I6Je8oTro#k6+|%ayEctB|p18tg@?iytel`DYAM<2~R*oLZs$d
zxR-2i#;Wfmy1F_8(aV9!gSrUcHKK!ix+
z%Q@cck34)G
z1VloDqqT>Bnh)Mh`nDd_9_Fir(g$E)Om14WH_D!@BX;FtC4V;;Op;~ZDAAveF#>u1GP5U27+sDx-A3o7!t
zvVLY8^U8i-64CHNi4|Wtc5&Bs0j3yU36^!%7J6$<+c|kZ!nNEncTl4al
zlFd-u4et@r_aKoVz*r@TUA4gh*>8+DO)c9u7wGSW`bwKDkaT`cNih6&xcp3rci+)=nxzfusyjOc{b@LY7Yw|kz^eWRk3HM|
zBka|r7|Iw|bjHr$zpF6&838`x-0tVAHE|vL1k%KPp=-j|%sx*ehS?tkV8Otr$MODD
zJKO*L=^wf9r*1-UWU0;~2JBTt{)rmEJ(h~;DsMnUxrDfzfgD8H{2nrqiwww!wkQqg
zDujSO*{tLR_+x9gICaPwrs){nnVT*h-Iji8EO8DEz9vp~OA3L#cr*IhI>Slym&u$k
zN7BOqYY?A*Hq|_fgla59@WhwW(L0#^r4e9h#`uhYP_JWG*6bD}+_@9fz|bFtNyZ>!
z$4hA`1Cq^y_q8@|-~2Cs-kB}p=`%iPtq2KL
zX{EpQhVQYUWG?M>d%Lw*m<$5DNaI69;jVj1_%nJ>)bnLbMd)b6^Z!7PtoLZ3iWllw
zr>VUP&~sum-e*ll7>R3$%>Q5)liP1*V@arCB!vgZ1fdjj|WDB&xvZ=+XGI%Tu!26p>r!;bk?r-j+`XhX+ZLLH4a=HwzA$`E#_2L8y
zlu~_H#6U&Bu!P&af50!z55fz*_mOt@JC(+VcW$#5c?Plwy%*posn&T+rW$)iHjI=iXsIjz!!=3Ex*Vl$C=}i
z$B(=7rgk>vxdi>uAYamoY_Z@lGhz9iKn;tQ$B{UU4q#*a@h}b5uW4Y+^C4(ahr^xLp+JwHD=#XN
zF
ssVUIJbhqz;-Wb!{-miVoPGebEnbx@ui7hOBph{ktD`BrgnoDv~liV(6agG0b
zMCjYgbX(tE5L*!r&xc9Tx*32XbvIxLG9rH+Rv7Q+6ge}
zbC_->WtlX%AcLy>K{wOjyn5atj!Gral6R!(!7wP+jX@4QarzYbJ6PsI&-xNitRBMy
z8%P!Xqh?oS;i4xh5;^ee(NR50hJd*Uvv+-}gbv><@`0Ev?NDuEFu2lbkBSpqYp1JU
zm}M3zNx@{$K6p~qHBMjE-otm%*=yu4Oh|RhtIvCeteuG}qkXr<%(nGhM$R*H`zAof
z3qP+jL+KQFgHc+bz|l!pQFfQ29|&bVIJ0MyPS@a?(Tf%qMUmo(9$j**mf;n3V0W9O
z+|;zp)Ds_f
zgJUwX1ou)rnr@+bOL(lfIKtp)cRXs*2o*CjxY%54*Yo0e><2&?EG_r8*
zp4kvgXsZ*Ih256cr)BTblj>dbDny$g3ewQJsI>=3ewCH18TItK{t~ebHMO&*MwkVU
zj?A)?h2<18kE-Blt+|6N7VqEJu?K6~mg9>sF`7aU?CYUEh!Tez>Kxu;d*htz6-2ih
z>8Tu5L|*!j1WEt_Y3wY&fn7s3`7$0c?)C9(!?M@Gl2@pkfDeb>wX1;uc!AS2<0aSt
ziz$YMG_(8+H`%Rq2hLZqQgw#PE@FW56a$UWSI+f1#RbpQ7}BnF!&2C_cO8|5q+E^Y
z`}a?U)I^$)-^C@E?>&{5qZtLGwB~atzeh}uzs>enMkQTEjNI{TaK6468A{DufO`2~
z*x#D5>5@ZFL=8=&9){N!9PKn<+uX{1X!KJST-N0MAC4Y0m>Z&E*sa{QjPh74|yC{eb1?MZnM{_XfRyWnSnV4)H6zx8;%8gR%!syTNd
zlnk+UZFTWRh`i<<968|&|G^&!7f|Api{<+1r91Z~@N*Iu!Z06VRPJcMGyM!c9fJYm
z&;~(i&FWz1QK&|Q^IXmI`;qW;MxI7a242A1JB!&vo=EQO=SVgsbZBmp?3@fr7VzBn
z5Y(MgHwmlNE;B{ga2gYE^r>fw61obAJ9TucrKMFcXMH1rIUy+cXe1p@_BZwi1w$gp
z++VA@MZlb!PD|J7%EIp$)BjD#^GDZBj$bwy)_p@rK2GUq7FMLK)boq#auMM
zU2vH6ds!}imUP&-YwQT)s5+9vULS_EZUEqm$+7eTceJ!F+?l;OR<16s6S^)rDL7e3
zZt>ZKfk#J{J^7G}mi=#ma2k39dR>|clftK1_a?MoETVYwJjUpDZ1F1%-?yY70T`R5_P-@^%L(x(+46L%<0>GkE)!kZZ<5yn}1@>nWD6mqZ+BQ%SzSvZ+Be>
z>j@w3>vU9%Nz84Qe3|
z?}+QsSI!weFULv-fE5A5=CaH^o+2OMaaYt*O{p`EcB!fsyIM2961(tf6-%x%@%t^Hzl|l7!Q7~FvvYgjCm=ROb
z2}-0k4cyN6kcN-j79N*%MMt2JlN5K4gEh2;hi~u+#-3@{BnyVxphI}t2#W7(TRr2fLLNmEx+##_44#*DHm_4cUSBOP
zbc@y$({@9q7L_rVOE~2aHJtcuRYP=;<{*RkHbi@GWJL^?qxi`2YG$pRyjljlx)hSN
z;zR5`QtDX0hQ$X_DzL!bhd{lEwUNmMUb0erK6&u&aTG6`D_l64Il7anD_>4D_v&O<
z^>+Xtry*%`3c>5_>$WM1Z?mepgG5KI*TEYZK>+7Lz+6ALJ{UerF=NYUFvFp9NbYM#
zpG{Rq{~Y~7Y9tM1oca%YGrL-Ls)*gZch8ROd?4|iq?`&pxrPm-G1o5@Rrne*IMs>A
zpabjLXg=)I1XOgMGM0Lg^C>KrGA4J!CS$9Gt~c%jtWtp%1l1->rPjpR=rj)a
zdL_PbY*IMSU~Y{h*pS__gl{?T>F=K`ULQXg|4`^ARhE=)r)_;&h>!qOq0>~#FDYHs
zGk(FIB3wG-a3dU%8=H6RxqE|q!3ZwAC+l&snRa9wlhl1U>njmoIgx01Z?YTA>UEeT
zJ+>J|d{MaBc}ur8`IM4n?pK2I-orsnM)Y~I;l)TneQ##=Ennu-?HM+8@c#6D>dDK~
zi`I;vvf-^p#bwdVD_~jxzwiJJ`GxTqWo7`Sg;TuwDy-wcDBGg*3!CaeKEa
zQ77vOzCWqj4`gF?Uz=^`9nO?ytl+AsTLLd>6>oL*9yT79|E+Z#>kkN
zu{Z|-IKUwkShBF4V$OEa8{hU1^4CDCPBt#%y%%S;b)|^o=jTkrK*g%A-ix}oFUXtb
z#}iWOq1ih#F)PuJt6)>zewxKT-^PT~r-O-{gC&a`fqz>xc|B3XOMDr2<=c{`NKS$-iofP}$)C2G5Md<&OeY>K=1~A6|^X`K&zOe^v(>rT9
zz9}LEI|XhY?3@Yl-q+qTUwuw4cN3JZ!MEc{%8>s3=N|VH{9^!&QJ*aY}}Bdu!%r%C(%S$wI#K%RIESefrmc1tH#|526I%1
z$=GwARl8W?Wb;>za{@rnZ57w^0ArZuKm2sWd<
z1nk|+PigEv)`R|v9dpO-oE~{7|5jBH
zMIEEuZU>{5Eb{{9bqCy#vi97D2i^Cj7!7R~><#E7C
zdyDdNCTDOsP|?=5y1807U5DeEATH;#lwx!&keg$~4m^5{HrQw`Zmj$UM9{h_%Ao>>
zOWZVozZH^bP3p5oOe-2~t22;bt^cyqMyaxAsWiGq^PqxA5Ir~<5Au=jrio@Di@S0hDxU^~bMs_dyjA^&_PP{mJz;Po?`wX-&ySBzVM{ddMZYHsANV4EL*!hjHaX
z_nXZ?i^rZPS5D4lX<7ihg7)%LGxgR|?JPJGX*IGYAeJ$h><(hpTCt+rZh46kwh}23
zftBGJS=2hLUABqq&nI8vfK{7YwdooJP!F*NMdC)u-9z(umOy-LsQJ3KcEOW{DwrokssWho{B
zZ07v?7m9OUQ_D~{Z{q}AX6XE!7@XF}7VI_u7+18+=5>lh+G%yIt>~)zkgs<=CpQ*OfI7}1_XmRmrKMljQE=}>|oC{Tz?u`
z1lR}+jnEw25RJ=ZLq;)2-DB)lcW!WzgIhl_C17a>Z{Tz@7CcX~+&yl3B;2SYY|XOX
z!)Hv_puqK$AUdpD$tLGM9WqARv4xgsf6b2zpn6){}y^FLXWHOg?I7
zFvFY1Sb1~oGl*wteqR^lo2<&3))GtzNT%xVg!jJ?1o`LSj20y`)6r#ApgU%{QIIUF
znHp$Eb#*^#-2LK3LW?av11rOCH`YD%BvmD_?K2ljEcq%91}%$SMTlHGlj<$FLG04T
zr6;~mYNdY-B={CDr&3Hzn9{7d?m}YSOsHs&d0FOg(J_@Kt$QBo<`8wCKJ3}>Ka28C
z*Lv^m-t3;;b&5%AHHsg4-C(iZ`Ix}=YfAm#%#BnHM}9@>?dIH?xjOrc>m(e`Z`;Lc
z>AkL~4bp8gUSWErrfXPJf8q3FBT<<%+nud-+G+jjyfcq2L$0N#U8Ee6wWj}Kd&FsF
z3@>;g@3493E0}VV+o9nsmel7`O5NN@jA(zalXs7DH+$7vB_!+Ryqea07B-;_rUt$Y
z9i;aEr}~$JiFoiWnNxlbcleFRey(P(yJMKnu9>6D3J2el|1a?eb^DLXD
z!b_E6!N4`G@10i{V1D45nMP>CE0fPX+9KOCM6t;EUl{O<7S;eMGoC&El-ggakZ!-Z
zjxiJU*mQ*c^m)Aiyw92E9$2wo#n~(!k`=&0KT`6q0@CD$h_H$I&!PdaGsS`Py;{s9
zlw3rkM@ot*8f?lOh|hw!v&7K96k(!?gB^X;*c02$4|dIwA^@l~?v
z^#JPp{i`BFh#=^_fiCn>_i#D*aSm2gKgdO4`!Up^fFYM}lC;J$6QOkM2skdKGJ8Q9
z?2-^|e_0*Q{F8AS{R4znU*Y^*&iFN(E(%9=DX)UqiO909Z&YS;P6XNeQ&*ull;SuM
z6~pm{I}7-UOuW#h3Pn2wM9+7{<0SjEl6p(XxkT9Q6@DmMx-)#{Ju4J1p(^7p3{+>D
zPp<{2NQ6(8663t(Hs~5LMYFVs+M9+oAe&V#t$6*a*!md8{vc`m5TE+XpU-c&BOyS!>wj42ImYt5Mo{l6$
z=9@*@qDciXG^&F}0@l`unA%^Ms=A)(!0n~*IA8UQxN%8A>U~L00chxEh&4gM*tQI~
z=Q<%TT;JXgJBa*_-=PHkHA54A(ONM85{*w~Cb*9GMH1U$(Nr?gitOUcB{B=7@mf5e
z=K(ZxyB5JL>D&>AZs3A}O2_M6IB;d&;e)rv)KZDK4M&^r{H{g#eEOB;==?mVW~JWN
zbAt^c^cSh@Yf5ufs6frqadT@#cZ`snF8Ut)0BM~bT@FD6r{1T#*rOw1V!H2{8pW(q
zSSG_mvENt=QsrYMr(}_==whdT&~_){{{BPDkDx-XEF8`Z1>vqBA%(zb9kLf1ktkTa
zj;!8rhjrD<9zFqt&zB|#twCC_XID1~8Cf{QbQ_7QVB(q(wumbO#vIk+ao}kw@+n0I
zo2Mj^8qjs~ID)5PjcPEQzVm#`V~o=RIiDTR;2O&`_s7l22eG)-1*iL9tM9~Bn0_ua
z|EYBy;6llxPIByUAc%ERI-jaBJdLUSmNlFr}t0mD>NQk+i;dS;G-EA
z;1{Qs_K|VD5i85CgaUID-Ne1ja@;@^{XocY9o)^Xz@~Csy`WhnRTbv2!P3w-jc3MZ
z?GRz*#B`u`Ko-q!EnxI-is3NIA#i1flR+)lC>{ebwNENpuK6P|<4)^!tclf2`1De0@(N%T5^M37mWd*wfu=!T_q|v6V?kt;H!2<+)
zG*Ned5D
z*O=BFDHn`~Nza|BtCI40|69?Nl!O8=+h>aMbeh#9ujCK=W2Yo*Bz4VJDmAF%e5Nrg
zb;YVx?J>DI5604KcKb+VDb!2ZIX4D{y2SDoisM__ZBC3h@@Y<$Wr)u@eA9J2&FlB~
zKMgUUZF{sA6z5W;OtA3K`!)vf2Qg}0=rO`^+$LB@&n)WM~233HjK
z_UDtYi)d2clCq8CZ5kh(E6<}B32*>k@_9*$dBj}(n=MPHvBoTda+Z<1;KDXO{lPd3
zqB`RNO^7bl)T)I5=L&@P_SHU0reOuCJ=R?m@XAp(n}~{+7t{u-8K0kYhG|7RuIO$H)4R&PXIE#QjB2i$%9?4k0}J?0v8TzDoqmoS*IMSFD>_=BzYJM#G9xNd`f
z)XSa6?%4QY^-$oi9@O)ApPh^kw49dD`_sWeH>Weokx+YqNutFBF=p7
zUk^FLt?752nc!zAzQ!I1#l!#m{qV_Z6n>u{7|=w^f5aiDd;5am!mFRpZ5>3gZqH09
z++ThuMrKWwHq~tE?cTFaTC@_*<6GOi{5>Qw0GJlIOH({IYA@^o9h*-X#}ht^Cdv$W
z*27)zrC%}J3?362k8vZ}T=s~n=s;pJcw|}ky+AlU%dY)ga}~rrjwb5d`^N)E&3cD}
z&vMPFG+9FgD-z2}ef%D3$W$z!FTi6VR)3z7LJTu7ycl!JJh=p=Z&r|04Rs*;)7sde
z7h}8zB6K|~pn~zU9DUg?KC&N#oR3B>m-8ryC7()WydIDcDd$l5dbsCn(De#|bMGH|
zw(Y*CxnTmd&qF3l?eyn{KveuQkrNU7ycZB`5gn4hzsct#J23^yW%axhfb*AWxUGBfsTS
zQdl*}Ok=BLlC~G%3K~83Mk)1c-I*!bD#ix|2MZFM*kmp)8o8RKDHbVE{+6ZEAwe%^
z08vJpm|}vf0>BwVTYLMOhL)lAbEcgu$chF8*d|JSyroTdC4@%0^y&EKTxu?jh=zL}
z;Gi{f2W`ogQDxxw8(SDVyhm`-TA>mOgNSX4mIp7`Qf#_Gb`{MCev&5@PWcm606O6m
zp8jU|6n?PCRtbiA2+<4&ZJ|Kwrl|t4qZyh+BVvO&>(CMb;^NYpi377y<^fiXn(Ps5
zLWoQyZnWI+w3hk}H~ySj
zFy`|o&wb%HuEFt25>^|;Y}X40hZj~A{Tmdf
zt0x#7?Zc*2%=jrSW$U)^U@8G`VDs0RXA{CD6t>?!SAIo~c1O7t25NB6-ytYh|L1-{3_Y+(2`
zaF=Mq@-r>Chv_zbb|Q4frb@f`tv)xB|B0T|PWNMc7Uk-O$v|d-o(_g{bFhKTYH^WY
z>#u&WhMVTkg1P**_WtW5Z!I3*uUUH>;_*6AoB8#!6RJVQX>9L5U8{`Xs@aRhn7K$pULn~s2N^~+
z6>96w#AH1ok8Yr@3i(zEadw~y&-DKowK;W=d(R&JnG*)fiJjXhqZp@7P%$wf^{z8@
zYlu&2H8WZT2SgZ}n2s|GF1fhU)Yvsh$xcL-NajA-2!8$+yH0EonpM``<5-9C^iZ8S}!k6uqazH8eg~
zhh+a=JTgn-V7tk>mb@Cg+!RNOV%)~3)y4J>F_Tnmb)TI_^em3)Wg)jPyc&(<Sz9s6YrN)8955sRVn$>oV?m
z7N5HmJp?%U?%9^Q@
zdw_zXbltj!)$u>IIPPH3)<`WA^nb1aQ~phzcNPCA|3Bb_-Z1zutRMKZ_lGXLuk#LX
z^FH>kpPKa#dP9l5EIVr^FyAvi_Pd^>r|&-QIbd#BOrwq
z24a2}lX=WV%4GA^esR5YO+qzFFRh+4){9khWQx4Q>SV>wx_(92LQBL^B=QoppFc5i`0V^AF
z-W}%%*GMu;KRx88S_+TSQ{B2Grbkd!W?{)YjA;qzAnGCGx%1a#hL$y+fLI#gO5$2r
zmLz+8`k!n5L}ET~LqbCxwF+0cdc5?|cHW%9JicuR^~S6(Z(Y9&9Sk3K9SmRkn>$=n
zz;p0__a)4RkrSQ$d2LbAYAqxyCrhL$J0U9tAmSr;5iOyBCt_2f-61#8s>j>4qo$)$
zKs-TsS?zMldRYK_vp0Sx?TRNsq>8{+eacF}IRP*5?w&8#hFfF5Vw|mRgiQTiknN^<
zxtGUqa1ZO(znSbcwLI+l{}m{DhdUXWn6d9b>QO%Cl(oQBEz2f`mUE~X?hg*fO&4tY
z7MOL&{xZeea{A3w91|{H3spuW`#`VDEtt!|KT6a8QEc7wL;)Pf21HDwOaF$YU^rq(
zo>ZVZn?+u1(e&64ROOKe9n#0D$YwT%*CE%?2>*(`ecPH+`@w0d-9J4|4z3yAoE^eo
z0EYk@SW(sbI(kWis!f*xqDfSId~JFy-;>71P7tAAifbV?L#AtR(vRck2F(U&X=CH+
zcWN9v6>_mT?_7G&yNr+2Fv{23g{>4SsZ4YtcOib8W*NH;`{@+nn$zG){j#JKJ;2Qh
zB&Lv$R`+WRviR-na!1P??c=awT#>8;hE*SRi#&2TTt4rRIwO$R!LTqv>Mj(xngu-F
zFRiUvu;c$us-+D7F?ldI(C<0q0nCjgi@-3A|A7inEqjvvtj%!PK#6b`HfP^A1PW__
zlEL4ZWdx~uG`=are(9O@ELGzRG=6{{<5YX>lWxB*nn%;&3->SvjG%*Z7fvo)z@@RP
zOKpI^tU}Gr-Et}Oiv&gDq6yM++&EieQH*beqIEe49JoZdCj!i>bA~e;EYh>Ho;&B&
z;=7JFgeqAo;y^T+-t*=$l&@7++rb>NEq#GGx_jLdj5XDh`2Z?UCiCbmNjY7netO$B
z8Y?Bis*#S8PivHrydv67_Lt%ss!6_-lpPFaB^fo7ibtk!R_NMPqLCCb-lCD>)QtjG
zkUKTs_*KR?95lL6|aCdh}fB?bWgS!rLAHGxP+$w&dQ&hq9bnm^EyoK&}-o>&HqG^_lH|aXMy4K#7
zsg5h2s#c>yEjcE4i
zsb%>L5b~JwL2{sKH!vlR+ey9)xY&kF;}>xOr%FAY;+}w{*T{~4mD!|c0E)T!(dF7+
z2>a>IwaA0H+??5Zjzzxp`m|YV*`YGIqN#U!o~)s|;Vg<&9^(n1B4xJTS!LRH*>xwA
zQigHIxJnw~3s^;|tSve2ZjKbnAuR7n*$%uGTe$L?I;ok|68OQ_+&D<&dmQ^4Z!H%R
zK~kz0N@njhES-LytE2vk0DDBo2E@k>5g>oX6p>WOCCv_S6O54n1{_nsx2085`Ms
zbmdx7XJplvZRHqbN70p2nk^1TF~Ei-7s81igM47Ss{)!^U97|QSRReyra+qQ6?Yw{
z{<+w#r+`V{EQ-64J;mc!(#-B(a2`G)zgH}Y`qwhL5U774&d%(F$VnN6q`g|mbd924P+$+)jUx>AcR0lG%pOixm@N_0s+VaJS
z7Jz;nlXGb*|Acs{e6Tvgc;b8Tm=sOjmm<WCe8r2O02
znzZ|oYUC5ie^(qL*BtLBuMyBBD99<`It$=rR`@|yHiRA3J#d-**Se?`C>&@Yea;`#_>^I`e;52*3+_bY;Oy=#7^%&qn|jm`D815=43r
z`Rf=M>^i-jzH|Qkw!d~Hb$%Y8h!^ZnRu<+nKIeV^sry4X#=Fmt*CU{DLJ~3n8D!k$
z)S7(pxk!q4N{-Tgf;EpKT~emexf7lVR;
z2W7nV9$ExwLOpx7kXWj73*(*35t6)`()|aCeuzTgBg@;Kz|)4E2>gEW!3xG~kPI0B
zlCj0XX+(MLS%wJ~MRhw2=?a0sH2lpEiJ&(%FlqmZS%rUD-RFq`9Umt8bi}zEuon*s0aS2U6_xkmj8v3&A(oCg9C*}iP&l(sL?5u!q|)Vceq07bYSqgnjC))
zvmBk9E<538Rw)wF9lwk(T}y+x1G89g^vI;c5lG^0113kcOxWO4)j~U(<)n1Q;Au3}
zC3t;4`47V$t1`4^KPcD$@UL(E_|Wv40dDTrkxj@;VfYN2nvIz*sj3Z`s^#$A=K~sMOjEE8wT0fkj6&nql3*0Wo3oO*udS+G!B2^(jq9T&_R?`Vgy;RP-vgz{e+Ps
zzeV<)VE&7yD1=}Xaxh7)_^)?(1$%}kpo!t8mB4N@bN3?~GwOuPm$4&~#!~r_0}Pk}U_O
zrvU2}qu`XZ|H=!Y3^8pC&E}MbkVjG}MA+ah;tC@GT2y8UP&T={yI)#cv1G+n<)t#E
zBO>!gBx{pnkjy$b28I|mCRBXU
zNKtczNH&<^U6bsi22aA^wzVW)i2)U5$BPfwv03yx$Mtjgyn$by})%nuGbl5`f!z3;+7YB_tEemt1QK+F%KrUFI
zWh>waFO8mYXD&9?r$BVM8~ybf**w!z8wOg2(Yu~jK8e@N+dXJ1U>`bU>s!k@v-hRg
zq!`12D}lBx7zV7w+IM|i8Xizfg4ch5_&FtTr#vBT^SHjdrSK!3s^Lx>ZcxOrjY&|+pU*laT_@{`0>5#b~@ok1w-iL
zXUSX1j*3~G`Weu)B8FTQT?#b|mUmeOCRMUpGJ_%`6?s%dw%Ii;3!Wn8GOlz=^)Goj
zd8@NqpT-qw-vfFI*@`u4B{MzUtiG?kUu&nlIK}ApqE=#wCc?y7Gnc0r_t0U^jly-90Q21#jr;+8Le#&Ck#CI&V`lF#NL6
zP1i}^eZ&P*n>g_CP<<|2K&UP}&2XSi@d%ZpjnzJDMlPV`N;be4Vxw7rtopiyLC*B7
zy$dE%4To;WGW^gjm@;_Exl+rlYOH?XdR9&^9t}mKDvG%@jb4T3mbu(m7=aqQe($Y7
zEV1}#Pv$k1Stq&5eY>@5Yk0(2CrNGz5FQj22_Jvo6YX3RI)+%Xj>nFvO3a*1AF<3B
z=~18FB{(CJixCW|YWzZp6LL?($CTLNHbW&1tvs`zWD;!;I3>hRd^;PnR9_&jooFi1
zk!#nLtHI(Re7qcL`EbvJTbc6Rxd0CnNjeJRso*8
z5wk|s^BGT~dbM%dxYOs&IDvlGpD||$9>YQ_(EdfsCVl_P^G~TIlt9>~vc3T&sI@$d
zt*v}!_B|$L>15zz37%EdW7^mKs>@L8QcshV7aKP7${y8Ah4{AyW}(Iw$d=0Au6%M6
ztwZk+g(76gR^?JS$}+YOO92`A#fDg3{#B{u*J^&2i08#)B!qM1k=INe{D(;4%U&lW
zggN|KnO{>m0kL09oVbi8eA3Zhf99U-
zkqJ*0^@Gw&xhS7pTS7p^u@p>&8Lpn40=KcZ>})&!WoH4pz=@I`yfd1*(}%YZw&7m{1eg3k3e%bOmztO%ELIRZkyor7Oks0~l&H+#Si@>#E>N4O0s1dGR_H~ds
zYW`Wzeir=y;etRX0jZ**!Zmo5D7XgLjtE}#e!OlPg7gBl2hD(6?ZDe+DsQ6ik3OZ5
z07L1G?f)hkq+@_Q<3pT(AUe9*+bGE+1uz@YAOidY+;^klV|3sp%jCxF`hP6~hfDus
z6}JVvJOOxQHsThuk7EO0X(0IjyOz8j_k8X>Iyz$0=NKV8Cjl77do6*G761nVEO)ox+*8-!a>=a>-RG;
zNnMZGfQx(V?WFgafyI0J(tOnRSJYnko2S;#V>d4iTnnp`IAxp=6qdZyVO7?gS4OBI
zRc96ThkqV~-|sO#Kbtzd{mqZMX`;dzi*Rg-R@H)W6U5I?J%u(kS-*rm9KzMZHH*`W
zR&))cvZaP8(flTdx`U00kb)+a8TB7iC|MyTI+Yl{98p6Ki?h35l>C5iD<}ViN4G)&
z6}lw4Vnpbvr3azvsD*-V2@>qD&aA-PEt|-#uZgpz#-_Hm01kjA&NI;k`rGs`%p=C3
zU1ioE&zO<{#8udADe&KMKgx}mAk`b2%)8o%*MGW-qQ-9*rD`5T!IQV3j5L$|k)G;v
zqH5Bclvfbf%d}>K6-j{8^Pky=@qCuOe^lZgGqqmvgC!Tp4x$?3JHHv-8Jh
zO`JOkJIhE}csV9Ia#?gbcHR2FKmI7XV(Xb#pL#Cgd*JReU9k_FB$z2My2i-TYKE`%
z9sGznHw%3D@7Je1OYg1wLr){B*cwtWjlWJsvQ6~ftLI_kM&X8c>O@YSneV+aD{f#~
zp@Gyfwwh9cSq!RW5mF|&aLs_?j|_Dqoh+S1xY)_GReTIt9V2R@CJbY$nUiPwrPuHE
zb>~XiaF{i^u}6fo1#*8JP}4eA)Vu}7{)f0VH>=nXhQIKW}}wj(^ln0
zF=5U6>iK=8m}bY&*_m($#5YFTe7wGs?gM|Ec}yLBmt-yn{z1%jwhMc~1}M20&zZ0&
zwmD0o)oN3jGqYY8Cl=itBEMGXw)QT_v#Fa#n52q$}#UYmiU0=pYxRjEV&sw
znhjC05rVrM-SbCr4T(=^Zl2~w9;Llh{l#fc>e?&?4+<4&e|nY(N!@PtxYbHAmPOh8+3X_R#gGatb1Pf
zE6R!r`x_s_WnDNhhQ<%|pM;hgA*=YyhgkWk(`hkf1_9Ye*Sy-8aulH!?q$E8jo?{q3r}f&U~wV1ibWf~clG26ZlD$-sA+
zn1(8CEL6J|&eYC$f>+JPGM*dP6yG(2hJT66NC-voGOXObKLCK+5&wk_S5f0gWnYKU&VrQN9*BY&fkjV5InVHe3o
zss>aU$4AcPn>pxWfvr-~0l5X^UDi1vq5ew;6c{-~w(s_Zt)nPgEi3be&;hZ8O;c!L
z&%FGYzF+4eu3v0>$v4%dK8P|-$Du{%u<5Xwy@hVt0pACH@9!5;UQ4$N3sKOjUZq<8
zd^HPvDRw!X+=BFKr*`{vl
z(L?^A(vx|4P86mb@8iIqAadVbuh*=+BJ^3Ni}#P9AJe##F{;bq5+t0wZ$_hIL@K$Xy$rxK7>cfcR9|5Of)H;%KFMo$mO
zaEl$4NQ^;~Pk&
z5&D@xRtD(0}#(M0}Nl05kRy?_n~q9*wQguC8rvmco?nNL226
zsJXI`P*MHnoOOd+dtUZDOrp**)fI2H8fcxEEP1_sNDiQbP-2y#FY!$L{d3-Pd{ch=
z9M1!R4=C{~+3`g0bl$9uXKECo8`S?wm2&v4q8(9ye1U5x$?8~NN5jR_am_~3f5`%T
zU!2~pME?U3ear_!h&Qv!u6HL(ZOU%po9_vQSf2F^20x+?%0jo8#*gFLKGqJ_P^08#Km*9JuB_4@5uBoCw4
zY7?k6?4}1CrVFE)+W<`AAJ(}aGLrLx&I9f{NNk@x3u`WAm!M7J_#9#1m^bACP$JPD
zk3Xtk(taObCaRhq)@MmDqIpQx>Z1xaxP9!U3d+%)Inq@)eDiuqoB
z3Za(nI)6$s*uN-yaH)~DI_)nz!N?{CPgwm?g?;Xwg
z;vqwjU};(pa&ZoCjU{BlQ0dAAW9do6QUf!`9-?WRK2Spq_b$@?wESiQ_RUGI!1?Ft
zA>)a8sfj?r=qyke<%lJQDjW10J&D&t^eud0=mZh*DZQTZ;}#!&I1W$0tZA9QqIX^X
z9EtPP6SMW*{UQr{47!8k?b3>q>*EaleRfy>%f0dDP1%O^s&BN?4UEqv&Y|T!TJ$6u
z!aZz-M8zu26@{hyvW99SH@gl+A}$i*MlNT+<8akFHWq(jLlA8{E)X;QKgBNpd?sNg8mG;W_>*f3p=mj2c
z58f1kOm6AF3&u97_6LwpRl4p;K>V?Qv6=lj4Zu{CJjSGeR@;YHg?-Gy8nj2Ht=!Hv
zprnn;L?~elqsZ=z_i#^H
z<`i8+2ufjNPtBl}N`tY~&DTJXGlfcRMS&>1C8{M+BaHQvpK4_(wG059VqXYEf!e(B
zE{pQFR_-0NT_HRed7#TM>M~)hb)AO^P2O
z!a5Beb`6e5vk%OK2)*<~`n1TBMoA?IrApP#9^Nh<5xGLgyi<(ZR?wlfA2hBOH4gcpAHf_?)TGDQe
zC`u>`-0Mf+ZAHk&FhPU~?He$K$;8<_uQRHIz6
zMy5DR;Y6QIN#~sXXPo`}zz0Z(ztlj1rttmUr<(3#IU1cn1!tMnNAahPzYRh(d$rB$
zy_i9za_3*Pn*wApWdZzzZrY4v%hJk=aEX!DEOPn#?uY`(1{W1|hPrfOO1gm=`&J
zlHMA&MceI54m#1JTm0MiVDyEHcxeI&iB=sWVzt$&kqUO{>iW3m!n5F%Pmo4Z^vJ*xq-^cPk1?oN
zwljD#Rj-CPtzE;*j$f6+-ats>w~4^CaAZ+=$M2?(H`C=Bl<&WQS0O)pdPfbeO6N^#
zP0aZkU>h^5$-LTG{z5;VG$;>3cmYV$U@^K5=ODKJ0G{!rqYt6C9iGyB0wHW*wdS|b
z-vJ*(oX5f@*sPT?f$pB)FK6-J4zwh2Z2;ow=?0*Y?WAS)6Pzw#`$o~fUo
z(Uh1oG6xyT3piy+GXnc}2b#5GsT9p$GFYoHk0Fu66=zXTOQGGSFRXzEuyo$~pa!I8
ziF*=drPLslo%1UHgK~*g{<6v7jm};iC7H_Vh_3oaW@&hJx-RWkEw1&2J&Q*F5tF=w
z1DJz&bNalI14}!27a-6;#&)YjMtOvPZk(l8C(PJU+X>Sr7@d_>vDC+ce8Rga+oz_}
z|J6PI^TDrPr?T;0{4<{N6fuqKR&K3nBV6+psn$2CKT>}F5{@!6j;pC^8t2iHCM_%&
z#^2OJ%a;nL&j`+^o^g&HdEaVoXVmb9k
zN*X5U+guY@GY?Oj$0L`e`l1Rmq1SP>;CT_0(M=9^F4lmQsA7|_z#2M3(nh6ptH2&
z9}fvEcCb}9LcH_0Ox<&EQP^ptj{9KCL|ar
zM3fOs`@E|P_JDPTw#e1Ns%-Z#2Wk%D*apn7Be`$wvSY8uij
zD7KeQ->VG$A^off*PkKmtCgYHXC{Dg?|aWu3xD2QBUiUAa+TN8qqHN+-w`D4@X
zMUxXHTAW2F!PsUvENrCwK_}1Bne=c|c$&6_-s`rO)4GHW0vUkFl}taVooSNN(LnUSU9>rT
zP*bR*Y*Ok!kwBCcImdplgDAWkfnlj#AEvoJe6$sK(AcciZa{-CTtxdSViUqSJoejc
zE_)9RKclg)EcrOnU=zcylXLf0zdE3GYqZ}D@c>@9
z<#8N#4$Jr_racI*-8j}$c5Lm0CXkZb$$@F&+I#*&8?>t2Tn(Ijiw!s2VV$A5yw
zi7U25lA}#YES5@;qS7s2l({zyrTq>El$97wvphP?#Eg7Zn`AaRZa}4Q&YH10
zGP#bUYxs=Ac>DIv;~1G-mw!%ZOQto-!za4%b|
zsk@(4zIYUL3ZdABh>2|2fD(Ha4N`2Tusbe;C!TxY;AmvcOIX{`HL<+R&tPwVgM%Hh
z0}Uu8ljq?g1zMnlb@WM5RUNPbh(74yz$~XAL;VRSx)Hk6fe*q=s!T
zb!G@e{HpVd0Q!vS(UeK#xZWq=o&caur@a(PXTdAS$xUiAyyPxuXNVR2`f%nQ&Gt)h
zG9T(H{eRF;^>Kh3Ukn)(s}Zj*6VyH_G>iQef;qK>W<&TcL5pabk>dWCs%gg^(erl(
zZTCAos8=UR1q}(zGY#~0=rV1d5Lbj%35<`K!(rtW%TXov12C>)JtqW-f
zla!p!-TR1zSoVevS<+(3U`;#Tw?e9u4b}(&^B{#$M|1yEe}oc0c@jOT2;8zCtim3e
zo~FINQPilTZ*VDk1fKrQUv~@qo>+MnO2z+VOe4VL+HB__maFAdNh!;4$eGdZdI!#-
zhkgoTM^Lcx9G##&5uZCK&(08)Y|DI{Q`*hBs$t9AYom@>*`aBkfY_@rp$r$husnG5
z)|m_%c=jDc1L&FK2);8(d%x85le#u&B_7y4Nx-T;!Sr%hWB*5W3Z6Slpgi1Cq^Lw7
zlWD+KB}{HwMq2s!J;Z;w2+dMgWA~ue#>zD6<1Fu)ljGlmlnr9;Ulg}c&VTgF=Z(&!
zGbsmECDKaVmWhDN)#3(8eM`@jji42vSBQ;829!d5<5!5bX=IiPWYn~l(k#lGpoG%K
zucvbAcH%~M7L>Y%$If>CVafLLmCWw8Et&3{vJ(YWsZN@SW(sRo2Gp?vHL->rvLB?5
z^Nl+m87XrhH;rXFe-+WS4Zm-$X!!RaRHHfmxh42EXD`-MkS4!SMZo~2l;d1t?W!D5
zO(*V=`&W*vm=xrB$XqjHs7H3_4iFJTzu3ptp&y5SYR~d65>u!!hI3W=bNF&WT8`NA
z+D+r8Eon~)U*qRcPFc?FsG#X|%Mo8I)t5A|Y%3vfvtf(9+CJEXK~8odkvDp
z6ELQ(v#8ri$z8!qjL9Z3(8bve^!e{{Z?9WR^4}-H6P4g7%!chUO20Pu~&px9;
z1e>*J*8^MgB_0?VW#{37ez3-^Pwdx3u>{2U6qzIyR)WJko?FaS!`&viGp2!vGikHj
zXf7IA$}`AWSvyi8t^cZj5K}+b%fJgL{gqTpZnoy)6B*$g=^dvkyC!^!D5UPgUnsSI
z>yQIe-;VplRsLcdjb_+9y#JB9U>&;VHb8W3u}ZI~nr`^8oKNlNOI~?8I}UwrAM(iE
zIW>UM?{E5GnVd4*y>J_#y&1S9piso5;l4lo*5370&zZK#^o)z15>n?KcTv^IOjBh)
z^pA8&D#z9{9a9Uue(>?6osDw+X-;YveokuPc0Y1t{EKd8R19IqomOHz+8iuXHEhma
zz$wPxEDkMLSDOcBK8?8QBO2cGYyK)$WLPk1EVR8ngF&Z;%jpxfeFejyvqJ6xcWM91
zeic%tw|NTcEVJmE_HpzokWF1}-4PXWTqH3Na9_K*M-exAt-YUehHOhgB-Wd~PU=S9
z!$uD?>D_KX=wa9EW?K63^}a_0<+&B1ll%dQK6nG~cX&9`?iuHp5*+x5BKY+^9kDq!
z<=KzR{y&29L0;f%T2ii1=h0e@M)j1)n;@{(xDd3l{(s}g1)x*B%sKZ92gVg3p+WNL
zS<29}Tx6HWrX%_2nf4vJ^a$)bx|CsnlLbJG*aRjafYA^K^DbBHxz+FCq6u3Mx;R%B
zp4eG=k7Fd?_#OQ%Gq3+Q8wBv1V;HonF97)>NekUb)cd@|v+lKnA?vqtN0-QE+Xn~gO4`JeEf#cO6>!|&MT31)X4~3s86p~W{0)zJUG}E*fbaf5i
zV`f8OET0z`y;E-IY#p`md0H@onYcREGC5Lk&hWIR88U1nbzg!hmT&ym^t!18u~|7T
z+j(=dfb0s0$jDI6+ehtRBEO98g`<#zc)f|*{89zg3>HNOK4}wgZy3CKdf6UF19g$1
zpAHmZ+{H(YOHw^+
z((mwXa4{;&taKc5M0Km}!v{P@o6$6ZgKJ~LOSuwAM&wTH$3cno602!?dAXmT8Rq
zGnfWtL!OdI%S6|d|1P7TbCavaaLHkvM%ZDu)UZ&{h$1UonB;3}{N-<-wc1cWr01EZ
zWl-4k#Wo0)&$&^W@YGzP-^4ABL0@EKCH;;4+~_IVVC6|yYKOyvB6ty&)gA^6OBa44
zP~&X27q|v;6o~`A%se*(mvue3yj4Zs=N{D2bXkh|vV^E~h63-`xADYd-
z@qMj+1aq@Tjhcz#uY5h<`Z}7*pzU>B#p88-yU=>E-f?nvmY+&3i=IUJL-xOJD)?b+
zGD_8K)KaFTO2ur8QCgQ`@{mikEs`VWw7;6==v6>MZ~D|j$IuYa1z}Xy)YSbsPo#|O
zsus+CI0ewrF{*MLgmH`O>-)=1_69xOV`u9LosIOfk*I?+=+$89#+@Z<;qH%u^o{HaE`SPE
zhE7*kq1E%_nI7GtRT^bF4q7rfnASW6+lGlwGiH9>70MjIq`Q4|Jh}A4^&-FM1KBsG
z?ef`B=o%UA|290n@e!&%x%b*xhSDGS45Ww3f_<}YRc2`&3?rp-i1yvXY}914Ez?Hn
z{FZ}d7LhSMzB3v1G!O2*5O*03FZn)IR=KJ0n_w1(4e%sx-V
zV>Xeu9a!iU+A0pR!KSD;-V>OljMEp7s7myt)~$kOC)JP=(xW>6Xc?~ID5f@_mO=%o
zQSDqNbghB7d1`==#2E0~t7nYM?x6*a4FZhJ%nkMR!!uXxIZElYQ3fKju?8gIUkd4J
zuAu9_uPMG2?d;!aqimY=7#Z($jrA?f%~QccFfr2x9Uj1v*(-$P^};gXX4&4L!zWhF
zw!TulV~s|ZUxJ+~sxa!gFtvaY^fvV=EFmcsEPODTz58r^zTXE?cD^6m`&fMN9i)(`
z_R)7BBn@73O#`E*QCpK!OIFte!6D!}G-}goYQc3)LzZ}3UZ2YzBl{E1U^6WJ$_0ID
z-@fi#W|`J0l4g#*{)tRMljc|MVN@_))ezRQJjo?r)l653O
zQ+xu0|MeYCNjd7|NMxv3t7259ed2!WSDmNXy|0T!4LX06$hT=-2o_YDf>_hVWE#KA
zmf@vGscAZq;Vv)}6-OqSYS;Z@_Eqd|ni&(XX6D#2S{Ob|#kU*kbvW~{JZmyUE5TqEA
z+*or%{hj7JSVezsI&48STQ5Tct#O}5O&>MyCTJ5>IJ@gLIj@g5DCP&v%5PH<&d~bl
zZlX3nhzcLo*P2I6=cf>v`~6sq{bZ*<;L^0h^SABdTHU?Ich1D0m2ga3qkJ^B87)PV
z%NqLnyVHPP$d%lp<7Cxb=CQJ6bORq^N``+SRK?;dP~f$MQy+xnmp>Eg@ncy+V)VTE
z{d)B2)WR)gE?#*aaCzcz#t-TOT9IYGj^SKL*n`h
zR|s;H!rw!kaqzr>z(>mSE83p!;X<^nA;56r^FF0`0q=h2Pzle(VxFbA2zj0nI(1$S
z3!41DNhBi@D2_y&2fjh_eCp#jG)5rlKo;WPCB9qCg?j)EXqfBM`y!_8Ab01mG|$T4
z|D|=JZLf8f+xLoqzzb`f?@YljlPBI6aMxI&UwnInH0%FY^ZINH65$zmXFdZB6(@gs
zZq64V@aZDm>3aV@Sq*+->FfnIA$N5mkVI+XM-4#B=6dij>DfJ?-F*dEC@ldymCM6~
zi16u~lnaI5p}g_7z{cZbsfVE#O+K|Sg9RZ8yhKg4IV>43{A)vyPSf~MP_!VPgu=8(
zNif+!e*NLKDXImBSHnzG1-ebrH!-H@|Gs}~)IN|51kHkH_jhdlB;5&Ez+=YegdYBsE$y~9$Q+_e_C
zbqrTbye(sgz?SV@xteqeOk>cc>~;tt<>Y3Zr|hv)rUxZVY
z{G_L=(31S;RH9K2RhljN5>~tB)p_0fJaDRG;nlzTFwdm3F^Ugn@B6-9`=EUh#M|{=
zxH-H67b|f*SOR-g4r{i*qHC4*-*oGr7T!tOnGbRV&7NmguGgiwu>nv;IEgMKWS*#I
z+gB!$tm`sZ(MM?tP(qk|DM
z5cIf23Uvu58aVOHGjxVcZlwntN4H9&&9*~sDdNPzs1Hx4a~9j`3Ce;jaPu|Q4v5)H
z;On}CaYKo2!T<5K$*F63gJCv+umi>`<+iW?164zN*ICGR$Dyd(sUU9K#DmvKzWTiM
z?BpiDp(xe7aisZdP(|{vQHzx^>$TJYutd>J%V9=^99B_%lRd4d!z{{Dm*bc
zV0WQN&YJnMCd6#m*_&0k}&L6e}1xCZ)&
z^|P`JR#wgIt>d@6V(3$mgFP*A9p%V+o0Ri%DFiy|xrbFSsfwl}`b>Zj;)n&isKqA>
zdFxT^dD)pz3Hqeb@(fuz8S+vj>Z33I9X&z$DoLh^N$KdK2)iaOF_8REG0dz|TTNd;
z%9&c?cekbx9rH<%%1wsg?^6e8@HS+9zp0GqCn#CyQcD&T`MRbpL2P7cY~Fb4^fk?mskX(Sl`XVlW68vR8G6Be2gMZ(C~{
zB&yBK%22$L&EItFs4e@4awON6j*2A1
zXG_~x-OCl{l}4j8I!FKTmv@fWo>&V7wx4=I49aqKe#1hsNq~%0A*32xu
zBX66q<`=Ydbp92jY;OU5s@Gkg+&tHX2
zvykp}@jC0bQf)6U-*L(3e9jc*GT%$g9k8=PU8goM#v+x@%z#N88
zm0PhXFPHgQmRrw-vZ*JBmN@(oR#e|BSC!rVvh%aXHsQb6+Sjfb3FqH4z4Y`1vlP3_)1yI_(G
zFK9ASol=(MEP9j@O_wL)Jj2t|Hkld#f90BX*{?;MG1OH05RB$of3C&(p-C|HTOaw`_q(VSTRLjP;=RzQa&p`@3Z`(ns3L~`JaAh5OXd}+fz
z*t1ISXv_!9jwRYo%OzPh`YWlkkwCpspDlU~`}FTq-||9N$jtW_=P@v_3B7ZvXbkw5
z&0)EI(?+rL@_^9N_X{apiYKeQUwp$I=G?plp@7F{vJXxvP)pWSql?m~ZQ0-gX};4Q7|ErJYcaF)zY3@i
zCF*;!9C>7qOH=91-KRvaMsEu>1t$vTHL=AVMj7pb6^b4bMS_hh|3MM9=|$P8C$-OK
zf@;XE;fQ{tCR7?O&s$evjes$m{VWWP$I<$_kjYeqZy50RIM=!>4tQ3Yos{P|2!umH
zD`55*L_mW(V{Bk$Z=W@0Ax0>frsC+PY29>AZqv9#I&l#rf`deMtHZu$KW9tmhhp0w
z`nm1w;yyim<2qn!>T+z5ChY+!?cY;sjb9iJQ212GGCBQd?}!P_uD!%Zn79A!+WXs|
zs-c|1N$fZ}l99!&U$?;LJ!rr-3%@?>_`N|koS>_jSy)K&ME{?b5)=s_ApjfIOW7?K
z0H3ZQUevI5cfWl=YDeDW+40$l=kXU
zwzdWeGFG@eop05>zT^c-=q;DED^Zh&r@Yu)RA^
z2-z-9c+huYIt=1;_phc{QCQHy-}~}Rv%#2!qLa-Z-0nK&-ARqHTK_?flze0#Anm=t
z6CS?*!nz&PJ1fUD2ooLF=LmP@n91!$LBE$tx_0
z-OysLmA}Op-1wha42>{}+MjM@IXaoy2(1-#c+Wfe^^UJOo<@OTo<=VK5AOy_Fi^q`&H!j%B20eoVE63sn_#Y@n=lY1R0jLmp(({NjplwNG8oyS|@QlvdO@8oqTK
zZ4Gu!)pxxB+06eaSoybX*yov!_@2H=M&Q%ghqA6;F8&9jg60MXYnPlL^GoPB-TUe{
z&f{5w@y>3ElvUPWmyI(RD7qdPcLdV5YjmMP+Gy%*LIZp?FD
z_f+#D1j#CZ*!O8xBl&=fwAI@+q$Xf5Kp2u0f4S0g`=2pnKbT>5s^0pIrFt?T?3(3G
zIoJCx+VA#9jmzVVrCSx<0gSt567qsfePZ;wgQd8J#~?o{d4e$IytF&Fuo$NfNtv?G
zcH@ohgWIzc#^@A(5i4KVnC@Rb$sG0?mwk7o0@sw_oF+UJqMk_#ni^*-*L3IR-l3_F
ztTL(ve=*At>AU*Q`jR{SZr>&zwF82+t6vfBO5=+Zqtqt77sW7C85Hl9mLKv~J&!~#
zdY9gk8)#lR2zFSCb_uKCY&iN8^PJYSm_Wn%u@=>ECz(vuSr{kFMEjLRvZ`O`$yCvc
z?17e<*JD3SZr~X~r`M}@wPBZ>ZW=tD{?)bpvG)dgtRpHE9fn3`@}BsN-$1q&^Go=x@mPyO=E}ZG<>f6m@6wQNU(gQ_(?ZaSJ-O&W(`e@RzMEFV|?h>LTfvZ
zsg{P#aUqR7nW`E&*fqEYSp&8=F$gU3wXm9K>v7Yh--6vzI7?-Lik^UNQSjGS2R+`2sT3xg$Z}Emk$$>;e7T&Fbt^-LZ#0ZR|70TE!7;A&ZQVT%
zWcjW5C+70#yc9ROdGTyQ;i+kTxPcSp0ucm!ed0~BUpUv<6eSS5%eMw18g23Lef0Z9TUj}v(S5!nDy7k8rEC3gF
zdWdgFisWVa{vD^HuKvdE;v_=cDWNq8l*1`YFcVjI`qRKj!0q=V#qQ}c`}B1S)0SN*
zmvpxAACz~pZ;X_cP3Wg?scm24G=%U6LF7?^g(3r&@af^;#W9#83rPy9hOL
zjijO^=+t5dr>g7G5NogdqWjBDA(|x}f;H^Ax!KfA+HNuzSLqHIUrkh~e^1urP{N|=
zlHrdJjjMjs#2O}OIlja9LYdCnt&Gh`zvDn()(bn+;3QIf&QRcln8%{Jvdtg1a6XYM9r54k
zn3J5Ww9h(jgD_(8!Jl}^1B@1n``qG{nKkcHBGMrE08H<8(8@rU^`p@}9*-kZWK!iL
z{GUj?8F`kjZ*synK5zydy?52W{*+=SmJ3T
zIT0I?NA8ckV_%+916*t9!Y_PSM|a#^F?9LVrp1PjItB8EiBzK;A>8h*1}xo9jd1a;
zZkc{xn@!Ev^msPRs(ytuZ@~GlFNqs*B}V)diAY-|yl?X&;mlZ^Z@vR9!g(}TZosil
zKMaiJ$;fUq^d)BM|L9+|S!txI^}KJC6tC1)d+56HFr}8zzZVO(8_w+@nPM?Lo&JH*
z?`O#&xo78F@>vUm^;eij~k3aGr@dECw1MifgZ$LZdBoN%d^YfQ3@zSFE
z>7E{c=#j3#C*?P~z+*f9w@Z=NkyNGM
zFQ%-YQG_2z^PYEU9>?;Y$69{-+-ULIRXu4*@kX3ekUYY_ZIV2Pv}h2Bybnq5FnS^Xz!c?om7LdEC9VOuK>=H`gpiNl!Fyn)
zlWG3D=hK_yt(SRzQ^9;wbnqb>;i1i2BWh?b!RhsD-lqCrUIS)Hg`iRAIPUT99aby<
zD9K$i^)3UlIzTT3>Cj2j@%t~Mk8Jcx*Kx7Tt@QAYTZOKHslJXQwzaO#T>Np>(?i1_
z&bVjoax~)u9G0A~90&B*a87CAtmr7!KFB3n{h)xF=hDmeV
z@XbM(IivS~tSXy?D%`)vt*!}7$ySd3w@2b&=|Cs|HpyUs{b8Z1A$Z>(dXU>Qwvg8|
z#=+v3Tx0x*$YW)XtA@^H!Yeqrf0>-!78n~&A}}mKOyY9xk3k%`t$WsWU}@C-HBCO9
z*n(Y1QMIx_BTS7-ZcbRau8>ihEM+ZUBNBYG$C{|*qExm_9g{3XyvjS(5$B(?WB*A;
zNi7n2_iNBPaew6(B{_+*I
zc|Ol0zB?K|2oCp8yAL*MF!W6_;IfKPvqaa!RE(4JD%aa_YVB&H-PHkY-X>R!oZZfP
zhKJ$Vv>~@-bsFN`he_w(`$D5l(mETfW^8C0Ln(R!(neH_h^IF^)wyVFu~H_g$aoYU
z^(Ye3;*}?%o?p^%G39V$ezIvnes*Rm$hzpVjoK@L;VSoBx#3)^l@}Is;xjby<`Ii4
zRSKn|F^j@~NBh!N5LrdTE#-AZn-k=j+MmkDTph@2o<@i*%NA&iu?B{5whhzv^bS8T
ztS-KhLXz(}^aa0YI+06+CAk(eidPWYRd6qt50RI}n$ZuZaGUof)Mc=li813?^qVkh
z;ExyX4##Hw-gmvbzhC4F2tG9zh*?+|n?a&WrGYvwSvL~VInQTC72_A{PLcVMuNhXx
z&{#%L>OOcXfz-_^y9bK%m0tAuZ(K)57?FtrIGHCMv(4q5kZhHX~yVIk?scRPH7kolP+l-4bmVn
z>4xX#|32@>=NspY4bJYJ{jU4U@Cyic&~id~3{$JnMgw4u{`pbswVe7#LKH16h&n%5
zH3N&pF3YmfEyIqBt~=xj6M0)tuhD6%@l_yyteZMWQm?4in*RG|nMh%|MNHbC@g^U0
zIQ8i}Tv{&@-iz>2EEvI_*kpg*TBmq|I(5_%Pd_{c4}oM+rJ5STG75H&m+pdh(#P1hPtK{bRbW>gKulWb#s?NrOFo>^U
zExxtgJJwR1Y--68=lmRmPj`|YIFH8tHRNj2yQ^M(hF-L`xW;xNmyvCAqVD_C_1Q8u4B
zEX53qFms)dyre{4+6mNOyGSK7q*47@6&1BGemu;$zx_23k3mqd_StG52C6YlX<*7{W}wyFn5HTTDuxOBy}R4;x0xSx6DQLUX)+Q
zHZ1jKR*y@4_Vzgc6aTxT%DFJplB2~|4NQ5P0Y@4VE5=^gS_y(*BN-axtW^oqlqGc)
zgZ&j(?suw#{Gi8}zh!2>DLZ+=e}x)@Az?Xwe7K19vI$lgLu0)NdLIEky@#6Z|>
z$2z8GQ8l)Q9M9@Ec3it-Y`QDoWuxbH??mi&kJlHF`GWCnmiS^cFr-btDuk%ZR`(I5h9Nk2dtG?k?
z>QZU*ezFN7);Hs(_NVR*xeXh)FTCvZ(Dy<1#jmsC2Bsb7N#}JkTh}Z3l+_ckdB~@v3ix)FQusb@Ca$f
zvP=G8?ZeWMB_FJ3y*u~uGXh=4C=Mmjp9Z+>Q`7Ok*YpPIpRtnMKYT@2uq6sc^0O8$
z_y=aSh#%#j+n5!;``r~NL#f;LAlL|D3zINFkD98cS<*i+x9M4Zq%tAJ4r3>ZDwAF+
z5uaT(!sCmZx3rAAqApm=V4I*XzOlwA4ME#B`HSeM{?qt`LyiRt>lI>-_xe!MK0!;<#1;dNa7G
z;&4{fW|9FM^Egf!|4WGH{s<_~SXf+aqrZXXD9$98=6FX!?*Z*yL-}#=>6+R@uV^m%
z*$Vqy7jpk=;{MgMal|HLRo!d#Ea&PUumWTJ=h@I2WqeO3;IP_Jc-leayFrlq$M`-f
zpxYzh%A=;^ssF8i!qchoyWZ?`V~lkw
zy89{jb~c*`$#LrNaXI(S#lx4ABu4(eJ>Uv{T=ojV!Sb#cuJ2$}WOJKZ8}WD}@wEGl
zGDJ`R6|1CiUhu02dy09(JFiyxHm@#vn8Z&o{Ev72W2x`zsP$Vv`-zx=xKYDp}w62iv^r+4U06b5jhIfcp0&MQ$0_TxJh!rSiy+px_js0mb$u#w3D$YL{xY&=Rz^_YqX*aF4uU>EhZ{_t%z&s;%fBN?6G@EF*
z2^X#kJMzwwV(cqq^>phJomG7R`Z+0arP|!
zo48Mx=q>+Ly7~^(nL&jtDvO0`riETZ0jJ=N%;o6qYG?WI#GD`@RzkNl7w76
zcGG*akiu(q>+C++^Bl2r**mf7(Rm^GXCY*!&Uu6ydiZqd;7Z`1Xj0sQ6ANAWRGeWn
z8Pr~c8xzUYZ3d-CO?+C5%@^ba-;q5g;>SnESC)}>?RDvo#3MRG&6W4^W01;%P2VI)
zyX^lBUVFT^c!V4KB|IHy8|`TOpJ77JZnQlw*^GBjqb3};`%7D0A76Hy-aqYZ+|)R&
zlfOONa6vQhLKz7$_v_r4-Nz6S5g@_#kV@VI#{7_?haIP+maIJ$hM@(rqkJQ$dyR_9
znYHQgv-$CcW#|c=h8`wQ2|o*6LOfIWFF9^`{mFp`tYN=QBn3~n!6(uYFC|8HJ7#mv
zIv>8vyx5MEIK@7__6bBo)8ZdJ7
zR(|1lOKQOwVOzLGMy;u&NL77Fj=h=_IQ;
zQmkAn-WJJJ1BGxhvv-YDroUc*2q
zGd}NPuFY@|&D0@fl>LHS@PHs&W-qVkcF$50ShoE`{Z|naS$Sx?8bxHa-(DJx7Ahl$
z&l%o?k@&z3`O+(Sv(=j+RGj7ZSY?fh#Uun=^=zifNM>dFiv&^u)+$g<k>?y)vyT-Y4t%jg^X6#aFGEzEPJ
zK^a>i1Fy(NIcIlES3cv7B1`!}#<2Ow<0{q60^t!(wqjEBl*!M!Bdl0(bwY6s=VL})
zt;lbK)f#5jrJ=dYA6)gpHQE6jbm;>qe>oZb$o%Wb^b@nWn`ZZAd?1^2;{qR+-oDp&
zDTYA&q@r;iNa-?~7}`>=U!#8s!;p*>WJO_+jxUZhU^!)=RV|`7@$!*~hO|s@Y`WjZ
zjIRFnE@0RcwE)K#1V=9xRJFK|I|BItT_2K}s#5|s=pdrdyYEtpG57;=&YYmgMM~V6
zI-yJlaaNvhqUzCS(W2_fko5uc?%zQ|QOY7dI!zZ;OSR2(>LK`lH>ICZCg!ajW-8ql(LT
zd)e0y>u8rYe=S6>y~}i%@bBPT9cnZdvu7MToi?m&X?oXL{Ls9;+dCiKeqA+CVR&0-
zD@XNK%M8)kxAV6;+jMb5^d7E3P_+TX>a~BSV;QDDA?AHIDQBtWhkf=0kfK^nD<+ZGDh&4%=OEui@{QOQFvo
z24~ra;~&9p2bwOHgP2*Y+{Y7TT9hoQsV;mjqLtWy*$>
z_=o6f@|P1W;K0xBW+*l}bXp^J{Bq=~soP`iRX6y22JvXvS4&(=oL4`_n+svP1+7iq
zD^vl*KcIJL{gFOD{A;AuQvF>vIpln?!!agj_Rp)Gstx2FU)*dlNetsW{=w<``$T)M
z0yL6)L=WBlfZqOScGGxT`DRi2?K!D)5u08!uzx*j{CMiok(&!_iabxFZFi$l-^cFU
z$(xB%ogqp5L%QDvKn}3)=>JJ5?>~t20DZcGll8bnTMWY5bp-~24kqgRZt4!kbQu;m
zg)D#+i%|O~@U*jbAG>?!-VdzqO5FEK_-e-o!23k+-QPa_=zjLYYU653+)l-v!XHa&)Dqw^*yrg?=72+Z@Vsb6yAS*{D~(1bM!6rzJw+cIdc!^K2X9@h5YO}q_i9G
zxC;$R6k0ywp8{Y*rcCNH%yK+hgb2vB*GTKQ1p7rdv#ooPo`}zyaCv^BY`d(B^NY@X
z+|S*4{-bxHxa>2$+i{P;6r*;0F5KPl#Ih%IUovjVaIU5&n?Zc-2$VC2B;^r`B)$6T
z4ubG(%Bgv}fc}RjBB`HFsG)2a4Qm)3={N-s<+-;k;y1%;r`nGr#;pf9BmhEOyYmTv
z8$yjAjwlAam?9=TY;*st0>FvwQt9WI&_g%Z8Y!@%bn*;i1s(_1{vc6bztGZJaU#F@
zP4_GC=^O|x34~xDoh;H67gezCM15Od3@<2hI!uir&=DY0kog$v8^%sk6q7Zew_EqG
z?5m7u)qJU_V7x+LXU1JjknUXR2z1FiJ){FK&~@v0cqn0%k?_m6@Klq|!!jZjs+96g
z2E~K;Ia+T8`=qgEm;iWtZQi)HwP}mToj3i<0(a#Dm4bik(tT%_R&YTx<%T%j^7Z?~
zWcE?gSD~-MWKO%o*(WFwoQH23l9Kn{_?8{@&NnhfbVk<;_ywy8yh+C2Vqv!NJlV#0
zT2nLLfZj`_UB^5LZTDEn*$iw2;RQ*3phI>uym3ZE)?v98X-H2s!kewLRIFp*@23@Vg||oo~CK7qZS{8d}(`F
zWRLcsexnmO8S2U~-xBb3MQ9+IkMQ+UFALD36(ObycHv^wn^QMu2}~wpF-*?-lHl}k
zOQv02*`O1>Af9u!XNGYlRy1{q3MAjAXC$mjQ}XTM(jR(h
z*cDB2=_Z%;>fh-U&4!OZ;NC!D3}-*u+2#3r7nHgss4Rg3HA`!y^O8-IQz~(k<-O!s
zL5&HTdR}2Bfj??tM+T{4T#Z&}&tMb;e7)qb{c$~R+Zr2UB8+rA7NiA
z$#J?p>lG_Cb<#~jfYqYdt!O3*G>y8t5d!0-?_Xeg)O8{ig%;WLvMTTYq4%MHUkTu_
zUcMZS;33ZvYnRVZwyi3wcx6V3r=eUb07?=@tX9Vznup;5?|y8vhX;);^<0~!g9C7L
zNMAKbnVX*J9WaUfjM?}TsZk=H^
zAAR484Q~5!0YN|KB!vlZLjzBro|It_H^=%4FYkMik>~?L6p7LE}
z+%2&LjVuj=Zllevb&y-1?7>X^?fy$_^synae9M3k}b;Qy7|6L*q3PEl6AA-oFiYG~*lE8l0}uXkW>p
zs(EQooP99>X24z4yp`cY4ayRoP**mM$**90lf(kF+x#lwiRK2FI+_WiAlK>#hYx((
zmmnf7o&)+KN4!+kv|6HDxeoFAeC=C*vUG3N+#{M>mOc$?1>7D%z<;)@r}d;UUuZwK$DUN=NreEanMF6+@1E$Rhf
zilw?OD_HLVJv+P%_LRgt%%!3(?P%8YXQVq?S6Ac#KOv{CYNd5L(pL9tmAL(>y@V;X
z6uDxNx!HG@FMhvZi}AnUNzFyOG$4VPW2+l+{~)@p=2@yRcYM88Y*2!hP2W{&g$WCm
z$DbgRHs0m1nHWwwHLwu%*^c6J4XgN%W;6})%6-Z^Qc
zYD~b}w{z_suYwV%_m|oUYIBu^6etqAcYxi5dyQ0L?(-GBNzA&B*WwbqzOX12iksig
zh_)mzz=n6BEtTodoLMszpBg;`qnp*8@t`}c#Z_GicHL}>z<4C0AYJk#?}3uvR^Z~P
zXJ~D{JD1Avnq8R2&|uwZAT=2}`?WNsj!+;@MmRC$Ed!vszd|~E@WBUm3Qb2g!xfe}
zj^ltIpqlM;Di7k|qhn+!qtoNPm;X&#KzhJE&^GqG!J17J7UzFnc;BaXj|>nAVO<>c
zKc5McgxrUPTt7<2cCn7v6mZK1(*KIbfS%RY{7k}#Vi^m)?dBO~
z-2A6x_1iceFAF{XHGcSUzZdt&%=96vrOE^7Ukxzd>J`B3oO#Udy*b;x38cOb6tVoY
zyzg~=Wb~*V6z~t{5#j&IScz9F%I_a$OU3(f?tkJeCv>bSU#hG%uj)NpjwP;KBnF|6
zb5FO`IyI=zSjhL#jCvE0ycF!)JAuu3n^mAL7TTTYc47&MIR32f`8vn@FU#Xr?tKy<
zCQ*B->`lUvkDL6;EQ0y{_&_-h#0?A45)S~UMX9e1)phI7L@~`h0=T6BFYUwZj#MGL
z*T=I@hn_rDUPfxmV*tdFxcDT|Pkk#Bu(z~27w}^P@E`9$ADW-s=N2Fc)DL?KHz(9j
zPU83LI01t@Pys~X^`8Z`EkMPyf@6xDUR*yJYS$Qb(xag-W8EQpIOig-cDanN1Pk;o
z{u{f0%J-%is0S(Di^NAIGMMsbI+;22obWI$h*%eT4Pbu`ti%;5b;}+G52|x=u`SOH
z{r=E5?n&n?dTE1Sy8jqzP0{_9btesDp-ox$fKF2d-de`dXAoTWM)(rSUBzQbA9N(V
zOPkeTIKslYRv+`%z2239E#vqLOT@3gv##DIR$`Gw5-rnHNE)PsN+P$FF@oK#a
zUUz<5u!$bSB)XT;R$Vq4S)fzPOqAST5MgaCDkSOYpro*eE}Ykijc8HBGoAZJV8TS{
zMTkbKC{o~dPk)A5fc=PruYGqvD3X_whYW`l`xCy!6eQ|>I}|!`e5_h+3>}C3nEU>#
zBw==mLc}F?aP5j@?Jg|$kI!Qq_!t^Md~RWHFagYD;c*&!MRz{<04Ej;uTs6FGNmno
z4$Q{Tk0_vd>;<@RGey`mo-k|4KgWYQAoLDqY*JQ2l0WJEG6_uAz#XB<&=9#bK|q;o5huhSwW82)0avU0e!KUKbo?r}LkDMEak%mPiYk
z;JbhF2R-ZBa#sv{Q(Gj#pSpY>+FKmY1O;l0&799HDu3tDu#qu&DkOf=Oz9yZsYzM(
zSxZ(}b;|Y`H;~g;jts#o+j=jt8FHsKl4dqetQ
zIZT-X)?`Np^!3+nj#^p;@`^#{>UP;dH7}BxTuE@j_mY8fm{GU`0Rqf6in6M7%J{m4
zJ$za`@vkFX)$MjuzIX_4{p}VO*rB&9rk@qr;M2+c@;U>L$@rEN%xwRj`DF60FpDV9uQ&LghR-DOM6`mLDhb;`wi#kc@=ETo`(+nhx4YrED
zz$@pl0Dn)(P|8o&&ZuwFgyher_Ap07mZ9$HS4X0~>5!Mq`Q^VR2|fo^yM*!SBJUtOWEy
z8++}L`(7*%56$l);TZk#yD~J-BA(ULNGFUsR}gL4`@?}WNYE1;)zV3kj{~x1$u>fp
z#{U!?pTl)@ZpF&iz`A<|thab{d1dt2pBlo&$L`i#6061#w6HA7Yxz$u*eBme9S~*+
zdw9KOK`zVXoKIPH);i;aMN15y_wm+}uRZ
z!AP-J0^ik?2#^)BL;(!)fSs_ZcIny&u)@<$e&kGb&=Lh9clx;C
zd~Pi><}|Mb82oFzj`BmUcz<2?X0$?~c8XIm$AmG<^=SyHPXjLH!Z!GRgSC<|^J2VS
zi4Z0*g{T}Hi7s1UKO9T*FW7hSclsTe4&Nm+=VG(F>NR6e!_5~BLs_^?B-in>sd%8M
zm2bxQtP5Q4WlyN$mQj>mfM|5GBC;NdEjXRi_L3Oo?(CbFR=X2Dl9Fxf;R?!JMXIy<
zJV%hMldV}_dR3i>iDYJp3kK=$G`Fzn?|YIlg3ZTv&Ic%Kf1il05@0kIawV@Iu;uSW
z2CLc5TyKSuYq)9e%-Dnn!q`6+!JNO
zh7zyXPV4&bgs=aerw=p79R(jB4bZGji>!A_Mq;%s3Thn=l4C{iHhK1}Zq&cxy#uM^
zM2E*o5w~}?=f#Dj^~wj&)BmOYtfIoo%1$3VF{IOs{VL2cxZ|6@yHX>8<}XG!;%^qa
z7p=Lhi0iH83NkL?A;~=obWP#~Z6fF@5x7U`J2b`D@k;vw3Ngm*!p_y!izel%+26a2
zZ8*vb`3$CMf3
z1N0KeseestPjAm`)snSv^;q)a%Gk!)Cz(2Qs?oD@2M1S=w^Aay?EY*hesA;=p^Zrb
z+WN1FMtt!4XzdX!e$_hRH~55hs^4F}Qx!g6eoPElevZRG!3M=Ixy4Tt;?5s7IR}6L
z(7yR)CEg22GjG=tWY7@BpNXDlPz4mDCpnNZB?0`?Stj%*^Bw>@#_;f@bAS=*r@o;N
zDDqg||6jY#vbyNQ`ON)o&66ExRA@r9cDSYW}9FG(8!@R*up#k&FKYOpUhEyC0$(rV*I&7`=
znWr~3%HmhA43@}iyXv|l%s^YMw&=?8po$91I-Obo#sJsU>t;Ku<8)-ypw&Bl*nE1Z
zV)~Qlc^}%%c#8mo5R)wna(b
zw*Dgg9YL;ZmP}3=RZzA=$8pVk(3MQPHqZ|DahV-PxGWaB?N9&2`6GN{-2*(?kZ5l9
zbKq)ubg$D&a47cd6vC%tpzvMjEk)MmLg&@latMm{TVD*2a=qPG4J8?DFl7yj79*u8
zw$2(epe>87UQ~mYU3M_h1dMk)BwTXTo)#^-ZpQfd)%?F-J#@vL#$Ng1+&D@?zN>4!
z*3iK(r|j0f))|u)@YeQrZX{g1n;;8NB7hcGov4&8m*6$2JMtyt?et7WPIrAdO*HvD?O^N;XgIsYM
znK5Ke7cjCCq0=hO5EduTB*73-0V8Xc@L-(}wH4SbN>dMCJ%N?Dc*#4l6c)Nj5;;;^
z#&@RCu9yO`P#4~Cu|ZD3t1Nv`ndx=vLiLhiT7
zB7-noYh=D}WWoL4`x5F^H=rVhW`2zal73Ex=f^ysgK>*&qzyqpaW+cV^|nJT)F)_A
z&3G#a2imhgCRUk32qAX?v)}4?=);rN@Cm;qu-=Doo}|CwU<&HPoTJvzqUAOan*Y-<
zu{Nm9nPXz+k`4EN>g+g}qMYW%15%wb*&arbO
z3cKI4E*P`f%lX*;06gYsAFyS=Ak^Dn}^sI$A0MlX3ahLNE0h
znfSd^n2U{PUy9v)ytdyQazYHwtLB4nKfXw*7Oi@02qP-Nn!*c=&zU5K=Lg@+HkT59
zgGh~>Kxi`y5rRbBkljIPiAIJ54yYSEpIa3F=-3}l2nO7srd1z((sFb0420N!h(|mY
z;oiZK@OlgPy8aG|lZh%8a6X1ULlOP-otltq-ye+*a=x*k9`*u{b?)z@;=xa+_pGTFX(r^b-naM>Ruenee%}947(S6tMc!_1k)yCc(i{LC
zTnTBWJP{gr1`r#>(4prvGjZ3ykhzqh>u2re3OMMWAw=h=+00i!RB0E;Nc+g#q6mil
zkhn&BGJ$jbdou`dJJ0eEKug2K6v=cQgirrXFPbMM)x{A?JezW#~kIpMQ9
zlSFikAI!JkA4R-Y+k}IDF!TIhb^u_a6udR!F4fG-jtwUzZouXG(3
z+~>Xf_9ePac9L$A{sHc5il=V`S?3=H$6G5I&3~Hib`Jk0nB071N9=xKaz>?cs
zUUeghI*_9nnY;UfvIZRpJc>6nbNqs1a|;Itmk-q?$cV=&h_rKV|Au||2aV3slrxBI
z$8j)LD;7W<4fY0c$dgypyQ=5y@(@LlZhw)|(#B#ba;qKtirnvpdfPm7k*26ARlqP%
zWY8wG^@Et-^LU#(Np65~&Vf{l$0L?-Y7CQubPgm-}%0kVNx
z>&7pA`)JTioQL(eQ@?;MAFn}?t9gSvcCS;n3h8ImVbFF@0(eD+zSON3+?N4SCpBgp
zWr_=yB|)E!+`0tfG6*f}kGyx7AQSpbXZVgB)q;P{9lwz|TYF$4>KJaeJ>8j`rm(D5
zsH;b8o2ZnL2;d?vgG4CXyTyMxJ3?r
zCTlPIC<1yVas6V`BUiEt>-$^$ikok3q5xA(X58{<>c0y@8V}gCd9TGqfi--#ELZpX
z58clNsy8ji{_VwMZJH2f_#9PXtxh`(4LDQyElhVaPoWj8sE@5d^a0<_0%g9
z7l6xxqQ%{B9h1!Yt~`;9(N)&hJ%yl}l)PG)SXY6=2@kI@<8)4Mqk&$coi{6~IpWl%
z=&xq$P+57UQgF$1qEdnrxPBnzf$vTK*F|PJEF1>wBr|ZN29JD@Lv81Y$f4c+n}f}_
ziKPMvD-rjEJ7VB>mQi=}V%IbRm(aL}VISs^8J$cz*@$FDX=KFZsq8*CLj{IJop)N?
zpI^d$M<-+@sMGlpDpMACDRr9&ngbcj>`kV5VfW}M>tGE`v-boKH!;q5SYb>p?_cR@
z?*W!_Ofy~>NwDB-0yO4cHq7MnZ@!uGC|)Qu;HR&+0%-FHS)p2NBv(jKm$?61rhr3j
zb8{GXD1*{R<<>e{#-613nQr01W1lR7tFELeM@d%O9D@h4%>rR!2)#(!V#`{J
z}srlwK6h_4|+qt|#Jczg%z+}eyqMkoOX2Y6XjjM%KjBw
zxkN!Q%_!DCXU-BhP_iSMF|ZLX8-_7Bp~)F^xu(DMzX5gp+mHi<%iZ2Mqpj>x&ir5t
zBa6pQ4_#X+ga!Y%DO
zS%}CBOJN}sAbXZ-v5OmtJ5^J)
zqMt^~IeIjI!SBuW1{FbNt8^)-Mm(CIFe))<|Ct}#IYR_-*ksfLgV75<<3Nr~X#jLk
z^!OsS@T1M>S=!4kSJROgn;0FbX{6GEK@E^($m>3
zEHCEYmUvX{_=~9lt{PkH$ix-h;i9s##wgL6;OM{WwOF_{PBCs%u&{`Yt%2}Qne@||
z`4F4A!*3n80ka*hMy>deU>*O~{B1b9YU(8s_NHP1
zPhlfWCZ3NCBzg0sNIf#XJ%$ESRoQcA<(Qvqa>fu*xv1tpE32Mp(
zxpp+`v;kSq0Nr5E2g0>z3(TV5y$th?j{42M>)G+KuiAb!db#cVjL*$e5=oX}UwH|Q
z0W1AD$s$YoXck)$^SRThlb>IcSw_<|=2293!#<)tFq&e7qCjd8eSfGJi8#Y6Z^{ft
ziEO2(+jEwm(@Q3>j04_SoR+^}>0qa=Y3h`bz(8KS{1c~WQXupR=zZ)aioObvqnBGc
zR&L?1;2T?fqh3v+>S{9u36x$6W69qSfXzXymn4S1TF=;w+W3jxdPgZPU?nWhIb`|E
z%wHT;$m*}<8`Swx5=56*#@Jvp*d7%N
zvP>N{Pk9oK(S8Lt9>p%39x7zU=c@(7k8(5xV(gu;OCM|tWY;jo5N^i;oB2+>UO$^m
zfl{m%*L{4zWsLj>Xq3*92zZ=q0HT{6cPntzKR}Ng5OvG|T68b5)g7de>2`qPX#h-k
zyF4(XwYNCD=y-a7qwQs*tfG1XRk#S(0|H$DWsKn1h#lDLw+-L`fJm&GV`Os#B(ahi
zn^OC$tE*??qxSUX8DZGj*}bTWrHUJL5Gfr5(oN?8bQ#otSp!=&upqc-+Q}s7ri#5{
zd(m~j{nq7dXDkKIULzn){ng*kyZKMqL#RFkBwC%n;^GPP{Cm@B#I;kI!Y|x+>U6@Y
zYHH4g=;O{0ZmB3c7CQ^jsD0ys3Wl1S75KucP{-nf``#{e?wgj`7Be|L74<_~zCv6(
zxvBAUjj*rF(dRovg|UFnM_(V)6ct5+bo2cjjmXYUF#uo-c}>`0mw@|r1tXo#e0#O)
zD0d~JTu^bw&6QC!T+TcXzH^2b7|ahSy;n#`%U1htO=rV9ZmGc=>oUGDvVwyrR0J&l
zTB_WYx)tHzS(BC44`4XRC38tLV0+|mb}V!e`8g=jc4O2f?IiZvvg;CcPrqzzn#WB
z+M^=0wS@%V^m*FL-S+YSgX0WGddV*j9liY%z&%v+iG9GJ^O7r#Q*8Vq6Xzo#eefhh
zg{-my!_j5A;D`&GS_{QJMpb%&LCkK+I^Qj#j17a{}o;lJ*AE(u5q`VH2l1?lF
zmg7^4u@K0eumr*6dKt2`bO2Ep5Hh1
zC}wSAegIx8?w`TCFqpxAtL^CAM8w)KW_O7zfRkhH@zi^$291()Lo}5tN6}!nmJX*x
zQ1EbdGeclxN7_f91lf-0j474>B(t3vPqRtnB8yvCUflsSo=knt+mhsZT1jZ#RIZp3
zLKGirA)BjV!2#wERha=F!vYC<+Jjlb?veTI9t0b}aZ-k}X|2>@EHe
z!tL$tU*%W;=B_x|#5Oqo6~9`&-&(tn|0ONVS?-PP8u7^kOwN*xRYn(7q1;f$CugIh
znc-!jGaIVB88J~i8F}G4zBxF4b`8^>AE{B#q=gLCN=w?{MUm-hqVr1Y2J@8_NdIEB
z+ZX=YNXGZp#K^QUPdSt@hyxo2w{X}rExBQ_&7`ZqcU{3X7Qy>RC&D<-VfqE*!=nw|
za06k`#_fB@&i5phI3zKIpXYKWGrqjX?4e;N$;=ZwS-^W*)@eU5<Q~YjIdg2{l(D;FJBIo^v32}mJjA1dm9ds`h{#$CNB6c#57C$
zeIx?ZXPy0N6U4+azvH*&77F*idawgmt7|dB$QnOP3d`uDeCB6q_l}Do5Eq!j&%<8I
zs&Wk^!Qf^19eTe>j7X&R?&Pwj4t9kw2@|d_Yu1W}Xu`fU(x<`}Ju?@I4Q*75v&9nF
zx=RHAxV8=YiC}D+%Bwf?yMOXgG0*sT?8t(fn?`kJaUx4a3ycYMoAgW_8}r4yMs~uKGko_+s>V`S4GJ@R30;-M#x8EdiVy>
z&mE1ucYbzUq2yqb+i{2J=;t*8Y#(pzj^<>q3z6gf~&$#dBMm{b6}tIjT_
z7NqF5CpnY-t}KdjJi7b}rVmmPwB;bj?sYvl)oXvB_^&ICyD<$Sj>tc*tSwEGqEd<-
zmL7Z>W%8rxVt!tu5ii5=?~#iW*^O5=rsIB!7Q@bmwr>CM!#t9)z>=0e+C3y<=T$1f7{v35hFoDPM^+{Db3LQDh}GVoMn-JBTqNo9
z1lX{;r`5Ztcv?gS@{_(rT*E>)+o-}G^mWSV_eps!pRXkJcemnetd*7p!&*P?NS4^s
zdCGRF*Pj{k#`<4-W4Pf`;K|UE4^F_0M_`Y`IxA1Ni_#peUP{7)^9
zWSlU)PtOxCwcjV0cU(s5^D6itspveW*8XX#H_&+ZxF~_9>*<%JH#@~xBN_>oshT#m
z;AFXj2>fg7&_II)c>zLZMy>%a;4Y^pirDiuz2+1MlGW?{B1@Dil{st&=FqUo|z8Rp)U{XiR9
z5&ELu>aA83^};XM0s58m#H&d%YZ#C%>*iF>Z&=<#=+#wr`9P3v7-!`Virpe6n9{L?
zy!RTvt<|e?#7!G_jl@XOrVK|LhnT0wcK!R)r}yc<7=wO^bg{}MmK+xXNJ7La6!d$N`DJNY>!5L8(*fxgGS2sv4<`NbXFkR-F=7w$Q?NCXwb0zd%Oy7auA9#XMBa^N0<2{
z=*!K0aT>)C(>zSr?<_t(GiG82X{1m+{l+F82Hn*9rn+ueotd2j9MHC;#7SsytqZvw
zXb{?HM{XBptYcuWODeeZ%%V%&7AzwS|1F+rif^=Whu(Y{@9qZl4#2@dYN6$A<-H)g
z^$I;0QqK;MP{Kffua6{1nc_tEHhixwgI>00aX3uy@u$zckXxt|>yU*iT@}+Oy@0%L
z#g$kV#VpE4=o7_G$a{+rtzwRpkP#gs2z_ie#Rt!|lbU}+usKCdxnh_1#qky~Wgzq7
z737RN#k>Ygr;ny8+NJf?>NAL~%y^3zFU?Tonub?*f|lpKqdV6-llkTL5KV2_PY}2p
zV~-QWn2IGxUIcz*0TRI3M{d)0KeszG&gM$P4Y)}ZUBQ#W#W1AG6lPu8;BK9eDGZL*
zNxYq7Z=EPj%yb4#i(!FW=+CsSdv+gznv?5&w)-vx<3C+T81jmi7Rulv>M5s2D|DjL
zLg8k!pZI&ep2U9C*?vlI8Ajq6Gc?IK4UBgMy8Pc7``-vI4SHPXFu~WCTX`7v!wbp~
zV)!t@@2D+A=qyOq3&Io_LO#D1EbPoyusMX&M9`ZzUix{en$^9TC%3174Y(e5O2)%r
z`&Y8USBC-@z-Qwzqi#(
z9-Xtp!)pnW(xr_QV*B)rPniKCTjq##V*rHxp80z9ivo)Q4fo5;()fY4A^>pPg
zI7h(-sqMR&wucroBB?A?uS;h~@_%uap=|t$9FbXrcvH#psgaA&2p*|e((<_38-l3&
z;ns=`Oo8C7?#T64SCcQr9j>wu+vXPDT+$2G?b(mPV-grFQf@caP?(r$7-)gs4#_ZYA+?&GjTsqlg`9Gc#0FM`{^DG{gA4bWOs)L3)k6W
zMqU~SZ{tAa^!=q&aP9kWtLw7dWB$Hu9zarC1Q&=e?bHU=s{No(bR>YJ*D{d~)i-_O
z?7&PL8z_t8;*fc~b`u)&{+ZyqvaY|l_@OgIMnSrrl%h`e?m?5~!7ZDV;kq*u?YK23>G1oxeWWL6ybQ#
zWfH`U(O>W-z#)wxAzGe(q=K$8Q^pw6x}~2^*8h0?DGV2{Z0W}QB8wVMOF=r^Y8F)K
zEfMZ=;1`;6eZLcW3sj^$o``VBKKG#f-)H{+J|u$1!l+eM$2wTRC0t;Hj0T2J9d<3k
zxN9$tV#41QIQ>n^#wG0DZ*lv8uZsNZQja3NLgpGwv9E{r>yH5um$`YL~@)I}!Lm+6JAR7EE+`h)qw(A>|Ql;4d?olnsB8
zI#6=(BauOA_>A_(L^gGo!6q~jQ})}>#NkY4pO8!+#tDP@jm$I;<{!bl+(i+w6y|a=
zB;nb4NQlGpB&xcftqqe(SU@0V~j84T{KovehC#3wal*`NObAQ}=*v^h)SjQ6u%ed(>?TK}Eh$L@FxCSka44
zhEa)aWFg(TOK94Rn$M);XbOJ`&{+qA`VAW>T2wm8n~68AudXJbP9vC4gqt
z3JMA{jAwpeFyqk=Tr`z7uJuZj&^ENF!S}nff~X044ka`C^pUp7={1M%c57tWzUA-@
zr1*8qW;Dr;;wOJt0Nm`nwTq8t8z-u&
z*orZGH#8*K-cT#PVN%5a&)$^_UX&7!4>`?ko;=Rt_HgWg%~&e?93=ulnI*~9vfpf3
z@vV)$HE8brfTv8M69b!9SRyTRKYXHz>4I_AIO}*?%wvB)^h8LHvXj*?Tc0W!z5>aTTkg_P$pRQ*TH|UsOxY`yb5o*Z$1rl|na7;Mua;uXRo!
z8<)ykSWF|Qne)pqo#?kZ6nYcd{I53&1g8u>fGEsnLJs;SBj1Q;q@5Is2b#=Hyxu}3
z%?MWz@k5Gg-`9?dIrr%0(F1arfMUyx(^H99%IpG2dx3HSJ#M!47~x8^Xg;de9ojRz
zM2a)ZFc%c;)-vHs@QH2vdO5aJRkyR$UnXXO>7!7&S5xd^6vF|TZ+$hJ!79p?X_w$7
zbehw4jjD6}#@iJ7JIHsfVL`Yci+o8pd5F7|%7qpz_^#U4+-%Cwpkt%9NrIrn4t
z#&7gYFSHbku~$2>9owE)6z<&{B*hkRTm4zf)4^%xoGM2r9+g7=J3iosWSow;sp8(!`-!74p
z)$6;d$0UwZ;^XzH9%^3{cnIQ)j!D6y`L>>MuXmK8kT&<}S)b-Q;-_VSX_Jde=l6MvwC4l0s_(^>!;r@`8?^cZW0}{K?;Bjk&(gn0@PK
zAoP!8v_Nd=oqChg}L5obwz6MLu#I7l@FKw1J@
zO=cZ(;AI@=Czc`gJbmNs&H3FsX3$(RyySco`
zFG81OWc89NPeQ-C9LM^!q=XJQIE|QDT0EsjR7_*hO)55XpQLX*k$<~y?-1Jnw;@J3
zPD-mQogtm18apSbH0;v3U|ObI5*#bVlq!(UjWtT#Lw-1@Hde465+x9NaLCR7Fi*^1
zG~88Alr8ds+ThTzeBR+a7S}A+aHU15WKeM4Xpy1)>c}wDI<)BdNsW-C&b2|@Q@d7=
zP=USWYdmQlz220YgJ|$GIXs|&wRzK9fAjl0h0POR-A2%R67FCgVHO&TUXh`@@o#fefib7(
zm7TW-@YC-k?F+4TAY`71%3kXfXl(qxOBP*q1a
zx!n~fLIlwwCo5f|Waya~2Z|t|6Bu66;Y2wB@eP@=Geige%cv9UD`qZl=0~;7qRB?2
zqx^Y9)(uia3Ht2;|Br<%NK$IFLzWlap1&kj1iCdOHxULYIcQb-VTeBg8WJam{6GZY{%lR_483vxv
zJ4a3(rwFfiEsOF1>mX}q1^xOLF3!q%$QUmdf3Lr7{8sQ4ArJNX@bgrU#NTD3P4^4Z(^G~T
z^@`RXq>+5+&IATa;M#w7325XTtz>_xOk^H=S5vWmzLLc}%b1X@DTHGJF{mb+`^_`>
zfnY2YLXlJ@5T|i2v+PHuY59d42!?4$sn4Bzki*JIe%+it&?m%R|Hyb-waw=>fg`oP
zuuLBQ8!_|FR$D9cor1GsgdRHG4xX1FBpu=i_r*u9DcOh?a};Q(BFjSh-ihrKoQs9A
z((lxsFwK`F;>Q=Z!aTQ)VZt@tF8=zYHL^nfWJ%14Iaj@YcHKX!K)4aSA
zafA4r>1wTEI^FDdRez(TkpV{FqhN8-V@%Ja7*F!S)gIB>ar6Mso|q3ONWP=@lA+B6
zcFzUFPu{&EI7xxFT}5y@Hx2-Hxuo0%O-2)gS%DnUi9jc(-kupsorX?>2)+v?t?X3H
zra`pxL`1E*Z%A^Isx2bm~Fz%-@;n?uFR0&4Vwgv_0cuF
z>!JGFReJ3{t;z;lZ3yJG7UL3phH`8=}skC-Q-#X6spLXQ5n-K6_-#Jc75xrroA=GEkD^M|a(bzH1|
z;l<=}&pxxxwt#};zu3oujQe(2yExOoHK|g&{Do~uWBSjFlamb3z$2y9*cs-k%)}+DapFh}ZGFe~9vkNCV~@U2fvaJ
zq%|c*dyQ^h5cOK(U8ra_30(N`sM49lyf=txZ|kFu){nEm+23SoTpku3A9{1YkL{DV
ze8N39&$^^3{;HCZ%uEJny0Nm2G;td9wEvMCY??XPSPbD%RrGKlSvT(d{OB}%Z+Pb-
z^28JX=-ndLtpOlG+$DY7%$=<96TUwpXk^jb+ae6|-!JxMKxr94t4;A8
zRWy$KRl%l@TgafC)6H-=P9&w<`18Rdc87*Opm#UBBf=JCa$oS<*0aWwdk?I;lRDbH
z!ton-1apc`Q97eC{;VLwxWIKka_h1cd{*KwXE~>H!@ETht1*xHX1OhWxMuymtq#_z
zz&2zd^5*T_O5X8&3fA~g!jQ|r)UiNb=VeG!khEU8T6yjSad|e$Fdx94H>;ubiQgob
zgcQpc4FC+54BcE}MBg1YuMI?p5k=m=Y?1>d7JILnm>E9W@#7y6y^Qj~c^Un}_dWWuX
z;K~-YML}>V83LEcu6+yWdweOrOSuJ9%xXDCU`5y~KGw4T4ARyn@=VY6u;tmUu6ObH
zBW;!79xx$P7TB9PQI?FwPc3B+U)m$_+lejr|DD27Q~OmUOR}HCKx(kGx-a_AaX4yg
zB4pv<;&o=;cf`u!Ho48kNm8Yj^O2U^B%|3dH&
zAY$WaA=5tfZgC%!r^5{G=yUpozWP{;Y>+g|NXT3|ATMQrkMF4bx^7of3}1+R0%DU~7Np
zub2VL+8GbB8Zpt!oamgqjL{UIT@n#;_yFn_j(lP|xg2mne!xt_?-Q3fJwhP}8m$!v
zzbhO-$)DWv9Yy!9>vmQPz<`EKgPQ#e3bW_UIVCgqhVG;P
zzR1U*V@%=OKY87UQt&>+oW%8v7zG7lf;!CjdF0b9)h+Ja3aA7NXDxK&GdU4%&53k{
z+aTd-y=V`)%WPKA;X!~RzH1EeVv16WH_97Q&5*`L`L|rH?$(Tyo7+odtE8F{K7Ab4
z%1o3L0}55jk?t+Q^}loRh35jq2{;w25>ERS27moLpG|EK->mSKH_Mp!K3AfXrO)W*
z+StdA)=BKiV4N~{kN%0Y@wwx3&A{0G?VOdpkgb+Ac`$Jx;02e7(3EqUgxgIwYqK-NjJsl{jQMc$^F_9CbGh;A
zx{@$`=fX$EJU=+Q{wepKx2Cro?Y71JzjVB-3Gv^9nGQ5^kjt`{r%{Xik4%oU#zNk7k
z1un6bpQxRhF(g4cSI}L8!S#=r&
z{sydi`PO}u0y1T`(*aEaS?dd4Uk_w8gD(E7T^n%gN5LR+`(f3>R@O)nxch6Kap0lO
zGl!O5vRz6k#o8=>k!~h9BK0NOkzY@{!
z7f1J^{$ZkqADyx+exe5AL#_??h)6nrpCKB(_dF|M;BLCW%V!;MoBR}lM1tLrhLr~
zN4Gp0_b+!>6MePQbCrw)$W0<@I^3V+3qBusng$>#i7K0VonsC}BjiW%nJ_prJ2!2#*N20yaN|!1p?gZ-
zr7IDbI3m2%sS!LljFRoYk;37HS!RsdFJFd^=%5LF02D&3l|!wK$BzgQH|~GPGL8f)
zX*xNWiShci4UFisK>E@j4szOr6B2Jb>)3yQ@gsF&S(wIuZZ5gLiAYzx2FA6_x7zV%yUhTZ)mPTn_X?Wr{U3`_{D1v@Szg~P
z#%CbT@TAuyVpJE#PI2ODjZ*2Vn1&j{AbR8!Unarb@B9F+<5ctrNm27VT={3hXO
zhOulqGi*8#r8HWn*adt`>72}4&K6{ATqr@|6|4}CTc;(RRX15rAVJB<#w7iC`Hn?8
zHGHID!f4rkr8Xp-W60Gl_{tl5EUb$93zc>{nlLtJR~ou)y+@E+fqMV}M-jiAlfauT
zoJ75(7-ynIsP*3qfb@&_NI~%l6Ip-O|13OC6Szg8z989*5WZ
ztb^38s?{idIH8*-;6Rlp{}%>Nz9rO%I^>q{GT16QaH=N7KDDatLk1+a;+$bk_!!w*
z=oz$|S8}$C7%XP%Nt0vvD6fk9H^-wk)VO!qsD#(5=%eMf6d9Dt_$&i
zjKH1n&A(D}$4F*N4%W+sPIc4{2SkRSaTVG(zp`&%=dp^tJ9o-nV7f@Y1gf@W%0432
zf*p3KJifZ^jwm>{*l{i2*L3s=mf3$8j#k-q9UMHCJ~~z#-*Kx%*se^g`7Hsn%O`i43M}mTEuGP!!d=>aA4m{3I@uFLHgHJU8dzP#Bs->9y
z<&foZc_jo`Y9hM3Hg@-2BT(E{G7G9k`6##gwz}n{s;ASWl4-m6n%5^tq$0Mz!oI?SeDbVWJT3qnJt%S+h{*p
z_ML1}5lW5yvDsETerR`jV=udm`T1?e>$$D_iao^Mca;7>C!h5g(O2-M9`mU%3))}N
z+LX>Qhy8ZXjlLx}N&sO~V%f%-ORBYeqYLx){b;6e$FtM+73Zi0kIncO=*|uI+mLa;
zMRE~vz{cYjwz8`3KaGZ0d4U5fe>qX>S?ql@)%O)W
z$&u>w<1#Ex#OXSfAop|q?lYtGyTjV7=A-=993|G4!NbRmP-*V-tT`y8R$fEmT5Yg9MC7Hcax2v;?^hVYs&7Ect$U
zzqCu-p?AaGl?4s%N6=w|?g}LpE^iNx>cWHCsfTQfhEAG6*by=h8=}3!gZ30kC8~q=
z)~0*9)-%Rwa4ag@!Ogq0V>pYNw-ahSzIfFPe|Krma{fHR4{F)yq~^;l+_MOvk$1<;
zSahlF8<=XmajE&JsG_Q8`(Yo~Io$xe%sth|iXtpa*w>bj?yXr8OROaFp27FV5Y0@KcC8Sd_DLWBCgXn%J~x_2rWO4=!V
zh@5d0-*~PE1PXV@BJJO*N|CznoQ1<59x_c5a!Z~h#U1W0pTsSW!s)=~F@MkU985qg
zv7^)H&D?Q1F|UWZ_;^?vN{XE>x2;29UV!6BOlX;FEzsnczC}eQ(->+r$;+V(x!SW`
zVKTr7PbQBC&zelcqhxVW7guOltB0iv;Cp{LZ1cYD*1G*sI8kYS#d|s
zoU9NzzN=p%S|ys0LI^14B`a0uV}&%ZT58eb5}4WSTKzxMvhtLi4M}+i)k*!fUf3{H
ze4y8ZCZoe4(Qo)eF<$TQS!{HgHU#M7MU`71==q2xNU$ehAke-?pTfV7g?Q%Uol
zSzWN5rs`*MU1NxZNSJ_Z5ri=X9GkO>tD5_?0(LBU-*t3cREvaB0y%hWu;HjzE|s6d
z*QZ!Bh~STPsB_1N-hW0W&tC$im@v$jFmQeQQ1FH;!Yx3M^)JKb;wqeAn5DroQATi5
z#R!}?pNf_uOUu>Ia9<#2mFPGz&owVfjlFMogLk52F0gZdr__3z3FLAu
zudN;avM3Bc&N~K|@uM?9SSDLU1-+pnS77}l#cmqBv-$=~DU*^QQ&hlX
z%nc7h(2kC3A-srBmaeGsqg8y0NM8>SPTpNZO})Cw?_`-()P9ILe~*X6EP+2xu^;|s
z_c^eDRl$FNRb4q>I|hIZf)CemETXWLNXW$kcDhudD}Rg6K5@3>Lrfsc9?qxYEt9oM
zD%?2XxIV|CFiL)lMoxBhZFwI=@Ubw|FaZEtjzhiRe+Ds(!3U^~mqk$T92yMYmT>M5
zLbLK)-#X^_tyNNdfBMa8;>~lx0HR`Lt8kjUu<6g}L0h`FKbuw|)|&*+CN-EYOjPHr
zfUP=N|CC!9r%#OMWV5hOkHtCS1-~|@8I5k%{5r6iO8i5%bYEuLg
z)yr@7m!410QU;%>-R_|9k26o}&tv^HHvC*2jl%mDZACRFc8SX&rZn8_as_PEQq7~4
z=XCY&FU;3=SR>5clwK6yi;0P!h|xa>as+Qm&NFU~o=8z#zV|M~479XzOn~yYMFZBW
zFdH|`H{%IZZP5ZF$N6Jfdmi1Oemwk(Jmd7fm-5lZKo4A(yHc@)*!D~&t6E%b#;prh
z9LLssjq0^sN!UH^S2__z)Bil=PEjiTwi9c*4poaK=NHB{)Y1zBNPsvwAT?D)d>
zFr9Y|2@Db;+^0W*haOw50vgU
zqAJ)l6s+O*va}RV7;>3iHa%We_p|u@j#lsL<6n{o1K8APG|n}lx|!qd**2{%0bOS$
zJbIcdg^-xf35(uzwObfBZFPn``y-hqgI}!?IMZ8b(lbNqswb_`hU_+VS*Lh-&!e
zS1mH)A34ub1;`c^Ur-%8`l=}jty4+D0Aoa|NlDv(px=2tnqB^o^1=ISo>Qqvhr%?8
z<8gpz`Lw#Bm!j4rtCX5(Di#Bge1&PRSiD*eYq#AvstNM)YB>$_anQ`zY-JzweqFPj
z8nQ!P8~8!|iE`+Rq23RoZ+&LBki#zAt2ZJDjfhn`$t-c(4uj9n$ZNws%bS~4B9w2#
z;B+@b&LE^LKtORL@f@Av9FtTuszv9$wJx~Tl0Tb2f9j1YUv5*IcUUuyj(?($jqw&5
za2|N73DLC@%p$?R{i$%8=CJwW+JQ6A;n{RIVhubLBAd!A@jMlV6Z}Yj8N4Ozf3%&;
z_{41B8brcl_R33AA0nJKDmgEvSd|LQpykRTOu_6YFB3N3*{=^Sh(DydGGZOQ9}N_F
zndu^}OXTxDW|-PJ?e-q?vB@m`#FlrFIq=x#ROrblb?jX$ANz~-wAN61cE@0^$C
z!X6IYH_CqlABdt;x0~E1gLaA~48bF>-R??;2PsjR2FwkBinsUZwFj52-*p;kym1?~nYr`bvLr63*sk~jye
zbHJllGHD*$_3y(qy$4i1dtjslURXsVIK8DT*D3n}N&A%e30)
zB~;&?j5*LtaCMT|R8t?nuZgcZ)2y5E$Oi%}DGe*IJ^+9#Ua$hy64I*F=q2+VDvIgm
z49y0c`5J}=(%b~;BpDl2x_mA>qa2z(e%quot5t>BP>#=
z>{Bj^*wek;flmzdZUq@f5@r^st;CRIzg3dW2+Nw*F5Uan#sjWs5w}Ac<
zS&XC(AEIUD`Ip%m?cDWt%U$z-pw@kg*r-c86N>p_Atf$BlwejSLkc*gkUwtB?MY#1
z6A8I;d)WcXTe}{L4me^c-`y#3*;&;ZNM*`#%)Xf@C3wBSC>NO-_TO7q-nBr{sl9raF3k?3M8?i1#=4KFGm8$mMd|UJTmD<*CskHrxo}3&uor)El}Z9%
z%i=$s5Loh)K7%5#a2XoAWLnHY!#xLcq%y*5fv_wIujWyO>ZUTHIp*+>2s8=GQ;fs!
zad5gDciFO2lpxapdChOnc9A%dkP#~rHgUXB=Y2w-iUzLO^7EI_ktMVERtO|hidmdTQ9w|Pkm>HG*#n#ukL`_Y%PIbY)pWDu>A|C6!a6~QN!SZUdOeQ!e
z{h@8X`h9fQ_O-xy7ThZpW`l>u_E&BCyQ|ik9wXF5KFEp>bbh+!ayZ3KtY}?Zs(dZK
ze}QiWu&StQizD2D-6D>U*V9VlZY&CENn?1;iQfnR1)@eAabzdn(;*&@#&|lak&K0537WJ5ak||3kgm
z(J8AIiP{joZO}+8nHPTfs7euBJ+_EXfo5WI9!~c$XF;U6V>=}(V!dim{~^+50K>c<
zW94M_g8ERWXDIC`?itk|iEMy)qZsw4s$+m>Iak^5XT!JyX~m#`No{m>vz8n6AT1LY
z#K;|$;{qI{U%bA9j8`3)Ds6iQZWW3#%j=mYg282uHx=As(K9ibtURv-vMoqv@#3si
zf!ziM22pL^HrD_iYpzu>aoB=?kcD!xa852z+}H452yM`CFmJ5V^wp-z`U66!*rtiX
zm&6kJ7mZ)5GLK%tT!SX+R16A_QY(_uw3NvU`15PQejsCfUk6OQS&~|_8Oqgz;n1|6
z#1GWwUq>%c`)Pe&G*Mn&;1d!#cINHv=S-_AQV{W-Oe~Pc2(@8IzzVjZzJz)gwT8_?
zvgEZzhQqm4pI+~lER@<8`G0*JW6~jc*H&W1Fcw!ic1K&4H$cf$DSZU6G$Il|Hul-=
zZmEl%3mu7Nd_M*6x)A5BgoPJ)dJxwB%o90NBBI>cVP(nZ_k7IXzec_EgVQPJI{dw7
zj~2DtgQ~A%shX|IhxDfwE$h8U67t(SJt*~}Og_5QKX98$BFKJpgD^cvj^i&b4ZDCY
z8@_)ZU41535rq~q;Mu`*%uzq-9CPU1Y-mB0D`dai2JOQELWU$l4pOzI)FxRI*RU!;
z$p~v^ResPb#mm~@Fh*gHE^awy0s$eMP$-sve%0fKc6)X+Cih0&?vC8HR4KzQom*)V
z=@x-O`j9L@;)^@B(N3OvVm~3%K3Mh;?c|pHIGBRb9-yS*lIBt;I=$aCppP6XKl_GxGm1Lt7XN){emeb8I
zV@u!N{xZci{UDu$PLW*%?x6wNe14F(TtPN|-Kx4>v{Lnuixtj&3#~b(GLn}Ibw|3j
z^FY(slu04m(nnWJ#RgGsx^7ghp;pYMr4W%v0@zxWD-s83mC`Mf9k1t;*$s%DnV_f%
z-u_BPB{pvnT8v6qr}=rPZL|wya`A}&18X@*{MmGcJ+9((ea8Ir)n6g}+eqpuszK(&vS;4s&7@KtB_RC68H#hM}ws78?$ub
z4Ny`DE<}nvkf7%di%B|A(X}#}UnT-~tRLCYIK?DTZ422Lp@mUh_gaEQRTsaM^s7Ri
zXmHNU<>xO1EJnGD+6OZgV%7V_cOL(JVF?_N~xa__GG88sOGJ3K`(s}^M;V(MZ-70(p
z4t$b3qc8kKV@@YFgN%~>y`e^bI5S?kKHM`!t)y!s|0LvR!X2<0A{=~&%M;oWmdqqx
zk`pMb0zi59JtuV>5!vHg=(?iz<5dRuPbdPHs4vsPUfqu-(3+`D>=$cF7`HZBiKRvk
zyIu1;1J}g~)$s$1{zW&LD$LayW+@ayTD;5;J%
zv+)+0CrI#X-&Qvid;qD*iY!VCiVor_;r#T|k57D7xv>$X0_@*~H!xAhM%_Bqe08FZ
zAr3XPGD(ePs3v8fQ}Ftv%=$M6cV0ouK^{a@Dd=L4vM${5LoS@h6VUg=m^tNyR3Gzg
za0PWZ6_ui$Y1`IX_BQce6vO4?OQQQ_LCr}{RTUg|TjyM^TKaxM1~tJwT3wBqfEndw
z!cO<3Gr`Nb-GH-R@snzqi75dr(nQj?x<&usc*$BlU*R~qDe@;XTvZ-7y_M#i*L!=1
zS$U|WOA9(bv=6;Pa-c&3Tdq`}|_P~;nOWfNjMc45TUbmSHk(CFgo|AVetfTx7=2F*HzEZ*$
zh*crUq;QR#*mMPsC~LCNUO;7xWigDt$CDL&Api!~ak(BD7I1<#MC51o>lERTBXi7M
z97jArg6BgwAhW3s-a7(URdVb;H<~facB(mp-~fAj&%tABc*S7UxuV@4bce2YP}jd5pQ^0B@C}JC4V>)2#Qv5F
z^P9Mbw;8F{EQ5o?TU&+TN;A~i8g5ZVh0`hLShSQ?Ikzlgt;__>dOkYT_LD1`SFo2g
zi_Wbe!Rv6MAqQ_YHXc}?9P+D7y&%K`w6=-q$hFZE2P`Xv>(fDSUDne25C3qInT6*r
zxw1|XoULf@kl%T96^bTJz%cbqh!<(KBD3+q)o(hb$joLT+x)!(;vaxDu$FbLMfS1t4nD`_RH@vB?=ks
zMb3kF3szCJ_J(FUj-#r9xv%#(^lj55~hlp^S^)lod
zVOWz}Bx`dxGM)y0goe;8xk@P*)rr%)>d-=$Ef;
z{QgGfn)dW6ni1E4s1E=7cIW1&*{A4$Psf(BU5qt>=N+AjaY@uZumbr*Ymi+06;0Qz
z8U}^;9&a=$BecrSj$g?@gAh3sK5$;7{6;?Y2k!zVequWG&NiNb1bus0F_Mr8RTZ>o
zMEC#~?5(xe!}-W`rnH^nQain;aFRxK;Jb08UwfxmKc@(8ApXe0GydAhi1h8ZPagIC
ziLYlPs4RMn^?3ctCptvN-`v_E9kg1i_a!8Ec
z2(dM)RgRZ!N~9_2cXt*;9GO+4bS4IbyAhDUy_TG}!|v>;TI3Q*)KI?o-5BKNEoIk7
zYE?^{CV(%*qRQzX$KbDOpkwXZxZ@uY5NiDs@xq9vArp&_(Sbv0FRXFhma(4iLm*>%
zNYK6mKWCg9Yd@^>*HjUV+A0=;JBhM!RAo^sQxzehnj+iW-uLSuB(kAF(zrp&@E7VY
zZk=x2=wUADDoj~JZYr1S$Zr}~uXnpHmoC?vzOL85<>Y38TIs3A95pPGFL+;D(V$gj
zNQw%9ZFpqNIZv{&RupRrm)<46+D6G-2>b;nZm?i0E!Go;GmBKDRBo{v23)}IWCEB
z|HW$ySXcZ?E`OD3xxY%wf4@t1ThS73evCWdVq%~;${z8RKhF*mS=4a{`?diZR*Tn3
z3nPFSeOai(!HE3LIPKF(fHlh6#zOCzQFOdx)ehl=jS&1rTuc2YXQJS}FhQczwv`)z
zyJ_As`4T<0*$CR{#P9tJe!%&&hF_3#G^?ssYEd5be3@LpAi5)ly2skCPizxwmBoxX
zn=VsD3W-yzRPZ^?0Gm*#?r0Mzc3jMN{+mlZJX4vSIdLYAaQw{ko&`xqC83p2Z?dMQ
z(tGU$!F~&s4Ns@@yFk^?j-;@4cgg7VmiG;g$pN!B&oP#n;Pf*IU(y|upw&a!0%Hoe
zDQoahW-6W~Q12xC$%ax!xFxXSQfqW$4M9<-o!CLC=!!bQ0h-A$*RJeyyVg{rPW2=f
zA2UV&0o?AIW1HDWeSlO-Sl%9;w?f8Q`p@|HdGBX?_jz;G64jv-a{}`ap+F(^HSXi1
zi-79D+EMnCBS2?E4&-~l2d}Y~rXPZ03cz>RFYfAvCPs#9*{3E=j9hGbZNXwEnUwjQro@vv3^$tmGdfasI0Hb#BpoyNQ4(A;n7xStB82VsXB
z!DJ}``IAI(D+;zrxkcIbE+29nCWNnHWA@)Dm|RK+->;#u=HRWhC7vP-{H^V>QwusX
z_+~*7(_=m)1Y!tAj70Ry;a4$WQ9){Qr#Kf2YX60OU}Mt
z$QH?0cFdCKKtJ#1Yc5GX4m>_eGNht<{{5CM`ZAKge!_j|rS{z8F~8tV@B^4d>fyh`
zBLceB@$$^as|Jg39c7JVX$^;_J|r@L6F0(v!~UnqnxANm8}_>8_q-Nf=fLg}Y3@0E
zbe}{!wfABUseV#^w;HuPo7&dBr{#S2v}8c~
z++!qLkRK8EIk=hi9)~&3WPCJv%B0*xpr=+-ETeV}p**2!U+I9kxl&4GgfZ%2Mq9VF
zwXJMQL06x17-${hRZNkNfJ{eljs%mpS)gn>LjoYBuoe2q>#N{wcOx43LQu1S%C;gA
zlKCGYcy-aer7lw}>LIiktf8t{J9CL{{H%B<`J?~*n)dlu9tRx`$y)gW^c
zGqg-OjQ9?;D5I1IQ^0H)F8@TaZm%H8c8t`;tMD3Ob_DV6>`d~#?29?3HWW-NWz$Yj
zf`nRAv$o$r5zn4+{R6`H`mINH8EW{KJ7o=H(<_%OY185Vw0_k8bzoup>A(qIWbU(1
zSAA23zu=PZ-Z4pZw}%-&4;7sZ`Y6m+VG&Qeu#EB!uk(lNo$;;l
z1&Xa9mk@MkB9%;5Bg;}?T5%$9h3yp1BdnJ`JF
z;v}cvAsl7vidtj+hSo6$91K+5&aW_GF@wRp1z}ppN0T!IW+vic9!D8~BNNlXgRDoC
zT7tUPS@CP~9fyj3?KMNiFaRLApp=EZv!Vp5s74HzrZYJ!l?lSrxm`h~FCG3+ydA1i
zp5m(ZY?22)6Ny2vSR$XO%9K)CFuwe*8x7cpsfm8Nj+a_un)P!5ocv@Qxkxb|K;^7!
zkQ9U4GHWE1*Iq9gS5_n=B9S@K>IqczaNa5Oyt2<
z?3%J&93I?KP1xb+E{UUBDy5kH%^|+-5fkj}zF{t+CHFOzAjUL}koT
zBTks7w)Rhizd+Fw=}MdA1j$fW5};jFfR_4y8wyKE*6Xju`0_c->9+loxfs|7ykqQ1
zi6ncK%s}g5#YPaOjkL0ADruQ5Q~-XA-g2k!tdDJWsqDO
zOe;!@LOrLKM3Fp(B#MVOw!ubqT96K{g;9B$1LH&7)740K#EtRZT
z|01Rs6=Q442dFZynu^AufuG-_AP&y^>Cru-#XVd61JdtF$`}1rX@)6pI+^-6ohB4*
zs{E2-m4R=^eV%mq&plN?mwx6;?rZfyIs$VVavn_B>mJopxW5y$DjobreBE>hpdp4!Kr#`wUpt-9SPWsgib
zOf1&TBU@YVT?5XW55-PCk`nw3@uh#wChD`<3HN@z5z!L;oCg;+2Xowt&_EG#%KVvB
zeZJfB2d2(_-pUG&Hk&@5mp~TwAFs3b+g|vY>e6~O#|W-~)oOv_?h@Q10NjY*q`Sx)7Hb+&2*ND>3A<_#P4NHc=`-LcOs&fW$n{0fwf)#REWyY
zTa;#1s$eyq!EGChs~HLJ7g)bfi{s}?^tW;l@WeL^L`j$p$T~A-*ZmavT%=gQHzjaA
zQSVg8aJt;xQH}$#S~_vF55hxr$C9aMsMUvOGtK|H8dB(7>z4P<0#R!dEX@|_PJ)bO
zX~l(aT3dp9Y=?mZCgKdxn8Fe@&racO=--GHi1cYq%G;%o=jD9v#mYLaoBbS?Zx{vU
z+(sFHwW3KiSLKZWJw&6)S6WG0xi<0mXt5y4U>^BdUxvU}I@SEqO`*sDqf$h6Dn>{!
z)&7==FuoDRDO^6AwD!Bxmq{U-Nfde!y#mMnqminc%bd{c6J;&xsuLr
z)0hS7ev0WH^yk{!n4fJotbQ~XY5BNnA83UmhP$*n!+f=Oec%sSDU8x%UdH8ubu2Wu
z@rO8ND3tx-TWkL=9{n}Se#=(X1zi7bo!j=oiKpjT8){p>Zn>cNY%83%i++FCTDh9rjH
zC)rbi=2ITC&K|vPP8aV2R+N_-Q#Y0~{*7Dw-X(XV%0qOf1V_MFt;O>Bs5M8MD5F_k
zmRAL_XY};opg3l~i|3JPzLRD&*!*kHLg(J=Yh2aluRQC2N5l_(x9Ul*8{x^l@~8Cd
zRpa~d^p)5^+AH-)jXeU`*hhe;Aohx$L1(`$>
z8vMQ~6pbIRPkc!Ejl<7FX#45pOY?q&UVfTwPLI{k5i>fw*dd)mG9Q2p)%3`HS<#OS
z3Kt`AI{i3H%yqn4}HO!ld_zjEtkU;$Tgc)K3)pJeaFe_=eNB7nfWt3)KfyNKGzLxQ}|AfQ99{+R=^zPs*A&)ra5*&0Fh_-ziPMP
z!qy?{3#Nl}fQD}{t@r_YYF6GG>B)Fh5l@zJe1!wd3>++~~UIj>w73KY<+
zRLiq#3>=G8kJ3?9b6EOOU6Lg;)s0-00;FNLo@e4huZ@bfsXG(q0+8j>Sw?+86^OV
z+;K+JW7LVtKW-*q=AK*drmwf(ZRoGgl7%Kp5U7T~c9dM9TDzRT?bO$)R?CT7C#L|q
zkQ9bu3yK{5IVs%Ezu`^3wh@1a?QVB+Hnt}4?>LhKYRQW-c5LqpN6ps+!P%p)`O9XKxWw1gk(#y!WPa^w*E?^cJo|8CWSN1#2!Eo9w(3|xM^b`iWSOO33{WbwT5Ao6*o$vA_imNs1&B)!|c|Oeqa7GzvaFS
zu{`UFp0uwWm77p8Xrz_f2I)_$lx64Wh^u%$*_s_}PyAg25IbWS=U*g7E`&z*sxelI
z7f$QWIm}ou(SJuK2u6Hr)`zy^inN%M(&0DW;-&{MG)Bj<11sw{bl3Ln*_|K1r+p`|
z&2DA~o!g%u$7WT1mn-#mG*+hLgE}1VqE|MX
z)J${J=Mj`|N4C-=3#wSHY7dT&T2~GEBxMn;a-mWg*Kv2>@}oYi=+RXVUYdW^S~4Tm9lZT_V^!GC)ElV9f-_u&AnJE~YM#`2&kv(^&wve=mlZQ6
z<@}B{Px>|!kpn|H%!-EXnO9;xGn}9eqcONSx?wVJlS{Y3HNv5z
ztH#>#InozTNED|4Ro+65M63Ie7q;XsXsczSRo^9a5*EaEJ-#8crq*Z8x4n5qdDIp1
z-^u|BE9;O5%gd488!I#G|4CY4JUdx>WzxY$cBrURG0&1LUpQUj@1A^UA<^opcvRBJ
zn5hDfeRg##sz+C_`LtKCXOK+FS5+m=+@@qMG^MZC^vWOczg^nQWb&993$!cd`4e-_
z!KD#?1{Ju8`*rVMw@)X6s6QQ3P+pOq5TSICJ{If_CJXxNH68t(e++>#R%|LI<{!)w
zoPe{0Pg?h=2!&O{bgA^uWrg46aT%&vz}+rMj|8ul*1@|kjeP05{8kVe4UgE7S(tZ}
zH2nIe-caz$2i0c8^RMXp%eEV@9u>t=DM;C`xJIH4FY$is
z`DB(iQkIPUvxBjvx!t(Kz!+?F@-KM4>)h6E*%uuf)rvZt*EZm#qVJFEZ=9eS>ZGzI
z;FzIVO?#ANji~qV-q1h>iT|)=eUEPO0f$DW$^JJt^!3zAdO#4pFAZDWWD9Q4m!JR!>GOhx`sP$dTeqwvTrqG)2C|}o8*rr-<|05~j_ICVk
zcQX6zaYH`ED`E~$N0V|`daO~2+HP#dcNqPO%Ygjlm;1Fplz2e7#nUo%6k2QphH=uU
zWr#B0Es+baS9RU|n)Mp%US~9X>JZ&{PB)|^j(X$BOZh=oPAK|<+45|%?&*?bx=>Bs
z#r%qGSMhC|`HB@nG5OI|QQ+SMr7?GXSv38pdtskdj{PJ^ae?sf8B{t6x(fY}_)ir%
zgDjrrKZZlD6lj?!8O$cEgXp)Fb&{)UfgNLB&e?DU>>5iE`e}^;M!0U5+vhvNx-<4v
zB|lje;j4TVF6aX4T-4t^)2Ag#0c!pVp@im;_O==Uklv3LFhQnVB=t0)r`9>Tv|O7t
zxm;6zI7$@!%_2>~A|PzJG9Vj&?EZleM1YegXPt14{ion8(A}!maO9PMepC9kNJ%=H
zmsN(jK8UGkVmOvrnIM_M|KLyK55h=}B)C*u5y&P}2F!*&qZwZov9xKiyyiVz??BLi
zX^lz$pk1gf^ZD`FNt``Qva<4g)`rNa5osb=bb8E`QogfgO6jdLOYRl~TSHYT)_
zHznd_ha~hlr0>2&K&24MfLQ9^=(OwDiObYhBwhA{G3qZ~Bm$ZsdP}9J(
zKxnbdC+!V7nAmsoSM_9U;4#D0bV+{m0xRhsx8n64596t6M7QMa$uD5(-iJqw!^3y3
z!#i20;2wd*1)Jv|D46q)CtjQiM}G9&*hAACB-97}z^QfHcvlXnxR%jzgDikecrk_r
z+^;Qm>C6!I;~kC%C9~2w{W5viz|!W4W7jQN=s6&br-8W|+~N20yQ1Tweg;62M*8d0
zwZ9nWUM0B@FPaTV2fW;b%eI1BKVDSmB7XgiNKBPJ5}Kr38ejZiVf7Swd^4mrFH+}S
z5T<3CoC2uxd1d^0xIUv9U_Z-4Goz~lMw7~WUEnId`YtjB$2Q5-ZX470+c%fmfVh=A
zAxp1XXZyA$6Q_B%<%aS4t(1iFc$})XDEkAtsE)%ph%$AQ@RW#S9U?ye6N38U*uY7g
z#vOk3;~lT&68+Gz>glW4csB|_g(IpC^BsQjYAL79a9ruNZ*GDZ<7^}2B5>fkD4!wx
ze3gp8llA=MKO0MVy_X8h8VrfgbNrebV@_&D<)AzVskT3RP5CCYqyu|~-N8YmQY|~C
zygNJH35yHyS+uo_Qv%xs04wF+&)8oZww-D=hWZ07v_N0XNHPBIVb{d<7f;T+5N}xS
z6p_-YP8<-#a(DV$-EhJhoMbSYcSANl9T=P|fZ2}I6mUtEQSbNKa7YB(HXN#UuFn|r
ze0mVQb}i3peU&YOgs{vT?I#40(Dggxefe|AlmDg~DsX@ER^-v#$qH57$~DxP=BW&2
z%u5R$Prq)xZe3e*na;oPOAv3en@Yvs@KqVINmjxO2j&}wt~N-$WHx>Dee!)J7q@8@
z_tV?1a13b*bIFDtlk04#4MRhd7W@
z>Ttx#uA4hZ#+Xc*Zufb+>&DJdzhzs+E^AQf%a*7Q=3No}(cjV-9W{pipiS#~5@j0@
z(w7}~s5a$kQ?J_N!rMC&vh6fv$t`#NIR^qK&s1$Z{MkgUvzw_&?oI=*CCtsDZ;dV2
z^40UhUvE+KdA!JV5&DYt=jE6KD;wqi4(i_c$zziaC6(b7laNvYr0Hj)xl}Gbktz?(
zP_ntcR}aTk*%zJlQO+QJfJLGkQ8|8?DIJ?Gxa+f+7Nn}c?W8%A$UNXbdxhR2D2Ojr
zD+Bhx?^qKA{H%H0GIrrrz|Dt+?KK?u;E0XNJ2dfRn9H!8srU+b-O&
zMhZoM#uY}5{-eZ(4;W1bz9gQ8T(XHnLyawM)>Fj4NfY+sku*d_^_72v674i}~6uqsVfMW|H
z>*`NnEosqzM^Emao42SJYtNcha@mc1xZVwSl2??8N;R83xZxDIxk+%z@DqPgB1mXm3a2s@f%NU1$LBTW-
zg*5GD$Ww`-yoXLPjmja=D1Hg?r0<2_8LNorX!GPq
zsGtXlzc{>YO1;Z9?Q7uefi+z9y0I2o04I-f!yg<4pCq3%_IAb1H*a3Qf+O*d$xU_R
z?2=N#VP;S-sI*`D*J=tlinaL78Mj)r0uc9XQg@(mxmuS^hYerd?A^B{vtpJDiLJF;
zLF_9UiC=Ssxp+8Kig6?gF`DL)+-Dm&jfe4C!vJSf~BFcM>Kjpfs{5o`d16QPfGJ^-GKPJ#sIzgaxlSC5DXK7Z`7;bxF
zmir0T-5dGF@Ai~edB;0rRrnwC$@0ean1}>-dK7eFd0xtgwtRXlV+Sbo!qdp6Z>%aF
zhI&*D!;RG!PN8ZnGx^w*e;iVdef{Q$tQ%pK2@1(A)}))eD@G(btAAnIM9evwH*>W9
zd%hJhV1EwrUNm%OAdju-%i#dOUKfa?MK2~ab^o3%hI1y+bMify?zy}#HK*Z^exAuu
zR)NrS)qD#e^gvLrvnI#eHFBQl*t8rpzADH1RBy+Twl2ST)6q4*-J;P|==&W~$yi0H
zRbaL3*4LeK_c4DTF~vg7?iA7f&4MVSm*5TT5AZiHGiRGr3lTjUGog(2n|z(Z;N3a8
zg;MoemwZmRW<-6EE^1+|>(LB8#`hSd8QC;n4Y^?rN?#hx5()R#Exs=L>vaOP9&1B)
z8p3v*1V#a-!ZGHg$*svat@JN)MJ*o!Ym&>;z8&mn5F1uw34T}>-xIz?!DqA&?+#O{>Ta;MH_f2{&6~UN%bL%u7o=_{K0kK
zw02XKrIHe}=s?Ow{oIGENcJ93e^hkb8b8BTp?8};%;ehyue`G&#m#lY-PVbdnF+6?
z!Bd`Z);ty~%ok8#8cs9?L`bY^tAw8`o?Bv2Ij&qo8_K&1!(G-;`Swn2BOXLw4F1j=
zmQ*jfxg5w*h@kwAIKfT3*^*s<{#$b_$SkXa?wFpxQqSviv(oN@v7|6xU#J{!YhEwWMZC^5;4Ip%
z1Zcb(?(e83n?7u-mH5Dyo2N&;xtss^T2P892LD#Sph=a!x$_N^opeVf^ZkXF59Y1+
z_xY!^TWXzO+0{T-xbIhhcY*fP*W>*i#KE+M8W>MdqU0SYvkPD)s^Ffx3+tG>Pnqz0LW|YtH_8x`cz}b9xZh_A
z)Rvp83GxyvA}4WjVa@H|Wy2G(=c08gk@vo8TH}{q)<4fLaH~vm=nFYR7PDOke>UkR
z=(@(cd)M*|qT385QSk4%C=>m6xR062-X*4!$@|v2g10{SZ&vSDw!yWZnLh+2ZKZ91
z(5bD%1D=P)w312&C#2F@L#VWNE4Ng;N^`WTk{=+je!82o_zaND%BzPG1o7xoZQut%
znTaK^GIJWl%m`Jj6!bU1-8u&kHIyte1F|o)a;The*Ku4WF0;Xg@77VsFMaU(peZ&}
zYr0WCCq1KhRAh;38K58_)oPrPXvrGJ^bPv=b0}o@s9duAcWbayl#Ou$0$nvZti)lC
zB5JznV1}4vc=6hc>-Swv>{490?q+>A|L?d61vwTTXmoKYHQWA4_2RNg%{}J;%1c1r
z(@Wp#rT)23_#AE6Md$|#1{PgHP7eOy;hh=YUj8WPAN3)HOpYW0lzTOdGHKzmwT<@9
z@u`d-EgV!KU^MLBk(+<`C|#?xqyNFH^xV`lIwY#0e=af>pUH}fq=kI}h@sgOX?&FB
z#q2s!10GWe2TKl1GaVZ#;noG@r37Q{o##yKCVO*J!UQuZeb9$3$CqOIC846PnzRQG
z81#O*26V9ksBY#{jm^Az;dQn&$fp4~&~q}>$89S3GU*W*?EEvMCs&NdrMo3=^D$71
z4zoG+DUSMz1J_e*XNSM2gusZuRbb`^kdYt_-@GcSL%)W^)w41U;Mc*
z;i1Ww;4D8w`SU#ao_$^AJ)-)^1iXmVmefTv)gtHVNU0vz8@SAMyJ#@*Hp#&wGnS8<~WmTh9`?=<{mDsF15
z1LIi|?BbuanqJ4U%zS1$de-tyl8B3Ft42{6lA8=sbY3JxZ@62U?F|Wze_m7ie7oD$
zs5g?SVJAc_6=s*4PM1cz=aHNErf=}7l-8EM_A@q^1IYTt=YCIZSaIn2863Zf8x4V^
zC>EKfD%tSpE~U6-ToGekm2s_Zq%D?&EX6Uu)~QTYL@O-IFthqPhb_AmF4NeA6=$=2
zmfr~1MXM0ViV|4Kt`#NxEi)cYdJm$AYuraRiNjA1orb2%Xd@M)sKe_CwTa_;w^!38Vr-!uuQ~6i!
zz(aoa4^klH-dMi3-*m@d=P2;-61
z)GoUAwVpDUL}c1ccX52L@?UbVTdIu|Bn)35eNwauA-L0#0`Og>Uku{Kp!UUb}@2X_BB$Ijn?k+!j>G
z#aJjgAUlLht8q*X%+l}n@60%j-}5C~7Tvg+Q*QRZMdh==2iGxMxAmoAYbz5}SvND#
z`}p|%y#aJ~D+97lw@8N3G6KX9hoqXzt-eEg_J*g4Re<$XW()p8Vb5m`!2Q1)54USP
zWTQi1%mH4mH8AYdCW_kKf@DzK?#qd@#Upl;g+<08p;Y|q$*dn+8Io$beYjjn*9r6&
zxXC{wOmp5en-Cqe(8GcO!q!7c+sO9R_)~QvPfY%u4mFiYkRX7CN>->hp!4X6U;~DwplI$dws?XI?eN$B7hEX6~Ynox?x3MpOxNd5Q0333U1?QfDRi!h573XBfXl_?y8{WLI12>0raNo<}YKYMXTVEdG(n>U?TxC2)=l;
zIwRBaJrShda4un~7FZG!vasAF0bB~486qa?l}ks{uL@w=!lKFx-G%iKf#}QmZc6sk
zku0i1gHPmUkiW8D^*Yob+O%ODHoH{{h46gOpiV|B_zYzPPZJ1Hs`2PDUDqiSlzqIe
zRbHrC{bzOcMQewxK;-Y>2S*;N_x)=euXIhz@-64@2>|iPzLj~`xYk;99vL_qsqvb`9Rb`mz`Fs#+BDlR4B_~=xJv4x;6J;W-z`C9r{IvHi#VoWM0NU$lN?u>8x
zd98pMJF~Z0wFX#M0tz!0F|!_nZv7XCpFk>GrkJo2W575orJ|I5WWQ{ZGqg>&Unx%$
zLjuI|7s?4=Sl-RN?e&>UM=$G-L*^VgaaeE|Rs&fDL;yzi_K-7=;5osVk4v8U;&J)3
zAC4BV4`ZO$3zuHI)HKRc0(Fkvv#rJ&M53>D8^_LG9k_u_5EnHt>1yGOdztZvcx;`d
z2Eo=kO?HS3v=TJp)3Sc&Gj!3PCaLqVUn+y^a+G6jwCDC;BkhGNO>C!nK=jk>wd2Hy
z`rZ=82}v0CiR-Qzsiw0xW5@D9?w&Ua(T(7I-D8FE?4iWu2)
z98+~>b+ZNOO+7_lvd8-0W7sjDkiILLUaY)tw`8pkzayX4X&uk+EzBFJv{(7+ym9(>zpIAvcQD3B0RnK%B&Zfv5DN|}Ybu(PO=fgJCki|0H%7-#I
zf$`+>9L4J>Jj&TXSwKln@A?T-`w?Tc#m#;zL#w8q6{nIf*cki1DrU-ktj10;VJC~l
zB)jM-Y4v1;GIP@>Ge5rZj_vttnHz1Q`B>z+`~ywm$D9nPTa=T+LqH`uZk5n3*2FY#
z-{WQ*28yZeEZnW|O3rj`vZhc0A$up60=w2`PqwU>)HNw5r!ZhUdiCwB#e+b8+3=3B
zpsb)xp%jp_&<_m-?X|t~?{fd4RxDj4HAYD;grOE*K$CeY*)>jZdLaJNZ}^Bje6@`K
zSHN*iBtbRo!$tq=Ovt39l+L@*lD=_iNd(KTC%L@*kHUlc1gYkQbLNQgw-&sCCdpOt
z+_CIw#&O2P=iw2PvBun7R{OZ5EK4%Far|p{E|agAlV%;?EBIvj`0j2k&4@f6jnIy<
z)4C0qTIuCp=pKaOgBAdvNrzNDfst9u2JcBXT3p(iaffKT|Djw%F!cn_UeoSDmY@Q)
z&a9I|#p3JTxde_59)0VR*q#Q}2KL$&n#g$~QBcEs6I+L*w`%h&U*uuz>cxhISdej`P-0a&3Iymbhg8JB8^k2myrpLjq_%S)V0AKS=bhs7RoAn9i~kFtC8W1x
z$CTZIcNqH*XEgk_Wwaox<3sfSn5rVs;@o4wdzN|c-GHvR_)BCe^*7*YbY!j>t90>=+LXeP80wGm$bF5H!>Ex{uWTg3{oW=s-iSNZCo
zSIOldW9$+Y;~Kl+*7gCXE8eoP0|Mt?GO|hfe_U4_OCcCjI)42wkCb&oZQuxH(wx8e
zx-Ymgf1v!ubUJsM+jv2{=i9QTu0{WO7R>Ni1LU1{g+oS%}OOZ5EiqKRr`#onW4?+RtK%md_7|^(AEaTY)QYN#+-lZ7iz{||>(o|#eR^`!&i9|*ta@I=JUyX8&1_0PS#WIt
zcf!Ys4!C(o4J&}|JU+>ZU4?RiBf`$Qz0@2sR6JifH1Cd4!aX2Y&!
zG2t90j6$nrxo|Fa{l_wVzdN3)PgQw+9aP*H@V!W%{l*+%cVDmsO<{rahEoPoiHl{ZH_4M!6hCZ;)Of|)2K}t<|InepC!Lg
ziF8@kcRord{Fo2vcj~<}?yb9v7;)Vj@|Sy@6tU`RYYfkdIo$qCUFdd|PaW;%E5~=K
zl;#7snC?8@H%Rb#Xj3FNTg57ny%eTQpPy1!$Z+K!BjMi)LXGV1rj
zjXfaiPclHxo}h3(3cGQ*L=21<%g0IU(i0C@5-bRrKmUMKv-K(KhLSF|0<7xQLh*}x
zd373m*phl056{`F~SF|-^VmY{bXcQ?{s7sNOETKmmGYmzS_S$?j
z^r)9u!bd>vPDJj7)BretU&VTVpnBg?gRs0v^0TKQbU8oS&9vE5tle)kd-uR)pnQRQ
z7Am&OFs{DlWgC{Q?iO1PO81n$htTol+S>n!D=uNuUo&aL&9q!a3B51yDhdSjec?UH
z6fcs>bd=4C0-DtC#7HMVaAHn{ENwmLIt9WHzLoixz$TY(VzX$mK`%cO3DDt?Y=?zG
zQwja~QG_$y=2;_a9qr%~WnwnE>ABvmHAGylA#zHIAAbv)}skN1RDJ$Nj
z+)MarVAKxQ)g2#%a^vFvC@}s1m{|K)2A^yG3gt~nEd6nua*h&rjpy}b6SU*O0LLaK
z@|U}Qgg?VI%fjPlAJ_O3C+}G#L;D+*D$mR!Tp>!PlL4(mvkhBsk_C6@t;(-pj=8N8
zzYA9@E5zZ^tOK3-UP75zn;p^Rs>zel`AsAS(=??1z0VEFr(OH;epAx#Un=}{cT1EI
zJ#!+e(l8p&nWveSuNdGzr&K*2stzZK4`6yVjOggQ$(xs??3YV=VgLKLCJI7-iRm{r
z#~aut?mnoX0cmufn1*+yGs3=uBKdU9y4L7g%ZUh89ve`Q>3q}9
zowFiC#B|ZxKc{CVSCQ`tNU^4zMmPE#t_Qs^Lh73982mr~>uO3CpzFdk7G?+4pBaI{
zr@$q!#eo{xsNs5VnU<&Z+Kvy8xq)w>^A|9r-uqyOwe$I^$`Jv}l;Taru{%VJN~KLd
zflz2QedP;Y_0b$6DPr=w!)PNQSVNq
z`M!X)e=vuu*eTLLTqE^cmZlLpaw+{wSWb!*VyexDvdr5%)ZCIPVLZoeK6M#$KGW)G
zuCvUn=lxePKT3DPHH_p7eUpRAy0-7OMK_(ES?6Z;!Gt*);d9J?3ODOl2`Skm@V|hO
zehwuo9(lEDWUsO*aiO(hAt<3XU_lSLdFVIqFb8Bid$PkI$oUc2&N_e;0WQbZ6`s`~iwC6xWRBGO7GP|H(@s-+Bq
zfip+r9hKsCU!SDCu`O4%EcyY9Yr#2En7YQQlBe?QlX&
zj7#b)E&KC!zs!Ob$s_28FSK|OtxHe|a=2fcU*_(DUL4LRpit^f{aG2yhm?ncNooj13#3shl?DnEfG4kzgmB$IKAjWNl*5J&r3AN;9m
zI*_^y@wb#o`)Oha!TU45SKlELaKHSOf@_#w08#;c1qj3;cY
z8+of1#ZJeydTL1HsLL82YTnoxrQ*tw{L;_L3)Z9)#NX9UmO|$my5~}y7#5R49?2{&
zvM6m9E#&NKt_*L1XIPU#FD~_+nVU=jW=S2Pk_+%@>Fyhx7w5W3@qpOZ3Pkfg#&O%y
z#;6GXWXAM;9*}GeX~OJTPEX^?Jd>JMe*PCNHIgo^I~B)$Zat``;m~7!w1c=oN}Yjezu|gw!Qs@;gj!`xB+v=#jy}l++NYBZ;oZT+9ZSxQo#)`*y|DHP
z!-XBbYb|}^q!DzBWcjSId0?eH*wHp0!LaHAaK81dSv2fda#SR5kzmvJMn<*ydh6H0
z`gw9FId5FMhAO4Yk9{D5%j!%efuPcL8NW~qiATWhrZNj8o81Gk8Yea$C-~w3GpT$aW7=A%frWG>9UUA2=cT%A
zPux+nr-10pDkjvDwZ#YZEg+M3(;~Cpf~9K`N3KSK)=>%dK56H9q^r@hGIB>!t$1hH
zp|L+Hd8UIvb76kumq(%edQisvR1t~ODkI=D?6O7sMdtjAmz!9w0>{4FA%3#(!8{UZ
zzm*r?g~d;d>9snc!vz~3stFPWqu#G9l0mQ3sC4+rNM>!Nr{`66fp!A
zCL3ZPt|EF#jcjaB#cW2Rz9*}I^iL71f`rLk4C#ek-nBMb3Pe54|7|?3TLcF8eq%xF
zf9`%|rs3@A+5h}ZHYfOEP+X>B)E**|0J1GfdL&`Vj$WB`Gxe#O(+0AIQ;^EU?(R`=$pe
z=?IZDb1O`GVpaYE9B$$jdGqn`0h0&j)?LOp#@q@NAl_eN4_i@^Uq!8;Rpe9Ia!XjN
z=WAJWA6(kzPfJ>I{E)aZ{>e`5&2vqUKxoBxQ2&@7U&^TY;n5MYdEssk;`g@~g}iQ&
zrTp`;A!B_hHa}m=3=d`nCUq9@Pu#vwh#cW-alIF@%ks-k
zZz6N@x;Jd`yx@3vs5(3CW(FFN*6KucYwVWy;L6%orS!$STijvLCdsao8q5=~7m-7n
zwU>8td6!dwXSbtg@uA*6UX_bBB-;HR`TQLZ?bmR%&g%yH<>m(SPT}?L&9(Kj
zX?rKfyDw&*VPVuGnv^|tkp;$sDsH_BB0IBZ5Rc7W>`)W|9YA*pHZ%fqI!aXV32F@+
zp$6!zZUmKSd3^OxtlOsp+vnEe#rJ$2{-mK%3uN;|lo@X7t=)fVvRi>$Kjr^ucjDan
zndV%ZeDUutP1=GPhR>4@$F)ktaY;GMdy9_t82B`1)Tq4R!f0}S=>U1NWv&mH{#c?ckron?yjsE^cfwJwmejnK68LG6jw8D+~
zmS@0}poB+L`|fF?Av)W-=trtF7A@^e%F>>+T1Fup*5wg?YWk1W?$DcHHxu-Ou{b9q
zj&xMbBbC^k4BL$73Tl4;wvcJ8BfN^9C%CkKbXvuaf4P#|)k*tTUFdwRh?SAU8)#kLTx#WUPipDr3Lm>c%)Mwy~>DM0c(a86WO495q$-ByD}S
zHX=SLJ5{8XB8yNnQvHSvo>J=eZP~_rlXk-3QCOVyGap>1R`uWO#svCsxT(vlFW8Do
zaR#Ep9+<`#L{m7@$N0Uy`s?n3%M@t_&~KBYHGCil8`Ay13tT*D9DQ0EX4
z-=U~SHKjTfl40r5+n7KEgGC9UQR;?|kD9O~bfOw3sGhc5_icxb#raWVGc2CW%5Di~
zuP{pVo+f;}$Mqr4FKwLV;bWf#Hm5-XfHq1$S$g@Tm}U}TI~@r%)y*bY*M1v8ML)uw
zr_${K{F;qmnFp2D_RY%IdPQ+EyENuINvnI27fJEge&usxTzc}m-uzc*O=fO0pXPPM
zWks{c=1F&qYc!uxuO|-d3K63Sl%=n#Cr>*RguIVy1li8+#8gNjNYPss=)%)r4%!y!
z;#~PU_CYJuKQsGh@U*m&)is8^koeQU(Zgd9*)F;{$w0ysU(cPY)~LN6K038f-PSfJ
zoj2NGo&a)Uohj&+ifiu0U7`-}pw#zz$eBp%-AhSF!Nn*vIR`em?N(XUN^GQaL|?UT
z7}xCF`HXE_i<^-O;&gd!^P|8*=@sE?ow(s^dQ?tK3oBP{V}=OZ&r9$!c3~D%cenXM
zUzPW1(Gh80#pQh|*}+l$$u3D()N
zvLhVR{PmlHG?~LRwk~aJuDqVtS+;OyK%Bj>L$UoqNmdMXivU?2b{?B_3zaN4!ob&9
zh)9ndE&^mK%Ew5x-kzuznnh;awPm$$HfWF2yDsr`kmc5`2G8M
z)a$`+Ywv6jMoscT4%fAf7egf?ewtYLc9;4~ByHVdLhJF?3|uVA_Va)=jqixk%90pW
zO0z+M?|ONGKrxh-cjn{u;~7%clr~6DX24ah@`9AGZm>Tik4|Qe|BEs4|HX~}_=4!D
zdN3?92}Jg-0rbpJQ%Pvy`AtG#2yE7QIHJ#7CUd4uNvjk+Vm65!P-0>%>LfofG}jQV
zH+e=(!yhRLmlqfK2$6|bF3+(}n~0pA!}
hATA#7oIerXtlo)q-e>v`?PWe7K?6};-i#q$
zV3=gzpM-vRpoZu0>}=U+vO9-cpXqrSq}Z=FxwIe6uu{@$X~5orN(QPyUlxpv+l_uQkI#?ql5Zy$=Wl;L(`oIz9g+e&h3bhSe|$jr=~UY1@|oSk%@20sf7ABp{<6}w
zv)ZAR_MzPu%-KK5f^ez&DG2*PH8W>qBexpnHN*B5@VdY{L)Qc?(wdRjXzBtxE3ped
zt4>ozw_AIVCqv=4_rN=M%o~9er9s!-ajd8D@slA##9)F+xlwt}1R5<-Xpi>G*%oD_
zdvaq%UgHIJn|k2{k&C%+0s?Cc-a3bi
zj)TfJ-72JKv6klH_
z%+ZX^Pb%0HV*O?tr%8vROu(z|TPj+)Ro)NUYi*&Y_oT5inQJ^ZpR-7=)mOGmo|y3W
zv3_O1A$-Xw9fUcv5qr1E%lybKlKaI=H#6U}h24p$arx@M>3+Vu_35gMJcRwl&^?=G
zA(#_Og=L>`e2!jNFT1*Yz>CF@MZqLR=Ml#oz
zHn()`n6`dn4E-W+%SC2Da(1B%X<@QhZgu@jT($OZ%kuR`UhjP2--Xwp&DK2cj^$i(
zUyksoWtq}gJqn@pKV_2`4SNLGsC^)RMad0XgQ85_dZr1c0scrQ`RLlBWpbpX`l4$i
z>5-KSAf;l=SnWZ@l$-Xrl-A%c!YH_3RrKe>`_t8NfQuRGgjwcT)G80ZGm%LogUq*%X
zABt9#f8TXARaq`k8DTC|esAMEneVO|s7*wg23H{RQ+MZhI$0b<+hD18I!SlE${7&O
zVx$CCzFt39|92C%@z&}AO?w_r%|9h+S)l=!YdcECVHeQ-Vv%wlPT_dd-BC&=j)26<
z;kbQ_f%3WCA*$6*ZnN?c+}zPG21*yUwciM?933v*u&(*(=P9XSC)NgE=Rv*7IjO8yjWYVkqYG*>MHl*Qm{hXlT?_ewu6j8tUpg)HFuk
z7iW={w-=@t$`ySL1xJ*lquew9Ex7t9XaY9tC}{@Tc(^s}7&qiJR>n#qzNJuL*^9pw
z0?^JkKy#BDRw|1@^qy&NRC?gs7mh3Vm?~3jlaVe44zi@v1FiNTIE(l|D~+sA=j_r~
zOd)9l^$e1oWdXo<=9QVff`LQP@0WV34F|0kX9(CsT7Qm?knZ#jT(BV&2iW!Hh-JFK
z87O$;It?vxkNlLndN5m)Dwj{&e!cE>BP1{GPrz$ckUr1jEUrK-S_5gl!R4%(fCY3Q
zh=r(RaSwj#$7#wI-P0A)HnLVO0Aj7-$0K2(D-786*6H)HxGRF(u9(|WLJa!&wo`>W
zo^g-6i|Ld#qB42DCnOgQIV-#+TJX>kMl117zk8cCK4!cv+NoGi&VrDJs?0WHy{R{l
zH1{=?XR#Cuy@zI3a3ECTp_dr_tCw
z;kAmFoq2O?q-tylq^rL4u8IRT_3tday@3WmV}&tGZZuG`%TyD@bwG`a^oXch&|fuW!o@0$1B^+>GUwW8M^&8@I2+3A5X1pxE^>gj*;{t>qu5mmf^m0eXi
z7=gOTKz+;UPEwqQ3w-#wf4Zrb9}TH6(KsmZ$TTsfsVq?PZOiCtQwpAXjMeVWcVcpU
z4^gr%QPbK_n49Tsz)v~A0)~wpx
zHe=+reo`4RMOw$AKoLE;<0iD|kksa@Tc49aAzLE7CF-rES3voh&Srk9kcgEo^ND^N
zqmhR7u$)PSu`rH7i{bf^z32wgwwR3MQmG5;z>#-aw@Az4`B|Rm`r(Fr!uBhwFBQ!7
zmEwB$1^*GBv-tDoC5hN#~&-fEjqP~AD5=J=p<
zlpARH0#o4SUlsCW?|Ioh?s2=z(MoV8?SColvq`-6mmr8EBp2DJVb5n6tWwFJVoCyY
zGT;1!Eon>AP~#UJjWMyFDwASu#qYP#z*azUpAtQ11#GrcnV~Z=;r#2s&*piheF2a1
zwIdxK9UFq%xMl;v+*?U*bZP3gWb{OS6!nD~i&%pG^KBwSb>gwSH8_A}5nN
zx$|1PP64$%{7bu^Wc6|tr+602=b=A$i%)^9gvv9DfT^37xHl4SkN(0rph#z@0+bx7
zQF_^iMZJUz(ZAR$VRQ~K=+#zsLVL?zva+*Zu*rcdh*G$;x>62nDyOAmn5A62vyP&G
z-idSPqJ;*Yr3TlK9GZUUZo9z%3sI;Ps$~g5i2&|ZMv0eF&KHUwZP*Q@CYen$5WR%J_rs6~PX
zQsSCmvg+^G^Pr6BU=4lJiuO2~6*r#_Yz--eRrCHojQv$un{C&H3qMr~v_PRal;Z9l
zw8h;$!994;Vuj*vh2q6MXmBa+p;&Ntg1htQUH|@C-)^>Z9rK=Jj&Y7x4;ch*2F-m=
zLq^D1hxoejIGPN4L!y}pTGRZU-=Ua2hq8V=C>jDa=UtYnL9jEi{HG@UTKd?g%;zYb
z$YY3(f3~kEm02`ibiv>Fuiva=@-6p#=B4X8NF&}WADUc(%G+Ng$T4R5FWZDYyTw$XniI(q&&I@^161bm2hY(d{8D&XwHBAr?5h3evWbG
zRb8CcV6RZwG4QmvmEDCGjZ1lC-627;G*S~q#m*3cvdY}vBUTlI8zV-b
z+gT*IvshFYH61}n)mPE%!1?I)#hJ(9pQ
z8o@tbX3#WeCj6!X3Cs!07}VLI<$D_-XrlLVR<4b+uU}ZgS>qt#;OkY*$=)sE30@9(9EwX0hjhfiM{z6#~ye
zVHPvrU*?lckGas_AuS=!GLIA2#w`gEb@_9k+9+xqP`AKTUA-9)8_2+rsbbb%?L!;i
zx}cq}Fm-9vRW%2wE4)2QFi9$}{cy^!IU*zM)Fc~SXgT^zX@6L|*`W;xL%
z3D%ZlFMVQ~dUuAFa1Pc5wdnNh92Zb#jHW5c0&nux0^6v5EBBaXu>J96(OhGU>%hL;
zN{{90uA~9og&wsx+G!L9IRS_eVU}~aV!d%A0ee`Szj~qMRkHX-#m51QvgV8O{J^kt
zJhqz?9Pig-j`@OFCAVQ7m*m{*R7ql9v^0PdPH
z*c10Q$1QyRGfmT@>?Ede+QIp!#6t0z(Qn_8d)lDcyXEbf?S-*(!28V|_Rn)G9&i2+
zTpZL4ebtFchB-+S-jUQn*K&OQINSnNs;Jd+y_jEH31TqiDCSF$V^?w=0+Z2bsW<3i
z%?Mo5!h?N3=8qM{_$xK{l$_Lt;m%P08r|`AXRb6lzkTt_Ps0hJW}vFy=#2cmcd>AA
zOyT0>Ie^{q-0-j&Dz>@)kg;XionHC*Ji4;A6Wnm^Jj7ASg7{kahL#Oy%MVnSCnXn+
z$QrOd47*+bTsF;!;>a|4JxDmL+>pkoh_0WUcxRA_ZFU*2VQ;7ICJ42hn=+xWms^G)#?~{OsXrGt8D?t6
z@(ZS0$#qrjbI?Tno&BXCcG;dwws<6ZOc|SA*^u!I_UNHtOiZ|x+7Y?RU0QgBl%j5Boi~2I|K2%x$8HHmEyNg=gR-&gP(9IDm>cs
zAa1bDv;Bl#Q#mbdzohU#cIe0*0j)2Sq<B23udZsBwU%%j;6FP|
z#E(*E^UjkhV;0j7LnKL%=zv;CzCxH^I$Q$Yvd%A;i|46o%`Ez@*-h{_C+tm*xu&qW
zpnVC2kFTVTvr)02);^9m9`I8kT3|r1_eAZ)zQu>bqIy2`)gBuoe;1CzxBxj;r
z=YHv`kD#g6)U|eT_YkKZ>SlW&8(6
ziTzd@4$=-so`zEw6CcCOx91R8IX9E-R4%qI+cuWtH=vo7JMt14B;WTXxQIiba>W;5
zv;a-2|0fprRd%Pa@hOy=KJ8~j`#aJ*DNo`y*~z1M1)nSzQTBmycF+7WuzTo4<
z6;8Tb>7(!47RcXJw0r%+lRg_au4}5V)V^iy?6Tw$Z7KG>iR}H|>oREB)|bYcI``)w
zZG^4l;6;Jx-(p7XR*_tQX0iQ2Yodc)Ja`Lfn)h-;-l@?%Z?P%c@B^w-@H{3&!bf&q
zG~lqmfqx=E{8OsYS{f`F6)oTLiGd}niJAh)xidYfn$*)2<#1no_BWKB*j}zb)dP@(j@;u0{VeUkOudNE_&(3Dpil?k)OMRQ2Xaat`q%no7q1N-DI)a*>T0
zx4|l}GGQI7qtYNz69rsU3WZC4%jXPH3cGmt-`F!9E#%8A&dc;#2)^p$m14|*2~7v(
z8%N6~`N_?k=Oo!-XaV?5tdsmY1xDoRwSKmG)dS+IQ-zUpx?hN1l)FnwNwvY3$y
zrGsdi1gA73|NSZ`7@cXFEz%%9z4y3sJ%B8_RYGW8P!RXZd^9cXLW!j)-U`Qa9vh1T
zPRS=7esv9=xiYYmEmHAF!mo`6py-+l*6t==%U&8mI^_;U9C_8|w&ro!FL+Yp{FvIAi6OF>j%dOX={gL%4^!ce{
zO~G+dBEzJuq_~3qXMR*HJACUOzj#lsX0-a-9UkN;r%t3!qkii*Efpesos2BuRrpd7
z*=J}~eczNI74^QpiIPxuFHlH|C-A?o0UA@VzWEGO8ZvS=&&TMUhvf!}llzWmg`EJZZV2)jI$v{o<>i3L{
zkE>gDc7|_wc9wr|EMa9$O=y;CK68Gv@%h6mHqi%ztmm^1m)lhtps4S_`OcWv-9tu;
z$MKot^I^vtv(YUV@-l79rPoIkJKmXJVCSEH=)2~HkN)S10%?|HN|3@0vI?HQALubd
zhI4#q1H(uPtCgMB=Lf>rBufdt+5{u~L=sjH$?|B>
zg9c^@3Yd~go!Iiq9jbsGgD#J6EqpP4XzOIrRM?@&H+uh2sX_lC*9S;p9-8^wn=eG%
zoz0AJ{6?a17^SX*jiXcXRRW(fU_3i|(nQHaG`Cti=Y98gm?iAYoJWH+i6LpiI`fV1
z%}W{QjL`h)xY=Y~sC6`Ma8VmB4KgU+&{??I*uU;;yxIb1oY`P2q`pLL`Ynh}
zZ3|bY!dFwnfEs$_IS@KWpGw*mTCZ>rCatae*Lc7y|LDBkCsjfDBbwP`)Z-j>qu0YR
zDm>%Aju`APbG)OBK~E{;_xye-c4BwyZ>IGOVJTwZ{FG{vbjwGIL_dL}|EkvP`@wBp
zJ*C~q(l2a4acbw*Q-nj&fcHy}jlGHJ;^
zVR^+*N@Px>d8_x19}FqWe9j0@KN=9p1{2!<4In8=2Yi=DK
zsmi}Z;!SEh8uIZgk0P^IWu;{eAURo;w`^@_L}q%)yy-*65_(j&3RmzqHLh8l#9Nzy
z1wkF3&pkWH`5Hr=5gpR3v6&23K08H}>&W<;hvuoSdcxk#EL8rCt$SSWCrO1Ov?VS@
z;ozsMZ`gR|C*YO1f^Nh^KHrVbD5M&kmYi=3%%{&DO^?`Jkb-dnw&aiI~uU)H`vXiGo-qW8_F7LuLOzdd6Cwe4F>
zORqbe9Y}vru{h4R*YS8%*<8W*mf%P+Ohaz}nFpx+!@A4ZW}w&`Qr<=0W~N?O!5ycLi<%O0(<=6Qak)N)NFWp!|Hj6J=e>2$uqn8UK%
z>s=He_DN~-Dw9y?QsOV|>y92T_EA1R_5QndF|9kHXp%r=y_L?guWJUcKe^+{i)Z8N
z+T*R>Yj4pZp-RVixe2SlmsY79>4y;H6{v4gL_3`~PYo`1-+g?x@=zwWb~XOHi;c+U
zXP9w@>YFZ9W}R?bKdZi*3U{c^jBonG_V9hVwnvr2i23)RN0+(e|xNDAat;L>Km7&np(l(bwwfcLJK*zznWaUG9tBV%V<$?y9quiwQ(nGc75
z^zhxgsW4%@I*_`NsPXMC8qnuN2Pks$v8=W~DV=tBI<4$Bf|TFkoc(z^`+Ud$$%>4E
zq8i9n0!AsNH7t581V9j|TEF7IfpZ_L3#*flf3WF@>F?j%qiAlyLw^fhD4EolMgCJ^
zwMW;*>eQFA;VKWFD_)104eJ~!dp#mP#9U(!!ukL|t6{NFvJ4OV4e=%ESFZ5_G*Gt(
z+G(DYk)m%fm+SiUJ;bP*gblxal;ZsF$2X;MCunN~yp
zlsncewxVVxorT*gC-b`q7K1!Z`M^^z^JY^`pDw#I%)*W9flCw|ZEE{JGb@1|Z~$lUejgmF!V?LY5Rvl$EfLs7~8lF9gmLZ+9!j!ZlN2
z(Q|3UC~3t|5gv1lD+_>?G&QH4KO98ujEyxmr)g+C(GV4UqAE38SXET;(+<=Sq-w4<
zcrAu~^9tzG_6bWRiZCh|D32APfF*;;MN7p0!o>+=i>36c$Et*1wW4bJZ7D9wZVHRd
zC*7VDZ&=2a#yhRu6M9koC5D%Kd55``-Y91x=I3B+lkY``({Ek7rhZ`lf@yoOJ(OrAVlp3WF5A1r5+(gTHmSvnpad
z>qX7=qCb*tl$+->*UiFb-L1uQdN|cFE$@iE=QR5EaXZ!LJ{GWYn0vnb>iwV8M*cHZ
zn;fEO3f?N}oAUc(e_8Au0+*}2Vy;{p2xL=r7+uQ4=f5Ig2#6iSy6H_c+ujNy<-clr
z9wNWG!ghwsND`8T#geVyxK)mySm#L_v{Am0B3%ynKw0|eu`p6bUON*|O#lXcA!V?o
zCo21A`nv?;u!W-U-gqw#xA)6kiE6k72>h%jMp#61vc_cggr>;&AQCUsGzN}-vMFDNtT<$sqq?D!b9~n~V`AfEhp7pGG{o!VorsfQYpC8;*)T&J%
z^~iKtXUd0p`+jc|cO1dgqIM$R5C+^VQg-#;g#wr>2K(`(??XS;`f4VdktM-$+zFfm
z34*nUD4uX3r;$k%9TV+lKZrgwsc7a^Q_|E|@|6#z&FlIXD;{WcedVs^L52@Zs=*rv
z)Z|U<4|>L~Kc0e*Ej74}?0#4B38z;Iq&qTHphJ>u_Y=C-e9T9eKdoenJxZ?Hd0R`h
zE`>{{32!7kZ?q+E+L8L!jPlNu7)^~54f2jw5|fVV${7j#18G(Ugo!N<2uLKOuHjgM
za|`}!+#Py8)%Sz!EJeuVar2skR?IxqUqR1xd!KyFP)BiON2=^DvRTBJV&*7%K{(jF
zmEO$g`{6;y4pLLT**=A)V>bC>N!tv~b)HEM;NKwfJ|9$cCpogcDl-oQ&_25_N4}nQ
zN=R1Ek}ONAw3FEX;1YHpb_1;`ahzwf&H@iRXToS{?IX{!1!DVUFF1dCYNuBIq^U{h
zpeXet~oHrPk)}En2>{KZx+8(@yuV!=C>cYAU3qQB?nOywLhPH=-?7r{()U
zO`$!cX8+{w&?qD)(B{*gZhUe7NBP@sQ8N;oc>-)B+iRThWi?!^cfYhPnXZh~W4Cd}
ziakaPgDf-udNs`%O9K7|wX4$5SiK|$;*#S7&ZV_r$)N4AI=;aptIBuVeFdON*eypq%nnfi@%C$wvPl50Lb%PT;74@~6%Fi^PlV)Y
zq*k}A^nBkkA6I(Du>?0GB2U9{hl{0=48MTTlTM!r&&vR+=@bv0*)1Us?f9P%&y%
z8UpVxxhfyWsP1l6|_?Ew(jxJEQ9u^DzMEsHzvNk(Tz-iaa$l
zu^J0vkHWxb#;b>_uo8^FA-uvD)%6R$hAwaJZ4l|38EsS!Qs$(=O%8>
zJmlRJY7KV_8`!Vnla_Yxv{|5y=^9t#bM|ypurnw1wRJ);$ki_VFtfm3sJ?8CA&RK|z*B8&-gPr~+>Ol2WbgN;1_}tlZPwRu2>0);76f-5s?%Ny(
zvR?A`Y!SImnJK79WRxOUj6aONnu+YKiB@dm>27mTz9pp1QXaH7(dw?xb7{3UJ0N-^?)_d|05avir!Jb{iyln8;wpCMk>A({+Oy
z3vA)$FOkB|vs7WVn(4e&RyvK}2Q#~Q8?d2&JJH5sJdfu%3o@PV5g)IQmMpna@4N;>
z7*2EbC4|5Bzm2_0qv5kL)2yd~6{8{ML*BhIc~Tg3Q7+ExiqmL$pJQ1FZ%p<7`#2uk
z#D9m*UvoV-IX0*jOx+!4R*nVH``5w#9!te1Ypupx*B8Tg9TF$ie1Tp9`Zteps}YIkoAN$N9nwaKlpYMz+@0QpBR?H^*=vY`q>b};0b>*zKcZ2ImE{k6B7
zLm}=SEGMPk?GQWw9K56pwD>ODsr_nx5#YNMXc;N?=NmFZcx>=IN_e;pP8!KeYszQc
z@O9w>3B`-~a4gBYKP~Ll+jiufc=k)i={(I-?^~f;s}%7SeC|y=xM%j2ErzC2@-H2F
zrxGypeoe^#BT~@QLKU@dpEIKj4$aWV0A8)F!`90*H$9W67dwGKUW8!sFfAb=p&eiP
zAMfab*FT1M^wc=g6p0ThIOZ4VDWX4a9cIt#Gl8q&@nEU=^2hNbTUk%JcAJfb4f2WP
zCMWT5Rg&(S@)Fy`S$umMjSg2#y7JQvVY4Kw0N0z_j~1%hA*cog+v+vDHENMc@K4#g
z=tOjeI>yh98+txpy(bE3H-ug4CpTmsLQJn;$occ%OuMe~hl6*{vjB(RxLIAdLpO8C&#$a5
zbu$v(+xamk(;!Q(Ez6_5JI%&4;s
zz4&+B?Syt%jPXY;!bYvG`w5@R@v)z+Lo~i9g5$-%FS&8N&0K+xC!`A5gV8JYo=Pcs
z6mZv4Pg>8wii(Nb_^jF6BC&Z2(#ion^B(_ByqsPE&+IWoY8G;a};Szj`4P7ZAfzrE~v^fisN=anyFfcBRX)yjoazH9r(RzDst3=CcKr$2>d
zxA>`u;(@H=56xG;&qv6XBM_QR(5y&YX`+mkIwo=}2H-lR*lz%-9EUpq{HyoMkjOh1
zdy*5=_dPrRpE56MtzA(+pb<@R_0i6o)SC&2*|zNWTDY8$?Vl`064m8>-m9JPh8p?F
zOYXOPbjx4@%?vuG{M2=gsC9kxC8cjT;otrR_-QXl(C^Z)pAOilK4+8@6EZ*1+j2dU
zOPATv9}ohRU0RSL7tEh>V~99bbtL;4>q7?3d)jo4&bQMYAsgGQ;?=;)lIWa&9HSrZ
zn=I1wTn_Ok1v+(4d*G4{V*FiywB-|Hr9W*j>WRMns0hnh2CCY#&6cCuAMx1Ok2#H&
zi};G7`a(7~b{3l+qq=`8aDf8fxryYKVnk8m%)K__${0?Cvv?dCo)ma|J~>=@7BqU!
zycQ4uI>B1|Ww4)rV~ai!`rI8Xu;?`41SnPxx(4G(9lEHL)+=D${1)mRYZQD+V!=bW
zOCP#u(%U))Nl8|5nK(F}?rq~|l{;4Ra3_Vt>*w)}6~jf1FK=X6C?3NuC$HyN*igM!
zHSTt{!`8xU6%8xvV^0SvH+f=D5=TpI9|X!_pS+YghsNdkvGNdtsJGo~I*96%Z@U_Q
zO&u={Zqu4n#Jt~#+R%e9OL7khOAYCURzsuPX$Fg3l1hFp>pZASQ#9U#i
zG4(fNfQQ6w9jGa!)@wg_^)aaL<&H~!I$#<>z>=8P^Ckd&#ERU*3=q+IjX%PqMOAFw
z@oO1af;KqO6M=AYFU#}+qj=r*-1FK-oJ;Rl3`~%bW}oQkJ5)MVshn9=J@;~6<{dUZ
zZ%!LkOs;?#z$9~-;5m{dtM
z8ZG(hXGuwN$##P*y=<(f{r$ldDGseWJjJQjAWU#gP-1Hd=s)2ddm~|
z5))c)Y@0v-Wec}WFK3_ElvyRp9Sc8fjB~vu0hPFP1ON7TV==Z8Y;?^Dec
zru;&w{f?jl1*BYM}w;yMPe6>9tHE&@)q`I5-jW|iy=fbTs&g2y*lMa~fF2lyX
zPZty?7yN}NMPorYA3XShZRCYWH4mVAxL%$*?F#$JIp|g-E*E58gEKIWP=oRd`}@hk
z^jskRy-VGg<&&+?2SI37p^bJ!OAGyrO}>fa)!9I?v;=+}
zba%%;%VCaMcyB(baPYJB|LL4ooO;85j@HWMm2$s90cp7ryr0vk;}Xy^CSF4o6JLbN
zLhsjCcX6_(8L?Cbj*4vU^wG_KTgKv`V}Y6qf=ZI6#>1>qb~D9JD7PERtbi_+c$CL!
z7cF!Q=`G<3yfnEC-pT`^iP9Z#VQlb4OHWp?(VyQ_Z7v?Pc?{@}2L%rqvHK
z6@fz3<*9wmvReD{w({8&oTL{&hd=*ol_kum(#O!2euvdh<(@B-+CARtWn1BRhwMYb
zb}FHL$-`)J(sTqtHW>>~t&nlPtok
z4L2%asAwe{J3AwPdj5x}DooJkr$ok$Ilr)r)=)U+JGAv6M*3M`4An4EtP58G$yueC
zQ&7GA_?3jHd)9>QX@kV`jxADli<3o(J>=P4#y-9ZwwsZF4KsHtdL
zz39{qJojm`1f{yO?d$MK)nU_6`t}(vai+N61TsQuvg;ucGP*6m>pzjo28N4M0&N{M
zq6RUXW%D0c9%qVnp^yA0iubfOa;4Th&8f$!Fg$BO-Pe11(a?^;0KrA8iee)G*=
z(5VKcUeO#Bvbe`iCIK^-WA`-9nvo0*9GOw#JY(4+W_;-(9_)vd>d|de3y@y#y=%B`
zY*)bYjm<9}a^}(BN!fBp&O6jBlKK3~&+Y#_{3MHVZ>PyJ
z8+vx8WQun}L)Di%n>NfpO$y2m!pG%Ur3=DhrEsOwnf_dkIOgHfV#&o!$yFd%8-`6p
zqa!5gvk{C;h*91>?~jrItUypvVN8ZT1q>m`QGfKtPdR!x(^YKg<@d3RA^A!LF;GJl
z;uLxhVb)8ebG~X}d;H=5fCJrIX+OGjq`#XNUOB%9XOMerB?^$*zR7bAfcIzN*ndC}
z*@emKS@)M$9T0}-9Sulw5Xdv56JJ60t%=0t!LDj?_|#D_AV_#;GS+{vU+h_wqWyjb
z1$*z<8oysb{pi7vEwX4U9)chDo1-KewqWd&0b^n`4H49U|N
z5$@fkbj#?d==V1TYh?@8U9{=dr{^_QPpRo+o6=34Y8+Je%BuUp?P@{r8t!T1Va%KRAa=b
zPXMVH1{K%KuSVorTAbNqO|7lwF#WX*>M=+}?(7_|*B9~ZgDo;eF0O*&s60M494ueS
z&2NGZ2i*$|JjaW38;G@!8>T^Fy=wRGiky|uH;wReOQk1jVNYpbVb44XcR;5*G6ibP
zdH08%e!oT4`nUCO^O*R0hBv5O>E%beOphfj;y3G&xJe-_=jJqO7@h0iF#}_jZB&QW
zGYZR9P`Fp~Ccns7+QJ?JCeEk`tC7AAbAVSim3PRODNAg#>%SH`Q6R}X
zD4CBrL}j0?!qT1#5%6EdYZqtht+cSnJ|aF!k{erC*AzZ(*nqrD*Qo-xogv2KtrBA}
zgA`~cBRkvK&o86|84}cX_-AqZpYspxPNr%~u3EhfJM{Xi?`tgUwvsQsqWBUTYGdMA
z1c|e`;kM<&&f0&Os@Bn~+=8b_HDS`FyrqVDcEL4o@pIX}SJ!iDW`@yAqgCD_Y|olY
z6=Sp_ZpXQ`J)W31()|aYHhra=3(K#Xbe|Z`xR!^D<#*=xm}Ho$DS0-FuJ7%~N%4MF{nSi1#&m`w(q;nu
z^U(R?Ww3rN>bV<alUhBHW%GlZo>73e(ke|M9sEV@~PIPn^@%@!KrL+=Ka}IB9+Z>IPmSfV*d_
z(tW`Si^;WUPPKo=O059>sHpVefWh~b%3oiPC#&7uHx~}DNghvBnB;b%mc|NeJk`j7
z>>#V-e`Wp>IP)fXmxBP#a|a{YxTfPC`0bnO=<1Gm4jD%2yuecw-_9bSc*6-EST=~V
z_dr$cPN-KX#Bt9R(MQC?E|%L6^_|QeDWDn^rBBMT7W<_1IeE>CIZih+>SzhxN^Lab
zZOTpYlB5wfvnI>hd@vK3KBv(L(5zhg$$1Ve4kuAH#xq!eCTc}w7Sz&jL%GmW=bvX
zJ0Hz%7VxiHGA#DauH2D=^I3N4%z_yCPo6XVlcIF6(WGcU8Qn`*wH+URnOpCi&*=GW
zgqDr?gEzL-qY1(2eHhPk<5iHq63tH1s4Olg%*c;u6~qN0#cpABxfhSv*1z+qsfU4O
zu076u^!s}ZmBWpfJh*G7-+pxT#*CqASQJQe+S#p|?nmu5`gjQz_B;C38ulRmO1L-G
zo@nA{f`kOB$(s6^IlkjVhVB+7$&_N|@GE+V_F0j}5gfLk);ihjHgpV9nW?&1(q$^T
zYBNsvj`~Noa|3sBI@HCPcm9;e$KNxyJqdgB5D<0vo)CLG0^H277gZ!{d(;{mA9PXP
zL$QxzKWQjd2=b|*6b0t5?1<&A(4bEp}4Cov6IotT6uA_uRRio2k#It
zRu6h0awNO@uvB3^n#=m4kiLJLdYtbJtOi~9S>nVq-l^g16wZ=Mo;qJSn!`M5LTgXb
zN#UNn^9EMJIGi9nJbxN@lqvL>vN@-al3UH{u^8*}B~=E{vC#E!T(>@$KUrjUjcvK)
zm1)=lO@D9~7Fs>0{=b9M4eX@uZ8A>&IQPV0#fOI)Z&*utS0sM;H*x4gSMdtSj*Ag)
z6sM301)lr6-q@05S|gM080WG!v^03p;ZGteax(*ekUnVPkUYtqx;QrHhX^;Pih*V>
z8?wL6PI7`3=eqhcNYt&vs&xuo(Cvp(<~qQ);Z@B>s3d(u9lQi@^~VZE{)X#J@Uqj{
zOn%t{W9y0u*S|aBcL8-<{tN|!^xTEHstMJGAd}J$STay!_J*k1gvUiEP3x?3m#j)|
zZCD|npQ26pxV6y_Rg=uE%{=rzlFwqE+1M+O2BJ?WuH4xvd!y&wk5StgPJF5uMZ(Rz
zS9ECC0`0ea9){kD4IO@omA)I;w`V$rJ1AAv)9Uym8d_|V$F*qeXa8U4kvh%-$}wWA
zM@!kK`(+f)-<$uGUf*2Y&fIPeHhB6yX%Uny=o2jw7A@|XkPgED^QZLv8AgB7P53J;m3ZC~ey^z(~JtI!HqNwt@6WzL{
z9-4kr$ix{?9y~O=Khr|D8y+vj5M=WbA*e2vr5fnv?QB-Vm`)gg_|J&_z)eq8(x?3^Q;B_lIg~0EvO{TqWhVmfg54pV=(5ymkkbpIG#3
zvN4-;$*X&LpU1d{bqy?=EVA_26h>Qd5eOOotMf+sT>)m@!M4vkSOvbXtu>}K@ip;@
z2s1layGYdV5iy+ZeLlBBc0=AMz`(ZkvzSZkbM{>NW0;!Xw%kR)ru~~bm(4l+(Ao|Y
zbzyV=J8C%Fa~O)cZrc9DrRDZgaoiT4N-^}Gk8i)R56CHNw}CTlh)LV6FPBYVWaeOp
zw68Q@mA6mao-BPdq^49CnmkguO-7#Wg|)+|696c#(n+$V*9i+uwaEfekQ2p=^67Ui2>#Gcwqu-AI_09>=gkQhBv?6(DFA_wTBW
zQ`WPz;`-+C~48=T0NYP>&Tdt$@+PfGz`0{Fqf06!#nU_*&kY3f&M_pu;_zh+GjnwWd
zK}SIHEh@c4!H%I_b?)q}Cd0JggxA#tkZ-QZj=9}<=r7r-JvCb7a$Kd8t$tX7EbR8l
z=+oRXO)|w;OnFFhl2D_4EDx!6c*4aySX)nNau?JWa^ER>Id=1I9yaO(=Tc80!;cjh
z6x$Z?9|-X4;%SkL>=f(-1XGfcm?-|<0lGBy5Q-^(bwVWUMV&5FLIsRL7|35aQ2rm6
zE-`d(o!S2xJtkNOj!P3Z@Rp{pP3%f*MBWr6R$|Vi^-my8#;r{~iqHY9@5dUI#}~F}
z_R@V{9U6Yw9?xHx7XWD-#enyYU4i*~lRHb1TyxDy36kzJ!v&9X-t8S|_SfR^jhlyZ
z_*hm%vG{<=C>iSHVf^#0z-bpLkk0P+m_}4-*YAV4337F$NcrcSj5>(KjZu9*fp<5o
z8LwMJ{e+5^fF(d$<;j?5t$+0uKw3GUBg%<5(-avCxi35e;TV$4Kq|rncUJ_)j6?nga4X?X
zp*S(cM3yi;EiT_Qt+8!DlAFz?PpYrERAkRn!K+mgY^IljZ=to3A@;7zHOY^E9jHDX
zQp&|LLu(a@KbFFrY!M6zeA_c|xRV8K*jy7>sROHLM(a
zL-`~+EPHD_e&OBIH~r&`O0*Kf?o}A%cI_y7IF`es!W7ZN3@CZC9R`;3OWn
zBA@G=^7USh>Gk`mDW`Y!5H8>1d@Rm=
zZYV9&>l#gO@Vq&Bp#|b5j%6IK8T2OXY;r_{g`$ZYR3T^I
z+8C-`vCVVkXn;Ho=c69|1Ru$>3nDM{qnz3vMf8rpNzFSqc~dD;8)M%*{2wVkuSD^D
zuQM`_022UhH}W?gMaMYh8?YW5hm;ND6;4J#enI|DU5u?xC)=Qy00!UXfq=HhWD_7-`VT9M8K4dr_||)-2(B5W%nxs@PJV_{;}=ox>beC{jVx
zWOXo1?La``;&I2>a19sC0Q4nh0Av3(TPE25HfOFQvQ2mV;IpCmo1Q$FNsz4Dq@B`W
ztSQaNNmRdjez=6LUv!^75D{@|-Ltm#!sB>x?dj2*jGUa33KdU@;{0GTbxp3M!c_2E
zx$l!#pXb5g7%D(^+1xV~^1syB;ounRUcT7q{1atda&mI|>nXgvUkb$Jzk*=VrA@YF
zbkBugXO-(EL$3PpLp6r&gkN$+l{mxsHhV&DqUayzS3fy`?&W~l>6dFW!QFUCd3g}J
zsuVrCp{k-_DK?fUQ?;FPCFw&Spw-r2LGpJ{tO_s}Ya|>KrKh5dttKq>UVm)7
zyvkitFZ1kSKUS}N#kl=}1eMXqLX~rLxjuZBY2lJwxcbG2dlHPHrD2U6
z?C7fjvM|h$Zbu5roGZk{s?3QAo1-R!L>+^by+qA(7fzCdq^dlqEHOI2ArvMtvC>O@8%r3dM&dbO*`v!O=!ClUSMs#b{7541E0ufPl(G0
zatNr)X9YbA*^<%grw6!bP8Qi=t(YZhmFrYi>-+xf-Ct3k^ACm_qQnE^uX;KHMcyiP@LAj?NCGD$U?vrZo+_H2kAt!?}OK?lm|PZs|qC0h$|XO1xa-nb4h4ANi4%&RR&?q7Jo{BO!#0A>kCSuQ%D
z$!Z*Iy@A>&yJSuh{HVS+=i^~pE+`{oF%qMC$J{cQ3$yg$_xxw$%TLT+#`UxCsI)7M
zP)S1n7FmD-a_jH++)~?%iX{m{`u8*UY=f}^V!ROlZ^pLa;%#j<-(7Tb@hez&mfXFd2>KIunG=pgt9a8u>WjKOPomcHD
zfW@Z!VPZtXnS4ddG^%(`-W1vU&fci$q|IOUMdNn!>#h{AW1FzMlBN-kWGm?Dq3i6l
zNFfWD1$lOjN(flKP{C(UBb;z6yJc--X!5xNTyk0PK(5^O5Sg%96l$nPu)?V~Fd)?2jPl;#UZbUQeB_bP5}
z?gMnMXFu=Z{~O@nB*o5uKV0RHu(v>-N24jxAWcIbK0hI^<1xcU&;`9jheW6PF=RQ4
zN%ZL43gz(gXxHp6`Adh$A@&fZnY8x)P=-`I$lhrYnW6HxB!<=~C1X
zxlMa=r6k+_`bF8SXfJ~uCq%qO#N*>Df=K2lPIEm3nK@$o79933dePcq&L(f6&E?L(
zB=Z;F672q`YBm&ffn1xZ-azx=yZC>Q>Afo+wEwrjvLe+awSST%{WUpj=XV@{zSqh-#&VBZX#dT)8W4a5
zWNKQmZ{dmYx50AO_-^Q4
zKzw2lzzXt0_y6k}RWO!6vp#E`!IGFCoo7S#h05G3X2?a{31R#ypi@ZP`{e~sEm;i9
zpr8pLbv2G|9t5h;0}8JX{>(l*c{Dm1{7I{GE<|+c=6cGMr#8BLfGTY8hwlyE
zNA)WZl(h(ZKJ@#XtQwsS)~;AJHE%5BFiTDgj1#2=O
zyEj3Jr2I|J8>nFA(&qBgLhIE&bL(v%Y3n5-eX+$Obj~gW5>vxs#H6p3F&0T>CHz^<
zoLIk1y#2gqvl*(#mhwNRtu9LyA~%U@45A`=_?6kjF~j-%&#G~PT6)^>bVQB*%RidS
zI1s9}yy)4@aR0uyU8Z+4v}({atMBoKq>b8EiN}nLw%+QbkSO?2rg-smuRTlMCUYf&DN)C9PXgZfOtUqsw#G
z+hqMd=G)DU0t6b+^Q3uz+NwW;h^WU|)YFV0gV5|t{s|Z>+qyKIIq_|nS6t|RD
z9UA}rXDL5J?bq_yF63XC@9rNy%nwuP-4yIqy>}M8w1(gA$@qL1-f8mrb^Gw#<@1R0
z@rcQkrHeMFX@==a3MM7sjAw>yl7<$=vK-`D6!lmZh$kIC&(FIr7O1r*sNva*cE*QO
zYY-rVqBe4BYve=|j|7`Y)3=XZCK7V+ZsV`-`%(i@X6HE7vN?
z+FRej07E&bm&%NudkZ;An5d7w0@7{xCP17)UT=L%U^%*M3^QN?D?&{qWW+QejO31L
zVK}z0%1A7bz(k9nybKpF)pAdy+v!65E`v^rRwVOa{L>yMndwUQ*X|WPno{#Tn>K@4X+gxDFkl=(aC{+Ax(-=w+>v%-1E7X6n056EE|ec3s$b
z2DaL8fc?zfjp#4N@(i!*#5st-Qd4+Hzi_^#zthrQ6&#U##AkLM=JEWOrz7(x^}BW-
z%0#31QKf(5(ezKAioydY=f6B*Ocs6T&Y8@-|LV^7IY9o)3XWS<-j6(X8>7pIDyxg}
z55&pxFZ}r9veJpD3kVo*IW;ybQ8l6DgY8eU^*i0?LtWpaNxu?rc<08&gnkiFY(|L^+(*OWuvrn
z!GZi<+~SC9;YaH1*-7`-^9AD8PlJf-k;{&i6h(H)coY)Ry;GZ#0j}@L6`{UzAdwJ(
zsIS0xoKaRj4eEn?+QyZ54V*8Zk;T+IGM4to%F8tTE%eI%kyhWUwJ#;)nEh8&DTxzX
z>SaEjht`BJz)A9!zB=H7A@L4!IzFWufNEdcKnMIo-|RGZ{lQtgWm~&m$6Eb2c-L`0
z-i|@IPZ;2WHg33xe0Id?f~|J0BWxJX_u!H{$4$Kl-3dky5~*fiE+ic7eIEB2PBs+(
z(Lq3u-CIs~_-#&6`gF$LdC%Trk8{SZx$sINcplP0_Lt^2S2GbEXxqNaFikzyd{uE=
zR=TjVy^fu!m8|7?`bfvg<0cHlEQr(juvHc}5wrwbz(B
z!T^v)s249g*N(@Jo<6VXMUIo#YW6N}5}2dL{+HqxYnMS@kH1N3c+G$oXCWRgG2tPl
z)>Bc|_dE12M2$Z+xZ1vki4ZGima;u%CWwNInooEo%+I0f0ic*RU#x_|3;=GYZ0U9$
zP{hCpRJDNeIwbUWSYl?TwB#SOy8Sm_<*US&Bcl%L>TcgUe#{*YZ>`ku^+Cd;
ztq`$nZLJN6+0K_*xzR@qPYBNcj*!>K7L|
zaPxfEFT(qc$btO`iz|UvLo9Bn43k{wl~1g;trmD&gz@AWTyxnF{j5F8cE_egxPgq2+EDftFAdajj2rjrs|vAN{FIq>^#Qw2h9(cnUI$PS{_lSAs7)T}XwCNl@(uCnw6-PjIr=H-!Z_9}yf
zwh%`*iBwX#-{?h2@xVUEh(fZ-SFYsNxFoWQCMN|qX?Mm2s2L)B%
zV0wMSWSTSwV5cqaAL$ys?6a-vAHA>#KEl{MX}cwd_c>#4m27-3`{Q`IlNm+%6aDgS%)74s_as0HyrdB984Qd03ov4fz7sx#oh
zkZlXUX*;uQH}M^-Jc7Ox37N0F5s95MGaOUE3*)ri&n&XzpU@9XloC(=k5Cx`bmfDC
zMu+a8`BP>`)f(38jD^~*ut#FivJuyZhh4vaa=Z}r7r%dA|F5wU6h|Bh2k2_aZ=MhH
zaEOKA^D40nRXw8nH#o<-y(LEJ8rjr8^<7%p>?N^$^?UFB6h+rHz31mcmHuQRiT9b+
zeomXV(W;HFz|GqsM;htDBbw1vnAionSlz)bo40z4V@(-M@8$(wzoIaM>Y+dro>B|VxCnH6;9kpaa1#>O6FxNK<=L7weLvnMc|Zl7B6@>Lshr}>Dh
zO?`2Us0X~S4j3JS1FhS6G!ZDRihoR%U@UkA}N_U
zG)z}I(|1LjIdyHfdeWBX-kQx)EWfuoJbWt6p{r<3jXN_Cf|YmvJmLwd0yVi`-{^A9
z><7|s5$iQw{kPGxWEV+Ou3*wXe_`vJ!ihlUtV@Ed7r{UA7g}BNLS96YWH(N?^DN!r
zZ+e+j6A0;_A37-?>5Kc_M>em5e!I24Dh!nFN9rafyn05cm4;tn#c*CrKwT_T;Azia
zw$Te3oZ3|E_ykTsldmCNZ{E$`#!LAa2#mTHh~HY6A4W4&GG|(Ah(dTq#kkleO}8g@qDo%x&=y
z@!NO7;w~`rdY!DtC!h`bx9yPOn`1fQ^O7LWYADz|;Nu;#%j)4=*WB9ttj(VA@~G6t
zpE+xi!&L4mSXwK!$BgKtDaMKyvu@zT}FzCq3{>+
z!~JcMueIwMIJ=?Q0$PsW5(Ax?X4?1RTbpBICWg%Eg-vjcSriy4I*N}wUjx~q{W|d4N}PBinqcSj
z#9`W8pO&EFsE|KeWa0JJBl);jJJU@saOOBVydBWOkTr+vi?4kT&>sK4DmD=zZ)D~L
z?Dm-oF2FJO8RTl&8uYiaJgL9(`76qT<+U`)#At0X3chtuKDVL%hO-g5Zdla&)InQ1
z2N$xJG2j(Mq%Gxs`tXFT$L{J^uMpY{k}bE*dHW;bprT94%A55gR%a7FtuI|xs0XLb
zMt7tzhR<+FURNC%HPN&)zPqWbfjC*9-e(Z4eSFP%)ldq
zn}?p?aVH23-3G*4BkEkOh|ypbhLM|T0P5o7zemwod4UGiLI$f)C#rl|3?aFh>D841e{nMI87L2+1AM~S7E#42ov;DjV(}yr)
zlK=OM@b`5lv#2WS>JAg)&35Y;%O^_!`^LX$RV1{5=eQ*w9}oRwT+evl?~)kL4&IRMlNI@nEclJ=-{ThIwKc(7#RGCV7c-biU$lm=Y#@X%EwHDVH((gYfX`=ZwBERysQ#BP{wlOkerm$^&z}wA
zKzB4x7y69w+T|R->4@thR3!GyBrWZO0S|+TP>27jOJ@z~sHG`mFix-oW2BM&UH6m*
z^X5Ytt&n_ZK8MT&vaG*EjuAJQC&}#?g<3fBYH-Ap(jrZ25S)hb_txohNR?S8xJ*7ugEk&{R_JLC=v
zt~sBpT_S>q^er>2#W+d|=?P$h)3Z`wjLY@ou4%aV#sx(L;yD2FW4s$XlC$(BuPY+zf>9bR1w#H4I{IY+x=~$B6CL5hf>jh=NqNJXvqyQWle7L42Y!6
zu54#&bEx|vNFwbqIJe1>D$l9m_||NO0&{n676FTEpR?v_d#1L@$FPW?R3oSN@j47B
zXKLOzn`BT_!~k$jx5PchKZAqLxfDVZz)6GeZpUMA1?$h_$Hgibs(opq)@J=1^ld9$
z#IKZqsH0mJ~5K)OgI!LIR;K(
zz36)N^zZ#Pp+5E=}H!+uQV8V(O@Yow&q=rFp~gu`z)C
z-@aru@q{7cv4C!y`jyG|_$@@35cp@}m$ws?(ch#SfyLt8c_ukfOjg-P{jjWKbZ_Bz
zo@Aq>;?pNin1`e%|X5oW)`^wqPQChoc$azm!N(wl+*&#R}@}Br5AMLB;k(r9)
z@vvJiV$9)KoNgoDDZaH28bK^1m!AtKj2oDk_ro#Ul;>pWNL60Ael~=sRpgUfmoes7
zV}_Be$;#!gxhcrTZ%sCJeSiAHP3-yY^#<5d6PVrWiJrC2?dpCnI?2?|LFqu?hfWDc;t)
zS9lXbX(T_UTUKSAl-?MW$!1}9!s$RZ-5?aHz*
zmWpu8=PF<#?r?Vc#{D8TmcsJAqGEjLsh0)r_F=M+f0kd2x6_*l?w+?Zk;t-rLzXg}0F|VDP|~)^$v;
z?98`;gn$bcCnvKfF6evY{Q%^CO`xcs^q;}%3}!u)
zRh~8!6JOs8ZA{SmQQ&=@yE&(9k|iU@@iCIi^au7BBKXJ)?$vA=SYo%7)>h?vx`~a^
zJ}CIiPB4+uvx_|)Oz>+$fVy#_mxcnHLbVS^BMk(q-|DmZ0J=~)ol1{JW@Q#50jnJx
zEhbf*YEeG;tD9dMkgHEt@sBjrhplv859%1~ZY{@8CKl?+L~b5htl{!CpB$R`)P{k#
zPB%I6_~jV|0gxA;!4<2!sjETrQcY)UV!q{h`f~gV9I=*KO&*2TYaCae(w4$eiLdAj
zzx~<*4M2U#25H$;1=%V
ziS9;zJs&zeaQ?d>tU|}!)3(L*0_YJ*QSsW#Y-*krpyQ&IYC?*m-&T}JUPxAnN6hsB
zjh>E>Kxg?^RnfaX>}J-V)AOy$f!Hf-1>IBVPQ#Dt$kn6Ri|*6@1Qwd&hxGb+J9gj|4LcPg!*C
zT)7NZnVv0NMMGA)d%UZGn<3{KtyRYu@J(eP@Fj{n4~D`fX>!XL>Sy6ClMJU7H{^Vo
z&Ms@FI-f$&;WD1K!bFXFu|7wZ4cFsrEYk@WX(T<$w&~AfwI!t2#4OgRQYS^83bgSj
z{nFSC;S1Fu_qwLvXS1kUccJr8-bDq@1W&OFlx>V3
z(sdQvi1IRwoftBtH~jK%H$D;ZXq^mAA12A$Jzu-BTD!(zyTq&Vt6>y7qNwk40W8s_
zMZwnVE0yZU16F#r4Ok^4KZj5>T%E&EZEtT<`$$<+&g%mTKY7~$XX*huOUw@i1%&u
zK=hYdIWlXJU(D*142az8Q>gWxsT6gnS(ZBGXK7hqS^^YV@|xt=RzkJog~qMUmIC|J
zVEt(8m*tb%tNHn+*U))89s2G$
z&C6jQ29I2l$y4FZ)!zX;oGZq8iwN%yM@01zmBpzuO|cE#-#UNpumyZvM0UWMB5Z=+j+_pzx=n
zn+Bi?3THv&_r?ZedhRZlU2gc0ZquWpm|)dg#Yy|t>|w&Q6Ba60+)|T5!K@wKHtEBQ
z2a(TxcNuymx~vX$Bg%84BO)^&Mpix4lxMv;QPb8W
z1o8O{2}ypj5h|B@_($rFy9Z1@R0Io@w*ejDV14z#`RAo5!6i064L3Ro6GBIiNCOiy
zGdGcGhvrWhKJiZWNS*0VpKDB=y0TJ}0@u4wpDpMMvMkRU;Lqj!wk8M{ZvGK(`_h@Ay`XUS!Svq$B52(DE{=iOZj?nEY{)dl2G|d
zBH8jw2~{P2%%!|iAmN_?nvwppYNd!gkV{5o1B{ECVk{o9f`3$>FyD30v~tC&fX5+=
z?A7UkUit+4GmA1RNU5tw>*<{yw>!h)25tKYA8)IDFZZ$k`v4CZG`}DD_j_haG22v6
z+;7WtF~q_=#0UNpG!8(ZeVlJta;s1+CZgmqBm!i!G~)DXuk&SD9NNhssNA#jKmPX)
z=k)L^jTUEpL&CyuM(I2WW%0#h@$BVkW0Tt7A{4)nPVtEjcR#tXB4NcOre`zDxQuw;
zzHgVfTM25bvRlMdjod|cH#awngKtxZ5-^>80qW6m?PZ~Qg@y)r`a7|Dr8m2JfBsbM
z6!VGmy+65jfNyyY-Q?%}#f9lIT8gEMGAPcSCgq{*B_6E^4s}8~;2Zw-8v%UcOFp@4
zJu6Sda0Xvk7jmeC2o#|y2|biwYNbrN(kYAP{dy~B;$cP*%2eKK@}V%FORTKUQcHxO
z@#`A3ad|DToUZ=<_xyo2+=DcSt3aidj)AY}yJgg4nkrLnvp5xKbWh$yAE4Q4I&Dg^rV
zd$_#O!FW3LllX-%?W^45o9tRTTP?bm>RY?F_HT&|66jr<%q*jyR(dkD-p@q=zv;T+
zqM^U5j%u3vp1tuHr)O~6+6*EB`u_G$@L|aA_a}+bRFnyFlP==IRw!4h!xc0=`0VdBn?Q@U
zu$>%kI=5J*oUw4R6^fezb+zwQ8J*rdMkjnoxVai&@4!rrjqT_5(%R-g&z~dld8pMQ
z(!$@J2ZuSd@WWfVfe52>wcI&p_c@cvTWp_*za#`IE+Ue0(dbo}9%Cecfmwpxu}IEk
ze8P)KuuO{@MixgwfSGWOQ-n44QpTwh>oQ0gK5FUst-Us-4br7hqJEANj>AD3r|pNA
zpIOrXB=pq<{kA^M4qIRvpA?lpI4c?EEGr`n1yk%8iT(FR{i;h4*!}qFd8LqFZ2>D@Bquy=I$YiBmycMb>K~Yv-&0x=KqZ!cTO=}UD51K^rm4Q>#M9tliN5Y
z9?l=Jxjy_K=&vPK0C6v%B)Y1(=3LE^+apNtcV8S#JYbxOm1A>Bam6PK&0CeqhiZ^Z
zt8vGX*GT^{*dz|tc?+<(`Wde?eXBNRGuwi+h@X4(bISk74R=^w?JyS{IY<71v~iQk
z{dd{AwUfM6Q}Zc=GsSC!#Xx1Tuq|Ql<r)Wy{P28vRs9
zk2Hq0&E)eau63|l^7}Gn`pcANNOyqMl!oIT@k>{si9RMO7}m(`&4;6wie2TvZ7}mw
z75g%odhSpMm7Hb}c~N
zFK%LDM8bi|sX;#fg(u)(;jdM?Cy^exCUH1AGRiECe_?^~(q?R)J_9r!h{uP6J2%de
zfaBvuAEi;(nFf-~!P6-XYT&&?9|O3?#v-kly*HNrr!75d=x0TH=*8>-c2HP~PEMM{8`istcH))nWg
zYVIOIAZx~j78d~1i~lkHj&IL+D1$cZu+hXjqNLBaN-JVJ^rRzuS^tH^Yiq|kgI-g`
zROOW0E4iJE$l=}fXkGI?$q+iy?T;Y1SR15KpuCtpF?*}2rP@>5H^hgIn)N!#1a%(G
z#Vr{jGvX6ayqBSPNB`$7)zYU@$N!SYv0bTx0`s5WI!yMS(bJ46J+R#LkQvdLfHWX4
zT1!pAieC>LY&DDCZb=}gIxHA_Y!phy=tq$;@$6Aorf+3C&xJFpE)lIRKA*L0F|dg_
zV?;5ZXO^$VTcP_bb>}5xva5D$60M@*$%n%y26a7n?jtsqn~ILdjz31)(5KsjpU)Yn
za*gI}?0KqFtgMhib_|RnXyS64rxz45{|*8ls{dWCd9nWgnSK6WAJF|yQg(h;(Lj^}
z1-2u3iR<>^++AWE3?Lu>d)Q8clGZ!9&)~RUm>Kx)`acA4S_jz?RX$Zfx3xyafE;G>
zGlpN>V@mk|(M2Fon%DMjhG-KaK#jATRNktjI(A>c>32(RXWv`_9qdAojd9y?^uSlh
zX$mN!2T{JIdKd88r}C}~XM)-hSB|vm;AHk{H7(@uWfFBxXk?ZG_mTnR^d2es)&1^i
zLf*&A#gP?7qfDaCZ1)Do$V09dW7Y$hw7k*K!%A9P>63Y9XFCD_Zx#Ldp#K`QB&{Lm
ziT}FFLk6niG1yt+Hv3>K+KqVl-D=2nybSd-#Vzs<5%CS@9YLf-5{V{-iwYOt$h!B
z@v)dcdF%BDkUXOsC0!(Zwy!G427NN|_WEMHzVwL}H`jn$DQ6jpo^DT+5$(0qD#L-1
z@J)dzuT*rZZAggY^j_F0_h`Jlg}@?P`U+92&<<2LMq4;|ppDh@j<@|bi+10;)+s7d
z@qo^FE>dNSRfK{xS06v-?45oLLp`@7l)rgm#5c(EZFc#Kq(5NWr6{vhvp{T|yYaW*
zibY$3XR*K8mqZk6GDWlaEH)u(*4*b&c9~eKV!E*v<&6ZGc7Z*ZcxojnbMTSad$hO1
zF{(D_=|^I`T|JuBAbEU;C~$u_Zef
z&mRWbL5q**>tpjeuGIp;;CSK!S&y1|-Y7xC0`K>P=)UEK)s$d+9OTV*C<6iWUlpkQ
zCMe3&AV`WW|2d8Fx_UNTk?!f3;qI8LXl)|-G^h>hT^+w8?3L2r1{WIo}#
z+Pb>Z*;eqK*v2++UtmBny0I0SjD_2y{h4zM`XuT%!ZrsMqX5+oW~P1hx4$G~M0ibc
zQUxjQA$<1-^C=bHYgqB&Fzcxfx9{z?Kmxs~W$cZN%<1|X%HJ*TGJ2NUxcYQe`x?M|
z!QP?wAdKRBjBPGwa4`}uo~C2AP&29y$BJ{)j&pzLp=TM{b+@_K5sEkG`#+-Oz)dqp
zx-(PReXmWbHmS9|kv?hI%OpowSC0|t`{bV~m5Bt|FL;{Gw*L00|AO_)Q$bSro(lLy
z&Wfzme*7n4v5-A48Rt&T3MOKAydDYIgf7bs3db}gozf?Qs@Y&`=48c7BJmzHXF%!T
zBqp#q`n6*^43#ojx`&HA_x9t5bXhP_$@s+14ow>`!
zUMGCe)a7Z|Y&MU|>==oqhsO9a19k4&p^sNP;;VmNJ$a$-E=1X$b+6TKpxDuZQS2^b
zaLb?BvI?(*u-}pxev`jhsYuarw_10H1=DTSho72x^TG1_tre&A3
zwu?%hyC}ATOC-t+9?{>LDxmKePW6+|``%dsq&DTm{m@*N`Cb~JwA!4clw>_%6$v57
zPY;zU97T7Iu8N%ak?98gXM)xGldq#9d>LWhB~smEqv%K-u+Zty|&s=X{aU#>&g*t6-2Bx(_
zAlwF?i>czi12<%He}@vLfX2Gsr3#zfZOP~S^Ny5OQebuidI6iXw3~~rzCO&`NG#Lr
ziy3vqE(y@Zbx0KRzlL;EzB~2(zZq9CShi&!y=`LY7$M6fFFl8^{&6HkHTj(p>h~M0
ziLmYIcdTeZH%gMIf}gpJ?!72vzi{AY%H4Yefr0M=%&*|?@uIUM8+gZUdff8k_%hxv
zdDwpoq;t5i?8}R8RaGA+Z2f?^%oZb#r|onf0#fDvSUo*x(dgdwM%{Zdqp04I&}Wmd
zJ7s4o-WczjQs3QhEiC^?dS~rt%o#U{%^d7bZqq&?^YKj09_s}tttg00e
zOz3NW<4zCpOf7$xP3iJpjzy3Bfb7E`;XwC)YF+64gzKI^sMC=vHLs(m{k8l&1~K8@
zv-k|za~vJ8C
zS|mlT8ex01!8QEY2W6l2{9?b^iNV2@Er4HDcC}6N&l*bB^A7e{^1^WkqOC_aq!F-8xfMLopZkSe><#ck)|giGl6YjPXh3WU;9}2e
z`-D$Mbz~Oej>(Fyus9vlczN^jFV=Jg#OxnZ_dr1rB+!AAvaZi_si|F1Av%R1kscq4
zr!Ze4yiJMC5J1-%quF0|B3h+5o(~-&Yv&;`tbx!+j70oTvUcSW%{`4%LH=Be+$E@_
zB1E+v?Yhbu0e&I3L~5tLC!$c5z(oNT2$$3@)|FII$WMSG8&l|BMj&Y-$ijHE|tOl8eYt7E;Q
zWuPQ{mUEwXTso2%vLdy^rr__jF{QX;$;jqUINEr4d_2Fp2K!O$9xYqWG0KY39XBRm
z2SDQ+<;{AHf<*48#e{Bo=b4MaM@`w~3BJv@s$I3WI;%#H8Rci#KL{e+MS_7Z1jmyI
zRKB|;MvPfO60ne9@-V<&mylAo^gVMw@>!vsk*MB)LSF{38yOhnf6q+L<0UH6Y4D_~L_9mem>6(aW=E>}Q~o
zitM%=RLt3fadhyCq+}zK%~QkOMsEF7IlSnS7re(DB0ggG7sflrTVSJ(Do{@h`~uJh
zpB-AcNfv0}L{XO26>z=9wfs(WNX>rv*d^sG)Fm#h~7WU{^l88O5W+n3=4GcHBdmK
zl$EQSKioZ%_i$KP2#rn3ZJsl3`=8C*J3Jp*3@p=SxRbIE^UjrBsYQvIFaza%RibM+
zH}VL0dh56i`M5^&e~Aq_VTe$tq538K4@;SCM+(mxwK}!Ul0ObHk$9LlUjy!`ww=^N
z#|V2UmEbjIx*eY2o)4=XJNsYaI4&3a1l286hTwDiCsnL_j~jG~kGWRV*m_Z;JA;48
z3wxyn!pLKv7t*$a$G$GbUp)Q|)Ef8Uz#E#8?^&8lcU}48L+59dwEEz3@ATHwn-`_t
zh4s|uvoYfCbOO+5m-$LU`km@Z^^f}-AsaSdLP3jcbma1j3H6aRqfrjrqYJQwxz}$1
zD!og)dZJh*3|{)axWDg~!OHDX{oRjJ?f%cIYW>Y2Q$tX=+gwvf*b!QID=vo!peS9C2a#Bv-#{Cwxh5e>hATYL;mXvrHu{
zOpsZ$+{MsAs6bNao$%I!QZSU45RE`F97J-+*{MKiSZuHd(4(f*K09eQiB
znuqRR(Tn*q=YtHToW-o2_^}=J#6O{ZV|AiA`$+^1D&QPAMEqX>{)nbRCJctPJt>yF
zVFaSICr;o9dyg>=yUbqU09i%(Nwd69UgGl!a=R6si@gqTVMc9npI>@5Nag1Ot7dJct5$*OBWKr2OS
zu{-y*ThNQBx{HY*;Uoxg&jNux;Dpy*j5JojWBhc%27YsbWN`OymbQz#A&I}JW&su_
zU}gvba-ntH0)sVcs0D{6LmSue$yuCnK8EwbyESXq$7gfP^;Zn;g_0W;-#E*oWdOLM
z>%_E`ELwb7WS8Y2FF`3?QYI;6E8XHjn57T75
zzpX>S7!xy-OtV0$ytG~#sWlbdJe?_nX)~UcYkg5l^59D~`#q9Ue0gZ7HPp}`x=eFD
ztj*ow+YlH(;wxinC`aQ+>muP9U3mQOR29q+EiS%s*l;5{f@tMt%OD18jRWYT$_zoI!nL?tGo?jL)_9cyjW$Q_d=Biiwr
zy4)2g7fW8tF~jUb%j)Q_w$d^7c=E*aj%}Q0VqeO_Nq~u;3rKR%iNi`@*~C4nVaNlt
z(gFnwphXNO7gWXkDb(i)Z?B
z+gq>4;Bf3e$Kg)rqVjrm9iS99U*f@)u;GDV1?dVu>G`lL_8rp?Oy(pt_%53l)>@er>3Ruh|>hoPZSktT=ZO0vHP@(
z9fG;%b$yYAo?YytILLoWd5k0dB3LZM6fcS6v#Rnf?!Tp?!AtlCAzl(<5>u_uk#Jos@Vhn9Kwcsnk}&d
zOuV=YirsZMUA>|M$US(l1TRHOT{OpL%I)zDmo
zg@Bd7QXq--wJW_&6fiNqSJKXrmRLQ0|55&8VI+4iW)d8Q^3~Hoe3*ZeS@()K&$EG{
z^XSAhu7!Gwz;=yMq$8u`JUD-U$g{zF{Ae*ME^FaG*~;x*jqaFK*Y47;Uk=roiw97r
zPy0;+oU*kh^@b1~ZiC(Y*gCP|-juxoS0tkFw%?Y8;S@@LV@2%D`HWn?<~%lj2=9%l
zP=iB5=9w8l*i7`NAn|Cn`*fW3T!Tkd!qU%?Ma`-hnhWi>T(Ak|X|=KPI|+(0C3FA4PfNGX9an8CkGyD4
zUMR0+{4pmM2Q3Jp%->P!*7*1lISI6;^nf&{T!epTSwPgGN*%ez*qJU(zd39Cm+&D>
zj+{P#5WMEy=fm(O#Dsp(TBS>U@8NSy!I!D|?2LrcSS_=K<)2;daJ17kr;&+awf^$j
z8kVLW+4v$%We2>J(xvf&cJH4W8THj|r}!{+=FW6U9Sap1y^6b!hmnFZn&vOILS)*04S^hmA;5&8c#QTCbmhNEDkL|G%a9|PZW_^Z
z!z1dR<0eHjIuZ~lou#lHR~qv9-nFbE1GN1M0$8N`k+ibU5^3!?*2!9|p0!c!!sWlQ
z@qQ;h18kbS^)pMP2SG(|eKBASmjAblAqzW2L>aIPS#GFmn(&h#Pw(
zxTE&ikb;~w=y3UlsPVYx_!ba6J!fn`I$6VkhYQnG37I5C1S1w$$Igc`5ZL)||EnNO
zYN@|tevPJj?+-5h^rFA*{I)5aA-KM`N*
z7)G;T_fCrJIa@?&biC(69XCK_N`AOmbO042(JHuGk*NR~W$=h$S2rm5_uAX+t
zlu}osZGY?res-Kjj%t&Cs34cLXuTG-K7GYXH~T}RdGJg>^jppm}umMXn-1Y+m7L7e~LzA)3J(}pE
z8{`v$#`9WXZC;61U&$LoO)W5-p6$f<^+4kk8(;%d8&W7}TMsX!-pKLV%IK_bU*b{w
zK|u2GQIwDHA7y(W-chLhf{PbhMY;Xn$)qSxm~1rf0UUX
zfoC>j$G6!DZ)~-MqSxsf-`bi6ol_%I>Osah
zAuv$n-@V!-$NiG}T-nc=Ym;j7#BIoUImL&R8|*GKO)=YDfjR
zVf>KjP5$o(dJZF6o$2Va?b05Q&f0ZooPMPn5BDX{2P*cqc_v+pAOBk;pL}M*&?TjqCxnefQv3v9ddS!5{LWu}|Pti(}}nNGJrmq=k^!fylk
zUb$JsZ#Dt8L`Y0>FR66+({ZDI`oCTYZoOh_(3E$nmLjL9N6`8KZTEVzA>+`4W0@i5FjHOrPr7$2%#sQ2q>P{e|X-czL_)^&>J3-ZYMh?o+>irUN0>VEVtp-7cc$f+=&XG$Us?
z{jm|iGsk4-BpUz`l{*#3$Y3DYmvAp>ue~WE&8P~gZN@$V^EI~Ot}eFoKh1LZz2zOt
ze(O#fq`fsU6X>aVghMVi+S~0ULnz-!_Ib?-oA*m!nFASj+BHC<-E2c!$N7d%S
z(VZ!MdUb-Y_l$Bl;Ap#Evw_x=lGz|RiNw_U-Bk%Hk+(V|Fa4Jdrfzb&iX|-nJ$_<)
zvnBU9UPX7QGfSRcsJ4|n=zn3Bt=w4`>qb)>c-4Pha+n+EFAkZ}D82p)<71EW%y41a
zcmLzm4m0NN4Um``xw&(l1l~Wfn1WjRUV9&Xnnmb!rV9CQ|X-u&u&T%
z)JB>7K+YK0DnV#_i^fBA*!wu8^SmbkT5C+yBZ)|k>Zp{3)E9}z_@NNj_VcEMJ$%_W
zY1`AnW!_h!Zt6MPkS_Hc0fw_AF}JR)uO6Go;Zx96XXYt&b~gP&C_orpBo{X~ezR=h
z>IcU;tDj{{Bx)mY{v`K|PMOo&+xvfjWqEZq_?gU+?8&&b6*B1JP>{_~9aO}w49F(z
ze*qKrTypbohaON@9MzZ(Z8ZbeCjf!JzUkG~F`OCx%TTltE^FJAKWD?JOXPt!n+eww
zN8Ub7fgiVUQ$^B>5EJdm{nZPvA;Tn8k*DfB>H*?HoVxb
zHG95g(X;jZ?vu@jb~6aCg%0?41tJk+UEgKfBfEOOHN3ET29{>rP0s&V%-a=L`FK1B
zM?VIBXEc%WCRSz~Hk-RJopVwVjK1vZ@1z5`qPrUxuET0YW9GYD+P-{^>f5r)BAZVR
zewcD4$!an(ujr+vqFPZpEl7B>*v`$dZAzHwk%^LwDsf9+@gBc=#$`iKJL&jbAe>^w
zpiV9(ljGB==}8tIDXsGzu2F0wTA2Tu;{9h*Vuk%6FvnhpACi4f{5VE2;k>x&6ySju
zF=E70^Zmbx(l5<
zqdTeqi-1Qq8#131RX#M#-JSP6q3vi&`6?;Psie|qG==YK9J&brR6P}l_O%3E`(&0f
zZbhB!shp}2w_SnWw~~d!IEvoJ)5N1aRyA?q(uY3H{Fi>XJGvs5BI68R8S~S%ugJ(U
zyG50iqV>BfT@CutpAN6S-|SuY0M+#Vr~;lmp?8?luu`Nm5<~C~YJh-cRcX0w{z9?g
z-MVhF9lL$l1=2Y>9U+^F7!awpMtegcHh3*r-Q{VqFhNaXsTLCKS{o)!fFNgKTYV0u
zf8}53y3f<8p;)2Seo_AX)^??~r^hJ3Lw8b^ZyWM05F!1WsF{5T)b+&F?88?(P^AevW9N`G{`NVm2JHQSF(dyme
zxwHe&#w_>;BtuIJD(W{6_tN2`<@~@|QTt2ynBC}Xknip5?Ff{VGMqjdOYpdLg}C(i
zEPOR(SIMFme`{2ucjg$kqW)30{C?i_PrraXP6wvZn+vka@k7#Bp`^C+SvUi)$_ExVkXuhl0Yy%_G*5y78;CByDc$~z%B(`
zQc2u-NK7yg+4{+ucDu%T?cuMm_|q4C5|YH{$!$UBGGPJsLKdWw(z|u~)EGT1fiJr!
zFxpm=5DjC|mk$HbkR&IS!7enkJLudL_KM^RIh>s0QxZkf-N*R$5-!EOTh4|Lsplw&
zzy{`~r*dxY>%*d;(eotPHNR((d>nYQN3qqCEidzzsGTn0^cLNhZ{7YgemNp(rgIm>
zMA;&d%Ot0-R)1u6cYOU6&VUjB3?tt7z2mxg;5OgB`u`!E{2$Z90x_k84T)?Z;sAtI
zz+|{cG)V4Dc&Kj11PbH2y%2F;Kl1aYMb~NL+CCHn`UA5Q4h*X~F{+T(!Z?TPcP&$e
z$`nc7m)wvEAl@!Ce8lBsp!b*nWn%SI`lvyDc^Mrp{VoYa*h+dVc!Xa>U7AfHz>CJ1MhD5oRj6kZ{sv!6x}B&L@x+kD_+0nqIr18NFGsQ~sqZUOaO+d%O^B?|L6!?N)>Y;;0DXz&<
zQD4N(OH8~B<)5Nyv$kwl39}I2mJqL+iH>o@?1p=Ol{XI?Ede$9Xx;9ImIK&oT
z;(R+{X{!qxA3BCM*}2KlD6{LiGpoKc<7v#kCY&Ji2TQ=Cq@d|V|EkQi*dk<8=zJ*s
zs=LG^c{NuYlUL6o`L*!KsZ?NkX5eiR+e>GoWg
zcY!{Z$-{RQYCgk8E}-XAeV`IH{I~=UIe0CB1s0MHd^1IGQ{SCe?sAc4^z5=+S#+wr
z#2+82%@2x%H2?fT!Y@zO#k#mbk?&GdpgL@^BRX*U0SdIf=+Ss8j{sxZy8egy82Z?a
zsDnHoCUE9Q&NZwp9$y?QAs8JT|C1rIrq84>O63Udv%jLfEs83rdbGbWh@tgNyn0wY
zfLxWemjsx0l8vz5<@f@5)`W@oPKRNCoSutAf*;4uE!}CrX70Nd;%1`sM{#|=Sv%C99E8}&ll@w|T!M3h31))j{f#xQ!bP-Fe&|r4
zH2kiwTnxMjE0<%Dze1eJSR`WMEi_W8*Vfs
zevPH1qqC#2`bV(m=1feD^{0sL;&JiI=-Bm?|2bzT26z~{D|BXC33HwCi#FZqWlv=o
z4+rEtpMC>7Ws@X~$(YECWgm8l>&VOqtcuO_s1Zz;6j~`f_ID*i7l7+OjkM9dl^J5D
z(pOe=BkKW7b4e{>djSgmqEg=3)pTNQ`!J?IewTzDziC_lw*jw4K;8ADG%iK^(8z*}^n}Rh6pQ{CNuZ0&@cP&2e|hXK
zt79ZvvORP+@DFM9z~g*KS6#eEimEv5F=E
zGy+OKM-K~|2Ld0E`qT(U4ndLzHBMth&)Lt_=1-{wamFP)>jIT_D(~!8F1H7nvPFEp
zuFZ-ZIz?h0E}5Y8KRgJ);X&~J(t3j$h6+SEWc>xBDjGo0hjR{{3e@*pHg)s2$KCdX
zXJ;*qb}=5Y^?H|Z&)rtcT~Ek7VwUcCzE4`tnSUGxAN1nVQ3(<(*P_RU-q~jF`C7TP
zb`=Vbc$r~k$_deFy*jFQ^?*Kdf+$Ek@##}bkBHT>)YL(nuHUrCg?Y?lk~gNm0`~o
z@d`qdGmhng0U4PSaeK9GhDesEBMat=IIZNI{5jghGH~7PCt&z4oiPQKOWb}4W<&_Z
znAZ+Y4A*RR?hQpyhl;W_yC8p=u1#4+=(9i2Z*itcKb*CUrSTbUI_3z
z(Fts0$(?H9krbhrx>xI3z25-X_|a*|lFH
zO!nZTtAUAMV1KU0u&M(;S4$>dOD3*vbUkKhMqYJV|2_i5y%J&8xU`{0~>
zT^f&r3sWbRnL2sg_{yh+k&mOHk{i=2*;pa5oJq_nEKvPVE3Lb@nvVVGSPfRkyk;{@
z$w~;Hh$_k{Qet7^Tl!JZTT35t7@-t41|%LIQRfaPW6E;?)v5=E`4#e2A|{$j4c#h?
z1J%c@_kcPFmoj2xH%vI-tCM45CHBg8d#rm)Q3Zz9iV7c#o4(otkK#&b_&+%{9CKIM
zQI6PyNV;PtxxD1fObZXoCBD<&jX7A@8nbkE7E4VmF>aD_(9pVX80@9#zR42x3KNM-
zh1zS=MigE+YRAOuFwDaT<8j?M*4A#lhRQMg!=;@#^ZpHz!e1rn)g}!UOAGoZuRFYm
zN{!oVCGzV5?a~a-L{-NND2C^vmfoky20sHcg(!jM_*Kgh&$M*Y7V>r@R7=b%#lnq*
z)5|9j^@b9;XhO9}XY#jyo=A^`Dk1*kpx~4Av@{L*k&yxI?g!ZHLYJiCLWxxh;q(nu
z>baGL!S99iTAk!-LVzYC%pH9If~D=1F3Kbs&&+vuuB1txO++g)>V@;b4|2eV-ur$Z
zKJ+*P3%Ts#v*nNH}c<0xTP$Kwp1NxsewY=nfI~D`nz&CGyVJkt&afu^v
zu#FveO=2=j9zXs;a2^)HC!UvnamJSErbwJ#^pf8bqy24gJL7u^OLFxR@XC_uR^hY<
zDdP%QM;(!5QhObX9LV~QQJmqF`8+<&wms8tU@u4yTv@^J?*_kN3=(j4zmZX+&5(IW
zPM_cyttsMeb1{Q1d3lq(rMJVx9RY8{qlek>i0;Mqp8RsNlnu!h38!pAb`yu5f&Rtr
zArTyM*dQuo(xwOR5T}p)+*xv}v)mlB8xQY<_kG7fDBRqzy5_KS%9}7t+xc+D=LOLX
z2saSjtM=)Gz)CtR*(|Gyl}R8{{Ye`AJ`l7Geg{vJxq4P&%E0s0HV&Fw541o)v2r`0
zoa^6ZqIFv_Gnz#BmlbV!yaN7}?A;Tedb3Cr#9zK29>dF7KPxA{e|KGv`8{EgLEJ7x
zpXo$5ubHCjJNbG;1xe0^+tZWTFr+25+J#wOtLD>Qy~G$|z5?@%1so53O)&13!xH33
z;bSis`TZgz@|}C
zIK}NyFXEf_yhQ{*%3wut2sK*6N`a(~N_~Nh?=+l1-WOE!QDiD#;M5x@|A_xQ+kY$m
zB|K4Q@hra~#fqbn4roSVM0;+`5zGS3~FD7
zfrK6=vT~X_ZVvKqDLEDdNX(j(KoXM@r6&-!v_aMYm6@Kp2`vrY=VBg^S6
z!Kr_gPIrt!cieO&oabKA@cNf1?uD%L=00F+YVcJ~!#6fO%4Y5SNo^4LO2|5&y74MG
zZtXT61?(Ab0KQU$-S!l?y`n7mVb6I*pG+QM4t-@v=JgcTU)k0N)iY|>e5bE^y!pHz
zqHHznwN%F)*tpb&sbRomr2p4zPx}k{nhdFCI6dH7BL-57y_6%4$Gz+rhKC-Q_+x>s
zh<4%51)~7&MRiNfZm=JpHoOQD_v5IvcCxdpfzw8%r&H_p-bEgPKDg>T2|LrZ&Pm88+7&tEMnYH
zj?oN0p~&}=SX`>Dvj>c=>;-J8JaBy(d|QQU(D(MkI`$Ad>1u+`F}%4UH>on_t`oW-
zD(f!$zDKVRVls8h)I=1J^`G|*M@mIC1-F=GNc7##!P0)9Fa#iX2!Qd#*-|G#=QX1#C87Lg+Dnnh1
zy^%7X3iQFQLc{{w)7K%gDtjAFuZw&7~m4kqTKbx=#)px3u_
zOqTH5wJ6|`KD>A#3UGeGI_8Yo|6@qTD59Pp_uEov`5Ap>)=x&)1`e&U<<)KO$Sp`o
zlpXm_#i=>wt0myCcZOUjTLY8qKbtge7@76!YEWvN807xyQ22{VLO^@q7ONMUCwRXu
ztvHoiV14jyzYLsYua_EHqG{3jd=PXD!i_4IV8EsP`pOt`yzvfzqk}3N{@A%RpJ2&L
zvL)_&`!}f~s3oV8UswG}-{IBa^Dim(Zr(xH#l;A7XdUzTE`z+IjuI5Z8HkuinI2Tsxj;mv5+5dKqC6UOEoh)Wy-G@
z*Y!M9susqck)8h%&~pwd)S&)X8{g8Du8c2Xv-WOZVBJ=0p+Csw9iFbXK3pxaQ>fh&)&!X}4ExMJb2g|7?P~W7x9@w3>e&^R66n8hR
zGWnSc&#Ch>uW=+iG>0nP3Hhj0O!)IEqA+Yjk@{Vf%+~e0rdCro_aSE$#QYrjLWsEM
z8L>!&fPkS8Fj|&7o<6Ce*2PG;6FAN5JB-w3Xsz`}{3mNu|Be;$iIC9-!!o*i)83NZ
zFkOv~{avhFe+J(9L=$A{Pd+>0G0%L(zqQA`j@#Xg@y;iM+5)*MK*I>
zD_dp`d*`{Q|KgeL{_7p{owPt$=Uu`&Le?oiG5U#zZhAX1KpEQB(*b#afDwzh<-%keZv5erl>BmkcEe3j1xP-cgTvc2^OwJZ?-yie
zIvaGkA4b7ZAr|_c#L6C9+EuOr=P^@^fS^>&=cV>@tE_^CYA~CWF>r<-=53lrGc8ND
zHx3McYwAtapf4;hxtqBDymv1}~5+-jz%(~Qs3Lr+EQ7x;EJFb853hXN=Se>OFtCvCwYWSIoE{Xav3nTa;kEvd^
zdq7}Mqw|Y1wBX`ffa~-}6&#G768A}j_`^Hftli(ve`A|)e|ZxYWdH~>lgpZn+q9hB
z{d?#3a3K&p<}{|U`o7;k9EvPk%Jkz;r|O-D+ntxkI%!K_6r&#@d!-oDiTF*>ftQ2Q
zU0LTTIANbgLqi4U0}YA=%S3ay4MUl>tR#g#`H)XmRmSKG3s%<*^kk(pYl#f^*jaB!
zeb_1Mt8zZO4t)&zZR#EW2g=9(!gXfCtKC}$)jyb&qI_W_`hMYFVGj;fMRr7RR_4lh
z=(D{Hgjt6uGG9EAot8?p`19^{7JT%gz%lBld3p97Sas8Pc??tnKFi_CD#<8ezT(mI
z#F9Uea4uFVTXHD=&t*FNc!6Y^>`@g8!yg&Go>@^~DnK?Kc~1N{ix~
z5mGXL;EYAj)y-Q3itp;v35Nmf#OIvj4*5Ks1RD?{kRpg{&Zf|ZI;jD>p#;oSVKcNQ
z%o|e=eYiSYGiBBF!A+2&od8Xo{`j4oG(ePOq2}Y~364*SmEp8`oOmxbFX1E(Yp1%8
z-hGG{%|2RuDfwhxH1-HEwG2pgJy`i|{unE;ffv1NcWa=on4MAmmS-Gj76u&hVI=vG
zet8Oty;3WDt%5^&;~Z*|LJ{&*kEgFhID~;YN9p?|3PUUtqmB9pE5xt{L#`#NEq`AN
za-*I}vUj|Pnui#^gy6$&LjuFF+7eVBb)%k*y!tyAhD&ELA88X(r22pyjsrL(DE>FA
zA+a{ma9Tr~VEXY=M&|kl!m%?MTB6{xOoDw^_pTh7B)5dyGK`Zf^Utn7A6BaW{x4^p
z=pOyj13go_uxdF+fGT%y8ND~w&5P(%%&-hwdjgTt{$7$$Z4A6vZaaS1QB3M6c~^_Z
zYv5n+1&36;!bEtR%_4V@f*ejI%>y{9eOkS3Lds_ajxd4si5sOB&OKk25Q2g2^_Q96|XWft=U%c*W+Y
z!_!eW!jC^<^Yt9v-G`Qz;QPDXig8s4ARaLlL<509PCR^$+KcTvuvt<>Bdv`-Ssden
z27}r~C!2iE?6aK$eERoRzOYG@5wVWJ9ZYbZL+Y{Au%NWMo56wqFftfpeYxtm7)Qr;xuC1Wt+TrFRELVA)2kaS-(0du&V*lzC
zUDR}3(T2=}()=O5*{X?a
z*nIFop8eqXv)DEy=Q(F_q-WFr_47T04lll2lq!0SV6r6uQMLHgI=w%3C5^4EqrqiG4)gPbVQfjD$9{G2@TO=5E$#v2{%!K#GX1o=TaaMSV6@F9|M7=L|{TNN1SncLI
zapSBT@xxeMef#YPR>ArxvK2$qo1{+vhZw-wh-;8Vec!gEjsQxXC$pQyA~%!!DLnjy
zD!QhBbqWGL3_Vx~>#x;vhBwK~N>p0xzJD_nvp}8+PaSc&74~r;lnfN0MxceA*
zev*l5cP7+-w}8~|cMwO;`TUFb_}2(xQS5_~)Pn2>$lcCgezHEF*YtmDhPcU}%cqrp
z<7%kX>JYbXFgC+NCD(2ee&CWwyBTqJq9^RrU=|8FPzneF0z@fKlft7Tl%U=OoTTYn
z)($(ns9W;eZr=hDwmfBRvf3-u!-G5ZShcI>nZL-xHF|1H{21hc`Es4wHG6nV!0dRQ
zm^dxirJO^?4eamxhj(xJT76|YuE3d8TxoG$x?vhMvJd{^3@_{BU114(CfztuZiXW&
zD~#o>sx3_w#}mBb4DN+sPorKFg1RZs#t1a
zQ6xgY8HHI0%Jlb0=Dh^}BuEjllF=dx^4Q7^S6ophyS4S8cz3Pq(AhgTha1B2%a&-4
zcHv`*jJ(aDw{~4QO0%2fvG3wA_6CwEfIC8geZwJNvRxGBi;%--Gi$#Q_pP+-`r!sC
zcAzs8ttYK$>tZmSk4*pk;4*qJ|6oHi^Jr$npSWiJyhb?%7g!DU@@g^Zs5z{ZRJ0RD
z_PZLr5)pg09VYWHDA1Z{EMbdcqe38i6j=)kNRd!dnITK>;!Kd2;k3TAt!E({^RoUJu
zKYzIh2V~i-V+9KOYjBP-RX8kzoHNS>8J7b~c#Mm_w$9gV@8j
zo=<2oE(;b9nW}VN>zx1Yp5mjpiCo8oKy>2_GK*;w*P&mgU^KuHrjdY1+Qq+y
zC`t9R%=>jnPI$kS{czh1Serl={LV5ZX2sNQ&uu*7jIsoSQuf)jYUgo^XF}T^de++B
z-nYyu#7})=I!z*)h2+A_w{1$gKjgUy|67_COnQ_95k$&-=3@%`jbYQ52wzC~$u~=T
z1(v#MhVTDkU~FMF<$dL%{mt_8e>&%IJX-#qpLRu>2T>$0BXk@5Dp)}-sb9zoaNn-w
zcD4a$ZJUzs@1n2$>Q9Fv3;1R2EgG>2ASp%ItCb7{uO8X}G6l9Q=hBEGw!5fY{2ea;
zDPsie?%uY){JZiWX*o)kTrS(#Q6Q^6jwT2o`u*sj6abuw1V9ex$%FC#R0*26y@by_
zzAJJIpWywP7${{_T8>hr9=^URLwfCpV*&LeKd=053%@ls3$%rj3mjUfMj=K?A+~It
z+53R^U%u#pq>9R}1Nk&l;mDwde94N*)S(diazngeNsi!oA@wj}Dnk6XoLZ
zC%}oH*)n*d{*K%ewDM$C?7|m6mIC>S^0gR*QPKAL8aUxwg_Bx8NW`|wLdCNhyj3N?
zWqqoRll%4}H3e>~CNv;L$<$YdG4B7?a|b4iT)%lXA|B>*!F#4J)kqy7JDk}p6OKPl
z>~=E9uW*a-_)^83b$73HkeRQZJ~pc^1xhl8lZUqEZvQsGxa=`d(X)CVFz4kC*dNiM
zX!9&fr#+HzOwj&0k{3!x7v6
z!33>6aovVFOfBKA69IIRxx;sH!s|?$dUv{hWYxNbE)%*pZ`RWb@FAtP!*$1$TdiZK
z>b;lSRNcEs%^Cw|7Z=GD-UIvy<<8BbTs5Z3wn7%tsUs#q?M<5aNL*f(Khm-$@M>Gw
zJ7`+qE(C+#4Q%{?z3zW0vL0|8R{32_rq;D;vy&p;SSZ@r5^mP?KViCh5O~d
zNf;pSb;HtmqEh~K5%IFg5H>L$K$gP
z^nNQ+N|*&}
z6Y$D+MFx&|+h2Z>+&)Mzf6UDn#&woKlVeAAaj|F8?el;IHS`_Q@B-$s5CS1Q^HzdA
zxmV~^Xn2E=%5l_eR?$7LtLQincv;RZny3Sw)~9)E!U`%U3S|<)in?`^ClQ-7ff=z~
z)o0wa!C~e|^rBDMoCj1n=u658=bMrTypVGal*Vu*F~Gl5Jo!lh)kxx5H-wqt#gKfT{D>7c6$^F<8)mUTVWi0AjR$CQqVv7a(`
z2$EosQh{F&ZBjvIWNQyN0)_OE*-xFi%!{YZ@U^zRxz}Ak`h!o9R$`9l73${OZ=S&n
z3D$y2GFAmuR*n0%U2vT37mZPKT{yO=?D)fqJ%_%c%Gu=pyP_J)IYK7HuhWGD@v`;5
z-Xy6ju})GFzrW<3hRXZpti-#|rf@3Y72qCAebcs#lx|RCaa)O4`C&*
zbaV7kq~+=X;lECq`S{?r0fgXRVPp9tm!mYVo=plio1HQdwR*{!HF&G=J*B<3?Swt(
z@WV2l{sAXva$G`of46x3x2@;jnZhFm8$JDhu}%3d{6eB`nUrzUW$9E|zsnTCi$#82(k!Fx#(ES^bg|P-CL{x;
z9DBzBjL-4pV-3iFbmN8k-LG^hv~)c&RJj&z>G5ej%)SmtW+@SEFYObO&-(vsz?0-J
z8zFKjov06JYtqq-)zaFVt(-#hFstg)60t4G>FjQo5{utspU@L~TF8XvdLo3=-2#`4
zDr^8YS|(3B@x2XO>oH4c)Q2n|c^i&9QCD^3Mx97RF?KCF@ZARrMZn9Q18q5-qd~-N
zpVwIK3<)U%chTK;Brg4x&;tcXo!`|`N73CPWANj=;2uWTFk
zUuts?RXfB7^5@WjVOO{rYa1qpJ>a!lvh#!Ojf?y34JWv%OUfyVzBWO8Pw#-MNEmcP
zHMT7Psdyb5--mPOU>nXI4|FHT9(C@Ba(U`8;eM^)fr4-EEeHK{oomx$7~0s8SR=)M
z&MzBdR^H9pN$Qi;>LZB)^ly}E%>$XL>-Mm1ouo?U`H>5^Df(P8<*B3y%+u~rAWW6I
zKNR4XA&Rjl+>U2MqZ5p__IsC!`JA1%D_EnuX3$}c($rsWdvb6Y&ImV&D)6yx0
z@pe(bk2@`KhW`BEF?#oMGnbFM$}}idxZB1gRCTK|LUo3?Z8Fb=nVnCWOviEE3PY3)
zLNKSX{7o9?V5DSzxxKo4%dGvYH6D-gbYM%|{r)ZTM%z>P-9)yN0V(~!&}|rhR~DO-
zWwdK%hDGHyN!%DvfUQTKf$V%58Fut{upSv}2L2r2uRzZ{26{4zZAH8wTRghRbxD4U
z`ivXVw5dnFLLmwOx{yLXbOENrSmT`#$d-qp3Do15JlFC|vB+M*7LYvq1pTO0zFND@
zNs1;K_H{0^O5E=0k?FtXe{(eM-QK2*E^;n;zJ8Hp824_=Wt5)>CBjZ)GZaD1&9f8v
zehVvyQ$ga?036M4oHA6>#-(NQ81W&Yfb{5l@uXlTsjMWd|_dBj}+FJekdB&&fT(I11-xdbB%XcD13Gl9BWKCKDfR3puGh!jbo
zH-JW8NrJ0MIpwk3l;YWR{end@9YVT@#
z;(Jw68F9^QlEESe9%(pP;PhVe=%h445
z7#9!i6g~aQRzY3W1^w{HOa>=h*wdt|@2xQ?EXU5%x6WKY<=_3@dAq)>J%b&{!{)d;z7uqh;^Q7aln1}d1
z!K?7&bcG)&+T?%kj}0Y9?&&k0^lME&tj?m#!@i{0rgvs*YyXPD9A=lc{*oe9Vm=9?
z43fQNEvmU(JR%$)~PT@aA*W=gNQuICID1|6#-Zzkliea8C)*e%szGMHfdKu-o8rBD|3leicP`
zc}6lk6#HWo%g$3z8eO|ULWF9#ch%}_#K>j4^*3=_CYS``8<4YF#pI$)q*pu~oAb+M
zX)`kwM(f4)hpWqixM?nKd47T1V?uCGfiEy%?Qi_TULXW0>_2wlHWY2h6FVTKQ{&Hx
z#AUI*Ynzb>11v@xG%_$gjW8@C84o%)?Z&ih?3{XtnzyXMckf{WO!dOj
zjnmUam?_uuje-$^C&K4&H(7Gg$ktLD9@ZJpG7Q^mUX0k02va((5+@xKH%Yh>_&xBw
zU-!rN2WcxbEn$fI7C~H1TFL8F-TiHm%^myhS3rSTsAJDF6{-G{Pc_>s)$!mRhj|=Q
z#Lf<`ZMZT%WJsg6>YO_?#nSf4!W3m@$X&SJK9#`Jt|F%|%vs4ru@#_oy;OhszRktMuM;YN2uXMi{&OqV=Tk@Y#{L$$3xu>kH8_%MAcduw3
zuzsv)oC%=r*o=3Wwe~A|
z++=ree$lm>Bc8Bc`^0y*QxN&9f72ONBlM^_4!z2kIu8)$0414k!<3r!iB|n8A&3`l
zhA|fy;Yao7Y$WUTn~S$>MpAJJ`s09CL6PKu(IA{15XT2>wVJp$h5dHO4*2H*Z}qIY
zVVW32I7l6`*3q17`wWK(v4jXZR*B_J@<1LD!KnQ{FEkzyws&3{!s|>E6W=>w_a^IQ
z=$W{DIPHT}uB-r7BtG?e=o>r?Fu{ox6k$u?kQ4e>Mz!7JArMu&#}A9rivZCTeQ9=N
z{H;C4h#lI>7j%|J^ayEprOIo{ttC}Svv{|Np3+Tkte0vzU!|WfEbw>${`lZh5FAg&
zs$*cMYv00R=kjLZ)1PwhTAr|vc`=(sntbQQ>JkX7xn$B;tC~bv7=elf=
zV~*8|By;+#H#EKaUbw*Ot;|KS&*xD`xBHBw$xOG!iA2PsX9@bE(sIONdaY#KsKLZF
zchF01iOsQP7@{^OO@7vLEV?_X-}B`Mpxara#!UAU@@Qn2%5gP^b<
zH)P9Ouyp7bS-wo@uVjfYMaT)+8~)CtaGF~xsGbaDm|`j1#Om+zyL!NL?$>-er~%iZ
z$Enxd@apJHoS&PqqVA{1`BU?LO@Kd`Z=*du0^IXnX|!%(wApvo*~f}8^0LRD(~B(K
zg=)J}Y(18A^tzGXXL`5wr5{D;=0yBbq3sUVr$V}OIUI&-uH)b=a$4bMJCZAZ8EH;q
zhL(B{leI8&8?{@>W2^V(9C_4Z{dRj&C$ib#ckNl}GVLPxk#lLbO>{gXDWk$q|C0A%
z2cmRjOOUPjUXzRn_XsogCjmMm2;C;)Px-eOuPoRDvx!d;bh^;G0iw1+&DjO?OPka5
z{s1!_%DML{45OJS>x{d!Flfkr?3$#8X5RYIUj(+RrS}?w(Uez}f{xamxy;c;2CW&)QkS
z1?`{w5=pWX56|NrGyMU%6CddQl~tC=m>ym%lYN{~Tf#d!ibGfV{q}!jzL|WO+ByvO
zhzX(swyo!k&Ip0yE(KFwB^QQXB&c?$?o}VvV$&6BK$`~(F#d;s4^W;5h2yj!&P?b0
zf3STDq`k0A*%B*G!-*XWvV)um=w>uQ&&==YS2jebicV{5>u$!zi^%N?oY!_f?5ZE7
zYEmL;L!!U1oeA3>38oB3$q56YztR`6FO%nAtk-d;@_lE48w2Kx_RV1#RKjpL0kVqB
z8XjrnMMluwk3b2pb)6PpeZ4^4Q43tlr
z)fkh5)l+o_QoJmkC2@KkcEJ$ctt%nV?u%IxoTA!Fb;t9;SCnCo&d(oO*022>|J>cn
z-V*0?2}y8EBw#YkdiBD@%cmo#*F4~D#J!)Mtu{D>H~FOt5G<{k#EU3wLbV|^vTvj~{Tf5I=g^TZ9Wo@b~;ED{cB+p~Ut
zHPLtuZrH={a=Sw?uzO)xc_xGBy)8TBlgm}6$)^+Ao-n#_3dB$;(l2`nbxH)Z`qxwS
zzN6X9{Is4VoTO1=bBZdd^+qt%+n(jzKg0}RoOb0!U*GhNaG_a84e=VZzMx_Szfw5}
zzuf6%2gI#0fta2Zttaa2Lv9!`vlUBAnwZ?f?poKLunksUP|uoQgD3n@r%(aUI7PR;
z7kb2UxRz@9$wTgFVW%u>)o^fZP`|yrg+4Jx$mf{BM9TL88Y^3%fBFkKtOz<*-&2&W
z71y-WCxTk|bE5(Sg;29%rh48te$@F(xrqNVF1Ueh`C0pVfZWw6G}H$&*afslHrJnK(XA;fR`MFYo^ib**kxZ514=`-8F(z)1Q{FR4f=vM*`0dE4-k3Twi^TZ*mWJSAO
zG0LZDIOW3x%F_<+*&3>6V-RKX6_Lizp7#J84Gq;)#HeyOpJ>$~xc{x~Lg+~K&)jo*
zL5FRZJO^gD%_OKK_v<-7cPu<~8tnde*msC~Bza%`{cdHop`$}P*`wLrXN0fxz76yB
zEB1$iAm=8FHa!&$)W&&3Mlm>~yDgs{Pkk!UAQDl$aXh$@{neTs^k-L*kwmY6S
zK}9u*U!Yx+W`xf2h1nh6^@VnmS8Rt~dMmDC$1zCI7b#t3X%+(qm*9CKUl+m!B&#(l
zJK-?^9uy5AvWKLK?b-OToE^W2A-dQS2Uc8gMm)b<^JDE(gW%$1#7Pd(_XK)s6kKF5
z=qCN5g-M!t{s3a_l{Ct-&2};Qf)b9^)rUuCuL#=Q*QC
ze+QlYG=>eXk**gVZf(PbP~?Cs%^)=>g<-~b=QZz4@yh`WQ}4@Xi(C7v**x+?eQQBs
zPB0y`;SdJ4%92_523Xp;-Nnj-MFTqz#wUQ@=lYGts}>JmWz@PDg1b7i^p5Y)mpYbw
zH!#8V2N7d}^YTgXI1RSy^(DjOwYFhSda|M1?w|*GWUda9et@3&QU75tqThz%81~zX
zod<3|cT3=6RCuCF&yAI_!9XKLAy?Dq4^@{J<<=&k^!9+X#@UpZ`plb;F^qv(KtUuFtzXiEKqMgD4AsfS}azk_0e3@ZWg-=_r7
z)MlnA=7=g`iXhbd3Wt8`Cme$~^17ZN3RY{nVU_1U5kYq9#so`MoY|AD}?3Kb4Yz#
zfqUj+Jd!@z1fa?~Ya=tqn0S46(O}4C=E<=`%_c+PMH+6l_iQ`$Y~sr_X7A
z(>G)|cP?;mTe5oMg$h2LSn!Tt|1B*+==T;ETk7=5)ri)9G@@S3r4k>}t=hj*uo)A1
zz#KNURlj6QY!vLiSyX0I>CMw;=xiar)wR77K$BlKkGm5U>ZBN9hv<`C@>kuc0m3s|
zX9Mc1NtQzC-fPl>3UF{jdMzcbnB|&EoAVBt73o}M8}C9#3?(*K1mExdjo9o2`X1Ff
zo03x!3+!3!xa{s^W1%QX)15%ZFsIPrNCteTMgteM9G8vid)K?)alF9$u8N$i-Ukc4
z?{~@Vl9`jsjw3&B5GO)p;c-gww$<$@5lY_WM3fptg^=*0Q5w>P%b-DznV)&V
zw%OyP8NSO62?(8}QO7}j@uD{E)Spd;=D49Np5^M86!DCzlpDgQ>Cbx<>yZNC4Oe%p?~J|i`~Kjhgx~v2BOd!
zTegk3Lq`=GqcS+4sT2inKCXAjJGJe3^*d!uRnGQ6??iLcWSpgw_LdI$IJ2C5VEwgD
zuU@B0yBa*Te62!-w8j%XEn9JOb3yeLDvTrT(#@!yjMyP5M7xquSvZTN;=$SJaL^4ZKicJLcHsJBYLRVb9FSQ1h?!T0D+PkF
z2DKHg-PS1X?(Xhx#oZye6?bt~l_BXj}hei8FG}uQ_QU^?UV$
zs1Y^N3MdLhokpFbKnQ(z3flXw
zhn@PR0H-MCf!;>8C?+;`IxuTGf(HpwBzBGal7zPq#3T+tpTC(KJ1tY7
zDfys9-Eo~RHS{!m#B}cpzSwmK9Jmcd`u*eUNiJ#ob$gsikym%N8m;5d=?|&X>_LmV
z4NESWQZA^ocS5v$6!C`dT%jnKC9|L$5bQ^tk*r5peO?A4Z^~@GMn8#*nf9PP@-6f#
zZ;F}lXx$9-jzx%vmN;-nsWz}VJ%jtN?z0yBhpd|qCiA)o=2|r5_1?k{%bsS)xbUBE
zKkC}np;nY{OOo}lJ>8@>yuFcyg)7Up5{WTO2C`|Jnr|1y*S!wh3O32I1O)kruj3=9
zUHJ7!AY#)gXK|SSA0IvxS|vOhgEosUZXX0Ih*n}PJ9
zHBlT#$EPSNg7@MvkMftJM1FWu)+U-D8dxX#PO8L=Q}YiQNdvC
zP>zY2*6##(SO^0WSP|rDQc@&4GLfIaQZ0-tPwo^6D06un+jC~gIYbGh(O|;O*2hiJ
z#u*uoxaC>QztDI>m%s;Bz8qYc3=fZ>l_Z(7U}GIv2~>t0p!A-C+2w8~oOi<&rsUEQG%5;ne)RWsIgf2o(uc#wtWjyD=6C~#
zRNKx!F9NH5d+Y#KQN|kXpt*H@hDp4*66qqZsZhRPw#L)k`5xrH&RbqH&{KVnXsFFp
zR8h+`;-VhhUl|2^0Q_0*LWHGrf%6;hu`1&&xtwo
zvjgS=g+NB56G-%_V+q5&7MqCKu0g=1DN&=)0>ApkLb0eZ4vZ9AmsP=AFXxann_3OY
zh33%muOt*Fy{8wz)1k89pey4;UhHzwO0RDo4KBD&>)n687kBNx=Tz{F3-{uwzs7bWVCIpXT{K}fz*!}mHTtNJlST*@I<>BNb73@__
z>Z;C*2)1ouiY+ForC>gDZ+>OV3rTJg8U`rg;4(O^e&7kEVEPhTHNwz@
zDwD5uKxoxmGNa+(@T+j*CSL?3x;y#`WHvTc%iHRIOGKeL3}$-F@}CLKBqKV`$JH3r
z82n;qy=rzG*Z;d5#~F#T9*&bD4hWgKF{VWtY@sD5YoFVv_Z57f9HPD`)y|4WABYGg
zh{w9?Y0UUzpMR>llFprfHhKSdPs~&}U@%iHpL(v(T=;;8
zTCBc^Xqe~YEiUHga*CSuia^#2pQ*tM
zRpx1_`|Ino-n&|y;rMi(B)~WpUspno(L^C9U}D@|*<#%Tz!>7Zen@p}gN1o$;=$(izi^b%1>!AdgaL6R>2seg
z6&>(BkDDcbzjPAUF%AfT%vPxWB)Yk=kdU$}q((eY*)(r7K+Y+;N1)Ij^Rer0I)K!K8;rKtR(IMSZh-O<;KN#Tx6D8_m?Fz8Q-h;${32Ly&omd{
zjGoaa#XV+FW5=Nqev^Ov!fjBJmhQ_(pFc5kh=NNysE{Cyxs9}f7EIcd{Fur4_x&PP
ztVRAHiw);-ZE;nOuAOWqxj6muR);zbt&`=&ZVl?=q)ms#+VZ{=Ei?0zxj(jEB>Jn6fE5f8`sEko;zZ-cU}_FcUZ(o8s;Y
z9DUt?xbdjDoF4b0?d2qQ1mAwVI@Pisi_YD6W4kWl5B4CuXZ_8(W*ZItPd+|B3z?i7!a47*2wtLkA#
z;esSLo4=rbTHp0=+xB=N;bril*!cGpZ)`N%TOT}b!(fDFQt9OMbWtd~x!L8{Ka4G6
zdGgn+Nb=44g00hCpWHO(jZf|Mw+iCb#v04h%^c#7AOj{N=H(a9u#lkA?N<-+H~B8S
ze}~JGKjCGN)*PSesW28FA4*Y=7d2jrvJDh15b14dT&&r^m^3*aT`+Xj51*!#btO
z={9wb$_g;73%GIf?c>L|km2p$Tm<5dcaF<=V697lZ;-9_;rWeIDRJE@)M{Nq)(;K+
zWjWr5CiK;5rR_<@5=%_&|3^6jJb{~sKbIRc
zqTz`YOd7>6`7C9zbn&yXx%7MJ1J?;+^zdEyIq5^~gwo%K1K=akC9f&T7VS6FGUZ*N
zT#oH{B!}&P|I*ufuh5Pc{vf(~8!q|$D7Lfr?jq%1TB30qpP9kb*gNzBVt-lEIeq=}
z$B#I$tOa$kvtewXbC|v&X0g`wv8<#Y`c95v!GODfT|h{PQbB8nMv_qmxTdj$TW$xs
z7+p?Ya=`+}lFJl%NvDqs=y1=<^j|6qs7BfxXCIHdw$`rT#p%?wH_zuzM(Wel2CE5C
z)h_9KenBqKd@?4jeElv^_XAthlCpkFC|r*J;;Yr@ShiL1{nY@WX~1=QfDLk!?ET-W
zolCF2I@?O(H6$q$-P`X?uT|zR1(|(*JHUQ}$o>y1(|;bJ%EH1zc6mjOVF3C8oV7{&
z(?PachJ~HMDJ$%>zuEfQ?!a;Va=77U7-qrgtZiI*~v
zYx2{_XfymMDy1s~-|^B-V%HA&8L%Z3r^6|-ib@eTqR!qfn726&?ZjY<bnvE(Q~S=As6jCl!7(wFR9Dhz65zDC^{{X1>UNW9GK|mv$k}
zVN^xJpGYkV^Pcus6Gz;u7MDlPsI&9CZUX|XYo8;u9kdbBY4l>@BL5i;h
zh+@dWAUj8N{k6N7ZwS9X^(Xg8({xfb;p$*FR%?l^_e^>PMbi%jyHv@`>
zT2lSkgBP`B=Xg}_jpE-CNn+a}tEkWi<7Q^GCgzZ{__{PN64cnO-ktWhOCh2V=2us#oJ|=NuY1SWfBr}(3pLEzT#eLps+5+W7B?F_z`kR^&
z>0(m^s4xe7nU>M;{F}eDlwaNm!+C&-xMq`
z9UiD1MvAJ;soN03EIPP?*F&j8FsS0Aj@!|T)037@ji6DGpG)V-OR*BZVxeYt&jv3P
zoiDR9NxgkG3EUjMX4)3h$i`C^_I4=KALmtoeIJWi9&^w}>b{_k6UOLF`IogYhRN6x
zEhaBzg*eKiqh)do^WoSpP{bLSP|s3AP@_5lCC&NZgctD7=5BeB4$E30VlfWjU{O0#
zm0e|5nMg>c%TtHH3oDh{(58+|vM8pZ&1a9X_9&*2Nk7xJXD6fUZ{tQ2#KItykiwE9
znZUMIkk)y6Mk^rocO^TEjZp9^2ICh&jyV>w?6yyQCv)zmd`~A4;}<>L7(Tr5TohYc
zviO;}gJ7L+&HMHsYG%c%s^VX$K%E-KeKhB3gQD~=54Y3clGRp;(vRC77^?BYj`FQj
zK5l-7dbwGvqDx5CYv+xNGzt-#aaDN0pT$)#mws*O=nPd+=mpLXe^)Z;_kfo3-B-H%>%@EZxZ3
z!+H9JhWv}XplMz4OBJ=HE`~iXaP{Wz-Y0Of;KRCkC;fu``VJ2ni|0?H!$=xB8_UK5~XjGPHShuU9E6V(R}pAi?0o)YTU&Ni&S|1Uo3Y
z2)_4ozQPCZtd@kPBoex$@=a`vkM-0U;d=w3$x5f_awDgs${(q+r~Rjpukf)7!g!vi
zL|
zIm&FaqZTGm4Hz3TRszo+s~Mzomupy%Velkl;%WIOzPk=^+|5Mn5L1%=ba(CseMX`E
zvT_p7E$(uEDH_KElYFG5dEkvvd*Y;&H|t8Yn0G+FX3;I)87Qxcnt
z28~?Kf%U{=LeWrS-_EF^Pux9^U_{ekU+@eSB|GY!s^TPGc5*!u8$Yhh
ze)kKezv%~O@(y&3>6hNwNzxA%A1-rHEH1Wdb^3BGE2nTeNV
zBb4B*@-aBram>>db7B5sQt+Apm5>fMhc)|6Hs4@ocES1mmk2LC>O3p01Lx<+RbK$&9(~Ovd-?uw0wrM&>AyK*?=X51
zgx#RU3ngg&tr(e#br=ygVBogFY6pr&Tog#|v4|$KjkaLuGe~UY1k;6L$my|Ii|;@0
ziM!tY3)!{3w4A+1iZ9~)19HJh;ZaRz6*Ob0UTSZD6=kb-adUfsI{KKFb8@DzO?SGU
z&l^aTNVnZa^kqd^v{`xGkSIlr0tqaq6-(0lTk<==2t(0Qse#KTQDjHX!0TIMcuD?~0ONesHD|JFcJ(RW?}>p4j#KQG`Ah`ATOelh4aQ=pPkj!enQ5!t_bOptR=K1yVM{a4FMe@(=~
zlV@Mz=C!aI6~1BbW?}u@KK&R1+#vA5dT+@B=K02Z)K9?eym|Ioc(_5wcm;N#QRf|*
zBR$I08!6#UU9x2z&){?`1!SW3c8}k`4imqgU1k)REQv5<^YQfoRJfK{lC+BHyiHE#
zv5e(dJhssjWTuGD=!rE^t)ST#J_%s*hp_oH)8>ZLjdgFv*xwNiAvu
zdUtUA6T}=>mPHY_kSjsk4A%m?fpIzGiXHXU_oK{l3dvG@!27rYEf-Ah7}A!szH#gt
zUN9>QAm!8LtwYJ02q{z9`ZolXT@l_)TA7qaBeG@yi~Q`*>$AH!`P6g))hu?ruU(|a
z6WJnb8hSc<-fM_q(SJXQWNdiszxT8vQyZ(l7;k@C0S=H~jUPIt$=hX&!MTxO%W(R)
z(y=2g-6alX2vZqiujOi~61gm05rpwB4D#}&D&}t$Ew52Z>)Y^JftqQ>9
zsk^Mio3wjTB3KoC)p8h`5wYS_T0X$_vWyBq27k6>uH~0E#V4^1lyUwUHM?FdlK#`zI>w8+m0QL#JJ=?UkX@}4vW
ztJ(G@e3=8}L%s$EHd?0X<&uIrhws%sSTbe1k2&9nEvfZRO$>z;kFytwdX_GWmM@E1
z1wBx!o&_aeM~4jV+_QSNxQ4xKVmP}IJ;q`t%{BBs5|sLlyZ4jFP*G4*y3tvU}V|LxqzqB$D-DcYGZBIycgxi
zw-`)YpX3x{YRmz{tdUB)5oV*&xIlQNvy@$HJWX0mVvnA
zS%k6Gc&F$Nv@tw?BQ)d%UC`QHYw*<4(Dr8HQ0&)%rn|#{9FPk?1Z#c6WQyub3%;JX
z8X@sGsd4^e9Lf3IZFRI*D6`?Fs%YnnwPhC{X3w=$z%83|A5!fSv_TBy#ng8Gu)8Rw
z$aoMBXJn$j#Op?4c`W1`@~W~@hti@q^k=I%fF{Dl6IjpPtLJU|=CFm^`Lwb0E50IR
zINbf)b*Jc27nb%HacHZh9PzK@IsC2olYPNKn!x=3lOp`@Ky^GmHMrv49nDD$D#JaePu%%7M_=sSMqOupHig~@EzZvUrWtIiwe*f
z5y-3-ctuv-*)CKhAu-rohmy}jfOXvexJDm+^R@K*$zMdb_o%E0*xw;F>`LU$%2h|$
zj&uKRaKCd(ec6s5DQinHunL5cnVS@y!E`8~6Ar4;&*=Z0Ou`0G4b1<_4_Tv@+KnH(
z1pd77AM%EUTCZr)f6zd}N!JRM6HsD%!0VyqK5OW;T|VzFb04q#c<2%t|VVw2c@OVgHb-J$t5R=dyCT*Ew2
z@?)PadQMv)2YHw~OQ!0FN|(m0f}yE02!DU00B_0KJ96K`!LOUk$T^~$CwF}N&P46u
zrESW0_Z4x>@5{PBLIG_Y`=)}Rscg4f{?*2H0y;Z#T^nzrpZRRKf#1xzNnpw6FFf_v
zgzh77mbi)u*cUz#kGk5i*i#)ff+;Py;ehzUzD_?K4uWb83-#L2RY0Of9C)#`lO~aw
zEZhd@wif0YAexvyVBMad(U<9P#BRrd4$mqh+)Qa_HsdG}}AVXeJsRfHLf^ybmFh!Q|
zH%z~k0Ka@oW#!T~7VF-v6BsT+_d@o+K44pqYDp{7vv8}@pnj(PTuUMes?&xt$oX9<(AM_>=h*+>j7bADAQ1+25bHhZkmYVntBH<3If&$sacYK>tU^EPBZG_
zSPXO9^OfrqH~Z+V28YYpd0wUFRs7YiE@ncGMbNxxgEEuf&M{1~s8kwGbYiIy#{KEY
zuxM^8*~DFUZ1>GxY$t9X>MrZXts=-8^p$5VKDD)7{sq6NGaqc`Xo6Vh6QmJ}g0TNE
z2l4!Qvy`Iz6rPP)CS;yg>#bdlsyDS%Q-$k7pK+uDy_qGFfkX^n+>;4^kD2v5ZFNTR
z+>nB^?B?Ix@BlT2wN?c|U$!NKo?at1*maH%1?x4E`9|vNz7G?ZI?8KKIO0(&`o7UY
z>o>a&LS7$e-|lTEc58|>5&eUs;&2st8sj*z1nVy~6nPAbQw4?RF4l3@=Lzv(0e)=3
z{FJBNL^6!c`lIeS9<6NpP*jy&=?led)lXqvVU1zMlJiK!A@XV6(TVxyrzAf@P<4WE
z?Iu_GPvMy&agXVko*#$meOCGKYIr{+Dz|xGOt7~!-$wL6q}uwPvhLnLMLcRgwyZzZ
zAE!cqMJ&iyj*}LC;}(rHs;PQ-KCpiQqXtgOxF#@%b!{TMX~Z`Bz)YC9j{fwb6BQiW
zCj{_+uiN15=|?ch_=0^}=E_Pk$=>(q)2qyBV>3K{z^I>*1{L(vh1on*j47=Xb>ZYL
zlZ3?R+J^U<2IW5#bM&nrpc`ITxEg$`Vo0pqQco{@AA|Z-X;y;@p59XDUhfo6;xS-M
zrK3u;Z>lP;BswbZ%MmdSI*frmlC9TQH8zYIsl_eMAQ5=nPqmrmHr4l>?8{69E$3%X
zTSMr2SKxLjrwqG(cRUYVM6bD|Mkg9zUWMZl1;Q=5kx%m?mn{^i=0Z{2oB=jy!BWlS
z-sldQJib%apVX2@1ot#)_%cg>nj_F%OR&jwAQSC9SXqc-VKx?TxW$`k7B$j=$0mFu
zaRj4Crw6lFV9Qe5qAGSt8BmqQP(x?B>`<&$)E97dAnU7lKLjY(to5$Q?^LHSxQth^Ye9wG(>2`SPj8OR1
zU0|y=8jiG`oQ!1NZ=Dp52!U>GLK~5)y*55FY|)vGXkwV^i*Le$i4m7S>9@R-JHQVP
z-`B!@_5mN>2HcA|%Qfg4m%hLG_3s{v21q#1iwifBgCjdfIjL5^s0F!?Ww;+tU~Au_
z+4^L|lS831s?~}ElkpQ7;_yHaeZT+F5!es$Y~17VehqWBcB%m|mpL|Nya8Lxu`PVQ
z(CslDzVlWn80bk<%naIvzI$Kme?0f--f|Zx>^Y_1zrsjjKj4mXn&llLIa9Z0+`G8*
zX=d9KzizwxVSE3N0+z@58EIWz-4lrKIQi-FWzhQxD&-bp_Sf4Kq9HYX*DQH|Yt;c7
zd=1d!b+0`AB*@wGdixfVum%3G>0)~fB%hx@^a*O}|e_Oh3Bcrhg
zCQ(7sdI&Efg-~n-_zv6H=516^xLG3BhK#U9496EuZ8f*H`V9xLNkd4;%7ya=IrJ~Vc*Jc<-ld$~C9*)CnpoW;BPzpJMW_gf%TD0cwfTzar48aF;l%0-WBycL#=_o{^q?-P}64
zKNm1~PN(VYXFC3LA*(^GzF4d?!4jm1W6F^eG&u8IVO0Od%AX@{of8t-DJa>RpYg+9
zW~dD-E&u`OL=wh{2p;F;lcli{d=spYd;9$`y6Nu!?;~EbsUZ6V7%@N54SvK{r=+F&
zp}2d>{|+_Zw0FS%4p0AqPvsIfbq|fokm^U#;6jW1-e4<0*|=!}xyf;>pf$??7#oZmS0@gc2Rfz
zJQF6sl~rC`WnaTt_Y!*?pkt_)64u;dfjKa$+};cnS^0gPw@=4E4@PEs^cD^FC~A;g
zjokyeHJNO#{4K-XieR@p0QpxeF?60lyU45@Cz37Qqn=C
zUcvxj6cBRzz^bt+#GVfno%u}^J`~#MTQOZ=Z+rXELW7dLHpSGWZw^(H5N|ANh{ZyO
zDER&|?(Gsk^O?Q3^FUm5wSkdY9Z`dcB|D12K4v!tz|8YGk#;f84-2~tvDGfd^Cish
z$_Mugj$KkQp_2aY4GHE(L&(7o$T^u+Yj5_r2S07!29r0xGr
z(T5h}2jQGL9}{OFdE<&-KcOTB;oZi#C;!Yf>GJ*(}|65HB3W
z*C1YB%7wAVRh!n-gwx>;fuxw?7#@WY8}2K8Rp8jjb#7BtgHBWvu3H+lzY@Pq|q!Qa?CbxNx~H1C}Ii-;h(0;WwY)MK2Kjd&vS3dGXHi^Nbs<*9jOln0K0Ob
z)HBVTV0{Oqd(?-xMZO*Z21Ly%)r|}Dns*k8Nai%TCtcCRHKHypf%QObb=8IZ(
z`Dd2Q)i(+}F&RvWWp=*+Egb!OY?i72$4$JYPmCf*3#`khXA(*8oba==Wg_*`T4!8QIk+%8>x(G-pEm+u
z)@~mz+DpuXMkSw$a-gNjrleVTXthCsf&))Q(?PkK0^(@sA0ch;%UEMj`v18&O;2`(
zM7iCcHD?Y}7%{A$ZK&+dZ`s&XtQ(nX?FkOk)1P5TBFh7?^tkL2KX20IO0>7#IIc}Y
zaL(!y0)jL3R;GLyqHZtmtWKe=Qlu+ZezrD#Prq+g)cDXhQ^)Z?|F8IzG`ZWnILR#~
zL2)JqUs|6pvJcWiFc~Dl3j2bGEt;YtlqmN`ETj=F@rE{1~G-Nq@#RwsPW4Qb2MI5`X@<;RijI){xGy$>OUsg_ovK#7at1
z6A%>+p~cilEmeIMQObMg-$gmaebjjtkyfA<%sPpj;Z+~RlPt1e;gXQ#c?j1xncsyPi;X|aaKHMyr56~#
z+`FiUf&q~+GTU!N5|(+lSY_F|p}#naelYLc#f`c*$7q({QM!Jzhrc7y{9AX
z#PeCwHs&9mQWPhnm0$AMQT_6PsZHz4$FuxbUYp^!6#Yj`0~btACt~$1Sqw{!Ds#an
zxa)h%q)hPx+Z#2by4SdyTVb39he@qw74<3!1>9xyU83`PLt;zEFgq-1d+S4Zk_lMe
z1q{AmQymP=7*i%2bv2eW9DJ5_r%x^FvZ*(3i4EYg%RDCiY1Uo@?tCYlaR@we}efKfGjBhf&S=d&>
z?TsKwbL^r2%6H-7Qe~hy&&pV6YGM{^l6L&*Tb&sz&p+?+EmzVN^kepDniuoD4g_ht
zPfB^`41COgZcx2?N+#{CS6%N~4z~4&Soi{p+r<*i#ql0FF+yUZoC@tQB9HaJEqr!Q
z-Zg!zV8rofBOCGUteT|bS->|-iqpH2K?|B^~$TMJ`CXp}~PhH=G(Rm?SaaW75sVP8nGW)hEW3vC`Ej4y#d
z5nRIzDdKpLdFd4_tnHM6O3T)SelY$DbzZoIEo&t~V6}KJLX&)sqeNB^eRvD)2+@dd
z)vXUKcywfLE3b46H(MY6wBHe+>o8=54BLKrNtrM<6_>;E@u<5CTfAjENwC5}HCTD$
zb{DG7e&PcwldR>SDfzO92Nv_6bdJq&P0odRX&4-*eecV0#R4#k@g*X~CY`dJh2e=U
zIe~Zop26YVCcPnG_xv#V5Q5Q)2(>ky45y6if9^vi&?S-H)$jYX)BdD4b!y^jBd*ET
zzCpPImv@&_dgnZ0%`5D>g6Co;*QfRC599sNDkF`v{KlSVo4SIag43fQ-18{y)prC?dr$xxl0DMo+R-3xPh>|A)J%bxk$>#^X+M&Z-qwp
zBJhbG>L!=bNl|x>ArRARhoM4{=bDwVVX9>k=R8~;B)zAG2!4LR2lJV{Zyr8
zs**xHNa$>cfz&O~bKD*T=278)^?~p5vid6=v99||@&L57CuJ#n_r@OBt}ZDqMqwSu
z5O!}?qPw@2+)83p7wHBcz8Q}OJi6k0e3kkAKQfnL1K8Frn&~=-sbLn$-xuD}C6q0M
zQ+59ijd{P_>It#qoQhKlIw~7^?AQuFLgq5=V$#ixNZ&?*d#q{*9+(lOb>BnxuMc9)
zDH*3XNra#+wOJDCwNK2#alQ48=FgHi0bjerBc?Dvx=QF0|0q)OCu30a|5nYhW$ZF
zuvs=Zh-faA=Hod_!0Co}h$5mF#MLWEWsV^DJjWEe`MOeYd646xzOYaaS>uLjm}WF0
zH0#fF0n+kXP$xG&!-h4dW#^?mKh
zQ2Tr_L>MC;OR3cUSqm(1*uurODT|_rBKyhm$Mrj>T=>5{b-un$+h0vw^U6dF**lg=
zQgqxhQ?XI;V|V~L-!ok_b?%m}H@B>xM9*@c>vKth3iPiK%9wt)eDKZ0zY7GVsZudQx-z&liuVSV#;qOFxMM0Ndou67Ke2z=}
zUsr%m1+%MovoC&@vW^Z=212rDGr6ySy^F=9f#}(qE+zj&&d$npk&tVEaQsVxmzrk3
zejes}Y(N=0Eqa0q3Pc0);+xJKS`N12V<6iZf~+E6_ln#POASXUos
zzxI~rtEO$3x6SH6q?I|hQg9@ndU36YlyKGKBUTU5YIYGq33-Q9fQm#ezSi12&Pv9~
zW0-q$zc#Lw)vxNffMLVey}Se$kPe5EGW`)A%E;e$(PGxQnO^+80xqy_#@)eKDuOkN
ztVu8yiehfor1w%#fG->%_pd%P1Mz$b?-4X}a;~{`+y>LG*sq1@KIH9R*xN8V`8BzC
zr@0}K1ho*=tlRO6W=DrQV>)cm`SiYuO6pf-bB>CZWe!peVNW|dfpo+W3zZ7Xm|zW%
z_xPFf#|vk12;vNGvHSNAgMO}OxLA`2P4gJWW;aARuP8d8DD6#&xOz#C&kQv>6lZ40
zFtztz?{V@8+>ey<-KNX%d95q>3rQj^hC@R`FmaDV?-K
zh}K5c$x`}Iw~(g^UkH1pm;TZx%sqlnlD^1&=|j$HZ63rc62n1~nG&W=vAe1&uXH$)
zwM{@7HeacyEr5rf&Xujl`oiOiIW7!5MeBB-M!V-9CgXG0@e|7EbUkVev>kqWbY8#l
zs7(IK(oWX)u{I)YD1ra1;0heBrJBJlU3z`;qQAAfa)y*|T<81mBnn1Oddn=3qU>!n
zlAcsN11(wp5=pH4Hg4zyK0A&LF~I^H$J8>D>_&j(9@qcK^^KpgR}L_}I>6c9dDJFC
z_e>WKX;Ga<9@o#&j!Wkro{zkfWY3uNcRfuQ8;#aDJiPU4JIamGeVARq6!(?Kw}L+c13Ks9lTWZ1@x&EhK5f
z9~0RCsr-BX_)t=y^%Z8$<`?=X?5AeRSev<1_v&m$BVPVcim?HA^;+`5U@PzGc4lQ2=2U$cGUT
z@3Ln8&C0gz;h)`@c^}XZd{;u)_-mj&J!90$%#oqsZ?v6ucD89eX0Eoq3{aP!_lg;Xfa>SM
z8+u5>EluUrx;diQQRud>{|AYGKRyX)*hZ#fBGyJh?&=W1>vhhwS%Av@ci*9OcneLD
zM2B@afW}PB|7sz$M1*SP;rQc`n!Ge|TH2TX!qeDQN;x|W`;k#1ECIQi15U!pigSeM
z-+z1PQQi*CJ`Gk#eb|%Qo6ejmpjTXo*!+z0SoCji!HpbM
zAT1!`Z=XuzM@e)%*e2XG__%uA$fU`*e%YCy^w30FoeEIqKU>?{JuPRL)3cb}Y1gh_
z@Ar5T)^&GwZTbv-6Abwmy=m2$6$Yboff@
zTCp=2!{re*sjX!#tsR>-sq4v_R)W%+OL}*o|G5Ynzf#*>CCCeIJ6d`&e{N;4QrL0W
z`FpNkO71T5Mqh^)(sy&P*#NttsVBTa`WLAIt^gxzC^#z}Jl?=ZmE;VV3;`crZn{DD
zTmx$Ukn6$Se|<`5R`S@3F!TD)%n+UWXDJ!1=xo35Z&556uR+xB>D>&zG?%$ZuV
z!df|^>?(>irxhpNudp|7$_&qE+=>|c{lmq7JK?%zTSG`7eE-8E
zYs9dljKlFW_x%?*FtHHz6e~kBwDq{bqMveFw<6-7N&dWtrt)Y7>*U$oVm1wk9=gP;
zqTVV07u#fAfqs0rn!D0pZw7#bdius0U6TO4<
zW=k*4@y%OOE=GVItOG1F+ElNp{}um%HLjCSr9sZQ}&5W-->W1+~t>Q>)oP3%-n}HI_bhW!pFH}XMDgo(ULX}W$LqUNO@WdSPl^7D+B^S)c@Ie!9KVD*e@o%`x<{zM|P6^vbwjKEA-
z?)oB+lK^3Q@H)?cSH5XZ(g3JZFnVVreEX}ZQA=j@t*N*TL+wloT@tmm-5)j&Z+ZRn
ztWJraL<>0MC8Tbi%D<8r0JFAmBnD?-I!e&MIt%J
zrrDMB^7?qyX{-K>9`Xv(AqUsp$4d(^@tvyQ?kgd^oJWN2`VC>^NDm}KW@z7jV0fy|
z&%eVuGm}7~%{z26S)=%~-Px1x%juw#Lur+ByAf2xzi0n6dYFVgHGCVb&tTx}Ky)$l
zZT#Q>dg8oMGM+!jLtD(F*zs_d2GwbO8JnFK^lB|(Eg$)c->
zVX2eWZ4L9U7qrV2-+RCj(~C|^&Jd~KsD=o&2%qSH@`1!ZuRaCry(s_m(oyRD`l>dk
zNMp+7_OJF2Mc|_u;mRf3&kv!}1wP^--sS^pZ_4u!F_H4-*wkX
zsR#_CpLzyTTE!Wx0r;;!-O!!`v5gKt1T?zPo1I??;dE6~s6hh1K6ScXw^v`V*#Z(=
z&(OJ+`PW{{hkE%Bxet`sg6_Xb^ghwwLx$|TMD*7s1vGM9yN%qvf`+dCB9!5HK>|X~
z9K%!G@Jx<_q@UdH`oC%nNCBR`Y=%D{+eO@_lj+?Sk`u4KbT^nMo7JA8XJKxNKluoY
zeWDznsx^$1l)9(#3(Egrm7Ja%gz3fO9yJ~&hbO#^OSW3C10EdG4sAZrZvM
z_GKmvhRI|KggM6
zcT!T)eRas4-m~7@Lda-%G&)0ESe3o&rzum*^EP!oT3B5g9^r$CWut&a51=KC;^j&p
zR6sRmH;YEa@I03IN~8pqRx53KhToUc!dfUHSNT?hI&Km9E|ItPZV4ZO3oZ9#
z3!2^ORa5l?TP=UWjLc9~i$1hx#M}^R3fNzc!}n`@k|YTWy6|Lw{Ami`K}thE3Cs5s
zrNzVbEiLU4?$uD-sbqhot}AbP#SBYJn&a>tipaaOTC
z0LM*eXEEc3Pck5Ah0pUk6Xhq1_%H?LWe8__T
zYGjozWW84VRK)AOPS7OZ6Vg((IMm`M)6sjgsOC&~##?w#PgoizwDlFHFD?&G9
zdj}kPDjocxSm>#(85RK2S1mP&ngA}&KUeSVjG}iK1Z2n|+F@P3dV$}r3-YfvUoJSs
zeY>5t;@>YIzRxLlBlD1HP!y)#G*H$m`9b%SH71R4k#Aln2@6gLEYJI$KUl-ejv)uC
zeGu0D@ZIT!+xtdbD@%@m5o0FwGk3@k0q%0C(-&U4SA^nZ%GY202X<|ja|6yVS)(a_#d+GjSTNjYbmGlT`Yvm39Tj+@C(OeWfOuyPY+Yyx~D
z8!+$)Qf{c4@~nq!>E(BY=
z&{vS7ulx|;?+i?zM>dRb%1O`gnfJ%FE$^<0qx5ExyYH?r7q2U4wK0-OH_>;9>*w?M
z_wbz=QD?cG28c;g6?!*c`a$j?+rJGC)vv5^qAml-bTpUwL;JR!UVk$dw7D&>Jyv{3
z;58$*zD}RprH-Sr5i4kWa%0N`YJDcRM{R4PI$!Xc9tp=nor3>+cl?|{m_a4t=$LiF
zvEhuaF-S^qqe%jc?e9#($G`T$`rmeB=3nR+na6UeN$ePD9B%f=qNw-}5hpjs1|^Ja
zNg3rB4|hlc1~1mh*+0Hl`+O{_h-}^|b&g5-C7D1h|22Qi5=zR1@%uhqblY#_>PVCa
z-MRI0`Be_U8t(a2?Ih~W57}SyT?l$LK@CWyhEADaE0V?-v9sh{
z$-I}0Bcu%ZZ%5K%=wcjvJqUbrqX$d;-0Jiv!(fc|#qDoy{SJl8naEgi1e3pw7F=yU
z^S|--zD>Bi%Yd*QgL6%V4SYCZW}GGFI1&0OSlf==I8>=vAXw(I^*aqrxy4;D@t-hi
zpkb4v97Nk$0JN}{Q0-F2gEf)jsgOFUkeXE~8rk1sa{I+a79v1~mQib8wOhqSGfQ-g
zc2YT13};Q0+F_3QA8K&~hrffa8ds)d)P8`=p5Z!fOlOFR-xiaMWFT{LOUY*|kdme`
zzRg}F#o#|W>(umzz)4sGJ!nM}hgLu1@^YBm!=riU3?kfpyXvqG3dkI?XoOM<&-M?@
zMKoIthR;Qyv$>1;nz%Op8eyIz2RF)r$x`gPszC+pYn>8zDicZcM3>yD)u7k87S4`8
z=do8|jYwgxK$RH-=n%$2p$H+Nj7ox2!jlzB;w%myoi1u@oga){bj)|_FuwTe)xYno
zx*)A8vfyv0S0z!}zntDA2HiP;EzdU1S|6~_a$k8XhTnwaSDtKjC@g-rJaC#dFu+}}
zz?U9kC*l2vVU%KXekJ8!gZbM`hqK@SlG=SNuUWe#Z-aShz1uKPcT*Nfsk9A%?6rG5
zT|79%{2@_cG`_7v|BVozKV<0UAerG$v_wQIi3XhFA+VN7I{li=8dWEy(a^Zb(6o6C
zX3_1e^?+io63ZcFl#q^5y1N_c?kF%K!8ll&pb}(qhU54}6%{ixlR1Z3_qWpD
zzP~W#N!~sz^g7#1N%Ke5SEtB~BS)9l>MN8lUfey@Fs?zIqa&!5%;E?c
z=MQ4+<`0k5LflHoSBaoFPUD*LuAB;94^?JszTEt~^Nqdqq%v!Mqy(E>9hW-?M>^r%C@O%Nmqx;(C8s
z-h-Jo@M7qg7yFGGDfZT1EOZ~;?8~!-CC7lLN#DtztGi@5k=(+4rVWlu;p7fFvYsND
z@jZD^Sf$}YP5V|YTc%A=Gk-ONeGmG0d6Fypci-iQ@`IzUiIkJ1m)1
z{iYr)hhUD&lW%AW%~!Chi6tL3>NJzJk_m;U7@mY)zD3lWjZP2kfm}sl#6}zDJW*ED
z26s<@lN-m;YU4;ogPS4Z=R(O5(q`|#@rQ^X?A(1;kHvAo?^6;aADf(-pJB@@fJo`L
zul3Z;6=WXk;jyK;$X~TL`1TAnFEHM$n}izN;s(b;1YYCF+fH*9m1nM4`Fr=!UVVbn
zy}Ok{IJCYPNN;?2j0_xKq=t-C*FQKk?Oxi-om4p=SeYF?-e?+%ykgRkrYMt1$#>kk
z9SuJ1qE_sI|G#k|z>q`kT4HM_Bdl&7ltwelD;9M$WH61_%+L1q_qc}9-W2jo+5(Zn
zqNwl<`Ip>9kr?+^&vlsKVG7p{pA5IG3enrM7s9$S&|@!&>6y}2mW2ds%)gPI5X)Tm
z%$nmH9M|E?uhNtsws}Hdqw3xm^!H_Jd%R+&=~7NRVGk8;S~fP%c_-M+)A~0ub-EUP
zC3b6zX4MPHnl0on4q2&zIb~{3>kF(ty5r=5QUoU$&w}oN?crRtf9%uY(>>xkSmc^p
zt#GI=v(+i{*N0dhz6`S@_p<7rU=}BnvFV_S
z#M`acJ$UA$kWE;r#k1`_VAlw|d2NDTC*K4-k85;T!m4FV2$)v|RrvI6D6&8Jxu~5?
zS#~4crF6N#R7g7Z7I0)-gk!LM7Oh-kp1(g+yE8p$Ur@s*GY%1~KSz@TObt#iVf>z}
zfhY*w&r=a%rfOy!3>>NvbZYqZr6wuK?F3-mV!FW$8nz|{EnV?N$*Srou|nx+F&mB#
zz>t;Cf~X(r8vc8U-RyS8lSvVn<9$n^W&w&V0tGcbe#Q?jP3+$!q#dFg=~rnsdU~B&
z5jf~`CsY~9r+1I%sr-~)zxDKc>fUg3uLox|7Vki}K$Jy~_^UmoWk2b%!(;MDM%Shl
za`5o1sVoCUo}^*;t!9v;YCg`C4tbuWnl@eA92bg!M+NgRbpfatPg-H*@mTf*oI4zZ
zcg+;qNHEjjG@3k7;@UIwd(SJuMy{__1lEMhJpE=igjSsa-}F0us{Z3Ky~;HZw%EM6
zh-1&H(E1~ih!G|IIV2I)9>dbGrgO3Nv8Kg}QGXS%Fl1(tsewJ8MZ%I=4VxeX-ZoA#
zZ3Hsuoawc))Nyc7h2glnRvoK7lX-0{G6Z~=UqJMwThnD?5V5b*XjGb&3U_dqs%`DF
z@Mi8*t>l>(d^mx(Kh+1W521+wO}^xhYduqp1WM+XrK^674tGXM^bU~hi#4rpcoxlf
z#bvg3@V~cs60dEn`M_@mVH+_ye$E(L*4_-Sq|x2u
zFH$Hw0#A?Er~KCc+66f?D;DF1VM{S|+G;q%9dHB~)KZDYw>oO+#vpkN6>L2~J>FAO}0-3>jsWhepBn%?3>ju)x5*O8`6#@-F|q`+n4
zRyFy{P85)r1SknWC49s!D!;IdxeqF92#SUjUPkEUB#z)nt5@y?k+0aIaq!?aS=^U!7&nO!W2+
zb1>a>)Rxq+SxU{&Fq{AD%~Q0Y1`$-o|4bb0FMfYXIk{cY=saJ?vZe>`Eyf&d&4@*s
z#F{n^U9X?g#b>Y7tm1!@VhmiDQys5jnZ81|Z}VRLNkjH4*iX;tNa4Lh4gBmNq|qd%
z$QGk|MQC#^04}`tu=E-}GNbJyti`NsrM-}pJYqHARPS*I#8f!DC|*$?yS%ZtOXs)N
zx1QZe9~)!m6TU;Gzk2$%^H=*i7SZV!gqh=Vzlg98r(9?g*$>}nJu(q*THf!ZeC2Bs
zcwnjIRinb|WArk4&5-N}`E2utedLBW8?ADuYI%v2dEyNI+T%f0fHv6UYAhFgDLP3T^wIMH
zAA6i`Wk5oL*9hVLTu5Pg8-(YQx8v!B(zD1YtNH
zR?n^cEA$S+GwyT8i8%?X$VYNQF_r)Ik!$!O&LD=(n_&LY)XVV5&BN!p_WIbO%wD8&
zw)w0VZaVMN7Yg=i)G#v>^YCp(95@5$oEl_>;{e^f&{2+ha
zEOSB%7VbU(<2w1}5^;V1k%2CAt)o}p8HlNss4O_}=$PxVUp9G8Prmr9mJD#NYEpRC
z{DPhR@AUl;*j*#qUAOr83}#ILBjln>HYfEPoxwbplA%Tlilv$=o7*mu?k|hffp!7j
zK7$#KRG+J2A}#m|9#h9;qxY=A4xjmXvUF+rJBQ(q(7w$9N0H0B)0xO@aO2O6Q6qw8hCtbt-p$L4WtlHXYEx3h+mg{vq4IrX
z-|Gi{l?pZ~@Vk0lc`|zd!yv4%=-_FgtDFM2&;R;7PUF8!cP)
zPC~tl@eh9om!59g&G^I-1mgxva}?*8G8Zt$T2V+9stujC)5dCNj((|vxK(+eb4$sq
z34G5bh*sy5|4EcIorZF-(!?;#vtWs+8SfnW7fY7D
zlLuc#yM;DM`hfaYpQ-%!OQi)F#@MP+^!i)VHBDh!!2;6NCyNc4NgF)+CL0L9ble+El&vW}V
zB`tr@>u(2>E~L;#?%E{ORA4+I+1YQL15Y9lBCU3mOIMN(F;X%?P=Mr5G+&jdiwHIN
z>r_<>xp?WwI^cKKdNYq<-n2c|_HNgo$yds}UTe9^(03yx;8Bf@Q2k!l+7-`HxSPwClP*y{V>85lAEUPR`I)su|#_PT6Emy?sm
zg5G&cYiNJg?9OM$q!$w9@n;XGJIKT_=7dZ=T)kM2X*g?Qoj7Lg3D`S6L2^N-+_@AMT)
z12QUAUiXJ#V{C)Iza9m#WzpY%JlDF-2+0s((KMgb1NtyqV6;I$#DV8cuk-Sj!c{8m
z?c9I{7*j?;DSeqPi(S{yIBAD&(MidWkf8|HYY;&0l3mmo^_%2u1@CM{qwWG2!sy9X
zZiEO1os-y%X`-f$H7EjnPngE`zEPaG5eFKlA2WljlcJ(y$x?f%Y$mIw`8}#~AiT10eXhaQxpB}WMPLy(`F5kP)wzk+4|yAVa*|)bO$_TWmFnouM{Mij
zfLdhD(>1;hj(qy4Us{3mc*H)EptQaww&=P`UeB+8`R?jC6S;ks2oR?-trh9@D5Cub
z(R^Zjf7AO;G*Om*3Ii2q$b^+0UqMxH5empx#zYen|LfWhF2$&XC+IfpyYx+zL7e&W
z#aqO)K8Wkf)Au&7{|tju`4KmH(QeyWf8Z#tMq0G5cm_oO;Njv*sXo_oP}wC3znWk2
z^HrHK^nNk%I&V@p8E?wlY$>gI!I$N4yw_t&Z5T6
zmEW2^B2Ue}@`|2Z=jVbJvGzjQH|h@Q&+MaIu{{9OBS)HWS?J*^lO^C=KduaO1Dm0Mg8^Tr#~S}L@GJLvi`0&>
zoqhJ^_L~VE=Pn7gl^0agq>@b9B4*#UBx!+6$n#JdzZDK_4Uh?!Snh^su9=sr(4q!hBe7A3~LHgI!l)
zrXjl_nOaHXSPC@4Y5Hqt*};w5yT6K?Q`&SXU>${AoIS|Jhyp)9OSPTpLT!@mNdnX%F6hH4*r4qt~~4%UA3ak&>)yy#B{%4CP8K{Jc1af#}5e1vl?_;)QIRzD(x((KfI+^
zpb8)X>sdT0VL7mjaQLb6tk1jmj!ake_e4K9>^`r6;uV81+g9T0SQw*X3byd9>QVF<
z+ht6^EpsV2IQa}#wUd3W^85;m5a76EDFzQ+fjc@sl2sj`N?}V%1~Z=~3QWM2NxAgmxrAVPmqfu*Jf8%g
zTFzqL)KORYi>n^zS}jTb4^!#1(-sN&r5Nrryxj&B$93Iqc_m{K;jtaEhr89jZ5qt)
zmCRD=({#zyG?hHR{o$baJ0RLMD&sSS6pZ!qf&)*EApHJo131&yDb2)x!jJ8HPue+S>?rAv-`G
zX`A>f-nn87iyO0NaSK%HQ^LPTop`@>fIn+yHowC0DNLElUUhLGX3-4dr|?Zg?2ej@
zy7Vjr#9ggs^qy>7L&+hwcS1&#%FL}7$+gG7DFS(|nh0#3P<<;3$mC6Wum9`p?DeFf
z5wKDH`LTU<5x3i!w4Fm_Fh&TyO?n=xcoN_+Y1X<3y#j@uxMd~kU*^CT?(Ys>Zi_9V
z7+-~xY)uNEIj)2g7(D1Fm*aRY8PSEG;3LUzSjaQIr-
z!+PQC3W$Wt_(C?3lN(_YKmNxxkTuU=C?_u*;HtQZ@NwMo(q2D5OG09YlBz^11WI=ffJ(-E5?Dw&#BdAMGw7AUo2@E
zi+n8P^eC*#Z8efaJ)+z{z{1kHFly|;cpoWm)c3;oXY;mbCr}Yf=ADfjEbKCUaolxR
zjf;vA$^xa9=j9XHu}Am?kKLx{JZ#HY=!P-C(WYKcW)dpQaLy`87;1jjjN}SSnlWGX
z+rsR8IC4IXb#CvyStND`*j6ZGaH@c7_uqO7nkmZdh6EOiK|8<-&`gYszWQBw#1Gon
z2~hAKt3i)?peLqi2oaYZ;jZ@jnsB;`Ok~d)Z1?Bc^!qFL(9-Tz`{F3)NOmNE;%sv-
zE@T<&<3|FCCTim7qPZ>pO=o7Rci(mN$EVYi(k`2F`yva$?zAhi5-Y#Qu(F+>;dFjw
z8#U(^B!hdT(->YL+KfiVv_1K1!
zb_g+|rTz7hC^9S_
zW+nhv9ITd>66xeBEFsjZBiU$iz#pqBUBD0Pnsl8z2eW3LVu97(AG1rg1U3#Jz|pM}
z3vz5JG)oaYP<(SG?3>rzU=-{~u<9%>TZM-S&8C03urzujx!g5l(eg|QqXJZ$kdWr`L8jNFR+%$4#aT;SDO{Ph31Hu%g
zvNAg{#`o$%#AI~esXFo&s)nlA8MSz10Th>-7TrWn`{2LfDSq_^}HygdR4%Z!alq5O*Ysq*Y>X6equnckMn3@UFsmWLHgzkLvWOt
zl?;8~fGR$*{LcthqK4Bo+h&l8Bu*Jh^!3PS{(MUS0)<}s(?>9hIA^E;tIOj)dostgomvtz!$db`=Xf^7&qCk5(w2=s_m!
zlKKMP8@^(7bX7HDHqwZQTs+*ChUXc*m*yVq)X1Jse+R$*Ey1RP`0}_Gu8`qKCCf=@
z@+3F9qA~GG^po??;?JVuyZUxq=q68L=3r~9jCc=E3DPk$50?`suZd86uynn_G06sk
zxFVb2aQL#2fVfQ5K>HuG+lt;s<7Qoi7h(&2HSp`u*oT9ckA_2)?qhQ*sZD)N##J>Y
z63}2)%B(JF~CXZyHd99yZ!X_L&>)faU1uX
zrY&B<7rR)gse}$Z^PEZ-LoerY-W(YvJ~NkB^p|M70i=mxH)m}8{zvm!-&41b=W4pn
zc*z#?*!BW1?_wS@XHhQ3RBmhDau%qeVu8a@UC|(}*Kx7rMZPx8sZ$-$T}U&8pH2I^
zbfSrbT-3r*KzQ@$EYE*@@22K(izN<8kYdS!i86xD)k++d)_Y4LAx-M1d@skodmbp>
zX1p(P5q}X5I=4h@krr~ZSk4nTT&*D9$Cv1{dxbq_`I8pPW{cv0Ix3nzM_YFNVQ!R2t@yTGw9HhM-Jkr+jj8O6&Yw);H%{{`pK7sQs
zm?)!Z(+@0-v)0GZx8(}o-4dlt&4;Tv-`6yFs985P3`HKT<*0Br;4C}4j;!!*+IaVk
z*{zU&*t`1f2P3B>nIwl&d%QJHog86*3MU%u^4vq*dw>W1%I<|;(p|^IsnE9D
zkg=yq+`;x+L7v0g5qIdmjh~ZkTIN_ZA7A4
zRoX@tiD%NXq$((@!~ckrDfvJ-c=#Fhkz0hTktpYa>l
z>5zdkCL;2$e{O2%eVS3`$fErh%{6N(V)}$T=RyFl{M-wuC1>(CZ}@jNZ2NkX(&|G$
z#AP=3{}$a|PH=E$eFe5|?momZVL0Y^40&Te_1twkAj1O-$5mPVr>I)*j`Zn9}_F`T9
zs6a|!+rQ&64v8q+F9ln}BI9Cdi9VU92FBL;{DE9ERU9=n``&iBeK~(&HFPJrbg@=E
z1rXtLxgEW;))#Y=-%lTPv!m8+DnT<4F15D`PSC&swkHxz|Cmx#u&1)$(H<-!ia$Z!
z@aMam6o-1Vrm;Ur49#uH0dKs|^5)5QQgq-*Fhc*^c&(b|1KqM|c
z$EW)&6$T|GvAy=Q`1Yh{0)j}ZO4?yb+Q_LTs%Q9q-c2uWDTcIpbl{xI6SRC2kQFzz
zQ_ui|E;Q1CpH4QTC^c?k5Bwv=-*
zg6EFtA=AaHkED8}C!(!-bGNe9$tIoL-B68|&-L)-sB0pw42XZ)2JHIaF5(INd2R0=
z@={0Q)p-qa!_)|F*4NV|^6R^QipE^XL2at3jsNa0zxaN%7eN08Pb#qwzzd|)j?RP_
zrqsiB-dTznrkX%Xi>aJGer+g%mLy+B$Xn?|hA9}MbAKHZGCF`k=wd@F(asLH>WT?P
z?29Jlab!sFNXrMzYH0T_q1M4Y*izErp#WCxLC+tg!fRA$;OLQTc_e8UFk|YBYd3=j
zz%K9L3adUxSXblm`cp{exEXe_Lj0hbRV%-f>!2%2Lznv1mK*UEvx^R!7aKg^RF*Dn
zdwGsbWfBOKvnI?$D5r=qFgaxW1N^sizA1TQD6=|aA*F;
zhJ%wloukpZ4)^(wya>QR%2h_V%k{~m3HHpAo>#AJAWH_KSHbP#X&lQHqH-c(?dokL
zu_&ftMzh?`8G%j;W@Hm1iruHnpnz5HKJwK&&Kur$yAbQ1jwKhz{{pc74iW{{Op9qA
zFi>{Nz8F
zuI@`edcfeE$4G6ue2BgA@eSJ7pbfDbC&u745yE!*!vJH!(HzX4C
zAAT6arm{Kj(|aG|SomOcv3VVCDsq!~S55psz9r%^lacum10?NuZP^N61+r?p%mv~}
z&EwHOu2b>7>98ZVDULRmeNX!8YoGUr(@Z*8i_7V0Q%H;G@{2r`5jAg^;H&Yis#XjIrRC)tn>?=$mjOEHF*Vh6z?=T3#2Il~n{ew3)RTC0Og1FD<^&`@7z;`Vkc7G;rTm
z|BdT7mk^ve2y)FH4~1I&b;#{?tD-+>tDW=~hsx7{`qV-iUEJOK!zqP!hwAOiTdjl@
z()ttdbQ&LeLZ0u7-F*p!(A^14%o6Z_Z@?4DfT9;SE%rPDv~d46Q=2KVgZ?o+>*{<}
zY2G+|XNrCs%VuAec3)esrzMvx0T86tWF+xRTXAzs{&E$Ed$0~eMdk<`f$W|L{9RxS
zxM%1o0a~A2BJSX)@?k3V=~;J}Q6cM=HPH;XTW{tK@U^sD^}!~Z_nd=(D2ctH`ZH|A
z=_61Yu0*G%V*nW--aN*~^R&0I-30T*Ur%&C@IPJTwD&iywfC|_dLwR9oIAXGo=$?}
zuj9M?#$~jLr3?X;K!f|L1)xg^xe|6#oO#2jmltU}sYg=}BuGda9q;Z4u?Mt7LnF)x
zg#M#Y=T6-|OB-IET);3_s08chC77wKNcgVkRWjO^V`{3Y!c)_3{VOXCh`z55nzC~>
zt(2Ik5j6;~l;bZ%<15}};zrkfsENHL_HNdCQQ7c(>QQg+brfO5U0WwSJu8+nB?S#Tx!)s5M{k*&{znWJ_MItXtlhPQbX>G-q-sIM^6tEa}
zs5!|hDp!6=<`)|*nYELXh^ZatRWHB@4JvT*+wbvWPww#V(9W;gysYG<21hhtKvzCo
z3#1ZVveLHgq>fI@>PC&yA`&5Kz;IPi!F2=Wq;iv*#11`
zDD<~$;jb|+Pi3q7RT%t!L*VLWAa$Hqf57hp+C-DM9==H&!N}rb=GGQ)zedqc__4!h
znV>OWRNrB-_Y;})+c#QM97v7{&JI>
z9j;Q0r{nwOqpM5Dnn*M7S4i5G&B}www{^oXNLD>aI{)Cs&Ed;~qfe%H7}y&mRVO{+
zOMcOz#bWmTL#LorN+`SF$3#jYv)tE?Irp?CGsqm$|XrcAlVGK)_`;$nnMSO(kbZJJff
zd-Qcb%LCJmpi1Wau;g5)eElLQ{%ZX&ayA>?qVgz&YtqNC@S^qIwuCP`1?#iLLGmv%
zt>=Xk(U`hgEC25)Y#$_SV(hC)yejKRaG$m_qZ~?UC4@b<$r-Ri1CZ>d`GKR^zaWWH>Tb@~eJXzl5wH~3=oZ+1kp4vY8oZAPD7
zb07i>_=?lya-V7DkeAI*0}}3L!t_7~SkDw(1QhN;U322_)Vwl#k2XxpUrCnez*9w_
z03Xwp>=1lT6^BPn4p4j<2n6aJ#;C{mM5fyp3k_1!>3O&aGtvRi3NmzeL$2>ey*@%n
zZUN-n=1KxfaP3Hi*MvQo5jEbPXnR+fb?OK4jv14+!4${`O@
zCd8@XpiR*aEx|C`%#w;1cxHPWO{5Jm-Q1X~cexS!w_zZYSd&;@nHt|Di2d832BY+>
zT&C$LhH0G&KxI%-VZZA8m?&M9rY2p`MkdvJ5!jB+3>5$
z=h_1uy&G(sR}U9f3^b~$Y@*0mN$fG^3FE!a9^-FLQrF;%;a$0Z=F^sfVr#hb%QH>%
z8Dr-*njimSkl3PIxdcb_jiW=3LME%$c)$#!I-+{rR`$`Cj&M}bw&FBm^x!+H!U`evXIsuZz+R}!8iVIJqz^R8nFGI-
z;$I20n3149*BB^HL(hZ6<*$g`Ml0?N0qn^H>1cpFW1H^(z9pvmrCOY$h5*z4qzCcb
zq)EVWKR=Dq40z&h@OhS!Hh0lGwRa!|jk?8{6*iiYPs927WlV)e36dyf7N2jEf!Jk}
z(FnLS7yQhFLGk#@rv>c2m?N#Xv%Wa^=+jKiD;>gYh-VGeFGv@hJKj@9vH0FuQ*Ka5
z=~NHsCA;2HMtk@85!5g3&c?pJgU5U{&
zTe98Nc_{Um@^o%qYz)m{hy&Zo_F(o|M)h)7U6kXqQz^-ogvz0?pbU97p;C-6!Fg
z0bw69)^bSU7f_)l*qPAyAOqrIShLmMSA3YNN(``RdG+=>IYryE$2TntoEtQ2AB46+
z0F*_H{W}Tv>5EHNUo=jP0h`;at*g)BJH(O7`o5Bke7v0@BXl_;*6a;vSnKd#8wQo1
zyR6p-HM++XOz-}}l=OLmxnUFbk9tknEk{KSp(lvej5&)2W5M*^JFp#0ijF$?{`%vk
zrMsmYMT%qKLPpaL%v&uox0SB^=#uANu$^ZD4WIK^!lQw2dXaUyi`jPO9MSs?W^98L
ztTA5VhaBDW=(D6vXbjiUMs9HnbaHfpePD%%!j<>xZ<9Y%f-jZn%PGT%$;7#qwn#6p
zjNY!!i#abgNZD=iz8%o>{G(vv94}PW^_A&%)@{yN@>nk?@i1K)17Uu5rxMCk=r3(np~#-voCwm*Gliz
zPPm_MJ8qA^#tVQ=HtswFUG{JEVL6|hN8b&)o6=XR@|!_;H?Pk}aU7NAgcqU&@xkBv
zUPEIVXWfHAW7#rkfsUKjImYIS2YgfBs-wu59YC7|p#lio(F%;4v~+Pa$xtfd^6#Ov
zncO|DOf1!*B^hBOj$=wtmyh$yBGZ|yoX9k-LwaRJ;i#?*nxGb2h7Xvs1sHOp1oX2}
z?;>={m+FQ?N&qcOP(@bsods5PP7lz@dc+yUWxw@nyv2yDlf%W4O6QIHHK7iLkpsCA
zVB=xr%n+40X4N}4hmTDuygi;JKt@T4_}9)qT?ZlqcE=R`eqM>lZp-feax-myF)%Wd
z9GXlX^qo(
z&WDFcb4Jbqwya2!;{gEzSGn|AfKXpMtTD{rlRs(pwrsMsdx6A!ce~#{d5*7blNGwi
z)8hLto$jC0e@NJ;hVonqytr$;VRbiX0DfehKM?uAyb4Tgl1#}kQchdZpe%+ODs20Xd6)JoS_v3mT5
z)SIXT*YnSd%@GmJr
zUfb&-8Pdm%Qw^vJdyQtOy0}`U*W6LJ1jaY`q-YIlm3=luf#8V*WMLO08
zRk$l5n3^*`JT~1pivLDQ-N87)zgIZBug|2y`q48`#D*}y@K;f?lo}bVQpn8mmOfSL
z#9RjqcQHdlb!`=3>Vhn)&TH$yctP>6YS`bNb5-@foyk@0_)~-@5#2Qt<#2yn@iW)w
zyD~>7X)+!M#Q)+usdPmu=x#o{1+dT%q#)9dhG
zGl^H+HN6gII|(}Muqh+;csk|Z_$lva0$$r;Cem^ur|$sn5_%>JeC=LJkS!XF5np^TpVBW_7ks~4jw(&=z_rZ6DyV4uW
zxM!0FZ8?Yz6G(N_IF>D`=o(gGMn>DqUO_#=rA4EMXW}cLa68*N4L0tQI<5$5xgges
zn26q&M-Y)o^R#{mgtHeor91Jb
zCk&^9?cPgVzpvgTORuH}Vu|u)-LA&TvxT|}85F3&woARK4g%G8;QJ2uM9P@HkNfT`
zMx9NTsJ?e-sHv01XI=iJe0$_Rs!qUjy%UQ&ghmt_A;Zn&toB
zp_0at2UNQ2{?z{5aqtk=LWH-(Ht_DYA0_NbPk2|=CuVcvhh3$zd?g{A38Q>VE>mb2
zKZ%%rKRrKk&P3kAHaf^V%4+MIuk$BQAbA_}IMgdkf#dA+OYwqBq#w3yHDH8#)t;CpQm(;IHtYC0B1yI5PM^UWp$@
zr_;V>Bv|rU>VEO}9|MBo9Y${dg5oP}gTR$2Z36CV7QLfbpJ_WD|9^7gC++J~?n4(<
z#g!PC%Djcbrb)?FsMss71Hi=xeCAAXXN;>1qzy&dFgMIOL
zQ%b=%%+Ul)!C_6=RNo8=XcrHBLG)Q&B2KzVA+8~O3a+KtpYCS{Uqae
zAB{jBZW$*!YwJ{~`m92&nYElXYI$o@u4p|(4NoH9161v$UM>ffX=fof&s*X9>y1!o
zb()ildYmIuI>Y;xCxYsyjHjT2YX6Cq;q2}ydn)+Cpvd<*yyg~ZkJl^&?XI2cu_EcG
zjq}Ram#NvOPY$OtEZSj2n
z-DXK9(^vaJi8_%MNX`Mu=6r^-iZ^o(5lWHV#Cnu?Wq3>kvmd&>Hh6;@|MgmibcR)vRc?KjaVEAQ#dF+7)k-4p229+GAXPk9A
zvglI)Vx*+~VF_gHyOr#H#?ESsJHMwG`8+NjFC`iUSXZ+sEmUf6drKPUQ);80XurLb&e*L6BkWUHV-;3X6Mg=A!EDsg0l83JSVJYLQNg&lKFxfWr35ovclX#Tps-t+xxCxHVNezJM1
z9qD;^+A^|Hat`(TBX~XU_21UW3A-bm;{T!uzUJEcG5Gx?yX$xe1niQyFi$)1&b~3@
z;U?VUBj&n|$zFYj9mEx`B_*_Sf@u_dLG>@Qp9E{Y6?cPDBrP52-w^-U3t7d}cIl$m
zxq3EeBsSoUc%_&92Y-8BBmGv!CsCaUd=&%wpm1Sn-yyBLxW!A38`l+Oa>v~sK}%>P
z@`v~?QzswI+YIl6pFA0LV3_3Mk6F7SdUvdW+GhoyPlA6y({Ju0%$o8>J;x<}fwICy
z;m>O5csW}e<46z-3^3Hy>X0QA{uM)5L;2(Dymjlx?4#a9
z4ZIFtG1)KtTpQ-A&`%x5r_U*<%Vl7Nsp7;^vFswOy{v1aY$aYU(Nou8?s|vC@-;iU
zA9bsPaHX-%U`~LU#jbg=dyCWy66TJj*BwmbgpFQZ_a=uj`Zd|E2=~h{PLrwKO>-X!gsFBD2uX3-IXGXVT9D
z(%@o{X&Y^6t?9(+yf~u6$>^+yhjKcUL+c_NAEl<;mol4#)sd{&oI`;s1=T)F@egm;
zB7&tO%i^Gxi}U!VoBCOO&qrb#mZkvnmVp}s?Uo&TF;Ch9-;yWUnzX^ptjv>(rsK1f
zrcdK>p1+~qcNdrOytN&rucBR76Jtp&Jlir$RM=aZ+w;pw^BMB1NMpEUOB8vRtrb<-
z<}>{T>|mc=p2y)@(*qbBZi3ji_B)SOpQ0`>feMkx=?LZHQ?vJ@kpNnsPQ_m3Cs(7c
zm{W=!2YNpO_I=F#zZwP18Uo5UjHMT6%bZI3;Z5^n=vxGHoxP}2I8#G>NaUS=+KvD9
zl5x)s><`*gg#Xjiw-S-TU3-hR(DE?dVV*VK4Fykv`we~gy~@*h^3y5(#LZtR2{u%7
zI~JU2pFbX!b(~7EL>vjO)o|m*<*SU4M*ROGg=yxnht7;^V2>|!FF)m?!c0KlOMa)$
z7h#*KCUXxgy;O)qo?&RQ;NS;!zXeQn0^|9aqJYvflXq$1m6QZ6K6QW=IR)0b)I^q?
z@x*1-TfM+OPqVsGi9*U}%%eAchu7CWzU_LP%LVbrQZZIOszK6C$fl$0)trK+a68OH_v
z{0`OoPZ$i;l4^Zl%F@Pgt&qgPANvYhK_%0E`EM;c+r&=_ZXcEZ9D#mjn>$Dvljg;`
zmev$JV`+A{E0Ioozn0Rjy-_>q({Sujo)<9o|1ox!4^c*Iv{wNorKOSXj-f%iJEXh2
zJ5;*6d+3(V0qK$$Vu&H6ySwk4d%xdv_NVtR*zbOx^;>JjlF$fb5QqD)4*^q6+hrc!
z8t;szlBjgDXm8auDJNNIF%vOI8#$^ftNeTsKYsZ8&jO(_8gJyaB>KDY&oa!V+yhMG
z(#|qJ87~h!S<+(;FCH%*VALmWZgozc9+zNqEs3;1YTShh1wKAsjVpg&nxc>@A1On~
zV?^C%hZ{1Sk7`m43cyE}P+Eq*+`TT(D`egQK~m@P`Z8=i@F`{(&uy%-;ZfSkfJY#^
z`D&$l3p534@}%DR1e$#vL>0{P43IY+edXE!Kg1ZplDzYSEJ1?9Hwa`-B<_(o)Svv4
zKuZC;SCn}|L4Bryk~+JqH%5p@XoTW8g9^0RG}w|ez!pHOz!*UrWU4?c@2N-%^b;)1w~%ANKRLl_PJYhp2MMQ
z42$&ar&Q7QomOgj%z8aOdXn~Q0`6Hl1@;(0FJX20Z7-jUSsnK+jp{bw4z3%|=Icc)
z{oBpE;+jvGv)ndHpBL{s8XFy2%vNvaSgY1Q8~*)y06rb`bt}GjUU}-Ak9Eu;oW>7|
zK$F5h7N>0l1$wTRK%I=In`1l|p#NCNFWebs<@KY|nxIXYv&u(+nLxpwYLCr6d6lN9
z9|5+hT{XQXF*H#Wx0cv>!abE!ySl9ZSaQz!r{=Ih0t*g7$QvxBoVkHs1KWkoQ=p?D
zJJ2_;{1btVP;R};y2%#K*O>hSiM0?P=1<@=whtG4WdHy3FCItyqku6|Q0i?=HL9LT
zmqu4m=SSfo&Tp=yW4>TQJNlxE*ipTrf$Y_3UxyCqxPPz%Mo6Ef&2oh{{;i6$x5)7%JD9DQYugMEedj
z`x}|)r)P|Pe5AYGWNf^by~$#pNiuC%hEUOH%#CfYW$JrEYUU?M1OOA<8BoaWfBf2F
zaUk4-a*iY_nW(ObD_IYl1K9Ka;w#e>%Pxm7!l2kvGDZeSyNQ8y~S*jT{iBH*8h1whgCT6&+u8kE8ro|Rpo^J!T5VR$`obV}qcp)Zkzgd3c5i0_Uk?x~
zN{Jyyo(g0@`SM*^TVI5H3WXx|R}9f<8+B%Lhu&&}2Z4eE)%pUC5kfPbmBU+IbM4i4
z4-tImr)71)Q0vE&67pij&aXUj|BlTvh$)nWe>lB;XQwyk{-Ar?{xe4gKsLA7F9^1~rfM337?Hen9db&A@5`6zJ}zd;2o=|;
zZ>nFW=%P{gG_H5L1pO)&1_-7{9@Q{ni#aeqM*blSl|njtS~6q+PD?>J)C-*FB;|2j
zr~+c);jkPQJ|A}!D0Yg^s2Dk@_Cp&8-f~+HeuvPj4&U6Oti8jUo+~rB%YBt%jE^at
z!KGQiH&9LW^e=@~6T&hc@k^N4tC-{=N4QlxNlbcbIH5OGGTFB$oo(bzq3hJ#{3mSZ
zHAVsK2~xDrB!fv%brfF>DRp
zT);2?Jeh4@LZ3phYN@+k4_ktx&zjl{Pl$B(OHRlnKnr&%&y#JJ?ryAJX0}3VNK!U`
z&2ti{Fnuyxx9RZKn=V_c85(ZF1{tgXN|6)~asyUnoSGJcjnRBP503=vBAu2X`h-3#
z;7f=5f-k2`f$O6tQPVsZzZI&$*c@#v%7RMEpH*jpS?l2i*#G0@ilFCtKVwRScN@GE
zll%gPDlWTj>ByyU9iP!}-e=r^+0)Y(^bLec8BR)Y8KMZ{OJjyHBN^5!+ldFKER$i;&
zQ37+lJY$24j7oM-4aY
zrGizE{W}nYt+Gb|S=DU%mRVa!#IXpC8W{2_QOmNw4@Mk5kyqnc(e|XbZ)}!sUc~Mm
zto&v$PWj{a>3kpY^sIklz!zA{g@dcH8=eR{#X={iDw4pGPIKlWjFrKYZWZ`|0m
zR7$h>>$IzHVBiuMIC*iw-nbM)cbN4f4tunJD$6bxp5M-gn5s|kPdl}S7DsiGGUE26ygE})KiCTbAf?>@}
z7i+3cGf+tt>nde4s*}Pl%Sa>Jd&6MSu4O+~PIK`S13^{8SRn+0bTk3?jVzS9QoE1y
zhs4MTRvy}ZrFo-Hz3$rYoGG0;j~rl5Tey~j6cSEo$O(9>D(p;6c!=1T=Dt-Q*UP?uhc#Fo9eIeZIgyGj_x#*^>|re_+0T{!gIwF^yQ#uY6LstY!@T2T$~auJit;FA
z2juE9aYFAm&Eia0){}e2&DELl?y1DlBrDop;^lMxB4rlWqs^IZkHNYCRqZ2v98P2X&107?9!~1%_9Eo3
zpVKHhf$#WfK~kGHZLxpPNXT4xnPyLM6A-rlmmB(|Lna?n#aJJqQwj|
z18k(JbgIThU_IcKBH2E$Jyene8b7X{0<6MkZI@x|kOYLRd
zQ~3(|-QDG_2yOn{g}Rcp&iXfc<^=3$6j3OAbcFwM9!M&_HiF<;LqxG`w_59YdfaLB
z`Rl1YG&h&WuCz5j(Xf|cwnY{6(r7q(#5O^IMi%?#=)H16NU66|%sh+2dVM{+1j{Rt
zO==LyzL>a+Cx{&y*#$NkAXwk1&HAx+#OwL>I2>oZ7dUL{tn4>3+)Ek|>H2uyPl1sS
z_u;=BLVUA`tqiqv4foh~gc|ZVoIKTZ%o?v68uAuG-0E2b>F#>&A_{pEZ
zvyx`^LtwJTLUdx`1;xql9gpU~1$L#YZfL{YO}|gPqvBYeb)HR*@I2iJh(h6830w(7WJ@K)66KTZfQzt!8DLYLKjj<`QW+TwFfS*
z-l@b1lj#HF57qy(hM)R!F38CQ2-8I3A?gKdyC&$?{jIl+b!=X!n^e#HlaQX3ow|GB
zF}PI~m7RK0)+prdjd=F8cN8aS8Q4HqW5V`)ZeBbTorDoHMDIe#Nm#;!2Xi@6e@nL*
zy_w*S-f6gaJw}|~x@aRW5%?<=wPgMG!LaW^v@d;1%iphk!%|aCt917-a`832t*6jg
z0v93r{$%j2XS6nlw!F8SoBRj0{aw(7wplli&CyLKUTvW3vSC!ik
zmEEdYUuBy}U~1pnybS%e0&`%pF0DY9+yMTmclU7!^#09^F$gE
z0IE4#0^10yjX}f-Z@R$ER?hSU@mP~_mi`xDzMlJS;ru}7p^ddR#VUqsy!o}hn%Huo
z;`Hq!RBDD04Ug=;=iRKg`~24^^tGUPzdNOee=pJsW!R3pZj}v(DUIxpc(rqD_;$8O
zYhI(N_ihVTM)+0Oa=7FrA7I1=e5EB`hEzPZ+OqtQCQ!NPty7baL2iko+mVF&-LfNAyzyRG+UUo
z$wGkfq#a*b9ov`+j1k`ecuII`OhbzlI35W|A82-*IXF-i;4|O?0YWK-cid>&r
zf5em2slt9O+Ouw+sG%?2OdP4r$U%SBPfs%y^slgzE-H#cQH24>B!B5Isfx2-X^XsR
zE~8g>(oP$ut@z^WEqT0tF^#DTBJ?odFlGS>o?+|)ti8yiabkukXSgi*36dxAVE8`3
zXVMZRHJYQ^MW%^lS0Aq;ouUBduoe2rLrVMvT7|T9O1B`pQ?B_EnPtnpwYRq3ZM$zB
zG(D6pjEQj_v@NJ1T5e(bC1WnYa$VdX3D%@~gc!AJTx)k3A=$rVv%0trWaJ_tSw}uY
zXFZ+&vddt=n~$3?7%=%KZV-*C_qKq;vHFqH)cV@@#Kbq)74PGs(~ycEv}tX2*k%B1
z*k^@~{xB##?Em^ojy_dN1wI;W79qAZ0m>VXzUG;81peWe-OeUbxWKtaOTpEg^=`%3
z2&Z}eN-41y|FTNtrw5ID#W`PlYAg;i`cwNHPK`Y=cbL+TZrx#$EmC?{V+7I7@DhUh
z+W5{wL^x_%XPL>H`SOXs)vby5*I+j0@!C&qDRjQxY_@s5w6K?gnt&Cy->Dq%YVJU+
zy8ISIwYtg^tJGWe-&Btn?r#PTzp?6?g~vIE7_E0l+};_Hlt{91ovwO0W&Z4lwT3tskl36n4i)YQpk-PFJrdyGX&R_9*OkG=sE1(&HoX&
zTQex1*Uy<9|7%?(pg?Pul7b^CnpGN6S!uTnrA4w;2GRW?%n-fd5~~KLQG;B>@2)Q{DDiK>@fq-)SDwRMc`A
z*urDv?sNGVkX-1rq4Wz0%EYSoLgX~WP-Edu3MVFkMv8)|75iwSiCQ2!OT~=M1MUe=
zg}2>SC|~z%%FIHBHcS>ZoKilpTf6@Td6#Iykb*Z0|Fd+MjcJ74ma%%La%!LN?+Hj&
z*;Uf;Le`$VqD~)I=%?_sbARE+DK%Z%j?~hv!Q0`mq@)XP*xo~dhVsjJ%!8?Hi>~Sy
z>V<`cYAr?hrda;=F*HpDRjF#^Z
z&SXX%iWu^5p|0bjbaBojv3cdsWX@unNbeA8Zh@Dv`&(mISIFhV$J?i5K~`RH8rd`9RSb-ZFksyXGAB=^YERjrFy+nkkZ*IntW_Bs&GajJlSdP{n+9Y8B>m%Vl
zvX^bifHA$7igC4ac5DWI^4%yBTfgLg{&p7=K=MRl?9yspn~AMWP@~Yak1h2;u#Smv
z=s+wNYWcNao!k8$^hRa3Wmgtt?GRx$_;3-U{5ALTO8M;SqDURK8xFWI6IQHjT|d&$IQA
zG%fwxH8dyasbsenf^PTPT8bU4Kf&nu#jZgy
zZU$Elhg(!a1q+Uh9uwHD(c6|PN)h|b)S?BmAXX2zDOX6$SVD0G%h8Z8LaMG(+CR7Z
z;JA?xE9%bCPHrKHX!RFB@lCDv^mVMpjit7f2~D%4qOvbd*)E1fz^$A7dagL*m9&D{
zUawSXNY6!gGjU3r!(%Di*-^wRyyDQkjNEiF8i>v$lMqvPohUk
zSanpT?x89D_ch?Eqg%mTP-!u{ctCZ))?m>?jXx7QLCwmzJB~Zr+xO#yEP`UtySiXmWgDf59`vJC
z6Y_h0Iw<6PXGg`3+e{g?t18IO3HeU*8xL-b&E1^Tc6VlCEqmQl3n)^<>
zk>Q!%tQ62dz~7)3yt
z>6rkf&a(zT1#+0!;oQ_S?B7ftxB9x+gqKgaysq@c8?Q2IQqGe9;R(Z0<@%#|nR0+B
z=OQSGRkAfGxd9|tPp@EL$DZ=Zoy8l6WKbG)RkA@d+3RHmO#Z*U$5rGoC8HY36PjBI
zw_fb}*{#hRgGE*d3(&L&A5;m87DbQvX7NcJ2)b^7GE;Qp*T)%z!OZm}=%V7h0C%Bh
zK6GTzp+i59LI?ATbIPiTBl}1fiYEc5XGR|CK!EQS$zBFD+w8huy`&pMCA~3kT$>
zrD>~a%;hD#IXj0nsv&A%VM|8w#>vj6SfnLdFlrrf>!j7!zOAjSIKzOMg9G}fr>A0@
zWm(Qhcq>1o#>7qQFK#WZ8%NPOWjuUK9mR7WsdGh-Z=-B6AV+1^HhPopP*o{3Ap7M_
z0Gf?0`edo%@SxVOG=1ir`p%n2gKYGL0RblfVI+jti{0MjcAAHWQ*{#GMb@DVdO^)2;gj-G#pR>?3}NkN+AJ_{JdaN+XCebdl)8eZYBi_3CZ;5kEWb&&?&bR1(|Uc5(bb99
z`jyD5(c7oj=lo~-*`<(%ITc(zgdmdDVJ#i+Ej_M)lh0jqt
zSbHGq!uJR+>P#hXcVag|)fIv2`(wR+BVo?x7qQ<|g$bBM$sQ40??p(969y6~;&3?l
zsT6rum^ieKaDBQ9XHrwEX2worG{{hK$9#VRZcR}WxXH(SBHRg44bs2mM*iHodXXheJ^IFwBh
zgv=&Y|Abf)9pKJld)9fbS?C)`M_)er8ax1FMJZ%9BMx{il4#7A!hkaZ7EYQrTG=^+
zt`ob)V|0-RsJ}=OxLs`WY94kksEq=GdM8Xn*$L^lwG5s;7j?f1iQO7Zf2|8f7h+8b
zrk}$j)Lh1390l7C1Siw?-0x?w3iED)4??brE=c5WSb7@}(Na?l`K7;S>!yRMq>sMZ
z-gcX{DQb8B3@%G|*y*(NNq6h){8PiHXTC&&a6Ve3_pt68=IclBsKf8zv+pm#so$lv
z^!JqI=I)VYv-Z3eLkn)nK	DY28co*nc1Q9(O3e)7z5KHem7?@&p~*^|E^Fn<%yJ
zJ79vE^~dkwnV07i(|6Q3HgX=O*zI3NDIuN_ufEanulwe5aZNv-tZ5A0vwr&n2EXSS
zcUrOsD9~VEP7Ja10)<-7aO)LY9N*Qa>hP>N^YS=*<#_u@?CX&~yFe$PLLe5KrGO39br<&h{~Aq3mNE;TFUZqV;1
zX8*1H<@+TbIu3)^Sjtp)A$9>*`Spxy?mx>M#aDI~=P`M@AM(emu$A3|S^);YhgK)g
zIUb2aA_&h85y$7FH{ynVUs<@BT&dhTYxsKxM8g8O(9r`x@j3(~Cd&xhzI#9z=3
z%($}1(BmSDB5oMW8g*2)aHWoxkr%l>o{4{~ybW0RcRneV=!9E+T`K_(j=8YUcY)bg
zQ&S6JK|vC=ii(}2P4rDc%=NRQHbzn67Jsbeqp``PZrRJlNRUnp*
zkwzL<9W%|WAA{Im=2)cF>(;=W&6}5(N&ksRa&_%<=ckVAES`6%_I}fL5sc+qWxPLp
z$Z}$Z=v(j|H5AX0<%H|o)`^!=^T6>!V%bjk4d(~+mM}zX4r)CiU9aXvHxwsbk%X+Z
z+1aHuGAK?PGg5O-`ofc2ZO8a1C}!m`#^MUa-1U_az+Xk6UoMrIbzIS@rm6CBB;+NP
zpy2P@4`Da1ldggNu#8YT*O>Zk(Y?Y0$u&<|E
z-lw5Dpw(-_W>W81p4IE4-xiA7Q7Ps@fXxjddGFWFBDb}kpb_^K>Zvo9uOQ(T4wfkO
zr1}DVT>A!uj+t0e>QHk7&F2@9=dd#6{Nw@5GZEhb@CCV=)^=5wdSq>vHNyk{b4kt?Nqbwk<_OaQm{mLG1z;1nWH-hA)@8;vULpDGe)k*UjC
z-x&i5Wur3zMG#d?i+Q+DF8so9zMu9D5`KN>c1iAkvv7XQlW|Pw;sIyVXD+yz--CAr
z{<+bA#{G8MiPkeIMWt4ln3C#Xmg4J@_udJ$yPB{B))~dAWVWj_c0*Cc>Phz}5^fdT
zC24;xAMEmlR`?V+TUze6Zoj*_&ALQH`0e{!N@pQlD7k;}0XR{)?
z+4Z-2eL4Dbm;E)S2}q<&RxXPM_Hu-2zIX^eh7qj
z-MHh{$>8!LNk%8Hf&XqiuvhDgyh`@m;meLXT`W7&Z)E5*ijn*pF?pi7C7W&bRNZ<}
zVt-{j$<^W!jNgzJQFUv6dZ#jjm5q0l@1o3q?Wf;*3+OSzWZNnt55v
zG=>_1emO}JMR26>8&8ECOA;5P81pOg3J_l%1j7Zw)QFzv^N7ZNFmuI=yK$%N^o=5^
zGi{eO@Hk?|=XMhMu}p$hKl$X}uj%MU$}8x$U1JzKle|~yiN(pfC^sE)*ygc!klvc&
z{8PgX!p`vkNSri~xIeA|DTt-{Y#>f_=6i5t
zfIV%&EW194KfeWB74Eqw*?hWj$l%rXa(l&Y*hMq(3YnI_t3@8rS^_!GxbdAZ=jb)mH^UI`_;y0lZWLx>A9Q@&>zMt|>&F3OrocyW
zBN3mHu5$V1E43Gv|7m4wT@wjJq^4fF47;$mM5~n}bc!JnFIj#k^V>ntyosBs>Eo=y
zmZ$35IVdn=IRQeDIl5dvikTY9d0L5<7US}It1Dl~3>)#2Chi;y?ab)!nBYm2rzV}j
z*Jw5TQa3f^J$R;NE^2oBYft}TrX1Nr+yJ^HM2xd(0d$SR50@T_Ics0AD-2q6e`1il
zbKf(YYWZp{$cQ%4Y!Egg_9F#TmR}56W)?F$mO94~!q_yN)yx5Yx!sw9as(TqW648w
zG?PrR$%m78F(vLDRK|6-wtFf_R*;E(rsjTzT>=lve90jz-If$8oHh&7h$!ndr;lw)S
z{{ap=-h=+#=9v~L7Nf51R8wxST0dB>>c$$#FDEyboix66XpKwL-e(KAz(Mb#%qq(9Nr#3qHMvNg;3c_e7uZF7~)l9-uBW)v~
z-IFIpt0A23PEdwo8it
z@>Dk04R3lnk#&k!zW`xouXgtEk!~LF~47hy;jDkp0JB
zp>#@#91R!lI?}5DeDK{n)Qi+_qN?GGX@11Ul^OdGa
zW?Qis4l!z&Dv<_ntmm3Jc8maIwx+t82f8dsY#m^q*aKg}0D>mZ`{lI3HjjSond}sg
z9-=1Cm;FW@L7MPvKa!TXT(@0ZA+h8P<`85QiF&EYU&YS&@Xagt+jD35V8
zuLK?+=Q0#k>(|z0Psd6?npf&rTbw1VX)$7XoLWaj9LuN)GpS$h4Hr)`*H^c=LdbN5
zrhMqGrE>}H=k1@vy1}R4U-nLhZjPD_wA9ZE!*BoQTCv}|v31t8DH@){^Hc;P!^r*hf^`D_7VlKhy>d0RcH-b
zbJP>N=rg(9y%6Mua8SjpcX^T4U
zGeP?6)vG?Y<{;os1h9j)|w7CGO^
zaJ^yvSr?dR%xtL`@g#z-v5^qB;lP{
zCML4to2moH@IKYr0Za(w%=@V(@_v@k$RpqwOo$KWZo!(9;7bp=i0L}y=)C0m_@`;e
zy20*kmc8~i_MT9W`MWSUtxQ(?US#lAApiJ;;IQ2*z=@oNA9UI7HF5O5pJ6@Wx0(OG
z&LQ`%A{;;=ObKi>cI;)f2(9LS`{q4+tM^mp#nCie@!Yc@50hm&iM-&3e_xS>oKho&
zEQETWdNjf#t3ju3X`4u1#XVgV5sa_cfpx;cv9dFGR2*peI6T={Q%-9n@9D@1
z;GL;JG3hFqKA>q9XzwB2j57$Pl^pi!NEvr+f3!bkdZacDN_cV<
zcTJUDyQbQAVDDu^bH>^|l*%y-NvOx-&QeWq;l{xl&*kDla*b$@gZPHdp+NN0ZPG`*
z+{fJHGd0s|yOHCpXGeDg8NxKUk6uI};G0m)8-WdWdW92Th_5xzw&s8f&Y3RrI5DW>q-#jF+xAzvzQ`f%XtJ$EfxnD`~1)=hD
zbUoMJE*3HwttcEOuAFrakM%U2dwkgjBDJUek!15Ac
z%X^I?y!e>}U5U
zDZ5|*2IJtxYs6C-r?8**NWQ=S-=apOPqQJ~AT&RMSuD?7!-rCyo3*4DbtxlA#%nyD
zM=OyS+@4;mI)mg0HKVI(l5ZM71sbos_V1mZDB{r0Po`p$C-VJa5
zVA<%g>ul%{qCYbH;wqo};-1|+e1iFL
z(|tEetRoVay5RNIlw$V=G;{-o4-AUAbmO@QYATr8
zc~?=Tq*?qbJYt?Sk$-h(<^&J)9(h>m-+Tv0auqjR4UK8!>rGi0QR})tmb+8oxldD#CR71lW
zkCWSn;}OFS8Y)JxgM=A8;SzpOu=Hu~gB@4KZaSxpV^C{|%~Ll=9M?j(S2`-9LRC*&
zR|xetJbsE9+@f-_D8`xcg5|%NMh1yBZXv(jV0`os@2c)R)rZq=LPz@Y(LpI_RwQ(`
z*Se{eO{!=Uzl0*BqJY8Pm;1uUix1U8sC<@c)7@$46MEeJ;PBnUbCDbAngCTGQs0&5
zM{VlrncKIl4%Tnk$RXbb*&9^w4GKM$ynla@BBO*o-~T;a?ZE_9;`sWk)oe>_oz2FS
znuhXb2R(YARD~Ph6~ftaLP>94#LEAnS)xnWsI^52PmLcUuISDK2qHg}f1ktN&1@!b
z&nd%I64&A>T@y~G;R!v|8EM;&L>exW!R1wRmAmVLnhD09%iHl_w
zuo$ImYQ7|76#C~W>urB(oChE20
z-!G22NT4ZyM~b9LU?mM_e~|d{`EtRbzH=(~
zIX=Z)P%a}*JxNZa;O(KR~w|?n(EA8L5N?3H;;q-Y=UNLLJ?caI?
z+&S56(ht`%d%*w6ZBQK6H<((2LZulZ0bWAWF4bp%4ae3N9CNtUPF?-2_RoaY)oJD{
zp1_Mqx>F6YS;NcQj=iY5TDvY1#DTpZ|LuIvVZX8#6tkQ4=J~kq=j2Jskh5pC{x}-H
z2mQbu@@F?6kcE>c8jmLp(NdZe2LbK{8rZA-1m>^n@SoJGFamsHH;>q!xkb%&
zY~(wH3f-WYcCJykKxe1C&Rg%wU-qD560I*U>!qFq;G3$4Q+&@?mBsnt-aikk;RV#n
z_b{5DVD}!sTbhU*M~ngEg9FzXK9G(k%5(-CG2uJGpx3ZeO(!q^@z7860(93MJ)PG3
zmlqqlpXuS0WAjyX;3Z(tzF5XqxNsmTkNIG9^ouHXxi0L1FgFuh?FA1W
z8e70;u%c%AW|A?Or2)HH{?W0^zPo@28B=9ddYTlE!
z?dN&Fpe2iiJ&d{KGG@+aDQu+s!Ce)mocKK7v-9xq2nN08z?PQW@YW?A<_oG!KJJ2>
z=7er%Y#UWpcHQ0DZtYC;W0e#-ou~cZH&Ey!ZhVd4P4Oe;!a5FO5VJZ?D)#JF?Fcu#ryE377*o)O&Fen$VH`U)=bZr7r6$+
zN^lxo`kwCM?+4579(Ky^yuAYl=wc0(KWmajAy-y1w^5!!-qPi;Br)5Yu(c7F5H;yx
z+O(k0vrx~|>RATsm%opdZYRSeU>eTDh1eHxhRLCMJQD
z^M+dg_HI&m(ElO*{3a-Dysl6?xPnYhlzv84%%4eSJc=$#XE#i-wnm(X{#!Ogz~E0j
zGC9?js$9~*Unzv>l5I<*XiX)}G{nM04QpkeC2`w!X})Q-=lG8Bf*b%y3|ax7Ya
z404qe!#L>!E>iq?>Jr1%JHqYwCJAtMrGD0oHA0tQ6qHl=$f=~-;qY0J0H>I~lhBL1
zzF$;vOZjmbifWg@0wzBRhC1w;g*{xDR{qq8$>4avcPoY~s$-jH!2Gb?!mgApFu?>gCU){qe5mVP(9kmCC+Qgf<{kq>1ixK*tM5i6IBj
zfp(R3FU~#dr0s4crKmIapmtv1kQ~N^9(D9sq8sq~=;^$4WxIc30b`Vsh?Gt`SH(Y?
z6&uCteYxqTJ&g0_VX4J`K4rCeeV;bVz4AD3^9daIm2h~qf$$@vUeA}cL92k5k+b~L
z>e|mNuiy6YDGyOfyX4ifkvI5(%xNw@9z?5ip;&SpxiBp^qk%4Bv+Y9blT;;#;>OT<(K!9A&1Y(An@4_l@i2jG6
zTLd#=Nc-l7iPhklOR*-MxR9`)!s`Bn=;Vhqq1|f~WCR4*b*0j?va?qVwFO3wcbaKf
z$|_Ax*O;knFCFl*Br*cmNgEzc2<&{ONW`o7Yd*7g+F-9EE(Wf0_+Cdr?*44_-w|#Q
zxjlwC0}}DfrHB$B(G{#^
z>ylzk@0&0sW}KbWIAYO;bw=-^Gsg9=pjNobjyS{Wg9mx0K~yp+)R=@Oql|IY%ie6a
zXhjzbY_yC-(FNR>|e{qN$=zj
zrIdr8E+YD6|1OF4EllzF`_=FIYN}{L#B?Io><_(VZAV8xLibMmo~nEt1O1M^+d-pe
ziJCx<@L(qEKKXI31CwfH)tEmrk>xeefUh%-&xFk7)7uuJ70m2VL*R_Ax03;6hA1D+
z>5zIT76Z$SO4shPuRvAegK0$~&?yA1qjH*?UQdc$OsAno1#!|*#
zJox%feP!^^`ujyhiC2OfmH+Qp{$#ZAC@`{@5QE_oMND11T);Je$0v_=1j^u0&5$8u
zFdJPuOJI7X7nL1dqcXu^EZxd@a2@n-2QF%4jKvA*Bul4O9J1T)ICg$d#-R~2EswF^
zGt3P)_!>g3j^ali3h49waW_IcsZ%@LUca?wxl};Fko!{9Bh~T~byZ%;--g0BD4-!V
z+AgNv;>NV&PSKFXQGM5BQ#L5R{l-=L#q*3AbPl=>w62*+v=2?9k}FK1io>RX5qAE)
z9OKg#kt_`G)b+ZBNeD&T3C$~3;xni9egmZ~T0#~9bVDg>Qj6EyUG$tl@BQa?+@hx$$)Y79F^Ao>%Wr@6<;kO3=lcsb-Ifc_fO!zvH#
znjWRjR(ua6T_3t0ig??Gcz|;Xz0uZ(2za~pWB9h-xIlNWZdL2}?~5c{xZQol>p>!NLK
znR{Qq^)gO%HfrmZor42)2^-_~rr}ljmM>uQxZ%Xe+Bt5spJ#zO$ZlxtwSCw3dcknW
zN-w^AHhD8KwT`Fyx@_CBesg8tjkbUPGUh&MEP1x1w-62)n-sUYSXccXBx`#cRf_E>
z!)gt=3he(}Xv4E1vgsh3Ro6Feb;Qy0m<{--lV8W9>A(+hhxR?Kkr(*cC7wSH#Y1nM
zpm}{?;V007S9hu^b_(8E>xr82U$MlR#ob!zvo74uFpyWc&o?;~82}QO-|tOsA9t^5
z4!kzs*m2sj;9|n1-pm~K=UTKk<(y7aZLoa1NIveWnhlV>Y$PDvR@53^>GvW5!DiB2+`Ogt>_(6
zRoHUeGN(Ne2k&wJQrxQLy-&KkVMwl2VO@gMw{L#{{(2fn`1I@yz)HE83by4L#u_V?
zL2F`6S|l3|)8;Yl8jefB){SPHuJCSosjz^y+lZ14YYmQQZt0?^VV%Es;}O{&1Fl8w
zBV5why`vHwiqsWYbnnF=N~U&Zo6POH<12yK4|5-^e%nRXr}0wP7guw&Jm4D}{PBmm
zY0qC`|Cu65BKXm-tcRm3n*dyg{)ngqzx=QUnTq2`N~lUC4HesnWtCPp;8xv)2~PRE
z`uCgGo5LjWH(1;nr=?F>OrZV+Fc%4`XLICC;?18c6!2<=EYBzLwQLCrweuM
zB1P40L)k0s$h+YJKrro$KAj)Ppq>&(P=PxO&Y{R`(3dv%2}#P*wxO^gM7NN#Q#e`W
zi&784&?rfz9l;^rl-ihHR+)c<(nazK1&$CqVD;QEZ4ipx#q
z9ERhu21c&A&bdC4N2NBrJ97{Wl8gaeU@0NQooQRr$0Wh
zAaCcx!@Ce;4r3Nk!G>8K7Rm;D1Rv1~5AiRYv30_VRPsmAB}6V{gZQNgWSzRTq2vQZ
zowGBpUswg*ZDN~7p3^;4W{Bc`jfKBk2k&~;TL|raUaR|U1*bv<#@vZKPhwx(4O7-I
z$WMl6xM~&~s3SB6+<~xS1FA^bnG7bq#-(i>pSWVOS_oZkVG^#3`#17o9GX&&mSGD&
zaRf0_S*5VunK;sL`{<0lAh!)7$Mlc$0^0knJ&@b6)C)Qj06`)Z`w>v$Xw?vFA^TVM
zVAOo6F~6KDDd!1|-qp?)O1F=zY|Ank1_{<|^g;*8`@g64R}E~lOqn$JBE0O}YCzS1Z8*#gette0Y%WS0Gh7rf-nZ|cn)TlB#Jy^{
zb7XeWBl!OO`XKA3v`UG7*U#n9LAeecyQYxY_g-}bztz11;L)QktGn`xi$jmRZcEkk
z3=FXck+v>i2z|@Y)->@7LvLK5ZFh89+wl=xYs+9u}XIq?-YGae`?zeLe#X~fQJ*a5e?m|KUWYkC#!H0I9}F|=L(4`XN9Rb{xXVL@7u
zmhSG5Zs|t47u_Hr-O@-mNOyO4Nh960knWB}b6)rP0ekz&Zw?)d*JsXo-PfHKc>d7u
zZY19g8%*(^=?|tW>R~joJ~{#-<~kPtv`Ogi^aP=SS}(Py6uN75UYc%p!g^Ly@1Aasv#B>{9YkzaQbF8YC
zI7-gOj&)cH+NhDih|t#|$f^k|1Z4nOD}X
z%(2g6lqfq(40&F3*STFl*Ui(}o42wa{_fuOSFj2Vf?72hMBRL?j_SAknG78yPWOA-
zy%!$BP5DG+03y7)y++g~Hj(|QfnAFw9=7sS^My_VE2bi4=X>A+wu(3cUL!F)Y=KPw
zt8M>(M)@Jg{noaZ0FSU`hf){Y++%!PrKY{?`@*Ij4mvhT)g;~(%fxUxiEo1HnuYTb
zra?L4NF%@t@%P(n_Z$D~iDviKv%#y=;5M6eQZp#EQv=dj*~Z=EwkqTh^2(J}BFN?S
zSa#u4+z8&c!<~HZbowwh;{L8%yqd8r*z!idk|Cyq%=&xq*|5L{sEgqRSyoLtmrXJ!
zPP+7`c(AByX=?f<;<96ZI5MYKe^xl2^89i7ZK@zXRiq@Mx|tR;x88=HVWcTvrelp?
zkGlYM*D-FmN(%&b>n~J>H;ln6)!uP;aCtTBz#s;eX(Vg>vynY0ULrcWlcN&Sepb5`*
zdsmg)S7~FkyfU-GP8c!6&M(bRU3@Yyc`OxzuvHZAy@jMLNEV{b4Qzui?h3(P&5*i7
ze7UKru^IRG9Ihl*owZ`!sYF!~khN(`&kpR_=h?83%!7jIh$oZt>5R4Lys(!SL12R3
z1)i;4l)7d>RvKiBX7)t|2=!A*cn`~-)W=aA)2+>=q#DM83{pMUoIm0=b%>u^)6Kq^
z^)EcR2D9NpITEtr#iT}J0Ls(~OI}8v>nM$3r-#
z%PU8ew}YLRTLS&>oNhm5&7Gebtq6UCwqG1^mNF1?cd}o@ryvEe{rsT+9B`cbi&0VU
zFJ2>ok)Co|cf!4jEXa}>T6F0cp}uM~;G9k9!4dQe-z@{VK&BII#$nS7GwSeo+9{5V^YJ=$
zA88XLn?FmTS*dx&IHwZ^yc}U=(%uBNc4GM^yip{OryH`_#kGD}U-;LHXvNf+Rq{rl
zFnzE20-N)FzOpO^U0Iz-SnZ-coV?H3-?e;ug%u5t6%_^ngiXm$>z~^-Dhrh?@3!0c
z@QeAmPR-kl(r#YZxk!!}+XEHmU=N-lDa3jt9Sr+XKL0~%NeNfTf4j;dgw>HS!A$8_
zN$RpC5bKWewl>iwgHh4RPa{*+%2zOv{>B=%qNA|zCt-o7j*I;+GNGY1sEzN@b){jD
zZ^F=R%-6Q(x;M5rkFBX9dhv94?JnOyV|r&Em;6Ce@jZ;iq|tokU}|Nq+EbqD4UuF(
zR_BTaT>I5Au2W}F=WGs}aZe`efrr3fc&UxyCBN=+0zSCjEgzw>K_E=p{bKkZ!2F6l8Io(l^7mKEA@m)Y$$CY{zF2*N{t=o5Te1W7P3;ZGLy
zOazZtH8!KhGnqJ3lr$7fVNh*a#MEwne2Zap5E?$Ir47y|tFR@}O}B~K0DA#4Y*9^=FNXm2+It*Q%TA()Chw`OR
z@nWdf)S?uR27kOX&r3x?x->@z`&JPqHCrrE7EdKUVK^&%lO(m-Rkewd;t@Y&T
zUdJ_z#SLV`x1&Gaoa6CXkEc=C)|kMnA;778wb@TjOmnO=%Baw;pi$EyE4+PVWTn+N
zMyuFH#2JA~nT6k(PK{#=dweUZs;3GiD4NB>mH~p41JD0j;O-!(Jc7JN0(cvpEBQX|
zHw--PKX~5o_Qu{|HeNMTg?o4K$YpnEGSBbIB6|5~~XI(a(!c+W-_
zWXge$4!C)gN56A!nb#WE(k~^#`B$+-$KzJuxKVM)CTby8)~0-I@i2Uv)lbbOx)qxP
zA6v7Bm3B%6Qb8j4z8!B@sZY5^q1t~seLiCrbC|98RJ`g%f36*#>~4}4KW~O!Rm559
zB{kS);A%2Et;HU5F?%9$F6`FioJaNPmw$~gWz!XghHsITD4~SwGo#cW963PFQN~EO
zfTygDMuq2f7|+3H$Yj1lUdU0G{Ie+T_ri9a(#NZ6TFmj;Ci5w~;fb^}^zul4{t{ss
zvR{#y=D-{;Avvv#+aA%y7P4#oJ7J%jP=2VOmV6b<`{gFBngpozt2o6E)3pzAnKT!8
zG{YAav|KG>&0MgYy_VK`_1p@^l4VuIJj?uZ|MwM2*vol%azps<4XIfo{TuXP_l>muLM*7sJ7Mxj`Z7%dZ>i5RrW$nDZBn6;Sg-z8TJ@^F8Ct
z6Bk;O{${rG-&DIyxld8^hRM6d}aHCkx9r(1^1U
zU&Z)U7>&z8ulatx3E5rM+dr7ft?r0H___}>cT5eiw#~Eayt7Y6fsJPN`$1MNEVI$U
z$p>=v5S&}{i8=p}8(zo;tZAY{P4m#Mw~Tgj=G(`aXSer9Z`9NBdIo_Gr>1Fw%cLGH
zhp4FE{+?h2Wyyc{(n(42+Al^SEpR(5cPrI%qKlnwiD@=X7C1#F0{epZl<{OSbXh3tih-9^U$84{JtxB)Rz)VA
zBct(67@D?a1So`9&bu}@V{3~Juyag;7V6`O*uP(+NE?{KJbX)~gN5nfzc}G-1|u4o
z<*Om-%gFf-j)*cSG)%h?8X-o=q7R@0(|uAjFx>B_=d#u$lgt~hT1IrBvx80
z;4;NA3VeAL7qB;2D6-7h78yp^Q}n9-n$q*&e=w)(hmN#GoXWy8H$gcbQ&_|);>u9R
zAcsog`m-2f@nPIe^TM`@j+1a4>zs&g!Xj-}X5&6=c>RI-s@Z02@+Z|+bc?@YU1zRV5FE3MftUuZpBhTapELpSbx?_CozZ}i4JZ5IW
zV;|VfYh=^_WNI6|6B=pQkNrEwEtW#`4I^*Q1Qb3`+_5YtB62S$)`iE4QWQ$&{%k0k
z!Do`hV*m1?QUBBI0#h`_77w`sc_J%gea666T6#KxWIj3#%@83mF}O0G-v2i|D{oMs
zmjgiOzhrNY2piVPvGR%*BH&?Gqm;4sABUS_*9(McE0NRF!DZ&cJ-2+7W&0KRb32t%
zMka;eTOCTdf+A=xqB-mxY@WCZ`Man-Wl+#+;Ll{{6M=f&$w|22q>UBYhEg9)KyT~`
z$r?&oX4&&`he@rxXBt?^?OwzuFXR25deGfbiPaed4@uzPI$L==rMJMWZm)RFaVYre
zfBRYi>_jV>$tJnQ)ny8tfX4f9RN8!>1>;ejAvH-OR_Tmy>m#rOIL~zm#17ghC<)!q2nT9UMDz
z%E~^X|G}}gdDhDc)!yX7S>iYq5Sk(#&C(_*V{v4Nn9=U+h=+IM)$*ScNsEv
zi*UH~t(OX*ym#W?(sHXUORB|YG=X0TFdnj>)`y;
zKz>Da^Ib8xJ>b8?L-!Otu{cm~FKChfnqDp{tl?lo+gaRc7|$F}jSc6m{CYEmWph4o`QXE58(S_7N~Xh_p3p
zwH)hz2}zT17Cx1~3*Py$eo){eMe+6nDu3Q>eCvcJ%sd(rBtFE?XSab!_ElC{=4X{P
zyOQ0nqewKO+Fnn$_jK6#mq9#uanXYSDZb2}Ps?Rl$v+rN^n2J1AqibA>Gq;HXp^)u
zl{zh7d0@77uf1@~7}K#)#k6(DoQdaGj}elUDdHcEWI!s28nydvyApemxrhu8@qKxp
zL+hNkU!jM0{>b;#g7BDVV>X?L>xi|8b^q%6zO}pVPJ7Dmxw+zZ^B00I
zg|LN9AbL!72M+Dai@yO%6Is`X%T#%5Uzs*XKKBlg8+EZ~5Mz0}e;*$)9eEnke15z{
zd~>)YHqa-$cmYfsiIzn9GSf*#^iDyMsq*F->->8Op;FJ!+n;^!dhzy$>PfzQ(;Da%
z$FMV=Ds*X24xGUbM>641k&x!?vt1w+kziAfH;&95=8kf}7R4AcDf>Qd@OJCB=l9_E
z%o0gs@;cD{Nb+mqjExC7-8+z(VRCpiS==Gih%_V1T8f(JYn1C4wXYQ(n^4m7^{dS-|-BpN$YIumwfW
z?sDrXMlB*7_tmnFJLcO)u(b7J^VvqD8>TzpP0a&cyXkL0*jerTIyH!paote*+x@wx
z&DwXa#|qYeeRPZB$x*Sj(Pb?|?(Q7o$hJE(+V?E*A=<>i}~M(3aTyl9ZZ5pYxsN
z2;n*K3&*4O312bf?a^XS_ZQ@GurcMjjnQ)>oz`W$Z6nk58#6VV?&)&!Mgox0L%zst
zzH*^~1HuXkK^#%79U%5h`DxKb938x7A3F@zK8`lCS?0Xb){k(XJFk&>q#Iu`;kL>VM#)gjJC7Ht$Qy!krUm?zJ!~FXQSF
zk8p2qP?`$rdoLsBYj>i%pb``&bjV~aQq>6h_)N?QNmD@l~E}ALE8&JKpH}Vp<@?VRc@%Yu;>!g;sIalWg??)
z%_w`g_bi$i*hLH8650w`1^44@yrZ5Yhl+r{gaBwzuZkjznesa6uLH7_$&!*0;fKVW
zJLqMd_Q#Migs+V>3wf-JfOUi$dt<}0ik(&G;;;1Ob*QS4kNx0F0^T-<=yAMl?T>)~
z>&Cvs0g{s7R}r)T+By(_0*4VhUagiXpgHS0m%JvTyT#YM_-*K=aW~cL)NhH{jI_#U6H5L-CjSeUnU&)n};YCUU429M|g78x`g^E60*-z7_H?lQr_zdvD)yaW1
z5=bT4L1+k8w?dUJ3TM@-SkQEoqD%G~%&V#TlbiNAL<*3%zsx@Fj*oDxE1o{YGz7$i
zLMB`~SKhwE*8}L+<3aH-6v%DL!)|OUa9<&I1(czonA8cOz+-Z(9Jc%Qw)&SMwSeav
zI?d3K_%xBiw3OGy!Q|fy^b9|w?zfnWb-%y|)hS+TS#-%9r12Yec$29ijL(gfi#f!d
zg3<}VZU%rU_kfeTw@ec_76*1kEm6`atb$M6Y_~1ek`g0Ia;cmKpGYnz{T57GidK}&
zkVlniS+W~!IU>UYZ6?>!YFEMGB`D?2cIrtkFgfGnGqn-X3UPNi@UiS%@r0IE-KWY3
zJ~$fAEH$l(O}qfI?kUT)J=c@In!D_@l5oGJ#%j;_%(MP@M?YxbdJ3V^?d
zIUtGRW*AjbwOV(s`C-loomjZ{aElR{ThY*o+%vX8L);o?8Ds^YVl`wr`)$1Vu5*0R
z8K;B#uqiIGuKTBiLE*J`juz;H=N)YylbSrS
z;0mzB?E?}{D1<_%acUl-sD$muFEoFrYbQLv&k{a+|7${fDf!jU&AF&A;>zkZ*Y9Kz
zWj3NcX)xORMR_5T@BE~?-t~029MfHD*LA^d2$~Jo_pdxgo6Yiv_Pk}tG||c}cW)2h
z(O$coP@t(+9Hh&Q8Lf4<0=q|=g>IHfl}68Jd*D-UTfMI#r2ajQkEk2HTbmrD?}YYZ
zq1DmJ+#UkYTy=xN*6F`iA(!>%HuNiIao0Pwo8L&)-BE|gDo~kMuHq@&cqc7Zx0QDo
zYWCi9C_7EGD7-8BhSn?P!mH+oRVf<#^)}64!w*gSZg2ZQuhcNQDM$;O64lcRs-s*a
z1itQHuAZ92!tP$vdvXLWnf2|ixZo&YR7TH&8wW{uB5?X~(9`|s|A)`+k1v*$G6_L>
zbBT@P$|eUpcrI?pZb46LjlSRErfkhY3W0Y9LSdSg?ZyYmf
z)vH^!9j=Q6W=o(r9C*-L`1y590>V_skT&CLtEB2Ni`yMf^V(XuWX8414}&f-O+6)Wx-B3OG&*
z8O}}(t~WPf&q?&Z^JOMTQX$aFN|;e9Q;JxQ&V0pMkn^O%GNY2g`Xfm-1do%UxSyCJ
z*Y$LoVV`VbzKx__3DA$v57oXC-S_+5JHH|4j;t@22$Lw*s2zjsa7?wba|J?_VF&5}u|Q)A(~AOz`cuD#<+bVXauIOPu9AF|+(TrvHunSK*Qn4rmiV
zHyNI+KUIa7rv%^fxgi;$v@-P4s+VWEhI_(YNcit}j~;T8T6DT#SyoY3D<{#K!bI1Z
zrj36Sy()0=kwzp33@g3d27bQ>u0JR$Dgh1cU(QJ3opX$NT0(KpdwYF&9^PH2zrCCE
z`>+d_kHPOc8a{FxB1(7ld*%3!zk~V}i>%NvW3XSARR*}=4|vD18B{JO!j;5Ao1Tu~
zAq4h?LjOi+=RzPm8sm@;rQGe~V~_P}KApHoW{S5O;ds5Pld<{R
z9Mg{0`}kNebP47fP9nUty?=gw
zJss!+=I7tpG5kaMU?de<9>DgMx38t9zf*#^0
zuQX^IfzJ%X-N8qFpJe+xu1+psMFc@QYi3MkSffE+pQeVuHg&OyO&fo_s=YyQ{
zsUk0*8Z}GwSexHT6ZYH3!SlrH|eVfvsI68?fM;==P#UXz?uF6X*#Tif9_4u
zI_sct-sV8xrp+K)f57bCsl6I)b@RysoP4JbC!*RvAyU`LT2Op%c3eeZxA(eBcSb(@
zYP_9X7X5AJ#iCqN*U8-G;#|XcMb1)y+(ES{Ep;t6==j>-bNbuF@bd>T>%1#lc5`;(
z+7a%4$1O8$%poh5F~>3cbiH%~Zd9qYZ>7_(yz3hLbC1@mDR!QNmaMnFs?LZ@uZ1+e
zuY;;;P13Vb+{d7F1V6tq#ruL&rkDkLygnqn{{ju5u_BltJH~(2u(|3Gl`JFs_2c$W
z{qAmAi2e1BG?6Zs>PyYU!Ep2G`j=tLfS(Iv8y}AADKc;$scnBgHR<|XAL-tc^N(H7
zFU`B^8hv|uy<1@g7N1UNeHi-%YKd*)UU1Qd`}thJb!d^(a!!Y$Kpa7a4?lsDPKFQI
zeGb>brr)O3F%s0>tlIu?C(!f4(ji5rh1!-*pdbID<0w$&-Vy1Z21Lp!W8_B2o59(c
z@CRA}ScBIG@advXnd$;_^OGD7@v56_#alO$)r)<;5eHE4<_`rj5z0Z8kQ
z?XG#QF<&t|zR)1=f00J&{TH7{2kwGtMT{w9Y0he_*^}Dlc)-f@&%V>dUJt81FdZwR=
zR2@`o;&s$QDRq}+>Q*uu^Wbc$w0xPt%bheYc!oPv9H5HQgqg(&CiHm(C;MV7Taa9(
z8S5%ENc9JOR4M%vEle9bE;?C0u1Uk~iM#@3m7pMn2yo+To32A%sIN;zmsQZJ
zd;UNy>@(PG;5XKNulW`->33^Em73~~Lgh47OL?uD;ozcS=T8lhVHXK91@2Y-yT5le;mMETu-mwBfw
zW($GscL`a_BM1~A-ziHJb#-zWLXt@yfwk>N^N|Fa6yV5T2e#F!Tp%PNs%vg`YZgyH
z1ZI2lV%C+=?zd*(RT|1Z)Q>?1I1w&8L>NC5aM{Bx|z{|%KAzG!?{X;_T$a8Kz#niAU
zcf&cx!?xa&w@ykb4l9V8(Ouoef3TEsWnujBl7Gz8+~`!use)8Ai#}@wh*0uN7Bh_W
zd(lHd1v2iz9$#uQv!f!;q>VZ33>!LI6zK&!7GoqFqe
zuA<`yR&{O0zpec=_mdGHI+#ga=m2|oT|`XZdNZYMJ^lg@5SN)BiVHm$XYP8
z*i(K3#gM+l-)TU-$F+VFhGyQ22^s8pfVDg706U-X9>ec*NYVaOiMz$E;FN3JK6ulPPHhO!LfiaF74H
z@JM3E)Q1(9r%F@}Beg8!%KcD8sqaUl3!JW5Qxix}@pjYxURy%T@Vi&*>cI=q?`j$<
z=8S|DudqWJV*tSJ2MP|u=6Ot7^=h=Mhp)#;U?1=-nF*9t0ANHWK4^r;tX7NNz#O>m
zl$99^W!D3SrH+oBo14V{H!J~YXC@mRY{uyjVI;)_s!Eq{*H-zyO%`~k;u^`0aihU6
zL2BFomwxBoL*II@qJveK|4%!4ZXds>b=hhzVpn4PP|1LW08zPyM?eYLx>W;uVv=d6vfdx;xF7?ejO+bu
z?~#oc(1k7UmO~wNJOL?@Z43!vQb?YtB}$=W;>zeeQc!qUfkcf&TiS-N-(r46F=zd?0|8oooX5(E7i>@nS>%p?m5@=cQWZgXov#fu+|c$P>$
z6A%2^XCM~t+$m@AoAmbm{qh9i1*&IMj=tF}oWNWjz9F)vD!@(pqVB?}A&H{d%l;|r
zKM?-d`pirLbPYi)B8TYefj3{R;jhY(vC%UUgJ>y0;^JU?(cIkRe>e0g+v7F!JKh*jIoZ6Y9x7b6J(aml}^@TxT!)Iex7c=ea@
z%HBNytYj>2Wa`A0enNW|5DhVBsV=&dFhE@@^7=JOT4=~Fo2noQ;Aoz9EMw>@4eTA!
z(~+2(aQ3R-f)44(!G~g>|2_+M*6w9WI6K+q+DLXL#yeK<2&7&`#6&;PWD{O52jUgO
zQ%G9%qjIu0aP45Lt)R)sREY4goZcqWRwHwl2LNpi&#%kj_RN}k%(4r?omiWfw`oI*
zHbQ=^+xfF>QmPUdow-+%5XQ^aXN39(&Yf-vHBCdL6&sJU)Rc3(RU*Ac)R!Kx8zQ>|
zrA5D&-^BezHoDS=)Ua@@BINyc;Y6dSdmKI)9E{0ZUxDTbL;_!`o#paq|H?1;1_Ybw
zB*}KsMe>>)asZj0n&H360_obr_9cYAhh3weX&P=bSUTGy=9wV+aAx+l4<-{zWik_T?W>EwqvmOt(tQO|-ZZkEQD+`^kBWpFSRh!5
zp3sOaD!M3c=!(aOZA2#Ka?eOmb186&G$v&^qFNJ!Zd+t=w9k^_>yx70nu-6aRPK{cj91
zlgFkjrho~uijI=rb-InBBB~8^4Fxg3@z-Os(o3+WF4P0K0zZ&)Qfst%!?8bxE8}Hz
z7>|jqAMeXFZmV0x{6Y71mwSkYejwGGj~x7E_!pK`VAln5Hnr0q!U~J_PqS~~vgBQ2
zu-GU$D%{*c#iO|E>&)R{i6W*E{EO1FJptE2=lC;f40ri;jvtJhoNDq#|@S
zWXF=v@2UIA1BomwAD35|Qbv?7t7bI0LK8qaQFJtcYZkZBk5Mg^gv15a3y=z1IQ>T{
zDw^`}^Axs8&G7O@FfYGpgfoLDD5H9at|N>`4nmeTU6Ip2okXP6`nZjEE{TtG)60&-{q6)4HIm?dy9+oYZ?v@$V%4O>WmD7iwf&=jx;K(AAfsB|zEH1WzqG7uq1?D(qP5Mv&+nle$Qc>DQusZQyafR7%E|kk2!qRmd?jAL
z3$lcl&s!Wu80WRkDK#mgrMZj@$3_~s+Smj12O|gaR
z+*c>i?r5NUtilKLX_!fL9u>g=HBN=pELALt;)<{?R
z%9qFn+q^o?_I%_B@KhAj+Lk!wCBqZ)>9U0cwo*ulu9x$Dh>4X_qx`g1zJiQdXxb*U
z?dYS6*dVAFcZ1=S(3P9?<@aaSF9n-9w!hP9<7^s6{4J?a`YqYTU`yc?kChZ}GZ
zsuJ*AFqojYW`zlZ*@8#l^}i>hena8{=5Rbw%L@BD9IrHRCq0S}n+Pmz*xBy`CtSae
z+o)5QK9RVczxg{piG4pBs!CCc^;QPAdzQ(mbWmk+SEskNH;S>gXsi|L`P}?=ThZoM!jtIjEQc<#TlKDaP37ii<@hhWi>NIdw(o0{XJCk4Da@Kk4U9qQI
znf)c67O4B_7|Z7D%rx`6eY`fF{j8Rl}dN}ifr3XinZfI6Y}CS
zqgML812qgcVc#X&U&d^;IN>qImchH_(BxQ6|7jef{}uz6)c
zuH_&s#GV~?NYxM+CnvUG!#NHcYc$nIQpp`jrdpJf{0UAKUY0qt#kDB
z9JS@5kcp=jgaWEZWXb^GlW@`)2+&`@s87t-8LDdiHl>n~;$EWHOu*+CT|-&N0(JC#
z+ojAYfl)6^ErNwWsviDc;Kf8|ISJ{q%OD}gcZAb6T3c!^;4T{i>S-G>tS^MW&GWU*
zq4j0O$_x4D$!PUI|D}mqB0R^$|J>9wAJ?wsr(B{qw@p<;7G&sHIGLlE;XI>bGjI{B<
zt0X(I!PlfkMtf#0Gj26?*hWq5;vu{ss)1h`@LUb17L}ZXmf(^K_XOkO=_uGjWf4(R
zWUmL#*@u*ZWjD&*Hsb8+W*Q;~vB>Qd9TUyN!{c7t
z`*MOs{DLR&%4w>ZBbV{BeQwYZ4;qXvU5FbqMCm9GtVrr38C7k$Y)0!EU@TQ#I_d~0
zv60@>a)ghr#%KdI1_CywFA+rGl-2G^#k-TU16qb8;^)W!`jemaKc
z-;fMK33TNGYkgoKkq69J<@0H`=7IFz5UqmZgFw8pr&;~^ORmIFg3=UzsyP-x@f&9fT_5AS{1%hr8=5f(%8
zXX(u!aq``D<6e41hM#xc4lvAUwEm;kekUNJ#;6vRnkJ=qyVP!uS>SCL*lF3yhL5_2
z#h)V24AwAnZ0J?Bg0KOx4D%;glI*Bfw^-KjjVWSjqB&py^a!Kt>cdh(o2L@pAteX7
zRR&=g(Q_b5W}4(drY>pF{KCUY$aYif7}1Rd-M`59kbKO=fd;{i2WPXeIe6FAUrr_o
z#@|G2ZKxMtz6ZO4eROZSKsTQ5%P6tg<(2*?e)ZhKWJz0w_Jgh5CMReXr|M4yM-@(E!Ke@*ldB*x*^5kp?lLxylMxASjK&m6j^)T?9_&2YO
zLy#q}A^QAf$Y132Oh?V_t1=hvX-{_txMMr_Aj_+%MtE5K$!G;AepZofD!90%TTKPBSTiSZl_1jbFs%uc-)P6e)td}!wQ6*j=*BSIA*;9Q(-<)@`5aU~
zflDdd^=|w?UwqbBT^Zukzn+X%U^WKcZ3zjOMYD9Wjl>THKk}{mU$q>YNdCRpSh6c0
zUioNN)znsUqCV^0-2@Gu$6M}YWqvQ(E^XT8Y~%Jww-1@vWnu{u*0ixiNf)i-UU)uy
z`tW+vdCp594BMGkfm|+X6p62wdTP5kZcM=No0d&aGvs0Bt6EXMK}&?;72$%7SOJVO
zV{1!Xg%b}LNBp}?*E7oU9>3b7?$+{Tm_;{fmUmd#9okkXCkmHr4#iK9jkvOLPyxmr
zJ1!pG76(~4wtCkWg1~8P1pnB$+XF4^yE+6N!704wrftzW?sXK+=3pq*1HR%v1h5?6sKB?i6O2%g5BovjotQ;Fe
zV~G1}+Z0A7(Bvd-(z$hb*7&(OJ~tRL4VtPod6(8k*$K6>u6g&Xxswv^s(Y+iA2FzIywQTN`%!vekN)V-P5(qw3qHtLsxRDq0mvJ8j11)=eZwt;Zx6K00x^hddaF6~I#`-e{|OnNA~@HSR=gV1K{Wn}}+C
zJ}KMvD~>+TBrKYB;3@Uip7lYPynNsAJB%3<@^eVVuHuXETW4_98!B4vZlr8hem=|8
zA&#Yo?JNVAB_b8TjOo}UhJTgKE%2s6H3uhSvnTZ)8iELR7DZ~11ZR{Gga8L9!+2ZKw;UYoK$
z!~Iumyhn*}wVD~*38lxN)f5nzq^F7
zW(6yn@T(KhelU_zp9u3AW-M-Ow*Dz0HaH;KbVy)=lF^F~{@9D3>mq*{igGU?$N3)j
zRQF?eG+Cy-X3J;6YLM;$9`#wnBuN<8&8f?HhAVpadk~2~%**2eZs6UX(RCM2@3N+A
zq(z}fX_yLvbK$8#N?kHl;~E=WKIb4_?NbV?`<2IV%U3!HK4Q!PwnIz#WG1bDm50D`
zzDt9kgNaqk-V@HhFIIN^u*DExGb56|E0cj4rlfZB6`7d0uNRjbHmMe53^dn~wwbo-
z#qJXTtq2K+k(chW@szqa4z^qj$=9$EyBXf~O<|K#p8iv-E7lCz++ElfF|
zYg<*!#gI=lCax@8K6%!1sjJiLS!8wZWSN!#SO$e8tNMPiJL7sF5JR;4Vt9WGRqP)Pn~_K4N^KrQ=5i_gg1Y&?E{ZV-6eQPKznR04mvx{V*(-?+ZD
zeMa-a%p3A|NcI8b`~=_`s4D(22J(f#ctqp{M6pPzr0;NtSRSg_RehVYXWiw
zVRB}dWoRu5WmN9me7cSNcGM|1a&e}0E@Q4*hZT7e(P340o-}8c4j`dBytb*TW&w^_
z`xX`g?he_Vot^6=g{u8vI_k{6BYamqYP5?pvVY&xf5BG)Ao$7%pfy){7af%NiKSE0
z4RMVpT(m}rL6XBEH<4_Xn+Jasv(i27=VhalPu3<_xnU_;z>qM1R9(lQr);F(wG}rwsN;YLmyh0%2yb_(wm_O_Uv>UcDhDk9{9}?&9{76cT
zJ48>PslvlQk0(3gtaz;~rK{{a!pJ*QGCHc=U4Vme1b`jPL90G?a8Bb{T&JEeD$q75
zxAc4+$A#uR-0%`J&O-26CH#7u&hK#<&+lpkFMDlc`@}&UB#j!h(c+)*pH4f_wV(4p1*W6^=+44xL>*g$Wd>hgy`+RZoLnA@8watI?rItB^3+ah9~6*#e)76-w56|ooZWmo
zd!^o$d)|K4cKU+quQ^h#t(5kAK^OXCMpQ=$rM+6a_ZQ<=yyGeDJv*B#4IO^y6GpLh
zAr$5L#_8w)t`x&9{#-Zsh@XSw+O}&#d9dR-8TcwyD!f
zCVQW6%ksHzMvXe$?UwZz{bKgPY2SROR{f?kW>uS|ERI>IaMyueZ6glixc=|~>v3oH
zQ^^VJ;6YW+DDr;AtxV%)3>VIb`
zuG-rcyE%6~93Z~jIl1jWR52p{54rUkOm%fKKzND?m?)=BV`$@fMnmgSt&xjdwZ@V0
zi|z14U80K8T5g4z?d){koB2y6A1!Bf4I^~2{a7<%T@>O!X0KK69`w3m^2VO654lb_
zz}55&3HDiS(&YMfvi~{AZD59u^e>Yr!xKzgIBj8Go@GZ6zKP&0*Nz2=TMKp%Pq`Y5
z3W6zNe%`R>4@D_JzvY<}XU~iWFt8pB2%Cx4->~h7)?3b
zKS)O}=jm!GaBVei4k$F`t^_RVL7y`fWu!0xEm|4N0v^zAn{
z%0?=;3`QO;xeUf&e&PVF*{@2iYHpySO{1K1Y&RZ2bp9#a1qF8jRPxe&O~xrhdQfH*
z%(G>hd~rJfEWQ5?CdX@@?~+xJr{!#af#OJ4I*a-{^Qi&zKWCTy
z3P5%bGktV?#&ce9t-fDHA?Opnm2BBFURy`eia@@ekAlEchWjJk^Z^w6ir?W~8)gFm
zNgdZs5p{xME(IohAz(8
ztEc*jN`lRzHPz{I4O@{tQxP{xDk^1@QTKCq>maU_Q$464*h1|Ld02r-NtbAnv{1Yh
z*N(&N@_qZ%GnAqA)-{*Jb)gEB2pYuz0vTnJwvqhDr4O`0Ka|rP&Yx39O#AEiH3hmm
zHp{EKc9!e+J0Rfq#6sToOHcJ^Xfb56ozDUCucbZu1=vIAR`KkkCsg=A7m{+KA${T$
zEetGM;m=q#T4l0zE!LbkwLwp#l)fD6>UMm;a|)aI@Gmp;NX@eXA+ARHR~bq0$xw3}
zlP-6GFE7Tz8Dmx=MBbRb?h#$L1i16H{&`U8~vvOR+O(V3HQY3SUWD{
z4`mXtru;)qcP$|aLKq!2@z^woxC>~zB{`ewffFl4k;sjcbvU>DQ;ZsOzdKa?{G4wh
z)Y2^VNP(8cfHJuc+wIe~x46Ob@Nyk_%sBKeE|GTE+jjMe`5L+=C108jG^$!Zj|jVh
zo17l5UTYp_^KBtyzZlpw(`1}c^&j0HGrgO`n#1ZP7A4Ay0A7^MVMn~O-)fi}C^CxG
zm2AEk?xs=~VxiLRcqMN6MBXFpkm35zV8=ChqT!a66%8PM2^QWUyj0|ZIrzaHXe9Fn
zeJ`$Y#*F%wb9IfH3L&A$(3)*?x@jkfPB?nS!;O9Upz~~c&iC;tCH`^8Z2C<|FH(0h
z>0IaIG3&CY*BLN-#kQkt=YOL#dH#AAVP#|Xu0c{9eReubqN#N9gtEv9U9%Ds$%}ZZ
zb#(5$=i|e}UE7_YwHx2U)Fo{fc11$92OaNq*~RUu+ZV12Ss*PO;W9GJe
zoM4W8@eX1QJ0QNPx_<(jI&?X#Y!iy%x4q0bS!%?sSMI77(uo!z>j9ng0gL-T+>pTi
z$E(>N%>NPgR#9y=(6(?(
zf#UA&E~U6jk>FN}ySuv;ZE*-%yl8QUqA5_UxJz&g?hqizpYM$S@!oSEl0C+LPS#jc
z)|_}V(^)zK|3M^^8kW(byUL^e)lSjNX571Q^tW*P_e^X=bH)yWlCvUlr)>Uphh=l?
z?nTh2eAg@$&l!eN7EhvU;VO>e%759L_h-1Ip&UYVl2UN9g}Vx{EMDH{$@<6bO8c^t
z-_+VY>Lph4*=&TNW~#l2&f;SfUqY4&9Z_7?fQ4#4@n=GfA1m6a
zywP+TPMT!R;oG3GC_Q;m{`hM2w85MSevPhKV2o-M&lfD0
ztMGXyg~jz!_e0>4&^wNIZ@o=guGH+n?v^E~UT^}d{3tm&RedlG7wZ?pdZB^P
z*+^jQh?PqA-~XDDtotnN7afX~KQNL+(aNKL05A|DBnbQqh?M6e=F~W^VJLL0Yy!o(
z|Ao+woS$pMU{tdwyoMiDuP4wKcL*>;LEfBT@fD<#j+~t@_m3sr%~o(KmOxJ5G)vpe
zOZh7tljZczWm*DCvs4_%P%EY_oKZ*ECe+o%8Jpim{n(_tO-?3T7skGRwTBKiImtl+CBkp{X
z_(5+jY;4Y`w%jj=$149-SDtm;sA@U#6$=*&Hw8|d35&wNGM`PMRbF$&(*XKYFX*pQ
z7jIvk#cnoUdiuV#alj7JpO80zaK7x$Ax_?pcYodRoUePV-Jz6iKA?>0sH^tjR{MQ{-9=Y8{sUs|+F`J8AA
zhrHUnTS)MgqF89Zg`7e_{m<9J4HqIzN;SznK!JUH|JwR|3b}(DZf{&-8aEK
zB(-gPstH;#IhpjiaXAK#&lD?eYbF%V&pn2)BzjN(w&_Xio4D>(zu%#fL|bNo$@{~1
zia>C|G>^w^r!;|t0$mD#0WU`O|MUooAT3&@#Qg*Xm!>s4Crneu4FBrh$@J_Xr7Z6I
zH9GNjAy{BAQSCFGS7U4HwW?84mu~t2<=02g;g?+>w}PYVe51j!pH;+^cm5l@U3}g}
z1pPdS;D{3=edLte8!~xjCJ032GEib$%y;tx^*tk}@X};^MBVHubC197E@dwaps;VZ_J9$~fpnV)^9BEx+YSLxt(OGAe?9SF|qXo1zjopY73Byy{$0Nol-z
z*&|jnn5ZRsmfZ*?Y%=V`OW5tc%cPO-JJ?+ehWXIHLm
zGkLStoMx>!%h&ZgsJF>>^1noj%blUz{Cu^QP8_$;y!sIWVkjJ?bnQdbp-*@I{k03)
zfpkKl2en-U-9BeCVnf2>qG72xM1LEP>v-y2jb*1d0vjRfptc>uB(nN?SEC?JS{g+{
z@)o91^ep~ga`{TNzHZ(8&3V#nG4(X1a%~wNbtzh3
zHPQl?1T(O8!>@T`aFAzXw%Y3fbwm6#22R()s=ji%FS%CFzPq-sk?=|p&BN}@&gD2i
z;Zy*K2?7+D)Riv;}$XUZ_xluZN4LH1Cwl&8IZe=SR<%$GX?-NeNi)
zdFX4Pk&zgaA~3>p%kz5|=of=|lkiU#GeE?T3w1I7j)b-TX@mB$!LStg
zxn|@#K4`76d%cmn?l60h6J<_-;UQ5+vytsgdj!9=qi_a|_=d;G;*W)sByoe2|JOex
z5kTO0hgmlppia4C!7QM2M~YI|aFE7mcznXztha(`??_Iqegr>XOlV8J=k0l-M=n-ft`J8L(IKuRd3l$KCs$DI%|LNC8_$dA<|LJW9OknLLv;Nf(55_vh!=+8j
zv&(m9_<0*TePtObqVS!-{?YG7jiffs3&GCv8x+SfzU|Y~4Dnz%vNh)yrT8h9)Zf12
zu6j2R-NVrRXODY{SGI*5Of)<+I%e|3i!#`xp*s15=4#q>5OAj!W)fzJP8)5xF=OvK
z7jk$04!z>c&yi`598x!g5wAxQ5g~kHd0s(Bfo9_=-&D*Nd4Tfe{Y*UP+kDH}n-Rhz$VCwz6&TT^u15!##D%?A_`yToWn-^NbEEVBrL({!TkQbt`wyHjU4
zLqe~2YJ3%2EMIo+c2p>XTra+&#}Yq=Mosux^qfXtPh3-=yYFu1JU3&F`n)g0`jXPt
z%PyR(kb#;fgbnI&=2g&^roPqx*%J3SFDh{NJRo~Nw$#QY=hr#1{DI&=v4SO*P!X66
z5%$;D$PmpAP+#=cJEo5cPH32bQd_MB1#-qq&SqsV?m4WrS*UVL#6)TyZkxnSjpk48C=FOj
zKAW7}Wr+M9X*%Td&C9T(_X}FTQtxTogGyEc(ycd8n%%KB7vO38Kt5HC^4Owby&C9A
zYv8}eG8aAr@{)QI`V^&lE{0>etpV>*Z~G;|dVN&pU-(n4(H^cTN*2jZ>~DIPW2~Bh
zwQ)SX;UBu$KCzUzX=|S^Qh)bqmoTX7`Ln1_hw@wXp7^o~2Je&tMf(nvzgwCJ+LrPOm!EL7R_{u^<>2WZl`lpQb&Rb1vjNCe37=DdD
zgTmAYQ3NV3?{8B`tu|K=HjevogqGQuxvlR!Y&@YYqdSyTjn2o8kD$UcKRMWOWRk6g
z6@kD#q|UXBZqr|~#}lB%9`Iygrz>)`^cy8K*)L}Eu?*~Xx}@_iqpd62g8a^l<}1@up@&1^vi1#Yu(sNGNT7igb*}VwO!4@bOs2lck1z_tg-C
zlckyRI=cs-8m&vFy2WmXVO}o(qUIa((?P0d$VH83&+92H==c(u5tIv?_of8-n_LTm
z)-It1=}&iZqdhHvw@Ol#4EmJ-hPj%b4k89SMBg+OR_0lFK>lKcG9B(lXex`-Uqp|p
zAfGcQ{V#9O{~pE?FA}OkeFm-H7J8)pldf$z6fS!pbbdf@eqqd0v%4K?J`z3HwG?re
zwiPU-O-E)e0i{}B*<_Q^~6p}ksnX)i7t_Ec6Gn!_W
zDm*55WFl~*s{L>dw9QItRPK5)?x;W`8d|gT=U~X)xQ=K)(`Qm%JkDR-6bs`LIw2aU
zuD?ERRA{B{w{E+bu)=o!He%!_w87q2=Mx~McNWD?{di|4!8ymkz0AGCjQjbEBxD17
zN^pF2vXK|Lf*?nych)wuT97=gpo%;s>0n@d=nA(R$1h-OyU<&uMhKnL&4~=;m*$wR-t&N6TSLBcovNOoIrKHVn;}eLlkR$mivn$
zNi(aEz`12{w=*(|9-=GI$ds&Owd?mO&5w#&
zz|=I3$&on1d6(njvjr031uQ=(5~tv0huXd=#4(f@wRjhv>;z=F;wWbM=+g-_)|sBe
zbOvAD9BRXvwg1#z_&icbqH=$I363qw+HId;IQ}LKT|iAzho{UCgutq|knU{6uXG@p4MN4(Tb@
zv2>GqHFr*Jft8g%k40A0B5_-=GWaiGy#WnGJ-YMp!cN=c|=ai${A>9
zd2ClIh5V`C_4^d~i2CQ*F@*DUnqg!q-DdNrZYgP|x*V2y#Vre2YV`5$B7CR|WNGy*
zZ?#0G8*sS(H7aSPDdkeyiEtFoV8nS_*oFBz6^I0P`VbjN7el)6Ww+i1CVVNvOiYy#
z2}{SWTW$k~n144Bn^Q1J>ls@>KTiIrC$*Ks;D@YRfOfch!gt5ZH6FSd^JrlQ9xmxvJo7SBaaYBE`CX=0OpUYBo=
z8SdscK9?j{$L5NuvtGs1RV`%@TDO$D3`JSRyLB6J-@2h|pu~G?YA}+~8ilULNjuSK
zx{Ir@Fp)}9D*f#D{R~qoTWw@u
zGmLu7@X?QYR(2J#`F_@s0}Gs|{n;bgy05E|GEDwx-mNvPqCvpE>!RM$S+<;c0_FGC
z*4||t=-A~0DdexxYqM8!mGa=#tdW$F6bEs|wm+?gX`vqLOavHf=+uydow+rP#cn2|
zs{_A%uybUciNvO=eYhVX4ZR=v^l|{Dy4dO;ca-%DY4^pm#9Y)F;s*V?Ki)`tdM+oN
zR7!Z$_;*=-uT`@55dL8kqZy!fy(+a^%a`yqG#
zG1kY_yo0DzfI(}G!8ni}g+_5PIV6+IALCA9CO6z)UdLt0o0>+LUJ((9T*H~?j+>ug
z`eA@1fd~2LK)suWU8>4lYFIa2q$%Jp`du#P?V0>u(4ya|@35$A5Ui_VcKtJ}>(X3z
z#gCrHTMEgmrTYL#_1~-gd~+(ntc?&-RNsQu(9(k;SUbQ!*EXL@a1v6MdgHkxuI$a5
z8%moK)m9hraO@^wOu?Q7k`e_0$~>PeJpO1^7S1rUoozB9p2q;BH3?W)KMW?9u^Mz*
z3#x2?#xrREY9}3QU@}JUx?;@q6pIacYWu+otSy8h7&dUe@Be6bQ>Cx
zNr^$^!P!MDzm75j>QZ8^UgS&wALDABE)i_IwDqVfT+KT%jhu~226$rWA~cpoLq2-{
zYJ3r4!sQKO&B*nW8R8QC9z7V&EytW6o31bfjS+%jVFgEMXJ(0Ps?S94;mkT)l44zl
zu%4V;j|RN&p`^5BxJs|z{o>EDwyMad!d)s;fyf#eH?^P;be1PplP5xbmv|A51lI%xz%th2!8VNX+4XVlUr_CmeL#y20+<
zx0*2tQijkVGz=xbl%`~oV4WDXAoCx)4=8xUdT^m=I|eS780>y7@DE_
zE(t%)hLBDBbWhyEYCoiENX>(Exw^BXwu|{cnIb3Y?!JZ=8ehE&KFjoQ35}}h6S7yHGIp*dmMv?a
zLYq~UzohNvP|CR{v58tQ)>HrvsyiV>pu&}sKF;7d+fxYk7rBX%j?JUWk`=JI32pVa
zk+0^k3?YZseO;^SUJ{L%O+NL9^>5GzTgi|<>X36{6g;P=tnN|Z_#vik5OYeTIb4g?g-z@UGDLTti9%-
zgDjb0y=-dG?1_%uK0BFZ@k>pEUi*c~K6z3OH*euuh+=r!OiV$Pl+q4y(1VR^h{VDB
zi1YW?P6~;^vwMEUKh~`(IrDCuO%F1qB%1@)qn=}f^UC*W<8mATL(_KRJf)d1Da46^fTZkqVT6H>p$1#{yUmp
zBH7m~q)U_SBlj_ndS8wh#(6;PhrDMGiY?G#WSBBtU2h+6_l+G&Ze3YgA8J8qA4xDx
z5Yf81=fUjbj_J-%Jr5f%ggJ^*yhe`lN+oU^R(Ka;YyrJ%RF5$$Bg<7TpX%^+JY2f+
z(QO2$&v~CB0Whugubv(}sU$2cjINN6}42720@5QV*cjRFntq=~+sBU>-#Bj!Y5|#uxbpoh2sbxu5uku5o|<@x$&D4O=OL
zB5!1xmvP)QnPZR3GlYkA>T-NXEsTH^m^L8>(RvNc&w)?0(6E7Iw{SdCPE6hT0(vR6zVJ3GZWE)+W!nO2pFZp!&oJN>xeEu^h(nJ*LySd}
zeyTNV6`if_SvwaTw2spl@erlq061j4P%gsv5$jpZnf&iyoWWyz>t?OJ%9+GzdOV^&
z4|@kvdW>ZixhMMu12DO5^fBm3$_sx~;cUYiDO2<%D(g
z*`0R&1s4oy2;H_8tcNtDm^|+6PSWs@Aj+V{e%AOEE~c!gTb|bGSb3FQyI$;T*wIRB
zcshK=d-Wut>i!QN>tHNni-;VTds7fGHjJttCZ@bx?a|0VO0-&U(Vlk$&M0sajaPNH
z)lj~nvwGn$=>vRZGOjZM|<#5){M9WAV0*BVE6%B)CHJ3YiGOUW(B>?Uq
zpXnUTsh=gXZ2bBcYy<WdGf`?aymhSPkFC_K
z{k_7(xmHQS2C?)om)M4;-FUy82#YikSvphoNW@ZtkBWuIL40)$rk+K}=S%kb@i$LQ
z#;EC9)eDX@9RKb4#%lT^)ArPFA2k?AV!A}ss<$MV6~nb+ImTIH<8@M+*3wU^#8tun
z%^{yoN?l!DBNLPV;-}zY{>Qf>I8x{{)tg)W;j`nh@5^SQE)eC0LTfT}L{{jTTtsp_q{wJrDroqACbLRdSL)q6jA%xmO))il^kXhuKI
z@G*{mJPO7%?(YxG9#^_J-rx^<;rQu{X@B(hpDMaHy0hc2yKFjqF%fm4HGoJ0@#ftg
zkzLgk(@$R16&4OIi^i-QDOarHV?!^pDt;!-UtdxUM-{a_SBz-EykralBe?eBvSUg>
zf*j(5)>7v@sb6(c@DOnTx7tl`+UJSG)tjo*8$ne+TQA7XX%S$w7u5(G9g;=+==i52BaGlcYVm
z4r`iU7~yE4dSnKp;*1DWKv5fhV>j;6NLLOQT-e&o63>kIPU|EZooMdNn}1r<*tUJ4
zb2=u%O$m2uk+EU_A}6C$lD3yoS-m8P={>8l8@P}I)NxPZXje!n(8!KAP!>5I%<9R*
z*l*;~V^zc(s2tWKCSA1Oj@+(w^ZknzzB@sZhO;U<@-UIA{c`i3_-U5f_+liLH!Xmd
zJ{f=NpD1TyDZi0U1|+?3HkK~kYeg>J1-^WT0k1>WW=P`%h**)1#Tbz0qq>*
zwNOl0YccOx2GI6x|uHXobe(3rc&mav>$di{dB&XYK
z7#1gVH=y=nn0PQMar=XdA=AaD7?PG2YPB@B2WTdj$t$3b1tA(
z600wONZw1*+#~%KI3`ptFr9AYl+%pgxmC^x?l3#LKZ^}+IW}z+EbrTAl)h0NtF`fK
z9g6yxNO|i`gH}NQzksv#Pkq05T+u8s6oE8y2~cLAZE?}-En?
z8qcPi)F(t*1XFf;425hICTuNV+ZvggUT+Veb8&I`!P`9{5Qvq&fPJcd!*&E6Wc@II
z2wHo;2QdtJkhzv)Tyl)!L0~V#O*X}1{$MX6ouc3KDm@_iaF&=vMH0pDvV{N^
z6(Sd1A{^>ktht=*b_>rFQy9a@4uEskON-Hg<*3-_
z%3q=ikLAPKTk)Ln404KEJP=H$!NflGRTQrO2G-FNTY7!CJbs>HobXpV_4{E`%HFyOQ{#gdn3TwMI-&4Vmv?Z
zf!yTaHR|6J0V0d0nR8s}I}9zUA}Me-@jQrDn;KX#lw18m3Pf=3+-vj;)MG@g(vi-v
zmS)9PzM&fa)I-hPmLs*jj4;=rneKkPv!kJ
z`M$>O9mAr3zJo%31n2vl`aC9*WlX-p73+2LI>u_KtCSnwA_L-gB9MDj(p0RWeeAmi|UKZF1110H`J`f#fTC@#(9kgh9}h%|@l)wGKY8j9}Xj3)={l)c=waT0(NYQux$$
zNvp*6oaR0B=fyTo+iMMu;1fA1N(xo7M*!2iy|gE2@Z|_eRmY-Po-InnX4@pRswPhY
z-IP#212%m4pScix>{!`YGD|r~)=^9Uc80+n9d9XQnP}^j==4a2HSM$7f$GNlhvlY_
zCxJokf7`_Vzxz8^X9ABq`+%g~g_p)+XL-ANTS=dTgg$eOH(-Lp6&52~jp{Ex3%$3m
zAt!g?tkwGki}BH-aLJ)=N>k9d+ks`+;zA5hWwIGfN>wQWoig^Vi4~p-^D~o3CDF;p
z=-1qYeFNnqelNCpg)8?)4z>TXWER&te!hAAP!~AcetvOdfXjihWN4QWVqaB1I8>Bq
zKlrgDuF)ch#3;}3ii2J=ywy47*xT*drz`#0Fh%tV{2
zo?_GawQu#7c9%WkJgg1*`pK=q11QkC*c|8wTF`?EFTi218$JcFvR
zuy`@c7lJCnc^~DLy4+1}`Q`X7##KG$+&7_3Oq092*8fN=$~~xA7BIUgo!@1>bIQkxB~164^9ftPD9;;7ooz+4_lRc+-;1nmjkf{G#~V%`5liOl))$V
zEB}GG*#rj}S6M)5^g+>JRkjhepwBRuMfc5y3eC}}+y%lsWE#C7xM$EHQcLX~!|w2}
z^UWelNxzLBZx3#e!mBpzE}I5
z=p7fLXK8<%2-yRb0D4q2LUK4BaA-eL-#$#q5Pa2&?HUHmSk>DH^5}7gWyM<3`EJ!(
zWYytop)6Fv|BpZjrkrLI_|%R(V%`QvByfb0Yo#fP+7$^o21kv8vnzb8)N=^%I)CMX;HR&WkL&M3ZZ6+R1T>qcgYvr|WzcXi
zy>DRekCqZ857)e=3A~9#VO{HSbndS%B0ESk_qbW^`H?0d>l7)v?l&UJLYyDrim({?
zlr;w{!LiNQSj)7#up>y#(B*JtfwJzYb7tn2V)+Sf9k<$p{i5TBAC=g2b{DxgM7r%5
z1DPhUrjb9HhUji+v6LJXn+672WhFKBhktkuEWPweCKq-6u?X|1+-wmRfZV3@7T|Yr
zt^t7;SpAnmlZW{`8k?(i*t?HiW>g+Q2+W>&-F}{Z;Zjhht?Qt8TrjUvxUs9CzWMeiJb00w++=+i}
z7P_=^7w4M1^Ni+PT>m!u-PVolwJKAy7yqoc7vgzvKDPCagGtbrCV)G55P^daZ$r#C@?7!((MMV1_&!KKq$CTHJMy>~E
zRwR_``>@0Yuiz`L_wsZpb(Gz_#Y%Xei|h=$gsOe{D7vy5NaF3K%1}h*y#m1oc&v>2
z`NcDlV(i2JICbV0*JF2o)3HBbq6z*R6Sp@W
z-ar-B#2iml60;Wp@E#5vJ~s-aN{ox9lRHoK
zc&vuR+^}J>M&e%w@b55KtOq*x>mv$(j~on|NPqEceU$uJp|K0VlQK(i4}C}bi92My
z4YcilTPMu=LVi{^scE5VYnZ5GWV%>;nppaMXR2yOVZ_rI$PcWYWr%sKM8C87FD64+
zRB^a5O*KU$oz=RfxE3r;^F6s%{jc9v)nA%kG=kvl90CXQa+4*ZJ6VgZ0ayL0+W-ne
z>>*YP}U<|BP
zI@^P?T;cy?C_cd>J4Gv5eX|!?D>v9N3`i?EbAG13ElT)#%BO@%m0xqeu!AZ2bpJN=
zDlqB(euNdOe${4D;9E+pONZ#6om?KPlkK6kYoyDs3(G#PG+(q6ah@8)*sd^_7z}SE
zD&HZriebM9|v8&wVyc
z(u}Bz*%}#!Uf}kDFF_Bbz+dZXm&CMD)TU8Vm57IOlM7%WH4Z`IepRrMM<0bx;tR+<3P?bN^E~D46fi
z&BaS7=*~=SU&9p#`J3p;Rv8r7LpgRnANKe#sB+>5VCe}SUFFLR-O&tw*P(qj%jx64
zc*>|5dmAv|JwIbF&Ly%!KKlNL!Nwvo3~M3~N+$F~xqXqf1;y?|LO+qt9vB6E5*6;D
zQ7Wj)yQ6NieJI8Me~{gZk<z3bK`{sT
zho#|W^RmL6fsX<nLGT?hb|YWNnO$zbTJBNgJJAD)$}4$y$=GMrLv5uGD^%0t%InD
zr1V|wFT;qTNikjP6Zq977A+>JHl%V5WgK;^qms=}iTf)J;vkxcqhi|TlS%ZMRyAw#NlGhSKMlrb9feDVt&yF0KIMwe
z$TEB{0rx&`DKtuydzpq7>olpmP*ZBd%q(t+C~gxP39vNPL8HIfMf*u%hrm)E9vuSZFYaBh!gs(s=cSP()-Ai
z7hBG#mRpI<<&aBu1Qs_s*y@1DT?E7mU>cr|^2&+!sS2t191L~UZ`t2%mBUObSVVt(DyDV*ve
z>~Nlzf4>(f>Ri=Ec~MTXzFgympot-K)4&wf+1BrSi0v5{LE`v6N0j>u13X5zW+V>2
zfhn}q{SQwC)FsSGrDQ%1+#MnH_vvdLw~PNn5+;x#E1kjgKX!@bbMoee)W9Rzcw|cs
zB~&#*Qeuc>$Wxx@%RLPHVQUB?CSeMi773)k>qN3NF5r60mjD)!s%4?h>@xoe)a?mmQPCTJ;re-3z2IwjZiY9t^N+~uKUpsZs%Osy9S@Az5VRR^ti)v}co(c~zc~vbz1T^kj2!qhi!#9zzpETD_Jibs
zs}B+(mUv>g;&bz|beP$mEIj8Otr=gw=NI!Z-`2M^Pl?h6!U*?q+o3qBA~;||3b4DT
z7+Y3%&ZDgoJ^Im4vN`)AnhP`d5dA)ce|&~h>*RpxugQl5)a^USP9tg
z&QfCxN-V_z+bdkfpY|;7Q7sxpCf3fE=P6CGiP@1Il)w?tB-^sd?IReg0eUMdQ2
zt4(;g8I8@L+7mF*KAF?6+$NN3S=8LWeCZEu_vG4$AVT@khkt!P&u3WHuYL+gA7gYi
zVu#}+?THQ=m54luO*)!ybV-~vP+oiaD*M=;Wp8W_-*WkD%h3tC?@D8JqO&q8!4bRH
zpzuT7AIGWNP&`^R$nP^9iUrZ8PFaKxXL^V-07#_WX;J?{6_5edp^0{sH?}WEztTft
zmhZg%H}{neJ-jCwPk>x5zr1<@sndDHr_BHmh^4tJ|>^?%!a
z;G?t1AQbb@bvt+JA9zSK4QbuWgzr6d1TBgRftvt*j)^aTdcCUqhuQE)$-L{hNspIJ
zL&gHJ+NEj?EmRfA!00;vQiWd-*Oz<1KuFGo`%L_`7ID^}_h9i+3BMA2wK$A;t5mig
zU)qP%bKis3{fMF1_fYia5qnWN+b$i(45Kt-effr8O>U7NFqlIz$8L$eBwyLH3!VZy
zHl8qG+rmYc+peWjiQ?K-oTc+6GZ!qWPwK?cl49vkTiubN(8E~Jqu37dVje#GnMjAv
zIUFhI;)0%#u=b3$kn-Qvi9
zFYwfP7Cj?;4X1k`Vf`4-Z}P-Z&gqUrib#jTEgxLob$T8zl}y8Qdn>?g#TJCKYjuRZ
zT=I&oHuD-I^O99ogNrc3A^pz^=LUq0!p)^Jo7Od4;UfxHHade7&ZOx7IKWj*^Ju6d
z6a!_^^?lzm5`V_I?eoJvq#%?^R(NO@^G1e_C(xDXntA)KD-jgv|I-*;G|sQuR9?xZ
zCcq3&1O@VSYa8OAG%x3wSH#kJCPm3aBfg~vJL6+ZS1MfkWZj$(*`!(V;39ig5iV!#Hmrc)n>eP
z^p~hEQ#u2I+=Pv>#wqz-VBGYeLIc#Yp`|2d_wpf@Is}p{dB6L119p&xP3US_{Ti{
z$tl?*p29g{U=!SS4h~cvLqyl_KGi%Zd)l}$nkC;cyE?%-B$`>;{Pem+;y6vz><&NQ
zU%u(HU8U#^Qadjb>8OZNA1b0pb@d~DY-~{Ip&0X@n0>^&j$&h8ZFWtGLft4H+OYlt
z9oDxgC*2<#=<|+i5Ra!HX@Ic}cKiOArG9CzP~}`DAdmHnXyl^wY|i!J^XfA$CX9Tg
zeY3&ZQAI5Rdw9sGuKB_lQ;#be_le$6;#A0Jvw)S-?XX8b&~b%I@JS96+u)eHX(7q
zQlPEVFTi$vinIb+)%dKdJ4_q1_ov__t7^6C_ds{;>y*(TUdGOWu<6I=8s;;>wdwYc
z@}VJ}Dn;KypKlG$k%yiwaixmzJ_F%7X85BIfKsf*azGlfJf~nt+<@M(?2Hie*t+It
zI@3#OW0pFc68mT^R5;6tGR2i`s=fF=F+FAY*djvuixPZsV<}~sNxNmU?S$*vu@0be
z4(04J^Uk%a|DV~vIlzUt(&V5NmMZDh;rHY*<~o0*TI2rDmktW)f#|dTQjJY!)#4aR
zICUmw=KyzaySB22yx8|mN^D$AhGK+XH0)3B5;Rv8ySS5NjJ+Mgy`3NlKIe*9eh+yf_Vf0f&#u
zHKLgd2bW}zjknXZ3Q0G#M@^Ir?j|Z$3BDwmdSiA=L6}1`V8d|P$g9*3`}5N&9M76s3x^NCGSc#JPDt#IJBY>~hW(^Lt3?
z`HlTYgKObv@md!`R}&Aup0}t~fCv1K4;u|yukkupNyB^46f3Vh!!g<8NCwrlB4#%K
zx*ikekYP=KuSH_)dRdy
zHkQM=Tj)-O22XM97ZZy^Lx*rp3!ms{q7j|bO^g2mQ2^R<+0_7Gyck?M5QP(CuwYfN
zi#<46M?0lXU6_4BpV;=qOU+t~jU6z%ih7X>TuY(l7n=dBYb|^a-`ie>%;(KZLqnSOjSokJk7aGu5ZmS
z?t_>tdfPYxsQ%$zsBI*D8s8h{yfbFG|9h8`lQ-P{TAg6%Btss}0vQA$W2EHTKA?~+
zWZ*VkkAuFM?r$We-yS@JX8k-pkJ~FSxKqX*Dc#&g&!*m#FFq2sF1@t$skKp-kq`YT
zUKN`5UHq}mgUusa@*2YTtv8=|R@GcUJL;?t@?zCpxG2s$yFnD*1$(Iptsn{co|QN1
z1)v>nB3Se8N5CZ$zxnz75*qu-{OY(n3S)c!r=CTZO2aCr>fZrn`O*T}^J^j#J&D|k
z0?UH9f*uc9g?
zh|%J^p?fPT;uv5X3cm?{t3?YNo_t#y`lCZ^W4CHe)Vilqy(|sbrI%tO?4KVvJ(SV0
zUfe?U_~P4p$aD2#Hg=$5NFZ#V@rbhfAMxI}I@`<%e^yp5xwOIE30L=*mj3}snN9y{{@`KvTl-$6S#5^+A4Cyzg+&Jt}x&V^CgUD`{aRcTPB$kEP^U$Eaf%mM7
z9i=?s9hV5LF@B<61O=U$lOWJjlVc~5=X2VfZm25E_ci4%^A*YSLAcK@Xldxuh!>{z
zAzQAcb#4X7rdNJ31ZAYILn?4W1nHJMxx#lV4Nef)XKtlX6`qXAZRLZ$28h(^_eVr1
z#q-4;Og1!&$N&5wQ3Y~w#2i$LOv<+7L^&k(M!8CxDHjapQ{*|ppIPhiHJI~EW7JVm
z=N@aB+ZQ~!*_c{}e3hA!H04FV|Nf^4p8p%Hy8NVjcRD!?bd3P~S!cKsb&@h23iz^}
z=6ywv7p%k?BIlT2bjqKN#yod&{OJ8b8e1dz0vA=1X(yBcVw$|{O5a>kEc2brk`wDemR`PCjB@k6jwTKqMRpg=ul-
zMx}NesCUyKAGKlHnByRQ@lCUaZ`5<%0|B0W0iH4$!sh$N8jAl!6cwA|SCbZX+KcVU
zs;A@8zjgm0qa+%^8*&_uSy~InD0Wkfmoen2G0ilgKR_lI8TGINrV=8_
zi>6>EfzH5w37zC}iVal10>72+TWb7it5jvA8hQ4YRGZlMwn!TR&0F87B{!4c=YQuC
z!aGkKuPf@LB$hlvni&M)Vwx`QQJFZjN!zwt#bi*PiDKB;z%P?u;fLd7hVhtNlO_pP%X^wl!d`;Bmo(4JP7xv@mMN7cryFXjyrP~JF
zLRhe~tzC;5_0^!7;+i9`eOHmIE9sNdt=w~j|6RPpYv=98pK@ri(9NL1yQ1rL;E}~A
zDMH4Ww#(IaQa_Nwm#@YRD^5A36g=fK1{9dw4Bb2+#@g~Yc}?p*{=!jyk+$eP9KO?h
zQ-dwF1d*H@$_mN1bvqhq0mbf2sh+8r2dho+<0p(UYAAynFs27hz)?qx1c*FGp1a0-
z2ThHes7uiIa4kK%h==RwSGvRZJH&})Q?vRnVdJ=cayMkTBq^Ut_M|b~IJDv<%Qjv_
zu#&Zu;PRV2&U5>!vwe1U?QuBo;QPDLUhMFVuUIqNU{lTH(jRdzU7W^<2M+K{Jf}88
zI7u;CAPHG+vi*d3FmA(Z4`mzoyV;bmWhUpEsC&4zU3}V(i&k$7|5sv`I#R?sNEVO8
zNO2I=mQZiYq}hQ^`0
zhxlL5_kWk)Id@z$^S-n9UTZz8i~ZM)G7@{^pfQ$uA}z5x-8b|h8P~=38z@$~$ZW$2
zjd1u)25?O#RI3Z9_I7HhQTR1qI}+byE-M&SR_e9pybyx%Xvk)~ClQm4X-E($p11c$
z?PqR7OV96FYN@fLUF)ifT$3mqkC2iBf3bzN&B2e5>+V}-I^seVeWI$MBe62uf
zoli_6tyK`u?7IKvfQvuX5B6hD0h;mmUoM{IkS~O2lyCtmQ2oRoyP98zQO(DMRzx(2
z9U2UJ6hdAU$b%oYhiBk)z4-vw+|6d+2OQ&mAh7e-?o5x}jLwlw?l2Bwub{tlbZhXc
zYc0X-KKpotdr~$j85mEhGyt!{4J@hIPM_jDgK$SOPZ{Z8rM4j!R`%-z8HL5$w>id9
zZ4rJ541VGw*^N&uAdNeH!Br0RXI0zL>)Jq8m7!ENdW7#_SbrTNS#O*c7CH@}oGaY=nIh^E`+*%r@z|6qZ!
z=4OvIio8j{q~`C0=b5dQWylLi2MOv0U&O)l&t6XG+?8+my}*~3R?l72MB6v=hlOUa
zh?y9PeInXL<%7#uHrcb(xgPwJKVP0WxgxTf+^C@%f`Klju)V28wwBuDQmseNbkjvm
zjn;-U#rhA6%#d4e(x_us*fLNev
zlF+!;F0r&tQL>4jBKsfRu#;VKCevJftTt^=Di_DIqF%*y(NHXzjFykG6yz?tJ!B|l
zrN#$mcynA$AkB#zOt1HTKEe;JJx(*%$DIKVEdRw)MV%N(7yh$tZhV$YAw>Cyl-k+V
zKH+@q-$G1^!mhb^bl*67v7c}?Kd1SS2e5v&-GK~Taw^GYG29hn0=1iR3k-YAL)#p+cV-v*+vut~_45ZrsalZ_-%1wO
z8lhW+d^B-{IvR8r^rk5p$oI?A?JEX1<5=Nq?Z4yj3n8Y$!+9+(uv}nAO=1A*H
z8}v5%#-ZW)Ml<98T$+5ZAkH#P;2Cb7A+3WWUOaKgU}PuHz6g%zxI4dPzgugexHx-8mRghP>_Qy8t-FW1ZV^rc9FhXi``+E)Q$ba}EGs5T
zZvafKcNoh5vKP$raIPG1ng8w!HZ<_%wIBI6F}S~
zOgTI_jvA#a&z~yE2nw;(sZ+$^X9%jA*llxbabZhS_%SkG474J+4~I<#UpU!ScM9Bi
zh0GiR_6vnk^%|jVP0#}@8?CrwCak{5&5VP!>+kAG;1t56A0LyB<-T*47Hz4F?!0_5eop`L{A!S4r6p9$_cyt%yJ_QLFFrTEFMSWNy;rXtZ`h6t<^&}
z;$_q9D6|@=F2p7&*B(m`0T8m!1*kXi_x!pm=yURNJJz9!8m9NiyC*BigSS5A^^zWJ
zlij9GZGUYqzsH|K(=ky`cA;6h=8rQT6c@1dHqbVVMdKdEvQ)>q9;`WLVFczvV3D2p
zo7GJ(%QRXaBgvFhRBT1sk{8_+{M(SSb}^5qIsl;DY1p(FKJ*yDzpc37$Gxnz@FJz&
zJ3~8NEfcN@`WdJG7M6vMl#MS^&=7r8`D8(sez~}^>#wV*@J7>CX
z*WxIL%0GTuViKF51ma(-0E}6**NwiLiGgY}pswItFtQ4P
zf6wMr9&p+c1s{JMq04vS^hoI~MuZCkNV$5_kh$FtQ>NgZ*#=dT7=>f<;vm%>Ooce)%JqhI}nZ
z-z_WCISGB1SAg4@W?&Vf$BFcHNtCBf3DLlWYY?HYxC)RwvluWAH&F04`ck{}AX_3%
zd4_HI*v&`dO^z)NoEzmb5`b09K=s%iP-E=Ct(KLY6miL?I9($$qreRTTYTE>7|Y2A
zM}JRvAW{(mWmN#`Ln}vs(2CGFBXA4!Q~LV#=+>th%mx7G_4M`6&o(81qIY^Y%Y1cZ
zWp&<)AfL<-#8O>ehFxm
zMmglllUHdxu04fNWGg&R`VGFEAF{oib#7@d8#N-#QFRam4iX*P3UWyw$zR|jg<9SK
z?tCzHWy_rS`GwZswPsg1f$f33#h4kqL2mUeSm00MMifh{Ba8W)&oOqNX#F3xu|!Oe
z3NbrIwv1ezT2wL>R!%-Qw!vLf1-&i9ly9ii_7Q#f7)XteDZB6n=R86(Yc6Ab=1*zk
zU;fk~sdFLbAl_cs5K{q6xagUte#cNXA|PtxHBof_g%!HGY$ZIz%~kwvq`(?WmB@s1
z$T*g)8_Z23_qphigc-?!N)9J!Cxmr;q&};1RqZPsAnd!XNL8kIwB$W>-=y!lR1)Ug
zQV&boDGc*RpV_ZIogfGFh}pze%deE=SCKAe3InVHA&^{B2XbDDdf9ab_xQ*~(ED)!
z-{S_Eb}M}Kz)Kb1)UC8d&{3-qqn7iAYFer7F7-q-^P8?icxEtce!PHgDBg`gwMaWk
z(Rr4Y4&*Y{{WQ;Wy_HD}RLJ@EmLE#H~?d4ut9{qOJtg-3N|Hwv!
z8=KRug7jK@uM7PxFtN!_
z*CvgwLo{e&w;$u5H*UhH|9XA-TRdW4IAXKf
zLYqJ|FdA6#V-9C||JrHq(h5o;p6_z{$J24@6INI9QjzO8EbcyMh4OUND{en0H7+;%
zGX2vduLX?XC7`CG9fYCw+hRE4o9GpW$+%f;CwVm7^9T!<)l0N~HeqtafMB5@SGG^c
zZlfVe-#!0w1FAyBfZd97fqqAHHzH32{CaHSJk#Hg|3rL0z2Hb&NQ4BY5K
zZ)QfrY$7`e{Pg92X;tyrDl22D*K;V0S<5UnOI$5Gi2L16jd-OdmM}>r0`8%uC0$_r
zZMIuJ1rSUvJ==rPghGA~$(SN{-vN&#P?@W0?aLk_4+#CxtDO*rC0%mAhb&w?=)T{H
zUCvo}&n@;kY)XQ+;xP=4xRxz7RhgT}L^Yef4u1A8<4@xs1Iy?`-{9E~EI8=D;AF~n
zYTh`;_I-4!o+fC=0BaD{$;c*%Bh**jN5mO^5kq?~#D$~|yUO2mluTG!WFNZ}eS|+&
zdBJR7f5OwX%Vx`0H}r?gbrGXT8+$K}*~O9{;8nW0V)>oS_ULITl@&ty%EL1gl7G)YKErLv8r%gzIdoW`2xJN??b;+W-yY
zVr9hevzyHPhCWAIG7M}9L^IPFpqU$fPpdrAP{s5=k+6PmygS#iQnz6gquMW)mvDtO
zR6LYBMaY=V)|RHa93(`2p53BF7b3qp!|NzMyY&=sixxMj1foQXMHtvxy%r|7)I@?I
z6~EvuJR2tdRY|y^LV!Q2YbLDkcveSQX1T8$`kwr3LovBZ83HcpuopsKL@lD5MLk$!
zNaEh+o*aq7mumZ`QgtM?+r1|I;;(&6#+v;7_nqa==9p$R8_Ztkn
zvE~WgYI|NLvQ_lCBX{dKyFG!yBI<^S#+(%(N5GpQbCT)s8d*Ds=CpMrRvw3!p%ge0
zt5pLj-sBE8C9vW{oXPPxQSjnQsS($)>y-FgaKBB~&22Qc;4pFUX6A2%+u@)ES*lT>
zBLlEo!hikQ06F<{dfR9>erW-co3-I(u*k}#U*$ksZ2k?A+&(2C7>d|6gfsF8NDGyI
z;98H}?~}56B8gvMe{hLQMu9Xn+Ckx`x3}JQIcm%EV}AhUmX5wt$4b<~z??grd=s*`
zGpjc(=tOEx@1UjJL+}$9*+JNrbo0Zp)uZ$Om}PLQ?zePq~V>*SLt&%0p@|CMFswG6*?R(qYm
z>r?@hQ!uTwYi@gh#i6b=r-x~9=fmxrZ!_$qM>C;qr5rvOoc}$8LXA5-+L6~p4{~7*
z@`F%)d4G4K_0iE1C7TN|m%Rt1jH^73jl@!f6&+*%C*$ZCci8nI3HL1m#+Sg{X)dF4
zF`o1p3(KukygPDaQrYOEe4z9@4DaFlq;Cj)n!jeeyWlfJ5QML_6ms{(YcC&=N%O3<
z)Sdh{){8}&+95LddE^#fy`9lOomM0&S&E24YR)4S2mi`Mg#3y&sX_9sN>g3NqA!A0
z(MTfoF$5oT?3(u-c4z`#1&>vR8eWCiM19RIJBiIsY+kZ)7yss3mkVG$5&vDTm6#S<
zaavw52RSD0vXEtBbmkBmbJ3_E1KY`IYs@Q++qV=}+W=$CzdH~>l@s#5ikO?5qa+kd
zG8gI$i2s^%zV}S5WT91qI(3?zK-a=X`OS%@{S{Lkk$`pMv8uk3eB3-dwH&s_$5P8Y
zLI#scTt1GC6GC@zB*iiM~R57Da!0qC^-SL;o=Y}
ze%cMle;y5xeEECIshMlCLc&6;!C3Ul3gRZnmHd`!I3ivAV_-6&VYhs|37adzO;s<6
zLL$oZhFWinGT>+_*a(10P31oMj=$Vn4VEtXv}<(aEsIkZiOYu$7Jy0StST1_D0iq!
zbRjb&lSwJ>u?2dDD}q{YUT+MLh@;fP8R=&(H5c%JX568fQ6RnMxG@0Ot^uXhkFeEr&3GPokb{PgD;T3_hz#9B>@cn>w$YV`%AM<=Q3J>SP?y?~YRq0x!AILR5yU!EzfJrgOzu;Bdw
zCwts__~i%oJ)?wvk4OYG4f#qp)YoRes5vx_+*5k0$@J%HJGzT(NHZ>?AlD}B}2{Me4qdZ{*R^XW@tTId)Le8s&N5;x`4
z-ep4m<9M532HHaDZID5)pIT0yY@hT(i#Zc*F)U-AF-zVl%r+Hy3kMEvxaY5?4F4J8
zp|My;LwZDK!@z*QK?rrc4>vZhBsx_wjP2_hM4^&xn7=JVpraGm!ar40*L`xV
z(zn0-8`2p_CHpTaQH(aCNy*$S*0j1Pbw<4U(zb^r{!Zy}C^NBhox^=IAY&v@At#QoGZaj&eU{y&22NtzhghOIPim1gY
zJ+t4-pZo1K+eZG^26%2j5iIcA*9KPmD!9__r2Mdj@#JMHe)>Cuo+)$g^VHNygHBE2
zY^2#dGnL%u3*XAq_O&@TgR-V~B^O=2D{IPUeOSe^KSHQE{#41ko5?lRSRI5}fO6DQ
z>3|?^r|0>9nuj-4eijW>NkRK9!HkkFhammPif`@=!
za3Q6lp-)rS!p!iSp*mf^i93mOZmQST5?AH=SDVueUr`+~3z%5Nn$;Mk{0>gp$qo2&
zcCOtQ&$0LjHcWZ!Fw{FY^!J#&tnWn;D_LzYYLw>Vk@l#@6ugX;Ybnn5+vIS&2whuy9EG2U!Pv-^rzF4~XFm`2OWqHd$Jh0)3tnX)T6}=y=dk
z+~;-31$NdTrSDLXyxte62wYmnLwrG@*MHc7WK-wRH;Zeg{Dx;`KTTDCk$X$1(
z-PuhLn=wA3*TCg?hAnt=ZDoQ|zPXzNZ;jVQu*A6Fm!%mwy0`Nb)#Rbml1JyWzl|NI
zIiM)aU+1*D!|fS2Q7bhFW`CBKOC>A=>flhADXe@9uW!b+K*~ENpFZ|B5EB$34M)=D
zD;u5&;#+xF^^@TsUfVPHCF=g^*SYO1mW%oq$IjB%Xt${-7hbr8+O_~ow!y_CV3-y4
zALtJU&=A68lB=g81d*@DvAXn(Fw`e&N^(8C3jMBQF~7IC5#%h?u!zgga!bjI@k#pM
zpjuuh<^puV3OMc@(|)<7#{ITz)IRp5&n!NHE|7`caOy>(=&N>N}LYaF>YS
z|ot%9&rEax`pEG-6%--raoR|0_>7aQ=s`ArG^kkcuEK)6DsYp&Db1
zL5yu3ZF&xJDk4RtNd$nxu7S#u^R*dtRof)#->7wf>rivoLn`Kl-h0qqH2oIfQcp5v
zOraKJxWl2A`c~WAU%(|>bWFpngT~h9E|S!65iRb?)%c1EFY==d7{#uds~({au2U<7
zQaqNBaf0TkQ*M@(VKQrvX8qoB02Wnk!{v`3bF0RA&6@xu&xjcq7_!znK-NwCvOj2<
zM&;yF33pN|PU}}^cqg&&k=9NHJfag?n^TIAORdD!b}BXamqZ?1<{E+N7qFmBEiB-t
zrKJh^Kc@eC@*y42w{#L@quUIEIuU9}M>+fqSB+W%3sjZ5tCZNpV+6A5MfjA(hok}=
z7zE9jY*7g~F@x!G8jY4#SzX-84x{%g2run>vxRsL-B%NgZEy0bW_s>xxC5RX%3d}(
zxHMJ7NQ>k@J&^aKP7k%J%u3}o!x=$me)wP?9m=tJ_eM6?+5wGBj5wo+rA+ZE@BJS=
z2PuT7?ll8y?k?AD@9mI_<@PJ1-9wXQi?bZ@ru7dmj}>m7MMzs_481sV4q#{uIJH{9
zf0p3b!S0WF$Y)fo-64X*p*k*o!xn{Wz?g&-5I}f+_c{OBbv{EOZ=GY7mM(Ff8XJ@8
zkbxTllAAz9xqp#dIW+={_U%mwr`3xy?=H%5!r5efBc6$V-{sh^;~=Wr$R)dd8faFg
z%~13q59T4TR3p!X?XCCw(=Ssj(VlpdXn7o9BLymE3r3h)mcD$I1M}{E!uI%s0CA5D
z;O=zPq)g~bLQ*2)@g~tG>WTeretB##G0TR0(M|g>5;#P317Om)0YVXAjk$gZ=^Rlz
zuz!z7hi^l{B>+No-%qVynYn}4i>V9#WQdfYPQ@l6YIgi_Q8Ckq=t*1g2GX22__3J{
zsJuH{LU>Z4U|V6sd##RC&niVn?3I&92*2#E@b$zJoH|~iQ0=iK6!4tIpi+IeF^W3{
z0*ziik`aDFE#kzhH>X2GN3o;Rk=+sZG5EN8j_@UIGiypnaOMSjD%rcHoqTz==2*y|!*F_v
zEuMHT@eqHsD4293t>j#zxs3Ly?P^Tax+bTl9}JG5#Xu0m;VrumEv2ep;m2>#&FlN|Ut=34nK|BYI1K!EDr
zA&}DqLRR0UE
zX!EqT&%*eSq4e69GP7^DnB8l)Tcs>Syc-~PPdwH%ve|ga4d4Uq1WsXOc?>i<&
zbfL4*^&wUA1Y@~wslLSaM)p@a+Kp;0tvv|)MJSu_OmBwxd73nPwBHV~_EiY+^jRB
z57I*@v$vBGqu#;xRCp5QEo4~z$W9neZS6={=e_@_)JJ1QUZZE`u|!vo;Q}|Aug8`d
z835TM%l410vh3w5GS}&+BwFB%2i$WP{0DCyG7b+}4*Z1Khc?2Qi<6^e?N`M~7h43E
z^zXXqu~56p&&-+S3Hx~i2PuhdmZ?K~Ukx!+9D}Q@lFaz6%KzXIcR->JAydeCnPC;<
zgn+Wd(d{|*BO24|q8E4SG&d1qeCU@#EAPnR@GDxlPqta0l#Zf6wEs^DLua30IMx(~
z!GUJhpLM&mdOnuN`(jsj9JoMt?DF*wMe0TB1&!jQ!A<5_pK-b|_6Un>iK=8sM>x#6
zu1@+;*^W2aGE^E^uPGeB&I0ME0`7gwH{Ctl2>+&1T>h5q;)QQ9x@8;RW(Mx>F}hbf
zwcjlh+Y??M_fDC_k8(rA$)j@zdvCzxgQMq&m)142@!gNlx1b~<(fCJB@MkyGa_EP(V_5wH7rWKYn8zKtaOe2L;3L+W
zx=v-Q(FTRG$=)%wf~D;BXtEqgR
z7aampkmbmmMU*(nHMY0*iHN`O)ahN#RV;F8WMvmfz_G1Zj}Ef{g`_k0ltG56H^lKbnnhSfA3&bR^UTM|Yxwv;ExataD?V2_S6|eieAU
zzc%7?EAP*^x5Ek9USJM3&n>yc&s%9rk4MoNQ(Ef9zW0OQ5dDJ}-=<^9BGcFX_DOr9
z!tO$9pom6NCk>F$!{{n=_a}Eeh3{+kpF|18;#W79Fuh7J=wGzi3GWIt5|GyO>w$fs
z%3}O73M<+Y@tRF61+^*X)Pb!snged%i@-HV$_pYTpBo44(~M)zL;>fd&4PRdiyw>&
zsIHI(wql|OF&lJg4=ommp~-3=+E@}JynI{7Ie8q%W^2YSyi|Ab3h<)%c@N4U4!!r;=0cy=E$LYwB8su!YmaMoi;k{b
zvWZ#<6B<9cde0O>JN`aNGO>hTJJrR{Z_@u%QWk(*vWlaaej|+@I7o~@1b2(=i*^}Uw^K?+s(M?C^6@in^Y`Z9
z_Qmw-*K=@#;|2%Wcg&?^Ay{R!V(I?aguz_)8?Mb6xlEU^tTJVcF5HQ~_gFRa0pYhd
zW|%!q&wr84T_QGbyV;be1xYOh1SQ=T{X253HldfHE+@{~xabG7P7U`%`}~7q4{v7b
zKB~usB2PqaXe<6e5Lg=I=5tmcPOav14&k)!%H}yD4XY@;4rqL7942h*w7MRrRN&IK
zyBJDZO)P$l%XvD>a;Jh!zvhhTk>K9*&n0HD)|!Z%dG51}`jW^f_O%6XbZmJ>F?3+l
zFH_PnNAV2tbh)>4VDszBj0hUeJ3o3m4s(od%;76SA3hsLtSs>EvyK1M*+||ZK?ihy
z)4>pr?0@7@1Eg
z0CqvhZqOn;%!gv6P(<_+)YXuj&W{RNxPPSMR`77bG|>nXEQXPs@i1G{Ix~b|F&1N~
z8#Rf6Ntn26r4w3_rki_LniNA
zj_^rkloVOXr$Uu|4+dN9=Fp}f4-O3wyO&ew0O7M4xDN}P0Fl+ldzXLx#z3g+`3Z(6
z<-~BFn7|}=PPP9xTnqAT#e(Rw0HST;CrVzY-geeS_ePCErRV~&U+?7N8j)vjoYf>4
zNC=OY1sJx~R9hM^hG(CS;PDL-nrb;7*^`oK$K|GrH1fDzV+$%s+m%fw7^jI=;&~*V
zZ^~%={+wP-E!U7osv$g|npOPL0d_F2{5y*7?6WQyK(Mj6H&o#XxWPQT1>Fd(+Z?oC
ziZe>&(Y_$OkAe;d3=4BMzY`0h%qQf$MWYr~Oo)C(i*!h%KrXO_il>f;RM$6-Y0ti;
zwzmFk6PJ+iH&0zIDJub*Wc*b%l@9jLEeWV`@&!19K-aLv*E>dsrvj|^Sc12GZY-bf
z8=nd^hjb{8zyY_^*lDGwej%UacMHK?0gT03Rg3oTMm3Mdvh>@4oIBM)xerOvDTU1I
zGUQ|<1MBx*4#z$Pk@g5LIfsVtW!kP-k4?dvQ~@`~3LgV~^-uTPQ`@VxmUuF+!#cJ2
z6aSLgVlctHrs1?*PA8qv~CytfLm2FA42`
zfEe&!B^Jg&h&-TQGV0_}W^JW1YScRL-M-Gw11Mph-II**aYDdEe!epb4+c{4o}
zUa`?yb^kR;Qa8nCFdO~4?9^7%YAwi|J+tyQ_T9>}o!v#5Ip~4EI5d+_8j`h^Hl?Kl
zF=DN}2%*OP#sh>yeyu5Fwgg~Qx-G4s>b|aYLWS2Fz@qcfs;vcWgr%I=x4Rf>sJP6L
zkwzU2PWib)r?eP67pJZFD#?NdmE&Lt7wFwr9t2{X7Jl2o_))%X=gVENpP0^~?u)u}
zUCiMQ6o?HctSB{7=^~ZC_hF7_ivr_IFV&O}gW0U=%s5n5U_ix>G7uN8*>+S*Ho;t|
zJO7*a9JDw8LlO~eUH}~(jq~R>`28)7f(~O`fHm3kF~s9Jdy5U{e?0MN#^iJY3ZGY+
zjSN#(MfGQ{v9UvLxr|T7Nm`jf?@}156-96fp>m$#%BPJWMoEGQx#xa^ga=EEr}O;&Dzapd2^AtC8yA~Ty1BE_J2}Gn-U1ml%5L*s3+u=>n?J@GnpcIH
zOSf5(-~Bi5MybL@%w_b~<5DG7tCXo}jCPVnz+y_$u8OH%7Y%j0D27<%3V3{HGf@3egG6FJ$*b<4Kar5S~Ka3NJmh+@y
zrF_8qe|_-Bv`gZ4-0K}&(Q>7}6Tq?1`n_Xdq6E=eCC!vD*+JC#$l7iEk}V=rW*4fj
zodSXU>+m}?`*r^z4JR4Gs?>?N%=5q*Rp_*lnxXv2~vR=n$VxFJHlWS&dE?G5fdD2zL=qpxWe&Vd-|Z|{Kq~?1Wv6(3$~C?
zX8ck~g%_X8t;hXgLG7JIHY9+k2rJ)OZs73rb?TL&@CUXLAl%aR8T8`|Q$;K_i-s+)
z&(v7d_;C*5*KxPB&lzZT3zfwjlm?SKxZh~hx87{f62>mGchm$P_YIS$U|UPzWOr1v_8SIKb73A-_vx!XkI-Pj&D14~54
zL08E(_S=j$;ik@nvlrk02w);{y1nxM*{EZ)czBIG4oRtG05)uiw7)
zf+)_i-Qg?QhpTv67@e`f99(j;c;QdiI0kG4^~pa%x~o-EggZ48M>JI^@YSe8sEs73
z)stnh`Y0>=Rxs?WixeHHKZHsdKlU@2#nl=hn4_=EBLQjYpucx|@^5j1f3IMekWz|8
z)l5mLA_3KgwDO6-^bPnnJmz75>;2n_>pl7qQLGhryA2R`ur{pIcF?NgHTCi@6$<&k
ziO$b>EMv~&{)RKWP0=}?GE}omDF%#mZvn5x`$+X9AqA}}P1}}~=+(BGvht{EJErj@
zLXLm)^MB%0!2B#bceTeK!Yr$~E>(;vySSbYd1jlu=_$DF5pPLDDiouSnOdD?)7c=a
zj&*~xRD2`w1}_7HMlj?mi8(sH9`)*?O)TL9*)sH8?07(&Q)|x1HF{s*bV7)HbYssB
ziy%39M6n?0T?&0;(k*Obl$SW%B=6I~fzeGops8vnriv_XK
zubxJ|di#}zFe!3@W+S(u+@~GX=YEhVltHD%RcBkWEotckZ+8xoZ?dG#utm|jQMOA@
zEmP2dReY6LLXcA3N3mRq`#}M;#e>1rCBbD~TJ@a7U&FKq6S+NtwGG&dq*Lx6ao)OP
z&FYFG_ssG35wr&FK)k#nOr-fVXK8I_)(t%a#I6$&ZURB;W2Dj{1EgN_L
zLE0i0iI8hv3(cIwuZi#-Np$l{WMT36uzX+wd%+b;Nho*1CIRp%&
z*WG~0!3aREn0erSA}#!NDD3umZl66ugeWTSC~uND6KW~5#vAXn#4C{HQy;O!7*5g>
zC|02M#2Aa^C+W650FoCoNGM5C%2BO~68qHSAfEwcTkr?%Mk}zhle?VnAVY6Y*DZs6
z_mLvyX>E!}8j>f=H(fPd+Q65c<;(8GKADAfTQpAKe8?>;e0x?FwdK`j7Pk946Et8S
z*0K9jiQ|8&dwV~_fO&T!bK4q_G3Rz%dlx*I^qmlnLP(7dZ-=g>$2KF)Ol%JENiQZ{
za-Z9e`@vrfS9E0j`5|1$fX(2M#c-`IvRg0<^j_KnI7qL|x=)?VElT-gQg{$&Z8z1N
zkHAyd5{%0YUga)72~IfBN9s-ZJa|32XGT0&Hi&#YsjWmDUadW~yVRJBq4`nak?rEA
zt=D53zn1{Dz>OIvV8uJR(@y1EzfzpSM}5nIC*;wPK|3*u$U3W+lB?Po6C=WvwIUUW*HN?E8B0C
z{ii+VangB%0^y3nCbgQ;F-NvVZHnZ2?v-@0R%7rQ{C_{-&|c^O@{e~{iIreRvqaed
z|8mHb@N)nctbPZBXP7?$2B2Y7?C%6Veu@u5J!Q1YezT*b77NYTn(pPMoi=QJLJjSn
z%)M+r6w)9>R6ge0d9iP!m!if7m>0#r;}(6DOQ~>gN9U2{D>)mxgPnxau%aY$xUP^E
zzpEl8QchF)?nrSbab_&
zKYchE%-(qWeX#tK8N8K70wKRyn~C8u@FF{`U@5QPd3XTEa)p0eJP>8z=jV4SD5eXM
zEV&(>g_S+N`EqWz0$HCuc_#0AVbgGUa!LDshD{j^1sU&22B(lz_FZj(s3!9ZTgjfY
z^$&}bLQ=oG`5_D~>x#o0Q@@eM98%<{BrvEYMPBD8KaPy-d(89alaR!p{>0
zr@#@xkBg%a9&5+gnPR0NJna*aElI_vww{ki>KX_RNc8aM;K^l=B2gtp
zKP0_8aq+t5y_%`DM3On}az^7y-MaqVBHh>h=ac6Dc*XDjQy%5w?Ja(Rq87bQB3>{X
zE4RqlE5ZIFQ?
z=RADfZ)BJvt4`bEd)}em&Xke=C{aifEwk9zt&3Bboa;q*4Y{Oke-72kWry;MfbAfVH`I*2p$?uwVd%3Fj?}6j#L0s)UB_+W4vWQCYFaoQsrls#_JYH1M{=)32?L<+M{h$2XCF*l*QN5gC+-5&UUXQH&TVVI-eimfmljZNi>czwlf}PUZ|82s9F;&cUG8d1PJZpyb>?M-A}Py
z8;rC2>r7JXJ-3M9se>i?1B~qO;aMU?58@tPI4h=~xi08_}^>DzD
z;R+m}l!8iu`l7)Q!$>jY*g62z%?g)NWUKsPb|I|tPwl#adYWeAWKKOZ@V-^Z@jr)G
z5%K)DRE-29LM!d-#JYXo$~A9svuIW6^6thp=nr1n4M(>Z9yycXIC#j>Zhw5gLz(`E
zJO7?ju|kV*q2+KD*R5UtZt4fF|04v^_q6bX@)EJURz`_bdGZ(8B|mW0yhHv386zc!8dVbj#hv}_GhznIW#eMn1ZpWAO9!nE37!t@1`W6(M
z6O}#qNYCbn?@*@FVKiKI>9yt9?94QK=?hc9w`A%eMY&;i!oc*$j%Xshcplr4IX^aj&qNO&{YkO3YDdm}L>4|AVW^
zGRyDB!}k(2<>$Rpl(sxh({p<&_pT_N$Md)i$-8ml-2c^Xgqp?
zGyVX;=zaxHgm+3Ak^%9}%azW=THE0tcjxz>?HzwKYvliX#KBkMr{G&qsD%R9`|p&t
zx_ol7-bpo3C
z7I`z!ZD#w2UA*Jp{}SQ)4*O9QbP)0KSFGqDvSz98TnoX%hge{C64fQzV{Iqr8XOda
z86F<@w?QZl0(wJH_Uu39J2rss1;P+zJgjxl2mrRMz~oseqc8kFyMIR!&@D=Y2vH>y
zen*bsqST+aFd>ckmH1xj&o`ddqcy4%V!_LDW1N8XcSR*~tFGa^t4uS_6lX%dM%~YC`K`?I9o>|F3yKUVx(b
z!p|H}**mZ{!4$dd8YP8bXE3aO6{sm@90$V0Kwn}}wT(fiXD`s{<9PuL*HGw=m;a-X
zc;b$V>L_qNwp+Pi;9d$)>VOHl!eXdU76wofI3>NwZ*O*w6&wJiT!P8cfrEcP1tHO7
z!Nm4~tLj1IYxbvl2E!V;VsUz6X=d;7(a3^qbkjEsE=-gPbdh6w>T(d*Mx}6j{j0SZ
z?GB2&U>o%8<@EyFCXhammYKR4$N9HMz1%&)d|!H_M4$FzL}B}cj~t>;K|K%0sf(`9
zjL4k|1@(NfuV{x*I7L=!xFm$-4p;JUh=2KhG6q>5In-X&DGW~6V-Z9nfT=2bwe~|v
zhe}nazBN)7QLBhhf2N?8jLPptS(>7K@{zoo+H~ezY}&26j#n$8!C<1R0Lo6~MxwJI
z)`JVzJYT=gxY=_B51=NdII86cLfQnGI`y}PF&Tmg7&i&5e~J`uD`*Dr0bQmh3F_?~
z#xc5ZVligHHABsJ@8tk8XcqwKK5zq+)-7wzIc}fY>}WXFPjBtCP*5#%gRf0+Xw&Op
zS3cRbw0G~fzwPmOnRS`A+f&tqg>^ODnE7X|n2u9_Bc#ZNahT=0{o1F5
z65clbQRDON
z&E~urw9{i$#E{GZaZ+7t4-)|{75CCd%-M+=E3q;^(#tD_rLz*xL;69s`}I~>YTJTI
z2R+FC?8)(aZtlh(C1Gy_dX)jwf}1}RSLk%)Duk
zx@xJ3B&SBiXk!+{?%9Z<)qy5Zpr?se9KprTTN^Q6LE-d0w?SI=JMWEK{#FGL
zyKfneAlrJu8>n5u?uO8%UG%<%n@0>?OOs|IXIDAVy;7StgfnDYX3+vTl`K}4vU0L(|I@e7q0XgiNOZCuc!^eBb%#lPx5x`;j2$;Brka;ybduOsws$=Sw@EVfuvp-Y0@tgj(A#(;e303NtiW;ixTOoQM>6~$17>yqA*vs^CaoV#13V6!O^;erYh*Nufd`8KuGQ){<
zUpYZb3Q0z!y#bW4eX@IOO0>J1-H>CyfupxeVTjtSi_v+wPU1Pf;?$yN(Tr|aTqS8?
z*_;KJuD6iv%N{FWqe=n{K)|o_KVG(dOQXI*sV*L}daG8WhHg;;Z6vi4p87o?paRd`
zS~U^8tLJ27Pme|`?+&4MVi(e2JMRuwVAYHN`bQ?ZpA&$|o0diPQW(IRcC8z=uOM>~
zODR>yV-$*ZIH<=&`-B#>fxX8$B&P4m)zj8)8W(4|(e;;Rn#o}8u6@uFqn;YQWQ%Pm
zauHEM80L0-z%#tzS05B>uZ!8EozA!G->KT`dM^8WaDt`Lcq97$KgQlFtj(?qyRG0w
zio4T7ixzjc;_mJcJh&Dq?(SOL-AQo|?#10*0s(g3fA8b}+v|{{Tn|rH=9+V?aVOB=
zsVSs{Sa?6~ZJLJ#iu@{=Zz=$032`(ifa1N+Th`_t(KR>r9o%D}
zRy!;KNb!};y;yI;CFIV7>lE^T=K>)ipCySsuswSY)n@rJt2N>OyC!sdDj1#+L=1Jz
zdRqkhSf)^kb&!fI$0iLR96bTKof?HBoIsHzUqYb&4Y(4?;E@qvwtV*sI(o;aoOR0Q
z(nIa2jnr~~XFbP)pkJR`Aic<6w+Az4*5^=rDJUZCZQahRN#;JWIX&1hJSv3?VsqQl
za*?+W>kq7Lh{s6m$8=hIQN{zOQhUP(Dmk=Pss2b5#8G{u`4FBI{gLccN5+7
zGOUA5L=1A{@Zn{?xR+Doe{+$~uz;mvz!HD{omrg-WG`IE`%W?)zyKw1p{1!G3gS+ES@^+z2J3Yf;tKtMUJ9&U_m=&R1Ml;2
zBF9l7d7K4cciFVR#Q#%ju4u!@HJ%ImGQHQ7{5KI971)QS`(jNbp~;oDU?k7%i(_zz
z8e}rmHr9~Sd^t722JE;jB3AKbSD^5OY3V9A1LXQ(lIx@hw(|_SE
zGT*hQ(tMQu5-0Nu-HsZKhR$1I6g~krgfRv0z@qv&%I8;Ww|>Z(=0^lm4pF4~U0s|J
z4Cy(sO6vf7ShVDu(NKyYLlO?b9Vd>!b#?0+NEeB67|4aNE78-eRksG)p9Lg|8-EaN
zRB)e#xcZA4e
zN?O|=3&?lnj!V%MWszB9c{+C@yX;qe+xq~
zv}?F%M*jIdM#LizJSBB%HE646*}C&@aC9yKs)^+7RA?dhpKJ(DazAb~>)K($EXii-
zH2gfwk1r0{S6vuf|5nYUjjc1#%NWCDhG*cCJurb3L?rOHd0qPS(YkpOL_t^ag2@Te
zASJb7JJD55-<Y8f`;bR7ZpY`
zHy{%cj1}3q2jd9cukop@{2D>`=+^c$txrmh-aBkupoUM|2SI<>rhzQV_HS>eFV=})
z+AY-QBKa0h2n-cJ$>pD`1t&9A^xw9zL&@FFD;{$y`b-`V5d-1AOptdNpgSOyHJJc
z*2!2Ke7q_w7rtp}KFvo$-bvI2&TK4a?huP!w~XduHJ1`~t8M@mVW)aLHRWo6@Hr@*
zl$8}iwDJ-D;;I|b>_oZSH1W3C$G7_BkGPM5^#53gpRp^?#S-jEGjz9ad;||#K+gga
zJeF>(B~CrxetL>V9mDtZ-iZG(o2vWho1}0+K%A$3@kVnSF~e9^aK~ly@|#4k5;t7>
zM*!SspZpk1H@@xbu(K1ZonwKhK~K#>K!omaXV;|tftcI4`eEsX_wFtC-8}+WM(+8$
zd_qI4#DX8Uq+Bdpf#e-qc|#AIemf(_CQHrEt-WgEt5VTXaB!^Xp{>lf#%Uwpnse)D
zSf~(ueZw1;51X%+5Li77D{53!_r71gh=WK>77#Rytfo_*$;`FK<=f}hP+WbY=w>uD`JO7UDdo&fR)5}>OBZkKOXME%4TPU+a3h9c
zjK$)ocFMTrC(cX`n8Viw^a-vt7oWeU@{8(yk{+oGQ8EHdgueIX29hxEHDTe}XWvp>
zMy$cb1>+L`3r2f;efjmym)ONm(*E_m#Xa%$UT!T2k~&ub>Oc3#u*djVaFd?{tPrVe
zU-q^o#N1}lQYf2upF8&c?is}*CL7|02jI}r&HAA>A9JTO92qzO$yMfmL3@HKZneB&
zcfQK*J|U{9B8vTSwZahRI8a)l2ky7g3h91SFS53Y1pLkKlwn_3^yc4s6Su0YbxWbb
zS3v3Whh|DaM{eNXpcrH~hR}Jw*Bg+}(A#4l$6|-C2h|RMQplg^d4Skd$8)=4VP$o%
zNAgDJ4?^K2-5xHXVy$y@DO!Xlc_`d($;IzK+b=tF`^2@gkkbA7`QLzekX(~nta!A|
zJ{FQ!z)cN|A(_6Cr1AVofzxwjUq;RvEQvq9XHi
z1R&bI0g6Plfyd&vIvo;$yfx=%nO0iP;C$ZR;A@`^LnH{{N|3}pjzkaa9ffhcT~~;a
zt-_~hDfrSg(8@yi-1GgiUn!MhQ2ro}c^tA8)0p@P!M#j?UE2>z$!|}b*lU2Fw|-r(
zkd>)5C#lflCOEtB!II(byytyO`!~UtmnBT(6CYx9y)lZh^lY5*u`I8g2JMYn
z9f#6;KF&-u(94>`e}ow12EX&^oG(Yn+p7faTB-jTpB$wKtpO5Pz`(72?%BD1_3G6(
zpwLhcys!Ri>{!tE8bB+;r7_Ir;iYp)3*y~T^miSM9AX>or2udBPa3~qm1&mX9|&6~
zOMIKew3t!ofvS9*
zkO#NyeeJjR?Ofj)f-%akUsO
zzB|aZ!c(F}lPgO#=s64B~Fgfk1C2L
zG})^YtbwC)$1%9Wp~mNVMuRp7ZcP4?Ce0yD1u<)4aCFo5o$MT7kP+!ekdr*73>ueY
zDXz*&`ApaY=Q_^T#WiS8w(dWW3$hUom5Aa%Eahn_DFYV*Y!}L5l8qqR%x^gGD5P{)
zDJ%K`^;R;CqjmLC-=cq%yhB3AmBxMTpfGXb?)1}^HMys*K!Fof5Gj5Z#
z85F3!4xD@qfdymHv1m&wCyiSGYcg6_RW9gjc%K7ygL6M^_x(I>FRtUH6TVntP94XW
z9W~iUwtontA9GHjCGU0n-lFRxwjVlw?4oWM9R`Z!SYQ8R$;GW)
z1U6@)b!2P$tzSHxQo|qqzcj#(?ueAkLE1JUFW*2?O)0*=wCfH2(gdhS!{KwsjSJ^l
z@}@WbzF&#}zfM1PnK+KdJ~uFi4J2BBp~T;N#wf3x&BFnEK%-rwk*h%L2z+4@t6O!<@wNQ3F>1hoZC!iZ
zoZZf4o%9b9nN{i&AtnUZW(vfZDLRc%?8e$0VnC=fd&Mt7qS21>x8pvZn!SW^$z_
zY_tJ`gnD@u*KLP9gc4#GMAi~CTJ1XileQm?Jqj>8V9f{AqmB5Ir9OY{i
zR?LSiLrv64a1uL@c^ow#qHPYj2$C>L`ssxtB??)H>gy>c$&4d-Ahu%`4JIAh%4aR7
z?*AF@VI8-~2Spy3g5_woB*`3Nya&-!BCbK6B*o7ZI4OR|YE!a2
z-`nk!#pDc2&5(ljRABqPn2gLeBv7%jEA;lbW-yEe&NLmPf55sj)dJif+vgTH70PiN
zjUqaN^GTL!zp9=YQ;;~|x7_!!ocmfO@^XsC)V$I7dQt&N30d0RY@IcNec29hbBO8H
zG)hUA@-9!euVf{5+snFG<~HB;bc9Wj2lc%$aCj<8iVUjbDOQ;h(F~oVjh23`*+|(i
z-9rC4`~8)*)BUsC@z~2XaB-sWZpOe{c;_w8@A^H)z{Ne2!bKmortvG_F7z3$6XHk{
zk7#GFaQ0V0KT=TfM;AuUg)`4-KVlfsi&-ljX1ERIse`AT1-|I_G^z)5wVl3VKg~=(
z|30$}{u`Lr3(=`G1zM@z;fXis!7+_jRKa;f)XNeV9HhBAnV7lPvLKP9p~;21^Cue=
zLl8u1A6{;D?ZB^(cIydA^UW*#bAlmGB}hy2Kb8-U34BShHMcfXoxHpqsSe-+;?r=$
zt9!)~Y5*N+D9_{H;UlL*cEaaR(fYN#sntskp1X!P&(s4Cd}PKr8>&q9sg
z3`;wD73Y<3rv69chN@wnKt`h2JFfkAV9_=_ju$}?L
zHC@UebIkx#$&XHqV}o0CGm7_>d=GV{WNFZ=yGCad#QER<{Fcw~slJnf9&dLq#I~V$
zrLqj5$h&g(085nx!h*_RG~Itg!PC=IjqCUs_+=tAoud$eVn+C+{8
zw3XuoUeT6EK8+_D_9YtT`@P{iUR@Xg!rwcp~9vBs~q|dCt+JBu!=3jYfdK
z+sH}z&HkgvtzRV6HA?M*rXwV(A%!JCgY0BzkY))qbR+x9|9R2j@xsr0WJBM6c_C+W
z72x3S$=|rz6wIgWdlD~Pdz$rT{+AkwoIwW{=Lw$eS*j93Pqa)WQnpOI9N7C|YY(B|
z7p;}ofKe4Itcr4EeWo?r|0W7Aa;NsEq^J37lcx9GVkHt@59+B$G?Zlc!0_{+RaAPs
zIib07{Tf~+mjTZLFFq2LUp2>|$7(;?kSAs+N>>5nH}ScZ2z~+ouim
z6=wY(3*={mT?BN~t=Dl_x}X^+s0wY&lO`n=}AU!`;&CO8|!;j&#jD<~(OHglc2uP;1b-C@AY
zan3;BiQ{?t96{f~qa$PqWgjz$AP0MK8c=0Cec~Z&d@(E8o5>AVwkfhTQ^l`aZ&iiA2nh`>T~JG6+`K)!=109fh|1Oaoxql5U|PkK
z_O^rFy&~RLf75x|`xsiB)N8G@tFDgi46j!*rS@DaDdKyXmEZDN3tNWolM=1A@|7i(
zcX=i#+%fcO%6n=s&TkM`-QMeeLm$3-khQLjoyGhMlBd}QivHhMFEc@%ml@R4=VA%S
z*jZ5TF09Xhr)Rj~=R6S^Q%_P^6vx*tnl5!j`bDcqR_}<;2HCB${#;{)KL`&l%=>mS_je95I
zXfM0x7paQB>GxZit&!pV6wY7NLIsxZh#VL(j|TCQWqDK5idq_@Z~|Zhm(S5;j0m?w&#>yyW%IobFSwIX+NLWFW3R?|)vu
z@&o-JMGxB88uC~o_NcQ$%G#v-95xt#*R#jA*{I8)P8)_V3M_C^&MkWS^!K6Tr@PTF
zvUvN1zuDZ#2=ZWB
zIJ@YRmt^;&sT?H22STe>;V$(+2VUijm7J3FpXk%-dx^@bzw+X2IvKY5i5Q5*7o8K7*qTcUCs=p
z54vyo4n^WKYcb(v2VN}(SJLr}0r4U;8qQKO&vM4khe7l#%!YLg3~l$%^h|NHCNmA_
zf6ORK=cFPIQ5vuM&GS|>9Ur^CRgu!ImCf*-K$A)YjImcm449hsoA`=W}8Qe%wJ0mAq&$TDB-aEo-ot^XVLC7xe<<(~^wBa{+%pVFZV@
z2GoANjg)6ok)gnzAZ|4fGhANGH!*j`e!!TMg4%7@nIhH~UeX7B(-p^yHlYt@pAY6N
z?0jSrf+4P*$|UkgSe!@6tzgOPP&Y_Z7M*`>tcFqw{6zIVqBWel87b$Ew|DDW<^v!{
z?C#)7*k#E6L5oaqccbOm&4GDkGEI!x}Hiq>}EEUPolJvv_*?dkZHh8Mbg)0gA$%=rwUq$tlTXm7E
zd$fhE{X+FbcZ-aw4|rhw3_K2-x7}$=fZh2{`TJu`=6IMC&@`$Z6)L4
z>JUeLpa)LaemfA7K6&m8$u|@wj&KP3_XAo^Y&4vp#cjA9l?VcA8S}x|laZ{8OQ^#@>$^
z9dqV-WTW^MfL&J~P4_;grk?3tKK?Snl-m
zOSS6abIN?K=}d<-1tlfjo)?$nm0b^=h$T^#Ws)b@Rzn4hZ@U0+Tjg-2$h>&))~LY6
z$A|T0uL5D}ASW*`FMe#9!UpWyYsKjS;5CF?rRFWc<_>=@6C`npz=-0%B%;VEIaxNZ
zO4J<$)wFuSDUo5t*(NG)@8wzj)}=AWjagnUN<1-65XC9-qmH&Z0%#DRMz6;yhNr01
zpt`%ubG*C}ME?50?@_iDN?N`W5+Jf8hj%wt#>g3mgw}u`VZEmZsOH-}>D$YNL+jG%<@X^&@R^tJ
zeOqC7(lvE1-EUWf9h^b!BtU8Qz!7Td5-iSOb}ND4OWpJZMHWN#kI)(B&?Mj@e@>mrt-qMC5-#FSl1Ewg{dLZ2shplmf3II}YyP)2m^f{7V@9#qA(IR1`4b+}5u%=GD}z6N=@pB*Opo=b>d)5!RIl^n&_Iv5Lpx(;Kiy
zQOu1@>_g}X)i!RPdK!M0-;E=%kh;}fF-YT88$ZmRC=xR=4>LRe8XJXer&3F09M6`E
z-tC0A{4Aoqy?EuKt*z|{-PrT~lxJ)JDy!MQrnG{c$JXbg0jy8EIysQ-*HG)=SX_j)
z8FA4mxia|2hUFG8TQiEUBJNkP$E$$dua%;WN8qnN22sO%sX3;85^Hoa?n)Z3T%pD|
zi1FiGy^iKb99&%*l90#MVkUJCR6+hKLh!L1T
zS`)4EEYZzZbpC_IlNAsDp!13L0qEgyyJhho|0^Wm&LIEeidxuc?ovT5%hRbdC62=s
zu_N{`2pwto!RMT!N?BZi+i{#1&s|sHv9Z6%5D%VeD;56t0%7{M+KG^ruA&5xdCE(K
z#QlMpMaYaXj&g_`{x8W=r|pp+0zXSNbKmp7XKX#9uTf-+_mZtsqAOM#Z;yXc#fK(}
ze3MX4f~i~$l3=7lm;l@k8}JTvTc0+vbXJ@H2o0d`hVLzEe;#7Fz??#G9Z;S
z%~K(v&8WIBT}Ym}>_f;Hp_?N*A{_eGIJ{bCew}A$sD#Vq=wf1k9T?FN+6#-gluVDB
zv69P4#Qii{U}m^``apc_JJ7-9ptYd~1Oof+?xI^nz~HorQDy^dcafv0f^j;Tcll)i
zjT1td1G5uiw+;aXs4;;;B)RuHr@+3zLvW5V*pqmu=@?9u_)Uky3(2gu$85y_1M*%;
z5dF7Mo&;b%Ivjaux;mG{D0@mP-zLuqu7fT3nTMef3*oVCl4oc6v4s9Yao*Ej_C|HZ
zWoVm_Eqmw_5(kvDAz}5(Aiz>G6nfkTdYANDNUyez^AteRebwDKwDjm?VE+v0mzvXd
z|1~AH(ku0cEB?i4)Ue$=!~cr;bpZ4jc=Y9ebYy6t<18AwSeQ{)qY0T4G4)ybO`=(H
z?dvYxVx+`cy>Ib-T{hvEJ&3d#8}WJUTw!(oOrr$rJ$j^{W4*svC@v}Y1PJ+c%W{=U
z#0E2U0xhMG&T)wlMp}|2IL24!9+oPOf8DZDh(W}Y_F2IkRcREi$&z%%hl1I}No9JNL!+6FHHo#+CepX?!IX@0{pgiYfX(R{
zaEveW?OJ${F`{>p0}E-BRV_!^4&BX91HwDMqUC#J4G
zr~bPs`T@eR%xJg8lEW|tV;u_^I@-Fv$op9r<(hg(AOA>JOwvTN4KHMDM2Yw*1DX81
zk0k<-*WMo=4dmyt+FOm$tg>6O=lZ8-*kAz5av`eXfFaFIf;x%?IN$jw=|Qx)zM-77
zf`XP#tbOL7-aY?_u{Xi;$DB@nF+g+^-T{Nmi2?_{kp{T)%-pBDamLpl`C=V*{-=q<
z7r4?sVaoN3(ab|*Aj{8zq9{yK<_58Iw|ulzR+z%_itkOkJH%47f5)zzxFgtE=0eV#-3Hw)(<7tyf8Y<$hs!^>j?g)q$NM
zaCnBNdQoS|Jmn9?(0=X7RZ04yawZ06>aFz)wGfYcS+$Kz%l^aF{N}%iVy~bo%>cPP
z$8EZ6%}%XXau2`j+#h4FWD0KF$kt&fH`?4D(wK)iy>2Z#6RWZ=b-R#4w(HD|g7sJ3
z+w+wcKv7-Eur}u6?P4w+Cx-}vYcgTpHlPz5m#Z7EDR*A-{|!7Fss+71R1!H3JN%gN
zn||a?$_YK($PWG(&Mwom1FOhJGJbPsSf*GzKiYMfH(7%pCq}I|AT50w+-Ue>-eD`iTyD|k(WuX
zj_)G=*E_I?UA|4QFEzV;JHu6KxLtNNISD^kz}5P4&kM$5dW1g|2!J{*DYrh3sp~;v;?=^qLdx(GD40`Gl--vW*LcD5x-<`BP
zqv(|DJV*HFFMG5riN%w*cUZjtgS*S1?hX6&jO0xR*40ropDMwu`m-FLLTC`vN++7h
zg@RD>rTSJDY;_&m;eYm)M0>Ma$CSUg3A=$^#mhj?XBS#o
zS+&E?l{UEj#YjD`Vh2WeVYWF|)9V)_ePX>Qad2xq_u}Y+>rDUxCO49~Mw@YJsUlA1
zs9E*w$t2>qGp`}q0OXRABd;u#TVnrQ744W=B`j{bmj11sqj~qvG=qwSW9lU7ly+c$
zboK}pVmT*#slL%a3
z&BvINiy5TNv~*!KvG?A!=lPC$y@ssL
z!MWLM5DXQt&)MZ4%W=HEB`hgX6o5={(Cw)+>h?{q?xR?}6Dv;1N_+dIBInZMKKF8aSIATw$|D-&>aEzmxE_Ol<@WuINVuulkDGrOCECtP
z5Ab|4CH69@ImPbB$ugK{$tIT)KaVS~YMP0FU-qkoW+VklyIrfm8`v;IG+#}d^$E}t
z9>k)<;ed;y7Vh9nifHM>^j|d!D^2~NS5oWUzOeC_2w>nEmTx=>d%W;nt_wP(`G%qd
zCm>(qUDX``jX>;HW}^hn=e&{LCCfrFAt=&fj001tO1a}@#6WDcq_!FX^2@(742oJC
z$!0n&JlZr!%-Vj$%Tt0^hegAh3g|UpeuBec13EmWx#9G>4NpruR?`f!u1cS^!RZrIKd$v+n1ZukDfA8gvHM)>9O{P&E^6{nBe{hxqy6yJeks??zIremM0
zb9Fc1BUd#yNa^strK9rR$pzp(FT`9JAkizUWcM<_l^th}P}QS>*3RGro~b!3rqJ&U
z>beec$BOl^G3p@6mK9CMP>Blr7gRg_@;YPuZ|%j2B+*kqykl|}^pHCkBi(-x{J+*u
zpD0tO6Pot4$i)n7W(WHwe
zpc?Bhp~`JK2}5a^+$!BcbO@ev>!O$#=zjQuf{qDqf^Je^!%wz;Z&$!6{Y?nc-PsWGTPgQ069~;zXrG
znEy^le}v8(*70^w9P6X7C|x6&Wvak>U698BGtbaBucE1|8&}a{Sqp$gWLb`fq>LP^
zUS6vTRWY!xb~#s#GBf^b7J7BM
zI5(#P^e{XBeNoz+A0E(!bIc?C$~In)nmswShozcRyINW51}J~|Pch0#tQgka-o6ci
zC3bi}CG9!$lyxs92&?z2C^o?vnmJ76sw-zC$n(Et+pJ5ek-SQ;7QAWSg>$!zzf4i;_SeB2j
zu5xGKBKpx@!G>&_$r(p1VS&S>s+OK#Mr3Qs_ksO*HZhVckkk4kIJ)VSSZgEV6;F^CZ_~Wg`UK9*Lvv0ZENo?`}*K4|JoQW`Ag&@IgX9x+)00ITLg1Pa-^iY%o
z!-&o+;B;%&`XG7{%c|arhV(iLIoMW}v;tfjwdi{8+Y2S+7dwach4Ox7PaO_A56BN+
zZeC>jVRwyI9sjnx1cnX4RpDqLYrMlp$01T6#OOb;a@xs0eE#E4dnW*m;94r-n(nLj
zNG-Cv^yKVCnfXL3&|ccOU|o&vBBSGONM
z3uZ1h*8f<6GIb4DrQ;DyT(K$)neQrUrbh
zWe$BvXlGpJHbiuI4g;#cb)%D37L$0=n)0OV)5XL<$*{q+Kv$APg;!IPXT-r5#+|TM
z>0Vml(~7bW2O^cf*gYibP+dZdoJ>Am$mI{e3kvQd6(>=!Q6!(^Ngth7Nhp*0C0Wr)
z{4t9?fMhK}JDFCgg+>dm$7_IXEul~dbtq#Z)mA~~TSWcs?t%8tw)8rKn&or1rFQfP
z@8I^+pOaMu4f%t!r=qQ4NNau$H3KL;8(3=M($)XDT$eB0U^ocg7&fdv($3k)T!JMn
zc;>)LoF!X}|3jpp$^Rkpl5;%-3|_9e<9*O+`o8J=5=N(JjrE(8@MZv2ANIZ~ELAg4
zXTnA;=$6DVj8SXtpCc@Ga2~?LY}ma!T_}}g&Ql5$!i%JRPcLC0NXblDV^ePu*~Bs5
z-P;EHz7!_K9l);;DDq2LiT@&U3Pj%>1I}S(^vXWaw*COX!G5J3ux6&%1_6%
zc7j!gZNSHTxCzX}{Mo_}!0RnC9
zubF17eOZRqq-TS<>EDniQ{SW%#@-lZ7#;1S!-RGVqgx{H^Aw&0zk-c35de%UKmCz@
zVW}xy$vxgqt;8J=6{1Ctw0~p`;mk+%bA>)Gc-Rib*T$XT*cFQ5Cc(5FwM^lI*?1c>
zTZD|_!R^&;u$_7$ldIG3C1Wg7Hlw(JP8OECrj?UbtM}Npz^v`NhK8Ykq{Z9+aWgev
zl-5V5@l5w~MxK|0=mt;TmgFB@2ThOXxq`ZN3-umnTr^QDF1)@RJ+5&JLi5+ON`47k
z!0l6l1!$0y<6^yg>*+WGCIY#+?zr&q^aB?*s74e#)~h=!TEXZ5EK}4y+d+#4;|A@9
z4VY|pFcx3N5lMkWQclaX31;W)|4k_q>iX+gdd6u|<;-7f0SKzWy6+(?SMs;L*x}r+
zjuhO$C+BDK`5Uz0nMtSN={B<>>r^$k196NsuGPe`!-;-~L)fCV_p@pf(&^G71=XAr
z+v7OOrDR0`duhl>dbyJqgSd;uqE8-S;Y{BT`g+%1umr`UpG7RT>%7PepW(1CeCKl!
zx6oH&{aXcO#1wZGqq$^ENvi(nx(PYx&x#NFdQ)f-4|Q?k4pHy2=X>(X>59y}NiWSS
zqpSpptKlf930+7+u=gIuSPPxNlAL9VicRISl=`^j30qu;?Op9XNOX3u_71{9`F6GU
z+1x&rRUdgOt42J34)_xj6a!BQne`aE!m;*2rZ)zQkH$dpv>l6E9(@`y^>+e!J*8{|MI0}x-HJ(Vpt~CK-N%|
zQY4%tl$Z8(cv?;}Mgp-npXI2U<1<-;;7UBjw1_rhOn#v^uNNo!;vQxr;hgjE(hciEK4vUdJ7DVkSF>EH3A4IY+6h`>3i3cR-NR-)cpF?IembS4uw2aN^SK-P8;ZmOqO?_#;P-q+
z?0pC9efwarK{tM-!
zKf(xx`8jwKng*JfD5M^_k7xB;QpkuwGYoU
z<`_xxT>4k~TtC#6J^Q_&Su~`nM`vteLSD+FyS3E_v{9J6
zy}KI>^N6R6eeqO&bkf{9h;_`9j9hO%1o?wkex779e-cU!HeIIqTuQ~~Y^3Y7
z-RezP6-w#q&u}XCJnOst5?3lH!AEqeI!Jzw?YVi>?0IFU3-P7g>0J-rNc4EzpU@{Y
zKmf%E0xuH=H!g-3>sEiPK4?6%tg~ZC`9$Q2$63i;U`a@V^
zd%R`~hROCm6>VVWGFf(}oME6u^Br<;7b>G|j(O6ojd-c*`^~Af&r;vSD|n{ucV*ns
zQb^JM|7a(5Ib}AC{unr1ww1KgMWlYafbA|P$jd_;5CM6f50Yd_Kla+b@V4-0ra=Ei
zFf}p_V)1i?zUFpu?Z5x=C@qNK+5X~|s$l5G;S!O!F49;P`B7!GPI{F=GNz-7dCn}G
zety2M4k~B4&ewiiy^g4TSx0h}d7Pf!#kRpcY{z~1G0~X+2Qu~dM`NP8*|VkCGp%)O
zEWrmm7?bFwcw``bZ5@B}hsK}$fp?e4JJ!X%Jf5QDM6M{`D{WC=maLrmWtN`Qi
zoV$%$x-D9?RnOcjHRxgY>ni5UsfC4LUde_Aw)*i!PiTw|yqziP*Z*=qA-3np5%SRu
zf|J@QA4TQS0+8mg2$L7q_>^Q4%@Isl{iWqgrm7Hk4^}C_lnYeRHzoBrFpIR>4XXC;
zZVCL}Gbv=AtgB%PiD|Z#p;W*5o@#w&gjG?CHJzG1O7mB>t$SDu?bxeJqH|^=_R>O(
zOmI53!Zj-2m{sF(_6Ue2>rvZScta=eic3tDeTS%8|p(#Z3XL
zL5d<#`rSAtpgcR8!+zI
zN?ssoNYgGfzbNSC#wSBX;+som@HPJu*x+hZrPKkVsJ(h`mi&HE9@xwMB0KFMp%!8L
zw;^(#ekNwWEw!b^t7_iO=sz30zty)xaop_fewr_zkTdV{sC#lUJfWP*OEkp&Z!Wb}
z=+?BrYpNp)72l3taoyiY%#)=_OY$P0e)nZot#C8j<3~9@)4Ax+Lt66HkCNHf-GpSh
zr_1}+&ceK$bOW4ms1BsT4)heL=*E+>j&uhYve&dBgP4!5Ub0^Iu@6DA9q7@qnS`A~
z;oG-%n!Otg=;|3e3+w5g(?84gGcCny&(ZTTv=XI4F_6VruPEENdEwlSh3lvjRsJ~n
z(yTi@GMh?${r|B&Ng2zz*w?(DbA`<;TF+uCGC6J_SVcX}osr0(%sNmq?Wb50L_QG3
ze7qClcV}gOg^)!AFkm`*Ga@QM3}qEZyKl|ef3!+=#UGY4zFeZt5M7ZGC
zqe;@eFA}0Y$xN!hOzE~XYc5>N0H3-8GP2<(p9U8sT+wJ1Hnob)O6QvWO1K`A<$^z=
zG-BU^R1ipUM)|L_rx=}hoVjk%BZw3S?ws;}L)lLC%W>zd_>{bk6
zv=f#@w>}CvsY&KPmBGvQ3u>_V&MsK}88Kload*e@|3#3&prRkSlO4Pl(jh%tOFReR
zr4k|ryRb*xv|(zu6AoVC^C!OgJ|%-V?@`5HHst?Dy^X5BLZ`q*D(0TUaW#+Pg9)h&
zG%P4^knZ2lnDI|q;J7`!8^1^cO(S@9z9hUFo2t$?&y&Wpv7!;@XfW$>WpSB}?b-A3
zV%|Ga=!z2kHt)`lM&uWEKWpkiL2yHAmtlNq_1{tp~
zZO?J>BVlyX{yd+j7GslR+M+r4{E(Fe{idk+nnpV&Bvm10+x9`dP;F57XM?3sWnraISPw4|4hptKy#gQi)+gb7mcd&y|5b8f7Oe*PcnXuIgZG$0F_#eFudv$H
zv)1nJ{ks!#&zC0{cB83v0=tX1cZ6Ir_T^>(cJBqseJ-r$p#Ls=z=`-9YtwTs@n8qe
zOSyJCrfNI+{O=>TZ%Aydv^*HS-W_mdwFNIehG6YoV7-rm22&}g=G-c-eAmwPHEiEU
zOVv!(GX!n#K|YZQc#Yd?_$9k#EMyd9<;#a~A|w))`|fE9=(uyNuCSj4e47DN_6XXH
zI2)|7-3nXdijmT>9eUW6E-)l0&&#RL&sn+K!a*}62b<~8YpK{560?a<9%k*oB>L_1
z`<6TA@~$b~{bME_Dy`r6{`EnFT&s0tQ;JJR=mGsypL=NlByc*-W6`KxjV>5$RJGrE
ze&HHygovey>bGDWuy`KHo%WNw0HH~G>4fo1esE_kUbVAqlsz+oMwQG&tmV+MSV_Uygv!~0oyL-7
z{=ccj=!5d&^P&pXcNv~=H*an^B;r$dEeZcSeRml*t=BG}!BPU0thw5s3FmTT(IaW{I*L@R?^a#`L3NYbYdAqkyns$j*XN{6MY
zno`&is{uAwX;`!Oe0vpuxs9@Nb8mJ>uq_(WHCah8CzBTp)|3DdofKoN`_9rFa_BOig3at{~yNABCL*R>$VXjxD#B1ySrP0
z6Wk%VyK^AHU4j#w;2zwAyE_NBgB{%URqp%y_crL#fUkDdUUST`>P=);{`&q6zt=}O
zd7U+lcbmh97Cf(d>+L6Ge;d9J)|{?v$01Gl1X?E5V&
zdtF9~%8vr+)jd{74MjeP=+La;W~VoCd+)8?Qs2C9Wo_*9I8HO+DPmPAZmQ^;j9^`x
zYQIFw%BCLnT_BHbgo+9BlIF6l-5L#*_{tZQC5of+{}r+e?SfJ|Y?=s68chm#zE@cu
zpr_JF1d3$*MKWE>!&EwIO~xd7cZrvutW1M%Jl^tXI21yQ$OA4Vkd}(GJnM5qFHBCdf#p7N0&{biz@qKw60F4>uY>Rx8qgD)(|t8#=(uEszdE6jY!aXQT~rkckr^QQ=eNuHcH!#b
zKkaqE3i5BKYP^b>elZLeChhlH*<=>X|ILu{wdR)tVJXY#VpVBXO76GbYPUWp6UF`{
z)>OY&!XU;mw?%oNIT7Csi6B>t(=UmgiMQ5UnN+^tV4Nf8VbkALjEr|Rx}97NkMqf9
zeoF)vt?WkhSFqB{{F1TUP5M;eckJ`MJ!Uae5msWME_WjnxkWuF=cT;;5YJ~>$T?F0xiY2ede#lQOWR&|e%F1A2
z%VRVDiwg|JwfPmux1sx8oIgPBNs&X{q@qfPI>
zv7=qYT=mSw#za-!52rws-u|-EAd@VZUBCgN;D!AAzJY=ppYFC}+x%a%pze`jCT!U*
z?%v%%fc}kJ<`=Ru??>AT43!~|BtItD^^nXn)$JpG-rYf?Ta2&~n$kHq&DFOIq8_wp
z-&LmLYl5;q%{5SkiVPMBS$-P9ZqC}iu)yB^x$55M2kP_l>2YuE;!YlO@9BF)GRVkw
z@p}mX?z&f+mSt8Isbl7Vhe>~D?WM%Jy!LOLBOca>4)}`=N@>-|iCx5<6XP+dSKq#U
zU^w>$vuuI1$@00%aqav!+Z4fS?FRFx_XG@_F(*Y-LS;^yUQM1J
zgk`IeLWmDq1JQ3?5Lz6f6@wF|LML{8d`BD48ruA+J0`>)wmDBO0-st+un;;;X|ZK3
zOuc`6O{2kbjk%|e8i31Zue-6X(qL4i|C=|vl8~N6T2^svxe_BkF*s&L#vZd;pqrOZ
z!*3(+A@`}c_9ehuzHC3?BW!J+DG)WRRsilQ1tzU-t*W;4s)ebWLs6z@a_amy9|#!W
zRB}4cTcW^$^(bIF34P7uG9kcf?pu??XpwsCS6S3=I{Fdc%Ro7iBci)ZMDxdQFh8t+
z^rw9`jImqlkF=l{12AghAN|%hA6;?55Ig_4C=Kjg9Xi=Z~WYL==chXvqQ0yXL}
z^PaJ>eeHy#U-IcY~Ovok0Xn*B*Sxrj&
zysMisO(QR3R981S>Q6`zC-y(M$xo!=`g~;jigR`j6YNGH)idNfk$)d^zm(N%^@L|8
zON$L)fn>sZ#j*L>C~XxLzDOP@zo0Qt`_QQfXY)Z9_*tiYpPiae3dfyHq)(Qw_D6-e
zPDXvPMOksrIBI7scrB{2npzg^7aXE^a%yCH^6dq8WU%-g8y#=MJ^*LtzffpzjBxGg
zj&gyAC+tJ*Qq->V_|VYLKH&Ojv1acQU=z-&;x#b81n-EA<;S*f_kqh=;(3aHP4PO}
zI)svFF|dbLCZsRda@eilkxdpn
zp=q-gGV14hoj8E{wyYa3u8kDEMBnUVx1_2@a?Z9!^MDTeotRa#xqd>N^=EGUD`~3Y~
zDM9u$8FH-tv@4^l)m+;40QlO$heg9%W9pRqWS>C1EuS=d*eHs80vTK?ZN)L4YR(Ez
z)OW`_Ikmc<6U8grS>=&Y>PHjw)>v>@v56&Lnz`q>yDJ~fzsK;;#D5=HbhF?FG?5<^
zyD5t}K14Z=B3x+Q&5QXp>s{c4CP}HpyKFLXnr~koZ&B$zV%MpJj)F8z$_yj
z94w(hU6AI~j?2xS)H)E8QdFp+zY*iC;aCkS4RJO8tvQ?PF+xE5CHByY{s>-J4wG+Z
zSlNVLNz7G}bl%dqx`cS(sF1u>ftc9*J)Ykz0kQ=3)Qsb2%uq)L-jC7e0wg
zBblTLG!#c960Y)sS;N06k(=0GjO;4Oo&1+@wmv-lqw)r2gf+hs$SNr4`uRl!+{th1ZhM@v`6K*!T3C7
zT))O}ONc2pxH1S61Ed4i3>bCkHxh4lET{kwp>Nnx^q~LJA#QCpCUaf~c(%`GX72_r1JSEckb6q?fRuA5KD+J2<1y?w_jd?@oYFGeS&kYdh0r_@c}H}gGOwMLd~mnXCmBc%```$&
zI$e5Z{|TWXHBEcK>_#x{IJ{qa!*TOL&RSyK3t1h4xL&+OIx=I(I28cr|WUir@ma>(}a9QOfaJ@0vR*WxaMo%Ol
z5jC%qZIZ2`#yYA0i>5xYi&4^3lg(;{i;4t8iLE8nU
zTG1rf<7esvhdPq$gns}#D9;mf!$lR;=Iyc`v*oHgl3y$I+wd0y6`rlk_heCu+HX(s
zj9bt-d4v|)$$O-T`v7m3A$y;|<;og`vFkb5vR^i9ZJ?1d&GqxcIk_xBKkVau@I?;!dUT<$G73u-gOBpl3=gxbVX((al>a4-4g~Mie6F
zGpo49n&r!lc%PFkSJ;!=imht>mh-!n6Op$d9{oO4B!xJ^qa~-vVSsl=^I)B43>9O~Imtq2#I+WMEzh7Iwzq+Mv
zy4wp58hYNiEPwbPs-1EHAJhno=Z$PiOAEW=CU8whIO>LeEQ)bbRE8Xx1Y9m|K5Kav
z0BUL#O@W{Upas!BlYA5=#pGq4daa@6_{S2aXeihJaPX3mcc+)+hY3WKYx7g@Bzg2n
zEztuSO567k#m}J~#s)a$lp^f$5GTIf$@nkJO(vm#5#WVMf4`=T$IB7(Plvo~Eq-co
zla2K4-`eW0Bb~#1BQU^-Kl}JMn%Q~f>y@2V1Sj)UB(%GJ0c_+%xKET)RCaA(%|f}}
zP^C^zy?G=f(BP)nfLZz3M`$ur;A<|0ri)~7$Nmif;~?jMv>ZOuza5XTd&SS*irE!r3mRP_R^#l9zj_yO|tOo!M@obXj1lB=F>rLxK;KgmC=F^<4
zvN7C|diuh?>xjRgIfz#yU|G+|Pt`wGDd@CLHY}Sb|ntuk+PL-b9Qm
z!r0JDB1t2+j1D3xbTAa(CKp;)NB4yzABQSM^Xh7Umz|G#@nq&Z62jt_%a+|le^Xa;cC^T`tMl;x$?_8JV_
zU}72?By+Pb=JGP$#BnmH$2;g2D;)v3dy>URA%!)Z8pXdNTC9o*W$a+1G(p^X{JMOb
zMS))p_zdWRUFJ!PDO|au8%IJv#dYWp^%7Xh6CIrR%{#Ba+A#f_YkQIXF$Y5E8v2%jj(r9kwH
zCl%ebUyZ8KZB-QHP>;aBq~Ap5k{5};jmAX}Wd4}SNSV#Fl%jb&zi`|LJSTgH7Xbjv
z+1Dp9K4ExqvC9`rDind*amPOyqo890Or~rCD{RVZg~*$!tb^O%!bm2FG7sIo?oNVah||Qp
zZZxQsVTBq)z}?F`teg2w-EghRR{}CY
z9!_;5n|`t05}SWi;ZWofmOgqOpZdrN+tQHZmY6+{a7&_KBER3p)hb^X+nV61oPmI#
zx)Q=aOy^RT*C_2R4S;l&+E-m1|-?OXsw1
z>*Qp@woOye+L94^K`cbF5p=ZCtGC1>hbF!HDMr#D>2`dSx^_!;nrf{mt*gn;_9O1K
zn}Aq)WJJ!O!*>V}OV=*hvM!*7hr`(rcd4($@`)A9F>qavFQV-shB`4eMR}ap=SR*H
z;HY6!1+smON{-)B_QkARK&Mzr_hqVvMeI#zhzCb{n7+hiH5;v=BP=Fz%=
z!y~zr(*~i2Y$8L;t^3q{Fla4?#%IEU2X##+H~B@ZN*;vRYNbx+2iS2
zRuGS5Xz5JHqn-+Dy@SnHW<2JYxsec3`aDNExe!e1fLuoDRDaGVLn>wayTtyr1!P7|
zv=5V0*U#sJcLXAskFPkrV;$d&zUz6`IWa|1V%%@VbGR4y+ZgoU)uwlRJzS2@aDGn=
zJJ*KYed}F^#A<}S^J1cdq6Xbt2(hH?0U!M(RbN5rwWn#B8E!E9YgW(h$ZFepXY)D$
z`ff>JUs=g3Advgk5#ykVYD>N!+`lBfhV>IIDK&J{VeIN0rzFu~RmebLely9{A
zyFBp~**=v-NS2$dm-n|aKDL9}zFfk@h{}QB*dtvE_<5(gTd!CX|E8J*wQ_3=5l+Yb
z!=DI=#gx7K2xAeI<7oGzr7lKDqbs>IIMa7?-t%+*XyJ^OfzDWZ;}
zm3&uW*gP_8Y_j
znasdWad|pVo86XHJAt!LkYvW=Go4LD1_BInRdJP_{c_2V7CXPFz}Gl)_Q4hZs27nA
zzp=SFoPRHCZ$KRaxGkKW#f2#{8+#=yC?DTIlFBoVJ`a(g6?}M;n~8*4&_!#=?X;Go
z%0F5~L0cj2*X_CePE5{5udqhx$DiS!@a7{nYIP^Oz5c9(-%Xs?mMv
z)RW@7G|mV7LH=1jfCGcaVa*X}d%MKsF=~&dWB`ReUhS_t+NophdzBHmWWwNEEgBEg
zEHInFvM*!D^~nq!#4n&{u>%Dep^2D0YVE}q`k=U}y$nixc|#K2C_{vwjEo^-(HN60e)j-+SDh{8t)8CLL&EJX}K2K%Z5uE`5)M6zvm8
ze4D9GZUTw)h#4WKSKo`B?$A9G!++SZd$K+2o#*L^+W3rEnmD2W*7cJ)hW
z{h@npsC~*RxsT4Cxz=L!HsVgtyv#j;^G{+Vl)qx;_jm>CimJXM=srW?uJ@&iH?g2%
zmijB!^cnIyzbBLgLfwX(20DhCR;BFfM&7I7$84w`PY}^6iM;tAZdNW*wYNcl&+@hV
z^o2&r)Ba||_G52r=B)Cm@-G{$=Qk<-3O6@ZRvnKrD{XhTTEd`%B$skxN&iu8D)-MZ
znTfh))r=!=j1A{36Y}d1Gg%}my?^%B1;%?cVCq3WpC(zi1!TAydpq09_3~YrV&U^|
z6o-Ya3Lky)E~c`V?D%Hq2Uf0-_APIyl%0MZfEjtP7BDN4nV|(^?
z=fpE>Eeodj2fipk?UiNYk6>l3B(;0hdD(a^#LO&{jAw~tf;XP<@AAsNfWh{Qrpj`eq&Qf
zL&1USVcB2X$2RxBOaZlewBji(cJ=oG$#5!0b$N{J7f*EV9XUFNHHLE9P?ZIRMY?0(>2>
zUbR``w4VYF^^-k;^V&&lv)#6!peHgR`g+Vn3F%jvaWN)wllI|DOPk$5RzD5<<`U{*
zCv@Md(OGsQd46Fs@%50<fTtRjqm4m9Ko)7>s~v-eEZ}4!*Z30^K2nsOy1Qs
z&d`UY+(m_-h;@-<5NB`J6NTs|=25~+oH+81&sbRjGoY*zc$FUyZGE}riPnAmt7ZZH
ziEOhZ2Gb+C-5?|w#5sIQYuMNx+wmIZaTdD^86o@cT`s?v;>NSbPSKac&Vco2xkbGd
z65%+FMQ9up7vwfn-Vw4IL3m4gzW?<2_P*K8IXlW){Ja^(p+btT&$(vf8SEkG_mJ^t
z)KQ&mzTx-68c*XZjz2Cc5rwWM)oN);|3T+{!rKoCKQwiAKFs$Mmr2hExb|7E&^sp!IR1(>j7tCD#v=1~
zE%Q`3rUNs0Kci1~fcv+D0C2b&u;sh03CNd^oWy_Rpr!#unK*Y(z^af&EN6uE4M#}R
zBVUS|8r0Nw_UyyzJR?ekhe^ce(>WPn;j)@OVCRY%qh^reWD%{BkZ$%;Y!CzBJyAC`XWT>lS|3MObsdIb3JZf|UP$%)g9nj(`FKP0@#v2&6GAmXka+N8
zL(QyPbS*ACXLO!@xu&o5OpX}QJpXTd@mAS8g3U*DpTA;F(qP13jlO1@bmTN;iL8Fe
zsglBsBjC$v^*FsdA0YL*Jt6)z1_wKei-bGEkvzgT;`Jrj>;s&b<9OAN2QzEU?m$j{
zgYgAkLEoMxJG3pEfzV=&vMJjcth7&uN>F*)c+f$%3mzw`L?CT+KWuj$V(87?xn^+J
z%480XmVVwSWg_#WcFd3*(CYQ>G;RggRNVjqIpBiV?E4Y`U|E9RS7Cx23A=Z*a>3v&
z8yoqz(l&|1ld{HyM-UfCe}d<%W)kUu<&ZVBiK`Cs*q+(NEbIC5UhW>@SABQ(^S1(Z
zujG@QE_)@Ltu(Z;{+e6X4nB=Q;p82moLIYpRy7RS@
ziV8>1DplB%fL@d{R+3!xFC8>_C{xB>4urWhIa{k3{^)&VBpCb@-1I>Rt~#w4vi*1Ta1#Ot-tfC)nC*OFQSpBSq|5dj-{&y&>JWuR
zvckzIa=0iaZ5IX&pU_4de1Kg$NxK<>pda%ss%RkE#IL-NNga;t31B=6D3cV;@vr}i
z$)osseTxdGR#h~F@6daz7~t)67XFj=SPP~48X;nah<8`UDc08*Ed#o&8?)xbeDmnA
z9kxwmvj5$46Fo5!$klqLi=5FtITC?c1sVSQdR)!KqJ0yK-N{a`@W$YX
zpBq-in%$4<0d$56bR^jN+u+tytjldN2zu3LFDUCDnTk`w+PW_>ktR9Vg$7!)5D|&P
z#tc*x+_L@#kjorEXwA?WqiZ+um4x?kY%xg??T{mz4?g>sjfGsB_n0qCYK4Xc#|Tb;
zi$I*h`lbubaLBw{$)gGma{+2t>4o1q!EA;nT8xSSzLCmkQjQ1XXAj&G0R?b%OLJi*
zgS}hWOUMRJ!8gpGo%0MFrDr8H-M0}NxJt+xiC)rKnO{Fr&Dko}H=8y#=dPU=AN`w?
zRO?hUwYB{V(%evOV0C?xAi`0I*B=U|cAEQ5`&;)$
zG*_(quvXcD7$!<@m)FWN{hyXJwpB=Tfv3&qG3&I$r^{o#b8D%#$e>+_2d~0$emd;P
zu9w>y71F}S&sBGW|Mn=SCZ@%{ni)O2;wB<7&rV)VwV8GQUmOl={hfAS*~IXp>%B>Niu4D*i2W%*}2z_{GxEa+d;8n>SVJ
zK-BN+2Lcs>;P*9QoMG(y<)#h1xuYoR^*PI9CGIwd&mlnj+`M-4;v!k
zjqLs$mB-2%xWl&d;Ou+T^OBz=ZDG*9q!V3_GTt16JTEnU;$*oR-}h}_@&0}?;C0_w
z9%t+Q_Qy7bX{}s$-~&lw`+oVnWaJ|5PFQ_er0DjXkBwjR+>^J(-uJsbv&H~q6+8_>
z7aoWNDI099?s_ueCHb;<456%rx5=-P+J&uqi$uuiy@nA$&#cGJct?$VV7Va1{Pfsz
z_g>}E{?)iKv$?I#cPKU$*vpo$Jn<)SWi?ypXMZP+>GD_2-8_Y;6-fozi8rA$-75L#
z$@4-rJoik|V*kVSjU%x22?#j=71Qg5NRH05OHj-mV+CTZL^({K(3>u9R12o1(^oEP
z7P?`M^aKuh13_SEs|L6A&;%fOzQliora$|J%p*eXO{RLTguh&6>-$t)(?!hQ=2d06
za$Hbb&+P@Ah>@*bo!__hyWtdw46li;&30A1m0b4z@S79|SjWPDbwCS=$RGDJ`E&L*
z#bWfvOrCZ>v*oEVSI10~#_ny4Oh~kKT<3mAf)i>{b|=aUao_p1X<)ASN@@_<>gD^N
zE-6$8uPD7IVnVAlF53;xQK~o5#_6+Jj)p|4t~0|%SMMlMLBtLus^<2F$(upht$t|jN#a;xa
z9}fVJcFT>}vYpzO0p)lxL)A1mjI@@^U{|{g;j3EH!dK(%v?Dz7LtXJQ`JHex!m6#d
z)fz2k;F}|Hm#88Xbqp9UAC)0hR#jRa4ksvdY+`U0_B0J`^9g!Ywx^Vsi-vj7W~4G#
z%4@@f&cLag?%bOUFJ<_;ce}eu)wkzXx@WaLY{wdSoo@wvgh2kGJuNtdm%k`FB5$iT4GQ2j-m2V2HOzrc!hu@rNE5%uI8-ZAX>
zDC$ST_h-N*%Z{<(oW5tg{?KwbGAz7E*jy(bKA_Pp0JDj-@95>OZbxaIx&ECAJA3A~
zUNojyxH1n4cr1DS)I-l;^=$hWl0iE^WX=Nu_=bXN6JWK?DG>W6g>CI7KA+i5lN=su
z-``Ho8k{fbgw7y>ITv$et)y5fc;9{uJAWV~A@%>lOQiHJUh-4=kv-C@m$JILjOWev
zF3Py}g`h>p+2!oF8ZwrP>i$VyB7&+nWde^3g_~FKX~=_<#^uUb4Wg|E%2T%{a`}%x
zYcV@ed=U!{D>H-7YgR}0hbExU$!oUl=~VXBB
zoY_k(cYxxYqnf)3DJ;>h(mZ}$jk!HtyV!aHzS~}fCbEQc87)?uUQ57dyd96dRnZ5w
zBsO+rA~K<^;p>42q|eI?qs}_8y3|29@fbxXsBqx@le-dkhkdfY@5x2Cy{9krb=_4T
zrLE}$RXR?7wbL1_sntgP9c>MM%!*p@Iv?REt
z9aJM2AZcB=o=Rywq#5toNQ&x@tY+LlmILW7Z7FmHDoh<5%g0borEla~Pc*jQdW!h4
zZ-&2c*|tDEUF9faFfDufHBy4=Q~7UvTe*Y828d?-(MV2_{droP1}rhKvL)_}!T9aS
ze;If^7-CPn!nqfy%4?tB>Wa78#&>MA3nvKx^6w*f8;clQd_WF-o6sfZC@iPe?b2q`sWh}t%P^aos!RIRYslN
zvdXnnh!t*--*gmBIT_TR;w81HR*X6fi9p$;ygqfoU+Vm7p^YK?d}qObI{w#&J84tNWZCl}~|=5~nXekQNjD%f9AF(@W5I4CRV^l_`z+vhiM~B)4_>`wB5c
z`mL<@ITDWApM!NTU8_}9+33VF%FxkV2W@|cVgE7|!h7H)p0zE5`V99zSC6;JW%kDr
zp7@eX%?e?7aU3g3@H_`QJZ`XR+AQej
zdLD0210S|H01*W0X9%fw6Cm7t40&2bt*_mo@i|VwzfRz@Po~rksVj}m6k1&PqRE1&
zM?6|X9{isN#CT%$j()mOcl`RLZ8ZsVxr`cH#mc1#
zp3#p*qqr73nn;V5O5fd43j$~A0iv#diWAjo8Z0<0Ff34@JRWaz>iKN1e(9l!vy?#P
z|5eujFIpXuhrpr}*K@1ZD2jrzX9*bxKM+NHhEvb5Y7zSH6Yvt?+x{I`kEe#tFI#k%
zeJcs?B)h%EQEk^yr>@@%EXwto0Mf}hEeN_@HH8*N#s}PBbwsKFRkD^h&m{-D2!?P>
zQ`<8?sZSzHOPc^k#{Z$pdIko6)=#IXw|9=CfG};Zmd`6b0Uuxj;@c?SlmJXhLQH>I
zlRM5VGvnjqIT2YtJ+D?^OeM{e@5aI?Gll4CY|QMoV;#=&`xw33)z{>h*gdPqid>de
zE{v+EgHvArzdw07xr1sx5L6HsBdfsbSR|TDC#TF_!h**lp53MD*Htue9<3+FuDJlL
z95Pqtk|p?$Pa{<`MAPP8@mN#WF-a6j?9+~PH?$ubt!a-pasNv^rD(V0YIwROw?_G*q&{^UajWHw_3@PO
zgzWfjjWfkc{Qa)+0g?YpL-nm#tB8+nWddd98tI8+>g#z|U`2Xu#HoyvATRi^r$0CU
znmg$={Hgv{S~venv!8N=_hN8wU%ra<0T~a{=vLR|#C%&G))2j>u6w>_`@4I^yg`W7
zMlJdMkg4&aWn!Mtr)uym;*p@0QW}CvS!tM&
zt-G13yyKS}#PjE5Ap+NIiG
zkQO^`B{2yTp*mFU^HRfNsyzESy$k%^mS-{9;{t8GG_yxs4-T(5J_gnPFG81n7vGMU%=q8lHI-7d_G?-4|%mr)5)%z*w#&oI}_ubH-J^r;Vy-N
z4Z$qnU^mafRwT$!qfN>Wz$%6j5Acwf?XsE{_}X7tV3!_3q=FbtqvY{Bn7L!Hgl*X2
zQ2aJ@K7FY?hn9U$N|6uzv%=lh9YsNZJ3|6qydZyWIxx$7s~+^6)UoHUcq&RqZ2Li`
zetl*ZsCKF>;9LT%bp)WD<5^hXR3P@v5Fux~We65$UCh}eX-)JAwnV5Lu+u%eqQv@e
z#k@!#BFJI7grHsX7hCR+vseRgw{oyLBu+_50n%Z6d;6xAj;*IYfKCxHuhDXa*$3M;
z`iL=n;`lDkN8VY&3Jx*1nfev7S1u})-curm_fd_P)@
zKrEw2N6tj(oUanAz|GNH(~`lG=Gc0~uu~t{NqE%wWVU+;ANli?K+eF@RAiR_k$(AkecLm~-lRB!{W_T7hA~)J{9(cV2y-Fj`U!#UVS>vC&aMg3iyk(Q?Y$A>9Jas=U
z(hrkcG7>QGQ7kjaM^a62-hcaIfW2=NI145`v>vf+*QXMNvs<
z@bW2t_`W13u^Z;vqP#nta?mhTh!zJ5U74%(=XbrV_>NP|$QeKElFoqThq_;!Z>Sw8
zEY0`w+sgOK0CLwgp&vdlE%5CvU7H>4#>Y^0`eQ9IZ;IE|FTZYU{`9vlJzE6FSt$_1
z&5=8KrAG3Gpp~kSKZ)g8d}|EFC;9DM^w!G*wPoNjVOIKfz^vV{r2i9S!tUsQ;GZX8
z6<||3CDvY;zCzEAjg(@rXquM2oonY`THU&|`yT&V=CkQcY00&hwTr~uj5j@u0ba!m
z_;e^2`)w~iM6c@39mb>pvdC6gX>-AtKS;RN-fHS}fmf012Rg6@ujq3b9yOV!bh?
zHDkT%C7B2#W01U?J4`jYpSFH>+At(B$R>*`uJC2cQiF&%=wyLJY#xjfkZg
zh7<6sjzV5<_5xlI1;UbAZdZ$BsyYyE3e0q3k_&>u$aEJi*dA!9Z}py2?W!m4S>(|~>;uAs4m>e
zNR#BBom>B=DGc$X#;w}s+G~>TfEl9SycCsG)~N;bA}qEUg^C2
z)ev0tKzNp^XG$Dw$f86gO=Jk0t+nTq#Q
z1l1aQRsWP?eBgsKc#QV11>yTsjC0U5iN`KDt69i)-4WP@z6(yRL!e)e+8HCCuCIYIZ>*MlcV>+@d*31G`+Z=eb|8KQnaxHxe=Xz21AU+)Cf6RpnZq)TDGihF%;9
z1H*Qkb@|7Iebso&cvqjc4CNLATD|7;lyrVwx2=XLxRJBWgVSPS@G>v5u`hK;uw$VV
zso}Pw9?wK;6axI)Iw1?Su#r+f+)6J^tTT(g{j`R}&O9E<{|p>J)v7s)DGqys)HLYz
zS!M$L5ii=e04@M|)1XXT$Q_?xM=549H|947S&VDYu<%-{B!@D+8i=2M1A0BPkpLh*q@&00mp+XfxUp7x0
z>Fr{;d~OAzGIG_WE0x~kPj>=V$x31g_4P_rkUZEQM<;E3PCe|=k97J=bOJ$zd1|(}
zo5xCjhp2^DF~^M^Yk##KBOqEy@EY>YtTk|0JPuyI=j&^&w$|H`tVoK!$ama=*cSYL
zJn6YFdcMP1o|IU^FnlW+1XPO@w&$wR6p8N5ax!bF@bF4i-=Zx?l-4gH>+o@+R`fQ)
z51|v$`|@bjIpuI^yUVJKTkU1})@fH1-|7PU58N(efJrC-UQc9;K!7eOjZ&6Tl$Ya+
zPs;JUGG$zS;OeI694e?@W*)bBaYj)8o8f5u0LO1mKW8UQg&oOD^L2Ph9XIQ@7Alz3
zzWJM1FdPi2*@6X4UWyI(OiM5DH|A3MYGLP}sG-hLBaYv!{-j?I;e3O!u});Rq>sy#
z26i|!1d+={OVEe1!(Cqm9mg878tw=R>7E_^LcDDyOpfe^ckeM+ieHB3`e;bVU*k{W
zX`*PsH?E=f(Lt+ZeKVqHRB60z?9jK*rl49jH&L1nNeltZ4WQ*fxMWeUbN=!STV;@W
zd1MG07)fE_<;zsvA;9S3>sroX;#bB#9VrQbE@%z-tt+eh$+H%ob8Q7|{c`a;97WM!
zUt3jsI_mTttCd-J>JjnnbubV{{5==sfRTOl7r4E70zsL>TH7XIIRvx96
z@C{LiGOA}Wap9YU-O&`e`PuFmbl
z!=C_+25gdJ$P)4alzk!wgO<^xjrY1`7ArN>Syl?D4pnzR#wdx5X(xNvQ|kR_`dKW2
zb-mt)irzX_7rKqFF&%Zs!xn#PD`)IM?ClnYPwx9XJD;tetG19VT&JT5Ol0$&A5B`^C-mHEB~=c-)IEOOp^m7;+L
zeW&_gYZ{)Qp&djN6%YT#>x$`zTI@df@NWL2Wr+$udYVmSEoy>Ql**z`F~OD3S_xnG_fAHSyAFT}x{@|3|Aaj7aVEI|i
z#OO~KIrTa*l5a|o1?~~uzC-x^o;c&P`s~^7BM%+Ax^$CSqAp{z*m&6UFbWFZl6L;u
z;j@d9P{pq2db-BB5(a_-SI(U4(*}?Lsqn(fEZ#U5l<0QE)=sEzP@*v???kB4__ykJdw-qMOchjzoN!G8
z&h(X?&+g(Xd#M*eP;aSveN|0Dd;7{IXafM=ID$Z9fCmxq3GeRg>}%!0#`SqkS8>K`~txLwX+eB1eIhI!6d8&)`S8V#ZPwg*Z(Y75p05hL!|=
zR5Hw?X|%GepsP3b_ZsSaX?-QVR_M=u-u54!p;1~)fnDHXD78Mlq+g5I=L<#Uu6OqX
z0hhTYC<8N#18@fYG<<#ddDQ!Kb_3Nl==A&dS^z`G
zN#&iLhk=c!BLyNky{2iT3VlDCSPvg46PJ}h3{eAQ2i-(BN3A|r*mSnv!f*P}Nu_V-
zJ}@HOe!sZ!OWtKvV*k?yI28ULV{a7{SG2a>CP;#Y;O=h0-GjSpU7;O_43jnhD|
z;1)EvHSX@t>FodF-0l6XTdJsPs+nufcRtS;kQ^q2cmOfAY4iCKx5xy>vgooSx(2V$
zhdK&SUi_|^)xzNXb^h{^ewLd(l>h^WPK;r~O6g0;01mQu?5Me^xJVl#q}8l8mkVJZ
zxVIa1>btl!ZXF%*{CjahsF%rRUI1FE+A76XtwU?kdy2NRDPfN4$<%W${uT>fa#icP
zmXd(HmK7aIg?!NW)BdDf0Gj(zZyup4#QpWWin^B;ou`U!cMsU+WZ__!KUF3ouOlk%
zaYVo&rkRywv3HUjECHMgGRzw3I6tWCS{&TCOw&}PPOT-TlyUlCzO8#2o_;a5yH}vH
z;e)~LuIS1RLnXde2l#0F))+L4|rkef0@jGaAa%1iuBe18?u?b^QaPkyqDapLs
z{AFfDy_0Ht!&u4Wrq~nwt=HqyLEmjVQpmPc9jXP#dPIrLv~mz9ECW(dz%Au3f~4=j
zx+>=jt)&o2Yb0bR^^eK{`9l8FXF#BbOSYl&_e#E+2n}fJ-=Lh9K4B%=M{RZE$KW7C
zd21H=mpd-_%nrov`Tq225XOQv{Y6N{C**o)V(cvZ^%54|0XI;$
zNAUIpI!EW+ny)-0>L6}lbw-cN58gY&}+0UF6tjoaXt0fP&!_p!|5
z_yrrQbU{NwE4T*4O8xr`k%sDpU+|F!lS*C3$yfh=P1Wfo*k
zH3pee-8>{e`bwCy1FcID&pD`q!r|1_8&}S(G6%zAzJMA$}3B{r;^^bl9wsry(Km&3zm7_sO5MAZs?i53GELmD-MB1HCPVVV&eF8V6BTr=J@
zMZCd$)OeaQ$ugvXT2kuh%6L0}&)l?@9+8z{KIm;RQ()
zs$})JyKb=Z)XxFtF3vAL^g1(MWx;gQt8?=i2rkLLk~%zv9i#VocybqLxqtr>_67yo
zU0+AEpUkdLy#h7~QHk1I!QWmvq809OqfnOD88^e8qpY8v9x{2I{ai1#8t#|=b{H%l
zUCgMpI_XRJI7ni~S>4l}Wk}93Ql718oUMT*q2q}BI~S^%>s(fBiuTK{$>oG_18g)E
z(OKjeOX74i7#35@ElP?zIvLT2H>2Yb7EZP~@;o&N3m
zn|pv~(%lmLt3)rx`J#=n@&D3o`OTB8)z1N;%TzH+KzO1taxN@s0msU=
zG@Qgt$4vG0h%!d~fTH)swrjCiSH}a>*;2+LZXNIvx7Bw#
zj~O@kIC6P+zxA1rG*1#w`h3BS`fQhk=SnQgdFBmk>d}vS6#fc=;8c$<=6)fB+?H=z=vu17OQ^puG;WlTfi;>&)CcLmn
zsX7h0Rw~QgUVwx2tzDK(&W%mqDO&zG!
zWU9FgnfS<2l-FcM`hON78!TYW;yI_54TS1${2xCA1UN1GFE?|wGgADt#e9u-`@r`Q
z6cCIa<$cem*7&*p$Jzd!AveuOvmX*u#{wt#H@9EzU84kq1_Uz5U4l#zc46uRTR8*o
zKKy{Ls(oMJ`5I%190iNP7Ma9$GmhSH+ie1P&6NQw?E*eDanP_2*oRglx4%xT;E}Zs
z4wa;`H5dq#K5Xw}1YF>_aehuB1%=AQ|54h(y)KirUQs$db{tj$GpQf+0C}s?Pa`^L
z4%oSb9J){CI8j|wAtC`cy=$S@pT@f_wU@I>bt4Jh^T3jgqDj@zI2h?`$^6@
ze!5ep2wj4)Ydc}fDM>rv-}3t2cwZ);Nb#6KefM0oys1gxwXifdM?Hm{&ysHA;hsj{
zPL6?KphEK^`0Dn+A%#Hu{aYKsnK3gC(vKRpbM9A8HuWwgRW=TnYq-hkKh@QAewAds
zYjWoRjy%bCTTYwk3j6yPf$e8(f$mTJaRIlwL|te`?75Ih;Py!R03^rdKj7Wyl;*e^
zZ_sT(M2F4=#;LnCZ$o9m?e}iq>si-@p$B%lv<&{xc?vFc#=g)}UHPeSBXd+o-SKs5e0yiF`a42?1O|>fs|}*cQGQJ)ikK9rH#^8(eVpl!B2@
zkm-8xVM`lURDmc=a0+Vl{*CkQqiq5@72=c7fgpQ4D5s{=x6_k8`@conG*HtnmTphL
zSWJh-*RFYAlef>K$95v_f&>Tt2W#?n_3!ebxhM~~j&;`up>?L`X|GP*v3YOpLA~a!
zfcv!v5SZurWScJ=pMdoqSs2Xq
z3}054Go7DXjz_+Dv)5rSV>rpDDF;G!xwoUX2O#zv?8;5eV^VeUcRmd;hLIxVk(2%R
z#~(?GRm!{kssqHm3!$0y%cM}E%~4p&)u-%=)}W~WP+%2LvFW5Oi3-Oi52gC
z@Xalq+MKXVx&Qdkq}m-z{Yv@(NohvDvJdLW_(as(G!Y~X7fUA;^e&@OtV!G4Hv}18oa~ei`202JGXbx}Y{|ng
zK}i>8C=8iu_j+bGXqhvlK|G^@h0a@&CAwKk=@K_cO@RwkcEA#gZE5$vaRCqNt8k+^H6U8XY`6*>BD6|za<%cyE
zs?%IRRj^gNiKpg}Wn&A%Wc0Z|yWHr3hTNweBU**3j-GtH8Vd;fL)5$7-6#0WG`!SO
zrxzq6a-7+!hrX3!s2ZY%MGMJ>iPR#_`bM?!4X)-XAiFtC6^ClB_!{P3UOt
z4&Td=Rs`;ZQ%q(k#0~GGEY{@KE%KWKnJUyoG-9NpvS$L7KP_q0vBpz4zQepW^6dz9
zi#}O~W|rD-s$)p)&QFS)m5+{UbAFLJ;c9_0$m4RJ1=G1E#+%}1-DU>7F}?qWbRVji
zx;8YsUtqf*Uf75Pe5G>v^f*sTmvl3I&11tO{+4@FVh*g}MOK@3BeOW?5I$Q1nDscg{-Nw%yqE^qVq@X}3p$mu|wRJp(^v-G&ZaWzIQI
zxC0D<@cmM#U`&;B?KQJ6?Phd8=HdSyMzfC(OY)2j1nxRROh|)jnCm(w7RX?DQ;){I
zz0A54nH;>9_uKiXwG#T)sPyvZkY*aX>Q+>P6@;&J(`wBd2Fh(N6dw6Iy$yq5KQs_>
z;Fh>a7gRa52md4dt6QsX*{7-L%tJd>z#d`t7jIW%ifySp!YSwlUV4p
zv>W|Mp(d-Yg#moHhu*qgwzQRRoH*Ws$%?WzyIuehvy*k^yPUBOgmw40ZYw#&H6(&(
zLW23@4G_nYVHU2lhiKua=VRf;Ab-=z>yIeU>zzl}IUamg+|{4-t?(+IH>>XQx&nlb
zZif)SMV}|VpA&Ab+xK5mNjfghs9!wm+p8Dh-%6c&LWF!nmysuTzsdQ7^BaOo*j%aF
zsr^0|5mH#8bRN^^f2VXvLIRv)20d2chC_!5%F&D9+=^Gc3Z&clBRjM*$m4(@E(*!o
zVX^YXQghD>`n)=q$eUa}rtHG8`lZX$Tk)69
zmih4myC-khuk1HjZ^#FBr!Kd|?+-&lT_KMDYkeeJrxeSlt4jluF?9<&LhV&nm(v9S
zRZzl)AxdiWO+giMbwl&{r
z`R0ThcvbGo7kPMND4duyyH!NVQs+3&&{o7Oje|$oh(@;?t4GP$M&##!(h_AV8l?%E
zTXZ5)9ZgE1S#&CrMcNch@4obDO>=6I7+*rufqpw;$fP4+$cJg=1fVdRg`^LO3MB3G
zXlEu;;86zFe>cM{CC$)D=57m561EXmQUi9L<#)k|L|763jVwaJks)7a)eoUw`{i;f
ze6*q_XE&|g?>bLE3h>8TnOg0%to|}S
z++*I@GmE`TF>+?c^`RW*%7oi1m|!hz`p@`fX<%ir^DzU9tmVgLz%8Z+witZFw}i(}
z&CZ3!V#4%N$!tWxJL&$rl;s*%)I#F=>AwGjo(^{c2JnMrsB#N2UYOo
zZfPffKOYFa$KNc(Nf8qku)&H)Fw*toSbS7<6@93QhP)V6TV%5#9<$^8Wy4WO(nE_p^4Ggkanwq}1a}aT_;(Nb$
z=kCs#=$X(oS_aU+q^w?Mkwo<1hVV<=TEnWaW
zwMMP{-TocNf8}~1BT7_=>rsGW^$vL8tShkVrI5Kxp|ilty7Gv76Cg_%a_k4@`cPdyNLM;*BNP8bhJc=J
z(dt!|j1iSjI}L(+if!*2sujy=K15>ypvlaL75rsMj8>9aAakUtN`xT%Zs$0vZZjJg
zvYe$7-`+}8)q-+1N>vsY#$K1g@w|C(yy@Sp%lwxNYDJH11Cr5qOom!QW_m&Fw)FR2
zQq`dI)vduxtI4tZf-StZQ+rS09WQ-~?|$M|@La-X4OT9!{JJDXXw0xb>^=uzp8Wcg
zWBL;lpZA_UBl~efmvzU82+izW7FxlY$(@3qoufJ0A2+_v`R-aK02(9}QIkf`aAwR~
zA^!Vq)u4Aw_UOC@=g2QVz(B|*+kGz#bn?qSyPc}}ts#loL2%+Xat>M@As1Rd@r3d#
z`&}Q#;cmUbDD>Iqte2lT?@AuiTP|Y%{ppYUX!RwDZ}>#en!V4SOKl}_bk-R6(v)DT
z@GGLYa^Oc`_O6mDk3=Fm8@F;;#nM(GdiokAPnT?}Sx$Kv1NZ%Oi!<}hA063DDnt7D
zD4y*@q!Nr27QW3882|tfaG_S(<+;S>;LcqJI_4}Eqh0b_FK&EXohhq^3EU#0F3WFJ2}rKy@xNexj5Nf+9>xUe1^AD+$80oy)XA1~+CBH}+xtjmKus=({h&xAFr3}_J&+%<0g!pu});{!a
zitBMslr&-*pUg0VbM}hH^9c96k+$gYj@9)D*g3?m(`yr%sojJ^17U`seucJ0I(kIN
zz#d-0dp3mJJAdaGBXJ@#N&6_I?`GF@_Is;;63aq>S5f(%h&h)qX`EO^SHP^D$E0Yo
z9$yxe$s*|A|A3tr@AvlMjJVA+W*`;QIHvJo?uH?NR?cJh;P^LoQb|g{&u*ALcDI=t
z|8>Fay72+4j`ZmVwcb+pY(`{yx44}Nri@sIrz`kjT(h+N=A{q$M`R5Ob70Hu6u09`
zl(W36s_qBMGNI>i2HdY(k}u^(Kr;w{64Fc074#d8!iIovQgLh`IEzEqK~wL3Q(h{0
zH9srO_A@RGrSzx31ox61$;7j4D+8}@r{{ckdqw;H2YL^0R32fm1pccEf1^iCM{f?&
zFCjuFe8doX85SZ%sm5%`lIQC`Zrz-}XwMS#eBlwN+2(lKDMp_AtMg6_qi8yveHgivcJjx2bY(NtU
z`u?7_4V^j7EFDi!e7?YyQ~fmaNE`>;jL~Tt9a+%){}P<5TNQ=n-nSv
zeR!z<*$-uIQ~o;p+&zdE?u+iLm(@898a@pFY{>JWAVv(f2g920*~{tNbl0VYMP(I}
zRHN}{gp<(e-yoy&
l68yaG0OVM?=x!m^9u+G~MjW=C}MUo|6Cxyk8PKqkx8ua};
znBOremR6&>zS&J!#RFg+x~;vjJ&1$80vY;a$*LLtAf8y_+9(b$@3qCU=(B0y-4wsm
zF>XMQEm;1(9k~-(jL7?nm_lk2nPWM7K=!Yo`}J43R0pTP*3(4YI2zyB@c;fiC<`09
z1`9lX%WxdZh{;z4SHLt)VE)_%dY82
zCZ*%}%(Dz~u_=*j%N2_e`|8kO$a}7T)PmA9G2+d$P(G#sMU4<%7wvJqJg_WIt0_0!
zE5c@8+#_!Ckk@#vrM-Zr`FS3S%wMhv*=u7-%CAMrr8Q4Q^GcX&XW$+3aX8KrIZVQ
zW{=iXoZu;0j^tP_TVs%uh$N5*n{lb#jV3_N(Z=R$2hvb`ODuD*%o?$zFDJUv52yZy
z^-aU#p2b-A@*&!~>?)yISDTWt8Z?nlq%wUB-(@wa&u-KQ($cNKQSn}@lUFwN>@&v%
z)+fe=zZ+J=?wwsx8B
z#A(bbLyNUvo$8`-%&{^adR=&w23y4b&ddOzSA
ze3b=$w}~b4{m}rpx`9g(DA5wMXlyN9w5N&r9&HF5RKqes!cAqps1)Iv)S2T8fr$Z+1p0nqFH`*+Op^=bSgBj$Fhk*2O4F6
z)e)vqPJgl{`lzfmtp!7$UoVxJe}l>^4O#Zb%?Hdo3s-Aeb0@VO5FyL)KDGsa$9w!U
z${9o=dr;Ewx-M{b*l|vP%)APPTZA@3f~9ZaS$uyObS2_7ASC)-;GP>c
z$?Eaq9!H7abEQNeP}XN)`r!2LJRmyV7VfP0tM`6=a)aAN^eKC2LouLA^}UI@8HDSy
zeBliZ@qng`I||whX7F1glS9q_a9Xg?+TDW`S+2qcPQO*OiS>5hhmmkD281NcW|iL+
z#*(~_^TBT&5(NK^%y~7h1~NJrE!mW;y-l1SuL@4h=jow|B97UOvt#9F6+d1R285Ek
z5u6U>b3%Uq{g?YG7GeazOrKdvP@^f4LX+3VtHUZ$7iDJ~O!$%fL1s#Z
z`*jEV&-q*S;SP=`AB-teEK%GL0~TWBc<7GlDAmRXFPs$}v(Qv(vx2@Bc%+q&?SW81e_*F+E_PqQSiu@
z&P%R^S#&>%DrX8zKI5>?egwV_gokrME4s6f8`#iWR+pOtrvGVcT-ulKq+)C5%hv_VQLYRP*Z`%%N6-`uaEe=1$@tgJcFx2(O~OHMX6d
zL}G47DjaC!_nOj}(FmCcde42@>7ONxr*IE!WiluarjeR;>-6+hZRe{sWFb|msK;yw
z$Knj!JsI>Fg`1a`Z1Q?-CsLvm#mJdBme}Y#01Es@$p387>iuy5Zy`R%jLG3})VhLc
zyIxA0RMTemmv9YYr4Rm}PWUFjer;MJMrT{?Rz$Z;-al#C+(DV@Mx@bY8l10g)O#)Eo6{603pf_fMw!k
zj8dGhW1RavO_1hsK*yNaW+TpR8!x-KilykuSgBF4V{-^4YJQ|iy^-IUEsOaB^YTD9{B6q^SJ
z>4&zXo~&l96VhlgEUBXa@B2*L7a~io%h|PiX$ysyf{8f)fEw5Dx%<)7s-=H%iP
zNxzm1Uv*#|P+?SBit(a;Hk@QsR#$IGHy|gIqf%{P0N0y#pjc%lmeOjcChd}AMcHVK
zQLdbRQq8y{*e;d-CHda)E7t~J_iXbiUIZ1c)YfjZE%$RsC1yqeUib23L1CT7oCWJA
zu6uPF)CLC1x%g6etJHeEeMn?x^fVOWlh})b)l6H^)iZE<|IeH^14-}q72bpW_~ZBl
z@98fhI_MPyP)P|p2z6t?>)Y1Qm&4ar(rxwYItPAM{cM}(ny}(I2VV7B=m4!vm|yK>
z^Ph9p&#>+9Q4U`>UF{Cr)Wh+=%DenPL&)ySLKH2ihm(Y&)byH;Ie%93np9&
zP~-Ugmus3~n2c3fvqO|R7<&|j>|(BYW1!^MKt#pO!lZtH+L$O#J=bjnM!Z`dWx+T<
zA-*qt{7CV?d;iD^!yN-6&c6EF1c34~m{I(4^)3{St~5E?7Xbygk_!5#xq+$6%
zDi!TmCvdh`gF$B1BUqx6+;Z%4h9B;d$7jUq1%s
z;S0s&l-EZKxyoZI
z&6%#q{ITrUlGV7`)XA3sfBQ;0$ojKuJz`vM+_`QD1RUFxD9Z#XNF|%;jPuad;xVq4
zzxO)zD0l8f?botnHVh_ZWag*FZg&KD9cnSS}(XRLYOmK4RA#d+_FDp`C@;sr;Pv>`gXD2tvOM<lRNcGdE%C(Rt#xfo--$pr8g?Hxj}fN%DiHS5(>@{ia`eMb
zcc}~;=)eq^!}=T{F2LN?Bl`qGc;l~y(7$lCKyfuY*zdZ5qJ3E3sx@j>D5X9Au10v!
zNN+LzbOVD5ZSMzfMj2f9U!T0M-g=cpi>Ud;Mg>Xhvraf2euM+ZL^Ch$oq*m;;O95P
z4*A_azTlgc7e%-jN}TeQgqD+P<@H}k2-Mv3;S^d(DSF1XA3<26vayN5r6`|l6=XvS
zeoJe!B^Qq`7Er({HolM_|MG2eblFI#Y)E*PV+U=p4wt=UPxinDdWj
zFlgv9X)_!<<&?xY}?#-DIlw$l&-9~pK1P#p~bAB8WAN*EBmFmD4MQKG&N2e;!VZ=cP|@h
z+~+G;Mg!7a(9j|l@b2&Qxrac6`o7niBHasy9SXGGyLdk3)yKV~-!0`Cx~-LVleXgH
z?G|oeA*~>|fxa^#{CPyUHI(pjh|JkdFUFEoFV%NwTWup~^C#8sQ18BNf!F#tjYnRo
zpRM*Y#iBbFjjjq0e_YBlN8Scnr@t`e9Po__zXN4~({O6bdSKbZQk7FlzqL7Cgc|qp
zCYhCI>vHasMw(pmre`7>;XVbgxDpP>;6XG6-}kulgz>!kF8?D2;aW{w*(jE8wKDL7
zI?44~CddgEvFf077K2i^h|YI>#zgD
z@|Xb0B4v|F`GXC`uZ!!00=NErEM#?vleSiiF-^{#ZE%~`
zDw7B!OeJ#KQZBQ2r--F$ARD4p_UeL5|PcaIP>yX0?zh
zBxGo70pVmGOp`;dwTGvtQB@H0`FN5i@+NfI7timq8CT(C%M{P6u5+3J@o-=bq>0|^
zQf3THHE92P%1!mbgaure_2l6P`E9$h?)PHt80|3yTo0vI?%XHx
zvU)+#+X-%iaY15XHOuPB?1z@(3zgdRZthG!!lVLHU?Ybba7tbu3Vz<&kceNAmCTbd4Pt=Nt@(02sS}^uPWsa$
zY-Ucqy{-0B^bNP(EfbT(Ss7z=QC0x;g;oF1M@?IUC3o0e5!|_-_uKxM%X7&5{Zu0I
zT@ITeYg;=xTYj>N{erH1z-O#GEb2h_!VgjP0GUP@;RK`3fj{}=+*x+&X<-MmFj`21
z8gK{+I);p2_3QG*rtNt6IM=|2Z{+KrTp4wp15>k)2O^ILLU&6G8CooDtPv;iDDA4)
zT4lc_b3b>j3)-1r;!&q+q%XT!cHc?poG7B@UYzVz8X?Ah4uOEc{|eOd%`dJyI65Eb
z>_j(SFlf_n2~GLZRJ649uKHu2qVqA55nUT%yaSb
z8iA1B+g$d)K<*kyN)T{!$cVEIL%Dc-cjN6}j_ge?>USGR8Rkfc5p=Duhk(VA2qLxVn983Qw!zRwxJ3f8JgP4!{xJCtJ^
z$JE9Uem8_PF8)7!$z{blZSJh^XR7Ptt&@@2YUGl-C3iRpNZ0?s0$7-|owK^R}NXWT5O*YavsA3X9
z-{b9hWb2cF@tQR9pskVoDIN+J2@-~q}=bxw^~V2Kr0=IW*8s7LLD2(o^$T;
zZY*GkGAFW|z|F@rUyBre(3r$sqla1zAN~IYc%q$lVYNpA?S%J^z*jT7=GXQ-u
zLk7{U?fOW|)S$U07R(@Z>Nl=c(?&9CLRbnK^jJ|m3kwVUtpRy_9_K7M@(^fvj@#mW
z(<)&i7+wi?Q7u*8Xl&vL3(Z
zo-RkP`#%q*n1jc&a`(Sm2^XGacCr+7j_*Nl9zPW}Pp+{Eq`>Zd`|{e?E;AGIX+0vp
z6fM$YauB84eb_%BjT0@E-Oe{23RodeJ~|AIe>u%ZKkRVW#nhdW*39p4*LCdqyN|S~
z=N^8|`Q@??Am5n&>q{qxnaC8ka>Ma(;A|eOjVv8tquMWaiq$y(;xEKB@7%BYzHi!!`<
zc#tFtP1{y^Wd_)Z1{>l0y7OYl~Mm}tl*4LOha*Di!Xa%W87U{&<^Bhbz<<_=Z
zeKu?58hIKK`jGiiZQpr3AV=j}tb$a(v#y?tP7WJ~tO`;@14faLF7bQj?Ez`HM3e}<
zNE3Z~AsQC`n6HU8c50s`%8K6M{*wL>AVOmxD$u{CP&k
zH}AGln7VH)Dvb#7ndrKVz7hPABlTVI1-sx%GP4OQO}=_xWVm;)us5gqyR&t-;5@a2
z-svH>%8NNBfG0#Aay6Q+%Ndp6yH@g1WXUn?75Nz)oxJ4D7_?K3fQN7Gp-tNktrAyx
zS?7Y%IWD~Snt}YTlUqT4l{a_QogW8t*g#Buw}CL%B``iD1;>JaB=PM!$s@FiOG
ze_ftw^dDD~I$WWL{O{wX%J2h@+Y@PYTs=+Xc&0OtB<+-rQ4HlYvkaP2ud$x9naZD_
z$&9zRZj+a=fViiF$*D&iZn>KmI*LP5k5_I;?x9iNq#Ba_HjFX!u+$exhn~c1#}MtH
zeLwp!_^5E_lS*zYykWg{#m{z#He!pOf3oYqamIfr<%`5BhG{dicu+p@>e_qd@9S+1
zu)ArbAq#9E1*9;Wzh_sQIhM)G*`&O@^04nY#-}H*nE~j
z(xFREm-s<{i&xZCfi9>zxkg7h;(X;RmS`SMQ(aL`XHoC`F9_x~U<|Ru$|kGqcnRui
zkV*MR69{})5DHwPEyf!~2Cb3}x?;oM5{JO5qtC~P+sWMc2I*jw+ik9wA!J!AjYbsP
zF}@&Rp`K<|cEOR5xcPXs`sf41Th6?~4A*I$0@}povmL@W{PmmCD}N{BSJOIS>mBgw
zGS8>HdK?4X(oz=%=HOcTW>(${B?+gRKJYdfRE)BZ=?o|1!V)i__x|
z_y14euEv-i_p%(gg@no~=^7ZeA0C4L@_F#^@*1;`PDd;s26UlR>Nqa>E$g4xllQig
z>hU##k;y7ZMRrLq^jvG7q4y9xcqDr*`8RhSf|LG*+Wva`3Nld6>p*T@7Btgz4fG(2
zEn3jZx$nr3M^Bvg$!c$N>K;etg?#Y>=|
z4K$<+sxK9cv(%uF=Y2A>x3L)q){Xx{G|J@p>#%
zuvhJe)%AN+=BwuBG)A2Nn4t9R^d~Cq)1irJFE>%wk@<^UrvJgy*EdHC`3=_9c>
z*M-yPS{0&({e7x?QSe>JwRwR1`*DrH3J1+v+~@t;kthdE0Y$D*^g$od_SIvL_l4It
zpD*n$N2i`^T_(|dqPI}Xb&7YWxn`jjQ87O}<5KTHH+Quh2!VE8_meKqCZnWwMl|;C
z-#XkO4Hxw`8xmgl!_VwDt<-y+EaLK?`n@J$Q4GEFiLX#>lzB`Rjs@6*w6)VDP~*;c4#=bN6ZH(Ej$;QD#Hs@Ktr&S6~WL
zJ3E3P_||gdM`IZmtlU7B(x#U_f?74te5|^ff
zN1coSC!5kyv&%a_47m>|H=a(=F~aPj?Nv6)Vym%ygDHB6#H-Myr7jvt0ZedxNrh0_
zw}yhc8L%U2B99(hNN-<)=Ynxd3&w_RavwX+9*}3g{aj%NzI8Kqb>Wa~IG$3PyRSaF
zx<)nNE5xUkdU=v*YE?kn+naEFQ9aZD$a=Cr@BT=%A^&v{EEL>)pz~{BqdcjZq^t0|QL?Lm`O5*bN
z^`f)*ictKCcbW!uhJtib<%)q)9s@E{!n0L3s&*r@a8jv_EJkgxW;xLLu!7_OiIm$s
z3pAyc)zL(!7p0JzsV;hFHOUjm3d^ym6e;v>cZB3;r57
zwNVO&7xzCcb4PO0*wyo~WW?tCnSlK#EcLgldOqzX79OS3e0CB$^|*5b1F5oMOru8I
zlC3sS12UC@+;OO@LofyMe>WRuXjuhAwx5Tm{cwnRSiS2dKsMz^QRKv>RNh9D}0FPvH@#k#y(iTmvI+3>02G0K_bWJ!A+tnpWTBe
zPqmO!fLFDLL8aoroc*A8v^3@miy
z83d{&sv0SE>$bE}j37GHY}f-aJ`sFg%8|02zkJ0-T5&|iZh@fEn&n@>s96n#keT10V09yH
zyX+6%00@GQAtA?!X=}~6UhCrJ(WUF@h`#IX$>v|9)w!8WUe#Pua<$E6bmS5>=`x*N
z{z&j-2v~!IxQxowK|KOqoz;H-d_7-ab2Rbvj^z=SFZ8zZbN_h{fk{}saQGUvv51`7
zQ;?U8I3{QACdhEfp7YPXOHJ*!bOVF>rE1!-2I^HWkB3Y6T!+lVKAu9t?*3n65VlGo
z0|;Y<(M%gLKc&&sSE*VCvWRM@CYbX`A2Bchj^~+xn~Gq4dm5h#FxRgJUW1{QzI#d--Y76{bI!>dF94=v*`YuOw7By1c4*lu27cAExIotE>BG+YQ
zruB_JrS*2=&}+xxT>aKgh9LP9Mj?U&`;9W~qR!Dt`%j;U)_$3H*UQ;o_?o0cZ@+hO
ze==CzF$eQ4z2AXY7Z)_1G8g>nI2jsiF4`Q*TU|mIAf-%**2>h~P=HV^;N1+zLDd8F
z%0FBCRD!85j7!ubbDl9p=q2tbsz(^gqENkJP!)U}l=A?j>^Kwm#vjdbivx~H`D()!
zEKf;-OU!NPu4#@+|3+ix}f33K!_@Cr{
zd@?>M`N^5{{POn!{_iq!*(L8(oBrcWEB-2VEr!bfx&!V<`NGOWmGJ`mw7I{S_ey#9
z1p7;?*M0n&8hj=6YVhXplJ`#hj{g4gk{S2*LKKh!4oi~soEfzX-rS0ZD?<8>`ujva-7Vdn0|Ell-3>}N2na}bcQ?{8bPUbVh;%c=uy3Dt@2~rPegL0lX0fho
zo#%0wDNx|xMD{W)ghq?86+}=d45kX1{oF5D3Y-VpNbN;*O@Ok<33$!~+;teFLK{6n
zuYq5Fn9!seLa)$>%4QHY9}M%vKk{kMVe=s1dsJ~vTWH>^X>RhhYCI-wG;I|8cwqUL
zQUc`ZoRz!+B3lJRts<2gI5Ewm^)t96Z#VMrUrI^Uw7t=5J0&pYe<0n);|?9XaM@TE
zM#B?1>FoVzQd)7FZ;iLtDE(LBZs+}@r;2>Bu8r&+=d)|P=id%pk8wix9?!yL
zfQC2B>x^+C(X>i_z*>WYwK>CH{zMtod^WSr9uG{?3Lu95?%lQx!^2t}Xszl2Uxu%3Eg%Mna
znf$Ycj)~T54?qI`$C%3!_8Wx_AW5mkQ>ATh6Vx9>kJ!p*#mP3#
zE#qTfbxur}6<53JC{>4LGEWiNQus85G(VuRAS%p8`*N8Jq;vcl!{5#CL;gl+2C4%A
z+SWZFH_UU$4bcuS={e`$A`y2qB?2BfOj^Ix8`LjVEGfqUj~`#XLLt0hf8PhCgrp@V
zYO>uOSopvr&^p^>l25?_X1Gjk)?*<-e9r(*vO&F5-@h@PE0_5FMsIb{vi;JVv|pMr
zE4nb^Cw^O!zL5B}fYE_~yX5CX
zJe`h?mV%bc=;y0qhdbUw^m@?!y;H<0L-@M^aq`-gfLr4vGr)`!&Db{pBiKG5k&M7r
z`xJJ%&96Qsul9FHux2yt+pq1`+T8hAn(>#==AVh*Wk1E&eW_>&W}h#V+65RHE6$#QEL?m
zDJ}e5sinH2`&gRIZN(8><7ROV(S29Q*A;hXI0HvusG*YMOS7WcjvnEo&2(t}oMhWX
zWI+;-s;__32th%VdOOI_F-4FuHUo;RQq}ad;%vwR)hNp{x~@OBAWn&=6%vzWJNi0@
z36Z(23%s0W&-d-O<~A3}uGeFsi#(3WZQ=mEv)2jZ>3+(Ovx7moFS~QCqYCR#Hs5Rp
zpCL0-D^*4f==S6sUg=os=N7u!$pkaV4bysj6eK5LU1XX7F;@Y$@+7kXz}*W-YX4d5
z{=LSD_zsVZpn#Im_oN4Jf2b(e6leFoehz=ydwy;7Ed03(QR6$i?!4i9@^3gj)}Of4
zwMt;fpp@ZVLKpU2Z@$7d&YoEHDb#+7pQS^~_5^JgC4h@HZBX|w`-(!YO|EnOAlr3Y
zALNjjw$}|0gRTHq^TCGG_Y@W9Z4-+ZwK`hO;q|6|AfArq&jEc=ji#;quLDNu0b5Xm
z-s-H+j&{$#MJN4-wvSAct;a&tDsc6yc+r$~h)yPoh?Rb8Q}0_?_+DQ(3;AC}!%Rk~
z!s1S1T6`RO&xh*_nSct)2dUjr9UDomJWYKD0VkbMOQpOdY_TYPE1S~g>%8oM7;uzz
zMMWc6`(*>2BnsWQuc?=plTu^X&Se;XH|*-&?76CbwCC*q4Au3(gYEqly@iDZkQioS
zBaHbqPD4zhL=ZPbm)T%L)}2*%96NAoDs&%MM8d9i#^Nj3KzliPd))zlzu81;?L%gJ
zd_WVxu^}y~Itud2QnUbSVmtX*_bFEKn&_rCqZM?KT=tbcyB-9zYm$LPG0iLiDEYM0vGAEoNj)|$jvz+5A{iS9#{kbL
zNm83zTty2Y-yocf!fT!>f4>k59+9Q3{oajF()|tP@2GY4vp459qwH9%uxx&}U5D#e
zYttPvX%S89zE`Ce(4blhuUG|+VQZ}J!ZHRH7FH0GPKS`-_GZ4!KP6RSBK8H-;hcQQ
zA=;$QJ<^hEqT}Cp8An7O_iVULA!Z=GCNmXX_>=p9&S2JTLT8-)yX|YI;g%i8BQ$;%GCG#0%>_o*Dw9zCd9bL|9W^GkI1?#{H0Te03QN)n|w9!
ziyC8(hO7>J^XY4QBt%YIJvcD4D6x}U=gh1mpaucCtlg#PEYP=?5vs)i9dS8D_VQf;
zi$0OS0WoQ_i-#ceSo<%$=+B-gL5uE4ww8zeJ3EGaB5ey2+t%D{q;h>xXxEnIV*e&M
zXMz6q=I8xt?Xv@)1CiPt@&6KMIoAk$V@Yd?yK&c-&F+83UOy`+B>GP&l=3($`pgOV
zVaLqtm%%mROQNSyKcDu`PB95+-d@twDQaoa6{beZ2>CckC#ww8QkIn|=}3U*k&>-xDNgk2_;Vx2o=|M!&#uPRK|n
zO;>((@t*m+YVX_NU4&R62-{g;8xa%d%3Za~qWXA0XvJpTt4=#^j@oU@VA^%y4f(){
zw5zAnOerSHU)ONe<oiEUKg`f^H!+4SNLqrSm*`J-qN!DjvO_|E<0x_
z8Ua}$BK`#-5m!?Dmp|0DRUwQSe_G*_xcK^Wd6p;)$}iS#(3at~DN8FY)+kAkwB#0i
zRC)msf89I74IxrT+9q)diU1*3PnoMData_O0YS0iF%cpK$h&|L{lg0S;4(?n
zuWEI5hmIwrmjo-Z>OVD56XoS7sEcsV861{KbH$CPt4QXWJLx*&N&6wMtSaR+VOU2~HbT;-st_4bOPMs0{Tih!6?UwHR`pF!RM?C8c_!K9CCoBbmO0Y&
zd6BXGb)(-ekKdRI08jQd21Z!tHk9+F2vN;{k5K-uhqlEn7|deJBa4?GIUpY&nN-w2
zS}VOwxTv3{4|wVc4tN;BdcN;t&(78x1b17&#yORXpMSj`>fSzFa`0k@r~XWMU&G;x$n}6z6rZ6@9o%m`UY(cShR%P_F`uGFqiod8?8w1i5b}a>kKk$ZL_*^}XGrEYljZWIWYk)g!5tFuwyJt0=sEU)Vf=1qT~
zZ!WbtxUp-puKRdxvfx3jd9dyGK?nmi9;(_B7P?^Q`8CsVv@Hq&(7FPdm?c}?N*zU7
zjlo1?cxDgJq(JT!8Efcs{|T9m!@@>Vx387#1)#o`x*GbV?ZTCn4Tp%v{R}Jm(`1A)e&v%ysXPZfbb@axajkeJPi&^oPn1~25
z=%W`(+eOA=_t2XbKOIVNdhCAAnZ1*%C4N|u-_Y8(q4ZvX{^#z
zQ|5?JNwh824eVtwp8NX`S;8<^*pr0glSPe46U4p$H_uxpCgX$5-eA1KH(y@kZjpOi
zT&+#oSzU!`qr8i=Q%PbnHe0%h)JO?t>%Q)m<06u>8XU79xK_>-+_!kbK4?RRD)vz3
z1IFE1U-tO`o=R&+#^v<_)Y-{tgf`pvBD3!uN_Vw59NoBb06z%uG<@x!A3%%Uc?r;U
z)|$o|bK@mOlaEFJblYrEenfLyOS{j-lbX4ep(C$9zKjo>Q}{#vwY{sICd*Kl=F9{0
z7Zlgod)J)YK<&M?kiOez3bxCWykvQB?Pq>5dyqQ*f;l#O}G^u8c9}1@%&*pM!_XYEz&FRJwk?wfzYFkK%@~gnE
zFNxldTpvTEvh7r(UimZ@wTb3-2<8hAn$GlBD%kEaeIklF-+|Fcs?<67$R7?)87gg4
zZ8GS>sG(W}K7q`&;`K5|-2hfm-1sr07lD%Bu4?3V51?2KHYcK%_o
zJ4&7<`f%}z`bZLYcGJ}X1bN1W8M;a~t%3Q+wEfAf%XuB>*)^|9Nm-!i!_FI1e<9Pw
zhwyoz-V5*%vxK*S7AxRE)WD?%WwED^r%sqF(wJTdNm$XEvJE2ITmy^jy#oxKu!1!`
zT6Z=1A4T*q%Q08)eoqeaNg2Tun^gcq4{Q>dl3kKrL}gm}4GhUM!aZWXkbUeaq%O+S
zW2SE+;yJ{b9S;T4)O2J&3J%t%dV!?BImfSrLIN)liu6ek)80QG>1d0S+NXIUZ<~R&
zIovTFC5DIe3q8p_r0LDg=t?A8ahwj9?ijnLw$6k(6IP@qYZf_GJ
z2VsP1(6&D$8dM{g$J-8??>cN4;_?k3#SKO^ni@-ri94Vo&|R*M(g55#~w7
z6*TlZm(fwMP!rP<2DGMrF6=a-6>b~~90-Ga#-AnKklq}6M_hlK1B=;EaV$N3*~+J!
z+nPj--}=}I%UcQTd(vnUen3}J*#g-0Oj{oHbFZhFcdj3x1=(AxOVpAsYu`(^v>EHU
zYIxupM^y)~LUC%*U#>&Ykb4qJW?DH*v1Ouci0olsWJ_h~Eq8h%;tR2*y@2q?mUq|J
zF5Nm%C0o%ykQ4w`0ieJCc7K2#W3|&Kg$azjkd{Ew^E%+AhkvNw9opUBfbR~^oS#L$
zj}t2H_N#44YWa6G=@~bQM@v;IcxBlU|48+n5W8xD!D1G@GAS7X?4O6<#E%T
zmUuW{&4ds3=nvQ;PHwRR9GQgaQ&zidr&7y>7@m?z@2nOpitd-kY4Rl%UnKcRiyvY6
zrvA@g#-2|WZea`A8IA#I-+vcXENZHjVkJ-&n2)dRE)Z85o}K;eJ{v!3_Vv1dos|s_
zEzGK2)PqQke9gEx&u@POci%;K!=3Nrjl
z!>Sp-3Hz&2y`*;k?+lqW#G+ff=ck_T_pG`*w=bibwCp0UKg6=7}Ywe<+F
zR)G3GF++VHQZJ(v`t?ZHL^n4(v87@DxnKGZ^O^p03;+Z4KO!|in98HW#44(%ml={F
z9yj0diiK27K6Hv%V9fY7eQ&h*TfEk~$Ky9Nk$M+Ov*Aco!#b?vb#hy$N4w)S_|>`J
zmB*AT>#Y%HEvo?p0KVw5;4N
zA3b3LE}DFYljYR)OIQGZifK~Kx+U|fT{@wNn0hFB&~z{nCF&3e{ZX&vH6fpve(27{
zjTU`VBJs9T1~Oqln9ljj+wk&4!UkxOM$FmN{^lsDmj;m+y|N)JPYM4BX}JPnB0&Vu
zN|Qg7a|vi!n#t8C*@4t-Q6rC$O%<8H{p)1|d*oZMt3Ur(RA9@bTuE4I@W+SE?3@YY
zvRWO;W_|DPvYHAs1+_+Q_BkA%6%P;1I#v>&$2C-f$GbAa1LC<3b593sUSg7Cm3>C@
z4K47aRv^ZT*ldwueuslYH^<{$5u1Y}ajZusm^f&!?pDa1jbmR0R!b$MVV&tpx_JNU
zHNZiAe|K(VJ9!GY?I{2HUz`3iGU75#_t+1u?%@7Kokqjl8y`8dzvtr`Y(8oBdZbU>6ngx=5<-x=%Yi
zPT}r{Bg)
ztdWD4Nl{dmmgQZP3VA9U|GYzDphDKmR4v
z_-y@+jB(kpEgO@63iH;?`P-*ejLQ9^&K^Of6%qFnkGCgw1;W$7tynSHJbF;SYLBQ2
z&!#Cop_dAuOOHXF{>L(;T1-vPVn`=KOhbQeMXs4k+>*Lz_CA0v&;p~oi>^EY*t(aM
zrA7l)EsZ)M(bSD}DwIUvCk6%q0FXTw?Q$fkTJ-+O&K3-%?U(lV!d;w23*Q^^wT7g=
z^ejROd_AaxUdU&#k$(J9S8Husi_iFm5NNS9hkV8^m6A;eKJb)GGY8i0$-pcI0OP#J
zfkrEEip*QjzsWfT+P?<}^!d}4IhY@!m49G7qneZT_8A$IJu~}0U^v2vBboF5I%y$N
z0(LDtPVM)$W?w)Fw$;-ML}l%aqYs>8YwU^*^&brAKPP$|FIRRZ=^I42L23uSU2l7+
z%4|Jd>8Sdn|M@x*S(R4?vIwsqRsQ0$f|h9}BXwqdwpBl>r~T77jRD>cTt2Hs__roW
z6=Byw^fDlEHEr9-$QgONHU00(^hmCar4Xokdbcg~L%zMwtDC~SY&
z9|ZXBspz%^+4}CCWp7<@F$4zgvd}g5x%ShZ}m){d#}I5na?^GI90Jo
zu~_*mgXVw{WWf>{I6sP(rIWP6{F6RNGa$h88ENuV#Ba?dYRx4P^W-r%Yak|<2^H6T
z0wCHqerC%gVb7Wvu6KWOHjRmjFCrKkLIF17o^@YDx&b_b#s?{8Qd^7{9cP4(k8=v;
z*MfKf3?G$MD-iY3l9^e??k#iV!hX6l=anpjFFGtI?k}+3w()1w44QPPDt^0XlC>F^
zEiMo+O}7NH>r%C5?Mp!R_V79q?EQ3pwke^ke9AV;nSvZ(6mk6Z_$S0}-A)aZ9Ac)W
zE1x}epJ99|<;yF1#d0IuTn#e&vq>a3>+#N1<%L68hao?)SEGCVEzBnAI+ln4UUzC)
zI@_0i*{#DE)Ol%!c^8}7;18A45q>M@h;L&x6u);g!8`t`-{#QF!v$;iw||$$;8Vlm
z$-XjK(nndrjV}J<1LVJxPK?`=-I?W*;4?Yp>TP{&k@`D@~6aV)d{{+!b9Pn6ddy+JX2Z?SDXL9;Jt8$_Fz
zE1`JuP^ubStjN4ihrw#*EuOIGFfzZ;mreRnwT5pur#3~7$?Ld%nW|+_Jk<u7{;!LEGFss-Wf~y
zstXwaY%CRZ324G2kA#w
za*7?0c$)PfsMofjHH(pDA#@%$x)f6@ZvaIfY3dUNj(k~~$k9qxA&whllUG<=02&Zp
zGYIV3GwIAb&Fev&q$&h4Ltm}@RV~!zf9&$hr6rgVz&I=#KLgdRjBmq)%}qbj7f`CB
zLXTM18hfYOdT-vWuCtB_+=U=?tLqbz$?mc6%<
zd#IZC;s4f+n3K$>Z=e$>{wjTHe&UV6iXZNZ{-uAI_EwPEtB6;}xJEdEm2YWV_n>~>
zV^mtiF&q_;&p4sSMdB4=N#gyW>%lig-^ws0sS7-4GW;LeF#^sxMV=nX$@XY{;b-_-
z#OIeXoFecIQhkItR%R6z`3XE6uvh}^0aQ;lqz%!H@2XaC2a2aNFl&P{+sFz=dOABbvC4+%~b{W^+Qd&P-oEV#~$%N7xgKSvkTcZ#ZwQ5V{M}bvhyBY9}`8AgZ
zV-hTpV=Tmv87ctD1Gt6@tV`fet>x5g{|d)Ig&yDslUB>I+k0?#QK&9{^`t<@gL(DI
zz&L6Rd^G63vvhxM-FtvS_vf6+-$-wB3EKk*ZBsvfvw#{Qm0I{t_{<-wLZ&87vmWPG
z*C(^at-szA|LRaary{gNoON7wdA=mf9V&Ll9ebqzxbOgTTpKEb3%N}y<6hIYQ|B6Z_~>T-viU!<(-zk7tdg(!NXbu>dcg(y_fz@
z?i;raw(I0eiswRp42P`-pEZmyhU07mjMd%@Hr|Yk0{Qnpk@@)hgA2>FJIZLKVt;_`D&9hw?5Oex$;?E=6~_~a(pIn%-?~6~^${}&Ro
zGqcR9i)*A)3}cRb9O)#KN#2LI3{NqPa33t~?cnRz@ha*tCb28@E}GX5dV4f=W`y(<
zee*DO8(d?wD4{-g`Yia)`qggCciA(Ywm9T9Sj*c`>u#4E7tX%)-{>SaekI)!XA{@*
z4w2{+j>}N0*IE&51ZL6o!RK_Q4U3Mgi*!!^Zm2k>E7P5i1w7DgBJ~0f9!%$bf3na8Kwl8~WUfQU>
zg3-}}H3qjc2AoHHvj8$OedNE%{eM`{03`6=W^%0^NKDdDjlAdt#r^Q<4$PNvJl$U$
zdd7MfhZ;|kQm!DW+h49^^NG_fE40Ho
z!gDFVxgVlrTg>ZrUf~(Y`mu*t4IWHo+38aD{8b{(h2}dk-dx`&y_ZF7u*MT^Lib2|
zGxSRu#a6zrsX=ovqS~M=W)_zysPwIgn9^X$YqEIwy$4k{9B+s0iC{J#ga1bobdE64
z4r#KAnYC&YHlGX|MajaNQa1`TT^k?`{PMHcd?-Hp#}R%bzeE$`Cr&aFH8T(TYs8v@
zc~rAQc4>aJJUwbEJxUW$C2mHi(DsT0>E(EH9KT@qZ@s=JK@I=&5kL3^FWKdTksko(
zRGCscdzAiCSxAQpkPW>4?|i%
zpDT>kmE2mMs^|wmD}}rL-j>D}xUjEuHSMx0ozI4%)A+E;@&I&epfe0k7`aM4aFrGz
z%uU?MaW8l4b#<4_iUW?fv3;GfeZZfjUsh(=9e!Au%Qv}OM(bUKV@;%S>k~kL(tw+J
z_^Dspo&<6o5t0}bPW7#1z9RxpnBxm#Kx9e}t{V+dhDs|eY3
z9`Jrj1%m;R6pPnY59P^fXWv&Y8adZ=6^S8%zan>Ead4EcI=GW)&YFiV^=Gm=k&YtI
zx?Jd>E=5qytLplS?rHFE@Jb%vM#*N5qxZJLsFP)#V(~{QddFWp(}>+FP@DKks;1x*Y3t=tr`_
z{NH50xA5bJ9*-=tFMwJ5QuKN9`8%-8&W2whsQ{Yv-5T+imm)ov@qYlLH0dB8;bLdI
zhv&r+zjUw<#)(FpQjST)u=lsU0v
zIj<6F)-n5Ui!JM0vKLu=^IbM?9k0R%{A||9pT`oBIv2%|jGL`cMn9f8Xg#d8qX6Tf
zYyo{tvLk-ePnsAvs`tJpT76%Huj##;rw7;dov%EmxH^;#?PpYpsT|$kVR*T+t)>VC8>Tw$B@uV}G0r*MTf?mDcIuz=NM2E>yanv4qKCFOkIZGx&YFiu!W^IR7HI=s#{mI;IJIpu
z0GX@((pgexU)?Cj9rU|MvRgN&Iyto-XE%s`zLWiiuFo0?wEz|OL+;$~(CjrNwyH73
zPTccAWF7CK^EK3$70sB(Is`qhl#|XoE00Z^e|61{KGDUyMPyX|09PsmkTljN17ZwY
zO~`y@ogHftB;I&6qlk2s@?rNB|7p)CA>bl~ik2)d(JApYR?GVDSw`?BI1p?$QsmfF
z`O|e_NqOwpc}H^U8
z598Q|T|&S;`-kP-(d@$`7#c8W1mD9J=?EBeO@MxO$9kO)WMP0~68`8#>1kNKplbt!
zZiq?v87K4Al*_ahoNE`|$&3fCUszdSl=6z@4%ZO0ua@6PVLWlW2arv9?5}aT*
zu6LuLjHo(G1yjiZ$Uj938O0)v1S&7;T4zu)a7yA{{=53^y6dBXDc5d7Y@KgP|$&aGboYD6Z0SRH(A)RIs@U!M;c08q9^qG8uR>nXZ`Zv+&)
zC?J<>pk#HkZ6EEi-BD|QBk_o2-{>=+C4#Tti1+G6$rs-3O0g(pHMcXI!4|JE{X5sE
z1-GxJ4E?Bd28TOu<(WWfIgP46QOwlTlPAQlOhk$o0l$3P&P1n=koR4paDi2zbP2e7+jEfD1)K3>S3fI}Z6lQfG
z6cE7sA-ixq(ZPF0y()ZF4C{%K_1+Kmp+qBF(@dRFk|2A!Eb@oBCE
zYt*zn0R6sUeQ}vsW5+PR5XZ_piLp-;BS+1Shy*fx0bE5*V3$J*_92<1!_T5ouI
z+eu;m(&ee7)@1Ri>hNRPe>E9P28Usgoz-XijRIH<;5P_R(2dUKb&TruDg}BI$jC|w*Cd{36&>c4?Pv-Q^5@^NpcQ?1m6&dH8YR>)vp+z;{>o3#{+Lxu
zA*2%EcPx_Js(vH1hKM=9NT(Pp{O;RtE0sE}$^5b@Ru+bg@F?P`P-G%#6u7pXMty5d
z08ad>W)gD493pfuuzo<-Aj-goanh10ax^epjQcV&JEfo&cP;ip^lgI;frOeZdr=-;
z6Y#1@>;I2b{MPLopOU2PY!Veeg^~*(YXy#-PlQ!I
zCv%;IG-tK2=q3gC?tT3DiIC{4_^0fXSe?v94TVva*(6DuIwndovuIp^Z3m14k@ME$
z-krv+pEviprB|FfTVsnBHO&6woUo=s?uPI
z3-9+FBcIKWI`L%Wr4PftQC)f(YuT!61je)k1*2ii1B$Akm>%G?woMgCMDn48Z-
z@;&MBY}6KlgBw3wv080gQ6e;a%B#tX&l8_mDA`xoIUt^Z*w<>KbybGsHLf@3z%rFv$;2{7pr2D%t*MN4Q?=f}M6
zrg~gk6i7s))PBz8o;tCee18%j5)Yl6?97M4p43QS(lrzc7~G~+@w~Vquwboz(MO8`
z1cji!y}*e{>HoBZpd4#q%fM&UT1IS9)k0xNGKddE6Zr)Nw*hQKhc66x7-o!S2VU?F
zTt|WJqR_8+3j+Cekv$d^%bh2#wht>f!tISo4M=}YxsB^&pD+eB0X@?{OyVvu0V{#**F@Ayb>vr8
zq#LfrCWucfxg-R;Fz!r@Oo;`;P`23zRc(2b%ntsseiSV@fe4TnkAUoCK@tU+!_|3|
zZW0s?TsmE}6+Gw_wSJB-NqSO{yU{wkn6x`w?2{fF>ZQF~9dm4qw<{qN2)Gu|bi^B8
zyl<=1L)o98|BMg6z_MW|yYHpLIxG^aoXV4J$fFCct0*Z;nD++!P=FbzHg&SKqh({g
z(+3b4P&TVvv`{B!mZc{!pae8yW`@c;_a1
z2OrmomkV2NR{m6(?>N!f8bcZ`oY(@zKu5Bc2!Lx%20XXSB4%}*
z@$+5S5r-b;<#NtS6El5hwGs6Bezw}?L$SCNGH1=G-yn4yX``S=O{MyU&8xaK)%PB9PqwT^EAIZ@s
zH{R9Ro5qQc>~39Q%kwc}M^0O;+`@CGRp~k;4TUkc=%_`vA$b4$!wwcP8QJBMh3Iw=
z2I;>>#%%HA-F^TZ`yb}lLiFJ(roT3>;t(pP;LIP))VX9kEJGO$a&o#!5X2x;ONt!T
zWDcNzQ$+44CZYZ}zR-S)EX@BkrgM2m3Tez+!7qQN2??=2;s1=9KyE0x(ZFx|E{0;3
zNPM{Vx-9YCcXSIJ_F)+byTinLruM+0dk~%XjgdGD$jEkTpPRRzdxaX_`97dbCF#ca
zZO2JbigrRn*zlqQVUQB_#RC+@o^ek*xOr&TW?>dnA7WDaSC?Btcp3;&HT9biPVaoP
z{a(`yO+LB}mLWS23X0z+umbwk8iTMj@Idmlx005g1^Wz;>XCmh9yJjtHJ`U2T`_~Q
z#ACm93)#g$p(xwa1VUuo1MXn+dSf92P6A+aI7nsXjoG6k$xawEJd-3uRzJ26K!14B
zakM~9kPds4!+3S#y!P9D;{(BWs~Gcz)n0*Z$MTss#Z|ji3nxJ+B1+f^U!}8lqc1XG
z6sE18Yu9iqAT|({tbc(Yz$pChVh9{@#3Up?iVo~086=cNz8@eQ0T`T~fKZ2L(yV8c
zs>zyNNF}3GPLs$XvrZx^gsy<&&moR?y_A8Ss>e|3gK60;JC6zgM}$DK2Rv;ip18&7
zjL!SaV@Yu(cfRE1`%~ad3gM#7ZS|d==tjlXKdIUg49T}bt
zF2HpnBx-2y9ADuxZYATaU_@(X!A+8%(#!pcj8?uId3ni6D(XMj3~T^*mk5eDR=Wka
z>oHExbO3Eg!1;dm`JYX8DHNz7zM&xzaN$YoJCJnZP><%AVwlJa`;Fh{-MI(M&;Qio
zR|guvnO9mC$pys3GyiyTWF}Y;L{xc@(AYL3Fp?5@YqvM8)gHP{@swdDX~1<
z;L=?Q?`eP({EIW13kaJ4CnFWY_7Z6e8b2!ViUt60=bgm>^n_c=dHa3Q9Xg`;n@gCl8PI=x`Hs+Vhbl#VS|eqdf`AN
zR?@enyb3Wc-OL7Wfm+qpy3f@*LVj(Ve=icEqFnxQ_&a?77%zZ;J&3)wFBNrA0Grt8
z;_xXR>W}}y|65#XdZ1nfcS`x*!n{a)i{xQB@oD%v`
zW1KY4gXb_z4yF6Gzd#T7c9|%S{hHg2Z)WeR#w!vJKVBb}C+jfJ-z`sp)Vg=gn_S7c
zPr0lU2A37t=T&9x>#TQo%h;l&+x3D3$@ctrVBW`E-^9Un!^)fW&pV)ftD!vDCw9`A
zre3h}ZR=SnfYV<&grDu-gnBf4K4twEkIq=p()`&0a2JvLKN9dwGB{e)Kp<=PW28g(
zQ>cmvtc850x9XEc_HOQ!xV0tBf-ahk$)^N!STE@fD7mi9uqJJC8V0yRw9NGK(rCXP
zTfa9D;<pZ#=yu
z;xMQqx~xdyN?IP+$M&rDRtU`^7>cpW{vjA*fllBiXnG1l7BjM`0%FYjM#^_CuVzpc
z=FO19#+L4T4t4#Hsh`)3Rd#&R_HQG3hkM|L=S7#fl0c;Jg3FA9C6!g9YL8L7oLS4P
zWJ;H@Ks}EQ-+>z=D!DAHFBsJz%wkHpRa~~j&T6Q$R^)e8&E-=dHoTH9k{Q6W0I@b#
z9?Hp~*ZO}o^*Bl60NMmlC~I}>ZM^o5A!);*{uDxATmu5f#ylo@pcM_QH8z1f(7(nEAmnvE
zT^BBzwX#rJ{y6fZJ-csOqUDzV;!f108?HYFuC6_#HeskKG6YS})sdN$U`3wjv-5?b|Uyiu!Oox5#enOM(
zek776gD1P*yitR#->)R$u@dR7M-!qI#u2r|EUZJnPJaC{a
zk0V*K53J~9QPe7`k)gx~1lq1Qet}o5qv@Qr@L;m3%F!wp$q9l3%--20--lH@^YSospjOP_|
zc%`cLCyNttA|{ijJ1Nh6gjWQf
zFl#d;BI}lmQS=>X33{<^4JPKZ_ez8VnoWeK4x~Ho^O)Ux-7Sx|*!ClGe~
z)C$HFE~$W4(EBF6wmf1(ZzOZpNVrqfL&Yt#ulJ@_+MZJ0gqL+U!!5ULx|?Y+H$h+G
zbq)cXqetU@_)M~~1J1OzX-(>}fX#H0%UzDX{|qAGN!}sxoXfK^Tg>6_C*?=~Bz5)PMOe@Tb{(Zt_*Ej9e
zE3aWKG<`4K(Ds86Wt7)s4{}Fu{BPn}C2aU}>Dsf4Gg)TKCz?YM5V}kEp+XGAduQz-
zPS|}sXV!=QLcdwjkw9G6WvU4qzQ@Q%e<7eC-04_~Klrfki)x)%V3&u{{Y=&O9$qbi
z2VS>x!z}h2?p-k1brVSk9g=tP%oJLhGh7}-Nd7cn4&4`3ZR3$5o4>%OK
zA(hmb_Hz{#wDWqjZ?E$^uixwOE;YK_GUsx!B!zaVyjfr}0|5;UGf>S^1%8|b(3H5C
zenaqn>jYW)|Jp5pJYJ377^o^4c(;r$)%&6HC&n9n>Y~obY7F0*u!#~$xnj`;IC{u*x;Acn?CQwflhnJF#+LmLiwH}cxNZ+90beLNJPkpk+Xf8k-u
zZ|gCi$(OK%9?=y~d)6l7X`cxmuE;iE+41w-6sYB{c27&q5>9#YHkpgoHfk2s)~!-T
z6?CU6r6)=yvC@m0tz%k&mCHg=v^OgoTiIV4Ez{MV^0pn(8x$OG$svBIIw#un7Ci(=
zhp+gJ$W`!4iqTp=_Z7KD5imDOS>{qdE|jwIp-pO`M2P#H#a~@Q69V9`rA437W(3Ld
zuWePvk))XI*K{;-77hbPb@x^R6Xgca`AicEzXA^@s`Gi6X~ZF$4Tqd$_Gv|yg~0T8
zSk9Ow?1Gz>4Lmk4!Ehj&zE@{S-H*q#kJrE^Xg_;(d*UEk3&kjtD9Zr%bLHFg8Fhl7
zM%AR`yoL(YzIk;gq0$!4`3z;QL?P512~Dl}&wOXVeG^Dj05lJLQ$Va)e~<(ucfN8^
zD5)p8zxrUx{1ZjyrfRI$2y-f$`^n9vll45$;sE(l&uz{q02<26KD4ZdRhS5z)cs#M
zTySeIi7PQu?C?z&D;Ahi{u!Kr4opYzO)d9$1+sQ8XHDoM1Irlb!OZ2?2}^*0O2Cwhu-Q16{ie4}
zaR9J&c7jZ!ZJ^^3pH8CzRoN1FAPvjtvBP}j2tvW{Q3V)J=X{KNqENK_;*FIeYE=oM
z8n;+1O9vp3;N4>>zb%#xHho^+^qCOTwJM_*g@{Sg?|X8lM%4*}N*N;Sayo$iEc506
z^o{P$$N|wbV8sanxF4{=I);otup+lf%zKKd<~ER5NdNWFS>GClk5}Rn3hNR-*D9|;
z#_JFd1%6@yf#CAKzP)-KqsgN;Zaxo761|6?{r}uD`s|&+@9G?sFk{0Fb%p;F1tjzP
zCe5DD8HMrS(d*q9sc*3Qyq&}qB(S_BjV$bAYda|8^=OJ6V+~9TN%%6rlhXA<){=+6
z<#Vspp$g}KfudZqZJ&aG$MSquul3D-r^mP_Q2~zHwIQ^ab=X2f&2D$4&9T4bnv#O$
z&8g1c$C(L-CVofzeMKxxR?n)nro;UE*6c$cyvw7V=Bm$DA}iX_!hgQQ2l*ds78HCU
zB3yl4A;Y|(Gg~|!<7Sfb0#&yWDIR-;J$w!$rY_jM=_G#nXZ-(vDKZ7ctHKZ2{`c&_
z)fc6HDHZmZR^1IxppEN308zSX6EbWwA$q@Rro}@CdED?N=@?4mhuzpoD#(YUI;d3765$@MMy`@lZN?`We2;cp#
z7f*?DxJgXJ8pHgfZ8mNXTBbKjf*zvqCwJ>%-A+YslQ|JEBRPz%VkUF>7#q`uHy4No
z`kX##bqQ(O8BO1;PF3dTdUHSs#B2Y$=jo(|HTNjMyUuu8A)@b&lB09tiR>^0ra7l{
zt(F4(S2SDbSs}
z+8iFCJly!n$SSYfH-C`&SCnZ7(isx~p`t*}JO0HoNMJ&4Hd+WgEDh`q
z`P1hU@|bFWI`yA_iod3jj`XwnK-TrPg&;?w`+E{%?h-`6HjSnr!klXqf1f{JZNj
z)h%vqd$@|@DMJ*K!j*Syngd-m#PRu!i^bbS#6-&qH2(@*7tJUG)q*pZ!w{r_TSSrg8y?l2COgJuUCd_
zwXkI4yl{6!Y6Rvs;;g~{)cn37L0PK;4cx}|xe`4^p>M5pbK=LAXTb=zShp9+4hFK;
zE>mV!b~ez}&@k-q>;cdogZQ)?ByqncXcTEpXP9Jc*1~a~1iEa79PlJ;lvaGI{Bsm2
zyFKRR6V3SZN9I6Yq)M_gS3i3JKrGO0+4TiJk{mjom{b}65~QoxsWZMRyYn~;DR@jN3V>rirb`g--eTs_EoCqDK47+jK>s`io3q{H8~YbKwy3t&
ze*EphoY4^SyiW&3=wFz$qXofb%(BfXSBraW(n;b7@^4KL^ARf>+kL;(8`rNp3TFfU
z_<_FH82^W{vuuj9YoawS!QBb&1b26W1W1CrJA=DB!6j&Lg1fuB4TRuL;G$*J$|oUR&D&7kSXf+WqSQmtqus-HX(EgfwwvQkb5lmP=f|vZc)dZRFVpc
z#?LVJv4lccZ^!_?0jxo0&v+rgU
z%Rb7b-^%SK(rtZqy^2q_8ieF~ZOb2X0-Aj-_SPx_4SpWk?8v(!`!y9*SBoYQEc()H
zz3JUQ^yaW@s9tUW4HE2@LchE39*cRu)G}L`q@RZ7PCOl=zO$``X()(lp+is
ztB)z_mLEn&Ui~TWVl8R;I!gw`)a+$Pi9F0@Wjz@`aQHLaVz;8?-tJ#qJW?KXd{am?
zZ(Bt)!ud2XhYWnz;
z%U9jcg$B|Lv9rW=6j|8Xxa@2Ru^muIRX~QPbXvOxH$t;Eb2^ZivS_5x@&yT>c-SUp
zw#jF;^4n37P>?AZB7&I#W&D1rn0{V&yL6WpAl&L0e?!w*L|9z8Gp&+Q2&rPi44aIT
z7k!O+!FUfJ>v_3X0z&B$Ag}N`7|5gd)bl^o3P!P8&iKTwS&plW+sd>EXk`uutrU=>
zu)o_+O_fZ{x8$YJea3EF0P6qHqX3X9jQ9Qkh9k}0S
zCCcyVK)h$e^2ODzB7i8f?S9xo`43U%-6d-~;c0nlj?9TQtfbO}yEi`H2kshN$
zxIePL{t`}Tc1Oy)yIXW;+)MOy
zqDLjNm;4`_ws%$Q#;04nh9sc{ixGL
zYWMp0STym;Je$~Bw+u%J(VXV?VYNup=6)LF5_de1(5p`dN{6t&X7>%0a8=@p9|a5m
zFcUAourij<*m_#q`vO+?*lwAYxH&KgQF6r&xvP|qqU0>a=3_5(Lnr0GutSbROh(QNhg?M#9!g!Elu~Yl_HR>u
zy-Z+lJOAX$B~5p?F#LzxT$osmMFYS%$cIPvYc(+M)UozWT}KH9>$msosf5}_Dr}6t
ztmnE_XM&;UYNjxPK%`~p=aAnsB$u2`Jqelw#TY0Ey3s+x%S(wVzvWzyBHcqSii}~s
z{d4V+{llvzVtJ#;OiYuh>(Dr6XI#GF%t~jnEH2b%^}B4P)S_p4`uZUvC#E`+VH0%$
zNx2R`Js?=(+~#Kno`>H@#yxsSC3}?Hb^EiI)?CkV^j(pvo|6G&)M)_Hm@VI6hgi{O
z^W3w1?n5`uN-XNwz@{nD1-%J>ibt#n8=OP2v?WNtM>7T3!LST+y{OSX*u;o6|Bei}hd9L|%Hgu-;c>K-
z@WZeq6ePxf+&D++JFuq_&37^G^65MyzIDFmc9NToA8#@*=qN`2Bot`&EU@&TX
zg%P0WeFd!`TcfU7hEJZoLmG=C+Re2UO~Cy2IFtmM=4Dx+U91RpQ*-KCb{1TBqdDh$NH1T8Pp{8m@4h
z((6Fp4}IhR3%P#qi$ZPn`AM{GzgV!42`t632&({Wv^RxdOWSPo(jZ-UflO#)Auo5
zI;1@-gX^mN9D+9e?O(?eiMn9R{xBT%XrZ>EQ-K|u)VR%R(V5w_ZfrM=S~0f;R%~})mG$sL
z>YAp$*sj8VX{f6jI#X!ln9^dh)fFrFw;bvYb5PE-jE}cKT&ch@7OKEGSl-g?js|8}
z*9)vUIGK6a9jwN60cx)PIhd
zSf^+JXZDcE#{Dc8XNa(}j@$gAIbEvfWL+81WaP_B03
zQ~pf!58Y4MDv=hxphhkK
zNo^^s7)9x+tU7e=&W3{w2rlCc3CTpDK9qaL%iGuJICQsl%V|eOKJ-xoOO4XWCCG)s
zyG*OKT2)I9yg+uFL=eLZ9vS6Ne(BtQ6=jZ`s&h!P?TfyiC7T_0&(Fa#g2vi*j1zzd
zddXoSh90nl0Dv?%I8DA#*`y++5=QOJ#8_v*UvfO31jGPntMioUuCb&NqCVbv2+^+2
zA%#%|QFe}9F6vt0irdN*xYPs0U#=^)X<@@!y(+D$1pxQZpv8|e1BfKTz7K(ObN|t`
zjZtPEIfR8}i|;qj=r115>r~#zeclK=3@C1^*faz6za+vV3gsv&`h^hYGvT-Lmht}5>UXD$LYcnntkit#m`|)v}6V%8Y`+?
zYVfQwW#Wuh#%8n_Q0%lNbMi%B5_XaelU}#(W5nK>>z~EsDwaK}E2L3IwnGcm`m?S3
z3zK64M71w(CwRcs&3e^}8mmdS&1QA10vS8MVqA(dIB1<3m8KpS3by`gsgTDVY3!_4SNck3+Hf0T(_?dPYrQ0*h
zN0}pjjH`pG&u{S-j%lIx)&9`f?;%};ju=2T?OCgvZWV|<^J)%nDF$>}GBOc3d~ZpZ%M!l{?+
zb!6|SvM#;`&D93=wgz+>*z7Y8EhreXdvs-T7?SSIptl9rh(3E4kX-JD2a!}xt9>_A
zmxIxo;$c&;p~nHUkCsp$v?yxJSQ!`Y-@?O?sD3F54_;2tDd_IJPs7YEHofaR6b%uL
zB(lf9{FU4H*jS-z$A&|eejn0^HUBpKl7+7(=WPUiIVVKgYacv9GP+6pq?8Wsb%U_F
z_E$H8N(AEtSg^(3W;v1n*U5z@RGdr=7z}uQ3mCGy1p`@^$Z@$!GtSZT#p*OCluYq-
zb;!iqwGY@0hseAQmi&HqU%i}@MKgYe*k*O#!VK#6JileI5A$+KA;}bUwYI|jKuF&H
z#%{i3*UsLvCzdI+=W}V!f2BX^g-!~392C79eT6%XT3&MA(I(LpL+#Y}G+^9)u{yFUiNB+}u$U4(
z^Zl)$GC%&>!QT5Z%G%Liz|QCL4=1_ls86?kQg;P8>R;QL-n&5M9`JmlTFtHO(!`6@
z^%g>1L!$`BN6En}4o6deF~=ZLg;%xnUAck6-OWVQfKfr+8a{wJ*qSrtB%*6vXWU4Z
zvhmfD4wkwEKUh#^Zehm;8kved{afE!(@htLbugc`@ei*(l;2`9gyXzeK%`4dCh~AH
z$5FdPB0(NBv_=i91#n57wEV^zuA67ew=$1P&q?FWjM7gfO}IL$F1tU&2#@
z)8?UNz6M{Bns7?o%`Y+lkKDYsyK_~MS9~uoUoamJDkIVUHQL*aLsWooSMNs%t0Tvn
z(OQ|8x4Id>4bv|rVp1Nw@fh}8gYEf>#fQ0jv5l!!53Ns{3@$-zU*a#=0`7U5@4zizW0mM?RUuD61Em-;h-H~Hhoef
zoz&aU0@S~867fjK>YVBg3GeigMP>a>+z~^zJG)HgXF5*XHb<_gpIQFPW|Hn}Gc=DS
z@Yb*GS4@1pvu1pd&nm=tqr3Tj9p}i)F;{@!v;)}Y
zLcF_s@sQ;UJ#Sc6XwJZWVu$v*D0&Rvj4Dtzc-vzA{*!;R&fK{(J)Io?5jkd&9@6-oZVh#Ek~Z`VBi)oM$Hq#xql8krKX^(HKTVKcJ0G@izD}ycd;ced5^=kpc2i<)OF@vse!2)&ze_5d^DE|qn
z)F`I3u6A){!TMO?RaP0%n%*6cHRWm1IZT#1)%1k$ZD)zd%dK7yta2`QcAU?w@S5MB
zt)f0y!<+x5Jm>d1k4_5(|F1%eeDFZb{HT&}BSNSds0Dj4BVp2HJ?rKS`l|np2TGY;
zM5s^V)|OPMJCp(oDFrP&bh;yFjV4@U-3x9SVko(XmLu#Fw5yQV(;R6@?lXZs`{IK&
z`xmn+NcA782|e$fz4cWMh15E7lsiRy=`Pl+ytFkn6{gp{z?n<8ReS);k(
zF8dLepDiR>bGU;$JrlPbgY3ymw~-{sPLURq3zw3*AetDfxP7lx7{jrUp`2!^pRsQ)
zeB^5yqUTn7Y8yS^bzi^UN&ng+gzl*LFgfLStk{4^Z_{Ptu4N-Z-K|&GkmE<23on4L1F@khQh_S5H~Bo_W9XUMvQ
zD^v>+?#XtnKNbYKx5?sfK$&GpRg_=$n*o_YoQxR$RlyMn<&eN|a}}NnvZOHss98Nj
zaIMy-8Fy^2)ck5&QgU(~=$qNuM7kP`JpI5L!yvrWPftke^^crO?^u5K7EjC;gyIrO
zBScoK2k&1CTv#A?>IPP8^cKU0LLo2U3_8lwSnM{p{xn*KF8r_z?v}k>4G20)3y6N&
zJx`3yl{AtQ5vI_n8hmUV0t)Gcp#k`E|C;ubg{@tYbk4q2Ikub|bY0Ho%PofI?SO4(
zPiJrnHoctb%iE&imf`E>{=*bFcl*FGMn=)o2fnjiYukCf;It?QQQ+f>!x?TfRlt;l
zv;@Rtz_U&sGjucs%7k~w*E^^t4&ML$8mH6K+tCu{y(AEDoB;&T?c3u;&(|j&0DLy%
zc4SoAUhU+_-q$CMoD()m8JZf#-eI8mZfW(&Hl#NTEPZWQc78im6XI#$8<69-+E#`U
z@aC(Dl$c*`!@bRsc?n(KE}LWCG74+j^c7L}zi%rIrSa6_IIj{+^ieXCic@xeCtWFy
zot&(IL2k4`*K;+F!aU;EB5vN~$Kegk^)mJ;0@R>E?9|7?Gla#oA$#LAWy1$jk;`h}
zhkQEXYU6K#8#y_hCB&FdYFh)aO-}z@Uz>;iMGah9iojE;TUWgp8vW|t?7P>7E#qOs
zGKZHh{ioc4@QCa)<1Hn-4gIjED;+jw22N*%apJ5SYx?i$2B@OHk31wKIW1Ni&E|`T
zf|?^D-?ybm#hyRNl^Es|D}Qfid9A{KKI?BmCJFBp=NoFPx>z1`^N~>_55K}+YXO@;
zlRWj`{3C%m#j?N=}-X~VB4C#iWY&+!SSHetS%*H8h4SjiJYS-
zw{xQD_yT>Kx%X1%hiko_tEGveRVDQyV0gCh(`1~{jTDmHI9?YyOzPoSb_lY+oHAf)
zOdz!oW_Pn0MqfLZ&1%2=M@lN(be<|-3R{}DAlOY{-OR?u(9IOf#=;xnl-})8Az`)2
z6?kY8^wF><$gI>WyV`}`7hl7!lNJ-dI?lI`3;OATOaJ^S!=6?s6fFAEX6(6i_||Xf
zGQ%Wej1UH1=c4w@ZB(N~+Q&KV$y-4&%HcIC4?+|m^aoz3y;_CV16KKIUgA3_`OAED*{N^L8*sVZ
z5rPNpM*J%CJ3~8FfA|m6scW1-5`J|-)VqQ!1{8w4%`e_@6uxI!yOod9LIjIlhO+6H
z>(kMTCphSR7QN}jb#y#dP&!!j~(Q3MgBUNnAA_%
zU~JZ;&5j8!NO1#(1U4$(K%
z=04i=VkFE&SQXN6epe3{W>eK5;lBOQ@JJfHi>=>*AQ6n5Fq%TIOUT)B^-syjEP;s>
zw<$y0$CVB_-`BD)RwYzL`+#(SKE~q|0cF{wVcn=n&xD7(kSJbFo!ZRj$4))VfEBlF
z4ngTL*4{M{{xy+C#c%BEUe8=&Sz)`CXF4$7PwSdr?fJ8y2#m)CUrH@)+|dS9nZ&|s
zrpY}QB}*3;d_|zwHLcSEl4rNCuCM`22-e
zU37>^7#IewpvjwUu)t=0(~q1nS0GjV8oYwPY#wdb9oYFxN?5-@L`2#{D+oR)8g=Fr
z!i147?uFopU_;bu0KM-rv&wj2hpZ%Yp%bhHL!=Fwk?pXX{?8Kxo?cIIqb{B87POu@Zs}Dt?O!A@6OjLG2`3*-
zRk5B>-~Q85(Q06kO5CRizKVQ#gL%IJ2Mj+x65&IH`!rQZfRcN$;dVnc>*a-X?hsWvrr3HWW`vN>G$xDKJVOd;Q(EeTjtu5xK_o}se38>Zaqus}FpA|~
zm3tNR4Y{-FQx*m%+uUtJloFZ6Yz&73U8yWJu8bQq<*j$au(Rlho#>FWC}8XdtdOwl
zhd5BIaHjHRM5^invfBvCTzyuqkyJHKkOd0q@OD9Q?e7fZAIy1%eEnnRO+u6>bZ~ip
z^>pN0Hli+AwGhd?s`&|t?UiPn{tU;8%n(&Wh@bkZ8moJ>Z5iNqzzQCa8X09K3;F%m
z{$E2rUxt<&KLvwYAyz68xbkF`Tat}cHl)rvx>9=et3s{1eA0&z)riOgx300^lp_*)
z`HqCN`Vh=eXYRhuIX1_K6O@!f4k+<@--~W~g9fq4u7juM>n=plSxawrQmc8<2+MHl
zDGAOo7TTSW)t7)(;oI0;3D93Yce%u^^&+--IPnK5-A0V#=(uOje=RvnIRvfxUs4@d
zx9;UEK!3c^?!
z3=oZd&5}ZwJ{8(JJCpC<5B(n}mz}1jHYs4R;5B~fLV$u6{}^2y0Xu{G=Q@mZXmxVy
z!Mf4cCannp-n7u|0fI~=aKhJhJ1cNwv#=}_5PmXRkBYcWOt84mUuqLsDQ?NFEcl
zBSIWITH`5#{i;cqfpL?8$v;w~zuXKzU?flI13&%12C`}^nE=o2vfYnelb|V!kT21D
z-$fu*E}ba(kEVSDrW)#D^$eg{g
zFpI1Q%Y;`667UpZfmh^Zy22bYFzFzQbvAKOF($~OcNbfW2{f02QVMtTD`WFJNj98Y
z)s?$8KBipBXuzA|cgQPRx#1gGc?pDSf@e(OYY4S1oFe5B|-wdwA74
z`ug5QPj89>ZgVxar}ty`Gtt|;;X4e({~E?CXA_8zby?dwX&-5ED>}v?d*L=XK
zCu2v1ZELQ!nDY9R5QSQVVVOBq?D@P0bp5Xw58Ju^(yP@Mr_p;~s0!m_o9*kgSRDQR
z;}&ZSx2LlB(x)t-pbOQ)0PlRAKteuCqNS>f!J$Q{ML79i@1jVGExJdt
    PW&idNn}XeT)2+b!9#o_GAc=Q90OqO-V5
z>>uL9y5@navxCa80F$|jV`tZgsu5R=1QnJGdcjbLkW
zuM+npOM8-h{wZ(9pOsG>_9}#+IRKj-NXSB@ziM4Gb5wRB7)T_;Gu$_(NGkFQGjCNe
z=b0{285$Z7J+{v8k6|YRsY_VGK17$kyvKch5;bf+_91P)2Ovi6C&%hOTxTc>%k%Nq
zJFCgd-TgCn=8(4nh@1`DwaLe&3
z-`?lE+f7`9b%fPW#U7CVLH)rn@%jhMH0zD^1>c4mnIn!7p&}kpBeznT5V1mDtcm&O
zmG_HxuG%%n^Pc-lXnfwv=8JQenW}gi-!Jn2@AD89JZnt*9(uS3FjlY-jfB2AhhwLfwZ_UoFqO?t9^|jJ
zLI=eO%d-3-W$Z8BvGC_`T)P(x9p%f^)7nyrOn*X1M+3No8b*4fpdWKbSSxr7s!PYF
zFYFEqyJ1`Si(WzuEt^kgw|MPoB1sq}ZhZp*ea`kjVI=+{k(#rPZ$7E!H$bR$^cZACt-P
zqu}2hLOu)gj#p4Qv2k0vv0%1I-LF+IWi}q~2+Y30HnrA+Ca{s-O3B)tSvg3)P1i@=
zq}2O6HQR?6S?Wm(7XZ>VNucPe9|2e}fxY0)fOm2r63P=$Bml;ps=9j5&@A%BJoMn@
zV(($g#>Y3l(U0$3JG;7(?y%6vbos+R?D9T_u+XFMG$=;2i9I<|dNqA2NNb7GX~%8!
z4LL2yMs$?xl@ekGL7xCXQj)ov1s(1eIE#0A1i`X5WN1NRsh5-eQvYWU!q=n!hKaQG
zv&anYSg%+d)toO!ULkV>A0T>_ml!qs5Aka-yY+q1HQjvy&zfQb=l-|wvW3JGUp7uq
zJzd)RZVQHrliS(YC3);!EkiE%mJO({_aCKUB(2!MMw0|CKmr=D@Z{ecFV;Q0JI}My
zJTSM7Y#=JsZ>A?g==3gS)TXxuu$BI6b%87gM?XJ%j4+!_V#H+R;cbp_1v0q2;%aOi
zh0=gTY!>MXC>US*pu5k87f?pmbQ3NQ)^Q5V5NJ;*Yx=`q{n}~>tcB#kRYe2L`^-Q=z6~!OB2-P!}JtF
z3+}#ZB|`|hpZZ*8PqazGMI*G?u>Uf53;iViApuUYY;fGWZvAsqvwh#B!p&rLGP@Kd
zNlTr#VbpE+&$g;P7WW&d4BKL?R|d1S8O+zRzv#RnWaK6
zu@?e%JEwy&Bh>cI$8|=CNWbZQz0m3T;{~r*r?qc12Kr0K^pZ+4R=)i;ehL)CuHU=By07dX(Pf-
z+bApMhEL(OJ&GFI8wY$7Ukz#Saq<7Y21(LnGe#uSK(G6Ij1q2O$Ny~PBOW>q9d@p=
zKaXKtZLcy>xt}zC+;H7C$%{tC8xp9gY10XlyjZ-Hsw?tAN2}JSlXj^&JWO*$YP<4j
z)h)Xm&rhpDK}gAEU+y5l^HQjJ?7{GN%XvG>p8BKpztAARul3+Nt--msF+U`lO_%@G
zt*YQwb@mp#%CW2lz+A&j(!%}342B*j58Z&cVQ0NyyS)-m+#=cg#0ayoWR!w+uSMB|
ztqu9fM)g@}vkn9KBC(-1tXp_U=RRWeFLgTU7ppw6rx`cfKLIzc_ZxZZ
z^Czae*^#Om@K;oGZU?V{MXM0Igb(w!_0zs2ogpUzC|mc
zWMm+dS=RO+=AF+C9;%SO1+lWwYE00!HxF4f0Wf>%Ii(awKDN91lF)AIQ|NYp{W|2!
z(cqo|y<)KgqJGMAA0j*%?&^zvK07{c)fP)(pN-6kM3PCALJE@f3Cx+cNT1r%Vo4oG
zxv>a&^o;{9cz|D>6eyAWf`XFTcSv9vXhF}QO3^@JZ)E>F#f^v_#I8rVjU%#B{fhg{
zsK9^IR3Xv`*idfR%XVRQw3O>^<1PYH$j|UWT7eeWTfWfkI#8_4K1B9Yl$E7<$s2Ac
zsbp4ql+Vk23<0;ArT&j>s(!aVxQJ|&wKWNE15`Mn+jMf@Eci2a9+uU@cV95U=pTHY
zykja^9Np16+m|*>7L1z`ZOB#j95Y|h6*yspj9Rm!Sfp}xF@t9u7Fs<6f;4LkfI?`v
zdAjP6K1E=4gbjJyf`7YM(|lNkyra@bOu;S;ZyPu1Y;JAA00Cj1*L!F{Qwr?=K||eI
zg^rm8hkOnxu1+8jfHU_#JE|F*6jl*T{LtB!mW?RH5goM~&vThh20k~vsS=A34|&~>
z=l&wN9m+DHmI4oGVsqT3a;CU4P8Cf@7C;}sG2-hUJLDU8>y1bH@{=94A!pJ#@4s@0
z!#xFE?i?&kvMn_Q6UCWr5m!*Erk?iS*TJegitn!Pt$A`bCv2<*YR$stwKaD;)inY^
zf(L>S*b6xBwusfKV;W;mg{aoVR{euuOx05Py$ep{8JpyFZtpF+X?}|qQGL6XWo$yU
z`tVF-`%S^M9#)I&2UTOO&QV+N;XX|RpoG~>(;%}DO=5d#8#`aY{5R}#h!dv@**W`C
z^Tg{?;deR*+xSkh@8+{>dLQyS;qo3BzfApa{BFAUuZtVs;BkB39fKg7<=DVIH?b=x=z0|5Z^TrDos!Rnj+OO#)t!y
zpL!{j<2ML-(&j|tlp)u|H{;Ok$2LKvbu@|7WA9tsll)^JJH-%3KkzUMb0XJ3u|3B4
z3?`kvkOXawsJ_k}Q|2JC6R#2(r_}hT6T=o!7_vninU2e^KSN?>ypOX^AQCP-r1u!>
znk44d_cm7XORaUqQC49r%lYRJcNI(!_SS6>x9=c4Qw{FDa
z&$9bvp>8)KWd-zZ0wyjFN)=fdvJ);s%**4lZv?y(Z$|U^bIs@v*d?EJIdOB;!zLd}
zr9kqxrZR#af7GXmh=c(iX`^szb?g-KQVhSADx;Me7qQO!Km64wH8D;=i~D()+{cSH
zP+H~rvOn>CBefOd@#Gt2H7yOzi@5|+Q1K_VJtA4RpykS5*Hq)~6^5Fs`jAY5QiJt}
zKod-zorKX-Kk?uuF{>*{6rq9x?m|c|=O{_cO0&#jGa&c@w>hg})0j*Ib#>~AyTV%w
zuQ4qTv~01L6H@xQ9z$n
zt7D%+fA9!1cZdu4aQH<;hJb`Wc5-I5ZTueyTZc7)A^UmshS2neFk)}84=>=2LxEat
zpfDkwzGuuYsSrUE1)}#V1VdVS+r#eiQSf0f1w7Y+_F3gQRgq|YHQ>JnObP!!?#)8$
z#kP_i&GC!c$^xnR-2-E^M_b$x2n9}V1bfRQG8?*LCqGM_LwsFgXtcogsdn_WcEoah
zSW1i%Cg&zHIro`#?_zS35|UXP_Grr~!s6n$UtNCDAG-s>fU2}@BPf+&`LH)p0_Ycp
z8yG!r*g!jq3RXdQ>8tH3?V_U7jn=LzE#Q)Ge)+J}7wM@+hVE!Kjx
zZ&!1tD|o2PT+)Ns@UQW__V&eVj#8X>gV`ndT-gmoP803Tys@|Way*(nc+1hdvJBDWi{HbPwVoP-NFdI{d8fa0^AW(NRP1
zx6K&PR|pv5xGwan6z10pu+Vf})Y6DYqMHd?xbWu(R1qc;I#TL@wC2t=!dRZyK$~
zT4@eTv8{ENlb}H22*(sS^Plc9b^km$yNNpI6{i=Yw%CNS>m`M1a$gI6>J-8l;PqNHEkmIl|gfsOnMdd>4_=4Be5
z%QV826XB=0hL9%$q!$lC#obdy&ZxqgoL3O%^nCLaNz%XPWeTkLy0th^KWxL3`_FwT
zEToq8In`&sJZ|z%Jy^YEIb|0_QH^#FiGOm>_aC{=fdLK0M`Jw#!gz(w+-fHtL(?Xm
zU^A2;G$@J?IKrj8-b`jF@z2a=f1y!bM=2ymXZr3D%txnW`A^4p&jOApL~5H(5%gF8
zn|;-#n{o<>&nc;e-f_z(#L+{frxvCd0n3b{!8sV7Jt3WUq4Bvy
z=hiJ}CHLb0;-61j?)4i&DJ1nZ875!~%K>o3!0j#Sf3m~k|LsGyHUO$JznIv_L>e1_
zk3p6#M39WM&)S9CdOQy-Wa|a@zz4ja_XfOSY(y_&X5mrrd9Txo>wJ`fF_JLCR`8t<
zlZ2y#&cHHD44KhB--WKrrEN9MOU_dgCncwR-fb!Me+N7$kB|c4M|`ih&CXm=ngue)
zHkx1hT8a%eyrq|A#lZ
zPVd_fngp6gsr&*q7tu92n-~zkC^6q>_-=j?X>H#XZXZTkfG`e}oXe+M+fJ`*gzHDr
zuL`&_LBKR?tPd`E49;Bp=x%DkHlO5Uiq@>BX5$4D-`m~KTFhEGEq;4tw&34Cv@0HZ
zf==%b*9s3jAinS2s!ty!bM{NeKX`Tbq*+=Fdw2Dlhz_`V#{yHw=Cvgy`q^vte@_M!
zV6MM~Sm+ok>={Z4yD-wra?_OJm<$uJJ3~lrB11P%FmuNO3lr$|$wtK@D|uo`r$MZI
z$Cp$|1M!G6>l(&r|jEj#UN$y2)unY%C5Pl7HZDPL$-;RFcXkP
zqz5`fqTvV?N@)Z)Oce~#i+$gk9fwF#v2Kvg^YBO1mVsVV_2;V7GrS=0YKMq7kv6My
zdqdMpav}fKw&p*
zz2wTspc-&4*htsB8+SAsfi2Q8K;+6+#-sv4nI5KWx}SgwE!*$}P2ME8$5VxFNA$c~
zn++DXaK1L4hMX4Xh??UR=guv*&v8GdRSvenR9=&%#ui)-*QnevF_&_^`&ol0|Bxz}
zAJeax-i@@dWASJNeLuJDVjgHx$JC0qG(wmRO6IM8+l9rH{?*%^TGlZGJK5Qo7b*kNH%h!~1*4v6XaqWU&spHH~+INMy{(sExLqib}
zr3!Wx?RJ=5eUT}oaz09qqY;v&3(2=ZCj^y-)($R;d~T;BTVFb^UNvfeBrhnZ@IuQ&
zhnw&V-V>#C`Dpz@N7a^s{Z-+wdRWOwM63ThabY3BXa(`9ELMSJgc!L1D?ZdxXWLBp
zce6IFNeVij+>ELVZMZUi=Ughdq}UqC#}hs+$zL}|GKBqUC+%?q$`=}W?hx;p%nLG@t_qK0hruTTbILUtjZ@aE8N7TrmrW?$BU51Rc{&e$|S4#
zNT6lqgKx4Tyt0eVqa!UX$8!Fjm#uV~t$(U!Djd;{_A=qnT-!r5CU4fsDzp6n0U7F#
zxIgDrFtM}{SXOk_Nt_#~z8(aCI2;zEPloO7Zc{-YL*k|HIu6N1@V8vc&eufqUJ*Hk
ze#yU}iMi@LTW(**Jhw_o@?rIUgoUG+a3)oP119xF+NOedTGJ&(KeY(TMBS^dX
zEXQChq}ct>li-<-T)c{3o$tbPQNC5Xj&p>bEwnFpaZA6n_EFk{TB&eO?Vk)iVVW7ou%LC#c
z1nTw--VVv(y@)cd)7k*UO3->|Ux&vzFggQ9^T-k~Q=UM#7;~vb#`|-(%>bOyR)5S9
zQc+6$dT7D0?6^*?F8s&X&vtd2=25KE@$x*s+k$}HM30(Eo)Ks^9lgAQfE7UfcvX$t
zFEtSRjy{uojGT0Eok0zykXs~TRBq0}a-~De!hJ!NWpn9g-4iP
z8u^vg?AZoJ{#}wC8qpx^Y7AJ+8*2{(_{+PPIpi|TCA4^Miu>57F`thpBlkBsfeO7y
zy~5|CZ#pEZKIH36><>B1x5PnEB*lszCYH%q)&oxk*46=*!FEqf?_b&Mkc-Sxn%z?l
zmhb75dC5P6&<0T{Ik_X~J+EGDW8!nDCKLmISM=xI-S_ZbG>4O82wkETfvhTa;&UxZ8B?o#z-cBS=BvbgBVa6agBG$x
zG?c*fjFe{ZQdO@V{21pd%xNj^d24@PtE24mk?O}CmhnAR+TAYD^`z|QH$I)w)6n>e
zQec-d2i0R*DFn%6GhQA_BC7}~*G@N%=pqNjah-;ZKlm>6Th{Qg_wRwnCR{$gIF%m@
zx=zrp#9{+QOObnuP+gxI9Nl$dt{u)|tGYuwUOsGMU@Dpm_-vY%7X5uGyCXfgX=mY>{Z}a8`_dol_<|gG)
z`}!Bjl)N#NnFD0U8*KhZqKnra#GXg`nHy3q*1Uh;HHmo8JaZs4oYedEQxC`kDWP?`j3{|S)(FPp$6?iE3E@f&ViZ(51U
z#TuKzMorH%4CL(u7py`zTJN_Y@)1L}G5|1F@PXEWdL@!N0UwD}s1`fs88cSYS4so~
zHenu8Q<)KvO2f6bH9eC1r1IeRajnrmAR;YaC!K|e4$b8BJs0tKjKgpNC^cRZT8nBq
zG$6{$n_E&XB}nt3`nXnywJu#{-WX>Bex-T-wk<&H`CBL!3=Z;lw^%1GS*{$Tze=jO
z0Ps02f0Tq&$$3X6ydd3#BMXp`3vFd6c2-FI46+gHj-@zIY+)!G<)?)?BdRlonzWIQ
z4Ei+hmf?S2e;Z@*!MOTbEKHeoj2CS8nUyB`-X&)7n_Ebwg|Zb}eeoO>cF@Aa$^4Sk
z&W7}?t|W8KNhKeHJf@a{YN{!>K>y{TX1cI|jKTNj6KmSc3(A?6^Ud=pkwK;skb}8D
zhzz%t8Dt?AH{d5|nA=XfMaIypye~n>HZ!wb7I|{cZZ9!wD7M~h*gEr$++oRluHnK`
zQ}EJ$x|(I0-ks9#_(XLO?#4heU7=_EtPICCdtwV)8)25zggc$e;QeS5yZ7Yc0#|-&
z2+fWo{>pPuPoT`S?U>W$__`NNk|CBTWR|JE+(aaV|2t|Xye&T6VJ_o4aK{MCxEyVkG2YkrXYnw5q)XpPfjk4>HHfLS#nJV)|7cxrxd9hA{$`5yfKlD39NxFk6
z|EQiRKS=2=bSbq7)*Dm_;Cema+V4xSd$-yz;7t#5@xCaXN3L!U+6qB(PL~4TGeH}1
zXOnX0=iTRd`vxCX90D*dSAX7M4z^PIblBZ@SNknDVnphToR}6j%@30^^vQ9O5WkPM
zhstgI@J|ROzg_dhqyn5?lIlN!iOQ~B__AHlkT;sc{I4KwuK{?Z7<9O(7#bRs6Kcn|
zf4>=JmFUeQhJ1mn%&2*jAPeU}H7;3o^`H$0f_mW*{@lEBz(G;fl`vhqY)3&00(()N
zlu!5_0f|Es&r7P?qPp=mZuaySO7Q-Zb@2850_e@6&|E!hhMqx=;O^8!)pmHuvlnoQM0B}kGI1@JElm6p3th54
z7)pXu#`U7sHZs*kSgKFd2st3#xFNpffXH1fws-ICKI3gq%nn>iNoTX{utZB_v7J%{
ze3Dd)-;Lyxw~Uv@rN`H^6?JbG%te)gLcGRsvXkH}rG36w_W6nT-`;{BP63Mppqsn8
zx}sS{7XfD52SBHKtZWC0l&*l#LN7i+Z_$n+zwBat+Sso5{WY6WdqCX9g&S~H<>VY9Ic2?`CBNzn=)E9pb}Mm(#k{+({>iUA<)RHxO=IS9KHtv<1lb&jso
z2;Fg@!lpSlf%9yXO
zqdbv!LffmZ=I@S23+jGj#I>Y*JMCecTw=T$rFh(LvSzPvTTX5i@5k6nRhRu302?{>
z!GDk;Y)d`bm4QhiwHp~cCSf1>em18j$B2NDj-HfBWH_X*w0Z?r*v#;qJZ4rIIfb0d
zUBGo*;2leN5Hc5oq#LOyu!*%J%e!lJ>@AdMX1}~Y{Abi?iJG*1@(atKzJY3?n{JH6y9Ez9fwy^8M~N|
z2GJqRoQ|J@3mw=mkPtp9*4CB)D5H9$<9Gmp#5IJA?x9<-LAwa|y?ZxF0RH&XD?QQj
zWL)ZBEFpjIB|mZ-h$XXs0&g=4YN2-BU4$>o@MEbu{dRI%8^0nsxyTA-mVi&^;g<1V
zANVG~D0DwPt)p^+=7sO@BR4jklTzy3e0<;Ot*Ud@b$J*n^LRx8Wgf)rz72X9Jn6$(
zTX?L1B4HuoTX3-Ob3C8vx(b(8W|A4+ou?E?acz9A@A=%+DUU!B6su=3bx^H>c3E&O
zz(C#kelPKj^Z1siv^_fi-QZ|x|DA!T3=hvjeJ@~>v}@}#6PMuu^f6xMVLND(Y8DZ(
zDz@4*A7MV%ix>7k#T1zwSiNaOE(^E?C||sYwC2w8h>Ly
z8l;0}`6MD6c|NNPp~6YPM~gg@Nmx6N`Fk22Q0%>7Clp2yzu+R(?2WaxoK-XN5btXW
z&H!xjN5b>|K-8xvC=Gi699!yvBq?4No^*q(#&Z<>A2!7?ru`e-+lJNLyA$KQr#ojg
zvHRCG=eS|2ix_9(dj&lX(;G*=!5$b6%~)Bn_T)1RL$orCNc6-SN>EN}oGm13ZPESk
zp+9Rv!`uL7z7<&XbSeXSV5`qj7Z)&KMJ*m8H*WKeXn(rWx#{*Y-Z^esa!Zo@=KL0bWx;+5OZH|$o>}4U$9Xb5;MJn
zJU2R12majjZy8b1pbNR5^X-I;=0bjXk)k0TTV@BCuLeZqwh99iCqP^UNPR#Qe@iZS
z)E79u0=$QkR-^#R?qLN~Th$gVT~kq^9t*!9J59P_ah-QDk~Dn-eXoZ;$KLMYO&-1>
zydbPW4nc;pdk^{3Zki`IY}$9G>qj~+>a3<_;cx}(qIgQHH=k&|tDOBsxQh(N^zSmA
zY%M}fSvOPzt`zpG5XoAQ+j>5)Zm0*L6ElZjNvQoOpbA%
zZMphDPdPS!zco0Vy)1yV2G+PX-9fWjG5hQ9a)G)MudMgSo>5AQBJGdX^IHchXjjkv
z{T21&9sgtdN~Y(baf4TP$k7hzeK;zrhhnptR+E9PBGZl+>?x|->LO$6Anfwq@)2)<
zKuNX!C{4CvCd6y+^{1a_mPS9`)Ed(j9CLQqwRLpayPs~)vM*eJWLmN+u9o(sP_&S0
zA-`)4xfj49K@#Cte=r&D8;}25As{fY9Lua~=@_?ro|Z~
zDfj^d`r(k?6zy#hOgX{{Zj?7+x7|jB{Sc?QGz=RvoXs6@BUh0Z1V5ukjzOfgE;w~D5(7YS|I!uda0nrM817DHEIsAv12ZK=^q>TrDc{w{iIJ4LKZ`a)S%
zV$C;O&igyFrSNQj?5@CNZOqKyS0S~DAIU6`jaV9Qp|WZ?Jkh*&8%s!ByRWhpRrHWW
zp;~%kt35l@KVEkI{j^0$O$&690}@PL8SrLRSi8O7wFiWqs+2_fadi~5^`x3_aC*@{
zb1O&&s7-ax7~@QOxK!`tgH`dZetZLF4!u0b+5cQ9iEJ{s4%9zyk=0zS(2
zYH7`cyU#{5_bvM2wr<=!J>{vR`2g({T=8d0
zO?lb)wsP}Xk+$0HwXIz@rzF@aMsFv0UVT!MU|FoG^&TrKdb@cT8P*O&@p(^dZ5626
zJldJo95vcU#ruz|DAr{z*IVww$R?q-`I49AUmH^ES(YHTUkVA4W-C49HA|rBIbU-j
zU`G=h30Cp7u`(v4ChL|NnwER{_CWXh8i0b`{Jy2g1|bQbqCdS#+Wr}QTKKfS^yMg+
zt5pAUjd`ZBc~KEY!y|Vd>~L)N4ka;5=v(zRG&7NE90hbp&$&j{5PmL@ASkK^EsS2)
z#;v>ttaX;>8>)nkF6~JNk^yw*9G9lFKP?mo2aX#5IPwoHos>lqiA1cAIacgJHXq$q
zW(i;_W1092|ApTR8$mW5k5tCTj<7hU6<auG;tp-~;H7Gxq+T17Hel$8Y#1KQc49(jPSli)2ew4Oah
zOL(Dp3fShw}hs7Hc9W8#`l?2q<
zdwaF4j6x4%r6!A5vLl+s*gBMZpLLJh>XxuZvfCTR1MTq=luLSC{Mz;2zXUtZtZ7vXeH7Y^5ALjq*vE!SC-xUXFH*E{G%rY2ju;
z!$15CK+WEmsty?0{6zEvDqUYaD1QIMzB5+_sN#e++Nd^+=9-pqtYg=-R2VdB1PnEz
zk+^0pmxu^ohyzAf8c^usWL(EL$ZIiihyzjB+a8PUVwMyT=1!^_6cZ
zu&2MoXn=hZklSB85kRjxHV%&ZtcPmE=cw+=j?Is2`?G6I5{Nf}@D1T8;KER>doCgXyXxY_(NpvKKi6(
zO!oweY6f~_GWv(~`~mLe>sQ|Bmsdp6psgl6rO@D8zld5d=;Y5tVdTV}EEKIStkr~K-;^o`hV
zm6^>5aFBq%bn0m#xZfUC`d**s+9(*_PKA%cf3M(g#$%;TDKTmov9QKDhJsyFg&^R+
zmvL@kCoLqGdnU-%f^E21Di+EbnX=Mp2iGC|VL$q(hrxHe5j
zeiopFZeQ(X{xppTOJ1sEfZ))apWf<5=Wn(RP@L{s1{j4FJe(urfQ{l)`6C;Ad-_gS
zs6*7YB6<0L%|2(|7bYQP_L|x)wv!hOB7(R7#My$&^44H8u@ho+u&x3Nk;s+wbn(On
zyj2#XF;pTQs9N`)ek*L@+5J?lh3(zVkN-uwKK+R3mG~Jk4-=cidK%81Qi8k-tv#lyJeX7-F77b3zi00srC){Y)!o&6=v^uA~|O{H?eiC^sknP|6AxXK^QyOQZTO2m6(zB4XBobYSM3QA1)
zs|n)?%;$#mJWfESA~6iY4N=opA~2eLy>mf2xlcYhzvLUpkFW>K3_w*F9*0qD2cVya
z&1H>gXe0DXpdK)F*ujrfNL3ocBSqzfkudziWzK4sy*h4}eH>lQ^*hIs6@OwjpN*9K
zXmxY#6Z#%k#fa!;vxVVr#q1{!JgAlOdmVf7a&5sZO~F$8|9GIpjAzV{7{!e7nQJhm
zK|t0CkjZb(`gLwDV5^(U^FU;Z;h?l5l_P5dKp921KaJ|#T+NPgI=!{nm@GH8>T?(F
zU2JeZJD6qx=uSX#Gv`p)LN~&7(nOTRoSP({RXK{a_IS}R`+)49J&Nj+DP*F&nv-xp
z?MnG_8SA}N_k%(^Q1psv-t#W(NxU;bS-n*}lk1_$CVOjJ9JDi>?ey`EUJ~gg6>cBP
ziEPvYbA=+2ZKnhFQ6TBpU^Xt!{cdppn{YtK75w32HCdVLfI?;i56pG*-!B~SxEOR$d6T)uRUYpk6qVGZ(1=L8cQ
z1FW!~zV9o~U+;2#SX<*H-WZC~nBcso#~tCX;&Au$kHii=rgYG`a=q*>V>k+qf%~N9
zYh0*WvM^#aGeF
zQ;yMvHLZ&dkA<+{%-5h^ZI3n#%w1v!t0&B{@XWVaM0>B!fN9{SP|iKYlEhpiTU{AS
zS<-xI>X_ETJ+^uKmOI}Kq03SJUe-Uch;;Fe7d`>4w?Is#iz6VM*`bg1c8>%rn24?x
z)?@1dOit4h$5fwu93UPDZU0RWk(`^;YtDf^(b60@sJ=y#MI;C*>a+6N*VaC(T7==(
zH4X?;uY<4;uY_if$>8uTZyBh$-7}G!og#j4TWIDTTEWqWF;94=Qol`%c23+@RZlzv
z9ft{=@dX8adX13_rS-nvBsDueJ9~9=j!xwK6gxOsNu4C+q<1qVsBm)?Wrrj?@`g=t
zP4T@$HcB8?+l(}RBwNg2;u47ZT2vq1nqfz9QIjr2T{49Cz$iIl7q2I=n*1Gisaxff
z1NlzSL9fv$GQE^cA)VKl!21=%cnzj|Y;rIUA^ANYX-rewTl4|52m)f1px6F4TPhbz
zn))~V9wb4MJ|G)r^EeF=h49~r>LBX+DfeV$8T{|8
zYr9jo;hv^l69*q71#pt^$_>ofMI2@Tc?IaB7bfYnkmH}QNez;GfMVw)13f#LooqgDJOuQ6nmkx;RRk~R-XQ@%wE48
zgdf_NB-HiI$6e2u;F@l|7ld=j&2OnW`*E~mIY
z-A*Rgq4#HgI0i~aBBmvb`AwFFqUe%L)_$4
zloF+8jc@P>5+AiK7i&OwSfq#s-^k1*cHXVI{*2zKTiiay}|o9VqC%O%aF
zP`%$+oE0J9yc0`{%%Z1S=)8mY`#T$*WcH~bF5*bYSJtkYicMVl*gJFFG%c!;7h6AE
zha$V3LfV+Ip^Mi1v5Vg4uv!O!UZb1f>%HqKkQM2NPWc~HrY#1+x`yfUz+c!ir<+pG
zyV~-kYS2%6AaXY&TzWY5X!mEvWH9k)RL)_w1grQj+d}u^-^c~?>H3h@m5WS#_u&UO
zAbEKwtZnQ+7CF<3|%Lt}F1(){=Gh9j=}cFhZ8SWJGRh_ELG#7i_vB2L7?
z@-luSK`7dvdhx#)=kP;wNJH!tg;0}`KIJX9A|ejdAi~8~v>-(zNE0M}j@JUmi%Li*
zNg(F;Hv0qXooo_Suuu5t9r@v?zMSEPjAB!V)dk83*;X$;9Fo-Q&#qyu-e%!rR2UJM
zEk>0|mL%~wIHxC%ueB%LQZ|c#@2Q{H$yFOWqJ}R>wX)53Le8jVZbyxN~h%*Dmpo=p`P(cBcXYs~C
z?QxaMf+(qh3rwGt999fXAWmJ@z%CW;
zF)ZnM$sp?n)Z{X}orESR!ui0zCB+4_niCmwO{<&IkyCerwac82i+uDzVg0rKVLkw&
zWAHZx8-nNqq%OPX`-3Je2$3;;3?K09>FhOP>F|Abm+>@{6l(`ct*gGj{k5?
z#Bq8Lpt?99ej)0QuCwFFGrRcXbV(wnL^t0#IXdr
z(mveB_M0-t0jtodFW)&uSN%bk`+uCe=&YF5vrfZ*$!VEr3IJ=}e-Z$JTTA|;xNkKr
zqXEe-1^LSc`o$r#_HQhURQLhOwWmJE=`e8YpU!uB`sq~@KPB~Ic4mpG2Hfv7=2#7pl4tEu#>>(38ZJ!*~|4%PlS`4vlV>BgT~#1a|%MKdWV*t8Jl
zPk;dLG_W88Uf<06I?CUlnk
z2(O5{VP4?3=t*Ad8L8LJrc+clGhBc(>T*Bi^;~lzQi{2gRpr08wp$zL%JJW%@-WlW
zsEL>cYMAz#0oxudahX6jpe*d~GT_J~2`f$_QZO+xUxxG+`bSozg%nH98cGhmE+98=
zdcdDKq>@6?oF1c*jjI<^V5;mG8E_2~6_IDO`$H!G7rEwhW__NbJm%`X(ONkSWn;pz
z>n>3vI!GlR=j`c#R2eM=Rh4ajkSR9_Bqnq(t<8c5=NO&yfjAp}dsj_}pXTse_nqWF
z*}{(MKh}Z+WNSX5+Oa$`e&!!|897{)4KHqbBBrcF^^pqrKId6}qpIKM_L)C#WlSQP
zl_WKzCh4sEEhi(-YoswEppl&RGYyyuqUB8Jl)ei3z#OR?qYPc9U&v1mBa?5*CTC!1GLGF`_E*Iz|O5}0ZV_)
zdv3`wZ9i8!I^0OnW4-}QDqIr?rW*FfX6KpTqx+E
zb}vIJ3eog_-eF255lDtG0ltVKPaCvsx)jcsg-&*FGRGv)?(ZDJ45KLEasD+0iA@Ozrn
zaq)$)0*neTq34&a43LN76B5)3!ZRZAsL#;{e*1sEN|rISy6LqAa>#hAqUC1^&V)Bb
z4Lq%FOD6SB-}+%%F^9dRs(pMajIruT=3Lnu(+QQQMZgse=w&jkTgYMNemLs=_%PkT
zaW>n23&dF-3Hnh-+hm^-;wlbbzO-Kis_EZ4&8#fAFS5?Z1WxgBqpV?sCS1KWvz;E5
z4x3rb6~Cs+t^3$t*CzD#Jh3>Je!sz&=~4^+(pb(;l&s(fp*70Ni_RU4hMuCuf0RT|
z&0=G2npXqyGM2hhQBow6svx9bluh-x)?&5q)@eq@rb5v1`f+@XqWBABopo!onX;8q
zSunb#n>VV6>kp-L@XXy)!;xOa*$md^8%;2I-Pn7~bEFYHTLoF%^eV8>nOCtafY_=%
zUn&vzJxS^R5Y*{8DdUL3Sxd;q6E?spB-rr+s{H&p1ek>C*nm`l)@(6g>Hz4B?!s~n
z^K*WR-|bZ2a&QXye(6q5WmYgx#5H--?F@_XktM(Gr91&0G&O5U
z-^=Gyj5*mq^$je-i)**HHVAk*K%I~Ar&o>77N=)Okk6Jf8aDeQ%2Cb#wS5&(UC|@j
zLuh=IY82yJElXvee-sz+Y`jBndtUHMUMxCz%?|c}`=?+ccL9dJM@^i)xV^Wx+g9u(
z?a6jauT$bwHG?gFi#!&qZLpdZDwFNqNqoA!(cTwG^`HAlREg3bkG
zx8-c^WvW5@c7=^U{5RGDq&T%Ge9iecJLjYt>=T}|a9RU3SZN}h^aTdFhdYU%k-xM2
zSw|8Q@Hg1YbXZodp^r>=Nt$>hEx
z>7JG39}^F@X5DL#+EMkjE3}|<_KhSs|9_^&XXc$LqhHKNoY5`rve|9Ayjd@q(nfPIv+*^$%Oj@@)T8(jd+WLydE)Ic#N8R>KJ2i}bC*NW0VaZ#m7;Rj
zT5|4vr5d^}f_t_UmF~P7fsS4+B9Af{
z_5%)nCLjwfWDu_x`Q0>t9Oy=-2LVO&HlbtEr{o=@sV||xQNYkqy~4^>N`{N}B0!Dp
zD!IsZL-*>TjygAn^sd+JerH{63m>3Am`S@IZlwuUB&g}B|Kiiog)pOoXw<$VXIO>p
zUtt1X^Wkmu-_v+NhWEvh1gttQlhZCh20T5*^~qvh5C{gPO$i-vcMleH=rFy0x;+NC
zRGz^)smiRgcsid$<19u5*v?1Mb({lxGJ(MG(FUGLUo8{re~x27LMHqrIGiPJtCrf^
z&%QvNMgrIwUIN1}1O*_RmpNJmOt!lueVZ=`R>lvZr4@;m>G;fhrE~FULLWc;eW=s{%+xFXA^>X{mfx
zV54)QDN-Gb@&hFaf>uD3pGMwWrnG5iuFStxdRmF)$@%FXP4HyBMyruUROue|w+BZn
zsF~?;=meB$lGJ!68ekg80{i9VNax*O!^s)nf6$^4N{o;;0;ccvr$%JT+GGe8D~!3x
zrOv+<=tRZ*zZ8vSnembWX;0d*YFm*qZTldkq0%mR(BU-c?`ed?QIw^37CymuYiNwB
z#%AuPvS>@K-3uXC-YInu>#!^^V)E~nk<|L`Wr!j4bw1O$xLc3mMQh8B_33SV6qOkT
z1OtcbWND!%{xPkiLg25PJIBi8k^Lf^B<&LF*Rs39Ih87=hPvQCj*CzpdN{5>^PP_p
zp<>e+$g;)Xrz?Dlh@8qFemJA&gHa0nO0Ow?2joUe`Bfzvbn$(
zPt9~oOqyuK96iAGfD24wU5qFiE?)Jq#!E25WfMTgbl+y{VPys^#p1}9vgUe*hL7^V
zCu%IyY@_S>hu*=dw!`&Z)K1-MT`0}?xNy*CW&7GW{?y?pqtG|7cgd=nUR2Aoy!h
    `^=o){e@5A{X>~>CTJO;p9ma@8xdw6^KM;vfQGBYL8
zcm6c{T&<9JaP<#kn`?s~NgiLD^qfuTB$-Cs`7F&>H!zo)hVd5p2lru9Pc0P&*4jV-
zS8P9SH%&~e>78%WGyGiPBCrlQ!^gwJBL~T4K?VvL7Qa&
zU?nw$!2z6A$BAniB{(0iJ(yYa58z}NcyH-BmX@fZRXuG3Lx1eHm4N68QD4t(UBu?)R&Lv^6@SFs4le)O_+WMg}8F
z9bhhU^9zLZwAjY%mv;Njo4MT*rb;}J49^oJ7j})c3-=oP-vfZr-$XZeG5GH@R2N&W
z0c$024Fhm1r@*!AM|)txs9jv^09QIQ^l!3j=tQx+79rf5go3rxn{SH&L}w48SvWZ1
z{_jB!fo?6QYHm}JJrvD7?I30@mPAV-V0;p@T6WjdHn7;QNp%ZEY#2+RUUPuNLZGhcf(&FsU7>V6a-6vBP{s`;ZoWzxYjoTgUv;1*mz
zld8+Z7FtcKqPC{^&yD#JT2vFx@@Pf!zixPu|wd_o$2YT!Ntt&Z8-5y#3%nKsGZhlg40O<^9{m4
z!R9wE;*voWteI^kgNjz`7O!2qcYC~Ln|oEh#~;4D^*mgA2WQAMQOBkTWmu9web^hP
z>YagA)ynMEx^>s~lVejAzfJxxyxCDd;?<&vf*w#o>U3k-aIFzXR(o6M8tmg9v(!Rz
z-zjK@Na9CqA0zCYm?#8}
zQSn!7gvUpl@V>^m31@cnzks~FpdGrwhcRkwo~?Wo*0|Yx#cE;d3Xd&u1hNxuS_De2
zDF5#?$Q!ycTL)`4e`TaCjA_X6qWv%AiNmYnJ#N{u+1ILsxV~9;;q?)JPo>!IrQ7sL
zry(2cg_#Z)y|bkd{Yw4gr(pRSx0&)CG!*e#pScnAHv6H7J>S1aSlQ?@E3?_Xr|mSI
z=9vp_ca?<}Nxo+!_xn+@pn0{o+KJew#dhK?l
z&pb4=6K0Mk{s)wwis|n5se;D!IJ_qYX`
zocabc2WP2Ltb!2W{xRSX9+?SKsi)?zq58K{fVsbT74Q`~R<<}Njv-DANhwaP`_*Z6y`6o+_`W{>gyNv
zF}r_nZpF~IZ>3dMxrA)jQ^r0A{-HO1mkLY8gg#6%Ygg~oFQ||tzpOM;$Yc}S<+F)#
z>{UtoExF|xf?Pmsjj)V5q~D>r%Z_HE4sT?LtQJvjl?hdjffo0WWNhXsS}&?Obd15f7x9fi&KWEK4Kh|9OE0<}-N0=mzC;v7RbQzG#a!kC3<(S6G#4@bjreU0O9zEl{k
z`5Z+N$7YGqja(mo_aloK-nlq+c=Xv%QW&@#vMC9d#r?8g#Ju`&cGN{S@VQqk3!u2lg}(0V8TX%9v|2mZ
z$#uamfl3YWrVOwRS1VILs^#rL2yAqM#FE?0|2)Qu2UQ23>>9~aa)KLFo+kc1j)(ge
zSVV)gwDJaccs-)6AU)d-;lO-kmmhUPB8*G_qlm?A)C5%Jsa{<07w>%m(gYI7|Jf~^
z%&lQp>K+o>=~chE2-+bINMT+xdEiW6gHX)gImn?r3Pm<4sB=CiCf?{K!g3#8M4b11
z=2Mo@g|@&%!ugY}3gDj1JGcPKYp+=_49el+dx!bbk})`lHuU-CG#e@MJ+2Yuw4f8Z
z-=jm=^SM><^QGA4l~vfz4a3vv@W=n)(l58*JZ|?j*3(G<8}|SBnE`5LuNnLs27rCs
zXzSv_*=Mno!>VkNKBq^dn;Th8@{7jlrb?)JFB26jn(5A|Z*@?7jt?7iRClzk4Kk>|
zvzvQLvL{PPAloU>ZzLBDrl;%jQ
zgPV`~tp)kXIUl=e_4Al$@S8f+3zU1#LSi^8byU%{6`3k}2$9*>{w}*YQU0KzP^;UNiBQegvYv*(*&x$^}gU>}IXRMxj
zNt-UwOt$$wVpCvtp8Yj?-E^{#z{3M6^xWU%_=_I;^5^&D92C>x>T81;h_IeTs@C}!
zdT2!ECa$7)_4_AKy)j}}UHV{JJJDOk12EE?X?d#sNUO0=<+gRMW@$2QBYJUHl)NL5
zQuc(Y%u)K0PdU`-t)4mC-OZ*YVvY<9k^3+YX^qs*dUzh&X)P<*VDu3o=V9NwqjPJj
zJwH8o;U<{8kxYuG+mvI#+ay5|DZZwrKWMTR*rE*noW!|@!VU{qLp2)aasH2wal^5x
zZ-)zy>{uuyNmB)~Qkfwij2PVFf6DR0P1B--Io($n(QXlDMY*2ELdh6c{Qw94<;V;K3
zh!E$xrEy#gpZ56S%Wn+_F&Z=vnMk}RSi!LeIYz)yYJZ`d58p7{{ni6Bu{2qyVTOH=67D_B(@)a6hhX_s1!10q0ep3#|
zm(D3w+LDVeeftM>5B;JS0Z?B6K?ErFs-1s8e2xW=;f9dY-rVm;{lV0F2wkxo|jd)xr2#`jT)#Dtny0fE~WUHX4A^&TGKSH9)47l4qm`KoV}_^Mbs
ziOAFGST>e)YE{}RfJfasgmf$fF?jFVd~{rZYKv>A;3vK@*$tM%Ic1AW*VAv8Rr=Kii_|ItblK1FkyIkOhu|(>923aI{1R3dtCrSNR1%$0f*2?8i
z>C-D1hd0t7MKqvUfL$J#347+}^~$oak}w;(txEPum}2$8Yr)Ce(s_i
z3HFU?&)C7a*lJw^)au&&BtrtKDrg*aU0uJ?^)2r5gYlHzqKMoD
z0rsJp*Ng{8mzBiGQM=D=n8>NR?8c#Bu(xx}BRYNi(n4(&;RHez>gzkyca4w#I%s3J
z_|gv3no%EF2qMcTLSb(3#MZZMOgDxtVKPr%;$=KhdpkS=;KG@Q@f1+a9sY8--tHs_dS7FG
z;qJDwjA~}AIUBFe{5K%E_W?=i;V^lXwXL{6*285T12P}&)F3pv^^cw?j970iHYjQr
ze>UZl9{%pbNK)-CMw(d{{n}ZB=vTK9o{P!4Yxji+HR9wTwrUP)cbHnOw}|sjwuKFC
zKHVf9E~wS8d14rpRx*)K@qvvx@vD=YR9obkOerNoFf-x|)tgSA`&!GW$t%T@>!+;V
z^TV02rg9F=gp2(j4$R?gCy#^OX)=pZgB8-+`45he|uF|;~d{HF3OB~E+j)
zNQJ2z0v0&{z~ZZIF8m++-148azlATIVgT^t9tQo-I^=&x@L`gA3_lJ`MJ^zOt$>a(ls`hP)#m#8|xVayzr_o1Um;IgwLe-Bu86_{hBDO!6lzppGUA<8w}
z$tvE@^vS?%{M}?Pms(V_@Wru5d_P8E%BAO*lEyOO^X7AXsqw4o<^vQZzS47sFMjcP
zBS}Qi>NJl}9J6~CC{;!z3bZzLSto{A3fCUqfruSsigCqbsb6$yeu|U-4XZ_K!6(bR
zDm$li3>!76_xz?~?fJa6=n=wCyDe6CtY=_kXeRtKuhb5h(8$Rt*G~6{kxa#8l!4|m
z`+*9XS(C;JBJYOyES
z@dPsb+A*qcf~S9pydTMtsfPCe)qW`PrcCOCm_&&PnG`!@DtJTm_BPBqzajSNgL5Sx
zD?e)I5s!xdpRj9_Rz7%MkwK~B$ouk@o!n!eE9G<8Ac@6nzg^(e(Z{zk>$thAjVFhj
z21kVVcRqUTm3tqhBp
zb#y8G=qP&qzcXNo(*=7PZz%Y$^f`qGt^!d8g_=)DM<{9>@Fs*-@@u(6UG-eB(}OH7B@|jH3eg@v#aJamHPOYLxzlOUYf<$^9guoHOBGI!U6wp!|QHb
zm&e0;N5D6tk_}9ZnO>g^TBY62@zMvzJsndDlNmUxujpUh7rP?Zewz77vr0nZ#Co7*
z^oH+jj{8CMb3WGZRtETp#KMYju){MaSproCrp%NKL$i64&1(~=BPWHnLp<7!=Urudbq!9TcDl27Td$(bbyKrp`TCA&YhTE
zNM)ps?fNrkHYk!((T;Y?pEbS{ch~Sy-`J~?6f5OS|JInn&Sh9urPr|67&(HMCyQN0RAecBck#L%O+xjLhqVaNQRMQ3F_QAHU`V&BrO2=2rYm;8
zs_JVV#Fr&$st!*yLEbz?3;7jm2OcG&G%s-iL{^v5%&6_f*lWf(QRiYLwBBPQ8-&}3
zSY$iln^4fdU8x>xT!~^6?O>R5(JB<$`&9h>ZE_h~NHGZ0Z
z&j^E)2QXq^Dk4|rE1IBcPmN3t*}K!_O@C5LAZ=91V>i^jCL@soUYxLbpV$1-x?q#)W`v~?P3u2d9CQe$)fuC%p>?}Z@#eIeqx3v6
za$suKF~W*1x=e7r`G6-5pke;MKMy@gG5F-u&mw^2q5FhfLHJC?s{M@gh6p!Lvw(>f
zcNfmm#3(;m`GfZ$%{IBu1kjbBro~WDHoy5e6(q5N=0*_
z*19OEg!YC3pw|!Ys-A5g@)%^l`_XzsN`aBXla(Mnll=NMS;*Jjw{!%|#Tc9FLYLQv
z1O7*^e%FTuS8vd$y{~QiWg9I;`9Y~7z(BnosqzUN9i3D#7y6IBa=0s_cn4@>@-$~p
z*MUf?QP2>C*_$!ImAsoTj5iN4p@L_ncP9Ow!z2nDvOUa(qtlLTc260hg&-Hg3p4M*g;l*?>4~BZiIf$O*5I3R{wTfCfr}DscDiM}R*T_u1W1rnf+%CAj;V
zq?ze@;BGnY$ab7aMxp@JEHDpAZH)X$7P4wn1l4cGaAI9`#F~hQZoN7)*}mz)1IISg
z(@t3i(++c%qVrxTfdOaOvLc{uRYjhRM-f=+)!>O>n6V{#zRr~v#Je^wCBB0|O;H8>
zP{qJEMx5+&ILDY)VDy<)$75n+r!eb48$m4}^*0z&%yRL-RpNqtw7ADl-JJiYIlnQG
zx-@Nj`2@YVvY42dUbco8694R#qg(1JT2Q(x@PNFSMvjg!foM>gN)f=n^vMET`I|rj
z(OXnnYkJ_qU~GL!&+7MwV`67B8XTBtj5F~uozLunYx~~na6~OW2FFH|pE0}O4+*Dv
ztWlh&lNgwq*jZ$%lV5Y<9YG1uAvMbf1P&*>%@!TIA`#Vtxh3R16LXm>)wOieMk}iH
zaGUEvaSDdXgTaD$Xup825EbI=lXYtYY*mh&=31zjnx}1Bj(=z
zkVe|3-?m6RVIr`_JL?a>l9fWl^hpNn$UY~6*xF5Kcz*4(YLID|^&kiV&25cF{opYZ
zT(|v^bR3EuCz*=Y_Kg54Rmwv%U--@_wEmH}=}}6{*y*b8-<}`S2dS47y|ok@9cr
zqj`QGt!Gokcx=JggXyW{lKizQGJK6l*AO#pvK}PWPYy>(<*I>!Biam2UVSd8bi!v%
zLWy5BD~~+73M;iqJZR0(Y9f7`g<%*6*s*_5qey6)SQt72mTR)wzgAE+l-Ts4&fSe$
zUhXFEw3#hl)RiFB^Bwnb{U65OIx6b;+y16YI;6V=>Fx%R5EKEC98$VFq>=7!5JXax
z4(Vp-hM^mV9$<*)*j`Zk`Th
z^N=ndBfn1!D%E=c4X&F$>SNum_~Anuj)2QSW$JM~r^}j{S+(aGkZxo^;X^8=0(5nC
z|K0Eb-045@Q%DH%2OakTZDv0efM^F~;Q)=w|777TH#T*&$hF!4+6X`x@m~qGEb6gE
zY{v@A-$YL{BTR6dLfYX09Q|NuhlkX>_gXvPnIVz5i|H8sMP;FHCh%nfr~+njzWsVo
zxqFL%0bDe_1eEk=*ebB27tt=+Yd<+mxaZ>T-lrDc*ed#2f@C1Elbwx4?SjdPnLm70
z)mZ=N+7vhN*VD6BYR88W&hh^ywcg=P({~v9T!oQ@Xp_(3+rQOh-ft(>12FV;#_DW@
zt%WI5bTGeT7m?Bb#7(dWYn;$fx%Ijw14Tn+Joa==`1b4_?%GlMOnr%_H$OnrZeDb{
z<_}kO$jP2mx0sdtYcjKVToOHdAsgqMJiD)LujUm(Qz=SNdcI3^NM2MkBj^6s`fyK$
zmFSNFU7AD3gA|Lzy+o@QC+NoVA;)MDtq~6|IB5~S&}+o?;LzvSSSaogsb+QWfuOs0
zgDzPX$KR}qeOons=}hhI+w{~j(gqP>lWlj~NHtg@jj?we{cf0KQ1jEfDrg4X1`t?T
z_`X{)fqpN8pKu@7aXb>RHsI-0{ogw91O$Y^K7y>xnuJM2ET3<8_Ylx*d)IegoHxuR
zpXG%yQ8iSo
zVe;t5ep+539FWRr5u<8o;-23;U~>$=(*OX1uD|V%rs_VM4U^yAm~^Mk2>y@V4H
z20;~f3hGDay%vLsHKI%`S%XTPYDP;^nd8sg*&=5uMJRtSKY_5DUaB$<}eieuR#ZzcDXCQF!g*
zmLppb67<{6sgV5a$(a$Q0gPkuz-@@&pAP%um=j?3BsAC>*%J!#)1oQyTp9tNu{wscH3)iFo4ofno(in!0J{=J;)dKNv(W}waHY?|?6WdF|i9Ccw
zEDI4Vs!{Ko9CYU^P0dx$z!BmA>5zskL6hy*?%*Hqp!KbB93JeR@5!Zoc@Tc|u_4J}
z06HQq$!;%~pzm72Rz)-D{uG1nwpm4Ts0ylg*)GVdx%Tv`()L?$N<9922?Hmx;RHEM
zf2T!4a794Qj3^|vY7b&*M{tGXTT)lFvB7Nvwj^gFOA7Ijob!}WW9F@x15WM4cB4TT
zFnqPltF?IYn3JVUy3D%IRkdQ`Jv)8OS?6RCtN=YodGH&>Armia1a^n*VLNspUg%<0
zS?}Z3ySCc^@Kf(ch^u^#|I|`g6Sn^(+R@%ivI_X^uKZOy?Qr73gcv7H$is%i+rj3&
zU#8YM%RVpvpo>S$ew*-$F9(~nd2Z2S?}I@qfe?kz7!1zQuNWEONMfR7b*PmjRuJvh`C`fFDB&kE%Fc%<{5L@<
z1qP}*Xr@A-<0ti1jRomsdri~49wFi5kd_+VwcHT@8w}dw
zqu;Kw^=(bdhw*(dU_HKT`{>QUU`@Q1czp$86~25TH<}w+>|hCs;9?
z!-#7D6t(m-|wPQPp0Lr3OZ2IU?RoX>u&qRJx-N4vXD?@a&i+zkF7sm
zXCf*0($_p7znmliBEcIjD9`nqezq&})d!3pw6_YMZ4%m}LnAn?cN*>_I-#A}}ta_Ykm&$54s=<_eJ^44VCy|$0Ev5o(&W2W<
zNyGEt@Lo
zw+VoP6|fPbI?c~7rA}4?)BQIdoo!WW>m7~4msB1`HM85?wnY5BG4>RZ(XLBpl8MD>
zwN`up&#abV;j0%hSrlW|af^0wi+*vlUt$3P17N-ThtmY~=fJ?Sk*0u1iW=Y@{iBfp
z_j*T2M<3Asd-;Odj60fYtf{NZE9SgJ7O=1KA=Vw$mX$w34@DpD=06%-?RL$*f7R{n
zX)=(_kvty&KR&%5zu(L}Tjt+0rF?RIhOY=}U0v>=M-L?V?`GAlf@y{48$?w&M_gu#
z(SUR+OW@i7sDh8>Ed)#*@ljF%+>@IgJ(%)8!qe}1uXPjl+XV9(f5pH2cAkf^tZ1qU
z1vzH+{k(iQzQOzOxn39)I!Sku`Jv@cN2iA6Yo&&~647CCelqPBC5tqO04Dag9pW;1-)I&jvCDxZ4lGy6rrB?H{rH2T`+!
zP0`qhV?)#~#PGJayLWkTK=Tc3hK@%b$e2oLRvt#?kX-3EH
z*_qG~g!>*{`Twr_nH1=@7cno6QARrzI07ToVK0wvs55l&i;xoJ@#(=z_*W>meFklQ
zLIakMcwWy>f=?%%kB~Fz2fSxpzq=jXTj(?V`Eh`2qTRH2&VIRwTsUFW%kz!`$X|7d
zabTQur}k5wQt!?ei6LTvNnASim$d5D{{2QIq>#bKB;*Tl$gUL*N0RAlGHsJT9h1h?
z8RD@tId^O)(>A`qww-5{L+cXR{(`O;+4RLAl-FasTLi+Am6A2FPu@uLrQnb+;I;NE
zBrDF9rl^!JTSw2iU3c1a?QUhh~$+r
z1v5fkqe5(UM~xK0JpSp_#eIkK+@TUZfF)PbN<-iWUQy_T!jP4Ikc!O>&d4;n^59D3
zDk1e*U3G}g%v>LE(eP5lPR6kQ3-a|oC%DY0#lt|-5tU=XGS`n|;N}@A)8ho-Df1?2
z_Y<{e96Z5X%xGy{i01MbxR-opGTbx-Ul(X7`7=PMHo^=kCScvZ%;$2Hj2IEEM?jpE
z&gx2+drgOfx#&K4futy?#G55UjN1KvN(!&!`g|{<<9xR`Ru&mE+uNp%dj{saM-yL1
zdcurWuEQ4#V9&E1X~BDqHps+QQG6nN74_5vIz?K`I!h@5Y68`
z+WZ{JNW3Ra0q2YVr7D+iCmL*$%=AC#LQvJWqm(sv&zuZ9hEnoOjCU*aoMn$um?-7#
zHD=cYt`Au{)-MhB7~f!v{68GLXtE!Q1~s(I%dCetoQ&JNqMk>d?+BkEfpFL@?eoAh
zW^^Cc^Dg9p`}VZecuTkQp1I@xa0C%9pu)3sJt~2YAKF}FwP(oFojT?dYxw~gVVuht
zuqNw!BaCDGvIT}L`9T(JybQdRuw^fKdjDkqn|#yeI|70zB&6@0w(;b4Ae1(KG(T1q
zmRCpE4yD`9<(2cZ`>R)&cK(}&oIGpMA9s1@k>r>J4!|py80i7j$)tMSGvBP12+?P8
z_#~t}t}O#T@p^v{Ewsvv{IaJ**L8_4R~;ApW&6zVuE^c@sa^}`bI2FI-_&=G6jQ|>*
zy3Y=^2fxv+-SC%Qr;`gtj>5K*r-tJG{H`&$2U7)z(|);>hjx=rL@7G4J|BRX5x{f>`m<)z
zZnOpD0?one31n!gsS7zrwqvi6zh(m47LePq1}KaF4bp(Kwf+FxeqIhpZ304Kl|)3y
zKq~@l(NsZsJy6DivpMuSHX9Xc&~C}@5(>^e-QjcZy^DE1%r6?w`O_yPqbwa7dB?rIb*-D)%lh5UzS&3>FT2HUb4^qOubIBYtL8;
z<6U(BX+tFcm~w~aF3_&@(I{su;A_45#?#l%xVdP1!S_k0JQ(%(>BF!eclob=5=y)v
z48piH`=XdZ)X!wgD_!V->-=!coHaxrieDY$dxcJHcy-<9e`M-PA`v~^$~$@6@MHPu
z;d&1wg6AAmB_e)k6_MFm$a-OTaQrsFepEnRE#51F`>7Ol|J^Azr;-x7j
zcQ(4*&9@dj)@y2Ay{%$5wa$Ti~D
z3Dm4{w^fT;IttH?zD3}A)#1*`A&5C*%YxR6@{CpJn=v@Er{P|xWXU)LM
z2U@oP0--zq=}Ph+FAkUu&3*~{r=SP+4%CW@!`N1C(Lubu2xzM?@ptg9{d#m=G=nl%
z(Bc(VN;=y&tX%q_MGTrn9F205yhZpf1MXJXrK4VwGcm+SKJg&tlz~aQH6~F5teO0~
zOi=oLziY}}zNR))DAVw|u|h94Mp}3_SpuQ^(_o1#263DMzIo)AymTT;?#f@U3H1m4
zJ&>q?oa@yqyhe=2{u&#CV&Ic-4I;XZP$WLlQGs<3Duo?s&McaKur6=SxH&8W2B2bXqt#8q(CYAHhPrCe}`Q
z8Q4jP-QN)gKr0K%eA`G*N&M+9BJ3DxIjb}?-n2T-h2pb)qjCpYX#kpBmd^&Pi@gQa
zzqZ0gWZc%0Ocd@p`;D}F*kL?3(3dbVyi%NZ$szJo)Y4t%E)!G}3VNPn5Tqf$gUDaM!c)dOpayafjoT-qdDI
z$HPZPNpkW+wpTF&AS#;N_4R$Se~N+R{k!*y(5#UM=3smP0+O(YI$WfnRsR04r5f3F
zZ*xH^RnOh{Pp!9;6S^P)7bn?ZB
zBTD%BT3qd^KBl7n20+4*@IEK0Xqog!b-ea=Hbiqx%yX|jK!$H-JqpvbKfxDW2(SB?$v3s4CmV)CXN?>H=nJO<
z-t#F6U&{YVJsMM1_Yx2M;Ess2X8gIp-e6l1k-c;*0z=G=)w@`TWs}CGUAYl`T~K-QV_Bk=LYil7bN^O%E9&=J|cj>**X=p6`tuInZe3{vU4s
z5*UXq{asy%<6y`Z&~cOosx2V9lMwJX3wPe(8vC7Vd+#-)-mCEOCb9)IjJ$Qre_Zay
z_sS$M*1~a$22k&ns}pZmzmrVTq6eUP
zvqJGu8-uSNkC!E~R9HHlMa_aU|4=63fP{W9A1Xo_RK&ZvkR%4Y;}7Yp-#$YV#@0}L
zk5N|?^ApGf4{zI7QaUhclS8&@XIvp~8(@neo8NsN5EjI%s1a#Dqxhq{dXC9Dw)qn8
ztwG~_xTPs*1n~^0Pa{;^$mAc59QCw|3V*S^OB4N8B$M_-wl!ImYWtpXcCzkCJCtOv
ziAIG_mf?M2l)aIZxz0)>WO){{HoKOP``Q^*oHy0yC0JZegg2er?C-~Rc#oekP0^2$
z@!t-YYMMPr8{ecm_}I1Nxlk?`l=WW%jwA0pd2@0g=6MY5=MEt-4FJS{Kpf*gZNL~B
z!iNQe@hYNlAlT*~DdHdI9uDmsz>|I`Xm>O&wih0}q)Hwl8&HlAkrM0e>LRnyrzc|Y
znh|L6nw%t3!^P#L
z5FGq9TD|j&Q(phohNZp^TQrh>ntn}HyK7guoaY$&^Dmy6xBO&;!|2}+QT3x$FDqrH
zsn*xeFmdtXCt_HEcq4@$(Ayd3X-~V3{5IfMNAj&G6i|+`Pkwt^3;R`2R(fC*G&IlLWKKH_uVcj~((s=uZuFGD_{!QFs_0O><*V|SjH))cv
zxrWVq58n&wboQn|`L-%}J2n6H=T#cG-Oj4#=ly73h_mI-yHy_V_+?I0la7tRzyLYq
z-H4l<@T1f!486r-6Js!O--__2w#TIZdna7~)sTW5BT#gs*Q37>^M(btnt&+7Y`&dN
zEbIO8&v$$NYp6*^6b{kxCwVoFaYe?X*bV*>x|T$7I&aj_>_sFyQ%(s~5Xif}#pno9
zbp2M<^db4uCjY_(Zi^HXlf#zP;Ie&FjA}#Qidh6pkFBxz<=f+2KW*O!8DQyjDA;BA
z{TOGzP~Jr}N=0=47zzfMU#;>rBS@BQzA)S*a_=;G4;Q@!s^@6vvf~C~DA-6umvlLd
z?*KPo%knc`p7V@ixAE|tL
zvSgIF#@2!rIwN|yLQv-Nn!4)qM8w^|=SLLdr*Yx!$?qhcr0KwlV9$LQ8q!A20rb)C
zDkPST%MgWCzBBseE_`4UXq8%sN1$k0BaZ+)l*QFAJ}8DW5aA<4rl!li6<}@XwhH`^
zup}<99y+OX>-WrkG?Jz9-CM#;K>-pE=kqZ(h`8mi`X3{|NbzrpGa9C623!
z3=Tg=zq_nfUy{S%2;fqE{lJ7sh=c$HFoMK#|M6h{OY#dW`N}Y3x#Jt5oQe1b`+w6v
zZ||so(?1}BzBYL8zzyr4ooM%k2z{D>+`Q&DA
zd+Bhu4ZlphlGCo{ellLn-}Q>K{tm=4EpgK1S0(so3L
ze|_0Q&*nQbVHw^rm88<*E?V4j?RBDKLz9waW;>xncmky#2=FZ@BN*
z^|X`T&_%BrA?OHno=xvXtYpoPn8Sco!pW#V}tc_Vss_
zGsf!WVAJWkT_Vr5&Eb+SazLv9k$``4`y2sGdYAU7;kSFF&Q*1T(YM8+IF=%3+I?CV
z8{OR!g}WqjAhrE#WWtUp6U68i8!k>PMJemThT#E=@xQpBIEFdK@;Zb1KT2)+eDr?7
z!8PbfY$?6Lk3=c^bbqnFcb%8(@&9ttlL3yD$w`7GkC4EJfxzznq^(Kz0jt=k%_|Pt
z0sq7B3$`w0wj%|`l0SF6$Z;Yv3$7tlv!F`@xF3l6VG=cT8X2{Gf79|?%h#lE^ZQd9LvJ-{gx))O`$*3~pyVyMasO91Y-1H{
zBfe1JPZY5K0-{^s=6I~KUN^P2d^+B?%I#@T2Q@jZ7A@$a#WR7%CDOL%t|HM2aOZ1x
z33^yWMZO~zJ*<$^AsOUn7D?5P3Xe?TR-bMq19B74hQh_FCF~W8r_q0YGkjc&_)QS6
zD}O*{E>}RV>Ez@U*+TPGCSj^++5EZ4-ryerrq`UWKY5Fhik7;FJT_rx@#Yh!
z(o&2z>$hv2iNnj%tv&mE(aCD3r?x-P?DsZAp@Z*q1Oc|OvzuS=P#5CxFMtko8cy&5
z7%Eii9vWs`^*Ht1-Hyme)L#=zARu5WEofGV5wLxu*g8jyi_>)Sx-UJ!KASr#8#^8xH!Y72iFkTRO
zGDS~j=~&aX#jvSZcVutTu6^(LZhpx9>(bCc7lC9DmGt|yyeO2vYidIltqWEFwFu&fo`#)<7?tya1
zslVfvu`#=T
    @6H0Z=C!K|M^Ml(dcqiH0um-{Ml?`rx$kw!`G%9;)WqjL8Tg`T_6+TXN
z>IIYg;AI5givxU%w0E40-ee51@y==Q~JmW5>1nLzMJ+&C2rm
zQj+r65cCNK3W}5&pKOHsvX!1~`(Rc%o?Yyq7qCK$^Ncr~B+w#E;_=6jFpOj|yJkd#
z`a~2b7t?d5b3CF_by!3?@9-Ejk}Sit9x(3dT&rKePDpzQUH1O$OW0>zGM7%O5?<&s
z*^m)Mjel0$eYUl
zhIxpTe)5VNALJgQNIqW|(ek1Xnhe=@sm3EF(hlHAbzvv)p}Q|hhc(5tE-*Ut8w%nXTQx0QSL8>-L>(=%^P}$*C{8<0G&dhtWcje5z04h2lRp+eI9|m8`k;ZSpvk~
zqz*VC6ieW#e(c?*-!UTTWa$waaUW`M%d+;9I`{>1=O}b^Ogr>84-B9H@eUcmvR-uV
zh(zd!#G{qkEl|NkqV~_UNgRhC_W|K9Ux&)zoHFRG%29cKjfkrMaPQR{?}H`AB$gGQ
zZwT60_RC^OHW8~X;bWE&Kt{`nL&T~>1YprV@rVH0U4R-4faHBDI+!kN>gebPuA6p3
z$@|sV5j}`N-wg=95f&EqI^RSEf^VV#7zgmBTj%?OZbBDlKo0WkETE{Os;_n}aX~5i
zHlQaR6B>Ti*zc5^S6Rd7U6#k%5zujI3vQ0$>X3`E4C8(GZ9)L(^`h>Mlu|0UMz0OT
z4KR8v;|OI70zVyYYp!9m<_AIVk^>U3Hz_E11KmdQ&rqWeUQodwX5QH`?D|FE<%l@b
z02nuJDq;8K9_rq%HF5#N+s&fpU|ux!WgvWh;m)lyOq8m}x0SUl(@1jPj4O_{j;R@qAB|*?+KlQ~1Oh%1Qmr{YxO-E4gb7$=0
zh1jV3)=}#@C2PKnl9$?Pt{$V>D+zDmg07u9&lUEaylj3ah7i%_6r62rF`ym+`;@S
z70vBiPy^*Z4)^_)2<1DI!q!cFz`RI4TWMTJ&p
zwEc4S?yF3l0ol#*C;NDK2nYY8BgWoMRX1Y(1BLD7q#!q|BD}6J=U1i7I1~3zVf~d=
zr%wohR5TQFelP#Gy+(p95+iUO?}6!QB_Gaz+l3c^7kK3O`055O`LqUs!*7@7JHj3P
zwvKM)1NZTyDA=1z;@L$o4PPNgYIxBv)xcL_LP(HlR$YYE&AVvG0JgFGsv*#hZPEN3
zDkp$TB;8!UmC>-)%pvyVqb#ATnI7L~yp`j)3Tkk(8FO9Na
z)^#mMD<^eD*@7@m@Q>OWo3P5B)zG*x$ESlictb8YYsP?7AUuci{_6x0U>vF8HLr^wWpm&9Evvn
z6w=qG`5C%NwzUxh0uh$ok9RHMW0y|@NCF7bel+>qkumGFlnr#WdplLTPj?1
z7_PH^mC@||?MC6Er&Ryl+ElA1>RY8Lp6}>Cu-sP*j&}7pt84X)Ooa>zsyp?-a8hc3
zRhZ2D7w%kPWAMA4JZ*$yWzEB5#bDK}6Vhe9g}cZgymv;Dph#oCXiXF~LL6oHh6{vH
zytv`WtIb4ZlazvFBhwnWlcne}$K@gz!#ag{Y%Vl#HVte{<1h83ykDpbN3`8xeqa9s
z#!2UVHFjqtSvsGOADAeqf~eZ6e5UqP2N&>xU{5Q?LA~fXlH%#1z{Dj+~-R$6yUx
zbB8=DIoq6GDwuXO+u<4KicPo2)x`-f%7X|0uVp8a^}Va}iW`s>JVKvwJDwjQUC;NO
zPmZ_iA+#I1f!FhAxzGp1hbityRKF8jDBeXb1AF@5z?0`e>vL`yx2bB@L&x)}``o7<
zve=(}*!^~ZQ4Vj?D3E0A0keSE4HB`B&%7sAzE78~G-D!ENM?g_hHE-G#HoQM
zu-1UrFxjG)9$EZT(ue$_qa4q*7e+oTsdx$z1MisAyIMC=8D>PcYUNH=zow1atU93!
zfAhgT?5j^7wh&J85LPQ~)9^Nq3hFdihj!&)
zyg+45XS?&NHT!lk&h)BTPa_sd(z+Yd-Xo%<>@x2_`TfqSs5r;7l)Dd`M3ATy2o+B!
zBP$KTZ53O6x+wdE7dqEuxN6oYIPo|smS`KIdL2YhCd}B!Hcr8-vEX%Z=P$(}@sj<1
zYF8xHLA7%ij87n|Lz2cdb!q0Es=&gc-F14oNX0?x?prSlq20jL0v$Ef<_!~5jOw$#
zabR0nO)`z1I!P)E;n2
z;BW<=|IL#twxRf^XEHWM=exST76b;aS>NjjWc|16z?cs{ZBSpkMgzbHxU#l})_r+$
zVjp>YJU^%YzPN%o>pOtx;>mZ<0E@X9MtuKK_~HJE7j=(wcz#lJ(c&;GtY*@W_R6-o
z9{gp>cmymfwO1Pd-cPw!Yv^^|1z?b^^;b$0O&+m)5JUWZ24?Yk?cnp;(}p+TyR<|{b2(^hfa5eY3pnu@n9*99A;
za->O0GqPc=yz34qYHACE&O`We_?vAjQAEAAMX?60dkx?ccF+x{9
zklDVC&!H_Fm|{-Mb6xGiIJ3{OHj$^Bm41=g;40WX4FkC$5eU+=84<7_q=BiPts@+KHMg&*0NHK07
zy<~igoO0aE?9HB*shKN9;bQb_Sezue%YMkdY6US_3I}f+`|n{z#6{buPpv-9fpQ+w
z4urCIxk?NPw8^cJ5CK(vAwd8+0VrY94yu(%^Fr5-nCp$thYB@B$GRz
zcmWzDsO$MQagDAzXTQBn-?q{le!10TnyXgVz*iY;!~!A0QI4qFd)21gTuE_2z$by_
z1F^M(8GN{QYB$gE`{?6kd;wo8Rc6fRj`WXBq8Xch2+?
z*^Zp`&n;l+Ii%E=rVQi7``n)Pha9%enrr{~iVC3EKr{)@@LY=8L~>--`IME!dmoX_E~ZdcPna
zjaspZl;D^o#TP_-Y=n|rroU+ME#sFTCCBR`6>oXElj_T<&J1Dmq(JEsGw~ISj|u+3Ut4T=v%z)(3$w31mZ{YOsi?eqoWF%`dn50q1@7f}Co>P{|JcqU
zYREeySf3*K+`#2J4#(sT#!%I7{cRrtgbw#?MEHJae_3BGFMPyfYcS&;+fpr~y7*_$L{dVQ6(-t{Mz28lP~tw288-<3)5&x5
zIj*|rE#D%ncGvScfx(xj%P*>Lituo`p}icB8cwr3hV>J;k_g#S-K3?Q=A)TKAcVQY
z$*&;~G=3Eps>RHmW^Xk6Uk_(nh6CEroeRv}OOD_6=%wG2$5Hc*Q*Z<8BTPRRR&clh
z(zHflz15a|@Pk0Z8COpGJo?5BEm3v0)4Q*%luMrPo}W&tWzmBLOooD68og5k31%pk
z$Pk3E${_FPgiSv2mAr1vHrzNI61QH`=aV|~A?|o7J48xh1nX2bPisnf8qpJb#ilov
ztQ44|VnAm%y*}c-n4aQKGmRoK!^Z@9pHJ&FVMnROC83?sAS5T&-nqu_Qd}nz8!v2G;?hWSvQC;HT
zR?GdPjFIM8zjBd)F}~G8HYp<|T(B+Bb#m!6lN{ldg4zwANb15Ofir|}E>TfU=pzWe
zjM80g36s@-WaF#(BTFY`i$r`k8zP=@6q7#eZ%rFBJY(PqzJV-tuB#<~9g(CGM2VRB}DJCy$*
z^soM?za4`-ynYz#yffaoxq10~`TWqi?)VVt@mu{UQl6u-
zk&@IIh7h)<#I81ofF?$TmlT&YT15>CwY>DE)WHOZ^7HeWbxkcWD8P}J*3^qdDh
zJ_VgP#IKGM`kz1umRDZ_jRG!0x9G*n9w34dkY57aNZ@H!=cE-!DWJY(XC%*lt%P6n
zXKv!tmX`z?VSOyI9iw>7Jg{3)0+$cF27_a&jm4UgpA{}z^|k`)A=S1M9@H(>it|fi
zx~g^V8JM>H1Ajk6qq%@H+wPLwzR>1BaEdT6vs>i%ECTj!!ss2676&08FImh_w#Oa@
zK#FGNyfDCPTcE{DYb(87?%yLRjngS|tF!H|xx-YSE;xCa!JP(_7yzkCOGgfqpyUi>
zsiU)TS9yfxYJo>|&?4I65H~waBT>8TU=kBe&N&-
z!uM5|_*^#*#&R|{H%@~jZ<&8dK2WN?$D6Y~jbJD_debP3_5FQQX*;KHor~FT4uh`t
z_dKY_85BDvs(Re%0yintNln3?5oMZCm!_zWukMIe=ktu}6f8?f7
z4E1{CgZ^5Ho)8ZwS60Q1GdCjQ&_$fH8KFzS_#I71-OD%7{`HaqgicAe$F_GM%xkfk
z@{%fdXh=w{0rQ|JK&1_xh_JTiCC+1=m%h`a93i@5UYuVus;ruKF;2|4ZIEiHQ)^7C
zJByZ?AC{F}GK9T+i=cP{VOT#A7WXdhvOh-4;O~+NgKE5TS0XQ1+b)@Co`uLoj}=3Q
zf*|Fcf*K1qQNRyPNQTBjOj4lr>zl_L8{l(De@C
zh78i>1RKo4@>Y*4rUJ=tQhsd}x?%kKbGqSm)|}!u!y4!MC6o5G0_kH!(%A`c7eX|a
z;>RR+3|4&Nm^$YsVw-mtYKv|;0w*=A!iSggS(ho5e4`${$w?6CYMCfn!IE$avSdPU0uj(4WpuGOq{Yei(0;Nje=x=h`Z3UaXk6}sSl0J#n7kP;
zRF4cE2fmBFE#U5|KHemjVNQuG3gJlh5u#@XUEgI%P+cU#nL3f
z3v!FM1gZWWt!pt*ZQzSf#70SVn?4y80?T}U`Ugfo8C5*`zun=ZuwKna5vrTG2@wf|
z;2QU}Aa)Z5E`9o2MRS)ur?K`W^fM%N-*j9CFIl8EdjCa?z|YbqCshLMp*dOpt(2
zl_7b*y>OF1a%sUjqslU=IKxiA08UkW5aQAz(&bT^_1=pFZ5}0KVs>D-T5@r{aU$DN
zhcR*2Bq%uLN`~|6-0Mrpf$rBKMeTcwiyWKb>lU|mrNuo^&`}n27U4i3XPXi(yXAA)
zj-P7Z%XBG%Q`36#dqbSIH2;Zr2mJdvGBJTsMti&6kr
zWuhUC8HoUw%P*HZH@1PM9x}{!p^2rWN_oogmKMQy4h3=f`r)pYm*(=sB#=rKPPuN~cy<|Q
zcq@&2L{1@=1aL#4`OrUX^p&T6m;EfiZK(~Ud})znzXs=!ftRdeLaY|@h3X`|A-jII
zI}TT;l0S}GHWACM25{!jBcDB#D0gALe;Bp(&G)~&hx}gX;~I2pD_*+G@u6S38=dw6<9R3WuJN2m`dsQM
zH*oLyYmMIyN3ywhSxMdg=#I+hkLf4*;7BwA1_{5=(*HC=&RibpRR}5;iLSD!5EqjJ
zabV~gCdg!pHHv&(QYE~#pK-9_!kd%papSy*b2~GLXq&k)>w>)BDWWSMVOkZ7iEFa%
zJ`7TK2)GViB-Z9y8L3u&Oc7DrnapW-;$xZ7^mf3SkDe8FMsUTo$`TFNH!OY~TZike
zAe1Xq+f|-%Y3Y_zye}|3BzF7)*0)WN?W#Pnj$)l7Z@OQ0lfS-T^%q6HpUYxN9Xhr8)nkJ)c#=pJE#u0RfvsC`??LG
z#Qv7z;Y}6pG5lz%P=egD^A0&DFi??hVX)6}_Fp5y!2w7Sbf|+kOwhs*5Hz>n=0b1;
zFqS?+y?RESIB6ON4r`U6-L)SxeO;|mA&b28C}fj-wRv+d%BmjP6i$kK4=@UV63k1
zM&v7PUW#oIB6I^I&C3xA{m-VSUL03Qq#&o1s=
z@*#<8vd`#*wb)E};IuTz&I53}m+A(^+JYZS3#cal@~nQ}Tyf$rz^VPC!S49u`RwTs
z9xS^rW>%EqfLVc&$&c
zp?jeZbkhg*VMH&^WhnxU8L4$d5I-ZM2-rvTJz$Oev>n}hG2{NCg>JZEeXPHIu0G5k
z7o)oZg**afDOR{&2(tbHO~g>g`@D4H!51S}yefnrCpDX{Og
z_Cl)rQy9x&Km7&&Rp0h|CO1~}GoA$QMV*1{pX6hsM5O%RNl9leZKNL-j}xYZ>6sca(wu?s#Zj>p(KklV<4
zXWCnwv{}0Tnw;uoF@L&e@D8qRG0nR*{WvoHWs;XUCu(l1nxkfaLtL1&M6lt)o^k4f
z&E$OZT>?|;o1>#`*H}iyWnNrS$+niy1*D}m=)RoY5gLk{
z?n@;iamcDvO2;o`?{xYY1)slEz=uU|uQ0-829e9%93@mb{Wvp-AjV&l+S-4rnrgY1
z|HQhi?y{UVtv51lTrGme-P4AZi6gm4l~H*bS)3V
z6JiZ66_a2=+aY@!yg~)cfy9q*3`X}1MoC|cwlp*Ismti-#33vI`K^`njetXJ;lyJ<
zUYSg9Go4#}5+qlO9(PH^D+I$Hz9wyUS#0;C8K3{S!C{rwRek+IvHS?De`a5dRPs(_
z;M_WDy!8<=qK76TgpqQ0Q4sa{e7`
z>zwRKWO7ji36Y@CWv9_Fx}}2ZpjByl#(D{eU=_*bn9_Csjs*
zwP81FFR{k6J*}433*{mHOl0!A&{(L>s4=3yh-QzJ)QGf3&SVWh*MRi$Pg^gv75wH&
z+ExjGQ|b5?fTS@nL*QWW){p;k#foC~o0-e%pgm9`xsPVKg^_&P_oNCsZp4X%{Rry}
zQ05s%HD3PxTc|p~f2%?J|hv`W@95nH#A%g}iB%+1{tIOtP
zvrjdW12p+w9fFN65O3}sv1)_#3xi4OGvG{#Y%~^!K`~fLfZrVW0$wAw-i{8=J{!{Y
zyE!qI!fhBQ9^piW(kaXsxc}>&!nOZe@T14r_blszY}1{Co)9u{oCD
zL>;CUmNtghv2(oqkfwo?YeYk**Y`5uuj{+d74>7JL%h58o7%+f-j2BS26(=m!5{A`
zH)v?EM$r#ZEt9=;P9gE}!t4q2Yte#MWK{3LcmB`N78G}ttb1G%#Dge%Bx$Wr>7cr|
z%K0e{NF0x$Lw*V07KzZ+^-cCR-rFGv7C>uQa9tRYIefaeIZ4S;dc)C~nqFc&UE!tf
zrba)%R>-SNQzT|y823DmP_&w0slLO*+Tzy5`tXH73R;3hLjqZeY7fTo9u}h&rat=Z
z65)LmtJmh=!)0!WspJ50mZKn8!=*bDF51Qkf;%%5r8CC>xH@bGem=&|I(hfnbCC$>
zYqUL6t17g`H&}>KXHBu?U93WrzvM#&4AE%>jSi5BxsrXQ;}LS(jkLbJ#?usvM|uqI
zx|s>)N67un9~|sX|oHavT_LWl3e7(I+;4|E{SBafbe+X4*K=mq
z*IAX@AOQzQjUgJabTAHBmyaIdbK6eWO<6}m
zbS9ny(t+3uOnR$0(3qsL$S_&ABoxb30=(PM@7nO>*D=PjqieWV
z)4rUWCksUs1E0hy63^UTFQ1^@D}Y17Ri~|wnPfg&hCJlqyilQHiB3FzVBl+s=&HMJ
zUszZGswXvAQrsT4_;owo{BBL7Iz@BOW5C{@Q-AAGu!by&S&GOs4`mHRLnM)r=*^9C
zJ;y^mzPzV|dcui8z+JHAt=(4kLA$rA+&a8}KeWK)bX`hcEN)CHL=VqYu9%S~_1_Il
zpkTzj8+qnm`o@x|`Y4>41!`qC{k!%hH1`A_)kxu84ERNX9yb=Qd*Luorw~%T9;zXu
zcyw!Q*!z}vwlSlG_qZHz8C*yw6iL=heGn88;`q=c6!ZOYyFL3
zCqmG17**qcLZq^v)!)7Z`EHh6%5~ikUj7>%x}I%c*B5Od_t4kvS0vwa-$yvNbJEGn
zxuBk!SNx_BGmgE`WD*8*1`M?uk@Uw5H~VWIZFb)`&Z{tDpd+pg@`Wu|pBTN-s+<7!
zNtqCaT^{-3H*YRY*BXAn>i&HpLPWehVGK>|5WTS3!VJK#;*jB2JZXTv$fr-Izaj|_
zG=$-pU*QmV0=W;kQ($KH*D^30SLkO5w++_w5((J5mk9hZ{fxyv%ttRxoiH-<_tTgd
z=?HS4GKw{J=8yz8M#8Ysqn1p^0Z(cnnrz!$jA%fD4wa=K4$D*$ufbL|@s8k%mYF2p
z$c|zxB7Bk?Lh_^M5eLZ`R;Q$JLAyqb@|UD0LN0Vyc=CG3X&rqAIkZ$N)Im+9}POVNjR0dFK-4X3LYlRg?5>B6neILe{yQPv;GP%c$e7JzETD0M{l)27o`ucu85RDmil4r@`Ypeht=w
zv(5nzhQUCz#oQ_IH0Fs8>8*!-@&hr%jVJkF?(Aoye?(a79e`{0Q+d;;k2&5+--@04
zF;5vR>$ty2tmVmBFTGAV4IvpU0K|s{=P`VD9kd@TJ_|GjUrHiEB>b;aCi3e$
z_X6>JqGUbrub2mkzONvT_PB1yMe>fpv&32pE;)zSypbc
zW?l8-aj&%I?buLHhBwxEbm0`3d~ehe;IoXt-?XiKyE9@!zAF^rAGHR9=)(HW0N#)?
zbTS+ty`}@LQ55%%8yT2$l*&s|CkZulb+#X_&43ex*k$DxZN3G@hn@V=)+Se}scMKv
zm&}{jmory12I3Nh^@4Lx?2~S?LVSGuTsO%@w@{=w;DD>~x&oVyTB<3)onEc)BAloZ
zFg-C~?ixYj`QK$xg&XN@>Ui8-B!N26I48EW5t)3vsWw@5g(*V?C-aQJ6Fxc1bsw*4
zWFo3I-8#K==N~8(4XRA4b9zxRVUrZn17-0tCd+*8@^M9#{c6QcSiZIKy9E`)a
zC0{^f>)aNbI1j~xbqRfuBj}iUBYAXfecj;XNS71-rl+4bz0<*x;XlhUX0=(b=C&34
z0`Wqz{EL_l8lWKJF0Jr2a-5D|#pTbu-2itAg=vt&ELUEJSwZj4I8eGW4A-ewmsce{
z`=l)DYfd8$mnB2MMz^M59z?dI`Uaxz<_>;;OyzWIr|+hD`s|{nDFk*tPmri{=|_D>
zhpt3HsuI^n^Vlc`atc(uk*m1g2gkwF?yyV5-#%RcS_mXG)q~~fCDZ6cKJ?!%=#$G$
zocXYkQwk+OPgdU_oSb3phw}{)hE$><{n^X}bLzIg!``a<|4CT?F*_Scn}F^>6yo@b
z>hrXX1*lQ!?@L}!a<;HuZv@?5S0<;mpSi3QB2EngbkAXp>^z}wynQ3e(+kRMFLxf{
zZcbA!4lLcbU)2{b0YT9sNih^IgX4$KX=I_1C?~2
zL>_x7>W(4Ru5q{dK4I%5=rr8)y1%3b1*-yGvt<(yvf3-&=Q+-DA}b3ZPWYX8s6=;B
zlC3pE3cJY3XIW5^GldcaC~HyE{IVE%{2&+sb4!BD_5$5M(s1#Iq}sf)KcB7u0qUI|
z(~30{-R+@@$#AbT{#w&GVADHe%k|VJpNnTwj{%JCWzZ*yPw@s2*-UZOR>X}XJ@aC<
zq0c^BYI40!2)EKzgfpgh-p>F02Fy8R(Su^C8_hxqgGPDnR
z*g~~-OnXMq6T)7CV|jAL+8r2r$_4vJ<(I*I7^9BC)KN!OpUbl#88hW#@rFN0VXs-P
z$c;fvstu`yr>>ipuk;jG0;4k!$Mjq)?qEs*VpqQ@TzH->Quw?K)@tBjN^gXtkIZVC
z>2q32skvlOJCxUa@h%z45LCxBIh2ka%QI??PPFHo3_N!AmO<5|&?oaPE!+4D9Vv6x
z1$j3%^)zKq+?Exhce)TA+UiyYufr9HrQcTLBr5#f9=W={-PIS?TpVSzim8E^B?#?P
zHG-L~x`Lh%BU#81;58-v@D(~KZ19FBi@Z)#C^5}b^}8VW$=d_(Z$Z`YJ7OZ>3Lm=j
zHgL(^dt<~2TiRsGY760-`Kl~oOWyBBX!K1>2N2;?D76L+5idn+Qs=vSxcATMN)k=R
z_M&XW#W|mr$3DNOURl30tgoXj{#xPSzJ3WE>6Eq~2@@30Ha{Z`^5S!%|5B42_GoM?
zP?-=rLnFeBzrx0iT)lc`cM@ZsPBLf|<5e`41IeWqtfxuM6iwLq?tGcsw-EFM{rD&B
zqIJ-FM^OI+p@#}%?m3*Q&+LB^UCjM$=baIvP%~fH<%$cemy}lu-z_C?^4BfOSCYL^
zxI3hPULQ0g7Z~K$t{@N+wy;!{?L~LUE77%PR8!@*zZIr#`@d}A#3a7^%rQvC0iQe>
zO6vlN2Ve&T1*inZWP1mySAjeq-!hbkE2I2EyFf;r_}H?dcE(LiwkLfRXj!@Ox29%i
zi!!qp9Y)|hFjLTbZ=$f|idHfFtQCvppvu0(DLdzx$hjTZTp!&ioBZS9kD+sBLH06y
zO5Zq^%utwAlTmlqN}6&iASj8C6GjFVlPCa#ml3`3TR}``u9T3cwRoW2vRVs(ZovRx
zG1VF}y(~RO9Ypb#8z0|l$LyFvF&bUrY?&%~aUYH?pA_b1kOn+Ra|Z>KY2?XhgwHL<
zZ@Q1kj>@GwDyAD*mn-YukByUr08r^RPN$!)EB>690?(e2uI!|Jc%*9jiZ}lNkxX`|
zBrL*j27eVE8ah7Qz!-s&1{#q0!CG8FRLslCFmv_~zwW~9szI3M02t)*T4U3q^81yz
z%Xk64{EsRG98|hq$@Z0^f8`pD-r+C;^HAe)HPC?OU>{@y24V&YhnLSGi#GEbRkk&n
zRn_a~te=90q~c$w1FBxgYlh8$sq6SK5+bi*C*F<4>SPFTDJqPb
zr3-m`z$l3GzBXqwuvE!VrM9*|K&fd|mPIq^??5of|lpizn~JS&_;hYH6+SE9fS)AyhPVab`V9DP9`S#i;vU
z{LHjIDs{pP9EINN1%O}5Lr$Y}#y>;xrJZcM16Xwjks_~V3rZ|Ai&TA0|MNx~1pFom
z_gr1~^p`TPi*ME+0|H&wl{HSk@=-uIJdlhUVdiqnr4-N@)Q#Gp-X~b15>=!OPjEt6
z07LM~)_RvNbrZ@aob5Gs9#zzab{4OMBW@Y6eHq$Vc3ISve82NRui@J&1HJiKdW86V
znHt~Br9?|j!OuR|J#bPtR3N&)rr6EP?*@IxCoi>SyTmnV2M*g@_=Tqv>eh`%lrbqI
zvN;hLj7B7R$Wgflqa5jBs-cE+_W9r%;Z-8sI2GG8IF$=rm$ZN-Vi+MFDyvg&+k7%C
zF?P&;xzC-p{@!(LH1o5&K(WErue5tKk18d-bC8U>X~FE;v=ZBpiO0ov(g5mRn+^SR
zLH%f9Cfbq}iK{Wa8)FI^0w!9aA|+VWEO>f;niyoygO5{Y{cq7-M_vfSNt6=nc}WB4
zlQo=Ngb!goAs&}c5Q}WCqt#DS-p4ne8JzdEDMf=<7ow}si)3}KenbJa^||Gw+N_p<
zm{q5#+XJY-tOF)(q_JuK$&#L^cjjsb%KS*qa>0!bMpkS+9H=*D2(*rmx8K$w8o$0Q
z+PW@|Er3x^uTT|MB&O>bN}Xpm%6k+{t5Vy(hJe
zIOgHX#EZW{;F*l4uWb?O;%%geIOMnJspL|lGWill->So$nomBVxhTTC>n!guge^}M
zu;cHpGm*qgpzK)QX>_U_|5YpF{i!MT5LiS9m$tQ9m+7tW(tM?AX}QFViv~-wMl-|Y
zot@v|+$IdIP@fZ>W^b{3R9M>}4fi5<1clcpO>8QVL
zb@A)JA{pJ~F})6=bwY(l-?eYyNHOk8QZY|Q1P&8T=GQy=
zCJ5xTh27CstyC>(mlj1<^vVujwFpg4HhsTJD`IdprdFDVFJjplt+icBZ|YkV;Qaf}$5E2_tc|U~^~?@y8pcaB!e!+xvizi+8siAe9C^NQel5K)Jn(^LaD=
z@h*hBKI|5;#e8;CJnS4VI3M#GZ(#x
zE?tpH$wcf#_t?^C;ajw{Vb7klgN+02eX(4k=K;L30L5u_5TAK1v>Rng@R|6SpQx$?
z)ArLf?#~#x$#H!0YliXAcl_D47sA@2Hlu|tsInI0nL2ecRF*w%!q?_JLiVtq=jtoFesiFlqot
zpP(OSu8|`2@5x%@*V9OTo$YP0>V50%mv4)C^)Ly;p=DGBIe=39V^2>B>VS}r2OuOx
zG2ZfYEwA!>{VIB0T7>K|wV0>brRt2h54&>>C1Xm__VJtFQZm=7O?n|w>Kn87d;3>b
ze-*9M4vm5>Vs)C1KXigOc~3)P-lPPC{_VIvy3U6~5#$2L9Guu%rtBFf@*BGlF$#9Q
z5E*@q8%k!Ca=sH(I$xPUkcyJ)s=&$&@IHr4Z|(Q0>AU);X9rw;bUTBNQ(8t%D~27d
z-KXD!o?Qag;wVoA>|1F|L|kKHYV5CP-L#HOT%<@Act>=EmtD1Yk@v5GOi?9J2J3YF
zcox<+u9rB+AK@**pWdC`^~ObsG{`H$2;0=c&~*{qdJv+zI)X$prQ@|QF15r$W9hmF
zY2DJl0Eq57v&oB-hZ2pddA9KSS|o~Q(6~89f%BE3r8CUNP}B=KubRq|nKXY^6u9(=
zWc0YHrr8HgRz0yC3mB%#3wqvXuW{`edOv%r4dL&k1Wz!;$!~$
zH*Oo)_*gTY7I*JwA39jDiXFq6WQ!N&1j?Ny!RgTNhsWUaZ8shXNfV6wXD+ry)x3sJ
zCfyuuDlA;fjFl~7bni4tJyB$dWzKk}1S+%IwFSQn$opMb8NR#I8Q5Kbd8pNRz82}h
zstspqvnTe!0#&A9%&bCK@doTo*_zt9h7IvJ=qSqiPm#i%QLPJMPnT8r5qiFu>b^6z
zT`%xbIi2vWfrcU4?|wRLO%`Sy{$e`M?oAmHKHy56^F!MgR6MxpR?W=q5{_H*tY*1?
zDekAgl7#MuFDThZ6sr=@p#3sd5=_DzLJprax1h_p*@Akz*YccsymuL*p%>ifa~t6Z
z^IyydB2mTk#Qb80(&fpnt^e3J^A+|@yCZKAi{tZ^l=+!ZU-th183$+PGz
z{~kLt8P7xkQsj&AatD#Fyw-7p8$fC0qeSC3hrXga67IV1<(DWX7xCPM2m=;Wz+L;#
zU<(zFY@D3bNZfDRh)a%l?MviE!+dR&;24OP`vhwZZ0x4iqB(ji2Mu^Mi)iqwxeWuG
zb^w8Ez58KrG~AOmD$*Sl{fbG*YdGDxlFNj-mm(^*1a
z%KZK*Ssv-t;Z$!eD!7D~N$S880~^@Z?VfL}SqQ!+%MW3O;vuiWXMXcm${wr@q)mGe
zs77ru{^G$DLR`P8!=1{l*ikIi
zf}Ml#zoc1JWF1RJE0opC*7kf>xw)`pHS4d)gS}|E)482Jm-l4QXDs`Zucc~eMW$5W
zVa!i3Nx8={(4)gnGGjAk1c$ny>qX{MSg<64-5+aKP5~xk4fg#5{ReJ?Y$*QY*xAn*yOXSN$1I7z7|b;G94ud`J<5xeO%VM-YJ+;Y@4Z7&
zjfMp1IYWB?dRFgn46ZJ@c}oZin1(jN5n8&rR3U-F5|@pCzbG(GEYmp_qcghUJ3wrm
zd?I)0!6p=;@0LGJi12EgK5aoi9z98aDopGn=5oYFkB&>!Rif6adkP5c4^yET%KYYN
zD|Gf7+1N$gh4nn+;aOs-V1e@Dap_~37-vP@ayM|PQd2s5aS+nM^8eJUl
z3Ps14De2x=8ax5c&$fwC7l_;Qiele&Uf!=x=Ry4pG)i**oV6qf{NRASZI$WL-K<_Q
zX!~Z=(%fU{9#na(jX!h^#O49g>t&l@8*uND%l}o9`yl;-@|h*%)P#6kKQwSp>_6aA
zi_x9@UG$kq@j0q9=E&5GZ;-X{iRw4jlb%a+`jr|?ZLq1GZ3u5t2%n|WmSi`)=x*~Q
za@$9HO!s9&%+&8gT$NJ7mD1SU)2g&Os`CtBS~qNIAF%3dKG%l
zAK41kNG_;S}ox^ilB~$0I9>evj
z0nIZfw1i+VNrJWt>4mP50nKT5D$!d3>43s%S8UaL0O>qFrU#aJx5phuk%t{4ucymF
zjjDx8ZMheE6;Q2ds;+g~u%M89sPrl2Q|EbJIpQe2t&#gPL3(k+^f#qNlQ9j#U!oK3
zaJ%QN4{dUwKi4S2X9TD2_XFG}l@#NShqn^`WsVS1*hUCPYQ{rDMIG%of(IY8SLJ2q
z5A8YU4|Z4fEE(tTl&^#ZRY7ZL({l!4Z9DhVX|#1ZJ0O5xi@zE~i{x9Hqyu%EPv<;cJl6n)FT
zr6Z!jD&d(?@>6OEt6qr*^(rEqTrdpSp&H~L!fFIwRLo`XgosQduxT?)?zqkUKjjxY
zCc6C|Aev8>r%d_6-_mx=8R#PN<5E#QxvDEdTR2?7f1-c*>zqLL~gdt3S(8^*r_&MZE9c
ztjxva&O)h<{Ef7F+)ZAT|FW}Tt48^OrY&PrOVswz5aGn$KK;U_SF*N}q&rSA8@)7X
z`AjV$5$62$dW%88Ajorv%=uTH4rNc@XDUkQM`~nDcM%j2y}+KpUKWg|F`m=_|1yr7
z`kT$gIRiWL54_x?n676(h`aon7~Wc)WIg&{=Q)b-tfkEqaVnzZ6V(Xvv2*SZICM9v
zG52Z)=NIUyp-#ASI}7avR^U3|)^;Vw#^Wiq-&is%7w*!1KXhZSGRxQdlZv2&;}s=$
zlvtN`1!;SPiw*?CBv;2L;%~^5uer(AWP+6LG3>Y*Gy3}<$xCiw+gcYCCRc!(>i4N{j
zm01RLI{LuF-orbh_a(N%#Oomwi+q89=4fNzrLjWxED7B+XP;Vg<-H>x|8vN4bP^Xh
zTRy9;m|8C<`cPcoc?+VT-BylG{~Mn9FAJ+_UISw^uo_z%#Q|?Lo}w2
z6Kb$j0UhPa3*%DC*$rnkt2V1#wW!kz0oD|LF)cDGOnzAAxo*gvb^7A(qN^dtDPtI7fbck^F0t|54C{&lbQfge3r&(z=li5
zC9bMP7LA>^K17&2xsto7;39*vIz`B`Sm+!|KLM%P@(BDD;Lv~p^9#enXkS?c8Tujb
zJDX8$7q;=DFPw!y)z2{BJjKMy1b+-br6k5L-a?)fE`0?xR_-)77D-67Hd%gJ-bJAy
zsPUOxrV3LbcY;sk^xHU{+wN+B^=Mj0=A;TT#+ruUap<7foKI$fKd}XXr~epx9hqBo
zQP6VQ=p|_TzH=Qaf*p$MXwY-#b7%!obrGiD*uzcSB1qwKF&yn!AC(gidW3s?pZ5{z
zXlgji>KGO)s~6$O4z1GB%PW1ac~wix)JB*pR!e89=eatm_65wpRcC)b)OfYrT)Kt|
z?2sL+`6_QeKA1%qe#2RM;Savomk>8$#~zrUdFF)~LA>Zrk8zbZ9h!2LBP}5gc2(8f
zP20+hyfJ0Eyp{MbK_#XUCLft-zribP{o3zTVHk-8A9AXI=a<1nN)i8Q&)+vx>K~Mm
z@=03Pa~CYly`wsLIa*IDRjGMNxtvx2A&2(MIC;^dRVBPli?olOQ
z!nv!_QbOAg1rokO)YyJ9nPzbEPFZM*3EkaO0?PJ-QsbE<1vY6znT4Y>KqGkFJI5bm
zYNXmTYmCcsXtf&TsgCr_HX7@nc}LC)teQS6CkXiNby`$fS+^(XXxX1#5Rt!Cu7`!p
zlJVtuXyQX%LI<}_67RQE3bl-xf*AN=SnRWslAh(AHm93A+V#8q_rEb7jcBazywBZK
zsIt`k&7t6v^C#!iA3qyB%0r=^86r0FcMhI89Q~StbROEJ$b|M_W;riH+?(NAMYTH>
z+?(9Yup}X~^STE%&b{6wQK^4t2Yh7OY4+aJ*bl*HX6kzDsMc1A1#qe>>z8Am`3%&M
zZCGQ>(lXbb{biG5*0$|Med5gL-4uFBMt>?Xt%G$lgmK|*qAF6Dgc$fCe{-8ulZ@(m#kidpcJ13P-
zxv-jE()}*f4H7#aq{ww6!t9QglYWPt
z@9qPqbY=qm*+OBpKD#j56b6QQY?hJ!LCd2m(CJ>JBEdtcy2O6xpkF_(R1I~z`+>|{
zK%Jq~+gK;65vCz&E=LC`x>uf)d7zW~u0+i%nC?P!G+gz);z>V5($jUZxFa_=%Nj*X
zb&(2XTVSbI=4f>wfL@fmPUHM!tb1<
z5S??>|L~WN3?|;ZD6ML*gWM&!<
zOv&eeVjj<7xU>n^pTW8Npvvq(Jr@*ah}fTlMo9mbjLy3o
zaZV&QxH*T
z4cT|Tf&aobbFOg7g%=>q`P`GvR%dVXtqSP5je8<<#bVKq^
z!tDeJzCfqmM$fbcu837d7WYJ<8t0d$q8@#J;@;y|Dh7m`5Q>#kAo+;1)-F9;R4KrE
z@V(N*a9irhD9WT{ve0=|PB-oG33SZcb*areauxYMQX_QvnOX-bUQou;Z4F)Tg~_0V
ze0P(%uAXGj6g~<)i82Cb=b!@qEw}dW%lkL#4L}*9vU;E<^exMQrTLKykg+gpMui)#
z(daNDA(M&jrzuq1f-L-V)
zZ$&gdrjnZJ#0(mzd&Tk^@0KXX{p*XYrQjx4#qZz~nX|-n
z$p#Z-#p;SxuA0rk)W~ktzAX2__S37Y@F42j&Vt#!{#rDT%n`fqfqb;k#Dg|I`r^sg
zAa^tY`A^z(n>mv{fA*avsa$(HRr&GSKTMX|fhm7@N&%^^4+Z|I)kq1?ABjAp&kQCe
zxyxk@L^2921Xu{UD!#IUf0CZuA(46dgHStQnSMuE&MWA)*#78nSudSbeH`~*F>(5v
z@tLnm58&KZZ}?6i$uJ%5J%%lk3*&SNn81Sy5WG}o7y+J8fEiql6FGgf`umY@
z_*8V1s??zG9j)jh?GQVGhmQye^WJ=KaTI=7fu5kM@EjrQ{pd|gO&<|-}sxF~gf)6Lj-1}0)uR_`=pCViK1
zJAPU6uG5k3eDw^p0WbNQ4QO%}l(O|jGJp|X6~F^z?hgvEYFb
z^!bTBS4WIMq8HmyF|2;Pw?B@$(#qbAb%yEt=@y2`PsEAO*>>zHazPROEyt1Vf7~*=
za&pE*J)Yx##zag%@cWwa2DvaBlU2ArwIzO9ko5Y6jcJ+uZ1;r9zU}X*5{u(5>ogBfyS)6Ou3qYMg?goi=RODzSDtl
zuq`Y_hWMuI40#md!Q$s&vV;JVC|G{-ziB~M>_nSCq`Q>9)3EnOr{u{S-^nfP`*avX
zWw}*YCf8;+ZXtA`8Lmvo@+(EW@?RI9i7Y8mEibcg)%Cy|01ar#)%3M=wUdT;Ue_Yv
z8hNc#xFUKP_fXt#qhg5fO;w5A%EyUp=8ET(sZw1^v4FU`f=>m&KCq=piFU|%S~4(P
zi7y@}?klw6#+PfI;dWojPH$YO@rq~Jq+vsXv3&Nfq^slk39^Iz>x38!x`h{OPKb#x
zy5{N-kXg2%6BmON1N)yR3VwrmIi%*`g`-|`+wKW@aNCi=IS}kgR2OpS=|;$!4`czs
zz*>Jb0C}P=vCUsr6L!>B{+~qq}m|}C1-t5(3I_qpF
zn6cYIFG&{kUcjYk+ioY;@Dloo!*P7V0Be^li0MDKy7!KbSK4oDLUTHaoi*|5C!K4%
zY#3;5nkC**0Z^!;c5H0uhvL7(MUyIFro4PFSw8G!psw`2*D{gM-{V|_6Rw19xBaxa
zzyX(MP@S$Ju_IZs>K)~;{#|>_;BLQY_hd(?Ly5E=C_t@5oVKpDWXULJ!-9W!&TYTH
zP$cg@509Zu*Q0y@xxU@EnF-@4_NJvq=B}^6Yec3h^;8?Nc&ez@{oBqdXL6-m*XN7b
zO{;t39L2G=?pYj1l7HnR3~K41FpiAqYW-M=5wI|jW46Q(Y6(1tDdA}G<5k~Eh8%_D~i$U
zS?s~PYkr1ArcS1-5&q!RB>*V?c<>5qJf{0~9E$9$W7N_vJ>vq5`~M^dm0tnl_SAq1
z;aRIR%bwwJwva-nDH94Kg@ga~BvikAC={2rcl9F+NU4>qss!{b75C!p(
zVQz{Q%RvRBynH)`0A5R0%Kpsma5IE3kng;+_5s1zyKBNfvMvbiDTq
zhG?a55YFiWBWWnn9P>de$!^E
zM79PRUGMp;8)uDZE(dO6N1q0wOei7-p}RmbqJz6ybI)C8GfPRy8-4B;Pc43uo?)`VA=k{%5%|7FvK;eyx6VeycNr@BjAke5}8+r8LM_Edf!)(11LP&mb
zMFNJ{2ws6)z?-6T4L)Zq86)CVR6>>(hC_W6Fr`49Clk651ad2Uga
z#Pr*CG|V`OqDp;vEbsOpFsV75+8h}WLF#nPpG#_G+ZDF*EE-aB-VuXRDPDzO6ljSB
zcg{hSyc%<=pGhMbCnC!g8(j3*7A9O>0i_=Yog>)kKE)TTJ?}&%IKQT_#XiP<-Nc
zQ4~YVsG`{IDKx>?QkcEmB&e7x)K~yTG$0XLgy#RRtTVufi-Z~DS^Hp1-{Livfl6v6J
zpR}ge2$la>@t>AnPtzf8FI)WJ3=BYh8wvVc2;(mdQ&|o~NCGP~V|rzsiKq+Tr*1+L
zxY{eY4QGf3hK%gQ-K6X{*=3y#q{Uc?NZhE8hU8=p+aHIDP?^8v##{hC`a%QBJPp`V
zzB2xqnW|BA?0h6o727JuTATMosszMKD@9j*)^voG*{GMoV|)+oPty>bgf;e4ok!tm
zFTJGE{VHcKgr4C(cQxbw%FBa4Nws%JJUk+gfM*vqR&6r1RYWH
zeG}X#9c}j(T3UD|PWcTpHT+^HwCEsaXX*k!YLqtx5eDeWqgI#0vkRwePZZ%47SP3R
ze@vLCGhg04vzrC(e|)%5(nH8*Q32WQ{-%V>eAoX9YN~T3+NBndTSWyqjm?LRBMUC<+Om|mE7ak_HukJk?!7&+cy%uv(}v*
zOD6UR-{g_PKi;Cgbx0+~*IqA}0@ZS=#Rgo41!dP*-`*(5dZHc*9B{(6uPP-w$_^wp
z^4cqumP*Xj{}ab)>loE&rtbO}`gYxMq>-;g`F#HCD7*trW_a75GpK<mWZtY(Pm9<^IdJN)mUM~mh(c}jAZfpA%=`KVrAWQi7<=aC
zcuebvD}lH_dP?8G7?CFs(0}0f>|+a&js%>+)+?JwkE~;aa-R2ZJZsv~^rCu1_y4@b
zRlbVT!xSCp+oxkvWG{lyQ+hjA{8e?B{FTH;&PG3*mwL)GPGjWlHR9>~QLr(0B=F*{
z`95vSlFc{3b}xzZVSllj#Y!nm=8DH=4K)NEQ!+l`?RrH6UE^5OGOPb97tF{~Cz#kaccFvd3PexNlRc1)hxl
z+PsKh4#vQK&z(i*!T)G2?$jqk+3{QSQBS1D7t+Lge-ScyG58WxQu?#xJ}u`VjR8c%
zizQ1I)hzhE$u(yA!?&ia3OkRxO>{mU+
z(jYoGAR}j9oZzF@x?tO2-F^u6PCmT(e>Bl3$)qG-R_A(gAGh3}xSzzkGn!
z`FsLV#)RX_-K5{Q%9N<~lRdR7j=lcj(VM4S60VBfG;xsmZ2aPu=+)QRXBB@xas85(
z&Uol)ku}r)cbW?_N-2aXKeS!ROAPp6Xj{SPbzfxJogPEBm>!1HKzQ#ichU4Jvl4o
zDyif9r5OaLR?$E)>G&Q2bQSS=T5y)xJl0uP#wpTwjsc_0b=o+5Rb~NgAXk*4=5&FN
z0(0>cpz%|{0>>2@@?R>-Q2~TnVa5;)tkpo;yrx0dia10d4X$BE=AmJr43s0h=;}Uj
z?Qv~xMx(lHLHwYtjlGouC$wH>V55eg6-LUX$o<0B(C
zU>o!@pb=sxFkimQ9~zVXm#tRurAed8re?*){=}1Lrj7Xani~#g-^{ODHL!jc{Rah1
zZlc5r>j_b4hSZqTn1E6T{#1FN${t^j-!Glj_zq>c>#oS(R`(#xD8?7(qzu5v2
zQQN}_CHFhqF_+VWlv}&k`^hjEx&IU~eZ`kohWrWNJqZf7|Aw{rOZ!AlG|;aF4WLYO
zmw&Jjwy(@m)M*>NEvs9y6c#cvR^U+w!xn_NU&0~a?Leuk&wYzJ!l%t96Mf@fHwpAa
z9BR+84Coy7>Rm!m;@7Whco-m#ZPY#`+G8DIg+oQI!+1rJ^NCSOL-l|p#N7^gr
zdv|1>g+JfLrNctjBy;ckv5`>1c)-530Q3WxN>yJ1-xBMWlv_!o~mJedDFF=#@+k7bxw}|
zvnuyRHS#d8L#Y0+;Jj$_{VBosj^=6E_vN*}>ScVrzfQ^L$NI}S5VYw@1ow2K^g1hr
z`C5f*WNi6cVDcMM758_^&?ofY*y5H^&ptr9Yqd5DPy?CW)!uzdZ&|;f2iV~Jx2MI9
zjV_L^yTknsG)y@uOD^WtC#$eUMPfoHb^?pb>~$7W8-Vap#0=+%agU(q7kwI>-oOJjJ!7{LXCrjX
z5c0>DA3%;G@Ot>Sy!tg-h~v%2Nd^{MD=+Y*H+~QVEWKKQDjrBc6oVv_p%nFG-haLm
zt?>$bG-#%{ub$gFh-|c;gyK}L*8*3ilaVo6
zg?f)l{l28=n4hoV^(k7i>lOBCKwkrQIzkr4M~D3X4`FZN6=m4wxwSc3uWVUwxy`5`aVJ3Y4pzj>)nd;xH^%hsefrDry=pIM9}!U
z9cc07Mhrl{jqKJ*%L#^M<@w*DYhmMOH?oVTz7fH5FKm8?+Mq=Nw>k4$DG$G8|NnNQ
z{7;vG&nPYb!Mn152DW=fM{Q383HPWFIGMEANLG`prt{_^G$`(7C6J`#iMl>8B_T8HTflEjbk8pB2cVZDX*ZyL?FeLZhhj2(TcOHkMQiQ>%Bkp{zm$2fM3PjU
zC&5r44E(Mv={}I3|I=%9Iv1EC@$U|OOPQ$-M+WSj{A%&~LT_issz0roH!ZIL+qm|M
zy#akk?OMyAdgD&@yLKP5d5e7pL~>v~4d~v?ISA=-X6SKNavQL}*ZEg0Kn-w?9h{<*
zmMMXwG;^p^?H`r2yDMh`Nl75sA=h9Upn^@`pY@*oN~HBJ|74c?KE*K!S2G13wEkgn
zv9MSL?jWI;_~VSX&JsswsV6bNU%~IBl`Ky@fMR(2pA}wSJU|+sa};_;n|)d~1uwQ-
z@r00K?$imaWQeZ)m=T)Kk5#tyLqJt|o>e&sLfr0
zGUPXWLo_3F`sBymW%zi27y|e44v^S!884Z7T@fyWuyt53JE)K14O>s7qWPdK78h50
zpEgxnrwntB!gHn~)!~%6yIene(yj(A*64OY2z@BCe~cOE&-E?2e`y+K5LsEmK0KUu
zRXg{dWaO!JI$?zGbl3GZwR8`qE~ye<^C$v9tkNhJg`I!>68Y%K86wRRQ@?APtI`y0
zxqeefRHYL5-_@m-Qxr0;3?^^^6$p4xVrA8(j=q4Bb+_tN)<9a--yuN_Y(FhBS
zK_@yBy$A~Xti1fdX;6!k@?-okrxC0?1MB4Y9az!~*@2-j=VaWA(m85$djDi`T;$JZ
z3!j4TR4sbHqG%wita`&MlZQBkzJ#!dS)ZKaZ()UyvvvgW&0neB{o^gUv6$2qh?=+N
zSPwhRrtX-NAoKLxnlxP+ep?*2V~>p=i8nX*`Ukn#AZVLlT`W#R|RzMabG
zj9ESjUWIk6LOwTi0YJ>a8}A?zpiu$@KCl7mR+XYJGo{F;WqM6|Qa|t##{>=g>)0SWoB|4q
zMMV&?sa{*PYUd;Vk>CG$i)x1d;_2KH>)cKSJLgO2+Z=Cx)trQmi-M-zvYs06zZ7;1
z%(~lNMSth{hR#|QvUr@es=NnuUA!1>qh2Tbs1Y82mNoFfB)-pd71c~T$lYc!uE{sd
z^KMpEO9~(py(kgwtyd1>v-9*N8xfC|xCobrxR$sKuGfVx?Oz(+7Y4U&wS^tQ(%hBr
z-#w5)%}de#1#ImY4dC+Mx|wyh?f3?(UzBl{CC)>nY?S&hb#?Nx3Cf2Q=uTR(bsRV8
z?Us9N1KgI+^!%0^dVLTEJjhT2hvzT*N%uM-W?%@s`fZT-3%RoL&%8gAqW~SiLgm{<
zud?zX`Qd575A0tKqjS~*I&rO2n~TZEvXyp6y8LVo+*LswX-t$`>-$NK^TNL7^>b*D
zACDU{!=3fH8eJGae*sU@?~aRu0owLJ(<^>4o8*}$^YNF#(M{W_p`BfcynF~@I1uWo#dO@b67FYbr9oR>9gU%o=
zk)2wvJ76*pg3P<+;R5pxZavND$DA*jw}E&W{T(`T$Ih
z3a5UAj_by1|KF0iD(LHW*}bh<6Z=@Iw0nM`e9qqXir&a{i$UARa-3iZkExRvOlYnt
zi?Jj9JY4anj>x$Dl#?0knR6MN`AE(T+lyJ~sAn#NEP_sza#
zlDG?l_C&x=@-jB9!Zl*R1b6F!pd^eVm?>^U_Mfz3n=6m}G$RcNi(9~_zT}))Dufzu
z%vEOm07EHK_)UDs71~uN8^}2Nzo`Ynq&5E{{(W^aV$A`cqk)0{kVce$J~eqD3Br`R
z?Sig{;RJ?*&xGK70M!=wXVyuJlD);bbYQ;4Kil@(1rXL$09kOs3i8PJ87HSNs&k*deE!29F(4
zd*|eA+>4e)QQwkY66aJsfiOvX1Sd%E`+wD_EodO^RKu#`j#R8n^9N#k76b2a)SPm=
z6RfP&i-dA-@2>;^HFcazSU(IMpwD@B0JByA>3?{I1+WhQ5HJvlLRgto`cWh~NkGP(
zF+vc-l9Z1Vi3IeR)}N7@&oo{08o5Qa*n`i5ZxP6udp40_7;+*%I3zbvwy#~l0`$?DP?66T-R@=IN<=9QZLP`((ob7t
zw^q9)D!+zc+`mS{%(|5*F1dv?_j8kTZ(%gM<|lm<|2?0F-`Dfl?)s3I%e+XRnE$fN)_Xy5|ITf59)sM4d`%OcT}4}y$f`!W!5{miAdOi@gf{u`v(V?
z{5&=m6L0Zv=VDGxY1
z3#b9--MYPrKMl0w#_F=8AB_eSOu}Wn0r2?27h>Sp3@+!>>qY0yUnY+)vr0tvN5mM*
z-C36zC$maTW3Z2$WIF}=fikeL{0;jKRho!-n_qO#dT?v*+ka}rqnW5;6b?y=r)0hx=7|xr60g$y=jYjFyn`3>;oPNqYR00i=W=NgR_+
zyWaql-enHzDo*{ouJZ#)Qjf#m<3@V`s!(38ejX8Kt8!TGZ|<;E*u83-$LhwsGd$eM
z4}H5%H%!40mYA+qN>hd3M0a_U3{tq6qqzp!i$J=Py92dLx`qY#uUU))97_PH*gKp7
zn0ua?m+GbCy!&^Ajt&kDL(yZB&RH`GT>`W}0C6hN!WQ>E7t9m)rMT!LI#Agd!T10o
z&Zm`ny7^=v%kwi95*j;upNF2ISZFDFwgps!%6V(v`64Xi?i*Jg+P6PzAH+Cvm0|t3
z`F30Pnde78_CJEhFS{Kx6aUKB$Qwbf-u!;ok9^CY3>pd4_{+Fzv*}M;YKesbX6^ci
zFtLD4@BBWfAHMA=h|2SlTP%tB=ngm0HoF0_rE@JBGU(TR;%aBXPBu?8VzXNBuE{IL
zd@phrvZgi?cvK&j(bzVzm6pF*3
zUJ`2aSt5^)MdEBT))6cNW0DV9ABGxi{UlDH3!|r#u7OFBrEzEor}bM`NDh{ti?4Qe
zC{KvA+&Vnse|;5SZ>jVSA7}e-P$eDJBBT$u+j!uceY|m9a=AYhjM|BEfge@YpU>G!
z&{F`Z#c2;sVDK)W)ToC5
z$jWWEr}rXJM4t7>grGpyOJOHxpuZsPiTPH?#Wr*p8>`6MC-Ut{eeXYv1jnDu#-}F~
zCEm4U7@|XJhh(8-*({>7Le>eAW@kN%xFhP4I3Xz$w5qzboKceb%n~sBl^GTJQts3k
z^8s9uVRNC}LD@-8_Z4Ee>^elW0%$eCg#AJ3h#
z?$(#nv7dHosSk{L+4(elwwH=@$D3F(J#N&rt_QU~-&fBG_)woRmG>;-06laIQ~
z&dIC8NQOEUr`+<;j@lB>NmchCp|hA${lA_$1Hjz}pzTh#m>&;N2mSwy3ur-NVj&uN
zvudZGfS@vPpuG2R6m1kyew5pZrp6;Ids02qsu#n2>H!r7#KU`nXOhIizT%uxeBvUs
zmIc;MsCDW4_0)bPT-%-g_mkUaDt8jgVVzS(%REkfX3}_o_Z$cYuDnP=CfxiR7{@d3
z6adiIw086Xl!dKsG`}6_MNY^42luQjbKWEKEOcNE80Zdb#EKhxLSDS3~D4ur0mGh>F|4W8L7B%0R3E-I9;w&w;LD-9zffh!e
z^Gt%9Rz0ywrf5=!*}grhZg1V^C>>5G?kD5Uy}VbSw5x!=T4A-_GFEn?yI{@6S=g7I
z81P2e-Cf4mW}R_XK}&GaFP+gCY`{cIlL-nl37XSuTmM8m^`aOZs1s6sCk5}T7(&k@
zXx8$$V&13bg21a5b73c*r=biyUkjICh@>DmGGN$)oj$leOe|ns56b4wrHOG<9DzVBBnHowmgQkNH1j^FYVLu-xA10LLaHS8uQH^UzVoMN*%nTmFHI2Ox2J?
ziPF@038YLAI5=c&DSdEw!co}_q~;5$vc=^1yEDn3$q
zw4>A-SCISQ9XdsqIm!93a0)i47H=7wzj?QQF@z`gUu{l?`mfJXEIA2Cw(ftIo}+r@
zhtz&kEUUY^+_ep`@cPFeMl`-Zo87u@I&Yr3rS6!Y2myV|?LU)80ia-_OjLYW8h`u+
zXd+QfsjO8u;z_y~=yt&40XIHMA7qpR|9-78T#jU%CM{DjYx!1WPY$}=cdw9Y?b=v&
zF36LJm_ZE)3imP-XusI4c0g{q4t;WDP$JzGAVaUx2Rx4`JT@)_63|+jiky_XpKVSc
z$;0Sx2kJ{53-Ybp;z0nXwsf++Qx1J4^Gj_R*~)jW>Ctj)**awUSMFtTV_#8K-j^kM
z^IvX!ds6|vZ>EKpy63s0pvJd5w)2U&QpE`ofW*l*E#(gHQhcBVA$PJ@Eh?(c!f3O5
zP%^7m_mtj-YFssLsxI}a+H99PtFKowYr`S*ELcM#Uz;Z$HFmNK5ngfSPTQ29losaY
zf3T}C%$ukV*E*DZ^CZ`GG&q`f7=6xVTijM}ld@|}WzKTbWwX(~sd?u{7rjOlGCt_}
zLsQ3HTPLjXxO$}mklg|Hm`NI(uCAvss~TwHFUqTtZ&g<)q$3k#g8fZ5b-K=4PzgL^E|%?L@!CLFN*oLn4NGvd`g*sD0bL
zkl=Bg3Sgygva*_k$1GYffM+8-P~*n2TRPYbYNQ7Q2MReZ&bp8@QKh#Jx86V+_c!w5
z1s!lkN|^i5|BN7K&9l^)v51WBCUU=*TVIm}+YnyAya+)3nC|7UQhj}Mk0Q$^l+#fe
zKoN1Rax-~em6eena3SWyc`Jz8MzBI`*?p4Z)=Ho+hL7HX=;xKL^BM(K)-4H+_6Jo4
zu+CwgWHBez1%@cSPIy91ZS_ZGI+E${tVpZ5$9;kKsq2rk^nD{a#f1N{qb)2j>KdzB
zLRFWRLF%R-Km98mP}N$ub%7+`u~S~h=}#65BZ>wH?TjNP%eJ~JD@&>5d7qbe+zWAl
z$3b?3XB{@5)MJ+482ZF5{5P8hE?j?!zS|RimY<_}CE0RpRE)dEroqPlZ&UaVP;zTY
zU5yu&mrDo2SK&>oe=c^pPMe;tgW@-9AZK%%8t_4>kfs&Cdlu0gdaeB7-L4DRp$h9e
zP0pp*NwPKZVR|ya?0p%K*jj$N(&oJG)zNoh1CH%EqVmvKeG5GO8oo?Q^tZ>f5b$Cm
zSSXKF=$*vv=Vtik@jO+g6uHxGaHEOkB^;)1{m@2(B9zv>07=q>FNu9bB_NfCtm(eB
zakei1+vlj)v{w8rd$u@XVP!Ch8B;B5n=B1NrOg~Fmdlh7%5X(SKvg)e%F;GB!_a)l
zvc4!m;&P|O`kZBwI;{aaeg1p(m2snyuR^HlIjkpqZh6xFD38J%L!Ed{o7-cS2*jBB
z%=ZUIILf2(3JzOke`p^pK54cRSLSAUTRBl~)GaKEybl5z$oESa}jII*qO}`}(!!XEbNn3WDNq!zcdq
zKe?XPRZSr}oR>nq3KG;fd$5(08}j%K)GG`Iggb#J05@{0_xpcRpTpyp`5Qc`ZagaKBSss9aGzp
zH)jm*SC6p>1`_JS&8rJ)_ZB|0cJkpX-J+gsq4+b8%3{Ms^%mLogfk)Uipzu;Nqi;N
z68VklK}y}Xgf!>ZbXCW+8zX<(5(gEUj}5}iAHNy5Q%}ml{RoQ<^RLD}x8p#VHs1TQ
zBvrZd>TX=_-=Eo$gp~*vvkf^oSpaaRm`$o1_W^#sMRKC
z&UrZeb{~5G_yj2>XG?@=ACFYUYr@00S)$cyYf~1+QK$@
zLOk~!wEj!pi>+Nh3lBa2*=OQ#G*EJOO*gG{0cB@vx}ICURUCepF@y0o8NJ13co4}2
zRQ1y$(m@(ME~{dcm-FCB!0*eLvE7(qnIog5DU!G#D5%dFF`&99`(XE
zx-@qfE*scOuKk6=wOSLp;0z>yX9!M1W&7~TV1W|
zbtEWU^7rkC?$vtfJG2?k471*S-_7T5@o9#20Q~L;Oj{XzdTfmqYd+#(&Y{e`t{HWj
zN1Q5V&aKH~?!qg0kG-KkrJNu47*_SWN0J%Zpx@@RES+}5^MZemp_3{aD@|;lw?13>
z_-L4)mehjd7dmYp6J}>4l4(?u91?R_$p>27iWxS5o~rHXr$@93{7u%kB<15(FcGzy
z`}?ltVNAi;S6x*GHNMm(y4P#ORS~tBcXgLNzFh}uHEZ9v43KD>!hT_>%--6VeP|y4
z{#ml>zX5gJyuh272`{O|%WpzMatwZ)T$@;onutv=piX{x7!!l8J-2m$uxEbYJ*(vH
znj&B%|Br4Oa4jj!Hr(FNf&o!D-g!<+#uF!A76g6VqEydIG;am>$_SL@0Dgj}
zMDsE4w=NO9rR)~p{MRNl{<%QFp{eOlL^&#ao|$AhVZu2CdG3@&Bl^|F5+wLrZbCJ-
zRqY_%zLA=L4NczPE)pW(l_dBQvnhgC*rg#4M%tnW>$yu)>IxN)NuIdQiZdl6lZ19t
zul?P^lIHTgd&3{J4}?*&cdhbdU62L#p^<-Hzh}lbDjrhWcxRcTBKD6nb+~9_7M9^U
zl{c*-Dj>eh_QZHc?eiueC-d$kqT@qAXThmuNO}1LKL_wiT;{KyG#9A9-Wt?hBKM}W
zAIB4YCpVwG{~f`bC5nL(xRn8$8z5qY`&1XB=*Z~w?2Y%60&25%A2=3ZS~k)?Zrdb8
zg9LrKnkOT(u&3I(8Qx;wkZTQX3QFp%wsWA?Mz69qEnP}J
zz0oYxPN+oJZ~f5u;oj#Kr(nqp8&wCuYJvws=A$@$~=
z#QD+lQWiS>7yHIxA2iEfuo6wbC#MUnI2Z(Foo0;6UvIeMl4%?jrLlaQjqUidW3l^V
z(||<`_HM;&e1O`1T-%1tM^&Ad#AZ7KpZJPYIp~9TpahJkFFkscX+x&%Nn)i9752(P
zP%2|q0;^B(%59X}WG;_&&RP0}^;EPsygM&;{jR4@Hq($GYFyG;-W1u+EquG#B2|uq
z>0QZAxTWRICTh%b*znywNpIOeydh4$#CK5%iC_kb-iMhXvLmg*Qz}F^M2MhYPOr?GTT{TgrRyHqH{{7|w
zm?1B*0)B3XmsXD05
zf}D9>;!aLn%XbHj8Ea(wRgJEwdxU8`+XFz2K+R>+=>cmXbRb+1fnf>)_F8mAcIR}P
z)9e%*&OxM)Onjm>{bY;K#Kc*bYUS%5z5q}UPQPs>m&PVcAR}{*@j&d>nd~TamUZ^3
z^3-?RhyK4L*LJ*T>lV{}3F-X8^M|=f!tWKx&z@mXC+75reZ+n8z&OHbs??q1Lj?HL
zxybG_An61@-8?Y^rrwA9swW)aTmvBnlE8Hn2w-v)7!+cp5AhWCr_8Ybq5Zzo`x9?}
zIQp2lX#JC893o1jWayVK9|U9iYXAm!QFA-&T8n+xA9Q&X*{l!l-8jydp5}DeN`<2r
zH83pa(bI@Q1b66!JW5n@#UN2M#@OCVnAl+q+xZLy@;ij;|Ll-LG9
z7>=D-3>X6fc&ayY@dAKI(dTSKYiVf-=tUzl-)RZgAipW;L)oW666akbX^U)Uy```3
z)*HXU{44xt-kU8RNVqSO?L3bnn1MW>ZPw1eGB$vtb;~TD{6Z(afcS8ulD{_r=YL!%9VVJmXPr{QzPRK~-C`QrdhPtQ?}%J0s%hLjc(iGSSf>53zPg>l
z&(!!6EL28f(p1;D*?6(NZRz=BbXPWmK_+tiE;wW|?HVNm()gR9E=49{&L&{Be(Uq5
zC4Sv6HPfgoR-`1}8aA>vu1+z^n&tmk9$|U^91fWFe>T<9$=v;w@k_Q9=iE#Y>fN`{
z_+)ZUwUrjk%H%fyBA~FXY4Z-<1)W1tEJWQjmH31Md|X1GW(ZPn+6JLh*om
z^u>nfhBCwbID&arA
zO8`)aps6Ep&Z1<2SD+#9)2VT}%ev(NeN>!x{&P2-E9(yrX*SLwdQMie8>J
z2h-^X#JQ+ynqN1hCTn;H60fe*BPCX+ZT`%;{(07r5rK#qFU
zzQ^Js?vcPOcq@Px?rJo3Gpco6@-9}hrstyPDfW8ky|J)TtL_Ovg~w5`Q#?NjQgn9
zs(ENqPbGF~;N7d~{R@Hvc_#-Rrlj8uV)}}BGC!NaB~+|p_&!CRJytfD+ea1{5T+z8
z0;=}h!swiw<}SrGWE-i#EYq6_abs^ixeZ)RP{5e>Wz?tVTNXJai^l>sPPA`Ei1eqj}2SK
zAnTq$rxM7TjD_Qi62;q-N`j|Ris;=d3rv56XuZdI0ZX@cR%eHX$97N0=Ay5kS5$l~
zs*S787Qv5;sZ*UdEB?=~JW$oSom3OMOFgJBr5pH>Z3pK7P_T*f&>LW21Wu#BJkzA|2#J-5Q3NCs7={8u5Eyn
zMr>q9AOWpF>cz#$OWoKT9{bgh4@B+t3qUVch}uW2nVPY;INrtKMTUDq@GFLDk%XL@}ZAl(4;hZp3~Bvq&~*TC7No{XMr
z{NGiP5JW|7tsnD5og7Reai|YD!#3(P
zMDk(sEED1p$V~NaKCzUyI+kLZOOWn|{4-1NT|vZO;o%kK_j|{zzTJfu-|%7~p#8ex{x&SK{>a}M#Rv2vZ*3^v_NRTSU2r1w4LWn1@6JEclxaX^H3kF?(n
zc5dmv+B;crPSR~o#nDZG+|E_Dq07P#rbF(J#`_cDnA*ab!if%n&UXw%*YBZFoiQ>L
zmGOSYW15`83((C`2FX=Xq};yQ+U95!&MDjM1ttQ0fgQPax$L6nx6%J#*8#6Fz(^lq
zRhi;%PYAp|wW<+_-~mW4!qoray*LWlE)cgZQ#UnnP111e`>J-EI$7AqapMrb;r0s9
zR6h~U-zM`fxy8+l6VVU+LC;$Jx>S3{rtO%9*t(Zx0npr|a=m_x`(7Mi_too-R2M;Z
zq;FwGYbB6;t=5z50zmL3$Kj_BWZgfFPkmYbT82S?jCc8M#2euaHr@(^MInUQag!
zz@3Eq)IkzRmYzHetC?6d>%zVoWWJ~8K<}lfp3{By0q~I8#3|zM)CN%6(z}ZU
z&ZJ5q~;$#GG9RAA0AfxP5Sfi_`rZv8~C6EKF&U&
zReWT72n#yJL8R2wc?E|Ct&^Ot=vHz&J5MywY#!`ml3ogLpt}W;qkeiq7k%JY-qDPa=_H!=MUKC*T+fpWw$DnIe0x8y=+pT%b
zrNG?Jo+k}sG-dH?=x3z&6!@R
zr!8qsh~Z-2r%m^~Z3`cAi5wpObJ&|ppL(v}(gHHE9$c4DKsVDd#V!#cTVx)K=6hgv
z$BW~|yt$KrBH6KsrhE#LiE$LiYtOS77jIH7`{We+y~^~!1H?4{ZHzS+4)S}hpxfzh
zlXPDi{R4#W0F=p52x8sQ4LEAotHWkW_7DQ!>$AkwF(o`z*<+2BOsB40L16$dvu1hQXpjS4g@1AP+2hq%JgEHf
z)i2%)jSsiH99T=M(PIZP>;Oh};_oQcFg`jp&X1c6T@0H2Gq_^b5|tU}IkauQL#uh1
zQkZGiGp;F=P90n)^~e2u;FScfNfvTl0B-_FpX%O6##zG$JMxweKNjt7IWUfxigT80q1M$H%_)rP$*XM>l7(OQl|68i9MSt8jK5
z*tT!HGuh3};~B6Gv?Bc_iEGcO5QESBhfWGqwj1N_=u*Gu@BJ+aZlIDXhA_U`+N~d6
zREOe}ye1{aTT~2DpdDaSDB*P8lSg`;Jh=>3VWmt^+=EAzmlqg??Vv5a!alaJ!dh`1
z9k{|j#}>xO+dJ|Um*89WWyvldr?_?d7;w|J`gn5q5x7-Bu4no1(kE8Lp!`n&jAP1|Gu?BuBRFBDU
    =^@
z1ba}p!2`ANNxsEwRgFaZpV4b?*Sb4IL6}C3*phi*Lto(RTN=QhsGnR6NZkc4p)xC4@FmRTio~4Rva}~;iBx7D<$-NYi)xUcq)jRt6k4bVdbYo??;Yjx%4lZR
z*_{OM%e&?)7Va(dA3AQ6Du5EM-cfFM{24*yj~UKI47)?zG?(B5c3tB&@`aqIh?Vt!
z1~TXR24vTE5)O?|Tdx8nOR!FL@~rc%m}DE05#~L~m9MEPaRB|g`~eZZNlFm=F{5DX
zyId~E(A51{!RGd4MR!#~&s8V+A}MNA=kTXPi%YgHmMFr55}D~yO`%tBtjI9?Yj7SA
z*)wnp6u-D544$twG_CeosL~4C1LBtYN#@xI(8;Fhw6ELRppoY*-k0{s`7OooR@AjT
zMM`a49bKeqFwUxT&0?Nn#K%TCcuXD-Z?w4O9-UnOcTU@)>jgll4j)G
zHCkr~`GVBQ6q-sOFgCUNqgH*?6KS43%%mwNYxmawV9igt)V-GX$JkdB`1*ZUxKMsu
z1=7i1#aho{X=*c5sMwACN{|YUhH~>D^{ezRG~X}@S>wp+F-@_ovDe5%vM->itd)^e
zjNY?qx)pE9EFScPuN0RriB^N;w^8S{eg8n$%1h5dby|Ggo<}$^AebLwcBL83!7}hZ
z`pm+p!|Nq_ue-bRqttF&}
z$c8se{W~|lX&)+BV!X~bC0me;qT>tKB_nvl;^%GL;%8af6w=}+0-^bjW5)>`_JVcU?Gu$aGuLeBx{Q_QsR>qBbcL85344`lP?~46lfc*LVVq^&G{LclM|7Uro
z{WIutQt`Ge=mxqW6u2pL`Unl8S%qIWohiZxXu4>gHiHb|izRD(7yS!&ifRIc;h@w949Yp;Lj4ydaUUW?KgK2#mt^C6vbulpJD`yS8
z+zK+0B|N`-o^OjOvL9Mn=T2m1I{vw>zcL@D)#$}pzFTrm&qufg%30wzSv^Aa$5tQ16tog+asw5$5LLZcn+QCjRX}I)LCh;
z*sBynV~01@=d~_WS&K9=dN|b?WfB)2g1b$z^u
z(sC5MKO#QY{tA$mo$b$U@z7!7Q+Q
zf-@toR10rs+JEEkoTyb!U|FMjx;XPreLF2kz_JAf3!W_#+Eh41I=bn(V#+iwNBp8tUkhV
z41)3~u}7+N9)vPX7E;jtT`JviYXb!v>+{H(yn*Ed5XuZ{Y=cO^O&DKI+GEBtW6^)2|7=m
zEu=J83Gr#{>x*Gt7T;>DO{zC`c&p{2Cwo@YlXh@pMEZ8_|p|
zz>^nC1)7b1o5m?DdI(=D+Ei6fBE9wEB6Wg`bWkU{r-g*j6JcYY=H^JX{bcv*bEZvH
zUD*$!8SpmXlH0HU>rXp#sT%rlj&yL^LNEFBpQ_+O>gghNI%rpHwd=V6SYHGV|0XNr
z|MQvnU*8U!=cd5y)4d7AAhU-#9e!)f>f^ULbv31mw*5)M-5!<4aw;oOpQe
zNQGs6Bdh2`g)-vLzvI&X6rPpX{bQv(sX#L+_-e%$#!9?MB^hu=*Law*m)99-eI}^%
z4{)Mgo0+#?WM}N=3aEK3#bFyogu7Te`9Or6#SS;V$_)dsDfb1;O6$DaqjwnUNgtK!
zbsU&q1?TY5)5X&sYmrxxib$mT&G}D4hTE3z2uDcuVfxIvZrEQsb(&8aU(87HOb^TT
zU{RqJf83%y`_@h{KG707Zt$6_&DdGa5hz^*Ls6H
zD*i3#B`>pf1B;Ka6v>v=c(5x$!w2MlfS2T@+N@bW`UK<=dx5dt{&IH;3R^QC+wev>
zbM|Jr{mXxaE4le2Q1pjMK82(@0m|5>wwTe>7yesbqsT}(y-3wP)RqpD!&_fw)m%gM
zc*9)#CJNq}N;uIujh>@D{Nw7}ADnC*57wP?HziWWt|mJ?ityU@
ze|r|sKfEg>4if3FmT0&n)9Hk(3emr~d#nfzIFFh@puC$o$xU{PywAKLM)sf
zNuFt8$+o$DwgJodcrbTu~XErX$PHp=@WN;(P4*!@yVP|FCc?4BKf>b`cA#{CJa*l!xeF
zdl_^}^MH1n2rqitOXkP_U!REo^KAhvqD%uA{@ZO6xD^QMzbck_BK3gt^!8z>>LG^m
z1lZq`Or?EMX#+gpZSA{p@~O^1KhBDFC9Geq4{Ma`$_BzygU)vmL^r|!*Xz?E_yO7E
zamZlq+WzQa`s71o4_?3RU(W~t2*u&h=r8UM
zM_K}%GqA5dVt5QU6%aWDsF_q4xJ4O9$s{47&#vwVAW5UPTxe&&?-+S}4|LRsAJ_NLCxnkrIeZv2H&8K#
z*6fY<7jYs$p1beO!|w#EI~ps=`Hdp`Kp}VM7&{jz2d%z_o+zQS2yp5lh>lU&|3g&*EU`VV
zdN<=(=JSn^UI^heTdF%%U46$+@kX=m9>&tr^~<=pgdC;V@SMYJfq5+~M+;P?rBXwn0eT2gpd}!`Vj;Wb*xg{Ucu@=FQ_gc
z=oI)dP&e<>C#Kd0grEb!4Pd%icJOBY7uX(mYs++uC%k{`4Z$^#3Cp=;ruGYPSj*JR
z-1z3^w(4)Vb7&j{s{H?7e>v6fA)Km%I6uxfuV)0^Wt=s^XI4`N#sPORU$p+)+XVxn
zomcCT!pDQ`8_mtY~puQigPlXUN`u#M@8BX+}r>P
zb>6-LtjEu^KNJu=0s6T+hrKh#V>y)%2IOuL8Ee24_j&*mv83DR3F2e
z2*SZmSb85P7J$m7pixp%f5QAQgQdmVan`#wfCKe{ks>
zZ>zTjH|z`Cr>74qS9{DpJC7U->FI?2R-eX?!#>u0E_49|)N`IqQ|Q>{a}HiRy=pF$
zn#`hVAmb%jMU^Cj(Ml&*7X9QHgM#
zTnC~tn!vTiuRpW1x4#zAG;3TolchHVdq}nlUo4u
zdx;~eCO!;YnXd58?M1eO1A3L7WTX;u_^^IjX`p8?fyWEj;jkCO|lWUIQI
zkK#Zc#j4~MoMcDZyiF;3?dT^(oIocO-z`FR?DZ#4vfg`%cFs)kCwAC<==gN+g1N=M~{K)8mJUj+{8
zVMkXX-#39osNBVj7zg*Sn#m?%YK#S2+m4=^AViD$_>t50zVaSNDW@$5j=CUPVTbfg
z-2oJ+21}K*J}vin*S|Ch@;Vc(&j_>LkRGguiYb^o?kJ`=r`@euQhukDsanI@Kn*GhFF9SaOok)Xi7m`jSPv+FR*fheY24-BFhyZBgKiWLBH!57#$1E
z%>NdQ7zn#=S{RMN$V(a%@3AQzJKUN2_|^3H&XRCZFT+%uxd*oNHa!&bh#m67ROq-_}N;MCMAa3)F0
zL5sSxO|rl{`3%>}2>pLB_7*^GhF_OB-k`+^*5bvzI1~be;OAyE_C_7!mk`=ia
z&8vj5dv6g1sfZpi+S)!<>EEA6M$aBcZF)m2Je-AC5NEUBuzI)TNpSP4amiZb;mh;ZyD*yQXz0?et3YX&TGN=tSy
z7YDx7)zu?yb%wncC?f0~`jm=}b!oMVG`(G|#ajalQ?F!PKZrC1;i$&lEN_3Sq$6w0
z*+xVmd?zHyqF=WN=!p346kU*AJa0%ar_o1o)ia|5`|>+&zyWi0q%+=P$vk#S++PVSnER
z1ixk&)&7WCGA*2az54NzrSG@@F6r@5N<@c|Ax=Hz(!|Kf2`38J!mUtUNM}PG@Ws3r
zLllFBiUu$Yh~N}8tuS$cCDv;IT04U|TkFp1
zu4>llj^Mm{Q){awIUQ{uuW3KxvdS^B*S#JWyQ8VSK?RqT!u)j6*yAXQUePEvSvBrj
zv9(&{Qz4^Ray*uL1|$MUa-RvL9;7`c27)Zec14&j1AGw+2k+nmATAO~EQ{%*qws*wPYJHiT`3J`ncGtctR#;7tCWPLBOh&F<2|_mBUkOId@ya77M$l
zZI$|IR66w4URf5crxaT8M>`45TMp~Mt)wG-EfvV_k;?bKx2XOoX_!`
zjn3Z%BD5&g1debQG8~xG6&{{8#lr3Q*(HDT}jj_6EMbOqXh
zsY;~!94u~~?MkYA_s}9@j40*-ma+{`^cYNR5Yqjlq?R7OoJVHf630zzcUv}D{2qxRB8GB_Wx_A^buc8)Kw(lOD*
zU`Z$%01|PLS@|^BCtt%9d`Tilu@WB$_r8TC_ndLs1oKWU>SW5)cFgRt#&nwz1ty^v
zOeRCVVcPhFn)Db|fiyko92$p@PTr{831Dw%6(OW+oxO6XdBUV~j%$w+8ePnztqrd=
zPHD8d$d3AvmX$STi^f13$!ih9s6g0dJD9h-SX4ThW*_ymtGC8je#=(Nk4~;^|FK*d
zB|E5{!xDa5H^zM%MN8>psE?PsUKT4WeWYd*gL%qkDZYDHULb&xvSKho^px!-wjvRn
zIVzfAK0Y>peO5BnT6E@LgKi7!vZ#z}P?PDur_cUN0vJl7-W(@rGOXW85rs&1vXIj;FBMz
zWv;fL-rMS@6U0ueX)vF^fPP1oqh&7^5lCn}XVDtdhDCr>j>R$J*$L)ml3M@hOVcm4
zd@+_}>b0CW-K=S4G+37#S50hKt&}7DrKf-3t}&M9opQL=yLJmBo#%BtUFdPnS*fdQ
zn}%~@AA6roU)E6nypNuG3;#uXow(q?j`OFb$zYDj8H2t~xqAHGrErNLkGQb-39#iDN=o&W0mc&^r
zJ_}to8gRD+oID4_I$O*COlJkK;T70d>s9SDTsOa;i;7gfvN`atFmH?3m{cXKGp)?O
z$`YQcjVI?U8GM^O=Z7*uk$YcbAsyS1s9f@*(d3EQ5Msg#6Tmn7o}?f)!9bUvl`-vz
zsa9+U{q*WFd6PWMNHgFmo6wIE)st0<>O;!PWUHV|2rZ$KT&lGIR>!1V9eTXC6GlBD
zhrP_90#BS7pL
zooOHxT-}U2k|TX69nz88f_t(m)2Z@Hj}`m);ANSeeAL-nzt6YppZtujguGAm*p&ji
zroEPpKN!@uePjBslikr`!*4(TxnuXoztuSUeM?ka>_4mN_v7XCZTjDi{eok*zmJb%
zoLwhPbzAxg)S^qefv;Y@%6$z&Ve<;TL
zEjDJJDXH(7BW!;PEY!=cC6gRPBMejkgv)c2A|-wAMw>!~aD-t1`&|hU)A6R7>3q2{
z8t`LkbY*FQsQaPL(i^|Md*^@ZP5BM;Zw3k3n8k&A+(zdwU%O3x6?(DX86?-cP%4Wz
zrKLwwP96thy|4hnvVip#$1(&%KZW@@9|^C^_-qC2V~8vDeZVUYko*iGc?qU?npqIe
z=xosUkzfs8z^EgIG*y~LYSE-H3k@uUmux
zq#r}+1BaUciq2s2oP@Llzcuoh_q;02DS9hOjMhO#QyVErA)bg+%CR_b@q+f6@t5|Z
z4X<-UB~_jBLP3WAmD|+!!IPF?`B$`_TPcsD=X9e!HZ~5W$0mn93mM&zHWogq5AXdP
z4@6wxtWTRc2W}&%_Dw-fB_UB-@@ftLDS}q@w>FU
zZcUOMDk)&G7AN%mdqTrD=iwXbOtbZb{H&aBF^!Ep#8|RONc&Wa3M4x_M9djuA*A{^
z&;knZF8_S5x7~&p=a-)h{W&M(p+C)S^J&rYT0W#E1|;yhB0d+Vs$_oLmcP>E^#%@S
z>OSzpojtn~+bxr_nK6HB87C9a74oI4w5yjTWja}}M*z3~5o(9lAi*A>st}!nzuYIt6kfbx4=>g+!Q6EYLvd;-%
z-~ie$Y$yV4`ph4AJAP0@Srema9J%Q2!p}ED8?$=jLiK5D?Q=UYu6B-y8+)Q%oJR7-
z>t#b6CA=SxE`(4)o2qbC$Z*ny0TwsJuz(vXn*vX9=o=kgs;|fGqd{-xdWe&MbH%h{
ziZwq>m=t$ami^f{SRR_oNH8&I8}LI9k1EzN=Z${}j8myxadZ&Lv&qo5V$3
z5>M#RiDl`GZ*k28_3MRf8;^HXOF`QFA|_}r6TSKnXU5Ih=DOECy_jW%Nz9~ZKUWMc&&
zkg^~c(m}6a$fh^6-sAFzEa6AWOr9GFq!9Q`+W1b
zDI7YPYIzO;a~z*WrEI!VbuGakEXb{rMY5O@#QKR*Bhyn-RNx56H8hJ&J>i3#XS57ulPzkYE38O}vF
zRcpY!#Qi)?zEB}_Ev`v31`Smj9}R&?E7A}UV*I-Sz?d~%jZ;^>zP{MRbm-`amEu~P
zpTbK<9gU=njC;FkHu`{L^IkFMniO>(ISdrlpeEU#cIsO
zMpgg7h=Latf-FekTN)hw_#q|vjrOp1$r+Cv&V;=wX>Yg1opht}tI>@hfUe9)En48Y
zAkS-cSUWfOQ|7hqTrWw$E#v+|R{+0$^2?5*umB5-?_n)lcac3>K
z);%CvOg=c{`IT#kv=crNW_n`D;w>@e-m!F_1)VkhPpd(dViQR34{x>|OcP
zxuAZ~Hp%Ujf&`kH)iVY+&q`KmEzB11iB?^^X2v>EcZoC~T=B^O^4n-!G}nJ`X^ghF
zE!y{bt>1Q-Y)L+RAd?6%TF^5vI1-*-ZMOMf7X4o&;)Uyv{)mFhKl>^F_2;ndK(CMg
zKQj0~SIfY$>EC;=+Fr8RW~qW(s+TVlV_)PU6Imrtry}d*67FY`%vR`R`ktki(=j?>
za%|0Udkhzcr<`i)nN&jU{E+uEr@}HnslVEa
zY0$zz0?>zKB!F$K2tJu
z;Cs2w$ECADFBkLl`H%ADEW#{aqBtE~@>3-;-&@8V>0e}fMsL|Msc*Yt?`IALfSBfn
z?geH&6_QQ>OZW3jvcU}*rB5EHsYS4Rjw#}F?_%lmY`=#}!Q>Gz^{+O`ly)*!Yn`>!-1F3SsXIoSfWidFsk9!hR&bvTAqhv;-`3#I
zT6@lStD_FkB`hR0W>y}%B0X@U7O1ujwzy}-UxFb+jQq2JocyaJ8-}_5XRE80WXNZk
zr!>z-FWh}*Vl2egM(~10VRirrEgiuu5ITLuvTj3XvBdQ~YmPdRIx!1D3%+IT@tKBb
zFwPal_~3=YkyjZJ&~@s%T+*nO9#HeWLyx^gi<1^7Ry^7vHiC``{$|#3ftJ%~CYK)f
zRf*v6=wNT&V>z37wom~E!FtGt544E+Y^;lJaq-H<8np>pus@Rtt?6~Ul1s&ABiivI
zmhePUA5?;pp@PMx+nXw30T-@p!Ao{@J+}LN)KmIT`J=`TZZM``cMku8GmREu+BaGrKGj1mmDYMg!+Ane@Q$oiWBEy
z!}8oCPT!Pt{*JL@vUDLiy{MRhuYAg|u|WckMCP=%7QaGGd6{n&HgqO_X=3I%6h~}1
z77KF_+Ml?o#aH=*10L$CqhiF#Fzas}m1)=6);p|Ahn+f*yj|%Iub9MZt}Ch<^W8Ig
zMVY?(cKKC!qxx&I(${fPdGg009_m$IultQ@)_y4z)+XrvEwIZ)qiZ_&Tt8vb4obB`^;`6ETqLj^k@zm6?pECR;z&)fp2eCqSmM
zWJVY`*}3TK1=JS$_y34~=m?8uQz~ZsjGI?8OaaB2$QZD=4P2@@rgItA9X$_{A7B4_
zVlNPA|KWX4X{#y6-Di~l1mgem0ukLAGmYkSpRY+({&9)YB0V$TMXicx1!W^W^{~D4
z$iaFOyJ7U?wUgKNWc_;@gNMGNqib
zFSX8{-NN9zqFJL5LKOhF&di{Q@+ef+)dYmkPY{U`(Gd6Z0%wZzFGbr~4hH_4XK$h}
z+0QCLpZr8IrEUE$4>8JBF5PA&fxy%lnK$wxOkd;lvQU+HM=i}=lTCJK8W(fYqg|EMUy{nrM<1$Qe$`vn8%butTWL%O%RF^D3iPrA};<
zairg5ZzkR&Jbr|$!X4O6cEj6i3-ikQ;+DsH3Bqvq6@CE2>8SrjP;Dp*j}
zQ6bKmQNEtKv-Y#GNcn5`Aq`CPq7e6vn7uv5g$&ZhP7Uei*#n25Nz@Dv6+JeF;RbQ|
z+_ZXW{HeFoUK}kB3b7(1ajX?W1vnB(C@0vfv!-(kNQ8pFBpB3U
zNO~J}-J(bo7SF+@0{9GdJQzNuZ!lZ{Qs8w}=HGY?J|e&Z9fm9XpCDPAP$D7_?Os*|
zyQS~O?6)597dO;sRDs87y)N`Roty1x@lhgw=ko2zQYYtIHAGo@%Y2>glsxaeZsTPQ
zP5k{!k+k*ellcJ=IzHpEXceobiPxK%1d6#HoyM{Hon?4S{&}>|1({dUE$Ac~y!$>$
zMv4neT-1K69}VTCdeIwyLe(Tb3GphUm5tKot+fEXmEsv`LIR9=80*J7-dk
zX#X6&X@$aoZ)=nJ#Tt@8y+~jrCqWvf|E`7_&s9B>JPltj-Nw;MV6M$gKIeH`Ezo)X
z1?W1sKYWSUXt~mn2P&C$>{u6}<(*@VX3vP_^QYd(@{7O}djPxUUS
zd%v5HkFWixsyfmv9RX0}g3XYs7$bF+UnC&~d9~Tmzgx~Fw3(8Nf)P$S=K|CKYpFgl
z$O8${NNl*#WG2xw#DvL6Q=GE+8pY)DM!Y0uH4Cw}p+$W@g@n&c_CKa)9Sb7Tlp!5e
zn4K{*(|QC_?Ie41z?eO3y}B=9!4j=uLu8q5tM%9oiq!JV@+VGjyajrZ<$@G~R3Gxm
z>~7TcSelsX$$6R>_sB(Y4ha+hPW1i|4CKnuC<3eq{5^Nfop>fr_9fTnvSvv)l=RAc
zz*wr8IkJY#QOWW5oFZnsp~%OwmG}f*#+_Y`>_66kiko&?U&Wk$*A6{wQ^hc0_exsq
zFgS&qlAd%!tBj^`IisG9mwyWScvOD$Jkvea_o$2nGRUxE_alFuGt)<95k4h-yFqhB
zwD@$W3gJ&iD!Y}nRIp^kX;a4boX9M=nO1-yT!A;|oq4t69SHIjB3EOI=&&
zI8~d(?&)>B-BBnkhm4E8oPTi|A8o8MsfJ;Tsg1<_q9pK)!2u=ERL}sl(GJqr2s1Lt
z)J=_Ln^%!a(Z=b)n=nY%*`Y(>3BSG-{1aOw&39%}MZ|-8sE?H}kC5y`1d0MlkSSF3r#r-KdkMSA6mNKpL7mjcq@
zNTZ2K;zxd$3gHH5m+o^fgzY~MXQUBMGKb;YTa{;wo|~8sAJ(g0HH|@XrIh2|VJxv?
zhY@x#UuzYq$&~R%A~?t55iq8VSLm0-MVnku%`qS2X@pI_5>UG(f8PW6@dj_f>(!M2
zo2RZfcr0yTAWQmC)Mfg5$keZ^u6q8f%AS|5$}jbTW$mjCSy~(p=g2HZmk+`+pCniMjnqme7?3j>P7+@
z%8COk(tVM77T)rHuRFtg+1G-(Xmjp|z2H6$n9?Tq%%9};o6oZTmojWZKBLg*H{2m^
z)`EjO8|6*Q>wkWg60qbtVAy=%rz8ihjm#f)G6$C*5lVs
zxYU1WoHddD(#8n#VFUlHQ%!1*v=#y&g*HyDSS*jbb#+LP^#;cUYpy->(0MH1&ea}x
zN(-|L*ZuJvfH|RtrzWE!eWo;?W7~IDwtk4Z?wBX+Q=?wR09ev6ZZsjz&!FBzF!5+N&7HQBs>l{dsB
z;@ACEr*Grbv2LB+>H@RCX|_<1=c3={|HLl;c_q=S|Bb{s=E-K)=l%CS8*anjATZKE4Uw)ef<2_
zACF7?w)gi>L8owAQn$Ty-C~W(L5S*8ajIH}agXQ9jXB`WRB4&!orJ?}ICJ))lE?R?vOz<3b&ZCFZYz>c}sFD6c|Z(1y`S
z;==&aB$L7bl;1RJplsyPpQUM0#ejG)b-X_{TQ!@Wu|Bv&{WavSB%&pP#MsK?X>;&m
zhaoX~giKew(t$eEBa)lFSgWRxS0KS2As@J-n;-1qAOvl4tqCD4NY64w3*T%#SU?a%
zSd55wbj4TkzJ@n`l$*Iyv>e~{9y~>8VYNd9RS_G%f^I~I?SL%%Wa-;bOe{W-3{jYK
zhd|)V4@mg#=JW}7POd?Sx_8>OUp
z-ssa#l)9(=@$qs0@LD8LR##ktrS(HS<=6Vv$EL+%Vusk`J@eVy@Ce49vH+F%6;Z#c
z`Xl}AE!u%zmj6ps{?AJ_L^mg^;o&b{Zr6349f?f$1U|S9;0of*vLbd{;@3h03*Sl4
zs3bnIfVOClw{Lud+<2N`ZMw>z6CbJEBb|t-@*)7h23K0pm&z3pvJCW`WEpytAScC;
zL;IQ5Kf=43S
zaEPb+Mb=o~h@TzW9QUqtG&3PWLO)l)ECYq4FJjm$PD3kuq6+z=K~^YIYDmCkG-1X#
z?m?w-dFi|Zo1kGv5*h9(VrAtdW?>~hj_ha@`gF@oqKpiVX{>~7M#h||Ae|$Ju9Tq*
ztkl$w$EY{)(M8Il(AiK(fH1A*91S&-0dC>YG83Ud?7k|I%ep1+d2W(O>etW0;&OcQ
z(ttR*mh{qYAb@~h=SAys9^YxCLzkUE{E6cww=UO@Es+4rZNbwI0-v*#9tzyK8e4Y!
zer=|?rHU<9?lQ1+KntcGr%!4u-+QL_jJK@v={V?rX=_v_^Rd#jFKAni*RTmVKFD}h
z@K(wk+H>-#xB854|3UveBw66+TKs@Byh2U{M@oX7n446kSc?_fVI=Kzj-sP$Z!18l
zL!2AHNln+89~;1tSgg4kp3}v^P+Sq6o^A!fOv@!f#>PWq?zn8@Z)@u6if0`qv5J;*>;jXtO~~w)$Ou*Nr27rZ6KKE^NZ;vBh7{7O
zlwNTo7>ZVPDv&bwU)ZL>dHjjt0DBv3;#K8}i=#l?1nSVqePJM
z3n}}hThr1qlgH7huNhOS&FZ>07+Pt%Ud(75Yy4%p=t-#ym0-*(yL3-H&nUNLnHC&8
z6CC{xc2CD_k(u9P)LFEC^OSgR3d|c>qEgJUzLuBa3JvZ5&R8|R-!x5-o1k8I@}Jf_5a$8WU@YfXL9U|PBOC?S>#W;ma%Vxp5c
zYQ+tB*$kd!*uCAXYIYc+=cvY0_3%0!MzmTiHzn9c+KeaqG&x&B;#T+IgdUW4Sv+*|
zXjbzQ_*<{Gy>lJR?+pOu0@9Wm(Qv2NLvmR0ZFxsNuxJI2R&j=l_H#q~*PN)~Gh`bJ
zR%3fgYn>6^kC1?Okheh$&9h~TjJ(g9mBY-<1Zj&7=!cMt^!s<_^@*ICyZ~D_fflYz
z9{ybAJk#z%K@gUK2{pN*j77p`qID_*FHoyI%O^E7^}6azlyAhq*=+6Im8(U3&9LEP
zjAvwa0k9*wX=~BowI%zpk%3BJvvvLAsIr)3>EeFGG$PCR->&1R#Nfvrp@189yw?&N
zJg0~w=l9FMzh6517bh4tQ)A)nwil5EBK3-3cehl)#xxM2nod!ot5kY-KZ(zg`emQ6
z8i0H%&!JdHx215ebI`*k*Os5rq#1ohWNIC&k_AN~x^a7t`FCvvaS%~;{Jd>cvl$5q
zPaVSM(9wimzu6Y1>+-+}XZ`y+FYr*;_hxnf{P)0L5^QX)AG*E%c)F%fc-=Rx^~$yM
z9ASJU0&jX>Dx5ebu?xmJI?e%<4K)6?i=}M=+Ud4#)l4vx3wU#-u@_P18mc?sPNK
zFy(YCBa`H#^Vn6WnW&tWbupWac->>F`8d7!89?)zr7i76DFw18@7ZSOl9a5PWDJ_`
z1>=*zV`UrC^MhGVGz>EB&n+@=(xz7)VOH{3G<)0`+$=ZJM5f39-nynWdc-B4K(elhfrP`XG9sJch(WX7}PBY=D(aw)S}9f
zIKN%?TTFRx_L=ovDY02xi5h!}MrCmo`m}jbR3UcqJ$?%@8Of4WP5cPbv;k$^n*)DH
zdA=T0v{Sqwo%8gYQujs2YvUSixgRE~Jr7}z&IFC484Tp;-eL=P@8Mrf?rXNJw9Zc8
zTQgSVr{DZ64T@zzo$3YRvVjVxlS;~&-U;`zwqJNYC%-l1eN@8tavZRu5HBLDw=>SaKy7L!AFv>7XZj^**0S#+#D)IpJokR
zUEp?3{5A-Vb)dIXTndtD@2-=xNKV(*awwPmT$}IkAf&Iejgp1ysN{Dg!SiHJ4pP2%qGUGJkeLSS_LiP`>J>-T!b!V1L3YIfXVXR!o^I
zlV!DKI%*+Tok&qJ7WlgUNDIzJ864cQmoz$(PW@WOIeXf?#<5znTX4!uojj_lBWm%pSIj%NTvoY*I2=*sW^54bPWYA_UdyZHej
zBOuT|HNp?&AdgTI!bQ*82ju*35IMyp<_H|oH>j{Y+Fj-DcA$A{5n3DK)Zk_gTbLR}
zGN=6N-WOU6EvNb(;Xy#EtMz_!v3wAFpPoNLh7YSTkWR94hWakDnRhHgQ60xL9(-jO
zRg7&WhP024=7>|nA9}b&YBtL`QW5j_!n7y)0zVqQ;dRjaR&DjSN!t?KY`MD
zi=NViHKxHtIss-Vb4(vl4S?#nB@)j}DAcPrFc(`-vrUrr#;1_ChlH8951IOo%ihkE
z!hA_4ULIZ+TL#TwqnNK!>$h?LqYkoPpJjMk`F8e&-Q5I_!rI#wF$yPe>2BF+eELTk
zAExN{KWz;BoIbpC*aLh%ej!gp7tZ(Tp(@whVKndDVkdIM7p=GyRvwYmBakQdk834z
zO6?zVbUaaPepX09QvkMb+Z{ArywN(MLfn)b+w)`
zOQ9NjksfO$V5W#r@{$JWmp)p42!S|vWZ~Z77YTe&;i_XH;uili>v&YnR^jlq8NVMK
zA5J3Z2=){WJ77(PIA4V&@kD}(qBJUF##V%?pgYXztUA9prhTswJWr49kMP();id;0
z3ybA0Fn)wh-n!Uk3U*F3lmhXYdR9b(gK;MG99pa|5!qTOW18MFKn0cl!g5zA?z@(Q
z8xbB~#sL~dr9PJK%T+=6{eB8a%ODc}Oi)=dr#51|G+zm`im@@bZT%`(+kN?3=daxa4v)v}Yqvlp-{_RVabpXgfX5C`f{uh2?4x-v#+rb^4cg;1QIhT
za4IiWt-AWanTrDq5W;t5OmZF`FvrQNa)(Jpfku9Buqo%&V*)Vw>7yeq2`H2cX&Jm#
zA}dI{-dwxtfLR_}wFH#aQ2q^udr3s3nN?5#U+#UzeLRr+q*lLt{^eJGA7U=)?{U*-
zEJSr%tM7d7wKLMrtP|*-V4||~+GqZUk
zsTB4Y;Dn$Nf(uj5qQ28EYb3BPRhTGkDK<`q;KFAdvAFAOy@{ZVaoRC@m4fPq?_ZH+
zOrZ?%q`}M8nYEE#g(ylYn9DA>1(g$8=3YdQ@F}kMjz)3B=&^XcBI_6^5KM+_AbaIQ
zM8>R2wvFhC1uQJ(6K10(nCy#A1z4b^5(eHlEUaeoQ;ka{ZNUa<(=3*wdRhe1QDsiZ
zw3SDmb4iw%Mf`R?IU~Z}#s@qg()9Z<1NT~lDfb>;%)ofhobH=r>oO4ubIK)Y`Enf5
z=H+8U;vxO`
zi|3fc<5p=3+6(rvdio|Z&FUZU{h_4ai?bTK@{iZ@kJ~aQ?wMogARH(rwe@s*cxvov
z2-KdA)Tt%|j0B*xGHHXt<9#>~zQ8CZifbl{c$cV=F{KAOwM2LqePBvKru5+-)Rns)c+0aL^fSv{j8dTW?V%x^5*O@m{PnxMHQ(Rv
zHP(%n7KTlm@I-a^e!ukFmdd?~&1%dd{?qmcU*kRd=tmB##y@(hZq0-As+z&FCtY#*
z`(X7v$S`?0(&~GB&M=;IKc*)}j%mY9RV7iE3ZQ{Pt8f|G6$TfLh3-0PmDAiJ^u0+-
zy^=lsN}GlLV3*+mouLl(t|Wt{lBW|*ss8ZLlPu4Eo>RjEtL9P3HJr};&7-sPNm5Uk
z0Rj0I5R#j6o3jexl|=EJ&@j#P>b=+F_lou`Qsk_=9S!#@1VxC^OfgByzI_)|~
zbv~B|TBFZQWKb)<)Xu71lqm&2c5?Sso(j`sUw+JSjN{?YQLd?DFVSPS4fY+I%;vpq
zT=8EvmN)3VSj_>XM#67iN{wLbC&D|X4oJ~a;BJmJ?r(iua+4CiweFwlq^
zC%2N0uN?1Bz381*L#=z$e5dU)*--1!638;045)Y(%|x8qtXnLL**(vr`v>k{ir|Yk
zg4@eMr^~KO!)w(gP5I&y-dFpw}_A-w5`h74=bjbqj7tj@B)3@37VLJ|@q
z!ygUNyGkcKuaR`bP9ebvukpxqKTJnJiJ1us<4yk%ijB-ovL_vk9_O2dnxsLr59|kp
zM1zorK-8^!6gnAX>MracoljpCq2D_LHTYPq;uf*Ei7aGLK)@f|&B;~JD`5qyG$pwh
zZy5o9s}wXxg>a(itrT+5>!MyBkh_)J1u&i$^Z+Lm+0pLW!VIHj>?yZOGz
z<&W(n6CKp(^^V|wHm!-vkV-d^PPtlbOEHN*DXTNV;xWY}Xw&>^%tXm)8w>|jqq4Ay
zOM!p*(ciVyelAfDOcOV=JhwS~%C{fx#4C(NeF9k728Czf%8$4ikU=Gj)R10(lx)<3
zE0o3Ttp1#WtF>66X09F&~UruT?2T7<~|_Nl?-ErC?*dWTU$HGl+p`
zsd151w8eeDh*nSC>160`z*5e#XvKxb!sv}L84apN`SPYU-l>;Awdc&3$ym(f{=}EI
zHpOKpzf;wQtAWiWzJ4OR>fMIxZgfA%7Q@SybxF>f_vW19*U2-WX4oB^1H8YTr93#b
zE5sAFpMkr%jir&BK>jAdb+&|2Fc)8eZRzcvYx%eF)92_;PV?Bz?{T3Xe02;o*|9k_
zsQYVTkmEluYD{a35gtzVP1oT6Y_>M5w!PO*34gibKDEIi?f
zYk6AvU~ZpN29AU^#WhQMZpvJA8X7P1iTkNI;Cjt?gbt!FVCNYs%VK%_h7sTV3!Zm5
z3(FyYSkSLa%kTSJ5R(MiLOZGyqjR;RlU;{F7-&XoqtiE-
zz*0*y{DvtRJ34_bw|*e%hvPR;u6R_AYy|)<#Tc)Su0uuI_)`7k
zq^xSlVUU}HKlx=DrFrvNAd>w&vNcn{J)VOmk3Uk}vn21AF7P?u`7b-ro~46DFCUXH
z*rd+zSqI^04ir-}6k#p?fX($D0kZ$!OOJ>fBQ8@2YaS<^ZrGQle_WAZiVK@PFHsGP
zyq{)Jud-RML(8n)l2?+}j5$K#vAd(l$N
zW$y1!A?kD1cfRwNsQ1qhpYQol=I?_Nx%0
zi~E^ycf)T!nLp%ApYy@=Vz~N&=ti93&5+AjEPO|t@s)(dJySW)8(y?_ob}Jo7*&gQ
zNH_?(s!psQkUCpF=^~w^W&7qtO-Wcb!{ZRgP7T80%`L-KF7pzE#mz12&#w-KH4L&8
zmE_CBV+idqlY+FyG^6N3%hs@_dM|K}2&C;SL|KW})!4O>NhLuk2TKX;HX|^?7&9Ba
zh_=%?*-gj~n{|mo8IZK~Ue1K=1^Jx8gU~L##UK8XlJCyG#><^+~D}@W|(5
zU0Q-@CbU#rev0pTq31~EdXlv4H0%h2A-;J9+Kjp03_+1JcLAxCuM;gylKAbcBR_(M
z&o~MoF4SCViq%b99c(_|slJz{ZN^xAq@SP>rpRN5D7F@svTelJ3)wIe$5v>;)u?kN
zJH#R_Dq)hn$2A|7`UeJ}9OgseWYe(2#>f#0o%~03c~6RBI}4j*mlS7l__`49M7j4^
zCT%)*8kU;Xh7JZ}c^Qg@X4~dO5)f+a*!hD{g*azPsF|gJ<~rxsPt)cfRP%@FBf4vC
z7=v#ZZ9k;mP)xGCAh4sC5Y)SbyqjqyC83~%#`o3^Wy*5)gv6-fR6e?~y9FLpgO;A<
z%S|VkVvN?I_fhQ<92Uk>?-P>i!c#jtHcQo4$K@Y@@|jcyReu+d_z!CYR}T&6o#vdj
z3Gw}rSUI>ndeoS|HlBxSY?Z8MDb%CN6Wi^ir%zxuo`O+C@|$#>@}$R0gN90br(1Z=
zn!R?|W^U>hG7#UOTCENu6(rL0QlnAmOU&JCL(We}0;fOXNFdqV9|SOIvWg<x%a
zh)_h!jYQNZxJ8W*NTnlp;t+!vRFVezaGj>8QVCOpH8QGYhc0dT>G^XN8jLw3D$?Gl
zyMXmfgcFA%bZPW4%g#BN6O`$36wQ#A(WH<8$jC&a6d6iPP7o4&uLis7ST+kt7C%;_
zI3E3_t+YyNZJ>43CjozVMHIVck(@kH5_+K&6y^_;DRLe3VenIM9@M73v3OXY*SYv)
zw~_S0YJ(H{t|Xdq2LC7r4LQ8y!D3ciYM6%h-Q1<*__aC17x}c^+n+tlEXnm-O%U>y
zJm;~~wlm8XE&m!*Trb@C``mXM@3*KJ~pgdQ=%gDEfOxZ}V$7TXXu7o^qspAXJct2?OM0yf|&MZz1RY0D|pw14G2
zbTNBaI`**&NMAZp2xNE3KkJluv?Mc*j9i|$8CUfHXpv`S795MGbiN9q?o&zO7vfhz
zW+E(R!J$79iU(Dm2gzx?2R9tLy&YsOcujx;!g?~Mi=pGs#YJdw3%a`Mh#kLb+gUMX
zsrjmtR{h<`eDN%Kt3F$1I}E((qA;kzoTkQ_saBZ)b#x>wb04~~DzGcr>HcV8{;3K4
zDOj{5Gr()I(V4`Bs>X&4x7ztg)3%b{p)dt@M4f;A*WIGZXn?YFy$XZ(mksIMgW*@2
zds~`KM85q>yr?(NqV?%g|jJJ?jFK*6G~MU@JZ(}GRv$5i*y(>U1gtUsd@rLNnNx)oM=YzhVDbs!%80!>S-7e_z1DyXrcI+aI?i{EHY3V;&Z1Z=8fZ>6
zvG;%sQ|qa{tE@UI+jy(q3k~z;=xr)dPVG04PM&flBOlUfhpv?%!Czdw-Zf9V>F@az
z)3(@By8OL0Py5W^>uYs&Uw-?Ao0*#<+f3y(_Hy+;UU;>Xzgw_>n$QwCaFPdK4ZEB^{v{+5z9}9|%3qq#NJ7jJDAz^UvP6G6bo9Q=}cyzFLi))NN6_
zHPebf#Civ^$$B%Cs+S{W{pD*;^pE|tU`S^VJ>CbuMRlfc)By81O+m5Hf-^_z`)t_2x_lx{;r;6{F3zBf
zMR_>B!R)?vdI7b25)L#euHeEKXNqx-Une0FMQ*F$NT;
z41^p(BZ)xQ>5CLOts*){Wcb9xUt^(w7!ow(7r^kAQe|oRmIX;R%UwE^D`JDYV?Y@!
zVGOuk!pMmhEqh>o)4)itu*qqT_LYU*MB%KtQ8kNB7(NzC$J``C0OG+Sc}lTsvR4#^
zU^77Kfkh(CeoBTZmZp7+97KCVQLxfjSXvaFY?@UZEnwlTI-061x%O<-RTe=VDm2i(
zQ(|>1as|t7-RuN%eAWV)v4eCdp%P8DY#7^B?Qx!rvYVFwHSxhUdq=$@TVQe@m4g@q
z8B`yj^PKf%R3;5=f|<74hb?8NwhZPhTI_UJTWd7^BZDE%=Y8p{A-9cj_MOI&^YY%N
zc;JI{m&j7_vs4Jej0n=If-;jTFsE1RnP{LKfCH^2=~ZjrUQF%Vv2~OF0B5>-WD#u0
zgs2hh=EO5mbmPG@YG?EVQP9i_i=|#5hczIaL4Dzp*o7T2A;Z}ua-wYWwG9pum4lP&
z&C!twyHv8u)L(_eqF_(dvIM0BvWg?8=0
zLoIAygG34#k_uTJ0PCMg%X_Bk&7C}o>m
zZ7`1iG3neXREjO%Y1isV6a~Uy{e(jTisdqYv?-TviPCEoh!9yHK*|x&n9G7
zlKAU=rqWinu61(}4aYQ)>XwXIs&NK`=WJ;J-di>}r1bF941E!2u7E71gKR2}Ow@s%
z{w{j+XI|6Db$v&hhH1U!pcOL7m+bqqIIfwihh=+Op2VCmCo?ca($XHS%&j!NF)>y291!{Ter#a-xWCI
zHy{ae9skjrVgRsqwf)h;k>G<%K10JvNvdEH18-_Gb#KuTm}Sh12sp?Uh?K9h+|4>xY%6hv>{q
zH_Mg5LDXjj2T$usg!5VIO8Cm-!3uGM@vdYqaB+dNLrUYdd7&usz!^!%H_qMi89SWY
zRd+4oRtpMFPEq)V(i42FJ(sb?<_VvONEAqVXwq#9({Y?LtiRp;C3V{9h;rNf_z&G{
z*3mO(B&@FO;eetAEt~ZyNm1wm=Q*+5;X==7nS*D%=Eu?aQmdC$*E7=Jr`Xy7oX;9*
z{$SVR1BbBhp!>^-J1XKeM{A?Ubqg=m@i4ajUF@Tzk`IOtiioWIfr8RyTH
zFnUKSLWIAHVK(=QX$v`p#E_Pk)*^tFH|%_EyTe?*7uf;hBgnLoErGs
zarwjtz7di8J5v_~VR-%Fo-mN?ajMN#D8HcKq;aO@4F^Y?&(q_|llA>N&6B@YN#N5(
z+oRWi`^u50d(O+l2e{^kgDU5TJr+0^9S!$45r5lWOa)=4dl9~6)(J+ESrD`Pjz5GL
zvH6m*iUc;6g=}@i;HKMSwqe7ocYU3kXrpxQlQwO3=>=hx5}1(@hCQBdR4$`14@Xwa
zRti5#eOpN`-9wvq*)&O%CBZ0$O@{X~9=WHt=c6@eMLh^tsA3U4#tNNcDwMYn{aEZB
z^z!|qa`q=q%)!xB!hYpwve!P}@eU+Wk|$zFhZM#1;s$HjdX)VRkeO5+ad;WNQLjs+
ztl-d4wbM>u7Y7W=40oL(5A1IF}tmzY=JXi%&ep&l%c(`ihLfT=h7h>Dn!*r`g`nkftj(zE)
z!i9PlA|iA~T3-io$^^#;wD%Q2T)q=j@Ig_#4e&N(ae;G+5f%&C7eKGaOl
z(Ms@D=+{E}2vlx<+$693Ltuo3S03rm^icJck3HxfcE5E-H5eq7|>?QT^QgmzXxVBjhny@H0l-?rvi>kIR(-NYF
z^NTFce=_v|$u_do)L<<2!uo8H)3Yv56pLA`Tu6arp^MAVosT;6Kzq}jeArlIgO5b93r&rY;s^_drvRFJ)ep
z40JwX3%A>YJE3S#p>f
zQ&FATlDZ_!Ys&)5Fx57Yf58Nlf8quN4C{F9XJ%&hF1NVd)j~h=x}MVl
zEaix+j3N>U>?L+Xby{*zUokvbs6($?wEZ(`@x%GA4?;OOaK`Dm-O6S%P356>)V+p=
zMtg3Q@cdO80x|j8au^R6&!ucXdxqVX&oW9UMuvx1qos)P2F@(LABiBK6L(Fie{RV?
zb1AdyD}P8^D!rOsU2=p#CUw{f`R+!u#ji($ggM!TiN72_9zNV%t?m9QBi?@ST5_1V
zxEF3nFt5=&XxhqfI|gf%^Iu2jr;k_L7fU&*CzE^k!WM-5_K0;$jtj1-Y#*=NMjQ~H
zuK1z^gE^5RvDvlDG%TglwBI=Ii1{ZbkTaG@$DyE>ExcL%G(komaF`==A@(?D6FljPb>8`r2nz#9^0qSp^;O-Y##cwKGbF1HPYXEs`@9KrB7gU9yt-@^
zyswIS6R>D;e$8^-*rA&@&4x~WX)Y-a
z(fKMan(3*{){V@pPBF`8TNYjg{4qL*36%nRW$v9@89(thd9RKktPnT}N~JOuXhW^P
zmL+Dy6t094wip9LGmNX{s8Nbj-u)X^))tJ_GzvtqZ^RgwfppsQT=}6N3;aivQ=&fL
zYGZFS9r`6fd58<~X?$afSXsZd7-9cqd;_m@W>MB(JkH}<@S7&)k3@4LesXZeeS(~Fgz;yUOm(0sM$p1*DoxM?O$8fL0B*|A%V$FZW1d@oId-WJ6{-YLwwwm
zeGQJRby&JSrQ|r7-;|S_{ClAN!K%*EXXX045k%tv>1!*>le$->=@I!_>NchBhSVhz
zi|SM|cf1o~pG0qyZVkfGS;rC;{`rr<^0YFWz8@Lq)W*uhbZV{q`ZR7XS(bY6IbABa
zK1v#Ss=!96rbyxnGaXKbMAq&knSg*h-Cn7ZF9`QmmhKadwMuE^;_H9I3;abAI>Ed`Ep@rXn>=RllRc-t&ng=@0>LA#DOFlTVD8G<5fRgX64F6>Q@|0i(1_AjvqA8>`XP92
z{vSZX>%F$eJuWKxbU_y^5J&38hhgvyAB~9LU*xVzWG`gUcFr&{0`+F`FNh?7qgZ`y
zeh(QO9Q=FObktKbqbG1&`C71Ib+xmz^Tp%i(Gzjqis#+O0TOr_J5NvIL_Js2nVs0R
z6OcJ}{TD%m7vta1bG?oBwRINpVHrJesTB?Z?XTj~x#C?0f-8}>xleL3p5Vjz1RRSK
zhC!!e{WDI{fDUSE%wPUcknREzzUfy)GmPL75C~IrU&S_O-iB=6nlf#5d9J*Nsd+=t
z)6{2Kx5SNeBGWcvs?wA0v^-Zennx7aU?WrZytR2eJbJXsI^cl(a744SVn5hdldqUt
z5ny>^5yxdH;7sy~d%Dsx_5HQZuc>cCTr=w_#xs>d)@}<6uC+F!ovr%9ZV~O)Ot)1t
zeDILrGwPA1v+`xjpb*ckQ2ilu7x!bo9@>(#d9tX_1RsUl5J=q1%Nm}pcz<)noI!1e;|8Eh~7yrvl^_iS>ZL
z4dI|w*kHimy?Fg*#U0n4PVXg}75t^nILuGtd+b>#(|7k})TUCacm4CxCI&!h$si}E
zQN)zf+UP6IX}Sxm-B7FX=W!&=|B~v$IiIz(*Ga0T*@Ed$Fri>$fc(93r!kDyHDd=H
z$e#gGFfyvYuxS$+5IfsTZ`k7zAKCMAmiWK?-L_YWM3fv%d($^avgfCt^1V5;DIVW&
z>6$;p9YAsa8nlw(a3uh@%R
zX`++lhGsUeaPJ651s*icPGh9m*B9ib5f#d1tg9L|rOm_*t%+L7tpP=|6KpksUJbXu@}bvGLyh~rbi%+S{Fp|9#Rip^w0
z(vOtBNKIv+Z>S2RtEke7kvl+8>on@|_MSZOZ{VNjOnMWAVvad6{JsZHnyx}syX$O3
z#BZla20o1fD`%v>{{>rF}A4uhXXji5Y1ybTRhur}Gs$)0AHQVO(
z)c!>4d3~(EzuZCqI1mDp2Gi%7b%u9&uat+beBC15}21(eJg@cYwD5h!m18;DiQ<*FqPAt>N_N
z64^F!Y(&WQsO4&v$=c=5yTg7EBy&!L=gf5Hado&PG&JJ({8l)0LDut2B4Kxkx{l3D
z;meAsY_(q-%*G$jo=DlYw|W80JpbdrP=-T3^+=cm^Kaskd^7&oNE$8DtD6yNHo1nz^)B(jmxJwX|xg`UD{X8q~9HsTE
zq1lLKFf+q-EeDnWekVWfZu_egeE8S6S4yzE_;ua|V66Sr@X-CT3)`3661s(syTWlG
zE=NZ{xbeC^{@1!9Duw5=HK#8W6qA&hIbyg5^fF;Slh*Dzzuhm1MLZXU-GUw-V~s%&
z${Dp{a_%)FF*P9KgSd?eNbX&mt-*H2Z2KsyrSxq6U$SvVfBaEjG==%;{)aJm|
zbMlvN-4*(VREAsQMDyGTzKupb$OLgs9esPhvLz_UO$>npZvF#oSjPiIo1MrPwc&KW
zk7nZY(lmZ0vHHD{2eVaM=->8Ac-EfoMV`=`_jG5zxVj?4d^7r|IVj?WQ&#K_hj}Lj
zWFrzvUX-6?VdAk0XA+gniF+&WU%g56M1v?p6-i*EMSA%Vcvg}_+XwkDx8|X+5FU6|
zF)WIk@1#j{r(6x;m%l=)vk||)P@-&yNHwVTOm^|+)5sRS`t|PMWpVR83S*|ntB4J^==6(#8QH`Ly!Haxno4(jlu|?O
zF$X%ajF#*&CUUq!*mpRZs&)r|rF~-76VJ57Ja@Gn|Fr8N_>}N38lPYVUp8xYaB=|h(U9p
z-WyqX6ZOwKa>jAS>U(CXWfeLsAid-MK$DW@efyaf*n}Hib}NHWqb8(3l9IOcgqK1w
z4np%&a$D8KYnpqCq?RockA9Q3t9x)!l$D3v7pZibXdssDSs$B}wisHX&C)S>&J9>}
zfZQ8qGh?3Yiru44RC}lS9-nE})6Zt$Vmvj)p?5Bln_B49%Z4!o1rA~<8vVDh-jj;v
zfn2xtnwBizUJOcfSen^AZTOW`MP1bkS}HJQg^7(XSHzkK499v{+!M=in7|?}j1Q;@
zqihLOq{0qJG<13WhBU)A%Eu-SEx=@roY*(#&ABn+uuLSzCAl4sDtAS8?PA2diiKq7
z2a?TGPZYXKOYGk&B5owIKNzmaL4sF4kjM0p{CM~rBe60466F8^$KI2|!)X7T=}^tL
zNNUN_rjvho*>>&n)vNU*k;oYeoa^7^a9)Ib$L`?K!Jk2wl`ftqtNEyGU^L9
zQ|!^aqkZGdo$N$)yYHsYkK7MUf5hUk*1$}V4XT+-j98Ab?-U8DVOS18OFXsg7S-xM
z9CvOxbXGunzjfoPqkf_lDAnExXWIwOma5B)Ci{@N5h>
zO+Q#|Zlzkcx+rWK%bw~oEP@Ea=QjwAF~Wg0g4OY8xy1^a)k3}&y+i$S8Ue%qO+%=(+9cUfOma9?Dj|mSQNg1<-q*07?O}B
zfbvYd1J0l#U#tzh28`c(3zmiM(caPy*XjwV<-eIYm)U9)Q()?pw=Jj*}GYQ;2KN8VAhCTmi
zFIO71eyOrqDbIY5V2Almf)NlQ&-XX{{N-jaYTBT~gz_Re?#0R~Zq;B!_sS`oux))69X~0Ts@kDMr
zYm%G2xUE7A`&BqImUx^TssQ^QJ_gHqV^LG5kVGo|0cLBM<=NRTI$F!<_hHqTLVhz)
z-Uu)81H5I9E-Ne1MUsFri=V7}Id-yWJCp*SWFU*{ZP;U<4=Ibm6nPFeT7f`-PTeDTzu3}T%
zX2*+6o2tqn*-tLgAk~`G^apzvc_&L&WhV!G>{1ln|A5wzRIeEW4dIrl?-Pc>uq8!R
zeXHeL^zDV=zzhXpyH0V9{k;CsO_0;pe<2XX2-|0=c7b9Q!MoR#)U_7f>*w6z3i7}Y
zU=C|Ltu9&e(*bn#Cb9N*susi0Cty_O=0kzcDSgjhV{Lu
z=D^8ppu?n~@+(}tg@`q2?Gh2GL^>H_SSI>u&rE8CUnn)5(ABp&8w70{w&KG2{^NPA
z8czfrg}7EttsIV0$~T${pL-foG*jsDny7{Oa6LHGon!h%=bJLpSp%kG>}m1Y2x4-2aN1=yZjs#hQR75m$z~(X
z`|JFc&ftuY*SnAtZDivn7bhsHMoHNh7Z;>>Sv=)J2Zk15o9g6v-8JmYuglln=1s|J
zat?GCRpD#t`$Mu^$5+|KFI!vi_fDA)0!GZkdr7<&kzY?E*MVVjAK?+6D?5#^6M8mx
z2X$*b1lG;l!Hj`%Eoyn$lpu*ufPED=D#Axp^4%e>Ih7#4`!y>uDAlqN!$r=s(KiO_
zG9I2w#OI1p`~Zyd1&=Tyx8T@d2S(hC*vFVaP&4i2=wPFw?6pio0zXa<;+a#$mq<3BkIR{>ZIzEj6&?`)4i|<=
zrq@{bEIH$B5O9c~DK?mZ!5W%M<6KNc?7E%Jm0mTos##NB?hUTrMuA5@Z`(NC
z>1HDly3*>=aO(MZmN^n%rVUt6`WgH!3X&VdpHpU~{4HL6&?;z)htcDa9Wk+=R6{$f
z-YV_eWGYbcFVtFgbIsh7J>HL4ca%K!dN$vAv_-5utcdS7`)AL6{Mzhk`EksEu%UvE
z5uc2~QC>X_S^GJoZeI(^4igT%v|FXZNd+S~BN-jL
zwy$&M;M?0>vhITRq-p7*<-Z8;z6%E#I*ERf9rT8bS_;L4;t8>lXK$jEi>192971@>
z#w81VRKvx^8!7W-Ra&bVWq3wu$KpK;-)Tu3#)e=)bB2MpgDV6F=EX3^_MU!eEa#<+
zaV(V?)?wD<8jpU)fr>f9p!2wrt#r!zgZM&h9y4`F|D|A9=AVJL*tvCblQfLzF=$y`
zHy=LDxmEcInVt`q?(bC&CRCoCt0cZk`RZ4X!&Fksbj9~tA>*cx7{XghwfHA4Z3V?g5s
zDR{9RG5Yi3y{#}2uFXv0Y(>R)?qywnnPEt0xwotXs)~wsg~Q$lc?E0n2AvAbL)*5d
zYi-L1wihbPh)JWkgKmNHZVvOGu(|9`wKL%4|+k%`-*NOyXQMRUIpw;
z?~U+?&@NS2JbDTIr!AQ>vo#&4dOddU_|u-qHL2mWxI@CIqzUs{ee=kMrPkaPu!3)b
z#=CxItvK*wGtDmQOn#=;-}MnP!Z+Rx8xqaq*l(eR{c*)>l?hzt*>ed5%^3TTbl6Hxr5UC@HOrkLf1s?-K&$3kJNpY|Q1*P*9L(c8Oq1P;5w
zlbgRCI`S1K>v=W=yne!v;`x0=jm7}sgA!GmQrQ@_I_OU6KMsXlht8L*dqv%;O)gya
z?qY`p=kZL&yAKuZrsT&^D^3VfZ}7eo(lPS&{}|N9IlPrt&ifhnCR2WfhtlAoS`Ab;
zTs>N`v8YyU50^eManmV~)Id-o=%`90fWE}@?oL;F4kwpX?*!rjAwPBvQ9+%Sb>>8^
z7z6#~a%gX4#%Z{08;OXgyUBwv@(2k>>4yx#hEm7y7vV|YS`Y_bNktN#tUtm21*K1_
zRf!GQo|fLF!G=eg*b1$@pH{i|Sc~8(Z8%;MWC*+|We3f0uphS7m3YAU{RWbBm33iN
z;DtV~b8IdfFtnO0Yaq1sT)`CIxpPUMQ|vZ9TrLFN5(>JUer48Q0%v49W4i1$EqDBt
zm4|1GLPRf5BkIO5X?5%QT0lj#v^C^8?E>0La;5HIZ!O*ubY>-N^Kdb%aaMM0{q&$+ewAL<`&9c2UOKCFcfTJFTfv|06&Y2ESQefP
zllzGuqe0y5@?MlY;d+V3nCV9R3%C-^
zinpc7O4f%V|5RWwP?cl)D3M+ia$=e>b|^;+Get*#r_=Ty{i&2N7Ec+F|6UH3uwpD6
zNgSSw^5Zb%MLzsM;MUBkerPz}ds*ofkm~e3g>okc1Br~(|KpZKTAxW@&gL?fa>#LN
zG`TtZyZ3=(UBlAYUlsiPF;7qoAUhmXm5MY9y}?`O%L!TY-%=AcMVlL)m*^a7vJ`Uk
z>H`jtj@eL}*ozhjmoSd*$hLctcMzDuH?J#N5B75AzXn~7k1S2ctLe$z9MD(0FJF84
zZKrbHDKBL5_uH4dwO=}@ST>y>$REU&Ejj3njC3Qu@ry@!pf<+XO*LDyt;gSu}=3&%~rX+0!b
zcu9IB=M4dsGLPYaw&OzMD-{K^mzN&Wjdn6@pEDY(=Hs+7=HM;cA$@NBS-T&SmE#?U
zs+W);cNH!l#f-mrKg>YMn2uEYJH&eowZ_gLKCdZtxsDVSM?>N=D~ieAwsGWfIjc^6;IPGMs;33fICod4-Gjl)P*)5~u%u
zM8e;qdpah;lv`G{x7dVeV5*0*xNP{k5h6cl#35QGN&-izlDctn%~e$@5Toc&QRyq*
zC`{#nixKRTdE|^RA&0NW3v{Yjg6EG%hbkLj#ue4TALs4zQwUU)e0!QG7Uobfh*;dlo#ue
zWNWEF=Ms;%V>th)kRZp-fsyK(a(pI26i63*`3z64G0v~3#8N@a7;tBcZK81PU{iHG
zH{$8oG6dmAEiw=yr^?fxr8JC+h)^PXK`=Np^pE9nm4$ne8p-Rnz-wORR%!nVY}$Y_
zNdJprr0=_cRUrEL`(7cB6Kd6hH1LYxLXQNOP{7e*iM+c&AHzIC;HcSdiN`mq9^i`&
z=y{g>dcEh;cDT|;0(z`kiH$SAmH>QQKMyC(juSFFLI5a-C#x-0k`7uI0*5VE%NN|g
za}}?-e*88%R-&pXFK1Og>oIz#r$hoHcsUD&tEP;!_PSeWIBhvgKN^g;2F>LePzCli
zKf#z|N6h?&gRyG8+4Q+}@*dOK+L$Xg1-`PRM=TbVpiSvUp!LA2rU&5NS^9-lA*#3uR38he7Y8?M?~X
z@aV~;uosp&+&sC67m{!hMsWJ=Y$?e7K%V&5ee!!
zeF=l<%PNum@m;pYyN`%9qm5i*^;8WpaFO2m(%FI4Cin&+>7&td?+~~1tmUf-?1r&e
zUy=sc!}Ezz45f`OmclJB*H@d1)P{R%xwlbslaZwx?q(c`4M_qA64iI($$K(tUqSUQSh(sw_UyVob*Sj;uTO-wAXa&!BLWqY^g
z3W5z+F%kdbuklCg16Dz#xL#U{Gf;gm%N5$?B2;s+ye~(ARFv9}*+n1!A@w|BduNTj
z?cX5ru-(-zfy-#gXAkB76oM(HoryB`WnN)NUe5r1$vMg~z9zN%PAuju4Hni;Or5y>
zmWoXOMUJfhC?U6t=3E`R;U2lR~oR6WFPEMQAw
zqzd_KFsirK*m-0OzcX29(i_6u9{ig^27aEq
zErv2xl8rXowtY%!YL9tbMZctdo-0JR6T{`+ks7(DYN8JG!mLWVzo
zFmgg{8~mlM_2y(O2+o+4rCe$H67*wYcsz5fGncIFTQ9cn?2;Sk_ujV@PezfAn!lzbLsj+5-zeD&aZ?N5&5cB&jt%_<
z4D0n|fpE>Tk$uE-fA91U%MslO?d9F#BB52oMuHEN)+_^+i(T05^C>t)
zohH3v@x0VPd4`;_$KA45YI-GV1PyI$FgHBrjcbttwo^`kfc8NdfUW75%V_m@x?9Qo
zCH$Yn4l;FR4O(^RK%PgZUa@#hBe7pl$fGu9aXF&T4&O@~VSAN0^{VOl!IK9qf6r3?iZamauZ)l^7
zW*42QKVUGAMq{t`Rq}nNosyZ3KOGh7+rk+F&?NSbD@+KRt+2!@TNd^ys)YaSf)s{g
z=X^7lxw?N=P;ovGR#-MkDv@V_BnBVw!Wly=6}2D3fBkLP;C{VZ@WdyT@h(6Ou8X*R
z#5A*9$J5F)ise83Nlk`~T|zDDkK?~DEv)mIKyIFjkdK0JP5yo4W{UqdxmJ^4f;5AJ
zJ+WM^gP<^RVYQeP4LdYMvXf0A7cK-)YbqShyp{PAssowY;7>p#yoWxUeY|(_3Xh
zz0|(=D`uY{@E|Q6Z*46$X}*$3MY|4aV2j_a+#gyry{^{h26dwif0t>ICnuhXVvD%;
zbM4j=GKx!?=@vswVSeJ7uuD%L-_xb2@z0c9lckpRYUeH7
z!4omEEcFj)(4`WiD7o&g!eq&7(_vpu0?pM{nJS7OMD!)qn8BYk2he|vL)s!!txfJ?
zBczu`BPU}WGO|D6n7kZFBg^xA^Cs#Zjr0xKtcijGrT)B=DW-jFrvX~v$DmJ<{`O<3
z3Jb9w@|*N-LgBQ6Y|irhJ!#e$jE8=?p5!XjGna@-<{Mp={WmGKe=a#xgy~|vQ>Z6O
z6Pd#2Na!Q>29qHCV`?1K5wjX8w1eMI8r~BmI%XE}Q;C&w)2MnJkkh9MioH<4R?_Op
zm2JA*7WRWT9)B+x6s`*rkBs+*WbV|cZ?TlaDqpteg`q8_DE{pXu=^XH{gL)v=rq%t
zHBSpW-8|u$m}aiBLz#rW@3l&_6S;%u#?PZz~8cDyhRWkBkBuWjI{-
z73Z}G3bzqJ5tYDcS|=CU>L+j95J(_9uwgBoB`e;E?a;C^c2^=Qfb;DsUHk?Rav*1+u&S*k9
zq&PbAOsZO0To)Z#DKwmKwcFIQ+Z>XDmPi?cQ`Qm-mpN3LjqHy$Gd&l{eChpBho=eh
zYjVAw1+2Lfr(bhWUN|ub)p>RORKQtcDiHj&$fd;F_fK8)XQ)@=$w|CC)YCWuPT+$b
zH>m?9C3?)M;eb~^mMeZ0(Y|bxdgVZ5x*YZJt@C++RU^0g;^kTP#yWR12{bDMc)O9H
zYbpRk-~gFoU}1n9V4ztdayq|Ru~W|1Bc_w{I|)=s?Xz7twH8YuOrV?8wf+VSfNMHj
z_^z@q$<~}`tlh*(F}jS&n$4u#WECt~X8KvWIyhAz^ap?1h`!iXqp~nwklSswct+*^
zRc*)`g0#-K1;MtP9XceZlu^Xh*;qz_ekr8%y+^-j0z_Q1ztQD$u)+_z9eAv{iwDUE
zMrPR(S@Rrx?Fwu5GERcAo%{1|{m;@9hhfY@P#-4_2u3AJDSMc~Y6#Rq$|c8APc3Fi>CY
zVj~&7ACnv${726HJI=5)1in~31&KA^6R96lCQ>>PSLOoqGP`)-Ex&Aa!*Q+PFd1ce
zrB?6hDORT#Ob@R=hOVW?Mpu)36b%Kb@wAm4=8uMUq&7Yx9W(jz>vxn;I!iyL85wHD
z;y%vY2}KSbe9Ie3rivINvzrw9_58?LO#ipSpZ%}bX+)9*te3Z6{_Z(h>*X=Dj|IgC
zBnX{E&zggs%_OF>x2FgAA8htY>PqYIEKi;XIa9DG?{LkZ6{!?F3tq{zCPi)|!mri6
zF4QUHSm7qCY#^AflDMGlL};o+mzBfmvg+joxhM>NMnO8Q(hv-RT}T|SJ;iFIz#-(u
z_#}&shlhu?;|lCZ1S&gU_@M*ifyd`HMUPz-3cODfe+_U?tnIJ4HcWCzNN~HK&t^u~
z>2)4sl-jeJw&Be4S(utdh5Y$uK;3ni0|=&ml{Ws|awqsVCjdHOfbTuY(T2Jd3%#ni
z*)IGKoCF;A+n;NssjK7>cWU)kM(}5ScSX?a>g16uYUo*>1X7Ol$wHaG)SR&IUi_Fm
z(x)}WiD+!$vNfpIE9j~O2SFhg^rC{{P#~IUDd_?y_t0rrHFP1YGj`99Q8k;JE
zp+#~^S1l8E5;2OB!Jpt^k)R&|6TDCNgl|6Izf@yx4lb-fm4+SW^<{-e7yp*u=M)&6
zKuM|m;ulpEyPe_M4%w7LzJy?W@BNl^7AJos5rcnU5h9q@7iWSTl247U+)7lE+JPbX
zX|w+0Lf$9%X(P%Dmn=*f&Yb#kJo#vw++ZxPbeMUG6C;VP4}9KYGnkm2mc{A(*)3km
zeO0DiCb2vbC`8!k)uIcWcr^0~{gsvjrK#)EreAqB3~iZIZkx0~NAJ0LR8R-C{o@FlN4Xlzer%iMdh%D=yVhpCH@8j(A$&+R97=wQ+t!Q^-ATJ4qh
zqx70%&ay7(&A)GpR*56XUSujquNzI%D0`FaH|82}%q)ZI*~Ktle8>vX<@0Zr^m`E(wR2}rXtFw;zWc{#St+Z%oGY@
zqb_I)&O_(CCoX(bg9;(M3ia|){j(M4GR5!V6=EH3NM#rr66s){31&dnbA5!yYszkvj_)hDh-^W?dJYYv9{IA&YcdTXZoYpQlf-hVd2
z_ZhLAPy1;EYR#+kt*p>!KBU!ePe%b5J#>4p*_|u`JuUIAzoJ6cTc9f*z~Y|0X2*u|
z+?F$C3-5S*xH}nkYQ+Yi1ZmiEV>wEHi^4sTVp~O478Poc+Z<)}6{7Sx7dqUuCl!R`(DOfJA&}2T}DKCM`{N^?~`Y`Da+&FUvwAv+w;d}aPP|*V62*w3W=R$
z-7kSaY^f3RD<;Rzfe8#N5EVqnRf!kan6oa}t>61xq8rHTnd>R>|H|rNf7oup5z&^4?
z{qP@>Bj3mk-s<}0b96C_uX4h(*pRe?hIbyH&32hV+*F)f;~YdRccAw`r{t#)anbxDU&&sUIr(OPg2@Q2Ua;rM`<
z+DPO>CM_#vAInBgQz6sXOjdvy68Bwra(~b$uWL9?h#{F=^d}7A#Lp&-Ju2=qDn#|A&T@fsa2t`oUa
z45?cg?;;*qP0h@txHL<%OuP?qywupenJ#K9NYjnp$1AuTBS)1nTc;#PnMk}yRTKlB
zpGiwX3k0O@1(Pw+7nP4tL^kVY#P=!|(QQ}yot^8}&6G~`5VN0HpX$&oZ0Ad)^&S|6
zBid^UM!CG|%}ZrxZdaJszvoq^XREXsLR2p0ypW|3bRZb$9$@2+Ch!rx>9R=9vmC$n
zGP$d-QXJwyh2`-SsSC-#yqJG|_IFWm8fuy&js020%nOcbhH@@#-R4gFu1cOnQ=ju9
z9&gMlr;JiH6Km#kn1VdLC<}BQ9&2Xe9EpyAaQd@xDba*!0Q1`(BVLk}k_C5Ak6PSY
zE8ku1*NjRcA;EtXGxE;i0s=H{fV-v}$QhH9w|*q*?P?y!
zO8(q^1I#1ZW_J_EHDw+pEVlpUWB*~Qd)BQa`=$g0ycE3M&aHo@S7x{nJ5zgmdf$Q_
zoKN%^8nIw7Xj!cR^Jo~6*X>Th$+HPG^XY!;?&lsTiQ0iWrIiae<=I>WI^(^DiMoMu
zEG;-wX1G9J3Hs_`LvFwvv%jNv3ye+EJfddcib-zzq>f9ST2PuC56t*J15Nj8S4iuo
z@w*{h05f|okM~V&cTI~9FWR%kgN`GenkmkbomM=q^nrl!^X+)zZK(1YD24KKr47OB
zi8AF}q^(^t<+N)pI(kJOPcly3FRL;S?({O#)4M?n>-eti>8=PmT#WQAQ@`YDMF+H%
z$isxl#Yh3_b1fCR9Ty0ks_uW`NZM0Z?v28uq88S|!oscrMg;;Xcp-~VO-RB*h_i4o
zALh5S9MdQ~ad36c{{r3~|0a3F05<%@yHaz2rJ1*r1;;yzUX9kN2q8c;|E#_QvKofK
zOXiqb{V~_){=IXrz_QQE{m@F+FTD$Tn06qbjRJ8b1#k_^ei)Y?tJs;_e_p*?Dg%xk
zsgvot)|!o$E%$q6D2FC2BxfXs3Yj92{)09cBD
zccuIU{4ROw3Ry;)zPvdNy>*Sy()~5{ih6Xh&YA?6p056aZzwj9%!hzgF$*wyz(Mnd(+NC-c+AYa2DPr<;tkj^Qh!11H=)sMFO
z)-&mn-8Cu;Wx@1(pT>q87@!$E;XCi|U)2y_l0bSP!W
zPQztGFhLWMI05s`FT_8e-~+J4h+zajVirUo3W}z27OP-$1>#NT8j&K4Tlr(LroPgA
zL8dB9mKPezBrmW^sd+|_Qa|toF=;U5djt79$xaCr^~+#sY`T}E5yzUe+4WfB3}OtB
znX#IKPs$f|pEC64krZq>6IeYe=uD@U2}bjT4*N4o
zM{KRNa4nXb9eaP{yM*&ydPU(<1K$;T@zGK{7Rydqpge&^XfpPb8qbA;ib)&M*n8Yd
zNCTREqW>YN{QbIhf%re!^S8^4tt~~v0-Bjhf4EG~Lcgi!v_=({RN}rZk
zh5kL+_mHLOWLtGiB!S9LXWx+7UVTt%(9L`W7%E}^L2{B{=X^g1M$G(nu8@6%Vlk(Wcd>>*=Ba=
zYXaApQ_8n0>iA-r{Gl^@3TRwvwAlj2YrS8s!6h^t)3`#3$&7DzMpmlbTP9(%#Mf7e
z)oY3`rRNFR-IDG8WC5`{3IvPs+6+*VN@Kz+R3b=>+}iPI0SLN
z@xUsHjUlI?m96R~KTh7mcvpy3>z>R
zA(#4YNPnTX#I*R^92cXWYR_*!)7&IxI7U5JS%uox*@|e*_1=!J^R>kUI4+MEc7NM_
zS{>cmZZ=a9BwaaI|9O>TCxXbypfU7@%e<}RooU>hW=diUfmtFwG|*DJ=-&x
z>K0?LHuM_GtDZ
zrXkRc!S1hs0|`Fp_z9kM0SoR%2!&;NdHJZ5L_>0Na_P%{0I{F%=4WiNeGl879&0bw
zliyZ81L%myp&e202oQliJBVT-ZWs^CMJzGAE~k{weCH;$!O-yVCC)`drg??+itzox
z%A+N{4Lb^0HR?}#2<&3%0F)hFDa>o{@m0J=+
zmsEoh=u=Wl49nG{%>=!;QH!X)g
zdiZ8)N;gAI>F53d&Wz|T8{MOjVoc;n0J&S}YH>mxXB{+fH%pslPK1K@(h2~d;by%8
zH80uBe-3RJTDezHA9+;r1eV!vY$bFo)FikiUKA}+M%gnCr>_4jt=+UsD}?&^n4(_0
z?2epSC8$}~jZkvb-j-C&Tv>$*+V!!z8@X&9L|ht2hS>}c3k3at{;Nrx$3K*
z!5^_xQc^IDksMw)PYwUUU|tI5xxWo6ci3~}ObyL>gRM`~T97U$km10{6cMtkKO%=+
zq*cJNFG(pehSu?SgS@zkaItJe_?HB92;cw8_aK&1uPU_v^B=%s#Vq2p
zyRVh5qQZkFbPeG7O_OM^l2KVx2iP$0R%87iO9qqCEeZK-65S4
z(%sz+(t~oc
zp+d8!j*W1djDB3s9RD0xR}h5=p{j${HMybFH%?-Xj}@G;71kz`B3NrG;IH#Brx;HfFyD@I-M+XPyuft!
zFQLjI>bCY&n|n=QtWys53t^y(Y9H$ZtAk+Tjji;wO>Sr6K~v$;ct?
znUW*RTDwzlM&_@9FKWc+|8dbS&+`;*_cr%1l~WW4n^imJ%&*h
z_;Z%Ti1VNJ^e>yP_HLiYtJ1$q`kVg~;fHFGM9{JLGIf3A(s<3<7>>;W_P2YI<-3Fv
z(IV+b;mjkk#cNHX*gF@x6nO1O7Y!=<2e#-vmqq0&+DSO_zpEwOFdui+JQi8HmvF?{q{wc@d^2xMsB!wH<~{%J%Ik~Q->!i~!QU^csXt&hq&
zO4pLWYv|SBc$7ktxvLgdzullQ6b&2MrFZnw*3R2xRVSS0-iLFKpSN7$2GlI0Uw4
zHSi}qC%wsuF>DMdE$Bb*$5h60j}DrgHs05ou?tb}+zc#>sWwwa&hG^1t&S6=@hJXK
zT1bbV^!~v{H(P9Zu%X=Y=&D=$R~z@2+mzrPzz}A!(fz^xes@(a91AA8u%|yjIe;BW
zeennv2gmg}@gHOt#N-AMU##$+E5Q$?}_-v0nd^hI+1cBs^lh|NJLGFlWYQXGoA%pBF+e`Y%RaG}$Mc&_XCqbfq_v
zVZF9u1?%cG10v{|S$zJKl*bCek|iTfVL@bblvI;)wAjNM?rlOJMosp)5A-ny@$e3e
zLy070O!cKoTWvBiys9gxkaH?K(@I8(jz+~jRVcX}BhBhwyxJFVC=u*)K5-~su*#)z
zTrXcRsp&~6Cdvk_#lR87QXgN#7g>gyk_}$%!hN*QAEH{NZty7jjHD;&_V-mz;P3pu
z+Z@6}nMN-hlX>^JbCX@4hh$L_(0NeNLdn2Z90tU{|htA3v2_4)d<6}|zZ?52l!$i?RO(8XPIa`+n
zVOgvFz~C+?$g4pLQ$E12Ia~%m9JitkoBlT`0$eZ^Mr3Y3^<3yc%`QlNzhqhaqKNh`
z4E^1k?P0WCBJ2g%?bmGAcpIpna3tJX+Jlw{;}U&>QxL{KKL(CFF_c(*N%XNHw80-w
zkvnd9kX@0Qxi1+H2+6VfW+nf=Y^@o(RT2ICJVQt|+B}wL=?a%=V@c@)_opKZhrzAW
zlAZpXrkD3VwI2D>TMs!%TgmfkWIWRMvOG1zNj4FVas|Q-T%yrU9*9KrC|0t@6NTa{
z(CMNCG>~1be3gA^M&UGt36~w|sO|JRVDb7S
z3cYu_SjtV}r2|Z?2XY-DfDajz3xZiy#|Lo+?T=~IYW{`v4iSgF0!b-{GGD$VI-j*H
zZjaLB4?-3_jsL>G-o0$Sd%$oHc-nb4B|(yvoqai27>)HDbr6$5I`#a`Zn3UzKN2i{
z0@rhDU%^_rV4sIK_`LMCMVHC5X9^?>98MRvb4M+Kk!U17U>HgiUgGm`WkbZ{Tu?3W
zEdCV=3-K=6=*hIM6O#Ppy2yVFOs6Bnkr22CGNh)8$UZ@W#Xx|{WGe26)YIDOq32HM
zY`M~{JRdhJp_Y3;_|JXpXvv553-s2J3!4;d%-9t?p#R5op~xWT21Bs+VORC
ztfL9`?J%f`;q=sF1$xERH}nz7
z0TAnw%YP2h26UfHHMQo0=Z(!Js)%6ng(UWwXICfq)^&619a#-v#m9!la$3PUURF4U
zEj~YBLTygHoKCqqQU3v3eMR{dz{DWPm+Sod&xafN_v2PRld~t`IL{9eV4}-FBur@k
zZMT2$sQZnPzB}_H^@YaK*HDN$W0$78VVh{Z`s!T6FJ>46_vI^wi3_hNCFPhzoNFS*
z&n@jx_e9N5oC$VXmt!vr!lB`h2VR}0KU$C1R!$PXHKGI!wk
z7%9}c>N8zV@M~xuRP{T2K}9U+M4O9}W>I@Ye45V~7`1SuElb*m2&-kN^6+$2*C`$s)z*&=7cm~EfIJ=&G}0mg%0|OV^||t
z)X%9TkEDK1WOQhz81#-!8MKowB|StWHIvE6oJ`QGLr
zcA~T@=neag^$G|u`hpgqlU6p#diCE74s#dOp(fWplVWx9#vD>?
zdwHFChr)Yxl7oAe)s`k^(KPU9Ci!{!sP*(Ft@Pj<9ZU7^xvPin&75NOI%7Xq0*Kwi
zwi-i3i++a|OKwvD&P-l7f7Y>V;#Hti&AV|t4)jGHR2AAZ*(vhL2sCnfO{)>
zK4^DYT-`)^8)5%)C8XL)JpLQS+BfBdD=xo7CuZztaAmqw%mjS=#+5!8g{U!?Tz`?1)=fjakZnb_ZVL*1zspJUZ)Sf>Z%CFVq>nZ#knV#v$
zAx3H7%ahDp#gQy4GBFpyhR-mpj|P9RlOnneR8k451*&uuQ8hWbMozAkBOWIE{T!b4
ztnBwQ{Im|Aln!p*Z;*em_6G82oWc!uZ$e2DzL$Kbj{W_vU#%CEM25P8bDa
zEd^NDPGZ0yxsmTK9(jy#hJtK}u8XToL&L*v2en`eUF~^ZopYm}5o1YI<06+YX#p8h
z%~aNqVd|FusJ}t}!?q=hzi0weNnl!Qjf4>M87=+xY;`d`1Ijs@vIHH^hO@gRhX#Nj
z;q*T6LI1G;>p<}Em8GAy;BzT_wFhZ+5pK|{$0_2cLVPxnaH*4Tg-ZzrLmyt(mv=7fgn4FmyaPIfNHF
zzWg2>3$g?<4);L_unj{u7vU{R|HgBqk{`L&dj9jNmEb}9%fpG=yaz~B$Pw^n*D{0x
z;EZ{lm1#YpuGhbW;qP$&C!5&Cq{TIi5dC8--Dkh6hrOpGf)ELerolxQoD{*5YNjo8
zXc99?l_g}uh`9JGXrZE$%Jj}~MAqW;D~_NE_OEV?Rsb=`onA?%^q6){qO27VwVsR#
z;@!)Md>Yqxx4>g-!X`;Zn7qQ3KJwcG8;Sj{O1`&U#6?eQtPUuuOX$_SKlDi0^!
zYg|N!-aq3wOI27XQ+_W?utt&y+3m=0FT+GC97)Uhwjqijmxl5+%5)Nc7RI3v6ob;o
z73vU#ls3a@mPO{Wk-?o&xW>84WXjnNhuS2|k0$qhZ(%i8`$2P-&?=UMm2k2s5%44P
ztf&Z2CF#j{)GhMjn}e7xzK#C&M5y+^=A#N6GToxtrU&7ZmEEHsyDj$km!q(`%mGNj
zf(oQet5`D%?`^yVbf=FlEOLL0rQdS=40pFP*6q%jhx#HK{RNa@^94s9Ib^JNU;?$N
z3UR)8p|&I%-2Lj|*7U1!tRK8m4eI@OgK%YS_QS0X{E+=~h=v+;d7NhX)Uu^KZkTN@
zAuN`E9|}kn6kOmR>6gD!d+44yE91gCTiD0fX!AQ?xr$lC;Bzbz5f3+5d}Z~{!HDH-
zf{~}XQIX}N|64eE=Sd9y1MCkR$OTR-*h0pG>vTR16n_=becO_j{S#tDvx$ssG@yoX
zH&s`iC<`gWbkHd)7tK>tj}OO@iG0k6)UfPY*08Dv>jrzoS0~*047yVgRhmO@BKjm9
z%P!-83W-Ah!_}eIEsq(z5@V*8{&!EiftgM(Nl|GXRA)rW8Ug>5=0nehk7R;`%S$1Pigauf`fADq1lYj8atK
z>D?AquIh^osf(LAO7xkF;Q6}ykY!-=Wyf;F((+-*+W*3eRKRG-&|a=0@@oy?x6WKg
zK)04v3x|skDG0j?fldTLxMB#Nf9EntceJW5Le)Ld7s(HxFvu`B(T+QUgE=B$aA}=Q
zR>@Mcw69;4tkfjB%$-zMFfoxMrx-2Xp;hoKKf(WkFDAXkH!ae|^k1<@8r2J%u$(&)
z7*)Wi4C+wBGEYmKTvHo;4Qnj1+;F+8?W4fjZL_*xQd^gTL&*OoYjLZT9GbM*v
z>x+}?N8vyOVc#2jBM;9{<@o^}^-o|tMxoQkCj5mQ_2169Fk*ViqYIwDvjn&p!##O|
zleYar@bR<>wsOPFjEu!({Bs;Uv=~Aeu;QaZE^Uvx%H4MklKfyXgN?vg
zYnNH_j?B?A@JNkCPN6)Q1an>Zx%^*B^)F6{s-yiPm)FhGm@5Y3V*Lb?O}J4K_zC39
z`nO9j1AWP@iV}iPWOvpexU=H{tYAy}5i%>;k{(Y_86h|G4j8E?-s8KmS3Cf4hIEa#
zUzBY@E@7s(f_`^@vZ8;1v|iSP19L<4KA%Vzl7PFd?)+JxSsi^wKD~af$p35`c*z~e
z@#X!f(Z6e>gE3EVKYOGy*DJ%~jr+Frnih=>F>7
z+@b8-DIZLLN2Yyanv4A9F8&{nR`)dqWEP)FDGQHWyJpoF<6bqHWDOshQA{q&0G1CH
z!eaiNQ&TE+#Bv2>9_o1h7mypNe}93{>3O=8_}KLa!f4m|d0&$7xmO(Az5+Y%SG=u%
zLZHNr&6-tHq|}RUNVW%hCP3uo7R|BZipeEa=t>%oWITRdCR6&wPpnc^+oUS=
zmcmrCLAjYeZxbfsvP0yi#4HL0d!{0Zer~)BUxz>)-H2emzSiE0sXW|U&{SBmm~@0!
zD8jBo#x$t(Kx-u~I*Iy_wGheA{7sreo>20R-QFH~tLUv=4zll)w^y^o0@OGguWn%^
zj$c6#y$
zrdiZ-QQw-G8%J9iDGH{x1&XyUiPO*i(p$|$JC|R!EeD^eg063@jbg;Hc*!`{L_XN@
zCImE@e&Yz2zSADSB8bQsRUf8ZO8
zQMh(F!rOkX9^95Cnt-ONTM$$@M}^-zv4FB|p$d}0ihd=ovI{RZWI)V6>w3mN!9@ov
z?9zIj@mR0+Qol@Y^BNG))C_S2Y*vs2K6uG0IIKttC3X6q;-~Ksz>o{Fz6G?_
zKSY1>JX;9^i0V7A3x1itTQJMZ$jv2wC_@GWJOrAo>wNd{GXmcBhm`QcH2C!tA7^7t
z3|7qHF$t3anli$>j1CZy;YSBI8)y)nLl;qvT76JJM
zZ~ZtTA+rg5ftKu*U7e{biZmi|&OFMCC^6_w@G^3wK0vI-QiamS
zOnvv>f{1WhN~3zdV_lyt67;5;ZBDN`0W|klwy8bm=}HSj`Ub+G(p>t1jtig^^Xji@
zYtv)vDNsS|14M%tyt2bau=LsOBfoyY{hrsKzSod%vM+%R|9TS8Y))0XdTO%iz-T@KpV=C4|6j=P=BC4zylui0Ye3m
zBBW?@k-001>B1d0?<^-4p<{B;oTzM@iDH{k!f?1XQ(KaImLMe724&_PG;r=~vmZB^
z;ATnV4U#<9|G;!=qNpYy<3bmmeF>(ploTzPj!~}X2GRs@v6D%JvTowlYkskCC0cEZ
zF2;Kuza18MkBAfSKEjRV1ta>vuwT{zed2+>h;~NBmIvUD
zf6zV@2*tckNFtlrVMNHF>OU5f<*bh^O`(;gSfJIczGu4DR#Wj437xtQ=PMt=Z}X>;
z(TS;MLmA>^Qy(J7ue4sLzfkT)w$V^ydf$i5XH+u!Wx;ZmR&>ARjIzLdcZqGVG6BPS
zOR|+h(UV+*&|NGdCUu_eK{i{Ky>Up|8n>qPv
z!%_2%BtkG=!~CPoZ>8H-nG2@sTfX@euIN_7VaJtq*SN~~E*5Rv{2HW{_Y9drJ8iKT
zuSpTaQW(te@cov%PjC-{)ETC|oU8`|VF-Z@A#SvisUjwpj1h~pPk(Os+EE;)J#USqecC3AaPK6+b|)d`o%KdZ
z?WbMi1q$zpf?;?S=eD9V#|@Vbo%)t8=FB(Y%t*h7GKf(5nlz`bj~A$w%J&pD@NBmt
zJtvXP*KZ?T8!uYrq|S%a=&##T-zD(M4-
zW*zvJL+ITK^3ic&fFfUywCni8qI7;MLcE6|{dO68u$q$#4zA>^OMeEY&QLvJ`^`*b
zhD4D&Axb9Isc7efv&^Z-Twj!fxD8#Z2oEcI9+f?(gUHI~Q7HFji=FeB)s2~{@mM2I
z1$j<$%Xj*yf_gzAgy|qK3&F`$D8{{NvxzH=f48(1H!%mkhIs`<-6>@|F-+niR
z{(vTer@`9}@_qEAZmpx<@lVKM-+Hd{FFkmec{`M|sltghn$I?0p@{2j;5___J>tHbJ+63o(XtraAjj{PkXCT7)D>E~*_KS&$2@?})$P95?}+X)RvOHtn72ghLt0~l
zI&U8!Zl>U9*^TQu&%Lo*puY1e_-Jpwi;b`sd%XGd!>2DDjX=`SQX
z&ZXu&K~&FePW|G6iEzk+iNU|lM1|m|6NLEbA<3MkZ+#$;HX7{W1d#d$q30td)Xpb{
zy01kC0^YCI!TPN6dxG`CzB}OnqSrgS&s}un|5{%7VADI>7%fmrb7Uko77{y%
z?plDxdX0#kDkUA6Pz@yH&>+FX0jXy0mF7uQj5;$b7R*~RIC8RHJh2hpel*%t%Cz(#
z8pZCA*FAp7`Trgt{|dhhd;Xb`W6|;T3MXyP?d3i3;{VHGRU`2vUo50o$g!v{nQZuu
zv_BiqZ`c)*8&#nzY4YKGvsi2K4?r)JuSUq`7zF6}ej4g}sr#CDw
zZ*+_@1K5X*D^tvf+l=244A~9i8*1^mmd;QGx6PC)rpV%Otj=ou0gkP
z=}JhGt|CW~WpdSod8TFW_kA{~Y7K{$LaX4z?o%xyq4SNY$mC!;#03rm1)Q)wYTdsb
ztvW;Y=6nxss+_T3mq^3b7(L8A1~klN(p(7^S$4eApn@>6llNMysj(fj|5REb_h24{
zju`Tu=^Sy~|7E|nvwhJg#U`Bj82#aF4tmr~&>&{g#%$0nNq+`b$HLuC>P@z=(7vyO
zDvJiybfpYqX2k(vf!ec9*pI>GW~aE*+#7zKxM~g?u>YJ)F>>S`LD`PkmwX?11N&`n6Cs!*k1L9(GiCp&Gg$-bHXZVxt~UI-
zY0JpF{+>|JxmKXTUb5}*!h&uA6}F@96Vba|2C+p4hx8VM8^a+p&7q|WhEjI=ea`G%
z%5?j`Mf18UX!${KP}FD7Zz}!aD6<;Z+!Y?*#W;o8^u%4HvZhM)KXA$4Bh&XY&6}=V
zlX`oXOlR9{WN;B|$jVdJtgbc6;NU;*=b*olqUQS&saW&OxInb_qD^VFb!;*8T5?8j
zBZn^9uCKoG6YoZJWqj0Ehi0Q0ZUN2L4rt4Gs(k1TXd<&Kgz+OeSWwv?m)x9RSH%
z`#1P%%H^+{rZ?%iox7Tx&*Vea*&7pqe8Ds%nHL42xnEs}jHO4v#AjTP7N7eu926q>
z7k_*^*6mvK*}CL2NhtA!g7n7-{Jcnla8(}VNlvm}LC5q@CA~#iu;zMaXekTVpsWYIkZ12-zw?UW
z;@Zv;L;~e6Mm=TMd)I;M02&rNxc_w^wCI*6ko_KDpAo|=|B>T6+fZiEL$Xf2b4xuR
z3Y{aVedsWf5=1C4(8wqK><}V>zo00UT#9U06AOt|!WkK_7dtZ9tw7hrdI3M+MX49{
z<(w~0=z_BQX|H};_}{_Q*7?7ef3Z8mJ<%Y#S64*>4Q@~}?F(=2-CXP+-W!>eOcG+<
z=%gcF$?t<%hGVfyI{z%Wem7z;5UOx;huDz^fjn*2wR9Pk*57_&X=b1n9l=!+K_P5h
z+C*s9)quq8Tdc7>b88BQ3gu60u4!v7bP3sULwamBA))Z2N<{n2yxuGESC|%Y&!+{q!^KSLZ?Ofvll0>StmI^Jq-1MNPDjeopb0fv>=mVPmUyO+tkJ_eA&aN
zECHh8*O;nRWNf-*UH6k7lw{UI`6jQZzkXs8KZ0Xnkz>q~>~Ga@H+C&YuT92o#_7`)
zvY9VQ%k^O;Y`C#RfN6{>9#;DnyfmX2k$@=s%dx9vJHPqkt-21^jQ!Mnp7{E>tTbWp
zDVt?k1=g`$&A6d`$*uV?yk*W*BTds(Om665UMi1Wa`2>dHLs4$!g-Crz=ON3m~Pf@
zNi)rp3+)9qgLOj6ABT$R)J)arA3X1-u$GvSJQnwptle_lK%Am%Dt*k
z0cHRT0E9krw6VM}Ls-zb@Q5H6JFxsANLO;R(PM#sGXbvsSzzAxIe80m$-dv+QPQWf
zVMV?ABhrN`T{&zRHPg6?Z;AEuw+{btYyEpy`ACY0r50jWSon1*_cQjG3yvyz^?ZZ!
zhWcz#s&@Fs7C&5nF8;tbhmY>tVol8mey_b}J$>HEHq=rfMz1biR0&hY;`;aCSH&?(
zEjx5n)3|pI3hDzGt>F0gjkYgFx)^mFoe?nEI{SdxR{DGbG>JN4)5PMl>)2bZM#cF!
zO&Zv1mk;TqI-R*U)l5bmj|93r4iE6mNR9SKMEH*~6yn+4A$7vW^z=;_EMO9+iEk2}
zSo^=nOUXzP3y}$GLDo$VlZ>|HQ!*wOfmKe-TJtY^P{}R7EsvI|Jx+S4h^|3~gBIhk
zH3e1oU(kZw7BJ41N%w#3`4mg}|5|s8Y^LNe4pTeg-B7q7xX$xS`>b}3fuXEsa-V`wki2)@v
zJAbwYzc68L;Ur)s@gUEATbW8Hvx6Ob5x!`1kU(B0Q1{AU2dwhX|rE8EjEDk
zf%Q-3@z$^r;$x#bFF&l)_7%qgja99TdhffJKN0O0fUOY*m?j?Qf@wD-
z&pVUD>JKo%z*;L!`hn5-d301pUS56^QeOsPeCPe^IX~um8M|p~gFI{s*g0zlqv<
zYoGmXua(r)>2=t>s1hR(l%iIo(2%a8o{uEWzU@0al&I=q$wBFuTt*6b3;jl-WuJ`K
z(6S~_bU$d*7(Np~s%bfo8LT-qr)w6`-xH$s*XAF}pHYZ8F)w}I^$|SgdPYPGhsb23
zN->bWvxDw0q%iUj(k~}C_3@yA#@{|mhu_R!#xQLDG3Uim5Ki~a0zE!aw?n24KR&A|
zuEXH?XzYh(ijVi^`T~TQ*Ro8u3Y{lb1~(Z_A3823gf#_ZW`^z2El-Errd;Z%4~E%=
z7>9CA644U~(3!Vk3`Rd(*b?geB0(y2l2p7{7Q~K(4PskUat<+A`M63x8{c1wK1dQS
z{!c8`u0FAD79leE1h?8m;qu3!`>(n3DHk7u2?6mq6*}8-&EIh}!(#FSvaiLXTxN)p
z*r{PYirR30jPcJw7>32cean$x2j{6!*_rN$Qo$T8-YZQY`KX~A9mbs97i(~4Ls_)*
zm8rpHsu^!3&cyQjk5;413t=~dy3or-7$|NV`Ro?jRrP*%qbh;%IPAAUE^&LQZ?tuE
zL|b%Kria4hk^MZU@r=(|=)qZl3d4@$5P=fQ=Lf2?4ki
z!6X7+XrPpF2-0>4PV(aO3l>R0MPmp*ZJ|OsuXwrd(@lXN4ED>I83%=vdkmP04l9pO
zg6G~-S#!x+#w{SQ09tS69lSz9?Yu3I%QhS+aAFg<+x+C8+y(JOWG!}<9r)>px5h^4
z)~0=9k#ZRK-o;fx5o6^;czQYzo=SCM3jGB>ov-V}V!ZIn-OsS8fu#zwS_wFB))?uJ
zAQP6*D-+9`!W0rWNqP=v7Y1-;X02`kQxw#A#jkjaDTMPaxRwPwIPw
z(*NB5H+N9M+}XX5G6b;(f(Q5-AD%(&uIn3GK`vab#c>j}*ed&ucxGlA%AQ3e@k;
zfy24{OGii4Z27_8)25z&Q1OKc`X&Pz>bf6@@76Y+1OF=nkCLFOVcv-_;Y}Sj}@TzGXBoRQ_!~3C$?BFUBDP7QztPs#tZYUXlvN+42^rVD{kCdoPk
zex3y-YwGE3{p!=)z${NnA8)ekcEam3d+HWXVh(S7sL^J#%23KZR|R&hp3LO$>fZYi
z7sSaO=KCX)DdQ;Hh)7$^sIU&ueLeecLdU%g;eFh2bE+w2
z^H20;z6*3Ko#VNfO<#nSP;WoZ`d>AK6QK$0W^aFayHfqAm%7SBu4~&|TatesU)A9@
z?_jbPXUJxpdAt8n`}Rv<0&~I{UjEi9fm_CJI78uSl2U5zB#*(#k%5qnhR}6EoaZdb
zWUf?3VSJC6SY(-cJp5mm7vJe|>~Qhh1rhSvYsoj_JFa=l!{YE)zNHt$Jx4?9sN}1u
zwb~a-qN^Z_B&jF-L8=;{NHC>3zUDi=q+#RckG^kDu9%uCG|Z2%=6t#qjagGM{VO8-
z-05;+IK<1N1aw>~m2XX~t?>p12Ji?7K9uUdI*62$OAD<{mFj*y_4WJQC(~I!`G4cr
z3*_*%PMS@(K5x&dRXc8!@9Uz|*7Li`oSx^9*oQ^GVxy3}-^xi`{x_nli{!~Px*Rxo
z!{mNOBA5L*;&5xY{_tK(vu7biC3Q9T-)l5?A-mGiiz-{`FYZoKqu=)#5jc^U67f=*
z3MV5)PMJp9V{<|u_Cksd|5#H9N*6w|wZ>h`)1AHg
zHASpQHmM17R{|e<%Yu@1kt+CqAo`sMfO(mF%kUK?nJP0fid0(-K%zD!|I^ZLm%qNZ1
zHdi4v-|!aBx=5{*zANj{5|sM?{K;6Qt1|qjh%y(0%2a#OZtS~{q{d)fa&ofZ
z&%xn>x@@1|h6XnK)z%6AMLX>OB0XIxxBnB6{m9SBVZ54YvC8*vJrfWKTo47S0Ru9<
zHcZ~u!A-M<$>rK2;G_d29T&hNW<9oFL~}d=0r-xkxfRIxUqjj#(ndxhLw+R4)ES2J2
z?3>~lwm+f#7<@5m)u9O(Mbjc(Y!MHB#Mj}yH7OYuM8^8)P4olCsbpWw5#|f&!z03b
zy)<&{xd!#%07^O$0uQR-Tl)>IQi@0s4Gbn(*wtY
zsP45`405t&BR54$R<@2}T5U*2v}(kG^-4w)Z96qZYouoWy%~LuF3g77dv%MSFN484
zmWK<-4OM(J^CIce5~Adt(_t04Np=<%OSE#rN_i!h%0jML99411FZ&Cy-MOX~7Tsp#
zV)2$XS!7CDS=1ku;EfuO;Yvbc
znV0Ai_y3fJEn`pSrW<@uYY7?4jeHm{rhz=KpxsAaPGk?
zh_QnD=*icqcJg;zLGI07D~GnKui2&Fo84yZzM7(MYyo-(=ge@rOx%8_ZO{6&+I@;=
zrc*qhzf>ISWS?*1TTgNp@g<>@lKowEgUZV5OEfup?>d@Oen)8$_{v(*7|&Ryq*-4i
znJJdYZ2@;~@yXJeCD_eFV@1!+>rvBlf3VowgmLje
z8JXRzl3EK||I!wUWr^NAQCZsM-L|o*8bODiFO<$wL6$d9bMMPj_*6NMFsTJ^l}vvv
zm(|5Ukrqti99V(hKTjC3`6>xeg`D%Pzr&{2aoskho(}XtRcPUX|ImoJe_v>Z4`#f!v`)s
z+AE3`i~8atMrtBZ&J}ni6*#BXuY-8GP*NH^v-K`)=*Qii7uk%o@Spu`C!DcG)W+Qo
zj}xT5li@eaX?!~5ZzNVr1~*OHr;eyPKDjEi^GsDl39&r>MJAf_P^*XZ5Mq(4_e^Eg~PK}oaf>wMz2
zjc~r&6*lb_swYM@n9D>`6IYQyPF-=4JCtS`^N(plk9qqKV^Z)LDW_>)w}z+#FV!5u
zDu*o`Jc)>M0$;qfdtS(&_KaSjKhJgV+7ur#lc^5d~%_8>BE2`rQ9~
z&+CAnrProB*Aj86WhL9|dEy{=Y(tE4szB&TI8l8AY8X+Pu9p?APiqH`AGd{jQtDzQ
zOuOsCFQ?H5SwyeHPV6-9(AyS<%L^jT4y
z>|2~$=FQU~j#a>$Rq0E0AZSz+G;E=u6YJ-tPldDhlFl}2IMSeW!k+lH&J*|WOC^VA
zK}3PyOLnv`2`*Lkm5eI1T$OYcy{$qJ@3r2GR&VjpsJg(ny=+>2GDil>9H
zP!PyYS4I-hg8E!!jT{T$tfXqTHnT>?lf{#Bt1FOE3UxK|OH?J2)O&d;8jXv~(`t2!
z24@CH>*D$5M~mZ8RKLqI8ln}IDW%%;extH4>o8~N*WQg0c`*BqW`bg&DlO`_V6OeG
z%?#?RPq&=wlSO1vsG7`TBndumMF^$vY4nebd`Ub*da5!c|Ga)boe+eU4nzf_1w=tx
zTDH2?1cDfp><&##T?U#AxWVM{MudP5+vhxs);=c
z&1P#g$MOt6UpsT{)eV1!-`&pujVl3ZaAq=nCat64?8zB{wVn`%LmETaLi`S
z!wBPnpNf-#m+yh2rA0lH(tM7L6!Cv-uvtt7*E0Uh`;Z^#+VW<#5Irn9`wO6nemDO=8m-B
zP!1m~J#c$jjcte5_!`r=qSq(*UoC#5SzuN=?F_|cWMo)pdvU*iKaWxA`g3Bu
zU&&9OTLz=V5HXl
z2=s}>QWsTl>Brpq)k}Vb$1T|9%!+?1R>ki-{FBLJ@%=j<{~ou?&Cx%MYRSphO?A$_
zQP>QB?0Rmh@!koef;L%1kPZLB4}xL?gFtAFP3O(Z38M!UCZ;veN(XrTZ-eebN>5=9)nW5AqC$6=x>2s3t
z6l}*>L+!HH+Y@AV{*1&Rkbi2g;2pxAZ9T{*83bJ#iF}>BQ)o`wCrza?~k@
zIP+#QRg_-~LrHUwh*ntvuSy1nℜzgJqY<-2ZK?cQ`!kKBtVH^xl=08FGw^bVUWL
zfQ)i2r8Ez0LB_bimT@GxWHh{366?4Y*4c*mfeQM^67aAM4!0KJg5!A|E+xsz*C8uK
zo4&HOG@pU_2ESa@ZI)+1)(3U)Z*qoF|K8{yJq_>A#ZH#?>Ahs_pHEB-^F-@nm@JQ5
zvnhW(O-B}RHWCYGCkRRl7fO_p%Uk*Uez*lmL=lQ-mEG8MT0=AJi>OpjSIr4TZ8}pr
z%d>5{T;v{;KhjEH6B9OYsyyo^WT#I~Oy`oZ1Tr-8GxZQ!mx+-Po9$3?A?~|pq+KBm
zMKaN=YUUA~kuzX48yR+_upS#HsV7ouKox^6*-%T|8XMZ@?RC4(9O0$)oCc-!^>{LC
zVX&H|;qKetun|9p($ot_QIXRQ^()^Ns!od^Y>}xmXi%mi@hFKV`p*dNupg2CC~Y&(
z>`Z+az8G4-@SeK*=&l=c=wZ-7Elh4beHa&W9WTGZqDviHLnuTuV3oJb-{hRQB88>%
zczub!ETa-Op;Pd)F1utcHt6bBf_kO#-f3S*wya(vI%WB?Nr?So2Bs{ZwKluimr4;%
z`Yu%2gIQGR(;Ri|tLq)s66O61X{Ob@)Y4lz(WW|uwE<3QQI3)*cn8B9s~Lj7;k_pl
z#C)~U)HM_$n8c~p`v|ptK4}=@ujn{(ht`_loM$90V~|{9(_~hYNls)gR^d;m{yav<
z+`g+^ke;mb*`KxY-_i-6|9*u!OD*ps?rGy2+RZFK=OE?B#jf$snWK*MlL-3hJh5lE
z^s%`8;nc=>QX>@&%UbhCR{tCU*zstJ!pUO63-2XQWBc=m9&&}`!@cI%YmXs345-U%
zZzBE&c+I%CWHLO**FIk6c0XKO;?9nL8I>7jSp%rmJ!}A(%%HRQpgBcKddOiIeemov*2MHyoAioO}6BAO;
z@H#7W_6=mz{dUV|z>+8zkHmQXAdH~F(siXe^#jjbUP)=k?A@bwA+0{R=U8ZHO@6Y#
z*rQ|o1cGvi-kZcm-L#?b9eOV=L)vP<7hsI8qm#QLcqZ?Eiv-~~M0YB&RX6fqj`CZX
zZXrR5tKLwX_XqhbDfRCX2cb7lo2Gut@q%%M2`=A*r1-23o7pl^-vI!Rn)vweC1)6d
zz%Zc5Z=q^Bj3!GX7`rUQIQzd5_a;W8!RLFu@z)2<8bcR+&3KZ*Y?{3OU%xnQn|gXj
zAJ{l|iSQ`*Phb#t@JMtbLu*BoI65N1@WAB
zw%oNCeS-w;3-=nm>;(?T2e{)w;LY+tALxI3J$|@T0+)<=KF4%VLp@Ixo&HbQH6)L3
za|%&HDGoC;
zF2pq@gJsLsy
z$@Z4|P}eUOtwb79bg$G)TsFN_U-|x6{o;B}01{Wils$^BIX@NKnG#=bxqoQCQp*UD;dHq}BIH6iCkDSR$nr5Qeh46FxNZD@{R23T{
zskL)!jsHi~TL(qmfa~8VAxL+3cgK{&4icKPl#LcQm%}Hartu<&TGz`rKNu|qk`|OB*zyK$sXIU_wX{5$YH!a8kld78
zS6w<+E28VuR%O~pB)=~z-*GqG9N!?LM8>|i9h(~=iGL=gNDAFgZj}9y7$S=E^{|*2
z6KyhCx~KO?E=3F4({)dwGOkya+c89l=*n2XND_#{iLj8>7)3(
zH_Au}wf5m^}sudGu@REA^Cla{|+Bg`@h&F0u@K)q$w^Ek|3X|g&i6=|tsOTdyTHc&7F1Ccy
z36Ts+D_9*2`wd`H*&;2Wu<$LP8yjU+)n7n-CbT$qeUSq|DZ!GP*>mh9fl4Ko7$q9Or8oDrfgMx`i~RH~0@NGEtI}9py%wFo+SMK^T1$
zOzo%+AWH6GLUTY^nq|LAfA{7`1Teo!J>r4;XreNh8dJZmL%J9l8Bx#C|J|zn!&_^2
zN!WxqNF5;
zJEj`Lq1c)w7wL$E3nqxiHADQ-r51nRm}*~5Lptz)DN72Ok0dS$WcxdGEOB+nx^6xboYEza;YFBffk-M~Lrx
zcrU9HTWnA#g^U|FYTbJGrgo0C)u~XH`xYwkjeG_1?kMfCX#M+|vujFgKIi@(F|$bf
zr1;kp(mB>#<5*nJd@@eU6sf_$Ep92yJC2gm$8@Gy6~5DdQ6DdbxzcnQs_M{a1(x=JT|xKHZT83`GdMO12IX@G*Ht=@6T5r+
zWd8^TSyWR@NhoV_b<`N^`9n&uaDk<4K`aD!af}1eD?r)G9P%uBpdOauYd8RUeyJJ#
zj_EFO3N|{(ejq=gzT?jMWFw1EwrD7m`BFViKKevx?2a)4O0PGzOp34<$u=a5SHCzJ
z)4bygb+)86$4iM0X1{H9{iinFL3A$U>Jad_RPPW)l~41b>qbIN;YHW(98}qJ$6S$m
zt2euo(W&9`KmqcGt1Ye*qiRDba3=sg@xT==}VMWJouBnuXe_n^vI#|T4_
zI)Q;0dvJbOIDPYW1|w^6s&z&|r-Y0kQ-^bb*pXM8>7lKfdEuOGY6lJ_SJBqR^U548$(MTZ7JTmWEe(xltKT|4Z!kO=90p
ztH4m}CZ4&a;iVH}XxK3?UQ$2i$#(WQ4R5b)X-pe#o{77eb~$8iGXRN`$M7hVLW_4I
zN-SN+W}R>LTRO|?al?l4Ry|oxgjK5kiv$y`nC?Nvm~LU*DY3nlhwP#U)j?{$)|NMx
zEL59kSChGMs8soLNGM|^2KwpM#(yQ$c*h+kBmMyp*$AB>J8*Kk=J&glKLbgbvND3r
ze+^^pGzEXuvTu;g1qgj-5YkP4|G6wWI{qS8yE=Q%=uL%-@f)_JazdHI+J}}LDa((X
z&_&K!>%Dp{Up>`D_sRuZHElbG_RefZ)L8L|x6XL<=xgFx{^Ia_9fIvkv-zn5A9K88{0OI
z>_Z#Yac2(CBr0Y{uibR*Ku6>oO(Ff*jwY2S5-8{Hfxq&;3-TQfxN5@OKuyxKQKP78
zzqA=Mz$cG-H>B#*T^286%2u8r=wY~5Q|QIbSQ=>M#XE$
zlCr}pOxPf>du^Lx0qPAfK*7NQ2|2)bZ2GVqi7*Hv0O`3<{&iFEvQp5>ehJ%LiKqdu
z^B4o8dre$)cO9Y9l$V#g{@#{!Tx}Wa5(o5QfQYP>-of1+2WUc^_VuIxx25>sO*1!>
z{9mTzB8}MXy6Dok=6PL2O7F*FLHKogjc@y|Su-p)I$J;g=?cORx|4Vf`n$r}8xaHK
zczxfNilTdCqox_b&m<>;C8u)EC%T1|7)xu8CXFBefb{#|Dq|+`AG&RPZj5DXbK}Qb
zWjY0%xJPBLSnQJH9tWN>SE%P!;_(iM9#jO*CX~V=gQp;&c*MnO-
zUPMnv?wK}_Y@S@s+TP+46{RevcONkgkz&7=UuN2PnFHiKTW^AYKacq7H)NT41Qs?6
zAFg_&EWTfz%&De^5?z4Zl1eY`lRX$pRa4VKcaBIoEj{l@v?FvwcrmC$>#Dr+%P@w%
zXf@3(cF4LX$8RQNGMyVu**#U{iP(#0Gun*dF$-<2_kMLS5jgpgw_B3+fo!5yyA+wg
zBvPojBH%v{ufm@Zs$)BsnQR_g3F)Z?Bk4}LX;aJdYeQaK3zVuBo@35oN
zGEI~yydL%x)weOQ9O)Kdh35O%EehPfkTu>C@1}3Mm{OY3?l$Frh$r_zGh=(Rivpmw
zN>hml-fGIr^HGtY&!0vku%Dpwg;GliQh~{fi)f{l-bk)+D{@;LZC`q;Uwm2@b2$)8K@*f+n@@{~oN3TIDAY_L(OY9G5j&?CL&xsgkl2=U|C
zySuCH%@m1fuTqnH$wj`}Lwvt4=0Y>pRe}Yb&bNb-p?I?^-{Uq4klr%J=r>dcFmZ!VDLW~7X+K$~Q)ElDMcF6#Ut)bsS+
zr}8(*?yBYr1-H+kxk;J4Z|bMtRR-E%GDJ2VtPfOLLaP!O-hY*J$KTK&*wanpO3E&3
zKs`R2h9ezY#!7Z_KNvSxe=?S?Z}sBuL>9b2Y~E2(N*
zb6KN^eV?A(?1K=TEBo|8Mmn7Hni2`A0#|>cU0V~qyGV1V0NuHyPyKD}e9b$1HQqG?
zO(ARupZ;nK{dsZQI20VDLy3eT=k6Ww<+z#iCRy5kY~SX~Y9g8@7cz&2IZK|fTY8my
zR_}DyZE<|fHJ!anrH5MOMobhFJKa}MjOSsYa-X(3%YeXcW{%7D&=?5|3hJ=dp2EQT
z9ar5`=GNZ6%mE3}vE@bXEG?Hbf0J5tI3sI!y4s%gCZ+F2D#d2ZlqFj&A6&WVkNE|k
zo{`c+8m6dedg0;aWui8ht$3JY_wVbUz63G9d%z|OAHeMQ@#A~Sr{xhCgjSB0lFdf+
zc1C&`knu~^z-rN(0YG}&Im6zTx%Ctw9okMn;rDlpzh_fq|W?X{!-J_D@rR`$bBfrp<~AT`1^g+
zu$hRfC=!D9a}=UqSUO3k;0nkBeg#}U9e!RD3Fo@IIpk_OpybxB+Sy~Hl8Wcig2a2B
z59z58;S6&b5(C1-3^rIdg)!ZJL6L!*;w!-iSFMrO=V3EYhPRx>5)`OxV
z2McH(Nd5`PQFxo|a)dBl(J^9F*VoTKebBHYwoF_tnG>cy`7M;3oFr|*gRN4cTCr?m
z#)6aJ$wOB~i9Uz3po`nl^C(%7mb5pY0|2W`jtFR`J{yx{5o;r)3>s`XNL+wM*Rt&*
z%uW)CzBYn;X8&u|{DO5ObbS*vWVjXhCV8jY3r)4=fGnMTfG=rlcAGI@=AYn=6~ge8
z)OxJFtsD&pCyw(`MP?0HJ@POZtszU(he!FWqIq|sZ@rq-da?l|Gl|)n_uP2xeitKT
zBuINCe7JLe8)=fUITL$O6n}Q0b|gANdZKbEo=34R7yJe4_*5-9Pne+%8X+7
z&UnLxCQSplc|~vK3t8ww9hrDq?EXUrdlSTkKeCbl;fZ$hhty#+barp3!yfAzR^(?B
zZ8d+$s4(?>^q(?FI6THnNx6HDWOul+
zTJy8%=55NO%NiK*Nyo_hfiv0dF!~JUWHq*|siIf5%p8H8m@UZ?Twr}ffS2-Gl8;c
zDcE#z|CkNsfke2Ib9TX>FkH0u#`WNOHlruYZX|XuO|zIfv)6@&I46TR8H8(-d))o@
zzRZ$&z>VwQOonmdLeuHAEE3)Ow`ls>YYw)ahspIf2u^w;`|
z+X6(Z6teW;(pCQ)`RPrMu;s;D>__)ECGY4JNqeS5WwUXr8r2@FdP}ZJ#4>$)6MH%E
z$MgqeM3$1vP8!}S-=UYw&!Ux!5er!g1wO0%x7cX$6ZzyWyyvih_K{s#VXPkAF5i>R
zF5xJbdug^O6x(r{F{@E-*vER&BZRt4sng`O)$y&+n6<*%w`m^xxh~0+8<~VGOVAo7OwPD9b@}j`r2|nsgmat
z5m9Y7Nih;-*#M6tk3-?+dxw2zO;4gZvR8(2g0{v&_f^ddfV)*N(F_V_lM>q;wIyw(
zlGCOezT5es|9vhnwZbBn)GVBwJtym5=s@L4HZP;Iu?TX(%+Pv#|S&K+7u2-RaJ+Q*ZM1Lq%@*2M5)W6aI?bRITZu6Rm0W
z0)AVwyX?8fk5k8bKI2tH#;r&ttZMzyX%%Kxp7=VodSN?BX~0?F%gzqAc7&WdIf-nU
zaIB&=a@sgnvLZMjdFToFe8g1}$C#@golsOQ)08X|4psCbS|~CHp3f!96S9+}ftFA^
zG#Z_z$KClJcy*0O;LW5}dN$Ltw|M7yO9TU%qC(DB;_}oY)jwq-4zKC$Ovj>0!z3JryffI
zxGGe$=@sQ36(r$|*`0I}-{j@|TkspFxo`2`aN?mKE@79~tUX5a
z0h4Gva+=;Zq}5^R7Z6R{4+IH>93R_1FPOgI!*I)Ow0$knFl6a4sSDd#sgT`YU{$)|
z##asWD(jweD!?qX6$jJ=k&iimhl&j3jsP6Ua7_HKzyQz$Zd&}waf`+~L$90Z5&9?;}taCZNG3!_mblavd14;p>22MbR`4u1{|B)|&aB*{M*ws=1
z?*$m}#AZf|O!rOz1t^Sjf;zp`^8=7KxnH`hwp7Iq85UPCEu|`9f|tBsq9?nIrnLo}
zR<-SVIvkcbVAXT;@eI~+t?iovg7x@DRfBOvnt1G}mpLQrpZ!iE`
zUR^yC<$r20Z0EQe_zIZAfRalAxNQf405n|Dm)p6|XN@CB7S6Aua^jkHPQU+eMTQF3
zx@p(W0Y+Y;h8^)HQ2wux1JQUo3!C`i-<&B-P84kKow1Ri@23vq?tW*O#lkTn8Ih0j
zH3kK{BhjHqhw<_`4&kccVgIOcdBq*p5d)Ur*l@PN9vIXzYZ^4l+L_P39W9U1kAwKc
zI-vC`zu{TzNf}P=CF}8qSl%t4c|}|Gr2H@dVS3<+=mxS1#3Me^RYaCOJN)&D+iSW^C+}aBik44K7>=qlX%F%zOcx?&aa+?Tn4Jdw3&3Riy{Jr60L=Ql+
zopkC6iTSk>Shdq_VX~eqA0Mh4%;KY?sFfW$qfroM6OCL&#HFc7j^0@$FF7C+Mv(p&
zJ~DXQqIXX6BOc8i6thSh!S+sv=t+|>J(IB8)hqARFPq#V@Mc?u@EF+@+mL!gP}u3T
zmS0LxeeYpO!Cc?NL1IKY3X+ulZ#7f-ru=ZZ$EG+2vb&QCtzTb$S#B0@$VZATy;Q0u9M
z$S~Zqmg`tKPlKJz@bKZRq?fi>b*;Cp95Xu=X=QNb5gU0e;oeD0Cy?`gJj+7fK+AdM
zc%+L4?JL#?q63;Ui*&PY2HOiJ^t2`!OPR3ehpsH
z1*2V%U?{fnn4G|qCl?T#S!sIA?5CB}?$wv@O
ztjYInDA(Iw+#rOK2Q5?YUxGbC{cp*
zHXirwdbMX25UPJ3L>dhxrC0#VE+9*+Z%wlY-7TzqqsLG1qUVo^KajG;R6FQCVsYv*
z@di8=2H>B>7~lWFeff+sq89WxC-pFXq(#q^4OWBr4>KbpQPoW1R5-6i+W>o2W3kE{$lIZ?X{(PNYM+MI^39F0Xd6~n{5wzc={2=}E!+`=C^EX~pg)iXFPoSn(57t9mz42Nc`
z`EmHP=*T(8!QrS}UkT&GB$HyxW>j;ByEIUcXc0dC#5(-)smmjsPc?HKq_)bYs&=SZ
zX|!AlnGiiR`Z+5QYIhKd6#+7&J*_%iijsZ0&%LR%?D!z(bJ+AlIjerIMaYR0Wh;z}
zPTu$aJrUw+nGyDZRZ^!7ot|*ofM2jP^X}Jpqm<>4`)(4s&zN`v@r=m)p^?$O+WH
zUcDtPo_~}#;u_k~puUe#QMUZ>^5)LB_E6AO+7wM`1lIz_IVyqB)Z7!=PD`djJaSs%
zp3+gMS&>B?x}K9Z6b}6GX(2xViBeb;ppfo}&Yb26=vt!n~U!ei<1
zyKAt$AD^G|Jt}+Ee^Yh;EdQYFyn>6XW?V7ckfYQQnFQ8LSW4Kf^O_9bIU2JPuZadxc_C_8HH4MHAU~$TO)SBg0t^hBv>!HfzRk@};-(n^ayXhcz;C
z8Azox`lKtOUn-Z}&i-=O>T!qADNuzX9PXULZKuRoa)O(Vwd0QNh){&txAbY&)ex9z
z`_2nv8JabUbH+bv#tJ6z7}o}>=5#(9Bz81SG3q1`UV$SQk4=DW8YVa3=T-p>bEu|&
z-@+O^Im!S$z>9ereFp5~U>O|XMX6ta^*(HiK>`Gdf-#@zE22ef4O**9p6S!Yqh&{6
zIeJUhD(RF!?O1~}15Qy2JE$R^+a4dR*@IORAWm5gw4WM(&54NDx&zAwv{q5{ga?mN
z0n%^nPx!|;(XtJGK%!$;tpW*n1UT{KuK~|sv&jv(UkUpkG8AAtY6r$^rqoN|LBMWA
zpH4H*zYhf`b(BzIf6wqkc!FIl!>(uB9)M-ymiH?FS?N&kppNd{I|tB$8oJ}t(z5|C
z7Ib9yAAb&j|q1^&~+ULe?oe`?f@{e7)K6km<
zIVjmbPPpsZrK|IL{`!YBwnMlK%2QL}w#R$#(Hk(IgW0)-b!d$1HEUI3jrC4aB1CbM
zah3Lm5TNz`F*?e1;xTeHo|MrN;G-(O&~Qi!l`Um%dqC`-?3V6ylU
z*0L*=wxQO=))J?bdc@B*D|wXQIYf4|7Mug@C;+2UH!y)3fW7Ph%MdjhI;N$a711E8
zEEhmq2-mIsF&eEATTArwH}FU7umtOoOHN?eYpfYCC`(Jjtcg-ymC=f=(TZijFLxrH
zFZw~FvQa>2Rm`o#{tph|7nQ=dBV7+oOI=xRLgbX2yZVFt-ddBT2&TF*A*$wm{7OTZ
zEJ}*{K;6ijNi0GKzoaCBnbWt}(}Wu%NxWDjh}qVANv;kom-}qyCiUWP`yazbo9HZ@
zFhw}ZQsgEIy?KW$o(80LtXx&)M~kSLMp70Ph8tE!`;;g4L+PC4n`AO(I9l?E;fX#`
zmQ*C=xfa+dM>y%(dc@KEo
zg`ZU2j*lHnh;^(rAWh3HA7Hf
z6R%fv<-7gk4PaTj{jX&ID@{DG6W5C2xLz*!_fSAM-u~&*1P1*yUbOMCoJePPI$Qzr
zk3fs>umH7!Mv*^^?K@|!()rcvbP<2N9f+aG?_)cJWo{PvgU1Bu(;1?ZQDI@w91c)J
zBiDY!uxl?VJN8C@^?K$E4vy~M15yvL1-rG1KiWxWl_}6wvEGIZ0#VX{gKiLRm9a7b
zu-O1nL@r&EM8Hm8oyl~zqx=IIK<>ptsTnn?H@^K{;B%Qj(JLZBVt!zk
z7Y*4+kH?jv%;_g{Q@@Zvo2d6$^w*Q!h%M#6(;|sLIcfK7*)tq0cnVR**BKeKG+U_E
z2C<6bZobcgs8Bqd_h+8N^`Qx-Sh=B
zqnx9a_c;6?9WVY1A(A?(IknAP
zl`tzJMkAAoYQ=ro+95a@N<5WQj`L3zZ*HHzxGeV0tR9@X>Fwc~U}E$7%=55gTqLTW
zjM0uTX&)PHG(9TGupZwiS9)??WqZAot(SEl*s=
zIyyOcY-f**tuM+$w@m40my$|8gjy4nMnMQ<a{N*fm+~T8h?o<>w#oEEpu_0;F0|
z4^d8TJTmjd;uXVAy;BQklX$1Yrd%vk|0(O^*Nzt*Ccc6igCKU-N4Gpo8M$sDNg0aJ
z*LVMnL^gi+*XHsz82Xe;VwJ{uvsh&ORv-6^M?9u|rw2CGUR8q4Z17)TE<9jfgGF7y
z9Bg{35;P&V!cyuQhrO^x7EH;|TWq%%LNlAAD1;Xm{e5{<6404Dh)fR+A;2PuDeRiN
z-2h+){`eo@kmseJf|FOwRCNzX8Ayp3-J)k_X9qkiI8D<%{{G$>IPLgJl++;Oq7iDu
z#m{d&W*PGah{QB0@&_2_`cS&)K$&u54A>Hj5WED}Lsz-@u@(I1I?7}_+o7I0^%CZQ
zDXRPi1M)0E2040ep;8aNJtiEhE?fUz|9sL<;p^hea}TIjOko%KuNU_DlKp3ZDQ8O0Yq(0qJHVT*XFY)dS^xB_8j^n$^c$U#PDY%
z#0TNv?8&~Ol})oX>e{iGcFpsk^Fa^VM9Wn(8Xj&`dnsALuaSa2n>AjtKhv$#OA98|
zzu2|n(hO45C3jjK-IcMigI68DC4bj^=8|j8?2{|>n)K+SQ4(cY81>#hv}~dbBfQu6
zS0@}B)5IR3Ce02SYdFj2n^xM)f;@0CR)jo?lB+;~_-uSnvi;KMT`jv_9=YUb&^k%S
zf!b#Y#kaRm8%4xguU|5CGx$m6mOEb8D0y>+QYkIkf#dpmrf8K{_Y6w*NA*<6`GK*t
z4TmkHDhhyn{KNaZa`8&;0yV?@Ypr(3AVWzbZ)Bdzzq`FX;m&d+%YbVnXwL{O!e>EI
zQybHIdrGJao!2(Q`_jku*X2M)+CNdIK6=Iqz3MNbJN=A~?cIP&RBY~8l!0wm05vES
zGTw+G8&FWi9@{gZMls+y&(WU=xh{dQD28tt&NAW^qQ)g@TkOl`d=KlWB5!p+mAtT~
zM?FkjFFyJ#2N9amuveh2aXBQSvRwYOVA9f%MOmvLVQ1+5?xBl;O*)X%iIg1~vZ-I%
zYRyg289Ngg;xF0-eGp3Kw0SB%hc*jL`NVsY`b>ZEeh-1B)enlTbTrW+{F1R1X$!$>
ztE;~{UY>N2aH+CfYz~b8FC{dB+DPLTjr#82v$mxFMw?h&Q~KG?b9HCM3j$*i&73v3
z9|oI2oN1uVfsW{VA9zPo^U9ZxHM&|?KC$G1
zh8_*UT}~%KN8c_D{+R>2uoMrqQjx9s3YC>Y8~<-?9PZYkhHA8XF15q
zNn7Y#PiEoXnCMt4%$5u6g$W20h%@477H_H_hJ+kZw58E%J)LK)O2}`O^J%)(rpQ-0
zj;p+s7HqEO_vcL{1ppbFdv3B6%)$)C
zovI(snKm{CQU$6zhql4KR2$
zipyf800D#*nhZLbh&NsncS2f?PVubW(dpwE<7MuUc*J@~VHY7z;fHXW!|^JWfJ8<^
z!ekERiAeKsaigq>M?100&STF5wP7^?(||Mviv(=A-<4i
zS6ot3vfShMspyF7$8{Vs)QPbiShCkUG%2zf`yWYDvZP)D+4e{EG$gGCPE=7o8?v*N
zk$}3_UNf;&-1(<}J-n~^-%&_1}
zv6;%hp9PtOFC?B;roFYDDPTE@c;mXBFNPNOB~0d3b`nO&v2k(u3|#joVZusT>OHxz
zE=>R_snGZLcE6N{PX>vA^SYj$$yTG~J6IaS_j7+(4E&l;)6AIf0}PqIbXy49qX8w3O}!_+)0WMVeKGnpJwq9Esk=9~V|8u>)E|ev@~QJ!B->
z7<+|Ojze}JSe`yo#T2~1V&!rfiIG7>XeesHbp_88f!|GLdXS?ql<@H(ALITkUnwN*|E5@F))UW!#$JjZ!;`al@vNuE9SF((ug
zia8Jn9WoTsipce=8)xFqJo@FS|NS{I7?FGno2oOxE_IeZ(phxTFG_N;PUez2X-jV6
zb(Y*}g)2ec>YCs151xI&x(~ConTe@=M5k4T0ek((QvzCDXD#MRZ~BaUmY~I@WLApi
zZa9W`+~n$eJ~z}T@
ztw{l?W3-Zy?J@#%^zqg5lcE|y=MxQ{D-0(>t~l}WKt&47~Z!eV@e||A#s)3
zA)(ZJx!!R-y+iS0sDduV1IZr%Niw>j1O+o6w8ZnNAh*nBYU<*7&dUBC4|8rN)u&ni
z5rlU!dp$mxPJ9rV;WGER^y9&wEF-i5w!FCHzedGQu;V`vQYCa++w6!C54+z%L@v(^
zh&Q~)j|F^>kxl8U$HE9rmOqe3%kD5WcDs(zXW)$k{##gJLf^zHT*1@&q3hL8LQ6o_
zWSk);&vbITq9o@C2g`$jg@D(~WQqbA3Pt`v#c&?6)?$+}6$a!2tB^Sdo~fUNrbbL#
zfS1JAX3(I{=E+FVq`N8Cb;D~3fBe&$Q@d9l9iNyOV3ZI>EqI)-c;8SLoeTnET!JGg
zd@I#AMFCL3cP{RIScG{KFv7i!zP|#}wQe1!HER|C-boEbLjK)#91j1D7mplMN
zNrq`4sLgq!b9*{7Ie3FO{U+NTRLIm(s`~_Z85ZXVYuE(v#Fo|o8m{3nz-NRVJPBA>
zQDJ>ik^iT4H#s%=JJXjFUDSP2N=lcf=fa~H`A*`gRx&wyGzLb-E|$xy5z5QzZq;vr
zFTSvq_Tj##UxGfMyh3ioL+SkAn_Y3J6h8=GI*r>^2l1=BXl@NEGE)WTZmizY(18y*
zvB4K$KckJiZ^mz!A;ir{l{GYq43CR8E&hUf_7c>a&=7d1_RVGM>))F
zHN}GvNpc?k^YcW`(nZHse1egM7){(#2?6pId?}=d<)D3xmVYA+?@7t{t&u
zH)H{pe?pYY<$QJ0f>V`)R=rf>G75A*ZGnhQvf8!~EC*Pbbf|a&aZNo=5}ll;Y3wWi
z#=6(`&S!{~3~-O=69-zn#Q$9DZ%zT2EIc<0il%C6#yZ|6^meJ-E@a^PL*o<~=h*d)
z>>D@CSx4tlQZf)v*M};-C#|bGyoGp{^3JLqZ2K-{;zsU#EggU-T_hvQa`YdS!>_`v
z9MRqEyO?5@ET0J(i|OPgRS@_tX0dKAx=qiHfvt%{F$bUhHlqq
zHO9BaO7}6>McO3Y_0or?*}Z~Sb}9Rdi3vt}{?^gzj6WI*8|txqwkj!#G!CD(AfM0o
zJB+GB&r%K4X?Fdn9Pvp%gkm)b?a^jk&n%|77e0Y>`a=0uZDjJNmi5{WY9h7!is^@&
zzz86oRZX5NL3`qdBb^Njuw+IW(iJLp)cZz#RqcPWt{g6mgp~T5_TQ=pJ`;^=@pQ(S
zFy|y^QRFgl+jUeY8cKz&iE6mz@Icq$hm=WfsZQjX^1u+iutn5Djf4dik{A6v=Bxx3
zCdVb$*O<$mGWwu7@k8PmBW%ftYpecqY=0j#Wgb@~?e8)J^f-WSD*2I4*&K`YKRpKm#b0tez3e-!Y
zuhzLrr*9@JF*vk@*+J?PiVmley@GxOdwF>g2Dc>O5@mS$s}7l*<-Q;Zj^4i(>z{Yy
zs{DX4>G=H)y1GW+!aXXu?|(M^uJZqMzXQB0w7@zjSmd(XLo!m>wjjWJ
z$%Bc5B^>@=XvAY52+m-4-*Hu+7m-qyW6$ctHfJmaJisTb!vbz*~d8>CS2;M$QFh#chrd
zXR(+|pzOPor5zFXUd6evwY;bBW0LLgkKaFD3<}ZEf`8B8`BSeNRQlKKt#g08l@+Yujyo4#
zx*fWm>>tWdZW45TIjw{7F$k@24kk6O$ANgP2xu`tWx{-YA%+(pyo!@I6%jF8*ugFx
zpw=xaPQ0GbJDD}N22S=ab%Bs$ug{N_7E!9++Cnnj4#f@Y)y)3TNcgF?-e-K?C*&+NabaRqCU0W$v_SNKXIGS$5|%pL$Cn0qf`q7La=1b;
z@@(>WR*U#uos~PlIJUc38Y{$0a9fviH6b^=aVZZ2r#QqRP+_H%FTU8gU@a?6UrY%0
z`rOnF?j33yf*s0#!q6Z+%II%B{uKsW*ai3GoRnUCjD2w)-?4-{37z1P`Wk-bCVK@k
z=MwikQTfORra*);NX25mio_j(Rk9a84e;2L$L6=Gvei}S9*%yG8cXkg_a)9yQy}%r
zXI;m14y6X23CDet+F0e4rdBnfC~NEr|YG8dSo+rH7oRh6&
z3#iIzt=WqKSpIXl>XZJ9Fh_-t$G36YL|e1jI*C}XP(7Z_
zMy+>7iet?^42Vk>IHW(9;4Zf$*`~^^M^_GI#%t$%{T7eo+lLpmoxMs37PpzQmv}Vr
z#KhK+N$$!1rmWE;gh5Ee&W>P!FgI=3`5m(Ihp$fU=9jw!wE=lbps=atVvKJ$=6Pqo
z2ei#LQFYvGa)wW`qWy`Uhtg_cDn6-J^s!yAO@7Nu#+F_sF=4G;q5ad`p`|8Pz@)gG
z={Jw>boNudDsRnvr^@4;?&x=Eavgrq5iRZMmqe(J5}m!y$b2Gr6h3sOv{}NaiCpkWx%&VG6A&nVGb$M{U0OF3<@g<@HnZNJuUR*;XF$QgI#Jfu)xP
zWXvr+g!`)eR#mfwraEd^zt%@k1%Q|OZ?O|nou1&dD4|>4)}K*~?!(fIy$ni!5N5zm|;r&0*Aq1T>E`+jmCI
z3VLHwSou^S);#|L3G#VIt$gZXDo5j;H`enl=~$dRsmL^+S&RMjRwF3jr`QYhgQ7%B
zYw!JYT5Q#A7Qw96zcecw&dwhoOf
z9twAc(e}KLGmVR_@#22*0jQYP2Mb?clD<2`zI|iBC%hGcwKAe2uzEWQa|?ii?DB*U;pJtikkR})&rQ_-EZf)54ay{(|uP1
zWD(y1eS#lO|EDVRxpI}TC6WT*l{yK^j)5+E2vDweJHYC{`0p>6Eyl95@@5!4TODMbX89PRsR$_8D>M;jS~da_Epg40`H6@%9#OpfCSF`&G>x2uQ9(Vgx}0a7Y}4A
z9_Hr^1idtD4F>#qk@$O^aoK;dVfr)Zf%6|aj1~bq`_d_#jbNFgHx*0dTmpB5&!35C
zX=#Cm7b^}lIZZxtHdd0EBldna6Y&s@mUfVcf7;}
z*GnjOU;b3&$+6umU)QluzqihahRwx*ivoIIyng*zZ*Xjv
zj71xo$ZMsXLpmO@LehqMM|FhipWzCgZyKJP
z(H4EWbDG5d1h%Fsv|cU0chz!pW#dw25xGq{)D?y;GTU5jn1Gh#qi=L~Q}bhktsIoF
zOCv$&Y*srERk39i(_*)>PT1Sr*<|!}hj!6&aE$j-I;hPgr(08`3M|jZ!O~viG`Bl!
z@-1*)F=JG%Z5#RU;Ctd5)(G6h~lh4X#1E(78u+vXS53ECwKsKsgf
z@5!&}4hX9~?U4TMg1F2Z`f3M2Ro{%}Wt`-JJ|L_k&*s|I
z4;kCWdrzB7j;mawl$ja7O+0X=oJ?te;g7wQoySIUN}Qud8#WtG>cu8~Uo^CbZ**Cr
zu_>2&K-}gLmtK2~=T=O1@B8jM-XMF69%jQyje3@dI
zZIGrK)@BwJ2=z`v#q
zDBG@G6jYQ3De0E(EIPXOm9;$`l}TD4b>EQL2?B0ThL=E=>QYv^GT&Xl
zEXGJ9$dCvZM*t}|#zvbBfOypb(;wiJ=7UErpmG~yCRg2FPb*DLpe%Jlk_N$Hf}ig!
zSk_^9k3VA088FA-Tk;C**0hH_8khcDP&QZ7%FaCOM=P!T_ViPG%tpvJX^a34fjGG&
zjYca{XJZH7q&1Bh0tbSoYWDF;D?tjo^|8u`yinOJI|L3G)xuS{?B@F7oGv~*!j-IR
zXIC*+xl>#@LZyg5rphw=^l+1}Wulzb1%3z(f)Xj|>BlB%^VGom?o`D-+8Ix){@3Yf
zAsukTJ6{K18K>HcEV=@4HfH%lsLPr@v_1~Dh60`{mBaQmKxc1&9Ogu4H2uM5l(+G0y6k!7wvzt=
z@%i`!zjxI8*&cjTWUgV1~5NJJI)As{fYowE}h76BPAld@;$lA~Wno}mlZ
z3BI9wN0@E&wjJ`XcBQ=hAfbQySk}S?P4o$XjUv@*(_Q+)UHeKv!h!fk9FwfT|PfX5v;i~K437Nd$uEUt9
z%2L~&w8gR#?*%!zy^SVVO56EIHaE3()}W);f-FIYB459V{fwg)u{pfLo-&Temj7vt
zx#fZ5ESiJZ>xjooiAL)_$R0J0fkQ~ZxYKV$7s80n^nE{|Aui9by%g0T&mhr&%%W
zp^2L{hzMucXqA9N5=J9T<4eURJfrWA%7^h|%jo&B+w_7b=pEYo4$MJJMb~4@7;JX5$OaK8_senP5;3%zq?!@<}{H0*g@A`NPL^8U!gD7w93
zduCrD`F$nf-DmOdFiyxn!p!hcS15yLk;|pqQ7=9EbGJ4djg1vBH4%M(@eE@rR#{4i
z8%doRSbj9yyBypn5Zn)%74oiB#t@7N^9}unoSx>2x0X!#UjE)smntBg!vBfm!oz>Y
ze=1NQquPwAQeGf$e=taqYEdK}@k6euVyscY{f8-6D`+XS+@-*6OW1&bQ&wAjnT*a?
z&u~6Xe0eOuv1VUMVSC;s2iD1-$bzEH*ErqW{JGGdPmo0@E`knd#Pmn;TnF;Js1q@f
zZk5O%Mrt?3g_9pw*zb`oQPA`**~u5pCd{g6IC!8>r6
zk2$3ihjT%|e^t^+r~bvakW-tBYL?@B{4O7`DA@uq5@J86?7e|p$Dpa4^dkkcHc0rYO}hbGBrWG_oY%+iz`9rSzRovP2DhBMyiuSid=7m2HuvR=^qQb=
z!fCTNpxCOsa_K7)Z`JSdg%GYhd=8u6`QsKMqN4w&-)#j&FdEKFj(D83wOrfMT{p6x
z`wDn*&S3r90OO?tkQVZW#~^|XvRncHV_lfD%;UC`;P~^)_HYVZiEn7OF17g}P{%>0
zMcu^+cM2a4(8`uOpDfp0G42}3qyp*#3>0o+URfEL>mkbKu8B6FFy?e{36l91W
zr{^xpb-4#`HRWl@>^g+0#TJ{+ux@cmu+5Zes(*GfDEf>#UiTf+-i^l}6T;_?1XMLB
zd)QE3H_qO*-&)Gc%RA^*Q<9UZ9PmBBJ9gGsLG_K0q^r;Q)*_>QKC>qY;}#}!vkjuV
z2j}vr)_Eqdzg;)F3RR
zKriPWBh_`_q1
zB&1}-77>;sMG}lGSZP{s@E8fkKy#ewj}
zy9SdK8+mU9g0`s`SGhujbpu9`<2U0fLFpp145-nV!Jmd2i0{X>NDJS0p3Q2v9|o(Q
zcDX*vi|RpBu4O5R7v?q~hq*{jEO@A0{W;L-f@a&!vD8o{yLua8?0kpyxT_{iWbggf
zDwE+=s60w_GEpq_ttVL#RU!OMtC3aej8XY@OeyJs%aG3{skhY2pVFzdlsl))@HJg`
zHTAc9!*BLI5dE%B52Vs2N@84B8AduBg!0P$Y3XNlXxl3Fg=T~HG(3Y5H!GB7)s*$K
zr?8l}qFX7kTd)=b3J#*G)+opuE&bR=upZNdW=0C#mw6_gFbUsFtNX(sO2P(LjS6a47^9*r)pYP`xs&uT$aRY%UPPvLu~im|TdG*;;gD2pU8cHim`M&^xx+@5f2}
z=c0vO2&8Y(?@3taXB@0)=W_Bpr0boKB(E4u2ouFPz)OI4g%toX<1Nf7UlaEZRb*nx#7av`HxuGBkvb4AnAkHC|{9;JtcfTftKPFbfBm)o*N~P9nsg3I=tr1W!
z3cVj?1kKw;=20#W!Rpq{)g*zdkgT(%21m%GR<{#mMo|HA*kei=w1RDT29QDNj~D&Q
z_y+Q_wCuob*K>I4J=D?6I$q0qJGb{PWK$%cKz$#!S=lCP?}H
z0?2bEt-9m>V=LI;1ySbA!G!>(cTMjjWD~#JO28y0Ujb8{wnP{p52paM&ff^KdV)wU
zMTK;s*s|P<99O_q@E?jHjl2v2Nx(BF&$AKdsS=cYCc)9SKH5P}2%CDS)4Ngc7KrNl
zT?lF-{E&*7a|_evvr|UI_7?I{F(Vh59F&>tC3sf+-g3j|a)5>Oq%V;FNj8?^rpLps
z@}%%5dV>cZho%bCl>h
zuz)7EEC#lHyPMb7V;RyMX1Q47_-SS@(joz^?~3
z`;Eto(O&R6@9uF+fT<+c*GbC@bf~|wSZ;C^e{7TsMqPr@M+c#rj3b;!RG}qaNOg*v
z)1UaPjUrv*cmN+X1vSPrK{EYLBg`Z)V?&->W>a+~qje0J%U|7G+A
z?&Xf(8#;p52^m-+?Ewld>BL4b!nth}u{MXp|2E^vyWHrOK}M4^41F4tXxr=O{Xg?R
zqpd&h9}gn0lp|T5-{-?Wk;veFf0F0Aplw`0yRp!X?vswz?{
z0zpj$t<#0XUoHqgF~JJVlCV(wu=qsSwp2#hvqUWjO`iu%$8#V^im{E@g{MkwVUP0e
z;E2*iH3Z&8g}_=)EF2Iz`VqWZaK?ux;m_wsscefZ6WB+YoaA8fVVuH22rX6ld4Fiz
zWVg9<-$MM|J$#bzEulPdJlT#m-?nF)E1a^ai@si*t$Z$u7a;V
z8w`umA-h9y{D3r}Kyfhkrb6|Zffo5@34ZJE4}X++UH701K1V!`K>ROh@s<(`?^Jcv
z)|+s{kx~MLFY`Yk|C~Yj?(fYmBdF
z$o3Vpa1lHCJ6)N)GXR5d1b>Q}|_({tu66P_oN^N7uA*1VvoZy*yG2^gU
zlp5hlfhalh;S;i`y(>=i<84vwf2JjF%gK|mBp^(tCY|=WR;5ZMCA{9zbmkxGLf^Oj
zj%RIH?2sKkWBC9FezkJj!a*1$>Gxg1)0{}t7ztEtmdnfy1;t8Saw73HioG30Nm~1esDCC1?
zeBE5(|4WAk0gT`nA1)COj@~$P1EFF{50kzd%d;YfJ2Zc+j(sGX7AT>BJuTAPnn;r
z#M`z+Qhm9Y1mC2&s7p;Pb0pIVNj%4tc`s-&QmJMdd#__s@MIeMhA#hsdXXRJQnwe6
zj5wwf>BT~>m4}g#BY(0jT{U?mW`<}?_-#Tc;=~A)mbRgQ*9knU)5`K$8Dq9~;qI+M
zXAv~E#c(>~6Ctm&nn_SU-y7Z7dB9
z!9(iMAgvIzoa)AFnkR0`i>@IesZbyLho(*-b&-YNi);?(7#}w&_}c+w!9zR*4h=}R
z7bj&
zvnKb9aToH&j%rtyZT%Q0t$^jHb1OkR9LP=iS(RmFW%U7U1d;Xo0>57wy^Mb%hho^
z;PA6VTLqGm5MWGlkP|*?zFsirKECOpwDIu31wawn@^wwE>p>z#y31>bq>U?`9Xx3N
znoSq}V*!+C-9c$17kP5^In9Szm$X%ohoOo(nuFTh3v_XCCxh@q&qJZ}H?1#EkYeWO
zeG&fp0+)Nh0wCFT~-#`t#j@ZPgG&I)pjW2o7GzxptB9;xV|hwDP#k
z-dJONz-rZW))PK+O9TM`*)(e%LbmMqs{W1jw@QL|5I7{BQz(!qLf!=Qgf%;#A$6l>
z%aDEopg*?R?2G!zgGBk_X|?xz$@6$>>LVW@^qn8iQ$S|OC)5_y{4zI!{j7ihnC0Rp
zWGVdF5oc6XDxZFjubdsWrT;1wS&RbAItUGXzEh9%CU2S*Bfk(7t6nHkzktB)8S@{%
zPK#RasX>`iYj2F~Gc0&<(&wF*$Bh4y{6Vu5kyLw;)sFtu(0Fl&PX
z)`nk|qh4?$JQq@mMc;9G@2Qj&pWjo<+)-|8xgu82Mi@uG&oMCR2>0avq9rhD>8FbBDH#^ajpa9+sc$tTB~yql71Ui{5l8)v7wFUf<{gX2yvr1BI-)!-m
z(lLE5XTDh4eLWklLD#jiQN;QZvwqh53HVz%fpkxc!xdCE{tpVgM};RGBfnly%?9$Q@9h2
zMUCzW#~s6uAUh?Ni;5-H@(*HvzaELiBv~P9q3y9IpT{VcsIs6PBNWbLD3pZ=sjKZo0&?*QSY@@OhO*CBK
z@@LueXdZS&e}3zNG=8bIBo-z1-fT~xomiWj-?NIIZ&z$U=Z82p{vfT^N8O(Q5^}nX
zNCa59y!92YJ#CXNrv)o4YeMzPU7FYpx^
z89~k-`cGM7z&{BWRo2k_a)=J-(lbybSGQ3iYcN6nB~4s|I;@FVy^vuRApLMX%wVt9cjyxJBF}sYJIgZx_1kSr8K;BuUy%4
z@4`+H!e-oq?@#JJj4@F%|U;LyeQw3W(FD5)QKF-0OC*X^2Uk#ov#b$pIoI4-Ts$7vtUSd_;)=~5Wf$h;fPvsWD5iHg
z*6!igYjY8tjp`J4JZU61O%6a4KpFr$h+09WecVDQ;&FOBzH(ny-@|ktJX;1m&oZFN
zUbs?hX^$2l{^9e~e$zr>j!uY$q?FauB8+B{LuE=0SGt!nl?xRrT3T#XV
z6cK*|A3eWRei&OMFQjO5P@H_+j&C1sGPYJxS>_SW_?r4EHD6mPZl=4;g!@9#LetV2
znpSEi#4Gb!^bucKvath=JecbEh@_@W
zvI)wmcc`+1#fCG_SyqV=p+{<7tzf)Y-T*Br={P>2lARiHdr!Hw{KQ)fbLP10D(mTRrXpV5&~w(k+X5s)Yyg!}W{{(CX%Fe7P(JAM{7tk8?RnI~`Q
z)9GT*9@XLCtGy<(tLTuvq2CKFnGd+{2niiyja9IJNR;gt9_3#t?7;08)2`vBQY2<%
zbPtMZwI#C_UCnDi8(p(F8L-K-%cCTA_6=WlM>L6ZoqW=mvYl2I=UQS*y@=(RY|Ha@
zpY;u1%K7O$--_`hZBS=EvA><(obxOpo)Fif$<$}u<_eZsxc#xUH6d(30FQz=Yc~+f
z?-V9v2eK;DIYu?@hgpH2_Fr8&#kg_BU%&z6!5sDX_lH|TC|%(VmnuaFF%W)^L3B<}
z`h1G?f(+?gewQ3t3Bo5eCNA{4cKBah4*>@B`^d`X;i$GY74VkTc%rk($;ta>Q>k8=
zF=ArEr_*I#)wFsG1kY;(bjdq|Cyyg1RdHURr0LYn7-(GPfF33*X)g^QNaIPTPY
zAYpJNj66XPGEk7tWYxM*r&(mUpZ_rqkdpzk^|o91OfX?92x7R`Oq+BDf|y#?DZp@?
zt@!uY|2UAFbf`*+&6>7K0z(bu+5NwG}`~5Z50WF;-#f&q~=RLXp7b
zfZ1?@u9RM+wX20n4!#>W(sdY)Pk-P*#t2O3hhkhlG;^hr&bfW^_
zYuvixY(iU?PuSS)3)LrWs>WfEQ>q(OCImt1qJVig;`DMd@N9$N1zO8xZ3pVdtd={T
z_d{*OMlb&}Be?S3)vZ3l{oV3z{`hk4so%EG$+AxoXKyqWrc&=g87O>&_}Sl1%>PS=
zcF(|mrAZJ6|va-uZ@F6M=pOsWvEjE-<_
zI4gA^BUiMJgFzo^8<}xBZRkj}ID`1%n=Q*-;g?vbjS(W`=qUGvwQn)m1<>X!9kK(0
z3*yuJZ%5C@s)#Od4;*AR4zXMmnRlvp2Bu{mLhsmfN*ZG@x69dnReTQB&p2`It7@ix
z-)rNa>2&$Mc+;4uKyl2Z?wXbC+lpCsR_sEi%4g9jiL8IS{w0O>_cv|?j%-%FqY<~?
z0xWyJ;Mzk!xOuJhM6e$@@2LmnT!cjI7NuG&7W()T*$1#U-@h6DROEg6c2L~-SqQg`
ztlu!T_mz?OOJ+;lD3^q^1Az_aXMaX1ux6^RMtamrN
zvWslxHDu4L>3B_yM_xGZK&BWIyKq#&V6C_aj=ZR|I>vaMFqQ10Iw+DRsm
z&H*poYw-5oZK<9)V|D0!u|A_f(qey@iiCvLvbFdc{z08x;K*{YZ}NNoAj!|#FkDXj
zOdc@`>DO&smCl)cH3HfLsJHh!#6>IBB-ylG_)hqV3B{9VsCXOgJTc#I=B|iq`ic!L
z}WMscB0Cq
z)n0jrMQfQvmN{gQJ((}#e~#vVOqxDiYoVmK+<(Jju+$U!)5nfCWshL|N+#y#&yK?5s>d*2<{yjSKhI&dbi8cT6Y%Zt4OG@$&dsp(#{)jHe{U^ThNP4;ku!NN2
z$d^}_BD|ZaK-rTTAyTwva%ZQwlcH{4|3-y(lrN3aovQM)=XD_+
zPwN!YP{QP}PpctHt(FopdTSr>nf9lK&mJ=e|EDU6o+?dEt{zg7Xr(vAs3Fl24|nH>G#|6g6I7hA9E`X7`^7E5
zq_=s76vh8q*wmjb)dW7berbCkk!uNJOB)ta)-YFLI4D>Y73{QV3+%y2yqDjQSoaa}Juki
zIp(TN(STgBe$0%eoA>75B+4EnT@ZV>-V|gw)B$p1VT9cT%z3=#)kC1dr%tWSP$>nf
z^nfrN_H_tU0%Z9@V@FTJamrfrtg0oeAU*!mt?|k2El0~`F1@4lGP<&cT)JfZE*A*w
z^9v3R_E*v;v<2(Gv*D2ECJDiw4_k>oWd;&0pt!ryJ*mwBs03sJhw83JzX7h_9q|s|
z<+sgpFM)T{ujJUzC-OWH|ZbN+FqL~>*TnFlk~tKj*+W<2uo(-;tBy(qb2_pW~tX`
zgn3_BKRZ7*dOI^))f-Zmwmxg?Z}d1rkg}i1ughAWKY+GT@6h}Vo}lb9o-ny*IGtxw
zAG_nURrfr*+csX_?jG4TL66{!p}jYz{fzcV(#9HnfSXi=u?Y2Gx
zl~1bFNGwkt$uE;>>{%4?-)WaRxaCXVN3dM7&KXBu85x)ca`apU&a+&1{yxa!lR&yj
zVZLHtdmkLW&X}m2tB+6F)LOCak@dy6DDqf?psikyn#njbvN%X3PY(
zSc5YJO;o$Pd{(;$6p~|&hH`w=!kFM}SZJXk-4_&vrrOtCCf5sJKYARoW4Re4Y3(tY
zkE$?f>|a&G5ER|}$7FBdM`bAd@p|(>P)H?yZ?va9iG$1C)%RR)WQDxM_W2sbfd9T?
zTd`dJ;>FFz#1(+e(_ufL%c(i~Z+ob2egG?5W{Diyx6k1dg(J
zo4*hE=leM#xjI@GaVE0}&dC&bD&BHWFpM*zE&fY%{pmHdwOZT1Qz}_DT6ADwk5(^w
zLj)f}I@nJbE7;NG^Q-+xQ+|U!n_2QtP?LO{`*<3%T0k%q9=wN|GJSG+%WULCKLb;+
zCW7+hp|R`fVjp^xG}aQ_1I%cuXN!aBV}m5D_=pa%xrG`%(PPc^h&|IFp&hA?OE<`;
z66elpb}*bcFX7p~vV9*Aptyx4SWN^M_~2-+%mB-vobbiCkm3yq0mFO5y=INDnR*ll
z%kTUmn_@QPkqo-x%T*J3{6(?vvC7uFf;T{s8;6L92+iJn^y=4*cl;NywS1;Y4r4(y
zZ#`QETzJx2X3jP6M;KkPO*2L{G*~iJrCIK1(z@Rr(p;kJaWt0gxZ=Z7C4`R=1*h?#KN=Zr4^tE_J
zW;6o?*5z_swDA+8y@P|ps9HIZ?_*}=bTX^?U)@eO5LNRM(%-+*e)6>3+D%(4
zwH|=O7Xm<=4pn%6!S_b!UFONN9OPZnE&RxVP0dXe$0KU`3J1H-sn$d^!dW`$
z8nc=|nNv%G{7Fp3vZIDzkp35&J-r8s=~AaD@g+s|6W%jLNUB8A+-a!BPunl~o_Isb
z3^rW(LYu`|s$DHJDC+lYtRo%i!zj*EqeXVR&{mfgeJtfaYAlT&?!Qjtv4)8sWdHFjK3g>JWN=Y}czxy_?>Q)`wGT5;BaYf;>u_=D+P+&Y@
zyuol$DMV&z&@J5IwOHVa%C9rMVimz7P;Ip#fYD`78^_yp(5`T&zlR0O?rwaL>aeE
zmHh5V(r2IG|LR&`tJku5xJH=Ptf5-^4V#8z4{e?t;o$2*nw)c)tPt|00kLRYX%_4Q
zxSRqhEkHtj;NbmRhhZBRbpRBulr`JBt+;@!>|Yq(Qw{J_e5HTF%XgZ
zMUIynFQ(LAAe+J2%W5(+>Fx0D8PZI$^{3Eb$2>1fb2VaRSYQ1SDev-L&x{_)=;VDJZ+-A6P3FyhbbFJGz`W!Zd@YI~u$oKo
zO>R2U#)zjy%q2h8Two@B%!%JAEpjhxF*1M(=drB5JhWP9inf{BF-y$g4qwZp%K2#X
z)L~7{p8w|+ntOX@p|3ao`!B7LDDCxxo@>O)zTb*I@h3RFV;g#w&Sy64VrQ9`g>o<5
z#Fl7iR+OT!mnm*dX>hC#hA@R3GbW^C2Qt;(SrzjF1-Ad9EilwT
zi+e5C+wVksY6S3qQZP
zfTt94<*TZKARIR(#gJq#5SQvnU4{n?F94AZMje(DSJL0=Q!B=Ah5;q0>@u45DETFi
z-;1@*WZ=Kcyfbe{r~CyYoz0J9Ghz$r(@^%>k_Y;o?{!x*va2t*{55MUFAv@k{oWU?
zYsu6rVQ}=g(5L}-uCK;z!qo=~WH@KUq2yu33mHvTpGsxP(2_rERZ*P9xMsfx)|xo0SLRd2Y%IL6TK^zy
zNq2p9Cnc|4TpJ^WUfbVOxGNY
zdh}Ey_j}1Yos=!4YMkPXSje7Uuxc_2Ih9s5iNjZ2R1nWG8KqwB?FPJf$5HQ|K(-u3
z^6ue4c~oTqUN8ap;c4@_tOYerhkyBTbnzZVd(<|)_F=qQt@YJ6w;
zkunSI>R)%?6_;f+)|vd%KOqXq1m
z+_~vE+s=fu(W^PyB2q@he1r`)7(AkXo(1kiK5Meu?X56Z_{R%zYLB}rd8=&3yy39P2ti>f7EDQ>FQBFBx74!&3&N#UZsU1ww*ujD=fSn=j4%6iTr*$i}#>7z{29X
zdAr&C^@Nff40-TT*inmg-F-?k>0UIQHG?di^eTeoW(Q2;vh^suSpC{R5%GEX9i^+u
z9g?5*)o!anwf;ob?4R}<>-UBU@&94XTuH`>9HHfs@S-tsi2eOq&!%OdJLmS(+EMvE}5Wr8uiieeM2h#XJ~_+~M|P?Ap}D#1{2tXv0W{CcS6R_01B
zbFqd9+s|OZC_Is-4?Dn>?L&VPoA~JX{Z`O7l=ON!h5130(yt7A`-*F*y~{4x=yc_J
z4sZ6u-eT#SX%M+&>W%dXg$*%pPFZy8WP<~wsi_b}m2md>?owCq@J+ZCYCk#u&bj+=
zV(6cAg`!fT6T_TeuONy;bqG}I!!_H-ceM|
za-=cty*~mnaH-f^TJ@S$ND-@G(Ksf2^b!KZ0w~v&1R0d12G`KQ3cd57bOJHW{8vAO
zBeXLE)CyHL8S2e$jtSMOJRm}!7IK#VRCRVZ?||_YAmQh~UG;^0C7{ztFSsqz=bfqb
z4BYwj@2CuJ*B&hR^_EIBh*o5;;L%F=|JGIAWyrn%Wq8!r$0{#Y0XNpzJNS459Ql+xeXyXRgsWoE8iX`E$)#MEPmKh|Gv7b0|9ZL1cve^$
zFvA4?a9S%wKO%>@3j1JCtydzy
zmS+JCB63^SUxr3kg^m)#ZgD$LPZ>ry9LYv-Ep-sQ5sVQDI_vW#nvRB0nzpLYRVvM3
zNWQ)&gfTKUwq4)k*`~_Zu>!swuGY8w-i{b$Ew;EouPv?@su$ea(;vZn98*@}`
z&?WbYmmvm60|JFDH0ZC?V1Guf5rr!LK(lW7Fpn%2&7c%P$DS2fZ;f}(X^RoJ%|Eg=
z?WLc-QWcl6rD3(CN+R=b+z__pfWAp!Zq(|;!C97SRiX$T#
zxctO0+G~kd-g^~UMjU~bI#wZgE)~tRAyu_EH+akPZ3G-jjVRj2&H@{B)e?3y#(X3npUH6N|DcKb1+*x&Ca5k=3i_
zzvd>G))G-}lrb^%vR2$%y_dmHR-#UFX$u$&hcOBHUl@<_%!Rva9=@NZ9`?Erd
zh}WO(>q6!W7*3{}%HU-NpP6`$t=7hwr4s{yAPVGCT<2v@641YcZk2V{qlGF8MmyJ}Be`FAc;m
zx3Es+!@Dj;IXgNS?gn)ajc%NHPgZ`pJts{X*x1Y|Gz*hjh+9~i^MR+!bF>~~n?%@1
z!-H{$?fKy+t_XXV)E-$j&Dukn!};Rd8iAR-N#ocl7SVUuw;n*MnIMIdU4lAM`J42`
zKN!Q(VUl&W1hq+wj(1|{&a{~^;_fcyj}n>~(GwzCT8!`K(VDfa%4W}C{e47e&J%~H
zXc3|7zM*u#)9*5yR`{kWc@Ih4|DKM;BmyOAkRcX>VKXk2c~yK$BXFv(pL6ad+_qQCXlsDE0?y?+0}T
zrx~-Uoll9}s)50$H2&|ANUG7Sper(iM&;Ss673SAvk>PK-x)y5&k%L<>s@y#Ht>;P7pCyzaYw|aj7x(?uiWs8
zluNq$H_G&nyHfrrR?!lBGHlz>D`@+ixCU{c)k5#P;9?wcZlEHq(FC4I&8HP5IVHy1
zqPd3em-G#N3q`Fy-xR9E>D>(?skeP?)WezD7l9HI_H5oT^hT71r^N9wRwh#h3J|pr>5vk
zMTl=A>y`WKYCjtPQ8GiLV`aAds1cfLY8;lqo!9p)?RVqbta-^9v+z>isD-!W-Nf}?
zccj=r&FGtwZ>-@L*Iz$gqA6oW!!7sD=SaC;ZxcyPCumAieNmXes8b#Goul(ogthHhXp-I03b8CeRx;V=P1$=TE%>KUl}5mUcU*)MCD6r^tb%-3
zP{?){o~8!)nMFo{>%J%wK$)0e5pM|=Uku%6FOKvBHE(w(692#1AmO)QZh&}fLJ(>=
zcT*urL-5aTkk)gtvrFgRmA+p&z@!E_=ljLEkJX5@Z`BDTQP91#lTf!Fgd&mvSK(P3
zG}7hO7Bav)k(_{@i{|Ec2?Vi(n3&fdm9<@%QzMHYqQXd
zN>uaF&0FLAp^cIvnf2814VS?XeGc>6&oheQSybUa1ym?FKN+ebN<3SsqkFzAE$H5!c{<>O|@mq#UkU9AI
zaBGdN!7*`kzUK;aq@LXOK`uXzw(jZphSnJAi(a=4dNsRtPw)ms8`?4FH?{n+pm!}d
z3Jp~=lv8fudA!C2(3Vr;8pPZS#Qn$^Xk5hsR6>J-k9vvZQ)Drb$WsxZYqxW|mX($@
zCUg#UzDu#-WbfwHXA0fu@z^!tR-dDLzCD$R2=_f7asAW^gz_XVPi6)0{XUt?TJ5hJ
zD58#%AIK3`l<4bBR4DUpZlA{E#Zx;vfk+d}It=>;+^BWd@@9O!FDqUw8h|eb1PA?&eW(Tk=?6
zE6uCt)~z^DBq}?!uu(HG**u-qSg7~=TWf=1hfZJ`Mvx9pg3cOT}qL1&#jMYiIfC#kU-X
zDeoO6P(@03f0GB?gr6HO!h0o>~cOeqz0R-{@?TPn$E?O|DL$4
zh%Zk3St8FRqD{FM+lxtglY~i1n)H%m$t;FusVeNAFEO`$5#J+%U#wy7ofdA7o|$yu
zA6cG61k0@HDsB*HU$E!Aoz9%MhZS8_8O~D*S8hQ9qCQAuL
zgq36Pm)G>V*igA3Uh!p0v)mL55_{&q4~T@@541C
zEj^P%Z2rLNH1ax3G}9jPC5!r&wns045yEC8V&mxa!zp~~ou8I1iC$lBawy@j#3INi&
z5pt;-E=d^Zg9QwavpwBu!Blg~XiX<;=K9wky*^3+Cu+|i?o1s&IrOvuStX>E_r?H`
zaopr#4a+>}qDZM^-&J*ZEF!2H!kvJ|4{qX;63s>opdkOE{k{@>aIVpr9^dPBtrX?h
zs`cqQ{JacgR#3aquncjPX&bD8Eo-eHKYf4xULV5r+;(6B36AVdx2?0D-s=dy`_Evk
z@*+poO3-)$`W*#Q8KO<{jZTPv1w1Iy-S!JwIMZ&&
zS9g>7#_B$%ZdKo{r4B22sMiKKEbY%LU0q#mX$mFUt@ud%j||txON$Av4xpY168DI}
z{%`D}LJ;^b)*uZ6vNvznIMhaVI0xTvzu{|6XQsq<8Y8J)2EBBaxfzLxSaq0|v~m9}
zh$vnZ0H-}DslBcWm63x`LY)wsL5Vs6H2D*!dm{A>?`Q8P^Alp+A`B#No%Ets@
z@BH|Pe5DF>L_~&PInzR1S+>oQuQtm|dLID=HC&GvZvk$Y9vTYAlII}WiYQ6mWWfYgloj82C>((4)u49(|
z!p4ZOk`tm8c}RI7#l<8sUco$SLJ0{0ktk51KfM)6h39=;F=8Ct$tRFvTZ(%zoVtSEEizv?{D9_hj3zSp2G$Rhd`7BqSY_`KC}JtL4?gOaUt9AX(h;gliK
zES8p>xP5LMpsXUw+4+|u%Y
z)Od~_v3cEp%&LoLl96GTx
zgjDn$-owvl?e|Vp1UZMD>_cc?CXbHpjrO~+RmX?
z+oD5N#d2LHx0e{x3uS$5U5Wcy@#06<#tiM`^#mW)P1}`IWJR$ac3Q~66NQdKR{W_k
z`QDLPXoeE$Z#JJaE^$^?ReR%+JFMkCwZ-4n8%$D1)6rWmKT3;JZnDofLgzy%Q;Z5(
zrQ}Fm=45zWuP15N(H5uLsYs*PVqJ)wF`2`UuISg%P%I2dMbhNw+Ia$0RL9Cg$#N!=
zi;b%6v3^L%6<)>q(Li)F8Pg}#i4DVI;ix^m)a$5-T18puS8*|DOku`1zq9x9>x1^V
z3srxMC>>PZN5j=}9cqSNlFOI|A15)1JCiD{qCoit&!$>9&ckOOd%|D9At=RVK3cmB
zs3k33yGR9VLOfAaG6npBBe!<_SK8rsCgGl_zVgUn
zLJcp33GWm}IUm!c4Vj6BmIc+JlNi?qWR7o0bu12+sc3Fc6@T99|FJ62`0_dCfk59u
zLe|VYXW7?*oSeMx*@aUrC0P6BIr_!JDQE0M*PFhiJ9(_PS@ti=kl-`sOsJ9>(z54&74_0908;Y8Vc-~+)a68W$`u$llyj=rfG@@rphGqFO8+Mfu$JmNt#ti(UCmrt#
z@0f9l$h^O|I8lzHnsZIxOvXlqznIQ(j|$|KKRosVQ%)0_%>hE<2)|%nmN^`9)(o{b
z`+pTpcR_coiLT5rZIyt{8cg_0BV6|phWliZZRTbvue$jtk_Kc+7q|oG
zfW(vvnM2@Tph32{K%Gr~+n6$`Va!fS0afb4o(V`Bb2s+!T9Au~G%Qwe(X2$U}
z?+IS@xHTJ7*M@MJERrXNA_QieKEQXPzxa&esvU
z9-FrdOO%?_1WFpfU3Albee2vRp2Sd{T%F$cqtQ+Zb{H-5!A>V6J#f8TC5yOxG$|t<
z*E*z1q($riQCTMX%Ghqf)p>6dZb&zT-SyAC3y+#g=INIvi(fzEt0n^Ta#Ig0&3e(}
z8WWOu#_(m~dw>2rQ)|y3&vep;S*5VMt>93Pgf~ZPWJYmL=qz!DA&@&y)a&Js0PD*
zAV72J$|+t-v>73kigh3nz7~2*^c&-oS|z{wpSwb+%#K|1zG~Bp3tDLi)dGrWY|WFI
zu#E#;9Ayl4hAslBO|hQsVe*w=5r3zU^z1J*BvK_d%@OL~Ghc6)AYviFjB|OxWl@GJ
z5%2_$IbTNYA0^(!M0w>~jIK{jP5nbBsDJ?7X+<0`(rJDQ_@#UXQpq_WHI%IZign+n
zl=;^OnJe|j8`Ho5maGfFBD@}&uQ5Xf=E2SN
z0itek^)s*xU`VzUhpY5W0`P=i^jTgt1$7yT^;jmZKu6`-E&%t-K7|nqmix
zpI-SsL3|^h#i_Q7A8ZyI(4Q$fT9DaidcbqZQPHaPjAf7hSf8t<@uEk3{s?o3~;+1Z1C~575QckUkNJP`r63-ZIPK{!~8hjS&BnXwEN~rTY}{UcDS!g&yh|4v|R_y
z95lPPWZHWUq^uwc_G;~2Z^+Xlv>G6_H4Fi@739|I^Z1}Y>N*;U79Y#kfp+>cPOP@u
z$z(k^1!TVk;~_;A{t#NdXNqC|Jpf
zsV0uX`0}kP93$nQw(IM7pyQWtQ4;2nCvqSlPze{lKFw>fmN5T|gTD^jUet{>5DTkU
z4{shuhiAwi^Q8%+-sCv#Rn2`a{o;-yD&`haV?;>dnKMs}u4Kk<`l4p4E0>KQ7UuDM^v;=82X1_wc^0ja>Vf?vl$V74-6mVd*yCBe0_;liS_(s?9ka
znTfd1VkbS7Qhv+yQJBVXQ6yIS4eCCnJbAYFxKLHx0n6{GK&NcvhwIuSyt+?iSsIf~
z#c?U!a#!ha`HPv?tP2EIi!v?n_*RvD=6U9KZA$)mUP`N2N4IPi`+g4{i3JEV5skm)
z2F>%=%G?Jpvy=>|FPKQDszuy8R?EX%hecw4D(N4v~?54{t(YIbkwq#aryL}podpqDNCt(l2&Qk_FP}-ATD^`ExA5inOrGd=X}It
zqRf9com|X_RUR1pTFwm`6W`^{*Ey-L^LcfdxX$`j5v8(wGJ2JQ`(XWq`=KcF@s0ai
zi((abM;{`tgT7-~0?FYOg>qY|twYi;?zs?nY@M?@K`{3Y(gG$SV+@XOkE028q^(db
zc}@o22esId6>80IJC}9$3Ly3=$sX^QkD0Od5H!#Z9aC)}$Vek!YX{qFnNT+PvCYw|
zeORo&P3h+~zd}0S-mqw;kF=n~YhF+N78sZL21$dm`oy-2+(sa`EKlOQp1FYH&_3r?
z`6-ljfL?6yVwxz%y7|Z4$b!a6t1?mL3f$Yc6`xFQFnzNH>Q@h|)^ul>+I5njU!C+p
zVDs^SWtxY##zM2=6|&XQzN*#awe7nM#yRKV`5vs|nI&dQnIFB8oo6PMK*8*#eC%B4Y;Y%WgqV%|
z=)FwPl5aI7mh=zv^ra9XdtQ(r7<~1MqRR%GPnW0P?fl!3L?)#~CI?Q_<$06Fr^?7-
z5z!iQ>0vZetI4$UW}i24$fP4G{JDgvNoAp9BjXXFWqKdBiIMJ*-tLBE>!0j#Xsq0`j(B0B6jUB4Y+^9RgTfbzNRM*NOd4==$uw
z+;(JZ-&2rexZvdYZg>`YbD}DU^=woKup5!snzwMCc`|IEj0+
z88}bKpx%e`-gf}%T-KZB|B3jYeq1&`Z#=bGk2XUXeb0M97pBj+NjcIB=cCVuQ#MDO
z{njEB%i<Wdw+8lLOskMw-SaZIUGP6M%BQ#L$U9~lyjYomy3Z<_X>`COAFsH
zfja{e2c;4&-DIQ~vWEVRaI4=;;4!@RqdTQlVzwDoUCW$E2HFx)LEV$>C}`5{d-z}t
zMzYR%1~@&dx5?FmED;XPJ&7sFl|qHF_n~elmLq?Q!zdrk&CZdkM`toAA-lih98;a3R+b6#`j+(p
z=9pgQDUG2UJ^z8<*KaV-RQBWoq9y3M)m*5=awnUul*nU4Y?
zytR$-ajm))K$SuE7E%;N3GJXS=~brc<=wj*+Ty3up;2lv&wOPMQGJa-PmB`GPc*IHrciUrWZ=HFjx%1bRZi(I;R~CWJ#HoGlH#1WmS6wp+dWhb!a5tn?qJ40#D(Uw%~DqRA`dX!GQX~S|qBj
z)U5RPAkNYuDCORk`UAH%LMf$~Q^oa9bQI6PzObk8hffWT7Lq&LdN>Nfv@$>|uMm@j
zrJ2mn@)b`wX|gVQS_!Sk5fcp_ygq+waaMKw)L(0gdW?IMj>~M4F~fu(%0F_5
zyWv!J+*4^*lD)J*i+KMc+{vML=}9%tVc97(HWFxF%%7xl!!2d@G+&lTeRCY~aW47@
z5i6ukj>V5I>yjkZ&*Hfm{)0s=mIiBD&l&tu_!)ZzY$fNe+mu8_W5mrZ__kVy#v&g=
z8g-j46|FL`A6>VkJjwlx`Fx(#eC!42nt^~Yw#BLiAei`Mzj_M<42EibPHygiXWL~f
z)0XF&E?_AwnN`OG*b;`}(0YUbgf(&?-87kU{zd!R8u$q5#WQYLk3H~(AL!`-q#~on
z{@-_jAN%hO+Mfq@3TxkpxXMS+?=kWz6-&54zi9M#d=v%eD?Ae!33i3Qpw4lYwEmxIBQe?kZ7vVI*OrvHk_OGBfQ
zf$mA9bD5Y4$-b}DM~rm*EHpMRut9qC==`%P+*)iPZ2>Pvp~l=LPQ0J>sQF(m%LO)SUR7{qNJ@d@QOxF^6TF5JlMlUccNHhi%&ML
z;XJM(rit{-b}!n$zooMkY%XX>&RjN)kVl@uto~5|bcN4#U*FV(5qS9riR-|^$BGp<
zYotweRy=373}RPt_+~vp7931*P|qRyz^iU^WOX2!NxWYBt71mUuveE_Ok>`BziO-i
z4Vu#abym@g3ryl%@oTPUGgrnqE3TpYsl6d=tgu#^%qvFrcYCJpi-}eDNBv;%rd=}T9f5R#D1MS5X^fNxL{ozJTD$6{(Zux{$*
zFr6}2vfQY#AC?k!H)WTL{Ts<`Iqer=VS_A7gmFxP?*j_ieFDV5v6}9lV!6ccVrB?w
zOynqRntOMm-xb8dsb{-BxD1bw^$aHD-EYTK1)6<&Q|oJRCw7+N#5Lu}uu&uXm2S(j
zx05ku{=FyP9}_7r&E7RS4=(4AR=EL`dZjSM+%(Tu`MuFc8WLmN;9akcdSrd_I&gRO
zpSBa3u(9c#qc7v3V+j=s&n~B7ILgl+H+At9G@?mBsQ6=SygitpEI$Uf5NY^tn4DGt
zjR}xjX7MTm`I}m7reZw91Q$@bSfCSCRZ@!22c+cxH^=~QVo-avB+hH$I&eBUWe~1&
zvtDhM7iEkJe7q#iA^nSU^oP#az&zz0En!z0y(%&}QBTP_zK?Vi3#5jxB1C$(?O`XA
z)`TcA`+}tIs$lOW`dvX~P)v3NdC|%4SRJm}F3rIb#8R4GakK%Wo$KflVjbIZlKzX9
zp+`|Lu0`L^De#JmK@{1s4=NMd;HWv5yY~-04V%h5VQTapa?5Lmb-}>_LQF9j3X3hhr*skkaU)if(^+qx!!
z(&dZ#1QbvZmxfTCkbM2GvO%^FkO5t0`1ym3%)Eix?v;Z$^^u?s6|2Ofj*=_-kx-48}YR&37#-Lmp#A$knaZ
zcd+HzJUmHjDqf)~DZ2|=^R|8gclX>WA`v9hu%m^Qm+oPva8E#WSFUvE%Y*M7qwBz)5CT(zayuErT=6U@rM}JzVh|kJ!ez?_jA`d;dsSlG0g_|
zcLiEqWf9UG{JY+$-k;%FS3N&RqdA+fp`!|AD3so7%Y6)1Q$?y7Sb9HHIG6^#Wa*Fp
z+W1X^{PbFBQieqnQKP7qt;-wNkH^AzWkt3CTb-AZ-sQ6{d|`yy@Sw5W($9A+#)hGV
zqv2=;2?r_`2a-&`&wt0U;cu@rer?L+M>LT=23P7$9nsc{YJcF*Vbk+gUR~3)TR27g
z8A{smmbHLjq=;S47P+vz$Jm*l`aO)wTLHlmi<5+j$+f=w+vNy92frr?vsPrTVHPq3
z#d6w&ec$1YjqX(sR({H3)0R<&kGX0+ZZ9
zwsr49l0)0@6%R^-lSg`zoXoe0rT;M^fU^ttGa`*gxfaMm#4OeS;2?3O8c_
zyYr-)?tjbw1fWMd0A8xR$~VA)tqOAi$UMFjuR&K~K8x;v=*BZ!9!OOFKcPZf!Fb?x
zwg=k>QDmDZbPhpG${k7~bhz%ItBIAM@Pk*Bl9I2AssD12hDpP%t)+k7Ef
z=S&2*%2`H;B6Q+2+{GdzXIHSoa`clxVU2__KQxT5uE6=pN&Sb;&De($CjQTIdx!d3
zo8S@AU^#CWQ+#jm5No}Prl{g&;vnH0Yy%~t;z@1##cW%=FnvEORW)2}0eJzb@7>G>+R1^F^OKoZob)<*0oyAYkeg0Q(cJgChSC-&;*PxAcOiEYQ!!b+I)>2?
z(`O6wOa8TWtb+1Ap#e_2OXX=AvIUNN0^j!dCi0sr^1|4Va?@P2>0-iIi{tBfsrj5n
zm#Rh=CKKhU7#0zJxDU+9!T2y1fY8%#lCQ5(IG(`ARrZ4C0_rz0mCsE;!wL+!0+@Ps
zbsW{)MMcaZyfXn_PqT1bA+hqWI$TOE*|1uQ7+Q|Nj*-33iKFC87srizzUhovcP137
zQKv0v`_hm%3+z&h-vfUdv&G({N-exE7Q(*E#K^bgCS&mLVP>8doaU9Y&MjJeA`aJM
z*O2yI=ubLEIJK2MAR0&w)0Q5cz4vQX%sM*`qQTDL;ZND^aoj!87Uq>YoB-q1
zW#sz`4yuK2k4^NKnxvWVePpjaL~yk2DC;0v$j1*b-^DhM7`4zbtJ7#~`_U4pv>dZc
zqVx6gUHcLO*VS$_)rd=I!GQIBp>l@WhbDHb#{KF`=W^rR!4lqRx_W>2i#iF@1^M=A
zTKmXhjY-dr1zvKU)-I!>AolUFj$taVmB~BK=ou#Utxq*8UnGdqK4{79$Lnkdk8v4A
zt`b!zReqnMasub8sDe`_6=e&(f%lud)`@?U7oTljw^6355Sz`Cu~ju0fc
z>QJg9ikLYohNH1}ZYE=U@k4jtFaG*S;g#-gZRDY}&
zz~Kj_Nd1tPk>#h^t^z3(Q^iejQqraRX
z?{bU1*%L&+ca%#}!SjyZ7s)ix*Db#KnJKSVZI$L(^M>+yA`%E9m)JL}zT$uf$vDsQ
zb4L-`_P@Bo4fuWlPA&CTEn#6O&mM?3uM_?||A_X2JFa;FvLq0mfW`snzwpGE4VWeZ
z#7#@#)qmO!qm=(&gkq@v)7cfW@?&t99sUXo(1$U?+~Fjfi)lY?GcfFP#KbBG5;|0n
z;sv?dY(~fCa5)Px71?xJF|&8~hms!=h6;ym7g%i@hEtGsW4Ul=OnXHctWriUsg1U)
z3pW*t*4ol2-g?}dT0buKqv#YbB&oiR8*@9>aQP<0t)4pub#dQ(3K;e26`c2S^U!Ij
z|F&M|Qk5AMF7%yAc3pwCYpYPJzF6@MZRV=Vg$9WpqmuF)SYS~B
z4y8OTFyd)Al-ug(I3ZEb$G@sOGAj!1p;O0_Y;po2Dx2tENt%n}?6^73!MGNr->1BB
zU$uM-Zt0Dgccx6Mn;(b!#xM(#Q9_9w*gNpZqZR8Gm{vIX1kbZi?y{pby0~ADfWAl4
zD#eux@l)L@tQe?kPme1d+D-THHt24sYfsaUO8zrwF$i0X9V!lJ`KG{sqRHVBJ1K)=
z;wLLojp(AzXUn|UclDAoqg^?rE`nwR*IyQE!c30}uHVS?mZ}I5=dT%s`0-HsT^!vQ
zEwxO=yt|lsI-zG^&-$XkyW1MS&;={e$te8Fv7oE!>zF%ph2^+Y6#mctgf$|IfQ2Y7
zI_l&+h($fBGkX1dSV9U|{8wkBwa}Bi=T^Nzq5&0cw9ytJ*{EK*Yx0lmjRNzJ?uekW
zaMrh!wPW=<^z#x>^AY3;Y30`g#8L(ln()e*kkC~-!%DWj42#V&s)hCyTs)RCEmMet
zVF6fgWYsRSJIIBBt|fldbEV(81gGw~68ea@IcT8$;GS~|3
zvr2mv9X*>_uTxjqgs!)1pLYA_)r%Tr(iKfni$}yA+~j2qj4rX{c;YhgT_IjyyGSj*
zO3a<8&wJj#W~P?vthdn^)mjxmqs28!10h
zV~0nC$;!6<<#Zs}K0!j#p7U;My}!mc53Xg>hKWcFu{F(O`R-~+R+dPGGnQ7s+YKJvH^
zD2ciF{GK)~QFfLQ45U8WXIDS4^aPn=#UgvlEKV|p1=1T^u3dov&L=#8_L#8bA=yh@
zpmH6TIFJb&+)5pU!7NUUzOX6Aa&w+?gM6h;-;4eH^!JOKHI2XMj3`3kga;ZDh3E^2zpgF&0NltjLtc2*-57}
z<|QMx0qQefx7*2@KXZaXVc0P-DP~4g=b#%BHzy}2Hz#p|Yf?qdM-t0ADW@1M5zsl*6g2Xzw!l0U2=-wOLe
zPL_0zTUQ_1o_beriH-c;2TA69XbJ=dvDZ(YmO-BQccnEG_^*Q`*AM*uU7`w`f?bX}
zu0mICPMJaH%>OO{{la72!51&
z^X{6_0ZQ+hXK^l?CEHo_j{l`;KtBo<{jO{9KC3hl-jBwWr#EEnQ2UMfBcs21Cr^_-
zm(5sfE~{&m(KDNo*$r=_mtP?9FJlZjBpVRNoGlh|%c=VFy-g8}2G@!V_7nZNX+n=O
zE-;q)>2u^7g*{0!*QDdYfWctF=dc^;M~E~rNn^3;=WvBr`#$PDuFWFhSeAGn;LhX^
zX>9y{T}!@s7RNuKg5F{qO_KXt${vHKmx>tYivxAqens?Ol0YAZTllk}qy@3p{g>t}
zi1UW^8UEllgb0?|+LgC1CQy>Mq{Ypug4dTl(P3>f9+(=`#dlz4iZ%%^P3
zdPW8(F6&tI-HTHdVmn${YE;qKBK_A}vxE`-*l=?d6G&ZT6f
zTBjk7+0mH^E3VaD!KaWh?{ON
z-ZRC&n(9hwG9m>o3vhd;+IGZX-o
zhTN(?#la9kH=ZCji}`1q)NRyL9&8tNyyV}4(+crDSA81z#Qqh*Z}q->)#L91una3v
z$x@H*#2+wWJxP^H|67{RFC7h*Q5`|D_AnPXV&_E%=w8-4mw-(82U`&jNag1B84NdS^+Ai0%PKvye9OUac=*4bX|>6K=Q8e%&3
ztq6r9Whw6z<@>(iHF=D4@z+OWYVNTJ!BR2t&1PCEEl2w9XW}3-_xBQYIt2vk#Ce6-O}#vA-1=ZnpwEg`+w)iMQ)5G
zl)5%2G6Qi;(csTkBsoU;WqYDoMzdjxKHFH%yqRnNn?yyC$!x;0e`^
zxp$X-#L5bAj^Jm5EZ+rF37>OR4PDxH^^x@3KAv{k5;Zcu5>944KbkYDDV_iMW7(7Ln+;Q7dk1|!Y
zR|C$9L*MyjW!%n|2ZWmF_kMg4HE&)raG7Is_ql&;gB|LHiv=H;
zu7?`l|0cij-ye4V&6HXHc>EI?JZC=brq_3)!IpBOZV_clKGU2(M0#+*Hi6kYf!JY^
z&Fd?8&PsQ$paWJ3LRI!&*QTjI0&`Xxk=!SZ$0V??9VG4RQ{NPhn<%Xx`^k1O+aAaoT(H
z;odzQ9dL7UCpL@A^GXnMu+)vrLs@aoLS5cLz3+DN2oLd;?Ju#=`Xvf#T<3)W*8>S*
zWu2DxjHR-68L)kL7oX{3M#^@x=Yrf}rlE2-@r(Ntizt;Js^WKVvqYeTD?U(0xW9L0
zl#3VpZnCYTAe8$yoZ;cIHpquJ7H9W5KBEsJx*B2IJumVQ-9|3HjIJ*s!%JJSj+w}k-Q<0^{Bg|1V=;?yj{MI|cB^Fx00m@mu8E
z9`T2s(tSMl3LtzKvOhPt-@=Z3p{pCu-YOTR2h*s`&)fg$r@z1umDEG6@AcTJ(W4e@
zW@e1#U-kgN;Y%yEYk{x0bV=2SYH8m6pJV}uNK52Du5Ss>b~9ke`R!rX6EM`qtcNTC
zaTswaBc0J&S^JNzQ
zb?;T@(I+f^WC>=66}-tmC^^_~NQ6-uXA^&oYqKNAIh}GLYiU#)UQ;xv$!1%&y|;Df
zJeL?R=wnwbD~(X9MO}+Xs`ZIskw^)APyRw8LD&|4&3U!gYD)eO?gzNW0b=+WvP8uf
zd)EoYZNuO1426efAIg-rT
zE2oX{Ma*Zh`-lA%dy@=*CxRS!PyXr+kC7vm=vYM6v<3$k`mzeUGPtJWvjMwsa}tQC
z_0v1g20>WDbXLOaE_Tx8zMz&eDq`uVq3wop|2!*Cd0EHmjOa{Tr|jBQDyV!-Or_ea
zY7wm8qs%704=&lsw$X)c->7?HuK(C+rKb>c!!cDX}W>>D4?z
zC;7{bCt9P^f_FkwZ`tnum~dV|MTOjrJloKqqH?{*#!xIqSGMV%uw!=S4W}7W<$1BX
z#XRLthvLw;C!{wna$*!(v`H;X%s-+M7}=e#CZC8$KQ3E#u-E$1cw<0#G0O|NwZ0x`
zH^v9?^X3ZY?1@xq*hu_RwO|1??B@G!abq{8T`5|ODbA@r4&goFpWRaI9aM*3{=IL^
zTx`iwbd7?f?p00Nduiy5aURS5UFJaUv}XU+b&4_{vh@tv5ahcp=GTErdmvTU9nXAfEHKt&>I%lSJ^
zHXy3<7pb$s7)Zx7k^43wV)mE9QSF{p5xsa*C+8xk1X(((1_q>{(#w{m8M>TgQCF#8
zV;?(Hga)P&l)aX*RTOi`ODCL*^hVdz1pkN-G`+VmNsyIsi#CY>Zgu+)fzCH3qoE}S
z3j-Sl>&orPm{s+t$xAAk!3d47caw$PgLGVX={n`%x`F{qRIDH9{QUGHzCMZ>kyS~Z
z1Gk4;=CG9mnT?u9jMyValf&kx2V|6wU&#aH>Bt^cxh#{xpxTP`r`H|y7XsVN7jFH{
z9kihxW8WxwV6=2>AO>K%9ym2r!ui5ZVBUar)0MCHi3bSU3ewwZ27<(Q^yk4Ghf0D^
zZ#$liOYC7spvRweO#coVGwb$GBcQNVX!~lbTf6TqCTyK~yzm2q7wK7_??a!KMXbfc
z0$>pJ@5ARS{-r@izH(y^52K*4>)l05z^tNi>`5GmHYn0rQdOT9?I`KrO^h#?085|$
zZ0}m}mSF6O*6X6&7u_BNC2*5oWXDA~YcXmPN)?RtOQIY^j&Y%;;8KfEmhMln_Ee(W
z8}Q%2V@<2g`;9R9=fs+6^Q?@FTi}W_*9nusmQ3V8?K1v~{&f#cyywSq-bwkfG&?`0
z6`y+~vD<;HjU#(r5$_vTuX7uuat{jHV>JJ8?*_X0_mywir|~|i+W3e{j5+y_C6`XXFl}of#92#z;rbI>X@7ZrUA#^1A65Tp*rnH`HrmSK~
zK4qSck~!okUc;)*-;o4CRlg(ntAMnaGGs87c^-m_Z#b*s9+56W23}M5H7Zy{ciNV0
z4!PZ0%QJLiW5DxK`Br}59{tOMG%Ca@ub?vTi$91%ETlfc*^Li2oAa#VvbPtk
zqKoe~xn;$E&2!_5qcUy$YChc7AQsiDekQ0b!nTlY_F@OsGwuwdB_j
zudccz&?{wg=mQQbKSE<1R3Ni5LxknMac0ZbD&VLo8YspJxxu1yp4b^F{%he;L2z&1
zfGC7bu-ef=k?~>!
zo@y<>km$^3%eq0!kIzd91-4=I@XIqU69hQHY&>t{Lt6Q2{~rCyH<*;7Fn-vGZvf~T
z{Pf?#@LE0vC>=xJx=;j1~Ed8I=^1QEKAPTkzr8PeB*3rWVmP8&w>kso;
zlJ>p`qP~Gg4}@_3>6y8r_k1f}P@`UXP=uSxYG+Y>Oh_9$)iq&uJpg-yd{>3TlZBHQ_w2zVu_s*(4d?jUB7crN)Z8N(j=;
zQW4c#a?yYGMUk%$K(f|1vmOamqMq>k_pqH8In)W^OQ?*A@8k4z?83l8N4v$=zl
z)dM-a>$2yTL{WM8J}dAe!1E(po}_ZJ7DkgoG6BB*-+=Vb@J(u%nBAl=S)#uu6(=or
zl{J2Qv$Nz&+g^;p+%XZ9KEiLoMK`ck-`isMu7Bl7+VoP;$5B-MjeT4m(T`eQ=EHlP
zp?|9+3ui571^Fb1VtVFdYI
zu?UwHvCwqa7**ELd6<|6L|pdghubjAWPw-&qW7?E1D(CWMOc4T_V-$r25HZ4Y1Gy|
zzYoAce|)sqcI@H7J=_`Vu0Qg|$6uI-aKJsn6@sMOHEmC}7QPR3oX`lZC~mgLdh@K#
zA(ssv9`rvEN2T;SLcN_6)7UX2>Nv}xiS(B>$l6#Zllhld(>IGVUV|5`!}V=slcFmc
zhC=5aJw8jA%&o=?MN_&P0WI@inYK$*_MZ7mqy)48xh
zL%&B7hkX*BQ2pID>h!rE@9U$*w&_uAkDD0gZUd|%R8ML340=-}t?HhV1@Xta&Dlp}f-8n%r
z#brpntgVhIC9Z>!an*>YwG*@S-1~f)KY3}Z)K*auk>@yJN56L-cr4+Gw67c#defc-
z7!*uyIlp*!2KaBSO{t3g0qiovW{E~27j~ILB>4iat;}yPf@}B|&Biut?uQKlrScTDm)zLk;edfTQGPW14f;OWIu1!FZ5CJ71jF
zVcd-tmdhe00zA}OPuc+0-NP&KyIL4~$79kH*C-jr(8L0Fs;kUSZMFe7YTq9{4pun&
zPtqGTA*IK!lIV$`$E%>^h^#EVo<1JnjS9yJ?;
zD4RyV-An_;?)dZ@!A@ZuBZL?k9QdSri>)^pv;3`TBw2A^B~C&q+*9gosv+-h&HFx|
z4fqu4}+b2yC1-&;!Lf&rAkkO
zQ*Q6j~xnM#rEPj**E#iaC_2A>-8n!7I+`Fx{3OYBqJpjTGc
zvQmPH=*D9F2W=I5PMI$%l3wM*)5CqiJW3tCCE9LgOACg~(M?GIcv63n^;#%XM|HV|
z&utnF#ltQiWv#<8Ic
zLt37%vx!1EO%hdMM=1M|pR|Loe1}D;MWE>o+R70=#P%V`6`0$)DcZw_nkz@YCBYBEd-2{`fL0X?(w*kicp<`Ei%wTYAO?C_lJQ
zr(I#fg)?zF>d{;DFLnAz5%>u@RhkWnYH}|HBZdYM9FjCke>>9M*2pEU7A9|mW
z!}?%nLm%7!6p|k~EmyMQar>yIVp*CNK&(Y8Bf!loSFwEJJ;#!E9&RS>q`NTfq9KzfM1|h#W0#EG
znx4~$eERz1m%=nclGi*|)UWhCiQ!h1y{z?ez(KW6mRIbjD4x$wn1>cy>lJbQbIb+Y)58Wfx`_g5R?ergg0V1em1Z#ghCc9Ar^&`x*6S0vbDLSrr=ObE0r%&lF4B95v0Z}69nYg%1&_})B%Abfj0Wln4r3)}AA#bNr8ucB`b
zTxg*T-$feTtZa^BoxJGPDcG)}tQp`;OhQ?7#j%{-Z=znQ6F43I7W}l<#+Vvv5!Wc`
zC@Wi8-IIiH!@1RIaPjc#Pm+96C2Hh#O1IVvW|*C)NLIb3#la!W6@g&8X*t8*h%w>7
zz~L?g%9K?dDZ}m};(@hZyMP^;>CLu>J^JBC3cT0VWOac&umcy%ki#C8w!47*s1DD1S+w-0!u
zO%D`-XE^0&Uc5%maK4&L&K=2J;5pI|lc1RHBQw)lLJa-L{woLkN^^LWPB<7neI2`
ztv^;HkDbi%vyqX4JM{)DJR@wjSq6Nnk#&PRSHF3aZVz?e)ZA|;h~XtHB%o|RT!a#3
z5b{`%qU5CD*o;Utq<_|tcIXg&@xkkY_+*1#9Ez1C>KWb$I+8UFWbnj~tY%UQQ>l$Y$L!z7$Gxh88cFy!9jn+fLN8ny
z^8}EIU-(>kKCgTn-p2I0${A7u->PkstidC!?lE^cdMZ`|6P2e2v#U(PXTl_aX$MjDe^%
zZjD^1)V0fQW@4V^H}uLuMd%{B)?2^v^Hu~ZU1JP4EmW&wI{V@Bhzw{`%bQ
zy07cHv6PqN*gf`DGYw;Ey!?G;sB%iLm<1I0dOPpHFk^
z*{8BDE;r`hWs$aep)Tz_t1i|ece|?hRC1GWW;Evs4Iyx<;faLQgk@)nf}5E@hE?Fu
zVvu!?0Mq$Sa7cF3=cGvxr#I`X&X;1o?B;VBglozl=$*k6;ro{zY~#l5&S07xNK?7_
z^!ouEh#?Tt<uOn_LeGNOJAw{v`lRjEjK!#O(W_u>z3V3n36ylm|Z0K$9DTp
z%x{LgEQr;kTo%b(hfc(>K|>zlM~iuZz=T$FHbDB6csBM(DVfIg@!xaeV*2g0o&7A)
zyJ_pPxgCv{cz#9Jrk&SzYfvXXCd=DfS>J~azuQw$%au=9b0x;c>x|+iEA*-{^KCax
zI^aPQYsYo_+dMa9XBRqYgCo(`NCf6zNCcM7=^cQmu_8%j&MHu^`sb&stVeb`rfyp0
z;u~?*7nYCOz9g-RHDvM(Fu}o(WCJOSHetE;BvlngV~CkWgf$*@w)yejiv;J6kgMzB
zdUAY42)XhpxyF7dq6Ge3()Tzo)&4~tnEm~)c;4&VpWO{+zFOlXJiOSbxZL2qUka1kpfXq%+i)l
zryszcB5+@n&D*0uwh|JXnxB68JpjhvWFLl!{WxS)B&pf#AfCd54Ea8>wA3H#@9oIz
z@9*<*E}6?O?xWj?Uhhk&rZAUtKm^nh`^e}pTr&}uUT-UX+DVcW&CJ>brPwiam+>s&
z2;x({dkZqlU^YJQ*s#);H`hMLVMlBVIn$JC@DPzulA6;W0lN;NN4cS*WIfX)K9-xd
zX0rjZ2)Um`gO;>%Cc|#ZDXG=IET__fYQzOcqLS5By?e3?^I+AZC$M%Om$C}x7sw13
zo1JClZ{h}$4lN(lL_wahvl+mV2|(r#bmsn;L`PJkT^Cx3g`R+L~aDL1M*5IeifG0X}Z^A%;L
z=Gf2ik#{P7El+A%omM$-u%p24L*_pFrhCKvgAHjvYifTnw2>7u?AjAvsoO}W?m)fn-v(Bs
zq>q|ue(aB9P4aZ>kK?)Z1(~tEPVuFeN1{EPz1Y#keoQ;%ZNBQ_swpY9Y}UW13Ii>H
z+_I}!{*chaA1wyzKT!Ayd|^HLd4DU}wLr?${zYKCS4SDV)xLD1IxFxE6`tklB73cq
z+h%ZBT!FfyZ_W_Y-hLRmNC6fEdctU-N5-ycfy1_<9^9M?S``SFNZYM?{CfQ0)rX-uK@>sH1J98xff1#ymn(6(ES;hl+rrE^z39aayXB3ZX$ZQ(j
z8pS*%Zi=31^fH@i>AasJqhoBxVUhbM`!+7P)sj
zFqHKNN(mKbH9iK~K4Pa|&N)%ztd
zl~f~_y6zkWkKTLQJFD4>7Ynm$DTwIZd-PIxXtmG!FX31J((-K8gs`*Zya}X8>@}&@GCjZMNxh-`$?J*LUnBJn&tsnr
zX8_@FWqT+`2qhXL8mf2Ofd1~!5
ztc`4@PDt^hI^D;K*8FVMDCu}sd$pJ3l*P$72FWSF5*&xjRwGxws%CArr{`sdiIU#0
z?K|@dsjyfcTbu>CJ}ohiC*me?nHRuM)GG2!A{`hf{XFdv1Tw6!o7nDrXT4hv4-#UP
z5@jp#LqbOCqi~2{y=1_C${VlEVp{p%B>IaH8
z*Penk;@XE&Qi-?E2%VR4T{a^3Mz{mG+8#8ArN0hP$=$>f$({-jS1eRs%_
z<>R)$9casD!sqR$R``Ipn>WPco(l_fWHyBa5A6sW{L>wrFZl&y{W^~S@vc9~nLlN}
zIE7rv2IHhdpkz0%qjv96XB%$!;$QAC%;f~Q`^D|c1=>rZE8NwEGV^-FKT!{mZ$6$2
zTnto%UjOim2tOL<*LH4oadz!zE8t%AUlV;4gl~oqeVdfVi|CyO^3%+;54XL5xqL(N
zFDt(>ZlxnCl_Ei56~JO%52z*P%a-t+4gFMm#S($Ct~@Of^!3sPIFqgCzg)OT;3SZ{
zdBF3#UFj@D#qomSvA}2NR_!(4bGu$+KHqh1qzouRvdRnv&d$9)pBbZJyJ-2vY3uO}
zy8bnMB0=mu=q;2KzO!lTRsD#f&m3-FxkvR8_GK^#QrIPYa{n10|L8?cX?I>DUEjU>
zq0z$yS!lJm^sMde`rYuj^>f%0dIkGujbK{QKLz?}&T{s9;86%P(eBP>p?JPeT~qf7
zLrhkmU7&`sA-*K(MY^|56Mw3r$V$b;gUzqxeZ1IjiOc!>vKA$#97_FvGVe&3E%dNC4xl(Pk@fR+RBB|rvqZh;c0*jBftcAbtLm5;Ia-7dV?|#t=
zlv*oQrg)~mGIn9s>Yk2apq9y@vp(_U+6h=Y0v@#5_=H4oNCYKF?dH#wz^|uu)kWbw
zgQm&qK##7Ld!d8j$1VX+T!LnPZT-IZpwEzVH~7PMNs&7@tFCkQ`t$T4XY!ot8D%qGai%ryk*Ep`X)tQ1xgFeI~l*I>Fbl66k!g%Yd9$Rci&Pv(pkBF~7
zK6gGvc}a%2uvf-6R}V~}q<3>SlivoKxZ8#*NE7N}3V=cO$*9}&`UERQd4t_qo1EvK
z;+N#~$t~{FPzB30M;7xUAxqq$xoVrMPK(~B6^BLqgLQ4RVaA8FwWfH7jJs$)YRC95eGnW+E-{dZzoj3jYAm*HgHcO{kpaor7s4=}5
z9}TGxcw%w5IeksWdF4NFg1x$cEbCt#>hHT>uwJR#3H`HKuOP}*4E*@dAl?lil!P#b
z5Gt;GuBmy4J|+Kb7^6A_x~`9!kpx{IC_3uDj@z^ozN;0;!tO;*8?Fz13b>lQV!Y-H
zw;lAMpIm?bZ~W|frC_{d{1KpNga>cV8e1DH(&6n8c(0bdoN5_OxtrF7yD#ZWc+ndr
zdaA!dIF3$nFjvEn5@i?e2Pb>^31J%Uiv#W#cewbn-Q5+*q{NqWfSqDOC
z{Yl9{%CY!HzR78qK{q)>(BikpuB8+1L4&t
zBxp|90i|J#GwuSBRu2D|YD)I-&B$GJW
z3+gnxJ>KmlxxsY~UHf^;XXIfny!D-wA@S(td2#a6SEQU0a78iWlCv;+r#liohlug1
zZp~hlX1^7n*fs4w^ld@hPDxt^1vuV3624j@XgkE15pi$U%p2r5bc(W^L4;j8FBFN5
z26sn+g)Ln0l>wYXkcsPq2oJ^Wj^NP324#{{@#IuzXhq>H%qQGCA~N0KNsk8cqB$4x
z$)@2|oZ8aq7i?le%Ait@zZ3K3PhVbBfWt+$6hwegj_SvaHGjC{I%p&}pHS+){7%e8
z5I$8cVq^JvO&D5w!-}*&I9tPhhxc=ePN1jM{GK^6D$q^lXbN0@@~wIwRfGI(h#tyY
ze)LY9wdRZSr94?^Cbh*wQT7`Tt+HI7ZBwdW{d5StTp)0&xGTfa?rkw>G;qp@2Rtb9
z_D~S{U=avSztdOBmm#7gQLW_QSRqbGO}xZIP&75ZWC2N)%%GA#n{y7fwv_#
z%XYrg$CCo#cJVPi=Z3J+A;~EzrYXrJD;6{)q^afh@so4@4Zy74zm|jl)kOaj-f#bf
zw;zQN0NtE~Va}Ej-+#5{m}|9@fNTDLSKP@cQ^w?@A6}dW%9D$`vOU{M2c~wNk});h
zy!?Rttn$7)Adp6o(#h7<*$Xp#5$|=^p>Z;-AAAFJLDPD~e9go;Lw&@7a76QObLrQn
z6#i6`?gqRfY~gp#DH^g#T{oKb>t=X0+RvGgEQjB#8_(S%wePdEO?gKg_(1aO>K>dE==%*JA{uZK=@Fv}@6Goxq)I`u_1>4}Cyag0rCb>##wJObOx~(`QBUp0`4i
zn{zv6q|rlimgR_21aXH&qX)!i>eswRO{TALhMy-UA}z{08#5A1n|8@533w%0k@%g-LfuEH2{>;*V9~``=ULO^
zYDil_uG@V?#&Ub`<(BswNv!~PikaLd_h9ej2F!RY(ON6kce5ZaOW+d{B>`%+iugHT
z7Xmr%aIUJxnKA+jHYz?7?|=Sl?dUPT#9SP~0d83N!ujC*yO_Uz!;rdleu-pO#WkaN
zl&VoU9q_>G*Lhf*ef9e(fJ?qM_Mv>bXN%SBaP9$hS5T|yEO*o8IT&3DPS~1+z3w2J
zw}zZC=5oWM$FI43DUPFGmJ7;L{tU8DlP5KS11VO-yFY@M<`tpDO`s0BQg1BoYX5n5
zfviCNH+OTNRNJ(t$A7z}wxBgm<@htk2H7U=fmIr&Fhyw=*jNZ}BS)CKtKpG{IE1yX
z>t5+m_N3%vF?QcPrP^nNsp?c9aO9wMT4tVajIPF;Rc(AsH!T&AS#&^XW1Mk&dH
z_c3cAU-bQ!{+3*YEL1BUrCpPB3^HT^rc8^n#y6&ATEH!Ze()KI$WJ%lj$BFP6`lfOpUodV
zCJ|Z%q+;BaAb{1IN-o9W;wq({%=Mtm=VyY-+O%`nUZcCCj7)I$m{br|cChJM1>8EAmavL<
zS(J-Mx&*QuS=(RW5fAL=y5s$lZtivrU;YRcF~M0;lVv^u+}%nSE*4p9y-L+9zn+6?
zJNk^Ng=c6=oHaBi_Y64AcjL_6l(286uOvgv*;_e?cu~duK$Zxk7QTIu`q;DW*m$@P
zXD*6Z&`^}`WZypj=;}FuESH%I=}|wP^{UR{c;Cd_zOyP!>e{v@)mmMWfJp7=#4TE_
zjvS}ILyq5HY2eHMJ~Q_r2!|W(sx~rvG)J;@(T{8w1U_;BSE*4C!W=mm2Ij7`jsTe}7WD!!l=&QO3or4o9-CgA$I+7P4#|1)UO!JDD;DHY
z1M?UBp7SR7j?XNJHUp;}t%G=0ws-}0x%mU?x>I+PveL?DKgL>SNH5WsWL$rLkyN>3
zRQlQ~@V=Y7JNJmk^WOu=kNe%^f8`Lj1Y*Tf6mkT*nCboig80ni#}RVk>!%#|ydB*`
zlu^FFtCkPW^VrQMsTyS$UzOc==heuDS4e{w&p^}$26I}0dRqDU_LxAaCj#GPr1#pD5L
z14acQxi>OunjXGqt4WC{&^=~b%d(MVO-O)WT6F}!$uG~s3+mqJDkd(|>PfUi1x0NH
zHr@R*V7r{r5Cm&;UFHmc3+{|kZNQ#cQU}yR-cVQZ(!pv(YGuD1C&!l%H?h6
za|uU(K2Yop;$j9;u#UL+^5t*ao9fk8A_cM!oLXOf5pkGY<_es&{N$px8GY0vc2daG
zfVyqcmL!x(<$mCDQ76=^<^1U3)V)ALLukQ)#*|(Ba%;-Y4(CULB0}^?wj{Zi82Y_-
z*K7dyN(kj+zx0hTu-OG3|DeOrF@w3B@?Zbz4x9#vg+?LsHZEG&sWdYz|F{K%WOw+8
zxWm%AK#abe)@-5c71hWi=7hwNt$D#fPTGZ^=?-sTr~MzhfhD~?m8}B$5S8Ut0kRcx
zdpm`P@A+YOJOOW-c6J^ODbE3x%Hi6tH;$ISi8kq*0`OIGv53~4RVi5d*~{sM8wj71
zYP&nu#a7X8S3mBU`m{DLsuE^dG6=sJ?@6nL5Eh`!kTxKzKz@}%(s$`pS|3w~__1Hx
z%(zmXOEJpe1}gY5Xzo?r@uMGAg>Aylvx;qWNVnO1HzP4LTKG_0uk8=kNS?u~76v8B
zNm_X&KfU`%QJV*;e_sEp!F6U+?e9x2+g_RhZ6&{+udY5x7+Q(u8d7f&h|fXrK%i)h
zrCB}zwPx+_TX|ajExmYveG&hxZANa>??g~@UVbqhE;J=(55{9}$VUhbmb@e>JG>p4
z*EysHClauur(sDb9u4xvUWO0up9+5PwMC;&1?fVn(sQs$=NT{LgD^C~i7W2tLxI8y
zM#k40o&9ImmXVK@L}1p@gl0RLiG4V9w$Vf$=e%ZFQQo6HbCLHpxmaIHFY(=^HC)V=
zWcEsOt$&RANT*-y>>H#LHeAGtIy)Xl@_wba4#e2Wle%k#KshIB$zj&dg-=heL_rLHV@1&%XBP@2B^$D%7k);v#LxG8RB8TZu8
znL3yH{zFQMRObOPZbZ+Cjw?({S9*TeJU41uj9t@o5`3Xg=d=_E5Z-Up%vP%boJ~IT
zIw=Q}PbwNp;9=I>$Lp=XzWi(T{Ldi!KQVvua*30W9}?Mi%>}s>;QkMm^uNDQcJDzY
zwSDrvHwgnhG22f5;v7B`zcFv@$E+i~Z%svaQ)-(f{|SE2%$>sRmf+AthbC6COYY(C
zPjAsuzxW5S;%;tkd$IFpx36PgIf8%hkhwJm*AU%JaZAsVPDEBWSra)GK7US)zHaDAb*)SZ;?=H(2VZ;4k+6a2Xv@<^rK
z*KFKj66gRANtDbTxh}1TS$Cq$y_4@JyUBi2LO)~xmHoQqB~AP_=JN-hIC@)Gf}8js
zv~8WJBWoy>V#|)G%qN+G!9UY+JM!z@j6ve5!u%BDI+S$EEJ<|n{7asoR~`xkU{j)N
zY4)v8mLB!%$lC+v_oPR)?bFP^hh9EBa9RNKuiqRocMA5RlQ|Ya0-OPbx~j4{#D~AF
z=B4h>CRrwP2n-Gk+I_gVx21E8L1WI
z=frfLW$l{-ALl}5)OPqQ(}BbKR(uMm=4mzsa{sdSiXp
z_pvJIvOUj`6#mBe>dBt@5Z+K(`9UtL_*bQCYV|M9pmHcU9eG#2K@}O0^|2v;@hWL(
zhRJ68QMYyLs62SF0gD~9iYky#NP)}ROC^2ZEiD@M^@hPpD^XG?4F-Uhsqh%%gRdjh
zfv~0O(iBtuW9YWggtVz81!VO!K*{egaCi-6-_T|l8rh0ln3{goCiyG&jLo4s0)h>8WC*CbG5vW0haqpC@VGd^@~k6A`Fk$0rIBhw|Q+xW#+k*Il$4=01n
zVkXt99bQ>B{&ZVeqDt9nx5bmaK*=Dpxw*W-L9ECZ=QmhM(45KInpWF4exSfA+~W8z
zZpwcpgX`3w>~Jy;AQ)e?b6ybtBRpv{YPdcy*cK6Trhny>l>6W9!ij7I>GQdyI9tY>
z&B;j>tN~2-STb9iBN43yDWxOE$usK5z`VjF2t)xC?%sTU7Xm)C3BvD3`(Kk4CiFO-
zWiE>f)%$hcZ{pe;%17A6yTOTeYPZIPx_B^iw-is98AC~t7{Paq?Y&K5R5~!+*7FP;
zd?ULqH+5|5gO^yl3NYlcBfBevaWXd3&2rLblmotlOAk*G?`6WLPcwQit!h0({b0hc
z{Bfuy0)Egp1o%|6A=Q5O;S+8Zm80M~gLXfB)>8Um%`LVN$Enxp6IzK+4iieMW(@%*
zt1WsTzAZyp@u()kzosK};h8&vTIX(|5=4fulY>GA%EGqU?bN@X5EGvHry30BLTJhJBkdlaW55I9
z?dZuGI}r=BIV8?@cjjz0AR}R(zR;tdT6U1tBXA-ogxrf+3;-g#WFG06BxOe4
z^!wUSV!e2@z-_bN2;a4X39NJ0wTHMSjO>fh?N-Xv4Wan2Y~J7Bm2E$c&uVHdXNoTu
zId2%)E#8zv#t=TpKq|gyJ4AY*&KiGp;1NDB*PZQBci!mTK}R!)%{H$?=0F_Qlg6fe
zzZtwXY9z1a>VzYx|M>28(Sv^z2Tk!C1z|>J00U}D^Id~H;QDPNYu{^NL%1KOMqt|5
z6S~aeNWg$zi6K-<0Tw$l&{sEC%$Y#CZ(2B5?3B$3MbQS&qPkNVaW
zRwDOxY+6XwPW?N~9eDW-O}<%n5JyC~~k6T$F#A3zkzTqm0iD_gw`
z)P{?jZLP~Ww!c9zx3|7sX;@e2?n^NZWxWw-xVs#}>L5?PTBf3@fVM&zCW;Kq(r6AA
z%u1zc2^#S1+=4L8*3xbP#Y7cP7)MGT%~sDeJ69qPeFyZCZF$lfsIYFb>vu8vp6+XF
zW*L6kvUkoHb~FWnvPO_cdYMgSJs9T$FmH~QsGeqGs-O=@31D%5vP81=i+1pa^^J~;
zhPHxni}S{3oN{lO`#5I
z#}1bPP7qp62gdy?_RX3xyG2v#Oe4}#@~+l00|;ndeq=y1`%+L8&wf1X4MDE^_EaUH
zba(JHZ~cUAQ+@4e9K3(x`nMuK&r={p34lJDra)_*bmnaw8~NO`QqN~EifLoX>ts@P
zJi4>1RA;2QP`HKCUaTl|81^`Xi^BfXvmOl8Mk(QdbmRQ@RntLw-NBf9l53P{ujma;zDYj;m0fn6+GUD<&D7jqc
z$cdEH%oP4<4Qwn|kN<1Vs1~DQw6fiymEhD?MH$D_v$7d|YCE&J8s!14-KGVMcSqb~
zrSlcFAbc;W|9MVPB5!?*qM8M+_1wQAcrtREQ4E7!f*u=p0AJJ#3$;29RdmZv9#uB<
zq=+&eRI8?+|6Rn!621XC5sd*eA_jto0#?Vh+1p~}LYBD{9Lews5bL<_mv6yR}8_s$HBYT>Xm=YDY8Ov?n~|$acyDO67kEr*u8%Q
zp8u(?9gHVpksdAl?A9M=9;
zr6J_~NNEcFEGAsNpGIJr_)MbkJ#j73U_m{p4v8&*;sMjBwt}TgdXeam|KaeB!N7CE
zoRe+zpzX#RWcUER_6ySQag=h`jFxK)>OVfE2kM!>V6eIa>eUe<9Y{6
zk>DbThD(|~=)z#Wx%t*3aFh@(c
zFnG7^KIvpkt3U7ak$2d30~h&}ST0-fZ&zfTb%3#Cg`m>>jZ*uNqfn)v4gE54$$Mtfn<(^egzKQ|DFWp#
zEi}>(=i+3S>V*IXdk@UEt7sE*!F{IWNqcQ=_*8WzKr0R7PV?R~S=zso_Aa|&sqWP+
znY;U^oqY!^BIlg)DMYw__)XXrZM8nMMX>qr^0U(ghLqf(!UfkEysvQ%Rwhtu$e(gB
z$YdzT+MI8$69?k`x`<1|h;1mPktY!ZpDdo=9bsJoR>wu}5gUUNEZY7)-nw?~RX!_87u+hPc=Cv(Ecf`;6DxD^v^W{{AT{28z{PW{G05+|iIh;O=3N
zQ|LIz`Ezxc-MYh5d6L+?lz_T&`KN<6x<6b;Puz^GpL+mrxZO;A?GdqrXPZPh-P>lH
z>&v_Bco?jes)%Ndk-bW*LV-7;!yuy>zt|>94kg`gUZu4B(I#exI~C|$#i_o*@Hm$M
zfL?Wfp-)5b4zyH&^aO{+_=riJDN?RRqo#P}9YblC!L(j>sblI7M=Dwexg_RugaB3tD!nx;?LLW$W&5JP4d-O0Q4%(LNzD^;*B3A
zxS;lAHn#{Gzkbf$>%Sig_bQS&(qEjm`s?tw)q`a~ZpT8NBf@!KjBKPL!q)09_J4-r
z|HSPKK*)_<
zS^unMVaC_D9Ul00M49CYlWOm5Z>Ti8)qpx5Uc42+vnc
z-`-BlqDWe4as
z1G$?2D$(ilgk!$()2rHUU|ty7!D2r2KXzSlAz?7^GONB7ZV7IBhiiumRXqG$PV0&rXRLRIG`HAjd1=x3$59(U*ud=a4vpy)#E}o7<
zLXrOSa%X2J`}{3mS6#U$ymgY8vNB5c%NQ+LlkV1kt%SZBrZl7{dRRhZg?1M!mD;FY
zix^i)nmk{P{@S~0{b*;rKOPi~_xW1y@$zr8SpO)BJ?WipAE88qO&t2w+GQHv3e$UO6skR&RccPig-2xJqR8
zi8R9CqtT~2(ka9f3ofk^)feY=i@U1Y{Ai27$tx>z|S8Ihw*y+I=d4cOt9e
zg9_MCO{@Ep4%JCu8ZgU|-CDVK=rUH|A^e-m?)(XtNy5e+
zymJ!~>(4gexNlQRN{AcFGqHq|6)#ZkMn^e52$K#woJm27UUR=9BsY*Ea!dW)ba|PC
zM|k7$n#{PvswAyk4L(VIgZ=D^Rt_VNiRVFXhP)M8ZSKB{p0@|qh4Ay>+vnV0(x3JE
z>y3d1J|VHLcl>_#_L~Xnh%qTgyDt-W4{qtanKSa#MmpQ)>&ow&!{sP&A>Kd8PwC~m
z-Ga8p_Dj*upF2VV$#73ffk&Q9L?hk`I6e6q?(?8rn4kHY+%a=@1}M
zrR$CJaSfyOTZ|h=uDa7_j=DKB3fRampc-y_ccyF&fZh*BZ+BLoLgm-WerSf6dK+TB
zDGwkpV*7%P6MSNmt;c%3L?TzZx{&?DV*_}DQM0SdOZbFLi)$D@!d#m{KRq58fht9`
zvdT^q#MXyiNKWp5I#KAJ%!RJqYy++(=SEAyAm>VP&*qDi7tWWId&;1SxZO+_OB@Oj
zc>LEi^&|)vRcH2}1oJ=rL50LWnouL&|1%2#?s>hn8f0^tJPC9jK-5lTU-Uybr!O7p
zAGE}|7E*lty@W$5cU$Vu6Xk+GJ1Cz6uaWh(e<*4=(B}7Z4yx@R0UpAn@|Nf(5%G!S
zE6hs)RYWZhv)kZ}PT8p-KKsWHC@>tAec}t&nr@$_c2uTD-h*itF!x6FUe*MN?`4p5
z3WZw!vYvpDEGTkikqkBH{uU4;>DvybImEs_O5n^sy;%>vexi2LzR~(9&
zh>r_9&GDIfDjwOO2fFd45d{aaM;AVqJ5|L;UPisVG-s9uH+$bn+@;W+$QNl#2k7$LZ4
z*&0n!JZCWvOAGg2%bu6a`A=gC*D{syb-RiF$Hg_U>3;6JN*n_v1
zLXeI;UzEiAF_3GyycNf>YO9BsjsbB{pKpf+;m@&AJ6}wbR!1e7%}5{
zSw*Wdd@7;VT-hbHx_0JPa;DZzoLDGgCf%XQH~3J@>4C4+Bx!on#Y;#1(b^X%=fUI>
zv4FkPWH#Xp>}gWzp4W}%S=*TDFseY|H;U&dKwz*uOrxSUF23jeYZ+p*b#vz#q(h&F
z-UMhC{u!xx!PfLdA?3na{)Cd{`J;Z%w|63M0fi(UxKv=yw1~8Y`w0lwx)n-OQF4kdr?t6rq3cB!{X#x%NI0*90_eIZf-gZmJ^e5yj$aNu*BvRw$wI2NoEM>6AgIp^Th|-)Uy&QtguxP>~|s7LWlGD
z@#+78Z2uoIw)aSl_o!8m&F-4|957oHm+Gp37zxwk&hL9**cW@)wE>)Pq;J2*-#-Q)
zuqc79qWur>b)IsH9oyzw4x4Ie8m}+!ZuO~jFio+B|M1jMu&2pYdhAX-bxqNHyf#l>
zS^OD8J0`T5-qZ8E;GsVxqTGj@L_A{}3O7fq5mSy#(5DI!6&%q`xriWPw)9(15-&<4
z`kmWHUTzmGIK|p*6ECIxBs-_lySuktRZ(46k@;D9x*G!R=gL>%?<^*;I^M4n4Z%
zia$KFq$y
zzBB#3)n!QKWBL&D&U`Xm7ChWX^ucEUpkGqBJJO9vF`{P6htJb+vMXcTMZ59n?EX$e
zTh~3loK>lx{ev044E?7Y^Y1dR-b?Kuh;fHet_$g-&Amq=ZdBe`*qif3TGia!MM6w8
z%T<+4ry2szEVC~{vj*wfL0PCfKiO@oc=vTfN44_!b6LC^I9G3-hS$rXac<9;UffM|
z?s}vgK;C|=Kvc1Oa+=3`XkLUc+U_?ZGDrloeV`p#Dn4Zn9ND=?v3R|Z+1RrwNN
z(<;55`AHF1Q(yds7Q#}aRPUn(RGqh`+UU#zEw&
z9rZONk#U#KWCb~ut`0#b`{$ub-OGFlm9A(mMSWKXdpnj(`JG=FkuK)HC$q%jHsVjk
zq*4YPT+fgp%1)J32wodhd1VL$X}koKA2(;eJ>oFU-_er2DtBxHZZ&oHPBX33sBiTr
zTbO@r+LqcHUy7(;__zG>|8C*`r^1Se#*1;ZxxHa&en6LOSf;*=?x1)
z<6j9Dk;03JY?&O0N4Wj7!d-zLG7Qh}ZV)$RZBQl8-fAz=xOq&p$gzO1B{TN^@mva?
z_`n;4{`5wE!^?L#ZnYMSe@~-2xQx;SJyi85G$eW`UL0`F;Ze?`uR+xO3NQO19W6od
z14fY{rQFIX6HxbxGDKftXLM3ZRMFwLk)nITla?t8$V++@|3GcS^)
z+iKUGtDb5HI%Kg>)l_-Qk-u&Our}F|Uh2ZMG46SALB~^VlVbOh7W#6sjnLT>-
z**01p1B=Rw%LGbp*>d{%iY_-R3E<&-v`B@DbH1{uhcSUm8z!NeCL7ZcL;fsU04b9!#DbTO1>cj
zcnT~L69v&Ul%I0JJXd#)?Zd+o;$WXe!;V8QnN)Zf=4huq1A27AgAl$|h=85Xo3f1!Qg=d;zH`eMJeN+z2vwJz}8
z*})u1SXZV>sy>Ey>u7j!9+H?&li2NJ9=J~W%EDcbop=HE>
zF9-u#8x5%nnJ$j8ySfl0}?1aS7o`+&sWld1tJAF%
zGtQEES5Z+h(`D{kh)>9A{nBhDvy&MB%3}_005JG1+0p+5Ffabw6XCy+@&7B#Pa0GQ
z4}zX7A6m78OG0&wyH0~Xep;CIo9l??1c&My?;mEh-RvUYX#Z&E@%dBn*@LsUlBF|hiM{ymuCb5wspX$2_KGdxKA-g11c=Pq#?SJb%`Ie7
zFb7n*6{?x>k@{5D4>}Wq@-!@|zE&uobvs|*TJ3F}>^pD%xtYE+(^6roV&?3Gsi(}CoxcU(8+AMSP8&3be!jkn1kEt%~OIgx93Em94o)pDO4_Vo>_
z%uVYlz1=a-;s{=IQq{8AahCZK7a*kBmvhj;U;vg#s2yS+g-HDl5{`Wd2QCd$GWt78
zjH91nS6-Z!Zl&>l>a9t)TY=8seg^LvGM)Rt`-QA$;!c+d0qp;VObBG%n^K
z`uFck1$=z915Yeb&}|f6W69-vSI>~;&Y{@R&3QovQ9OTgZCR@8f^L9BTUsV?$Mtrt
z^5n?0{@TEvoxYpv$BCJ*>#?O;ICs+TCYhfI7Ez{~l6_Kn7`^-#b~C4jXl{4bFXCNj+7?
z=v{eR=xI91^HhZOX9Ln3l!GMgjH2=fujv#SI3lMDpVYE5&g66m0r746Hy
zKmN6?i3*56F3&eo3gqOjYNe`8Ju6$D2Nn-57xb7QHsS(q*r!Rl$rn@QIk427(tiX?ByKx8KbhiIM~R@T
zE=7~LGcb8Vc;GvZwTnS>n8+REfJ34Q)4|{hQ6NiLuk((+OK!v`av-||fc4(bB9)m(
z4g}lstmbmiptyf2iWgCyqMPdCBQg;%^5KN@AXpOA^I`V(CtLXi
zwCC#KcJ|Lnr~&r_P5Gl!NQdf))gLcln93!E4%xo-_lB!Cyl_@2Knmwr`@nUow5v@5?hU^
zAzUP!o2ZP?+zq8s#l9ELuPVV;P=~BINTi7?o#CX>8ih(GQIHV3sj%Lr(ttgijY1#@
z!$18?&|dquJCb1UfVdyxtu6{#+Yvz}`Aq+6W|UHuY8!JYr^Go~t--X`B8Cc+t`oc>gusRy_7vK&R)B4b*?NJDomj5M;=!I1SQDa2~c`B_3(~+ThB1?P@|r
z-%iR&xT~w@e|X4B=QMSw(EG-uODq}mtm#%df_uN@*kMg=Qbr~-Ff*c+aaNeEZr~>M
z_y5J%TfapazU{uGAdLu!bOb1v~)>JNyos@C0#Q#3`5t@
zHS{pB`L2Dez26_!vEKK8c<%eYp6fi%&$-)Zmr(w=%I7f{7SPJR=}N#H83wepu_I9A
z3At7wp`Bug2b3Ll_QxWexXDq}7t2@+F4Ee?DjpoqX$vyEawTc~%@?E=4t7>!I~SAp
zBoO7fpd`uX$aSGNo6pnqzXM$X++XF*Uo{Y=PCPm6$7fPK#2rkKsVEU4W#=RI`@s8n
z^tchJ6^J!R$BMT4=QF;OKrG?S&{UeuP~v)Ru8oZr-czidY9homx&I854XqOvc-boc
znA_k-{_zbwI2MJ!dJfSjO*wlY)p-||!@SyQXq1k!dvKgRr+3#V$E`NNU94rli+gA7
zlC~dWtPa{T++zJQxL^pfGhNaNMO9+{Tzw)e;`L@sc(mU}v^m2)*3tioO5K@^2$gUk
zwVC;CPOhv~NL*H0_S~gs!E2BNX*lPAO-kSVjpdW{
zoETV5ZB-gyvKyt{2Sg73m$`jK9L=J9(yK;#g};8U^$&Sb&k`}?Z$Nb5fDz}sl&DMj
z9s!-^$pg`S*btf*DKWiw75f<1HCY
z&mW$CdvV8{t9x)a^HJr7uS~V~fJ%5y40*ytA{Gx4TfOzxSC`D$*;|_DmZ6w#t5m5;
zu#WwHaV7aym4O}PYn39T^$sv}1YyU?TSp#B0XV;ppCl6JJG{+snJ912_#QrT;`(v!
zaR>aj#w3;%E?8OV4?Q3ZHc`)6d9}pzCJ%6YEywG~(!I$YLGxJnChH7TAs+Bo>VaLf&W2xo`YF9W{Y~^Lra6M6o2)Y8I8Dw
zi5MPPDF1sdMr;~fzv>I^O_yF2-)usQKK4i8r_eH;mPppaTZP_VaoENQ*$
z3b-(n#J>*@+pFG5%w|G#9FTnL-IU#>h@wzk1niV%^WwGgC!j2!zMYsAstr)AeChyQ!7*9CsDOVD+%Uqqf*jysR%%5vK={v7zxd
zi-YSex5?M<>l=B&9pbZsEs7fl3&mBi?cW0;98&E^%k1vbw)4Sl*FWxCXkt>JdfX=n
z+Eh$O4|fd7tiEEAU&7nYd6S=*_}V^iNR6QF2EP80yoadp)ZgZU+f-A^>v(TBtc8`9
zWix^lY%6C0F!Jqu4-L-_DyBqwJ{M{QoBqM~IxP~Oo5*Yhx5H4xWS^F1v$e)eqiUm5
zqK|GNV*p=QWhyoq!&#F<;HOWii)rf?8#z_7a<;P4t`ezvhiB>~XOKW;*-NHc&OsKo
zY}3YOU%C}#BHNo*rn+XoO@n&I>zXao-j%GTCWV=QpevFdPgc2q0s33FVD3|+=C0J=
zeQ~Zf!ltR_=k&tK^(pX>->?i`A4`~)w+?)QKilc%UC-|KvF|-bYMbge!$lC|laOT
zXXhU#)M&^s-p)zu+xY5A$8^EfO4HGC7vg+`}IqpJrM4^ZlM6g{~$eYI+1i&
z*zkd3T`oc-JD@#6Lp^xde>3DoO%ETAm+oge-1El-%oe447M^Uy$g!HdAMjWr(!~Qm
z$-GB>4Hd4?d)t#@k-t?pn+-G%@pem)NB}eY?#4#Z`Xn`{n#1PkSu*Wr7);$lg~2?i
zybhR3qL2C%>jljuOmN8Kr;v)d-r>3{S>21{AXY!3_t
z5(WSPslUpWvWz;=>%;GGuNHI4^x#$5@>o~ZI7hQX4GnA9&G?Bn6G-+Xk7NKWV`l8(;iA)(1vw;
zCim^WngN=wVmTM|K3A<{9Jn@s9aN7G9wci)#kl
zsL&K5F2uab`LhF+7KInrrZ
z$hR&GuUd~bJ%T3r8L@`2kHao&;=06&H!=M-=38QZv+PXGPmz|@sEw^y
zI7at0@?%gQh@V3u&92v_1A0%7#9Jc!jQbsH;1?b)m0Mp-kPU~%y{1ctVS9w*4zj4R
zMqIO>fuJb9Qum66Wt&2|RdO}KSTjX|>CAFhpyb}5t?RcHZ`%~G&RX4?xwnd9qf+Vx
z*sl(z1Gv}vAc0H>-Ymf`eK(KARC=gmG^~#+5~aj_)YtV+rZNQf&Zzl{&!*nJ;#)@J
z-rj)?Snh46Ma$UP(Hp`{L7Jf1xsUA!Xc=VcgMaB8E)53Cr*)=LZ5HB&g$=H*u`N08
zZZ&_EyQH;K@?DV2nJSz1xa+k(&dGM_Bdu5V%RG&?c9D8mL~r(2PKBhf-ON}cD^;O(
zXxFa9$IUh(j@%&EdiHab0^9hgCV95p-D6h2BSo1;+(j_@VV2)5_>PUED{pt6=-euA
z_l#SJt%bghQ)vZ2i%b&pS4z1QWlgAUXuPk
zAJ!BO?@>Oo{%m@`VxzDukER-})czG4&HrQug8cTN-{k=xk9Ew6K(g&X=XwQAQ>*N(
zUtZ3&*mV{0Px_AzGvTb}wm#(?I}1OLy~~`9NR^+bD!_soG|b&T#l9FFzNVDr%%L?^7BAX;UH>G_W$WU_PwCfRUCQ~1apM7>UbP{EZHo*u
zM2fycDJSTnNjVAzcbsq&DjOl)M<*%d^}9vEQ~u%#~J?<&=WXU&RY1Cup8O(
z2nU%O$q5ELid7kv0~(l+Av0KAgZ2fB=i}Gag&U$}e(sT;xYN1jKRBO!dBUqoZ8(Ia
zlgm4E($n~HuAN0Yw%HKfDdyu}`+*th7(WgsN_|;966VJ7jzLw`uQFlt+bHPm=Y|KN
zK`uUZYBLqoKE;be{)aI$J4Yn&oZs#JV
zY*-4dl>CKkgClgV__&2pYs_y-2rx0)7!qW_&B^<5}{#1)#p=&WO1bYN@_)ViG+5Xv
z_PADP>GfRKkB1|a>8P717MPKH^m+rlqYV%Kks*`?+I*2oj8vAY6Y;Q#&<@JZo(5=O
z8D3(_5Dzxhh?FIzXY`R+hH+ORRtD_b#p&V2gS_vlV8F|y&SC3)c
zr|3dIWm;=y(hu7NSk;WQ5^1O0>SE_)CV1n@P%nRMp^U#UqoE2_k_hiE&xav732BJV
zzazc31vGY{*`EUjANtGJqb|3Aei7oD5k*={-B;%_*aC#-jXNiY@9pMN^>EBvXJ=Lm
zP5>hak^Je|2Ej#hV;>9wkaBZ&)mrX##dII+P=MIlc@s1Rl8k}
zZVhQAv2B_?>2urN6i0X_(}!__im;S`|G`LpDcPZ$M(kt9z59g@m8U+eb6{!V+m`sY
zsqO*n8tm;I$)lCEyt(;64!a!c`6r@i>5|T$28M#oW-1&PXP#|X9)a2fx8KAj;qCX}
z?c=VyYB9Zo>r0xyl9cgmB_f1_)r+Xfb`{oGo|k1>2<>ER$JWx&1i}(K`{s(49eKEs
zZj_G}FJ!@|!a`=#tDj}NUK5&#fX2f*)veCuZI@GJsHPsS%W#6xq7ZY&3pc8^f)VG4
zw5cZQ?;Z&XECGbxH2VJd8wgEBisQkqb}BF+(qom~&Hb%8$BT_0PeOLFfN6B
zEo(=S3Z{I|BO7=y$toP;XU~`LC0>t*Mp&~3wlm@~je1;8DHf0(iYx+L$^K?-tT3Lj
zgF;T6%bJ;th|aVvD{$MgW`KWzzJD|pBqQrD-mat}omoW(YBA~-dTx$imUhZZs?`03
z)6t6-ma3E(7F&e`e&m)w+2%%tPrRMitjv%aU;hv$x&k>ccX?ey|Aa5`^O&SEvG-ec3yHyM2?%K0AecF*y`cW^tbXR3W
zRz4c@`Rd;8olt0$9HqkA1
zDAR-?C37=aB-p*iyG71^tzo({1E(VG*O^!EdQyTCJf6O~h{+R32
z#rn~D4N(B^2i;C=#|`f+hW{{`ggegdca)rqsX^sxyUUYm1Xkoq6K#%+6{xv_h+~57
zw#Kf1K04hNiz^O7eAO$=jVU_Y+1#XtqJfv|U92lk$fMKk^jRocf~Wq7kIYPEG*ggO
z+NAFGL}(>yWTGm5=+O!S^0o-B0GKhXMdiSh{-NV|jLY6PtqbBr(R(!U&$!!148lk-
z2Et6?-@b+l;vQeR<^8ull6!D(U)}nSnwxG~zA?8I)%N~Azh)85qR%cl`1CmDc|<(OS>L5TzY2a
z2b!03fz}~^R@c~A3I``?n_0PZF%ef$9BhZn3^xqHZ?QAyC2l3cZG5QLojzvl%d_XpitMc1ZVD9LS_etX>D>;TV1HF{_tRg}6O_V%vDJ6IzuJ~*ig_r4_ikINnumgP6yBC{~J)u8!pkJ39
zoMr1`{oRz6OO;b3yLZTMZnh;V-ws8}Inx+X+QnN$21O3!h8%NXv1Q`4t}81H8bxU*
z-oEaec~UF8AueXGvIXR>Q{H;CIssG`>TcZurqe*gLgOlhT*tlq!{|N~0*fk@^kWUR
za6t*Q<8_EU^S*BlR8^C_PG+x)2JN2gP5ZOPCCK2OorMsx-09Mfu>ir}j_R
z;h{tBHGYO5_6dB+y$!AA3zIQXhJ6%pZ7(%sy?Xbx?3E>vdrW-2);E`Np`q2?GOas6
z_)Zu8srv9M;k~u0Jc`m48i%PTpgH+-iK0q(~77(Wi3x
zW%q^Qe6#I?ajv>&#>(NO6*6Pvy*f|#uuqZL01R~d;#!Z?3xB*Dzh9Qu?VGFfxeeI@
zjI-&6>m$HghHrGije9}EfyctZWteZ)_;36GdT;h6=FHM(-5+zjpbF0?h4XSA@7Y?K
z1Fq8tHZ-}%8n_$V$+AmjmKyw(rJdRv)GswNBXFN}dtGn2lHAs^d=tXeSPzr
za>!0wk)7kGC}CSg==L7*b~kiYY&@f1DIw^^s^65{K%$C^{k1Iz2-OHAROZP{RIV2<
zY2bM2bO(~zZ0NGv`@zM0Zo|*p*yN1se7u`NN)-Cj@#nue*4D&L57vX{7ay)XJ>^V#
z$0T-FW?cPbQ`$YGRnq(F&MNsl8`FY7Npg_vd})(|xt@fZ-bLM=IhAjRYGuBQcUA(Z
z*F1ry@9Cbv##-GRzdz$)9@Eclb=LY-Yw7pAT@P4wL1M%!qpzDkh&M^=Nr?{6N;(wFAn&bH*V
z%{J-{*t?J=!U`utCn*CgrL*I^MAW6K2CBZ?E#%EW>Q>=i&|(JiQqKbk$1c)HQH>q(J(wbu*WdsUR!r{we|
zn{go>4(vHV7lvlTi*`^;4??6~c%wvqL?1{xA#RgO5TZvO`t_jOPy<_b-UfKEgIj_sCwDk2_mgIZ9UY+&+>K
ztl@YY>i#R&NX$e7?l}pJs(WlfiY^xND>jgRY!WH<&~oqB{X22TY%b9!8YUwl>Qrz2
z&om$p(WKe5MExKOuNk|E_jd2tV3fzA5G`>R*-;
z;q@YEc;B_~aY!`#=gE8ab&al7c3QsmTECBUfGdlgWve1hDIk?4{IyxPKl{T3=k_g1
zLO8nRQwl!ibbXFH9A<@k4zl0Qa_l$)z0D>MFjIp{#8g9b{8(%#@aZQ+;_zmnR*Kx~
zf@kDn+*rHvDj$MOHnV``c~+C>!ORF-aiS@tZoeMB)aC=Ns^`4O#Bk|V<)Pb<0vCcl)lsYSrkHn!4#Z{ve7J*7R3BZbaDJ&Y=7aV5i;4%)XMZ!xY_hc8
zd}@dy9^ER(Cs{6&s*%U%@wy81nc?d4A;gr7E+jqEMlAf1`OCR&o@!ll2W#k_ZxR&9
zSo(F?l5wIt08+@MEY8|(8Rk65{Ns9c>?_=0q?fJxx^zaP)DhaEE$&7ukkX8^I7D$;
z{cWa#Wbv>|G}%SriGu>pP1t_$mzs~WS_8)r5J
zs26C)NqGTDdVn4IxHBt0k`3jbgGbKwnAspry@_t-b44&awq4gxdWO+GVEs=}A*9xC
zYSIf7{S0z$>~@vL`+H2Z_Ld>w?npnhw{R?7RUK}(3k;aHXlYQ|A}i_C5&QYoWV|j+
z@**Rl6cUqKk{`#W|NYMVO%c+5w%=74)VUC)IO^Ow-kziN>F0jys&442t^_Ujs#Rch
zaPKB!#4dDpMnEw$^@@
zUom|q6extDKPGy?`LYihsNQqq6Z$yay$Gnm<4sx2$tat%EuLD@5@jkcX7`E%fO!*k
ztrl{PhCM-P<2&~9z;f02^#~O*{fx7@pl&w@#g%qp5d$F|g+p~Zr$Agp2%}KC
z?Q~yUQgecSlg2!NZp7Y4BhxZnpPjeQd|2V4_u&egFAY*zsRg+DZ*dbTw&7%3JyMYA
zJ6U)pN3;;BBpa=_W8puAw=DKaUIO__cjLvD!d)@kMDx?RSyP{%e#}&c#cC*;NZ^5o
zw&ZWaTz)h0nSXk5#*hjeOFi{v^&RQfc&(2iK(ocTBUO{57>O!sg$epK{xWb7W!)Z%
zwLS&aQQ?18OX`c9Nrbr*E{I>B{WSK|7Qr~NGg?aX`>d2uHu`zc4q`u@n9dDI{^1nC
z{<*PzzW7Rd?^KfgvLxxyxP0nA7-A*BYuV3aS0n||0T>X3hBt80#(!~1-im2P
zoEhSjI693u`st$TZKzz^K3v5Tc;|3d7ECoN>4^UY
zA7FOCwrE;;t`9}R3IcG_>yEV_XvatP^eEY74$y+hPYcA-;fWvkXNwinh+pw}8!65x
zLu^cc7JJsT=|N-;a(o(F0f_Mfh|e`<>>PWwE?0PX6E02Yah$1DD~ziR5ddAU8>0?M
zmRnTg^A}^H2NnG9qd44*0PH3Hyzez&(Fa5R*Ipun3q#Q`5q?5~L+Pyn(#lr;KGFicUOlI>|>?k7*T1_=W
z-tjXoZZJ?kH;9e@Lzcu5LUSg+7b8!KZk#u2JkA|2ks3CXmKsPUUSGs7e4<8zVsN_J2p=IqvU
zm(C&q?s5_i8v4cUi05`VQ~aK>cDQei!z!BU|4iCo^Z1cq*KyI5$)7-{l?VVO>tD@I
zyYpARh<3O5_UlZfJ5swnc~nX19qxvV&b$scZ@Q-X4zNb(LWsuE$WL|m=h?Emt!9GJ
z1#K6iD$`%-Im(-~xk@fI6ITX<{0f>>YJ9If6B0h
zcpYM!m44<_b;UEb#2+q5+Fx>1638cauzO(f{iYjgtY$PlqDz5Vu9=QBe}Sfbefo#P
z^prS#KYeDjVwtqosgA6*;xE%y5xtyo@xKC+y%|T4y+CQvHDz&MQdg?JzLMOUZw%}y
zKQijVpsCHY9ld$#Qv)Fd29OEXtf2AgW_oYOROf4?6?=RvbCWVNFVW8(e;kR3C#3EV
zHyIFtv$~+?)d!HqwyQ%Yc+K8R@<}ElW@ws{Tzio`a?}(|%_p5r^ZHIiI;Ng(fAgn0
zW?O<&&y+pTP8Yp}6kC#QFqfckLB+Lp?_G`}Q_l#Lt}`
zNe+$q+eqDk(pom0TFWTT6s=jT?Nco@FmUK4_`qxUbAwa#`~DTuts8s6nfYku3(gER>b}vzt0x~#%du4y-);AsWNB0aD-U4cJt7~Ue9@tq
zc}>@W{rKK7?Dpk1hK2fS01jFZA`Z|MtVrpMDap
zX#XC6>ixOCWd8+s7ZNY$PA}eJKMSPYld3!lHYg4jsUr#>V5IXa}Y$)HU
zn(`n$NAJDTkKtPa35-qhY5^6)bwT@DWVOim)2A}SkgKt-UiZ5Qh
zt(~CB5W?el!E8)ykILDi4z~?R)bx*ia%tlx$(s}ca(0xZDfj~9Hg_0(1CnHu?0>rM
zOv*$rA9{i;)Wk_-*BV2vcUTcSHgPt1J_v$aZ%q@t49bwcUcB)WiJg1Yg7Dy3fLaT}
zHq}242N|{S*ckMFqs%R6m}81zyrv7Rpw-}JkL9$GtTLpPpux?f`CxILcHN-hw0J$6
zp-0r6X85%V$)Y
zosAV(6tgyxfHh9wTt8wY*BC$Ia)o}JR^63S<)B-hpGktT`4P^k;_1uwDUe1tN57y=
z(Id)xK{{nIF7gbB$v>%(WWg(Orpnh^C;9P;MiZ5sa}uy9?+!$=I4Z4`%NJMZunx-<~s-Oo+d6ef4bkaY5npkP1`RRf=b<%
ztzM^%@T7s}X82^>bd;D=W1oigNFm2+kKw}=0KV7seMZLs?baz4*2iRu!UAC!|20Wj
zSjTJ4#wGb-sTK!EUvir`O3K2g{4MiEU7}fbrQTy=(ECjuJjeM7@J>LhCZb|#_5SNV
zx7nF#ptP$Bz>+EXWWhVOZNEo|8ywc^w8Z4sTFOG-j{0l%vTi(eo#VQkzEZaZ?nk>O
z7=vnX>DGu0NToKO$E2lJx==#>oYJK^$Ga$;C!wN;aJ$i0zuW_jErn2`&DGXB-p(S}
zs2yLEUeWWKJgvdH01XT;^yy;`lBu8N?X7q`wrB>mr0d#7xb67E(fJBI>+vcDC(O1=
z19lI;<*fJNJ*BoDC2|reChr>>z6k*;{cx97a(3Yc+&i?&qP|Uk*IDG5$a+3vBTkiy
zQHCJ9j3-W;nG+yuiLaNm(m9cDW?v2%uGgr<2DRT>9!>_|z{ecIDib%ZgBd?moEfsnqjDMQdC4MVV(4f}hgC_cJNDzI+ezscBN*eA^
z@`g-4`2_(dDdZFRH%^pZ2iQ-Qg88Mfx9og#gXFfUX&oav$zPivx?8YS^qoU;uBk6%
zQNPq1=YQ6}_4(4&0NJnRjcf{wPryMh&5-@d@9HT5cO%b0;nj%Uop1y*U7)4Y)n6>{+eKCo
z`bM&gR$!|DdTc3*rk5o*8Jr)HkkW3&+dGfTd*jNYcYRb#srEAcZn2o^N@(ip8@utP
zI5NFHH!SWIgb{hhzyEQCka<4T$!g_X!J9fEwlVXnJ^`)gFo2<$1{y#kKZGV$RBq-V
zqA%){%^{rQHTG-wq4D*4n(M1K27aC6Y1h7U45D*fTTT&TLsFZXv!3?bJ&N-K>TQYa
zGIFMKO!|oAA^{rht~BENbbkM`WSl_tVA-y7l*lg%byD1`vcyO$M`{5Nm1HkrlMdl?Mv?Mp-`5>@OX`$SB2)5e|%(T?88Zj@y
zdGp&Y*})RLM^OKA_NPNDgN}~%MctpERC^1((peE{hnF9y1;?J@?VmpBczAtz+cz1-EK-ufk>zK)#W)
ziS{p!b|^=Et>aB3bbltPbXx1b-)w*I6k5mWp2@^Dpu=4+-Mzt%xAJkCCv{$ZLZ;NH
zXTyPiUuLyk5oG(GGqGaVwT^nF+r#=CTB0HI%jf6G!3J*$I0RKTOWr+a3H(dnjZ3eV
zPfSsP%JCa&nmhzrbv@>eRXE71=Ql46{)0AT^ZCJ(Cwv^>R;vz~$Bu}mkMsYPd_Exc
zF{c6uRpxl12a3oY`P!qIm8c}3>G&^Vz=)5LlB>`i-wBC_9fKT`#AWcKE9ykgl9*LF
zh-zZe=Io#v$a;m(BLt-NxY04uOR7yK$`3ZlCfPk@-$sU(ndOZzz03Tai5-6zoJ2l5
z$5k)ZEjevr72FG@OclRMAMCf7PXhYz1%uw&R$aBH8GoOjr`v|WA11JiJ;)OvyxD#8
zE8*dJAvu<0W?V{G@;pYAA{f7B?ad@bs|*GpF;H)sxUo!oRWPJ4rX(gU+36!Ug|D}Z0~a*IsOm*_vp6nZsjFzh)`eK
zlj)&@*m{GbclZ)>Kd=OhFxBGd7Ke&%0UxqLz6$tel}~X3)6BBZI9#t*|0P8YFOb3e
z4CRxJN}(H_XQW)nS{$2po=|vI!$tcqt6f5nFK$}i)4l)tmoiKLNRtj&+A2#q)U1YaTqs5=PkdZ^@cLe>Cf)iWx5wu&p?xW26%HtQ#rgU0=SLso@rJxdYnyztWQh~xdEev%
znY%5MJE1IJQLSaeIaX;kEeq6q4%l_Gea+gg3QFaFHvEHgwY#p0Gnp9*_m7=tfc#u-
zh$RL7nF}NKG46e%_FJ9?8y-2No&htWpDr&8NUxaa;B&z@W+*uN3S%YqDvdjG1LOHI
zqc&P7!6>AnZu$-(!SL}?Y3*=*Dq5AuWjHq#k)zFs48w=3jp45}YE9p`V-xR)-h;M_fWwhB>ixs$&z$BAn~R8W@a>?r)FOZl7$@NqvaQ=#s@V`VlT0
z7%VBMKYCWyZidz<-lOx@5{5}jYtedxW-ocebuOv-r(<9`N%>p(QYXliMNvgOvP!x}8B7m8sVYEnK&Mq9V`E
zpSgQ46XwcP#(<}cB_=;!)Er2=&&dB}mWynH|K9)U=98OX=
zt<$$0HR6^DN~Rs?)+;=Q9Qx2~0lAvmih)9aF~%1+MpbEBh!+c6Oa#l51cMkB<{WGf
zN6<}~)D9+>*c!Q){VjvK+y@tydK=Iz6>f7Hh3j<{rJe@1C#|tf;348#yMqGHeUQR}
zHgeJvi#F>HRf}t8-_yQJ235wKS+|pay;Xm4?o3sa7I$9OQ6-z5w;$C9t;QlurXQ>N
z9x*#lfVk&0mR|EbG4wS2d?_+RGlGtcISiCvHcd9J;AcPy81J?>K)oK~9LfP$`%*O4
zNFz=nDXN~~*7joO6=%xdd;UzeFozT!*sY*M6L`#j?aBbX=zFfIHi-9n@D*9T1vJoo
z77>55!}rtDri9b`B_2npOQhpZzA-5~ERUx!Q%8;*wct;c9#95$U?v
zaHEG#C+M8-Ip2r-a`B>3Ex$lAsX;$Fq5{mx)u^_krZxZpiI@Jac!c-UsbY;BgfF>%2UWoKhBNkf34KL|&c1gzfrk&I62u@RkV_Grr~FAEU`Kyq0AA_pF`
zH>V)M1O!N63MUIrEBR{RZZJbFQnf%TyiD7b*w#wgkqSx(F?4h}3{{FJOd8=nGW!vY
zu#jXz1VP=-$HE_;&IJ9ld?cz-X~!;T&~yhkPk1pm9|$d{xX6o96R)8Dz7c4YSv_Gb
zjDMRJmWFgv(655Q>DZS3mRT|6lufxBZ)epFb;l?N>~JLH|2RuM{OTf~z{lP7d!a?k
z3i+65WX&zU7#==%7_6fJChwfIs$l5(&I;T&@8ZJ4tgZX!I?Q}jZn)v%CKY^qkoS#X
zr4%`jKh>E1bbutZZ6sZYnf8yjFniF-BAio%KsX}lnf{xNlOZ1e#)GKq*cq_0UIHj&Lbs7oBWb}|Q+yXnM~2AWS-*#-Onzv91?DkLwhl09-uKB6
zZz5u|k=hp}ID|m%?R-qcpKo4HO^}guGye*)wgD(H_+yk?sG{Pmu
z7FZ`F3!;5wG78kezwh3NQTXZ$iL}R?uC+$;7n1a|iDo~JrFmtVKOVk-d_HToV8Iogc{V&uM}qQRqUp1~T}g6;ckT{*EqYQt)VDtUE}x_Wov
z=DBw%1{bP;*(^*K#iS&Wy1<_`B~FoY?KNt8ej1xiH8rYC|C_(t_~hv3UtzIBJ4%po
z5KEgND|&Laj{PG-BPDrBP=K=a(ogNa=)v)SZ%X5YP+_(33YKi$Z}v)|^bkq#^gej}
zEIl?F$BUcGn5j0DA0>ozUo)HlrM7=QUE;XTNaf6p%{*MP{zGghv!E5ZOTH~=wuROK
zGZ84~>(u7<>!;t*{W1TVontOTTfQKlzjBU_IYOw&bsIX&FK>>K+dIHfYP>@
z9;o8_T7KqcW2MT*HoKW@-ca9Cg~1^i(V?+?38c4s&!{UA_{~Vb9tnhnfA1>dS+h3?
zItbJ8;Jgz{?OXm5m(^Ytypa;}KTx=09bbr?8e7J!Gf%;t_YR?UmwOQSQEx&Y6?Mzq
zgk0x6@~f+P>H3*%!+i}akTQ$F%1rE)IaoH`fMoSxzuG_pN4zf?Seh)jXADc7AO?G)6jAl>yWbx#S^yD7yKqtu|SZ*d~
zLEmhYU_wT{NVIg)3V<-1-KEuqZg&|HZ`X1sd?(ty=gm9r!rlAicyLA|qR(MKN5)E@
zayJ3Kwmy?4??U`cuAx0?n=;SNiP=rp{QU|>MXCIiy^Ne)hQL;)TvK`c2#DXZmekH-
z>BQ}RrPg8sR-L@Rn@g^*Yc@>cJ#-CCyFCSvPf!G24$)mV^FP!U9vbt30rZ>aHwCo9
zL-u(e(RnH=zDsxhn&XooF-MLp7`6)*_va~`uU1fx{+`(ZGgoZ!*iaAVle@6(*ned~
zlk;(B#y2Zo^nDJcJrPHaAiZplqXF4s%))F!#t3+{skt52ecG$Bt!GJ^|RmN5+lm`jx@T!8RX~j
z>Bl3ew0J&NoGlv?_%!9^V)TM!;||7{9k`kp5M`
zJS*UYp=X+=wo=IZrzP`=3sQBJ)TAtu*QJ^4R0#3T0k*#1#D%$hydwDr4XQpK!uxA~
zIGw?!0yMp-Q&jO?QM7iM;#ts$c{X_`j#;zVb=AzQNgSLYo$57l?e!e^_k3Ygbx5iL
zS$7K-RQT9wbQYl*1_)oBd@Np$JAOaxQy$$Qw2n@rRjf5+CeQ~6vQ4ev3PxOsUn*kO
z;QcQ@FWVRBUf{(&;E^dhN$LHDy^$bdhsQNi=ze
zl!Tt>g~OaW8Yr3}lld#j*9U%~VLP58L|?q2rA`x(E=Els%ua|$q0T~PCgH)a{{7}p
zH7dMWEF>H*A=>QCv6gqG>MS>IqKqehOHx(muSbqMVd4j#mGMjy)_@;)OHz8;AMLp%
z%MMu>JmBG3?yrshQ90o*mXqW1%uwHN`*$>Mim0U`jxPySR{kU+HDT2qXZm{!xvZ&qUT15!xQ701)rhb)W~
zNA7orBf*_PzP#amr
z%>w>Exe4jB_5cqJ{xorkFrU99uOGT_$bY*SmeWuyfCd^XGm3@LUkgp9a#0vwaP?o)
zy+g=nsO9~_Fe_oT>)-Uq=vpZQvg4}BFdb&`5?0E=^?L<3@g(9;3mp#jE%XzeexbZ;
zzF(L>dgtnkU2g6bXazG7%Vx}cA5MXN
z)_*5kbzYhopQ1-BwRp_X?|#+a`=?a&43t#X>>#jS^s>}u$g4|k4w1M&EcjXCBr;dX
z|7N6Y4|4^hA6tq(Czum=2deeBetI9gdc1J%favb)1`A{hflcGYDr_Vsj@5i@;osM;
z^>I085m&BRsy%Rk<~U~~8YF)$`D4f(-`;@d5+OPCZmW&5YwkC69{QaZ$%?_&I*Tn*
zy6qu^VY$M4f7bYX6hs}2RBeH^P#in84f0L1N8CKH&>~S=Xg)NS>BcOVH2lU3w*ZvN6H6LXvxDH
zgvpbO@~5Ui3at8hkqluX&xUoW
z*7w-S6fdaq$Zve)d#LIogvb^AbBhU>%C`Ho3C!|X`{SXOIN$Q%U-O7jz?DVk0|o6K
zRk8UTV7?p@orM$oAY>V>wbXZ&ki9z6*D+GO(qfls9d&S*XrZ`yc`nN)8Cv~5!_1-0
zhR?IP9)3B@@OvAu2^{qZZZa1OwXuL_-i?oD)iutyY-6&hWPPA;_GZ6jX%X-UCMdVJ
zjagFaAMINc#t|
zudK+E8OD7O&VJ&f!4W;fM=$aE^X(3ljjebK5edl$Q{vO~BK~tCpzYzDw=4Y<>O=t;
zbtQLa93nK0KF;T#sMwi;L)(c_r%ddzCn$ogUT^0$E$4R`#I?*$J)MXm(VvcKR6R{7
z)qtFOi$_vA0;Ea6T~ewC<#&4Xz;*Kb>+=`9M(|Y$$m<8myvtncvEQlbIw~N4VWO)6
zM|8tlRu^U%_VkwZ8@b|dyPk7-(xH+%V)CosQa|{`zczJ)A|D#M+-8MsIR(Gcl0QDZ
z{klW7S$NfLu^ByRe3B5gC)ah&?Z;kxsN+$8tHR8_0DIabvWuS;ok{P+eR@qw-EY?E
zFmCk!F!okaZG~O;FYZ!GahF0V?rw!5#Y!n&pcF5`T@tjomLdT{a7uA^DA3{%q(}%-
zqyz{q!Qt?pi!=UbjB~#4z1kQ1ZjbToXRfv8nm_#?-}{fv#jVS&c~Pf5l&f65uuXdK
z3&b}RJ|11D4z5h@sgo408J#QlfrCgto6)9TLkTKlgn<0dh*Qnxif~Bgn@$bjFOT+$
zYtKWydB0X8|LGl&By&>95HF(5L|YIz`|q6Z&^>TzFhkB>-Oh;SUo}>yo0aJE1M$3J
z8q1Q&FJzC?+%->51;p#qJ9=2UGPF?=hl=xlc|d{M3YOn5dGTuPIqe4Pr+X_$_RFR}
zRwd~!&|5k2>s6RQ+9c!3obYW41Q)Sj^M%|A-}l%n90OUE7O7X&;#GKm9shF36=eTf
z=ApSWP~Q_M{}KC5j0*nEd)g*
z@EsFPfeqUby4P(!A6$H;<`1;mbpo%p@VA1pmou8GIavqtVTJ
z?bZxT=eJ*!#MC-6X|2CzU4ghHYhq#j13T?0%1A%7P2eP1XY~zSJ5HjD2IOjNZ~Ur@
z33W4}S@W6L%FaQKc+Xqz?HI^P?t1*FpyxRVJUk-!UqlhUTElgB`MD;}`MIOpB`3tl
z5b)#j_7wJ?1=HK#R@bj@&+f>NHx#cFj}8RywfxVvDe_i{n)l@*LX(S7^M6^rmMzdQ
zUJ+*j2U?4xz3$Oh^|!PU-*>*6s0p-#tA>s%bhpo6Y{7e!yo=R7{XHXb^!>#5grIz`
zg$tq-wDL)w!%mFL3?=h-5o@%;a*2Dnw!^9Zl@b4SN6u3YDVD{-VF0f^R{<+bcA8P&Jl%B!RDs=Vg`H1Y7xhjqOb-5dG|edQYpZ>{f%`$RZE=X-UT81d(z3O=
z8jta<$Q|Ow6w{4$fAkI`WEL=3U+{f30SY$BKwt%3N*&4APXBD})@&rafJICT6MsKI
zAZqg(6sU=p`{?^VB$HGe(uEj9IG00(^~Ao~86Ett_CT*`hi}o4M3ywsDl#BT@(8)+
z33w{guCTd)HQq>{3`1gHq=%E!M3r~{Cj}De3J;RI9#+X4XIA?wF-+4a2mVb7pT(eA
zdscd@Fetg1Qo+%#`>M@9n&%G_ffZfvY3|b}-sOww)iLia8JHeu0m|M40G
z64oB#bJ=K2qzTr5Iig_L%^qpNX+F$?RnjhAoz$^;5f;)NO)WJIEP%HP9bZnj*BM=r
z+(80QdtFg3LXw-Dwibgu!Nv%-uh7F)ouWM
zWU}L1sg-;R-Da7nBQtSO0(HAREY4{4C6T@Fjh4ur*@?jc%DXuS_+vp5)SISFm}6G`
ziK+g2TtYy%m#qocH`dm)(q}gK|tu3w4LeS`A
zqs*e2@+-al^>5thCwYwI^Ct3oXNR2lU(Lah&Xa-e)29lQ-XC6Z?t3DcLrVC1y;-qa89ue
ziMXC8tUZhCrak29TA?;+nTe5D5iyS`aqRMeKhrn>GSeT4!$)In6KEHE90GIradIh4
z7Lpq6Bk_rgMFS-UH0e|Rs+fr)G$&yx=rh
zpj}YSI{p{&is%a#=&1A@w!rd(TyXeb@=J?CxT`5&amA@2gBZXWSwUohu_*TjW=Uhq#-baj7l%+qJ-xM7*MUrV-lxqP*@l?ERad0#z-2~&DmH87W+
z(3UfqGQL4GAzxcBw?S>6G0(XALYg%|oi$WJrTPSRP705y;Ck*A;i6YRxZ-B~MJ*ye
z%~+N?gdZ>g9yd;d_g>Q%6K1WX6kF_QbRyjcmau1b%YKE9m)Stu8xHKwHzyf?f_Wa
zJ4|I2VrhJSwypJaUe-G#3Hhvn%(TMxVpd(YbJhBPCzJII|{CcqE0x6$S%c0t_MDyDD-~3|*cFb!`zC6nLmDYXQ
zU>DNhznJ(ews@R^K;G5t_QD>%n9C`Iou*TYn~9w!<*dABAHld#Maya1vGT6eQv2N-
zSYgQuUbd8ns%?srny;WEx3ff2SV%CriRt)O&8WfW!HZEGAYsG4Pp~))D26
zPa8tt%m-5N)#@WZ`f{TR0ABDm7g$nvY+%Sz5EF4G^E81zbuT|8bv=Zl_ik5jbaCZFOEE8g4=*q@=iqJlde-kwab3~V(v7_=LZUI)FABcVa2ULmy7{l`jf=oPh^fZUvj
z>m6%4FJIQ$`o!rTJR0Z@Q`>?-(@dwu}dCVp0>Dc4)Wm
zQ_gX(kK<^gx6ysk(EfMpS%Vg|c&59u(@@53S1@V^*^xyhCTFbKQ-e`^jwC=Jw7FA3
zQB&g+*iX?8L9xm31@s%|QJ9GLT)b_ElUt^6OiCQsaCYMHYRdvQQkrH?kSKUM{ouKa
z@I^FOTrM5{+26C-x&8T9l&Hnwv+XB0lhd80hPS}^2P&h9h6;2<(xP-QECm)W@89(I(t_%l_|jm9WC-gMV-|bzwou^V_DV1${o-m
zG_fuqPg_vjFYf=V^B4B44a%^yeaZ*1sCxWNrZ)=Q>OH4R1oxyfEv+misH-c_>qqz3
zAB^1W>%~p!WmTkT9*q13qWn|UKPV|%Fwmezay~9!`NoR@LnA`zZgHRMJ{&4h&2Zk-
zbM&gVfU?LCSuy$tpLC(Ipoub6G%w^D(^c_GI8ULicl?o4am9$A2p~!m!+Qh(oHmga
z$#)AXH8?A=ieXFN)C1>s1n&fwkuuFSQ@txgk6;!!V`?k;=IYqNHL1(%n60a(QTeXx
zY{u*{a#@QHEGw(NS)et$w0NtiPC?l<_}l+4sL%g9BJG*wlliYu%>1S~LgH`h;%Rjy
zg%Xm3n2EWm_(*KbU69~QI9S7*f#-n`5UAe?txt|ln#Cn_>E)6F#%Ige;T`;@H
zY1Dhwdk~rBDWjhuhcTx=WbTVyOVv=WWq(p9Ki4TfH33bQ!VCPj-iZEYLRoZyN>qPvL-YY`cBqZbvH1S_vI2JyNyAI>}EdXju73;G3Lpinfz
z(Re+dgCxUm5tW{ch!q}i%W?7g*h46UpbfYlvm1)K1@^P5MhuD3yVQ|RUxnvvUYkyK
zr@j-T5@Gin&@IJ1DZBC{v}fU#ySjYY{2ObQ&STZ3x?%MvXv{Ol51MJc1*?y*^O)|F
zxZx{r=cH2?q?&x6G#`BjSv$Q8QFA26L|%kxn^xoR4BC8my8SXbDc`{|^hXB8G1~&G
z0GY{KCTpz*;3u#?qt^k#V}5-d{1qF$Jr;b=301YKW%7PPjkYKNgYapT=_q
zWp=!k`=+`gs?K#?4$P791J7lbX)dxBqlW}xQhe$z-2le(HN&KT&pUu{43BllgebuzVb5q+9D?Gcf+&~b;z@_AL;KwK
ziC2d_TC%23FeC_XOtoZO5K@OzbZYQm1%?GJZS59KjD(va$bRK2fFsRL1ZU&9
zA}#W>LP(=Yrj^Mzs`nBvf5CtNgpiL~fFcdRpqP?vzjFXhyLvY!B(tXKLai5cz1zma
zp=c0!XRH~rd%?LXSSojZEP(u)%rbxJmsi7U`sUw~y=l5f@7ARFy+Us9cR5AMx~^n&
z_G~xgpX*P#GB4?!(+thl^|FviBt3;vbGZ%7wtB5hjL&b>%z
zCv|0&TvCTZnxYllMx1=i>T=QS&*@dpL@7vju9e0e(TvyTk>g-(;Xz}v;s_MaY^csw
z;E}No=)aW5+>}-G4f>McfwFZZv1(0ZkyQ=MIB38^_yhF~Fc!E`>tvhi5c#jj28|d~Zh1asfOW
z0uDK9?QSPt^2V3zVeX2BYV~AKjVa+jkmuRl5m%m|{GeXSd*d&`w=(+w`vDF9cl@aC
z)otxM@jMFeEfE`)wvw-C(J4Ll^4w(w?A
zvc=UCXNb>0uDa-P6K*G>MOqfJnV><`;yWK1l`1+t=QwH3Rqz`3aXf=bHL~IWeqnb>
zMT$z01#M2B272TapLE>pWCM+_8Dcp8$b>cc
z#U*F9Bud?<2;+an%oM(EXGW#FMjh4TmUO-3Bj94XxDv85tK)|30ZmokUx8F3it2jY
z?*5%e6Lsmy5XF;w*0e(NIaq5W`Fm}u|7;{oXmnAO$(70i}$sA&&%u?16)8nr9VM~a&4_Dg)*
z+8f+!W1y(0$~&yu%KvK>q0#
zkiBRxnD!!l!u(=s1bs>ilskM@cNCB<$Vf0OC}PHMrXc&J;ZeeWv{=kPy&Ao@_W>{+
z!56MQ7DA@wD|$F{YP_vIoR++nmwKo$zoCeM4bm=xyE#Q+2Nh77I?%U;~oyVE>
z&Mn(Y0E$pzxfU{Im7v}%wvSbwi9Z}9DXMiCotwvpQ&_YZ1Q)SC2l2H|>^XnxMaf4jOBCi+{Xj`&!x!Z;<)=ne{JE~@3vij
zP=>tbOkrM_%P1ekUeu{6=D2LfU3~dTZO`J#l>+T@89q5KE`v&}R8p!__AX(1gQ86M
z(f(O_$ftHVt{E9@v?Q!nSUkS&C
z)m*AvO=4-O6-)NN615>Bh@&=Jr!T_To8XwK7q#XVeqmxv*XFiYAD)wjj*Ojjx~M|Y
zN!GXoL=3e+Yx{@q~y)IDr=CtRID+jpZY%cYDn<8!Md*~as$fPvFgWDiB(
zz|jEUn~V$mE_zD9c3w{sw@SIhw=w}X{zv_kgF(8N3%FOLn2kfTeMa0}?U>YI*+}g@
zb9XBuuj6nc%@@BvDDiXV``v@h?@JF?r>7(MUt1rS9@b@dWo8;~+{b(ZLx$C@
zsw=(l57=*dNGesA#aT`g@yq*@)&!(A3;K^cvE&Xa=Bx^xiq4AaI!$)&Tl(^#L>A69
z96eB^E0<8TN1IUWNy*>-oovSUCfbm<+B>gUXP9$ERR;qo;A-^2`i?%{)9CxeiGf7=
zT?O;o+#v;_>y?9_H*oRY9a}{j+Ss08*M^3Kcc!a8?}1kEkEUQ*@H=He%_+FUDpCo(
z?)6E;Y+>~)D<)*kn4L8qxm0)zV_4`P3|?!{;YwjJ73bq!6zccMIps^X
zUAzkKMW%FKot&=i?omAx58xa)IKCbK_h_rLUmm0v@w|C&@If!(lrg_IN?B)DWrZA2
z{ja`FF*wQhlf2{7bZ!gDc7x0LxpTb>|IIZs$Y}zyGO0tkH1y#xMrXo9nB_ITM8Vf?
z0n8APb)SdOcoyQotycZ0d@k=7-yxURwIvfw6$_ysX%`FU+8yf#VY9ki3o1aHg&o2z
zDs#b_I{`M8q$M6Z3SMT$QLAF4dw#g6c~f<+?c-WRaS+P9$x2=#2E)O{zx#Y%HTH$;
z=AIf#^)J%@evexU@_&0`l(LoF9>-?PRM*^IFzcN7yWRC3^Knmb%F;y{JSi?nQNlcVQ2~g&oEgP;970$J1OM7vwKH~J(9gIi#hvkK$hleT(gjZfBX3{*
z7Noa&e&S65{XgmQ|3|?2|1)GeYWm#-esA>eVB|7av1Zq@uLLk9U`{nn-|RwsA*~UD
zc-GR=@}G8j_uaVFPuR)>M#y!@ZO?Ih$j(EY
zxPPksQ@D&r%KTmE52}lgzPmxp(!>HrJDm93l;m`}zc{Hb>4gZ&iu9Gqv%AI2v?I@d
zh7fJ~^@U=n{Yi-2Qtn2`O!aFpv(e^a!6XxV5!CO2Mr2k=*ZcX;Xltn7F=Ho{f&M5u
ze;c`UC;+9cAoaI7(>OQ4LBe(wXukQEU3~a8Iiy!+4DBY(&M2+98AL%q>lBb|>mqvv|V`%4PP_ld!!7o+i1@c;tCb25j`VsPr|s(1A6Q-&c#?<
zT&XJzIFcW<*qdvcNa?E!whJGyY-PRStLMn5=MQVE&i%z^;A-9&vPro$iXS>De}bN;XG~{5>ZfhRslZW_)>|pUHIk51%9ao@;N3~5v)Y3P|ro@k?
zpP$MoeD!lcR1gvOGd2z4iV!~3jtBecUw#=_YhAlp28Gpi&RAAl&)~jhoOnQS3L*!U
zXAN0eliDbxzc(Td-$4aN%AAcdA88JYOwrOzWKxxoy5SQ>V-q!7j-kZ6-r;O?J#i6-
zAN&rW6!dTjoz9i4cXh|ypvQUpiCR>625rz29bmtU3}O-_+A)^IK$v<|$ZMUJ>+BxL
zaW8b2Ei`MI$ccmL(uTtrQWtKNRhyTI{V=#W8I*G_>^&l2nP%Ii8>ULZEXzuQBJ2#F
zTTUxBx3@8xwk>RRH06v}0ZcLx1KfvpKRHca-I)5%tgx_fzcJ)5zIdLcs}|Mbe5E~(
z6j!TmFt^G)YpvDy>ba{CSp3^O6Ih?ahdlc5^QM~hU?*buGm7810S#Q{8MmIy)%3xF
zXAbzD^L6f-@A`c&JB7c&kdqlUvDt<(^$OMk+l9jREHmu>?H5k3vPN_Letux@UygC!
zgvh`7LQM~G{(ZgFn5H`pk~oC*xAt0%TH;BLLQhve9)3yae@2T$z>eY3AyGN|bSnWjI!aV6Ee=1u
zK!P%nJL-}60ZWS4IwJ%l9(=Kr06XnzT22mij!eqPc7gPm8Qrg?H&@J~r}Z>syQgz`
z)k_QI)1X5^ogoPwhww9TfmWeoQ)r{RukB%-+Og2|sc~%Ot;l~-A3U?fP~Sv4qnMV>
zDT<8YrB!=KsG_=a_6x@C`vY{?HHt#`Mw{u}#lkA$y8r8Tb_2hTR0Yk=~wHkaYP
zsvdXA6Jb{Twwg_|pVy_Tj(_wsv3!jG@#oaUL}_;hO)qj52L1#PeIyOk5AC!|Fknq7
zkeA9`9OZB@t(9^QW(aK6@kamdju@#q+f$y3$hFx78{U+=q^*II`9l>`J5RIz{q;Na
zGM_&dR-$1Q<6dC4oM2yj{;Tfro?iLa5SOmlPv+48NiIF+ib`@@y~;X|2DHl7-{K-3
zJFnTjpEB^V-
zJ|%yjH6y_+R!U^Xn085YyyITAlxeLOgh{M~EZodg$!QOP
zN8LDjn)_}^Yw`_--0^)xe)oVSzuU-z6eXjb(Xh1=D&nI&`iUm4vHynxPoW>Gz$kgCliJkw{>Wlvea@T
z^^|q|OzFYc2dXBpX-`eukpAHCK<)6t`4>8{qjCH5&k~U<}!wmRIa@9Ch(z`o5I8rvuE!ZdQYy#YvB)M69Ceg
zo&~S(^2oTebKk;0!}p&mzX)0Sp>zjOOFb6pQ-ep9{ihkIEYj~R!EP^OnBeBy)(cSJ
z3A^ey>%b#Z$xHFXyHT&n4$hg0tr^Lef<=w5cxD=|e?RNmrRAXBG?G;5xyV1GfloPtKX-BV
z?mN^4oYQjmxj@NQpX$5`RkO*Qhht}!2EGZ64x%%4im|M7jnBu(nRfxd|M)bag04?6
z-}{}q)C{1;&u$G>|C`Hrpy}797CZ;+9()GweM~!Vnjao&=a8=06&V$^LWD%m2cdQh
z#Qw9-cFXax<8{!9fNt_iWFw`w>|sO(zfx9sRM6XP0#Yg>e?#_{eA+UV8ogBwfXoWW
zch+xRuBu9-+c6GV{hXWxTpr=WF;2n~zlWfQT5pCZ^%;SM(0h6i73~T1+sv>
zDl6g{ZH8o68-LVz7m2`D%k0)ANO7|%#|QS~c&T8G?mnP0olPlv{negVjukH?ISEfL
zZ!Pd^8j8hx!&>TT#z1N;SgSao<@5TdJKn#jUAA-PmGz3J-PC@phZFV(<%1>bSC)C~
zevCfR38vdGh+9`|*jXe3tX;O(vu~g@pS_n9Qu>imYVjWo>z)^;zYE|}#T0?z6uH)L
zfk01x%wMdh?)-t0+kUnCJwe2Qy*pV>D%j|9s{IjJq?_{o(AmE?5ds~1I8hLth|bZK
zAKx*D&y=T~0RVf9!S&7Q4Q;!Q&QEUC2CwYceGkOI!BIId!bWJ|!*xI~txdOy10%`%
z8}NIalw%1>hIk3G?6ohv*1BWb4KjdK7}u7AWC(crEJFnwswF`kuv#Y^iY0>n-W{U%
zRl>VFYpAri$K>;x$<^3rpyPW!1^EdN_(1Luam4VIeMkp0dC(dx@Nna~_uD)xk^Y+-
zMsfue?X|QHU3`vAS>geK#l@xMwGTISD}^ja@+}?$GE-DvbZ3B4
z{lKvoojFi^T3xl=2J(d31V|X0#Al&Au75avyJi0Z#|P0F9HN4gpn^PEiR^SEw}+O^
zU9{mYfIAVzEt#fPsLlg~L`_5zRW|(RDqNf-M8SFV=r`n&gk-h!F5oX`P7lVO5re`k
z&!ABU#dlc`vrsfa%iCP2Vt?7;gLGbA-o}2J+uu&O!4|D6^cKA*4U!Z>oHC3E!Io~#
zzsY0Eu$g|!*vje(&<#$NfI4eNHOV*l!f?|Ngf8t%o$)!Z`WV
z#dH%R>yIB&P?%%Y+RjXw|Jkjy!u_U$|2>@B|C%_2B6%ek8>Z+M24D)$Q~z0QHVGo)
zLefrY&z<^vlJSvK?5l+1>ayS^iSUQ1%kq5xTZBP6IcttNKo%f34Q6_Um)YF4JEWyz=GJPcH#8
z`1;&jst%v3-(hE8H}Fs-e+=RGmmkZr;PwHXP0RB>!*Y6+y2Zc4B?Amkt$ty-Lf8<%
zq!&KWIbj;oF9n9!qCb6o@`AXysh6(biN9^-O}&YObQXD~Uw!EXPt~b*?KpV2ipb$B
zIWWiGtQ?#5nvk3QIq)>t4;mqr+^UDie4&Qxf8eUoJZz_t{Gb^0u)?SAY8x9dNOAF7
zUB3A1A9;D34(I082aY>?m(tm)ukXpJ<2Rolj)xwygvusyu%B?5@@X!g*rYY+nwt7A4&fAadrQNaXg>MjT`J^6&t6N)y0te+~^(+IvvlP)t`-4g$T7y9^u;CNQ7NS8%r{4z-F)Y5<8N|X`ebE|nIg`PV
z$AC{kGQ;}B^*cn5Z#A~atucoH+K4a0X(ybwOR$(1ZM7HN((r4a2THnlop(V3b>8Aw
zxH7!ywXd&*2mn8df-_*VuQ7Ovn4#9b_t*=7&xZR*o%xH!!9z|kka-)2ky7@?E@R-W?QRJ6g4zQw$(q5BZI!^AIMc)!*1$g;~-x
zVmEHw0s)|YTgK^#>?`J}Hw%L(!p>FhJGw6F^hWANamNH#UbuzVh`+=mVQ2=kFr8r8
ze6Z+dCBo$(nPM(M=`6
zGFlox@WJeWAjW&V9rKQ)PDO}WVOLFw#R|snrv!iaiM=U$XV0nT$lZ<(q91<7GyILu
zgw<8^&i%;ou?b`veb+9(_JuvBXJ9ax?A2-OS(qj)YzgJ{@enXR|wc~9`D!_e}pQgU`(|?*AztHQ}(@<%2
zND27tfIfV=%wwYWQrDgpPG+-vbL&6Xtr=1RFx)LRqw}%}yXA_oOSF^`ozs+|uB|=-
zv~b43lr5YrhA$bVu|VA0CQgf$YSemY{~$IyqiuE!5T8*TaB{T_NH&FsuX
zMKqo^S4TcJ!Bn1bvz$O@BapBls_}Zpgs>FRfBI{2R`Wt;%RY}XH^A4|;Q7-B4F5jc
zyZ((yx7K{Jkd+``Cx$f5gw>1NuAR06x(d-gcaLU=(>7;`ihQ^_Yz_cY7Ay|EDMedS
zxOCj~%>>|J+C@d(Fk)ErDX&5_Omp;H=%V&(PqxIb1-RnR{5Z7Ejy(2XCBUI3Q>
z6LJq1oG53yEip
zD}bIh&}GB}Z_w;_1qyKg1d|zy#QV~+<6HuGej7QryxZ$oFBWpt@lI~KbU6Fcx_9Y9
zX##WCU%yx@Y|UWZ|M`f;{!*k87`_cweS}=f9j2zXk@xExZ6|*@m%QL-PuyNItzplY
zrTdPG>p(CNghE$A$@3TomY|4#yNkm#hO@C`y1IWoS|;TCnsD(h-o8(_R1Zp})|LrJ
z@|>BdVbtOCUtH>cd295-r{0Z~Xxp*g$fGSfmwM=ja7@O!xx7-2#xwM6#Mo~`Lgi6L
zhng&;BXRak?mb+>WXeT?Ef{fXX{Ai=$k^u2sa2T&9d-n!aA?8|`N%
z=_8ab5V;wz!wIGfglc
z;Ua7TUCi1o~YSapu9L7Ox%h-G@ADxWJV0vj!G?F>%*?e
zND$<|v^T5|Kaa^xz(*b6`vqxy8pzHhf#018rBy!qkcPu#nXHi)e!)%%Sy&gC7|{{a
z73TOD5Z!MK7gkKA^Wo)hfXDDbn%IQvX_)-TdZk(<3N9m=M1tW&r_^QX
zaFi79W~Qc}8KR(a12+@~d`lIo@&_X`ZpDiB3=dT{+VDvgJb7$3BR=~65AH1nv{v$-rzXi-_5dDmk}{-tiYd!9hK`RN#RSBO@kN+b_(YIZ7oX<4B@#r
z*QB>lQ2>P%k^}CJy!HjF1X^E0LL?Np7NZ0z`BBN#?NH<#>;;;$FJS0|X#oQoF@3Xf
zj(9jHP7zZGd6&`+0*Mz@ti`eUL{0maN@HH%ye94kmiUcsy_MGM
zu=V0_gP#5Uc7iw~xlZ60a<}{NOh4iSS2qutDuI;NDI3B)Y`Z+}aUzS+C-oA393uD0
zk)l#GtN32jTkPKm7YQcM#&*XZTErK>Uebwjj=YKZ7Et8!!xdq$KsOp6m6LIFNHXz<
zaimsw0kJ?^vB8sbo}99C)$@67n^S@Nf4pR4Ro9_U_3S?0n}mzuKNaBtL&yIq7}|3w
znj=b{{V^*Gbvz$~!Sa^ZuCIw!A1-OOcd)x3uielL-b(Ec=)kU?<$Tw~-Ulu{a`(sm
zkKa*I7`HN5C6EMD2FyP(9&)-9)te&dkAMjo$?$#g6h#%*Cf?K-JTKc$GO$a6+UH#C
zCFE!A(oH_tkI-*wdDKq`zl?~sg<+RwGH;F_s<8%<#BC*vbZnBiyZipZ#E`7#F4s8;}2HBL%MRt($HfG@i-R4|qbY;xU
zW#zW^(EEKa6OLZf?+=NVKz4xMxpYZ_ffrP6LhT!?DWLkhAUEwygQ=c+k`uK;$c5aYu=%K*t2don$F<)a1A5z*(qwvy_bksvzrdQkw*HlTZYzeGc6W)_vPl{gL2LJt
zvSl7DVU;kwR23Mg>-(binJq-);iJTqDA|lOTmWmz!$_+$%T}RmJN3fGw`Pcwcxr~Ccyri@zIySxFPCKX
zjhcm%$nkH6=IX!-mwR=51<8KNhQ8psv>pF3V%Ga*!6*@J5R92_c22^W#0tCylgZnM
zSt9bKIW}nGENjD-lXG_K-KyetBy%IXBmryR|Li8l=Kx@}A=^0Kbw&t=2h!#JSG68*
zoKpsv2znbx*L!vM1I(LhBr&V0pG=8JP?uKv1r9%GB?$K$FXZjp*SlL4nv7gd+$>z+
zq*IA#tX)(#?K~)ALIpr>Cxmy$`sYcn2D}s6dlRlLi{yz&!9Tt<{#5{kIQ--McHms~
zVCm{>pg$~a+^$@T#uQkdF~d~Co!*alb8FEu6gLxXe0g*Y^FQ9h^J{7O-emELMx*&|
z_2}l8#?VtSJg(iv!$zoF>b&K*489}%
zft5^7Mb-410XDU`vT!*8PM6Auvnxu!U5&`(4_^p}$rA*6(_=YXAD4dv#)W4vP`p$7
z+k~~vX`}Lc=-}69(Qpd_-OS_U^YIW9(%JC#pa`!Fg6@vy%D;LsoSgfn#lF{=z=YT{hM7*+jIfh|t_xrZ1i
znQ96$WXC_0A;nPeL-HU(nC47_hpmD@~7}m@MSSgJ|u7|i&XoPYS+t>Qgpsd+pBI`
zw%K9sBKSjV)>G&g7`n`WM
zES?7FLGL9;+)=~N$vvvO{?fa+k1J%{Hb?MpZNp>7cU9PawI5s&YSqYABtJvZb|^5z
zzR&0V=^{@MD^_Nnf9i632{N9qQ-@;MrdfYMWV>NQwwf!Vq368Ik<`w1=-c+C4##RU
z-(}(~7%hhscLBxy3Lxmu^La7&FW0@pM#UaNaAh%$zt`sLG|seWk;B_eBO{;2CVMEF
zFg`f}<9N*gvQ9E%<8nX!osU>%X$@m{ZFk
zK3(%0Jm@N0O#
zJn^#eLFwrfn#u9kw&xe;wmw7srH3>AIdaAIicHws4BX_hrV9q$DE@@jPJMJo}&c<3eh_7a7>1icqGnYdFHx#7mx_h68Or+Q>Zz;Zmd$2
zPQsR@q1iSe_ScbvS2Hh7EPF4~-2t@(-5DAwdudH3g~~{Dz}dK9*0@mOj5YDCWy`GE
z(4ZFzcTbl}$|q-i;)8tR%DwegsIx^m3{GO@rKUQm51Y>-J9)>to8{`H;}yc#Xyr2|
z`=sG^(F8Ct&}mMMOr0_hRZb{#NT+MBO4pDmJwLch?G$zT$o0<6+u_7uq>a5%O{3YA
zMLA?<5q|dCWh{~&1X2w?>UNwI4d|U)*N$rlpmZNNsmg6XjYgQD7EnR%=9r5WGxzSm
zMMkur-DiW+5L50C-{#A!f|7Qik|wm0J3(U`wcI%kn!g%cWIbXBKd*9|*KDJx(+;*Y
zTc5O_f-m|~E@EB#u-fy-C~E}h7WE}F|1wJNe2412_t^78&R7c4T%4=+Y9RvNuyoW2
z0)iD91YEcN{zeO@&Uk}Pom#)Id_i4JHw)xdjQ(EW|Nav`UL-5wa8LnXHgf0ktaqam
z;8o@@Ga?C^p9uV7)V}olx~U-+#`+mHG=bNA+8Lif0&t+WTJHnJjlS_+3HHkvB{KD0
zV&}lFvQukTyIK-i4N@2k2+3l+U51UWMA?OsdG-ElqWnG7LdSki(nE{w3?yo7C`87G
z?RgdxDe>mMAD5}Q^P@^0v;}}7?T(m5sgn%y9N4aIT6x!{T2tL!nRD;vShhUVyhJE(
z9qz_mZZ0M9xz5XHE|?Zt;gV;d?;L9&jT*6{oa>;0FCihKO~%i6LkAr_517sczUuzF
z2u?32g<|F5XExl(mRP)L{H846iYlSWI&YGsX_q(mn~z3d7RY}`cnpb2)MtXidaGB(
zsQKqv?(~Ab{qttYazUit&X|w`y6=)2R@;@|l>}~9OuR$~luA?x9~&+?7*edla1)RC
zYF8I+mR_6LJErqKV%-
zUUw0WZKJ!M-j8##wv8nF7bI&<_r=enaEot7*_Z3xOin<$Ru)*w^`WY*L1JM0Sx3uD
z#`gq0KEYKJ?LDkB4e58d#Yn4n7xRwD!!Q4h2xmstiy(q9{NtA7(h91QaDyg*W9Lw!
zH;odXs~l{%zB`9G!l;*5&6ZIWxK?t4mpkXqop)=IsMR|`jW$=2vlxfpF>2H2}-)8ffX=Lo5~lBIL%($oVZn?5}$#Eu0oek*(-Bf@BW
z5W3mY!(Tcd-U@D&i)X^xwA(fdQx?J-S60Wxn@Zhuc=eifpaTZ_Al5xnI~|bo!1G*YbMzrlLrlm(wp!OTRNDRhIB8+A#%J+_`eeFPnLuzWhb^bZX`a7wbx&{^b(-S~fa&@so
z_mp4uL#gbt8+wVpO|@I}4d@4{LYlzNuEFRBwO8}Y$;d^8dS$3yX&J+Pmi^h$Xmc)B
zG|ldwIK^HhM3mNn3MwC@kO2R>78nOk`bdrlO!IKnL1te0-2F;aDAN#?Tn?xd3-!=N
z$LqC+C8(;>Aq&amom_x~>|sXj?sLs&5v5ouD*>qz8HMA<-_wJ2#7uoCE}BrAutmDH
z?nn$GC%8^WuEc~pCf*GO?fIA=zXe7|F8*)tiaW-QRm|eo;twWo<0POd8nX)Xw*?Q>
zW;bno7D?><9Zoax=+$SwZQ3^g+N7XGz5`J{D;MB#eG|70j+=L#u71KOutG;$#wEl>c+}0VrP*w`h58rX
zhN*mTC@DP9pJn;*Z4H}U%92aZ1Zox#dzG3CUV2fn{Gm08YPn3*qiMfono0K*=AZ~J
z^U~0{F7!nI&3Zw+;O38jy*W(0bnCf--Lb^*z(C}NxNqFFKd8rf2rDtp*stg$?Ic(~
zWbS(g%^}sff#=dSK&}qDod+wS;8`rv9+mt!D)++rVeZMJ8z9f3OJ@@|KV@U{gOzDU
z9M)^Y-bG&O_XN)jad@d;5FK5To!|MH?8+`JTiBRVv&$Dzq$3S=6yq-LlZSQ&!)%uj
z>{#`8Jz5=7Hd=wMsHV4ycI>^L5S&$||2xsnQ@FTo;
zX7x{SzKh!#{a|ysoO0V9nL@hzt#s_j!>lIkzmN^77((G#iV`=c`V!k5^Z%QSJ|}6~cln|gRQtNrPoGfvNeClW;Ek0*Z4igQiF_j6RAjwje|U?v
z0S)$?rXb%5<4?tSd*xlDIRcyvI_aojH$)g30~C0$c_?7Rwrm=z;HXo-x1?^EU(Fvc
zK5Acoy(lT4o>dTNS+mwL{2Z~@*Ddi4Mf8#{sQ8~mE&!MBCvf|%kZrbef|Bdk#2YGl
zRFZ2Y1=m~AFt#UDtttoR_`R_kLji(j7M7x~LOzfs%J>F2<+V<%EY^1!qW-FQv50=K
zM{GPL96*n45}bSQksri2R6sW%wNlnKI6)n=Eb%S#YM0ehAcPq>R<)m5JN~2k++1ta
zDE+J6c9*{W|BJD=ii@&+zj#S0C8R+>LFo_>knT`QM7nbXW=QE6LIr80rIqgP25A_i
zVd#co73p0hYB1abGfgDQGiu5On&VxkW%F#L1D&f76#q+NTP7W$T9#d|Ngs
zyhDoI7q{Y10?EK)p*e?HT&SexIzAa5DE`&(G3^3Hfs6DC$e#TQ8`b!QjWWC!9W6E*$>TCtH&lf4+DmntC)<`d+DS)e-WcJU_L-FvpCQ
zf-0u1%i96wh
z08-2M9_Av`B?^zJnHY83LLJ?(1Fm4=W1jYAEDd6qpzXlHpkzecWZc)A74o{_M-eP%
z(*-yiD8oM)orRZsL2VU(?_-5~AYpc3N3N+lj27*SFZ7;()vF1ql*?r>ZF5JXklFEs
z`je~Wy~^+GOYJFeiQ3j6u8=r`Fvzx(&ga#|*Nb#To(`zixu6@;Un?hp<9J>}Oy%%>4vAFoA&u2piQ8_2qyAYqC
z3ZK(6ac1oz6~ALr5lU~gm~oSMq?_;RV>X72)gz;!k)tcaj&XZr(N(|DjBeKO3+U)u
zI3Tb3*)rmKUbLSXTcTHcaa~ZURtPH#%8F`5Dt1lkEZq2qr~Mo1$L>|7n-6mXYbe0GskuXE`625J
zvDP_ICjbuLvk3U%$rmFfUG%Ikg7!&2LG)Xlyj&286=)F-B-iZ@h+|-uUwf})JAwyi
ziX}W$Sc`cZOp*BB@3hhz*P5D;K1h6Ee&`7eU~KC=&4ReSY29ITcVsb&)-e&c2EO==
z@z^45TMj**ZqJvIMiGhc8v2}|QaCLhmR^{iX$f-cZXxXe(vwCgeST79Dl;8*sTciW
zOdeC`EH1}vBaoGrzZg>)6{==War;O2Mh=)P-^t7rjYxTseravP
z%luXIc28DlbrD%zR6Oc|C+6(*2!TIA1%g9ty{8`?_)Q7_51W&t>PMG8E!1{8CaDdiZ^Bg!iXW8A$`Ervw@%GH+!bCoz
zzEu%+RHK`Dcqj6m%3hf!NVWunn;vj%mjnEqb<@f%$AzzMaOF_p^R
zE|bUlo_c7WQ;0uBM^10xh8J3s9yclz5hr=gpqZ#_d{b8IIeu4npwVFQm`$s@eofa3
z{;M;e!fXQ}vj2BC12)#_6}X8dxm4Lc0FzIbwm_>9utfQ0W%spo{Ox><(aPQns272M
zb&z(?*)`BBK<_2e3Iv=P*p*vg#)Uk)bw^~TJ&{@Pbi@xCwKl0iTN@+fYiO&{S{^fE
zW4LDadj1h(tkdg!dy-%4{vP9~P|%O5x{e&Ca}3`C!nbToG#(Y7{V%=q+-0Z`RpT>a
zTc1DAv~A$v)%wsmS1vv3F>6miqvI&{v)jq2+H?gAMuY(0(p;YqTgaHW1_pi$A5?w&
z=dN3x7PB#FoIQ*}a}7f-Z5{u0dW&;x)_gtN+!<&yFFHcfg3FA29ld|<38AddyG29?
z?s9AXWSvv(yBD~+GOGoIOxwa=$&mfL&ip?l@E>>^&vX`7aW%*z*7ihWM-Wy5{^z>x
zbH-v{#ex8wDThsSn)WvGYz*x_UbwzUW`sGKBbTQV(HWeCFL!VjNA8yAGf6X49uoXC
zG#l=b^{1EsyLVeu+Hlf6lL(Y=&LIO7){6Og*dHtY2l@;MI-UH6GlP4YvQ5|nmpF8*
zZX8|3H}rd4hwseTLa8e&8khNFQCN?N;$?+2J;IQm&4VG*&hJ09
zMS%7LQNdmqo9Wd0!YbOm{P(qc`A>26KOdC~z;=F?FS|YxD+Lcu6S)Kh#WYocdvC9?
z@K2y?Xl#M5I|i(*E)yN3-Lmi1C$mK-Te4K90|08t-|cwiG|WmQ=cz>znLp}YEq0~R
zp@T^w+Xp-oek2{>7p3II~gt~xShBnk;F_?
z9J}?}w{-OQ#*KT$A1;Zvbag^QvwtLS@a{jE5g9SOowLT2;KUP0>^~nZZB!3_eTRPz
zeqLMi*I$Ntp4zZ*;@+IcL5AH`QC}&
zvIV^BzH95f;BLT(?!)U?M!HOR)@>c}UGq2XtjZJoRT67ltF}vk>;|x0S
z$dGKf5?mq+5&=RQQ>W&Yzr8gwrhYp+wKEmun|>w2_3$|eh;j(2WO=yptKyMWCx4?%
zk_s+(A$Awqou{Jy!hwauo{qISRuvHW1!{{?FXsnsIzHR(g0fbb(#>cj$;hZ5+5mxD
zz;O9hi-qH0IAkl(B|Z*aN(FG;EF%kgByyZAU2UOo!iGy61OA#rl9Tuvz4k32tAFqR
z2c!9a*NE(%E}}>+743r}m&XtS$y>#6qerMFtPd2Q0a5h)}C#
zm8a`N&Lr_P964{Sm7NR}(ka#Yq+5x6LcBa($gs2QAS)m_i(9
zO3eYi!Dc?5^h82(=$WObpR@cSPolctibR7=4?1S~9|J?rj)d(oB06bZ3ERoQUxim$
z-CYEKEhCzFA&9&f5>|5-vc3ePKgHUouY6_CvU)J^PE6~=AYJ&?WsO-KZ{l%153>&`
zYpgr;E;r%Tn|>Qqwmg(VI$s(nh@TlS+D~gC7Wx{}UKmEmFfT|c#JlB5UMg1oBJ1dm
zaZd1{d-R`QQ%kj=``;%PZV-a
zd0}*#&X*JmA2Ue<64EIcJj0v3B2t0m%n#Mn0N{c>iaBgwphTFZcriS&%0}>
zO!>OE`XUkccN}*oD>)&pn}0`XeK7a9=I9Tu$Rqqp7nfWfoSVhUM+-+CM4ilnyD-`i
z_=*6Kd*3nq-eX%2_+Crb^W}0v``)cYX+UMg6SYEhPp@|^eL?^p3(@$S)53*sxZS{>gw
zDdUEAW;gx2dPOAn;q9((#gecb&t0zGNM}2hTURZ2nGNA5Z12$gE=I59)EY7<)*O$YY>
zJl1S;5p0cg`eD3$_kB_(mE0nHx#sezS}7o!7L%QB4tG{hq0VnIx@I3t8Kb*msF>DN
zUB>47n1d>ry5?J+XWe|IoZ4O6olzwcF!iqm6|@p(7D@X14K%Jp9s%}jA(QtIzN4M4
zn3DLI8nZ)mPrCW+XvyWO$j%rL;Dsn>gR(eAk&pAASzd<<+-FI`HirmG>_05;KF;{;
zBs2D7sX(rgd+=u|cTmW`5{uWKdNc2)%8YX~hm@E8O3jZx6tANxd;((D0?_y=c0GN4
zY)6~VWmtl1WI)TJ5~stVU}QtIJa6{9asQk?VVJH8SjDh$t>O-9bq_8`_|`gIee?L@
zYV-O^4IlF$zlAz
znINT2mtgg}{@3!p1LlGJ2Ja8#|FSLk+k%}#TFuR8KmPKKKl78=$d80(Fy7r~@U*vq
z!&&YxFnyPG0;gwUn9v=^o&z089b=o_&nx{^wcBa9y7(UZTv#Kf67&35E)G(pq*~cF
zmGEzY0CAGp4S>qe^~aW1>2>dnUi(nsZRSx@-Lj1Gml2o0oYOeQ;?!N2a*o-m>qc2e
zJ3Y6hW9-?N-g<`LA}RQk>aCXiVfGW+2qUkC27zfGBN<0~DIT`1iUZwd7e=u1&_zX-bnI3V95e-^pQ>{J93x*N{!=N$pT%prT{Fj`pb?NW!6r@B_F}&BAe%Jo{7nw}B{HX^X
zsi?H~s(Xb$H6?60YNWG3uWs;<03h$G0=K}cFVk>wDbVW&
z98baOQx>EIhiw|J`a4aFJ2TCpTJYV$eLQO=+ZCIaM3*h*xY?@>bd;PCxaqMzf8`IT
z?AkN9LA;n;Se(zVJ!h_4`w^cf%MUF4ottn0Z(4cr(yPnhOXENG=@yAcD3H^RqsvK&
zrwAst`t~>jIe>45^h^@xNJ<T`r!J}
zvH}jRE~-$fgWS=RQ<$3Z89pra?nPGbP5{dzxDUEFer|VPA5n}K&&$#Fx3;(5y{p!N
zn*?Jzg@B=J)!XL;;=UIj3jOqhAm3v!(1k{}yCyXsF}9`9tiHbz=SbV6z1rU7*t}dP
zI#Qs8e^G{!`W-y}KV0Ts-HSgV$>Jkt?FRNim`}TVMI)|@MI*^LGb1xPK-NA!%j+)^
z*jUakm9Bc!NTlp$(IB!Jz3JWD>KnpZkXXWl&oKJjGq_D>%tAVh0e{dDkVf~f2R934
zLKb^0pt9SR;0D;&P<&QxS48=Evnbb2`TR(Q7;jb1DF#;ZraVpVcorU_tKw})hoF-k_$$#&^G!CoZs
zF7739PVB+F8pa_NxYsPxZqJRbyl6ub>z}0XHS3S;7iM9}eI?K@1h})MQ&*thy4Yac
z7aBg8eB$13k1>$a%D9|TH!%$0v~kL}n~%9AL23M+2n1i$+%Tk#K+fSpiP;))jVir#
zSRCGh_I1O4o*k#H^Lcx&8*)E{-l)tPx+EV8MF&V~i^a*dZ&`XXRE5tXdzGyVyPwn;
zHnye_?OctuC3DJJ*sj-}-2dAB2`{K%$u?n5JPueZ|C9CS>#T5<_>Z_>Oh^KT%e)9%
zpMLQ?Fc#r^3N>tI7VYcPHiPSpsx|AtyWk9%(~QJ|C~If|p?TLW^V-VF2F&H2S{aX{
zZNEE7{L+T~#etYCj;&63
zixilP9h(4$fD;AV+Vq!PtSj0x_mVw#G)zKjCaDbrw~ipgE{W`P=`XVV0Te&=knD=8
zG&Vuh0pGLff7Lxl#jDAW8&qq|3N}@J^p!-2kCye`rLyN2-}pV9_&4P_)LCupdPa)E0Fus%(0TtI?h
z5cK2SI*jP;vvX3QFn~ZBKEo%(&p=2^qahPpvH(&l5I@*TM@ETs-&=k5kRS|2AAs+~
z7)M{5lSy*w0D_FLuxdzZK4alr+u`(7Wj#KQ7jaC$sE2~f(jVJXDkdfddbSybFaXcV
z3+EaUw_Ivn^Y|RpQl+{2>xG}FIQ0o5zetvbs`AOWu;Rz#lyFFO`hUAUA%;{LEI(`v
zh!KNAuY`9sMz-pl`U&ieQo^#}ly6pl$gbUCj|yH<1N?SxUr%|6kvG!Z0TWLBMV^*5
zK>mE7cl4GVdZK`m6ab)s@4q()cQHnlFB&j;D2gHoZDg|(Fk@|9ukLlWum2M{4>{bIGMVRSaDgojZLoGL+`cx;DE|h{tvO4>Hja|Gt&RnerbYMklMP<*+Af`A
z|C89gc`p6kOOxd#Wnyq(fTTZURq
z&8>!`2p-aaEl&E7-~rYLyCA`Rb}xfrn#r=1^I93|5BzpL@e2R_QMf_06X;d|?4YjXINXlo4rO5O)OWJV;GnDX<2VG|y+*q2*l;khV%
z&Fd*Lodma@Sjstn3yok=R~4t=T}8YG?eA#)#jx=>t=O=G#j2b`8zZN4G>v&x*7$*~
zm5cBQ9uw`=!k;D2Lt8Zw1sM_|q(yKineE_U?Y(O=947#d4R6W;Jw2y%==v+MFvKae
zdg~&Z#l5)qnjR;igpI+X`2Fn;K-g*?DCeVr`Z~&w-)uYyc8T?co+rb7O+&p(6gVc?
z4lNwo8QVlq?Y*LaH}3Ubd*Uf?zd83%ZMox^L+oX_sg$8R1ms`UCyv%`$}O|P*S<*r
zr@dn8ajx1Y3jhivmBRpki{5~xC837w`kUN7XHlVg(4OenE}Wkg&Du;&hUg6I#83zv
z6F+3pRaY_yD7n6~9a`OC%z0H=a<5gc@mw_uksxnL<45}rvM;(u%{bEC<$+f7i=};{
zX0XnlaWMl5rt(3?WONEH`e7^`7=q93%v`Eyr6Nla-RQw+OhfQNY
znEnV~=s2PFFCBIk|JYZUN|1eTn%EmcsPqSYz0>RK2s@*bahI-va56F%w3pqI<{8Yh)+Lc#p_^5Gc+8g$R>sF*c5%xUQ*ix7vil>1>jgu*J)~U*j{RtI`
z(s9o}VeFTL;~-7=_*SN@@Qc*279McZx)UkRp6KgUYC$w`r#2WgsxKKA9=22A;y<^Q
zC(R~A$!=-zUS#Vfz&f%1=rN3K7Z~_HpsElQ!O!W9%brRNY6$mrF{Y
zTTS{s^w`Ii?`HORKMK!>=5%aI@tgO(N9>8%K?vxrqSuHeYT6Q`h#S*8d|6tS>$N)g
zoc({sT>>wOD?-?)Z|=C@d_T6*kUvJl1*_`#E(9IQ#p1?y#U?hSN(DtHGnQaAAZx~<
zmsQdx3v-E26Mp9p^ucz1q%ZIR-@4i*UJFF2nwnW`
zlvQ)VjK4~Z3Q
zA`$zAX*Yy^F?ai}C#yZy1#>TMn%=(5o7M8--As2jICt>5I}*D@W>ZjW$`sev9PDQV
z#<+@JMP*9CU~dG}56~HR+rqw*fu?1RYqZM!qf~iEV{fA62oY<1D37*ln>L>E)pk)m
z6BR^?yx;X1Z&@v+T4&Zx$q``5c!_Q2SZe87`Uny7aJ+bVL3CeW53w*P;A6Hv)N+uE
zS}&Uh(0@FU)iAL3@Ud5t>`WYfG%|s4Pe5g+1Z+Xk6iy_wDLuveBak
z#tPaAMB#$_wWh09v3X#5ei-X5?B{RF|1zk*JF@d!#~FwznOyZ#e1|_#DvR6tZ$JBTCuW<{nS*meHg;gLJC{Dy}q)fMo4gav*lV
z2f4<-XOizeHigtkF~8_|6f+c`Q3unBJ}m^r7G1ryj|RNh&m0-=*WhT;kl!iq6YrKC
z`pcvg^qrs(+ODZA)%F}VgmdoCT$q>0UuQRACfVms!RTb6*)hgi1E!uZq`utWI#1Y-3Dla&o*Og&+yTyBh1id0~!*-zy#RLhxpi)3Xf&Hu(t
ze+q|s+M?~wK3>s(0O-^uNf1N@zh16`*vXV17IS|3HvcuXK=9H27ybc*KkyGOYvO`g
z)!DC6gB?$0&vq1;K)d#b;&-C8Ejy2jg!suCdb;?0a|9E_>(k(rj)`~N^IYK6Ykx~~Flf#3dd1Pp
z<+^GU6aK-|wz=sx)ZSqJn4mF=hUy`=xU{^bG=|VDB{W0%!-v!ROQcT#(xAOvN2;qs
zem2tcA^+J#_5!yj+mr3>(S6Dcbb?}J=;*SYf{uP+#Pwz>x%{egzQ#RmsS&Xn7rOLy
z-X$;=V8x`v=vJ<{cRo-fYS>wwmz=HrK~fK$^)+&9CEE+Rvy#Gg14&vL4L}R3Ol}^a1cs4Ii93X|POh~(oHpQX16#A3di{o^YrSL&VG1#$!jktxX
zCkoI+zPxq6*M)OQal0uNUk2#tm%3G*;zoCeGxS5WcX2t{x^x0
zYUZ2ThOVNMYE(4EAP@?0%B~>SasXy$pOFYn@ZaRF+Omaa%g4R@mp0Ekqg{P$#0~~{geLGoghZgq_;##T-)<@cJ80!NAHsof`MTNtUzV)yX;X|&>R$xf
z5_*k|INgb<+%!SjS(@ebuKyAPS-#A5yg5s91Eon>+t?}u7$WN45@Y7(J~@2_mFaCp
ziHE{__ebY`TBqzZF|>0Ma?NkR)fejWrd*iR4G(ON&wkYD)@5VrN(^8(gh
zVl&*|{9VW{(@sKQ9w~@tIc0?IEs5)@kjaD#(Q;d{OEO=2T;H)J1?u=u_i<#baTV!?
z#IcI8B(3-dIA^m9dp7o{>os$EJI81=ehHG1wONabU3fj9e$`VLJ+)|s3I-Z3Fuq3x
zotZ7LFUi8Cbkz>-*&pf5p@m#BBc_)oin%53gvv}rvN<9xd9c~A&&6M#eRt#_RLN25
zO?Sd|uxYw)&9V4{v9wN;g7a2($U5xRCm%4!T&zj4njsq~a+HJ1{#N$ue5P+bTN9Po
z?fA0!$F5ZrQ#`k!E%Nhu0&4Qz;M8FjPzzrRKt(|)DYoC0cq^4;y{-79)$6AIhNFO_
zp!AgWsmzu?xUZ{Li&ON%{N$VLMfNkFg>b=tV+bBQlhwoO
zCmG=r=YxLrzFUvSqsI|T_dww}VK0d>4(O|F#dx^8;=(*XpAa*^Nm147`u@*XfqYBf
zua9ie5IaR#$*NKu4nEGw6f;wlOYv$k;lL}SPo4Us7pp=NWuiQvB(RM?NrbfhYr^k7
zxFGZmh`AiMIN5yU<*O`}lp_S!TeD=6j;
zzJ17X5~pB#ubQwm|6jnuYr+7>tG%LHdBmmF+nhz;s5CBP$$y}(&Z`Ij$;XO@y;OJt
zk}ub!efi)^XvF$W9l*!SJG=?$a@q$Dii7Q0Y{Awr0z$^h?trmk#G#V*{jg{bpn+6=Rj})*t$(so342aFIx{qZzIXIHC
z&6;7F4r8LsWLHJ&E*#twHm>3-`V!8Q*_U4j*_^DH5PWiA_GMoD@|TNIO*^}}t$)}NHyAo{V-8-sQl
zv3)WSsCL0h1oL_KABkz45qq6d>}VBls<|=biK9A8%H&|1lV{
znpt+QG_C#AUslBboT67qY5`zE?-dCj-&s}4$^Z%o5ob*g-f@qO%6Alv9O-8q`MxCm
zE0K3EzPP7cUsp56v2CqgfnI58!8+C+W8w;&<@7$uGBkJCDdn#JP2m?4Uvj#c!b+BGd@
zWVhp9iy+H_hS%`}?-e@^u6G&+JEJGn;mM-3jUT{m07`44^O^b8f#0rc%EmLa83xVUJP_UJ4>q8!|
z2+(~9QE>IttLDY`YaK+xT--W3d*%h4-8>uC>nm+jUr$i!eg#t*{JWN=U-1=Vu);ZI
zuJMWgquOBGcw
z(=eT1-mL=5{HZX`O&g4hGg(ez@!eYb#pD0tKI~#lBM*Fl5?}R1``hhCPoZ|GSQx&ZDZH3XUjO(aD#q>
z({nb3E~Ky6FXU^mYY$aZqzavzP?2)EcE~BsjM+jX6Sqmq8%T21UDne2ad
z5e0gWYrhXZ<_TmM-vw(v+K=EtTAr3@qb3$_-}aSS$6FhMJuFe}kNHOVku{XsM{71%
zw-e18%qxwO5xauqIfIt!wRF>o7ftRb_n!dMWX6o6hN~eL-DjoX7t;U2S#@04y0iDx
z2(agB~s-EmWYHd06Z$L&0-}Hc*5U@9k%LfpzT`n+uvJtMaPJ)U?FR#K|O{!
zQu)=Q{$VL1&Rj{SCW9>Oe+@+<=gEu`<=vajCT&_YE33gp-d52BHfu8(#0+k}bsQY&
zdx4Tj4muB(tNE9Gt1{v4;w#Zyn&)H_M@Qi->s6Y0ZYTCpeS|F8B
z+*Z%ed5`V2Em9b7i`Pi9q-y3M;7`~T(Ba_GEdPOW27+SjA&hujxUjH)F)T@qs-jE0NK{))f)~(&tfj&D#YR
z(Nj{D$1HN+#|Dg?lnzzO`8%R#8{FFmCZXm|U7wZzOc3wa<{f9=p2v{=>#W%R^ZJF$
z17baJE3WSJqSXIQ?Wvhh=U&4|l&`z~f(sp4bw$~s6SDwaW!|gD)dB#i-A=-z~2O%sUX>Fd2^BB(}z#YrO3kZ3(F@?=QXM%
z@0l<;!jO^1%xj#K6^5%3lQfHGL!*J9ou#T4BjGxQqqPW7
zqATOnsL4k2uJymn8acrpN_PHamWppzNAJeovU0LerpDRT$*Kx@v>2;qTlt%-hg_U3
zO)HlOQ$DI_ND8HexST#v#$*rDB>VukGAHoc&My6k`;!n+oX`SA-HS1M8aj~8Gx3zHdZID#Y56MFUI1z$S
zO})VC0_1T2(*u&+ia}8^&F}iZ=iWBbSX51NXZx0Wd38oUa+eQ2CjGn%H3&&5{ROd;Q&7xee;3Tb@u5YC^Wi5Vn%_C}7)jQ}0SZ8-$kdOcN_ToV)k
z|1+K#^ADM$MBl>5!S&9BF$Gll1tm!NN8|gGQ#;5VGNi2k^qV^6Lyi#fGu$ayyI6bg
z^4_WkqYb9%5znk#C(0i`z4vD^BF6fuYqh)uf~rVA>BiC_=eh%C~udO-AkVK
zKs4?b>4%($5g5Onxa6A;{?m%Ia(ZqZRP8~R(kW#;0hi+lo%aXce$j5Xe!u2lk>%Km
zi?K+KIyYslcmvEQTHnr3n4SU`IT-vRb5SK*i&e6LZ9eg-ja}zKMcvL>MHfDWXRCAq
z>9DxN<0&^O9fKSn@VmBfH0bu9=32n8B9R?LMB||j;Qo{M0tXTP7`wZq2gfXU0DJyKmrciC(Dd|^?Z^_v*J
z*a70^2J%HmPnB_&ib7lux)g!C0dOczir2tgXD$tP{z4|BP7wL4usC}es)2=PcM1a+
zZ7eS5ReNSJ)%pQ~_0nY*fT6-llJmfD782wO?xh_QuOM&g-(PydyXJ&W=lFx|wnW$d
zU36m_k2)B#Z|)H|b?O^l5dmEi2REf}9?T6XZ#rtvLWPFb6f+q`?9zica+bi;nwD#s
zZM9GKyb>NS0|&=e9p|@-g&AUC@(#x$caIv@7^CXNjK@^sSt=@WN);{YMgo>QSlVp6
z`4!7LVx|V|jf(XKUpp{NVT5I+y~YgO&QV4vNq2DtqX
zj-JJ6|3Rz0@mDM|hIzB%Ap%ze5RsNBwyc1ou8fc@h)qERRh8JPTi{{$ZBwn;cR}{k
znwD?JpvQ|>+Y91O9QAoWYc_vBg^@UU4I*frwsR#}QhDGS;=n!5Tox>h+F~|_{=cvh
zG8EbgImnTtKmy}nF#`FCwH&`7B3?r|BOYD6iN@At
zmkjT3jSzAxuMWJzI^zmni-;;{g5}d;qWq{?KTfuNY<;Xmhm@Ha>eQjlv~wX(pBdCp
zTzTR4SQ;hCT1dsit0=dx-e^I);P&_4=W@jbHg-KK^f`{Dihq1nFsNXprI
z99i_W!QG`c278~#+q=oI(nZJVuQAViHy-L+~5C~I+CeGc~L*`SRc!)SkW_4tba>%_|I8^EixYzNqsv*=ECvCr}4G~Pi?;}_>2
zb0z(%EF66oS@5|6&k0ZR#y)xREZKuoiCem7tE#Ss8wWC^psmv_CSU&a2KA*y(K^7x
z-qi4+dj-xpRORZX8YNofb=tFBMd7x>j|_`do;~TAaC#z|+d{sAc;#Cz-Q}9k!f$Wl
zGz&L+3x9D{I4Hwr_D{Tk0b(oTU%}!%9isIpsV21^p!D5ZaIL@XVB2C=-WcC3g>6fF
zw6v5w`8Xqj5+8q@xoRJR9dTn?kZ76A&(rf0;&g|
zA79G}bxD3VLo7ElIKC$Z@vLe2^!I)Khdgz8a<#mtK~?`C&t%nSFQsA_vc!V36eUv~XoS5A~+8lfNV8($`91~1?Pq4#1nqEk|3_f@PY>60O&*VUJ@s}+a
zH(E^gK(h546T{m^xdJC+p@As80O#IwS&|TmACUheW+6fC4tL|J(B)vGn2a{|lqB8}
zXE^?b$#Y--$l(1QN2{93qKGYm&a0YRT6&E`
zJoP?(g0Mf)2$_<=X^Z(}W%>pmFsDq&3a}2x9}(v2`+%}wp0RjTgQY9UuDcT)i1jht
z)Fdmh+)Q-Bl#Xz}BW@(*jB1COu$evMF+CtD3$TjlktEYc{A(|eEi)Yqi&nky{7Dia
zQCF++_(X8-G_J%tFW8^-!uu)gVp;F)cq)-)dcT?!3yUq@IK*ijYLA0!hrinidLo#2
zX}aDeCtc?HD$t(>_IfRh&N_yRCuVM2sw*RUwnK^lXp)ug+WnHzgoHO0E)R$JlKEVT
z_KS;U`RjxL8Z+OG=&@0(>?v>lTK?K!7p-)b)}utK&j5p5HA}CRi>>y}(fmCzVa#o{
z0#yT-NgE&@eWI3ib0OomeV?>>6z0A>x3|w61b06Vo|BKy8P*ChT|j-pilX9G&)AzL
zx;{lAPWvn{(Szas%PZzKT=Q?Uw0z1VKjIZk;sshub;YzL;Wy~3Q7o_WUc;I&q}1yf
zYk^^&T!v0;`!~O6p;0OlH}&_jG^$Rwsr8rQr%v@ZP5VxYGN@!%t=+DIZ3aDFSYY&o!^DOCza+YA?7DA1r&0$Ar{Nlsl(U4z#WkZy?EVdLJjo9J!t}qRqJzE
zc_$8Ar*^#Ih{Rp>N*=R#1%mQ~oYZd=_M_K;t5K%3{oWK602
z`uv5MOvgb;jvKWy(B_`c9+HH%i@y9V`Gmi~&X0dP@!0%}b>PS}pnb~k4SMO<<{GE}
zdWZkl^*-pUd_{{oljC~I}(Ujyty3O{%u9nXqcD@&=
ze3amCQt9w8Njd5?7QBDa>%j}(#G4Y%64PXFEVClWF$zAX%Z`_N7W%2Db@6VHm^MzYq-qSOB1m!7k_Mxq12xdT(sk{oTN4J!P)Hbtc%lz*4&#
ztE-I0z0&Sh{bKI_v62l~EX)>La3n`kMrv~_3ddl+se1}qb!fDt^8Ib}DsQ!6;T~PB
zk!8IlqYtRIPe_@hx0!g|s4}xTcS}3?FIqp>8c!uk3>}CMbWsdOGlSx5k*rr3ooryZo_BV(2j2>{(LQucc>rgr_yt@$4K*1nJ>d
z3Wi;|4b(%}n=q}`sU}$B9GPZ2B|u_SJU))x6L0DpEN!>@i|v*u9?tK{XUNOG+uFwO
z{C~t>Y^*}ucyWA-_
zyvm|ZLwj+OfECshvf9(mE!t^g@z1xc0l=FNd%0WcOTb3eyYm+Jy<6$6_tIRW$BUSn
zm}ebx9o{xM0oDd6J8y1jk(3VP-oWW6FjkJ)Lq)mY+xgQ)-#=t^Oe=TlThdKA&xFR~
zSUwD%-JHrUDhQOZbrtzMK>?o0^cBSfi9t;vH03q{0i7mIyQB52e7f$EJWJ_oL%}$0
zM&E|Jq}c1w0R#2I$-;@TMlSg~DII>N=yC8bS(ALg8*>vzgB1dChtXq_a}QwVl(1ML
zM8}9T;R>4T$ku!5;bzvxA9bUdrDnQx9nP{KFttxP?DW|zBhbYK@0&S4YkZk}=Nr~n
z7L&DJSy;g|ZbkjOVSAPOw-q1p45n_&miWntWpJ`^Mb87U2@)Oh;5ghCenPdu1ic*x
z0ZY1p#Jw#8TbAiZQ5p~*vRXa>%ZK4jvfdIEOh?yxhwRlH9cqZ25#Ql~ReAZlin#v~
zNaN+8iu|0Yk$gg9W2%CO=Ik;{eoJJd4UMANgkzWNm$3$|OHhE!U(xR^nTxC`AfaLz
zFQ)YJAF9vhLYqqd%vSdw#yecS3yYl{+T1*M_S;x<{i6jNT&-9d5DZcznbk493?lxr
zAoXTt$q%SV(>19vc{Rm~uSZoBE@rqE7+!Y!2$CbfiBs-*cbMwzI-9gcAWE6O2ZXe%
zShE8v!%hPD0r^&_Bav4rInXp{-Q&S2NhTv0(ss$kg}(_INY>*~-s9sp(QZU`N{r^-JFl1dqTZ8j$%ri}*rF;7om?^1m_mR~QIb)19teU3v3E-J_c
zG4V!q+EdNA^B#FW+1qYPa7MO?nbTIG9TdWC<`CYdioeQw+3^XYiK
zu*Jb%&+^UuKDoDCHd*c#8VaK6Er?Q-)0fpN|L4gu9UGLwwrM+!GGee8qzHwuXls2(
zkCUm5{kdX#TfoCxPnw^t!Gi=3
zg9Z32FG;p6hPZa9+C~(~ZRliC^h#7LeUoE;^TowA>=h`;u}k$boAnu@sUUn!*5R?=
zE)%V!k8fp1osj8F_H9seF#<9tz&{n0&E*<25y!YzK1{L)XsRm0o7ysH6G{Wu&-1!W
z{lT>fQn`wQ?p2+86p;Rjq2i$Q2h8YS;*w==1erF!fIqGlMt2X)o6Ful)DlHxNtX*q
zuB}WNXHXX_KLn=4#7BG4IOxnsw2LI9ywlSQCN8JD|9R7|_5M>lSeE~%Yr^sfm)L3E)oxY|gF
z8kmqOpalCVq(l})7_V;nU1Tb+nDBJV!D~`?si>V-PdKf;=26q_b>Q;
z{)hWG_ni0Z^?E+rAH3z3+vGn}L+ka_jqv=R&(aD#HN!9zNY>P^I-f;w`gAp`wDwEyeA#l1n&tLzRVb^dGYz84F-_)XZL)Z
z8jVq>P39&FKJy{_?R9q$*j)dW_d=N&S2p$XUj1xJN&8{-zg$2cL;t?MaO3_>*l6xc
zRMVWdtofIlTZ=9udnrOC5x-0M$!Cw_k3Ivn0lTbr!SqIDQ(tzm7q@wiz=}CSq8T`KIP&>z3
zhq{iRC3s5S9=|D#9tBrQ6Oy{-u|l(v0;%rV5x~=1K?9Mr*v62P9YD%vX=mlyx7?i4
z!nri?>1oEa49KB6-Uf$RIiTd@*Lla%#vTJr#kK?484f%$pd7+TQqViw)yDXzCELx5
zH^TS>(R)im~}`K%9_!PKa*aq6lr9F$21vX3|JO0KT8
z50y;Jsq-`19j;tn)5R0qw>n;xMF4BQsQ#HgWnL()mYpoQv!-j2kxjVqidmXDM_N4(
zoc1VQzF!9?E8dxHey_BbIg_;orJ`KwTY5NQ?zH!+%xM~}pp!`CJm2*^U_l=sFf>JH
zyA;$kBu>L>F8i0fOp~n8(bCb%#xM=qB^=Yi
zWQK08eS{WBw-3@xRh^w6a2L@rlXr;fqZ!^B`|2qy6S)1|`uo=7fV8XSD7Q#thg6qZ
zF=tH5asm&fu>V~P{M-`4fh?j)^vP)NFey)7&t2|UDcpA*jKA(B3RKbHlmWpmLLNob
z|6^D|;AfC?J<4d~lRI|5BA@#4>eGAqN_s0pz`uIDdaN$0X<=akBeDv>
z(k?4Z;=r&u_=J(Na|TUF36G21%$(*d1CSKE>O)xCO0>8{`dGRpVD$TpTvRvo2Oov_AW03|05eLs`P>gz9$_f67zk<`CN5KZM&
zT_fN$UIugy7mf%B9kes(YO@x@(;c))p{{Z12c*@~e72%Aw_bCQq~u~)Rt$@G9E{El
zE0;+h+1BvJPqh(DgxKVYw0*E9=9Jb_8}sDTeopqncs+lg3Z@YJ);xXBa$S*A5-z_8
zX5MSGys!d}A47B9R@-cHa?vImr1@?}?7Jb^jq~MBz08D+``2(T=cRq!lJjNS(vK7?
zLy~CZh1*=oO{UPXqwn4rTll)4Kx3Q}X;{c=P+rOw9cmrgHFT^ew}FC)zme%2TJz}n
zOg-Of`3q}Iw&ZuK9FfU5N1Q-!&Dh1p6evyeK+oKO&>8doW-U;maqHM6fK)Yj@knez
z_}3F48{>0G8XxbLH_2{hnO>2osHj)y^biG=z~1LCS;f`shY(HF8;Y!NcSF4&Z*EpA{)L7UX030!0KS6lP9OS@nAjqX7Q#ih75d+GB7|B`gRPjO@1W@6E
zm^MD)m*<&_VWZO$LK7YCbhyv<7WceddSVk?p`?f6)#d;#M?d)3^5k>0YBK({PQ9rs
z{yRn}vltABC42pNh^i;AP5%G6Ed78%4d-<~QVc)q{SNpApvs{q-4EqzfD{R2nL=RK
ztJk6rZ}NF@o{}M1ukLt-h2{E&>9is#Q$Bz5z{Qu{ct9oGpu5b+eygj%u$M%v1KZHJ
z9rNI3YCfmn3r%KLuzEs3k+*dOxs>AD4qRJ`{O+j0^pKeu0{TjJI23v<=ccSq8V<@O
zdJ;8Izij}DR!)_OTx%5LeUgu@Y@0@|KN+gU63F8A+~$v1NEC$%cc-78OYs+Gsb3a3
zf0;QIf(}hbS+6x-ZDa*G!E$2QtbvW@$UME-A9h*IMXx*PpZi4*c?l6LV~xu@m0gRk
zY)R0+{6wnH;-K$`WIt#t!|C{!p6$P}%?x*v#WpCI_O>Pqf61?5WTm{59`(#3wlku`
zhYL6#{q`RKmv%;B+A*r~IXlO#Y&s(S`pecEgIDz}nGh_X9APm{?A6#s#U=5<-S%t0
z`6tI2*Gd}~_;EQ10til3uK_IEm_6B_WCpK@dGj^4%~+}PPTJza2}Q^>5C73oGpy%M%@
zRT`3&J`9^B-}o!aAsO#%2z9X)=&RRpVh{279kKYFzA`N?j4>Pl-5d>{VX
zO=PseY^bLBon!SmbrvdOTJtqamf-68o{8LJp*SVN@lA5LoPNzO$Vr&lpHEJKYUA;%
zxYO%3e}BvgDwO_cCr%9H*e0eG&bN6+?ZH4LFn2g;%)5F9ZgFr9inctgGhl&zCu4f}
zQY~mlGm;+X0BcYOq++RZQ7>t}d0m$i2aSqeXqcV?Rf!7%9@>5e7B582;D$6x0aj4?
zmUWo=$A7(xBi*5Y_!%2TbWTnf{@v{=m2KWT16HToCMJ51Cods^$teN)OY6C2j@p>=
zFxia1>qXsMwq%kX#?gwh1d=>@#q~3w8O5&1;XoqKUfxQ6*=@3wx#b~u{JUI^uAr#_
z*KkqE+N8q{I<}G4aY=z^HEDc5ex6LPd!-GPFSjs-~*BmAC?%MsXCuN*Kw3L59Tddh`yI)Bdq)WpIe&E4le$2S&&j
zCt29+^%3D)P55(`fd21~?72{Gw3Z?00TSyWj7Y5P^2WmXe0D0ep0+YYxDtd>n8Mn2
zI4L5E6p^cJ7eqdi^8(;(uB}V|gtY;8xkpK9K9rXAcGOnb|eYN2hC;S~q
zs?dyax;(%kPuz~6J$%scEp)MA3+>wTm>DH#JcNx7>z{C7l5remKl8?GDV3-OaD1J1
z`^7Ke&HRNGlNAA|FNk3J!os^8Af2TQrL)3tSK
zY3YVT{;CZ%_~IE5)|v3Wufrcvb~?mHX}UGzaZ;SEN)l*T8JNa6KmNhiw4=j90SrplH4^v7ml?nY
zooZsdd4$=opXKq@+ikNOJ&}$h{N1>Zfb0(mGzB$Nu*SuHiZ(Y`SqVhlsrWypxtu?&
zP5o`@?2Xy?!k{dfi}t9@(SD_WVp}4J2ivcgOF0Xu4?1+s?)vc^@5+j6anIF4VveLsa@7wo
z=vuY8c3oG0Dv0j2S0ksypB~rh<{rFEk(vcDKi%nG^y!j%4?R}6=9CxvsIUtrfkjfw
zsoqP+P*38m$)XEH+SgVaoe|0z35ze_JW_qedpVuA1(|igHRuec0?f&nOp+p4x!{*6
zlwe%S64|0uTod`aAoq1!r-$B3slR(_-8c3aew)cg*~<7Fk0S8%F-d+q*-%kHr{x*;
zosDpP*Kpy0&Y=RGJ-D){hNU0~|Gf*~1r(tjQEfK3DeBK=uXK_+qKmcJ>dAIjX4mzU
zx@W=i%A?{1Oc6K2MSh$O}BGxOfdU;i4q`5|*@-A%DHZScF#g9=huH3&z0u3c03&OLIFx-jM_!F3?
zeLRe~tKF>(8D;k*0xQWnnm5`07sX~>@G`CcMgP%9dzS#Q5Z{Fi
zVnhYTba(dZb2*n4CWCs-W5R6R6t)Rgm$
zfP`XIXD+6Q4JaZuijUES*NQecbU35AP{k2%;$`C;^4jrhZ90WheD9Y^7t(H;fM9uP
zp_aSyUSSU(evyiVMq(dUKm9_F1K1iV6!6Xa5`8T^5`
zQPgoDuKP8GK41W%$DWvbG(EPy>@E57Y5+%zleI>6Uv$J_%H0zD)Q|X{J4j7V$_l@q
zH0o=8Cy<1EUm24tb^H!oVe}H{?kk@h|hD2R_^DVY03+5nEPUfr!|a
z^lKJUU5kuPz|U@Pl~*@b69|cCmRX^Efo^&YRr|NA{8E)yYc{70Zu{U+c@BoDL6ta=
zcKD=CyTx}6hmUIkr1cz$MGYsp7_!pRTga}5Adnd<=;{(vuV`cU0-*oD@g{j97LPls
zn2}E=GD#tqx67vpuny~({yTnslzqIN7L--%
z<|oT_LcDTdzORL!<}8%{!h_Y10V$R&pjX>8bS1ytl2kddSJ!2T8A{tJcPCk~PNidbYQ1{96y9(Mfls5{n#2+{TZ*n}|sf_DC#ftoU5
z7%r%lGjyfluScLR!yGf(FmYJ?MTmL76r<$1v5eKmY^#=M2Bh&FuO36C{+_G(2mY>hBkY&6E>s+>p8a%sZYI>zD&qAmfia%Yj?S-g(RPfQ
z4%vY9_vP6bt7fCg-~KL46AmVW{US=^O~TJXb4bwn*`3j9D&GB`PtVo&2oD*HN*nGM
zvEIUY$){7#K3%Ufjj!E&;P`>xG4pwqDHXgtGKr`jHz68n<$~vCBf;s-ZIo@D1NRKe
zfMUpS2N51+vMh4hT?A2iZ={y6nMM>aTOHdECHgN*Prd%^1s}K!#yB`Dp>qn1X!jD$SX)Q!&EN!i2oA;B+LHNRf69y7n4
zG;j{4UX@8EcvAybO=0FXwMkSWI#jc2(j$G%RS-tbMa{$!N5}?=>=r(6G`AW{W*QMt
z2p*Omk{p@XF=u!2&J(d2#11D>Vsl*>w3w=t=lXKcWX5Y%SN@*M$&R)Ct#Y@_kb$^C
z!?8^!V;S5ASM%YMewM;T**08_OR@q3%v_A&vG6p9lF*dNdSiyNoBu=y4gI(cA67D?2imNPRnWexE3NV+pC+L|VB^
z%pf{#5{U5q@=&{1tUjUq^!3bkUl;
z`=uF{Iguv(q(=$skYlqI?ePlik3dgXPWM4)r!WU|V1FH-eysRb?D?WI&$i!m+;~PA
z*He>kxdxsdoNZ4_1?7|TReRQ8qU&A2F!)h)b>}V3>iT+qN=Ty%bJw4IPhdVM*iB^|
zp`vd|9w8Xw-L@;%5(Bb3{#hkvBk<=^n@Mb-PETpds~&kelJ5m+FKH(-&JFav#Ymp&PKTYE}_`XgJseakmKIU$P~G|@>U{7<4qJaF!I$}CQr?xadf
zWVDqo0PrHq{LxVULM^M3POKbUseI`H`aS5GGP(lRT|H!^d3sLQvfMHN5I(v2N*4+N
z^Y>LJC-p2(cI|=6k@@dz+-BmqF2D!9K3L7g-n$s)E?-O=`EX$iOa3i8obCj~W69m|G-#Yi
zsN~Q0a1iwmfHhkM>gAu(-1#oZ9{|*ck-f|}vY3afgUcpl$r0DM%>YR<{OAT?fvBPR
zHN^)NX-~noX}pKI@ul2{TsPAl?eK+8>hj+D48kA3%|KT^ELayBYeo<7b)ZI)E&E{5b8|)60IF`yN%TQ0
zV9mf+p#1i-V_oX=_c^|IW)k!q5
zG2s1_ZfbqH)-DCKmeDncjAA7OfN7~dPr^n(je7$`C_4wa)?UDQ=
zDh762@<1B93oNU6c5nY&WoOVEEbiY=wD~Wz{e*u`
zzIwiRSKH!5nmzqI670+W*04hGY0slBo*m|H$$BoiB^I5$IX}+l#1TriMrvPdRlM&!Ar|x8Im(Gx{Aj#>IxZEzGA)PepT5Y92Up3!pkJH&F0o=iv=V{JVL;JXbmHg}!|@!%tBBEs2L1=ZDpGA$zsq;+|nL
z>n|GSJX;(UcQspUS)j9{A^5k)J<+OaLT4A3+EvrdhAMsZPHPO|zvbrio~p%fFk8aQ
z7MS%UW89T^lwrvC*v|+3#9v$ASc5FxJh=D5AI{`lukT69aLeMJm#iS@-PKpc8J1SE
z@6v}{l}Dc+ez6f4at7x_`0W;3?I&)u8G)9nnNu=iScXO6RDoX88!CtUv#$<&E<|=G
zoBWnd{au@?e;@?N0bZ+q7w)#F0t**OPHw{i@=fQ)Lv<#GOo$?}-M=5Jl=mYW>m}-N
zUip<$)m@~JbfHlq3e`EUa`vQPd@6TL&5%;@_P)DJW7-}Pd*u#
zTRTQ=7x9Vwgwvhce}z7R$ON
z>Q-<%V#(XbD{g~bxir6J{RuP-}z=?0NQcGvV4ZhwJdcFGWtrJmo_n~ntRfkx$$4LrZ!aPL!AFA&{
z03I;n)M4hEv9Yi8?9YF>l*-Ax$(yZnGYsDsY0NV$t(v1nLYQM=nC%0!MEoMjgi3x-
zj4>N@BPlC%Y8NT;sPx7{peTexMdKE#4Sv4F@(^3zCU~}W4BZ6B2x4F^4r*c?5gxEo
z!EyZ{ZX|!H0xVcEUSqJ^+%
z4VY%;_j3`=cht}=Ih`V!YB`j_x5J5C*CSqP5?zB(`cCcy*^ft0zs7wewRBxjjdv9u
zq^x#Qa>Ug7iY^FdpJ8CAM};O#GgiLia+qDcaU<=AAD1~VS+lBLNQz_ORK9rLVPK{?
z{R%7GW=}*WQQ*gFB#1pKkn0<>6${IAjeJH3St^J(m)*Se>7j7W3x^{FSYz6)E!eIa
zi#+1!s~L+dsjhDEZ74cXF*q*AVmXDMZ<7WNWPt*38qkypLbFc)_{J*zk6ha1syy5_
zQA+r1v^RM_a&mGWvkhiBQzfRpMJSC>z7`|uf%##C6>Ac2jO89bPa?L(A
z-Eye;fs?ok2yYHU64^8Wtd|7#21`Peq5XnPY`1iEw>7_lQKj;U>jl#*xLf-^pz{kK
z!4d7ZHE@v$PoA5t4w7V8@W{~o0iF-t&p;8)%svyJ()6F};%W2K=_#F4U2-kiHhfTr
zAptIPP(oPp`i$e8$z#(aCuDeg^NQf^gvM~;%_Lad1et4yliO%;>>0nDtM8{<^73ri
z#kch+O}QnI7goS*P*q!&C)+uc1%DuxzK;~lHQPQd|6k_a*N2_?g5QMnLgRGeq@95c
zA$U&rW6Ps6Vex^RZ7rOhyj4_4b?QW&Ws|*Kxmk`Qcw9t=L*`@v0v0G-emExpR_xxa
zhX;N_>{S*bcRh!B
zhq&q!jss3bEqv)Av4hm82<9EwuRA$ltR{UAs|f~$g%T#!RbB38)tNuyq`SkP)%3v5
zjbsD;!tF^Hue+*2wj3+kA2j62r%nUN`b_*R9N?zRRizPoD)UciLD?BIJ=B~ILv(Vi
zg7i_9kZ5~eXi!H5v6N9Ell*>gMQf?{0_QHm2gnl}Dky)NktnqHA={BYvbzfMJhCza
zi*b@axxqVkFx+m}5eCEE6)81EuG02&N4{^zrHt$E9t$7g(CB_;*#cO}b+@Bq7Z2SV
z5%cY8kn3=bbu}=16>Ij}5n;S1qZrdWI8Kx#HzEIMoHHm5AiR_r*9mi1oA9Cb4$wqC
zuXvSOS^Yyr6;&3-&r-U@YJMfYE+Qn@9e3Q9^6ka>LA}~0;BKVms#U|sI%6^}n
z$dO5}RJpjKBm>tx)uIUb>pyY7DZY`_j@T{R9_(i-)Ku}JJ5yM2=yrt5Bh&FVyJ^7H
z(;Bqd_meh5i?SUrm|2ABr7vR1$MJoj3$x3KU;5DPUm4o{C#}d==DA7c$0c_Fo28q`
zu@gkc#kaP;=HuI%dpcm)63xNRA86l$Y(d;wneS;sfFUA*Y3NgyF(9CQX}suSh1pCp
z>ErL&jLE_oM9|Vd>)-aqLw8x!2@ManpU-P@Y)6Zb%8E5j{OE_Uy>NTHiFNO33@ELy
z#2qV5iJ^zCwjAp-RpVtWT2zt6@`8*q48mDS$YdW^t7~4lB2b*+C}Os15o%ZOg+cLe
zT({f|F{~(19ZxI^zJGYh=GbeAQoo#6`n|>eM~_3
zFWX))q&n|>0Q`2k>a(+MW(I3GFm+m1PFM)bB~Lvj<%0u`FI%3|>9YtBSNd@#L3ZCL
zPIrnqR=rrH`eX;#fZy8F4v2H8o9P1c!k8zKV<@48IR0liwX6!bDbU!{$_}vHPkY8w
z=*yKgqghmRr>PHI>{(^JOg~%Adwas(8@q8=7tM#Y`BdZ$VNKwe)rBYLW4D%)KhqnZ
zR`F-4KjO!F3MN$|Y+gi>E}+2QTvpRf8#LErsBJ
z7zk(=bA7Y64JhBX{n1?)u?5GN{`pGuTJZJyf(_^AD4S=mTcqD+?qkZja6&X>g=T{&
z>Vi|rb?31cnyqyWu{Z#*!idk77_tiEcOg#PCEaOSWBxiGznGc`Ez%n%cVO8
z@MK(rMFwrrxC6L%f`x>qWH?s0mtl3B`V2M7@NTxS8ot2}6
z&A1}O8A*gXR1PNR_h6>=F0>b@3+44k*Z=&AMXT_w!_#LjYI~B4UQn{+Tl-x^xLh^5
z_QgowvCXN_m30w2m0TLf&d`H6F&;`AZgWDW)3iiO*~(+`sP&
z2Y_CieUWlCZmqUU+kO47>7|xBm+$p053MhgF9Tl1*XvRwEi79{&I3ee8A%LPGY
zr0no+MyxraP-<7w^LdOrDnZ>hcE?UfcOkcaQP`#)4Gx(2#{6SafevTG@@|cM2FMj^
z4?lkI4;%+PdDIMwJ&*Wa@G!nlrkuJ|6CdwB2n4&3ftEBSS&m~?D9wXmZBD6P$qu
ziQL7V%RK0Pid7#9<@WrPO5PB+13UtzSDQAPwdPH@H1aJL_lb2re=M`h$T6yRGOLEy
zcHZ({qIg%=`2YHOKIK89K$HJMkqCUoNM1HYUMo{iXG#2(!=!n`!l9&FwSxZu&CgMc
z8-5u!Ji(qBKsNZdgu12`;W(-F&VVrBBC9%2=lr41sQqs4T7T?mF7WoLX~3O)D{ZN`!6u|8DDj&DPd#jte4hjVp}kp5zCaMjOD~U4fTH
z#APz=&vVF4R^f%9eahY-Q%zG;*N$*vrtI?%gX_elC
zmi7eM=MN7u0lrt*rjIu=M~B$XRn=$r2>S0!!;>X*rOARgpCc_*N7tVms^1Jf4Ph89
zWy}t*)fuA0Zn-|S?|nNAJ)|f=(}#yAxJ+EHPe_)QmTnH)r>zh5&M~d$qdH7mV
z*Z_c$2VK0MoUAJ-h{QJ4Z7Y~EM8ueM$6dCC)5b8j=<>?ctliTk-CMP!I5ijR!bHe;Pbw%0gFNwVA
z?-dxD;K}Q~f6>G`WA)B_e68rqSAQr-D&ur@a|%Z}^x1>U5qwiSQn_;x2{ejcmQGi)
zsq3bd+u7AYu6s9{g@tfB6Nf>8$gsUr>ks`OQhixviC$Gqi}>pxP!<<
zYb)dH2WH~O=Ecg#-iu>K8Qh~;2*)Ejt;$hQP1tMG*L5<(gR}_YJob%pFC-uEzH)>c
zAZ<#shpJK+0_$-OH|0ll){C(_Do_~l8*GlKC#v;S&%bfI<4klwPU
z$q&x<)>nk`zpxCbS5vWMI{NyKyYl|Wq;(-e!|fS^w$5FzI_l(BIzCzLx5-G!MMZ>e
zYAH*O1a_?^J^FGa?bhTv33TlT*x=vIkc=vS0k+p67ln;LM#$|gp*(9L
z$w=3vN@U>D{C=)3XUcb4mrd-=(&H7**80~61UpjbCGT>+EP(jMa`A-XO!|r
z>d9cjb>zTe)4&(JXXxy!+bR#^L4w;vr_G<5zS4B4Zr1s&RpYhCPlwAjkzI0EML#@h
z4*um)8z=o#q4)R7@Qv&-v5$!OeZi*X;cLhncp1A$^Fo$4SyrfFTx@|wRh&;@L$Ly}
zKi30MV^3(B4wVjHU%mIDWT{a>G(cqW+@CpXL~LkLDkGipNQuwk^H6IOBBxv7!cpm)
zb5_*i`T$9W`cw*_m@=6U=SR?>Lu>iEHlzEq)YRC0KZlG>4nTAtsoyVms?jV*?cCkW
zq48G8t{Db~O_3E_N`5>z1={Ejufa|
z8yJp6+Yi@3%;y73f!#4_X66Q?!vRkGnr^Qko}3dh?^0G&{fc4R@At=;o8*yu#&Q=)
zo}VN(D&X}ZhC4w=T-Tf`b7J<&6J&=lwMK^u=_s=M=vl=0hr*A>6!XYjy|I$Bd>E%&6J@}FvQw3Lam&TufD-)cW
z_V*lKmVFN2DXS`3!7EQ@zDt>1vA_DTuj6iADq55`Vih_&OA|LCF79kl+re^YY)9h1
zXOh;0X^B|NP?AF?Z5B4&egpfdAlmTBn*x4!z4ov}-30vn=K7^5tfg&A4CS=j=-iiC
zXVock7`F4MFa8v`Hp7
zA-wZte-f^}(Na8b?>N9%G$d__8!KCr>n&j+9|mQ?*0F#_cyIj}dwZIFuA^)BNPFk;
zD$Uwm<@uVOSN-7l$26Wkkw%NI9Mm(mhZz`L8SntU+qst3m1{2Ezx25`dnN|&?cAma
ziG2!B=9kiq&vl}P+9I5)wJ
z>HBc96Z4Vdt(#xjiZCC6LKecn@65SU^rx)vv=7e#j`>yW)GQrQk%+jqN>Ckc)mPF#
z)CK^{Crr&U!oDJuTnH$kN6G}Qta^Zkzgc!DV97@%d)%4QTqnl#QL0XqJoDz
zT*tgierMuU@hVe~=ErbCOjm{R5bA;EPEoPrL#fY~fAqAi7ao5m!mYnfGVoL^%DbT+
zajTAqPBgDv#jfQerLa%0=UFjTS&!rq0=kzmXkB(asppkXTB+U=$
zP6exo%Zw!wibAC(x=%zahCFkOlgFASG`}&EjWEoxSJOu99jL;>-@^OYISEf93(AmcF-kEln4bSPMqn#)pGOJ)55=YV1U`jIUq+
z_A2IjA)NE;lWqI8Ofw&@VD8%H14<^~?&0I~Rp`E1z^;4T+W^@Ev*XL0>%&R)E^$Fy0suC3&*+TiChEQ&vq9g
z3O>U1{w+_;+ATTzC%ZQ5x!;r5CByur_A@{B|IoU0954~8wgE)c@Y@B@P(V{wwEh74
zOzwPURHwcf**dimH1JJhhOvTbcWUP@gZ8Fak(IiOmgT;l3I6W27*uBk4cLdmiH7p!ur5vIwSCpx3fy^r}Ab
z944a{{$)#4JerJlu*b!~8F(wG-I7Nh4@LY&N`
z1H=Evo7mnafUg4+!~;?6spymh&8^u$-0JC9Ro5W89V@AH;@(YzEXZPgVhv4bX`{d)
zQ8IQ*qna#xK~|h(%YOPIl6E1B;g&d$SRX0YPjXJ%q5U*pHXa#MSg>xcC4YVz!KbKS
z-z($CFzc9FP-W15W$lf)v3CVGHgnLnIt1UXh(yE%eSd)^ItH+Ls*I!iT4CC5ZmZ?R
zvuwu?42Rf1+ReYyf%9%Msg{DfhoOtPmWRM`!q=Y_9jevfJKhj+!s<2?7pnkAW3lxx
zE@gc=@(cR9mm00<-nIEPN$M0=Lgz@tt&=n+7W{68!d`c-$pp7
z9Bfy=2Aa2^JAgqR
zCU`9$itE=h@$Sd9;?_=c9wbZkuNS-`rv<}5tTjw!-7y+Zema|W+6>^#k8dSv0#utc
zIr(caAK09x!DftwcBtdVSl!**v(-fECsvYO08{W$@tm=_?KqirejIvei)i7eM?6cH
zyBsOmS@J3PU@^QRt)u?H&Lkb{Cbr2-!~lFX-sD$TC?BpSSZ{0m(D_(t%ldPMuKp;G
z?!rd~NsIrl_6GJ0zsT3tsNIw-cC!7j;`^;)@k6Y^ieJyxA{Fpc-ezOG@MPCdU(Hs~<K?s<7?-(
zM{!SN#xCKQ{h_kHQ$$%OgXG!^kEDlbTW-lI$(QZ_>CNY3Py#`rw94R~qvX()vX-m2
zoFOisziDu4QYoNBIV|s{{_6=5lLsMRA_U7@Wh|lO%W?ioQ|Y#W`Zk7aO{xT!hY&qE
z6z+GAnE#F5P6yKdc6Robp(kq%p1q+qG7Q$3X;ieAnD5kkC2?W@B0cWX6hkv8dFtZ!
zf3<`RkhHyga{HG2*hN*6I&{472}!_G{v4f3dH4v16`w{stwCG5dyM9!P+fo1$2F>c
ztPKDgMsifFUH6Aq`7NsRZ8US_CvbQuK7Q}6^7NZzj+ak(6#snSY9?{+>sim2D0!vl
zq(qMyDb2M$tSRm1PrX3SoKi}B?LwCVT-6q=SWrul23};6Op&g?BEY5vZ+NQp&%4-7
zUJ5l@z;4{wbuK$>E^yyX1^tCWC6@h>D~D5xwzm&75WG$d^Yt+SDX$%+4P~XI&-4Zw
zDp2$%4IFwOD=zmKO3Jd1c@<=#sm6H~18EC}G88;3#u%F{;5+b%R-
zCMO;^GoSmz-YictpHCwfCkaY
z;$Gv^?zV0$Fg{sMwA0qmvE~XS&MnJpW07mkC|QxBAMz)`a&K~#iSs~AVo1K+zEAG5
z-2wRNc-&AemyV&v31R>(6`AIVp%Y7XG?M+U=V9`)0+Nx}_*<#`G(wd8CBSr)NFNhwcyC4R8gNK3kTqx
zqd@VKwz49-z&dyhKw#;P1@ADJ-;VFPHvMx?Yly(r6g6ri1F((C7%)O%)!a@D;EEUho2%6q;1b&41J5Q
z(wd>n1+FIV;j4lStl15swq2-~8)>m>%@j)}s$*}Qw_lqq8&`hq$|6ktT*DXg?xk$I
zU)WGHbQ`@pA9L8a12f^E|A5(p&|~-XT`4WL@gatK#C%}FuK>p*(iG;hG
zmGmJcIxw@aur7mzMI*6uYLAztEbg<=uXACl`q-L|dTeT!qOT|f6=UTZH#ssNSjh@<
zEM8R#vRP9)T@=nSK6&=u%9F;tt!#6^{zfNr~=(%
z(IblHBjZ`CI;a^iPYkh^6ujBqV~m0Vo!=KeCpJmF;TQHwsmnvbGHI$<#e`U
z{ZAHB?n5i@yoRF9jnfFfG%isXGDC{2oh-;Wn#w#k^A0Fp)3PpEow?{3x@ZA*)^|Lc
zGnM$cuPWQ))4_<4%60qV7j|7;lz+W%ox77obeOS(rz4OBDIL$9fFAqFqG20^mY!DP
zTaJpbq3D*iL))+j7w@6+kc1-@(Sqo+Y%{*!L6MRYSySE0
zPne%@*mADvg^e)cPtO-aSfy0nJ-2DV)(Y2mjyQ2o(zHDp{Ll)w&d8?9(cn3+urkBN
zI9d@I`}wnS{PQ#caGNje{Zf`CY`>7ZhBBD4twPzKxbb+|srWYV_2foZhVfdIbzSX8
z#ls8Jh$<6k@_Gl$MOaaC>aSe#C~$z(4Vd6c)Z9rB_*8pg^O%tSlM&=inHZ=bg~#f;
z5Wv|pMHC|lti3Rf4wb2iQ2xUV_dm6JjsqPxeL52kWj<=!*BhwPwC`#RdAjWCRDnf8l4j+dnAGxh@=pw>FK(Dd-0(jFc~6*}3`!qZlha_kw(TV`o<1
zsg~H-gt9i61j3JQU?o@;#ftyO*jom*8MfWpSaB)P7AVEF#l5&ofucoV>z1y)x#Z+X51z}I4}Hu
zW$TnMdyOHsDm&j8M^tt8j;N`%xu=F*YHcRpgk@LZ^SDI8AL4sz9!|9BjzRyt>*gA_
zO>`=X?A~I8H|^1n6Fi$@YPi{L^wa`XcI?P`ici@D#?jO+YuU^xWG20j{wm<2f~6gz
zMuRM)Yqgpjt%KXrijsq-mgH%$UWkqrA>Z}wja@x9%9CW1{x=-nEmp6p^>gyo3^XK`
z*s%KyjQ_NoPR!Uk4rmm@WHvRh^kn={j5TkN{F&|4lVf16$DPAxA@{o-`y73z4xRkj
z@e-9B?lXM;bgi=o+87wTyu8s5T3bKuclo!xjB2~2A@4x6OO20$C+uu#fVWKbW8|F#
zJLAs}IjXO!X%W=LV;GCYru67nysAp%-4l5diJYobs^6@Ib;_~&AJx~zS*<=R;HB8O
zrhYtzg%@{i)3NvuP5wG6uEmU6H=O*r$Dx!KJb{YS=RJaR^jr~WiROo7j(zK!^THjZ
zmO{fD6L+8LLwjWtx4IQnz(?&5RCgNp<0nDYmX3Ni6>e}S4NoSX9I$8Ok26%Ae6J)v
ziRSULQ)a%~1iZjrc)YTl+~@2cRp%_ob6Tg1S5GsaqSgLXl~`_BE==*6uzf1P+$~;6
z1@Ln`Y`7c1?5Lj3qJ>Th!W5tj*bP9`I`8$|#DZ#E>*pP#1?U^49QqHg!iiw3n|ljY
z|3UQ5UZV;v=ry&r_K#nR+E(ZgHRqNP>P8VlJA$*zI{8k#J1iU^Bshd
z{4z~S4zw=`dzyvD*!QE9yEHx2qL{#vx4#wxK*nPR;FG#-LtNe
z!U(R~%PEELkc3}0Kt#4q4g>S)*mvvs&nUanO)`Oq7=eG`#Cvmm5a_!~+;Rs8`^@bx
zQFyh<^!k3udxGAvK21pKSQq5)f{e4xB@SnoEZO?ax=M{C9_Gonbb+oW-iGvCg*O!w
zv^Tg44c1hb!%syif3l(UqSZj+fb!(?R=$pbf-{!m_AKx0>S&9duy%uYZr#M*oE4jZ>prBwJ^-V
zfF|RB(~@ei!D$6hC`%^t=fxk?Kjz~1ei;KtNVU{k{RV?D6Oc>@ron#whyO6M9_
z(x2bGVF30Q&zaO7GA?J-<(&Uh1GgaU5wo;nUD$)cU~%i+eJTSj5qfiF+gpUe=#5cDVX5VzZhFf&>>eq^RwOJHbDY@+*WolrX?&Q#CdYF&?DR|feB~`&!z#c
zIVc`9gs2xb$4P&qyyqcsF5SO^sV#vCl0_H2`e1*N;~Up<(AMZE#cP}up7ZMI4lBv-
z!wCx2n|tliNtHeL?qh>{{(~lc7jJa>Xky7V(c_pu{d|Og-jDc*lz~Rt(zG3dj^}!+
z!1ozH_*v4D{1QP=G-%`kZs4jxipZz5p8Z+@yjBnu`Q^)bT#?vJ4({3ev@sqgCF-?soogF#l*sDIx-(zXc=@m)FmauKdEMp?r)jtm-X8PbVpn&OANQ-+u;9sDdLpiR8e>IH^jEB>(f+un7xQ
zk$H?683W%vG=raY&#X&PutB45WveHoR?Tl;RkPmBv#4r96+0WipzIlY#eGAdujB;i
zm`^6~)cG+YP-m29B;M=x@NPlKn{`w`10xs0YcL0iz-6%X*&TF!Q}yPoLd{?MHEsz!
zV<0+Xa>q?81b$``(cMlh#kq|3qBJj&!z0NCb5AuTZ5S26eQIHwINMXrXI}mmi|QCX#QO5G#~u1@}$$`TGNK
zjpqR^z0YLenDXnroLEwZEdx>k>lp)`%vWDnhXVNCyc{;SSxtHM?UEmIWF>G2mfLBL
zD-r)m3`dAZ8x{s+=*!Cyryh;kE)4HTknuE|>AP6j9Oqx)w)mK(k+Dd?j)`g7=FA6H
zV*-r0^2e<46p}Th?w18Odv#cP%1b|rORV>X5j~eU5*7^to=ubedu(HMspsk=t*)h>
z^f7g;$II3x1v
z-f-b8*}8T;)>`zOUaz0&Dnz2AdOa`$2H;|gUf)udR+mZ7m0}ew1TV-Ychhk|dmq5Q
zO`lC_?b2&+bC71~cNxp7Ugd85MCu-{9n0H+A$-Jp~tOItJf=lA3z5w*17>oDiR#r_+U-hyiY!D55b%2~5Jnb1?{xP2a@$FiQ7n?Y)
zbxUg!NJp{vA$zV|PxC_7>QP@8_kyu
zIiQ<`!dLt31EwF~vywM6%&QZx3{}Qq=>ww;QlT_$AI^O2Q`0~%^zUI2I-W)*
z`CS$d(hb5wa*(W2+>FNz%Di&OFq9lb!wS*)$lY00IXNRfazHS0|3>IGD80b+NqgiJ
zCi=Rzn0N{ne+C4*ERQEeRj#5smt`*6f#MrZcJ37j@dRvjF!(Wp6T=LOT^_eTy=CwLJWa1VmE8^HLsj!*Zx_-h``PB|loe#myti
zf-i-~F!)*6Mn?#f()k5-9Pif?D+C3r_3?ZKY8;;E;XQ#mo^RGa?5;TfE|9W(lnVC`
zQ<^_>DxUov#rqg_`xH%{|8vX==&P{1ifS`fdm(f30c{qBB}V$9`Gd1z$SSa?pVqtZO<6|Id`
z=cgx>VgXHIQm*dq(dp@wNjsRh@)fuh^(Vx1qKthQ^gHA5zMvm?L~I~N?S~#BGigXD+t78vi|f$
zEr~p15tCTnid=g_)ms3g#;Z9}vq1lS_n9`Sv6YYuDC85Lmv1euxa&UccwRyLoXIkh
z#pVrv+?V4vofdU}NXoFbd7MU%w|2iB+g45YXb@vsTb6_}BdOn@R7G7nxz71!jh$Y+
zg<_k-qoES}_sb$pauc;FyT`d55{1CZfv}8e{{;GAHhYP_W6H#D=@5^&9*h-rioHrJ
z?NEH)>ZMOEBhykAi)7$MLr7Sut=-oZ!I;Y)?80>Swow~rnS(6>k}|KgA|H+eiBQ5f
z&LAdBhfMNf>#4b?Ln0qPj-Tb-kYdP{&GsQABH0>{*|%cM%My{ie*=|g36e?nqKo!g
zYa>v6=x5xtsTr@mid0bb@949RS6~(BEGzX&@L^cinLidfS(5EW!@
zL%et&exvzh3hIj53})~n6%akCr6Rl^4UMlNbMuGI6@FGJYr^zUIvj`L1e-e)yS*1+
zkue(YW4&VVUZVogDWE6bWI6BygJIL3dSH=zx|mu+1i9O*Lkp8pzM+D*`CF%|5GHK@
zrSb2oj!&U6vBzz9l!9xjk3tR>630W-QSm1$knjOiHovtG0d_&y|K`1#Nl2rkRS9S-
zO|K$xleoyqfjahsBaN=%qb=*G_h-084>Yt#&&*cit)&$XIgDp?%G`}yW0kYp;F#Rn
zcjn&HyTDyNc+IwKs{i8>P%i?e2!-N>(odyAl6V5fva1#Sc42u@cg=g@OJb|F+5Ya)
zsRT|TyP>Q8gpQI8$K*_SAuqHUE*R_vTMPch5HFvR@~`ss
z5k#wlqhP&^t5fcM@St{o*t2TW8@u*xF1U5ibmI=x*?&?CVqI83F);JSBQgDyM&1Uq
zM2n*RPBaE{ZR0rBBi;-GqzZGdZbq^lyli*c%yQg}6W7QZIsRc~L*6ZC*;#3|Uomqj
zrN3B0I`X#)bqoEI#;->4M!uV^5E|7!`~-Juxj5}S>YIB$VS3_Bi3^ccoTnQ#7J3}p
ztY#_(4g8*IRyo&m%B9kJ(at;(QkG4zrCuch#=Uawf*OF6lM>Tg&*nx6q3X*S_W&XH
zIsFMY?vGGPb}+wqwLcXAIv`Bi-(Gn_*$9dj*BTkRD#9%^HqnV;qcyQAvx|o`qw(uq
z7CnaJ(4-o6xAiTwJwQyVX8p&QF&n8sIO7aXxP#8@eK-_qxV*BmIS%!{y?lD{H`MVn>}L=$C2hX}2Y!;d`|dlPh*
zvhbmZ>zvgkcc(r7T|P%nZ=Y2IM2B+|*y<|^F+0NQazXjhkVC#af%O>nwR5+1DJ}o=
zKjFa}f8qovlvrvk>=*n(~QQXJQpI6H@E6D8tPC8?2Sh%vw>!}tZfCNm~_jnhyI
zL5r>qAh(EiCGEs`*02Rnv}lvjyR5x~ts5GksG1X6s0_Rl
zIRp>77|?&6NqJD;xrHh<3JzM$?xq`a1Q)ytXJ^&Yq*>iW9Kp`co?d4eqjv&b!^+CI
zG!|YLY3Es~+#IE;^}hGYc^@DI&;~%nor{{*ASzzHf}Cd?c*7Gz?sYiK=Z;P;w*!cW
z-OcRG0HKcBxy%p^KW2CHLzQ4ofwr}{EWq736X%9hEtcDmTlvq1F-2*fsed)lby9MEthZjHGbkd;j}m%2t3;7lawdF+dK+i0WA7?ibF?OmY(
zu6zyo=cOw6TO;zQE55YTBMtv3WV^+;Fp6BUlD8}$^D#EMGtz#4lLsFnD!Tfz
zQV9Hp*n(_6UwSu8&`CE+8w0Ni(+=A@H=T@R+#HfIt{xztL)!O9XekG#>D
zm#6z<$Lbn#-woJSR{~Lz4gK$+f!(!S9U<|Nb-H9V^RB|EiFhjN&z3_C%v2|&JW}x^
z`EG#l{k!?0)EY%DL95;IhcAZk)%o*@P-<9l`X~L&4mkif_Ot8$0;5LUbnI0Wp9rKZ
zkEfc0?Hk3nxs$Qg9Xw?kA9aZWcX2&TQFlKLpVr@8rm|%9iZ*lFP6lhgI$+6Lg5D#c
zl^(4MG4llri5%4l>Fe;)rOkt1Iq_Mbn)Rm@HnFxRHjLlTDP-5n3kM_PmA~_^=ZKO7
z{oqW;*~xTY9~H<7epU?qi%h;IYu`U3uzX1+-x|{oCexvkKvT%IyCgntCQ=WDb?+&j
zuxqz2%9=N*tf)QgeFEohwUrqY6_oWWov7G3k*7B1&UKpH--?|3o-Az?fHSR0m
z3Jz-9XE?QI+z+-)KRc({%yerRSleD^ZkjkaRlL?;Xt=3*rff~?M%aK(TmIc$Ro_&j
zR%CBZ$Tret`0AxcS9j<-c~-W7T}zcy-G^xyeep685Eh}w(?f7RoA!7!Mei?UCC}X;
zFHXPpm_$xA&uvb7%|S=f1{T|R*`-!R?b9-6%fBc1qe|v#N#=rB=x~?5_mE3se{?nN^b&s9%nYRYXWv*
zi@u%4zU~dyGeK}~=Egsi4l->|yPKKgdnHCU$Y%8eT&Q&*b@X@ZN7FP58+zI?~}GVEV2wA2n$#|C7@iC#P!OS%p5#!``Bf|>>aO^zI3uY9
zLf|hEb&JO4z}v%O#Gj5zR`xaYJF3J*bGb`=`^wIx>i464$J+`WJM+V
zu-6rc*X4bY9K&0vJTZH0maB$zbQ7xSr_FJ!;ZiQ{@pB&*shX#IGJk#{m
zr804KLDfrJfF!%+o$4~NT}bh_-oZsSWj~>BAqN@`jOtM8+G}+Cuxm4Sb8vL|&PCiY
zpLzcRET4!u$;YB0z~Ok!!%2COA?Hl?Xd!|}#AA%{##cP^d9SS{H;ek8L_k+>uQ>JT
z+UB!#IXlB37nFYFz&?r=k7oSFwxZW7H@+rxy;rR|Yd6O)dxk@F`yE&JFvVo`siSm=
zFKu#P@+)3ap}t|4WpFa~_l_~uTiOGq`F5V@3*|r@$kMb&LEYZK-TvPGqHxUjfoorO
z`R6#{bL{6dR$skyaQu?1+Gy>5lSi2XM!@3w8loqvxHPqDLmALlG1<#S&_ISh<^7=*
ziZl_e!?N+cl6qNibNOV00TiQq**Lt{u0P<}2PEYPaNM)=ju`B5uuJf);Yi-ly%{;Z
zmTDL|ZVy3gr!3%$gP1`DZ`_^3!(bCbBW_fU;;Y!KvRs>D|0IB&$K}8NHCjS3k4*1=
zEy+uQc5!|-zc6>dw_>r@mZL4mIsAS9L?H2at2nd;Zfa@i*#f%+-J@PyP!73SFa%eK
zNfK2YO?MO%4D9k}oICZEBEH3#y8%cQ)?bTo)!$K8j>2={Fcl%RKW~)iipn$Do)z|j
zC8Zuj)m*|?He#+7wRl9L^hmOEShN&}F=hn>r#{-p4s^C?P^_c568P_U1Q`f_svl0E
z93+t^+wbs}%g^`oPzNspACcLfCm9Ym8CeGZY6+~zve`qsW^qy&gGf1NLvTTv5FaN!|q%8x&0&HLF(lpOH&&dS-L
zSQ8Me3YwzHJ;7@Fj#T?5#VhAQXI|T|HgbIJ2K!bN+5TndNnqS~{HkM+|K?BKn8+JH
zP|EqY8V}*1=Gl=k@1yi0fg*vg0V=^qFJ5FCeb^y*4i>HEB>W+|)0`Z70^$nwxW`&M
zAJ1C-MXcujZJ))S`6ViB%E?BX`_wAFgZ|Ov2XrmY1R09Adgk$|$%G4~IQ|il(&%{;
zI9FYFQypd%yRVoXJ1wRUG61Nqb}Zy8wCj?sA!k|~n}0<)bJ+uEne?ilRu`$rSQIT4
zpUq1HG8Y<-DzA$Mu|%D}@s%eXa`YgmzaaQy+{rJeNEk7M`142rdd)Y3K0q{s@zK0E@IHHx%(s1{u`n%
zuTX$2&O9RBEvW-kAyv)0c~@Gi;hOX)Y{Ghn$rt=tn#@RBD-WQmAB!oDT@UrP0s}-T
zz&k55%1Bo{zk0VTV4*hNrJS6JskkD9h1QQr>%#LOtS(yIw?T!)0}8=}&LW#gMOsdv
z{ARXtBHw6-_4#?(H`*1e7@~h4JnJTV@#v5UmR%HpI-lR`X)h%q_Kz>-U!e?L*ACI#
z;drq}hgb=#!iLy+2aSzo5xU7!hJ#RU>VXnEukM!2vitlAt+$#P*w8{o77CTY`1AWa
zQi-P%m)ni7&duwY8>x;Bx90T&zwWR-JnV%xcZu;Tr`L+Q|zNhi3eS51W{2
zp)k(dwq3iCs5e+2R-n2^(=Hz|p43iB5|2?c&lUGB$tAqL$TmzQ#cx)J=1zgMt7^GM
zSIF$yH82_SPBWY4)_|s{;pCLqC8Tu@RN}Y*`tKV72N*ek_)^8l5C=}h62UTFWi}~C
z$n;g5G~M>ZEd1n!QJ_oR&={jJc2MYNbx!+Qg5%-oZx~a5%08b+C?>Nnt+AypRZ-pN
zxCcQj4=j3OltgX1I|7gWYfnk61I<5p&rA#nNKY8cUCd;67Y4Eq0Byl4))iBwh*i%?
zyw9NNko^LM+C6QV@&*eCIISV}h$=6pjNZwbWAVmNOm5Mc7q1a}w!Frz#DZwj)jR?W
z32>GyDluh7*$ns^?v#GplSCp#2`5mx3x)1=uAhWSt~UW=U9!1uPkTulPSD*ZhMMaH
zK5tD!+w+IAhqtyG|0EV!i9DBE-Db_3z{HvwDJJm%w!XE5KjV_b@JWZ4jswB;A0sC$
z7;Wv*gOBArX){&Mw#_b(DI}3Dxi@aL^4d>!z5D^DjC__WesNchM@144_3jho!Q&UX
zq&-vW;M;fq*)50aVzoY=hB7Y`LT-Va>0Rp1?$`_~lO{Bl2j
zQm~xnK@2~b9I$>TZcGo38sVG7d&wP50Oc%eJ&{x&`nZA5K0Pr}ILiMzSaDos(Tk(8
zkq7V&YgHn!XYUz$epy8Gy|q2KiMC#Bfk-ciiG*GREF<@s1uMeoLfwoiO#zlV?xWw
zf4377G2Vd<9osRhql{DMV^xcHV}+7rJ-c_nr+~u5qd_imBGccRR}4=#tK?gBYkBvg
zP>IsXVr}j6#OF@#*6$3qg&|3T8febh$WZmbCvV3)tjDKxDwM60>k87Yjt4l!SO_@#
zN$BaSQS#fv8NmOUTKBEBjkchok|^3vIEG;Q5&xSB{)_@61!OCj@<0cYnf#)ImHX0@m=^SE}@ew6zfZmXEdB$LLb4KN4l?n
zz1mQ!XeZ;qc-%gA2Bn#@*mAh>lcy6kyc1e-cI9Q~&sS
z_^-8EQ3))~{+eFSl^N88Hs1Ql&;k@r?(}a-h
zI7pyJGrg0#5|mIG2ZLAc&-Bhgcc5K1$qQAf53>3t+XT=F!ncISl)hW2h}lc6$c7Uz
z_ffZ>@b9tAFLYSP9Dy}7XEy3-@m`H4UutOe(8AmB-b>SOF7@8Ht)`EC%=s$;NcI;?
zUUkC$Vo$8{#X2Zeq@?F;&+v=$`~^zLoI>A9*6=^s#i7^pvXU|2tTS`4GavLxiX|{#
zUSdj-0c`h{Gy~XAXdFX{^weh{_T%nM?8Lk7p+Im(f($`2@
zmz#JsiM8$Fp61V~lRyO}ia|g7KZhv{e4VAd!ZfUdlo#i}p-{G%=T{?ms8KH|0hPNU
z>E8(@3DdRtW-aaI@St{5adhsJ&%ZUlllNK|woRn^HJ3`9|A+-(iJ6}RdY38k_^Xu0
z44fxyULEzM*F0axfQS!EgPHNs8?B=@GdnK^s-j-k
zKJYcp-2Q$I+Kcz^O%Pvjg=L;>WfFdk=$H|_&YT^LzWQr?*%JHCpXv>;eKrDNY9`LL
zSbteAEt`tfadFlb^`v~$dW@=b>Tt2zWxB&KHq`^5NwH1a-XnkO>I4RKs+0xUKOX+-zZT+TV)aY^``ysV3I`s3y*t0)=TMU5B3eepMYF
zM-K|W_lBCbR8?ieqwVvTzC4^I|6p)`u2vbzt=I@z4j%_a|0AYzXZod!CVIVwkyI+7
zi0W3U0D%OghXeSnoe5JZrQ1)}@;Ray^xC-$nQZdvdB`x8ogO+0oD&@f%vT;<8`*g5
zkiXYhi=Bo9KEOKaTkz6>{MQJ99L@4;-V$^2)a$BNhURjtTjgPLSu$1$z1)hU+<{CbuIzR>!btJuUJ>l9u7}k%q{T){>HlcP~M1Z*LqXw
z*>)I43FmmNWu|V!fzt!oJO&r$20t
z7;M%;1xOC5=bW?Ff$zs$4DADqBL
z1b+z#Z>tL&sQU1pJny`XEK`v%dLi*VUoH|}k`-IQk15_DrV{rIRV7h=AIF;I_R^F{
zd}gz&GB>d;q`dfjg}9X6aZFu7z#?Taajs-%K;Kc3=A=SeSmRP%g_Dkf+?O`55P@;LIhgiy-R)mMowQzv
zDolX+Gnno%Q`F0ONo3eCZ7PolkGP0f@?D7xZe%Jxy7b|dbv|`OZET(z#4N8VrY1U+
zxiQen(cqhFVsOtNkHpaXim2KR7RHa2%jB{WJUbO*@y8qEo1ugpB+Ts)e+Ft?0QyG_
zHWsej$mryi{(m~%Hv5)-C81HPg?@q?U-|`C2C=JqW*OWI*t!k$!!_sLxSg5SdXjZv
z#Wd=*1w4hT(*$-M6+Js-K+L9fw04rCkVGoG
z`FE#=7Pyk7XF!qaP4lKh*i{A}c1TeXiH^D5*P@)zPBH3#FB
z!muwr`jYiWN!w`=RC2HK^k$2JbEPPXy=ss14}qZ3ERth?H8!6X`HfH8Ddioo>%1jP^?*T}!
z>8hrsvdQ(y2PA;NsB8R(;t0`GcWjl&lAV65&wff0NkbjoIhzFHdhPsv0om2aCHlPR
zH{km2V+=l}74`TaRRJwiPZ-Fi4dwRguIl!7Q1Tn#j!NS3w9_K+o=^#JjmIR^Q|yvd
zm$+a4bNr*pv9Vu5`J6Gnw_A495x!*7Cr&xL1oG%(yois6zv8d4qPIm>!CkQzN1mm*r4`=RCk7A5_cvVA>UsV!a^7J7a%;3pNsVc
z@!rFW4tXsJHq%2bSRa}_7|cMdv(_g`cy5ddRmbs;7-n{N7g}^>vA5LeMOKG9LswjQ
z)hmzmFHOxCJa0YoV=1O1HT`C{l9wM^|Ii6BLkuj3kJlRpXJe>}DqCl5zW;0gv|Bs=
z;hsgiaF4Puiqg<=7ZZi^GjdjG>~kms0g-3UGc$*<%`O`A?T@r1#_Z3p&eXD4WJaRP
z$4}h;lBt`0w{!~FZhWxs6!E86Ep<8$
zFtJI6d|TZwHr@qZ=3l%A>1g+tV4=;|>@}ibXp86moRg_y-^H7!2$*
zgL>u2r!hEBEh}p#NQ+iD6G1_x@+`kA4bG^EiW9jYe|HaZ?$LL4s?Nwmw!5s>$>ZsC
z^rwS_O;>hatL_ueH_6}e$&+3Iqq3~iwmB;hHaokOL7Lao)CYW+JJss
zp1wbPDZf4tdZ$M|x(lL4|(Vt49FWP(mnBG5rO&leD`
z-5|+cD5L>8%yS_|AsvAmCjycCK+&|~?1Okjmq46}O9VbrH#Vx%zA}OnJalaBBuFg~
znQ=z%8+`|`jqgP)W6%jhZ70gIzFdrvhuyV}tvD8C0_hP+Mt=jHUWuhM_l0RHoW%T4
zm5zLrd&|=fey#E{bw+pmdj^3+&|mQ}fy0tYJ?r2LG3}5bdd;*lqAe4J2o_J{Z|2H8
z-$i+wUWpl3-O?c8T+jd9*MYaszIV_lUjWhtsov)QKycBJ_fG0Ph1jjnk7Bz#)8D&!
zNHg~Ft@XzXbl(RfhfCXxz%p~81FK!c*N$%&Q5TL+oZ#~Jg2>{dcCVEgz}HZq#KXy$
zx)7lelb|vU=X?{uFGf&o^uY>85`6&m_JvW7s~FJkenkeB-|QH@zwe}`1P!fol4{3{^bLA(>iyig+yO0o*+_;3=_&@d{f2iuU
zJ??gH-L1)DQvvP+0V8Ls;bVWwjV~VY0R6N_Z&w&dI_0--LM34D6UvmC{
zzy@_$(gli!OQ(9~f=c8qJC8%Ouv1B;a27J-M<|`M1G|b~NYWj+xCv*MH<--G!~~@h
zTSs_^v*e+o&RSRx+{USh;e(2=
zmannjLm)vRtC?pKSYcHh5ckEF8~kf7aai@IaiI|lsRfQRNOOt!i9EI0w0Zz$6mz5I
zx1NpH7i)A${@-;NvRK7fjI15XPUy|tC8%&T@SdPJ%mSE(7)_hnu^3)Z8@#zV$>0$B
zR!f~`bCdb?kY^{7bA(n(CJ)+~4zr%^yWTUaVZR019@=uJta!K*5a0Za(j
zBAmq$M0UKuHXR@h;XI-Bv_B!HgrM)7z&z-`5*b7ie#_LtXXvgWOB0Xa-z0A=Ha8O{
zuN>Dy`=ucI+MTI%3La8NSUhHL!GspxkYyBJ+5YE5iF=rOu2Sw2@ivtOkqgK`EnG1foAb_(=It-`
zJ9G<*sj`AE;`q(F9zW#*#NsXq?%<)&Bb#^6*Qr4QCK@T>t_-@dzfLMj@9QsQqH2>#
zWI`G%MqRyv)}j-qbjx`0T7I5%m1HmxEk`oiK+!Dv}32O)#KN-Epo+fB}s^gofC5f{*lBV)7EUB|BF&_Jj9a
z0>VbaVBecNP~=z(Du{dUmm6i)_cg>~CVpz1;P10eX1{%grlFqZ-gLO)Ug`KN)cL?G)WL~U%LXs3lQMsSCK7MWpbSNU8EAzg9
zDlP}yqEACj&+GU&qsPB0$e3J`P2_RCS+#s~XlQMD%=&=0f6u|GzGe>1)_`FCq?M3I
zL?ysIka5`7k_>>F2J7mGH|%!#mx-DP6=hFk9sL%(Is3!klwuMNCaq)Uzh3EMEj0WO
zRObb=aJzXdsmozVyunhB5l)V6h$#s@_X1vY8bXw=?&K?c*5ljZRH=$ZGF;p~*PicU
ztK7tK^Mt{;+2tBG1gl@IBsMkfbhHQV9X5!tICUQTTl*ACsj)S4%uA9f-Kt=s0>UI}&{
z-5P2~f5g$>Yo;51vD)`f@F$teNRQs3=Ro23EqvpeWf?o%V50lnv|%Bm#yOVB3bfQo
zGi3P19>AsrGH5+o$Jg3xdRe-2lVYGad?`hl&1jK*D37LydBHY)F3yS!igrhs6$y&jDX74UKI*fZcs&s0SqDxv-{w@t9!`xtsMdGoZ3Y
zk4gK{uTpCyFAJ1bC-HqDRFL#HN=z`fb!EtWOOJMp>UV1Ab_@Dd-z`Q(3mX8b
z7I(@~_};og*tR0Eb$cZ)L5BGc^`U?`$Mn0eUr&H9nv>q7moz;o?%2DxS>c4yJF^iw
z3g>FX{C480A4=#b|IWR-XMa<{+e|_yg8HLXLMG!S^cBji)d!vQV(P#K#uhbOhCevE
zHKAS=BcoEWGK&!;bun2VOp`405JSa?LQPVfOn&PWu7V*Sr`}=e3@lWcFY{n!9qtf<
zQFZ@f5C$(i_~j)^K_Jb{VQBiCJF7;Du;a_jnv3ii_!KEdQ}lTE-7Kaat~kh
z=h3`zWuyPZ_T%XK_t6ZgM8IP>&C?@nQR3;vGZtm->~TYgR{7A7EY7nyUDiL*uGK@h
zHX_F{wA_dCLD3RGcMsCN3w5n%FljTS`DZ($4^vJ?FwZ>Kd8dPb8$1bD%3%dn(y&8M
zVVP(Q(S1qWw}^5kp%~A~V+Ta$?V4S2eH3Rlv;zMy8u;tF_0P@A+leFls8p~`A@!&D
zHVu3HdFSCgLAxZ_Nde7+pCShBj}d~qYp@UE8D2b*eSQ*V;|LAi8UlisKDm`mwhVv{
zssD(WdqV~Nn;5sX&dz7D&+dD(yJ!N!id>#vcp!pg49OUYk96wj${@NVLDRVs-ZUk7
zQt*W+k(iSg0xKW$oxvu~^bZ6P?1m{0+lF-Flw0oRLpuyxu-%z5>|&2B9Ew(KfxvESQX)XGLPcY!};}!L*qn(BF@oL23(6JWP3`M`*IkNw9GV=
zh%WKd^Ze(2=e8EsmBORrtg1TcY{4GqWL
zsRwi+AtK2rTUAnjjZp{UCx_Pn%Lg(ZElum`FXa(YGq+u5euq!OfJ9PnKO0_EfD{jc
zdpY7!lE%b)M6M@@mzMVwSdPf{=R4*yQ#B0?>`6INL+@Rk9g5VVsRu>mp))u(w#kIVBrE$LOdaKL$4r=PacxNse*fC#3?OHW
zl(-*DjdN^4LwL8ImnAS0wa_qkx&(K`ebr$7l$oHAfQp!=em3n>;feD$hnbvs2}`pV
zO&m4Q*u+%5pP9=NzP6W^+U4vP9-GboM%@#3=c_&2YoBS+&c_*gd>~{5w%bJ&y0kyx
z|LllA-=5CAe@3S;!5B6Srk&E_hK}tZZ_9Ow6!HG*5UnJbm!OPbI}JWX@Nd)mUkRWE
zZAYxZ_Zy0*7@P!wWtF-tW+M8_ay*m&mFx)I?DC9E
zqHIKnZVHnS(x*r(bcZxsu<4>%C44O*K&BC}Fzc5x)%dJhSdRg?n5%mX)t1IP{TY53
z78XG>q#)rddL@>><_Kj9BT+T*7_kgj;?R>IY-{h>I62`nkpw;qQ#5DycMNV4|CoAa
zc7jK(2CsK;d+Ou8l+$B{WCKt_n%F##{E21eDI(4IM^er#&IpIs2~p*7C_ju6^S_hL
zzk`{~@4CPCxyxK!kDzw9Y~!IxTArfF+YhXm=&)Ch)0A-otqx0w+&+p3B$EZgxRu9EBar1O#J+0%icDhtqCat^uu
z>}DX>5ipO}%j=gn5rpxw7*onKU-ff#j{Q`Bw;i=TM@+vJmHur9`rgj-^v+-h=plT{88fBpYk*pfc2(2y+WJuQxK;2ZgC`PV8I9}JSV}`mOpen$t0Jt0
z)`8eX?|89sc|>%(K@_M^$Y#>T?pY4#VKif*
zb!xG(coSR>H>cUMj`%#;K$ZxGr$dF_`g#RN^)5k1$MfKv2(hNI1Cm(cT#y`gqe!eD
zHeH~qex-2l#|DgBYFA1pNyk6QPO=Ouqfg+5ze!%MB$|=%l>xf4Xu`_Q7e|xkO0V2B
z`I+i)zDjO;xk@`*e1hanWg8+33sotPc(L5l+3Io83PPOJbH22{`Md6eKy6eH94G<@
z4(P6)jbFatqTyU@fCXvYyUes$(Idp2;TrB$Vkn~LFl#L=3!J*Wi^E>y8t!qfdHuvi%}~blp}FCwL{@_>9OFM0
z^iK_>TWSsO+=TASp%_E1#&?{W_kN6A7dvu#;>WFiy6q)=9Fwa@s9Q$3ZszZr*M|gp
zjCMnuo_)%WbvI!5^}&mW9UN$KP+k2{;q`aFdT+_v
z+rFQb=?N?Ko>`N$;P=G13rY?~_z^>Y-o+-TSAQx5n}@&FAVj4Agd-gJCF%aSZ2y0Z
zz4cRD;Tp9KE!yI=#T|-Eai_RLffjeS;8KcPfuafSPH_#vtx((@0tAW`FB%}ooAb$!
z@0>HUGqeAIooAkVt!u4&j9a*fib%blSaCVI;x+q>6=R~RB;BL6?ypF73L~)_)NGu2
z$t#wW9Vj~o-Ju*4S!&4+eYr*?852oL4$w8<-QQnca^pY7b)t~l2(4R_Z1W4xnLJvd
zi&Rnw?dv09c#vJ7l36tVKIUy;R;r&_cW|>De*Ak!#Y@U9zc>`YF4*pXT`?e!mE4;7
z^>knof%yFc=!~BT7^&81PYD&pq*s2nJ)wUb;A-9%l%xSjG)S6`ivGE5eToj~FuiUZ
z)c|>y6PJB6ZQBX22Vl2sf1E^bQ5#-9zgRy*{#=2OE}{pInPv@Z|Ma>2NPyjeoCUS|
z>ZQK@Q>NBJz9B
zftDUKT3U14D0jpdu0y&8Vd;BV5d{7#Sm_EuleVr_a6TN`W<$@|@3Coi4Dka2
zDG+W|@A#y@ZT9}nfMRN!wh)P}6eU17WXjU@HHKdB+Ul4wyn!OKi!&y0<&K@FS1vaI
zw_J+Yz{i9`9#g~NQk1Zs`7lWC>K{Vs6ro6P9pHm+5EUB}gqy`>_!ps0cP}VWDl4a#
zj?Y)pGZcs?%Mh0s)O_ZzLse())zKkP(p>XV3=}s$E1FT>oH&i?`r+dKtJTWuv8-9^
zx996|Mtj|6?_Y3Loa(4n08{1RoP;;=Yx*o4z&tXCs-3^u&MAOgp-3`ChwXw15`XFR
zWo7zMSLGv+K-K+w8QVj^Ixz|t%Mku>eh(&9zFXnTrCfLA(#M3uo{X}tuywu!Vd(xw
z;{A9mKlw$BV7&9%KH0Sh+bB>A8D`sF2lgC~?NJosLAZ77`L1|PS^I8!rzHEQOX9
ziP;%}9s@C?o)#_ba>gcBLBn8kFHwv#!eQ4Q`-nFb8_>9m6Ds{<-tz%W+Zq#e&C`+6
z!1z+a@;GKnu6oCG@p1!azJGguwEj?c^nMev6FD%hiT4xU&Nl9l`tw+e$~Ue%q=srl@62ie^L2(Uj-Z(&kfvz9p;S1y
zlZNxj)PiVFN4mIQHyvx_WE2lA2;P4qll+zg=_fAOO}vsNkLb8D`V*!6lb@)xB3b^M
zOR?1yb9i5p(i)$x&lUE%j8CpRtfnVbq+f8R29sxsy|Kx3fNuBLu+00~&M2>q=sY@!
zg>L_4Faio^0dFHow!Yon7NsQ;uX`y}SORyDcc#%GS4xUeU$EqL0)SMU7ijIOz4RCQa@Cps+Oi9v~VMVX2s}p_|
z%0-h@3!^vbLA_x{0pu@Xcu*y`=Y!6(KEc1poycTr!_~1)zajkds@K&GCT<&On!
zylPVL8#jhC?UNzZ_~K&$MDJ$m_agcw$Bqw%N3G$~-|+=u`BTn9(XrQj9(=ko
z@sk18gTN!<&f|(3*IGH78BVQSoq518%XsVVYPZTSJ*sL>oy)f(p6aeLHYe!~y#UFP
zQu&MIhWNcQ)voG_owxxZ#-NwGiJMUPCY0G&yYISj`&$=%>E}lZlFJTFrMB%q9FMn8
zEh6~%4uku^C;d11D@HWw;$I6#dUQJ?mrr$@UVTeDh+)L-Tg6kP3Di_K0$u7=m>}v
zOt54^F=338$l%P%5iM20*iG(3P5>Wf30+hq@k5#V%oy9P>pXP_H1+P?^A!
zpW}mA8@@5CoXn;@jPpc$TOaorU$;n!LPkaR!}S~X^zV{yMR;wMYOF2s;17}fS?;`*
zqzsw_R5s=LyTyfZ60sXYFCIGHqy1;QSR`bU+DwX!sIur~8yblBYHbhlEnh}E~}
zv+7lA{rXV-wWfNa_N!OCH-SSvPe({v)Ms-W$sZ&>p9gPc!=5OD;IJo}R`DJ$ZGBdB
zW7}VEkS$t5zt-O2I3=>mRy`~R$hMobF!o&-Phtrcb9I?+!rD$KAjqfbh
z+(@nLwGnGvj;oT~*PtL76^ewM^Mmc@!J(c9&5|*puXhu{{k#3HealBrZ%{mI(fn@u
zYdhrZ!uv8HUy#LSvX$V#Q*9*ZcqQ&NyOR~c;r8Qlh0(!
zcleZJ&K+Oao^La#AOLE5eC!M9VEK9UX&xuAmO@Gu<&{o0V(mKVl*7JcH%_B|L1c|<
zhy_i<1*&z#XCE2@6s60?p2**Wkb!@X7~yOtz0qx0`alXZhUtyV|2pUwCwq459}
z;NRl_HD||vqNyF)9#aLFSMG7Y-tH~tYboA>8ze&L{J;=1{WPJ{BDBDa0
z8-Oc4{4Yr`gq_|1K6UU{j%-sKwCm;&eJWslDvrc%vA&HTx6$gS9@1Ety;}Z+h%!Qt
zakRdqLO0%U>wF5?-Zo1z>lyk^VOlF@YMSSEw`YVBw=MihWk!{X#X)NIH_hskyBSB%
z?lpWsBC*p)khlVpsOtXW6OFaAC0qxB@$6Q|h0u6314(dV{NX#gC3uXu%DI1@<2j00
z@ipS&sksVHQFFSPY2Od3i!(E#sVJ(Iora}yT(gorz6q}r9&;ShPKtR+crNnaQVgNb
z^tY_Qd_5RV{MEzoWn#?JTF;38^LxyR>kftY+ZrsWt9?d-!A7Dj$(J%LNu4Hw{3dvJ
z-_{QuHnlWU{VWw(i|yK=;e#IF0|`y$nBvEp>tqq@Tvch-xNVl!-8hD&ztfq-EP!Qh
z%~tWQ{|sP3ZI&Iz^#C#L)sGICEOc7&1L>>M2P&^NsU5VBO$HUn+d&MSRfm;AtOg-i
zdz(E;L@W6(^-L8LBRF>O=r!ex!zS7Qg6*JBLdQZ>R=Rc#=!Ur!tc)|eSmb`{qpCao
zd5U-yErf>n3;t{$i!|zZt@Letx-dbnl&~eZ*sAV2=)Wu3CuRw5C1cM5nTETstx+c)30zsd(8;-E+DE-gYS9&jnTQb}N
z1YTidB$VZFKReK%EO3&#mL2y@oB3N8vRCgmRKH==?ntZX;{8%J5O8P~L4+lSkoq2*gUoTs3xOY>BxrE-hh8N07!
zv0jdo9x(3VyWagJiVfba-tVr#mvzX$;aLN~h>0pqTX}l?(K0L*Re{2QhJJu!#7@};
z#FlM83l4E||*kwNOW&4v_Ti&DwECEnyDw{D4|tkWurP>8pz0-I>B;X197Dsaos=vo
zSKpIQ+B!t5L1k0&yq;wmS>(#ANkX9-JE=G?7tytwuKFVaEm!GswM)1~6ry1H#!`UL?ahJfU>g6Z
zcTpwmWO{f~;)l;Vc60lYVHeZY@bn)cF)|AE=hLNflN_DvHORZrY-ABs_R2#-MCwMolTk?MG=_-p@6
zvq8uzW)Mv62%7lxc)d*k%nR=NUs6=hzyEm>6%4`aqysRfi^UWDQW*D$oX{FWcR!Oe
zjDSW_Sls(Zv|luKC;EY@?w2V4oH1YOS+~vLH38<)7O8h)+|~Vu
z_y7Al)3zdLd|#C=$1JIb%$leqpaF)|n=8nfa2o+oixF)Tnkujc>dg8o6m(o5kJ7u)
zLEncOe>+fi<}fZt4xoDfhmZdkL#VXzCl!TB%mk!%BD4dod72P%lk8;*SAC3qS0h50
z1i@uxa%QEA(ef8tS?QBum|<=Eh;uo7{^)PF*7FxdoKRO_QatLzRCWIk^G-IW;^j6*
z*>?RU)&Xjd?Ibl3!CmJ!c0d3aes{b~Xl9O0$vmZUm=Or`Px|m@kLD=Eox}?4TKXaD
zYes9kOT!3elWjSN?a_Yf2js`W5S9ASS&BxF99mkSp_=(-$crOl%(Y%3F#PcX)d7ZD
zj#qUZgOCbWZ#St2ml|bF44gaFFMb)ZVvGG`@_pX*@jDYLe$~C)1^*aw=P*F={6evj
zS9WAnw|&aK9kKJ1;_5HC
zb3%F9&P$92Q5s{hamjZblx%NWgN<*;TmNR!{bj>9yxeB1T
zA``B%+cKU~rXsc_yY_~%P8z`lAK_N(IuaVr6-SNi(@@8&op#;VAG{bg8yD_Ty2J-t
zPiqy66h4>jnW@4WX$k9H1-60SQ+e7ZCO^)zBiWU`H6z9p9{mCA3{BMxqP>Sq!tKt7rG
z^4xQn@{637Ep;t-$4`dQoAO*xT2J=~hidP+-Kt0%Nq+x?
zQ0D4H@!|wMjQ4W#xhMjnlm2?K|V=?UzO`xDW|oK;Zb|@cM#qpu|Bg&uu9&4dlLi
z(#R(7%s?GFmozkvc_bTfwX3j9bpzJ+G|}5_`B35fb)~3ybT4x9-sFMsK&l}
zi~w|4LVcan*|`wt4n4FprG&a_Z=};c%qfwK9I*z{s##b%^^B@a>rbwY;N{*;OD$7>
z@rYu0la%BZSf>Qch!?B_01erUe128xKp%~=066Eu^@L2TL#oNhm
zcKYpX*<0j!PhPw|G;Frgcg)#NlX|mi@nLx~GsIhTN$$&rB8`B@`0`<(-Q~KwVmtna
z`{6&}zgFggGb(xD*wY=85png5y#r;RynF|s?F^#1hnR?IdXfJ1B}Kwg&!3aJXH-h5
zO=vo^cyMc2wEK?s5_uER4>O&xid-^nv-pdnMFXAZjuYo=Fx9fP6mNq1qS2rB*kuUr
z($U)#BA1qzG*XYKu9!$1Ipk{M%-!z^BeS*_A`Ry(J>sNN1%&Jil2+LjNj{6tG=F|k
z6Mw-W+Bg0hg~ixNKBm6!)-Dsij(I??QSMLOVRk4oE}nYM68L%l2a%S^j)_8SwDaC)
ztFBdddD}-z+xYJj0Lqy0y*fnXtA6VEiI4S9@8sf-jhJ~~O8oNKBiAxeI^nqxT=@xB
z+7Tq@*NS(USodj;ll!SrgXho)HnV_GaLVG6F0RZE-pOE;M=aHoDWM={k(#xWs
z=V9n7(flefP%6R=HB)IXd${(N72+b?&
z5g$9!y?f$uS(hDTy(^e9jRU!zf_sc!@ZDd2(`1{pXH?3*Z^Rdc3$nPxv*M$^_|^u%B@ei82ttSrJ!Cw9U}|Q`YX8}BB=tWUI_Wq04ZZfWn5hBf8&V(#$x0i*5W|BcIWa!_BtT>bpRN{3V(+GiUxuBHuc
zsUYqd4~1*2)MG|dkjdK%ef!t5GMdB(j$?LaF(BDAW+m(w@i49k^&BcgOyKAr_Eu-Rg?TBm4#YArw>
zNFqs|E<1ZIDlXk!bTqCkB1;c<_F5Q7`wnR~rKkM6N=cgajQ@uf-_CVek@Buf`-(CU
zrGzRk`onC=pa?JaoqLZhV%-Pfzs)mhJmg1iuy#&$G_5cV=W`*(EbztL
z?^j(tFQknrO*dpl%rg$n0vG^ccaHjgs2X`K?53p)i)9a|~gOK~aFM{VQ
z>|WIPnfrVOKEk>4yD0*qKaLmLx4in=yV%3-QW67AoY0r@g9KX?n}nVWdff)jb~8{z
z6nr#T7h(mXksjf{JA@Q{@B?ukBn~EdhBVZ}+BSt<{#x&+!g3wKJ3Jj`0^NQVINJ_%
zpTDfgj|Ks_ta7MdyC<~=MWRlk2UwTL-QHx|5iq^MFh!m~!H-AJGJ90eOW&{edo9cP
zrYOhe^uY!%D!Jc!)IH)?`!}yWBp<)!po5$-koPs7{U*)*8n+iNjPNc;pg1Cu_TKpc
zU9f`~?%twbUobF;k$_L6;0zXpVBo2u_*ebQ&E`O}_ufKCD{$*Tw
z^0X#puFdg#Kg)6+mO2F?5s1cd$U3g_I+1z4+qp^GnLBA@KA2UsRP9CB(YwmtNmxJX
zGMfl>40jvf9GavF{v=M*aZ0cD^FCR4;ekyY`YqUY4s(5QR<{UnpW)?S7hhU2d+QOp;{fp)?bS$#}2TjFq?FzyVLZDK(
zeC;Oa?l^4~Hi{b+VUvoErL7CyCw{|>3un`80>~~ttMa55@Jq%eAvKx_21of&3MH`qW|#7F#qQdQ)gX?!G6Pv
zkLv;Ho*R%H0!NFY_pi@OazEPjE3o9M(+$M(nWT<~X_@(%HizSm^e*Dl;r;FSE173_
z3Ns3qPne2OiO9zL)FL~KL%g7er?}Mqio|V;TBMrC@iVL_0*@dR7y=HR#VDe3;Cntv
z*GSTUVyGx5gu`E+UITlptqT}2VPYb4JQTE#TO%a1foL|4SeaonRXNcfX-T1}xE%#$
zF%2CA(v_@6W>}d6NjTQWi@Y*kpqJb6CA>LxXM(-7Q#`#TY6k#4%kTV=eJqg+g3~OK
z#;;UZH$6heLUZtUwtL;jgfEcn@Fu8l_akc?knJ^=0PnZiUPmq&(;|LCg6hIY;}*G>
zZImBZtEiL#)$LsL@FCw!D~9pJ8lmrS+do@mq|qqf8STDg;@#wfS|lp^7o-T7C=;cR
zSqAV8hk2$PXt#k8_CMR=n&s00F^saSAWA^H2gvTo@Ng_iDbx`|bBXjs3@4na+N`sK
z>sn;>Vl2xt-=CW4)U#@!PNzFSll_kv*wn!H_}hT7sGn*E$LwGZUxIN`=BK}}j3sGJ
zBscFLP@zE%yo{^GCcbf6bs2OmY{76r@d>YnVjuJ_~K<0&Zo9^{Ze%>8=
zyO;Ecg>QOMSVLV78J9N2JWK>nc?Z60tZaP&{q~j8&NR?|@5jESn?L7#xoDXQSM8hY
zPmF5TY!;y~b1d>M^D)4M_0~y+z`7MjX&TNcxIWjPRn&3Q0;?nk&8j~H>E-lfOU%D8
zx%de7Bw@Mg5e>OVR5ExI(O*-jA9i!@hs8z2QRU?I9$-VS(ZNy+W6tIBHGt~=*3Muc}3^x4lkcatUkxU{;zf^RppqxwUOeT8`jx4Hd!bm;+tD6e{RZc
zM%G3tiejxFB)-16-2>Gl?I~H1iNNt{!}xHQBfVzP+j=Ek2Nr&0LkN-h(xzRB)1c~@
zpfS4q7@w40(UA9Ah?gsw0bLJz@D&CkYKqj)9_Jx=UfCBRgA&sdC@0(Z{ZiX|kB4=d)r
zPOl+9Ghb|uK59)8E$i@7Y>`8
zRrILk6eUhtFj~cx?s*stN|xArc|QTnsR{b~?JmF}iSBN%tCESCnBUC*%1=$qtLwtW
zpUHon8ty&oJ!z%)uH!~cpf@3PSClW6pvRmqk2qU#_Hv2-qhiT^FtyzOVv)yg&)AFK
zB<&mQ?4JKST#DN)o{5J6mkxJ=7)=v4URh}4?Lg*czvpmdA}pLBQISq<)D|s|b1{a$
zUfPQmP!XJ>rN4UzUbU`qk|>${OISqb_ze_qF3!@;I9I=?PxaGw)<;E>muY|{js|Vs4_E%d1XlZ7JQm1n~
z6<9Hah3Gve%_muyUY5!`@2w5`qq;MdBJFXgBjXtW=4joX;^;Lo$nVJM;;pxbnSQg8
z*DOCj^hb`GN092);8J`;Iw3rnZb_;>N`fL8`B@iDTQnj_Yg@z&pmo)GmJ}L&Y=Q#%SMM()0
zRv)J{*Sz^>86*15|FrBl$g-l-NCgf2?0G_u&1TRd(p^JnVhvbMKLw4
zjqCxxd?T2S8SaV8lnGto#&mV?EMv^1s@#MoV@l5Xy-z>%NU?J_tWj$F;(_z#NT^_p
zFNq5&>L^WDpQNkTElV}GXHqDGGvn2Lu@2I*sib$f#Q&1K8D9O9-o3#o0OIcWz#g%Z
zFkhtEWLisGM2s5A)|QbCTPweFt}R(HHhg2-l+R*+ZOccKtkA|)6JLV@QC|QKBVi*e
zdI@GGcD`hWNMZ-ZzlYoLROHwh1}Bn}=q{q}N41XJOopn|4KbGok5VSt3uih@vh!$0ypvC;*zebiYj8jaH*Dn5y4b
zse$Av|DL$uCGSKpnBx2VFI9Gz8X;K)3umaO+YbN6`Xvpz#dsUCk1Bnct0>wS{0!eL
z!LoR^?@RECZ&0dOsi_f-_a3)y>#WL)=pCsWP1uK2kx*oKX4dc9P`DT1wTsWZzsvu*1{F6f}R6lot^NBIaP$gqlbtiG)b4j!}Q~8So|$
zKEGOTpqW-Th@q3uz(Udba9uCf+gTcsf~C1sLSLaU1^QbqlK$>PdqK)eksf+Gg-J43
z84&P!l`c>NZTh=P(uOoEU1`m;x*J}DXePBuhy#|Y_
zv}Zmrcj_vOelb&AC#78r!tmp^r}k$_bv+BlI)`6&L^GDs{~C{FgSA1
z#@oEsvmJPy!{8#yLs;H);e6D(KjWj*@##guo??b7_ZNoSd;5uSkd=;CrbDBSvTPY!
zBI;TK&gMt@Ut?o@-NnyN^@HrJ
z;*2w48zaPhg*Y$28?`iId-$tZ_YktHD=ZuvEX80VwR!PCHM>g&UXpmLZL(eQhVhRm
z&B*b}^l$~BXb-|nAO>8q^vGVm5i95(RETV0)JadQ3U0hi{B6Jw9a$8RUVPWI@8ZFz
z?a4fX7hVfinvoMAhhQaPEi~t@zVmAP+g%2Y4zYf}@%3QR)|oLVYeJaiH%`orTDGgE
zna_!+gbm5-`hqVKSB$JJ^y4k(bw3=E(h%=#BG#n5us;?DddeZw@hd8KVK9;pU!
zrIV#D-p=cvmY%k1tx`LyMo`72kjC&OK<(BfA48x=u#jzHL2L1oi1ed`BOx)!6=%q|
z%+Fh4(33?B{n@~4kHWiO3$=;a{#A!0!^e>oupZpV-eXhM~0E+v|L7xKSuG-jOJ)~h?VlwJaCZFVd
zFZ~u2iziBcn>^7!6U9px3eIB!hs5uP^sNP?IvH3QbaM|LCc@7syGr}2r-QLDXhC(Qbup>%#oix=}Wa=MY6WR|%r
zc7-~NMq$FMJ2);v(7IbEZFry~C~?_h1nX!=tY9wuPU;#V_pkoy`xjF`jrZ|U;N*S(
z#>qnoCP`Uk28Fwr`4&(!{mnW;H@_M5;YA@xW*p^smvg4LRIQ%aYUKP!|
zUk%qYYte>(Fx#d)KQ1QxIrMaz=V__te$)Jlr0Mjl1cooni%b2z+fG<`=r{$hNrOpG
z-}Rl?vE~#Cp$fAk!h38HW@vMt18DTGpL4kysuj+L6Jy2T+V994z~yT3RyH;^T=%l8
zf(+Ldy_-hZAGOn?9ev$LbnEy;;8~(y%sO!aeN8jst1Aaq#eYD1#JS(3a)+OY0$6oQ
zU7NF9c+MIJnt?bJthxc=1i!UkB;ix$_Y9rixh{_~8@u(}u(vv^Cz%Xku3hFjUo`~n
z5x!WG=;1ok-5WJyZ6N#ir08AgB+?wH-9;)7$2Jk);ib+H$Pvj#H5|!o!J$MSZViDr
zG0B>l)|+rGuiEg*t$C-lp}NJEaFp1bl2mySv0F>&c`s|ne;b1l-84b2x3qB|<#kFGoMepuNAK|2^oVq_joRP~m+ZRM^
zZbXQgckxEgBK#B^$$k}2lPZCfGn*euSB5}&Iw+gl)~S>esxDtJ!tA&tzpVd
z*UPc#5&Ys=p)Be>Mo6UA+iv~c^rKI{_mJAR8(wb{a0kWViJ=nz6nv?w!ga%7_4w9&
zJn#~$`yJ~2KBG3KGJ+=Rf9SmV*juQm_bsZy;~nsH!9jYD9}iFiK2nxdB{k`-J{_39{d_J<4DwcWO_hJDR3QC
zi0!k#t&;xdrVW1RsnM_9Z_<-wb@^uJ_xFT%C$6pQ-IxhyAJds>B~Tss01Vft$))mj
znM7X<8wo&q#X3qsXy%j};u7G1Y3~ZVtwn2r{(_%B8YvqN_tiF%D@@0H_Jyj?08>0A
zgq;aZ;;k&xcJ<=BGp^VB(i-~~F{26(@rl8>ab*bF2tqgNA
z{xr*oQ5en7#TGmhL@6ODg@Z_Q?8o8b8TObqqFVW#G=7j1Fb@;@pMcTmf3-RxVt#prR{Jfx3d)AOkZ<{|JvSF)X!hCet%??J3b
z#bv}m^V2bZyt-RI`fu5KMKdtw+~;#(3-2-RYEd)OFHTQp@iF~9TRwx92tA|wvxLKP
zqT*)^SqdF=vX2-EFDp93E^vqQffwwpopk4+5~%5cLy)uO7V=*m;4#RnrO@|1fuW*`
zUV+;95{UTAE@;=;{;r-s|2k&;pJ6rQE;U6sGu!tyV~m|Yth@kc$#;Kqq1?_MqNI)B
zwzUf7h2XhQ#VK>a_oTTu6W047t_;WU@BGVCkCNm;KSiL@O^9w$20-1tXLH^(?}@~0
zQ?2FW_@&x(+L#^NX2hh|haAqU3f-THzBPQL`hp4el~!
z0{!U8W&pwz|B1OiqPqB-d25P_F#|o1LyCw)T`b2cTk6YY2@|E}Uyb@sC)vI4Wy&ez
zd~tI67LW0hd^=Rs6lXq3>o|
zS2`5_sm>L8(TVw5meExvzJru}5W`3r|0%kB(~TbkwTvtq_ZJi;W6knWE+hn0{CDYuoAV`hP;%LYvxdJ8{qx9?#Lb8yU)@FUKn~iy#s%?Tb#h*1XXo&@h^Qr)FfB2YK~spqn7I}
zxJzTDCZuR@j`a
zb3Uo{<8VU#tR~U@uV!-l-?lNzd(u7fEnyO>mKbn`+|RV;&NoaPXEhT6m&xE5u`~(Yav`sv(69%CFa|cp?ndUHDcMiZx_1%_S0-7IsjLJNC}$o-wUUfm#;E
z1!v~XXLxATsOO1yJrVyuB=didCOoNb&1`>%U&K4WH-e8ZRrWG8FSA_#a=)j!*Yb$0R>J*+uKg|x!2&6VeBm{sxFA2|m3DXJG1*a9Bi)E%LNY1h}
z-3&Yqv7?N6+weBon*dH8uJkV28GUuyPdx${Y?rR?t*a+u>hq`x^|BZb0R|FV8Lj
zjHWD{4o}^Atn1-fa&4c@gB0Lrj6of#*OCoos$f3@xd+i5*e*0;VI@iw^DnqOE76xcK_5+9Rh@Ctb9k
z@9B5)c4b?ts3-RoeVbAJq`$MelH9KmdfydtF=944+5W6Dbk4kO&o(O|sg*X^HFz9%
z6nB&yVYop39JWQJ*Q@&^b5kNU1qi}7`z3eR(75niLJn+^e=6~yg-LGT9MrMY1@|XV
z)cW_$C)P|{vateBhPprr
zWB$Qh|2*>FE!sM&
z7CYKWud`f;x|$cWKT6OyxmJ%H9j`p8c){V;q
z+wGiyk;IJJUJQHhFqV_RZStbF#T}Vb0*7ALU!i3)X_4C_k)nPbZcHNKp@B|XNjj*f8$QRy5xlY?ZE<*I1^m12%H!47^Uti8%gcuFjmc4tE
z4d!Mfx}LdOU!|`62wzz{2C)brTkTom!c=tY9(5eO>?GYfgMOXd{O~{^C1~JnZ(6Io
z$NS?7g=CC$$tXuDt!xwLr^Rn)(l^noM~QdUity*NGpiHR(;cxr@ly~d++S_9_t+sD
z(-UEtDsjr3e)0XRn0wI`f|s?*f988V+xR7GI4g(&C($mQ`M^wYU2KtI%8Gm~>7Rdk
zmOVEe`EJdX7Lhy3DV^2#MRt%@C#%
zH&Uk?&Pi4qCd^X$E2@V!uj?gHq3Emtll)odPph7(KTpBQ0k{XY9^pkl
z%N@$do3MZ2=G$TqjNEKx278y63`kP#F|8l;R@%~xsgx{u>%d={=7I381D%Pj>alUO
zV!G5|pk(~A6X))kPxc%zZ$Dq4WXRGKA7fJ3#*nM@FI-{C@C41?b3$QBhOO^M
zdG)Shv%1(1ZCDl0YfDaM+DpYZyY+^hO@i&w6Z}E_NrJ=&3Sz=kY0%K~aN`?V^pK#t
za45_Cm3yofcWVV%Wo9g^F0&_7V%qMvq*Xa?VXc$eM`Q+3
z89YJTHpT|aA#txPu2T6QE{q%Tj%}P@<`x^T!{Rwv8(Qn{%Orp52mT~T`wi*pUTcwA
zNb~5IZ9*)LVzKIfO8Ut2Q!E4fb-X5qrGTQcZG*9Tj!>>T8L1b2T=$_VzP
zs)RuNQ|I^8T$-$$E>tZjheEY^%dH7$Sng`>%N{V;Tv6;8CzV*1A)-$ZfUHVIvf}#L
z)VmM*N>g?l$)`yLnvW?f`CUIZ*L4^$&FFx3`|U(s9X!KM
zZr7jWs(i3Qz7PTsWzvit@7u3~xAa?{Ip!W^H09MPU%I|-_3ls3m4DYE*=xDp2m31)Ysu_4TSNssaT~-54TXxnjX{?<$@|6-7hk_$nx96Qc&ocp&yj`X%0QquC3mVL0S!*v1m-*
z7ZdZ9Px;T`0Y7{x8o(${ZJFT769-z$AL|;$%i&Kba?yMbSbNsWT;0B;tbgog9aJTzr*j4PftX
z=PBM_|KCGBkLYzRabA9XmqIfkP=&f=F&(83=H0J7E{gYJ)(1<<|*`;jrT^yO6
z-}LAAwIqIB4M|60qJf^MGw7w(JH~F5ZFSi2fKMmvLchpICMrDyS-tbLo=F0Gq57&V&%LtQL2=~J2pB(p78-=fT`v)
zF>RG~JQ0$$1f-+!gi2%Pnu^wG=I(bNa&#%QnU3EoY2vO-Vhlkm8>HV;2~cqC9$5R-
zTLYAv=ax*mlbOGXpTPU};l<0(3|C=nnZB~k=V{aQB%bN0l>jz%5*{HUP65lhEam9I
z#_RNGIQl}}|6%N{zoL5Iux*eQP(taDZjhAj?oc|UbLfy5kdTn>l929Z=s^ML?jEFj
zqz0HF-ubTe{`x%6=U>=sX7BsHuJbq#zP5lplbHZf!VVEu)d-u>xAT5UiOb_3Ss(`1n27uvUp2PIWScS`nJ@rp6%6iqMcc<`W{Z5rM27$g4dWgA=+RYp
z+b`JT!ti$Sxa(Ql4E~+*kDlQNb-&nqyA{{MByQ$s%1w$?sj!*h{qC55;w-IK5Y4Pe
zh4&n!cXSwcoObV}BERZsAz+@Jzh?PSL3TeqGhr@XYDe4*&027)DYGGr%B>g>;==%>k##{1EtHLqvmeN*xKt`Q2#WM
zqZj8CsC(CzxV1Ttkf&6^f?17kHVCgy=lcA(TGWWIF&Nr`_+YU(MzdDpxO+)nIU7N<
zFbAr{PFFoAHfVosC0(IV)TL9$#nwJoHkm$#k_I(SB~g
zBB|xC^=e&hS&nmfJi|d+zi_IHnVm9apk!S6gD`_d(v;1b3>s=5!RBcag-^lAWF)@8
z;RX(Qoes#%*{o#JHlTi1zDj5yJ(*KtRTi5qH9k&KNQ)(?HDu97#t|YyxWDPg#`R>%
zv_fdzTk*Q>X&x@}i6USGBk=^+&-0z!u*20VgRLrCUNnE1Ag^8gh2*9J(kB>uU_aB?
zR@|@v-t>{
zZic8k7rG+{|JClbMD6MtU;WZ=_8khppIx-0Ds8GciGnNSmqMn=H*JG*0aVr75^lA}J<0ux3_y2jcUf?|(_C*}L(XE;
zWi-^t-F~qx>@)jh`Tjc(fybQq(0ezV`n4y*kVTyE_;yxOY47LZ%_SX#rV72BL6awU
zddZXT$7ht)28#k@ao_J?;2=n9$Fsw#_x#s|7h^)XrTakz|rBathb2Nb$rX6j?gcpvVv&i6o2j<{6S?<5Fu=FNU3tBEc?VymWhd!_M`yg2lP|mJ(AwgjCrevwB1AWKi;J*OZJ{+K7Dq`ZcmBfgIKY_wn-ne9-4~1l
zgWa|%Vb=m@`2)|-5VB0JzfErKrzV+tN?A{3SH0CX`J<)0xB57FAA;SwhMTXAf4S9Fj;!jqv$$uIy8pu
zG)hL_d%i!kN%z&a{{bs*-niaiWvtpZU33JVT&+>>N%%`);f**6XS>dB^h=SRW`FTB
z3>kk1A>FMfPKk^!vBRCovac&slVBPZUHl7tA#EHa#w6$G6M3B+C+p{ZA|Hpi5ly-J
z?lPnNE#191fKA6kBfs4uYPofZk5DnlD3YOoOf0S>hre%<9P4)~fU)@24ctJKEN(qF
z)C86D{~M@i#otVD=HD3E62!8H(tKVsJeiW%gEzt8Q$!rDNG^NW;dz>itc%UnNviuE^rPVyIm
zoug~YJ)kzk^>vqAW!XA$evZU-AyEtReHpK@tr|p9;$75_Q^H-XWua>j7DrR<`AzE+
zkd$wZ#(beOe?#Y{myujmaisp&8Naq9!=8c5Lu3RR4hAB_n9=h36$l~VRAOxKbH?xy
zzpJ-6q$4nuis+6tYL#6A>{N6qk5-TTwGy^}HDZ7fufzyy$m2+=W
zj;$l7X|Y$^m-SDyiVVWUu-ehRKH--;zVo8DwaF1i#U^g@tQ7*~4?dDb8swv!gH9c;vJZ@M!!>SdTxJe~F=tw5Fc2A0LXZXwB_XCod?oBhoNF@p_Kt76
zS-gI=+?RH&)|O?2=khc_G5mbuohx-ilZmZRe|tupKQ>t~n*W`{
zGW?cK_Mg>Hc#iz`JBjC|f&WIK)l&vY!ybdec%QRCqk%kA#dnQ+=X{#$N2eF=9zbr^
zgwygf76KyCGkIkRS$SRn4-3{$`yy&Voq23F*cQBBfC(ZgAWQ!H=ofAtid&UxLXIY|
zEwd>5l>BZb?Gb@f?|lDMDoi=_^yP^?`I`LSSt0B%@*S2L;T6XFaLcxK{}-rXu2*2-
zuhd~rn5VGMwqi|@dbXC9mmw;tsw$~waXeGaAhm=k8#x}IoTkovrQGKWazHbV++S|f
zn|EMwW0o{AGb1*m{8+}Mir^~-+Jyj*dxyRoY=5p0GNaAH?u%d7sjho!J(^)+bb88`
z&Ujw>k;*hgWW?u@4j2>hiRbZiYCilBC`l6d-p{I@ntvRr}eSk4E%F)a@)S)$;4qU^|?0~B_I6T*Z)O1?ki@L$`h_Z
znlCwa8!{{uYE5rWoGULROuMl?j);6(LiilA!ZJ${_k#weo9wU2e!A
zvq0APJyNjO4CYBcmxuoz0~fj8LrT1h*2IN}mW{H2amQDzJc9L|T3^eXKCF)x^ytI-
zWs~kqB;Q}3A%2mdn!6MrAB38de79m6SKQ7QWHyn~bx3E(582d$gb0z!#|De~FP)Os
zU8cX-LPb!OQJ7rP8UU3k<~l>h@q22(B~_H~U-U8C-Pi05ba|3pMk*D8_9QcHUod?M
zp`%tnCom9O7@pkzZ7YC>@ZBe&HpV#Ps`7G7PL^@d$zC&_y^IxA2>u|@wdjlDfPU6D
z3ncO(+aSS7f0M5kiw-GrEJxp6e^8sfPvPI;GEmT~lDD=SJo(*DjlqVBlvq@_ak)w2
z9f=yO_>P&qO!^I62At7lt`4dSx%!h+#CN7@wL)GZL+uT=Uv*L2zj`X}qYZpWYctqg
zqZEdG^+d}=i^y?<5WxWbV7Fx$zR9YLcco-o8i+d$HmN~-z_>16ZHv)?CL8HEC-XcjV@;LCheaEmjhW={3@h8y?&1I`Rj`&JK_^YliC~|nc->;*dKH`
zvhx$=5Bc)kUd`&8MB15XWMpt@hNwgg`4fi@~-E$>krD^Z(<;@+8`mXQzd#|
zJ<&w$BLDfGgoOWP(`FNNPH>q<_b;HJ7-#c72n4)V56P^zjvMvmm8f*u~e
zGoZB~W4D)F-2YRWA`t;IX-71#Sb5{)Rc7CsGCTK+htlLbm!WTaF2+qBsy(@BTX;~{
z4#CpYXl)$dFoE_w&F^*=gMme13F{1Cbr?D=`+1(K|q*#SJ}Pe+sNb-@(a1pBT%sJ?Vj-{hgL
z3Y})Ml(dAm;ldujAC%jh7St)PVK`O~IFwOvWv3|WXx6T&qI%*h)dRl4V{}#XV&kVd
zy+$T|_Elfh`Bu*)JKI=!OX;_t{no)5TtEF-%)gYwan!ylQ$!p7bu7d5&R{}QNq$z
z)kySzOpMlAe1S=K;`>B>SJx}4bm;a}r19_l3aAQqt<{pWW1uOi(Bb^+2HPRzHe2aq
z&nGec-K>tO2ZT65rjPLKmVejzU+;52dKye2`vt9_et}zC9TaP0F-BmcM!3>Y?Ld5jnRcu)nG607Q>YS`};icf^Luisvdn
z;2DGokqjN>AbNGmx~%&Q9$>hzZ*VzA`Tvt!0iStpfK%!6k7uDe`0q;kOSf9t31O@?1OX}jHS
zIGH`?$>wf?p1!JrMJ@93n>h(>v(dw)$Jtde0%82Tu-XicinFm&Mc`!7RKEL0wPbx;
z*i>89)z!Cu-j24o$hhN@DT5yI*AvC|^N2RQpn&S+mU>D{$Z_}Z#F4pgpAQ37_Qgf(
zXuE5K#3v%S;sWmfHucn4+I+eUyqP;Fcha($?eCZdHMa#{616MU)iKB{{SgJLfAp+-
z`t#`ej$;2K)I~}gAx?Q4-N}>Gf!L(FMN}vSlm3849Y|!RtFicdXRxiTnC8ECY<{Pf
z2%em9ZUrzAjB%PN|^`HvPca621H`fL^wH4z3k#zMoDi0rI!q0wfplMxS1KiE(fbVdD>PFXs`^osh^
zOge8zc@X~&MxM7e)G7RlR6er{(3NVZ1h-Ny
zV056PJks0s_WazO>|>HQtRCU9q5a}tpH!~+K_=i|+7x^{I0i%AJSHW!jHAtAN)Zw~
z5@E(L-b|2q;NxJvQ&KS0+7EM7tt7%!lR>tSZ@3C4#Py-O7$7n4Q!Ql#|11o>41ZbZ
z{n_UdSEa_@27c$YmfD0C@tbPd%Z0;guX}wPWVij7w7B^NPe+N4_*~18T<;XVSxidD
zRTng7h`6A?&QMRf&8}wc;-A$F%evTlX;eD2ps%W!qm=I5VTn_$L-y}_E!EPy{d-QA
zVbnPDMb(Rkll&LeFQJc{Ym!=9Jakf!F|U2+Pf?*+EWyb?!Rh*Zf?*Z6$p@XCxPk+>
z4Udwy75iQc!IREGQFjv1D|H+HVKUd?pubVw4$Xc7!B(5Eix9-sy~FNT9u;znTB?;^
z*a}2mGLslg_Pakg$c>F<(2MX}&X!@gu(PnoCN#FtM!Hh!G@_U6mZE~st$>J@R_q&$
z4NG?hJbr_)E6_rt=dS1)PI{JlMFgbJRHd_0@ypY-{=a}Xrb2yVr3xUS;5NmC4Auc%
zH^8Cdyz&Ad0Xs7jC!4%&{klk%PxGV7U`JNBm4MT(Yn`!ICE?HdKx$}#8%DtD`R7GkK#e0eOpf+jeONy+jkehDyEvO$f!GDwVM%hvXB*Ya
ztg?VNmVAu^LM;(a3-t$tSwEi4EARGYcou
zM{(=n&?G#p5zWbuH|)^{(T;H1Z3=s;F!*y8pRMeRct*vT4{?Y&1|LIgB-b8PtxSnt
z*u(CRFSu7`Tf)l_~iDJhY~bnl|R
zdw@qSEq$Q+7J3%;zf<-#7pR=nC6h(FGYmwHE=%_!Q?;c!LMW!tcul(mf3mJ87ygAU
zPL@42qfiw?k`6yGx0zLoCQYy4J6`o3Qc|j7;E(WK-HYb_QSc{8n*2*<(E@
zue07AmMNfT*IF09Sb;7qG^NpOgEMt=!cOGSD%IsT@mgsh75I&XcM@!II*nL~SA$?W
z*|dR4o#E<~X}>d4L&zUZAr_AF4z7;AD5LqUT7)~qHJUzW4avtqDHMChP01@;!at>B
z1WlQN1Um5#c#3S77L|xyeeQ!W27Bb@4_V~7+E9w0dGa=M?Clze0a?tvhFONx`emZ4
zMwFi<0My=T$2(b6@u=(OaE(ll%2o+H#FHc7o9y+)OVBD!Xx2@)@
zeV#PMS(;5>KLxIQ1v)Qs`Xun9wV9qUHxeGNm&vzN$C*@0_8c2Ch!M9&Pnuk97@bWa
znbEJ>#X1mAnaLYI^fZdfY&A@}@Vm$q{&t()9VDj?drL~-5pR`81mpT&-!j_*t^iTe
z`l=oG<$LS?M^{HVm3=Q-01$g@Qv+fPEj%H)d5p@K@o)xxoindE@O^SH)IQB0Wi;B*
zPxWt2RApsu-QKUll*fQ(0lIp`L{s*Tgq^6sRd4r@O*tRx985Pdt}aG
z%o^0PG5?3mVzvSp`2q<3Z=~MTcOHqVFFeo1yN8J6czbd^`7yYATQ>{Nnmt{?GGBo9
za!QsNxf-%b08LIP<3S%*IJXSeRh9U>k>!vvAk~X&GYnHwA
z63hJfc9HLweTutR8uGYGR9Cz42)IV8SXVwVMS7Aad*BRnV(6PD_Bqi&=D$1&hi#$j
zAJ_UGyBHc8Ze+rHrTp`8g1piWe6o$Bl7hu%jFc_nBMe_U=PC^C`*GbHJ%TmluI945
z4+EqhtC`8IgdIgGP~&qaJ2BJ20obT99g1=j@#21Cak@k=0flE_XYepU2-1xrjd&}M
zJW~?DJ4wW!(?O%6ZX!SOcSZ4Vc0@Z8%k>9$8_(-fD_SZVAu1AQ;miLvS1mK-7VO^$
zjsy14bPJY5A(gj|Ve&^@=|8d{+MDLc1`(Z0j*9gpVQU80UeJ|N#J|MuS$EQJtEOzy#acA4
zJQsuArklhcuER6Rih*M9vS5j{Du1ay1u$9>9MI(DIbkibj+Yj04*ydA>|!D-J7LiN
zRZXTybwFJMI=`>QXJ!%An39qDh|f#lchb<1`FGk%V+mt8B^TjPbTLO>X=G6z>ozD_
z{uoEQcEZOYv6vnw*|#K^aq!pd<{j~Q;N~d)(^MVrL_Xh?T}m6;<9FES}JSF60&|fsByobO?~^f2fUCa@onl>vhj9o
zXC?EEp+ck
z{b`w)4p}yomN(oD%M)koYt=ZW#^z`QO<@hgfL8Sq+RYV=p7L{FZXOJOjCI3`&}
zs5UCV)n}9JaFQ2tPSoiCbO;sf8y|^P&-4B}yS~IUC%Fbs!U65#8$VvxysdV-9ht;U
zX;E>e)%l)a?$3a->6I@5d6nJ>3EtzFgv
z@ZEPs<=&(vH@R%?E-eNo>`AUn?tpzP`KKmosBdi|A2<!I(GjQMWz0ncT$smPS_Db%LvO
zJ~pTXzmzDmGVrZq^Kws0&O+t_a5+Qw_Q1uZ8!$4F{gGut+PrS(XWI7Qm*TjJQRm<2guwf62(7O}P-SBqDc6t=yU?*v%;zknJiY+TZ3s)^e#ZNO|-4
zvv*7Su>@AyeG?mf`qX&i!xMNMVs`fWZsoTh4{R7FGBa)7HAQoLOhN%rtlR7NRNEl2
zcP}~n$qmSS22q$MCgnXEpCX_`@35yrdrAM(^^-Qtf*uDh$vZ()BE=B0;*h6#$vBld
zQ}u=EJb>u>BSd<#NvTjQW@7JyXY`N!gR@@|6#lEUMNq6Q0J-sDOe!2N`F9?bJzf4x
z34}=Cz~3P{@7gx~{)z60w`lwLpV)h)$oySH;9&Uq6+zI)p1ID5AnATTy%V{++3;P8
z6&EvCW()`tRihVA1~5Y@*bNZ*8GxX~BosmfFV(z)g0
zc!n$>!&&o11>wh07^S1QGi<*_ap@DT)(9;A3|%<1{^oNUC#QJkGU{F%yESwYsP<_A
zAE|eJONXM&73M{|l#{BBVANTA{T}Vd?^2@du`~SpyExB4g^>@1>(ax>{2!4eM97r;
zfBl9T6{!Oe=vCHoPd_!&C@(fsF064T$GYQ~erqcyq(_Tb{IGmt}s$yK3mNGdr=6`dD%|B&r`mf}`!r6L|-Sty}f|_^J`*&smcKHHn^AChDsUBQ|?2{u#O_PP$^0
zMAnJQKPsT4FEjToTHKBeYf5%fPJ-CWXan8=^q~^)jWlcoC|p-|^22+6-71|@wGm;@
z$b{EUP3;zj?;tT1ZEB0=avL;T5Rp
zSkLt>F9WHh!L!iWW7hn&yP#z&Q8U_6Nb0QEdbK&H^R$$C(1ap;Rx@vT(M@cdfB2@>
zL~cA_tF1Wxm+O8w!mO2EVy@!gowrk4^>oK0&xT1Cq`H@&>1vAW7Rn^`?~49EeWvK{
zS_ulw0yMrzL8KL%@394Qj1t3~W<_g%_TU
zb669R!(Rkyj9`1M1?RURyA879P!qCz-g@;La@Kp4beGdDG4G1INt(dm0OWsu!9(SYplDCAe_-k`$%H%Ob)5kdFuX^?(XfDW#()qwdJOrIHp&`n61Ct%ZZoFCKN}3gEv1wFi3!V3@#k+7h!ynNl%r9J@z
zb9tZG4W?3l+-O_tDcQ*E1X`M`Wsoj6nC@D|OtQCWLAk$ctRz#N@hA}FdiSD1V^KSh
zk=+~2uYug(0?jRAZl(@PbbCV4khHJpZx1$;ZeZ9$GyIIhe6CvdR;;(wW=
zd)Q{zBMke(=qX|MwTQH){pE4gQ6TwAi_IG)NwO=pqX+|A5Q-L&hAb)wMNbg({*-Rf_T2a}LRycNYIS9(W3^@R~lW$8W%(KcD_W{M6?hj_7GCCxh#
zh}sq7{(T_%BLn~Um}AZ-{bm*an;9aE)szCJL?)?QT#*dtQ1ivVOCpV>@x0FSTNj
z3^mZ&E((7^0O1|gOUVl|Ib2%G@1VY2MgN?$Gz(fNMb|?*2dH^gEQ$9YT->UXE#hBI
z2!px3=#r4usLY#}1X;kQi#D~#E|MN60L9wR8IgQGn}A$S9Iv-|S*zbz%QQByIz*ij
zr=~Ajs7dk-O&;H%A{)>(Ix~}VF#T|y--RW`-prQD&&NQDhcwb!dULr_R`?A~PJA6r
zWsDfy#k$tCp6K87Vy@g<2YrS%Bd&g}%`0B@K+A)egc1`*ngf^sjDwnc`nv{)<_S?P
zpKRdlc7J=vvwV+SE~h)=Hb@hmjJf@jwD@Rm`U-zTF1|R1*3r(2JZ?vym!}Jhw@*~|
zG}ERlR#O|;u3J=AlbRud5Su)0$vvDoT;t;WrN;|>;%;YX+B>-6euh8U2zCY@ztX>fuy@ue6m<2eAMwmSt85Wo
zd)=}CZKL0UYgtxk$qJx49D~=iA<=DquuM`3y5>5@nIvZP=z{Obhpkn57K*ds%qePH
z7u5`6oT~mnJk$uh70X4xB*7Fl%xaO8KVb|1X#A~%XaVn+pgwwiEyN!PA|kGr1^m#-
z<1nB^qMB<>pNUq$M-gM+s6IrU$Z{gi^-ZV!e?b~P)ykP7H8zPbel!#&6xFX~0O!l&5D8sx
zTPmTjkF&hvUnBhM_!Ga<#J8@Lr-{0DT>BM%y{|w&PYFdr`|}lD4lt${AbDjrOEsLz
zXC-CFtMjnZ`t>
z_t}m|HVOAlO$gA2N3z1@A=Yd8z}Ko$;LI
zt}F?Z{-7@itXaxkgEf!0Hbb})e<$W8t8bi1oh7wCJgS+uBzQdAPb~GaKx2o1yYfM0
zbg%UvmB2NW2URjqTsh`^xNSv6`4#*{^rzo@lI2x)_m)($aFUMN6B!FaHfRNq?@y3R%MXE%Dbb?(!hO
zU3d#}!zf;|4wM}p7B(!wM<-y4Wj(T|x8;m}rw2^Cj)M!la)K>)%aM9v0$(3_R)EWqKi@y7-5_-k4ZBA=tvh@sda*nQS7~Klei^XdvznyDx|St
z$9#S|X&|``o)TsX$Q#XN+=N)9LOr_s}N9|QY7)D%}l@mQB=(Hd5gQ}TX9x=%m
z!$aC^wa&^xSHSj5E#JwN8n07VtVHc*RbZI;L`J0+xP{hNHfQ!^2LyaK7xWO=R%It%sNXb@1zgqnaSg~
z1UkySZ23<@MlOmaP90-yPmF~`Li_gxo8(K!!u=gTr*oEMRU7QE#dwA$b0YY7rY_uBQU&kYk?pr1oB8m!&Y7L#KUpxb$IDq#q#g!KTXGiJ?>T+50yqH?IFGU2r@GO
zbNq7m4F`ph^eeR`dzAM&V!SDm(=m8`d}Jzo3?dVD+s(e$E?%n=_nRaES?eQzc_Dao
zX`Bw6+Pd+e6(sIJV-9Lkv#a_>^4zH8nm+sQCh^obApscjnn7^a9G-P>O?EH6N#-->
za)32672VIVt%#@^D1-5YQiX1t_f*3Nm4g{pRz`V$Fjl9r2*&x$sge?F%oA`23|Q@T
z)vSG3>|k^mKT33e(Ost)TeqXkOnxkyzE`Sw-%|Cxw6UCR6HuZJWXilVq9t!SK5utb
zJgk8CwDzYc(eyCxsKw0KLt*tbSm*+>Ev+vNQmzUu2xSFNXam`zF1rlp;-p}^ih66;
zc_zSk0!-_`S}Us+jpV{8S=GbbVxS*=>+hc6s-sV0e0y9f<0sma3KBl~GHu7*1Dsli4Y)T}5$wxa#|l
z%&fdKQrx#r&0_e=0a3dpj9hg
zpx0236!6C4aMf{_c|eVYQZ#NPIHUST}l$l{4XNS%Jx=P<}}
zYISI>*SGgL=qb;puP7>Ee*lpd(OW^
zdnWsj-H8^t=s=#SB{-}&disn2lqgxw$g4$V`Xwyn)QlpRJaIq%wjY
ziC>4|-j{%Dd<+$&=j3@#x$i~r9n0?3=uI`;G4yxeQUd0@-eZdPDcijRy4x~A!w7d(
zhFEs7=%jtRvZ=yfBz$B8G%~&!dXE}`ZP%fnm=Kz+ZLhqdUpV%h-(9!Jj;44c_(98J
zauYN8^{z>CF+A-#tOI7vGu;3Q=DfH!_h(|i`Z!oid^x#35zJms40o3e5SpUjOqvnL
z@c<&Z1;YfOqqjpe)JY$(R&E`187LhwoS0{(0f1M;6XSIqp}4KE
zH?sovR43LNAn;t!1;xC<61z2f^(zo%j%LZB{lnV)+nEO3t*3KkZEh{M9KxEt0q4)G
z(5rahl8u-YV6<5gr(20#e#(Wa=E41$NDC?EO8<=F64?)8qYa}?v)TZAH}dZ
z?{{!2(ccoz@Z^d`qhE#kEMq}DwmF7C>XUI^xj3scv&SkO*QDa+&xHK)5hx2$vb|6IE2N6_VRSG#>UObCn*{Z}=t55+JcjK8j?kM4P<0`dx)WJd~WAaC1`MYl4ozAK5I_ECKNy7UY6^?$EA
zLG;JS73ZmOHe!~D_JXuDOM+)y(r|Hfo`I7;k@2e2pfx!3^S<#5?JB6c$%}TvTGX#L
zTGb`%@`6@?o6J&#K-lt4$)nwwXWi^|L6W}M%94sUBDy{^xd>?Y8%U*GfP|0+tSt_+
zlH7`WKUlwTkiWJW^r2?izA25Xs0kbd!CFRl+g``mguOyFq;!h#3iO7c
zKD&x{XUaZFk|gx$c4Iw3|Fy}B>j0>_TtA#bg)eazxb8M)dPqV(7@21=>v~*l+kGg#
z>T16szPR|VJhoLQgxC3g>$MVPG+i*`K_PfrUvD&yAtDkom*5mogItK&34On8+
z2{}~`Tcj@9?Ui5C)l-cX&^x~>!sZaGIqvX_4&v;cL~x^Cx*j81HJgG5e%~s+9@AqX
z{BZ)*T2yA$FT%|W36uxZ6^NUHpMQz~E?ocUv8V~_Nc1xGez_OAHKNY{Da(BEXih<&
zC~7r4?Uk_O**m4FmzQsoB~`vc(rlLjZc0scFN5Zlu$_cEPbHdt*o?j7oJy3BluUdD
zz_xdZ_Tb!e&M}q#sbfkSoNFxU<&RFa4g9_!YFXLgKQh$-xx5EY%9K7IA-Nn!KEGxoW8G*Icdv2H~q{P{@c#H#AUV}EbkDHoVgE_swM$X4tr;aIf%OsR2XTaMS
z&C7}pZz-1rD=lWNPJeC`UD^4Fj3{T>>ilf|G>CuGS(ywzP(zE57oQS5Ec`l%N9qs0
z`~&v*hdL$>?T)1pFG*xP;RUQ`Ck&W4|MKYmHB8bTJ*t^7P7R6DlPs>j>S~IzyC?RhB5+-3R=g_4m6lE~tiEz1Z*(%x
z62l8JTTcf{|7l*ye&Ll=NXt|UJn}iNwY@C7ic!TdStCW!v*d;3Q*D8`i#bK?TB
zQ%$84jS2|=b(#yLnw__{HPG!C*8P(1Y&6DBb`!tkg!d>tc55Nc*f<|=#7K8PbDCUZfq-gpyX7GZ8jC%_tO3`Gd8@}QXLxxt$(Xtr
zts_fu>65YwG_%he#4gN;=1bzloc;FTZ)3XNwx@jmo={#3?#din!@t2G;tWcD)5NS<
zDd`vS`(M)JKY~gdR;$s6H}uz23SmfxEG}QsbZVOXaNa3i7~Ol5XO0<#R)d$M2k`bc
zX3pdA-^>Lpv*bJ{!)fpu=VT+=0NiO?LOZeidlxmh_0<_onCArz_{aJ~gbBP2N%VOM_qYd;h*$~SP?3Tx<<#sHP2m5eQ4W~`qiAKW7Dpo(4
zxj>9k9c2PhlY@MdYY&)^)txh&^jgu!`(2;Ag@4rvavZMDM>gca`$@vadov2n%e$#@
zxk6YRb9vWoN@LzvBTA3fmKBG%NyQ)#lYkXKygB?Gp!|-lErmkAeFHHg`>I3}Mv6Pl
zZsC@DDBS}Ps%p+FuX{R=GAP-&ctPs`=5{`IC=aKu;9v27FX3pE=Ot&i6#2J!!#UNi
zNjLImN7CQkp0?-bz|_1D*Olk!fEUk(|LJU9Pa_2r-Ze`(N>oPBzsRM-%g`
zaE#s;8u?(8q`4){5H9>GYSCKr8ke_EX}!5n6Ev;W0>TMA?o&^x^ufWg+n`}en@$4J
zHyidvR&+6U;{{~N!GxwNVs-9cop}!S%a3%(ygqVx{eZ?2_4~lbsoWkKos
z61_KtQq!b?Uf{<^jp~4@>nH+oPL=${GXc1m??u+q#mP_`Y{As0O$_|mNLb!HT=2qrxLN8O
z`lEHs^tRR?sj+}ANUl69UdWboVsZig#tqNbH{8OMF^m>cFUNqNrHO
zpgw2%tPB?Gcfu#5J!!c=_-PkJ{?p+s_k0A7Jb#*b(K1N#1PSELJ3pMsf24`d-}y$M
z`bEh3N453hP)bRiSc?eeX^p=Y<~4xp~d|JDf0c@pt$7
z@_Tc*-Xx+bXy+Dqyr)69}`?ibRGLta?t8x{6uZpTOK?!{RnKJPV6zi^w+=a>f}q6FI##CO_uKj=S*J6?33*}ojtFg@w(WNx2G-Ii}iAC<|ti-d;l-C$Q}=Cb%OE0IIIk5fTuY
zL(C0?5!G6lT}6^F-P#}#Si->*f3ZENHeBD9PvU3}igl}BbFBCk{nLQHVSG`Gwr(F*
zFEOGAqGaY5dgV1rGok*-c^Z=$v3R3BqD!bi_|_N&hz5Lpt$rUgI^$tZ{}`#J_1Rv$
z`dOt0*TQ3rj93ufb1F%fT`qN2*~(+)K{_~euCH2s;q1vaQH0Dlart&_A?UR4mL=8e
z%vx+e
z?WI#|NsvihH&K-Bz0oy?YGz5{pNZ2zBlOg!=R;`mewg&#btxmyXeEw>Smq}(KgkNr
zHMbu#tfso9{v7Ljn;&@msV}4;7K%)5
z%NeGR6iH1NO^N{~Y$$t1u}h3VHV7uyYyy*b$odk}yT10lQs#g(RIv9EEA-e&vM9rl
zoJwOSt%j3HYGYxyx-Fy@bs#Vo*JUwzF1FqDNm!rwrImu>!)KooAgx-dS=)6s^re?^
zIhhoTD8kYSa_X$t0F{dBeTo>4Tzdg#4?|z@%ClR&BbauDKnlnAmeLdwPLG^@9o>nb
z8%)A}{93!=MgnH=$mCuQ%X00Ig0(nQSk8?d(d9>Cj|8rsO+UKyu>vJ1SQ7~{>x
z?;DFLfw~zULF1~IePkv1459b$r2P@u=znT`Ai{Qtf}Zs!pIlZ6d|1{(FERDBum>D9
z<91pG8?E(sNDS-8=gXT%&-v-pX|J*%5?Q0xHxueyBhZt*rE^|`hiZ)bu-JL(xrBq=
z0!WtMY8I*qGNED>w!{^WCc2nKL2|#`m4T;eFn_i@DY158?7p~}lIL<<%s*_}SFby38D#R^3bnvO
zUGwa^8SHflTm;^d%x!$Hp|QTYYFEE`D+4H+My4Icf4z<`;W&WH|4*cH>JR2G4^2N&
z`PISGJ7^rB{e~{`-gUh4d_|37+GW+6Jr8<%<>_q@)0e5m+TU99_D?&5xh4=vY89he
z+>I6iDa#uli0v?ZUIfQ-xWApY(rSK5Y&J~$F=}F+7|$5lP9P4b>YQbA%C7HMeYFS9
zA#azWU(Y@!N#(Zv#YZs|c7?I&RAws(Su;;6dR*X`Gbgy3Y<u=ybCuCrgGLP&KkS4{~vdDEjms@_2ItoUB(
zp+qL;w^Ik}W8RHx8$j4BfVBD)-|}?SDXB@$?#js41}=9e?8iyr&Mh}Skw;{PBYyv4
zH^uE@>)eJc7b6-+W|qR{Z-Ju?&;u`P=xgys;EO5p@8Yo?IXYFQT=pq`18EiAvzWFX
zyE9Mct5Wu29Hx!)ZSo4JiakEVhB9B<(KAlWgtGI$cit
zigu`wOVIP(kGZ}#+JzY8SzuJVm}M0Xw_c4zok3ywdb1YNfJYryBnZuQf|Ke
z8S~8P)_`VIUsqv$e521MZ++Uk5gdbeiia&DtX6XwEpVx5eu|=!DT2TgSDBWyzlG0=
zX9)sQ{tm%-<9ic?q&i<``Jc>94u?)`vJ_-$Tsk+-lZvIjMd2W9xPF0
zL%AZ-6?17%KOiVUzC30*U%BW!h&*@D?(i6w7^4H?>QA~ubHQD!g=AqN?o8hdYc
z21(QVH+-S--H(%}M~yC)!ihd!B^y@?*x_$f+(mOXmMeFz{+4sEgrEOU$lE&9i&vM`
zy(uO~FTtJuQXhD*@)hII{bxhOKSGF$F2)FuJVH_795p3O8wuNV45<)40
z3mH;AFyyh;TYt%+f>+i#V-H`I1|t+U{`-&@O0x{K(Kmq9I1NQRpv0g0HP%?IA4xtx$QkfKotr<$DnvX?bzQQ;
zlfRc;s&j_%x(p99t3Sh0nOQG;`uU8ZcB03v^^)N7w%M;~w5seCZTQ&5Eub}k>%lb;ynhyU>)z>@@3w=Ay)A
zGU*M6nK`ONtCT+|#OE|xxJ=e9%Gl@yO9bO_QkF3m1p&$p3gK
z{rbm7^Tdpgo^6hlD&-vRcXXdOv*z2KtCPz9cjx~*erO++Igj(Kk3}yv418ErCQBd~
zRb~UYD>C6=1RZZqg&
zPp_pu3SX`avs1QG;HjuEURK@@u`nX=&u&`F(_ip^M)9E-LIQmuO8Yp$MbrVIwc|_~
z>a$jOm0*PEBrj;=8?4h1m@^EQ^tE1iY`Sa!lvTN(Klu8V>7u!g+ZGsi=qT7+wZYsWAX8m+-WXVT4K|$*faA9b9y^4Rw>v)E;69(qa0(}B
ze^p5)?A9Yf^VpbE{0Nfz*K`GIBU6V5EVME=Pe2@*%pD7g6~la%ZO`N{vqyb%kkRdc
z*G^RxR22Vx?>7G4@hBq@)c24taS@A^>ltXGZI0(Ox-}Si%(~<5%O1m+w5dJ?ptqZU
zP;%C!6R)15Jq%)`oXMv|&5N&HxZrqaT$wahgQl9pUy+~Qk2$*HkE>?3jh=rWcO7KR
z*OrE>Jhd*?xCkDV|ozTB;>vDCz2<(Sk7bwkFs
z(A#BpHa0kH2MltT*F{q$iFY>B_g@_h@9e*}w~O!q+&^_&S{nQWtaEMGeu=TV&#wcx
z9gUM-Ye`&fy?WU%IXNalD)m$Bcz^Nf{|BPKC$6{+yeGPx_laI-H=`d}@?QU`p|srv
zSQkmAWWX=U<#P
z4pRL9AGY030$i@%tYu32#f8p}+VJF<<%?vgcm;eM#GfNPGH|_g0cy>_@`)0%q|qXr_kG(P
zbi7@821pB160o|1vpd0J-?fX~)Z;FU~2i@eBw
z!tEGJVOiF-fzKJHeKCPFEr^XZCd!^<^4d1z#Os&dG5F6>`)^atAvZq4WNb@hXQN#y
z9mQf!I>&`Z7fJJ|r&{0=dbd?@7x9_A+3UF*p2o$Qa)&!;p^XdE$Z2@LO|IF>pse<9
zB@i6SB!IHbsCINQJ+s0($@&Rf>1}9x#(OdBV3C3HHe6aw>&^_R$5oBRLCNjbn25ME
zQ4lgnA2&4C4d^7%J`dUG+Kv2{L5n=oj>?>PW%r%
z0}yQ0dR5b~6p75cCLpIqPNyd4rgiz(HcGQCnN&4REm7>!ELL;NRJkX4Uaf?nSj{a%
zO#i-0om*gU*&)Oo!QUWrw$A`O$&+{O>*4s-WBzNmFWdZPyHvVT-GlVb?6(=2C2UWzSCGKXLIkIdNa`
z?%Vy)hlZq4(wS6q@hjJot^;<5?Y3LQCV!<_G8@H?G>jjWt)A#-+Py8%UhVK5ryP;3
z9Gn-+!hR{?*PkuSU#bP|nG5*S_I%>>(M9zaneK)x{ihQkNzw&9QnG`I2zhTPD)hhE
zA}3qEEU)U_37>geDQx6fSPhoC?0q{GXAnYY84)1raOhXfhBmk{gd3mI91-*MI@7tv
zEIZbPk*;N%77qZ@=3_Cr>TAyv-#{
zvL3;@4H&c}wN==s`*0{5k|+P30&gdJ65yi^3;GS*g$;X5&mEczht3}QDZ5z&O^?wn
z)%tbyLK&+b^xFLcSs!647Uf|5xh9j1$aj||o05<7jH{iAO$L8A)zWMeh5E@Y{Hj(q
z*C`K@r>IUTXo0*f__=B&8%TFU2Fx^1jv0}#aw^m^jAV(nmJ-%WO2gREAqRcJNt)V7
z3(x;_1wsm7n^0RRh$UB0bDK1iu*&@E*M7EDe7$sg%=--yGxIlx74eqYWU{#z5HaTd
zuf4s>eZ1RT6Ztyk8+3mA=UGY~bx*kJDwx;Y2iOT9nUo5{UO3~I4_wB?U&|}9r_T1*
z{XWpK70nSuAe~Rbon0%EYG2^sUv}qq;XqaJ$rJCKFE7&*W4S7yzYa3Q*T>KBPgddz%5=8r5AB(d
zfhJ(vzL=fa|ML2l6my0|%QlQk%|3(M2RrfH0P=)-iHOP7-$)*>*J4=d>gh)hJeiNo
zC@l>jZ!e~ujddA^Xx;wSl+MDs+fVxX3i(<$DoFAXOF0$UUbxDD>!4mMl*CRJM`{zE
zKWnk$iv#LooYlI*oz~@{fj71C1#x`k@hqpG^2%7NFxlb`uJvuUvE{ktteCwU!`M$#
zii7S_4!Nx~*X0nb#eQyemqo0vq~&;Lb<`st{jM^EIV
zmrR!kP0j>g4X4~e7Zz6-)a*GhPD*#6888K>Q1u@;1N_xY$H{^
zwy1B5t<_YCMCT9Mchq*Kl7Ap(Pjg=be-$%UVZz=RMV`bx&V}?e@ETfnZx80MoJ7{N
zol9~F3^yV^k=7ac0Z8>q;v7ij*K}P3%9WxrU9%I8Evjb5JRAhlt*p9s`*qEeJhjA_
zpMO2<#mRH#wbxbxsYA>eNv6BJgEyA<&sf1@3U1g9six61()oAO_}}qeP5sek+JE^;
zT#U>cS1A$|Y~)o!wAXB4rJcUOlk%C81^nKMvcBt|5URo1OUKojc_h8C3{~A|XZ5KS;ef5rbmI=L
z6s0#TQ*u2Q+R9v1J}cO180t(1RhjLQCK7!qd{#GIIbYQzOe)N
zbMV7|AjkdOd!@ds$l~3;O=lL02X%ZOb#$GxK=ZS)r4>?ZMwo3dCS~YP(-snuZJ_S}
zt5k@)b@7$@dt*)f$j3_>0r&PQ|7UUFw~v$M$(RGqjhAt1x`wX`I3#JQmcwaza6vz_
z`_&3G#^%S&es_2-rb!xEl+!9WT-(WK=%OVy30&q0muw2BpOs0Pctm3Ay>_x!f~})Y
z6u~n%3=!uNI~d5^{#AY{t`dBivuE0<3l%6f2wIo;F4SEn*)GoMBC0;1&BhJF#vh5`
z-|<_20wug{Jw0u?pVHsPMIpsJW8XKD$*Blt<8$=0HDy8eJ{zxv#NB=(fK(Ff*S!&b
zD3g|4-hS@R`hL&6JYSznHD=TTJ;>b@gZIbNW!Y4W)w3;KiC6ZAH}-g1DDe7p#yd8!
z0w7J02B(Q4nK7upSE;lpuJLR83#5uUMPtS5>Wvn&9bHA9Cm9*bP}MxqqS>_erSFjj
z*0De*XZm=Cv=9$u49&({&M~LjhG#^mW(di9D>Kn-E6rBwe$kOwJ&D4rrn8n&M{`UN
zzAm0M=
z9Y|HPo-`c;^8#(ws@egv)v#%VXjQMe>MA`hb_IP=vhntKg|V5Ww6$n^hQ2+wJ<1b`
ztb5Wrwr~@P#@cwh_K*2!;^r$JU3-omPof@)9s3F#{lXpn@(KVvlO?iv+-ujuN|1Z^
zA5fxhO{Bdvu-r0+@gB`jqEl{5j{vL|IWnR4Y_Hg+
zx%=~O1!w11B~FJm1uDq2tp95BSbJTf%%m$1ak+eS_-C51(M6zQM%0&Zq50wuUXFw$
z^;$d6UmHM{?|+eGdy>ZzDSi
zw|0JLT%>iy$F?h4sykS{!AAFC*6{k(*U_?g_uUXBv7}1cVy2}>3P6cEOr*Ruo-hvj
zBHs|Iv;e-zB;#vud%L{x>+`5BBR!h2$LE-rJpX{SVbUN{z1dTUzN&8Aw=DFvL}s`r
zWCnEdC&DJO}7O!n-Wa&;qQjAWzxfQZ@5;^cy?!&O
z*nnM+tCBZnAot^^N_
zBh6+)K7;Ur8@$1-km3b?$^#euo-gkH2e=^J;ErT3L;tr(+eYFKCv~l3*&NmkaO`Oq
zv6{u#a6x73^~PuG7yW)~h89JLuGAxWO7>U@S=&54U3rG)-%wpgpyPlJA~&a6i{7EI
z+9WwYOfAlZN9|~hZ>iDH)5fEZZh1@=aHDn~7H=nNlcBu8gGhU4j96C*rJ4-L-xyiU
zL8EsswqkN&u@(8y*dVEGk6GKYC6cfb(>ZvP+Ro|sHrpHl6u?6{W47~9LWsG5_#w}_
z1uJ;=o+X#;QXJY-g1$w17Uy)$irZ~iX_mQ#)ytIcw#XZ8$M^GZ@P%xU_TGT`uXR=e
zr4n-Ac=FE2t`D9?h=lqQ%HQq4CkbtH+n+-~6Md9nt^2qK4oqf0s
zifR5j3863ent73{(nL5mb7uy{F<@x-vFchNf0
z-YUX;i<6VcGR0iLulJ;IvxZBEpFVQ`Osd?!x{=E6q@C!I{mL04?~F1nb8oflnMM3*
zF5TB9_Q;J@Aj>uI_*}x+VfcssgIlSa+S92aI+dTjG8FQr*Ynrk)TX|7+By9Rvpd{W
z-;Ma&BR@qeY&;1$;=-ZFAET_CAMvSZfX1FvC)^%UX%XCFnKJ23Ua0IfIkx=mz8JQ@
zGfw!Fe!;_}*dI%9%x6fg2ChFaz8OP2td8pdhS}rghx3Ok`qho~J09J#LZ8aTkc7sr
z$dAyD#3E$h_N3Vdz2?{b%7_*Duxv_v*(Q-2`&>Z03zNS8h#}l$ee!@JRf)nOc}Z6>
zIw!7z&C|(*j_{(-gPQnR?Kh5JNymt0SiOj$6w9!`+qMhGOyjde*fGp^(di;vM}He*
zKv3Gr#%c?Hwxm}rOVbgblFZVS)*+IZk^obQr39tnNv9{NH=0Q%=X_okyHgK~I!RgZ
zA~ZOhz&N=o4CLCbwh0?Ca$v^rZk3HcS{%6ZXyO(dOP;LOEiYx)xtxU3UQc?EHB4#X4+us8et)9A*96k2LYx?7$;!GZuMa%j_LzuemIHqkO-CpU%`qi*m
zs!-0%O?-4?VEzH|#POCORJa<4Rps4GF_#HeJO_W*szXNKF%JHo7fAy1n(Z3HEer%)
zm=eY~Kis->uA|Fqj^eA2-L~S3n?gir3Z|(2_lyc3bT%M^7t;Vp*HD>MwR^heFb49c
z?wRjgj}01s;|_=Hxm~ajsSs51Vyq1KKa=W>$8+!yIEN?B?Dt0~Z!YC@W(jogJ|kUi^X&*=`@&!FY%@
z`;1+n;j-6?dsT4iU0&(jk29MB5f9p;tVaQ%UMY;vasR?(}J@mMlyE^AMDt^;w)8u-+w>g<}Rrz+BVzVwg1Iu;Fly?0$=py@DG
zrR=n8i)e0gtu||5ulqPfAe7R%mF;E~9}TxP^KP^cp7l~27q^G7I#%Ia_Psiu7&v26
zgpvA0y5G|h1GM_`7kPBr$o`8K=e2~yElajn7)WK>B@=8){H&FCo$q_}_`ARokT;mO
z=z3h4_{`hz-mj%>&349su1NDN3Q2YF7^U0JwtvmcINf(8Ue!Qme{DY*P+a%?PorxG
zTu7rF)guKek+QC{`xsVH-?s+hW$+=O!J&zzt*Obe;&_}%Yi-7$#L_d1vRtq39Rrym
zI3+bA%+^?3@Og$@6}}+laQv{Owe8?8f%cEe@(ZN&p}Ve6f$5lA!R3dnRO3Q;*TtKK
z#zDIRA^=|C$!l0tEGUr;=IiJ#IC~Sb85D&NVA6GwZ}kkKVhKI&7{;rp!1L
z&66Jy;}Jh@HVCC{=URx~tPswP@MAguDN&9l_z?DWj4XC0f1-7~>4TD*3)92T
zE-F0{?BSaD0jR$(GV;
z%8XHAmaG1|Ua@9q>)4EtoPGW6pC{Hop^7+-M{cy`wfw
zBTvvQ-RrZrHdcQ6p+Bg|oNM@X=N19o`eUrG%(zrYLkwxbERImFnh9@8=DO|gk%S#I
zLZ)NQZpJ@Rek^J~D!n3Q#bkexG7x52YVy@xm{qMx>g}@4#>DPw6LKy5(Hwm31wUOV
zJrmixrIHKeZEi4565MSD@E8=N#AR^=jQBhh6eHS&on}7hlBilww}t!na7W>%Ozm7k
zEj4YkeDc$})*&i5?F;_BTopC0z8A=?C3mLVtw#=Ppr7dfkPlV|e_W_ITXxWycR>gG
zUBOg;?d?&cvXae7$eKNso90kZ^mW(f?q=M={Y4r8b8WGZ
ziyFg}NY+xciNfW3&^+SaQ#S1cUnTxtD=X+oXKQ85QLF~Y&V?5#{48C0v);WrwO+Q2wnkjb*_pgGQId!>uws)CSl*1=7fK)RGpCj*|Bh%8To
zFQq$W8Y7do!_Rm`*2_~RsP5)b7Nk0-y%qR!lt9e#yBxZt-pb3GW-MGDIsotyY1=Wu
zRZ~HOW)()=(Pk5(2C9OrjjlXe9m)4DfDH_0a;vJ-NS98s$wvA17O=BW*RYe$8G4Q
zu<1HY$dv2mKsxV8lZVQQfb-dA4zh~8LRt9Xr5Ux(qkvYI%B$K=^`OtY(nWb*~%nHHogz8
z8)gEjN5ny;JmMjFu`?wdvdaSXleoEJXMhA|@!1!tMc?L_%Q=P?<})&8R{OWEFx{pn
z`rNoVBRA%57Y08w@gLh=?_AKZVES9#RE%HqljTSMmSO>GRMoDsmdC1#hLOd^Hc#h?
zS-Sn93%T=_7K)5Y*YhE)ANw)(@BsZZNl~nGopA`|d$??L
z@ZCCb38U>&VBRkWt;h?c+I&VbYH
znM}i1wMo(Wv$d*KD_hfPjCe@Z)$OM!Hk7)|;XbF8E#o6t4{CVBjQ-C6PZm+H$f*umLBgep**2Q30g1KKmVeY!o+Q_(2sw$v7Mu!o;J8ANU3D=mZ(OLF&6dQ!v
z+sUI;HmtRymMY(QYC}0Q$S8j;sz_IWyEzF%p8BzQsZSR&0L#Hl4(xZ;(9I_YT&sHknmxg_VK7whdN
zDD$md+&7=Q%3o`_pSka0c}^a768koeA}A%mWy|H#Jb17DtI0!tDkQ6breF8t$#=0+3NGVb@|J^M%~^G)mNHGDftu1$y)@a(x4%80tIrV1noek<;Gwy@nGXgl&vdA^qss#J{gyv
zWYos#loaOZb);*lt-H%ItPe!VVRMsxfhj+C+;#-r*5^US&YhRfC
zPO=Y;jfb7~W@M9{jVUUl~NGID}F?k;Yi
zYiK8>rm^%BFPsc2M3>Q)NBm({tJ_Z2f&R9UP5!9|>($YS6PYi;msjXistg%Jp%>-5
z{YTlP4UT#Bq+7$6Re)XxovPec6Qk*9S>)NQvYpw$jCao%n+Nhb6SuujiSuDo!sgwA
z3FNAU{ni;`g_nHtzEES1hXVP6pZCmsvF&L-FZoCrjJDwxmLyW|1<6ivr|7A%ESdCR
zUJ6xts16dleIf!V@Fgns^}ajnf)oEPs4PjVM176~DrQC7n*>(LtJ)oH3NB-^V6nqB
zHEeMaokx9bLS7zq%<~g;i}ds#x{4!})2n>i<-a>W7$ds1J#>cSCmT{Qez9^-iduB)
z<9UAh8s^ANfX{!}I*1lZVU*gn7kXhRk~qs{sk1c0IW(x}(fN~qCuwG1yTSd(bS^T&
z?F}RMBuDm6KYmk=e!5Usd^zQZV-c(?_Viy%9bGc8pqw4OO7Egvb{uNEQI*RG@>ug~
zzSvA-#5qhfQ60wr6P}$H
z=^r^Q+Fn_(`N|-77lGxOeaoXiX{}n8_vPwk;8V~g3Z4xS6wanTYkmMtLKGUY)9tVxYrnaFC&EIT#{
zqV|#Z3=dHWmGQ>E%NM5a{tVO&Z#KueP#*S>T@T+5j^)$}Zk;)BXEs5@pG!x(-Uc<
zj7XBxj@!^LUQ4JDvOKXST_0t9T*!H;_W;VrgHXgvZ`;jO$~MyU2kc+0aK{?{Sp#{q
z{4Djc0ycAq17+8(u75Ayid#zQ@cSwTLO&bv=;OlSIj;=%4Nk80m}hBa)U0+ABCr!D
zdm_r^k;V`K>y?C^nDFvZ(yNQT{yS%FYGm2f5@RC1J{%C{=;K8W&o4@~AdQ)vXvR9#
zIG!fCgp8In2ax_d_$*<$Q`ABgXqO${IQ)6Rp}&1LibNu*{)s^6_I6lQ#(Lu7*?R48
zMw9{ru>qm+^D4AyjJEJqdTm#63m1t+k5Dcdf@)RwoLPmGmb3YFO=k<{NJU#7V-VS#ba2=rXAXUI+gr|wofDG
zNQwx1LS&q%A!5TEjDpmfvtQ~u(*0zAL>F~GfjRFsM2{ZjZePG-2ybsgbp8vXVV>7YCcGmE4v~W<*!GI}(
zHHRUJ!OO{X`6^L@&6z;CX3JWb|3(y*RCw;sPwd?I)i`!1*KH^8Pe)07xV$^Xp(HkN
z>@^mZ)8xJXw;!j&7ZFr^_!qWi>Zpyhwl3PVI4J42&Q!>_P2rt(GpeeWjL&!DFF$<#
zntj1$slmqScjZF)vvi8%qUWJ`SFCT8lyQj(yu9@V+IxDK%ig28GgRj8?y&hNypX`*
z5nWk%u<-$JJ)0@X23h6j$BYKRGcQez^ic$eIJ3dFAruKjR2
zioP6Gek?t8HlS6L;!y=y>;7v@DGQ7Jw
zTUf#QWFz#U0j;@^R2uPApk}g2_5FOg;D+Zka{|A7Oq5VQ(7)Z9mx%bj$Msg|8QG+2
z*}w>ED@xfgpJUxFz;PcuC6Yf-TFq#TgEezFA*A^jayKcjHpd)w2~?9I^x1ab{JYb&
z)-qAiltPLEBOw*2bpC1o`5dnWJx`Dz;A#DIG4J7w^Ocl$elukH%`Q83qfs2i
zwV3N5OBZ?ok6&Tuh5@)Wn)v{I71aC>59?n>5iC#EE1f4+MiAwwB=w$J#Qg;{(Sb4n7RW-`UekJh&Tp3VEaA=vv!dKb0AhV
zQ7dUMtR237!4<|z=~S@cLZPUx;~vQ}@Yhh~`nM6bk^zuzVbOElQATdzBN{7877M&P
zpNY*nkZC%t!0{5Ye0Q94(6ecoJvI2i@+~=71V<;DTz;DC_vM$zk+XLdRf7GXws{Mo
zzYUq4k(gkmmCoCA#xLJFzMa^nrix>*@sz-)FCw{#_JMo8R?np;*F6UFG;s{M)f{MJ};c1T0uD
z7v^KrsGu_$>PTa1g{iNITi;hcE5a#MXO*IA_<`oo075(!~)NN`#
z-d?$ud`vM~;;Df)FPcBr`YA+?U{(Ihr#v8Z3%dhaN@5XWt1V7VgXwotYbQu*XvsL~
zKCHhqjf0u*S0S1{w%9qulVStO#I9_sWB*uFl?Cmzdo*4Dd1LPyCXwy()@HynrBnJo
zW~f7}8pYb@mz|9jx$|4{>*2r5Fuy}vhs(cJpRDS+#G{{7KHGkG{dS{?7Vr2`taj48
z;EOQwA>%Uxh9e_txZSGFOh;mHaF5u@84Tk8x@dGexAkV&OEUFkvl$gx^syE(Xw-G_
zS6PZO0?uQsWXFBH0nOH>er+=2-PuIFU?NvA^>>i_WpawmU<&wCCFR1!y^+%hLyP2W
zrQ}M$hOwm?tzCT%-Ui(mUz8VKLez%(P+0pXEf|?}u5-WcKl4pCX>vc{PHOC^u{+vY
zF?aA~Yv%OVqIY!+V!i0N>ZM`l`|UROtP{MZkqX!(Z)dm$bBLH#Q|zoyev9$WD0l*u
zzaOaA@g%FbI1)~w~%ki0o?g_slO|dr|a3J
zp=_siAGlKxVb)(?`@C=En>c)m(i~<%LjzX7tzL>ZBqTQs??uQi^8^W4_B3TrlN9p;
zeUtNL$Ud~W8RuC6GqxRBz9+1;rkGYO(P;lDj~z50nJ=yG`>4b_Kl~`3C|5;Z&sbT!
z?YZW31{5n|>etS^cibOM2Fns=FkL|q9x5vDuQzo}ZO|Dj&k;9rTkUkH-d|kR;5E6S
z{X`})AKR2wbHNmCFWGPpL&b6`{c`qv(%R~2Aa`m@0$`l9>`mj3I}Zoh`~IP?0z(Mp^7Kgt%m(yE`bk}bIw|F&g`6C9Ud(3J
zX>*B{J($XJfA=0Nki%oy-WTfdrW2wr&*+9C2=iTFn6fT%GQqYa9xo5
zB09hpj@bcTWgt_j?=DxSY#VwN<-~9Jk@d`ee9nlrlBXKlh|(+4p1&%C^A6IoSc-gU
zI_ALF48qc*$9;I!L<|Eh&9oPs9&&BGHCi~O)~rm+j(>Ds&r+R{iIsUAY)J|cVUb@d
zC@7IA(eJ9gc>bB{Ga8q;sUXa&=W*;>06yeE!s|;s78VAjAMLC=P9>u|M-e=oA&SiU
z)skI7_y->}nc!X^7#|bUGFwL$&!w+Y?wuFb|8z*5>&mW(O!!@%+rQKYQQz7$XoR-D
z+CUFr^zg^_9ptTMB^+DlSALQrFK`ix50P6iTz`K?8fc{ML
zvdRaH_3@ZEYo8O9&DMEBRGi6MLD644&0mIpvGJB#2_8F;JCHytLN1$nYIyTd(EU{5
zfu{g6o`>ZZ{iOBhI60KsUv16a7LbW{flm(~z27&*+L5Aiy-ZPy++ZM#gx$9k)R#zx
zW);4jUu`QoDZYCm#=b@yB`elz9HfbPg=76dEZApi&fv0u3;S^t?XZ};~jXl`?VHpi_t<9Nc351kvAbXZPyO{j5S!a{{jvtGHIEjiM|
zUdTwxiZztH5`?>UQculS4Pb20G
zZ`?!R<`oUu)|#-8=iz(6J1&>4WonI<__0PI?I5+QxDp>W(ZgCA0lc8yI7=&Kd(3EZ
zW1ycK#QW^@a!rj6al2aU4;%=-$u_@0%<;TEy!y2;b7o_R+aK<_YxIA-kmt)sgj*bE
zt4WYx5fzK!W;!~HiHG`Kkln7%V5Z=NCxJlM)A<6IQs)}wgY*ZHE{mcHcIDt%*1=K%
zh475G_1mweY@0VUn5FL)+lkGOl!w|rjm(sh$#a>~r!0IVk*2gL)HhIVY#u;kpia?K
z=h=wNr~Fa`3UC1u-eo3xDGGYrvnQ2d2d%c6j{MAFoK-l7Lh~32dP{VR@hXjo=(XFl
zm!2k4%Q{_qa5q`w+ZpbP63^g-VF3ZIrR0o{39pUmrd)IC8*eb$im;ha*M9W
z^!41fjXn?=DC^fSS?<(h+wO`u8=Uv~;ZrM+4!ZW&tp;r9(f!#C$YLWBHVnEB@N+g<
z--dz~0Uw$Ey%0gax&?%v&|e-tf%x-7$l%`|Ua>6me>=zB1b`C{P|-~4pY=;Uy!zp7
zH`h>vowa%>2^YbkId@^o11`Vm8>wrl^zSDcDDIaO+-NrIiF}7yu9r}8OHb2TE)4w)
z>z3!chiwAHC(`;Mo1eyO2#bCuQjfbJCAc0Gv8~j@LR91vv#Eu3yx4H&lyU%5UTo#Q
zu;9?_Q170xgpOQgQq?bsZ9!mgXu4=8-n32uDL-^S-pJXp=N2rlR@=B6Gogd#mdXA$
zOQpthrtwjz7h`-dFC<-^f+chFS!9N331-%Us1)~u%io(?IDb~nuEy@AVcDz<&Yioy
zzPEdKy3Zgu5LJR69D9q{8*b(C^Z``wF~1@Cw?KN3@(1my$G$pd`g)?Dka-R5ENFiw9=GuRFRmZBXdge0Zbrc_&{rJ_|$$4Fu+qC!+H)e*E)YcgWr
zmk=Z>iI4C5{*1Z*+~>LXJmGj(D@OFb3xp5w++7?Ii@
zUKExT>|VoMJ0`d0n7r|f0y*DfJha;C0MndlJIlYJRAX>dFWxmg=bwHP=o5=sJk7#r
z>4FYpjG(|?4>PCzg(~^Yn@rduKcK~&>t0Zrd+yr{fBnt!C%@C{eCy*dHz!PN8H~V`
z_HquL%x#Ty-qH;CEucMyz*`Nt_=cksqdz=QYnUOoII&2`SFMfqDGfVQ)S1q}A7qxc
zM-S%>csYKo+$8Ty$Tu3Q_B&GE*I4?25h5MVnCtnTKJ0wxA%z@>bczR~4ynVVD)@CQ
zon$jvToS=@Y5nS7j0yVUO6LFosF@0b83Ij@;i$>A+$}j=_X(L{`=agcDW4Tq+Vd*4
zXUNH)-i?*i)Fl4(G>bDvm_a4)MYgV{MBt&QVULYEeTIa6vY{#MB`?fDP3oT6qtzO5RdL<>ZO)~EsX6zI=J
zSP6ZfsSRsYN>
zqq%b#mun?Gl{M^Hce@*lVlSxv!#J|MT7uN;V
z5KkWMyV~4t<~n*~Fs}+c&3hwZV9t3UNEnCAD{if&L&R*w_V9crl!w+SD|e1)M|vkk
zz0-sqgxD2R3>C+9UfiMRfzEwXXP*<^y8ay$4juWr_
zs7kpQV#K@b6UZ3PfLw!BFc0s;;jV=G;&~^`nRev@gcf`zI1R>q;DCLtG23~&Zj}=E
zI>P5HER(igeZaeO@Mca;&6oQ|%+z33I)7_tB6i!?gqb`-jZzvX+cu9p^Oj|W9i3*C
zdT*#C&QQw~6LHyIGJRe{FPmwmU6NIbi#=92`i3n{2Cg*ziA
zHRl!RUImngXNDBDwd$OMO$X3oq`Y>#IIB{SeyYBEf7EckE3-T^i`BvB2Qozpg@ca(
zt{EbF-NJ;3-ucH~e=`@QkIqS7?~O*TX9i-xPr|JYM~QyyU*L{&1ANwxG!Lby>CbVV
zpDf=ZA5)dg{fL^=g_F#|&X4Hy%UPriPCR2!oL8Ks`}lnGR!a=@LQNI@_|un3Qj(RO
z8{>KbmAZ|nQ>2hkC0E;Q_N7twpm+3oHRpU|35ogT04A8V2b6U8EykOAmL2VF=`PO=
zPm#ADa%$>01fjb~MV+~;q5gh2$>*YjxALmfXGTDxNzPHr1LOD1gK*khz6|7w>{aZ2
z?Lixk+^X%T4`fi%8{$2q_kPDJkUuYHVPSeFkvGg!0;0lsFGT4zRx4j85o7ss_=WBv
z7ExpU#`SHnK2@MeO?Y1j5RnS?CAH(3ig&efBAi*v*y#A!uY&q{x5ZQ{?Uw>rkH0LF
zDzN(ljhmP^%JxZfJvCvum>RU`8`bb>$oRq2f@{_@*XL*p9}1Q<@>_cHBF@H**c`HJ
zwxo4L%*iwv3E$uba+9E!MH*#VG)%@wR|Qo=J{4w$X1)igt30InFfbQWgJXQFz6mn{svGmE8)`zvd0*Vc(jUEV3(
zNvH{hAB+Ty^g=PAY?(xT1PHA$4euQ`t8%R(!ajtBQExCv!?_WK(@;h|q)1N^VaR;9
zWX2q}!$9}&hY(Fzx&+rx{#Fy5)iofJ!;KaJ}NU!kx1w%^dzb4Xk2O&mm^Bmi(=UaUd)F*%)u~#hS
z3KSYpx6dQrMpUiX9(hPR1*T_ZJVBp(yLBm474-z}eGiPPTLwsbRS$V$Myk?X+@V>3dH-b}0HWSfShLtSXiv
zh-aNf8S$u#<2AlRJ0m>dX?D|_YZZ?KcF*{y=~lm5hUxOQ_?kCs=?*ALQbdMDWH*y0B3|4SZ??3m
z3Wl>1i_;YbHQ5#QzXyhtB%FOfa#Z;;Svv5g*!9g6RsQ}2Omw5(6SRxn>No30>8DHN
zA+PdXbUwd>WJkegH9TeO5N
z^G0wV8o6!n4{peSYxcvvEuqO3lHBvcSB^lu2}O>_(JlhGawrf=;ERTLoB&Ir{veqR
z*QK~i-LVldNkK@kz{0f?09tDsu41J`0WWUN!0%7snyGL!j2E|^W$J5WQ1W4X@Xtr|
z?EbC+N$#<{ZGcIm{%-UCnGkYffowe<&kowJ?E?U&tZ=K{B@;gj%8aUBu~*uSdv%w?
z!8`7}MIXZy7oJ${P&5IrRCcsTmuKVd5<*Eo0zg{yfO5v(N%RhhWt3NE`j3YJ0Hq;3
z<=kaOUHtTS@uGq54Um?DT;()ppj=vbnBEzS(Z@K74exJmUdDKf;=Zep1PVL;RRwk*
xAiQL~duPN+n~s707}Lv4&J$fuB#Qy)R`e$Mm}X_`spq=`&N*FlB-;BY{s*`*th4|C

literal 0
HcmV?d00001

diff --git a/docs/img/kubeblocks-structure.png b/docs/img/kubeblocks-structure.png
new file mode 100644
index 0000000000000000000000000000000000000000..d277a7642fa8ab09a8e03b4062fae5773e14d197
GIT binary patch
literal 503152
zcmeFacT|(x_Aaals5DUo1f;kvs5BMnRZ&q9Q0ZNY^xhJ>fPjL4jow9S=nz`y2na|(
zdP@S*Ymg41+!ezv=ey^eEx&!oH}1GY|G*gXzVBLd&H2n{K69=3@}ZnGB^e{xfddC9
z@7=xi82p7kaNzKVBc$LRwt@rC!CwdM9!uXmkcB)w1>QPv@74`PsP-&M>y^?u=X>PYYTEL2I7L
zCMjz_NKz-xP&MOCH69>uAS)LXdgSN4Cdv!7IYAS*Rn!`r#bDT~?a~@s-mQ(xKhb^`
zx4XHqE2Kt06NQm*+nh>?BRzJIi1-LKo8+?t|AQ9~Ppq?7iwJoBPp;j2;+B-;0@C7t
z`1~LDIC`*Jr01Uu;T$nBR`c*bFvDwV>M}d#|FIP%pGn>XMsWCLgkMJZ91U(is8^OG@h
z!ewmw2dfdUjy>sI_JV8(AA_CQUjZ2J4lduETRum)>>3?)S@ZgVJF-0gM~3EkzZ&7O
zGvR@RKApR|%^v4^O)rtvZTXRl`5^bv{ne@$qa`k
zE(EcNF;cOJ3cg8TY*$&xI`fKjp8?8tn8*tvBuFf?xBQG}Ur!8aWf;>Rf^ev0nSU{q
z4m=?RJxq1_BFREt`g3-uz$Iz3hx>$-YhXUj2T<<`^Qjs=GI|x-)7Qt~INSRu-MS~k
zXl)*qkdneqK}E%H*3>TQ1(6-&XV|CF-d$va3cM`#y+H8eNRHP>#)sZSo2=kxTW2(-PA3@7r%NZtMi|6>!(MwQz*kGn+NuEM>z
zS(Ku-_kJVw7KqdfdQtv_NDbHY6a_b@E9D3fIO_`5vt(^xa=+jv55Uc-+i(xuE~v51
z454DhTx#B)moe=hYioNrIWt4fE+0#A^)!7P1np#W%b^q>IISK#OS<*17<9+KMCcCJ
z_#{cd#cj5-va-|>NtPfdMBbO$r)$}Avq8g_4&{mNnXX(iv>Fk@q2jAL$317aFjSr6
zvNk`Qqt)*|<`@$blk{3Ax+s)d+sewSx75Kbvm?nAJl6l-*?m4%T@qM;gsSnxo-5cB
zsgYpyXiG;Lg8feU!d5>uIbK~?&aza=bc@8$bE%&!w8F4(SFC@#-cmc=1*`p&4+6XMDXj~BdIN>_JqXx8P0SO3TU}Y^cd3ky0Y;Hq(A*|>I1twE1
zKiPDxy1LqWV=T;Xy)w`hI9i$s=RT*KcM=5fW6aa1rovp)4
zbz1Nt9CeH7qgJcI^8+N<3(n4)`*hUT^k80P(a#U}~51Bm0Nt_PCY331BE-Znd=75QwS+WT;A
zslyCoQRDlw7MKdL47Gg2!R|oqs3EPU+OMznksiT~JqN*4#&%wCz9w9EmjnM4xG6+R
zN=modi$oWsjX65T{`o)w^buaeFR6ltnmuzx7?w>Ojw$a>HNuxAuKK63X)g^w1Zh~x
zRbkKafwabk3zm&<&+E3vij7!487Q>S1yCD^GN4RMQqy-TK2;H5CgwP+iaH}&jbNmb
zR@kRs*m8mBn;FR!BwQCy=V?fab(|k$uq>TQ_HV`wc^|783)9hVUSv<2li?{pi4TEJ
z{f5iXtIv7{1~lvr$S-j7<@@dve?~RIG$#%lIPLkPvu7g}noG0z$|niC2_$81VBWB1fHDDuCKNCyAfx9O9pGLl#df_!aSJho?aYU6gdCKe=imn4SiE7xyBx4t(h1TXL4
zuzp=I6~CRGopd*1Z2Ae4{br4TBS+Bi&o|UA5ZwQvR{Wc41mGn;s&GpHuwCpdUEg|h
z^@mpntg=fm1`dMulMmA#V>VZ2Y7euzQUl(gx}=!9sAXk^E#4-^ni^Ud`&B0-wDo=c
z+ICgLA;;LJXHzz5EPERrne$|)Oc~|xu&XH9~qi&Yw|JWIxE7_PiuEW
z?V@cTr4IQqG{h?1{kG-u;!BlUEhcet^h)`vi03D2uP26YtjfywfbnP6_T3r48W_^+
ze=ksuhZ`?gwnXyj;#TMQvbSgZ@`o#Tah0`|Tg{cvWw=YuX@d2otu^60h906qp-{RY
zrW}Gd@rw!4*wwYLrwj8XwLH&jTX!7?-DIQq`5wgaXLcIyJPixoIXS#hvs>Z%iJwYr
zNim$)Ncni7@)`@jRUZqLgvU9}r6gEQtE0HOLv*kE&@4t~jynosvcA(lpYC=?06O_5
z)_U^`rvCKVjRDm|5PSRQLirLR`9k?T6k%G9Q_pL>Sg9zU(`0X*5nYR-ubJwI%S_w)
zGdzz<8tYY|UV;yG|A5uO`HU+K!#Z>Rx?
zB|dC(!S0#rHn(y`z%Ig|@h!EK1Y8uK{yCI01hZv8MmFdwDsY*a3_lTyv365+X^1@P
z6Wxv!P@5yhBEz(xV-C8;lbXHDi4MbsN|=oMtll=bShS8QJ+`i~M?0(E$liM~`a71j
zx8gjfj^noVJ8VY9*|L?@I5S3GQmGzyD_PYm9@^9Hx#_41o%v(I61gHuaU$4BZjuFI
zv#s5OX{0?b_JQ7Q-vBA*a(@;oPbi7Dq~c?TP+uZ=YXQl*f2QD$D1exG)YMI$!0_zj
z`k$(X0mrO;#lZXc7(g{9fG@%y++ZZwrhwoZFm?#mwC}P_W7}|EHV@@7w8Wz3V^-4}
zj1^APbJi|vF-fS?B8xMMN+`4o_JOKRb7n0OT$$2!;}dK+i!#a=w^qK2IIdgh>;)OC
z4n-}%7lz!{l*M9BmEz_j;S);50n90KD1d62LLEhSyb
zpU)wlj`&iuh^9#A2cAEC;;HTp$!$k5S(P3Cw#z?VvX^-=sl$=Na+wFq@c5M|9ZZ#d
zRlXVl*;%Z2a=aRT@{saXSNm@vD$1<5rS+C-KUBzu?+`2WHG7WcuV_b_L`*vYTO#anm6a_kP#l`&eA@AJ#n3pQ<
ztlI8lUB?fWsBX;T%n%$|6Sbb6Ip4-LWVV~-qbItpJa{!Bb6saQ>k5Oh{mdY&4A!x0
z^X_HIP0ne$*Y~3Qrz}#C17s9(*4U&y%sWeBN{MC`w!g@!S)6mB9)fWzI?2y;`ZUG%
z_V#cL1bt~6bK6Pphz%B9o~HS&0#RD(>+S1w3>_$*I6+lO?Vh)?GhB(@b8*h5M}>3Z
zrJ>x%4jnR*g6H%b#~VPol|5Y?JvQ64osfd#s%sZz7r0AeAFq^l3EXwB{e3ThS?v_B
z2gkNUMiX@QvmLE>2w0!Aq8-H!t)MfV*o>01gLZ4aIj(&x7A`vG(fe@vhxb5l8MQ}6
z$<98MS+E?b57e#oU8c>Hixp*Xc6Pq0qobo+RzGl8Jr0dV=QzxCw?E9JD+@4_BmH1X
zsR6IMZJaAxf2Ge~!;&j=vyE5YN65}F30>XHu1yKC=)FF;)B8rpZ>l3%X^xfSGne*e
zpbf5zu`IwlkY&wtPNSaU^L9fPO1H;jm8w-o=Gk@R8^d6wFjxzVfMngmPq1
zP^X);fcdB@oclN5E?;B}heH(qoR{z6T)kqmCqvkz@9fix7FLzLYp{jSm2(sdDcaQ9
z&fl%!;}hHNb@bVk6G%7BOo1O{T%AWT(eY@D
z&d={r^tQLv2m$$WW=m{^9n_=C3v%_=>N-vN@F$LJ--LwL8q+Z&kf?m&7!4Y_FL7bh
z{I3MMp$FltyT;ou?pt0Jg|2U4LdRcKXS?t#WseG?2Ro8wDVwc|#WOZnTheNt9}{b*
zk&D~Vc(Kh=hD8sTORO{--Q426V!bMw^3~-oN$7{KfdbiNDe}8q8JpcLX{YZTD{*RG
zwO-#in^ajbE~DZTH|JU~ozZBmnUtel4)O*cBKM|4qIV)(c>`7|v=1yhNMykYvRA#I
z)eONEjEPLZe68udAziVy;Th_P54&ifL1;TqC0)_4f90H>mapvXNvIXs~YN;bG5DD?$235GC`JBI;>ULhoI4Z^<67}yx
zZP&7npQ+644xZG4flbg{X1j)Meqfdjhc?Evvpao_c8-wH{e>v|P`Xg|e{
zBtEEb|LG+cphbWo<8`D!#6)at>_V@8Fm3(Q4@^_@^HhM!FhRmOvWkD(>haj_dBG2~
z)UUTvG$gp4w#3$at%wUu?RJK#N<
z%;R5TY0)BX$;&CHpZN#sjUlkvb1ZflUzJwBsY1|Sne|R_iRVUY+h(OH@Eqw#k!5C?
ziA4y!I#SzpqD*GKNOTCFD1m2jY)?$OeX4}f>B;yu=`GA6P-cEoZZ=kShB~LjPYY^n
z+JVgBw|1O^SiBFxvBIEYrivbbPN-VETF24ql<`N4tl6ZQz{}Y=kac#>Q+2iaa*wPQ
zxEV5-U__W2j%=nv7A!KUuZaI=h~gigIScgH!6SPG&lS8=;W@rk#y91uODa|z6!#DD
zpD6z@0F}?6USIBz4>o?g1_U+bLXf0ttE&3iOB=}e>nd5bNGv*nyL|?f-IhdyPNm|8
zHxYhP8Vx~P!Vh+LJqry+vXYu%$OwvwAGzFdx59tR#2cbdvg2O5^k%7BHC+{gbsJue
zXuaHBCpna`Egx7Cw(fFGQCWNR=@R$RV`vZT3-PZwznOU|tMcuM;c~@vYL!+vmtW`Q
z82$dM?{te!-FIEs$)rVCqVvQ~yYJkcT2sQ53JvC))P`Hk97W&Gu9FEnc6jCv1_71h
zuq<(aR+yftF2MB!?IWp34)KCrSHExky7)>GK7?ut;iwH_7O6*Bsi>$-VzE50?j`c)
zL9pl?==X2_wRM+0k4bDTyud7-2C7CfHcO33E^dcYlP<0t!Fr5*yj8?1xG`NXCCpCtF~aFJ@n+S*$bw_x7$u66$qXC}-vR*s^uMK@NCxDx
zZ28-;e{Ii+vVbM5^9KP@fa<;tOwA7V3-d9RWp4~!`uEjr`3Te`nWIU20`F@NPt$5d
zj0nUx%Ca==+a0|ebD&XdV2nvQGz*L$2|^d1)J4bx0usujBmQlT(;EV(Rw6^yU484B
zRuIzp*6og8*V{8C-^T;(*kQGS^qVp?WiraG0bTR4VmUmtUjY(07Efy>rMon4C#OC6
z3!a+PyfUFvN9!ck%Vg!Y3yCe?mG50!-f3Sk-QT=`1Mag8j+
zm@8^pR~^CV?Gy_3BeZ`6JvAAtr1J#mer<=Jv6LT`4`?JP58kF2TUl8da}Dd`%zjag
zIMrUd6f}=7xcyUg)-k)O&s5XC4NM{%CA%k^Y)x+3j(vYi+j4=iOaqinwX{3wKA+y=
z?$Nx3@j+DrZuAdmqji$3pY+=_&Ap{vHf*i9B8u4EivPBkcQZT3e$PF1CU5V+GtMgu
z0~l~=$QVlbrp5JkUt**wzEy~NaEt(kY$-&5Q8Waz9s?;Rvv?EK!)+K$TnFlDWd?|7
z>%rSwGnG3tFEIHTPRu2$W24h_G$~#gz)*n3O1j|uIBgp7O~Je-)^>)U5C$N^DqeT-#FQ
zTBZ{TrsD6?7Z!%nHxz#h=<&6i5a*^;NxP{*;m)O9>RNms1=TG9{$X&J6Diakd3laY
z$kAM(g^!e_veI$unI3$lds;K=il9rk&v4(N36*4+LzyjZ)XnJ$E)>uEYo(GXZwK)y
zYfc1XwI#;$^xCrW6VGdWce!(6cQ?7f1v0HIHfRfkktEo
zhQxvzNI=uBh3~98GVdEo^k)beg>NH@Hv=9ofl
z&T05I1l=fM&BB}npsq26{Z=M{S-MqM0wd}3XDOReFBPSrh^B;9IM@{%b@)cMratyF
zD4vhx(V}%PT>3hkETIa4K*!cRE^b32$-Ek=b>K@GPY*TT_wAau;~FuQ*#{=GQF{%=
zXzYn@KNfC2DNLJ;UdPp@Zrjnp!3QXFHPGv}X9MWC$POQ4$q*cEe8*n9IMyt^63uj;hjn9HH`9uS=N=aualMUgriYoiRz&f{rIH-k(X7+_l0
za!$2QjGFNr6X0<9opUM180L9F8f}bzgmK>{3#1aRs$A8=3Z4O{o~{P
z_ki2Zwto43OU?75tpGVSb9P`iR6Lu
z0=HTS57KS+b9U{rNeAQn6UqHdT7(P;rXy(sygzXGVk(}EAY(*i4i?^*|IrbG+aBQ{
zFFaKm;8~4CBCUt2JagQ)H#tL(2eC@9d{e(H2L22W`?0cow`1{!*mx$wS_0`)Wk{1v
zk7fiy?l$HqX0&&I6*~XvN>5u`$>SnW>iU{xmzSCjWr<2h4r%4JV+u?ZYQLXtRVtUh
z`mq=~wK(ixvH7*sdLZ2@wuh&A2nBHZveJGTFH_4k>|Pw+7lo+98{eGCFN1txI1*3)R)(8Rq6G#kLf-cGB?Qp#ZlO_INkxWbnRbX
zw>WiWHL$u|A!X7yM=fLN1dgo9MJOV*jW`
z{{DsMhoeW(v~#!S{C?!Lf?VM9S|6Q80tIS-URkd+Jv~hZ1gwaALG+~@Jn3q~i23>X
z91!w0TtG58)t#x)^SByOoM3Ruw9M2o5mr}6<|Kjoem7j)qTh-8DzU@5KqV!syCC&^
zpS`<7-6>-YpkOAs+aQ7LNd_`&WSf=@Q<;qK&bQ|wtH%fGtkp7_=;KbAyR24KTwN3C
zx0ze(evYpXArdA%8kA#)LA{c_wR4OTc3Rb>;&;#A6e6W-?wU6}3X?Si9>9Ty;gv1`
zJl_}iv&zS(ttZyHqal@gL-Pw(LC;g%H(`vC|RfQpOI%iVh^vh)1w
z3(&9&cliYcPvLzZY#o?ZZ@49#bQBfRfSsG8)F`ry`D{P38t93bRGNFKEFH{ZQ8J`;
zjhKF|cv?dsm?1Qv@_en{)8JF)k-Rw8gpxJ6xT7g(XBWXQ_LC7Q^%Py_rBrP)PCc8M
zofHff3POGT`iN|@B@SXz_e`7XM&Z{pHG(D?-voB^oRT(l+{f0E$c)F>edpavN4^4v
zXjB=zV({oYH|c45z;xPeO^;S7A)58K#W!Q$58zd0f89@CM;yYoSJkYpL&kx0~^iE6tRN5f$qR;stfMVqv`u74Tk{I~9Au^gl+hco0
zaYTHzSO1LffF5-=gFIIPA|WBc8n~O>{ihY6Hx=ULClkt8%j$L|^7QP`P7S)|XMV-v3
zxSuTok~hGH4l>)>Z~gsJbdG
zd$nR~-Dxg+2M0=o0Yh29jf@HRrJ99OkBW(xiY_l`tK5s{=Z6b(7YEjQl(QD;2PELw
zh<43JHs#Z7%2WZ{VW72AJl_xFKKu=_4YoNSQK0;3pH
zURuU&Jxq4ZOBHSA9ljRvcT(p8Xn(!cjT=uZsvX9ZQRBt=rFrGDlpjqe$J&(<I~)!@(akom8GH6zmAVf)nNXKJ%FFa#);LM{)5NI}12FKikdFXix?
z`L^o_wq}96J(D^%H8piK_wtjC#%oZO=2E6Ue$wb~yhg|jMWr6S$$6k5)q>Pa4GY{h
zpGc$)qiG15)BB7K!R+c5qd@(yNLkcX+OQ}NSuM1_;{?Zl6d6?pbSSRk2|>IXM}ArFwa)V}>O$v`q{3
z?!nClJgIgadNYd!|&t%sDK9brQMm6<)gYU%M{7&1OEau#n~eqAI-C#Znp27U;yYAJ%0cd{?mC9W93aT_5U8e3H5gAa~;pi{v-_K!4HY?CfbP_o}vnLo#)1`tg
zH~Y-&ET<0^sPmdVgK$#pTR=FTXHr+p{KzWGS?w2qIalr}m(75tX%Mp$x4omiV!2%_
zv-_DXgPH9lM}=fp?$aLMt<&9=4qH}}-m8^r6{W>@{k;%52K!obWw1NX%ts;b4&f@c
z5J@1sFAgyYSWB%%R+u%WktI8^~V|4hXUBtIG$I;0lj
zM2T`+s6Ko~xn!W+1(Kc!vcJT`l}x>Lv0V!zQvhBy0B4nf6u@E!pc{`exNR^6!tw4h}ZKDa8C{O
z65FS$c?S1X%b?$jMKXbF9DX&7h}#-vA5Mp5HHD1gHzj!l;{H3R=
zTtMR*PubIIv^J(*bPa;?cZmm4Fvl_2bs?SaFXs4r+j-Ul?#hR3zq_}z^+fMz%c5p2
zx=vK!`>tSf!9}0Z#Z|D&YVKABf#3dRa{s*~@N@=LZ#twSc#qP=NWw_5zZo2_fp|em
zaO}D_{(gA>GMCOvAZxKlTkZ{Yhge+*O3$Yu26kA6ReI1U??-e2?HWlp02(VLQ@07U
zZ1iD#mp9~1LFfVszk3I`gRR{OFi;^f^8Lt=z&cO7z>R5$JNG26&%u(%y&#WxW!Jf_
z5)j3f)rhSvjhqwv!4F`D3m1SabXB5!Z==uYqbEUzyVpRD8M|sv-vf>zBzjPU-rNVn
z1Qw`z4k*-xPbq|2o$WQd=Wis~4>PL6h%(>#Z_Xsx#pUd3^`GkG|LGLrFP;sdZY?jL
zCpd@g`G*BVTAyTnv5Tim1I%c_W^IYX0sFNm5#FLz9U*(36efAx8}dld7yB*l&|#`k
za@S5%Vyvy3C>=p+`RNf$QiA*PJu$hz=K>n>snrNq*g^Qy@4xQ_*wuINf@Io0?!LD_
z+e-3*xHrA&zPJBT{SJW8fy5L4?{3}))BSbMVXq7PuMYU@NXWn1gAm03znd!l@dEeA
zQ~2C6{`k_sJgS}mrT>y*{)DmreR#$ez{R`8C186YeF9*|Mk&?*=*$*dIjC2hIk@)F
zJQ}}^dFB}qMjTj__ahU4QA#?4VJ$1G@6~@opkLJ7bMPp*pJ;~fe=+sH`IYA~n9@{c
z!QK`wTkeMi^ik1v8Z19;qdlvUaXc)*cI-}d$V+S&(US_p2=FLCG70(Ig$;7^|
zzQ&K_)_3IO9L^r#-uxT2A1f)zP3*s__l{gK;D2|Y|Nf376B*%N{AHgCAN-h2(+^|i
z?UN)?v;8vGFVFf5G5_UR`(WC?jP=V{e@Q~W0RF!bqW@1u`;W%zB;xtDyM?JH!XmqW
zmI<<%dG4-kATz?AxZ^!C{23u>M^!y-BM^XsQpvB{=IYy`ixuwRNL$?Nm7hG35)TKe
zggS03BNn8I+cE^#~Q2Gl(|A%zd365Q=AR<`6^gqAb<3Am9d2~M_w9UZt
zLwWD@i0rv!nldxQ*&^Ma4BYGBbX8W`;Dxh*Io6W+uey~EY1L-GI7r|xGl0?5B|D62
z*d>9g*U3^@_UDL|r}A+iaE83g!%N|>xbW=LfeqZm7!Z023wxX=A|~pi@6y{$Kg;-_j+?IY4sZQB$D1;3;~P
zK)OUx+-8KumZih5*1L^Tx`I#TGPEah!m#Uahe>b>V_HD&m4zo0WuMF#$_Ap;A$y@=g~^mNhIH654?Os3`mq^sawy7MN|=hoEx;}dYp;JZ2Q
zmx!^vJI^G0E=rtOK1l2}Cr{C}Idu1-BH4+Pt|y5;XdOBT9}gn#%({H^Tj%=O8!*Hr
zns67KoItg6y7Tynf>`(DWpOhFTyZN>$Y@vC2n+9VpNKORnrVQPcMDHx)$J5hOR^~+
z^n&Et@-h$|VSDI!umdd*4&<+%t$B8Y?b!jMBOj%!3Jymte@@AN3}nASf%H2E4iZyK
z{@joD-#|lyOXPM8xR<0Z;oc4Rq<>t@R!fvD)@uOe&{qR2N;N!!_~-9S{W$}C@6s1b
zF>-J(DdGn%5-~$DXAZx6`eVp!)OYY>zCybEod=w{s-)#Q{*&YVF)uH2>N0Z6JIBB*
zCD{pMPU)EXKu4Ih9^I9>>I;SyUu2~hxa~BMUi-MWrz*)2bX`-60p1*aBtM3&)far^
z&~JE4v%O|H*ui4nel`wy|0p`i4wK5b4~yIl02V2jJ?VhImk;6I%GQyjlCw{7nQ@jd+@)Du7CXOgGzW3D;X
z)=+t%T4Yzb;MUK12El`la0CA+W)Q>wD`syQj$CP3}$IJ*#
z8HXEDp@oOCv*UaoBj1AxRep%Y=gl{|+gHg2Tc2R(wH|9I(AbeYpiikVRh^*c345vED03@vd=z$e9edU1Ca}kkauo
zj`A>K>=&UK6U~>Ub6HR4TdoN^--%4|kXD8aM%6!kXm`L9ayaNdZVsX*Cd{gSwj(l`
zcv4p3ZwCHzKC~{2Yw=fw^5BozraZ;l;$b({GE=?$SZqTZe?)oD1~q(mWf2v)|sM*Se@v
zxpi%$TOhgd<30L2kvwxN((0lyfqOAfe_EE#i{MPw_!#gaZ#(-!GP)YE?UsEP17xYIC})gnK>$=4dr%^Q_{H1A8B#Zk0Ra-u#1F~5V@nR1+V-W^+vW0Bri%+v7U
zS?alFtAMRA$)_B?o}VgKvXYb4SLfZSj{l7vs28Yv^Ve=qP4CQ>dlgqJ9?8yh@ca_t
zCY74jq3w}+yW{9V;%n?6*z7d43rVofI}%N2gWW65M@X3O`Yc##*uuUbrdqun;r%J)
zI}Zbr-@PG9E{~I2Kz`P5`ap=r1$N-rft5-fmovaDMT}m^cDbytsE9Q?qSuoc1qcqd
zFsqV9N{nqf&){|%hnY{gV;XA%@9gfp#i}@RytB`$Vs%!hsen72J)nPlLP=yL9oDSu
z)>Gdw3c}(k6OY-Uw1?k6rzv#j4bUM2%3LQhAWgW+rkJ#oqsY}3vwjEotfITwd|bFr>Dsc
z8dg&bTtDqtkpI3HKuPK8qtl5X4g&k_ew)xXICHTOu|_h5E~~S(!8oSuyc&v`c=IyT
z(@iQSvQ0j|S6-KVfUbGrI}JYk*2|eYTZ;?u(M_=Wznyf|xq}lJ*mPbNZF5-DU?`JY
z$h*}|H?FsLK-5)PfgEFV(u+(KH+aloTc63>5oU~YnJ?8E$82fo*nKQ_*9rLeD4Bq^
zTFF5?0k?lVjZFujEz@dI9+B{2YWEX
zZZ8NqJy+q)6bBl1RnLxi3FH%PIxqhlZcTk?GzXtyi5En>y4kk7RrLoD{O2DZA3ZUi
zEG|Ij-9og(((CW7)i;P?tnRvO_g5=W6CE)I30w}x)t+9Z-yr_F&qG`u7O8l3q(S9g
zYlntG`$uw_?$FguxOWO0*IxRdYw-V#V7j|MSb)LFd|p@<@Ym1x=k82MnHi{eyzdq8
z9nzW|z0LmR@d9##p7X|z#S1y`Wq03Wv{D?v(AlN$e0)0JxBETEp;}a}lV75OT-os>CZay6*%5pqLqz@Hpt(V
zm4(luwlya=Ckbir2YmgzAJ1MAH4k84S0`f4N(y`%>6^#1D9@V#3{|gDI0=@83oOgb
z+}EsPpMcQM16T@Um9Wc`i_MHr8wxXj|D57C0)}vZKYRNi!EWd97$R%?wl7?*Xeg$1
z%B*e<(lj~AV0P&DOwelxCBc%CV&1)ZqwrXXPtG(QmQ>-c_fdWBdnemva|IQF8}%wm
zw;7j~CsQeo#l21#9(k-I@Q9I#`@2|dfACjcBU`t`?h$RB*Xq$8Q`G07DV8Ug7&ab~
zk_O+`uQf1kh=?C=1wl&GzoR$8Ov5b3zu7+S$U`)eVX~
zk0iR`Ef&7(nvWO1gqiI|uk`BoW8TEOsHF|XEUn4+9xzO;m16&IkqRQ~G24}q{uw(g
z-K27sv)@ovGfxO72k;O>Ik6*b@p#s>y>>T+EZcaqFj`zJUEyAddOgTL0=V5$0}GRE
z9T}+qeN6%)op=9uk$?=Iqd)kI+%+cNnUiG-2Xnh`%Ve-cZ9Ym!oX=`-BWaCJ;FJ&;
zbt0LbMe%gXb@g>^e_Uo2zc%jiNkz9rh%4U!Im_hA%EqTH-jNun!^KgQ9lbJSd6FuhWswHXrGV
ziCok)|B!Rzk&+IcTGTpblxi6A>(ESdwtN$)PYlLA~
zM8lFniPbts_YIQLQUP)8M?
zB`rsBqIO2C+&{CmBwIVY&I{uBFz>aG%Y$qOT3(0X#O_2dS6#2i$eb?!F`1{9&4Jo7
zJnK&HhhmsXQGi3}*ukQx5dtApjiufY98Np;CS*SD=0V=dc^`kS)i4wH22T46(JrHh
zFPbmNp+5CogT(UPZZGoKRAaaJul6LSexblNHKf|Pk+>Rc8uPf)RGnbz9<3Pc(ENZ0aLtt8*cz*lID|Gmt{ZVv@)D^K-b1uef
z90gi7#8}1fCaS@-Cqo5k3Lmp7(3g!yNAJzlm+~5kH-$_%XhewEt7y4-I%uf7p>CN`
zIJ{HV!Q7as7g$Lg#{x{Oj;b3xDBM+HG{FsvV_a+ZmHC*Se*ccN^izk1@)E|FZN>6K
z(wJtHd9vkc^GN40D>H`BTC1`MY0O4jD2!Yb>8vE)|G7Fk#XZSk)~$DfPt$
z3db|$QP7mrovlM(*X;Du`rrF4Nte);G9wu#73Y+mrKP1cPG_&?S61q5QdNX9rS}HD
zoR1GD+iWq3h=ECUO=-%v@tV^=oTaOYuT1_xjE&~&HM$$KmY)8-y;o3;lr-r6?y!KZ
zkpld?FxFAbG_FUi;EuYLnjY6Zw=hPg3o>q6F%*=?G>X1-^~7ycrLsb}Ok*Qjcr*B?
zMeI{|0|(WLW*$29O8`O^oM5+G5#c;W3t~3ifwW8BV^?yqiyX+!}ZJqv1HB^#=-wCq5S#Yvs|KkhZ{a1S>Ya?
zfwIm!sVg0<2Z`J9)j1e(l0e3yQkWVF-5EsS3di8Cy=+IXy+
z4C{5Vu4(QE@7Q(qNGUnLF%61iD
z$LPspbyp(tSyq@?S;WswQ)JE;rx>$N1X`KC(wpdG@HaOv{#5Fg-5b2L7AQ;u0HSDU
zvok`qVlhZFyioOZ`*!(@JR=`d?jENFJ40jdKdt`nUP(q0zx_SrTtfa~Jietumss7-
zFZ!VRB(qn!j<#$@)uHH%c}4Q%%SWC`eRh^jJ`P*NW1VvrB8BvojZIWH`UGlM=Fxl?
zPAXrNoKS+PKS#u+Cn09Of4FfG(w46J`Ey^EqNIG+&~*+6m`)p8gh;`d!=V^lx8$0v
z338*n-hYuJN%ICQUVA<=Bu>P2(sD!`n}m$U@mO@WSVtlF-+#3A9WAS1mn(Yg*%MkK
zFhP`{jX7=7X$OQ5S}>SUadF~(#ed{eSi
zoHAwMcD_+%mWccY2Q;U$ImVl`(Xh2#2vV_yepr-akxRiUp&K$r^#B=Z_pY~yD*cJ}
zTJkr6Lw7bp^s+mwu4MFT-YBo>o?R39meGKP3~ZEZ=enkSu_(zkjzH$C%?=7RES1h7
zV?Xz$w5|CMY4yI7lKAg@g80MHbIhlIFodtmxBhn`Vbflno4O=D$J!-6wUO8@?;%D|
z@nT{pUZoQ<>x8Ity_mo_EernWZZlYeqh*GyE%?AmDyF3;TQi3`~psxRhbyXEf?#&&`lzz9C
zfcv0ecf;#!MMn3Oqkj)TI3WOa*vs2XFvpDS;19JmKUi
zAc3z&M~!}o2+r`ZjNM^YL)m0cP>~#+Os(D`MFKCPSDkj`G0MDLrlg@TS0dgeNfO-|
zJYOv`u?b*g1ePw^ld`JiQ{tZ~zwPvTC{%=^3)JaGlZ{d@-
zKq%P@WVB6SLBZffI3Hid_@%D94MSSH!)I+%cXLvA3*5*u69xhF^gfD70@?EN{fWV-
zn?-#kwc+lqpBmpNlswTrQ7w|qxGIvY|WLK56Awd?lw7jS6GnFVJCGvabB%`
zNjfub2Nd4o_`-W2{~U+h@4xJ47xU=coXSoisFWQ`(T`{ex6VBRR7x6hbzk;-;H2s_
zKSB*8E0QS1@HCf>k@JvCnbjg9RVPP}boa=<51DITEETL3n)a^EuADaed_`c>iDZbb
zkSla}#Ka&qCvX?%
zKj0;20`EBkid42)&9zjA+L-M`im1qB^&$mftJNhJU$5CX8YIIfr6O`b6R5|`c2nIZ
z`?n0{FI04sEEzl+*t)tpCP>4f!ubw}oKE_Ph7}xY$wlx8dL!fm^MAFRWcOci`=65W
zw-o3Qvee>QPN3FtpQ_I*k^|KZ85f@pPVLHPHyM>S_3hL0FtZ>P!
z(n^TCe;C^tZ6?C{rouaWOg5XS;Zq{W@GDj=)B`>v!!0%`6mR9vs0pB3O7EsDv%2%%1yh{$v*
zGVo#On0YaMvdqr+W3slBkLch=yvjy}_^}h`?EEqYU53nRo&|PBuQ`amM$E~_^c~`<
zJZb%Y%Z_zywSY|<`I~qDhFGia%%dCKv>e#?!^1S)l>RAlZi4lS+a;xeS*n6<^FtZc
z%r7X*9#zggoC0zM?&O&0b1l~GzbVn$l>R8uN_hxn*x6^2o5V-ZNmi|IGX(#r-{O9v
z#GM$55joIV`XrXqG)l+1NZDbK#}5PT*v!yftjjAglWNpjVWL|cKnnFGXR&{TVEU>;
zO1Wl-c38)|zEMrT9~tKYJ_Ab7H%LELc=s%AIK^TYIb&ob-$b(JU07$|(Z}8@0|^Nu
z?as|_HS@CqgBz0q-<6k+xy_35d&aFa`7<
z()@|Rb4Ey41BUk;%Y``>HuL;K68gfjO$-N?Hyw`iKQRoR$q;#9-O9pl&U~Y_46~{`
zZ9~eiJzLU)7L-azw-BUo}rOv+J?3gGpT85VVgMaRrNseg2ugYKk#
z&p__9WkjE=?Bu&tg#|gT8ZCJtA~|GK<<@gr@|YMT-0Y^`2YdsryL;n24q;_NVHcIa
z`x&mc@i=0@RzKD5_=U-Fv`98JOt=B)R9{GBW!%DQI`j8*dv%YKrswYRkFdnEDE+`AuQQ;z~c{hM+^rjq?&PoAoH_3tGKSM9q=mNEs65q3rJ&BiCm
z8q*{Pljqr=8jQYw#RPGmzr$ZyEus`qDr6zo%xMC;LE5U)EtszI^E~zw4pL!Dqyl`7
z8Z%p0NBX7nu~o3VOrsRg|9fC+W?CPuit>UeSLJRa2Pqv^H~Z<39P1VeGXp4uN_VD~
zg0)+E2O(xy_8`pD1|GcX{**Qt@*#jxnG8f%(uWgR+Vs=?o!F|FBQ69
zggYZeK#v9hwpv_>u!?r2?QG@H0?idpe&z7^D8^=|=IBi$csGz2$7J<2(%kHJ-itsX
zsk<-iU9ktv$Rw&&cvMzOaK6G07y8367eIl|(xH`^U$jQTz$mVLWx)(`5fa|Y(o?dN
zI;<(d|95>9HrYR=i1PdYu(UHdf=-*W@$K|S)xZmvj!%iKtD`N`3*mfS%GQSV%db|{
zP@j&rqgINq>$vcXjsa)hq-2$N!=!_%Fxu_a<-o-$4FD2!*(s0Ar$aFZR$=+!IzPXv
zo1d@m(-eRDyLe6uW-rn#!Bn7%f}2eGqX&w{8j{6j$D-R`L%Kmz~Wlw
zDBx)xg_UQ0DALo@GxEFa$KJmcP$)eVG0ztiE6#RC;LiiM)M;VXr%yXq$h~JMr$T&=h~g
z8ASGoJA>$EMT)e;3Q6~7GN0_F{=$b(n)-@?n6qU0TY-eaHot?!BU{XD;q)uKE@ihA
zdkviriJLfO8mEkbST#C8b>E-0rUk8vn?a!I@`YTs)Jz$}1lQq%cU9cX09pPZa*PAJ
zU=Qg@@$!l>>p3ZUEIo5GCeIPI`PQtULD5VLWh4J};8eN$+Jo{%GNDc-TvDU-$@`wKh#UIf9$)qj*A(MW|3r(q!$Kj|(3j`n*N6ux}+`YV>?B3a;
z?Bm6G`9Qu=D>>*oRF_Exl2v>$i8zm!+FH^qR~}1D8>Z{s5Hf0UEH@3B8-O$=)Jogh
z^$K|K3ABEWs>Kp172oX>Y*2LTDxNlko5(7Vlg`h6uk9Dch1gnW)jBG8Bzvi>tk8&s
zrQMWsb2~hWs$N@My-niJZBcwHGBVQja2Mz;yb&vRS*mee74#vpbF?wVbUNd3$AXU%
zr9V;9m-7A+?IiovI)t%-MGCd)xE?65)SYQ51VrDYFA;Y7Uvt9PJFS=mt=2ShT}@I4
z+G}Rt5Cu!
ztDSFszT`@~iAprJG>v409_zwB@74%at@RT-{}`%6P1WEO6!_ydg#ZS5z}|>8(ql;?6smMq4drBk!~qe
z_bt%E`muHCKmB>w>$l-R@tA-nOjCVN6e_ev;2&}iX`G1Srt$9xUC><0U|$129S
zvpgwN&eZ?V9gYHTuo9Q)(S9R5+b0A5-SYqWo}?xb
zXlbwmI(@Ro-gBHmVBS@zc-QYd?qm6mB-?fTCbV16hnO4KUQ|`aY;rL$i_v_PF}DD_
z7aBc;wi4eiC6({=UWvG-za
zm2TzT>C3e+YwRa{E~J>Cm&k10eY9ty#-sSngwY24hTuE-0^JNlrs#S*SDu6al|=?H
z$M*8q&$^CLGKYbb^%+;Bm_C3tHB0|r?7eqTl*_g@u974P2C|5WAW=a;K(a~(B}&GP
zfMj9FFhs#blAMEp#34u$W&i;NBuh>+pyW8@Aqadu%-Wd6YL}(#
z;eC6$SFc{{S@^l@yF7uh*q@Z`&%a&J!IT&=t+7FwOUUV2LaXtkFF^6y=SodQIVr~BKW%~C90$9I(
z$QnL*%nXkii*TziR}5ct6_4;9M>(We8u-8Ho{fh)$H
zq2s@rZqK83o~2y@thcnUJwt9doY>;mJ2O9=54`}$HuXU@g;)Y{>mI-8f03){fFSJ%+$zf)!V4OVMk4WWzQ-a9XCq0WPc_G6lZ@$|Y^$2@DdzK>$$_3)(?aV{K7
z`Y$H+Z(Qu>3w{9p0N^qu7F`8g7#gY{xy6Q{z-?*sEZkB>M-h7PMLx>1NV4bWeDH}=
z@zlWXUfUK9T@7R2mpa^QE^5=aue;XleJK4^){WCfc>X>4^kS?r+xX#(7Qbar;5#h6
z>KacM{(`+~Un{=$i+Nffv-Bmou^v2Zf$wSWEX-v%AGfUUkgo0yBWTQc0R;ba%j1>u;<
zmwB)hg3PbtFlTVd#bW0p;s2He;NJKrp(_vjtT#Ao7wg2^5W_JuyiuQ_t%nr#wW0kw
z?z{YUtbcw)${cJEGnYl(JEu<51C9um_CO&L$4!9C_itjy{Nh$xm=D0U4`I)+#N+?g
zbFrVJHUR7MT)+eS3!?ZjC#%wx;~qd=ie1KA8)kN*{;mKGp#^60-+uNNPEfViwqwKA
zcUpOI!*2DyM^B9#_9hltkG!<&q^-Eho4o&^|920TqV@w+jhIN5LmBUl#PJq67O0K$
z!Bnpr_)mh8pTZXBJ6;)2TzogOV6|(tOtjY35q$hev-3a&E2Kz_3~Kp6z@J4oz8O
z6zZAP(wfY~fd%rt(Hfapw%pnWBWLH#!$)E)fFPA4+YJa7)EqaYRznHS-JcjLp_-DD
z6uAy=BMH5@zf0n{xA`pU$KCx<7Gu^(-s*?7-n(Y&Fni@lg-G7bC5^m#Kycz#Kgvn=
z5!&fJe%P&J#`{G)XZ}Ak-hXpA`n@3s=cr`TUDmqY4tq`sy7PE5%$BlMDfcSh_J6Oj
zGQ9uZ$yt)YLctx9OKk+)Y`nmJP`4nEWvIx>0jTUX@ZC{1Q9))acp|f8Nvq~*bOmc=
zDkJjHceP}K-9)Hd+`93;>
zvl9m_f&P59A?++U=f4_Sbo{OLoW5-G<}?S2TG~jr&_|M*M+KT=*JLM)^gtmX5{yZ5nTLV1{SCFE6r;
z=18-qMnCT|tZe(0SjY5<&EM7U+gw^Ye!#2YkIvx-d>oj>>TAzlaf|Iwwm|-7z~L@_
z#s^_XH`ou+b#ynvtM1MOjxpax?luVM-L$?^>hc>nM?S;XxfkiP
z@6+^RrBfW^Iu*u;cnB^<->@{G6~vvT)bjYW;6Muc(`xMI?jOTrqrT;KU#Nw_jcVnpZA*Uv>JPVwyj7XK7f4Y$1rHbmyuy1
zgDHC2oR{a|I58c2N(DTenhs0q4+Ig?}cET!N_<+G-=yMJ^T=5C9%t*7gQhlC=y#pk~f?zGA?zbjBR
z&ke+D>VPKB+doR@xu!re4qyqF=WO>rzr=m>jgt9e5fio4XHD$!tc+wg9Js%#8k?MU
z<~NotmGpM_o$`rjKvu~ho#%HbSU-Br3@^|4oQ@dOac
z;l5@mQ&$79%yO({!ku4VNI9MpY}ikD7pGESmb-+Dkp8cPd6&yRi^=)0rFySh79Pz?
zU_`2%)>DHng@xi4EbcOH@>Pap2SA`PjLk7&2cgv?nE_bjyg%&lw5ANst!Vv>y2*w!
z_#4N(8P=s-p2!J9tIAL?^1sbFE)$#zw1X$E)*Zd`n#UZP5o>py!+o5C7x!)vCaljM
z!5$yB%@Hu+`3e1=4fg!5rT>Ob?(g4w!tdcf%)P317W(6E)sz}9nILU4&X3>3mSMq*
z63b!>h)xoIyaLRQY2Tnwi|yF%9Yy%hoSMedNT=^I2jg8p>oJF(!DpcrXaK8uGE9DB
zcP-(cM+4Arz4nIvt*!H>^ULX;%KK*Fw*SM&xYxtA5DROfaxaa2`twOsv2OMqn27s7
z+YDxpmjI1=UgUu|)6zxkv(W#d>+C9iNGbZN1h`kX(*Wn$G?v_BY6fV}LnL{>nUj-*
zW2XQ{#qEQse~w}?UeEU7o=wKM7vrX+1>vjHUC<-_F>}_HoOYf5FjqbodRBweaV@+~4|yoS$c%Wc{8
ze{EztiKepaYt40?9U`?0UR62Xe-sGb#0sCmT2+&|{W%?&TDShm@-Wtt;4dO71llwGjENPCvFb;m3W{iDs27K}%FVX7kqybSq`&^W$+bK(H8%WIFgGq8)BBOqcXLin3UpNU`rZT`8Mc1mDhF5wn4@d_?NX}=R&a0+)eJUmYNWzr(;f%(#vzO
zwS4?PHow}i%OhT}$d?JxcL=2k{~VnUHP^^h>R6n8yj`5SiKj_19xHaQlssZRix999
z)1~;0eb8SA;`+3Xr#Ftbj~Si3=ngPZGZHf?U@mD5z%)52Y=%~k&&2WK<)srT7^DlH
zy#*-#v4WMiFtYJ3?tpy8fmN^ueR&XlZ?!XL`ROg(k8ug;e@M+u!ohL{2t{Tc|KBJ>
zoQIEPjyeerJU=o3o#)>Kmoyy*R^E*jk<jrURRV;x
zl|-T|_9gwv7a2+<@?g0)SWaBSq6oxf*Re;NH$i0->lI($A7HX6#wYms_>RxP*!ajF
z)Esy{HdkAT2{mr__W(gnfQqQ=6>%+aHDj02BY;`{+i#DZ3{BMuAh_!B-ZvZ|G-P6l
zRAOS+vExs@rgIe(sttku^(nw0`T!>~VZn+=CP+d-M!46(hn&aUXPm2NcQs=`E}07U
z4peAmisG>mitjuwJR09TFoS~qkb>P0+hq}4ipO>nU1nxO*Z|BA3m_M>`+4I=Mt>Q*
zW+(OEZ=E~F^)LJ_9X>*I-#Kd>>X4(F@+qIWO
znPl!Q%I5wdW2q>$JUTyQgIiGTD70YZQ>B5d05)wmNx}u;!gjK{K*36nZ9%ca!@GDY
z@=PtOxR35F1`0oZ91I8NXQl|kqp2a--Ja<{)uRdO54V?T%l{TmjpL=(S?v^2%Po$X
zM|E!9CpC{ILy62jJXyItT^Wt|+Rl)<|C7!~=WzY7E^4E0=e~6upmNOeShX*SdD$!Fxvu`+AlSU?YFZr1W^#m0<2vs%@_BL
zW*=gW#S&m`u@%fp6{zE&CbdAK^xDJ6xFgV^2Nw>u6l3=_1FH3LjOr>ctvTD@3z|R2
zN*RDVI1l79Cq1bi3!X)Y!3C$mZ3Fm*1ILW_mFUNdUWgPQ?zEm7ACz%O9*
zI#`TtQxQJ!%(so3Kz4UKSk{=m?ysxfJEaR97k9LroEjU$R&b6Eb)I&ippG-`a_V_P
zuJx*=KsT)jMSh8xMO0LbxaM>X(y>WpaO^XgB=p?|v0wu1{X+nlUAYOJaUM8A_pHeg
z=_9WJquS4hB{$
zNk=_5vh&#JM5moKI)u?0baEJA8HWxgulGy-vdWo2j&#apq9?2e?f|u1^xj^>k2^Ho
zp02AoK+N!cP&dSC#xsInF!*Dg8K-pq6R^w5nJ)n4&=sOzs5?xKJMu_H@$(m$;wB8Q
zJBk9FlMEZ&n?&B`bikJl8MQWbZ^Rl8aEMjd{y)Jlkl+ohmDe8m>w5dUTA6i>!P3o|CwO0G2~(M
zCl>(kY10yYbpZceaiDxxA%AH_B~1Uw9T41X#!EDUUaC#S^xmDcvNU!`3@*xR)&D~W
z6A8Ln7^sFOMF2tcfJF#p>{NM4v-9DP`R|YBk^Aq##)w(H{-bA~G
z=4$oZciE1A*0wRM@x*)s0*#bK#{mz=r}sQOjf?H;ZdPXF_Of
zh}x%dIV~m&xM9$RaMep*t5@>J?%Ue-R8PQ76*1K{-GwkvoxJF_oAvS1E{D0xmB7wd
zp5y%@*2X28pzy0WP_zI>2rJge&RD3dLrbA{wGOfcRiY~bH{NW!deZ@-pU@0^x=;fu?CFsb
zeL!OL4oiUj#Mzfy8*9b9f_|Wbp=DrZ?&!}N85qzMt2MP4?81gK;lwAm#2~w^Vvd#H
z7SuOD#jb5eh3Sn13r&9HL^7|KJTqu`bOXdz(Gh}>u|Es32bhb7+PR1Dtx$`KR?zo@
zm=mC)|3e7L*Y1=N@W8X6($A5+c&US&L1ZvPMnbG&di?p!Y~V^`kq_-{^~v1WkwFQB
z1mBk#7a*Qj6?vAPlztYs1G3>4Did-i!EU%(rq$qY_vjQMm^unR1L~G`n~_
zfWv${X`A8E?6CE?7Dpd5TY0lC+z$F(AY&xwL<(~Y*9Al==Hc=;k=dkcp4+vLqwWU^
z>x{dq)pzxaaBtOUCil5QkRyUb0WXX`bc$LCb%oPZj_YpT*Ti^>HS<<{x+#1+P+W5J
zxI%QL7{{p1x+oiM8;omSQJRjvEa{!n>mN*42HR)2`OIeon>iZ+$BKBdGsd&J1d$Bi
z3bXAan@NG&#qJ7+9mM{0qfuDxRC?9b3!(!ouGjN?A?GYEY9_;GI2u>jf(~FHzz99{
z_YuzCX+ly|?rLcDciV+%omXTNIX;Wk)oWIsU8;*SyjvO^ZW)ILL`h3RoG+$+VU;Z_
zELyK}fQobx@J~J40244X*YXXG_s3K2iv#+TIaQt&w#?_Re>A?V^W+jub$(&36m2L!Vb{kLl>emFAGnZM*(6#vdDWYe+=dtDDw!Il|{Fj%595n%AOI?x@W#T
zEtOZ^#EXYPA1pz2*dTi5`WCHF?YBe>N$KQ(CuNyT5R`-8gs6gDdll@+8zbByiiQsR$o0w8i#_k(+k2ZP
zIN5vM*5m<|os#bg0(jVY_+-XKTxk0fU@7%%s&MYk`>VWf;hhP83u8IaoIZ!pu;y?I
zh$q`AYfrY|DH&=TcX5*|zu^r3hCeH{=j=dZ7cyc;^Ny--WY=h_YdUsMxgQgFwRW2L
zF)9W|i<}o8YgiADW8Q#{F+bfSAE%#4)S1Zbizj77a80AnpmN>!_s+D`2NYePxC5@q
zv5D<`Dd{QlkmzF{<#Suu#?PWWU0g5$hzEH*uc45i%EEFe(#7f2h1IlTRZh$$CyT9o1=U$0xf&PixZr=WH
z@@&(k4ATW>z%8&e51fNWJW__OkmeRg?J1t6&ibagC(^7I5%IZLGVdvGquFlDA1nIL
zU|;>6fnYbqapa0G5K;HRQeN(jhx^q_KN9xU7JZT3;Iid>KC2{M`h%chJm%veKt6;)hl@p
zLiE+O@4DylCjYJeJq6C>BaSOAq>bDi3ah+I@D9DdH+ec>@?13h>tqA&(88o7VaP6J
zM-_q6Hu)4-hL4y=E}N&75aJlcz>GJ`kX+xSaYnKb!`BnZ2^N!0lvVSGi)!N$b$j(x
z1wJk-HRBV>;^8BM;x+pPCLcEBh*m`h3vamT8{AUMj<1HQ)WkhrvVaU)SUXg#oj<`<
z-UM2#w5_g^axGQZcWvCC#f%tBj*|+ym4xRA|carf}>`0A|>MbhsZl1#~KQ(I4?zk-fTr7iarDz|NN+uOD<&Qujob+UA2ii%_B{%ReoP
zcmwi4aoMw~A=$h3ycIK$r)KSrFcq+h*hfNAQt{l+`>}>L6#ey!7x!SjZT*W7V-jC%
zYp3Hn5%_};uz2WBy!`-v`n=kWD0uZlzymMkI=3%=0C*20@@b
zy(Wt3be~MN+7#KnH)3DcW@SGfHFH-0?Cb5!g%1_w$%`)WMq;*@3%pcyw
zz#KNt1I60}S;wMQek%Hfc%fnGhPjwUh_}qEucrd0^Nzo*q^BGc&ccs@9{UXUiNTI+Nl)!o=cw=9Hu
zZUk}DAmE6#KXgIUC1<5%vU4k-fe4Dy@(FM~j#|BMw#CeLc1ZJMYGaMtN1OjhWck{D
zske*ayCBHn_C@S3QJe#_fQfn?n>+5C7MnjfTU)z&8`s<7xb^~;x*mSw@og-9WuYd8`){~xB8pWwb7zn-kq;fCk)f6
zGayDqdI!WKZ=iC?K1NHwhPQIWZQ{kc_T(@s&_9=N+&`D2&
z&!G~rIaEo#nbTE&R|ce8Ovs-`{+CA_8-7}ruj)T>0#$%5eS?d1@}-PF57QZHNh#7i
zVGSf}ZJ?Rd4Vur2qF|+Ctm8dPwPvw#KT5nStV*|&%PpIGNoYZ+Ayf-9wS7Zz)eetU
z(Uzw&jcSV3?Q1U}c0{ga0l_zQzY27l()a7kdS1W%YT;w4?~81*7KWLK3!&&w=uB`kCgnI0#Dg_knC6Dqojeow}IuF;m?4E
zy&$n4w7Gf*Qbv;Pb*7ayGN$44`&GN`(15tox$6P0tq|bE%z!Rms|VlM$5i$ti)vQPdc06z(e4n%|*BH%}9{}ZxHHQ5&$1cW#q9M+sb>p>oIXS5?8
z8cXO2#G~}dtCq`0g`1EA|M3wa4f?a(wifl7?8ziC8w&G&2
z&)OxsWHreBPF0MZw(*?1tcPldeFs$!@nkFJd^2r(9WNRn%P(@S|?AyDO-Z$LN|nXq)$Z0XeG5A
zBIOU#NzSsnv+0;Du~99bu4m};Ql0J}Q{`I=(w|Kv6DKxp`h|zhCE==gzesUhChWHD
zG1nzTmkpDV{XM)U^^*)$)u;&3H&+5YdC;+e<@_B-;t0(gTk{QyeAwQjWuSa%0|0?K
z$LCp^!;;D|d*6J`6KxFq*1&nsl(o0^f68`ErSZ@Z4$fuzh46Rn5GahP7}%|~gK9+H
zu@}uRYNNWv$ut{Qoo)v@CEZtpk29f@Hbw%f0v=nXYIR%h>GS!|aU*}EDk~5so
z`Uk2_g5SZRAhj2e%5A#gM3CS(TsliMKU|eRrKibZXE@6^o-T%|-Sy)H_s&8dfECK>
zEZtM?fER8^TY`R@JTBmtjyxFI=&&$R^`Kdp7U$DXAXphjmG+Xk)9zWuJ0HZIc#rH{
zgkz(D>MX}6qd#v6<`qw%kX^gOs{i84BRTsm^W52O9bsJRu<=_w8HM{c``XcA(Yj
zbk6OLtPbote6(n3m2j1l!>mg~Utgoxd}|lzRe7k)e$*zz=rbMfx;s8?Z`PX3FK~#;
z9<+V#Vlj(Sv9oDs+E{OT+uZ&m3EpRvf5OP;=pEvq0)xo4%(J*$+&z-KKQqgu)q?C|
zSo2KLg4fm(I66*6X@!cWc610hT9?jpZX!zvY+dejLs3XmcbLwWWS`Gpyh
z#E%C>X#eqp?a2irQ!TA%pzmw}?K^8?nn8i^AJ#=k3&Oj&;4|3pP8%QIA@cs9m@aw*zg5V
zG3bC=0Q#>GHNx-Y9Q9x9_n*W24}U8GRL!>`3E&NVe|^Iv@CJN<
z@;~LmuVLTNmB57lS!#Yv9>pP7HXwdzo2e}YwA!0M3~)>-fC%a0dnx;Ytf!ciLh2>1
zPJGftchHZ@<
z($x1L>cMK~E8C~PBbu3{0rZYQI2u$p|GCQl@7EyyuXC985qu}k#kM&xT&XNPojXg7
zViLE-(n_%YjZuT?5o5jBwTgH5|6-T_93<{!!S(xl6d(XzjN!xwD3JPB*_za$MO{91tlVunwNkEikfH~xQE
z6W#&+uNc3kya~Se)2EVjnQJtbCJD4N5Vy4Xlqo}_={*G4-RgqgNdO_y^yXJFzazr`
zht~VigtihoE^U-04CdbC1Uh?!u>Fhz_8JXN5*-D@CaeaNlMffPar&6Yoy0jqTda@#
zPCF9ba2mz%@f}`0^Y0yHFeqbNjKzGs4BTkiAAfc`R%)DjYX%t%y0yV=Q1%!U?fM23luuXI)z(Xy*EY%EwPhLO#>1nt^E)U4Z52G@p)AK9r=2ZqaaO5W
zu(A%u(ZIcU252iz)7-l#4ry92(tt=$q-BzQ4Gm5@M^k2F6L--nxVXHR%I^;F6JK*X
z46tr!7w?94yG~!jF#^IFvFX1$L28c-xFy1AlH+gFe=teGbl_Y5qAdlKoNjq3A|t(-
z&Ni(+xTns4DXa?z
zIu#1RGs&%>xiq#I)aoikk8>=3eb{19WHqI{Zav`QRy>LNo%A;z9DA|xF!OYfo%(%V
zLD6M;Zp0X{;?)sW59903A?y&PvId~qEblrvIWh*|9GT-5#Ba+n7FhDc#s^9uV*Spd
z{6>G?V-k5-tCP9$L+<|aR{$~woe7%1h$tP*301@6NX<#tEpDnoi?CV!_wSF~U020H
zR}a~IXl^3e;Bf=?^)3XX#m>%aL{)-?0S{>R$^Ml+zUSwUO!Ww%ww3RKJu(+O1TxwzD*Xt=#rid^&qY<>0=b|q`Qb=`^0qrMP<
zHnE(XZo=MP(HF;#)N@LAjmJIYM(k=j;x^LnE~TdVeK|sJS}wxv5>>0`8syWPib=j4
z$0dQ{q{kIKK;AxqwML4ebFkRsmf-1keiN3o6a%-96_SqxNm^CmQ|5x%;Hn6g1Dr
z;Ir@^kc2n10Zb0=0k{O>8$UO~q$>#ki;SAF?y5~2
z2fN>ho-M_yp(6ss(RUa;<~h?$S5*UN^kz^tfh&3WeBszWH1uwxK-Imu;`z<06ae?8
zuafID$dDew*k{WC^Us~G#T&kz6N)h=Iasq4S|zlCfZ96
zzH(@94;R2-@1UM3OQN6)syWXI!^<{fQ!JzgtGr3ez8-sjx$|#S0W|8d;
zssal?b>UOkx=Is{uu)IUlyB`DX>cnLeRePVh?GSi0oui1z18KZAUcYp76t;gq}O
z6`%+jK87C>mCu!A8CAkbSmI!ImjoX`J>~=i*D4&W<|eGI1zrmovF6u{kG%t|CLj&do(fRrS?E-pG%i
za%@*sgCatW*vswhZ*jiV)JsdKhb#68b<~Rq^fA%Q35v}UiK#2**!|4BVdPu5WHv=23IYYZXKU5OHcqc8D(Rt%l
zi^KJ$YpM6c?oJ)}RN5fZ2MWh0hFsm+{7cAAU0nVw?Z_w@RO
zvOlH;7GRaGbOYf|P~FP*U05Yb+Y__QM*$m@_%xsJzVOL87q9(g!-ds%87Vbh)v1se
zd%enWp;~IMo`DY*`eo^C0k4W?d57+v_t-HIoh8Pt}5}
zciEgBic;Q9BI+;%<=&0tR_I!)H`~&2_xq$$y&UDm_lC})t$Sryid2X<(b>dj<`$PF
zb@!{cy7mjfWkAkW@*383&BbcDf@%q|wGW0qePrMCvI%gl-4tJ6}A#1oX%pFESz#m_uEw
zUVGs|?lLV-Q@sG!qGc~xq=QarWl~OelGmW?#O1B&*q8mQsZ?6M3208e!zLBJ{C)FZ
zJ^P-;0-1yx35E6tM?SAajj1?JN(w~j&n0FXtD}SspKryw=R&&zj`(WGp!+q8N*x2D*_LAY}m}D?B#o%jpP+cR{Wcl9?JVW
zFEcT|3FyOiWI9@t
zzKz%U#SFnc1CCyr7I$+;zlfnj3;dF}PL_?bhf>uP45t@pWQUe)1(TVxGAL3`k9|6k
zmSSKauN?ZFttWRa=ACe^@tANUozQJMhJM`+M38Mqx>NlvGBQGO;gh~@3x1zpc@?>haxB%%|>k>
zL^n+&6aIVFfrPXr{JI!JixG|VtHWAM__9&0^!kp?OTX4O`qQ&Xtdi2Y}{>T~a?P87GhNjVqSKM|P
z->h>Cr_+o&;5#XFD>lqcqH5i8>s$8AH&!XP`5fro+lFLLs3W$-uS~Xzr(5$5zR7Fm
zTS&m9&VG6%Xs5~>%w>^<50zMq3#(nk4aHe@D210<
zZ&%D$7P+M}&a1Qjh|6k=$q(VHSqikB>aBvX3(hU%_wAHaK8UfMf=wdO+PwA$JgChc
zu7!BPt~>$~iM2z>{x)SE#w^y&^q%vC64?Xsp^u-*&o5LxP&#G_kbuf~%PIV&KUn82mKOoH%!UV~8HHZsbdc9`)AivGi_?$@Si_h9e
z<}i<0BnIns13M15lY6y~W<*kp+Namscteo-6?Ps}LWfJOGOJ2O#up_eo_FiNT?(`_
z)VIExT6~Z>$hV*Ve9U-;vBG;fMRT;+!K_QZW=g=yX;Ctcnofdzl;z_GFi16(WJil{
z%)DFnfd+$1_t*z58ATgL1J=jKM=NdKP$0imiGp3Y-QP?o=8*8CUyioy%Zf^3jdrf7
z0ru8J!z>r#NnDH(E`4(Fk`4c3e`S^U3BwPqU!VSMEgdj2g~7#zQ)D=7F`9
z8T}?V=DshhAYl38LKK`AFdWF2mjbyIr+?auNGv}7WMK~PvvWgaiyrhY1lNhi{P)p=
zRPCI3VeZN|)6895g2{>kJA^Hb&Z9r+FC#0*^?Ss7Kjiok?9uXcUA)HaW})|>LuK%)
zmHlv>B2PQ-YX^+y^Sf*CkG-dHp7Q{9g{rUe&U-SnQLNsJ=A~bH7mmsk?0QhB-$Y7&
zb7A+3?O6dS8k*<)UBY6)_d~?~7}~O0h*C&5UKCLXU!3Yp{$r7W=D8w&6k}_Z!$~QK
zyJN`mcz{hRU$p@aX{P33hhWo!Zs`Il`477Jf5Jp2H
z9}g+5b#`a0R)_=;<|%bn9gq~+KDkqFqMHmfh{ywneZ!7gsShGXx#jO58D2$9qKM}#
z_HK4sbK=%UoARu3Gs>6nL63jD53hZj)bw7DMWiUX|B_%p>NA_D$u)9CiAoOexw
z65ji3JJyP5KYpf4f_^XRWBJ;ffQg*BBE@5Ubj52OstYqEzkK(P*tQ2LgWy~8QH$JR1r%cKs(wcq!`mU>8#lb#RLM|ZmpdlJKVTl+9OAH^7k(a2M@_jx6_2Uu1+7ls
zR@c2>xG=+II-mB`fJ6+UEHE|XwiRvFxinv^gz`KBy&n`t;EoYX#`Cn6Oe@MG0n^@S
z9GHY8O~OfzddU0}7e~}7dO5f+
znpxAkCNZoik(pSsyNh@(avjnwMpAseLO;e2Kffnqaj>5QVnKV2*^ikfNif5)!=7=4
z!B56rH&){4wD&)`$Ny;jWM@AkewLy--mf|*)ZN3#us>MHF=EofnNb$R@ry
z$R-r7ZJ%W!G&q}Mwpxh(ENw(mV3Ky9WWC{(_Q)^=QvSL9bmR}_nvHd^~9SN+Z4|3Xr(ubyX^}WT5Z2x*3`WWbdT_Nb{J3eyG5Ev)adNN@0j#rp*`%^
z@>{+$azS2BzPIwu$J*ZHp1*^9qD+yhSIb?&WunR4e&VsFxbq3crqeF`0tA1+$2|ng
z;OT^)8>%k_A>lJ_kJ39bO(VuOog0CGHz{H}_M$ZeR=m|rg!cEVTGkf|+?(ImnmIeL
zraGvy{c;ICArZ4-(@;?_N;GJL%67ETL^doe?~C7RfK)d*)ZKDqL3y>^;o~ll>=9RM
zC_Xop-`re|t<&|J(mAN$fG_o{Adyf}e4Lz247k}!Z4JM(l1X4C{6&_3#e;ihPnwzba%qd=H_B-6m)8;Kac)&ky;PuFGo!
z=LUXs8$9i+6>hG>&~+%T)a8V|n3g!$6j!r>*Y8xy!jnyr%Le?01UO
z2h!CRe}264m=AY4$T_7DXF09Wl~c37
zO?T!UwE@V$5J#0}qxq=ue86Pfjk#sC!^z$ESV5M7>xLV9v;1Hs4Oc3!lVoI_v*awd
zAb=H232f`D2CllX4|;9DkVcil%(Rhp3PZ~v%XF_VQJ=}eR3&NHt#*#hR!EXncnZ@a
z$b;-UCY}y?_iubpN4~BM(U2uEUQGNzxOxfd7%l9dfu7}yGyiBngNDqc>~
z?KKT>-8=QsHHI7dp2L@YW52&(KAletNregjfG87c6e7n>%
z;l=Xxg8qD+E}8HW^&htqLtQONQLn>RyesSuHjJ`m4{l@;=w&V0h;b_N2-^?mZ9`X|
zJgGfL?`6`-ZG2PE&M5i}>Kk;hyIW#SqUO_Jptd1VN0qVD6RNEVf>Nld5;7r(V&jzk
zxfbZcx8}EnFpQV_%lo!x`)>(YJ~HPRFt=w9X~vw+lupwI2;!Z$WJ2?4`kEx%qYoqF
z6i?3r(4cR3)>%hQysEu9>4HZON#Y>6%a$Tp$X94l5z7EkVbsB8ztrTSx?Gey2D-K8
z7hsglyKz3L)XM^UNE#)oJil3}^dus^T{st>!Vnsj7U{%j{cAW{+wVp%C+`GZ!D}bc
z#o%`0KCEeUV>>oyM5D(PYkO
zaSfRDlc2cwoiI(eb~ZP5zWer2RtaTGVqPV|3-rJQq@_e=zu#9PJ9&3>#Vq<`Dbm;j
zb*?FCLrUq2FC9g4Q-}44cyIqGeA`VndUTD~5;?$hdrB&?q@#vbn+wVkM1*NqQXH)I
zTuJR#!(L3Nu8e)#_eu1I5AQ?D{aptsnG4Tc
zJWVA?JUd4Fr<;0N&I-Wcz~s!u_?^QY|BV8>Tct|+ar)20b3)%#c$TTxi^azp2+(i8
z(XUfk<;`6*HfR=ucyW1UMcIDt*rXhtgc9G*YkV2TL5j9}4CmZ*Z9?}S-F+g@-*0LR&^;Y4
zQUm}pN9YoIc@?C!MykUQ9%_
zK~>DPHeDO$-bt=JuQu7QllQ66dhbqedDMhqv2{Ako;ASL5G{O^%pyqduxR-7UFapNOSz}*EYm_LTn|2XA8C9Zo%G)52${%PIGH(?-6NT*5A*H0wB2154
z7WSX*HtZTH-!B}gaMxJ6|5HY)BE7_TOiZ?BYugSj3PNf1WKow*!(KLeGnXz#_ukXa
zZ8=O()9ZmvKWo(%R%wW$oK!@*k}w%mT{^QalB2>&uqR^K4b|-as
zL^Qn`RTi2|oMk7fv{|;<))_)%G=(L`GFXM=2f7=7U7jtT(AW2-iBR4ensKOKO+Ayu
zMx-@Z=p)VJVPp9n%4c0Si(*(vF;>2@$8&R#tvLi?RsaegIb`Ic6<_@zXLDkuc>GjooXi%(nri}D3nsM%y?6XI+1N?66$%t*lmfli@AVF*vW9#RNdNIg3$IdZ#a`KZ1
zTh-}pfVQ3>9=mTb|g{swH>8=4H2
zT|cc*1LJp?j3Lk`%*&hhkJOX54pN_=6peCJ1I@jr-z?{EH_+GRjzCQCLF2BM!H$`l8gu2ri
z4#|q|d|rC*Ja5uxnVp=sl02}qn8!|{6leJ}+0_$y^E&sZC&+~Ni`N6If7ZtOs`Z9H
zS@`nW9&yI_?IrJ=XJ;JSTcf*j-d(fjHW)QHiqSFRPKbQ5a>=duYhtBW1Mq`t2AKNH
z@Zmp$?m|dc*$t}+-&zR^(peRR=|{RhI0qjR2a}@|y1|3bTlICou1IZ|2IaipmR?QD
z41XN)`Vt?f8~ddFa5az9cx#pDO~$1&aku%Pbzk~rnqoQw)=9}x@q=^Tow^HtBBgJ(
zW0Sa?DT#F_z3)WK*riyzBnl9e3jI(jMb)2qS-tpv%sS%%Dl_PIxl1P5N~|0m&m;Yc
zl~-fE4vSM_ggu-OmnV?TEG(KTgO!KGB(hS{=Sw$+f@Za`pee`2Kh9Ub3y5scQ%Glr
zN;7xQdo;~2{2%t-Gb*ZVT^qF(0}@nF5lPwzNJbD4ilm4LNKO)p(BvG79F?SkfP&;C
z0+Mrv0t5s^k^)7h0FfN3h(+ekMPavRpMCcI?ik;=uCj-b?7XhE+ZQ*WZ0v-ogsnk%n>hmRKKmu4^nithB?^ij6=fYL>c4F0L@!q-Q-KFU6wmFeO)yD&8HdFcv5n9K>jjzu<1
zKz1BlDTCWOscE#)YM-QJn}o~T#!c7hm79Zp5T~A2OIs{N{BF2N2hZC)8dyaSHGRFP93Z?pISBY
zSBRzb2wS`Ln5V#77nLoS$6@iS7NB23=hH$*IhwtE=F(w0!oPwL$Li)a%!;
z_X4VeTt1soopbpT9U9In;di`*O(F$nfZcOHEmbdZFh;KVT<1hbxwE?kceQuS%wtn5
zWRoKzCqafVYjb%R<3Doi8&gsNh%n^J?JY;&yD!C5C@&(M3R09Q7qg_#$4GrM`?2Jl
zEq|djpwjSQ#tE|=eC6)z^IHs*trd#NtOy$W4iMl}PEYDT(5ln7Fl^#G^TgKIt>KZc^=Pq?%;we9SF>^5(ya%;K
z9;Ydb`{I6$dySr^t(ToBl@x-%i`Rl>q7LLzRkk#pYr;jpIFr3!7+7(i-zp?W(|XuX
z^`Jd((5>-{9~4vaq3Wn=IOa129p``2cyD3#yL8T~S^37=EldMc{s~Z-u>6dn&Z5rS
z-DI(-)c0ja$d)f-JHV0SOIZ`7pKP%CY@4jXi*J`}SrAwG#^|V?V
zvL$D+f=_Pmy6#5G&ZllJ5w!>x?`WX75A`vklKxLR^fU+1VF;N7_ko8g%`d@abPOeG
zjm1gYCbbHWIE$s=#-jA=%ggqb2bXn=7j$gBL~An2`OC*KGPWRWeXQHAAtQEb*3i39
z1glWy4$gY*Ckz$bXk*&eP2S~rRK)#?VdexL!y{K943A2uL9BA!if4`gQ|0G5g444u
zEs0QH5~#2{W86?x;t&5$JRV)o!*lJ{L=xl}`n9*)r5X(QB8({<-}bc3>b)bdyp@yE
zXfS<^OfOtPvmiW9gz=iw`G6722S#M6acc{e$rf)G{Bj$K_f9W;Ht)@d>wU(m1#|O1
zjroaDG>E<_9``23594LmRv3BX2eQYJYjQ~&$Vl~?6-#3h5AS0>OUEep!*R)LpG~%+%wN3ztouTvW0I^QPNlmw_5+dEAoo
zHl-!gpe4lp=B!?YbcoaujLwAL`dJDB`x1yZ29X?EdCkE!r3H@=`yg6XYPv)gP-W9>
zX@iF3a+9M4Pq~iLC#e@w(VZPfa;8hhyjHcmpcy481Lfk%gwT_OJcg
zh(2bFI+M0jwaBkB5xUR>-TihFK`
z(@b}Mt;3ydK%<+Qev$Uou=5Z4U1@Ej&?gB(nYTw+X5w~r4-ne8##G+nJEaZ;XE;!E*+;_yFh56RYjoQ>4w>W8Peb%lpZ4*-u@Y|D+
zX@*zESEtxi*%=t$A76U@
z$9_bfZ=!cE@<;RASSO5ENI8=XVcKBlC~~vPk<;by&LY2fXu0!P?a8-GUg6}ar)G9{
zHTOH@1$G%WKaR@~DE+C>F`hvAuxM%7*okBqJCgO_b_a3$xdgwkGjPv{epQ*1YWXz>
zkCJ=fDtdCAJ@v2-&YQJuoYfCYZN|_r?@1X>iUtcs1Ru;?xT-)R!TX@H_L-6rSJ&b-
z#u_xt{|;1*Y&79iV+;c_bBHxZDV-s`N!xZmU-jp1QaF+7v~*=Fcn^*e%a5fNL3
zjoQT@52DIbpM?{|JwkKK(cIg5)pC37aAkYO)!;NqieUPYLaDFT6MNzVAFWE9=$INP
zGyo3sjL?PchyK|O)}n~fqN~DrW`N1kTgQ2$H@xiuh{>M1USk5={(?r1?;&dT*TEaO
zBEm?`S1(m(gg8$N-
zy=E?N0Q+A&_%rr92x!85S_%0c+0kG=W9nUr&i_$RAoY&iEiBIg#C*8|1mrDhSy!({
z70h*%hmZ*CR*A@2)yj;zA6jmJeEMr*bnCdWZ3i;CHuTohtBx1GtSGwQ13|`_<|Q6N
z8j&3=^0O!7u0d-z6XaF~mxKw6m~5_p5hfG*Zmb(SGkejn0HN27j<~Zf=u30>6Bbt!
zwN9lpcNk)Oq~#*eH(%rPGhzV*+U^Cd7d;u%!L)&JBZKb$wC|^&eZS2u^APMi`kBEJ|d!?Hh*>!AeB>my22h_Klq&mU&Bv{wDKWJOka_ImU$a4-NJ^f}kR#
zIv5BaeL1}F3BKm#351V|z)zvMyG%)!8m$$M9GMIcTvOBwav*&MD%VKHHk#&r*1Cd7
z_O0A99rp^h1H(Pe@aj-jo*^VHb=ur@>gax(qgCFQXGiAJLo>8l0
zUH8Yr$!pY~n;$=jrsA*pk-!3P_4259l|Z~LUG&&@J{{U;(8TGsFI66`UG6y2SCf_?
zn^JM-rw1XZ*N~E>%CF7h@?N4UWu$~v)NNE5+RSrBqeZ)A<#lC3!sXnH=;W72DotaD
z;I%Kat)9Y-_T7>e-0!w(_`EFYPi;#uW9H??#H7Z)tKyt;sVo~AR@{~ByUk15DZAIO
zsc9QhB>5RFq`O(yKaszZHL!0}j8+Pgu%a|V}+1{`_+
zTH^?r(D={BL93JjJ@o%`pertFd57z$0dBO-^ESmuZ7$Keb!U!Fq8Ck#LU
zl{E-EUFts`_`v|>?Na?vgJu_zh_Vv4BRbQ6$=Ch*D>oETojh&0bMD4C5_BNV$h4n4
zdT$ocQIh8qO$?d;z4t%ivp)Ldc2al~tNSSHqp1x~I9m@>XKkzcPg$P@ko76jxjzlE
zK6wCph?<6+BdUw{Vc;KFGFy)pvUD21kR{wj_2Y%`9xdcd-}hJGmr4rQ{!?g}8_*7N
ztT7n=0TxmL7Scl0zIyadEMR0u-iLT%h>W*a`*<
z!jQ)~bfubNAJpIbKjAA6m>Y<+uwNq)@HQY
zxtj|($6~YUwf_T(PCT~(TUbc-`;UCO0`?|PCCm|gz0UFG>o{{SKAQ|K6PT->klON&qHje&50IK!8(qmR3Q4tbF{_Ir3pK$NVK?EwD|oSJ0jkOxpgw
zr~eAg_m992nGZ3^%V0V~`m0@j1MkN3h|~A>`;Bp1LZP
zQ-gRkDX!cnq_z@(ihk9%uCLeTib2u
zhbrxl>qU;`4xTPGkSj?VsIu&LwXn|4}>hpNs_&ic}
zRcN!WtN)+P68k@!g%?Ku4P*ZYS^w`}L*@4ilv_?$_L@vQxr9$jmr;1e%B+QwNglU#bth4rH$|sGCPg0
zKPwF=E=_Ypt{bRmfk?dFmKpQ#6d+;=M;ewVeJzT~Wya%`hF&0$Q)wyJeh)_iHM~lT
zoD?XDtY+WQ6islsM?aNOj
zfxk`0L8#=WCiV_Qgnc9i!5<&M+h0OXF53cGoDk<|q}qg+wjuEU(Nc5)+NQhk60~a4
z_G5y7O<;%6yQsJAUv&g2IJ1nCh`+y~25=QU{lXZ`0M0tG_v(lR>n5!gwPX-
z@o1{4kF?TrY9gqVXi}-q5Ol$sH?&J!#1i<_S08SU9Z+%eUfjw*+~FKk4e?kvC&~ND
zaJ-!Ni6;+RLI`Wy$f0-X+Rw<}@0676;Uby-M
z^xjVb>fA%L0_g$>*!D7Fs&+$c-3>*Ul%*psJ3*FN871^FhCHha3`n
z-ES2r^4w1;@G5nnTx_*ucdb;-
z$9X%;IZ=J#sIqJo8HXQbCAJLRxmM5n*~DOlUq2CtKD+sesB2@8lmIh7*?GLD8R-B^
zbs}BK8-u~We-41y4?zB39jG@<;8hOH$&qb6%U%7Ngy^Ykw5v|;k!9AlW^KH51dB=v
z=*r&MYm9aDN_$p6%V48jqLlCLL#sZZRF
zJ$LQsfS3M%bifL26p3dq(B^n;HPU7AbP4hB@NUhfg!a-MZb_DU?e7G3g3b;J0ZM!b
z8Jkc0%#7s&3y(htf7fjGx_T5iQ*eqw$I+EEoA}&tj0);|8Ck(RRsb=leH1~k0`$Q
z7{rBU_`!HYq{+QM;{d9PH296-z~h3N##lWs*d9N;^QcPaiM~?RMc|#X6ZhCFy&vKf
zk{Phx-CXJ%*jO4a|8oKc3|vdXt9TBuXQkk7h^m$*iBFmDox0dG3|PZA*xQch2t(KL
z4*0qO+TVnsA+UTC>YTt!R7b_QLIEVF$2!LmDIF({h2ucZQGG(7_HX-RUWa8%u4Z$t
zB3iyW=fX5mUFLNTU8?%#S4`hY+A
zZ>c>1)WsU?60&%hpE^Pq9K~QZB@c%@UMNSDU$}Q1K4OBxrgk50GwTq5!GshqG6k3v
zIw%rf5L14*%Aq|QKbR^N-dp9kFjT_f<81X~(h!diZb(q;mYIvZ_!_Zw_C|vZIcV41
z+t&L#>MAx5>G*%?q_hC<;l<+P+^6;86b}?mFgR^cWd!sVWH1z!pJY1bxbTF8W|8h&
zhkS72RSS)aBV-xVL|w(k9ZFD=MN3aQ^EA=bj2E3~AL4(xh!b+tHd(`#uYk~{vuAXI)=8Adgll5}NtR?Nt%P1n2#qqG=epwj%`Ty}
zC#YJ_-4CwIgT<1(yu77m2Lza*&n>O^6EakSfMSXAAe}m8#pL#@7J%;A%UTKW{i`#+
z|Fkk^LKa9>7}w2T7ZV7{97yYBoK5``w{41igD2Ekzvk(u*z$AXHKc7N^nL4V7
zQ3g~(k{fgQJY!M_rAQ=1U#Y`-{7g|Fe0vPrqe!Lqb63(}G`4i_><#qxY{sNBZQDNu
zbr}I*t>zir$HVK6ey&l!`!f3&ROca3DLtRZ
z-a3aC(AJKO6#PTSC4@tYUDpMaVpeJS9)m&w5`11tiQ+vg)Y90zALW9l{_ZfXgeGVQ
zaTi13$76F)5;_Ya8EohG;Lf7U^)SBs|0`D>`fBtA0!)Sa@q><{zk9JlFFsB}b`Jx_
zzWHy1dzBts712`TwBwBemwXwz-u>^!6Wrb7uattVH(>fHfxp~SMl(fn`1gANUGC&B
zC_vq$4hK|-pDvI4&&=+FNx<%4P~Oouvm9Amtq4=WL~%m2>MDoH@gz?Wh7$#FSXH)A
z`==&-{74eX5~3^1TQn5GfH0WTga0ot_e%p&m#)W(f4-^v=AxpktgI?0P;*)2h~w4U
zGeBZ&nGIMX7QTI}(9YLnwLAW8aauwar_+8k=rcPOtzt)`(-!2h9Ah}&wrt?g*gI;(
zF&|@o4F6;Q4)bsPA*)ABFlyNL3tZ
zUS04p1}BscG;d;dQhq63^Li}ls^N9yGa%m2QhohR^WtGv>M^nXMKZtW&adx=5PooS
zjXE}DMnMoQaM0L@&kg`YJZFIdRb*_9e`6k$szT{9FGCRZ9&ph9zyVM(-_R+sa09>S
z`GQv!xA6O8b_p{6*z@u_ea92BBaGl?M}W%$@^Ui7=vX}+Ysp_ock)(_K6cl@H_3x<
z+UT7b`nBD_$}fRT@3GKv`0ysVNjelq
zv5UsVS;y;4)7@5*I-@^R4)e`5A*kYWgt(Fjiijp~Fv6!$-oL-#Bg+_N}5
zy&I^e`G4h}J!eB=J#eGWyMdeAGfCNw3w^3JZ~hK_gKhc}0WH6>7)!m=M4`LISYy&f
zi|XxF+3?hAn^6F%6;39KUy_X=9`J3=7;0!-{s;YXkS$IGY;h(k(HCvET%sC3%s8~-
zZSi!f7(guu`DHhtOojq#9>}HaJith&ciwQqf5uHUCz4Z$ZhIEx#vZY7UNi9eFQ}K>h16W?MT!>2qLJ)TLAW&qL;z9B5X?+1jB;-&yVbBRK
zdDM`9v34D)wYSb#YPn%k;}-ATM`=DK-l0}%xAgw4b9sg{j&41Y*J-P(*DvlmPvvK4
zk$dkS$tJ2t@4kRVn%L7k5v)F`u_Zep(z<+*^|_r8y?y1MlV@1AZvGp4t568Uo7bf2
zA4#wb`78Zl;|VbLxQH!q84KxGLE;rMHdc!e8ccYU-aNwI&;N=&6Cda0`aydLFNzP3)N(?!_02iUy%;TtGGnKr|Dp)2|OaB1Y>31Aef99nW6
z4>$%o@o22C)^$dF&JG`h_yCEBOZC;C*&F}B(HT5Zc^4fDh}5gsDe3eS#0cwcrSP|jP7#WtIp=VP!F@nilwdNx<-kJGAbQZIn2bU`x0v5@l
z`P{rBFasl|1n$y6c>CdPrzX1Zx6?7-$5CsNh=P|2m9(RhG6o!Z$Ar~9OT6=)+q+(j
zWd)4g6$~fc6?Ku9jT5Tn!gat7-n2S$&8-tv*mtGs#!`YV7Km;ZrJQ7OoO^?|?;9Va
zTttC$F)SaSpacE_HGgM=bj1ZCas%C^A}RWs4V#$b&0TjV?@JIdc-
zO*OARZ5_M3R$Bs3a_$PH%aU8h^1n^Lh%81qtla|_wQBW9r}{Xr)bSHwG(EKmFM->?
zR;Sxj!J@t#b9acF2}}VA1y<~OuTRXeNdfJGLKeVK-L?8^CFhGRCLjd|S7T`s8fL-6
zd#H;39vxoE5g$eQCX@{SngtDt|NP9WKP&LA&pT>CCY!MgBaHks+dZ*>JggX+i14Fy9qn6a;R`t
zWEVNRt1C;ZQOxnyiIw^AHiC1=&$+Em$+0u%r;W5gJ*+0HLij#5Pq4J>YX*dsA4gaz
zB}Ug&Eg6*>w6Oa7cO}&I&TsB7_V~`P7!eV*Y|bIlsy>XVs?C&byk|R&`6x$AQ9Xij
zoJ|G&bc-~IK8=|yx&+TW-cZ(?AK1=de0Im!G^?GKRHJ$T0Y`74IFsY_;Hc^g(p=u@
zaJGvk*Bje%}B$VQv0KFGmWQ9&DbV;G3?y$xUr
zEbX^1ra;02_P-=NK(KDz%U0M0H~zx}6&sv?ckM+*_A7pWKAckM5r(N|!My*=RF<=u``0t4s3?p>$WPs;lHL@ctO8mB2N1
z!rf6}CORyBSo2Y4ZdcwZy8GB=t?JZWCBXpNd+U`ut6%D}ytl>;Ho8NEMYb}=2z{KV
zF!B_gw?Lfa&#vS-7t&lKP`z2$I?s>U6919wxwmzP;dzlL&CX|-nhz&}PBZKJ^&zH2
ztyiatE+eT1W|$b*E8*h4J7!%nz0r2zWWsR5OGbBGDB=-HZyEFiel&ldU#(SVq|roS
zLTUGvJRN^NUho{ATK_DphH8CCtdo^XOkeApSFJ-*HN7*yTy5Xgo5G<@4rXpc9ty1V
zFK8&qkxT@C5P(hImH;etp%tL$q!abEDDOS~QX7Qg@BbT+ZUwD!Tfx^?k?fc<|Z*%PhdpJy7i
zc~6o()>G>?Ij9M7lAr3l(`a0l4D)oHb8}K^bY|s`5ezkSa=PWBu8EbCrXWW7JCZTD
zsL#mi%z&)I8pShUU6E+C&3wCi7DKCx6tf!IS`szH4s@&XUbtigNTRMd>_AeYSo|z%
zbBQxMbQ8mF`GC+&xXdzQ?dG2ppgTPM@!UD)4I#U|ilI7;Fpw{8^Ks-%=N%&yrlJeO
zd#~*0Wo>((G6qQQ^H*MunPdvd+}}m3!^-M%4yLmEIJ{QhVn=@%Ws8G0I+RBnRPB`E
z3xu%Xd&F@C7v2|!MjBs`!sb1o3es1C3=l_fAgZ7T1foS(9lA}7x4XoyLVk6tk89-2
zV=PA-%3xXJ$lwn|t@_Vsp2pPP+sQY_4rusy>dYiL254?8AV$lql(2HsyHjty*QWbR
z+nB9L@5M}(+Tcb%hZr-yrhKe<$2+iUd@=8pwbVk-sS;k7{z%6r_6BP85E)HG=_v`+
z(-}x!(C|=DT#IcXQKipY3V%9WMtx(V-+-xhCE&Ir-H^6=!?Uxbp35{tm6N*J;jq2n
z4m0!}(6~DnJDArKU)_;N#L>Q)So{Ywu67ZYqd{{pKUM?Ug}+)Xcy_C$RFsw*Vf=zw
zEl^3)djZ3}>`N`c;8K4%^UVP`YrHPF`LZaK&vqT#jZxn1gQ^!VI)$>U&0S43%)QOe
zY;H^GTl3L0^VMQ=c64Rv%D9~v9{$wCa#T<I8x@6L
z2QOSGZCA=)8i;op12)3{c3A=`MjL{DA7`?RYrK^bKsI-wKfpXOb=f4LT_xv>g)6o+
z9I^7bGrXH~nb&vW*WAxq4VB4Q4HXWig()Qv6@zGqF7wkTei=qf6GPQ_bis8);J%UJolC;jtQJNmp3V!6QQ2hsO4
z3*H;84J>#*S0%hu#^#?{+`uv^6hP_byc9uKM&$dkzlFJm6h!&L;P6`8o)zk7ZhNCDh_@olg5!$+*;%
zS_t1t1QER!$8Ad6+)4OFyN2kAdHE#I
zAN$)(!W?Qm#@@TydbA7DFEirbtHSvb?Y`SjBTf3L=A(G(_Rp%Nrc5Yj^Y=M0Pbb=e
zI^mboLmy67=8f?1j^5P;jf@icX&0`&8&qMqtti?PD?4qhEnh2HJ6rAJi&DC&-;~`
z2Kn;D((`M#E;_Wdw~)0&OnURW8h)=xBM5BYA2>ExFZ&1h=zp*9xllKKK70JFK)`G7
z3Epyw1C?BB=WbO@?KUeJ+-kvSIqHHln>v|8l{fn9gJ)drQq4Md@&z11>}H6sKD)x!ij2WOv3xgc?w-sL{!O{JCh=1brVvPhAZyC(9K!_fvX^kZR*&v@zL)o
zx>>``Aqq?(bhu~pqedLy;MVr%zg)7(6P>lMGB-6H`dZ(;X=e4hLc(AV;N-G@5QV1OAQpW|gQmbLU4G@Tl
zf@eZL9_UMR5YQ}MDfMZh=V5x+%ge$1nCxL0NnLb{l<7m~$?%n1uN^hI5XfXv5n%p}
zCVJMkeLg*iM4X(sOGvf3JP3D$En#}4mTI#PF!F~|Kv2ohh^{1|E68fK>MS1_nx^%F
zE}`Tzp$V$zv;emOJ@
z?8R1VfL9g%?b>&{-PHMJi|q|IF$96W3Dj1Y@B;TyB-2zZ=mR;_5d0bgzv#PgxBu3z
z<#N%?P>FN#(;%}**=*S8Av
zV%rftK^CYGE6(iv5ZcWIris)E`bXKc6go=XP8!ALuCp%PnK|-x*(#H~;^Z{lSCK&o
z^Zsw|y@uAvc=wvMWJoX1**Rl&5;)&5j7v>Ml9hxlutno!9DjVilOHL%^S#o1OEx8F
zUNhHmo+yHQX8fo1^}YCE_~~>
z$PZnr<^#dyj0eUM>-4SlpcGpgxbm57MB#s@F+eqgGzgw0fyt7nvWfS^AokX#_dVKP
z<2YB7WA#Z2FV-t=0#`qnN>vKO0+NX3eC_SR&)I6Q9^Fz*nHt5$nK#7)o>KfZTIFco
z{F-upA-}FnM3U|)5arpFNTM^|x7LeIH?0_4-(43ggf7<{p_IamdPhp&Nd+BTl`HG{
z`>X;YEbUhpKU^7BTv~SZzcbn30k|}L5+)lSm>d6L>FIkvO>S%#GYNRmepB6?$p847B`d0TBV;Zpce6~pXN1#@RNp|RW}3sbkK*&|;j?@jM0tZk
zk|(`eLJnV0?1YcSn33m&=Dwm-2pUS!w|GfpIe4&n3&k;ONktZ}xk)FI%jFU6%-mVk
zw>o_HL#BL&2^aQipN4{bhNe4{OWh-XPot&suBR&Jgw*L5&6S7UXu-(8{G_xQ1z`lm
z;U6#Wp2iq=nQlI~xw>s#?UZG7z`YP+lNO+D39E@|+4tDWozFpZZZtgA*S~>yA;UGh
z%-lonZGY?5+PA&IhJ>=RNrPkyigMWH*|31Pb6(^b#mzJaYAj2v_vb(fn
z|E|^QR>OpT!a_sy`98zZp!};#B*X5HHowK1L}tr!GCGrl4E0Y_gNeR2k$w{VwOjF;
zYwhHPxL}9cIGDHDp9xEep~FnCC?-JP?3dTTJ7|9w`0Dp82;?2eKL9cP&#D00d1{f4
zr;kgp-2X!fR?%t$qAO7t6)rE5dCNn6Uh;u+`G|cGnL>F(=luxxjc*&Bd9IwrGtu{QwOMkTU=hNb1
z?d}RYBB}G2Py3qMC2U#Z6r}ha$Mt%8KFX!(v^hk)`3a&&kDOHAx$y?aXv`{5^jET7
zOAI7(irv+krla7H&C(s%@tyf{N9w+^0OPfDa`USuebSww=Pgz3#|F?*AKR9Au4eZ;
zV>S+M;gp_67hg81G#M&Cl=~~A1XD0QVExW5ByBl?D8I6%09jGfW3$!UfYDKU(V@z@
zsK74M?>0HK7@>rzE}@eL`MKmAp%-6s;0I@Kbdw>&$__-n%pAn9GdKnplT^;G
zv=O4ET9}K;Bm4^8)&Wt5^X#|#8!v=ycKBSi-^h{@2lU*syglhCutbddmRBRVK6n3@
z6N$8A@W2feslJ7$0#PV+HU}6oxBcvBwW?Sk<%vmz+qg{JTU!gKrO4)hYr-bJIyw%>
z)2)fNjAo`TsEtTgUFL{Blo`8eOd>}u`Vq8orhJ=NS`|5k8tpg;`BHY158?gUUo1^L
zg_0~BC{ej3`vBm?dgdcWl0+<)KHo+}kcOmrSHEkHWLX0u54h$UXUgb;erS>j1j;t9>2I02#S^cCm54tkZC&+P|0FQJW@}PqoQ=j3f8hXLt&Tlg(gYW!%NCFq$>6D+8Ipk>Afcct+tvr?;+%jPe}UM
zSLr(yg6ss9kdxZY`0&dWcW{pc;Mc9p4Xe}zUDQJ?u<;o+7M;V*&3t-KrEfghJ=68J
zV6Y6u{`L=X&qT`JA`Nk_XjAekD+^hxWtke`#+!nI@EgS
zb6oi;{2z*O&AcZny$=s`(yOICQ`biOaxh&sIDyN*%@74Ug
ziX67Hx~0lPH<<2whMVqyustnbt#Q;sjCnH&>d3WdID{O;Kfqqn;#(2equ7;1MP8>l)L43PS)We%1tV|cx
z86qgc8-2?mVl|GZW~P
zS}Ua~zV6;a-LqMz-j4WCMZuUWkUa+Cbm|;SUCG?TLZH@dgA^_Ge!k3(KG1rhB1BHV
z?vJvY6rR?VfWSHb@r~Q_^<5xh1iA*Pc*gXUhXF|Zi#Q4|XwA>pZmr5--i5;`4#z^JX;`g}XAP7raZTf!5yeQL
zs;tbLa1Kt;a291`^4*c_@fuNAdc1w$P_1XR;9P1r%g42ie%)+PLYCDfYD4m8lV=ev
zlBfS;qglPRotAn>>3JJf`kb37dv)GkbSa1@ZCq`B8hmTE2wzhBN%ZjSbSqGli)f
z4pO)$DqAf%ul#oTghY7k#$w=wKdZQLm-;tV8}llKRz^}?=P|(0wzv)}zSzvtghEO(
zlvL^uwttu!Sfp6HHN4x-hcpU-cM8;BoC9fJtq3-kT)o+{k+F9fL2;dG|y5niuP;J@%l~T(&pN1ONmTf_vQWR^LYbu}mvZ@`MZ
zfI+4|3Z})-)c$zIFyBHEZ{ltq7n*H^m3sg`OX~4KLK5@qYe$+%gUrtSqFIj!>1|`*1Cnx8<6?uuZ4FAl-;Krp7bHV>q}d|J>ZaeUJDy$!sqJE!S6yrt?XkU
zlZa03v)tR1Sjgzq;nTksXVB0Wae4I)d)#pZyn^ln>AC92!AiN#@J@ZQlEUrB1*SiL
zR@R)Wh3;
z32~R7S+xz+FGK(vQ_KvLIEs4tQDpunsW*AG#8Kc;ZI8>74L95(j0zKH^wmqYqh
z!P4{S*7%(ZNovtA9Mnp;k!$rW(k~(i(KVLh(~EZiUORe3PbhYck6x&X&pU9KCASHc
zd1kZ~oI-S;WIIl_LVWB7|fO0Dr1r(>e*Lyq2Z*Z@tf~`tY5h~OpTmS
z;T(S5-&vQ*iM=}#%I_8`Ybhl()^hSvU!_h7tA|kz^C_6PYYRVF=k+aXv?j(FyUzCb
z>PU(eD;p{?3B@6LBtfkE$F;uQEIg@^1*Cy
zE0XGG-~!pzMcRSB6Hf+8AIX|ApFelH`f@W$%10PinuD!!J1zP{V50dE!ze-9_ASR#
zY$i`U)ic^T}@VYZ&?VdEK4Kr2h6tJ4R8~xL~S}krUcI{oBzwZwGvYTc%CnI|Yhn2*35hQW4&;`%_@$Tqqns9*+w;}xtH(7hwS8MkHk{MrE&A4|1<~6Z
z0ADP{?Ra!1RMI=#V;^9*p2`YN9!>SZ70KRc5^8}E5-(8ZAV+22am8Qs^eI1MTvf(J
zxUHx0%c=I_s_e)Yfx14A&v$*fw)&RTcNbTa4uWGEo`@e0CzPUaHA
zIptouP2^kx9dAoPz3gPcvXOCmZrK@^4-ce_Wp9lzC07Ro{D`{j<5=tJVo29z=utY
zH~7zLTG%pT$AD_}b#=aUOz7fU%Hrv!GwfE&lyMebTlKp-sQMplYB%@`yjC
zTiO-R)aq%S942cA9I`P_$!y~UUBj;TxONA0GG(jV?>;*&B%HY;Ctfq
zH28A3FJCoABYLoOxSx!ek(>X1nmtRaoF~Wp3AQsZGu8L=`lggJvT6^|P)oR{rPlvN
zs)Jl7EL~rjQk&?MhATJYnKdFtXkJ646rfF}`6wxQ&3pQz3_e9ddG+xEQ5RW!xUKep
z2?#-CWo9xf3NQPc5pGD?axRB&jE|-XcPa)MkExYBB-E1XuZH5kxCGg=MdqhHzml{L
zoWVHUX0R*1cGf|7<1(*R1BpcXEytn2sz(L%y*p_T-VaZMoV{Jd6Z#|Ev*
zP?73;zq{0WA*+(%MFgL<`!EGZ_sgOebcq7VsVhIV=ct`p+ME0}2mBPQa7s#IZ06K>
zq_>pUbXn%n#5)Vj7~XL)>6zZw>#KM8*sADma_
zO^RW$geANB?dbcFBN6qyU8(h|*Upy^Uo|
zXJAyiK>J*cy?mLQ{cxrNR%5GGnN(mAldxc@o!L6FEtAV}GvQM#Q|QL&YYfkDJ&92n
zBA)prc~&D$VIEONrd`Z|!MivNxeZEOx*@WU
zecrzUe)GejZo##-3ThP5v8An)is^Q9CIYf7)pgtvrklgOs4MzWXOQq8paEzM$~dedDMZq5)j+5
z45@oezcsKP=KFXL4oN*%TNC~g_mkQ!V;^G@C^cGSOU%Yu=2l`*7cK2S9N77Nr+;kmP}nB;xn`e#^tok0sJFk$S{
zPYN8Si|(5(UemaMUUFOc{^drPpTIImaupn;hKNUyTFep+pYieW=N6Z(Up7nYum4VUCD#3uSE78KOG)c4
zB3WxEfe8_K9v!2;y*24OKdserRj^&S=)D9Sy-S99Q!oxsm!m;VH0B<rjHoF=U&Mj;
z<}!h1b`|r)YH)*H#ok>7>0@4^7NypS
zYr}OD8@tQto)+Gn5Yx&RsD)bFTFX2f^rKuCK1HE$BZyPTLQf7ja!JEBb?}dJ3Fb)U
zo{zHO#a$w>lA1FZmeX6HYnqi}+V&3e4`SgM%kO)XM(`Rdc`9dz*;lscpp=H|)>=JI
z=vv=D!p8~i`_UeuBQ@i8Gi1V^bDawxul38M1t>mGa?9f-Y}9s{x-gYF<(=UM*D(-g
zx7hZf+xaNyF>*TXL|iLcP1EElw$pv}FRt0ESGIEQMxMbMkJyJ|2#}gSh!puaCBgl|
zNUTn!K%&F)U5#yXcd-Ey+StoKoQ)4MN*IMbS4_&C_G)=DqcoQ#d$4M0yLaZ@CG!(-
zpU7eaW+uZOpX7aC_^|A0rQGH$UFn(GH)!bH*r#Z{Z_uo(28O83qwkJZoA(?3nA^AB
zJ5UY(nep~)jMC*IntIq_naPTPipkRw(Gzctv2yaEv4WmzFEFP_PRwOGTYvielc304
zoa5tj!5*i2iRd$!ksFG
zuBp_oDQ-`Zq+tr>G&^kjcJBa*z
zCKrm4Vf*I~gez@!RFyrFIdKl!_6=aXUAgh)B1r7-x_>7Ar!wr~Z4i>}>FW1YNI=6q&4Ilt%=&eH!Mh
zuWr6gjf>r=tw=*2?p+q?Coxl7>yn#Vez0zUhrA!NY;U*IV
zHwO%+Bt>@=O?UG>mn@1b2aAe42Wdf`j<=KQ&?v^Cc|hTU&C|wP?Z#w%-=2d_Vwxi9
zo9psTju>ZF3>JKz*$1W_Kg=j7h>^O+cukEP-Y+!+XJA#&cRW{f^Im^1)q3E0y+@-i
z^|9UK12kZgOTxFQQ3MUn!xDOwH
z?4fU28Yyw2>z`nFda#k@7|lFFH2KGAot}b=X1Qa`{}+4j9o1ylwGY}r0V7xd0TC1h
zk)~7y11JJ2O{7T&rB|hej(~tl6A_S-pd!6WF9F0tmEK!GdQIq&nmKoZkG$XSduP4B
zSu<;9t@-|VTs#oC&%Mt+dtbYpeT@qzX_+R#j#1F~li||fcbgno(saY`&Z{#06n4xl
zfRwMvWbu0b3$K0W8;1uysKwQVU##?e3>e?&)rfkC=cus?JFdB$T^je)fJ+=sEMz=T-!_g7tt?)#rhD%PDNXO45d+qJx+E{AtEd
zu2zfR_z7A4M$1VGi4Azh{Oo}n2+xASyKj{ekFBnh5@PHhr{3+E4r6R88%9$k=EV9}
zWW|Z+&2fe+7mOS;>1(pi{w0B%_GK&S!5c$xYRcj#Y_&@T&OqC`=s-t
zW$bKvqTa`~nbPR?d){qYxWF{3cq)K`=>g)k`>Y_p*EZjWY%`q?g2j#d;qM&otsi_y
z79B7^oZn$;_?XbAbJ+}ewog38cbJl|C8rMx5ze|SrRvws&UQ7FI?j8|S@oQYU)FV-
zoVUL<`xG;XulET!8yUgXPjKVkKI1s{i+_hnpU;i4KiX7jrEzhQ1LZU^7BUa_ZnT~a
z$-c=613k(EHC{*MYRasZq}wX>D|3e!UG4f$D($3w(-A@o|H@a1^YZhUp)gT9X|wEq
zwo+uijNfK%(byLuQT#MAXb1nfiPJ@exBQMGiys#Etx3$)uUX9eM*5RyTbnf}<8WI;IpKr>H;cKb
z%CWhZU9CL!wup(=zP<^aWDlfgUn>9Fb3SQa9xGlg-%-VDde!l3j;5K0EI1D)C)={+
z8%r1~==OC%2Zjy!PpL3qI}&ZtLs2%%z@X*Oh3!iJ#Sud9{Q4
z8l^P-TlOLwy;aYbLWqHg!RAIIDG<+5{!)0Di3xv6&Jx^;P8>AgEu=D9^qag_RhFVM
zA^LUY(lx}L2GttjQ(QauTUh`=4
zM3whn6;HGaKdx5dM2@I)-hbG*t!uf^WI2Ec)iqum6c9UVdBUFpMQ56uW#G&)ce~hN
zAtO>{bbyTPo5c86Z;Z->e3!C$d|Kv*oCf`OBgQoAZXW#q1w0y!>k33=q
z&S2d;#^|_9^eX}|ce3o|t}kJ1$KJ5W$Eq$mb!=|*P(1(Y@tw8h8?=nrXj)@Toz&R6|i`N)2Zw#2yNH;4Yd
z6X<2~pn^3=uZ!lTeSkJGZi1BZ4ovRc$5|!E8mG&#--|J}~U5QJ4Y_5;_qPw6aB1yNk
zZhGv-&-E~Jq~IxsA045D*YP{DQGQP_~zQ%+PDMHYPOs!{B2|7>ZNR367
z6h$CCOBn6=em6z58~>7;ujkJ0$G>EPJjlNO^P`SMH?j4T3uhiU;NrRoj>7H#YlT%0?-c
zZDQ2~jDOdD
z5E|#%Drp{vvL>WZ^eNOqijM&o`=pdYH@hy*6PTT@{GQlGiWq%LHaC~F{w3x52*cJ?
zu;GTAY%x3>wbRN}YH5SLI2n%!v1fdT>Pqw&yn!!lGq+%TM}!)qC@d&$wzhm~`T^^PuXTWeGhtaG
zwEm-XAkwRg=2KP!d{t42{wPX@PRJIvG41SXf$|g-Oyc;UCMwjjqAhG;74BI-Uq%jF
zP1sx;Sf0lT`(^5TVKWh#DzHz?jvc=gku$i7K@X)=>Y|Bq$1I6u@>8qvgIb-854h%Q
zw57&x1z}o{T{%tCII6)Kmt~ye#78$}+t<0Cb;I5
zM=Sqn-vJl5U+ZbOD(aTs-jPu1f7%ZTNegDwb#^BB*^|Ql=o#8m>8I)8jPT#nDJ{$bsIum
zMO6t~-N2F$_l
zdpgZ_v@DhzzH6^$=7~}g70TDWGW>CNV@HkN;Yjkdr-$VXwZ~`_73Yj3BkN|Dnn+YN
zcXzgM%ava|2%fhmLl{~OQJv3oc+pD%OC3uR*B+BADX*YnQrv_WS5DqidZap=$KK41
z^f9qZ(<#Yzw0+i`8BL+jz{3%A7W0AKVL*EVK1TJ<-|uq4V02W|r9rCFCY9PrmtrOQZ~Px
zZLr*W3152~KTw_{r4s%{X(9hwxFeaAm*KA~xKsnt&svJX_VlNY(nL@lMm_C&=rY?V
z64&qtei-N%97yws*{KOtZglaF{n&Tl2h@i!WFPB`V2ojK
zs;%)ZGAIaz|0-^I-{D8E;N)XlMbkVrT75m)z&vFlCS9z8FbrcLQz~8#nYX5=%V;mO
znc}wM7Ew){U-7anU6!>MjZ>WrnY2#7S>9f=NhrH#yHDYpiv+=ZlhYFuN?uh@7J%kx
z^lK~t&b9z_er(>W64M>hUB*523EWsjok$^A@!Q&7Xet2O72YVufoOoIjHd
zD|12-Lk(oJ;sn2hIM!$;)s?jb$Ey>?PZeLQNFgLDJZz?7&;>fv-^J#&mCmClX1Ju3
z)Etyzj#4$bzz5oo`dW4kgWOL(A?!zvu=3`#Zh-$m+fqaIfK}Joi!*}tGVwmASK~Vh
zA!a*Xi<ppF
zh|a$U-5Io-oL^rec%R+vsb;#>IxE=0@6f`5mkjq|kDYAKEs{c?q
z#x3zmyoj@@&NCLFpqOltt(2_s1NwcsJ*9$Ub}O}OgxoDc`I3k}tGpKo>xW&<{K~tFQ>5=zGM3M
z^-o8c1trxp^kzH!@GIZ$Rmrq(Q~KiuW;;G0?A;pCJJtQUHs8c5c)2lR}-D|pcyI;Fx`y(vWZ$p%mTH0f=W
z2i9+`k~KO0lP^NlB*zJj&lJlZ#@}I6WD|01z9$A+{gJgJ_GXDWahV{-odvpfCsSvb
z&AC*gTeURT7j^hU++sKknIilOcLuHPPRG%ItsN4tkc
zKR5N+=5}7tThG?ExwKuZQ~UG6LIU!T?;Fsl6jqLBI3v1kurK$)wVzBIt@}(8Jsq#W
z?B+k44jntE(0>0xj&9G+)M@KjEn~vL`pW!|_KT=z@4ml8RKbJO_-qGnBLR8$km$rG
z`q@5ica;pg`O&aiwp4%)C$c|bkY^#{QFpywBCs58?U$=sKg5#TC-t2B(DJfai
zoW``zRxA&iC!XX;hdpqeoQ5|x&1Mf#vF&7TQ=_8F5V{B-gmQ%TIvz|GSRSclo1T%;)s9d#moi5juutNHYlzO(XRaqF#!;73PDeI@$cy
zN2}_+YL9ioFKYPsor6zp?l%IT%aIG;i5;pNq&l1JEbChHqSR5SCxN5VVpMw!dM(oJe0Pd{5T!69&rX6&XAqN`0JeDM
z<7Q`}x#f>kGcSpEPKHKzGWsMR3RRWmT!}-p$sU!sZxiErb6U0&>q7HB#x1|i{58h2
z{N|%=mnTh0tM@bW%q#nNo%{5A@J1baW)fNgt})5}>gBVF;974Z;)UCgr|puav653p
zwSW@1VYrUqN^kxky$R4VHO9R2kd%d?cj&OWr5mhI!{jGwl!#4Ad)txWSl6`~wZGWp
zd3$qMf~;4aMhhc(sbdqYqD426HTb0H=GylMSMGi@N=ABy1c@!aZOImbwGY5?P9;W5
zh(x7$&kxq}Uj5^m@OM+?p#-rt_r{zmy-l
z9<(#W2Wtyp%SXvF^G%bI)k?QrzD2~thwYoTnYL2=$W4a^axnOGEzmeqxO(I9yF)ni
zkoEY1M8lg>NR2kY722-Nwb4v?y7R-hGH>N&QV4^Q6!*^)we-gx)Bz2SA}DmGY=$Lt$U#fMn~cQr&`+^wx;O5sBEA=k!X
z>xcSFexX1f^p59NLJ6(y!En~sb!aZw2>TyVP_M;L1wgTzJkR!YUTrqgm@8x}of}b%~?BZwVDQ1h28CDuxO!uxa8c(-$A2l1cotp{^v!slT^Y?HFT@~cPAG%eft6Eu#061Jl4W@&MiYil4C$G{X7o}5H_OM7wk0@%MYpQ;{M3Pg>wG%s{BMXC*vWI7)Db6Zg65ul
z)fe(~u>qsOD%~_aj`8#XGz_kRbJbjUI|?hE5@Iy!b-TZ5ZfSGvE}@+Kr-_b8vcuO%
zb5OxBa+_^jtL#bJe*<=SzQk0*Wi;5aiNZCuukW5THl%f6;K7=uoRp>tFFgCyK8HU*2Cae31ii(-d~xe{ZgGY*>+
zS<<-J$1=?NX*Zb(N?u#Yb{*XQ?PC1R#4p-UFc{Cyq!QCSzS9gxduHIm7C#pg$9c6k
zRHw!7%atc9OehD>v&o^Zjt5&!!fa;x3Y@A@za8}@0iRcwqEJWr5ZA_H*K2IDzs;b^
zq4pz~Y4>Cu{?B7JOGq`|+YA(U4g!bn2Nl+=PKNK9+Rv&AT}Ao<9C~@OVo4PrOx%2m
z_wtm<3I3(RMN|%*dWI6?Pmja4qq&vp^Z*^aYYUYEEV
zL2Y78Jp3fiK^J6CHdZ5kgmOQLWTfr5j9*O?U&dA>J1oT%wr`Xy7hbRaRi;(5)|86z
z5P>0N8Vh3jD4E*(y{^d97aq1@kv+R8T{2y*naCa-WBS&z12?)6YTOUe2h(dLw(yZ`
z+;y59FktKOSPAMpcq3bu@aAYqb$#cqF9U=f9N}JrEMJ`eXNv{wTEA`7c!R_-Zy1ZX
zPLuei)pbyXX~2H(E3h=~NKyW{zv5mznhf`b7Y+-_x9L0MBB}vje)ae$j5^D0l-Yc`
zGcV0R2R@rJ(Wlj|GwY~q-!A#q)1Udug*5ZI%5KT#`p)@t-!4p!hV<9Q)V;Ax47Q)O
zsD1BE>zf)p;Eq3=iTEkA+Fn~gwwY!UVm>F$)`jhIE#K^YQDXFg8bC_j+8KMZ)}4K~
z!7S0g$iXA~Tp3q#qup0N5_CA1Fkb1RMaTQw{73dF+-c<@IOEwEamU=PbuxnfGgz6T
zugIt(00r?+70rqY4&emvIF4v2agx>7H)?!y0LiA|O-F!R3*$klhVQ+KjRN4%ydAQ5
zDC2Up`5h~49B4lOVX=|P?wq_$elP$^H=Pphh>3G+SPir6@cif#9MPp?z0i~y{!|}+
zaphD+&S!LTdqiv?%E7shVX}5mq9oIS`ZiKfBUK-kFi3b$$X_0QX<=yp^q?u`y>-Wp
z`pncm*B7Pn<(1U^ML911e%E|Qrx33&98AI)e8{bLpqM_X)`*BKnp6Jo`B|+t?H>N3VFl;lZx~o}41h<&E_D
z?SxFZZx)%EnTg?;wxgJmn{^WtdLNVXLE_@8qIMvqh4egfZ5@eIGG66U=T4=p!1BU7@JZOGjP%-UK#X^dC(LdjwgIoZtE&cmAX^7>#bA=J2u>iYN=s)
z@d-0YE$NOx!Ch6Pkt+oV?#kivP5j;&U=YAhP%?#y475CX3g^L9Y+?iFqK0)Z9Ea`p
z48Es?ibxq5;sJE?)RGq%(CyFaMt)F7Q5sG-+NT%$-WL3xP<3*9Wv27IO0w5hl;~iH
zpnI}mYE9IQky&kDB@WN=Q~F=ze8n~x^6>^kb=nxJa(kzPZj6`R7QEeps;}t2_b;a7
zbA5=?+(|zcH8rq$3Z23lQHDyny16nv8}~DbB6<++?xmn2#HsN$Qhtj
zz2#J0$d?603Qw90yNlvCx^-4jbPxPIb7xD{_7sHCPlfXp*UQTv!L5mr*OsGlmc+F*{3I-P3x^3
z=JbsCYPXCl&6L`w5iApWUC%%C#sEW?*idUiC^amjE0-OI2uMvNr()S!FK^-u$ELo{
z3f@Et&Ghj3phBV4Kce^s6U8&=x
zHl%S(@UZ4F0h_s~PhORtiAkN$-Q@dZ8aKr>6zO4@X<4Q1*e9UaWz{qJ2`>v$Ds8LE
z-pTi75xx$6?5C%w%6Lx?@WU|X`dG#lhf?WKZW?$A7ThsJ_+O=PkI)W4z?7>Lib7p~
z4v2#uQErs8X)g8n1hX-tBlWw|nBv%h<<7#-#7P{T>igzNMem$DP_-+tkUbn?Z?3a8
zd=N=#wCZOXwOP$%`xdGm+;J&smnAc;aYXe!mz(cmz;feWbJ(wE2{LCrGRAgj4);@hi~>~
zvV<4j{n_sjl;LF>o#pEUMn9DU?=;g`Od1PM>U_z7U@Q&{A6uw*P%3M0Z?8r^LH{J+
z;$Qjadakv*RM5oXEy5Q<
zu7xPC!*sS)6RVP)56E;#cOITyJWvSld6>%+txFCrR@%T$=G|uny?k6nsB25h@dO?5
z?P;a%OhTI+U!6wfhFWj&(*$MxL%5)KX3vQ!`h5rdA=SLE>2-{!p5|qCFJNgkqlu
zq4%@4v#LMIA-a05#(wzc`C7J$OCp2U(q|lH$xxR)c;2C$lQ$jPx;bjWG-M=pK=LuD
za+Np1>Pq%ET&+P)t(e-iX_YuSckA;OpGFs+6)4Jk&K_
zDxP#dNF`qt72!@Nh?@Q_RPUjPK`>7h`HtEvo>x29I!E_`^)GX%-bi?Y}
z)%7?EqisLa@s0&##qS#?Ki|bT((`DI#W|RQi6q`&u-eZ?zg@>ca~`B9L*J}UwtU>$
z4GmqCS#i-f!u^I;gam?MoWTM)(&
z{rvakK_23#oZiZ+CYpUx#mdJf5%+l&zpToZ2P|l0icycBS0zQ&EZ<
z7#DRWbo*?j>yoo($&B%2M;X+XvcdoYm8${FTO}IpLrbh
zdCl9r`JtCo+JRgn#h&unwa;1^5DR}Z+(&pxJY=lRkT#CF@GV^d$q$#FixXynl3n`F
zYocD8{X>!VP1j|$w6ucOwY9a!3;S)tXS2$helr%G1CwLxig(BU9=Z%2(a|w5nDGcS
zw~|?3`>)f6qR@2dV9Pzd@wH%du^2%d{EnL{<&=(ds}a-?@e*~Qa4L+f;>qi^5g3=6
z=*333rb0G@+t;WuP)zm5oZ(W>poCdS`+1>}(fLBx7qj#$D{_B_kugtP$6?)!t23DH
zNo1b_NdI|d%JASxC2<}I2~b!~%Myx;;)g^k1*5cl5SVR*7$r6l5${N7lEBTiTzMz*
zmM%^^(e_))>vwe`=H;6^8>NmTsNWh{7)R7lsnIKY{!ah_A)xx!W7@5^REnkZXnO{z
z=iz%D#v3t-#;ZQOBUqmgD0W4PHL;#od&{&)MVvCNv}e07o^uCH5J
zL`OOH%g>artYYhi$A$xgxO?%@(^uZlg=FZKsYmkZ^J?h17^@`@xQJbFHZ`ouyFkiD
zJ%H{N)HJu-r!*2rguc?JJWAqq0JdT5A-ISTA`+VVcc$PeJz&}^4_O_Tah%N>x9K`R09mn3X@c2Kb#P`_GMkL2
zvYOfM`AH?uBGEUnP7=?LyBNQO2N?T4be`QTOcJH%d@L7NH01e_zkAzDy6(3NwI|oN$}(eTn=i;u>KU{NUw$@@8llyB+oYs)
zwf)O5v2#b1+j$b!;z!2VFdvRDwX+80m-Z{e#h+^sW)DD<3sX*q;lFx)Ai%8mJjnE?u9G$q`E10lMOO`#)+CFGCF<>VcVHMZR)BF%JoN{G-VbA
z@LuG;#$c85tIX0=SHEi$ax2|B9>!+(uci*`KD?yT{lTzmywdZA1(W4YIWY^AUUI6K
zhJ*vQpsGnYfw^vt5;S_adA&pGLS@kFcY)mRS_sgQyxkgf=^FPG6)$BCpbQw{Nx&NZ
z@dVE#&j8y#az9aN&xOV}#F^=ZJkH4{vT|~PD$Urw0S`)hN`FyDYsQBwYmO_j1xAk&Frp|_;iP9i
zUH)0FK`PMP=Mtm!Za~TAQoEbd?AF%`F8MC2UCa_aCu+UhNu^4(UPYV&4WN@MI@91h
z8BHy?r}@+tjM^;ro8BGB`3UN?$B8t57@w7>gGMO-*f{&B;EykmOaQF~LigTy?r{Zo
zAoDad0Oo@&(f)X$NxJ~9S%lP_xyqleQsAVgoXn|Jw(y+Zgy^(R@Ur?#P4YTO|LFxF
zx*t|K=l~kkbyb&uUWTU{nkwpKV7Bm9(}ne~%S>LL-02s;{$a^@e-rZUuWtmH5C_Zq
z+B_-!$MaJH4u&iPrY5_;Eu|s1HVCa3IaCoAT-ZOesgDseY`!kAf_u?R{2K4a$ANFR#(0KXr8O!H&|BTd#
z23kmz>mOkOYpx0nFIJKPqj*n@_{2ev_e%IrAI}<0kLu3QWCQcB^`)#u9cM0xIn9}w
z#dE~F=N>(CEk`Q8A&M_5Tv0SN>vN%X|3n$;ByMJ8?9pz?;~aQ$rGoj50O67Aye}Wg
zEm2E?$3K!~IQS<(%7JD-d$`rV`n#F0pjvQc_aWRXl}%}Fa0!0}5jU*M??y72Z&I0L
zNWuoh6wr>H=>2#X3`Z~%(7ZQ6HUh{a`tr@~)_P4r!k~{r4^%?!1|7TA+@iDLRb{2W
zi%*lRfDSaJ+EzOb=$MZsz;j{4(CBg1eCFx_6@jfUqCk!D>7EMd2s_b!Aq`CnSKegd2l`O(}Lmk&Gz;X6n5;FE8#
zXps~rA7IFz4sxskt}k`u8fm}A^nob1-0bcN_{Yol2G=XcDJWy-vLdCE5T3K
z;Mkt+?Enhbgag}KC_0ih2h5h7SLlDdy-oBe$3?kF?l2{?%Nkv9FJjw51kMkBAW>e*h8}mS;yQ!C=&JY`QPSKd2=dfE;23
zyGLB7q4y9tjuyDg#5fM(Ro!DE!JOR}NXlh#`5-T8w$hhl(EiQ!M9gJ@0qRb8|L9&0
z8a!3=<)mL_Ot$E&-5f*HcMB{=P5AK
zgQ-w+`L2e4xefOC<%wO3NYt%bh<}Vw0bbq&$)COfe0its9qM(En1qI8uBV8Zyd*VM
zaYGva-RE}lMIhNz2M^RyeBb_>hqwqG$CF>%y~f8AlKZDCv1Pale?@TlLhAbTE-{e~
z1g&jVP7t>*qzFWeI_|nx{+ZOtx(sb!rL;WBNs`V;E@WuHeXn9oE09Dpoo#}q%@W;d
zLN&>{UCM#l{m3x&w*dng?{O3In_rIK8Syhs5d*uOaGCVRmmq=*0*YfD+NIw;sA1j3
z3b%(>|3sBE0U%btraBEq-u|%x-bPUK7!uS;RS-dM(2`f2@qW$J;Ymr@I;H2wkH63)
zb?E-kx#9rE3xLg)j3l!H(?fF$VD@azrr&sT{n1B&-L}`0fAfx`s_eqsU4!FoMf5C(
z!Jyv`@O_5?1`rV~~614E>`{X@(0igS0lo5=de3#mZA
zFBVilX`$XnO65XsQWLlL6G&xxCLt{LV^$7##2Gk{>qu9igCSxGy8$S)(-1q@v?^dF
z``>#0>|-Tf+UYU<<*Twm+;j)hd7hK@g6no~?k;!AS4awm@uOh3=(G9C*@kHWXM)3w
zH=sAg29X#?Bx9gW1vFtvDuAA>?Ww8iAE|$VhK3i;Uz+{LmC;;!1MQp*V10V9N^hCT
zhc3Q_ai3)#`Ur?-(#c=tYLSwbZ|jYbq!acWN?Lwnui*lvZasZh*)I@V#B|5OJ+JC5
z2tn0VmtzXaj1-u}(WaC?hx3^=SejM;UV`cS7r>f!1_2Yzrx`ws7K@ujs%J;
z4?hz%k?U4xt_WL4R`G4x+;GJ!o1B)M#-ue4wlSb0Q-GS}!g)uQ@12uN`m}cIe
z4Q8Y!tAMfZf2G?7MO)pI*9DOlfZyRCZ2+y3g+L+mtY!CdvxPy*j{uVp!qO_80XLfl
z8qPQJ%9?02kEg#ABAK^^FzD!6S)qZ@MphFMhtOc~eVig3*#8=&GU<5Uf!GxFK;&LS{FXQGo2e2Vnb2l>wvYiv=$KIqL?oK%x4b_EPVif~fbr
zihy)eHv)?Q_1=&E!KRDvrTvcTw^Df6VtcWN3)x#ed3?xe!3Vc?M;p06A3mocT
z=wt_b7|@aQ;5(6xq`}@SuG0jA&-206%?3g?bg4SOokutS41Z;{V}?9Vn9}nH=F21bSln{WjYn#`EBC!wfUZ08g3Y$+rEZjf#fI
z9)`}Nr1F$=K{EU^^)5Fj5M(Xb{WA3Y>1ERU5P=0)5IEr$tbm2nFb}LG77B(imq!2{P0C($rk4uqY`F-i
z_gaeoP{jbQ_9GbOLwmMk4DgH~yoqR42Me6TVzHmxyzl4lvOWY^q%A0a4QR1r8t
z&j?Pcz@v}H55NG%?xv9u{l`w8hb-F5rsC9Iyl-sN{26noK>8OnLTRc!`$K%~_jJ%E
z2?3@s)#Ke`5I2FI(<_%Qw*Mq80Hg%P27el@lc1*;1wa$cBkKM}{jB$u!CVoty|?T=
zp-`PbkYy2)dh#I*Jc^B-@mvcHSS`%(
z)5$-t#R`$F#lINVHL7bN;X~3n`3Zr%xvnc$|9lYqkR+WW80pBPFgtEwCu(AAI)X`0
zMJ{%=?;?&(=!ku`4ahAx4~X^ps7)Rhfv1<@|DkM>bb{<69ztUl;s3((0P(w9{4gUy
z0TLegm(L1<;81r|Vfv~YNwEM8Y;_Q6gYL+KfHd~|Fr_H46+SjM8Wf?uxIB0C4?h?C?_uekCA@tDul8`1V07AAevF|12fX{m0$h+`>WVgKo
zqZ3JX8*&z6jC-6#mDtQ5_F$SjGC;0F0J!lx)?C`7tlvQ(#d{1C5dI3Al>yQqK*Cbj
z!JP03;tBxvS3#YmzZbP7f=FyCl1Y$U>rQ3)g`aaa{gssa>A&t#jy!Y_KqRTDKFocX
zxR203@9%>*q%0WZru*kxU-+Rngk5wt)%4Cg6A!kB^5YSbNVATpH8AdCLZHOJA2R>X
zEB?sq$=zLg^Cd)CgxsoP`l?KgvfLfF$5II}L6ggqFh<&HNf
z!Sz+5FU5*%>)sX>vU|9(XZAH%I3?fv%R5|14md$zr!=7cFBV>(n1!SrjdSV^Gn)%}8u-AS$d#2Bh|PYgxm
zp=b6c;s_nEygDoxV_GRvRc16B+PViznthUdxRB*I+Mny$_-a7~>-555x=SHO+bJ9w
z<9bI$B~Wm6Y`jGVOX0CciUyaT{L0*sc)J)*)i2LikUPjLhHv^=+LfU@`7_b6ovt_6
zIB#o|#d^u`JBzTnIrd@zvz>uS*O%M5Ad|(by;)^7a-5WkTI`@G-%VstLDIDIMvdMD
zmGs4ltEb{uWxsZs70E#9j~tNy8+K!1TI9N;zyD->Yw
z`>9d6CbHh3-(7#K_jt-1=A~XG>Wl95?d9P$HlVQZwELZX!7jY(qkRA8(h+fqtzAZT
zoTP+r{R83>xp}N3iBq$nJ_+8zTmYN#fNk6c&Myk-0`zF`&*~9(&_^D1W1C}eb*fjJ
z9J|`FCcqE_`CE1%?TMs20J#!5D(E;us?~~3?KsrxTD)LWo1Y<
zif91epY>iF)-^rei>1OiPWLI0!>;F}vP%BKU+}3%5RKL0CQ!xH{m};2#iL!gLDvba
zA%hE`dh6(bPgs21%m9&9!^W|zl`^{qT1J)3gCv31D#TM$3M+{g1_^X4+LHy6AWEwe
zqpE**k@%6*z-9e=`NJ{!5n43!b&*kD@8@3fN@j0>QkR2u3
z4U&i@8-Wema&V!>Ok3zVKlMYecZpT_InMW~h5Y_%DAY|s-vtlaxSQ|=YnwO!(4t)A}LZy9Wm3#Jy_0DO72tj35SjXs?7GkYX31T+t~Kk-9gO12G5}*cng&J&w?x
zPlBTZsuE|S_(GTEJ0scY1Mj=V$F9(=Ye;
z{EYQJM73NQ2kyw|cAiw?gLG_4dpO5;EeeUSNiEPR=A8thw5#xz9|9e5#EB5rV!NGg
zIVw;cQw+f=mJaHe$Z`XAs9p?G&t4IEw}{FNv6z+?07>Hjt?{6%k;5JppE>?;eFWU&
zo_=qMXYX|z@|~T2W4ULSC0WmbBjsrp>x0lPs^F9J+KiifqIk)c;IO-m6l(O}j6l9x
zx6A4I`uE*L^hnBy-T07|1M)9p&*4SwAlulV`FP|yh+<_Icms?Z-V&c<0sdcuO~;oP
z49N<(;N1QG>qn+N3w%KHaD8&74EY4{s3ZAG`d6VT+Iv=#BMzAXomlC34H+d)A*
z#H^gqs?7Vrs!BP$wzYe~cvkZdajsDvgAwIVLA~Er>7DZEZpvfS^7&cPE7b-=z^TN$
zdHOamd=8`UoOpqy068(^U;M;+pmz*NK6HNX$_WGpo|6J2l#xGmWRHvZPNrU7+#xbC
zye-2newchuoDK%NLV1fptD_h{q$?hn*f4RoIX&&xCu%m}8RJf~@0@9Ot?aADWKNQu
zEF@al{v9K0IB*1oY(HzD!XrqJXW4cU1N&iUrm~InzjFj&u?aPM?fxf_IBmk&K3-7N
zC|-%4G;WB@O#C4%f`8M@VrNkLqh{(QR5^@`+pn@)b>cjATx!?V(?Uiocf?}$#t}?!
zXFOZU`qX0!$|Y-mc0$Ib0BfnfI(eh+oNIfkfThr
zf?>+@?Furq2{PuNk)nvmgvx6J`SNC_FM&G93BNH`Jw?JDr*4qMLPktNe$kGuZ3D%j
zuRmFjTq7>!DHqXro|1)5otwhHIdwqiL(hU+PP@Jc@Tq`XiDH$NpKsRZ{^;TGJeQ6u_KcIfu5FD1ZGyz|?ZLvxR(*?pT4p+%Qpbg=?>TAf1T-@|nIIzqM*>
z43JJD8E*0AAedd{AM=*Nni~KX(>j^lpjRZ_3Gvy5+y70nl;~k>$e=&(Jp$GL5!v8I
z-8QJcXom)oM!FM4A_yFz1;F!c0U3bQBvFu#%^0IwwUPZ{2v%UWNOLLO;
zvmjIoNWR@JVc@>FpI8`)dT{P4H)*ppi5DxVl7ySRhmnnC4Sl(w=m4^hT^4WFiQDxZ
zB0CV*4hJ0W>k&sV(W0VQYI(7guw4wO3{iU+#3T08Xz#pmtX;ap{n-ITYtDwKP>OZHjP_XL`&#U!tz
zgJ@Oc%8x{o1pWvJ{6xYr<`*EF>PPx>dt6ipSXS0q_`ldUUC6$9flKjB7UjMr?G31a!w27I|_s}73
z8?^j4<{l`R>q_w+9Omr+S&v=)J-eG{gfs2B879Xc+{9g_xg!lFcU5Vq1VL-Z0c&A7
z1zjS1=m>mGK~%sCKp}ar&a{9=-*rt08UBaIfrM*acf)MF)h^Dn>Y)8ogCUyC2RFG}
z-h(7+elNEW)Jt**_kiyfJJI|=Wva68DN={eJ5QQ|Hv9htCjWm==SYtDPj$7~f44#;
zqkEN#WVjxHAhC*__WJ=CKw^@yjSp0SkCI);>N<~J=b$piZs}i(V#y^Klq{EW7C8NI
zBx*VZa1L@uN`GDfs1^M8zhH78w$-VpU_u0O!=Qhrfw}w>{lFn^>&1;WH(_wnpely&
z%qvaOCttsUTzV=iof+sKBkQw@j435zi~>5)bF4L940)(Kp!rDGbxo8Xcmcp^?=TzI
zx(DKuFV)AGq`R^%yBliac}QYsRoc@6R3S+SyKZn}UNqU^YUSn2C}wW>laO
zu7YwijA-9MFX~6$H4PG~5DxZtIfBV_HV)?2$?&dCZ0jpb{Qmavkl@rJB-|xYZp}$9FgScv_?o
z$&=rgfv~u+=&mY=9Uy)|hU?G-lS94;tb=$;ncp%YnOg7aW>ao3D2QHfqx41}*~0S0
z%Bc*dYIF2vdxfSYVaqGhs3a@~ER&n~KiP)2CgQ9yyj~ug@wwaI)+XXNRh1HE89PDE
zE8&9WYyhzT!Na=Nj^EY(YUUB6n+EgxJKV8-JHL!@uEIF(gW{{v-133f|50A>;b-1J
z4<$}iiTQr~aQA;30xpt+L6f}-bdB+bs<)ua96^j^N-Rh9z+ICUVKPU?ZVk_OSufR8
zdVnzdE{kZ2rdEt7Q!`TMr(?bxlj(|MpfB_oxYvsCTI^1GBJ#}YAma%$zPt1lv=TDvq&
zP0(2b3%Gu=>T69Z!3o*CwZI$eSf4@i1b>UM|9mwEh4MrCJ_x67w^j{crlx4`2d(xK
zlZKBvc+yD_o|V|JT6q0mmFo~rkqME{t&`h9F;pOke=R%N_!`Va;i6*!Eg-l?^m=vr
zP27>5b2-*!we4JeO6mROiRt!f1BKMmF{O@g!hE%Z=+`k5^VQY`zh+?CKgyF*Ejg}R
z#nVmqT|@__+y{RqZOVS=9h9O4@kqrd&3_>4f7yU%lE)xlC<1ixSl~NOCII3El0F)S
z>_#{tfnmEn{P(jVbGYjI#KRbDlj;%BItJ>$fFC<~VA>Qi%>?{&#S{4GJDR&t{0bBkO_Am@r2G^b?~}RoYP2aSn@pCj2&|`|?C5lVN6zPbHn<@!x-&BuoLa?7H*A~2
z+U!t5+EqOd`*ieL*Jh$6n6~@_+|UEE(gyt5_=}AP{;jO+pmb`bImm51{4U=M4kGd@
zEzF*ql%=-Q36dw}KD}L`d{An)?<^P4s)SoKpxuOcW_O2jsiBjut;w
zK3({v^+EJiJm#2p+3-G|M)+8Wrpjgi9dx(5X&3WS%ZM>;;C(4OFpMhWvNQgB+fV~-
z9_~e4_L@h{9H&|USpqi3!Xsg06c5E^&-{l`dY@aPZ!atAj^%i`Uaz`_H&qxEy>5t>
z@l?+mx@
z6zHwYHdnt7+5?z7l@F1;xW@A|<6zP+zJ&mde{jgIpAUHkeQ~2o41eev{8ytJxKWuM
zb3r!v#-;ot=@kAhO0o80Vul@0rn^lMlPnq;dTdHO1Yg^@dymUt-)$&^3fm_=k{QWGq?$1S#~VL99S@~Yz(4!3$Nk8U~^
zbdYr#sp~?UjRegKmu=}t7Y893+?Vubo=!>h
z!PSkMWx_;qnPAR=flz6J0bmXnvPLu9G*Yp&ops1~GQO`$vVVKBBi3j;7ALyxiE9%u
zo_iM-e91nUT4l%|c1IRuxzoR}5DGOhHiHvEPl3ZTibBoHHfOrvJv0kE)0c6F=lXR0
zSF9~+Z=f^>*{!uD95&j%AjxqS04}6UH=@9KTF1W`M#2jk@BKYUuhs9$Gg)f;
z6j*8{Ft&tmB4P;K`FJbr#8$ec=>z8h*tisSvX=B
zFXCSr?T|FB1L}ZQU_PZ|0VU1$y<|Tkcn9d(R+ZRbT6hyuo$?Db{)2xy=rglZi{NQt
zlPzOtx>eIEV@TO(Y2AmL;@jo^y-jEb|9oo;>JMg9^T3eLT`5HUM9e*Jxc93##k0Vd
zDtpUtTD2UA6Gmb2@V>;KAw9h_tNxS$M;TcqV%^kyF9Q_)jdl8n_t7I({k%&nuY#N8
zOP%PhZ|S>5nRJ$}O05rdS~p3EIL=;RJ6zNOe{B0uu|^E|u4J5+IA~o{&YNTe?_3UU
zn<-i6ogiro5TW;@Bk13Fulf<8%%!badWUoXTVj#~f_QoWplhxbBN?S@!_&@G;XF
z#jk;fX9Ar!%3a)luPT4B^PU&Gy|aZ?cNs45srsn#9BIV6;GeD_1%FgnYM9Wc@iOeE<*YXm>D{6dfua#jne^*!Hh*7oIl_a_!GibrzefY(;PiA$MdtU%a7@m>`t+v%Cuj`~$?4r^J7?<&;S=dkpwk2S@bsWkXA1(kfha+$e;^%A
z&O?LE5ay+|$oZvj^h04kEc?ei6o%Cnr!k4)L1$aI>F0`K=P(I@4%4>}WK)XA>`^fq
zs2U&vhO$3w2yRjV*f45|{EGYe-JC*Lt|1LcwzHh2(X>@!ZfkEMjKGx*UAwBX
zZV@z3O-LnL^;A{+jJ%H_nU2ZlyMuwRT+O4@0EoOsmgaO`kk^h6Xw+@T4
z?be4CQ7}Le1!)xl1p%dD2n7KZ0cnXLq#GoL4g*w5NofV?l!hSUM@rMIj-|*=Mt4<#9xXpNg
zvl;f%;!UYhIa4pPQx^u7aOXc>B|b&ah+>5t)J^`1i1?|6t#`j+){8XO^%8y&f?x-x
zsx$SS4uxG{1nyA-lnBo(MJxz*53}
zvAu=$cbGgJePMje45{2wcDiM5HYi6A!Y`KRXa7{c@DHtUFYx4m=;qUKtG?UvIfFME
z`7Im?oC3CI^$^I#E)}1}b_r5!uZ+~j+^uF*`p!Wo4KL;PKl)GTpvl1Du!6&Z|1S?G
z53CKaj5ciAywy>ZwnxZ>y=4Oyg`|?NvW6A#-$+^9FPPj=T-9B|v~Tr3J`hDp4_^Z!
zlIyT@rhrg@CeEv^yv6n{^UG0X&FYgWJGc*(Nc*$!CBGc&kNm?s)mtbpII{6WW%rwD
z4_rj^3Ql%$f(P3rNjjLKCZP)&8L#ga#{xpd8no4h!nw%*Pzt#ZM=Nl}jUD`ajXyjs
z!y*or<(-YqCkPOFPmsJ0AKekdk7_;Ai{13(37jaxF3S8lj^B`O^uP$TPZXB9y3+xM
zP?4=YDR{)?3*Ht_KDw?Z5#mQpL(lHLh7cy#A;Qc9Nv|K`NdP>w=PT8TM;U+{I9jTT
zMV$Bzu-+XVLy76*^!zSg{pq;#(#;!q16M46U$qB9*KcQL=j`Nj`kqlwEPbg+GLKlg
z#v7AzO=Qo6#?pLySv#Y*>&0{J(rd#u*F=t;x
zj>UWuQHS;h<&BuhmuDDNetgy22HcFqMC&EEu&SpD=Uu+G9Uf)$FC@sA(`_@-5yAg{
zU*})`&>b7DW(p)XA2oFyK$mTQ5okIcO7jK&$sgY5&X|;77PAJYTCrAPNYx4!PZxwa
zv(Z*RJbH5N8uIr(lCM4+=0HX=wfyJBgDk%ZH7>xyzF%tQYcZ98(0u|KGVmjCDW)8lmMT6)%Ho@_s
zNOqC)XQ(;3s?%^=KiQA-q0z}YPt7*AeTt|0mLnqe`+2S!11*YTAG?fQ{jZ1
z;eMsT%s%n90E)ByK
zNNM|Y$>z0Xo0Y4VqJ``wm`fQqbdyM|P~EWMB?8t&rwE+#Tgqj#fqBXPdWkxbCd
z1pB_%WE5PJgP9qvaCuiR@R59(2Q<(E!rh^DjJQxRwaMG^{*UA2vE(K%o^amGp)u_D
zVP9Dh+rM=%RLSTXvesmh!VzmQ5{n2Xb9ec@e2`Ln&H@uYx5Xv+kGUNuT)`Km=7iXl
zXTO^Ip+#b8mlju!iqaLWfTp9OR0v;`hP-NIgo^y@#5bTGQUzF8%Y*1BRTE_C1Iwye
zXt@(EDY7A=P~RnGrr!4>!p}?w_Jcf0HD8FkloaP4%C+D{}uM-a7<0MBS
zx4e{v;SSz4Abx;=n0FZ2IN=3_z$Ic!0cGt6pIVPc?%mH)5bz_#*7dOw8a(p{@0ST%V!OVytc4gAd!
z%F2&H;)(rVCLWg>*gDRqckMw5gxz)&Ih<0eR^#MSdOC$3mJ-18>D)x?9
zi5B-%G+p{ZzgTuCxX(gm+c{~uP$73jVuRr}LYtpyEqmqE@DmAl+o8Clx%gmmCnJaG
z0?WNNDjS;-W~vf%L}JZ4AxxxRPV9)jU|cH;n=ft{K#U+fdJFS^|E{TsRa&9m1FOeP
z+TGT{wG35YTg{3}Rnwo=4_L6YZEP?+Qu*U_Kl^5*RO!wX&S14@<3LOZ8y^RUf#s%{
z>l32gnvUk^>nNmsu$;5brPaRj&3gzwHsOv;rRU2Z?B_X#yy3*_%10x
z&!Ed&!^~cn;4ib3eG}F!ln)9s4%8?La9}NIi1`t>UNzD%u!H(Tj1LxE9`KSBTs?ub
zhOhrnoXrO1!tO#(hERbW(Z;FHHDhS?IkVj63tz5SZY<@CM)+BGXu(M5hheZ?+(vGr
zQ@%3&XRm@G>0C)H`ExMo0
zaX4g!=OF?hLptUIKqap1nn#1RCH7p}6>MMxAxF0oT%0sBA_gdqY-6K8qY)qIyx9lE
zLdB|f7Khv1EO#FCsPjCtG%YJ05~+F
z8cs=ZGwO}~Djm_X%>E(nuPB!bkw~6)ppem8xGf6}+WZg6w=c2bM6VB-{3zXF;};_)uhhVwRm28+$&Sca1x|xG=STSuaBX;V)ma%
zyzr&H^8g^m)9O35nh9S@dWzUvu22`bV&14EdFo((B43zyRgZ#&HXK>^cRVpcY=$Tw
zL`=N<5pke0l3R1^iz!jq_gBg8f(m|8|Iz|LDu!{gC7+q7GRNSL5*q$QXM8rki#t%G
zI~}+C0i|maT>^0TcyO~co^l3*;0z9pxBnbW9$T)%4yp70B72^JY_u@0hr-#tljz-(
zBHy@s#jPz|RyiADR{5ia0WWZD2Oro^$L97{sqqU=l8YF0RU3hxJAFsjuZ5u18!vxc
zJ*;r?gw`a`jKz9}0P)CYlgQ_g;L9v|;yW#njNqlWml>3f2FBWrjtsM6El?CYrxDt3_Vx`~B3&Lb^dn|4EkSDKIQhu!h3J-g
z#3OeevtmspH9->`Gu2MoaLX;H*!mHsU#$Xv&QB0e{@r_t5acvT*1=CnDgF=1zloO!
z>ergNPH8`20E)iY`l=M*Co5qJEn77cvs!zY*f|V#sfR<*L#X5Ck@=elsi})8#}05Oe*K
zq<~JxuHg8rBW834m6aj}A~vY%Sg!02+->k2H>1_>n1M*XeKA)A0kgCU)_u2^TV!YL
z2^V*y0W(eikEqu-EEH6aOJ4h2VxyjY^`>8CD&@<(H}{^NPW_zcME3N4skQ>&)cd^j
z%zkz8jSIm~xMYmDc0~6?yY-mi`Rf{}Zj}+2cpTZR(#&q_OiXt1Se8zpGyjG1iNU{x
zikk$rf{Q2-j|#QkY}etm_NWTWu@TytaF&4P&H(A~^qb}fRmh$qg!x=LYehL*ndkXH
zY3F>ue1~;1CLs%h@QzO5>#r!!t8D*MpBd=+X&?Sf8FpL%1j6>;cY&mRAhQ2p0Sd=~v*R_^|C
z*F%=r@7l6RjplwU{gt;rq0R>6)%RWyzLc=**AAj+T9VXysIM6`JlxTR7|A?c%DU`)
zSw^5`4XQK!oMeYwPCy@vyxeY{-`n|*lNtY1cpTGZ@HE|ylg`I)O
zt=}nMQKJdG;^)j?UtPLjGXOzNMcqz?>xa&zdRk!7H$X*h4GFjlI@odVpP(0Hr@F*p
z3-kR55S{l=Vv9aha&i%-S~uwp6}3*mK~m+zoI-}e)cFZYa@U=^1kN;wVyS^{uZ^w#
zxhREj%k7zOYa$i5S;m$2zQ+=Rx}42W76|%`)LtTb7Pwvh41C%I-qSY@(#|sJS%pOv
zI9g};R~7#-B*cU_adwYM?i|23Q3HJJ+S-?iNBhTC48NMH+6Kxo#OIJVtQzc!Q2rAg
z0EtZt#yyRPNeHoZCO4rW_;0Kl$EDy1z*6q;(20jk$hSyI3uPF{@N4>p(
zS%3vehsvKJNcmH04isYxK&M%{^S1zCfn7J|1#Xn6(8X3Q9}U|$@aAzWeQZ}=X1Q-d
zrxHbeb8azR6;)8KsFDSDymj}fw4PC!9DQ_34ujUuIYf~L)%!n{(ZeFhGJ$eIW+TwR
zjSZB<>S;0~j%0`LW+5*>^>8DK_YGq3Ze%YK!+CGiz|`3JLuB*tpZC%5%X9cF{s9^@
zJpGg2;3hcUOJGs}^|xq6SI0XXNQQE=RjURIL(ftv7@iY1aj4zIA_
zLXhhpn(zmM<0*T&kGN7Md;!$hWHNfm2S$SdZj&1PsE98NZUY|8_^!=c3~0IojSUk*
z>@P96m;1@#OpEqHYm9=Dage1S&r3x7zJL?1IOr9^k)2PBQ5HeZEmh~=QfU6J>+n8i
zeD~&_I5@q?QlS5{-?{tjCHR8rO3YaB_0a-IqswGSbNA+!lUh6AQ>poY&D-6cnGsgpI8((JOhDOE9N4HT@6tLfU^XKQU
zemDl+laT>1`#$?~A9=tRj}nbX36EU;^pNnmL-X)Lfmr{e)~Zr21L!k4RYSM>KwJRo
zdKgyT4F7K4LSnH^7pVan@W!vyARi(w9Bn;?lE~22rP*5d?RpsRquBJ1DUDBXH-Qx@
zo#l3z{cSjBNOj4Xgv(8`gUQ6S0}WJ+BgLz$$t1kZ#=6u6|=b5G*dQ4
zJX~s3A8B(Dm^WFu%=m`B3G^4Z~m&OgiU=FM_JrmL`-YZ%5%7Z+z~?mrXq9~;(}ohMF)_`n-!4}tC#ap*%oGckvP8npHV`izWDb^Q2_
zMm3}<4Mz3=XV{1S!0^{(O|=I>Gh9Sy{-7wMr~;NSp4TA~1!I+V9_Go)v5aYcB2MKF28sNeK
zXo3ji@weiC=X7SeY4_AY=F^mq%6_9|W4{%Iz6i@}2T9Y+4>~lyXI(U%JCi|V9VH!4
z8r2~074XaZ^&Y;IhVxPCOi{CqA3pZGCM7&7&PgXV
z53C5_>JWAr-F!i(9!%cu&huA4x7rk(e-j(fP?^GFwg}H=4(3WC5o!$+%O(S%@2JHB
zLq_|rwU8g?p_pRn$fGOV^v*(gsE<5FPZ|7zgy$2?gbDcQ&XFGQLl$qYx4^;+j?zY{
zf15j5mVDXQ%j{$W%J#;HurE49T;?+C8|1`nL{UXPr0|snj>Y*a`;+Ct3-<2wxzA(o
z*03;oMf`3QSjH^N?yT)JuX~d(_#kVio!5QkWuI!-dWWC~%Jzgd>etQuTG~DhtL4ebmz&`OsE#x(>nJIM
zq>MY<>^3gisRAQ|Dp_Uv4uzr?=%)v-_J!$9&F&{tw6b@9k?c=i?!j1gs_6_Glkh6En|T~{trTb
z-N9m=5q8F>6my2JXDh{wLz5}!iQB2J6@_(YZEv}{u6w{WbyAy&Yu)p1tBnYjKS;D|
z9xL6Spk|qYKtBjZ+o@?vNX@dYf$qflAlfAPfM$MPPd&8|jD`}qfS|RwhY$0#@oet6
zMKecBPhYveUP?iBw(R+wkrXx7s2#bTRTtqXcyLzZ`CA<^!L&+Gt>AE^M{b#hRLPwV
zb5+|SjA-`l3#5~OE6B6({vZ!_(2o#KgN>(hM;I(7k2{j+5I88{G1w)-p^HGOha}E5
zd4JCz{1ng%mAomz^N?5HfQZh7fI104hWwv!O$Fg_mStyiBip*3zOnKfkT_(F>6AZn
zGjC2dK0LeDK7}(kclY(kw~goPDKu
zPsh$;GSY6q;C?d!JuJ18QRrP<90j|&LRwVOo@D^})ETZ$mGF6z*VsxP!`GxMBy`o>
zFWqrfRccaVwIZEo^X-?LK1NM8E}TVCr7Vi<>Ai~E%Do?rh9$h}RD|ePXOLo&W~GuXz&F5>7QFccW*x4G_3}FjCE8yykss1Pup{Pvb@c)O<3P}z
zLx9HvuK4500)
z`q!_ceWim=$2_NWtAKZ>M~ADWrx%*{?Z~sXc24dy4->7Dyg8~9;D{&#CztC#}Egy*Xq
zaI^e+mZ#e9&5@_QAjd<=fDR_4CKW3IUqpGJsz4<(QXnrSa7Vk>n0d3nMD-mAmUS62
zf45(+CP4rz!fzD**iz>`RBWCCE)4C?vv_f6*Z-gDKt|mdKE%YzV*ohldlQWOe|NBh
z`rPh8&*weIoxSbBK<2##R7JYgqM|^1krNQ7q@qf8&%t3C56;$3k~psnm6bUHmrV*-
zUv>~Izo}W=%-Q`w$4INmutJM
z$ZUWHL0#8V-dRYv&Us`GM1W(W#rc?^~cC;
zBxNY5&mD!4Nq>*RD?&?boYO$~XzEF7yLqxFXXHwW-E`BO!bY_vai8YlQ|0445`_+5
z&h5*P?a6$b)gr|95fa}3Iz!Nn{<+8}g%8|N?Tnm+Mz!*RNPiOZ_tb8#uNT=Z*XbXp
zPuX}0^hFb?;fluEC(CcE69r<0k#54O*uL2cZs{Zyp#uZz*TLhBK-xW8TVm5P27PV$rKKe%j-V_P7wF=a=3Qx5lpB68*p)
zMAe?GT;*A!k9@Zn5Ah{1^~9%#);!J$Wr}6WggRZ26c&gGFM<{&7<(Icq+!GtwKp`u
zyMFNk
zc{(#GLz-d1yf=$2eN90rB`ru_Y6iV8AAVi~{fiTOE+(APjp}}dxpBm7+sFQf#|*pn
zg8FEAi!3z{si>Auqcm-}{6QLr)s+UP6+!|;4o6Jp!Zow^^SbtjOQY~Si&rxgR4HGB
z**(6SQ$KHCaO*(D^ye42G!NX1ukdWHN8Bav^lm=NBhi+38V>2MtH2m`^gEUfzKAag8^>k_H;Vf0IovI9ba3Rc>M5mry9h&3Egx3YgrB
zheZiNg;Ex%qi{wMvu8uj62;u1bPFMCL%ORJ`j2WBk8lP_m+gMI?!GsgaDU!?>6HdV
zgLzbi#!a_VvQW-e8HL?MmDs**yfQJ2Ghd%4aS-})TfUb;kky7&`(S=Er8xatK1;UT
zt2g8mLdRnN;vaM&`Qh{%us^Fa;IX>GX+mF&3+xU7qPCl+ntMNL39ei|WT12e7?NoM
z@(zK;!`EJ&9(^~yH#O@6dUX}eje|HAfUsvp>YVV|vmO{)gs`hVpf>&uZaD7^ftLfT
zv>@}lBk|mib)pna(4E9TM35EtWzbCn%oX)0pg2LP^ajH#>9KJJ75Gg_&~QDfT!zig77gNT=((sVtO@It9vSn
z9d65JWT09`FX9|iJ8k%Bo9DZu+J1&OTe{}XzO!l>%Y_d+0|4EQHeR{l=%=Z$C$u6n
zpg8hH$Kz>+C&o%ux_DWBLx&}(xsy&hh|1<-yaMj>LHY+N{UH>n2nd(RESWDqh#ydw%`d)Rq8n^DeH6>d9hvNo6fBRSSpF{98vrU$Cox}Y
zzVP-HzLK=J~xmJL~xUd0$)h$&$OJ+u&wL;xsiUr54bOO!v}hYv%a)sYYG=D8_ToaK@9*y<{Tc(7$LdqYvQ**
z)}FXNw_IsrJtO+hKX;r^5n4>>+z+tFBwzAu(8=c{sCDs+mHIMCd56SRL<92Uz@ddg
zk1J;Jd~$T-0(eh-Kuhw!03Hu3tK~=1+O7FX>3#f?g(e9386P^LOLmVJ$L|#{4h(k*
zS`-^PYokOx3rN`-RHdp$wRzSFM>^XVd*7o~QAO#GT{X&l7Oh%RX+_+=KU0dPuUbpV
zTHmfgqo2v#MR~Tbk!`n-G1)GLwEXq*a*>dMo&l1noxr5ZFNEWlfq{Yy?8>eJDl(}1
ztBiWrby%d`qlJhLX3`t>cOkPbA|iMP!VM7i+ad#?E)N(efK<|@P#5gKHa~*dbP3zD
zH+wli#P!p>_Dg+E!==lFj$5d=%h=MjSkBo|^S7;k2$q!fcUSRw&YqWdOSxXUy@E>-
zXMai5q^s3==-s&vic?-?)f`aKMX*IV;9$Jz#AIdjblCNL7Ph)*BkKLI_eSso$0QA;
zi+Bsu&?)I~gZX{=Koz6yQ9F`ihJn9g@CWD`t)>jgC5|vh&ccl|KVkO0gPdcqjc2S!
zJaAgc*|ei|tpZSkOE%`&tPwGYJ{BbZlN78=Tu{i9?ug}Bw?|}TW!>`X=p0pajrx}s
zpp$WH0>=gGz(PoZT
zLyFfP)Xoe2j!4s-i9jzUSSUpt452bJbOz}*?$kw0%}tjkv9`!Dti_#253J87yAKQ&
z9ZZ&uTY!|zCr%X9L|xzxkc*XuLK0v)zi0o8Z%W&XisD0+Cn&fNr)XhDD^*}
z!whVVxtdvJpLg5&7vFjH@p1X6j*bnY`TYFVu<*F-gz;|;E$fo~*NQ<4vj95O7l7YG
zJsOI*eLO=PQL%c>Eh00_aLDAypiB@1xWo4_dJJEGR0sFDoB^9CDugf1+yTc6S^`B$
zk)%+(sJ;V!UocYS4|pweJ-M-nO583JF~gc*nk`BwUqwO~`qnkw@=tD8ts*WmDH&Y7
zAUM%>^2g&VDaDXu6k4Ke$zk>eMHH=U-38)yx{LAv<=LyK^i9yJh9()-g{Ix^g^kir
ztd+x5)#iV}Wzoa&2cfplhHmMvI-p_{SR1IX^bDdplFUbjvT){uKeETdos%@2o_eS4
zC2y00QbBPZ{T<8FMcFJGLCBQ9d2(bg^Q)wD(D%oLW8eePSw<Z>K#_pSBKA5PDzTB2qodb|G!y3ZApl!ZbddY#Uw9%{
zwK^oZ<1RUGLh8_f?JV8#2!M^at$~P^<3RlD>{`~j5wbFwcGSV`H<0o
zP*rY14`^QH_kab|#DJ!Z_&Ex9%_`W+Sy3*pPsWdK+I;h>+Pz|mHYUGV4KUYGz+XHy
zicyyk+>_#%nu2T;2{B4Oga9{`syU~oDb0rN`;w?jN(bqPX~DBnig=l^vB4+_6l!1hUjKXe<3Wa06~afK+k#*>jgG8(UkIl$2I;C(d{K
zw8c&})*hocqCY4sE|wh)vZ~;-fy$6lA-ISnt?Wft?0d*rf#S9+Gi=yh8Eq!}%TnTS
zr};9)()2@LotvKZnNBzh=#Ian@@p{As$#zG1oUfd$GktCEV9;4+}Id95bN!%plWDp
z`k0doIGQqkrOKDVpX!qF8<4$!pLDX4whB*2ELfnsve+^T%7rQqEZYQ@t)M<7JFSq@
z%(|0u_)NDRV?K}NgYe|bMy7e368bN=d7fZ0g9ZfBu#{L5k8u?Lc{Wn#7cuP$K|1MX
ziyhLku>&1#Uu4%8U5bljL2a-jckGG5lGT1HEeha;i4|@|2bt0$Osx;K1HyrDdgNJ^
z8Xa908iS_oVPs>ICr@c{%%Z}J-Ggsc+rc8Y{R6xp4MG1a(9rpyOfkz)$xYx;wcmY0
z2**ZA3d>^IpHXmM?`V`gE+URdD<>afUE)
zy}|CVy7w
zuE?w5#pWhey|J5@18EtpYui``9^BbY*$A9_W#53jGIZ47|NvZtSb|BUkjneI9WC=JM^NF(F@byv@
z(nWcsJqy(g*Qh8~IT%1Cydm1M$4rVVsYTDj3JeUxhBGYv4BmdQY_#LHA5=&Z%Dcg0
zph?}QQ|*-8)4QnP?p|0XebX(vV}#b#6&bsL9NEwg#qY
zrS$&&z0dq^y){&Wh-#Fc*XE_LLq6XxS=phtE;?+Z7caa4;
z*mogzGY>)OQHp625(Jiek*1u6%RWL(b-#mg@>bL}&wYVguOftlR|9mRD-Eijd$*`M
zM>#g=An(VeBfciXPY}hdIyA~vf-POk+di{#pqmbc`JwU~rdGY#If1|qUELyTX`gu1
z`^i>Gu)U?Uv9$>dc%1|`oHCm5>&qmiBj4P1FW{pOT1_~
z-;>ANy$#s;LPzZRt(mVWbL9Fj3Qaa{UN%mtvlCF#iS{2wy?Cgld;hZl6m5T0-8&Qf(LA>~(J(s>cDR&SZtX
z8|*+JwP`i&KgT7GsJ#ptJ0`}ju|3v(sri&1EoI}gRL3ET9soj^AOAVjA#0^;4CU`{
z`Gw?WVC~my6$8>VXjPwg4RS-nS^L1dtJ`dVet((eF?ZD`R!tuvyj71gwe_=FZutqr
zwD00(n?D|u%qL|gI0ba|LN_*rNOfHU=1W=7nS%@aKfCva)^{Xj3kRYW+KRQ;-{qU!Kx$j
z>pqM4j*13fZzDLhb`&m=Sp}YqvTn&QiZC<>In03&;Irf
zJ<4Bju*GNS5hEH5_C*3Z`oW-YE%ED1#4(==^!L2l;AQ+0{EXZN3_SQc@?p%e
z*2SOdgO*=wTj(J%QI0B4T~0
zb+!_No%tbd$2h;?tuo>?WEO}Ka2+E_VEeBf~@Fp
z8IRR+uKZQ|w$T2TsN4AvTSR-c+HUZ;-<2PiHi{hP6CxZ_6O!1*wL!HHh%0rjmyW;s
zwJ#ean<^KnG3IrsL`|qXa&-CWerhDQ;+xWC_E>KlO`sj30J(PTSO490Q^IpLDowTC%OXkbc_+@U$O{NgXTO30szI&gEer%P
zW=1ERavyU|Pe5s`!&HrJ!;7@Y|Yq`j*Grw6n~z5ul$7rMVB2R@G;0A06;k80fF79Lw+
z6SAE-^0X;wZ`xyHb{atoKJ2tdbY7`OtN3d$-r?m$5@(biF@3k
zf(l7rr31Bac0987OA193I+y&*Jlt0V=oBt#w%v&;9F6;VR$0|T!5u}vS`E^N*ks>N
z${%m>M|KT-cHbKb#|g+FSQrAj&rWA;CHx7HtgLGtjR##aoK@*QwNjRSxwqOT{HZ#X
zM_1=2L*&}(S?F@UO-(5T0WG$fkkym>NAKb!|23w=D0qPr&)Y5|ADdI__)e+
z)5-YtEi@mFJ=1f=`0K5BnIUoRLv7RE$)a7!xU2)j=a!F3Q}MXik{(&GXC#IU0TAUA
zsU?na_Marg=UA7{UOXX2UrvfPxOJyI|!yG{p@@W
zM$z(h4Kf7$ewMnx)5XdIg7;A)l*{J
z#2*|3Me2Bh?Ry~PfjXhLDzDu-x?b@az7(K1iy?sspaL@X3vVL;iAQ)6yWegdLyisR
zdNE$;zlPryF-i#GDDBc(9`b`UT&;rmO^G64U;u=tQy^)c_}`4L5Vgj$=J
zeCOnx-Bh!{lA@>u{=ILIjh0%*7d|v%ViC!g-4VJDd)G`{?XLI2o8uFoV(lD4lX_qs
zjhU{Va{$byN@veydq;13H}yD-`cppLs0?!rhaZDs1E`C;J9;Jopv^IhnP~78R_6#!
zB>S*4U4@u$ZU}!Wdm=ulVHxzz1A2xOP8=ZT68Lem46tTS77C-}`KvlUMwgA}Lkyi+
zTUXPwvowRO!7bbfzIuVvcY@6y7?o*@nk(KYZsYrPvP=k8;4ty{5s{qN6B(Qtx+zh~
z%_tNS-U=vr%@lF>hN&}wwP3ORKpy*I<4d%ZJ9@5aw)R)?FP^C+8LN)|vtpEv48XpV
z6TX}{YK;(CJWj?pXd%7@v_{CYPT<}^V{5`J8rLqI2NcSqGsC|GACd)C++)BWT0Ld1
z&=q5cgL=vW#7S_{^Abxf!#iQ<1d>nMz}iGLfhzk89}k`HSJ)qOKKrl4^Sy*E5$zMQTzc^B)t8eILyz%aVzAx7x{F
zZC*{Tk7lM~+2M8ANM-xp2FE&Ow8m1aX7RiTJl1MFlgY#lGCTq*1L}mhW#&O!Gcq8S
zzTDOB7d0<0FHW}1Uin*~fPL~hO#m4I!7?#wU2%9d#85BZ4d_TnW|@!rDYD3j(nC>V
z+aWg7{SVmekI+z*K$U1B6Eh+jypai-R?cuBP_>I!R6!G4XyJS&#ca5I>q4?keQ864
z$UiGW@!1f#8N6up#}oc2a%@674DrG{TP#mG&V!%EyEKU{1GLs=a#3GJjlJZaPRja?5(mfK0F$RO
zU8xMHCcNa|mQwLZv1JQ&W1AFO=?T2^0L7Bfe{H;crH|0@E-*rU}aM{*l;H??q(e)a5wq$;76z=Yx7@p%%$1^8S
z%dOMj|LS1u7KJ5D&C&ITtjyY4e(Ns>%-L39iQIhJIeIeu#H?veQdLfo~g^1JqQC(ManB1$XJTISq=Y0pe;n
zkN#_pX!Wm6_I95vb)<{4Ju?)om5a^^<&pL5%-I@7Dc)IPYRV#^E5l@o01&5F)+?}x
zY0}kT
zNxp1X+56assZL!^mm^k!AFL$|K^vr>u>mBbF`_(QQ8<_bO&2i~b!lpEmz1AF2_QlP
zIhyJ2sHy--z(EgvR=VH?9<-(X*cw^c_=oFLPMxQQaH|L$`eT5@3J%8JI`NSsPE>J3
zsNAVq#1f!h@)aOgAMtASZ$m9g2eg0E!g{r;b{Ckq(;}`0H1M+#%rz)2#H&s%=(O?qS<;<)8?uf
zro%%*vQA2``j-CMHKa~{H6prE5{`1f!IeelDa=soiT$scjUvn{%)ZijDB>^p4@8J(fKx<0UZK(1ofCBN?a!#hjRP
zij=%(pYA&u>l7)v+FJ$P)(oMgAWekm-&sT}Pghw>F
zI$kq;VCV@bjetCH=tBA6w|sFLK5K5--6aVl6MNG~7SZpJzLz%9y&MdD_l#d-JUno*
zuZ9`W!_Iayv%Tj{y&oA@bc{q@N
z5t7yeMM10kW0eicz6!ev>UrG8%BFI)W^YR{Bs}$XAW`t%NB7k-fz^haHl?Uq!8&d3W
z^`yW!59xVH`Ma~RQhk_hMk2AxcUmOc;b({~G;qng`+H+gCsr;~WLOL}T}q9)2!#g3
z84wy~%DCq@ftHvXRt6rhX){S}B*a6{7ihIfS&>
zb4(c`W=oVo!jPd9^*2SVZ>~oy26!*Uc8}#L^lRe-1}BZv`&fl^TY?cRKeiK^*Bhr|
zbf|Coc*1C@BbynHQ%&+O^F>0XTCA8xN>D39+d?}KR)IoI$k3v)C0aFg?v45@$8~MX
z=xkT>Al4}4_YeM+@5mmP8)~PCa(k0EC#c!?#KMXyGJ3*Kq}*Q798{lgv;e90pJD({
z-SmK)IY=DQsKYYAd7LlaPwG9>W=~sR>V)W4nqJ|s=4wxFd>vP%U--K;q5p9m2FQeY
z`f+^W<-K_tZ>I;Fj|B2rAUkDpnrU)x$;_mjJxD2y2$$`6dtKW;sX1)0e}bR{R|FO;>=#J6IR=~9D_i!G}
z%Q_$!z|>Soc=AcdY+!4?HQ|=q?_qznF|h?wo-*7jT~|)aIknybenRb5NujPm-Uz
z;^Bcs)uUU`O@}rg52GuhT~U%(1+Zk`wjl*eMl11qgaRNUf%RBs01L=vtv{A3l-k4u
zc6V>D7M)13P*?FKtK|M}Bmr*Eph@TcZdVhbFbEU~r(dsrFU{)|DcfFp?rw-CR$BeC
zjkPRDtAqn&)&WY}v2)}~(W@yGXf)&F(2qTyMRy$2-imaPj05
zfj|g&ezH0P*%$l>atdJXlyMQt?KO!n4p$gI^!H^bmzQpiht1AdbLynYa3Ej&b~Szy
zpDebA)&C)0cX5{{U7yhgF><1+qqB9FH&asEsmIu1YAWJl^;sD|sc%Ppsc#FXFo!dt
zSARiLa^_zVKQ^Yw%k%dxDtvwg0y9~;U7Mp3D|8+WXAiA0ROgr@+K*fg#3<-0UbW4n~_?97f8wCeh6Aw`bbU{b2c
zq=P7x1VJXjGb}GuQ3FW~RZ58wj=UM%P2>LLlaK_;+&C!FW~voQ
zzi9P)n`8gd0)(HEYQeXf1fW(E*Ymmeu5>rs`kp4DV0^TnB3WcR&kOc=a~nPeAK!}`
zxCcz1E>aTON*akmT>@xnfBHSInVA{iwaJ;8xlhjdA^F$ngwSeLj-{%ZS|XDR{z_vW
z8=Z!)JUo_@)J(SCt4IvPtTD8E+h2%$!u$|Rm|qdv-2r~qD*F0L57?$xr_ASq!Lr+W
zmIw9>0c3BFoZio1{a^cce?S#~tzM%(at?G&QNd3?yu!=Z?Upu-%p#1P&40GzL!mWugwL0*{(5Techs3kQ&MT~aX
zetUaw2Iy7{)Z^T!On(F#$Gk9pJq8M+-sx>pl_JNqq>L2o8jR83IEFeR{$nyU_%h?f
zD=H?zEp0H@*Ivr{o3`>Go4!gr+oT7C5>p4+=qWs#LIZ|FN)un)X7D?%2w@GgY#J3t
zFE3ZX5+V_ck3Ue59j40noa^TAn&6K&)7Ivb@@@Kbad3JZBkKJRg1Qd8>icR{
z0u^_1TJ_A?*NYg}(HcDogp{ve_Er%`*=V`mPN4A;CN1S4frfH~q7%&le-YB6y;=Hc
zd(?=~=cu#aHxG^tiZS?+J+1Q4(~&@Z^BTG}=Q!TsH(BxD?~|@Z>&A5N4N7;!g@K%y
z13F=@7^U>Qs$-m?{hk4-P*5PI7%gq~AXcjuc!B|PhO+&|$5eVe8JxhA(3@_tVh*pF
zDGhvhwcxH0k)4r+lxBzTnDdyeCp(AEuiu1;$o}e|qmB%tSM>y{A}~C)vj#F4*LKl1
z7kU@8nH(DW+<<7?_S&6K$675nAkzMW25@^N^NQ3SQRwW<$7)C6nwq}au6A#Bxej(x
zIK||p?N;Dgm6?EYwcn)x3DecW;`K||v+Ei9gl0l=z0L{$@9{xfp`19{(pty!cot+sn#bIkF%8sSzQK{8)`$mD?9;
znrQJrO1Ts9oI3KmW9=Wl)Yxn%kPsvvoI=_ro;KKCo4Z8<2ROMO;eMDDD-&K=#yUqlfvkK4VCKTI7#
z@vpgEuY*|uFkIgM
zVyNoyj8}VB4NO9>uH}qnUxDSl%@&Y8=R#P_c1yo{@63-NbFT%Q`4BQ!-@`-Zfd2}a5vLc@GZlWtRcI`?
zNHxcB$r&awMF~2TBQJ~#5B`v^;AxwfqKrw)*tZ+=8kTUUpLsFM@#|zJX{|cS^ZhC?
zQqY(LyZ#!BX-^0!dr>yfmgPPH)g~=e8Ge&7pZl4vfwW&;VXc6pu20wtWaL1JA?FEu
zC~hKT6pR|0Eu!0hXRei;L-qA1F4zZgiIQ$OZ0&8wH?MEo63A_EC);(Z5->UWgtn~v7OYbI$tRr-{Zp$BXTqIcNn9T>pnT-#yV{~
z)OSBs7oOURGO3O6r8p!kGZ&Yj=LJh~Hfyh5hy@`|a;6pzC73*r23cf83+s5b9kh_e
zvNCF_raX7A5gxxOv{0x<`nYARe~|t#A+4RqU*e>N+a=1rpi*Q|PX)`gOZ=1KyQV#N5@|13@
zukZP!kjPWy=AW0(jlw0pv#pgE*`q|2Y!-cTQ7@Zhj;&a-)5DROFJX#
zq>jRECaC@odv6_9)z`HP1EPR5s3@HxT~Z<;D1rzGh;%6--E6u;r9%(_>F$nAqbM!i
zU6Pwd`pmWA_gwG$eAjcn@0{v>G`2Z!L;!HTo_}=k1>z3!+@JZ
ze#ei;tgCxsuM(-TxkL?Wgta^xV2H`|R0<;uYfC&9J`^`f>nXO14M^%4@5>>anltyh
zs#3Me{2awdzu7YAD0v2GYV6p*ArAAH8cM~msb=D4L6
z`GGX7?(ipR?a6=726R6jsM}rY?k%KprpDp@N=@+Xvb~sOs;2)Ylp#eyeGVmBg6T-w
zBqIS>r~XK~^%L3tzx-F0av95?xK`+JM+1$qV{zGFEgW9Z-#>4zTdafJEUS(C0X4q!bnMGK}SMk1IryIfAGpqo_SDH)#I_w&AslJ
z+&S3Mvq1Q-R+rJd1(&J(UQ52s$Z3Ha9NjS-JhG6lHKKY=@W;tgqmtdtK1Rf>Y$-~Z
zuJ&*S>z#5$0_9yqHa^*2Kg?v{H{pd6BB=kQK%h)Drknhp5pRuc4d5M_g`B$}@0AJS
z8;7w^+=)-2Get5W*X*)BqL$n>E}{s7!BQ&Hgj=_zBJT$cM@bX2<{GIy^V@`9JP|%y
zx77O*<^0pXvy9%{)wZaZ$~FBspPlrDU~&-0W*2R8!i`4;NrMK8X#
z|BQNno_A00F!7H+VAaSeAXnubPMLHra7I=Ibc!%f
zO#g&w7t^VdiCEN$tBAPT2*Is{Gy~*O%m7>&vwVc8*d#knaS~2dq!kF4bJFZEl~1ttOy(%
zA!EPE@I&j8TmPi^u15h>n&Ux}P=L8rMkFU7L^5}IuR+O4YfylsAdwt~y^VhgJIP@_
zvxSw`%tzI+bnLHwyT17K&6m+-(|CuaQE$9@U
zm#?TakR@d-UP>Wv$NA7OFRu7a3Xrql>bkJ
z=_bzJZRsuczh!W#O|pVNCsb~4IO;1D$tu>@O%3D2UcMATNAjsr-5qcS1#^M%5_s981f3Ha5dO2xXSy@a&0f7n^#+A{{SZ
z`His<3fq~5hLy?GuwUNp0jkYA`W@x7DZ~AGk#_3#m
z-2TR}!%u?|pXY`vC$^F+;FzG(TpwLRl`LSx5LXA@F9-5|`?u51^&trBAg;ysU>q9e
zH#zCnT3ug%E%DBFM$#j6n$YWU=;KH4=LBS9AJVvn-BoWTGQ&0+Q9Tje=fN&_Is8Tw
z;QY7g#BSiU+Qegg9
z=>#uZbQ|)15f3B*UURz(b8&Q+FCICRUJFCgpK4S@MM-h;w)_&_nO$#6DsxZGP$fN;wJs
zBg7QJOf6l;S`ABg|6t{53J3adL*n^t`OM7ht4Pjcw}qd9$0D0rbaYm##hDq1?~PM8
zJZe3xsl$d{gbx<5Yv$`O?)CkONlZ;2*5szn8+MUEQpaHTpzYvC{x$wP8|A!c4kYg9
zW%X_&_Qm>>G>+$1u0ZhDOyqKfEM`7ld0(o40<&;EvNUz>_U$E)106^
z$!!)vWvpUrmN^yN`C{0<6fn{4Bj3$D?WY)rJ+|XbfgJ;YN`wQnq^Qn&$&3n7iC+N6
zi&pNw^%To%s(O`Hd~h*Xs8w*A1^+a;>FlQv$P(_ACgPp5w}U)Q@*>L{T(!B{YTLcF
z07^{W>mtvo%`P1g`+9oj)r+5@2`ux6+a+N%CEj+DmN9BXZy=S4h+0|^EuP++45r31t?UP0DMG}4sux;lA-np^#
zZeVL~T4ycSvWP&-Dyz4Mw>i4CR{rRFxYthm&szZ$LqQjnew3+%m#(t6b5!;aN}uq7
z^5BO15~(~;fxt{K8x^R)5Kw`NsS$I!QyZ;(N_Cyg?z?gsabbzc7e@o=i;jmFh_6B}
zcB8rQy{7uV@fjp0vW6kQ`aBXj-_!?{oy~(6%!z7`B!_-Y!JjfnbJ{D9rP~K3X>r`1
zWlS6pEym$OU4LoOAZIY3&XH3NjxGxAs)IaQ&y<}KL-CDyakU>~7nW2mnjM=pE+cwD
zTa$jv9?y6TUn+U^kYJ8a9ya>4_-_W`kz9oU$r+8yn7(Y6Cx`v&_8wlg=z~l@=nk2$
zy+*I4?3d{ThaMSgLW*+0W$)a@{;oVSvG%b$0rMGO4-I_eB
z{Y;3-S$UexWeV%Q--_aSgoD?J%PW_KqG735w-MCP)6+4u8_GsD&@Ir0=qGXIuo{2G
zc$;*#Ik2#rj-Z~$+owm=;iu_oX@!y5Vbf<^ELP>C)#*LRb`ZjaMp0pS@%@&75cj;bE!K?@Zmo579ot2L|Q``%W4v!D0L`V3B
z`f*!EiOY($FS(KiN7NV{WJur*Bm+}DuM;0bSo%$v4(g8V1(p2}$zT_8tO025W;=78
zW-ocpN?0OUW|u5W=S|86>{71&B1!FBrKr%^BOd}P6s{z|P+hyTU&thSJ%#-JYj)7qz_>uij{q?*RLF^UAW{5wBmC-IW%wq_*k$YN3^g)G@w;!th8g52
zs#$L>zCv<&@|tqCxHntICau&>$@x=OQH1R$+gQ$6o&=ml+^)s8f1!g=_n&tLkrujl
zn9u$_^Q~^xX?dNkI?q_byobk>tIm5p9Df#vRWQWARdO7w>y%oVx2Rc;jYwQ2+$DPJ
zl_5pPZ4x@!RAw%%gn}y02nX;PeAz8m8I6-Ffg
zKs0WgqzD_Y7T1AWeM@Q<9pOAa{ST&lkacGpuJI#&lN72Fy{D4dVt1UFmCNC2y~n;z
zVj;jfhN5wCdws*GYDOzfvMITJIPtXF_}BY2JU(m~$w8u0RmJRqJbdA*(6)d*21f&V
zv3*dzN1VN|Ke#>P$9+_yq#kphvMFSGyxWWya^gMfSU7KoC%?TvZ?kxuE(*?t0{Uq+YYIl+|nkY7F=GAMfw!*v-$4(w*
zrueXDR4xbnAY;b&2g{ts^1SLvSrhyp1gY0+#tnEzrn@GZ2)4K>v&)ZIZe5Gx-Q=|6
zDoF-SYChcKRjI3;^sjV5lY^ICiM(`g$HOT48~LtMYJ6dQY*23^=wTkUOI3{CfRZl<
zGEcpL;d)Agz7(=%6E9FHzZ&4$vzlx+cyZJ(`DaJsQ&eKOVQ$TX`}oUpTs
z0k#gy=PwOT1e~Ol0nE1CU~|5yLoIL@_WL{5vDL|Qi7D5~^66ecNElAL8Gi6M8sV6M
zE{m2fnSp@;as`T`Bd40Pp&tr<7<8l`S_{AwD5x__vo0YP`O0SU$G
zxu3SvyNkpRceHL~Xs~jE>`&_#+lRA7@2{Dinc|mA)@`_q`#nnd
z{odpF`fzrN4I=4D>-)73L#>k@-icT5^x^SCZMUxSCuK5z#Zdm_BgmhutDBUiLMCpl
zoCRJMN^n;lh!ajn2+m1uM&DKjGkle3{bi2tW(IW)gYivRlMHSQ@oGSRMg=U;E5?_h
z)|JgMZXT$;G}a9)7C!yQ&B_Y`Nb`{@XYo$PBaS|14<^+hDgkA*b
z7np=y_&6VrE;r}An~gR4)6T-)KeDq^G}e#l^}lwInk`XIAh}?MbD+j1-EEF
zQB}ZHf5P@EPsL27mA6ae*QT}w0V{mhMaMe2t
zsjrtImprNFC#cX~3x-!H>dz}?Y@h>8b&wH%0yMR;2y#o*ABjc=C$deaCQ@JA(g4WgLVaabseuud^+Ri~iG~o|b%i84&vsv;;7
z138N8S$4<{$OnKEh-=n3K#i2UJ7BWioXhN7Mskg_tZW6Vu>H0j0zAY}*5+a}O>E`O*9zbTa_rctQ_MkNh{acDW^
zxs2fw8X|AN`FIsd^)%4I%q)~JkgzzzP6;F(NRkL<1B$%>$RXEekouksYI@^91IbCV
z7cntgb25di{&6u`+1Y}uSahziN#5yRa6W)v-{rDD_zrq7!C%lm08LwdXL?1k{_~7C
zf>^PtWcJkIDXWC-;WPQ1wX;qE>CN?i(O#9O67Nd+%qEllGnVQ(OK$Gi
z(lL)j>l;Icbj&2y0YTP0I_QQq2+C+!P<P2BoH+HY&9FD5&U8tAdnxxq@IQG5l$sn2zCgpl5caLwN^kks#_RdIT
zM=c^J_u%snSzN&*qf^{Ir_zJoQZh4wX?AfLX`i<_lG{1t&G3i=L_u0=n2
z23DEJfqd+4%;7@QK2@wk{@`@=8D3aNgG1QsjkD20+^H2Y_x0Qf7iofo%lX+xo@|{P-ui0bI+l`0fc=-{`vREfFeDwFe&Qbon6wtSLSyw_}S=
z&q77b|CT?l(66qne8;-lb2`m3J3p6kFX8k7YLa>k$BpvYPC`c7T9zXwscy;1W`W6r
zNQoNmaN1gIo9B)yu_E71cd)FISdVz=uVI20YodP{U&*Ot8+EU@&3t)^Fg&vDz|#0+)t!7GnbOc=Ce3hoZd2h$K;FwTg5x5ky0ccmXHY
zrn#DXEm$Rn8#TYwLLTkfRU7VnF7c3+)eV)?izmYR=C(QF%~iiW78q`;OM~9dFf(>!
zXHm;iNjHLziYW#pFU5&EtZhc5`RS=|$QC=H0WI{N*;)z5WhoXi8Hta-29|$sVl3bO
zdgPTy7vG3RKMI?TRUuyI^Ta2lEC{_4PxYWfC3%u9PrE-Gn
zu;BLy!IU5E+b2QjnnH;{)U8X6w;-hlI$?~i=~66BdXe@caEsmeHr*6Gv6y4P7JJ*7
z%sONmnsiATcQeQ9P4_P7F0@2WjJVMqT@by`j@K`)3uag3#t*u9tSHx2S`BOMICEwb@>E8u
z3(onYp!dD-tWFqekAzjD6HHyl0$N>}I#TGdBu)8+?`4H|54#0Z$JWJ#Qbxo
zb)^FFloWrVvW?OOBGK>s=NWh5L`oDa=;KmPEJ0I=?u{G~M;byT{6+tr4r$5)2EM
zX>GoXS>+%?F{_0m>xG3hBTq3!tg3^zkpqwD87zEZ(%HAHYs5ftsac5R_zO9-%~j%J
zm7H>wZYrxuEcjp(OE3|3_M{G4i~x6sSzfvW%`^o@UZ5Q13N9bU;;hsvuDN
zYXN_wu*3BXn10y9W+A17fJ8-1NB%A?Q6Ts(w|-6bxOY9GDp9HhzZpMqTG-l_o~|F0
zEu_cj1m8L))H!Kb2yQs_P}I_5;5a?x2|p;p1n1tZHe4Sy!v$*4$S%>>3v_R6{J5WN
zAoX$tz>5+yn!P7Vvyd+guZ)WcLJBK?PYbG=PeP?O>#P*n|VoT?}AX8;nXI+uvT^a=L|%DtICz#
zZGPxG9euvdp#G#KY|A_o~FzF
zgj1i#3D*OV9s`s9i>E0E9~S#FT5+(QAGitAKsjlOrq(v?v-CB{+KJKaAZG(h!6c0o
zb0qN%+>(1&syAkgM*hIIrj{Pqw1AXn^@gs3G!|ImSD;ucI}zmQ)F3OgRN?cCfdTrx
z>s)lTJbexVAH5o7artHXEh$5qBBVrS6Nrw^PCoC4TUK|Vr#^+n1UI~9&l2K@{>zC5
zUJQMyB3y$;Y?pT4$c0u=HzM$ca;*>Gl9n{)FFl}#1XF?&Q)xFK9I(E8R(W!(7F6Jx
z(VjaRRiRohkK42AM#}P}q
zg;tOt5B${&k|K;te>{T;c?NQ(z;a{i+R}=L=u6b7zA#J#xtXaVWKChJmrR23PHl1{
z;nejfKt=1@KuVyk+vkIV9uFx&K^i?e5V#x=xTBY}JQ?`CCuNw3
zvT{;?UP4ufEX+h~_YDncmtZt($xYfEmabWb8GUoBfA6v%Suhg~PsuK(vABFv5?FvP
z{^gT~QA2{I$8@8Wc;*=MtAx-_91HScF~5fx%pmiWBl^u2)NelBmh%0}Eue!hePLXD
zBpHg&fa+#5^S6nEayJ2Z)K^ax0o;O*s}pi;B3XfuQGk%IJ)`#nuY`0Y`5v-WKx_V?
zpBu}}!Xi3RQ-Q;DsDPzSQPNRml?`?cV=sm*Q-{On2JVA4(Ku=X#!KuAEEOuce&uZM
zBAu|e|1DD64R
zejnIzzQ$x6?^4<6F_WBfD0dh3@nyV~aqr0p3aZwnSCUi7)FrtRDnK3}R1~KaiKk$=
z=MJH3Gm;p@6h$*=B85tPDb$s@?_uOXs84}VAKtzXCPo0EvI1O9o1f@Qp2|6bC3R
zs#$K51mXf{0%#@V)8)f*3Adi+kk2d*E;QW+oH0LT;T;cBIn6fZ-vVu;4&%F2&dhx6
zA4|ZiO5x0E6;ph7#$2Zbc>4$ytWm54NHS8HBgGF~-#C$h0iiT~Ad0P!+CHti3s@}n
z5ZY{laGf-LIqqHV#L(^*VIm&#k{Ysz&&&_K
zss0l{tlu?eZ*a044KE>l+emO}B0K{ERRGBhL{^GE1+4ujH3Y
zq-~KGtkx$p^YxUzl%G+F%hr`G7X?9{CwkVNo!l4J{W;2)^*$7>B>dQreh&#)@j24p
zpAFp3`{fJd=S5S?QsSkG=XC*$nOpt8YT$T^xv_CHXclbGQ7(LOI7Zo*UlBn3w*2G#
znts2(eIMM?BbF;yAt%?=OhW!K6J%7@+tQFx(StS7c1Zrd1Hlt3TC))L*9d~Qj8@;n
z=4DEb@-5vZENW++M;rVba%0fLWN*_z{NERL+n6pMU>@|C>(n!2fx#57Cxd~m!^;_X
zPLNh^q@zMa2sXrP?-Dvi>pllY3j!-=^o8{v`zsUtkpgDEhYnq$4+silu6D&!xb>NO
zwe2QbiiO|W-|U=%=w71)#?;pMNU5f;M8w*!`R~UK$(C-0rQV==hwVJ)iNgP~{~GiV
zM3SC|8@{)Ltn!iJ<4s@KP}zO2?@~#svlNvRKQO73TCtGcG-3NrGYwKt#2P6f^BR#D
z>XIUus#a4eDTzeV6$gj=3c&RcNHJc<0{v>`f7F0CEfIGlse`Fvp^5@`+KUg}HUa|r
zJm*n@jh*zK4qTot^MmId2KoyINSm~>>spX%PXOBFk?d=m
z*t&zZ8Llf#I8{A3t^W@4ww=YT#Lz|v06?2h`a(%&Zk}(`JJQ=XHFRGqH%I@}H+Nwm
z+(;iaYkV$>vvB1c{+Mv;aE(2@2Ae|`pPJjx6BU5VwJ>W`h=oXkXy>lpTV`d5_;dts
zu`6#Re$`LKgt-1c42M@!YC!hVB$f>mBW_=A2{jP}DYOh-DG|W`vfQH}qj~iZ2`Zl~
zNcq4z0d`&Cvgx(?4|=|^CUIIo)rzHC1i+@;dmbVINy7l>Q;Z*neH@9%anfnd_8SgF
z?tLv1g2^Mj2VPUn)55-g*zGCAo9~0=%D!sFfR_FjTKc<>>2{Do=kYhg?e;y?OK^#&
zvNbj#6IdB6HBt`?%WS#ybKSFs47||201P92a`VpO$qvwbmQX4<%`2SChh8BWLRyKZ
z_9YaqrANB10nFhA4)OHceV=xM88J13s`=Z{&twBoPLGgTt~VZ*k0yz^Xp=er$iR_7
zjHV`$)C~UbHap5DaNn4>q&=&(=sBY$W;`GlQvr;Y5;q3Z7i;s;(s8Tl>1k8?aV9Sb
zGHjADWsKy%l&#oP;O~;NoPl8bJ4n_DwvezG!iaWh#Wa@@qA>kN5IRT@uwx?D*OgLA
z5`lKC7ZMUe%h&>{+K>N(;L9YZ>^JQl-^xW%lE(~#gS*@;_)6?7=NoWi_Z2zZjW+gVaIW;ZFg6Rb^oK4et1JroTKv885>(ypWVge
zPIJj}>3NIItJa$vWc;)3B~{Uy1bp}Xfe+?Nr+o!&5GCpoUoeVSA0#Uq2}c!ms^Shg
zz;V{|uoYI2=?q_(^U3u!$ew{U9^-vU)%!lr>l>@K7&>A%ng$2W}_8&LELhZHG`khyZaJ
ztyeKQngMVl%Sl_C{fEznE}qmpY}Qj5{F@~#HdAsj9B&%_$YD*j>NqIB
ziO+ayEPFW35XV;#X;Ch`a8SK_t+L_B4<`E6%8wE4^;{K~QNr?Mbf(C2ToFm`W@K%{Chg`kFB3uy0cYRK!T1P}L
zk3=3|cKW8b%(d#Cx_QH+@rrz;Q{}15?pthaNCzT|zT;m6q45IX)ET_guSfv|{Ij-E
zX_VQ71bZ6hI-pD|i-Go+$bo{61BJ}3)HSbb#6d2hB+?GFD%9G_I1VpPex@z#Avtn=
z?)T_sB_wE~7rKRKPeaIAgzzafdoKgM!UwhJ0?&ctCTfPGKV+j(hp6+LXul-ngD`*x
z5&XoE^xeADNUq}(V&Hr>Gl88>PH2+2&d;qN_Tdw4ouixwy;{%pzK*St9BT2dJQB2D
zxThBBKAW%%`wm-JJ!VSKJodA$K+NshrsjGx(_j`Lf7KLpUq^AqIq-xib
zN6D5KR8$yIS5OTEN1L;0Jno$g8VyEO&^DA`s;CdC4zON4*#iMmX6BIoO_0iX0vMho
zMO!oEt)47lB03MLDfa>LW`?pwZ(QXVODl&z;2?F3Iw=97N@HH>6R6GbD1MX`GT_|0
zDP|L3Et5J+WE)iRki)hLX&i3-1iPl~Cvusg#?*K_0kWvkH)plps>g`2(E;AnaZ_9X
z5~n!8d*g6L%&fTaCB7+E%RF_$kpc@7p|!l-$28gnp%<-{afXxODp|1Q_&9^!=775K
zbyCBguCF^FXC-TU$j7!yudNjL*%9FpEI2qmyv+v>DjvlJbDkLF9ccv^Q1(1-Vj;Ls
z29CuA?K(w2-F>yuHqyh$%$lyfZVh6D@qFh9TtK2l0FgZ5y)PjQIm>F=+Wvkc)qKc(
ze4Y&%Zodp$;Oz&*QxNn6IJ*s`6Da#kFQXBVsav;VZ$XTw8+TSqyI0GeV{~-m!HUg{
z*r=LY)~~4(wSE_LH1M?~T6B?d8m+OZ@B~C*IQ#gUk7eeYYbp&PHZz3v?YUJ|J0$Wf7IT6cJhiLbeYo8=_(aZjF%Pl*C2xLv2U20@1$a>{Q6Vc
zjqR!<<+$8XUXZF=_^MQYPN8xE^aZBW%EWELW!(GiDjyKe$5%gjLfwUl3doi$736ZI
z?f}+Wvihv-0Aj6q*PDf0oFk>$89lcOT>Vw0C@*6I8Et$bE(p*7CEY!p6qFsR{YjD+K~os^^(=?2)UF+uLFITcL&7E&7@xd83zVu`r-*SBu-jD%APlWPx*izg<`oGud(oB(PPKKuC|A96XYkOf&mHnL1$LJ>g(_-)$jPXb!ho?Gmr&2V;0VY-qf7+j;cCT9Kdwy5TNSsSor
z)Vk4)MnY*F9hD!-%%?VBGcbeY+047a5=dF377_gc8(s7sBzs4``@!}iysd7IsJP)WF%wgl_NxF&IOd{y
zhAF$CU}ZrhaZ8S^b3&8f_0kh5NW_Tq(UIxVpg;AkmL>boKmaW^O1!arcLgwu6|@8B
zO7KwRAa)H5rZXt!!v*1Cw0|QxJy`v0vk(K(k*-?>5A5JbCA_}ZJ+9nmk(6mHU8FzN_;dI$63_|jpE`eD{PJ1V+ca;DN?0mMmR~;Ax0-#++m?68`oy8DAM+rgT8imd8wg`gEids?e`)#SA4G!aD)GrGa9_{m7R$+&(s9ErMp(O<9^6xOX@NF25BDMUH}(GAF)Png8YAJ3Jy9l
z;^hYZNdI-VP8GzeTLzp%Td)`tSm)W{;y|?>=H^JgP=K_+1aNGPU;p6<9&eFpv_*PfRl6(E?B3r
z4rO?OSE2S1eDMKbB6z=+hn2v3E1a4EK*u6FGuzAl4V}ufs7m?w#{dF9fNEp~4mNdI
zBnT1T}P*lqGX;EBtPbTe`G#EB$DL
zT=4IE!HqJWG!1in%W3~$Q-fIqWyHb3n^=AiJ+Z1cpSI*i;{^)312wP~$HvF>|9(LH
z8r)afnPa7|Rf;>MeY%fmkr&eP#}!h`$uZF%?OQfDDp{`6*nc#&x=gZRKaqLvHkhMg
z&^Gtec1gQ!Y<`>_fHZvd$wSC2nt`}lFt>%#ZJ<2pg7;PkBQmdAa+xdADme-Nb49xh
z2%!WE>s>QEz&zP+5n@vlK*a}4#9Pz*I7ss8DVFYMTywTofe^!fFN+Ft?;OoS`5XS$
z0g!>r&P4zNnE~mxib#XOudn;vU~SFU@tyhW7f-U{HZxwvjY`|as1>5W@6UW0&HpC<
zmuV_^@4Ly7q_Q`%l}$eI}PEg>F{^eNQ98OzSN%9YZfY3_=o`UV04Ux$PEP>
zQ4sLi*q`lBIA@keMw-S~Cjd(q1lBJHGBRL_QeFq3NbJ7|X0?ul{g8CwGyNqY_`H7K
z^(vraehCqRoAY*4!p%a}Y`=L?2u)Xp+9a<6w2uepBH3OkKS~Uv^;VAgswJG+gQVH^R0&
zqCU0@1lX@l?!p`XtsD*MU(ox4(=43J&2h
z-tEZOK}2AmzXyY{@F%lSXd$+K=I>;=Mt^)*L>@n0QkotT9fAjs@
z=lw6BHjEuiMBg!e7|77f+(bntY;4nc+OG2Juh1Z_@Isub~hurIQ{DB-x*W5bPC~^bceS
zWdaisR6*4ZZQOUcOV9b%k`U)A#Ty|3sl-Bnfr=6T_PJ<2L}6G$
zx7hRL(V+A~!DWeANaa!FVp|K
zZyc6;`D_ZZ7l=K78JEr{w+5PN3@lpCbg_*L=npf18AX=o6Q4o#hFuld=jTnSQ^-0W
zgkCip4CAj~1zYrAp*+S%c1bjEkP-BxB~0M~Iu9y(8L`qx>VoqxH+%QROc@p+Haj4{
zT{9^yN9T1|QOAHdGcRSV7y1(>v6y5LV7ljTjCA1-Vo+@1n%>Bs2Hj|oTwt}Vg`Cnq
zpUbmA9I2hIR%R_yYzmkk%O&o0f9|Hoza#--SPzqAc)e0r63so3xyb^xxrg`eU-?KR(*y`|#ka3amIktFRGQlPaHv2R
zW_B5q#jLy2J73*d4S8qm8*vgKILChW9SsDfQ0EJLiilc7(qTGt)~Lfc``q-zU-VDbl5
z6t58gwGxgb8a#T)uDNUf?1vkMx|r!r0QQccX~mpB2-@KPcxxM^n*nqJ<9~}zFnBkv
z?pt<@1zUD3MISDVq~4H{RaUt-fGA3NH$Aaa@X*pte#p#KsZJ~
zT#kHAI`fTjgMEKkZM=(vsFd>{rCBqt2$57$U-QGsbNg|5-L_!=AP_8&%_j67wA}ko
zqJsq52t^emn3xDhPDsG?ahKZ60H^v$dAasicOVDV?%lNX3b1K$!eSJ(VkB2!uP|&5
z)3meqsjW51e?E8G+p;+JaoJDtsLCAf>axz;H&)T2;nI`WrjBmb!>UCC+Abr&N+10g
zC_SK#b!>VdxLXz}KCet$(F*3
zz1^&k>>j9viTpjj*WKjSic^}{qiscx
z+_#dXCIY(>_tT00app3Ni6IGKf9)k5V?yc~W&TZz|LbNK1p>v=6(*P(9ZY-*>eXUN
zs+UTwVun$4C7T&gm6YXN^J}$tJsk@mWh~It~O27y&PeIK-tN2*w3?2r{)&R
zZvsy(;@q83vKvAQIjV)W5=tOjdR(_R$jbA1&2m`|>+h>LZif(I6VJ{#t;F(FlfK%0
zEw#T3ee1X%S35Lm8{e9AjXC~{UTnW^(qe(wU%9g5@OGmYoI=_Vl0qjzmH7mvIt5o{
z`P>fyfOz77_T0LY5JiLH^%PXl7;9Y70sf?gd2Uuy+y<>y)J(oG1)%_Y=*#O@eB9yS
z9>;Pb+5GEukllF>`Z~HBf^N0cg*86hL9<`{qSw`ggk*d-m_n@OKSQt;Zei
z@hvebYLyq_th!V&HhU
zS-%!mARKidoPp?X-+^$TEHh)7l`mkFY0*gGu>SuR&endZ+HS4JbTnHPn`P@Rl@={-
zFun_#Fhh;0g%gLvtd``=(DqRI+x8@jP9-$1{iApNvI!OrYwu;S#<=;X2jvBxw|?VI
zi4+3f2uFU=%yqJ%?K}4K+mVJ2=C5ZH#;gvijHkMsATtf^Od#DNdFGhOhwd9JxE>XS
z9{1w@KSmaVBf>xivPIaS8qN?b{}qHDt5|&LHgHTZ57~Zl)frY=+VkP&B^c}aLcUBgDSZSB)CEWq2ZE9H
z-EPNz!F5i$GR3#0&jMqdyz_KkCOpW*A^Xa+R{B$Z-@-`Al)Li#k!7utcOC#+Cl|&b
zVWkkV$DAB>gKN{#{?+(S)5D>nDmV6pI!~7Sy9;C3#;1CiZ~_%?8XUS1XE@2USZwW%
z3t8_iPO(|64#9uO@;s{Uqp2lRg9S5culH_uM5i9upA#l*fa
zCU%i*G7uAkuAA))+Yvp<_vscI;|&?Pu((Nnd2-rH*@jxBp*W4H&z8elEQ87??K^&z
zGLj#bNVME0Dzqer23^o9ID7v&hLKPnWBqr2%`#>=x?$@sR=%Nt_Yfp>Ac26
zYD(BGw2b8}V-M-y>;Deu9Q(mi8-zQFhq8P6t~W#h(73_f
zgE_`cTdWBZ5;V5N>azRpC1y?LPAdi$E^7}SDOE0MJvPp$lpnq8GGuIX{$(~{uFYtK
zW5UI)Z8ALfu(R*>sK%+g$O`Q}UK>?i%ZX(pWtbYQ&?KdsLrT~$7LCsn_G!mH#eb%_=#od3N#SKxN
zVazYzQZvq~-*D_eTvXRNVvLE*)#+Z>pRfpJpVN?7FU|659V#~RR{$VWdSp6b1phWDat?o8HMkdH5VWof9c1AO?
zr-u)KZqTM6VoXP5JTR~DAyn-7>}1LhnMI@0rMAZUfOb0DOhXC-UmM{XiU;x{zh};z
zKyUN$=DKyiephRy?Nm-ieb2O?txJK<
zC&^`iG}(`E30JKI5t*w%#r^nEMpY2NPAAYiSKy%vB`c_z&|gwL;Mw3{vZt1VWTt?H
z)QV?|jgO#zO4A@k6%+M{fDsZ)7P?si=w0%@4N86GscOhG9RG{3@_aTy$MxW6MqEi{
z$K=E1Cr-Mv{feTqPAX+w_}}Z<^Hx2LSz2jg}Ap3DXjk%Nd#f
zEjb>4wR*WXBT$p3?lshY+krAsvb)6Bpw$Xo3gbuHO(dl=qt;y`2W8=PJYN{!W47?J&e+t`Rtgw#tr;qoY%=wU{oajTPeMnIIMHt{f3_rs5y2szT-
zcM5a$iWVfS{A4XIeKv|>9%0mKAzBDZ)a3JJ4
zsM|{EF{;BY2_PY^GF5CZs4xZEw6rmb$K|PGw&dJA#*X&Csme@R@q>e4JB`hmAdF#;
zP@oGis0;ud7T&_;O&qL3E~l^6N@o+~_-FqId+!<4WZSik3ZgU%9TgD-DT08gfbytCgu-^?@b-g{=x
zH^YzHJ4%vuo@=dR9c!)QqSrou6ya%;e$@Oqd0JxjTb|S*({AJ9UXbHfThg_b)0fT-
z(o`+;vpYV}5Gy?$UNN~91WPk4chA|k>z8m)O|&XLH=cSlE$5m%7O^3HP1vr#Md5D&
z9hGycBWb)D@x+>XOmt-a3g9&}Q=ZQqH~)B$Ra=`0;K}Vfcr$!?6HNom;Dk@VR|N#-
zAOq4AyGDWrInXl@CuON8`M5rl@HjeLpsp?4`**|PDk3;puI%>H?<-?T6m
zlXpY=4(M&}IuPw+Mz40~ZT?Ov3c(Ftf%XaxIQTN^cq{(nnRqL@aWo=sNoQ?NEisPeeDamTh@F?5e-%LV
zl6;}DUcgL3&3y))7Z|X%1dBbewOJ%mxst<%Z+GO_7{7sr_JO5Hpc^RfvjHfZ>;!@S
z%fE4OkXci~n`9N-uByJi2UxBGd+{p+5f)HMJ<{*lf9Xnivc;WgH0h?O+NEk|6@$?2
zAmc-(jo2<)Wn+F2(C#!lBz{QHEsPH5YG054gQ(+!ji;_|^t?1~g$i3=XH+R81AKb#
z`gp1|(joQM17A_T+sb0YqMM(HZaBF4Jl;~dwT&qmaIcHj{B%yptv^ol?+p|{62lAu
z1C|fEOI3TRi@=H2X>e|dT-S|n18$~=cgyOBBCC)kRdM%Gpe~NX8CX(>;FghKx^bR1
zF~3%f0^1wm6rQfHf3J;!GoNy=p6Wee;J6N%#-v|UU2X8#b56%#dvJ2da3xZF(@N93
zg_JlBjD16s;Bm8T-h?*6HC%(8LqGNO@$uBMEDk|RKrG0+Te{hwnD{#jU}DHAoax;t
z9(QlLQf#9WnJA_O7L#wiwuE4hRYZQ?mV%?oMUk09#nsW(fh%Zr7n=im?U_3V+s@|q
zkY6dBbmT-ixLY~)la6L-tgXBUM_AkzQdd1zGEV~cXYTB0hikNZu%`e-Z%idpjD7C}
zaz;Zz05cJ{>{QZehaaD!zY-e~0B4axqisQ+H!R)gBdW60KIghKN9LRZu
zWgTq@6bzFmO5L*lBjtMc(
zjok-6D8q+J&GXa}Je)9SBI&#$%PrxEsY4IoP+w{MH>I
zsB!6&v@j}Gd(^Ebg%pr1h$pcr(iZL-mmf~knIN8KBRwrG%dwZzqgagGEK)UwK;X8I%h>zAlS>IpR`|rddwc?q1SOqq>omX
zM;esC!=cI`gnWC~6w}U!_3%P^Oi)$Z@BaB==Np_+Na&j}+nnG7tQh4}P%LM)UkCGm
zGVdnvu%u|>IXOf?$+VHEK~>Gu<{(k%d((5_K$+gN1`@~+pen{^7diIY@ah#oJsKk9
z@=_XN=`|Ef%e3zhuJXNipn3q$;;h}@)7eP7O20J_CH^sUe~85237?CML{SZop*H|;
z*@)Y}Tz1lDk!loQP*r$-2H*I6?^fu}6BGzn`BbqSrTQoD;0)GfB$Yw}%qyX~H|xlI
zN~;-YY;!6Y)sx0Uzurg({@fcb>mvoTPq2(H_)dX53BttETS?LE+nuz1Mv~fqyB!(H
zYqIzn%*@_$Nl2!$yp~I#@DX1pzu{#NX=<^Z1V~#e^kZrSBT#wRTQ^xLSK)XpRFNu8Bx4XtB;ad6nftjq@h
z2bTaGoD(q!cIadpJc&P^jeWam=|e;Ytu@O;<4qpc^u-CXNIxF`l+_<2A%jln&R>&*
zo!-(CgTf$iskfSK1yq!_-F{YS#2b%wzwWV2EanIKDpY}f%$4pGXuNe_I(M3|Gg2;r
zPfe`fa2?iXS0gebfhMO#3XW<9Yx+@g-Xvhqt+5vT>#O32BQu~R%m~Q1#gUl&tmVb(
z;F~`uQB+`
z=Oiy2?ho5^b}XAHliKZKwwW=;J$(owhbQQ-Y$aihAwnzEf6LX$yujpmP_8b3arMW!kZEjekWw7Aj;-HUpTyO6Vkqr4}EYp-*C
z>EH%2RPlY&1+pdpLIKg|E=JDFNxo0usecfNpwY~Q=K>sdpk%?KcnF!rGti7a?_}iU
zKV*7~4yhgngf?A<@zlIjjmN;P2&uz(ctQ0LoTeAv%SQ9god*n#H9oE-Z0^31215Xs
zhzx;OBc2q2(g{1H`c$bp5T2ZzY(52VcAx9jtdWD3gI_7Tg>>)b|Q(GxqYwUeO`&+?m-sE72Y+MaWnjTzW|%Y2JxsFG||mw-W%2?oi(
zU;r6}lTdJ~1j4O-RWLsQpQ51%*Z$hxy`CJL_3`6F2I-RPtT0O2{vaP??mf`VdiCpB
z%OyVK=PaTwqjTK#11k7FUwI{EGa+k&$Nd2c9X9+U3M@pe+TJ<&1I$c=|8%N}yF>4v
zuJ)E43cUpG;>n3vz?A;%v}kW_&LD9I)GmLpo3sKH&gXLvpx}}p635C)rSun(ot;eH
z#Ddr=4l0=8bh1@IpFHBgmy(735w&=LhASl<%zuE6r6Zo9OV4v<7XYcOj!;TUf>N@s
z^OO498QL+Q(P5tKg9UDDTib*hY&!cuKepb7UrnEJ)93yE%|^uhJkR-CjxyO<1qJqy
zbjMPtl-8TMF=SApCTET!@D0DIsu!2@*1L`W-R)?1Iqf&O~O^z19P}*+BTX?
zJArVX#=YcAu`dm(JiqY!KJ{OpAL`kgxo%=KxD`~|NrsJ-wD>f-w@@vb00Z08$roVJ
zIKI
zNdxlJ9V1^!+6Kn?7d<|ztSz^x@E)x(dN@+4s_v?{V}tzqC)Py1;8nsq5d8o7=F0z*
z-yC2Io}bi5Vk(t(Lp&|aUgd2FL>Hw5{{_`gif4UPO=Te{J|d)fEH;{&g}?>_y%hh#v+11I`r3
znf_Ck;ppSRI54@iA3Gf5mAK45u!V=R=EkpxxSQq5_*N6
zFlI$>uL{PoKu9>|KMH|>_Z|N?>u7HO;vusBCxZ!4h+sQdp^y19G;AyhAtG|d
z-Kd)b_Upf3q=8%+(qg9;g|MakRC<{PqzXYm^TI_85-9JWV;c9-v4*gy5}K$OM<`V
zj6_7L&>LI{!>6H)Fxg`03$Wq8P5ATY9MB(W6bT&xB|wpH=!-JhJs3x|6cjiOBO~7c
zotJCj-c@bPf>Z7e>Q}3N9sCZSl+;3(w493b@{<(dg1a6^zj|Bs^Jb?M*2mbYpe%@8
zke5Jz(c3Fh5uD&Bbe%rlpzE2WA2;%}QU4eR*at9@jyT$V`<;tr5teCM^g7je-gu{(
z4Y!(3e&yGn7p`mq=o<0W`NySeO-I+0LjRj@EH4ME!Se5@_n^jd*&{FToGx=dI$N@2
zuAu}DF&%#ms~^};SE}}1VI+&yz(Lm{mm8ZHGuVh8En9#L2fkOL6OhB7-ezxPRzeB(
z^6=zRjhfp+Q*V!bP6K#XpMxf4;|kNe}J{$
zu`bJ{!?pf?U4uG{0&b$;NkR*y3#4ae1=f5~+!bf5ci^L4d+!B_7P7Nc+Qlv1z
zZ^d_`P^2tLcQ0iJRO;d`wNEMV2WWJ^
zz*3=q9|rH~TYr<<^5`mXiXS!#I5CQQ*2L+U*Zf7JY~3{;@GHlWU};eLo0S=EEsiND
zPR0^&Z;Glfz~lVBMYX=+@cYi`+c&k430C-k5+Z)tw(zLSr75K6Pj9r_QNW3jv#|va
zcfziDte%B-ZnNqN_zZ)gh0E_>lvPnD66kc&zXzGU?;#oUU5l{u^}^IetD?h{zbVd97z>OqnNOixFz
zHCmaOJv&0eNaOb0)KkOQ4XS0bK?$Vv(W3Uw>{t8ME>gaX!=a;`T+w@HBtB;^4s1Lh
z)X#0%y8@4?14bwXXBup$my=*K##p;KLcfF4N
zV%#ivlXP(QjR5mtnVq0(it{ZM`pNZ{(MnSoVP9Fba(^bOP5~OZj&)n^zk-?3iKaDu
z&j32l(YI%;QWQbEITK*Fj-^K391>v>k=8~<#Gy!5*9Y);97)O}?A<)QqB!u7yfH(+
z(Gum>fvAOj^qOb{T=7gcPzcQ%o1W(L*veyCKY98hE0BH(S$q*h^-GZu#$74g2w|KY
z(pgX^e$6EYbW%e3S-x;VYDi|3*r}eatk(2GN*5GifsK4Yp%IrsOo;RRc#Xz<`EP3?
zpS}1lMS9`1+J9Z$v8Dp-tQC(J$5N7Do}klKzp>=wG5v~0!=r;5)y$bYLg?z@_FFC&uo01G0q7&L34upPVTYZAROYP|@Mw(!Z8Z6$p5VtG68lE!Y)>gbNtzGL#%(Lp+qF_UgQ);n39g*&{5#d#$1MFX@Kvew4L
z)^5Ln?qB#Q?D!%lK6}XMgK#JI?3hD5H2bJht2O(9#9h+!G`C^xd+#PuPeL2Etm=XAXKsv@Q1(dV!qcDA
zUSv`DU`}KU;V;wE`pU;;-Dj*!?zTcAVPMBZ+i0WtpxeAP{LA45{3ac*S0rvjcTnkF
zD_S?Kb~8^#U)<}WfK$_nX8FGVvRkNsAqXDU05mT`_6?h2++P1s0)17!C9oe(^?AGy
zYzTB`JNV@k=0@w}J&sdnf}?m6cUbFH2OSHZd^Qv=is?UFb%Jes!e5ddEt4%A&s)no
z?iki<>vU32f}C)wx#D8%@z?372cae
zT&qK9={Tg{g<7JbfpKF#18+v)*Y^i+9|%@q5=WSW;JIggZ;ewxyKq_$iN}qus9d`b
z9n&srhv)@M%geQt-Jnxz2{)mH0E-7<0Hi*68Dg1AO=h(ckmgZP$NM+AIjCR{D`1?!gI|)_kr(c#1)R(Sh
zre8o}G)}ySr1gBi+1hv<15x#aO5So|BAN{-J=F$3L?vdz2EK=gK%0!O(jDX0=R{ba
z=!dR>osBg5-#Q@D)$UMIJH`g1cTn@sF}r;N?jE@#JiYc#DyK>Y#`f-{6SXLH__b{Q
z=5sM&I_h7I_bxp!br^c=g<&_I6S;-8S2oFz3WTRx&xnz=jh5eqso_(>J*K8>H
zL4{iqtjvp(_xEA$mN9*74R1Y)b&U*R9~1j**WSN_k&ewLfs+ng;#YZuIr=ncSP457
zQjge}MOhn_Y~1vrx{dP?l8ZZ@{0YVK1?Covd$FaD0SfTVWBQ1GglybHJWoO!i+#%W
zo_qH=b*xMJ&@tI=q*Rdi9sefw^dWx;@Uyu>!qHVvI}1uv-W~4D%UBn+o#1L19c#Y@
zomm#h%hqhQ;bjP%%vFN~9t++?%ezApEx_CKp#=mc+Xk@;3!n5P;)vOpe2F(KHO{)yytor?kcFgE8$DXRTjQi7#81H@VrtjU%sFTAO@>Ijd+o4|tGBB9wrpUQ
zFg`vd*7KrN3}Sa1`gaN}Hm8r)^oDG)K7W1*`1R2`&FtCvE))$V$Ni*+nJp$ukzLG|
zo~~pjj`Xe*d*+>LH(b2nbGI%kZ_8;Tba(4AS1{GxI5p@u(d@3rxDBm#h|N8-ItuVR
zyS;p1F;r=;9WR?xHTDg}RJtpKxM=J?bL7?escYu@=TN<%O2uy9$Mi{5l;vQD1lxQ8
zIAxhWxbL^9$G#02;b#d5B->l;WeEcBkdJE#2|ZyxO){Pe=}_^4mXI@;Figr*U87|4
zd+T7Hj`y!7s_kL@p`o?Fu%JA=Lp5+T2?8$ceK%*Vc5p)cHIGqRH^l?KiM!(8%|0w^
zxT3VfSrVW%BC)GWDb8ir9&Q&`X`mtpu8%|>fA-$L3@p9xXOOynMQv_ImC8xD%177u
zb9Y4Ts;@Y}^e@jVo!-dVc`7p3mFK6$KQ^)%UkegNnHSZVYx7LHX&WYEIFBv7&l?FX
zU7?|+R2Xl;9ga7DdI5?EWSXbt9Me7G8~nWa=i0W@3G~Oa#4U4C)%MuET%@pyi=N?#
zHnhDyPz4rdghcE^`=@t|PO%>q^pY|=C0^{sEI+OEvYkx%pp}CAXDy?fKvqEQ@#@s%
zYL&fVo!jTSXZ0XhynA8`0S>L*6rmL`JBqcy+5P^1Xm@4z>V8%{cEYG}X$H!Nsbxu1z_x=6fP$pE@sjEnf9Gd6CO&vD#7#j2s0`u67De
zyWK8Q>9jxcH=)N6l%HrX-R8Zo%9^t#uJ=W_(8ECL$UEVj_~Oal;sAmlxIH62qwQ?C
zQ|;ZTU$oJyt^(8=-UK1NfFR!B*Y8^%=nRv{dW{s&g;sU#%U8bc_MQlmPk|pa``_U{}4*
zj#GoQ1ESWG(-6I!Pqbjg3Z}mKu2~yE8X$&^;zd8Ce2mRdlilW&q(KGVvt)n`D~@dF
zxYQE9E-eq_V|Xtm<0?9OQY%76`LGOIhj+$-{n%AlopyRLn(gsV*^lw@7S=UKFb|WV
zweGP>RhNVvmpsg6cBQn-Kru^uJwj&wUVmytm6;-(**;Brlg5CM7wU>K#?EC2dWSPe
zTvKI`<{x=jNiu;!RUuPc`#o;#m-j?SpqNiw1RG&bG&5tOq(|e|+z=uINKU^amw`2-
zC5*i9bqL2hIC!$}Dc7Aj%6-sBcQh8T%wLNVqqvg!$z{U*(w`TY*XTW7|V
znhWP%G(6iig#}Y$c?7GtB8yrN>n=MZj9MP{8My<#;8;;(>&kTnSX%@M(>
z1L>ph`}@<4Ys%&hbqJWvi*?pK-?%STywq|ibMC@6Xr5B(Fy!~MWJ?vS67G5L|3*q
z?@o;2RdZOhXZ4hlkd^Z(c9X5UB
zAfFSk_zHFV1ijXUYaq^J8i^xS+U$?)mQdNqw(7DUwm^AXEA34G6#MU=`81oZJ5ZIo%QT%s^
zIa^&;`R^=1ApC!ao-Z>K7<%Q|=_7T4xRpXbEb)$Dk}zqu6zD$!Ved@)s>s_LHYX)^J-
zX0g-#hDiMMeDqV0y0cpJCn4RG8BeI5Sp<7gvPnP&dXe7_@|o+J_=2--i|P`g;OxiE
z8eAxjJPBqEx*Ffa$KklnvQj~EXyzP@f8o_9yGs={n850(&O=bD$hDx^eD_9oL>-*N
zJzwWcs@tpFaytM|nVIF6_XU|F?KMApthNaHD*F7h$1N61+|#X=dd$VCsOv_c?qstC
z(-0cpp0!Vr4;`;o`3@AH^27x1Eqqjed(Cg8vac-9@PQy^_lLo=jIcQ%P86N)!0<7+
zp@jt&0{~d4x^4ZSy65^l%N^gzq=Y|fozfeJi-fW??o^Awl(tguY0r7iy8F^(*v21!
zAjr&6Vh_tZT0#xEheM#rK;atm*hoIr-PQdieI)aU^qVS?y6A&RsASa*B~&!xRnI@%
zciv07OFOj@SLAsxs8#I0T$tZ-t1sCo5v>x~UFvG8>yl0Ju){vkTPm$h3tU@((t5bi5|BlK7g_`2p(9fEKT0Z~kHgCpgDWUAWAPe)SNRO5%0Wv?
zE6cz=`;Nj6-vpS0kp9HP2V%YZ^SJFN&5O+`q#t^>=BZ!@nJbXMmSMW)ll~RX_tz0W
z;dV3$yPWh
zJnIban0Uf?pKVnZ7-kuvUe=hTUNhW%r0dA|2kAAgW2yL%?23mkJU{xVIWsruc-_&Z
zxHUn1VS*V2p{v#*7Gql2q@KC1|LwU|fpUQsq=V4FR-S@{`-d3n=!*>Ep#*l6W
zxo^CZ8UE0dTWoGChG|WN?BCmE*fo{TG=x`U{S~tkO`;}AdQFBl@{GS*3_1KFFKtQl
zI$64jzF$SCB;yqnL$_i>1lF|NV(XlwfDg>kCjqGYvken@B>D4!R{MMj<0GEzND21Z
z;`}?3{+XQ)endL_OapbX^p3I4ruYBJ<8TJY2NOPsNr4gqQf0jkpmUZlT>=R}W?p~+
znr_oWB^r&?X=Wqk=_t9N3<4))kVwC(9ZW!WS;0Hsm2Zl!rlm(|3o~k3P>`y^pTQFT
z3@-dXat2{agAUYDPOpDB$fHz5W;&Lc#(TdE0(A9r%ya9nE)|{C#CFYMgq%qglew@G
z7-AflLf98g%Kt%snB=U9Bn@(23E6*F_RE@>#qYCvCG7FcAN|>v?>GE>WDj>bjYJ-L
zS7WJ$EYZ0cpS=%=flMTMeN$M(I%gt2H03=0~&k46Aj-aFS7P12aUOB@+
z>o-4J~ru=eY@!EXuNDfsV;x!m-I@p!@Sq8alW=3_}02vakQ&cxk?+Elm
zrXqBBiN9SGntpU1J`^{8!a~u8{ECmSB}J9#p7JVG-MdQUZ&5=L=4W9Z0qO!5N%TcO
zu3z`42BLWb!9l*%WQ~YYmc9oU04~;D+bu)-@uVLd(C+M6rvHqrjG6)os~wOMW(BZo
ze4C6eW~U&I^Oi2;HF4=9W8xb>^`?gJCYZOPwZO_Dqp<7#PQ(%8o7rtq_1Jo|h*apP
z7HFW#90*JNCzUxE)-*Sz!SP+O1lis!^Y9d5faJg)0m7XQz%0;&^!0xvSoMWoY_Wgk
z7^KG=;*xxCkVq`9NO}xX7(^kqPWim7gxSYur|-Y0;$J#$boiq2r*H2g`4<-c&iJMZ
zpt{V~=0#=$YpWh`KT8v{`zQqpzwA#HTGEGgHkwyOrvI#g{z)x`*7P}NM5(i(jutb?
z<84GRq@-jSo_m-KX`_4%Zwu?Cb~N$L%gB)!^IUV?+2g`IKK(b>=+9g4vN}TcOFeOs
z6I~@fcG1Uc(i@W8>8J-RVe4cP4|`!7R;?@coI~r#LMgGt##Ug_Ix~+-px)xYVpd%0
z$Xib^Y~Cw&B+xUfJ0cHD7dHlK$`Z6~>)e_eyZzUz)&?XXF6%$#A5F$l*;v&CdtSwd
zx?$e+MwzZ$GQceGl?a*pp6V~@KWzG%pj$bCkE6)rU9g=9wS
z1Ra6!sM%;!o(g%mD9u3
zX;nUN9tmna(OtbJ(JFDzO_r0o=*wyRg~HYbn8tnGidFQ-nwCpmDuBiOL;sx7(2x+e
z_^b=(M&63wlt`DCff{7cexYDvCE(SDOpxACnlr}}Fkki^WY
zUDB3_N<6?|vWCsm5hP=y%MUl>-#;zOClFS%!mg@mH7goEo5Qv&J1WTcC80dApgZxN
zgKTuvu}Mol%fxNwMafSJXYmws3&lqOdA9u3oib+en-DSkx&F56g{e6Y+2`v>X>X)K
zrTtKB!GK9!)a!$qOH_sjY7lpCOh-iOY}mP1xy!+jpm~l-m1WnD{LZzLf)gY}jMMN!
zxlhU1v9Wcf%{-~&k$k)T2&f9N2nC!!qIkKZxASfW1$KX-x6@&|-FWt36qMfoGiMOL
zT-b^L3WVtbt2<|S_aboES6}tt^--yCPl)<=eN@)7$nQ(aw1uy`EK|9B1!e1#x%NfA
z8;MlNYFu=4%FKTM%cKhUO2!x~-JfGdIV(ykxgmlV94lQ68=H^bf+##(271)(M^C;ZbSoogfxrdUb~VGbz*KD(!{k6&OR=lG4Zo~7J-@I!C+
zG3i!t8mH;C`1qfP#J+V(80{@9GFGyhBJ7W4UIAxifmMClq;S7VA?
z1+l+u#xyVLB^(U()P6iMwtsdH539sMt`4$=qE^{|2&-nmBty|`ArNd_bc=Nnb$qv&
zFZBTMRr!nZP~U^~$ty^^QFTPTbnQT>+kAuva9gQHXAiW#!Rql5N}ri_9#Epij$~Ld
z5vLX~2N+iTLhVqpZf`R|)Hzu${dHxfZnf2-m9>NITE7KsaLu81d0z`hP)lDB+P!#Q
zWF_|=0-PI4bjHGcF!;x=V
ziNf_oDvkOl&C4ZCm3+wx%=fj^!9$<4cTe5c=;iZx<_D@n+K3x1`?|no)Le^xH;bbJ
zW9+4WtQn$Qs_Yk(v?dmdk0N6YIsOdqf0patl!RwWE!&-h>I#B%E_qk(i+
z;o8MXAMyPeF|FPFUrAsV<7cGb%RED@(klu1!c>>HGXZZO!-Pf$J4g8+wcJMgO)mcL
zYQNt{mRN~~{7)THhhv!3GyJ>^!xM~IWcm#XVNXcywAd2|*+P^)(9sIX^eRx(Q-0HD
z-wrJydMZmNNX|`6-Rm7dM|nSPg263;Z_BHrVKlW`e5>QF*G$S=(JkrDn5d{a%du5)
zY^`|y3{rq>`{i>R0pWV0JS$eJv8y#DcUnP>RSQ&b^vV+Ccn{?s=$Yr1I0h3bp!iqJ
zQJ1-?juBBUpYR{%$!HJfE_Zz^n)2!q#nv9XR^`Tn3(+Ov49%gH0&gNHDlgaG#Igm#
z)$@k84V%^|DP~{Y?1%{IeadX+>9E4)X*G%gt$0A|Y-mMNdc9>_g{qM#^DmogQ`DBKCtj%JcAYjdQq}&5()qY^8UF
z*c+FYLLYASo5_>`D`8TPhCCg?iq;rSZsiHvN4-NLiyLxO^Yvn7Uhj#}qA;VCf~7a@hUc|f(pVK@
z+f%+vku83dUdMb4pcg<_O;*s7Vd?BEiMuJD9$8@5t3BF3TlGHB>#j;3;M_RGTCn^f
zA6EC99XXsG(FlSsGTtu9f$s`fJw-O%IWhpDQ$jO(*!EWyW^15b$_5+ZcOnpOp$qob
z2F~aP93y-FoQ3=o7!zz1m)1T1l$Z#z{xPDvBoDkwq67KNQD9=J)G(?^k`=K3#El4n
zwCrDzantj2JBAchbK|m#UpUB~A0s1)l9>&5Rj@s>ppbaveWPndPeoIy{JKWD&s+oX
zG!@WXc?mu7si9fk%8qSxAo(nFO(gvFkK3t{DCeVc1HluNi>ymcE2eBcOM?zPNc+Z2
z&Myiqjiusu=~>vXE56BKItsPI00*
zrn}d8
zL5bDsz%czW>f^S)O;`6mrG37e%YumUJ)J+A!^(EUt}~NKhfZe@Dvo7Q!#hrUdnCJ+
z(z}(~Us;=8QEhb4_J#2|8T}&8TsWk76n?OA1TLET`Lri7@qm))V@&b
zBK?@S-Jw$4(2tVzkW+9;NJ!VbnU=_;J@|y~d&%v>&#PiJPkSrWiTzAEwAUNgG?VHt
zX}Y%r($F?Xd^{c(H!Qi>&|ff#4HM15PSUe$4?A;6z)Q5_>icVee?k
zo*r$=xwLE=Y|1_nIe712NO{|=-eGMM2w}P3H8Fa<3twNKAKka;-3&3kD$a%cp;T&I
zSz_D5kb4cyw3Dxse5N-SqZKKFYM1&sIZAM{P%^B3fx=Nrh|g&-fu4N_BQ!g*xwl}!
zQnA&HByJSH2~>5qZAp27H+ZKpz7z)L;D&&}
z)Fmn~H96Rb_yt!D6|NoD7d^p2C$$lE=>g3n$U|is$a{6QtTw(X(2rc8y;>L2sC@cO
zQlgbfiq+nIlM}zr1LUI)__%QF{
z+b6}wM?}%(Zx1>L4!Bkgds8W<^A+HBVKj*KM9jkPN5r!u-d-~xN$+^>Kk&Zx9@*g{^tvYXc*Zh7V;)EBponXZW5Q;8Id9K1YY*7U5I6*L{;_q8)f{fE-tIV7`0
zDeP9lzI;CGcGl>I@1ou0+HV@*a)(O`ojJQQ293XfYM$58En+3KTam*W9Y4rLvBjJr
z2g#TVURCH4SN6WXK1X}DO03y_%9Z|3)v<@AZ}QxR*d}9b
zHsNXMde4|z8XKr((9no=`TGI{S#1J^yPR5)_TdX`wy){C?S)A
zpDR@NFyEAjOJm(Y6|LeR{U$*TOddX`#2&gbYHf@)=gv7(1{u0UKT>ogg?@((6&Ysa
zPPwP2>5U-Zh>A;eTDX|m_le}00f?Cv$>8WY1HHCTiVr7J#oV$MTE>1FJyDBoR&d!B
za+&pTX@0wjeB!T8nRGNS@o;r1gEOGgRNU?TWy>^GcK+HF@iUdzko33%c*~HrTd6p|A%R3g<@Lyc|Y($NWYHhni;Qslm|;FUgtjJRU|mFqTMv
zC@L>^`6*zd*lt*^K1bezLnvL+0&jPpc+0*kNgL`6Ih3&Wo>HBio8NfKb^g7>T9?z>
zK|&`xng={lBL%Eqs5DAjz9@rKE#9hIyCfqTDQ=%dJwE2{s$Up@Ff@fA0%MVpf
zu)x$11XsgB|o=gCmN+8q>%9X7!3Da-nq&s90
zODrQpLDcuyE>lL;gbOq(mZ#ZW=O7rZ=FVln%2xMeoC>CvU%{b%GJ5PdV&BHk3%1-xcVswYh3XH`7fSF9}9$2jg5!m
zU?i2*mERTD)yw_5Hnmh&z~8Z)K0eNzb`AnH_hOze9yc#N$f>sdPC+8W8b^+M|E$$9
zHaK`)2>-`_y}DfyCFgEgr~=T=&Q--F@cDUwUqiQo=K+Tk{_T$WlaPa35yN_NwD~R&
zInubNe-a3rfOQe@7}}p?oRBGH&J#bT>`~Zbv^Rb$M-@v3&~P!;7&*_aMBI0ZBL6OR
zk8>ow#eMt|WYI@ltca+en=@k9&yhgi8N3de$>SB{l8o8p_}^IoT(Wm17_I68SZq>d
zIf=6*4<5U5Re1kCMF>dyiZWoD!>>z6-~!LR*;!Y!4yR{pMf=9&G1Vd$AaoHUi{i+*sl;0^EDgZ$t0-JmEQX
z%~%9T^N)ki?Ej8x*_H(Ws7JB@A^r)KOM%|wiMkAU!T{CS`)=r;fFIT%ob4J7uX4DO
zVKbk+q^0^rCgTpa<1U?m-u0Nj(qGH5RM4e^bGDFzPMQcZ3pg&E*^&@1^T98>{}%TN
zjCrtpPNev=hVK2pUNzq12B;-BaG5_Y{r(iVj_-)^cFLC1VI#%89TC-?*AT`ff7)Q5
z0IOz-`Anf^{>PJJs?E^tepoISnJkJa6Xku2tM~)Njo4rfu9Crs6JX@G}_4U
zU)HFTR8%_T0Q*#d@WTQPg_DqPt&+R*snKGL%YAml8LmYEPuijcWNM#42JV(UCN4c2
z`(v}Em1qI_eXQIhW?NcAhXJK$LujFQaL3=W6{iMV0YFAUgxu5u(TW~$koVHKo5-Q}
z=*RmbGY>QFLjkKNLiV~^8+ZSH@UjMl#cd*@2)sLL;r-+P*X~F-V0_&AKQ8!xT=4(6
zAQZI!-^_^rb1s-m&7F5!hDq2V>$G8|O-`KKqD)q**gY-Tw8;kp4#O(qZbx?(MEo>;
zYT#Lo`x5j0)h|rNGaRl@KZ;FP{)Ty|g}#)Pfok-nA*p5f7`iDi>`ODNJ)sLi=m>b`
zk4miM8<5TK$l}9rLV#gdEconn_*|d8y?I#@SdQ)rF7^NGReW-&UIwshUsk~rgy5gF
zcnI<{;TjDN9)N!~2<3(^kHbMmNY$?|K0~a8OYYg-v?VsElQo{B$De6G&B&HL@aihQ
zd_Y6voztYM%dh6`c9ic?zKo)e=O^7pgrWzA@m@1=sO=H4_0}
z;)UEBU)xSLcdlN;xX`73nZK=6=+-G`oz0%(um!udnsL*vx;2*;xxGSwkG@07$Y!uq
z;$g{JyJnlk#t)^-JCXQw%ArgOe*rNWM|jM*6Lp&n!s-4~_&@*K>jczB4=S>)MOHTG
zw>Zk-frsdjPySyR`3(Z|9|kwK`I*VE-c=lAcTDg`yYDd)8u6<7#!639S3=kF&AmMe
zfBqU84zifkYqYIKk}w|o!F``5S6Sak;rFnXrMi!fOXuM_Z3lyu3(7Sf-IJW+ix$E{
zn(6PdnM@~s&c(De)9HpWjG?5RFSYv|NNU5r*&EvUiA=XOIdrNe|GJ_f4y45$^B<*3
zP^#66JZm$j%D8lLfdFF7w-Vp&faeC)ngSqI5I`*e!RBRZd_5pVfiFS7T*h4+#*|c<
zObBQ-DC5vsR1~o;L<`6`k`at@84NkBsa?Jg8KsyFU$@?yf&K~jdEE$Iqr2k;v2Osx
zy?^~HIhhpzK`)jyvXsAY3x_cj3K9*zY&V=c@H+}-xMrY1AJ_MA-k0`s4ri??E!j|g
zvF-ZzHS1nQc{SpqtXAo)J&XEJ~MI<~v{%T;*_-vGS=D_l1sr+C;a;HM7=
ze)`X=1jl3mQgL;bwCBOy`~)C2gl==dUgt>yJi15?cz|ZDRsF?8fQllc$4^q-f&L46
zfSQC+V$VD0jjP?hT)~$BUq;6V=+LVM~jv)AVEK7t9+IRjqr2!^_I9iLd8WkslC3Ai4|NNU;fUwx`lj|(YSd?4~rW=O-?
z;}iiu@0}h`P23H}+2A$diu}mExQ#Os_(d!L>wjww4vC{6Zxqj}r2Vm-g1aC)wF=BZ
zGYWQQbjFIBls&1Xq4qb^jX$Xy@)mvSu5W94bbN0kgQ@VMMbdQ(`q@Uf-mkTyI#niN
zKbD=(lS7xC%WrT}l1HK&h@uCO!GlJzxf59~_Ns_I}?gXK@yq6_Kc
zE_U7CQA2)lM=m)$L8c|?t0sF3+g>VhA2Js@@w&icd}GV{aKg`a;>INgrE#N-0R9g$
zH~OU%%Pev0
zB~}zNx8!LTztto*%)PayEzhyiLkY*}
z`fchwdecR_tna67RP7j(lq{QzWm|9hv5pzGO5PeO08pZ}MYOlfT}pb81yl
zbrMfss;446_j^N%toGDiZMPk#y%ywe8BuFw2x}X-`(qvPo3mY@6$`@SGn5V88-q|X
zU`85hvSyW+dhRp*P2vo}$t%wGZ=x7O8S00!_=lwf;cnuro6f*By)W0&8ajYn4}9q{
zAs-ZdZp7t>5H?)!;tFG0#P!?s@Mf!>pg=;N#~St=Uko#Ynyr`i7y|b{gP+^X^jxlu
z(H2=I#^r-!#PwI+f#?4TT;`<%Owp?F&5Ke~ymW77Xw&u;0o4H#tWNX~#YqKscu)GJ
zY;%@^zKg!~Q&(e5C)Z0SD*5I%nfM{hVQo8Txz+A1$mZx}wmSITvWEJDRg^j#lWP~1
z`jZwC6P9a7u@(}@=P%DuAfJcr|P$ODC#}{Bn
zK9q+R1?*p}f$t0TW-I3~P?~<>B!&%^>LinFcckpG3(os9iQZg6H3=pt+l^p^TeTUN
z6vK%yOMTy7T`Qz7lD2rrFXs-Q{vJAfMTU#`i@xYG1$Mf&GVo)E=iQsMd~GhP*NXQd
zglu(s+-h}pAC)h+N5)wlfzBW1`CuowpkOXf!>;a&7<3)o74cX6-%bO^&~d|xDG7%O
zab#;@y!*Jr<0eo%@QaP&TAIi3CwdXj)@f*<8Ypph{21Bx8(h;nF#p~?si>6<%7Am@hL|#?P{y=$?`=)xnpZ*pjj@MYzQ{-ekT{IX%j^aOmQ_-10
zk~$)Evg3k~7Hx%K7m8O?4aUZ4YRg~mTgq8qMw2r4v`nAvnj4ZoyzbTi>#CS;`QuHsNz#2B17Zx5^Zx)%
z?V(~ZpRJ%APVw5#0NK+rqYh{wI>(lVQ^&d+Qhu5{4
znLmkNscmf~)2f<2--}S^Om%*6#?>M2GC!IACOzKvI=WBYg@a<5fDb5Ex+-S^}SH;L{+%3$jFDDBO6<=^PHmFZ(ih8x
z&)^D_=*4*%h+f^;H`^(hoU4NFHx_W3zIpeW0B3uQc!>&+PT|~ND4@H!9gJh<(*5qR
z(?FRM>;>U2ym;)B6HsQ|mv1QYp=%U(jsP3V>6Ry>aWR{XK>f1mJokbKD+psyL%l8T
zQoefi+UnKBWlmyVp6^lT9wB!SA%hx68F7hfj1_{iJ<4k=QB1#n@p4*e
zV>3=TDs}xY=H5D}3bkt=7C|JW+(?Los30vZjZ)Im4I&`jvFQ?|OC+QlY3c5gknRqp
zyPMy=H+s(Vn|HqVkMEmj=9_Wm7#s1dd);eY>sr@Z_h<_deQFqfTw_GE9Dq?Jyr~e4
zn!_Kz3R%9nF@#^+r?4!(PDNpKG>Wr~4|A%qowU1r!+c$r8d9Yf26
zR=+1JuyhTuw3F1u5eFbJiNS-D<`4YsVFqa+z5%Vvn9r2qc;Xrx&adxd84AKLiy=41QBgD~V6)XfZex6W+bBF%m=Jr8BMCnajHY+sDErhlv-5i^|ynfR#bG~Uc6E{PiBDdK|jY=i5R&Gn~xr}HlP7e#=^-kqwsQJwUztvpWI3r2quHUk!alD;H?2qR+zKzB>c&>6F^~cu73vy?%Yg&
zoSSLb-3f*k!4I-I7)U`qu{+^)1|VRToFZ>hX^>xQvmg^%Uyu>St~7{{(ICFi*@e>o
zEZ%A*7n5VCQiZehhtv~;ozJ({iq#&2Qmse-*;o0O!n@cIOWSpuFCCiNt_kU5Jr!^8
zd0JyBG7CZ|x85%>u}yD!pKiy#CtxcfowoS>g4%HO`M07Xd}-y?1>U_Lf8+S<%oP*guKb>zC*Sdb?bL8bWC7DYG|SkX@FOg$pur_K
z3J({iEbmHrEb?J+wN*tv6!kRT*qw(33C6UOL?*81$NLB6-nL-cmBO!J--TLs%%!(Xk&|A;@PNhwAM6)3P(j_j-Br(@pR;njkXmq8@x2b`bu{oHY^j)c+
zvmT8$SB?MP`NTK|W7bhDH_f9j(9O&s&xu@c*Cicl6wvDL2Vw<~Xc#;Q
zw1>Hvd&muwvMbX-4KpqZCc?Ojd|Cb61`;{EfZ@GyTxZI{>v%0XJ3&)G`)vU@B^N-z_W@8uIPMw^j3Rj!gy#+k+wYrez9;$?E6#aNpSI3A%`2g;K`
zR$h}ISvK2V#Qu5y7nOMpP9U2y|H%C8&kdWL=$F&
zxj&WRzm~h{jlJaZc=Oltu-+Ky9e8WmWA1NnOaM?i%U@r^DVukHO2gzH*q4y`H=K*}
zrBz;Yg?vhMqYol*B{^|-AAm&&8OLJh&_`a-i`bnbDC@z>azsgbP4HUF+GY$=M40eU
z;q#BKP-otF7kM_fKIK)Kn{qnGQ-X;g^ZgSVXwB<_B7a1ik#Bwz@}&rwM``U>&QFvM
z+6YoJt3k+(-p*;4Ovi|=AI+GEP#{ShTQS9v$9tcTi6v!^Wh9(v&GA9Xxppqg5YpE6S=
z0_rWi8OB3H)rwzFHc6IzP6O)VxY2hhFe%~$-Oc|o_Xsd|iR^!$`g)C$t9nT~QRi|u
zA|gtzd>~(5b;8>p4c+_1YI21bDw%x8yhf;?J62%2%GP$0dxMyw!APh1Q;>cWL{LAW>vw=}HE2b@+Sezk`7X}24c
zL-LWeiu&WEX|JkeC5a^B>SI~$SM)sl4M>2H7LXVfk8%@Zmrh7_AqfaUnJ3=Y$8Fj5
zMbiK2@G-yQodtvq6uM;$_*#4_v2n^1t8X0jj($;a+Z+4sf~e0Nmw(zzAR<2>$YV~O{{;1
zGtIHO6>S`{?O-+4{q=1y>eARydHC7UQc8*Gxm^s$IhE@79m%f4W}kR*Nc;7Stow1S
zvGCv;Yocu8Ozlc#roQ#*ApYucQN3HCU3#ih9$w~3YxI%qE{sOT;I0*oANzfuAzwcsjxbA`vKL!ApE_3t=z+X>cC7)eq_W-9
zmbkX6#bso$JM!?%*_?QI$j7BLsB-o3T0dtva+Gd=Z$NRDc(ILEtIxY!I8hNV){9&)
z_Je*opdj%zQJF~))xkxEz~jGKfcLKpbK0ecTg1|dx%xoK#`Lvj(L>+BqFxhV)B=k9
zK=NXPM$H;Sdtrep3);w)PI^AgdE#R0*+BeNfCP*w7E$l1XbQi21o6>ea`bQe=2ER_
z(|eLbM_F8G%P#Gx9y`-cpMz>EOi#P))TW_BRTqF4R=d}t3y?zv
zHQ3cT;@Sc^>;2rywtDU+3}@`ANVXS4W*9%VJ<2_5-BNiswOn@wW5R3SI2UFtj}M0%
z0pg7S-np(N(LF^~y@ct`7y(tS*yxM@$B$sG{{M)ea9E4pBSi+MvOxrfDJI(g^}|2B
z5ZG7M=#YU;1p@}kfQyA`H@;b8qhKRw_J>+d@9g&`J8iR(zSN_d1Q|d%`gs8A(t?J>
z3iP_fhgIgQ>jT_ha-!!)CG$B!VFirZ!^P4)MogkT3{BO?QLb;;XE{BDuhwL%T$=LK
zhZ~kLlQ*V%kwo6f5^L6Nk<=RpW<4`w9e`XJR}v^j}Cn2IG?8n?5YvD5P$g>^(!
zTl?|mSRwO=_?I2e90$p20<_vdePh8>qWt9rs1LhW>gWnGN9*%5+_ql6SwB9Pw!p9L
z9|vAC!u`=zCQbkbyozd^f
zmn(tIGmK$l!{t0l
zs5s*f;vwf}7p>PLUe^679E(UV?HC|)7;B>e{%HHGYFcJzQjo;e(Uq{}LBQUZPf_Uf
zoFzOOJ$){D^5gxx6t1%C!lPD$p=dtH-h&5^{3%CRwGZ?xGdBpg1M-Q>LQGWW!bD2?
zBVFqEecoJtuUC_7TLiR#kZX54Ab;5*OJ
z8UUXF5a?m{h9~+4|C>TeApRr2JIW5qc1mV8&-npu7pisDk1iul!s6YIHCC2oZwDO4
zAs)G{6rASX;xs*6ImW{2Z#49o;FrUK(6iQ&?&!<3551k4@xHV(Y?U0lTXk7j{ZrR2
zek2YC)EFc-oJzJkC`fHMgtinxivNm;gNMv}`By`YN
z_eL$q)`mYN4V_@CgCWbw50k2`3T~s=*bh*2-eY$-$L;Xe
zu1UQLfCvO8gyC%p9=xxbc*HiAlaVbgVr^9BXAdbhV$RP#`&uxZY0@+%)O|wz0cwg=
z)KnVbdl{4Mnk1)=CEw`q#zR8YqyC9#C;WiBlMuWd-a9>=w-y^S
znuDq=-b_u*6}Ec$L)z6zl%DH%CuS1FOL;ENw_4Q!W-w9?$H5CqYnDpDSj8LI`#SfL
z%DrZvgG?eca#s`H>Ks|
zgU*h9VV05bZY<-leG``pF@)=KX`(*{#a~j;={WCH(tr^GJU8D)G2IG@C?ryGwqEK<
zE!4?3e%~%7h-V%_#`6jo
zQE;dHhZ0qK_D*oe8s>=4s!ukuqk}jrPxbsQg%MFu9v14Qgo|4Yj5$Lu&Hhy+#Y+#X1SG?Fim?$
zSC~lgm?@H*J4Hv^X|JQ|u*BxhWR!p$DxptfX8aG<(x1_M(jHG$#S5)I95R1*D~M)o
z@bcJruRiY8Xg?Y^8u?b`G1SbFc7{*u6Cl?%uhMhYC65_)
zB@|Ng^Dl}f<-$?nj;p!NTus;Cs@>en=d)Go^m%&k$MPymJ7lR*yeX)XM|fu2BLE-AVFiHlZC7HHSTsl%<9$?EsdIJ{tr
z$^v8Tt9VF-{eiBe4Yao+X94Y25Tu_XuyV*QC2s#sP3=d&9vCN&{<`&|COx)lrWP*{
zCdH`J)TmRYjX0j20N!q4@Sy|4|cay
z&+1ZUIKCro{pfnh?U64RY7nlgA0n^`ZGy)1H!?mS-k9r(yjjR6^_)CHo;_!g!A%p4bKOSJdi9f-
zBXVdYWLfIew#-C5durk}uLDGzf9eKR6<23-Ql(L8ii}y%hctLy6
zUPKxgTzLxFET>ho>eMt*0E$2;DmV>;h)vdTl>T6iZz<+B+nLL+Yd
z-yxQ}>Imy#Q{G13MM?AA$$u_yhfj~7eAlMQl{g4Q{W9f
zlP?pbI2J-*8bhiKU?3KnEZ^=MP`?;immjFB-K+!Lp^k%=?Iwn5uHu3io=s6xb*veo
zG%4+QW@P%&O^~hPp#>L`z12*8S#*C$e;pBCKN^A1C$^zZ_)+vK?Iu
zDj}if-VJlZm^{$sm8UNFmNF1byaL*}l@C0(H(u;c$0hyvERrOEqHA#MS&38%qAwkEI8=p!%2$-~aMRRS4(NHtGd~DjL`6jn
zo?#IA`EB9RI?J7NP71K~_ZbZ&7H%omwm@(&N$sS-1m!@~KsuDVOYO14uY!-kkm6-yr6kF7AKdEA*6E(7~<*zg!
z0}D~SpqFeu!Ny^ttqwk39u+M(a7Di5@SUy{l&L@D(@$rhMl-ayo*Bw?Qrg_k;XC)Q
zn>|IObB4Ai_j1`K>61K?7P{k*(zF|-LD8UAYxa@tcYe*nEg&_Kk_*-67({zQ@=VTe$0IuXKAcg)b1qMWY8GEMn?mh5k;B6MPq-X-O;h5{bPZ&g5
zNo0xC)YPmxg@RW73HHYOpZZ7pF;WXk?+r=5QeY~J^DoE+uLl?HN@X%nyg*wL6Pple
zF*%DF7yz*myxYnS1X%P5)_Ak2SkggM-t?^a?%)TO+5`YorLxU6LXao6pK2BaI=(l
z43$0B#~K|uSe*_?aB?gw)=g1#qN<^xVaRml6cI(ZV4ttcLfjBun%A=c$=ILhBn39Km
z44Dob^H}ro5Y-W51)$GMK%s4UlH|W*=A_5NbJdEVT2uPzh}2|fIHSBvGWG8HgZ`||
zqLK>U3ENOJH5uXR?28Bb`?)Nf#R6>8_Rb`wz0$`I1Qa|yV0cQwh|mF4yYB8pK(u;;
zlT(|Urk9tM
zCf~A_?eIJgufG0301$nm6FbAgn~N)kAMBbKD94}abY-1TbLO9-;~Czrw5MJ8eg86!
z$lKRMnt+c&QSD;nf+B@2RU`8@E7u2Zb>-=9tsRP=jtf+DxH%JyBk{w6T-7dia!G`n
z0hA((Jkj=}Hsye0FU%2fPI!a>BJ$m}CYH-$F-N;*1f!kyP_Ds@@28f7=lJ~&BFhF^
zS_h`A(?vCb&IHuy-oE+NKU3AJmgO{&DQ2G?+pyMxCu_kwc==Rg@R1lYFevq>lQ09L
zwpIVz&_5n*e4bayUf#BI5JKM`lP`7JPQbNf))@)ML3j}RmK^{fMu9n;nZnp{mX{PH
zI^mlWgzU<(7)ffCV{L;a7D}cpz!iKJFWkOWZKC`{C#fS8A-GwA=
z$~f-VkHFXVqqwjdVXF0`H!9BCSN6x-#CKQQ+^z+Gm%4r_BA$p80dzAvCi?dMi5IBs
z4?cWa*&U4OH*EExIGDEzfpZ9zN|N7^U_5doF58I~3#&4MeP{0<@RFm)SRAb_(=l@O!0cJUh(oA4{cB^)c%D(#l#`ua>(F`RtxRY!R=*z14d!#D{)18=
z!Pa$M+PLNo0&;mBmuF`68BDl0RcGLW&|sNY@$ScGcd$j4iM84#<<16o*xr3ykA&&y4DBS}VzlFvr(mF00O?+OrM
zVR36Dud*JPj#)QAh+rh4Y~osEDft62^1fOz{rnm14P(kKrCODQ)wUTfL(S@JZ;wl+
z1}tmo6B^4-mu*g#W_g&zXw33b4iroOEMi%VB?0*WVMsAWukqX
zHm
zgX87@K+>~r(JM{V?E^y_kZ~iBu{ftie}t97dRTogiC;uAEIzF6u#o76E^*)+Ft*$;
z=Dh+F~@IEP(oo&pAB&(KuaA?^`CuJDZvS!Faau8SEl!
z*q5(na&qqftgUNJh$l&5ybc%93F*x&X};!32IX9z{)2;|sJqTQhi1E#bXnHL0`Fg`
zgI5@L=;xZMhp`}=leko$ZiecYr7RoZil?-P95@*HJ$yC2nOw?hD
zN0Mg6E_%YofU&`-Is>&GihY0f3zt2G4Mc{Ek?;3`%1Q4wr7cFkgG=p$xeoY*qrzp9
zbk2qRV|HwdD0k8`8yjJ`)Y^{Am&8m2B@xJGSCB3Sn)P7wd~x!RAXA0cZKlr>KU!cm
zaiA=F(#45?xYpDl$2lU;|V8r#GCW)sp?Xb6g
zRzm&c?^c?JO!)z<7(q(*N{X6N`U%XhV{;YNossAqoII%L7QJd;AxdY8kS082H!Z7E
zGgMj|$;&Z}S8A$GUJgtuDq=c2nH7i3(38v)FaF2?or3qYyW_BCTWeI%z=Ch7xnpSk
z`6Uvpfi|I~<986}I}o_gX`ubC2(qmlC@eF-qrE&(>vk1x8h^0ql#*^RWSQ^o=ggzq={_eg%?=pKG|G8ivIMwc(;&+TO~Lt8WYHkGl6R6NF2s?v
ze6sBAck}^*ef%1%3y@EA5oduKA}SF7QVW3-#G$c5ELLwH+VV{h;lYP9@g*O)OvG}2
zRKiAKLMS-5Vc~*HH0(<~Nl1QCq85I1XGgl|yE$lkC|klx@Hnijsn+#U`St;tOmwTZqQY^8c=L&;k;`3keo47Pq
zK2TZ}Pl|vuF4o4S5|)k_bRqE8g`sKTd*+;kdOhg-D_3_(M$8Cb&fy=79o(fA_=S^R
zC<;Z#t4rTRuLfv2S?QW_RJRwXLYRMe`4H7{%8M7Sd7(YG;d`bR*Mkb7x6j*U|Tzz$;E$+Rmi#$qxe!pH0@}za$JP
z7R+df)7TNGN!yJMz=CyQwnxq9xyzcD0SQE7+*!^R#a{uWXvCY5`KgHpHrfnkeB!!C
zwq}Pab~EnUfa-C`AZM`7WX*TkIxRR(7q2Ypn0J#GZ>hL`My4MJU;kkal(6`7ofz6N
z)!jL6F5~S!4}it5C|+3c`%2MAI7J8Igu%!87xPo4&(8_@ZOG=Jw>1w}4d~(bx^V3l
z&(yXw$^s&S8)T=*2Of@hH!k#Nv2Cd+I)4V&K5WW*ho1S}ZLD6`E4&b}hwY)$&6i^$
zd9JSe$FVh+zZbYKwSIv~Z@~d?Q#kOJz7a*C=70SMwLto@Aulj^oL(x(%NyO!03D)$
ztb^Ls$z{Q{$xN9q?Ow`&Zb<@FkAd5o61U~UecNGH9r6@vJFHkEMZmwk^b+I@
zr*TpsU;HdhEp^espR8^~yes%x`_Z8b)$<>j&{<7vx#$lEw#_T+c*HT@wdm*0a8HfJSNJ_Sfqa9#nyOaNZ4Us(ZS_-~Ojaaoo
z-h7q=63qesm8Jc`1^?|0lX>{81uB8lNjp(f34@rlolHG8#s*2s_u@r*)aE@k(qTwr
z3)g<+Yw-S{-=Z)aB;L7Ek5N|S%4*n0uANqEXu|q-$AK!+1G$t515LhoQRACIBwUb6
z$w$}c6p;NCYn_yA4(ny1i`LF~{ef8Ga^6~}erxI8U|MVHx&iZ=m4UDcmih^
ziet7#ZypI>Uj&rO_;h2rHLU3_Vd>8{mU_r{Ix8Wgb_i(>6<#@}Uut^8f8a
z3e{Kx5#nwsA0@Ju=Bm!qUUQ~Cjb$3}9of!F^&^eyKSFd~;IZp%I2va;I&lU_QL2*g
zy^x`V`pI~LYRsYAU4vGss}5Hlz!1mRqOXiK*<_oNsTSpZ+B!>{LU@*vTY}BQ_l@w-
zH1_ujFKw_S7`y=M)J)c<{hj>QvL%U#w_b#P8>|%kT4@i1(BMZd%8(+CscZOaHNF>a
zF<}DpuFyA$cI}?nbjlD@t-Dm9|BVJpQ+WTo+H+hxw7zbNUqR-2>Y;_RFZ~+B(X7)e
zr+jQMNz(B6_QqSnSd{M@==;5?R>i(o7NQSJpNEQw!JFj-q6dZ!he*<7NoWXsAbe%O
z!&m=licij-{9`OY88n#RxDUfjkMQwM&nA}NwgpV;$ouF(B
z)O9UYSPCS|9MBRxz=8S8-;puO3X;th$-)*Y&*sk`ctNTnVu?jzb8C!^QDMv4N~;SB
zUweQ0eu2xbaP__LSIEYtX9pquD5`ifTOpEtS#^y|w_qZe%Q@I5Rw#?Tb4B
zB~(t___4BV?@t`U(%YQcgAL<0@OtqA_Ped=lROYzUxi4zxwo5kvvPK=RJ@;l7YH5t
z(e>W{YOG2jXEeIv1*ysFWJWg(~cAZ7z#AHs0dz>+b_uwU|4=8XzC=l`-$D{kN%LWyn^k;P1(prGfQwT(ixeLvP^%blYfH_Y445c=xQ3#=hsS
z_Sr9*eLm^}F6N*rQbK;|t|P1U%K1Wv&bs(GXqn
z^Aq28T^a0WfrHhQeIYERWrx=kWMr}KT8h?am(sO)2!mAW6SadxLVffIH5Ss??Cw2D2zB|NyYF>6d_o0vnW!HIq*y2_kpbg&#=fd5;PoJ)5urH
z@#=3tb13E_?=(5mwNf`R)nAYAC?M3wkXay?+>5NVAMLWz(*&{;3pG8JxRWS
zAg$DN2RpbfE8w#z6b(JaUhyPbmIUEiXWypl5wBe3yqmj+RKemo)pI`+5Qbz!{+52t
zH>Ka5DEw~9S99#6ff>Wbyy>mxwOjVtTKCN?!GQ$B4+Jq+nI9YoqxPgZ--X}RzE0jE
z@#-W+c*X%;+Q3o}IVZ(0waRlcNi`+W#O3e|s}t9bWj$##dCY#2
zxdBj@y7I^m*T=W$PU|S{>kB_LJf4~2Njxj-U_$*}H>M^6&7-EIpto_1AL9(avU6lM
z3ZrvqNgU)yEFx**vh{-I{wnE!Ch|$kD-nmH2}Z-tXby6lkVBACeHDy%#CyU|sc3IU
zboeDE7+_&$LfOt0HyB^mcC-~*Gz>YS^SiP@z1mzn@o2_?O=E)+KZC&}Y14`5vHB2$
zAfi<0K2ab%bz`}3dXv(pVcw?4*{PBi$$ShewYfkOyJ^yj!a|6UqTq|VN+tl8e)q=9^mm0`DT$v!kQ
zMA&2{n3!+NKVc6@Lfam>QWjt2qfe#NJ49{dN!CNwm|~3jL>w+4q~e1qFwBZ=dXW{J
zaItqQZ&piWHga@OFW!2!YH@GHw`RilIJ_`1rEv`ym
z0gqLq^4d`N@kJ8cviR}%3la7@*Q;6bTlgCwaZ})N;1-Kti7nM&5&uC%!*!#)V30=DZ*OnJo4e)Tx=2n
zt7yq?YaXo{NWjmf&AGoB@gS5;9(#uop}^@T^yTJ>U>)xYc=Yj;H6V>iO9@{88fI`A
z+L}dx?P|iG6Cq57EkU@I1>c_9C=~ynXST$kEC`jVifvPOqYEOo*6>ucAVT@}3fBH)
zwQH?cK~k_ko){gr>5V}
z{Y^~~c{L&VLX6(qFhb1z)|0nyUw`>3K2&zsd-CwWdroSW0G@~Re&YN|DtF_<&#E
z@*w-Odc!k^@rE3k+w(j{LahOv3Dx>}num-R>ePm_ysu~~Fp!4eBPS257iMWapV#Qz
zF8ATfN&9-8XLE-ZGICj{gaDdgiwD*ApNs)~Ud?o@GFISlInyznBJkfISP8rO`3*|6
z-%{~_3u~Yvq|;z@-Va(Bc#nfA){OUYqFAiHf!yv!z6&eHgjcl)Z$TQOt?2VDOs@BX
z`(~Vv6Wgw66T9j)oU^V#*B1?aTeh|$j1Oj7q)>PG4o^-i&|E8Py=ks-t)m6=_OTtZ
z8-9r;E-`{(n%dPteiSU?+cT)u{E9$j;|2OJz!~T6w
z!GwwFLlqEmp0SQ
zr2{ZvqaEs$b9#JXn`&izqwMJLb!}kQBV%)k!3K|8ys^NAg_9ocamD|S9_Ijq#&6&V
zMikM*qXG%|*CZG>oj#Hw1a)c8@bqOO9-+-Ol~Bu7N!8e`N}*ZB!69iHP`4uPzL^9<
zEyk>A8Shn_QNKGjj^`WE;mZUeel%SMm%Px*1Cw~sK`E*foq+CzS4#7lzCeZzrt`yO
zMPguQhmCSO#oH1v)1i-Ris{(_i>VGaMC3#2Nm@_@Awr?xGZcWyfQ-8;9Qzaj;{Pq!
z3wQM(AnhMB*wPSw_24(Uzf+X>DHjY}E6
z8`BjGp~LoTA}OidGzA)e-`R)#V?tTr+ar)~BU+*&$=H`K`KX?4@*NLeJtSEraznt?
z{j?GcL0?%2u8dq@TQXG>;^8?^c~*O>!8*)3Hfbkd9~FU@qGi{aOeFmylZ|6UZh6eT
z`g&Sf$nCVm(`aq__W(JUpxe2SHD{q@hD(z&P87W1G>>z$x?u+k0lwDJ_A>}ZqhLV(
z13l~~ep*=XPcr4mVWX9re{IoTJq!Dv!R7{vP$k;eMT?=Ez0u`Sbsv?IeF?z04+#PT
z08O(+m&tpu8jha@sEsAd_%vO?4Hbpig5$;(SAT6W`oGyi(jFfZQoCpSVEW3kGKZ@x
zqceVpE&R&S3Wr$3fNy;B>!AY6etkqqJ;l|pYO|lrE2Ye8Z|yku!IO%_CQ0H^8<_>J
z{yg#Ybb>?VTtQ~*3DQw^J*7X7ua96Viz+_UP>uvgvBw(rCiLFjhGT7BcVzhZo%gv1
z@tXubq~VemAfaTnaisgBb--k?{P&Rr-L&qTYDt5Q-|OUH46^$U_zCCk`m4aFszhes
z_U!n&D_shfX$@aG$n(Oc_Q*V7ZbS2I-0}y2HIw4A_r=iYfn}cl^_#>NQ!g6W_}MeE
z>H@IUt{SwF{$E+DJ)>JvMJ`_wYQ5jti_)LQ>un>`{>ZEtCRk@Ok2tbtI+#9OS5!+wE
zqQraq#Wj?Mp4RB8yB`!yx+Z5_Jb8NE*Dmi#HW$Bq9Y#79426+3px{Pg>;1whU4J!S
z#^YO*Hycpn{!EpCa-yzB;tQ#ao!%xZaIgVb+DiZq_QO$X?k)($JbxW*SyQOtKIrbu
zf{7cg!o4VBACTW_Y7aKc&jh%^cj&Ugi(2FQNogGenu3VLO+mINp49B+Kaywr!CcE#
z6wZUKd_)WuqgCDsb;aKJyS+^$c}#{gp-W#{+ji!~U>kGL(6)_^?wdM>zn79$4>07|NbYqav=~3PDVoD1G
z*fx7vu##O7a|AxN%J&yT+eZF_$NpzFgUkS$F==pV`!Pj>^}gc=erwKJm&lH%L=TTc
zek#bTk!H|TEw#oSDXQRSTyN#2^X1ZJns@&6EFbhQUfD*BRr_?-_3XUF(4j2=^>AxV
zj-RrmJ1vf2@wc)%gBdr!w~8UlMRW$_Omk9J65w)cNC-v6`N8KAMjfl+uT5=m(2IX&
zN!-6|?YlM=-;lNQ#C-@BZ04{%0@YI>!ZH=MSkivJc##0~HbZ`ke9=&e1=ZYF7Le*9
zmO-j3N`Sz&)>wr6y{wf=II!0H96Kzk#8q3*9R7&cc+0Bn|Y7m8$%J?{_MQ;T>sN~0^sU@raFq6a|=)y
zIF7q2K}(L`11jZIR_g*xB+!;
z3^)k{H2;4yU@XApOJ+9Es-Z7+-YiAp`F;>cbM28W>-4TD;V|2yvy}y@N|t;l-qH?J
zYMVjx4};rFZW@xLCOP2F!coH)h=8b)27bm}0BN^r)>o%ei&g`l6zfQR(b?U|&CXm+
zBu-K)d(_Ft@I(+RK~+$IQ&+A;al1#lbG21+%j$g~0gbvJ1iVF$F|Q%fuki}id(tlR
zZI>(>{v**ZhFdg#N|;-))pj24_rK+(ylp+kovHSCUCs9-du)UyOtb#bWG^0m+d07t
zh=Su03O`}NAFv)84z4`DkJmXoNP)8*qu38i>z*r2CptYq5C<{P6-y9pIyV+g@I0yhAx2J%1zu{PhBI_z2UCP2Z^rR+0l+|el^2Hd!$0dzPBSz8q@S2X@#MceW-B*ayyA2`2oO`P+9VquVwWo7+W%qV
z_^af@7sA8+j)-cb)lRGy&s$e>?%;O&8-;tGH0F;Hjo)J@vtGbJy!dd*CbxQk|
zOu4vhQGOT;EO=Vre9I%BsPTQR7WFks|CemVEU90Sn$=fJ>#!T+qQi`9YmK!y4jveh
zxC?@*5y;v9AA;$A*Ky7-Y9#m~KmI2O^#}bcJfQh3fz-ss4R12uojt
z`0-%W`I`D77^5yRQKq5pVGh96unxTqX1u`LF~_a@xD$l~>UlhF;`3}gvPqs{QNrdTqJs!(@T%myM2QQJ2Hb`flBf(IAq4`(-MZ`^z&&2*r0f72
z*$e+oNZd4JF98h3mV#Feq5r_;`8#*+0@syAwI2t*9J$%cFk6s@hZ_eH&vQvoU9mi5
z0@;WhymE6%K+XXJM4;}3qh);4{{=0J+8?)-58HI@F)2Z9Hg->0j`SVA#GB#GTslbJ
zxpZkp5Z5yKEpun}pnNb^X88kQejCKpWKi1nAm{jaaApnJD9&16^KdjAeKjiHN>VEq
zOdfn(r(XSfIKuJt;laU`kj+&;hu=)X-Z7vpz}o|{cE`Qd61IV&P^xQXL>MR}M;fW0
z(%sWI!H0?`I#Q}C19e2OCJgF8sJ;qr!Q2Au0bi$vSxeA}1PqL!*o*(QR!ttcJy<}1
z4T=U(V+H(Ut*}z0NnkCo3He`Z{Ze|s4NUz_3}&quh8t^Togt9}RilUe{<{)@+S45*gFHIbv^E`r*StDK!%qXP*@w%8p2ELX)Qj2nY$y_0~s>&pTTMiHd!4
zT2+)!byT_7Hn-wZi()sX2+cQ=3%b@9w+uvGAM)56F@vezk3|FBO|`n>$zW#<4RmyX
zl#eQvI6_9}I3@7dG*)!K$3Xkc$R*lfCWHEHFndo|fN|Pw)6(IKn>A1yGPG*}4s&-^BzdFAXTS>fb-3;KCfg3UqK<
zRa4b&!~x#~_b(;_*O-RN8;KHFnhI9ovpi(Gv1)qDqVH4iH^6p6-1uM)?flm{miufB
zG}mi|qY&>!QWoi#s8c8aG#KAbX3L{;>eO-}PEGtM2L@Ssa}|8)o`ZJ|1kVreo*NFE
z9V4CQSw~ys`%2ODFq!^NSVQJqEhRNAj}qPFWEIaurr
z3s_YQ1%*@$8AnH1u>q6&Y3sbiv#ubJ;c776`KpWUOO!U=b?vbW#-*zeA@R2WR+?V6
zRtog!AKL(9v46#0nUXb{0gpBSy_Uq&B6GPk`)1-^iZvTeBRpnUp?b?h7u7^lFqzJr
zUDdGvYmVy=S7aR5B)Q_^$->M)p%$`3}Z%mZ_4NPvtKCb+t&2z)8zpNu5(8?hXi
z?>-!@%YOcg*6EVd=D?0|e)7NN;OX!4h!!hPQ0XAFqMoFO2l(2|KFq{Qnqck?T3?LabFBI>
z?)u&hEH$9+O!_0&22b{w=OoX%opHB?erRwZqF3U){A!C}hNJwMt>K3
z{qkH3H``kJc?Sj}`Yd_@cRZ#82?0i4tYq(nXo;3D4jyVm=B
ztIA;9%Ko@v?Bl9Pf(h})yq<}8qUQa4bf;QZ74LrG^*pEmLjQ>=aq<{vB@k6*^xHE4
z>es*5qy2jUCi55inm18E$9o6=KMy9{UjPgG8r}F6*&zz-Y{FqR2U1h@kP}-NBe*cC
zFs9j<&s%^0dP4)%UVeDoRlg`SR0#DAnLE1nIs+08l1KK2vihxbTXfY)RY7%}K1CdC
z(`U4{RI#e;3yLxgc|}x42!$00W!QN5WY|w%fv+~bTArGxIF{P#o&v9*F*8H!&bhC(
zI^yvz6g-tR<&efA<1Fc~#n&=g|vM$&4>;b7o%uJhQz;JXTgI#}Y(9hU|;*
z-krb~6PbG#^(Du7p?gQsmMNCA=BHGJi>mph5{U+#%YnW_n_BJ7Ql1F3$nhNgE_?37
zuE9R~^en<4+uqp9(tA0_iAd|t?ersr=DI!~qQ@yQ=rFn+@6xPUoY#lws9ENXnb`t-p7P{v#Qze=N=gbwC7Ts<}q?p7G+VP
z-l9z0D-R^a`qJy`b!J4iKthn5X2UD3~&OwFFBl~8omb9@p#m4e`^pOuKj@6
z4i%IBP*eQ^L&KS~wYI4G{@^ua%)Y$}{b_q;Xn%LmWEFe6!t&F{jV{^0-H{FEgCWY_
z^QK-*c5q}x57jz^j~cD<*f0BpE0?6+&OIkqv;MBvTSta53LPPGM~mJy6DO>rTGFpE
z##?UMoR+hgtd!gzp?|vSK{ZqvEf6H@d+T0Hd-C{!=qE%$!aIVk=`Iv7mmyOmL@Cer
zZ^I|n$|I6_nX=HC4P}~H{YtSIZ~41}*S8V#dW+*%iZgRQT
z?zzFOyJ$E{M-(B%*Q3zaBK@RI9lsvNP{;x)8?IS$2SGql&;O5ctOESweh
zuOU|u>9Eh7wPl*|t3iLx2Al^Qaep0TWe$s5
zl6f>Zkf^b_o
zvx@oi^h;(|%uTt4zD(QOXC-3%Wg(W*vA-lquL9jj*?1*SyV(5fEq|=Kj+WHdV8#Yy
z7uKfXIf)y~D9eNhJh1)7Gt42)B@?+DtbN7FMOb{6=J|E!LIX8t_T%|%VuONloiXX?
z&-8aa9~w4aH#O|fld$z!#HR$_PFdh|DmJi%UyKudvZ*TShgvqy5i0N_siOR0pQ#@c
z2yKg?TqG0zE&8s@42$=y?c@E$ZQU?iHG+lj^!5T)>MNF!=p7zJu|BNQlpnVz<=p-K
ztxP75o)r~yrP{8St&C*_%s!K@K33cqJBrvlJ2x(W`8DR7YV188Msh*6e8}#b%+xhS
zbn&9JLG9#tu5aML%5RN3{Ae4@S3_nqe0i+aYKb7)9AB8ubl_#9=9(A!scV^g0$a2>
z)wI3o_*`kcH_6WKQy|oFB=jR$WS
zyl29^s&Ww(@z6r(qsqds3H+
zd8d8-`}(qKdcZW6p6xfzVKnKC-F&wXl8j^ndO;q8v(@VCuvk=jONLi<1i=+%
zeBskF*)#aDjD7g$SnOecGC@xe00C}E83+#M&eOe}8ISjKsiq{(n(uagQsJ$uv{{Ux
z7PODktB+7RR?V57FQHHI{bs8_hzp-4C$Tse`
zG@3t!f68v-V*=}Im3Is5bh?82MZ2NURQ|VDejX)c&SlQc|LhWbXQLMrZXS0X`~2TS
zrH*%GIO5*;^N0A#qF*+*pS}#32v8g=B!joEZfB}FX+Ldf7$n9;m$d`>sN>OHHoLMB(1H^1{5L<%A%3r`DT4})
zDBR}0@j`v-)f!^m>+joiT=@9pV`E*h_jsU(%{kJhTozfSP~%AgjY8sak4f=ymUi(7
zd7Cl5RvSFI&DaIaPJQCiY?{D>N%O;;;Kjzr&(%F9$4n!qYnYc@*2B}wuuhv*v0-uu(6SX1Jd>i0s#Hml?yj5##|j$fZS7+!ulbVcaP
zvr6ig>mpWe?e|9HY3<@p`&y#{V*-QpWlrvW>?1E7Z%4!d2Smt-dfDV&WWhyX`NXoS
zT0ROSUs2GG0CnC2#+G6frpIQX?+I>f-HXA?PE}z=0lRT5C_Te>?NHXLOOM5HPo)&K
z!$bgyZIu?m*GA}mA(M*nb`~n9V5w6^S@tp`j7Mq3n056i_XlHXWt@?;kpP>DYqmm!
zv(YP!ds+K(O*C@zXAQj~sOfWUnu^#h0px=k|Eu`QO@%!+4?_f75f4xdG-!I=sWKki
zUt+^^(+(VO;nk0M(;+kWF~s_tx{VSuDm6`z4@nb$3mrfWBf~pn8u^wvr7cXBF~j$DVBo|`0>z7{;*Y>JW5W#VDW_Z+`Z4mnR6LM=H6V`1KRy@OGx_Sfaw39
z2h2r5LP~l9
zKFp*w{9>SZ+a3YIUn*B@2&f(GW06k5C@Xm^^_I7B=cu10PClTSzLIpR-es>~g^4fh
zz_h3ce0BT(~_fPfyy-Nk_6-Y1Ltm?jjUxnkLoqqFt_N1OHRS2J
zo&ciRMpBqT*~b^}CKDHV7t1HT;gr26=qq(x1&#jOB0~Xr>|C_^S5zT5l;Mma_j*XE4cPzpDXsC8o)@pzzVHvcM-ryYOh>V)wa$5rJH#<
zwe;
zX+n!!<}T;y9M;@kk0eJU5`Gp~-yRFYL@0ImVrbye=XreR=+JlpUc_OiRYs~sU4E6s-l`jLhso#MSz(w_S(E@}}kvrk7!R$N$wfbmhloe^=DgskT-_Cw!d
zv!s`zWn?5-`fh5ruOpZ?4quP>HWNQ)%2nq6=Qz3zgp3O89Nc<2tO}foYj_wWV@0g{
z0zgaw4~G{(8Lnsy;SZV{4T~b}UwrRO@bA|Q`}M8#)K`riBa58`mw4~S;bn`DeKZaI
z`l1H2nf=2fOjqw6KE((FnBg_*Q_QVMx5thaiR%c<>vkrBkZ&>|p^5u2N7)k5%*zJ_
zd8CA$Yp)#Ol%@iVZwCZ#EtSh(d%Spr&x$#WqAh#r^lit|(PB`HbgAN6KFf?s?5?30
zV`0Gyxi{hw>cii9@NKvY-T*30#3qnHeZ|PMYVJ$jiO8vfud-^<1Mi#$n9YtYTOuDZW7{6|9Hbht$Whprt0|BO
z1>PxBtz|QFE{`qM2c``bXz?-Ez5oWvD@?tzE*+OQYwm4J9BT`|M2HNZ3PSjm)z#Ah
z&;kEcU(!C`O^ep3+ht4BKdIt7KRlsm>+0f@$*kB(i*>JS44^?Lv^&f|VEveso&+At
z5+l3gy*q|0(e;lDm8y0brFZV4APc&MLN3c-Fb$;Xg?>aoxKPUKN2TAeBL#)&*jaac
zNAFlwmtc7Q?I5yx;wD*rzN8m9}B(i11Rvd>lG;{S4(35Lq5wJd&G&6v|
z0iD*4AjM50H4Ji@{l~eCw5o;~#@guWB6)De;K1wR!@=FC1lon?I9#E~Gh0TOoG@0#
z!^Jlc4l!WJZ!8?851=s6N+GJ+UcY~Gc%q*9psJB}qPWhc<9ytvuL6Le>+JbW@h_~4
zc9wL2f{=0*Z>Z@Q_Jof=XoYR{&T#@H-BY&Ol~8#;P3e2ikDW)=HF{hRW}XWz1oZip~hO~
zFlJ${KWpK$Rs8G*g7P0l3jdu~LaGBV*o6oS;*nBk(otT0e#sun)
z+tz`xjekXZ(o+S~$HxxIwh{BK#+UV6=&yYIYsM4LTx
z_+oBX>VXJhzqO@s6!c=zu3VmK3cPZ2!>d8m?z%*0QoK{2dV
zR2U*{9t1`ub{YK2E1ldRZ`DmRzoSxyDagsWDYO*vCOuJZYea6J-(?p}UjSWuvmh`$
z)C$NG67Rh|iD*UeNEsyLSHiOO^Nv$1KRXi6)E>kRinvt>IgK*^}vMlNtZJ!3CC4o8+Nn8Ocyy`+|-DE3xoej95_
zp(;NfnZIvXXXV5UvbBe62Bk(s
ztc0g7t%^PE4;J~ZJM9}eY}|_x`F>SD=#G%An?>Lz%@L`S%?WCO?zV2-J4^UzpSV)lriV8tiG47@}(*3njdoPOc3l(+Y2byayc7yZEwZvai+Q6;g
z;M5Vj)Ukug@Y>@WuT=F(1-~T1$6CVl_mz{4JmSl;B)^e5X8{`pw(!BE`Lk+cy_g7R
z4o$O93Q9_Y-x0htXsfcWthPX`Vdv}a*`)YUeNDHR9O^X%DHEj%wMrt?mk5jRXK?fWLwqC79
z2c?YxE3ce%D!-2@EUTt}5P9qKC_N-W4{w=1^RL=EJtYwj3er#JyZ1aK;;0}N&_rb0
zpsXmnyTmv?)Xe)&eXz60hD#aEG1vEJD^=48E$Xvg6EFG2_4b6AlG}U4{H+P^@{UDcdxp+VN|e6d4w{>+){3v4u|J?W
zmw0Py<*{ZW#K@Eo;ge6E99EB-X~mpHdNj?i1aq~7LCP3QGrXP-8Q*MqClBA3
zrm)I8-0b_BNT*w9XUe8Y``h%QG(7)5OXHf$r?3pZ(uwrVs2R*=Z9bzHeL5pjD#)sIW8jMZXi^KngtEJbzhHVSFlJ~vDP+zcY>afoKlk((G6|9`1a3QB869RHdt;Z5?DF0i<%KHNjp2O|ZD(lRm=kB`y
zsRA6ZxPMxA|4&E-C36az#-Hy2&>R@}klH`3X)ol6%zqC5m60ff(r~IAg)ggrHo@nz
zHX?J$m)91_E`$u6Kz8EA?S%)
zbSMPhKD?`;nv+)aH*a5v4jJR|5BkFO0{+;hodA{A7Te^c*|FcfAwPK6LY*G;gduaQ
zcm|TqTXSgtal3v^$pQZ{#;)i*fA^B9z^-AI
zr06JeO?NVpvedZ^`T_w<;PlaXf+LFr{~;l9wN`4pb5Cq!lg
z9Pn-rn9{O}dXNA(_|@MTtb3mTcD21XbmesBYxkBMtzF;)UNBYyU|HFJVEkjscgUg6
z{E7}c(Dc>;X1kXv1;t5dTL2ea8f=HQR!RU@Z-~IlZ6^t=&&%a6IsFc#Kl|DL#HH;x
z$m|ca
zh<$_o{6PS&M0KDXZ#H!S3YkY_ssB!oV}_b872kl#^sVqGT{e{)gk(2JJ&H~}cMg2@8EsCMj
z(Fe(qXK-8>%?+X40ag3_NA@i
zp1vEDJ2D*5`()z9F<)(_ESiCXRnzubOc=gO5$;kGwf~d>{W5!FI2+3tzW-$m_{scJ
z`n5X=GPGQXGAUZSLJsbK${!NXz{XJ_g2>^2t3cB&;NnnKSWQKW&_&ejTRW
zdIS*Mm2p7Fs`dajMX;{cb2hA43djR^v2MEp)nnz>l{0PCqXq%T*(>$TZ=TZLqS!8m
zUv2YUaIF`1-I$=UuS;W66;|WJy}J1_RtXROAWpPI3mkeYv9-Oo=7W^48gh0g&QPM1
z*es!@;URp5GhP4e3eEJ0>Ive4CscRHr(9EGlHodf=td$@c1-04MUDi;UEZD#qA;OW
zG)#c-*nVbhcse$rxV(!yZSv|#hOV>XX5tT*{6T%fj5*jAM`50{VXT$o-l+
zTM>KKqDYJ-?_6dfsg&~A!Q*%RKcygF{D@xFQRy>5JTp)Xg8+p?QR}vF+|YUAtzsY*?%baMf)C
z4+2&;Hrj#Z{vEF=o-;B)O^3-@H6ECfmt`fTXQt9VMZg*qF@3zHo{b0@EbNPASQH%z
z=N^F6|EuqQhP!&T;C
z!FOzE#4H;@OUJY7+{^K7hA8Q*ILevr7KPnu5EH7Peem*P+0iM5Mf}3K?kg56sn5i?
z=1OXWm61rt<38%i0y2_>I>t^+J$H8+ACOx@
z$z$oD$X~UCW@E^^+6d9OyPe0S;Y{p$XMi4D#k`ck!qrr(#x#)M*wt9~_3N{#l(kb`
zI?&L~~zuCg+#&}?o7MiH~`)VRUlLAt|WElXAQ5s=|+5`?!DL~Qf*O#!WU
z-Q+^+WdyX|k=2wXivfpX11F$8$9J2$@^6^Ik~xy57uXxoTP_oCu&?)
z-oJY%&fGGBfEr6N29j3a&xB62Q=K_o>*y?-0Nwm23qV(o84}uJDa`_REq7MWsouqb
zg^&KTxS??nV)^_L@$(u?LCHvfut7W+_`=9Xo%{mvG|P-D$$|d1NlEP?Y-^7D2-LyJ
zOQjgCJm+a~;m0o{>?dmNHb)nrN4~7D(1V%7;
zRH58(g16FUReSdq(vvH7+Ski9L-^rXAn?ykXK>h#h?V+of@52j3SjEH|B$(PgLBeXJ(G!Qn&L
zej~m|hC#1gl~c1{I)}x6mjzGgDCe;vG0hl}CMlEOpP@x5Fahn>7kf9Oaox48{;a|z
zVz3sfkk+3LK`PL^!h1r7S6s@c`_+0cks9H&Tj~~g4twl;idadSW~BP)W=o&
z=B4%b_WuZO)-Q1dYxU*73brix$;*dp626e!i(KG}#=M5PK6RxExGJqtwdZD{zcPgeJ?H%i1VtvR3jO(okEzYgrs
z0_MkKR0dc>keL>+U-Q>_W{S;RrDOo{`u9xFF`#769^o2P)J1>++n{!aP1*ZjHnk)R
zE^AME$!zzL?^qP?!foTJKszs+x{L0c5o6V@`!r~;dV0mvU%b4-Wbb#xS^AQ&s4OT9
z@^$l4hyt#q)BlyaQq~qDVA=EENWL6|iyVchLR~6O?H}!uoHYE=*4phk?uc}!LqA`%
zdfqoHMTN?3J!R5C!Qb48xS)NP{013qEYw=~Zmu9iD~`~et9PWw28NO`(U2x-b
zC!o@}!~``nf@Pk$Xz5;{@&w
zGdF{--p4sl8Uu0hDC>C-{V?B2P)wpr&@?{!yS!v>}ji}suqA@wH
z1dK9WBn>JpS(Id2#PwQQX5&ZUP<`pM5Z4;biHe?AXWq*%<(hgE4>zja2a^)e?|_&4
zYwyEb&6TZgvtVBxkhs5@47F`Dlh!p6HhKMzaNGG5S4SQ3|5ZoEeNued5LY#NcEa`x
z#dZDENye99DGz9Pn6R29n&PjiI~R0ibo?IoTcF8ttoiLvPW3-gRn2h8>rAfCG$JwE
z2M@e|+X9CuyP0#mjkZt3{~s&KKO_+R+8uEyp8t!33)0N+A?
zTRkD;EOkP+|57+}V}FhGAWh3O`K;&6{-bx@=7EwLpq?r9-#5AWx3o-Fk~~0w>F7xw
z12nliH0Ul3(r^}>dcjBI03-(akdAq!10{UKRTf$eAK|bz5(f9HAt4cu65!#G#Kin!
zpYqD4OOe(KKOmUduyH(T+ani-fx!W2gX#rE{)k78ZDxq(Xv>$^h@S^wA~?~5(!&6T
zA(s3cys;QZO?XnR$%cUNZjSA26zg=IXVKGm(%p6;`8Gtd7}D}VOOSNW8N85$f=)gc-HRf#TqN;
zQ7S|lS9l&=Miuxme>Un;#95lgS!1UhL!-?CIp^-s)wxOhf$cd&p~Gm`2NE!#rolth
zQK{UTUFsUpgGU+#<2h>|_uD=v^LqS#+SD*1K3TV?J)JPHO+I$i&6PKEU@HCMnc1Rl
zsw68kT7rd9_kc}bzgqv$(n|jJ<`*af0onSnp%qxG6Ztmugu+@?U-c@>V3rjeCa`})
zvBuH`|8g?S8tr4ck(Z1t@IH}}zVYj~M`CZodh3zFZ>$9c
zsqsz|FL9FJ{6_vCR4;ge>No!z)wA?Tm8@4ZAf-!wNbsj*K2O(EI6stiZiNB9s>94s
zaLyxc&6;V>w@Hy*U3m}R0|GGb&bYVQ!+BUOEEdZByYQKn)mFW)g_3pN+H^{FXn(O9
zRXtrRqEL6ojyxbu8>P3_RRLx8{chwHcfMy|r>+mOrdP+$J*Y#7?$spPx#^%%-q-|6
zAw*B(w)=S5S8rjJ8V$Duv{oshk6d>0wz5cfC8ah$z|QTj{#QYt?Y|rUm^tUKME57p
zwj%l<3wc}r3x(@z;7(rbxlz!18N
zx!0BI9VQ1zS}b`068AhB6QO3LN{ySYznU+=Tm+c8T`+EAO54wIG~z|%=F&Y03DPa-
z*09i|w2o?Fn)oX+ei?*r~-JS5H|T
zM-iwT%vtv@GzFSqabKl%!Tf;$&CdF{vf*6Bjkg|Vx9!%W0c+7>f+kYwD65J9at6CY
z$ktr5d3&0x#RR%AuH;Nn&7>TH^9O`;(>C&C1RQ~{gvn?;DRXRz!4nM>|FjQSFD;5D
zx|I|FF(hw31|co8`2xmML20iQZPq=&7~&>nSN|}`1awuR_{&wkZJO_e0-7ZgP`fkr
zdVyl+UZ3@e<8K+NeSkhgI`fVKmNn$qa-jK2|(N3+JCG9#y`3Cf**?4
zz7>gY>rWaz?f#u^1LVhfmC>uJI{>_}x2pX+FOS&p
zWdxD1Im{vb3Uu2RMONHz0cU3e-mm}~wGf|XneK4ZCH9928qECN4EVU|OI+G#XcetT
zm(K92g{F-?LIv$ls7uY2)exW^hECRObM(WVapvl)&gLj!&MT(zF<>n#9?-nQ^3ais!^=r
z0oHCZUCv2`VQ0|)5zG3goNdK%Q{&7&%zu2<@+pYumbUE<*jp1Mpo~G$g7#n6xb6$nL
zgw)M#eT0
z0lB#ix|9iA;7mE(+@k1``?$NAO2j$!GYsTkkB#2rcs_SDTd?$lh+udB&Fy{Gdq%CJpK
z^1{Z0pMgT}dBJcs!Wm58Of0Qg{F~B@SXW_pn}5c?we$sy1d@CI@}4WAFGs3vS={7k
zieP;Kh4njzz0B!s4goIAl|*Yct%`1XC4N2HTqYK@SsR@nLC8h5eX25dQD`g^c{WEkX#4Zyc%s}*xJ&|G8+`%yB=JrB@Zv9#-F?WkNf
z-HZcJ=p%{aK%qBtXrwz{^m9?H0LS>ooEu0vJ?)*)U8jij(+2*Boesdd=E}ABiLT2K
zx|D4R&O`-H=GXKs0Y65!B$eTGD%?JA$FCfD%Pp6;-D?{tcQ{4<;!wcOHyMOzX{74J#dC-K3qpmkrz4^8XwKn%#36RaD#353v}D4#`$t;ZF|vO
z(uG>j(lU4KWW~b{dgtQ>3ckyO)owCQDLbF?)1axzyS#*)t%dbx)E>uCU+VH|?J_$P
znaRx(F-sfNFGS}z8`N_Uf|e4iqfvSZZu&Ru*1w`tmbmL^otKJr6i@T+oQpq1xG$9e
zeMJDtz#$SM!(AI5Dd_#7#0l*wqhrNKK(_p0(hkm-p1TH7gB$z+ZH!pQZrXP#R@k)Y
zz-!IannsGfhDY%?%mMJLRedxwR$hsT9JMJx4=<$bRrwCMD*5yjQE__G=~iyoXuZ%d
zQ~0woPO+~97c`82T8a=Q__3khT2pbjXfXn9HBJ<$v!wdD;buNuw4{)v?UkP~NHrJ&
z;N{jYj=Ur2L&G()e^N&ESQtX0K?Tlgg{n8!B{vKaWOdaJ8S$nqH%5S&;ZY3cev6#sk82T?=Vlf)}YHN^{`n|n-EpHju=j4qBaxM|G>QSsIss~BAX6J{F{BEE{Q~C$S%h&i
zZ$`_icxjIj(%Lxa^H~6|0Qxg0IX5a;lDY18P6Tb*qUeR@1Ynf`hgIk(lqvY^*WFSG
z2i98qpW=QD*&@2f;FdI?%_7P3$=m`}?{#
z15R3rNgSyeyKiik
zH>1>U%`;WZWj9^Y+dN`>HLy5L{y9)d_cZGVY
z`-rgp@-$i3XU)7riH43C{eU!d3$g12dqQ;&)5Fku(@H{nJN~mz)0Lq_D1xd_HdA->
zNAmNVj2;%2Gr`S2rcUP)y?(70H8^-Ob3Y-2y|2aCZGA{M$G`UmP1kRP$P@*IfCv`|
zG-JVX~w=?
z05(^ijG!I1rYCbbB%EIY*aU##NQM5KZ*CO+tvY~5TEcXMtIz=lw0E;K0864uFwc_!
zDDof6yf|;J#C2$WS^>a3YCz;fSC}8CAo#bL{Iy4Ndl)-){Z2IX_v!Mj<9nbxUykfo
zh_8xJ>pv>?UQ$lkB2-LjLq|qyoPiyXW4g-<=2{TUzw%ioOkR5BrOLGP$nxu#Y{05L^tFI2H=nDN7a-;
zDnqE}`GntxcjeOrcAvhZd#U2`%^gl;n*?(NlFL1|w<1F)9?q9FHy}g>eW(3nppK*0
zPnKAVvqay*?DUHN5C-`A!4vY9-84s4n~v@AG$;D
zqJ?e7PReH@AUY_`$q||a7nDU9qwp)apxV8$ubPg_FN3%3eHTTCp{-&5A7olyp-jpx
zzjuKd){*)!?L+&K59r(=d;5~N`L`58n?^w9{XaXr%u+l}I?$iuCg?36ce@hQncC;e
zAdS&z<{G#4be9$ILqWlFmWqg$!mseJ^6sbMEg<4fHJ?29u=F`)@z?tSId>4K@O5Ggiw;Q2^!jA$|
z1sCk5fItyvs@vUnb5HnzY$|cJFlXC=KKS0XK
zAN02|-orM=;wmp%yC{DP0c5e66Vtk6aC>qDyP04=6=*741aw^pq9mI=IIq+go08Ys
z-{GM??EXr1ZG$$Q7X5l&?`4Q{#+wp4n(lXyYOpgA`zi)Y_Aq)BS~kEhxW+~8K+Cn!
zN1V6ApqBvx%mpCIr7zRZ&^X%sJ!zrFtg?0c0mwB}ty@)-6dm~@`;^hY%cSeTb>-mr
zm4<$9!U6zWg8_xK1{xx~js(0Y`+8%qW5e;3hhTLSWpbMX8n#jM5R6bt_}*+dx>tVG
z+wepCQ>~lzCPjkF;POK{!TCVT5WK!N8)(t;Bd+dRgT$sVi{9=p6IZ^tTEpBs=v%y6
ziQoO)S>kUjvVyC%(zOh3(tvVvHQigTuTObiA|yS3UpZ>?+NZ0^*J*?y(Cn#iEM40l
z^YRG*dY2(YSYp!Opit{CrJiQ9|D?w
zrRPk)m^tqj5DjzYgcw-rspRq8=3*d;V*v&*F<_Btwe+oBn16zQ^?lOa3TJaXgr2B`
zVdg?1r*x&A-jj4ZLLwqwr=c;w;xtdQTSxkRlXsAR(_>fZVL0n2)|5@ZNUG%UMjTl!
zb|0ZUf$rn_Tu=Oi>L0~RQOr*_a4Sc)e(6z>o&nvSjdt-F
zK;c!y58XqrIoX`%>V))vx^?sltv=QFU3mUMzFNn_K$`d1#14`3^*{z>j;GE!ZzcOf
zzL`y1lwE4Do+sJUjeC@Bidh?mi-pMyy6}_O>@41ZeFTBe_$>f(
zs@#AG8ZIPamU+2a`W~w2&4S!4HYWmaI&R^CbD8nL^CF38EaCTF(;MGC!gUl4PC6m1
ztd2KeDGH(LcBI7z6i_DL{}PM_9I`EM<+n281I&WaX`LGVdB#8t!`d8AKdjo~W5viT+
zWJTkIZ~naBZxYNuRuA;A)uZzZwiPe^N?(5tLW+*?Zt@kN4I*WT3p5WVrd7n9Ix&zC
zAxx&8s43zQND7Cr8SMk>G@>(_84>7lsq7)SH_c4yeT<*OaVj0wEledms
zvmvE1JoUac#8nBuE{MX9SrSTX6S!KpnNS!_$T6$CXvfdgnKFj<%Zcrq&}H;k(V?|z
zL7`)p5J@>IMKxX=G(9@R<;_u0D6DM#>*($|uQa*NUDR~j)bT4V^*k4tZk4ly1p#)Pb+6L;I&r&WMyod0eW6m#(TpyUSHxn6
zZsKe|rGw{v?VRp8bbgOA;<;1$EJU98iWzns%xPVq6HoR~mx&$*S?82y+`eJzzYagJ
z0wpK%+`72;64uV;!rHmq+-F^oh5AIl4(DU7uTj`7sn?BthQ@y6_;VwwrC;#%;nF?h
zOv+_WWPt6t`T6xzoDi7#&oFr-k)HuM$Zw<_rY>k|m#
zwU+hD7gy)d?%wW#kfnSHq0?rqQbWLgh>$4{d3@>*izF`uzHu}YdbfYWRzOpi46g2Q
zInw9xS-d@Q39#3;&87GvdC_Zx+);)J@g4_8c;XD6Kc%BVyKRm~8mOdP9pSZgJ)?i1
zQvNM@B#f#}QP6Om;Sqb`dsQmonGe>mj>DusUDW8yG^Io3Jc{LNUnIT0y?x`gQFCfL
zRdv0z$a==7NdPE-Ly!R9&Ugozpt0PUsC-RcC_U&wQuUAk*{UyNn#X|~NbMv-70e6P
zg9(PG?z?TCs;>NKD|niEWqeH|95RIA%H)*(r!2;){>KNO$Mch7vQ#!l`jYTJz+r7-
zv(0t}|BxvdG;O)l!SMT8eZi36^rn74eAyIR656#F@sANrN?;6#hA%$x$Blu%%W>E=
z7cd6M{{)TP6v00Jc9d8l5ujOP`mY1d8Gg2$ZHo|FjuJ~xva&6T?j3nbY8V|Y
znLhe?Yl*|;1)B6Y?v!x$m*XcP3JHY&0)y1yPDHq@*uY9wtc-~5lHE7q{vKSPi2v_CAvRGSw7t}X?Gwqt
zHa6qXGw7jP4-vJO@wZLWpIAB$5b}M7ZWe-PMt5Jk3_(b+;?gq&>=kes)5Ttn&tGwg
zd1Of8u<)Ke62&Z=u_8+>#%C>p2&+r^ze|Dv&`jyHm+&;swLVA^31;gJ3EuJr%=)~~
zPlkp(wsC^&
zy0YRm&oV5Y<20*FesU%c5S8F}wa1u-sin_7UUk-B-5i01tDvA_3>#A(Y%(c+$K{P-
z3pBf>)(Pq>CbF?#qXlpfo3F72-V?4x=X;~7y*9yEx4|(%6%+md1laS%ENOyuZ&1@<
zB7>Gr_EC1`GFjD9;nt(7e-i5>k=R6%>99Cf6Zt2xKI{02qt81k>V`??+6$f7pRp=;Qfz#G^C*isgPe-=dcxbCUI`nh_u<1+
zs_`(o*=I$Eocak&JZHdl-fR1=6(ck+sQBT?YQ4eUVgJR_;QTWu&8H{de0}cCuGvT&
zGf0OlJ&$qJ=z?IQNGIxk_io_>yLr3fO=8jIQ9#-40Gg)tuJpHH-1QIq|CF7^7z0G5
z5J}mGuD2s!3p_n0*b9an13+@GX32^w5uJcOY6$Roa82}h>bT^1c
zt(>P`68+eAEU)4+*SnNGKgtjj+b8*E1Qod3T2xsUA@un+x2QNNH<`6RD0-@+er8NF
zxZX#*fcg*AAWwKkTsXICV`>pDttLi-k4u2$&D=aweEZ{&oxSvdQz302yrrIlB%&?ayg){;IY%KCAQKb$eKv%mw(;=45{}KxLjYXrNbBA8$)`mR
zdB0SW`?VPBTbb&ar%e1;e6*VW`0EyhrbaCm(*fFT7<(kpYx)V>pU;5gf*WQTx!#Tq
ze+VmUumF5kV5B`O5i@nOC8
zzIz#X&3=~%^6Jd5y=>$GQXPNFfS#zaaO+~b-Wb>wz`E@pUa@{RHKlgw=SguLNwgUL
z@n8=<6Up*I**o9*n&YX`n{azZ=y|8>s_)gOme2V3&33`(24cp*)1G$FmXP8gHV
z!7be8Y1v)h7Al2HGOSo!+t815;ro**@IsxFFcGRA`-sl=oc&tEh~@%C*Ww@l7xI<_b};DdYRG-{^%AH?D0
zJ!P8@GCqHvJ}ht~H!}hvWF(;n0N~WqEO~0im)~!T3SlKtvbCDU1EXKn^pO086AYGO
zVUV91_RRr1X`1tq48k|hCwExfHm4))#u@td;dX|4z?BPZgk64BBEAtbjuTw2c?z?q
zF$II+0@iSLAh4#;I+MLM*Zah*1=>1KLhqR!9ZIuvmo8h62(?)4u4YT+1Yl>PANk~g
z;4&(pHR%kdCvttLpjw;_{*T&0a-ntr?DSu?<5o9eJAo_2NS%o)I@$eD_!Tj*P#*B^
z8JWC%t=^xB^2;?Q)qRgUhYi1i3E+_xo?C_7mwg{A0(kcP3FMXZ7Y@C<^u|x`W5I~B
z&)>MRuPGq8Rj*knm+{>Nwp1|p|J_mnvZxx}8Sq0s!#66wEK|VjWLG8G!XS235nWJ`
z<%coA{t6-IZQ7^A*9;q0Jp=vj5KBh`!<2{yZSsb34j1pe8~7KwTyrhCSY}JjbyH-pxEPnAFAujs*nQ-fi4wC0g`thDF|jbShVHMD%U?yo)(2r2kYmc`B5gC
ziQR)vX;BEK1me&%mZ3x(*NFAsh3tMQe%4fPG=MxWMZm~}d_Dlq%BZ2v+te>qeOs2f
zH;pYE{sE~9GB{sPKIMP&^+aCo{X}xDORhFXFqyomb5EI)0v702PhZox@scy>Dx|MP
zZ*slO;$7X4xFU8cCH~{bY456|;8trCl55lD7QXTYk$G`3Y+`^@ue6BR!ChZN{rTVY
zS4^O&O1OoL8+FkS)(q+yEuXm*CK3d!Bh=?ZiIL{jX?g5|O1jK5eB*ci;vbGo-8yc=
zY<{9?j7u%UfLJni*DvOc`+M9ji68Yp^J1v|Vs925bhB|ny**8+>9@%e4=xF|M{NDP
z-{K-)h&PC927i88rgIXcSipCne)b{OJYZ`Yx6eZp^DU5puc69KZKT-~mx@TmDO{<~
z?5-IFvva&A(a)5=j
z&f^7&wc_RPZ#wu9Xh7;zqU#^?GW#g%lKDK*cQX7Nuj2`Y(nt$J{K>w%SJ?`?H0mu6
z&>M2}BBR>56EG2p``a^bgH9;cXt#+>qQiDQCB}}L{s!hd3bB|ZVEAg`DApT@#@^r3
z0<1%%2be(gS2SvxV?X14+1Q>rR%51ymboLd|phhouu!@`5%)8
zex1|Zp1{#@bF+;W5d>kf?~&vpLD=~YI{U~wHQyDXvO}wn9;#Sjvd8Z>Cr|brIe=NS
zqywmiD1j+?X8rhb^EsOR;Hzfpxx(j&y7nQ#XrSdJv)!Tn4JF94g66z$16FRiv54${
za;fyLxsIHF&3MGlPR9IhIJd;sUiPrPo0*f7B3W|RM^sV|$E|?tdw@7u0Eutlqpxw;
zjx%;NHrVYcnbMq>Z~91Bn`s~m@-n!IFy8!baP_RRp6`s~W#7q%ONp4Y6!j@&tvkNk
zn^eO1I^B1t6t2wq?B5dUKu4K#C8bFMhfQ)X{Vhuj2riWRdhvIORSCIRUy0gxck5wR
zzM&bPSsqU2EJGBqzaVZX@~{1Tu3w2^=P9Luy;h6fI)NV(15437$nYwoB;&iC9X7$t
z_#VvSoHCUgT^y|i1>+3F;g&imD$qVg1u+%Ex`}hwnj~r7DBqwao$;^Us^`>
z|LV6$drPbRFq-+YQQx}p+!HlA(9L@P)fvdS)@A_VpJ&H{=8pzR>CbYF$j0l#Md*?M
z+@s_}j0!Myb~#R8_~P24pkg~!Kd>DVNa}CJDK4_R-*MJw8u_24yb><#Ql96qu{R}f
zDk)f5EfA2^c$_~ehW3zd*dAl2ReDudKy6UoDC7~yOLSViwuW)CzLFBF+MhXM>u?Fi
zI0Z?(i>pOgO$`=n;clree|Z}pLT
zw)tVXl`Q9TmKiogA9p?ddK|UG4~+7Ix%LeTtv|L`t)6WmQ&TB}
zAilBwzoCVg0yrXIZIq#@vh~1akf_3Fsp8{{vp&j~DmKkbr&8DJxJ&Kf3J&;{smbp|
zJ(o$C8=W#nr7yN1iSpll^>yPx>7`Q;fRM%2UGUx6sU@6B@+yyfGNyEs2Y9rv1K_2O
zfX~!>5dM>%B~3
z5-4iM4@Xx30g#%V@K!}fvftuCB+2!AL@f>{iPL~kqh)A2+CYaOZUL~4b;BSn`s6{Q
zlVM=nq#doTXMVr0(2lm$KO+o+yHddG2P^arfMA5NR|<4*J^5&~dpA+_VV*VpS_ot|
zhp;Q5vOeLogi)TV=Ev&nZb5`a)D3!~ty3OvSQU?t1B$A{=9m#`R$fciS--mRD@P){
z_%fzp8)lzEx1pP4KoowT;A=QmVmI&p3ukn3xR@sibsxAREM9W+zasXTbIJqEI-%ZJ
zp04LIK30P9a-TD;A=*1}|BJIVfn|$E;jg!vpJswCs$&6{&<4cA3MUJCT(B;x{qM#F
z-&$D7JuHC5-o}xM^8_Dfw(xu$42Q6;i9ruS`8Z}YKj?TaU5{ObX4=zoS;c&$B`cbT
zv7MlNSsO%p+pCgVsvV@4Oin~ZuXm%~LeZ^kzrFR0=tPhtVQUHRgFLeRTe1`waRet7
zZWXF{-^}Xh#M(Z-OipM8mVLETYa?*9PKh0`^>}6-1ow9A-M$v(Xwk$c4ZL0Pp|W8F
zB^I;aL;w*@t;0rH)2Rdn!moN}?0Q^3a(@EE+Hg^jg?tt;tVx``!VOl5y58^gBMLur
zLGmUUU|LkQ8Th?q6Glo49r2le$`<7b#eApt!d?zxf*?Urkeop>DmhBdK@gCP5@rAa5lNDfAUR47l3|9RAcIKGnGujINy3o1
z`#0$M&iC&BR=rns>(x-FEXsqrd-v+qtHVyZNi<}CculX0*JWM}3CbD
zSntW>snV*!^z!z${K^{@2g?bY?1D*w1?`%iAqkp;GNyI^4E0rno>sVz|5f3p#S3ft
zT)P^?iWL5aHa-;KWxNXCb>ojr;7#v<*HGWP&GXXB>P>yn#AEiM1K#63ifwpd+%z#x
zL?e_YmE9Pr?g@|)7is`QO)}hgMhmIRq8Ro8Q3ZqwHDV?Eh~Nf&5m$
z{r%l3bA3%7-Uvyonc+8@XqEUUOA1+w@>+0GVxO*fzpsSFJ0~ndE#?CDT^qVlocPIK
z^FiHrB)geT;0lo0g1+k>>^6<@H)Q~_$|dV1qdNW4B9acCV8BT(xTX0Ver(s5GAD$
z@S5?oFk*qX_P>EQ{b6}%H?}Oo4H`_a<*vn|IkGy3U!!9-Jy@AD-{E=W3}Bqrr5o*b
zvlLNLwBgXR$@az
ziwI}O~_Gj0*u_XteZ1pis#$_P~X{2IGH$b}FZxd7M1oY^0B6}w#|?oXNMPFd%D=0B|3
zM4SLht0ErhUh?_+ZX?uo<3KF=>o4~24w+o*Rr^8}w@Ou~IWk~Oeiim}{r98}5m?F+
zz^*FCaTjkttcGfqRO18R21Rg#75uYyLjo7s$)34Z_bUAMJ&E^$-WR||!M|%in6Y#D
zW>DYhKWjhEQQ>K-9JsugDivi*%PNcHGDb0@Qk=oYAR_b+yk#eK)zyxr=W&E7Yh2W5
zM#6>2S&^Ua<<~ut3rVn0LLCaV+sk=XTRv{Tc0r8?eZ{qO>$O9`*_)8-3b}qiRa{Zu
z?<7?^#`fIG=D;Sh^#4v|VejyI1Rr5ZM29$ErH!ykmJo^`(&r}*Lgxg6&W<4HDZ75z
zMf{&;4J6HzR5bq(#;>ohLqpSazUAb2f%j$%6h+qCPngy*R16NKog`|=Z|KNGKy
z4_63W8|SRp3eW|eRG@cX<-BaGgT;?f|Km^)SEYu2?ZE9Oc`~#4_`kx9R~_LXf0(WK
zBP&(UKRO%e{oMGUb9HCIo@>Q_oooB8yJ|ggX@y=|WNwMGs)*K&+H=mIOi>LRJ_hi^
z6}Mj^KP6ZCZwxywHLT1<4D|XoPNTX8l@%TEg|%UAGW;()D}$sq1H4b~sZQD{y`l%_
zXdjrYrmh}9=TF4UA>x+IP7ylopJo~%)|;Usy>f#%tOBCqV})W7U?NDelzFubIZ@#H
zJdV^R4(_c0*;TxzH=Td2>d#$?Q`>i*Z9(zu{?#`x&d6^yAj$W+5dH
zq4;RpD-@t>2j7;S)fd32N#q~dKp}>cHU2NYu7k1|XuQ!Xx%3sZ&2#E#&U_&mV_H8f
zA~6XJ`Lw_fvkgVQR1J^$v7#%k1t-VWCmYpV-!cVxC%Ny;%rQDf6~QcXTgin}$@aH@
z`9Z^kIXUF`LGMalZa1ia>SN~CpIHi>MWB8WzxXFaIX^yKentH65ak!`^K6u5v;x;H
zdFIi4<)jk)*U6(AF#DtJpP}OMlYEWO2J@ps5oP3*F?aR5V1|bsm@zc;T-1ruin@(K
ziP3U%Gmpkf?4_7M>cQ|5vgiId7g#o<%7lED7SV@%7_zp~XP$WhQmszu45{Vk!OIoKIBz1SSaJhQ*u
zK+a8_OzCvxGT31ZRcfM7al`V+8~wb`5`8c|Yi3v$-w?Fuf=RBsGdX>0LlUlAorr3
zL_*qQ_a98e427T_@Gzvxcp;$emuhx9n7fqOlH7U*(QofR458;MRS4LKXB12^+0|e8
z63mV%e1H?Zblc(MHQY%BLkBG(H{XzulFI2mtDPwmg672r|6S#95GDA`Phk1LNVG!&
zjMkVt1fu{m^Br;FJjw{fVZ){t!yf&R`fu$>W>C^Cr&O
zSu^-vv&9ob6+bY>8Mqs&P)SDNLjs0^sYXv5kDQ#A%hx>8DAo}3TaT2KGIBdp?37Q3
zNbKf!t_d;>d!wNpmD7b<5unu*rSF{7t#5L#8d{P3ca0fXN&xnk@d#_G3$8=^2n^`W
zOxqge^)NRg#g1Yh1r;xb9tBHNLvuh3rpXpv!CXL|l!9gBTpQl!{p)e(`GVS9GPu4~fFE{Yl<1
z*i>@jb^Rg6o2?r^5oHeY7pUYQ4tJ52gW$A9DGIDiJKgO$uyO<2xP@JL@(-)+aR@Is
z(E0pUenK0&>F4dv$dFMBM9(I)5LW}j^rR3Gj+HE=9R4KjyPwlJHk%G>h-Q!;$mrC5
z*KfC~O8DME9Ucm&`0MV)PF@Jz;T)jy#+c)oXIO8d!Z_XMcxFF>{gJ{AHuu1))?ec$
zjdLGdPp;xoXLy5QZn5}3i#wrVZs^0wG%DH#Ho&;VMM2%x0EGjXg+f6e2v3^-|9q%t
zxF$aarN*6NFW|i^{of3*w6j
z3@whlhh-ND3jfX8?jkqk_~qn8b{!J%eead#H83i@yb}&9qa3TpPu4N;vO056ki$P^
zmU-?7LE+`ca(~Ns?e>XvHMYiC4I0|EJzF?M1*ndn+z;f=q^5K
z{;;t={!RCS8Ho!PkR`uiiJYaqnPk4w!6X@&qq6DzUS-(a7eu3#C2cY!RJJDO;^aLS
z;J!NZ#g=C&XTo_?n;DQeh{g$Z>uf@ay%f~7x!FP(CJe1j0d2!fL>+{_XQ}r|3F>tW
zup^cd{eQ>hZ~VElI?<0G{3UndxDd@qxR5!l!q|)eT-n@K#9`AfrDV~ef^Nq}JV8QO
zY{BU(lB(pEH0zsSn2KHP!PdjnC2dp1of^vI8%b0e8Re$~ea|?52v$-KFNs{p*Uw$~
zv3O#kvrZ8D+%yhsJ=MXEMOL&zmHofVPUyfo;DP5ZspP|I3t%PsdA?792p>x9YllQ(
zx1l~W${##T{Elj@UiI~)5mFMr`Y3@t+6%Mb>eYd4cuk$?rBS1T{J1y3qOF_Pg|q_tbj}82~2or-y%U2HK{h*
z&cl)2vUj#ZX5dv}C^ywu?cRGoC;t#~(kIeBbtgG%`1Nb7Qi5M58C+}*RIB`q1v{sA
zm%I|%D;Flqv_99aEU0I;M}Wl~gRK8f6BsV@Ym)RjxO$zL1v}X$3IA+gU;v^aXwyvh
zszpgb+qPQx*`eWqP_2%?>-z!BVb^fn(vbM@&qaERU8Hp(ZJ@+5e4`Ut0p2;_G;?IE
zBppi4V_WxblwtMDd_eY6m)_X$#Hn090wbL*GfQ`E^UjPx6y;jw@#P1V*=gm%R9@$w
z8Bqs*loB{i@b)hK@*penII(IklVpJJkJItm_n^4I;^R_mD6)S4UmSJB=UM&DU!u6b
zL>ImsXHZ8)J%mfw)*I-Piw*mV?~T@p4Gd#9?S9$abkf9I&o+{tobNYA;j=Qxld*ly
zi!Nps`wjLlN`-qBX=m4ZqN4&Y;|cAo&2MVmm%=kwLVA6-{r;h4j@8N}Rc&)IxsT9w
zGuC)w308s~fP7&k`dC`u&plf-^YDF~2z9c+Wof0j!+y?oVlIoxY5FUtPz44AwKru;SNeMRByAFb$W^o8#lXhvf#v%#{<<0Eoy4L*ON3m
z;G?f_nlJ1egK-4X`B}3rrimkU$I6?pvWD6W-B1n_?eGUvgYFzQnVX*Id!Q_8iTHQ_
z&NzDWwk5pJxxO^;_^6aUKixXC);Bq;}h9b6$%ObYr*maKQwK!{vRM?8Mhw@bZbZA=
z62OlKQFyq5BsQj=;WYO3j+d*h-=%!w^67=^&wYAur{pbyH#ptzt1$++6D7=Q!zr93WZHig2UR+CuFTT6QRe5
zTI!bCj&G0ZBd>Xjg>1SY6E@`P(E7#HKN?f1BQUZ91ts2BNpnD*wef=}H_={OvP!#+
zMY)|%JD;tJ(=M^$E7$183ep(^^8}XW_Ceu$=c&r-41V;7y8mj3s|mR&dZRs|uol(s
z_G`4N>I?`cQ76=budc5q@ny7%c6BBAZd;?ZJd03Wt&1VHWA)t~7R!rbpDgEvqX}IC
ziWa`5;q_6qdB=%;U<_||b1SJr7Fr4OFgz>cLaaE1>J&?TqN=j&!^l8Xv&d52{d*JuH=7!T*~dFPpS+t-r54Xoo3|ez9ty
zGhcU~UU5^8#_Xwzo$!~2PD~EP
z+%3+TOFFL{IPYE~wS1+_!?jqag2EIL+_yS|Pkit0ZYDWyCz;~&?w&Wli+DMu+Tj=USF
zxPs!d8$K}eL9dUkxhVRXm=75``Z^a5?M_{Ko^ZyM#xbQ`^rmwCqjcsQ=_wLg`8te+
z2@)b(i_b8YYaUkz;QVGL0!Mb6LW8r)?M9C*Dhl{GHvJFA7b46C#oP}HCi1ftZdq^Z
zHh9|2f3_ZZ*5R=~?lNyN?RY{7>yp`TvOpH5>?smoJ~$%TsdFUAHzJx7DRSG`)19;C
z6Jq~>(SCjs##LC(Cc@lloq3?uZTN9Be9&|&)S-E8Cv9rGq
zly%u-$6p!*QHNqbow%6?@#Tj7lNV-^60QT{#Eid;eZN@wTvlDm6`)f)`xQXnfZlL=
z`x?~K?sUJ?#zi6Th`zEhNIVk^yU0KJqWRVSt&dD-*pSizjoQBZ%Gc$KZ@}YNwBZux
za>eZFM)&2lmKL+ft|oc-?7FB^wyC#NdFS@3kZLWZKCN|p@3+y}@S9-cj#MRC>-2};
zl`qxe`8+8OW4}eYDE3I&s+J=o7h;R&Eb7Zl>l(NfNhy7J^?2?f@{0l2so9F-FvV#x
z!D%+-rv6U;;PBFQB@(Id#Zc2(liDDz!q5(@FkwNo8Qhd={%&?^u5)d1WTanxb)x=Z
zuzP2qtf!QOP!4!#k86Dc@ny)V>eAbh>e-iBFA7;D#B$|Vdl9nzk5iOXA`i+j^%p>e
zZ>qqgwJ$DtWk`NS_JvKnX;ZNgd-*}C(MhS}Hy07iaDv={&YXxCiQe#9iVd1ZxH`p-
ztq%35>Ah687=ku?C!+8~=&}cUfg9aAMndULJWI@~(ZjKr&d0i4HEWNaXZUCMyJz!f
z_~VN;W%%ds`KkpN#>kuZy4C!QS=QZhfIb)lpZRqw>b#CV#EuI2+BgO$7*=zv+3W`1
z7$fcudTqIu-2~6p^0-Y2eFJwfK>>pmMr7l^3zLPLPz@W|KTOA~Xs5+{c!
z4e3R0#MO%GcMHgE(|q;|;Q8~{uLp6DUqn4$V~*NCw1vf^1-yHcT7s03Gj~rd_%IK$
zV4l){S@1(#8hH8ASwu*3J1jQ$I0Jo6aaY6TagyzanO2xh{Q-ihYFz0-XikCNHhR31
z)4W(jO1ey;E8#H1t^XSu9i8y$&47|=6CL+F@;V$bkt~8-2)WFzhvGQKy?HB<7I{@B+gWoMGirNQj6{tUe$x6!72f$)p|y{Tl8
zr@t@7y0%;DZgQ)R4G9go9=uO_{9&Xv&d+tJ(Ni_0g@A^Antxj
zu`jUvAcW{C8D?4Rt=}X(VQnkr(?tcBU!TZw8-vMjF|gnjKgIc8F{J;rRf>0VBrCcu
zWhK4N-K8g?xWU9=%|f9h1okwGr2ZI{cC^rCL`~a}t5)gymI1TTpL^GfphtPB)XQ?>
z?oygE15(+2jd1>LPpZ40yIh!|mdeT5p&{Mr#0wPWyA>6+bCo$Lq>a?#C(o3X0~IF|4O~^J@C(i@PGTKkTrowt&7D&BCTz`
z40^lUZaQo_a?7)kdPUYfP0*(d&ATvF;}ZRR*?A=c9Ay^~3i*6HSObE@ROYjNlgj=Z
zN<(sw4WVyKtn(KOp^8NjDZS`7$3XB8I%xrFif8>_o#txzw2HjDK2;#^eiz)
z@Mo_m6~qXU$yk(?Y24B--vbT7ylFYA2cQM8mt%5I%P`IG>+arf)AExi|xdsQDIcl+OIWK0+y~hTC<{St#
z;%vVH!U>gd(W$U4O@VtDYMvK&{7;-JXvs>#{*A}KE9Q}s!<@NIV
zczu`5RGL3-r_X5&IiEV;DupYxvxQkH%@nwlX2RyS0
zeTlPx?|lk9Z*n!T5w0cSA5w@?0;X~VrcxOEiBwLe+n?tz8M<)|0$y*4tGk^SCFaM8
z$1;TyqOtbM!`zN;$Q$+D=v2Ic5;~D%?hhhAtg05P-_gzte9@@@zh>BG&L}wJE|JYI
zr?RQ`E&i@nfb!48UWx3K`1E`>g`Sa<7=yj;r|B5j$HkSEk&{6eq3_iXw->h52udiO
zIU9qCRH8FJj6cvEE=Szod)xfU@k-lbt7}6lZ@y>J$h2eO
z98->qT&z&&F*C+^9z|@8UVS*HEANqRdlRLLgNGN$y9XmzPRVN@Q
zWhK8Km0o_7@x)z{2$P-i*230)4zLt20ja3g9z##?O9|uJe0i&SlteH0(aO{FCuO>a
zzg4;Gor(J5^t>Z$gZP|>9_d*j0lNH-xD0nf8?w2Q)Y^NC7!f%kLQ0`G16jgMoPr`{
zKD2g(UIh^?Qd#?w9!BpFzusnh$v6iOJ`(e1aWr003)4$?+~$3D-TeR1wyM_|2P%0jtIl-;-?VN+6v
zctweV^Q(b_@b5+%ijDAjdcv6`uaBhrlED+v67rLRQbt<2c{YL7Ha}*_jF;^0+!_{;
zH0bIv21WYRhYOH`1l
zW^^CjwXy?@Vd!y6QBlF&ZOgSf;z4`+TccsehqGs`PEoFiSFz7B6K-;RO)%eVIj2g`
z+(5n0&z573kW8Z>9TjpiTM)K>z%aI!4U&&VqLh`aJ_`Pi7R;K`o#>GuA2)k-N7yr!
zkQ;KC;?zR<`P+r_po+r#9`No$PjcjXpAmVNI7D0`%OZNQhlxn%bPa*ur@
z=R!k~qD`fNpvZxXEmtouC4#4-cRb(qz{$+Iz=3kwk6mhFCrIf#7Dm);Q7n_&&>2Qi
z^tpz-^Sox((2Jm70FgT%L~b9BnF;rV$uxiZXua5Nh%JO6#z@oimN*RDJ4n6J(lu-8
z6l*p<8yF?)M?tYgFMO{oH;>fADE42*$&qEXUTlPueLVMD&-4*Zh@5yc_9FK*1rFN6
z(HK4nqOj4VmwMUhv1V=Jy6pGKNL^C_QZ+CW1VL|>40a~JI|G9QU*Ur%w
zp9AhWv(BlAxJ_%totezxXWFX09Pf=j(6j3!ep>48kE;#x)_mBHe4+>_aX_FGb+22e
zM}EH)k#eJfmXP$8!=0^l#6Zz^&ucLe<(1`zw;GDOo0{^Jy4IlEzFp8hseoOMTLuTb^$xFwS6Cm-(s;bOhTxQA@qD5R8Th6m}VU(MFURkDr
zNKQ%@rBYKr*j$k##>%`e4Zh8ROS0pO-HU(RK40PF=$n<5+(OeOB&F
zkZc$`RGPE!<(d@8R!ADGX7q6V6iJ=PlO*U|!$M%PG9Eq0p+>TCf)8o`l
zGoHg8Bt*h)1`gzVDmEly2$#{WqZ8u{%vQ|~JMLm6PQz6gHy(CdrUE|$-
z*~Gf`T&j5>F^Cg&9T^5}5&+m)f3VC>W^1sY!*2nFX5-qnk9j3Guz*AN2XM4Zzg_qP
zIA1S(04+LrMKZp|wamRfEtRIX&&*JPdgXO$2`Jq;gZt`Y4qr@esJ*>hbZB0+>MSHn
z@C3t-qZy9wpwgNokeF=CWt!gd?8t?L2KkAryn+m|#1&RC>1q_$j_fQv%|8?BWiRA~
zGFqu~D@c1jc4hLrNtx~!_WK+~3jueBWGe%UUy@^GVq#zFFz!&btlDy^a^=zFKHBrh
zzuOOFnpshgz%+l6h}pZ7y{~wepEoa6rQaF!OmY${@s?SRF9P0jR4Ao#P*{fEje9kc
zsO7nKA-`**WrUE3I;Xt-i;LJT(Iq=Fav^-afup@PoT6p1F=SAvx`xn7O_yRy&1YEp
zB#6TbVI8l&>9(M1+*gx+=#tH1zWlssx_E@~eBT-}v;J1d;&`$()8@e;P=KsgMpjS8
zH@!4A%C|*=VOFd7eFr)3XrcLRhoGKOvKVzhpMsPS|48xe4q&-
zTZ;SM%m0Dz7CI5Mm2Br+?*
zi(ypnDB0_wC~N$+0kP19U^O)-!4ov2dfVM9py|MbdSh;L*xW&bfKg#1*MHo8J?Y@+
znNs!Un-CM5fS2SdM&g8jjb>k&2f*i=Tltn@8@-6Ts)0}BH73jMZkSwB9brRk=yI=e
z62hXwCR&TAa@%#3dd>TC|A=Vq@sBTZ)`b!$LuuzAZ8P~)+Z0k-`4s<{h=?vK5yXiB
zv8LrGmtX9VSlU{O7nI@@)n|Bi%TXW;
zq%frxonAX6Li1b^bXw3C4g}n$pCg{v`GUh~i)oN?O#YDal-qnuhJp(&uBoX4-Vr?m
z3$95jqEbKN&lM8eBVTh_o0K
zicQsrhdek4gowwKGU3&S;#6n19Gq=5q+(XBd)-t0Qo9CD%j&W*q}IINGg$+9cCYhju#X`w1%G9!kyE8QZ+m1|k{K?sZ=tu0k`
z33gM3^6j#5qq96A%!dilV_-
zg}Vg*V=HQ!V^V|qfw+%tT{pGUn#koenMGYE0!@n?^7PD6*
z`yN$pqT|bKZV&=%aD{PCoP3sG)~vkfQPAKpWkSLC3l+?${C-A?f|CTBnN9!&VW;*H
zQ00PZ5VS-P8*QZxx7)!x_sVesb#ap=EmFzqtOzJ800lju63aOAaf2r#UIfk1u^HyQ|8=~
zo7B!EId}~?@xA+GhIE}E#IY7??e;yAAbui!Mns=VFFN8QRLxPxrW#-1WEC$zohE;G
zPr29^c=kk+c>=yrGTJ-Y!`$0t@AhrzE^`Dzl^yz|kWlx)CU}+2UJ%cG;ND<4(vnCy
zEy3?{wurk}o0Yk#i%JNrc5v{Am*LjdIjey@NjxubaGlGwoX)~}KQn30lCQOchN?UHr=Dk9BXI|GVyu50)XoMwfWm`*28TKpI#sL~$4Q
z9IY`elwn*FzEfCv-G9TbJ6u)J;fFEmxdr8@Zl!Qgvd3JjxM&UeTB5Z@SG?@JbB=2X
zvIr%V9oI$4J@j}s7v82-knX%}RHi+gBG0y}&}4`BVs_o>$V(IL8n#{RNbS|XZ?$8S&kc!;yuD=&zjg?f)qbG&zH`5?g_@ufMyYcAFlfap?pJ=FxK
z`LiERjhxB1h8hsw)G6Gt0B*H)n{9LfYTlYj;+b)>b=<;TDqu#uyHFIl&<*I(VZeLbkvPjB#!!A68Xr24u
z{_%91+q0c=_bx;0m#LeZV=%ra#iBL+FWJu+-#D72{$mG#A@Dt@O$N-&4QRY-9N*tAYci|!UeI|6o7643Q27^|f+y;up
z6P`bHULPoy5}=Sd(LT=5UnrH~v>b5-9PKI(blh(fzKb%kR?^$ny1&}dZwnYG
zZPwZP)fm6%f>TT-2O_zw5B^TJ(3(SnqscDtfeEKJhqgo5i&!O|l_|!!;s-5X9?KQN
zJq47G_MCzqYKOjxc{P|wxAuN$vqiw;Nl=0~$3^nBOB|CP?g6R^H1zxPDq6FdT?=yk
zE}x8f9}+;x=QXJxG}3`P^$NLN{0Nd0B@a1ReagbF)HJ0k7TEDl`G^mwSN}iw$eDA#
zbbl;U>_F-*1CADPH5{rO#hzA<9#lBzi~R(;P&keEAJ*Z?;FAv}y|jt1os8ZO5;8og
zI|pZqN*;l!vE#9<}JQ06+^kA+c7KUV5u3tG6
zO7@}-p>R!zKB+ygtas@OO;PF3cwTF{qXR?5s<7&cm_s^l&xb-TtLYvG%&Z3H6P1eA
zLkLM_Btyk|tmpjodq##cE#nD}!!?T|dEulGCP@IHz(=v%z*NKFq73C4-etxL1q+*7
zNVDO~apksUl)xJseGgDs#nJ36&dqHt@Rqsi9@wOT))BZe-Al=wu`md99MX4K@Fh>u2%C%j88^c8I1C^mn!Q&do
zH^$xSi0?S~&)37bm@8o++|qo~zttbDF(jyP$%Q^mR5$`_20(>9MN}FT4kqr69=Ajt
z8w#nl-8g^*Y411JWR6@Zdj^dVCRquF5>751j!z$+FT*|M0+aEOKhFrFPwJP<7%f?y
zjcYf>ZUBDW3l-xR{}p1%9XoBL)l
z8ZR{YTrQt$erP;6YFZRBIZ&0{8?p>!vINMpaZy4atIoc`mhWGl!**~Oyxs`y(68Iy
z>LN!SG6i+OFwtm(<@^njtnT=VV%!^alxtZ^aZi@r-FwFXxl+a0#rAYmg*Yw7qTR_R
zp164o-sx(ASDCi3xjma>=aOomO>3b^P?}(GVBt6kork5a#=<_j5+)P;Y%#BlP9~e}
zvhz||xO}*%i3!S2+@ylVA}_s}q=kZ;VS&?XL0hl`glpy`wFn})P=97
zX$iBwBrQ~kw*zRb)qVVrnJ1Qr_JAk+l%_o;b=hre`G-unT96&ZFP{bS8b-mm){(?}
z;|h!w@|n8{r`ZUZaV@3ekhYuqjD11u2a6LEDf5@Nlwau8&5_2R{Ol{A9+Jm*tF?q_IU_>_)DSt~)*F~%AHQ%z`gqkz~>g^m79Y&mRo+_n&zI^#N6I@gh)
zx@q1~I{6SJa=)xyyM&`d8OAr%#U?is?CVV9dLfqaeqqub)1mA70MlvoGI30L+&pgU
zfxQr_(*Wsl^OL9-{5MsxWgmhxgz|tSHq`NZ>d>v$g70$08Sg3h?(+TYFj$Xr_8ZiB
z`+>?fB1C2RSnM7iTB8mwc^ZqclTBSs%}5nfOJ+6%aKWixZCfO5mXx
zA<_9V&!hZ7T%mb?N7E*>E7ds3C!5Dq#U1AHjuhAHmTlA$&^*ExS2{(Cyv#HTbK(3G
zT!}ls_JV6)*gU3g`Hk$X!3eD)hW3Zh^<`fwPI3g~h7ecWI419n!DILP^dy9|2BTB~
z0e~Tl{Cgi=fr4x}%ej*Wa=Bri57jG*>DTyh7?12@gH|7(M*JA;&AD>(A5Hpi4hDqn
zQ1R1TtFBQ4_YMdN3sT|;uk$NLf<%@g^Yt_RgN4W8MBt8i;!gi!=pF5KLmbwBFMi~}
z3tf_a;%*>*bOB6&DnSlI(A^a;g;Fd)xzEx+aNmzlc^E4HLD9%&{iSDOR#ob9az1>L
z3ti&60>YI6XDzBCwe{tQT^i(Vu^lgL1UFYh8cb(?0B&6R^XR4HT-V%}kuvlCcTMAX
zhI;XwrM8}!5M66Kgc+{K*2ax#@-ERbS_k9|O`?Lep)T16BCjd^&9YV^)e}!Gmpu$i7zP?ZOAx48;Ue-DnXC~>SWMTh~UyS72Ou#rwPXs#Ox9Fq0ZW=1T{oq8y
zZJ_+KoA%r}T4KVpi^rtpvVTdmP|O{l`v5t(&izO0Vq~Vq#XC!UU+^4aONAY(Rs;Q=
z8KFl)3BJn_{0AXSYTL{j<7|IV=UP$@7GGpJqOLp8L2(zmc37Sx+779v*BNW0D@Ttb;@>jD~%hEQV^y&-llnwR3
z#uWicj8wB60PE53@L#4>a^JyIuGmZvzMo{O#BoXc(u_f42y9JMW~(DDUhWqt11j5@
ziJP4=Z(3#A?QOgCF!O;6{5DF9G~bl2YG6s7Up7``vTFMJwa-L*!l!?h*UfM)^0#Vlo@7@^WhWDTh!;_*;&-+TB@Q5pD(
zGT3S`WY3h`2Pfm(U`+^Rnn@~L9AEy>LudKz$Js$G%!HV?Cvs6_IYr>)eu{N5)ekPl
zIW9%JiT$C)DNAk^oFDnc>IrA}9RJ8&uNa6%L3jV`c1@{P)OBmc$+mkfGUxDE-)yi=
zp^%r850RaB_w77p6ulya#hSl
zC-4s-kfuFV^9`q%&P9N4IMLBgG`!hi|0_2wYkGad2RvHrU%oM`b-S7Z;`CM*763W2
zfYQqg!LVv;EaM*e!>iBGkx~}gA)MdxGKPo=vle=x^K*^R`LCFVE#Z{>I}H7mip!tn
zQmTG1?yqN{hqxRhdA-e$z66%F?#nB%0%6#%akHc1&AyGpD_g&~3H8O=?s<)Uo+o2?
zx6x^t)y9__O5&t6thd`Q92;25CSZ~9DfZWn-!JfYcozW~r5WxQ=^n#1d6`004*$xn
z%pjywx-~x0rSa)T$)Lhq2gP`vEYdb6{pCoL+I(S$*LXy^e1ig)-}rJI&YD=wTzpkm
zoR>_0xZXwI_c7Lq-onmP?R-+!7Z91!lsw<53=)7@(U
zW5XYHjhtUm=AE(@{Qq5krhW3@FEb_3x*7m9kW@-6BzgUXB)zKr$D+W~_pbiQ>KA_>
zFf|Z-8GyY1$zyTQg&;z53QA1|gjlJJ{#1rCi!(;eAuYM7Gr~kCBXf0=Uu*GgxiL`*
zzE!ds=Gb)1o?^8Mv`B!HnbQ;Yej9?f^er&Ouw~qnPpQhnjL`6X&9?
zE6}FO(`=XiA^jw)e80|uAMgPyphlpOzuni;Ddmd0bhB+ZmS44!4#uY_dWP>~kb({(
zlvRTT9s;IptfWu0q&rmTHMA}H24z+=ZOB~bO?8&oM!d)C5GZ*XxY
zNkp>np%2Mog|l_(YLCFj4zLv~Xc~-2GTOEc7V>~-tAqK$GaPbLB+~B^wQ=Em-DQN!
zeJX+9{7c=(`th5)83HM=qAWh4*vLq{ttICCR=#P0&6o63?HiFnLVX~~E+1KK?FMPHArCL}FDrR_4#f(*AMI!Jl
z!1QaNH$a2?=_2PoB5w{Uv_F`kshFWoK8XFlE_;Ipu(F{+^0j#0JNt{qfbYe!H
z!89-iM2l09b#Jf*D)bK_Ub=z7~$H9)O&`f|-*nVU1dI=t-+f14_hghZ?obct+iZ0|r`sFWChPnsk5=Z|8}L_PTU4;V
zLE9AV<>9YG*_h~<)^IT-=GR9U`iRwb1d(KpmBF9SPiv5=jrIG}l>9co%qL!H^WugG
za=o=+-Z+7#>E#yp`oO)G?sSf4XM+9T
zz?E);qMQ*=*&uaBT{JIxTU1QaLvXsI^g@7EIfPD8G8&o2I^(Z
zgMc&B7y65a=v@ZrL=5tzzR5$#u*r@M3sb&1hF^gM5rFiU$EaA=r4r1XAiku8{x}BZ
zsBf{Kzbjgf?pi{oK3bD#Dpo(3zlYfB`E+vS^~P%Et2V+RC_xA9VL*+>^o0PK`LgC@
zt9fVqRI4)(T(X<|^Y4$TSELFO#xu?L!zg~I4_d2}Xjj6Wli+0Ah({o^2Kh4enndsN(Dfa8J8cqCy?zL#wF3}^)
zlVdq+iXY3%L(h^vk!{!hiFBaR&nM)Hf@0^xij7V_56-6T;)0ioUWC)%`z8T_NEQ@^
z8+#!(L#@h`fT@LsKv^Q&;s$G$4}QZX7Q$pF{^RVK1*qy!JL|qceIh0Of;-qIEtP*fwnH1@b}CjG#MU4P
zs)3);kjLj4nH8O2ZFv0Llb|O_tU!7@j9r5C4zRvBC-H-Q)}9%5stgaKfnP4h^I)1p
zdX86Pk?ecsc6WBr6k8+pLvhx#t9^Z%w=y1wdS8q57)1&#iZX#tNp`pD(NQTU;`?~#
zUWi%iop|Lb4=Pwqd!=yKKDb0Dyb_u{IfZS`+avxl@31Jw9h
z(?Oaz|9AL2j}9+TS%Z?}Q{tp7$VezFhj%L+=k1oIJ^R|=9Z6!0*(r*ypJEt;i(QMR
zUE?81r4DG6`#8m{7462hqvfUT#th&`r0V+%W1s|LTj9=>HPH(y~M?2^9H5G@r_tJpI6HW5ITc&ZZQ_PHdu`q$%LTY*HY
zIA2g-QZa!qkY}Y+X?#tP?`NSN22qju*dNS_TB`7ONR_kdRqC|a-*kE0-=@Fpxw&fY
zhM9on{{H->y(@taG?oZDrOLA{-WoC9Z6staWQ6UsgkI&jxekDT!183r`PD|n(3OBF
znqsp#PXWW1t@pf9jsJ_dw}7guYuiQ@kx&p6ln#SX1f-=wK?J0vloXLJk=Qf>Qc@}k
z5}TARX=xDwX{2)_Y`V5|!uLzmqP9lB9(9|Co3(RDh*OPZ0p}9+vF2<#6%mtCZ!)S%`h-rTdJC=i$~V
zC{*{H|9!Z$l+9~oDk@Yag-qosSU+pjoyGDb0Vc~C3UdBty4{q}W0J+uFR|fv!s+~xm~xDtG8EBAHq_dpMR^>sj;8WqOWI2ym8#jmeyxy
zB~4*LqiB}O({Lh`uEMm_p*Q@2h4F_!TmTCv5&@zr)azoy8IGrS@b5^@j{$=4EaU}D`hG>Pv^Y7(BMw6B;1YPeE4QLxyuE0kmTR!
z$-sDu^!|;Ly4Y=O(O}qTVcPT@tOUb=YyxS4X$^Wk_m*oA@3d;bKYxupu@(F}WLexs
zbZ;!sPd~%aoYk&BBTQ)F1Q})oQGc}0a0_+OHU8@%sKeeVz^CFLRh>B?%JG}6)D}rZjAJADCw8|YAJn|gShU5riETQsV5Uh5Hyd(p_y9>sDv*gdgG8|8!Wj_tqwqwHqJ
zb}?T@JEz8@&{%w>oI-IC-`wnVA$>;?rm+4YEjMB%>Pgx{g@dy#QCov~m|pww(=+}l
zLGWxI<6FqR%1&Os#tOK^7wjM1=Uc(<(Ta31F`R&&B>bDU!bC6jCxk}CzJQD-atmdY
zSGjZ~z21|xoT2U}y`~y4gp5kkP*Cj??-$q0^iBj}3RK)4C!NMLr7GCEi;
zz&rUEr(Dc}=QI>0DIB{bMJrVZe+rH%{UjMm1(QSHl73;a^{?*kpsfUE$2QvKD^g~Y
z`ndgcM|acRH^I(fZ~!GVhwVuX*tpd5(A|mSiJ*a&HFAQ^n`q
zzUL#<0*-2^`+JNn04|UqCt$7d%EyCh%5}gng6Ai+EdpC^1~Qjin~weRP2wB)d|J>X
zz&(orylWtYob(T1zvc(OE>r5Gw*a#}K|nzI2XNU7fM)@ewIXn_gbV2`k;9mg$V!Lc
z>K)gjjYp>;vtXf>QFAQY2Sa0(@XMmk+>LJ#1uOf8&Fjlvk$^t5jgd=kfX$bqmyHj|
zX^GhUHC_EuPu)(i@dy!Ghg61g=1Hu4fi1__?ovih|@)I
zn(!O4Lb`^EZ~gaw-g}S~$Q~$2VdZ6j2A@O@s^*h%FL)ahe0B1Fdg7!Y906D~t(4(u
zoP&Q3IJ}V`G9O@}q#!+Kea-V#`vxEg_|74IW*9ILkh6F}2K13{24p*;Z>sNCr>irPVhTs^l1WC
z7dga~xpAF2J^>g*;#a3g*Qu4WM#G11Qgb}-cmK2r++=7L#mj~0R6!Dj_oOz$Sj;_*
zUk@Rv^2Xbl*$(bR73r0nxf9R7aQMv{Dn;x;DdG+}*Q)fH0DAfl3e*oP!8TymV(=FcmeEG0{N|Z0XQu|&PuG+<
zYKlz=eQYX-UGpDy>^jABsq4^3-$-L)>1SYS#MnE8!-JSTk)_=_ik{up?zw7}{uw%w
zlkXJ%;x65q0C&k%>xIU#h|x&3DM=9gPvAZyykGN?Er&&(3?A1@h#)s3u_A5U3+evkMb8Sr-J&
zdg#4Wg0p7p>YOy!nNFPWhTXB^=Ml!K%U_KP47S=odz3$)epH^IOLJ@W@~_G7`jNc1
zy)r}H?p~Giyq|2n7%5N~gNagPuo{!5>#$M?qG#_Aj%z;QnX^l?G_ohSw7+O4lu;Yl
zQ?znDJqG7g*fEZ}cN;Qm?O)tyh=2>AAr3}7{D4mf#ql>xjL^o%-Bd`2867S-UG8dH
zN}`giP=I)O^CV=-|5^^1kGnMpMkNpRUgE&3aLnDy$iDO)Knmb@o;cYCC6o%-WEAUHYc}Jp
ze@g7P-&l$8;;1G?ef~)TwJW^v+!8wFz=qQFaI@C0)*?%n4`9PDm$*V
zpZE)eB4MD}9MFuQ!JAJvNQ90z9M1h7M=Qhp&kcZog38>*TN!@M$JoV+VD5qZl!WL>
zUFeuE7lP^OgQ^p!p|wz!#ZG3aK4uRfhRz50iiZ8M-AJGkn7jVYaG4+Q-l4oTO|>jE
z>Nl>Hn~eo?N+U~1QcHz>=k8AjgE%UDy75dq;s(q$8z-O=_Kk=ZV+dVm#)5&c#
z1hE!bDYX`-jUGaDcT+z;(%YFzZ0#7_-`t-3SsZL*&>(E`B(D6nMOV5OUiB?Bk_Rtl
zt0AN^mofE+H-qD=mw#5_ss;o`zUpHAaRGQqN-$t{|M6u4Xp>E4B;bWudU^FS^jbf{
zVZ&%4BjEu4>cq(^Jt(!`oFYvU(J!EX6A*RW@w1bZ1dbD|*a_STaCm$a
zVs8dX(lk^&y=}CpLV;ZhdG=M@LXQ3sgOobF++rG3;i0DQTlq*)mQg_U=$PTPT(lR`
zadww4ZW^Y;BrqN!m%%u=!A!&jhEh_8i?&`JCVrn5fTBdOpby(=STo<*I?d%?I
z)S4xVP=YszUpV2xd@BzjdacRm7QvfTU-v2EH20V(lm`vr#Ahc)hbknU5M3x;z9tT^DmqEpLZdr^PyB4I$L%S8fqFUyfO!h
z)a|R^@h(Wgn6mx#e!e`bBz$9z0wNu>bxo+>sIU)pL{C~<*7WSVr?qSM3syoEwCk3U
zio#nS22|fPLnaks7dr_Hd(3YL%>F;ZXs17g=bgjeD%L`nY-&h-D^l}I}z
z2;1v@nz`k9NJ6(ab`*1Xm`u}q_Ku|X@hP%BHYm2i=}g4~8GOaxT}(4>YXMJCYj
z`~9g<1C&h`+h{(sUiEHF`Dc+ci75-;EA&#(rbo4TtNi@;_)vn6Bg1I#7@cx?_Ivta
z)@D^J#nB=v#=Y&O?`;8fuD{`z0t&yGnJhLs*q%~&rpW9c9|}Q3&&=)sgbzRjWN_g)
zPk0A3MBs=6Sq5oL+BYJTIHP4d$
zelec!W`dxuMAq+_kSvTZ1|bbqTX9JY5{4?9C|Q4%}#YUM)SLC&X#K$RXb;GLKS}P
z8E%iknQ#5RQCFOa8>4ETS+NfL@-&vWnkl;1(1Iu%eMwZy+P&U_;bVTMk<*%9okP*{
zdxFJQ&i8VjgTTY0Z4+ln>h`=Lsq;XZk9DX?0pw-ux(BHnd3hQS&hir9EAs>!vMoj_
z@}l4(cYGki?O=PZqjQKDbMe{{tLx3lXmC47=7xG>X~!K1r}fQFw9E%MEtuU`%v=R?
zNFbmV;KIN_cnTprprfn=r)e211aJrjDGpR@{$qcga566D3JhoLtTG(QKVVA?;Sasu
z1K1$Vm>>vMiM)--?e!;>s;;jZ#}Fhb5BaD;t~ZtA3N#%+I+J+sAaz(U&De;iau76n
z85EQ(q>`;?W$1=W6a9!4i&Fa)VzxxoJ)D1NP4JPgbXbN{&>Zb<%-$L06zkyj%Fk$y
zme?Ah&IBPT-+7^zDQsIYj)wQ?Lt}33e#NC6+JVXXyK5P&L-+FllZO1k_VNAK+a_ckF#BSQl&IozFocUB%K?c;X0i=)^r7
zMOWhV^o2z8hIunrteb*pxU&nWE?151Fi|wH=Mi78zWvA2&`|d?ccP&A6*T(^=<^xP
zK7f!2>1MovsxPH~QY$DAO^aj1I>ZP#`}o&>_luoDBli{!Dpfr>C1Ag*c059-rg8Qu
zwhQv||+feO45gfPJS$*mNa9ojlzUzL>Tr0;b!~TI?gN3ZlE68W$QyJ>1z1
zX)V@ouy;@<>@&~i*X9~l@K>jn-y(Y};GZ#7+Via$kDb3xP#l$EJs7O(Mi;X>{eX&&
zvaUnzN6^wfQ1Y=f!SJCRFfZtdX@p0)@mE=52!GNb-nbWlHw||
zkHR6oO0MxID7B}a{R-kv;mE9}W3v>+d{97GX~(HpnDuBCa}pWSN65#HHU4RcJ3`C$
zAyW_wy!^MOMVG-i`VKQYfEjL~>K_*>c7GVV#@*h=AMCKy69-IH7g{Uf>Ra>V`Jj?a
zq;swL%$&MI14Ytg*ddu#739Bw-}DoqkLvrOd*q_UbCTM
z!4WB%p+f^Mhb^w2-`*UAr`wGb1Py@<+JAG>y-5K4Y>R-|PXh?>Z;#ep8)9~OGJS>3
zm!DlREnJSeN3PM#ZMewXF{jB8(43sx>~1#u<)mbxJ~44t>$RBnXRln3kj-9gxm%nX
zF_nZF)luV71$4n4yI^zgRcg%c=B$)zi%rvy+jeJotbFrzE5=3Hti>9H`<$K~=JEIm
zhH7^G!ZjbB7`u6Q0u*|9=3s*Lz}2y-=bHXklOwJAtBFoCoX*#0hMF!FsUH1|xZa+p
zaW{PBCP8%EOy$wH`77o*LtX8%Z&cE5PM344FZ^+yV*f{z`2TQP4VgtGX~u@B1y9$^
z8J1Cwc{8K(%`670x^n`b?;C_a9();>JLiDPFm>!In{^eS?rkLpSH}tpSm^My-6WXz
zT^hcilyP?Nww)n*=rs#VIfsF)Yv~hg
zaC5fx>JQqI;}W~&)?*6dJxiL1o{oyG?&GQ#Ea1Y0uBf+!wH=72`DprLJdPwCPU5!;?RF%AvoQb
zTc~k8Kyy0b^$egZkmS}o-LM~jV1`wAc3$t+b6Zw`L)|$)ReYkMqVhKtf(!Nx>OGc^
zpaLmkzw>Z3W3PVZbSk&UE>A>lmPOhouSv6Me|dU?0KW+_z2IPZpT#F?M+{NRulm#+
zX`aTi2L>gN$kC(Uq71J_4K03DX?p->%?(s%UWm@H`jxVhbz7D|X<-mO+ZLbWlttx&
zx==g!X1?U}@nk~_1?kBu8Ls%j;4kzT8V6CqGW@8xizhNEse+n!R0uW8kq
zFuC0}B~rLF>a1_SFydvbR>=Vu2(4Rb+L>cC)`U0d6>Q?{v
zA$`c9tFKlR*~dOf`$KevFXnw_H(iRKYW@1e-rgZ7YSEoJ$YH1sjY(ab=FJc!26ylJ
zD-XV;uf7i{;uD!m2CwmAkyzXV0z3!3pJFn%tu!(L7HhIWsmx;E*(Dv|?tcv4D$t3sPXnc+u9!x#m
z=Y01yXYe0L)CnfPf63+^$0~&U_f`}Cj^>(e=cu^?9hVQjML#T=XbODX?F35aa=R`a
zi;(`W7kiy%V!v)^)vpITb7m&~XfH*V<(U0Up2~F--e6Erm#I#KD(|VY%)5#@Yr{{}
zmMmS16jX`j=ex7y=>xOr9GAgVbi0FIw^q*qw3O60e28Kd$ztNqxAHfvs8xf|{({SX
zxz?g}1o4~G9^?|$4s(++IgSNLpYG?;ZRF;SA8RI=s9hAcwD?GkD-Fe}c5~fEmp^~EUNS@hjj3lj>bC=OpbL@(eI
z$3IgTL6h{C#^<*`PWEUh19$#5I`(}W6Q$C~YAt&@MWdEKAJQ68puiZMjFIa7bJAl9
zA#6LjPo{bSBKf{r;TU`UZ3Fh5EGJ>s()y>+Y{>H_EW`M=-e}V=Id~zlC~tSTPyswj
zSWb3aI;#TVHa+y03lM;1|3|n@S^WoGd4caeh5hXV%Ak9on_esfFhQ+q*`RZf#E5Or
z;yVg~&h?#^n{x@*jZl@b)42*+#wJA16zfR!>r`i&@isp*&#BYzzt*U!zHYd9COX%I
zCU4t?Um#!dwK&xIkk@k?jYidXcMhDm_$=;4!!O$1t?~h_7QxCIE13qjVz)yns5{2y
z@iD5&n3TV_LvSO6WemVvz8sjY+VaZzA7is23fec_#n-eucK`+$+n+!8n)s;)SvMYW
z`~#?#)>?knqDs_nIq*IzxS@Y{ih_w`_tak^>Ce1MWq9@Ijk6M+t9&XRdX{U)wBb0d
zZqtLj{7%mUj(&hsr2-`oj$R+QvG|4-YhX+>*WO%*ifJh@j{pFWB$cl>pk2Lx4vwIa
zvEdS_&?z7dd5HzY`hbS|w23hBPd3w!Eh
zH-Ed$t9{J?Jl+V
zFBQfqUa>IU?p+Mx1~Oc>XNa%@|FPlzx-XSa{MfD~m|Fo;K%&!pzXcaAv+B2xueQ^f
zjx6=d#AQ>KY~B9HtF^)%a&?RU(yRTqd9uMz_3=l&t7BJ`>YuLteD{IITz3YE*Vx@3
zkv;Gxs=!z&z9K<4CR=-I8$v=Y#U_+w0oVOC2VJ^;-aMhPrO4B@(K!Xhw*6L4Mw&K_
zn$6*0u{;Bx$Abys?XzB6kzx4BfQRa(e<1ePImi%n!V}
z8v5K^I+Gm1j@lTOvBoy)_~@uf$`X>%n!nM;4I6DDvHJp=-|o87MN463L!17*bO#Zp
zW13}+Um?1N@p(Y%R5?HgLWCAaz%glJTEQqm?X4=yS4j8WPW`}h>h+%VKcHMx4<|$}
zfExKpGk){JTPPT*u$|}_Qjs;FBOoU^Nf%!nRSmUwvgCbJ9!i;Aky|^uD|P*f)|N_i
zpXYua!#3`cyT|KY`^gW73_2tp>G99T#D|YQ_Dy%lV{!gH@nD1lpaftip%{*XnOL#!
z1IQtRF9|r9O`HgNKnh$bKA<4}b)~Xdk8P^;(7Ih)KKr0gG{Hprr#hU2#p^TYfxidh
z^&Bo!EAn4CJwK+BIG(QgeL<930}xO?KKS*m?|S*Jw-d1w9FTW3
z1Li0eLq@#(kTas@^L;EfCx8z`;P>$wwBW!JhHV9=>4#wTSAbh0g(qMrzg)TK4Zyit
z&|&@y&UKmh#c&2AWW^;XNpM`w|0S#NVz=BEyX6!)t0hMiD)ripv$JE#IgPNhOZTKz
znVZ^}9q|nF_l3zSdYQA`D?L8ID#{zacl>k9VCzBeU;*t
zTK5H{k49ZL?-h|w6KFns7o}!(7b0fTn(R4YPgLxqI};C1S!&~&>n0pQO-f8?Tz)FzZQfUh_31nsJ~
zSI}J`!btI&HShE-)Rb?>mmmljYX6ed;0GOhAVc-1Wu6SNZmj8$!VlC5;JsRbKF_Ot@%W#ZvN1P_N*n<8eD^Nsb6o^MRJ>B`wqrn*4Dw4UF`d
zcL3S-Vz=;1;C}$ppi%gP0&>=Be+L)$EaB?1AT>7sum0q=jZOUL9;@iuPr?}$kQREn{Ue?q}X$^!A|5E#gC!PE~>F=b;Ig8soDqIf3x`g5M;
z%ZoT**!0bXk1lEQ48(hwm-U93_QLc(-JVpE)f00;$CCB-@k#x~Ie0{_dNXAgSw-MYEbGbdSTaARAC$f9mF`b
zD2P1$^#o+2(Z!pTz&Hz|ypXCr8?=EnjmY~jd1T$}l=TJ6lel1}2@$_(tq&O}M1!Zd
zTe(Y*OT?IP7weWJjlAitzY@P@
znwrc8CS`u$T9Jb&VnAoL+9UA<5<6o3sfNc
z`N<^`C<%YZ$EF{KYyQLN*X7zgu}B404r~d0osD2|AQE~>=snvZhtfoKg5yUZ@Th|0
z&CUR>61>W@3y7{nJSYdG?E2{Lz5bj0Fju*mU?0tz!jD=w6Ti%*85OJl1gnE8ioM9@
z&EIp#%prQuKhwOrBH2dNyjLw#4x;itIfI==qi?ZH+qo{aMeMzjOF9!Irc3ZT;3O0=
z;y1hg456(*A8`do$(NB7alBzBujusx=E3djCr)w#fXzGrnpU6_!iC=B+A-}Xa?)IC
z`OgUis!sHltKYp^?gHLsT8R`WD6oJsP&8Q{#Sf_7DNkJ#LP^hsYcu5#z2$^{k*VaL
z3Hix~U9nxrWr{Y1|6HM>WAD!Jddxh?@lrB@p#eHDG_#|!Q^*H+WBsw?fayEo_A~9Q
z6{lZ~2w`}puB^h4tP>%0G{p{2{~UMcb1tH~Kj&GY!^?@i%-|8+9uj(n
zZ4a&G{i87YKQZDxh2X}H)qf+r59zi&$LVE*lHlF=qf`HJfC{km8gJZHJU)_)QV^l$
zvZXW|(7+f<20z3CnH-?8zq?7A;<1*Hwb*51&txuq+bku(4I*ZN>OeR`b$%^m*~4Y-
zF1VrLLYw@5ej=6rS2P=TkVLr^A6$k}$`-_5Iu&zXV@2v~UsaIn-s=yU1>sv!iIkZ?K`+kRUW5!BK
z+jFi<5nMbmyGZkWP38ZnxR|i$0uekB3`HdZ$Ztp`T+9br3Fai=4uRt!4D_!=Hmujw
zk~}@oH)__YMgz2Tq=9#oWs}Dm7x%yJiYi4R6cg-cRwlrmF@)Gd_y#_N*S`$S?2Tzw
z-DJv1IR^%J8HkRF3^4b5s9V
zWdLa0yu5Z2D3@_aoqKHZ14hO`#E^o4Fr1tKX$T10{+sGYfkEn_Lgq2|Rob88HL_aQujoSo
zAt3TALXlBu#&3XmbTCE}vL^ivu}`a->cQZTHG#rT-wX&`C(c((=z!-P{-`cwWRXe%
zyAOzg-y-kNThJk@BpE;n_W93Y^A)`=1<{kehOlqZ!vnEt{u2>J#SWpn8h`xV9mwCE
zH7rpW%U*visYOg70j3^&jtSk!4fDItO!t_q;4X1SY9_NTuA?m7NH&c@MT3
z%_@?N{a$kV^T2!ypxdjWfl_XIIVUMd_a*55oVNKJxNjl+g&CH8Vf!Qd(x_|Gd33Z`
zl~y;7Ss6fGh5UPD=dq$vBc|oM2|>XFSY8
zFXf4F%z4jLF7}KY0t~`z>u0
zBo~0Kmz$B#2-)NPTPFt_xCd7h)(cjDyw1ur0*e9N@mRJt;X-=1*Uno6{Hh58sT7vX
zwfQ(D9T`XyPhx~1y^phWBP;i=gV;iOaw}cw;aySi;5dO)21}-0Ry#!`ChIw!_aJl{
zJ0%%8grcihZlMOj-9i3oa7Oh`L85dGTyeux!JvXwyFEeh>Po_JkOC{^$3P|nRv%=sDxImNNwuLQo=ms!)zIB?Su3l5qt4#w&3^Pe1T2bVQQE_z789V75+q_1CEbMhxXc~Zav
zQhky0zSJksx~y(P3VxvJE4&mAE4-943$uGNfOFi-udnT%g22Wd1-#|Lc3;ONNuJtSGak^c0s%cJV+WU@g6)_
zI`3w?Q+=lY@<-Td)8S4eB@em+|AhqUfRj0#k`3b83xj(>bqn^+_a5{9Bv^6Es|H?q
zdo{&@a&k?I+#HlsiNDPRKyD5cWi8Ba!uNk%cpy=d|G?3WtO`EJ=2ZVa$WMA5t%l@$
zbK0W5oz?E{Ninn7iys~eICCcW$oX>-1KkZdqlRKfwzR$pR3<&&B*?h8DZJ9xr5yA|
z*LKWIXNwrLD9_^jBva1Ow`SAO^`Cd^bC8J|&O#@Q>%IDIM;buk#kwKoVBNbjvFbjR
z2EOfKYH*Wak^$NkerNDeK#kN=V?xFTA_BNW(|QKnRn?s(o)1ssfo`G^s&#sXei8b&Q{Gn+qlepavE3+!nSx=C@wS%u*{
zt9eFpuEH?!UERp4R@2DT2~iqfQD$~ge45lsAJJVZysA?vA8t(%CI?9{K4CdNZPkm_
zeu`*Wd~jU{Fc5(ke@(hjC}Dh8q~;OfzYK(}(jP4dstHnQ*goj7??u+T5&(Aj1kO)V
ztWE!1KVJZ$5YV~23ETA9bZQE~6B%g(hy}+VhQX(eoAWUG_FwuSkzRbRf$_14QJRb|
zE5vnA8iO4(#aja!iJaMuuBNB-KcQSvvw8W_P5u%}T3F1w=HPexP7tn?6v261wS~aP
zg^xLWyPv|Bcm-jgb
zS=){}30qu8%i%k1ZDvE2JGzW&Yx{V|R~|7vzy5~{FvqS2BnE-6
zIf2s*tw=KMlQ<@@7|VWuMi2>8cGL1#mKR3VI==*_8<43ih$W?Vg!jdz3&az?rge^E^~YUhSYh-x_xOZA{I;jdC>h5l{i=L3b5M?praf{nqPG
z&3V5&NhP=BrgC4Y<=i*xZ^N4W-kTbFZ$E1P=C|1e6y5`qT+`5;MVV9haZ+`D@%K*Q
zBPRoYSE`J)38rRO3LtOIN>MxX?dYsjUdYv1xt6oP7TzZMbp`d(b8Z
zP165luW&TFwzDhKlY$0-bIi0R|J-bDxDS{lLL1#=#f5W>g=0&mXx_R
zMfLml$fU!TEQXW8-UtpykvAacE>^u*IbK7*=zKV$TP51CK^}OXBtU|M_n0H3m7pdFj#rJQU-sTl?qX-OQysPAtrHa%B
z>vir*gJhkw@?_9Su#RgEucKA4>-hSAWgR)uy0sKB0Vr*`o^o-soB$WeuQ3uT0!d^I
zs>9ADgsODF;tgRUI$)`8?A|*1TJ%Xr7*z+(t8&!j~1FtQ)elS8JVJ
zVOM*griW^@RY+Y~-uAki?pXVD_#@qLNoVMiw&|Q9iSa}Ty&a)Kv`aXr*kGz~gvILn
zm~C%PPCyV|{Pd&}y+j)fS2Io*u)$zukw@(hrXdFfgJfJSdKOR#EKmRA
z4#GgcVSL#qMb<*H=Pb@4p_{7+mB4$#?O$c*PpFiV;_R^zP-AVM4@#k|-&CF9#s?`0
zA4o|m?oUbv_q0=F^yAkMcjY3di`Fd#s!K0zr5-J7)Y7fd>@8%Zr?gY*SDKJ*RqC9g
z@khzCi~17;G6@9?OBO7OoHd{6%(BqCb+|jw>{0Z5g}d?L3o1-2Ve4%Hhlj-m)oijU
zvvU{ocD4FO9`mjDmG=h=UX?-0bekKOwR=uY&7|8HdB$81BMvQGu^t(C+XL8H)|FQu
zI)uNwPlq;}z6L_j+T!Pde-8JT8_0H#%y-$z9vAOSY9)?)%=t5rfYa-bxng@>YCdo*
z6H5C#IKvpxiQ|&#zA>kuA1*NF7pgBr&%0F?K^d?2N@%IwEm(bULKd5u))#gt!g{Q1N&zi(B`mQf2U4Vg*
zc0}p)Tt~XqmJIOCQLG2fCTH&E61uO^#l^T;a$I@Xqc1tv4A%>}+8XqaZGDy_$DE-o
z862+>$)s~n?LnGlgwav*%`wx@8Pd|R{y3!NszxQyLU(^7`L6E6>Zg;`I!?C~bi1|Z
zK9AA_M6C|g+|uvq`o(sbm_)uLvY~WmVO3osw6+qz=}`dFSxk~TTReA?F3%JmwcXDY
zPmZqa5(77fjDJu?JwJD
zX7jFdu01%Rd)fUIa`_1W2urJ9(VZmTVkQ9V>WYwUBrtX2rRI=paoW;y5JQ!Fht^@_
zz2BV$^9Y=Q@hj)3a~12*nc%ABK?mZ7iswRMaFIaDhs~}j9Aqww!^sZd>f8ruTnvHA
zw6r-TX40A`81Vv2U*eJO~1Zuy`@yly?`u@5${D@h&y38KR`-t<6s#CuKVPXJG+wBDi
z1*oj_edsARqtOgo7M@TuzfL2`UWG>^FqColnP#P2pjuKFVe@IZ0yQZ&)@65#@7w`h
z|Dz8+`?S{#1$uQ=M58zP1XLe8+w7tXJ|N|$_)s}Ihq$7jXXyp}mJ%UC&hIi+l%o-!Q;>C_?6wU6yqXz4q
zL5j}V{7B^+>gKHijeG(EKf&dje21YSY;sTDRh0=X{D$rBX7mA&1j+_{1b~
z93vu7Oo`_cfm1x){wI}Z)7G#BiV7LlgQ+0MSO27VuW;>M^8-nZ^gaOyeVP9s59YP9
ze9)Mlux#bJKwRxN6iWHN$Wf|_@t0h}lYZu;5Rw58RNr2ORHHt9Fm+talM!AdQ)SJa
z=kMSuOQ#&FS^NgL^;Hcj3(?TLxVseX_a{d#uJzjDZG-WP#ue)|VafKGwWkhtv^?V<
z+zzjN@L1)&Si(P}Q)gmQNt!kEIN#&kyZjXmbWFE`LuG^T+Ggc46VrNYbIo&$a4PX^
zu2yhK@!|e)7-`(iUR?g@DNaml4|oH;In*FGc(&#-e7x%S1Rj6AtL`m*#zefauU
zAjv15){+0=U=}3T+<4-4!BJs3Hq)`r_G|0c*H<=kqDYe)btcSjPolC-&^Oj4hhr;GGI1$U`%4p_*p7zvDtE
zh6A>2M3$o@nIVh-g6-%a0OTqDqeAZ|mEjeM;A!jt4%3KJqxF)@frj?*cxuHbQQ!2)
zn6Lgyi1MA>rJ~{eF&F=3P(o(b?H9KCtNXK1-517cby{;`h`nAEZ(y%mVl<$cDs$Y0itJUbDaj8XG@Rn%D^*BqkF31a_ABpze
zZ+qY!o`(8Z)*Hs9lfO@hHc8D03$Xzi1DI8OW{pEDI4^=W7Fnyat|>$jneuJs)1ofHU6e#S-eDm__Bv@QD?b@2FlmdtaowS^R>@K6b!RvuUuJvEcelkFa25j&xdNDPEgOpLMGCaet!51i1URB!0rG-h-e(goM!3(E&_SWnSk2?wL}(UBP%kE
zxO1qP&SiA?TJ^xf!I*O9GE-ymtyANgA2e6RyyY5pMDI`Yp?FdGF@4NRW!DS+mV`@`
z?S!mGJ~uwlTdWJ#0||g_Ubp$KUAaN1FzNXwe$-(yGk#Zk~nl~pb
z&XrK#8@a7T92$00+kj`uXzVjc$k*#sdmf#4x$x-{$i|@J46>E+-qSoW{Z6y?!q@-H
z<*(!w6!!0Tg1puPeXw;2JL(`=P@3Ml$SY#n?;Agp($x9;a+K!-R*Im-v9VvGB5pRB
z0i+akZF+221TMpg`pqc_B%C~tb#SXLry&FshN?DU2PgXb6$Q8(b%gkD4GZM9g^lkF
z6|#3bTDf;HWiEcHzif5@CB{IqbSC{tV-Q`Pi-yW@Cyi*@V|>bKQ4xXak5g?=P99v0
z_yA)T)4N7fZ!ZcgRK
zWz9)~A%v=oi
zLibGqqw2EG_E~n$IZ|sl4vx`u&OJHEfKp;LEZ7tDgP5444|By>vTkZ50vG&$*(N{Kr8ofhus?3SUrGD$PZ*Pvo_NE-?|iVk+{5e9Ak+)3Syp=nT$ODQUN|Xg%pJ=+bx+eS_TmLhZ8ckFqq5;$
z@TK;Up3GXCXo@;Dw9Mrjct58)CWFWM4KXD{@~
zGZP@TH`K9s%@qJs_-s@JJnnohm`ZHw^m&;)k)@U!*!7DFn}>-uuR5QP{dlGD5>zbc
z^}P!QA#GHw&6r>vCXgqk0UngeT|-J}#lOGDf^R`)$%!|QLxr*=1ioEtT@?f@#}^3D
zmn?(HK^3oPFn!N@JsaHM4B0Q80(Toukv(IqX~y`?FCV#GnSVA?YljYs@AEmmuK7r;
zAjGHu9kUSmK}KWb^In56_+7#PQjZmu`7TKm=ku&@aWC}t2d(CN%JO-&Jz66%r76B4
zhQ<}v6{t^jhh<*n7?*J2TrhqEB~`_$4Kx<*4(QR9Q~t_J-u0oDkAhReL`^Bb4Y2I-
z^9v;8RqoS5p#$A$TeA|{`z68gTvhq>>0~`)ZV|hl)4bFEkx=&5A2%*K5XsdWe87d|
z-nZ{}%F+~<$;kYgauBsgJwPgrodDtGLGAD|GUez4
zJGg~o5F%hOV@R>iOzPnIqn%T@dXNs=5y%2@2ip+8QlvPikLi5BKZ3Vd;$hkRgqZRC^GEG0(ovn{mRn}u~x8&_w&`I3R}FLL)x=sw(4
zFZL)?Ps3vWm)_D)FXweghi-OFF6!ukqOkM#t%K`#l)R@kAq*sHOtbTMf-Z$eTqN_|
zy3wOrrv2(`Zb#B|mq8b!?IbvYpeLQ=@$e1kJD$M(cI@@XcTX1n@%ld@uP?i9d@TWB
z7n9b2BX?Z4fObcY30Un{e5hB*FuKkdiJpVY!ZnNA#)uzqFC!OpUa-9kGT_TwCAmvy
z5>sZLpk;DKUAjFHS-(6i%NIuh^loP8wh|XIvgU*bc+G{V_tW+ZvGH|ZJ#0S91I%b5
z--J3{CR8XawYPPmcKP
zE9>Xiav;NzFn(CB{wi*}Wq?Zy0`s5!+qK`eu3T^m^&T0ZI=sF}3#}@+e&Crs)l-;o
z2X0A$TX4D(_Li=MGIFskr5am
zo^KKGnK|nDplMxYgR6Yp@L@VH>aoQW3%wUdqw6c7ISu+IV0o__5;Xfx3*!x9b@8UnUi0p+BN09q9KSqfm_gg<-5O3P~d?%H-(qawq
zi9mF`BSIfDELrxu)yHp5`5g})od2y@4=cLHYi_*l-ZJiiOg@JQtQAlVofN|lI0vD^
z1%$JXVt{)&NawSS#fce74WTB_cgo^RDYZ^42pWL2{~b{_6Af~aWXxV+UUXXox}QnQ
zB!fFfjRF;Sp`={hfe&OgfPX89hzszcynR1)K4Uqyu|u)|vPJw$qlUP)6}IgFV507r
z+Z18B(I?pdpv^1oxd*#rGJdpD7
z%i^<=s6%IsL%DKc+710Vww8?T5Q>nlYOxiyAz>HqcJ9TBZWT|jLjoR4clqT2T
z>Msjm2!Bd!>~IpsN5iRE+;MMAWRFPw53ThAv}~+i5sk7?kpp7=aghULri$XEs~Ac9
zoX8x-93^aURJQyV_d#qLy$pBs%0d4T+tE|{E*Y$Y-=yVoE@?tEMy=RAUBv&j+^mGz
z2a77%>J{t85_bEw;OhX^qA49^*C_~YZV^6O8v+%ku~>!kF8Qw0p2k%aLu5{pT4qmp
zO>a?ybybBO>dL%!@0bHU%~A9Gj1bj2n4346s+tRH(GY4d*{CU<(?y$9Hc0Zki4q@n
zYV>Hgi$V6@o)Hje0c|38dFk1YCTm)F*Z$hD>n{qS(I{biElbVgcR8r!I{hU&
    2P4p`haT8c8*LmLnu
zI(un;FRS>G)HO9B0ND{Y2bJ_pYlt#5OH^FvME*WgaH~RL0S9;@O>Mk(16u_T
zJ^iC$0Im3{*(i2aFvi6G3!4xNgs!xYUM=M=mUCdPhJXZZuV7mMyWVIdg(clgEa}$Y
zpG&&LF9XS)^P}6uLzVm8R!_gtRrH^q^EBxs*x@d@__oTvA3DBSGNuz<1u7oQ{raVA
zha)$<3TKjD+#mZ^o9DYbB^u9wqwllEZJN5E;0sf4jFcb(dwB*9m&Z5$I4EH_7PhFr
zmPr_nUO6CLQ-BX+k@<-)-?5hnP=qK0Gr=}ous{eU)GVUuTm&S4%maM_8EjyY#=Yl5
zHO&vc_2a7;ve1%B|jK^14^*)dcmO
z+LjyYdrS6ka=#M#QEa7OCSCYkso11)yxGc*H(-^ZL)E+Ab=8lBI;pAd)?WP3A5ah3
zQud#=6gK_sg%Zx1lUY-)%cup%Q#s61(I$qYpPl%VjnckKSUPZW-G6klHZ6nn{
zl1WGd%WMG4G_uI@M3#RXvK
z1V5*Cq{Vi$t)(t2omtqUF9*K@3qHJ!%f4`1iuafM&5M$Qh%G0qj)<5BBD^o`CZ)s@
zAbllB_JMIs`8;-&Z<|PV`^lo7PKGg$PCgs@=~_uiNHpbfx8|}w8&&Rbw>Ps3#bYjWI}2OzCycH(f-ws;lFdt;U%yjR$H
zt7_YIpnq4E&Kd8?RIL?%UN{RNro_*XHqqKoDR*XQS?ba=$=aP(zPpE-QJWWP>INM((LKNRZY&;N<*@TT
zhK3UnquaTBWD=UINuRmDZmcPvX-iPs?Gb;OK<9-v_Nb5pGF{O$
zo2`<<|82=5$VHR$fmX#dO||>EXQTAhOF7cLf7rf|xYuxUs)(vmHrsuQt9r0GR}{&-
zav~mAN`Oo3x=hHFp*e@8Tw}m1c_#dkHhZx!Nl4_5Ltpq+O
z(wDEHW8#1Y?(M2WlF>Pr&2;?mlFZq3trzdBbleGuMKGj8n%YEd#lexZndM62vcpEiA6
zV0e`8b^ZOt$beZf4%a@NeyZCm`WZXYy=nRCi+Mx1E2+L(Ngc)rAtaytx_AMI4qe<-
zg>;H-8h__y`4j-F_bs$X$$9sRw{l5bVuXx*?b0tj72uM?#&(&SBGG1+`tTv%9!Kn}
zt^05SLDb45A)^9?Sh>V&+`s1l-
z8^eOK=dbHS&`si)&slH=fg`7yOhr?KL7y*xhlA
z@`7>+9fb%&=-=HSJi4Q>>cfVcz=I$0JVg?`ZG6XOh={Li?;_YMA03==0{*yDsLxgs
zM6&4U4zo;8H^k$aP;ejvjKtKP18Yzg%!3Xv`dzHig}kWhGsnKdwz?Z|N%)M{3ly6A
zCf7bYzeIKO)iRopKEKNDtNVx7T@7nl_zpo@sDyZ1%-F09r^>k1(;YIaO$nb12{WzG
zfoUoC5Vx6i)Ryrd&1jocm||;kd>9-Cug-M>$(4mZNhZkBM8ysvE|-&gO9gqX4hh9i
z*2x@XXItcIQ14UMMJCZ(kxBHvD_D%tA19@dP?U68hvdaO%NFiPMN|UOxin#Yg`h`F
zF`Xf-o(^($mJvY;t9c}09K+kqQ1$!X+Vh&TNimj`zT9vmZOIz;u0k{
zYN=WnYiB|@N)_T!K=^U_JeVGDj4jbxq4FSn^aqYO6bmSx38EC+XzP5~(rGBv5w-b)
z%{E2&Bdj?OnO63!ie!C9rk5ViDL7h^lw_N;<2~@Y7isVqXPM>L8kgb1bFN*Y^&ar5
zYlHR~CLi11S{tbt4+QeJ5jnl?hXJs6>)E4x`RF{ynb-R<|3mMjFA7Q}
zsk{18^!eu!d$p3S6_GE;%r0h8vk1QWkPPO*sqvrTqXHH&>BN&GX+9H|38U(K=%F9M
z>YqR(**6BHx}sx2`o`p
zOregd9mWy5y5nX;Z)~MSxGfx$=VgLj(mRC1_oZW@*J$B^6p+QIv11k0brgdWU}6hT
zFZ!;JIKyiI6y%h9H@W8+63KyZNJQvQBNr(OPlmc;I?fUlFF876;$hKVtv_kpLMWGS
zzQJgW3F%}>7Rs;dNxVOgQByd#e$sbkw$EhdsYahE|L8pv?N_7s^le*jUl+Aq3
z9ISR<)Wc_@{X{18@a*A%cZLQi2o@-E(Y*B$2>K6ln-E1N?U~(m#6=oJ4$VfHZpbaP
z`azsG;Q*=_HUJyO-E*r(dROuvj4Ll
z5&_Axh@15>1@UN3?sWo4?BQPQ)maVAvmc;Jqs_BF3SX&=T>KnW$?fwdfD0FJ(t465
z4TK&Dk8FMD*_SyRRaRcM0|9;#Z9&c%H+!(qdZd+muh^c0Xu5vfObWGgJ1aH42*)J&
zR-f;a89-CojvLn0{
zJ*ef=ieF@}M?PezAvKFt+Im|t5%FY*Rg@sto+J8S5<@c+P|lPd5&R<95a;Q##i;9z
zAeyPf-dE;!^Y(RhN7-#s(L&5?P@();!;^!hQjoVioM${M2)0`d7jleL$u|)M02LcW
z0R2a<*zEyMgghL^EB0+d(gCz?FhO64@;>!^DvGR-6=%anOe~Ui6eO*%jy7+4mkkh4
zi4D5^L{Ti*nHL@GSOPV%Q;tjYYfZ7!a^^7*Y?6%$O1@f%1v*?Oq|*%`s%f7W5zk@i
zssaZ0nbP$$JOXdcFhb{)erNzAbh0QwT`o;s%*jbxA(fSTU?h#poRq^0*K0%!8?v!D
z;rPh%m@C)@AJRY!P72o2b@v+=25=^ele<}*EM$0Oe8PP8Rp$_-HITZPj<*J`(uUGO%
z($s`VF}jgpTor{8Ca0qnM^E$g)jmIaTU-k>^%kYBP??yZFt6;m!7XU0Z0bEL`HgFw
zHr{jo;)YrBq+UWkPaj116f_u7^3F82ZukwAq6N93K-s|YIVJ(o6sG5r5q6OUO_+wJ
zbAe)J>Qblz+07hAbnAnXd5-Y9KRBW~ZiFTFfy{y|`hI-SaJjPKZ_e0hRjxNZY?67Yf__8qqQwMP~Hm`_H;4
zcQ&Xx{Q2+}4%OYEDUN3_{v6nSzSW6qeAKBr`+j5`!l-4sBL>$N_{~I8TW?j)C39)_
z#*`>CI4@|oV)nI=smw<#=ZD83P`yq{d|VigF395lkjpr5fqZj;5*S!TNa!$Y(;Bfa
zWJ2@yFb98dVt^XoQxvI***V!h+;h%$`=>RT-W^(VB9-eeGOm5~R1aGnq0m
zz6{32YlSb+@bEmxOI^$gwqZjdAcxWXJcFlnAz0l!k|=^S)_Xe}7T_)u7T+~CUE|(}
zOD{IzqH)rAx7eB>#Q@2b^Tce$b&?CY`wGRWo=4B7;}9L|qimGDG*Mnrp}a}u#`+kI
z@_P#QTQKuJZ@p!b=BH*p{5kYRCd%96B7%ue#cGP`!o#0G#A9XDG*sdBnaOZoH3qXP
zwr!jd!k&QmNg+_pdG<=wK2Who=TvO6CtRc&%S8Q8s^*|2{kem!^He_L4e#66E0lOk
z@9zaS+<2LgxVd`Idho)OSGuxRIv8hHmOdx_oO8U-@Wq?k>oHmx8t(XBq^A76YRd_?
zP^45!zwi7aXdE(%r}GZQ|BG_zkEQ|l79G$rJVLb(3R03~0I(8QGzvVB>o~PNq}Gar
z<&x@&h6a8P)qKfbOOZ;Ftm|MRE6WA-h;14Fd@?-}9>(Ek-2ITf3pN9Z
zAxU^|XgP{eTI6ew0kjCTf8xVJ*M$4E_7#YXg~Vy2i)twJi3V8yULXu?&|{0}diOBx
z#oW|$&L>4R!NU`=w2@GQ@~eM7Hb(3#A)oBcn%FyKxHsVB)HYB+;q00YNj0<=K+9^k
z_CH4!XUPBm6|xkJruqCAB+KlP(0kR5cMrbXf>`$0O?*m`7c@e<<^FdB?oR1YuVYfT
z?-?QR7$ZzF^kqQ*vcl|t2P?2Xx`h#gU$H(D#qLeG=QG2ACn4lN6($fCA?``tgPYK5
zRi^jp;JQ#hiB(z+xzDjW~g5cv0EY^BMgW4l6GpIUyY#h3C`23
zwm=2@&4hpMc%Z=lcaDeDr~gGti&K8ydIe{F|HD3nIK*+kVv@yRaCZ2A2Y1$TKApFa
z>*L0U{s?c-TzZ{OkcSbUR@f}`&HrELiLjnmPW-%2k8kHgK;`}#fdQI14B_O#u)wTB
z^8b?T$yMIymJmJ^dsY-dyp6xjBA$~NRF%DgKn5gxR&Na|o|5{Ao)pWmKRt>8X=k>@Pai6FIabBe$)Zr1)V>6!QY1$hVT?=
zdkNXeR=P0~TBcDEu0dSmc3E3pIEdbTB6r(o=tI+0A8vMW_q98k-p=lNc=sN#Pc-!Nkfm!>&G04|95u%t6I>i`r0Q3I%BQP?_8}sUDBkj+Rktg
zO=K6D;Uy(KxxVK35><+JWPaHNlxd|wtel0ETf2S!X#ghZXxCLZu7bm1LOUGMm{}=1
zk}-|>FsdYho8Ve^S)+!#{?k{CJ_Q3D_P41To)N
zin(V&KimDRqwOBwhf4^mz)e2HMQ`xEbwO5vkI_MV;#;uY4O=8~lKvDqJk&TxMHH^b
zkq#vz&%Mj6K#hyWob<>_+QSJ8C|5RG&w(owu89rs7dTCg20T!yz@X
z60#?emr5TSv(fY5c7L!X$oYjP_~51(4gRs#p-1ybP9QouRed}PCJFVMQB4`kZkd3x
z&vkzZc|zeFCmg)XY>b-e2j^i@V>x;N4>EszLuN~#&q+~`PA?OzBP!8c&k`1(v=HT>
z$-xw+4D|NaCw&(HX+om+YzS;s7at_ilh>laf$N5#?2d?w6p-4wg@ff~*lU+F1pTr8
zYnKT_wiVp4;l2|XbmgaPaK;MMJLnkDHmlJ(z<56^n;3f~v798V?uHkknshAGJx!T`BlkJ=bMI>W`abqY*jU+DcWw
zyE%pR$ZGL6q*nGY^p#}}j>yra-+Hg-fztD7-AgfG96~j*S}Mn!x)NjFJ}MLIXeV+x
zvB{vlq=V4%DOzeQWF3@h_m0+?198yC2gwZ`7=rQk$2@&q`&AsL)+1=iA_H0mokTnl
zCN)81l>p1Unr(~hay@+#Lj+VQdx5WZ{~bXDow-Qpb~{a{_ekL1nzW8%j45=s@E&-9{zvO(h}Aey$KoQNvA_X$LYluIZC`}2tr*`hr+X8>YeEq`E9WCU
zzFn9bh$S4o1tlUl!V6Su675?qe*)ngKW`GY9!)8vw$A7F7%)No<*Ddf77dzC1*bV_
zCwwn|Ft~^4oV&HXR^wxKLT-2|lqCsqxdC#+(5s;^6e?*5RGvUG)E^x&0Szn|zI?iI
z91BKNt$Bx2?fMNm%Bp+*5)9#9^Hw_$(Uv>u6E2V42DQ8abX59v?oMh7LO32yu$G72
zZivwcb9*we->ght@-4m=&0y%E7P1(qo{m^g*OM%YidOsSMxv
zEissiWsD%*b@X12y|liC!!XBH3OzLnG1dvbuQLSD*v+>AyI0@aBXby0t0LnIpeBCg
zCt=onqZxL%UkKV;ZeJ#}kd(Q8qMg=z5fsNdSrc@0ty^Adm=DJVkSd9N@~wSx9`8zYQgS0)24^Q}w)I
zTJr>H*1-VApYTA0@_uSM1qGyL*hkjb0MovNoTgJet;`1ut+y`I4X{rPsDb*aDVU~$
z;*&Bil&ubbDFT3~YBY5d1w@!j)w6zs4F0dc08vX<3FUqqxB{Szf!gxF^BF^_)mrv*
z&LJ=I0Xk3wV3^07viO6=)KfmEwu%!CvOtv&URiV!_CH_My=4O+8ZbS-)
zgkY10wKwh~+_Z2iDvyz^}Oe_cLF=pzIfV|Nk&y5$<Pd5Sm0V}Bt
zNVd@~d(*TmD)Ji_&_cIA+snP^KnE*Ky(qg5=_kzpoM(sJ#bJYl!LEMG3p_C6cbFOh
zaZ&1S0!(A{<{7#Aa?bo{Oruqw-feiJ!)#$+{5n(;#$F737~+HMk96;Yr;MM#OVYZZ8w$hv
z&jVZ8al;gExgVH@NumJqC4c`i8^~Bjv!?(xIMR~elywp2b3nI(0TX!Ch0QVOl~ed8
z=BAh+#+WJEX~+cm-)NtYj^!B+7!*>8UJ76PbymDoHi!jMi!ifLRM39cIk6n5x1$zx
zO0c_H|FBn3YcyqBU_gyOI8I4vEqv7@7+BPfGu41+_`oC^YmXSB*cT(B9dZ+($q>2q
zR%LY+vcB@{L{nA$g(1|KF(@B4cFHAQ1LoK&h}_DtbwSS_E#Q$43Ful5ytgnG_nGqb
z*8TA9mNrt^JHu>?vt{0W_)%53rrgVYV<-%gkrqH*K7U;pSUj!R&R*6%VGm;TOiZ8Z
zHoe_&6cuNT(aX>416r)Op_w4N6nI(TL2O(NT^918ZE&ulx{Xc9xk|@qw
zx4-Cn+hbG5KffyynL5$+2}Yi}pF`D{gg%RdxQ8~D2~yoPn9<>93=T!W%>zza=;I5E
z;DlVHF|DDk{@z1O@aaX>*Of0vxR4Hs<*!ZsveYm4`xR5afC6#=noWK|`ma3oEBXFn
z6iAo;7ft#_X#X`!zkKDWuSZiBD_-?XO4_xIE>{N|h4^rkW-sboWWXUEkBH_AzW=Z)iyeLt%d^arWX;+|^n-4_cxD5_=Y
z_H8TNC_Ntx>z1Pzje6knP;apla;eyJBYgG9u`M^?s~1B7y5Nu;Ofx{;tHK4Hp#S*_
z{{UZh;dETqHp*z4LL$dp2FKQdcY*5jH)%x<_#RQwnpm-TXQUAITn%6OW!aX?@RiOm
zzT_O7N18`cfc=U`qVQsugab%D`afQNB$kA>?^aGpNf}&aRp)bR>Y-JZMB_Yc
z^T=1B$ARBNl1(Vuo^r9h(CTu%7gc3&S7+YnhMh3S1lP9VSj2Vn?!u>GioG0}Vn?o=
zJj5{>N7jfzFJ(Vafg69TWMy8SZ|>vNBs{Tzre9{$G%DHN>S;QEmQwv(lOw01nX~q;
zj+FMa#pVS=X`Ta-Ft-5R4}F-lZ)hKcA1^?=C`ZHvxmK4CaY1%ef~$#8&}z<2x}a}k
zReH&r4LyF?;#OpiMLa;<0FP)93e3Gl?9hyX7c|;Kc#%wD-g5L5{x}YC5=(u=qSQys
z*Pg}>ZO4Mb?wSiKnQS?1$>#UnILNpW2xEU9E3U!+ON7fhpT1xtW#FORyH@xY*FczF
zGw1j_Hvz4A!Y)pxhFuFOkT2lttT71~O5bJ2vsN7OwV%fVKNw~ghbl!a*Z1+JN1bx`
zoGsfrJ&>@)Yeg3_NC@v{@Epfout&xhf?&i#V%4=cI9kgk;5AFOgb9L>0ZPvo_~idM
zX>?(e*hP=Ycc<1LSx3{~-!zl2Bzx}Sm4CF;d!eRjdiu3W-|tb)ek#6`%BOk@g3PGv
z4kA;QB!3vU@Na5Gq!cR5Ull0x^1;fAO~GwUP31P=E$2f3cmJJtaMYfV`w3$~S=L1v
ztPh!?+Y9&p)oAlb-P{+IS$e0=P2KidVA7VaFz8V}cSg6@}o8L!s*^uUluxc?|{xl?KOhyZ~Ki-}^Fc`*8{SRmG)8Z{-8A>7i-VitU&q`-)67W5EnoE9rc58XhF*>7La6JR$nH3
zTQcFNKG~J=ZNMT~J2!tt>7B-cO>JvFpm!kuilPv^e=xQS9(apYQCVbwj5=z$N&qLT2Zf_X5IS2&LnrTQD-%JcB=q;S4sMPL!?*GTOD}%pjFQ8vM$D#!0BL&^M^TF
zk-3=#6lDy&-!)624>z=lkarz~X@DKefTGIIpM6b>%8aNu?!JtW~;#z1Iuda_b`>PpY>43n-R?QKdB#^O(+eH+ROu>Gk`Ho{qL`{#`;R-QvV6M4zkODp_Ohn;Gb10`pU`CZ
zPRKtc)27opaPf#B?Z!1N-ezxq0C}B)9p|)4;q7}?NWet-`4+Xjzj2-WfbcPQv2Dpc
z2mFc;;cAByWGLd+^hYx2)_#bRE{KcsL>b^a+bD%P^qFbt!bkenbm|$0LLbI{-|Yp=
zLYh|+ADe($@uy&ow_HeyW&e=;*jY67@MS)km{1@7ge*DT^=){{*D`>*2b~bq<^7@L
zsi{pwfW(7?eMjEG{s0mnt2^3hn}q3m{fZTFytM4&-xtARnnkX=shf40k!`ilqBvpc
z25SrdT(UqC-)Y9;y2K1te{Z|T(^^fRj9hmnWHB}#?AFYz5#kGsMi4Ya1SXK1lo!ee7yrs^ei_}eG!
znvl<1?&V+~l)r9=^k+Uub+nHzKjsQ``WhP6
z&e=X=Y%+7d?<=uU(f9XAe_kv;MvpXY&|LRE%%N1}hegqS{b+C2XnG6L9>=hlKAq1R
z@Xf5>bOXr;r#Pp@S8WK{u+OhJ1~;=xa_49~%yu(yGpu1Xf0(9;@GB)>+N7`=$GJcV
zHF29IUy}=3XcYHyJ+ZrZmSE1|PZ(nk*G&aRPd|T|SL8cBK$sd&K;B3Bmqq>6)W|<|
z?8k+R$$w<=vlCqyk?mXv7tiF`ND05KUk?A*@PJe^%f+7%xJ4YJ2_!}28=Ftw5eg|K
zD>_epvGP^4o2e}F6l(1+lPtE~;eH(9EdOY`zy3`n-^Dw3KC&dIN?aH$^&0OK+&X-W
zFYJNUTeApKf(iDeXVR@Qk&X+g1o_SdI=zE+2JT8KjGehJ-9_(n2&GmX3Bt!mHfzzxi4&aAw|-oyfe`eC(mNmdAOpw;6pByq^6$v3plV
zE-&riyO$n+O(6f$gJSN$8g>sOQ?$h=XOGL~Tp#k5Ajqm4k>fhWo6l?1X=d7qge$fG8ahTeA1?4;BG}i6A6ig`oPidHM$1wIoz55z|H=pzimx{WuDS?
z_`=D}xfBD)yZ+WI(6h3XL=cI%=;dc~F#Uvn6qx>BKIqT?L}~+&8?ui_YH4FZ?5SGG
zTG$m5%Tvfx{U@)9AMI?rv`OjIzx2bue%F@GF(dZJ?XMtU??gh8Q@ikh3&t*(OTkFr
zGzzN^iBRbem5%>4x&F)3jS+*ODmbed0sD0~#iB}oH~R5qhEmOM&Uc)Y+wKbCy;U6P
zz5W{y`uR^~yc;Rviu4Uf{qO?VD!Ggv#AB}+oC*bs=-HOX#UmL_Q
z;QXp}AOQVX3~G9xuT!^9cA6o1>+fxz({m{}s8;+dB%hRXVGbJi%Rzwv{BqD=4*Cm0
z{{e&kDW?8H&|e7pE5$=$_$$Q&)%(Rke{s-X9Q4%RtQ39~NxzDue^tdp7wVt#t$)eM
zNV8>jNIH)yRGv`M*&`xDzV_APSvl!%dL^jEdH9!8-*{=eE$)_LL&EEEwTg=CMVtSo
zobt!gi)S!3rNvkF0;aa-iy_6o+Umbq`2KMSq@!4g+W2jsF!o~T2m6q!pI3;8?)PsN
zy{Lp(JUqKB;^GTfwO`5&YSh)kw{HBW&x+ZBmH4jN^ZYTvEM6FP`0(RFXg2(Z56dXP
z4nDiOKpHg~eitIYT;kcj`fnR=)^A+a2n_k;my(Z)Mr}nORlE|hq4TnD*ssI*PgWKs
ziU|&N$_!+ngIj=G1{5(N#p4_Pv0?Q;J(cuk)`mYW3rV1ldZ3Hg@Td&&Z;C*Fa;<<_
z$zpb%e^yhAOZ-ot6qAIVdXk*<6-@oeM~=!>;Wjor|M#s(a>SOTG;FN+|FQR#0a3P1
z+w8g^DJdZ$A>AmQi?pO5AdMm+qBKjxN(vGNDUC=vO+9
zzCZ8tM^{+Sm^o(Vm^sT9sj*PN1m7Sw?f*M0%%h)H@>>M2on9?qhJgB8jN9VBlkm?y
z8A4CtiBsgSh}^Y-UUm_H=X0%pIU;ntw5z9x5?uIvde%Py0|8EY71UD;`WIpXeHYUx
z4X-hB9IG0@^rTJxt)qe_1fQyIKI`iDn>gqpfW)Jr7f*ru+v5FSYC$791-WNn@2gXT
z(E+AM9(8)L!e8I}?KI(nr@kfgL-6;jF{1GRi9Z0VMt!3
zKK*|9e*yJ_X5s$@)Nf(=UqJl}>-)b4sNL=$MM80mpZ9Y8U}itZ7Gx|gA=zDzs2tq7
zOf+mCD_VUTK(7HSHOB7R|27hPR!soQ99f9dUO=2n&;eQ*daE@0FBFFyqM0_ZN1$U4
z|MKa8SGj<8x=-&W`2BqTGNdMIU=8bny4$}dy?=RU_)TzTv&n~a?vHo>b!_{K=|F7(
zsYJ^^_55pW|CR^y4|105|
zDFIez%Y=`7X6#>x;%~a^1U4sWs*NCyp#Ecy|JhI45c^A}RcWaHHNE|_!!#y9m~Sic
z{`Ur(AOPUJIYkoszn_PS8X!zXH=}>;LI1Rh|2mC7rT>4{Y5a-lKR*1|QT!1${_80I
z2%rCT6#sP;fB1(#gY5qYM`8F7-aP$5MlYe&SfojbtBO66pW+iEet+nT
z*y(z@(eKZZ{`J93e{_19S5MEN^6AFrBrOG{*h5hh(3$K#1dD&C@EU??B%BB0wa9%r
z#FZldkOc4(Lqqt$K&hRR{Z??*{vs;{1;vG|d-G>}IYAjTbL=*)*a`fe^cJua%wLPo
z^w*Q4YI|CBbRY#?U>X1K^gUftHY2gd9fQ0I6soPuTrR
zDs3@P_r17EyNHwZr3UxCK-gX*EC9$-63@y~sw~j~fs-NyE`t!biDJ_k1EDIa>e{j|
zDJ~u|4{vB_ZJ+z{MWfaEOm=%mzpD6TeOel^;}LEHG(F|({3|^oeLp=>QH6V${>b2B
z=fuXugq_X$y;R;SXXZuIBv#8}m3jDxVH;in+TJLdKfQJM7eeWkpz@evKj|89bKlE^
z?JdC0mNYp+(x1H;k?o2B?H&|AIK7S@@smHI)b|extX42sJUcr_ptD|lY)W!+ZIUSW
znM3}H8gmdqK997Mc?7Lru>9$UnZMenO0n}e2?bhiH&Lt4(@x13)k&?b^?Dzie`fRB
z#8%6de6hX=-L-`Rbldj@HPIOW=S+gKe{^wLyCxz`q%q`1xH4)O*3vV>L^8H_W*XUu
z$-iUJJz$1tpEDk4f6rnIc_`^uU=gMcHK>^37@Ded6`JDkSX*{2k9#kVIAidW2)_%p
z<>(#K(~!ynmI?jBtA577+oHcCFqSTBV#0K8bMk(8h5hMG$G=ce8OP}0NCu))xGZ3n
zG{zmezxWnn3{V+xPft$!!t2*sSKZeRot)Rk_p*#;5$6wnH9tJrFU6JW$jN*I@^7ie
zw7q}A1Njs91<1PQ^14B}-grc%b3~>43~(Og*Nrw^9Ub(zqKP~v1=RzvXMc3&Qim^L
zQKg402M0TG?9QHZ>;+t0Vu}vKUk(*85D4J=uHNsv4EQ8YM8$z?<8k*H-%SVr{D-@>
z({tX}#(oCxRJG%oDs(i4kZS*@^6>P4IIxNR8n}fNQctYn$mej}e7cSNpWaF-i3CDk3B-xnMj9OO;3ay?{?jf>&N52X6#
zU;+Gu0FJ|@DLNA2_6;=w<&_3Foh1ppUb)e(%$~Wfg&v`TsDm+**7#Hsj<{BlXTZya
zzcdjjKcK-eRr4DNVXcCI=EKh#HWlY<=xSoYz{8Mr`!
z)4&f7{OpkKDCu$ZgR(zEp~*c0^k1U`@6Jttcc`}4Blc>*yMc;cK^*nk8KDR6i+be>
z?;OC#IHh2g>9%V_d;7u^m$NQ)lbk*)@=tS&)Bu{#k8=i|00G`k1e)K975jEZK?rUR
z-ot|J$;0+IMx+m?IEJS(Q}ji|Sq)9D)-#$0fh{2bLFgD7ax)78
zs7Jo25SEC{OT?&tC6_-UEY)fSz}7R*vU0p$t?yoMc`&;H?K=QN5(7^cssHJnEa(7;
zu+ZZoxRH)&*`J?pbScVV$*bYJrG|>>&zA%D
zZf2_AT=-xwC@kDN{vhO^@<9Fs{tKuq28(>`A*PH9(
zQ@H>J310=S09{Ga6@_Vvq4d#
zK}n;>UN=Yi-nicJx}H1P%2`yznb2za!Oq7gb-?Z!_!K~L*i0YMo`Ku8l&ZQZ
zP(lg79eV8dA;9V2
zUx61geX*#dLWz&BaN|PzgO%^7O#Oy+hI5uC0>(dNvb*cal2d1zRPBC*_>g0q7E6)LRO0{GcpEYF)RGYk^R1Pt9yTW80
zcZ?IC4URX524}gjwg6AtlcQ^6gC1gh?*K95DFWdx|5-HfZpANsUauNerEBXcGA&{7
z!@APLDWN)8>A-_Cwy$&m^#P9S_R9v-sfYtrH0X?FfuT~izr-0ZJ3@3_eZ3T*PJrQj
zsW8yAHMqTRZmtN$oMfBk(Gbs%)u9kv{$e&|Hp9N~;uvC^skbk>PZdg9NhuszhW5j$}4
zDto2>&RO1F9uP{r{$TOR=38eV-G<^KV)f@2q-WWm0=TdcHn%=+{&u)wd3$k_Tg1hI
zn-@5s`m0HL0;k+sj%Fqtmc`=V#Q{yWWdJ#Z#`UYGk5&IO;BbEUtA5gS4yXSJ!RRay
zNebFU&PTt-!5vjVG-drjKU-ah6RO$(W9+%7@#(C^JQD=3i?2Am{cWhJ<^1_fy9!a|
zJ>#;o@`*@!pn2W*E87c$9%QD;2-RK$RQvA!4a7rBznZlnHT-a0#AA;AWJ;%U4|Sbn
zWj$hrXr$Tl5#r6MfBu(ej6kgF1ANZ+B>6Y%RZvuTV`iqEi;L#nNozl0QTK2qWuxW(-|}%9}ZskC}r+w{u%VLOQ;D
z2|U~L&|d@!Doisw0IPaA828P2K#mK*Vakozpl)%#x8TNzdf`~qaTE9(Z;TW29O
z1%Qw|YU6;=4hLW7IfQ}{F5kxC(^-2J=?RKTOO;6?A_%wMz9xWg^=SzT3bM}K(D|kG
z@kn)n%uWak@kcQp;>xSHh;+0d4S1j9SK9T28i99qaOcL?X&&(gq4JvX{~6awZ&&62
z+HMjgf##nMZ$64WP`1pF0S1y!11w60DflCf`Tq(OO;kY6tFK?CfQy^4Y%)4j+UVt_
zE@%*+dU$O(Kc63B=Hg8=vO3c7UU#^hU5sw%#Olr*#fd;wEPDem+o<3A@^r3T(>9o~P{uXgj%
z;mA}*kg)&Hyw&VR!gihR&V%SXXitFJC>O}|%TX3q@jQbj4ILD?oTC5WpdEv`jOAkI
zX~eZ-9Zkrcn8b+vvlHY8*W1Vnp_F>1J7h8|UiazGdqX4DPWzc|%C5k`>ouq%TR(Ov
z(Eh5$<}^V-)fSGJ(H!&b+CTOJr2hxAJC9oSc?o9yS+BB~P~ay;afU&c0*jYB6vE_=
zfTB&T=ZA?o)`(@f<#1v?V2AD!M(6NIL;&M&Cnn4sYA`DL=GD5#(LUj41190sf|_Fj
zhI>Du)$j(qpTu@3HHtH;ga&E|2C5B&R2qvl#U4_w5`ezv>=%+89)1<~e;sE%M1_#D7Plr6QBWoC|%ZVV~}F4);gA;4U@iCS>Fznub$
zpK+1j>Bi~5{xq-^j7gHh3Of>`yPu7p2}l223(pNXmEGb?o=3oSXEIeSZE_7~#YpqaScNKv`~3A=Mf62QaSVZD3^RjPa-Iq(h;FT{LhMgAhua2TffDW?@Awo>V}UibcqcMM}XDwD(5
zK0g{9j3)aD1FPbvO34Ef`ZV@Yiy~)C6D;Us)8sCIGB05w5b*W)2q?q9lk#d^N4v5J
zY;&2f(Bx--D@?C)a4-9SyPFOKwnvS@=-D#3XD6V4qmLADZ&y`ZJPeo-c0GC<;
zuBa`~)2k-x9|I2|S~8hyN_ya+aMc1G^Fe!b@WuWi6sPu90Zmsu}eKo(@Vv
z+6{15%Wvf5g*8d?9(Uw$d}x$NLP5-E)=qRs0?LWJP!x@R-ycrG)L{3cI(zGZVNtYMCbm_gehxk35FRrK!KWygXEM
zgC_fU`tAK|LV!2n-+08qATM(thn4NIFd(aoNLju6t*nGWD|!u$dwwPsYaU`y02jE6rjGx{@~!k4Qj
zObg#kxr)rvMIfExInOO*ebDdZuU)+k{966#W(gDDcC3T@lM@vYQaK
zd<}YN=g;mK5j+r~jQc&Fw>Z@s&kuv44`Pi~yT7t#2?BBpK+5gSZ{=2vTBd>%H%j)_
zw7JPML;j}#mhj}XliRW-)EL;*qu4NiZe>BahbF2@vB0}%ib$Io|EwNdW67peZgs>3Hu;
z{D~LwB*))rDyP@!WST;6>k*J@S4Azg&Bfne?{1@gCI~#o8I6?Kr{78pgt{Ks;rB}Q
zIENDMhPvKEfa}U*Z6Lbz_pX3M13rHW1T{AdbWbQ15c=USw5*w1j#$RYf%kVYkfAK~
zw{SzJADwc^_11_Bj#v#NG<5uNO3(!8X~4vMkqTVP8V?khECm#p0H%w)dMq~!IVX$X
z&It}FbH#ntg0f)11OB90<RhL})KNAAcnHCY9&3_x6(Wyc!Z6XYU_wgfjsaefw7RR;@Bc#1y3js(pNua`JJy81k#-wl&UxP0J0uV@Yz69
zcr~mprRX2&(IVTa@LW=rB*5o@3JE3u-o@1A!AhEnM1aGK13tmsPleJAI2rvv$LqFq
z&xn;+dUn}f{hIS~Zyx-}YvFCD(j#-CEefl
zk_hxdLC=5BxMX
zL$>Y~&$S=P2MQb)3mNBvy5u~ow!tmhg@x;rW#I1dvI=JgXo4idlhSSvYW#LOQJy4m
zv2+%mR`0%mURVVDh=m0CQ!LEc^r1&XQ9-6z7u#_5Zh;=Qg8>hxh@|Kje@D@QMSH}O
zlW{5~yc?3$d>S5q@NQHo$h0O8cznrGyQ*)#Lpm<)s^Oz<*8yiCg0-`!F?ey>AbJ76NQ-#gR~J0nhs&38NapBStwOy
zud6K6)X6@+nL3q6&{~xynT}#e(GSz{tn&PBsiR5PjlntqrbnW`dR+w2p!tapDAa_Y
zBN!;X)Nhqj1Yj+QLDq2v-4Vm`AEy_WzEd5chvKx5egc5%XoyN~SOeZv16XteehYMR
z^S7zTCF0ff^07(uB`>gkjn2S(C{^S{U1XQ-mTNfLzB&HOQXU(x|n}kkngCngy1T*MtKGM+{Ctm_3H~W-(`wI90|SZc^(7#2^|)$?qk|4X
z?>r^c<-diRIQ%fI^l-gB1iAw5wpk$~LflgQHw?^boleGCU*sMGXMyL3_H?N&P5CPA
zY-Ez7ngQfE&G6rL!v*4CC71WN3P8=8yLW@)M~4OiyI`Cq>iQ2l{Gm{g=^M~X!QEF`
zbJZlCbA)O12uKTB|JFJsz?x)CvmW6bn!7vk>szsZei_13Aw
z$%A4%V+Sy{c@fX$xaG)bre_g7p9yKCNxx&%W(Y=&_JYd054FrYKDPJgo6xI(3I;O&
z6el3n0_wG&`zTG9TaFsO6hsl2Ngy2OZSmh)rYF>$3+bQz*$v;xwC4V_j1CWnATzhO
zDP2^=V4&`grKdgUSOCe|LIi7IV;4|}N-zE`0J5u0Ubf%-rZ+@O>IQgS_tQH8|5jgw
zKR?Y7O(wfcmnE?H&rt{-Nhl3RQ(jx5S4{9l+Gfmmx&|DFuiVr;o_{UiHq|)t-2Jej
zqGapvxMk$%VD4e%A&X_)$;`~p%*1~02#QB{dw+Jp_a(RFp
zWw}QGc$p|GT__e%)C6x`<&5?$5Yw)2M&9oKms&gVPHpaz79TylAKW7P(6nk7ki`*F
z7B7&pC^(hHZQ|X1hWIziqI+GNm6h9#mI1{Tn<*S`M@~d43vF(@;hvOM9=MDsjJmjw
ztjL{=#~;0X$FMO+|m8P>m>yT|UhzROCiy+>d+ugm8Bm1CE2lK9p*Z=P$R21S&G&Fk7z_s>$xk86aK=%~5!RD42d%NmCU
zlXd3+Ergn!`!4stj{^@rhzE?gI8o9mfE?tEZZIevvoHUbj*#Dla?vF`@azJM&|$Rk
zU(b5$fgIEaYlpsMuj}Mq~wQM;-Cq9)9vn;qGKf!`0*0#PsTaFb~cm|0H4YT
zk`DpxPXOWj6-!KQT2(IEs78&4RJf&G^+z
zbZ>mn{$uZNhN_x>qVCVx&q26xoK*?(#ZEjcPE2P>lu+a};k_?Z(3Bv7Y_CPwx-M)V
zY3IJ-r(z~{o(q8bn71_!qLV7n)+9THvwVl54UfXBr|91NAngI{HPD+?tRNIOK1>`o
zbp=QO7aTe=au68j4Lc@yvo4u{jv?Hcz(qj#5|4tyC+Fj1_S-Z0NwXS-SIYBNdPd|N
zH9d0?e{crMspqTQQ9z3>k^4yx2q5h-BrN1pvz~0G-M6NKwvGnYGOAJj3^7J_$gbe$
zElApk>-nvdZV53rbpJL_S+XVzl%%VPo1_=!;Prhu5iw$DbEP4l$fOm*L0TZ9bDIQH
ziJGP7oqcpbUQyjT0gW+VO~uZYb46@-Oxlj7k>aCY1mH=@d{xRCEdb@+!TA8;0(nSH
zI6|#QsPcaRRT3gRv^;!8>@75Ak=J7%Rl~gq%@3E-ekaMKd49G%@0_+s2hBj-+8thx
zZ6B*4a#0Ka&xP}~1>-Y=#T(=H3JTnUutm>`au<^f)6QND8cNvY#Elzqg&NJ5=C~X?
z8%JWU*$ZBv6bMk__$V_W9Re@iE5E+vwm7u&n(_MLR`qOqx|gPqX=`k4*zHN@mah{0KP)*m>kofCeEAxH~e60#YF5CDVBXOM4pBfGXEaAzx
z*|g&xDGcUaH)p1_{Bn8;2?)7nyY42*I#?5Zd-c}ji8l%x5!x513h56RbjMfWsmi!!
zCRh~@BGUevXtTBm2VFb&EP&lzjRq`9bs2q?!!}W0s|G}{1xM$npq?wep_Ukl-^L^R
zG@|OoIH+sBbmIkf!T1S^+!q0koSk{H+(}2uJ(SPa>})X_&`cz}S4bJp|7gjdpL%9X
zD9d_3n$%G#bi**o2jyYjkwf*r<{1@q!<(dpoo^Sooh6sM^SFABf>J-=x87K{(kMnTUOjP{?elYAAO)
zJB`X924w$u*x;D9Y$Jjyi~%eqY53h)CKU{oemt0RXsFZFot$DSYV5^fuhRLr>%}?{
zbx=*?nroP6mGPQqV(9Y1IJn{|VZF2Z3W1-oUvN*U>lfFZ26l0scunUd>)M94a!ty@
z$Y9fG0k;gi$>JBOvRtLSHUu2|1Ubfc6ISc@);X(L~Y3a
zD*GPlzv4K4Ufr3#N4~XC&uMVQIGMyMVry5}NaI-hxo7)xLG!R%*78gjZ%M~e&MQ0x
zJas4?6qiWCVU}?f1wUa$!Lv&31Bi-BPXIK*@i5`?2u`>iMw7`Gtu!{!$7p4$tzsrm
zi_Zf06n(5m1S2b)D4U|gZS}R_SF|wA>q-sFD=8KWyKzx)%wPO{oK(8#vzj^8+w-z%
z2Ck@95+!#~{dqjYR*8nms&9sSAMPvUl0Rs{=
zPCIU6_Zwn+hDrA`N@EGQxb*k*o)iG4O{nBjl8wbzey}ji(x0uKS)v2|IPrRg%gn#_
z+q-*pn6o{l5d##V9=uaSbNn91Pj-cq4O=gIa;#}PG-2IzY$&$1zC6=jY~*q~9r`lcBg6kis=vn-Mzh$;$Kx-$%0LY){z6-Tu3I9t!(>;}F-IkAkBWPdL8G^QUD9kkb(
za-RqHE{B%2x5)a3By>u)hPZ(^5Jd*>4y
z-F?s-p(|k{6?N?gHsXli#@%b`iZUSd0i_hjKb)eb^aYyVgq}A(Ydea`cx7{FH0()d
zz89aQ-g%)C-1g%fkQaW_XCIg}XB~QxH_U;>@WYJ{0}1No=l5en&OJL}lY5HpNQk~!
zuiLLIas36l_f-VvYPgjmdXX3S#9>i$TMQovJ771D6%JE^;XV1#uY_Jetanw&`g%v4K5>KT^VV>q8rmK2oLzmer7l*(oxJR;B7(CAdsVodUZ<2=76
zg9z%QzJi7g8-_}T*M-FQgxO0bf*hNW48-(h1!*&4_a!D*{ViImEMmO5L)J~fJ@)z~
z8tt3a+7}edHe^09{qYj)8>P?Ut<&OYL?Q<3BUW2JT;^z{_5Df*-!9boR_2Ri;`0es
z=UzkLiFXx>J0f9G#W@9}(VaEEO+tY_gu11=PtWMYWV({zh*1
zkG%l+xx&NB{BD!HObqVQo#Weva%)azjnDFlCk&n5oKJ|f86cXvddhHc!5J4;dHf`?
zBrrH`GTapb?%v?%iu~uc*QJ$m8
znj{xdy+sa~e6j4#@*LKlt``Amqo@XIH(`^5t(gG(+P+Vy*0JW%BK2*NP6ixVE2eQ)z-
zqHp&=OK?~Kt7v{SNm%o1;HuK6MBBAk?*rz{S^FGR*l+^oEh;*ots6d`p>4i5bBP%6
z(Qkk^DKKlq6GJgQQRK_KXHhwz$16KAK30cf)Q2*v31P*kA4^}?uHO+eqhpz5!_@Oz
zi@?2m_~7E?$uY`@RHqn%=WgoSeXpOl+cPZ|4#pR5Gx8yHmq1FGn^9#zV(jXoTb*NS
z9OEBhB@!e)1vY0O(K?@mBlws#J<);gV$fstwW=To>d(oN2wsTp1m-2r$_av(IE#pl
zVTY2b)YPtJdsmM+kW>S9?6;sd#T@Efj!LYJWH6cqB0voz2%7x
zIVF5cB7f<7l_+5EYS{*nq4A8v%zSXFM1Q-~#%k5kPO-~8Q(5C~6vKrGA)TRvdBeRI
zdr`7e>W&dQc~r2vgE>D-o6Au#_pkshr*duIue`gIp`heNX0&4mYLaWygmgqkqpKn^
zt~ci-?*8E8CS^x$-g~E1My)oIp=#)q+-TY2P`*o%!>Gp}0UL`Q0EP6t6d!F^aKP9U
z-gs*9jJh6Mct?!9f9Qc!mH=NkBljx6+#qo%BK+8gq$-dxanA~QWDHOSzm3Jh&vSwp
z7MS8l`!AvB@zx$!(&u(BV7qL|B#<}V@Q%oN^U%X=lehj*iEUnk>fh!V->Nd&?5Y{iSfH`*;x7KJcN`kzAT=I1rukY
z)T}9iXrA40Y$%$D?Sp7!aKE`?`9!@!J8)LrkL*1w1#6u*esXP;-h~1nz33LsmOtk7Lrpp!vYNJ0V|e7Sn-2=MvWh=
zSg9%-WN?LN{H4dUqL|tIV%sF^N0yB(OTnfSG-WZOt+S31mSYr_3ir~qG$cXTMrUF19GF8XfN6-;=EaOa-5zF@;;?U$<01s`*yCLOr(a_1O(3B)@u
zlyNIMAwy6xeHDj|gHQtF9Jmk&bgi*BsNE`WlwI*TFG(J^xm11Lwj~
zx)irjdXx$^z^4+gsRC}33OpuTm7O80pl~C)jyX<7I`XOt4gF2Kuwo@NoU!24w{NMJ
z=Xvh15|d)1y$WD^8fhkT!icoR&{s1tiq(iN=wuW)
z0!~b0?aImOT)EvIBuT^0%hkLRmWnNXXQ2W`-b=sjtES&$=A=p&b)h%ACx*KlSdoPa
z;16#c8|b)bxAr9cQJ79$kf#?)<-p9_6v9d#qi;FVZ?r^+F)+~y`1sxCd4+}_J@*ax
z7CnHH;hB+Yjx0Pl0a^R#V~3B0sp50H!fy_iL2sH;z2+K@rL?`Kq2|%aILE2FuNJM!
zSkgVixKVBm&IJn%3+S%kkzo{n!7^0r2PuHLadxL{b3Gh{fS`S~$_EwzxcV8Q8H^sG
zQk-0^L>;!aC5PD{!E=1wO{`@TYbP+G{vL9wf;mJ4&{;Da9#eU0$Z#6X>3dDl>trA{
zKs2Yfm3X?22QNcNW;p5=24H#!4!43Hy4qs!
zzcUQhjC9JV6(Wh5e%j;aScZKQfTP*IW#n+QQW=_ZIDV6S8!AdA38^1fF=oq``t&GS44b${m|#ySlgE*^FA<@*7x
ze&wR1dV=;=GdRFXb-BzXY9=N!`;|3->qk{)Q7mnq3YSM?_3Mm
z+jXgcc=rilJAWJtEYG~48CBR2BEF!(#$~vw;(9QoL_G*p9xmSe^l8%RSOvaHi%2E~
zvO8G-TY3`W*=*AAmZPGkBuK|-F?b{%~8fENe`Xi@RJbkij#2UwVYWdW6n{wS9-$t#t3@eKc
z`7&Lfmgw1KYR2x$RIkpET5);0ouxrO&8qvK-Gnt$i)LG<
z+-EWd=vp?KW}F_tFplm=C62K%J9p+KXg{zi$I8}{`i2K$0J$p@qh;4}$4OGn0(AtI
zSzmAPS4z;{CZ)f2KUsU&Njpm@N@Hz#=>B0T=*ZIFby``n)s=v?c_NCOWVj$2TVA5U
z_R39USRGd~Tyj4*d|3Z15Q7z#Bvf%fs_T8q!@E@^Drl0g?d{4NeDf6B4V&Eix-}Iy
z1&~?L3FP+0k52O5s~6IVIkkM-^VpuL)q|en>u=37C{Zu1g+p(s?jJ}`6AfGU2%Wq0
z3`|@t1yh<4dWpYSpAqWUIws=TxaIIxe;hE5c^|#oe3%!rp
z)N#!E&ZjwM?zk0nfwv}#RVoj#ooy}|o}bhhO2~J?7=0po^Da<0K~&HbdzHE74S_Q4
z!7ZI5=9OugjegnD$LOb9mEaYn_x=8qcP;H61LTE
zgSc)F3JCWdYT+v<`-a@@K2Y1ZJdh|M)ArHL^j3C9;1Z{6Z>Yt>D$$%V(f%HRM=|pj
z9XD`g$m0d;Dch%wy9+=mZibQFB~9FjmYzOwY$D{|_A^*HzjxqjiyG&b8C;m+WR0YO
zQT)tPDbyR$KsnQQbM+R26nmq?KfSsC?6TNJmu-sAtkR8nxY!CW+Lr1}zg23IjiBw#*wtix4h1C+1FUb(;k%b5{sQU
zR56vkv|1S|G^%cT`dFL_R^Vd$mDOS(uTjsVMbGHQE}zrlL$m%>{P-m`G$j9Hu;Nj#
zDG>|mb+7RxVjO9I@wDlVczB;x
z9U0C&P{x+g-oOV1sRRi5M9u|O6R@W?h}V>Q24nCj0e*B9PxiyPO!w>eCN)iZ_f*Z5
z3hA)@Ip-M(QKCT@_r_Y$V$OI
zyQ^3?qvzrn9Zk@rZa@M+AU(hR83WmcHY|RaxbBH}TyH`Z=L2qR5ka9(HZ5=BMQ|I{
zS|!A6+1>J91^O2EvWH`>5ap7d+m(90M)^|0lR-*G!u=sz6{-V6%#kgNMz2Xd`-=av&ZSLSV_os#ogapXj6NHa+1eMI_yfR
zdW4qmtA_p78o7uQeUI}?T40?gP~#`@5`W81%!8=$n2&c}!AB=QU16g|B!7{C>kp3f
zslVy5^P}n!Kdlxvy{qCgHJ3JRbFGo7%4vfn*>WXLUn`pTq0hz)2Nga2C4ldLRgDP`
zjr9MxBsc-o`_t~^fh&ef`Z9wJFAhKIKNOG-agqPjKNA)=+mosp?kFtfqh*bKeoU|V
zQrFs-hRhEXg@+~dXk8>vo-pA7dEp%@(sTj@UZBAc_rnL2Xd#i2a-sD;rpszCecZiI
z45ph&Ua%T1ms*a)56nvk`i>ecN%<|x25trS@VK?h3TvOhEsV(4rf<6*9X+Own|LbJ
zGo--}RzcPj
zI7_pXITHCCUuJu^AC&d4&_#g*oagiRjJSTrDB&
zNyDxYy5;vlRk=r4^L5<4vS&1+Jz;t;Xa{w~rQ#3RijvE2)@zH(#9qS#(WsC@USwSo
z-JKGgK6ItPaIKA=sK>h{O*T-fF8$6%8usBl`<*tsJ*_)?7mVeqjntpr>8T)4xF$eM
zQs1i*3ycQB2eGJ1;
zMOumQoKF?PU^MPtyN@FjJ!m4+Y;%dZ&di@{j?)yPjDHxJhfvupbsdKK0HC#d|
zO~2Me$pOk#uN{&*>bJjR;E2&bJkrsl?*x}6%X)*L#EB@kFb#R1L>27@2d*lFHn5&!
z(YsJ&cWGZn4W$e7Eu&`^N&rpV`INvzJ5P>n^NiXxC3R3A%4QrIV+#+4o|pSHf2WgC
zqMk7B8?6LeI{Ozm&Mz?2O(;mk?x06DQwz%yIX^}}Er)+;X1sL;>3tCK1i4@xDue)h
ztKO@=8Dt^c_Lk@)LxZa2!B3%qhp*_+(Pgo3mir~z-5gQ447mlz8jmyUzF8E$)>#P4
zQK%6-7KQPVi)^z2t5f8_x)+#`;hDW(C9;nFnR88W!ZceUxn8MiTDcOB!&E6gpg!oz
z#=N^Y)`|@QXx-O#;Tcf=)GVx(3C6rT_FjrIA-v%R^+`+wA1n~)cV}r)#Aj^~PovgeP9+82?_?t)7O^#d4sh*p8N&mZ*uCA?D>r(0
z#@6fOEu*@h9uUk-@EHxXVgS46MN`kk_$<
zfW(hC_JEp6K>cz@6Q%a{@-RrGv@Dd)P`#rqea7&zi1k;t%VY3Y{!eT6FWRbnRG70k
zKRub=VK!pKX~Ekt!d-^DqIenJaK&Avtio#Tn&2j>H^e<;o}h~hbCvbJrSsifxphd7x)lk`Lzs~NHPu-1Ice&O59^J$*m*OY
zFqc17yNrVu%mf({bv5>i`1|S7Y_V1@Yf8m$31mA46Fs^&Yn6KO(c{9T+mKj0AnHL;
zf!u8MbydI!A*Zv9@C(-M%^KF|3L8LjVFN0u;P~*&nopDGIpn-`>Z>lt}AZx(%tG1rNRjsFHaVh}H)5`B3
z!C;W@pHe3mBBm_
z^R6uFui5)n@2pBD(z`mExz^rUN0n%*)Cej2l^_<2#z&7vtyeutRrS4LWP4WD)HAgm
z!IqUUEX?HO6ZFb>CjB+p(iW)4s
zJw@~AJ7CeD@6xnmNOiF3F!)M%1+kq)xc<4bI1q5G2zU1fD7n_(?RZBnd+=CF1-=cL
z81tJ?vZ=!i6%OVP4y^Q5F0>$t)hfEjmeswfNQ&kKqGWTY?0Q_R<(mEAQppa5Z{-CF
zTl9Tej3DyPS|P&sqQ+wE%gp@_OMc#x?w%4QZC<7AY_AV4ESDr}e8hvfR+3vjd(GTG
z#za%|Y>Q<&X&gsa9*F^)1-u#O?
zg)5S&4y_3Wtu56eo2`25M>FOeM)(=`7)}Ghe5(FPL=oA(tS!b($>Jw
zc0{^f*pHo{7|=!G-Zc7l1^ZJd%%?Ts#-bpsAv3pQ;=1hwx{>^40<`yMc-
zs|Z8bLFW}gK*+-DhFD|-zms|KZDxZ-c8o-0rkr9WpS3Y^FrU@An{BKjd;Jh6BBtIz
zb8(!=*}6IGVt(mWf)zfF^VjvNYMd)nW&kVutWd=NU;9%HU~x8mlJ^7^6sB9~7Ra?xkyW1j@PZygKde}mwI{dUt2r_VdBbj@RlVW=1T!I=bhi{QSrDI<*^xMy)yrdU
zq*#fh*^oN#w;CLHu!~#CH_c_ODf;cgIhVWWcM3`=*vy&pIeKm!SqN;t;+`cYH_Q`P
z{&?;wv(*rq$Su^scPJfvudBa?)352>ReVF0#&?=f;3V*-rM-<8RuJ=hs+U>tN^fDFOi6cArel^!mXO>;uuz8aVD~A_uVs
zlP%q+)>)7BeZyCLX;zLqk2C!1%G<$sTF**pj%J*1G~3u_F3#
zQSUqRu-CaJU$7YuK@WU4XAJzlm5*@Z77EFr>90g4x_!%V>adWAF1*hZo_+kOFW5V}
zEA!selN~Dg7ilX~;>oKA@XDcJ_pSNv`j`h3Zua?YG!FvQH1_^OZr_srH+2?rIIebh
z^*KIiBAd?JA_#T@WbM0G%U-q~`S@?;z*Fd_->G5eYQFR9{HO+!Yx&5Ev9j(PW&>=c
z!#jqkW?|QUj9wda_nMH5d-fEBJDhcIyCXeKLwiHsZboYhU~$g}ntC@ws_h+oG#g!+
zCZ1_M2EV*}|OJECWlY@WK6
zad-`LJvr4j9)z=+doiioYR{S-z;m0uVnQr$ih3&wlT>kB;%Jm3mn{;7mi|UzFDMg7wm?GJrew%^
zjRIEjQjwEA3S%#w{=!|#lw8{HRZZ5^i}pZ8@o&~~TFkU8M5Nh>{bpcUPpl?hxBX%(
ze_Og`(guoQ?Lzap>(|&v2lDr@0FIU^!~v^V7)*ByzQ@YWIZ(R7ck8WvCUM{NRImfE
zqey1z=EV;|4Afls!ttr{BS-ctgj}zzN_MB-#$^dMHm+5ykve|GB4<~yvz`LiYZute
zw0s_zG@|Jh?r6^(so1YZ^$@o96H%^?GaYfwgLRDs?qzg6lqzs7Bj>Z>2dgs20Rbl0
zF<59T8j$xWLuky4Cjj9ZyQy2|i1G#1rj^(7*)55KQm>KyEg0$b2fjKnBxGc89G2;E
z-}NY|%kgf-=O#v?*w!xvp>fLQ-eF4`CK<^86nZT3-e-N@zc6nh)h&s&^+dOko^(%J
z?70{Z56+{~&APfq>t%kKith8R1+Fx^)N0;I|?+LN0VBHlX7kj?9K8~B~3b!w7=5xpq@#y-U;X
zUQ|7nD5OEuT*O4Q0CE%Mpk1ChAq+-Md_7jI?9F-U%~!3*tQH20R+nEItI%K5>&!D7
zvBw^WL+R}(TYrs>i6(ZNAg7xHm9UT~<&-YbLk{tWnNF6aI-l?wGWBdo2V$+5b4#nn
zZvk5#$pyYD0`ZxPVRb@|EVf5x+}wtECllLo)Z}7doMK&bo>uz$p%<{&e0OvPzt=(X
zOZ#4M{J`}BtxL{>mUKNrCn{P(4SC!7oE|MGlu@|7UkowKczM!bGlJ}!-t67-~f-%cLC82rQs+6!s
zsEfKvzU$LbUV9D$wvnHp?iQmD-S>67y8`dh)DV&IPtXJQj@UO^s`!1m4tlzYkXiw;
zxaPRZXa~%!Ca!ALdd2XdMdU&?`ODR+a@gJB&BDPWQKxzC{vA^6*Uj|96Ev@I-&Xn@
ztdt88z9!oUzq_@s>{BWiG>}na!8=w}T{|xKu>9aXEbW7Kmf21_G)B~Am?`Ter~58d
zzT^8uv#ZDR_W4WwBt3*LKFy^Hx6Hki9{;wF>wUDVavq?|jF-Tb@u2YK>!Yj2eRElJ
z{cnI$TuVIbjl73d6ntipo!qE-^f)z)ISGVDxiN<%5%p#8$c~e0+Qyn&%CvrpaWa92
zj5C|KIIdUa%KfN!$gwr%>YoHZn@4V$D8e{&`u0RWUcdTzBk`DF43|_C4(Wy&tMd>g
zRs}~e!*Xbk%Lb~2@T&j?4GM)Cx3H_ZMd|{Bs$UCVUdKU2dRJX^h@>Nf0Pu#roFfVQ
zhW|c-@{JLWE`TU{q{lOp19sx4${wKIW=_AmwUM5tCa_U%EiMIP7r(KhXtKD^w?O(;
zYGJoT=2`~PlhvGredrdTBfJ_&mWHn78o0B(66;$m?IdijM(CJ>V=k7pwk`8IHlZ{M
z+PVDl%If~?^P8Du7lKCpRwU&v6qcHi`D;||f<*D98J7&;gC$t`A_n4mM^}o%((s19
zypT{~xV-Yoi*=3l>B6$NVAPjTA>4DN8`RJOiKF}YojKj-m>#q}Vm8c^P`*WxR7xvk
zx9kv+1wJHTmkYhb=lP)d-dMH_lkoRVVHmAWQsu~%>px(Hj*VO$9C-MBXU})Eq2EPR
zVDi0oJ{u_x;j~0^;p}v$>rsI=DfuBDizjNGUF8o=HyzXEgSY7E>8R-iOQ8K1!f#>*
zKFZzXD70qPDj{gp$Rg-lNp~ZeZi#kQMyq|DMjfnk?xT0e&~*S)c0qt`@8r1C#=QcoY$GzGqY#U983XL
z{bp!N&I0tbiT4#`giGHK1vDEs&uqAeYVFP*%Ke5$Hm+uz<|clqNSdf9c_BgQh)c{N
z;j`>N&p43E@FL~lWRj#zcxr<>5;R`YE$TeoYh=9`fsH7=?KH8br2EQ8zoAa;O8EF%
z`%sL#C@l(BGE1q0HWBDXb0T?E!+t-;LMYDk)`fq?n9>LM$79g*;7qbUb!7{{($L7b
z1Ta|u&KFkD*`fb-ul6CR*Kh?Kn)TJ@2`0W&p2GS{)$Ta
zGCN0u@$1dVk;PF>^CqAICb{90m)>0gCTb;0z_`9JlPcsn)8w;m*aXvhRtxXSSD$id
zDlJpi3?$rcY?4N7F*SySM@C_d{Wf?jO*Wu>)=D-Joc>5&D#;l*CH1K7CXVszr**R-
zuM&PP>Sk*bFySXCEgpmS(61gcs2cp?7pplf%(+MQeg3#m0K2}&mt7Io$-c!344)n~
zO|t@cqLK0EdGv=P?(x^Jp{bi+7nO-b`j~hPz=IwK4otkZua*r_cU&du*G-AWp8jd&
zEybG3o&F!NlUM&MQS{0DuWS7u5Xxl!;XIDl@Y99Z8i2Y`OYd7*w$*fur95gOmN6^a
zk4PeqA|18v7MU)R+E;?ki*d}ngedkn96S{sp0;01c~B8#?+5c}S90TOI#!R}`qFIb
z12)`Sg{@NbhxI<=b9#}6@a}TNMCXM2fu#564H9yARHjGZK_+03^)^b@{e4e(=@Hq1POgc<`nPzQYQ`8|8OeFATWPHL~=XOL3cp2NLkIe_{xe8_Y8j
zq&F6gL=qj)WS3f{c@Pw%CsVvSd!gQG>$gD8m|Xo3I<1!mXc8Mb4K*YGys1zNfEx_klDuot@ytn;+7uqNH@SLRBHg-BCAPs@#ZKg
z->|O;{wLatv(+e>*i06nmk)?w-PTQ)@~%2L22ck?IY6F|qg_`?+&^bNcQslH&#Qf~
zd^xqAk4k|4@?+RQb?Gh%^#mXdUx*65_E0|khARI(+w;86#??1ppJC8ZR4>6pXNyKx0bP^*U+tW(~sLQh>M0(?)#V2->b
z%wJV}#MxGH_5UME!V~@nl$QU0Kv~qP*=T#HKYzddgKU4U!ZZ1{)hZIDaW}uE7a~>K
zjT61C2i{$f*sXOw3(%U!FDAz7%=}8TN@eAPUE7
zW6y)9mCRnql2|-tE*&%7!P`}CY}=_acmq*&PQzuTGpMSSE
z`QSYeTby7yJ1wa75-O7A_rqpp;MQWFMDgieL@7FD|5lxB+c`S1Ft+H;5x)C*@8RKW
z9l9SV7(4DA2M!yJ^E4U|9W7!vctTsd5#UB-w-C=
z9{bj12v3L88gX?`Z%=tD<+MlLB>z#s%31II8mVsU(utVBge}%q78=SlMqXDw1WGP4
zgVw#svkc)+&7a;mA;hWxX^cA_cwJZ_+~~{1FziGTj{(YXo3nlGJv{G;KLv~xd=^K|
z!LnBm(rDgT@Bml(;U&TWwhg(SNnB!PEw#N)x6#7UmJGDRwK(?lDkOjl#EC*L
zzv=s~M_zcczz7G*-w++`cmF8c{E(ZIQQSG+Ad0rRvR1~?J$VH05KJ~q!brF(|4ZuQ
zmRGAk_OH$&O-WkNaTyq`aa&HBH>6jg>qPEwPt`!!D#Z)`7l5gLgj#Bz{=?e&U
zw>$+uhf}~tiROMFk{!%+a5fMo{G`&?EP6TBE;;^kGo
zo~;Fbryde67^Muj1SZmeQcbz@4WbX(BzJH|oZM&4OPF`oX;CKxEAG@3pq4Z$AXLOHyhi7qu+Ccg@$G;!Vc{hq7J#m#Q|ahFL_XV
z!sQwC@ZPK}^OIK!L`$T5k4U+3IDZnKuv{o)NLWA!lNj(r3{;3(F!4CVlUG}t*I<#m
zaH7g5V4eeh=aP3MhWL*$Qi&m#vuN*d$ecG0rg%k_kyNv~H?$-L
zQVTjE9(UH%wu!^}S)p`Rsjz3ewM@O{4~d7|y80sk0u@EA!qyecs3KT08Oc|c%72>D
zvc8gP2I@!$cz^Efe`SM|(!Xx>>Ho&E01y0g;}OUa=Q5UnEVsI-(sOJ%lh0kusyKdD
z_ak6BP53LmJWkg(xCtb83P@vSka5)bx8=|@>!uC~^MkMLPlswA+_!F(P1=0O-}P!%
z>U?pl|LkkbKM_mPpxc-1e{FU2K1*0+#Z*J7`@-~r=H$S;!GHN{(`)xRa{gtmS?60c
z$cQP6fO-b2F-gIfT*S16i0fsk>S%*8UnqI>#%K5i^o1*duJ@q;#UP&$I9;_
zKDD0wb^>p=
z$v)={T6!V1dw!&mv3$cZA(N;1um{c@4oF{IKxRY`LP9vxBQpT7y{-?(efJ;
zO(1bD?<@XjkzwSi$|FJUFVuE^w6zgK`(PWY?=vL++1qrE
zc9r1c>QL^VH%rKy11iuhC@MqO;~Kd*V+CJTO&|mnQYVuX{elDC?LPQN7p0l5nm_NCK980k0oPF6Be+5}&6tLnS|8RQYIqZS
zCrSoVGv7y;wgh#KQu=Ie^r@VPp;J^O58@6G?(hAV_^lX@hL;r+kP`^(bz2$FyAygn
z(Y3NV6vG>vn=I)L8=q7E2>gcYNHE8MIoI=QSGho4KlSS;6>1g{6viel^-OVY>S5y+
zFMq`O8%wNXJXckGtMfe++~{)1YjGVYHcacauSw2N
zfCN1n#z>sBmLQ~$_$tRrt;El#-pzpiaZCUs~@r6`vG047W2?^_$!BfQ%Yv
zII~eRN>Yj4{0CqDn@1v}24|Lg89>DfO_PETT}t2ezMeo|oxSHVaJlX-K$Iz!fvcY!
zLbpCS*yppfKe`??0D2>|1mVlGG=OEze=^UWFH6m!lM7gQOEp+T10pt)peLS_G>Y8%
zPT9>g$ORJ3X)(zf8;48xT8-U^$Iz(aQ|H)nfUT7#AL6!a
z)d~>$=a3rrk|@jj2^CGdMcc97@bqxm@NF^XM?Lj?s2$6L7N=K>IkA|6ia&8{8~U|9
z3a|8zN?eb!Kbas*UJegWppp^kPvPwH$DF&@Kc@Uc&;P(R?q5O}U-181p-3@t;_WDa
zGu#~*)cp{R8{UFC#GU&2mW<1MQWGsKV|&hNT3Nr261d9(NnlLNI>-}<>!g-Nx^4%T
zhOH8IJnH!(sA?B&v011}w-<^Tg|v+8%SVx{wN@hxd)X`zAkO+AlpcgOn)~jdj_F{b
z=f}fZtWRP$Uau3#HiV9Yp1&s7g17ys=UHE~=*1ko>isAh`dkmeybKP^Ee}UErSL1n
zuF=3}zAVwWl|8kF~Zcrk(iStjUUQ-}X%U6<)7=986WDGLhko?N#
zkDP2u-*my)k#EcRbLdjSx|Uot>~{Zk7X~HHm4`6u>OJ*Ej98dQ({j@y4-bO>@nUDA
zvee+a_V6+?z>2U!(cl_;6g$Q^ERM|`Lk@ZdWwlRF)fk-jG_md4u}5hWS16*JI_RKC
zR=*sH>avr@3XiCiJde~*j0m#-`a|q6;iiloDNcVKP2tEpKZt7EDN#CUhw;Ly^c;SSl@}LtU+6#J^_nHDXyG-#Php?Mril3IPK9vfZybg+l0w
zcS$7NBoxe5#g@dPCvjR@{h2wsiEgA?2f!YE82DFGQ)qq@cpg^oC
zqP<$>-KHso?n7Lp`=Lnfn}*JNXF>=6>+av`)3hCQ())9;QFXGG=3jAsG%y0<;iW5e
zJBy{Wq!%Gl8_*K`X6~Joi5Od+sB8xr1ZydlQrx~P@0+2|Yuik4fS?|Srn8RWDK?z@pY(EM`UEif>KNHc)ocztm|o(Dka2xJ9aoKP-mstR{5dbTL$GuTcSS(7)+
zY!4;85+W1uZ*15yt+0TQ3;anrvoxY?M5<)~F6+Uv&BdZ?E;&hTLEJFBKQJC6kTZX}
z)2QwnlkeMxZKoAkYD<7`Pv1Jl?x^&(YekY=rcJ=h%)+9U#CRuYx{mf#$*<`AIR1k_<{$XUz%}QQKt1VdF>{UMTxhC
z2etF@(ETOrTz$xl`sajv
zC;1H8my)*T6r?-c3}Ya`aO0e|R5KE1_@#P!q1bPne&V#kpcd5G40CwxsjvM5ks#Uo
zPObp1Bi6aD>qUp?9`cUf^HmeCUCk$G2Hb?#jM6R>WIrZcyPnrYI=knnyr>mMM;za<
zF_1*vtVTUL4_)%>1U)tUE+T&+9Cm>FY($?YD}>?4eefEPrA3_wWn1y7m4B9_azLW{
zOD8@RZWbaZz#zYZ3209s|8JAfkAyDamC0sqn9#!pr}qiLS%{PY?!1
z)C{X-B1vm4mk(%2rOh|Eo(K3eCbYE5&7NjW26T10TP_GW*198A?&7%T^4#sb>ee`2
zTdK0h`k2j~1HT7f?zz*^GsZjPv%FXAL%|QgN1opmPJ~5`lcj
zD|E6VjZwnrQtJ1V%M<0|5n|ik0^d9(OKyJGN{jR)uen%hfpm5o;>we@9`CkX4s73o
z&lX6-{nqbhNBZo|gKuaR5`y)6=~Qs^ID;tpU8)X}TxXgSGlhdc{Z_64HC*p4d^b%B
zd;)J08nm8Uo3y1aw!T6-}x^*`ueI$07)lR-w|r?sUKzL*yM2?p=AkbV>_C
zEADl9m5Y&7V>WzoP9(v}Wk)8bjZEXutITx61+d4v;#{W&MLnH2ACPuioPWJZYGb+|
z5tJ=>q_~X3chm+vLkiAAt+RSR^VXDX=SVGTc}Es#DY~#Z_D;@jyT2r2xYx0gH(B+S
zsHpdJYh;d%xCpwxW-aphgv92ki_wt2qc8eTE__PY`L3Gk&CHPI^Ga$(xSt%_QG+;~
z|Mz>%&okjhcN}}AGK&tj4N~m^Fz179nn{o-k6UA(td9SUPmf4B{VgZ>X%i+oT$bc~6JNYTKBT&l#VO?_!0
z((X&JzmLZ`O``&*2y4g8F`BGNiJ~z+hL3dHY#{!ZxTG;lEmOfrtAxat{s_9`_c^|o
zMk*7-CDh^vf0Uqjdm3}A7K>yfZ!t7m!{zv^+Yd;!Kl>F?EVn1qzrLHQrP1h<4
zTF1$UEMhu7LepCpx1+P}(ulC&45;VI4dmB!j|vuXMMd&5Y_$8I;CcH6^=nAyiMcRk
zOFY-+Ok~DR)pCW3?=~%O
z?L~oc%!q&;%-Z*#Ez;}12(qYz9A7i0Jf4eCr~DJ!AtAy
zOKXjbX?MkF+fDQBtkzw|eOP}ARx`LPn5g@x1!W{0bXQIO+3M&mftb-E2U^KL2<5>U
z*To+gNb394bYUrwiG9&w#Pqug^;P>gg0{~4O$i-c%im3;gR%osqV72FcDM?LZqt+l
z7?o2_?kQCZYM@8+qd$2wuIFYj58?o=KdJp?@`L-4+G}!_rE4ArJr;sLC5CPdf>VaX
z`1R3H&fZCX9G7YiRMVL?h88(!P(>-^?UP;k=UM58T_i8v@X2Ck)p$D?yC5`@Pnv&6
zz5Wn|(*yY%j4)lQYn$k#bzTc`st*ZYKm7B;nhNlY*KvN*wYGdGN#_Lm
zE$iB4i+`F1eE+EI;Qx!t&NK&hJAly19-IBAc~Bq$f%!9TyLy+~S+0kKq;<{zOC#cFuwJW7;)n>OYCqT
z4kZ`p4w}oZO*46&fDJ}P-L#ji?q}el-%utRN*=CZ0$IaE+*|nd;9|=kF0Oj<_@t6A
zD3DRJBd@R4jd%U84E|R5uG9#GGv2^?ybn%B{d
zc2nOelh~yO!gJ}Or9h75ZW)t3-k9%f-|r?cwlZsm)YBVQv}ReU5&w^S7&oFd$72)^
znslboCY5+cu4)J;jjWj1+JEM6<5;0N=*;dEVfuqS@s%BuWJx_(s^a6a~+RYf^Q8YLC`E5cyHPT?Szy*wf?6dCrt
z3O7fgmlEvnG^?U;zG(6zM){+cA>40r
zjh!ZO79hp|2B`oOY^dYao*9;=XPk~0aEJSiZiuPor-6Y43_8tu5Y(tHtL&I0Z7r|a
zdZ`M04v{9aQO}W)ql3ggf~ABa(z}4T0MJTa*c539RDYlq3sB}`{X&hr0|?#p&EQ6t-(MG;;NpP=
zD&$Khqqbs6ij)<}DEKTi%*@!L!NC3oLe8emjqV>;hkEvht$IY=EeVW=rxuOnpSx%=
z1lcHd(>XglIEncEzalFn2GEYJBDANBw<)@E3W{q`w04_HJ~9!IfHw->-rmY~M1T%w
zUnuRy?mAiZC?t2srhun9?Me1;l_3mKpcQ-n)#JtRxq;Tr*WdwGKc@pS=auH3b#+GNOul=5
zv57`#0uJNwNLI<6bkA~+i$sGK8w50j0R$WlV6&8k5C)kTTmBb;fGz3Phgl@IIw}o7
z8zpO{9fuXa6ll-RmL+cZC@Q(Xwi;b`)T43wl-v>X+=xRsoIt!tSd}d}qVZiP>QBZ;
zRVe%QIljEHD5Yu@%8HU$KkQ>X#5p@bP0`4*%lyf$D+rY?O}eqGLRI1T7mk9!tlMkl
z6Y}i9jbhOt^lwIQeF>JE-fVA(2IcXS01KRUSq+2kkeN|;WF)*%zOn(NbSdm3T41#b
zwqFLDfLab6!f)DS8A_3NtB<#Dq9lN=XoZ=$jqi$!UkLSUj~#f{d<;e@evQ}`*WPAL
zHgCrq)*MMG4wzbg_=Ns$6i84mddWAnXro%$`2EzwFObz
z;;<0mx0?1R*cT^oZFwMHS`;VDJ#%kn9Del_r_GtL4xu@l_mX1@>Gd+uiIQUt@B5@S
zcuhxL_$DDr7gV;9l!+!okj(~IHJ(3A_DGa@i&xiGl`%H-0(E;hS~y(~9%+2JA7Sc6)eNhIEQJJ?Lu@dn*oN2M#xs{wGC_gW^~q5jfkqT1j0s5_hc*iCF=fIZ6C0J
zE8$`_{BdbYvttRv>E=0PX48TyHdMEbxO8j|8Ua4N%sk?G413jjpyrr+LetMXYVsSe
z!JS8Uo5sTvlrt}()aFI@9D6(}_aqz#4JYT~3qEbar#gg*aWaHo7xYBn&_7h~Zropy-S%
zq<<6kmudhucmOL9Gk_k|-xY}eFMnPG$dNOHUN%FBzAI7$A-!f$J0`E8iMg*eQ0mVsUwz8r`+#2Pj_(u=+>x
zugrdQcjmU!Ec>-?(6elwxN^3hnv^!*HYO196L{LGk4-*tHYEM200=mOs(bq)(ijuR
z>SMZyPtz(*!X!C@On%q|rN@&;0((NhMg)XN7-aLDDiYu9#ykgx9u&J<)*6NUQ8pTz
zt84}x*2ywRqZ?}4@1SW{0Uq6b$13%R3cg{SkkI
zL?S*WOL$3^YLINh?;L}6EvC}RWis`T({s+w&KGWadF`gN-<3$(JfKWYkq#B+o&C9tW&3QwxL>ssuVimRT?WZal$2H
zw75nI4{Ea?%I^wIJEGouJ6Wt=BBK$+KU_MPMWI4$$WKPt^RL`O^Ql>IuBm$p1)VqM
zrIh{=uIQlGz2!_0Ef>3&OiEzUtX2rOKX@AM2UN
z9-7*e&XabZ+RLKT1jisx>&*&w!_DHPj{O@q={Evfn_Xk)g?_oV*6hC=(&#r!R?scb
z4_lncJy;n!;WDM@>7@|4znCKpDQd|4l>DNB#t7B^ZL#yZKwgvgk>ip(elD_J_@<}P
zQdu=xw5q1q+3{w(ZM|{Zw|q96J9-U8H3XR}^p|?)r*a~%pHayuf!U~my)??@94L*m
zZ{+;bVmuRZzQ|W!VCeQ?G+>$w1sa&{NmGk+J`~5y!9nw@Dm*fp_KzRf8T3s`#=qoq
zTJGub3T&%PjCEiYk#~A}lXHs9oo$49QGUTehCoGQ&hO%#$>vAkBibH;;wSq2X~yYH
zySit6a`Om!uIs;YvJ4CWPo<5Va(Xfs3hRCwj%g7J#Y;!Tf0lqL0bx@7ko=X!u6&wEK9sZ(%0Y>x~D^Y`eXHB(JIOYjCY5yI*r;q3BD%xV=!
zEg~>!>I+N|T)$y5uMYiuQ@G*cL%^HBDFItC_x7}11U)~aY?8sr3!|R!6$fY}TZZDA
zQpwA$gi2=lg0R(-uuIgQdODzEjYIQsB5gAdsp9Da*H|E`j0mMe`gD9%n8@e7-X)#U
zvWi0bP0K8)5A;|;ouK-qJWZ(gpv%tF=s4lEk$~OA7-0=v0y2#RWG?8x?+__8E~s6@
zi`xL4!6fZ62A2tiYXPH56^Z=uT*1;X0<;cOp;U&P$Ffe2+^&!auAfH#FrX-*JM8;*
zB5VOW(tMC$B{B|I*^5x^D1Gk}RBW#|KK9M)Ari9UF)O&?O&LQy5{7}j;FR_jn5
zv`ycNN+{Luflo)hwa3kgP(w_xvft$Kt7Eito%U>~v`W6(o5sk*i^5KoxW;L%Z%EH#
zskha-iqE7j&(+_UEPyI7R_c6aQh`fTTcA3L_sf~0-1%^PF#or%pq3RW9H|`
zX8Am(xHq4S*>Alj(7qV8DSYPj>CTb2jwgCKvTiKh{Ls9?QH2w$gMj9XXZ$=*Y%}@T
zeZl2fDZzk@*hvnYjpiT{Q;Ml^s?SzXO#Vm+r;&CJlswjHU%nH!?^JNzkT124VLv}1
z5qEt_jNks7iXBSENDzfuNQT3i-;$DwO8C$=OB<_Jg#Cr9Tk8`-#7GWReD*+|9#gQK
z_TveN(Hyp+|r~+4ij^)Wve2(7Ykpp*U
zn`oUTtsIFF6&c6PTvpIbXbjnUFcGfZ1F%3LHRN&$%AnwT$D=GBcm}9+U>pc%g}9zl
z8h35*D|FNm+sB(G+}YrpNDH-Sk1ZyX*Q-(X_hg;hvk}hgqo73zh9~nYRRf7E%=IPD
zwfh^l+w>ZYS;hg;sW^BnPDk|ZmV&hdZ%$1IVAVL2OzTJd1fZzCgLhl@!<;ttFRme2
z7MmJQfgE7mg+5SCkIkAKxAN|EKAZ%UKX^i!H?Ac$UqCR^PSxA@5e!zckunp?_I0(s
zmzqw*HdW+my#8JdmyeEHZUD2TH2bn5UAuttUCL^kBvOGiA2i+QPCK0aY}cnAqxkon
z|Ja%|9LxQk#DwFwQx2`LrTqW-CJL_7c49X;5`;>~*_V(7K@gNVnE4z}kGh*~(Ds_X
zHHU9~8wbNv%#4?QC&gLazy0>D_<$*~)(M$_!RpyjPY!9dCj};?@)9TD*
ze2G4MnW>(4lYF@+Jqg|$q(KychICwH@8m1mPDHbQuNz^&B^b_4K(M!=w-qjH7boRi
zeW4Pn=}#Q(UIH;lpo{J=wzHy|JGsW+wpq-$X7Ry09xpqW*=z+|E)m#^%vMY3ios4=mE9
zq=_-vjx@6?b65++Uc17E&m!y6FSgNjmX!4U@)wwiBLzzduEMKvv8IPqQ@uA;^SzA-
zz=bN*rQCi~Q_Q2B&97pKZLOFXX%&%7t~vVBQuQgp=mn&_Ji~8vk=HfD(GkD0Gi6#B
zDYTikZrIZ4C>II>AUX{08XVC0P)Hsuw1l0PcbJ_3uFb~2vB^%M=*i<4-*+#FYl7!c
z9lD{4#&daNb0ZYawrTe{Vb|#swK2!D$-@jbG3}2m;J6hCmXSjY5l$*0{o%Wf6XV6J
zMJ|Tb(40qnmV{U2>vdwRYJCq?M-Ha!aZZ%xk^ltc1_9ePg2(dW6MLehwed3f=R`eP
zX?I17GH=y!mL)Nk{83I;o5SLDVZiq^Uh=HLJ#w0LS
zOWEB(hlu=~8ZG+B@8B*lKU49!z}arvHM;Cq#U7O-T^)8cT8Q+b=5#%5(1FdejOsDT
zEGrTaVh9@|#GI)UI(keyZENOFv=NXawqlzK?09n?sSg&^xt0Q1;teZ}-cVbJoTeBwsE>%NM99v_OYap{=C;6DWvFc}i8(7#Ig4eX-@k|ZcxB}Ek4HFe&MI-VFKOiow1rmrQdh-Z;n=lo3`Qw>JUqX0<-&2%W+-k$g
zc<#E=C#@Dcbilh&cz9`C^CkE^_+|rMJX8HmNhpY~Da8bXaF{TZakw9<=G@FSJ>wGd
zJWsj&F3Dk{dn#h36~x7eKDVEWt^jPk{KJ7ZZhza8k+w;3`Klp=v+I4sF5L3vsFMfA
zPV3bh?OrbKE)RI%bXf4A2fv~23pdD|KA|%~OPlO(v|I^iSanAYjDOtM-Shu|%>M=Z
z0upvXSCrRf5i+$M#>Bw55Vj-=D;((6!SOwYCcxjPhx>_rE!sOpI!rXq5a|~hGt2$i
z*rQD_4)RSlAAEl++;_XGSKoce&;W1z5Z)P_zHKo<>C#EKTt`b(VIv3GAy93vbBSG(
zTs4eA)X(b--IQmXWaMh&bu#2`^7Qr+5@ioHYh?Zd3P8g8_pNsBt6TWxzxFJ@xP28j
z{L}HSk3`{el+IB(AkxNdRY(lHR!EN6D$$Wb+xnAdG)vBTF8@F;SH6&!X)P)wR#CC`}f42Q??m_q&SK{eO1Vh`rp?>4Z4+-ADH1x`yl
zr}OOU@GP*w<#cDetFUS1t`F2tbCzea6~Kt!v^qgik!u&43J2B@N02>C*L*7gmM}xU
zQAsP>B8$-Uaus71C!n&Mi@ZBbrk<{bhloPDp0^zQ$&llyrn7+hJ;_3H$dI}c$d@k4oRm5p@$Yz>cCqRE*}0DB)IcwNkwzk6^?Aw
z>gFJ=L7`(UDQ?F{@x?O5^63axsVe&-6t5!C5V54@zvPrA14<(tSqBBCs7W->wUMAi
zL4p8evh25#7>@W9)--ufB=Trkg|W>C?-9Y|>Nv6S$)Mc|-@_A=fV9Q4>`>~?W5IAx
zSWb9cnB2MDYMbS_riX%v46kdV=w?X&3tHSH%$4#Zf1&wmrmy)Hl#t^uKFMfx!9tbY
z!|yMpt)H{Lt;m)aehFUncY}odYzkr33rd2IC&-AQ6lsE>tX#s&eUlf9Wu4poIK`R6bva@GgrR1HonpsrJ$U3;Zp`@g
z)@moWFHOP(3N1zT9Hb-)I{k#7fM$J@8e##J?C6G@)+tS6<-S%agitG
zALylq>DX>|`~4U$q(`&>O&3tf5FTL*vfYe}O1E;o!&|#5NV2{ZWL%qGQp9EbN@w+R
zd*;mGA-k3c2SlvKM&dH^U*YrD!BgVk_R_9L5K(GJPnf9_aBYYKr_jL_>LgomCda|<
zerKz+7|!0l>!{c_vF{>SClkOWKj6{Pp9A4=df6rEtZ(&DlbT+kY}CJkNpaNTK6fU(
z7xe1M>?G(dhDAVIeRCZTk%@=0A=3HcWK5KiX2xO$OU4Vp%RMp}73H*tyv^;h1wviU
zGAxyPNFED$l=J}NoE-Q$Pzf3b`Sf5qs_B89Yk2{EvT0Xpp$ReH5YqFoZ?B0-Idjdw
zul{j4k&f77%-AxUkD#&&t2#)wFyH~!f}8rdW);dXs@YXC_XUJHO*=qk>easE!#G_D
zKQ?8(7U(Y^l>P2T#yM1f9ZZVA}J{Vu_=!LYAzmn({q?ka~Y
zo-70`sE}zgDuIu=;(ERoE+}RBA~_a{tb+h!JBS#E2XZ(ejP212@|%yxe
zDntp`dygyt&vM&YbNZ7>~dy-`jnV(+`gX#o}!JiQa#K
zT^lH$T=Dv8TWOXyS!`k-8qRl
z%}mV!NjwL;$a}+mKAzr=bx6V3fh|=o#I^|o+2V@8g!$D8YsUIH$O|NG{%KA!sG+4;hUR+%W0JxlL24i2f7!G0h-q?M7gczK
zHyvNoX&6H$|5aa1wMUV
zU+GPJqL5V}!wNRN@~?(7eT~WaDr^&Ub5{QcsecnI
z3mAH&Be#d~vjQ<88F&&M6}*XIy3MJ3;&N*Gpr1W1$(Uml-W#~lgmD8Uu?mZZHm%x2
zVGIiAQRf--!U$qb%L!zPUK6H)pTp|ZO!jo<(o*)4`aPLDJ4XltGOd}Kwk@Ih1$?Z=
z7nH$1yH3eKCBh;da?Pyae2eeY;uX2!Bg=xbnnm>nMhupYh^n&?`4oy-Cje1elxiX+
z4}PdNlxukzHU(Ytc?sd>g0rjTS}~;Ep5;DCrPj#u&P$?~fKcYnUEDeyJS|g7p~iQC>o<#>r56rBVFPI~hI^=7-zEe259j&t(^1
zkJ^R$%LWzKp5|6*N|S8s@jC`^_#7>Ldcq@E43|wC`aM4}c7QCepEvHChUB(8y4>aC
zwHEl`qwg5JTIb0)Tm8^(Q1FTToZ?sL4sT(wIhy}zaM$%E$%ovf@J#;A5vyu%IuhFG
z#&H9s*11My+$uA5b&m8J6MQcMHnm#<3Pm3uf}ik7uiE^)`A47cj!~`^gE7f6CH;=#
z`wki$XJq1q15AH1yPo&z7xT=e9sbz)4dFVEEmJvlU2guw5V2Hi2|N4ci32PyCps^l
z_2^|PXA=823WxrS!c6~X6b|qVmk?55Sy;H*5Sb}s3u6?gj9?aMyOP;0vd3jK>=wOG
zS#COf+qHhx{N=;8J{8~2YwA|k&*5Yv?6V~AqfcGBiZQu-e;rEKJ9V77p7sKbolQs+35J@_c(`pPmjW^`4r
zJ224CLi44CMm<8#U;G|J+N`}Q6De}>!;X>QRo+6T@J+ZyLQ&q(y9i-OYZOD`Xp+zl
z=Aod~9_?81#De|CNxopwrmlXWk)7ZB+oa9Aj8d!Po5S!h8JynM=YxyJdUVuR>Q6I+
zv^=44423ctF7o%Gq||FfQX-q$o;xxtI_Pdj_dUA1!J6pa`$u=>X+;kS6Alr-k2&5o
z(mSrmckhMF3%{u;hMOBbX3BhK_*f7W5!vf4|3M?}WDg%-CiRrVjj5e2&Mn`U;Nw}1
z%oo{E_vuZfE}-b+d_aj8=Tz8~m03?EciQOs!H>l(d8b)~DJv@iug9Ci1APIMj(UpTPEwEH~$wKCJE9d>R`?a@#K>qb!+
zVd*8B*vDfYm7Qk1=?5X?`;Ro2f5pBOtrTML+n`78qdZ~M%CR!!qhTz*b<@tPq+ZY#
zFdW`qVvH=j7OOy;@Rl>$Rx|jgegmu+#jV5($q5jAZ+iSXk3*?cxG;_oH$7>;yNkbG
z4RyT|j3?%$^nbkG2~{!a@4y`o=w4=GZB?N#~RIS#>y
z$*T^v;OV-VF#@k%Kkk0YPBW@|LcVRp0-_COxDJ#yDq%T#2#AR?sbP$rjAEt}*>gS15{
zM(Ij15&R_+TB`N1m@&C_wGB4hswYInLMTTn*U)gG5hPU*qatcGJ$37I3{qa#FY0(N
zCIloy%xZzna!EAV-S-F20hdeiOUV&&4$YsTCtu@@3lFbeqO$5!OmO`7R
zr!x&~Q}bzoO#&z?oMb+UUAESn9ab(9bowxcvw;vqih-TghDhOUWw3)%@Y2lFP7LnT
zQM@bnd8B?=bpYcgo~C3A1;J=G+P6Wxy5mJiaDOqTT-Nam(pUHiW;dCFFt9y>YG8lRNO25D^;v?(3k>-QoaiplOJN)DB~SksLC?
zPkOqmlgsug_Ed*w&tm5ez~a@{fE__q06}oboF%x0BGnVNFUa
zRG9RFYvZD%wxz@-~6rX@d8Q
z3z9IhI$d-B)~o;JPk*}yx?xb_u`8Zblm#OAR5ltP81x66yO){?Ox|z=6&e=ffdo^2hq6sG>37k2Wq464v-%&?xZKmDKRDCR
zc#rVl2Y?-^4~TFBTr`a8m#_gd!B>~kT^I90I&Ah*SYPSjJ0tfUPg7lhQ^s9Q-82Pe
zgHO1&AQi|ExXGGWEH#Eqmk#XoR7}|~#)Or>(ILOa_@VQ@f+*oU(v4s6BMxQ?cL+Og
z7BW)&mkvN#7k6vEwk%trPgFK`QVZV^M?bs5kZ~ZPkA@ULxJ_cI6~tC^Q>>YSPOqerdb5JIud*xi#gMmu-37m0n@lPfjzxr9@WX}UhTO)0&uZ
zDZIX_N1|MkLhO103X9$6K+|zfo{ihs7YXvqgM5NhccLA^`*5g
zvxF>mlEr3YE{H*t_$L>RiPsZk_T}3(gi{<1Fi+Ss@0=#=<
z7$qtLKH-;}Vxw+k`Z6XY?b>QECD`A1zZD|jY&W0`Y;Ll%zUj*D$;VsPuuCu8EVZ5dG*K%ix~D+TsR?tw^CikQx;3j_;#t2
z275_+8Bi$lGM$zJ{mP@RtU>heY$z>LcfLrZ)pDQ+*3^jsPo$Mow^
z#6f1Oe6Tg%m9a+&leq5(f;{nRo-M)!hYjgwiLh4*ZB<+92u_^4szJ6J1Brhf@1GaC
zW#hP2vQ*!RVj7F1NIiU_-VXCZ-I?LuRF%D>P~UtV8tbnLKIWu#rfZJZfh%;63UeeB
zaZ~m-f0uMGER3~};rg-=2hmWHnzw$*%0eM%_*xRnf4v(zb;+~g$QwG97E;;$!=F$<
z{MMZ8=|ofwPEnEK-9=u-&pQsC07a1YVig(+1ZEUe@5(&H#V1Ty+77_Y69Li;Ptn#^
zcf&a0d;$(;rer;cG&onXKFXA24M-y)
z-Q6wHLn%l|cb7;b4MTS;B`IA}QX24KqmhcjNQCdw<8?`wz##4`zmY)>_wj
zopG)5D_3hYQ$6|)efAd=Ju@xZ4kIM7QCS9cD!IUOUU%xoO$lwHuTLi}iD|4p<4>stwea!k#dw#6N?@C848
zH-Y$!af4)`MejU_tm&wI2F!eMuzt~-R&;#PW*i7ESaJo#DtS{u`WI_
zB*vS3<_fS>P3*ax=T(@Q`ZbDWvah@Kf&ocRVRn7@y_Wc7
zX{bjDnU4hU!>-_G#2a$GPlwzgxI$kc_9<-<+&Zh{Fg&OFg>rc&yca^U((&-Dv0-y%
z-BHk>$Kc`}%J1wQ|C_UJ|C?1^#=@lt;2$lOlz`2f2#b+y2c0}@Zfy#d8J?TxRBdCx
zW*~T_64vzbWK;J^8GJ29#v;oyq3n@C+c`4gs-KuUz3+(stY$y^-Gv>ueex#n)+b^&
z#^8u~JIAQ8_|>~TJO+}{b?mOCYiT9o65$lkSO^+@XuEdv(e7o(>n)I+0{LyBH9sk=
zt?n!FN*(eRF(Ek;Cz5Y6dH1Y0MZlLF9o@w7>MVG9XlX2I@*l;|O6WRI8q0Ki@(n2u?~)^yle}l;NTf@ICSnDTIK<2`U)VW#N6NhILUq7*j=bq$>1a=
zKq|!f%ihWygL8q#!{MW`noE1W3oX*e*vYwvcRIadNP8JSY5H-0L420R8s9Cu%sLt3{7_nSj4jSz4-p^@$_HPT5E!tajFA1LwTbGTKZj
zjV%!)aI8%)e8(sQU?2^Lqm{@$w?goxG=>!Vy0DjuT7F$O#VllRy%D%tFRC&<%`P^s
ztGlx8J~vVp
z!BUW|G^o*`CG0evpEU`saixXF4f)|_Cm{K*{EH)_8g9dO7*o*F-NSM{(arL7xq*&L
z(bEGVg-7~gi!zCY6;fair?jo#ofU!Z(Jil%DvDX!C|81jFZV@4JTeL@`}56x!}77(
zX1=%W`sIO%zYFW|_
z#l(aEXj^~JwTwcbFZA|X5Dy5U^Dzk6Yn=g5)OKxgyRS$#;+2;zG@7$N>zdmohxKadLn(@R9w-xwHW_)<{HG-+Rm?;MYEF>hT^@S9B(|R^>5v
zvV0PKduKCiW0?pGW1Je|*|yowZE^jWR!+b32h7
zyrfJp2z>(v)Qx64Kgl2xMlh(Q{~=sa%-|*XlzfGqqpaaUv_Nz|+oXwc2AwSmDSo*?
z8s0RU6q>zq*W^+8vI6jYwC~CN1z6ukWBvyl35NltUujqCsuWy_i2PKt%=SBw(;2E==3VO};{?rPTIIDQQ
zd-WqV^b=P8$KrD`ZooHlEwFrM|6z9-!t%<>J{2c$M8DqbCL{F9t;=GLLP@R1n$yY6
zc2@fKJ7>?bsC}<4<{0zKUgO8ZzC3L0wU|YFso+=D?|WhQV`Q+tY!Dw<6gS+!Q`a_Q
zw!rmd>A1yz^R#ld{zdKjun
zWsU4xLG2zJ_0l$GGdf$zDqMnIZMU&nJ%JbEBrp0Q?JPRG-(xBH|F0y}=V)`91QZup
z?VQdK43{*Aul5~~Cc+Ox1O9dNMa@hPyk=f8FCyG$Vo-VIw7KiQ8Q%UB`J|2HRj%BX
z()WSdt{9mFov^MZUsw9NS0(FovlVC#-WM{Sf++IRK}4oB<`V
z20U^^&axWr+C;o5d=aSaba~)j;mA$(*V6G8AyJ30HF)u5F1
zKNzdjOA!W1Z7*TONsFDn4zqn}vt?$EMU%`NEj*KOi%%-SzE1I5?l+}~j?HknUL7;d
z#$ZXD&G+)kdOC`G(DG@$r=R)U)F{#fsqZjTQX=a(|zmxWB$Au9@2>
zUM}Ll2xFCh3~io5D-+J)=$*8>qcXSDN|gr$D?QHKbc?sS*ka_adTbmq0LgEK^^YEj
zjU+(LsO*8Hm?lLupi-tgfBx>Cnrd&B*$2}MZKfEz`sdgvD~rfvGsz2Kfdv)nAI3G9
z5imaHk9Zt4tCLb8>%Nm2d1%ni(4Zd`Utu>V85$a)4lYT~HYm>6G)$K`OPm5c-_3Jg
z-?M8Bl?iiF%fwLj7(Z^Ut#x!Uy9%1{lE!6fV((=6G6M6kjaX(wf}aTVJTzb&pnr$+
z2m0q(%9}1NbYPWl@OHY;H*yZ@r8sXvnYv$i1Ai#foMd86L|IV28(=_qUTy3ubDX1!
z!4Tph!hE&yN^m6=-PVXwAu%mGwnnJqq>`fi2j<=IvArIgd=L+Gh$LQG42te-B)v=Xw{tQN`2^lR}Xt
zjlSvdXi>S?fwQlFe4cA1+?W$E%GExF=oUz|FjZR!Kg0W&WxeKhuj)90&!F;;u`3}@
ze$@0;;5(VMlXzv*qb6u=1MWZvOV#}N6Jvb+cIa^;{T6?hIT^k;yHiWr*AvNmg1wu3KC9@
z(`Pv34UYb`o#n~YFFHX3#GLN=ab$##n5qBd_tw1N&RG>@xS?T8C{&nA9nN@=M*Cq<
ztW(XK(g_>E!(EN0xVMWO37564@>pLgrfq@QS^(Pm?)?2$A&Fvy4pPNeuk(H69STJc
zqDbWezmk>VuQeUjHN^E&{P!DrvK7PsaiWqSY6v{9o0bKMAR@#h368Yl`@fx!2oj#hXh}9X4-=hO7%bZyvl(eEu%Dk
zU!8{lbr|e}CH3Wah@z<`GRWaQL|YkCXqo-Vl*2`jA$ub2_E2RT7io?ju!6=gru8VY
zEM_HZKO{$}3pZf?8Fk1-=Yfn=gsSDkBg#;2iEB&NWCm1@3c>jIazxVyK&_#!Rwx=1
zsAJzmb4c0cGV4vmvAVL2C90+xqSrqzocwCGzY@VY#eO3U1BF1yi)Zv;d{*#$Kka{9
zqjQpNm1j%VP=u=8+Igh8!xadADNJ}*QT$3X)O5%>!c$0P(zeS*)l!1<{%fm9=b9K$
zMv4s(vxL+lBKfn83tp%xdV#0uD}hVIQUff;qtfYZi4V$*~fj90tLcuCQaM>SA{vuu!lRg
zL{b0ORXT8T2*Zr1($6$>!%TQ=#*Tsp3hvQxXC|`KhO!R*>yjBBvwtNU981Z4udMy8
z_6_YnB={aiFjF6*5>4QT--V$&tp7NkO-@jMph}wKt(-T50iV@UdAUk8IrL3WY@+Sk
zaIrr1G+r^Hu5e728G3?HFfaTdwtfTb8M|>~sZIq&i%g?0U@`g3nOVujKdS;1G%DO!
z5A+kI?!A{;KN>iJ)?O3voB_6vcd7rEmdt4DLO1y*)W5w@n0_2xF#l+-k&QWuPW4F*
zvuFkBAG`x5Xo&Tzp+%hc)06|1aKYBL)sSo{1ht!R2#wPA@^AuEv}slA4Sk`u
zhsshP!{Ht9T|v$GBNjj^A4DwEMT8V+4FI
zQp#8geMVseCs-%0Wz|%$aqy$L3
zfUnGZe>U64eZGTm10%&czvPNIhVbVre84N{u
zidK&|5Q02=o3RYC(K56UeT-7IW5PYp+29zi^=v!&9bfub8X6!3pWK4u>#T@1CZ690
zG{OuVIl3i5rkT@ZlMmv6Zw#_
zhT2>$y@PRvl9p_n^zIb=e?!nanUx5|8HYBVpuR$35
z`0M}#mRP7q%Gx$GZM$_VPPQpxo@DMSMxSJI3x!;QuJ#4OKlU@JOl`Q>l+U5To*sH4
zN}wASw!N51aGtLPHcMW_K^Y??+dR}D%yJ2)6Wf6FdBLK0!{AvW^DPd1QFDDd$Z?wm
zcllal?dy?+!lLc3nwLFM&qX{6F_g1qP@Els9UOor<=w01IyEc0)J-#WyU+Ev;)h{WVp{TFdi%mx
z_#Ofdftw{iU=^!tMAv8tzi6E@>##|2W+#x5Bai6OSX=D5CBDrnNEd%87CMU8!M3*0
z@b(~9NHHEvksx6Dx9F<(aA9Uq;t84~cDlH~1EE4{ayomI*RkvZ>wxS*5`l)uwRx^t
zNN+a#x}mNa%g5sqRtQEW-bf(U;A(@l%a4+<9gbv_wrV5orI;rmqB1dd8Cd}|R=5oP
zN5E0|w}ma>eqtZkmDedatl7W2;hJGm2}^-k7*Ar
zIWyh;)Brs#_}_E@5&alYBPmJ!9_+ILLUR11oeLXA3b330jZEO!_G+AEFh?@0C9NSw
z%JWRpf0m&EBLE*!u>L+0xEZXg+J%YT_5Dw!iA+^|701y=?iSt`@7t~4>M(|f0V
z@sKnk+`avV6Ak(hQq;rYr}~urRg{8xh8#y{%g0EH3j!YF|+9Hi)Rc|j5PFpr@8siNxAk1j8N
zjiqHt)yvB54ouX@6B=U8+wW0ZNCMVo
zyLI7os~=-||BD|_i=4;GLMa~I8f0H_(6M`(+0TJtx>Sl#*It4Bga@f2R6dtnNbfN2)e)Bbdl3%l+Fj+x
zK7^ZG8GSF!TL*Y2Nhjy_Qf!Y5qS2FfmiO{sWNT$!Nzkv`5=kbZcE-
z;A!-C+xN1CnVe4E$eII+Y2fd8si>#+3y8qoz$FwxDmXBHOK})+Gp&E(T@l?5eyUqL5Hl$BQ*t*s-ON^J
zXaszqwpH`E;DE5-RTa&TrfIu<%V7^9+ZjLYH}rE?OVU=W$E;_Ep5ACQ2E#FQ8mQew
zex%9)o;#zN?fN&JzI>z&wDKEF#NTk+6D&FUHc(Am!gCM#2*h$EVL!
z_=JDhbMwxg;$*AV6JGy(cCF%)3U!}F?Aj1#jje_T+VWLpyDj#)?hLNRnc%W@+q?<^_)MdZs839AhAkW6;%6Ys!
zK3>{Mnepk#R!Uz~Z_G|1IZqjLt~Ccq_PK34!tb)m<{8h1@C4;ie$?@aA?^Bw^P3hX2>
zetYayU0_(OeYk0#HjWZGf>gV&x$&I0Vt~hcaSSNyxdS}PQ$mh8a3@9>w(t^^6DAsr
zj!;VST<42tBuX#0Z2iN9b2CBqFt
zV!_Aa>w1W#WIbCh$)4Ot>dh6Yk%-ac!QPo+Bge>eD3*vZGCtXx#BKZlY@%W+;o+7?
z-{2t0o+0tcwC;ZzI^{4}%r=pqi9d6`wUHwpQY3(~jI;H3Gw~9Yix@lC|nCL2UU-D|GH(;YV)M)>n5gqhO8)4h7
z%@L|Wpc(4h<%R#*lX51%&*Y1&JClTpCTft^q-Oy$r5B5e!j?PO9F$5)fkZ>MsquDr
z6L`%yTEu=g`|-Z;llho8%M~tp^y};2ZPq5-r)fUHALLgnUgeB@>JtM+Dc2En60L{$
zw6vgqf#)(Zes*bSpy+^C0j!^PYiWsvX39Nv`=B*<2|it#20PfaZB$U-3_OpgQwDo{
z2>6B5wFt02nX&}o4$?)bk5&=*^7+Z@qeGjgIRfku@VT~@#c4l2IF6vH@O|sxw?q1M
zrpFEejrdi1Gyw(?v@)>Q#7-QAkP>*nIwFs;iOe`1F!Br(wB7y2IoYMMC=;WVjI91>
z!jDk@>)d46Q|XsTXkPQO`wQid0-IA^JUFj$QY|!2v|Lb{W~M!CcpHb7~@N91u=F~E)8a$$UIWNyu1$(j3Srp4M^FFOnc_1^|U$4&8D
zUsA+`4y}oLFz@ygvDRS-FA+q-@LvaLRATMN_`~lRf?iYWt8r>(J%v|*>`GEww?~kI
z1Q9$gCqUG4?=sK)lNu|ms7tivKkEn&wg>;{b~-`-0mMftyen7wM?{OABKgNWkR0Iq
z0rrYQM5U#yA~+X=(1F}5)HU!WKrw(@3;UJslvFT`>D;A<^LxH
zD)zsQbB}!PP@-7LzqZQ)YWNN}@;ozf#6B8SF%UIzZmO;15B?CFiiDt78*xwMBsM~j
z8mifYNNTJo#_N&TvGY-sIS$EnJ3+!AQW~L30|de~QNQO*jBUtH=a#Ypsnmp#$JBAK
zimRO5Jdm*JLZ&XJ$2L;`i2j?nT>{Rht|Bl2v334H0{XVGs&N2GBYY~1I>)~0g~N&g
znRa$UGWWZ;NUbZtLwx9
zgVr#E1-xFjpK$X5K{7M#+K)l+@hsOVOs>rM${beUg-bf8O}WYIMNnjWGP9qjr*sMb
zwabSq?^fWzWh@~@e18HplEBy`!)W=xK(?cjr7u2Q>n0+
zXH0d35l=yn^izi7b$jQxh;j$M4!#1?yzaSaulr8Hxc$upYd^bqjONWclM&EXWSfLv
zG*rZBC5s$@_IhZfVCrpse!F9Ay#WffA1y5<@k2I*BEm<(wM{L5-;8%s*Ct#}
zS}fIQjbRghnR05)y2{t_;$>hIEza@wOyiUrF~6&0Nw|4}N#$cpa8Mp7N;G*<2lA#2
zP@A8GH$YF|JkPmt)NXspCGrXDXfau*0$6f^Sjm-F&&_&UfC)2Oo3T%6iy)OUSu-_;Bu8pQRZzmUN
zl6PayJgVtC{cOR+x6e)Lo2%D%v=nmQ;pK=l_{lq;OkgpsO?isrA>OSvlBE7>Bm_Li
zqkB;LR&Sg07Yg@1SRRJv7WVx5us?2J8=B26zTU2d0gG5e6M?nN;VfuwP0nzn*CVLf_=VpxL4d`r=EoRPHbX0ZndWhn^I9swjSCCcb{Q
zRwudAh!yl5rxlrF9rTkxL)}ZRsucal8csor`0JY;f#R8cSe7?IV%aIcAa&fR9cNdK
z?qZ&-VKs0aBgs_G@)8N}WgFLewYn^JY9Fw-;Td|ULU3ZC;gDNUL^xQoBpK6TDn`YI
zmwo*Y(aNv0X?&gJ=*evz*mw#obGlIK^3A$af$}lp-W8<>nGxFUvbw47ccKDc{Wa?e
zf{PUgiCX;Sgn^Rzuh^d0Bj9J$12BFYj8zh%&=XbDn;SRO(D2+}f@H^azMrDmC|6N7
zv46r%fb(`$8Lrbr&*lJ06S(WpzOECqd|6)o}cJ`*c$nYa(iQ(i@zXoi?ihc__truJ)61ssJheE6*p7JN1fb-
zc~jGXuIBOuKp{idyd|E?j=vOgAgCcxOIFWu3xF1sc#E1)58aNRT@QkfLBzqr*fd2)
zI7QcuPH~`t3bld&){ro43S3ns6#IFMAQ7occX81PU}3CTC(B+$
z{b`Zt`oYI%OI@e=_?G&U&YhJa2;4-opZfdUyZ8Rd)KH7|7st5%A2`Moor^Mj^r>;B
z&tZ*IvJ2(<$wE!SX8E~@8yF#SY5H()9&us#Q1HSSmO8niI(}IoEfGKVrbBU6GO4_X
z`yFKz@Lj)?lQ?freWV%g%ML8Q!{cM&e1C|(_QfHjW&JGA+sQLa1yFyz;rjZ-P^NmL
zWnU4~s1Z_Rb{a%YQS@*~hMt!Fq>1rBcit?_;8kPclER8Y(_jz86ykLK>I&O{!It^g=O!ki
ziGv{PqvxqJt(YhM_K&_<4=Jid$3w?K2m2i=JO^(LT~7y3xGTw@wFlg(ioY(ID*j+t
zNfst97H$H{J3j#Bdwy8Y9K-0U<1*E|N8f5`L+n{8oj`Z^X}f+Xg3x+s@$jHYK(fgh
z?&?7Xu{1(k3}*WjE#KL4W^D*pk20zh+cm%UQUd37kjQCiipg99eh@GqD(M^P(#Ixi
zvv;=w02{Rb5Oq47D)9z!Mo5I%buQ4CQ`r^8>Sz^}fREwDc{GoW9PKcIMCCgb8W853
zUk4&1#?=jCg8)5
z0I}wjxjA-a^fX%C!1wiCjpl4h?0N`&bw@a#kyG_~WzN%^I+w~wq&|JUR337nV6|TT
zXWJ&7b*@zbMza^xua|*|7KC_Qs?7B7Y$xa1l%>o8r}R6iVi{MN==W$pIzeK9y`{Z@
zR~whq!qSqCiAe^}d)_Xzqe}LS&vTMZH4|==01`Tya)nKC!R>yW=gaF)b7FR&#Zlvd
zU_MfB`@^NE4XZ`KbAydxhHgdh41Evb*o=PgjliWz)o1}4mkj>U-tT(TuHH31@@QkEd-Ok5Po)X|pUCo?{}ow=sHS#)>dSpn
zHdxRiV#q(Rf5bG_(x2l>hhw-Oen(>IiNfH4VS0W2lK3(0NZBH{`^AyRUS4*)Z7v!E
zLQWJR{?eaU__ketu;On4_(+X}8gcAzGU$z2*f}3CR<7-mH==N!YV!S57*gw%n17H7
zH@)+ACC?KEIY;I}e1S}A5WcHKE%V|h+Ogkb*m`d|jSgKySwTCgQv~q26RF^1^WM+(
zg>(CVmuy?42=JO__c-lM+x1Y=n+(?fIUD#OkG+|i`6UbR<6|PKw>6PeF00_p^8K)h
z4Kn~BuL**b#7Zmj-jqGE2K+^HVf)lRVZn$uT$pjnn-Vud^oM3rn{BxXy$|clN;;Mo
zVZhjQj|JD@`(qXrhG1il%S{o*JEZwF@UaHVR~WXXgCo;|VuHC!?_
zprL}WWErn)vY-8c5r+gxE(CdU?mzFNz{~kyk%_-Pww|Po3*H{517;W@`
zGVHV|9Bv?IS#l)14S9;FQo@n&4PSvZC-}V|79wzcW9;`$2|-Y7h|Omoae+#5l+(c<
z0VWa;{mh-sRYUS357=c6tMGf-8!_NN70|dk6Jn{fXGsB(5f{2f<5J+z$E?coP=Ik;
zB~&}s@)$dUTO5!56io}gC)rB?0Z~pv>Gt_ORaAS>FqJ6MRMS_#KGqu)>h^GtaSS5?
zAI$la7%1w~lZwa!(?t|lE_R3TFJEao
zSpPhy$glpMyyhrl(HLOGVIq{4q8;PM8jrIRKgXdT;A;53o5!qYSGrg(8jdcVPq@fJ
zfS|?ujIxdR$!GA}hqRR%0D%!w0aBWPAj!fQ5hYz})Y6ouZV-OaV|bD`9m~*-?f?en
z;cuA$HXF8KAFz7~krev`1z(Idj8Kd^vQPT64?3B^#~C-Le}nOAdG-*eU4ls@SwNV9zzmD6d3hPH;n##FWTyeC#Q$WJNdt}x6<)`HI@_}YKj;*b}A!-GU2
z|5NaRbHFczZT!_kG?Wmc$Y%KNO#~lU@N?nGQ$B|mbwnqU^99CW(Dy7%TgeqKS%m*Dt015
zKjv`KxKe^3(%DnRm-;NVdWao|B`Jxy9hmz2CyNs4pw*Xq$XUghyp%E5=ivQYO&W}V
zpXUs5VWNn8!#@Q*>D@Tr#U!F-@F28NI&M@j$T)J3l`I~`NHJ*h^P2iq-t_Y_h?|ZU
z!EyPxJjZJjhp+uy0R|fVR{7HPk{CV{K1T6g^*RX=9AFp8F_;*M5pIw5y{`Qg^by8$
zwQmdu9Q>w6=d{0ST=6gbsa6<|g3OZTJY+~4hJ_uCL=jiEGie+ikqfz)-iS0&l}
    &fg>O!E}80(H5j;rc3~4Z`R~@y`VCjNEC62a9Y^ZsWbPiI24imS8~yUGaCeL|)mXDlM}@Occ{i!s)A7;5qXNA8
z4)d9kf#6nJ#g_^GnVMKshUzuQr7v|lG(Hh)TFH+`r;~PSPfg!C{~=yz=j)WNgneck
zuUg3t8FhOx#WRE$K$&-P$FuEwmb*dBil+>8@)RVTz&4BZj}Ju6E^#j|nSTEKA@)GL
ziu7xN4g_e||EmK*Qt-kEV8ljYmr8q7R+fU)Mz&91SRNDu9pr4f*9vrw#k5?5v@<*}
zwOvWmLDM)ou`njgq6WsN^iI=0_f>S+K#={E->3qg%U4BSJRM&5Zf?Yrpd!lAxp^mQ
z>EeYbB?MOjsRsa$XYzN-jeY5K21AsnJf)}5^L>Y`s)9Myj)zZ%+nhBf22uQ__WKPK
z)_hA^X9fzBQjmvH57W}#!7Y+E=Ji{nc2Ly0;3WD)1C${1l`Z936TPbO*rv`-6cN06
zmB|kC&Xnh`V4-<;f%#nGKo@IkBQ7C<-n(W$U#mTQXc0J8vZCi|+-8T(@A2pZoi^Dh
zKEQ-LEQ95qh|H8E(q0to+s&285QIy=QVLk)!a#>FkFD1sv)xp$+<&CwDTeHlC>gcV
zOQ*&1WV1-If2^hd93k#jaHxHu@at9(#NG;IA5H#(ZUu>dcPrpo17OTpvoX+FFSbl2vQ{Tc
zzMGfX${GrHFR=~s&JW2C1s-Wj8EjGC%Nu33lCEBml;g!zio+14h8c12k^7qZ7$zp2
zZs+OJlQQqrJGNU3+ReYOy=f$CWX+71ucn^Y^P>}eN~o9UrE&m^nXe
zA;#Sw>oGxF{oZEfz-cm4G*IBM5#dtNvy_m)HhbC;g70%IlbdGM{;6rm@>_?GVj`J0cv|qNzfmod*=x%(4UUJ
z&i!O#@>j3=184xOQXZ#4hFYrzxDIlHQI%u-O_
zmI`^ffkdxTzSX|IRj88HRSfrLvb=Tqh%wTrd@4`>Aa139
zL8Fq3m2W*7HW*cEX}|>!g(Z69DJQE;$0cTs=@x7(=eopMv<8otXq2(DB)Wu^Ee|}|
zj~ab~gcs9*?nAIy(Y$?jufSVchLM^Ty@`UT51efD0K_q3{R2qAi}zIm2S!w0uBS8U
z4a1W>kejbJpe9^M4ve41@zxsTMok;
zH)uD}q&*&*(_WL9EML4d^&a;hmFpYjr@OtoJej{rl99{epM>7^!|!)BBqN-W5d(?O
z**{Q{{^cT{9-m)a0Ikn&n&s}4WO@SytRzXE5C#YQUa{>I&li23rG8|Y11ALCw
zX8Q4EY?U5{LAN{hz^TvQ-R8|@POEzrGinv?(a5wN?VG|>p|oo_t+Zcj!L@j?Uo>Hq|;A&
zX}6z3^l@^RN&z(lmICBI2L|{=5Pr>f++JGtlyEqXhYatbisGT+mAfDumJ@+f`>&GZ
zO3)|B&1=3;vbSyU`BR(*=*J0{iQW?y@z9=0`9k9B#!`?3F133TS1BPm9(bi9eH==u
zh#yb3YHt_}2mG2y8qq(l0kaCY_R9@?Mg~xo8n@F=&`yN6x^FGU7X1AE)+bJ#B9z>T
zf13ly>Q(!Z{6EV5GDEI`Qb}sDU#!aKufBKZO&nt)hGaDS1
zK>yK$V|()vB?%rIdAAtDd1%+Bp1{5MC`Jo_p7WDI-PnCEi{}kFM
zW{yykymjikNu4D>F2QFH1h8O5fp6nua&k!q3);zp<>xX5#Jhsp;z?J@@Fw&=(>z~?
z)|WgCx95om$)m7k68NDUgpMnQZCxcQ1+6Bh$bY-ypqrX|p=ABC!d|OPNBS+~
zIj`FQZF=Eado)a*E~xPYJ`rE(KRR~!%C`8N2t%k1zT}J?xe4yD58+;)dTK@tdVP-n
zeElTRM-tB1*S~<}wtSX)2huk((lfgwhIE84uaFbaI%L!HVFt4RJ2JhM1FN=6X2S7W
zl9z9(VYx!!b>9MLqWK1As7n~AeJW|dk;P%500;)d2<~iP3V#v}t*)5%91Wz*L}W!e
z{$0vf8{*h^6ds%-AfP~vdar=16I)s(w*84DN9l{!T7zxii8x9ih!i+(s}N}3_*}l$
zn(s@sLYBJQ=@qHg(Wq8^HsnArRC&CS+dDG@PU
zd$UCxjI=>6hR@be#)$fmGOzm%q(`Z>ZF^>7V=3lhlbt<`z44~5m_Q;3pb)hYo1VRw
za$I!}OJl3Bldf#q09P*6k?JF_Cd#+$E(?!(2;|cOmu$SR$Y!H$IAliRfW?w2Zy(Cig<|6x^
z0e=%orqG)@uAg_a567eJjdsQjRZk}*g08Ec%yN-_$?nQfgUi0se$6;>)ls48>|pP_
zfFi4+L&|bI=Sz6PDoh3sLzI&LH9EMBUf}0l0lRJTs-L;4y2zSX;G
z)w%mmNxHTawzGQB)be(>UzEDyA{drY$
zIj3yPk@y`qlO#81eNtLYhZ5`D$dz`1D5{yIc)Prmows~sT$oW_*pt-LUq2#ZKdv1t
zaUhriegU3Sa2$D?4U7!<2k>kmXo(3vJnNzEavfYPoF^Or|7Fy%R?cq`^LWHjb08)i
zTt4d#1y|M`(2Z0Sv|Qgn+n#B6Dp+*NJ@w3mv`o(ddLdGeZ2OZgVq!P2H;xD{q%7h>
zF74E{_YkXkBK03^iHuY^dnyTDh`&E4_YT2dwt8oW0=T{3PC{s%afWk(aI!Rbjq=2)
zbA;Om?V%5RV6q^fJOW~uUKfbBed5%!<5G*?!zUBEdcdDhLa5ZbcFNT!0HSJ6fdm6b
z)dcRU0|}+n%NU)tP}UK@VUe`-DUXuXNxoV3d0xtN-c1+h!$4WMQmb(daqE>Y?m4r;
z(TkprJ7&}#eLqocj&9w1GX<7>9kc-ncLsVDDqkWDxW7hx<30KCuj*(t5%v^&Y2CJJTlwMSRdb1Hk(cCHwt_XztO-z;)B8k
zC)u{B@XNmsfgmgP5D1+A;}AgLRvqZiJD3kik2=+6Wp7ljEgWvL-yEGDL2cVCIlru{
z96da9-iTn>ZT;?$4nGI1hXqsYO*XbI52{4Rw(4bn)!Bf?`tf{XJaI5#3fzmKr8(I4
zmHCW)s421CI+m(W_(!K4aB5WN*}3m|eqMB^XpM9IIB|Eo~&4=0=<
zVmk-!$yiV-{Jtj*-FU&SrjCtn$d402Z|89NlBVxX>C^C*F7McKVmMU4zvr^Zx>cJP
zHM{h7htH|1tzDOatAPWsMp)e|1D!O)>a!2UlszuUzQ{1GDrN>7isaAhE~WwN$M+mHMow7$;h!gNwN-y;WS
zr@i+kWvcLTVR9PO;s1lesZ{TUHcuN
zoG?hjE5kls_VlT2B9Nmn@EmXLt6*f_xARGT)qz7S96&tp=foR>n$Yr;kP|&
zSOv3%8dFJV+NoJ7vC{mkc4V~Z)p
zxc(+OVF4wh+gpP2BK;~OoL+NRf=!tB9AC(
z4JiP*BN+N0=}*uG*!3CnsP$J2Kdy#1dw5%(5#YcsOp*gkpOf%crnyvvTNtH37iORC
znqBtGQEIU3q4ejv32HtJV?hEd?*Q~IJ5k=e=1DccfD{lJ-Mc*GvCqyWYI?e%iSO`@
zV4cG^O^<@40_FnLzB(J3)q{f;uOvVmJ~jF28oBH#XaS7kaT$75Ej9zagX(+@T^F!F
zT8Ct${c?UEBfdd_t78Gm&?DQV-p#hp9~9fp)b@1Q`A?U9+q$0Me+NK4%Ev1Qq*wo%
zhK^miR#-Zh-6I#03=#=Pp?XKrvQ)b3@c!H+{O{%J&xw3kc9B?6bP&G&q
zROqxxO{=lU-qYa>`^mteK&p;ga&7PA&Gw~Igx<30)%jXn5;hL3cG^_iJ$$-+#MRGg
zMm{lW43q9+Rb?-IwGOK(UUG>V-W3wBB*BFwv(czXlHVbz8W)nP{tQVZXZ^gTgfC^q
zvp+02<2g^c^gx9OkWkG9dal3UaY^K3<0bJD7flRw9pxBzE>!pj5TuKTk_1LoOpORXqijhsDp%n*aZDSXwP>V2BFa@aYQb
zvBM%qJUvmj4=Q9_h*6=*hZCpyd6$px;Us%I;=
z9-y%Pr-^4PAlvtgk=1NMGd9rXDs!+{6BlxgYksoI7NpvlXybjs2ij5_Orp8>PlXcT
zzoQ`kz$E@gBcAW|PEt4R(KhH{eo^)1uOf%i){*~6TdT~-`_2YKm4>!A6uhE!C1aN!
z=|YUE89P50R{-uu}p`txfCunGE)w9k~a`
zOgUoFxf11WKvog4NoQw!!#6V8X4*d^v$Uf8+(p?_|I2Oi-zG{Rmi+?}aw<68X8TnZ
z*8*i4Pw$Si6Y>AFQp-y(_nSt4dDYxID&emIh
zmvaksc|S$YZYG5`@!=*bpkvmy`=jA0ItR{q^IayW_n&gl
z8}se)kJd~xMQE*o7#FB>s!dt1NpyD2*Q@JjaFgi3Vbm8jW*SS^Mvu>fdHv``;zm&+
zmcX)?cWEMA>K9~bR}PWnG}R-ayyFkZGCG=PbXs>v$K{n;x0Z@L?S{+5IpnLdi;>;!
zXO$_cXJ=?wHxNiZvFR0k0K^YLE=q|W#u=TBU0+clb#sDmp~QBs
zZe9_&GRXHMs|ydv9WP*$T7iE7a%!auB7R{#3HksQgaiuJfNB|kKG3Nq{)H`6T#zdt
zlYA09e6yjV4{9_JK?wFcvDS%eP>lyL!Qbo%P~&M##B1IzAMW(zcHxEYi2mUvRE5)R
zcON=~o$ST*yYq_9JgRITRxSlSV`a=-3I6Lc@09j^zH)M|iUZeO<=Kktl-h{-UAqLG
zIp-I#g*Axc*d=)4it6s+&rQ7fL3+yN8p7|>JNFnYV3F9ixW`p&!n#ZIN5TpbwoDvN
z%!d2HSMT(haDtcrZK5vUK$=^kM0Lo~_4l=Qw4f_!lxkc)6n!LUMoVy?CCjpV6Sb}^
zU7N7>cogV-`1*oF+)t5`3T747jT*UpN}`x#k~}?0@J#UspsAoYuY=wWwZLKql0(A)
z5>*GK9&5)Wfp&~rC^-A2ll4aRRe-8$FK=VAnd9)gtlKq06VwQh$rG;FZS4@R4=WPhh`r1ln2qfp%IkR{
zp38(Ixf1Wme%K}aQj_2{5?f-HUOoYYhmt4F;c8mFiHN`hTalD=WsV>^rBdPwb8FF`
z#PRJ1scGGc>)Wt1Dpi%Qv)Sw4d~Myscj4w>^2I{qr~ZV-ok=EN{>;zw)PxeSY(Uq2
zU$FjuvS%@6a}WB}@UTiE`Odfot?%0rZI
z^_9@4@`$E4=5qH2>1(@YWB~2Z#T>)7m?LoObiAGPtM(-VJaqBFMORS$Ngz59hxy7o
z^_@}0%4xSIyc%>wq8;wA&1;Mxh0*3o!;3C`u_s(mw&0qfhwr^ZC`QBHaaHaTa65dO
zd0_cdkRQv{so!|wUi439T!zbxAO4vc`_KUlcIJV~n<%Pfv26NM2SqBr@3aaVtt5hK
zYWE243m#e>CRuiCfHFM2Y11mn2JZFQ_5*W}lhA1~bpi5g=n477<$*yC=C5wuD=&OM
zWBkV*LaneGEcj7msSS=UL5F{7>JbFLGKk>18soa4Vu*7k=J!+WInOnK`>1ERhCARA$ulW!3uUzJ9DoWtoAN=;_
zjokBKkJz{TW}UOjPP3G%^i+mGW
zYofJg7a@deKI{|qGOA%=v-$#iL0Wy@SX2v7(&|lDXV0LV$`0)wTH?ecq-LGP{`*L+
z4oj~Qz_+I3A`$_`|Jm>3J6t#Wt)GK6&e7MGKjNrAAU~)r&gk0m+pFI|Tk2JM(m{!Z
zXVee}-{C)r#UmUk>&BmnMKW1fi#u;zpp&Gvq#8Lxo^=wuwACU!(-OoJw8w{XtBTBvIG1Na|^$uJnQKM%;ddIACoZe|CO2(JXY2}-y0?}?z
z{k{b?kf4gwrJ*i^@c`oi8%6+wKP~YO1}`*79qeRXV;Qjf1WFSvL3>WUu86;%oc76)
z;@Dwf%+f%jR1YK|#hyQ){xa$clCe6wdmVNi)InLFoS9Nu$tj?FtLK>CYq$>49ntk@%AS!x+w|FafWFUy=EW{^OUziS?v1a`
zrWdzenl9NRKQVF53&I8-SJ^C;CTws`U^a>`QBPQ;>)xASr=H|TT8puPRBu@(+(fmi
z@>LYa&*&R0tre-b@qbabsPv+a?VNw-{uZv+RH-cyXZYmBwu^AJCoejtba6;PdYJL~&UcDv;`M
z1(H&klFr5gQ?I5m9R;7B38EVC_)ojU1@h0@QwSfHe{`0DbHtVZJf*$XUYSq!LpKlC-)wJNb
zR98NAC4U!PmFqi4?L?uo)*WDKI|>-XcfmgrRKA2R$WTU&oR6Xn=yx{dA=STYK{=|w
zrkcO;=JI)Ly8fkJr;A
zQe5T4w0w<4+X4v}x#uye03zi>hZq@jRJ3zrO(m92s%;x?O3u^i6%1eo#_WGk=>#iM
zv?m$3=j!{uYJY&LtJt}b#lxrZzqkO|Yc2bAKA<76vKN;0^@WQe;b$qC;{N9}|D$df
zz=(`%U@NsK16-UsI-By&%~UyU`3(K{oLoa#J`dbsjdSV1Y}IPlY|oV2{;`*kviHg*piOLs;
z&|Cn$sFl+J{FHT3ctw|6MO4s3Qf%i-)NeNO{%Q_+^5zd
z1^uMLfj*r%HX5!lc(ki|2DKd^3X_HuIKPizGl=IHovacLp%H8F#ujh{w{>Pp4BFfV
zW=u15t_^zX@z?gBOhFDkToxY#p!5x$eM<+)QXVGgVvt*zd9VxDdd{Dc2
zUcB)aow-~bx;g(U4Kbwzr`#XX7*c^YHHfa!(#83TI$2TC{nTSmdD2~Y9sN&kFdcbo
z2e_#w@*!0WEsHI$IST4S{LzSn7eO4kOG}CMl7P-3V7eL4n2;wGc(y;}Tc7uUR=pl>V8cj`6ds5f5E-Pm
zr$7pQtmqEA_wR}t?regnwo7BL4-C4hfn){S!5R(>t_5)YRt=iwV*k%AqKc-}iVh@p>M`9EntKr9-IU
z)o*Pk=Srz061%IOt?-aM?4IhB5ftI^E!;*ucT#IUi%1mB8I$tTE58DB{8fmRg@U##
zZ==PkoB1GZKHqeq4qiy^Ooh6QP1WF?r&^*aF-wnVvFE|h!{k%DX*#b7oJ(}DTv9tx
zj`zmiMZ~t=o9(){2!%qwBX&JnGn2*!=NX$-DK
zr79l_O{Vm5eOJ0a`>rpx7_=P!LcMevSRw^(PBa%53&K4a^{ck{Ax_i7eAgEui6#rj
zrVtJ!b`0ur8%;wtdckXcTX*j0yyzd;urx@y9-@QN15oi{NFh&36B;po%yGC|bB
z4iDZj-%7h4uzaKY&LRDAv7g7uxf=
zTP~h71hob~h>UQ@kOwv#XQJ19K&FlKC`}3KYrwCg7%)7Fx4|WRJ~>iK##W>C0~ebPW90
z%|LC0pspoTCg|i)w}ZBN)gRC(cB^~$?-h3o7RD}p)2kl63qSa#r_;{&29Oq2qvzZ`
zE!MCqUq9c-A-Y)i1CN%nc*l**XGcz{`(FbmwkzfUIdpRoq`|V)j9BM!$z!-ds_(um
z3AWP=e4?;vmTR)BCZPY09Q5Jeb$TW+b$+b5scmhxBoK*Z0>xu|J=0*2lm1Jj7x1r
zqvVxXSg^C0G}ZNzmeK)$n!MySCT5P}k0jIqqyTbBPY0kJ8f#-2jrE
zV_90I0WJkV|2C=vvNS^1P*m1c$7)?e%a}Kt{oQ<<<@BjWh?-A*H_$SEqb5j^tg1*|
zWK21&dJWIpH`>%fuvL<~peo*IqT_{E&#z52OCKmDvJ>dayMVW`eQ-^qv(q!n%p)-|wjc5CJ4%Ii&iFxBMy_r;ujlG7cr9&!T0?UV6;TYH9^ReCK
zMZi74IgG;;xyc$`6_2|87GCrG2Vh0XQsIt(z@R`|isn>t_&h>TRR^lZTO&lwq6Q(2
z?&=#%&jSOQ5RtmCgcC?-P5n;fo#JR$A}k`0FcEI)Fig
z)CP3N6j-p(rjYB*=kd<@I4{4iH0-kzK*_aBmh1_fu-uR7Y^${?h2ZayIaUppyr*xI
z?$Zp&dzU!>AnCqZ)G{gADF~vcHcVHdjOV({MN(U}sRu(HX*vWhPTifCbQHvRS?_jK
z7`N|}(B6HHh~pB0CVw_fP_&STN_UB@uPNV!vRlDCh3*Y$a?;U~~UtEPb(C
zyf9OJw#R_8G(6#1R$B>jjw2|2p-he`oW)IKP%$KT%O=`Fv;#fm)_c*uEqhApn3A^?
z4plVLy#@veM{(Ibg(%?4vQ`Gf`BzpMqW!1%P1?-!4N){9QH=!)@rkO2menpSsOf;l
z^6olDX=hBTrq?PxkSNvP$$E#H6ebaJ(jct``AbUr7`yg;kDOi8IT#;}o}h8f%Jeds
zU=UChmW-NwR-m;rZM&mjrsUUwq8@NJ`P*21z{(;xPns-)CZr~iSzojq
zL*1-!CEU|7+?v!Xh$TX_v*q?D6ji`-eE`D1(;-7=aUd>cHay5-Z6rU{sOhg85J2(u
z7@%nBGiR^mOr%K<>Q|z(8DU12IQ5ML$F>eqs_H}^-sz4HKP<8Zy=5+&vUZ));87t!
zTt{v$%5Y8hQJ=QjG%Qt$SF;}BMT#h;^JyS`hAIh8fOPAiVD4H;1^b-#_EN0c7se~z$j|CB0DSb2H>@EZ`5r|G#qPr6H0DXC
z#?4=8Ofoi&;Xc2{1ndQ#Z2?)??=Qfubvu>=rm!HDC+H5FsXtS?gRQD
zJO^e{DtPJkT&h!FtjZ`nuQe@0gG~t`Jiq9rBrU0Gqo5ie>m3qA!{?Z4$bf2Q_4mw5
zNCC~_zCT*^ziMbjAg6ge#}09+!2qC}e7ulq4P7@#RnOkIEFZ<&;M|{|Td|Ta6IDQo
z!g?r@`$rv9<|{sg{u-Q_40#JEzPr74g%7Na)1P?8f44S&Q08lbdWsQm`GY>{WwEo
zr;OEFHtcfvmx6b;hT+4+)$IDd5g%7l;-IglrvS%M4A1ML@LG#MXl>C}$ghPs;YxP`&FZ|WIf
zP@@rwE8L~okk4*}w;Dm2z!F^Wosl{htWiYPf5kbhkNj@_%x^n-1t{;^#e*{0kR|H_
zHla1hAZ1?k>x;3-AD$L9m{ymB%8FnWY)|Qe3RNL}Hj{_xFhmAFaS|>q}%lum&B>)R%DdG+llTJpS
z>*PjYAaJqM;~lp^4s+MSwK!pJw9Xq{nir8Qr)io>Rv>Y>xl52!dIEf`@Ku@3a9F1*
zQ%bpc%^Ig|@je9`jUo_WwalIt?G9|K+Z$lbmpg>KOf~*D5l|ASqc#LGq)W5a7Z4o2Ro*ss}OK7_bI0W8l3K8EnpgE
zrCH&Zu2bJ$ym&W32gs9we`|LnQcs!S;d|ec1$+5?ne$7eYCR84_oGLj^bh6vkX?8`
zWWkGaO%zca8#$D1=P8a|ne!9AZ5^-M;6cU-&fHpKW-W*z(=1l^*=|f(OwFl8
zY5$~sVcFWXpM$Pog)OqMsg*Kpe!6DIjy3o|ev{>B|9l=$#1+zH#fbp-Y%8V6@nrkJ
z@|aYJo4x$>oc&OnGZNLfVA2&9_K=!;Tauc0-yVVmhQ8WzVG8fLSbPWTG?73*5pVv3
z4VdV?O=Gb}Gy`uCN`Zv=i<nq
z%x7KFk$D%fqLC^_#a~^Ind=?C-K9q+L*vB2Yb;5zD?m5ajG>A@E=*q%M2^)}rY5
zcX1&%xqiG~aDRROs1c5ocN1xc@S8^q*E~iP$QKN*)F-=gFIvLWVJv%{pPJjt54S6$
zTYj8dM^xXXUZ%`I(!2R<ri
zwUpN#<{0<`uJ%@0l+IrvEkVtzRe8Y~bOn?qZTdW{l}Af~2}S&4w2m<;3BEiSEt|1o
z>IKMAwqJ?L=G`7BPw!%`eQ6aXn}F)Wm}4j;85jV>R}(D{xqtO>vKCRYrN4e(*(X;O
zvPtl{v%*CHE+{@CkU(sEydEUJgDDMnUKdy%%iHZ7P7Qjs!e%b$>xu>4m2eg~ADW=Q
zL5C;=@`w#b>SP%~YTafaXZTHKm7D*+4R>^A@{8_oT|Iv4Tw!g8C&Lo^;I*^$)#~=a
zgMHF!4?=hLeDKWYVlXi~TwKi%vs6W%=B&eK{+=`f;7piS;6_^5X1o$mKhi)|>@f#%
zV7ffoCo`|;OFWBa$o2a?0zVw*Fmld|+Co%n>U*~Lex6eI%cWVKYR~>q-dAgK`z@6?
zV*e+SH|2BzRdayKWiPge%KN#>Aj|>#$6-O1TLOz|rZwn*$d=Nc9YrW
z5UM^mH(j~ToeLZpih>Fcfe7%s)~I3>JwYWY-c*84TX$_v>l7!0ik4c;*#`vo`@
z8(Clx*~O=Pm~p-8loX>_UL$iV^j3=O;*XY$?|XlPH8I6^#-!A`UaV(dxq2R*IJUNM
zMaPg&*PmvYlALnc^>eYI%CTkSB^(bQS-OZjJ|N$71ycBCX|&!ObT*`>zVxru)a%}5
zz==Auf3N-n$P(1^=&pv0+D8(-GoB+3`__80zY7CnBje;*<=0099y>Ovh28IDl-G2u
zIzU?Z?j0nj^^|AVFY~O>UE?A>d%F>5N|>m?R)A++=_%sykjJA2#`6CCR=OygPK#cB
zui1AKg&#aAPnm)`#X4O-Rda*RJ#ah)A}+Zuki$)QXwOEI_jI-tv~
zM8T5mERYxotvOcXe2i!-%>~%^Dgspjm2CbGk5uIZsQs(29q453)cm=+Y_tTJ7wp?;
znmHpyAHE!mETMSrOEJE6eYI3Z@^t6!_2e5D8LQbKGkc0&9h_yh%CUj6iv8JVzu4Uh
z*tRc4?ob3|nVt71Jo0}Xqd3W)oTpPz1*Ap^Ea!a`Z_1Vf^H(4u9ZJ&J3HF8F@{LBf
zZVJejc`jPo4=+%|wsEFUe1rZ_K?G_GJ+Jl`k)S_CeHAL$W#|ozi5H=3-Y)0EGRVie
zfx*DayS9yj{auk{<+(}5-af%I29RD>qm7i
zy=-La`aD-Cot1tzB+@|_nFTIwvg1pe3ivv-bb+#^K|Ww5e=$GFvz}(Z28mQJH=ZY2
zt}-^$$_LcFtX~bKbIF(bY8JN#Ru1{_3mhlcDj+v}I>-%wHP7agb0ivls}G3kLfwyv
zNISx>FC1G2h&_gs5BR6OU+V(v^@&gCoyh~8ygSYL!6pFy@>W*2?_1(hE=$r&0V%(%
zS}ulRty=Pcv+A)`%Nm0;)`A+@{Vv;0PJMU}Jz-z1CnieAH40g7(%xL0t`DYc`zb(z
zx}O)#8;u#OY0MPZOXuHo`ei{2Z-$&R#>z3#JePqbz(qFwDOuCWRi+tlsVz!TJ5F6^
zT2$M{ny+s>RqmrY#PcIoca=LFN=eQ9y&_LR7!
z35w!$ylj&af7^Bf6)+c!(clWO^c*38mWb|niK4+0*?N7Co1f@>0t^9*o2tud{eZ)D
ze|IV*^i^>OzG$^oYze^U8af3l3vCP8!17<;0a(ca&F+fi2ke;YrvN5_u4$mJJeHy$
zp|q-HB2TRd(mg!+t!-1{ni!>8dtGHOuT2B&bG|V>eJXlk-@3}$eK_;g{@QB+nw*~_
z+ut)bu<|3}w36)TN6QH(fr<2-MRFROK`@+jzvQaUO7g#SjLph;aU
ze@34M9@qO3VATO^Zm)AsCCdkCTk*XEj$MkogF(?TRN>dR#w2X=+*sFuukGtu*9!tB
zd1^@a4s(v$n=Ef4^;f1QJDo1JwsIq}>uLhlb?q27=E_KTQ?=6WYH0~PULhwC9n*ft
zU9t&dx^i0xDZq~LP~LDl%`F61)?RXTE};Za!I+eDf7w>pl6JbbpvyanzwFHJqLn_I
zl5p2}u!C%Qd)-JEQ||Km8^p;3j++0*t{iv@R`VUc$TU`#878gu9&)|0CmoV1ERm_n
z>g^+}^BQAbFW0=C8k|~YSii0=c@h{r)~uR;^gsN&RsR2I@R)g5)}&1r2uzx*`5xL%
z(5Tgz;sKuhT3N|QTfnpjba%o7r}Dp<2X~&_T{j@H5)Dj$HYj
z-7#as>-j5AsLQK~t#q7+pkRMerB@m}3+^q}bapA}@lKWm_nO@$O@2%+
zpNLkygo3c~7`
zMG(X(i52Xs{G@nv8|q<^{sMm>=R8bhtQopYVYqZdI5Kn9C$&_cmihOL0LaMG^VD!+
ziVrQjW8kuul(mE)oNb&)?fJAXfvRWl*$xB4Mx6O-KGuA-X^CbTa6}=Cal56s%rj3u
zzA*l;b{@{dBX+v-XmR_@%y(oF8l8=FAR#wbX9~Z}b;9vzU#f|4#({C{L-M(%lKzLh
zK47TAy#Sh}qARO(r%lRF0e7vTJ3@CPJ_9qOg%bLO2tJbzXNNZ8cu$0aLrQzqb11{h
z=fiL?HaCE)3*d58cnMVgw4dTNxVjyQ^&HRT5U~xvYlK}kT#53w0UaX8uV@q{452@i
z0cB6gT3$)nec;
zvTG;f7XevQj>`6=#Uk_q(k5Af#dX%VWhV+A4~+NYhBH}xcx}917
zY4z9%`G4bEX)EmCm~Bk(DqypG%n>01tmK-KA`c9Gt0bhdPzL3WIfM>)@zIf!hloRg
z-Ku_aM&4H2r1Ry%YZjpnmj&-yVbnCk?yF>{_N+t**adfc8#%tvuc~aJ`~+`j0oAJ3
z8y*|rAfjtG;uEtoXzgZ2iV};QZ$j{K_fWxH<~MX7pa_nK389r^f%{j-r-3AVpBb+X
zUBH%mpbTv2UOuhrn~taK_gZ%NLFf2zn1Nq#PS^aONjR(B
zDIHHmOTG(Vq)wox&aWb#0P}F>0W1NHNy_r%3K}UHLB5c;@6S4h4(6n5nRpvy=jm5e8qxJ?3wbgV@?J
z{lht#YtD44Y(rj9v^P|HNw_NsuBKK%EDUM4Tn2#P)aamV0h
zP1*w-ITCoSV_j)xWT#|IK$fudkJma8pl|rP9dR&cx(#DoUle>OV?2F5j~2KXR-C&z
zQ2N?z0#drD!C7+@x(Q*NK9Bz<9BpXBpzfy+kR|Ahx0WmQW!7!|!
zy$h7Nv?niLlb?!^#toNZj~?~Q78fHq={WJFG#sf4lb}5wZ&4c6z=55q5Rr#N4#E5;
zg&{KI&!HzUrLc-ym#<6Vz6GQ=y@VJ8FMiT^;o#o}uW(8SLnv|mw{|F`fldpX`3~h2
zrDX#{1&cGwXVnvwd(w2ehRy**iU4OeuMEuQdzI2MqlT1r-X8&S{Y__hhjHNZ+{3@c
zHF0
zQili|qw@D>fY+M}vpTiN)4;1x+b!*%*;^@n61O-Q!3`8;O=!x09G-09Cw;xaydbeA
zSRrX$08WH+3L2FKjBzpTjd!sdf$7*41uY_>ew;5Ry@Xs*?k7#!o&y^J4n{BjW4iC3
z`7Jot|Z#}@O0AvYb)D>%!!iLI6DoA}=MlK#uB>>LmLn#FDn*Xt|&
z4~ECb;SQhe!ZS$IGwXf2-s2o~%OT|M^eB7%8tCIvfC<@^FTlXDCYyR#jXwU<(A&{_
zRJbEiXXHqqecJmQ{zX)*d{)>)?fNC2iFCouN(V<)QW{U!pGh3?khk!i%-_Svx_KqF
zI2gbeLTPpH`MKmC3Ty-NBW=_v;ED0vSH@RF?Fnwo24J}U`!O4SE5RFjjqT5ttD9vLJexpcv+lxi
z@1fU@F%xdGf<6TEZB#3ZBcav@kW||-HE%FTchn(;gb8oEt&791T)0p>iz;UCfT_5A
zd~-YeUVK
z;vIyHM8BrEU88xE%0M1^f?J58!apMmD|K4=pU94D1D&LgJZfXj=IzuTY&mufo!#(x
z&f;B$8M{8`bh6X1_Q!)yc8q?O_mb~}SwNSMgWi9g1(X3Tb?<%@ply&7{hEd*)g>hR
zH<;}C5Q^9p
z4F$5+LY{lRU`ru^|FE47SXtjtzZ2rn6QA?H55_F^3MY1@i*te)lLj##hrj>6yflV%
zfi{|fA^z78yDq*nMBsy~3C}I`RO3UOUO?2&e>;u^0f1*7M%)}Uat7Rt8`gI?XaQ_O
z9JFw${{x3uqUnOMvQhE|qUfQ5zpj*Ro4)ZU#OE1^Kj9O`
zzG=@IHYb#`vJXGdVBTq4o!}k1Y9fu^~dUt%7Is
z{p}Ju==vpBi;TR(F9fH^{biC84kUGSbl6$P3cjlo7GF7WlFV+7cIY}sX0(@4A!IGV~@bR5W=97U=>Svp&pRcf8tV1axoc&hV
z?4;U!7{CYbDm0W8q8#1N)Uwj**p}WuJ)sxMji~N=AKCYUQX5Sq03Plc{#>tei9>OI
zq^=Y>(J69Snv;u%>hOah7qj`Dem&*Il3w~9@L#SF_7o?YSz6AgP421J{B6jK&v3$<
z?Uea!O0yf7;VX+jL95IPes;2)$Ox<`La==P?E>`l;IoBrNv_2(=3%*l`N
z?qHffMS`4o`3}Z7KU0ew`k)`t0B^)#@bZW9G#E?11~Akl4nsI=R;_k{Ih*tR;@~Tj
z4Hm22eO(6c+X%}cyefX;lsRM4X#Lkb1Hm)a!ber}S@C7NjP!(J40PVEn-td8*0m+>
z^=~wy#kG47as3vF;)u;kM2b2z6Xz)?f--ncwP}3Ox=LhfI*y<$tL2^2QS}vJp9N+u+9$?5u8_{
zefgUE^0EUL%D!8?2V-v;%mba$SlIIjI7$*I?7_qRAneIWG@CckW3PO}
za1Ng#l=sxoLC(l-=~_qMT0wQfka>>iDu^84&t{r}4b#^r-@29KeBqN7ba?deySiio
zw(!B(B}51&XhT1PQ+yfPu?J&@`xytJHv*6QKfeg%riK>RKaw0fU3$R?m|of?i?4tdk~w-<2dQtTD99BN5CAHmm99Sa<`*$n!VVKu6p
zm;-eEn-c3K=8z`#(EZQ@J(f$W(BVFDG64TstigGgT{peI@ucNF=R}#ZGmUkNL41KH
zhqB-|IbVADVkO_cP#UF;eAQI#bkbR|r5-%{nO8g0(;m!bZY2#Oo{ufdmL
z=&^k0s8v}uZ%NO8n~908y`|_Sox~@xBtt@>ZF_ys1El-3D>CSOoPGfw$y?1T?#^lI8V3n!^@?
zOJH__Kg2JE$M*;3s)uByj`#>Qc^Rb0hR;zWK$@L0?Bw&gO!D-Vxh?r0S-<&w(t;DZ
zE3cj-CGfS^lBs6b;I-^KonRjx@O`V$9NDO`i^52|=8l}~b`l>xmSCFIf3T%^X}mj6
z>C6b7-}*^W@Aa(80rPO;<-_fYh8oJ1AvPeSs?+8BZiVjwUvi5Gv#-~J{x>4~xbr|l
z`O;h9vi|au+1KV-wv1K>=RSK&?k_0p$e>RuA9n+seIgp0vTtac=i3b`IYNn
zBuQmE$ec(nLLdMG*K0SxU5@uXdy9QdOyVl747Mcfj4^rrK41E|R>_U@Mjs^!xiH=%<(I)Nz?Wn|G-GHT72=B8Dpr_lU^+MkUex8Ngg$yNzzxq7s2Xaj
zu;;uN+IX47YZKFy3YY!F?SU`2$@Y=TPw-?0EJ@fR8`|^DsnkHrwPq9hW8coa3)r=T
z8ME`l31XK?L2sVgaL^wz^HZ+!aHE`qhfw75AB&d+8y&1pfj*KBLPbuLoqq=QJwT{L
z9SAB6tN*+#&v3$d_?Sb~U_ajr9&tiY7~Ztd*8EB}*^|HH^c#$8Rf*iP
zQ4=I}SDl+~wMUyK>yD3ZWwAZoTiMB<@$paH-r`5u313VF_c6bGRz5MLt2IU_7<(Sr
zw7z_LZLE9Ap;Dr(gzmldRE%U-I?KoxlbB=I-zmqgKbZMbABXnYzfb-D6N%L)lw9#)@
zmTL{Rm?;AI$iK(=uhpesE|I+UR+28se@A%RYv?6!OmWEU5%Q9`Uzq5waPeo}_kjB+
zv0F<%DN56zt1~^Q=-X^czelP~Ap7jElPVp@zVW>4h1p*rTb=g6gp-2T_+Oz{P@^L+
z4vv`RKk7-Ib2+5^q4Q95^>)?9fO&h`+X%GcMZRk{zllw!<%W28hRh4Eo&q7KIIq^@
z%wK*v07pR3LNEJPF92GQvwfuuB^$>hJH9a=dWg*4qyO?eBay4t=GjL^;!_}#`{ZPs
zcx*Z%wx`>Ilxl7-@^q+g%6+1EoV&@
zG426B^WjD9G?T2XQ*y@{aV2MROxZ5!N_X!k`@_kebTAMLdU3+?D|YsDj~L+1zZV7C
zuH3e-67A8>4K!9lWZFeDVBT#0+@7r&Z;BIj_|&wYPojoZdG)rEu#!?7j(#nqa)3~5
zU7;kRX38OnmO)iM1h>^QjLM0cl8;f;F6MB|4P_!9EalpZkZ-+zu=gj3wbQ5n0+Ghc
zFzD_4O}v|>&8No;*d#}WzP?fY=653G5B%W#R+Qg#CaJ)+J0Yqt@<}@cx1~yp?`eP+
z2lRHGcH)}mENrvXu)j2r1$_=$UT@E!wzbdOnlN^E@@qHHt0s-cy!dNh@(IX?cIZO>
zM7y8E!|nT|xjev6yDqd&N3WdSVR<~ZNUS4b_|+5I;rLcmdD)V!PgP&(3|E?#l}}gy
zH6X&s2A*>CuHR`Ww2V-av&s?Mm#@g~nAbeJASU52NpuD@E#w7Gke+c=OTKdmGL
zI4PYYsLFGAKPD*9b9(POfpr@=eAfxsK*>kSfxJupke77(?IjJ_f?-AlP^4UHy^6ap
zT$rM5);g6Jah!bgpDCZB+N&D0Xmh&Hxo{*<>$cdS`-C+U21@@~%r}0)8{8N^ft1@r
zGG?!Nt8sH{p)RSbLes20ZCorG7`9nZZN^8gcJHfc({bU=8@-svW1&5Yx6(i~cs}0U
z=TUv1W?vZRJCvd!QWy#n46Ym#Lyy$NXsLp)aONsJfHuG9&Lbl_<9QFG^`&MId7RDb
zzgCF_tqn3_nxs+X_jVF%I-+OV(%AzlJ+|Vx?Kr?AaRUjQ9YK^9LP{5R_xx!ulOB|@
zfMnqwwCAe(_4ft=Zn<*pcNDEW#YlW0z`P5dEe~5!-ZQ%d9CV20u-0Y=u@IEx4qI`w
z_4Ib9zGjBZegmYn8KsHd&(A^L;v3HgDC$48PCfuOqE^wbc=F4I4f_
zD2DUTwL(G5a-EMt1w?gRJS#x
zDUSKq)^>6My8396;F|DFxob|rZhnb+&iqIm5VN4JdHhY_M+h~sa(#e6H2PyM
zr#6H(K7u+E$cJtnV}$bH+A~!B-uM3Tub;>;*?|L9d5h4-34J1yhf+|uV_?G#p*3(K
zK5IeA-tue7QYw(`Z2X~Fvs1uDi@c#xK0M`;T4nz(8Cd_&v>JZzsM~Kor7VFSQfdAA
z;+C%MJ`Q~;z=Y6~YDMwdKo35Bl8nZd4<|$ExpLGq2=)IPnxvqh753qTEz3P*zE5hm
zrWDt6naqg;RNF%4{|IFXT0jwv75eIC8I^C_3CM0h(hhunFoK4?nznPvHEUuZBTTzv
z<9&ioO{~pC#Xi-=j_BQI)%TyJeoHQfBpvrPhn|guebJkm4Z%>CW4MwM8g3y8JseO30hf
z%b9SfwQ45HsySWZyWvx7#8nk*D8RFe$VKe)pj$
zP9-1NZg4d+zXf8NKyo^=6x?Ylx);=QPWyG%Uya>(1rb!?jiY8w&(`y`Hk4yZjd<#|
z6{{xLLNyI0YdFlnJAaR={kDhe3QS!$Ntibo%JAxRK#TY9R5h5`A5cwB-c#$GWJ8g!
zM%s5Il=jkx9!%|3Q7@Re>;OFw=)i{myfP>W4dQ(T3V$W$kr+kwNq2Yp
zN}PUE`zjBwWsf9o=QTxQDo^q5Ryb7t_X@`lKe10&g+I3MMw;A7aQ8VV
zSD=I@8)w$na9cBlPWCGfCLXJNX=5WzhY60AM+aUc!__H%Qw0vYofTUEIj*j26sA8l
zAJqg^fuWP63WXw@pUn4whz1)B2n33YKhH#j$CxarClEsM#Isz(ij(@~zX>?*&)_Uq
zZB7aMpBd)yUB#Tx{w|)^2kOYux0~pngQ!&Z#Z9-0l=vVtYJX5UA@AaORd9k+549zR
zhM(n|WapJ|x8gY0-MdhfrnN8)BES<+yVYVC%OO{>`0g~v21BEaUWeP5%h4J3M3kcbQ>W_X}HT#P84csoFkoP5Jxqja$#Eo_y(&?bzH5&RDq~pzHj)d^403AiM
zH0=Q>Ixn4ps#zs~=&MeQBIJ*l2NRRxjZ}?gF71rUkx3PHgY~E0lkD1u%C3TVQW5INK3t_du}=SzTuj@x>s4b~gbUSzS0icbv-tClS1G7Ka*d`;x
zmlU1n>lx82@f>)bTXK7xGuj3ZeFo<>vtSg*we!>?{3z2c&*=;aGB3}>zBrMg{lyJ%
zjGEpRZqk1W8S)_jmE5DUo*IH+SAN&_sg=3;l#6XJAez{7=pNT&bXZ+a2`38?tZq@22Ah`-dk$ErH|jjd
zbpq5jckq0k7Xdq8iQF9W|KL7K-6-AssX)Ky4R}*fn6lJ;y*WO#th~+<{VrVg+Ek1y
z>*mGpsunR2uSh=-q)9
zL4ThH9mmC6R9khMi`v8KJl2ybJ~&{7J)rk_%K9o$5j=aww#)zBOgKhQIqE<6K}?&>
z9_$b|aB^7r-v|WA!Z9|H*2ziDmCIpYY)K>nGxNj1h9)uwmqoKkirij!Tpe%Lh(?^N{J4IZk
zL!%_V@^CB
zyWj;@z&!Q;jW5*ORpKC|$-1E|FpG_%8^Zlk8Y3j88X~a4%={#+bZP5|^N>F4z@cJ?
zlVnF~aTjBvC?>=wLEE{iFw3=DY-V!1exGXJq{asvu*Nkq;;%9Ca7zf{a6e@BOqtn(
zqC(K+U7R~AuPnHS>O42lVA`y}Kx?vZ+a2VGk6pG#F%p$b>B8uBdB}BI<%vcw|u*
zj-X@@>Tea*%MsymW;>z!+RzDlM^7b^N5A%EYbK$UNkL6x1wEaGyH?&=sVF
zNXy?9hIq)J*}*6F`FX5Riop&c<3noqM8p8rF#*M;3NV4O-tAf-6!(6Dhb4>BoFv2K
zcAuXYdj79pzbx{D=kF7O_-JyC_n$f-4F~&2eM{A1H>|YDHl+jER1C}j+}xBiwO$UI
zBYzmLKRt2kH7HnoFVbS@#z@_R)PL%9V|`#7OQu(|=$3=O$NAP8NF(Ba3-uEH*PHoqxeW!Y0%~O~xoL9S*u8SI{cEV;t*?k2_43n#d!>X0$rUwi1GV1C=Lx0VZ9TgCsc*B0DJoy+<+3PAry
zhyHhjpbUOCVS8Hm2Vm1{J75cJtX@u!s#mz(N))
z_M8ID!rvAv1|cLUS$V=USL5#9?3*7<_kr&Pk?oN{;1e>oSEG1^)S`~2BfHrCG8A6;
zzcJqjv*xkwf3W>O6J7p}dbweMiL3$5xT##3Z#|{W_f3onL@lropCjtYrNW6_0;fyQ
z(riEZcpbZS27=}ERy2a(!sOY}Kiq;Su3V>xTovbR-PNovqsR(dNl<_N`wH7%Ge^mk
z$UVLfSWNL!*2!^3Kk&UE%!0Pg|6X~}C8<_hdQ-#5jUXBGnP}uR;cof8_`{g7zR-eW
zb&)6xCz}+u7Zs9uo;DmpKomqNZ%-DxAEYD`|5IXjzY8Zc`%Upc;TdvvoK<9^#Hg#U2~2LmdjH}5D29Xarm{4aF;4VZ-Y@Ib
zwXb}s!d+5mJd+?W;>=)pH+o~(RwVK)uU5|g3$OP-rsYp4+|MSyu;35RifL=3aY(9k
zt43C)YV*!YtIk#vNVPZQi}cG0G_2nQ1GO$HzfJJTi1!*S{&5P|SE!wqv>wqR1H(Rb
zH$k+3YI&8yzNTH}dQ~nZ*h235?+4red7ZsW%jr`nH%kqN^d#rh9KD*`2NsC}bRy9r
zl{DiF_thka6MW~sKJjc0$xpxtsh+w}Kt7%Xkaqq3Ot?ZdQNKF*VMXOqzxoZFpl)-{^7=XrX|a&L
za4wXGv{)o^>JY1#d-CRLe~@k3JfYR+To8vbuSD9-^P9wsq56@b{kj-dw9@fmLoUX{
zVYh#$h9;EM5Wop2HGE1)1&v9)&QRj}j@8d&>7+e2XNpJn_*KBXcBv{?`bHSkxU%op(x{L7R$6{HNboAZgFW#a^xh;)}jB7Wey{#YN!Zv(3yS27%4Al@TCP-mv
zwHZ{8Fn7-9iSaYnC$BNS_$bCq@>4#2h|*g!<0r&+=4I!|Z!G}mim?GV&i?dMkbASRmj=hHJ99%+)UqXTX1;GWo_Q7GH
zkIj8V$@V8gx+wBFe!jzzHBQXs&4Zox1py1@-kivnXJ(3>^WM=57>7Gs_-b^>O`V#D
zyQKusg$FV`V0$yJ3ERALT5+?c&|(W!x9YbymN0G|*updBbEDd0jgBy3Y5cC9PxGBY
zew?mUqFyW8iXwlF!Mes6<4Mzw#t@-R*=81En~<+G&d2xC6jq2WFdi9SvltUDfMXpe
zh!Nwp=pFWXZKw7)HF^hKbQHf(&=%~4!?X&mBGeN@_pd1MTVQdPzsjo2dgphp-At6#
zbY=PGlEMF-SF}lO_uzk{L~y}B5!{30>yL`_BE%eP5ynZJ5mp%W05PAs{FOcY(Qw?6
zG~^}I*dqt3u&R=eTGZjfew@d+e>b)Ls_@C)($uRl)&6n|;kv;uHd>*J*pVzC;nP?A
z_TR-d;Y=GC7Xwz9gU*YHRqT7Nd=cn<6))W3=rOVtn__pge`hphB)~Z7k+W5Kp`_#n
zQQJ`I;F}G&9f#cQ{tcC!weTUZn-(Bh{lniY
zef?*$AzoEKD*ClM=pQNw7Ni0#77ureLw-Rqsk;dssIAn$?FY?gpl9UYWk6~$UNI?HhFFTuu*1+SP`4z#IxsLa>K|D>`85z4r?o8pZp8`TuVLuq3pC#|epOf-A@gEm;6%d_#desVWl?hhzp
zb;7brnZV`^ih^FlvokLkQC5hFR9C%)Y2m)lPEpg-6=o@=m-8R#pe;hNjZ5!thkG=A
zn`fN2QNF)&9x*t_=DrreK<)m(dq&d*9k}lzs1%&*r8_F6
zgi43v?yM!nav$;1l}n`R>T6Ak-n3s|rSDg~#J2wQx#-EGsi5U9HILG1)!&c$)altn
z-8JQiK6TU;WtF}qrYdjRG%#O%#eCz(VaEAlY0fV#gW|>J{$WlIb
z%Y7t$L9WK9e^7urtx&>}?QB#cOtr5JXYX$Dth||sM8|ho;|x~6ozQ!X=j!X~(up<(
zjJ`Both@$1C7nY*dg1BNX{3rj%K}o1bvU8id^THm@YW&oV&SbWCeqMKG5icMV8Hi`
zE&j*7)%BHpnfwJ?pTBVIaRU?Zp>NUW+jG5ARHuy?T{jeh%;tw%kL(!Q7;nl(=Npq!
zB`(sq8qsnjx>~8shDygC&g7xA?;Y$VqnG_q5>y~*{V4B@DnGS1d!Oy(!ZFkST
z##!9qkm4-9ona)t`IEgXh}!syv%1sIR*@=BAt8g2KFb~MVbqL>=dA3VWfrB6IbV^~
z=+LpmnX=T?FDhERF-Hnaj(r>FOYQ3JZBI5b@C*{P_0&lGLHdBiZd<_lKTEi{p}oy+
z)G&#s5^r}QGHck5C+l!4Cv~X$L2a(zi^)xG23IOGU9uMqtMA1|De^1eaM2YeSTPfL
zv8IAfsb)OaOgqvJWBOuzXz4>DGEG(Kqa$C9V-HtXefIHRQ;TzU-=DeCVS1@6)gh1d
z-_OxqgzzS0Je9VPCRG&qIZX`hX4z&K6+d9^Wl-spf~@vQkuTX?>nMBV*eTG^CDB38
zGVQS1EHW~>zvh(lLVrDVo^UzMcHD3a(++%ouDD$HRGVEY9JW2M1V27IYmj%&y>$S}
zsC+8sr2XaV#078mk2uxEs~?f3i|i(i**Z*lj=#;WBn;0Kyf${gHYUjCf3`SjI^m5u
zcxS%%j-Bh`ff&}ZiH;J!nq@D0a^Itl13+nm{V>6PNHz|-KW<<`+FyVxBom{e*ayz!
zZ#~j2{*~Kp<$MzN>WW9x9=^pepDWHTEnKnFMXfvMqKp<(3&WS$5H%XHPop&ww$H>|
zSo@syMJpam91!!QcfvLp6pT8*-?ta?QjptCOI+*72b=y+T2l1nSUr%2wD~$l
zxuDA*l?kX=;OUkG8Rxou_8OJEFxxI+svnfjix$~?;Hcn0Ko+V0`pwAFO7J5E1pl*Yc?_a<#Cw>;elBIjE?xu^_&YBVtvud2js3
zINzhwX}NLx6`bp_bqL2W`hmnFFJvN2-%i7l+ZZ3~Pde77Xr&WtMZ7DfLaTio?iwZ%
z1s6L%|Fp~L8GL);;ZSbZ%9{D3?(5jHNIG_bTnWXKi~HudmmB({T=BxELb2{vKsy*ou)bkBRGOYquFSbx@JBSl=K(4taG!
zwn&M0EWzbQL#n$)$)2r^mOaDEvr@fZjpj*{9W^^$mN!G4$Gikj2%}$FTwpg|t#@#)
zG8njlog9^(p~pw+vSx(McT_&fADnSC$^qjGe9}^Tr_i30ISc0M>T&PIG47rZtU0z&
zl_tZ<_`lppPr6427<`4hs>Ngses6C*d3&}0Y_Iw{CkhqSPta1r3g^Zx=%+eprn_Ii
z`F7KbqtE`ShhxnG2loO?Rd2rW*(pqJgk{v0+Gz#SGvpeof{M@C6x)Nro~sWG;csr2
zz_H0s+=}tVw)=^R)GR(nB1!wphnG2AP;%zIeCD(6#v#QA8`9)bTsq2)qqb05<8aer
zct?21juD)RI638Tgb_52sa(#Jl9}*1A@#)G5VNE+vl=v5KB3SU$8hQvzUxd1%FM^4X$FMoOId0ebOyk
zNTkFvJY)GrWGl7rBHS4De&rjRt}Ze)FHEmJ$iB@{7yCF}&<8mzZKo?yOJB68;g||{
zv6%kt%1uay9{44{iQDkU5xplymu1dAP(yp=4g#DlmJq*%IF9qcBHyGEYM#uYgSH(X
z+A$6rN-CzeR~2^4SUX&bciG$t4G)Q9E@MKM^z-5Sck|EOe#jC^W^@+yKwxWw(A-CE
zI+L93`n)<#n3`9>+o>|YRQ$L2MGj`%YH#6r5d&$L=xwQ7J1s8y7#|-FCaG#g>AHIc
z%H=Q28W!^6d{Z|qI$!wL;9!H>PWA27Hv;m^kPEzw!-MY5mJDsMt%FD3RnBf`mR#6t
zyfaEz7CmLd_t!h=tvzFHD-M1hw~gQi+cUB%ZB@7fjC)u!@hN2}Xs99s-3ap&MW
z8_P93p>N_Pr^T5DraoLX;bD8Jm{7eJ)W!1nF_Ya5=!jPD3pOFwopRgJf}wuYjzO`P
zo971_T{r7q7`_Np+xURJ(rD-5`C`#6yM7S^z~#p=ZRXY9K8EJyf|Dm-M?5{|A)_Ea
z5&Pz(NVCrgNjhlGLx#K!lNMdRzdllS@v(;+^2iO>QizSFR;T`Z^llzuN*WhqFbJr&
zH#dhd3nICZ-RP>q&!3zuq#dgs!C|-`zA-z-PCLns)V^L;U$j5A8Emsm!RKukU(E)3
z;rhC`1azu(UTtg%bcEAsDHS`*@ngqzke1-1Jq~+v%*+I&L*`m$*7fQ%9{;(~X&Zz*c+Y~y7#|4NB
zF$<&`CA;thHip@N?Pvo6Jkx&8NK@SEY525MmYT;BwL;ZoEt`P~POn>-#b}TR_1@B9
zB*=rs7~~Eyc&nQi^0QtCvL0LJj5!QzrNM_93vRlfAu*t6ZTXB6`_t#B8J{o;*RoNh)l@AQJ>d5~B`%VlgiQ`r_2x!iLKvOZwC^-3h0!qc
z53D4dn>o88JSd28US+g)yg6%52Bi;b^gXJOm*MC0R|Et;H4ema`PmZ(1mlFu_&bqY
zJj!%LRWLK@6aTqB8MUQw{lYE?miVi^mtCAYR~*LD>M27UYSzBHd7a{|&is9EHOu_1
zCV$Xlh6j7}p3jvi{3#t|$2pW-%|{|$_@cp0SsxE=JJ?|5^-=uyCv$Rb=|9dz^A0C8
zrw&^79`2S4eucx`lTe7gZ(Kbcm>_+)UlixQ#=Li=E-e~@rKQD9iS^f7?5R(?8r-};jIHzZ<=t`bUOBSh
zX!tnkD2F2dDdwGa&)cK)2lHJSe?%WO_93YBIpVjV2qh9T0A(kL!A)!(W;Ai3By99j
zJ#45-DN0Fcqp?oNb=34nyZm;FQ%h#j`#pOPoZjwW!E}-GHr>~fE;iC|h!x)T9Bm69
z`7x~2FT@bu_*`z=aZKo$Pq4(8SGC#fC~vJwkPpw@bnlFZA1-)$wrb)d@baYWx>cDp
z)GZc0i(7ZLxNT_rEAl3>$VZ7e_%)kz$dl`=!UvZXk{_0_JoqJ5dFztNQ4fgr=v+SN
z?ml2l@VD0rd{eAN+)F6B(u)7U#Q)0w(&9wdL7$24k~sA6`%Ex9B`TZ3o${qBjHEff
z5p#ICRt!=fIh=Ir!?FQMnN54wM<#(x-NU&w5O(SsCeU}m8VW`49swRFj@Y&Q5rjo|
z%&Z%K_@w!xq$l=OpWS;dzGXKOD0f?ckd^A}5Jg<-O*3qWR78`@-T&2k_LP2gV)@jZ
z+R2gLsZkeo{NPj9&1v_-)Kty4(ek(9YkA-KKx1gz%F}Et>zH+dYX`mmz_do9@p{{N
zb1}RRI{En?`=8G&^gy^w80N<+>#_>A1;YRn9SQ;9KKbPF2x@Vc-nIR+gk_Wx8eB?r
zjsu`95BQbi*!^Q8a-b(RoQ@j45J#=Pe_ZL|
zvj^Von)~Y$cMjA#iQd3IL(Emn5tJOKMRy|Br{x@c%h&tkF46#blNLIMSSXMmsxEv{
z*GO;8#PZc$3-7|5A<;#P9#PTPK6hT&jnYx^ed!r=g|!B#^Z$~R9?)caoUAb`cUD`v
zk?AC^FtUO4wb!`0)Qb?8DBQQunAw8;D_tu}aZWwSAxF`0&?{`*_I9-~kYzZlUmgI<
zV4VUj14ha|t@vhQWLDhp5niN{P{b%7dqkRjgy(92dQDQ+;KUI6)Qwq?*t-e<5@vX5
zj@u(fz&nl|r}B(l@yxB#QePvd^~}aoK0|a$A``WVB}q|U#O-Be1;fWqci)#Xy61rD
zEODHF7n~zvfz+hY*Wf{UTzfJjag#S6P_E9c{fCQ0gg)fD7WF$u$)7c@`V~<3gJ}FD
z(aUZ9qyNq_z7RH=V>;5c&~dI(JEGSKkTvEFt=Z<^=N^&fo@ckBG8&q6SlcQaI-(D8
zR1Dkn*(=wCd7xNjx4!3}s@7>aX4UGZ+a}p&RI%*!&JniDg$fjsMJ7U^WtdX;3Ii<4
zI$h;R%nr_|w3v?_e|K3fOE>E%tf6`K2Y=x&jNAxer%%t7Z4^B7f?=%UZ5p-^5qFJh
z7nMd4%hh!i{J860gJ|FlOG}6kHgYe+?^qN%d9}4Wj3vqOk&t)h?O2EqmbXMJmT=nB
zRwX#&Fc%${B#sg*S1kSW-P05$e6}h|NTe2qi;h(C7Nf8Ir2~r`_elqo21??tDrNFqvo~ab
zAX}urBmUlL;_WT1aJpqBF_$z+-#Yslp~6*cjVT)K*o5u!Hr-Sn=d{sQ0N_X_Rxty!Oml@Z65
zy!t_qTUV=h2eRL!8#AmNeyx!lrQ%*%vSjB|lN@-hZ_TW+tA1k0dUB{)2exDd6?#1lJ?{WG`(05#x$vGxU;*{P-lu~x
z3uZGcaYv+^BgFkw#BkmO63YloUCxJ^WGHPYd=SoY_jnwB6){rH@m}(S`z54gg1mE<
zmnTli6}8-ZtNp@QjbZ-k>Dp|E)tJqvj~YKF-H=AE(L5``?Xz@vzk#uJ6tZfUTL~8*
z8t?f4l;&V^l^W|;
z(SQLP)UMz%FFRtd+$+{X#4n!c$H(bQ~BNOeQExAH4FRR%JfX=OV*
z)z`p9nwCzI2g*ULJkXVg!mvzB!`}~&OUvLYepWxT&R-3vy2IOYPClHOY%L6swZI?E
z6hv04|4R7o^*;XMxO&Nl{MPslT=nfld+(f2rDDfA7~MP!N&I}wr6O(%*}>^ruy@H?
z7@bit$?8EQWm;UqOV=GdPGpN?!(1&s_ikECWx7vZ>GO>G9KV&V`>|09MtAxc4;d(W
zIk6S;i|FO@F`#p#5`o2p-2JJx7_Prm0>+Aw4F;p{GC!t0)eP)q`@x#K8b;)DOBY)G
zm+sTzLy+z00sTI(9fsUICP*g(CaaNWjT!dy!^v`6pdENa7Dc_vJCh2DqFn!1+P|Ke
zxT#urM>-BL#4&MGdDZh{32Bv|Rda_ejk#=l9R=Ov49?@W<9J!)eV3
z5nA>O;?j+V-rWgKJ7?9YooSOUTzV4>_`nt3w*%@vUNcAsRb|uY%Joq>0#+#3U#5ha
z*!BSZ-QsA7QniJVba*mgm!WEdmFA!$e9&8crGENJFiaQkmJVkxuWVCX`r%NvCC?wT
z*U(eA2_e60Q4ti2;{8X9$}mGpWTA3O@jDoaC25?(&No_j!V19=*$QMT(}A#`i0}3;
zI84@3rs-QbHHi7Ksi!htiEHTcdDoU?D?4nM*$}Ak<2T4JpDq|XR6Z2BpC{OxcU#Xg
zk;}2V;AXYk?0)g7VMA&8jfEhaXYgX{;|KXVwYYAa4zQi7$k1(evt(p0>JM$_P}>QH
z8O%oVAr$YIY7{<1rf9+J9aHxLApJ1?0(EWOvkBYr(40B4K?G(x$eGq{Zw%&lpw%y0&9OVA
z^MdOa!HB$kCIDfa+<@WiL+M5GBVhpLzKO_p*{%|My|r2q
zl#Mg8gqx=)d73{m${ZNHF~5vIwema76K!Lhx*6xGGBm2VAZtRp8sjqflu6brT7GMq
zx^Ql2M~HC~5;Pd-_ul{pBEORhLmiVI8*ZdWioezA{d^f>X4f5R$kh+TeMdD0hBe_a7{-fa0_zhLYEQ(Vgd2Zdt8-bJw{3h%}B
zYP#!feHYd%cMF`Jp8FBCW!CNnYst}S8CUpskWwd|3a1OvFb({VQ;Yn=?9MkNJ*yVS0v@*BT78Url4oRV1U`jJxGZ=wLc
z<`iotM)L193PDI+$D5lzJ}~iH3lO5dFdXzLGndrr3VkPF5pum#29ezBYPqryciCrl
zlP$)APoeP#>tKN4^Y{PQXjR@@Q3J3#LSivxb?ABJ3DNWmt^1-yfWhAryWsP|?X^2Sm4nVI*)X|3*-2`7$3
z;74@qmIWAU8K)stPhFH&qO5kc0+B;5FjmrQsgUJ}MeQ99=QzC4S=T7vs^#87aN#F9
zN%sk>Y2C(l#29S!82D8amIbV?a7}f0lWvY>^!Cpdk*EvUn1UxDr>`zQ|M&~2Ft__Q
ztVd}0ew2#2bn`exFZjZqOf+yo^BsRf@w4#xj(o*mKiS1??$9w^tX5m6X}4B5k#G@Z
zhnOulEFAJ*uGR
zXqop~96C56hw;gtBNPLs-U;cj9YKmyVUxCH3Lh$vP~tCAoL6H~^t#skz
zGbzlvJA|Kw7z$7kNa+O5V@v5dmE{WJFyHZSNrEx}W~;
z06_h$XR;LitREb5W}2#{JSD9_lMl1~U#&L=vhl$^vsq}vv_IWey(dgxIrnnMku2yW0YB4Nqak|7{rQFHY5a!wE_!
zL1F9jR-74`wYo=Pt$0@Ltr~SgVZFtz0TDA@g$0IE)juHaTv8!G4~%J*a~ox<<7Xnu
zyDbRIUrY2{j(L85CU3*%ZbYW>#uS71*+otFR4UK@QQK6wB@QEyn-cr6{;QE25pDKm
zMky>;NGQRYReWvBX*3?)`l&h4MLn09
zZJKVT8?5KfdIU#=Dm1ptC2nwgoQ2AzfIrB}>7Uo_LoNhA@0*=jmQJx#8~XtaD{JZa
z`qR%B#=AS&=^=zI_Zwl$Wm(tfJ37=2R1>fENjkx&Sql);tkt^7(!V5293m2@jKhtx
zb*^`h(sXS^(02{b1a}@;sOkjo-3cy1N`%eCvcGr}4w<_}>c6wFe0?opoeZhOmB+aH
zVck@tm(Y|}dJ1+*U(xtL?h`*iaa5KmH(nhEjtewBKYmGeR9|N8Vj+z-=rN#-z51v0
z@7%N|U%^CVPxgRdM}DtNNIpb2wMrCCLg*q|-XO)u0q9u=9AbU_Y+0P5w57a;=sd_LhLagc;fT8rFCOu~`KwARYJKf=XBU+^<%u
zOl`Rg%LDnwHL4wn-ZqaC#lCCCe#h48B5}vhsvdy983g|7{Z)>jJ8ZSRku#bEJjM3l@1n&yll=^6?@$NJ1NAbu0{WiU|P8npk
z;~1C^ig%@9=bgDyhkt`Hox!JO=9Y*eR*VnL!&pm8beME+
zeuU4~Mk|)ucffw_HqdI*9QO$=>X1);XhiL)ZdAV2c>%ZPENWX#KoMI6L~8D@PW_78
zsAHdAkn#0c9y4FoGk$d^z`p>LF`B~0O>awmQBhz41Gw6XQ4oqtn$3#TuSS1M7f`Z?
zG6bpL8A2&dcYu!ZTj)X96yn+0Q%6(YY9=RL?O8p!r99n;
z$nJw1?DJ2OoLdMxOKlJa2nx{7aer+1T3_ad$I{H_Y$z!Gx5i!lQynN_GT)%3%bd+S
zN#N}J!)sn@!OAc+xQ&JAPA((w+}m{C#mSc6_vIU*`J3#b6UiVzBz|_x}|X_Ry7KZA+QjQd?xmnkNk2@HE)fqvxi9!7AgnO
z5x1vGK#~Sun+^-=h|txkzW)j~@Z7lR*`B@Gq?NedHpeG5>#=cWHS;H1IwXcUyhr>T
z4pe4F@_Py8Jo!d_Z{qWl=z-}?RCz`}??Et-Oh_|1L|e!R
z{};m&h#SkZTZ(I-a;0+V7NWr0sU1`$eiW`%ctH-qQ?D~U-1j>)yPKY(Yk-~TkI-Nl
zXuSCjmu+;t`o7>*J7Tw>+_kso8PXrUfIi9>D6kc}=eIfqP#AX=J!YWV8Icqi78wfY>B*dK)>nWMUd8wh&XM
zCFq?aVKmR^Jatc9Jt$swYynFWUqCBa-aM0kNBW@0QM#B-ABc~=;dIzm`NFWiK5oV$
z3{b~kW3O^h8`g6DN;cvd#zJ)0aRpQhskDK#8hMZ~vrKdQaR!_X=gbk)zjHO_$Jik~
zgl)v8*|4drp*XwAGuN$do!I)+d#|`ea?xXFOND9Ecwz5#
zaI@d|P>GJ=raMT(pwqd@Pt9>DC7)-Dmf6Q;=o(TP5h6R#E
zsyg$o*HjwP&MVWfSH4nRs=8R+wbWYSxZ%Z7i^}7Hi|-VciD!5bKx>p>e4)JD1}TB4
zgsLyjLa1VMhffQNqZSp;9a|o?wK^JjiQt&~65gr^m#N-UaEk^pbtZJ(^mq!L5+}ZV
z*&yM{3ZzepQ%qSvt49gWwqZvUcKYM?N)N9ETSghm8q+2&Zr28TZtr#F9Y3G$bK}5K
z)^i3pZ(Vld_50~D=+-ZeNQpoq6!p(A)IaLwQ{)+5o+^t#NB^9yTPi>6J*?5Hd1|e{QT!zfue%laPir+w8LfTy1@Q2pbDh%Kw=osxB_vcLCQ}xvz+oi2NOJ9lTQ`hd+&8t?iX>(2UOwGmvNepP#m&8h^3d}j51pVXpT99tvR(35J}80b^=qaL^cRRoo72fz=v5B*qe!;_SuUQU)fW>`9TiiE2Mo?U-wi!)vYS#Z2EPo
z#Gz#Iev!UMj)9QoKq-(qi@eN>vDu3VN#LCEUT@AjZfe%(gEEsAd{``yNGv4hi}}m)
zKej8TssxX4w3pCsZ7ZpvHNUVVg+8G
z-P0*gNbAi}h^sM)VhqP52l{9`{Vbb5`Dy3b9&u~Zqn^E0(shg#{N5UETPUJ&Pc4J~t*h|n
z_QA&>*z|W;1DVtNyGlN)KO-xDMUVC(ud%y-n8S?fl$&}>FS?tyNAsw&3k-5vDu(m2
z@k(gPWJ>rtR&CcTOucW(l~K(siS^cVTS+J+5#GL-89goM9EZ&bNxFSP^GRZ=(~{PO
zgNKp~m(%NvnXZ`YD?(_!HyI)fSceT$6~xkz2-m7kS*Xe3`KA0-)1xliT9VF&vhAhI
zr}cW5qvba{v~JMG3+%#Fpj&pVp7
zOwISAI?ia?8&Dsp`V}JfeNCyS&4q9wP9JdtPf$0n%?Gu(>r?NoyPWFyQmbSYLhLeyQKIIgGA`1Tm1Gk(Edt2qlO
z%ppOcr1x*_aOsMlc=FhfTh5I8{8TxiOI*ZyD19N^#r82$7i@6R8|(5l8Xnb!m)8|*
z1mBOm1&3weW4Br3kJl#KabQsXEj(<-13AXmDVNe4EVRI;AU}Cil%1-No7^l}H_QC@
zikAP$h=ZuFm*CLkA@#uxC9FaA^pz>bdaT4w72qYlZyl)mcIm3wk5yjTpD&I4cL;m^
zRQcY4!kqud(!3xK0E9;)%k*AU-=p@(c7_i*3>iqbWDo2xt6L`F4z|Gk(wl!69c*}0
zZwBV#Lzl2LYW(J(@&I*?_*&&T*_SDxWm?AKhPUtX?0Iv)+#YmKK$=>yoz{b3lsqRO
zQ3b95VU$9My6gg$F2L({q!b4)1|7Uj^nq-rckHcuAJ<&f#tS81HelPr~C
z*7a61kSuX)wJ%dN9b#5S8*1KfYstrs-!+AP7-2g%`x?c=xbFcMv|BD!P@9{Na6}u-
z^1cf#lQZ4Xs@k;KgR^J!j9}-ALj-xxp&z9k-4t{C}u>
z3!tpOE?iU*1ym3Oq$ETEY3X=LDQW2vP`X3nr9nkHM7ou3q>)s*yHh|qUb^G#AL4($
zGxObh&v(8%XYQS0#u*0r@$S9XTF>*WXYC(x!9vdXcL|Gehg7q6%UpVyd3Ji&w`ac<
zgcv10{`^*YllcyK>l5VZA2s<4dypHXzqG#B?ub`ePKrenRP5o1nTDaeGg-qHb3(I-sjXa6Y{LD9%j{B
z|2Q1|!z0XCf$9^Zk;@mMXz1h*&R+xL98snvTZ2umL)<#D?=~
z)0i}TX1utA&^AxXA_0q<|CSKh9dJ*A3YWoODa2m2-%A^QGU8i!=BjX+*X}x&^1CLw
zi{6@mo35u-qVk?jswIXG38jx~-QRlji5pr+bv#JpQ}Jv6M3f(xB?0eK-@x)XhvV=%zdz0NwNl=D9EL-DWVy6$O)=mlM?@m$xwp
z4}Su83#?hd{?p?*pi@QL)r7xgPc-=C4Bp-PMNax7V{C2qd=VH+ftik_+9_zMvoEIV
z>sD6ndux_>(!^kQIT%u|@4X}0K9@a^5eUV^w7NX15rZaBKhb|H&w()js~dmw*1?33
zXpLb5i@QpDI%Eq(xF}7H?WI)>o~M~j9eL4Ny5>Z@{K*AYS;~S>!T|kP-bI@H^4?1j
z_k{n%{R05ToOK&mZdILrdQ)iAF!hnqk`(qeZN=PVl7@Gbf+X(`LN3i*wHlV+n@{B_rTi!
zd1uT2%AF{{t`g!2Oi(txJWr_3ylz@Hwl9jjt*Lc7+1rmQ8@1E>Hl7g|z>_jpS|RAn
zCca7;0>C-r>CyV$`u6@b>>Spy_)EdL)lF>gE=w2_W()E=KrQIWitRK9Bthmkw!Fo*
zF8A#B5x--rq)Y|jUIlUX?4-eg*5(ZFLCB5chjNGgV8&11=7gxw;w1Ycf9>~GJGG*-
z)E<4?I*Y`R$0zYPPCPdr{Q`GaoGS%IUU}fY#(Nqk%dc;M1Z5#=-(u9lHJ;uOjk_{&@@{3|@
zpA+8tp<{5ZNh`RHP06?cda=}(s|$vf2heH$#L+uNa8J9xG|eN!r{LZTdk*aXP7pn`uX6oe~W%z4lJ2?(U
zwKzaZV7=TtE+{1#jytHd*N8;-Qw`IRuE9S`E%OWA;<$f?nGP4OHu25z~ETv
zDx?%$rAX@_wWYUr<<3BlI3r}jF%eM+nZnT?{^`0_|2}IEOxm?ztJ|hCdW?9
z+A`}UaQ~%Xqwwp18WUt(><=E4lt?jEX#SLzdc9Fkzx4Xtw$M(Z6QB5j{P`NW=mWMl
zn3y7?1Vf`y^KPjf1<*L1r?Zn=<$A*1=-YJZH$R
zA(8!JL3&e}h%po2egE!^$DcPndq^R@aV`jHWgvDHbfB^yepH3s&S5n4sOrNegAtd%
zgCO*Z$_`}VJMF^bDhHxrq;mH9^EjSxr#{Rn0>cC%Jf`<_RjcQLbTlL((xaV(hYv4k
zM4kEbl7$k76hjjVfN00v;sn|o6
z&G6$Ms?U>OqP*7k6U}M`ck%RQCOwu=3CB)vwY9gX^yzA^GtlIod+=qSdj<`Wq-3db
z;p)RvExv0f7rpHY^k0g)hN#}=FtR&Jr0b{Q=nL2|;fG~@=@z%p!9YC>^jJIAFLfKW
zZrM-QEF-GZ5VD~!$|=1G{q>iP`aU5xz=1XjW0SnNkt{Wm^rb{QbO;Y0;na2OykvXL(2uvmmF
z#0l*Y_#z7x7|}A*E0h$|#+>q@fz=b<6usCaa+h{Sb4y9hb~hV;{i0k(dBx^r`C{_n%3E;*7ar{1f)Z8U
z$r_y0Shl}jfM}8GJPVu^;wdfJ)x^xSF`6AUk!qIZ&xl2<=8s?+S`o=&0GrA*a-GBH
zJjW9*w4I+`T~cYJv4hpQ!EjH{`_;MU)tYRHJ+%5Z
z8QjXo+_{_wIu)6FdS6Z7S%Zs6uaEAvn{^h>eOi~TwBT^j@MUlFekBC|I;2yf&%|3Ve7DA`
zmvG=X+<28L`AS4Bz&<}ewVHIWcZ-)vt@%fftTIXO(J-6ehU!5D+fG)P1)K-Gr_}S-
zbSM!*r+MLQb&E!FE57YrGhdU3z)Q4pIoY@z&C+SIg8S1-&&bt
zJwry~dqP&}7X9cGvA3D~m|f+}xJ0qF?Svlqa~FDr!VYHC+eRxw3J=_ujrRu^MKfI*
z0xy5>@b>rb5@oQ2^gMkHZl~to4tOy3{P`jqx<)t+3dsrn4J1K|*JTM$K!nuV>pJ!x
zZr?lGg-1LcJ3IE?>ZZ-!6H5Lr+`3S9T>nt=M68yhao47FXc=X(i61e_)Tfs-=)zt;
z>P$<2rt*rrW7~$EA3n_9c8m~}*wwQ8IlO$BFNtaV|_iJMK6?G}oGks-S<8R&0T0YsB$(QCCvs
zSZ9#zn|MS5|LK=ZUy%Ni$H%40=8$v+$*hp|h8bA|{)wv{ANFFt=pK6bs%YDol0>xF
zJ%z43b9o!Kf__c^$VIs+WoP0e6Tga{5VZbT>!L1r?gMr|tNVBj4c<~N-YDQa{|I!N
z4GT7#E<7W!_I2C7j(b|0E`NWLsWM|mIAcaemSo|hug6*iPosUGBl!ZA8HSJG(%bGW^qi!kCCuR9TrtB(s@&Cqui;Ti?u}6Whls#|uMWCFNE`m_OZ7jW(`P*(Z8IC9&b-
zo=|4@xWvJ+CV5AW@3%Tlt&5&Z9xe*I587VT>wEW5J;ObS_8O>(Mr-s>B!ACFECr2<
zpj&?NC*0WGeWRrc9&cKBwzYx%e)&_p(Zpx!7mM$I+2AxB3hl3ga^rd!mJ1>7}oSK3z
zC(HAkzU>#x1zP6eN1uMe7v;wj2)3>(*YOXn2iaBE-#~hE`S;g}Ld9Vxbkm}>;HWtD
zZOM!>)$M6j!)i4YLKqc+w2n^oN|YmgZSFejO3Ycwp(5=J>1q>PrE*lJ57S0~4D&LP
zJpCt;oQN0qy0jMIOcr-pnPPvmBnZbH*q9+zOF*X_YR_QLpAl=8PE*kIpJUJQbk7+l
zUUh=9hs0&}2!ygn=Bo~2INRcW`eP0@d%eW!(_G_6FINa35ODN&+=p|{Q=9%@q;ssxm`fp>qEnlurw<}f5sreo`Md~RWJW{hJ
zYHe_-Pn(x3Vt=dJ@(Z1m?91gkKUcge!r-?qDPw0o&CbR?EnVcPa(}4QOmlZJ(2_$E
zuQ@M9cIl}89$^qK%i^nw?}Iq0u!Qg^#L9!RJ`aOwzwsB83ny~kR2fR5&f}|f$e5@|
z5IzdfuZsR)eMuf;9j^c5JXSaG2!#8F0r#QGhL8cbWSXu*U>3IEJC3@L+eNk9NSv`*q@p8}*seIGJ
zOkROl>594da2pc4cJn1YL$*8>+iSS`@dO906)V*)dKl)}=RckWHDEZ(yuEKD@FV{+
zRxtjF6#!-xwZqSD$u;T^-^PvCC*c(~ci&N{H_tU4^}@K&djLKbL33LeQ?$PKQ_CIo
z`YrIWq|3Y*UJ!_3$k~Q&m1pf7D`>t+&iLlCR5CD|>KLX{(b{97AkuIXP!0(H
z9qQZYokXn5hAjmZ2tnKDwDFAb)m46i-}x`{6W;3?O*;Y+3^U5H+UShZ_e%{bf|C3V
ztAFc$24n_1sJZrZ^wOnA2`x^(dM*0BKsBhCy%|yg9r&1nh^58G7^%R_?39H
zv95|I{7o_MCh!O*t{*BMnRq9W
z)6lK`&N}->Xp@}4`R4JsCFa={DnfBzs%MCwJIv39QNE(}`;n^Mw^eN7=M?l*eRYKd
zyN#I_G#1@$e35+l#)n1y=|cO_wqF+f4|kVE@;~DS$EwSw%})5oSJd;MUoVj3@dgrTNWgXIP6O{s>Ggj>QhKd
zn+;fQsMlr}=gvgA&l@p^^uQ#MMO3@BP?2>fy_dz(Pv$KglC-Bd&E+?|
z3@NH;N#D|kYxBtmZEjSDhg~dWdo8Cn&8&I8j)=HsQhu@h6Ch?=r9#*C?Z}SIpnXg!V&(i4VtJ`bEvZZH~D&rQ{xl(`$jjpYcSaqLy*HQ>|)g0C}?%W$vM&B$t
z`HdF4%!}QpwP@&8UK|~D8gKr3#QM62BlF9b?+X3zVo_$|FG@SW+UO}=?ihI3tlMwV
z7&$q66#$qedOKWn+!0ZOY~Ticx;v|dBB8KTMV
z*G%GMW~HoYriNV{%zKT~C^$4fzGmy>_p)BDK7Ju6z!AtC>DF!g^zGWQ)8}D@Z5Yen
zGLm5XA{?^<=)-pqFYDtIo;&Y=ahzZke6dQoT45^y>|}B4U|4FmVlTCu;6IdyiV^+`
z4H5*vmx~yg86$hmjfaIEWL4OeUjgr#Ct?MP!%Ywww&f>!ay{~Qx4c4r3^#r58*Tsj
zfye$Dy25+$A2VE7f0!rTALgm|ugud*(t&E*3?B{Qqa!TsR_40>REH_4#rQ07zuRn~
zr8()za|ihLZ!{;{E}`A2vB9+K3wSs^vXrvP?>>0r2RT~zva}6*YdWIiKCqXld6^jO
zvDxKL^cx`GLYh&x4@=$jGx4EkGx54ls%%u#s70Cl;_?t<{6(s1c-0c*p74x*X*eOC
zn`4po6elF*=>JpwDk;V=qm4!DVCifVG+h8;Q(`5d4_FOYg)G+7k3%b5&;qTncQij)
zmLWf$kwY<9VmP$KmL1#!R_Rnd=R8&360snb+{
z5~LJA`gsE>^L|EVN=qvvwX(8A$PEo28Mna!oeR>D4Hs`789Uu)(<;cq@u9dnwT!Lz
z^L#I3W#V6B<(4NL-K=QaC(n^n0;V$D*DRR>LyYF$IFL;cAT>XK89h`ZgoR$aJ#LMC
za)9`1XR4wg;`n{L1QB$^7x;a>i;^LafKT9Ptlnl(A+ZK;4E-m3?orjwWJLK}roq;%
zi40x)(_6iy;3+1{6EDI@p1K4LshoySai71RDa}R_fN}k7N|NEA@VZlLU-aVC0ontR
zfA&DAxL2Tm;dzAIYDHBPcG98jiCM>!fn5IA|vRVL^b7WexZRdBs
zBYjgcX(`CHGm!H#9ug`Q7iM^QaGhqNRz=V53nBqPp@fPM)*~>3h>Hi$ReGwLct`c8
zhG1?ILsjGu!>R=dtks&=^S^|LO8M891tE#&?^c-oa?bdZ#w0YI8Y}Kntk?=NrXmXytY?s)2crzv>F>(yVd(BuhSF-amFRdQ@2DMwVCQ?Xb(Qu6WT!Z~0
zmITcCN82=Rnl9C3_Q!S@YH}yUTx5{{h&>np*CyHn#YlIUnFE`7TSGE21kyc=^-j=}
zYQbZ=TC065EO#+
zmCO2>Sry7qVGxqP%us@u=8Cun$#c{3!kv*klD*QagqWN;GX{<}S*u*Owi7sov(~2z
zzKVm{wYfWvNmir&14$VqwsejzP1`dasr&Hmm6H9=a3P;=$)wMSKwT`fkQfCTs&
zUS>53BYTZ;W80j4vRqpC5BtMz8+vzMMTZg@i{8UwNa@;QFC9xv(AmxV%!WpqI6QyOso3Wht9W
z+If-Sj_VhowkofE;i
z9W=n_B7lN=l}7pSrV1F|TpWI6$n9BXFiNKr-m*B;4ef##Sn`EctrBl@3X6NBKB4%v
z8yCk;r>;slR?{X1^@>6_vL$Z-U~}(2z!(j0=g38wXX0sCVE{;AnCB<+YOF|oG-gPx
z{TlsMVH@Yoq#p`xW@+1mf*e9>g^87x;ghbldreIQ--%eRz5_(R?X^6oPNQpVitYP+
zF8t-~S)8*K8KveyZ6l4WGPPvJV(Oi2!*x%c?!E|QK0v{Cm*TBR#4hX4Z5~b2sSYl4
zc>%L`AtZ%a{o=@U?r!?-`*Ur&)*EAOTuU>@Vv`Q%R0z6g5KrsmxxYj#a;BcJ7mm35Wz<7K@O
z!F(y(p;WlBt!qV0X4-tP&i&IGyawGjHT+|Raz~Z+)jULqCm1A^Rk{-yR8=*_bC*d*
zuMVbUR0?p9tJ@0pvFzY2g*P#wytDBvKvXmd?@b_qar$QIE|i4%B5o~c3mMMGrAHZ9
zTV|bY4Gna7UvxO79!!kUa>x5*!qnd%b8_+o#H6p+?KTJ4ejz{)vusVGS`#
zDFshEbGMyq`iil`n^{_C879B1Qctj)!^%1GQcn+*)xF?{&9!N&7Az(Qo551e#7@7;
z6iG`C3Ijnd>pD08xszgp9;qp&e}a8(?bR3lPsCp0f4-ri9(yGR|0;`Bg|4fqG=CwV=_oiR{e*#oYGhM
zyKHeJ^$ipDXMJI5ymd}zWbEZo|NKkZXL8b0uchBAfwVN|7>!+36K_AZ%zb-uZRn>6
zfLwCs{*WHW=tNvNyEtZw@&Wu6LZD;E=?l2bPueQUfM1*YPTaf3#KdPbqT%{`b+FmL
zJXWo7^;ZV9g5r4PbbfLmV1qFN->e~>
zHr_qO{mm?u*ol)laH}W@w&?Xa{wmBM*
zAN$_Zc%zpI|9t5K6;J!j5PYpht++`pVheytfz
z@(ZYomzQq%wECd1?k>xo-S!*5?%?t$e7Hk5*(F^0S8ROJQATD_t=0e+~KhSd5S^BVp&a-dTZ%ClI#QPZ94QxuDqZsQ$K!LxxT}2cGrW&(c7K^C
zokb^mN9_K!rHg~;hUH|a4&K?w+edz?_Z%Po=oD=ULIZDH%VNA_$0GSc%BMjGe67Mj
z;ssQIMFt;8yqvgG1R@O;?SqO<8A)`axZbPbkEEiSYa-T+hbWWo9=#QcUw&CaJDJ6$
zr0DqMd#4`X=%Siq3hIh#Stag0CHY+O55ZFuqXVul9t4X|Mul5md?blK`LxY86mK!s
zF*-2|11ZVQRGAiuVQ+cp%xq<9&TKwkgWdfF%Dpv(^Nt31(E>YeUdFj<90dhO2*xaaWYPtzu
zhvCQ_kSt=ZmNs)wyOtJx^(QSVuQ5)x5Rf0NCM}om7J2}(j
zJSC0!e1PbrtTTuM)fuD2qnq2}box89y94_|eEUID*PO)JgE=ceyR=CO|x8hX86+6aUkwPP~ISKf04d)0yhyzmclC
z>g8YRAn@Z3QrYAU(!XASkjIJlT{l&eR~O=`?KAE0noze#{Bz&S-b3T$qB1E@`lz3@
z?yu#Yb10H}4`9qf$OaMvpxy+|)OTx=h-uR#mwwwcU8GufUM-OlrA+6lKgyj!pNx}G
z`foY^F7#P}%I<-ajilf*GsQF}xuK<1i?lZ?TJ`il`u(cz!@wPVZ^!AAGi0al7h-MYT-U=LsQ8P5;1E~@JS>%|Y$`Lh`)&b)L#=ktX3)_Gv;!KR5?SUMV*ht&
zC#v9Q@k??7hBEhly0BQ2>85ICM=WOaa`YXF^BaHW4&5LWf<#7N-TetFJLkIDs*4&l
zklv-}j5+Xfn;;O@a49!_r>?Ro6!Wz)*r@Cs2pEeFLOTV$_79+Rv&C~nTxp4+_I*Ut
z77uXs?dI9CiM+YZwbBWmCI|9B#UfFPVD2)REx(Je(Z61La_Y$U6^!d
zhGt;Y52(fdG?{yk(CrUfa0lxPs&jHo5<^gg52&?4aP!sVP#L>%Ijn*xh)Cq6dH<=i
z$LCeT_LB>_@sOS%=+GR?g^zN1maY)t5lc5F?%%U?SzxJq`bN;8vqbu@XG!@<1BWex
zoIUGhWfg`hYc}PoG7b@6_uG%sp3E(C&-9UpbcBEWqcyQ};2g5jH
z&_V|t8Lsv1h_>&Chb4q#N8RcQ*k`q|LYkl{P(QwEe8w_J?Tb*<_)}8jnfFH1#BRBs
zCA?~Gm%ypc^it(0nNe}e_WefJ8v;Z?rB6~g92Qc(%)cvjYw~?|ii7vbM{(=(zn-MY
zSn9}_)-S!M#n!2+oJH3sSw-fs+Qh8?G;pWSpsrHyTl9}lcAG3QiBb;fn&}}KK}QD8
z?8c|~jVRcX=hm&26`zXvRMRumg6W>Eox{(T+QakgM6FF=#ZhQ9B~^h9pIbRck0t`0
zbea>nm9M%&;FB%r=%L@SD^B3aG=J=J@epIaEYn
z{w<=1ub-n^wh%J9Y1N(1^2R+ScGsF6=Ds}>e_n~|3AeA!*5SDpZGyn8C>)*l|
ze%aculN2(93Mb^A`QuwcqO6i6x4Nx<{%Gp;{cp4`1u(uSMNBr-B{a@X=ky(iBmdI;
zrblhTPr-{j<%xk#m_P=R`*}d_^^%Hau|@~Zh}6O+=>~HaVzm_ni)VjIND9W&MCryO
zq7L)6Uj%d?F3YJo3gb=m^(@OQCmaal4yv>6;wYkR}DLgw*S!q|G35o57n
zCVN1|$*9EXlMj;_R5G)d{({fGU7-E+?v}9FOD)lwQ2KAY=~*>#0M)n3zf8Zqpt=a7
ztbr-Jw?rfY&BwxoAKiA
z3RfdXZ~tR6kfBt4T5I-138Vb|O7dH+QWhH8j803EZ#V;)@41GK@){*&UoI3dYoKW+
z5~YR?2P$w+QcvzY==vH!vlF>q&wtJ~SYl$>j$*<&UCBQ8Te$cgVZvMPS3Ii86
z_w{r~ge{3>L(+P-H6&UyFxDzILrxQ*0$Ga!Th#}&ZspF*La9-sRn)z5I{%7SPsBV7
z2S3W97a!PxvlFY<;OPW3x}KoHl2?<6u5!-MF2+_lPfsb^!`{yd5lJxi1mE2Wm!VuW
zX-O^jXuhdJIYWEJR%uqZ+2GjnSzO2H#&Vkz_G7cf^kT~kE%mK^ZJgfd5Tl?2^5|$I
zhUj0?M2Xll#H;7~qd~I9%|WuhN5@7x-hSdRkX49JikS*Gs4o87-vrA)l|1a-)ko#7y0vHcC#Lo^`>^_
zJ)#_N&Ob$W%Ad^#pXONDyT@mtjC%ETQawj7za@2r?wq8$jO}N)G$51v#-SoW8nF+x
zV`0)CW-J=f|La6Cs4E(}LsBgD5m-(#-}XGqT7kXryVjlcA(U{b6^lc@F|{6eOp;cW
z<#(&Y!EPR%<#~meam!5x$zK)(`2Sumx$;0F&_-};Mu-t-x3R9HrpWv-
zo`S_U6BQVpm^_49Q0?uEgVx|VXi$-nv(Ri%WvlcHLd>iZstmwFx%XOA*m+rAEh8Bg
zGbiN1&9s1r!uH&bYamHM80**iNqG>4o4OdTK2uh1fSfdiT(PugVKNql0ega7oi$x{s}K^IIfW=O*Fj5~f2rX$M1j?5~oH
zKxJy?d4LHxS*~j%%w1{dC-*V|uRZX5MYJMYc0{Vd`KNe$wYvJ7GUKO#SfTi`XMz>?
z05P8JLlJtGcA!{`C|8nQYkD+p`SOg{(THX}|19o~rS(Rmt;JNMt@4Irp6xr9w{87<
zL7gz#MIOT5ZF`yy9!acUu39^!kd(l!4)@4fr47q3=`fLX3~}LRMCj-gu>2%GIX-UV
z@I^b#AsHe-DDrUJFgYbb
zyWDX(R|}Rbt}@d273G|U9G|_RTFE1KZHw;);OQ+5#&tu!kWg^{+2nKy!HqH#0u`*C
zAEcJAWSYNv5rqn&*Y;rE?PjEzJ%fO8f0)_>9LzB1!D$^OXOY}{}_sQ7@H
zb2}uNovCG>DIbw>l{rkPqpjtZ8T0`(doY2J74UYI{MM?|L4I>~Id_xs<2Uh}V+rws
zxm_(^BI21dMAGl3hQ<-yQ;{k_ZG0+9U#Zvg{c&ODrYS*0UNo8Iffx5!K~_^?d%TI>
zQW%d(FV}3-TRL+i5!IucH;b)$<>N8^Rem?deZwFP?s9#`AioM|8zH9=>fVwg258r$
zP~q`p*^U7#vBvRn`|iojU~1F(>K6t$D6!oYyS#h9EoU{?$KX$s1k;Cmzh*o(AKiUa
zU3^H;7ztW#h4|6y;_yJZiZzO+)=Da#I@lC45!D9qU2y}4NvT3Jgua)>sXQasAl+m2rHN)O*J4xS4O2G=jft`i>nDwaWF~^
zUJKL!<4N%$;(T&rZeV}$_baTRB#vD1u+8fiwR>PU#+h8Fo{p_TG{C)*f>BB`%?`xwv9CfJ_u)_a<+~Vvu>Nh-Uoq>}(B4J{|{lMc>s$2>9&i
z$&Z+n|EX7Io|{37wmNq&VNK)}teJi#r1uxr#6hse^M<4i3V=1z*ZzSu|68d@tKpA9
zZdoxaGrPkuetx9BqI15C4YPi**@v0%rg?gsuC0r#>jYd{KOkz8VY%Kbj=kQK`1=u-
z$rSbT^{%b~Y5o0sh>W=+M<;^|#KOx+$zA);O`&xjSNdd@>I)ptEx)R$rH!uy3b;g4
zPwfFsQdAug#|u|V=gNK2ArZE)U}?p1f2v^P34EGO8S5VNPht(R#2Tl9py)4w^vdnD
z1s}c)8j6lnCC^O)IVuhG3{yVwZb;b`C5-sK0ND0q{MC2A2ojCgDEZ5(L%u=0xy<7V
zt2r3bO^qwMIK2bQk4sRNj3$2`at?(ArQgt|dyxDhDNOv!fSqq6;joMYt_NLZ$rc6_dMv
z6`}Bk*YQgu3d33SgJl-O0|3FZD=A}Q2W7~nNL;{+zKk%I`!r~m`mV&{zBGShn6*6g
ztG2*LZv7&e3PXd}8R9!oGwX>guD+zYTZ`=0`qY*lOkBvQ>qE
zhW>d9^421S!#%3)_HIcMjj2Nx$x_CHpyKmJZTnT*S%=zzUOj7G_wH83NW!Hf(A9N1
z!Mw!CFE7on-YK{KSS;SLQUY2Y%Q_`NEeRQo4gR}>q}{+-Rh_`+vW25WGJ{#28uR=4
zB1?$(4%;q6YubiTp{Hn+(JNZilb@c>#=CYL&EA{05f-E@F&JyprX!s)Y?P7?k>y?u
zd+Lo_Eo;%c8-5%aYHobIWe(AKCy6G_nVUhHtRx(1!c3uS97!61^OM-woJw-qg`9d6
zN#lE-92TITvSF#xNmabt;sUf4AG`wiT#kgbiGEC;g*7d@(LQ~qspK$sCmGdkfI(8z
zv?ijVAq)gi}7~Y!h;~
zQ+Zm~oph+kAV+5C-lb^FYU0qZ4Q_-GOwd+rIDr&Ds*CS|@#^>~1#X$iz8#R40=ce|
zES{#}HbiZuugu>qZ?9UFFLa^im-t+6SRq7IbiDpHzpkv`dkG$zGyaoXQ~8)!9s(=K
zaSPS`j#XIyo~cmGl8Z{;?Db}hPdXzVPK{|w=o&t=hj+aw-1h0J?V6N2&%ku)Y3N}bpyrHi4&H0tqgI%WA@
z7ZZggZgWayQ;#8xFO}9yTj)SK-omIJ><~dLtJ_`I(jA2rAo?a~R1L5;0U21TR(msQ
z+i})X4KC!6&ivv5is5k@kCZU)44Opg^s__V$pxr`IiAJ;OK@#6DrJ`YWF-*r4bkHA
z$w3*wD4M8~hMw*lCv9Tbjm7Fx0DDuey#2$%Gvi_UI=HC*OT>+
z6cdS*kclkpDrWA-`}GR{{mOXri|-m6@UNdxT~+Tg;W6%)iU
zasI0qB4^Zrrj^E3274ljPU(I~mTwWkCg&OMKk^gtAwShbfD{Tk$Uw4^qMq%lZ?Kwc
z3Q}gcEgj1hd;1P7nowEfuGxs%y8>DSkEJUrdFF#sewUI_W2{cxPSkn5$V)+&MB&+nY=l3R=rqh|q
zCNqn^;&R0l$RFgL{uVcx{_2z9ZXMm_MN|hfe<*5Q5&H2|S$GIL`zfFfp|!4HAV>3v
z2v~jDE<~MqnI9ypK6x%fLytWIpoaP#U8v#s?>!`
zeFa)mmBQyBXy40@^&dhM;5oTzbgy9edbdA9C1XZh(zfq<#!SYHVAhN{_SpP&C^*|8
z9|dw9EE1b^0dmL)F#qwR$-G7SNqi+c9r{8=_q2gMpa61~s{CVg`rqM*t)Owk{cC;d
zr6f#wP<|fE?U6>7p@(TrR;9u@hp{PzLg-{|h)Cn(Lh4F_NRZv^6>C@vGGC((7qKXRqUK-eX-T1eQGrIBxqGkTLyQ2egI&0R?`in$*5FG
zt*?064lz-m0(eD}*f{{i7gTo>e0M)vd0=Y;=AH`Rn+ZYl<08YpO4_NHvo1P6lPIf5
z;&(oglPD$%&%{SWMt_mU4SYt#A9bipsQFf5f&@E`hxtG-b(O?90NK+5Nb2V1VwoBi
z)uE!{6B*+ir4djEXk)S)D%4hs38C+pSA6AO6`pzZlCFA`eK*0)g&o}@&gTEEck;1Q
zPM9cViRJ0%Ua|qJ?YllfU%qx)egS<#s($IkFEUj5{dcu=;zi;uPD$c00QkT5u^=c0
z7mdJFe=;;Fc36ID3^&X%r(=*XGONG~qYph5_JLeRl2hq!`Fk1$BJn*rxOS-=Xwd+5
z3LJ=>Ys@CmrBf~Y-h>uX*VtZlQ}|rQzUJ;$L|~h!)7j2v#p%qot^-M8>3xL9BEtBP
zK(~Sqb1HSUOLr24{Tm=aLUx~z6f6Z8SIOH-0CH4eV7~T4M*?Dl$NX#1*unVn(c!iy
z9Q-6_bpfj&^4e|fwIDnk_Kx=+idSg95%}!?jnG7KLM{p>vM?<3(?LUeX>*@DD_4T8
zf4w`nH0ULuhGEl`&$X+xBR}JB2KcjN4*Qsy3^j>XV#GcwlDQ*g)P!u^n(#L-b4t;^
zeNDW3+n@V>93@+i&v+06JPc2g7Avw}y?#XDk=6H<)yZlXs{No1a`($$L1
zoOg-#=^@lsz3yI_6Zv3)47H7ND?C^jj1k!Nb_
zE6wUpdK`67;=JlC;I&KbZD;4O4rpM>hI0hQB)vEE+pQ|OmlZ}
zizOilBMy9--Gyc1D0-`8DXv%eNwcR%Vt&37^1D{hZtKy*_83qO?ea+qI)ul_b<$+SB9}~+UBg*_fasedb)ME
zwZbjrk)2FbP($iq55_ClvX?`+SYlBzqRXv7tAvKU0)SR$T@fLV@>hJ#YTq10mX}HSl+_yGt@D=n?+{c>y_HP`MzkbMR~0!iQEQu
zsX<(tna?Usu|p=nyn~T$qlpM+RL)~^Dcp1?Uhl$du?A`PYkrn>r>pF2Ka+&
z?4))T?+ix(kp3>7&Q0T->zfnp^B5{Z6GgGi2kU78E4P7Ry0e7P4fvxJ6UPHq{3>2*
zrZXtkpK}O4X-4+=PBi@V{SQ6jqZONE+bmP2!MO3@EayEqcn-iE4muc5Rvo$i{!
zO{2}+4N88l0)N|4jV8EGDlqQ8yVm=I&0FklpwFhwUdhq6dfcd?(z})S(K!}F@(=3h
zyTXor3aA*+I5afv!w*gN%#jw(1b{+Ta02vk)f&Mfj2U82xL~_l!z%Cf$~wHtxoYi4
zqk44_#Eq9OuV+Vb(3REwp;RMZ~$qjU*7N_~kud?rYY
zJ)9{EtkS6%wprTywV*GYIXN2bXS|RsVrU(|?$FJ%df&G2RM>@c^_8L6%hY)7Pt$P~
z2WCqq>8>M&}r1C%a10X*f|!2
z&_&x%TbJSeS>9>Jl72>O{P`y)c;uy7HMAj
z6DyCdoBoB5PcPvk&gr#VcYxzf8t6Iy)4b<@1p4QP{qZvw_!m9^ODfebC0-!#ubuxk
z#NYw$$QE*Flr~rL4{BadwN!Jw$q)XBx2O(kvX?^}T^MzD!1Nb4_DGFb?Qz0B>*vzm
zqs#v8v;-=`ES;V5a=<1Z45&gwt2Gjg$TuJ&1N{|Qj(`kJoIS18
zYIwnR!9)r$8Q=A*(5K?J`Gp0(@!)b|Tl30d%IMnOMtcZg`X`|u?M~#7@$9WVrWRQE
zzUZrI&?(TX{_mzixdRWf&%BrLY##J^b*5Tl$KM|B8}~QlVv8=_xvcK88sDy~9}dgsRma<RFpk%)@Vx|l
zEv@?GUjNqiZJLd#kKDK=sa+q}<=NK9Tpr8~>LSvysc7xkflEy+nBSqy)UB`a(CL4tb~?*
zB766378DZj4Lz(+^N$Uc{(1qxK*SKm-{>vNsd`Te)`$zaDu)V8H|$dxqRiX`w?oAL
z9pIz?uLk&F>0IsH&Gb&L6lbpMqC(h_=qs|=%|R(zyp^1GGhcs2>U^MZo(p{$2{eisqApDHbLD`F`n7I6^g4^q03Qyl7G*5)Q`=!uu9@+
zsf>lPcmBgk`keBp7K)F()Y5I
zfW2nudNAA)nv#0I8N^}p3JsVps!EQA*Lh$CA9z>NF5e@|2T31
z?P(p=x6J>V*sYl%^=5J|!Bx*s#wyatbZV#FXbtYA5mYcYy{u}x_*SCJA>lGzDEF$P
zA&Pfu#@v1KNcn}ZrtN6VBrO>e<{3OcsRN5+b)en`YX2mt
z3(gD~bAiC@q$Te%cUd0F{NAfLG@J48AoQKAmn;3gmp$N@ioY|yMgJ^NY%2kG%Gx;K
z^B-X^=im2v{vUGX|9yU3j6|rsWk!RFVinou$hK4bxU-l6DtbMm9jkQ+=uY|qX=4}v
z$K^#Zs8oMwE}AR>=bx>=cKrebkAI-Mxc6gbu=yL0|7G*H7II3*CmE@NljT&<7vfDg
zH-cJi!t(i9_2bBWj>lsHSxtY^78_o?nO}8(^(@|LZ|#0!J9iYln>odq`FhhK{{9>5
zU>2j<5joqVU+2=Nr-(R1W5juVLxcL-oA_>SJT{;Fu4VgMyw8>mCLgj}EW=EfK6Iu^
zyG-!U7*v#77R;#Q$LiA#t(Y-r6@GqDT}p0|?kE`6qy?vt#&^!8;3&dpAR0()(&Fe~
z^$KIJU(mb6!?|?@kE@CZ+l3v+`A$BkP1%KhHqakf-0u$0#I{H&7)BrdSohZI
z{7(Ba!jyvsNpb(1tG+YK)=)@99Ut4Yk~>}LP5+4IGlF^V70V-->6~>y)mzL-{+YCt
zBozbq-c{Og)j{w1i>2(EsV6AWiE2CAsiE_@UWOk#N!eUJZFB%R8o&0)pKv7O9*l9t
zDR)1tQ?pp_V^B8XDm{e?exA}Kzrn%icZIv1+0G_7wcL-wd8O^mE{{p;tQY=DO>N6s7*#eE=k3+ECz0UmpK)eWPof7{~`
zFm<`<GoM$Y{z>Q=&q6bn?MKAgFppv+q~g%RmFxtBA9N1E~DwAb}mr9
zN(?W^AdjEfH&q~j_|Aj>@DKl};(}dO2#K;vJITaf@dn&7xEs%UI}%*ImlkdCR+{qE
zwqYVx|AV@>4y&@=*2Mu4K}5hQB9elD0@B?`sgy_w2uMo^A~EUiv?%FrkWgw8k}BN|
z!c@AuInO&mzi+Lx&iS3S_x_!0pX-``F{wG85%;*q{fsvy(6hvof+j^uPqfdkYd2^=
zRvecyI@|gE?Fx3>eUKYTE*?$`)da^sJv7$icH`T+yx+A+pJ~$^>1$*`;vxzALP;8R
zyMaB(V~rh#mX5d+i**`y7wbbon|V#{XZkb;Y>;horej8!l(>iOBmnv8bL&*`l~mJ>
zxB%H`)}C42x(Xe+a&f|Bx~9}nW4#{Z8F?9k9wY`@5jwK*RR-{l^I9fpd>KQHaWJ1
zG`>y8Oxde5H`wTH+^nE_J8te6-cC7d;1IsPrx`@qkK^1c4>v6>d{cjcVPmLAutpt`
zLIE3RL%$3o0ysG;e|P_UApZ1$mt5NBs%In-rL4qef3mt9rb6~=sO=oTqIME?SyXN4
zkS@Mvm}&iO`E4$hTd7U8QwDK^PcF%Nb2)v<3A{UvTxu+SES9kT^fa_FN@x&DJB|M3
z45NAnCC(+zmFx(kToc-6&-x9*0n6swXot#5h+@HC%~S}b;8VM?jG@csSA
zHg65nYL1&EFhj7hkbj9n`nmZ%9%#zfxyJGfW6ye*V$SM)f^{(KJUubKY?<`q`fLXjM@6
zn>TSf+lbZSiyEIS-mM$kExU4UImvxiX$fC@rbQc~XGnhDjYml*XN+JRkD;8N;(#S%
zG%4qTxq=MbSXZyTV_mLpRcZQGa~x>1IG0pyQS`?2IuFEs>2;67&lY$jWj%nVeNNgo!p)_
zqzrcrvj)8jDK~%VCVp9iQX5`%3bPmz?0w)&#+GGfMo*t?Wn`o;w3_;88aXx5u__LO
z=0$Ch^=9lFf}jxHsYi6x957?kjj{Xw5rej&0n@<#m$jH#MZ9EOGjr?jqvZ%?2jcx7nja`>e(#=J7(fnILb#TDhkyVj<35e95U
z`7!d8r$VvV{};|BR~1d44$))R_OL{KG(`r69;E7Ta0_iJK!!Z!qH`;zAVQCtF(Di8
zMFy)aLiOrS^ZA|_|K(n*Y&?#nx8ZJiv)}KorI0rb7oN60jg2%2&Es~BG~BWHBR?V~
zsXB(dmo4{Yvzii@sEp}4_4ME2F!ojPv?v?e?Vbnia%Ic
zat5{t@hG&vv2u-b&D$8oRV)lD0wWA)XcP>bK-ZvXGg
zKy(l(=M{2I1`Sdv|7~OxN^k7o4G$o%EU^pWFG-`mClkE4UupDcE|dkY(!H<{GWIpE
zhs%l;@g%l4dM*YFYpexlk#Jq6A^o|Z_FG7P80?{JFNe!8E3#ZoU{i)-3d(mYCb
z(4nJ*{&`=8oG;RT>Zg#Bs@9D@4l`N_1J~Xyx=f&caGTL;TT!B2xL2+*daodbgU}4z8^z{0TNTkd6!__E>5J$U
zh(!D18$^o+p^m?i>do(pseRyfm8CBL19e<&0edls2*2W^YUD|)&GzQ|6u@)c{p32Y
z4t@}guVLquupw%)l;^e^cg-j(TkW?8+pnKi61~DJ_NFszN&R^fB=W>Ncey#6D~atI
z-m2=c#*_UF_{K1xOsW6S7i!&N^q%gw-_X-S4{66$h;3!<;LXW@Wi`1J?)7{#-0Ma2
zJecc`q`f}Rx^BZ{#HT+K{IT%VZFm1eQUA}IH=rh3ot}KYmrs>^#a_|?_u5f(MD-C%T1w*S-1
zy3c{EFIGH?-t&(ahQ4B9c8I)IbsU;+b|VbG6i^;|Ppb9eifpuv5Kd#<=n}*v@>TWNQ
zTy2x%I$#C-BGnkSvMNl@C11OAvANm!{uio)d(i{OKqrz)6zeWJ3wY8Dn64wZCg|d(
zx6o8+Hye888YV_O!D@wXhSJ!!f!E|!@LM)9g@+}twwXLJikZec32-(TcrpG7(h>Z}
zx6mQmh8QiQv;CC+t;^cK*_tXgPl5+Vu4f-uO^B
zqA*^uH&jZiZ;%Aq&+y&Lwnje=O=Pg>Zm2&FFJ%l{fW}qM;&&QrXHHJ_ue;h0+j>Ie
zd_%iK_|it5L*zwSf_@hD>~
z@ggbJYgfSk^#qg{>R^K1LzUVHvy2J#`am&JO!NY?vPFJ|k%Xc6Mm>o){{G}Xa?m|v
z>W_xRT|JgO$B0y0^e(Sbe@H4m|GdcUf}mU2v|sn!`>PoP7bmrDSnx-H;s9wppe?mp
zncc7;ln?~8JU|Pwt*39fb^S#o8jubL43PrbNJv|k5)T@=Ao%i^rnf+iwDn}=AO-Vr
ziufJ&s8nE=$+S!JjDjo*#^`%RahIHnPQ!4CWj?(SC*p$rRPQm3dKn^c4w96
zpAP2Z^n)WBYf)}KJE8B0(dpJc)Qg;kQdd1$gmR
zKtmZX?G5%ZEjcYsXA;eYeI{NAt6g}scTODM+m;&=5&p|)g>YRD`NtW7bV&Iq8D)ui;j}y
zGY;nJ_Qh(+K?gF|K=d^~1(M}}b${~5br-Qy1^%3Yx@)1^*pCdFIoJJsNjVK^e+kfpe4)
z(qg#%y|D5ams(87@yJGHU8*sE8=Vq5ymTTf3a~qr$tn{9?CL|m5->ZeQ!9w)&N!l1Q!o8+UD8uKR4Cy
zyXXO{#LY&Q9G?;6SK;$HYqTS9Mz7QS|G$s|e40EF&st
z#*>Qc@!d3e;k-2T$m+{b_3FH|yQ9GuY+C
zaiVSctqhA6GQm34tz_Sz7PBW<9yT|BY=Ju!OqHSEw~ZrYes9VHmEiHs+VOhB6GQ`O
zBZq3vQ9&D+!;5{&p;P$5bvE5z2=;V4V*plPLv5s>MD(?Ma3Ym$e=Ees^v0jG(sMEO
zvgJVT62wR_@T6rV)v#)}G?quBpjy`|Sx}HTgsZL6L=uM=UxkFKaY=%Z06b6^
z{?*{A{geT>eUti9ctp;-C!fq*%+~6C$*KD+qM5Y8-p+B37mYp8a3<2+vadsOgm_6>
z^$;Prgc4SR)knxn{IA|V9}e28!1vn@?Sx~s5Vj7snFWLFv`XpGFfrav?YtUs$yAGq
zUUi80yTO5YT*N!+RSXPWx*xW2@&=5MRWr0$Ozoo-uJvPqaocVZ5Qu{qiC#*51Fl2g
zTnh^P`BNC-E_xl?UZwu%vGltm0e}|4R)m-Bg;*e4ZGyX#klZBAl3D-PSpbMacqbz8
zf5JP!%<5_dIZd*FU{d*B5>OhNtoblM@!I*e4^ex7gWF6Y*?3<-ieqN)EnvP}+XZJl
z4z5@%v^TnI?-Nl|th%OuD^OoYVOr*BG&6@{V{L;E7vj7=?EfH|i^9o4@w5~h?s^mV
zsTzb&8e@GLyk(8_7Z>%j^_L)VOKG6O=he{0SHJEmnSe(%>5X;7$8m%Mt}hw)jV%*G
z|GRk_hrZPg^CoD@c+~F|kr}`q6OEg6WT@x1;*&O0qyVvZY0$wn#+m-bNp*#ys)tP~
zAzKm-k}%yk)iPrnqian29Bpp8>Vk(2Y)qfiR&tXL=tm8{PWtuC+%c{O+{9*kP%$|<
z$!X*D`d>9(4pMjbor1?F(oeXS;o9M1E*0q?d>{gyzz##xj!@|6vo^x%8z12fr)a^o
zd{mMvB)b=P`QSm_zNjb}I*6=>zHiVG(&iI$J$>q0&yMXK=dNRVT541%3KQOEo)oF;
z+(;SdZ}PVdA%DcQG9jSbVV#Q)_D{O^1vlnwidZhWT$zpSp*YS_
zx0i#Hw*#|~^xsYDBk~@?dAG0roaw47-{B!-T*HPB9-hPpLv}9rt@vm$+MV0R3~vZGDrdLC
zt&;LoBI^yxry-I==33Z(`SXM1qpyJpkr`vqjhp~6d*gVqTv+qTQDJ@j(n`B8%4
zWwd+3yv@fD~BzIs(phMkIzUeot-$QIl}%6YG+wKBNkW8f*U#KKfDos1n)
zMle;$$s78F8k?7HwO4a%b8RN1$M6`@0Ner>To4W<1cYPMN}L{&^wwInSF`me$zwX4Gvf|RT%*Et8>y14*Ms_|-kzR7#d77}?lI@nUPSaNn6oqG_tXf
z=`GdtZePBxcuWScJ_n>FpvuG?JpZHMGR&odM=C}n=#HDb^D=_pYd@A2?o-6_{@Zp&
zw?te-j2dY7#cn0$^{l)IQ5<^U1|t79>TrW!UEmV*r@zSN<^B}I)n&lwD(cH`mrB&}
zER{rUTnDhVh8gb}m&h5NPk>4_DZ1(aB|ukB@+T8*`piL{?$$rC?T%EX`>KFi!tEmH
z$V#)_x1f$J8SKNXpbDb8eM|!)<2u7G+fD5L)O|;j(Gy5f-d&bJ1Sb3Xk=!|wf%yDXK3a)l!wPiMsWO`cP@ovc{
zT9G@_*4CAn7*NozA*<0Pm_IDg6i!0>RZ=F&h3iH
zHQKePG#(={(J{?-E6Gr1h-tT~vAfEkY=Vmni|t^HEW$#RWw
za6$N<5%qV*YFDxDiuX)w|C>q4TjS3C?(288(^j;@ov%qj(Ik4#gH4=nT$ouu7VR~g#rAx;G%#?zoK0e;F8q%!S3ejbNc@m
zuKhGeDlYPrv+QdMsl-i@hABdc<#nVu=k}|;xMk&aR0Kpu>1OuuT7$RjaO|}*V90Q2
zLr(E9W$(+%p&;lgt!w5a3Q8i>ja2K0DB4IaOvaK(W6w~<;ugJR?mINB8PDCnN|*y<
zr>yB$1iXVf@ZDA=bRAS46F$R_M`Rw7XZR8O66EHXSk9v7z~-;TCa_TQAGTBf8yDh^
zg8^=vWg*quJm$8kL1y^Ap?V#U$;?N8Z=#y_ULGz&vj$b!N#01tZ8J~&Ai?yMs>QeE
ze-jP$i#kPQVt5Kt*ih(ZVje!ON1gYgL98O4bThxrqImN?Y=w|{?1L;*Dev3UU&7Rm
zG^D{18UwYj9XLWJ)XlB4#fvjFWa1LgM+Pq{5ZqwbyoZdEG*rUv|5n0L57PKIE|H3)
z4EhwcB{V+oF=oror5jczfQJdonP*~
zOJ}F_gAmnTdb7P41^WW+!0y<8d;-n?`E_uuY!aV6l)EcnprUF
z76%y0LCe}v|NG$sa?~nocWZDK(7ogxup@1TFEkqUOU@|fILe}nY`<&2G^+L!HkDXG
zwLkA~wIB7sh<`&sm}PVoYAd1VQhcp`L^qNkl0}13eY}(xwIc@?DELTPQJeeRuAu(+
z4SL(t$5yoCobG{{P{Lk`o#8bCRm|O;@=Pb_`jw&K)cMgcC{Rl=rLXri0b!P2DcJJ*
z56>$ZaKbv&`o4)jZ0!RQ)lvG#7}5S&9nDlj6DfOIYt
zy0WCDP^g;z$3yiW92VkC{-X|?euCXQFrcKRMP7Jvmyt0ggp{Xy_>AsYRSJ5!z(#Ht+pcX0b-sz>VT4^&iCY&K^Vhd=6`JK9J%N#!!@@9fXh3~-tL
z86cHnvs^yY-I*jlM>?_VOWmMPU4=jh>LpB~ql=u$6b|2O`2$@b3QXOCK@t`NXOkHq
zmMGK&&z?HfDQMM##|X__F+^T?A|_y*6j*w?J0=3PJ0<|NJLXXu<~4m$Xm?EB-*?9h
zfG`ESW9S)ro}r$&gP=Wi$?3c%O21OXy`k?
z!v}Elqa7|!-i!YjEY!EzUDeenGQi1uQh0C90{){G{bsv>-D0Ny3UkfjD~3CFl0Lr}
zdE63l*JsyLk>I1uX}?$;gANEgC|RIk+ysl5h+_y|gs?+FDHE~_zN@&6fs%E!?5H_@ef`z18W_EN=NGRFa~mBZ
zYR;^>l(5~UvC2r0F63=(ZTUss-1m0eS^0m
z?kzz@NO*p$=e|yc#>y^Of1J&`j)*O0bPq*-+>i2NLJvD=cvvv}B%Qxz-eGoL$F6*=>5IKFJr*)f
z^ZZQ;=mWM=B_8;&k8QO+Id6k13+tFR2`Lo|-`)$kmRxlrQ6|~>ce%J4`}WoAFhgP6dU2WY_#!|fJ4{JHtow+
zjNyJUI~Bx|j>}x;9xAZR%YV39KlWan34520pme847qM{{=hknTo;6Oxl2=GMqps8p
z>v)zH{q>dJb*{St!SF8b@vh~&%B$N;-0CWv3$nf0VJ45oYpkeN_m~)-?4)ajiPXHy
zbQ~Kk_WEtNf`gwmdn%+Y$}^>`S=lKJ1&izZAzo|k@&ru+HGy^ZP=@oTLmy*#KbKD|
zH$RKg)FNP=E}`@JGjW%sATS`xi8CuKZ95P|XKRCc(A7AO#1@&)T6=Xm>$t-JuT(bu
zN8NfX$K$=3IUi!}#4zD~|L@jn*i&xNDJ+^EmAxRol`?slupx
z>8oE49f6U2a8`O0L}FbI|_h|L#V)C!BL8C`qvrNhFE_`jf$RC09zMX=zJ5Dc=igK<`?%DI1-*aT_S;vPS
z)Z1ssIAo%3NjT!nx+clOg{n9eK_7{R*?VfCjCd*r2+6r?V^nk~j%{srOK?FJeWXB@
zJ##x4zWvE>P5ww*^!yIy_R0i{MY+>_R>o|!aeR{PLjTZP&gY%(00BtP#K9@zZ2Q%1
zzTe5XU1Jj3W|yOym%KCX6sGNVWUKMnoW$kiXyh#^MIUP1Vjf34C&vYXRSc-Xu_c
z*Zf<8|MpVRx18jma*4alx3V>QIERwJ(^^3wmP{K0ck-=TVBpG3MpDLERu)T%HPhws
zJKD-nWx8M><7gbNuaFUv5TPOC6kB|ZQ1SG^eVO>X&A>Y>4^hYltlsX+*2-`p@d)%6
zC`iePJ~orN?AT33fYMj^68O*aT?7yM6lEPJDN0(;Y$F%2>m0GHkqZ+!E-N#%*qCk$
z7^1Wr)UA8oT>5>jz-Kj($&v#zMCl@wS_VoM!5@Bx(5GpdCX%2Wu_T3QFk0N}
z{diZUmoGFk^Ny^2`it7k?H0P^qAlf*TQbRVEdih1VnEddPE}p0Nt}etWaiypmk{j}
zR?pmCq19^QQn6M3wUvsF@AVQHM}8w1P9foa6LQ&SXs01&Tze%E)i1x4Jk3Uo*RfIS
z9L`~*LX6`XPA4^#t;(ng`mC=u2^_H3+0Q)e$5xef$20E`IT4hL;|~|!`w7!~7qeAj
zIiWpD2XE(_@gMn_t&)>o7Bf1xvp7WK=)90uI$J&yp8}t7;bBXB_3OLMs8zEfsLgX9
z%<_}ozJ2@1Npj5|=09>U$2#7qSrGO6qV&&O1f}@gPb@xRg1BlS{^K+nUS4T_RKXbM
zNylmT2QKy8@^wYt|B-sE+*MYU5iQmn|55jK0H)ybJF4UAuRb!i$5bcdEu8pTM>%iF
zz$0CYSjWqihWpu*iB9qpi6)dg+s7m(j(=DPCZ{||d?1wkp)X*VOiA=bD03pe?&5U5n91Nr1c#KcNsVjkaBQ?1pM
zl{>eWMofl^3^xb$!@5L{*YNk(nkivOQqJe1T9tNb-`>$HrAUROZcr0ojuM<&x?J6*
z0SX2vtG0r)<|Wq}LI0}9V@-bPZLNJqM1jeWE`p=kzIpfxk8hsihvyT_J*mLbqKOaj
zl9jN^s^^$(=R2*)8r=B|Xn$BIWO7GjwOd*#sz|G7YX$m(yndrefbZ9ujHug5qUZQY
z@;wW*5Ph&L)+Wu`rjeyNhL`Qw`=h>5IqdX1=K
zrcsvKS~o60IlqMqzF1EcZXlnZxD8C({^7!q$)o+Xo_DC&CgC1*V}CzI4C_7ia-GV+
zizYdrMj5S<#il4IT`mo3E8}@&9I>sw>}$+H#P+z;O4rtajzW%F!8IxF1(UAi^I8tS
z!foc#BD>j|$;4H1RPl;zo|W$!33KCCuh!vi?jpl_K5&^`DL3oQ6r+!`xgxY$d#)Ft
z%&$>c=VAn}_{dXPtYp;N6IY~mo>~~%hJ!G#Z2wakTG_{OT$aY~ZT_0b
zdWLjL|Au#TLpCwFOI@N_OG3HIg0PpekH*K(NndR^_$aU*LMUopM0dr_~XF
zznuU}vOgJJflQl+pE0!87p8Pl}km$hAbWl=kthf29Go*G>k_
z7-Lpg>YmK0H6v>@np)3Lqx;nM7c80V2sBVa_Uit4Xs5k#fceffhTB^5B_`*)#^WRm
zCqB@LY?2ycNXq$V_un!d^ksNBjAdLV6USjgW=NFCLyvKBJHS*wdj9YlJ7iT`jrl}Z
zP>tel1;fk3x5no$e%Pb4D91HDiH^$9nZe-CG4z<}PrBJLtH;q<;EV9=Jh~UX?AUt#
zpe2WYil@Cr7;?FD1svUh-u2U=*4|i0<@(BDCYk=xmEG+T%d8h8pGyD|
zm&gY8#VsBa0uNNDPjF@jZrWx^l7|F|R@|~`>)M!AGx_H6!3vCYJvYPkXut0DK0;g1ORdnaGAi(ghHd|eY%8FXwW8G6UC7V|J8d;vckx@wbwaZUPgw_P8g6
zdi94TXuk^SgK}vkKYUSKUEW}tPphDkyll<}HYphGxMc>gYq=RsN
zEatJ}me^sYBNnV*Ox|HR+nK~^*3UFt%WlZpL
zPXbRb|I*ZH$eR8Fz+z@1$SprafTDhVMYe}&AN7D$Uf844=p{u(Z`z&H=xxbGys(Q9;eb^3i;=<tA#Mn3X!3n&k=E|XKTwZ_=%Xvbe_X&@`FAFf+~qgvNle9@LwuFL*U+9
z`Z^hbHgOTY*(Kf66haCUHyg;8Q`2^!9dlZ#vgncIKTeOaNN%Qdiv<)y#B8x*iqz(c
z+sR1T$RRZze^^Wqs;2)cjAFBea2N#efyd=${*%XDZW8F>4046yoT~lI-}$hPj*$Mp$`uxiKJ{<`R9;zuXezvLo2
zA4M@UQWU6{U6utD}q*k(W`USK=vA#L=#kp*F}
zhd^CFJkGIC;kD2ls^~OaJYczhK02ml0VKaxtqm%Ho$LfENB=s$4<-}FoqVNKec`=`T%<3gUz
z?xbJK5AjRb1^@>_jj31XStXzfgW6I#3YN5pEYKNU(the1Q#Cx{s4=luuUCu9$OSY%8~7&@UTQQrA*cC3R7OF91{SpeiX#
zIPtbl=S6mbgps2QM^RPdV(=C4W*`0mSsOPLbOs>k9_4SYIv*MU5DC-%+qA*GgHE98Cw`;DJdz~
zbw59su+hgihS71RT81QRf6MXpjEel5Z+)LMf_zH@_!d$M_*Tn5`&P?}5%8^P;9H?%
zv-GptfA_8F8S7%;TN}W)NXoht9(}48mM(6Cay-fP70<_l>p!VK>Nj6@+@H&si+5U1
z0_cP8-bksX!U|&JvP%9_J{B|z-k0tb@D=%js%;IuYH13GTVtMFZW6s`ks;
zTVAxMZicH$`4x!sQOu_lA(XIEevl4A<|4-X-Gh(^!_!T!9ib?RSe#Snn-`MJpS?cc
zeHGNlZr1TMM$1i(3(sYSl0;4tD8gsUq$;ik@+(rWf@l9;m?pnp(wj=r)fUM|x{{c5
zJ5R5y%D(Rrfz|@(?bU=93#fVt3&yY{Dd}p7PnmrKsXpEU4QS;;S%z
zLmXN_KzMvVdwDR#)pJ=AB9*imSQcv%i40tYc@FUR+1#qU;yp-@FjWK=tvN>oq3qzX
z#(cYxMu8iHh={1}DID+~J(VlfGUmd_3_t~Nz8L152duI77tJb1uA}YJnlwP7+;5Gs
zJkmcmwY!F#TP*HK_ry3%dGA2$k48s|3Qb(mtM}2*Y?h9K6-onN-d97Y_wm1%wz60R
zWDi{clF*90r%3-0LJ~LjYW#xXqa$8w0{;PH8Ri6a3}=TE9YNM=Uz;t+SiTH7*l8U^
zC9nF8ZiU}k01dJKPiHfrspT0h%rc*DjV^IHup_hs{cKl504z&7jH1pyLIIx?f2Jcr
zfoI&F#%1qUq~3gEV`FcDj)<=OPtsMuh>~JhIf($Az>^d9G(S}93VrQA;DaYvlP>o<|M6BdUv*)LuYl>`Q8IEGOh6L@4%w%2CPeOP;|i&AD|E;7eHrmTxG`^Maoby3C-r`9ArN*xn>1mR0+zcM*N8qEk!3jvV@%@Sd6A#64I0*ijFP&H;ZG
z+Nh}Fbx06BZ7Tz*N9OE3I3)jGDHxPF?3G=||1-wLuh!VDz2cE7&q1GtFmb`8F$QOHs5#+KZn^)v;^lX^ww~(BwP%JEkg``C_l17UUvXrgXb&pC3JPI
zuTAHBh>D`_iIy@04o~X>^hci|-l<%k8olyNf9U-krP)b8%5(r0*bZZ@+moD@%l+DU
zk(aU`rv;9@FASV0wWM@e{c^#k#ZMFXVk~t2vq*RLh?KRvt+aA&JNRL>eK&_#Mf?ecXGz@PP;H>YuneOLSG#iJkyE_
z8ij{ZD%==~T|UbIj27?usSpcL4WGu-(ok|`BKRR(@IwPKS{WY~F65E^#PxO0Wi~*1
z&RtZOy_u*9Ech+|9QdOk(|%*PU}b^5I9?S4LP(lt!kNS6Xlp?4`8ztf=j%Ve_2#It
zWGkll#@G~wjD0dAZj0vd^Tnr>pKv|Mi!uf*r^(^&T5pOpezta%qxno|!1=SFBaY>M
z{rj1p!Kk2Xy6dis2cpsL3q;PL7|dv0M2o2^gXujYJWVCBu?S_w-uIcNay_^FT~qEq
zTajnDt*JnysjBoPJzT=@!U-~ci4Q!dIA$dgd7GKrab5PG$;;exk>_*~#pk>LoD~I`
zt3Iulkb(e}x#$w@-q3meEhxqAB&Cs|jCNsgPm+qEqH_$iKoQw>*)clV+?JoApyUWc
zab(6O{AW>~3B%Ap^dK3;3zi23$Vx%a?91mXumBVEOpdBH2aY#d`Mh
z{^(2<%AQ+(u5vUV;q+Y}YLvcI@GQltWj!Dwx1`P1Ri-k1X2X3KFp><6Je#ZZ6CK@OUk&GaquhZsU=URZ`xdEFb
z{q4QK>L)=FSg{0Irhl<3Pv=iS_ut8_aJBVZGesFx{zd6m3N8%YxqEP}v8~4ge2e>v
zJ>>g*_fdu6#|Irun}vIDp`ku+K%j!&SN#{2d_*niA}MvLW4O#z^}YgqLL;6%D4B2N
z-lpEu85P>>6c2FNnd^~`A`0W5&qU~Xe5;^8g|pTL=)tGGH=4&F)PKheGRaEN
zgU>eX1IXUHIz2^zwX{N#Ph8oCrg@X;0zG!>!TU$fs>5w?B9WS&y?%jNUMcEYWilN5
zG5Qm^+|!~d4K?WG16+w{qxQxZaFALRACvd5NQr8qnLNFMs+&(g`@HnR{*&{XsREKz
z5UaR&ySE6!ttC-Kv*oI)Rs#E%myIaXWt3^9hF$mg2pled*)gky%~yx4IbyH&aoaYzkA>{5iXtp~_GOL0LrH&_3KC!D4U(+
z@KG@Z!8Oc6!mtEj?AUbK=zFxZv<@xjVeqrTv9-3f#2GmzYL4pA&50SPVP0UOaPQAV
zA*gIER8C&A=Hb`uV~NBzW3&FF#xiPzrKgZ!KkQN&gj_n$4yb3C^``h
zo9uMlrS?zbF0FeD75g6}U_B9HmKrDd*NU?mj?M(5uNQhjHE>lS0hOuW
zz8DIs1ASC52w01nBRd8rCQKTFulG-TfD1{Gyb!!ZDL~9#h!NC&cvlI`x9=ML>6-|7
z#_!6`6>JzJXLHHDBiRpuc~l8x{b-x~Orl~F_rcoNM2cjVS3a=zDLdinM}=HYRLhhT
z4@#v94k;*E>QEza4KxDEOE^E<|ab+ro3na;TKv6@Rw^_rsq$X?wBKwX;I3`WY&anWOSwH(tf4umx?_B0r%0=>Z7H
zeMK-M@$6`S3j$*MVD2P&J7!5Lj6&|*h08WK0u$&7?WY3hpDzv;)_FH~WM#xZ1i&pN
zT`tBG_U}+-^}+DVSoA%cey(k|e)R87agWO~%-+CU#6--GFYu}#gZ|Ma6Ow5#iPTsD
zq7$diJ~BTuv)X%7?cn-Kh&!>6Jqa>TAei0bFN?@_UUL+bKz;6KF*<*Me-=Y-#a^1}
zh7luZA$d2j6ZOH4(ci7ngTbw%=a~9^gpkk0R>;5M3p}lX>eG<&;fpX$&b
z<_Khn4S4Xwt5>f+nrV-l15>;-Ks;l_<6?j*QbvqLMZmc8hFmsilxX&_5Hc)P?}3~B
zq=wzw@DUvySMXb7)_EV~@S70Bg4!c5NmUs%2LW&L#QL-OwQ1_BS-SgF1(ns*1GIwz4GV6CBft(YAYiVuS
z^s$L)&aM%lP^QNV*FSxATK32J>B#K1m^=v0x6-(W9ufTMhv;+%c2Wp~RishIoadLE
zzJ2AiVT`W-<0g?owLpU~mz|TJoDuK%(7P{RD+UXng2}=nM0OO+%k~NdCmuAjZq?E^(
zy3nzA!4}aS?dETGx%@SKeo&wEeSL!zd}kaL$Vq`vPICPewlFVADllY*5~MDxEzd}q=2-GtuIHd18#B4pYpU1)jH%1B>63#<%HUTH}Sj}
zLWJ&X?J({+G1l=1Dc^%cudh5qwBRP`zxB9sYTD<;_pvZ;60T-D6(9{Vvi_TyRwjZe
z36wnAd;IWaS%9@lScAU_L!6^%DUfIlms!z3Qlfx@wiuof*BaXeRoxPOFY*cdjEzas
zmBH7|QV4$A`9~lRW&w_9ig9n}&;@{!M&jmcSFheL{e~q~{cE|6XF@jE0U5&M4^8Qm
z&xp0>zlBHZQBXVZbM3KdG@wRI*+9Q~s%rqUGtRc;iB+63uIjQ7d^g`&An*noLm@*7
zsK!mqi$mbnaBuHSpjK%k3H3}h!_D@Vr#+M9nC^R=zZy_|&(EF6o2cgJ92kE6E(J0w
zx_xadDlt5o5Umj{jYfWOlOO`n8ejx$D_57lB29BWClrNldu;<-6>puk^Xy;>dxrj*
zi9H^?qYEG%-px`>!9ww|b3GZbnBF2G^H|Lq*QzqwFgKv_$lG->F}d?%xa6B%CHaJ-
zab11s8100o+T+Lf0DMbL3FQwA72bcYD`DQ98o-JybYK~&H>v|#&K`hJWGdk+*L3xp
zf|I&Zq+{ICg-p*NT@Mr0d(^T^aDDhT#D&M)E)3CUfhbBapgz+AvW`cmt8bDDqn9=j
zm#$5A5RkjOyJ2YCuGq$!H~%*)Bdb9=hn~|N2`u+^(@Vj)vJ1!syM0&NY@zA!U7hM!
zP*a3-KOO*r$@^QVn;}R1gr8-E{<){?#H46SH8cF?FZGoYw!-&?PWUDoW_Io{GhcfO
z8VyE^Z~7eLU_A2xjOx>13WAEG?<4nvDMjMx6yll6R;|Y!X1ai!{_wNs8r#`CD=6tc
z8|_)+`Wg%$@VSdNq+)r%G~NM8Cjd2nkM{^KAi=C*FUh8r{XT|82#W>MYmum2l#1!p
zzKen?Go|~FAkB@st1A+;5-@MjN+?3DM8S6b4IS0bantR`q+FJ+WVW+5@=spa=cAfe
zdkcbdGzxz;uWyJ|-SIxS%5z*q@o)abS(m8(IPpc@J}%L~c+CdBWfF3%CU4OvGl3{x
z^E=JrKHHm|7dLookY(9X??{36kO+{P>VH9*$Pl!NuT4@#z$^i;uie$?1kxr2Vr~1R
z-S`lFMz#D@!-l%e5Zox}SjpCIymkPgMw|oV=TvZd4-P-a34rzISmM;}(
zxD^2fodswu>JEg%#%vR*
zanPyNp^^1*eqZgO4aX0%u;t;eHon$ReIGAJeQYN}5sGOPb}lCm1GKSXC%QCYJmiyy
zcH)fIxyzt87gE3X7y2@!saB6K)t54
z+8B8=W*wAFX&twtPpse7Yl_NWBVP%PvXEl!aSK+Xukh6D;l%-pcN2((x(eKm%rpx1
zB=vUZ`*P-Xk#Q#{o>bGJWYcG&w&)Zg8`MLQz|nc2S_5#~At?ls44ns~=u>{}Om0w5
zkAl6zha}182v)&v%jK%jcS}!fU32guwmbaS3=jBD0Wr`A5D|es;LudZ4zV@WdFO@v
zJMy<2%kapZ)D<2VdOa-s@;8oJ>oKD1^AD#WW@z?9SR{YVqGQB$yH^WJCR6X{Lhndj
zuJjdF-z&5^tXEn|E4MzTTjypnkvI{s4%Nqdhw;~V!7yYpcKa9(C)@Jk7(THGIh(?K
zdaT>GRq*}?`wy~azIa)*nX;2AqWu{O&5*Sq`Ua}P{pd;NJd=K3sU{hNNvaCmr91HLMIA?{N08r9jzgS3c
zVt;EPXaoc3yzUR^xK&*@6LQ8KZzR?svpM)`%zH+?PkKrmoDC{|_(5fQSJ$Jk9rGE&
z+UO@eAm2qcr+BVGVw$N8)7JZ3r^O}vK-!g{Q|$Dt;@-_4s!~EKX~vcgOGw)335{JD
z086*qSKYeX-Hm4>Su;+(8Sb<4?UGfndd2SD4@XnQ%1@ri)=Q0xNlG?`>L*`Ck@FHJ
z?O9cst|YrV;Er>8cZhP%$Jci6BYT$g=^p#Z&q$C2
z3%?WFRZaMCB5Jng@$#jLt*pv;Ep;77o{6f%tRNlj4%kA2kU?S15+AK{K}GR?)O9H4O9xaw|H87CV7Af6(>3T5YX4Q+|ZdqD+`|t#d
zRE50G-gp2j7=FB?T5Vi5pPe&X*i0ew?4w1ROx4hmH%vYeAo+v}yCnu$IJg?ufF*Do
zI4@@>R&bX?mxIDBJTek@W30{xcka`Gb_yV`TUD81%>co6fpfZHqxpX^0PGR>uLV;~
zrQZx^c_$>erTsx*#JsGB8CPogmfb9UZ1Yb?!Qa6+rRMRWVJ+}z
z(wWj4q4y>wr*16Xt*ydvu;n$d+
zP+bakCLM2&6Y$|<*_!K43u2v{ME5W_1NgI~XG6RHN`OD-jN@f_5XX-9;kOn5)Lxhz
zr{EMe!yS{@{zoQrSi_(S@#z;W;hLE{>_Mbdy3%lLgf;G-%Nz5)z%YL^F7rYE50p-b
zjHUU{#>{$szxa%hvson63Z*}0dtVPYl#^tv8~&KsS&)o*$kPq7%SD&t?47-2uH$O)
z4C~PnlA*yrhw;R-jxVnIQ?i}BwI6Dzyy)>Zi4D|uHO8~~R{z7f{2<`sWhYAaUSP!{
zkN4(<)#~^=>OL4h*=Pg@F>*4O$bGKe^2FIT!3@j^uXz5|Sde-^mxrL#JrV{kc{~Jn
zVNeuGyU@;4{>6{=XpcN=LE9z)6?#bTx4B%o(hc~A7b(!N)e!(ETK1c_63>kq1;
z7|)o@RFuq*>Epka`TaX}H*JbGI9fsI;R(+M8-Mv$7N^yBnuJMtAAcSR;{^)I}g==^nbVW=sz
zN*}YTbaN?o#O&~=U#4wLkECq+UN9-gXZezcaq`j=?{Q$iDBj)|}V8<~6TXcp}!tdvrfEoQ#!0
zS-SDFaKCGvTz~SImrLL^eHIZ5%A3T$_;ATFWUD1$2U8W4M?$(?9WIhtxyeE`=S_!O
zd-y8?I=jynIb&%m#zn%27{5z@6_reSxSquD)Dx|=w?4S@dy%=*GEj@$ryjV}Wgc*#
zK_oZ-5gcNuan9{3w`J~vIJ6lxe0jtL7Md<;NwsYdsFMpk|DO62lnQl+?{kn_3rr5`q`)Q~HKPWqj?VCeHQ0`a#j^~%L_b#~Av#jnpYStdQa(li
zmTmH+F#u^~Ra}8s`i;WlL4*Pp!)%*v-vTX7a*be>hTh|EGiiR&M5bhwZ^Y}eSmpf)
zz_J?bOZb5+Yw_%^W7h-WtoNyKam)Eb$!q!3dpjDrdw1$-%XM>g-=hf7VgWJ)SZST?
zA4+5{Qb{djhvV$?&AN+0`a2jMol(`IEt?@jzoiiw*qw{x42QydF>{g+Au4qCvY^`~
zGvR~xL}%2oVY2K%GgpA+_=)~?Dd<3#XL2jVaQC55!(L`aAQW2r)P2D|a6#<@v64=p
zk)Q&7&a-})Dm$7}^aPaQ13`5qp&M&`3$(-h!O$C)9{-gr)$-3Mh7h0EY|5=ww$jFU
zS_Qf82JR_DO>L2F{`QFamvc(>cHtQ#UA4+08cx*#HS|aCMvM95K4Kca%+!jGq&7A_
zQfD%{L$9*ul~!u-qy8j`c{NZ4;XAg=xz3ZkTtDWNvuMktc2F5-9)UME_4#!9$Kw7G
z$%!?K6SmE<acq)7d|5b
z^vy{_T}tZ*=_UttCtBx2Hi{Kaq;{lgUTxO%ZTPzmb~qgB9&U^<9EVa%M25AtZB!*{
zm@-lKb?EVS_4<=6E*YBsa(pWq&Gi{Gz5COe&#q{~l6C=z@l+8S}
zPvTF(9sT>s?>ufW?eEfuk3-QNbER(m1)*-yWxWT2zZ0&ef*ip-XQ%l|>Jv^Pvw(y7
zoF!HBB?(mPcs@6S9|HSQj`SGXkK4R&5qL*pS`~EL@h21Q$(q1D5FjK+^R?ceG__u*JFA2h+wg;b+f>>@
zbT4e;C+-R~fnnlP?h8mNvfMI}MmxacCL@OkTqW>RjzZZ(jz3W!z^JwsM$hx5i&8d8A25`Ohe(B8QH86)X|#J+`gdubD|w60C3~zKxkxVi&J|N$1Hh
zlN8MTiI}PSQvSG}l;pedPFGIJxyX!vB;aMMJT+M|*aSLt?i6xIFq=Q5SKj@&8)jM-
z&1qlJ!P?>)XXH|*zq^9*O5Fha_DLGpTcHvKe>*sB4?LKe#6FBU88;G_3Rq==DiU{wg8H74b(&7^c)ehhg6AI&TX
zZMpMbmYr)PT8TF~lUrnk%KZOdd`NK}#cnJHh7?o5x?FR_Y~-i?KscOOfd
zEv(q6CnPSOAC|8Z;%7T?;E(66Orv&>WbN%d7oypV)qHdkk2l7E)7|&wX^8NZ`p3c?
z0thn%Z3U72@;SSoOm6ebvTlyKc%)6P<%k&uU|p|Wa(015%*1Om&nmTjUj7<2SVg!N
z)BdvDf_kzpJab<5dRt1iYKlPZ{H4Ghs19*okdwn+)@ClL+_qz!D!U}rM2kp
z?^3A4V4=cpfjn#M$PUzV#qN1&;CJeC=oZr+dD^c}`E5%J{7&0q>8EM?q>my7Am$@W
z1?wD6>l9SMzM&vT4%O-8Y=ss+3=9pD|gwvVSZ=*E*-RHD`?~%n!3k=
z^0qmPZBqpe$lQ)PCc<@g^WO4@T0mNl3;@AWv%Q|=yE40}aFX|S@r2^mp2D&o%okG+
z-jFOStLPCI!~0R0Z0zB2w4jCeOJ=wd?7_es4T|*ryin|^1S`0$#`A%%t}9DFiEQUF
zSl0q@H!vK@JKserEh)A%KVuZRpG7e++E=h}{w5MdwvYWY*`98PaW{>itxRRPIZoD6
z{TIDiI>(3P&faf92Fe?6sw|mnX1{TrWpW|M+=oum4vvLw>!5h=4ftj8#&VEN&?NEE
z#G{bgYt-G4LeU(I2`Y*+IMHbinGkS=X_QT^Z62kpS-|avQf&#$>EKh895T7$E?VFB
zwiGnFtp)de1uC7}pDlGSlD0&>#Qu8d!ldkc}pfmC@pKgp4FZx($5=QaHjNrG9s`{t}&AT>CYa
z+cp?TRqND_wz!G8W`_DtKIbS6g-Vt?d;28|V=ec>BvqaPlIFdcJ*t&@Ird-Xd~Wr7
z;yLDG(R~&sp>lNmcmgini*?TZYh)aFPmPHY?#jKP=^7Hd|W#L#OWE;mVY!dVLik1Dk;}N(G5%X3_iSCQ&
zam;55=nQU9`qr)+HgI)B%R|J@JMxQH@u&T^aX9`bk2ECA1d64iqq4L#(Qzl+`bz}e
zU_4r!CdLd64AEm=x4aItg_oVTM|3v@##Aj_7`93$z-9K>5prJ@vz_GXVDEp@U}=h%Qxje2`pc~fS90EvSa&TPe$WN
z_%E~YmjfZ@5of#-Wao;wSo#Jn{lw$G)Z<9F%6`d;h||HufZ*|Cz`GUPFM!!Y%Tz-m
z7g4vu%|*19B0(zdd#mQo0Mn!%r{VUU^x-u0GrH%M+~s+zIRcl7)f2`}{hSAo5Irr9
zCUW1RQkXMo%V~Qfk#b_cA56I>W<2UNk3pz!tKEQ2k?z!9&FH+D3J@~XZadCcLIA#P
zrD!N|-x83d3c>^BpR-v&y8=kNek(7+{p$dB_RKeF^u{Xr1a)|!1|*meu>zy*-S+db
zBB`-9#5V~JPEM{*7ZE!pO^A>?Kg+E>=B@^6pUu7b_&%_}`X|RnCHjMz3W0+;YM2+a
zVr~f}l%Eb@qT!(f8&K53x%r?VRTCRyx-M9@97uNjdxDUAvm)TzH`^d`pv#MD=>qxm
zCdV9d#g0YY$sexx%lwUa%otLZc(nF%F?Wg5pm9OA(d7=6GS9iHee7)YLVny9;Z${S
zX*T5%Lnrs((mQ{F!*-QC&o~$JCwHXHUrOTfU0ZkpVzO>WaQaRdeUwbNRBGRBb>-KI
zD|$q0yJ30rH_c=k}#@1zx#*(Ed~C%`u6XYhM1c
zYulccaCDeHqI^=HWzI)nesZJnXz&rOL=!0Wn7X5{PC!Zuky+AMn
zM;Bt0!`R^3eI(1(8fzOYi4g}&Z=x#AFqD?RIZN6tnED(ImJ6y7o9Qcj@io?-_2HZP
zikYbk7d&Cve`5Y?U7c#F4H2GT>
zaHMM+e!#NBoUfIz)Deg-JHHljtEI=Yt_%3~WRG-q)GE(+wm>4q)(_*&EW-#By7w>G!2T-XAZ3{vloPJvQX_
z1$6ad>h2RjRSztr+#xb~t>)kn0F569b2X$@N==eKe*EYs0ZmtJ%y)H(t4COwoDp^R
zQGPf8ef$?h-J?Aidv2*56^V_y#vxNj%#kuIyKb&jsZS)8X!2p>h@f;=N4b}S)v~B7
zF%zSf-$0{UV{QUbA;GAhW7~zY_cYZ0?RrsB`~9j>{45);S7u2g_e*#4&CUBiO?KDe
z;^TW)_6wJnf^4S!^JF7;aF+DD2Xn_{y6$aT$y(pV(aEiAKkY0~S{214etb2X4#F>Y
zYkIOzsMLf?zUSN)&#YA!k;{{m
zS2;spa7@dn7H{sOSSj&n(_j0$s=pq;cSNPgj(=7gfnw(iLa2a?fnPjOqEcF7p)Im7
z{p;8K<(0%SJKgF-^8RccZSAqeAMaNs$5RHAml^Z{?UciH4+mAcII*SMnSEjirdK<>
zHNiDEg*E9K$?gG2OQ*rrY=UMFMUmzY7AZJq;~WrYaRzwMLm^RMTrOXn09Z`E|+k!hvf&+^mUk&Z+mDD{~#$I
zPHWO3tx?J;OA8}}c~@xD)n2dFE$ogLR*wwJ8JoSUk4T|!Dz)BPh5L*a>qnOH1!9RA
z*-XzLGY#ofadj@$i?v(t4l=Scz#hMsjY?hJX17;dkC9EfK6E$?16`
zW`~7E`~I}AxA{0%xmDJ}9())>k(1z|SKa4HgNrHvu%g$L~kMo(iaUzW(4(9;~fDnkcx7
zFm5?s7`teNo_!tJXj!~1`TbT09@F4$L_Q#mq{a)vNlA)oJbHss5qYfizXE~>`-GsST7*_eY|n5Sk8htxy2U
zy|{;ze+@8M;3^jB;l{ci@A-U!Bsrb@2S)o~M?>ncF3ut#d63PNr_WF-q=vM7QrwI8
zxQ{mb7X4T{uj}x#Pa}=(sre6A+kx0~0XS9Nq^`u_-+dghT4FjWH9I?71ZH4Ii|IB5
z`o}Wl=<0vO+QPp;eLtVY+N15U&BFMj_GhFEN&qdQ9Gc$LQ}CT3$Y+sIpYK_uHK_w6
z*QWhruK@}}wrB^#DDArCo@|d0O}dfIxwl#0b!Id@hy;xI#hi!z`l}~J&3SG`o62;U
zlA3PS!HO@#)ETbXrc3|geP!Avc07{db@>FRMZ)=B^w#ct8wYy9kM(2op%kXVc4gA|
zMPWvE`g|^*c%?pJ-M3~(WUgO8T|vMUX>l()x(cp(-_fNU5^0662x^b}5fEJ6P)YK(
z)~{#5$4~x@6*V$Qs}vaNXW5ogW5n@|YVvxi-S!3cV7(=m4(s4V61A^Um*`oHs0$5S
z;6s>63x|;iuX=`XWM}WALZ{;&OeN*_;sr+9{n{KKMjy)=)Ek=QR_t&~l}P_A?)T~1N7
ztIK{uK?<6>gee4nynK0&St%Z_mR6PJ=diucrsljN$;BgGL$;RkqqJ{tfC9nz#$C&tY*i%X1g}%tb+Jbtg~hte%~LV_WnJ#Psa)|8ebgU=ZcqSfl`gjbP^<73
znm2lOjd8Yg;f*0?22*>R9`upnuq_upOWAcg%vuo^2ryGaPvhG@|=2LQibV?6yq>XnQ+{mHF~=i$G3?nSqbP^9j*4=
zUhRm<1ib{SpG+bzpEvFY*pH*GyKum(LI1UAVjeseOQY64Qgy
z6}h(O`~&>UZqSs{?~3D_7Uns2E`k2F)zAEvUaD}_r{U@j4>Nh%rDTP}fR?j9r>Ik4(3VG>*2N?#SM?SPF0Me^wz1f7(&z2FclsKm{
zam=}Bu4w|gPXLk}IofvUp%+d+Y?NZ}BWt0G?kXS}voK~{_Rcnf@uiZ$IBg5w1)5?y
zT6}AL_WXN$2)%0w{Ilefbo2*?5(Rq?2G+wJ`qWd#&Sk@SwvNezn1lDa{p-K2J()w1
zWu~rA{AvMR>JvRm18PnqbNXD^b
z>FZ#g#Je!;SOZ1-LZk&V3-wyQgfNU^P(=tEwYP1s|51~7|0aX;fU>+jgq^S&`i4rY%1=4Be_f8&{G@Tq
zW%tWLnqetsI_PFTBPC&%&)FAkN`ZfF*{Do(WzbyZ&TQ<`-zt>?DIuc-ezwKhJw#+XebQr#sVlPqmc#pffkmck$1&yr7AMD6+VIScmb#L^
z^wNqLbaNLAP5a)(b5T9GL%Z5>Ck6PU2+icYO2t0V=78SY8wL6KO0`{&5!
zukQ?)g;9M#psA!`RD_t^$b|;P6P{cEUaYvK(_mWzT@99_Q?xpmE%EZ{+8@}j-r^f7
zl`uE*Lrkhns?`3kR=1CmWPLY`XIbHx&y|bMD6JBlu}eHP
z9RZHHfIOVA9#Y$!7dVN8W>sX?josV*=iM$}lsOwjNiaWCe6(sos`%g*@pSDa(AsMT
zvX9@Ysmiw<1x&`P?#-+SFRvo=zFW~TBW@f9qLW2q8Jwc9%`l@|(<%i~A!P>yO)-9s
z%zS3KZ5%CBB>4A|-*lGGMOCx}OFk(#`Bo@lnGXB%_EEXX3q(bjrmt(f^7ppG
z8#5y)l9<0$y?Vxi*(1+hrWxJEwQF$3|p#ZJ7jWR!pX67W)rtEq5GjPqo`#SQtNf|4BW8WfaQ>rmLrR_q`!h7>s*}b($}GdnvAK=WXbV>psXeSLr{uA
zaDBv|kZr>ILvcVd2$r|b0cM+_E10m#
z*Z#JKL*$2RV25JW6^M)@0DAmSCCWA~NKBQc(XexSYBihn=5Zr8o+%Q4=Mft0WHb9L14s
zQ`CKe2IT1+{Zu4l!$$puhBQmIi+fy>?#wWroak(U5v0_$*mmhBagt|J-2w4O-xXUp
zdg<+S-Y9>38?3`#__k!Lq}?dSYQZh8iGxAMQKxE`xH^9ZQAC0ryz{wlCCJ1oD1H8S
z-5J)_$=r7H=3s(?uG9_PcdxMML{g%O?N{3nEJPTf8ON}kZQyrGY~cJW$ii<`10c)v
z7}pTx
z#yNz{b?&La=v^Q!<8Uhv-jT&&BhCb!glgn%_C^2OJZ+_m0hTDz)uoF)6yf0znI*mU
zsNqK2J_)-;k?i+2;ofNPNvV0;T2TYl5ce|~9SYT4(ynUca_*KO>*rdT%aTuyXY3*c
z0+&UR-vZVLD5`_Qo`_Z{Y-W$1_w8Yx;yU)
zpMkSXW}V~PM$*aF!9v}xCP`D>cxQGXICZuHmNL58rIzzj%H8?qf
z77O`gm@b$PQKk8aL$Wk{qos5}N#l@j^2Jbrws?I4XuDhWKBwsc-J1I*GvBwJfS-{S
z7**)yS*yDHbhxaA1H)!{P?B&5wBKn@AvqcAe`^z3Z#Vx{iM=_GvZLZW!`3$aZ3Zoe
zU?zC}7yX8!+EG6n`e=?MVXCdPEGLnMi-aM_N5SVm9mH@sQig_-?+?v(>h*p0-duZf
zXt;FO@TvtfJU>W(F{XA#H9!@ehGrwUkGpcO9O~EGaa5;$Sb>{{bW=n#s1$QDYhlcF
zNSCNA7i8Q1;%?jSoU5Wq^^@>iW`%b+TQg0t@3ndEb?w)P3}J7k1}S`uiz%A78k?>A
z5&R4rzT71cb9|@j=k4Ixbh=KI&^9*wUsZkN07a*5R{AF@Zrp0ott$5Ic&m}iwHrtu
zLfUgb=|O-i42F|RXee)#s8X<--E=RIGZBQ??V_ym?8mFK6V0YQwkIo*rtv(<9r`{s
z+H+d7Vgbez)CVW&9msTR-;cg&0GkwSe@8mqy*=5g%a=VC7giS8wPs(Bs0P#U5flz`
zp^d$&FE#T+B7>|IPREntwMH?%qtFl+mBc#+UDVfvh$==l0v#Fg{HxM{L4e6G^Yva0Q_!p43M--*D&fXJoa<#DqYyKA6|!1bF+6VrEo9rwAB{)3n(4Go?@KAltno>{-Y
zyQZZ4dz3p!)G_^w+XXYxgEEpHQgbws)?xlAsS?B3`;VM8#`#^J@_W-s!uyjz@D=q9
z2dB(F+x)q6i?VKsu{Mh{_lnnyClsx9m%Vv9+c=&bn@{6!&h^e(okSQeR!OCPlwoGe
zQd!$Q-0l1D&dSMGr|3r|x!$Os{Tp6^5tD|H39SGMw#%oJg?+%|^M8BlD4Y$Ry{AKO
ze1sGk|3!_-K5c$U_HigJ&*Uw6U?ZNPoNUoK0*2*7mq_P~QZm$3CEkTWd^0!~8n=U9i)w+9{|K*F!U^EGG4}LO##`b+`fm$onf#EXW`@vwRFVSx^7cfjy5Je`i
zrR1Q~c?Y)>J!Vws{c%#H-y;2xzipd$L6D$}Zfh-k`XDF?tu;KC?fV*E@<
z8>{_Y7e+kKGmMxu352s<(q8V86llAqBS|8e9bg4c}{$KKHae1HO#m=*xxA9Brqg0x=&rS^M^2j6!B~|7f?NN*Pb?E^s_N%B@KFDkQ$I{_5|l>@OvRQ^8|~d6
zIcY=!{ifXq)T~(@a8me9YRSkcT5pIC%ypo8o^!a5TU@~C<(cD5aNhDk&lOn4UC8{2
z(>nH!Bi&;)935stKfi<#
z5*`j&8GD-hk+bJ-txd*5&H-D?JSRz2Pi$I+d1{VK*5K6*ZY{OS8iPBkfs!Fcc?gYc
zhGj=ERx+QkF;ZpE0w&n}blM|=%JKs{d?DAUhK8rH*$H7!&R`$94^H;)sorkX%IopB
z>Ov|9WuZUL1Ay3)FDy3!pR{@wh!^Gx!#bjSHgEzvhd-aL^?3E<{m~cu7iKd;M;{j`
zWmvv1*_`X7HrO;Z{9q~*N9IA
zrdX40;y5BD7MuIZ-&CCARlFhWi<~C|G%BlE`U|ygu8xbtZAJaBx>QX;^OlvHsJJn(GkU#Y+`EoqLv>B7CM;7(SC)W
zBN(DY+}#MH1XKr*gA>zZ3?dK9xs^SnVwR)0TLv
z-&kv#(Y?OLaRq`tuxt^fS1(GAhdr&H)J6+Q&)e{Z-X=60`g#cg*5YM6
zrB-ukq=aVMTxzAa#uL6uTNxWt6hWylSSC3A7RG#yzVSXuZ&7p%!bL)Y-qC*UP|6L%uo67I8TjPOwhi|d{td5(WX^0JW8MW(ag
zj_|3`#UPn*Bp3wb1M+*wzKvQBlkfH&)H%)TVEAJ{BYRVR;!vj8zAzLTueZP0olpLk
z`^>$c`!1~R`AaA>0x7lBR#Kr;GlGYfN{ES+HP;0HAcQuPIw0Yd4s5>EqhN9Df|GliTGOb&TSp$aE&V=o&T=nX6
zqV9DuT$T%iDyGShrO&e=&>e@S^|=?n1{Y1%(DXcddgtk>y#oQR<`=9pKH`c8#7ukE
zGsYVLQre~%MDtU(so7HGs;jVcGDnz`gb^ud!f{ko*%D-^$4?OznQr#u}y#u`&b3YJI>y@M_2nDJs)ZZhTyjX~8%8u)P;N!zuK(Aa7^wP)n
zC2HPvES9j>&WD_Z0(-LfDr_LK_+$}!IAb&R3lZa(zVht`6
z$Gw1}VjHb0adsQ-#XAaJt}(lupq>`nMLXCm)bOylFfJaqmCB7ec2L?D=<-n{82f3iH3
zn+h(D+VLS71H+nT6)PoU3*ICq31A%QS?0(G>q65>en@nE3FA3J^e3Dx|omzwmW
ziu)Hjl=32mhx^hr)ke$oeHZS6p1`OFZ{P+`EbXtVnq4iRM963C1o8f}Sk0oJ|BEAr
z723IP(n@>SQTv;!nhprv<8~zbU0HB@iDniFyN|qUwN(FW^y&)hL{NYAI&@j%t5`4y
zUf#S{^W+ne#qCa_egwqn@~81n%q>ENmBT!?@nXIC5Epirwt*$HdjYj6p=I}nX4p`i
z8c&ZsfUVaKe$#8ene{A_vG3o=`%i$cW-%D0)itUC{zt#roZ^_4*_
zN3|2}RHGPdFp!u&oa!H#7OulQLpZ`WSN1yU!YLEB2rT;lef%qE1Yd(|r>wx8BfY>E
zlBR(U>==*BI4QsaQU>U6v}IyLgK}LV9iIE2boyZi
zrmf@8Pj*~SL_E;3#5^_3>MvP!@$I}A17#DmrqF#{kju~B#ykxnFVGi_D{f^FCmW9kG}-|Px+PCD(3P6syATOO`sy*?cH`n6^ko?*GN=8$4s
zEZg@YMHTeYzv+=mNWlPT
zsD*4{pV)|s9VtncJZSR>Hk|8K{YA)rtJBW>Nn*T2#is$irDs`V>jf*uInHAo|((
z)ha?DKg4}7Ki2dKSR({Mc7L^pvka)sK$lzf6!Q54qh_*&}^wnW6**@k=mo2KhjyY?OaY*-+L!g^u(A9QONQ4YffYCL9<=SxmUB2U8L_Zk9{=c^WLs|B83^MrH?#uUP
zBb$w@Y9bRp8=$eMtBT^M9{EEfJ_Qr|x?6P|2uR4}66z)J58WPf)(w14)+baXPwJp|PCjScD0hx9(b1FW<
z{D=g>Wtg3Gw<59}&zi$K#C2EBz~~2*gx;s4;hEJP$ODD&KX{;w7y+qtswovp?YJC`
zL7Ps^xRqWGmHqt%La!WL_9J3u=V~e*GqlteQO`SUpUHk+M4K9HD9vt7mad_9yVUcov
zx{8Up@$nM%a@^_sL1#5OBih*<$s0kG!EPZVZtNAHbR|Il8z+;Z&#nTr$%`t_Q4w@s
z$6YJ~143aHn(eYLsfXxo$g>nBIsEo8jTy$C$MO^kfSZ_Uk@%6dFu(sydVf?|5hZ`%Mt4=F%x$Qh%jdy>JJ9d`W9>5DP
zxU9i9I(cxGs5!Q{7ZN=D#>S<94Iz&vv(8V@X3!eHx(g)9B!-V@X&V4Ww~YbTaYT)7
znATDh23@IrcJr#3G0)NNJ%+x%&|ELZVgEAu$&JAc##2r53=ykdqv@#xS
z>b0XO4Yu)Tuxa_8?HPIRimKIw-NXI2kwF=zYi~AJCh(mdwiu7IYHG~d!og_&mW+n$
z`gUe_R$l%yePv6|f$Bbcd4eP!mqcted#mQ+Z(S>##B1#r^{X;7(-QbSf5;!itoga&
z^KnKZvba01UN~R7NfE0+8!sEjsOQVJO7&WfK)xl&SO@pY$46h6KXrz`2-UD>5g+)W
zk}apENJhYqmNkrZzSfE|R)Je%g%8P7OX6>q!;dC!t!W)oBSysa;o?_o%GfSiOYm#I
z>QxnHkOmYAzD7KQuHn0$jlWJdy)|1DIAd
zbZ-F2UY@{@=Haj9d9(@q52G84tnl)oi+4Yr!+^dX>tQ?t_bljf+_~q~XP0W5q*vAV
zox`X{zbXm&$S6v?`WQdY&?%n>esZ$_4bM!vE`2q&vrsvIb=Qa6YPRR*5UoHdLa&Mi
z|0^;lC((WXy0^-TWep3O)aKONfWjqCx@#8?dp)ew)174O8xrUT%fj8Drxz)S6r#9a
z@hL};TjQym%WTI(x%Z~U(4(vFUdemI0e;BKN`29(c&BOY(1f+2eE*8#fY*TeKCSi)
z1>UDYFDw4ccKs@r;6!9+h6QIQ_B$YY*KChqvI8A3!xVo~E`aT-Td5|<%uga`_{DZZFe8xK;8Ul9hPun(V0PA2
zOQ|LHNQ*~peDBTCp1`5v=Q}UKzOqLi+Gi)Mw-j0)r^T(j_4mng(#O9kz8+v;&#l)H}$|vb)yRwL$C-qwBGgszn>sF3^Vg<=(n^
z^9L~BXq>9|X!E)F5cl+~DUI4--#L#*0lwbp;e8mr9Idx1VAD
z3S$ShlF*+=`)ggTPj|3SUt69N-+H)`qrWYg%6X31V7Q$KmX~?yR{u{~L3P=?M~@?A
z`QsQE`f;-fZ90jz@UyFT-muwhB7|zZ--vGYJj3eU!?U-?wx)aFgj|UcT%vJ0$gF(B
zvyZH}wa8B!o6F7+Cy;yd#<87EC;1ls@zJ)!CGgX7Cbb88&dUkllX=5ov%HZSjz+LYLDmf{LbKt
zI-DsmF-UzEFYnoD#Mjjy194E)4f;xaPGLZ9lvplauXNMj)?vpTgng&RTd}apPef}0
z(dTAvc}nVJafn@{{dPT?r?)6^x9&MsPF}!EZ|;E4cgn(KFOtRtKO>xgF;5_^a|JV8
z)EZL2M-BroQ|w?P%OFd!c4xVklh^yzev{Sn&*m2-riz8ktBzk*r8Jn3xe3GtL9g?Y
z-77C~%M40}Zy_!T613!9;RH(`c)&-3}es;0A_axFnX0~zYMOiY5)SM_rQn>pU@3x3SWap({5kiG6kiW{8Qz%WlRR1Ff_*Ck$>Tsa~hM
zRldr7_}kD4^*3Q@i2D!R4D+9Y8`qzE4j4b|Bh%R7{<>6J&>;l_yCOvx
zp{91znC`quw7RhaqRc~!)9|rFV<^6)g%l%36(d7H`N^1Jova+(T>X~vt(4qwVzI9K
z_`+p9p9q}3lHU*h*0M3{#hr-XxXP8}l32K-p4F7&Qq?;0;#xX?RN516jXMloUreaH
z=xQb`MUOlUU9A-i6$I3W?oNm#Zw$;eZ_F3FH;{F$?j^Z2R8Oian1XN4?oZ{ELk#KnD%y>CjBlfzg%Dg2!fpyTs+s`PJBNTveFhq67}x56i{5&s#?e)ud0*d;
zAFmZo1FQI;&`=@ad#`;S_s2=60@ZVg<#iNFIfwBq_;Ejwd|QUQ7t(@cym|>lfIV+|
zkhafrz?~1U^ESPz3k*gvkd3+LEifCX*#=22q21K};JkS*PiqA}a3kJpm!F2Utb)**
z)N5j%I1A6rP73eG-?pwv>4Z7hx;{O2;mS{5P5NuqGZ!u{^#&Jo8wyl@MC>x+?aq))
zAxCsJW^wu65ZY8(Z&0z_fW_|Q=rFL(ln{fYbZ-OcR4ymNI>*DO+b_+!J-I>~v(#b?
z_ns5qeku3fs%EBmgLFw&#_kOd-}`=PyKP0?WT`ELwq&ofEVl-&Jg=)d?ft6!m^gtY
znkvUt?OE_mA*?40N$G=65V4V0{cY9LVwsliu7kD&5v;P=mw-eN3{iW6q7eZua}Y5x
zF%kC2OiQZ+mgjyD&;~9F%!;HIt(1fUF1)}eJ)*~`%+B#FCTwrVCp||+R#6HWR7{YG
z#;oa@
zAqC$3F|4YJ`vE%!6IMg_amw~c4S^}>pb*w2^Y>drGs)iL@^m1By#*Ppbm_{DQz-}x
z7(2(6+3;+a@i|kynz!U~)}r&`>O5Uvq^K?ll#GGhd(kz}T7E%ZKz)!J+P%qd!0t_U
zZy@--0`1sCN+CH?NMx*I;{&mN#;P5
z`UPBTNrbifp4_DlT%lwRxZpuE&XuA*Gf-?!r3xJB3m5*FUO7wsBV(=s)Mu$*6#U)g
zXi;OSa|9r$!zI7`PW_LjU_4Z5Cwv~m`#2rF+%wEgW82u#D=`G+)B@unB$XTM}JP$T}3$Qjj?=v(5!q?GCXvvULrt
zQU@Qewxz#*4$V_r?rioxw&S++r1)7
zQA%M{pjEAT=fz9(EW4Ze18Y&4o%)`3Cn~*@HsKLMWbgHdlrq)O)MN39*LpvvA|KHw
zzQ9UA=KNUMH`6EWn}Z1!r#>81K@$x0{i0#sZvMicm8qgt!YY$8l>+q!Q_I$!xa34O
z=eW2)Fxa%3vJ#ToM6fCdZhSpRv#OCboagHQ^3@DxWr3X@Z?;+-kP*Ak;>(TtrZ*vg
zr(Y8En#?Q!Jk|RN(5+b_IfuPRzew;92nF^E`rz(mi%vNR54E5KWab#SD_SjD67rh@9btgI~2+*sd@u(LR;;><{V
z7H2&_LUGpfs>>DMWRSm7075v+U%)n-`Hhh}+C%n9b{)NJeVT-Y>rE;)m#^~lc@mUd
zAnbPt^+|h+^=2&W0*L^+O;)zb;+JqrI1HZc+i(7YBgUHRPQr3bd(VOM^KWG-bb&c#
z0jyK6@l2~&1VO1;{h?nP5w|bJW)qr9{vEf6vim*`p^bZC^s2$D1nQ$8vFTfonJj%)s0!VQp8Fg0?o5y5kC44Dh>nQLB&(w84Zs
z`t}p@}~@q@EIrs
zBqKYcdP_pj#tZQLe_KFyKf)XUC>=={X!6)e@*Zn8F4|S2~j!BWK^*W%+mA3Fnx#2Xo{P5_)cruye#Zv>4
zDA04hsG~jV@-fkhT;0WZRPk%!BVDo{M6u{s6{hlTz3Bd&i`d;==KN-JVYy^9x~r%D
zUU&Y@7oTuQd)d6b27q86aKutOK+9<}1ZRZc4aXpj?7h@aL}G@0Z&ch5HBV^mst
zT<`xH!1l!n**`o-gLW0mX0an(-r>!rL03Zd@_hJp{13JZm#qY_T1{s=Xw$SzIhn;0
zuGpDNSwZ+;+UNt6?8PvAK!n#PUi%u-!V9q}6ylHf-!U&b6|hG7Lsi@+{wL^>H}T)1
zLq+i?Dhg(ImdyW-wSV+t)<>%%v+CCKLC4V
z?R#-RWtCt$S)*9(P@I1ZjDDv;Rm6Plqf?NZ+S!(=ygFrvD!k9*eYpNE4cu7RJ$TXL
z0ra4^{ygYYy(;k!MmzFN>`vE=RMBKZ&QlpSQVR@{yhZbhbsI-ZNaL7Z!A5qPDL8iY
zn}3q48|}sIHamM}iat=5k6Wp%3>*OL@VwVipV(ml6<(m4M{NW?
zN?{Zg@ERBO8r+4V2Ot{G%@OU%N)q@#cEVshJIcDAn-`D4U3qUnFIm(jV0%D8qZv@y
zejspfp1;GISW!U?`5a2|ZS_aV0Y`rX5Gyo{=*E+M3AK0tVL6D;lKG~2`|RWqX{q%KK@&uQRT~5sst2nd-D!D7ST3Zyu^gjT-UU+j3`dwEDQIE;+x=aZhI;~~Y*ZcX!K*BNv35(tuJ@4`B
zdiP5(V-s)*NG1`V%}>Q~aIa}=TU-8hN^U_~Iy!S8z!A(AKoT>)J_lW)MQYxJ+oN<}
z;g0ax=A|a1{!OP$p*HUnZh*WHw5HO3*7TI5V}L%uvq(48N%yJlff$qrl0BfTmPEvc
z{E9$*{I5!tGC`H7(xVNKN=7Nh*KIT`7Y8Qq`=Y5Qe#ev{B!X3fy
zioc|D4zz86dQ^A{*adr5ASJ8&{Y{s;?CtG|ywa+wNOj}vsr&t1mqKG>jSu&}yubYv
z0^}br5iDc9OXG`|Ld0@ZyuR^!ji^v-G)wK72Ex&%$*n&X4lCSCvM?(g#eqni;Xe2R
z3iqX);yx)c`*uO-1A@;Bp*I?Fpb-Si^;iJ49MS?$bf8^#Ug`Ad9!j1DfTN4{crc5nbz!4gHh$LQEav=qc?5g_B7y@pDMrU
zx@HHFvo=F4JE0HiLEJ7r1%F3w^tALmC0zv6W)S`lZH9--P)2`uAHN%_(gpH4PSeSQ
z({vInb9)1Pv`=mi&!bZ9UqS9MgrfVTKM|o8Da;MREPt)pyM*nR;D2zGM#LT$)?IVF
z0+DIZg^(bHs(QQySLt{YF$AlZPP*{)SK#Lg;_%(T5jbSr-&u=)PFN<718^`zOYq*f
zEclz27zT^vAvwJJK3g5^tyRMRjlGp;ju2J_$mvDtxE$c0HC}PlRwH2~L|zLs!P4x6
zX$n$O*iu}xH+Vr`_&zP|xBJ+tS+-}f46g4H^m}tzp*EKl$@Kzvu-28D;Ox(mlq|5q
zd+Z3nu#`GX`o_A!wv#cb~c`pBVHr2Qc
zOIS0dy|3u4v)}ZmC=P1vD8BWOa`QT(AiUsi`As%&%
zQsXIDejhr3y#5y0&k}4LjOR%A9nm(P-CFgI*@$*H<(~a0lNRhXI9EjPo@c)z!okl4
z>?yPjcSpKh!f==8?`&N^W%Cz%Fp_>R*nQN3-A{O5CJHUs?LP}f30>!V)oF|N%`*R$
zvr?YtHU_C#hAlep5@(aa>%}eH_34-e_OQi(_}8a9Gy~$I|L1mScpVr;V0_zp9h&3%
z(1L)GfcvtycPy@P&U@
zD=)yqUygq97{@#a%Co%SX-}JB03`YvtD`w_-%NXm8QrgFdgIE>jdKEI-geRO9C<;q
z%7&J^9Oe0Ae7tJ_HW~Lb;sJasT*q|QN*
z8h={rOM#1lQ6Ubzc{ETEkQgV!lV_s29p;(lq%xP!47gA{#m%
zn>-o5*XrOQnvi%c|1;YMQ?-x7|A)7?j*7Bt`-T-%L`p@J21P(XS{ND>1(8zemhO(B
zLlhJVl@ckD?v@6XlJ15XV33BPdzf#}fY)_D@4D`_-sir*^{&Mq7V4bm-pBscaqK2V
z-;brv*&iggU_Kg#?@E6Eoig|(DDJi%PpqEB3_52k-aQ0CF
z|LEpmj)vjG_Li0(AXyYVW?MIrO!$M47m&+waP3D%PR_kc%YZb1se?g=VWRw>&KGBO
zpYG(>Qto#nP7nENY?c5%c@vJ=0Q768pn8w!g&$_gzX->cGE-8#z;J*cBqmReBR}|=
z>yI%HkpB-11%=1zQf$2r=^n5!?7+qD`e69H6krK=<3Ka#Q^?Oo1MZ=9e{y=-Fe=DF
z2f#oB0|QN;ffM{LhS&5faKP{1jp2L!=45SfJe)Sn#QgictfK>@Y5m67;Cy`P-N8;oM*D$t1_xxT?-S#B3t3HGhzpy$p`%Xe^m#
z4MvS|aECJc&68f#L=$6AZqMsL%kj3oIDN_Ia11{E`i8EAn_I;*-Dl6Ph#jp-E`#-S
z|L%~pJlu1W2o#J!piEE}ZftIbXrVy^(6kUg?)#Hx6#Ds(qDu$_ND_XE>d0HNpb~MK
z{8bfkz_D5O3guBEqguVdVU>RO!M69lXM8Mjx_Lb2cu*L4XSXJw_g=LFLOEd$mTer>937^@>C4=}LMUh*4|o-fdsS*xh=6J*Jn|)$jfo
zH>}IA_MSlkF@_i`#=h0sI`g#11sN37^}sWclv4YqGd0;+JDRAApuWBx##Y!R-9Tkj
z)XwM+dH|KXK|F@(tD@@JK3JyINCKT}>kSObOJRKa9*AJaiNLeG_4zWCr|DdkvEAC{3lp$63Iz1;V%W10rB
ztYP55i80)R6`Jndp9CQ#i1%(4T)#1h>`n9T(PSgI0|Eic@lSA-lYe!!h&9#R2Z`Nj_b};#$V@;0f*5Q_4P{zYm~O8*!Y0+VOt;smZV(|X+f46$8RGf$YAsB~tV?Tmj1
zv_gst^$pqO<>Z(V4EkI%@6J`9YN?wK$b#>mBF>wPn0e*+7^90SV-1gWl1<8+}K?KBthiEkT|e5E}zE%MT}3&@3$oa
zQ_+?_9xr}Uho>^P!9AI=9JVn^ROk}(8%z4LHGpy_E9GipH)^TLj(yF6sngXrIDYiI
z*u9y56II|NJ>_Iibo)!R#N|Wi`vWr`FYtfR1ey(U#AxdPKVxjtrhMwc9bDawY;b4J
z9t!Cb26R9gaJvdki~X{2SYXv?zV5O>Pl3dZU!l|wgG>NxcKfZ8I9&_O*JQ^h%2*tdv?jBTcd
z+7X%(y|1fzG8FSd8~oGAiy3%rx?0zo$I&siEq8z6+6XZduoKj@MRI(^7P{f&J{Dgj
zI12ljE~n7OTR^cU$*83X-5dn~sRgq1W+%%p+tz2eZiO`(0dU;Q-a3EaII)442*z<5
z<~_PyU_HjK?J{EQy54x>nU0Q*(i2esfrzF|NWpV1kj8}^T&hhe+?!R2_1gTO8l@{%NYN_@7dB-jC$HAfe~m+Gk^A
z6I(N;Ndi4
zk&)#&AoaY)X!GF6#83(gCUCXZ#fd7YG{|)Kq$|>GtxiyZfk0*>MT|v7MLFkhkf++L
zY84us$6VD?Z?$`Th?LCD1i!`Pk**MLhq3hfGoAs(?@C4x3(u+3&k^3=R{Domv16+q
z(cu44_5Rm+M6B6tS7%*0a<4vibv!|=li9ER*4oeNN(g+yK4Cl;ih&lhhGCsK#SH@=
zeTl&Xm{+#^xSY&d2dsL&;;^Nse1;z-K!Wr1Jb2C$gc2_9q)?N^5xTfHQs>_-oqHrY
z3G5=LFC_AE;0+h6eBf~EOOtm!-k)19t3wf?!R~wpawWyO#zjgH7?25wz&PawpalkD
zj3PW*SNhR!)=LD@t|FnpX1a>GZdD9@K
z{}B&VFCzZvJ8!UD^i6kQ5k|tdq52EJo@j?dJ~-GU>|xylRSBF`{F8Jm+aQhjvp}px
z|9j2_5FL}30HUe(k1HZv^^aj=t3uy!z*ZCxFYR9uob&`c6?G1{B0dQ!AH0qVT7vSd-mk-J@N>{YSjgh)-%UFuX
zU?mmAG9k8~4^ytx3$gfPj
z8u#s=-iIru&5CX11#GF?8Sgar;v8E;=U!SUWk(r-Ou^C9_yNfL4;A~ObbH8k*@4>31;5#hqX
z0bu=UY|(?y7Df%ou>@#i>#20ML_{BWO{;;+l)3?otKW*+i%d);*_N$wLBgz$o?j5&
z&d$jp1Osz(xkp&~Fqc;n8S1R)wUliz4yD`7Q2~V>Oqr(MLc;05rIoLE2sc1GE?9>c
z?a_(lwf&z*dJfB_&;63rxdoPGS=b*g{hvHii}o!GEBS+@FLmM!1QWvL;9df7EErMx
z{a*5`YtPEla7iEh9%2fj;;MMdJvR6bVnBYqHE9ZpAZ{-S&dI|R=%NR|dgI*DDgp~+
zvP2EgI}a)_h+$vO<5pO82G
zs8UiCCBs~0L@YRSxx^NpP&R&EQS(F29c;q=hD?mZvs2*wA_N1R)wTGf1sQ=2qvS9+
z3rKW`Qd;m=%xjkG5gJ;cIa!W1*%2Qee;3D5a7H>M3`8#e<8sW}@h3-cwm>QwCpSdEPbsJOVnE#t9r~H(zp&%I@v?$P;J$)xX
zlz~$?pgM=#q1eFXf!$rOKJdUNV(xWg+?1K!kicP)ERbHh)mDD{MVQ3ZD3rC&>b0c&@XS$rSg
zeS6XP(=vvb0)D|DE?zeqCF-0^t9P6J6v#2>($D_!bkAUIu`@{_Mp&y;^)JTtzs|19
z=7LU#H~NpzjAR4DMcG#sIS*+ahawHeR#{7IB0>y>?)^C5$lfC)PW8Ith%Qf6-GLAT
zX7asn#jEio8gX#mdr$2zI%@efSQ=;b;3{t|fl(B{FaPwW3G2ym2BJ@*fWr#dHmih^jYFD;*3j#n*IXGZ&%|1;&vUamn&og&H<3Fwx&Xw*vS&F*stYoVxMP?{m^WK
z)@oCpSIs{8>Y*MAIO>jn@4+_p6g
z86Z3J-|n`@D!B)lEAJrDQ>po7hAmM#!>YQk%iSw;5~2^1(%CnQ?H?~mkr(OlQV)JR
zBk$mfn=l6Zxqtg9-Pt;@;dEO(KUR#hf=|wVzmplmqxuz7EI)KcSBHDge#^eIV_(%E
zkN}*Uy%EI}M^NZX!|!MO$ohGzbgccgnt+Cw2260W0XssBMsP0rRQxK}G&DY~ykFum
zmDda}Y6>4Xmu%oH_)oh(PIk}-ZEH6b>P^UU147TLakTfzaz@4jXWW3QQMQhtv+|C`
zJ()7Z9HIqe2W6JQ)mAW{xo^#OVu0;Ts)2gK7Nf`$t@Nl^8oMn(isLcrA;{x0*R6N#>S~(!+fQSN>U84Ivt`HgjISttdI#UOdW{g`yIra>1LB5EWSCzzB0OEt<(XW_L
zfA>Uy#~4A)%?L2Ws;%XxwC%?LCEIDuJiCIyC{n@&go@y#)jbOs@JZ@T8lf|Mugx9B
z{9$r)EVo;t9aWqV}b>b5jqx+&ad1jbPm_iP~n4V
zsRh-sPXy$9-+>x`%SKYORO#%_?w4nn5*+17l3p`1^`jLj(+S+wj0e4H%uM)8Y$w}%>!t$#;=<$NyuPs#j${;EaZ
zJbOmdyjzuBV5dZpaNt0tYE@=h4(2kxo^{jMORA<6X3J)$qXUDv2k(;H*tL2qW=+t2
zOPs30X;Ndh*$<Gw=PF`f4zPaG*8sh
zd)nr(U*EA;YYq&Tp}%~NCo_7#Vz9vkFX@8HJ2)y9t$OthqU+WJh0}UB26lYRRd9kR
z*K=bw`DN{sj8N62MM&f+iSl*95YJ>9^#Ep|ROX-gpP_|PP2^8T*`6|J;lLENslTrd
z;@!kL|C6bwS1;Z@Y1Co8_|NNMA4M|ZJV%a=BUk_g!z~4kJf3>dJp6Z9*~iJF$~w9>G3rUrw=q0`g>Qm$!
zqGJT4HQXqwxAgJcrekg?Uz-?hs#1HYk&vI}!v&Rap5@0O=9^}xAVu|El8&#UfN&`5(9L>~gpB>O$&Pu_Z?EGvaNKX=p*IutoQ
z_VE&exkA{DuQhZktY&m_+@@naEZ_TkqWDDcgi5DD=Gf{EOMBxwI5{J{Vh!LwMY?_F
zjMM~}N%8jv6jL^z%!ao|=4kB9%ZdEZb_D!%K=HYKWZokoy^YVP)($b+ZPJcy<6D@=
zYrr0_C8T$`R*lgDJhg_t4nFC4UpmnafZcY03HaaDk=Q&KXPst6
z-DZ>uvaeCUr*=3{sAD~=ajK0&fDfN%HcW&pWuq`gp8BY;_D3Qkt-lDAqT>MY*|y}7({WA<5c0c
zD{X4U6k_ltA&haEdWi4VlVU_i;PC%zltVLfce%~Bxi>QVS+&oNqop2h)ihV0*{H*W
zh8)P;P{U$hcHsIU4WqG?=-VNGL+Qy^X9L|o_Jz@(XdQ)2v$ug*a-wxC4gI6m5wYb@
zzL;JHU++`9M7OktdxU<*au9jA-v}+zTM$9?g0Zn*j
z{3Qd#kC^d8Px!%ek&)zNccBW7tM4H!kOu9_G8w?ytDJ0h>=RtBzTr5#;5}{13HGNq
zipC!$Txl8&4jys$&vb&h%OLS&@&sHb`HZL;6I+n5J5_gHX7j0HGiRV&jbcSbmlqTX
z7$X4Zf3^+sXKdSWj=X)BUj?A74)>N5p|BNJ
zKJOhg#++oHqc{S!svP+hciDQ^982nz)1@o+z#2|;2rP<9k_rAGt
zsn*+}_OM3ZL?~;+9>W;ofTXs5%u@Dpn8$Tmd50n{)LeX_t$qUZKE>N}w;oGWCf>e$
zvXo@k`v;O0mzCcf4kt6XE>@&)f$SqQQQ-&d4jfH&EDj4?P%cuvS|AO+Yj33ONIP54
zr@4EeZr_%1r;|cD93#zb-4l1v)W=kb|Ei1=In-D}teU=tMH{Ti;69HMTv&+t#Is_I
zs9|Q-4y&zs=U*M?Zdr_Jc@~R!s(uf@|JJ1PuELzg#F>At?lR(Ul58rg411&^vgzhl
zrz4}uB*Vw=-fIm_a?|~0&oJS~zPUc~ZK8ZLIRu=8m#VU+@8o7ky~eAsSyD0@_iP4E
zl>`bH;4)vYE>wTqzwOA9u-qdub^EdoPA@9u=0{9vKUqxc^9lNS!QhX+O{{6}Dm~bS
z67JY0*kcyh>0$x}^uJt8?b{sMMb~bGRd=bstR%B-IPWi6L5;0NlK8J5g-fuX)*_y>NZ8f2`^9%t9APUb>)2}>i%Nx!O>e$AqUad1RGJlM&dGR6&SL9bU;fs
zE7sXHx}ZP=XFf}>DL(YM`?C?zG92@`t-)W|5Jqcax(n>8KERDa-36ks+~_nG
zcta$l@YX3ey3Bx~ga(s5_pSh11Ob_%lU7u~?Nj)&ZC-oK#Avuej}4aZu<-U`cHcUR
zq%m?X(NAGzw4cR%Fm{t0(c$*AsTBBITBdP6v-H$M=*#!mtgJpeB2!EYmzVhFdrEn2
zsMo2QWyf`kND)kE+;>rT8!L&l?ypJ9JmTZ{Ty;SY^;9>V+}OKn8q|K9m@wwL@c_lLJ*XSJ1#u8&V0vyJU8Qk6;5{{m=){r#MT0Y}nxS!{y
zG7&h~Y#~4bHO|;s)AS{xn$&%XBCt9PrS(nV8hDM4m;q
z97-k4oz<~u8h3z)KW(Zy(4`qGZQ4|7`qU1+p#!0PC#De~R^vXI`8JuFeZ4JT6*0Zk
zJoE`y={yDBkv5^QRRwAS+M&z
z;HB#-XbfSN1*I!GgRoBwe)F~m*f8;AkR6j1fd7Y|kGIJ=LG(VraCo>M~oC!9oz<(^8n|B9yOei~_PFDoToM*9-O$6&fsZ
zQ4Y(Y<^!;$iTgMm)&o-f6+(hG}ZO6uxfV
z5E`@?(YR%n5GEXE7P#86QD9?wp6TvL@N23zJNvFEfqV~w8B=7%v<*~+n$BwY?Sh_H
z*l&1rvbpYcTH!R(2tkFI1CGO8O$KTEP0zTY8o`57APUgsH?w8uGy(mfUR>?im*E{H
zq7r7^T}zLrcU(Y~e8cK{(2BDyLP;=o)@16N*rPqS^ao|q;hQq_MVKwb^YG+sDU_Dp
zF>}zr{o*osy9@7SfgabDk9-E`bVT#64E^{yp48~`zdfRE+wLIk66IAD0g-;Z6aJ9h
zdq%GLriu?{RDm#C00xK|oD%+%f+j$1&M<1vb8>*J(qcj;?GPabaWW9d3RTt+M-c2tsWr9eG>)ZJCeMsHaj&O-`F&;wFvls8%kzPT7_)}n>{J=KuSC9tjo5NMGaffs!_L1iSRAR%vM8=V6@vW=E|MlI6i_A`;SgY>oYzDyB|%%#zs
znD?a2v12A07$i#hB5W|3o%vbn!Q04DtX$bH!3sGnC;L4@S1;BnW~0YZ4nCCpE>HjYQ94lWapB{T(SVNru$QNfYT
zXp&*opsU@oN>sZf7t-IH~irUN9Io2&S)qV9%iHoXCXx;_tnyhg{(0Y!@;hCi<)o3#uQD*9P;G{ImiG-{Dq
zRTZly8C`f@G18oSr|Xz}$9~8DFslb5nU$EL>4#Akz4KyFT3s>vXcP2c@?FzssCjem
znUV*A|CJgLZ`1JyXFqmq?C#^9e)%Ix>6FLiH&v&}+0}PazCj$)y~H3!s&p|#d{CKF
zqb`HCgKwHVWo)}MvRx}!JUCvNcV_Q6a5|cKJMbWXBD(Uv?czIIgTc7nkW4R+8=
zHwL4zFG~OS&q^8km=ib>S^E!B6A+k4p>0qvC?te<@6mi)$$P@K(j=%+zBsA6r&2ci
zE0od7fy*WpKn*{PstFm2kU$x^Za*&@sc~=h4oFf65nS1}E%2U8&|2?jcdZ*vXX!-t
z;LuRc-D11YfbhMP4SHI@H>TH)UvNIusZ?t_vI-V15?J10w8=G+v6S~6
zEv6Cd!7TDpw}#}#7`%Y-BiL9Kk1@hTd^#FYB}B`x-H}XiU&+X5!lQDbEQ4oiEfHJA
zgw-Vur!p<(&*n63(77agA*h3wCoZ6ZD3iIMUE0Qj9@7M5`FtD2hQsQrZwcT!N{0QG
z8TX8Fs<+p^0bSE1z>o7Hrx@YgO)k4>KS4?)rE1MXaTygq3}d@mF`T^Z?6_mLW9GG3
zM5pmJQ9btw*3+QA?NA)1rILUG6936D7VuC!Gv6H}GaVfW2Y8~7uebT!nX8MO7fwt2
z74xDRQ5UIB8h%nK%`lbnKR0*!ay81Dsvde_6oO#Ltp055rkqrHm6YzQAIwBt?8#yX
zom)~_d{7&BnnGNI0LWL!0-(8iNH-+MzB)*us6hVO(y
zsL!)gKhDx%+~fS##d7`1!=>mkLWc31)|3OUj*Z$ls%tA7%75o&&{xe4R(Dio*>xOk
z?|n<5MuaWEC)AIJ02YiCKD}3XW%OEQcA=nbMWz
zQ+n~+SYt<8jBawGR7%^g&PG(J%AYpt1D7kSl~IKBG3W
z2Lw#_R-`^}2E@Igl0Is3e&XynUK5`9NM?b9VUVhCm2bim?6yv81LN^3Xs@^hp=&dZ
z^F`W91gn-lGoS!UX*E{u`%ayt-)1iu7ho&peGTR%`m>PJa8Kz!rzxIH-w*DnVznPY
z=Qj5uq3My%HmdFAD|x2xJnOX~)968cAokx;)Gjk|bDf_Lo$nT4wvHRc9gI+&%tC7g
zstB_ZO2WH6xeb!_`<>a
zt@lTZ8m#0v6QmjDZ6_gJq92wg$XadA@K~o8u6g_5?Eu*mfaw(vJQxnkRRG9n?ixMH
zvBsc`hc(eMyPZzLj6Z}eFb0W%Dw%j-y0CG~W-ZuZ@_j(!b^)E&|4qqzWfDf|5UKxB
zsjLA7JORb$OjHMX*3_KrcD2Ex-K2
zAzgR<9;!7zYO05YguISUU`_exD5dXt)X{v(Vd2Ko?^;^?a-mP3V|{fbeZ
zuc)MT%{@_NlAJ%f2{>|Z)KCf|8t2x29PRE{Bxyn8+-7ael2g=#MdBifcB4){0b0F@n6+9{`QWUM*tBhz?#Y*7%>Y68_tOFe+X3T?ABuO
zP_H)WI
zUgf*NZ~ki6RAd<+Z{cJC$>s;{KbF8dJsT64B?WWcvXKLAR%RIr(hkPB6kb@y#Y&we9t*=H
z(GZEWxs1${oK|1RYHf>iyNBu!_j;}H7dgM}nuvAK>t&nmCLCk!<=9JsIvNag@}5AY
z)M5*Gy6_E;5}49YT5FV_-i}#~rgD$-E~zlvdyD|jLoflb?+U=a>VIKhsJMBC1J>Bd
z)qw(e^q?~+s2$YqTs8fjuS=`U-dkKA=XOFd9g9E?NPjY2ea`P42Zha%%i2P^!LP}F
zab-?Yb9U`5y{#2TbtaB@%j>7pdPuKd?)#HjeUFjpN6Dl6^vdDo5FXY+2~6Y$DOIgPbrk9)_+#g?}|=T89HPM{E-#L{8a!YxfCextsV
zTAjBmcU@s34YHjoSgya9uLOG>&2^P2c1}6B7#0_nRCgp8kmw`T>V}}*$4jf(ih4HZ
zCaKU6!r;x|2j8T|T1OtbcE50#N>T{3S~D~c-tWm0TbDQfnA|>n!
zBc@#|#%jl=G%$EU-}8`q_mb~O5}S}Xx(=@>CUeUE?s}sZsc)os2``tVLbz4*2MSCE
zT%#|-kQs(t^C^9H{l12U`t?HXZU~hluTj#bu~w1YwTMh-n>-~%UZue&{Yp!%ep)qc
zw_I+$8d%~kZJOw&K&@`Y_$PFKKSi3>TuzSU#3;nE$x%wg_%gM_PJRfb#SnC4P2AvD
zjE5L+_Ab9?87-UI-Fm}tw3VlPO`T7fDC9zxTGbBCbJ)*x^D10ARj7C-i7%#%RXT#m
zv8d{D&mirdaoAvX{X?e-R6{502_1L<1$9gq)Ul{Pb*vjqh{)0wD#OfP8q&c%EUPX4
zdaKw6Fm9DkHS?8+Z|OwaY5i)FF=J)Hl+DbK0-X0JCnXOk>S(<88Ogbip1(Y6H&`KA
z=(ON)?FO`&pi6!;vL#xNqrTU;tvr8a4*(VHm)@@SXdmr?oxi9
zjs|
ze96qLo7fx2TuNuNG0ayx3~u}WR?wxA#&2vRTTM$X{4=FaMzO0Q!klxcS{#{L_sPj=
z+nU4>z+oyR)M|*bI6Z{-rp$PGMy>1kv%twD;k+z2y%;^bqm5lYABDlPgJ~e+B1jCG
z2da_3S0eS{Wy;&a*s;qL{|0jH*3(sGPnXZsls=smV9QS)gZz*br$p-Nmf|9=+dfbc
z3H7?2)RgZ<;7H`J^K*}XGj@f%%FIxHuh1)q%eBs=DRH3QD%J%o?ofxjz-!v4
zaxoYPy@dEyNkwe^u5Wk3u)1myJxBpr)^KA{f|EA8yyQ81fXYJZ7k&Il6Y#+r5wJa9
z=>rXp6&Nv6RdUwi&dIcQ35IS@m_>VT>avnfV$?jyZ$6veb3U>B#R|C=qBn?Xg5y(D
z1NPbVLyL=u*_Pz&l$&bj^hP%+gx8-W_#j_NcW3WTdE0ut*aYQun_U|d^qattBwFU3
z`7O95_>l#dVCAZ}fhi*fa)-|V?MT{fr@1JsIBZNUq?Y2@s&Q#1$iO^guLOaQL6CGY
z+OgfSt+`ZuLk9|YR|qhhpZ;}F_dra3{0y^OPgUdpBoR{`HD`9Z=HFbyXjO0fXV2MT=MG)_SDCm!
zho^5scwZwTBF(#7i=ub6@_735*jhx3*{H1JB2B&=R8hV>sdL562B5RS0{iepK0?Rs
z8lVGTTczC=?i*|U-MOGm*K6#9@=`-n=N9^e~*gs^BPaB48vB2*sC*
zir+`m@X9&1a2jV(9duFxaWk0xNH8m3UJmh@eE8TE@oqcsF?McyENAF_;iKF-)S461{GLA1xpG|l7xZK
zNdq`Y+aIBG9@zdW9GHYnR_RinA4&_Z@$AD0228TH>tc#GuwTQz!+Jmfp{2)lPFV)l
zUTjb;+nGYBsHSWG7SbXb^Z?90eQ%1g$gpqIwhvo%ol4w*z$P5VXg&!psVM1t5sP1Y
zC#Gq2@VW9V@&{g1vPPe%IQjD$lXlPZWuXY8bD^?9Ym1XsteZvO8|@)n-=6
z{=0r!Y|T~Y%ynd^R&d!Ydme?}a0+**Knl4(9?@g-7fKiE_~6!3b34`;6+ARLL+&ZB
z37Re{Pg{pnazJP%B{|7vDjS(f=xW<_m~YRZH=Qc?l$q8`-G7t%2CGO9b~5zT?{ggS
zJaHTcMl1(gto$}UIjHBeLaFS|1V+|g*_#e+d(0ul4)tsBe(qEzxzt6%3DL!Ap*iVz
z8t3ff-X(x?l$fEFqd0hd%`Z;~mh8Yk8~Lk&;t@YqUP5=re_H-_9z5=_U(&bECFBbk
zDayV?XEpv7SWwB%$yQ^V%|6u>UEbR@zugq+;1s}VEE3Tj`mRh8&PZsnxKqAPn)U*X
z>z*Q)*jW(uX(n==<5_gjVSh<6Q+?B=(m9T&Ad#Cto|~(%
za}*4gFV54&;Mgaa6jxvTJJHd~+W26jFa`8?PLC2&W5L+us>I>U6Yc1()0_!iS+dBX
z;Yc%pwAoT{Lh!vwt*O{Ju*(ErfzOgK+cruV7oFR1R3Eh`sGWa-QJmzUr#`M1Ggli6
zW;aZ9i`H%Sen-D(uyagQMsfleD;Ix!_sNqdAt@v|w^N4F7*c6jddAuLO6XR3_1R+>DT>7ax*)XpIbQIGg`
z7cKL^lofbt+|m>edFzGKc~Bu8b5H-&NJ?yt96df~EsC+yKo8@8oXdz|-Rf*krdR_cz;hp^BP$DUk`g!Q=@HDhP$-vjm;=q%aIe#O^%2Phlh{J-tn
zwKf^IQ-d!BZm?DZG4UGN%=DM2{il$DOxZ=yFTq#$aV`rFt{SQQ!rJ_NlusQM0e>V#
z*?g*d$exU)w!;fqY=s&n!L8#vBpVmk?^1juYCp+RZWFQ3ui5AR+O}FYk8fYK;?-2H
zW!Y-Efh$Tp)48$L*md~>UNECNAB-_}mZNHe(rHY*6#rRc9M8Tt;LvU~{$S+FpmA1J
zV!s+TAC3WpT0>>wAC`N-aJb>*_>9(lY(X;q|FIy+npVpiJqt)L_7QF0OXhhfx;1xL
z&GOy<4j34Jrl5O5^UsO60GdJ-4}^?qRKtdMIBHahOqc{sv!L$AfGs!TIN(-i23P
zo1Wp257||2mp4`kV99q*j-%v-CnwrA0{i;WcG&($QW_92q1_)zpc*^!WE7X%S)0i$
zpE{Vj!_Jb)!m@8tG3uOF#lpc2%cepu@)j^Qm$ml3UGKE8g1MSTDo@x~9?Ez{$^btA
zOKlx%v|?JY@6FQlryxiV_FQrmpT}XK*Fy&*v{Pv
z#%avO#Ye_~%wJN3prIM&q7BFX&Pex1C&T~q7mcW|u7#iYG|svIH2l9ms24@wWM5p%
zA8cK!*Lo4^+00b_r99QXejJagwZKT0!bPo-hwWM?G>bH{WqHj%QZX
z%hPmu=w8=F9aWVO>#FQ-9&xJkBOY=dKu&uxESz0fGwx{_Uull?J4@4DX6>h+b=a03
z%k3fm)}8>Ss5xsWamZu_7r^Vfdq}%k%O-DLZJ7`qQeo=k>>;PqrRe=BM@BzMzfuV@
zk*i%ut(UJInz$oI2t+W280k@02%^z+eRxKksx+l?JE$Js(XhKCOYiG3sF(%8JhbAzd5c^E!~A0YW`1t~^P|Huzei*M;uaR;U?N3~
z#oh5hj1C22wDBJ?+G#gDfHfN1m^JD=H2DZ!>ke~|Q2m6i@kgr`nL@A-#M$uh1CT|;
z-9LF<@-P8L0uu-uDlxk8c7?8U*oSKb60dO2ZO5<>s8`w0q{;gb6I}Otl`0!O6wJc0
z8oJqaNOU35-eUBpihSSkG4*whgNvZ|ZGXxg^hSPv&#O@1z1f-T=&vaRpj)o(_$$0%
z0RUDA83K?f$hJo_JS?>>D!E)53NXoNL@8reo&F5p{77-2`FLEpo>I$yO5U+lXtSC)s7WnQ^1
zz7<7xJ>*V2nkQ8+pReOfQh`zBp&5jn<AD09-=1HGEe}1WSj#C7}u9F!GQ(Uf?
zeh#k{ywoledJ2xHTYRz8YA`_oN-Q{{thzd`0&=@h+y5Dr|KD61b5fvse3~vrsHBzi
z5OW3Fux}I*^~JfX-{pQ11EUrHMk)kp;R|+!a1Q%I2Ulwv<#T7|cS7uuA6Am&oX7_3
zt1HsZDSB5B-PkgFtGBFV)&SGcux|IV)yXtpU;I}2R=QOBzQXmvm7E4f$DtRf&L@U)
zgj}_-JHi2==_*6cc_N;hDPUi+vO+X1Z;n}9F72iBM4d2`s1Q+0Kyh0D`if-dk$k*%
zZ&TH%xV<+TcjqYVV9d6PmaZ0U>jXsB@U&qxT`y6tAqZ{Q4Xo))CG;3JBQnl5kDBPh
z<-NY)8h<|raERC|jB>xw*+`qa`=Ur1a)d!Jx~PHBJb+*{La-f$8uOhU{|B!FrA{&u
zW28h|tW;XAef5EtyugM1u8T~G;K1ZLYHh5;r3eb
zWh(KeiNb_SgSq)Ws|)FqtHRCdqkB$EPq<=Hqp|@bwu@UnDZ$yUUEbDGG{uIJk8V3;
zb7#70XiDJvf80gnjY%8+dSNNv&~2^TnSVN{D%64*JC=gp@81sE`48If01omA_CyO;
z>nz8U`^l<9J@GcJN$MDjwY_k+G%HR@Az5F>~en&#PQ}K&LjLH8zZXqlcsa2
zHcM4?K6t`PCBgFEpX6dzo8;qyfBsC9%0rtX6~0@uEgi1a%{ffzGLhlUykt`GAI|!b
zA&}N7NmE$bbNeeF`pGpE^RtInneSeG-dHH3e6@t>tBjQ)&IOr}ASJ$M_TO?JhlI4w
zoJsK=Cy4eq$nh#hPArXkb?SQLqiQW{kOy&d`{Ihnaiy@S&Ep-U)xtp$%xQ38%8;&t
zOWF3P$8N>k%KjYUwcG_-@BLSEuUpD~$#d-VM}rO-OzavA1*@~#Qmt%x)PCvm@mRVj>(NXAm6R=x
z=R#)3G2Q$S6`Tsm&w&_58P$v^b;UZ=4m&a~^o&6R^!CeXbEOs0!b#H7`|e!M-QbTT
z#HZ-WF77k28+7G{bt$=R>Md36H2ny9qD+@qmv_B}!9{$;Oeb?rK@?t6)2u7`@puIu
z85}yeYX`5tu~OAm$>}1#@HLR#g27cSnzUN0B&;PN@Po{rfvUw9*yooFOPe{DAiD#E!{d~A3dfB
z6bQq-`&ZGvXiC&%a9qt2NGF?8pE4d!YHFXDDcE@@;0-C1INC7$3
z8c}5Rruel^FL+`A+^da>vToaEu(ru$Kx-lEsR0?K=T5P=`HV2}Gn*1vNS1AH_N7%7
zgo20C)Gab?H)T`JT(@KhCkM|rkLx$?0Dk+Us?qXC{JD#l=_Tkr{CaDLjeO`u;l7hP
z_;2ReDvpR<}t+ZXLa={}#M@s5i8HvF;$V
z?dLWGd1<$DWpJ$tkM7E<>*Q@Mj2E6_{mqr|4_EjrsvoQG9xKgG47o7$FD+h-47i@R;b|NMZ56d&^9=Fmm5
zI}B(K>G|lmuHrX`JuoG4|5aaao}O6Rb+sv}V*7!)iU1O#=kgrj%?Vp-?R%~eicl@#
zAr&+s)xY&hs$7NxQ21op@vPX%u6vT36S5Og>${h2=Csl
z`4K-NQ6U>5g;3h6#P;IweoGlqy(dsF&xF=kw5FFfSg97%6VDttwXz3)>o!}ytqB-L
zRUIf2#MkV=Bab%ge-a=U;E{V!#;WY`-8ENKC5b4z3yGGp;#wXDvul|dBl}!!%X{T*
z{%`4`&d9O)TD$dVsqnbn9|SwqZ>^Id!77F{&i&s;$L5=|b8wWxe5O}@c12fmq2WuP
zc1$W4YgsR=qQ)1yC(u7?*1Oz)RISI(XDuD~#deL7PRCY5RV8vOT@ogBw)wyFb_rM~
z@~K+-9Pok^14(+#qg8uF&(UA5H4q%t-TaANDKaigJ;P#)U?oL2d)H3uw^cV{Sic}!^&IITZF
z|HMN~XU!(_w?4RS5iG~|(Guo*RvW!twxutj_FZyo9C@}j$pkQb*l*S1!Lgd})jy)$
z@{!(wsuEvnzJGPDoA;7eqxLvj7c%G|wH#0Lw$3N5&L+47lh4l>CNT(x2|<4oP5lN>
z|JrBE1Etrqw2tYLKiRxjiKqCrcBpFN^DcGOY;3B%PCwpCnyP4~zRqu}i7yVT<)uc$
zYpHC8E1=8%5Zg;W>wAfp0_n;Hm&Z#fVVk*@^?IIj=m0hNF7eb^sEFCMU)ew{Kj1`j
z9c@GNsX`{U(%lVVRCG?Dl&twn*^Vt8m~QA#nkSXFl(oovXpPl*46aWFikCYqYOQB=
zPsGCZMtigtn*abu64xQV*^T-Lu#InC(_{?$wY8$q}pOny>5?!^R;L%T;X?p~U$<2v`}8La2@obnQpr>ZZwe3sl$MP#A?y
z7u*SIAyHq)-Woi%W}a1SvWV?JeOI$pFhyN
zy}qB#1fwZDY>{G96ooGCr>%(YP;Mbz6C*8*P+I6vE&a_9u=JgJ>SsxZJxV8abQIQX
zdY))@dW9zyX)qkmw&`9?K;`JgIgeD<<-ar?(CW`B%g*((oX8hJrf?A-ciZO*IR<1X#RbAReW5$;%7q
zt6lSd=4!Lj1xlr4Jw-DOEq{K_D>k1Z-kh9Qc7eIE()tyN%j0C7sKL^yFpdf)WlPHg
z$L?AAseSZx!CG+Dgvr)Uwb)U+`==aDL?)-#{k^84TI#G9ico&V$YazE<8W>UK|Rdvm1@t-ALskn^H=`C?EH?W$`tgXIwd
zva4ddBRnZ#Qb`$Zhpoq)Wu32HLcMl$bGcj1^fTld-HfG+xCAv6J@E$S}C-ugf=8$TJ97{
zCHvYcgsfv5OeK{{cS2x0ZMhSw3=M}Heh
zOIzpO_|)c$*^cF#Te63)U;6YrnXNkhwfbC`#Wqh}6S{6e-e6lVWo>Jh1fAVy8Z2_B
z%u+)lPjG{)oL(wC#MiF4vQV^(~VUmrTjNjcT?Q0|o0Z3Hodw&ptGgwj#NC>^Cvu
zcFNG8^Tekb_dA>Y4E&XqJcBJn^i!z0?xSKwrpY%p07f5>vtN(-eriQYB}N1s5F+rE
zxPoO-07PKN{}mCazp|`xFn2v=G+CWja&m^=+0(B+L>)=sZCC9|%UQyniIx;y((=M$
zD9x78DtL6JMW&RIR;1kO{@y0{T%%2sL!IWXOGP1|D=Q9m5Z^sxU7f=9zF`4%1H(0pEV0U*`JxWk
zhN+m(ze-zsBVoRy39)zEUq;`io;5JDNbSx__xPO8FlcP;99dgxd(^i~
ztt9`Qr72yXoD<2S5<`#q)2e*R!^?E+pRvQvOUs)M?3gh-=ia|bzNweGQ0>R5PK9*h
zev697W3Es8rEQfke|)4BbZ2a|HkLr{uXXIVr5@L%2X7eAEC~HHroH0nBNIu76F_KV
zg9f_B474dKC+NY}Du)n&&2P<|S=GBE)_0)YakK67cF*DVzDvRC!F$u*DWnjEOoJnJuDy*S
zmdao2AMyw@ec=@u%^KHGTI5#A`%Lf578!Sg+Y06b%{eoA=G8_`vR6w&
zUrVS@sGUev*FVKatv|;_i#IT?Sh!7&{jTcLcdK;nc4qmi*qlZMR^mPfMjoqSM4VAb
z{2>-#$J0fq)Z-{j20u(=!iUSM%sNBORuATEx{73eB5BNOJ&tWFUPqnji*HY-V
z>Psx9Yj2+_U-_8aZ|6ZMqKA(31YG3Pka(H*?M1LDBP-o!oA;26#fnPT(jpbdHs$1+
z&mjip+vxe5MXh}%Yqop`gCvW>a2y5-WPUFs4*4?d9()GI?Oj!!FL}Ua;2pA7Y3+}cXn=LkB1ho
zTSy%{d}1Q_@Xb$S@e%Ufrk>;9)h#{8?R8Z>C$qBNUf(qJE`=V4lwtWy@wlMsB+8^O
zBv8z0uYG*KxkQe!Tdm~2ZB0=aJFm8gPU|+P|FSlIP}*!f!{aOz2*!}7PgZ|Vw_8C;
zKA%Q4pBT4_*y`#0d9CDV#)y|g8$|YFeLxa4(P!#>c!OoYXGv|kr13TK>?LvBGbi9V5j32ljw5`6l{iz|B
zaB@dfv^_>k(^@>Yt8gH^wC`lhOL@h}(@QL!O31kZTinGOm2zsiXr2<3sqy4}O+J;>
z7y0bKvi?&Vz53@HOzsv}D1=?RG-|>a?zGgMJFR}mET^q
zB~aC8koeTtH`v+y>xvR`{-chz@y(iWJyXPIs)}8(X?jY>R5Y#NTWDTI{>|!D!~xmP
zR*kkzD>K@6c{X}q54Sg3%pUG)=MOU_pAjw%@9kAwI4DM?Uym&{vv)0ys%hWDAcqzo
z<3tQw_tUV))DkC@Pm=Dl&W1YnXFZKdPcAyn{E<>I=7sa%TND?jm+%QH6uo;+iNC92h%s$wDoBgA)-vJkHl_HoXLK&D4?&YEC$q>7fb@+SP_5QKH>0IkQ
zSfnzP^_Z8cr(Q4KmQ>snD@h+w6<--p(m+*_X;Rd>uyRfEKAZY*I=uI+qqgM>bxN8h
z*_L11GI<`j-jr1;&5pK9n$pEjRjxlf)ZFOR)%Mj@FC)b>xuq$OX_l2kh#tGMxPALB
zFEuv
zuOEq%wBv8jchhihjpXOib07XJHnfCVIi_2%mB#R$c{>uLBC*azi9f=2;ikL!(YJm_
zb(H(gdQqO^QtJ7Nssf}>tg~wykV>@uD^(Y?6aKIB?ZU4JA8OOdA3SF^oR!SJ{n(fP
z`x(2s`AhqrB)D#~3ho+*Ux9nC!z?)I2BDHtLZyrX$0_7*af=fNC4U!>yM=IhO2(HP&@zfPFM8zSC@L0
zr+>Z_T~F^VWK>UFWhUh;m;oB3@Zw=sdfF92{RVSFLpbB=+TO+0mAiuv9+ESDgtx9v
z;G@c>?p$;Lwbyr{jYEOjKGJs^Z*tMTM1{@eiUE(2VX~P2rFRP_LhB!E_}5Prth>tM
z|B(7AC5Mv~?yyM4!dr{G(JY&wRK7C>!WE7=0a67(FOsL>Tdm%;>?qD)iRG-hs0Aq){lz_WDrl@-LC(JdMSi6_pPH
zuKdZK7G9(8OUu7mIZHqSke%@npgAVFz{A^Te?#UJnIgSDQz#g@8Hz(T;=8
zDPl(>vi;uWkXgOXR0)Yot*rCRuL6{CmzKWNt96P~&jZz#`*eNP)hJ@>IH)C0oPYgr
zmz-h#thEacEINc)dnsaVweKEjCr9Ll^LVJQ>6LaMT=^`dmEK7%t4J_6lS!^uQD1Y_
zYkHr3-1&m_+}>kF*FtE-C+0!?Dr>LG=CpyY=3cGK35dr9`W&6Ho^C360kzsgOe!dE
znb-}769b@ovmY*{ZJwS;k~bBR%OAXUu#;TgK3Q|A$1xxv-K;>BGVYMR(tqI5OzZjg
zK45*dHLbmFCS|sAGkbG8B-6q@Y(?_D#*Z%y!g=ZSRr7P;GP&~mr4xI#l`SgrHs}}z
zMVZmkr0#~}4OnL%=^~)(;8bl^nHtnP!INcbIJ
zKFa#OnnD8AyiCwe+CWtp*mI4>Pt?KR_a%
ze*)5>-7fYO$rY?I>yn)K60(H{^)SLCt;_P_Bga!Z+FyF*1G2BKXm@{?*r}xBaTqGU
z;lfj$YOIISSGDBwcEazNE$O7qO#SASmb%XsQ?%kPOFgF|
zjWWQ;kXY~ROatxF)B<3*Fz>dUd*fJ#nclH>cOApLFV6Ij`PJ#y>Qz>To8%ceyurz+
zPup-?OqMk-(a%xcdeuuc;d9z2#~soJ=VWV`&Vu`{BZ+n9$GkI$Aav3nVmV9{V%ft?
z?fPihkV=`18bR;J#~}Mf;ssaj-}}gPUY4HDyfiRh(cf9VKyiAe(Rbye0%-?Yl!Z%m
z%eCH>&W5?e@n)TeeObPxcT5^2A6I;CV@xc(*6z`)k=Boav-QEaNy4IxO|2P2Wwl0eC&pheW6Bn+aD7MzjzV~
zOXEKne%(%YLT$PFa&yMDoLf{I16B(0ebcT!R}5@@9i$v6-*ci_RU;Fb)t@tef0xs(
z$G+sWs(x9~RA}Y|&pW-xX@m&)=Wuzi!;8yb!gVODtJHr$&bz~GdxC1pmP9}V}
zb4rN%{$(UW$R}}P%4)Zb-`p$!$lUZmqY|=MVaKbHQn5V_(eu4n;#5IIhrT^C+3aVT
z^~NC5?Vx+-ShSKtielayev2PrQ@KZy@1$KO6wTbz-LEH8uXE0^G&4$B;X`gz`Hxh;
zdL!0^t%a&58}||oIvFPN!qX2j|*g&9UZYpUbuo!e3+AAOLPH_wS$
z()3bLZ3INWdf!Rb{i1^H(=2v@^>8!!{o{pKmG=C&p3$z!T|NDlQK~Jzjy3GmH`DoH
z;mW0V@+w0+&Jmn-$)}1DXr5(Vwps>35+ebh8v=#HBN`1i2t81%)LRE8d2Phd566{yL
z1yXs^xfUeV9^-Pok3Jrx7gEtn?dOl8O^KqDv5HIXe?8`^O-e4
zPKr{S7}Xh0OvsX`H!6VM3)R`T`*@oP%U4fUPom4xLQltcyeiYG&?tTBgFs`R`xD;z
z9cIZ+mv*2;{jCW?)IFjPvJK?j#Td7ssATpi!Y3A#kT+i+jZMEc(#>pnb1=)T@A&GK
zKG7MTO|`36I!Ujlw5s_L%^XD5VBlP|3`_zzTDv#L<6RC(F%PB^>!1ndPfWX
z)_hUH>vZMGL#;(xiI;7k_uNnz-lvdJtW9+{&_FoD0WetZ&9y4pVTk-tnq
z#oj3AW6-V(&WO^d(c-|9bCO*ip|004>}H}A5F$_t9S&2*a9Q@Cn7M1a%GCfnm0cG@
z^TTyNckdVLgWJbQr(sj?BmE-J@f!8|35$F9M%LMn+YwZ0>^5&~NKrHYn9yW?O>XN`
zbELiC+CT?XdF=aTsAwHYj(Hd%0z3Xh1R%3FY|$G@_6+mRbO4&=hk*dV|_07JFREMVC#h2>E
zff*r;Zz{fRk8DOxt%oZ-ui>Qs^7ffK&Wf!%u5F$YdhY4G?Pk-j<)yff`5X-$Wt?7{*3>0GMx(w&>avu3&`Wra^b%gG
z?YZvw-iDjT?5XdJWDh*5H5tlhCQ?%vm%XTp@M}E+Zc}gd2FxQ(a{iI{8&vF6JqxQz
z$voqFG5oBv7lTpumO_&2l}O(&NgGJ`n9uz|y5^kvVN_9%JRsQ+<6F&G?x;X2j>tB)Mc@IYtW&#u=Vmt?0k89W^`?ocaAM-xfbJ&?oy{OMR=}_d=
zJPH9_xuG|y^x^>PWFv9#iO#S`rjcvgaSeOEa&`JU-Tl7x;op;@Z6XK!anij}Ze}9*
zJ?txDx2VHPQ8a()$Xp2Z!lZW9p&!yiU0WyP3pXD#taKgrVqYA;mffG=dBvPAO0GB+
z8%>`uEAYS6P+(iWDm}+3+|5BKv*PS@eBtdX+bHLhv{c*!J+l;oe%ENyJac+>@$;@9
zrlOrBKbfgt?M*4E#lu*_2MJk!+JE(CC&7xIW6
zDwW}3%gykqYIVprA-lSo?%UQ>mXDKFmqrP^B>B}N>4x0I&5%OnWZjhtzB9i|9Tdq`
z8RAptmYkuwJ`U_N3(54Fj^tmQt~xxDQKZxsXQSJ*=}1hn&plK=j7{m%eeG
z>b_ast#98D{=B$9d*QW9W3
z**7zW{zEtY+umCP4pb5Ik;M09?A#43-|^0<0K3Ij>Vt1Dy`2d^K|D~oKV}yxjd{4=
zL$@l-Db0v0QKMS$yBCfn+NZ027|l`o`@)>lQx6d-?a{}R*KXB{c@HNw4B}-bt!%}c
zfsaL2#NE?T_#&?SK55V=zY5PW)i(-6Q31-p*f&EHUQG4VItPE3!{4h;ynd+U(w%kWf}WYn|5cTy7O;8
zhUrT31F>73BcgYoWsi{g+eCesHS^D#58!Z|4S#}Ln-eXbE<)=Y1^N>3(QrlxYkz^u
zkMJ^H-`~ZW_BIwFZNQmL1b=c)h;n%3E7`C$PmV=?m+(q~lR1~TnU@K!>r2QR^tv9Y
z@o`h>lP)Q;{#d0G_iTkT23=7obR~Rvy+sS4>-bOTx^;z+ClXE~q?EkpWD
zReK(+HHOhK
zmWD<*)FOGfg=L|VJgn+aF?ph@{^>4y-v-UJ)ipXKH6Sa9RB)EdMVawZ^HcfMC!h4u>ICv^L_V;YsljE+3K6iS%^-(}Tu~JLAogPg&7@
z4=_Jdtk88jYXs_H~k=OpqL?dBiPXX!?f{Lcj>4HjdI9
zIHF+RlW3Q1Rzl8m1jcbOBHqomZds`cGHcO7vKFZsgjQYb5aOo3mYp|OOG>+knN%9f
z!++5foh=P7q5{B4h4pi#p{=C*u7SMpQj*CDy+jXgfsMa)yjit6f~i5aX_+kOb0(G&YNtUgA;hpBq5u&+=vu8j69e5X3l_1c$m@h)DFls4twulC5ZaD#91
zH2L1TID0|dYaW;SqNy5<@bL!qnXG|8l`#*+b?!<*p|)^
    ;?A1uJANPTOd?*xY}
zjMnz9YR6Qv=^*0CRk9pC+jjRywX0D(ig>rl5HRwq$!&4CO=XNb+Gekvj;U>TMO4c>
zYszjs6keLFgME~>hHDauxDCRU2_IhRJ}e2vp^)Z};tgDTNPF#2K$uMYX?R;76AG5Q
zuiY3;l}MtM&=1G>V@RB^a{*e7j}VZixzFq{3;LptVpRW9>wNIC6;9*T_Xg;50ZNg+
zXjk3vca?ueg6II7$3HP-oPRbzT1av%*{eWS#ej_78J}AUtuh_W;Xn@VJ$G%xNT3PJ
z^}VfgicFT3C1R$}vzgo~NHC($d763=li6HT-|xotrN4|FDW6ta9
zzPODQi22PX*7b*g&CO*--x%tJcN>b4Vs(g3c}De|keR;WU>qd
zmoZFFbyX^S>ix%~4)=G`&D;NFjBY0b#>07#3qQ5J$pLG^QDR
zCT8n+0f__(rpqVR@;4K;M{K9-dr}1?hc_zd8!j!UUrok5@JE}vB(PlK)qV{g*3nCV
z2yU(p)M?lagA{^aze1k4!Pu(-E)S`0(UgVrnf~4-V8y%@^B82pul*3Zh6-#-K@3m`;l
zp)*SKDSjpcNHv^K0c%d3Hy4I+^}g_{x8n&su{!
z={G7+hP=jG_tabY+$jc?5UyrohN$mnAl-^it&8rcmA9_3hM0Hwo8E;=eZ)ma;SAZi
z^Y1tVm?!^hR*hh!82#SPsXSXp`$_TiII$88#CpsMTgs5lZA&^@g=CnLBN6RpPYjM7
z%KDHn6@YyM)#@V|L+F^^I6ue7enfJ}mp6ri)LoE1&z_84Z5kAeP+=8|HB?YO--%YX
zb}4%K+}|g#M)#htNStt`=jP-3CGb>rc3^AQW=eOmvHg@_jZ2WC>yB(N)a2t8#Cbn3
zA&`*}Ntxeiy-Aod^%@fOOJ6buAi$r5vM+C*(&$3jj4VU_nE;&}1c#RI?2Kjert$OH
zSz@*edmle;MZC8jU~pmcslZ3+kz6uEBVZGmqeqhH6nctDjDXtv%f{@0AOb{3|pr}?1kec3jfqdq(55u1;Y2ki{75#9%j4~g5o!h~i;{V6
z)4)8Yq7M1oc(Itzc9kawk#mo}X(1~M9%h3nNRVmhwx81x2YrOqJ
z-%!y=35@y%#)8bgz|np(aJbS{-A9lGiwkWK)xS-|l%b|Yl@)k*wY~Ccyx8IzZ2=M$
z)tHVB8`kb~>OAJ0nncYDiKnnx^FJC?rq(HJZ}^+ZdLtoHjk4$98th?3v_y0nkUxu9
zyzMG`>lf_SyNnZi_iR@4ejVb=t`Yp$9#oAAE&kY%Z=3szPW5XWYN$;7wE-FGC?2e)
z2`Th`HPi%zus3dhG#|gTs$tttBU~DFHrp(@(yHe@b^)8kiM>A}0CuG;*J8y^+`|vIDQez%9w3j>72U&~Y!)GWQ3-}de#{g*&_WQ*w
z0}%YResqw{_DSG3?mODN@9c)7ip5L5qzX8heVb-xY`O~Ob25N*f=>4%x&0OFJp(>o
zS;F|6?zV{SL6gQUSZbq$SEMx|ANyrgrb@N@Yx{$N204xt(Jm?~k|`tAaJNe}mdGf3
zjH*HJY%fy^%=u~CHAO8C1dsE!o%Q>!M&BnUvgYw|+&)TK8+IxgfW`V}5Fh80Mggz`
zP8>wq7J&Z`{Y3-=f3!IJPO8`M607sZn1AidCw;S-GXW)`XOwYh^q&9-!}y8*vg_&n
zndKyeFUZPP&jNOb&v)d%2i#_4Ir96Hzw2%MTow^vevg{!u9#0Q?qO?yzKEHPqI_d1
z`?qw_QZt8gXQiU4cYKIwH8}FituKy(1Lq=L_>e?}Dv^v~?=Z90b+e^Q6B1ttt17cYlDmWu#`Z4Lv*$o!!}5kJ2VU6mO>
z`*rolv!`A;+Icim)d;aijHyN#LUJG{l5loy
z`5X(>mqy`}A)p>Y5Y`(X)kf9Z5;U7$6STEx5j^8`DgCBzevl}zu7Tu#qj7R9Q|dZL
zSBCUQx@pzp4_wGfN1)7-%7p?z1Lhmu)Qy^bvxI5M3SG_=j~7$eCrq%|Mc
z#^SXL8%fAiuoIK1m8XU}1mhe0`HD+%q5k0z#m$I(kJfdGRTt{$1V^A70a|ocPIQOWM;IG@*TDk|Tk9^5&Y-iFV85Ys-&`{(Y4;NKl1ukk4aT
znpPd4YW}j~m!5%|pFZ>qCG9q}(au7Qce?p~VHF93(qhMA9^hqi24OoA2>2aLtSAS^*d~P!{
z&1Qa*x~30IwZ=wjw_8lV_@}h&$F;0%sfcSNy?S21kZ8w782IpJpnzkj8XpxgV^ux<
z<04OG`WVuy%CE*2@{9?iTt988amproAz8Ai{O-~G)8+j2m7b%erwsKAbzY!T^88k_
z-p$2_s_X?bo{rz8PFi4Cmk6}X4CZCZd0_26!xrmA7Xa4*s(}ByFezx(&hJ|mU%r62
zu*rHOrpcPGMxDN89qs-9@dC6=6G56E!qWLR
zzv;5FALi_hDX6~odvLF(_NZT$6UqT%FwAGDSyQwD?Me(ddO|WnQ0&T5A4@-JE58bo
zFJI=pi>nu}qmG57+&ykrD|v}N{M6xw?CP37Kr7LcIpW@{eOL>BSFnu1@${?_}DEt)ic9KMbByI!gl@Jw=&
z+iyP&@?Fgnywgkn&M-ZMk7@-aoE(x|ETR6n$LcL|Gw)@1U+MO)JMY~6#|)O1gm+D8
zsn;xj(caC{Rm>y;JfeF+00D>g#px{bW*1Y2NqXk0w7a0o6ifN!dG#QjnxOhOIwqF7
zM~R(U3KGcA!C98uZ|MXNTh?ErJd}F2_UnBZ+f4^FVVqT7@y>-t#pB?|id#vBVIc%s
zk9s;Tv6$$;Cl2uAN}^qf@%-TseAedBI+Nvn^erFx&RxA>^m!#c?!@p;wz{=-q>{*Q
z4YysD-6gR9X?Eoq^m0}6pssm$YihK3qOiL94VS36q3vwhOm3T)!7bI?&*Wd&7)1NV
zl_P3?p9qCd5~mW=(+>{Swe(7NvcH!8`NMNQXDNOzy3tSC_pp_Zecn@)ZB=Q%JPqB2
zPw#YMtg`HdK`$-X$VbQHUhFXAV;xL{GYaN*Hq}YAt5u4H@3se*ie)T)-n$2{sy4eH
zx!vWcCw@)(9ZX_@YQ!~cE{Zuz4OFEv1VpI&%vZ~$rvopB0Cw}RaAWT;3bem%wa*e@jw#}B-wx+*xdwuQcWw#LwV)scwZ#_SrR!?|1
zQNcsA6I1lFRMmZiK>!KT|~=)6O@u)u0lKJtUP`Cd#uI&YpO#*Xh7Oki<|!GHgdFr1brY8@l{E5iLGZ)VJ)-
zr)}o>eM>C}$fK&-xK;5h^iJ{JLKO7eR`a4+Vj##peKh{I8;Y;fq7vD8q;kS(Y9H{?
zGSJQBp69och#%ZGx9(&pC|KwY+(`hp@8zrfrYSl0PRM{jQ93EY)Dmt3sa`=sCucdxf
z&E6=h_wnfUeYLiOWwsS>sk}^f(b%?6*=V!-#MX`_GZRzao-#Y(6km!#G4@Jkc}%9T
z^}?|3V1L=oc(jyAh4`vxcvkvFBG$fd)p7O}Hd0vt)SJyYAxGNn*+d&Msz(cV$~iA)D!uz_ICx`ppmP7N;b8`z
zLAwilB^0`hfb*Y2Hm723Ik`wlGw;=^z
zMTQYiV8vJxp;kPoH9})ma(v>|Pe$eyOk;Brhv)T^yt9jTDJ}a)?T*M;*$7pyiii@0
z9!-%1d<-=+2e(uagrYKF4h*N@lq6>DhhfQ513P(!XDcc`DZ##G?GnICdt=O@%NS|T
zG%ezPewfuQzWd4l^I{G{l+H({uf-)tQ2Gk0Tfzz^^A|vCn(H)3QiRebp?qgewO`5z
z`0P@~(dj`J4AyE_JB(|c6w1e6X#V~a_7zy9II>Ler^(HAw7Jt8mr}z*6aJJ
ztvZy`m*3fJTORp7*lAJcz}y2FddIi5b{GtYU`>O=Zj=LQqvW#fLT5m~
zRIS$RizWzBztpG;l5b7ctnNfBPBOTvcA=^T)_Y^ZsOr@ak-6no@!1^_!RlrlX>ucW
zRw3+8AkpsnGzffRf1h|ehl=EXrx2-hTY9!M@b37}C(@mMDsV&4+sAo#*l}%h65j#;
z0J?RT8x_mEJyg%d{_RlW^|mp-PMgNSj5o6v%;fijZ5cjsl*06plRl|9r05@yHK8y?=roT&+AIJ
zw5-gvbRLu3HOju>NO?3nVW0r7zxhw+%fbHs@vemINk0^FX7z~*k9nSG5}`Mri0+G3
zmZUv9t|0tHm5;I&K&E5&Q2Trl6LMZ$#v-P5Ij-~Sb`^pNw?RgCd2x9Di
zE_#P*xoVl6mUmLWTIzeTYJB;4uUcz0?_AWvdB^KmNwi&^)vK(HXZ9K{uoo2vV*8o!
zmi7q+^J=so+}?&=xY*wQ7oNzW=>Sb7?XNe#X2SY)RKHTQ?Cr7B;LZ2D#M)+*09tpO
zQ!hyUzt%h)JWG{nuJbBYiOH6@=5(@}K5IbAgzkjsm}@K9^1T9s{x2im6s(xQNru6z
z6J2v3@wb7Kp_+jpJo}gMNhV=~H%pbfS&|vKh_fT=4`p4Mf7A!c3<-qx%;s~KT!|Td
zrQqOW5zD&0fa6_;#9X;Qs{CG86Pi(xe#wB>x=4)f{>NGWq0#GVbZdVwnAJ1)=f){|
zfywlHm?A;Rez8JQ)C>}be5Bz5iC4yyl2kTcVL
z_6H`7o92=z=tR-Nz2Wq|*37{r^XW2L^(%dwE-JC4%r^PWU@(<92Z&BNo=i-$`t3TS
zFxwOM^HKgyxy59qfFvMW1er=ce{RLTR-ODkJ{&^zwbUG
zLwUq93NlJk1b-*Vk9oiQaUWahCWkn~p}BJ9hn|SfRtG&{75|vw%!E~;dvY8uH0Zng
zt8_C1^!CRy<29e8NERPkf=$zT#ud>>-26`%g2|DbC
z$0l9YZ+^2jgyAswp%5|bROMM3Xt{nA9E!uhrv^>(jj|qLHL%zDD+Hz
zm`*vfQN>VYi;?AXiem|%vj^{{{k&66Rn_&7qfue+0#3*V>(@aydBVqnS22&}>hRD?
zJgRAUQEX?{<*#d*^PFVfElOn0T9_&>oTqFAS9Biro0S4wqig4#1~XG0FAp;C;1Sb;
zfu3-i8{_Qh1H!TcW-l-8-=h5o(~OU=W$ulZSU#V+Ls}IUy@j>-PwSrBTdT$wiU=vX
zh}|ok?r0y`*4Zy#%tYi$^6nNah!?ax^y+J9)F_;8bE(j)ZD>p%tdwsTy%W9Y53ZH6
z81{MV&?>6$&?4$RWm|ZpdLQ$PPup;2x74R3%6r?y*#Fw8{Imtgsf~D)?1#6wom*~s
znq|yZH2&dngRW>kp{lK-!ojmwwG>OYIf|9Dd_Ct#P<|=eo37}3R@}&9{n-}2|JUAk
zM>VmpYb(M=u%JRj5y1+Q2q;xRsvt@iNFpt$^p3R90)k=zMMVQrLT^b3y$A^)7LXP|
zYJkv`UIK<9p?w3o-RJJT?>T4PbN~9T{E;=oS~HV*e{Xr-=Xo=;I03vCPGe94@O+?V
zJ)z%n*(`a4JJ1S!8GiFj*EvNz6My@zlhHT2tBZV!F?jEqbpZBXpIcO~+dE2>U)by9
zavIEKy8K+MA|c(iMbYYXhhd0ugJ?m>DY=(9`3f9oihB~ZQMKRJ4-YpS9zHu8=Eaws
z|BkOaY3<9v&SL`{K{qI)bqIHi3YuT@>WlD|!?8Xa+U2@wZYSH{A&*=_k`>a20GJfe
zp^;V~t8u>GryqxNgCpC^4admK@?83)jcrEn)WKa?dehih?ahAusNl0J8hNRN>u)>m
z*WnVWf|`(pmLWG(!3kXZ@KvFGUMlA`s!JFZO9g(mED~Q650<3`It*JGLQXRZIIdjU
ztbaj(pS!9652SZ+dOg02LMhq-A9O*$XWrdC4Z(i8W&rA;y-om;49;Xlgdw2*PaB~f
zJMaN|RSA0|ht7ZS#o#^_0N}#J50Y!2bYQwnNNn*P&v98uzCH_b5e+g*vYZqK_X%&n
z=FI0kRb0KC@r@g_wJDk5i#Okf=u>7ZAuV=+!}6{XSrO<^K988X%UUn98fk$+P(?7n
zv{?N^m;3TJ?!a;8j;!n0A+x~G%-qwiizBG%)O^H|oGiEta>lG)x55C3UcH(3;ttF-
zCWnJUfCQ*7h8sJ6(opJ)Xe`45ouxmM`kcKn5dJ8fYUs6iN>CyyaOpX_$eKp&K@6yL
zos$wRBU!W3W-G+g&qb>DpUkNXn7cemQ5yf45ohm*Z3G)l^NQv=pRXc%FzPkU_iq75
zr9`8hh8%rIf%jEY{X41bAyP2X3&OBky~Ie|pmAxkH_?y;5fW;dHDI-~x}BFzT$YlT
z_lX#3!r-gksgDgoa&D`5Kk_z3_WGw->jdhk9~D)hpTq+3AEc)4S%3Le`mg;VdD5dHncG(qoC*t5l*63>+^f{_B=iw|O*
zJ$)CW{A#cp*VLUS(%9}%J(m(3?lnyCyv;56t&=@0jgd<@4(E8Q9pHu&xG2%WHtysk
zuB^xz4*BTv?bcU-8d@li=c!5I6uhAP_GNn{N@DQMXu38mBmAg?XJy3zhHVM;b$qn0
zzP>$GozIOaz;UO9I?&-r)fgq|0@30Zq-sbp@541LFTwXnf^_f-t}oio9lvOzVNFZF
zp0PdDlP^=Xr!W5c3lr$l)VI$Ab7r&Ad3bJOfQ<#nQeM$n{#0i4BbuADSaB3HXDd|9=sim
znd1hcRzU8gy1+4OCQBoie$mqhitZxoE6|uueNc^Aj%|Xt6{|StYGN*MERw3TmlE?BuwYkYM+__TqKQCq=Pn&}!~&cHfHKoKZ~cVB=-1d6{GIZFW!h$b
z^UbMtt(Z5a_C2`)<(CuW4HeItW@={h;KkMBrVG#4c#nsSm#y8OY`0pXE(}<_SQxM2
zACH`@!MR6?#LRYC7CH1I1eljb9C9Y^qF6Ubfao|z2TfXiEN6*ohS%(2@1x4hAhdn#V_L222P{w)
zpFNplj5qsEVe;~36>MZWtf&KD9JR3e0K3xS^f47Y7=7X-g$VAO(fhC~Fhw=?F8X7S
z8N0RAemp^G`~V9)FuVCHlFN85D0uSVm77FsPokkqLzCP)eUg(R`lwrFgpZm1qWEEh
zb%;9n`r?y_!+vMnhFJ42m1;{I)B$?GagluwMf4ZzAfRl26OC#f?h4J2ncZ&>qNk-o
z=5T@ihJro?n7aO+G_dK}f@<#`^h|@o$aD{x3|sxC%pNCc8U!P>Vle9`fhcExS>p&&
z8)7(XYXRM-tTIvU$N^*6E&PQ8?ToLlhtjh$D2@h01v5(}M`b)9hdx<7C~n@f=AF>{
zoXmS9XhdMM3tc7db!UQu1p6{WEwVEU!UW>bD_yQ3Bb6ztgX`eZRml*=K!i-I$jhS*
z*Ty%ruaVRPSzu=kXDr?UnuX$ry|PfNOB(`mhhXHku$Oalt>m?2Omv!M&DzT@E0<^u
zmvs9$s8PD*qzZb^8ga<%sa|!BE_kIBf}!;+6VG%MB}6!R8MxL)v9HZO&-(hXwaFMn
zi$s|$z>J$O&zbqvk{jTzluq`8fs>@lYCqnzPX+uh-Wt{UE+4e2Sb6rc5bfBimN&DO
z_SlpqFfOGzxhYk(fvw01;%yUd#}q8nWO~BkoxM;T*0M)L!@A@-v{X4gCV`DgF)9V~
zd&DL`z(6=}_2v&@Fwv9uCiAVS0{DCtFP$K9g!}57I(}Z(5GL@^xmUL;c~5b+i3gFU
zY(!o_5hvtI(uVwa!9+FI8u8bT*<5(vkJrx68benX?O9-=Zs1WCZq#uDpK~oqu?Ip{
zfR-g`V5rFoO+YRRdUy7&M}Fe}gE+ztP~CMEhwn9OS(O?dYAls5auR%bkUhkdmDZ7M
zlyY;!>p*NzeA&K+nLxs+t}pTn
z3NWEa;&fhyz(vOEY?AHcishz2OEAT=C*wi(g^i`Uj}2{)rTe_VLutVQD2}e0@xXJ3
z^v`hhN*o1fr{)m2soHFIsd-{gQs5R}+3593b#Ks*oVUqVUfyDRbeOB;N`isem$^_V
z?y<9JY0=T}&R3VROpD6S6J1JB5HN%*%tw60`{Kyi=RMul3TxQOc%oe=xdBw8W;v5i153jHF#qNP&MX$sAC)*)QPEL<&c%^L(w&nSPDh&loto6#f
z2ioBX-2Qj>CA>|i9^T>Jr0rEO?we31mWAoaLGVgJ9y@pCH4s8v`Kgl=c3(V2FdD16
z&Zkk>tNW;f72M)xewywTg=s~2+0q3ArMl&=jZ3hp!Xf*XuLdp9xs2%{!T3u3WBGb8
zF@3ba`85LGIKhrLVoE6z3_K7qbS?WCQxQJuN2i{qX9$vaHvF*eDVp!-v0(>%4Y>z*
zFnZNx<8|OtqoA<}M!zB+pW5euhnM$=-HBp!M+_-iL%urNoQA{8qa;sx*etr^sMEyC
zp_A%Dy#-MPSGE|bt`?@2dj$T@4+{yyXoiPf$hwIs_Vn{qjnCWBYMi0fLmH=~EB4DC
zCZ=zGN}#?qIO0qQAEQf$WcCIdsM`Jl!sqKvL4?EX|1
zuhFo;7b|`5BmGR`!MN3hWw7GhWuceH@s0$5^G}`)G>;gn#qkTuBxD|=RGk%q=)!7o
zz~Pc%m^bD(mP_&r*i$VHVTH-ndJ%+fy#MTu&}nzA_Kgzlebcmz`gY~9K8E2ZKIv5c8c#s&1s
zhp3y{MZk`5=sCTG4SSPfjT8IJ=T|+TFQYnl_Q}(~SOUx%q1&BSE@8!Z%mbjdECk{F
z=E2WW@`YBJK1K5wY-|p(nvHJeG2C9))C9lC*IA&e-Gb~7z&o!@r92SO
zzJ>^>C}`r(E;Dsmr$ap_^Pb9j^+8?A*L5+-^$=OVHe2}>;BwM4BGcVzVFB**=Y8Ei
zSuhr!SGJffZ8Wm6#Rx;^$h`H2z+ijB9qm>N@MN@w{+tLkyQ_Msb26kfE5mLzt_4tm
zzH{F|86~iJqXKDO)|=`@R>oTIazEB~`Kxtaz)e<;cK8E;0bW;M#Th-Z^ZSi9apXFx
zs*Hiip?$-4CcOGuSpB53Y58lBk!agt=B&x<3oMfpe)3j!;6?2~YYhyQ5b$_3T?e)_
zW9Bdgj|#$mk#=i4q;}%8$sS>;w)<=QY{8q*A1yHoP}0oB$8QL}EFIvlV=zRF=Acly
zn5NK7mpvZ#bLRF=Zf_-OBQfoly>wVl7qFuxj*}Xs0K%zm?}&!=^BZqchZ;hwUS)Zg
zW{vq8wMgdIg@MjM^b<)qMjjM)7E)1#H6C@ep2o0sJK@cb+zonku-q`9*(tPSlGq+(
zLk?7>Y?P~9#os+qQz1OphbWS-OJ>|D}#D@#$A$#)7FDNh4*VYJF|D4&!D%u4kzzjj2H7Iab0{89J{I5+8f
zbb;VUV}sQ9e609<8il+OsBKl+yh_ng1D#+V9VwNCxAzyy=-wD!s`DS+9&!*hbZ`5#Yh=VU5&w0z
zU+Ch}9+uEiIj4Hv@`Ap@I~)Co_NE``mQYno8%sGSi^|o&(4*cF^dkDk>L6F}cZeVKlFY
zLo2){lG1bcZk6l83p=soJWD~Yb+>de#rUi@F$r;@zK@Fi&UD+n3J$B(cU?9;-7}RG
z%ITbSo_k?${L!D`fw>N}34Q=;e$;cy#k1b*vvkOq)(xlS{1a>IQ-j7vB+}(s=Z*$$
zPr9cC8jVgn>_SQdL>J7ySUdV>j{eDD_oj|ANe7SqcTY2$au~A5Ur&lus=8Fpl@*wS
zd2PlTAFQ0>WE=G>d3+=%B~Y%JVpB!hu=^67AqS=9=NDis`Zt!;J(Zf`jZ3@;IA_eyR4C}{1^N`voz?J
ztyfYQCkCteTTr`1gOMh9)spgHq)UaQNt9_o(Ovr3d$tL8*PP#!+qtvwv;`;V4lU0_
z-{G<{MsZpYb_Ur!FbD=4r#rDQcqEj(R1(=(y!E>7b;Yp*c_>r;*uG_DBXhWLW<5Zb
z*J>ED%<>dOCy1pp`Ct{{F6QzQaQI-F;kWmN*!58c<3>0ra;jMbH+K7ZvRR;@T;cSy
zt1~?|VA3KKO_M{Qsh8W4zah?x?)&unEC5g%nJ4z;f-+50v)YkDri;N@u0<1vP3Snp
zbhht6cJ=_KHp{<7WmudY{rFz7ajtWHN$Wi)JjwIOY1a7E*+c^_$JR*7rA@JLi}>l%
zKeiu@)~b_&2Hic}lf5@i>w>aO0i_D3`+zd<`LR&4O2kC0rqGEZFc8@-YAhh*But?!
z1O4cQqziO;xtu2v!gB5M(zW-PCL`;>CUB4n#FjSY@&TM?*{~eX*`e7DxHn7kxlpv5b22gt?PsVDyg*JcK%@!SXiiK|62H`(#lGu^w8cINxbB6i~4*u
zzM!q*Qj&mv3l&0_i45{z^45b!C*ZGEV3x7)HuYgyku#R=i
zErCy-yo5%BWL?JgtC&D+c&xgK8P{L+0;Uen3R(pjZ
zh&WM2yvB{l;?z;F+I2t^Ajs_7%OtCa_f5~*l|9{J$6lm!@1Xf(iSJHXLm)0iC+9d8-XO@V7
z>|G^Epdh-*Y@?2FAl(47@fywRqo^It^7WFv>hBe~Ig%aE@^*X&5H&9S01g
zFc1ql(y%gd&}_tQ%}GT&Lg)t>=*=&}9UC3SdmVf7#l%2kUDuNC`a;?WL?$E3S>BNt
zwVyWnPJn*j4Vo6oB@AGe6YpBphZ@3XMfk$TD{dnvq6G!zluH`2&RSVp%iJst4P|Fx
z+K##1an8)YU6TE`9e{?hYhhi^1xwF1n}6>v3QS`-?d_u!E36No%#a__>d0g-gy>XP
zE!#V)U(Ixk)G(V(@1@IepLv@MtHpE5W!>*P6X$$0P9T<0OYNJy3_ZQraF#sf03L~6
zhxTXPNxQ`0OOK_IdJgZ{6!V7L&+Z-Dk~}2B>hI|M;6(tW_e~X$27wg3IRMwu8ZZ}e
z0<}U-cDhevf+>vz-QhM-4#?)t@Km^og@!TdlFK!FitFmShYT*tk6-<))BOr?go=)(
z*%S=tZrR(3&Xo)4#)7fu=^**p=J!0xat2s}MU9HuiTx-Bg$(LB-EB7a9Slk&lJlLbNxzTWR=k@&H
z{)g6HyRSzw
z=H;~lNfvQT+cm{AbUW{@%s@M5#nT?@@=gy=Wos87`n&A^`h5h6*xDOl$@NvP0~!l
zB4PgNgNcjUD3m62t-h(r0FcypEgHt?Ql(Tc`W{)IX12yJ6G=yQI+_7And#9@NAcZx
z59Xivm=>?+T?{x5J^I9v4v4gqbjIVsPQqwp!WZz$*N0E_fC*Oob7ZwFj
zjfEM5tOTQSV?s}1%Z%T;$}$TJ`)_xl+spIC2*yP!kxbyJHtglx9N!0FQNjj3YNaYw
z&t#F~LnJOjmf^J){pf_>rptO}ioo5%X8B#%fX-5Sw$4}%!wnJvO&Q
zlekE(Pka8bgJ6b#+@MuNvEt6ya2yx?E^?-BNWS0sJE{Z3&Zk@Y=INg^Tg|TqsA^G&
zw32?G&T)#nfgs**DLdBJNd9GUkt@*k6d8^=np72~sj10&4G58awA;%avMWqkmX{?9
zCw7%MJt?kOZ(m|8?(R!tyvip5qlXx9fWWh*?ksMHC#8=>xOMI--Q9#eH^5$iyw4mH
zqLk>7_f``D^`6_!0uK;?=pjG=pEz4=v#)9ORNJcQC+3+0`eJ_Y54)b~dz8FD+b%&U
zRa4eCRRsUIjlbV|WB%P|pHyZGUIutez!5e-1_)dCtgbstCLEjIZ+jx>{%j*G~e&tbh!f1PRHw+dUGr6qJipF?dE6n%2e5_hy6V=*#@G`
z)5YQ`#Ror%ucQY=-@#C{9yUp^6&WV0NefzXL#8qS($@09t{4LA(LdnUKh~Om$UlKU
zMw-@NhaY13{T}Zq(f4gm;}MuP`K?0r#7E3L2S~BNlk_f!^Q2r!eLvm~Xj7R@Rps{c
zxyNft-FwSpzP3qZP&>^7Odak$+0)nVx!b*2x<7`XDN^Z+M^Sm4P*raji2isGt;~~Z
z(wY#hpMuxb@-hBg-QPKOTe2nXz1_ad<8SV&i0lN?JYx>&LonE#BXKJd^s7o4QTG9$Y_7b8OGnJh!Im&vkZU8PP%)9Tqolx%WW8nVtG$
zIuQq>=_i$LD)#esk~=J{N7*>!hj;~x+CeeD&(4xXOz_rldH==Uw%Uola)&1suk4Wm
z9wmfl10A-LyE?wRd^;4{JTEugn`e;}A1hT#4h_`y7Z(jzi0B;;tJ
zBL#TPjVB8Y2giwTuhn&;=Z#ldg5*dMyB`h+YXgyf40QmLa3i>_XAOlocf-%Br@
    wgz@(|uq{mu-`@L!eDq7Za>UARm<6C{M(R|j4fcpB?&2ycmj`b_a
zmPNKLlP7^2Y*;b?QHw8rbSOY{d8|)BX~n%R&5w3?HnvCMX@lx=qLKpMaQQZy?!F%-
zC03_}HyD{NjtnQl4dP=)=JDCMzbxP_8vYw78f&cI+#b&d`oo=nt+i6gwgP0{Qg
z*5d6nEpNgx#eOVoY3MOLj%I$&x()W!A=q?X(ny(9Xab7%9$*fU{~D`Ra9sj
zU68=Guqj6Y4veQ4jX&_2{-RvddE`)5-+lv6m#;5$<@&{H@O&ikQ7!29Ec}L$rUks*
zY~eFpIuq}L4RDz}1f)Fecrnk`*DL&s--vC;n~JN*SQ*=F9^YBpM%
zqi_8qo|@nW@>Rt;pR*swcNgBu*GfFQW)sgn5qz2r%)|doZvSLUf$Mv8_r&$ox@7x3+#6Dt2Kkc({wi=Wxr_vC(22GQS}E;@ju*py`?_`1L^<6<9{AjaR3mq@#B`~{^K(_v;5#%
z{=+lf$zk5vw`?zac=x~4%>T2C-7x_?U~_=?Kk%ZhwfjHgN?)8FzTCErUFWK*vf+PZ
z{aJ^W;6WQ4wz(*Yd8262ZNT4EH67J#l^enT4~p@cjsO4v

literal 0
HcmV?d00001

diff --git a/docs/img/pgsql-ha-after.png b/docs/img/pgsql-ha-after.png
new file mode 100644
index 0000000000000000000000000000000000000000..5c916fea7dc3f2de7f4eb07fb131e7f5b37a5b76
GIT binary patch
literal 168784
zcmb@ucOaI3+dqD3(2`Opt4O6qvXxOpB6}sHsEF(>qal=$l9kNtnIhShP-K-=X34tj
zz4^V)&%E#F`Tp_ywKT@_i?<|aXwH|kfGkhxQRp}QOll_QX!Ej-;zjc
z`!`VHU+#CUjF3osNwQL>)f}G>wL563Y5o#lnccF*F5rN!v=YU3Z#te$=LH^?YrQ;c
zn37daFV`@pd*LRNCC__LGlw05*O#yQJ>X$9{O8Y~XFMq)^&hW>b^OVcTlWC*eD%s?C~4cpqrvXcc=-MvmFb1fH)c$4%-c=I
z?6226f52l{-u3apj!@BCZ$g#%9%v1`@MPGt=h5fDgWXH>lUgq?pEi_9RNkfS|3z0Z
zz&uUg`|sB?0RhX6zq`6f3=9lHLPEO={r&u?b#!#bS}jWWFMr`X6m))i)jE&)MA(*@
znHg!beS7zkNOk#krpnz1D0#jbDcrepXKZRp%G{jm1TQbsNs;S@rly{2o5sr2K21Qa-&CMM1}Ito3Hh_LFbVQDLL+A%*r
zA6uc$6;fJTOS5(BR>!d}3jB0Xw<2^ZIndc;7koND6fkB
z-qNy{hiAL!t(lLdUNk&LHOxA1El-;Yb8~Ya)p@(EJ>O1M@yOAmA3xmR{5Uu`!KM{o
z3mZ@%(tXA090#qR)BgSYh1`~ef{$J_P+3}DT6*caXrDenMZ-$`$%P;Gu3sJ(>lHb(
z;X`sQI?Fe67e?EJ4SY*9Y+WJJk-Js`_C$wLl@o(15(5ctXxDYOI?!pCPL!LNsA|O26
zCquX3Lwh^Rp+kpu30re)+Ps;+Nttt}R*ngsi;Ihrw)R6_qnfUv@2_8GW!*UwBT*YE
zx}BERz?u<@$JbLYB)2M_w$6)#Q5&CSgj%QNwt?3DInjL%t)i{lvjkt1iQjTP%~TQnoR
z>nOTqH9v7}ZehV-ZtSOHPmpF^?Ags6`}Ys{z1HE9Ie&iBk2hvM_>j2x`1chRR48i&<3lwdRP6
zJkDRTfh}8X#}j=IgJpy7)0_V;#`S^g_$`ANM9afUR_=0*&XP#$ifOly+!v&|sHmu1
z=6-thw`A>eTO1br@?3ZauC!UZrlW)P!Gi~umX`0H?3a6K(J4lfkdj*0l&MdHgIG9T
z&U8*eK_hct(D^LmdUmWr7cMmP_?0yi6BB%IQW={n)z+_HAIfhUyf7Ol8+hEjO(-z;
z5`#@b?at+e>3b0oEVxHJ_}2)@>hGj00X7otEl*@kbpp4P;9kjG{Z_F;;{E@x!Qm*+~b#*$T
z|6xmcA9p==@EGro!YpK6%j@crGjz8h4{mJThqgpI2gMH6>A%yxPz
zJ{KnmSPKgaUujHH@2`tli{j+9oA-KGQ~D*5km;EjO43pNVx8sVyGUF*Z*T8$o1?I@
zvg-dD?^7;<5+Q?%g`IdDl_K5iwM6xpdI+~ZJ&$^?;h_N=TibOcl-FP5-4!AsPE-Al
z%SW*qNvrOC5e~;~`;~?|OZMOmuN3{P4&_;kcBAh$A75T-6n%Xz^X}a>j*gCd#l-YR
zUtYs@(8@JuS(%JlZD5L2@}WO=_e{(?9Q)%oz4G_G-tBlM##TYG?O;@xq5GN-@UQH3bF)0DaM+uP1-$jR9JX7u8NvII6eK?
z-Fv2t{<9rMYQOsHUyw|SXX9$IY&ooAokh18I^7nJDMX&SVSTEou&|-R0(F>D
zP$Tq-SWLxfUzeu3#n%FcfOK|w*Ovu7LU85kK;a##l$%z{<^?V$rN
za;Y>MyALn9EmAzc)Y6bg5ib=xzBv0_#meBi$5!6XGqX?C$(<8T=|4O2ctyA`KD6{1
z926Jdld6&aRalWDupj7Y1REztEx8{@xHwN6)
z(W!I#Zmp)N9y3=-*VZz@qy32aL2&zv%Y#F{tfoz|RTCvYNA4`YoSgAYf6YtUJK7g@
zc4J9?zV>o}(&3=kl$4YXyYq5#$`>yt+vrsVAEO`vlBfxqfB*jdeRVZ8DIhHDkaTwV
zPPy~?=zA#SL;)*YnyBjk{`%mWwQG6Ie(=dZ7utSnX6Q7EiQDS3Ltv@ObVgEIT0?79
zd{WYevuDq8a&a{kM11|~X=bZ2UBfdIu7EZ9WVP)Gu_kFO{yftjTJ!NS-KY~J2@MU6
zBFUekdAd#9`4`VGpZU9D#3@E}cBRODk#Vm#Y0G~_F64ZC!t+v}Lcr%9(rv>X72O@5
z1K);Dusy#gS?4;eJtx}AAaG1OdfNLj*=~e+iM3?K@HOYp)tOcc)KYo@i;gW878YrG
zMU1Km@~35GA4Cu5m^5x80VkzL;yZZ0dUc}Z`}YT8*;=ZXN2Zcld(TJqyK04}^xYM^
zapT50Ik|gLQF}bM?7j;qT#8*l6rT_-U7|ie4GId|sig!XnRDlT)0B
z4}UH24G9Tp;sa>?Vr8t*b52vVe&?i5hU2DcZj$Bj5B?WkyRI(BUQ+$?Z=o!&Tcpm&
zZ+Y9wK32AOdbHf6ZO!dpM+W9@4EYF+E!l9Mx;67?phU0i&WhC>IoVQ2`!(bGgYKtm
zW~nTH2ZdN&mT-S}J5s|l-==Tt_1Z|kuU}P&gK>{W>~=&%M15=4>(`PuZyqIeVY@R<
znzy~BAoVwszxySuxZ=^CwWOOjZD~q`FTW#ATO|#$@hg41bhXrz+WXO?3V_}i
zwZq%DZM%K?9_4_JUZc-h{gxcv61{IsUn?s&v543vIn0;GGJ4jux9j#Tp_Chb3FRU2
z%_fEp@Jx<6LjJ1z+!QdjSKfF3V8n)biA~*s_A;<3(2~Q*ql4Jp~I<<A3dT8=F&}>JnbeKms#9h$6Wci*f}=2hrxB6??6fa0@u@RY5Ljx
zIZOZS@~cNrQ|pid=6fzYpC8=+tSj~
zj=L_{%7-4O2nq^1ckbM~qFb|k!QYE+3Ht95-%4^^U7l?eC7IY*Tc3HvD*92@H!O_T
zHDz=R+xo_ohqte<6aZmLYO2*pn{fIxI0X)cuThjMJ+{C0B*5}ETG}0NeRTzRw&nkNZceJubI%JKdPj>fgGrim9lm@C9$&u;JazaO>lU2x=xKCJ#@~5AWYo
zkjCCq?df-!yQr*u$HQY?nob@qF!MQic`0S(?QYA{nomPQO22*Ear*S>Phf2ib{r)8
z8oIcgGB7aU`?XXrv!^lqO;Qr)&Gvlt<&*pOuQxKK-h5+|O+sc|+`7e?;nLd_YXbrT
zZudpr+6qRBcS%Z0dIu^23`Ao)(spr>;ugRiXr|?8hp0}T^?D!~&O?XpczID77#gw#
zehvrbs{gUY9e$UWS;`5tF|
z=&zJLq}wI#b}I7Li171MPPCvk8#Zi^(9+t~*52MoGs+|hdS1M;VEVD5qC3ne@;w@#
zvWm)G0ka<@(v!fzD~5*mT3WQymk;r|o_TaHbY7lQvg&bwb1aZa);J$WNz0EP>;eMx
z>g?3}s}{WZt&(s%`Vp7z1Im6esT
z*;y*3Z*pv+qAZu*+7s|`e
z|IpZ|zPyOWOBpZ674iajo!sf~?@yU-#A%bj4MxVr#pV6*;Td#CiiRz`Vb7oYZf4->
zANowNNopp6y&@uvKY#vYq{&maI%U~K5-@M`MtO^{N=-}q*p_GW7&ppod4Y|KOF?uL
zReEl|FDj==*m;VRo?FlN#lDEFYt}THjEE51M)G+RjfC6BFb8;K6A?X%Y$Ksyyh3rsXtQ9i4#R
zdg!P=is=ExNZ-l6TEq8u*Wb=ibAA+mxWMD9ScbkHmUisCQ^YJRnB|2PQ@(1n}6}j#!Pcr{U(#&{I
zL;K)M8FoHc=9ygccD{sJoYFsNoP4-II#29@hY2bxH_gm`VxjYN%)i<%%#J>~`1<
    <5(O<9dci15mWM3}X5Cbns+sv~IJdJL2t(>Y=W}73zg7PgD`x^K<3$imTWsU7@%
z^BSM5NFE^$F+VrzKKkQieR@v5*`;D-Z(rV$WjOxGed$F^waS69c4
zckB%_>LQbWwAAy-+8l8Q3v8Z0F2c#VffNuGmD#wj8?aU+H|WWeyD^epzkpVDipSC)
zJ#avJ+1=KbFHYqUAQlsk^e^MiVo_RJTGZ>CVt!);*S=p#6Q?{PAJc6Z;K9&9%3dUppn
zL@5*e`XSa|0t(ICcy~*VDZ@xx-aY6^h95jOIxbH)UokWD!`@T=kP6sR?7C)aV(m1as4*?C65F}PEh-Jh?tM|h&L51=lAk+@Q`(F
zA}#K-QpNP>8apQR{skFqc#`gHCh571pB-w+T3fO_b4jw?%e!Au@o!lg1@RAnE}4ygXeh{dnGGw%uV4+Ix1w9-Lj3C6(N&Z
zElk~*K4H^WL#T)_gT`u3jwUggX6TprgCH3$Obs~B_k?7D8T(zS_3#ulY2Zwc`AsbS
zw?sulZ2t{3G!;q8FQiPnXr2FF%xloMT3#DH`aHfZJb5}JuQnIJtnQBrGWjP3dYpHT
zTDJ~0Dzjw8S+e}Tw>Qam7tfmEA31ADg^MEv&bQ|InwTb~{7%OxMcWNFp;49N!YPbr
z`d_K>@bGYQb5}6vyW9!UbKZywxHek&cP3Yg$S=TEvRC_I@s+uKhYufyQW34hDfOW5
z{$_@$@&RB7!bG4T>FMb)T5Z|9dF>y1Tk1iTVUeR^VjaK7Nd3zSDi8x8gshWjixI!i
zDjqYi0sZ7Lo2JuCj=G{GOES#=<-dzO@ldr~ub)`~64k5=pJ!rwv1
z-VEga^y$-RpzFrQM#rT&9*7ZaLFZA+?*8c)kVr6F?xds~)+u(mSi}O(ermcw)y_@+
zI~T
zKE9pz@83rY_%@&Z`rzjt>$&-PNWm0DAGEUKfgr*+6BYW~S{V9tj1niIh%*VA12IWF
zH5+W@&Jq5{Yax4y<;ecOot9&jbz=Fu0&6*FA2>UPFV2ppdNn}K{oi&9HLd%#N7ZoR`ynG&+PLb;Q!i(W4JcGne
zMzyTZ)E5V)kufyZp;N8N?ajzsg(}(g`*R@Cr%Z~DXl9T|BDVb`85tSB>}<1EdZ3!~
z@wDh9Cxrc{A?-t%z76AoyJRUgGQ)W;;_>4cIvPbq#lrdCaB}Bb$}R6;8f-B+SI3~|
zs#cL>KYu)^u3MAA@T<|edDA42+6|(eo38xt+fJEL6~g@jcNQl#RLF`QOxVPd1gWqT
zUk=?m!)67Qb9`lS^aVC=3%tad=fd{@4%m5kltp*eqmueohYNOdyU%Y1SfRw}hTByL
zS4La%z}7ut%z+0Lo96bu`qy-6nosxd?>B}nvF^xVf*H*u-UfBAA72!)ik_Olu@;AO(cfvlBSGyj(!ch=|iy?bj3)ub=Vot5^1
zl%bxi>_(7l|NU~op*y94Is-yOy#aQ+?=$Gt<$b}8Ah0r_yK3cG$CnqPn0|>Az0}9_
z9ork17~bEx;kZB)w%znLVM)&|r*GUi1hnP#_N~ylcz4^lor`mid&XyHWh5j>q%J70
z(&?rqVmq|m8MSMD&KjQ@Ocjqx>;Cc?1{^-M`I*~T*J&7GN_`^ko|D(jQYbU9D1=u{
zSOaHbZUY)?XatwPN_cKjt*h@2Mc{5zfmJspKvGv5~9n5)QhM22R0dB)$@$vnT3exPVy$&BT2VO$G-Chf}Ld
zlcd|-6_1#*{n)mGpd}QktH-XV`ENe7@|dzcb{1^dpt$^=J`_u
ze!L$oLkJJaFh0DS~B9w>gMT2%DZM$Jcy?pZqD10NAP23ybzq0_~#z1SJ
zMtk}A)Vi-|n-Ne8M%n|IK-m!UfItX|=jGMUv_#(=85ywz7>p8g9c$FoPj2?{@hOGH
zOigMjaNy?BFW#cBufGNRDzk86lXkZ89mpF+AB6Mr!D70u4;kwsGilArSj0ssNWRIUdT?dEgs*zDQ_SIXr
zWf2!w*X_*AzR}Tp0ljXdP4K|)6;mHJ%6Ayu8YZ9Muv_)fy(|KxP%E4N$SCcR~Hu?=nbZ(ri2v)y2~iJuV{Iubr+9;N7F2oj;`)F)Wa*_HH4Nh
zj$QN~ClTP{W3BkAuS16u6|nklbk<#cQ-x;ki)E_E`Wg;_?Y&O+mrG
zMreZk_pdaz)QWLKui`Pecl6Od?$-b$wleA)&zRHUNfc
z$+tU#?pTL@L+qmLZkgazd&w%m&-E4+p)Lpb^fxE>F}+v{D_GeX3xVPF^=g&XNWL4i
zNG}-Gx{s6@XVl+;T2S6a$D6TW3
z!?QR4Luxwo1joVj+{=cBYtaTt{G$2I>6g3!Mbb`tedW6!GoZ>7xAee{8Dt!2BW6YS
z;>8E9i!)aSo6@yKIgj1f>boTAZN=<+X!J~}skgPdd4+O}s;ro-EO0!$*7Fg8A3A1=V-cGSmVi+
z)xgf@=)V37tHeeI0xnxwUL61FwWo;*sN^U+`x-1qsXI(h=h7GAd*JDj_)8PRCMgv
z`q*n#!Q?9f?(;niMMYmf($dp=qVj58x@3qZ#wfb>&NDU&b(7|6yw|_n#yb#M0lUaN
zh*~)IgDy&X`i?nB2w^+5w6r880+A7i46aj4iMxBC>F*Ns0_DSVTk5G@+Q6{~?eq5kq1Kofe_ddhFNMJ0
zAj#P{-TnZ(GZy6^sGELj_yve=(yAuEHV&=gGRxhAok}@5IcHJzZ$D!Tj*L?~+}QnT
zaelsW_Y62R;}ly&C7^lCO1X>~h#wWnkF5}|2AWwgBBO@Vi~r@AMjkH2_a@{n>GPPsh6#+r#9R_bH}06m0@*s
zby@tyx3&P^3;Jf$LBtJ%x}OEw2fv@$ruWNx)661I#BI7eS62iH;sU)UG&IRv{tbsg
zgatL5wA1l~!p+o*U;a9uSGP)w&$qm_+J`pak6Qe5ATg@xyJ-TjHtDvn@0P3AuQTQ>
z10VM@KRMf6dMF5lCdq;fif+4=={K3YiFDx?<*9#7Kvb`xNAS`b?fl;_4UB2
z6Fw`lG}@xeAop)=2WUT(mJ;h4(W!o7#$_=sK}SFL#y3iIFd~?Uh!+vigEsypT#yEe
zx3`baz4Y|-j_xCBxOG|As~G_(uAs2a&2>Mrd9N;1D~3toUA9{o`*5@`;K7{BLE
z4eAF8CINJ`kz!CmDi9R2Lb71X?i1@aZrr#_$Z{|G9HjoOyK>S!FPkio_cd819fP_J
z0`-xQq5%KExYrY$BH0p?6|#J9nPfE@C%
zDI9_#i#`TAx)^xT5Co+wWN$+4Am>U)#WD7FV?U9-%u<$m=nI_``qHumh774u=b(x51J$Uy{0?HdTp5!Bx;ndYLQt!-khlQf^A
z+YIMX!E$X#%pc$?K9b=>gX5-T@ndIg#4Tn<+L=g}zdmkEk2Ln$%0h5L0QM^u7Mc7U
z=~;cP2l;yzoY4S@8wVX8Jt4+ztk9rr6G;NnDzt`surda3F6|)dCK1a*oB2cXswod8
zlTCKg(Y=Q{g5{P55G2SjVo!uBrN(W9RAipc*67|#wr5ItKY~E|;jQ+;8U_0D>v*@{
ze!+How^09U2pPVN{F`9kO5htxJvgG|urFBt0Ix7J9b4ZeQ1BwZUeZg)af9?5QLZz)
zH8SscL4hVcqw%Bu9tHhDBvHS>zPg`{vZ}JOVUK@?h4EpYT@wJKf9On!*F`2zk00c}3d#|roGn&ji)$GhDk
zeHW6&!+GVBYvWXso#)=8lK%=+6sr%-Tu2DTmXD2%wd}8ByO*0j=wf>4N3KQY$k_>0
z9nBY->y^HJi^T@suyw1nyu2?bwNA?Dh|{fGhMmQ(bNw$Kk#!S+eZVvvXTBQ|4Htgx
zm7dBV#;S%9YSqQlc>!ZvV#w3Y2w5q=IjANXugIY4FX{UA+qe1{Nvd`0)`c=Fs^7EL
zm8AUf{W}GT^_1gfYsWJ+x!Ab`eQcUCigcnxa&wNHnwhO9qU0JSk8lamsG0-jp%CG9
z?xwWsJ-Pk7ysFF9{uFB|jjJ`O7gZM}%W}#pDpE@xu$L?CLo$tuqy>~SL3zXQ?+AeL
zskl@Ncj&GxhULB#dpqim37WT&NK@oSxu?U8XZgQe8-Db@S%3%vxFvJD!r25GS^KNI
zn<&!gzxei->!MGKglHD!WO5JvcqS+IA|V|l8z*`VX==Dtv*Bl`?e=GxWl4S2$}5a7
zR5>z7ed3Mkhf4mqwSTC^o*6icsGxmkkF`5Mm=dug
zbDR&|C+#Z%Mv1_}o!L(J&S*Qgm3bo)zh5BC>z3>rfg0Sq8)q9~(lj=+Vz+y|c(ISP
zVT9#Ku;=+Gt^{VUg(T;D4|(JxQtrsv`PwQsQW1;d#rnJHhN1Y+uzb$3!(`W?@Cstxsfh-*W9dW+JPNTBEe(Prs0BhNMu!~EqKIk
zyAk<-(tP9tLbH#7&qdrg;YBENuq%jM%$)s06ZgYAH^B%rPV0uA3w<%gqOxqE#xsTD
zHH0ijFyXV`6U0&kbuVQb$4pzYad4~~Eu8ePZ#@cu59qvshD*(YEH2AG70f0o^zn<4
zl$~-BO9!@C1ow7tS>s~*7m&i018WDPtY9|}BA>H^4UWIU`A1@x1gDTlo~
z4mM=^B^T@kzez!82+Fhl^o)A0XqR>IatwAZj6O(w#4T+U-8wx!IjJfbfE;3ZOOh#p
z<8D8*Fc?^z(TAH&>Vi0Vz>6w$*5$~Q(1v3K-cI_v@^4!wXxRf}9O{fn-||EiH_4>q
z72NPw&I{iZ#r^1xXmlZ>HU~k8qG35Wfd}p*GFYBKv4s4sJ(>PbJV9?%;KPRxFK^A7
z6B;#u<+lBLp73Tr5CYDX59`!4G}zhLPIq5ufdv+mpU()wO9X%sa3GQ3e-UN(w{_kE
zwQUyTTO1faC?9J{9qy~H)8QjII2bWF_6NM;od+9hY7|~1s$adz0cuP15)khj=?4Qp
zzl6APYH{c_*;OSRL83Ef&X7og799<{-=t8anKox=@1YJkA5W$_blBiGZ|uSn8Z+71
z7Jb3_*5$dyMIYM80GDIZq89!WM)J$^Mur>vE~saw?UV6`#ZF#Y1lA!n6mlr*Tf`H0
z^%rCnvR9==(R8rJzVyVzgVMgcC`eEq(mIBHnFO~o3EX`5>RQ!H+X0oLs`}8X7>I=R7KIWbP(_X`0{-}x{{|-R
zhj(|@g37@Lb_*rAQ0|H<9`eRF*cTcX<;xEdc7BJ
z&E30qiI523MMGZt56K`3ewrO7?xw`S{Qr|cNu}ThRYM(m>oCfip_6y_`EzD`6(T*5
zm6a9n`6_ZRUxni~D}6iCh0GyhKp)|#TFs4JA%fAZtxRx4KUG(kK@DuE@_AhAy@iM#
z5j_kPP6io202m+QSfr$E+H`+S?WxHvtYR+S$*PIZn1lb>ljoLi{24i^*W0#DRZVRt
z=0s9fBAz@^F}io3=h=dj>Yttk={6jTLERD9vZ#ihuV3>L22Yl|uV|-$Coy+({J7F`
zD-?W*3$@)pDp-?}8O%EhA0Ul;^ZTpygz-bUW!G)kcSJxFon*tNO&_3!b^ZSBfmMaM
zFKf|~<7GDlwF(EeY1Wlg%_at_v65Lc^<4B8J%kO_RlO{O_26AzcpGAG>bo!Vvx>X<
zA~0i%W(cv|eX+HZ&r@OtBa!IR$k2HJ*^ybqb^}E8GsqP|E#tw=Og0cV*gQd#(02yg
zGGaWK#Z_mF%*vmNvsJvWuiuWtUxpGxpk^%gPoOXKYY8Ag*td_Pq8{aTILkd{+p`BB
z4$MhRI1rN$h+q@9=XYP<3#>bV-67UKq&p%te8;F}GCiRC@_8o)+Q>tLe;WK@;j+{^
z@ZFfi7KeBXR~81fC&6h5xHdKPU3Iys8|rrevNR|%$(;dPwr=$VwA;^Uf|6ARB0?lO
zk+Ek-(pfa~?b}^ATEZxC=?=?)T+<>9+HN(Q*z
zi-m%%iYfo^_zzi7%aoTtYWwb3TotsG3c`lKfD+jMjVVjD!$D+L6@zV0GX2u~qS92W
z)mO-=;#iYoS3|FrXJY|
zc&)d0bTre-SN-aVuqNBa<5wlBbKMRfIWi8|77JwwUT{(QE0n#t#m-f|zTAgfnT2k{
zu0+8FDvLo53C{E$dHKf)(6ZzsMCf3^OGA7{Ek-+#8jf^ws%o5AsfrH#VGOwe0I5((uPdJ~dm(}f@jK+ZplW;=GIXKhQ`{VEHC
zNTbDzT#?S?Lw+4aaCpn1*%9jI1A5lW9h2#G6Mvwe+UGsNe;~|HJ84PO3zhk>^a$w{
ziCKqGp)3g9Y?$66#)^&|D}%W3I55yqX*9vSJ^y`Y=N`CV&8?j5Y9(&VM4N$rPT+nB
z@}5CKbO>lun}-q>8S<8Y+IblxRz!Gn3zNVW$TVkERGuV_QKhFkrcy8a&r^Q&?_<-b5>ALfJ}A=sXGKjl$DfjpL7`6HND*$>RMOVWf1HbpmYqt
zoW6D~zN6>DXCtX{D5^vx60w$OwZm3Dm2}XOee`4~K1}57nl2Qsj4ECTd^X~w%E^O%
ziZmNFqGyN#L2Er05Eyvx%NO|w?!f+ENFgDb_Krv@70(~|`t>X0(F`1v{)XfokdKLz
zuaj>pI~DQ(N+^cDaAYZrjEsngDlhL2L>*!UZt~Nesf^01AZjsw48z<5!UGq~4rFQs
zKiNe;Ou2LX=+P~2-nR=#6`T1Zr{FgClj>Z&3i@R3cYe}eaVX`<
z_+>hC6Kfdu-jDamd?+uMHjTvhfh$H0CWv-#^L85qvJ)*(7_XR``k)Iw^Sks{{OYQ*
z`~oFI7}3@U`I-n8pn!Y?!d6mKTLQy{!zA_{-{hYKO~+P$d*R
zokGX$*!<>0oPSog?b~!3XSww^>J2Z52b
zs~|nWOCym8PK6>-G)|mHZ11tf(UN@_4>HttMHx=8$%g<%j0K@(V$h2L^+H-hgB}?T
zLaB#cWqK5Yp_kxiX44!FI#0M<_zsTfU`6AkEv?)Oi0ZOq5+z+bXAOyP$Z!tdH>PST
zV_rh9reWFO#+G(&9O$vJu_WG3Fo
z-xi;^VHkADu+|+>eZN|r<{de^t=mn`JN6dVn2a3l5&7D%C~l)4)!0oEHMUm2-
zDK`
z=jX>ErE~5!_~hpF1m~H6&?xOlfMK8rdf0{K)~Cw`vcB!y+=XLn2-6%>mV3FlYNC&;
z^{}D1K~$*Elh^HeAL=6DJY|d##Hwh=~bFu7-PvvT)|zG*2lPo3Onw2{J@ub-Wi%k$@(Ua4(sHp=0*
zbs{}moo{-&c*F1;)rNxAwelyo-u^k8jJOtJ2ULiZKwyTwyDp-t;O*P4$y#wQ*yxDh
z6OP8cZ^Q^d19Ri|dvawkC_wzn#8<{?PTv8lNLqaa&t2_s`fHy32s4jTD_p*OIl@92
z_v0r(jC0?Spl8EQ0OME>FVtpCL60ND(~VhFq*&HMz*#L<+!Y1ymjmkv+HfwP|cn)f=C+;!?sN0Xk7B(a*M4Z7iAl&Upr{3MDD-U1k
zAAN9Wu}sIe+6XA5`FpBD&@A?fvW`v~&nSW&+EtJP#h0gcU>vWjzkdT94dicP)KfKx
z$w5@2_~ICbDiC($)YqU46L=BI1_;!yj*RTAY1z1^TgH<^G>#Sey`_Z<%7dgk+rW3P%hCS2wNZ>~32lnV4
zq_<#jjX}v{Ky~-VNG)M0KnG<3zZco#80M^Za$Ld
zM*sYy`TI%Zd`;8qF1OgFhW^}L+AuiZTTYv~?|$2A*y2K(fX!1c@v8Zzmc<*YUk`d5
zn*S2I*Z@zJ;446#M8*#amIuIex*4J*tnhQu_L0>K;k?0tw({yKB_S>Oa2I>CoV4$P
zX~ioN0F^&=6*&tTXe1bWjCPoZta=$k0RYs^&onz*HIUcSdUf@|d#iAaj)G2DVVVe|
zA*c6!SzR7=pUx=35St6!7jh@^KTz%~3Y{kF(k}^qi4xbhaPj@ljdlP+jVf@Em?yRR
zCFREy4XgSm+BzC-V_tFAVwp;4zW$f87MB?ux1R;0#owN}5h_8v3Q>nTT_!IS{oEuv
z0xOb@5Z+4K+u8^Shq^-rR3NeftZUL95N?2^0OAb!3%Mwyv^#cS4A=wM9?=V(zVJBQ
zZ^#9W-%DS;0sk{u^8b_XXzIIOtAoh=SIMrMs*X6xfVCsw1f1XTxp70%JEDtViXI-_
zFJvRik><1A%3xTD`h6Hf;&{7lP-L-|gJxYooRT13q%K^a_GGoX`TfY%P|{?^;d^QKBu;yt9}ETI6jI
zL&9Uhem3RfII?pSfm=z0H!yom6n8aNe8-JtpN5yYxfO$%CFZsyoK`}kx3kM}W*+!q
zfs{pmzc>gsQBM-#B7tkj
z6iO2E+ulQmWK(^IA&Mc{O{v6d8^u*^Rg&a<)eg2EvhX4y4^WAu*iZk7w0~T-O3rM1
z_FB)tjg3|YE4LvpQt+T$30mM5RK~vLsE5L%7S-{>&n*V?qv-d!_OkyL%faQKv9444
zraJt&P#hynpgccu4@G1sfJfOEcuyqV8HD#ox_zb8Gg;ilgtM`11smugq4K=jBgs1u
z%lPATv!6V^F^CP*?+ft&^~$*pOLJBve#5I&l$0jbg7zgumWO1C>?Vl5M#Ed}6LPKG
z27SCX&Qtx{0d2m}cOYIijs{6eD!9pKm(UP(A&eaJ9{)I9%u)dHiG+Oi+H7ZKf#&5{
z^5gx1pU0-YDGusLPdDWXv&!#MF|StUMuzD@Ra^6pa=lyudbwg;Iay>6fBAA0apB3r
z5f*msis8@1Y-pXMoV(sp5Fs@>iw#FV9gN8U^#IzYoT}L6@OO5^f5v
zMC9<$AY$;hpN-z^w}JOTx{clh!pYam-@I&%IS1$!6m`tS=gI($p%9V2e0!&^na6Kb
z^l|A7hkto~civjXmdq&F+OG@OxO?K3I4$xI)oPp-D{8J1+EZuPz+gB(sd=e`E>}ku#T(yL+qs)6R1dhM4!Zqlm-|HWS9^K
zl7CUY18mN5o;Hzv)2>Dg!y#zh1SMe_{6RDHy0LK*iv6$~SZ~#_5~EsH(eaNvP?y#2
zQBpsRjLbNsn!t%dsl8~dcf!VL!E%8IF8HoqvY0zrS#LeQeuR_3
zV*I0(2Fu7oL1IFJ2_6VRJmUbEYVtQY~fra?oS^AJ8iy;94mmNeE`F=+vi<`cN
z!gt@xOYSg><_mV{&(h19GcGwPracDNirB6Qg!_f_YBX{A67UcmyI}CvVBxez(Z}&M
zZxt`-C<@t#j+*h?MW{DB4bFaQoQZl}*jPLroIiETG*fgqV|kL&AzQVtIy@06AWkF_
zVY~)&9J}}`?n71~o?^6;MpiHSw1+@h2v?pJb)Hg!yP6G6@O((FedkMztal$u0X04h
z5eFPxQ%cIC-RE0RRfs9_{bbBxWclV~5urbgAJgZciTGOG;u3(`?YcOe+mn!~U!sRk
z#Df@K??i?PL9b-UNDH$itLcP6*3VEZ0+WFcwp5N;tHwcv=XFOi3suZ_%T$&95E4gy
zQHT=L9WH#@BcIn`Ppq$XpOuDg%}To$%^vf%bH+cilL=`S1Ze;zvjIMTTV!n|M9CwZ
zoad0V?8#5G>OotdZXQ;st*ZJq-K;PWuihaVBUpRMqCt+C`}QrTMduIS2c0ZZ9}K4D
z*MyhIWMr@E{7j!o7i9S+en+ZqK(W+{v$`^_P8%)ndkM;%f_4S9Bj42y3gH{
zq{zT}nu6q!3ocxLeG%`V4&k03y6TI;2R?QKU(aH?$D}U$4nkwL(~X*VfQ`KP>e6dgr{9mQ
z6}5y_xtg+dcXKpYBRoC}CuzyM5|iTgoyDC(f^@q8h|r%^l!f#uG|@fE^RwyJ<#V!UxKC
zA^L8~zypW)`PK1of)nA@+WEFO?(2$NymtM%3GRpgNHE6$Tma{R_>gP}j2E8q$s)m^
z1V@sNp8mS+E$b}2Q+dM4hv1RAM>Nvv5I-CeOUHiKidw@LUj5MDKV#y0ylBQ#IkO&r
zYb(LP&uTpYJ`;0wj&E%%
zb}hiuYkDm8(sa9)fE9Xr^Q?8ifxf9X9UReUUwHlIBo5k9dLQ`%fsFhnY6GvJ6+Ts)
z_j{SI!gURJP=UA|h#%PEB22)6%T+r8?1MU@$C{ij1s@#T@6Yg#x6n%TCwaQCpz#fH
z99m0cYTm5s^x!xoHa3i=rw-pHze#K_a*zC-VNXVK)(-b_{knB?Lz2lj4UrndJa5y0
zt`adQbnXeOV;9Bz{P#|eY4&N@y2qM1vohTAXr03x93Mv(i07YhBtx&xAIvD=kj0`|
z@)al`Ci4~aGLuGZPT~o%fm0nuJM2Sri(T^Ygs@nM`~7=X=e$tC6H(LETIlZZ_Ndrz
zYX|l1b4c?ax6jk+oJk()eK>9Qf^7L)Xi{KMH5~XZ|BXc_S8{ei7*J6`mC?0p1L?UP
zDO0cU2qv$|?R0biU>rVW27>Q;Oj9noH`x;fytL>Q(-sN}3haj`=c$%5&gmWsc~!hP
za*}T6&T|VdI&xPo%_LhmSWgw^RP30ht=QakF|shya@x6LyL$q$eBV1?JTqB!N_8E)
z!9RU#(Bx!v{_qWl(WryTqJz3=Cpn`1AdTtxhN4>zvR~;m
zb(Awk?FX55<4~i#@Nhk^62V9lvBGlPMQVs(Ro2vS39k4byhwE#!MgBg0%j@D(M*sS
z>j`c}ymF`BMx*9+(u3N^qMjIyyqt!6jXY2FJL%9?g^$NRMX8gW`$B^
zk*)m-tA>u5InAoOd?2fW^+@g)ROC}91T+EKID!NeL>uj*bJf+|7aqd@<2P^90<6un
z40|=1lK);wv$8BDDd`*rtTgcarbf)Jr|T8{c$=m9eeTkTO2p(F_Q<3VAw5#<#7Cz=
z|3N32rHh%b$s#gtk4#=I_xTZp_am{>NHVCs9pZ{H%Udz7AgUlb;{urvUXc8g2J|EA
z$-pO1d=pa`1`ZAO|>}Rcd&C!&a!Yzsq$&#_6&ePO_Oi{pV$uPG;27HOk+$&Wk59zx>TC&ExUM~%2&n7cAxi$
z^m{Uy2dFvk=*Q3;t72>8WK;VeTg54-C%^HcdlRuw>i^v*cFVI(?TdYfvJplh6;=$m
zxf0AaHMGAc@}3wvzkyoij8V6U>J&LF
z%+chjUmfZhPpgR2mpmnX!^~KxfxhD?!yu^w%>xEsv-IndhB-;?U9Y%6huc4q$$KjiZcNp1ECl)VxS)hRyHUqyw$)
z9mbu`+PbSn(;ruyHVq@?LK?MPs9eNqEg&K@RP$Oahj^6U#iR5EQNSD^pn(;`mwzPbh;?!XfL7M-TLjj#}`F11S-L=Cv%XoB7|z
zW~?-V3sy#u{dGqAYJZTwk6owhtP@8Rsw{DKi6@JBu{xi7f-$H!o2tq*9}QNL74^r#
z$Qc#u70@uyG_t~0aZ!Mf3aBrQtT&6ep2}U!%gb8;DL6Rrh$}aRcO_V`=8{D-flUTo
z2J#zs7drwJHh#2=;~f@v7?)mb`bJ(DO+PtPdufGB7&{1@D90ps=iy#BMRf^gxQ;Vn_AIC
zR3y`mLMKe2guS`>J(d8e@OY8aZNlS4_)Qv;tXu8kbX@$7^cV<)A2}9+q?t6r+sPcs
zFJCHwrZwQ{V-1+KE}E!3f+yicK&oxTQ&Zv-5-P8rxB=XWW*wD9ZNX~9VAJ>ivG?ZT
zSg+sP@U3?9JlIjm-ld5mp=7KyNtuVtl_K*@gzVj1sFch@ndfAlOSx?nWek}TLLx(^
z@SIC~@89qqzvF%Xd7tC?<8ge4-LAMl*XO#{wbpr_>s-ywqzT52K(_!ftUzSz^DpOX
z4mf-ho~ue^;3_#Wg;vKVk%TonHO^Uwz8P2k>O9gIiF45CGgWV#s4}sH2M#dN}lM=;EMD5v}Bjnx=5
zG1%o{TZ5exk@IBo=KrwL(>ZhIF@(4w2yO56Ik3b*-|@#{fX@M}t3;EBAiB%bJ2KIx
zh(!^B9vs(61UmD(@d?Zt@{*wdC=r+D^^&-@o|W~y)3Qa2V!!X{tqu-4k+{1o!W$H>
zLgFIz|fmT|l_rS||Zk{!|DLCQc1AlE|P85oPmS%zFo4O{p-w+T6a2G8nF%zFg`?BCCYrVTjcI
zAu}1$m-Bo4y&gOWMSk1`_Fe>#<4J>Z-sHMCDok))c^9622e2*`a#TInuaZ3GUNZal
zyW&Ye-h`nE3NV!v5^E(CAJ=+DKU&UJR@2f7fKxye4!R-E6f#Kr@XpyrH~eDgHtU>Z
zPuQAKHtR!Z!gSMk#njIme(@|Vyz8G|{o7rH0uj{z{(8RR7nsog{nes0-{RljdLFsF
zW!68x<0|+2^EvlK9XR*0xmg$L
z`@3jas{&2uf8^^8PM4^mA)E9_buZck%ys5F_D<67MT(Id#Mx4Qb#U2!Q>As?;b5F7
zl|X={6z646QDNbBVc}C$OyZn$JQ4$}-sLI78~uIY>j_aYF)gIV-S3k^k?V$`ve>q1
zlN2sro6_&b;Q4R*?KL+Ocyl&so1~;J0xW<8lO^r11vbEeuMy6IT~Ch;443dEe)dw+;m@ODBMHyZ|T;;rrUz
z|EeH^#|{)N`}x?~C#z+kQ;nuheE=fDx$e-ti;sPJkosgDBqe2;IWrFr*NBI8CN=2O
z4+XxCm#vr8;b&~qmpfvFn=1weHsOD4)TQoyWaQL-cKhZ{dECW#`3Y!LiFFw4a721$
zByxw>AI(QlbA?bT5Rv?G5cTys(jL-Za&uFLK?p@n{Hp8z(GA;cnN~+m>I8oior@Lt
z;ZWVps3n^R>F_06E-guf_{-d20V+=B63_~?-|-3`BFrHbpvrWqdFLPVHtcG8Xqk?-
z!AW>SMBDfxra%>Q329RW;8iVIYuFuOLs20bbn)W?1a%otmmlAbViOvHGaEF6|qz{)5sP1u>oh0dXMOdHDF`aJ`NwtMUk4>_3gOYKWDi6hfTAUNLcDXTq+^hF
z)3!s`B~%JFpZ>bfU29ud*r^NS8)DP&$bpa}pDH`II~ozt^LTkMu6Suh8=DjYAo}|)
z@IWU3);)8Wm9xX3OjE5iJZoZJm1>ntl}yt>jiz}Qz$+z!VR~*{zixy+azZ{JEVX&L
zWN*g48!C5THw4Zs9%?RFi@t2{RllMN)(_I|V3Q8KqAuEuc`$a7LzBs~lW$9F+dDg{
z#a{)Wm1%_NrQ9Cq$?!ptW1|()ocRo3fhsgn$Zt|npIeD@3ugxf2M4d+V{(euoB}kW
z>9vVo6UXmWHG_?-Ks|>cB1zFnKo3haR<~Ze%-6A&)mB`bUguYpUIx{Uf@os~*;D=1
z$#_@yS!~EEWGkj~$J0*jaD=TW1-Z7lwS@q2-KRyZ1|Nf{SPN+{*2&y{3Mp6&rH!6d
zU>5P9Qh>FrugURxY!KH#gh-}6D~^v)T{cfF5upL4upiiH+aeD8Z9wRO@hYNM!0kM|
zMWc}?Y~Qpg^ustJV=a)EYPzht5a#`0FixGv*rE?NDI_Z2w`7vYn7MyB8{=Kw3&9TzwOwytV$*b4v^g5
zHJk{ZfaZpPU>ZaAsZRDB#~BaaHtDgY~Fg@_o^*%&~zOhs8u-tX{~lB5dJEL)}nGOmAm)q{)X21$vPmKYsjDW}n8<8J88b
zfnMTBtg0&yl6q{K2X!W@E-rcBztn}I{rd3Hqp>I@jVi)!H4dXcefYiJU1?9s5dmv#
z1HMTwTHC%c9B#lDI$rxS2XN@@=6ON`nOJgi-9D>xnUR(k07&l?}j*hlQ87`UYd6JR)FcE)1P$=l6f|QJzdfzuU}Vzs;SPjHlTC@
zy?yTp=>egzwiI4xJ(-%m6Qlo#$;Bl(&&8RK7NC?t8ldR#&2c*4*T9@$=&wHg)H{
z4N`vLF%g)g+6B7+mD}&(k&)~ExC`eA5!RM099&ebiNwcK`lOl8
zv3hkFoCIZ&CDFe?E)fnMypYK>>aL2$N%lds($YvoQV*nNPmTMXJ4z-daYcjbwdErv
zr;6GaE?J_|#&kDzc@Pk7uibjVF=5KX;nFnigo{VKSSXc7UWXWsa`z-wH?3<*ufOF5
zl7kbO{G3RXFNm3rq1f%l(GltZAfSMaql7liREUM!q8kweIfO0td#+|R7KfV2}azRz-HvMAf3j^!3gNLj7x
z?5*{L;5!_-8U(#6NTlE>QMa=*KDqs4!sKkDP2HQj!|S<`tm2P9RCcVa^)(C8wxyf|
z%}l`V_B6%VtDx|BXKGjHOgB@8W!t%=Q-ZL1K5t+Ui?@TbV#{rDA7np}`B+?J*atb9
z>887VX`kADd9Q%jQw{fTWktm&P~zkc*J$3pbxQ%Vk`i0X&d-jh*(hU6nxX<22gE@z
z*mlw>@i}3A>xK!SlZ@xqF^eru28pRdH6sGICZx`-
zM)uYHZ>5%P|LwQ^5Mn$&8G3-bw}a$+U~OTpBak%@0)Pr!MmwWZ{5=7;>v~eQ76WZu
zK?(xn5QtP-4Y&@}q6Z+x>!|sqp?62!rzJtxt=)CnRWNZr1~
zUUKs?u7llj^QuPvF#NSq6KGilIspyEc67)>xlWns5G<+IRzlB`iZc
z8N$(h<@)xQg`ff6&2z!htS@;guBK&q4T&B$$iYpY!wJIK5U#
z1kwt&*21$%Pg)UK+#A=<8vj4=QP2vjhPC>TJljtA(v4F1qNmRWg^MG?WLG5l#ZT%+2w5W!jf|Q%j1AEhjW3s#`ld3Az8vWKR!p
ze)K9fd`RmBUc@$?o@ZK#sNHz6%^mgnRdAW+ua_-eyd&KsksVtDcF0dq7e#>&0Dx8l
zX-%U>#0;;2;*EQ+IPee8$@Jf~)yvgA!isEW_LADW6JRM#yfzjj#Y5VWoADyC=c=s15clRcb{uLpQxxNj#;W`
zdHtIcVD_L)S5a0zXXEUEd~ad8x}6R_gWI|?^38#VFO@!ROg(-52)YH)2KPE%TYMj_
zV7Rvq_g+)n&FHuHej}(>rfE=8sEu^*NR^CNsLr^QmDMu>M!I&>w*s4}bvzQu3iF}>
zdSToedDn@XqQE#VD_es}4JNZ>zfyM*@-{rrwd>cp(a^GL%ZYnjZ-T@d^e@M2vj+)V
z7Q{RBN07HJl4Hv1(4F-dO9N=l=-bD_4btYg5A4eGpB-3XpU}DEXhoYym@+QNunOL3
zuDc`l^;Ok;Y>(K*yUQD5Dp|J!{Hvi469%yZt
zxJ=ZuIf=Y~*Y0h*eEgZ-4z0K^Vm$+jUBZ@x_6@X`4ud?_#@!BZDt@Tt$}*!@H&=`!
zJP5)JD74`>wV4fvZ&s~Q<~3O3`}5XZF|61g2xFgzjuQkAbnb
z2avRCpbKW7Oz~oPBX@e`?bG9s+b%BNXf|>$0j^cBYK*MPu+$?5W;9i*0@T@K@}A1V
z`cu3(L4d^-yL{yGi~#;1KG(oNsT$hY&0Dt`LkEDQz{>?*66p~@g}LJ-<6f{lDE^c-
z9MFEjQR~JrSWzH12RMGjwX~r(xvb>M-Oex--tuo4yf<;9;h8~ez-X#
zOF3d5ltY7IRT<~*SF*OYX6tB+KPp(>;yt4dzzjBXalLn|wM=?82t}WcnahzQ*O60H
z#i#(e^Qfw5b%@O%l>m}Fas~E3PoR*0d(WS+
zel*0cA)Ep?mD}FoxQwRMyFK?L{}jhPbL4H^e|~J6#k@W4_imeIXxmn+FqQJouX}?S
z6UtQ#l!#89=57t&m;v8MkMxIWH-U)XaKNP&^g&2Z+PVw2KCkH%TOiB0fSu6O^8a#;M)8gojnnxIaURzxc2Cc=!_l4x(7
zoaL0Kb^|5TH`Vaegq;(Y%~RVx+z;B6uh-NhNyjNj#n2U+C_V6s7x)@D@D1aO(0lS_
z&nObp3xXR6$hU7#Epf5~(4qA*0%e#ZU6g$_61AJz8+`;Xybbx_0;)k(PVC#&$Y&-t
z4hprwi{qsMCr1MQ^z!$QMWXP*Cl7?5v?4Ob-n_c4(&zz;ar2+qDh&pUwWW<{b{EsZcuveW)n!~Kz
zIHc#uEcWBI97^bi5<}?m%N_It(yi0^A6$_WtdMO;7E__
zxh?3r>%RjVYmc0Tu0y*QU3Dn=m>fDbAZ0o}ys)LI=S#3?uH$8q{^
z{GyRBdxjxS)xGiGblLuBnWTN=MazyH+cYh$XkCV`RPHA+{T*qPV+WrRAZ7}G?O6${0c(-Kl
zhY&8W7CbfF3;i&*Cg
z1ZOABWEmf>gl@wYmIt3(s`Tc|&%rljpz(YQh#w6Q55#U4&
zo&cuTgjNzH49h(7wAh>I@uM-;dWxWB{7#ld>z|q>6?HQ=UARk9(fz??{eq|-HWw!k
zS0FM}!MvjF5}fhFD)v^uV%#M83leDV#TFu=pm~n;e+E!g{pXxT%4lqC9XQ{7JjLU3
zTl5e%TsYD-b;Qgn^hORtT@?k!4R5OgBALxUnXN3X+H1Bi@G{+d>QPYrL#H*M{bA?k
z&4nF4K>$zTB6i6;U31m2`bh^AtY2iC!MHW<9sN+}5i2YH*~rl&EjFi4mbeOJS#vYO
zDvpFH*9+eQxO(nf7=RcmicRaD9fpBVf<+^sjnC1MK@~_wqtFYBjm16)*>ozb$((^)
znY~|Pi1XzLZ6?uzc-PY0y-eKW+VhsKmj~g1${=F>^5v&Lu=(=qT}M3|_3+_p>;vq_
zesTOUDj0x(Yxjx?8dd$JfA1@$8W~@Z4kE7?J{cgafRL&JxQ4$Ek}Kw_zoPt#&nY4s
z_-3DeY6r266uJ;_>a`6}v}WN5x%-Qp&)&8VsVT(RhBoZ_wygzo-az7xc0@IV0MoTS
zH8$usibl@|#;;i0RoV{q=~RyyI6tcr`R*zz?AQb)PC^ym8tYF#W{bHy-g1;4Fv&%i
zTO>L+tSV>EnpJWVexp{JS)LE?-#_zX2@+@(Lkk(tZ{HtrMP|*h;mU<;jr&@m6-%TC81?`QRdBg{{VNO*SDiqM1v!pRjIujAu)V-
z?_LNHpwHm+kewx11wMVc3qOC`?%j%LI8wqlgEXMy?Sn3yE*_C-n7JpSXi*1{SdG^l
z8)#=MY~3WH$;;Z?OcaIwYc^-dgB@Y`)|hqKb*ns|0^g*%4ODdC{WQ=tceVudsbs=Y{N&zf0%I9ZsX8A0zQFkMNl4m@(G7
zwp`|<)2vyuc5pfm-9L}o_7Ut;sxG>SePc%kHuEolU{zn#8SRfQyMeuN3&V13+XfAy
z%8_`Yk)2Zl`j^(1MYWQbhuCL_sItR1&@6XY6#?s!{~DC>s0av*CQ6{PaVJZZc$7Dz
zinYoBuaMC)5wrfvqWy0$@}L6W#I{^~1fz+x3C~6XA3i#I5jLqOAu*xS3gP=nGOO3Ei$&%D#G?l&Zv{pn
z7>gKhpl8Cr4Me97epWoVNR=l#PjqIrXimNU`#X#L7oPL~aDMfF8?D7u$UFgtDv4BvVv3qD5(37`>8lU_+kq&
zbPw>%YGm%lB2XUu{bj4+#gYK}Rt>#RiB?h+eUVFQ?p7hzPsrrGynb?Bo1uyt!t7@0
zof;|q7d6nJs;I2|5G7p#&h{m-8J>i`oojtW3B$V@W$F<19SS#}P0H9Tm3ILOJp#x`
z!gM#1$hReIyY|aG+VKvFiTF>1p8h(xbZ{*|9zeHs7w3$kWpPI?N+Y4bCG^|$>ZFKhyuV+%(F6(LNvT9M^8#C_S6>ncBi4P|axk)ECx3+yk^jxjpT*ellRT2&jTU
zX%QdRPJ?pfanlgf1+G)i5nwo=jYt5(+#uwU-Ng#Vqi;F}VG#1{TIdLEU@Hu^(_6Ew
zyj%`xz>dASkPFfYx0v|4V-3;oT`1*L5F$}EKgM+fp8>i`>_%V;hGnCY-RwgYFry6d
z88mF>QL5YTGt_`3qySz9AOf061_7<{qdm+no%}K}JwXvUL8@po
zh7(QqVJsCsW&{AAM#~}&Tue2jNJ8bW=S#f^a1$u`>#uF;@xTL&K^7P_q{Y7(f>Q{P
z7EQ|2-%zU6h0=g0JrKpKG5&(xAWvqb7Abn3P-q4x-`?T%@Zo2wxX>WSt(6aA26cxw
zS3KnPAa?CFoBn+(>*;aSF$0L0zK396u}k0!p$+Iq6*xVSsmR~A^YE1F2O@fd$C3kd
zfwilL{U4zH)o{86vmOQTh&2KjrZN2qthO??25ma<5wVbW`XRHUO9X6Z#VLt>`_yqU
z1b~0D#Y>`3MVJv%o9ak8&Pf_bc-~t(g2q4epyBfZ0Da~Nz+;dKj&WKNKl-I`6~B*@
zLD|3s$zPXr408)LMexlQXj4_6~mj;hhr{dYBj*Zv;bpsz8A|u3owF33;8sq`tc{^6sygzY?bhHBe9xaTch)
z3_+ppX&hSu9v5xH_Y2Q%_tt@81ZH?=a@bR0-Y>0405N0@+W`M#-S;*fLHSt=CDdv#
zYT%?VV9dmk5m!_;3CQK+NgRf*8jW1XR5;OaO9SeB=fkh(X49A5CxJDO%R1a;kx82E
zBegdG0FgW&xvp1Wpte~9N~1l^SmIc;nGxrV4Vv6=?JSfd;bYqH`JNiD&5uE60H7z<
z7d!u15&6&Jo-Bo2?4e~`aFPx1HO$1r3F70p3eQ}?*#Q8jra5hwCth*UsALe&wm
zl09tNI4so=sP~%nfvT%Pv78766Jqhxg7)?8$1*cu%bwmCfz<#j)RR9wnW%Ii1|*I0
zxIJk;X1@9#8O_U}$_3pOH&*!|lr(<*r94=YFWP1YcPz9Lf1!#dYGZE!_XEg3@Qn3=
z2E84-fFnY%4Jt$t00@^fgg4O^{^vV^ITo0*@Ir9P`k#Pes3EpNY6`}l)RWYMp5BXg
zhM!-q>7x!-ZNJ_r{Q6$2#zC>`qOxjDdMv8SZe>RVSAkiF)*#f}nrTirqIMNDjORH4
zW^G|fSu$@QH`;GlAvE=a_bh4giG)Knix@gnc9EopIBT3M&5C~oXZ>^U&dpdqgIA`A
zzPgQ@Q8cv|QBUTX^QIoCay`Sqe5hsU)H&3&J1YQ;+;Hg&KaTq`_$6JZnXSl2jet>)
z!=)H|y2Y<-<3A6xKeZ|bAhK!n?OC#F+O}S1`IC0i9=WZnAvMPim-a;FsdqvWSOZry
zDst8Y<6h>|F9SlwAr?Vs?@+FF{vW|H0Ac6t8#T>y0S3Zdiv%03DUCacQW!LUrvBe>GR?z4
zU0Bzx>rwRk=g}Bn5uv<`&&3S2Eya{F1zi6mm_^(#0czw~MIx*~Q8nZN9ZTY3;IU>5
zjZYTzK-54%)hJ|cyE^_GmIQLue;5iH(t+Yx@eOKsPJHGz-XDWBI}3VM(+-?)P)Wt3
z8_+XSt8?ijT8a2ep8Xe`NySGWsQ-+eI@;@quKJL)?%&TX8>`khN7S}EjD%zbV}ze0
zG_)w{!wEP}PGam3wF@?8r&HEXN*a_2o?#$<2_^w<0j#^d295~fkQ0c*&F~O)A0aH^
z3fU}8rxJjgS#k~#5-_@gkd1&JRVSyoJj`whgvl(S%pk_Vd8T0CN-q?*B|jn|y#UC}
z9KitH&*<;74?k8ys&`TrlYEn^UkVCHZ76u5;Vqo5H2lftQ+arT?zfMxkSjXm$KAk(
z;qajZ2m(N2ICu{v6NV@iStIK5FJIm?Y7Y|72+nt?d+d5JP4@3EWwq4VT1~qFSxASQ
z%K^UOMumwZ>L;*YZdT|&S^xtN`bYqR)iM%(q6bh-aCyz0Y0yB(A`0KG7C$KeYuB}xOc
z?KE%?s!-|O+h8!_zB3kFH}yG@c!FqRh=CyXq~net7Gvj!F45!1j>%K=7W#})r)BG@
ziERN3O!^1l0wl(Q2SA|Y<)*LCpd}{)8cR+`fDi@BJ6
zCntuLK$t;9K9HRX1TJ7dj{%a^?=aZo&$?>WCTU{daaVJ{mcmj~`vWzZ!9C22i$o>^
zgAdlu?$c?RfKo0t^#iCg
zf;gpxl+eWLq%kN@>LUV9f!~IDJm5<{+?CsBK#hC9ORd)^s?Bf=koMr@1G(-g;>%oV~Alc9)xgoD)Epmk&*!$^_hAkswYRZ-azk!rr$j?YSgka
zL2wS&byib?e5hT_zy7_q^!${pn%ms)=l75H~JC?2KG;!-r4
z5F`f87XKs){PvpoTWAe#vqlhyL{uIO1l6PHHKT)!qEHp}kNeyU>IqfZo^-z3rU3+d
z!*=Aw87zAhw}+N4pGL|tq$g@0VvhY%=*(a7HmXaDa7k0f{6LP
zI|(jhCCs@RTzS0(7x0ddQvQ
z=&ZG9*!_3y&&Rb{LF^NQlTfX*EW;k_MMgr)f3atb!RVLEIj|uXWKJaZXdjT;h~Jt7T@i64E68SHYONG`ySDK
z)1GzOXS_7^Yy1vx{|-+X{YQrPBQh-(JPA9fP!
z_~yE=p);U-Ajn#x(@Az8r1P22A&7EWS@8ZvItVoTUG3als%1+LM)`
z%%uq^(H35g<$GRxgxiZ8F!#3$myE-|1~Uf?O(WqV3QT5J$UZ0d9sVLFE^Y~?Nz7^V
zZAePgK2cE`cFN!mJ_OQx_rAaxw^=ZKFzxCGr@b~{3uZsSAFS&~Mn=YGZP_p1k3C3g
zf34MiI2a>qS_5^mfI)mLiRQeA;9SF;5N@M+Zx@g^9)@rgg`_aN
z0yWVgzaOb2O$`Ju>Gx1p8#l=f{Noc8Vi;+(>iV20X#iRMP2t^G&FerEd$d(we?R8W
z-~KuP4-4W2qd4G~s%w1lkW$Ga@t_@bvNv}I<6&2bcw#;k4OoZhumbiKu#SuMk1M*0xozvcgBTM-G6)MWz2Bfk1B;lIZsCWv3!;ET9%-B_}%-%
z)$d<~yO_XUxwwT@RatrdWi{tMFW&Ah6>k>qK9MYVvjYmEiBB|}^K)h7vaikg5ax`|
z2KDAxTL;!x7&DQv$@_NPEybxeNVACvgvl}r=lyU%-*VFLDharS
z5|%|dEhPF~Bocn^H5qX}z4!O%mH6%}y(0k|vgggi3Ecnb*?|(So%Z(fI)nUBC{Ih{
zjn5=z+802lxqvYWr1okb+RSOV$bpUbLy%fyx{6$U!_3s47~_b5X~26YM&ai)HHzE(
zTCJeZx|w(WSH01f7RBbmjqt+j?KORS*BvQxlrlFqks$Q;OoztCMwZW0t%G>OWRXXs
z$l?4AA3t(@2Y$P@b_@eZE{N$*^|wl`7rVHeOeHQ3FtxgrHS=A>NosF&DZ4%d
zNMPdjsi&t8xDQsndf$8B?D``CYwtDMRZ_y}GLka(i*CQRLGF>!)>n_pK{Ap*Eo!n?
zNVhN!6XplkP;RbG$$311$yIv+YT;nXw>3H2fIFJFwrQnn-yeP7EhRqC5rDr1WPbMF|Je6+HNXvWP~QGLBv$G?`MEp8Ux
z+Z_3QTDKAt6AzTG%evg{zkvXQY6lF1L-P}Sf`3iroN-aD*@BpXzIsNKlSUUDf95fU
zFj55~k7xQ7Dg9zJYx{LLGd2)N>T2e;r3)ACz8K!UXabzhTT!`c!z__mAZS>X0ShB{IVayufhwePkmvS-B{9uuaV_EWwJoe@Fuv)w-JpY)
zPX&6bMWx6_*mBk@n3{wKp31V>4!c84Ch8~EaE_Mg1k8PJpqXqIjjgzfI!vQ2bi?f`
zc!2{4)c=4~3SXtSu(g0ME9mHWfS72*6~Dqv13+DRPy0T`^O*p=`=mRE7V>3!+l&@;
zNC=G1dk3={76wfk8Ro6TDP?%c-9i%G70`*2v7i~vHv$ijhaYIz`!KU2+K%Q7W2zm4
zMmGyYq_~(QKHi-Y;G3Nj1=jxzHaNo-fsH-DY@^isCkBES1_N>h;!-lRZbEm4VG=BB-T8yok?}AGuK-hBDjO!}dUUfQwzbt5m
zA9g_{FS4&AxIKhAfTW4w{>NMcV<%q&*z=LyGIHN#Yx`yktg#MHjsdZSNl~#iV_Ykx
z-8rS$v&1XP-&*N=3_K~*h4&+F$1|X;e0HB-$Y{YE5r$mPz0_9&m!fiQ@72r_0}K--
z43Oc~7Y-N5KM#~vr(EvN=0|i4QH-{6p>7?PBfzL_XxS(P4saJq7M)b|FS;HNFL-kJ4rzo9L=A3c9qWW&
z5`3JO(6q)N9L3MQuNkgr;cA8AMZaN#?~Um8b|kJfW0N=^G%~iQF=1cnqUy`&xN^_Z
z`|URd_Tew`V^2(-QLx-Vs+WtQ|22)QKUX=J7KQof~&w<=e>VYir`a>itjGbp|0E?M)$m~sh#0UI$rsP7rfmGh^kvYESQ;_k3E
z+5_n&F;3+=3IVaTElbB6#*GQ%X#6aur?lY0pEv8Nl?-mZU*V|!8H40tdcop#{2KZ+
zHf`QK2X%v^H%++3qP+`~UImMrrSR}Pq^|B8ZMaudQ+jLlvK@RgKk(=O)SR)Ix8^_W
zw2`(5u?rp|d+&e06r5iy<)!`n&)2iZutAPgGXfIh1&E&bdm9I>FxM75mSCPz
zv@#U*2QU)==kRp+Lw3GBdwvB<{R%fWxk96l{CRaXD>lMk<>hNSj4LCSU;?P+#CKC%
zWAmFQCMKk|JotU*&LzjgD|Q$<;tKi(lC=eCh!1)-)oY}e9F;QHz1f{Serh_Vh~GLH
zhNb{wXt)5BaE2e8lEBSic)g#qe`@sIvHjQermB}M5$B9cx{UaMF`Y5CdEf+4F;t5L
zkBe=>2wijodDqzBp0w$yS%Pc)PjBxvaHB_pMPpC^Xv_hq?^xU0(n6D3ZMv%#05Xw6
z7_;mw;1zOqp#+-TkXSl
z4#1V~vvKfmOXwlld<=aj4`$qs&3{I(EW)%A4DiD4Ww-*%+sMni6bVQnBGEMmyp&&V
z9r_cz!5Jjf1p~L0d>h;oUcRks`I-o
zUw@*UQQbrbv#G83F^Lk>;hokl^NyXomU@NVd*o$+F
z9_qZBF$hZC$gD*e?uhfr)AA*pI6+4gQO}i7IcS8Ij*bZHG4$E4q6sDhdt5l_k6Q;+
zF9V5rU};mf{WA0$9Yoy_KS6%ehtd+USm=;nj~sgxlG!9s>&9D)w!~{^|B7UQbxIdr
z%|XuJ)p;VKzS4{u`E9E&R7I_zp`>u_1&DhZqL&Nrp3EUq)OhQ=ZMgLq3_d=-)@-N?
zk&rVOc)+WDn^!KtkcxYl&zBaN?ph+743962Lw(*
z2=yCWE8S5G(dY{Vib|iaST`>l-%XsSIOB?b?NN~0!I$QUTPZlh7t4W^pN#)aXa!?ePh9>JJZkYFAF
z20mbkFqPzUb?h3%PMQdf&Z}R6)=+?I6_uq$+Y^mNTft5vAO9*V`*muhzyCRui!^_c
z#^&MYrAGhGTg9Q()YMd95Bg=rlo|N8{&aXNhbbDmzr1`C$=@Fkx%=7%;`=cPqGeIm
z*_P0hIfsFy>5=Tp6$-iOXq9tc!RaKX`pGmd(EmCULXC0akDfmL8>4^L
z))n`!UA=Kb50XdRbOj`-2}pPBeB_6!90!N-dBeaD2%6UhYO25+jveaAUqT#Sa#A=G
z(h`z*vDxc*D~9&LqS!t16n4q-l`H>535N0Qp#TAx?ub>tVW0?J$FY+q4{K;>#J`+_
zyXrAGwZDgZfa|!2l_aLVOMW)opmEr5+_|%e%2ZtB5JZI_KLpIRrsKget_u}m3mDJ>
z>}UJ<2D3Gg!JxE%iOkb-a@^K^>(v;~;&UJm1mG!*(Ir<La0a_s-*q5n&$0
zRm?fSb*9kfbap`Q4aO|ucDBHrMr3n%49pGrMo@i^((sR13A5bs1-$UN6&4l_0W<g4QQ+D;P(_wQ#w>p+tUkV_@UX*OF%mtdeK9K9D_JD8E7;3f3&Hh+8Nddot2w?4PN
z8b!_Zm4{z{v$3^hHT%46+qPXIf!ALdTBpdQ6}%mYZfYJFu%bwuC5!YD_xKM;Fhn||
zOEb=P_40f8?)C7!<0BlgB`xQs__E~$>Gh9{6ydC)hw9At;n>WK3_AU&=V#70rkb;N@W7oV5vxssVy
z`?jnNrQ2U3sLCM6x2H>0)$uKh1c$z#P!ZBbqIVitru!ty)Eq!~iWlL*84>x6YMV^4m>s
zFrY~@!^3kJFm^uNDRy>X83V>|x4|55gdafX5TeUjB##=+wT~`$-ysWb)V^3T*r{&E
zt0a{hg&drXt6(LdSs1pnxV60&`BzU;F#>&nVt@_Ut$77TD}4TAv&^Fw08bak(ArlffQ0i^Vc{7RyLeB&OhMyUta_!9
z(Uyg5Zhe>aoYEIdwVkp(+u(p~wzaRmX^oN>zh!-uU9l>glEkIq?rVS}TJhGgKnY0?
z;JLgMy%x_YpCFTX!$&uH9yM)f^KoeNiGIc8-|ytG;?joGw^8$M7b@qf!52
z7pW7!Zu&|4Du@T-WK9U%Sid}OBYYd4g
zeST;N5oe!(n;Jzg)LRVd+@YPNtS3qa?gtOff;bV~Rqk-cZH2C~;My1zrI?Ulv?Ce0
z&||5CuvtCBhHm~E7cMTY(p_`
z@~jYqjSQqGuwH1zWFh1~frCd6A3lQydWI`lt2Xp@^$!ld1k~LRh=vkIsM2Ski`_Bi0{i{d@E}}^5?CXz=uN{1Bc_R$-pnw+|?JJ3>
z)l3>YE#qct&r(`D0!2rhR_02)dW?8uf?T!)WeMPLBWf=?j3Vl3v?rBHBaPcO0QqlLDL5@W^EOcfa=?q5F98p!WW0&6s-&k7OL7A%v7
z**k`*EP_uuA#a$&$Rie%2lXCJs6$b35ECcJ{^gt5RXEO3dXJIJ)Fij>m2Yx#GIJVy
zvz>uJDv
zSazDd1-nK9P+_vxZ=D7ba2nOJNUgfp_F+{MiT5k@8nmAe3^$gQErr|2ABl;4;H3FM
z#-8zc9W#goyJ<(ms?rL}!u9QyJ{e~hGVPfoSzGo5p7n@Sj0Vom&3OLqox;V6%^-z)
zq#wlavn}V0+*$l^{I^F*?a4SWKwAl`<%eVggFX&--n)REz2>^DQoixLr%qaD*2kVJ
zwS0Xl;uU{Vd!b)%DL~Gje#j>nwr1=yk;J1aL6?G&RrJGhMNz1wPlQc@V0FzG3G|T
zP*v`IKZ{#Wv-T)a<3LHzI|BM~WiU>}E<2kxkF7Zm
zrvHRvuwk^fXcl(kv8`7`GOFYpcW7nEXN=BK&TAQl#Pa~gJ7So@Mkmg7>+YZ>6mvJ}
z=V<8habk8Z9LFB?Ur`l*$Y8YIFomN<)9>eFbRYUiIA`^bKo548Dgl7!zy}g-n_(Dp
z1@VFg)B)N5RVYjyLmnVlaL0{UE7FI4y5GExQ;q4)X}!PQf&&-og_<~a8Ub*feTB6a
zWAKZv)E@KdPt(FR#W5=%T(+~bqce?K={iO%zJ^|}W$0G15aqE8g2RzLOAl~yz>y3}q&g-;`#Gh?u&fLBeh_)F1iInRP1hbwUGN_e5
zk8Jhv4{y7@uf#yRR#J#L;)ft6Ya&v(z8quM-PZ40xxc+X6N&0P2F+W=-|b5~ajWvt
zx#P~uWloGiva%KDehxkZATrI)RW{HSFc>DydD@{TPKhM!^61i+`MH1i+Y9S^OVN9<
ze9fA9s828@Cy(iH8DJv(tmh(G1~o-j8vGAhQ8J@hu$49qS|Z|wY`)U30dD|xbZ=U$PSd?~PN*K(BO1tZ;@g@6!_p&d5k
z((4bHVR!A!I_!*{;~(Eb!m5Zjy?1=K&2
zDac<`s5Mc04=VW6a5G@zKl{u0_uf0;7zr(d0CM%
zsO4xzl55G*1eozWNl96a`}<0Ewy2Yn)4kx}#1UCpsnXPNU&Xs7?)<$8+lL!5a1{$9
zwaRvLEM>6iq#tiQQE7?k5E
z&M!XsV>NAgc^a-4BEK!=dSS8+;z*^wyRg1A;s}3xxI_vc_@7ieGnVH_7N5er3I}sJ
zjkcuGQH$Ak{{o>tE0uRRGw9lE?O+SB8oEQ1vv6)ij^;RZ)>1v>-?@
zuV1sfNw>8FA$uTj`V&L@P&j8@`il*!iHh;BkXU6vq{+00LHu%;
zis`#^owbkjkN=Dy5mf*TEbykDG97Gk
zmrG!OGvai%80i&foJ|-oJTbY-|1zWPk||sB$xV9#ADML)GP{cYc<-1E1=r%Wg7cx?
z{{?c4yQojk_+?f0VBKri8kJXAUb8&mFZ9u8v+v5HR}a@{n(G2Wqp1?$7iU9_d<~lw
zwr2g1Dv?1>!KH^dka5nV2E{J9ed2O5n0eXODSX`?3CLyWGIn}G
z`OM;;7HVId*mPj>Y(H!{=ig0%v+rVtU{jL3cC}&m*dHZlI3V${fQFT3*M4rw6-5`V
zJF%A_%jvCNJz9M-JddZb%6O5@VDY!|n_h!y^_GU4gy1dtbwv1
z#9*9&%2Ujl*SOdC6~7E})>n_buUF(z)(7kdxh@CQ9*q4+ds@1?7kV8JxBh&1_^lrW
zREVx;g$(R1MYU?&*o5kfGFYlwumnJdP`DC;iJ@=TFHcW2FL`2Jid+bZ1x0+*KG3JvS0JxB(|R6p6tHn{hv_qEDR}7*Wr%fAHufef=$T0d;jn+h@j9+GwAXla)P!S$9xwdH~8>yg`f&TI(0M
zNohX3+XnG0=E!!ogU+6wh^6)$PzpUrQ}7x$+X}Q5+1$
zirzccug8>eT&{n*;_dAn)o#-soOPuXycNFp0VwKs^+Mjv_$|Pvrs{=?u5ajuIuUJ(
z&EjZ5`TW=y)9mmcxKbXz{-<*AFKXh_39`;bkrF6rYqkZS9VlP!)^{Bt`lN}-PVKv|
zm@~-9!4tV?9^^qz|Lucy`5EZcfut9~+Vhua<=coAzoVy4&t<0aXk0tqwMU#zkmj+&MO>un&Avu%#*tl8s%Tvi!z#^^vv+WB4N3ria#Qb8h!K
zj$yy+iPxcFgczeopeSy__i#yYKYwh8^^XLB>-!qkh>D70bn6;?glj0oP&lJ>a>G^+
z2&8GOciFErzX-?R2`TtoFqHFLl}}DyIXKy-wr04kU!}9Q9a;IQbTbZQQc1W-fCi<&
zs4ycrT4Y$eLq%6P_U`z+%sfsTIf=0_i*)sqTSKn{=vd2m-qfV?`SkMAdu?aCCX;PZ
zX1)S4!C-*pTV9gp_x?xhPELH!Yy6Y39anJU1!K1LLrWiLyK9Z
zX4ciMW%%QVa!mWSU3opo%^xKk
zJ=>PDuVJ$uyAtQ?Vjo9;l=}kDT58%*y``GP
zBJ&{Pr|`}imLJXr4z}G#>y~6%k0JSCaO=L;nzX!xq;SWnr~8H9Ew`>!w?%VE!QCvJ
zxd*lLs+_dk>Lg=Q@QlRIGj^eC3Q+6(%2yr1BAjqYOyS)@h%zd-f@u|955z|Z;IA%X
zX9&7{(goFml=xIao{a^&l6lXQ4RtG>dlSMN1X}vMrR=+_q}jedONhE5Y4{-{Ni`Fb
za5%^rCA3z3%|9LTeLaTxBC;^vUc3>Ed9|=vK)*4{0GNBIb{ZyK@8+N~pN`aRcubu|zoCV_
z{oYc$?vM7`Cr=pae);l+hL5A{(1Y^e%X?&D8~=z}%8O$AM35LeN@Xbrhkdv!uZd>5
zmhfjODJmk%qC(>!ueb4z#5|>w;fjViA9T3AKEmTg3Nib%xp`u0BgFlokYzJMM&`MV
z|H8L%<1A{&>nN+QdyJ!7Fb&ztRd7|9I)-ht#Y6xzF?rMyu4?hF`KqnVkBBuP%NB4TjgSfmiVKl)a2J}5v&)Ox#!
zZtXiLJq0O>f;MZ#)StQiBZ0SF4ad}mWs73YKY@Fp-m#GGKKA5a^}2Wd#>;7^gD8j}mI`}{y!mAxA#0Jw`*0kpdg?KKpEG;z;(AA;cXw9>6@
z*%$9BIrU+@99$&spqoe!$HcKyU~}N3I6A`b?TXjCnR>_U7My;qU_SDtZuYfgm78w9;IsF8zP)Nvm1>g1J`
zui`xV1SAlRiYI-VT$#6?>LG|m*8sKOhIx=UGS>_t)i}5oJ7M9iLr+F3mw)d5Oh=JM
z>gK>kW{bUzFW{NoY0gQ>$uA)`x;s5Jl24=w8Uc$vcFq(qZ%wJ^xd17+1$a~fwz#+s
zU3}IK@l=fnI~Z9E=3MZIPuJt`fxS=(90lV9SrzS2L|Q7kGisph}VH-<&1UsGq~jh
zv7T2TAEv8;=BR_JfWUh;19j9@lt?(~R{-6#dBknNE%XW{Bn@XLf{dp4PW>2^13^GT
zTR-FVLDPBd!B!=YJHJnKA{EC7QZ}1E61$hLWmtd%nk&%ZpaP<)0Kt?rMl=BviTIi(
z?af=Dy=z+*AmO|g@$MoTKCV5uiYi~sacKVZ$mQu9PzI8usgf=4)x#g9B_*)EmP8qP
zEvc*;LJjW4LJHn;7U+ICi(^+X4bt^FAZ42};}f}aUe#{WJZq9|?EW2X^ox;|5a32j
z0L1J!Nl8xVE)PQ&10<6OOH9W=UYoaSpZzFgHc}(
zs0BbCM5z)`8BqccN20kXe4*r(LJ!7Mt!llYW;oR|O$jHU{=Qd|3L({B4d4lcu^+%bJxqy^F~=kkniLqi}KWR
zDBWM++w@c6uGN9Oiw(BW7SB|NR}4YZ0_1!C-B+>jD4=2NpX-vhV*3lZg^zb+D-aU$b7#jL?0?JJZ+CJ`MYH_DnLI%z~M(+ZFJ}AJbETyX?-+Dw$afA%qoR0{9W-+pDw3IMC{>lGAOZ(Hk~HVPm3&nLBl8fUq`^1y49O&
zqQfAV0tJONn5*nUgF|b33$ufhLDebdrl1nZmcLsTEO{3p4icFsD5)6SMQPs9GTn+q
zW@E2~Hf7_J*>Kq!AL+Y`t-t7Hd(|8O`Do8-o%Z{BTqvz+*2jStk8T>;$G;1Lnf+P7
zU8LulyLA2hWy_YemYoO3gi5x%_kCQ+z3=A<2*uD}XlIdbLcXiU3reN4Jh;~3My}Au
zg{MW^AAAe^#<5Ex57N6u6L#o2TFWQl?rGNg@m(=oe;E^@D4h(YBP
zG)c{aqqT)3V#j1*48HW13m&UgJS`7k#T7xp{8=1(Ea1#^Or8Y4h067V2iPV^FZQz0%}3-0woBF8A?268NUqZv1&Ob_-aiWY=6MXFLZ7`@h)x@^CKqcHc*e
z5{)Vuqrt3*Oi`gy$qBzhi%?171XBBUgwrM;!a6X{CeX9^%%p#vPWT*qBONSM*PE|QYs1Rf{`
zDFn(GVjxZ)Huc5J@E;ylCFbjJ;O53y_wlg=NV+xC_m|Gk?Unx=Xe41w)}?+3>*IhI
zB;CH9^^;~ZPy7e6fA^=6Ak(q-oBE8dg=@~c^EC3gx$bdkBk&FN&u{<3J6zmnU3cW&
z-^e(wMw_i9-Nw;`srR0s=z;u}bBHeyk>jjic_%+reL}R3WuoB1mg|`q2{ryx7=~7W
zRU!5SgEP=LutnZxoVo&a>(u=&6I;d6{G8OL2Ql30({8nnIvf@BN!nQq6wmBXk8
z_Pzi!+JR0zvTI?3_SvfIZnZMDppvR;6ySRj-JP_*Ab19BZQzMB8_+`@``Lf?2X)Ex
z#hb+o_4Dsv9ek>`qNH5@pI%*P~ANzX)hGgHN$RTN5RcT&D*Bc_cJcA|365~9^Pw&G6}WpdYoGTirG=2?ybXB
z78DY4aNtNyjQSVWg_nkQtdZ|?fqn-znLD6P5p^YiUgPr3oBmL;G~*(YJLrTn5%cK@
zY(-xwe-I$06B`s_8o1LY#9tVujQkslj175%UJGQY!s*>$cDQsE>0`ZLHu0OUk2jKN
zItEEQgzb=-LCSOh`}sU%W?M96gM#FF-HAc{`j|$Kjc@c3xQ4q?nV(gku~A#;M3JPMT+El#L&~7*p%%zC*!z
zwCnR~sH9>ZJ|DKeNdGSt6<%abASFWruOn9WBZ5vO+T}K?-&Z#Yl&*<7Z(?y+R>*&E
zz~~4Zj^02y7lSH<3F{dYr4@i1h`K-_VuHC|uQRwALLor|g0$w#Q3Fszz6Gd1Fa%Zw
z*NZnU$VjCBUhwr@hsyIl0BC=tg^^G
zk%a4D{%Kl!)ZM$23Dkzbh=qawz8IKODhA4|Mj$x~g<59`TM(gWfTU3QLt+*HPn`gi
z8}W`e=o-K(gjn?l3gjJ)b^8Bk0qB!4#{u$zYTAo?lj>Xanz+jthMEIZCFU*8w{G6N
z57?Cma4_wbGuS7V)vV5dR}=C%BX-d1vw3)?GGr1AHq_KI$`24!<7K`h*KpdcT{dve
zlFF80&>(0agQ=(wAsX=gbHkGJ#P(k9d93wG<@}Zg3mo}ektq;)VoMkBa~#>8a48{-
z&0s_hw-Nn(^tF%dTQ3@vlGr||t7$G=R{a-@B)k(7VX2Zp!jqVb5K3Ga>)?SBR#tu=
zA-n@+Ef6=7Pf!hsA`~IUYfViTLLhZ)Ag#339`9~3Xm}_gAVwwqo(pq^+iw7FKLYAiRqfFJ{d=*0=$pIMcVLw}2`EJf=Gux+IY32XnUE
zk-&mLUC%Js)i^>~7%(VQ!QQyR_pKq<56Z>Gs;c#O9YPat4*e&gSMgikA%@37s4Qq^
z(dX>2cwm41=1Cpx4ku0|Ixdtgt^^^E;o*#z>UcmCwk)m!TrPLeEI|tc+6M4ikpA+r
z3YqsDnx(PM?xm%rIKz!PRp9A~+!p^GKxX^RSecfK(V4|c2O5+P$XGUqguUyr)E=?a
zrNaSrEfQQvo0i;8f`Y@tsGVa^2p~Ih9r}KLetsyS1Axj*uLMuvPaJNI%5M1Lud^^9
z71rXWkRQm!fpmFuG!#F$C(aji_Cd#lkSv0a(4FuE3lgaCU@IN;2^*;sLn`GW&=(-d
zB*1UU!f@+=GA%?DYZVKnX5>!yf2Co1|Mt)3=5uI+n=p)tTShDqtF8&HI@;8{P4$41
z*sn>Mo0w)HJ{^&}0z;jqm1Bkf5IC*ypTZ?D~-eRtq%#bVD&@|Tfdi>3?3
zc7M!XfEf;Kdz`cPN&1V2gVuSPcby_5UDDvU%0g+ByYm
zoq2S!E9`YYR>@QWGeAN$Ap_|5{Wk9cnS(3d6Sxhv0VF*F;D-vD5~Z0ZAS5`yDP7qd
zZIW|Z7D*0Xq`kuFdgtwWx;!(y1!$id{WroR8?N>_vu1f9w`#4-bvo}b21jGV1~#mM
z+5G&JjT5F20L&L`*ita{24k>rP{hj3J;IR1F?mbeliM0CaEx$;prjyvL0
zTbmCS4h$p()z!vUq4;!9YioD3WFwwzFR4d6p(%a>I7A`O@8ZR$U_#-%0fh!#68%*y
zf=>a84dfhw9wS2W6@->#bVM^j!g1&|3u|jhqHj6V)1v6wLt?k!5P=?r&XjH2=HLVX
zUw#ob;3j0rt_6=lAVWZx$zma$d(cfTgG>Q<4UUZ~D_ic(8%L(5q)
zd=v|CXLl>2>59=cqcg#wJ{(m#09>AP)^3ZW>b$;=>McJ<(DI)%lBjmpK%&qB
zELLbLaW#_^v&I>sCNEv+#==6SX{=>o{#gHV68T^rNfEK6xE-SaJOh7uf_jI`g63ur
zQ$vDjm=OctztyI-(9cJ=U57Ag5Gm6Klkt2ANP;niVbCTN;;#x{dAWkL3n^w1J(k-n
zEk$sCV=Q|BLUs;vj>!Mrm(-jR;X$(Me^9dQLuk&c$~@OI4-e}l{kzb|sj2Awd()P2
z+~iYxoz9$zf=mo|TouId{i=Rs6a?tO%3!F$C2(xhE9vCJQrv_2CsgM(WcwlbxdMf+
z3+!;D%ENtAjs^~&DMWglX(T>_*LOulhbDwlpz4F4&{uz
zcssP`((oP~^$V{~O>ptn(=02+znliLjHY@L1KieI)Er=Y+HH9dCKW#$v9^$*cou*t
z{D-7R2yjAX$63s#dznyN`QJ-$RFM|(JvX|Aqs3(Du1i)$a!W7yN-wWDyqiRbocv0&
zX3UuJ6A}r>8Tlp#24_*gP!V(bOQ`jIv_#2H1lKnmLv;@>H*EjUdVG#`GS{up2~o{J
zST$}sK(u=x%43@;I|3yHNwOggUo~QQoL;<9eyXS6g*d?;$bA_QzxrPt0HZz|EnS|!
z2;3hz-b~;PPzLM17N-rOXpiJ-o(e7fH=T*C`DYf1@k-`UUqHv!Mn_3P3}Lyqjcrv!Z>nv0zh#8$lew|6)N`s4*$ewVs1<$&U$
zT@hfY+(i|fTMy8xaHEjxWYAZzD(}UCYI!pKHDRlB4AoD4```=96uJM%;V?A$BPwNv
zAz8QBTr|!8R$1@2@dHP&)6sECJvm|zDLWLc1$2hxnpUHvr8mZ-V8adT*j?GxKd7>T
ztcVT?+;Wo{G!bE4qnA*KzbGx8FmnqF=^=eeVe?salqD!4eQ>!Un!Bu~K#$&7MnTGe
zDXMH<5~k#o-jV3Pzth-9uxDUk{`p8#ga{fe$X0^~V&Ov57Ez$Rw#xkjc!7&oD;NmT
zd^RBO^AOywWnyA->qseja{w3kZw?8%0MZ~gX4iLOu?R4|4rmo&Rr>g83XWidHb
zJ_Ab?6*G%oOEW*=#=j5oAQrnNRtj0Be^hJ~YC_6KAqoeQZUziqY?+-@38S7#r
zw0H>6JVz7*v*8jy;e`bSkzYpay8irg1?+;bhtXmHGPG(7smLz8IQe~=jL02lF-X9%
zajKNJs^weG8uj$rAAr_O?xu*GUK2d!CqM0D1bez5
zL2=GBcHD`c*1N^_rOS$Fb))?U)kFU1w!Gx`c;H@&@7CDI9YK+>_;4}WdV0lxQnp&`
z))dKTI9i{DSA>9O3>Rf_(O
zz$U`tu`adH6&-JXyA)bdc4&$=;67!c>@&7C=EhgRTPzY5W|H4cQ9gbCd^eMSQm|gNw~tQ?o&%W3TQ3h$MJZBW2&={&=Qy}tpoLb6MFJc5+SZQ`
zJ0cyc^xl6xvtYwkj3==X(1;s$hQtgdiD&+%)u)93y9|?9dKJ)uO8_`~6xI^EBhL
z5-Jn&E;Rfxv}dzxTuHghilvRu`AJ}7KKF+1iR#6g@s*~PQ@V?T-#On13%iIqmt-0+
zp31=iz(m-aJVeqCNRq>tVP>`PFJ7Hi?xw^1b9Y%BPIiUVR}!vZ4%BcsqytuO7xDv+
z_%LVrVI0w9Rsv2eiy>l>g=YX2VHaNc6JQYBJ^qlBAC&Hfx)$noRt9n=#~E!~X!~;s
z(M^3=2kKeopoNsdZikm|-qdE_%*@Pu-Hm9R0Ku?Lk@{GnYnNRK-GAYJUEJ~q@r|z{
z{Dv>tyF#lS1QY}HBK6xqOg3Py;R;NTMP~UFaM~$9)YQ)FH?~R#X|LG(bn=y2>#4Fs
z)>F+}rB<$-3)LCK%vITD)rZiN?W~jz!hUC$Eeq-{;$HxW!Gxq@&ik}g@c7g6R&GQw
z{H&@Tr42na1jF)^RuWStbC4&D)`M&Tc-8V>GAFcSuXyds{s4QHO7O1bk&e-18X?Vi
z0CEn?8N(WG@fE$z=)d1?vi8-2T7NiJ*cdb-KBhH0t^j0bp=;hz
zR|sLr$IWc&^GOm0THr!Jz;ncqz%){hQ3}vHX}A6Ah`ru-Bs&3SG!_QUb_G=WJKOHv
z>kD4lZ{A5Unb}FsKVuVm;XE1I(>}cCwTt5CM$9f%NE=7y)>CP`BD4?4h~b
zHQD}xoBr{cK#d;Z*u~w5G(eJ-0Um~JdZgm{A9gVKTk$6f>G)b%miWCJh|UXox$iVm
zlvuvHDc(q}DpW&K#wV0_J1fCdd7T;}wR4*axy2YdeqOck{
z{&JUlc{f%e1JBAtDZ>8rO*DK{7wDGBCUJfroN@^MSt!#$0q|#l4vKG+Med1wk>ug1jvc3i{$0G
z(OoPU+mU}`YTS+gP4qkl4zPm@!KK~;7HO68R&?r6HE-+h%EZyneALTVR#KYU?UC56
z=|2(L^3f8$-yp90aG2uaA+a{l{l(zxpm}`S2h+Jmyb{$Gssg5^;C6bW=-(P^U0zPfqo>)UFH={JGkK^=J>oyxT0
zy9xhL^&S8#IzMC#Ki!Kwv=uimg+WjNZEV~RbGSdy4BmDha{@G`YwHSLfr1WzSHBB8
z`Ch_C(i&3kwK%}B($o>YjH@u9wffYZL?a1UBDVc(KqVD0yl7b?hzF~Idom7xzMGA6
z_c^vA9a*`(p~U4x{l(r3QYZ9u@HhT`+5!hY1)C*03+iP;JyW~38K3_G)}RY5E)=7m
z+S|ns+oE4_&O8Lt@#|Qq?V0nI{44!sK_k3VV!I%@+_HYb3E3~FRa8|wf2m7IY&IDn
zll{oiRaRzsE~X}Lpfr!FZtrYu^+vTljX93XKe-hlG<&RfIEgP76)pbUM}U1$SMx+?
z6A8=bUfg2jSiuH|rDW2A>*H@4ZWXULmKkFmFMD!-#)Fp<{*n9HK$*h
zZ8&f~bR}O@_+V#i=h73`+LYUr-`L-7yV<71!Vp#Hd}|L#qMz7G2!{#;O-;O)n6&hI
zs6Ux1Hzm$
zG1~6qKbqHn$Z?_}J(Mm1Z9M{z@ZBBL4L>ta{vq{oPlUVlOuNLdd=4v*nrpNO>ntn;
zG2SrFY#ip15YdCAxB3-VUlf<+22
zc$MG-&_-mXMk|j9g(o0EjWQIhRZK+IOx=1)xNZIQ7Fa_x@%uUQKCu-WS38RIrV`yI
z!h=v1#w8>qWZoA3A#bgYeZYegh_tT$JhB?^A4?J0O`LefJt*Ru4`Gq=Di*^LIUvcY@zdGZ5
zm6b-yg2-0C@|H6<)!R-$az{oYbln6$kV_Q7ugZG{fqqdGLx0AuN5_RhT*v>a2Qjy{
zi_aVA%QjOQh=*tw2~5;X#l$UW4oSd8-etG3A!|?`!gzNCy}`G5eDIq&jK3KcA&!lC
zhAxWqA}`$5G
zqvc5a!DY)Zq~`^20@qNS>(jifEMAh|-ieBujdAmD#qVBd0T@O0Hn3vx0riwGfyi(&
zOJey_%V+-{YBU^)sm}HMYR=q3F5g+GutY`&j#Z1tNx+@ZLgv=#ikz5he~Uc3)b2f4
znemN#J9@Nrh~%{A5Sy~)uIJJo&%E}dFPsj2{RuTh!igNmm+p?Bza;~^@SE>c4d?QW
zRqmy^2R{nfh0x?~4i!QjGV$B*-!GoeGMAs96N;PF2ZUSyST=mu^RoS>b+vsfH{?yB
z$o3gn9Ot!{kfesU?8NyR$zikbqN#dk`$O!$-f#=7Tq@vIWySH&qDb5l-Ybq_n%L6L
zwN13HKs6BdVv{)?L2hn_i88)g+qzFZPxE95y(HNQJ{TLKXZ-LJYWzmfnch9gZ22)T
zpNYyb*g&OC&6deeluq={njKUy+~4Vg4MK)hz#~4u+YcW;d>ON~PHfD9-Udo;Eaq6Di%=E^*Q(K5D6j+G^i)~Z_xa~V7xe>v;N&X`p=|e0=Wxfn#kR#T!kf9
zWm8s0oK%S-)f2`FyeKKrzQvciQQdT?CFodK{R5G7JJ|GQfbb
zpfBJN+6a6&D0Vw*g|#|T@U+kt1z}E|U6!Jw*R?H05yu`+{u=I9^deJZYVwXgHFeFx
z$_1UWU0yNavotznr?}pueH!VY77qF4!G;`Vcq6e_;3~TStb79S_tX{?6a?Po35XrT
z!Oo%c5fK%QEhtc(=v_QmftLns{&?00d4N1e@`3VW$dCX)Ptb=K&^7TKY24?k<9N%|
zFIIR>h_eT&%IWLH+h<(&(BcMs1lUgHHmu^+yAD8A0Aw6c!2-Yp$Z)wg#Zr5pdf7~R
z$MmJA?gnh;>$FeskD?8*0+Xb4>Ki|{8l3^aQ7Lr=A~76!8c-B(irFP%qZ1w;(zd(!w?EQlo(Z$Jg_>vQ^+`3#-?`!9WcS$qtPs)7hU8{AiM
z@$u^p9g1(Lhk!TgFO#)TRMcXCrXa5sa9dPPPF-rd|EuEx>Dym^K_)R1<}X^qL^C@$
z{PUamU!o`XudjW^&#qmXV0VeLqOTxzr6&`h^gn-SyZ3+mEC2rw(6iZR{QsZB|Fh@t
ze0$|2oQ|M8+tVMGlxzmu$ipvwB2@m+91IkQcFsT6kh2gM`h&Ip|15p_XPhuE`y(pE
zp)~uN{t*4*UeY9PEV=pF{&CMZVg<3=a4609|7$b+{lABz>^u;lLh;!VWg0(t242x;ma>%k
z2$657y!}6WZQ`GSb9bps`nUfwT>g{ox-3kuA~?@sAi|0LG+1(nK5EEgpwuVfUNG9R
zBT>hl2h=V|^+-^h&`jajU%B~ok)`%&d@vMbo@k*c^a5U4{PC!S1P+8=_@FC7)(u&B
z0Xij*JV;GFo0ynb;|Tq@FEnW+V#~_f*=;oul{j(wf{v8GGUGVe(~(**b=ow8p>jmJc2t>K1L#e9
z>6@F%p16wRQyBPo*t!n8MST8HEYpxG3T`3t!>&oGMF6ZX#Z`b=okJ4w
z==yaZe2&7(%DJ;7jFbM+0+65-SUW_+`_jIbIk|xQp}zz952#*BQlTomxR@OvE?n{P
z_8%dUp`Zc8%<-r>r`52eK<&xOxc&OEF};+Q!7q2X{88EK3?Jb8ZLI`}hwSqxEh=Jz
zQ0yGu7+l$2NF|MW@L&!@Tmu<>AfFP3s@x9#C-@OI?>~dXp4cMP5fp!i8kvPbKS2>(
zAfZ>lMBZAjeg3@l>lWf5kw1t|6ZJUZuEFh-(f`v;kG8Y*7(Db#sgxn85(V>-UxkH*
z7090eYDdtiO-t^vMs@OVqXMCbH1^Ej_SXvu->s>Y5PM+uGe7uc?aiCF%@`9Si#dQn
zR)+L)4FjGgOg7
zrN=N}%(rDTw2=*1FNvx~u5cblx;X#ZuE8kz_-vij
zJd{{0G~3~%>T^Cl6ai>wusMh4@LIgNttLFjS0Ahr3dhcQUr@jWH#R!|l1@Q#$7~!(
zBxoxwt+dVmeTml;;lrVYU?LocbF)ghJ0&SG3dM;{rY%Al7|=WHCGjj=Tzb1T3Jr4{
zDcNydlSD%^X$r9m7)B&$DuC%p42@1Mcl-UdlR85^v~uE5D)!l6l*e_?(7t}YeGd{`JX
zM4!NTs&oxPFy#YX7t+V6x4hyL>s$UEY5!9gEX)3wO?`-7fcG<)bypZ$BAEhFv@=gY
zZTR8P0rMYsbLJ83s!!eB_bRh~`)pbPL3nHg?1qL;`%CBqmr;ULAY{t)j<1)d+P9Xl
z4O&}Sd7>?~LW>0j%IpJgZUS$7%!X#^$`Dq+xl
z$|~S-5vtEd_JY5UL+m>uU+{Dn$e6`tx&qJs4%eJ8C~la&Npc6t8^hpX%i`#^s`@S%
z%M-EfZ-nF6af++?U!W)q2sdxsFQ7#NkS7@M11#%QoUj#$KVzQY`r})@jNt*kYro>?
zi_5|k{{YOup>9I<_4ihaOI`7jS-=2lqCS(2tpDIKtT33UWdA?yWv9G(eJ
zsF@Z?N=}DA+ol#bg3vz~Axn7{0dXJQHz
z@>uU*(_f*{rEr#q-qurvn_B;}zP2i2b>@$D9b-Homa+LT08E5>I2vJ*%a+M}!juM*
zs#(tx%f?TKwq
z9F|oXEc@gleNybYySp118w-nzPXjN-cr36u
z&4TlW8X&B6XpQ&S+WMjT3G0_u`0YF2DLTh$b^u``tKQYule`gFH@xE8O>ZLz3-Xo(
zPBUQe3y~sxr>C2&PtX#+L;lPA=B>Z&;i`}e%nNCOX;V;DL4i?KRTV)(!wT7j_&3aM
z;6;;1Vj3LgxH-l%WquKLiBnm
zNo1iGYQe3NfBSsZ%wf5{pM72TR;H~9?H3B_pUZN&
z^T8cJ$;jsMLr7J6DP*w$7znVdH4}18Qq|Z@Cs(s2@Pd;3a6qO~5cM*bk$wZ{|Db;<
zygA-Tl_ISwoQjMGBJk(}GS3Hq1Q-1?ux1l2!((+ke#&q_P_Y*Ft%XzTJQNSr^(l~8
zn;_K}HzRUGVtwPtDZ4O!>0wr3io^FajL1ij&yQ%=^Yr|xNp?K*)tKro}||$FbGT9
z3b?6~Q_qrw#6&2wh~Y$VSmNPIj25DIhI<1X+BVdbFzy`Uw6gIBlpP&2tF
z>$huXVQP+)MGg-tQ+k{zlxPeBdZ$PnZD{e<(aMh1rwQqD~hhG
z^!ilVUC%DNvAK2&adV2Zzk07emsc;c-!la-PII`Evwh;ln&bDJRP>ti%Fxzcj
z?GSBy8l7MPdALV!hlO#Zn)u_)+>2kz*=PK&35L0pG@^gRYkQaHnyuTmzR^L56zpAH
zlto6vXwB-~(NO002ZXvo?7D5N5nU^!u)q_`vjXypMbmET&k7T|@U@*nTY*}LjSu`4
zL<&WWCq6huA4({0K3~(Vk-GX!mi)l*$8BBfT;G(^K6($Oa{4uKco6hw9LLbfil4KW
zMpb0G9yzE+8B;CD4KNz}h+?05PrPsHEJ1N64Ft&5SZP(6@YKDD=pOYvC)^sLTyL$>
zBA(_}9voS0t?hnGc=Tn`_=TkFk0R&M@k2I5W(7=n0eX%SydTzfCEmlQU_9WW3*OI=
z)QFrt44Asz$94k_C7D3o@xD-*sSGN{?+_TZAir)o6M>?rM3jXyKvZ8fzZ<)(|Kb(l
zG6BK(*kdXbM7Fl;idculPD60o1Uw_-_UehPwXS!x=eh>!X)CYO+IIoGo+ZrOfO5}i
zFVO(!a<$aoAb)QA)4GmRCsM1NQs)R=Xl3%VkIoahhxq1HSW*c=3l{wm)~_g+4Aj
z$a3xtW=Hm`2IVD1x@As25x&57Z&Dv=0#Wr-Q;JfHSK-n=v*C<;Ta5!L&xP`WJc>*Y
z%$i-j|Dqe)*1?%Yd!)NoI^OmgE-)H>sa5s!kiaM61xOiF<@Q-`2gL4-zG`FgGC&QJ
z813k76dN%#0$78EK~5;rthc}4V4zJCT=^GxX9(97rZ&^g}xb?BJUynb{{^
z!)_DqfD+;?=zsJ*qxlyuKA%XuS9iI9>$M%94+;*JxKXi)=4mRL&G8p1VTfg_)RnC}
zX2_lJ+1N2{lMmo7lvHdKLk4LbVvdh=9Hdz~KpWZh>nj{+W)84@kB*%3z{_1ZOFem(E1DkJnM
zYnRN~3)O-OLbYE`!#QFzj-K++2i=3GvFu_BQImd$T&@L06{Yi1ECE&y?wG=TcK*1D
zuuWlDC(|veoJBBUtu5q|a=nE8{6i@cgvxD;zc4s?Y|Cy8DZz
zZ?C1ae~=rtvaxz~9?nU$AE8ot`&MUR$(siPWfjlg5%o>nl@2u?o~ZC9yRdIUf}(f%
z)3ip)-nBYv_G?;MA9R=NTqDxCKk-Op^RhG`uz;LYy6%4ybJwf~dR7QC3Xmt`Rj*kV
zDJw5TU$z0BBXGOKAnk9Wagw<)Ds~PI&(NPzPe^hnkl1X`4R7I`<%f)X1MXf*1@iaz
zXKSfOebM?WBM=}rFU>%)+a=r-^rQX84CtXl;0g!v|Ea0zEF?>SL0XXTO#OCn?-V}2
z=Kh~Zg=($51=r37oNtJVj49g{_SM-%qr|s(;U9EcOhATQn6t8+94B-n!qouSd0UEg
z*|7Q4Yu0)~wuz0eo_}b2zLO`+4LB6f$^)DXiHeo%n!VZ=Z`}~JWu%V3-vr?3U-l=H
zV_D1od4TI`M|_H>iL*;rsIOe_l$`8#pY42~=)KTGJ(#F3AZ
zj@+d}<1co@=Rg<@sv?5lAQ0M!dd3ce3HVh{astPC`{)gdW=ehrgiN+>fU8Y#ynuG!
zgEUI0O)@H)<4K%y6Jt0a=_$lq`0q8ljve|aq(35mJBIHqp7_C!+qF;cbJ2L)gt`TR
z5n`0HiBp^l%?>0)n#dVMLmvma`2iBHaZj@_7z%ALb{jDC!j%WGkcA=VqeQGC*MEDU
z-?5Jqf0;u@0*J15@9;6yx&1f6G}?Uo_B)%D0F
zXB|YOd(^Fd;p~5bm({9o$ar7(J`<{OccU8LUO@f$82=AkbW8HhZOD8
z{%b)-J9=#S;cylnFaS}yLd#_&B+i-1_-HSI@FKg$5im6qw_FSpzCBSW3u6!fpbHeO
zum)}PG(1keht~c^$c(3x=3Dk(YQp3H1l0X(;;kO)M8gF#t1Q021Vet+})PQTXom8D@Fr>tiCYH_7l
zUBZ^;+?tOSL92zi4=KmGBQg(i&ezp`i7Vzdlm_pT+%Qq4(1xB^kL|l=>`ZVyaIt5n
z`6z&ov%q0(KYl~cLJ`mgEh1OT$M)5@gY)lHeAMAqm4V%i{f*AoboL?J%UhPr!M(<$
z3%zjtfO?K$V1cCUD;5TP@-fdQs7|e*Eg|hJ1UBR}M^V&jed+4r3eukZ*evBF81y*V6&u!grVP$JFK0bc
zc2&N5CfO`8Ng%2pV-7~7cup8=j9l@3;e5!|vlC*xlGtLXqArJpB~ahm+SD^Nn=r7u8;EctNeH;j(Z20M_$hiZveIt0g{6o0aywXQGX>_
zBUE+uwl|j@^j$urK_W`P6LLj?xpG|$1Yd-zVPcL@|Cd?YR|i>PsuIo&aXzy!c?TDb
z(rF?hkvJCbW6@ArB~G>y-Q82HYP+?DADkb~6-GJFr0H>e{`84qD5FnzME*I-z4I}R
z=yHPxw}9X}_Fa6R%?Dl^suQNqn6dq+p@l`tu`1sSWl!IYa&moKGj;#Bfl%>72|~~Y
zq`5szN}XL7u0w04a~TD2Lu5HIm#Ok`
zBVL=VdMkDrmq2(q3O(&O7%p8f8I|$C^&D}kyFsEfz&wBMw2F6~gXhKA9)bT;NQt@-
zs4nELI6$BHZa5?>7<9@-LbiyaZq3X62jg6iBsl9x&9|zGGhjU6Wi;$Qjmp*!l`U_U
z`>c`=T`LWp-gdAX4vNk-XnFg|#pYuBasV#LC)!3)dM=t6pca7Ir+TK5)CTC&nNN^-
z)|ixI{`qpWI>LmM!d3W{J44-n-kYqk?(wmixO)NUC_20FX<|P_wnMHsEh3|0e;l^h
zBnICMURAO5aq8ijbS_o%7~{iVe|lY|a)>NQ6Gbu2Bd&^cb5jSP7
z0xOU^xyCbNggvtCqx#BV(PXz3j_z`^6=b-sB{_!Axv5|~pTwUhb3viDj`#NV2J78=
z1^9zX08`A#Yg`Yr2I`SCw|%Jqu_*D&ZK!oY@X)D%=O#arBw^+Rzhrw{ElB%re|U-m
zSD?<8T`EXL`{cdAqFkxb9+mXffIIMTzxwn@>^292RnD5p>=^0BQe`{J){o10`m@XJeQ|KMY>I14$2evBBMdo|08<;7
zHb7BEOdlAQ{m#d!F%Xs?MVwp0ukG)VI}8)_lb+UTk;$${%ACtkoMM~OSjYyaHUS0>
zrLdx3#dKdy*Bn_W7~3@#DdSqQ?nLGruOh>bCx>fh)@c26LG|;QaX|P}{9u16#a1In
zXfKxhzhf$^Pk(2ikl%Ln3i8{4D|12vZ<2jdnk?i%&tYhyToy#7|3vIWP{!`o!@_mPZbyC_7A1v$_~yX_#qd=i%LpD8Wf15g>^-a8)9O1mPS+d;yXws
za~n->tEt_-I&6Lgr5Xk0BbJ(Slu5o1&lSaSl0yW032H-}Y`~u5KfJz4HQ66-z~S{}
z!ogsW9@K6GZSn@S(b?6pcwcDWf1cXIP3cjPoA0)|X%aZwz{*m4ji*Z8Hz95ZX)4U-
z&UsTjEm^4SIrpez)*fJAxemOaBoN(8I%`-GDDeh*&k6eGF?_y&jEpnH#l)UaT}3%L
zc#jl5W^amSm_kH*(qf}SL~i5N76R4u1NgwJniU|DBf1VXY=@dJXO;T%maeX35@fD#
zbG6y*100=rA1thrc&!e6f~UAOQjtsN8T)eE>e;t|fgn;QptHKF>Jt!w(}46VFxC;G
z=?4X_iW7?;VWRz%WlEtvc6J36+5-XuxW36&Rj;JlE8FsId*QH)0J}~ZNT8`J;Ers4
zfCo;_CW2i`R()B$yyj`SYkS>4T7Yfs8w2Q6NbN|)Ie|Dn%mkQ;D!B;vAc&76RV|A|
ziqc^X19B`Z``MRt;`*)9jXtS0twOWC2}5t2!z95_Rym$C4>T7J@uMKFALZ1
z{`Rk{gHXT|;MsEn6Lrx1081wVZu;{wc{q;G!A#xs_nfTgD9t`2*;8YcqB&%mqzp*N
zPEtXBTq%-7*@HQp>VSi3fX0>c(Se^n?a6^_@WrcF&yg}wozen~82)9W305Swz!lN(
zu)WM6tQN#B{5Ryn!`KkLmQ~Pdd_|tSio;hLx`erPUUdHJ85ah2
zF)!3%s3%Bh1Supz!Nh5SLc=;wuSY*Roq`J>O*sxd0fD1vH^r7OSNtLzKMz`Gc6N3&
zP8)4^PTjO7QY!S@mnD$gW;xn2YcXsLMpSA3#VvFH2rLU0;(Ywt>q9KZv^U5-BjGyk
zGXUybU?yQqqS>`bUsw-f$l$d%q`%=ydy@0oF)|ZCZR005+^0~o+vtA?TXlp3yi{_V
zu3fh=7)Nx;gp;5jb@q-rGET`J1XvHA0=4}F1-%`Rpn(3MS?Bxer@rjgD14YP7A<0ZFFFmtoW=_THv*(Xj+)UPuqIGj-OKz)0tOV3=)K}Tb^}O~#h^FX!P}j@
z>e|n^aJ*UF%Sq({T(Ak|>NZtkI4u%>eL)m3#(U2~o=!sM7_9y@L6?RSJ)NP&P9F^t
z6&5}|KAfv;G*v#=5h7pxYnxAPL5jz7#DFmj%@u|w+}mp3YK@*CLY~_lhQbEVO?-s&qizKCbwm}Rs8|OoRC@tZJkPVF3{sqWFl*k|KtVN0o22{}g!MgA*
zcjJp3GCz2_=3Rc@Gp^%Y0xH~OCCnQiJA)M64{s|{H)A1fcN+foFEw9~dXnsD$T2?l
z7|$jw0laJ~t
zNe@ZAWAJcz!MjXyIviJ2Dq$yF1R+2UKNJ&q4wJ~N^uQbwPq6P&fuO_v+Jed#*>&r&
zCnhon5rRy(5`V@vS}m}5VMrcVc0+L{z(>+}5UGRWjNmKk!;rVSb!$M$Du{>3
z89?)D@f|EttI`=rLJ!0-EEE2Z-Y5QK~}=7N;s&P`<9t&ssG{;fTZ#E&Z`Bpl?tnQUi-x?HLHbB)5wZkc70X
z86OIaj#}4eBzJ1e%B?m4GlWpEdqKuZcCY>K|5ip^cy5V~mukmF|j<*zXk-!3n>r;GMkXx`uvO_Xh
z@@e8-gIV{+XVMf2`ZPY8KTa#!=&=-pKZp-3##-q-vH(aVo;nH3D1#7LHw<%q`R*dH
z0u*$Ns{-fkt0jiDxkytiQRGvQT{7w=cPeYwHefhc#|P1rq5y%bLyBi{;j~v9zr3EQ
zac^bhU_6P^#7!}$I+JG3!A3J5fC;632CRCrRDQJV)-kTG<&X`9
zg8xM8ae4sW;Q}yMaVTtq`P8lKf9>|V{3RI?0UeWUdp1gQ^-w*nxI2$l`;xhOA&gP~J~iM#xAJydTD3Ei_Ez%FhA7t#slG
zCsQ4tB&3MLlX-$|5YNwV5SNm&eFj@PMv#Hil_*^ao9(v|6nsD7p<@_GW1aaU4`^JP
z$xwg<68R83YPSD)qA)5q7X33fFTPgu22?E3xeG&Y+k)h!kOYiN9>E!havkDJFZazY
z`Otn;99mRVE*r5kd-!4<8^1o-uF)IxxYzS?XBi!}wdpb4OQ=YOypHi#t-e>@x6(Rq
zzx&w(82)roZaMz$y@q;oj-u0=uP!^x`lHeNy!ESEzZ%6yfZ1SlLP?w4Q*{VO7ae;D
zR>4t%naVo0i=gGCh(yo;>v2XSbg3&ViHzY?7EJ44^pBa
z$cbw1l`9_>DBE+XEzc5cyi#UTtFULR7eVMaAvqBagU$fs)~0}#+3-Fyrgz&!b^t#=
zD6PmOht3CDdDWsq3l}$bTx=CB)jOdtF9Bhi9lR95`x_b>2#CAtUW`ixTJDGMxF?;L
z!@4$G&Y@9aZr+<55Wt1z4vIeGJkhssA!8fdK{E!#ZQcI;(Yim9&bZoyxsTb=pq0Tv
zADn-fuTI{(urS`$n8_aP@>a32uD-tT{rkrasqsY*+SVPxyC4IQCi+S=iI3u
zb*1JPzIv5x@d28&;9FZdLVB`}a<-;CUfAgpmTe|?_k-&;u^#=8oeDVfNj8IF74~Y#
z^~R(ki`K6W*SPj8{K!d$BTb
z*C*^ajd;n)`XgO3Anb$X@#++?f7(XA|IvK
z%Z51OKqJv8+0ZbfqonFP$N3LFHy-9Qv#td)rLF9a^5)htS$gxF%_ZNaA04$7i)@b#
zP%o~9n2@A@3JUY^5Nv9l!iJ~!MW3$!Mk+Sf;j~SKa!UlfKA(zgGVSfZW@VTZGqb%3K$pFmW<~rg4*$AWzu<@o+IGZ;_l7IXo1sNxGNd{nuV(4u4s!*_*6x;x;z3RIjofCVhtPlKyM2
zLS$JX6jyTY6N3eb?gXsh{j4v;02;t8x7Orw;<3CV?;kA1AxZBV2pe=D^rJ~n_a0i)
z$2OHb2Y(K`tp3J<0*j(qAtuUZdFP?Bui;+Jaer&jERtJA1A|*l^iR
z<+4hRMWY=YBDw839Cn(5GQE
zU^F3R3TGj&i@LOdO#9An!#{sUpyqm)dqxqk=G}2$o6j^j0`G-_7!6}|3jSES<*MOf
zOx9zeKql;DKS+1hNzJG1EYf4Z$X0fH-0OVpEV!!ZiQ+91Wzm4&0kkBKOJn4s&+57d
zvm;~>4n;$=Q1(FXW>o+9q5Jq4+o0if=(V5)W~y{q@9M%SZ0~UwgosM_h|u^?c813>
z!nCo4)dTX7LU_8;=Mp+FEwoYJL2K;66Gh~n2g-a5=3rrftv)nR4P!yH
z$2wEN?)pr<7UsG*Dkno!PJnCOFw}6LwRIFm4MMSu|3j8yR!P=>y<6_7-=q{TKr|h8_#_6x`W3L7rgy4Ik4_l|Z*Qclni#K^mhfs}qdT=}>enB3Q}Xi{`-+91+rkFO($OgCnP$ZR~qOwo%Oy
ze}h||=tlmrLZo+$Wc`Go1s)II1I@gOgHr=g+{9mcJ(@?tBw~;th3n
zIa*Oj%cz6L+CxwLu&&?A%IXK28Wx6dIG6~YAjucYtca!~QLF6h>&q6tz)GO;icVuE
z?~3uEt1tIaU4pzXN&-(ylP8NLs%5+g47f|EJ+${Iwx33if%)*4U7GiQRr$BZ?uE6J
zse1k1yp~oW>RvPGD4>ouK6w~AWQ8!c3n`rFm|MLr+wVA-XH0Phgoc8JLZ3qw9G)d9
z8_^N1u&Ii3R{XAU?+udTlT1=4krz~2nrc(vo-=VbrrFl&Mpx4UFl+2RjG1EzNz7%Ob}v|hO0j2xWF
z@lPKlwB{1(Xr9*7*e#Ss4+;lk@3wZ`H~|@E$vW=!eD{UPCgj1wO1IX2>$vEj!^4x}
zx*jE~Ugm*q7332(^+8uFk_#16-WYZOLCFCAD76SV;@&9F6J2i<&ms{s5Fwfr(i9b_
z{x&Nm{;ghI(baF~5xYt#AxqUsnsu>NS^pg%ULsd`Gv{s76$^Qozjo@0Hg?A}Tvm_t
zst2=9VqhZYm=0NerT4AYQ&iq%1tuQw(twb3mqT*o1wrICmwqEE>E
zQ>B|qp0GSv76s?uGtZ7OdjoqD4b9oK$+wPk@aJ4kY&eqm!vlpxGU!$lkn}<^Z8Y!@sGFdI
zf&y@#XP`D&v7LxPM*M;P(ZzkkkdKFJHUE^DZ$yz5pVf!&R=+A6s}G-T^ig_N&$b*nf7o
z?S5wTW10IEnlRiqS*^-LmE}Xq#tWV8r?KJVTohvl2~I577Z>w4YY+6$b^d(OQ;F_
zbLG#^IkNx=p^IJRQRnlEw=PqhHr=&v=P&-Q@|lU61g9>hT5QFEcG>=p%!&R5Q}@6A
z6>%w}vy6K%KFgvO*T?D@-CG>E*C9t`jg#I*qvu_Kj`7C!Jd}6s?E+Fy>4a3=0{*aJ
z&-4$TK`mM%mkUfe*h8c(NS|3{>X}u1sBFMjrAV9IwI{p$ds43ZrVrmG()H@#`=$XZ
zC7Yq!u-Nd$
zk*}rUbPHGOl}mUh#$($D8(Z`=@SYc-uiG)U=1iNw{OTY5Ke}b4q^6z|F=knR_r)9|
zEzWfnaR>jrQ>111Owi!YJB@~S0UlzOe-5vlv+JBe>hoE1tqOS8`(C`W!QhDo*DPP{
z_palg9xG=5Df6dv_H?E2w#RQ+Gyg36pltoE?B+^qL3J&-iV|~$Ade-A1sdt$@Cj1ulqG@L*j%LmNW$7cXPvb=Q^*~v#)Jf5*FZsxtMg@&!0b^csVrF#(CMp
zX9NsH~@qs6O4`m~0Y1|UME=iss
zy_y01lf=GQMx+?xTLwg>BXTYW05m8RA&R99?Aupy$Fp+UFwUfs;ij}j>+BJ
zz*kN3-PGWmAayttpuOr&&kpbx7yo2oX#FD1$z^A^?EoMd
z1{ghFT!PD`t*XL-DPN{PijZ`T+v;jxG^iBpYX%`XHU_55RZ54-
zZ9ejpT~|Mt(0#bl_RroeybB*qh|V0Ob~<5n=QM^xI3?~~1_)SVNwKR>v$C=z4j7t2ymVV&_E<0@X7%M!W`AA|jRuA+!_ntlm;zf~DMJR{Xg(D8^Jj}ozVV9?-
zLn`cWb4!Vz@3~iJni5^ME?j2+s?{~)@cTKaI~)a3&TM{T&~A=#fRc0fxkPTsZN0Ox
z=U_r6Gcp{ts4Lc4dt^vRr&Y%-Vm;RYFcDdVP5{>1gQBC)F~PG@pWWKAAZF~*i@#Sr
z6?>RJvAHe&Rch;w_|#xM)Un7})Q||auk#DsG_ol;9tY-T)aCoG
z@!>p_ALW8RknEI{BLkuq4sM&JYRh__7N0m=;cbR_8GIQO;SVd+xs%Hb+7`b4?AnoK
z-u(6(LXf!8k~&^l^5L1fl@S(E0PjrclUN)%rf(kzapokOpNX2uruIMbv7^fPjF#`}TRE$aS9k6SaAIzPS6V
z_wPdmPb0R3m7WUhE?=kR!k9{8MtC{fsvlqy-pr|!Cm}gT(#cMihUIK-1MD3Dsifph
zc$XJTI}jVfeouO58pA}h{k@#$#n5s_Q@;=f_qbj`W4>VQnn@miNGZrW#AllIX$x-c
zpTB3fJ9B7Fr)OIDc!AMtgd??|y%irX4kap;^~_o`vzt#NLU-CU@F<2HFmyNj?7cO+
zbp0`LXmu0h!V^>)54WyFa1sNB!plQ?V#~VPa1`00E2LNooUQJ}w-SClPNn=_~`tz$$&x&bfFgKYk6Jz{F)fydzKqi0JZ`Y``;I#B9=9XLRZ~
z)Yu4%TreTdw&FHU-war9VWUeg7i|C-5~Gi&4|{QJr5YRxNl;}Q*2Rjm--KbU;O*Nv
zNOb`>y;j!Wx~!GXTEJtev_R$DX%a$$cbMxrY*R=$nYtXS}jr$lsblf*P&c^3GOdc#!
z&%YaVHW9FK*Pe4bFAemxZW)eR`+UYd$)pRv6o$A~MPqhuR1~FR2C5HFMdL`uL;y?l
z@&SwU6UPdff8wT^POqMtILV@nWpJ$>au8}&*luhbT_c(PSF(SawZN1cgHLx1g-PhN
zhLw(#LAKm87QeNtah~7vz6Yjxim`2c<{i=t)c)9Lm^M86IjD0#eoaQt(~OU;cQV%o_*_-V
zy~`HeBjB9Tvq^17tyku<6^GWG4GW%*D}v@Bp)|r)yrOyRYybADtJ88^H?Ja=d&ot-P?N4B4*T9{qqT&
zZeX}6TS1^fXBWT}k{H2oTvSxV;dIBLAAI@dxwSNu!o_8^l$6wcP)uyrpD%a&+*Y#F
zXf1gTwiAO)SxHQdktXZ;9wCg?1Ko%);?9t9byu_wc3muy{ahRzPm!Hr#H%VF4W4a<
zqN0z^vO?Uw!6}B34{?44&Q$-&tlcW)2B~(R+TqSXX|CLc$K??7Fi+5m1b=RY4i~>H
zgx#KK7#J)~;D7c**Zen-;vvXAf0AF{fwyHeo$yNL8TOb%VNVugM;n~eF_)asiXUYN`L+T+&JK4Y{gH9oR!-+Bt}O;`=V^;?oh;Lnp)kLG59**^=zCIz`d
ztPF8P+tpJxbNn8BNO)RPbIp*qNB^1`Rsymu?w72&GI*!6M`5YaONHx$L(qv73A14g
zMSH*qCifOB2y@XFd$YmutH!aR^2IS&b6ZhSe94*CCCaCa3AR+JLB-07f();Mrp$5-
zzBsc`;qHv7h9Ol#yPWpcf9r|N)Y&5$9<~~>1BK$p63C1Qx4y!G(TNTh&k8PD8jU~=rImNWI-he}4bLv#cM0iZ{c9^Aiw{|U}A(u{x!t1>;1
z@fAhHL-%Fe92HPJlWG}BIsnB0!pnmnulOJAy?I>DdH?r)Bt({sWlCfn`w~XhHX&=I
zqCt{9ErZCCqEOcCiAaT3Dn?n0gsf9(B~+GZLA0Qf?&sUg`J3~+?%#b~kLz(g9{1z^
z?sSQ7aj=P@1b5CQPk71MUhM#R)_T}_T
zx`W4Y>waY}-EIa83T4ehj8R$W*t4`rRkH?}qnxcWntg+?G+tlu@x=?{bsgJkXe__5
z+`i0u)tqY~n?_vDx>Mu&^J!Z6L+dbh|;#I4NYTQ}5B3e#63`?03*4aiU6u#=e*pJ>8ffy^b=
z)#lX+JAJHde5yWIp2Td5*_^w59s<6KEeCaT>{K!C){`kM@pJ2b#c5
zNWP+?ApVvJHV{+6&9#uM@o}Gv$LfFUTaXc`YL_yTTKwZ9xFyL%NUQbn@%hS#6oui^
z>r+G86Dc
zbkmj9C(Z>rz~@sUmBB7KV(bZ~gjedEEwN7bc#Ny?C$5+jfT-C{iiz?`kw)g1;`cOe6v91n#%
z#ay~|lKbjw7fqDhD$ZAoavUuo90QpQCm}??E`Atj*?0Na?Ds8wS6lXMEh;wphscBm*Zv8JGiD<
zMsGu9W#!n~R`*M;Uu$zNr|(?1u+s+9ZTC;rFNyDLG<=RI8K{MN?;-O^$Q5<>MQoRu>r@_b&(jO53=&11V|6AmH?wf-I5f=WKmnqA%ig++*#v6;E~k;QWw
z_dazMOMFiJ0FuEK85Ogu+-G@bB6yWEz&FoA<*AF!X=$w=mQOm<)T&{_hJx;iI0YFh
z6-e*j91f6gvu~dgupM05qjPP0^;DaOHlf9JGfaGJObjY&f)J%SLjICes$}@yaI0To
zS!Lz)oNp#QTc@ciW%QkJYQ86X_2`sz?8-R02mJc$NxczYHOGTB%WOQ+LIOk{jB2N$
z5x6Yxngp@9?S3z-ZZQf=LE0%ZkS30!>OX^wTip~>A{RQM~W>?|9>u2${S8MuXp
zzX>-MMOII}Pk>oje$wOcf*e|YG!4)yCP5ip$cK>tGJ
zJmhs{t|X}esWnzSjmi;f$zISurACQY0G1Q2Kr$=DX<&|wV*W(rR2Ja
z3*e+S$}ko%4<=y0KGs12_vY83qc)Sw9oxMuEWA6s-@o6cPAOcG{e%Idiw2Uqk)D3w
z^O!m37kit%HSd0ahFle#jw=7Q#DLTOVAMnm6$rpW;UX{~_}R&iXJC0HH2{cCQlGhq
zA>2G;V|83AiyN>L&YbD%GpXENvvfB?l|)2!Qq~YQ#GKCt)Rn?oP)j2^(scLRUCCuN
z5WS`}lmIvKlMsL@I^n#d@Gi_et-kE&>kuYYyh!vUDf9+}k?@l*&}0}gFNb2I2_jxW
zQ4iQ#)7nTs?8LR|K15$TISnH2!yVh4P#Ww?)`gbo%$eKqDX7x1kAukac|F$G
zrODasu5N}oQ`0uo%W*VVK*b24CI)*t$|OGEC+aM$gH31+HdE;Xq(*)*ZR#+uPqstB
zU(mplgc+2i5>uRpojYh1;{g5}}%lml`G
zq}^IOrM>L3Z`*+v6SxQx;+{LNUY!J~gSz-5RRz?E)~&GVYB^~Jj;Ge38WnA;3@c)T
ziINh;O7Sd7OHuw^Cn0LCq|d?Lao=wKv?Yg&q_9-WbU~bw-Hc-ouA3`l-dDdG{0&9`
z8Bx~pJAfH)W@Y`%sUECl4*|NHm+PggUH9jAbcH1kv#IPPsJSc$jZV?MGi(b{QP^3k
zpY4}NS-A_#(>#gFf`Aso2bmEd2SPYO?5zF#t}Uh7L<1audtj8LnH2VvQ+dU=uPZ=L
zW#WlE1%L@6Fiw4J?^|}x)1>y?pFQSQ=CN6=j#=lxS6Lk9Ie9Ai-Iq8dg*&95hq0#M#UZnk&M%)ky(d0X4k4LZ%v%Ka
zMqn{!o`GS1+8DQ(Y=>f*cp#=wnbX9w=|XB2^Xk>uXz-gh4i39eaJdqu5%=&{4qZ*y
z=dZ=5qgyZm4LVKyxiY3l=Gmb4qMzbywzkagM}~>4_c2{=aNw?Zd8@H7?G$fl&Lm0w
z6R-n>p#REQ4#gfjububrVRz;SjLdn8a)0URV8#?uVo=Q4?%AI6jDd)fvi^Lyk6HtP
z=%Ma|O`ehUBIg{kBcm?+dQNU=R)hdBTF7y&fb&Bzt;y#i@KTCaK&$j@K1Ro!mJ>!f
zi?O7+y}kW!vuAguOJ)(@M!dOHt^fV~4^>2U$wEZv-H*JEiGZ75F)hybY+Biu@SPHA7vH0+@9MoK
z+?Cw}pjX6Bj4&syVTm8{17ROO^hDy8wyt+v3)Ay|oTK`iSg^6GtQ&`;Z2n)9cZQIk
z^r-YT>R0E_N{!`rdPJ2$(uc}W<847RH
z`!(qOi=Q3!oKW>Raa?Utmw;wk5h-;PmoD7`dP8i8t%o7;L3ol>cXxJp+IUXlj6Gjd
zw*tMl!TG}lKVZNBL!9|*m(5)@d`iFm{f%)?;5M7Sdoy*MoIuuJn$DlS(KX?ezm2(?
zD1zDgl1cdXtqP2FwclTiJg%#UN6)ngTS2~RSOSU%@w@jL)5fK?S_cAS6SaN$p$?|y
z?*pjvpqpg;>jFR~)jkgv4VsA|R>w0^|7_-(3q?9vRo#>nu~9)Z9ydAV6qi0`z>A=y%S(B5sM7vjzvl9JL;_H
zkDoqy@ECBwda-=L+Ri60As>*)$$Fd`LOcVp-G;sxyt+gB>Z-3126M7Tn{SR@v37y`
zX?>A|z&(lbM~0X{q(>G^SAS*UVW^-1CUNQ}-zgo?+OpS;kL#{WJU#DpP(p~m%`7nl
z;F0m7FodEJ<0FT*-J%ZrHw|im*zX4Ds9YL&X|vwyJbd_2hC%|eGyLcUrMUzWO>Fnj
zxZ|b5k+W;<9m*#SavpbmL+wH`0QQ5NPkpavQias<4@%@G)WQp~W2Ud)5VHc|k@0F=
zJ7PJlJgz=RD`$Vn&{5WRgQ`a5q=ZDPEt_96p}EuY;a*MKcDcH$XuktzmP<;L;hK4$
zy1QE59o5`v@8fJWzuo$ufw+(1-%1V_=5}m9_&TM@s2<
zi4P6BcI_H$PcnWA&5fpx0N8LYPl`|az%gtc(F9g)z6mp3)Pv9jO8e0BY
zPXm2ZE7l&k{`W_}^eJT5&C7h!P6xvMn|E)~Xp8Y!|Azhl{!;#jf6$I(tZ678tFwlFk~G%G>iWz4WF@q_^b2Z~>}buyaEIG0LQT!L$>
z%7TwOg=qcjmGAr#|92H4lf^g#PG!16OTCZ~8jd9qHb;ismMM=k!-Ai%CwiC8%Q3*f
zCWjkjGKIGITIk*L5nIft*~NDyh5}|wQ#}3ajN`&2ETTU#cUzp>mTWeuOV7w?4N1pX
z70L1z>m|4&cy(9A;oOqF14>|XC8iP`s4iqTUgXitU1FBt?Irsx=pP$lts}HzM9C(q
z;;{jhQIpUxFu`0+0g+5~aqjraTLuml<1MQb#wJT~3}+)C44i5L!ee>Q6nJHR$GHoE
zAo6*?s6!}$M0E*!9FQCco8LB5a^2Y
zwUKu3#MMZlAWUIaR@Sbx@RBcjdU~`yYA&v8-X?2F03aX3yhd-nayvl9lfXBx_D^+7
zecjXwgWhJ2YhG|v_s_%VZwjyf`S2kus||?3o%TN+PJLXwr_k}Hdxo+>;iNh~s>lZ>
z`weak1WiVO%&iTpTy=C1U33ElLaLJCJ+^pl3k~f-x^CEKmnl=4OY*Ro0R~v+Y%Wyx
z2sBs~xp8vZuR1!O!NJ`oo0!D<_mjSxSeib@^)9eC#9D?knJ(1w3Y2U2mfOYGNLkzS
z82X*TD2QaKQG@l;BMsGq28QC$s-RRKHc?7M>Pn46XMcbE*#PE$8y<70x`_pq*j5pD
z_B?hzCgv91rZ*PW&a*Ao
zp<);j#n6zI$GU#EIXMpOzrWKP5KYNU<_iu@X`rIQ6u_1(C6g{UZPLVvnhG|)t(xn7
z4j!~ga90vInqvEQPp#?BoQ9iO-6v=fRNvNAmZ&I@d5?e()d)V>d`P9@Y&Lb-IkzbR
zgNG|;Q+Et-csxLmRV;exYgTJ^W0RVFyjDYB^6A6X(&B+^^}$KiU0t3I=v0}Y9nl=hh#RoP`V?Q-HRoY`
zASN~>aV0>MjB4^wRuLvmI%G-^QQ$%u;`iM{yEl)PhDlqmgWoyS1h`dK+@P4XK_VgZ
ztpWUn@%Hp=Aa9Zd!-TEgy4<*;1Q8YvR*#}Cw2??
zL2S3ob{7sfvgI`afGS*)bnykx`}XzI0E7)r8VinE;}V0dlJTeT|8Ukpd?L}XlIAy%I)F;=
zCDcS*oVxp*+M-#@o@6N#?A#V4B=Yl@k}Dx5oQBgil}2LUR?zdB&uaVyL4s!;ef>`K
zwKtd*AdRzxp!{dE`{=~jCe~#SE_8Sdk6Cf){?2Q*PJds9xxdKR1-7mcFgP4EPkkJA
zmuu%Uf05}D$XF`V|;JHUz8+G}&6+p3nhqWWw
zRmgQb?H#I@09!YL_@x~eAv-%>St0RoAU>kzSr%XUGwaVsH}yK+UuvP*o%b7JB@oSu
zKnUQio=1+f<+PRwH_3~ZJgtkla3M*6(U#jk((K8gvY_aK*RMMv=jYU&;gU@M0OKxG
zcu-KFQc+Tf&yzMsG&!&_KhB|`5)m)CEIK*j5*+EOIi=O7z%wfHX3Utu#gRxIRB^cL
zhqIYo=aeYV9)6Tgo@#b4^{%iQL$V~^cr6W-z!rie;nr52m-0AdT;#!nZ-GQFt$wd5
z!d6IGPU-T+`7~-*vSweXH2jGXePLC
zZ}O#bu#n{T$0ntBJaXm@9W$P%spH|O_oE|Q;iK;MPx*2-lsiJ2JoHgA1Q9$+*?oT4
z7%H^c$TVc6mS~VD74q`t{#6RrOA$#EhycUK)uaSl?rl=nS4@$T{{ATGryU5kb2LQ!
zPgl^#>bi5up`5&H7{ph^>xwD|s<0_e-u-e{5QDS|4MaQpUjpe$P${
z@mm}3?Bq26xBb;c1Q3vXpOM^7@!tF)%DVc{tY>RfqssgzIRa6fOX?~}a%BI=Az-6)8uRSzc3=fop$fw3
zY0O3!q^DZn!_ZLMyl?;hA-={>b8-NQq*E4ll6h;bbzKVJhL8%kqLr0yk4D60!D}Zs
zZNEd&y>9wiDPP$bK&L40M-b7twH~}5JOKf|c{9FPzaGf#O4t?Q619(f#&vXgFb4Es
zwn=B{U!4WXCgX79^4!O)3W#cb
zAnG{WyOqD!v!^wthvdc%iiv5>U!rtpt;ZS-KXNFA22X~_Ozw_@t|g=uW2b1aKGbO2<^NFKz
zE6#Gy)Q_#NbTC-%@qXBpxz&NH&Q8WA8@YsrC^|jp3exM29y#)8k``$H4y5T)qaiwLq3UTDa1z$@2Dek2
zAr1$iL^vxzj!q0ZG{E;uVhn27D5qwH-`)KD{O0pIDl`T?+~?RD3-;BuI8DMT%PQN%c_%0rD$)qYb;+hFHQu9eBMrS|Rm
zyWBEPmJl(g&0ob9MpJ8cWE|$wcTX>>0VFapQ!a8XqeWnvk1sg36YHMPPaLW2$*+V#
z`7|9zVRoC30IX~gvXWu1XRI#ocdi1LVp<$G{5EQw%;7egh_?lFLNICXsJ(Nj5)~M;
z1eC*$WSp?6iFZVehDEFy)?yyrVLmW}qUoMb1rXMC2Z_fDq>I(<|6b{@mlnQ{y71}a
zM|TjgDFOO_R=Ev&KlS6ZjeZ+K9UUECl$P$HY^K-UQMJ=9e|eRD;jFsHA(kDXZy}zI
z$UCsU=E@+CMnwGVEWfmQ)1Sk3n;85tgtiyx!qc#mp{r}${d-y=wZBS(aeKZ(XP&zS
z*}x|4#!JxEr)A_q-%%u-4mx5|cOIMwT9n%1CeVt8UJau+rTyWm6XN6S@)t&wdZRnF
zkhBnmJ9kw1hmV*s`(Dxe&TW^ec`0V*Mu$hn@g+^}CscB{bOO2-pAe@L2RJlW@bc)W
zJ8D)oBX_6!>W3NmAFKNMRpEaBT4LO~avRP_WaV8MSs{noiWNaU%ATCEROi^0^`gk<
zO4rmo**HgItNpBNBp}^NO8U!{xbeHI&zGgAJ7|tXNwn(mOy}B}CP!e=boKN^k{H{0
z)aCy=JL001)B~3Z>2Q;j0Q;=y@25xJ>+Y
z;qCbwG!WYohtt$KpqAzXudOU@BbGHtLd+&2MM6;*QT`n5fK2|gS$Z*X%DpeOM0
z;N(}K0UP9GAa+pZB;oB(qAJPC((xM(c%V~TP5)u^Lojn9)ELp}XUVwm2
zek#AQiA{KtXCV1Dv`{~3YDNrxakFB0RCU6TfMrfsOGp-=c$G0I$N|OXo#-4=dcbkx
zCPx>=`O+AWUuw&NWMs;$f|;Y$6Ta01+Sk778lS^nVS!0oMYDYya&N7F{dwuy`6D(2
zlCvRQjO3#6+MmphZ`ve!>e2Y{InWC359e;o**4$NG4fFk(;F?Az1HZ8i%Z%9LOkn+Zf_Lq#qSF-(bU8Ydk!b&1Q9c`71|qwk;o
zHa;f;UGFKEw>#*db6raMDRf#@(Ts;Q^8{qR1?-!le|qPq&DT<6;cfd5Ke9PAUMoDaurP%Wb!NnZFb>iaNMl5=EXLly&4-)^6H}avGq!);#7Iy9wumksa>c
z9JB254>r8zrJEt~hsK$UI`a-|tFSI~@)4ubP}7cr5HWL99)rYYarhWhKT4dl*xv*i
zldT7*(sQz%Ot68i^YZeNWFOFoHbxp^2g6tQdh8xC6bR!h4V1(&I;GF2qsAg~0#n|E
z=;j$i4tAc_+h;z-THN;w+!R=Ch0&lb6MTS@j)!`KGEsD1GWVLR^VsoMC)6Cut3obZ
zE%WLZxH!<{eVb7?D%_lkrk-cCR
z8~^6BXK~5PBWc|PP*=$CXqSL3Cyo_|wW#4Tx&s&fBScJRS{tZ@tuVcwJ}IrmjX{Hm
z=2%kA#97GYNLhFjYB(DGWjP6o#EULbwQ-OPMI*bYI_b?Yv4>+N4=T(a-LIk#EmJ
z6W67?--ShR781db;Z>GCbaLfqR`zXV^+JA0JrlfQXS53p%8bWD&w_E&H$IMEe{tg0
zD@pFI!>ir}gl*d6VsircF5N4)xaGXsiy3QkBzOUreZ@GXuqXNN&g=BU7`9`6dVKd(x!(2|)AN*&|V}Oc)GM(v%
z@9PBCd}#~OL9fd_g6XAa+l%ggdvpR;Ybk}pQT2sx~>)T9{l&MwXM?gU%#)HPV4-Xxg=R
zI5q0#o0WCb3Tys>U-OQKBk3OV5V
zc%B9BZcjv<7wlj4F?gVyJenvYLXGKi#E$Ce^QwB&tQ$VGj3{E}qm`%S7pIk+aMXCn
zn_6-ATb3joIdsT4`cjb5m${dn#=h^}{(|TlDY(W@nUb~Er$a#HokL!~m0AVOh#;0B
z)hieWe?0|PpA1AVyPPy&L;jAail8Bt_m0FlJeYcr)zwNe8bHzFRi0DKDFr5Pck1rM
z788)~PuP_IRda2(TDdMj9tO=)aa?FjjK#f4nPzJxy({i7A-TlAw`?;gY7kG!X$d`Pjuq9jR
zGbJIHdXxj(<@(_Nh2t-sd`=Bg@b}@||8VX1zxpmx}fBFAZafG0j@Jy=u
zy^-maQEt95jXPE+j6wZ>Cp5QM*}JI~Xe-6M2LU=JAl^8y`yQG=?-?IX~-wwnR?xc7MK&+@+H+0(Bb=s~z(cy^}QJ3RVSSt&F
z5}b2S+MP6GKPgQ22YY$-uROG~!@ZN?x%J+asC9b3#BJ~Aa)V2EXPR#f?~v)W{o45g
z@z{CqiD;PF8bn{=&Rd$Iyr_G@RUku#xzFa)#iEhF<=J?@|7$^~E{k-0dGSJ=5efv(f&ubY%UGmzTK9`0J{#u5R
zoot{0h^B@LRp=E<;#>6!?)f1v;PQ((k;d7jSfVPR`T
z0V9?eiWb!oCd0o;X-1tbeXeEQWhNa={@}TFD{*N*az=>fmV7dxh1kl|z32o_d|5gh
zG2eo&t+U0W8nvcBa#1phUUAz@ZxqbbFn6v4B!&{180>L+;I58%!ra2VL!5}w$6MYI
z<%YSrIbWzmH2nyQZxDaN=tTISjO9Khq^Y54eZ@)^Z@TnE;AmjJ8b$SS7f901H0>9iHwXip0fRKpC6}ey|9@-
z&AO^x3mZYhxI987q&oRRZfs}mD`UDUTQZw6=SKb5_V1~(E5>>3(6YAt?WZdHvh@u5
zF~jVLySQyfw~C)=hEf@30v9Y}>xcsxCbK)~hb#}A$!=T_In8pHqX{99>4p0D?axF+
zOpLU&LoxxZZJRPebx5ga^k21z@0TO=RIh8zkLFzO^YhN4MdzE{7#-IC-%PBQ8_SCJ
zem2(=l&QZ{R>r)DaS_L}hZI*tL^WI6Yxt|;=gBdyR6nAT0g=cPO{jiS}-W2%bOvQvAb|RGUlHv%OfOJ81r
zB^7{x)}1;iH-A_AgC0%7W=;sTD|iY1CKH$?;xY1K?ulkr5`BnDLj+zx>}jnWsV^0F
ziGMU~&_INzc9!L~MgjV6HZ+wWVXT)vb3%;*=ubOH9;myY{x=%AqVcbt8OP(F3>PIJJzzglUl3cjHMR6D1NqM{{H0hH{lsD_{nGVML&w7*p^c$AZbJ+5b6VhOS)=URzv9py7t?Vlmzh`KjmG^1irTo>ZNjcD
z>(X-ju+Y(+^Hf$kJ7?ZaShQqGXqIjBjA6Q2Zy>f4a7qo!lpc0=Y0g3QISIqhu7fjG
z^qi6`$Pqy#&PlBgT@Sf`9%Xbh0vBZrpBDfr2E#z;LUDs|N4L<_+(GR-WnJhlm4gK{
z+R&vQOWV8kkJmVfCm$K)IsN;QIQ2Qc(Y=Bu%GpXqI-vQ$->j{tEPG~Y8P@b#>g&x$
z>xM*(R(E-@wZlNfxJyB3bl#fS8pL*^XqfJj@muPcWac`pojR2FA%F>#n7r!nm2V{f
zOx_k*c7pkTJpShB7`i=yDG@U82U3scw{~@1cC?Gh<-FcRBjSgHf)o&$Uxyh|KzVeQ
zuV}=Bhd=9Ax0;;)f9&6@_uu*Vyj2SR-M<$d`oHk+z1Imid^jv7qBry-mD?6d2^iTM
zOO5Kl2Q*9Tehcu}F@bD~ff*xbqgsf6`2-`9umqQ;yq{^iaAAG!8kdcuOl%*%Qkhe%
z+~EJ&Eg9{#FQ8}kAuIFK-@|+kRvY=cJP7VkjQ&fR*Bt{ZH~j5^K|Ql$uUPf{Sy>_E
zPI3`5(0r^TGboK1N*9#71Vlp8U4qMr1o8rfBQp2QYpXGiDhYoxuQ9!}bBym6%;P(O
zfL3x(yt74fZAQ1(7LS`&)lC0XQDiQz2l_CH4-nf2oLi3aA43j)@%-kssW@;%|Kf%l
zkkX%wcl!Eq`N_CFNEn9AzNE8G`%3@0=A-h!;gT{>hgE~H=KL*LU8UGE$C=aXx6dk4K&nEj!^jtnDSxDivf<
zm}q+5Y7yEOi>8O2-kxy?7ZsOaX^pe8iH?9F+4HmE{Z}W3WRxv4(ber0-4;+f_-uRf
z$t9Mr-)q;&pLJco?)_k;rZwVVd8v}~b@lEAUB7L`C>k55sB?C8b9McQzPY^YXmOx!
zGr|?4eo+SkB$QHHqb>yCubs;&wedCgnx<`NX%RGKk7kpD$JW6Wo0LuXr6pHaYMuGK
zfUhs+j!8AtKB7i<;WL#bum2&R|lnB>aU~D2Lil!2H-$oShJAwS29M?Xym7;0rS@p#qk`AYXch?g4%3x*V1-Weu|`3Q(~o+)efu}p-pXS)W+aO%@=G=W$sdW(>TNX?
z7!{kRk^)nFsO4w7Nm1UrcBL{aVuV4pQTBVCr!HZ8sLaxvcLnsEd_Jmvo3!T?+V8Co
z3t4Vg9qDQqnPf-F%H~gauqBp0LfI6@c|2JtPdaD
zV|v}HWVE%|)6mq^3w!eV!UgqVrmY5@1kyN`ej={4a@oU;vBy)1@P@75CWJd0t8Oh-OMVX!EpZhJmV@pgZ2m>h9
zJqPS?NME^P#WefE!N4>`wmyM?
zfMO@((zjwF2TpcX<+rCU{g8w~3lriNOkrFv!$VaR;1D5qUQu>3bCdWArzt!CUb5%h
zqB;le8@ZfI_}s?24Rja@kXk6PVtN$&>DV6}5dhbZ-bnE0=xA-G&+p{3oFrqEl9&h*
z4_<>-Rx}QtOXB0>Um#P4&=J8T0(l~|d%*xdBE1oL=E6mb!ma#NGS;pU1r^z#(i-xY
zw+d_b2}BhrGFF|DmKfkFZ8B1i*zh_z3D844lRM4j-q
zH67Ha$JeQS(^0g4sdgcxrM|VixvvT)N@p`UNvI_O0&bJb(QSA=vEDMLMPUX
z*zBQ`^2xzA{=yhHSOW?r=1A9Ia01Q3rL}v3
z*zDZ>yq+ruO|SWNtP3a8?d#twwM8q7f0^DT5e0zK_>mtJwTQ1blh
z#pQ!aWMVkht5vykI(@FGw?&=e(57HW;9UsJdsuIrdV3pm(@3y(ZDi^f|g&>)Qpia
z(q?(_uN`{(sG^EFlGNIEetJ2MThxT&O%Nf6*!t080R+gP_aN{&Bz*Gl2On4I>3O*9
z{Fu0(&6@))CSmmyGe6gr1m}nbh#{&Z-H1#SgAsGQa(kNVAM)|(47fm$=x8Lj=SajB
zb_i#YJU8wp$Q(kCce$wK!bT3Hbf7Y^LCqnToqRzkim~>c;7b;xT1%SM&~72^PsOYm
zSTtc~N?L_xKGVLS)>|0&o8G%kf2&LGXc}Ma*nWq!?-z*Y+KdR*39ZLersl{#?e%U&
zGF^(JX`Gc+d*18dDGejik@~35IP+WQhbdK;DApt`g`VzT{mO93wuXkvzfQ#vPSJgS!jk`Uw!A46MHM?tiw~p6BEB@{>n@x6oM;
zI}d-~V&U~|%1WCD%R?MChKw&|&W0KXpRhBcZ)20TJg(HMTk{?7)iulvc-4K(l0z~v
zOYOJpt&hJSVPF$CtDRdN)Sd(cjmY?k^pfdKCYg#8VLZVgwhf)4%DtAG_>A*99?K(JldAHsz{=Z<5*a-W}$O5|K1V
zz!%Yjh{(w^<>16_`n472C!c$pWAerW%@H%MWQ7aQl9I$WV@boAWy7BB9o9LdJz?@B
zH8+zbLxBXoleh%SDs<=_wuASO-x^uWRY(#`+g}$4qA%7wuL>76PXMa8b>|AY32S?0*#GuLiiP&xO0j)Th^hl@$SCtKNODX0F
zSBNf;usByUHv3DvVaq#-*cI@L|GAufIvfZ;24hkApZ72Hc~Bt1$J}jRp#PeS
zzONdz>v7)G9{$1AGp7z3SlhUlrd{1G%?{^Qn9_lKP%kt$z?Lt55~fz%KBN98fzDn2
zwrKH#$sJ*y&ra^Q^SUJ>Y~a-rC{UcOTQBJgE-j*b9+7;nBD^Da%~>Xdd2k2uJ>TF7
z^TEgc{(Dre`ko`zHEmCSdpNWCRKL)oucPeNld5$Sk%|;ZP|O_(7VZAP9tj$z*nCu&
zM7Cgf%_`h{kyR*hV4S(Ae1j}{i`^S{#>wUmI<4ow+!9ofdTK%|u**ay?9gV)NFX|!
z$xa>2O!N#jI&^4`#EG+80+G6Rd`AKcPB$TO*i3(02M+(zaOYArvz)7sKQGz6+J18H
zIngYjFuJ4@$N>G57af1?%g!z4)-%&$*kUFQ>z!}!w2tPWi}>-UuB-~bT|Pg`H@vtq
z-E{8Pw7T10t6rl@8o0r)rZVPYSM%%a+mlok+w2PN`}~~JWUg!G7Z=S9meSTxQhFyK?_PLwehbDc7IK3Vp2Sa8{rcPuE;t-+YBrYB~
zS+lawBL`_`sh_xhR$5+|V~pqYh_nK*IBgk{%%DM_d{s!+e6(xs#f
zMW~3C*~ub`Bu{Q0A=TiNoY?7UPrAM=RfSO7*Y<9KlT>F6eo1^Oz#~}=YbR@U>Ed?RRL?SKhKQ-3T^h(RM_iTjlN^p%
zQiX|8L6#|+SZ-;NC=uZQd(oLA$AHtY#EPqcLwWXyk&8RWS{wXX=~;2-zrNL{>1n$k
z${EQ?D}4nFzsctWt1jWk-{iI7zTNM2N3EI^SJUuS$D4QW@<(sy|C}}Wlg?nLNlV=2
zo&W3C-4^Zh`J4azI-RK0|DXTIMgqv-
zu>~|&#wi*zuZ#SKSZ;-w>lklaVl^Y!tntP^tyQIge)Q;E?%-QdOB}fn8*;u;05pMj
zY#8}8W~sS?sd#&P{b5WQ{`aqitJ$QPF|_??OHXjC$>==77sN5;R=tYU*e1$t$JoKW
z{rr75{XeA}2VZ1?sN{(*Yh1VS^OGeq54Pm_wa1lb9wEb$EDn+6f;0@UeJ;*A=BG&Z
z^LD3k|9aUmV-{Mws4egJ+ga%r66+|6rA8Lv0j!S=w5Y`RPw0McgmY2a*so8~+u-hF
zOhb&#?&DJcfQiA_9(;b(zx}+yPUrf-wTL?2w6_mP4B;3$i?7|i-+-A#-0!^Y#Pd0J
zxsQbF``52u4&|;{zOh%jF4K(HFM
zj{2>0eeW&AK%k-^Vy>}yDH`rh1g+8gAzW;TD)!8Zw@+WSR&79`A%a%mK#A+7P75oq
zIkJA^j-U=?t=>;$QHVcP!p;^{2&ZU?$$6yvQVD^^
z$R}E3jb#4sUxh_ombyR4&Fz5ZQ{fJ>Xg4Q>p*U##UH~wl>ajAkCEA^=`A0mR+}AoTxJjA
zEBq(rUgT5WdmsI}Z!3Swd!x;KlfFRETW6J)5}rs$e0q)B=r7{KKfCx(O?oa7yvrK^
zgn#++<=`xTeYX*4n@|1v9L=T}K@`hB=8uBO49<;fNd@uOuSe+<(iGW5HrwD*tz+gB
zV`(ER*U?*x!BZiPG{vc8kqA{&8du(&XD&7QxhX;}BN0YQEz1-|%`Nh6l=zc~L%IOf
zQ_RMuG4LS?fZzq;q9Tt)8bKu^!vdds{Gj(8=C5w
zN+>m+rGzX(ny#$6fqul>@3@9p@y^F(J9$%7u2NTleo_+uOg@ztKbruv;DNS{6$anG
zo*dwQW{6FKfn4gF4P(Y+O|4K^@}xy6Nay&iWNuwVqv-PugP04;9J62t0G1waXTZ-JEptN>|J)29kN5YN@Q3H9ub+GjaSzY_7$jX
z@FDu1!z>{>E&*OeIKe}Tik%GprIY*X+wTX3)1{99JH3rtGRQ0(>w_2CuZ9B?x&>d`
z*3Uq+>OhZlfD*taF|VA1FUp}TUU(19O%2lqF8b;I0}wQ#vpJ8mMieX}={0A7@FUCskI;OsGgb=?c>(X-jVItn@qp8SC}$bz~q2
zxKjIP|Je%5cl^!Qx)hX?qRJ=(lJip~VwJFdxgjh1l-$;M-;+=}u>C}q
z+LkZ$VOwa3<`EEt?J48PcL4lPfRQ(|v(*WeOsbjx&l;_`wJ-bn_*mh8`QsuCy|SjX
zd9c}bu*lB+`bB2_L3@b8zk#WqPwp#=-aa4u9w&}w#;fy?1B$TY4KZu4qu)L=blHzs
zb-it7#@@a19#ylz4bMVoa85#GEdOSU4dr-Ta0`Xfc}OM3^$
z(?j*}7|cqqP;y60=3adDhTRBmEQws=iFD?naN}CGtElk#S9HF6Pv5ny!p@i*}j**gti8>k)2G3s1tJbDnO%v104Qh
zcgG)jepB;1NTr5a0U@jgNzd{`GNGZWp_JIDGNq#oVXA
zwmDZLfWY6s?mpwII2{76a)zn63K0PdU-PP*14daP7w+5R>!+9rv$-3bT`)K%C3pZ!
za=lCRw)Z!MA^+<9`#8PYS$`HhxPM<^p?1kOoglv#SKF2(AK+q83d>zqJ&5<+M_1QM
zqlJ*R*#DDGzX{x1Jw%D$n3HaQ|?{uBqAQCjTk~
zfE0#r*UYac2)IyZiUHst-b+7_zwt}c`;5-Aod5G_A79@EJVAkj6k5$Ya)PUo?o%`th*!#;?T3
zKVWwc;rIrPcp`SiHp9beEY_YQLmbZ2^k6Q<**JGyLRl@e=I~}eo2!~QQsic8wR>uc|a}Rr1bj7
z#3ooLWK6cG!xUgJ=dyH9t8{saKo&uPy=!;h2UsSeve!iKJXkxwDSeH2r?3@o;eKoD
zR`bs`t9ad_vT6TV8IXdKzmP~B#0Ktcoz6r!8df&0qO$#;Q;|37#d5WY$dM{>*v9X=
zz&*REsfcI@qPRYL{`_dhp>R}nJ#*45T&BiK*)8+tB`TK!=LDl3Sc0~syYd_nO?tyt
zG%g`VrBPJ?6sEOX<@BDdRkv)}Vi;~QA6KhbHZkx}Zk&48+rupU7Ai$yes0~mC1POi
z^D#4K{7g?FASfiJq{A^SHBsvC`9N=~>DkOAHGJzN%5%iZ^d+2Eo9F?GtS-FCa8sOJ
zQ)(^q#VX$fU#q;8rEX|d1wL6{bb}b`gkqexw3K
z64wLy%b+-V0)+TM1IH^_1h`9$1Zm~Ly0dkKWb<5;=VqyKFk-k*9?d9Kec#D9W~5`k
z;(Df}`;F>WSs`9qT(si;PW_VYk=pK;%8v4t3lYYW
z@51a}!^*)M!SESOdSUs{+2H@eOEr5(S#JY-CKNYnLWwDPAP}RiX|R>Uth+)lvMKH>^P~2tF^=*>ujR;8$e*0DC{99{$Z_%Ou
z#521r*f)EG@;_^bfAh*$5Ww`$ubFNB6E@v9{l))R|G>U`ARdX?+1Y7NIt}VYr^yC3
z>c8aTj$KiKFPM3^6xHe1R^f
z*{nO7I(8Y2uvTc6vN?l5fx!48R?Zn%(4$*C8`veip1atlPn{a`_N~{2*^0gYiTTAI
z+BYkObrk7yt)NHu<`J(Z&3Ql9Qa3#q3}tO;{1=%a_qPL(o;mID43qzY5m)$snl;%i
z{2rMsJPe~_DGwjs;*gKs&;v<{oxWE4_7Y}EOCv%VPB=;C-YK{
z_v>6cV&IDK#rn%~<7O^-N1(i?QBbO1%AJ8525qpr`L$}{fHk#)o!e&^2O#>bcwaiv
zyVd-n*&FQU?|y}vt0@RWe3p4=^y2W^`M*1ibG9#he>c^2NA2@76CQpa91}lj&Yo@C
zPVG}NEjk}aI!egYfZb%k+SiJn9hFboT5TwXxf`m2vpObI>_Cc`eXWC@PJ(7;N{wh>
zQ7jfce88c&l^cWzvw3_5g*$A?s323@Ir+i7Dh9mIOt@
zU$m)}ecSNLhtF!#b2MgcnEx_x#5FO15d%3ZJF`tnhfvd=1E{G9o|FkBtXh~`5lex$
z1fT0ShqzJ!h3z9#IIL#TJAUL5**!PFXYh0J6~BYEUAhGj5FYM*=t?nAQWA*XglCv}B?@f=MiWLWc|fAS
zKwXe=rTweW3fDobry&d9LY!c*Is@;lK&B&;
zbSv+t13wJ_QF*a((mVB)3r42>shP7+zsmL7*BS3h7n^u}m^$$Nock_g#{A-My=J+N
zV@_+}NG}$dxS5%s@pF$U#|+61V&>!{68|q1ay
z6_*UdlAtBHYLKr5b>C}+;|Gr-Xx$4v!Mt_mcd9s~yRz=kf3%#EJh5}PDkw5LoshpDtRIq=EcMZc}V>46)DOqSKQ6nVbn|`iDP`mLR=K&kmoVlr<01Uh&5eMbB9Q`=
z(JV1Ri=^sV4@^>K4-mE7P;qJPr!F4|Y9?~y<+6=6?XVG9mwf)fW|ssNlti&*=6TcG
zairWtKIB>V{@uHe_xnWKKzoaQP9_qA^|k93pcueJPf35n$Fjt~%EfBRl-s`m7YVq-mJW}?I-e2BqIvXXV9m1EaTeftg?_@<^1vS@NMhXT^^w4XEFa(+fGq)
zZERhV9tUv65(SaOPLjcQJb;%noE*cnq>*P|j?&qiX3(nD+uNsl#LitZ-m02-fsaY;
zmL-6>iQkI;GipT_=__Oqv_eE|=nT&Iba)lIdRKs-hC$@W4SrWQ`2E^ke+t)PtY7VO
zwy6#8^pac6g^%mNeEn&vVZ2k8`f;oa;K>|J?10<@@~%
z@AqqZrfit~Pk^%KzZf>ARK+&{@D
zlH}fdLRAl29oNGb9O9f5Vm1DtBA75fKT$&si73zS$y1LEj6_J0aVS97?&%1AXm39c
zs-;4(epS7L-QuN7_i1Xb0DN~L@2Q>Hc!OqW!Iat`v~h*hRB4-~FrM#5r7?%(B{BLw
zCQAVbB`gPoQqccF)yCH?47veZalWQ^xPIi7oYk(1NZUzZ?TeQ2I)bDbqcHU8N)d68
z{yYU#0F84g=$~wd$glO%>y%tArn2On8#SEEo`fw`@QQ!i2G%9NlsX=UvL#Gq?^Tg%
z`QfE_x63I&=1+vz0&F?f4VQz@J;*Kv#N
zjnO&+&#)2Ga!Nw6R73qnWc)_rjve*n4LNu$!_Eer>8TkS+E=#40)NZEX{3
z2bO2=Kdu~>9K#*jJC>t(wA$`UigobKRr~g7B+0Vn5+5odqZ#Z}hCXou>Qu`mSAe;r
z=qTwJdv*$bkNu|*5hE@m0}F1}lhE562Ey1+ySlgp;wd7ckqbEN5->Ipby3=CB5CGEj4g&*r~ayuffJ=bdij>jOl~HP+fYX(o~*1wub^;6k>y
z&HW6HTpOO8IG~CZMEaUy~H()U%en{(z!n
z465ikIvd{f(Pz#{?sSv6Le-#^91O{C_g9HoS&o1iPM?L#&OI0xmd-AyWHp+;;hws
zaBKmY%}T^0H6}AAvK(4DcDpL~;|xI}2IZcNx9eWAM7Jh%>XXO17(92>!DQn{%T;2u
zhNRe*ama+VN4C@0pNM^j^f_(hB
z$Ic?r>s;Q{NWBI~?Ap^i3nj(K)2-I&FDlLrjELBg6@ms}EvdAS)r&(b9hX;1^RX$3
z9iy$VbV&kj6*t6GfgWB6RNh#H`wFspk`d>0tit@39GjyRyy2#
z(n!VNQS^w{H*0{5suqU<4yGG0jvc6qYeIp2D>QTws&2$Df4Ylj#E&!D#*-mgUHE3?
zhFtg_TURUhGLA
zqyuzq1duZET?=zPQJ}+aG&{j8aoG`f!qW*tmEWT%8-=T`TiO)i)a4>E4iNPdp&}0L
zXTV(mKpx4GXS@LDiOr9y(BR?WbB1*dmjxykbmK;(Oah$hm4)yETe
z!o(=wMF5kp33ZIqcP@oQRS&OUbt4j|uEg@v>Pt^dSoQu+H8mZV!+NeMe6atFc5yqm
zIgYSxEz=o?pPZ+QT-nL=65aLJ;+DnYzCFxwv@bNKWB#wT^;J2k?2IR4DBVFx*V*|<
z4hF}cxC`4JFo$-$o?7X*`NS82vkolA<;$1Z;PAWX69zx&I+uzxB&}g%U%_bzw1Ev(
zXh5KbJ<=m?Ld6k4aLDS
z>^@Sd@6UTyef!2+k%uIb`2qYjdP*WCfhSkg^DL!S8%)clI|)FrPtwphPZZ9%H?jWr
z-hzSbBU8zo9dTcGdPyzqijNGw;d6;cDc(Zu$dRZ|GpPbMH@Kwhczi`BFDmCb6w*86
zjrN)XCH!8JWO9eLkLhjragKk!yy91d{$8&4T5ZL4FZVvMQ;g5;8aB&1r6Vh`{tuAtk9zG2TT`7&1;9IVh>lcksHDAME#i-`+kKz#e8c&OzNqu0~L@A|_P3
z8y<D`m_Fl1l
zM8hdgwA*ubThQ+k<`so8kpf-6{tUF(b^$*>yO)#oEzyCIVZ~0Gf-Gt}`jSI^qkT+`
z4)4EMQ^uspobPYvH}ZDUSp=F~2mLZ}q@yNUf{Gn_t=*7FK&WovFUyVQ2IBNB03Fba
z>~Aa9(VAGW>|DMtAw3{c-R7ogbpAhH&?3qV>g@+$neovKO1u=H9ZPdIEUnHL_2OXf
zuzq%=cfvM7dRy1no@rAC|6MzJU&maQ!<6r0%(EiiJDpBnh>$L{XEQJRObi&*dc+u)M)ZT8&&GcXXn
zqe!>n(7(P;D3V6zBcrPJLnBrQ8ib{2APZTybJyzx=g|ChVLa;(daBL~Us=!8e933z{O{e#AcQ2E9n`~FIE=&5j-4jwm|n^=*D`EIx$SP}bh+t=wsfrZMryNjMXH4FHyt+(x
z*|O$Cl6JPrrP9TcfudEUlCsIg
z%se|janu2vz$!uNLc)}}^v`wT3Zngb>I&OVLLp_uuh75}hP45JyZ?|4t!sypg};^;
zZK8A!<*Cjh*Sp7c+Ln#xf+t@^x!~TrET))~g^uS15Y0I{8r@J23DM3p#m8N%+f|h`
zS^_pq4br}K;#Ny%h9hSw6cRI!`f`C4c6N?Yple*Wb@;>f8RS@pK$aY%9b5`FYDBP!I7nc
zf}CU!F!#d%_P(cQ5z-LuRy0f{12QJ?ab$N_S387EB-9!U6--1L)7b@ub=>HjJGVU=
zEuo7inMTMQC7AHN86lmRop|D_Sn{)|;jHpFn74bb+CPF~4^SS3%mru&yKd~Mtb@}N
zVOv5jOF1_?heV2A6)p)|desM@CPi~J@Ym(bms1FkXItpq*CKHgT`x~&;Ydp~F^Qtcp3@xk$W
zo#)?{aLje(6c7>;@5^LUdxD1S0KIg887jDA4l!tSy%>ADGxWY!nA~#UC+N5O8&mF>
zxxG>ef`F%6eDN~N@-Z9cTiP5B1aqRf4Q#g(WRX)+?S=ZUI#jzzj9?p
ztJ_}bhjeKU1q^V;J0E`BAaf{%5Z1^t54Aatb?t_7o71`9PX5#a
zBzq)w0#HRSLph5(l|wXc*_NWXizYkDP`%EfAh!21BZfbx_0r|bOY{@m>2!$V_I0p6
zXI3w+Xbhi8jS@w_aOcg?Dpk+T8MxDALk=;omuYM_E%sPXS+R2EX7_`v|Ee9R`$ltE
zTO(4{1~w7a5K46aT5Zh)ou{pHP-X3J9stuxC_C)GCxQc~BwR}#>);TYvpe3rv8tu>
z-U)i_CuZp8AJ4K3&vHF5g}l`yes-@cI$dUv*V+wwNBBt>k8po*Crq_P0J8S^7^PTfB#gD*3OQZhAky=q=E
z$uz`C{RHR^iPA2rle&;TWLUqe&0`+AMqv66x*u(d9+fz7;J{-@0@)}}`#&}YqhW1^
zlZNsb5~W(Ko0K%rbMxW{ZtrTa6|FS&irpAH^_-}N&i2aGZisC9UA^^Ja2gG#z2Ut_
zsY%2mOGFAc%820Zd>Uwgn{`EMYQXQFD>WjeO8_Kqs#1L`5_{jv%5Ne%QwR9tr?wI$
zFqA%+6H>aZSwd2Kbcq^(&L#X}$(AY)CvF#Tji^_zo|e^q&Uf}}z}NaIbOJjp2n4W4
zfb^`X05$2O1Hwa`#u4cq6=PpJK0xqG^rmKoZ?-jia6IIe{Cw(IRfn;51apOP^$l(`
zS_ia?ZWVUv?uCF-6keUI9iRp`PC=t({~qJUBd%6~G1+YH{Dxw~1;4N%I&xXSG9t-z&ok#4LXE&|~NsEG@s
zF`w6pj*oo#@)%y@L}#8coIAg;*_FmKAt!_w;Sw%Nq*;?I`%v@F(!txC)LKP$UPPII
z`xbRX&-8J>O9$0t4kU<7#9CkTx?Ak@N|T%pHzHYQR<_6G^~L3N+zUioFx^V>GKpnx
zPaA`)|7A<(C@eZlsy|WH)iJZK_*2Ig=hC&>7IGN_r$$aIzTj%(@YhJ6)IP2t?l#T4
z`F>Nc$HI^%6lniD-qMS%D6qdU^vyScxyJDE=ialE=Brn)Ccq;h?m>=|c@+?XHy<-U
zxFPnXHH{`X@vUD--hB5R2*?`8JEV@iTmjcttxeCZQ+~*uCDc5Af_>feqZQRPBT%$S
z9$cEBaTdbY#c2CizB9*+Vniv%r>Ab}mqHHS?e3yfwpy!W|ML1Sfo&LY|0@8~*
z>EA#;3sn6UL`#90_e5xqcefGcC9&WmT?$dE7^MU=tft*G;W#&52KFD
zK;}SVIC173#i$!15eCyI5iFWjOu{MQ7lMfWYNIqeg-E8IW@kJf^sn7J-1GS}gqBtW
zO2$CojRcs&gLwj)oM`Q^WtO3R2EN__0w>v9x$yzQu+aTH#%2D8`ILIaeZdv9XkD)7+Tyxew&Uwdhos@0~|^Y6I&1WxsYB^})XtCT8D
zmF5h;Aag@rm9e9^BZLo07DcwwMhLI4!~xaZuaeu%c>T3tbIcU1(XJ*7&Z40v)Sm2c
ztd3yz!8EJXHe=_bWVu9d?HtoaEZ8TgU05~b*ax8-F;HT2
zKeTU_BH~hz#=mKZNK5B722}&sGY13kbQZlzcCcu4Yhv2Aau{-XJkr#WXbl1Q1=KdA
zA2rCbxiw?nbm6tBSJ%d^1J_vSmF`0bhFR*nArv9GydN^LHj)j}=Musw-@(diE6GwJ
zIe35&cfbJ1YaaUF7kgP4t#@BFq@iopoL#(65%YA
zIjBW02x!o>00V$*gy2VbJWl{0p5!lpIYVTDh^_IZKUY>KZz?K-teL>*m(gnjut>)k
zpn&MTVQP$7iBl$X*TJ&PDjTyc`dS~((ylada)d^j63wjy(*q%FNu7u7bsgdxh=bOJ
z@6tw}9qgd|z3y^Qrgcdd2c0zAdt4!NANKlYbuo(IUOmnK#>)5$dtl>M?CF2x0|Xq5
z(Grjx+;1&B&MOvp`;_Xk%X#J1=og_i1WQ3O-0{3%XT3=8xR;qJPO_efs}M*jNQf(6
zmIhzi3eXx~lJE&oaT4Vfj(%C@uLb7MPj`h{lbi&|sK7YG%Rqq;4l{LfYE#YpV0}lO
zN#yOma`RuF1_n+PGdLYlL6gCmdrTUJv(hhlaer^f71u{JjM?6tNbr?>zDLs4n+akc
z^&1%>fKG`ZRe0yuyDGIhmBZfL>}CGhd`NxS_|%0r^ip6|$7B|=J3>05Pz73_WCFlN
zL3XqUq)H@}5vAMqyOJq0-yW3^u_%cD?s9MV1}I_sKL?;C94@Dhy9RvXye9j1E$P0QJg0+SA!VJ?ca_Y6#YF3J>rgf(T7xzJ
z1BO>Hh#2#DY2k2TP8e|^L6Zg_*1Y(LwKHf=NT3qP8>`2i^@2*Tj_y8KbK8
za*%&~TMwE5Np$?xGj*OAC}b?SFlEN%@@MDvG2$ut3ae&`23&rWjv_
z|G;3Qa;!Bpv3MX`Agl&~%)urMrR)|!JpA-#jgA@Kjvo^
zvH=Y|S+>8l`CVDMC8ss#f7AkLrSuUZfdrQ#WaHYRRa@8b%^sC_E^Sje8E_?$kC*o_
z`c8ygJ;pLT-t+l(5Q1Mx^e9S{$8fX5s6d9d&5U)hbHSJa_(hiK>gxIjJRkpftjQ$?
zZ3sQ4a*W#e+^GFR_QwE#>Dx70DEhd0$dpZJBfsuTn2#jCF=L=4I5*JZ{`UHot$L`~
z0@QsMA#4O)LNrR`)Ik0AXGKigr?ISl4YB1|XyBkM!2OZ(mdGX0@nm*G#zE}KB;6#U8u1
z$bBPgf%)d<<>BaA=UT9AI(LlJo``=#P*O5$7?(pkBChtMdrPSXG!9|NY-Euf`}K@>
zvr01@r<_*Q7_3>n_VweFm0~7!83#i*oyzFUqG{21QnKWG5|01s7a>u>3fK0^oVuG;
zsV1HsAYkM0D#%WG8DERRQca!Rx7#+Dh+n>CZ?*2+lkSfE+#Ox-6bC0GCTCu^jGj@9
ztTst{C$(@H2QSqKCOz;aI+l42k@|6$yc8NfsxVenbqPAnrErAo4|&g_=#e{6G-{S&
zkp4UR`cs~lJN$9Oq~XyTo0UMX%(3(EZiW3WmdiFMnn03K3ps#<*VO50REc`SMTLd5
z?CGTb(@$kmi@q7DN5~r@1B=Cd4H}(fs3Q6J`GxrSUI;WFKQPJmSn8(PBN!2CF(l^!
z_BO;9eJHb;=*D%8j1sY(m#y2m8xBC-R3%u0TF(p?O#pSYD_FAJ?f~nsg>(cZweXUc
zc?-xkw3WRm$)5QO@*
z2#ayWq;wY4+TiyZ@RoJ}#e)nn?e|bTqBJ}r3WZW8nLmH+;WW&dFxE2M+&lC!m=`e&
z4A%p>fbZ*L+S=d-W#Dbz4A
z%WLQ^?HKhh@MdBWK0$-Oh7QEUQi8+v*|J?Wxa5K@iQ{9v?|!rp8X`p_{(R%EP!+8C
z?Kc`O_=)<+nP^m~My~n?rWa8tHPVdYLCMD#^9l-*u!sDQxpvHNh{Iq6P)P?!WDSqk
z*pYrrhVXaQ4{_xKCI-RAQZ
z_4V3#ih=?H=^XE~aTmdT6@dKdz_74HM1s+lEW(%!7ndwtNv@x2HbHV6{WPSu0@m6d
zMCs_-*u<0AKBQPd&nmQK%Y#x^yJt2Mi69#wpkqLk0d0O9#64@)uI+>0E4Xv%LLbPG
z_^9{_g2KW&ksoGeb5+?`}Ke+K=qg^Rouk0)_8P~
z_My9*-{;B3gwLK?`nz|Z#k~%VNDKmcu)LFym=4-X(roRw_{zzFm+qG)SkQLZLCI2H
zU7g01hA%H32C2KSn*orBs?TZG{KkH^pR35BGQ}SqL@I9dJB?1x&f2?o@4narCyp#;
z_}dCtP*}4{6E2I|2K)cLPcLoIeQ^?WQ?G|??vWJBj(2o3O_<46Snin?=e1)5GPH8
zq$%iIf+;7C5*0Ny9n`EscS|7fgHoYssL4ix1{a%uG+fH@_wMb+XsHI6Ga=m5>!e#D
z(HQ?Ei(?)0m{ZZQlGljKJ!U<*k?3##cY
zwUYHU{AEPnCPY61dBdyK!sF@e>q|ngkvbsC;7)|M#lsY)M_;b$s;|-T%{}-54tqzw
zH}VVhfy7BdpuS-Kd{{ixA)?Si)5QQdRT-gR3|KZ2+iiP=8{Eveh5ggg(n!n*KB^Gr
zz@SdykaoN`GZa$tDZekZ9<6F}`BkhqjLm=~JMh-6C~y(I;IE*)(8oRnc8s1`Dan->
zbg>N#)h9bP%*HFteDz!NZHs^a6$n69*l$Q@M_u&FMxqz*3gAi!l*emUuDp-^Bd#)H0<
zzb#AQaba~VGO6S$+K@Sl*FO3pyQmsp+plx>jzc~bWc8^>iR;nikZTgJ(a_)03TU-|
zd1=G@bR=|u(&@6_%P1#8(#L2~wh);v3
z&a_J_isg?u{X2Sfy}kxr1t*y^k=9>57Ex+y-UPm<0d#6g
zjtBga9(6+t8d_UT2^0jkXg2(#MMLF6!B%fDOyw$R&WcbCT=Vl|VwGDtb(YtsOb2p1P_qaEsZWf_Sw
z<5((^UQ=VVvVG9VX5b^m!PjgHT^FtZ^5s*}w&)wA;QfJ!7m5&BmtH1F{rLEev#RL^^XTg4ZFYeRHo
z;LzKz@`rO6t8*Mx_*9N99wtLq>UxRuci{S6r^rmNX*wxBP>9#SOZxes15PVdoKk_j
z)8HJL2=CcGjHoW**2thiqj%NS8lXiB)wTuz93TY+R#SOdS((-eE${r|e6NE#clw?j
z-f08PSuaxQaCSU^jXxe2i+Y$uEdhdXFD>Je)`UdvoaNp8Wd0r2)x+B(MCMR9Hs4uk
z0=(A0eBnt+{zuUn=Z?&R)UkoSxfh&(B^|n2GH(#yk%lk7W@6pDT&D+(2#t~($ldD=
z0!Y|&#Q`nY>-ebPLUd0paCWczkk=iNIW-FnOAl@h21K97=^I5+2>h{se=po6=-a~g
zn&!9;v=~C`9ifOmfFr~c2^F0^Jqh?pTLgPH21(9Vm6vNF2m|nUN*Gh_qzP~LXLAdr
zc}2s&rlV!}Xcd&K3B0z@DdvL-4C*77Jk*$XaQA0JFi%#dzPz4to11dA_-;YFymc#9
z+#?e4ry|GZJl4aB6~c>@#nLFJ9evt+li5tiPtVmCR~{(iDjIYgDSl%(Dg!L60W1XB
z?ifMxCfh+x?FL@K#}6N(Q2X|w56ucAuzkgBXWwJu*TolQBHYhGU>IgklzADvIq^!S
z=BA4LCreqxwj*p-Xh8Ogt_Mk_!!Y@H7{8J_`!XVO!xgGCPZ!Up*j#C$nMJ?Gm$T}5
z@JQZy)V}kM*PXIYMC+J1HP?3LX%l=bN#~3!O&`NDM#JmwZOO|_+0+F6-t#)&r>Kd$
zzg!*$Bq`{z9uu>oi^sb-?LifZRW>&przX+0hnWv^;8T9PVj6c>oD#X
zS0I<$5d8+9^~2YrLqi!xmnZ63j85mx)+X#Ie-lUKNV)@WU!xVvRO8Kr{Pk{C%Gpch
zvA>zXXB{SPeK2PDb>%IzXQZlv@70
zLrWHd06eEz%!AV}bP@e>Z09-`|JDZTry
zo1i`SFO(cv=dK&k8Rk{yipy0YwF0nstVb+9%H}2Oo)C}5ds_FWX#>+Xw}vv$n}&cB
zB>}4--@_K4ORqI2C?p^NLwI;mNb(Q-0fLc9ATd(>0GH^kS
zM9xmd6+fHvzWud@$-1SN7AwUA*u2Z*IdZ5%B!lzXB@_1i_OJ6(Pu+ay$6$FXd$x2R
zu&iE*wZ8>@-888g#PaxpTGxiI(gnubh*ay<@aduR@8Hr=X#@~RgV6-jCIr;Xi)^5%0X5;{`Z?-LKgufc2R9UmdO;+fg8h1^*5jt@?9^!AIUTswo|`T;$Pv>
zY%j?fY%B4OF+C{ByR3-ECA(wMNA)Aqd`3i?7Yz`6_
zYeLEqTx_^wgBfB7Vku$_tz=nS)M-YnVz5}fpfBiXujkDd}cm;5A;(rQ}5hVQAtqATf@iYFo2y>-R&rlj`E`EF|V!(##Njv^h!e
zSGNw|UA^bnc>X*#y@XR4p&JJN9|s0f`%(*1^_nv}a~oJQlBXoOs{ymBMmFg8o^Bc{
zmK&CT#vsxPw2?-|9(h2NjM_d>Z04iR1lFQRt;2u|jwv_aOJ?l(O<(7~ADg}{J|GwF
zZtlT1;Ed~&lwv)&R*3%QxREus4T>b5lWuNIWL&sRjMyewIuw50Pqj?RL8-d=>O`h!
zVs-bvs$!nr%!v5w(sL-eb+n-Obi~4}S-mpi#WIxUWd=5}G
zE;z_IDuCD4K`Fo8T&+^LY=`y0$;x2!o0v}eyVl#?F`Wg}!16G2jXP46Lr+W$XJ36I
z;d*@IJCD8m)98do;!N(FOz-&6@zbY3kUm}80SJWDEy2t#(TAylz3)baoFbx}5U!#J
zqd87cR+{&861GOomF>nkYFcl*b#&S9a?I6FU^DGJJ$GBjr|*x0D^KK9spn1fq3AA?
zOiL>()d8OR^Lhee*0VUOYYh=Y@Z=EQ84bymdc%l9y5HdDSW(FwAv=Jb?nP`#d>PccM;{>5Oj+%u
zsCA3m8)x&+$&=wAM-T+5LJAU?-QNm^0C;{VzXcBSTTc$h52Y0XqBS;hLV}de>8hO=
zn0N8gr9{+c-~xge^1T{NT_zs<#J?YWY@`$7wf952uD^djWH5#B-SJmYB0z_>Wuwte
z>QvtvfJ!|TGi8k-KxL9T56t%=ltDODb>XH&y2l6x%i6%=qt}PkW{6^55A+9#aD?hw
z2n`MT2GW+J)(nb>NCiKYT4z8+|4?VfL*Yd(E_{lu1TB$}&_NM|PZ2i)wXdpRAIDb^
z&ssPW9MBhgA|C*F1?e97s1Pd!;>tmuV;ZoO$(zU5fD?$rza%vzc|$W_E-Tsmg>VpQ
z`RY}x9sq%~f#54!aYxSQNEGMz$0xMsrg^M#OZdF?NpNu;=XD(HH1uP=?}`d|=$|&>
zuo4r~Leofxr+W>?-(WqbG=1%UeOxBurWyOq2y2^l1aE-Ot-=l`E7h{(K%vL$v-)Q4
zG3A^x#w|atA9N7e2z|T!$igqbMTt^%=uY1^@Xr{Fo=H9!pj`o2?Vx;^fH`9|2M5i0
zDWYjJ1W8dd0gXRbE8i0RBsR{8uk*hs;eL|oD1H_(dDObKdnu3n8-JI4Jk~65SSrTW
zL2ae8lisswO%`K7CT&591~b7N)}UmI12%4hiB%IwiPS*|HBRF`h(iOhp^&w07}fXt
z)rtC`?^RLTC^Pz>9U417Tqxs!P^7M^;vmcq;^a>Y`{kboz8I
z%6Q>n%!$eZ`;EO)E|wxy785&-sVljaPN=8^ckZkiq64Tn)$eAp(|zbz5uQXZ0>3-y
zn7ZPJ7^VbN6U4)W@>O`GJ%AlAW#9CNNy|akIyP&AS(8u
z*dP=KB)6LYFE-|q9|U*?R1mxXq6xwXCt7)BWpB?j+IyK~ypql~@3=8+l|r(zMqr2pw{5F|GRPImRGRU18M;Yg
zQwjJZJN$l)*ZRx7(0M}Jt%91`wJtweis=KuBXv3Azz(5oc7YYZ|1PEXLamJs9^@Jx
z9@6pS!03aDN<~9MAK&0^Sw}DH@LstHN}?WP+CV+rsqIL$r;~j}U+sg-Shipw^92Kz
zRe_a@!smy7OOvGb;Lxxhp67w%QRnaDanTuR4t@$n|9s`5)XO~d_~p-N>u`Ey!&QJY
z6`%k?jFJcc48J*lEkOG3zZ@kUSlqik>AeYYE=Z$!g;MXSJOoziw|A1_2|QR*rs(>g
zf19GIaHsS=Eg7*&ETlluKoB({7V&!soORVU9666#PF}|?t7~FH?vqcbu~^(Y{{H7K
zLHxPM&0T`YFG+}01Wuz3fQ~T*Xv!ABu1#2J3V;0a^RJ(+(3M*lf*IkkQ6}Qb10u{k
zV&IyDQwL|=U3?n`GK8@Tdhs}AL8^FOKJ|ipX_zp9IQxSb(MPrP^H)QB(?35q5S?6x
z+s{9uP|hkGaGdk=w~u4~d_VW+Z_cW2da~f>Zzv(ZpI`F#Z+2Y#*B3Ndyy%}FG4vP`aQhx_ssRRxiMr-}Xa9bkXM5N2h9(+s
zgaQGzoTS4ilyFS672#|C-eMHZ1P)_IT+jy>`aBt$xG6V4aB;YZgKphA66%G*N@&%l
zzrUd7L0{qA9U=byn_-tAc)t-Y)HRF$S#fn2c%4YQ{er
zOFs#sIcQ`aZC}g5kqDuT8Ijz8eN1dI`yY4~oOSUh6@V549w}!s^-<*L0$jkEnB1Np
z{@>C#PTr)ECKVenXass{(dqe6rRX5yqCejsGMO0Al4z3Sfh3o9{=LCcPaV{7L}6is
zs-_;+V3_2j)gu*6!KnrDdV)STf=3&Am3wH-Oku9XF;q_gAw==D(&UgKsEMi-zDyxu
zYYgQf&5fX%AG!kkSIE3fd*}q!z@Z?drg7i^`MY={R-sm?`7lS^7Lpn8RVfb-V*H8(
zL?rki`A-XRz`)?C2GvxO#BwTukO8>zQOQ$-rasZ+x4)mcWq8DU!NK%09&KF7c!^P%
zNNr0}4~pp7Bo~m-QJCTWC-QXxf5buhTbE(>e-4&Mm0!PcqoL4!_!y{3B-F#3V-64+
zJkTL90jYUiVJb|{|4Xo*H^~(m0a&(pam`pBJ4OgjS&y-YI4Rd~bEn{##<1PArVdox
z8!BdZwYmi0da$Rl{yyFuB@2!&N`=NmVd~2>I?C2;j2rPp2}w8_*tO%ZihE&-sk87?
zh=u!2VB5Br8D<6sw^3{p9+)h0bRT-A>xzNykQ_t+v%2o!j!D@I44b@b!g}f*K^`3w
z(p8~kCcGHXB>dq&(&Aj*gGxQ?wJMcbi)SGL-hfPY#eAzruM2?W5TqS3Hws^$uL6%}
za?t`A@&PozT0CXIP;RfDJ==@!Wn&$9J
zA=DP0kkaYB5>7~#dj@zLMIlPzcZmg0P`#-5W6}UBRfC?KBu-;Be>4~gs4xVP3=3{I
zUBK+;S~{C_w>hX~3|iFfvv&e^gND#W6T+)&lkNh7Iv)FhtkhGV6Du$a+8}=t#qj9y
zgAQbJ8tM}*L*|7L@lZ7oUBSHB$KTe`-r<|`2Dx6BcJ5LGhWxhcbVWr~)z*JKk;_7j
z=lhUxjCu%+C=~RmC+{L@km?WA4z4f&p3g(}CRQQ3mOM
zD%KS2qaE@`tZzyzR=+VSiiV=mcS(={CICJpr7rdwX`I1%F}tLH{-Bh@1>_q8QVKe_
z%T3iF6tFv03*DQkWhhffX@9Wr>y$^RZ=$gck!-uz8<_&Ncw~VgA=cRru;ijp9eFKY
zy$#62)&MG0Z5-lOHxMkEfS>`Re6#sKztP!Yll0$AZrZ#n%;W9B4%XeVii_)W4INeA
zn%Gcd!@qzgm9-*fSwd!16?Ju@fgrQY!H0f|I5TkQ@Zo#FbQ@s#BI|6q{Xge;P{Dl*
z$G3f0HzB;)$Th7S$2_0l^c*~_%D&dq9lrGU=hy_3-Jln
zj%9z}ZR&TdsKAgm)XlD#92?svFW(|yIq&Dw
zFDGX?;oEF&M3G0H_DOJ
z()M3)fnN6IKR+ae_wCMwOzeqde`yB?OPO?>G=}-p0ny}(VP|G9wgQj^zAFStp`_A1
z;5jjZ)PAjF$D;95u;6DFLYYiHhpyWNc<}1cYsCOpYsF15fl$*x*jRvnsQ8lrTNJx+
z#hU|k*Rvde2!tfQfSZ*#`uvya5A5DGc%HcP({O@p+q!i>&h;8xkI9XF;1zHPCBqa&
z(%x`J=|k=fokc^5w>PX9f(iT9<5sF6dL^@F9WIW7z3_vbX)9U2O3=(ul5LB?NpdlZ=QBjDs
z60FDq7C{y%7;pe2**BJ0;j+y7r{C2bu){WqNd{Ny-|}~9{P5f!|Af+*UWAxmAQ#WA
zTZim-d50{)t%au!5pfJKHKTmqa0y#~*zgh|BtsXHQ0Lm(+Da}YoB>$VpTGx^>iwdg
zc_#pU0UVNOn!*1e@H-f3cTupY+-dLsryZgN9Qpb1DBJ1}uQI9nh?RzXOM=7T`Z3Bo
zY3X_S?-PN-mWN8e>I(uLd%(j6goNl{bcdoDiiz_$$B3#0b}7bQL~#E2Ey)Hk$5ydff|E>38c1>9c_zQ7}#g{~j3+6@^Np85R%-W*<
zu9+TFIduQkd-z8FXBp`8|6nW9eq?YlpnWa0v<*<9Q0oA|xMJqlG7MM^K0v2wO{VfF#^XB}gx>hol=Avbhgo67`@7(N;TAwUFHKvFlrj
z$6ah;uqBW_C*cre{ZfMp9%$1Ih$VUaD$i#t@c%$>r9fLF|J0aAHz<+ZAzG4&$
z`%?vn{AtIaNvl&8!jBAiq~x?nfDq9Ja8w8k;)BvrQuiB`F{J4x5ndCvo`B}WIDDHx
zS8vsbFzLx4^g{a4i%`lCED1ETiGN3T+?MVosbc@vlcd5J}{Gm}l}7MIageBVhY
zigdAJ^!i-L?b2Np^jDEMg2)(`dq}N`vd{#PPWZXaE)GGNsnhAx2+bYMSb$6Z
zv?dOl06;9b8UV47@{Wit5dm6{@(`PILEQ76!ND|gQIL4F<1#n&2mwcsB$>8jiMQRd
z;tC;0R52F4?Pm&&7M^T!7mE0d(*Ar|Wj|H)K!{hq4+b+4j%Okx0NN#V+#2Wyo+6O@
z+o(efj}#Ym$Ayw-rij}q;K|{Kg~c5=EF@1HQ$t%BPFG?iyRo+4;MWYN}~oip#_;GlU=z6tw}1B0gKNSJ(7S40j85@-a6
znjzos;Dr*QhJ4lB+)OJ1$!>;<8(16z&IXdeOszu?a-USr04#rM8udOce2BA&T>W6<
z1)qyU*g)$r&9w4GZs!sAejKSp#{lJp4#Zx7-gt|?HQ61!s_oC?@nsKLLKC0>l6?D4
z!tbT$^rpOVhy+o2BG`0IY#ew|&@7NfG=fp?-_gckbmeY0FE+tk3~lraq?bTCu7Sb<
zKNQ#4zkF1@J#8G`7wzAi@q#3z_0tTiv(w7$~eDg5GgQfY{Gc7Hm7`^n%}Ep-)HJ9
z)>nd%Dq@aGNdCh>_jRNY0$<%zTqDs!qoaw-l}&%S+-F}x~XC_BXs5GeMyv41w6
zG5WXUhsVdOaZ&{~vHjRf?-Owplk|E
zET|U<;WAsYyS0T-$GAgiIRB)|jzU1Oi)y)Ov9oG?({tOKJwPb_)MGsIAaK{2|ur6*V!D&q0
zOgX8bT;byk*^UOb2L6PBvDkc6BqYS2ypi9lbIQxCs&;t6!$0hM$@~xWRgkhF!p)ON
zY~iNiOBkB3KTdC`oxk?yE~NPNT1lOFXW}0i5b%K9uTpQmD}*Fd0`A-qf>?q4hI*imu3cjYW^fG=1E;^W@)$GG0w+LFmx8Z#nKh
z-=#YhenL9yho^@64t%G@A(zK~Bwf%1>Lmu+gS!f6^vGZsoO?+vRybV|AsqvY6)AlQ
zDua~e5u^H4#MTI%Z;o#0<$|ivODJ63gk`}4#a496m<-4{w#!cXD`|@oHpRQ{{i$H5E1ln?7IC^
zeRJ>ey<@5OMy9E)PT)}wcoyv=Z3
z5`HYjXonVE7=63vH1AR96X5O|5?+u>t(tKhXlOJ|$ij|E6N
zDF6dBs6m6AP7>z=`4unq1fY=ZIIiVCF%H^p5MZGgffX2`2k5!a|OEC
zQlRUlf4rs4OT~|ols<*}k%s?A5KCykQ?s(NR5nz}a#-sU?h%pMy^Y2v7)bVgkmuap
zxX#YL!PVVyxpB3|J;gW1gJA*L@FQ7V96k-F0%58N{RWQ1G}m$bTJzSQT7W%uvx@q7
z!{}Z}8iiq_A@o2vjpE29iI(eBZ}E?V!^Q-wv;R%{v$}FZ5#lgNw|xLMj8=R9B1X~Z
zc8Ya&gELD&1Ee?X2Qe)EbuV!PK+Pv6E{-DJ$l@(Qkp+J3vI%Dx!r@6EO%%>V!0TVm
z({3h!8;@2UfpgdHTSEcBT0o}(yhM$?ru4bktj&{(3ROVZb>A@q?^J{xnS-b-I)COz
zgLW_dWRimw!X3W@7y11;&BqT$9B<9n^_MvO_^cpvJw(%xg$-|jU!l$tbC~SLy-`H~
z7t2oYFqXlJ!KrGChQdD@#+dr6x1??iQ6#)_8VfM_BDgUP9lIvZwB#9pBesS)9*@F1
zmGt3${auUauh)HcFP;&Vh*o&1C>k?mu4bI6#_3&h?PU^HPy8;2OlP
z30n^*sx5WY2K5mB9@T#P%~3DBa~jiiv|vvI+IpAg#NKV}yF0AxikgA7QtNOcF@Q;{
zKyEPTt?KWP*ZEss-t=Hc=~P}FI;#B5KOV;WKRH&J0OcOx^%%HfTNcL
zkdx#PgA#;P8`%fZD&PlAfdfPt1`md|hJs}%g7K4)&b${#fODOL^nePC%7Z0+ZL>3v
zc)X{atS>ijET31i6Q|NU!6%l#ad9Q1T_F;T5~CoCfru}-5|DYR^VgjYQVF;DBXXrd
z;bYK+V~@C!=%~FV;o~PDHt>XG-JeurD$hdd)-!sb>U(hvVt*rUC>p+pPcYqlCwK0m
zJo{IVPR>S9Mog(wan3=~bMA?XdZ)rsi+M@PARh<@0WX0rVC=RJKhKl1pAY`SlTui(
zQi^}}dRQW#_xuZ>_u!7yzx-L_FU*7vTZ_%2`(Qd?8TZ(=!RSv6W@
zqO{eyU5o+XM1;$;c`nUwIR6NdWAmJTKWBQZ!;d&@JTLnQ@q+J1cs_CWn7XyaK5>1o
zS>m|@FhmSwXk#|C^4>`Bi;Q
z9fc0Jt*RqZz?$geP@P^Qa=q^t$7NNCX?+=9dpZcegwj9|+YE&;i~Hc+_mf)BJlgD2
zs!jT*3()OKLTdw%RjzOuk|~Ll&wfSgE@UY;(7*psD6WVj17q7rS%YXNO_aaHp-7rR
zVqgG?6~00g>ViHy7pMcy_Wdv_8n+#x*j$J=*W~;U+hh2|ayHlKsNzB2bj342kViaK
z0RSQB3y{_H@QfVn>^EXA-pPcXyQk~UiPTf;0bt|Zm;kMW`^P(_!UMb%`B>Q8MI&{p
zXgGp-iQE)TABWtjJV$79HWNt{%VM8==JN@Diu)VA3_DX8ukW|d)xq-5MIwH)o<=N+BYL<
z70wK2`s#WXd0XWavZLl%{L3%#4;_Fz;-a_KmT`6EetSk9l{Nz@v7oa!*w$fph2V!!
zuV|{Ukq`uVhsG2qeIRj&;XB+ZKLMH&22WxpfMhW#KK^Awx@nb(8E_4l28qk0G56sA
zz}yGZn?on9E8P1LgeiG&zbkB`<{e&5XH1Sy%O~~@r^!PTk3;2C`-|(c-ZO>jlgHSN
z-zh1}Xq7Z{<-3lYmYtkR?xBmMDVAJ*vnuXbv%9@lb%nVf(L@4f)m6QQ$)Kj*)2t>;
zbtkbc7KIQ&k~Rb8VcYsKxtwrj)x?ZpbG7Z6Ha2C3a*>O>XaC^f3H{eOI2}xsZx#<$
zjTP%~-fZcPu(euXQh%u^M0we(Sw1vAT(9m9Rk(nxVoP{R7sOku@gDOqi$S91c
zBEg~Blm`m*f=aLq1sq0bXJ{AKG@d|!=DX7!%UnOdsijSTW@3G7K4AwAEhxAd&J>8D
zqU!dzsH&=BVTr|!%5a0cc>a7J3q5w-_z$gta2ifok~Tsms(0|e66ma;3Os=2p_^(Q
zj0u1ECrNBa0j+HC9Jz&PPaF3?jm+$CUN4$t-gYvZW$$+F(AqE!e4pyZp!}2=OCQvQeBZGIC3ZgIo`ceUv2Lv2p7~QkI&!q75A%IW1po+BH3*;Ls
zDoD;hb|V0tSQTauo*IHI*Kl$g_e^2-LQl%0cm5y4r!%_?F(;tUUU884-chaU$yUGX
zI6Bx}EBvVCKFYg3Nc!pl6`I>FMJ+{|Ue;UdNL%z=QvCtKpWSDfkO>Nsj0qKNMiHV|e%g0K_$3$*D9WAA+!
zTg8_x^dY83{OFaT;&Dxv%f5v(v!`bSWM*XgUrxS{pNJ2kC8mkAbteX=cX~b=l|4D)
znbtqbrz}H{wD@i9+Bg6XD*gZ}2x6j}_Zd2R+L}Iy6^PjhO`RD+NYaMq6J^>vdY>?~
z(mFsL-BhlZa607f@fS>)#8I*hTaMWGF&Q0TEU6FST%(ynB6e0N(|u_w3Va#{7vkX1
z#OlOt9rFXQAzU8O)Z+*=2p)t-5w$8Gz8L)4`yfWqgFq9?7fB#d)14yTwGckUL1||Y
zvy;h3Lll=2%&
z6{&m=1gwCxSW0<=iJv&<{LY@GkR-}hw>F?GY=RXTM>L;-SM?Qw-&6$QA{Fp-g0(MoKQmw%iH@FX$O!OkBzc8
zMejO*0$yI;_gH+6h!ezrhcj1mz$z#Lx!IX4N2WgiH@Wf*PzhKFH75g4*ns
zqh3}CI2|(Eva?u#
zlEixXVw!w{qYGM2@@{SI%nFIt+B}VWig&6NC3HN4mQ=_M`bK%Azj^KGU|SiW%++nh
zd?b(x>+}PJC1Ws|u6Y}aS+|t;zzP=D6N><*D7^<)WRU9^^=0=3{Fxj{j`GO(T
zu*h(uT-g6$4~CS0__F@F8w&U6~_YnGzm1u_|xex&J7y>U=Aj%v2^**F=>vlPC
zp?ekun^TCjx(=##$9ZpWgBP-Z8+aSW&g6{>^{2N=zHb9x~iWT1Msw#7BBPEs2L7EJV>n36u{fob8yR;1H~}M{q5m
z(hiyamL7p0_Yz~fry3>=m7)K+SKou!(d(!dk(Bm~!C-}Y-@`DcE5A^<1kIw4Egitw
zgaLxPZEnna4213rgp+eAq?1-7saWV_Fz~|<{Om~$Avn)AK+QFW;sJzt;V_aI7E(ZZ
zNy8%`_d3-SJqw5<7p{%l{mM}p@^a)NZy&8S5coQGo3n@Rn&E8s*UWE~zx7!6
zxjb~05v;5mE5vLk)X4KGTTkrkX`4Gv_F>-FRovXZsNV3?T0m*q>0maO@7dO%86I`g
z13W`GrVl{yU!EDGpEv?^ilQQG6Ub}jjQ}r_ctvZ7KL_0K2-eVVo=VESGZf3=Y~&+>
z4?T|;h!buNN(RZ$uA@%{x;>7mszgT$>+H{Cth%EV6YQ4IZJZE&LHl4aW;QBZb?9}!?z@!2@Iu^l%GFdlqE
z%!t+>+V#8~Mg6ZcJ6fMeVw^WZ0LTL-gBU390XLvw0kr()Szu~K{
zpXdmHNb54`zb?7ew3
z*KPYQ{L!R|21${WQj$cYITb2|3=xtciAae;rV=VC%G88NLWM-85<+M)gpes?QiL*u
zeVp#`_w2pb`~I`{df&Bv*0b*WxeMR#_xfDdd7bBR9OrTNbsPgM2)jie!!f`-+d`B<
z224>OwXh?b)FYK|zs4hdyZWoGsfk|&8Y}_u@Oumlga>QnOfIVxI(K1E3Ltlyg-G!n
z%@c3xCUJ1AwSBuI<*xkD$BPe(ABI-_SecwzlRg?SS2b-g-A?+86d(De0n%sgCzz78
z?D@-=6+x<$Y$(y>Kj35GR%BsbD6{3fap(ARE5&mQUMALY^oY{E1Axum&xw2jxG)pt
zDaA!<3lDS&`3xIV44B&*Sz?5w9X74%Ry9$4;iXyj7%zH+ZX76Mbkl63MC(6(Oskco
z(R}ZOe7poz)YJ+<2l&1%+p-Fua{}fnO1u8n9ccfCJ5l9mQeP%+u&Hbv5TP1@FLCHf
zt~|$Mh6;po2w)C(+GkI~(LRg0xTwd|qL=1jo~2`&1#WbB=WPhV?q4r15h$NF8WaB*
z?Pdr!Ejl*>9QJ$}h!tu*(i;-rbu+mveqCvHP|JN`<Uq+~FI9;6$RWe^m@O`{Gx5IpO
zipW!#dWMz%INCX+sU#lNe8eTX2ao!at9QPx`B^c{6Cx{ArKlVf8?T@H3v8wU{I3Uh
zVMtFUed3QI|0=tu8&8YJOQRJt=Ttjcj7Dyk*VBh38L;l-FlhE*Unx-g>@=6;ZFc-t
z2;q?^<->|#5<|nbvU|RGuPZ;*z(F>D#llV=WQUUaUPO6J?>A(?p82V+R;N$o$_Iq%
zI0^s8{e}6ZO?+bmthiSv+*b7l{bmf(i)=vQ#A$N!@Eox|wL)#XvKW38Li5)|yA3_Y
zkXO)3?>p>TT!Bz*M-9{6{3!H)|A;iz#^wzGKf8kEV+1yr;-s=NRZlFg@i*Gq+k0Z*
z8nZLKB(|)h8x!TP^}1g@794cr1}rTrZ;o_$i(P%3RoXZBx|VJ8iI7c*V8`vnJ{9&Z1X&5
zYdssARQzwo=cR0k3LTGmkL{n`m3LXQ;uDKw%TBBGnj12=bZAHz;_mk7G78B9&{?s$
z4n2KO!V%+Y`~_!C*3FwYMK5)s@rIMn_9@1TVL2wMm!1(v&N^wzlqO&{Q*sYyq^8P6
zmN0^YdwK9sN|N42Y)*Y0vp-`Klk1a-{ka4xM59F=kCoo7|G5F&oH@<#fc9$!z;kkx
zoq?+!S$8DrK|70hv3yZON#EflUf-ucDsyUo@B
z0Ncy=Md{sDX8?!hWh2@Fn2!OkSGwsndS-!J#8Ar%kG{Q;=)Jue4n9lJFYkZ#s!4Lm40Jj)z{eWx14}n-n1@=1
z98j~@fpHSm4W;~cpLx&b)Jw-<-v>&YMQ&ihrtV*wXHnH|Onc`f>!O
zeKn7l8(O(IYftV?7DrC7+j2;Tt;6Vz?cH{?conAd{YTv=OywahM^Q2`0NrXm3&&*Eaii-Gy7o0B0lz!Up7#6
zi@^C+Pg5?d$))`eNx@8WCGkKj^9y3oO2R_92@RGnka1yk^#-f8hBZ~Cii7@H-J_T0
zw!G5%b9GUbTm%$x^EV>!bh*Ns)FBJZ<5IU819&)@0o}1Z(K?8ek8f6Zvpz*bO30yA
zpBdyxQ!)ZF@5HYCJbn|b+sA7Fp&2{ecl>YPwieAFfh3t>u)kM?894(bU}2!!$OXeG
zR9lJQ+yh^Q9Sd4_jNR4WH*`lr*zrUI5`9KVcAmk>kjbd~kpE5Rl2@v|l<7S9_?SO+
zpBB|EHj*`bxrlGXcTjK9ADh|g_Pf`KiV_+*m?d?*r%8dKVe)v@GcYmp$(r^VPNMGa
z?$d)|GrrRO>dmEVGrQ0BFY&q)c<>h69+$?8o$fjH{am%W~bQ&y#{<;QS
zh)}?N#za3q-o=`CL$CDZrlRpXAZ{hpG?#F%?^`rQ`A77_GPP+YMGvp`SgeXF&WP%d
z4&B%(aO6pnQ0e_W%WzDWtzXYEHk>tfq(ri!sPbNQm~HjG6zMHh>4sO!76)AY)_Q);
zj;QNOr}Q-+|K6vkrnO0|(azR(?(^d-lTW<6I)Y?v_u&J8hvwi_H-vm|ZS?|HF;TGN
za9I`$`HA)kUn|dUfk`KM8CNA3<7{mMft<~%3M(v6AeY-7?)?m_!1?*-E6TQfykFKa
zQ#K`TZ(I+AY>{ACr|@Z3#%agRr`fAas%^`~l7x=6!tmCy?jDk5Fok3ZB%=dk&uAoRT!JBAlm6DW
zC1g#W2YC!uKi{=$i+hgRj4+Qw-)UtAiMxq6S=go^m!yRAoJ;PHJgR^@ucUs?cIQ)75A#LqW>U)I^D
z)Yf^iI?Vk^?*^lqUXO>J0x?}%V>%vbNt7ln|75)@=Eur=q4!lxa*<~378ehwmGoaH
z93C|cWRs(=2^Zg*V0M72ieS@`BVMAGc{_0gvh%V0K8sglD3*xpKqG5y>hX#2Q$7sU
zayQE-s6{ht6}(lWbJ5(92vsOe*aAXU0KKFi+{JOh(&S{M2H=XbAw^=VfY3>TN?hcR
zH0>U9-sDwP>D+r?HHVAK8{H*M
zNY?YwK;n1png^~{j8B*Z_Bki-{w;^Z=7C$cZ_^v{IVu;t*mg2Q?!UiDUbEa#Xkzci
z-i_-0Ka%HZl&W<&LzN(f)lnUA!>~DVm!T)w>r#YwC`rJD99f%syrB&90)(`%UOXUz
zoZY?<@A1MwWgGaA60;dIX3XK>@Bl@A2cI)!i<>M?mp3#LjR+{+y}jF^cJRM(!!p@M
zQCHd7SsI1Zd*s8JXo$*rlZ<+m(m%lB-9LV8!)5=*b@NLNjb3})I6dg7Vi8Qcpgm;T
zZHDlw^mF%?m^NymB9m{3GH^;~8oCae9p1nVeerBCL?L3PQ)qM>Fs-Ei=%_)>j=pPp
z2lO?v0uEWEX$#5b`RUcQRD`+rcM3>!39mAAY~h4OZWETY3KFHfPTU|SHDT_3CxfoG
zd0#9F`)dobzgoRRqx92)#Qy7{$))kaV*M|{o_RXW7woW_tW@W|+T{`>dZa?wN0NPw
zz=~V$ji>ti>hqE+MgdJzNfm)h7OB4gG+aCIdCBpfrX*3m1q>*NC*5!SzBL1$ge>03
zXbJdHmVgcJaPvEls=5idj$&n+UB_Hv6%7Ic7Hd#Y1R|NEKGg~0@NA${6tl5gJrq-2
zvryP)qoT(I)+2UVxF0Dy0nk7=4FI<3y^EzCmPpTudJ4?~BjD=_e0E@Hh$fr@w#=q%
zZspSc`lEYjGO-vu@-VQF8zO7C+|O-=$ABro?5gK38KS#YGqvoTLD4@Euf@RJDSVEq
zDk}5rYt?vKkF*{M|I%_K=H)%Ua4Xre!y7OVe*Wm%;qudF0m}qX)k5~Yc7cTLaw@E0
zM&MH+1-i>`3W}CPHtCGOacSrX5X0t!7r+4Fr`__;aYlj32&c3B+yGiG8+jdCtXS_n
zmqigwlg3~Pbk5V0MR1|5(!;$v3v@VRE)8i;U)8pZp?or$P?Z%Lg(
z<|y&9Sxul!&C+H6o)L|p5ZgF;=ZgX<<3TGWhJsci5aG)RqMxxalp?kFBLiO5r6yPH
z$gTeEx6!V_B>QK}Q)C2JergF?7SmxER5hgX?nZ~sjSwe(cpo(A_LW=Or*+^cANsYL
z{&MW+d}defgo_^!I5|n8Q1YaaXlP@@(A`YY_=JSnUBL#{EtBWxztmqc-4-GYJlti5O-W%r$sPfjWTYmoj
zXK+tla5BNd&@jU8U%zO`JC%7b)ru@0!dP}x!BotGmhfCT0^^;Fct6G{H_xM?#-Tn4
z0FFE=|L_l%G+r3j%4Zt|T9@5p!%nC1-AS8o7ei*e1Hd1~68WG92tBt;AYf7I
z4ikdJo_-D;jCa~KYwu|(j;sm{=!KYVCBVq&D}s#mY_2F=^;%Co3{Vka>69jsWS_TYQAg5S}kFE
z4=bSiQNSp`co4jmLNd}71LTxLf1saeq7dbsw|5II&FF)%)AN5^7svt*`~~=;J@qp7
zC3LSGD{vmM`mK9(Fe(50J{vdLxGndjFe(Qx{>MM|{rdGQ@_lt~@y7wK-=eMp&TiJPQE00j_orzBc$x4vD|e5=8eROwV0)
z``8}E!rz4^ZfAqt@(&52lSBjKfb!7<3}_|PF&i4Np^#W6hYU|heBu0%cSWZ`k~hMt
z8x=2xDTX8BWBh0ifQ;j~*^1Z~?{gm<`j~4=n;2gQvqC%AAEAjAvr{Ic{R>T3lYm#v
zY+VUUacHYDv$vk~9K2z5X7B-7yL}GSF`P5{uCOqhFg^&Kv`y~soY~rGKC5^Q0v*2m
zM>Osne6TGNs01RN7v2TTQ>#REQ?wuIZQOA9J%fOC?8J%fIAME!zdZ}I9l
zE>u3Dc3&+ezA4-Jjq~mz$DrPQ`vA8nLcbmUho3z%#-5nbRLSf4&Ldxq(ggG!_g04;
zJl~-@{&Dsn8s`B1SVj70=qC0+UWPEZzrVz)vSG0C&(}#$UU>Td^L0??zrTT28N=Ze6(4A5S;DBxpFw=2UzFJgZ>msYGgpdHCXHnJBp
zTF>K3Z-T6_0GZ9aStyA68`|2u0o&-B4cnqOZaTMbetRZGb-*LKi6)cL6>Gg{-RIDs
z1LfB^-XhbTAeW}P@9$&A&mXI|h~@RBjb#}23lmI+#)#6`gQqt2iiU=Z{~#zY0+nb$
zqPJ67S#yIClsGTJXCVOt)f3;I5-583{St90sm}ma0Nu)g*MxEne*QB7$s;{2s>S{u+v3XH%hmdMC_Qd){3DSv%M6dn4>_NWw{RTr{Ghm;4hz$8%?
z0#kwWgjWO30Qk3eT638ds`@IlvUO+aJyb*$Ck8tw7;{YlF_8xnR7mXHyUVa7ME6nV
zV-G~lzrXgjs`v2cf68>^NHTw?i1{+o($X|BU2vgD22_vd0fjV-^Ix#qTPze9dv*K2
zgOQfK?(ZqVVmgluvLVu>Y9ay=924q%3(e4;F=zH{1~@cA>Jm(l1U88gI|)TM52Vx`
z*hka=h8#UVA_h-S$8?
zj~&B+3-h#&D|}n>5y98gU5@JMK<$oimWwWH1Pz$e2Q+jUY9WX~VI(kfF)R)nWjjB_
zd8221k%a9e*lhvyJe}Lg5>@Z93hQvVw2}XNcTSV
zxT!}65J5o7V&v^Ua2(JE&8X<^?w)F!4Tk4F8aSbA^
zGMn{DH*W75cw5YBx%a*1=`8q*-9P4%*s~868jXBLkqRc=JtISkD15Mj9`W*I&<=HLCzx>x1)76D85kdr2An5`I(&pCJMSbt3QJ
zAdcjRU=R4cEuiVLm0g%h(0vM9;?CZ~Kb>*0EEk3UCU
    mWd
z$OJoT1T10z(s2S907^DO62&yB>JSsF(h@pe`ZdBAltD5X=7PwE(84c!sd+E~5OX%|+a2x1uB&Mh&Rb;oN^B2H8
zFZ1%23JcFfOv%F&2OKjKQicZyvKC_pL4>eg;$Yi_*22tuB!@AN%iqd&iA-*aVceusm4+
z&TRn5r;11ofHG&Kte775?4z}!g2EkKauC++z++Fdk*0CdsG7GT0{6ggk(f^=|e+kEwP&L$|Ffc4O7LNqb8HzG1}4mbBjpr!EbPc&$o&%jyqGr**r
zG25I~p5+6=4?&|10Q1IFr76aGxR8E+ufIZ*jFDfj_~izWh$e_(U2qdb0dVf?~PNyuH5s?$=)E#m6rz0
z9<6_FZTks#VNhpny36n;(r-iQ^TvpeIfnLk$TJpxsz9zy!_qO~DHmQ0&9*_bb^oAv
zck6=U&BYyG0-hSYMzK+->%cQ;#g;*GY`fsWt_uUwfQWz|=^uTDjmvn|?_a+HxCGW*
zNHb;f7#ep^OSKxYtK?*Q@HVQELI(~
z3Qe1I(9a8)hOb`@jKu@0>dREdHfiC^?iPvUI~m{iBkIBhH5WZez|+8CBXXPuf0!|H
zQ2#4Dj9$6omd2_%|9&wo$Ak9Ah@s(KGA>K#fO{!im~HAZ4R$C%$ORRW;P3dc3Qdq0
z-w^~?&-9)xr+?O;^y&MA6H+~}_*c3I`r_PB!%<6DPT#D!J*CNV8w{yv*R&o;$Wcg`!~v;(uUjx58Jf;
z{hcXgNLWRhJ>fzy6Cq>GQ<@!60^OPd;>g^dRXZ^4nC4320-k2}uPAX|*u1Ax1#T_3
zOFAI>&Ou0=c@O>H%P_VPfLT70z%_RBCrzGwCp?_N@%6^1gZ0c&Tit9VnT@!M2#yAi
z=>>9$ADS4-(}4UOZ+pvoX_16R;)u)}dZzA7kbzJ&@3o!S*Al#ILL2Cd120gwAp;{q(v>1tg
zkpG_>gGyudnMsSn<|h)
zRwRL|o*Do^Mu&wjvW_L9qO+k@vVo*d-qv;T5p(
z?T2w4)@rL8jj`==I
zUb$Je1sT+X*}=d{PX=@SD~%JnIo8`mpbV^#T3pGCIiM
z>tDTni@NWQLQTR3Szk05uYykK98%Yb3@MATlN-Qp$-BDBGIGFuK|wI?s3gHZ1k_x{
z#89G`7zhn_phx3wU)d2PmWvjRe%t(^!wy2gJZx~9Fzj>XyG5RzTYBlI4<9(;T_^`8
zX7=pa?g)t>vnX7+jSj8^G;4g)MY*Z$szXWT0kh9}_4RykJmFltgS0G9G2jz@5vHP9
zc;iq5V%w}wW0UAG7y8ZVZ|nS1?#p01f*=&1j60y^n{S4Z4ME?E3o{3qD9g^ZJG3|Q
zhH^^BYkhN6uyn*6(V!1U1NFXF&k!KRXUdcP;pO(;T2@)VeY?2KIzGp4xUIVKmX1|P
zXu@*0phF$~89>6p0IX56K_?64GD|X{6;*W(T5(VITD6Enb+JdS(^I?5kZVCfUMP1E
zyYK9G#XW;$?xw*GKGvw37VWf943Lbq)MD&{L~1FWM9g8_!Fny-w5g)qmM{hMYI%d)
zX+&@6hIAE>WRz3`lBHq23!uKpzrwdxs2l&0r*HF2;CDT$LT3nX6nXgZQ+W_~C|K9u
zLcx|>Qj%EP2lEr+tzppr0?oT-CwYSM%`KquqO2VKTs@$ZfqKX?+B^!|Kos%5N0zNx9D?~s=#
zDHdqE^^!Cp9DJHriK7WU)`ckZa9Qh$7{V>_GQboq2Vp=A^F-1Sm=fW#jcx8cFdK|@
z9*lqvd|TmntM4>A^+i#~;^Jmd-E(^Bs1@~moCp#R<%S7Va_zR(0`?2?Olg_pU7Rki#T=N|zVcEeChB=u$s*j9_=!OV;D*yr{4GNI(6^cHG)R!jUf-*r(wWP3M
zZ;p3VniNR1^X~FaZq+Gb9y&
z;XH#AMKd@FJqJcpVjp-}O_>s5Xoy864n(gXy+!DSKS*l$`ej#Ea?y8cY0U$8LiOfd--@3F?j0RbyaghBaU0oDy?fKujD+$2IN|~=t3!a
z6RdYRR!>s;h=ZZn+v+yXn3nxVGXjNHtbiPlOKa~?;z-U;ATBHC77o3p)Joxg85lRrAM*y5>1-pl4AiCq)Y
zDtVEt(&NT3rUeLaIb%qr=xM@~<}9KTA5>w2m3Nr)SGZyZHNW-`qgByAAmAKmb353L
z9%K!MauT)_EDX@?02gN`jn5o@C;1dqGvuZp31&^B+Uo?{C`_NuL3fJMdG&+%4E*ZU
zRB;c=I+$~y
zC}9bhX>GKd%0@3iwBT$`yHz#twLWVL;wlRR+Apy%dZ(`oSEO*Vo-bf%2#gybmCi}n
zUICBR{{r*YHB1veU0}EQtzJhRd^Z>YwI2&J)WsAE2Oi9XOywmK-#xmzc#T<^XlV`P
zT`m*}yb8C}1oa;WZ9n?7LIL%nq+||IL$M*P#2D+nEkoBXeRj+f5ArtmsB&JDyx-`E
zRaU}?#&v8%Z*=vCn4$3_y1et0OAYyP;H2Fs89~t{hZYYR8D1m_(Jv!o
zL!x?O%3bCu*G$$jOt$<$D!$0x*e20zeCIoEAApd)3BZh&%E~T&Q6E35v&kj$7XaoJ
z^>3?j)R=JGoENuz(hW+q(F_pSBnvQ>g@JHB2lt?v=XcoC$>Y!vQTQHk)VU!eFzY!I(Wq$J%VRjG
za$9>q#3@5N4a1cYV0>HdSYC7bv@~VE`o1bxrdCtI#yW${-slk3Uc!+gNhoiD9~3Ve>kZru8UTu+Ed!e;n7w
z2W>wC(2t%bK;hY;{7HeOd|vh^P9$|af9TtB^hRmNqb#E?bWssLQX;98Z*C@K+dS3%
zs%b>@6`(v&9w{_)wGO#FEE7USAVH|-@V{yi8@|)l5^+LY2uDbmV
z7}@sY$Hl;!_h9vXj^rb%IRae>~Lu@02wK)2B%!H%jnKWl8n=7@LMa)by}dW_)0;Oc-s
zC~<0^bHEg@odIv0&AhJ$GItm}>
zIiLe5t742GHCmCi+xW6T%lQuVjoq0ViLq6o;aR)VzWJ4k+@JYJFoL_V7#u|ic!gRK
z0W`$H|LT?&!z@1BaU`lm`Hbtw6K;T0d;7MJ0UW_mdEb6AQ0bI%ljenCL6hpqwpt&4
zr~}9YM!ppE9P-rxXtS6A=Ylr{1+2iy;y#+j#rA9?O^$APc9h%0QRE6anp&*vi{xN&
zF`Juvx{b|ur3?q*k-h^kfBk5d^!V|-?+rI(T_lk+9~HHR=shQ-G5@@;8Zs^B1#X|!
z(u$gxi0VeYdW<{5&aP^&MX1VkA0M0sJ?SL?3Di%xX_m*;kZ(bWhtl~Z?WSVx%2oq|
zMU=?k(I9d*V2%tKcc1I2=x@%CMt%W}p}?vKK#Dk@OYNzEG~^gi{*oHF=5idJt^G5B
zyF&bV>=G1jNU0TY>9R0r=p^>9y-Qc`atQlR-!{%-t>mAR0$28@Z@0<_f1E%U>%*CF
zxJex}gw7-r$1L^K_r-*Cfz7*Vpr(?!voOpyzMca)>)5D^gQMdebT(~c4Mwero>S(<
z@9uLNW&4$!i_)&}N9|v-LtA7P)@>UaF;L7bdzt3ykaNIwFz>5hqPBz~LSyX3=~AiK
zBi)!bK5<|;@{!!GW!5N9)MK>pw=d9ofB2dDjXgsi;Za!yMweV&JM6J0yJh_&F;36a
zcHlNRWOR;gk1V0l^th(Rdbf-rtK}xaNsEX^)XQks1rtXK?4vM;W)_-Qr{nklONQ*M
zB9>2+)JcdFsK!|!rv?lXiHwz|+spp$-oDF5RC+hE;tRV^rM0XT6PtsK`6aGp+^aU|
zO)$c6E9#6uzThc7sky)$`=KyeSpUgkuIzyOk;)_e51rHa`6FV(bJ(d5JO{V{
zA=?XXiUa$H2PP4}pfVIr9Rm;4b7v!^tiT8H)PRAk%Xol5&*XaT1i1zYnfb|Q)YpV-
zSFz0nhG~!xcqfctoG}}!-~t?rhL#pWL|+>@uIh<#i&e(Mtgp|?@tJE1MREeF9+gn(
zg?L5_Hd@XbzpSO4IHL~W_1Q#1Fpc&c4w)%rtyjai-EN3}^5wYs&9<12kw&SCSXF@c
zPp}>E=tYEWU|&$9CxR~H#}@|A3}TB)9D&bmI_&}T_Q}}LoXJ@iQ!^kSAFN7=3Wz;b
z{5Zcyn-i{ND60S$rVQ}wT8m{5>TUF+EL**L
z2Atc_@INbjZ`_=P3oqa*zrO$PBXs5}dnANmIjF=?^JPOZaPDaooQ044Ji{N;A+kxf
zrNZ2#X0q)CU#`~*nAk=^CzKR~Wc&{%i(uj|(SC6voepzgHJm4QwW7
z!V2m*pnXCqX|t?5IQ8eZPjOX;xskNicAjDgo(5~u2-*N9D4&6mZX4M%WW7q@g0GrD
z=X&Mp$tj0y+F)9(1JZ&;}u+n$z&u@9#@mK?AvKmuRe$U9CUP@0FVk%`nlg%s43X&SYd)eov;xU>oOS-k8Da
zMJN7RyWk=Yupii)**N6LtaCsXaDIGL{(Dv15U-%)a5thD_2yH6!M%yhY=Y?6$eJjv
zd0`dz-!WxfXN!z%+G+?9gRtgs>D_54d;xVBcGR%N1dxAgFoVuSUNXIq+QgbXY4`bc9(}
zhJ@E#2NSUtoBhQZMf-l(zz04M+^!ZLAeO;LhE7PwwFzoJ7&Tl7;HR1>NW*QiA!
zD9u<}6%);s5E6e
zBR&c}-j9OvLg%TGn|d%!d4ZySX87GQ1-H5<_bxV--RQrbc#>t`y}pU;I9%Glr4C`^
zXi2<2d90%#QP%#;JUkntzF&mj30hQ1hzD4yj$M3?(jWPg?@@iXMoi$4_{f@QU9$-v
z7BJ5pWD4jIL2C&V5ljL`v&C&b2=>f#)}s~6mrpH=Lf`Fgz(6eOrS9J`6Zb*)Cv-g^
zczLQuZ^wBQ#WZY>!4%c3*s~p@ece4hEbg{89D)nO0tMhlfLEQUzD^~?P+MsWzfgAX
zbQ$I5=FW>DmR!=-xOQZWcN18NWm$dpbUdta`ycBQ^K2b
zA0g^aVN|ZKn~aotN{w~pBVLRWV7sjv{Yq2?rp-B;MyPdOoOmZ=F6xE`Bw{^4jeoqN
zvHA}3b!yP2EFX{EV^)!puNp&e=P|?p<^_5?4~oV=fgZ>w#}O}YeE%HKaRP1UwSJTw
ziJfpSqYK>aXDIhYe%*67M5ncAm|84pD$BIqHhqg}!S%
zs)?Qck3-nCXS`ee&MVgNXx^f*a_2SC`;C;x^VN@lNiI#EF&~k2aW$@JUy+rY4DUjP
z{-{1FDX9nW-^78_=arS8XGLkH-}F(FXtU6KojfBz{JHrZX-KjfXgP78TlM@Z4)t_c
z%FqA!)QYTZ_^Y48L1aYh)qkU1$?x`Uss+OF)X~$i0m2xv4uIK8I9lRK6EWV85*@t^
zG7TiVlvMfn+88$jfc=3K)CZj_fp2OCuTELD5UPeeM3S%Y~%j`CeOo9!UUK9e9sXy1uDWh?LAz9
z08UQ9V2+o#DOU@Od}S+p_M;TL_R9c#>_{KC2uUy%?*I}TzJ0rlWcLm_Q%HV{pYXk`
z91rX%0!QSKO)MFSe4?zK{eS#>b@5yb`XVS!eE#Yi!pw?wv*f7922o%>W+hx2=j
zl0Js|5woCtMNRd2P>o0eC&9~(+_%u+N9ihxTv0en!Y`f!xZpZnMJ396l?=v<4csk^
zlGnN?%bx|PoHFgX9u~HCSei|@PDC&+HprU_Ry+63dMBEVn4P;3byjkO%@3I_sw(6U
zNv8Ix&EC7=vONk^>a@l<%gMm?Nza5%!O@X{9#dzCz~SUN8I|(V2b${YR5BPN9xCy&
z;)WB0pCm@f&cx9yfXC31*Q-&qLVWh%hg|~jKRhBBNPzaA2vVs@
zri437w`Sgu7b{E-zaVzwexm&DuL43B^5=5SRD7L$I!r!Z(dO$Xhpt2(o&(Cu`6J}#
zvG|6|&#oOA$+lVfAmhciua76!$G7!39yCAH_WIWK6SAk@zLvo7R8Dx5kG;svorvu4
zCB*NSAVa;@7M5)M{H7YTj)=KlK<$j%ul@j}kZ`g~50*UJI*+Z$mkql|RUBMzkX)Do
zRjYbx!v}2VUSjCRX1+NV1x}uZ2fYj`z49mD%Ng+O@rfF!%o_Csn3oG%8)O_W$rQEw
z+0xdb4f)@H;n36m%o%&rIM_?Eg7NxL=Kk$?wh!8^A+XTUU(LrDd{*(O}
z^}TKE2iGmcMMs*ys*fM(>f5<{w*pe*8S
zy%D8x0lfeq*0e+Jf7E^L@1`ZzI$Uo^HZ+QgIZ)BydjYPX_k$gcq{!c*%T9%SalUbb
zKHk156|1#07c3JIot(dx(^O*5`txbB2hP=h$NrMnl8Jrhf%&!ne7y>}+p0=Erhk9=
z%l?NZeeZvY*0uk?_zii>c@5@Xn1q}ms6JN2V)8Wuw1))oxdq8OYI-#@ELEQM?9_;p!$^23DAn!T=XD
zMRM(r?-wsvFqPrHZPt3|$dMV)Dc$DrDbf@Q%o%I3mB7|THhby1^fPQ-O8acnYOO3B
z_5gt7Bd;IGf`pyt^YF}IsG$w2kwbkhN>Li`s}l3GDqt&FF>@Yo$d>1FJiMN
zT&4~S6DvU8L;y?|$~=_&DDmUhFK>DNn4s^Q3r1i2;{)dB=TCwAk)?fj48$tgIQuBT
zBu{ngORiiQJ^iz=HMvz$Q)jgOLN4US^tESqbnk3WFikIT#L1wA8Z1#<&{YPst^~c{
zN>6vIsuGjb1MY9(EhFUwba|4|Pfiw={D#!7gZ!if!g>?-h5%rMWTL$+dKQ@-Z;#Px
z!d;pV+oDV9?+7{oYzA8Z@vmK7_SK5~{ziZ+P2&yD=b
zL~Ce@jw7|_A-H>Hxu8qj&(9qp5W?I$NGdruWnyft32fxOqJv(&CIC0h;4>E_k@547JEuTw`dbs+|qb-l4o37?T&f_M5aoTNjFyD7
zF0@lj11{2SZ9zqldDCyaAU^?$7>~01Kq8A+;L7}UvwS_13|4!QR@tJ
zD1Xo$165&Zqo<`P6c_=}kJ?^A-;!7vSYPdP7yZaH`*^Un(D%KlsF)8iF{JnI+wRbC
ztHABQNf`~!E!AyI7QKM6);pJAATmj$!E`~jp@2I5?Fs(E7F!5PazNz(nK|p@qnv!#
zfArp$OcFt%qWlnqHv9&}0?tuaCc!<8-hNGeXW
zU*SP@p+N3}r4?Y7kN;@i1tpU)fG(;Z)KBuE99x8mLLBH-HVJUK?nG<#Sa_&>$EHo2
z5Yf(P#Aw~D7lSmD_+M-`YCnut3)`D^VE-5R{7wPoEq(f@%7O@v_wkF@KY6~!%^4yW
zdVH4Gc>UTAN=TwzNmu
zKdNB_ng)6KilT6K5eCe?IG)K==qel6gIK`UGc^5#1lH(BbD_4YIY?>hO@v0v*KE4y
z1gdf{3+c(xn#X3!Kt3m7i|Yk{uS%SVj*E))*LZEbt|C6A*|hvg?e*
zi6o2x1&HUn0HyK1U*Y}f^QL=l>pY1ML4OC~aI#47&gF``SB^Y{835ysLft}m7+_~3
zM1lB^r7KoU!DSD_rfoZ}>4FL3O5TrlHf+QOv`FGdi~)uc>LkxkoqH?{WDn&TpC7Cv
zodD9*b8lCKe|5TqP>7(D;4*yjHh2FT>+#33XsBWbs3vlwz}v%Hp_jzYT!tZWy3vrk
zY2H&QaoWu?^ud%gccd18mvUvfkG>%q(PJ9Z036kj{h+90c
z5kYvzDUpZFkXkj+dGz|m#u@lMWSoag!}l@Y1^KgN#|W4S&Dc5QHntQXK9d0b4S^dd
zy{aIbA-2VSI4Q+>hzSW+=O%3emk^Omf%?i^G_Rc9O
zDHPgK??oTC8XJ7)rVgp!-RNjj3>K%>C8*YCAzYJeg;JjCO~3-@*Hz)yt!BtuFvq-Q
zEGDRO9_&vjY;+@+H3HkFS7~e@R}qU)N9F~%8qcxrDk^*Wd!FX=1&_DLWjWL#O2NgR
zG^Iq->m|>0&G*~6q0TLHZ2Vcz^@z;Cd_Ryoxo_XHL;Xat4-!M-v5ap6n2vNC8j3|O!;@THR*e&i%-*lK
zh0NHXlZe=KYdiIEfs++#JH&td;JWLgwPirFJ>T8FbxRb|2Ejn33$6N-exnH8fSQ^Zf1g=IpQ|_JY!Y5)pO5d89JIpNz{y
z>YX(gRC9^C$n5=8P>FW>#>Gz(6SJ{dDInlYn?muC!xKb3A#i{tj{|gekWnTz@h}ZQ
z@VBKySVyT7S|9*U9KKr^VsrOK4UbLcnZ+_rXdjEtdHQxCm)}N92n-Q@1v8_7QeVdJ
zj(ZqsGq@_nc}~U_xiN=PL>Hj{taXlUye|t1$Z;hfDcqpqwCnS}Z>e(|PipjURo)85~;k{U7dFb$Lqgb^izDz_!I;|EWnak~Lddf>Gc56_$A
z4sAgrSQ^G1xn%okK|S9EViVR~p9#~AJu~uINE(Yv)YI43SM>W81XoS^S*$3@pdZfh
zi9Im5nXl-S-6i}KQu;h(zvNvo&g8MYKvN%?oPqjc+-7Oo!`H5Hz3D!L5x84;;M=#}
zQI``5p2*M}gtCwv;gH0vub%A*Pn%>sFz=9|`-q)_)Co5Kk>-b?*!;Uc)Ry0jT)e)e
z2O(xD`klzk0Ro)-XO1ArzoH3@5k=A8*bBRo2n>lKwg+nFN#o>w_r5{5*p`a7xTkQy
zaCJk@r|8Lh>e7r?Hv-&Du?MHXs;U6o{~6Nri91``2d_2x#oEsIZVcWwBw2R)`Sk+$
zWdunNn*hJU?$OWV_o>#kg;5^LE3UD)?+JC+c=M6$ZvpS)5ox9EhOTeU+_@7xx9#aL
zSQ_~8i3JA|@~ECcKuatEGW%>W96*YU8W=JLPc(?pnr?b&cxY&P*l8!H2vC*`ZR@?_
zTe!5t(EBrOOh<-|4-5$c>9VzJXCX$;396~CwzM*3M_hdgSo6vIS~`cKQ9^E{OYlUw
z=?53*lP6E+n8!+FlLh?r`o&1iRx<#HC!sKLii*1DrxZncqGnqOGAl(@XtDnaZZn@Y
zo{F;4Qc;YVyQ7&Nv3X|ysx7{Ip0_7$N$ink&}l<{XI6(-UTYXB7{9{W6^sG|-y-tK
zeN^A(m5v+-d7>9k4f2D9n-(Q%NW>GNJ$!iP1jP6(q?=8(?+mINevNiO^cN+IXv)m7
zHaBNQW$RHl)uQ~q|7`KN(6_FJz!sPQ8Q=I)OzPdad@iIE7Sd6Q_lT552*IC-OG@Tc
zRP0@#>;>MA!pCQn5LZ0_b?fF
zb5>==cBm*zvT6eUxMt>R@&#_j-TClW@E2phAG~s{_X}c&2du2D&b>nm9mAlt;ZgT+
z(b^ePr;1jK##Y=#hJ-1{61bToE6$8c=TAP^5z0qGq9cS#9mfg2-Nf!vq-GkhcIc~`
zmGpto*gGz`irS%!s&gH-X1}edm^*ys7ayv?5}%h1n^OTDFY(_Qt|jDKnjNJs=m#SM
z^k>dPG2)F&6&=Xn7relXH3AOgk}TKKQj5}=ud8zpBbsuS?A=&fg)qyu_?t@qlYJHz
ztL9Cg9H11Hp9Lk@5(s=8OxOm72a4VmD%`{Grt*$4+w%I)WO9+zNfm2dVeD(;)J%X2
zNq0F(D|;RvW*Kl(+=x{6g{upWJ1DIx!n2OR-udu&3|)+$k-1n_9=DHuG_p_s`L*L0
zVy5Yv+QfFn(SNmkcFu?}G+%
zK}te_O}KOC)h*bAla8Q|Hdt?HlH1EuAK46A93G7LKKe4hx0-e%`a|14#Er|#fpaf|
z+6n=lLz4ZxUEbeI`U*>`we=8$Du%tDS{MyB8G`@r{(c3Zp6{U~C6EIQ%MPFUYu;zM
zxshOpf&1GGX502Bn^4e2l=k3CnmD_@T{rZL#YS9ZxT|M$#9tICXu|}nV?+0C0b_xS
z$%T610@^?U$xOnkXzNGAxZJYy=62u-=j_nfs@kvFRpN}-
z461(E5jM0__(X;2J@Myg*Ftm2{WnK`W&S9JnuZNGA%@AUBv-mimo5p_7n#sX-BCBf
z#s`?Zh5UFSXElPQ9!yaR$j!fF0u
zV}00s$WQJc5^ZM_0nsgO?F^w&V`>nPhDNh*{M;~2lK(YALc$4%LRhKbA&FJDI+csEp#Md+~2
zCMT)ybu!8J<$6X%N8gn8mHRmGQ|_aII@BodbqzH%&QiM-;|~1g`UdNYS9(8g
z$A*dY`|Ad&U(s&siQ&FDVdBKq2GPIEN~^hpv~0p+S~KD-v~0?8Ad`v!u=oj_M)nE~
zl_#hXV=(9_Gzj}nAaAh+V%rwTzSg~j5VxICv)hc)@Vg?H-{+@N0%zrTOA#mIw#A>FL4L}@;5IqD!5Oyb
zrs%=UsUK2VJdDmvLO@ZGlk|0IAuEFfb;edlY)QL1ReNXPUEH~nWR8nE&g#XwGdS+k
zwwqBnL+WE$Ork1~$&49Wz;w@?W{XMG#(;zTTPVFw+w`MO!L?f(eUqD|uB)2SQfXb$
zF6iOi;<|`|uuQY@=3&f(T09OTIP|sq=i7cz-@dWu&0R=~rk(I?FC8dIb$^iE%hpBZ
zz^9DO@Zi6MmOS`0e-IoNQyh$#h1)>AB4IIj81z^f(RYPG7&d+wXur;
zmcajj0FX0P(mUDQ^K+Il#=)3z
z6tP32Gw>VuN-q2Z)B$(Ggl{5a3Bk0PBmy$|2hHT(FcmcwNjlJvytUxoRRl{|0TJAR
zQ&*b)9S6)5u0A>D(1M-;`T%rV6iV#ITGG6e1Gj<5<|3X$T4{*m(`Gts
z$UX7NXw>;}zL+|7z>z3Ao>zQVw@+12RE%YtF+E_T-o~S~RSkAGe9#(mxAd`}O0cmi
zWtJrs??)^#$UV&cv-z_1J6%k$sC6%?!6%=Y;G4Ax1U2&|_C71iRE*LFW^l{e%dhsv?1C|_r6QNzPte&HK-}iWuahU0-o*JGio@&9fs9Sz_q9~2yaXb6d3gq(CX)phC}ZVNpm9aCf!?a|NNTSY)!rkW|y5_2NXnW2@!CKB6caLYx2~??=fAQ%G)g59`Uc6?=FJQ6uD1f!k~eG)CBAhd;z}BEE~ExzMA`lD#F&N$1U4
zjfKb}e6`rRAoG?3@I}F$TH=9QP1)$o6X{$M{Jh5*umGfTQ$VxeU6K!pd8FjODvZEy
zpBIg|AsS(Vgr1k3HV$Z=d%$w-p=nUp-;5Lh1(vTWTg<^B=as$d@8SRl^oK}X#+;8M
zRC$mQKTfc=_2E02nbN2)rf=u9!1$Vr*sCPzhC79vmDPiUujmKKxfxWkvmZuix4k96
zT2y=)^!oXuGE3JhXst;*$Gars+rpFoF8h2Zxhu$4cQ!Xqf^ZT*MYHWP2MFj{6UidGf^ooaYZuNKiQ1Rf9^}NnQY0=IZlZ
zh*FcOKUMUqGTu1FMfH)D%T2|$0-z?%(phSFoTKTOAr0?84FT@ceMoi)?~nA~0&
z&j+JHcJ$4l8evEF!^$}S{Ak{eR5^dspN*TlZ)^ErW&P6+ohH2>d&H8UrEoRNc-t1HW
zbHw=G0Ul57av&Ugz;4Y#>RY>O8Er~k?UTn4+ou9Ci2dP1J~AK9kXIKFvNc!!mh@bbCSze4+Nu-Tcw00nq`Yc79hQ+xYVB)5PdzGV6$4T31L2N=U%)1vZ)k%o!QwrBhAnI)uj
zKDgKFk=E^e!|V2KUVP8e;NrNJ;gu>O?wBSJe)sMqbOI@W3<16e;k{+W=G`;b9B^>q
zX6@Af>~#AoFASz9>%(Rbpb$53M`l>J9Mi;G!PD((YrBC}mW
zR*|4wKNyDoxB+CXgoI=IdIJ`3EmcIIW|RBpuQhz|bN&moo098R$Z}tk>qE}-=g*Ho
zJ(<%IxMo=wu4QuIg<9!V?XIQ#%hvCGW)jmfsdIO+k#FU`-LF|sGz~s+#SBz=)Gyu5
zg&gFw3f+K~{?Q{xjs)2)jC5}7w-B#*n%@t+fd~ieY|sHDbw>5(f9uvIB%Yt4x@%SC
zr|_LulIF1*fCC87__t3o)v@7;?8b(V#1^1d4U955`t9JphXe+Ly=+xg^u2oh`UQy5
zLZlYTAI#QO^;^{~xolk+94*6`BFS~3-)kojOLs$Zm;A<1l)-TI?SWK_UrcNc-U?xa
zU_IQiB}1ef=?KmDObWhH2;nv*pbW5eqqh0u_shPSbP=CB$1o&hZY`hv%mLCJQX=A_
z&IR`meQD{DsaDvR5nN~9+8paB0zUTh=TE^kYZPz_qdFloK;z1`7%j`&r(V|8EryQ9
z0}XrN0$?zF5gCVXKkA!?rlyD<88A2mqQi0G7&2Z<#~dV<|4Y7=2grur(`nGGqY7bR
zfCOg)i1rPAbP=zW1Botp5iqcFXjcIw1g3eTm*S+Cya#pz>z;I7#S0m44DZt~y+FT-
zy&R~8JrYO-DB$i?q;=T6VFmC(syz4VQ;tqf{L<2ku&GGALy7@(nlnw`SwD;nWMYk*
zu(q6`Jf?MSIC4;i9J8}~3Eb#huO4zWybdU$HI>mzJIa$6ZKybzzW7R9T%6u>ta$^H
zQ{vCh_2W2Dh=Z7Y9vm)8hV~;@CNBZZ`XaFqdBaM~Ik83honP{M1Vkj?at$>Q*#ZAkXiMQr?@J98t*sYTZ4@ll#>ajZz4t|0
z_HX8PCdzWX$~}1!NS+6c
z5)3FjwpLMPqz!9{qBs0O=KElgo=0Lu`<3^W^jvsY-+$p6Ti(##YZ6~-Q1%e_q?=;3
z5F{G)`=ISa=lWjLyoa>!g5I|C8W1!@dI1!@2oqXE%Q6wd$eC6IM#|9Z++WjK_e=^c
z@|2S`u15_3J;@x90@|G$s8t#j>m4njIlR1VsLWKtWO-q@l_N=*JL)jdnTMX3`hs@@
zj{Zc|>5;l13D3aw?KIsNChKBkYU%;DW`A@;`2u1N@p?<7r42fNP?QG&Y8iGx=J^dl
zp50fU2z_yh6&_x_}_X(QFT{hMoE>*g#}Ve7(uRn?_vtEp9V
z9C&UT&O<5JaW+-QWSX3kCASF%1GtjU!X3q!l~SsseG#
zfvyanix;UO9D#njX_m=1R@2+T=$`?b$-=-*=i)Su745dM8>*_S`Qb%_aN+QKK#Zs~
zthmJkpnw+;zan94m19oRdjN9LuItahfJ$onCcE-^+{?9blOAh&InBeHd%nKv0*_Uf
zl+n^w0szH$gjzUgfo&%eRSU
zfS^q_!&9Tp$QVJENavNtZvf?wB=IVG0x@H`;(KyZQZA&@CGeM^J{)pB1ee~3@1(mG
z%qGa-NessPErjC3vgQ{?#-VfN{XiY4O)HE#7?6Ta49+mbD|Xd9r4pKyKQM+z9N3QL
z1sFnacwSIavghzr!;F&;cZF^vLwnp+dtfd|?S`m(sIlwMKKbM&pZq?rZ+7O6#Y3Yr
z0{2eJ4Uhyx$~uNZ6#&H#FPvX;%<*wKLPc#pr6=pL>KGXu)9p}k?;
z!x@ZxK)^ggo!6~|#EcdSMOYP=qmn4+t6e1!ep`?jQndeL;LrR201@GMc#A&EJS-MYttL+uUVMB|~cd)EY|auo<8y+?8orvX^uL{K9RC<;OZ
z^h7iQ47N=uBK#eYdYe_=QAQ9T(5vx)Dj@Qi1PeSez9f$aRM_hDZa~i1&o_}w3qn*Eu*1B#I6``zEFF6WH_4L3%{;!E(WPBUastX;U!2GhuyK!rGsgU}Fe^)04u&
ztq<8_$Bd!$Dig!xZ)!$vZgRNC=TNsy@W?y&ed2?D!warZ3H5ER3OFE5u};YA1H7~q
zi+M-+ZG)d@tQblTx^jgcVU%{_?O54;>;3ea8|VAlV$PxIrmt3}&y%|QeRc%6r~|P0
zltBJ25l?V{k&XM-fO+SMqXIS`jU$MMoC@C!#zJYsE;xZeA+gSFU^Sg7Q=8ySCOZ#)
z7ueHnhD;O|9cu4yEf{<=qi~%#?sYy59%ROV&vrtFc7di?hFHQrh$y7A>F#*r_lp+M
zFp-pzUMZ4?F{MyN@!}fmeb4@6%AylsI?%ucIwyq$qF{v9^~av=mben$>^(P1PF^WE
zvq7rRF~N>yDty-9AbVEM$D26&M-EKlY*;Avd8UfZ4qZy7Wjg-|Dwc>_FdCnJ)zGSki-r~j=W
z5aYT%Pe*=%e1(x6tGUsw
z=W6%9-G;-BH#9S8M||^us;(GcN9G*$Bpq4xni+Whl{GXpB1iBNN2mMOem8Tobu<0i
z^{rQ{-({wN_;p#a)mo4B1KZSd1ZD~Ou@ZF~tx{44wW`m8@nzh6B5;0*fQ*v+U;-yP5kYrQik?ataNH|yk)&^SFqoABp9jL!V
z_v*)Yk@DHC$~KS9+Xu^ax)1M)v>QIW{g!B7<^lJ(=P%pqzZUf&*p|<|*7N6Wf}?jf
z>OMyEaU{J4hdR4reG9r~5G%1TCtj|tK=rG=Xghf1Bm01U0I5-QmQ0b0+^75gzGv)y
zjm4LJkm68NIisC;@BqL63)uyqupo89tmODoVu1ZYI|Q4XnIUP-+I+JN^is^vtdTd
zI&onTfGuZV-WjRaTH;~^@4kxFgL2hdzCxZcM>YdsQ_Ut3nZ&|aF#@-ysOF2
zk9gTt(LW%%3)?JQ*rks(El@3|lI|K;5&Gz5U`mTSn1-OW!6CN=yc29?y?~yCUv8ND
zO^v(F-Uyw&$nbRQNAjE<_iF7c^oq-W@;WBkJ+M;u(ABh)(=52v+KdDADehk~j%s*K
zywB>&&Ifd=tYAAtyc0D$IZ-ecaE{=z`pfp-ja)cmKDGZ1=WkytY(R1v^E6~Ls@h0tfUAvRg2Xl!U=??u)(S+|BX^XHk)i3qo
zq3hZb)ick)l+YF$EU?qV47%Ui+7B-_7vnM5CgjgaGR~RKkSM8GKxR;Fwb}Y`dOEX&HWnQ=&>T|1HFrNB?1{3WZZK34u
zzhZg}Z5~^f+G`tM`{EjTXYg(C(Ue4;pXFP+F}Qt^3RQ&Img!<=HlIXuHim(L*;#Z2
zxsV>o9Db9FVk561X9VdSd?^)uc6M~T|K(UY_rci&zJtZbd>`HtS`4nmUg$bN
zsu{J#=!_F49*NqVi3$sH&;6#6#G0hW?yJUi>~Q1HtEtf*SY(gQ{pp-N)rWY1`+!w-
zYon`{&T+u?!1Jc!@HiP16oi4+@o%MFVY1?*6q5>=LbI596V0HwnH
z_3u_z3LT^`bUoUeAdhFff`fx6HyXHsJHjx4aPrXdPp!KxQSOIn4eQ+A1~HhaLP)K+
zW5*;AErG*?^-K5vi|WFE>{iV3jdoZ7r`3qG>@RLopLX!?PoLr|!?b`)V{5RZt1CX>
z#4EkePAaB5y8Xhx6%Ayz53-h53KP|F~+?LS$}M6
zaD*lBp1d?R905yjD=e^oWZ?W)+V>ewW(KzzqP5XmN;2Km=gv#;jlMjOaNd%GFbJnW
z07ytJ_1UUzZm(YCz5*tefdhL?#x8U6k`qvqN=W$8dzKDtV*!wgM
zm+tJ01p(+x!qIJjpUZD|_cQ(wv}_Dh8oE*6^7{(pA5B7JZEQnZ`XBVZ37iJnV7131
z1BrxS6M$v|bik_mV{fUAIK!1vVA>LdB
za28ThJplPIkUIGDjpy3CnU>v~i*c40JR~fcrJ4ZumJuB;sK%|QLkrLbZ|4|2^BqTEEd#+oxu17K@u+s9?%FMtNNS@oIUByI3n*~fzUbvs6KSce88V(ab4(wU7P4ub|H3iF#6Om0gCI@h*
zBp}D6uxivz5Jt50gl^TUP1~o<_nNYE891|VYc6MHf-2txKI#`BA9*U1OirMaXmD}sk#UwOPlt&1|Te5ti;75oh$Rr5$
z?iXW(2V$HWOEoD+)}d+ruFA1JvJcJ-6It6Jr$#FCsS4IGmNkhXTOXMhkDkr@
zK3hfX`DPkBG`}LpLuTpjIewgkW_AXENJJ2gVaOvP(SE?)?AKJ6x3tMV0Z2O-tLGq>
zLO1SxG)crBu2|$^26EtO@H?<$^|qjA`@YFo1go1=NnN{k%@lYq5#@kB!9YPo)D_gf
zFKix$P6v?DH$km~s$G4`iOEyf1V*ZiwyrRn=2~3G2pmwvjKTh4h48{e^?Lzze`f*H
zW$OrG4}kek30_ZRoiHr4Y&2kH)O%yC_SK*KpeIEWK03)42*a=y%@@JNxb$THktd1c
z5;!Ui7;x?y-=)$FC<4#pK7mlRX2>Y=xb2%aJYN;)
zIPdj%7WA9v!^iIz-ioJ7xwZxD#5-M;Ao4$X@nRa9#3FF4E|Sj!WRQ9G_{Nxi!f#<8
z2hJZ+$+~H{9AFU%6h>vMk-<#r+-Uub#4p}yc%1Q*JG38p8EI4*C0jBnH?=6}(j`h)
zz&jR>H7!o2>#C3z!P+5J{?XyY6R-F`JK4r|0QDbGE~vnKGW;Q0*a$S>gJO2L|!r7tn7%Yw&O
z=VDKZoeJ;`IBzeasT!SJ#KuUinOFrPWDx1mi{bV|%jrXhV%Ux#&_QN1?WgJVlwY!a
zL3uZwHhEwp_hIBqHfeZ6u*#6Yx6_}0ZY)EoslWJ4HTT6#FDA+1#G8vv9_q8#xpNaf
zefo3+b(abimV6R*@)wkC*2iY(7~Pt^?(vq3x&te8r{Yk
zfKE#6_bGT3iJt}0MHUxEZ>^?BdkJ=zc(Dl-*3-MOcf;K@iN7DG+Pt0xW)#payR@~5
z#m)BEZ_{M}h2n&a!Fu54sA3e~gPj2)#YI!(W|wFbwE`2MN)ADgyAo?LolwmlGRv%e
zj5Sn4Jf+uXC$oczNxKlcN>CQ>YkA0X0EuI@5zGFM7QuhRZi{ioC#
z2|9F+Vw`Ze;W=fs;ZMN&@SL|#G+$x8YKmR2Gd5J+WR)>8+>c-_voxvDKXnCgdjPKF
zV(l6QOLKI+?|_v8|mSm~GGHbEPtIR8FYluzLE$K=9C8#z$e
z=#f=beM;Q(5K(Gd{mlMaG(oxClp;+CLaDhV5q5ok>
zUxeyk9<3C^jn7-X^wT@Z2Y3rfrj7bf4!Zc)C$avZyvR@Q?(n~WQhDt7skt~gk3x3@
zDjK;&QwAu8FIp@CT0$^d(Z~F7Tnn;OBlHo_LpMTj_&7tO0RvAZ58hI3p)_!y2c(L}
z&ru6;49Q?HS|1bgr{=*l`EUU_5p&@(*i<`UqVjcx<(MU+Qc|C@XIotX8figDtxzCa|MbtIYM3mt31`B^|IlhIf|ghRUV8B9?XkwL&7~g4WHQ
zg$rldYOEfwyk+L!f2{~xlxdLnVv4Rwqe5`2i|%W~zVV;(`qw;IlQksQDtEB_SlO`y
zWp?E~??79w7)%tgu&~&AwL~k}RnW@Xx?qr=!^$^r;+;J4dC|os4EDnML1S>F8!n-z
z0n&uU#5~_`}yxb0)#6Zh(KREww#i;PsZ5w}XqENN=9I!50=H8s`J{b0fm8SEGE
zZ1@z3;Z#O=7e>xg5S{AF$=7B!2wp7h%xH-kTbh+UG1E%?RJNQv9_rDrz2$*S#m73*
zgv9Hm72ho_E-4v}xMl#vAs)BF;szevB^a=QT`k3GVk$!(0${N8M!m+V3<4G5Wnk5g
zVX*1Hi0~apm&Z7BP6u91FMQK;3Hy6+$2Tnxq(~QMOivDpao1I&*7(E28ofH!u~&8n_R{57#CjNzPw(Lzm!9P$mjy?5cYXX`aS)IT(Ec+F
z-ayI}nA*aF4Jmp*xI27&;{o)DY`tFfQve8RYuWjaSP#P?NxdTr&8jHexTj5wZ>CAM*@3A~CUj%o;~c=&KG@U{Ywl<)NipRIcXiaLlK@tlyzPE{y&
zFixVgJ5}a@O@eb(`o~^PQe|Z`h8aO1)qZc=<2g&Azq$LjcZSc2
z&;}j@4u>NA3v%_I5ICXVpvdhjh!`wD|0TZ=wj03-BWuWdIh(Bj!}xdWP4?3(cFd
zGA*`bZNv)WXE{30|4Q`
zz`+zhGXal6z6%cQV9`U@m7kxFbBnWkjL6lUkH59tq_Y~|pXhFwF{$GMys#C^_t?0&
z4Uq!y&;>p{9<`6+KyU((%@-IfP?J-L{5C3N6O=t)CZU%ntra06xG$a!FM?g
zTsa~wjb~@%EF5a5$@a6PXoBm@z^2QOhS_f%=3lpW=SrnG
zrg2w~Kq7m(jUm%|5=DSr)4IDJUI){nSa5M|aBiA*dZ?CLr7WK;KSX9=7%^ILp-g}TeROjMc+c@&WiR?rk
z;KBjNqWFYga_Nxe16(ET%OoZDxpl$O^k?q<$2U$;^3yHU)#bxT5><_(F!OHzT<9WB
zLJb3C>;8ck{M~b$(OE|)BottE{srP-fP8$ib|pxaDSuD199p^3LI3pPmnXnl%}2Jz
z({eB_5F$XmKRjiUWO6l)j1sGjfH)l1@G7Y~zWnOWWS5=;D`rowVzU>^%NvO-#P0~e
zl?OGJ&#SrV^v1U5-Ot{5SRc*|P<@o`U?VOqElu1ju^T@T
zgoyZW&j3U-86E#PhCIl=Z*xZ#MSBufX-Nv3z@A}9wU|_ed~3jHY?1r(K@JXWIB%pf
zDewjM8>FN6>bttk@LPenhe@x5_q$TmRfojQW9oz|xS%hIdewdbpM_yCc~8L!8gbFO
zLo|Wc-P_~P6wG`R!iKN!KuKg7@Yh%BH(g#i%;s;yPcX5w88jkUJNJJA(5kp?#Bz}g
zJCPwa8~Gcg?Fu&k+xtd0?6IjzFaVTje~dZ^A<87OOL?fv-HER1gDd8(j@j%o*qZvC
z3_-&f*Ro1g@%9F%-BZPcukHlW7BVzNKTkej#rGKmic}hh~iaB>T5uSUR&T^#YbJctoc&NH+Rhol<)v6z6GM#P-*8NKp3t~!p6X#?|QOgv9&
z2p0e&;z~_2mt0$6M)%ARoT_d_$&O+6sjDZb?d)-Utb8Y%$7)z}&&8k59#@pHaV^bx
zqhHUoY7M+P(5#a?Ez?=Qc_RX$Fy!X}6hG4c*^t%t_RSlFF@^Osx3sW3n=>>4(pX}H
z2QmT9rO_CRs-gJ#N@yh!vo|A)6#Ol0Q^F)AB;Bc8rm~WbU?&7cZSz5QZvA1Nj&BWo+Pjz#?n*%ioC6@?T2sJ
zoFBQ@3DqDdyIlBguzcxlF%ys16o;&l6i4KzbrwSnc_qL+p<@nzRN8}p=3$_M@kVo8
z*{_-|#|*PyI(W6!1O;>!ykWFG1^!iKT;*QYkE>V)k~2BX051SLa<8_w&~7oo4aZC0
ztqP_c2B5vRH|ZxhR%A>3cHaSgZCzc8!CeBD0aH<1yCN}N;tX0%LdEbn+=+@B4;Q$T
zWy6DmE;^YGG3$~yLRdz?kfWnxEes5bi;H`s*4#7lu0Mz7_I#W-fo`CIELyit=hKoI
zu&LS%Xq-kIoZVnRLd-_igUH&wG~l*~!7ZEBSW{CY@Vygl2>iDz1fG2ZNZF0W3|Wrg
zGy`IK^2v9b>uK#4VDsz1cAD)3i1)*TtoRpKjxYaX6YfgZcmLtNcv7Nz^{wx&;S4TL
z4a&@veNeCRA7!}K+$#=%f~KY>e3_m=azMu*VI?s!lR$7MOL9(BLR_5E?%e@pwy5fF
zHO;dq9?1kWpiA4Al4bFYn>Vvm*8o4D;9A^r;2ni%Dlv`(pcK6WAK^g*&wlq|Ik}$F
zHF~$Jt)!7~=nEfB0o8+6ED%zf&I^SyO!wMA#Ub-v^wv0yfushAXAf*X7)sWiKvyWD
zl`wZSr;t{Xmod<&0{zQq$9)gU${X+EHE}JHns0~`0}vO5-e|?^oP=9N<$jO!Qa#R{
zeK9l)qqbo5*}VAjJ8{o%Q!pryX#(rXgk#rVmU*v*U7by}WG@d(LXfZCZq=@XWgO*a
zN)yjIN`aRG&X?ZzTab2wL>TkR!`JjFgOw8HigD4+5tY
zpu9?&et_Wd+F5Jq>rX*)6Hqnm$WB}EOIcph8Sjo`_oPs$^B90R1E8$>qZ~$fK3J8k
z#UeK)@O@_XyS7|ENGvC^DgtseP~Z6DK(+l%T(j!+xFcA>v`KESU%)AJHdoC_lL}d1
z!yC@RwgsQvs(O>V`$oJIBuJS$Z{2@2DCpB#Y--4zfe~`8>g6?0w9^V)<)jm7H9M5
zM!g$lw>tbPjF!&Yaj$;OfddCjL1@&Ad-AI|`}kz~y(p8_*3byUFe+yMeJELz!Jp500}YLe;2-
zhk^Ov@);@xzX2oX@wXyrE!=QeY~jLOVV6E>)DB`)0CP&
zz!UGzx%3WOOD)gc(=JPclk?7aWRRCxC@NZN6_|~Z{vmb|vc%w|X~`0F{UmX;F(Bje
zCtpONkNoIY=$?BD1h6p-`AW(BDGg(NrZa9GpflEF1>dCJ6Uk+@H&zp2JH9V<41jZcLH0+I+%HvK%<~aWZI-h
zZD=1yIeUu}gW=qAYVRSCGbe=&uUBDDKEjoI@V5JS2IyKw$9{9Clr!DiwR5CpLYC3%
zLn9r?c`YNq%go}%Q)(T&;*yhR67d-1;j~b0GoLMsB?j(yh6MgRRQI3pqu9?qwiZ=&!wlY9ZeprF2P^$1%pdXw7adQQ1rqc{{RsdQ3GeW;P}0}`?1iX=T=oE~`&%M&1mgLgjv0z&Yljj_xK
z0_4Dh*+M2(zBJ~aTesLI&@rYT?B+36A-ygLe&hA!r{ubYgV0RHZ1g^WI@0XpPV&hobAR3+=22g
z%*~gAjZS9K*h`pTa3j}M^w~$SCdP*HeNfK#Wn@)~f%p&n3F^uCoZYq5ZYL7sP(pD>xq3e`Tzw*0IC@|!>d390rWs)!|7(}dn1_o-iJ6fY{Jn<
zOdQOg|1nZ4!N(p)qKQo!$2IEKgkwVq>)S*hWluEn9CoA?3fTIa;c!R(0pQmF=GhNT
ziB{ad-GIzhxzeb42~3b8K4-X^&+WHw-)6;4+_Djwa<(KKLUs@NjKGlVCMRCdq(E6$
z_xixC<%V^2b)*77KZiGQ%*a5I!V$y;<3222xKQ!2qtQ02G=7q%07a^gu|f6fdnW6u+3BH=sSk*fxT?T;L(^6
z01mett{E8rf&L2^{uabNQ}8cXucokDJ22w$yPhmo-@#a($og}OxjQYE(W?f)Eq*r6
z6+$0lV>e?`x3aRzRPDk$1keYq#|JFD;FRR8m+<-IU-jvs(Yz2Bk$FdQk>F{_r(9gt
z16Dc}8!H?qxNQNT82qhZp77%1(Vhrq-|$>{wWO%49N6KtJ|~rlod_tJ0M0P|`1sSp
zA|jP2PYFfKe|^e3Km54F{`tiChZK>Q4PyW40P{~c5R6DT({@q(KCUtgalk2U!BZ(y$dzyA@6N4SKA`Gi(g
SSm-kN+M&FAOXBAJr~U`2^>Qcx

literal 0
HcmV?d00001

diff --git a/docs/img/pgsql-ha-before.png b/docs/img/pgsql-ha-before.png
new file mode 100644
index 0000000000000000000000000000000000000000..ce3718844acf012be72eacda84c75f1fbe75d004
GIT binary patch
literal 132261
zcmcG$2{@JO`#-v*5<*GHm_kBCB(s#tJTCJP5*dC7TZd@Nmq0XaZZ{O5#{V+f5qM^}We|_29pWIgRR5koxxag_~8*3-=#B)Erf1u`G@b3;6qM
zCjU!y?ZwUkg#-ykLL&0UTF>1AyU9B>eX~6o6x7_>KE=fX3h`nUp{6GkV)&2w5>fO;
z*E&oq%?}jbT3YxP%}1DWC!{{+n)y|$K5o7g3#!76Fr1;IqX4^NtY2Fcuj#?wHl8ug
zJ;_#r-%0r9V4(-g-`7t`#FcpRft3~M-Me=O1_yC9p=oJs42+B`GhHbn!7(B)q?2}x
z*x!DxuRn7A`t{>BDNb{JC?CyS-9S1azJNmS^k0*cghE0>56sM1u3X8jO)=%y)YN?9
z=;)`wp1qXWOYga+>KSn+m9dTH@3*`#Sx8#TZZX*$LVM#$B5rAGW%7CH8D{3NI8hgq
zm5HVcEuA+^77{LpFbGpx_2qnC9&eZv<$Pd5CdeG_%x(1dq6IX5rdCN+cXt=*%QrF_
z?K*GS``?x{;7Z~@e--|4Ps5skjXz6+1TGXM_2|*%crmwR2OT3LqoYTUo);3L2@DM6
z;N-+O?efQ*^z!n0-8S{5wN+Y9j>O*H{&v_?r;>%2(K!1iC=33UPg5(l9c6;AC(h1yl$6N3dwRmd!=HWp$geuFEoNn9<%sJg
zj*pN3^7ZTI5PG5IE9bzkmtCTl#>n7PIl`hd6Uqnhc7l;EBiYs8Cg?~PEjWY%@u;@
zy|P$YUt4QPm3`Y;WUidMmLQoS=fB8&f5`2s
zKpe4fyvfu>mMOp2laDe30#0!2m7dJX%Ayr_FHUsrduQ0Fl)!CNY{mVrl~g*8meLW?
zV{tXS#KB5gj$ilr_LJ!n`(CRYhlZyN6AjJFJcFv_f`U`m%-b9BbNzzvta{SPC@Cq&
zW@Z9hrY33R;cL&ITqYU(hdqLO{c7h}@fB
z_QLKP^TgiX-YJ_&<>j<>hX+--va&Lv>gwv?k;gU)CEm@;D=Rmxt+{h@bCFHBHr+AV
zm#eqDy81jJf!V{;)7aKFzqGw<)QePES-Ipp~uWk7c=jUDN!78UMM7$;qx#8@&e)9-QanI}Q8$
z1TpbxciQ4NZBzH|-SbgkZzwYFc(O1kn3|E%Fg(l%C(L4`x}p#7H=po&O+`GuIrW-n
zSO)F(y?esO*6ydJE!@~yYfdz+rrAaD{#|@~UC(I{pkuY3lOAp{8c5&>P|I#VDNqNi
zkt(I7rCYz=Dt8u|-W79SKXL8hw@IDe%lC
z6DYQKY|Gfwha^qcmqwZ4cYOK&y(yZ{wAeZAp{Z%g#%_C@C}wJCc(^M^=NeKX7Gg04
z1u6?J8A@>)xn>VL;zQu}Xa0^8bE70CCZ-jzIORM)@a)?+g
z)4=tPJK=q{h?*Mlz8oE2az>G{@rIYt5}qQ9Llr4bIusNXs86rXa%u2d5uCq#nYeCm
zl@vBeSDHLciIYDWy?4
z74xjk|F~8%3R^;RzwZ9$Bh@>&>>G|VrzEz2z13}a4{r!%c6p&}6jQ@+98+1=`t|Fx
z>};;&loTiDwzCWj!Eo@$>JIm9bH|BGEV?dynX*q!O}*b7e9qlvh28Pc?*oz42gir6
zCW%jQU243?n;wwSxqgAFjb?#Pep~wQ?G*BUeVh2*cW%CU9Q};okt*w{X
z63X^ZZEeyhP|xD3i6}n%B!t~>To9%8IJhvVoIr5x%9ZoFCj+FHpVPW-%<<|DgocK~
zCYO3i&bU1GnXvlsU{{1IA)(>iS)#Dc2
zyDswsd?@s=TT@dL%Z(f5?e!l&ete!_gl<}VMo1*^n~RF%!wtJ5Qdrkpl^;Z$XkZc?P1WPmx?&
z!J!VfdbKx}pfnFux$qi#;?Y>rC8&xFH=dw&H<#{udDU7NJI!eG36l^_DnY)7hko%yeBC+*%*t%H+K3Cg
zk
z(o+ATqHC!0SFR`qTL^K4JUWCmJ55g?1ki6Hw`@eZBVK%ba?&?TJ;S8lk9eTYTN2vH
z^7^cLN`qRd5(h7D{|`A4$di)#};7rKNui&;!
zT^}E+IK%ybANA=g@#OOIypDe0;Zbt0gPB?{aoy@(ig({*ld#LDF;0t&+vmJJD$x5;F+G`I_!fQib
zWqd60y?xwKHY-%IhN$zx
z!av4%*0pOb`cL)z7dBF)gR5@rZX%Je-NJbFYfYU&QR&w`8@*HatRcTTcqoYG@WVli
zA_LQ@z$L>I^;jpXaEA?-2L~RsEznb6rW{AP7i)C)BL@MHIo#^cEiFXM-g|t|kjX|u
zOG*R*6r)gg!#Va=*<*AGyTG+;0H7Ex4QgvSs&pLe?7Bqi
zo^CGE3R+U3@ROH+AIv^wij|h=Mk@&vuP1ZGOZNov!IfKD%>Hw~V|jcz6XhJ_mw#s7
zDyNGI81-k)DQmria(x&1qBp$=M1+!`Iq^tKk
zpL()gx@0<1?FP8>7T{Pwky*G{Wh2|#ZFYSF1636jA7-zun>O+GueU-OMJJkQjHs@R
zkiEFPsr5vE#NyP$ooO78?Nz672n0opfu>%W*7O&g5ZZlfl1n
z{qmaSc*|Ff#o;O<6kzrzPEP3e(}0!C+TK0#>@uX;)%xjXL9%^MOSr4EJ1EbU?u}t_
zh&=02IO}9rYEJzJ2CMd%sT`w1rFuT4r#Y}Y{%CixSH^=bEr^D9qJcuvPdZ5onr#7e
znwr`%p=;L~iXZh{_1bfdh>E(%&VHPNSwcha;b^Vr4JRBGYAhv8Le2B0Q5>{U_w^Z;
z_V(y#xH4h#;F6LOUHRvahbvilc*r9oBg-9U-!!{ZQ&SIAI&m4g&$30w#60?D+m(E5
zZ-1WvRc1TdM_0Of%>LfJ6Nawi$4cDPvZ$UtduG{VVrr_YuOCc6c>Mm})(YLvYjDTw
zy!Nj?9x9J@u22FZr6fo_2%uS`h5h__6tJjg3hX@=T^Xzo4ugd$LWG%T)s+PN*46aH
z$Ygm<&{BmSkE;I#54?_t5y%Yg@66?IZB2$D0Ps9NKjrrJHULvCy>#G!0Oe`;`T6fZ
zdKBp6g97Gw1SN5>Wtz)-sjap3)R{8~bWhPMd#qhy&$ha{y4=F-od`Gb@#DwKc)Sdd
zw)PLg>ZS}SDJiF!nDVHnZGM(wD{YgKk_HM*U*cRmJj8f3=QcWN(}(Kn>!lgQTu-U0
zss;oGes1HfI~k*J9C%wqWTY|h2q3CNA})(LD@Vj`+@SsZ`7?sx@{H<20a8&>Q`cj$
zXW=Fn6cj*l6A5JyP8NJT-I35z;V|8qt0$CWP(|Gm%6JQYCMG6^dFLtsDZ3Yb{0NZbDCcb)
zXXKRp9=TG{Pkr5Q8cD2rK
z@_4E3Xy~ZduHv+_yu3W0?TFZa^aQ`6q9R<7CzK1Q0<&nAU%!4Gy>#gkc8NGcu
z`0m1BiTZwNQBjYTxMz(0A)HrYpvivz{>cv0?I=v8YH(1{Ej6__eeX^IC>IS9mxt4~P6sBvxzXN;jrYl=rwf`1#jfjtr55ca04$*5&l3u|;j;(SWCK`#k
zbv_;iQurb&iU!~udX0pHWEtOYWD2LJgdwsrE$2Z4mx{w2D
z?5)U4+BKg?Iu~)8tpug6QnX0Ti9mCkS(zUbcZAc!X|-zwM*h{agosggJ0cfqzC7T+
zapT6PZ{JSp=;$1;vw&kL17&m!hkFGm|Gc2!S!k!(DFdh4xEjgO2x3smkH
zpnKpeW1UI2@TYG5Bi2x!Y4N#mAz5Vl_iz6~(-xDVa=UW-DUvsD--bs;5mo4i0)ad86!d(wnF^r#;4X+63?CWo7&p6uyuNW8v&2iE499Ch~uy
z`P=vx|5K)WxX}l`6`FR8d8~b%af>8$_WdZ4rjYdCdFw7eT9F6m|B4vkLlzKa|^{N+;4w
z%JH{v-h9^X#K^$VS>a%g$Xx)_%RRRFDbJhy+PTL;@>nL-0He8!p(U5TIrOKF25N+gkA?Pu%eesKkjb+=w!nL{a
z?Vpv+6v;iSa`8^^r>dpepy$tzoMe_@0BB9uh6sBsECk1n9SeB%3bPiV){w5kp>f#u
zfFIuSQ*(1Vlc~lvW@hH{iV7Akt~OR)kARcou+8CmlR)aQ>;c+Afhu=hxsa|Dr^~E*
zM&{O6?|;QH;JYFtXn#e?k*?_>b5bZd`Csskw`wom{msMx%KAm7o@
zpqgJ;C~2?rK9u;~kr4Ll*MktZgM$O@vd4_D?&-1A=gtwM25LP;ORaxg&~<&fG5^fh
z*SF7!T{%9uL-LT}@L;p9WGVHu-v>$4zH4YPMK`g?4#r219swYX=rfD=UObgOBshiF
zyyu1Tn|4VWCpP#?n!ZO53oA!XP7b1%fW-c~x_Z+~x+D-SO@QEGMi2L@QeDU_J
zy64><4#g{nQ`6H{p4*FabEGIJy?qvic83R^xj$cl4wju?`^w0BAH5fF-DG@x9CO%E
zUw=MQ4s=qTFh0t=aAr=QIYS5~x28Y*9;^1p2gH?*v(JB)*_uLCN46c5vSnO$e*Neu
zGc>$LKjJgD15FboJYmC!4Oh8b1icE%^h;k~!uo~=KG)^@P`i;U1aphU;>oZkeHU!_$S5et}B4DwicDcwyKTz0&g>
z94A1hjI}t+F5QvPiC17$R%ceebxvzKq+yUgGs;PNA@b-}keI7Znw{PA!HqU|>
z;kvVa8TAR|nJb|H*ik-bxim9NG>-JP6+oFzX&_8=_z>UPD#sx1PUTZ>phqI^E!}_3
zNlejGO!0d}zn``?j$$3&;ejYHH6xD2Dq7KTXcbIT^+*c^q|=h$saP9`p@$o0f(KLf-0S
z=@8!qT|K=NKx516>$kzH0Q!)$Nqa&#3Z?|ur&VC+pnF;GNeEPCRQd}KyQ$ek$Ew=a^|f9yD#X>pi_ga_4>`5
z=N~>WsFvo=m1~%T;RIF)w61JJ5%i@?H9pMLx}2Vm-+vhrQf0-Ak`v`@;K
zQwH9~ddh?RgK6V%adA0%>=*zvg3ZO@hJ~MH;c;;WmsZl6Z*8M@w%4ZZRwiU>`p;a9
zJE!i|r`aC#HQH6e$O}g|+87bh|7kk%-8&4Yo?~uLME-m1Wv=%JTay$+HBV_kRl5rn
zp|{ct&Hvlo3b(og;v#Q*ym%0h58u8j0C%WF;M9Og)PF`nAp;C^gyh8$ICOj>p(WvM
zQ{c&fRrYDA?l3Pl&qFvaHWqW50-75l24Y5G9R~`GeGPecoL*?rPr2pBO9r#8?-3g;
zx$XRxEUeGY;|a~kBCZ)BF;?(8^=9;izn0eayFcCSV9+whr_y3;O0JQXJ?1_=fUihq(P&Aw7^6T=PVjlIRBgl8__j}s{cf4hZ8&Jb&L92ItyiY(q
zL@c1L-Ao~YxUUcR(KJ^<3c3l!_IG<+(3daw?xlHOm!jI;Q=Ob#SO@`$gb=0l;f8T%
zWpnzqq^XSBscLP%=AzQowQc_P2;E|fX9iWyj}#>Bf1RM_(a&^>tE+b^5-~g0`=iJl
zVIT+lySI3v;Gc`);s!Q4rg>%QJo(-S+vYdt(o$0ag`T;hTm0z_n>?Zqz?QysA^biN
zNR$uksvzC*#5V9s$WU}=qW~-&O|0R54^EV|jg1|UA-@+dZi1($`2ISEWI}l2-
zq1)71)F)8()^+bP|0!{pYpjQXrwyH)pg??$o?BQ$IyyRz~Wy5n}Ho=hsUACT$fRZ
zVRPby%F`d{VFX11g$3qD2<7(ng9of$TjNCB29;DumxK=pu4HG1GPB3#unPwNf8q~$
zWV=lDq}-(d)YYpq7X!rjJj=9@^3dN|02Ts7Kvz_h(bS{{vtw*x;yJLsU|K%ulpr!-
z7=C_7fx-dqJPG9sVd~(nj89L$gnb%BcNhMGpjF?eo}M!R`pTb-AK@{qJ^=(6Jc0H3
zA7=n5CHp_(0GQX<++066NO$hsxf|XGVxYntsNc)yogsW?6#;2yzHg^>FD`&y&qcLC1NgIpQBbxuzqs(R!jzr6=
zqp7K^&{IrdqnCJ1&4b_u8h-uI5CaI0($>~Fy8V*O9?Jy&6wG;Do7HdMo*D62Q~Z`7
z$(*5_(Cjkm`I7pIF5=hR@Z1qV5)SY!nlqP_G{nynNFh#{Q6O9>y?Ofw)u5-+(npSy
z(fPsgfy1V5^#QPPF!&?LyGb7IImC%K{u~&%4<<5VjDvq*2M`Q}Hn?Q-kI9!a7<4v9
z!N0ux25cn6LJgw6qPF|W^VgT~XkmwR3sa*1TLaL7y9fG`YW*3E_u)Z7Y3ZHEk9puT
zr<7!K($mwIj6(TWy)wPH7z*?`DKSyXN=vT8TbSpLNYMY2k%zOMC$bd7X!!wEl3#X)
z?k3$+^4h*uVl^SFO_}9?T8-|xyISRs{h>-+2uCssTKR@Zmx1bjijGbf)S=gNHcn(U
z)e=y<9`K(-=)tE?Qu8A<^pt@|g=|L}iI_bSi`e|NrElHhb6M0!eF7aWXM=JO+8VaT
z4DSjX!W6KW3I|m{NQex0G=E9&239gp6vxp*_dlWb_Lsb0g4;YlKabKV2^c9bCam3C
zZ4+7G?2b@~JdDMyP|dk#Q}C_&{vl;f;kJLt4JdKWL>~kbOi
z%NH+>)$VWTl|)%cp@je_2!k#ow9|Cr)vX01H_TSeyVMaGKT-u
z|2Lx&cHU7a8W9mxAgQG^^Mp5EwH~+%umUmBfpPU^dK|$RO7yza9k$Z2Kv5CdP4M>m
z_7>Vf_JL8seFDUrVi0jWk>If^n>!vD-u~Rzw-Ao#-e%oluAtb?oSn1taTTn1G#_sm
z8QkISlFq;Hu=*~SzEI>9@la;TJj~WMIGk`JUx96ULsax4FR!wxI7|Oe9NjIqIPf_j
zTOnXOqK^&vg}{m;bd)y0bZ~)49UL6Mz*fyQV^qP)yjR12wBA{tg}g~KIQiLf*N{3L
z7N#7+`_YOt9}+wS@DCJ3tN`?Vs242Y3nBJEaU?r7hVxJmJ9)VeugPpOMdZ0w)w$f*
zUiKIJkn0|#$#Q2&o4=5mZbt!RJ-~TkFzMq*wax)is4ZIi`8gU0P{$-3PS-6LBhF<4
zvk!*HFFpfk_I`AkMOSij0kedMb_T_Y+^EM|h-Gjfj>o9>B+Al2qoS%R+pyMVpx|?2
zR7QJy`w23#+!fyQ=cP==&0HbjaNFMg-(!X3Qs0(b&2{h$n!(z))W7pn3ipQ~
z=~vh{B`zy0{L7I=zt9B+i$p;XY_lVitvzeKzk{di0iM@A7A6?u|MBcgK>c8}_4U2n
z=@g11R_t$%Sd%^xlEF7+ZratG{~nIEFRWv{gy$>BqhzN`dTgD7x+ViY5v)|M69e6?
z{IU+KFH=sS(Vl|c4#5Es9SK3bLgE7m#sdy@`_7$uC=S!xXEEq(O+*U;k)+URP7?t=
zx>0T4zoXHo1_uZ0ySpjjZHQ1Ft6wi!8vN$0+EdevCLvai+S~I09d~kds*RTa!3kKh
zt53$HJ_tJ;e@Vf7-9k^}utog`?(RwHyfTohFe@_{2S>-FD971unaPngz=U7Ee7VZ`
zr#S+GU;XQTb*-)E`BZ`wAIc>D3Q~L%U<ZSN0Jly!nZMbfXQ)TJO&4>si
zyQrwBxxGs9e~j9zH&&C
zqerP;t$qCt2&5nkBMlFs%`-oWE_rx(Af~%Q^i=}HHJ6rt-tK62qOe;>RW-cJE-5|R
zT?bH<=;yO4JAj5~2czu`Pnb?#PTjNSj@@x80tAlofq;rd@h=hgbq*jGh$k{IFksz0t9PBAiuNOxYF>aVg?N(%7IPK>dVtQM=Z}GAguY^KW_ApGokUA+fu!--$=#*87ElFG
zQ&IU96kOGmIww0nnZ+GsiFxo5{+}}#=cjKgE+*!vX-SAcZ-I2r?Wrj3D(76)?a!S+
zoeu;5ZbZd=nPH%-mnOoYLA)nGY6O{*+?-sz#UU+;zkdH_XN}jrTqh85k2U&+-34fX
zGCxbRO00iS`s~OXc!p*mn|T&88^>LewDk3J=(NjA^P(CPJ4lJ9qP8ko+1W3Ef`m|j
z*DSZLf_wx?K4V~eGu>&V02WLgD=p+^gX!`BcW2c~Y~b-tK&hw`n*egq
zv&Df#ynL4b6016*|HC>`gA7fjE~llXWi=Km-RUz3rpS|@lLpH@@
z0|O0>2wZt@sdr3F4B|k6dxXeX98@I5pt<@UC+n`PUd?tfFxKkV=sqWCT~Sei=u4Egl%8#X)!}3>eOy60;q%~Isy9;wgq}O`ykx!V_JlSqk%0wshLm(
z7+p8)e+4w}L3DEu)HIY26yQM0-w;PVed?4Czz2`@uDkBJ4m<0QQ0T~)4<9}p_-Gb%
zhwIGLmLbG7F;6LABR7C^fjU~o!{%hdq-=I9!v6grM;jqur8c#F&G)vp1|+z05Fvit
z`e&)4=?KUt1SdKuq>(-x{PjP5xRO=_RKP=!h*dyp0?rcR2I2=HBI#IFYo8<`sfRES
zx~_@#yeMLc^V@(We#fS5a8P$nBBuVmZ7s2umKHcWnIXeTV(}oof-7J-hlre@BVb-W
zENpgD(Yy*Fw9~w`Tge0p0?UrbK7~mhWOkiR
z=7H9Y9*#oV07$_lgB}3C*XQ~vPzBzHUg#i;q;YIB92bL1N7bf`%b6X!nuTH0(4q)5
zLcXS2%~!ykMoEEx*)>j(4kF97+^W1S%ok*hqcOD6~!lAnB<|YiCruX=|xXU5~h;_z~ABRF@_A~qj)G63iz+|P^
z-)7a0QD+~q{O^H+%ZC366c{^dx8~o;&CSK?a)J|21|m&nP7b0@3iK5f6np}~V+9U<
zR;a8KR3)FfI!P2l{dF5OHC0~t`B|Lgyb>vg<`s!(yk~y9WE&(*^wbR8Ca2Q-Ap23X
ziLuP;2HTHNZFjt0UCU4B1HhUa3xJgzs0V%tASR&-{`4)ITu;lYk?Yj``9@kb>-)Q-
zhk2@jIB-uOF-V944xZ^CYT6w9IbHF6fycHz#Ag+z?ZK?ZtQacAixCIjPF~BeL_(Ij
z>e_mG*`-MW(6vAUgd3rj`(7$(yyQ{O9dnsf&?$a@iy?$<4_o*?WKu;o%L+5+fLu?;
zfZSwx4(gMB$kVFMPL-1kH%@^l2iTThMC?;>zh2VzO$#p?y7*4~UlyF@h0Jr^uoQd`
zzMMjP&&wfP6yZ{ol+O{3UQCPuUJeHYqBmPDk_QA}lEW@jHZQy9Ux`T7S9JXVY#75f
zbcJqXw=u5+0fS_?2PaOR^a~0K!l=o-2Y{W$^zlwG?KAMpf;O-=PN2LsLk{wFcsPQ>
z!Ia8uE<|D#f}DE)FUbKjGYI6I(9qBbplpFE)Y#Z~ACJQ#))|t^uC5l_TWMkDxN_wM
zI02C61zjK%T6b@GF3!M?08O!rZLtSkxDm{?(`V1Vh>oVM2IdC3JJJSqbk2d=6~H9!
zo(P?Z?yNP4swc?Fo59St+uv~l9)={9p;W-q`2g<%*Pz5%=ZJ@m)kyI~rPz+v
zB077zLX3%{HZUaGsp`wC2@+EoKs7;Syq*1tM%}`K1FBNe4kI(Ok?D))EOa|g>hMyy
z1t=eY;IsOR0Ny}u_5te#fsW1|Cu>IfUPAH*VpezYKf$_}Lbna$rkOH1X&#T(hJgNT
zJzC2MuayB;O}FfE2-Jv+z^iL(YhA%D1zc^0ylK1c=&6qT)m-6ceOhdbs7>Iv*5#2~{~Zc)<3XJlN`
z%r^{y?LG!^m2!t^a>&FW@COzp8Pq@I`~r#rW7}ZT8PE1>?_80glm9PCmhqS!*LM*S
z=b_?|((v4dCSC}GAq=9q0tQ=^2C%Igmwa(3!FJ31&p^|^CK
z2Lx9kP}0@SjYUL6S1=#I090Q=I6xT=`w+szifBQw>uu|$M9UAS!I{9?sd7+35k!J0
zc{^`*-BwmsP#8`JNOFur)&lrSV@u2AP=y1Aw>Q!#^i>iBg1`-LSfB0T1MCeHI9pfj
zG8M^Cm5XvdO%ebYjKuCUjOlJQqAh`)4RB~?r~-14!ougE>VXN}g(VcbP-as2Lzs0@wwDHF*oInn~;E7a~!Qw}V
zVhxCb&!Y2u3H5W57t+}JdnaLECQa?FnPIY4jclf-zl2|W!NC6(4iIKV61AdB#wGX*
zdYpo?lF}Vb&3ElnP|y)ow|cde3osdy+SgbEoT~W!=TE%_1}+_KZ3_i0=b~d8A%g0L
zhM_JaZbV>VAQmR}ouYz5W8DEhpRT1<7(PA(z~H{Zu#i7Al1ZSCeD-0zX6ELmtr3?2
z=3ar1>7FPW~
zvl9rs2JslVsGJ<`8m>@8$6Z@;^5E#q$LcBv`A1kOks#P;gFP-KBT_^$e!IdWrkETsP+Kc8ovlxqTVE9dRMFPa8HdUSC+SX{Cqw!JaE4#+-B<9Ui9J;K
z539+X%`PJ0*ipyRn1H=XZBqru7K!9Rh7Bn5Pu6C^^A|2qV)F9J6vH(*n*e(q0r3aG
zm0$&l$bYgu81L8=QgvU%KL)M+2$+a;YdJYNNKzH)!-7GbeC$-z$s7FRH%C^dN74RA=Ij08ggMWlBemekPD5E(oI_=HQ<6N498>ZYn)
zM!F6J7sp^K12(da=iR^KNwFN7QiAu9_yPoyK+y69q6Y>^*7rIP^v93uYiuUKG1
z3bMoALz)$QvI5&tiO|`F8U*-bfEy848{!&}
zXvy&%fc-6~)NE*b$jM4cOhqzsfb4hpQXT%EGk|isqo-j_|M^Nz|G&hI{(lb_@%LXZ
zgh<#Ed;59-li`t(X-+b5Ma95XfvuE#?w0nM!3qzS+CYrr>(`s$nKtu7ybs(R6hc@u
zG&OO-IaO7nNL~lP&jpavZFFrQ%T=OO2J-gd!G0G+S71jcrKTR8ot*_Ewgq4e%nl`$
zdcYFG^puV*sCz0J8c2X?5}0gBGhyda`3xn1SCDy;kAgzReDv$Q7(sx(ZD&_#DQp@h
zZ2?8k?iS&2B9bV@LwBqBwPKcCq7zRK2!kal0Eny@FM0=kx-+-J+7K-s8SMjt8W0@Z%XQMs3#wNG
z(Dh;-b#mLm$f40U;`B_
zm`R*2!D}ZgG34dTm)Qptq@=zYoVkS`FY@!NO%sB;tQ=y*0%&C%*u2Oz7Zna~4OE0n
zGsg$YAwm>D7@7q3<(r#(1(YBPxO~Wx&J2w}AnWzJcgI0IGdDMfv-=tR6Nt;+hVzPq
zfa7w7Pm1#(Bn0Lyka!y$TvPbj4TOPma)KP)1>z2fR)O|#uT4XPvS?%5VGy;XR8Bp8
ze}3K?kq*9Bc^}p`uS$39+Scv+!)(pKj~|~QtO39o2`?hd2sAbfQ+bm;uJfOaE*QJ`x`;Q;z;a@})
z_DA~9z)*3tkS!BbbqIA80@
z!Pb&)(lhn-U-#bp)!wpJ#R%Pwn9a8XVq*o7Yrlj`004vW(E=L@`rfRb@DD8X6i%OI-jnYUmtUG6siv-8=s2rZGkG0`3jhYN0MLa>uSL3cc(8P6X>bTEY70{N=(;Q1
z+*e_>zQu{sAm#-CC&%WN*OM0_B36jV_Ey5!NGno
zkpnhwM!(zD;ynQfH}wQ{?!O0YzG>BE_IDNl&JU6q6L$EW2rUTp1Ol!g(VBoq={lEN
zhJ@>Khm&X>3?2nQ1Pw_qAif=l)*@VqAL8ObEOS2x-`zy1a(C4`eZ3;j~7OtgyhsJae
zjz|(9Ki%nb-TH>iP@{-c@MdpIiI{F7m>qiU790sO6cECfgEZ=L{!CFIout6sMENuY
z(ai1XYPb;|Xmx3a1`W)9>jlDC09ylw9uO~BK%iKup$$U0z|lYz^alfa9E60Xc~VCeds6k0xiy5|5+2ZHb+&HBhjMi~UQ-q%=d0;kPI1JoxFe$j-cIC-~g2Vg{x
z0r5i}%)8&CH|sw($Rq$C7%&?8lF<&NG6E}WfMpY8;!=J9nK!LXYB2**9p!_}Fi$iE
zB{!GwSy{Kp%sW1mL!@mOj8mW6MELaP1l`|4DD<
z8P-7Dh7@d>&xvGQww(xysSJ#0n}V4onBEGl>4p*Xko=ZN3bw%=X==IcBL#?GvAlfB
zrV2~MSyEWo6>a;)Ze?TR4kU^RG{~OIq3L?;va;d&A(D(C>Z-IZQ5^<|L7)*g*W4Nn
zn_v#&mOdCeCP!g#a!{ZvB0(d|wOE`TQW8MWA;@&;NO@pt>I-2Qf~$(hG!(b{BBm&N
zNW*X7=Dg}Vciq&jNH%p84HWw?s!JXcq_&kIts5r*G
znW|tYBJohv7(f(FvH1=;^AjRcb_tlgV;+M!uO0P%)hj|PtQJIy`G`wWf
zAI#Xz4pkK4y!mztA9Mb_IPheQ=P@zopnM)sK<1g9SY4PW0$M0oBMOx11Agxk$l@Tu
zj>-&KS*`{qimt=APDR2R5gd4&-kAvEtsm0&3}^y*5gA#{%>2Tdd)=zFES>w$PElpv
zJR`k0t(bdx2(W{oL72k+UdB*qR{Ve4D<<;}mXMXip!Y?Ws(nFfY=A5>5J0do-de1%
zLNVb_({1q)aa(1DCoG%~zjqbJB5ST=7^bC#ukOCRq>M~_qbhAX+uGXfetkKO!VO>b
z^Yi=E)N~Bu_KlE?aPofykxIaXx!y1dPuFHtON?NBsLM!L3^;6`6*KIm
z!keIfK{_KI_9XT<^n_R;+q@m4ZRrseC_~Ls!ILJOPT}qAEPZZa!NHAy*C1{k2x5FW
zxoJ)wU`8x&WvQl0^E|xILUEoJ4aJ~zYlr_gNJRqXMq!$rI@3VMMIWF$^5qm4$~i6@
z(E{DZww+-!qY($@aYsidrVp%Y6XskPsw5)gJ6v#hJ?f*=LTwOL&i14;B;C3q8IK(OrlO
zAerXL)z(E~ByX`pG7IqO7O0A#6Twh)4*vHFSd0*FriFYZM)%J4)zuzlN$fz$9Vsav
z%c1D$@3DfU5JBpS_JX+wM8byx{~X#Oz!@Z9r7XEm1<7PhynG%7&@d3UkoyYhq(&I<
z5q!tDUe3WB54ajew(?!{F=qf@iSUI6T@H+CUuRpd4r3J6#3wF|k3R#a7;#2n+VBpf
z9%fc@zJkaBN$z7Hs)a0S1N?
zK9YEuktPkI*JH53fFeqRR*p#2kc9e*1mUE<4=;hD6Px^}xJP9zUGf`27982Q>D5f<
z*7nR#s*8`tN~>3l;N0V>9S#p9pSmzH%0wO>9%6}gW>4MwLP8uw96-Rv-E4gNRNxI?
z%G0+U9Z@n=syCZoY(FPSp$tC4bAq4L!VlcQvtB{
zo5ea_nq}kaXc2aLleaK}$*VLd8$oYs3M%)igHMA0dMROMkO~c&
z6IEi`LT&@9tgNgo-L;3oOztx*FV}Xqw`U=!7rEZjGp6>nY3Gcx7$EXgP_$>@HeM;3
z2hrUTZXDu2g510VkB8{3aZ20%pVPq~1sDpC`JddG!+#YUcA2caJ
zlg1Onb}fwj@4wSYdQ~IFE6^c2MJ{~|^tDy63!$vQIB(iqUX>E)+dI&7@B^k{k#5po
zBlk@df5s_k7!324#9Z2tcXGhraEQzv-39DsBSCZa>?25>k&A|r6BDa~SzqDy_4{|H
ztQvbz)?`0c*$xTk)$QQ8s)SOY<9}#3HM;>xX6H1~BfP^Bb>e-qp&}Id%DYXN@^Wq1
z=e<|ThLRzziGydZgg2R}LNEoJsd~|%{K@B;YxVIs5!+l$rT&B~fmnWJ-ld_6_JIJ-rkzikEcJ(AqB*>Fr(0t-|}X_GO4yc%r|n<9odicSq?2EM6
z@ZQIh^L8Q^6#|h?Fw$X4;>{21KFF0YUe|dL3E~1QuBNktH4OSG<<}ilzT+{#$i8LI
zg2XkhKR*GEYw$yH9w(T7{3%ej3>?@B6wg=)?hR+e>HpyE%b5B0un4LsXLXE4nCNa=
z>1w0bion~i8slPl4CZ9PFjr0J?7pVOW0W7K(i@h)jK{2r+}9GrKKgKT|xk!
zt92i3%&ejb(T;wgb8NB{;_I)gtLJ$wYXbJ*+_Y)X<+Qmh45A^S?;Ldq9EnvHKvBQ=
z=Fml1-9i%?v!dB|FsUWJHRijs+(5x5+i&0!2V$G4%Lw50xgX{USkN8UVXQ|HypjH*
z4vD#C^i!=3`B^2+$%zU5(E|7M%FQ6WP0Z?JFYVF3tdr)UMYxBaxD`mu=Yw6tl9ehP
znVHE^wPybCp=DRH%oI7)Bu5cuh#SGO!Qwd+syHbki;|iOhu%|6GP08~nrK7i@~7fU!ELi5*|iS%&CGP1
zHx^x&Ybl6-I`>#S@gx3`vaJz1`G6eChi2NS&zsb;5qJ3-cf6X1M>R~LvHj4buBoo>
zg-o(+w_v}%diuhXIeIc6voM3g&tokw3da8i=s3u2gom`ihUM%zlt+Jbe)F^Dsf!Gl
z2oPn^t@Ut=*#^p|H~T6)d)EwbP;-wVcpjk5zZHG2X{)j6CG34rox1v74R~-AD^#9R
zwYi*mJI%N0$_aVkJR&CsNLuG8%xLki!IX1>+>XCxPkKn|n^se_V2IW*C#U4xD!7SBzlN8!!tw9isgTSDmq1|5ZD?P1`WUTCCs?(nNRr^ASd+Sw_}7E+ZG
z%tST55np|cr++SIr#)sgB{t^}eHS_PjmW7-nd5rmM`|(B`-~`5B6kK8Gh|5vfXlPX
z;zsMdV`kym4hZgjz
zY&HWV+W;lv%{n~cL{mN%o0Pm-ln35K5$mi!{o^ODi};0~T54a_6T#x;bn%2mhs5*{HXz
zc#S9e#f~TnC8eWOX0(=YxOW$_p07efd9HmQ>}Z0RcP}KLsv02j!j+YJVPeMV08y%d
zlwhFvjZN_|5gQ!+kLSDM+rZ?b^ux4zEB&)>Z$)<;hd
zNXg!EI9{wBQ~&%J={l%6@^2tMU@JDI6A4pPj##vo?3*qRB~MkuZ5=Jh+iOq<$-tq(
z!MnxGf|f3)r1Xx}^I&^A67VkbhWsONz4%La|8z@nTS=XYrz7qk|D2G!HWiPZnx;IU
z&7xFF%327nYMu>m--Z`Spls~4GQI3?DxC@qiz94SVDzBnB)7oS0yHKQD
zo!H`h6l54IU|+sT&4_nz9hZPCbfM@-%)VNb#yRVg
zhiJ>YCX>QPcakf^in_({H8j-JI1mHhXms=T5K@S#ux&%$wGf^$CHU|gt@!Gfi*Hh)
zs)-KqI^oNC^7nGM5IJ-5axc;}02f~G0hi0Jsj)E=2Isru?D}8stLZ8!g~A}@q3lkI
zY*y>~p$6l$jT5p*e0Fi&PT5?z38WaG=?~BAJrqk5!IhzNYY#`tagxzql_H}PP;qy!M230F
zR;O@Y+h8k)Jc5b;9OMB6CAOo6NJROgSz9s4#0J%_s(|KkJsDwOX=L!3Hb>~>U)xQ~
zvoo)Y{{Gv3vkm5XGLgpxywOlmNdyO05mGIX!0droP!?ETpb5!Cu3iJ0A@q8rDnZq!
zpUZ5YoL7tA8!5B-nGS6s8{+SD{11?_kPgo$bY)G5DowF<8v8^b4s(X&QQ3Ku?(nRR
z>q7N8$|VxZp9m`D+r!)V!IymIx-TSHCZUquIJLq7D|)&u(xDR(;V9b
z1uYNb9R{tl-(1j6LVK-#$wp`Kal$pcrF8p5rKFnuzK-tPumvlx=P-xDN97y35=&p{}P
zKYi-gFAI1&7acqZ4vwt=h;VS*K1Dw+)P~_J&azb<8uYZ+_G%k1T3b8QJl^dN2xcXD
zwL6xkx-b_F@zrc)iS5RRF*v9M0AFj;VXJU)aNJWC$4A@=p_}Rwo1UKL5*5`03%(P%
z&-PPtaG$$O=UmW0T4OF1roG8X&zX_!&7TEyAI6_z~*EL65eW+SuhI<4r7|0-?A|M+9$pQij0+JGi%sHR=e11~NBDDBu-OPVHP(7ouR()D$VyS!!`>@gSGj);q;$dT|V{7X6
zUc7cWYAn29tf5phIp{H(1$HBEFPI=l|gD49h}nFsT0S4zd>ezqiz#*;M9>l*GtA+Do8e#H*xG&27D
z4Mpzz@Bj8^EXV(!|3>HFN_7-yg>XG1FG|F^DJ3ZjF`062{aqDyJesPGj-sTjmE;p#
z)LNytO9h-2aKapny-CxWv|`@`v*NE9Vgf4;fI(BPone&HXZgKe37GQ_OxC}HK=3)t
z(4JH13j8z;@-Lpr8Ca7aeQ|L`iki;YaK6DXd}RS0m9Wo#`saH&!lmMqsBEY&aQs!ic6*mXUqM5K
zZv{;8WCj6UF_W^$hJL#yMpAF-tM}5tooB$J20`RxPA`y<3+Sfz5|KJKKm%9E1X%bKvgd_+a*{*#OgHdHuFm-S;
z$gvJ3omF7iHxV7WNl7%il%^}ZYwo2rp#5~gSHv`f8i<8PTmZYIZ}aml3MFKOs(7wH
zs$u}+V@D3Zzj#$^m_fohWi=gB3O&bb)Zh_?!kRG~XIUolqaS6|Goq*L&a||1W
z;R
zEWh$OJ`m-DLQRhOfDZ~CkbzV=5)j4-`z|h_$AjJ#@NRFip9$JAPBxQi>`|6N3TV15K
z?F
zr>Cda$9?wm^CO3JI}_7G;Q2$i3Fh_&#?k3p0OTHp|stJkbvZH>uB^sDX=db*&t^$@$U!}sRRw_w215$KTq
zYDQ`LSCueLibEA=oAlR>8^iZ+un5Y4|2KtY!*x~lv`pZ&jQ}*#z@K&axzgNQ5w!7x
zpb4HwVDWQ97nuR}pCoP++vWzW7hS{=q(m@?uhIo<)&+ve8y%z8?lVm-Q^j0NC>O!M
z=8hGO%RmUK3@#*3GLE}1Q*y||9dGk_U)oCwd4j_3)8|2z){HsZJpQanwDEXcb`pQ0
zRU0!a`)C9G2ZaKc?&MVuMWt7sprIB
ztb?e!n-Q{TuePaih>Po^C87(@a_oENVp%FyMj=Ng!#;)!QXQb&1RIDHwX=3DfDO!djG~3JBD<&%c|67-F(sSe
zF;*Z?nm23fG=Wn5>I}GEx;Q^9*n1+7#W=|{1!z}Iw73>&GjP=5)EW=jf(||!xr(ZN
z57^06Lxul}^nFLKpT{JDdB3WvYBQ2TI=79B2`~;3j~QDW&{KHj5)M7ri1iwZ3Gy|G
zFFBArzqxJOw$V@kbK};dQ*VGui(qf$7Q*$VS$1)P-q@vCCsM|%P*nm?lKiZQH!jdp
zAib($rY{~+%x1{`$}1~T8o#z+bS-t&tCQtMsR_~J5NY@Ua))9L8b}i^N!!>$ZERlV
z*e*4@<^ad%bOfk3GT>6wK}_!y1JaaV@!xL~$cG3W}0
z`J*OCLWNTHKDb|Q9-f-DDqxtJ(o9f})91;;R*y0Faz8wQ0Pi$3G{P20wrQ&GcJ^2E2OjDro1D?~qz+Rv^
zR#?_H{r)?1`q(Hb0C`>MR;lB_BaHxiZZGjFB4?@bSvdiv5fucz=$uF1)n7K4Kn80?
zX=S6*0kjFO!Wq!iorf@@LhBL2o&@GHS*u!GTQ8rTBVjaF4xuP(jtP)85>+a4C&0kJ
zAUCkihQP~wEM
z!ie<<>>QKu-b9d_p(s`4vP$bqO^($>Gux7BoDS28mh)leRY7b9#qK$zv4dN}dE5h0
zufNfGsv#dhm=yx0{Fyw(yAVc3)s1G6{^ivCv8S9;>3M(`{BZt4b+=qF1jPlhn
z8LACMR1=(~0;7Mv@jWDjH|rv@Mv6H+NC=90nknowlDWq|{|KBRV9*jap<7sljO1r=
zU-^QTy87SnKl=?OD;i?NUQStAQ~WeEuE#_MjWv^6`nZ-0&Ofs1~E
zLTz?z)yYg4s2xA$asjs56d%Wx>R|YEQ5s@+GXX_>c7y5`DPSA`sq67uGbatcP~meh
zI>~?_63>vKrW*pAA=Egb+6^1wC+hi|BB|6ul)6uoy-c2LK>T(4(
zkK(u-Ui2n|cu-nroO<#*ip7){M$KL*sDqRwnDV+lg(jd39)!|M7nLnujhUaCqM`5e?uZ%&`R}|
zq%+1muEu>z?qdr9I$vi&iv_k;Kg=8Ha;wU3e-|NjyEuo?Z%pcT+n&3Y!kJ%c4pR4Fccz+MBF}I#1VKf`~fxo*b^61KnSzl^SMgs
z+Fh!7@;-m63XT-A3=rsSZOwZl=u(54T?BLRY)ps|*N%Y(El9UC=M&uaxR-A@Q5CNqiBYYpP%2?pOXgmZN44;ra6ff+=7ocpNR((i)q&{EjSd^F*i^PZ(}
zF6kPHQtbvceI!)0G?`?cV5lV{3f*QRbX}(q(i;E`I!z`X|f^B1WOfabu7LjxwX0{v4{Q^9SvVcHD?>c)iBC6zDX
zFd7nl!OM_Lf_gv+b`8Grjrfjq)%3$P9>G0QZi$yvCThd_%nzS??9EELU
z^*sy}vxXX7lZ5(C!y=lNdKT|%4a?$d4^4@S9grGpD!Llw9O%Qd{#W0I;e~RlSW(jR{l+@R+
zQ=aCj0{7Uldv_Wju{2aeXl@!|X=^J~hp}EeA#X2!q#KgxkdPwz&-^)&6t$zZXvTqg
zQl6h(QnF+1dOcU^%#!3a_|*5Vb;{g(c$kZ+b8`_aYy6YVl=ZA($|kQEgpQM3x4h208mTz-ztK=qx=r?TfFZzWv?gQ-k0ncmx{H<{gIkxbMX5HzX3mPfASZw=
z0}v0^gHf@(Q3f7`RS{AY{tCCD(SwSFAZ1lVM^RAO;+6_{P?>9RPZQcAf(H{6Kgun$
zc*a1Ht;UQ6&<1r1NrirN~
z47~YQTHOko5lYu^?S?;G2m7xi1X*yy#nC$N_e-DO?(6MYL^DBiHev410?BO7#X?9@
z^z!zexw&&ZRp*o9HR4a0=f>GYkv9V
zGZ6F;?X7E_nih18voW6gePOCDi1biA1S-b|678thh^35rb*xfAe(E
ze3Et@aK%0Rx7BzDK}~Q5fapcpBCD-`-B<(aLlhN{qU;M5fiHm!?CR<&glm`!(o#Xp
z96ApLE-RUY>Bd5D=m1Eegi;5?{W?L5pL^xzd)aTI0jb&c7IlRV23;^(Pwsr)lw}~T
zLJxV!F7oOfrC0?O1%;F#XTCM~>boT`Jsc}tx*rKsMfNn-ap
za!SAZZWc-exPb|DfjP!S)Sx^FM_>P(=oKK;FxMX|3e8;ZBLeux0
zrapz&9jjbIlylTzj3D!EM9AyQi#4RW;EI+-1z}>s79N=??4j2-z04@_aCmz!2@cc)
zcX@EnyBqk5bg;h7mCd<5ZB^z9BT)-
zf(DRROQ0jmG7#;+L)19KnAU0pvi5MiOR9X+}Wes
zhz<^zshB%d1J4x->=CL3OR2eURPIM11*vDBoDzoRBU0&6XiQo#e=tpy^O$tN@N+S0
zP*xP_X!&ix4~tgg<{kkC{1!+QdPy1ZRx0@DfJwCQ$9=uM@enp-m=w3$o5aj@aXT%d
z0ZWB6xbX7C1NoW6^nncSZ{;zQ&pNhk3z&U-O#19|pmFUlBV3x`ATU
zD`>0k5e6QX3#ffWlc0ELx%}s@9YPT*w{P7NC~?K>1i@$n)Ev{dF`edwZG$&wYyO~S
z;kp3yL0lpt&F8mP4Xh{306sgklQIUO3)jS`GJ$|blZHpWn$*u(dI>hR&lDdN
z^XMwfBKMoO@6FsAZ4oLPl`Yk}IuWG@n&D8!@!no7i)V3hag`0B37{`}CEp>J6uda7
zSka*tWMn;48!-*wR~YPaF-RHf&cYmgY)ch*u{40Fe+2tZv6vf$J@ACn!iPnaBjmOc
z>M2b`Q29r$=}~RAx#CI6c@`fsBRT4
z6b3UxO^qca$N*x2Dm)Pz6<+ktaRuTR`t*3fp5)6RFcy;GpZ@#2L`sDen1*>AX+$h^
zh+!xb;EedO$)5}+;z^|<>k;Sx&w|T{V7+Jq$o&F>7=r#2qX?2Tvpj^=_^ELy^_mDA
z2;Tv`w44Sjz3bcq8R_53>t{&8L%0hhjYl|y@8UbwX#a|Ll30B_oY#2!wr%m`fo`;K
zadyYY8G835p43-}0M=->cC4-*D*9J2j(ab_&)*dI(UGH$
z;op9x+04eH$yw6}XP7@HHVP?)i=unN5F~&X?kr-o2_zEf;h4yRTGDoSs
zccr=<&lEcJYM=x}rG#A*M4J>fNYVx>BQi79tWLMW$etsd(zGkYl6-4*FBL8;U-}dw
zeDWq>nn+IEhY#l^Woem1Ym);0?sr`N^>B4WXGKOm5HBAlW$E^TAOT%=52~_dt}cf8
z-tQsGA3@MKnqF-?VJStI{>$|QP?LFOF#d;vL!z_@a3Zza({7u1`~nmQ96>-1E;qMtW!iyKbKniQxq4RibQ1?nvAlOn~*Tj
zG@+nU2wiTGmcQ5st{W>j($Pc@gWz!z>pc79vhY!tDpVQEV3>YZ618#nfAXj)lhZ#w
z4F1LzaMS`t$>hnFr;2Id>*s}
z(0~lgk+j9w3r6@Yfsx2Sw@jX!Yo|*%#JR#&4XiGM4dr(S@KFdCf&rB3hlDwHho-HW
zXc#6?u|)CPJM(hvx>fJqwZ*PDEE#o;(`J>t8Ap~H{}_yRt}w3H3n
zExPb=5HP?YSQYn(+P*~#I{4%nLGs8ipZV(e6)6pb945>J=JwiF{w;1)|VG$7>P-`MGdDhKJSVlSd^=m75kQ}=X
z(eG#)pk34@^eIjZruQAE+A&hpyF?`Fz@kFkZT_W?G3w&lCy3Q}N_k8vmyiKGBE0Wt{Oy?u$=9hikRR>@Wf{;v#$QRfeaLQMZqnBd?~vLNdC
ze@7Pd+OdF7Awm}NWY7@MfKuE_XYdW^j44Sdh;K_h@t-T-uAFe|BJaI8-h(o22u
z0big|&+F_~N+uUiG+rU3SX0U!@c$@+bSyg25s1cpqYWPi
z7Q`hk!jvVQ@E>q*d2#T^;XpFw+_Cx|ev;(n@;e(Te;AZVjwBB%UlbIL7_;+pK=L*K
zL;_en3BtdqIf&dJ;va}C1;BI)0u*=g7J{b}#Z}i@nPnhg8h9i(*h>5?f)&Z=@IW*Z
z9vtOC^>i`8q(M(=9`1AwX7Lyi=tSKfTC03Kx{Z%a+e|E>-ONBWV@(9_0_RTrH7IQR
zPCdwZals8m=)0JMe0-1IyAboRyv|Eegzv=U-}=U1etxdMxwiO7c+F?4W;rGPm~emx
zMl1BERKL2asyk-iW>hh1RsX&N^V%P^zf=toKh|BQo*EVlIMAWd(YB2tF*$)LXcPt!
zFE6!zB?B5sQTCcfM$zCqMlQ-&ilg#Y9e}1l9Ze>#-!y=q%EaFqh+yD*?9U5G;r$s#
zlTW(1mm`=y=`iYo2AVZ=`Y*F)9MuRFc8E(PJ>%1)U?fD^`%bGC3*6X+Ps79MaB?70
z%OGd4gg1r{NoU|WvTTqe-yA9qIv(g{(KKlzHsDtXW(qKNI*8i+>${ns4})SvP3O|J
zesXGLW|Mggd+78HGjlJ=L5L%G6J7B{5BNBxqT^8tT1HiKAVOtV6!lUuPMa1UUKn${
zRGJ?TVhH!n{qy}B7;|JNU>I&9AeP5uWZ_`!<==tovM6r&`5o_NbFGsT=VoO+rm5)U
z9e}a6{OmDs63S1#;s=qaf)YTT-lz_ZwTLO@CE>k$Y0-5P+C;+gU->qA7)7Wx_Pfp3
zc`rhuwG>&4ZE-(-rC3@Y2U?taC*J2R+LCn0leB`IF02KLl#Zr1C?UtDb$4$dmQ5&<
z2DQvQqh>lZwv=7Jy9@IxE>?56_Evza(H+)<^WjBj1|;43xzCE&H|#&ZiQMQ~IU?)}
z+P-IxVNyX6*jDt*&522aVeJHHY{D_aOG9R<
zKG5-**XDT&8cHI=vrEF|h9L39><5Dl{nQ0cNN_6Ky+~
zcl5tJifFonfr-GpipRHH(_#*9Uj{t`bRW4G{$bNovF$3jtUJxPj~pK|TT6(}d9j
z<|J__U=N5LM#n^?b5P;1bn^auDC$ev3~&U-w-3>toY8>(3vTA*asu?Q1_@89t}=5~
z_F&jEp)EX+%KDt>M0rj^|bB2I|4BHd}$#RA;Zg(U6<5I!g#;2Ctz6-Rf
zyqqx(3I9}TU$g@`19$lebVyj@5X3JPNr-}A24(A60vw3oG&rOwK;DVyx26!44cShJ
zf!HC8nc@ut`v=s`YK6Y-m$;vwLoV?yYp7RZ%4+P1PXWLPrmY1Qu%jgtX3dXGW&csY
zxPSO#4YW&#kle^~#8D4DDFjDjl*lASkx~u5M{2f8p-?_)kW3A^4CG8G2-L7P<+0=i
zFw=hkaJUD-$Z_2(Z^0{8#>Z_*mQ}a!fZ~LhQKx)QN2^oP2N&p&4yb|
zR-$Slg8~~tOR)jtd#t}xUDcn2xz&qyTx)YbB6wdQ+7
z>x(-*n+2DNbS2sg2$tX|Fp+Yc8m3PCS;!|NQ=&OV6z>qvg=!1+UE{7y7NJi>kb)Hm
zqHE{AYh}A=GeL};&>xso`8x_
z7sye>VC^TZpw;v&4@pLPJv5Ln>=VpRxm%K1c#75Xp&F6TvW9WrRM1JWA*k
zV3(zg)7{d&ICGc?PeDXK_E-@V9X-O&kELSij@@+F^;=3-Na0}r{r6^wFS#LfMF-ft
zL(k|w6cr8q8QWYZ86lx88Y@&LR{PWruSRSeUUb61gIB=GDLU>|4|O8WWi66=CS#in^XLsry?c$GFfROZH=>~APLZN
z?Z1o)%EzEQVt5$SHs?{35IGC#G$Q?b_of5g7q}9IAR;e;()H(~cVh6tf80P0D**z~
zX70v@vzosu$?1XQ2=H2+MZrIEV8nOAui{3sD#|W=xk^;1_@mE1<-kd8Pl=Zj`XP{b
zXwPoP%IN3XMs#^EdG%(YtXP)7Kru+D>sTxTKm%ERU@nB5(!s0rGH+4A>Drg@V!HKA
za3D3!*4@;L7FvQ+A=rD^t{uCX`S1|bNbS@(*t-1KS@^)S)}!n{?`hxiFoIz37({`x
z$_f?WrMSTf@|;lzS)pzq5+;JVqXH%bQpGgE
zXaFma+7#^(+dq$*HsvV%5rb#&!;WShjIfM#m<{uzop{EIWK707f<=?rntJmBStn8iGHDF@;dEAs{s()R#Gu^kFl-vrj(xAu
zIoNDj%VHTTX2J{kl>jDKoiO>Y2~f$olG#-(Yn_j=OHMTQhh)P<6j^gW`UhAUw#C#n
zkNN_5ggT!IixWI%^K>l81ElTF?b
z-~sLqVkJ_6_5c3>lQe@+X-`i
z{cx4u8iN-^AcDBy|3TaC1Y{gO^ELJqE&&0^lUt;)%8L9U
zX{Z3XqCh{LI7Q_8hz~_ap|>pSGg76`jh3#Ja(N~O(8z3>(|nwVLoaZbQEXaK^p~cI
zi2qbJmdzQpm7oxYDF;Co$y+8(uXC+R`8IsKoLlLwtw_-g=)zeEnGJO#5sq6M%kUjv
zepQeDD82(tOHcw7F$5V1HP9H$JIrN(BPxVot&4WhKWau(@!Kn0CQVYjLb675Oh^8F
zR5Gswlmwqrd#fjz;wLj9p#FxCbDX*3X7Cl{iL{gqHA5Q5@TfRE83f3QQrU1~#z4^G
z8`XtDEL6vSv8hiJJqji!u>c9sKWHV*$+`xR&zwc_4wJZJ!j{ke7%ISSH)q(ilhI4i
zTc-=GB#OLREDqzvo_>~?4{aU-M=E-P6U&55ld$WMH|Vtc2>!TleB=1)TvhSMh}1|s
zwv!S<%Q2IcEmeB-87yVRQsOt`rq#_GQYkQSS(5Jo7S~gY?0;k>E}n(LsVaEx_EVa&7tsYO+l?=jfc1&
z6?Haf8v14SI~q4O8Ctg&hbFo;8f7Y<2(yYvDp-84RsU+!igoHMS44F0m2x^8y~3h)
zVxF-Gh>}L4kiW$2C2hoZE^gT%HmFDZk)Gt(|=t|b1iSvGI>>86)}3PeXDWvl46@A(_DT+r-Z}>Y3F6S0-jX*nOMH1xCig;+DD0MJ2unB|h=YQHL4=FHa(Vr}
zYny++sdQxY>o8jVhE%PC6hwXO(HrOsH|@{J>vDA7x?+y`H~g)&!}^|Uw9$%Ax}t5?
z*@G%XTOw1bNc~Z>H696Nr|C_yK4|4;M#d@trW9iPUj5!1{y5NWoW&UE!srEWvVk2-
zfvara3Ckxo!Y^72Me8abST&3T$&yN>>mB*tnemXvGJu!Q6=<@2eEn&D20Zt5_fME#
zIOylfcLj=7vg?RrcfbKC@zZD-(lWiO>)_HVFl;26*$t7tl(e)C^Nr84nx=R5awz$oH%j)W|FI+jUva
z_ZJalqa|F{`ylmUo95XW6DDQA>o4%>+Hg=|9mGq!#VMbVl@H!o^VBH?82y0el&cSa
zKG}m0<8RAJK1?|pb&g^9Bc3Xb-E5x9HTnwE6vQAoxp4xsZu-S0`jrDcJ--9xpg_v~
zD-onsRf;tw!5kHr!R5o2b>vm8{U`;38qCEt>`R1O$ZHrer@q%-03M@F?WX)zx{AC{
zkH5<4QgDo&Fg8aa%@u7|80zU?eNa4Qzj7hC?!C{P>0kqVtKqpxG)3RPqRBVN@wO4?
ziaCMa|1^#w1#ZMOe{g0neJe85H>_ncHu1x}FN4%&ehN^u9Er}~PsOKYZzF#jXfOp6
zk+F!`hQ%gUm8~eG`%;fXHVXqXRT;1k8eu##p6NeciVmvbHBg0br2ri6L(IZ_=M(rB
zP!QcuAT)Cc-eL>tUV$6=s`&mZhX}I%T@H2D!;;Gd;4C(-UIj5CS}dxkB`+@yJLNIj
z@U48a-k6J0iIE5e&+`-wSo)0J2KmISd{nS^Vcr4zCFa;J>f_R)J2`z1rKlU^jEtCo
zysf%@<7lc=eVHT|*7UL!m!H$JiCN6YW4%u+7GVtY}cf
z)8*mag$%Ud1#j9ut2?Ol)zfu(g3T-a_sAa&h@@}YH@79*3M_dXY&CtQSeaYI1e<-@&rbpi|n#f@Po^K;WYc)hE+jE6k
z-{e?d9#(|H^F4IQ-?bNh9)FS{_=rW!GLW^M>3RAlqt^~p51@!uR8Y8w2!$OBWV7!@
zq+7w(EN4$>hnv#%=sDcCI4Ppl{oTTX9e|_M!k02Qq`WmNF?qyCn}sza*)2g;iT`oP
zTORj)BZhnvXdgK_Z-KCg;3Gwi4
zhM~xEqH6j6>0
zMgUY1G`Vn`E6{lnG7f6|``QnPMb%6?CUB`NzyfjfQpCA^BctFocuz@uTzgCh>Me^&
z0R-K{5o0L2X`<>*_l%kGQE$<9zv%
z!BaF<`|yq^hwxSrqkOBO9o~HPg^$T2OAK$-sytBr=c9gYpscpp*)&W8ad91F?pb~Y
zG5E~^u--2zS@g;U@@eAaiq!f6fHgK)N8=95Cw87mQ<6B|_gmlbh^Anbntn
zgn#>=Q}H7Mkv}r~k9R`;?C~GYazBpsazI`FpBq7R9RKyZ<$wC9-mrp=MBIP-_e@F>
z_x`7Ub?vC)nvnneTh}gf4Ek*SkN@!6W;(Y2<;Qz@KVg;9zkhjrX|a(1%O8&Je*O8i
z8#a_RH|IPE-jA=yfoxa+QRaEP7t-8){`9YJ`E_I7?UOz!QY2iUwzU%V!ph1+rX|l-
zYJNHei*8I3wwgu0I1Q8Q8#t51j`Ij-mNB9xoDbm{cEE41Y%D1FXT`+@aGiubH`Nif
z&a39;P>{m#w~tq$U$^c^awFF8DnOokw1MyjUm%(#i)|_bn1}ziyIoY-*;TSuBSNQQ
z-qn)HNf@hvAR1HxDFCSLNmb1j;O=B2Bwe?qKG!*!wZn>wI&`icGTN^uFtXP3@uNn+
zBWkq++}!DJ`us|T!1sZ9mVMLj;qKmmVt}mE1?Xsy5r$mjfM83O$D(UfexHjaCCKwh
z074-%jzR(IG&gAuV6_^0Lb$OX67pJpi7yMrBI#0qe}5PsS95d}M7~S;{7GAX_0=PD
zhL2j~o2@6NryoAwE`AbM6v0m8))b#{;VTTTQBhMx-{o>a`AIgMJZfDhbA6p1y&*^xN~oDseI~Kwr3r>*8eNpnQYS^wVQuAo+WJ+-XDl#Q{wn9?Cj4rU{<9
zbV2k18f`<}F%EC)$^1P=c2GtP0KCHDudNIWr2gRL-Eo-WQ#|(@?fzMvTzBoTqMT>i
zIi8P0vjQ*R(xZGxc+>Y!yizhIli~}QxvRi?1LXGwmRk`#S#S(Rmv|_137Yar+;HMY
zRB77<$d#;nfiik$E0kee044{}ADOfGE?R0NX!aY5P?We`2A})ZfND%8?&E`R)8VQ9R3oQ#4XVR*u+wh
z?y1>Gdhzw=DOhBomt-Tn8-spu-*b4^sHWaTyr%d7S=!SiG%t)Z>pbzgGpR4GAoK1K
zSBtF2^mun1XmK#|3_=zgA+r#O7#|xm**EBTOh&;gMNH=jOQ(NdB3J)a9Ik-Ap}9C^ceX9fU5$`X%WxOfdgjSb%dd3A=Nt3)IK#q{K6u>
z)5w-gGM;yF;b0PU6ja$DR^ef41zU6PLgWE;#f9&GY#_1arlU(MF8vu_JgPgOvshu;lB3AR&?0!*U>41EmJ0tSE=*2faYH2iGR|pyjDTiM;M^
z*WU;6iP!GqiY-5QFWZ0z2NOj`$0yFeQ3qj)iVB#OtSAp
zAU2nw8%NQN1_cX~5^x5wNIWPcgsiTj(lp$QP1I0|e3_i=0c?d08;;**?2ugx9D1)m
zjXn{7A6c4$t3UF18;=QAo%e6QTRb%fb_3p?NunWG{$tebM_aIOM5|w
zL5=kmo{Eqw;h^qB{e=x4BySmSpp?Os4soF&M~7WiJlVsGfye^6utU-
zTT_=Q>sZa)^vsN|eLl|kWAT`lS61@b$~u4L^fg{4?j&aB=Ai|u_||yQnESa=ZcY`-
z6!Y-jTFJDhlIgfAlaQvNNz)>oIuX3UF!r%cD=+pSjeZ&31G<}RSVJ=n7&*u
zVA`v&@J>_{YPIOxDrSvBM7h6*&rpfFQB+s{N2E?&GuXb3DtM%MH*L%0PsX{>Al$?~u^5}@kz9h2t@e={$KF|2>
zK^Vua!s@ITGH&9s1{Whv@Y)aptDV8uzjm2W@()S~5(#(oIPx~ZLU14fR+d#)`(pBu
zz&_@4fH9G4UIQ7i&E+v1zFPck)o@2RvM=C2^k;@xvlyI4PUh7o5ToNkj9OF*U<(d`
z2L*YAg+oxX+;1HQ>GBbp9ac8BEt@yX;Xp$B!$o`lndT0Kuj}$Z`g=m?@9rXp_ph>sY;CIWN|S&NvA{XMsf`x;yF&(hx-XA7PkW3XOX!?hNG_!l_p=jQZX&;;h?8~dM#cYgWM
zActxeo7H>!)swWnm9~kZ0e{R-YHExsBZY^SBF7%dHn59OyIlTG|L}*OT7Zg*hI>a=
zp=yL%vLFfouBlr&7!qlOxMy*;m!5RnI##rqnIybazuhA15g-9)R^zE;n8=EXh5`3h
zhxw0lVW&S1?Mocb!MFEUfwB?w^I?X
zz4gqR+#Ei{Zj?EzH;PvEFKG*oTZ^ceV1#u2MGB=UfuvauL!@IZU}OmC77{c(Gp8_ci#9i
zIXx{rw*(=NUf71UYww{}MV2&Jc;Gn@a~HrTS(?x~EPn9=1yTiJH@{~n4E-2|l3qC^
ztw8*?YD(FH!*344hyVk4ehnZ}vuI>MOKOLp{d+O=%JFYHuto~p*#l6qXilQ>vN~K$
zm|Q%+!kUNy8&!Tsc>qaHY@27!+^+50U!fdzaB@0_zTr)zh%d}|%ojZ
z4@nOvG{Vp03e0TGrF#Q=o_4J;w<|cQs%j3HlDY&`ge(o-nWrBu{qN^-tf%KC?%O(?
zM;;!3@T#0;ayw0vfOOEc#TnEFa0hV2<-XeCq2L)lYtq$}q*XHqruGwl9!h>u5x3fH
zROpe?sTxwNug;5&s?0R4$+6{$!==I2m;f~JjaYjEsLRUXH4#{ImW45P(^l37a&N7?
zt=%VP!R^s=w|!Fc&S*7we|gZw_>^QVah~<%7m$>IaSU17uzo#O8=l4!wP0RH0Z4)4
zu?8t(&x2e~OqO}VPDq!Y5wF^ikk=HLI7p+ovU1s-cDsjJ
zO?`XE?YubkrUF@LBk*3Ta39H{6Ao-~Y-|k1e%3$XAV~m4GB5bNWd5Fo_i2xBjwo!P
z;68>QGVuWEaZ`GfMtuZjI0Z0V*VnUsdog7i0Q7H&N4gmXoQ*K@P0@SA0;%kqSm_Si
zk#1y*FR<6*G!^p?q^_dwssop`^@d8p)Poo3Ktf8p;l6>>R*tY10h_PXHJkWRJbuC)UGd(!UgVOp$oP(U&AYenIrVrx3Dx3${@qQ7JP{gloyLMR$`dyElS{Wc-
zQq%@fbBMsqs#UApFoB6(Q(&07GIG0&*GCG!ROi0pvn3?4jw@D%sw@eub_#
z@jY>hW(<~r^g-O@>WkAeWo@s+95b$y^CO+6WNYX8Vvbe}SPJ|8f}*0xDz!sliKy_R
zD#u-gEh~4(hty4GdY{~&jXp14J{{Y-*08dh2Pb6_ndsSB#2$G(R
z?L;+L@-y%=Y$Rv85sw9S2y4*$+k(V>b20_hm=IFc`}glpVWS_&^d(!Rqvo!bm5rdC
z=FmStHF2Z-WHq{_Ng!|$QpSy_I
zWq6b({5VYyjHm|0`h$A6tDNx$AE(T#s}syyfn1)u0LymXf*U33XR!x5}}1aK?ray_L+ws)H<
z%m~3#ujLr>!N}W6RJgzy_8|D9P>1g<-^57;o9djbHm$(4-G=SP9z!1J8m&;LrN#kB
zjLfin-;I~UbLpM#&RRe20Is;)CfS@hOV){Ev#s5u@uD%c_5;logC2iGPDb3h^8zU7
zhV9ZHFD0Y%ftXVjn$EVz7Z8@VR=&|W#^l6~_z^j&$=DvAot-Tx8Arhe&>EPvJO=>B
zaM+&7`&&o%XOshRUNemNZ0Bh#kI2d>-YHo)8km9S=O;LV$-L3X2=SOjnoA~s^F)0CjN4Pn#jG>NE1xwmXj6&NRiRy=uAC8avOz%v^()
zq-0#tu%WE%xf$XZ07tw#3$3D>-!bun1qg31GaJ=Vk!R?#pYR#6zLz???)FwT
zaqeO-d!}RVDlMHwoei~eKNX~Bx6nA>$iy!6M*c9JxY*c1!*L4QEl5hUr9kS9o%a`WCZUPu0C!f!P50)2Dw!Hqh#aoGlV}cJl2}
z`W0-YOY0u;wbojw{fPXdsp9~u%<1kp4P0`OU`8zeC1rY-b~K|Nla7pCs<-vzquDF0
zCE+SPMP5xwjcudP+Qq#1)D17dYQGF86$M{E8Zgqs%GJ9D<4~cQLEIA6uUvkK<~z1j
zMl0GeX&IV~h0U<-+xG_$oO!8_JUwzT@_Ros@2i9!b{Yt1#YSnfx
zOGa@=vxsjU`3?tQ{>TdC2jYgiE^>h5e4%pt(A!%^c-f){Q&r{Vf8jMlhqYq6^rmUN
z=62zPdoNCNJYnX)G;>~fH{M|PaMMWLor&+*6N<$o{^+Wrp@W14!0(=(QY5q>#26)B
zXayX9)8o$2Kq%_74R^1k!p4&Q(4mcxFKlh)mZ@tv{smSEPEa@D;~urOJ%k^PSVRI0
zx`9$b;PRIbnw3*|*C*tUK_7Ih8ch7VoWLGG*9}&}yD^G$pX+;W^e)T2raFgFvA}aC
z0OU8Bk{Kwd7YR<
za`UdhCNAm*3$(CoSYH-%avo7D7^i~0T2eAofhlXm
zekm5cMk)`Vbw@ar@Z!QYEq*{eS6#$&(v;-Aoz-6zBZ<+6XxKiOD>xkJ^~fPELe{M7Es{A-jrS>-U$4%Tgf?S9&%&sQK$fvo-`?XFO
z9f#pi#(`lC9c|4b!!0yv0f8+XX?%S(vkkN$ssd5ZO=-4FgjyOE7*dm8xppXc=BXec
z8V5F2Sjp!q=RQr85ewp>UBZPmDt6zIp{oou$-&h%8Ph2gQU~wM8-J4z&R&rzooVFO
zIcutK`%I#spiFTPwi#|O-jjkELbfZQgXy2c20O&6Me4hmndhI>%T6)%=q`=}{zv&~
zhQ}TtmaTLEtO>aVUBw<#NR_1OgOO!5A`~N>y2~zDU;Ok{!ba?Y7u5Lo5z`1B1SDRg
z^okduC`dG0oCx&3yD+nZv~hh8(pC)>48+ExH9(UU`RutTXb*}r%vfJZ7DC>$;W=tt
zK$9mTzQM!i9H1(2W=ylgwIh;@dz*9nYr8S@PdUyvT|DJ5iK3lM-Y;xccWP~kfzwea
zpE0uOr3>^*vV6HraU4&Ph=FvkeU5Ss45ns(YK`nUnPbO~M|}T+4NPW`K~tB?O=HtU
zmjnd`LpE&ksS5uBWCNH~2#rg5O$ZMyS87N@M^m$Ccw}bdqPyEgV%&Cyx!Tv^av<9O
zsL*fdf2ZJj)Ap=p@+P;d@?6NQeUyhl$*{8ofC**e6aT!t!O8RC56zXm9>0oQ;Pw}x
zVTJPFq#%BakrJ>U6Es>M0N0U@yteMEC{8iA@|f)#M1_7PDRxq`w%L8yT+lXn7Z
zRZ&?}W7O4!PfSCT0I>cUbbY{Qf@%J)?u;8(2ZzVm)m!75+Bmb{PEx;zn|&OjD&2K>TWM?B@~xA8-N
zwU3vV7bx`;j5!uH5&<~IlmdNy=gik)akPoYs0fZ;)tfhFW!E=RpFx$t<`3z|?@(II
z7VJ4Sbd#7CpDMi~4gZU$oy;1`Exa8$440uTnuA)c8rUgX*;r>OQk3dwr{)=g#*0J!
z;56942Jp%XBFss%*O()=oH~O`2xm6@DNF;g6J#&;Ok^>MeNksnhA2L@bjeeKHsr-|H{)S>|JZ_c9f5H
zm&Tz&1IvE`Q&8_gYrJYAp8c`nQ^R_=7?tyY_(i0XU^i^t4$UkgsXaMpZHu*ZQ9+Idzd?kW*qN!jv&Gi5F9o6Nj
zr)KLw*zNpksILAT=JqF0LgECVO*#<)>T}bTxxQV&eej{7-|iLh$hb%p5}|&mDG3q}
z-4Lg1*Y>>)Z%kyD7KHHA_Cv}@5L1}ynVUemgSm$tn03=kYI8E7N|ZqMV;D1rI`FQa
zq!)DkQ2|9yxVib-JS{O3hD_8$+
z$RI;^?o5j4k{`1&e!9a^u4sBN*GPp}JLKbO>Q$3R2jSE#tB@}b(u8FTc`pdK0k)WN&=j|X2PYa)bh
z0LC$RcC>hZ&wA~cNzMu59?5qs-B|edkg0`fK;k>%-z<~o-p*-BSdGl!jcqv`<0cF5F-nsSo
z*vQSN)4xPyi~y@{Yba5uBsyLNo&`!8qS+&~3QG7tz@lPPGwZ>F9nLQ*fHHy|
z25DA{XZ>)#XJ~X(Rb72E-gre#jX$O?;lgxwMp)hoF%YA4=m-9ejHIF!=LCI|Z!0H9
z!NX?<+6P@Yrz;CmeAM(Ita|?lm?q0z8jxSgvpy{ltF9izR)mnDUt10MYiMb!BD$gT40-=eqCz
zhu>5*&`>D}(LhRwY)YL9Wh+D(QB*>NtWx$UTlOANBuN>SkQ9KFxkKb|s^K*QU?{!?~)v3?t{eF$-^Rb@pSgAy~f~|>)E8A@~M9H?v(#)L#>Kyaj
z{%?XmP`!=DwgB=pq9eoT;}ED5tk9m}KqrI{_C08D=qhR74)^9s*uEuuZ}1a%}P7x8l~=B?E{4d{lpbI?c6fCI5@QOtm^Ck$GN!{r8_+yy@G{
z|9`}3dIz@2%|8#Xk^!-$E8q
zh1PL9=Yf5#l6=N9zP`T301n}ZxQ6;d(&-m=Jb9^rg<Sajw6lqN2DBbxlnHUEojr>C%)hdt&1Ce^td;8`e?oD
zSJU~{G@{-(@>IJ5S5Y!E~wl)X8v+prPsy9~u`L(#`1&A*{0v{lQ
z;YQ{FsW673>TdC>F@>2KR>^vywnNVwb+Lkv)!dwpP$*
zpv9u38+V($SOKvi98k+p5=|qu6wf5>l`||M5m<=os6H^N<{2bJG_dvU2BS})EmZ4W
z0|QD}cocwYsj)iwl=ICv#!%yTf%Zn+U_ywutb)+FnCm7c=DOVvw0HMHis5b_G6j1wS
z5K#D^(2k!P+3DW-+dxBcqI3WwM$yvJdSS+MWu+$oiwodrk<|57F+Ay|4mJ}$1un7+
zIQIcsR7-`gxw-tLv-5@hGA%(vfBcALF>Ah*|4lh9Fi<6OAYD8T+X3(-)m`JDhvI1!
z8=D{e6sYn^_j2sm3&=l196Jb9m-E1hHLxH)-VG<2P(k0ofbY|j?I>Pz9x6Lj{H;q#
zO7}MY!@Sy8WYCuzr-2(<64Wl`Vf=Py|Iz~BJ-+UNJTXQ1%N%BW^Plg+g`X53PocjA
zT7k*u=|V$j!P=pV*pXM$)(7P#MZi9UC`PhfhH3*a0T5o|^0{5G9%?11_@VSL#6}QJ
zQJ7!V!j&0
zp(*NKv{KC`1|X@{OWN^+Vi4B53p@sO>OEwCJg&D-`|o8=JS;taV~)}o?IA(|XfiP~
z8%1jH3T9?^-|7>~`6RjKzV0wX0Yns7yU(|_m^eZhiu1x4zykV!m6IneoOAYYoPB*M
zXQ)W*amj5hZHb|?l70=*Dm>=len2Zu*-C(Ra6v}=A%wn`Pog+A0u<*qGpMd*s5A)?
z7dL4J0I0yJALh6hq{XKq3?d;{5D|#Hviak<)sNZD0pg+o@08B5Eg0`E5BpKBCUNl#
zX^JUwYY-RUJ0Yl1L2OZ=JS?mv3oP5TCE`KdoDmuVX>c_30TgvRPn6)j*hZjLfbxML
z7XE*fk-qy04v*|_E%oO4P#U=7l?$RJ#h=dN)1NOi5}ghbIk~3L|3YkZi&1A4)G;{Y
zA;*NPh^Tq6(-tpY%*n-d6?BdGZh!RB{Ks8=O*7Y{(4>w#5TeKnzILeZndrTO@C=7JLgnUxYFy
zX?8&FMo-K_(PCCt(H5YiD1-Y;4l5&4#6sKqOw+(PY0CiNTLA!p$ygcHJ4v3q%&KT&
zA?_OxUZ1dR%w(Zm+l!~3TfnY`0eB-TF>
zec+tLIq5hEsIcXt&h^kcN68Z&%q!Zt}jsriygCMc`)+S(0Jkn4WBu({F#f#`xKMh$iWXp(gTW(S}{%@GLH(w$N^zC+P;o61e2ndNN>w
zW-A(cA;V`2DRQZT?+bANHfDq_fY6Mnt-|duokypRSp!BWrOEY=E9xyYL%??ibEX%3
zsEz~}3B`CNWOz_`>_Pw^ph&kw-xoxjh@}HoAs5+k!Iy@_BstjBIMA@>|Mj}}o|9;d
zuEhlbHh+QJkiL1g{C}>t5i4rPf)lsuT!^T`P9XcH
zq+6<%f~)go8`=s$pOPl_%}`81qPJewMGCMjF;C$h`m0jA?WyXF%B!#WE(hukCRZQj
z-(`&lDQos`K{K?Nkn$l%Xm}jeMiGZ1r>x9?zfDNT3~!R$!q_XQOw>E;Gz03>R5+k8
z#L)IrxVcSP4CB9C9VTM~L^htv(9UM1fTV0;D!1(||6Hrz^*whB>K54Vl+0gY5WrifD{^jTvehaQ+M9i7-_s
zkIoIXEmFVrI!peG!sgKHgfK}osk-Jbd&yhdeR>6sUKDm!oy@0+3WFje9C(@MUmE{y
zMXOiZKb{MUbf+YK2*+9W7W@DvwJU%6a4`)UV1lPY3ADu#fe^jgtx(9wu5J)DX190)xpN9%$29A3Z423X94v+tgf
z8<_o5IV-lL#?G*HLlwmuII-MHV`c7rhA(ed6lTT?`u6Ico4qpF{Yoqu2KjKs)}!AY
zaHVO~+!Y>u8at&Loj-X}rR27_K!8NaNUCT|K&C
z1##yEFgR}@f5DPZc_nP8=!}7lrYyNJuBxi3NsW(x(piL(f#$%aI9AU9L75cw`p4@OgF`|z
zdsc{FI)60nEoMy}c2jiSn}Sjgf9K5`H=aOePF%4V#7P4(L=0MgvlX-%1i^k@k?r;n_89fjhciO>}R7SjxngOkh9Dl|l90Fs9nmdxEVKv8>@Zjx
zMD#IC^{SHi^DrO})x3-7B^%#|qfO3*G#bqZ&gZQTXQfzBP%}B4XhdZ|j!3*TGu&Z{
zo@2m{z>2MswC&%IIFfe{t&-kN;RJm3+~0vDm3@Rlmc5cQPAKj
zMT!g~?mu~OTJTq2$Qbg@_4vN?45~&1>{rVMk*k=D@O|A5d0{mj~&_JlQyAf_xari&izt<<5#J79CbAH(7M~_;|EJLcAyP8R2#Hg
zCwG4?au_|0$^{)6nE+=*uL-0eLnm)5p61;Kl?A_H2d)u<1f|pB+2ahR^DziHXHukB
zki3i!*{M{m)F^}MRJnj2l(DpR@UD8Ey}P?2{0?q+z34U!GVCr2v2_C1)q_%MGjzyM
zV^Qu;Sw!GqBnK70bet*dKOje|B)P^Bo*t`#+5S2&$FsZ~ZSNd3bo-Vz^9qpsW-<
zgjQirhnJy7``u+LaCo<4pr`cge^ocg_I->hZ@*Hf^V=^dfxf|sk!&io~?D8I>O
zB?DoFzro#JjFxdi>1mh7yM6mIp8hZVeGjFjq^N3@K;R3!L!7t2%Afv?YH-bOzt9N0
zbDW7r5c9zlwSt2~1yfywT;ViFnn7nzPX#UlD_h$|@eo#^7ov&{?n+g4rZc63HP6MA
zL<-P;LTjOZG!#-;TJax$2^kzmH?;MVeF0p_*zu}B382Ax)%4appUByyFl6qfIPV=C
zxoA_@xuFBsX)XBvj0~a-?Jyz%38z056;|CR$0Q^lxcq8RbH94f+uyHDXdUvZxED^9ytk`bdG=g5ih(&c
zHA`SQ#BT}D%F7bm$XcB;8&?v&gu-C0?yAZxW2M=Q1Q;G3jevZ#TCG=;hM)32IGA2O
zLOtT(jTK%FW+im62(e+NG51i7Ng*_uA~*io890HDANSd52%YHr81{M8OKtWuK^@h+
z8~!r~&I|Z~D#92RQ?d#ihQ-PhZQGjROfUb=H(*246_1{_}v8yCE`(sN1
z7ZL`=37DV^!g|n~y};}s@+2Z?;L${B7x4Oo(_&UPROzG8sq)O3ojQdZOAZPlum2gf
zE5GS~$*`~WbfY0oN$}c9SVGeSC+0eF4-tPqzCEGDU;u$AKJni4{|;+zz4A8KJKY)-
zmOxuf0KDEm|M3>l)Z@6g0gnTEBq}wuE2j-Zi-8^SZwtN)sMP{I?4-e~srX56Igiw4&
z5)mTg7bZOzwB^BP_v}G+tEL`~mJ^m}52^rDy@t5yI&d>Ig7CnG*fX)MJbG3fDw|M&
z0Jwg_LPD3%*Dts-5`Rp5GpU|D4(T^k4AA5Rf>2up5DN|E-HYinkb|3|R5KkwLTM<8
z6rH<#8Aq$i=N%_XT20=8M6@D+pN57Gq8<=4o&ZLOtR3`FHt<&=OoTK|K=q)c%e&a8
zp69&Q)$#SjC!ycVt#C&6@+Ec_t;UJv)4~hNBo<%IqCL;Q>S6E%r_ZUNz{m_8bLr5>
zrl}9ZYHPQIuPSd)+%Y)0_0*(^j%jLP&((*?Jl@L`eR$4u((zp3d^NP(z=ng))FToG
zLOI!?=IUmpo}Im|m%r(J6aN`oO~b>L=(vAt@Fq@93QBJ(Q$iV1s2d?6L=t}bqbX=a
z{IhLns5|=lyrHi^N6d!fnI%?J+jZH#r3lBvIAJ;nrq5vs6Rq+-h3kqb&wzU0mnqcM
zz^N=EQvwBIr0xLfs>N`AdinUwC&Uoo`Jl2gB#@7qwl!cV3|SytTes5TGG>ht?HBt7
zEe_Ff1KQ577(}@dAz{l0Ikz_=1R;Sratt;p$C=8m!E96Pwd2Hd_7k=h(JF<<15o5NJYpJZD-DECj_|E-AL&E-KYTc?fA!J>
z*=>z)`)Qt_oW6o?uM@W}(cZj4gQ;tE;+~0j6V2A~4=U7V2X2hl_VRkALML}i{Ir+X
z!TWf2h7h#_QhJ6L*WTa1bi;-XIBl!j1FQj+p)cnW5FqNEXTc31jo}Vokb`(q?3sh?
zm$q%&cK__VRVXGOI{y;I%mr0-buI@nyi}spK_^3^g53*jtgNm8OTwZtw6O3+R113g
zvh;0cYnCno`qMr*=#PH}D=Rxl{)WO5e}6{uj{#R>%wcPh12X{q@UPh$a+4p{xMlV^
zqW>if7a)HwjAE%d#~;Jg^kTVDz5Z8+H?~d?%n)e>{65}z`%3j)3*4YCMFV>hOt2$8
zN54CzqAJlCophZzG4@8o%O
zTD@4_!;U}x`$s6G=$n?8S9%GnwBv4^F$0aEEY4A>A?<|4@;PkGyZ
z&A45%E6AtCPa_BUz~)bo?96~O4Hz@okGMNot2Hz64b0%M}z9q?{&JK=M^!ck%c4@cJTHfs^T9KS3d_?^gm_`1>bOVHrKX>OVgR)yZf7
z*Uup<%IBZ|*~@El`G5MAtN-sW2>QRzfb0LjGk754dcnrV9hIk=B-#LZ#7mpp6e=M8
zR1l1J$A1CP$pKBg{rSHa=$AKY7F0iyOAu{Yq$Ww=4iLM1I?4QdQ`DEt9pwvEp#0AdJ7YdwJj5B;i(?YC@IXZO_p9I9j)*(3q*pC7t{BWypS)#(1
zdM!2_}Rt#BYz9MqqCtU;qVd19}4nd8XihJ??|6tLWoF$hiTN
zdsbDo1P#?bRNQ!jc&t}|qja{-&F+b{?_2_k|iUL)aEG*%^U<*&59m1iH%H=7(ARvP>JSd97q@Sn21v}c?-7w32
zvE^t4YwVUll?+WT*dP>BR)LU@z=WWRiw=KY>Y99^!yf?Dl8P?@5Hu+D2G%)bf7JrM2<7_NOKJ9`H@$Fir}{`J{~wikv-5@iTP*&g-v>g18Gi&p!9uYl^2
zk%CcNJ$umE$!W#9bzVraSnRGs-H441Xmk0+NYx=Zuj*Oe=kKp$r8nT|5)HuCBhklt
zFU3&p5CR=Ma)gc|!j1un{(?A{*-5jQ1xnC4lcH%CrXGSbEUmc!9`f%Wm(gJ+s46ahu)%5|!!n&ph
zJLme4Y@^dMV&ib%mmmk>F5#^kT>kwvUmd8Y;elj~s*@}}K7Oiyyb-(o`*4SR_6LzO
z^$V#BbF=*u7#LWLk&$d2F6Yw(?Pb&513>n5)$C*cBVq)Zxz8~lkB+3UzT!0Amm#5~V5--CNcaBQhlt-;BnWXS)(ZU5ch&y1Cc9I<0-?$!~|?r__Y4NpPI%&
z2`V2H7m|Lk!B2vIROJh0t;RH$K)gQOFNR3JwEb3FQ9%c#_!D9k!RsQd?WJ$6f_nBn
z5+&d~lkVMH_29t+A_+Kq7HSck=8$JC3Q|>z~J+QH7nsy=x@<%tq
zi3VG)(M<)4D`*^uVW6t)b_U4I(@2OL{)sAm^D@i%Zkw*~zjWJ`$f={D&}&O{rKx8S
zuOHw`CLEU!F@0rkWoTHK2NdATDxd$yvv1pa>Z46b1|X`9C&;r$1`%4DW&!;14*Wc7aa^V8+VU{7>h_vva4Yz2v`_
zpz)zuI!#+-LX*?UJPJ#%3m{oW
zpXb?g=W>n8m-?1;BPsXq9}|8iIUORJF|@P{M1~voK0+h@Fm^ynD9Vi_!5mm6w@D$2
z|5+O*{Y*f@my29--}0z`eaV9i>#5Lu?gUky_OD<4UcOYs=r|oo3DYk8H^87z(SQa7
z1iX0tx*Rg53s{4?1Hj-Dp+>?a38*DlaZ;!vx&Qhx=D!Q2l`Ge$88FEG>%XI#lm9RE
z-Sq|N@BaDO%**s2{*{}_V${FC?*5=o3Nv{A`8nkC|MnN$6S-K9&5h_@M$~Z>0jN7*
zjezEx7KfZg_E3Mn4|vgd>`!bTL0D=0@891I`OhI^X_U&q?5QWHu2gk&GK85>0gxmbK6}9_iL)16+9l)
zM#sdEu>o~5jlJRh%X!~7S2P&g+1Y{Qimb3)&MqYf=>p2bv5L^~bt@Y$d5j51SkG}D
z5I_CxM`VlrEp^#-)eT+34@zqqW=)!0_lI#Wh_5fWiYO@4+E{;VZxq6RA}w775Mhxx
zYF%M#_>ZuXxu<84i%Fn=if9ekFbT-)II-eD)Xps;vIaK`MG=RcVv72}6zGr^_yDnZ
zuR}niTkY??jUvYy2^_iXwRC?K|&4LLVQc+xC{9=OoE
zp)_5OQ-&-d7=O(!9M_4JF6p>`o#L`9EThuwtZPihOjXvL7r(;L234TTm59b(h!cz~UR;2S0yt#JVYiTut`iEP
zeFTerumriP5L_^%sCz@bdbai9O1uY>enOISV76qy{jR&+13sgeMdWNB3-B;AG2xVy
z#7kQRco(IGJ5~ec50PWX5CSdMq+hAe-!?X`A?*~2E_O~+xed6f6B^Fu!s98K((I>|
z?NP1kD)ct??u66~ZX4EWlEm8U!m$P@)>Y-L%(E`g5CbJ8O_YC{m1XNgH48n#K6aGl
zQ0_d#aSqy@7z<&@p$HCn{v#{If!H+6P2oZ_C=s?!%L1&jE0^N^=?a_{_5bsw*I{g
z=DU)n<>I%>rlz%o;-yZ<_+Ra(7aX+E7|4ZEASpe4F)*&-v9bN|r(>){AQq=FMyip!
zs%h|8;7T=TBSl&F+@H(MQ$yUt&c=EBr?pHyRZV4&S%Pk}uvq06lkPV6n
zH#}ANc1fa_|J4qZ`eOFOoT$r19z+55MpK8H%^g~SP9W5j8$y9Np_@j@R9-qeE&yR@
zQqvd>M5!dO=H<(m=H+R3(r)4?3P8(mFrf$F5x_z@mil%j~a&>8p%@yd&C>n#1pYawtGmY7y3$5AXY8{q5fjw&5f
zfK2Kf1Vuk8__HnoW{A_#2>Kb;<{v1!)hwKO-`)kk_iPRv=n8iB0O%-+Kdig#WK_LG
zBzOLRLop_gl$YTGkTYO1ui%0t%Md+w!^4|``7SKJ-S4_mN}7k7%$KuXHkd!;$h@cm
zWryz2qt|HC!Tk4eD+UQZa6V@){R13ACmidmXU{_!PAW?PBIw19+Mis}ZF^xP{m1_C
z>lrf)6`y3&Bh{I7?>TQ3jWrsemydLGN*!yCtdDy&75HNM^P{yz6P>>BQ_)gj=oET2
zZuZj&s*anRp`xV-b6~ul0+qQ0O~3*Q*{r2aB{+HC;55qaSSP%Zi3$l1@51r$)OB{0
zgOWp!f9>W)jjkn{uLUH}w(LskG+CKV9a(YSZxs#?y)a(%mUg5
zNQakD=+H|)@tVDu@;h@PDvA9uf{h7rfzOl#111R;@%NHmd9(Y_$J+Rz*}Y50i=9)e
zlQd-yzKc`dU;kE|A|srsCs3JP82Nn2-7fX~!V@Jn2Z49bN<8xN
z@+w68_rV$Ty8U6g)|^{RQc@E6QluJz#s*;AIykM~SP6xtlyMI}PbR#nTc%HIWo2db
zo44`jDKd8E8RuCj{_$dM&l`XJ?!9B1mzVM>OG|-^pY`u+@AdlQZ3!5fmXe_DB_rn0%Lkp+YU;HS0FrlQNw!gk^fZk?&
z8}rSJ_nG8uztruWC>3l|z@fB|@-Ni30F1^g)1GR;k?yEYC<9aiIO{9txOl>oU}-`S
z2@U2z7+yHufM#-We1DQv|F$}1?=%Ubp
zeCxubB`^daJ$S90mIAI?Ptr3|w?&%e|JR`r?lhnmFj
zQdjqxfg&OjC<;0=AEL;Vh8`@?rSX%??@_hPX#)0n*a?#;W+d{q>O*O%{WSXexP
zg2khyCZe|C$C~4hkMY;C3;f>E(b&;GV;iLGbA~l10A8vZ7@U7F5pw){&OeM`2Y|Ii(UFc>ZhduE_buEYR%qg5KKLvNOg1|!0T(Pu
zZ6#L|q>pbPX(iG=i0>*9H*{^%JSEX(MiK)8?IlL=83;d&@yRy|E((rKJP%GGp={w47S=eX@^VU_0XKM@hjyu+PfD$lyrKD&gQKGn3?q~q
zdV~%rXnF#7AF8(7gE=*8Q_
zL1=__R37pkLT>_HeS%bD@RDVSrn-PohK+Lqbl
zffvs|MeILv|6{coPYidE6Ffnak8pwL%ScGszF6VV;$mswXABeyhxp;kIE>KjGDRwT
z0F5TLFo~arU*^}J&=%PhZ@M}VFaP;%)U%)%Ho}t(^y{g5!PIpk)CZzd&
zKmG4gYvAJ@lt~cNv8+iAQSMVQol!7x(WQ@XNE$^siO!79xm6T(3;=$iNCt=onjYUl
zXGIOJcfid`B<>Ju--wD@hH@1+c!PL3iDMKou$%#WN4k2*5sr6dEXnB1K=~M#fTNlC
zeuySy@!jnHpwQ6kz!3)Y6*!E`xJOq6Gl6BSgj?t;e%VQs{e-rL25k`w3ya|+Rt;%n
z`(3zj;R$2|*@BgWkkDO#mjsVE9yN@us2hX0j9~Zv!>K*=T#jQHDSR2?P&rftWKPNT
zeOFL4!)a__ES)iM$v9W)vXe;i6z0H0X4Tm8TN$t})p#o)B(w^PWg&&UTW@olo8yg*
zB@gyXr^YINLB*FJmdnnaI3?%fLMK1OVJ`H5?Y{P}S8Uz+52kRqGJ(%X_%sO77U3b3
zSXpwP|GB@j+uSl|apS~uy4z{$=FUkjSHKUOc-*ul-c0Qd|&Mg06Xs`F#jtBy%X)z^evB)uyj
zfA~};Xs9}j-6Gnlpyx``i@~eB(GAU#D8YoT&1@Stj(@AX@%u_&uQbyzL8J<(R%z02
zX_qxOADw?jT*e~TrarFd=N%bi6v~JT61gT$Hm~B;yZ7&naH>$5HQiv!Ua{#3g+F@V
z+;gWAoD$&Zc=Zh2cA%^2)yIU_InJHrQ+iPQ>d?`d3lKay!M^G^H~0zrMzDGF1KCKZ
z9|kt4&TCdmFhuPSOn=%diSphZ6bihCws$m~b*oLnD_3w3L3H9T%nR?s{{Z2mN{RQOdWK}{%aqS$Mdq01Pl{XI@
z{8+2nJmY>W?0j5g@SUkmI4XzsKko`xb`If()_
z6%KrPU(k4G^wXvo`@l+Vo;#MS`CYTRrltb^OB$k&4BM4D&W&Z!a8Wfwa{>S_MZqV$
zwQ?I0MD2C!VH<~^oOKSi0lq9f=SPbA{Sq>{-3wRVJceFw0{JU66dnzk4YNB;NxziY
zTdi&2=c9D`c$Ct%caj1E1v%)$r;2v3PEC%3xDl}mfMyFU<7$SORm;?l{(66WJmUDy
zN_anMDA(D^v^U?ka@|{<@oRnmBi9wx^*D+g8v5yNexf`QljvpCP#GfFkGo_uX6ev|
zDlS5+4ApP0-AQY495WO}VgqWf5>q2-aRgz$LVx@wKF5EPmeg&eNdJ;W1zlEKt1>dt
zOwwX(mhK!GA5LYm8h9vgIMz(gToipU0bH9ehv6IG
zm(pJxk>;{(RTBD|DI$51Yb;Tv!0+omZ;UPWb$wxfS$@d;p8E;HjtNj?tiXsP0X
zOndE1(Agoivan{{LbyNLl2%)Wu+#o6&xMS
z5UQ9m-Ucp&qM(eak4(FLdq4suzUDS97tX{=1s$FB$Z4dFH!m3hN4mH=IlZksObG=L
z2P*bE(7>{yjio3!#6A7}{VPVWZ-~N-FI3TZ%0ej)<VQ5
zznhX(2`+5~T(X32$D*X8P=58lawRu?$*%mkgssj$mp880VA6d=n!ooCgdez^KKT64
zejI11dX(g)otWeV^~hpGX)lFrvl71x7xWbfB*}sI5FiaEU#`UNts_J(R#!7`Twf5iQf>Wrhz5W6ri$zIP8JpmTg}KN6bC0B>ww7PuS0
z_?e5z>2dqeoHGD7;+if21CP>aLy^k4zV^<}tEhK~H4&_11&nGKO|t+v2#GTK@}WQg
z&@-%Y9%#DXBzG-kU|3zUoFt9M48(v*1mJSP*7g92TSh5s;FQ#lygR0BMX6CL`uJ#)F&X?BEww>8>wd7K0P_Zv(lC
za)1EmsOI!|O?#eB%b7@r)wot%`w7Eb60*EoU^GRB^0_Aa@`{R1#FZZaY>f)ftSQHD
z<*HRz3v|@(9R5oS0Cg>Vsu1lqJPH6@heldeTiXM%<`O?)p}B$rXw&Z9s{!>wLQ{sO
zei&D2Uuy*rR7h6#0Xu4kh!`&F4xEA$=>IS(=qh}CZz_)g=vq!iM@JKk5X%J^5rHdE
zCJ^chQff6_-AvoZSldb1JR}SVQ%?d22r4eV9w(l5G+dnp(c8{uSMrU8Z%o^kvK7IT4ZTf|=L0QDWq84EO
zHUt@lC$<#{aFH0`7F6ylSQ*?_?qPOkHbudd$^@y2n5AbpS^0ykIJ9uBO_ZuiZ0JAD!piDHP7d6pt6Xp*Eg%hqA)G>{jB{#a;5#J;jwy?c
zIWrK}svP*lVycPS2gehcA#pWVh~v+7C%jb(w+8vdu#dIAxf&C*5!dCbdT)U+03HM%
z#*4&2k~>X6(gVIkZgo?jCY~Rnhl@8yLR;VXWB;L=IKB-%rk<>4$_~Y&FzkF8RTE1
zq7++8^5^|AO!qIwX5)DAQ6$llI}yn>YtbY9aXo1;qPVDN7kvECikVM;!muvt35+D)
zk^Z@UO~6j0U+13pT*~c;%yaF#x!#Qx6Gn-Ln0SH6NY4DEx18Aq#kyU6`Q)1dEx14b
zmc9Ob%gSbW%LuL~Zy9|XAgc;E)h{3@?&8p=jQ|jy!GUwq!s2#8QuKX2;4$Qvlf2OP
zY6*_t0Gsga*l`{DTL3I8+1MDc{V|1L1K87jo`>Ws{@nJjhCBvPD}eOA@TG23a&q!%
zehaiZL{pCx5ci*Cx)u@rq9ccyo)kR3TW*3}cgu|a%C
z3Rw=Uedl+OtvAcT%)FrRxWRzL8L#J`xs`h9bDJYuere`*gocO1bGwlgP*4DY4JVI@
zD6;`MJ%hvolJ4vHB#PDs)OWHtp08q#6{$%Rgf#CylOr8H{Q?SAjzux=B|ta+r3mH@
zKux|FE}(~(hS*CcI+u~O+BIv|K&SXl+e
zANQvfo&a$j@8h1v>Hi$pJdsimN6s^JkYXF-ZW2%w4fXT)Z{q1rzPon{1pk^@yGxfU
zp*O^UCSMrO4Z-U%x-4b;`(oMShvEX|X&{(sqVO_)2md9R+5{R0iSV63YS)Qfi2pQ$
zgb8j~+;D!9*_aGV}c2#YcJ?s9vwnUADI;p&sz5YWv;n
zd*1W!*PGF{JL?-Bxfo&%NT-hSy=bgnDTpz`>#%I@2OWvnFqyat0*3rI#MaxwVziQ&
z4APDvU9cUYFw-@5>;H#`+v0tc+JPevoEs|?@y*8&PqGK$
zHBkS#Ji!s=Lx7UH=*+@Ym|hl8WMr@5@V@NC#C_|s&U`8o2Q-qDiwaD?w*--)fdH~M
zP;gb8d%m)NYtIDLPDEtT(V*THp^NKZg>hW*{bG$Vmgd-a%>5tG#Bc{DpGTm_6R_q`
z>uYY;&lLXiF)M+sAX-Tv{T&;k8r!_g(|Yud_`*NawiY^4#8`dVS&E2u@
z8GyxL)&w|D(zw9-Q7rlMKZ}SPC#UWLMFAPn(bcs8m?AOB;b?hLRV6rZy9@^(pST=%fFfCqb9l5*w%YtHl}s$`;`#WO)Ee;sRdlYoFqkn@0qKqCqvpc^zP
zDJlC>PQ-|eWD8Cn^l`G_35(;G-LY4{4~9L$;<8qw*BOSaZK*3h)dr+Ng(TCQ;!XX`
zjR$&sCl`#!d|LPA0>DJcoz
zPXa(y{5zm;u2fjDUUvHNTGhday+8K?w^LCuk;HcJD3hsAeeII*+PfBonEU6-7>S_I?%2|7l
z^b*VQ+2n^q=`}4s@>e&Tf$)ZO*b4_BNn6XzpN_IDc?zOAATg24{l~W8s%uA55e6ea
zAb~lfPq5XA4+u`?_0vDdv;ow4fbcFLqQLz@hHq{qy=J}@xz4C3-~;@Y4gq|jGD|jg
zfBl+uvjXMYRV*8Pw?=l+P`qjgQ5b=0L*LVhT$;3|@mG;9Mv>efvkyqOT*R11#N;p%
z(aqVjSrrYv=3ynSyRJt#Ej%V5dpc=ALb%yEvt>Xx=1$|l2@P*@E;Kg47eo|#Ted91
zgEw7%SHdP*MMX#zWcIMd%?n&xIZM
zZ-nN{T+x4bXqDixB|Ln%4!5oXhSq+zF02`BAP*mbljcr!N-63c=ol+drli{L)YGWJ
z
zI&9wECs}DVMs7w7WWlSfE6Z5|WjJ{r7Hm;CQb6u;5*5(4BhQifbd7f(jy+FY#GT-h
zDsb|`bbvfFFVvh(tS#P1Ps1^ZGd4ja8dY*9Vl7rUJHdL^g3M=VrjTMg{wW8o(sH6q
zgnshXj~&-5uYd$`a(1TCw3W#4&bHLU;9UF>`4sg{5)!P?u1D;#WH&N0x;6+!2+3sf
z7ux&h2GBYhhYdiGU;O*Im~kCko0dDcgtxx7&hh<+4;#=_x0E3v3_X>)KDDQD3JlFQhlw;Et-f2)cu
z$_CWcl&Z*Ue^IQvXVf1!o*6&=L{=~z6~we~!)2#=gdONFCqy%3tB-quJK<&M!^D=6vAev6sJzQ%`UUF}|-@Vf0qzt6d{O5U^E~pAzcyoOok}s
z0D1GjKn5ctlbbxEZQJOH^qve~~f0piMNvVxnD
zGz_ezi|FXcr-2rlhnM#$3Kzm4ApD3M0niASdUN?aRyOfjK6d=RcWA2PYH{yk1<;n1lt6a)
z2gIN7EH1(7fj$ed7LT;kuBPa4xA{~lG_H@bJp6P-V^3IID#kJ|R2A7~XN!(uH&!m`
zH=$=)x={sNfzcf1T*2=ATM5lNXU=3)JW+-x297(1fUXUeMOWZkf#2~7{ltiVN#kXvlU?v}8Qkgj^7a#xW?B9tmw*J{DjD66_T6Od=XXt~B`zi24zt
zdP8uR`4T36A8Q&=O2v*Bw~D0U5FjC{M3jfxjch#s?uu9yY`xqLnu^A^WYfZUi_^mc
z4U2*u=h9szE;$!xgSm=s0T{m#5#a?{A}+;EGBO7BqFxbpS27^E0ul&O>cRkaX&=;N
ziq=v9sP>*Y!;60fPMXy5*TPsOhBxFt(q$>r(hbz
zylfKwQf%4kjEEr2yQaZQ$g%ELge-_2=pzyRqe#yjKetZfA_ySV@lpun6M%bt|Iniz
zMcLebjYmS?7lohA
zMvYPT+`pR?Om0=dT3wD7Zgp!flyvS~Gidchw2K>O#<*I;olhS2_*
z#Vk<0=U}qGL&1{dIyc+$_|n}9g)Tys($muupA_Dt?)LK#$NHDvJzmI_)eBcIR((QF
zj_SbnT#+~mdkR3&HOOkP(s5^Jf7cxOEz-Pu={z2uSt1f`2x#7P_e3C6iiqwxKKbWf36v1Cf^#NFhBd>snG0HyQSUOi&U}d<V1$#xeMwbE5z1$)~?>v2lT_@f~C6Q*eKRzv5c`
z9u#{H0ShTPU<=pVh@}SAAQ7qIo!t?s97CrxpVRULqL1BS83JF>>%@#MZko1x2DEG#
zr$84B{4BAnTk|c#hxNu?MdCI`OJ5cx*E8ZKm7~Sf8VZ<*$_5`7F&!B-uLyc84<8Sj
zOICk__4a+l>Nsn$|g4pfQzO-)ljcfb+RL8L;s
z9B?WqDJovuyG)-WW-Xe|PMx-D{fvAlfnnys`rnA7c^H15c>^VxgOdH$bG?ns;WH{;
zOP22{iN#@7IzR9UnZ*+rs6bkM97LV4#){bvb#qkn>^MLqEfzuEnHi6uLeXalsh{w9WcR?75pxCgz
zKcg=YKGV@1YjYuxr9prkNb9opg4SeZ+%CqCq7-LR6|AlA?Hb0*rlCls1Qam;eDjKL
z9LlFMrd2icn3W>g$+hRz262NmEmOCaOqjzmuZWYtfy#jzg;*}7dPDM-5AA)1NW
z5v?%plv(XhGFX;3DTZ<|ydE+t%hQ|-`b5J+VSEb4LTY
zbJ`Rh8DNo=*4^pmWB}q0Ya3N)GQ7nF)Hk*zy^0ukA2Qkp+8Jzc@f@IT^K)rTL(rR<
z_B{#m*)p2VJDDyaKe~fcW~SkoOMk`4sH;7xv23L@)++YW=F-#mRnx}^}(7(mwb;k+
zCFje{30_{u_P^m1OY|Do@igsxYqfBUTgKlq!*|}^C?n%^CQ_JMx
zD(`EJ@6Nst34Cb`
z&B>ar3j8~E5D@iXiaHynQ&_otIzZ&wknkGfnj>S(Gcz-HxlG#=f_rl@j+>WN8y4BN
zJh-PHb2ir4N3^e4X{PB>K-#NU%1#JWA2V96wf%N}hKlISlC6EY4l1&)vuv0oq7`hn
z|I9C!c3S5FN!4HMNQ~nq04(Mvc3_;Pme})K+wb?WTo7@=P#rns`2eyBZQ7w%GBCVf+Ur+FE|~@xsUU1C7d$X@NnRi`&`pXV?GEFXVy1m3*A(?*`e1==nC;*+Wy*Kp)=
ze{r(X!#etNV=-qOr*554w}>S8RdjUL7<<0Z8~nYX`EWNUFL$VY{aR}>>fTRiS^BP!
zWOO2lB+~(6FH|FZ1rUH0;Dsa7!I)D^1y%whEPTeta}HXd;%8>SHz7d{|CI&2`o1dE
zk~|xaBlmID6WQDDJP{4cj&@sQp5ESkw*M`f4(uq+UmH)+>)2rpQL9Zd5)xF3PbDeb
zJDpiTjs95RHIud6NOo?oo;X5w66rBM#&nmcXa$I=c^9NLvPrkw8cVw!)058h6`iUH
z!PDnqT$FIOP6-SkErqNYBilc2h<0BF@lSX+G%~njs|KA&n(0DX8X~`jIZCLe_VEiM
zRaIA~g?kX=g_qD?As$hQsA(PK601udYdh)FXQTXPYw@LoHr+i9pHSez?<`uV^|8M$
zRp#B;wut1)19N`X`%{vWG2weDUd!oK36l{;91rE%rp})_bsZH~0^Th#OCbrN=F^t<
zKbAebp~4k7*fMnQNZ=yfIjN=5$`IoiWgZju(oS4V
zQj*vhJ+Ru~RLxuw(0D^N;U|v78|ZofPQ2{fozJ-qtY%nP7+|%OOSPb5xzGxM+fsmy
zhj1sbr4kJ$xn|M#Dl02%KDe7oVz6Ykoke%4E9u?x04vk_ao#4lK=&oAz^s&lZ?RVRXPp@z%FYn$%hnAowcSoZI*B~9~F!8az
z=(jI?EqKvc{jtl#c)&w)=O!%m^z>5Rr1l$`n=8gCY+IJ^bsJsjQs+L%>;WKzhsvD3
zJ)}IYa$;1;e4{F2vDDtZOM9SK>@ssp^u6X+b6Z1UlcB2E>zp?a!p!6e4B8O)4CD}m
zxLvVvqd!Wg4x$#pUMFTcM&DFc^L+P5>uPTe_q^WVU!suxDD%!8?ZV7Fd?3gr#$N7T
zx^?`S*5oa|_b)}Fre4+5Xdjc-2!B+%ef845a|y>e3~EHC8uUaRH~XeKF56`x`|hIG
zj3#4l++2Ki^r6kGRZ=
z@oq>=+8=wXA|EX=I$Za_2^wyYF#92O+_cvBW^61I76`Z!8VdHICw#I)T*Re^s*x;b
zBFn`#BALzz8QbT{)4nuAGgJLj`XRbg3wEd03Lf|{cFVdnw6>QUF96*novNOTxmHo?
zX}!3!Q_RZsU74CJy_2?_SB
z?H#z-oc~~kcIRgscve^VmXmo{8%vaqk`on$zPl>VHB>dnV|nzQCr47AJxA7AHVzsF
zL0Wu|c$ei_H5e|M^o_{Tr)KqF}>e3?}6}wzc`jm|AU3>1cpRW~Pvo)DT
zbgl5{p0f%+^_b$kf)E`HI{VK1LmrUnfCB)V-jAQ#-9bgRx=f!K`ZVqk-_P
z=GN1n8;{AD?Xz(4{{HC|ZUEXFaZ(PGBU=!j5igpW{Z;zZ_9P4pxdHkB-P1{^Aczka
zB`Wbafq&=52?Eo$avo!QTtr|-4VoefImbar^zg
z!*zY8>oce3427(9+H=M#53OPDIOLxhvVvzkZgbzoi6bKBU=~u-)0L6buz2qE{-K^j
zc=2kza_I@Sf(dNuLZ!^uv{0{yNGv%Wd=hFlnuUN?O$46_+j1LLW>vK20^l>!S5rm2
z1K^j$&B%qr0e>5%zgOpUe}VNi>DKcJsv=uWO+{dNM`q&j{rmTmZW*?SV)R~h_XETY
zn4*~%%6`ocsn`yWgA_nq2)311`hG#p3e4wF#94*>T{PDM^UCJph~os}1P|j(?C)&{
z4g?fC*jaG+4syLMjlmI!l-)gshThMgtE!EfvdS#-S&nzhxb5dBI=tB~ITB(4k{{zT
z>yiiw6p?H83kpgaIHwyBr`HNN%GS*usg<00cj1PpLUu+@ULFqkMTsju;Np>1a7$I-
zxVxd}rB6v^uSWBaQRYn(bjjTx9_#cf*c}T_OOry!)A6H6I!x-8Ff(ozzsA=ae{9mz
zx6$kgjIbA({kG_+>)x4%?ea2^wZ`}i_RKWs6Ex%P9~F>5hIm)?{0h<{;4yH95ZMA+
z1XBE~#b1OwHT0v$3U!+TqGN82Mmblw@4c)o_S1b;w;>Pg{sfk1Yd&w(5j~Is2EWC4G`ww@w+o-+_ty
zD+a^+24v>wzRp_G`2+Mh!yRww?7+Fs&@WX9)j{Y4YX#isi8y^mHkOUu)68XuAX+37_zF@7f~xo8R~2?J?#
z!`YaneiPF*KYx+`qwNPTBqqq|rkSg!&zvcObA@Je{%lj6EP};#f9svlbFk@>{sEb}
z>{JTIA&!TU&c0MdtCmdY?AQO`G<&mVQLCJrCpbaIc~F
z6F+crK+4uiEUvxgnXCQ&fy{v=JCkJ3*x2lyCP(J~IjilN$b?QirStp<@oPi_-@>o!Jx4qB$mayRXcEto1)5bCNUU5>@-uVr`rA~u;!c>9&&VAr))TQ
zTRMY408IlO6{G(%W1GOlbh}h1tskQhGXST@LAZa2EY7$*Ny{$O>H*DC~
zEEK&;E;jyfp_;-+XjE`Qqx?a@l=q&39Fkx1c&So)0o}lE8YWgcGbnnR
zr!p}9)bfON!3wU|fb%h8<+TmxqcTX%&c3WF5;iVgyG9lgM&F4gmUS+!a_V*m3dXeS
z&X-m$3o*z(r{ZzY!w#Yibf)6@)VDCQxXzEb{1Gj)V0`4lE?1H?(n7F*SfpF-f0Nb?
z3izFO|GIe!zs0HrGIU-wK2LecUrNxDJm+xwS>&{<$%_vTHrGVm|
z#XQEi-rI2{;|uw#tzk=nxQ?OJs^k&4gE>5`ziHu(inbb7uG*82BTV)1oFDA2DJ$aw
zH-hKJyf!VFJr@{1_V;#dIY|gBJjd14)Cg-^q2aIOR+33;EbRQ_4IWQOqk;(jRql-{
z+Fi-v?3OCZ2wL#AJLuD3B*tIbGiCa;X-XK@dCAi^M)4Ki7QhIW?g)L
z@^~fIu#r8rs-+)4D?W1x>d?d*)A(5$gVeA3AKsgnSPuspQM?cVm{?iOfCs{Pc^i)M
zIkH#q{NNxb)cz@wybwYF5qlUk1ZYs8s=Tbb@&)=Q%5HjWj>N3bEiE+aYKtQx&_fhS
z&!3o-R0Ubwv?;8RX07_@%wC&(lge7qD5OM0pe`>Ion(juO=VQz;zhpC*q)I^u)l%m
zB|w8vA;4jT&@%v<8=IP5L{!Zg`>UN0W)bIy=w$K|5>Q2;D>Mv6_=TYFKrI#?G}Eec
z+r5GL3l>BGyb$F6dQ^plC9Aa2-)zQ-RSlPK`^CqvL^Ev+;UuOEDIu|ia;e&@mcsf_
z3>FJz#H;f1`&<^GH^v%Mpuz^xrwpkRFGf7Eu(2yGn_UDVM^8_WRA`7CUqILyPE
z$)GTy@)}3K$57!-7zs(4PN6VV6kci0O&g}zM$NkHm?AwzY+dV+jxgh+_L{P23{eD7V}leJCVJV@c#8C_zNuEEX4^&d58%
zQJ8qRFj9CkAe5^wOg~`Ka@X_}y5jouL+c;2*7oN3CFwBs=h})(jDz}6Mk7n=8?>W^
zPz2+G4HH<1clc=2`JXu-ex$p4Kz6#@x_~%gFqcuo$e7U3+TaSLkz%2Adlifi7Qo-Y
zOIl*6SSUCII5Y)|fE$>`%}qKWTw_#2jg3N_h||XS=SV-r(jn2DqJzK^(hCHO>5gI-
z@4hwIOh+MF(n2wM^lSXb1tNIbw*GGcq{aXIge4Ex4U$N>01)YvE6jKP89H5qd3BZR
zZSF&02(+%Mig^Aw%CNU7IYN8(+_`hq`r&vi27KDj?9ie971>K@**yS<)!%v(`>KjXpjYZ`6LW(MYfsX{0ctEOXO$o2W^ZxF*1#A9n?j>u?BO<=+
zxv3OJXw_8&S#5(7lk(bmsry*BXN
zrU-tOG^=;(U>A7?%@p=JGaqGCchl3u$vVZ+MxCpgfEvn_JD_)K{ZfP6%oBROm)ZH~
zy292pMWPI)Oq1*dZ@o%rf1EB^9=zEv24)_{E(T7@i+MxXjf^Z4X`CBWk44EP%?Nws
zKID8v?CQj_A->9_i8ruhLtjOQ5l$G&ePQO~NJunzd^bZ+`SL}tMQ6NrEDLX0%l4sH
z4b;+a$typPplW*YrJOTV-O2StrSR3xPPC&JH=-?3Pz_Y
z|D1#Wg9m1iG$LYfHr{>2u(irozcJ&h!|Ezw$xGF1w3R!Jt3!;hZ|Jl5;#j@Nurp*$
zwH%Bz7~Y3zI-YJ5U56n`2&-6p#Ec_gh%57s*4eXX@dP8Hqt%)jCpD1->XRVHc|j@k
z`{VAgE*Ax1Tz)!YlPF_(p$tXBiDi(0=FVNaBD$7AD`*{Z$zn8euQ;4;i+$C3p|b~V
zKDJPuL8(gnk`3Y{c`@X`URPh=3~|lqFg)-Ylt1AdO&YS|u`Yedzot=3c+{$0+bkP+
zva{&Jb@;6a`h$8p45y6^*Txz0=PRc~J3seS!2K==%OmG`pK0tsPq!bKl1&
z6e0t(;p#6tkY5H|_zP3xNOaFj=H>&o)+yY3LTHE>ocn1IiBq(`%^oL|o!|f1*z_4o
zX%Mi6o`
zZ~pvdti^-eKT!izg{g349hQ*4Ku1y>4X4F+!mP&SLrG|wKRZQca6>YD*Y_?Rd
zz;TffI%%PG`;N0Le~smO85vTTL%m1e8wV_bmzYE8ytym`x+Hv*=TJW++EfYZW9h@?
zD*)slPw^Q}X-KQ1h61p8(GAcMfc(}%RAv3$@78+@ICSK?7-WSgybp`(U*j$Ow(}Ci
z36w#NXmT3N1ZP3b0i!{w-L3tOFil*<<5<0VwYHv~C+@0dbgBm;80i@V0=B#d+TG
zcg(?N!_1Jef=_nx)q`^uEm{MSRlF?@H1-Ov$FKC72G=~%f3PyDq!36-Nfn_G3H;D2
z$$Ftua?d$-T^$&H*a`IXn9fI$r`XhVVS&R13~&W+0OS(%{wen)%{@w{-VtDJ7QU*$
zGtK)tDxpwlH*W7(hALSN(nZn>W2g^OBN{5-E#-N24y6VC?5(rDkGhZw0&#RI&d#!s
z`k1$`=AGu@;NeisVh1UrzrR1q>M6Kwrp=srKQ9E-GBm2Uz2>trMAlBZxLQ?snn#Hz
zs!?Pxs!kq;*$BBIeBT`Wg7mXKnqj*^xMTG;l?7JsQ?=&oeE8C-v$E1g
zzT)pO&tx->D_BZr+?JTAgkvW2RMR3vWlExB#$`)wC4ip=vL|F;0$v838JnD3f}@l>
zNGQK>Uvkq?rzOW{riEx87XGwtJnqz|Ir7yUIN&I01?0`{-8ai;>MrMbJFw=d{_m&0
zIQ0_V@THe6TUYA&v#-EM2nw%d+4lRn{P~gp`sDN40T6A*?WGqt+#XuW
zGh@3-PM6O$7<|^6Y7Kn2l3=~?F(+&>X@mZn&yo?|@9KzvGo)~nT
z1^MHFTvrl!U5v8EvPX(@JNALK2ftN2#`zji(EbF4k34ha9_6?#q&4IykFc6a(
z8X5V#aPyZ%E1V;IJ(2K6z?J|b-_zPTe8X_x+Cp%jBMZyws^30)Eqs{B
zk9I7(sI=G}_dm;7)ym5y_2*SjEntrn$|=$wdkN^rvQ~_M6JX&mD82w-X&byI%mm^N
zR)qPY>Q*t2di@334Pd|VPePDW0N>bWhya}5`1S(3WMP2RTd+xVN^#zmfgjqHhWKXQWpyZC;2Mg_B6@5Qq$-rnj=y3tnSquH)7xZFzWICt=Ao>ncKL%j
zSVL;}DiN%X++>V701B%DYgxv{;}rN-Ffnw9Y7)}VPyQNK^!!b^%!~iXO`>u_>ltM1
zM9nti;nH5?@a#e3NO|6i@QAjtwV40TSH~U_-U08f#rp(Yv35jj^LUf(2&K8w_5KI?{MvLrOJo
z!8LRem2MDPUaDwY^=hY?hu5NxL&E;00kmR%jh(<2pmYXW8R&N_(TdV`32_;iP9Y|#
zq1P>(fl+L->*NQu>4Bq$5PQf^Utq(cPr3?a>0-2K(B?Kr8Zo}vYd5txtmv2@u8;+^
zM>Vdd20b$4c1TD{s!lJShs|VOPL45}3);q{4aC_4jrYH|o;W9L?jHV&VnHphX|e9F
zOXJgWa-!iUx`RH2uMj!`HH@|n%hLp&L&SkcZfz5_zqa{lK$C^0;+Z_$q0%?|8=
zb|%V!J^rHRxYV|$o|8KuU*9=t4Y~rn1=ULKn@wlw$X7p7ko3(U&)l-vGys9(e=W?b
z0CZdcaH>6sLWq_ldj8KV3jZr1S64k>s^T!n+H
zWf`>M9%xK=HvhmU!h<&hxIFR{`6a#;KE$;^h8A)x3M4
zOjp>yQ?7>TuMi{d(vxuWcX@R&45JD;(3YJ6b%cwQ9Z#Emu`uUtrN!-jkTW8NX(5&z
z+C8xD(zf@h#we6RF8PJ1sO2E0ym7IWeKf09yngK(D}#otMg$4)%gD%NW|ROXLu!uA
zFlf7kfk(o)NuH9Sjl8bgrt3Y+oZJi~7~c@S9?kMmU11Q$4DCa^EeE`b;$)83zka>5
z>~8HXjI}}dp%>6YPy0Kbe?4?D>Q<2a>7}Y1k|v(FG(N3Z_ZBm0md4ja6f3A-*%|2m
z_nL4RVd%sJM)^$XVlra4#B3M2^O^Z#JX>W
z-}cpuPOaitF;*@uzhShNKT0kXY8a5}dvtV`KC+~3gj-S!iQ}jn5Z)Lu^s=YB8>uNX
zacCnb^9vB>!IP7%KSrtD29S;%U{CZ_Ku=ME=*JwovQ2~+WdR#*uAh*I!s5G4Q$!iV-wYMS>4Eq0o!jQ(X9DAgf463>>j3~MjqSVw
z&I#hr&(9u@TMKabN=cQ#-&%l6G%f-0BA&E~TY0@s1zusKrIUal_kt4=Q}Tz#YqG!^!vRov8I-J*si^
z&-!8y*6G%%n6=^Tq4wDIw#3w$VF?{4ld**%6<5fG_X*3haz
z=$XAB@4arQ?T&p$K&-VvGD4nP(}ZZpXBZbI>Tor)VDOfObKv6f_qBazqW&|Ga$faI~X;LzLSw@tp8
z^b6$b2fFQE-Pe&m;U9k8ej~q626o?ecpkSxijERXA2m{3OCpLD2S=}
zKHPJ!2E91vC>_#}H>yuIftW%&v^B!(e6)>bRzmlp1*Di#7RZieY|t$vMYq73~auJm!T1`ZdAIG{2C=^
zG{&c<1Dl4Ka26OgT*m66hckOZpm$Lc<7bWKJ{xfZSaRcqDrDPxyP*pGwUKeUbK0@$
z$!=sudMGaeqDAhrq+nNcC&0#Tl7p||{;l+8wa74*0}4o;}OVJBw?Jp9-kzAM#P-^EE)kN!o>XDOtL0B3#2
z-vHQwtH)!TmW!AW;N#WdZ_}BRR*U`-yb9+uuEiRAMO}_R$NTF
z{NH&O^p)hdLy{pnmZzAV&V|gUv}A$g(gvTZeaS2n4KW}jAm$~-1Ei7U`9zfdMd*8Q
z86yuUQZg>|6mAp^=8n16gZH_gpO3%%R!iA66*P=JE{>z|kN8)tSV3F|p3QeagqV+l
z*fqS0nZ)|SN6*jxR-=`)ub>&|7zVC`LHF`coCi4}`T<6G_koYjAt>-Jj)tuE1HB0aJra$%fJ!OjJ?Kj
zmzO756;l7qP?%El@xj%)tA@vZ=YqTwI0DeaxFAAi$IhJ=={KQ_2L!;2)FY%%QG^11
z^zYJ(N~E<$PuzWd=irv6iWP8TyWwQcj+8a%bb&XtS4VN-7ol9OBvJ9&q9Y4m+W%CN
zLe2^f5pp$y82TEy1z}#``L949h+D5Z$Q++D^WL+@#)A<}LhFu9qhxmMd*}3OZAA65
z#0G7;Z^xvNTQmSP7N=3jfmliEj}VPbcp2;(MHKvk&)nT@6_sc5wi4_GWP2?XN`bS#
z?!3M}WLuYOd#!L)KgZ3<5o#Yk0R>^^(iA+pxB2HDm6`{
zrUQKdwk7Lr&nBHT#{BUkV)B^H%{!}s!_XKHl$WJ_sdXd}v~bR?e4iU@r$q1u29XDx@VK033Y8v1pkAQ&r<%mpxdA$1lPt1GeVj`Q}O57HP{n;H6Yx-u*+E$NU#;)rJ(s6Evbkq7I|X959JYs$l+);r>n)#9Oh
zZ?#zE{2?}WGBI^{*Dj%*cUM+HVG-f@&_km+pXH^>icyX*%yD3&kl+Ne{jG}6bQOAf
zdWbf{rPPcIKn;|n(7<#Ck-h+=&W1mN?{*Tu00~TIfxF$~Jdy<*p5~ZArN1KU!7ZC7
zknAER{(eo5g-FLI?
zharCh1-uFD-rGo_J?LtlgY2UT?=-&*3b3uX{v^+S)1dzswxP`m2o)jm=Ax0VLz8c#
z^#xF>qTDD3Oy~A%R1}KrI2-+GeEv6Da^`^${`A1E=ySDD@#ZV8j}F=r^6}3ZL+=
zu51jF0OE~i39UgfE@tp?WU9iC&YCHmLG=+d_VCBOu4^r$48UN^acv673Ec
z4Dd@=NF8|QP!sD?&DPw*PtT8ZXH*!&#e=uS7Mmpj3ZM`IvXl+FhGt)|y^!|uJJSOv
z;WqQ~7&sZy2TG1ks?IlPzXZL%w7g+^(z=n(la24*(Y!Xmg3;a^w9hf>QMa)$z=KCC
z-a>i?xE~yjQ1qnmn2n)4qhub&XPD~l?^$(r@vk&BZP
zSaRfPe}Q|>EIHzbb+-Lk=7G%sHBC}RNZ0sRY`t)>?QCzv(o*}1(-iZ#B{?m~niKpz?zlZwti%1&Q6u(Jnv=V>T+
z7}}ff?Cd!YLyUj9eox0;94GYg{ozF<1p?~7798
z8ad5TX`$9Ap^hE_DXesB`d*$@6Rp5);+L5P_;g|;eJ4h%GGY%x&9A}QNU#qSB$;PPb4@N@NZ$m0p>#NNKe=EqsQAN&g*ZpA08NH&K?&+WTkgA$^J7Gm!_?0Lup;pO))NTa?C^Vs;SYKn7LFvdr{mA
ze!SrDij+!EHlwDonYE3BD0{;JT~W8LFu~OER7rfS{NR_2ehUX2jL1*}@^{$cV|pu0
znJ=L{Fb1-RFEPb#AsC$;)+jJq(2m-)X~lEuW3Jy#6wSaSehxpKB*Q*NC_eqkVJ>srO$B$cZ~a(srcU0I`n_-Ff-^(lC3=$jnf`Eu&eU7CuvAa
zNnJo8O;Cpl7>TobzoNV4q1|#ewnfbBLV0I?z*
zna{Q`QzV!V2gFj0q(-ocjhqB^?I2$OL$%;Yz44=lhu#f3AsQDBq|F-T3bF0F>gt^66C{lb-B<7i_Mx&;
zSD7mQfWLLNYgZmhaMXq#q>jL>a}XM`89j|dDUbgL8yU}ksf|o&#s4=pGVg5r@HLH0
zhe7WGX4sm*(gOf`dlpNtUw;WJXYLwy#2I37-Lk*<|aZ
z>~y%~NvlVWX!6Nj-loHy)wtF0hPkHWk8j>;O%eMH_4M@<4C)RT7~BB%?0i&k>OJS?
zco)q%)~$h>uE(G~O$?6P^|5IVrzbO8J9*aHHbK`;i3Rd^x9Q|h_0I~ilKtIs{u17zEjqJFlz++N&CL^mgX5Whw*RluWPUSttZaauw#mQGjryn{2bf6S_ya6O>t2?WuL%8_+(P0!t;O;{q
zQguWI<8h8!+<;Y3N+9Nn=6o?#_q(`kac@kGz*dIQO26|)rH_K^7z*cwxK^M5(#z5E
zy!`9yJuPAtSz>#hX$7qF{57*?$V^@3l*0z{*)mS|G|EhM+6?Lia5aR%#+H`xpZV+S
zch>@XO;)J#Q+O5b{XaMV)i?cV{&RogTDd`9a}!3O0&)cThtNxnh1#o|a~0woGI9&N
zAa3=&ygJ=A6@bX`ukOgE(dnpVw-i`b|NFv$cz1utrR=eC-N~HFLz>1*V0;x_
zPr77$O{}-Ys&iWJN@Ib8fENsd?{przYy7Y(XsvL4g`-BVk*@utHEk9`kXt?%dYYdO}_7-iZx^P@2WrP0I&7I%W(b$Ko7izDrt3Gju%_
znD@EltLLuhiJAWVyk&Wwv7+5_bIqyNjkn^SNM_|3v?pzl^wrV~&PYo$@!fcoh)NXr
zROitlhmZw|oBK%S)r%l#;qfrW
zzy_*%(W6oJ3sBVriS;^6YrmZC5GeGSFi=qf4Q!XFu_(;rh;36SW&O;o&^67TpfP;Fy9UVJUWSfxnUz{9?kBgXoUf@(DI7pa23|`E;
zLO8MFLL^w8D5*?29c-(7v$tpf0oVKFX$1wCqZ>k5HRd9xseMdTkPEyy=L(yB+S-g&
zt5#7V1Wh}nBQHRhel1v<_dwNNqC8(@N-sCSnTcfL$!qvg>P%k
z(A_`nUEit}R8R=nSPz8@)y}97k(yh%YFd|alF@DX#hP<;Un|CH$O^
z?6^8nJJVEgI)7lM@B)Gc9@k>cuM8QK7vSaH8dCHu%Rnb#?xl+tDZUd}JEIUGY@4{b
zt9c;9`O{do@21!xt)kZPZ1Jkvo?ak`WOb!MZRq>dum_O+;|VdQDGrKS44D5`5EImQI~VVmYxr)f
z-o%OQfz80s_a8j?1<{z5?*jG(0{tY70^^$b0d8T1h2Ymn+=Bk;=+*AJ>sF3>mQ@$l
z#je*94~i*X+E82j8HhP&Q%Zb%F-XxV49+%9%QAFYqy+L6n8Uti`Et)+8ur8YWNRrr
zKqYVei}&vzoZooxzyT=5HGA(Gu2$>ush*-|AD$Mf~FX+Vddz4`3}
zwA;?LwzbumNk(^Aux4G_bMNCCkFn--1#u4k!3U+G-7EszcR`0h*~k+ny9HX`|!~niXUL&?evOCYnw&A&S&Xj+_7yz{0@5
zs=8O_fGB&RbgyPrPTC*Km6d>gnpzd9=y4od;8kcBhP(U;jo;$nBPb|3!?on@(D+|63r1}ru!1iVy)RAARvZ0t
z4aUc<%Vt9aD}do1Toao^Dqcb;#KM5eIHJ1JK2U_86XWO$Wm8HWL4rdN!WKHDWOoz-
zA|R@Q2;}J9sGZl2ZKpe=6|47Sut%)!=xC>k-nh8RJr|pIz1+N(duq{X0z8dmjv-)%
zrFP9o^ADK^^}0uK>=<V3A18FM08{s;ADJ$+;P%CXCPeVV)@3;!1
zN80Y|1C3ms%8!GlAif<6dv{flB2c_@g8!5`$A&3&i=MqY;WBKzv_Ye13SVBC?x}!c
zT?N<|XNKAj`gdD?#6>8K>kclNP*K&|Bn2}x4V>-=Ip*PAql4T$J*<+40&4+e!)GS*
zzxJ7-e{dhN_p4}&AXp9aC@2~atAX#O4qr2RQF623I>ePnbZyz@z?|$$gM%q7$0k@I
z`N4rs3@+rZ4~@|kvQLQ;+V#4=^%mTHN
z8}q}Idn|K79U(HOx(jt1!!qdIV9!w1^WD&2-u-gq(C_UiYO}79|f~yB`hWL5ysq*`JF5_P#0lx
z0S^z4J2XkAkjpVFI>?XTBv7!2Pz3T%ole{c4I^e`v)gGa63NL|kIh2DP`@D@rE
z1oKDPv!JGtm_gztwDu-uW^53XfuAe$l-5>r?a_xU0z!^3?0SloaA#p#JlhK~hL>Sp
zA$>KpS(idWLM}c*SI+Xy`N*`t4KD_`0r2FuPo6q?68mL)C0dxT1$l5Js%;E=shb>7
z%7rbZb}FC{TMAL*Kz}go+6*T#^J@lJJhEPdO#b8mFrW(Vt1fuKGapC-MOLl4e!l^#
zxYNOEDqjG3FwfaRhP26PU>TWUI$huMKnLWLApv^MQgoB_3!XmT%Z%2mjZfVaiJ3&t
zZiZ|ToJolbRsP&^4e#E~2UBthCPSPvIKH#hhHqAH%&k^5j6`D7W)KT=Njr&v73}ZZ
zK0Z_2KjyGi7PxUsb_vW7l%9ry4qaHibMbYHNh_e1SS>}slfHj>!A0f`v?t7BKT?ka
zfA%|PG!!&}VHa_KG@v6wxJeR|CP;U@jwU;Ilgkh9(vKV(Wy4TOl7a@^&Pz<7rtn-a
zJ;iWoAjKT28D%K35X+;icvxH89nNVQ;{hM`mFw5JFjn&d(vUG|!30kR=>7IHXVN~L
zwStf4U*6E?uU~u37CuneD}2
z?#J@>^#OEY8?4?kkXb-`npP~dOE4;UQhy2w3h1o6T>UTpPaSD%O8RoM8(=R*c@0L_
z1k5h<8O6}hoyI3b=3@js38+n>^TtPrk$X!ypWx+5goehG-ac(Hu>Lfp6(x$}#Fz}e
zyhK)F#aN+QQ$Xa)Z<{subh#su31*SV{PSoVaWw(K4AWm&+lo~|F;E|!d_Hy>AqW`b
z^gLAxz?-Ey{Wu2p6)7TRn)Q1+pw$<|E8-RsPG5um!XML`u#AUmLRQUa;)JgBAt{&roSqgxvt|dTjT4$qe66aB2F_Nt&r@Yh7l%m
z&9I-inALD{BaJb^&PaN>goJnUJtosO%asw`dQUwHkPo)_HrO{smM-l$yM*O4E-Q2g
zQN^kwsB1`62@Dt0RQQl5!s&E4-#M!>V=KJtIyuJ`UB708sSJ*S?&}CTU5ei)2bqEZ
z4r%V@D~S)rmZHBx^5t$_`%*g2{;P-mNZNttjEn9IuCr@@P!9imSJgCpkMfE=KQv}L
z%m`F%nH{tzqC|C}P{Z-sH)6L0g}rx&OG%X+H8{h?*^;cv`(f+!&+M6%eZo%+Rd#K(
z)z>>UF!8XaP04zADCwM=q28m;`^&wox7%c10cRkBh(8Qm1QqYt`?&4Mt+i-!Y0Si5
z2nr1o@Q>M-AO#llUGOn+;KW!D&24Ql!&};M{rfAr6~sE!&l9nqe0}EE$zDfm?}48P
zD5U8+0La3Th9es>K-nhH$51Y#RR@dj(+NxkQQo^3cE{N?awSMKLq!|{A*xx(9A0*`ugN5^z5dWgBpBKZ_yH3x}Z2s0bn3AMqSxp{ds
zARR0RdCOpp!Jam$E-NAJ2J~3)SmG#X0Tqe9EA7+KiJw1dW)C3hhl*}SIW?9rXHyI+
zxp1LMpBI!Zg~(m{N-*E#6$c{t9E914CkZ-MkLQgc-hA*G`ud*UoAboK#x~T)u4T~8
zhD+rcl-dNW;iASbuHnms~XaOkcv$T4O*-#_`tJ;(mY^cx{JLYA!+A3Ui3kcZFV
zMdqP2c-k5XQUpqY0YZT<*8IeP9lAmwu6sraC_hKeT#r0a_Wai;g`PyahlA1?M+{i4
zhDUB#-Fmwz7NNDmhZpE5ViXVYU=YgfXGU+plajWm{%ux5C>EX!O=N{)Vo%>eNj;D*
zkhXZhi7@Yph2|O()gZ(H`E)eGg*YzMAu`D=0JG6wY5>*==0>;N*XHBkt
zEF8SSCg{q0wz!@pAoy^SQ5ZQzR?-L^D<^LJT;vT4N*{Lzxby|BZMsA9N`6iv#{i>~
ziu7QT8pdrPE$B#(SlXO#H~H#UYFWSBkE0M_2{io;ujx~FP(NZI@mcARk{0A9bO(jg
zgD9&PnH=IhhF0V2_4Desj6!yJ$>v``;TKXx&%Ar8qPk
z$Mtm8Pg1tw|J=&dGE|(ku{A<>r3|2qIx+|Aq!`I3w9&9pr!mFqmyW3_3H|KJ0Bb!8#6^TDB*m&Llo%JfE<=$#IeKG5Lr-bDT7!P
z>-#3q_rD+=|`cNwIV004h;9`h}xaO`d#Li6&l270I({KqOo^^_t
zVGCV)3nm5>p)aB704L17u6ACPeaAA0XHbX=2oN~cy{2YncHe|yR018+<~f_Sa7)phsPFA=HU
z8)~y!L5PwB4nI}E=fvb2D^IHfYS?-t-=fusk_JZ>GL$%xh<36kF(3$RKSuhHCKzRQ
zQ{%g|X>o(OYZ?UBEoy12X)LSK&TgrfIl&
zmY;frZ<}@k~s|`RC8wIdY-@`dj?Nj(y$k
z|NJff=|CDAhBW^3=g5oy_YMBvGq`&7zvLN&$6#EI65K(!zY>FfZ?8WWU%c7EK}}oC
z(=;{?EtmFIy99kDO=Q8?pGP~sy;2bORVQ0HIcPAAO*y}8xu_^5Jg01MMWY6C$GfQK
zx?xK4XnVx1%BOB_3pd+LF)O)|W^ez`N{*db^_~UUk*km+|NOH$R2I+BfKXOu#?c$y
z`Z&(f$7339`cW?AIm}p-P<7nnsou&txTPT#Ie0?Qv>{_785BVgP@o1(K=rVX+J&hO7joER9?7nfyP3?>f(8mT_g$Bcyk}%HZfmy
zX!CUsouoL5T-&(slQh~H|C?5i->PB}?)F)P0HM~Ff@btyeUPG?0Ss;hsg5f*UC*icBTo)4eQOpDKQxFvJDluT$IE1$X&3zDB;OVnMWRPxpvojiy
z0b204jqeZw@YkPk*&P<0oN(m(BYhS}H9zN6v;}YOtWDQ}+Uj_9=qLoAhD4R=%&<1E0Rlw|
zYIN&txH#(OudCQM@6cf;Z^k5}5aCcj)ME8h79r+j513MPbtoXe07|-Ba0|i8et!B{
z9q?=@n}kPtvY=;-XM98vWdlp{|xsx;(z@KVewpj@a@Ej2SxOraqpFAHQ~-sP+Q
zSwint>=DxbGp3V+fnsNC-#S?ADD)SZ;*y#XRmRARqbYcw99S_w#2AVrypNq(PMCW_
zx!v&7?X@;SEkbFk$fpeIpqXSC)k2Z1>+^v^kzJ2wp&^_6(5YYBK{jmsys8DGwh44|m)lFEe?A0qJ6hv7)xoF1mqj7->@N;^WKLL5X@Xsu>
zljG2FxpunRphvRjHje#}335GczF
zRPr>74ERVhkUX)~t0`H~2xA?8wyt$6qR4#UG1o`qW%kDDqEAtvkt?92FixBg5HW_5
z=6`AhH9=u-fR0+QzDUeT@CP7?B~(1YTW#MjoNDEy-VD<&KvcHa?j2NWEK)o_0}~{G
zMoqo^0}hhJO*YvA-4(}G0V_&Yi!&^sPU
zWlQhm@y6oWf@k|?z;@9;%lg30*FxGr`Q~AkBOzhX9pyLvgg?01;yPueLYvC_4I*rc
zi*84F6%Cz(sDR4f
zkwcE8fe?X1OT7`g9LHSqCYFzD!f7t
z$bM*UOM*lV=;VC$P(_Ami%}E~gp#xdfqY>g{S8f9rJZqWO@~8ZUWdeQDqaD=
zI5yBTz#s2OxJ%srN$0n@=5^rE{BQW^|b}Sb@BTeF??8i?`oTXk3M(4PZO-2;8T*yDSUNCEx
zy%P7LE&Clg_M*8Z89FL(V_>t$ppMV}iB4b|*ex1>Ase(@sWsJhbaWId(yhwM7t6n&
z!qvmUpdJ(N#QDeG!D4v^^p|wu5f})~b3*=!W3U;LFWShdb&ku={+EvMc%B%~UEzg+
z6`1bf34}1a{wSP~lu(GXnw0%@vTNN-P5KGbSpUb}R&n2PV^iG6e(nusFjQ5?Aorppn_Yy;Q=A7YW-fByqa76=Qk^Mnp@K78747hrbv>jjn
z#sTT!N=HSZF%SE6l+sri7=3pE*F@5L=@;OGM{ppEayKOQf0FolN
zE31(Sh?J3~XzkEeE@ZFp&(4$IX6edm3F{-BM<{l1`?KT4Q}`QpLfw5=WbQS$?|=Ln
z?^bWY<(Uk$#Ej4Zi}-lB90_`$2Lk>JX!%ya2RJ7$lOMLvvmcIE1_MpnG0922HW`cR
z=bxwAOU+$tWMJKhrtA#HqFC0bC@Xu6mZaM{ssB!~&B^%;y|=7N=11%^8fb(A5?sqg
zw2yo0Z8euf6tA1e8?Fb>w5Cy|i6U5BBm(EoG=XJ%);Wt~kbSWG5
z@(a*FVBT0lZT2g;xlLM@S%%a*;VW>~gjOAocZDADwy*EO7i32yAd?3#npZY*wdS9{L(o2%{Nd7uhKBZ@fa`B<
zsQv!y&$`HZ(a`wP*RP8}i~QEMt3h#V`)v}`ErWp~Xv=ta#@%)H6JvHR6N3$e+(Kg;
zR&~5G(?SQ5m_wq94Eo1Q@2h66-Dfw76f=gekjNNC#EqP^k$+4RTNu8r$5!J}@5duk
zzG4W>0^z#xKjIoKB3$>D3O=tFQ2OwzMnUFCteNPNC0iXI`Gh&|18G%w+R*os&%b}%
z>X(r-#TaB|r;K0*x`e9vB|WH!m1B0o3wPwTF=7{SqmqqiM^2W9R>Yj}XnL6FsLGE3eF2px}~I{u9d@FZnZ}B2XkJ
zB=vp%><)9%e3i>Hc%xz0Gy(7L*b(LT`SWLO>mYO$OM$gAXbhSOx{CK_e*JW!i4C|T
zwBvMTyDUN^tBy;PQ%}-xd*ah(K)Rme4nz#&k|%xYYv3)S1C7A|A9^$^<|TtP
z+Nc1YluPfrAG5zPCJ;w<=&-yV?o6EUY})TS1cZbnHfu`#`Re2lyx?Biyzt)LyB-~D
zpL}Pftw4Ep2=Hc52bJ$sZ_kftfkjx&^#L)uE3ptvY8_I%d~EIzy@;~*8U$s{Kz)2%
z$N#O|b6SzlroKvEUI+r4?I3+nZwKNt7L$ia!q;UzBW
z>mWUdNX4Oa9A-lxgJeuVObBP!9038Y(GXN5Mwk~8tGg04q;k?eGtB*V#2k$(1dXF8
zk=mx~eK3g>oWC_`5N+X@Qik`O;ws_mru=hsmPk{eT<%n=xo^OPX5yHQ3t@plFQgf;
z#9YLkN#`r>$$19Zot3tXRP%~Ou<57QKdH?Qemll`^}QQZ>=e*ig=*-@k8(L>(cPy{
zOM=4bhr4$Ux1e+(=7d|F_AdbU@Dsz5$V3V0>s7jVAO<1C5X9LT96mSZNrji#pj|0~
z7R+N3K?Nx~zxI}Hn~4i}u5A7?Q0IldEa*RInkl6_%n^$G
zt}!hLvke5=FBM7+;2(=!N3fp`mn0AIIKJBkvkC<{mLc)boD*s+-i(ir0au
zNW<$Mx&EyMSdH73$eG0$&F7ECNPZ!hZnC~g@%Vn+PJUH{I9R{
z-|J*8(W2ijJh!Oo?NqY406_x0!pcA$eH|@5qA|Hpm&av4!G6bppX+F>q>ooJU>C&X
zuN6Q$A~POhB;4-wwE#Y_tu8s?(O-f>X%Xyo=(E@u=#Ik9ehIHS4%D8F(g*>;r3{{%
z|3Hi3_$|Pk@D=)R$y1X@nacR_h}>ET@;CYZa3gfhJJl-ylwbv9MeFF{Dn*N^aq#6L
zT(+}l{Aa*EOzs5)v;{q7iQS8CMez>L%a`i|=Q5CXy8wa>_O=I&Cs~x@bjcUWaL2#O
zKX>lFLu!`t8MaQ1tIUQ&{5QpVua(JiQ|W(mo4l3i!nflDM;#5-?Au!$Zs{MwJD~%W(yidi!8~^noM)6m
zgvc|-j#F+Wpy;0=DwBXIoDADV4b8eP-H+?n=sa^Fu4AXT78tz5!%r6hVWC@v@~6Qw
zWtD7s&lNXbR6Dd-MqWbaX4muQQSqb54LatXRt7o{^w}wekmn<>N8)Ay|1YW@tod+8
zTiC2zc!t!FqrT-sp0c+SG0VSgNd}>Zv_(lvdF!!*^(zH-naRZmBt2TH^Fw^dA!DUf
ztA*t`zuZNY;0NZvuR!&GdaO1VjzK!(D!JQElYbS*pjW*vh276A@xeog{t
zX@RbYgFy)tD4u2f{qULysKhR$sRq9lRvzk6)p;&p2r0fAF)d0-`}7~2QdeLxh6;mJ
zj+Fb4G+RELva#7lBsfLTD0FF&L|0McyUb_yRz~0{}t^kcKHT!6A9ZIZtO@W$|S&(X{
zVtgZ`5FjFX9|uLy>cOA;UtS{@AgqZ~5O
zvxJo)t~@S7LIo|%WkgDJBvqJpUf%~`B=vZeNPcSuCS`^tpV%obAWqt&7a*6h)Dj$m
zz&F!n&4_&Tv=JjPX%H=LdD2^8f(nCyws!*<;!9xOFh})Md#0wh`QBSx8k$<_PR`J%
zt#upa!sque^P*>f`-noDcfQY***knGu4k*IvwO!&JJTJy0TM?oa+;nZ?V`6FT0DsM=sKo86
zx_-4IQ*b|{y>W^5-^KOiGjyFA`NYR7YBz~-z>`W|Wqfl~G{1uhsEs;x<#9cf$&h(m
zq@RwsLI|87;d&0IC(pcj`+LHp@5c=;2Nq2kjPu$!;MM|cgwTE64}Kd9YhBn8jsO8L
zKmekj;jHom2S8cdq#OfFh_DaO)<1+O^^pj0*e$hl99Me!IIj4XQ&`K4^f5=3h~Iz%
zR3b*qG(&i$hW96hxDGOKBDJUCfy23_ts@#54+ZW8`?Sawj^u!b@PJ~C9E>1VFb_-`
z&1Qo2DGUNaF95A^5q26g`t^voJQM=H^AxS`MRmnQeF;@4d8nwXH|Y2XQ$MhtRnhRE
z=O+FhJO(e`pL4^UbH9vN>I{e6_qTrl9tQ>Tf;ZRUg)lwT@%`>X>MfKa8-FZUt
z0a{WGqR-bwZF!OHkk-Y^TPMRV7mB&jB%r3n2V?EL4Oaf;vwi?iONGP;deW}Mjc`3-
z9Z^s++GonJWRMmH5e;i-CZr-X@~}pn%X;aDL+TnZScnl551zNo>!OALL%1ToKZQb%
za4eFY;Q%d!=$6ub55TuiU=XFBVFFdHe+q)@%)7om+g}nr3=*D(EhcS{myqDb?Fwsb
zAzB|Rn=!2ilqvWW2}j@EAm@@oYk(r3GS%SVs6EHBqx;u-nS6{)XirSZU+)Yto=mS|
zT2FR7Zt7`n8?D}g{^mvKoKN{Wo4WP8d}H!&=T2&PU)qae%oJo1MX&-Dkc1~UXu|Xr
z?iQLW^s^7Z8MY4Djz~@gWt(=-a~{e30-7HO8HG&As6&&hz<*=!J)@#b*LJ}|TWPbl
z836;Jk`+P8n6(I!b5ayZf|4cE(l$m=LLnI?gMj1=Mv#m|$sz(42q-zqT#v22zw@1Q
zX3dXTv*yRyYwfk$tu0mW`-JO8d&=?x86xQ
z62h<5<{cH1EgQ6MNxac#~1ttO;FM!#}=%
zovOEetZd*D>vsiQ7rpJP-l%=7I8-C%?eco?%~&WkOU)K4U8=Ka3*|bxAJoQbaUG3-boR@)4`EdwKrpp@R|!hqAba
zj*%QGOQ>r!$1YpYI^Ftq*)DE>!oOB}GvtjZtr#dI7-0A1I}?Z;^w@XKfB>ED)52HR
zpfgdElL9t^RyN0BET)$Bv`nXTS)Uqilwxo>IPM8juB7CXna+MYddlO`cob{%b|qz<
zR>0xtW*8~hJ6p0^Fzy!e&to9%ut*)74d?xK_Qdj)$hR5Xo;A5I9A`K%<1u-C3Rjw=
z=YdS2CNF+6GY;$~ZjU`}_5PE$hRy~QX0Ta@b~T;;V!I}FqO#TU#;vza^dWa3n#Xah
zGpH1utk(b?&7dGMoU|L4!ZL-KI>VvRa0;2gNjZwLm*6O&wi)~64O}%6izp3^jjC3X
z3?G1g2wV5923JE465p{)Tr#oh0Vbt`>sGEz%O8K8)s~RKNW!f)it=JjY+z>Q5pkoP
zr!VZ=Co&nnd-h|53gO96o?YvHxeQSs`lFLjM%`RHiJR?gC#eW2x4g$_Q7*@y;7BrV
zhHaQnD^PKk#*Pv-jcJwQ=VbW$77EH>b>yM3-b-$-|7!*q`7Lbtu+m_`mQi>UEOOc
zD$tkrbpX7F5h!obWOw?GU0&++!tPS>(A$vy?977`d4Iomyu6A$OxDjs?xx5<{+j(*
zObX+uzX8iUcs1NFJe>cYZ;CE!T)`nCNkFj033NZ-gO5f3!CyH32LKWbL$t8b%gEUG
zj_LmC>po|tB{LnMVcqOrE~a1U^3j|=J|Hl(UKpcLjNd9|-%9r&JjyGVHdO+e5J`+!9
zj4E-EX$853i&h~;Phx&#@jE{lD&84*1$Oj9A+gUXo
zrEXDcbcPDEa{K}VIWq!C0?p4k;g4~&L0K`_v9~9-;KyN{v4-|Eyy=cdOqkT8vFqv<
zSe~fkfJ^XDFa&ceDBE=7*{mBwZ(Z9SW{z_9{dlB9WRvqlpd>L?dFW+Vi5GR`(R_???@Ke
zp;P^S_T#|7N`L8PN3}w>Rx~)efFhPfKrC1>4DclbzrMB@g(H(%J4?LrHJ~emx8ZaE
z-yU|VPVjFLuT?zp2&T^QXxh@1ZNT%RD?D%NXvNYjQT9G31N{;-b+1s{;n-b50oC{J
zgV@?_Qf4L7oa=B*oR6P1sS-9Gm^smC^G5FhZEpU>db83199}Ery#Y3+H0{{Jqsm7s
z`LVEdu&_5+>z=?-YZtO>PufqE-m9MO
z26KcmEw96S6~Cd>Et7RIQTv)LL*M84A?XZ0v>r4azbVYD-7%y*ID5dg>E1VG$$Q~h
zFaDUQ53;On7-KZQ$GLz1dY1w-uQA3GPdDqszDKGX%5%65u0ctn)tq`MSA(hTp-UE<
zbx0S}iGNP;pwitZPfi}c%IG)wO+{IcOK0L#mTK3LC*F47eff(e5H+(DX|+W<7-}L%fkNQ}gf+;RVnw-@gLy}i$aCsGry?>M?+<4jh~+(a!dR1J
z9K2~etAfLW^PefBxS@oSx%Wy4gre$oU(E7Ii25RRkREtkHowv8AMTNnBKoS7$S4RS
z2JIG(rl3y`WKIXpc^RokR`Urn2c
zRq9G5ZDxFf;;}e!^2s&TpA9+Nyc;-H;K*sMRxu?jkR1GYtm^^re2=x#&O<-fEEM+
zumM$E0#eNb3jx81sesj(nh#|a0j2vLAoD5w)_t$B7@gIJRrq51l*7?HlKeEHmBSdA
zd!R>%VhW4z<=3@Vil8e2fV3L6e`!909cSK!>RFQ{;r)H{$D9fVVbugz9h!$6b~2-DdS>PY7`G4h
z2ZYWM?Fqi=Q_DciUPPwLj*zQm({-3bL$(dY9nEZH%?k1k&E9))waAG1^vpM@Sm`vc
z*RJ8&zVFtpFZVm`kE*)R4bGmurMM#@E^fW+dDGb!c2-t@l+c>(Um_l=7^5@T*uKHK
zxP$^UTG4D4El8KI(lug3>kjl=1;v5
zT+ac%@l&1yGV?~qToH7d&=S~j#9K!)vkW`3LSD0OeSagt{N(U%Szi3$7w^YEi6*C?
z)LF7kYP%Hg-EZ}ik0M>QxYynhajPwXuT;@|+gtP+9}1^B>m5vwtp)Qm4K~PZ9#d!Q
zKbr?0Dgtw00Kv;~EmQ`+=56vDv)Ja0&sc?Y1xZ&AG8qg8a(JEi@!a=C5cH^f*t@n?
z0r?BtGgE=~3Zm1$S8e%GczyYvSnel0O@f7m*4>7-a0vw=eC3YA~Fyg-4pSC%rl`hA7IR(0Z!$3
z5JaZVXTstux9h_sxl3nfAW95f@g!s3D`&F0ygmsR3*8h+HoKK%)OT+qu9veTJ%_M?
z-yl|C6<2e4`gZ_oF+bxZ`D)U3Bb6S_Tx16n9)gk$Ad3S?ad=?v|F{QSQQ2
zvwT;QP9++q-|d^voILq9tK*-GyE}iRDE)5ivMLvy^L(&pPwg*W^~f
z*u(I&EA{P_c!X=E5s*=Pp8teiBVHM{ex;?3kyp-d2V4bQwOD&%hkVjqwM5rSW9zey
zEC{?@Vzcd~E*tzd$%hNxA~Np=#`FPsY?!Vq#yva_USb+nEt``2{&#g|
zo(^D1fPCF)s(bFSLc{byS}OgaYwPIb=;(0-iy{^G8-8;M#S4khD2N(!!s34v|G;fC
z?H5Vyg05`D2=Xur1%-r1+oo4dxC>u@kxB)&Nu13T1lfwCeYNSsprQ|%&3aG+#oK+0
zg)r~dV~`NO?VOtbiY&T%$mr$5ia;fO%&p;VXbbx2jHD-cR`&c693Kj%BV@&FF%CVX$b4Wqj-`ZbXo3{w)om;XTmYr(2~
zMa14?5F>!)K6e(15hCW(r%M+?0fQ08&vDSPppLwdRcWHb=bo2d7G?`a#X2dl#5=1c_WSSN33tn{f**Pm;_E&dv?&zj_Yc8h^QdPpp)
zzPV#)SprmHB#7bVIEG%5ly}j=pgId^VQ*TNm^muap#lcm^cxU&qHx7T<^wrTIeApW
zCQr(?Z;s5!Ab}W<#2p04phrYc8Z3TaAbM-S*r;lt+$5a=+*(Dgv$!r!A&?c~%7b{F
z5DCaCxk4nGis%p$s&ut*VhfSD!I?e4bJ5f#5RO=oHMMbyY=xGrl<(iX-9CI@m`n4&
z8&4uKO0hMGGm5M>;)4ZoNE~-~z*|7t{0)#BOYIxZ&1Ss^nM5p&dIAm4?%8B3e`(d8
zsW~ipB4g-ww`cS4I=%0qv@RK(lK)F5KC{oXGvfLD*`#g9JQ
zh*$6L1Fn)11TfwBOoy3=lk=hTcOoyS3>8Bh%?2{*5thaWfq}R261F+tP^$Q&m>U(w
zBfF^kiIf9mg6Xz{FVA;2m}Yi%zv7v2GNWEO1K>0x3jNAlK4($c{H6ITTk7#BcJL7AKF=m>naahQ2I*GSOl#Kh+B7$udGb&qpM
z&FRcJH8t8-0~;575)wC0f(bCJnP`;0>9^kFDO}_^h7~c0v-ub}Xwog>5Qzlf^0ELF
zFVTb$hA4j+;O|%BNJnrHN$4U0e`WS|m;&@W3~GPp=Zle(3z$|$O=F?8
zx}j2s*yPy%V1O)Y68?-hN$|&!FmsSF9=zV0_U`Y4aNpzTXG6jXGW$!kuNK~fzJ;lI
zgEpUmIItne-6ha5M<~S;a}S&vn4=?;m0$cHqv+@M3-Y6AD#%0wvW=m!!qN`Tp~obr
zdiIEx$j@N$vxGHGo7|p__m+5ljCbgp1-ykiIyw)~!h?+>FcoSowW{FE$(?iqv4l|G
z?8t;GGqZoacPs%=r6Lyj!xk3?AFgLg(d2IF%KaTc)gCA^-d5@pKxxB=s)4I8MVm4|AG=KAG>
zb-2GtLf=;$25h{%Rpq($y};zrreG_b$I2(lMAWlMp}c9zpv}mV!I&l9dtz=-ofBah
zDcqAn#L^h=g{!C^bp4?QY+Uoh#;5sH19}{OMSw1;7hWw7cF@;QCB7rLJwUKiRJ>$d
zVafr`xCxzErq%Go^*cTN;V^p|^ejj2sdt
z11d|D)5k#oGsDH3%z@zmaWUlN1{x0}!ViY15CZ?vAtSHP=FM+L*&Q4JU7@Dl&&^GS
zAgYeS>HRml#E+nKd|Evuv%W6edMjwqW)6h{G-jU2WTQOC1x*H45LYx*D2~nl%?S69
zW6|Kf0dq$GDEkv>6r%7Ey~GCphLi&>F|jSZ7+r#zED|Ug2#r6X2O<*3F1JM%NM9kL
z5y(mY-wBE?skGh;za;;K_TN`_B3^}P^+}2{_Awd%w|d)w
z=O1T?78=zW>Bn*GmSyQ`&$ynvB%V6dgdYek=P^K-HZfB`BPe77Cw$NfBYF5<%P3gp
zCk=+*q-P2nkMvScF7{%E5oCXWV>GkP2K(QTVP9aKR^XN?97A;4ZPfRhuyh;y5Z!eV
z#Dr+1Dx@PzDHtYBQdm9c0&M!qf
zEcqG8z$VQ8^x|FqqUZRp@f+qq^D7Zi;PB&n@$8e7QGZxrzJ<#WOUWf1sy8bQ96e={bKp
z1rHjNj@$S7na^$6WXcfR1oy4WGL~OW_RGUTtu4hrj53bodtta7nV^K2006@$Q5@;H
zE!dLm8Zga7`52*=cyofxQ-Uz-ShP|%Xi5SA5tI^q^?7i}gl%SKVZoqbA5#-N9bG&f
zve9qE#w@+_Rt4i*mngI-j$i8ENFiHYR7Rx~)|73~CI5+21rj4dg#ge?{PgA@3>|ym
z6_Uv=wZ!ncNbM2udT*Dg!u1V&D;XdO*9*}%TjDso;0j9uv0uQx?$HK+*&Ec^hBK@N
z@0FOHnaX`{{Z1h`6-dy*dz*;Ck7xvdr4qXUR5Z1azGEQ9stS^%2OB>e2`^Bc-mH!%
zdjL1bHX+kZ-{_e_H|2Rvjr=24jJmjm@AMpMG~2LyOKdLc6T1%T1R@U{
ztegAI51S2IPDSJsk1eUi6-@FB2$BaGAk2IUJtaGYOvx;CtPVsEtwLB{g59%e_Wz9^
zGgGVC|2T>U*jz9}@D*xW1(YfX-9&i+IiS1-Ix`@g%PH<~=c1Ftysx8P_wHfd^**vZ
z;JIhx=a;{#1%MKu=HIv~$fz0=9r#+YCG%9?trv^+b-OXf>*jHIH>844CH*hWmQAgE
zvbzn7@5UQl(glje3()Lutop=0B4PiBrroQSqQ~22XWl!v;o-N}S=vL&5+u;)^5q!C
zK{#yz|EmV;;S~YH4E8Hh$=bp=R49HsZgxgP?nIHQVf@(IcsjI6t0BvUG2{efv2hvb
zK*%^4l(^WaE9)~urI9@Wzulc`QAGU_RQ<0-6lS+kJ4k<4Fqn51)fg0gcyI`P6ay~t
zT4O;n#Ply)6rEmpaj@rtb$AuGL9LQy-=~}~4vuiBSdW0sMX$T#a^<5xk%2)p*idJ_
zf=bC&@4W+Vf5>(;M~>4#Ay+a^zlTfRh@s{{R^|mfFxr5;rsfKR||=?T7{f
zBmG@HUS%4)h=GEnI@GM|4xo1-vbcB^CLMWh!0`AtMETD(LiV3;#|z2NUS^64N3@RY
zf~zfICBJ5K+UMKC-Go1P>)yQvF*jU|(`aHi`qw?k>Uz^~D6I(1fTq|ga$f%ggLyP|
z4R%EQyuMn^6%-zPeQBrM8QcIb3z-Yp7=trz2T*~-B0CLQw3sudM%fbvO~rQBcW#?=Q1;)zLz@NJM1)eRN`c;C%|#Z>!#OI6y;VriG>
zsV(c4FeTm)i#gs+>-C8-kqdmRmL+@FM9b7GNXxZ?(|kky3hir&fgwCb>`-ESGH_x@fK=FRBM*xz1Z+cD+R(d;rjXRY6CD42lr
zk$Ar+j93&m*4$okg>&0PeE^9Vl*idYIF&BBrJqekWnP5ES7r
zRd;mE!`XIDNOa0Z>HBBz8(5VV3+-W^t^r~^rKz+m-Re;INV#$uHe%aaCuacAiUc@mhUH%z&
z3hg{i6GTB&UFwta0icSor9Q^~F~z+~WBiVO_2-LMz>C)r*@?zR5DHD~lw;uH-vNYg
z1T%5!^l4L+KTJ@t0m6Sui%)j^sN^MC^Ks@6%5zQs$?VCV?BVH1hnON2GyotLO~LjH8-mR
zrYAqIDrU&l)m1GV4EvQpex$#n)l+bFEvkxn8DM8?Ajl*Hgk<~u(?jo5PoVKhH|3MC
z`K)k%r&Lj$U`i#!dCkj?(&CUpEd3pZkJ)dz6sD-ob*+afH7(h-Aj#F5v
zYGdfNsi+Pt+rHaO=7?!mQ{pqs%0ap$#>JhzTJY_(oDt(uUe!s4zu4Bz!)+jhjbG7^
zTG{mdvkMkg+N#N~9iJT@k^f-Y_hsSHp(&Q+_O1Cnl14B*=Jj}q;TVp_eJslC4EjG2
z1p*Iswyo=X3>az67*yffc~e|%wvI=Sx`#RRZ4Z>)3&9ey1ZA*{@GdeNf*Zm!^=+m0
zS@k*}*tQz901fS_5Sn`|e_3o&+;R11cDh7iJCWUpsaQpZfdfzX@%6qP;|b$upy()?
zP`Jd!b(H8w!}~t?s)%0u49id=aD_y)=d_WHfBew^BhuzATTHQoduMZW@wV|#k!=?F
za&9vf`aE)9IO6e!eWE~2sJ(A2i$3-z!^B0Ta+vEq8&~cCqm2o^?;z~c=_^;Bjjjp?-?$l}nOH`NeLdA*ww6iY-J!cU
z<}E&o)o<}*{sgrS=z|J4xo#`iLLAZOY~Hpl4y|k+{>NU`*!*zX=SD{y{~V7L&f8SM=c2s45#cDP>NlkSuE*v
z=x2%L55q0bXP+twIX7`6E#T^>jo>8Mv~eR_e1=Sh3|5ki{&OTLW(D`0*K5igc
zCcuf}k?+-d<8U>zy?Xg}Qk4?tHhK`Pg6oz)*18=`Ow_cQEni%wI%2WRG`=TC1?*04
zaD0|E{LeDiuvSm}$Q^Fbrek^IpeO9G20PdCt7n7NS9Gclp33SJoo&B2P`e4Ko>b783jb@OT
zeR7paUtbo?9trrDNrD)?j_1&!^q01@&?YQgnFv!L8d|9aH;0SnENq8bRmppRCL7f#
zm&-(>x`oTdctS93Tg`?1{ATIb8$im;&h!Nz)^28SFT^Fp
z5kFovCR55#R#8#^=Tb%<-H$q)$)9jSkVn`5Wq~}phSx4$n%FMLK)A{YyCp8+v;o@F
zr$|S^Ys23-GdD-vv5uf}IH)*-{#FgR(w967n+v<5Thp?}nUJ?E{jYGpn|FE!dpyRo
zp^cd4g*}`4xxx}MmNlST=iOQ^6BwKsE@E9$*BIt-tv1c&gKgWUi
zSP+$a@l(lwQ{q#-vzc4aVe}F{L0hh-|Do(wE8Eo}*!ZnN;kl~i01j0l7xXPt86F`_
z(X|imy;a)|8BN~J&srB{F#*Wfp)pu~q
z>g6YLx3VJtX8bLiw??ONk&;f?#Ch}f@aK)yEpw>@MI&*y=a}QiHDBV$Cc7QwSN2OB
zU5@&f*0Kwkx2izRpsu3ghv92+Pz6>CgnN5?-#Z-{Rd+CieV`^UVsF?+5huw
z7dM~tm9F?0O>b0@xG?hbOjI>l6<1E0b)Sh-Bk00R8x#ChrK$qB#D?&z-k8|vXijv%
z*gj3TNNmXx#LG@=Fc6q6(iZw$cuk-qq^i;OUSKej2<)hS|J0C9WJ#hUaP`(7LwE0n+L|+=Z-9
z&F40{k4;@{bm>yuI3?2e%?zUpG94#O7kYLs##sfTW`{!QEN&0gD_5R^*NjCFfJT*d
zhi$WF6+adShkCj7lL5&^QyGaDV;q#Dh6&6|%eTO;GXVtMCj-{timAT7V%$!N*EFq<
zxES_l_rgCH1i}L7{Ww_m3RIAgzLIOTYNjgbS>=MHdP)1r#3x5v)*NY%K!}IF&%_R{
z>RFs898`23rp?KrGJ|PG|9orJp&Fd%GU0Cn%-mY7+dDL?uT7w}^~2!RaSRFn3}q21
znQV`uvLr|Wl?f@zn-{fvq*EBp+P?qxVSeF}*sxf;gTAxKur(Rfi>JgTA)%X6I2oi{
z0a?o_CHng#__!j_oyu!!n$eH*c3nmB;L|`lt*)FL@LKFtNJ#6DC6Fjff|#`sk<{xMncLeKU8#Gg
zWvuTFsCah&wU;!rb{vh*aQFr`mS^iZZ6sRYvvvHz3|JzfV$heXi{Iy%z(%w&>K4Ye~6FGHR(g5F3qnqRJu&&fy
zfw!ljJ-N?ask+lUx2Nb;;GBVddq2b-Tmk}Vbq)|?3F0uK)MXyVnX%p+Ur%r3AXO
zmrmE@>wp6!5fG726n02!W$=vW865migSup*rEGA?UY#jG0+zp@A9ykuWaT`IYJu^%
zDpXAUbmY6bv|e(RE&KWvQ>I|Wh(B~&f(
zVXm|EmHiljo+i%rKEuQp=hsa)!%%UDy1JD}E68rQ{u{cf{iv}}kIjC(%l|Y68&R)u
zAtWwuWX>!tEY8b$3FW*ReQ-UtT|F7!R#_>FM=X!VB6IcE>FnYKIVd_Zg1RmVn81$X
z2<074*L^M+b=sO|n+UR%G#tPo$}b7f3Bw2FPZI6^-Mh*@sMCR0tm{)rE7TU68XT;ZM{P+H;&;i(p*EcBR~xm
zfbv-6_HJ>3KOB9K0$wyb6&C;vXDsrylYoTuPk!OR){TV3g>-%76%oHR9j+CXW5@nP
zkxIhu6T#86BJ{Lk?oHSbdW(ii9)OKJT#?)?iC?#L|PQ2O^bTlJrQ
zJzc?S-{}wwb#f*=+0ju^!iB77RQe|wuB}tAZVS+Jq$03M5jCC=WE33i>=@A?hgcMu
z+!r6X>oGAbF$4G+Xay;eeCn3WB-VnuseSH?mXofPEK@9AsOkqLjQ|(ZB5Eb3+9Q!A
z{QgRZiX96{v*POTlv)_~#Yek1BMt{nA#fFx(}5&z9klgvsI}DbR%~CbIf4eAj_gU^
zW}$G@j=?`_$F3>$=)Hrt2j1?ZP%fi4TfN)^L@?})9PxtBGhjX9)ho~dOjuZD_#N{+
z2OgebeFxEA#Ez>4?-ID5p$|)}4WtR=qX!r0QIKSasOx
z>eUgeuDR)H;m~t<<#@)C06?vl3hSS1L_TqQssGqv0Dov^8hzy@<56@aV4)AxMc;kp
zaDeH)nn`Ytd1tG}I=!;FjYl2Gf$G@R{n;f|Y}gvu0nZe3zWqp6zL~@s7>i9HF(g|A
zb^Izcq@4mtWjLEsFe-hMf1`q1(>bAUA#X#c7AE839iQZptA%|#yT8+^$Cx@0bnqgK
zL!wdW47m;*P{92pYWeZ66Aas7+|4v?ltvR=i_Zp3TD4aI=>iC|Q$UxG355^OqS;*u3lCo>Vztlyd`0b@Kn=T&5j4j0~wK07Fd>vw@>C
zL(JR>C@U8*63}R(P_03!l8oC;t+lnhWkKoPC7LyYa
zC8RP7u~%?g-BGoXQrm3bZZB-5|Haf1>^3k$KNNagtC#1<;2+VtZpOlZv_lg}X6hV+
z*o4uLN=MD@gt?;Py@M;x2*U#YyfLYq(%`X9Qe$aLf@WuD=PXXAO{}cR0PT}BQlG@6
zf3o{6g7Wh}&wD>A)^CqnAus;zFZ83t1D?_@h-?A5M+c5UPmT*eCF>x=uRl&1xFcQa
zo{&fs{pgv@Fq5GbW01*0gqDsI*b^44I_8&Rkgy;&GQLA{O!}%{|GGk=jz2>jjCVSy
zxk+O{uo)yJXvlS?Ui5?%HI6hN3TyxUd8Yiq7muRtO~lm%&`J@2@0v@AjtwdL`UJni
z4+0K@H4%-Hb`W)#P-tdFb!q7t04U_e19VfaZ~6Ccv%OVC+4SqPP(JW@2O&P@*MA1K
zEK&IN&y;K2|K}I~=NVA24pQ#y&9LZbP2LezCDe
z{@mARdINA)aoP9(yW%OCDmQr}u!k=l*v1on73CTF)1l5nZ45=Djd(*3;$Z#zufNb;
zEBfUYg#ZMA`JzM}FXv7%?k)EZ5Yhhi2Pkn#;-;cY|MN?2t}i!g6oKqO_@V??H3FP@
zsQ~?yF?IpiuUQ~JsuZ+ep#Yt2AuU|lycHvRU_~b76TS_pP*C5Jl7XO8IUzW;;DYTP
z8A$;V$BkK~fFjgp^85f#kkbcH4_c@pgj7R(s%kL;AzmU<3((RjfNjB%hU}{t?0qg>
zH6MbgLv>DIM*JnJ)$4wJYz6HcUP>dd2VT&vVs>9@5u!y<#SZ7ODxW9}w$UfmF@m1P
z!B$kfdUXOiHi89>qj%CyiUl3bgoX%pih0Z!G>8NE5ZqHPi1-5#9Swm}i;e{#4JHXk
zo7@ouy*x!B6_2x?Hi80Na{lWHa-E~p?4{8Oq$v9+B1SHl7ObYE^aOe|E)kKI%aZKz
zP+pKLmYq{bNEMuQ{58E_AK$&T#GD~6jn!pb5n?4dyhzi!sM{z7Qebo@ps^@WC49mt
zu>LTy$PgTirx}Tst(om2IuSsO6BVQdn
z5Ex%&e68f%KBnLb3|vpAvQ30(kKAjO&$M$_;h~^
z-GhC55G@Y*lW3V5@@&nP{q~zmp4K$aisA?Z#N#YDo#i!VF(ofHog8a*!o;gWUve9x2uhTk$psrvfzJ7zvZ22q7^9
zg1+zxfet`;fFa6)`{m4Dhm)VH;|vlC-m%!XW27m65~i?!*!b(q#3{G4x|_axaueR_
z2s{@VAB@UP0L^MkuR8_UH5{lLZVnDAXqf;N(Xcw02$J#2h%XuSaE4eR>T`0VfvRUZ
zVi8isH+IEgmL5x51CndVHO=mC*BpT2dOag`7*tF
zIMlu~Rc!iGPfsLNDX95IbkWY3;I;Tfp@GzvYkPLV1F-MnPslTWOj`}11TRn%|MMS(
zeYu6p8jyE@cBjayD+S3?FAV|lb5JqP{sATr8`x_ht=R9yc?CeFxoWchCO{?%3Z##?
zpxo`v>-Uw;|HCf|+dXlFBm+bK`5Ar-H58+bDE|oo2DwWebhXj2(BzN(d+jQ0B|jLs
zffvS{th1f6pvpPx1FMHmkwJCmO|B5whY)x}9
z@6oC&P@5A+?|o(wC2#|4d)0kVy&@Vo2G7n6GS(J-fIe
zzKU%MUbAQ(IjHEK(?le8?|vcBudndj
z{}X=B{{sksMu$N|8MgvzYeDE+fpJLtAgL%3BE)iIK~o^}K2Mh__>k8EDM5!_n>p~0
zleyyfg9CtM8&TZycA;o9gCJ+1EjxuHDGeLwDQrwPG;ND_=!=u#oXj+Scicx1G}_GcbQ~lXTzq_4($;u-gy)4a
zDjFK+L3y`XUt**~l1Ut35>sUUl2aW=i8e2!{g9pu1A&4z(uJH+xPUi-jRCSy0eA<8
zHxz-oH`~O{Kc|4Jqt^ZGc_I3F8}0I+6&oFQfIWqv{Ri4T*-%kc
zc$a#c*)Cz3=P6YvyNvh>-r&txkAX*57YXX`{4RA(R#}SIN|zy2c9Sc
zDM0A6TXmZ1>y^QJpo6O=XC-220KR1CB*v#gkN3y)+%t`bsXyzJ#!JCT8{Q5H#+psfoM$b3@-*h-ATAb9DIMkCuEcQap*uIt_IYSm2WlUC{Fn9=%LTM?c2Etp2B;5w{4h!i=+MeB2qj~1
z9exH8UJH_#gUAs4JWCfp8t?U`0%K_eglr28k1M;P(^b)Qs_@YTw`{2O?u6j;_K7(nxb)ur-aP@EiDh8Pu3PQ+3*me
zVMspj09Ioa{TF04p^KYo94B9k^La?i4S3;69i1U3A)XKql}XCA0M35Tm1CnAX%mN<
zk*o%!-EdRM9z%IlHItr}HV$l+Q$$4b)TvXwF=9UQR7b=nPauYc3spD>_IKDT6}-)u
zgNwlmq%sE$o!>i0kUqrT#9eEv%(EY{aM<=m){$6jz_2FD#y%x|@t=&W*Ng*OOIj92
z0R;>Iq4R%d+yI4}5T<7Zk1b!e%mkq#P>hg*R}%9Usnr4UM&+xL^?r3WV}&sM08R3*
zn6S_UGLPV=hwMMGlcxeZRdv;gR0R}o>uBJwdq2*F%bcMRQSgM{TM6?S-t2_8+x!V~
zyP!RRt;4p0H+#RZ@O#rhtgnHj>`otiF5sDj0>;2%8EDv=Kz}EcvO38^Rw8}w9AAJQ
z@EdOD^*~0)q`O85WRJ>a4~k3V5&+#-LV^>Ms7)M%nejxf2MWxh%m^bZSe?HfD@0Vi
z|Bz%bc*bayBdIUxdzELdN3hW`%d_Rmvh$txpiF^yWCz*JPk-ZT)@~EY*T_8eYse&}TZFwZ7~fu=vxqEJ^ND=3;T8r@0U-G
z8y^2Eg*~hVpG#C7%{qWasIPKdd7vm%V4~_6d>ZGxq_NE3@d)ZV&_1e!V3JKw<9))X
zI+-T`-$ddEWzB%5vR_9}_{yaUV1{TaqFYLdp|QEV@Hzyt{#H;fH6*L@xo*%@_v$hQULR?YG?2;U6c#2Ie|MzTk$%2g9Oi`pb}UO&UN1)%AP!@vLj0(VS~7`t
zP>R{M@6Tvd&Ao0o-^nK>R}B6Tx!CuGU4PjvE|QQ}f5HKG1YBPr;
z_fZLbg<$;OwR<PSY>v(3shG?DPvL8E%JM?vXWw!Kqcs_+d{Xd
z^4&(y#uoYdW;5rcr`d;c_Q2~QUbch6f!Mra|U5A5XkUBv}z!#sv
zp$?jx$;h
zi&3g2YBL3X>AlOId2&x!pv?iiMGhF8Y@Z=`CKnug8$gnYxO9g7|L(!buW1v8Ka@p~
zx@_JU4awugrpF@hSCTXj1O<&{CVqEq{L<5tE~*M-7Ar!
zj&MfZ1{~-Ii3PgI
z94tIy7!$6A5|AI7`vzpTqPzPHL8A&t8sfQMz#x*;3N6lA;<(^A_m8;TJDp7r8mnM8P!FvLO0Cbwr+JXMzs;V1}8XX`O;A7;HrF#
zGG4mb0W6WW?c6wyko`~+qK8qt+Lq59ei44tw<*eiSZ9itPM(khPM7jGMb+)D3vXmz76>OF){f
zY>NF9{U-~4C~4@F?Ck7vEyXI=9lv<-;sl6w86Uu339@cjyRB)4cTqNw`*9G$r^7qL
zM1yPYkkNiP2Ag@a+n*3P11Fi9zk}^592HZ~vYnR*y?BRHRu;#iS*}$9lqR4c2^ELG
zsUEF2`@%?`!GUdr$${*)Doqx327coB{8)_{XjO}jU!A@XK0ZRS%xP-J!CyB4PWz&z
zX=j&3npc7i2+Oya0IWqF8{asN=yPLKRC$xFW~k|%Fc}HLouu*%TzO{d5#=q;YL_7;
zB~%Ui*(5ZVXSrP7I_9d_J6;QA!)a^b)sPSPc>p65+3CpP;6Jk0cP9D<>Wg^TcZu?q
zeQ;13+-)WdClxuTex7KiL7qwc*4EaUh&ZR#q4G1ezjhIQy-yT8$tpRrXsS8Sl4#58fUUi)zLM&Cal;3$jHAqoc-PuHIvtykUfu2OW3mjnVgVPpaJEKR=h
z)d^B~uoDT~X2}op_-=&vu6lIv1SuVLO~hX6&hQLkm*t}<0JTS-oYz}{Ptk@?fknsp
z1NlSuetB%9Yw>$Fdpz#6(XZ28h9%CrCK$`5JM^<-wAH!EKy+B=Fb~p$oxrZDC^D;N
zuupgv!--5H@AJ*AdbQ+i6e4OXOcaSm^iARmI=`xdLIi@5v`Ch5<8_tckIBw7TSVs&
z^?_=&(SdDT3+Vxc^v8DA>JAJE23HHsEE@fgM&Mg^X>0_`Gk{&lxc2!#tKwwdD1hB~
z`Ebd#y$}D(+2+MyI0;FHDVQ{9(NwLh(jbA1CTa*c?@_ts_0qI$;6c65D$$x&i7>?R
zYOxX6(WGHwZh%tAANVMwc;w3q1j;(%VcJ4eQc+~-pUXUh`zjT}?fv@tT`T+P?^7cY
z%dFEqPRe}<3+}enofzt?9=(xgh8&yG@$n4j!`Y%lBQ{*T=vpyFNCMPT^>kQQwM&Ko
zf1bEGwt($`3SJ$zBiZu6mgH$B1V=$n83|Q|8SpazAPQPqL9R2sTW@-Lss~B-|727Y
zoK%?H@ozUHetmo8bNBVF3Q&`E#=px-O|xj8xC
zp`NitjBWyYVItl%)WTVZn>P^`v=XN85Kg%Phg`tuPb}a=`Nap9(u~m9#=+qg2QwO;
zGvqhnuv-1Y>B#<1%rPlK@<)dcXL5v76apTlkZ~-;a)4WrAP*3}gyJ%dN&ne+JYQzV
z81ZxC;(wg3H`pxy_m>LU>)ro_wB5Bg{Qu$?Dq<*Uuu#kD>(gV>mqd6yc4bgdSAdy@
z5|K5QKwtf#{Gj8wF;suLy+!)as*zD_OGTFGoXIT`fKKBaDK5zrHbk^G*w|eQB)$yb
z&e%3}AR83xTXGGa>0hx{cad9IVvi{-?;z?xOkX^D^ky47zWls#Cr*IAaDUf!TiO#Brqy;WqpSme*&1_TdU6cULh|X|+c`q2&Yd
z(kf9bR=;-jY87}THtpP*0$7ABc4Qk8-JW(6=rx6o0a;3d!&o0_x1mc4neD!#wAZ(6
zwD_Gd@kxL&F+Ffb{YZJHlR|QeI4RVL9&|h{_KkyzwYY=?pw3ISYi}$lZMSb|ZqzJ}
zuz>uFbbuh+CeXa$StQ8m&J+?70-hJ#gPyefKfLqVCN#~(T#3LOG@T%#3dkJ{7?}K$
zS&JmUPk<>PO%S=>r50!8vp?}dxr^|nKRd<{8#RcTAT}Z$fQUciGEhQo#DfjgrSFsR
z6HnMI;sv+cl9Of7E(Jc=KzQr{B4WnM2&z&pS2$JfEM
zt|X)qBmiiQZB~SfdSN2Pt#D7?K5^(nDh7>_LrF%l;USoJ7HGnJQ;SB=26c@YLf0L?
zq?Quz0#X)^eAh?LoxkTgX0;;GtXJLQ!B;EPlQjS{6{183e+|Ef1Bif?RS#
zkKWe7bks&Vz;6N&yM|1m49lL=kea{mo)AX?NZVpCObZWB3AV_KuXBKMorpvCqbifgr=1vO8c>Mu_D8pkiCDbiX7
ze#j|UR5p4!P0-O1k`G8^_l}<*duBi&MzxC5Zbd<03{xpF{J1U7byfI7sDt+dbb_=1
zvAO7IM2PG}M)W)jIt4JjWYBCOY3bok*`LM{#u%3P%qk*mJ)!If=vH2B7s{KxyLNj8oA394<
z`kRd%Y5qw+Gy8?_HWQ*l
zpDnMc$u-CXG&KVmUfbK!*)#ms;nNpO`_d5(bz$r9i%kF|fRTV;mBVXEzE%W}%UNWO
zM4}f9DUU?6iK>-=XHaKfEgt79~{ehhgwY<_uQLz
zLx~mpn-@>yCTp91oThrQc%m9;W3F`~KGE)*TNBz#2E@st0vk9B`4s^J!J-dB0uXnt
z1JywTg6lJ!rYt1Bz20b1mV)O<SNbjx$uuB85|c}ek#^?3
z63im5)mAaCmc0eGc5Zo*&Z1UD!Hus+)*Fq|&*oa7SHUw+AOZrgSwSM!F{HYq)}e$=
zft8cgt&R}ikOu_f=7&~810(WfGlLn2N?&LSDe|kT
z>g(rKZLYdlV`)DwZZ(;cKkHPcS>4(Hh;+n6wfh5=Dfi>tTn#`$D_|7bj_agR$8}N<
zO#W=~
z;+zyLa%A>tnsh8~e{f(#uLC|V6H!SE_%wv9!tqI3IG89B4GGAPZ=vC055!;qz9Zgj
z7Wi3+3#ic^RXwm&8lZSQwha3bJtB674H1-GUSE(ISH}^oa5U_uNXW#`@CSzuaRV%d
zWqtE#nVO;fV5br#0ARDt5W0d*rD8ro!Plct6;InA
zEyo{5rne8+ZxwV`1ku8K1Y)XyGKSmS5w4N?GTXaNDs=x0rEAWH>nt
zRinMSE5AquOKA$u8`>gqT>|3Jr#1P5e1Q6^BO2moT@s+%)_m2QIPoAwsYi`V#MLi(
z2Uo9H@sOQtPqnTodi@#9+Ns4$BR12DRjY8;{1!o>98H1|61|Z;35hCroC{A4G$=YG
zEfIH$BW8;7UKJw>0Q9f^Sf@=X`Y9MI@ow~aFI=xp_~zpSi78wtit%ddu?cuhuw%%-
zOLzGpHMBGdQFPCWa}jZC?xq>h^xenj`!1AA(4bfaKYGPze(ik$UUa
z3E(r07y%3FO#wM+ir|;PZvg`p^(PXe&e5h;G|rib@2beatwpE?kY=?$95PQkpIBWk
z&M})C;3^mD&Ere->glhvM7&`W7&P8+JluE$y|iS?HM+3+8;+2KiA7?RAegN(WM)YN
z4Czn;5Q#D{Oupg;ZhyP_L->5yU;9zRBjU3bsUa0r)`;hzX*a=D0nPs%Z|`Q@p(I6?
zs9x|`ktY8TYq=r&s(vBHGV=A;caGyL7;QF25Kk=*7I0M2n1)7_>b8%ug4ow=7kJ$_
zGmPb^oV<=%_`GUz#T_rNCfLFEQD@y2M;2RF>tS8$&NQmN(v|J9pJEEV9l7y$+uz^acn2O!fJb#VuUq(_7WS$=!<}+
zWL+oC8}g_Of-oXEsxH72{*3mp)2d*XBJttX0mJiUcM6S7PG;Hc(6kGNC}13#dq{&g
zVRFasqJY8qHln)tk_FYPt~gilHh^N-^@ZPO2WdMid=Rn3=K|A;PZXdWts;5!`?%3m
zaiK%U;qTOD588oHuk3UT__^Ddf#)pagZW(A7n&IZCOagN+b1#lmd)|g!!v}Hr4nS1
zr^|ogko*yl2ti;j{lRDB35f`8IRgs`@sz@1kW=TH99?%L8e>N%kldi%Aub)=F@x!-
z6=*}-BcA;9=&CCwbDzmg?nvmFR-Lkp8y<_}ahmGKhQqneMcpj9Um*Jl%(?vSqvtPU
z!WV&E;Ov2ITVuHUiApQ4Ld!RWa^i;WbeYn1!N#Kv2NNe7wv!Cknur`Q03H)WPLauky>C|D~W=N=kqbUJphOu4Hu-tE@cP`tEO|PR$XuIWn!U?&M?8{y>
zd7~P8qm=AwtG(gLeuF4WjUmMV>6#8=h~xKqAQ0qXLz10Za2>(c`WK2Yoje*nAwpFa
zxtf=}sBE+v`uxvjo7l{Frr~7ar{vkzupYi#J61Rx6Mgf&x^M~re%eJeS%)sqFHR0i
z>`zGZUOH|6r{S^3loRV;)vj0B#s6F9wbuoo7G4Tpym&}1CjNHy^%ZyH3my~>d{|b0
zI*BusGnC3XDqy0eEXw6@Ozr?xVD;wBZ~qWFcCBWvv|0qht+f6$qfK8
z5f-o~G*6W0c-Rc?KSta&JQW^tU*b+;MYLAEYQ>#Nd4Bm0o8`Gd(C|W{d#E9^7;4sf
z`A^l<4J}ZCVm+`yJJxvd0WU7qpuOkOy4PSGuPWr5A+6*~B+X4$
    GqyH%X@=;RUI?^x1DR#nOm$UoR>UDj2?rA
z1v^TZSAB(eC1fb_ja|Edcl&1Co*~1);Y@k^{X0ZX(Sq6=x5{MitYSf24eo2Pjy&)t
z*WFN?Iwql&BuOwWSJVm)-w)}j|G#Rx@^~oJwmlP-I!CK;DobRkC}mG*AxvkQoY9*Y
zMcRfa6Um?)tv>sb`o_pP8DtxYitLq24wa>8v4yELp<;*=-s?H%{eJKJegA*+-_(yW
zGtYA`*L~gBwN6OaUII5wa+Jf6ZKoRswR@Ct@0Xnj=UYVKwd>>2&&;=Vv)kHLtAk
zMl%D@^JWkLci+!n3i)1sElCwXXeh~!oEJ%arTmydzuJ-$F@Ejkr4~~g;^%UcHG_p$
z85FoOP9v(Q`0PDVW}0Z8!#$dw{`?|<`7V^;Jb^x@#I)E2fT{zo3`(U^$`K1IO8w&b
z2+qgHO`r13&mgF+c%24(XZSV9FjV3L{tbH4sRKo8IWdgf5)8
zsr};&862$l!x7rI{PgckWM`&kjEt*a=#VBTN^kK)X0}|?EFBZxd3Ve$2#uH^;2AXV
zrp)=%p0{HT`mE>>1I_jue>wJB)FI*Oa!_M9+?ulpOl9KsN~ItvdLA$Db_(4$(}GCZ
z9q$^RbsZ}^7ma@
z)brZ>A>@1qz7N@0eb^oIsg!GF+pYT%)j5$AyV3T6NTLxtja`YvfPEQM^C$}Ka*JfC
z$k#{H2dT;au3)X)2XMjn?KZNcjU1v!86Glyz>g0OajBKU!(}a^om_YrI+nLsGs|ga
z(;KAr46h-^QSK$z__l9O%noUu^`%TEon?ALRV9D){bOi&5>a}L*=Wtb8*lYt)%%=Y
zS+PC0!s>BGV1>XoR9~l)8X&g!t(4@Ovvz?I9t9FJUIeRZo*1K;qZ$MaLz*Kp5~bt%p-W<&5p!)rC)
ziYopCydnAO5xbXYXXdG>_!!0(T1rNjMr)%&)+6SYh~}PX?8xl{g`C`S0RJ91)Yr4K
z9eGD6ai@|ZVsoU
zv3(C#NW&q>LT(0hL#ZIh6X#nNWc6!`LusuoE%rY+zF}1Kt_A|)oA$(^m&OO5{vVM?_|p0w0z~`VyvYg=z%T=~duLmxXD9XW<=VS;=jj+|jX?YvmHh(<|v&iv4<8
zz?WbY6Ei-}#UvxM?rE8!Q012Or9D~FN~;fUYqIkT#P7S$^4rqZVb_T&k6HX}@373*aMe0{!u|B;_S_9(
z@=nKYwkj!F%Jx9;poF1{{T*Qi3R*H0Viv;dP%icFTRyTmblrB-QSC&X^P
z6Duw!_rD6E-iwEcQbF1;WJua$q2sFev3?!hl
zL^UyhyEq+Rz;@=ygM|#E>pDoTU(m&s@n`JnW)6OD4(D@n#bf@LUWnUU$CYiuB@jk7<$5XU@O{Ir#O
zuen#yjJ-M+PdfX3%Bnx#fZEVfNhD-ZdHQsWj*dP8w~FGt{@w!}xxYX3-6}geIy$ac
z2AcrOJC7>`;lkH?VWr>%g^SqKU65LC0uKolGHy}5>P
zq6sN63NiCD`#I(1X9o5(gEpv8ko%#jsWU;~6@!!psA1NC3@8W%lVuTdJ6y=(wq9cg
z$OK@w6QQRDOW7BJG#a1m#2EpRlI+rEh?1@lJD2&QBnNx@
zu*;W~Kve(GL~llGhqyfxd=;Yp+5B_>Pf!WVxbIGKUFzVYZ(uMF`fQHJWUmq)GbOq2
za-QAVQ7B6igGC=7{mQAyV9G|cygLUOY}SQy=ZLS(XM9{(3n+wS0ehL>pY$JS1$NzZ
zLvWu`=8pqeLHCKO>mdvQcx_d=U$O%y*|n{l+}5$AEe=7VYm>5Vd)KUp_Rc^`Mh)+7
zpnL;Yd8;V?W>x^pv~^N8$t{Z-S}>O@+D3bGhBoAu%URCR>XZt0jwLb~CllKI&tJH3
zzrLPRZ&OzsC>$6BZKjC{iK$caUzb^0s(?m~&lR3ecRu+Ocf+8B=C?tJJ-G8Tg?lPy
zV~3T%p)T$LcrWs&6p(CyS$JZc1b?CjNPnMEee_^UbrTLT5RK`Z=G{0@uSiDdCY^Mc
zdv4x54fPpuB4d@EkZ)~mC9V)qMUUF53FI4G3Tv~UB_P@mspsWdS{?{ypg0=f|K_Qx
z#(>=p3{VConymX+O%X61;sgnF369sQIk~ydu?S2(TzcK7aLyt9jukz3H&3DhFJ8Y5RddJmmW(0?xb&U0m~m_`9Gtw~TP
zdSj%Hf$%3$b&ryWrdwy7-;;;kfo~YXWlR;gl__HGbbBo7s@Q%N>>IYi+H(DgPjVC}
zT?X_8gUy}nyPo(_;dM5`DFpoY)4JZV629d`8>UmJrfH{
z8~5*Dj_QT#y!bj8q%ZFVtioWb_p=3&um_;8F>(0D)pgf$a%N*n7pE#!vJCk)58P%O
z8f`Y-h|l7>?o)6^z6fOKEei2Dqpo}R#zLS#9bXTcxLTl7f>CTwYDViW01ZGfEW=S1
z#)Q}JA&31POWF5#!8RS|Uw~b7hD*D230;0*?}@eG0M2
z#o{Mii0%$AKg<8tvTiErg^1p|W~4XCRlhNd#i=h-l)NGbr-s58~@5
zy1*Fg)I#Rz3uy265d2Mj+u{OxYGLdR@c3kRgbsOIVDD&GEmT&%i2Sk4$VdSLJ2aX$
z&=k#^ZwE_5z~1l{--3b;U_lu2*o8ECJcr8`IAV@0mR@v{Yss-?7_@|epXY(3vwHB&
zBBVah0ybf2a16}{3mrY$$>Zb2VNF9#FTc=v>i&}F6pBoqgUu!Xp($VL1vkI?FET?eF%>oCO65=fPH+kIJc%4}zZ@`N?hf$3O;0*Z=
z?^!9&4=Vr%JWKuw(Os1hYjGP~mRV@D?m?fg_e=4ITWb+}+uEvboi#gtUq4wMrdz5OSEX0FTrTB^pozR(yb!R~k($LbHj**bb
zAU0jx@pqR~H#N-e;SE=tdFw(876ZA2$-Rj2Sc8&9v7-9(qPm(A-ymmSaw+g=drGNi
zg_t12VhAK(^eXD(N3Xfdx2XZJcnC?sNc_c+kxmCZ@-J_U4XH+Vumy%v#V$0|a-gF^
z(%1VT+^?ZTyT7>CtVn6el1T84s@)B`FvUP2cuJtunfY+7&~;9-Yty!E?9&PQ??@v*
zKmOc(B1-=skBssSsg(bCuE8z8A%g+SzaK`m|NBRp6#kEZJo7It{eS-sdg1@~FKC&f
Y{*kIQaZ@MruWft>+wC^O&F0x7CWK{5)}l2D82V4Eushr
zNa&H?Yv>S4LXvwPXXc%G=bi7n-@V^jcipxAEN20kkY78`Is5Fr&*O7mTW#}3?u{%g
zESojd&s=0-VXa|dS#fdwT6iUR+;S5O%Wf8pGpF=iL&rKWjyv>Q%csq^uiUeG{gvGt
zaN9Pm5V~5v|HJV9ybrq^NEdW=e7kV!^IMG1MzIGzEBV*(4ThdtH{i=7=DVd&?8S$N
zyLTVC=DYh;;BKL98#au1F)JM3|KuA^Nk}o9p@&9#I~r1o^qnT};S-8YIVe5e0MSF5k?_B!}Xu{RV~1Dr3+?fXFq>__wWMYcw{Iu@ywfuXr`xCQU0eF
zi^(i%dX4r~C$N{r&er4;B29U?^e69)Ca5pH(fMEh6L>Zgc~Mt4tiXAEJ3N$s{|eYI
z`;+6}UxMFuzj`OL_3y7fvpN^L?(Z+2k=j?Y>hCY^%H=Se{pxjQ3w9NzqqaLi%qYGp57Y?Nl6ZNcJ{FZ7Je%{%U+eJ@=WzE#ogQM
z>{rYVtDS5(g+(X+g>y-sWq0UNzY^cT!kiKf2|6X%d)u)3$-iyRG#J>>B-~Y4e)V8{=2VIY^VNY7^QuN5<_n10$yQ|nV
zx68{tE%BZS9Gj%VX&NQ08EDm6h$%_&yBZK^zUt$lSA7E04PF5|l)fIoi};_}AL-dj
zOdxY>of66=-r+9ysN2z&-7zk?Z=Y&z``s6J@7-&PRP$f^T9i}VEGZ%)BJIQ$XDoA@
z8138H0A3|ej*)VO`ds3Bbvx3>P|n9qrAsNM1!r;f4GmgYgBLpwzj^mA?(yT*4fvbU
zW<_rCSSG`!Ez(zr_bV!z`S|!e_xIoGtbcAp_Oh%=
zuTx$l`$ys5Zp_D>NAW|Ikzr2IHcoVziujWcTrJOD+prs?%U=&De&bg;nD$IU#!wz
zdY&3}o^B?TOFm!28lh9DKKxF>J!j`(<0x-_xBWl$SIE~Yu2|hxh?#Y)Q{nu?yj<+@
z=_7H2x2rRy$!nnv4*qlFwl=3vh_6qStC{?nu1u7lJ(rfa!c~Q~dfS1$
z3gIf$`4s`ryjb6!zRWqX`44u`92vu$8Jrz67us~Cl0P)
zIkck^bLvvZ-`DBmSa^sxEiBIBnv;y!`&(I9Lbxwa$O!Q8BsM2sPRz+M7#J9sAQiYw
zra&B7GZ}Paf>+T&QBj^GPCmZm($X{j#Ke@8=AN(K^aSP8ryt6?Oe8HXy7I|7s^4zQ
zk9fcF2&m&`T93+-YhpQC(RgVS$#A}HvT^1IQsd)s&aZYoPYN1
zwq|k$^kz}T_!tQ3tUp+GKLz&FF^)+8xoQLL75m4ahijJiA<^8lh
zmmyKS_|Wy~m7}d)o>%n=n^n!O9FLrCuwsmm-7Z|XU_(j?6B_9*c?9Q$i_#90Js>A{
z8UDBl*7e-itx8stokd=DPW>MQo@_BxsH&{Yvg?r5*VpHB!I;lJq*jLxEj};Y?O4|}
zZrOx+t3(f*-m3K^fXr?&V<6gVNx#%`$zzi$Zx%T}jSutecrm|AuSm$oUQ5?fqVss`
zfrAInmizJPs;E36-Go#S4=K2^=gu&kPE3uz@8E0?F33Cbyuk6~+8s4t$G6*c*?P-j
z=Ph+x#fxn^^0Tj9zwR_#`*AI;isFXBe6q2%?MN*slQi67CnTOb<8C4NWlwlrN|7zR
z6;hc{Q+Dpj_4CWZoLB-jjeR_Gk7$a{B6rYOIA{M4pBbIR!RxIrolRp846b3M-Rxa%
zvBW2jG{h!GMV%p;x`fF(jYh@m#@f_=I5lx|ezGeS{X(ZwCP*mu49z{8`eqfW!-862
zvQ4oWnHd>UcI`5oxg<)-w3JhM2X@Ii#%*L}r8MX$jU*(d%;Z;_x^zTWi4~l)FPwfF
z?lJRy?c~7VpiO<0##tSmg-gVqJ64c7L*!giNJKtK3*WMrJh8`ax8|AOlMQnli4K}u
zo+TE@1=h{raJacj(bWC2vc~Kp+Krs%9(BWByA<5ACJ0Qg+pSp|xt(r_J>!v{Zk%G{
zhakUK&o{^jP=j7s}na95)$;zpZ~xiYT$a<9#|YD+=cR1vfmFCG*2C7?fthGSv!^(uh=jIrGdERsrVU5NS=#z(dwk`5e>V1WF};{;cyl;>pD6rn
zr`i`=x7^T0*}i@I+(g0T*@P;BsHmt@m*;|Ye?`D^KgM{8CAm4dfzPGAhaxG+Mc*$)
z4G>DDB&DPnoI3RYLT)g2z53YRzHIc|3=&9}p$wub!0d0-D^$-UhN|ID$JDBH7LiMd#=GzvNQ
z_1B)-#~aw;uC=b66cG{{{>EvpXJeCK<~AbyBTSCexD-(5p=dHQbBBFPUyMRiAjIdVTBK`Hp5Seh_xt)
z$0?f6>WKadfBO5FK+eUgK%ZHP6S-OI-cXJHwQCV)&z{Y4U$EQED{~$)Z0%S}rqeeL
zLn}zjxKVCX`W8~OCMP5_s)>QIamZjzSR!Bmz7u!PKtgNwJLW?Mq=Je-Kf}xPFn=;k)aBw=+fnB*bt-DKvZHTZQ+xR
zgXGB#0Ik!Fd{{GQYx7aJc420jl4$d+H^=lXDC%c5G%nn{8Qq*>nwUyp82@;G+z@Ws
z(@k8xM_bO!V6kFmg_jN-II!8QV5U;kY2dA)o5#E_m*m`hoBjLFVg{W?Mppul6lH&p
zrMrCcuIF1?T4Wd6?@@-7m`1VTZV47Op(9S7g45qN^>VF5izTQf?}nk-6)Cacn^|4l
zKa_q9CU4lVAq3!v_KATV=yJ%URKv4ppRe7*YvpGCSde?CZ
z#q)#JH+1Myx_JfbLw^RpEWM*^_F(4F3fyF$`H=u309@}+bc_*3W#N)qphn)Bw~etJDe>_QHU*Au^v@7MxGLAUsZ1@>&|
zsURF~d)U326Qix!iL`88?GxWThe#ily^p=yP0F@4eEaq-rf5}CD&|w2L(+j#$=2*x
zI8)D=?;`xE*yZVR{?wN|$F%{!q(vH=t(jTG&=@6lSw!~P1)7$m!*@T-DKSPr-eL&V
ziH`HSm8`yLPQ4ygfrs@kT^eTGEUS;!6mC*Tumb=NH>?4|5g_zljrw9wnv_HDnFVTY
zhx7ZH*;&tnqI>q7Qc+PktFB(#nr*9p{(NnBiUrh=?0wpy`(qoddv>a(unDT}7ZcN_
zEle816CiD=&$y}SPoh%DLyYW-{_n4|-6%E-?xKb%jqRWAIZc1#wC(xw~^iV0AFUJ@74(
zWah55#asFC<($SUaYE9YH#@@ICq5acU3uUh2ETCZ+?imMY2n7CFGyc6+q57kEj$9t
zvuPm6e8OQFt|gY7N-*0hKXo6_QZ!T^%`iEaW|aSInhq)~c0WQ-*Rkz(E`<1Z-1s9W
zUw)VG%xU778$*Jfh$;t^AUF;@D(kVI>8+WT{^b%1?(?^%LIUVZef~=MeWzo!!y03>
zc%TDw_grm2R~zQ;C8Qb^t&*66vov
zv&+S|lvDDV<$9?|ddGTrtTlz7txP1oS`nl}u*Lzo+QAYkv7vXLGpHnj4cyU$3t!d;7OZzgWz;3=W`kUnSMQgFvQle*~l`!u+U`9nl+pGZN<6d
zlq6pNwlZ#@m2u$L%Gmk)%5a30Q9^|jE)-v4>TK#FmJ%O-!Okw_+O=!#eTHss`Ngye
zxm5gst(l!8ukFTZU+3}f06yAsZsSJ7?fPvJzkU10ud_Z*Y2RAq5B}!#Nr8tC<}ue8
z9t7)sVlC3*>Jow2kYIo780TEgN-Dw};~0bt{z%K6=^JH88B_m#b*F829L-|bi5
zyz9tkT*k(Y8{Y&5#`N}H101nOh`)rr#yHzXSR)}R>DC{oa4xBi)Hj|ZUhCMby1eRP*lG2jpiHS
z>1OF7ckla;A593F06vo1KLpsTDVt=Ln3W~9K(4E+qgI(SB64oG*(}WrxWSt6>?{TZ
zZ{3n{6JZizjp*p;Q78;+xAG??rsSX9;{BCximVh^IW}J+FIjFXp~z@CVaF{E_2#Cf
zWnx0Y+1DaE^NSLy334kS1!U6fBWd?$`oB+Ll~CZ%)oW7VnM1iV8a!V5qOedtIyM&l
zm84~jqfVg1(p-ZMrDUDJFdUoI;nJ1NW0zObC%Nn+Zm>Ii&nw@#g)3>2;%RbX!6Uh<
zP_Jj2LqmBLXk(dfO9MgBCf3*bKkV}NZEiSccBNmw{5k!!**258h1!uxM!2Do1OM(N
zMaCM*Eh~#>as`6943nPYlP~8shn~3SI2s=dxBeXB5d4)G!|9=64(BtN`DtO8XDwt)
zu~MBg+j}h+%ZM2n5*#~r3hz1b@yv%x
z4z=e@GHlIW0T`_II9E`Y!ees~^WZ|=3R=Hv;dB{~+d_wnf*Jf9SjcTZ9-$knEjwE+
zDXc+Mq#b&VJDhf_MMA^L|J0#_2QO-C*K8nJ*F`+Z$jA_CQh3G1CONr5ZEpyOVw@zO$Uq(=01jl7ri3GCc-q)R`%}
zd?x89-uVu_ekPNs*iiClU6I&q^JRMt9Axqif5`+Qe(~BGsPW&`vAOR%Fb_I>=PySt
z#p+|oLwM|j!*=>CQRx>#EMA_`RK|0q46Bt)s!_t?!Veo~-62@)6HlA<$W)wp*d0Oe
zHoz_n6;?i2=nLHnTK@57JwBwg^=s9{!`2D=~VGa>XcOiT+93#4wmR)^dji
zL#-b9cDtUbB4?SSZOpw+ghgB0{5ag3(fa5YPz8Xwi}a#Xt)aF87x!Ao8rnQu)Gw}y
zly0`MqfkDymup?mSFGizc*Qw}wWDH8hZSkF^pnERn=Vhv
z5L_nBegsRFligFwN3yNzP6O&%$b0`(&s_2~=TA(rkpSk@ow_dNSOU%6?B+Mj{nAD6ie7sq3CWaqFYyTW15x!R
zAmF2Ef%8a>+~i!|&arDQyqn6LuFB1BOH5hVro^W44Bx|}WQtlFpfrZ7C5d{_dS!dj
z-G1Pe2z3K5Lj{cX@ItGz&95&eHRKB>uj5OP`}mz}IZTyNuW3Uqt(RvtHJea35fv2&
zg%9ev3o~sZ=Dl5g?MGz_d|>FEhbD1@13f#4R-~T8wdJXdg{>tgY4S(S`#u23B^B<@
z>{`Y3M>*Ofe&lPjT#ccDK|W9YWSC>U1}^HW_ty8}%R|(+TGrO_@x&5)-Ywp!9Z`bN
zHvV)s8rHgmgoGp1l2H^2o(mmA)9R;4sqxUs?DPJ*C$?c^x=Wm}905rG6E^sIj65ev
z9IGuiE(KLU69-I#W6RAT4IJ%)lEK>{@m9;eF+0nP9pW@EI>j)R;;{KC@GcW1lk!Pd
z^AE>k((e{3)HHm9n!;`2huY*!+jgmg0Q|<=a)X2j+if#x4PqJ^8aOXGuIU(td&+?6
z@=I|GHsUsQRa!yESPko##V<^sTzMH9U#(<;1H)<`>-(4(kDCHuf5G54
z{sXX6s@Cj%LZDX+=Ur1-c9h|r5EqREepA;%?J?>T-RsS%=BX4i
zDG@qXZs0(~KJ{Yj4;mj0)?#o4##B~
zMt}o=6^n+?2DG$9iIXd!1zZ67(f|YjS;yhpVF^kX>wKyuT}A$plJC4rfr^`)WUKwl
zL3}`XZtF|09i2%PtYb1=QqK!N8_z9F^^6xDtS7EwNyL!VRLeO;dxsvZT8EklSV-o8
zuH4|foM_{@z$$*Z-g3S$Wmnv?tX_({_ZJHPZoO?vo}AT-EwMTRFZtvQv}1J<%LZLH
zziamjD6fZi5A2aT)0_8w$-MjV%tC9={x_-kCofR*gc%G*G|z`qkFu~#uLYqz{X~Ni
z1V~`{y$ugfnVFgC>y+8OcU2-9``9kW4SE=1Ym`BriPkEY{Z_($=@vX5XMaOHG5
z6dwgrRTfZ+um;|X8b3n3$M?CcxLbBStHJSyG)a9U(fr{qr@7$^8a8v<>p4yCJfO*h
zS`h#rjzUeia-~!I
zOFyiE|9guaoSd4tr(5_&XJrk>bV#Yz{w^*8nIN{5`y=z#l+)_1*
zz)OmtWZ!d+>3gIU(-0|Z9B{yIA)i9SeirT#f7O##Cl$FF^2xzH&O_`JHYGN@XL!a2
zgXMF8FQ9WtfF6h5C%~*tGRhQc@+3n4V`+3^z3E3qJF2dkuFTf0Ip>$n3X1Pk1+6?`
zE?EC*;*dKCopF}La4MCY+A*1YDg87K`k0e|I}3s~=frG>96F!r&bTN85Q;VGF{ncIpi+sBY>oJX<+f1o^A!}
z@MJmXMtV{DY{HfBb&tPztyr$Rglgj@4yD}NZ81R1nM+GcSJu=RT)p~H%?Q%yu8m7h
zyJxl)%=%ED;nyE8)nTp{^iow(8TshVe-RcW1^59g#`2OIXfj!jBNsJ*`hDFPskT+K
z>df%QCWi+63{*(hxb@MK*<@Cd$AN9L(tE>f>)U{VYy>VK*(kG&-U)&vB<@5-PwM^?
zCr;ehS0SAZoL>%FnR`z~*_-cPeNb$Tjbd`}aznz>J7?RZhV23Tj1h)=yoR3H_JlgTLA_tpYISKD30C20`{?G7N*VmKx_
z-7Y;!`CndubLV^kRLFPStvVd|#;}W3yl{8CSht#*8u`&y#VP8|82y2zgOGspvLqvm
z7kgO&tJ@-mnnsbVkx_;gsu5%^r0<`_K`(hTU;caN_#+ns0hh==^A7;20}mNR07Ykl
zn=574$-_-U|5@$-kRBU-ADsRSgT}PlCy=HiNqfXeeEjx@Z#98nkv^1)+Cu<
zF{Bzbv}SIhkJSRJREN<&LR@wF#Fmu+#DBG7zIc$QviZFhuR@{=^P-L%+`oTnF`~Uj
z#>&O(whRAuUym>93J8?=^oK43hpQ6X5
zE|1xl+u7MUKwpDS`-*<%19e|0&9&Qnmo$5J*XN4wO7hiw5W;z*$%)7O7Y8!*9MA;;~JYIgvnSKJ)nWelHhA!@|GJYns?62lb=O0KU4pqm1X
zqZ$T-ksS)XH#hj+9^r$Ie5cn!Kpn;d*^Y{a^2|}T2Ny3~_y|v8^dYNQEh58vo^T!n
z{tVzW0*o=*VF}RKn*w>e@KYStK;`%E7mSR8b8fXn)Yo4GB6t6R0~(fi&m548vagqK
z@tmtu+uTe8J{56I!IGAvftT&-o^YDQ0jRqO2S@Co+rmULu(^5SgTaSWi-En2hdzLd
ziwhS;GRGFx-Avw3jE)Bd-NsTi$vFG0Eh(i5bU~*MC(9%-ic4p(Oj?&tUZ+oQuuxNDqZO<}Hvl@)u2WZl
zZ5sukQwJP#?C7YC)7Z}oaMdILqFF%xX*AD8O4~LMC#DwHLpx0Y{=gQvi`Mk(-k?jP
zzaQ@`x(vCuAbuP~BP3ox=|Bab2!v(hBDd@<{PJ;-Sb>!}2_OOKI3QH&&Bbl!p~z%7W>ONCHE?be;^3p!w|sy
zl^^f&o@p5T)jM3`Qi{%hw*+DEizfs{6EToK!WG)+#bFLdb}eOxVbqVzY!IZZ9y_zV
ziMZGTG%%uJ;e?QwG(XWn_5#dyRyXEc-NQAkZ$M3b_;97N%R$N$@g?~r4!gb>!k;ul
ziuk)FoH$83DKxdN5lIm4^J5o4QFWf|Os=o5Upt9P!m72KUPaySG-}2>9OCVRK2j=l
zSZ4Kg#>M>~z#Z`$ji67uy$twMxHbEz7IYDa)B@O!+S7c;k-hN~FXDhb8HK2+h0q$D
z9TE}}+Ak^DK+nxJ|CA@z**`D<3@21g4Oo%bwvd)dMG^B52G?LDE&LKKi
zA?QC0VTFVwfMo5H00ziFZ8hM8H4llVGVU^$h*(F!hqmLb*;J6l3dJn-F=AQ7DJx5h
zbHH%=%g;PRnuDJu999n!IM84*&~a})4cIJ=NXYHZ9SNdv@{w#xw^
z4WD<9S!CU?58pg191dAnI-kj)Nqa53nu3b~spfSdtHmO>Z
z_|S=e+o;1ft)Kv}8-s;i35z2$z#*a94|E|(-_SG~Dwj>k!!;8N4gVNGXpKM8Dnuw!#(x4AL0D>~y3ByNfTB{@E1s^skT3a>`!!m&$I_ut$b{l`+wW0Y>Kby?H
zJW)c3jkDUJ0a-PWya>w!G5o+yJAziL8k8F$@U5&2P>3eT?MOC3lnOXBs)zSUB|s9c
ztghC3f9!S^T=!ZRZV6F1HhMO;VOYj29<@Y=4-n!@Gu16!3g>eP>y*=4Tc&$)`t*)p
zE&jo}Aq#@lrMSDZqrXYma
z2FP~KMAG^|^_O^q^=IH=(?l+|Ssob_UwSw{g1=*hFZP=(3FW-8k2!Z!ao>2WQcrEl
zBjH!1rCWFzEEJM8S%*FkAWav~T(-RZGwl@c=&a68EK;YSew6))iG}VIa{8QY7;||@
z36a37^aNQRoW`!Jjucp-YK^HzPCY
zU%qyG9DzSL^!w6CxxaHeIERowr{_)_NNyi)ezYEiE(EoM;O3-5Wo*rNN`%0AWO|rI
zE3|If-+>){Ty;<3AD_JK`IjE5^z2c#@3(^g)983I?aaD@m47*d-+o-Z{r@Xh<9}hk
z^u7!fX%jU1XgPtdy#WG*W&Ja_XmD2@1n%~y{)v=3@ggiNT5Nq%f1dwiyhTaM=jnEb
za=!e2oTj{)d6recPBD{RkGa=$E>8Eo4GxYM33ojW<#U58=IGI*`2&HM-@JdH$gkig
zwOEgx?)`S~;K7aW`(8l@3t$Zyqat32ZFI%-WSA%2aG`Bd3Giwc^rRJjG(E1szVBSE
zG|42gDbdgtx}1^}?Z_|D;TE;bZ9Y~ctAr3^{+1nO)_j`s|E}}t#5`#*QvjPO)FI90;tMXQ(l11
zg&lagzj|4R{ALeafs+Dp(-aIn)c)5x4^7|pNn9iXpfe6KbBi}~AD4z@BYHnuS=ZvX
zv)IEs`_E?~ztR@g4Q_Ow%(7Q~p%7>$ZKlF3EGzAx({uy!_o3-a_KiVqD_LrH9M%5Q
zoqKZ8XN&WiZM<-wAi2H~f~Im`MfTr~y7xaHbt+#1tsIy&9l$lMpvL9cbs+U3anvFn
z?0De*NPsW^x&J)4xu7BO<5x(itgKWeX@b)e40aq~HG<-cR7>zYgWawUbopEDD{Zsw
zI*b8X;i8}(uZ{Bk`STKb1>OUE!$SVK|0fQ<>3DuZ=!pz8
z&@MsfJudl50GK|!^@cPo0riu#Rae`d^G~k3Pnx}$i$a^DkVv=;$IYs7hf=j
z!Z%u12kfW=a8l24&VFl!)h*hxk0$E#fR9Q6E)xN?(Z=}Ks6jmSXziz<
z76O)let{sqoF7)qzW|7*2uwE5{rsB2qe1~=mJ~G@?9a_8ufTbXfYg?qRm_Mmb)OLl
z4?Zyg4WMz}osW>05O$W{VxX@d4>lE1gQO{HFp~Iz)dO4Y0OP@1O;`qs%;kIVF+VEo
z?npK;eM^06!sN)%2TD=ErfVM=9r3^O
zQe3r`St$A5lIyCY$B~03HHIwaXKccGwfh1B9Ap&~OcB8bp)Gy&%WBjwCS}&(gbrUy
zJ2^WgUKSL4rn;^IQ5d#cb#(L!ef^nl2-
z3OwrYQy#nX5Fb)t_0SZxIRo8_G^8s86OnNbT4*E*B6Mh&YPPx9F-83F;a{FHcv616
zCn7**vxWSFo&p3ls4pTdg~H9xz#9Ws-8)S!7r>59w?af^UH!UNE@9w*;tX?&BaC*n<6Pw2MGMF!$$m{{u=&=CE##*#(HS4owRS30cjLwa5;<(M51H*_>bo
z35;9j&Ii)KkJ4zQ_k+3#H9}8A!v_Ir!~;PLBX3knUhl^7Ed}b(b5=>G8UW*@*
z=j*+F@lqbU4;ZNhHH-^g(}gKg&%rg(b0
z_Cu0%B1z_gzJAjcCFhZP74SVNHl>1l9uWa3bAnakuz9h`x~=?r&dxbC3JWH5Xz-fj
zW5W}Gz&J+bdXO>%jyHhu_1)Q`sf8E8`4U~8fm$hWvGD~?e_Kz*OVhq(hcR+8smpu<
z)q$L1L!$TD0^W$MyDl%mZC>5aj&p3#I&NL@Y7ZiDv=x_byo9irET*L(lN=t)Faii)B%C1XSGwW{P=8X%(>tQu#c%=>16ZlJ3XhH
znwxD(pRu8^DW4Da(S)R=2T6`r^I0=U>@D(dqp6F(n7$g?ZuI977JFZCAHDV%*|{MD
zSwk)7$G?C5I~zysBQ*M
zPT4{uZw<}rQ09SkX9CO{BnJONei6CRU&g97{XdOWwot+Ns|@t?@|b+u)8F<}@h+80La0a4MEcELBOpEYaoBJyTmJ@
z+wFN51hC8LC@X*=EOHr_
zkEH>x3%0Kt46O)=fN+T94d}@97FH!e-o#&d;c1z}Tw)jTt*ENf1)CNaUZm~~o?D2*
z1nDTz^#nYo9@Ho?Jvu<3&hL53^Y9`OmR$jpj&3zH=1m|;q=Es1&!y;&k2qlMLdgts
z-yF%vzDIR4QByMJE2^Jm$+t{QSDf=Wns7PS4i-r;s#qEqq3r{KGY-UCGFa7dUwiK8
zwEKe+pQ;}WwU|e&~ReUd=w3H2?$Vr?U^E1^?^(~
z0^0o>A!Ju#JC2bIJ#;ze91fYlpj(60vslCc#=m2!Gj7U6&Wz>BeD
zNe7~$8JLU_uOH9z=77f;27eUA)e~?6Zgk>}ZD3uW*!PaWb(?m50JtXYDh?b6T1il-
zJ3VH#8c=%fkcphG9&=E-z`IQEh%znjK@Jli+~72h*qFWkR^J=oJlo|WZz|qoFv(v{Bq9ca64e$U1gNmc)X7vwE(W5b+7fQuE@
zfIcOEETc+D1G&)Rslh4*cA>$p7IS&D?du{VBHB{{cf^Bj1$U#W)POS+74@s|WN}el
z`_{SXG8ITOU~6*=;TQ#HHl$RG`4@z0KP+Qo&@9JiU-hcjrh75+Hn&WE0%a0O{x_87
z=GEcQ>9p`2BBR5(?KU#MsYh4e)l#H_^OoB*2B5~%W-{Qc@2%2&aIY?Y{eG1&l&@^|
ze{YBo$QVyMnx$jAMnWtM#!zrc!DE(lc?C&;l#nEZEUVyY@<&DnlM7k=fQUeV>ZO=2v*=C{t6f|;1_@!a`je#Ye46cb_6}gZ9igo~
zYaiu_7^T7(~zB@Mv-<)L!=WA8QpE-(-1gFV_|8iLJ)wF
zsR<(f*6rxqCuvy*A9^pEn1mL1Lgx}2qeL8bNVNxg7#Foi`6<%SxnTxJ$QpQ$Q`nG|
zKC=)_6XN!8J$$b=8eN4x
z6&D4TPk-^xnVFf=n3)?3U4STNFCowkOzP%lO%07YV23-{-S%2mTvSK{B!))R
z?I&KoURhNoyu^%p3NDEUCm|z{=)F2}nb($QV+&D69Rr8pT~B9!@Vscr{Vf)>LW83I
zCLvFI{Z%&Bt9>S3ym%Vd2%bDpkZR~uF3Zr5NZ^CkiaPamtFQ)W4tEJmXu%NjgW`x9
zHt<0UX`pT&K;fHLueysP&7!orH?1axx%Bn+a<``=DjdizsDrinPGtG!7a#|Nsn`jk
zDc}E;Cxp9!Kfi*IhSKts@_ictnJgW$5myj=tRu{gA*(7B({~6_0$tMJB<0qQodn(J
zmEQ3cM*-wP9Y3Xht~uJ&-BMpZvoBBFDB}!(?N34@J|f|Fx-@c%fypPsip$Mvr><;b
z`Woju#v}3tz>>mTtuo9ISWOj^wOb&f?+sROxOkZ7u9ho5AXR{!4c$cuFv3u6ofAkT
zyHwzoKmX&F=bCzdUD|jh=uX9JEU{*~qWbSKSVgh}pSq|#
z4Dgi{bHBJa6mU^7;(?gfab&SXh;Y*%;V|9NI*U`i<3w991{^y1N8#{U<1@0J!9j9
z=g&8S7i|PcZv*fyIFJ4O2(n{E&Nzi*yur(&_aZDi3N#)zz+u@yy>z3DMgvQsQ4jTe
zfKtxaxq38@*_m;(3RQ5ZHX8MSD<*X5)e)vVmMZ9hf`Ilv*K{>2vl5tGZ-Hoi!pfP?
zgz=*sF
z4WWy4k!)}`!;F(MILEl{J|LriZQS0*f67`jHR{|p6N}
zi7>;%`T~L-8ouEjTFw6huM1m$d(Pq3AE1Vul0d}kLqjpK0PC9qjg>V}MuGZZErWp&
za0`86c7ltyTmoshg8uRmD5b&^!$l!mAhdZ<&~*u3bRi8RBO^rif~+5vFa9YyUwq*;
znEv0qi2^(2nnKgI1AxDxt@ut)mBao+hxA}T4b80+uvm?HAoTR~^w8(J72UJB
zeLFeaed5Gso@3`=L~8RsBXuWsE->S-QeZC60#wU#O{|XeTtYoC#o1je*=|HBkBXRBCKh`t@>0
zqQU|S8`M8Sb1~vI2@X8ytn?$vYekD!Z=@6DL;*gGSC1`PMwzao!7~$vXGXFda=l^E
z?l^53dhFr9VJRONpL2rRR8?Dv$ll7zi(uV?bwB}&I7~!jm6$$|1j!CICm<*;faDY#
z?nSj$cA^%+ZP^dRxSY#GBlvi_XWc-Xy9f#I_YvtPPqjJZehM;m4n{!r$Gf9nak9-W
z&{K;-REQVN;kNL~Tv-}gr->fa-Mffrr7sYe6igA^Xi;I&wd+8U1Z{L2csXFY(1Bnd6SgLk8C(Z$Duhx1EW_;dd%(#&V+Z*R
zAsrwp7Bp2r<~m@=u7l^oZK}9y+?O$Ax|?S1Y0z)P$dUv^8!#i(HM>&uiE{Ddm;<~s
zHx9Tir4XjS*_Hg2A=zfV2|!#l40@x_g7gEp4GeM+{IleHFj`%$m-qnsPYosd-0Lnf
zY$2fr>WlT1F`Nf7wrR*t9D5IKt^Ml$dymH9CKgq{5NOJ-LLWcoF&LcZ+L
z349ndN#!vg+v)q)xC1l4Wf05Y}(+*Oh6PbXaR2XEtWi{e$ISG9tsoAtg+5d*;$*Mci#G#Z$6A=
z*{d%!Snwxyx?jsS1R#jU31B&(JcG&_aLB6*
z@~y5};kAg!2Lypw1^56a`OzdNq|z8@n;tirruut3kH;V2by+PiUM)Az%JTC?9_ydS
zO=D*tB4cKM9B+zHVMUTBw5JVeSHE%>5@2MS4g&-OtRAb^jAxE;L3RGc6Bj_~!qVEh
z=HMZ$OE+i+rrBm=IbVsI+^ipcCCmKYiN9!$V>^JB9c>4vRT`P?BIJT-L{R}(Xw#Ar
zjhZ_kA}+wym85V3o>hmif%gbQPxSox^DBnsNV-En7JjP-0wD_cdnDleY}&ZKm0}A@#An+Xh!h&49CWEKO~0QuH6)~7JC~g
ztsB^sRiO8}!N&Tg@b^6NJP1<#3lsfFSG<#gBm3rp=ovZN)G
zg-%E2Vp_2HbZAQ}4ju>*TG4qvwbhv_(Wona$`KtLyz7>>p
z<9v&&##gYM7jRf!HKr!_P)<;(r=~(DPjZHR!gu-c@x7_0k9=nB_WkCv3nCB*i7!@)
ztA>~C>}O?hU0a;Gf>!5upZaKi%kqPH7M3`F8O93uOV>6%6iAycqU~QHJw|+|VO$gE~@apv@J(H}|&GDC6`2G9L6Pey;f+>n+
zTmuK%)Oa0get*69StdUAmViTP%XZ5*zkXnY|6A&?Ea)dwUi$0Jp_b+U?ROKpFFF=c
zceG*9XBHSm7%*2&uDS)o;*d#%Qv&l~$;QKv`hgo*SzrH}MgxZSZKs^$@GGDP!i=?h
z<>mb31^qns_`zAu7RwZ;`Uv_tpMevp>+}(pq
zDpwwWFa31yUq525^J2!|$wb}?m%#(RL!QItn@~HSB?_jfyx0jM)YX?&+IYpH$}WVD
zK|rgB9}ud^?5mLMt>8q5kePhZM+^CN2)GmDVKTkGrFH!1fAl^}^mpK8!jH~pLS6oL
z;NQ;Scv%3+&quNnNa%d2}Fuh_|ym*sORSApP0#*n(5@klvM?CE`5PXP_yvqURnsJ
zIL$mu(5!Hj_+q~11xY}>b(OeTq3d*lzqlW^(6xmb!8d((h;!+CRhW6dh`5Va0KTw&
zNyTesri>Fa9uk!LN0$26_o-|DkXu}(|6vcS!hhPs%IwD!xSfADD*1cTOzc3XiBTJAYrxPP!h$5n8K`wm~L
zKp`7u^B)frzITVelbN&tXILZb`m{u&73A+f$8DOe8kKj|ZzBKY5~oLz~w;eE9I}B*Kx<
zBxxTOaiB>0EeqA~^i|mQK8vKo;es9dr5#F9Mx~tC@~_)5b>^5(D6yud84ZY#C7`V?
zZDNBRVpw5!v$V3Cu{wCO0eetNW1trGH{RWDsMJ@X(_47duW+BAOe^H-U@kQ?c9f>s
z4Do&c`dAj7es(@S(~$S?$)k50<>0O~!2Er}GCiY8Oc!h%NK@>>GK=cnksJ^7c;@qG
zZogHn6{L^!5~w&CbMAnjsca~YL0XDYh7
z%z>r=lA-}d$v#4pMEcbE;lpkqfRI}o9303b53V27pn#TPihwI
z`(#g=zuX6=Wdk@J)I7i(zjrE8cUQ|+)rEq9y#D?h;Dti-34jeMD=KP1otcB3H1rDm
zfUsM$dUfNc&2S39CuZ2V-#!(7q8Y|{q}=ALZg&H=lU8eFeo^w9VYgDh}*Ls7-}kl=NUs(*P#@U_P8L-^>zg
z-qxyg&!TA8iODz(;7V7nS_Q&q9XHL{qRgm@eY7E*e#MsAdT7oap}7g8>Rkd!6(N0D
ziB$G^-|O51zVlafJh?{}@@#lp*Jo0l{3fu0k`mf&U^qmMFJ#I6{=E+D!l?d-$z$s0
zW}3XISt2z%C+?Wte3MMh%vB6?Fge<{*qt&IPr1pvW=b_nkchl0zYGjODoeq3iiVu!
zdqUsh{bc;Um6zML=Q+r_PJMuVghanHme$nBrb-ysR8*+L#*0xkdC%-rZ
z+x-wD4Qzbey%U!|xrb+4H>k3)u_47t*DLK>&i(r=K=d7uk@-233m*ajS$%->Y4za5
zQtt$quMab~Y}pbBh@4y6rvJ?4_tIYiDw$3-k26*`=mVJDAB+-0mm=SwpOn&#FL+5nlryoNt
zF&QWV3J1-Sp*k@1Vh3&b2i1
ztnVEt&aliBffyhI1Cg`b5e~5+k75CyhV|$BE1b%ZC<^lx>F%Dmg$3mbotcLHGY!Re
zark~qrwU>#M$?OXU}8wWvl0RV?tLQI$%=qcK+6*Jh#lPA37tjmwy^&M>WN`=!f`yC
zG+G@j8JGtHWmZsmPH}=7hG^x+!@2Dd&_)mY$xR?zAIRmi2g!vXa?F9eka+d$
zQzVO_Q4ecibvAC^tPl0v5q>wcaU1~CB-(5Z#-chuIz4RV>i8{TATI_;hD}hCvtUCL
zI_#8^-~EtP%BFEY7^~2LAlj_)n0=?oZ6lpfnU9bon(_{37Y+S*x>Xv^yi%;NAZ8;vn1h!$$(}$r0L%165R{QW
z1@7rcPMh6pJRXl`XKU}u-~t%IndEQ7nDQl1bK=mJI)mjLJTEf?V;f`ZWZY>`W+!hD
z7q6%^^T(Rj;g86M4{Y#lj;-_Z7yY;J_*`DD))1>>2n>2QaDQkF6=f+Pe5GJU2^q9t
zOO$+|gR~Ar=8|F+98~kDg_Zj+H(RQD%J$`5E1NDcCF+vD%4}pIT?;L;KyylH-U*b?
z{QUj=RTs+BfdGf`Mz8)S*xxMhn0OxeKTsFhL7
za?L{yv`ZiOCXoz-#;jnv^dRSU?v9@7$&G6Xcp^oryOK4G*|Oh)X?C-5Cn0G
z+(e#oad9znWTHWJ2*MEk&1twg`V~h4j}N&eGZx9Mu#Joj>>Go&coeb*Ck~p(hUf&;
zrUn#ca#=y+A(Omc?je0A;D_4Vz_pjy-F*c)Mp5M$xG@7mmd)TVl!CdO4j3>s&a`;h
zrnolgm@3V(CWSL{h;N7no(j89v$(^)Pzj*tRS)H&+p^quk&sn%RkZbQsF{GRi-t)22+3PQ)(aCIVEAZ+t;!I~3e*E^
z74y?n+dfst(meML1x7ELQ;11a$2o^@NqvtB!eiZN@Ido2Vg5`3HeyK;d#ZM?)Zvhj
z&>3j4n_y=<@Lhee2}*bJdXPy>^bO23fd$9!Id|cGU#Px#_`^RQU%uqb>*X`_9&!=~
z?3)o?X6_A&n=&+}1wscTBr@vVhs=}q8-h3k#KPGG2HQ25p?L-Q^VqGQ=dF*Vixs&q
zG{ey-Xp=dxc%kh^qC&9$S9|Xr74^M+4U?uxjGAT$*pfIl5KsXr3Yf&8qaup*s?wx4
z5$PtHn4qAJ6zLjJ%21Tvu_Il&bft|j;3z|JsP8^#jQ984f8Mp;cdciw=W(rj({z~m
z&gXN^KKtymPxJdF8}nA4d+xT%3G86toK7^DsniPvr==j`HE(@?APqpI`YT~!k|{B3
z@zE2f-p}t!jXVQ&Xnk7f{av#aOGb~36vX_ER6&d&yhgMp+%q?cT1u!Vs;_byNqtX9
zc9WvA?nQ|bE6ee<=K?Iislfd2|5{lkY3dmzFE
zUb@s>>l{|I@|ezC7A;i=S@Y&b94M9_3gBn{g(l4Y{TGmLMxli@Sl$HV5N0iSQSoUd
z1baxegBcV_w*>W~bKHc!z!kLS!n+PhuA034%t!7^PItFy?kj%r>~w02IXXbsa8@vQ
zrG9GOLQXCz)6>+^vhhBhT6pKqov)#eMh2t?5uO{((cwfN
z>$>*|Qv)wU(Hrxf(+If1k8#WJ-W)1Y_C}SWV6PJrq_gG6ZeK-kp4_r`mfbt+?
zg>tY$kz1?Pd8#pMzVz5BiOIr7ndS1VzN~#)POZW?45%5tzP|8u(r)%y)pk`G
zelz6O>~*hkrtRWc9)Jp{Z|9db%{q=d&M_mp8b2
z*EKci;Lrv=xP4GV;~435r|VHn($`0sd0*7F^~~VM{77YQ{o?5{1AlYpo>5J=Z=2-G
zz9_kge%nG!HK8|B3m`a<)f;#&S|@jb1Mkd*=$<%Go^Q^5yh+=Ut|^n5VPR%k|Gk@-
zV#s@*$7L`((agFwE?ESJrFruP010W1(pIb-kJF-V*zvPBCtdi9FRtFYwVi2xo*!QY
zt?P!bJYf}U=IUPX3z}p=x3Si|do-|ze&5fav9`EIlJEqQgrJBB!=Xvi@u7}bpD_4C
z4Mc;G$R?ueOQ)8`U(s$h*DK1&(WNnEQ_#8>;=
z-f!3gy*SC
zu?;CrOD1nOjveCp+;A=4GuCi6y)|C~W#Q_C7Sjh1NXDVaBk4C}9vY60H_!^Q3EkDT
z{@h4)Te5MI0`D>)7JiLTXvHvjrpJ+AYvc`19P!p)x^q@k=+F+k?o&%b-6OW-s~oPD
zzb+S_puD_+x5;x^%R1?-YLRL6(pM+kp@joQOV$i((aJ{jg&_v?c6Z01ird9f1_n83
zf2ez4Vh}S-RNOG>4CBj~5e<>mkG3$(zHu!g)2RoUGApIJW8&BYr~Bu+PZjnqL#cwn
ze8Ly;SFT)XF+Z23;9JNVzLsRh(kpi~CGIn2vY>m{({p!D5B12^-A94-ec<5wRl#-1|MzR_LDTby0CNn=I
z4dQIHQ;m(RLUDz6iP=0{qQ2b}A{mVHTbBn*WX?E?MwL?TaDy}#lQQBcxN!L!eC&@M
z%P=2SKXKwjruhn-U!>?5RNdrRfMA+2SylG@3Qj92_Nl(bcsZ$Zi3T7iF{tN}f7UY?
zu^JU5iG{3uw=br#E+e&Q_wvyYONPgd)`S6ESjk65MMZ_YOM%}(6A<5oM=!omBU0+S
z@3!MeAH;8?${gJ@cx|ddk?a9K0rED)tc;Ey5@!R1_yn*C#Ua0uC5V|>EVS6Jw?7iD
z&ncprBgn{*Qs~CP`6~$>_IQLHu;0I8+)|a~j!^p7BhXw&A;(1zHr2Ij3fD+t^jShu
za){p`$cB3l9*{jHRW7*Nf$-*`-Z7*Ong9hDeAF{{&P>n!dwP`DX#(BFMaDGTVQY}W
z$>0INb3n>ha2EQfzY#~8bo&4wX=p^)=-5Sa1wim1Uo?m`RM}Tmv-8Vui8PzI$y6bPIC;w`E~ZKslWzAChNmW1O;vGbS=tS0&BJ3SIHJ
zVc~Tn0$FM2$l3wEyoKFp|DQhRSE1TSbDNB4n9XQ{N->TqD$;X7zQIP-qDy-jU^_uV
z8I(tt(^lv-Spro{%)&?HY%h<;dlXk8Vy>#N+#Qs~+SJsniHz}Ahg225dQ0;*<(#V-
zLAQ&_a{3
z7@~o=k>$C+{yKnuM`pCqSC%H~u9g-nOXfkh4Bm)leBlG)n#|2I_xV-4njN9E_)f!U
z*FGw2(OP?xwCbfKBjr2c69W;7H!GI>ejk_>3?#BiG7^s()VeL_W^9_t1r>;`J&=0Y
z+cYw!%;bSRSmeA)r89MmS09J8DNNcj0N11Ug54>;NIOrpt8Arv6C|Fa{dGG*G^qv+
zx1oFp_MQLXhwE^K%1rxUj*t0`BZ67VKNgjum`f=ryY<`JTuf3e`Sx4N&1AgYu;_`L
zb$xv8EL8lKdXm65ph6DdK4;D^;PZCiid}>Lt`v}v6k`_EQFiVlR~TxrCXj1$JO$v%
z_6i0AK$f&nk6YoS&$)XdkD?3Lj~s*f;6OTj2+GH%-mp=4s-gWMW2HEIJeu
zAsBf?8FOOrptNnWaB0!>q0_0UsVh95eUHo0j}#6PCJ;*0#0_=uQaRXY8JON`f+>m(
zZmU8enM8?k$eo8_p&En)*+aL4Xod$gc|))}Vp!g@zt58I5B*D&#&sCZNI)dGoHfg6
zrZ(V8l;i}*g{~~B7Cg8?J4lhSBs6&4*o~drhn=jo0EJ(;(L(0M~
zdKIWBaTys+mRS3*5~!$q822`BdVEmoB5rv@bbM;}<;;e-RTD}9qGz_;=@@d+o~`?g
zq>4!5vaSP~B%{Nb48qq&h#zU=vc_?@_-dL~fUR&cW>ZFBs-N2%>s~tJtVjS*-Ppuv
z)T;XB?RKEQ{yFqFChz%6)wfb|9^UC-OXjpcnAs!26s(Rlktm2fL`rv~wJRgp#nWn-
z^9wP5{{So(;hua{a4!FO-X6f`B&W%0&LB78dBtG2E>aQA&=9QyM#VLyvqkU`3Z}Gc
z5u63U>nIBny`?}NQ{8C9<&XJ%3}#@iv^<7}w&1Uc67#P((4x__E-Dq$x8X{g`}&@L&gV7+mR|Qu28C@J66HO&&qDM^pk8n&#(SC8HnnVdBGuuu%h*0rh4P
zoy8%(_VMwd6r7SbAchyeT5IXS8eVXA?Y@Zkmj|Sm%=d{+`!&{?NCI-Tr6IbiMRca4
zyoQ+wRTS7s(`PH`0Ps3OBDm!2jzrBO6wd?j=z&Di_FC^sTmbTEz;p!Wd*WZ#_%*Dr
z&IT~kT{E=%pFx6n-`N@R`s?LEsMP=tzMkypmMNsQ{KfBe3pauNOsnx__SxjoADXjQ
zo}>K;pi{IA4U;g&q7y136=ZayrN}$~=kf!>mRTyyJd~RV)~T?ZEv`LuLv!pzC7$f>
zXU^E*BFHLB%0fs6FV`gU3s7vbTx`I%#ecAFNDM;UCA+5bSb;P7^x&CUhw2`fc_vmW
zn7j>!yWD4EUZ;Qk*`BE}do~`U!Hm(%9fL&i14WIHWiL)0E0(60aZyIQc2pP*+;8Cr
zkm?PSI!$=`15*vfB6F!q=1WU;AjT)tw#!!rRzL)prgC6q$tG^1ditbNz_Pa&X9HMa
zB2FZ8p1d!3^=cQZ2>L-W^(8W8%LiOgkmk)ya1kWU^+L9$#YF@tqqaZTWU1!e;^O5v
z&ewEP0REeaP^vP%7*kR{lv~t{iDB>Q(WW_NcaYi^zd!%n55*ge^ufC5-tf=AB6YJ>
zvL&kuq6Vrel;08pU?R$r^-GMjt9{eP59zJ{Bf
zl;Q6LDsDe&-5Crcl8JJrFqz%qX5PxZ@ajFPdh`ICT4c8sIf{`CB`h5>q$n97y|1K^@_g
zCJWHX6UUplc<~|yeVXY5RNA9)E&sc~+2p@b6dBrppft`%^0kd=3n-MA!B<_M)@d2yJiTR3R*TG32tHgU{amyEdD~UvBqiQ6
z{1h6{#AD}T@-%IN@BcvLDRb0HrYZ{P?ABznqVmy3rSgCo3HS?TCb?dxpW>w4dq3Sn?N5Zx;9mq2QsTgsIC^(2O36wBv
z(Hap~doJO-SKb#*T@$YWOPfh`^g8DNN7
z`=%%(vMZpn_fg(?KBU#f+XaM9!_4EGdExUyOGBFuwukDKY{K_NSQ(!co2d?Ds&^;v
zvEwgj>n%u3agLILff5a}n!WnHkg9TlUsg(Rv>RlBIhGCG<4}d^?=lk!!zo9rWB=*G%
zM`kVdawQiDWuQDD?L7`Sm4``wPIADARHx&b#uzvhYN44+hE8NPL1TI}q?38#<&kM5
z-Q%h3sppR})ohdD(`@D6yROr+_x#0_oFa@s{mk78D_noKS~1_xD$vNX2$eChhxHvD
zeH#0qVhRKoo(`qa)l3>S1S%`L(0CBh(#~D2E7OoxHrRMz7D=Y-jVr$Icy2@FqohwF6PU
zLHWzvsf&-unRo`aPJ6`4FXH>Tv%d}1#zMPGXuXLzS&XA?(2<fy7YunmFSt^2)CI-4#ur6%69i44QR-5GNlbX05w^LSE%S?(4aqgrqVbG
ze0i1{pJxxGD?2uhU-=>`N7)Mo`Rc3-$W4_p`wbuKW=S&iZRF<==f_u>e`C^
z>^`~Zv(J)7)~W4WztS*S{^5zD;jMx7_e6$Lr|c!^yuvJHaAup$x6Qtd@sicV4xt7m
z%k2j_emt%5`uT3J&)jn9v3v0+Y`0<%oJrhD$CvZMV&5dx*7
zr{j{VAC9vwcHc!Mv@j?g*lCVSFef00%GnBIE?R5$SeTjZ+C1Z3i8i2tylF)Hx|r_X
zs-mKP+mvsc+}3!WH$~DVi&sEHwt&wG*SJ2=C!Evm9+b}c@v@Or)X6EC?1+FJXkCp3
z*Igu%q{CTJ!@ENLay9z>4mnTfVVwjODX98nhFr>d7cRmwS6Yrq>&B=sf*6IU1B^Z2OTF&lRPPh722vTg;20LZ5x~CgWub=e
zadkr97DqsIgMn))LoQ~M;>G2MageDML$kTA4{~C{hHb${q4Z&Ax^%*UVRw1cy`auh
z7TNQq$Ma=JyZpxN?~$!2_4F{N!vYggyLp?wG0lq^>$8EvNU+7KM`3JjN+=O|FDNij)9-iA=Dg13K~kjI
z1~D?XZwb>Y+73HWMM|9ilBuqqtyuL{1c(4qasfuH=|8`9TXgEwf!`b3tgcMi$3Tmw
zHLr-o1sL)zpPil>oK4;@m_V};GB@M?LWr-G#vYsgG4F)xxo@xH2b1p)IfNf4Z}bk9
zFQ{iQ?S-0#hKAk49apY<+%#9YJR0-c@FyM!S7i9MS5Ofa*m*kv2G3f<>meK>N}w{j
z|Ie|I&`>LpuZd@`G1U0$
z(a35sW2b5edgY_>cr*}Q#}4HHs|;oTG0zAk9G~oH6^!UH#lp91*PSp#Wfa@O`^j3u
zEneMbqUrgqgH`Np*OZeFXDBQ6zB-gKo%?hCM^AE>ZK|>lta<5dY0`OYi;iK5%NoxB
zQ;7UT!8K5QLGCWs1LLmaR2={3RK(4liWsVQMS72YjZ+~L6A_?cf>YsAc29tA=124A
z?-2_>jn2ryZZ@qCBDam8RELHm69NI5+-jj;0eGGX?qN4CWVRUeZ*cy@;02$2QiB?m
z5!9X_z-ZQWJ8{Eu5O$(XusgIaufg~tS$VKxX$@Fy$PfWEG<0fe*N72`L350Pi3w_k
z(>fB@rO#E7o#)0B{lruDxB10y3SR7Ng;oiMmPDlpJYFbn2H
zifP@FkpQq@CzxCvV)!@anw|k31plSg>ozaF1O|5T053#vq4W>7Bj#qFx0Sruy!KT)tii6xRL*Mt;Y}%Y-{(%*AEfWYn$ze%ctARvegRxV3AJ|iOvk0wO
zW-Sy8Y%iT%=AFWjUfyiyiWR%K12%&DJ|yr2Qz3%}F)k2IYH4*vw9hcCfIG<4g}hpE
zjI;y=^A?KEc{@DEGUb*n&8r8>(t+P-ee3Y}coQLEU$ABuV_W!>L>qdJzp-PjtcM-GGJ}9zM=FXDoPaIa1$mmC_PAa8*3tFeOdL@R35BH~%Q-apO
z)*pWOiI{9g=DHr;*x2Y5oS9&kb{!b4Yt^76Do(TEh;~aeitGXV4TJzCr*56
znXq&rDJ;ZAc~r&1XcP8r5=MC$8f~WMm9C@m)SX%R<(EGs42BGphe)Mn3}VE$>*D#H
zV;`!HpQ@R*m$3(1N)j36cJ)$mcnb*X{A??~u9QG7oH8F5LlBjzx)hp^g1{()3h;Sz-|`$S!`0PU?o(;q7mp}^--8)Q7;Yu
zRt$zTKmPdR>uya;>7yk?)u9+q%Q!!DsO!q~5vdtb%a+MtEoRGZLT!~GPo8rW5>+(xG{PGZ_UhNFf4{0rP)gMqqr
zmd5kb;@)H$_p7Cl(zpZtkF&vG9`t$q%#Zs`l|_&IaX9*}
z$5BSLTZ@|J$Y|bRsL7KGzL<31kECpAwDF^wk9u(|kC%)dOOfF^rHj$#57oza_ym6s
z5;4Msti4
z-tkb;qcl8MruikQyLa!B=pQT>^{rJ9v@$z&dcC~7;DKbi-Vz>Gv>Fx)^?s;-5?$Ky
zSSEPBvuubl*5S5eH&2$0LEG;aJrZnG0y}fNVU}?{h>!oLL4H^8Ib8S{VAWUUhM{7x
zO1ZhY+6&hq*lB{)%AE2=qed$@6m|?in6$J5FVAF=LJJZgk~1`PGAEplVq%bh~ld
zUcK-!ztB_JrX~6SiQM3+4leu$irtXSWE8mdRfy2w{q;-cZl)Q&7OI72W@ft1ws@D<
z)x#WR4YCMTzfE5wU#pf}D~ObyY)W}3t*+p^(PPC6fS7*x)YDr@L{Y$oCOIg$5=v>3
z#X~rKu&lpqBIBB`U50XHj{Var&U;G~Vj`lPmp?ohuvoPqQt(?;dL`SQ!ZLH+jQi>0`x#!a*I`BnOSc2(5;r|U<%KJAIi
z?L9U&fl}_$)UoAr+<>QdM^yC25OuW7tssqKmmgRK?u!aL^n0#ey{hQacaHQJSpQ()
zef0;BC_uO*n8RrdVk2kY^6;*l`-{7F@@-5atIPCX-Q^<)|I?`{~o`utf@q2Tp^uCKomo
zc6s0Rq1yn95L{A-0!=W;^@u2Mqb+dd17fOy!ZC7K=1;UIfYsK$Rs?2D9QX~rYtsJ#
z$~UaW&XO8DK&YJKkBxkL7g=#T399JX3$nXQsA=oAA`{6Km)N6+yM
z7Uvdwe5wPX;ZL59sk8yi)#bZKziX6aLH;IfQ-v1p+Oj)5uB)e}Pe9;%!6%6uTulN+wnRjAzV)961XqrFf1LUh30SEp@qrZK;KYbB9nv~>mc
zD(OJaLjzrple#n`AjSh-cob;d8b%4ID%>j;
z6|BJQ_M2Zh3sMZ|ZyhatXlCt-N0Xt8Ke#fG244czC7|}e*{lPI#|q|ebcCy~fz6`D
z5L}Ef%*oslV2P}R%^PhfrAJN31CFJOJU4v)MDplctM*$)j_u?|aB?e+Z|9I!;Tho
zuHQw$5qkqj;(W69&cSBb;Qg(feFj4{^j{(NY5*$h8|eDy4eaMNaNfKF;s0XgZd}>a
z@X)P1AAZ8qaD3A*Q3ki4pZ>qNBUmT_9NG{u$iNH~!
z&MLHcv|Jhd?j!L2Y6%OZvr-k|G4m;iG+Kj$cq<>9#KQ6!X%Lx)mlYvmy*$WkybepfmuJ~y68K^a--Ei-*mAVw%P%#CCehNMu{%JNC^4Rh6;HK^Hhe%u
zp}iFCuWxA`(gZYjdef?+W{u-||7GmzVEOtWH4O|ahU2NOgUlHFQ@qKu#Ks>M19e}i
zZ@UUAMaSI%&n9fMxgSUiYkiTzf1XXDQ`W}-s=znaqr;FEH`=5Yu3=IwjM7%d>(z_h
z<)5Mb0>X`a&OlfvPA^;KZZ|U7OHFGu1i5Wl{|uwKitOZMLgZHFv;6HjfAY!PIp432
zbUT1zQX3@%74y)V?gkzj!1o6+)-~Eh0@(=}A+yhZ`ec@iiw~emCA1h086Fd9R44$0
z7{}n_?mfX9YOt(^)Hu3b=d#>lxLc9MsV9W8jJA{jFrJnyc<~c$1EZb})g;uuz-g4b
zW?V6$Z1ecW)fHHmg$C@cKxeHuM0SFeXqP7kAeWH~9&vR;?fE(|=cZOI(XI>8MWnU&
zKyA+p`1=l)JsU#vIyKU0pJ&&yAt=RyeSsspDR5CO&|4nDg_f|Mm>9r>^#B~?sg2*4
z3AN_#=)*FpL(@JAc&|jQcV{(>MfJQY*wQldtm$oh7IK~Qy4bGd3f4vpT_4bwbssF=
zb|`e&E4o`aKa|_;!KuS~UlkWpQc^;Q=>bu5CKlsCO0Pe29K)#8&LZ*39;w1uXx%V;
zNi#^nkOJfBi5J%$`vqSJoU~(tu~Zm$1Z}s9jKBC6-qF+3JYOB)K(+M;UG0mfkGEsx
z!!T|X8U0`}E1Dh|RXNbg)7}`0kMPi?5`$R_(weqFklq^g9|8a>yc_uu9md$e)6(24
zBWk$=7RXqz$M*gO$WOX9LVO}tg(ZkI7#llrgI>_8uPfD+FwL04B7hKm?Xa^?uyI-p(9
zIsiZ&p}$2)wiQw2T4~V|aycUr7Y+Z^4!e0$$YLgLPRKNbzc=jY#F>D5TAAgK%aVBM
z|4!uumlj)aVP!|PHHp`0(GDg_7zqy)C58W)y^)msZ?iYAPf(0gz-IVjW5{@}x;Q?>
zVvv0{YmO&xuBQ43!g2{F6jqs#P>K~g;NS_Sot5XV-OhIc)Qv@7abC{0vNm3p1Ca?SmXDH7Asy?`@X1}
z;c9E`U(#>J=pA@JYuASdwp)E_`rd~YKEAI^gD-AbsQO1q_p<8H+;JmgUbv@JbOe8@
zV7}1fdC8xsp{5t7-p(}_FhteGGu&eBshNgUA6di?gN&letI!wpw`My43tdK)5(|iD
zg+zecl+`f?wGaKbMQ|ishNS=s&#s)RMF#|&t(LV8G7?w>%QMT5KQ>zkE2_l$F@A9>
zQp0(eb^n02cNJ{CbgqpeJ8T2|Ekgq?9n%-rNGLb3^KK1>SY_;5WRPUsp@0w%a7f{J
z|ID{53!C(=3~0#I66V@~DS?vex39nc+6;mN?&%ZUvgK}V7h?hx#cyrw)
zixmq<#4Siaj*SFl`c_cj6*<~;Pxz?^O!1WFuQ+iCn$C=x@SL0HQ=DYCr(mu1
z#FX9HXzuHndS#7hlN(}#t-I&o&7G_s#8hKW8#IxT%
z`gff$(7c%HJ!G?f-MaW=DY!0B*xAzWUf8(f?o0LA5p^aDjSeMbwXm^%s?UH^$fB)!nB
zZuanTjWay)^!Hqhz@~W#hVhttuoz_@Ya8ZU)=NkvyZVoME1g-mG#CRrSh5WDvlv_=
ztLC*fj(FJb)#7m_4siN(P1Nw+(()uG3V3HU&n!T(FqIdW$zj?Y8M1iRyU-gVhe(|1
zYk-q5^P9BN<%JbM@LSJ&{g=mxsX
zPLJ!DTZL=3jv{@?AXYxa`0NE7umTi-%O0Fx84a!(2Wo?kk*gM_p#n{&x`ahGo9%)@
z;E*B6Mc2*h-X_%_ERw&{#UWlQvUvEWf)=MJrCKpNqi)|;2_*DNz@G#Cxkk{p!0sTb
zC%t$eO&Q*Ui61wm^4c|o$6Z}JrFO1xH^b`f46p8`oezJVzv42e3kT387|i&aI#I;(@-l*`u00T=pE5!&-I7`yn+^II>pCYp#y9)UOzrOe=9_OUKno;K
z{9?++g!PjM@3CekYxdRo+4zjJFZxb&$ZhFtDH;w+?LoVy8lg7?RNrdBpwt#aENr25
zEorFR!ld#78k_JTBXGc(ESyzaN8x8uztNg>7>B!K2}Roow@C`S3e|!3hM~lku#9xo
zj;x>gB`1nL`67&6Q~!Z(I80mg%C)rAa6x4U*i{lk7(c9RDxMkLpnKy`WN{V!!~y_j
z%%|;NwC%;MHk82Jk=ZJ@qJuA8u+9N0r;E4~H@bAyc-)o(6VSjHF!ZfCRfKfDZR*9{
z+@Y-2VVabw^Vw547j?zoMqSC*mld6*aKGrLMY`k`O{oi~qiAomvJ!I`f48PoTlfaK
zEubkMg}Ko_NF2)$hQ)vHle170;z7bv1-`$rwU2KK(U`P4wokKqPJgve*CARqzY^0;
z)7*u+;QQs#_W2o>hRlJBxkkEJsiEtmU5WyHx$t>C~c$
zfq6}3pb3k80JZj#;K3#(cMWKjq+>-X(c@~B*;%+v6s-}o+49;@jlsq12k2nc(TGdT
zemojKG*P9%iO9fKMm!@&Xt_)Eo6>XPcp;0ExNo>w*q+rR;6$g&Ya5uH<_aL|?!oGZ
zs>MDpP8oW=ye}&nlb7gXl6qjJtGyLvIz*uY{6G%Yb{O+ato8T~oo>aG?Vl}^zfqfu
zp_f*Lf%q7Nwp7h!29qlcV%Chis&z89IHb4&D~5ExkZ-;G$yaN7kSr^(#%QzS
zlUsY?{n2FzX3Y8V;WJ42+d%Jj&f`RGJbXv71U-x}vv#BBi{+o3WR#+cy^QZc)7@KU
z830TbrZPgoJ@u%}@>N`*%1^GlE*U+ZSdpED*10)&8K$!5JWRQ<91wyOdM^AtoE|;B
z0cqQ9X1a5>P$L@1Ck~lr>GEuJTaL)RXi0Jab+3SKOF9PJr%HUMN>n*h1p|$G!eeM2
zV~BhWkXSjWyeNDs3nY3GS!+#LEMM@&I@xMSIjm)OdK*<&V;$82P)?fcO^JzljhrN3BCIfFrDo?y_sgiEe*52yCR!2O8)yCN+A{&MN2m
zjOR1-IH@`HBSk5tljdp*ZCX(+xIp8~G3cDoM#!XG+J{5`5T7t4)}F^zOX%lkOL(Qc
zhU+Kue!%X$8~%C)C!A<}4m0GqbQU=pWU)^2lTz_DvIeqeNCma!*f~XN!MYD-ka!r{uK1x|gnnIi7olg`Xc@eJy
zNHN(1N(ipC{eBxeIJAv)$*ggE{K0-NU7Wy^z&;nHe$?*LH
zXa^>C3++p2m>`4E^eRPhrgQ1)kfg8r+5_>W8bB`_|Fz^y&3T_d^grd%_Bswf-%v0m
zac~zp;`SmeF_o)lV(n=nXI8SzW$##RcBHnK)=N=*96+dv35&tM;+*Sd_St>A>rS-N_0e
zL~buBAzlL)Y%Mdh_}g2JsmRtMa`uLFyg>LLjyc?!ccOzWZKBnN=q$47$PSo*M?~9)({TT%x>#Ke8
z-!q4S05*eFYBh_-XWWYK){Q;_u$*6UFpB}>;ytVa)Sp4`zXUcb9ZVH;?sTYJQ
z>Y#?|pgUqDy!2MW%A|8})j#E81gy+mmXDl0%r==$RITpJdXf8v>>BWUvQ6mLmZO-R
zqc3mK*1;PiyiOJT_;P$h;+cA$+4ad1hcee_&OmSLa6=7@keuQJP!iH;xeWq`DL`m0
zedOMsgE#Ij0`xT9eDu*3x^j9&UIIn$85BPjNbU~_>PS~_lLp*!AOMdXuvLIhsxf(g
zHGZo*r98w?GwjhwytJuV@#}BC>4Oqh9n8=$WI-G(25K>jZrR3@4NVn;9Z?Y8X7Q1=
z1G4y-7bzIqB!wlV0ul;00Ix$9Rj3Ba@cro|7un2s>$hUv%OysroP+WGo0aY+4$r33
z)}tTk0S}gNECyT6)K1bp5Hslk7@doc)MVxI@cT1lD-0aprDzfyuIOsb3M)=sI`Kd5
z%oU6nP~VzJvrMlS3G=8&1)7}5@d;N>u8sq9~h+L)?6N=AE!tEA%G^KhBtt@oH>
zyq+RuC;%$a?03Ol>$GD9W8Vp*hFv=5myq%~-ueKemqwNR5AED+i*Dgahb`Lj4KeG7~Q+2)4P2?
zmGn$1E*d?OC~=CL2zIl2if1Q_QMgx4=3VE7pa5OV#9a$yP(g&C)H&5B$Iak@5ulN%
zaQi%~Q@2OMivHul{yu7yo%40aKUx4l2ZIi*w>#ta`8efDg33WvA%THCdrT%YSk=F|
zfG{*Dc+jGo&K^4X-Un&04%bp_LM;aDrIPTf(5XAoI|Kk&r4p^!P;7r%%}m3WoSNMZ
zUM`x7TITSj=qUyjSg4Arz`EJOeSBH=y?YSft-|DpZ)8BF@Ik7yUz}RWU}7xvDx{KW
zyQi%u3Yv};sWqd@rW@nuVa>i;Tn)bY5*cx~Jef>{rnmILY?-Q-_ZKOaWUc^b6UCKs
z39TR;8udaF4_boVt3&&qQHtWuMrj;`EhTltXa)Ddn
z9?(~dE|$9E^2U|g;Dg)lLMQDWVFW_jeYl4a{m}sWuEzaS6n|;t0O$;=FkkOLU2&)}
zy;x=_FCJn8j;50rYP@PVy#`}sN%O;3je~o`8IbPgLQn5PAPR&i_gp?owQK6p%1jn&
zSbN7Az%GVhLyid`Di{|Th$;>b31D0Jm^E_MYuH)!cxgA$#fEO|YkIramJ
zWk(7H1hIvSU;TphVXC8ncUnP#=FsofW`aM52qcnk=~0xRgT6w859r;gd4aIc!(+ZxdYhO+IsswKMW6}dcl6-&o0)R^
z4_gUdI$kBGtMG!?SZ^?j{6kKNT3Xgy@tZ49X2Er$05TRzn32jjYL{QvM%D&z4?m~CW8HoRBH(p);GEgcF#jFhj%=!l<3
zW7*FI0_!tHGAQ-}k*6UPQ((q8
zTQ#wVh;Wohf*x#;lBxz;4MS6C3!LF!&f`ep-+<*|0ccM1=ve7$7@i*fI`A2l20!|SaK0X;R~clkY50cbsEPXH=tSm1BgS5KtRrxuAPUlfN&qNa
zTddhr{ka45tHGW~*PBHH^`Wt~`<~nIJ}*vCM&G(q(f4T&K2{lYZeg>6=vh67;;^Nv`S^EAiIULDYk!c3Ng5g0?L4n7;En!bSPXWr+7V89Lb~C&d_whisOTWd2h{@@CbwaT7dbf1DLk@roBMS#^Xqrc
z#XtWRf3^Jv5agYI7P8$(1=8DJGDrVE{KxRs@=$Y^gYk26`KKL{N?3b9X(?S{7jvgB$YmAD@*XkVPR@
zTH=NXvVAABw}JEikI!^VJN3r(<=|tTD`lHv>E%(K?A!M8-Z@Nqow_OjiYM8x+TSKT9-bfXC8sV^tGDMM2jJ9{kJC5Hs8rV*bGoGE3#+2$9
z#2E95hKU&>a>)zMn}XEYiPX8*K_@q`C;gFpG<`&BsZiYqyW#^rK-kxDgGPM6&BZ1_$|z3;6QL>c
z#)ugBwOckh8Mh2`%s!a^&7;UgwJ|iEW%?#>
z8?1Ic@g`AJNPVWP9{5^S;8R85$!;Z9^YGJIyNua?c$IoL+|w!|Tfpd}(Soi#2NMqu
z(L~?xG>B5O{QEP@SSTHIykK!~nZDF-)2ELu|b5au*OAq!jWJuy-W;Y=tXu^A#4fp%Iih}uF0*VO)(NeH@EGZD76
zooq4u%|wP|F`$)5mhr(bL>k=a7yi-Llmlg$1^^ubABXB1|HgOR=(jKYjX!_+vHjp*
z(y1N;4;CJCT@iFW2f+P!Ddv|#9DMlUg8NcF7geU7E@rYoB>Npl4N`{a7*0J$sN<{A
zTe^(IEJ|i-geR?Wx%j_PMCis);&I0!5THyJZq_Q&07eRDD17$q#_y;(v}W(+3s8SM+-S#7
ztAnJS!V=!Ro-#hoZVus=ZCF#xWT8+~Vv7Cr!e%~a97vTe3#DR`ky1Ivq_5l-1;vuX
zlWOiDDOn7#n<-tp-}yRU6XU;hIbJNFiWrvzRe3}mq8;zp1Of0Akq!bB7b3XOE!NXwYr7Vm|^nOX&zpHp!5dg0(k(7
z-RM&PtYf4uZsC=SG*6O|RiVJ}CD$gfc*%ndnmlY+6PW=7RZu%}^xT5GARLBDTy
z%waYd?CxA|_`JL1DP`|cU?mse3iv;lI((w)~
zADk0Gp?ljRG8Ktwu9{oClE$#80uBYHvxxla-=A>hD!_v>^Ejvm>cchzpYlQb^rDMa
zfnMt+G~XMXLKmV>IPLsE#4;9tLti&0n4H1YG`Z>h`%*#%h58?HE|Ek>&|K{kJa?~$
z0mv*cVv&ol1U4Jh^;(e&H9!Wt3?#%^ApEK0&Rm)+@rX*L(kg*7@Xl+B{_!Al`5~MV
z0rfKQP4-eB2sdmqE(TG)zwyIq*E%+sYn{{Q$8
zWX*Kc6qjayk3E2TaMTM0>kF-da>BnA1pc&Wtm5K#*JRuhW*+tye&=O{!l}mo_t(O(
z|ND>s2R{w{?!+w-+uVoyl1#(X(UNI$`{duJA=LWef9za&bmfzG&({CTf3ypJGKG`O
zhmgQN;Hn&(P%5hZeMorTAtYXW^K1KZD?^0Z5G20>kZwLmP_$*3*ipfCM9`B#>=G0MHj!
zDM1M0#Sib?3*`uu6+x6#h=~7mfnCKr2>nK8qO0Q!u_@zG|xPVIWYLonCDum#q#
z#wJifGFTjBKJXT7R$(AcN!>OcpU?&?o5g
zUO{YL9a1!MZSD)3E*K1HSEA#&Q*Dclv4vb#bq$336idoUqu~G7Z8Bjo<8rq
zr2!rPQ+(>9t&fW!>YHr&&@u2vt1htBEeZcA+qV3~f@-E>8PX@-&0%oX53qL|7C)$os#Yn~wq43T9x$i48tW{*z2^BinN072(!
zfSX5t!BgCfJrCCEuV8OyhUdne=HbybA1cN>wSb>0l}q5?4TjuRU@*LnEG`nWxL!E_
z&S(j*o`Zy3!8-`&@n%t-?7I+q%+)FTfNj`HPjU2v=}eEe5v@wPxKrNLVvvoCaNZr|
zF_1v890rr|V^cPm$t{H(X)`JW$Z{Mn4MB%hO>K@P?{2?3DOH4;rxRig1{0{;TIMkB
z3JE7hqikBUhEyl14L}7QsS-(;M{Np!Yh6=(7fj_=wIZ>`BzK@WC@(AVc1+m+@foI2
z#F#+o!J=3o8+HRmj|4ZgV~Uv0uBcg#3R&0rShIZ?+a{(wM-SdryBTXp_BvWhIU8)r3*fjjm*2e
zn1MD91-Q#yES$s}Rc8|t3GtEx9-jCxn1$6$s8iUofK~+;8L+kS-o1NwV!b885|f(o
zqP(1UqU5b$lnh>T(H87sXZn%*eu`4@VkCwjx}1qV=y}b2Uxa!R<}pg)aszLtxpHte
z{~dNn$m^xnR(W3p(>R+H<22GIM
ziBz+!Ir(5F=N(Xb#Yc0iJc`~{OCUgXRvREFP9j@-78uiiMn{Y2TR=#oUIE2K!c-Up
zFC|WZQHrW85_9aDRz)Oadf7z=fnWc}*Wo9FvVrHt+3F62xEYcFQpzZRQ)9P{V@|m3
zsIE7*e;|gd2R)d0?p7R5eLj4w)0)d8(coQNJI$W)D16jreaLe!|*41%o<_Ki>?
z3f4^0xgt7>;vfFkN0{EO`Q6+6<42&z#iHdLI@T=3+JM|z9W>Rbs
z#|lk&q9m6smx^)cB6u=bLN7+AP#s-##RZk`#4&{36D0y!YKo&BCO_AU>Pg=|4LlJ=
zF_@U4x+8MgRGUEiRs|M*GhC$@Otb;#+SF%OcpQzaGB0_{jDvrZh;g!;e-!P@IjRd^
zSp}9?Mc@nL5HCv{YQsmI5S$G9;_yMK9vd3T5pU-Lf7Wz!sa0HSl
z2Ja158@A)xz{HXc%_Xyq4dkkIn
t+g~T=sX6}NfBffh{QvnhhGswbvav$#%=?RerXR=rW#5jJpAP-@{{WS4#^(S4

literal 0
HcmV?d00001

diff --git a/docs/img/pgsql-migration-describe-task.png b/docs/img/pgsql-migration-describe-task.png
new file mode 100644
index 0000000000000000000000000000000000000000..a28ef5594b1487b47516d1e2384c13e08f8a6096
GIT binary patch
literal 235162
zcmX_ocOchq_r8p>Ldb}sqKK@rva-s`%HCu|iBL8nWJE|vSw%*Llxzt}D6^265t2=|
z-{pC}&#!+zKF0g~e%<%E&$-TZUFQzdRKK{N{17<-0l|J{B?WB)0up>l=t4?}|IjzB
zj}j2vAy8J3)$w>T{dAv>&X1Kjr{s^6m5iC;cha)m=ns^Cs-ve9*1FQUYNcTn&AHc`
z>=9RS-3o7^`+vGg+*^PucKx@86+}G+2mDTr}qU;XEt8G1Ku=`ZrDC
z&6_vp$&8C{aFP<%3a~Z^?wxD>`E#QtXrG~>;g8p*N`>wmG
ztE+o>tj3+x82|cI!us=r_V)Gz$1hkgeX=SPpme=;>(Zr5z6+zuCMJFP)!Sc2=Fcy5
z=3MUR=om3`UYR!3(D?p*W1i&;Fa-vZEbD6mj?5b
zFH&u-ttpv>h-(FoAAdG9H1zCZbQFu|*TF&m+K~M+n{z~U1Rn2;iprgbFb7ZISzQcT7S8vQMM;T3sJN*+WJ)6vo0`_K^0Hdx_F
z_x7Ve^S5vOCr+FY7Z*QuYHFa;+t1I>du4j<@B6{T^q&Qm^2*A}9v&W6R=+N?`4S(o
z`})Mt)b!P>SC?M2ElqY5y?HbJ_ixg<6b#-;V
z8!Pvj1dnlY+I)&+W(t{{oIEV%%+AlxFDWT0F7DYF%ZUy6+SKIg>Pi$ZDJAtRBg4wd
zii%atd8V)Qq(;j1-nU^bw&oQcF;RT$JJ=t3dU`C)-KMb=NlzhBQO1SFkHHjDQc@8S
z5hvB3kVd_G`}X$iz|-VpmIDWFZf~wlOiVm@Ky&!;Vai&(;$e#_LY&5(TuOR^4;0`$IZ>n$Zsa;HleAa@}=$B
zMepS)l9;)ze}gAEIF_cmD1Nzqyh~PET6!^xRZu`6?TQ&$c!mIF@>zTBOP5;ahHLE)
z6U=4gFIs%FL$5q
zD~(G^^2B3H`>wZAvAZRUI`Q%GMcuo1FFJa#@X9`gsH&>%FuIcyfjf9T4^PkdhYurJ
zM5#DN#Abf|qQyTJ78ZJXx4*sb7Jq+pEj2On(9zLxb{4oS?;(AA+JJPV%6H?4lo$P>
zL*(^zPZSjtSf#ukGB3_u>BzdIpKlg?lA29=bNx@gAf@Z=+fmWcY$DeawN(UV>>L~p
z9XhnWy)j!$ckyC_g!{}UzQ)015q5aE{=7zDy06p~;kCz(L;1By>6_=ztq~jaia1~^
zYm2`vEiLi8s;a6uG(nQJkw4SrLoX;Q28V`XF(f!RMu%#GriudM<8?1w*!y{HbMf01
zD=RTpst`-OlceVY$La4)O+07LoH=^*sI>Gpe*3xz3rF)~;oa|vh=|~OIK2Y{1EM$j
z49eWNeXN=uoz~13N+*$<^xj;(>ETh5nmXt{=$nv~M5fDEk|g42glNFo^4nhfjpHOw
zv?u36>?ksYk%0lJaOsN|Cnsuci}YU^#p7UZ%vQ5W`-lk(FXNqUZEZCJPI7WO4pu#V
zEhj82EH5XwGSmOl#-Y9N?np^y=2(3MV^L92ZEYe@mrCUJ45<
zeD|(iK+V{=JR^fp2oV=FYFzCnWg4fEBFSua+PL^-ZtkgL$A0y`yN0ljJ|cDKj4h={
zfW`XST77-J;)M&_3r+lZ8zm*B?gGo8$C6}&kHuYW5q-!}ugxkSKYU1k`0!TEKDIkI
z#G&t)m&8sMNpLh+6MJ;+%n3mtRv9&#dRQWmPI1U2#
zV6NDIXB#_CMn+asQ?s-EuNvzR7#Nr~#D?5)=FH-FOS1RB87ZGtXKn3voXA_ZZrK-J
z7x6|6BHQn#0V=Afu&{uryN^{O+3zJLCT?x~>sV8emmk5~yi%OLIHVZ@w=
z&Knpo^;TMTN4qxw-k%eU=w_dCAB;EG&6BIpVg>iAZKR
zN;Jer6PoaOD=RAv`LABRdM_+2MkSN|3_JUWmX=p#W#>3J6gX&~N_mSMJ!<*q`zsFR
zFV;eV?8ugCmoAODh|AKT(7k*2jwxh6t5|Di=MvJ#+Z%X1QDNcx_wJ=FCp3-OeM>og
z>QrxmC6Q#n4&LkH{X;g6j!)y`4<0(?fK)y{{^-F2O>=YV!)au6CrZl7TuS?Lb8~t0
zU(+8vC|H@Cl(hfwxlTL^7yTpr5d#AQJ-r1}R%WKK-=>GEYSYV?FGH!>zKku*k2EAC
zB-q*7%5gZCny|64G3AFVWMpS^rE8LYp*|w%ff)Jw_wT>iLH^UH_ft{X78gvu43)h5VgF^PR;CEVETOZ()KYo0QhBSogaBp{aVq9EpBp;3#P6>9Z
zv$GRX6M0zT`{&PGf`WplPN~StU+F8kX>V`O$;nBh=p#KiI9Tm8Sj7|)|L75x$;R6H
zv-KOx+T9YzO;4Vr_Pwo5%fP_E%IzVQ2Rn)h!}C
zyrQB)j8)y*nxo@B^4Xm|L~q}{TU=Pkc=qh5e%=&8cnyJpvGK;gLI2R3-Yzae^nzJM
zdx-XScXxjta|zEtkf1_L)E$=btEj5-L!owaI~(1=B6`C_MMcHlo_pyq&c*%v_woKn
z#aFIe@$vET@;Z0gr!!ltrKJTaDz|VH6?tsT=K6ILUEPCpbVOw2UoS+l_Vo5XPE1tV
z$cXQ3YYPhx4>ZceXU)!5La{*RK7IOC)5E}%aVIXhI6HS2*-`7LkOybD&kh9b-S4%&
zWUmqX>BkTKbj`Z@`hc@Xq8Vvu*khDF*485WQ2Gsdu|;~wi5D(hu(V9iz3aEN?&;-)
zOs%74(DaDXQt=`)8zGAi1pfa1VG0YV
z<{v-GNl27qN&70j^*LzUZr~yJ?muL0VWEi(boj8Tt}cmI_((};Xej==b4oL=gInl&
zcLB0!QP1?|<|blV){}$OYH6aatE&sKra8kEe(0QA_IY+CW#!4&F&S}P?d>kp-Gx5V
zGN(?R5*IH;klwz18>r{$)2Bnts45K&q2GS%>+Am>9R;Frb94K^Ygg^J#e4cR5(^2T
zEPfmNYU=sX0`F}3_3Kwwsu%d285zg04Mjz9^;VY`
zVRDferE_%rJ3c<%+^lx>a^lkLp#MZ`x|f$1LJ=Sd=f}~}vEPPvb^WDbQPtXbG6Iu_
zK73_mWqNuVpe(Bhn{)q=uu8h7f`WpzjSUG22{RLu&vKW(o1PXwTkf@s7cT~~YhS)R
zGBT3huf;`=Ki%Hm{`Bb+0Q=h7T24+5zG&5T#s_=%PFO%-VtSgFo7;0~Ld(_l%w^t$
zrr-&iLIW&aVPPSl>{OgLq7y;e*x2|$Ln(G?dHL`3w9mpQ*{dn+7#3~J#S>)(kAT=Q
z;rsF9$G`dc#;;$qk|VMEQvTb3t)?xE2S|wt=c8y?7}*$83vT@>=F`6uFk>qppHkiU
zJO`+&D^LH&OT&)-$XTBG#*?WWCrbge&$=6LtxbPx=))d^5ii%izP{Ml
zSc}S~%$K^<=v2Fry(W&#ZN_Y=IPXL
ziG$=MOsyYjc{VjG?%d$N$(qn;Z9&cz*LXF+;)^0bd84(E5wCKrfDOe=sR>_EP1fIc
zL!zhYo^v+_v^|T9qkEZyc4ZYnNPzj98`+l+v2S9F3$*uqoAe`aJ!c|JYmsB9%v(pP
zsI|bD9YW&M)~5bV;)Cl7MPkhP3Ie|lq2Z&IkJM#in#Rz;@iH`h`efVl#wI;I9TD!9
zrgMDHb6L4o%RM$a&+g?2C@?;yQ;L0lC1~wvuTz3ml9s|X#pKk0ug#JKjdh#kcL_V$
zN#!O_zhd%9h+3I(IUuQJ&Y
z&N4susgvI4{H05;Po%o(kL5-ahi7<5ro4WA-i_%CE
zs_*EaYnVsLJ}hy&wKn>Q1I5$mnT+gmrHR*rY8$>e21Y!YC4cz_be1REIkz+?UpYLi
zT9VHVn4(>F>bmxmF502Oow1t_nIb+u9?8dT&-6@r|Nil=li80KGR)+KwVqeeaF=Rb
zUd$ej_xhl+kQCD@FP{I9?cr!1xz?=<{9mLz7wUfg^zMCYZ&k=7Vq;R~HVLpQaA@-N
zr#~}vrXlK=RxZ<>TYe=^9bA!HES1Ah#_A<_CO+1DnWXx+uxG-NuIx@uBqcy-+A!i1
zFA_WYkZIAwq4zx^?Ng#~tgW?`>(r_B`35%29~Zv87&SW->QF}BPoi+Yd&a|FdQv^h
zA%3}?UIFK@u3TR2y2*8uO@?fEY#Zys{WK+*{W2<~_+|H@9;*Q8axlm8`GNQ5j{!{`zM;Rg}i%K-N=59PYVPkn|
zwTEINo&O8j?klbSK5r=&#ZBKW(YC_v*s^68cx+F=d9Ip1Z{<{lIXj-{5gK#Up9NkiSftQU;kX=`t3sFknOZPJ6A>)zXcC)4cIhSD`#xp
zqScwS`k7)rw>X}b&QBPCzuHY{qyZy$5zrUrq
z8MrZ?wc@{jUvr)>K{Zpm>ssM9``NQ+H@CJfsj0Q@Dcb#%yKO=>6@Ao5_)aA8!;L={
zRo(gK)rLj3rfAW)xx+Y=lhn{3wr8s5<>h^QD)H~1)b+LaJJ}Y61^^|50co#Z-4}W6
z>&g`_BqZc;?bzLg<=0$3+w~W`Omw}hO?Sy*j@%lv@z0aqj64EYauPRwFU|7_X@8xczJuD
zVZ1%JLvcBFU%jyD6V
z%gM_t9y_2M>sA)YCL=8-)(rr4$%SX^*y}5r>h;eWudPK)(w3w;L&}{M70x_x9W^}
z)BkTTNfoHmNdq77(Dxq>El>gQrb8IkE2D#tZpTj<17D3LzDc+YR6)-Gc#?#yrIts?Rn(-`|G7t
zX(=J0SSoe*g)T-;VXZv@jyG-qu1@EL`&QD1|KHA^U!ozer5DR8BiJ1LLC;AWW-Zk7
zMBMe*@#9$iM6r*jn0>pWV`5@}ry}Ydu3n{ys7EfnF7mUk?s-W`H+o;RNy5UqC*lC{
z!s>qk`(<53KUiv#^5hAnuN%R+n)UT{Fj1fxCV~I8EV#b3x4+5E%uG+OL1yphC<5FS
z6pRt6CNAvl?FG6<8w5;-2g=II0^fMwz@R!Xjf|L3_li2rwJTRf#>c<+^*wp`u(PkP
z*~iAgVd~c}E1+lqHFVkL=KS$0Nmf8U=8YG0&_Z-}o|Tbdt1SQ@oSdBeA}b3^KqcNBi4UkbpIIL{{3hpA`m`!K_zzl-BdqZt)LZhU_#EHpC;560-oRFlXaCXvb*ZzF}{vG^-yu3U=e?E2#OwPGPc3CXm
zr%x9yU#2^@n8`{1-?T{>Fmd#uT)=_5U#Nz5n&tevqg5dJioV69|wJ>qc6ve@pW2`ZcFWNt`^n
zueJbLWAEO*XvBVrRo(vEdns9@ztTHlrYDZ0)VAeu&gB;~Jw+$ExJX49ZZlGPZO+a5
z?`)tX1OCg(%901mX=u!%Wdc<$Z)bOsOY-Kgx_BOa&>W8+AM*9}l}@d8=*n$v)r{rP
z{b4$s&9NIO=H}=WMUwyi`O`G!Vrdfd@ZrPMRO;02AJyB!P6OzFB(XGjQ)zk-UoSdrtcl=^Gurna}ys_=*K~c?jSyepT
z`y2gJV2!~;;h1|*v>s(`2nq~@g5f^$Y5(&4CKW*`4+y+C4e7P1ynOJAJhWkn|Bmiw
z5qXJr`St79;K{x`xc`&rlAKD
zVp03{?hUX11@8Bny}fCr*Dcfqe71(D!*t}qD4BRO7Ln`7C36c4=Wb6Y=I75M0IS!h
z@*X{6@b%pSwT{d`%M{XrM1}{(Vb#!Z0;86|XBzl1Qy!Th%LHOXn-7yXas*f=)dEupin1~dgecv3E6XpD4UTe@W$-<
zSd~m9F7zOv8qU~#t#JJvrOEaDqK#0}_3tkYaVQ3h>@>g!fr^2Qbn?>U=bBu|A_-04
z`c@tbUmI^uDsmqFh&-ei!LTbkVAq=)8|UZdy320;NjhVvo+^FcvOAVjRm5rFNo=gP
zsp+MB>K|QQAdouIk_reA%OVLyGYMJ&r-C;HYejcLZ53+(_B8F9G!!oIyC&tg|28L`
zL28?snZas2tIXresgZk+!-;;}VsmI_W+2~O#-J6ejMIeu9Ep}OD{&qMqY3t6)n^-_
zCh9b>i5B)=y%il@96m1cILY34e|mudte8%7$f-%L+m
z7rtFA@Z8?XY4D9rL{!vQAc3Cm--{i87JDv^BOPL)vB%iDd}J*F0Zn!qlzl&ZOfX02
z*70M6EO_NZXO)h{czAoWopZZaJ3P0xi3)VYF1cFJnbHOOgZu!B_{V#vK}06#_VyQ|
z;0UQ%L@4z_WLAfW3JsdA3W*72QDZ;}M2$2h@TZ=&Z`Y97didlCZ@dcR9*_}zJw21Z
zehFRs&KsjtXIr$7iuJ0K6ZP6?ci|Nr15K`nkpYwO8i*OJfD|4xl+6#}IJCHi=(M1d
zP5s&T{ktA25ko``&9mRPKfWzpct7ZO&C2SHZOb7Kz8Iy4k8WSVcg
z$o6-zdI$Mo&z?O1PgU--W*~)LKFBw#?5(RCHQ!m+zjbSQsw?kza}pf`1D4qx*#Wez
zfsxUTz7j2EI<9ok=Qw3M%RkHkRffjK4l*$@(a;c-2jeIc5)t_$uqYV$-WC_Hudco{
zdN+ow1LtoNQvWrNnJfEFOMh062e0&^htd2_2~vw#jDYvlf%No
z%zfrbw(OZ6KtRhonV$A7r_6Wbwyo_6R$1b#k9ZStpH+bnOPspl;bENoXTIVR5-3C;
zOZ7OE-{@F;3cG**64cv=+?O9pw~VQwTZF}v$%`3wsvfOK7WWM_MVD@;%BwL
zOmfSNHYAY_APrR8yuB*AoZWiimfc7g6y)S!Bd%Io6Gz!KCmvv6u)T66$g&$caYhCj
z=Od@5R{s8kIPpPXx;0%MG4Mg)`nMD*5PV?F%5aQIOV`j(gF8!#*`boRTL1xA*U<3d
z#S4EtE)GMpeyabrr-4BtnDoCCR<_`fJv}`~8lqpM@*5XV<5+>*oqG`+Tn}!hYI8v~
zGa8Kb$&)ALU!yl*3i0&tP`GeGZUv$>^bm2cC9Z@f-DjmBBU4h$G`^}5Im&EI7lD$;
zOV0L}|AQ?1Bj5br(h~7L4&@^{ckx)t>
zU}i48_JhypGhq10uV09-tDOH!i}yqWJ~|WD(mPz|23wTSbV);_8V~`g7)4GaIWp>y
zaBgMg7B=6aoF}dkkk%ueD;F#ozC$cK=TT8oQi62B6sw%6&6jY^!NCZU)%6}BF>@n%
z`5+Eu0}`9p0@dwro~xbc=&r5(mZ*
zzV_8oN*f6~97P@I*j&hgf*cMx}h-I9`eAjvYEHX5CrG&C^i?(HRS
zklmvD%9hbFgN)$~u6L6WrGt3yUND*g_JVcdQbNSXtpnLFL?1CVGV+=okP`dOGSgs1
zKFgPyoh!>feuW8iw<{hcQ#HY?%14}+cl!G)W8lnOGsNTs`;tI`SQO*IgTU(0)FpA)
z1O*EvjIGAgj84P>q3=>02>hiZ_BlE#jLrpVd^|is(;JlJ!ujGfGq;+94&S#XST3uh1(=|>@Z4k=1!vkyYrb+O`13z5?ndtV$^7*N_
zMhN+F@$vQ3eYJ|*%^g-6a{P-cH-irCqp>Ty`_l%Vw
zXD~5IKwlRV6N_XOM;M2Ogs3M8WkTEpp$_~~RJ6$EQg~*+a;&1FB5DvnH}@LKHF)iO
zlhTYbmM6uHc11ntg#^x$G+Fmhbef#)>ThLixM{l*wA{V4w8VVIRxL$x4Ym)E6#&TY
zrlyB4lRoUwS2-TY4!mHhrFAq$34sOA!%bAtnUzN!u}2HW#>SR-)|YI1RkgGZ{H3tn
zH*s%jhQpPHq9<`pk>=0P5J9MIn_cpmXQclt4IZvstd}b~pU~b=L-ruuzyGk5*V*J2
zzm;ib-)EGRHz7<$)I){fq9mLg=21$MN!R33ihcR|^|-SbN;ld~tu59f#_dB51I}Ws
zu^husj$(tb{3TH&Z{r)S3$(FONFV@#e$5Kc;5w`e9>d9qaS{LLF$b;`iX
zm|U$;OM^ZN2zj5P4old5#cLtkY3#eK%q!Kc1bXYdS7i8z7!v;Uv`)D3JO#GRZ*%^l
zt}hURGb@|01AU|;#J01RABj;j#7?ZIo*@V+3A86kQo9WZhI2^8CjI0q(F1a~vJZ9p
zt?jv}=oOKKOifKSG&IK5d-)c_7Be=?&&LiVazkPs8|npKENOIN*qO-0W5@=C4cEZVKD^<`Cc^(>-{wDG(XbhyP0@70SQ~v%t=r_xDRLq%GC<1P}Kpnk)eHtbNCkKZwn~$=xPHd*N#7ap@
znl)0=Hq1lKN>bY^dJ;XD1_v#$_CW?2^xhl)5otMT8~3sK2*A1GCCu_kL`C4mUf^74
zV$fOCP6QPNT0FU|sreu-ZgDKZT<9NHEJx$-#iI&ux4{G;t1+yD#2b4-y
z&sw?dI{qCRI@JSEboaKaMfZn+yc0%(Eh-8k-5HJ3$hwrg_x*)sj)MQOO?9g2qvYZN
zas97GX^3lvoztS#wR8ZdfGpq$+mpOC^dLFaZ+#Nj8yYV%LMYYYb&!IVtg6(TLj)&R
zTWpJJW|85cjMK8R`k(Ry}aZFLo4%B04Jjse=C-fI&?*i}bIDX*mobTCkx1CHq47TpXwpoH@PBGN9^
zR}UasN2l|>lR004H%<_?3BBfrVQu(N9PRB3i;KZvhM#RJVl
z7e`B*;lA>*vWCrP4jr!SjT^7ANv~hepsbd&UTOW5L9fW&LZL}tKkUp-3VGz#&6_=i
zS1vj`JIl%f2BkojP5owGe%ldjNYn^~PhsJMwFO{Z+S|3E=R-_Ou}Q2JEm%^C9nICv
z%gxJ6mGo@y?6kgqUGmB&N|%LxecZuyviJ?i}dVvtp$6gcwdwy{dSd6e~0MYKVwfqAP$d+&4%uAe)%F=)jIgt9z$;m{B%Gwke|83y^8&H%X*Wt5X7r|*`2RB(Pjlz#J4BZIZE=5U6c?Z!2dTZ6k
z4~=|=Vg&37vWsBBZea(|v;1_bewCAB_|73-?Zn2!v*?MutDVOannXdGf-3^;1i}u%
z2sNmi{b@={6*g3t{|?DqU7bRK+1JmXvtPb^BIR9d!23vv16c#%*Zbay&DGEej0%S`
z&KSDk`wb)gWw%sY8kOMy;^O18jq*psXlTfMH?ye76+Z$w^Hj#)5B;NZEXX@}Mu-ED
z5>d$uDh+uVLW6@Lx7HA>K9$)+LC6)a;wd=*^m{@hg_kyLs3H@*wKSr%c=?x@8KAA(
zWILPRs^xCoU|$<;q@%1wLuYAgE8@9ejR=O35{C+Uj#G1@VDs8NusyWIf=&aHO06(o
zoX^LYL`gqGGA9E$$7Qe~l8^5Q5?{XnFmbTP?Ke9>!zxh|&qFv_j$Ro}FDk%_V#_H-m)qkpY>M7HmJoG%}8J5<%v5#(o
z8vH;{cQ;o*EwK|wbRfNNnMujXIH9{-qjUP}_E3TnBeEkX`VC+VKvwx@s_H=jt*n&d
zQe#UX$w#Y(6#nW^=$sa4zws^w6?m86=`>^l)WXs6_@ik(rrY-UMdK
zY+Jwe_hDgVi$WaWk)o^a_JyV2yp(dcgZUd-e>AvvZWu?dbn$SKH3N(1-k@E54yywJ-6)B-yt4!-00{t_tnL
zSF+a@^?e3{u;+k7+V@|!^2`8TB
zyYUIi(ts{5_@WZa;V4cvUWZzZT<5pF>FMVu4YQlOd;ZIpeQ#_id~bEAYCUsy&rj4p
zm-FdP?qJH#qwXhvX9}5n-ENcF^Z9-Al6ERzGhKfobrN6sQub7&G-H`68GN1X?%qu8
zGuzWby#1a!sS#eQ&n4{Ag1}T-SPT`}eOp)rk6)FXY>bi#GHD{Y@&6W==AI~V^!@v_
zHzg&H`D{h}^`Cm5vRyws?*W@Lzy#ddy8B6!TXuQS!yl~*#o$X85`ru-=5*Ma_}9~b
z+Tm8tw?#$JqeS5VW2Z^`|B8vnHN+=+iafypzxT3P@uYR;N&ZIn@BU9plTW87CkYvu
zZC)TZN$-5R#wAPhcA1+=e@FgCl_LB0x|^X~TkrpuAy3No7sY=>LqE!ApO+iGDNnxg(8BCs2VGV8J2ldefathMWtZEJg>!A2D-ACbwGi`4k?C1q^^cnwHJ
z!(x045r3wp9w#T;2=%BQ4{aC$!`D`*;*YL&d~A%Fh2`H=m+G;YJ9@;jn)JHxju{%-
z+t@(8!<<0_eRf{nPY{+13oZfYE$r+XM&>7Y6=)QY4PjVhduVXa|Ih>de)F_v&n|0e
z#l*$MrKG6GKEHo|e{zfMZO2LN%V7U|ddLj>vGV%*@3XV-y(YWK`o6gMduM0eO%uKZ
zNJGfBZ{NSavOAptdngE$KBX5r*03FZFigPg!y#s7*%<2;6Xn>uj|=ppC>_#I(czIZVdofBwp
zom0VjWSe@rx?SJD2f{b-XLc6B6FK4m`z{dke@w3+D90Dq;BQ;%EC87YCp${%UAc6W
zpx|JVMRKq34D$cf6coBwuTCp!5zCSi2SUb2>zG2nYgUEBvg~wL1c!1g^c={3kk7A+
zBpc4j+Qlf14h?By2S9t;)cok|ysLhoXb9&5!IbfzLo=gku@#y3lAzY$=G;&X>b02L
zL@V@BvcLz8*7S;h`$~%n3Y<}=Ih5t%%dWO%fK}gBiNMgIBj$*Ob@(L~8ek4`z};F?
zC>_bkKj#^zp{cJ-_rTinIwvRK6idTR=-D=3ARVS@a>48P*7di#bZTl+k}^%*bLBX&
zuO6eH4^Y;A0;LN0h9F{XxaH=i6bm=-ZB0#@K1TAcx8>yU;Utepc|!
z78Yqbr&<~tp@{xtH1|0G7bi3@2$on__z!w}Rbrp(oDvoi>V-e9)b;n!xA#YuTklT%
z`2z%kX&M<$PDL(yDrO;Ap=?o1u{7|G!!9@t8h_pcryi&SSS7eEEhlFZ4L+=rz>DXl
zh=_tfq610oqg71T1W9=0oSPCa!?#4iXXx(_t)h{ZICl=V$T$u%hx7W$wZo8HUZka2
zpfd%fjlH5*g!&3f^1*`#&^9fZiXr{Lbz5Cg@ubD}&*UUb*Faq|Cr)IQ7t_3X5EN7k
zIkVK{wZbt=zlRywASz*dN3*w3K3fHUIp$72+^i@qHB?pogXNdm*<8e|!tXzSHr9Tp
zGBPm8`J3`3;Nx8x{um76J|Z%*@2!1YWu-6h*puy+-;1CvAMqNt*Jj!i%k3A7Dl&Kp
zyjcMO4H^(m4VpBo!d0{<-&pXIeZzy5hQ{S8`LtJG&D2?7BzvO
z$1l@!#;b6L8-XT=)h{nMSBw?2Cw}Nau#ljWv-0xFvBAd1Kf(P?xSygY=St^L_QRM9
zY)P6Q;+K$jk-K+M>CfKYxgA13xLshJc@MD&z7`m!G{IC(PQH8pzV_rJesew=QZQb-
z?**E-Rlnxi)vK(qR)J%C7A<|~?A2DhCtMpIaNFYqp^@;O8``5Y?XSJgzTI}7Q|4-G
z8p5Xd?OV6ow?9I+zYh$vmxKhGCr$$0csx=CP7{na_{7*Lb0eeme)mDRi2a~}96WeX
zGYfSChe*J4VH96MXMhx!k(pWl;lue;@#MsW(y9J9XNa2*Ce9#CUcGYk@+vdn#pDBu
z(l2OyY4J;Qb71DJE?(R>aR!hcIvDosI1nE;~*rR4JCM^=R2ULdN#j+G77@^GyJVQ*uo4D8a|l!q>1r2tT`ZYYP+uT7HomeXG+wXDDj{(k-<}Z0}+0
z#0NbgLj;NZ7D}|t(pjTGc4%Uul+16e>ca~P1n#M!#8TEi*UkwE`Lt^?=qs%qnFneB
z6H~Q0Bbae%{jVz2cSsFF8wIRDWqd1{*4dnqfoMJ^|2}~8sfqD&viLwXWHr(l$&^gGX
zSI~1Epa=>f)(*BSS;XmHmOK8Zx=2Jb3iBTikRs-dJWdrXSMW
zA?^c6;Ti9GQ9gPF=1Z35Zeig(6j3Bj@4vmmu^hWz3*a3Z(u>xbMI&q&;31txa8e)ke+FHVN=+IX*>k$9ls1^7UYOVjFE@GLE#x>&gL0!Fr?iI+6S}^o1
zg5fmA5O>Ctogi8ua*~pmU2o{;Xx=gK3JQ`-7xS32(AK6kefZ(+rDHK*)B4cDp)r;`
z@~y4yCRPSXXB%ANHd3(Lsf<%nkbjoE2XsvajL=|a7bhOGn)D+Kwa$N_8uRIq(dkChEQYeTPrbIVQ7
zr0iBs#5C;$XbI2vJDa9QeEnh)61&nbBV&0<2^6w`F2SW8oEq5R(+0CJy@Ww4uNvZp
zn>jj0CMIg4r{=^3m|fi5Bwa>7BSoi+>S<~1d3M-2OOxwMQ&WuxH~jFBUuptLuvU<@UOc{UL_3{
z1clD2_wD3xZKy?J=NzFk(f$32Y`@3HrzR&)n^#q!hXV&EU=3RvA3wi-7entWY*vIq
z2Dl7BZwy)1q#g2l%M^k#R%q}NS`*A^z4_q4Uk31m1#j<(8lerl8&x>glwe-wW)I{7
zkFsOJmc>2MbWMo1BpSd2T;X5f-s!sINyAj;?&juP+V?^uRgUrQ*OS0%a2S>w@O~0#
z2Ewkd&#bI`>h}{3hSu|Sd*Wa30wibu{xvl;BztBap7Gd>fXcs~iewAF
z|2u<&69qaRXE@EA>WnZb0=KGH+1Yal*@cBK&CSp2Q9ugL+J@`Kz0WfvsIIrViDG1A
zbf{wwy+gXDQmi!!`PQxMJU|Qhn|256_OQY62=SrA2d1#Brrz;e$C&~=BtAH+3%g>Z
zY=VP{nOTb-;jZ}G^Qt3W4X3~x91>Jl&T0X!U{jm$rxO75FIf1Y3blbg)Vk8Kv`I3@!r*aPTRV~Jyafm-SsLtTb6~zE
z7xE)$YcKEutw-6}?|l2+VJwg~ly^HE&HAs|uMtm^`Ue9Pv~*y6a&cLJ%K|gH6H$lb
z5C91eA3~9AM6k5A#jLDuuDOC3gvUxwchB23#VurpUvDfM8vRjN^RPR6|W@UXE
z9AvvtUBGi8=jgDr7_sbsgeL&(5|h_Bqn$%=BLYUDQOke*dS!9(StU&BAiy!0lW=6~
zqd@F7-?VlXS9ptU9RF(24W+Dl^+bVTIEKL2(wy)8uBFT-ok}g?WFy49R6QwGNv~LF
zz#GfK9ZpzCa~9GPOniWjIEX+{H^o>5gin$N)>|DQuKB%KGeBl_r`}5NAC+Le&}G(G
z%Daevkf=4awJ>StiAT^%f3Yn(S_uXrx*_O()e<{{PM{LO+FUqCBMO^r&|dgYWEW+u
z>h2+ov--KY$R;zNqj&E;Z^*^Vizs3}dzFULZSo9ZTAJ+Z@p%u6GtJFi1>A(gI~Uc8>+FFyNI!Np9viCfLu!Nz(vLWh%cT0RaDwk+BMZ!4q2%(TY@ntuS~&m
z+#z>R-F7`jY%)i(8ZnQE4YFY5JiX(8S@al9J~=(zIoEdPl&`-F^i%n$hJsscF=2F1
zAp_>6Tw7Z%Ubiy}n~Yq&n4bflLvQ-4YNN)TVs3va1%Zl03ezGvO+0LP~yPXbL8Loz?p|oGJhO%IY?fQSi|@@&LG;#
z)9IC!rt9X`Ld=l_5KzFgP5u2_iF}U(yUtF|Ib=+x5G)79`>BwP%Ju8Ku5XxspQZU%$>^&>frrPBZ$;C2i`uGmEZs
z4_01t=y5O9NKwAPBiu4SPwkc@@b}AA^S8vhhaZmCMtbP!f#zQL_b;??9+ekk(H8v2
zO7?Lctt@LgMOv3f=m`Va+p;n*^aU6~KSEE}8=$RaHSX|v4Wb(CMDUT}XdhK}`TXE$
zbCF&8!!QLHO3%Tc4SQ3;Hy?X4-OH_=-qWvJwdgr%NKplmFd@)7vxfH5foBkSPjV-B
zC#dW|fCuM92(8e`B5vzqiDpS^2`_9Xs;c14UkYFpGIQ)W8{V=enpe3^#-iUs!vi@b
zeOXJJY;URnyiB~31{fE)j+cXlV6TPp?e`{<8w8sNzunU}*qdHHm88O)h0YN34m
zO_L3RJbioqm*Q30tMyn%9OY@ZExat0eZp~ndd0Q%b-W$WALON7uL#_J#6(BX(uXM!
zUObjZwQ8w4b(f3|OKA~U{Q5PYm{{V#o(+cMhl!^qCa&7s+t}IF-agzFjaqGi;4P#eEN14d|fp;n5dxPZVMlC6MZ=0JFQ`G&q3OX3(M1%)ea
z`S|w_v#p2E*fs-HPd-~1y898h9CkJ!PpVK99-xu~g$9x2hkkt{QQrn&Tmj^ou>6Sz
z5M7d)mn>g$OBK-kTl)@NRj~@I+`s2!K>-uc3NWs}48LQF*v(&qm^%j`srkV4=@A@S
zkR>$(a2$z=-s)14aAoIZv&X&({@T2f*RKAQ0Tix&Gy7?J(q*!WCsI@`X%6Rz=aaX{>Ra;n-JFSX!EtQvUSZEeBQA2xt=t
z2|sq5V8@?(2S8)tRt;Dv)nlz;^u2a14?jdlYB|I3NVJ^tM865<;1R54TWz|MK7~#C
z0$KY>Ntw=GeE=p5q|=L3zuxwCIqLZIe2iQUODeY}Nn41sv$G#NMo9IQo1fp^)pcav
z!&$jY^*%z>Xf&-KE
zU8irDhg9c-OBD;s|57E^3w@TF(gL>1nCC5*{L`VWoQAr^jcaBN7B=$rUnM9
zy{6iwCSS)~T!Bs&7qK`_eIBG*C#_>6KD)>5csJb#=dyS2
zmI3!=wwB_`%e~RLbc~zdzsTAEM|o0eYBgvzPSr=N@M1z-TVbi?Z~>@mzJ@8Ief#!7
zc+RqFZMowr)kV+gb&;0o$j0g_*TGHD3n|SFxAwXH#pl4M!DtF7Xf$JJQzxgUzL>lV
z(K}cb@wG~hoo1GjexS>Ri!PB1#)uc{PmM}lY|fuQZ*0uc8I5^oFcNSTS|n^B24TUy
zkCV9f11r@&krKc8dPdY&Oz{Jrxz&@?y-yZ#3=FbNOnvq<5g&MTbTq$e-2s<$Foi&r
zB_Ee13oIxqf;^1*b82Lspe)NDU*MSpdjkgm#zSFqh3N;o2kM97--I^3jlF%2*Dn&O
zK5`fvUB@G*^4=fl`tr8&Lth^&-$gAi7z1Gl(m;5wEsX6>6D&0D;%#TEGrlulk<)O`
zOUn|8?^Snq_ao8nxU}W!>(}PhC?DD^EXlyJ;{!)@hcc>>)8U;6TdQa~!SMa>u$Q)2
ziUZm^XUT(~xlf%64-4~*JCI|-hdCWoS9qhKOF}gsaH`(=bUoRS_d8k~kRREH9{kwm
zB9tYfvx95^LeUS$39U3v2xd7FpFDYK+ffcj+>ak<5O9ghU7~>*TupLoN(WcTh{?lS
z{;Z^AHQA}^rn@_CVreL_ti^nKmz@z8?2J)xIm39O*=jMr0!qdC<$AK4#!c30ujMIt
zML_nL=;%C*i#uagM|zGH%qZAGWGmf~-|wpLov#qLP`SM@+PJ#vk*Uw$^29l(4g@6J
ztn)~1=xN9z51FlNWiudH&^b!t1`%{zA6lXcs9;cLU}aU4ID+Rxv6+y$$+`oLeu&`o
zy0SYP6a&~lXvl+MXab3IF3A
zsd>kSZmix4zC=_K|GAodAZ3y2!42#Yi$ZWg)|vR(mgW1F#lS(#YbiS>rmca^|G$Vs
zP3xfn?*MG}xC7=g{pQ@if3VNtkXB~Qm`DfNNt+o^J>wolK>3Ue--06@EUcYOaI@*f`Y|K{m25f49
zB>U;p-rH+1K;udP7!&}J(4(RJ;y^)!@9yc@rF%ViMMPEr6HtS>xw!7g-${zCSzzW3
zOuX=lfVDi_lF6jN{`z$_YQuCs+myqn6Phr`i3maCjR1m+10)~p>F3#*#9cU;h+tTa
zW{2nm_t7Lsi30%P
z73EOi*pUmt?M}9~A8zhmhXY;>3O(jWRGt#7?U3B}JK%J{))e{~xMwr7BTs(!9exl1
zcA)p?PcU=I=9;FaPm_|6=pAm{P-nCpLbXTMf)%H$vvYdEgOJk~ziGU~?<{Yi@iAjb
z##NLYLwk;nm4G8a3cHD;c9ADgx0))31fKzAT;!U;}JT*?cJE^WiB5GU~oAGUb{+2(ceyHNl#N1xC2L;|^*&YRTv`rcY7X
z7ej6xM5{vjD1z`CE%8m<*MMK5U
zP>I6!(e~GDo439g9<86Z15Khl8L_z`NQ_#`@06c*Uk93nEGuvuV3EWFryzI
zqzJ0Ptr=fiS{z~WQl4L0a=UqRZKiy7B7u92f`Ie;w{NphmoRXmkbwRLj<~lFV_+)(
zoO&v#hKrkfm#l}%Pz^#8QP%NvEIVHFo$tF?wVb~t7EXk{H54xUI4_RLI1hb5mqkLj
zXUbkn*?{-^i%Ay2#7ED1jQ2UbE_h23^>Fey4ZYen{4y|QUzFx=)jvpkQ5G)aQj~wd
zOI#^qeThcl{CR?eCXzgwFog{3e+oNOHM8%-Xz62HY+0C?FdT$ie{eC5NvR8o_@MU2
z>XJ;GeA}LyiyQYR1EZ^+VJ@5Vh
zKdDqgk`y8NL}^H+%$A}Ol9rM}NLI3nj3$aB4W-bqnv_vyN*t1u5s^`5nNdch{`a@@
zyT0eT{^xexu5*q|_0rO0xN`ZMg4eBHja;p*+Qrj+F&G}QbPQmK
zv{}G{Q*oqH-3DQEan+z9C9tp4UwV7(;@Uf&^-HufNijt;3Q|?F02-C@B9Y$ITyFfp
z%a_wZiz)Iz`wceg$)wA4m;dWLWjQ1sz9_GBv5oWbdLRdN@!W{f$_Xgd
z`?pl@C>FElDWBUEg0V3%b&ApUx(V5i1$$2U(jZjpzN2CTbEh(yGDS;Udnsq5tmF`V
zeVhT5~hp=jhc|B%6s#oe)xZJx1WHtqsY4pWC`+e%K!871#L)U$JUS5MCl
z8iuAPb5A~Gr?t_VynFW!K#`axq>**HRXWGX#N&GkPuFJ=Uy1Y^xKTbiqBE*Ag??=HWySw7$?VBcLFv}4QoHfT#X
zBXpT&fV!$j_wMnA&A?UEZfc(Ibb19*#yhzrQ2T{?MZ>Ki>xGP7sJ|BgoQ1-rZLjCI
zL0XxYBl^L<;m4R=uvA&T4LLJ4asO^6lA*1hTQ4NUS<1a+UGLGICFUu1su%f9iOI?9
z-rj2j@}&mznK*?K4OkA;7P^N7Jy&PvNiIh8Khy(rE^S)IIY-%VxJr71^Tmr7p&CG+
zFoQzzGmYRrQy4A`F9=H6@uOi|yjNN}
zQC05H;lsBRt|Ez|&VZtHb8vY6?j1noR@-j9rV7H8qZuPN4~P+sTisAc3;y!u6+F^W
z*C0&>`}emiHV=LO^x`ixQoO0UZCLZbL7|>p+})=D|*Pv
zTK;WWHw|GfDlB4eHUtFND@ddmi3*?rXyFi?ldKR{-!;*)MBDU?JBDJG#`!w*nevmpxJSA}FcZX%dFSFY4-y-bK%DMqw28OnJ%3`DB?)rd&B%hfLkm%YO
z=5peGyKYO^Ap?Pd|MqBb#9D-_+M}tYl+>=KmKG#sniIU^_|dD<5H&tb+1yV3`uOma
zzNaiZALh0ful(lPG*anihtYPSXG!2FAKUQS?Rn?tgX%vGPkgPEoN{1~>4QZ-d`${(
zT=6pLoRTJ*rx;lx})%ecGC_He!oI&x7Bjwv5%NJ|NTT;Sy7|PV%{P)T0B@Pv@&f
z`(&H+eQT2{YWOki9p%Bg^jUU}wc~^A>uS@SB?oJE_f-F>7oj^;qu1e2BQA#dT1odA
z7_Yd~>8Z=N{JQyiN_O4mT2xx^ew*;cv0D0PuhZts4YLO|jq`$;(WO
zQReF>b)-JGtW26?bh;$Cy8Xz(0)NMuGn0#6KluLOQjqV=A6g?6-knR_C3A5>QEt?a
zPW28`Cp*bJ-D5q)-hEp4%jw_ZUKdG)wyIP;9{nu4;_d?b=4;cMe^^eGHa&AnM_f4S
zINM)$BrmyAJ4R1QR(Ki0-Rgya#6!Na6vmK6W{`$Ah!q*=#Y
z?)E>w^n7lNC^NeJ*=q?mo$K9NHarR0bm`N@ESof?)tg6VZZ>GiIM6aSEYx@U?|ARV
zMWgmPY&iJZ(A{Zxd`ou*SMHAO@CuMBX;SymY3iDj`jt=njUw&Nk4TrbWqSzd#SE*
z1Fl%ukIR~L^r^IrQggwEcnkO2r581>PD)R4ioDxY6gxs=_0G2|PtQ+d?!3?lS6V!G!C$BSc14s)HOY}z=aH|7Z5pP#(0jDl1GsLOG>S1tXJ?_+jEf6O{Og979^7}}
z_<=#ro)^~xWgbkr*Rki@ukGDrcY|xmA_THhn-Lf|X2Sc<=5lzw#}AyYtyr3=Sfpuf
zi8Q?O%@k1#M{@NK%AM(+o|kfR?1snRN%#y?xV0251@E8B%_1#&Q{RMg$QO}RI)o0i
z)T`+-rN;}yk1Y-yJIL?TFQXM_YOpMD|B2+yKcrczicVx^YKE2|W5u_ze6FJJ2(#*^
zF-Nzgo(6SBmB?SO?oQ`ix_*6IiC0{Fe41QEOQRmq%Hb9>+}yGp@fEV7)E
z^d5s{G(0T0Zg?En{IaqfV(bU*T7(|YUgK@@;?~&qN}CKm>(d(ktiGjrVC
z@qrvB!%%Iv^^_?~5LnQ(+79wEN~XtgdlN|uHL}1J`Mp31gv0;f
z(W8^)2(K+A;TbQ$Y!qlQZemWpV19^NkFsANhmossWojr2~F=$a{ru1I~+YsDHxp?jA
z(@pqNXjv4jq{A&@M6j#lZ(#$&$Tm#F)z7a9&>76#QfeuCurxYh*Ec028h
zh(|!4BtsDxBl>zqMsT{-QGOC=UDOqG((DCxg$IanL8Ah;qvb;_9Zg|Y)?27pf~3uj
zG2n)6-HN1z5(&%_!1&0aLqTH71UM3e9EjHj9pCgppErvjg70LzjDS@{d9mBNg|EP)
zz7*6NqbdcIOfg997`hw~b~Qu~$&(4qY(w}7zEjnvGGG8PywwHE-03VyQq9iZ+zRNq`FEe?&|2@%*?~A}6jO?r
zpthmF?xya@+T|6ql&Gdc7j}|A=T#!cXH2u-CPLNCK2jOu|8kkmprmM2j^MX(q84a5
zG!|m>Mqdanvq>8Un0
zV~{G%m~nfY)?CnDmJB|k{nnz8C}cpa6Efv*16$K{WX4zVrv+iAeI9LmPi6fp=(foU
z-m{0&QP2l}inAvJha?=T-^g?XzMpx6uU@`XHGc;oKo$ZN<80l^vN8aJML=AKY+}59
zd^B3h&`~04`c1o6wUaXqsRb#yUiy2X)_De;guhaL$Z)yKzV?)usXI*R-udH0I*^#K
zxz%=8O}{;iP^x@RZ(1!#QP@0n1sP8AhMTLC-N))J$}#NNshfTLxR6L;69YjDgnjys
z@xU8ie>lw7ymavLMuUzPi$3Ho$o4m~Ba7K|b308+gSJP$%M*ui-|cfcbY?{l((^Jy
z+|A}7&kg&Q{(>8EnBv&5N2nOVfh!h%!Zt;X4|Dn~sJTK&gr2tlk^HEG2p524iT96=+2AR}%Ci=L7|`F(L3ZHLi+LYcTqRsM7e1UH35v=n6!NAV<{0
zPT9_!UGyQ_|717`yyW?H+(*BD{SvzS(t6s_8LQ@^5YkhkqJ_+-PsAaqtJ`De*SW2?
zVE`l({HhSCB4Pk=$2qtcp>)vnp$Z+%{{r>^
z#$(rh&pYDJaW){SO%OvZP0h~gbLrhUHwRVTk$3p^`Li-09v24?
z4DJKRl;11j>?K1-f)ql{V$4INgmHAsrqRUZr`euS)V^#1U!D~agg
zEB12}Q$Ynvpjr(_iIY=20(u&IB+iQR@(!M!m0o%flF%FtK#N>!sv?m$$ClmDW*AO!
zb4-I*A3kj5pHaYZy?So^T1}D=XA1{<4|ze+0ZB0B@Esh?;aj&F?7hjZUcAXJMzq&%
zA+AT7d~VStx(UBvHeS7coqp7(Ix8MqhTV){JG$r8zf;XEIG{NmV|MRmQ6D$ApxNRh
zX(&bs3JP@z*JURF9azqQ0kHk469j)Jxd}e=cn>c~(m;^*^yyRdo~!}Peh3qi)K7s`
zV=1n=5NP1Ah3kRvo0Y(ok?F*M1o^VjiL64`TI?E8=6(4C*BTh8kR7x3<71l{3NiLS
zw>mZ6PW@hcdI<&86O6!UK4_ou*p5V%LVw)woQuMF)~vIO?+)d^!7U0-1*SC2%TG+t
zAmT|-fN-mz>|me<8V`x6t7A9*G_;A~Q*uxm8yM97p*@&yATDKcbNoDaAE9y&n)G<;
zg>@q|7bBd^+dA{ZUqjAy(?W>`1$Jw%LD#>1vmP9bdbrdmwrro?Tnl{f+0MD`2bXnx
zKiR!4{zoZ9CSelOo)v9u9HXDSk`Q<Gk{fIfEAjM4~w0@ejCPS#R67FH*iZ;Q8Alpx{$_s*FyJvk(%5PM&1)hufT`8R_g0
zRy$(KUOjl>V+;&dQNYN9%lMikR#=0B}VZl>LtH-A1@7}9w|F1zXrhKqhbP+!e;2i3w&V}G&c
z_U+FwOYEE_+rx&dj{~VjF3RYHL=6(31)**|sgA7-|3GDw`bxt_V{2)kPzi;1t2X%s
z&?{gAg+7Xhc}NgTE7%QSlB_cnZ%OK_!DI!2%!Ay)vrJ7PrY%obhlAq4QQqYp=*~XV
zY1yZZo`O=q)>fuvan>kPWd8*5$#t&KB>jjamsv3P9*rMAwu2GsdYr2o@n>UvdYZc$
zi}z;yKLG&&z-yFql1HQbZBv^GDzlHpI
zyJVy8hlLaj06S_PIq{K%wD1(yuAMf0y60Jc^^M8`I)tc5hs?P0S0R}L&E+`U7heUGtA**StgPhb
z!t6ny$e#6F6kf_Qy7%wiUFGN3sZTdYh0)nzr}i?90@1O%I|(ppvMFyK8#v~0i$z71
zgG&McqVL!S&G?w@Zn2R#ZDDRfMMd`~)^e5{;Q&>%u4IAmGOVq)5%_;c$)yisM)~bt
za?&r1kZ@4!y{_p0GMbadVyt~iUdq(w8?=MdBM99>yS$`)p!|
zVkAO+q4Xp@YXj&!<*n#8(V+}4!u~XJUEZ%cmpgO!
z(sl74bu=}lrKHTX&H?^&|IF3}=B2ivxRQ-qWq2QJ-_pBBy4Tcy2mTa^u9TK`d3s#(
zo-HO6@$L38_Pnm$)@~YUwn7_9CoD>%Y(mEnmc%fSRH}boNnxRJjJ@a(56s$PMu@gz
zx3eXoMJ~<43vh7ICl&zl$jjHSi)LC7;YwN}A%hmi1Mpb9;|#@q^m|f;yo{-unp(}$
zZn40gEHa=!lSSjSg{!Bn8!(WX_0?KO%bLyAAn|1Az9MnX6}uIo!l3tF
zL0uj_p!lXkR6+J+ic7M74&(^E5sIo9Q7dW?1bBi{go{XQ|Ex57SRg?j2g}4lrF0w*
zZ$KU9HKT!JJnwy%(9kVgwg4DEtgXFN5hGW04Y-`SMtAr4k@@en7Zun!T
z0aB&DLetUs&V_C0HM)~d8jFbSc&Y766hs~&bLMjVZb27ugqYmkefoSwYM8b*hGWag
zNgdwc>sM}CUC$z%wwQRj&h&y1OI`S#hMh~=w4?1Lr5@#lNsi=G=@zAF6*w*7EpY@n
zO`qO#cnLAcEDk;sT%zDABPMSe8eYt14bP-PT=VB$rRt+)q{%-@lBQrf;Vr$ib&|Yo41+F2
zZ~HX*VwL5RqzB?0vaw*Y1KW(o{4{y6B)s<6dN64X32aDR^8!9UFBDL$#fUl(4I_PY
zQoqODMv;S%oILK|*77pR0aT&ixm9smh?2A9-@bZvni4N@+@u1;RrghGfq>A=mX^NY
z`681OJHG=S;39fi3Qs--fQEkr<_-3m`gzcgGFfbkz>#~=VxY8co150{no7>
z(wof8wm{LMOJ&1XIT-vwlT)N`zOKx&_^?HGS>_V}28Jp
zdxi*&8dd%HaT`fcDY(jZh*8ZGA=lkxzVQFq6D`u3c<7vVfCe;9U;@%Tdt#3$IC+wP
z9dIYaQmxm<$EpeCSLfv~95KR6PUvxJvR=}szkg3#kzApxsaaN0@!Mz3kKp1Llp@xL
z?*Ny4X>G%bBT(|!u9ZsQIpH*=ZGbL>3m2MWVjMAn#l}GhTF#$;kam!ojq`bkni`zk
z8?roON)&Tk(BEQ|=Muqzj=fC6YApJ4My2686gn5uTwsZ7_gQ34;
zM&@xYDXa6)SWN_pqeD9r0qit;c23e#s%;9#l`He_%y&>2#o@{2!DdEvz$I_d2Z1a}
z!=%s9?*yBle>CCkf{J3Xep?QS59P_oXrwdu6{rXYGJYiu6mx>W2jkcQE
zTuodnTrQwnOSX@aiLqyCvxk7*K}ZLbZNq$QX&D9x03Z$AMW5X5s6}*`|Jm6!ibeEo
z8DjxBsJ#>K@7TVbABm26CY{sVw1FapkNdfVXoqNEJ1F0L3Kgs>5=WLI#=ANXJRid@
zTNx$FvN;qk^``~Y$Bs5>@;d=WpZfn>(t{~Ur$9%{pZQyr7)
zMglkljGtp}-g*3|T2Wl(8~0hWN^afI5i){oiL_5=M>i0wCeODY!rSi^s}zjjJi(B9tRzjD&7?~wO*>Sqf?~jyEOk=k3%-;*IK*v`TD)f
z{Nu&N#gq+ArMa9C#m(VGrFO5tTIBENCrpq#
zvvP>J9Q7N0K$1?+&Pv;m!?3$6ii?@Aa4IWHCcG%xS_CMV*MtCl95@I)-)V2nE_38!
z%N{d-j#Pw(6?yaRK?YUjG!!B!F@F5|MbYUYk|%M4NQ?B5hCjy6LA=51xs_r%eURCM
z`}g-GB+OJe45Q9HkCK41?Q~wmsOd@9UwvT*Q%3&9;JYzt=2Kr`C&z=AnyJ>2Po%r&
z3GeOLDR@W#t{|C0h1lCn+#F16S9%7g%|u0?1jEWFvJ8TM?khZx^cWy~BV87)}K$=Dlc9Z}F8haWqbd
zV?xBptOCFWSIft;A6-R#w`X!DF*?+_^d~aiyGsC_($9+k7(7q^$!~nYMYFjRiv-%`LbRDL)029iap}
zHgwPz)7g(_BnTI+yqn*QOo^7iUcIbI?tH=h>gctpjQ-K9&ZIPDD?k2xoN=63B
z%&&m+E1>#pV&Hi;dP-Xj{Mtvx3h5J61xV!BHE%_7QgNbFoOP?ENxCLd@X_{^6i)()
zD38+ibIF7#h4eD#jI$Pv-PFcd#|6ND*JADeP{1a#L(O;lmUb$g`~+{;lP4P~2Vic-
zJNV)!ugpb17FvRy+MjN#R}5%5RSpFlM@noN{+%q{{l$}y#m3&H&crn)BK2)l#u=Sc
zb0V(H{odF0%Ep3Q!{&-GnZ0Xh2>8)7_w4LBvu6hZDhU=CB_&4e^b$wfDwux>R#A*%
z!Ub`E=<{#iGQ`gk8+#JJTfj|_8XjY*^PKLxc323LSj$+(hK4PabuDiE0k@0mN43#6
zKESTblc5qctTjxZ31rd82~62*@6{iNNNwI2m1!s!Iamd^_>}O}O0#mjNn9aZP0ef;
zdkqEaHamklrL}+*oH9vzb1zfQcibos&lfzS3vcZ^i#q{iui8l=qkBvL8#Cw3X@@OJ
z4P_BiVD#wOlWt@^S_^Ky@uco~PyfrvX1LRU3A5t-=gfJ^z=66wfb5_%z?ho>l=Mz?
zoK&tkr`)gKxlO;R}~i7bQ3==i~(9EW1UP>g!@_N^OOY`q`EXL3@K
zMkF{rY8&R*)&adz3Xz`)gDRCzZ~Tmac!)Yia+NF=P;KOr>J=IaGDBmT+7}xg-N{zr
zR4ft2k?F-tH?ba3Juln1@gik=a>cRi>|aWww!HU*z@WUPRtMweUI(Utb!E5D^~bS2
ziVIFPjpKvmvaBDb1*(3zFFQqL7&3TUcdLhYZ^g33F$#hU@-DdSlbKvrPg!1ngAGmL
zboKgmAD(oQj#%`i2+AXzm2?y7#YPrs_V;CXUE@j;;Lw&atm-++)(u2rCTPz(0c^)x
zxm%7laN4LYe@Wv@xIhdZ%w0c)TRji#?<_6cykLK*)_-%YI7t(tm?h6Plj-p?c+>Bn
zAm1Yq$N{IbuQ|#E%*(Fz;_)!J9$g}(EmLV`xQ(h90f^9Y?{lq+V%<&+PViM64JN8}
zo>O#z_{CoIbCM2s{Y}P?svMedjJNC7ea8xGltJvnJD+uz!RE
zC4vaT%0b4d`x#{x1--}&HLrq!va{-=YL86qUlmmxE-n5h2+be~?X_P*kDq7LlGIc_
z!CLXTO4c0djwNf?w_K9ip$P(EE-gO&5YJ9ZN{ZScq`Mob$4i&o_U|peH%viEb`to}
z`-t$bU6qP_It8AXg#{77i>REqQ=X?xdd_cQ!3(J&G$O1xUcXO8*TC~>&#dIGKuZJ0
znZQLcNlz));vroC?Pkd`O3ee?zAUquJI2pL(DT90nDqvnC+eGcy2+tA~Ay`
zeqfPmIeyP{{6dO%L8hFMxxTxKOXRZ9*Q*s|u2M&l)3=KTl(8t{4CYf-)|q|Aj|`yH
z>zM077qG_jV)+Nz>g~QKT9TLzCol!{_rk$X;s~5^Q?5iS$xch36>$-K
z0RlZyM1e+gneP^5W@nSdJ@?{B#%aR(6PiGy!9~KmM%^f
zyMup%;2^vkIyDJHMsK1Ra!xhm^&@%4WBB+}iLlxj{dabW-rS3czd8;%L0
zlnQ=gg-jaIO*l^-LIb`y+CIR{u^HAHq!o`R?`?#x?vW`zLY7soI}!i_Q$liHJ&Ryw
z(wEs+z4hHS(k#-dXlC|GrcOdM*tdWG@;Ah)?Otm@sic~Qy8*$*apDwdRLsnZxd`gq
z5j9+!mv0s)3=!#R!TChj#YhgczZ`i47%&L0`$-*gsK~1n*))_P?xa|zqN9tYC&WkV
zkD^d6sK?eWWt)~Ox(br1R^!q`)>*B!$2Q$pp5)lHd!e2ZwGB-ttbySjQeX_Do9Z{=sCz_5M4VC5{Fe
zA@1&5n!By3LYWOCE^+_P9kITte0}D;y0{2Kmo)~gR~p`O=+n)02>Dpn9F~CGZ~R6#
zDMn}aarCHL*5B@ZoP|o}@_k)8jH&-Q|CVFi5p_FmH#C-BNn1l>TY3hBuj_`DYuA#U
zjlNq=>N4;XFK4bT%c|siQk9%T)$Zm!st7XZcIF_5J^tSsE
zH?D-63r}(HJjFD4<#ZM8AIrxWJ4GMEl#T>&yiU)DcXKU0rqO&TwsQ;z;nao^aXa}rn&US(3U%|e0{>^caW`VwE1Y>ZTiEi3ky
z9{_DK%;pYe(Z~L3eN$6<>a|dcEq_+@n_~u>sSy|i1L*h{R7__`qsmC9a`V>T`zPWE
zh|`T6WhBI@1LM|Km82v!hwokh_Qxus$?dt}qU!x#kGl;SZ>?ygjEJ2mKl2;~4RY=o
zzqEpCGfwh5HxARO`Xx4|d(Tx-9yDkLj`qYvsUC-E6$q!slsj*pnzhxvGa+}TOVYfuht
z^CL-V4DM%V_pkymrCQ^b#Tm%8Mg-Y<41GaCf=iqzSw1{FUkJP6Ki3SAnFVsejI+b4
zTG#u_-5dWSi`n2}6XYhnZ1VjRSF|@fdx@=Wq-3bg=s6A!^8iX0FP1(5$VVaW=Hk-z
z$QRpHlWaBjjSoCQQAwW?`o^j#m7GZMGJOLBM$qq0N#Q#h=<6eoJ3yPlr{Mx8o^0-1
zEK<4et(I91e@d_uz^T{p>BqGWzw!yU5EK)z={%j($L1FBMdTk@`;doj
zsb({MI^QJ_PQ=ClewE@HO$Etaab@0M8<<%yZ2CR+=0?@zUZS*yogAukK~Iqu8hM&c
zM=<%TuB1zo@j_l62QMN(P*>;_b`mfT1L8L>AFT98TsvZ`P02v7K3+HD_V54Ay-gq_
z!P`X&GfYnJ}5GB!o33+}fWT)sIele?v{SyLP5?tR<)+;6#(u1V)ymQP$Pg#I
zh;##9Vcz{AHYXTAKWfw(ieqjEbYp<$`g`AV7p3g#Ypc|qvX;C?7V1E|iNQF-+3noc
zotIvw&BP-K6>EzA4owH#DWXLV;m#RPmA|@Z5~OfD%Sj8l6N*4Wx)e5Vt5^tDt|7JI`-#_3HrRk(eE}qtN$6#cAM34f|62K|>!UB039@Jp7)VQT&32+1f
zJV5?efz#0?;nPfKHB&%_x
zJ$HOI)IL)31%n!s-BT>psJw6x@kO{NF%q8(YSs7Y1_MDqBUd(4r(WQ7d}xHPA6Z?M
zY7POpxxJN`+;Akmyh~--zv0xmP|Q5GN)Y#wMYZW54kW
z%{@43nN-#Y_n3MjA}s98*|S%#Tp`k^IpBf-!_1q7_2}hG1twO_n875z?dyIz3xz+4
zv8;{r)UEUcD^`30h98KF>_Sr7;qlW98gv7Cfh3{i>>T=%-+DQc!dl2JE*{${&V9MT
zbzg8VW~UW!ZIg8pyt2voSy1VlW++a{aT$O~V~9s$^)m2m>g36b__pX1(c`E|scC7w
z0}-Qq={MTXi!IGzK51vaHH0iua>R%L`b_io!rcX%>0#eT)JF4pS*+-h)z_Yq2d>uE
z)@bgGvcCwkb`33m+7<967<;d0DSlLu9on7pjrZIP;OLu<`I3I%$x;tJZD>)_%Z|)Ny47eYv
zDv~0xw~x@!P(p_Yu!CewGDb`uSjzF|qW~+btFWwGKRi9sw_wQ
z{<{~AP?rw(rs6u!q;t}R!6qy7`hOxprjN46#lVlPz+VMFvr-0bjq=2tgz&N0k|
zauWUpoD!!bd;^?yUGE;D(!plYXb6$S(aZABF)?B?dOPNW&lNytqO4+os!=ir$mh!8
zvR!2FW3YsGYkq#UoA)Q3SRq^tA(l|y95}$+_qNK*rt^LxdGva2Ps5d)pE`WfNp{QLr~sngOvHZh$|&3V@HhW>3popq@Dtw&>&(p
zx43bMqNya`s4fHm7G4wJRI+Q=ldjs4>3eiNkk(4M`{J}b3(0wZIEn)Scd<nPtaFRwWvZ?ipx`#zD1sm96*>M`q&@eQ>
z5#iy%-MhIqh%=`wENn|Ta-~ze-{2+7E98?XDSP$GKcpONR{YAJU<-c$wZr`xDd=S^
zvwmHb0mF38oDB4?GXEM9!^=sn47z4PJ)3NFi8ybP9
z>RqlIGQ@<&dVtYVrR0jCHvWiR>OR?OC;-|l#qpck3J}6&=|v;oWHC6W^u|KfgIuKM
zers}m3y9_G=I6_wU%axxX370y;k}A$C+X-ZA78KXgp+v5!iD!Q+s)8LSrVP&Tclux
z_TlY>l-C`MAYZm<(f0xpBfwZ14sUr{{a2SpYAces&YV9#-$sKsMy5j?HC7CK1r>ofy}dr+`u7k_
z$U}z>!*yHw+G+F;X%U4m($cd)g*|%o$cpcD2~#J_LQzhx>hj;)9rTQig`9&a1qpX)
zC{cVP)wTL1VX_k{S@z!4+q+^#jiBxI^}%9I7ZSo^L=2YOw%%j)zS4SXANm$X{rbdRVQb{YGr1B^F8<(+0mc2~{F{NJT!05qVF%zeCeH-xRgHEMOeBsoI8ozN}P-?T<
zZdOz@_VuxLZtfrWW@nbf53T80s|&(yGIh?rwtTAfd(ZVJU5~a+n-$<5vU=+|9nGT=
zVoR=1DSf!EJu~U+vuz8dKitple?{llB6Y{yEj#)f>kb~=`JUb*HRG^H}QFz|=v
zB)Z&)Jk?GGf9ah49ol?2#bU_tmjQ~MdRNi(P<^ZfcIJk#tv7qNS5x-Dy`NTh{f6}O
zKYeAVeYccT;e9fEFHAn3zQ|FcouaoLGK
z2F;*J#WGDvr{5VSo6R>-zP^nFxCosV^Biqg$*n{}B@W`4y#`2|cW4$ldJZUO_I#
zpKl<1*eMY)<$V-vqD5Bts?Q?_E^ucb|GpV2RqFl^atzx
ze;=VWJ?*7`;K`moIi3Iei%yA1w~kb`St_yfzdve!z$v-WQhNXITMK_HTFu0|K9(nY
z8UEKV>Uruam@~joq^TnLKOaH==Ig^)*avqUof7iz7goACIZVwa=4Ykp1+oA6DU^@k
z^3+lY==ne2acQEr!T;}hcRv~UeMC4v+kZWm@(4Al7*T$E)W6>-afC`_x<>fyW2$-o
z^LdNBHRw+H=6!?74f)Rx(yTs_!98NNX8-+Fr)(|rimsRu{qobA{x&f!*KHK6dWjEC
zxa6reXjjLZM?Yh?t_um04*9X+e?8zUczo!iCoH@uj*1f5uk#&5mqxcyPWn3Q2D=A$6~>a9V4EpI%;uDQqRlJDF%+3
zsgm{Hcuo85`EC(=j<}!peZ_1Qlra>9)9sb6I}VLqd3H98IseX`aLH;^c$~5<$9^o`
zRzmqk09U##^k_CQ3RWDsxilA~t1O=SUmsV`R
zAJTag(^r4DwQ=;uCnOyGlTS>M;5loS&^b;&7_nOID*)a~CpTp@Dt-6Q;B
z#r|=EL>b`a>|?YWXNq|0Mn(mP4q1MPITMl8(Y4U#-^F&*RAOec$sL3$aOkgIeMR3L
znT|@HJC9?Kk5Z@0a9KG^3k$K?$IrRDhHeT!<$2n-ruO3kBN};pd$DDft(Wyj9BNHh
zu2u^!%33e(al|oEQ8yde#(Jz33=9zBJL|8b0IlB&9aW*A-mOE+W6Lcr>TrvZxM0BA
zo}FE_wZzN%umi@)15AZsYE!1k!YQ_YIeGrOS5}1DRP7n$G&kRst`u_;%0{L4N8#P?
z-MtI-q$Wi|O~H}}(?s$!6!3&M|NOe-=f%`e)(909gl)Vn6bCH>LuHup%hW4BKf);c
zNQonYLT)WD6HdKE=P}7GgPxv7aBFF1PL9HMOMY#+uWyXe5J+IL&Kb{IR8Sq_(`L1l
zd6OM&?Gbj8=|ooa
zDpMKaE{#vuP`B!L42+5z2J8Hma)r#wUyPp$=DpIR6q6JHj4Z{m{1rq%oc-Ke-n
zzXimrcmeziV>}?=z_J{M_kXVfUa(V8F=+r%!(_lKOzulylBef0S|3bqOHKYgbKAQY
zfAoy8KiaZQT&GL@-`q5&~iD)
zEs{VFsh)UnfazzcIoaPa>Qjmx)`KT8pjAp%_VInq#AU{(q;1hc{U9itkr1HT{`|sd!m3DBFWgi0)j`|2`CV5(QCi}I*F}uWo&Z9
zQb>-#TZq35V2yd*Ar6)W%JANOG=lhTZT`&Q@#cvl&@HW(9WXYo#0&k@KaDjXiEmBM
z!$(G2i=rzs9XB`Qf1~up_G4W20^9~A07e2nsWbO|@%K&?tO&PzMO~E%pNDr)EZ7UF
zjsD)H0A(Dufe^Tk!=27glId#x>{fYs=;TT7PNgoQ)mXs#VPyz28#v4Pkl#Pl4wI7t
zKA9wE2{;Uc%|atZiAgides|=r8)!d6R9pIKZ{zmjXmRmL__WOJMN;!_NkxqED$YsG
zOYA-A{iKmJVtXbhd~h0Al28?Ui&05n#R!3(EH>=RtP^VItBh8XBlQ*P>EN?9LZqe
z+?dcnSV~a6(x+yUOk0RQw+sS>!BzsWdG6XU`=FPK^3yX}jWkMTxQ=Ots44d@Yi?Tw6
zeiFAYy}TA)<_#wuSfOAhXk-lP6gjpCjyZJVNSsP=wmGS~a`
ziKp{nA4A1;6AwWKHMEDsl#tz(1JGmwrk0Q;va`Y~r`$=p!+w5qj6CtpcVH=N3C#aOsa
z?QOFHEAi$rZ1dKfe_N%*6`h8f2W=O75>X{l5+)tOWb{7UEg>e^89aoya^*k8#7v1QgVpb^@NOCU-!?Bfj9i{Cq{qt!EM`f4p>
zV;7>&$i!Wv7PpZBtvhf
zP3#&Y;vU(Z@HHjQqR;w)wv55acmv-2yrChZs##TO{)V>e3t~VL0`6?&L9&
zQ_*Cjw}N$Ztiedh@g|PHbhw-M8#Qv@n7IlCSjlk16wJ
zPSMa<+U>~7H+Wqw#3X{7vWLA2adig`=1_+jL;0cPYcWGX0V)w9?$_6M!{oQ8uJTFZ
zc5^`Hi6Y;v+NDtYlumeoMdidOsSr-SjI68^`T3oP5#&PMQOFC`bR7pLjuV0ULB0Ex
ze|y;#+ebXjla-9ReNIvUH*02rW}yHF!mN_+CN6ep)&e~xWW(<6m2DyL9oIJ@I5hwx
z)GLQNIjHfLO+0>JPt6s?yC@hIs$K!V;L0Lf0UI8CefL2oZ2-N95c>7&m-^ta#&9*Y
zbl_x)bh7xMbX>7rpYV>j$=NOp@IuGRhm_j|E(p7Hh7z316sZU|=NNF~GZ-_#sa4$>
z1L~=4l$=}VT{~P>@&!c{U$b)JVxTQh(_
z(2#|kA`;&he(d|w01agBIm3xeHhzz~7oOtcg$pEcE6YxT7^UqZ6-NjvLFfyUODzZ;
z_@y;S?oe7K3t$saAkvVaWfA}PV3lqfi?v$)u059(d!!M-b#%U>O_qX{kXC>6=)@}v
zt=XasG!SxLZW|8h+t8hW7#9ER*=CMOzQg!|p9GY7+P+Tv*J6@mTwBlrkYxJI_^sPRWDpAyD=(n8
z$nO9kvp=8SJ(x5D{3|KG-$@G%*@@p4iNL8m2$~$D6WMi
zTFwSmGw29FV|x#qCge}!o}lOsEg4NBI!!5PJI;FQ7y3%Vg^pE4Q&jN`qf0j@)EobM
zy0*>S9{JbK?HkW-yf!VcqxO;?_125pQqebwT^dR=J-p!)OpZ~!0ZJZiyqbA7&4>0P{Fp>c1udP(D{4oCR}Fu
z!jQ9GvoBQs3vj%yjc`2HX=%BkTm8AVKBLDn(^{5kUZ8@41ZMc~w~tO|6Aw=3%#)8i
zu`F;?Oi+ZqR@}1kD}GjkXSt7ZJo8jP&Lw-NZ}$3SuXbEE7wt6^?Us
zgCf*K^4pm_d1661D-c`F!sW}K!cxOTgbX+n)UM}H^F#SRPot!5SMj#(8*T17q9OXb
zd;j8tUysLZqk`w1Vj-Y~;Jr~gLK(Nm>>GIl^IJYcT8)jNl4!$dxg7mg{QZAhCb7dR
zef~VAwG@7;`8?=L=RIA2CAAk8PuTS@8YHrsA&R=H+P=QC^!wo5WuIMg14nM|bLP#z
zPw0p`zoE@v_KxQW)v{j+^CsWznBrIYZ-;I-B~fAONa>K0;*$RXj&vsK`wXqPvfe`W
zb%)f;UAcJ;b?4eVT7NwKGUu`Fymf`LMo+VTtt%H_7O$1EsN5p{)ZuE`_`VOOxpmKY
zdSbRj`DS&epS~+Uyl;E&g%-X%nBYE|aFK_E;d>-_2^XE6i
zhW%MjiI+@8R^_hZWnRx!Z4x(&Y_FtU8Vt6f6WA#!RW&~~>ce8M?7@%YO}D+ht3NBd
z@MV-&zN`M;oLt{iUmeRA77ThN*2nRTPKm2ceB9lvx<>~V#gE_ic1XvuwBKJ_=a&^d
ziY|~mw8E+tb=YkD#RaV-GQ+
z;9jtx>=}MTVURE?9^%7M3=nVyH1z&bxW!!balGL(vzMbqLOaFgD$gA=^)B|t*}BF<
zhIA1rGwzqMizzWBUexbFZ9ibmgZ0W5o@9Cw+z96pUywgZr|Jl@u&Q`VXa2DKb|D#+
zzWqxppc>@_x?KzwaHfGP_g5^W*1Ap{DJ_Avwzh#m@5R;DF`UrcZ^r?mFiAq;Ve@@-
zP-JOA)jR;@#>o>W_;4?+Y$MZ=(%*Kc
zhY=-SWM_e2@KJ=|Ud}|$K`3e;O7iK;!Y$I6Im9APsjR$tbH}!A*UQUe`(9L0QQ@3L
zk>umkz|mZkG|XI1ELh0Vs5kt?!K`H8`j-PKsiQ^-P{agP`R(w1RY;RF9g{FZAQPEA
zTl-lsF$Ghn=2Gm-ba~{DEIy*Ja2@JJe#Ab9)>A(+#DE-&14w9jVtii8a$OxAr!Q|O
zOLwGIKXEPyGI@qDwX7@?=9#vD4Tl)YRr1dsd)+^D)QWbo5LJ`S?Fx$hWX2^s-yPY$
zGv@r}BLkPcYO1#Q+}X54yS+x+cTG-xB{~2ieRxL&)~(k-ED#ALq@+xjyMm{c%GE3;zp#)F3PU&MyY9oq
z^hZKn`_+d~wp&|ENiC+-;~M?e($Yrs_E5V-Jp^i~KFCMtB2Rz8`Aio8)oC1-za{~Qj
zM#sku
z_*vfEE-95qHEtkvMrQ#p3cF6(n47kB_}BLjn{lP`R1~bx^^@ER-%A8cVUvGw#lqw5
z7XP&BP{P`~t+|Qs`YVSiNKPV#4n
z_yFx8WEQ5JM@}?*#80zfE$<$nt!cm~eLN#0;P=l_hKF9sRT)kIP=qccnfQpF1g@}=rY<2*Y3
zmhL~I^{9f;1R0k)bIH(P-QjV6`O^~S&0qfY{-;N7bK5_e&ky@{@QZ;==A$|TB3Ke7
zmrXUjz6u`sA9=0EjDUjNR%waTqkg|Uu4NYxuKD1^qfK-T2tH}wtv+)fzD$<=%7{IVnJLq!o345-d*d4UVPnUx
zr~aY;LU0+@A#lk$YBpaePC?KO4GcnC;EMfpj^yoU61r#t^
zc9)&IeRyy@g%fFY-;_#FjJN33krm&6>pe3=nDB*jcs9#t+m0RMS2xJTjb8H>T@m1O=cW=3W`?|B+{AVKjx^sZ3Tu5es8_d
z*Y_uCD_pu}T~UK#kQpG!0e`cy^7yM$liwv&z9Cip`hy29MFlw{mY;Vdj|T_h7s`b>eCU>V3>{>qw0N~ge^;3ITIqF&2ZK)kRfXoKU1>)w0MnG?cCHG#)>8New0^01)E^>IT(q%_srfmLN!>3_Zc>9Yxyf^Jk-es26G2ZkMRaP
zBkqQC2(*CYLEqPS*;&f{diO4^_j_5+fyG}kkF09GGZf_S$B%>X$q_uh}y~qMoVktl$8U4OIy~49%gZXonSB?J$W)gT|HcNo1j+dwbbZ5KEknMA5~V8kEijuQ2ZewJm?7mIn%_=2^^X9
zR4F+<0|3fe6vHW#5Ffu=WOLV0qq2S$S*|c+e)bzMMiGA?M
zY#LBt%R!bSO_B_)FdCA3Uac*9*^aR+1&D?y*~L
z8!bT8J^$|=y`;qDw;!UF89%;_OT;fhRK}J;Y7yGt&(>i_mwsG?Oi^3$9i6!b6V?VP
zMy)}{Q=`Fjug?VRP?2*%85ml#I#CLak
z2EJxNXEN|7T{FFR&CGWzq-1R=5&^7*Zkw;Lky!(LbWkGj3d!DIcey%5G?vAx`%%1wS#82^CL4`3Kc;j1k?Hgcqae
z1`BRRDgR~RY9DL5K0BuDC>WLFaQ
z*8=kH*deRX<}8v&Fib8=5IBc$K3&Gs@&EC39&kPPZTnA4OC3G&0&!=Hye5{#oOO5X=t`nu_T`}k2%;=rG68hQ8LJs+Mf2l6sg
zN`HR`#Xs2Nwe}cC9*))?`)?g7wM&$Y-Dp(FFkde*SfJCTzRvK8nA?wtTFwYlrlYOn9;~`IcIA*Slk141pp7z
zpx&+-9M2SG?p(YZyl{4bwrJt1A=sfZNg4@Ej8KjV2T29fE?~&bWg~hT#_7Zho@^8v
zYud`wFhFpg!M2Cg>aG03N|YEhdkiW%ZSO$7oUx49O@$vkb>YNlFAy_uCFX=y_PLg=xtEN6Lc#<&1wklu_Fa|XkXpC
zAv;En7(qYX$UuTS6Sd}n$H>99iGr0~?|C%wEC5@Bs_{I530w$^sdh?wFsZ^`F4o`7
zo$npU2AYS%Me>_$Pk}{h4O{x|jwbRh>y^my0o8VU7P-!GbDl^rP01eB(LOh~buTO|WXQ+|qvCKoX7x1l
z-xhZ0H`FYv7y)6d0^2=&mFi@Bc7i!w_MOk@kKiU9h88=7p{jCg@ay3Cb#CtNBbm8z
z7r*^&h&0EJ-M|D25q+SE`npuUH{OgJIaUl9$brCVPqF`f882XKWXa`5e;9Iu>!X=?
z_V{s^q0b?Q!M)_?&O#KVh9Y85ghh!$l-X|B23BAOBga6vLBEe(AGBB=dw3Xo;0fT}
zR1NE}BhiZ1nJ{7Av}r(C3mTGmoE&qYQ-mKRpHU1UxvwDxMSm^CU>)^T|GNwboQ+ov
zZe%`fkt8|^T&Lc!$~{3Ve4Cca&P5fkZTyHJa!B(9_`p-sGTF2
z6vH^6jCW8jgIHYG#(O+6cJ*zR0l|k((r1hi@s*!U8wOSO%EymO{1PSVZr=CsrR`p>oAbe-OE!R(?usBckiF7xeqmSM~I9<2y7wuSKoH?`A
z%vN~De-}=^XZAyrV%ZC{8ff>fc!I<8FvIyDvq)Fj7_HY~I$1W@O$N<+8Y;%Z$dVy(^0r115G0rt9U528o@fy#q`*hBms%UURiK`1>u*vz6XNZ7>p-Wg!c
zScTDRXzWmb+;>JX5JKG~EO4o<9lYO$F-1}*e2SANm*BVJI8%53xbfq8rtLH*BsZVE
z6JC9KJlfA5-?#G^KOf_KG7JVP#yo7n^^I%)P8ivp^NJpje06#GE5OZ>pa%d|>yEUF
z6;m1D@XCswcHr@>S$)H%L~HY9d}%pIdF&ew`#_g_IxX#lH*qMH8WHwK-P2PFwojpg
zo9$&MI@&lB>d(9~sumsw5s>29h%55fm+Cdk9wFaMw&zte{QN2SsOjnH6?%%?|7f4{
zjs6euuIPd=+z(h9U9+rWj%!uKEsUc!vL6X?NcFvG_RsJhJ2~U9s-mK>N~jL0vtGah
zOMB(Lcm5zaQ{Zz@$A@X){eb|5C&%2$^}pbpn5d|9z5|fU!-K}(=->fElI}-3z3MAJ
z*uWtD^l7j#+1-!14k*=QSaS?W=;;ZECztuTo`+yo&0Gke4*okX_4xtcMfEc&FrsTk
zn~Gqt;NqjEs3_7sy~BKJpMQvLi*9H6?9$HG|5ny??Vl}Z!#N~m)dM&-kI7HI-a!Q`
z^ppbz6rPPObiK9)vN2491
zjS`yDh=;W35_X-q3s#<*sm=p?DRb%77fOKfW5#Gq){|Sq7@1p$9xd_U!ATYt0?j=t
zDhj7~znXr=zK2*?#e?B0|20t-Y#@#XQ4;WMHQI5otmsuR?iormifu+~prl+vkUc|N
zrY?sywlPjd0>tYSjBIa}*vHpWv|@}{!VH7PuI-THNYm&?)r>4*I*>l>cDV<9q9A6H
zst!2_>umH0YjNQ-sQ60BhVo$UIj3lbHT!MwYQWf)xyz
zP}q3jGQNfUg$&Q|&kC&{8F-Kj3Ux={B~xe2n4I(H_uTnleZryTzyN56bS_h}Z`!%f
znDG#qqnjBG$0n{hnGsWB<-8dXDT9H&xY5w%?Cs4J#dGdVI!w+4AH$%4*tMEN$BdX+
ziq_s$HIl_{BhyKNz`eqzhlJwV9~Aq|oL0VTkq!qhM8T-(8vQI^gjtE4tm(LMiG^+j
zH|!gq<8DmdDXyQ%YK34<6bqgw1hw`4c!XRL9$p0X3OL;@LPc2_CH%IlxPSeoJOffi
z2y$b+luI)6mxjsk!~id4Ur`+4e|5}5c`~K7_dnuM9tIhy$?5+lD{aR#JM)jKUFw;p
z?36>o7Wgt80IJ;Wd-t+5^c|KYGTkjv*vr+<46EsF`;;?E@6C~I$O+4s)*<6HSimZR
zDVY&4s+fQmvJ(W@ulUbq&2}fs%=qaaxd-;WHRU6j(us6-MqZWuF;%2zCAF8EVxdlEx3Hb%4C8|sPx+EP-YECZpQ1NHCb{uADbcRuh9w#||F
z;}7vN#)EMNY)}oofVHCTA|h9Y9KVD{nKu{dirj{>jw)&O`t?&=U1{x9u5p*h?^FP|
z`4VgS$LpAZSy+VKaK366`}V|`6z`4vgkbAKvSU~p7If~NtcK_bb}(LD_u%Q%AJ_Ya^=N!5T!&WBc6v^0FM#aX4SJp$w
zTI%6(occ62cGA-(d^(dZTv*7&jq-zB|NB$fGAxp85)0f6Fv}J$DHSCG-**)Bd|h6s
z!{WsQ>P(01$Vg9z=T$}LLfCLA``odBfw1|FSNXxgJTB`P4mm6c^c(b1Lx&7;#;TV8
zT)423Ud+Ma371~5LxW0(o6|s}0e83jL3f6c4bDbx&95IWI^jKzHE@N6i>Pq=^y%Yv
zVTPh56~JVB_?1Mc;`{dn752?VgRGTyX@tyK+THePqZsJIM-+#0>LzVOA4)v7{n}Vr
zg*4xWB>@>9=vweJCsAQ%O6F{_fRM;tB=C}b8?o^xa!8fgTDu~^EOIjtHmw8<5m|c)
zBWsxmxKmc^F};0!&GVuYKo~{sWZ#B{%^kBB_H0hDn>C9(gR9hnaEoq*?)k~dT^@#6
zgfgil3DyNVZY00kr={oR;_I=$S(Cz*}BtKw_|aT2i?cELyulf0Dw>$B`1
zyQEX2qy22&=xzq(D>>`GcpI=_s0Ln>g+scC%{k|UpM%2E$b0H9x~z-Q=wEX5rqp=p
zZrucSRYcLy8q)z+ZC)mYSZ)z_Q%t~B(bMw;fP0|aqQsM$c{)L^oxjB=f9*1=idFzM
z;229seHFMJEu&7qE0est2{r`Dig?CTI|N67*a?E0?NKZ&XnN@|V1AEQxg%HTZQMe4
z$&_K_65n6vW49P%w^n%BNLweGnDn0?>NcZ6bLh|&0K)`Qej&D)BA0J-8Sbt;P~kB5
zp1Rn8U>)Txt1xByLZ6QvKw*fpiL%i~af{|WtVt-}{H%uG9jj3k(y;JQc%IU%wD4v@Y
zG3!{A4iDa5&ZB_UJ>$o#-87=2piu(N#ht16cm9o>f?IKtkJwEa#xLHHViMHmv|z!W
zJU%=4Y1?$%<$jgJRK->eEhwbdjy@7*A)ZDtjFjHUI;yV^Rr;?J~z&sQS&YiH_
za#nlg=Nb0T2r&Tg|9$unetconXW(7r?s59g82X5=>u9(#RbOS2vTuu7Y1%K5d0T|<
z$;x7x$K?oFF^Xo@!?m}XQa6f=rSGq0i2zXJ)c7@N?tm^mtslaWpFMjmxSpS~k@n(|
z2@PAJr?H#F7cnga3n~vtA@l=laOtD&c{jiJGg}kamiP?U{G;AbM~bF9o>0%
zgIB2Jw=L6?_@Z#Hq~zP?LYig53-yBFCCF@3vYYDhloV0bvG}?D`OLVf;p_rB&xlQe
zFrC!HP^o<6cuURMTu|QJlwG~QWBGFIzd<|75ws)=U#|vG>9izf3U%#fh
zaF{jgghM^sf;G4HDOY4PM+6ELmx#rfvbDY!kQhDBXx7IPOAD7HQJ9h~;&p_{(mU&{c#o26-TD-X~Tu@jP=bA05#JU7{V~2r}U}Ot)Shf$!G*`wo
z?bHo_l5{->)N=3MM>9>^m3#FXtbWI-!(4lJ`l~O5C4M6y9gv`niC~V!_y1Z`6Y=b|
zj5AiaSzVf>F5s`qG_!ZCbH8qg_d-Or52M)fjQwk#$1sgd0M_vODpE7
z-Qjmy&6e0*QWZ!UNq)OUk4nb{!nH5n@bZaulU_%-hnpwUcVOQ|=){puE-iVQbqu`)
zL1_5STcW>ZqCPW7p`e3*T`gf6VPQTGxSTya->L2GvR8D|6SW*dZ4E;igP*W&qn`Ek
zIT#&{`TbJXNF*gC(TIjiU*5TMnuCLq<;5xf0VN&L`BJ&gPEJhhF)~=#B5H%;FO!)L
zZl{FJ;!~0Dj_KHNWHl5+mQ74-OOw~xl8ebzFx%sAEDQUqd7uOXo?r8r=>RRG=+Vcg
zlOvelc#TQyvix%0D|~;T$!y9)pgPicwP~
z#8NAC(hIxnyIMj$LWdR+uRQca=KtCPBR8e=pWYwlPq2=m>qZ$q%D)aDi=`Kx{G2}1
zL-QSYHfHvasjmCi+`|*0U;%aKGuBUND}ln&l2W&&9CLRo>f*bj@mArU{B
zoLTnvp#$`UZ*XL>P%rB1vu;7HZ{ZC$1FCK2(@7~QVMS{@R&+^YcWm`v&rVp0@Xx4u
z5k&|&jYyBfg2WJzQmR6MW#H$2$TT<)+tS1!uqkcpumi)VAAvm1_XX5_`Ewm$2H6h#
zY%HrmktwT|8*M`K!Ok=ZSxhi0%o
zfp*PpW6=_L_4Py!`iseTR71_jjGQEK$!}Xe-=kX^43uMhE6jz6m6DpOH^+%
z@+p)pf)ym-E{o$RS;;zVL)kTsaTM|mk9A#EFfdJ$8Dt%E=YzTVg#LRUED8U7#!iRz
z^14HY2)jlD^qRiCWlhrzD3haT(s%5TX|#+OgntPSE(pp);}k7Lkv{J?h3+4}M#JgG
z+niNJjH`%C8t+&by#gSYc+S_rnj_jun@9xis`#~cSSSG?xy3A|8X<##CbGFh`z8ZK
z08^%PK9W!0yUL}2udC5BvP=%XcO)5xe5m2%Z=2Aku8eC>vjqAmhF~d
zdpx(!%IPV&@4=F%e`o9fSNKs#U2iw_7z&4@=|@AKv`y3qB+ZvAoZ;VByfB
zWy_{1TGFxzN|1buLNit8ZAc!VB+B<46ID_JN0}9Y%a_YtyvcFxsj8~0C{-F2>fIXr
zWmEURoUi-AU1B6F@{N3FDUSwUBDnA+sSr5zSZ4g7e)s$8Yh!Z{@Q9_^kk=0WB{|M@
zak;Xh!YF+Tl|d^NF)YV9uC65?zLV(cYHCiTrpjJr7jznWkKm+%y3vJh*V
z((FE{)fFpk&@W6%Mgjbhj~UoLK68X}7m^6@4gft&aKbgjhKL_-i_TK43Y*=qaSjK5
zgOms=Ah@eIIZbksOurF#-E%hRJUFHxr2cRR^AoIF
z;iwFYEoO7MjQl6Td^*SP9g{fPAdU=PpVaC=0MT_U>j8<2r%n;ZSBKZ4+ra_C)5k|Q
z+DhZ)H!?V(g9ma2ApuweV;jaEH+;a2l1d%a_78L68va%Eo*?;MFQ?EEzYb#h}
zB_<}GJUPhW2|k$Z04G%L0Dt^hmc{(x|3I|_$e<5{3xEqG^HgIY?M|Gy8MhqwivdXY
zSfa!~a)bzyvfaAjme00JVbMF2Ajn7sQ9r5SH55?l3j>A+4{Whks_w-~b;JohyLa#M
z-j|sTpo7^{#43n<2&GkM%5O+Pi2o-`>eKJ3;M&44i~tTu67?UpTiBW41X?MS(j3!$qdxh#vDhJA@r-xM$vjR(b1I
z4JQnSAvH2X$_n_sG|=ds5^lL$Ybz;kIdu3iJVdin(O$dw7r+5hYa6NF@d)r3`HDvs
zC323;b>hTYfU=e@I9(+C<4O#lf8`nN73YD<_H=407B9{8NKn2T%YQOpBb4F%pxkok
zBpSNUoF8t4V0+c?8O0%pl{PQ1H&%Jf02*RHMc&*km9
zO+`3|WT(qF+cj&~a-KiDdzXCd7+>ND^$B#|ui(LO_+_Sa$Vncfg5q6c{l*?Hs!141
zuOUh?W`LeqHG3OQ&4B;Fw^*-7xs^`$cvq=XbrC)r3snM0TueprfM=2ze-r^p>GS8p
z0t~QgkXZ;{gQBfO#l#lvk&u*R#!oqUkvPPw=qq1)U|~?`ScO_|h}LU3IY6ZBotRHU
z3fBe?B-u#swE0(6`iw6%d{E<0v+GBiUw2#X&R;s{kKd8qd-kK7I}#bO;$?b&Z5fHI
zc%Ti2dDLpz+3P5s7^1CNlj*2Cc~~&g53aG|Jsn-$v(%=Z?ks!y6-jJ*0CHBA+AF-!
zYWno!-Yz=IpuL=dvtR(8$$qzJwjVy6F>BT+D5?5O-d(fXk-}z9jyIQy%AS+LoKVch
z`~F**-cwm11cwS!H#@CpMmfU%KdNh)@UKXgP|>~f0|_P&mu_7E1j&ywLZO=_DUiGW
zIsY`a;9SDQmhdAVi8HdIy1GIKYc#epKl$)I0Ft4g7K1VeR!0z^OMF!OgN*c>%9Gz5
zH?<_Jt%`=C_FNDJEQ;JOLm9?1y@TH{S*v?w5i3vv@bq}uc5Ep@5kCDKX0Z-k{6~HP
zU>J-4@^jf4YvQ(fj%o&4rlr1fC2abhH9x6bOOmzWD7%5XcN)%2nW()q{M@a
znBcqi=rLe;dy322y3#_t#0L(P5GtZ^W5!Tz-YG3jM^rIt)DN&pY%rpnu(2vGwq9h0
z<1bN|V~Bd}DlUA*2j}ZxM^{x>=j7$@m|;GuL(5;3wzzm$jto0{wG{@xiS<=_V(SCY@Uz)(W4c+O{D@Cg
zTs@|*Q!?igVBgf#)W}F>NJV#af{4M44GfywTCnw89u$>)_xHC
z^;ZnmvZVQd$bFBIMHZ7PltT7D9jPmKB_*ot+wRk6jt>eL-Z-<%6k(AsUQc~cU&l*k
z_AShQ6Vmq1-#P~0urR0bBKRD0y?A*q(p-NZ=sN9B{?Vx5lQ-jTIHa!cyrG!WndWsR
zxMKK@eCu1~qlW2hb8!t^J5*15YuQpXc&v<-5*c(fe0f{t2?UqcK^q_
zUMcODlTw#`D5UehSjlpwY7ZTCiLOcoT{jdNMSK`$XHz{?`fNz{x@mQ*n#|vn_jv8&
zJn;V8Zqz5lJP@M6J+m@{#BF!BZJ0ZBud>6)=Z)TO4$|Ub>OsxPNQQwql9CEaOZ{Nu
z1e$c>Hfpe;Kw@rj)1>4Q`^x!EE?wuFI}b5D<>};R_Ri5@pXAWpaqDaL?l#}Jcwm<;
z)d%7O^^$&9Kk6C#ttIY6l7!yE^MT1WT%5FWUhQL5%TeYX
zJ6-bLg4clEzVg}Hm^LL=QMyRxt(?sqolg~UdXAc|5#`xm#M?gnb$hsfS-j;!nsl`YdZP
zG&n=ai!R|?O^qiLD?#@~Va`P&5%eqXqjxk&=TTf}M)ieEm;bUqow*rlmuq6v;rnM+
z_Lu8(^fZFqZU2O9c2W
z|DQ7nKQC@=KT>69Tdb;EO|*CC*{KHcE7eNpUe>*Q%r$>~S?0X*p>L%!PgQ0w7?|l<
zxu^f~tn7skex>eqP|Ew0x}xR!!tud5+4t1j7Qf0~YG!}W)F3f!>s{aNN=HSTUdu5XfvdC)`=TSw{*@7_xjWO_^*bp)lT}Y?j1^{TRZaSNB?^ADOFutH)-o#
z_3_UdMdGs)E04VK`Z@hswst~Xq}&kg=1<`=F)=rym*oGpo1>kawefQM3h&xO=67t`
z-<01Nu~OhIcdgl5Yyal-uYF}_*SyXcdhhD|MbGvb_K0v;*|SBlJ=meOUE!9z*D}!p
zYkQfc^V-AnT>m}4%&svSy@RK;Mx-62dLx#Bru|X>^5n!$MjwJJopWoK`)_v#m;Q3x
z-rC(#B1RkT*qFXE;5q>=UzujlcFOvf7YD=^pG!?8JCR-A9MO}R9oCA$Ib43MG$
    e^>eoea{W-1
zPamW=eE3RV-?;o+AHHMn`x2^$0EjCk`vRm2KRX!}k+pWruL)9jt}!5F+8ZaP`}RNN
z8aX*R?HO}6J;_W22xr*<`o@#>#BHMVoE{uFItZYknx&m2HQXafIzzMvSW`bRoo6IB
z6M_l(1powjamV7A>R_+6dZwqRV|EAGgu*f>$0_LH|G^>}(ya>{zp0N3-Fi6GPNx#r
zzYQA%vO)Q!xgmxWhz#hC{a8x6BZd6sSenMx!cM5Z+U-wp|}tiwXLC&Q2X@<8laH(dS_-r
z#R_ofaHa`H7PzLkd2dzK)<1tzv3woWL%du_%M$|9^YZ#v7Ym!RHg5b>R|oTTyQM!3
zJyo|^1m!1=RvIchvobTO_GZ8lJakBZcq^C~9cu$+0`2~m&vqxMtm*2evNsA_YM@G>
zdk(^+Tf2LYNC9}qk>+GQ?F8V;b=ts*3q_&TpFN8t`z^Ca=8_c4jS)`_C!+3yp+wZ0
zr97Ko2EPcHY*Hd93yyXuQ>SS~3r!&{tVXo%Wp&If`(Oyo$E22)3e#af8VD9Acz=c4
zQ$u*2vSQGR#Ke$7uVeJ5D8UGZp76(y&(F^E_$z_Kc^UzEhfg-ljK<%;S#5!|!1^a$
z9RFrH6F|e#19#Qu7*k
z_v+*!0aHdK+@~j?cHx&^4wO;4{3&1%^sT*n3c8jBrWS~cnAa#YCuWascbhwo&SAvj
z+w@t8_R~+i9yHN5S4gMV@`Z1)iL<%cK}fIk#3g5
zFxcAvZj`2#&6;)VyAolS6vPpk)bSi=52m({_Nl-EejKVowK`p6Zt32Fmnc)K;3zvooE8M%ycgh
z$Ix8AfL;ZwhsiXK+C9d9`PYlu;_bnh{p#X7Ga4Q&J0sH(`~9MF|EYfjM!Ep*=qP9J
z^E2p_y}LIj#`%U}7J
zt$aTXspREj`|h!5rF}R$e*GTp;y7o)b_lQ^om`Zdhx?*M-`ix-K;se`xsQhqTZ+!f
zDs}eA(W7_o-07O(VO`=%hm^yTPj*zBN{DtbSL88XMPr5a`}5~}wr%6z-6IrJkSUA}
z=}oL&N0
z>=aqKI=~pZL%^JO&-qXcidb;5e~suoV`JmJd-iPo>Z8}^-~$tOEn-bUbHzujxc7s1
zhw5Q#YtyGPy@?ZF!(Gd2Wh%{`6uEA^v9C$f*Vp6GoUk@>e)OZGcRql0EGBA+o3m(i
zh>_8AbTrmevt(K%fHV+72?oO;^JMig5CnZwP|u
zKlxlXVMNx}t|@&xKil8t&Aq3F?8|{Ys@txWKTGxbrtHpNeh
z4P$!iP8z~Sw~cb5KayP2E6L&H)|!3`#l*yjy00oLm!kl^uwkpMY3r^VKfs^KJ=@MGB_u*1US<3S!0y1_lPYx;t*Ur#W4;
z$kP;%%=C2jzVuR_YX6KY#cHRm3hF_%5BqJm)QT-O>mfBFrC=U{2-oeSN7lVX&$i^z
z5iZLz#(5iZl`|5JS1y3!Pc%;++OR;R+~VmPoKC325))O|5PDRnAH0y5ZDedbX@#aS
zW066z(pj@R&LI+LQ|rx&;pY0MgYD#@f3a97Gb2N3!F!4u21}S7HIh#5&Yz>9h%n1|
z;J_2<>2@85XQaZ7#|9o~k)y^CX(MU`ShdB$uTs_zRD8O#*aQAf_={Bd_;4S1f97cu
zdBi+LyiXW3f+QIZPEMVh=JFFzVapbA9r-0md@W=>Fs#B06{W-*zgM^|DzB&j900hQ
z5oDh!F#-Dstll6|D#sbQBy(6f1RNfyMq9p#JI=(hod*If0pc;Yj*$?L*oy_A36@&q
zO8dJiUv;NH1OsR&y=~+6cYd7L$12P%^TdOY2T?2nfv6HqADe0t^$CYV<<6gR+*)GClW?
z-n>V5?}nJR1m<1qcNG#yo2bA-n@FXfpMq6^iIiX
zl$L7=x%?lKWB_AAVEGRTi3=-}F7u}a@-3X3Qp2H2+lfh-{1w!zep&tvxf${r4pz$f
zcS4A+X@1s1`(BSljHLyyk%duP#NbFb@AL9z=WF1Yz!
zSF2L$GYSt%zM@`v7cP9G4icQ4DDAjz9;VanpOLxgMDsd3HbwPbWJn;uk^VWPG%!EV
zSN=KPs^Lh)qgqI5uZoI}aXxjFf#yF`bFS!S?D3LO6Wk!#2#XkPVLIpGA-+%HO2nlj6SyB1@}JloO_P8;7oshu&=7+e{=j1)6bb2?k=KC|R`eys#i`tX
zgtv!c$zxR{T!bECMrX}|g#@plxk>*2xE{f)tzO=JyykPXx8_6K?aCnXGb~{y&yVXI
z^GXcjGteej#c(T^mfWGO=35D)7oG|RkZ7BK(pJC%c@tIY6^azhD
z0-B|cX6{5)%7Nra7@cvHsb{PTh<_Pb2cZG1AY#%yj5ZD&ctLaimq22I5`=QtTo#D2
zyOI(U3@BSl(EC+(Vh^Cww=W&HGh;+IH!6ru9wz{<5YG=DWV;X>Hcq}s((`>4M@>$5
z2q#ke0t%zEMV*VkUHg!d^Ssk^RK@!G$%z_C+xp!SRdKYuZj^Fcb@9%Zw{%oey1PEJ
z9FQ_)_lVm!r4lVKTIx$_<%T&*^z5$J4;HKQz2`00hdugzLQr!FYj
zH|o^xghw0hT})@rB{pA1)T!7v(j-mp$*aLf#<2$58dKWSIOZ?^ar9oU`Lc
zE!_iT;gV*8aG0Q;R6*hQoP$TyX;6TaHOTpDlPN%qsW2J-@?`>-9TAI)uGZ;v2s%uC
z88IJVrR1`1L@3CQE@|0R!?2%VGbK3c9yVCS`iMSKZaY#)f4Jwb_V-^i&OB`nCEaQe
z=c-E$EIa6N@T{4}OLPky4v3Wp4)i*sseHHg2b07%P`IP5;KKfF+aF|{i-@=d`z^1db(xj7Wamb5KD_(!rlx{?v{O~1oD(SAowzt?CYo&vmBKWn!mcfD=o~tE!-M%dx4Cv2SYSx4Q!y>JeZuPB;%CS*dzJ7eQK(Yy8<>XJzwqh1bku1d>NZYe4C$$}*X
zpOiSD8K8@YJnj%<#*c@50Mdi#w}v^W-*MS
zTU%RS{|ku7ygwcwW+b`-BVna3oU%hEn^p6^5%bt|S;j}~{?^+XZL0u=GylWOSG;!2YqfeX?LUd_q5e|Y?5?7ar6igjrEG>(Lm5a&sC
z0-@yq1O)yYaqQ32nWn%qunPvuWaBaV2bqDl5BdDQodO4KQN6`90eY+9D
zW~uVTGLMH5ve4bs(lY%ua@nQ9ebG$7!FDAtsA~w|4q+8B1d}2gcZP}BtvkY
zXlRHZy)4cmnYwIHVRflA(`||u&cT4r@=N-E-4ZP>uBmqjn8cwg+^M!iC__kkD*8?$
zO4tM}2i*WVV~XqBxNwt`LvCdVdqerqyylq;78S7c6+;H#AaUa@1QODqP-b8+%$mqI
z-IMK{pvTOeduQ_`O{+Z6K2tV*vgM|;WO#nvbc5c`BzuVMS7_V}3>4?RvKJX36{d0b
zs(5E-n8BGP`L(dl+3mC%%#^tqPM~=D?Y{a}a?C`d1P$?Z+qZa`HSe3NqwF_AJlQ*|
zoSi|t`cE9Eqk|~2ua@NZSsz}DeTgD+4H-VXmz22b?vv^1QH$8#tiRu;BGdZ^Jx)}J
zb(|->Z`0Pg0h>1c0m=qA0M&G-8K*wT#oG+6+v9?UESZF$!uL0AzfADB8%5n2p1k+v
z9jpC@s=KET8V`^{p3E7>Hiv@B)$7(_xhN>XSVgX*!r<_XBNNS|@$pE50|=7r(>rXk
zd7P2aFB1RFXT=fATgz{>$gQY{C4wXi6ju7A{rS7s-iSE)QlUd>0d@)gS~)g2IGFwQ
z1T82IF^?Ix>nh8b)xpmkmI$i1FW`Shhe}
z{8ePr$8Umjf(8bdft+Dhm>V3uH(jOjI%r(keN1mWy)
z73hVjpj*{#MGc`a-QKI7&+&UN*;%ez=m|AMf|7L}s9%)Hl?>c<6_=MHJuliJE
zr&(!#VvWIuNtz>aSSvr-9y$<%e}Y^$h^u%)^Rbyg{OnbdmIkwwq6;_~)xP*1?Yg#I
zlC%TTC<_yAGFq&^ohB?
z)x)|l=k5BQi6<#4H4i9E7eisTd&(?VS3Q-TnyveGnKVmyWw|ulU~WWP1K{9^Zrf!bEP
zUFXl=Onke4|2{#5=6U0$O|S%ng}`Ww;4WZ~%fS{DIQ=JX4C(!Gw1%P->li74Sz}cG
zvZ4aZ3VhTd%n-Fk%0b)-m;kW)n?#AQ4seGWgs{k^s{il5e#*+#b#=`|A2tVytdsVA
z60N2St(c%bp2h~^A?d$cCes1u&6_9AV@1^`-U+yC9Um_JOIg4T#oW{C0a{ybu-a}o
z!CJ|7l$@A*8ONG)1ecxLN3}t;>}YyA6Fl|Zw{PDBg8E$#WmSbfUzA6~i={+GmXnp9
zZIHxIRPaQAUr0^G9coq;gYYrl)kkcPBz5}$~M^BRg
zIu0g9@7b@&01g5f)#E|foex5F!GCZMNtK3Hk?At@+EhQg!D=+g{Wg?1#W8}}(yKFT
zK!x_rRpFk_U)5MugUatan{NZ^XeMeO9K=rQ7%)AV>=%@D{IyGm=f2x#sr3rt6|0ec
z|DO7D=igJU8z>_haOYNulf(nkBu8gq;CDQiHFHEiRUec{o96qXvuBCHeZCr_
zJCreSFVuSWlqhKl4$_OxS4}*5%YDR`#||jIXeDXA=nD~`{&ntN7Ux+ws3=3f`3$xF
zjqCoo6DU78j|j9L5k{yNStxmkLf2=D{yD`g5$WBK)JG*H
z(EJqo*agfXtpKBu+@J;utMlZgP#O&xI#eQZ^^+~PFKufwSXF>g{Dl59wW8ZvKVOJ4
z>U-n~WfNwkf`ovF)1Vu@tcK6R{h=rNx8jNX1T)TCNv+Ps@!NoB+knm~fP#rZCE)Atn;jHMcf6)r7Y
zm6`BVt|-%|?CkO5;#xtVcr+cj8K5qo`DApD)P61V1~i6mLq%uYU&+LT^v$ks-r++|
zC}JnKg0k__%O{I@$1>+e!(I)58h%nBw8MY2Dqm`4Pkx%;5SN*qy|w<7!x@O0_7Gx;
zceKD3CYn-vSOPZsyqjsvy?Zv$H5V00GPm}vPJXz4Xp24o_9CnW$mIMI
z^ZU+P(N;Rj7LLcK91A^b;+;fMif$_D+_t!{h;il(3x-{`-lWIOt}mk_4qBci8`}$F}+G#s%_HdEL*wqEmb9%UvK|CK{e{y13vmf;KUJq^C^4&!W)+(m7;EmwwAUJ+u7Y
z-QajL*qOI;k3g
z&0V@K$HC@sM980R_YWEmA3HW-!ZZfm;(};|$=ZG!iPs&cL+N(`TXFLTQ;r`R^~L3s
z6ZIs;SzAMq0-VHgCzp$ZN0`4E*lhSZY4X9b4-D>;|G34%Jn@dO@D$Gov{1I^Q~R@P
z3hSS#>YFX?1yi)DownLaP%TicyJ{$!&7t=zu#=SKuHdRJgFP|wVa7&6w)Zt
zm1QW=OEQ-88(a3ww6S4`&XP3I_ENe^pjm8P7!PwTS!ah6+n?rUudH(3$xqr-kI=Yc
z54Y7HpZ7Rqgfz9&Rt#n@$LvW*X>ml(6nKp2QXywQ>mRgU&p%8XV_DONn
z6)--g{(#huj*fK3?7n2sGK{~IQio)CQ)7CiFZ2W00aV_L)RTbPuxxzCY!B5h8JtOZ
z=A}!n*Efz{{*V=Xz{+aM$~O%#8r*|GJWN71udQ*~ejC9VfW%A-CrFts9|AUVs(x1&
z_h@j7Z2A`EE|^__O>=|cqB3&ez7rAnFKjnJ4Z`rsl$9lII~TdTKVeU8s98ht;G(njdxK-Gc!3NuaE
zJr}`ka_5;adgq(5(60BqV|++{(k0mrhVFc(UZ_3+@~o1@e-xrXfh5ai19Qemf1jB~
z$b%HNy+Du!NK95$wC=-^Wzp+=A<^kmND5)|fGZj|GrRUDx>Y=Rh&~D0Qqr
z8N|RAfo{y3US_B&CKk@$#<;SLedUz}A9xA8SQ9NRcYNBalQ7HRLG2HCKBOyr81}kiC`H6xBu?C^5?EL)k
z%mR-FwGP2|?Zu1r+(LfzA+CT3`MKywLy*}BySg`CWx)vBkDna37TjORDX=j((3HfW
zr);^qf*&id&S6rhnIF>-0{WWBDTI-z^^ztf4&4??*iLiHa@ovb#L|MS($DAwPC+tK
zQpmovNACB01(HJLSoGq>0nb;+O%w}&D8`8%JAK+pcmIv~{H1~)`n9WkBpG(2Pz%xe
z08_C+bdPFtzwkxz)DH9`7oAc^sK<(kgos#MnmY!95JFLaTV1fUos7j}`@`uKqz3o5
z!(tI!Ifp+am)OP|ECRTzI^()4m&K
zxzxb@`*$#OGC69xqTG-g;Do~vrR_`>P+^$g{Nz+obwW&b)U)H93&k%oL(T~OFEcA~(2CPN0JhlX`n0LekX`O
zOun^lTo9GW92CInh()rSvk@7HtIJgYG$Hcv>Tyosb12Bjxa0>-Y~Fpr>7uAb7*kY)
z5Y(T7sS#Q2jAN_dmN$MK0;}lYrO$lUE?98n;>8hi-TGF451@3ddv@k2hMHF8gBv8U
z(Nym9mF^X=H2n9-KY$V)04Rf3Mb1DDN_Tv1m<6vx?Yq%>{zqRuyv_;?r5RupOi2Hfb9#vW+^
z2le2#i%{fjPH1Xs@*YokU*UgKpku1>!R1u$H`y<97(g~t!@%;Hq&R-ed}d|@nQl;)
z=#4Fs8;JZ>b>>eqS;3u^wc=WPPur&+MLxTyT%u&=gWpdaJ?)GmJ2RY`y4=cxhJucw
z`Oly5`NqSB$)9_L6PCdL$?r7TFp0=RGpKofB>+h<$hk>#l^DzC6s6qk3{`FU!l2yY
z;bPYQn#wXy)e8E_aFG?OUu%vSGFEehcSY^dqFxJDcOHbMo__gqOM7e(Yp8fX
zUt)^j@rNA%bI(eN?!IT8nuUp5u8%4!t6kt=R6F}$r7Q%T`k+>P9Iu!v!a=e8iH#aZ
z63A&pW!nW$$ud-FhX?i#Bhn!tI6CGsJ$&VBP^542u9vcpd-!+BnI5~<@aqQV*Kk>+
zx|WTF{47lA>5#G-l0zJI|D)ohI0#<(qj-=tjX~K%
zI=afCO&>V>!&X2&{MaBq_}(7EgzSa<#4iJ0S319Mo}~3o5IZtGpz7CNB@!vv^3)mH
zXe*5!GsZ8~<{{NhOi*bdmHQ(#nIR%5f|@$_DhLHw)cUBK$Ez)`i#iSd;2SF)VC~TN
z>s;Bs%LAQbm%1ML-Lqx5sN@=uM5MFn7cYAICaoQ`^+?OfrLBc4bt<{FGz*CB{O0K>
z<5k3J@?|Jat3Nw;20q=On49zcoNvb12m6Z+!(5jyPSm%Xsr2Rch1s!bUSBu1f4eXv
z^=Ibf+IuQ*POG=ie^%}2-MD`B)WiuLbCy-id+@VYYOvW_-K0}JRFAIk9h7;^X57lS
z8U1qdnNZ_oF@FGrn!?Rjie2jKGvEH%YUKPyO~F_gglKe5ZWyiFE&uTnwRzjVY5jh5
z`-P#&o93h$O8Q@&+cg*64IZr(efQb3eStPc@+St=FHoDA9yim}e|$-X?*4n=$r>6}
z-#^^p{197+Y~5sKajrC6_2HTN<2##&HShC3dMnK#_JQ~Lp$}{u$-RSvvgFCp8hx^l+ALd>yuFWe{EBCnA
zyXG&Z(J9-|`Ez$!LrYTUILA|W)kK~S!j2BM<+W?q2F6tVKH}4;8ewsf3(J~yB7}>R
zQzeI*(+$K(=1YU~PLzKd`{(xF
zP2mZpH}va`Qp<--T#*Viq*k=Oj669
zY+M}Dy;`C1fX>XrAwN1Bp8oIOinplUD4ia@YS+dW$~S_K9q(Lu?#Em8FSGysthl*|
z$>hi1j0e7lPLNryY<7|1<6)Ft7n!j2BZ8NrbY2DYs2v~m@q%im8v
zpkc_bg{{ALr+EbL&-C(tq;y!eLtZzf#{J{W7v2~COX-w(Gv>@U*ZH#o!t>haSM+Mn
zy>=p@!FRgB|9xbckDb$Ju59Wq`@}+be^C3+7j%$+W?ngHG`P3hvfaX*$u#&)S=sDI
zOVn1X)$O446Kt|uKZ`77fsH?sWI7Gj`1IfV`;J}%O9i&PLC@ayDfI%|(#|{`?Dvb7
zOR!_!+byu^@zikxL+Wz?3&~uCUnqa{v3eK0Y6ZaghT-LPg!^dG>jbku}21H|`_fz@w
zW3R+irS_F?a7#FGxU%=upNG5=0ouiLCHDB1i-5L@u(Kc~;
z35gn75!0n|*79`i8u=owHwmR|Z!8vAX6S8o2_d#t_?
z);YYr#PIU~nI(qj!-o|axTL9c$o8TSq%3zyn`3-tMDMN1-Y&-;D;`r>!*}GO2ag`@#3|g!xt1zyL9YMgVF!$VCD(ipNZRI=h=kbA#MtS8
z;qG~_VuGp{+#fpUj3a<_%xxoipNNwqA1GY!|5?Au8v&V8I(2
z95*S|%h7+l`tJn=ll}8fy|cON(MO#m(SwKjAJ^%v;dO3KD!R1CUWS$S112GmcHWV4
z+`*eCdN5TiVeZEFDQdqblw`o>pe*O$E!=10`Jvcs$BMoe&X1aTcc)wgjJ5U5WH>=*
zp@$!|>??##{%2cI?;VSe`^aq@cfN~lY8gf)TA5W-`LqsamIsaxoLMy0w6diA+D8?4
z>a5t}oKFWlof~r%jAIf3Fu^fG=PaMMD5?0;&5_I9-7W5Q-R2(J2@`Olk;lYa{xy1!
zPg~9f0sOSh$6?Wq6uRi#Q%OBCU6@B)w9@uD7nCf`&V!dPd$%T3o}ZIS8J5^1$PX_V
z_A1rO-Kz*`8oCSP(e7w=L$Oxr3EuMyuME{7JqN`tc;YKRwcCxH`qnKI4cKqcTf`<>$3O5MBOk_NGAt
z>0A7|vA};v3j9Wz7l|^jzyZ@W0|uxnh~Mu)59m9oDgZhNrRM5ee*+vt#J7q;v2T)Z
zn$>>$21<5l|NbmOFA{3-fSFYMkRlW})eB|Ox8Vm$S^|TRn*bl>Y}HUaPZf4_!UhP{
zT(UVj$_#B5ZvMI7BH7Q+PvG^VA7#b>CxvAQqg5Y3vk*K(2MxgulGbtjxN$yN(n$p)GnO;xEH9|UKB>Q^Qr1=%pnXaM?g2~lWK*((Z+blOe_pSw0HbF+!a
z!)t45vZ&?&BGd^o%6|)FBmoR_pD%u`bqd+{1nBJ1R>$Y0{*&^L_&QW|_Y#VBK`V#m
zbH)sat)_LWL(S6Z&avDor`-ZQz;L{KkE1y9Kqw+~(Am(*Q2&Fo7uUrfJP0u;oz_Ts
zvVb1%+b2wRAgh*6PEx?N&oHXp7$dVoV+5swR`h8{-3W_S^bq8_%OkEQ1xunGp`685
zj4HVB?c29EZoH?|;~CBd+@EX@tYB{
z(`}Y#WV{JT@P!##B>WRmsDdrz{+gb%gsCvaHNe6I6QMttMcY<(^k6;IezlwZ9ro_F_t2N#Dnqelqr%K+iYHl$rJ>^H6dSgQxoluyVDGK$TV<3~MR>KEYOZc`!XbVz
z=r=-nvHPx0QUgZqzUm(Fzk3|)S}@_&a&b2}wu}Lc8T|nWQ`10(i^TKe?1c+Q|LxYA
zH;g?IIzxuk^8{#qbB)7HTJD&3Tnz3El9kWqHPB88tyHs24P?AkCryT
zLXbT39U8l*e5k5w`0w&tL=K%orCnUi-69F$g^DMR4`u_9hYY0l7o;85{!{+7K+1oFIRT()r#)OY6
zoM3Vz5{$q4-)9h#R=l}}oBb^1GHNyyN~KbS
zg%2MDE+!yTKpjrD2y|kA`&Cp>nsf4M-@g}Kd6RKPHtwu!egrmzBM5iS$492UfeH1^
zTen(TTeE%2K0(O>Hs@n1bn8ZdDA;}AUlkdZKiXJHsfSW*_U4YUf!48^6B`!u3{mn&
zZ8Zi{1C+XPBL!HNc6{BMHQEsZGQ2RTWngZ<9iJ$R+}*XJ!;~jui-2vkum&(wpqFs1
zaL$xnN?erg1(m-aCxTDF)AaL!_ewyAmciO&dT)XBz!d}_zNXgOByfzAxiobet3bQF
zDl#PdpE|V^{QwNqVo&0gz}n@k>g<3XnvZWInYR04oY|xl6xgWzVDr;S7YXbfjU$k&
z#&*1mROcYBr*~JLoaTH<@LlA-{cLCmxda42*QT-k@ZI@zGJK+_SF-naG4k2HZJUP7
zQpA2BWD%o9OLci@T?_BuqI9}*XDOH|g&cEcKKi7sa(ys3;WkvQf{uYa1;JK+a7LSN
z)pn3h8{D$2tOBSw-!+3R5+BWK1lX6SR{2IsUcQ6NlRT3%A9dtN9(=-hBkl?~Gg^)T
zFM4lcW$y62=9FR?kRFU#yMOFGPocg3<41wDjDgp_P2ZRG7|!BCWNgeD1ri9TDIe+k
zuV4N7RNxCqrZ6Si;aWlrh^|ypRlVX_qOv2U_4haRUcJ`CCERP;(R&8coN-KBV!Wlr
zMR%l7eG%4^v$ePDDXMCXqHo^a82MB;J=#*@NF>VUwtLj
zcsXL`(^mfid`S@$G~wqb9Kij$Y)A+7BeYBYQ43g$-5rT`Kz|lPj##{P$}wy(HBDlX
zq0E!qp2#$#lSPB(noPnN?ki6Qx=8|wB|jl-?X}&KCbqOx@jV8~g5+cQaL0M`rhgUf=-P60%1s-NZCD|9I?
zl8r2aK~TmkZ)FerwjMzh(A$7Ne}m5&K)_U@F~Oi%;)>(a#;DP<@uNsog3s3&i`wdH
z2EF2OMonV=FJ8WkZ_@3>6dynY-S3l5?V{NJZp}J7HUbhBtO(sSM=SdNk%{AZu<*Jh
zfhEoG7#jf}ft(_AXa(Q25Cb5ap&{G2M!&E6NRPX84sZxDh$!sWO7>%V5U`?Q1z|(K
zT{zfzfe95wye4e)32#(*cKRh5WK_S^tHZqyv8ec9fUr`74_qx-2HS$b&oVJOArww`
z9k%{oN)K*Uo_6tEcJB2kuS6(9{XLrAeGlN`)^6=*@$BdfoBn=5Bu2kgtBUX416rnl
z37_flHF9L&ovD_V1%rcN$m4{5=o9c3?-mCmYYDmf;4c05QXlu(8d^2UL3Nv%?~Eb+
zSTcb+7Iu)8*%!c33Lf*VBjm6>lK%MYX%22=8!ao1+_21`Q`|M*ktj}7{dY=1!Pxs|8@KoapLF&{+a{LG03&diCCn4fS
z8eaqH$C=V4{)*eEus$Uk9CUuoR=YR&+pbM_GxV@&tLNYVOoR?ZpDij9Jg4M#_s4_E
z9$%f%dCTJ~S4JQbBTD9#hD>GX5XbRd58KLy8NFl(rIV8v`pwX~d1>YzvvbW;kV*y4
zs$aGvKdx}~#h(C=9AHeedUSqkK1@&~{U@Yk08Ke8lf65?E)bFGx5EeXN0
z&l+3wQ~K6JwNl2WLMd^l${k68q5zo&tx9D=MTHcd0-GuKOr?FSmx&Anlg?#Xo4?>f
zk;sf5Pkx;~yZ`u(2cIt*Zy3DIke^E-)YoJ|nt3g)z}0B!gEkB7z7UE^xV)%;r=btv
z0rg#WY*Xw$*F^2)ep`T;{T2a*PP_Vb7kl{4di9~B4;fh3t$IhCMM2En20bem$H^+)
zLwwl1VUKBLPQP=f2Ky7d>He*#^G@1RJJ~sk2c-rYzPWXKTI!+c^6K1MkWO5Pvx>I=
zLWqyr;0z>Bx<%%*v?xC~>DFtPdg)N-8YN~iUTi`6Aap(UO3GTHF}q{~#%XFYTw?yb
zd280sd=u{0(&Ou}KFzluiOk&aDRSM9NS7yKM{nSGK6mcOAVz1hE84$RNHa(BYcS6`
z*Lj3Ycbs2Qx1L|=mp?wTiT)aiu1#t{kNeJF*PKu>OrPT*wt2H1W`9%D`VSxOjxj%0
zUVMK^N+oK$3f?-B2uaXsfs~O%rXl-T@;H-h@Q|nUv{ve
zDP#wMvg{~Nu34pR0s!R{$4mXP=GTTvqJA?=OG^m=z8)}+1MSP_&%=jSD=5iy_cu^d
zoZ=a6CUUt+!iTQo8uf8q-5~wprCGE9P)lU}G=8YDUUK3S#TNx9g2RcU^4Lp2viaKM
z7G*kzF29-U|k3QQvPLNm5(OUr7;VC}9pKJkff
zOI)<)%CoD>L}wy!;LO!v@&&aYe7Fy9-WUnLU&D7S8~k7*qaqfw|HFL-C0sYX^gpf2$n!wmhebSE!D(4+Ta)KQqof=&UWAw~^qB-q1KJe4*Tv^b_+>PKdO!
zsf{RoNdw9NB~1^g9;)G&-^c1$$Bm|KMaI{<@U2VlIZgK@ORm5CgkTDzCN(Z2AP-B>
zi_#6b+1^EvOe>nb5fR)kC~VqXHlBhjG=lCPYZzHdAHkdN!QsYnIdGb%hNTB`9-D*8
zk18*H|L@VQ5YwA9#`%-xW)CyFbPF9?bxAL5GktIIFncC9LPWEbfd7n?;
zJH?%u-BV}F@}LCHVb;h{DJgb^O6OcWau`t9ICYJay?
z$(7D~^s^B2z_s!;Ll@xb1n{_bZ#^JCEkFIp*l&WRdhl}+ca+H@Ib(ElEL4Uj&@WO6n<*M9`cnFAqTqYbia)0sD
z=#mqST}ww+>X)g#kW%b*gi>EY7e1?1sbpgP-deO#t(TTF8^dKS84(YVmXN6No~ll=
z#Dy1~{sJ&nWcKZb&+KQ3hwQpvuBGh)f`Q|7uA2N?&$w(f45m@2ikprW)2xIZSD8C!2KqxIG&TkQ0MAFVUUTk)@ic3buy*
z1&fkpW;~I=r_M;LyAP+`7o?e-M%6e<7*oRw#+xl0V43eaxF8R~4)r?)18*$92Ex<(
z(~Xf-mup?x#rA6LNjUEGouOjbVc@+QP~?0eKx)8%0i;c-1HM%1(!ZWI}mRX3pses1j0M{c^Ir|{GtvYW+383KK8_k
zORn{58!0lc6YnKiBN^G*6^@#*-Yu3$9SSyKuXm+5MZ5KxI)#C)rlw^`Uu>Lq-fe(E
z-m#+-FDMBf_$1Nu*~XIm1$tapI7?w7XQ@GRH-1&ZAA_TdNg
z8xWJM4XPWuIRbU7!8T_6Xv#*Wj%3u_JkRb*5WoQMeDt}AJU(`7Y9~<4$w20`J(s_p
z8JWvFr3wUv$QJQh>$dGQh$CaSNe>#djU5G;f#A^UnwmNK`Z-EX{gw*?x}EJ)YI$WM%)p3|nW7R34G7CNIBrt+%PvwukHeSwN4>
zR0R;vR7DuO&Uwip39w3*8=;gv79Nu<5e8kl7}*q!=$y{sG;qAVxN!x2DFPDuSz(+2
zo-*?ks%=L16nK4hAT53W85_$70u*%Z+RdA20Va+aW2_?6ixFd#+B6MH_YsU`u|iQ3
zQOv+U(<3P6BY8bjP_X08^}BZ&X)tUGLcNDPb&jAvnR$Y|yxs{B7FziitzrN%c;a1|
zI56zs?G_uoOekWY2HIv+PykIKPY0e$MdXbR-PiXbSe?-2vh?pQm)?0ZD>XXW+Q8uY
z>R4|fW7DW5L+f!ajW-(}cs11R$&-T2Tj_Y@Dod;&H1I&V
z47wRHawIscZjq2L$8SX*q8_oDsU0ImoF8#Tq!KmXO-f@1k8q=jCE`UNRx)tl29PQC
zUxKOuaYA3zlF`7>Kz`k#WIB4YW92VizGSwU_3H5Pk`<^~p<4`8!r1B$9V!W)4J{}y
z8;h!d`xxhAQwGX6ZTfWnEdV<3F_#RInk=;BbP#C!k?6^rECjAGF)={^TDbHmH!mzq
ztot0+Ef<56DcBqIJP^GNTxP34HC4KbDjpuAhTHyn|5I#O1{JW*eNuu
z%6zOp
zev_@V9vpfR{?Po#-KYeg<40b-|Fvvw+p>O+@1KG13rZ2IdN!Bef(wl@D$CsfV%G?3D=iNc^WRbxGsHI&c}nVJlAF(
zU-&{M$D1H0wls=Vw#20fWziFE+BE6klC|Q6{?_BG^}9(P2)?_1&6RhGlf^e5yj<$G
zL=3nE#spEQhS{5@1;?|8N+(bHGEFp2dgN2(lp4?P>+Y^U=2v*^M$C=Ny-rVnJkvJ0
zUl{RaO#T>CIj0_xhqNc?c5_`TZ{q3tPq8>+5;LEMzC9-7CmtWVA?W3Tt_8C@rVV=-
zB6;uo=$-#{$z!$!*)_hZc;H@CUO4o&_KGvXtMqsu^z|>JgA#`L8Xl%R$nfcC2US;W
z{<2M7>&}JRR=fAEnZa&u>F(9ZT1oIF1C$-lJa-kM{tXa8X{+oKH(1p7!$-aC)?o3eZqIY59$xZ%ws
z9d^Hjs}%AHVq##PI^LZ5`(g2XK5w>63tw9`?xo3CxoyKsvQmT^oP&Z^}|=h}xH~-^pTg>x$Py&5HhFQ+_{RHjfH&NX~aWJUiv|
zkhs+uK7XlJQQh_~0%zV`QqSW*W**^){0!r4nc0gSck2J|>mEDFR$csjI=$;}I`Q`p
zKDSIZi_=alWU1-#{&JI|2+KD^8Wt&W5sRr^?%!tet*-?vBj%pAl0Ee=9QHPp^k<`M(DAEdK8!C|sXvtS7A&KD#f|GXMS|zb~U_+iBB|
z|Lz1XLWJ|L#P{=7Idtm{L1wwf}w07motsRl<{0G=F;!|6CrIQo2Yp
zxX=In2)!yoI?28J-*e$Z_0rSYy&`H(&;RpFOEqL}o64kAI(NMJuZQI)Ubo&A-#)^}
zGQhU||M$lxi+fsT{O|IlUc>~LEX+BoS+>P)XLWwwMvX_Eyl&QRE%O=Mk*V(2?flx_
zUalVQW&hnDuX)Y6sp0Wnw;O|t^aFpc`^EH{vuSbdE2Tfq{2X`H#H(Sh)SuEoH>6|Q
zwkU_F>@Os5Fm-|a4vkJ(YV*`r%D&>v1E%pwH}$DdaOfgugj-Tj5hHqqkeA88rSpF4+h
z$SLXqCkDqjU2$x?!hd(Y{o2!=&UVSlO>VqRl+;etlJuCK(~c6RoS?dY6i1HlaGeh@aoAG_Gw%gD-3rtJatVjp;8@Lub^&VL^##7Dv=AiiV7(32e|
zwL3zD5EMztf1%HFYpz~>B}~x4NQ1X-ckr`kc0mc6ay?5lF8hb6^baxVIFa75R{PbP
zzUv1R`v2U$c1OZVI5C)e<+nYZQf>q{%O52(aSD|gC%UqjhStg4DTkEJS~n+;ib}E_
zv!iyCO1B}`)+X)0HuiO~vdKluH1i(aNCTO3%+_*}qM}~{laLry7^fUvoYp&kmvU@^
zhv<9%;c~%Un{J(c*LixEQz!O9Bb+bS)%~QM(T{vBe(I{W#k3nM^MPmf5r@i5<6Be=)@6D#RY3`B(W97{7G*gxP$%lW}
zxFlRIPN_8RpHlJp?}znM&~ENX8yg+obAA`(0E!1Q)WAoOtH@QOl0Y0z;t*1OgaEXa
zSnrEHe~=G=X_1kAWek}>%&lqcMo~+lfN{Lv-YY2KX^%X%Z21l(Lz}J?Q7VKG<8AJl
zqga}Cd(S}6@95+qEgfxa;>=T)s8*8v1tl3*?#LTrp!DpcC#b4|l+H_Cr4lzPS7Mf#
zeYaTVD@)BAu6}7IY4+g=hhZWibDq&KjM=lr|2xi6#I+Q%x+#^aGC=?p=megUtix+V
zWa46aH8i}i*Lsoj$!`OgX-gWgkckQ<>R-~?o5vsHM1t^9{d`dLdeI6sznbt&-|
z>%BH^{K~HZ^;Zd6#2t*f7H>h`8(wFgEA!e2fq`>@SNDY-40Q@QHzq
zV88;}joZuqMvqL8XhEqkvJ@PavR!62*8yEfcXs*OHLRJLK;UJWkdw!&=-W_noIvnk
znmQ}KRUnrcSQ)Mu{@+I1_QBuRc*n-2aS~R&;
z;-VVDB0!PM1UDeSe~pA-2kJEjKn$1Z4&~zo^^P$753{3d%`QtfIa+!TdVL1wylrU6
z7t9IUvT*X_x<*Fw5M@WW{`pzn^r++sOvaT>E3~X=`?551j4@c*f)Y-$3Q(w%(?lSI
z2H~E=hp*)7>ERnbw`m#^kkC_p64R_l&>5?#r7lS^r}@anqqcw{VDOUIEj%$*FjVii
zLj$Qynbwh>rYgpj
z>ArZh4ASG#DUTe<9814iiF&tNV*3H?fC2^9hR%MIyZi3FdqLcZB8%9&x5w^U9^_y~
zaFfvpmBYWQaLy&plCMyx*q7ww`5oPbkQX&zz}$uZTBr>{`G7tPEcOZOskPNJay~II
zbu?otL`$}!&SAUv0)~Vf@7Mhbi%z#Cf0`XLTz~KTAszi(m6Vi~l-K_F*+XhV@Tx9@
z^7HdC(OtwJu)DiRp#W)9mxt`#Oou&i;HX14$gg6HC1wO{59bCA7xNi(7TyPc`2>b&
zWo-@9HifZZD8uV7zkT%z1Uk>*w%oHP__lF~z#JPmi}l{<;XxR@|7c%=LjnT=&=*ae
zdX}1ae(a3tL}wBD18V>pg+g2F_4G`OZE(7%zmL|*N=bzf!pxYG?k7|P_J>sqfT$f^
zF{!J1{pU8mYj;o(#`vAj=^HypoWP|(5EHvCjVQVR#+J-J^P1Zjw^C3(hKCE7$_exu
z3@4{}7~bl^xLR%~c>t2%;>(tqT7&bjR#83|_c?ih_)P*Y*(XOxOVru9xCn_KoHdJN
zDtQ1XI7qeBQJyeax^Leu$DsJ3j<*ndkP)(22C_5k@Zl<+&96|VONe#d0FP{{wR!V5
z*c%AJpakqz{18IpK@XmOr!nC*-}y*>+T)AIexR|yRt2l$ewlGHpk%mcdD)0p{6lZ~
zUt%NmwUr%mAcA-qs8X5khBzCB9|QAqd2(U!&u|1~q0!NAo0{tCvWp+w)VTF0tcG8z
ztD3FGG*I2${U_Fw$b+s0++{iYnzdh8*b+8c3{y{Eynx9-F-L>#^7Tto^@Mv8grSW2
z&^de+Up>R6Dg(*COuv52s{?a$ahb1=xm)p|BXluD7bFOL_KX?Z(V4;Ga0ak1@*=@9
z*&IbILj#EtuFQ=jX11$B_opI`Q!>o7BKqyyH3tOg3|e;q>|@+$;x5oNHcD=p3A5PT
zJmwDSDsGm7S^`f#2S*7w9o&I=N?}_peG#v;t<=-yY%`opVd3FfYHRH6|3!Czo+8bh
z0X5`@9f^(Y-Q~^u_cK7+5CRe6rjx0??0C7O5CA)58mEYm=tk(0^rN+WGh!@A>l8Cp
z?sarDY;*;hSXK}bkWk3NUFhF`l6wEYFR7G%x9vOg$J8F_vB6`9D6RWqunrc0Vau}L
z;Rra$4_sqs60CY0=N4*0xeCHXD%*`7x1WE+lxIP4Bhx+1XoO+k#MKkzO)UX8(1YaR
zUXkq@I_sj((?-b>lCVu>M(`9L`sU}&Io9|`!bY7*z(w!2aFrmxn)R_FNoKA!FJA3X
zW~0UD#oR9OT>#puVL^3pY7j}RUZ<#lEwc>JcFO!F+%Vuku7`gMn3)))KGazL(i>90$YR
z%W31}WkzZ~;jwjvDE&;KA-i<{BwKS20)#>egzun|y-{Llj(F(CcTb996HaoBbApE~
z#GHG#VeXtcj8D?dWUQm;B$?3>rBMEJgI=~w`>#dY-D_J^*N0EE>QqrrZf~hx4YTw@
z;lMRcmF)a{P>{<#b2HUCuhnze!8_^T;J~7D`ZP*kcCS8n;27hgOwHrAv8(J(S2m0v
zW*XTP*JU+RYQUSooJt?1OWYf;6I&-B>sQw`qTr4hd!=U!re;va2brg5t&PvW_`Q3n6fYCF7;o5%qzvVf`?y!^
z3?De>QU+w_=92nr4#qxhf9i>l)Ow9Nc3Q#+*m_KBSO>J&)va4Md~?z_nw~$OfU1$slM&PVT0K4H`c5?g>2Ij7&p=NU
zbM&8NGu{p>$Q?@~CX@ep;mdv7@?+nuSsJ%TWSGxA%@|@Oc)0~BD!#*(L1aq89Aq*0
zop}kHS6aNkkoBU#LJd-my%I1D3T6te8cyxa-{f+wZfoo#3c)DRrX)2zeLEZ!&3uEdk1z^71X;
zQ||_dIR$Mni`sE->!I3)qD$=;+MYZSQ$Kw8)2;T@)YP}PBE7k$678*^
z{!I?4;jb9vOl8BPb+{!PZ~FS#Gx8q8VLzGNIjLK}z!C_%X+xM7wh6fpeM)tfnC4^t
z-#~eJ6c9@V#UnFWc*sIika2mHH
zkhe`vuE48pZQA+p|-SmFdoj
zL6y@=Gs?-DH#KWYl>`&8nsq25nLFagrTHckll2%i)#Wp1&r<$`^TRNCercg~A^jSN
zmKO~Mt=T8p?52^!!y8;gKH-Ec-dV#=65i^oqe0v!wKxn44pBs5Sg>|)6D+I0|6A_c
z^~36{i9Iv|OU7wwtq#*R-9ZaRjZbEtC)#R$1bl2$6NU*)_vtHkj@hMT
zs8PL8X2hWzd^{~~Inhtf&O+QQActY?d+JV8l|h6hv5)aH1R-4i{#P|_A=v=mr;N8=
zvLw+wrHIH&hR?9;?$J+RwgNefix+1@6s7Gd`;U9!Vez*zvf0&22$-ax?@4><3Q$&r;wX~4hXD0&Uor*OKT?6i(w
zAABA@e8^;Q#yE^wTF?wHrT`m+D(4Ct+gnH$4ofxpDVQp^g%JXWu?D(2rQ>sK2cY^P
zrAFRRm{E+8yUPhh{o_aDH!{6=^5j;EN153ip?9%$(?TF{6V=`DzOD@!FeRm;AyXM#
zxe;WlHh=z%I)ekbo!(Cv?6yLR^|u*og{L~x(kM7Ei=uR`e17we_DBRdux(oy3e&8E
z@KBhETB7a6JB>s+2`b8^O5>ox2GC;aOm1Um$>*FsTt~-UpY|?$ciGCSM_{X-z?TKj
zVbldLcpHr%A(&{c;SAj#o$0sjIIxk>K*1Go?%Q3=h~*H2}7smW1||IncJ8zBc5
zE+j@lqUaZCh}oGD9VUCP4xbfYcK7bs2@@7itsAfaQ3xd}#=%PBM;#pCih=GCkxbUs
zCf!KhO7+QLzrPI4a6^+@|J
zc9}wCazVl2^Mf-Y2+|llbNe8**zeO5;yV+aSaIUOv12P~P0h??3`2Pkr`eqG4t
zU|XasSCHPrfxt>daABF*e2jMs*(IW88b&lmR9OR0y1Sz>BXad2R3J0Y5RuZl5aGN4
z?1lyzr-#*R%t!N#j4b+|fr>tRc4~wuh=o0*B)
z-hk0Y7eKLt=o@jE&>B}Re@P)^lJ!cM>
zfv{DhT}M8<$-{%55owjnO4AcVXKqKF>P>(Wy%1q5E7bbqA|}Z@Gb}-O2}7h#b61eV
zvByEIKm%Stka_s9?DgTlUOkri-F!gi
z9Cm5FO-y~VvF)xs7t+!Gttj`v>wU%@OMm6bJE>*|AIKf!QQI_v>x4Ha8OtDhZCtPH
zNvI<5uZ8BtdSsKU91s|IKwc|7*pBOa&6;7aVlKMGn)37Yv!n;J4a_Brs=Ks+5*)1;|W9{aaI^242VZbI=>$H;NpLK>!m@u`Cn7wuD*9Yz1ef{Rm@fsSh
zVZhIRxF9brEn-`ITQ#NL#w7Ik?^9U&t!RKIJ2$}%+ydt2#hb!8eqir}BQ6APo*Fvn
zzt1Wf7vE`-^{rK7ru>`LJvwjy@t6n;>wdobE^y`$`J%3>>NjbOp{rq_(
zraIuNw$@g5WUNKNBm2^(e#LO{_=30mmo^8nOHCA~#Cj4U6|+%f_&F*m#!i6jRf*ge
zg1Z=#Hzf1F9`on1J^O<82klwjPx@2s(zrgygHah_g@)N}ZHTsVnl|JvFAm7+?V!K(
zl+=rNJC(&}9dl^^8E5(6(KnO#-%MT&T_64ZgUpd5RLhjARX%y;2DPUw{AYbcjQOzN
zZ=))|@%l`usGn}H8)I+H)E)V=p)RR?YUO@)hxXDHqyKB1LVOf7bwi8V1B1GJ>h`(Y
zsFNqZ{YTZ@o8#E?SiHE=f$rs3Q?K9dFXfuIDtlwV6SEUd>tb?ljR`&#eCE(m>4Kuq
z%fB31yGi8h%;co!PEnR}x;6V-r0qIRUt4{=@obBgPk$@#6JO5N{JjPDR5fQW(w7fP
zn)PnrN%acNThi)ohJ!2hQU>JJIAurVSmiZnm2DXQxXE)nMaPo8afkbEopS8q)bo|q
z4O51I8?=kRl{qhzIu{3*KNxZ>-{+|m*>3wn75%tOLC8Kx0m(x
zIu#p`;S(_!?KVKS{q(jR9R-rtpSe#4pP-rkK5utpvwXYSYs<=}t+r?c4jSO7Vmkyr
zdRy1`Pk;2uBrr)OR3U2IgiD($$j{_$=k@swfiq|vj|vH5n4SXcO&qDC+PZhrUX&39fkW^ppyukY0gnvqe9lNjo**Z2tC4aHZ-
z#Zn0`d7DE)%$!*dh(}S_tV$z_ALA-@5b}CW!uyUP;5SOd)@2x4@)S;a%!&)=E?hXO
zz?9#!Xpb^2&Y>wH>A1B=Ghle(*|R0F{uE^D5k1!Lnq^C%yqPKxUsLayKTT2ekKGOC
ziWr0)W_w4wcYrO1TaDFs2633HN6;wq7gnl>@bG}gJ^!qF19_)??Z1|MTl2Uw{MvI|
z1I2Z9(-=CNntF?@c0d=9`HDLiKzEt*K91;74z0`q7uY&D{lE|Tx)TS1!l~#%UNmO=
zX*~xsrcL@*K6fY$WJ{cZk{k+6oDn7_2*N*o{7ATaAi!YlC|xPqWGsrbDLH|G-CDn0
zT8Ac^$_=fL3pgHPBQAiZPV7j6%1wNHj~@MM^f8<_uk^+Zr;Yo;ooBYz?~IF6aPKa*
zkh~8$pjV_*7Q-tcj%E@dgtvId&#|Jq)HRY!@`*RKx3?#-oY4T>{p;USNo;PAMW=6}JP7=)JCzlB8~QSvWx&=Dfkq+s5T>6adh>3qB-r;m>LQyIMQ3F=0%
zxUhU!%0D|dUG5rlsW+pRaVXHnWWN(4u|q@e0d1MfZ}9Md*~Fk!jib|L`9_&E8X7;}
z2>&I009-pL=1AE>u!!m+D(T`HEl^~7m9naf7l%!nw25t{Z1{%nEQ%3IN|&!*-Tv*-
zW(6tjM`+EU>Ij;OdYr7IFd1|f9$>}hpm6nNB!OU=RgOxEAHX9gAO=@g&!IC0ZHX{S
zB$~zt4ID#G?cLmZ_lMVK$weGF()9Li>0^PMBj!!W!;`wqpVLT6-khl`#rXw$F=hRW
z-J3q3NT((r%t)q|i(CK`B9<1oU%YflB;W-@B@Q2cO?Ic$E}}X3kTmdJJp2_-i=F!c
zK#jGup#6X{Mn(f8qxH}XGsIKgL#+90eSs5jm78D^YR`;PaKYT(+uPe3V0aXI?xHrj
zdik>8+;8?B1#*g8(fEn_-wnChVTa~-=)*d
zVlqWGmQF!BS+=ArU9fxM($<*~rTzBk59-xzgMmi3!h!;R4FuD5`z(ROZ8(^q60~*&
zdmdIxm(&b(?Xs{u2b>$rCzem#geZtdjB
zlXv)*(udKjqoA4;V=D_XG9z~AkRgK8PI+m|P7zvEoYv_B1E#Edwu>Sd^f?E6J-JEM
z6#oQ|OrAP*=P)GUi)6&w8x#+U%axpOZu<%1Qd3ohpq<^(p{N)CV5AyG3gFx{^O6yc
z?X$OcA%Rip#s#A$BfMoCs{#-Jy!muu;O3s@F-1ikY`DPmJN^6^RN-k@EceQNve0t}
zialP(=GfzG<4E~lfhAkqLzG6zbhow9K0WC#I+!E&rsLk&-0;Xb$I4a0jDiwKk%#q=
zFqbRX#481ihhX)fQ6k`5(}*`6B6F*JvKEAbOw+D`rmLx{qKzclw5C@umaH^$kYE^4
z0yW6>+iTou4N+w%?hppmo!MB5Jp5L9F{c4zLz;n&0T>a6=a_cCZeQMOuu^oVYfe!M
z=Fit0H?9%I-O2aDs#>rh{p86`6UORFLDYf^Qextl0x*y3>S&@+>koD|d*b5F-S^XW
zwM?4W;qNU#4ix5}le}YGEKoTfDrC
zW3L7q`Icg
z9M-dG=cpi}(8j2%gNs>$V6gr3_nhpZymS^;R>(0}c|wvtrG)*6*4EbDSDr!OfEfUA
zO=>x;bc01j2LDzt4LUk}wEoMcru=Zylta|ZPUjj-@R0n#fthxZ%EjxVlE~zPHwqNI
z2m4@Vn0iF}SSO}Gk}$t1I6x?IM~#xW>^%CjQKIyEOWjKutXq_~d2=J@cYQJd<%r-L
zWSWyO@lkaymz&_Ug<8gCLyl46v(IfHs=TFa-tMJwZ`O={A)e69!V|&v6c7!qC)={I
zy`wpg`6qdV&EW5?t*m{#K7dK2t3g6c3f|{9wfURU%$YOU7H{5s!#0YgG4Q8*t7W%8
zy@c18lXKpVt!P=Mq~gNeu}MKkJ3v>8Bc3JC&Wl?;1@iK!;`vl5igh2K4umoln%*Eo
zd-R_*Hn#b!Oi+oOFU-+lb^HN7eo`Z#aS5A(FoU5xGQej2`-6jb6%}pvoK`hrv`i3?
zr4SkdtV`SZ^ZnTmm+i*ARg#q3>OP#USW-e_orlND&F}0oEff#RI(8m+L(`b9T5v)k
z%B`v@fWgMc0)GOV+i6KW3=ZlTaQ1ko?^Iu0#^#M1E01;tlE>6W{2f+;O>ZBp<(y#6
zW@I<4FXXVHJ|H*_xZoe`#<+`W7KP&~bXdI_U}8NfpPYZ#6)1l(x~Rx>7xiPOGW~~x
z8-L~$*Sg7)NA+vOQ;$%3vBzT?zgklA9F&fjU$NHa{8j4&ECs{?RRBN$E;|K<9i=bZ
z@95vwG=|vsX0=oKK`!$`2&&go&%Y4eE`MdiHygD=>@o`&Z<+cpy8|+6U;RikaEmOC%VvKJ8vhmku&hWz-KZJS#%U@E)8s
z7sezTyTIeCSKpmuhCJ0C#t&_NHJs~3#Ybgj3K6AnuWLRI0fU6ugMQlv@bhr%z3Vgk
zS!{RC6@=-~8Aob&CnmlG^8E7eW3h%_@S^0ElN+HEOvUXgL_Z+A=d(A-*UZUT(b>SW
zoyW?wqgSJU-qQCp;HGoIuhw?*R+W6UhVT!Y;4q%#4JVK;V|Xr=Fpx{D1!;eAUq
zctVMb1WE^c3+Sg12{3Psgys@AH@81hMUyB^iThTCONU5Fxifs_olZa!O3I68CFaF2
zfKteIDGrrveDZWzoQf)QQz7uc3?~7Pjvd=Ee6`5m<7R7|&!finQ|)giR6VGy^esw<
zw}r6wn)XgrxO`hR
z%!q!1F6%rBGwySl6Tv|VeTEz$&jULp)6;vPwr^-aqLRxVU9up`_An$oJ@dW$_j693
z6kocx`h0<@%8So!P3>583zt3)?Y(+&aLnxhvGFKU8g50FSgo5VOu>K{$e*vgvrJz-
zf|`sd7GXM%=?+^ZO5FL@HIeHr+whVijlAP*ov9Oh8qIrk*@EQT4uQ(z2{;ol={(u?
zx#x@9%Hq%Xzz_y8<5wB6%MHTsX3at_ujSdWM1%y84#;GHi9c3%CQF4g(0!Aj-3Ul0_gWgac6e!MPG5CXrdQxfCnv*&_c60lA!^FqMrBAht{i1(dDskdhOlT82xM~Mu31b<5^WZ^$zlF9ddSv=E
zTPH38SLqe#0cKHEIO}@)oMQr#OZY0?Dnt>wB~njKO+&2Ga6bWNY@*=Rm148xWSeJT
zU^DuOpYLl5K>nnD_7olpr(Ex662J|dV24PNRsYv?8w2QiY>^Qc<#0%gnq%9MwoJtW
z!Hj(thaGhmTPkdZZ;^_ple1_Uavi8PD1aSoJz!f9B))ALU(sr-!WEfg+M4D=FMYqiN()L(wKrdOK+7j(Tg-*v2Y%XHCad1s=xEuMPkVQ59mRT1Cf^74a8jYJp`ysb64
zzG#fGTgY^3EQNyxEMBgz<4kA8jzg?`W`{6^{NDAUzf?0Xp;Z!CvB=0#5v3a!Ah9t@
zyhj`bo9eqowf_Os{a;^|vI#nFb!+~yz5`R=N@H>&Tlb=-Wp
zm)gjYfiXe(KSp=@+Nio|
zO?rcLfUVV;2=i;7ttLFl8J=gJanwsMB)qIcwe#(C?O$%Yjt*~n*_5vlvu|C^3jYmv
z{zCo+)*MQEPDSP?8>p#nt%(cW@zbIG)IWwMMFmIy$X7z1bTMv!_|Jdr0X5%aJ4Zjo
zui>5lN2}|GVyR3~!D8cdOGG>Gl%?O5Cg^#`F_)
zR$b~7ohRraONy+p6lxKX=1yGABiU)8c$*kYo-
zp!Sqsjaj8_>6e*y>#JyoedU4KxJfnR@smff~rrj@XE;K
ztjht4wYC;wFKMHNCVEY>qMN(x({N0bNRoA2cnhdnu@C`1j;U1=FGz4*ptOhkXH|^yd2LAy5FJuc26HvRUk-Iiw2?Q&W?aX@?OM
zlrkt)c7cfuow2Rz(W97tS6W0K2uejN`#}_w)SB9BEw(;_?4cv=3dNkGn+@6m4v=d#
zolrpah*FJxP|PX+bOwHa40#Ji{vp!3f3HA2smtKt#Lo#3Dlys5kSv3+TIMSFOPeKCHj=aWa
z&!qeJN3|DwP^k@dSfIZ@yvNS;H7+io#jp6Xtcrv&{WTlVS~8~84uga7$ZqDdkSXv9
zNI$hkhZycCOi!i=V@d*P%Mc*Jb5^>!jqShp>x!>aeMiX2$ZSCTMM>L{ck)sAkVxWc
z($dI(tm%fh;ZjKnAx{JlFo&o{9b&|5%NFp;*V9}*?Ku${p~YE>_?$uMxsaqdic$fnyeYXx83^qcm<+0c
zoQ-eI6BiAXm2CxsUS)5Ozyt)Ixj2B}N!}OFBdE8bZRgy>gkU_;T6Q1c>516wR;Zz#
zw^U0rRUDWHY!miwYul$p2qaWNjtKL}Md3wxJ?_jUBE>Z^8qA3GVf4l(xm^G3L)G{D
z=8h07g_hOy5nL7#@C;Q6SF>c;?a!iQi?tEU29I3M$4q~uVHK6c3lE+z41{oS>*=uM
zN8~Jd^VAljvhXo;B?f>?Q^dMcI(=8lBOok26_ijSn*eRmfc)pc8-}ozO{iXU~{=&
zRu=c}{Gf7iT`4^OV;=px-{QeE^+Pv`s}?Mjd0p9b>$2u7sa+8f|IC|rzERuDWAW}k
zmcyUHj&+A8y%{kkx88m7`oeQ)yaD~DPxl7#qKl=`U8VKqx9^N{<(=aa^WMO+c(rfc
zlC-*{EE@%t&u$$#|AFSN-s%32RAp!8=8kH~pL48_xG3mEYR!#Jcy55cEWD$I5rd>(
ztY?1$Kw_|NsM2V8-=4n!7s*dWRGDO^N>xU&j=Xp>PnKV4F8?rcV1akMDQ>SqZaJS8
zku4Dh9k<+`FNyoCq^vCDuN2mq$99(#v`_5tIEK>n6)g!xm=%u(40yHjgNEb@l4UXD
z0D{}pY^*8HflPArIpOwGpExgVF-DG|t&vp0Yt!U4mou1dQ}D%*;xsKq
zMQllBn!?ft-*GHsgW*FW(PQOcAfbk2V?GBijWWppq3(g0vhwneNV?cT_U_(|kHomB
z5$7A!7o#tEB`9>w8wLez;AO{EFHB7#fk6(`gc~cM&7}KXE8JgViaVa}fBA)Nh=Lw_
ze_OWp_r*qf1<&bndgqS!pm6(3a?|^HQTZ_y-W5}MwMI?;$=yG+w
z)}@Be(cbO~K}v@2HvT!Y{mzAzt^N4@m`eBgo0JUykgjzkmnaz`4Q===OhEte!DQjT
zv*IVC1*JW*=qLfbTR?|H#|?xcj`r5|>jlIp
zRQ#N;ona+vTtS!rD%@6f$#C~J);Zk?cW7V$m~0!o3+;xyo>$>gbw`=s&WRN&GUT7b
zw__kgzr*;P1u8Pqq6b4lGJY#Ew@4ww#%(DH;T5XH+GNNk{idhqWaElKy#ntxRA}CQ
zx!$thY|^O0j0(a_o-$ga9lUVoL=QTc+vm?`JsNtcm^EFJ*K{;pu%MUc2pt1=6nYd$
zIy%K1DArMmWl_C`&QOaemGHwyqP<1tmN2CTx0M#`1HUep(xOr#FwuhHU7Xs#^@6i@
zUoG17E^7PS9Zw`Ydt#eKX3Lj(1IEE=vxsu$9Am)4GDO$B1eQakWUtcKqSzB~Xfst?
zIi_$Dz+?L~F)7KlHZM5gBt9&V!jf)LK+qT;vNp&6Dm+y^jN1yk0V#%G5t5DnL`D`e
zh{SV#^QU%(J+VBOsK{_AobmjqXwW6vecqIWfdPW!m4bp8H1sMYW2P@Z|9xabgWewm
zw|$UGfK%Ouya&Bn{4loCVwIO?GOEf}mO}yo)By}Y@InaJ3HYHNJIajH$s@osfsEhB
zY<^sb23Lq;8!(_-S8G27>N=@`18JR|ky=oLQ9~|e-UpSgii-I6EL>gaS*gkpI&zB`
zD37}C;y%efg^Yp{LV`&%0R<$0Os8kptxX{K3^aU1CoGgJ1qHhT0%qmdGrNxm4NzM2
zhX&G^<)^JAB@V1Z*ihBfRk93(WzB=&H^vU!I9#e%jf>H7kcU5GqXfA2PW79`ohRDx
zjP~{)^Zi><$1+PxFA6EXV)>a3H*g+}>=Z3_mAncso<7tOCC>qJ!Vs~5?`-{J2GhBA
zSbL*0mN-|}_naP>cc=4$BxO
zd-d*hFWo1zl%h3kXOS1fy$0?ux3rYlk_GfY4GYo!^Ul-{>=F%p6~BM)zJ2$5X$zG~
zfAysn$d%FW(q_Yh>1a%{SPI=Z4w#8ZB<6s=wE<0e{yj@yqjZTv7BW~P}5!W9)$fMOhb0+GmBaywQ)6=`I==AITv-O38_8q#d
zya2HP*1!S=KrsniU(soAVxJim^U>#mfN+E%0yvLZ8~%>iBLR_mT-$noc@Np5)2GLl
zAe}jK1JG^{59;K}>$h&XKssg|uGVs-j)C2Z`?VcHpT)3R_7ztTJOso?t)U?S-$}s4
zwi!YwCj|Z4Z!^jqH34~RvsU#uaE-$Tn;Y5z6voHo$WH_}VUfTSB0*g(>%h(D1FvaZ
zWqW13B!P#A@qFV$6p$$`t<8AW*dqeRj&<9z{BQGHBpaoaZ2m1awS;^i>m0{EfhEWs
zC{ZXYFwk@`WL#nW2i63WnD_`#55^`Y5OBGfnF8HJO(fwbjB^{^3^&KoipW!lm0|5-
ziM2i5M&zccUB}ncJ|8{L@rRrieMS2W9r~fM@rLq!Z-6LH1UC1P^ADRcnk>MsDhp3W
zHu^0zV~B5+#D(Az!mfID?@nACP8KOU{`a0>Oz`*T0*jK8h_uA7#(W_J=;sGBQyD6T
z+mg0G@o;HWlD>%XkXx%bLbCGnyKRQnwN6t_Eq7!CT*?cMY?%Rr6dYvnCec)|khlEY
zHSACslgOAyEsRx11_>pPEK5E89Tq$wFS1&NR3(+p2JwWM~`hNNJ
ziPrQnl&4J^x`MYXRv{fYNa(RSTzP4PWF=(j>^nlz&5IZIljh{3W1jZ8dafl|()RY`
zZT=`fZncIC=@f!BzN%>=6kIul}KFtW4k5UYS~w(VRR
z_f%a;de4CaCiCa-V;mE&2sMvDt1ViTuqc@lGLJukZYPd3DhYQ2Ag#C`cSN*A18!mk
zB?I?nmY$w&9H&QO;t0`+aQ4v(yV?$PS@kP
zbt}o*LOL@CExi}Rj<_#zJ1z!{&p${c-rX5L)Ogp&k6(258s!%G=0g`ree|3Lq3Gx+
zzD%bJW5?M_q@~HGMy5mAX{JgHse<8nr~uU7%)sof_$QT;9=`G3h}XZsFgW@tw~i8a
zb^7!NmKtR(RZI4boSMmd#w@wEN*0|hNM{>d>#o;Cd4TNK7JMk8PLQ2KUKf@By^0DN
zvE$rJ8c)PR%S|41hZk26ry-2b
zSiXMp5e^9SN1;l|i<0?7xY#e-t=qV9;_QX|62x_lC_qV0n*d``UvJYory|FFVeRPm
zlh(NH`Y@cp-sOY{u-_FQTdsTvAA|VT2V+ANb|0Ce;5;~B^Y<@K@4CA6w^#{%D1bCv
zgGx})zI`;3baU*BVhukL{Sml^ZD0`ru!VK3Sk9qU4i0`r2xr=8j>Sy^ZSgW)w>ow4
zA-F|c^|tykJp&R{R959+3^U(tpd^xMx{F7;ny_t~U4#kv@O$a6zs?JAbU$eDFR=q}
z_0!DpUGNhP6Yt@InBP74_p1FA{xN0I#kP%otDnv2y;O+yA%sh)6X{V<#1yB0p_1vm
zs_Qg^fdNdk_5S*136jU+f_x(C;J-z?bz^g;@Mg&3Ax@yCJlSc7W-D?Y*y>rslI7=b
zUb3VTtf6Wr;-#&}#!By@^A^IOADpO~Q(SiW`NtzMG3-3IqE*)q#4LM><9Ga)R!~t$t6emr$PmV^P(+<4vo9nh1Qjgr0?AQ)yRS}MOd|N=7aAfd
z{a8`*)BF#2)`Vga=p8g&&JxAF3&6`cgWG+7zNch0vZ*-ybQ|L=B
zjud%txMjf5NfPF@b49vm6GUfcmoQ(s%fzr{#V$)4I3Y397LN}o(%AB|M3*uFO&3CE
zd<4z^h)j#6`K{`{#PKh3?iKA=_qlPQk9?XtUMZ9dHU~1IFF_eHBSraAXa8MEPGy4}
zqlkP5Cryuzu0@Fw@zDhLvtgZQOsGlWE_{??_o*~HvVSE>(qO!nc
z%%Cb~7~5pCEtH3xEWU%K@60NfLy#v*n&TZHX*6kc48cJ`s_N>b@!RLF2vveEnOl*Q
zCoZj0l3lvX29}VSsWoB3JQI`E{cPTu-Kd#xS5=}u`q7;F
z;UaZYqo+roKmFU<)brDP`gl*h
z=<>_;yu|FpY0JF!Dp2`#-;`kUQlKiy$E8snjG0CV8gzBVfQIQD`)2CeObZuB$HO5Z
zam$J=Pq?Jsu`7!l>Z=nH9`4}e1llLAB|fTv`Z>Pbkt67;YVmFLt`9_ePARtTIlDck
zc=2ndLooOJ{8p{`aj3({QzU*6F557aBj6~t9S1hY6Z-1t@NlY@YuhJ51?=48_dyKn
zNptZ06UWSZOwBIxns(Gom6c`RFtlAQ!>4l<_!mBBvMIMARhnm4T{I*_VJ+owbC~h!
z7@Ik;D0FKf`I1oyIz8^?-(C4FIc^#21=AZnDHqPn!iy-+ob*hRLWwa!81>Hb~JuL1RfH|QIx-zi#o}*6lyly5hFD63)SpOB5brCpx^wg5^A+N?_=#agh51%e
zjWru=##kQNzH=ab5SU1wveeWjOvuRjfeW8f8{3FZm^cwrBi)qmCd1X!i%)dZ$jIK9
z#+o36Hzj4Xk_kgN0LQ5#Ay*!%M_SW!7aXJ`D(Yze!9x{UhWCKB_N4sf{&Pfb$0`}Z
ztu%`$h(9BiVB>}jlD&JoIf-0s7Ef^{JA`fNwzF={5-)?$TPXH#-0%r2ik;l!|1Y3K62qy=NY29ioE0heQ3@docKASAi$8N@#;18NjW-1xMsA#C$4C#@Oy--E1g*M#iOfX>h2k4GBrarVj_yAri^t2&
zq&x!#Wa`iP@;!fp;oA3`%JtMcC~BDZdi2RRWbg#^N|uZoX(apx?%Kjw2^(cr3sNckQY>d
z6z>FiFq{^R+8>9c#J3)MQG!+hjq-OgM->n1JhbT`S0dy1(|ilHq?}GCHh2zKm>fP)
znf!<=16IIC-7hK8tF1{rFWv5VV5vnUMHv%J*!wvd0BGv4-yjOa1B#Ao*!vv{yXk#&
zm%rs9(1;ZHQ=D^ezP^7{3uCC(Um%@Oi5l-6Qeq{obuVYg_-$8qBzMX8qrPPIa=*zm
zTCCRJhd5(WFoX8&DNp}&#Z6y9>dV`QZaA(HEP<+2w0nvkY)G}R7zz5|O&`Z4P#V~*
z%3JgBJ<(B<#*Onz)9>_$;Je--s#*P7ByIISYh7G!z5ifY?8@ntO+7FcktTg3&5Tgq
zFJC_7{&WBPH-)Ll1cpW)I)q9S1xb${J>s{Gc6rZ>55x?oMFLGB=Q_JQfR=DVQVj~@
zmMHKzW*KZo<`<(bn1f;Y{5bnO%F{a`IvdZCBLct>)NeEqM^G
zhqv{y@+LZPb|Dm?xMg{&H}BjL8lYM67N8Q`Wn|QcU=O(Tg9aDaO6k?j-`%pylpVA=
z`Y6&GGErKM~}DY
zcaZu~{fheDMCp@u`ZOThR&E*fE}b0N%FHIqF;2#_V6;eUnFJ6K#ZMJ&Ek^>b7*P>l
za+7%>5v#}sv@r&Rr$l@Rg(z=3abS0}W`!P~ul$3XK_e{G@JO<`+a4aD-7d6s&6!b_
z2%L&#NW*g*dA8sO)O75+sFSbI_bw$B<6_m#_0{Rr-rSs6eDR|28gRg|;Dq{QsN#M9
zb`TNvl1K?2rBf_I3L7o)j>1O#l?-27DK8&E4~&{P7iE{qlTzr^7BZDgeb|BA(cX=MpM$Dl!#P{NQEdw
zh0G0PD?*v4>iyh%@89>?@BU*y`;qFt&g(qaxz@3c<5=En29}O)-*B0d5w|T`!l-S!
zzTBTPv~LC~e0q1}_^`U%7@KnNHomox=ccAr{fh_JkFOBvH$tPY+SEVazRx;y0;z>blp0bPZ3@=oE2XoRrsvCsrCt6
zR7KE{-5eeJG#y8ILB0XdL*b9I>=#OWn1;QLx8Xs@tyuNe_jKgvW--SAqqsvgg^MCE3I-}J%L4Drf5
ze|qE56EwaT)%}#)&J;BojiBjoRo2LfhZDWo1A@`t$08@6{q8p8HF)vHO#*jlz>`nt
z@<9boe|yNW{nvS}yc{{``6Iu@1C9Yij#5>nr`eLfBN(Cpw;Z7xnRhy5b(#7dTjb8d4=(uH5@KVb|w{~fhT$6y!!WA`tcOIMIake
zoefnaD(Wcs)O!J-)qdg0a~C!$5Us-A6ZB+AL>hNXjCq}`f8*xO6_-K>3gO&*>s1Fy
zth4OF*+>6sYFZ7W1ddgSX77OqZ!_!)8%)r<+F@Y;2h=5?E6}f+6oJVC0s|Sp?j07}
z6ZwbwMF9mJlm@GUDFy}N42X>qy2@jK;*Ietxt^r4qD(IfKT7fXIcuZ_-m4
zGE|y;8@fKeW?Q>6)c;lEeYEs)2z0Q%ll5@8N9<x90?kKRxiH6cQMF*;z(+@
zVszXH`MOR|salt!^uA}#pu300y&{++QCNBK^Q5$Nks9B`W5^}nMLt~C&^C7hV3nR;
z-SqLOd3mwCnt~;dOP&PB$G5$GD;PtFz_CkjO1E|3vD-tO-Bd-Ax=xh~=0*UJwJe9leT^jK;3K*hX+rf@7i6Hl-S>);@>*vKLTmNPbU3
z@$cv>WYF~ePuMiJ<*fqXyx+gT;lbN}E|OByFH~VPWE^sxec)CCyeEI5ebT
zc7x3OWnSU$;wIMim*{71pb>aq?19{=7aH3A-fKS(HYxYM`?yDsZ9Qh189SVw_%8|T
zVCiAKwVK8;M2E(OPfw4?2p*f6=;V0sLhIFZ8!fw|56g`=d$1g{e}qbbnX-%#)A3m#y;%is-yiGogFs(Ms?1^G{Da
z`SA3DeEtv$E(#lfJnVn?H{OPVGV1vS+71PFb`SCfxVXqam11Yx+uMW3QLQNNQ>?TP=HV>(ymp-=O#A%`y-
zRe2o$)TNo-`)hqk?{=BTJzB9)%3jnKaCZ^12%sY>uh+cB>(YWSea9Cxyn{$raj&S
za{mr!68yE55#I6^VS^+7S(LwvV8vhE;&=W$eD#0sP{vV)GDYKl9sD)K@xQ!8!yb{!
zmN90X$h!XD^4v~Qk6fgy5MiYDPeuLrCs`T6RF9fodo2EczqZy#zSoxjcvcrGMqV2}
zTbe5O?-)E_KCTSimV;rLrHm3T&f0*tmAL^+i^dGnFODSNJqS|5G|L^fGzqS=B
zi{hm`PH8GF99e*|K@7jcfZSVJgX+3;`
ze8pe$I}7@(?N9fa6a#xX4E64dUeh6cfC>4_e6CD-jSmAki;(gItJ70Ha;*sx<9of)
zCr?UVBmbd!E-f^UZsR<^-oM_~rr##@^f`jpHN76%i(^4%=K8KznaO06vg_BqFiX}C
z`9c+%%{lQAbm4fCwbbi|GmRGVO@?5X_nKjT%}tyZo-6-qE~O-;^NOLC^(?J
zlg1GZWlGy6=tLUQPJ@^YZN4=#+#`*-(mip4%(~;UMJ!a{
z{EBdKW(@d=Ab+2Gd(GmU3668;e%`dFRR2{-w1|+9paTr0fncVuImuX>^++r#sMwq>
z++N&R_(I~}lZ*|>PQ>TYb~SVWdIrBbu{?#E8|4VsF^nX#R!;&FAal{Eme{BB?PSWk
zyzc>9D;|b|p;AfeJ{30>THO9-mwLJZueg31dO~5@Qw_n8D~w|cS3I)0xnIF0U`S1W
z(c=$|XPo}l|J$TLhjf?O`}#7TBjIjwPOP%D=n=<@PlVg@TLLp)B%ho!`Q3vTZQB3O
zYC2R|JMh%oxm--f(u`2P)`X%4@H8i*^vf-UsQ_b*+K*M7ulXYM+GmYRxXvDkglEni
z=nQU46mf8P^sW74qCvPupjl9CmTphulj4u-dtXu=_iezYob|JHzn$6TKG0Qr>;d`3
zLS~WnvaRBwJGaTSDxL^_cd@&HA-56zC3cOSQW76OYW;_4Yd@~}w(NIcptR8N!Xblu
zA?L7)HH%TXT`}=x`=O=op2C(_T-#r`8`I)Cf%D6ebtcpFXRt>dWEH+b+Zs2Ua7-?=L+_rHx>tRpWe99HJR|3ugvuSv7t
z71t8jhJBdXh>%fp^%M8$^_!&<1218id-sDewE9tov92{S&nwr)4E8~eLV?K?(Ec~6^){y>|_q!Hs2U>T@s!w
zoLD%%f%=^^doHYdub@zNbp`Ml=cuNx4zUnPF}s^|S;&}H$6`U<_^nMtQ}gMSEtkN!
zDMCT5l%{UjR*QEnJ?PC_wxDl{KXN2;&Y=G->>hC+%s*5dvU}V<^ZM2lCtRt%GsAS;
zqU(?+;FUnRdi~+Um38l3RAXmKoADUDG4OM;%9NDhl4hQ>J`Y;BSwLBW0xBGKj=xX2
z0kr`C5yl*aDr4tMeu-!ZD2Fzh7pAs??y^8;uQC80H3qc_&~h$aDtWjh(ACuy{XDf1
zSu986`B_XRq?fU-lpmHcLm|BZLE3s_AMzci?1OP}^d#`1!cvhMPg>|4v5i;Fn9PJK
zZzY3u0z+O+O~JT268D^~$H29adeNH#33nEGG+@_t1`^RJ&{cR3c*I-4p&3KCB#<1@
z(8hp^^?r{@t&bdeil@J{Me0-xj`?5Hq1kQpl8i7N)g|XfE1PD5>;YBdZ0q9aER8W77o0
zZ@OkW8*U4jY^e}WFo@wLzkN9Co1~7*cy}svkdSZK!`-9gVO<@yw_a5)xO{mtn*cWu
z{!vi4`rBvokk7(D03PcYQcU3|p#NC5)0IS7c%H5iV3quJ1{(C!n@We{1638IBC01r
zMoKJf2`pl=Wcc$PejewX5GMJ&n<)7T0UplX?r3r;hcAm;i
z`ma6m-oP}3<w7nLCg6iXM|?47~(FFlO{O
z<~>LC>uPI59an`a10Q6OXdt2Tq(&Z5uXYb;v+pyg)z+BFh)47o$RA
z^N(PG&$%(hWavG@6m)UYk>qmTBCID5#N
zL3Q4FPgG{Y15Mv(H#OE;dOKpTgsOE0!
ze#lK}?`RqHzwPc|s+O*+@qYBA>Ra_kPHhN-gKU|i{##`&)(5=X>ppnfE`d}b^1$+@fs;*dd+4N%uh_El*fpoP
zdv7q!V@PnaCos!|EnBv5Q$4R$PaB;5Yn|Nbwpf8Alzu&|D8f_JYD@pWd%iU5w(Bg@
zeF{>D-r39a1A%Y$UCmFuUPoATk$g}h5w@cYcG5#R2}umCzFcL4OF?3J_
zdF7&3Jo(L6IIH2u!)|@mA<)iFif=n4x2@-~=cNPIiC){0`|b>^IILRf($M|(e>i%B
zjYa7Vk0XU8V~&-;O3dodz5^IDIq@z@ek#y#OX|ZgIYk0-~2N`ejVGn
za^EF(G8Br_@4WRB&!nXAl~35ZzHV;TcDs<1oh@at0+bA{NCWYY5*%uDYg^l&Bplso
z$}xT<;)=H1?S_VKQdddg#gyIxn`qIOyn9*{G?aY6Ra;t|^1SzkgoM2r*|MVlTpq{=DF}}&_zd7OA1pi
zFT$RIoE(RjCq!eI%A*F%G&U|H?9b13>9by=EZq!7pdbf9Y69Vt*9vru_b0zh{C8$k
z(JRsmW^|?E6#zI5?QO)&Ph4YsnV*SlR4lA4huL)mFF>-C%>dR>r$(#KB(>2?kO~g{
zein8oT_-MT`~x@l)A`TJZ(|RS5FM{K#=Em9tvSk`N6s;-ZDS;p*!EG*m5qKFgQis;Y{A7W@3v{IN)0nW0JthTrzs
zPjdHBA`)@9{Ud*LA+w@ZTb)mu?$s^9^)xwLqAF3oXDaf|=_1us^|!Ks>qW$#$d-c9*~9
zor1+;mVCL=J5O=wP=PO&mR5~P5OlP}#6(_z_{GvqDgV3APRs?#8yFi~xbj(FiHnIB
z_qACae<`l;=XEWDT+C9VruEkFAFc;BiFO@r
z)IQY~Ic93~*O>5C4dJr0E>0`T$=ahg;P%foF?$ul#R_{?MpQeG*Wc~1EXOKu*OmCa
zG6TedZ$)Sy%(?q%M8dF&^S+mJdtUvv?1_;~QiXfYp8v(1LwA!0s{Cx|)>oQ#a`L>a
zmc*E#nz`@U$}P`M$eH$5()JqPhW+-FWKG@keoW;gS6sTv8I_|ES$|m9RZIR5RkZiJN0^X6kVJ
zr*g$}qqlmz%g~=29DiqV*Obc(lg9kMDBk%#b@_VpoCeqP2JcoB)j#pn36^!&Jr>`U
zW#VXfy>wHFN5k(EG2cIUo_o2tNHj6ZYLvZfp>lu!@cqx<8E9IjJjd
z{m9P6I3VlH53@7Hw(flrXNJ0k+slmFR37JMW|g2uk9w1_!=u;-vo9C&X`6F@4!bpC
z{ocCn1EJ~*c0`;R7rSMKO-F-4Se9Yit+SfV-=DbY0-e&7TDYHD=
zQa+8s-Q({W;p&mdY|k)ZB~@sBpNFy5hwirgvTU1nOudh!LZ~uWa*yGzE-vI~G&(uPXH-u9rsCtjsKpug1n(8(^P|Gm8F60fN^nkBt-9Of
z&)tJ}Sf5^HH^sY;hiN{zbt|&Un@OPDZyHy=)mK@r&mW-KTwt)lZ{l^2>C!6o)iGSg
zL<5b2X3dtz952t`?A$8CAE9iQm^vQJ%K
z1-%<68POH|dDN|~rtXiZPdd=?oqOUNs*GKsiOE$0J|?vILJCW)$E`qFI`iboXqA~6
z_Yr~96(C_H0Wi(D?>Q&jNt=Wa6Ssu{0ppFOfk6Q7Ls!{r%i&dY45|@q*Zm=vuF_nk
zxB+`4=7>-9@$)NzgD99f{pJmpVRE}kqoC1J3!yC&J3H?19io1o(`g90pMP%79JI@9
zJjzL~U+6s#>*_#Qj2A9cig3l)$kg-|G)dxm%KcVqV@w-iVB&V8G7~;2Z-5{1Hxv3C
zRVXDzrolO~zB@!3s^wYA(s-Lm
zi&FX@IieubMteiguU;~Vg+x`^%2U#8rX~MRjmkG5qC*~~84rOf$4T2sW%%Fh6WkB|
z*RFLxzg$*mOuQA}Qj9bQM*bO;37YXin>O7$JmDr1(!iXwwB`iAD<`wEehGc;qnVFE
zk^orJ(>qb)Lwk*D^95|Eger87D=8H{_JJp)Wn{oVz#HkXvZ!7{F(iB+xn$2K|Hl#e1y)y?J(oCAOz+7536r9uGT&
zl;mp8!V%H70&{r^WVXf>+82=d1iWIfF-sBd)`Qy40Swl~$C~jm7cO1Gi(6etKd~Nn
zE7+EEvQx)(@7~sf3Yeh+rGLw^qAkBrY|`C50i7DgCXGgRyO42yVc~@O9-3>ws<6Lk
z5v2^A2*gIkhmUgz2B5WkWtQyyNb#+03o{V-v)pSVM
zzP{RDoFiL@V(*E=WJ)1EWF!nn+~jum8GQ|!3O6J5JiO2MQoee>;kENZo}ws#ze>ld
z=8D=$2f?s#dI8vc;j}~9NQf|X^GN#hNx1yFhfZ)`=7BFTYB>jd;%Y@wjvLRe3hzygV_PLp@~>X&8#0iVc8X>rfT
zl*C=^ixqh*aA$~Dz3p1dhgU-6DJ3=bUI
zz))d1kC&H>wuBs5V-iINn!1>F&lBGQhQ`wrqt#73ELFdL;@ENmnqU+n1~u@X#p348
znzaJ$GEN*3>t-`8_tGU~tF^6PfIJPs;Y#9XXYlaP=mN9gR0lG6N{x`)&5wBd*IzeY
zb#NgFOUhS2co6ve+e-b!zV}1m?NnQXTtGF+D=zNj7x6NgI#O3xr&9`WeET-@p56mL
za1KFr|>mQ;A
zU`o4MMG0dX0Py-L>YGV#XS5sK8ny-hH!dLod}@Wa_qYD*JvW;rn}|p$
zrl+L|rR@s}D=7&E9K*_l6M{<_vTS7HyezD_5^cV>w=>U%J4n;%A-TJ_^w)!|A|w|W
zf9yv+$!$RBSy@38OfzPPB-$uVT<*aBfI@KS5%nrIH{9aY)$I@@q(#HZjMj*-l+@_(
zL|6LxO&T|DFsf6kDyV*<=kGfYXTUQAWFjTh`~N#wcrM7yvH;`a;B9PBACfuu`0;OO6dVp1N0ZFfjqf%DeSX*VMpi6^cm
zPMkd1%0_xQ2{%a4Ix~FDCL;?~6@|@EQe5n=G5}CcjgiL0Hf_06Xo;%4!KZ%z{0X@r
zo42oTps1K=cidvx^5x?XTld}Z>dhNK)b$9(Rb$1zVNN-Cf%eruuXCQXQJqn;!A&?6
zzEnp(r`AZ?jCsZ%r4V85L<}=~Hk}+oK~BbE5J-Xzr2gJ(LNU4EcQG+bc_-eWyeUMU
zjDUzP^AHffRc4yxFyLERxe);hY!j?NZJF={XM4}G;T(pAPQAm_-~s{x5*3Av+4vAT
znAq8l;3e4#`8xCD?ZFjr8RFuixyqIW#$!FN{Qh9@P}q9SnhCk*Insg+AS+826XBZ;
zA3fUgK~2p#?lzzg5J$P)y}sP^3V~paC0g4BD{4i-Y$5*`PhLp~`6YkMZ;|
zG2;=T{D{2PG?Z-)X%5Pj(OHT?hM+f;1UJ;zAG`SqG0C-ChqQI%1U}zf{A|D1Tf>D=
zZz@`>Fan50zf3B$XF3U^6-Bj;%@tIp)+J?kKu|tF(m_`W-BKnDM$0nvP53wz#;0%I
zGqkT70tH@a)YDA+ss&kGpcC6v-Lc>ty4O8pc
z-NdR)U4Qihh@L_m!s0Xw=EGyar9XPhRj={EK;v-KG^G3i9f;^T(gd!=M0dtMz~-Pu
zbJ{>!7BVhs@87?FYjr}bNaqJ~I~mQ98|@O8lob5-fri%tT8*$Mxh^qt3|HO>Aup^r
zv~06mvc@qeW{i9!7mA#8*xGyEJULO7Ue_Pi){aNRGG)q7o~(Mj>A_-99_-=2xGmw?f_LMvcDJf)!NE|Vb
zn>cBbU{WE;d&9_4^460=96+#FLg3M?2Nj0K`}5`Qv0h3b_$Y5IX3T(gR5p-lR9Iy3
zU{2r8Ls!Kf>Rv!JnTUWpr!6;Y+!?5N=p<=h9M`*ERW%xS1xV!{r{Ep$9X8FS!&E;J
zB3B8Hsk8JlAzz7zh-e+LNIZu?iSuFWY0elWui5jkvw^M2TAEgt&L?u?pO280kD77r
z+-j75zb6_WDjDoorzGlVG0ZxOWcqQvWP()nE&^De-|BRiK&#E|KtwCOMtq!ywB05n
z&q@2&kSODsfaQMM=3uQuXNH*QDV43Y?v3GK)mWy|FZb}!!L-zQ&~)2XCMi)^-@Jc@
zjrM3mL#3w*JqqUj~_1Kx#w>@EVQz82k?T#UL_VOWe&KdSwd+K>$?>{&NoCPQ^^mtKuM^
z-^zadz5S))!@OcGj@!wkJN(6rBlSq~E#aOPHVZiqLlqS{XY@_NgA{*6m3bH~u#CZZ
zXT}_ltijh9-mDdD`T42fH^HZd?eIq08Ecfv-`jH$)nLJ$7r;S
zc$4dfxioHVGA79HF*N>bT(`QHYko5uo9!=VU9CmXf*ao#YeMK`z1sq#tq)@_|aacA)MfLUlaf
zT_-_qD($=1BKF0S-tUF+cUqoAnms)Cs@S(fzDa%b#Oz0TrElA4YiC#1V(u9AkrmsV+wy%qw3Br{;i85>-BS*;vB
zrF=)X>5s_BId(Gkn;(q5R_LqeI_~_TU=@`9_UW^ycVbHX%8fFoPUd+mi4}@cp>L6S{P^d(*&P7HoqKT!-0C7BAJ|7B}QMP0?H%|{nd_|&1fnG2w_>^W@
z+!Ac5Xf;0e)yr?reai4n<42%E)JaHMY+oghaIzolw=%#k)K1Rmw#W7cT<2nAUoqVi
z&^J^Wk}d~9poUt}W|EZpU^?AVlg4U;womcz^1L<;bUwTzWCdLWfgJ)>0q%552d6F&
z3-7(W&qoX!AJx}yMKR3Au6rWk7D=J?8
za*FKdM^pdJXS|bWwDRJ54|AdeA@~HY3IUPb%bq$M`VfKRda0S?AF1mfV7v=#IRx}}
zhC|Uuu8tgI6KACog9{dw_HVF?-@nP>ZhxCbVf)+#EeFhLMv|eCzWnEzxsUEs`~gHm
z4^7`vv}o^!pdb(#E@(ss`iURjyb+nL8CC{hgGt!XiOXetkDgH11dEnioDz>+eaoY>
zNp!*X@86F=>~vm@Pd>>z6dVC-g=C)o=aTVk1soot-)H
zpl#ZqH`E^fHx89A8mypzQ!0J=+Y7?BZyT6UUoWUL@Tw)Fs}5YJ1i70%#*g5nvbu#3
z2O?*Va8~Z%ZxP=8hgmRFMR>}`X(ghb%)W%@`~oTfpQM|45y>snjVFefZh?Ib*ajok
zN=rs{k%y+fva$z#6Tw(KV$9ncw2o2)=Av*7&gbV>_jrug*X`HL^_sf*M-HKI^5hG<
zg-@A{sHv?*0UhB1L7i_gxYU+%?wp;r9F2~xdq&vxG8a^;^wl06QKyjdzJ3#OT4;E1U-Et0!QhLDCP%k6#`|4nG$bqv_CDbV5#LO5or
zC|4zgP3zv|aa+Y!qOE<3DHqubO_ED5Ke}h_>32uN+zyWXnX_m6iHQvtz53`;;l+zT
zQ5ZJ{OV}^LxB}BHER3n@T3b&BcB=`0R_lE%2^%MlHB*zx_EefiuOPv6UM=20k2vEgGh&{;@HLb>@&x5$2DgE
zqVTaAC4R#JAT+#nnv~RDfyIE~*2@V6V%zR;rHm!-25G>~`^)$5hPZzIm=pA_P{zCu
zH%586oeV^orz9tocn!hbo!4T6*yqsGs0IXH>Bz4*N1VTIq3>4P)-yj^_)ulctYPMM
zl+QRyz-UJ$jA)d88I_FLqev)Yh@=2lj#rg}@@yhsm2Y76AbFgH;<0iijv~EFR&Z-)
z-R_x&uzdrZXz6`?io%AiLzI-xe%Zq4+cVikm0H4?4@s290l`5~BYo6Q9LABv!}i@`
z0OJ`mmg0pQ)nnNH!r<;M7B$QNR(M%pf7^VVvZP!-vD-~&F)uF7Wxvs%#tLhI?L?dX
zyLLI}`$VUwFWI_vdRq%4il(k>JM_1Pv=WrRVC9hUvJQnw%=nmEI8y8Iy*CW`E?LiZ
z6~o2-Q}Zs68Vmy{-x<)ubUp?+_Brn6vy$iwa3M7}fUUv$cj>ynTij(=s#*^<2-I3Q
ztmna?(@y7l{c%6}q0b>}Z)p8oH;@RK0qw?e#ShfgX|6xF7$~cQP=uey8ekgy%ZL43
z2u*|ojlu8N6wBF-y~5GqPhOOK;sjdld;r$TUTQ-mE@M8xX5P2YX6{_*_~x(dH#jRx
zZ!0PO-dl$A1rFG7@17sKoRJq7g1fgesCIX`$pQ!NE#L#=4hSNc$$eU;;BaVFxQ(=K
zsEYonz!vBb9o#w${4L~A0n5^)_5W-o>(-77qIj$uP0
zL(P9cOhG@i$vnI)bDPESP6`KsDLnxnNPXA3CUO}#KJO@UV8op}Czw&KuAT|%L&fZ$
zIT9jM-lRix_zl+gRhXDL9xU|GAxMzxoCY$Sl{?vIUzEjhm;p4v1?D;2<$_VvoUYPe
z*t3z9uw%E23y^iHBByjzoI(0MU9d#THoKD0=4*C1rV@Ykvt${poH>XvCZ3_#@tEU?<-7IJL
zOl}`&DBQZQd69TU5VdKb|2ioZnf#rkK4Rd&fRfN{82Hj+n4qE}^n8Qz$0zM8;C~UH
zY*$az(=!=}L4Fh7jIy$QdV$zq^r{rx)}`%E<*J^K&DseA3{?D*LwTB)8uq!8bMYc@
zw}$SXoCs!5s0fF_U~oh?dDmRICy*|d0w8T+*&>beyJNF_?sSjw=tJ$sJ{#!riP(Z#
zPJ7a%LEaN&4SOn@js6Md*tp>V1c9=1hPf%o3sx>#G@d#E{oMJ2f_ZCuhksdte=*yWJ>C~hDAe#o88UY$se|@-TE-f}0d;lRF@;@by;e%-
zg^mjzUEM%M|AD9R4;RP4t;wNy#sHxqA6{Xh^p+Zs>hp2@%6OkhXu7j-RTTIk>8(5x
z5>f_yOLNYsvjm(2C690z7It9FgniX_$D8?q7$Ws{9I(6Gs`!%W`WxPP;U=CmwvDCv
z1X5yR(T;yOu!=;u-m=VioMC1rsi5ma{mLbbEC1&bQ{dSCr-l_5S
z{S_tkNbtnhfM_gf#_WW3z`tl2&iJ;WA@5;4!^q8ARp6q;A9J0trFw-ljxEvc4wks!
zmX8-Uj|H(UbcKgUmf)t&8TRhgE7KS+(l}x$YWlgnHi?f&iI5e6A)dxfZtq@ZC-qP;
zEIm@<7N3%&T#MpZ5E;jg{m3;dkgyRuC1ihblaVa}1b~Q&uksI>$0WD2ygd+S(j);7
z>M{&Ma=Z3ehXo5(@%g>HwqO2pn|T>Dn4~Uwb?YXl1EC+I^q|UPkm}Zt_0x2ug*goGxzxM30_4^J4H8#e&xnFZC
z;-ka$AoSgmJ0ldhetqztALk-x^|8z0k3Qs@sAuD2@%76W9JrU({%G13V!v=}1kvE!fPgHiOgXzU$BIiER=H@*VuYfPH-oj@RPie=qSne4)ympT>a2ISuP
z&2D1~fq=PF1X}X3132t7xZqVV+LoY9egFs%x;qlczkL2dFg=X!&L9JxSZ
z1`%k~tOKPlW>N6x4k=)p#;sc?e`lenwH-Z=;vN21xBD>*gZRk>fX9IFz?1I#3t=^K
zOLC5Zp?2-t*TuZM#?N%2F5E(nf9w6-9tLyu&PR?Lk)+HPWn*#ws?0>471p_5ru&+o
zO0V*6ApIi=30%W&PELmo9MD7KBZDh258%`>uKP`=wN>WZZyw{M&y2hTxrlLge|ssp
z&5w)xDcsVl$<3fB>=KwAz+1b5{<3zvHQy(Rv~|%5;;k~`Wx)GkxLA>`g9hy=hM1KJ
z_2k~YkHBm`@jr-$5U0o+`E>yXUZvl)9R>$12vwL7CDNV-Rq#IW8~uehpFKBF#t4U0
z+={@gh+H|gk{eIW+7nt4*F5e|`=7I
z;iE@4a|Urb@c9rorkCl2KBK4XE)P90w9X!|mF6sNM9R8)2`^xElY&w)VDeR!Xi%v2
z_Hc)A`b@{p;TKZP$1ErtGWE!mUi&8#L`em46`#7?vgVEdvOYzdOzwu0;1BAxYOiP4
z{BOR>6*~H(gfzFExLvbXtOOk}4Jz95Vy?m6_g?ScGbpenzEpd@?pL4(Op={w1nc&Q
z*r_}%@>y2&$+MYOEy9E8j$~xKX=>`}YKf~tP}rjShO8e^gPs`JdZ_7Xj!3b^zW^gl
zfo$NKltixQh#ileI=U;X#X=p%{fc+&=ad#ctNC@Ez5beoLaq{xrb16AqEv~k+bi+*nNDh?S!!L~Jh}u3EosVQt|cU_B4bX6yXw)SK}Ua=?cnf6%sv#5
zg7pu_DSg%)o$B~4NT}wzx=QTr>!2>Vp;=SlPE1BY`9KU$zTb*$wt;lGtCJ&4!QVrJx{3}8DV_42d+=eX_>#bDNfHF-!E)nbEcS>j;5~9M$&V{=$w}T+A$Zl
zy3d|1(9%+!a^dvpSE)WDH#XY-WIjYPdKa|)`1#VOfu~6%X!JChDEb$P8gj1_Ww^OJ
zr}U{^Yb1=+nP{ZXjSA0C>qD5yFl~*daD)+_$f9dC@$Mk&cHE
z^e_7N)m~3v0ISVD@_^;Rs5QTr_6iZ@>Yox%cI4KpvipoyK_#C`%p`G5{oE_@=Pi?a
zuQg?+ZP)C5?=XQT(=TH1&`@QN-+Zz_$f^jHgF=SB7Z>U!AF^0SNpbR&vA@k|ubu2+
zGB3VSiK;+bODkc@8gF-ZMa$Eq%dn?O_t~gXXXl41GXaiv4Kxk`XBw`e^E%sX2UFQ%
zNzI~b(Fyszl2anKGC9n>F3P;W!1j&O%>Sv$nLI=RUH60JBu~6kZnztHCdH^6wr`sK
zG7z4&irIArf!Mi`R6`$(^iGO)7r9xLyk6MDUt)NeMdqnfc~9D)%X!_ot!h#CRrk-6
z2M^LRt=a?NIc|p{iXq&HfdgZ?mTm`y%==?@PCf2Gzx*D2YAe}pt_l{2;XS+DDvd6B
zmfx%vYYo0xVpXU`r=Et6fd10loR*$Wbm!U@Fxv%E-BmA4F8OpGpuJ!OfXXx#&|D>_kIOmD4|J^-Pk1(BYdFi&E)W-~5xp6~v>QuM7
z>oYAa<(UR+WYpvNVd3mITs)2QFNBHTp`|v336>Y!U{!}1gn+}rZa^^)cmgS%%|!`~
z+1@RJ1MWA~ADCizVUE0``cCPDD&(RU*^+F#vh!lT8&=?S12q;(2HPhOhXT+v27i4L
zn8J-79iFhAcH-0L&+mAR=gz*!Gjr@jqf#>&GL+m=J44%1w3_Mj)N=4p??JvhC$+6iS8P!5?1^>R+dwy433Pm$zV6$`f0gm-@aqI
z%W8S=@KkXeR2Vu`FwF$Y5DOfwg0pA>nmksBFx`IhYy#HxTCgKSbp9CHU(RN
z)M0vNHIe%X@b}}#Lnr$iabKM}6?W6S-F!8tUBJQyGZ5gi`6|r+5D;XF=;D%eu_#zI
z4rb2F{DoUaX*~p>!46csZ{Gv%MixyYjj~`yDmzL4;5RW^K7}rMr5FEAJH}mIW;uq=
zmoCu(I0kh4uKq`}rp$6CaC+kk&IJHW#oy$HZ~FX`cmv4L^QNE2O~ABdGf3)A$GyiN
zrl$_QVUUm9UW<1vEt5C2D<4(4xhDM3cj0D898*ZVs$=k1=J4nn7@T!EFRt2Jof=7A
z_i0d{o1XzYka0C`n7X`@&dd(U!OTLR*fJx^9lLRXjNvQ>0cVcyh*}aBpRQHEaLUlW
z%LD8d+}N+=w`9pEX;G%b6ZJ5;K)Nh=AOP}L9Uf{n`JW+hux5R71%@QL6DKwuXr%U{s*F*gpI1dnQt=k?;%*#;XU|p~w@4^(
z4(JwH7Hh&T4b;ch^0kb4dPc?!3kwXWiI~OAFJ@X?N=g|{pnxLSd$KIKsw1p=C)&)=
z@Y0b7dZEM`(T$R9CD>8MY2b
zDS$Q(H_0WhKKTb(3m}eV$HJ1HS4<-NttjEfW}Ty4nDdB_L-U@`uN+7IHFrl>gJy7^
zh)dt7FmgOlQ=YNk5IxBPPwWE(!V{*#y`M2kG!@IVapU^`+#R-q_zB{S!vb5eI*iUp
z^0SE`TM9#loXam(qVr5WTVWq!deL7Znt`_hLe!t1Yjm^6L|YPYWn^Vpl7mc%(b|z)
znDpqu132!51qFCFz(?VS(5N%k!b`2j$^rC?F*J#Ziq@lmXYu90U1*yMQK>9W#Y$!B
zBt5;YtRjlY8uRNp+TT9kGYq@!bI$4Lp+h(???E$6r=ZM@e${mA!E&;)Kz5njPXM76
zA-09TPW$=#G8GDJ4~y5FHI(ds(PQzC60ZCMd4ZOR^#~LDi*6ra=`{&Sc)>I8mbA)hIi^M31SR{a}HZ4S`^%YYlAW%Zp`q3f76MmGbbuU
ztzT$&StMt-0em5fOX@29#*G*oB&
zcq{{k`Njy%VmhjvjEtaNJ*9A)4iyVxx#0;aCmpb9fv`B->G4bSamvknG!!A>+7BGT&`(@Y*SjEkH0J+M76d$1F0}-EG7nFT-B=%qg7bR*v%Eo
zv+>soSiW5I=n({TjOM`-E$jhoA%f)-)2s4#e)k;hWNu#0+AVSYMEVl3W5?~M!6qp>
z4E;Xuog79o5HNa5(!G)m9jReqz5ixVg)MUqJQPe0f=A5Pj*gIQiyXJuxVSG=pzE#l
zwjVxt@aCJ(HJWm1b$9Qcp=2WCZ9Mc8M7qiw2hSWj$kV1N9&vth_YC`T)TlmvZP`VH
zmWrop)qaa~f=3gp*`$#+x;=m(k%NR7Nz=aZ(7~s~GG=>tdTQzj`1HbJKh{jy^zAed
z;N*?t_4K}d`o!%LHs*0<<*u8r%Iuu{i~GyVPsU{a@oq>q=eBkRO^#e>ZyysECp05K
zc^W65-$F{`F`m$e^h4PZk_wSpYWbHprKd5wQ
z&S31}7!loas731Cqp;CuGW~}3#dTu4N+s%CvYEE*w{u_3$?Pi=e(~azJ&}el<=kX0
zz#~A!!x+pRZreKSs%gyY=t%D_=fmfWyW}b{xKBNZI#%+X1B<2$ZN;C>EAI(}v=?uZ
ze=H8E^~Q_nRR@CvWj|JR$Av7m;iJOeI(6z)?6i+U?tj79E0t)>(CZmemigDKf@Z(9
zd%AmSlzZqXUBC2J8N`0lqzv_}q9O(18&12&1uw3+o~9C$X5MhA&@=2)&zz&t84p}#
zebr+Op7rkc`lac|(m9VeZ#I;f)$c$|^%C;{kzXleszan?A7->X>!H{_Pxt$hq;30q
zmWw-T4#{1yuw~+tuhufTNBz8vS~YU_?{u@Ni8&tmMQcl{?(ald%cY}|^P3GjSIs2D
za<+{ds8W;@=F(v%oj=P;V)i)Q^nl0CIgwRyZ6R7vlbihG4&=3orJgC=D!pv!Y$e~p
zwl1PAr$-8ZZpvI$+22yR>0V($bEH&q?Hv7g8bvX_<-+sKMTQ^T`b5Q8SySIY@~^Yz
zC%^R4^jdJfZ&pjZL&C3Im(Rn>QXgm@Tlztqs`Z6XqOxWBnA`n5wv1TPa%a?@)Ga5z
zr1<729rzTiYvtZ6Ik;qndqr1P(JilOaupK?CvJJ{{2;)&M~%s<)atXryN*^=Uar;6
zJkx9YqHhLT(t(>@YG#~E?Ys5m@r4goYtGADRR22Qsk0A{*5NsSd$!i_YZjen+RQdQ
zsymx^z2aB>r+Q|$)WOICmsYip)8g-!uX9>!=HAw(bExFN_X)=w
z-!;0f{B+lP%R_JL^$EZJ{^*L@j5vwk(=|u_`60uQ4+pl-x%s90l-Gjtq4&G_4EnOa
zhGia-`?K)0grvoR?qx5af+{6;+fC>lw$ZSq#VV&ne-x5~&tf67d;TN%I#icKY#GIBH>SBN_1L`LCb%%1?Uru)f#)~)W7>a$NZ$L-~ayh&)>!euZ|uYyjH)t
z<%H{kuKKXz^`rCmyDH=+h;|JdrJXQR{n@Hw5vOnWN@n-GwV>^b+VxSJmhAT3l44L=
ze_qp3MC*6x%Q+Rj^@2tW+2@w?(Zxz#sQkM6mFMw)G>dPVbZDh_Wc4k`tKGEX$+`&p
zizfZo79^a^%WN2HU{zw@amvkG_H4_BMXBfBkqft2hO9qlxb=~`L)@QDgF22*^gJ_
z;|G?9bR?a)9qhfk<6ezz;4Aey+w}fhUMgg%tNBS4{Qhx!qD;udFWG_SrVWdGnXj!Z
zNy_EJYbI!xcdT6R(~=b=n*zqkFFK&dN3txgSMf;**2UTRo1y@EZRSMWDH$nSA
zb34)ul0h&yG9{c?LAapdY(?KGDlU$v*rM@=!BNS>LDh5jUW!yH+X5sN0s8Y^8yLK4Ze|@WpZ}sB
z`3bn`^XCcZ8>{JPxI)9o%}z<-GHdzp0XFRdWZ;x4QBk-T)IVK&7?P)p3tUt>zooZP
z)}i`_zOOFBr^fTiYiVizxV;vq&Yrz@=g#(HF;(9E6m%;dJvwM{oKHGRMTOZ}B@+!8
zHd9%tF?lkSL_-aYR*D_2?c9>Suxw#!k=lX)kd|^aTiWpPut$e3WzTTI{{8J798?Sr
zzh{y#^M-wW{{SMPaQo{o?}xj7I&Ijn!7{xTN+*28?EXx3|GUvi|CxON5=m!p=-|PX
zZ$782<92V%AdiJ5NQ&@q!0u3vQ}Pe;UVbSH%^k=`%4{Y>rHrTrnWaTzZf1t-wD_`2
zzIyhoAB0jC@e{`S$B&~u=v#vA5{*RxUQ+1=%!flABU_Eq+#lG2Z-ec@<@@w0AT5Pg
zCs5Jcw~bK&7jV$R4}g;#U%WjjH4|`~VY^eL9^SvdCNS{Mty@9w(r~~C)PDgp8NVLH
zn_Sg2f}tp1V!h)f&I1x~Vm
ze=Q$%?H9H(f~EuGJr>
zBmt0@VtXhtu~W@QegDmOwANroz;pxpT)zA*$0Wu0gnAaoh=2Gvyn*T%=2@*9<-~+e
zEWO^nlW_k`8V=69`)wb|Y-XNJo;;ah$$Xk^R{$jR6QLY7!^&8GC@BeoYPgFH$Ocqf
zw%GmFFQmD!I(+Icok<~y=hQ&=B&}&NrN2sw)nZ)zTV*DlCWrGTRCJdvTu3$?-pOHR
zG81{a!TE;?KQHn=ur;8gu19c^~jp^y*oHVkf*u
zsK-39z+8hBM@>^9zV~g>0R-hc%U^Piz=o)@j1HSctj5@n`r(s8kyi7<)XevCJ(W7TC
z`<*VYkxv?I1GeD^j{SIym7znnwo-pfPZ~@>Dt{1(na`NPu1XDv-Sm0d(}7czOBlvm
zg;E3hI=DJcE2y~sBEnGpNQ-{?`jzk0lH1L;oPE3ous~_RfS9PLYG-E*LuabQz?Xn#
zkPn`J`kq@Ic*et-@*>Q0fLjr>CtT#S%1hcYWvG+}tzx72^{7_wT8yeHpqa
za+q8kBQT}MEB0dTTuC_bpAh|k=^_-cR$aqSU=#6V98CL@fsFE@R9#C?Xv>yjTEeWL
zAYKd@;Be=+Q4UhG$LN3v$&iLB9O-h6S$M^5-n<)4{#=6c($dnnjem+Loy5Ehc?E^~
zD_e&A>Vgo0_`U$q3a(z5`ywoXgpp-V7l^@0t3YQ6WH9frO#uL#Wo_;2=eNTF&~X#$
z9sc=Sx*Yk!Gc_D(211jHaw$csg-6Ed^c(b7t=Y6GTTuY~L<9c)ZGl2?M>84DQ2WPfu6(w
zLT0{%6}d3V%kBY_k{9aPCP)xr~@R`3F71tOSh*6PcwwsK^`79=?TNLm^%oRZlj^M+5gn
zV+hRJ1sR>7=@Il4Vjf>WlYxs?nFpr?a9nfb$lhPp^ROh0aQir!DA~anPi1Gr0&R_q
zAY(;i3d0cEJ+jY!7F%?dzrdJ)*r%XHQrXqD}xUz_E$_$4Lr}0Ib7Y@HKQAu3ZrHj?bSZK$PzbToa(f=WM27G#(WS
zac6oG!Xgk4MVw>jNip*6Et?TSIq4fi9$t`$ffv*0QU3#KhoF;%Mq-P96`3gw;cHjV
zqAG*S2afHIXM?n~4(|%?9*Nh^)!ZYj9Vi=c1$|ruRF+m&q31eX%?weVXlQIid6KLa
zsg(T2GTLjR_{oJ!m#hAE>rjLRz9>yyJgu6&VQAR!^r?b*)`JfpgK}id`wNvVItL}O
z1kg2ZwoYrky-|{m&IFN-oKx$~n=cc`G^OJvUaSqfDtI#()h3t53-9iF;XK(YN!ztR
z;A=)kotrDP%Z>3GFsY!Z_w?L)WQ}x`Gp6EgZEb8}^0=#O$WVZ7D-
zrf~;vOZDrQL`Yg{pDjQB&^!Wr2{FJqLQevnMVWN*X-Va{^cwGy8?Gz>AQV{+fK0NrA
zz>^#VaUApMcH-xv1z?+S<;ryH9)2ksWDvOdZLLWlz)Ih;?|rcTMMP&v&yTb+@7%e=r@UC+6{E`N}Dmzmc;nU9yA`>~yUd6DxE7^obLQ)Ax0K
z>FB^v$LZXnKATTvWIO`GlIg629{ySGAc;=d-A#Li%Il#+wHCLSUacFPR
z(KTegq@B+$t@i_KC#hsbORevNVuF3c*%$T9ed4P!C7^#w5Oz*Tl=&{YOFQ)Lzj(om
z+w!7lZTmhZo)UCRe!bQD;g3BC$xYNCg}v+eW$o?$J8`Wzia=CV)nfkqLB5^T7ToM1
znRMXZ@G7vU2!5ndvA$QX&Z6r(`68Q|&{Z58k@ca&;2I(iq|;qhUgI;Y58FFAJ%)Rt
zqNZ-tuit+DE@G~A=BCKFj8-czgqgAhD`DmasdpjDJ;T6qbGYYgd84#_q4JuxZ}59FT?=Oak2)KpZglcu}RGu^vqPqS_ysSt@Z9W%I=9`o@l_CEe?ukubb
z=hTYa5M_nG{I)SP$K_%G%A6B$wOuC$vQ}hqFfUE~h$g)-ULi1sQGi&`!f|^#-R;`tBZL(@#TXc*^>dsrKLTiO;Gw<%{T%MC5B7nb-YZ0VTxz5Y_5=ysKc&qY`K4ByOB_PF$D)51EX>4(qeQ>rIRbJF{rjs$qi$35LrIu6zG+kWmX4IV%jA
zgu#nyA=Mw9W7n%uN%|N?D2m$m&sX?`)$!#b#uNgPA6
z2-Mv_N3i32%)~@syDfe#j7p$~SsF|^?~xTAKp@*M?D!<3F9C<9jX<6&7!t<3wK>Q3
z$n5*ulW>$Y2GO^un&mHfDL=S=UFa`mvF%p*HIUR$=}>6BX>6<>Ws?4AyVLf4gD0p?
zVK94@)XEI_kBrMS4eW>1wDm#DA4i(#|x;E7DI8J$1Q?eCg}bYF;o%X%p|^TXugRN
zAo}h)S&{N~hF+2-wp_BT!AAm}h`$SCo$t+#^@D$nSPzxz6?T`OgO^w-eRnswoKSODq?(!1?br&XU3
zbJY$v2~pi9l73CNdhCX&$IxY2s^}+K6JU7CiT)gKtxdo{muRg4r1RsSG%H`+b0Chyo+1
ztaQ~|3b?gXQbO*%F?~_AM-PN-WNc-(u$89mPM*w2S$)iyh;(_qERna2ZenU`xJk79
zoW>IW5~u{H=WUAP7^3eDQSH;qn3+Gg$Tgh>2uV)v@iY`YzM2|s;MBuc40_ME$f$+O)I;57Fe`zMjhB!ULPb!$oQYZfjJLtjKyHj_9RsRyd~Omsqhb)8HK{AnKG)AsRz^O=HT{peuzOdCDpaF&r{&ZEfA%_wLE#yFkdi#visy
z=Rsk<-fs<5*jx8%DP&79N?3>>ES{dd&W|2D_T~s(HU$GuxE#$tY|icPvS7hJb04q(
zxfG-xPDDU1OswXM%rXNzDJ>e^1mp6Ils7|sBn}MEf4jSfNw~@IFUudFsyHyFzxW#|
zUY%$PA~KZ$Wet?f+p0-1p?|=5P$pBB1}f1Ev2FYRm^$;g8uxaMR~lB4RtcffER`f7
zM72tiX(41@C4{yzPf4RBsmx?(AwnoJ(XH#oE
z&+m5+*L_{reOCFdBVRFS!2*_m|H%hs06#*gt>Q?bA3
z_z`Wvwt+g=MuJ*mm#CMu^)?qvhfYS1o<9$PYYk;h64xPpnR*L{g{dZ|O~lWphfq1U
z-P3HZoo@u_5@I-bpughyZANPIyO^|*^GzvK_#hleWFs(1xZ`3C->wkGVB1zv5fKxO
zvQkr3Zmy`m=xMaw#YotCN;4+-ZjAJ{nrI}ZM^v?$#pqf<(}%8$C}Qb7t{#Dl`RM`z
z`QbxcNcKkeLmbe}wWMEe;)4n2DBTi%Uv2%Ccc}M{9h1c6GTZ5wWI)-kzQ6_PpsTwK
zdPk2QJ<=5S9a+0}?W^ER1DC!5RV#7F!;@a?
z*S8Ao;F#L
z&aSajU6p@>{iyW*1Bk+Js?mw}m~Fp@NgHaCoQ;gyTRKl!eRGRJTiIpURELJeg)Rt+
z`4>wBcxoVkMA?de|Ba|@KjuA(Dr^EEvBJm2(1l7B9hFvjM};L|3}`Ue-Uk=QKrB{x
zH8$a<(<7JCHdMG@nV_)pWm%+;&7z`!QtNRlDn-i+?;tTuPLsXt&qK#coFWu<)?oc(xg2~Ec
zM*GJUgJJpN2ib#N8gNTrOX^B3U-&^RLh=j}%_V
z8GSai+M)5Ghx9in6h5V(FXQ}^vWBa&c9$mR&TY$^tH25wBLNh$9PI8~mVKu(i(zn2
zQ5I7GK0pNwsMtsFbh-w`oY>IB0=8ZM4Z?KmdYqhe3cnvbq#1+&%OQ^uSb4JU0|#CP
zR@}L37gF+!kk)qA#v$6tcUd+fQdh^4BZ~$pMinX-EG8(T_)w50>|VA0d=?~dL-zm6W#
z(uKndtqyKk&vQd)@z*MbblOADic8q`P2?*vm+aqt7<_-_D(Tw2>2O@l{YbXwwjWq$~h{1m$Ur
zHrsBo!sNVseqFKmza@4u
zgs9R0gj+xW
z?Wn6u+))uh)Ha*C4V39=QIT4ItANq%3m#227WyngH`T}k5q`O8v9ao1i|4H0vMdvv
zN}&bz$=lbjT^-`3md|n3+Tq17`luu4NB~Q4x@0R|L?7$wQlyq3W&qMGh~vJh);fl=9zbs2egk^%grXI1O(J^W3Z3-uD>8CsOt5j
zb|crgz%}0nvdOCZeN$nj8N1eAE+YN3U1lWpn%X0+44ojp`U1Y}}~l7@R`zbIK;jqZ$U`{Z9MNa|U+?
zFf3h1XaTkqEV>UI2t}EQzJP3O()lpY8<-Ka9NHUy07YUb5Egm^Y5k_&UqNcX6+3qB
z%qMunqO`&VNYRQ53QU<4U+6+O_nBtzFcC8y)(kR;V8vDL@jp49DbLs;X=G&NGHu#m
zJad>-h|ff9>CUh32LT=CbJOqNsN*p+fcWH7TnYNbyoy2-^rxb;FlBoDI6Az5au!&S
zECB!_FwMeU0A?bS^6%Vf*P=Qus7uo@|H&O$v)kN^NeS)|*iL9P;ZP_Iaz
zRn*pC!C+agLf816`}fDIsGMzYX=R1Xjf?*tfRv@;(zaSkLW|4-Ceq&%nxjW?Miw?2w0B2x;O!e?6X&Wt}
zng_5T2rhVhf^h)BZEJBYaR-yHdae-joi_K)Grl73ep6f9)ZC00$SGccAg;T=O!E-4
zUM9^HR~t8qR3m*s6LP3?;j2?nAmN45hF1vN1esod1c&_?13@3#UfwO{E<;M=q0yE6
zT&Uz4s40;*FeKQ`;xRJ%ra+`O)cRZ**yjvDese|OT9tl(4AJP}gzZ
z_Yi4{Mj2LWA2B|6(#0rv@!~x|Zv4sx3m0b2f*$a$w$@=ut!I6LIMg2PIQIgM#G)G;
zwQn6)wnA1c0rg^hyUZd?`QJhirHYExWYt?ox6ECK^qK_5W^BNu1J3IWE%j2adwYA+
zT3|=;x~9gL*)DR3;l5SZ6je3DJnNM8N{|Xy2Wy?$ma)jjuFBfd^3FkLiVyiL8a~=f
z{Uf|IKtvq&YHD1~bbQ4NgMyq{A5Bg}TLFcP(t;hCt{W{#D~w=}f^>{^=O$5;B9*$2
z;*ss)58u(AQEJfCo9VDUV0X!DpefU7^#qA55O4BH+6X1NHHVzG#
zGM!{naV^tH-10yoPM*Zts#?Iykbbb+o6HTEeSP4?;Ua!Du~8T*pdL!zyEo;2n-N>j
zo%;qN!p-a2L50$1WaR
z=G?noIibr$S_d<5_9LwGWkwEAZoykZKVySts^YirjE1PlNtf>kPnb`@DnFvk2SZRo
zVqjAmbumX$yd^(WNAftwlir{qcX^DI(P#rFXV6e^rV&DG#Jj$@<5tW^_W8E1HaD6MOlU!XDjl8E!xlCJeg6CyFXn!^Y
zb|=HuZ(@xHIS6+DG()T4n{ZBY4~VDVIF+8x;i7QFkej}a|IJGynEQ@qQUUL_8p95o
zS=ml4acr$qxL|U9xKQ*9@H^0_+-E}9cvaJ1NM;$Hu0-%nJ=mxtPb-H@&r*SB*ej*=
z;e_+X73F|q*g8C72+4KN6sZL8AHbE6beah;bz5Qc$HkS<+1Icmy}Z2c^=p;7-^L=+
z(mp%id_o4Si}>{9E_{Fb^1*efX-M+Gr9l{eIEY3|gr-
z0(&bfCBsXf#3$0znE{7%ii;Lo@lMmzH;bJKYQ6?9=F{+
z$C-)otXUvFI4xL=+Km<4X%wa;O;(&^N;l!0r`c4>Ps4^@!mgh>CLLD?whay9JpX|R>P9z%Z0N^K7bFj
z#x1+~gXnCSAqJ@f2H=pVx%(t%fO^6;9LSiaR9FD{13j32dwgM`iTCr(kK*EMC`Vuz
z4_3@0C3BFywf23IS>UfTkcc{d`~yZjd3Dc#e%`!M9eEH3ZFMV2%I(B)(=NToYVFuV
zZt()fY1orAShL&DC8F_jaffGXp*pd6g@1Iex;O3QQL)D1{^Q3Too17pdclRCdv`Ac
z2!IN9h<+xI(Nf@-mdqnO_Pd~VVN>In{wCSCmAqzyClq>ZIi67f!D6KXf4!}ut}LdY&+~nIfU$V9*Qu(%qB+Iw81Wm6eD5%
z>eDw+o!5G?8v#8%YPWO`gq-kOy*5EvLVSdJ$
zi@v<=?%a#(w6V|xbfU2iZEBFGrbZXNDtos~rly$h8+`s>^yY~VC7HaYndjze?)ymL
z;0S@H(I;_nw`*LQ<(xu(mWs29!`hnQOArodMJ&wCo7$*RSyk_Pi
zl{&X8OH2Drl$5*~2Kc;tXe!9su^b&)hq&b!r6A7$;!wp|bLc_#_DkqwoD4A+gj&)*
z?FciSPBuOnuqL7SaV~@RHg!3tvwa{p6QGpBnrF@Np)gfw1&PE*`Tz(EnUD0ev7m;z
z080kh?59yQhfL8%
z_+`(Y>J}+4M0FL6qe(>q+OhmYr$MiYY;0ISk;GL;sudd-hX8KF6JHahIFRT
zh8%FXM2SXJs=Bs%DQG9G$&{?=7XtS{&B7d!>MdX}4`;}1_
zxs7T6zT(-HC+NQ_D=YKL9I$SfXS%(hrZganh}qy+H$(yzbl<+|bb>xMz@y9mGe4e4
z4>fICwRJs%gri1r`7A`#e@Es9(^`B-zB@1=}Q3i>R@o-hU
z(f1HFG9Kc@h?<<;#1i5uWUF5xq0DmV4G9dIlM#|JDqlxm!by+#@RgFD`t;zQJ#0zg
zZ35b(;X-E&V9v8(qW*WH$bkFD_W~r(cofEQ<##aK{jWvuOzBe>D!m=85
zw{!u>CubYp1}*dz5;Pbj#*oQ(kB&dC;K6znzf8PVD={x-DDG~l^5Pv0l448|F@vc2
z$0dqS@u@uX6-@;_hK@)8k;tnc9GPy*xM5V}sA;fGAq&chV{@k=s$rvMwpG~x*@kSLf?OYS=H00K`7fmoOxV8!L;q=mzU0CHmWru
zC9z@)B!?kcChv?t&ku$DV$M01bn
z^ISy#0Qv_7;q6&PA;Iv1SR&d>3`re-+^!$jV-*x7XkEAHZpj-sBie`(SdI-#
z$c}l|y$_iu`!r)b?c=V4bAjK24(yP~MbQ$+|^
zX)j!8JtzYU8)%7bmgn8E>J&$(DT~?c4@rgx44#BNw6_&}A)ri?pbqNy?CgGi_gG#%
zZ1%#1&nV?s8TH~!;9t`RbIKKNzqaFP^JO+BN6h%ZcL1O!-^LZaLc7QRtuX_y|oIfh*7=&(Wj*!E8dca~B2WW0!!R0F48rO5uqNY@S`d*4qR~xKkXdM3%Hk
zxoH>^Q^l1WR}L+qG_7
zzcTt_evnazvGBtPNWRZnwk|&HG#%W-04@9RRsbWIx&Jvf2LO*DW
zCWWgHndm)Pmu?cGHTbRo=#nNW_H51)r*-MAb`E$asfH`vn^BJ1Sty>>3H+LE`ByZ}!rqh&{$cC~@m;{FZ*Ooihc2&z)
ztl+Xo75ig9ZyI&TmLeJBuacTzTwi2H0{}=>nG!?41sZE3q1Wd*D?Vx9
zx>I%B-f~7!F>27jfd)fE%1ORNXP+qAaMlDzO5~Q-)=XXm5&;b+oH_r5wPWfZ>FeqW
zU2KPLzWoN2QCt&zA*jkw
zSNA)#6~}%zYJx#hbgp}@S>Q3QBZa-c{S=}iSQQf)#;#nf+%s7JC8!XjbTw4W=SaGfM7d3Z-$JSn
zT>!kuoHp<7fgBn?S9vqT$f2`HnQBrX44dUz6ng9>2@f^#m-II?vToz$>ps1G+c~S^
zva>@euxHOU_4}Ip$rcJSQQ~R(lnFCtXv#~R=j*qwRyq%ShWBzW$S!?GqN4A|LLGxi
zLKuSG?Qh>@J77Ytf$o@~++z3$VI4L6R^~-)#Zq%hS@rNPU8ZNYj6ECeZsKadARA+j
zz9#8$g}u#PvQik6f>g(L9dD_-eEAQ7EOOL!KEI!@$hq{l9|&-{+ldW5#3rLYo!U~NVvL$0K)7)<}fka@k9Wpzh>EfV`FSC;ks
zIp{N)D1!kF{6>qmR-==z>rezc&CkzL$*>n|xT$x5F;
z$I}ia28{AD1}(=PqC%naCKKM6%cVYW3Mu!E9rT{$!x!tBUeez%=|n!Wuh5T1Cy-~y
zyW~g*AdFsenP#~9y6O{O6_se3h-yWp&w8MR)hC!4FxQNz2yN0(U7U$|v5f=@8qO^=
z4vKRAdiA$o1hWVI!jmMHy3Csz4dry0;$K=L=3H^BqY&HZ{*wBbU)(P9;aTkkfq@|?
zt-!5WMN4{QXcAZsLN5ggU(>jx1OgRR07&enF-lk5
zECWoftv_%kQ0_B_<}ih4jS|V3AV7tu?L*tv(ooZ8MTJGL2RV(q8x@Ud7uZ_cVP^ag
zH4->YD{Dx@J4lEiJZ;|3p#2AS>{!l32o*{D7!pDVuRnM)@Gv(q43q!K!3{sZd>JQ|
zI)bTCr-P?}7eC{Lz51d`waFRfbQJudB7YF*_GhohL1@;wYukxb)_xi*se_aP
z$x2d3UqHN@9T=$Jp+lRGP1tUQyObJ8P!h{_nCNOy)U8P8F(fyB%O*vGGGtXnv&qX1a)K4IP1ojsUYbeom
zMA2zaF`@~UUjfx4+WUsK&Q**WlidF<0TYr!ZpD}LB4H4K0w23DejzwQD9#3KA7kF@
zl7`iTVZuSC3!rw!25W-Ex8Hu_iqV4Y8TAGQ0+ujT=7R0ijve6ocm$|-JIM^52>yh5
zh3dB#E5|slODTX3X#^t@Zt)?mkLn=5-~S?(F-v08hT6I&To?|F!AR8*d~`4FTLO&_Cvd}Y~I
zmG}!v;bF!8etS-uJ#lbx-Rzw(x1Ua|H*?eDei?<5Qct)#fo;sV3dR0Q=$RRPQrTo<
z1;hlT^-2es7?a}z9Rre;PY$Zv?$8y3@AZ#QOuV%sI`x`IR=I4tORaKgs*#TB@nOWC
zn(l>tQ?8Xe8EQ43j*|5}T$&QrVkh4>OZrxt3Lk{(nNDb=I3cnl+c7$NhGmouXEDq1
z1=%XH&wF@g)eO6LmJjGs+h^l-93_Cz^0ekPjRt>H7ZuIUIl#s`+>r5=;^!|U4B5l=Nq2EfvsDW+CK|8r##Odpb!)n
z3}J0kgR$6-vD2H^uN6}ejlQd{&LB2Ej@
z7Gw6Y_gJNyEN~+&5RP+4z=0$4!mfL*#0$~GBuVV7ap$m0>4Qqs(iH?7%5f?yyU;`b
z*o8k9N|K3ew4~$%0gFh!-7bQPlNEhb
zko?xJKf|B*-nx6YXq^xak$|8>gJ+Q#U&>~Q+FE~?fP`Kq-Q)xmA}@+V#B`MvRdY;r
zPjv0x-5CU(*Mi_icr9!^d;dOS_<(&KP(S6*x3fA?!u~2X`=2@6!8(z=LEW5O&Alcz>_rD7stDDcx}GLt!cxWF$nW4KSlaQ^$qk!{(MHhw&o2)B42
zCr=(8f1G#5`+y)tna&>5FKqR=yA@jeo>3e8Pf(Ibz
zwDIHpV1OZ`fcws<$HBT|r%v3mo!hr79JaB$ry%iNY<;y`NOIUs*eMv{0*=v_v_JZf
z3yIg}&e!sO19z3Y4tC@}r^c-BC;Dl|Ol$lecpNdlOv=NTe`4xYv~=K)3zCMN98
ze)QtSUs4D?CEPn@C8}Z9apU?xZ()uI=M?`#!v7#)Tu}i;lvM1$+igx;^&Wqnf6uD8
zInZ%-RNg|64Hllhj5hO*{OQfkAR{rG#-{oBlmO!Q%&Ggx-t-J;Ss+y2Of3s3U%
zmCE1eUH30LGz=6+Ap?p(Kgc*J?FCDb;?ez%T1x@VY&^N|y!N+^5vhl_7&r|8CV=3<
zMorV$iCvq1kI=frd6Ye>Tet6K&J@`l>ic-83Cw{Jgzi&4TsipQeMY%z&68@A?fh>g
zGD(87Ps7V`vQts-0O{dvvE{w`EPDOuF9#o8AgcMpWoBr~!%bMq@ZjBs-
zHW$s`4L2Lq!)^gqV7u3r?(=Fjp;2m>Rc_F0#ueS?$(hOib*)0f)c+s0kUwS+S#ttW
z%C3_5*jUBTOEk(jaVpF|u`I3K2`4>As=S(ehAK&QDW@YYn8;>eex%a>jqSy;Gp5eV
zNQ^o`!y&1kf-acawsb$KUGSFyKoF1fTe`$^^2Dj%7{>BkW&H{)u1CH&otjD@R(Tia
z?}@JQdFQ%CbdJnR+_uf_$fDrf^GsG*@djEOn3?jH>WRAbG7~)|TL!n3ELdtQ
ze`YlZd`4`a3P4ham~6+{>cokN31br`soO}BEXpF`1d$;5e%lOw!VQ(~-Lq#37a#N)
zpgrT&F%&M0>tM)ew9$Uxv9i$bq^5&6L+k4M5yLD)wnOOTq$aqIq6KG9*U_~B7PlnI
zxu3P=XrCh7+xh&%;J?-gP<;p@7yP5coIKs#m7DU|Dy%p6BYGD&Fo-_b+OpVj9|d9X
zj;JX^VFpQl3A!W_B5#9<%Hbp!x#3FeQMicKuDZIKB2Ga<6)=T28$QE%Qt2|;qD8{0
zcnqg}%3MEyYaAT7RCQ{Hs}`)u(v2xd%Cz`wE}Knm@7K34V~s_Dfl;+L>&EaH;i)fXm=M87bpUUA9o+eU4(Jk2Ou
zVLeqJ=|Vc9l;zX+T
zs|OMjQ`n2mt`b;*B+iCiwi2@HQN+NZ>f3PHK=mhF|6iIHko)ZSasn*7S5P0fEX
zbIn(`3MuOBtE^1)&)V<~{WQynwif%dX3*@qK*z`CuPtX43;cKEaA08HPs9TtI
za~aSy3>YO_Y|v4Zb1d{BOzqt1VJ6reL)J?gTnxq^quK2zdZmmvx5~#I1D!vA{w^dX
zkHajOvD$GZpzII}CIX~3E6p$Tc8vtavO}OlU<6ReI2(0b%|T18Byre=F~L{GhOoW`WGaLKAr;Na8>^~pN-deG@&D$(`$P?8$$M#O2E)LO
zGoie~WfDflFRlsjDzc}lNNMo0Q6on}*nS|wXXIQ~R@6{-zG}8>2WYzCyXr^YYbYaI
zIO|=ybsK5qU~S!Yy!Bx}>(GjWilMI@3Nij@Iv}467;8@Tu0VF-_93=%dg5Bz3
ziP;45?UN@LdU|?txc8m^xlzC)i03O2Qu@4gl(}E~-rBMCS>Z7cZ(sS(ur*
zm~;`%sQI!AQwTObtorrUoL7h1l3RmqP;F1-X190^0w}q9K&a~dMCyHz73g}Jq95$2
zVC^X1nPW*sWZv#9%Tdr!^U=#|POg7E=|b|CXK)N&X0x4-mB*hE>ps!O|@48?KM&1S?I&GA!*m3<|6mR)Z*O+fpGp{v!)-+7{N;O>{
z;xYcVBN@_u55$qmiVAhZ$_xVtGThpgoIm#S(Qh+6i%h9s1RnEPT_
zKVB(Z7+Nz!r#v8o9XWawwrl0Rd!N9;xiExM#$GeviXBOFl~`=Md*=>H<1vTJpq0Rs
zjM6tlCdC-_0t(k#KOt~l_bpJZJM6sU`|Fd+ohBmO;4#AG%;DdU8r3e|qZ=!T!E}*Z
zb31f9`?w!M8ixs0-J@jP4x0k{1IA@lEN8cjH>?!pGJ
zr>CZ=ZOLOh;N7{nhWH-W;99`J+q-MmMHqDy@5EmgvD2c8Jd+8nb5@CSrHcXrM4=b#
z_A+InP>LyeO((A8n@s_VjP&b9)irxbmDqHmS`9p%k%${%yGuO8
z#3QPp;@>ppfYUGL@HFR)Zuaz
zZ2LV<&12DcTXKo*vk?CeF=2uF~*?RIf78{$bBkeA^BRNE;vF1-xFXaam(pgh@C
zl?h#=CD`2Oncyf56;?(ZXZlV#Bvf@=CAiYGx}Ihcvu!P*UB$%+d3!HM_{971!(UoA>j5FUr9TDoLz}sV$w)C
z0BO+H><_h({P_M|Nl6L&7+A>wm)JW99s}7ubY^a(3Azx%hCfnZ;+=L8ed!bhCts-)
zbv8V1+B{LkyDWD)fGNOmoCT)OqO)&a(4!5EWXIfl`Dh4^c^q&UGTK?
z3j}J)&nfCrN~L^wAwcmriRCddF_fkZbMTyi1gByUk(a(n$2BjKNez6O^79LKVLt{l
z8Z<_jB{OIpuG$TuR|FE`2)&awEAQZb6Aq=MVAL%eb<^ZKC@W!gyIwJo6mGE0Z1`tC5b>Qd@(h6n|WJNS;FvYDEn%
z&BJCvCbsU~8#y+R+%Gs>FawO^r8=?7&${p?8W5cC-5Wk2519(0Uh2fNz94u8I6H5a
z5>&XOL-1IFPx0X4Ll|i=C;+k7I*h|lp4(WDf)o8AR#%|FXc0Nc8I}?)zW2F`K=r8p
z4h}7B3gfP`&xl%pI1e2X5;$%*YD>)C>BzYS;LFdMc`~YFQi%BK7hk1(3|kYxo)1#X
zvA>?C6WKd$4NNp~wyjtMm(2H6w8vMXuj?Dn@^~n)Y>>*q5tj!I-J5*>Vd5n(e}CNr
zE~~yZ|KP!!?bX}X82nt&5o?Z0zUWRk+?%S+f$Al@kg_L;skIO=8{QZ
zK_|OkVlgna4IK`joiBmn9hmse+;e;hgi+xkT0Wn{9NKO$`)_kQr6!sQo)HGxC=)Yh
zdVsN0@Dr55QlKk=t5laTRJrr^oAD!!&3k3%P;33|JxI;NFqx}K@Bjv_sHlMcIc?ft
zq)dZJj9##WFI|PM^MM|HimSC*HR0u5Jbfn&0y|6~YuC3Ce|?)rDV5}jw@mzdD3Z7(E5*e!16ig%@Dv?3raa6fXd@3z`u$jW!Absg
z;4{(&*}b^9m@gW9oH)wNH?aR;J^}LbD_Q*Qm;%L|IC23wi9J!{k9)&c;xnS_%IU52
zXby+xLDV4%VJsqEMupdOxfZJc>2U#411<^U5u#(cs!E^#_R&p{NuuuEZ}fwdcsw=sH+8VN@nqpSFA
zx>r8FbxSnHJEM?*>o#bxUX{Lqfm%DgM6MZ&-OA*pJ|yPyPns7LHnza{gh$t=i41dn
zeQWlB`m?g&DDOU&e|m;M>riZpL*!59hVAbdw7rw^wwN;PaoQ~JIE8~EOB86$i#cWRyjM6ORRPl`}07wy4Wkgs3_U0>lJ8A1m_v-
zYISt7Ppkn-fD#~{yCF2{4z}aNu+Y47=MEN)9C(HgygVF>A}X=c;Gr|y9|Wl~ZJA8H
zt-6ufCYW_@=dJ-};GG0`#T*0mMc7O24P7DcFvGGs18NR}!am1GI9OTgay_s!cWXIT
z@8YE((&%{8#DV+tye;hj=}@-%;hce7L8`xIFa>rpzGr3eY&ge+bWSc1T$ogSePB~^
zp&z0oPTWZ42TL9IQAgmUQFQa>AwP~ad{CCWw2JoS57jB
z4lGzQbwy=ev)d6ab~{y5QfC}1br<;gDOo3C#7S1eDvis}_i?@L7A56Hik7vW*o7jG
z8}H_&{he$YHqitpa2W0vF1-1&imHUL3KdPqI~v>!IU-Xi>a9lH9HMUT-s@|uyfl?h
zoRl<+lB
zvV1ong;v+p9A=yyu)wf?pk6|e+k~YF2V2*h*;@^=8g%pFH}R9@nty!#{DgIS11mCf
z8pr!rwcIt_eKzh->huNcw|a&g&a%iEt$sG{;_3yYRFzLz1RvpRu
zt>2BmPxioKcN`NNVI{=D6AC)>qwqDxaAvb!pF>)5=ZROQU%%CqS|rJSpV$5B$`9s}
zIgQ(b{&pI_EBvV^Trz;=xI(3|i(+%fdB=X*vQRwRYRT6dYaW|g&4@jD>qDF0eKt<-
zI&$u_IhppSy&g?cP5#x}wys-E&PItr@~-h4-dr_(y2G~9xchl`%j>7zTD9S^;rO0|
zj^yu5=+!bWL)_}(IxA*~VN6b$xj67buiD<5CiYHTai~1wQ;KrL=+5EGOFn11nlH+I
zb7i~Y&hEB8<}#HuQ&2<4$1dq2<+U`v-FFk5I-`b|RcNTMzqsNZTgXcbYvrTMJ(y7&5`nQe!ZNAapzq84-ROhvCnleokGd|h+e$LD9_@{k(H!?CD0P2ZK
z*AX7YI=iks^j$gNmTF1Dj!%2_HvFb3fpAXYU^Kbu-@m4A{&l0f_IxnFvS-g>!-oqC
zX490{Oh~_>`C!nlFUIy)b_TxFb{%3R{TWs))^3}2_=Md~Te|DPTE|!w`lx0t&WxEm1hG%uK7dQX2qa`#7DHH^C*ND@*`oH(K0c+p
znniXO`Ss^zA^UYqOb(kIKBTo~jn^fY#t{d=Gv3)|9*MXI_6iuq7IJ+wGK1XSoffZvs<(H-yZ9}
zT9525`F&(;=*GlgzkNP7Ta(n9Cbxf7&rb_VYPH1~tw+wARS%vH;;$I}zu&?Sa2JUe
zJ%9QSe_8k?|Nnn#*H81DH9ckVkSi0y9u^L0c=-L6*T4LQY}flfyE@vazZj5X@TW0n
z*VIuG>9XAW7Z&;L-}|@Q{Nk&+N3N`Y)-BWKwa(`qi{Gu?RJcOt-K1@Aygz?^aPh}!
z+ogY&zw7!mIe%?y_2ksF9e?&zr7OQ1g*1z-bZ?#Wd
z>sIE=2Z(l5TCNlA__D`7%j|GD1izJ$ZhU32#`+SMx{JN`{2d^+x>|MH@@I~&yz=^|
zZelN;xW%((J}v0flsviiY?sHO962}id
z(J=O-cU8`-KaGvQ3}06{|1RS+)L&d#$ebwei}m;>Pu
zk5+14P=;A{!L93ETuvgnV6hC<7XVbr=*lV
zd18@N>Iqgbc`|Gk5)Y%6brUoBVdP`ll`B4(KD0#uI!3P1?){xDwpBn$;_fSjGsoGy
z&THmOe3xA)ig2eOK{eLynK8HFZ|J+b&vNeGI&|`6+xQ26hn0Tz%iJxI10~=nt8l~U
zYW35ne?BN@M~t$dY=^)1?N>;!C>r)L!dCG#AD+S2nXu9qw=NoIQ9AL}_1h0ulBc;Sg{R=Z3Ny45
zIBzOqK>z1|O(iNABU_K@~H@WILPU_4KNP9qT=}7(Qa$
z_22+4~2_0IH3
zr^w|T$XB@TQh3TM(bY(0*zlHIaKUNFi*G|juW@SngF!8I-@1>ImxG5<6BQRTH)2fZ
zm2hQ-M)L|V3}{)*v+Gyr%&Ae0@>-em$zw^ygmZz$HD4Sn?kc7VJD)+`ApBcdTDrt;
zp3_z@2=lN(J|ph_cJQbr*_W^W-|z{|^<-0q!85f9pr8`m^QS(7I>203<$oD2z_
z_N%1@@@s3*(74E_m>B^SuDiLpHRf_-`(sxUT8k3SHI})$-@0>JYBs;r$TU?RRli3v
z%wh9p%a7iBSAIBs_4p?pP1E_~J?9$x-+$#;)FtP`({Ih!&MY+;-?;AS+uy7ImZdhX
z&YAV<*P0aC=d?b&S@4_F0JEcmNExJr
z6T<`=E}Y-6(Yx)!#~j0ne3SW-mK=}LK#gb}U0nwzKv=@iyRrcT_7MPkmDwKwjsg%F
zsDF=o>*uG+wG>eYpL*p@8O2-=*&Jrr-r>5OOz$?njIa@;y&iWq`=X0J~9T8uNF&5B!=}GtflHGCf3erpth9?4J4fc_&dh)NjgLMS
zOD*)wpZBcI`{T=Vgg1hNe5yNGc`&jJA8
zz@}|1_NUke8r*~Y7&JCGSSpc36x1l3aVek7z>#swXJN@aATXcnf?#&9&n!6R$L(9U
zSV9Qm+l5i6PdbBkJWNgYVkG$(Fq_#aQxlvBf-
zJS??jdpJQ1>u#4i3})$#JUmqI)MUQhP02*;WqPs0OQp|4W{Ye(4XSlhQ0Wv0f&#um
z#Lft_?%5My2?h}82F^rpzd_y-=ueK0mbx12LPD?zu+wX&KbzRX$)=hN)sg$#S3G^X
z5^6V3A_T9P2w`<0gY#Ls^58lwJZ>~9sA!QkfSJ?0tsD*7QSdk!?*C|PgSNVLdzph{$4)RM$AR-hy2gbF*k
z2h8#J2g!iSWUZ^gq|0)Hbay;J1FGi@jc~6%)z>45Yzi5g3;=eIhPl;P0_q4h!R__6
zK)=&WFDgMiJa_+OEv7Q4o6IJ-xd_DQTWFM0wAVng*}|XTmbQdYGfCE
zee6>5nI5!u7JFH2o9*Y<6cWlz;LISigmL>p_5hBz6)Q)rSiW2^Y{K*?S`yDHcQAUM
z;w`UG$SzFxH_xcQb*f8b-uItB(Lc1*6vW($#y}UoqmH;4K>`$x{8ey|v2(*s6h9h9
z8*jhWu#(qQSA%p6n$FDD==9mM_Ly?Rr0JpUl{y}U8Q$C=>zTM5;sNIoi8-dU($m!P-ZXQCiS
z;q?oSP|_M6Z}y_fL5{Nluz;RvrqxAd}m~Tm;yGlL~2_A$44#^<@4Oi~!4SEHEjLwfa!ofoV*(c%!}x1=}Cy7whL*$bN+k{rN0x-Dk~!
z=U^=Wt2rUROil!N?DV|h;^`f&`3jC5Ao?G;QsbUV%e@)a0>?9GH$)oIyFSi+$KQ?n
zH%KkvAP%7;LjGC3%w%f4o~Ge^?Jw7RY&@X<3%f}nn!+HesscuCUjI>5>7!!XZ5V#S
z)aetccq3=>sOKtbXNga!Y?wTKZ9S;wV{eeI^euY$@Y}_iF@H^6C-l_4&4~rtgdo)f5fp}sEKg}%kGJDi$Boj98h0qW%b|zr
zcIM3=(Z{)ZBqk+&88Z9pIBONlySw|CLbJ8834Ao+QCdUGI_2{L2q>mZ88LFCun2Hc
zpSsRx%j5m?wRUaqgWlknTwVje6CjELiBtx*%0nff=(T%8_lkrc_8yv00%+x>#k1G0
z6^sJ={T!-T2nqts2K8skuwiq^3QQ_apE`w2rJG!e#Dpm`@
z16Afou{mNx(JKx$T5}i_Xh@a=z6!HU^cfE2+L@9ZcQ**PheGl`h*-YfzC5F
zb_!8TxDZ9kt4^n+$Oyo4seAksN}F%rz9HU(6g0%a;l|rT3
zIeNnef;J{ui)6A~*p4KI^g~HU|3%2ruitV^pcq>4@LBN#R`Z|aW+d6k#VEI82xKJc
zT&Ti8?fenG8S9~##310sH;Cy_KYROs&(3MPXB4+GV!|+r$?$MjP{q}&e=#a$`p@W#
zi?#k>s<}xr){5*Lg$>zZM?%Zcs3(sf3w{@;PK`#EQ&O^!c)$%`7!Z)_m&sIt5y|iG
z?*P6rwqY~m!him`PX5zcslmcpPyjGDA|)RRZ*oZ6ph~xIdtveb10Hi12K#KM0>le^
zeksYt2qs=P@gTguP*ve4vdzYk#2L`x5&t6FO&!1246xR>(XI))@Ns^wvzSD)U
zX<)38d3R9U@CSHqqyr+sArUi7KM`VFJ_qwL!)_$3)x3GIOQsPb3A;E7=N{j%0VlR5
zEKqRBC8trAHZ$HLG)Rs;B=3f%sDe&&mN{2uLjo!1PGi6JU#(#qn-tMdWzC(q2>^2
z%yj1SR!G}JsnSMydZwkP!>y=dRz_jOY543R2%6sBO&-o&gzQO#tBS|tw~~^dkBV(G
zB`X`42;KOr@GWO&mcx1HRie2g*w7_#*w&4|G?d*K;
z?p)Mm%vS$ryBQH0y7?DwFi7Vp>E;rDR3nWUxNIop9)yQ0D$98c42EO8id-yrc;Wdo
z7s!RJ8+syvqtLc3h0!RBZ}Y?$GLnwS--3?L-yW$yH4!hCs~Ih+VFJs46&e%VUdf;|
z!pvIMuWv=qmRU4x$`s+)nCXo0Hs^}2APOKeA|mGHxrV#oQ9HsU38oq4rKJS8A2$`@
zsZs|ob<~H=tKWXW65;I@`!h6S-#yoW29F=kC5J1Hk-?kz4heChzCSf(#E728#+Wxgfsot3|IU)v5em&}qFy00y_>ED
zG=5TcoZ0$0;Qr#O06sjmozecnS6@=Je)TSlu=m9$k_W}C&@*m#tY8sLJt$Vgjj~=`B@5j5d-`Sm?H7!Ruvo(MQ
zj-f=-!=T!joWTZtwtxQgX(o~)dH~{4Lg3Bv>%Tkq(*5mD-AMgWyw%*aZ4aGTfzX7u
z|Gu~KiEdFmJ8~Mj2$i|543J6(r=VZsm;<`!UE0`DSal$jSc5(y-2-?adld!fI8B}~
zVJ$TSqq94A?=m2Se=4i|J@Yrm7=t~9-_P|5`KeS7Op)e3T3P)vzMux*_??`YZQIUK
z^sVnRia7JXd|}!pP9oiUlZXE2OHub=tpWD-R<^btq4<(x#W@xFUD^Qp
zlgqP<+8-fD+`vI%)oN-5Dp^um_(8pPH<5b^)T?6;*LU}p564B0GtURY_9l
zwHtd@8fU219@!)0Qj`X$C4ctq7<(Na^@r|lkSs=bD*
z#wcz|-
z3@T%Z$@Z;Vm%P1akg3CPlT^f&U7tNZ%@2_y@p+&991SOLsZ@~SQ{Ksb^zGNLSI?fV
zp-hgMal@4wDie41NnjSrSLR7!T)upelS}EIL2$8VB
zu&Y?yFt$%ql{S+t!6q8Ha#GF=wh}f0#>cBKqMYBj6ouPE0!9k!DqSaAu42L(7ZyFwa`e2$;XlYvB8X
zWskt;We*-i%%Y?K(F-3*S`}hw*8FV&1+4w8NvD&-&ToETU^{0L7aRqi^MO$o3EppF
zwYx^^c%?|I)~cNr$}gG&R&#es7Jw1{oXo}2@4$MOlBZs09etXNv!dhr4zsI#N!2@VG{K+QlAdUjgei$2nB
zzyP)QBHG74e}r<(7WTs_5I)^g!*G6}xv*QZaYO-2gs3f_oSuP7(}AA(Nbi(&7~eJ>
zTlZ^aRPcBz9ePE^$GD|Fifq5n7A6!|W{;x!TDP>)b_)JpNiwwZQ_vtcWb)01yPI$oiufA-Y-KPGGP^uz&fX^!yG0tQlr{*@CtirMgU5$BncGrCT
z_|N?Lkaoof5;D7*Y&wH$rK++r#ZcQW(%qqiGoH@LlOqG0jsI28e?|XkmzHd2O&G#^g$~cXrn~c)zoCr+5!x^Pbd_k*8cLG70v{=KI~DV_-3U>DiJIleBx~DzTmAMi^(jUkGY}%9D5sC^b9f2_
zXfvQCUPfk;&TA)Jzh8aHUA_DIp&Wcy2sgAU-BoK`Hn?pUr|D#CH|R&^+3gvXrnKfQ
z4z!MrtDy5*Di5;VFj&8RkKVl#g4=AkgV%6Y)_0;^!(#}r=qjmkH}*nhfk3@xu1Yt}
zgZDq9N42&^!f|-I<6$iRL7c@Vu#kn#kL&;H|C>SItQyG;Rf_ps6+Xdka}Tiy8vCl&
zc#@{}MGKu+2MR-o3vf=49eexk9V=-^%BA8ctS!(Ggzwo*B7-i8s~ELwX!%sR)Kb+v
z+(khukwx6;pcLM)O=O@RLWyX(g0wl^wh@AC=@L#hG-F>oz3OY05~MXO`q;V^CQ;N1
zllwK(2JMFdryjatc9wFq!0f{$m(&h)+$xq+0a9~;En@SqCki_R*763c>2;K+6zaHH
z`YcE3L7%K!G08nvB1heu|7(1fPoH~{{S9v}Bn6}920q4^2um<`0(%1-eInPe7RY}g5_$lop
zuUdAABs=61rp_8Te|QVeanst|rLwwr@{AbEdPKrQZn10A*EG
z&i_Z%dB^3v|Np<;)lgSyORjcSLJ}o)H4zGj?241@?9A4cHf8UWm0by$m82Z2aLBBT
zBZLqtso(wm`Tq6$+N~1G0xlS~D3?LuV7q)8;
zaM{ZAF(&f?Xq>Kx=FRXu^v7KKHPSUUfO2_#C19D5Qz6c3bh|yPaESnp;%q1m!5Hd%
zDEi;*=vXtlB?&w96siLaNa1r=`sovhGMF5Y5-kpa1`;Q+CTaLEY
zZv9LS7xx}}+`&Ok%L+(=h$&RP|1G}290cxY{+mV@2Mnu)(-_4i5`17}^xXG`b|_8z
zvms(Yt`e34f{L*#H7HD42YEmto9XGrf$Cjjg
z=SUm8*9@h{o%)7`%V^Y#8V9$pRqD-J+esoV2EgkLEjo(&2&{=NLaKKD?Aey)=F5x@
zwzvOGmBWQXLqm-Mn8es;!u%)a2FGB#?355sx-92zh*q?CY?ERBc=
z0x&^ES>o1+@4(0B#_ij^?CrG@NbV+M$ViWJ91=iZhBMi8yjr3Z6~>9-kaHP;QXgR#ob7)>M1Lbyyv2YVBJlRQRh874&_ZSQ<$E-`nkHWq>K
zC2rdVo-xHiohQl}9BZQi)iYS4hXs
z(_gW8vGkm3Gs>7RRCXK#5rMv5{Ugq5Xlg!Z&n{awAX3SEt2B>QM=A^FJO&NJw5c%>
z(|mlwD89oTTX|fG9inf{B)*!NSpx(|v2@jN&-YoP?wymm{?p+jUdQprI;H!?Nl;>!
zY7#tl+O&35vFsrhX`_^{H$1
zDlHa{ey`pDk62+!#`~eUBG=+tL}X-;ogRv9=l?lsZ``^y0vT1GK3`TC1zAk1p&o_8
z!Fh%^1cw4dva+cQl_iqB0K13Oo^n~R58S|Fvzv4TtXjm>=H?zgJ_uwOUQEcWGk@7T
zogTe=V;EV_=Z?O&FMJ^3FZ3oAGru7xUYGCUmq#DQ-@@Movg2^{!^e*$o2Cl`Uejsq
z>K1fQ4v2BDWt{ZZ4PNz-DHMh4pl~>6x>+=R9N20uHJ|1K)kxtK&6UKa;*0_Yy144c
zoGQgVd{nX}*@{e`?Tjj!Gk(q5wOl{s&JbBtQGb*uL~3Ecsk8$JEc^Lq8*%uxp|xM>
zI*rM_rXgs9X^m?&dsth;46k!*s`?eKDf`y422%6L5%hM-A3yvOA3J!E!4Vhvo^Vmu
zlV;F7(B?+B1c!uBFlFNmBHV*|9M0r3%ZKexC&TUooWv1Zh=iFEKG=F;yN{&(C8!=J
zJmX;xXJ^Njg~HFY5&RRr9y%8TIzY$k*RK(2;(lpz`5EB#l1+}V^)U7bQ_~X+f+sdi
zH%CanS(C;?hP~i+51TcM8r|4{xkR(G&{BCLC#`@zb$`dXY#Z
z0$gMZneFUxpGQZs1D+a6*`Z!`sXu;+)!SlB_%)0+SVqQiaM##?#<%QP5}x%xzNbJW
zmz>kz#{xE7U?gd|dKrwEryZ|DB+4fY%ln3`rutT|ioPwHM((wo?IE72`8O@mj%1^$
z6lz>F#7sCiG=8}o8DRPCw2_p%hPQ}gp_+ZABBkGE5fe&YH%ViT5d&~3jbkVOps8j!
z?rNf^qN1W4PB5rd$h4!aqNip{jT`rnlJWSlV{%y>klfbdFJZL7+)CVkV+pPji#US#
zpPuK!_;EX?0t>c;8B64P>WO3rhBeil*|*OuH~)K66EhKY74O0GQCqWgX^Q}_k@%9z
z{hl~-ocS;}oO{#^2)`*T-gxHtQRl!gwjYO&Jo3OpCa3kQMo~Z16Z(G*V{RM&H79MT
zhrZHKwiB(X8TDNLdD;F^HWSCWg}N4#w$ahmRn_A;pUK{+o&WaBm($0N3C@PhoiqGoYK;;mn}atR
znVG39KsL|s8bN&V@I$exx47A(Zp3MHH&ACB8IZErYZt*5T3QN9oPBCa@YVOJ&CIhk
zo7JzKd&8yw?%j6k2SP->{v}1ggd>8^gN!)7qh(ma(1z{>vl_N(D8({WBgQ=)i+)3F
z<3dmuei2Wuw~c96BcojIr~eHn;vm74^3T0{2@8pLN9{s!i1mW`wK7qh*JgYCtmKim
z1LkDMnAP?}f8qi|DC!ALBR57Zau(6_+y}gIJWr?qI9*y#dMI&XFirCppBpfKhf#LX
zwMdUB5@>?e_YuoW(Ot=?g6k{-7N8Jr$sL=x@*2m*4*xI~5KsWj5SHydzWgDtD#eLe
zTege?MVK_{DG(MJGxbYX5ecsFF9;G?IRJm$_8(s)MpqHN?hnYJ6+u)Vap5`DcZo9h
zCV+^`ph5gfT5^Q!y{T=vE|EluS1K=OtP||-eT4mtsczx=9(rbnB)~#938}^F2xmTR
z#f9NMJ~k97T&vl~lvGtoxo)$Rh2~YW-F1`nP%Erh=5DutfSsLa-U9X^n&<2R4;Gz^GAt^(_wIDUusY!JndinaIWMC3yUm1X;j=0D87u4)rjDP|L
zDNOhqN|2{+U#*Sh2-Xcq`F!){HX$cpJM_Hm6xQ@DcLiOlt`4GqE$r3u`#0^Kv11xf
z5xkW0ymkWSe<-oDAe}Xaa{?LtR}viU7d}pAvC4^Q!u#CZ5wI3CNy3s-Qz=(lA&5C`
zGoqC#5=izKHge=A0`7ShI9{{Xh|IKlUceAa2vfMbriv#@7M}wGeB>no!*y}#;x(?Q
zi{M*kY$EQ}D_5oxo_u*s^Vy?EDLi(%OO%w9*0eQO;r{B?tAPa)VfpFfN0Oje!-6Xr
ze=~d;j=Z5#yLjoKh+22v^BR#NOa$tLBRM&!->T;=4-O9Iu;phMnDEDrx#APMAiY{PhaP((vc{q3+e_{HOk-LYf5;uTJYjrN<$`8(AEY0EeJL^_W)Kn
z%V2z*)!mzo^Q>VgjloNJ{zBe?Etg;RcvH)Xj>}c+CWtijOieYHe+wP+ro-~XbXf&+
zrpn_2x@9Z5l;CnfOb2hE_CN>$1vCc6c#6S@J0AHK18JC~R`c>D(@&zfwXp~?jWn{Y
zxL7XDhl7D%$>0pkavatal48p_4{u104^1eIiB7lEV=oA1Y6x0vM(s})&I}3BA%})w
z$bW{(ilr}^ml69F*#lI?T;LsmcR!^UH6^9MF>xQWEer3KzyptbXm=59&BbM3c0z@Ni-_nFUjNc_aRB6dFeLnMAW@zo1@}BBLeyxQuf;MH5|j*A7J({}
z@9$d&UWVxePxdg|Ps|CDQi8Cxl3WJ+Y+iS`gbjnM5C<)rr^2o@L5LbAuP)!^+GxsW
zj$FkN-T07@Q9N%dwK#*jj~^Qrcs5=Quv6-?y0*6-m0JO`#j3NtM%ofly>6YGtN}wA
z;d&1^FVpAxpp{y_+$qD4%!urL`=&wJvOyR>6xDLu%i?SFr<*wgjq45f%#chl_@;Cs
zdH)J}u9^LCD83e#>%Qtr@H$tEsn+9~8tqfQ?3|@PxOQ*7%?~;O^?T{U1;H0pP*NAc
z0cxbF(qr+86`2kpQYhbfyMptZT^{+^Jpht~Ogep_=&nXIfl_#I5!=5c
z7I0QoJ*NOWjPW{>acJvO%6|U(HP%kg-LByqLmo#7$#{GP@+osSq0n-;ry?U~d}Eh^
zpYa=u|LZDlbrIe88?+M0E9}e@S_q^h+(}%4@h;}!YoU#Cg$CL^M2kgSxz96j-wGyNSs^|CnA?%lg*=ou~~PWKjUqVwlN_TNMoz}!N)cUSF)QSubAK8@?+!+*N5a)Q{D#d
zy#>Ee*iv$`>w@vw|1`m1LPc8URiG#3b+FeRjL`Ai;dfCi(N_OtwS&GyQ5qfAyR2CT
zB3u=kvC6#A$*4b?E^7w|vCVV8Ve9Lo==8w*HCGPz(w
zco#u3$8-5Qg^D5%+dk0w=nqTlP-M6#$}MUuIjpncNNv9~CJtMx?MKnUl0mh?O(x4-
zkXkJ_^vfTGUSN{|;B;=IS~@Jww|8ivFhPwQ(P*GwQv7oN(W68GjrE$23Kx{(Ya~WT
zb+e6h0;sR$Vw$J`kpfC?%jw0J*8TiaQ&U|n3V#3jGuc$)Pyem%yD&sV?womKcME;`
z!ZG8=`$TGd>`Iociq@lve4NUK^c$YDxhni^#?tJkmapsmalbk|0o|CWR_*Z*3Srbj
z(OndF`d?-9(ke%*&2zeRTLReBrw<=mhm|&b(OD|&8@h$TM7nXA`);kW
znB42W5usitw)+KudhpVUnwky@M0il-aMYh4KYap?7Pm5Z2R~m#gKx5TyMg{T4px+0
zTKo9o#8)#)`z0)f=J{h+b=C;afV$d#J35!vKI7MSyLjGXAXzXN6?O$m`Jm|
zk(c!{AoIhC#s=XKtBw)Xmv$dJ_5q-T!ja>HiBw9A;ccCaiVM~s#Xndn0qoqlumvRY
zaOXM?L%T!U0mK_qv1#}rL@K!!UChjc{A~)F5hLc#o}IB}%fSG0H;RFoTwo)!DxJ|*nE6We+Wwgppeq%Y9AUUo^|4kb|E)%NVN5FEg`qeA1mI@j-eOg8!Di=Ig%@Ocg|ApKaML{>#?Ny`
zW0|K&U}Y8+MUfpvIDEXaaV>qpA|#Jo7t>t^>BzLSwWG8JE^O)Gy?4S!?{ZWNZK07P
zo0MW2#ul9CY+pfmf8ZE8t91gC_sAq4`~BnWjJsd&Z1p%gW8RTW$R_)QuJ)c
z_1JnREbiwr5Y=JBZo+YM+0mzP&v?}Lv8_GErOfL*kpPcJL?*=@)GIJm?u%|{^9l;|
z#Dg6iDjz?#O|Im}D(90llY6Ljm@K=h9(fD{HCmgS6TM%nmd|URJay_HB20L!!obQC
zZ&3DDKhi8x&?ML%bQj=`az|97qN-o#Gm
z?);FMl}kZdsEB$QXz2YA`rj#JNK8VHR#Fmju>HS;rqakjk(3L|bfa(BYTPjEEz`kB
z8UK}=i+okFukXTzrvvN=r>7F(HGp*U>rrHhqy&m%fq=A`B8;$4EiSWhb-jG%Oy`Dc
za#`GV>kzF=U?Rk;2D`eR0D)p>Q96QmGF+P|&qCLYBZIayc6@naG{mr7d+nDA&R!Oe
zCj_2t|K;X2uiX1&i}#H0w7%oUjf<|i5MT%Qw*Ok}3P!>pM9JkM;1=N~Wo&vMYis4p
z-7@pCj8X|uU*N_?tvs=^dE#$MK$TUd3r_FJCZ*x5C&Zu!9VH7;qw
zU~h2NY(>DQc+*bh(VIw>!pRA7@BTYkn#If>ssQziN2i}^S5CvwtG3hsMrfW`N9Xb^
zu0<8Y;F)F7qsfG0W&8aq4k3}eZh!5YubW5kbe@Aza*A-4K`wX??0sEZwH@86pzdD0
zI3KAoGIw`5t6hH&Us|cG;1Ljz4o1qPa6s(jC52*bX346)`PCW=)z|=Dtso7Nd*Kb1C33
zjs=AE9J;Yi5|EN97m8h$Le~&y0jlo4EjBPkzmu;pGz94g8yb%sK>4-VD(W(^Ay?VH
zfeWVrD0tCgup59X#(l&H~YSowzhU~VnG3-
z&J`T_D^{FG!wRiEapDu?*UDwrbQ-&DXd4aALkRxHax6axb43S%A(@d#QQ22Ua$?o~
zMZsyxqWCMX-)T<2WF$p_nB6F5Vf|ryrR!JiS`N*Szi@ywKb~D(4c{kXaJkIi8JN?68=H~rFSAh~7I_PRdUF;6U
zGq^oeRmqZJ{$KF?`Mu{fw|o&@JIXDUmholPJQ*`ak*>YGu%!IOi>|L$;l770rtGy?
zTZ1T{s*Qt;+DWXkVbTe^cL?v0q8W
zRia#t(7Mqia{b^PK6kY{RN?n)qX9^L293jm7f
zbzE2f)ZDaLUYb>%r<1%jU7Q`Do0A#m5Wd3q@TT8&y3tA*o&RmNGz_!a(dF8oR|_5U
zO%GZ`da!~1`G*{s0x$@a*E93vbo*8(&N-8@TeJRD$X?u1l(}fLq<7^xh53~<2HfZf
zV{LN6^CG^sPdik!CtagQ?iN2Q;@e8}Oo7(>$5EVZXp3lgB3EY7cSg9Q$;@x=Q|hu-
z1V1`{6i7T`tKN6}kw%Vqzn4m`O_=j89k##|a3BwM3(`}XZ5QE-0r>=|PuP&99=EiNwR`gy*~>uo>Fl)ZF`9NJ7Xq=e20
zm487+LG^tOVId+&^Fc=_r3k%9w52ctPo|zXe`A>+CS5K5Wp;hOyy^LM6(PY|p_>%P
zD&IYvxW4A;(^kOvni1VyKiH^hd(SAd)Y%GO2JxM4X9JQz(kRC#!VQ)4hBN=|dN#<*
zNAc?ndYb5jVs)>FqK`qe$~oqT%M!~%W8&=d@220JNvc(LQ=sC|YyWV0QArVxf&`lb
z%_A!Igg1qtk^&huu$m;$+yHy|Yg=g@#}uoo9j;9I40S$!4nTq@O^XYg}GnEx$;3G2c`ChZ^
zxVBblq5)xj$>pynoCuQd-&s7E-krC#!7JzfKm~2(gQitGl!qo&Dr*>=2+FfSA9w1M
zR*de_l~&S;y$oUlsys|5Hqb~zqmhL*9!_z6B?BrGEs-KUM9ArLfW`s(q7l<7B6~(}
zaIKA1QjpiN+>jiQJEG&UDE@t?Hr$T5K*;ywUNrd;nP&7dWr&`q-Mj+~NQ6eHU9EkF
zRE2U%Yx-bt=!owgZ(h`R+QoXL*TqILh@5MkNjARo%8O&yGR!(L`rB%MCVWP`_7)Gf
z$f%v!&q)d*zpS=~fUu4D1J;n!E07(?*1yKeefHIr|8t1gI=w0O|&oH9#1pXW}u7s}p*~2GqLF+#`S2Qq^0mKEuHXttXGB
zqjGyP>wb{=mgvF{(f_{8>85a;6dHVXNRt?V_lrZOirFysD*!^H1%n--v7D`b7DJlS
z;5ec7D(O5gSjlDeGB}=}ze$?qzuT!dt~=?H?=5X~&+<|P&AgP`C6ZW>YSZad%$36S~CAg}{Cg=h>2aZ)impE%y`Md8SiqY{%
zbxDH8t~!~|p+bTqMInwtNOt4#A$G(4U#GRbc<`X~-FF7c#>U2GW=^jgsQa&C@nEe)
z_gC?3nF-WQMNst1G`BE-c(p{+bBhI`Fyl
ze-w_{WcWRL^vF+ls3dV&Qp7Zg_;%*dA~q_bi8LS26O)VHn!-{+Hm?b7MI^xA~06F#e7P
z18)weD}5u~QOGB5WaN;-CRxPt2VxCCiAFgPc(Oa4&#o(IbZ*
zAHyAY;pViO=;5cCMP&<*Mjb?F*xLHzxSce_0wIp%qK@s?Q;tg5t1c?}!BK3|8$$gKlcuLbGdb%3sHY$xzbEJ)exaQ7XK68H0)xJ63okO17Y6C=;rDbXZh{P*oXQ
z_xRhC4sT~RBys%+ZG?GUA5SCSJaM9dA1o?O@8oKJ#DC{vI3@TXp>e^@O9VF@vgW|z
zPDUQ2m5bEW=olkeZc!dNGMS9$PmM;46_psULk-sf?TVKhvlc*Uz?|T*hV7IuCJ)%U
zP?Y9nXOP11W<}@q2!*N2-7+&bZ&ozdHGe*^-b^fu^;f@sHTY1)>#um2igW8{IctjI
zT~B0-;{WmH1JP3M?28&ieKm;jW3knh~3JBNdiJ}_e
z9Vp!O=sm8-GBn_!lj#IvzC(tgFqK%!?UTfO#PSRp6@~Vr@BNPA?>v8#G!;?2H7ky(
zrZK+vQCrmzcS(4~>d45%vJ?YvyTawAH5}~p@kn7=6ubjE<~YaBN(v#AI!JaXOp6@@
zWzy%p-0iMg9XopT0{M!u0RTKbSMoL3E$Es^addaA1w0|w0ZhtMwyuB7w&uRl+@J>P
zB)s@cr^|F*so)~7mB4+Lor-!E5Mx{~SxGSd)@1#j*kyk5#?hD{BQn|t0mI?MxR~fO
znIyf^C;!(
ze2VVu`=LQ-~3
zcfzDet0<_U0TBQ}BGE@2f+>{9A@ToOW-wT8
z7w0@{5o)^rosRc;#KF8eig)CVx#YJqcLNkp5ScsZTcv8M#Em-Q8R{-Jl&XnaE)!Ub
zSWJ=z2g^k=EibeTAP%v$onG?Zy)O_ozIy!{Kt({zGc*71q@=@P+8A{VV|T$(H~4Mw
z?I*u}KmU?58C@#y3avRCk_KKl8;m+7X3Je0l^m9l!R>BE-Qa%xE>iOqfEkI&t!3?s3jyK~+HE!9PNH);w}`
zIY<|YnMB-q=}Za;sI99DnLWFOZOr*fRRQRMBaq_+W*5C4OS~gC01K9-CDh2Th9_}_
zPidlQ|R$Jx5^Qq)Ku1@^&~xltYMbiJS<+`E*B
zFsR+f9>%?*Z)n(o^kZnlQT{1Rt<^eAG{{e}3lM?81{AAJ4-BL#NjkJg+*^LnZK8O1
zJ1NCX%e>996Sz9B}^O&TXGnbGY
z-j1Lp;c?e5y@;Wo&^g`B=7v<+TY6u
z@0~V&ywkD|8SW
zO?+CeJ=7ft2vLE89dYrp2V7IXaPUI&H{6#^wz
z%d7tVm+4kC$|^~_QCe)881_*2(dp^k^!1mqNCk-TMJAKytbT+qSir!02H6|OxFZ*p
zJ7d$_hJ!0KWG-|2rkIzeXOf133fm+)mVq?+kX$BFGDB;P50~OUu+*
zZl4aK(r;hCs%dF4W#$YBev#yi}`tB)}UWRoK_$fOyyo^Dk%lUjOLq4hZ&fg_-ZEgb*jD8aQK
z8m5wK%t1>sRUeEg5Y#CDxzIQOKYsb*+os51cl7QbX}a+Oczn;etP^ZRzQlS)Dxil6
zuyOBKJbjddGI*JVg#{uL=BhL!$0K~8YuB!Yn>{y5x+E~XO_)197&tZ(P2BYmb6iVgN-F~%WDjGmtQGA0!{|C@AY5BCM>L$n>JOB#(o
z%x9nk^{=}6q%&ciyd=Lzu3OL%AKj&wK~D@i`IDLZB2oy9p50THMefW2+Ddl`Sst9g
zN*7*pmZVoS0}o1`%{cRf?}vU-_D8OtK3ZA+9p;b~%aRK<
zJYBvr^U7!27_sYz$2Gso+e-95=U@8Oux9!4S+l}Q{%Sy<8eH3v_h_!&NOZbgFQqN@
zT)zA#bts~WzRvSG8r&o_UZpM~^0jT9dqG`c*Z_
z=WAq}FlR}tD*zmyDq({^V`10QmF!W9-lM@mTm0$V#fwi_l*}B^tfO=1j-ji2btcKZ
zwKMIF(IXDjikbOLXN)U}L_v2sh%i1rZ&mXWFHUnp0Y;7u
zzD(c5sjge*J;v6I^z*ap)5mb};_mkL$Pc*k%Ujn9#`$k*%Y*q4<4;c$eE)Xmva|V4
z2&TKcPEMDNOH=nA3l`j>${J?$@xG3#YKIaA@uOL`rK^`ffIC?-GY3NZ541A
z=rmYWcVjsU<*n`#|2@uC%p3E!N#L8Ss7Nho7X-T|&PVV@QTYp!bwuy92Wn)ut)!$?
zdK)h$4W@WFSWYzmfhT@5hu@E-mfH5elUlr
z0sJsJ$&k`q2Auq9{Q)~yUC}}%Ow15C0tpmx-<5+KIieBsike}RR`YV9{;b6!pEMKh
zHbi;ttsnb{1~3DU55X3KJHBSYSZ_7cpY-;?!yISD(fY>5>mnBxdF
zq5}|VUoYTuBjE09=oxOwomQ6@UX+s{6WCo$3Wk
zAYEOpw_9d6l!Wibho1S(GzFZQ8s>Vu5auoj-QK&qFK|LOtja1L<&Fdm`Lo5I{;}ex
z3_|1NgQo4xIpx^L^qH0@zIQtPG0}L}_H8|V`U^Ctyh6Idd_vKv(kWAU&<@|JeEIh9
zuAHzt31c|eR!t!Zg>la_hqIb;IAFPZ4*TU(W>kCN`0+puF|POM36>5X0zt=-78)m?
z#X3#DeDXYQ`Xi@Kp=!w!#kV2s-^!a~?7X|BFb{?gnYzH%
zA8^#(!Pf23_J!8u9UM7?&<_Qbdc1;2*x{4P7zv%dV@{P;D?>5Bef<}UQAVT-lkaRx
zOR|OXTgc#4~O+8m>{{AWDV94=vsQjo#;6oiUN#ZbZTT^)a
zc+Vy{z)P!L-MXqLbpOn&BNkI_cq30WqCnmt3xRsN+?e_wJa*K2^j-)A<)kI5s-L`h
zr?xgWnD0Y%pf`WL%ZysUU4z;rc>a_#VIzkRS9cZb>LPA)9WsP^OsA4fDt}*O)()M9
zK!XH1xYzXmh_#pe>l)R*#<98Be8Uh+M>q4n=6z?^xJA7=_p(%DXpE%0sBPP2d0@Hy
zZkH@w&1#;7nnx{MaX#Qmc7{s5{T`$4!=`rZlN`<8p;~(|?d8wz2hSLcJ!}zqQyIvSgpzyKt_v=EEZiziAM05^Kru=k+o+x3U}Rc5{F7QM{I>8xg(#?Kei
z{`yrqj5OGvW_gBDA9lp4^M)7+6m#ZMd=Kv;${vBlpLbVnm`F?R%cl!9>4DFwY(qY6
zPrDL6Jf+Jw3h3$h$oSR31J0^sUe+YsLC?mAH`L;L2w~woT}9?yeB(VZ45#sY=;Z
z;e0A_U)xoNK8wwJC+&-|e08zc&k37|fXA_EHOr%^r%n&YO&Whj4IjRXG^d3+H0TUi
zTkVp!B_lTp-tybpOpS>ka=KjDG&RsP>&kf-U4C<6Pjqxw3A+@p(JBTtFQjifG*sG_UOU(A
z+0)O1O?o;6-F3gV&m&GnAd9F_EJj@#@ze8qdX7fVpM3*Iw|~AqIwxaj_rX&OjH^@9
zi#*1T{my|(Jr70cZ?mE$t;W!JR6}y^1v}Xso%!EUJk!H6#UlP-!KF~|6y4Sre{a4x
zFZxYTvDT5P$;tK3l1b9-hGiqR2Pk@dSw7)Jx_$kIsaLv+uN%hOCl9)E`cl@Iwt=Dl
zCP!xJem@<)Y{+6V2Q#EQYTHEfPePYvihqw9wZ`b`uW4_Uak`t~hLHuyIopeiL)
z++*;*5%zmJ%(6n;KP|BMr6GR$R`$JoQ>5(Y$H&_Ky0Qx>{xYrvdsU&3O;<1Ve4;2%>FbMDqLVp5KcYcg8$HS@vVJ$}u-
zho)c8`PLzk<}JQ&Eltog25Au{w6<5jm^~!$gJ({2X#A+tI-BMk9KWjP|DJ1eM>j9D
z21EB)dDsnJ{dQ<l6cbv6j
z;5~wY+PS}WJIL=cC&Bva+oEB|TXQzHTPmsE0LN)_3yKc2gBXn{UwHqR+lK=kBIeP^
zE2_9h)GUlu4qs1a>6PDRbMnIhw+}mYbvNsF-n?pQ%xC`B0k_sKhciDtd*sXS?O0Dt
zJ-^*$ESc@~QDU9o(SP1uT-p@a?ayi3MF0B!4uAIf*Wa~0vaz*3s>=`a0{-uc!)~&l
z{D9b^?rV*fwDo3SC*g0pb@W#7kB<1J)jcY%o_c@4P2q!Qc()%_U;K%+=XaVnbsTw9
z((&ec?$FbXZmrv`b|s(O$T#}`{$G93Ccj8m4@vWd8(sRwrwl5%IHBplKP}xxUin*k
zdeg!YK_-<^zSBjAc62GQwnGelf6vj*+GJU9eoAW}ijOf?Gp~I~3i$eW`#w?psE*UVc0XG>tVLTdpB<{$
zyjG-a@h9_0hqmFWb)-0h`K&2--R*OLs^Z6otuMzce7B+K*_gV7%3pmwsIn0*p(LL0
zsQvfmfL*_*H!kb=TeGcfT8(sBNP;v=O;&j_YHTl2Qu;N6B{JPrdACBXZJn1(n@&cR
zMyUN*wD7pNwB<{fbo+dF{bBt^mi*lH=11r(&8fo!SF9+zHH>I_SC($T?4e3o>7VP0
zIc_T((UmNP(!ahdF`BPv|^5>p!tqGb3k~{G@tH-(?WKRZEmR
z-bxhafB*UwZoUJAn@ZiPrJ*B#t4iaOJ9mW6i4vnH*%x_n(IW`GXS^NuI7mSpyVLN7
z(_LCMOv!r6PRyVEz(!m{J>UHM_dw>4-5cOo>a&7fma1}{h>Pm?unRy&V~QL$w@ICZ=u{3(5--@G7$lLz*`Kz9FkD#
zAsW!PZ%AGcw}Di2=jKfvZ9D335(3O+oZ`SH2deas!TurbS@Y}jR8Udg+2)&o$b3Gy
zx#1}rHh>y>oMkPM{KR_WhxznCPb0GB$vQ+SsIjMu3oogcs;fw&Of!KaE4NPpgbYc2ON$K#0;
zCs^@JDBXRB(N-|o^tEc56&SO8h=urph?bv#An?nds~}RYWqG37F9Kzx1In2moIN1)
zx(Fc`U&+_U2W6jyK!JmK$!kJNHDaJ~T+IBJ3zsf|#Nxq%^QNOkb&@kqSEjC{0FpqU
zBfFL@D+}O}u*w5vVSoRfq`HM)ObYm;jT^lz7`8wN{DG@+$Iz@0!N;QDPj1v~7&QRc
zYLuv%s+4C=a^V36lRak+*ju2St<)Su*i82ofniR3=?nc(r>4%
z6miHmIIJF93K5nd#N8N<{W+`BNlMzn&#c@fZEP)*F;5rA_dDM;8nbV&L4
zjyD3<1Rl%#XLI&4_y{rJaK^)%oy-3ZZ?HKnv5QAXC$;AOTo#P(a@s9qeD~Nom@#@Z
z(hUvsEHV-!pS_q6yl~mC)bz<3TZjMo_EsaIRLDaGP-7`<|E^i36(7YGhDjFIAsGIl
zaiu9WWgDk;Uehn7Or5CKZ{JYFA3b%-XHDl-V^XdBs>7OvOxIo$^AB=Q03LJ9@-E0Z
z0_-J~ihK_mWll3#7aft(*u(xr1^5I1t+=d!)gSW(eIU@+e56$;PK*l-{QC1JVv3}+
zG_)u9emMb=Te0bJD=dkK*wJ4Ycs2^TeC=2MUfu}#iu@(l6LkR05WbP`+LuWJz+1fZ
z|C^)9?AP%pf&?GOUi*D{l}{0taI!2UK>_BPc98&(h`E4b!lZ8;fKWrt3mtiiI+i}*
zGWg%}MyT2rf;7@04;GRFI9ttz2q6<-fDZlpd-m5aNEMuk04IPj$zAW|{n^l)5IpYG
z0`5+j0OE@AN^>`HP^fy|1D8kp_?3mhiQ=^T3f(e6k)RoUmv^&U4wnJGip`am@h(GD1tW-Pd2O
z@I~9?vbGh&^s;G;rHN5Aw4bsAXBCq&BNu@%*^b)7#V$~4u>E*$z)y=SEXFq$W$vHJ
zJoZ+z^Qd8Bw5S;J^RNpM*bRR`(CvlK
zUp{@h3X?z3ZLaIbuOqO3vGw4x`$K&`U+Bef;*}R*b}<61ij0)VfRb=gQ1ux6+;-7Z
z7_bP#Kr$uHIUOMs3w9~^h6bb`cCT&rQ+I9IBI_+Ci6->YxI0|Gxdx@j!ua*%RPlG5
z%gqgN9YbDd!?qViQAJm;nu)Pe0Mw>&7~>kLuQXRxRpo@oN58>jfVz0_s-|#63p%U3kiQc(-=v^W5bjKd&EIZ;5)S!&b)`tC?`przzRS>*PXpCFvIpldK|VNlDJM9
z`s)0W=s%ZSRE^|Q{cTcS>aH3Q{z1E_yIyi-3MI$Jat5qg5%@v75*+g3-*9QOPooeT
zsHv^JecQGhPk(#wmj6o-aD)Ai7;*g8@Vv72_FKUX+lBMM-u^DQO7Bw_e~Tjp`X7k|
z4#F4uJzE!Schbu0$xe8O*id%YQ`HR-^^NP-?Rxg?xT~sAl0eoTH3uXG{;_Mz8qB{)
z+jL%Sja;5I-3+dU$_4-<_g3*ndl%)ysW1ATo4Z6=h~RnI#|oov$o$-JMaPJpkK%Mgl{%&5E!
zz-wb;!!eOOTlX^A7~Bh)ko9aG2nj5~R*FRXkn0HVzpADJvp@L~I3295QBgx}lM4KxRX9cXZ@xTY+cfRQYMy6E(ZkYG&w|%p&zK8SdmtJUJn;ZXjznKD=FrJB
zwQ~GGGOyP(j`i}}vi}Cg2P<3KF2=@LwrpD<8gAWAYwv`e;eR21Ht5<_@B)P9F$D!}
zy?X;sgU<};>(S=B$#R^IcwESW1w?_=5kl8N$#mq%P~OzJDUT+Aw~NtoljI>nKl-Rs;v}SYw!Ev0mo9nc
zQA!KaA|h5IoT+7>6a7{>4#bhiqZ=eIjwI)U{kR{2{xX#~`SXmIjfjXqa!p&b!=_}J
zzW!D&BThzB=ZIQBN3Vm2XPmcPN}2A51EhAq
z+rxW@)>mffTId~9!Qwq^^yulZ#}41QI%$L)`4fJuq85(9+t2;&$2i@sE&@)@>vA8-H!8
zlbnzizjF2J(jt~UO_!pIX7tMo97Zg}U}qG`LiiS6#$z71D$fG?N~l&!zAt5`d%kDo$?AW4s){`*dCu56pWJb>u&p=QxD_q0SuG
zxMR&zRdFYCZL_HmDAsagIJ9OhLaH6}!ApeZaUwU;nph**t{pq9tgOgr8tW=0if7N(
ztz@w*X5_Qc1Kz|;ab-Fxc35-wpKl0wB57XX~W}($O
zi4L53XCx_dMUjoDDr2o-@Pg{2aUZ(!o06PX8;7R;F>k#r>V2AMA&9IXgweSGs}4`)
zDgi;Dic~rX$awem?Zlv<=nYqYr|p-ADMOE>p{<=TY)gE+H^xe|A6W)CBzZ$Ib3O+;
ztHpLa7{5rTxT5#uhZ}XA4p(JQC&@LTalN(twu7(!ZTd>#GuGwB?Sfu#1WsOBfAxni9
z$ob4}Q=kf)gD|4jIDF{dQ9P3PnxL43FC@1$g3V=J%Q(N@T@paFQ?2A92@<3+Hr%zU
znwk(Re~BK9&j?>W?pQ|80lV)U4zMed92Ob%3oeePTgTJ0BRRy<5CjaN)*{8BR(wr8r8g6>xmtyZVT*6C
zw|&3#qFrmES17t+bn$3ta0H?cL$D$VRrUglApO$2_Ka1JL$z=1mgWJtEDwzSPC3IL
z$W;eYqsN40wbD)UIY1OQ@gC?L17;7I^!1XXwvI>!z5&LEvUl1vt>tfdodmETt5Pt6n~Oo*EWY!VP4ZuH`_+@}Am+KFI;;on@7v2iREEbU0(x
zU8KYUmA@8b9!resg~(^I@Kbw=#-vH(oP0{<$akP89zGWJS|m3Rk9zPl`20DEkr&V<05wgu7`If$@zAq(Z^{%9yb&XqB^Ps{SK$uijLcmjT#p#^
zOHCwVWmw%f<|30$R}^ONV0;rxQE|+fIkSMC4%u!!75O`1U
zIojKf0`S~9{u+dcib?wF8K`&*=3pe3{6zSW9H#
z3}6+Sxog*?#UtjtwpCHc3a5dz&$qzER$M++|Jxv
zi&&2&2CBq)Sm{!S8EdX*QN)#q^~>!iwatIxFt0iniRhFTAC{6wSM8p{ybQcDlZkC10r{y{a6v1wgl6_q`N^Yzs(E^+JWVwUnilgmQvwUMgh5SLS@hHvd4TJ}&#K>5od}~|yBtQ-t`ULt
zAEh5UCgt7ZhW@)k!=l`@Yga(nX;;@Io$X0V?LN1)t7w}!FuB$d!d^myiwNI5SmoSZ
zVcMctWHWLXZ}$^|cp7w~^P9qNdv;aQb>^dU>v3ggsj?_nuXfp3-UahSTG|~7QmZ!8
ztWX?R=8r1w+yQ53pgxmW(Q{%GI1Y~F7%skV-Dto47BWN!M@QyYMXL>LEF!c+-zgMg`4=)D#C$MBYh3cWhvJ7LJR%g=%oKT#n1QAwuk_Y|A*+UKu28!f`
za12N7Y-Nxt?%{{)dTdo+pi{gYRX=C5-ajJsF%?T`R9G!Z1|P&T8s5AiezWG`LobOK
z6T-lG)d4|4K-8KpH(A~^+`u+*c}uYsfDOq{{rusBAiLSSx6|T$G*SjB7#-zj&9XvzNe>L$PX!136y<{yi6X3TPF6~
z6AWnW(nZ6LVYEE2@Qqpc05M|Uy{i@0QFFwrc>D&j`gmf*E
z3WG?4QRQdfzQikrvQ}-I$SBQx*ZT6;C#M&2y_2w^Y3eLa2?*ar4$QV~)YvDpcdIzk1+=_+I>Bvl+r$cM;5-9>$<+#bw?5JlS_P^zR93xV4*B$;rJ##pYa#sJqPa%c
zUAXvM$Cr1~X}^I`g>X|%!@1{$>W}AK*utqIytjh{3e1{2_i|cV5rZ@=w3r)0ZDnDg
zGHcYf+wIw1BCb4}z|92f)T37~>3QQs@-DdQ(HGb!nfQ|Y#Z8Sl3pNhKCtOj_z<@j8
z73ac__GZP=Hrt)V2L2%Loa`^QEv6tiKx{Bf4g?WxI6U=RkZc*U)cz!{(%0s?t=O>P
z8>pdHmP8Wp7hU|DmoM9px8YgAveyXZMQAO4i
z-k^lCXIHn3H#Xj9ts`L7kh=h?w13;l7;<5nKV0bTY{RMot0WBjD4<7SBclfK%UFPM*6jmsoGpQDj#6Z_6^
z$|FrpVeE)LSIYANpkY!6HaiL+ng}|vG3BJ;)C-8d9N6wdE#5q|6WH_q7KU@1)u6IAcPZlg4KaDTHI-ec~^Bq;Y
zttwT>y@H_5iNC)s#O1@P#Eiv&?JfnqJ{gG1ci$bwlQ{rKU}0ea26FA9#lDQXl1M?4
zu3*KY+;8NmssHHNvn_;xVPDD~Ofb}}jy>N3ujc3f;!cM?ks5!nHYUWr4-^Z4Ba4ti
z#9Qf1M~Lm)V|eJ;
z+c$13A(DUtboJV`Hq!@Ic*rESk%;q3EXTRA7d;tRGuXDayKyc<-8dr9vcuS%5vF}G
zF>%~DjqjGwFKjA^fdGCI#R2Oz;!t7G;p-v!>05^en!9tm+*u6rDEebanqRu@3m`lO
zL``*aQl76JTL$$z`u_L8-^4R%WyO)NI_?tR6kL}JqtJp#7Ub&1cQZ&*N2GjB>SD8%)d<2@`$A6sAl-zt?}lCWz~69!5$cCVp`W
zA4IYw68V@s!4sryc(A7B%d!P-Q7U6koJVH22bJNwx~Q+RQJ74Gf~OUCGmHd@GJV?hMB
zcPo|6%gV@Tp!Xw~r$oF4Dd7v4XTDV*_9BBiM=YMGlm-Hn4CjTT$-{nOSi#K2Fv|Op%
z^*3!48P$Zn26b*F;6Nr^6IUSXg_bryHnx(q^5}#7f+AaH!qp1(-HWUJ)|o<}qxaS7
zWC)r2S#Q}rlb#anLrS$*Q1XBST&{>Tslou25RVrADfd*CH&2`n%^5T#y9iz2KIeAMpNBbpYEro;;xQDEzzs*&m9Fd=>?fL?9XWOHqp9nkw-if5!ik=#K
zgc%x2mzjaa|4^f2g>#iL89kaQYg|_!!MCZ-D8f;9_uALPWFLDQCJHh0sf-M3&ISlE
zAgW`pDLUFVX&Fj;@K
zrv$!Y=~ABCh#!4uv=#e}Q_ZxL*gna|1|)zk%y3?gg81UpH!!*}hbEnWrgfIv3YeD{
z_l(cv#_TE@rjR$RoF=$-=FTy1(P$U##5P4i|G*qt8DA#><8X39i-DWSQ|Wwmb`_l!
z$f97K5|H_BeN9`|w>_{q5Y1>K%{Z2Kviy=nlEeIFLai_joJ}*@
z$S9j;k@FUzyp~oiJQv6mGy<$zpbMj&!J2p}m_EAsZq0T3NdJ{K8Bs)-voa!;?Qaxo
zCImEWN5@28M<^{pV_iJkX{e(mn`aOD5igAXWO6axboi_tEyEDSqFLdWZQH+|tC@Za
zFnu?g7KYkaR-(IsVss^&8C;#LuK1Lcpm&dkks$|_kA;*YoUjvQQH&qMJ_+xH+Qa=)
zK-Wnd@BL*_|5>)jpxqjymZBlcKDs|4LA`%}nlYiCp#vSy9)MnigwWM(U?;W6Sxf;8
zf@$aLJ}(HGXd4O;!9B@Eg#opFr14y{@nXI4*7sJYYxKIQft!K+uJMH?#;^!B-Uf~>XDp_
z-0wUf@UtThDR;e%ffltTCMNWv-lkfnw*nbWt4c}Rl4{SrNNdA7vr@5xe!>c
ztR#l@!c=nAgQ&MD-b!9$l*wmF(a=#%@k1bS_)G)SYQ)Q*)zm;Itm0IoOOe)gS+HO$
z_yKEagP9=nA2Vil>){?Du+E&DYfTErTiN+=9bE6ZRhGA8o2iY&(
zXqJ5Pq#wi#4md#e>`Q%wAUN+bm8z{9zqNQG4hnN}s_$<#SpiAI2Tp8GPaB!=i?k}d
zwjFu1YiSA=T3d(l5JEzV*yz0sTwCyDxfX}mGu+;T7shG?!q4ul7gm9Tn&J=7QK{Imt_nSz+>3C^Co@NySH!Ix?DEbmc(^-7tbkqtKm9K
zTU596;K9B?$$ozLmOS>lU0&mBX(=YmQxJqETyF%f11NZS)bfrj0xE2&h-j{r$Pxeq
zsHr)LW|HA7X^vXHDV*RKjCcSRTT+9+_Ma}3R;wmCav%d0Xb*~?anC}F;}q*RHbBxX
zp4T{W@ojb6vrM-+nx0NyT#M$T;msSXbDP5}$S`HY5hkmC<=P_SL>@T>D#{LSAs^lN
zu;6lfNNTR`va*-fH*1P~IP{k+;m6dI)eBAF^Vmq`8i|J0{e>f|la&=U3Z#fkHTDQJl37_Zn?r?Wb?tPF5AFa!$jk{h{`=
zx3gp5A94wn#=+%C2<(dZoQXW6fL{{tC|RD|yT^W73KsOe~UNjr&Qor
zYF?GT$!|5~*tN|J@I%IO`HFRYfo#!Fi!}%2bB2JRx#Lp$WR$%$pH`6PfoigFiUIKi
zlD|Ro0O-kU@3w@;$i;xE*eqAS=$iO}-`4xP45wC6vKQ~is9#R0!YOAP`xHfEB$SPH
zA?$tE(c{y<0IIjs!Wwp=>uq3Q^FnFF#281sm
zWGvFe6^e-soXULZ4X~aE+muyQoE;st%BrPm&FHQeqmWr!>+yb)7tO-ovs6giPC|&s
zHO>jl1@U;{TQQFq0zi2w&Cjk#_c74B9Z~)UiV5}>3J}?ss=*c8OT~BFsJ{8tRJ`KH
z44h&S(+G(RrVSokC8+;guLh3IUAFUvOOza=f*Bl!qL2yQ+mlK-7liS|E-omJQ&ReW
z-p;n>ghE5~u)6vZbA^~b51vOYS~5@RD3L{48Q+P61H#xbOzk&CD>4F>58+GG4~+i6
z(FGJ8aj%bo03ZR1Uc2TE=0j}96Sxy*0Fx>-S=3&0ZxVv0_dyYaJMO
zL6ZZ3#P^ahk6!qwvT`J!h2mjSTSV^T^}
zA8n=_vL7@Osj8V-&5IZA0q;Cl$|gM+8kTM}kR)_Evm$p=Ah6LBpHdYo0_GwCK$*F9
zq-jz(VT&|T!T$&#VA@ld7;ihhuu6Jk2GLpM{UFmil9l!CoU+}ZSSoE+=JR{^s6xQ2
z1O+)9Hvc}sHV=>jw8Cf~VZQ=bxXaNnzK@bpbV`bYt_mT!w651~Jl0YpUk2e76(4jd
zEvVRuyU?UC`nwwtbB^86l1dx~BGk=ptHnj&rhq#_v3-TuJdlz&;&$%(?4^F3lalMOHaBJ*n-gIA`2hpqbZ5dW1$PR9Yvp2j#Lte1Rw>j
zDD$2V_SZ`c2C2O~zP8g<)bah<1TCpFDv?W<5=CiP)SLJ&5U(sv+eq?G7+hsKems{!
z-}EFD)xUpttjxXDF^Re_nn4XQeJ{OQICC|z3W0%nv~n40{mrHa_qcM~Dx&3M)hgtu
z(JmWQy!NLp(yE#N;73rehuL9LZvxa;r0P(Bn5y**Q@q`PERU5NEz?J2tR`W4g*b*|8L>7%H^bP-=Ehdf1W>k7IlKZ
zlG=H-b(gd~82mP8)~t2ZQhITO+j8Mb4cc}>%1H9IjnWnQhz2AKq~lwi0X8uS>8)eb
zX6rQ|_#^y#;YLz}ZhdZ^RH#S6!6KKr5G_grdqIz!>Zy@q!YDh`03=_JoL~5*i$Yjc
zOlbKV`d#(Ej2li2hzM!S8$}+|;K3J{JvMxhrYe_7wEnOMMN-yBE0y=BZlr1BWB@a}
zK3$H3P%EP$@`QiZ6Q=tj`~?kh6jZCs16u0oDHedwLpz2{7_ljZqqo@)BCabByZ}TzYxmqTyCy<{YFr{cFskV>lu#soF`{hLF&Mw@+mj4
z_p0uOp30^t19OnK&yH3=C8#-wK%P$k_3T;5H)ec`j;59b(|!C#tVms5(D{2aXA$n1
zA7fI3^(WzJpc()KH+0x!HQ0A;)0dK$pSZmk`d1gYkQEA2mrxJIMn`L%-(&D(
zy7qyX=;eQ2nT;3$eLjI%Im0W8)LG3VjEtz1F=T`<3Ib+0G<~>%fe-|Rg@F0PG=joe
z{w`URW~8oAy8Lxh(+S?z5MWnU5DP7%fZPV4qhdYO@nZtiR
za7U`C_@hg#Z3c)O3kJ-WY(f3Fw{PCy#i0YmcHvLCfKH9ai~tW%F{B^!cxWZXMGRvk
zMo};zpy!5(?Qb5qfr|$aOc=OH-b!qlph4sP3Sgl|-$||bGFm<*5;=sW0`ZraI3<`z
z`L8KL4UfahHSA;6nUc~{=Bvv7_(Xxj-SABNGnGDe_relpK$j_x^RF#LEg7+OfKym{
z%un8hk8Y%adc86mQkepZs429FVGNt?vhg+AL1UOhF!Pe?Q}%<}D8r()_DnjA!UBN=
z0F`a2dAN<FM=KFPLJmKqt4dAHMjECcw~4A+JB$)*
zEsGEMZNioxChzr7UA*KuQ7h3Ey68V--Q2phj3%*m9H)`Y_crh(n0zWFv=8*ZT3Vq+
z9=4&5C~}&$yO~5pPaN}ACtRd#=kWODu8KoGNqfIkNXUJkRQ0${I??_QE|Qn$l&Z9<
zds$B|RNuKYhHPm^$1b~M)>-ZiQy8cCt?P(KSs)T$Ufk*It(@`HvWKO4%IUN;F_qgd
zKcP@W-va{_^FeNyY0NS4E1@>4Et!|X!}}~J*`)pRaG}KcVW%@P9%K$x9lc2PY=+m^
z&XEUKc22O2Wra|(35=Yz+=0I0CWc;JWfVg!eEj^ZceI`v@8K#Mt75T<*yTNwBwoyZ
zUm~j6*8NU-x%-8sagLGc(qS1IudZ4(Rd^U!ROeY#{`8lS-2IX!m^cke&xnzO31%VT
z^uh&Bz#->*_mdN!^P98?GDpIH1)?sSs&!3z#C`FTj{lk8MFDML(PeaLw=Kp`ei7*p
z=t5xthI!pESUJgYzQ@+?GBUO#sGV@v?a5R)M&AO{BBkLX*7JepE)*^>S;B}CgJ9*So(Bew7_o}33v91`Nfl}p
zLNFH@iBFJQ*7R{f$bxNdNj%x;2B~4&(qfBxbXcFiK7KhV*$9nHE|^
z?4W00WT>WbJu*1JOjz;akLje4y8LK6@|d>*lLJ^7*5fhx>{_1%6{X{Vi$kK9GK})I
z*|jMadO5o`TW5h@PysQYz~jrV6>E1?AeKR4LN$OXLc4v0!FH{O>o~mG4nUP!-jWcj
z7cO4pL=O^)iaD)3mPC|!W5lz{1i&B>J&Em0&f^9lo0SwoG12DtF^0-JbBYT~ko))2
zE7Obx{lC^=8%fWdG_o#ZQr9u_rKd)kB+_IFQ$J{^SwHlNDp3`r_5nLl
za+W35^b_&-uuagG4EjrZs)`=VpZR;_y~2fC+t@HAjN`WIey}%`>(cCz!^es&M+xI6
z*&ik*{L(I0xoHw!UeAg`fohy+kYR-~XZPFrSHs;dQ?aL$m;z)n*>hosU`(NqK6Pr(
zbuXMC2pQ6@UQM>kRO{7i+gXW3VW8EWKvLy&Vq|tS;=uU?5siE*#x{dW3|Cnf5ra3m
zOP~Z9+m*tDQ&zUrQlhS6421CgRKvPm&^M3%LKUV9}||Lwo#siU*mP%P)SPT3a<(c_-(G9QB`WzhueaI6HxA
zFG+NC;tDZZSJVIKHctXN<%f71_s+aS{M!oTle*VBTk5uv
zOSs+X^iFW8usrvQIrMq8x1{I&Fw;~t))P$xg(TxX>zv}%a?O>5OG{`vmD65n0WO5G!A9niSKGg3r+^Ytt7oHsqq5f3809-!G4
zAH|dQaFs$Gacl0$@?ww!mO|ku+s4D2c0Uvgz3K`^w8Q$zmG!3^ix=n}lJI^2D{r*T
z^0j)?O39A5wPh$&)rQTL96dFyyLkIEE_m>pzGfSgj-yHB{~PvRf+~`FS|GGcBGS{0
zvp#v$lAbT!{zdb#S%r$Di>rIj;9&>uV49JUmoL$uo_5^k^ujrk)7VrXZ^D&-U1HOH
zOldUOH2GGe6{5um=M(1A|QTWcrdjrzEdz6MBxb7veAH-tm916yFe}Pf}q?fW=(HkT3Va~geBV+9c(L)km
zhW?@`qkx4!a%7LzYw%Kc5=I=RX5pPU9!Bg-O$mD7Z(0M&02VI9G?D|W_lRTv@?{P2
znOpr7x?4oqWukfH%od_)gB@-AkAnZAph^hwRDS#MBVauL&fuEEXDd;d;3F}9J|)!A
zwwsKMO}+uv)qw-U|MIS?oeRB5d}x?>maEcH8ftVR$Dnz^PW&IUBdO^ji=)dH({%u-
z^Aq1Ir*WFHnOo}D4^UmeyA`NC#TG;NX5}=;q`0`5EE|?OrtF;chz1o5)PGUYLw
z4AlJkGcOD){8#T(9VK7Gcm!ku!iinOLN8CAGqv#W(aB+qloc{;aj~G#
zuDVJnap}much52+#^BL!&HnuflRVB9WZ^fqVWvJFAjF0De5gU_w`=ufZ@2-mi^Lyb
zfa2WX7z=Ym>%i@#fED(;U}v!Kwa8+04A+U{`Fo1mx_*3F6k13}l=c!m4=+0s%>@u&
z4T=6_3g>U8GS?@q9lNh&*5Fy^3)f5S?P1l6u7zCO
zFqe`0`}giW7BC0Vo6O?ZZQmh1Aqm7+AD~zU*x?oiW!z+~7iULX)phyP0Xy!cQB{DD
zU}$=^xYt10wG0rN$x%xl_pg;<$zdD18RdFhx;O73`E1)~-hua^-%(Uh*u864$<`xE
zmOZ%&$hcLKyTB+Cu4(a>D;-ONXQI=gk>#yu&o_H3Zy3M%)y1
zTrFIpFsG(x&u}G_I|%3Gk0ut5h|RWic60k#S0Zhpx4VH1aaKlQG=d@;K>zzRf3Nxn
z{bYK;9WchjUcCB&%2MK*4$;>4#E14QP`)X)eZG~~X_;Y>9_}I#8q$j24R9icfYH%Q
zALzl%l^&P=+UMS#Zr*!Vz-pD2qCiwu%Esst`|L^cwArcw0p&8P?%sgeso=8CT9EDSG6ACKin
zYsq5PcOSqo`7TYN
zTB$DbGcEmGQ_ftAU!u}uTHuu38k_BBY#Z{2uEw4upWPbW?p00ovXqQ@e&zGtP;s67
z=U-2~`ThK4TY=l3u8uFC7F<{}e>x7n#a|CrkKFuKjro><3xa-x{+KIBNn7c@XP*t{
z1{QOa04va-18ttpYrP3I1b}(Uc{p|q3}i@>Uq|>PLDm+C(!R0D5rL+Pir)2{Q%_*X
z>rWeaSrPd^II2_k;DjME?MJMG`KQmH!<2Wh>F2G?@4EF~ir>>gV*0!RitbJH&6u7y
z3IgT!W`Ybpa~y&Jrs+`(uaV&>l$zLs`uy7J607A?BUGRrkO#t)P@&@askM7R_+fDl
zXjkDot52}-{-C%}0Q{5a?aspVrf1;pkTw)TVC0#09d%7&W;!K)=JwR(=Fn^tq7s|
z3;Nxbf^Y$BuCjJn{AXQOD}+1;_aAnd)cpvu*fnUc--pE5%?p+5s;CTZ+0Q)C)a@9y
zka-oJaDZU}UDVWOp1(Rt^O%skZp>1Xh%7jQ2ZCJ47%R@p{Ti$khUu?9>bai!A=r`^&
z<9-IK2=AH?3ak^`Z;sND>F(}9=mbD*pIlmvE#UP_TfL|oqRt(pwi}*2o#4N5*DVyiCqmtuzJZpMC>+mJ1i@EMhTerSaCYgB2?Gat?sPaSw&B
z>XT~&T;UXh(SUqe=;^{pIn&-DwKMZ$K&z+f5?ZoN2;PdiHaW!GQgr3OERo56;6+9l
zZ6B7#*^Qb(RZrdP%(sWL-4C&ziP7nrW?HRs{QjmXXVEnR!7aEHFmOr_sXNGC!t+9+
zc3gb16|F-KBP)|al9M&xA0CZqC};UR*xtp>HU51k4Om(5YxK1S>cQXb1)cK|vG#lt
z8&-TnfUD5EvIHd}d(|eo*@(37#-jI~Cjv?rJVYO5gMvEa7i?<6CJWT&vvTF|AwznP
z@Cu`1w$)3B;%X56jQjj~jn805%jSa4tyo&VolumO73&>YgU=L?SOL-lYAAj{(8Q4-JjiR8utZZXn*@yS<(*)iu
zE2Dq$nyP2IVZFbvx&KOTc(9%KKrO5_vbC`?rPuO&AGgW@zvVjo&=_b39~h#Lsy8a*xAqF
zoY`+TFgJdN@0*sjeZb`)o{9n^Y%#CIo<2Q1ree(V4GRgC=VCmamE|}zE#u2Rg(&qA
z^_=XwzZ0K0v<>ArYrQpGsMa3+Gzm%?)kV9)QdIP~oalC1bdUPD^Gh$NY@*~r#`z!2
z-s(Hku&P4;v=qhJErM-EJ5c=#YA%=Zl-?)jl{qZT;Yq@HvCttq2z4MF0s71$dwo^4
z+zZlpDk~H6
zCD$^((Kc`(D#Dt3_eOx^K_eoG<7DvWFTDur+yaHVd2`2-lr4)_toVIr%!}>U>&;u^
zJA5p9D9Fo`^6(kx*yFba^(#KdgvTh%K=
zvLr_adjrPNm0y6m+^*K~;o=LPSqdTV*RJV~AK%esb4s*|j(QS>CK+hX6urm_Vve0af6W68WZXS}**1v%98tQ;V5xPh!4)YwEB
zV!F~FF~Uht6-YObU&);ebze>264`&NV9kKwLa)Lk!R_9h8=l9zZj#hnkgXl|%`@QA
z&v&N_NJt!MViHzAu)`$v+_p21uQXrnczfcs$lwIGN3ouuK;n5#@VmrDF*Y(n9w|BG
zDeDX%cgxRDH;^>}6+vbT`dd%WG-mLf_l-cy@F5HVd0Zq0k!XR9;VQUG8ADa;0N1bQCkM(Qyup~J3(xr_+q$u3D
z0U=d(k;SrfDQ4v+^rK{$Y7+jMmZIHfU(ww^Jk`&*{(Qco-^MQyKsTX64R5B(NomEiT!EnF+a5Na_!#x
zPoK`yiSc`J^^n1FE*q$3qES(qx=`7`P$Ppsar$cL2!S(4z9^*olJkVu1}6-y8^ctb
zOWI|0Gg0z_fw*&@K?Hl4yXdLniBb*yyl+AGL2)ymI1BVk%2p1k4cu!92?@Y@G~G1z
z;9{}AOiiXy6vS+q(?YH?*`Tekq*2$=wA@u-_wmXhK{Y^_51`M$nQFzF0_3HaQ258g!(5Hj}kM{J}@$aA85
z;FEuWxT!QL>+ZkxisjJ;Ch}
zD*7rrIMbx%?~?N_tPv&_FtqmK#r1SIKvy1;;v-wh^YWZ3d`i|a217x%P!QCao107U
z7z79Z1+ttY85us&j;@+(0s?+C9Zm5VdI9yVG~eOt)0Hf>LR0BUD{CK|KYF$mtHP&G
zF?N}hWZEjE$hXlL(;w5<(a;c*R7>sA`SD6~RmYA}zf=y>L@h0qaeh@s#8R^?;DU
    ~)?XEw!B@ab^K
zV^nuMOwu{S9qW8>b?+$i4MPIRoCE?LV@nT+zZ_K!b8kJXNVj@g8nHMTe}!)tu7;r1
zgcGn)njISL53~Cd;1*2(@-$#EY7``7sXhoXqXjt$`uQk$&k6%GEfo_GC~it>
z>WPyluPIMq3I}>4w0Q8qffLQoUS=T!dVefj$|@=#EMz$lX+nB6g01R4dP&ZV8XL&E
z!X7~i2njnVXxERPiy@Cbf!|ip{|Q=qV%H#yNO77IN1nmeTel`K`djMA%ojL)Plfm%
zuy}MPL!tDUS>aO2vzTg}dEJNm23p`|?gh8SD2tCLys$a6GEoaW3(cs{z!9dV^^94y
zH;`|?BLdBd5O%>>HqvmUiK~iir5Au*m<~(
z-Fjs?8C0kLTx0>}_U)G$_R$N$-FNP9bt)u5o}n6Eg@My&;9AfDu6W0*El%INjTQYhpS2A@
z6d@SS;CY*p6;tZJ|8}}G>34rOBMEGCq4zOFXu_hioOJ+m=&19t%=-(Q6{q?5sJ3I`
z8_>-P?>a0T7*&5uOR@Gvq)@&Y+Um@v^zM`sME;B<5F|YFomxsrT8NIbn3x?H6r`%z
zhA@Oz%JYz&m%E3DQOgDyJ^ARG1~=x~2(n+K@(_@8qVLZW2O?|a^2+7a|0LFtD<*9k
zN>_z2B>d;(%aj0g7)dE9SNAR;j+3_-VlkHy2WFn&9aKf8GA_KUw?@3GefEs{OUA)y
zbo8uj5TuOt;MR8^JRpw?x0_4w92b`bJZCz}^2zTRfKf|^{E5wl5te^cH%4bzc>6k*
zv(X7*-bKt?8il(;xo?IWT_U)#j*=XehZ$&AQE(oYNc;&H1EFm8@^XQI1x_S}8e#$j
z33;>rEtL#6A2zfw(=!s{)MTw&;UYQy42(>w!H{==29Tym02;Mo%9>kS6_j#z>{tW(
z>S#JtFbra{0@|znSsk!4BBEsH&d011z{w0dO5z94IaITOiOH)MF9zr+^Hg9Vl6%ek
z@$vKL4#-{51(_&K9i2BHKkf$W0#vlfl+F>No$kD!J#!`uthK4BDeGp=pMMM%d-J9>
zh{?6rKq$3`P|H$Ev9FLr9Hr|Uz(a?x5p3pWTVJWr+m}&
zQAbgRp*#Wq;quAJ_<`^{n^ESzKm)*sMW4rsaE>FE|NK
z-wB{1`uTkWlmc4>WZ)u`@{c^E3MD#h9NMmvC%;lEYxfuY8E$#Ms_GrM6AR>QU`4@@
z2M;Eg9j1|l6*jUzxOZqh)jsIFkIXKB%DAYVZOd02wk{K
z-+s6AE{qv-8S5B_2Dq(RMzyaEjZwpwFP+p|9yUlmeM3XRt2M7rVmpiAf8dcL0TnnA
zU*?X)S5i}N0|`>3S*;81$HdLC$hL1$Oar+C#FcUu
zZB0+FLtBVR0a{|zh}&Metp8oM(!3+Jmt!}xd_}K=fVJQ--f|@1!2^SKa
z8!`fXh7{;^uR{#~2wdvI0wMtO7Opq30{OUXJ
zX@kJLfTZ#n1)X4R=ut{1T*lH
z>8NV^NeMnDZ%ZAs(v;7lf}c<|z9vTg^PM*gZh5Wuw2q&EN+&-0HLQ;V
z@)oTUYDbO@UP=y7&U`K%3buq*6xtOimOwOlOg3%0Ozs+=2M;PQF}H?HEpP3lp=5uc
z1Rx9^vWRz%qQhQL-!d6yX*mV5Lf{S9zV(rx-p~NsoQanJ;f1`4p1FmE+4HnB<9@X#
zMf>t2)&49)egvus!^ldqf4R6yu8s!*u&F)8#fBSK4X}N>H9*Jy`~2kSKm8h!hjN-L
z1h|A;7~$Jfp>iVDfn}ph?^0eNU<42JfB~%Fl!^_9u1u%^t>#QdE5Vb^*ISEVl^vS@
zwl3SWY0Q879kOx$NZgSPQVDRKab}{xoL%UDm4gD<`-0Djef#zyL-rI6&6(&B+Q`}2
zJm*NrF0KyvhY!qQESOo$yLN68ENq(y3#2vRUFLG~^XXowA$(rr!T!!1@4>HmlR(Xo
za4~MoCn7p8Patzj#!^6$8y?Qf3qcGB9bjbioIN489U%g1i$dmOn1xX1z@v5HJ^I5a
z^*=d2B&@cQ9K1bk8Rrbxu}rGJjXw9Aip%Xhdh8gAhRfu;$mojaG@EYGRJ5D#<8!fU9lI9PzkCGA?M%pd!~_-qagi2k(Lh
za-4Q8vtOI2y5#J^d^EFP1|r}C>`va>nsw+Qz^6IIu$Z|u*~IIs#$y>ix1VB9^RF;_
zP+J#dFHP$JTW8l)BVw?iFRHGf-QW1(UZ0lVIAr%aL=41GndeEC$zxcSyx+p*pcX=n
zr=C>bb{erFT^foPf0MGlydAU_f=8sdLSx$>NGl#Zn7XS+$P=N>;d|6OIAS?Ah4|>syEZ@rMxQSbDJ5im0$K-aSS;u@o_W
zkZlM9+jtmw^Maow8Lw&S!t3{gN8HIE1^pliK=RvH@gI{Fz>F>vRlspi90q6Zx;O9M
zVG&jMv*)-~$|YZ)obJmCzs0f$pNWQIQXI@(Jgx(Md(EHa2tM
z){oPbp~<8|dQ6&xNW;2)|A`ZO*>f`C;r;ta_8*erS(!|KvWb3R)-F|r-ouBtB09_S
zY`$cpIj-1LyUWujb(2Fzuh>IyzC==vBfxohTQ7u9*`1
zX$M*5Y{{ZQM*zTiLz~V?&SO8mTfnz0ssb1Rqc!+ov#Y{A`DeAYD3R$~KPB6_Uz%rE
zqUv(WaXza%c}a!XbKm*#Wd)8cC+cwx@MT&@-&(ox^9>3Za#~sh?HGkF6hMUm!{fL
zH6+mU$oA`}Yt(qHg%0%-=-zA+11qve
zA@eN!H;)*%E^i85AtjZRVy7ngGeiTDN&kgrzP6_@UTS^*$y{F`rRz3*dTV_}0-aNj
z`Y6Z#9rXz&Na~n%>#yhWK^~NG;R2+48C_-W8%||m$6?~&@Xp`w
zQE_b19~>_
zK6u(r`t#tWLjQE&0E)3aY=wh4PF?ov0oJ%jkK$KesYnqVN2)W?Wnc`r8`yjVB4w$K
zf{*$-nW`S)1`gqu*fUBoEGxavDpUGg@ansF$(02X6T$#}r?VT01l(AOsrBe>S&?PF
zNMjSLZSB4iaI3yfi$N!EnYXNl7748htE0F76JTp_?|IB{-HwSOO-%idmc8!ZuHiSA
zICk}g56!i^Eg#{cpN}e-iJN7v1YPqOdj3nga;d-G?F7bMj$TH0VZZJZDY<>!&Yni`
zSyD1DJHKvWe>=M(+*>#QTY!|BEB0LxsJh*~gYla-4@O}^{@t3(_8)urZ$X=lxlrFJ
z&j*>{{K^G
zYWME{v`XJ#m;W*Ch43Qg+8>avIGbgR&BsbCv}#}*0B8V)rKy_`DBJwv(*X&3SakpN
zL)te4=?qO`ZGE(X5U3FvO1Wu+I}yJV4|-rv4J^iYcCPr!od-{!{#_r;>Ca11x=pwCcFLK=Y?O|+-3qvArBdsn}q>WCgeyv{yHVuHczFzz+*#s@0sg%YmhWM?tqmtvRyj*6O#;yaWPiNj*2zn>`
zOx48~qEyrhXCId{HCCwQU_-m%c1oAX=9e%UFi2mGHtDg3o=AxTThFIi1dp2fn%2HW~%m>pl)fITCu|pg`
zaboOk_X3loueUivfPRWdG|E*9xRMxP`-Y)L^ydHi4EG~X%tUPp=L6pPr+`C7$0
z%y3%*y()bzCk2Yk9pCeod00$AUSy|y++IY_ZKn#sM2p!iPJ(e5Ct}%H*%*`uoFZHk
z$Z{9+p4;8e78H;jhxdUXT$06sk%N5)BSc5R{5LLLcVi=t6j2WU$<}o39hzSEkUYD(
zbNi-xdKR2NPj_gmC;3UF#IV?^h8>Qk(BLU!bT2rhMi${Bzx7R|qww$o+(
zHe-`)f9EQ9xhq?{2KgP_Pn9^5!~A<>YQHTuS7cQhZ;~lZ{dqw;yG166YlG2E6SPn~
z4KR^6k-`2*`I9FX81X0r1--bOn~2Tk3Yleq>(PVrREsaZ`|<_2g0$JC-bS_Vq*E=-&^hOra#_8ERKGYV0Rb=TN0pT0+N-TvC5T_sfG{jHM)tp
z0T%ix^boYHY=zJ{Cv7du)?hM}sSCO{_Y=E-uo}ZYgG@mIQCQa*5X{SHIg{m?3
zI1doFN$UP9=~2o0^EspO%{q;aX)v-TLIXG-F`bPL^#Tlla|4VZad7FdrCb)^MbL`C*Z5KwJa&;uRKaP>9vZ<5k7@%3k*zH3V|%|+G|QY@cyC&3(5k)d6>34ImUx^
zC{c_i1MIA~aCRXJ1|P*(b}lN)kw>Ca)g3zz-9gT-U3QG9pgK+u&8BEvbJ6Kj+A
zu(h>Hx6Q#3d-sl+c(^A{yBW_SedHe5A=@Apjh!UOPP=1@I_Y53hd%om5E6$stP{@|
z>TSpXl^rW=R0s#)0jAvG%LZ%YsU(!>{(}eYY$!!-W|t+Wps+tsnfVtCGM(0L#}0*<
z^}EVAvKaeVUrGj5)UAck233`nkt2p)sUYE(n
zSH=$m{`sL{I>y{QG~Gm)KA^E>DDn6KIIcUlyG3jevOU2n1${hgC_=lO-7s~(?JOxOuhLi-q3
z+kFAs1H6t-T`!+-m;(dv5$z>f=*8^2X8PoofdzXs(1oWNA9wPGIc@F6y;RdjshN71S
z=pGG9Ig2)q=OENNpbI)he@>rIDF^zhscA;I%in?|(bGbAF(lr?`>Cuv3tJ2`
zDImBuzmhCUaN1SRIeQ+vW|ff5nUWsN~>z5@;-!a06C{>suiq4
zfe)iBmB$z2{F0Uo(cki7p3CV~2_u!p_?62sqbqHdSagfsHa5kOs=G#h&a`k9aiIJG
zQklRrevpo6DGp?GP|ag|AM*3_+zOnK3zTTVe#mM*MSci-)?S`&daCZPNaOCw!|V0u
zF@!-3>UAOAy#zEF(bai*!wNt&5j;F?jjxOU${#w
zY9VxycgFJEBX-4B)JGn#KRKwR-{thW&lE_Q4ZsemgI?>cz1DdmH)?v4>1>8;qFkb-
zuWzZT1dCRCdeJ0f;~d@|qI8Wm8;)LA01<}1fPP@9zyH0Gh%SD6q5+xOBwV%s^H!x-
znou+T>x<<*;qFNFvkj~fMY1HR%Fyjlv*C%?g&y~zz8Go*D7{ghN2yE(e6J+Z{XdA3eO>LziOWuf4~2(
zyKCwJFRLbkzjGO%8Iv_fCO^0fvxntNqzN)3#8XA-I9lo`%SFRF$DL!79s!D9W?YEq
z3TJiV(+Mz@`G7C1LZf_c*`m%m&%JwJhU<+wI`t_R8ujjp+2_?8qIIUf|1XSH{=^MA
znKHJ~PTWAn;5;SMOvqXR&Zy(SSu)SjyLy2V$s~(*;)^y(1rinL3#y<
z63DtHP;9^w_8O`3EPP-M9arGaUM$7czE*@7uUh7CiDyzP3x
z)P9V_cS-y&a^eJM)q#*Kj+e$ry$1@$vl%fPsQj#)_)znZ@I$nmX){4C;S+c~I1#~i9~cBB
zG8~RVO9Fc)FH;7qNV$=Z!~W09_b+XR<~0Fk#=9c~qUcPr)?)?Yd@D`xyb~
zpPsg8uN_Y0IQ_&B-_l3`XP|TrdLYN(C*mtTYfbVeiFLUvg6vjDhkk;TNZGaiRBN-0
zBd=!guH5E50;p70|7+!i=VX(Uz{#%xep^uJ&6oD(bp{%A{LP7lZ{aui8lHkRpB^p4
z?F%u05pCD-WwW|EJET8Zv#`xtkM0O(p%4PEw?=LHLghGE#F|Hs-o1anh!4@D$9W<>
zd0gEN-oqqCU&5Ik@%x{vcxJdHTsIC4eiw(6nvzlxwUyBUc!dy|oQVl9F0f!MXe4sn
zY6InuHAkoPP27zQj9Tcq2&qBH0?ol}QJnhcT2Id!#8hCunLBrq`_aR|fR|M0*JMPp
zoA@v2HZW2R9*n_xEiJHvO{0T#Vm>*#0QcR_4Y$Z7aR6dh-9b}1PFQdT$H&6Zy`$GR
zynTC?eHKi3+yLp7czgKWX=^PQy%w
z=RyZh5VnC3JIpAA&cwr7PZ4{c!#31v!VrU&Y5>tcBgZHPqk{sm^Y+uG~R*5+
zWlh7X8PbxNiz)6ZXY=y>=pjhn@fpZQqdX=gcJcB&*KE!^;XvQUwPIW+0RT^{sjJ>i|#Rg&ZJ9kcqQPK|5FXA8HY?M)AkP9fRIyZcnH
zaesyFxE>%`E|cZa%2o^r>~-tc3*v~|w^OhBqm6LP`L>SG;i(XklrO5e^2FJ0gCPT-
zK(eutgnq5RI1tl6#%OL=($xkOl#L|0j+9#Z#ccoVBiZ2T*glAI;~$wNa!TLju~^!n&6hZ;i`WXmn^?^z49K>Plgu8JpG`K2?q
zS95Jy_Pc^_pP-SPiC-Tay^l_dNJa%!{Q~;QLT^GAdi3n6@?||z
zCny_qFSFS$ysIX6I$$CMrhJFEql=+|!i)h2b{B+LPF`LO!GK7V3vzJoGng=+6)-?4
zuH2!k{dCUWskux3&#gguQ@mZq_j;;29K?!p(P#V)seyTS18Zct%ot93`UNKYEiXSRb%_iLjSn`i>P3J3Kwov
zn_j)S-tqTlZ_Hb
zl6TE($Uh3+`4>7O7%%{3>^a!k*=`iX`Y);`znV#bUI93Xt+&)dw}S~9@5@5
zLkx$azdR~P!QMHuoJW|shjVAf2_NY_X7^(G@tntzhsO+Wo@9~9!z~zRAwm-BvS_Ap
zxKekXZ~}oG+h%RK>d(}AjMAiD&$LWoa@uk7>yIATVIh(VPoLhyW-psmR!48RHp75G
z0*)AVjL@6s@;PO
zxj#=3i~-M~x8fywN*rw{Z^^gSofT!{jHet8NvZUKyu5lY2X+4w{WRwdzmW5W7gDB|
z35YgNzwFMdNSTgKHSE>v3;n>SPb!L&gRVXpHpN`~No_4Vw16+8*EjOEg8Ryxyg(%h
z;uUP4i9m!$opwlAh0V%7z+T_iv3+zNtY{dDB0EX!}hc}7O
z|;KB;MmYc|^s6sFJnf$jf{L@TLB`;=3zxh+D
z7q(6UQF#Z_uBoW0v96GZc^?_qfCZxUQG7XiWZx@&`}CQx&;CV4h3qAH%$KMkSpEXo
zbn>L@jmLc1v1z@QEm*Lc^|7ehIgL!u3tYHRVoF-)wJdezCT4MXW|
zV^dREX)Y3#@{(P>8po0~*XxK7fvVoD{LSc3`}~8$$~Rw%Tgk^)%baXF?60Z)=9~%4
zpjpNo=C4jhK*1=UzYUnym@JIcT@3_)l;jN=FG;ZabCWt9Y0WmUmRGSg1wPep|Dgw>
zo#zu$dx?ona%S&<5Q1#u(Er!;Gm4V6Smg=(6EZoiTmF&$4N#t%V1*1
zmWh^%fE=-oLU=lNsr{ZZDs@9R2;<2cUalv5HEB321D!7x$?
z1P4BoIX`vg%rHELke5{)K6zQecR_hm8^n9awr&6O36syTG|Eax$>M23Wo0pDOzDPC
z>WZ-`tVafdq!9$=MwT^LRrTCI&nL~TMma!StUDUu0+psxuas?^L6|q9d-X>c@m%-!IilKTY<<3rB?6Z^Ueb0
z;X*)k<2>5Q$9S5cYq|hpj{ueK$!_EUogm;y8Vu?ngvBEExvG=S-;gU5s@oClkb<`{
zk}!XY&e_PRnbh38ozTwjlNi4z1$XPX#BmE}op9>>`&T^CXucZIs!sbcs_o62CrzHL
zDiaM(lfN9?UZ$jQ->tXk9XR2Y!$KyOqOByz3Nsu@YUu)eLfB%21qAY$$?4_>rn9gm
zd_34hE_^4h2^##U<<}Fx&*Y$?>Mw&=CjJHIO!Go1N
z-ctT?FTfgUu$YYxL|8(_S{ov}t6~|ioi`xVSk$bQ&UJ8fjK+qH5N#9MC1Wi$c~QEn
zu+)10p5+kI!&K?4vLqF>hmZ?F6E%?^u`V6`D2XP$0+_gEH>rqqz+&?QVl1`*1|PNw
z3ngnf8WxS(ccUav?A=9<5j#x+Q9D3YQ-MiM8a-+hcLZ=CQllO*5?W+y{EK?fz4M|Y
z7Q;LQT4^Qyq9*Bt?IY3Q)Tz*Sw#cTm3Q&j{rOpDaLsdzv54Fq
zdwp@lDi3lQoIb2X&11mf#!9
zhao*o^#o>94*uRyBQ}#F<8+E{A^u_=HCV$GWV9SzE03)XMd(oURs_~ON7
zx>dTiny-QMPb{1Ye><~w?#**En!gqB3jr_YVj57@gbx|^QCkM3&Jfy=Q*orvL{cO!
zx`Fn}&pesb1>w_>y8>TkAG13(H3-P?_ar|wH8nLF($uKQ75HY`h?utqhtL|p|BHw+
z#8rIe(Cua*Ianp1gHdhWxFAA#ONjSZQ$yU{+0h}dJznE$L4p3!U)e?L{%JI{nr|Gm
znmbf}P%UQ@H3T&?FZX^qRvteoXwzmnU+KMdz&rnxNSj@7a0kdZLPd-*q)c-1qv`!9
zxyE9xy=CC)0|r5=(KUW2#C&5_4F4>ia1x
z@5MUiYg^t`Ij;*M$7~k=`NwAcI7dNPK(BFsvA;3j+x7N2bJKYa=L-su&QgW2Dv+9Y
z<8ESoSxMi$13e4MGjth-n>`yOhF#V_p&-DTFrL)*K9hY~zKh+AuUM#rCMKh@#qF?*
zK0cIGM}4zJ@lcjIGay`@?SDKlASe;%WEZJJmHA?Vt!FB}?ot!C?}xk=C$1fCk3_2E
zV(evEzQ$ie^zk%NPj5i)chHMPrl1BPnkje`b`ojr-=oRze-=v_DoDEUJ-!~
z31h0C-ILG6d6Bl@DVRJFYf{3(k?b;9Z
z=Z#$yq7NKsG+j4WYO4@PCXe$0w$MdzHenyqUn6Qa$2gxB7)aDpM&Zfs06F=QFKjAz
zwy~+fV6cb$GVkkCQFgsudU#BLw#f*p%PBKw#%AB6#DdZQJlN@YoW}&N00D2iY4n&e
zfJ-OVk*&^gZ20udJ{i8f-F>p~{qboCqhjo>wi+i*Yba${BXI3n(M1|bj?hRv9#y#*
zAm5OQ!f77i>Zjg@z&-;Du^S~DvX=}%g)RlykQaunna2IHC2xo{%KfKgx^c1lMpH1h
zqBEq)>L)0)o^^Z8&g1`-k89?kR
z6Y7GcRu`MJ`>b3^7!l%IGgGavzyARuMR_7vsP;tDbGBgTy$`uwtn)N^(_mgk`Z{R+svgtBb(?oXH@-a23MnCcj`#hQdkUBilsqga@iTvZ%=i`M
zyAIK8vq%wVYDl=Wa}(Tq_$%lSx7&`W9j4Ztw>Ms^^HYUq*2$AJGf(hFR)0c$7uw5~)vJ
z@@oh2-jw9zu*;tE_lN4@ekWL3OoPAz1`W&Ap74s?#w0WJA~F*8{G(zlx*!hqshphh
z#y}LNADfE3*N%w86%o3EsuRxHXNcn>HU_Ep
z>*sr^kGIo4XFW9^gmOr_(#;RgR$XtUwWA^Og3!Y6YlqC;NEhJDRr>3GD
zy24xgzuicbe&u&FHmkY`k{uHJIF`4>A6YE0)eIGxicY`b>Y&f#e6CM6H457|Z5~1t
z+N8aa>iohS_o?ZwM4YJda1p%@bUmFJi2!a9H-xaKf_#e*OuFxtbIt=gKB=OCrPXd(4Y@=|1W#FB+fO=%jdF>_UCOo^P*B)4VCvk?BH@CtIx8&!81R2|KCE
z_2gCr_X~0om9(oVcrL236k|_%rl;&hyCK{qlVauQ0z_mdw?%T;kiuKc;g)hsQOKeT
z7-ea>eOoVr7&;%Z$N>eOM^aOTOlNT*j?DCARcA8gMoV<#RBp{K0z{Ww?ei^A_!3*8wfmcdg?ZMRI-^O5hC3>vBCD!nno$1$yt$(4>wN8NN+14>TY
z#}V)DLo)jEoL&gEI251oT7Z%c>VF8MMC6K|?9;DWod8=Hi`wTb5vAxIv0d9p{20KZ
z&vQB_L?&e(eLgBQN_h_39PU~7a)8k&dXvE1yEkv5Lc2T-r34WU!JM0$3b*Fr2Y&|%
zCz1ruPgBqiwE?4#lcvz25rhJUQc^}?vYk)vx}YE}^QAYb40JXN2jnDHg%4>5Z!3Rm
zr@If+R0wXA8QVfakdq0TH4Z-|C%4MM-1z%r&)P%wx3fyHmQis2z~d1@>2)(74;YE(
zy=@CUEGVvo&d_2}W+_mSNJz;gZKKz0c)jPbTaEJJ+s%w&30qJ!PUw{9hS$;zF&fD#
zABrgY8%;r>6z)oA3krb3g)ASUin7Qk5BV6`nBnK(;u5MBUD~+2swv_;S@;dlmf~rz
z;c)s1V8tU|2vbnsKIpXGjT<2Wz_9qe7&t16i=YSSm)Lx8n%rP#xP*$UZzWV$J|1Q+m((@!K`rrLxp
zTs9Dz05%85_{pusI~i`U>Qz?zBm|`%o`Zj0oe%JzAy@
zq3n1)#+Zn6>55gWZZ9>b8i*ME4@3C$4pXL1jqER+DiKbSiQF)f3Sp{a&VmJ(h%}&3
z)YStt`ty*O!=}!oai;da*VHC3k-Jz(>!zRcZoxP;zqm~6$>Y?!+ISW;HZu-eOyStB
z=)Zby?Wj5>A`fCE7yU%E?MkkB)Cqv|Lod%(zUeP#y3jm%HMfSEF4EYe^#pt6B1Xvq
zshDYGKNKu#0}sPhbL`qT*QjxL>E&PJUuO6oxZiL*M|4p{b|23SdqgOhM$8{zI7{y!UUHB
ze@CAwqW!G`)yZO{MJp}gE}WZ^ntwOC3;rEFchqg&AU48@9+KW*<2nq`83L18eyDNV
zxyr41o+eI>f825&-Ms04bFYqpkG4ctYu~T40{1%EICQBotn6;|lWwA}k0?~F<9d6O
zM@ljgEBrN80s|#`Y*?7ZIGU?c_U$AW#M48w$50BhY*#@ke?IuQQ1auFET*QxHGP`)
zs_xW<$yfHBjkqErR_A&{f}LUUvInAm%5)EtR5v&smHqF(PcG#>AmuT2i^v9+2IupZ
z!hnUV7jG4}9&Te320(`<8ucS9crV*42o<#|rw%2-@{^nh33M`%^DL0Qvz
z!BCZ3{2H7@L=9Jj*Y23N5-cCI*ufe4dC#q-i**WHb48{%AXq7%`6ESHGdUowGtl9T
ziapxdKp(q=+~`{8s(klbX}!ctNLVJz>RC~rl*ybG%cu&)z9?4zjBa$;YORKqK-oh1
z6v5v9_G=D-vN(M7$krA+W27Wjt&2=kOteAW@jP|e=;C*FBc|x>XEIyIVLL|TqeqNL
z0B!qdoBJ#=?|4@*s{4d0*qUi*mhap_<0G1v%jt-}{sxs5z7vf201IeM{e&k6s5UdB
z2!M7w{rLHF`ivQbr%{TrpXjXOuNw^?{?D2qMWKIEKhtzS%IpCR()1-}
zKWU{tgLO9i3ZFf6@8)Lot0++Dl0cTqWJbgG;E{kWf$bv&6&ZEDhNJuhqj<@cUU;iC
zTv3a^uDeg>l-!o>tD{>R5#lAeyn~wVXZ|Mbd_T>k0_tFT(`aqO%vgtlcowsweLH
zV(Rokds?;X{g@rWSjM$O;eo^HvIM$HELl+5e>N>M`;C^H6BuCq9tbEqim3PfJADqj
zT;-B9B*n@ptQ!4D?YKPLxOr1XLk$jYmeC=e6^i^T*i7nL7D(&(=R}^@u(GE-)5+Uz
zo+LhOX8V7D3GnokCTOU94wreNj~%2x(PWd{Fy-i}8{Fe6z8W8#S`asZ**O>$m+8}W
zW{SKJzn3(U`K3;E^l9$A(I@v_fuak!OT-#X&+CZwX8j|Q$u
zPrnD?3)Fwl@7yr92feGuHx;K+f#rgzEk(1h4oEun)iCf#)b_GiDjtK~EBQd{)_vzD
zfkBA;G+Gp4bZ!a->+9>+C0D(=xc>bfa8WXF8syhRT#x|bsR4kN5GF4J7c5(PGXJnn
zw8AF>-7EODiD5m^-ow=6DXvjblIdbw71EiMo-%sKM8wGpKme9^XT+vs)dr!Fqg@aj6C
zV-aENDw|#~U(R&k&Fi*C2e$+VC#9r_yN5s4p7Z#{ix7qrV8UYLXS$x9Dzp2zo$spc
zUPiV(`}ghBQIn%g6Et|~C*tCsiEiom>AuSN_ilfC(f=jRvc@JRUBK?M=141O;t2AG
zT^@z9$8+n6->}Cb6Qq9$1nTkSzYe9O|>qCY;qvT?io06h2?Nctpr
z53}pXn8uU(`irat&rYfcIM7dOnU=^o!;CqotPv1ej2d-mZPRwD5ws+TCdo(g_pig%
zGloW|hlrDVC@!;GZ3(JEmAJR0ht|A^h;q10odGFWH8a$QHHP9Dx#}9}S^K192i&2D
zLlq47DH<;Rd7rWf5+31X9Myeg@>Wt26XXQp=C)w!)H`=}-CFvUVeENc&Uz2Ho)Dhw
z4J#zDQ}(L1NR#ua7b1Qx^uDb8s0SWe$P?xJh_<$$nScHi7C_F5`6tJjO*`jla=L%w
zx&qgcklf6MZ)$7#mn%2g)Kqc*Lg9o28tBCZbx=O+)L;~T#~SUaY4`oPZJzHV2<9A@
zym{yJu|MLQLGzMQtpgiGinA04TArrsBAg>yY|35z>hgfwv+yQX?b~-GhmW#|n!JB<
zC`V^nx+{GTerdG$LT2{J5&88@p$OqVP8jD5kV?1$2g3%(scCl{7~BT)cu(>LNQyTp
z2hLr)emx5d@gOQJDmpfHy&q92zXuv#Xqazhl?0?r2ZUo2bRao?bLaP=ZYKfapnL&4
z0doAsg2ZC%M~)dI{m|-7N5tQcBv51x1_n2Hy_i5Ox*WU~eA-#(VO<@g()LX4eIFAH
zi}R&^j)}U%(XzdJmDzVfMMH!}12!YcOgK+Xuw%=Pa552NZNYjCeXfvY%5G0-Tj59`
zjrQs_YtE#-;voT`Flhhu=^;m52q9oSthNZ<{7xh$^f*k+qlI|)_ARO%_J-2;1Jp&n
zHvb+)DP6WP+(}(+2-=?UC^=DR3+>WSyKYAGCxSz_)8{D_=eIDU|3kWks%mDj%F!=YXC}0unSDa(@-Mhdc
z-`m-Jgj9^P9=&yRObh|m(QLch1PWI+?V#lR
z&C4t)8PIpJ=zB(hRs{zayBe^3LNBF;t!=1INae3l5wI%+QYaze9zsQ(=r~(}!QPP~
z-CM^c$OYd2^6eX{_}hNx=<;fs*O9;gp!@ahi_J0SxmipX%t$t0#@*lBMDfWS!fN^(
z`Z^e6BxsNyQR$o64+ca=fh@eVcq#02W~sCx-`O>ezH7sm`kx#fjy8V8Ee|r|*|x
z*H?5%d{xstlYcJ?JgrmD>^-PPIR>(-Gj?nz(gp6JP<~QW^bO#a&&D?!ATRagvCJ0n
zYCB7za}tK4i!KJmP0i%#!iwgfJ?mCE$Lxh8)-41A=H4JG*70qXjXf7G3~eYA_58$q
zCLFmrSB}pTxto2KZ9G%q>t~yfr-`=ZYLw>MT9XStzU0y8=bIs@KsiySEn#P&q&=Rc
zpyXylMADQ1-rj>QUXh56r{d%L;eEs$CYOz+3PbhuAhU;JH$@C0r6a{9ZiL@&JgNEI
zG~?A5wF95NUy?|uS}<>3Ol)ktBNEi(e4eR#`(a1F)D<+LQ9usB>w?$SM6*CkCb)(p
z+v3U#<<}Lz&JfLWxFAVNCKUf~ghLgn7)uyE{+ka$){0<&zn3B}ly^pRO;3jbKI7wj
zh`LPq<}R5&eYUd6liCE)AyZ|5i_8`X%)emMZuW+AhrJS9M7dAdX4x(=1{WT11G1Bl
z^Ajbp{bCk@SV`z$h-mVO`@B=qo0_Vscyc2W=#_qcw2Lo6%(;Ag3hl^dHI00W*CLfg
zB;3_Gtw`8>shJSFKz^VVt8J4C*-k(!SpCeLWoPr|jYVz8wWDfs_-3aiR3KBdGtxQZ
z*u>$=GU0Dk6hwC}0^WDjZR3j)nYkSdSg`^^IHtXB-6?STJ$o$8&8?}@soaE`R)?sA
z2Om>Gp@0y-zr4ZWFQ^i_PnLYz^mHsiA&CqNE}?kSx1(&gVA}@k$^Xwdh5im0*+EZ5
zM_c-$q?f5(E^X$KP||e7$urzKSZ30lldJNdm1y6f#857X6pcvw&Q-7+Ia0Wnq{Y%_
z7ck9HtF=&Wm^C$#A;1HKW)VVXNugR!PX)eY1`o|>-+nL@R7UhTOp}e<@X;dR)p&vI
zU;XI+GpvHw67^DIod;f1j4yf&jozMC{ma(hI&~ua0Y_qp41oY5Lf~{FKVNyxFA@78
z85t1|@moWNW<|M9PzFL42IY5mrVy03)gT#dD2pKMtI1t|`Ena#3fs2bKfm=+Qd-2{
zWI%a`f=#l^&2kLSQiy(|qYB{0Dsa|jLM$vwgR8uvo&%MI5VJ#A0_NPJ95`qX$>2O
zZ<5S{IH@^|t|1;$Qc$=O^kF|pB+w%zXV3TMX$;1Q-ebO$a)jJ-@7_#A2qqO`5kmL^
zN?AckiAGz?I>?|{);9YoEJnD4oO1&o7VXLX2U#=Ro5X7~us~{R3vPHCsf1(8=ihEQ
zL8NK>^gzb3mHVh^Yb$OVw70VQ$rBI8lD8U&8BfF28~Y$|&T=kt4dzan
zatqy6xB#=d4JfgRCy|GqJbGknzaF&j{SP$LbUO*&eZzxI$cHDD>pLk5FhSwiv-{OIJ)RD7=HHm;Vl+@%aRqfa9P%tM&7+1-h6^9r!Ax+dC3dgFv_H^07XF;
z6|wj;mG1b!e5udK(knByHhK%?Y^cE?EqfFL?&2f5&n?{JCifr7%q_86WOZs=zY=a(5PL8gj|NE6h?NTyT3#i%accp
z(B@653_~j*)H7jypfOqzf5=}wpc)*c#S7S6AoB9TiOEWW&};J-KrS#`4q@Sb9zU2-
zqnN{^p-VT}0k?nlY4DaUiW+7j`|U*oGFVFZ@RFoe?JY4maktG%QaCXh^C1=-S2sE-!Y@wT&!x?Ck+ZYkoEP4
zZWDGSCI6BPp&WquSh^S5<8j98!?$nWzI;(w7)ZRJn7>KM*uWd6~XuahL0%as}}WC*j8DV`Wq(&bE=7lH;tr-kq2|
z*r1Uc*%8vi`vhS^V-su{fl7gwK<{1yu3FhB_<0qKw1$wJfgqc
zE#9}l@>J>)1-0VWnF9l}eiP?WxN;r(&NMs69#b2tLTPE~q{Y1S$dmfK+JNa5sl`AbrAx#QNpiS{z30_j4hu+01!1x>_;XUM;CbwTo6t<$0$
z@dVw2-yIEsD&s=bI3h>feU(^Sw1EMe(o!46SX@Ece*6$0r4{{!4G#`_18eV>Ya*?n
z<7S-!gY!dGrYu=9q&7qR2G2}mVo33oD-*mf;kV~uqP`?KqCn@Y(KwI4VUGHVI)ekU
zA22pZ9v8nB>M8A0w~-heUdHI8&n0((GP0XmlyZ0PDu#Tr9ox6rDsXefg98YSmVJF*
z#5fW|j|J$3Ma8A%7xQwFUyLa2icIPRp;-^|_j!%b%U?U;JxA~3M`j?){SF*D)Spd2
z*blOIig<+~7Bi?tT<|hCSx1q@1;WV^vmEfS#>)~X|E2D;JF^q-P{}3IR>DO}c6&sY
zE(1V-XQLIWR-Uh)_~k_jvt^HzCGT{0*I8SjXfPJA7LEujdwfCkG
zKFbtGX|E*2%75mT%^PW8Aaf^1#@^xb91jom4wWG?PzK|LklhUY6fZ8RQgoDJrPmNO
zxxUeNZ{2!U6mZNu`fhc#TX?OAcEdWMi4h$)Y6v6Ql{uN2qgOo8n>%-IiPu%4JSC6s
zw|gS?QSs7l1GN-4HQi7Ui!cKkU?7x#<$W<@qOd*6=>(PM-G>iVWvnoRmAG4Gp3EA_
z`9=q4uuslbP!`iGERi$!T3-)0Ogwf>m{kY>WhJNC?#su|46HhO5E?rFtB#eH>4XV`
zzTaaRpZ%JZtzC28KaoDg>QX|4CPD*d9ywbytN4odSNu3$dxWm;Vpw;$XNXKr;5BIJXT;B;799E>qKf^w%g*!(-m*i{^}F3bd!Be-{KY
zuWZa7@9@AM$#vxAjGmoo`y=G+mkk`<=k1%kQ>Xrj&VTu>pIpphfT=M?QisJ*0+%U1
zxNGOA#Iv%0HMUtg#~o*%XqPyft<@p3;lvpx-s&OWv5`9rh@d^nZ`bzMwgV5YZ;udZ
zaJ}jBF!cVzhvI3!rrRAvz#!+Lqkz0`0tgZVR6*iWRv$oyQPg{e1}oT~xbJSMJYm(6
zCF?24xf+#|Ol5XH6iRgIGZ5any}?n*Dt)qFe+&r%s=C+&BegKlE9#_uOHW4tsa>dAWIU
zoAu`k+rf1`Lp2ksQVt*f-WhT*B?kYVBiS+bjhK8KSfXyB6+X)gYai+%DFbbzUg|UE
z4eBralVUwa7jJ0sZD;HrRQ<=qTTU)pUTZrIs2lle))?HdYy(wI|)s=VVx?DM@-aHmZ8U_9dR8H
zlfE2R87X#9X{Z9WQPvMXM1uz$quMWQct$Ua7D$mFoq&H0L3J!g2IP8Jqg(#Hs8gG!;c2AV{8UH9ug=yoKo7@xqGxAz!NTOleC8H?CGGsX^X1-#_?#-QNb$D|153vPv
z4n9=4bj|RdriEPf`Rk`*BhAy+R(+Jc?Cg7G`y#PfyVw2DxhnHqeM5lbze_sE42U5>
z%>Y@H%sgNY0y^A3Wn4j^~hFTJ@b^iyux3@U%$UdC)Z`E
zwLAWp`u(Yng@;A6Y?oi4=4pYMUfrUVb-&bXA8&d&c8J-jvByt$r<|F(c+BdtI~r}h
z=ZwCjZtz>)e&FHfXU31rf2OJSNHhL^-?)Nx(+=K#)Rk6IRDFKZlJqs5v;PgSew;RO
zZg8sKgz}Ah*3Ah!ZGO1&SI@@l{#jdcy++>eoZQ^925Rgbwo7v4{Yz(
zlp<#1QPQ*co15A@$qMz47u|oTWSv>6@G(O2P~B^TQkm|#>k|wMCshtC$n!h*jOtP7
z^Ly$PO*mKqFaV=9QYmXOX`;`IvG--1Gx64=hYxA1h~=6_j!{NEm6I36SQ`B{%1&ID
zRzK$HMuoiDPeTqSPV1bvdj9H;xs{*t17>xeJC^mT?pDXs#|xTAJRkApj%H?;`oR$u
zzjoPwI&f&jh7X?ZrlYHGAM(5NyXNJ*)VS%D2cBNuyYiJnn2T~2xYutWb86?j7fVmS
zGcI+nd{KVEec6Ho(_F18b03N}SEq^fO;kRuVw6=fn$`|_pu_)dW`=LhK6UD&&8L#?*^>rMXy8jJ=j
zT-=R{{o1dfn=@4uif~&?oSJ#z;ri1K)^E4Qxaf22d#(@k>DXy2u-$XKH(0C2#sr)L
zSoZSBaT)xf-dzpPzfHU^{A&k&SDJQvZs*LWZ{Anm^NeU-xB2>9yJJ4nk1!|rU{-in
zkxqS>r$zfeKjwBW&wa=rA0p=ar8j2G&_Tb-6+i6lO^lqpQ1t)4L9b=!$k#U&?d~Rf
zK3V*E?#B?vac_?uQ2hCIZ||fBcSTBjmpm9bg}=XN=FXkt%l`k*sC@8D4fV7*vbt|T
ztcM(>`X-j2@fg=|_5vI=`*n5F2!}cXRow9a5sRX8e=&eSyLv
z!?4^EmzwWI3z|Y#PADyU*mJzg_Lj}~t2QP!IhD_U+!*YiRz0{&?|O&t&owiXd-w0|
zdhL-s_|n4;)2}t9x}N)Q?a8fI>@@BCojYd#H}Bv1Ki(X>oVdF8`%mxTf3}X9DCzjy
zTe2s^uJ`@ukFSRe;&#scGUC^<=-RMLT?bE{+WPDDldi!7n+H4n7d9mNNs##Iy&rAY
zUmXxrdN5JfK(%{GqHpU>yz+hK+Vqd67#xybIum#y67HH8})j$@J_9Nk{Ph^?(3V4
zd#cNg7XF>r^f)eZyYZ(B-;HaB)h+L*-R0ZdyZLx};^+}4dOd4;o4g31cFW6j*GxH8vh-lXk311=h2WkPjqh6*q@R|p>*rANP;H}n^MTAS
zEBwaoYuyy_YjAk2HNa{p`#%7oF3KBAh(+MQW}r6lSVHQC*5Uh{A>mv5UHH_zZY;t?
z^JG00))X5YIGvSM5U;u<<=3ua1;Vfe037F_`8UDQTRwg~`|R_v
z`BStMakE{qc=7SxIUOOPe-HlW(32wBUIMnq$SrWp>z@KdEmXhBE^rk#=l1j-)6mdJ
z+wd&~=PQa4C`g2*AodAWf2Hrvt?On8#%EM}HZZZOt<~W}j?s1aYspANR!q#joE-YY
z__O8C%0@~eN}+xid}=ggJ^TcbU~GNV)w;t0*qhoq@OFUp)lT^R`i521h4EOC#MHJt
z57V{>k(P`3{@dO^8$oGz(U441*s7x$nvR1y9Ouw_u;ChCxYu4I)wDr3_{UpI5!XH1
z(Auy4RJD27>3*7yB@f+S<&_y2uPWXyRWVyGYR81z+2UW!{si8cC7V9_m~z4KdP&WR
zz+9>3%ccS$B8LX-2cYGjg|x1F9OXU#e7M{P1?e%g?5Y
z`=OnAO3)6pgqPA$Nkx*Qy_2bR)8CXG2^h|hDLB|T-+@n
zKS<*xD_8EUY693mg0U50L3TKZJghoB$xp%C2A?QAu#~xb8k0g
z0%j*XLiTfpOtit@6OM|f3Ig=2y*)bTc4HvOIt+!v`i>Oq&&(Tx^6cDq^k^tH3+>ia
z6EvjkhCwdDQk_O2WVSuz+#OyDbtab8l!%~EK*c+5H&@hx2h)r)oe(oM6SOTWBZGPM
z`3}iGHG?et_vYn|h^>r(2UN~>hi&m=gEOTZbBn^=;27|9jEui7Cok_ZZ5kSpgE=`L
zV1&4dE19(xzUZJ2DhIwXPWJWdnXIh2Y_Q?AFp|=p)e&oV;`oUZSh4{Zs>wS+93dh5
zpCQYfMssP8Ix0+HOQy-XGX6LexEpO?BpoRDfV1AZk`SnD%znDYyg0IH$5i@222VjF
zd)sn{(SyOW4;%6EjMl-?tTJMIRhWStqcWG
z!}*zM7_2o;c0qsl12}fig955`28s<6~4-n``WX4TLuK
zVtq3)k%!U}>NI(BY*iDy1S(wxO^cg2S-}%H>aHz$0Y!KfMO(Sl(vRJKntHW~zq|&Q
z8)CB(jyt2?l*m`GrMLN{LPnNvA2dG`nperKE`
zG&lmR8PL6qTsGfs7Jel5!IwWO-uR^+nObSiUjGaX@!?}~$gb~yfS05otXMRii;f#8
z@43b=D>Ku(I8byV)H7h>$T&xmSVYlmg9BU*pf)E~Ck6$vd5M#cxA`n3N=qU4YaQgz
z_!oI;ae*heh$9AH5*rI>iaR=jt$Nx&J-zq4x9!)fS<_9u2m5R4=%o);z8Eg>R_j|H
z8LnIBI?Lpyok^OTnxyEZY-iO1ZRNKuZE2A^y-)jy+u7|kdJx~PIbWdDe5QHf>lZIv
z+b4YMzS>zCGxMmBR)77u^UIZ&LXm};2KQ|mCX)ylhWYDvSr8hb@R&a$TQXZXQAbB~
zQWBK{QZDW+d>_uP6DRhd@V;4_&mJBY=3t@+YXxk}aZ6tbUqEk4s6_7qTFY?-i;o)@
z@>$hqy!O5KKMvs*#EK!}5cn-;fqh~Dc+3*rsbs9Q?oycCzWoVR2@EM9A|VMr3>HXa
z{Tz39UjJ{1NJv0fDWqJjTn-$GcoRG~-gO8y6NYrOKzQ>pjLWmV@$jbWNGy}7s0fbO
z<8oB~1G!|JotNpL6nXa5x#bB+;kiA06n$m$mg%%pXl6-Zfc=b6&SF-fvXWI#MHoIt
zm9tdm$>YbIQ_wqluP5wv2xTlYWx`fVLbkFD3KQNcMvizqOwAOU)T*oV!X5u-4!N*o
z9pDYYDF4}KzVl+UBnHN~t?YK=ht56|u5PkUJ<2+fbw`Zv=x#s7^r~<|lA&$(Q-5EM
zO~hTqmtEP!II;+XOfb?w{YZV$$nXZ5uBiU0W@QcVAH*3-TE=97$b=4z_wL0$GDLI4
z&oRzh3^j>9xV05IBZOPi@SUzRG3kU@b|5tOoV&>jxOtO#`RtP?-w|PuC2`wH;bC0B
z$&;(F#}T5Mk?Fa)m!L42r$-!yhc5?5Nhtvg3IoOt-J5S2QizO-LX(P1UP4vWGxjKM
zCZi)W;Z4C%zD;b{b{m8g$1+bA5>
zbaHyv+>A2=ahs_RDu=i4-!CaZ(xHu;8fhC0l=)fV5D1Z&h$%UKj%AF+2C_GWKb9FgcUB`b;EE#J(D(&Yf-Zpt*DRn*NZ`Z@m*Jp&
z$>jNi&SKX98rWtuitn<-U)kKFv(RE?{)(=h8f*J?@8YA-hy=;E>q^)cb>jL~$`-Q9s8O@6VxNrI`fkyVFv}e}?T`8NNzkiovFK1{Ncf0vxOUw8z?aoVw*|c5jU*dbmNOn`7`H?e^ShYb?
zkVyfM0|LGgKOQ`A>pMZ|_hct05{r<-oRI}9#0!XO+%+}xfqx->S}dZgla`W{6zW(=
zG4S_DS%R#)(K6w~Zf7Typ|O<#^j@IMOn{am8<52ePcVT?`G%<*7Z2?f0W~SflLO7-
z0menxL4;VVn@|UQOwopX?8FHb(?jFT&Bdff_jpRKO>MP-7k_+BdQjy>gLuI`QK)a_JIT&?YS+UX8o?a-z!V-8<_L`L-B+&1z6idXmVi)GsE=HQpuAornDXde!e)_~>DDKLRavxTi(kU6SbrcNVn7f8-t?JWCjUWa|
zP=}a}wCzRgx)G{{2*3kIQ3xvx*mU
z@llcWGQwTDIg}%!}
zDRlVM)D1$hLnI4^&}Pe;lpph1$!OFK6T_VdKkd
zK4vURspzHS>>=&4VlCVnEg1-x`$#F3LB2hiH&Qm#R}?11ZG`tgkS4W5{>+8
z%t|1tF5UhZfTOT82q8UQ+5AJF?K4!#&DFd-x`2JoA$V7j?eEVNE>;*Z;uuX_Qul}#
zU1Fcwm7mr-JN(T}SLlye^mh-)P-wlfv?Jmvz5iEIaD^eoj
z3JG=6o9+~~>er`#1^-T%*D#byA#%=No0Vz!)({CPJ9=i{>YaIs&BvIZ{)I?usiug<
zUw-MbWs@u)YslZYb<6O7ihbS&H4Z78ty%Nm#`w2!pK1MXwTz&)!3B*kw5PKH+Jgcj
zEbATW5Cl;h0s}pTzN`I=)d_*H8tG1w(MeKEz{fXs}xZJ`(I
zIlEzreG36Na#Cn}O=-dbq>nNC+_;+?H(GY6kp_A>DJ|&4vMA76lH+s=j~!ctI3URz
z$2Nke==;jjc@3#RDkx#mq)>eQq2>k%sh!CNXm*6@c#4dtBs!Rwn4x1f$cL`li2
zmK|MW6FSuoU%&1-!`wTDQ~f(6No+KPN1!wgns3>$h{tqFEPKJLm|5;+Itn-wZ7#AId7w=!py1SI&GxhD$45in40a7Se*7
z+gVhOsbh41^=(~QmMr&ira-ya%hbFGco9zmUIW0~Lw^g$4f%zI?1-V4If~Goa+yDX
zn{eY=0L8{iY0H!=OKQ|`w
zK#_=V!mRMJ!|~cgMp_r*U7(+&x58Ja+7PRsD*gM%wq5rzqduc?16%SrfAGKozDvx+
zHRQH3A2ZuCPD1jWrh1RS$Amqbg!FS~&jwnt
z{Z(cEbNW1{E8f3A7GstFIQ)jmEA;3|Q=6xZZHs7xFZg*}|rFCrJ)7CEQPE!5Y~
z_wcYZHtu7n>N77TDG6{JQ5x}__CfG^V1%uYzrX*|rH<*Yh>NjrAq;IE&tSdSQL@d4
zPoKn9CaPI=W2ZRu!
zhyCBqreFSA`qqZbqEuWi)TP(~*S)Ls*OqyUoN&-T@bIqK^O3gnyxiK^Vl&a}&Ih2kx
zOTE*^JNTbSN~$B;aL}1_MgWd@BrL47;vQck@|sYHYj2PA9MdcoDgYi~M`W48hNqKx
zd9NQnWZ-L9tsfz0wIy!cPV?po1-rl&#l^+h*)qF@VT_#5ObDmbdfd3i<9ybA&L+mj
z*q~sH0EI#P;<5P~GPuGM%scw6BB{D@n7O3S44{b!~G#ZvRxayhU%Ic7H3r)
zn4}yaD;q~)$a_IL;wf~jYwUK>$0SE?n_}FU*vb!IzVz!G07F5M5IB5AK!9}3E-A4`
z4e;4WU}%|{l-spMh!Gv-Gl#0_F1?qwnI`Zg69mhz1xY!
z!$`-uhJ0Ek;c9Ey({Zk(WF9U0oja$=69nN|291lLFtFoB9%j0+>==Gibj#rbx!ovj
z@o7XA(%S9X|Ne&{_C)~NeEquS+N8-MecgweXrZwpc97yw6-sa0lO{92nPCB+Ng^|H
z0^9pHdXKp?uThZNGNHHT8e6{EKk;U(Z8YD8I5%@F!_ggZ=foIKanRk+lOU3e%N!83sk8X5#U
zHsJ=8^k(+R@sa
z?2M73(dt;57ZvrQJY){uQ-T_>jPU>Yb5GCTfbcyOTf)^y3`}yGcI=Ter5|YGTt(0%qQf~fP(H8Om%X)f$bD}9QGAC3fj4mc6)1Y`|3xA
zI=DC`t*TI!oBrUw@*j}6e9elPRDy^~sdh17@gFCxRRFBSR-5SG%t>Z55|{4!?%pw_
z{#9Cr6OY;O#I|nzOPeP?T_ih1`PDKcbW6=K^3STs-he!X7eWnP+-s;>xcH9l>9pv-
za@`n9h7X>R>DH*~X~I^Rr5I2)j1WJiq4PMdM7T^?h)3aNeV);I-z7^bo7(<0na51P
zd#1&?K9tD{&x+S#e|P@uS>4226a(y-gE)!KyMthwh{)qbT4`v=K)oq8O0=h6l!?{A
z10k&XlC?64mQ~j-)C(-}GEXMMu_NYZ_uoxcteol|_jv~hD-B2p)C?*;#$QOky8p0{
zhZj%f%eZr-`ZT?bw73ybBYY%xeHfyc;F{x3E;BO$5+^sU^7p4f1cf#*GD4{AAW+t_
z3QW6nX&f3lQ`0r=(;Uz2kk~CUr~uFhSyV&0_}d=}e=r^dg(gUY=E}qwss$_kVO<34
zY|mf1G%&k?G>Xd=EnZNQy)B
z+@eaHqbcrbYnlus$&ssHN%lU6i$-0Ea`h^<)NB}Z^Tl?Bfw*2HizFUaeU_Ve427=J
z_@TkAyj7(AR6TuUWT+h|!G1Hdks8>?Y>$~6NC!I$iQ+N^5i@{Y2;%^|oVOTd$D6Cf
zzCJ_gO?ki3`B!VJ>#SLvgapjqgfwq~rnXb;aoX>C2bi;4Y1ovEV7J2np0vjO8l@u3ry)wtLSWr-43c
zZQ*z6PU(09oSc}RveZsshRz7b#^B)`FfXva+GUVm5-V}ZQ3U=z$G5yTehMfp%EYi_EA)P!V6JV
zeU5V(uwt2C#>tcFDk>-|z1`gO(ie?pjcCGYb2$SMLEiVL-|vuXZH(Dj~=EhLM6bnXXc}SzkWDs
zGCjlr`~PZHhWq&V06TvIIlkQ0P8jEKgJ3cOi4@?B4$q+xJ2U{Ciwd^xx{&a#Q{@BQ
z@G%1hoK>kHRb*cPiKF&M`|_ZL)`mz!SkB3+3o7l1p>L`8
zzJ9%m8xIFRe1i7)4A&hyLjj2f}V0Oi5w?7=W5-mOx2Kf<&DR
zoJft`B9q!GoQ>uuY5%(OiprH`{u!S&%=0N-x4dXFMEY{%#Gn51*FrDhfJSPjBH)b+
zyBkpI;Hd&-rcN&4-QV603B}S^W^(
z?%A^s;7TZ`XJk?Rv(thZH*PqDWZi=Y-{GtXyM$S8B?l0yA^M+{%a`LF$x2m)<69!5
zX0o`5ySVq*4}Tfq_$zK@66s9Sf8NYWwsqiLLQZDdG*GXZ687S5_V3&!(iwcF;EoSf
z-`9Fqca%LBZNY+Zkab2)(4Xv`mPqFrp@j$0LnB1@zk2oOmK8(SEpw5%vh21xWp$vo
zwyL4dzf}@#yNOHn42yZ%4{IO&QeV5;0(y-MPW0&ynA94|`CeBEnvc{nJ1ZjhEy(9|TW}W#heoLUx|IIO9g`mpz?@Q1>GTfIpZA$4=590^)MLVgZRk5Pr+
z-Q?||g@Eq7zh_Cdvg>oF{>u6t!puz?RemJ8Sf25}gk8HD+Bz7=vuhcIfwI}!^!4`U
z$sw?fq!8|=;P+U}AW&^ke&i2e>ALh0@KY%-y~JDYM*W8LY{SBDbB(;!Cm8W6s`4(tG
zpph0;_A=h-gzR6<2p{BOvJ#ODA1cRP!>*MChB)QHhaf&Rt(K2d$)!ugq=){(ZK7BZ
zyshfx6M-|--GBK~VWV&a-62UGG8Zx4(4p6tn)?QX;cF%&&-L|%I~w#qlp2RjJT^5G
zzOo)K2C!%J_+lXm7gKBeKDD&q@fJTdGxEvZni^KrW<2}=|D>FuDd(c<^UcL9i>9z6
zWa4K}nqG$!_rdkqU?M!d+6nzxT;gL!Ab`+j+;=S}g?Su$C>|$!nTtzG_yh|t2Fc0~
zP?M7kat6kv#x&Eo!wM$K{pive*oC;buzd@Xf^@M$KBG#gfCI39!3Q
zD=j@%?sKiGYVf#?cPcBRYW#Sh7?hbFLd#rF4X|Rx2izCRn#3Y}sH3H%2@8?AZY2|3}lAfaRFK?Z3UJw3jyR
zNNAG~m3k_KqzEBeBq21`kk+RSQMRlt_L*TS``(J|51Nr>REQ9Y?5X#2&+q?#kKb{;
zbC^-jbKl?ZwVda9otKsrtqtQhR=IIdMZFGF6mh@!3&2mr6%|$0ls#%KzPQ9TJbwE=UmE*
zb|hE5q9e!ZsqzoI#Un9~5?|nEk!flLhyJHOJ~pj`XoKY8#HSU=se@a}f>Pih@#f7u
zZa8WjULMBGW_gZX;5Jao0tR3;kZeEVzoo-H#
z@WDv#j0Q%@oMYRqBvOHNWM`+ITEPY@TR4S^u-&dJ{C(uXq3wc88j!!kS<=dBibO6%
z9Pp;{DgNy8Mc;@59dVZ8U-dj=k2OEG?AU>BmlRJ=Kyh<5>Nsw0bZm|_lFQSP1^8H1
zXamyfpSX;Ai|Xpm9bu51U(alqS0n+FolmGskOHz%mj={^8j;M!)GGdg*CNM^9-aIb
z+f9Fvpo&Az9z!F*s9x=1t{8Sj@jt(e>-A32Ydz)Ul%Sv|$7Xj6>V6XkT;+{d%?bAW
zCfG>=uU@h=X$wb3rmw80le06FyNzcvOSO@og^S#Wof&XsK5eeWy|4$aj6>$#P!xgR
zl(DIE#5#T8h|*wN7E(D_oM1l)l@L60+O3&ZzOM;{W%5PbSJ4Eg!Glq~aoE5o`Mz3t
z3Sm9Ae*)OuZKJ9nr!7)fXEy)_^VmBoU}jw!Oe`6X0f13)4jQ%U;v^T5EF>EZN?7m0V|u1k;&9N68{NKY7*cK-TWPbL4cwKXZ$
z%hwTX9TU;$=w{L#cn@z)Q0g(F3A{>$Y@9v#u2Nh|PWPHe_!;5Uvt-4JuZXeE7*+aJ
zeP54_7;8>U+~l0aPmYp5!A3J}d;9f>`}Jc`nimMBVEyCiSq>~zc*}8Y+PJ>uHkeT`
zSTA{vV3Cx~xe#m<@@`NH;HOI@cTMbct9varxjWWE>0<18{@b{qo@q|7yJS9ohkUM#
zd$*<0thI=UI5zkGL2ekFV0Ir}=`htap25BP`b9A@7ubIXMQ)#WBcc>Oj=;wnE!m0Z
zusCCD!NiH5xDDKOgfN0ptXLdQF@}tlM9y#Eb!mlr(C#sC
zN6|;$eD&%pLO2CBt>e7U00saqGQ;##0xDj8I^Ai@-_(**HU0Yg;lmciD-0ioUa9t;
z9)gI1A`E+T*q!3TV;h&qlz{&*ZmzvgGn*^azagd(LbQA@tOl2M4UZiCQprLKn@rz3
z8_m{{Y|y`Q^#!*S!Z9=EHO
z(5?OnEKrcRjo^q^YF&_+^fc7It>-6|KLHB|Y|{MW76W9YF>;D*PYyb(eQSr|?NE7TM|x8Y66qwv1B0
zrde>u+{C0~_H5zw$YWKkFH!)8qV*Fr;tiRPYirv%Q=X>ghK8AI*Pcu4>Lew@pDTRw
z=WgkoLqQYZ)aqZ?uY6Nq+kpVGwEp0a#0SoLLyuI|pL1lz-4_npc=i6@e*@CuqOlmj
zw!Mnei_HPhP4ZAiF~RRa#x=Ac`yKstA=}mm=VQW>v^}7`_qGiWlU3VG?5kP
z>8N7Wp6!$yu;>|+0i5PE1LNTZn(4dR%#vr$I2E&lz2KPAXl*?GdQNiLRkarL59g)?
zs49^`D3#mf$p=BUE5fK6$opUL3SRvsCiY(gdtEjAjaMg5nIdfWiCc$DrN_#T)pzd(
zT4+%&62)rj>&r?@g^}Zf{d@PGjqU>Lfe?Gvu>}Lqg@jxnCHl3YD-gSyH@~4N2MQTI
zb}Wv(KY0?d9hs_`ry*sc+i<7hNGwF65MTo!!zNhP)t?NnN(UMgC4K{Cp=t7(h1^8V
z{ZnEnNLxcgA@IxY-64z$mxSLfM!3wFE-elcFd!3zhlnmGPn_s21uGH6MCC($&Ia>T
z>Pn*Sn;1kg}iU24l^^0aftGr6Qac9gR1+PFB$L=|3sY0}Qxj+vyvxPox)=#i5~kPZVyBPtxhDcOAqA2Q?QmhLh8#ojV10
zy%#NX{V%p}H*EE|iDwuJ`W-ur7LS&%f57ikDZX8)^#ff5$q!_gHK*dX&9YvuLx;X*
zd+5-iZEe^QJ-cN7z9NxgW+84S)gzvOA7X#z
z-DpA__`I#XefrRkV@2wb_6mIL?XM*;tdv`09R-G>^tEZ;{
z@g%#X8sV^y%`{jeu7~VDpi7`#LzhEjjG8wOFo$2+eo8dKOKTWp*nbm2|1F~CFAJ-~
z7c2lxmXruv6xgtF2Fwh*o60IfHS2)`mjIGlX&JZXXRaD`ly_~be*;Y{ab4>wj=Zw_
zw0`p*xkB!YRleIwb~mE;GRvrJY=4jMq^Jv*ei`b%PTcORdZCzb}cz*JSYH7!01Kf}GE
z>Sc+!(;5FHU2~U*G?L)RoPj5o)=Zx`ar5N};BykUyAHs5Bi=pnutF+K)fxvx{
zhp1?Ja+GkZ>sT;%F7jK_F{p3F-20Co_XvE1`V@^4)^RLblM&Ab8kboeNJ14qCT=nC
z!$ZdLZy%V!s3Cf)lnZHhUV#O|>j{o`8`3J){eJfB!}@yjC$1~q^3&2r;8D&Px66`l
z-P+F=DB7+=0vVe>F)(nz%Q){9;_AUP%#0cBKXk}Da$vu$)22+h`k(z10eSL~U`uOj
z-1<&Z|B#-07#UgKaqneg_2t5q0>$BIjA1Q#aPP?m!xK6T{kH-i1vVGc$nu9q#>QUG
zgSW@*+q2TMfDA#P0Fe`BWV9DqD8&T@)zSp9Re_1EZgaFbEW0bm0)qbXv&`64bZxPBQqcYBfiBGv_XX;xHU
z$sLjzNd_U~f`8AQM59Y#51~vUm~y}hFc($;7lNGS&A2*Y@KxB1p*nP-fMy#SiX3SB
zB+mYbo}f;H9oh3{tgk%8?$0
z`hqe!G&B*$%;Kq)nzMFA(wc6*rV4pz9;+fsJyS6QPK(P2>qflD`BvC`O`+^$m(yxH
zM^{ErQ0TFx^a~*8md|T%G_h6|2M{0|x%Empvig&;YmENhS`1a`(7&g(H5(+f&CE8q
zmGX?jUloJpv4>#Nq_mmj|5{`P$y)e=B8gP$G~k?$A3eISy}f?AwXTAZ+`n?-F6}v*
z*IJuv%-I99@#?*&$#QH=)
z&q+^-?BG6SK_YlvfH9()O?eE`SQ1`N=*_r|Xdjrq^Hg;Y*T2?0@YP-StgFujF~>0Lb!8e3eIPo#@@-
zD1HRe#)8uvt;(CFDr->+vu(&W(c&q{ljBz)YsMY4B9zh?E>L_nBQWqc;v}ozrv?Xe
zdcSZ)&`4N0UudhGoE%s6y>8K3VUq^q$*)u}`2f}9_y!*gyPL7cvAsv;*RJ2L+egc-
zyFA9%mkO|f4~@7Wo=S)6xV*H-{srgxp@cL((==C~_?Ez4kV6-{5`6GP5i
zN$PcE`e}dMvhWBC-p_+~Y-(~|C99zWBT6~TKxx)QvNtK`+ac-S-M|4U{&v6pgYsA`Sxsl$xC--nzuHn8Y)c(J|waLrKoatF{DA|R_EBi$;_yRIL+(Md@+a}N1
z|2rr*p!AE^uT}QRe^3zF_388K`Ey+Zg9rESQEf^roh^6+y^*zs?^n1M#bR)WZ>tH4
zd?VxlYcn%tLMLjA_OPOn#ih&9aA{ND5t@>uB}Q*yXGvCmISsm&D2S~$YfE4fX>HS*
zWNL%h>|9pn5zSccoP`U`OikBNp#lG>SU6iCIX_1^Fkp~;h~$CQ{`SsatGCa`p$G>f
zjLu*{JinvyU|(S{UNA5nX{Qg)@`?_>@tR7WmRi^gp|vVHTI6;Ui73SxxmDlXoF3o{
zJ-lA<96ynJRI;U+iOCkw_w3xAJ5}|15RNBKlvg)jG`G2>CFJbl*~pVQevM;Lu8`TJ
z-63?JQ6#)Xc-HZWisvnNR_su*u#2vqPXz)*QC;Kub;X;mUDdB?Ve0bMnsn{er0${M
zBWowE#o=pXR@Ufo<3KSLE;K@$JS>xW&-_cxoyw-gG~{Ff**>qBdiAk_>!{eC5_W-A
zXF9*Am#7rwb4coFKb02hJ+k!QeJdfvJS-;$OeFRLC879y&%&I({rdwa0Vr_WmH%51
z*l(-q@b~ei3Ua90g*Xo|6ITR2Bn3qet!=S55{gyvkF_G*S+c9z?qP;
zlPA)SppylPBta9AIS4Kn(#GR9Aeg^EZg5#i(6IcOw`B*&Yw6&wwSnFTdV!gh6uHBH
z*u42gDFIjt@xV3g_OR~CTPZRdx#U><*^IMKfu-8wYteKud?S{9~+b7D55h=dqY$Z8p>snC1`sE|&7DLcB)Vxl&<
z+|BI6^*brUCZl0KqLy=Q;+e5@{{U=AyUOF%TgmeO$uDG4oEMp*yL{`G`Zgm_EbvAo
z86Z7QJD$+NJ$;Jg9ut2n2M5hd6@XhK_YK%_lZKyzhW3`f+-LA$eT!mYZK+Gz-{&&J
z_`&U8z6ks3(b@}@M@!E>eX!X?^QOMAJ8~QJnp!cb{)aZRVI4>*N{@x
z2m&Q&*o9fyM~{Yq!a+2;xtS;)GAdr?t5WzWzOhwdpIv(NW0}CwlB?@(H0>Q|<K(rJ;BE>>o|3FoM)6o?o>dj#vF^>2f3M$n2bW
zoj=siKFM{KYwIE$Z*8qs?n0pc
z`G{J>!q6P0?|o1%b1|;1vZA6f^ssSVhXXCwse{o6^c$-;Y>V#|C|Bh4@~8H12EH||
zW;yX3yUb}BCnmGusvL(*Fwqy!o*fe$amQ|Bw-CBBQ;!5UPN2_Hw>F5ruD0jO&CYe_
zRX>@_&fM2?vbJU(jgD5r;a+?E@iVH7QViacwoI=jfibaDESDr610JsPj7B++t_w{H
ztkhjC1_KRqbRz#T()42UAx?}-nK#S%KG#{IbYh^J+mpL@!{^WM_MglPGqXY;=(%vw
z(SrxS0I)DfpimTOg^N6II-oz80oo2wERJ%Pi#IAO8CkTvaLcxBj0xOB8^N(Nw9VGAcNHT^
zLAU9i(BYEhBaCCWTm=9LcL|szzK~Y0qQ4+a&+9VfI}>V_@k7>KP8sL$#1^3G$Pu*@
z-EtKKVv)iYCXfm2lyMe@{lkkh!v-4O>Jx2hJ00Z!zsPgp2&=Vk81zWJdDAaEP%?YV
zwMeE_@M(Pv%l6=b!57xNFNWAinl|U=_EMHofL?n$8u0GqlBJZKffa;_y@iEZ6w{;h
zN>0m(1O$P??CRyq$Zx%20R
zocTVFbeeLX^$N$)Q~(5`)i};wZ2Mg@$#QK<&GR;y3~~LSF57+cAA%79q3~e&TNSYG5`Q02!ZPBd%c~Nb98*n2HEKB
z%-r`ndUR!&BUn?;1EI`maDwo#puEr3Z0KKeJcQ8`X3SZgRXH)@=1?VT6TN;AN$VjC
z)ReFHz86yd5XXB-az;vdY3n!CIXw1%DR(&uc-N$>Y|DY{vtfC|<17LXt^-g`WB@K(
z?Br_$?t~*5Fi-3NFIH+XP|#&*IV;j>axY%EfVDze*764D)r@R_q~Nk3-%ox~4>L1i
zwx!xI=_Que8~qPK*`XBi)a*9rm3L8VK!aFsMoeU6_0y-UKR4as=($g}9c9fRY>qLN
z$1306eS%Hv8HlJ9*Vcqyh7D!zG@#mVz|7dWfEW^%TXAc$MI%5jPKTAE@X}!F8qO7Y
zgilRp0gOR)Yd>_T(LaH@x|`8YH$LodYZ!ZgIDg>4Y!Wj(3=^t=`pT;V=v+N5cro(w
z6-=iWzQtAysfADZ&Tq9E3wBFZ{H(ku@)Lw}7SB}28f^a>pcic;p!98RxtlkmwUs-y
z31lUsN_#`O+f0@;7UqhRgQrigDiU^w=K2N&1OTrje=<=`vjQr5F|zqjCNsI0WnG=f
z;J+|O$m3259bw#GXR>na_Sl`*?@rViZR+GGKm8|Abii~hHp3hPdM(lG)*kv{b=vG6
z7dD|e7ay3w8j+emcu-kBKZrfSM&)4&@}(f7k=m*Ehc?D8%}Lwp+Y711kf~G8qDhYGuX==xlM-N?cZ1=tQ>CS{!(>*O
zoEzcdTiiOn@@SIEgc&nfR70_tUej1PV!NSRRI)hZfO_ei#HpSal9!8qm#xUDudNj-
z_C9?cNTo7Ee36y0p)bhIXD;5N@G@_}PBcwi*ckK+d@kGnUH@ap(+2sv%xjK&mOWE(
z@-%>WMg*$+=1_kg8day%WnS=gua%QSLp_x4*174G~A})
zu*I(sl_DT`m>3e6rckoIu;C^3Ivt&rRg}eF(W0aZg^ay~7nr`V%idR9ePfY?B-O0#
zDoBaB-|XMG0#C#0Gsz~k(cgbQc%)>8dsnuiqtNw;*JUT3J-yD@Fzc!3zfcqUs%f6_4_|P)q-|bt)iX*P4
z{rV0ANP}$=^GYXqfsI!kx>Vp4i*?D4l1>VDfyG+23Q)J(%o!A%Olnx1z_l4B!YAim
zoR;zKqt@wCH~^p*=u_$!Pgi9TZpa}ds(0_~)QYgVC4}-*l{XB;2ggmss}my3j2YQ2
zyUHX|-N!Fwyl^XCNJ)g?XWoJZmm^B~4ipNG4}2_~nLcE6G)B}lNTO7jV&R-4@|kHR
zR}`pevF;Di1fsg&6wCBGAZEizxH3HZddT`j7+)OeIa9t@vNQwnWJA!#2v_;SJy5W$
z;PKCQ8!GN^{m(Mgg3-~wfF&v9KEh
zp8(Sh76{H9vCAO+SC)Pu`Y^L3%+uxUz{>6+YR#e$^%y#IQZUFnVq8IzQ*Dg?V_<^wkm}$Fy+jljXB`
z`SRO$?kG>YeI>gGK$&)(j>dL1HC@RphDpF05!F}ZJ4Yddju{MSd>Xz2=Eqi_Z2*92
za6yo!IB0NuV4Q0^#}lWJ{_|le=XThQG7t@PVdyUN87}7)&Z3Puz5;e#8)>%j^wQT8
z_|^dmhp#q>8Pr!iB_D4$HbEFKm%%51W$Qpj(s{9cSqU`0#a0Cwv&YO*gL2!$O4Ro}cm
z1x}VPdDr$mzBNcT!@e>4C-xwyI~gplJm2%%=n*5D$gZ&qd)m%G%SG#!Gt)#xNt8-v
zgKR{N9pS7&&$7>xPD;H)kU*tGbzWF~vcXfk-&KA{fN;#m%JI%y%qm
zo+GQ7!%satfxeF6yE$|ExL8|O#C3JL7u-oC#4~cU@?~2bLMvf7j^AW!CcTI#h9~Da
zQgH!`aTvk`BMPtyM6pIqxn+wUs-ceWIDfOFo5oK4`(iLV3fv4Dtt|P#r1P?Bzo1Y6
za^Fnol^pvD_>iHs@VTSbDGWQVGqc3<$7RG9VeyS|7Z~aCyIWShFF@J%z6S>w5?#I@
zJCr0Rs=`K1FH5*cT5);9Y&GKt6+Hka>7d}3Ene#Pla35mR6Kz0<#Ghy+VZu|wTZXb
zpE%4wWT?Lmpd2EIPBcB*q`{{UDY)eVpC9UfSr{vZAF)!aEHnlP@Qn_SKd^s4W83P_
z)N?IiSvc8%w_wSy*aQH;VzPlC2+Oi!#gGffFI_f?6N-ADVSom(Dw(l>aRu`Z>#pc%Hjx|?
zCrxr+_C{Evc*Xe50BwbyZt+&hOJuu4$crN#T2Cc`fZhsoltzv8QIizogu}7HGRFCg
z&Gp*tiWe338@P?4=<4NEbNe>K*Sh+PxanA;`}#5zQu>yMMP#Q&3v>@OUz<6kgD>am
z=Eh`IQuSQ7w)EF(ftm~*q5N&^z+4^K!vukFT)iRvl$HCZ8y|nB^a}C(XxT&2$qroH
z%gaBY2E^(wtSb=c3=8#18l@b-X3ew5j}P3xGh-Ta$Oo}BPoO*ZF0F;)@*<84A0m04if-bDcA-D*ROrtc$f=}VX%o3gb-IHyUN~)4Oefm3O
zx;qzm6(P5Rrh`d0!t?C)(xv^nogM76A-!|o`w=VWHv}z!zNj%69jA)iv9T?#CRgd4
zuggY_S_P|3<;7NROPm#=d;Veufuq7;bGy!#Cu8Bwo%U8$u(RhjaTCQ>`7QNo92ITf+hzXb>V(sn8I=~YVq
z;5{QO+5uTmlvPiX%L{|d;%Q-`kh&Dh0O_#UZ$8i2t`a?G&R0~cpy>|V&&W4`-
z(NU`vjX0sKr6BWK_Wc?baJgi8wy3EoJ#T4w^7LtbZf^3lF}-x)TFVGC;bT`J#1Pbr
zO*bn16i5{^JjU
z;dv-`EpjmyWT;G#qu}lSo#KP}T!so^--vlicLT0=(|~5qAHIe-h0+b}myAj~^p%A}
zfoP}l$Mq#mi~Q!u`=+wk9r$j2gKONncQjd%kuL;3%+3zRgDMu+bb47nQfifV?ICKS
zl4E=SU|dA2)Z;DrOjv2T-N3
z#J#3{HD8C4kp_fK3?e5z9UVCZ<}v?5oq&Ng{M~DsHF~$jv9To
z!D4a!1O|ix0+MbjF!K6~-*Ke47-|9q>jc`SC-eDH{FlIDhfC?@C~UdUgE
zY^myOE3wh`g0|OLPa}wH7VEqW4IRML8JK;KSOs_kd$IH9f1+yDdXX0XXUjT?C#z4{3IjQqvo3IvXHq065GQC5VB
zR=CDJ?tD2})@uSiF~_W!R+_tpY@f^uEC27w`FY97VvK&cFHFtF#yZvs3y~OaZVnGitp(
z-Sq9VXZUw)gAjoCXW9!v1YW
z(%4O?=oRHzxD)h(&WKH`yaRFPNua*8#M*|*EYv~*iD*ASa_PQQ3StTyR($wyn9~jjXB1iNUrJgsg&5H~#04^%?vi7cvF-P#l{NB2(z8s^*Oh8oL72X+I2;`xw9J=I%3enfYP9BaX%MZg=wzGbQo46!ck
z-K}0p!!%P>_Im-`4ypk(rU)Ygy{t_%gtmI;&Nmn^9yx*x!jU{FoMPCOrdDK2kC5Kt
zZ3R(_pQO99v%-KGceRq-GymIp0zHbI8n|ay#>F9;f3BS0b@i*;iO1^aSLToP4V2jQ
zy8K(Ds?p|^(O#pCbiCw8L~WR5eSNoT-z?|R&U5o$t$cd&aoO9&q4moyCaykxIxc_7
z?N9Of?>$0Re)zgN`|G&}^Z$KoMokKXyqyJ?efp4#fH29rh}z*Tu)o;7c{5xw>p6z<
zxjDI3wGY_tpsn55(&BgMJ`Ep_nQ=-aO^l|UL{$M(Om)1$co6IBkJ055uAPmMoR;6a
z^1O{~)=)S8(0vN-FjvOFY2X;{HXXWnKl&9$0Ps&4yz&*v0Z&R=^3zLR`IoedwT8h8
ztWwq*HhAzqNU~^IU0eVORti_};X~?sWGcAH#cV==@;#URWv5wgc?p+2LFusi3+e!7
z*B8*g6G|x|d>$O_SfXkd-`PRF`p&PBJ;s0E$$V7K!aQX!R?Udxa?QP>`DOzKl%lq<
zs5hsU4u7#~^=gbvK&`s@&Smr$mRi)8^_s!^mtvD@J3(UU;quR%?-GBZ*85MU9J
zgdN`}FIgTNE3;o>ae_8_?b_qdE+oqGV}5r>k+bFRI>7J$+kK9@hK$OM}G{_`h`HkF)aVPF9klVU1ZxUgL8
z+3VNf$G$}Y6mbSD8q1hx`NqR)aZ9zabYB5BKe+z%>og}IGR&8>p&uGJWL`6%`Mt^|dlRme!rGIn{znqv#$3i-17@vPb
zk_ePpcqEvaWPp<76N^$*6fvx|=@8RQ7y5Qz^s7*u*u()YkIFL4rD4h10yaVmC?Z$(neZG1B5VlJ<)1
zmm{a+8cLza?>XEQe8q9_U_5zg4cejo!eg1~Lcx_<;W%Ib7(LG#=HD~fM4B#psXu?I
zc_ns4ToQ;G&Hzp{oK^$034@QqO<*f#Xk?UaGSq3vkOz0}Xr`^#IUx;;q~U;!0-IqZ
zp_VUCpOlxUtox|8R$)hW;e3u0@q;!eb`vY0#*G;x_fOOBioMDKvO(ujSzVv<9TqBP
zi`alb&>X&cNqQ4i2#ql{D)5_<{h%NI02UO>CHKTNH?La`joQ
zoxO>%$B$10Lc%yTirdL25lYR#w&p){j)eaXvcOZ8g&PH0t3H8Z^l|T*h)m}4a
zn(g+`zR#RT_m=$fJL-e`@>-Nqx!zdpM$pyGpYJ-a*2cMv@QEx8K{Y~8biBrc=IOP_
zp*Q6g+332uv4R|v69jtDe1ZqJkB`F5MN~B`6#sAFYww8@OSw3no-&g!$O}U?@rU^O
zb@oL~vHSic#^H%GaU$S|{N2i2b<2_!|H;*rLhSjh90t!TBQ!ETy;|YbB0y{g+RZ}W
zU6}Rt+tuASnm}yOJh8t}mb_Fu
zmjUkrda8BKWI-gKNxCMd6mcEll&qFi{<&2^3Yd?k!2ECO^f52cQ)}FAm){_*7~(W>
z9;s1%Y(hYQ;eZH*yc^kBSwop>Q4LW(tYn+b0ppg_8D+-jQFfw=3*VUD(Iq%mMuz!V
zaqIMVN7D!~#A>&#M
z+xF(~{xOy;gjjeJjT{XC#^;vT4TjA<{;&m9=F6AmC1(`}MesH)#t+`Lhw&_tup(4l96mY2^7>*U|b93ZN5Wf+jfDoJ??Wn~LNJYad4
z?7)V#XhQgj;Z5=uMz#`$o0R(0F+~T7Co8dcaPWW+SiHnkuP>sd11C=CS+pMx+((k(
zrIT|}3dyb5T4{+%ViyzREGWdY#)uCU+$~xX{zuG
zw=>p-vFN+6!o-FQDzm%3SLE_x^JB2dIeb_~B!sxyT22>7j^-o3-p2OxkvA+`|`SY96VyiixLW2)aJiiN@#jp+4wwxmFAB~}P>sl%lQ_Rd_2
z{GQ|F=$PZ-*kfDaIm%GdkoD%d(b0#CirRQZ6hvE=8n4kP4*xgHvL%{jnCmVZqv;}~
z(nHY?YhJqK_3|{+f4zIt^I=^YeCjq!QYags<=S;LS!xr&5C?=I$y4>7KF&2Es|5yZY(4hFkEUff6sHRl5fa
z+oSP4X60JrG0VafHu_CWxOHkyb2XbYV+IEg>S%dtYh@+3%j!->!)lpf11;~iejBUg
zJt(%dM{VB7fb6cdsroKPd&4XfKPPo{m!zkabz3^jRPAP5!jHzB&i?Bw$|9xah5hua
zpLP_PN?jZNusb}`_gw3p+YNWdsC^R**~xCW@V|7JBh^VaJP_Y}^jF&Wgg?$9{->nb
z2jczqi}!&ElJ@XqT(Lv$(it@bw$D_t$1ie`8
zn*3K?_E$QmTsLYLjQX&CHumbQ)b+jfFG_BZ9wybu#2sHAFG|(_IHcsO^!0=IC^7NU~a+dh}T99b72KOm{zAmr)bzG3bw+d9$Urya_($>&ndv2HcmO6CQ-p+h#Aa^5eMW{4`
zh*!AldDrWVi&_&*U9>xDAF6)VHW?`ywfTx
zeN0?ZM7h;V!*=f@$u23OZohq!Xbqlpq*YT{eVm((%Hk2JrK!CY^cSXBrG<@uUU#ta
zXW`1r*6)&DIW~2))h9{a9Z#)jb^4Lx5umIly7&2`o@?}u)aB;49OexbP3@F5sGQ#M
zt%qxfn`WtH|8MJ_X}Xo)U(-77S(jG(lak@3y2+2!j0c5{=zJR9CwEl7M%nKFU00e?#uiazhghOcF+Em9^=rMy9OE++*G1xVJg4Jgj+9
zXGEH>T4T1X?nzLR&i$Y2lDyq!ZIBDH$f~GcW9uE^nl`OlL2IIGbB3cqSl5;NhK57(
zJ(sq=s9vGtBzHKtU14kLyWxYQb}{5ZFTjl}fNFjCkbC?Z#Usl-`}T&;9ob8VOG2rQ
zQv{tZ;sTOIq>`(5EKTV`wb9+Vv{<7yBptPG|yh3^Q=IG4V3FtgCML=dVU>Tbn3{@$D|o4b4OQ
z>HZ8APq*_}-7_nB@3*8aNuuwW;Zl=|;i57mte!(V*6DrDOguVWy<)t~yAjRTD>xi{a8#APp6@7NvcBabwh?lk*$PE|!6uxr%
zQ|+_d`Ge_Bk6P_)^c0fsh`&yh=*uK({I>1xu@Uzx%#Zc(n$_z3GE^e%@X@hSO*|u0
z^4s6(-z>-flU$I!Ir-j(Zr@{@XRB_UyWx$F)4r59l~3~`>MNFqN>2y>DWAVq5}&yy
zx$I=>w!RmB*4;HbF(Fd+@}0<(*p0DQ4EM&qsDC3Vw~a~H%-2kGF-Vzds5(oOc>9i_
zclZZ!pSBEhjoH^)hfIh&F+TrM>8k;aIa6|~wY(7Uqd}{S_vjc``Dl1}kHOWGdhS0N
zJ;d}`hNE0ZcBA=_cArdzE7#YE<=3fJEXT3zkAnxJh+GYt4wjaz%rFDXeUnc%rH&^b
zK~kqGN-{E(>z3fpA&&X9!feg=pEU#5yRCT@_)BzcO~2h^6_+L2#3UIeDTlQ^koy-u
zI6%UyN*lFnzltwkuD|&-jScaBn(zI#A
zn*Doa?o)c#P5`i%qSIBvOx%{VCTwEE}{_zDm5K#SOYmwjC#DiuM*IjTis?-9p8Bx1?&3
zWw!RmMy-cE#+EIVbN8!q_N!{2qFa4KvA#C(oL$#!a8$@Yz;J8=5^vHX^rqC@qW`w6
z_IzX69*ZYK%05*(s^2e5v(dYb`Dm$izq*{`sZYm8B#!gs1m=|4Hba=zQ}jALPD+!-CGA
ze|3$UYFWK4a(nvR?VYQ$+UI}&%0CJJd+o7JIyLWV#1d&i=L4;Ge2^}UHpwLK@N#i$sbTx=;;q+;|J7^XTsR}SY**?j-R?h+uRJ6P`Z{0U
zwcROW_{3BVjWVTaabFb=4A2_BFg2lfrsnr`O)3kl;vEwg=w#FwmR+z{TChH6M8Ex>
z`F-kg8|Ucc=QyS&e7lkT{cZChuZf-0hj&b^RN7$@apF$6v2|fr{Fly?#(@ekg=GUB
zcWJ%8l%r!vGpJ+|!CYO7YgdQ8q)?eU7Bp8;&(i%GQ+kCg9
zrdGd#c6jTp^Lnc7v_Dv9>4xsk`@Q76KDxc=ik1HM(=xB~V=vX$mjmj4X3YNRoIdK3
z+M1&(zwh)g*;`nqIOy}79-kiM#&%zy^S)z&UDv3+_L}#L?-r*yb-r!(D7ct-bfmcN
zk@Pbi%QbksWpmS8SG<2x~)`7Zh}YGN;Gc>7t^SG&O*@Xi3_N
zCs~%ND{AN6V3q}}0Rn`%-U-`^3Iy_yO+T||W8+Zhla_K~5H1m5)urK$^7XYroM`3Ne
zZ?A|^yUH9ovZ2}-Xc{rPXYT1uu`|wN80;9kXpuu={zCh_vHR}QrqTJ)VwB{Xfq2uT
zV3-7hTN>_InC5IeZpsvR&_cHyZzEOl`s7up4_Ke0H2Z^CWjF?ud_0Dw4;)A<^W*+Q
z>fxhXp#HMrj%A9@Inj58pBkI`P?~qe1dNNm1D`W$y`k*AMa|||WiePWb?QSPsIn8e
zy>zbC)-J``gEo?&%r8Np)ul3JSXH~Ze|ay!O`ZzR*W7(G)t>2dnt(|_>nx<>iwY@;KVRtcMK@eYX?xvn(8t(z{o+V9z^WKLR7Ws|{LTk1c
zWsvddBE8#plsX2fRxV5O01pNJK~GT{W4nJO@ASX~{wU7E*%+2@60HZol-ZY&$L};P
zRuo4gy0r9)zE?QDG0dG!%A6INaHv&O`uFcIf$9Ns@*c+y->Rxvl@gW04FX;9@A(yt
zRaDrpSiy*fhQ};isG*{Qt1lcISm&3|pAQ7sh?V0XP(E=mEc^At(}a!00I)ELwn3~Q5gZBRHhNl6
zy(uay(_QS`u>*Rh;xCzpQz_7S=Gxj>xUIxC%4uW>>Kq>K`UJeVzzd5|86`7T|5tFS
z+yrf&k;(A*GY1Zg@ck;Th?+BJ2txKFE9J9%goRaO#-v2>%|E<&!KfTH*Xu2Fq>0(?#!8-YmqhQG-g2dmL>ce77r+MKH=A@ILmn7
zvC@S;W*}+vHC(CwE5cvF_{GyxPiXGMFb?pVcLrAq;R-mQ*!;!Y3~(_7_XxGy0JMyY
z1}y)~>RJ5J7{+JTi#8cn0)|iXfFEj36!Cbgx|xNdUHqQ}tfz&&y?}GFxC!Mz)bXxw
z&o>H#%XhoU&Dy?e7gLg0#8mldixkj5BQOE}TEY$~pcez2
zG?QQC`6otG9706KUF#lgTrN^pR_0i4Pl1`S?%n%Wag)V&2P^&q>&@Sh1SWPoHB)Gz
zp&;6TZQQDIWKxzrF$_bUI<>9fA|D8)5YKkg=FMG7QS#!(XV0*069m;ozHVq3`571w
zuFJ{-gn}?>lYZxDma%6xie{nXx7as%~f5gy!4-5JDvxvmW5Gd2YCZPQ{ej?8
z9UU@#(X;{)4IDTS9}Y)HDQ?T}-_L}TDJ*Pw|6bS}aEynA)GRGM9Rav;Mipz?m#{A0
zrij&bOl_uK)XeuBh=Xec}yD(
zDkdgi_i(Od3np>v>C>6pBXBpEGUXLhdJkfRWiaQL+oe{3RKyH+B~;ejmH!*zM+hN2
zNr2(p+2J^LY`6H{4i052KFlQgLihgtH}-P$$J3n!0&51h+wU3gj44U+&udMKvHt_I
z2CP=sdQ@MJFTz-t6&W$~Ovrn9KEyci%iIfIKT(Y(
zOQ{-mDRpBJfpa4umqgz9H#x1A8K6#r%xnZ@}v_
zfB&|8xzbd5xo6D&k;+g_^AjlnkUfAP^W#7ObL5ykS$jExU5*Od<==2f_TX6t`v|*-
zVvd8lP$z3C`dZA%Q1W6ZiX;RvN;PEn)vK?F?nsKt-#1SysGsNS?LCVx2IV07ogErl
zy(!k95m69i;!cU(z_MkyoeX&LIsB_-!u-_$n~-qmsBn;}w7?oS$;aY6VtTS|M>0cg_T4sZ1m}Ad2y>z$L_k7k4DF@37g51kgE};
z_`?mHS9=()tvL#4mhVqo1Jz|0)N>d)+OojkYte!xFJFmh0DVu2yjyBT`5Oro?ZT%Q
zm`V0Mbf2I7mbk*h|JXFP`7(MnxD`&kdY=V^p8W@YBC7><1nwp{uUmHmJq?$WxgsV!
zj~+R~sLovto(a!d#PZ8jAaDZqNZ
zd-pE{IvIkkqMR?x0>Z!cIg+jHx1+SQP2g*uJ!3r_N`DO37zSdy{S^d_tYaX^AyvWs
z69X(^S`!nuKmTM0&1C7c)J;G=lysEY$l@4QPvE+eq)>}hHB#qV`)w}?uSzGN#qAh#
zfTie-$hBZpNA6=A;Gn*Jh49V1YE>gdFQbTc62YMb%?FBEHt0Tv=U=`0>*vobvIzZV
zChw}BQ6+wxQh&rWzjcXA{5spK2-X1K6a_@Zf+jzKJ^lKL*C
z#h_3j|IN46mL?*vY~C=>56u>UYW=
zSue=ePoB+Gi>R>G)7;}MXaP>LMvwoB&r2l@IgC0I(K(HQAU@*$@R~_e=EpW8)7`bJ
zhsT6xi_h}L&{JV`)I!ll&m*t(=-{I6mL5bWC4v||)
zmDfC|rQ@UP{PH?FF%%=1=~pN!ZNX&qB_$Jl3G~YK%E|}=n2U)$1{&Y~!A5aPNQm0W
zS(fdvxBm@#F5oAk>~C~Mvc-9z=dJw78JT$|wvoH*k=V(SM0VQYe2VXmdOsPlJn|D?
zs3X%va4r`g+B+0o(9N))s95+56kVH`BR_Sja_#ieh_t-C2YfTZ_)p+Wbo)&Bz1H09
zBdP3HN5^~6U>uidXBCQA7RJIp{)1rKZA>3Ps!`nGERAR1ru=+%3?Q}_dK_eQ?sNpt
zstjjsCyWdbTHe)(M$g?5s3x)`WrSoQKMnjDBhnXyH`
z61Z`2!>IL0l;#&sr*2(;fOHLmmLOGsPT@;K16Hm}}
zW{+GAgU1K{G56)o9SFwwR^5-?mzhAficD^JS22qV*fE2>FS*wHwbLn~l&1t>1;)ob
zeAVg1sQ2!AhlXPEUMQ|#rs|((~!Z{1;)~t;ik#sg8f@>VQ
z$A2T)56bL^BeEal^DsU<8nyup9l8?lbsjJx#fzNO!U{H@6SQL+Xdda)dCYQ8b^R$1
z$leg@8Rsj=i(&7&vFMGh3Fv674$m~Sx8A&C`}Q-`HSHY;)aaGNYTvzny<^K3MsQR-
z7>U7wl73)DO?1^Kei;+4hJC*cS@cN#;+e^Ul;#5wq{=UkBV01L8hnNHY}crqPgw^^1rh%3VXo1S8`5rU79)Riyoz2CwETU2St?Y$P03-=
z!(&Z^5GW_k0IM@mo?4iFU-t`@A*zwch=|J2V*GM=1yIp$%^BDw(8^^NDzV_F-&Upo
z|3*Z{t3iM}?nfe0o59_^Uv?!{b~>|)f*SiwhPKnz@<8+n>o#sI7&QN%@xSRMmPtIm
zH;$JYBGusMF=qQ|NL1!gPHP`7e=1{y_-VsFdO*LYASqe=F_?60(WKCOpL5k*yN)l>Cz8@P!}_!F^m
zA51p&no>lUpn!w|O204D^#j1Ta19=FfI;=JyC)egO-Q(umUdawvK{puwwaF62N*U$
zPfgl4^4hqvthW|)U>m+PD04StWtGz06tJfbE}CGw>N`3!t}O4zqyG_V;!hL|CC{H9
z;yG3Y-;KuM#<8WEC8ew5;zaeYaU!SS5Pr~X!P%8wrsW0=2yY7nt}cUtAC7i#r2TGB
zXKj&$zgtqR@aXuQ@iS+#Gwgnnd}(*K199sVw_pR^6TR!|Gp~^DjV#2h;
z8POiTjP&Y_W4HEtQg3`>KWVe9ED?QPi2aa0eUudh4{jDK9M)&duNP;6&297#j@L&-
zX_{z^!mDWCKKX??al3yKjgi_QQb?Ms+Wg67(Lu9Y;I%
zXU4R#7FsrXs>gPvK}w_UcZQf8ym;G;3l)7M>sr!k;
z%b4j}Om9Q92eL!6EV~BnzFf5m1iJ9iX~T>1GvO&
zH)DMjzXz=wBVQEF40#ho4j*o&#^XRC!9!A$8MKQN29Qc@pnj1g5b-w{4SdN@kOpx(
zae8Hi792QPE|wca?%;7ti-CE-es~M~_sIW5ZGQ59;hv$~&cd+r&ma)yG4IM-S^)wm
zU&?PbUpGCa>!N8f^9}1=a2aHB>&XfET>?%(e`4_@?Qk>h+azWxBYPyi^XD^yrp-;*
zy?ebRb@%Qk>x~9NDuE4y5)cz|duwKo+(jm4U@&_m05AT8(5ip>|eX>yjl
z+`_prG5@pha?AS*g8`lpju-OJ<)mZwik~k&%X6WptG$1p2T5VXtoD79le07S0m$ox
zsJ{7HJT(Kix^S7qZgP%7#(SKgz^zz@jnafLlu~c4W6eQ9-nwP$)^{ELh?Rhw%UNx{
z66rpWGG@l8co`9(i>SF8ht)pnV-t;6eEqQ3>fXZ^JaQhq>lYOybut=#UZw!Z_3QV>
z;n+Z*A=~)OH0<d?~NvJlZ1&JsB&
zvJtnJ^2nxH@(a@Tzol=2TvZqu;Lx(MmAM}3;MtCb=wRypZu93Z(+zoKPR6wHe3+lY?7i74oy>)o$RHxb=G66w3GV4(THRJ_
zZNI&DhMH?=@MqN>e(A+IbFZK8MzqIFLQJ;rD#m~84T@LU%B6y
zEalzc9_zRF7^P)A++@(aAd(PE={zKpCrm(*rFdV)Qfr!z
zPc@g%(z1vRQQeA&THuHD)+z~+bd03k9pD^${LNJ}T0Ns#?0{`KiYu|@3G}3e4q}Bg
z=BoCLIly3%<$hqPys%l8OSyBSMrDv5m=}8Vxx_b1*=yqOL5^F>y_#tQ
z;PSqI`xZ9c$woq-Hh%2bs+oh9xR`(n<6Vh(OmHYS9n_dEGkQJT0XQn7t<=bL=qgj*
zw|X|yX;GW=t>mXz%&m$#J|iRqYJq800{8CyR5FHYsDKXvKBrdT(eb@#XfF<52$i17%kcGRMJ%xZ{
z_6bUZoQk)M;PG|+2w|B7jvO-gS|pT6azS6U*OWcja@9m0;}kK-M7_W-A_2%8lat^a
zeT4x8iDZ{i$SjFS!;Q>K=|xyh;Ow=0?KyIN4c_F<%>);Ua>00?M~a^TbMOQkX$D0B
zT`e()Bkg@fNh>PV(*hkG%hm1K@~fqE?Bg>dr{F2Hj18`Lfps`%xat#7d9ZA2r3JUj
zZN@~O5?o)O)LGWn<6^7cx|!uWw-GXPeS1lDD#W6eV-P(QiVREl0OUPT;kU1D#qJLZ
z$EgS8BTQo+f|by&v3f=M`X&YgMESBU$d{Q1y=!XgtBz^|xi08Cq0hwVhx|8^@o%Vg
z$>lY!4R?74Ti9my@?}aFi~InGMrI88_JAQ}06qfCI(@rJ<7Lw56A&rjumo6rJ-za&
z-IQJszo}VZDG0PgE}kjv{m<|a7ngSyZH@2V&42*FcJ;)fik!7sSyZ`rjHFyr)!ztJ
zK%WZc4Fmw(dAA=*dvuwSt=uro|Cg~AiTjS&mkC~P+47XK1<^VrN6r+AV>%<%aB$L~
z;ql+3_4Rkc99iH1Sau7Z>J@ZtZlbS7{$=WQFW({@_aQ6ZI7k`O|YgmxiR
zwyd2LLP$ck){}}-4aO4Mm>8kPPPEFBr7&czh(gGgkb1xWdEV#qex7IEk<&Tn|NGs`
zbzS#$|DXvD_Mozf9;G2Wl9SWGr5;wOV9h~-5yu!(M19%b5
z8MEpy3tQOy@!{8`>IO#2aGxb8#w^BTmH$XbnjP-$`4A?NnJohJIl3~^zgp}+Sj4l;
z0DOxHR34@w8)*rit<8Q!Y?35^3lo7V2?f-OH4z=7;96;D&envd0UrRf^PQ585XKts
zxQ5D*Y?``r|K`oYb~$WZF-PWmslB_?DN~pT<;x(~V@fORz@)P!IXmu`P2a=qC@d^Q
zNbTeljVu(OaKIWuqDI&Ld>{dbM@%c-%Q=Mbf|&XFFw4(5iLi;yC;W6Py(E{+u&(zoh@d8p!fqaf(`O}}~8
ztU*N5_P;3_{2jrm!Hak$Zuaep3iQ^yt^7*(IhA2|UN5#9llHq=C-|0KXrm;i7|JS!
zDv|yjUKw=V25$={Tj)DOtZZry3j7HEq5{PhNraOWdh9M#x1=~I)IUb(61ZoS*m6a>=If7ZP
zTUluZtPJ`<39hTFi}08^yL%5Gj?J#(!;l`--~QqFJCPMJXafj`t5%&%PM(a|Dn9-i
zBo9;2a(8GFdyZPq_%$&EW~dawHhE2w80ldSR*XYkZwzi8Jm8o{SAl{t)KkxsU@;Ce
zLllFxavDK@VSw@Cg_D^!I|uo*;#M{oJY(e{6SJUz`QPD$tK{{r)HZ+ruF^%pb~wg*
zMgK-26A}DILPAo;KH$xqSy6K!C8c?|%7(bVnB@wG13}!25qBDt0$ug>R^rugr
zRP>tQ<%))b1BW>UkvuxmO`K}by*mpjgmq^*IVz$ZZJ)!`622+*fUo|%ci7UR`r_-?
z7X$o6>hbQ=ckRkyQi=uSwTB5vY%vbF+0@ur{rvfJ))O&;N6yAiknkavp2PxX;I5Gi
zZtS3hCTUWwaLKPeeYz10w(C;(kn~3BV=RX)6vdSby=^46`hPX(r;MK_G(-th4_SHxHnrr^kp9UNd*-lHZk
zeF*83K9ka~99*IWxRqZ3ckFphO+>@(9)J_#>HQkFyi)nIpT!>&2`cnAP;T;#fE0+eqI|cI
zf{EUxB_*7LVwdjcByv9ycal}*#v1Rgq8H#o>2<#qUp8z1-d9?O(e85g&+wi>b^|v-z(rM=@9{tq
z)zwH>@xV!+Tv%P(_iPgc+Gb7D_5apHV=lzB-VMNR9#(KFO{aLv#>xqfm+0CvypPD8
zPYI^?hK_$iAliN%VKC@vW@cvas^eK%e^9SW(TLU+ORX3%VEmi5iaBu4rDQk2EV%_)
z-BHsYKYH{RI3L-)Txvx;VY;`|<3RlO?%&5DF@z3)ju6TKm5cEpbCVcV#iol({{0(w
z42qePkAhB3W+9{Ck-uJ`x^)Yp+|9t)3z?e%Oa}`OHjWw>knbaDl0QBedr&$bhEXV)
z>aw&KruYUpi!U>)$rHtu*4I}{u5)4w@(SK2TLFb&@$4CGS`Xea8<}cqw5%-5%v>BD
zqpk-qJ@lS!Ql__gHZOy1gQI*dtsf?f{QT`(k6}x8Ffdrwrl=r4WKZJqu~=|`tDJl!
z`FNXs6YT2cNb#&FUirLJ`vLSZw7z%uZcCp$Vf7H_42JH?e8xu{RD27<9J8ySTw@e1
z^kzHx<2uZOZoHmyvGhplIc6TU5Z#tCYZd?;`L^B(#;snTOS|j3WK0NW^p|;=2Pb?x
z8moHw6a@Cg7A9@)UW{-FUb|C)h+r-L@yVsXwfqAb2nZr(_Zp|^fwN~D?5Y?Q+UUv9
zJk17KkF_fKD_^=bM|}D8iKi|~Uu0WNjO3R0A3l5r6*y^;XD_*_kbJe*+8Qnee=k}q
zkM`_L3%40{nCD*veaHL5@%H*Yf`OpIj`orNGkbiiJ|tSmubn&3q%IZ=T|Y3JA;cn7
zVF(N1+QGSiBj_X)MRhz{`0;H?Nxxs+0~!7YCv@hiNlTgX6lSW__r~}QV<;13gxi7xhnfddTPL*K~$6>yH
zy)QmQb9z$T6QOt`evReZda&i<)r%K`&jCsg*xA(5df
zhaNz$HmCOOTeY~ioC-=QxHtxdKT^U`ZByZ#%+DVI$O^)+Y11VJBjZYWG|?#%=ksdI
zmo4l4-|AE%KTC40;jqwsFCEQK~RSkGsq5y~BW(
z8mS!Yo942Qw7nOy46mk?g{ATI^L%-EhDuYXOz~f~tcR|y
zHP4CKm^z?~EFm^lY0hCIrhdV8b9+)e>eAg2hPrfdHQ!F3AlV+{A$K2tRra!|ENVx6
zAYEbGH!7)-t#eEOGKE)HA|({?jm;T|L@bs`asx}vqj)BDEVhPteMYT<
zG3hG06+|1LJRs&l)p>q2lNP2$L}1-6T`0wsC;BRWvZc%b1yOY$EjRoQQjp^&?F@v9J0FzXh$7X8i5
z&k$aasX-4Q5CGR5F!LY99j%3Onf&oZh$|W_Dknu<$|=9S%=!W3H5+deJ1N-6&3?+s
zq%^v@Z-kC<=z;|(S(mL@V{`Kj*L$VyK0Z^Dn}V}~W}Hs>L>6ljSm@FmauiO5ag$M52{5i?j$8~6645XJ!k#H^
zj`y<1rv!t$6Y&ZvpYC9;MLU?8+!k`6(7Q;rwo&EqY5+jKQF(cJS~a~<>rdf^7Q^y|
zDdWa1CYVUpEFl)~1F(PSJEf7Z@{AnvJaQ(cne(1;`Z4~BN`*TYKxcw{$Pp-JJ+NUAuIdTNu8HJvzEfPZ-O6CY8`?&_VXq9S@SKH>s!|A1q*t{B!5@_WepV
z%EEybD=L;skGjr}QFuvDrvCXSHLzRXPAps@moQwrf6`d_2fVvqbeB|QBnh2uqX=e@
zgTxBuLz5`MVf{-2h%fwWpNlor=gin2+>h*p`h@(3LxV6-fBO{=0~)FL?)6)@*jeH?
z;Gs%yOxMe|1MQ}<2rVW?!K}A-moD%$f(r-7jfR5wb2Z6;j`3S_oijCZ)9bITBOMnv
z${DgphQ;9hcJ{;HX8H2VEL+47XTa<;);2an^fZVS{5q}=SRNbtSb{MF=E`*favKE;L-9)Mz(E0{#2`S}`nRiM?mq$Syl&?k8(}z!nxE@~sh)>vk{gu<
zIu#IqFjGo8rChy0)PKi$OR%JBMJUR&x9J5CAKtmd;z4{
zqCIT;(@MTd%V`8qaXp})g~iSNBMAT}(HGplc{9LDkCF)P47DX8U?g6;Rt8YM#VR3t
zZY?Zz)L(75zUtAV5dd+NtI!KhgAPyarx+|iB@-&peY|RDKq!a6qBJ`CviFmi&@Rl4iE%Wx7(T@snMJ7N}!(vtx?V#<@
zi7G69uIevGw3%Sk|2)5%)&JaQ?Rh<_x5MRVzVKpL!Jb4&x5xlP&|-cE()Y|Zwq8<{
zui(K5=*@E*zHyl6@Lw&R{(dRHQCb(%<1SMNSc<5{ekQ*_dfLuymdqLCzX3qQEm6-e
zAV8Oy{e}h#7{2cTvrYw~gUEXa*YOm$vq^<~P4Gp`l*^vfra#N7-Q;s1qV1E4LlJ)g
zg~PMUoA}Vyc6N`iUr(rbMcth_RbDzparqBnXUoXJp_60F{YWIB(v#iYLG5`x%6+1S
zAUKk$NJ5KHXysOdKqx2uDssj%gv#vENtOWf3%DjG>&{E3Ms)mMOF0E_5x+t8lU1FC
zre{opPLmE|W?~FYU8D#Xeh4CW%XHfMd}6bg%;3B`c(Gl6#2H2JVn7Q4-O
z-z^uM&9wXKZaHycP-_6~8suS&zUJYJ7Y*M1yID`+kNR8tM?M6+<1U|BT_2?M=NI}r
z!EbDU)+1#WouGsS*=PE0#R{!i3E)|fGBDnPDy^j@`Us%Wb$GHzK`r#8e&3jkm^E`|
z@idU!nG7H}96_|fr3RWOf^gEnOlD4+g#Kd_dNn{s;(l!()gP8Asr?=leU4M?qhj)B
zk0P%(HRG7Vfd3%ZDJVDunY0-yF&+&_z_$W-Yy5Q5^5q}klpNeJ`^HCPOna+>^)&_-
z_`Mu7VH!BtmMjBaLrG7QKdg3CiWyuf86#!K4h?m6gy1HR2S(vY%>@&Qgzs7P8eMRi_2-Ih-*|gaZ88m#SO1feX`V2#+U`{s4oKT
z8`<)XnI=yMZ@ok-VUVldbL3)>zX&PIX?V^t54qR~xYQI3^y8Wf+I%+BRu?aGcHG&C_Ph5an8@*ifXjJ!M{stp2^?vtW=H2uLb=>|s+3
z`aF$8kBEP12qg}+B2Nrus&tFO5nU2AC1t~l7bV%UKrBh0X}Bm0*aK`lS$w3GmEcVY
zi5!z8QS7}fIax3xsJCO+EunVju3aE8@`lQGV*qIg3gmC#*_fl$D^Hgj
z&M+qk7(b
zrLmo{_Ml{a-5X4mO7t3cWYE*Ckn0+Q_C0y9Vi4ci&
zt_=he08dT{ZpySP2M#>n(RT{P9xb_8F&bM7SP~jpxpm|hYhvGP9V2h859czIOB{tUb~veYc3(WNbk7B&^z|jjbvuoeZj0*-Q6?-sVqnNoXD1D@)_}SH{ldbUM2tKaKhzS}o^`;Hup9L30GIK=5zHO(M&3
z3U#`76DO@mFDp6cQt=46a6h%${*2?$qI&Ks`5U5#2_e=zAr5A6jI#4XQeBgdjt(D9I*?u=J<=%9Ty{^=^rdRWLda
zeL=s5hNoHQ=>Ug7!Eh%v^%J6cTltdl2R$u<9SHW=#qn_5#VN8e1s#l1k?JHi$`#L3
zj_ceGiN8eo&9RESRB_|RqUflFe_FTiQoz6&00(-O(hhL8}SBLLA~n=n4V-{r1Ff-HNES5Es{@
zf4o#RirRT9BWuZu75Lok+`gTRk4=9+y}MAKF%_zLYIEFREfk(C)r?a(>yfb%5#Tva
z1;#N9?f^i+cK!9&Yry6rOUzf`B=47(zfK<0F4zC3Abpj|nl*Yo^?C5{VQB<67Q<$0
z#3E5J=AEPoefs4D7br1nqvW`A<171CCWs5=-Qa^I^lDzNjUy(fEYedu93PjR*Blea
zQf4&1X}8b@&^689SY+;GzTMATyc(%(1s%k9x6)W6v#U1
z4pY`c-(~1MLOOAF=3p$_Ns(N3x-h2K=)S5qjzHtWO7(?Z55sF5f0Y^?8L`$)7-;du
zLWs7-UZS6Tb?M^8$7Y>=%U%*fF_n>z_*TFWdUs(TsD*6vQTOH{m&hZ~gb;Ib!d^{3
zldpslIwlNon}5Yk*U{7e<2M_}AgQDEVcjmB4-LZn&JD_9bpmLCb}Ow{Py-v+mBGDFHlrm=0Regl2SUf_VhK=!ae&kZm)R~~vCT;k-(?UQlw#~HeVafHkzD&(GO?q5ApQ8*+!cZEGc%aYQ-NREN-_79-^O
z$?Z0~)BmR$!)v(#Hj#pW?hjHVqmoL>17CzQ@swOZIs^ymLY{-wll<`$%XTu&hDAM{
zdEM8Fp(z$i9-+@>Vy;yYgUp1;W5A!(<9++GG~P*nF#kZ&D)$o@3)
zX}%YPvTfPYaFT6l`8`?^ZSDD-VeAFByEEBzWq#&+&kQL-FepOa0gl58|#pB0szXucEMkyUv?$30SUy&z>*}amhmQS!zm(T-6`^*U1Zl==fmd
zH~H22Pe1yd_~g1uJm}P!GskD1R8DhI46<_@AfLF}&o=CNnQ2&_e2k)oPs!s3|C)vs
zO7=P4M>wvOZoxI`&YQ2kG+Ey1^X(VkL!Gjs>lZbBTIg6%brv*S5|iXxkUv?hD7v|n
zr%UKYCxPu1E9$@Hjc8ALF(748dsV{-SM`xC8<&Zy@gvag-W@!~9#8XL#2YTx3V=dkM<%_zS8r%>I
z*FV>pbLVv*fk7i(}fFnXz$ZlpTnKkHTzrtr$N9Spr6?H_(N7X1wT9g_mIox|I{<(~ae~edi1*A40*6e*i9j);lc1$b)rWdzxJMovlHl_A@}`knuZXS;H$DW{nbM{b
zq5XGBS()O%nr-p%Ojuc$Eg=9?Z=nj4$;|-ynKSS^Ftd*s
zmFqLs&#+a6)$+Vya2!TzJ~;=5Ya1%6Us!!(M_;C*U|nQ7;x%J1JAVR^f4F;w4tgny
zVmS+8HT!=uVqM+4)m5%b4fkFT{;K4vKYd(Gk8gZX8X}a7fYwSL16jR#=@P&M+kdVA
z73Qm#@kfHt`Q^*vdD|*pm3*z;#W2eeYDlJ{aoJ3bNG@L3*(Kauv^~OfoM=WAA(j#M
za{=_s@Dx+~b-CZgqFz6!6ljnKT{}BgacaijoBL#2Cb=;m7IT@NN1>CNyLN_;a!l03
z&4~^N$R_rt7IGK!gCn~9X@>~5Uha}g)
z<_=!CU}t3|iS5ds8pd30Vb4fBicu0B9ib^^D*5m#0h2&JP*LGEZ){YM!@r)E_r+`a
z2HB)BbQ{sm0?4jk_Zm0TsxyaxL&}vpmv;oddi0Lx&V_N8!OIygi1pd2Di-Wzjh!merU<$^N-dN^84$Du!AOEMDM`d_swk_2DWmPpvs
zc6~+Pt0znOK+71DWry|U%SNG!h-}|{{1|slCrfp}Ex9?2C;1H`O=Wpn)D|gpn(FDv
zoQ+HSmyDG0lP4EK1)>>%CSq$KyRl3~`!bI&%=f95CYoMQ?-#B2hU1g*+=wkP+pm*s
zrB0%ToB7{9G%C8%w{lmazR8E8za6;8WBGsFf6RtSG8w%vp4f_Rov11-9x1bRoH=B88cc~oUK8w8=_{L4C0H+D
zRlQLD=N!eOZRt4+7hJADK#;
zv^9=$Hg3I}JqbTgp<6+zO_wc9%QC4>Yqmw)yscl!=FzKmlG_hbYI1lnLt(FgL}H`Z`&GA(7{HUR^pv0lR;=hLkwpHZEDqQl
zH?4IB%Uqbegl4E#idytfscdyf$X>*YbYGsvT4f&L;ew$y7+6wlCH-HpuRg`evSpnr
zKG%pc8O(zJ)5ewTqDfaLs5mq|V)e|)2*Qc9lq%GmR+GCZXHxP!JT`k*dV2bwf7)~v
zcRI2*_PWC7Go7@wh7TV612g`*1vH72g;5f6#}l^j6-;X9=fbpumDb!TaSF6}%L`KA
zxTm9Cy#RKJVP$*;YhmaxT@QKBV3Y5f66ktT=&ZqA#pvg5Dg2uehfWa$r)Y}S(Rgbs
zD+WXET)!UGj*a>yXhv2VQrg5@T?3S5N`-uVYrU=im+DQ;lHfmf?Ha!N?Nr)>%G&1K
zqa_1p5v};R_!#e{l-bN()d#IaXn-PFaCl$@lCO$R!NDAzfN+&Jwt9j|^ysl{$&xL`
zr*P{nYh>0RPYXfOA6(~ze%I24G90ZymR3<%XItgFex(a76#8Gnee2-xwj@xEkHI^lg@E=@iSde{iHQ4KQk`o5JHuyzKBUxwNimkh=Bcz=6eH
zYU8x4wV5ye{`IRvKXsNmfy@bw#nBZ-576+@oZ>2s`bRE#h^!pJ39nJ4r}fBU(q3!T
z55tT4X(bl1XooRdq+PJetb9qCID-pB$h0|+ra=eMi%29&S+6l9COcl8DP5*f@8ACl
z*_d$vMoxKyo>+qfeyuJzPxUB5MEW2=It{n+?h0z%u5peV&nWg7Z6Y#ydzTTk__t-R
zLMT9ljbzHod|M`IVhvLYhmRRRu2u`N(g(OB@5WkRxuzQTZH%nk31?dk&rCOz-}twUa2>~xgW3)xo5B{BF3-oZz6q@gb{Q#g
zW;dS8o@F^$83)@Xs;9aG;%#K0&s{gbUz9vd{Ft&P4Ta;twqW;mVw7q;W}H;y}V$ftF}#A
zr>HgN9pf5>FDN&s|N7g#R#%!3SSXSFJFf^$;~(c@ncSs|lF1U|a)bSo;tr~|b=h=$
zU-O*(NvTq8lj(l@`8s0vYaXV#KYCPsK!v@!Zi{O48zyCGQDb#qIaqZiSY)ggSPG5|
zZ7eqy5iv_dK!^cx@~#eOO`DYC96u|t7VI3?BViD=2jT~;Wh?PTIPMGvj&$GT@~C}V
zlm;f!B&Gb+KGR1zhFK1FiEtXYMl55fnxtl6@piUkLg=FYDH5BQ1I4e8nWVa7E;(}j
z*G7~d`MXlEnKU;S7_W=UQ4W#K(!ZK++6ae3J8x>RrWywH>J
z2m4t{VB}=LU=N#%R7XYr#UjEL@VXW{>x@HzUe~Vu4oQP#(B$uBVPUO8A9CvkQzMZ1
z^c`2U1Zy*#r>d;b^F$A_wlz(NtPD;^{jC;)6%oL#rH0ecJvTEmO#(Lb7CBAcYrQy8
zvL0%K&CpFfUH5rq*7+P1W;hXSgmy=zik&Va5UFuw<=A4z_Cc_w9P5g#mxiV5Xj<9EBj-lIKI(2)y|c2&NA
z8Gsofh1EMsMnow{JM#Xxc{05u;5<{>gf3Dvd+gMlw}!qearwlT${S31Ecblh&_$ZD
zPEps!*4ERJJ$v#*{lzH6CAiVVyM1MX&PN1sT@v$Nau@&7jpFOuAj8f-~S7O{6=5*
z`VEbf_u4F*HFfI!ox2;4LbH)^Xz)Vu@6)7kO-VgK-xMU$l>dZEVmDb`p!JoomjnV2JhP*|RlVV=g5ML&{hd7V_q)
z$W*ezSSrK};6C>I3>|t8hUfY7aDs}vRYn9~n{joI=~krs@ZaAa0^>tAJN!$(s+hAf
z*!EGuI@VqH{?BEFZ2*RM6OD^eQqP0Bs~;sQTFago5MWHF=Af^HR(kRTwDno=gegpa
z^El{3DWKU@^cpNhu$9O&`{v$;WREuo#R=$hm^5Lg4ZQ1g$eVHnTM93=Skc)|Rv8rI
z9BITBk;jyBi%~M3@o3!6gPoXWgG<>LSURo}$h@$5`s5-lz2^9$%a&ay%$)gy`3Sx+
zl9*-Jr5Mj3xtBie(z9p!jvb-=zo2G}b02Rg7qId6Drq+iK7cggv3se{P(wSmp
zkAlJxOJdQFtkMjGH_e
z^Ot9Y0Xzp_mndh66DAqftq(U%*N$*WH|V5S#%i8rNLqMdS`*pL-n3B{fKP~FwD!a<
z0d}~w9)%vB7UAHNx;OW@n^9ta>7%UTG`g=nHiu`huHiAbtKWb2Y#F`Tn#ATVicw<+
zv1icnyXY*qHADcl!?jzt4tXEpOXt|d++0+$_xk*N@-Cyv6oo}Oo5-pX*CXT>IDYza
z9(=^luSd`|AbsRRI8@mCmB`>FDY~*QUp{a>z|Pi|p)L9Q!SO(vw9tdaFyq}eK
zCNeRVhs*$0|AV{eENV2F;kwm@AUIDnxEb5EKx
zQ)?Y4E~1ipDLw34Ydgvf;xJM)W-FEpNC-$v_=d4}?H*kdr6o!oW+)Lq){{4Yfq=~5
z%0ckPo>`OUVA9_#_?E+DT7<%m_6>+7kc@Wi)=i<`f_xvOfHA-6vWqGzCZIV%G$^^S
z4r+|r#$iNwN?7Fndblk*Z8{iUfXK|bARx&<78X2&>_k!TEIWo4TzuHgSi&79%0Xo)
zq;Tv61&cr+dRnlH0FwZuChSaS?<41m$rnkUic9C|A!@PF1>f_{kCfP7S~t%>-v^Mc
zaYR7!P%A46YWdk?0#*@r!he2!lOFz)T>?Fway#Dnc2L+VC5X1aMt%W;2yG%9S?2fW
zU&u0vj2T#8f{+=B28ENb)Q%Y|cBcThQqd8R7(v7P2)hC-k3OR#rV}J6ktDd3M-lz}
z`4{44YAbxjMlfy;Eyqvb-T`e6ImX|9f+if<?d{hY(|_00#3m}F7gs;DVmKiz5RP(SD$_}_cTx_Q0@`-IFzWl71P8hx
z=LEPXY<*&~6abK1J?P+cvi_NqC;MAkmN_*Nps^>D%e^e`1QUp<n2;PK+NzF(@s6D|Uj+rT6*H=z(3RICrs}6I|tr$EU`f
zQrGV4ggnh?>yBEZVJzoEF2OMO5Cg4Foj@zpHXh7WwDiUn%1!b!Cp{c7Z=~fXG#e~?
z;{x*^Tmz`s2M=C}OoT(kFqp987fK_s^W!F9ChQ>yLf~NQES01WV6lD1Nn7%La-mQ7
zoH>UMjS6cnDlJvW8XN*GrYz2yRX<*mgd1+Xn+4*f$<((1hn0u
z3|J6)NP^hfpbE4fJ$jz3dpv{)qw2^XC!yGXS>B1TjLz6M{KujEe73J+oZuLwjtLWg
z?|(BmNbtI-mg74|kYa>x-~Rpd>%g4Fi`!^DnBc$+ALJRY5#A5Capc`ZvF@3Ut2e1v
z@o*A_7uatK0M+iZAvc(*o+7coLB#;zqQ4RsAcIA-G8ydaX^%sHsa9d&*xRmZ1png3=(Bsp0@+o
z4HQAMQzteYUO{|D*Fu5Cm(Fif9BS3%bqldL+X)8^(n_-yiKbxIH5s!xBt3!#?$m9l
z@E!MHbMP*Pi6_xlTUj;7p{;c}Te2|eY+fGzn6kuT3R5(C>p#6vdSH2#h6Iy9~V_T;dnzUtC6
zE#<`^zVoAorrHU7Nz^_nt6Xw_Zzk>HqKDwwVJj)b-fmKB+W12Z1UZNttgZR7NKepJ
z{5Zvdjh!OuUC;GV&()9LcBoN({ZGo3%iIS%28IsRS*Sj17-5~x3)wp%46&QyxU73`
z_-$NUYyD&Vs@zXv{z=xakbb;;&Yf=r9ou50#gj={U1o953UwQC~zC
zrhEHy=KB>{W4eJo5qE
zL!f4KT`buu))`ke4>#~`17?9wtVC(7oj^kTR)1aR)HH`Kk{_N}tg`Zli(oNx7uS>k
z4IfUMtGN3-gbgg{88x`+yzxNj8=7n7lOH8xc}V}suOPOe3M?{Kvnwkume)Lm`3#3>
zfe70^g2n+WKPMz`oc&5&CVQjVjfAQ4TOBSBOemij7pq4MxMp{2Q?$$AaHCq3cN1m^
zPM?llT1sAELbLKah6l8`=;^O@+e9n?e9At+%xFtD
zA>1B&F4(HXlV24Y%6GXQ-Dg;t$#X&_FNEeB7=VeG&rWzVeEIrSwlR@Q>!e)7bs?pd
zDL(^uVfmH7KumMbkTK>9aUoyX{o=PKVz2RA4cUzt*>f5WZa7QHPzpI23BqU1AVm>-
zs2MCVI;L%{FAN4>e8_&(loSch>0noKuig5;r@RC)A`>oLK1*#EMa<2Wud7p!{Oq
zD8;;R=mtk+<}zWH)E~g%Fs|leSPXu3=%@ttQpAG-#2gM2iyT-uQg)^
zuMzXxsB_E?q0Nr$&YR&jE5{%Q1d~cfwSFmAA=;`I2Ym2%RMh_s`SB##Ba04%2u=Qg
z{6H?OiueP=SxT{Lt1Sm2iO2W>R38nGtVE^3Q?QrV_3!_V?2$bwSTPgDQJ%Z1dM3&|
z;#u0U(0e~khqSPkMrp)hpY>Zrm$%v(Z^i~Rmko2uTet_N&
z8$O&8*x-CXiJ_`kocCYhtRNm>8vnZ^y}P^arv*_ZVb43}yMMhSJyJNy|4wwpJ;TmD
z!&g4}vG(F1>9>#bUvGW0H1thF*J0(p>$*P5-_`fU=|Rh5XdF{qFOwFX!jlwY-ifuH5*j6;dAXL34wga^5eXaw^7nv9C$q#?aOrPYcNcwQIO-
zFuuS{B+?)GEjel~#uBEWpMtF4n>@xudGygb-<`5y=?y=8)5m8W{q)d2;o
zRQIO0Zv`Va7Z(}v=|RAZuWi4Rbw1{IYoDg{@x+M@C3^?iNpE(u`m!K*OY0J>)22lbWYMOIen>@!+9e*wD-T|RjuJGHGP^rT>>-Cq2E4|7^O0_NNX1S|t2P+;Kl%KEA`lMayMbfi)*R8|v+=%S5B3IsM
zNsq~^P0MDt7JXjxORuI(drkZNnXX2KZK9hehpW!{M|SL#XvPZL-&;KI#Xpb9)B5q<
z*vK%p)g?ME%R~LixQ_V`)=t&*84h5Dq_@y2$!-C%5J0hf6z$
z$^Ge>*xP8KbgN@b?8dbGx)WaQtAcmC#7h0N+Ax?jIWc<4k)nIUe(o&%*S_HQ3FpA7
zu-bH4rRYsfSEsE$M_Sh!26?5={wv<3);Gw@#7EJ#MYG{*il{iJ==YACyMxEy?w0+a
zP@MT=T%b{HeuiO?eaztSYrR&b8}xDZN%GI#U+Bi@>u@1<$hgxG0UW1cIvq&Xz$e{&96
z(zU8|aQXB^*Miv9=K>d&4_VNvu3L4eZKY$FsaAr;5B1&hI&i2`HbQ?BZcmQ
z*+h0rl#XVr;Dx{{Q>#U)Os7~}RkplNnO{HXYJw`UK_
zKfhblS#KTGBirV`l=;*(EV0z?ou^uIFy8lhUZQG%kMYowq8Nvbv3kyro+(W?i%wc8
zPHJy|s#I5_FelvPbE?NBC}@)uASP-6y}KL@-BC
z|0-{kvr=F9zm7Q!4;gR2PH1oYB_HeL<>}e@?p=&Dq@%A}=u7KKFRj-gUtr)d;+f*!
zh6n369@G2puZ3!NJ15?D&o=3v?eCG557%h66^@tcCA=`olwB6KO
zzuU)^KOdK^3UE!G8#}i?%i?ra#gC7f9k-|Rw|=MRd@`v0|F@rJCOX;vu;h{OQJkDs
zI>;7my>fZP#!XAHGmtxF|d8}#wdOFjB!`0XJ~ZZIDh{arSKB1ysPuF
zi|Wc>_cuCSK1iW7+tknR%L`U`WoZ_Pl6CBD^$hkKj&wI2U0)Z}B;O^hZF2kgmVSO*
zvZ9`+=SXcAv=3Ql`iI{=AJt!rr)nSjr^f_U|GMoRhrgW9-4eEFYO%!2vLk=G%wk)<
zhq%o=FRIPJrm#6;`JH&9-}&+1yPb0KxfB1_9c>$yxA+WhKkHtmY4UhzOHb#J
zj{aV|uD+>gtZ_V;|G$s_@^IDbLgi0Q;_Pk1Y<4#vz7oFtKx}@6Uf``TQNtFuvnfjz
zldsaBCSNk`to`kdZ{LozvSE7cpJV0df_BzaH6^SnpVv;7_ODxY|B8#OdgJsx{v+OG
zDD>$Xu`SibAv)ph-aGfELKMCEykS+vCT3fhKnBz>RU7G}x)%&Ks{npt@bUiDAAG
zuops83{X-YuE86;@Zuwy<4z&aY7dji|r1^
z=>Gh=8RJsSTm!95>k~|+rcBv>W&SW58%$d?E^cKKj767lg1vfuVTO>ea%wcv+&eZ9
z7^8;_5xu~y0bLxUC1@L%>4!I6w=bLWiSLs-ZT8HW*I(KlI(#^A!2*1r@O?C~93U$a
z)=uoVUHJlb5!5*B#=)vU8V5>OC?-lG|2dD)!Xb=|jdcZS0aVA;DpbvF7h
z7EoK>+&B%VqCXkqA;`BrT<^DTn4=6=4|d6}#OLOOuB6MB-l^``3sDK=2|$*>n++CPZs4=FxeFVM)Bai>5FooAAbKmQ@b&?7uvjy9lyVWq_SDHb2(6pWt+R|Z?ev!<;w@M
zlL|wFxpRNdl;FO8m$UC~2Ba6!=|54={CqY??W%Ys??tZB=*!SPLk{cKozH8Fw`XJ>
zcR>enrwH%^J~WCVmqvezlI
z*=*kC`xiL`hgWf^cxZGkwT&iw*D=b?$1knprSdkpnx?|2!%bvvY`S7=xQ?C4Rij2k
z`iY{N86*JJ@LDRnEwC>`hyH>QML1@sVp8wFU;$RAZ0`OEd)zbwkuYtINHnGn>D)(AJM2F#3#NmUi=*_Hn`WYdS!
zF5|yAZXBdaI38Er6Q+lOUow2)BMdd;Jj|WD&AqgKHJ1El+KZ84@W-Iw;7Gg{KL=!!
z;XN0$6m}oklyU$5Po6!ygrzros5;+@`+Bb~LfdBhMfgxOP*6yQhK9`RLAE5BE2rHu
z($}|xX@O=_`Y^n?^5)GT-h-kDWvGj{_aBpTphf$bnH7zYBwt0I7RXE$Uk984RR_Nt
zYOcg5b_o9y!4V}4ohl5mH8kJ84z)ip7ERY$++ci)ZG&rz0NCx_@
zB8^=Y?183dg)3v0gFDXt+EDqU_S!W1MTlHKTJF&xO`7^uW!-q=}IpIRUn8uiVTK>}3hK`31xCUXbM9g0EO^#4unmyLLB=ZEe7mY$n5t?5
z*gW$krJ(>PcdM#c2oC@Y_WbzLhT&VU{1k2aQk}lFvH;sFAPD9e;FtF{I1yuQM~)O=
zbRpODcFw;1y9L=!4Q{YMPVP4RrS8C2!-eM~zKdGfA+lG+}3}Y
zZt>sZPkO5-yjT}Fe?IONbX28-E~mE{T$w#*&PGUlJU-wx?F%{S;4)}Tnol&yr3xed
zjHNfpwXIDU-tr0piG2m(i7Ha!3|NnT5_uF)8uVO>PXFeMoAo|spPMr<*6GKqUA~z8
zlFJFw;*p}JQ2ViB97xy~!}=0{IRGg1226@qKaBhV)zikt2vSP$ZsBhdt>^<4FEp{;
z>EDMBL9qLnn=>T>yw7eQ)F#2i2KBg75*NF?jHfc_NbP`)Rtu&UK0l(8_po
zJ1aoxp(nyWy>JDjnK4Um>;N6NVhDN0P%>7sff7VqFz`a;prO{842cU7kSk}3(@L8C
zfTzT*g{+asQW_5VORwGocP(AE?DxA9h4W3mVm1V=Wc6nP=5M;)x~Yu&Mh%3hu)X!A
zmXY`v5EvQ*Zik$k_S}quf;PM$37B$D>|NtQcQUYv{Cm`h5jgQHt7q2b{iy)4&RP;S3GB?U*?dVFYrE*Trr40W(0r%wlMth8n
zH#7MD#7%#DIo>o6`_BH6!_iEj<=HU>J8hyZDV7k1!Bym}AKSNl3OzzbWdtoUIBZ*N
zC3NhXkPsM7oh4V8gCd`xZ=O9nEjrq1ij5;JP&T;8ptX;aKhQ3^
zwbakAj^qQ^fQvUVtp!{i!Y$zQr2i&3C-1YZ!gQ6TBwnuUHv%JNH2W!hHNoC3`^T?e
zkS0M-&vkw-0CZ>0D9?>&I1oo6UMbiX!yK`ura+{^v=g#neqjK)-0=6g$BwPEeae(S
zj4&V8LU&6{Oerx+kXI}ih?<&F@T*tj+|7`r2{;Xjl;7#I$RJSjZ70307+;Lq*a|x|
zq%9&4Iv)`dDIU@gZw;oPQD3Te9269i_wTp<`j!3MdGYVOxl0x-=%S;Oed2^xf8O?e
zJR=Ctbv@8Zkw(DLQB^Hkglt*}Hl0fq+b|Pm7*Z&vsv_>9#DseVKW2Lm@0`+O5uWF{
zxgKWPhyqFAv!_qTPtRe3T8b%dj=U#7db5T*YePb!OZdzvj~x1pLd3;Oz^safGLA>+
zLpr&SeRW(k1ZKi;uQ4jS&V6vK{mpL_6sB|L;3YN-<9K!p&YUTX#So++P5rNK1HWe~
z&Dj|q9#eE`bLPr3%aJ5YHy2AW=nO!Kz@=flK)7N$VruQgzS$IMosC*-A6NUcya#P)?IC2cIHt#;X?bBE?D0>0`&oJ_x
z4c!q}%61DBAZu=RSYB`9Zj#z;2y0m1Lz|DKfn;e+N)3_ekU@p+4{
z_vCCHJqoDoC|HXTk+?eupb)%Kz3|^qC$a(W@r&Z-@fy(OprTaS$ZNHWHI|rr$k$IS
zQ+Sp)sJTt{txH+E+7Bz{nt8KyF_RZ_G{%kVOT|l>Vc%q;Wk=COg6#Zs{?YRG89x18
zwqvc0h%G6pQ^Z>NF>R)CuT;vEFI~Uawbj}9tZ0LQn&a*39S$0J3Ucs
z>KQ-neixrqhgZ!J6*X(Z-$eV5gMLeLGuUu;YP!mpwHs*l8+b~W*WTXp>Kriug6>!zG_fGM;#OoYarbz*(q51
z^l5ne_HV{PO6yS4NMnMj>{hM97Xw2~%=z`Ft|fq?*2P^64U)w=L?~2k)dN=t2OrMO
z#odCF@;A%@D>(^3$VmBF1TCv&DRfyp5$IM5Q(lW#5A@r7fkhKgGT4ul4wV>NDT*8+
zqqmf_%hp(1XXhIj-jRN4j+`4&*;;ZI;u{L1uC7jWMKwU-o#v>1L$fD&`{mst<2^4}
zr=c9Ks=pMrRH9}tN*4Kl$r@jLQz#H{*Xrv;=
z>Lb=I`;$Yj_I#EBvYM$5M`^h+rR#j4e77hio+bV8%xSslix)4xUaFTIJpUIL;uOV8
zK>-_hb^a9sFWe~Zag>W#<>DThuza*VIi)tLlC#hBi|p0Uv=x`-PbJGOV>uDz9kx)K
zactdTKN0W({RWf6fmcTqSy&v?r*a>LJV*rp+dMgxIUTeTCuW~L{@-Q*
z8!M||E*NA{WSFqwfM`cBf|_NUDC{AXZs%pP&xCQ_I>HRPwL>RPq#rmS)GX(|GdS@R
zAwAW64x+&9DjZvg$#q=$bew1Cm63{hTj(fYs=|Rzzc^NmPuWW?aaWio)U=+@;Q~e%JNqVrm)cZl}gYO$Z^@vqOc4Kfh;I4`I1j3p;_gLq2d}
zJG*ACr*-)g4Vsoa}Ur@tK7Y
zT){VmV>!hIFqw`bGy0Ss%QU|%bL%HSDe$ImSw~A|y5D0rr__mUD$)ZbKNdIbD-{IJ+*59d<7r`5gb{BZ
zU0XEm^I!0+@&~-8PDPp6P?KDHi&~YlOMmtGCHNmbE3j1P3g4n>>ai>eb1n-tJ^P(KAPlya?X?~U*4as5w))4f}qGQpb2JnmR~D5
zUNl|JJ)yx}S$dNQQ&3_7R${*>qc(5L#q_p(5V^JX}5God?HU)CNpJ*P&#p`~2?!C|0#kny(qP_dp?|DE>
zYuR@lx=|+hF9pk5uV9hYF_u5+8ySV8m=^!C24F%D+0?X;c{dOWV@YG^PxO<-
zDnOFO*7{*)oV92rCwF(ToqOW=#%SH3rGPd(!XkMEgR3~Ui*COSOzqOG+oF6QKu`x`
zB;Y85iOS~NALg9DAeek+>}B8Q25rZ6p8wCdo2fRnZ?2r^Y!xlmasrPs9=lk~Y9MAE-&!*P8@BeUJ=XDM)gNx1s
z0)q&>>5D!AYy)fsWh>G)@V;wTud-o^?039QafcVn+h-dI_36WhLc%~!4)*GJk8n%z
zD4$_=cwlr%fCc<41h=rBpVigDPS24@A}sszII)0$UkAY|-xktY}f#g*7cs`u)}omSNxLgUF#0tKUDDzcb_{S#zIMhBFq=zl0SS}PE{cKrS=Kw2=H
z%(vg~N);Tg5=a-K^#YS=sH}X?hYPkKtj+XIT`yQHt?;UKAeasjHWnBg-_jP_scYiqQMI1e;^pBYxKSp`+mJu
zz(rtzVYyN~{!efW!MP{(#EIpYryV?~x91KIpZ!8go9s*@N}x^OmO~>Nx(A^5C9%E#
z{OF;lH;ypaE@0@)hq6_$P
z-Lp$?EI%L4(O{lgX6BaPKi552PsR{9$_ykB5bo&qrS}ijnu|v;ELn(4;i1e_@XrB1
zAymlO#YILIe%{^ka=YEzHCIvT3QDwK&Ne8t?UlwtpL+7-IBV;@Sy_peH+cFG-2k!+
z^7BRAyMrQ71<5vbxu3jb^h$njAC@dvvzJ*KtB~(P{sOKc_-auA;#f-uD|l-X=YX1I
zU#mh#<{S^tHrlFFuKMSeL*4+U+KYc8bPLiXFd@i#(I>4ReS
zd^bmQ!PE{-_2oPtvL_M;@+)`7=r~9VkvNN+AJ!PI&(jy~0
zrJB7G((K+jA4@JHkL-f6!n!Xg{V9_nowM?U6O>2J47k}%pW6S&EsU1p86fhE@ctVr3z;VGKtyGJYWG
zIuSY|!$U7H+k@}F>iQ2C#%Ps+{66?fM-CtU+1`#>B+w}E0}HkVb`^vw)`Q@APHf8$m)v9?&yyx^u-*!gu>%l`0I^$SuGeKJ^f?~a+*
z3-eB@FcRhvS)E_`Z2K|#Ni2~K@`;5DYj5g?*ns&!|Ncji!R~l;(NKgNEP!$tz$`XH
z5i#bcJ&Ip`g8z!Bqar|$NTNBa$7m%%`BavvPL<5
zA>%IED<1B(Ex$Cw4RjljO3|rc13_(Re=cF1)0(Z^QdIDQ1p7LM!howdIph6L#*dK5
zFh}$trmIV1?NKPhjFojoSUG}ib26}W_?;BhjF*p2`xlcksS|dCO%HB;J*c)PM{1z(Jr6r5eq&wR)1
zbQIX2BX3kn)c`)@<9#lLH1iiX3>F%G;BFxrhy#?@C5nx3@aIo!?!XlQ3C_v+*EW7^
z7~3-CepS^fMytU;68f%;e#)qwOc6x>-yub}|3d-IeZg-;y~htB8F%fdLp93jeQfo*
z?hGrm<%AhBMCD3MlrJnoXD26J+fea=k3>4eIAqW*QZKUn3|rOKyMSuKJcr0l0|EC@
z&|Pe>W7td!CS>jkCanY!Q(Xg8Wyj45!+8!-*Q~;=TS(C65U{sr04mtI->HP-hZ7d%
z28+(nMDU;^x&k5HOnAdgdU)D%F9u61ynr_^*yI=ZJBNmbQfW~}Am>p(p05v@W%O%)
z=%GLVP}DzF@d28*_yh0H;xmNyHW*q>{cUcCC0nt)D9l3{B~o_w^fT4
zt^52BleA9w)gX3V!o($`4(`KQN5FN
zr8{Q*@+W7P7z|*&M>c0aBIPBO7YF
zl~x=FCr1YJ^qwtfn&Rc(s2On@W>L0?qTGLpW|&-@Kyjw7HVuO_kQyQu8^LOf;gB$pV?S_tO8-5n?$;quEpi8AU2>>7B0~|W
z%3kg{w-RbH;7HWD0NFsQWY2@Pj2|(|VTwYNeVM_i8J0im$-<#$r!&A%n-Yy7uGiP+
zArW1(qHnznW3I}MBNK;H+*VJ>s5x{<#{D5B2OqW*MlBoo16}o}PfP%%-_6}SQdwak
zRonQkjuB>N$1<0CAai4)bg(>ke%L<~$m
zfB%j~Ru^G&>M_vjw6tm1Y~zb-Y1u}cmu=kGGsUx&x;{rw%s+c}?&$!x;NW~T_9xDY
z{PE0%QNEl5l`ikz(*^YzW{1?q8xDHTY!u@^DESy+@mi?ucr(O|+qVzG-5ThW=<$n3
z?LRlZdq-VgufMgu_{NPFar#^Lj@z=4xKa$E@O2Y_rV-5J_Id=;V`pVOM-9d6k_@2&
z=QqTyqkgDDy#RmwAN(5-fxnzKaAd!J3Z(*1xp5+vWt=w5Z3xV8>6eO&7-mG+F&;3d
zZMuTh0-YJpb?dM97XamGGXPmo7L=5{q?+%JhUsTl!T0B@Ck4*wlG>-V#6;}STbwlC
zIq#+DTf>Bwd!j=Y1GBUB)UO!1Pw9ClbZv|8qP5EtY$C(g%1t?DaP1EVMUzWe?|xqW
zHLfsPF5vd)k6UJiwHLpxIP>nwvas(@j?2jxYX<*pTVlT6`PmN=MYn9;Oph6{g(NH3
z9u#AY`4ybP9@NwTYs-qp;(37U%Ej8=an&?j;c0fCl)lSxkkBaebQhcss2B!iR?AK)
z8l;nx1m$0J7UiWN`od|;K1h4#diu`ig&L456rweiVqiEzM&Je5=SNfqLj59qmt7iv
zr&%UCeWk2&&3vf=p%c1I`oiUb9MynC%y%1}k7%1iN6hp%;z4B8L+_2W+
zEVUbW@W6F!RA-N*4^bDXsNnXAZF(~`Ctw5I;_8b#^~M4lUqZhzG>@;W-m~X}x;l|w
zn0H&5k7#E}6Y(%KvDg6)No$dnQTr5n^?%Esnj=z4Yp@TY5G+DaoqdyXy>t}(5=RAj
zO4f4-GbHAVgW=1mOFxmUHjY={TL~zL;e1J68Vd+axc2^iwz65z+e8!D*%!V3!fiQO
z;zgbD7ur>P_jhVPByu%@OY8NA4>%$kDLP*NVDzV>#srqZFMT$GJJqOq&F2#`8`4=HFHnA0$BsQrLR3!(B~U142961qcV6o)KcP?K(~z?mwD%
zlxK5?5~7i_W8AnbI`U>lWS|YQUDEeS1f?zA!Uf7vOtA@C!>z*A{U)uumZ^b3Hq`f>_{172#`X(K#vPcBr`v^=OpH&{V+UyL;2~#tK
zwtz*{6DwZmF7o#mZba^ldp98fF#F7L*@;Uga&JnpYiM6BKA^&-0LUD^P@(j4J*PPE
z)&mCvF=f`e&W;U$*(?#xhu*s`9!>P>1pQ|)L(Ahk0epzI%KCO?WO<`AoVM~lTk8m3ie-az5W8y%s@kac!{qj6(0$>ZT>J|7+f+i
zg2NO(B!b|Q5PL_*h16!;_#EZb%(6D8RFe(lrXk|<@VJYB7uoa9osaBHG+=rVrg9iZ
z>71JEm$*>z<)(RgwRij&QLk8%_+#@{f*EX@CLU9FNMk9C
z&Y6GkOx_*AFosVyZ0ZPa2P$;1L+X}`!6Oe<}&oHm?>ln=DIkGP*C@A?;V>g
z$AATB!N$O#5uGWOc54UsO7LBro)|gY2=N5Kfa+llMldlV$=EIIR{BL?;|nZ$
zE>S+wjYO-hc?|KHSkRAAwF_Qwc6L!hBGs{Db}DO8BK{MqxJ590+`b)R4F2!4$lSy9
zzA=Sjf^h~hh~o?xV>m0#FiS1=rLa81BYNap>yJefJQ>WthB2cZW#&Ls|9EiKE=8QfFX05H+F@wVK~9*hg&rtu2oCGP_ES2IzG8`j)-2=3Y~#&`Ws$
zpwJ>W4*8D=9yLc3i7Y#d$s?znmz)E?*(f*K;A}?5V~%Hb52?90zp_i>L=x$Rn>qY(
zcbs)GSOOu^?hOMuJdHJX5C?CwTy%{+O%MT=0hk~98=l?2U#CMzK{T&SRh2wmJ}U7R
z8%*lV$Y>T-8tpW>?E-s)k3g}n5LLh+dFY;rzufbeioGL`50;a#YXQnJwmF)cOHvO|
z4F$BKRV{acK+j-4&-nyt(Oe781Hf1QFURAIO5xIpMr|oin?8SjCz2&R@d!Obw5w2}
zgmvLtXE2&eb59aW0o=j`AvE65*hUcwGkfr&Y4S1=crinbDDUB9ex4otn7f69Oy&ew
zQFTAU6#^R*zV4H!PYc0xX%uSL{OIiCFW9zU5!|%>RQ8rl%8e>g$i68vL*Q?Pg>`X4
z0(;}>0}X1|#RsLO0aQ%mf42!a0?@MwT`TYrIsE$Zqxsb3%Sit)n7Z@CCbvJ4%uskTSd>dbeHMw=+Pu#1C2|
z2Su4FYS!pAhp8^stQql!AlOaGoi5M|UYH*8ghyqc?YVYk+m|oufxOHF3MKq})*B;fYYd}1n#kd)IZNKG%)cX*Xp~x!CI9+Ze
zI-$t8whKVhYlFiQY=)_&x(6&E?96f+0Q1+zV%he8IRoJj;J-s#0%s+>acT8xU<2MY
zH$;l7iOrZXRr{?z5_DDV8DM&aX*WBS>nXM3A6C35589WEl{57bqs{EWaT9d_`dW|3
zA5S69g7ZRYUNpm0B&LMn;5!b0bh&kq;hF`$>(p~b`X<2qp~B%(9yaH^a;YYfx5g~;
zQQA9Sef$_kalsN+in;OnOOs9afs;Dj>)hlVvw>TL`u>maToqqJZQJE6>ytJLX>IT}
z*(Wo|wb-$?POg6k6n+S_G;-^>9m*n2v@lIpjM{dtPTeg*F=|B6!p6&bhdfH^$?tuF
zGiu_kL>((ndwU@YGx!32(tq$XY2t*uasGjf#0@~*0S5uoo>efmmxg6m_EeJJA0+4d
z7z#1}8t@_M;6c@xg*s8UUM{+>@kmW`tqmY&D^~<(4)84183S3$
zEGn5VpDE}aUr8;mjh9T0@G?6ve4z_1EdfZSLHsSk*gx}EIGDTW2mw?0JP|{cmr~Bk
zH%LYNfRAw_r18rS@s_?ycv*mp~W=bYzWEma`#$i784ifR;eT^$serD~BY?g&3Wa>+@MeV(~5$ZqM16_rq-xZ$Yvxsp}x4JhB{!sL)YP!u7X
zKb$OIB;)v4W>+EHC>S`xWJz&C5KAn7WVp&`rR~yFJ}}ILfE)f$0uKUiA{H(TyJ^{r
zJ`iC+X`NIPfVUFgowY=|+lb-Aw3=M
zdHiqEs{|y4x{T8}GE&BCQ$+Qz5hl$FS~fl%yMZL2>U?LrpLLWj&a***Jy!J>MeYY4
zEOTt;TpWTzG~)8*a#@g*Asp7CU_p*@@iHeEG&O|k+<)?!H$WQrvlMG`(s6!DRANNc
z76+bT0=yT$)>~7Ys8m@{a8}37O^E7cGmYqWb}=K#33=WdZ0EE3n5oU36}6UOBr_7q
zXzURokb!QV39OW{veb$@a4NdFk6b7*n5+gNIF)0wXG(ATH~%@^*5e`*Q|udW7i8zo
z#+cCAkRP|Tqg`EBh3_ejj(x(Qmk6MIG0d8N#>TQzlQBdst+_@(_kYv{!|>^xid5ym
zypi+mbRC{PdE(#q89M^(NTONf3)Ye~V)3p$duAe&!PAGz0;9A4B?m&WXMk%>sJ)43
z=ea?e_U^WNYM+N;^ZJ6oXf`Zm7-)t7kbZ2)k2TKgO9{N#-WKGhRUVr{dtBH4^Eq}^tG>i
z`3WF#G=`Fl-HPme&1$6mq-e|_BJm6E<89H`JKTp02Y>vS{mwnVZn?X6IE`328|CLXE_?7N44t@nbQpW9@uQLL{2A-8qGH>EEC$A7_gSp$mrb9VwX^{(Q^F&v1kz1(8z6A09_
z1ltm*zcj)$$uf1!UP3o*>iT?V7y+f?Kkmzm%no+NIvFv#V0Mon0qL6NvXR#)dT0sI
z6qMe)$(hm0;tEc4s0iuyPk$|MuT)M1t;g?jrmL6@Bvd0t*Y5kOa_>S~gki%zvRq*D
zbOOXoSTEj4POgh#dW4^+#@XA?sAcUIoujq
zoJ(|_=>0fJ;bQ(nXO2S(M|M*kaacF}3jn=D$-U@|y%sE36&QH<)Tu7)6XsnFRbSZv
z_Ko|feUG~V7R+tyA3ug*w8(cATTMqJ$$BAEX(c%s5GW39+%|(NC@=ZkY`y&b)X*F7
zKIVq+C6dFd&(l*ic6FUxKiNi`D|{9H_|QGwSI90%<{A0F43Xxrpb;{RGqaO&^>;Od
zMZlf4y48Mv_qfes3x1<}_`J9~&F9aawX-`-1II{`0~)HQU_Z`xMT!9rU+P2IvABG>
ziF2r2L%F$9<%<_S2Z|s6Lriokkk2%;{rxw~B|(v)z#(0_1^xoV`MRleYRP*ABzF
z8Uii)tkM3q|J)gKME^vJ4;
zo22dFU@$`b_N8M#!OO-_Vwrxec(J!2lH}0S@v1
zjXxdmJEleD?qMi7q>IETo*1c@NI;}WQ`h`PV|wIvnQxu_)r!mxI#b91;FdGwAa4@J7zf|e$B!d+q|!QZ3f((6c2L74
zS0yFy53FM{!a%Ng{pjilIs!Bx%)yi-pLXMteztmOI(AK(Hz=|>NkdavC25Dj=W2x+
zY0>6P($xmyShcnM2iGX~LI8XZBqFc$8&BIAgfF*ieb4ZN_)MxkE>j?AR7$k92F?VlMhlH(WM$kWg0rXp%-N`N4&__=Utu5F@Mu=A;d6rBCK~NHwGs<
zijpbSx@%o!rT73$%DEf+5J~}ViO@d%8t7D9a4E_|SUxNiC>d~|q^t7?QM;BDV&p8|
z$gwS$-Ju-UtSWgeENc}8KzfdgpuSYFiVO#T!4fGgJ^jX57|a+>bn!U=B7KFHjgMaPnGwb|
zPF?GpQ~#W%y%Wlh;)Z-FhmyR(%1fM`#0rhDbVu?R<>-MZBuRou$J>JGnQr{MV`xe|ooNzq~{scP%5
zb&6midSE`6@5QRXGmr@Zx1@R$tuX9B9E(Ihdz0b=+fV)lKSL#6?aWRP%a?n|%Ke+f
zN$z$-_l0L)nFwFda|)F?C`~ZO(j8bmTo?i{YTUkar)g>hr$ltQ2b+|t+BJ0BsyMG-
z)cok3;s)
z7+gg=b9uv;ei3#6XTE11W2MU)V-xZ`SYfQ@n2u)@)6BKa&OZuIkS^GPI2U5>Cu+;^
znsfX1&8D$toiRA5{|bXvAh)J`hm0lB4R?O@XMB!akQ>8j3syUgw!e--imLNAN11Eq
z&l6AE?Zjz`C0(n@a0L}PQ;O)zYsO`%bsLYJm4EMy<&yjAEoaSmZU#=|PGAn6uOJmhEYp
z4smPLvHlu29JdXSx{&;0j)lcQ>6)77&l78ld?zd~DlFR+;U8+Ql>n8Jf;CjeQLa|!
zq1ip##(gp3@eWV_3PM}yM%O0HcaZ@K;J744V-M3?H2s)L}3kXxY*xNzYQ
zs9W(eYwfikwrzLGp-tr;@-G;paS(HN}yt-ZGQo=d+UQ3j1zGeG_v)H5_6=d!BlHz4%8`_EjqlHOQ
zso_~}dW_0BC*^7(#PzuQm*kgdfJ9nf>_DFjT)Pn+4sMSKLNubm7-$#Pu32;c;X|e-
zYPuE_U^HT&mZ%+~9SkF@2W#$d$r*wUkP;Iz2^?ejD$(*nakRp_J3zII0|VW$5|Zc1
z)ZD$Rd`Tx>t#@QKxA&8d8{>R%DcRKRYzDe_v3VUCn
z2rqY)9+=;s)1Ub&yjnqWZrE#kl&oML`JsOFrzA%=N{!b^WwZE#Dtj?t
zB6e)li!U3c6r@x)O5j@O)D$&IYb!X)6)QOH8>l+;_1z5o+-3JP1eL_@9n04%(sH#*
z7#SZkPG%vDau9x*g`&;{kp~u<_og=I*abqYPT*Cf`N+&>3UPw&DR^8&6(kFK{V7wD
z$z|jiZS1wb5+m`*p;SaLF^$92NJK^>t{PYJUzSW<3D$A4SykOuX8;DKoIQU0UE@r_
zRe{Te_0QN&aA>8^2CgK4nE616^#Nd2={dV}taiEQkzskATZ_p797Klc9)IA(0ej@X
z4Kzm6EW0Uke)zaee0ib7*|L{81eDR9pUGl_{K2d!-#a#G3b%0WyvGsZeR1#XlY{rQ
zUd4MBgo6aIzV)MbjL=DM#>E|92TLIIv&oZDHJm%wD`Tgdp~NfRVdH6^6HHIKXeWKQ
zOptgP%uq=iz4U<@zrp^#0|;KQ>sUUA7F6_V{Fn~4_q^xFCQ=x=r!))
zgWB2~%m)lEVOo770e9P?j7x_PgNPnSQ^p@}8MNicXXny%jWKf_-#!R;2sP6`oR`OR
z;AU;D#+5_^S3Y$3Xv4eh$as=6QmnUGO_wW;TDEBB%srPc|75C`{fh?#C@%f7=i7ao
zE@ng=_Z`rCf$S*dIvw#38hOK%4Nya)imUJ6cWl8iHVC$D(z+lq6CAVv&?K>;xCrCt
zZHr3uqE&9G@!x_%<#%(f8GnlvC5IVqQX_@2=TaAZR3SvtZq1|g+WaGA^)O|(=x2Z1
zC4_f?-HrwHYnckI4M?90_^bxpI>7Ej@lIUCI($o7Dx093fB%aPN%?h+u#6}0aH+e$+teb|C~
zfoCMaMe}&?MlEPClppn|P5=nHhelPWL}t^2$I*-tscvZ~aSl!TlOvX$^K
zPOZqj{}%g@;Jz~HeZ_4ny+mLgEfk;AFgF?#X2tu50myYP}PgGU>f8%^eyv5pz>mwp<3tEGd(t
z9VGND=g+Sv(#Xp!Au*9q7c}K^uP0d72^RZEXK)Y!YC-92W+t)^vLPbz_b;5M!>d7{
zA_gtO?6k3wsj+mm^9$)vyD9RM89VySnbY0dk+sVKhR1uEKO9crJsvyy3-h88R#xnR
zAFLNu%~%Ln!fkkBW~V!!-PbvgTvo*NwM2W|Juv!ySy?lbLwOzB4m-JDk9ZdX1@E37
zMle_g+_)j6zn6=;si)aC_R>t}D2JPP`SM1Aw0$FLJ)uWT;b7YEo#Xp-AK(Z{9F>90
zL_`rs}2?`$5Ye^Xe-_wO^(#?4B7c{B=j1IvP4yF4k85lDGx)xHNIZei*P
zEuuV^xs79Jug^1UA3ahfWMk>lpI~Cy8-=w
zj;@K-wx39bf+b0)+B;pq)|7=J;pVQsn;2eG?5f5wo{;U3O#wK0$8(}nF>j<65SR6Q
zq>(#!!h2XQK`U3ftPjdR1(%}i=FQJPHdm`9yPAM}lHzS>g_FWIrxz)v6YX-`a%vFm
zc+Hr>&W7R2{lUb%s8jQXFCcGJeS}z4Zmy9hOX4MSel%hee^V47iNeavS4A_=FUHN^->6jG*CGR;4xkE$
zV6sFOvF_SRXIBry#U`i51zeKsD=yjiHR-(Z^z`#!&*{*T++Gn7
z={K<6bi3|9;qx*UC}!Bp%sqPi_Db30SlK#$58^20rKt37uhpoaa*9PA?6R{
zhO4Hhrlu!O?=S=A&qsYEK4cc9;)jSMu4EQ?5aSr+gls)xW~-k)L&*s;!O(~Fpb!fu
z^<0x#;cD)(?!E)$O1@&rLZ=Gf$Ct}GJPYwU6;_k?uF62jPoQqjJ2c6F%FEC9a+37h
zR?)3RKEe(!sGSK3#I(HU=q>7%UKxlrfaG!SAMUva5NFZM(()N4n;9DmP8ThPi?$XI
zm8zQqkbX6=@!pWpqkn*-(?drUNj&?ElMOw_NVFfkHsC3u=yvc9K*@Db6e=kUxR5)A?5wqlMNm!Q2>)$>*QG4WVXsCQT
z&LRg1cS{WC9;d_$&2E%1oId$!RcJx%6sS^^8Wzej)*8eEP%YV-T%bcyf=7WWrAEkp|I$
z=2~Rl`xgkO?4w5tGY80?HnkmWW=4M^7Rgml1B3<;6h8G^M==?4ljqD4GJL0Zw~F#ujJtZZkPw*uX(@{Cb^Uy|Db))x7kms*BpR
zEQ$p4#>Wd3-IOV+A*LJ^{rQkbX=G%?GxaL
zh=5IyYBbqQ+kAG)v19Pf65)Kn5o#Sl(l7%3;-OIwKgOFi8dSAXu3(^TkfIwes{702
zeS6YHOC}{l*A?IBqr6xc_!3o2gbX!XO^x(JaJCiTKuPgJgdxP<_~CdiQWAr!`+XA(
z#$&2+O|c+9T|7G_O~*jVa-SZaFbKu!88Gk28OpS>USt*{B0~F(=aI*
ze#o#<`k$4t)0GZeGvouP=7xd~%g~d@0OLny@rc%c($C7v^uX4OX`(*|Br{j&*~P_%
z7zB+Qr#E}Y@1$biqGaVU_l*`g>=~`A8t~m-5?dY+0u30Ti)*B4z5Lw1Ox{q(p2Grw
zlXUQe+P7=DTU$25^%chr;SZUbE@D7pt5@~>xqfRg^uPZ_O9ol)!-^0tCYIQbLc;9?
zUh`^_kNJmoE#|ShQVf>_|Nb5Tai|F&`+v4+vh(qz;1W{!y1!l82kN1Y)xC!gciS1*
z{X5$re#ll-K@YZ|A4yJbdi4sA3uZfK`GIIXn8h$tX%$Xy$E4v1;Ql3Ek>n@mZqH+(iJjUkB1WdI-`CTG!h$ITS53h{mKTQ#Nf3`1tlfyXKqYK0aYex@yH0f
zi(7u3$DkcP*4()VXvR4+kcS{KMasZUgj*9TQyv}#J%&`m+T(7UzWv(py)B;mM-4GE
zJ4&!pWgrxAI8HdHKz;*}rHef6GuT%v|J`*`Cs3FlIAQla!uDMCza=j}F)9V(q+7&n
z;a+9s?+;f#qIg^jC}}@sRajUdNC0Ioy%7m+$XmLWKJ^m594m*LkCSN{y@fA7cxSPZD&_J9aX)*$d)`f7T61*UaAvd
zd8STWn=)G!%16RxG++PZ;%ejUbnw>NptDT;O`%86nw0^MxYt-qmr}buKw{Qfx2H(o
zbLF{4g|Fwfk@}311|nJqmn(wRjJ^G+@yVKEgbG&*8
zqfmpRUG9CZB&uXydhF5u;42_VW7fW@=1U6H5k4mU1O7}A0}QOUt*F09BESixB=z;S
zq_p_=$-aRIk(?5iEcRBgNuom>zJ34B%8}g!0M$oFhr){lKmW&@xR{d;*5FU0`GD5-
z`}?PV;4%s3!)Xw|Qpu`FY^(otSapT!mXoybaR)c{R8Mwg!uI=nVHXVG{b^|^RJ%=F
zLRc2*bj*9mp+tA{Psk-f&a-_?crPzGnTRrfzcbUZ;bB)J^%9Kw#Jw~Q>RSgJ*xGjR
zeJPr0i>Y1EQ2z7f2dAfyFGj=ZB~~%(6PpWFw+xK6=4+=k2ome+^_biTD`jG
zHm6A2#m{Z><0mI=vW-6h{3(I)%YBQ
z7=WgX=mjI6=4K{E6*z|tQkg1cFcovx+91Rlbm`bI{^T0xonq@*8R!_K=z8oUc@+dY
z;AjB(*w`pNlc}wmvNEj&o@|Hj
zzvTofI?3%ywY#!6Vse9(@`<-d4GV2=^%Hh>Mh7urk;_ZoNVPOJ!f(JHR>oQ{-8R#c
zijc6zGKV+(HpI?nFI@Qe}66wx?Bv_#U
zI?%hKwffW=wy{(r5zKQI&?+%y|5TOy?wlw(*0|3`1SnA(F;Y|RE=Z2Cw5faYWMgBI
zG$MI+hWIkAN9|>IH6zGmoJN}b;9k9AoZZcHd9et27lwpTm7P0t27?5w>52mPi0$2j
zt}w4{CL{qlOtBxDcww9yo(Aen->wVI5uA1JUhIn*tZ{}29oYdrByVVLAgHf>n-YIb
zOLMZ2N3CgwXqtgQAznka)bi-Jo$#3}S%)2o2QJ)wP9_!a$pq*g!%@
zS+aA}pHAPV`hI3j!KC_*KOFTKS`kVF?i!Zx?rFx#_W=DlhOp4n>fQSYVllQ5{VGQ?
zW8FHL^Ale!{a10v=C10`JW^k_YgopsPHundcpX~vjnrscKpH~x9H!sW1;6a!vA?Van
zL|m?{1iLY>jM8{ObNc~tfq@w02tkfQn;n?mM_gDpm({v)goDOkX}5nPFd->D;TM5o
z;2T6D)%fvUT+nobb+U?~Aq?*+Q6J0-Rh?x8A%aQ^CjJy3R~l^O4ZglaWTC&yfGDpL
zCLadF5?LgYeC)IJuHis~N@>KXQ3}q12nP6-8z=TlrLzL*&CiN&69SlNAIva}YXj{t
z`f{f^gwv#{8#glb4-3wBYxuf@a(%zR41hLN5uCKUUcGYPs|
zS&0S-j*P$qh<)sZvBg2UPpW5pewbLmATrpjYBQV9##o;T0mX)pMLu{}b|$Di2l-LU
z7R!jLSPC=RU%GqJm>(tMy11INdhwaZ^sy`y>8LdZOo>6?=^27A)uYm4RGc{eN#ySt)cItYz=R8anAqAa+4bxXcNo&>Naz9T
z{JL!taM+;!>Ca0Qo7F)Y?O%BW`%Q;41pSFvRfz+7+C9
zhPTEuDQ4aAKAt@J$+BfC5_%6$fd>!py{p*^QozyBOH&hx>cg@!`N&@-gM!T{l>^tE{S;P+Z4!t>pC==Nb2(PpD=L%Fph9A4VM(=LKbF0Ru#9v@x*()WhcziWCf61#%GA+5n+9f1el2tP&hr2g1a
zr?O-F_M~d5#4-Q*-1ie@s#G>(Mz$NOy>+sQ
zhM|T@xjs>GY#1)F#Q_0!=Ihm;FKm5HUomCo%(x}ZUw%+v@HosY--XM4?&jT0+?}S*
zv3dAx(S@Ss1clMa8c^PTn^{D@<~TWZ2Ar~ol_<@5HQ}99!h99B1*nlhmQwCfX?;!6
z3D{A`b8`ux!=f%u^8_s~PSYaX&NsGR(0CierlCq;bZ>7j$OUjVj#NU6h`9$&ixmO8
z`iyu5@rHf?kd{Bli$8ycAHYK3{sNic)@RM@A!)^v`OzbP`ElAyvA_X{x{hilP135Q
zncY+L7mjmQsZjhTySx9KAtGFTH<-wb`;K!hxT0xMOf=qd)TS7;?H#@_$tTq_Ou=E|
zsw39k4lAExAtNOLqTn7aGqg0wV@m12S`->bk?45#5|yA_uf_#tk1-m#y9O9R;{`mYfHD!_O5=ANWan{GTQSXol`@v>F;HTY8+)zq!3lwN6zIS4DNMsaT
zAR1qOZ<5|3g$gTbbZEaEtIQT;55*n&x9#g!E-(H%C1tnR)c7fgQV;ya!+r7crK$ZJ
zJQlp#Rcn89((v)o5UDPWH*=7U)z&+>FD!n6%BhaRq*!aG7w^+PH|3A{`q!7Qf0;Do
zhlO_J&5YY=WS+h8^RUIvZR_idyQ<7%<7=CS&B_f^+_G$8<6kIkY1ai2rSItHmbuR#
z{Pw&1`NAlLARG;N8fJE87CTsZTiV{J+q_XNNN$ZT^XHb2c;2rXwr!&B=+juXL!YOo
zIdN}SnLoL7!P|q
zEpct#yL4BB=YDN`P2=0>v-a~VTgPm|2fVrn@qQe3j4gU
z(8>Q;ZZ+AYr73*n!WZlOt_=7rasA%Sa+ZBz@pX+ZFWq+c)j28NhjTxG8P5rU
zseD^2xu`oWdD*V@lw95)DF=@^N
zV-y0ETP(M!^&B`$NniEUa8@Q$T+bsVnP=jQ^IfNTBpdc>vT}2@YMc|EkkX)PJ=JJu
z@%0qTl`;_q-v+t$kseUC-AbbqW)2tmDx~1tSWA{bs@-R?$eoshVqP_F4i1ZZ2Qpb_
zVm5jIsGYZ7+JBqK>cy&WW7zLR}x<&`)1`xid^E-OBdE|8t%JJcB^Tp)8@n8
zg((y=xRO<8w7#k1GR@iKj
zwqoA)gEA8x>s{g^bDVsp*d&F|BwSbbo}W!ok(P&x=>~x
z80V8yKG$`2@7g0v;`=)dPR_ObqxxgPoR{M9J>TBVe*f!Hzb(hDlWG-`lYbw+>{aoi
z#Uki?x8)DB_1!;y&0eJHn)2$8YWKpgKeGF#jqB~$HKR5z?C@RbZWC(j-2Yr1nrhu<
z5qG^l*ZgLX&PU;E1>owf_te|LYpDcA?+UKaLoP@7}xzBHjNHMyt
zlx)*DXW;H9_wPqNeAf9=MOUJ(Eh$WzX^F>(qVGro`&!?&TAEWX_(Fzs#*)b$gfLrI81Zan=SV}+*T#5y{YBi+obYT+H2X*
zcfAx=6j>A~efDe3PkXGdv`Nj?dz|k44DVMC8-u&-`%F(MZBMLj^{?v8P%GS>ue!-N
z_O@}`q_*c(4e{42ZueZix+bivV42&AcI{~gjH)8-zmIe)W6V3U?b_$U((4}!%`ZI}
z?_wxe(zl**FaLJnX!nDpGYjW29{J40P)AL-_Ux{@uRQLKg
z-?sB}yf|^djJX4ieonk=#hgVP_M^X>`-}9jgPEP1X1owZ>$x;_*+%O%ltr}-?A&!&
zwfW7bzr+)j4(YT#^;_mwy3W))DZl*vodqI^=})PWTWGGSNT`8r^@dRtl4pF*(Ljur
zjsFMX7hAGrki-ir<yG4t6+)zt<)mB88P|6s+^V==JKHIKQ3wHe=l;XQLSz&?u3+SiH=M59o|
zwwz47^^&rlI&il;O;N8VMG}Sq7oR*CnCF310DwQ`&UGfTjE06Ez($`483Zr%l@#y_
ziw1y{(HJJX8Kzdis^OBx4&B+gj6h*6V>)0e_^mEEHB4=sZCs6#X>5fi!I1|BpTQem#{S>w;1ObJnD*%{>Og^w}TJYu>hjtrD}GW|vUX!~3%H}dd4oqNo>f%AKlKaczJ2=S
zcjht3S?%Pn@xJ5}+@?U}Enu1Q+bXk3;xlmfzs=2*--JA&7}$Vq<7$bOL)X~01tW*C
zrmLzF|1)y1>-ekZH{dpY5`UNuysO(@m1BFYR$*?r1*9>1^g7hod>foc4jh}nWUM?J<6jA!EPQ6fO3%a$d_<88?t
z@<3i5EGAsb=d+%032MnTQ&(^43qF&71fmN(yX7I%#$^E;tN8aB^d}W_SD*8MhSSV?
zqMvLb%!ptA|D&UI1MIa
zBKqstvF-EPyPg}@|9!TBjY5BHaP5dk299hQc<&`884Qq?J0Hw5v(;NLYt}YB_LoA`
zlv%UTtK$P_JoE*N1*j+KC+hUlii&NnCV^q5mNF@#>)K=1pJ1>=TQ_&Eu12Hl4L&4W
zfyRU3hx#wWZB9gL6YPu~=Z0V3{t`t$0`&|DF%pZ=JMiA4c8wZ0V8C0XFh)y~m;hu|
z3C=~})o^Pdrrc}_21qyv9Eg>GULtY`xI*-{hOe=hSj1G7wTGPF&Y<<%xA|v-rm!2x
zG=Zmv-WGBVl$1UaFJ@_oxDEm#HHWZSiBU&u7V@oS*4u${Z*IaBfnYdg?%caCKY>V*
zgN-!Al%wW(GKcz>}#Wfm50y
zIsO&&f2N&qf?%YM4{bFF*0~(qTO>!1&Ig87%noA2g}*>WWo6$HfgTek#PP;Zev(p$
zEg1?+eAGNWccFjSQHHr!#ofDUDS=Y2{Lxtee1Zsj7&SG5tMB5tW3oH1aO*YXkFzE+
zn7~AZJ|E+f9{c_0=X#KI%>dE7zvJsvrg5>+(TuUdDgK+g{7py7PPfQt`S=l&`k_l?
zqh9mrVOPe=j<=1oK1)Y~Ro!#m2S%{iyw(F?vNUsYzTijb5y?78qh_zF9`cR
z5`xygwYc9|SzUnXHf`a5aN0wDKSUL#FBGE44S-6nvx5M8k%(U05TB5MP^{Zc^HHPV
zPk-fURaNadUFU1b;lMzU5R&lWs1Y#Kg8b4{BABw^QAUVPESDUft+VSN$o*R2I-KAv
z%!O(>wXH8I$jLF>QaD@!Mh4fJgO+20$IT_7qNEgg*GhI&FRg`)0r?Q8jvrrm=5eSk
zlhn*&Uvg5R#H>X7ZfIzTmT)H-H2NGb!ZKh2+OOZuq9Owe3k{8`$WamQd3pa)CF<7=zAw
z^*0==II&s!bxe7~YdfpXSfyug&kE}l=t+1g|CCmoO*%$PQxhTNJTu4%Sewx1nf|rI
z&vIQ+#dL3PB{en1d4Pac78YnkcyA1@d+(``i72{%e+@D+)(rzu6psG{Tfga*=?Nb5
z1l1b$xvFvKDsZWVtN|9AZ5KVy7c(cQqPtU5v5V=iv5g}Roi?&iyk&JreTa`I*Tdn{
z0XGwGco~SvSj@`jiVE4<2Q8O)Gxbe&buw3cH{Gz}w)R%2X(R>|h^N*jP#oRM81b4x
zrjfQyP%Mr;j7Q14VT>TU5*w=cB5Yz-j`Zu^nWF`b_rZgVIJ>zG*<&Hp3c%Vqv;Fc5L1P{5TVD
zanqrohdw}c1zh#|_3M|x7a^ap&J=c}UA=aVadK>A8V27u@D1CQ<}-K(6J0i#di1yd
zY?!oqI&7lK3JnDb+U6M4xSlv5KwR4O3+K;M40S?rhUe+?r_rB}TXn+1x`d7O^gL(a
zE+?Z;KQ>cI;UxttA50UGC9@c2Vy>55B42~zaKwCg`&K=o8k!ZuU(muQjGt!DnKNY=
z(?cwfC5#INE^AGY<;Aap69euEsi{3w?33!bw>TM*!R(q9P+3
z3n`A6^t4_e9}MRfCnv#n&dW>de-4CjZ3KU69i4ZLjZ6_nudKJGGSOB219}jUAZ!P(
z0b>z#^Wdv@fS~s?F=6Ox_a0k%M9O(U=`;52h+Z76<|o0Av(8U64gs1#3*?Y3{Pp3A
z+kz#CrI;h6_M7A|b+m5=towQH?zeGT;`acM^WPlC)D%sN;UYmXp@1IS_JR=|ze0WW
zbq#&*|2S=tEAArOu^2G|!a1H;hdnZw%ngAlo>0I%kmdI;1B`|7ZT<|~dG&Bkg0{Pv
z)N-3~(@^sOen`AoL6qAxOAA1%3=NBmilol^2fv^ndQKSSZg-+5IaKe!3t=xv>>#gB
zGd??-w9-+d(u;ioD`0mpOZ&pppn2g(#aT!a)6>)0J=M$5mVliwmEArpq{UA1lIYh%
zs1Z2ZYHK(0I9b|$mb$u~6|2;-Z
zk=w(wWi;(t>aI*k+}N^R=}@GSOzo6?K1XYf-3+2l{uuJ@+L?pHBHsMU{I&I$TJLci
zZl{eMQTVvoYAC7%ZYkh$)M|qVpEz_#h~OJz?rfue5<4PFMI`o(!Pd-{>E-t9IYNyQ
zcoljCBV2PWWl&cJU2yyOs5_L`uSTNAWQA$=SO%1=Rs_=+|L{}DX5_%3BV>Y$J&DAG
z2SAO&vy(L^aI9yaK12dCX0XIIugxICqjRwjvs3;SqD)=~Tf=3CUGb>CX_~ud<+j!x
z*358pdeH0FAY0s(X=t1p6jm$o68S2|TRowHyW~I#{#3hbhSsOq9ruYl2Ml9
zFnI9h#oNon9IrJcQ#uqM8?s@kMlv#(aT`9%&1kLZsikDx@HJgiZ5Vg6xMV@o>ffa9(v~gh&m5=toQ%@+iv?NNrkK=S(T)cxQ!&0
zw4{YpDwVdT9iq_QMs*sep{;2Z+B7v0l}czMYay-i2iyIkv
ziYQKefY^kF)W+yMgI*e2LQoBWn{6#9hSyAuuV
zlDkn@SE9)@!sU=
zpOcW_3J*z;=3Dxo7Y)`-VwgGYA($bID!v2udn_UAXkQod15an95e4SD3wF!;tzo#1
zR5b9HBaxO_-@a(LJY;bAKzvt!B+-Pv{rmOz
zn!Y?|F*bXaP%qBU9S04A2wO8ievF~`=AYv1l@xtV5sM0{CtH#zo>8kv+YreK@_2Dc
z6_{d-#@gzgJD(F1cwEdM%VOw;PS41lel}PsRPr)jUN4!_?0QO0xFKto_4Bvd|IebGfnyJKJuA9
zqAI!4`k{p~4WH`kv9r3%fccp^=aS+8DtQKmnb;pnT#fRC6~PRcD2D}G+{(9PLU^fo
zdkZ^Uz7%)4v;~6=T$gb&NZLDr!;9i8>bi_h4(36(8{}U3B4}`MOmqH4&&;=E4-HPX
zJXLJ!q57!m?9u!>A`H#Q^|j4fYu2H{CT)?_9P~h0M!NMy^q_E;ZAMo;t%7FGgglWy
zT-s=7MT@By`ZYGQn^VE{=Yv35{x~2{e7X6;z`fTF^hcl4YQ6Ml&rQg>s
z#Df+27OD#nCST5Cl
z*?y0MpeW-^^OO!8_Az%eW?N$yl0r{|e`S|feG6y#d0|yT{O|mT{KZzrM{Pkv5n(vR
zk6P*S)O73ZFog!2I8RfoQm~(w|K=BUy7bdB)`izQMm^lLBXon8`KeQtwH|Zilg9VB
zG15?sd4RoZRjyIF^@lKrj1H3z^Ao2=9jR2tK3P{+Dt)h~75GJ>n?!%(%Z;OtoMT_j
z{;12I6DO_+2@$nzeDaBg$)})VSya%|?(v0bw}VRur(JRF*!<61s3PDQm{67r;ITUcQi0HD{s+l
za%V$e#I1`l;epwoDanAOZ$6!JsPUb6&;8lxmm!{XR@Gxljf?k`^C6lwip$w2@aGTR
z2+q;`G465__NC0jF+kMY<#iNWap*pvogpm;#g_J*d)`S#^ad~+LR?&3eZ84jL@eQA
z`H4muis2nd2aOPi9ELk(8B4s=)s&2M_Ih{yAd}~>U-xg>67B8JX|1WKlvT=c5}Z7j(*A2erC*;uc5DzvCH%!I
zkSB*|-MEyHI&kb@9g0G)p3$W|P=Pz(v#xdL=?XW?rRDE}3hQY|kWOUCQta_uaNd*o
z-GuWTDk>fWd=l``U3bQw{f<6IurcsGXgWL3n5yDSLqoh_L)j8f=&g`GSkFa14HyJl
z$`3BtnyhL=7TZ8Npgf#E;@jfFx16NM-t5#dP5ks^E94K*Q*}`)Dg04UtTplNtm%%iE=I)1IK#
z&SleMJ1zGqWBOXBiZ^er^LuWkVgSs|!H$W=6p)}sx!-e)ExU!Ev?1P1RK2@!GsfddcTFR8*Db~<~kB5&>@!DITU{wta-
zX&83FOd#vh+R~h{4;uggK<_r;nd8Q-PWC+iIoQh
zLtlzYFwgixtmf#81r`zdE>tfR5I#Xq2~3c#kqtOHIKU0;j&sm|+eo+blltl=Grl2%
z^O1C3ivWIem%P72O^DGu1<~8wh;*#e?`}#=6xKx0D6&n0qLHsv*#>k3kqQ|k|H3Ji
ziK_gaO{-~pSu4@a$G|ShIK;`bQM#t1I*!Z
zcQ5EnW4PndjR~u1PONd@CE}TDfB5*(b&#}%H|h~qS92C9OR%0Nnqb4E*LURa3Kea?
zf*kynqb|GCc;0>a(
zDTSA;P75U}zqR0{l~{z>E6eH?M9(gx$QU=eAwz*p*OYVh>Dfst?D
zj*>Ys{W^4rP7`ZT2W7pFkz~rr$>m(R@-8eVKOY-(VGAG%P`{OK$|l;%s~ipby~c9_
zRF8%B&j0vvUr1Ao6VT~Ga|f@cqusY<%USHO
zfe@XtDd9J}WY{7AWXOLyvucj9cu!%Kp$P8bW5&obnY{YI+Z&E}xOp{piY`Tsd-kkv
zKVb_7uM~!sjA$SoCMFe9u@ilL=|B+M8yFZ6^89EgkRPnSEbW1!jl)4Xj-{1Wd}-u1
z(ywW;maMO^S1$lThnnwU^t`-u;u6+z`;LevFP)G6PMXGnSYn
zB`xWf0R%~voS~N4wahklrA(|sWRz8|kNIoLEq*In2C9eTQ#ieFl?h=ItAfHp_8ywf
z)#=h@WXPDsB7Gne;?QQ-VDs4L2yW@3U7eUQI#V2iV8V08e4qW{?*A<|U4O2}aF)UN
zA;ApQ@sZO>DMpRvPgxG{$}4M|_XmwOkijE_rI}L=#dN48tnOmuM0my+Ic(`tdUO4)
zx3TA<$x~NRnI95DF@^*Kuv}~B-CGt+?!Ayd?cYD`f94*%iA-r!_cAgv3?xaYc?0?#
zzq5Tu#I~-tTvR`+#0eIlJX<6Q3f42MtWJ{L8ke9?b(=C}h_y8@TDeXw!_~(N5CYMn
zlqC9c%IG6k0m!aiMM)+2Md6ttey!1e+s_W|+EA@T_7C~an4$3$cYDamIfxJlsC)Kw
z1Ff7eVbkW#H`pQ##0rAsshHd}TAZ`>;m2|2+u$qm%K3UOfl^8&z5BcS!;APujU|><
z{nzGo63NbqKTQNHY%B8oNYqGh;@%Z>+PN+|a
z^3>w0Fo2#>o}P@P_ThyCU-#K(!4q-^CSgF+A8`gL%Zs@KJ99m{%!u~pBckq#n>;|k
zlLV%#$hL1^pz>({%CH=wG!lfbP!`B)poi2qxrK?*-SDvRL|tuUR7DFP@X5tqv*{I<
z%lvWR9J<}Q$=r(cS5;T%LEE#im(HFd)>aJ@zow=pOjlhJ`spliT^}8;AoAO9l;LhD
zChS3JpsF_8%5>D{w98d1zw7{-2)eo&_zM4I74_)rRKTDSV+?9b(mu)CeYSjdG9Q!S
zBTHopLggDUg`;Z9&;_udV!}6;NQfIjOPPLU_Ub!d1C*fm9Njx=SglHX78
z9>goh+#B~_JWV%{9Z%q20Ir3hZ?4=3p*3A@5E%!(a{k&^mfo0!x2?G
zjjhab*UI@Xe|mfZMREO?FIO2P@Vi)$fMc*>eihq832Mr>L2st_+L+%&Deq4CX_m8F(SVHt|%#PebmxH2@(msr{
z#93H4cP^HDEZs#x3!t1~%Jt+mK+X>*e~}8t{!9}x{D(4yPL`r+Gw|mn_TSaNy7Z6*uO=qdfGD&_p4TxCIXbX$R6)G+by5V#
z6!yvHHa4`9pJ`l#sRiI=z)(G1T{Ie4LqdI5oZ#{pX3Z`_K&ERvB^Q=(-}&?Sen2ha
z8zGF$>g#Wy$fp`&a2xqeJLq?NL!7fAV_;xl7QzdkOq?#SkDn)$3z?&?tcN8G;I4%$
z3H%NZHQlT15vu~nNsA3bFWRZrC||Go)2FWb`fu2m_qdDXEhc%EmURfOE6qJbOUSQ~
zCpb_%QQM&s%RY&27S`5GiWt8mf7~^Axy5#`&Wod>yUL%O7^Z*J4k02NR^f5lS!`Fn
zIwQ6>Xp|tXsUExJ#^KpK?Qh%=5RZQ697M+COUAdn&w1YA-2Hp!S~eA|*6Fr#UBe)TVVlhGI^41QoNNK;t3kq0*zqeS2w9k3%kA
zk)@ISr6oA|(%3TMqou~71vw^#!G9{bU4(%4(?)i?y>SbLE7HtrQgN(-TJI4)o|eAd
zhOKv$c#+E~gNy}^P^7n6%o#1Y)JJ?HsosIZ6ve9ve|&1G%*@8rqpu%53Ys_X^Ds+j
z8GioaU9Ex?Ij*#H2<1xS&!58TBbZFFNd@D|yAtV_!;!}U2*5RUV(&n|K%?^%9#qcs
z#mI-0(hIYmu{wbiBY0Xq{l)~Dqrk1B+(INm5`6@h*JFs$`;~H6FXXrHEUpHqq89>z
z{{4GJ@G+N_S^KXwW#RTh3TSKlyHZ!4p$pX_XgO>acx*&`=jrm_MB?8gk38l^AS;4Z
z2h{0on9ZKeU%&9co-=Y}5|MqTU$%HKGMJJsJfE?R>GM9{SUX>D!E^Oj8r#OK{q=c5
z$oy5SnhB$9OhSuJd(92cC4tkAYHPZygwvg|y96Uo%ZSF&h-+ArjCs3p8b!XF!wruo
zs*F1+k9e7YBor2$5T;H*S+bY*;uMsai0ccxVf5UV#i*#FZZ_f8~?GEa>Xm^)hPtfQOQ}vq%Ir-^PPs=GWC=6*Bp#8vFVP)faC9xTc(1doY`>=-}f1Fv@_a&l>R(yiU
zDVOll$=(ky5~jc(N;SY@N`z}f9Ij902TmT!(t+S(0Qk^h^YSW7GhOMyc5K;`~8rW@u@7#`3pM&57<3F0O(gb%EU?m0DX5&#?HQya)X
zr{KZtXTdR#3Oph{{2CBCiWhK5@1I>m5@w??xq
z01kZ7HR0ic^pR32^{wpXFXq$PID1`RU!M>im9RzFIkGBs%NAwENsk}z9WY;~l@aUq
zEnCn$DtWgPEWt#e`i$IR2@Vk4?*)c4E1x~@Sf;kYo(RuOs2b-Z-*MxrT0n*Zdc$Bd
z9$Qh_V?!>$5IBjB&2*Z$@7PALgXH7K_=OAS&0Dg3IV`|$B*4HIk`Jmr&zb(^Ma;F^
z!@l*3hz=0iW-2kFakk$HQt!(D_R~QjKz&OEfJ(ZB?O5MgAKkYv2lsuLL*A>zl1Ib;
zCDo1h^%ZG&Ii&Yw
zuz;E;nVMXA-an;1xCE-DuG-o^(a;kBOD04$?Ww5==Av_Eg!~Qf_PXoottG!RDDrz}bozLJABt%`ef>0#D2!~WgVEcODBp3W1+eK%P#l{V
z0f4b>OG>&+X-y`iU?8l6&$V$A$^8FgF!4#@w<*2fV+;>f9k$rmu}T^miiJJzb@tJv
z_@uZ+ri#|wH4w53ub6EK@YwMQqN%`49nTV`W7Qu&4g_|D0|_}CW6qB${BVWQb9DR9
zR+TF@>QRX6EU%H34_5u2yUb^RV$E_xJ_mdf7yp;9U*G#Eu!#krkZIY}sg~-Q_;E9~
z)AWsADJbk+YMHo(bwnpCWP%^Fi6^=0%a{4+)Hxu#&r4+j)Z)|2yuE#;xLZmw9EzyP
zqvRwr(;h97Fztru)Kd4Nzq1$NlDti$LE@l4to!msAh{_iu?O;?y2{@B9^!6BJ)gL+
ze-Fk%=HpMGj_}bb^Z{kK{_zX^5b%8uY(@c0M+th!d4gIz)}0gdrlP_|t6utyJu#Af
zNMz#pc@n+D(a{AU%6sqc*t#_-Ir$m^ow;V?jzCxW;q#T{GQnxrO_
z2jQSvWuC$H27@@i49gkHRp7Mu-@+kjQV}7i(spHBMq;J0Vp{X=Ke`*t29l3d#ReL;
z*mM0G$1BfD7-kowmN}V@cnrSh3^F+Pq5FD;-xf$?)23lJu#S=5Yqv(jY-RmHOKO-M
zmgfk{T=tbAe&$4j9Rb%2!A$3$L5yE1I<*JV3bSqrVx5ADkT>XsvAN&}u&&o=#U)%5
zsjWamn1RS`Q_@@)a&f}Aajd+u-*G(>-~Aiwy5Er5aF3fuN~6YtyA_mlg)qi{;k!6*
z1>45j$nU0&wRz9kESY3B3tc2TiQPs7TpE1%Bm8m<`zKADcw_k&oj3{mL*t5K=X}}V
z?iO#Lp>ofQXqkeapQmRN{jksHe7*CS*+TK708Z+4p6kZYVS<|`!wg*xUjqRi8t_ua
z+Hf9;I)#Kq2(+T
z;q75%fQ^(-H>>oAz0y1hm8-y(_LW9%3QhAV<_;66OYYvqMztq9m5YkZ;2{Y^ihRTO
zx-Ih2HBrAbwFL+S<*D<9O&5Ok+wD!Ajq?T}3AXb%bM4^o@eyC*<)=96cHsh1sxqxX
zQz4F5YKXNkSyN6)3wU#8VQj|*u_zW$8;R$MoLAhev$O0pIv_Dqinoq87TJIamwL9aJrKN3N%OqzAPhTm|oF+|1y%2O5>Q*?*S^ZB|2&cGZG3b`e>rHsaE3c;QANm
zjvx$|847nP@?Y6*#V=k=9zI;4+vX0t8BpZu<%J$X=9jg(dDyaL+RAa2*8m2jYcs4LEJFl?6lpxS5ftP#&#N}tUF6%M2LsaAepF0cQa#|E)l9K;aA8m
zZg$EW4EB#6{&A3zU?x8^is}bg9{)C(g*}__dWb8mf_B2ZMXK2y9K!F||6Fj&Pjt}4
zG6|0DOQFZ;-{{(9Z14}+QZaJw=ywgkEXzh6_6>~8i^WIPEBJ;9`Rm{apCOUURYP`DwvIel=4NOo@nWcR
z^Sgu8S82=Djs%zM%7?+6-br~e8N@_yz1z54Yu
zQMpA~j@;g;L3^}@yPjCehNl2R>uRb+xoB3E5&EqRS5Xw@U%`3A;>i13C2nn0VK-P8
zI=DeL?;HyWX+&@mV9tF7OoP*tf2>ExVMpzd*s_JFT}USey1J1@3h~cn`VP7G$obv-
zhK7_AR*|v!YiR?Q^AKsVDQtX{f_La|N~I$h}bX?caZW{&VZtR9xqG&0$T-teG=AiN#X$t@e*5k19US
z0Q_f?L-_=F)<8Y1m6eQ$3kt4fAn`IhYwx60$pt
z>&0`!>=r{zxnd+Yn(>7vC!d4Us1xLAxP(IeBMlaP2nGDhF0&s|5jo${JO744grffv
z2E=rcnj?!!vpKS@ZGS2(b-th?LK>3VX!Ukm$UbI?6#kU1!m5GMU-2h%p7o8|l6KQs
z+)8GHH;r}Oi#Ko7zKsl7A63+2aWwrU&y5#HFxUqp6=!_Y-rPY}WYK1S<+hs`h@`I@
zo>FTZZR^N1mJze8eI8Cm6l2^2K|^=+C`KtIBtU2~ewI6Gevb8YfM@W7?kQ49_3lkb
z!dv(3Sq;2|HfUyN+4TeZ_47yV#P4|Z@@0lif?JS}k9d>EU*>VZlFS3)+Xl~?1z^Wj
zhbCI3oujG3WQ|IS7J!+{F7Mfh!k|`Ddv1^oXK!Vaz%N+zwzbFpw^7;xVhUsdNF!t{
zQrXs5m6aK!l{m>D)CIw>d-m#uic-K6$gAYn_>Kddx=Z2-&-$x??_Khh#%d6Pazp
z;qCZt$$l@m)!1D3F%LfXf&d=2!Qf?VhtoHWiIv#UoFqxGgUCF^*$}lQ%W6pb*d0K;
zAq&|yt}2unC0DZ#oJ<@d*LO#%53N^ma#xc+edY%QWHSmsZ{}jxZS5XOrm#vSUfI4m
z*V4i~883e2hHcb#{dOk|buY*O?qKuf*ZTS*>nM}axf!y?lOi4DbSwi#NWVGg1L?PC
z*fPEJhY{39jxqEiBnKCZ39aJ68hJYt
zbiRPjkKevs?_JB6+o2W$fl2@QJR#*^=V`HS};NC`c!)?+$a0_;r)F4%$db@
zWHpGrx8A(bPhS1b$<^+e6x}8Sey!9L97cul{oA+G5rhzcV3&&hRd3JF)yO0;k%loF
zYbHv9_A*TZZ)7)GA8G|7np
zl?;i6ya{-y)JG;JpBc+e#};ucbL6W_-Ybip63sjBs@kkKC^~2u5?|6(_$}mC5Q-xr
zA`lhwrTAlduBmeq$*kX6n2y1py2yLty8(jO6z+4DG2~$9WAc6W@N#g5iqcw%6DSgRfgWK^0x~g;kbX^&-t8SYz)j9^9`hlWD>bE94a&h%@J&qPZ0Z0
zZM0hOb!xpRUPp4G8jz0QPJb(F5m)k#-V{m~ECrM=a`)tr)K<1eSpu@uAnyaWQ}EID
z9UePFS~@15y-(BFM^juqOCu>f5Skfh0Vx_!8SYm=!@Ph1sq>c@kTK&j&+atUg8&Bn
z0Wbsn5LfN(Cb!MZ&@ce)VD?{mag0K?=>0mH3r?Tj2U8auLwjp`1qB6Rb5?2u7YDRb
zX15&MVLyM4zNPz*Bk1JK^9udCZIwQ%sr`X$MV0NB^_zVeFz`GB49=Xpb)9r4Wgs*M(NZ3
zmzT`K8LsJPY60?h*v3ZoBP;!)ciELo3?Hl7PnP%DbN>9Y=4|J_R1Wfmsuz#ZE73Z~
zzVHfuS9HqnN-GEnBq8sF6DE|=|
zF99cy;ZdL7Da0uI?t!Q%>5w%PPK^Xxd&y}S1Me*U|wF4FThHijt;
zOg7`N(O~6SSVk8FDod`@f&17E(b9ep?_t+seo-f+NGhU%I3m#5XAi1zb!`fFn=VyS
z;yn0A5B-m_cYBnZFlmOH3E6w^{S7ZgS6(^N`6$pfzQMnxd)UC89husXtIY{S+3<1QHyC8Kjyisvk5MvWw
zFs{2St=)N7-n#QP+k5Yq1V+UekQL-)q(?1G@Rj~#)bzfxb|f7k(fQ~h-REu8
zo)Eo0Yb=GF+Oi|Y!xCRxk{T{w&c17YC3@5Bf`xSuEm25lQjSkcOr4Ccs9=go6;4gb
z6bNZKl`EITZIo%&+s4`47^#Yox2V3|dvq?Fl4cQkOr_`GZe6>sYisE|a&dgmogNZX
zt*B~>{GsI&Ivbv+dBjDQC&8d4@>OT61a1Tvi-~bhP0@Wi(cB1P?+D{GQ|DSNhq(}z
zGj8n54?y&Lu>a8)p)5waB#NcJ7Jy@A<;Zv(*UY3PEK|lhNo_^$7W~F~Dv7(j)H7Vh
z2x;-+zNP(SeI@@Hg1Uq-2J6s|T^Jh5EaVBC!S^E!#ZZU}i!{zJBPb<5_w?zI3__+$
z&6DV69Fs3~x7%517w3HLJKO;_+UA=p9MN&@@Jfi@ki=w&Phq>mz1vi1W|Xo5Ryrs#
zECmD!g)DOI$XktE4bBM&*>6f;)9{lJD;akppja`)*;MO{(T;a-rmZ|*{
zj%uh~`Wcw-B(mzLF4nN$d!2j5(4sM=p!p+6Uv}AEAO>80yuDRZCsD+G3%>@=fmf$i
zBEGJEIBf6s0V_~dIC%Z~{@cOqe>&-Z!HbQSppV(Rh~XsK#c58felMB1G%4Xx=s
znj0PJUOssE5HT$Cf0@wPo~B!_UR^`6Lqz8lySd%`r(+B;TVO>pd8C8eHQCk?5
zKxK*grZr`y;hipJgSK@YW@i^15&~aAO<{a(4!u}zM2A7??}UG~9WJ<82YvB8@)#xq
z-#*U(n!VvX?JG)qL_2
zcY7XmnO|rrwRUvL;`os
zcP)e%7_BLee`=hoY#SZa
z1eu!NGX4}Yt#j9}>)*SQ>%GBcR^;PDhSg`TT=AzhH4e_pT7*uMg{Fr)N4~cjIur{K
z)+iT_*k1lHQT#GLx6(#lc93Me@nOj*x#j#!rXvEWcmfM<8IXg(MPBH7X0_MR4>pHE
zIiHRP;9`lzm@!oRU8XHtu|nY_t<_ZPF6(JTpcCUxK$`uKPCZ6PW_TDwXBL%xWBEq9
zA6@u@1t*cerWD8IbbpYY;5^jQyLIP?4F{Fh8lz7}0{HZ4=etGHKME19#z|bRv$w|r
z5}Pnb=C5Zg`&^n*#?s&rUsMBwI&k^n^2!9-&k9W&+l5hQ(
zvE-eV<=N!TX5#&me0{TqTI#7Qs<|#p?kd`Qe?6=?U{|K*weR0!GVP2)H?vu-^FPrS1cW%!e79AOzdQs60xrrJ^>xYSYrz)Lk2y;EDlL`Fs@JiaM&}ufaUVLy3F|NfuTMYo
zVZ$QA!b+))teB-Hb8j_AoKi?Jwpi=plFw+X)NiDxCw3u7AN*J<$`77C-C*Bb#S_Yv
zeVKa~JnAPKYmjDMh7%|na+S=op_~TVKw79yeTPzoOO5FDGMToRMU!RfaqdGJ3Frzj
zQBV8u_De#yINHI>Vakb|V{_0pkN!u3a)sTdDD-jjXQq?4oPa>mO;z{{$_};yVlUG>
z#>X7A0LN2czzv}`IDp5MG8=;Hp{IT%JG=DGozFOnWShcVnKbD+LCaHp^7aE&Uo*4r
ziH}*>{mUvko-CCjl_!iibPEkg_7wA>NI|xxrWO$A_$o?6P(77m1;5FYq0*J}F8OQl
z;DQ(~0861+jx0z~5Kh4ojma<14g@S=an19w0W~3Ne=t(u&Qkc}LzR({z3RtoXhs3r
zQk{BhAznC>l>4(9`yw6;(8H0ysCfm=)p_fJLPHU)L1xF3nCFjo5utAQ{86=UnB~I1
zlUB5mYv+r;|M(FrU+Mh)X=ymfAM!B=*@<2o;2_)8eKLFrM5uUHD8?DX&*U-*2KtN%
z-Q8!=oHBo*>69Gs?Y7B`Z4OGq-tFEIpeN|f`GMI>i^f
zsj3PwUWX+CF$r%}R(BfQZT|q7m?)2fyFo7Y?2(K`CXS?qe^$b6N(N5|x!e_u9ukSEJJKsoj)sqZg&
zzeJEzR(}K3Iy(M1_$jFn;#%s!nhzf^Uro#p09)}nCP=NJT~h0G54B%oH_s<_35XqK
zbYAmpP@{zl)$8IqX4dOE-8o7U*=Oh5dI72=^)o~*o+?L|q(w!=_KCOXFHGmixSDU|
zc*5}!kn#OPaI9Jn^6!w@3ts}Hoy_Kdr~m_D0m9pm8b$K#)V=%sprC%U@9EN$gZc(&
zcUpBBRfoB%+c%fF%%8#b;0erS(+>M$EHnIkr)E_22PW*>mpp6R2+QhgL&GpuTX19p
zerb#3Qj>dKUbnu#4KcF>kq<%Q-atxOJi<07B>_B+^A-F_%Bl503Bc|Y6%_$ax!6QB
zp<+gB%+_XB@Eoe;#0K{p^dCC743;SjsQ}q%4Uj)*<)&t496Yu=zbGj&x3w+3d-qH3
z`7-@cc{dr13a_cW+z;HC@`Kw-eM89-Gj1~*P~og&q+@A$uekUKUun&WwusMj_G-Cfhhq{kx2rW2#1U$OCeVr{g2dgZ|;5>5FVI1bY
z>FFB9YAD@#nsM>*0_~@FZ(-8|9M|X1W%s`V-pI^L^!cUDs9KSe!ZQ`am)Nub4
zMxX6t)bW`y5YNWGZ{M*#J}lY|Td_iC>>~a|pG9HYX@#0Wnjq&i3sHoXAU}U%UG^f>
z{@&P*=>ywW{+*L>f+cf3@Gaw{o|um?fNp_+$Ol(dDCEwt(vkJIONzQ3zX893XVcOC
zcM(DD{3+?&G~WHaw{%tffK|UfcaA&>_l7q)m0j6MYL!APXZ8;3GSY*|KkRw{+b%|r
z7(M+EUINJ+C=Ube$gKn@6CpR}2EhgcfL(Fs>M3#xH
zVOW=EUdFVn`P00ssQl2M7kQ
z1m3o+ik-bYmPLHOd$`9ev5|_+A$CINT(U%P663bOi6S;XJ3T|g@G~O4E(9Nt
zI}xU*K3-T;d+gZUx7SwjAE@A&+NX|A$G+m=!E=Fy3XC3j8)&hHu4~!mad3p;9S?T~
ze>mP7pcR8gL%{sTGL~hYx^^#tO9gN)|DM=bTD<2LY^k}4uC=^-_%595$Woxly!#5b
zXV8Pw%9U&`lGHL`B2hqR(O>MAV(;r_xXjOQ7wdX+V@vtQd{h%j2<>B!9z85$i;QX7
z*ljgJt~Y<2^xi$9*cA|+nFR?0`@#B~CR)g;pEiI
zkBW+TpB}@9_d*o_fRoo%de&ZicKL6ZlCC&#^PO*4mgE3V;)sGHn!oql?iJ^cOClRt
z40`4l3?aQSR#^dq9oj@(Y2!r#lBQBWjy5kY_+Y?k*YmT{H1O;(W81y^1(s<%)k?kl
znpZV8d<2x`4`ARm{K#Xj5gV@eNgMT>n@}vpI@WfgB77E5VSp8c&X85Le)ZgUx+v(3
z0UMFMm!QHTl0%^9FUD#(su13(kT``rD6kAfZjfeM$vk;mzqK|mb*SIvW=sMg-jD+!
z02zzWhDQJA#tn~jRMa6tw#wm$eH&h1?xd*+_JdcJ
z(H$o4w;w(F&ftZNCtqP1Ux;Wy+B(Q?9ZPX)QBE-FLz?&(E;L#hzC99D8fdOGyE|N_
zVARIq7sj{f>#D;XFcqq;4JlotInc)PaIWh_@58=>^hRTDqBlWi@nX6bo`JX7g8^jXx$bMQ)k+
zYVSQGc<9`wfMc#N%$yHRpEOBva!(4=gw2~3taY5yajl6juFjv<`^E#MILZCMsHLLP
zp^R^VF*wiocR9bj!CgqmnAvYPi3lzz#o~!mr=aXdou6l^M18k~4p>i9Au7xeVYjDg
zR>_zrnqqZ}Q@R+~NqbF)o2xlLO6|0}=BSdhU$B+jwM#T2nLU(yuH@!M5IOkHP)t&9
z9zAjdfiBfipFVxk&fIWAEOp(8OUq
z(DV5SYbU62{nlyDGTvx^bB{fJGVh&CIepr+1uS_jEfsg~uJTV5cM^#UR9#G#*qBL0
zLA5KZ)?4`nl9≀BmM{WMY3$V-C9NW
zQMa0{bQkpyj_Cw*!dN|we4e}W9-(Dut`vgf?gItM>!90mwK?#PgvG?13{=Ta+33kj
z3j2IExH11DI{6~J5fl^^`OvTp&aL^g(9;q${$*~&xPr<-Qc~+wM!30*AC~@b3_I*Y
zL$MoeqfG`bqrfU3fDRLxHg82*v5;%8>!Y%|nepA3Gr}yB<#HIt=$iQaZX%bEonJk9
z!i0kOW0iebQjG%xeH|Aa`8Noux+HS+0Y^qTYJ0;R(qSL?65h(
zQc$D`IK})d1W-^Id-tA8@Zu7qlbpL?f&1jM9mNIJ-)P7#E~+r-(L+P&p{j!HgBFPN
zAZ~XcM-try6|8|lJqNCFk{u4-Cy@RpP5@d1jjT+NKEn=OKXf%
z^BD^MU{GLAGNWH-G_VvFSzTl|q-<<3bek&=4Cf6EAv?m)c`5hq)b#6n($iBWG>~^G
z59q2*g7zXd^&Bw*LsHNuZZkKOYFsM4Aoow3R^rWyVXL)Yt>%hzE3}p0qsOH%k%`K<
zbm@6XNiB_G*u5L9v8B()4bEnFx-*kgQXa8FAOq!NV@EqC81BI(vFo?(i<)}8;O3H+l<#;_6q2k{xF6P}xxn3oMqN_2w*6DRF=@%tI
z6>gv!*XY}J&GvT)PjFm)Nd=caLbMT(2=Si_3Wult$PwVSzGgHYK2Zc?(}tB>c}
zu3h0(1#wg-Tvcdp=Od(2@j$Eo@?{DTzP7e$RGWI8HZ?KPo#`_$IcKH2!%yd}IK)6b
z@*q6H1cj9f1vK-*g$Zm|5#tp+0!zo%00%N`g6br&Mh-ZR7(KekL(}*W
zdz)E(_>=JjWrIvCjMF!mJC#>3ru|+2W-qb#z=7!#Cd^_}PkA}6eWOE{MHXd8ev`SN
zwQ+)5)sjZVoT2=}tCa0PThNu*V@?C(V(PfVzsKmsu&|{3og+8+=t&NGt~W?gRF4Sl
z5VA3HYji{4rd&p9AZUbt`Fm{3mhf?|rQeBP=)BQC)x!BtsHM1qPcON;xbFFx=)x+V
zb)W{FC!*FlG=_K&OkMqMMy$Kw>qWx?6`OYZ;p}s?
zpiw8fxAZ0clp2~B(U36olU8CiDOI)m@Zp&MfYbpZvOL9hpl7!?E}5E(rq9{FbCXXH
zowuiK;l%K;F!ACXm)7f$AOX0It=Vr-*CZ`REE6f7FMst4fhP}|5eXGcMdqDIYq;SQ
z9x}hj8uk}Qh*u0qHSakYPM@AdAFXbR>sMK9r^lw~^FuW{ZQ7>g>x1@+Qi_xI6-oaR
z3>vbtK~~w!=9>R4yMcT7e#vE8H@=ARn5a*0?%W|IX#D#QRx64Cyv`c?m?j1;EoUIs
z5+>CxEAivn%|5=vZKM6Ln@}`3bFRmeYC=cdDSAa8$=x%;5dALf_}P>Zpm+!vcO@5
zI*wC99~+x4Bc8D${jl#b44{KgVFSaT^6tZ0oumN{4nU}68AvnyiMzdvs=w8iEDv2*
zys7(~z`(I|S`#O3Q0Q{>5@Cur=VK0s=SzLP&fEoIGe5k4pJ*GPB6xs*7-Vb|-vL2?594{a(P4S!UI#sOfC=AY%+eA*i#2t_E>Ep>QCf`*gC7nxfyiYP
zIV0M+87B)ai^d`0GMUU?CO8b}>LMBt6r<&fu?XO?=ZCd_eKm0LjUeP}=X>ONm_*Aq
zhF^m>aO%`TFni8f+E(SJ&H=X>RPqDMw}q2y0TBppSFQ}cc@Gi_<~_0Qd3SqjE2nwZ
zefm!A}QM%&#=-|72S75dPb;8jEzav!!H|H>%xwn;mVuQTdATzva#+%w>MR?c90n)T>#Mr+&U&(=g!Bwde0w
zr8mEsq$P9q%lbdfPWy&kYkV+t=@q>IOslI
zjNv9v?wTOqq+^=mth>R}lr@-eB+Fw;-%lZWW~dk1D@@&p`=HTZ^(W}Ij~_V_Ul+C3
zDz@e!T~S!s*m?6TpYyH?U={lD>
zAX=&2c}&3bo}WH*MpN}bKp%hwk~#njKa;Nr^23;rK7mjN97)fR`a)at%Ciu}#;^%)
zeP5dnt6}Qv|A2xoj94o-zv>lhrj^Jr2)5dIiwiJ#diPdR>Fg%Hw-dxFF3z3b;d4Ui
zu#?fUbkAc03yc9wOwSj3VMG7ri)?H=q;t{GeEC`P=G}>ByYpE_ec6G9c|A^^Kflc;
zO>JsZXmBvx+D5pZPTuCC%;s+1#%UpT{>*Bh&Bbh0mHq^r6m~zw(b`?ghRXoW_dRZj
zEp<-SR!-`fP3<}$=%#4!{sTLAB3ixlHriQUkgUCay|1_$rU^H)@t3A|3%#PGk6aH$
z7WW}(#t^Z~@zU-tJ5F#+F(Dl+8A3uLqP9-jx=CtQ~8
z{5$NptnT3apTpXY1>a22Bu~~(v>%@h6i+b7dfFKMOE#1l7o%8Vk>=_f@cGFl1ep&t
zH6WXqTS`gd*45HQV-VrEw#l|)f@}GywjA4`HHod|_vGrE2
zj8lh>hN0%s2|qV9buqG=TYaT-RMh(yySqvz9W3qV)BDZ}QMzXGQyqWhX48!LwmJQ7
zsD>LqbL}|d<~FYRW$*LHB)M@`o7eWwcF&$B{yfPrc%^CBX-FjUdA!(kG9drG{#fQ=B>^)NFqI0ny
zd0C{@;vW25^02Cxl2Rh=Qazewu0KR;J4{WVKOY8VR<|p+l{c{uEPU@?c9pSZ54=Jt
zEj}beE`Iar7whbs8nWI_RI3P4`OrgW({8t)GFIkJ7e9^qE;ZQ#;$|4_!C@rv9ZN4q)@67~BPpgWoyd~~Ermrx*G*&Zc#(8m(`q_lc
z?vXPu8%jHjA3GM-<`eK@!)8(X!umObl(l2|FCDGACd1fz4PUKkt?5yv(f7>CpE%9ZPB+zjqZ0aWOUKmO_@>oSR<9n8sMDA!
zYCCFpuf2C_^Eno!NJ3nUg{{5ZMELKr6Gy9NjPIygv}l&1Pgq_>x5kfdQkG@Q)pIH<
zrhlBF7ZoAfw()m|aZ&BVWvUsKxeF8`j`sxv;QUmEHKiyW%{>#I?p7P8V3wNcN&crU
zLl%5bB_Ev{=i~Id1cjJ~hxsN`s?`y01Cf(P~cr24smwDU9_$_m=^KzW*
z>l?jKmAtWjeLJm};)dx8-mDu_VddI_qV`*~(##xe;;g
zCduC}jm(}A6)UgQLCakc(J|EC{&z&$kjI8wWcp9`|p4FZ%IwfzId_y
zYEY!2{H_1~)AQ1fMe~}*44j#&FgI0Ugec1D-LN+sJJ!inE1cDDFIrvdvd8P^AH$<<
zW6BKIXguoBk(vB0!pBF|`jJbwLmghmZAZG>ddAp!#yshxcIB{Ya#QrY>|ueYb80^>
z?-KJfDJ1o};-?6ew)E|tW0IDoDtyY_;@NS)JcHl-|Nm}TGa}@HuI8fl>vpv>`w4$q
zhrMy)X5~)an|tlrq!zbRWOl5RWy1Z60g=}N%k#FsRc;^eG~&i@(eJEF)}oW6viD{w
zp6s~ep4p@GOaIJ4x@$~yAGY6H)UnsXE=M#^=lxCjtn-xBDoP?-c?&K;-_2K?~cJc+h>?*CazVSo7z6VcFsAEj^WSTJ8pHkI3&wG
zOC)WbmEogl^>efAiPqwYj~@Q)AMw1ry<*$6Tp7thwLdfDGvk6(HEXY=KW!I6QT~>I
ze_Ilb+dsxAikC|NcE)@~pkFWlZzGqKm^rY?@|IT6R}{xI$0ePJ6cH4y!KxxME25yk@QZ
znJVp8%Qx1`_n7iy-Kk3ll%I7>S{-%g`?~RN`O+!L0WP5*E~j_SS$A>kqMVkBC+mYu
z+>Ea+IAbaaYW;C@*4uMAFs;(cwW{qs!X=-0fW1$jp&
z*u@$+pO-my>8?BFGuArLb*Wo19Z9+c*1EdAbrqR<@L8%5G$0}K=0S)}XZB2c5|wVQ
zy^+q5lK0A$)c`_hPkKbO=FdJy&Ai(itHg`h*`N(z7YwC@`8dPbUY!J&6()=Ld~xF_
zm}*>4HU42}O$l=L+_}Fe)KZF>tAM0EV2UEiq;YD+NSlD+qp|h&`E%zWO@CZ+8C+ao
zL;bgBgHoNzPT_aOI&+juXa|N2VXX%|DgbxEql{gnW@i6fyQUYYi4GZ10=6Kg%_sn=
zaN2|CJY~oYM?IP$EM*#PlcEhR$JLx1z9h*u!)Bv_FWy3#nSqS%0s@f9Le4}-Jbm1_
z7$>Y|AstY7K_$VGUvM{=vwytB(|h-lQ&f0QJVbDD*za`C3{a!`Qu}r^Rj0tdjJ+Z_
zkk6g_qO9y2_#)q@ah-ZQUS%5#tEk>8uh{ZanOQ7YfF#W4KX>5Y03U3={CR^BGm|wu
z)EFI+jaIK7JTWR5U;)g;%uJ_84+#G%A*w70*|G&iZK{zg#SWGF^zq|k*F&6O%7vqn
zFkz*oVlwi#cZi_S)zf2omH7bk8B0&bDtrFp8HMAzKs`0A|DP?MrqDLf4FXVL{lk+y
zakc`_0hq&+CpSX|<}1iKKAEz=_Qr6$E$izS=GeyIf0~#`@5Pe@OF^2y7JqiQNj2KL
zy@C6eF)LxcE8homvEAMtYKB^gd?ylU<5^A+&eVvb63bq{wp@L4y?(w(&oLatunbOZ
z?bkcehfA-B)YL!+D7By4Nc?@Q@M!=%=L_R!h;{;uc2Am7O~G%MmOrJr(ko2%qm&eF
z_hPhhvb2j_51HriHq!?!HN%wU4{@A*Lr$OT>sgw!tl{m@vKg^6FuDU(!MK9Ln;IQ2
zVs?@K{dX@mLcDB5%_J%`IkXB`F2`9sK)%AU!t5BC9Or}qb9V1-2soHlVk1#E!*CHO
z4^#9l|eNbVyllI~d?gWdiP{-?{U05X%UUOWzmyZIH?YcZ4;+NL_KAt^mho;I=VkdEQ>2;%?{dR})9?<@>4wnfHM@j&OoI~Wo
z&Bh$Cw6<>N$Wrx);!2sO()Llow|z2k4uKhHu^sHS5eSHZ2F}gy-
zWf3d9BZZ$p!B06FSsKy$O)(?MDNt+c?I({OX-Q}PZ>b9fJs2nVhnZbWc2If@V|Uhr
zapvjMBFEIm0q!$E;Kr_4@tb2z->s;cNfE)SC&+rm*xvMQCSN$ckS{R^;R3K_kdfUrDMLJ?aQPQDZ_w}8Ci9W`S^9c<9?7~4
zd>eV&NRBOI5ghKrH7(5(OOvMz$7y$N0-Xy2fChsA<1}~%tPa6es+@y9X%fTiYxsUErj07G#$TheACcjb?E};2X+H~USCzkjbT65!{C0CPWm4{eq6AEvfep_
z2|PF|Q!0Kb$ohivY!_}WwLTfWu|}{`T9n&15!9=&>e{tywaZqsa_!lSgyQN^h^qPU
zY?mf#8(YNgq_<>a65FCCj~iF`ehFU@8&mcLssC6B=w4MA4|K)eXXfn~CsH6Y4Zrv3
zQ9pb~9%<}>wTYMw%c{ynjs_h#88F8OZD9z@r$({fbL0*oN!r!LR1@hq50s01xl-lHq%PL?PUVOmSu%Eb?(zDT@;JQ};
zaS7{x?j$mLa~4$XbGxB*tNI(m6UGl90i;_V5IbmET$p%>!U}+H13vQt!W4AQVCZUz5`=x@
z4EYD+LB((%BUMDf*itOOaF9XXtkk-kUS_1bfHYkdWpch1JgaLBl#lFn1IuG+LD
z$^Jc-UeUH@E*o=bQVHlLk5Qv*KaS)>ghYH%X!4N<LSbSOL`mfmpNc(OppNP8ag*Zunp;5oq+
zu-Q(Y{0jM7Aj$3DPkaVbQrqh^_{;zGbmjpyu5H_IHEWh63acbZNKy$6t0Z#>vC&`(
zQRXC?Rftk#t`KD`La3BVks))YWQY)oBvF08+w*~GW`KL9m_79pZ=w!%J2DX)QC;3r
z5|)8LRmy{pP)>Lv7qhZ>+bfqW>9@TUWnhBYVew=bv!o{kJ8|m+w=-xfAB}Q1aYq$n~JR@QnBqY|(#J9TJ*t
z7u1}I8XNUZ#mxUWRt#22#Rw+at^GO+MkZS?q}g$MEf&EoGV(2|7_{c3@f<6#)Y
z)6->W3>aL17N9+V>@JHzkWMGFMeyF#)^;O>PW@46AH1c%;|=2Z{(>HdrzeXu%pqw=
zTo2I6P^3Zc!}9H!Bmnc4caW3dRTd%T_k^`bD-OZ6qkZ`)5($g^{yBC7C1h+)>->PgsShcKDvZ^$SQ@
z3?NhYxtaWA5rq#9_@Ri?OC(_oQmYqiJ)g>#WH8IaR_SW))Y*}2
zS1^%(L&r)jQWAQpx#i~x4*wdL$T13VY|%74luR3doQvS}*FGD+Ls;23y{knXzURYN
zxw886wB%}MdIF)?QdrcQLSe#7rs9V4!U;E>i$W_jBsMZceGtHS1bcbo>@4q*3l_X5
z>8Erlc*TbYzvN6X-1S+^Fl>`b*RXD8n9T{OvvQYf!9-4gTIgW%4-ju@Wlzk0AUp_W
z$t;$1@WX4(8p0tQ^6LxvWh!2DtFFEFC)e=Ep5?8#%KiB58)YlfHj3@49Xbe^pi%kC
zS|zoIK&j+QIG&1Im!m90IbpJsyNp)_Bh#Tv9-BVg%}NXjX|tzFKt4(ZNFb7c0%|{e
zNZP+&<+?w!ZHD-;3PrlR_wF_7)5j0k+>7~sevHMN
zk-ngy_KAU!{XgPmOwns-1@wbF)3IrjSv5McF^)<@uP==xBqsSz$5>9Fl!C(0Ry6N(M6(McIqvTnr1Zym9i;2wbwYc22YrbreUM)`v2D%h&wc
z@F~ID@+BUBo;u#~(9fTvQXSO$(EBPg)eAY*4-ea9eNaMZn`hOjQ!gBE^TLnYMVw0T
zs}Gi24;M42rEw88>eX#|e}eN2v^uD&wtl(zYU8xq!sYDL33{zT&VZKT(>i=fjCQW#
z$-)yrwZ}q{H2r`9H@b~!BC9S%5Slc4_t(A
z&o6F=QlQFg#PG1%PP3~EH8Yp
z!0*<;^uL||4zz$V_km6p$+M0(s|>8CxYqtOnjkkas-pe7gAZG%l*M>9glsQBU;-^5
zk;HKyRxFBs{3?PW1Ocz2O`-^SMNu}9opUXWZ9Ak=vcNdLkV2xTM0Rk-EGy=9>
zaKjr!!j(*~+r~^dHdkM8E#gbV$?5PCZ7YYZ{rmL!1?3mzgvMjf$Z|+T0lo6rk#P>S
zR|J-hiE+?qfqldO1BS9eQj3cC@C_|ckUe-j$Q4XewfA9RVc~Qbx)$%H-peg{+pD=2
zZAm}|+n{F8xpTwk&{wn|RzcIG0t@OPP`Bshk*3BDimX=djf*pb1gtZel8j_TuV<8y
z0fUYO96oB;W#~L_D2+mL7_CtP5icL;&P|c=pEu9&mhu~aLKl2pv!;U43-QJTpQS7d
zP1wdWFg?w!hAgMr(^hXtkM;SVmkJ_GJLBS{{^F1fFIxxzGy&J>;O0}I*LO1ef*@P<
zRV0ewr)64TH2gmhNyYf(KbO3f?B~|MzZR)XF;9pyoq0|9s)CN%ZE^}jda3B;*0Uh1
zY$=gmU5%9=kO!rt=)xibRi-ghf6^-1yIzeuiF+s6g{2!MHqswGehl`p`v~m^phm5|
z-|)hk6=;=HRYS(7z*9XycZX60*-+B<1{&X!Cl>*tMvUlw$_8xdA-2zm$^Y|$;n*+p
z@sTPuln!;BF$1XyRz?RH8DTz*mYC8$4_HGyXWkrqnSkyBD}b(a`gB&xz_O5I9gaZN
z$w*!uEXj!zg;rpd4FX3^d=|QUGjnD)glVxxRxga2`{+!lc<^8fLUp03j6&j#+P^$|Hd8~`o#bd;qApsnz$ho--p5t6=sbhEF-6x$
z7Wgx)_JwB2*-7bz{|tXM>#LcoZyy{ntbhM6gjrXavR}-^rP`IvX{#m4b>_138sT@ie^#T2F&==O!&&bT}B~cqGbtY?Yw0R1+C)G{E`A&Uf
z)(uBQQMzI765B;hS2nhI9kM|Z?^Q_AO3+Q1DXfr(<;A;bjgJ;oE7s5T
z^ZjeqGt?9l3sL)jwoRM5zyu{E1uCEV6uWWE8PUOxtW3|#sa7`I1U5vaC^aGT>Y;L9
zP72-%nbO*A;)EOGFOyC<$95y7t|EV#9iz)8q20Mcf(o_WU02ZX4&C?5b2giMt$qxcU9q^bMLzi+!9k%M^BdP?=7jo^*Ga
z%8znSGl(xDrE6#MhHM&`tMT{mB8zUjK6@q_qSa7boFeg^TVa4G>=f4f6RYZNUsZm0
zs3mf0Z;^8xwET?|Ft}{$>)lKoGY%gPXWlV6>^HI!kc!YHf`nImTJk~Bp_ZgYx{|&^
zrKuEI0btKzYEAqEfZ#4zB*1puE-eFo#`Vk8-~i8bT3WFYvJ8~fQBF?cBW$5_?BGR0
z;HNS|qSxGf$C$LR-Y45NF4M*e8HS#a?2ni9UnT?^_4JGYjKAM^QBIw7pt@AZFHF}V
z)Jg}T?nh?~N&o;;oro|LDS!^m2FlgFQRWVcBLLK3(uW8i6p}yP9cP3l2Z)tF#C61L
zi)h5(Uv3iPbo16N{D(H3+v%g}mU#HoIn16W*bvvuV*Kxi&RFB`uY=GSw;ORn`v=9|
zS%mqKLDA(k7z6(Xm}D@
zN^;_CK94;p!dCXaQ#u(8uj@4^*5}DO*=N+|u;3Zz6IwulA*wh#%UABra68T_1jgNf
zBKtqaf7g|2Btd~@R3}Q`WrrIjhfsUe^&P0DInjQnWD8-8_`S5UKpvi{4bPpmju
zqq|bu$J!UX65Dz8Z@1@qlD
zqz4gI7xi=-kaPt%O6@I=@Yi1P&V>V{6uD#HKILnP_Tp@qrudC?RP62x*|r^GAh?<5
z5G(ODK+G&l1^<-@S_6hqInmenT0TcCw$KytEAOGY!}TzFY`(6&I=Q^&&OWQ{P6nZ`
zhUdsbT|v{9N>`=3jQ}qE(}6iQvCEJOqcF>7G^y+<@f>Bl3~`DcMAHMJ#$Hu!ZV+3A
z=1onFjTV-c>`RXwJ-YB`I0++}-91$J*e!~`-myH*Y!ej(`<=vC<-aWrZ%6Sc0BO!W
zt?Ss!+5Nx&>;fAV&6Xnuw`kEva5s`#eZXH}Lf()k*!NeUS0o{--@SYI;KAV4pWGOT
zKvULJv*OVxMKU4kf+E~&e@q5&#GUEpCi{5kZlj8ym#GSIg
z&e73w?l`WjByOvG4&$(-q=rIMqc-yLC1|F6&w|(5o7O^a;vzB-3%M?^r+`c8gXm@f
z{DHoZoas~MQLn^To-}RR^=`M;_tN~>o>m%uHC2GK;U9tEQUJ^S5*418H!WS*TP8*%
z(k&PXUO_V~|L3Y}*qMQ|a_Tu`hMJ3Dp8(wm%jTeCp6&&AE+CV|-hhIF!mG=x`$6lz
z;O_2CCo3EaLM>XkAPY2>3U4IH9Z}8m$G0Q&rl?r(RXEwu2Z?Yrf4oY*@u*QzH|9ys
zb1|8sp`EqvZ*x{ul)|eyfW`v5m)bQhE{^h*?q1Jc-nta+_MUoKz%GIn4LB?8z@|kx
zGB0}7Mn=!Pf(ZvvnP<~X>=86R`^wHmy&H1^Za+OAc3Zb8?4%}2ztDrD`9GPp^*CVj
z%mvlg^`4_B8ovWUTnpAnU*IK72iStL@M1O*lOuBEj>LIc?%FXVizZX#17xS8I~YXNd8yvOsl|v031xQ
zzGaE3uetdO`1U?g`07FZ_xhjgFudZisji;c7&ov_&r?D51tt6yjF(u})|f#fl=735
zMJ6k-EE_8^E}TgJp{}mZ20fRVKYoQ}C*;50czOO2UYwlVOS)UJSju8wK^2oY9(oMm
zxsp;K?vFrzUbB6cECHj@sMH&v3bu&@DkRCe=JOPF90}R0XbpK~
zq5I<#6VabUGd2OW9btHzY}a0U@30U*emtkF#QzB1WH}EoP9(Buqr9ag7v~XLS@#5-
zD!8>}e1$oJeYDn~_79g2NTM=`e0k1ObpPWjtWY?GXhAchKG@UnVV8*-1F%&i6xDsy
zTC!zVTwF71Z?E#!uVu@DUcON)h_0`0B3rd9!W!S-Ry-|eNs`z8ndf!4_J9R4d=n@n
zQs~P06R2d}i_e3=)1+cRAk3Rt`M?g6twj2aGSxA&W(6T(2_7;6;Sy54=)J+J^$d9!
zH6kk9%w_H`fE5E;c=(B4$~g>{c5O^psi4PY&IKJOzwT;t=IDV!=Q+jQr3q
zW0ScY7xhqwIv$y!<_+9Hn$(}2Tc}>v)xrl)vH)HzzvcJvZ<+t(Z>guGQsK;ML?G5D
zl}gE=E$MY68{ZP>d#M_MudhW?sIVD$NZ%sXnXXTmYvfu2t^mU_&O(qRG!@NPENhvw
z3J!3G56|_~0QtVhDU|4EGn~G)I{C31C4p&#v&(4X2zk{ju8OL27WsT(;HS8c>bu|;
zj8S&%Pdp3%#f=Q&vV#aJsrlL4+am!8hLbs_iRzXUiYAo9KvWj*@%kyKoZq)O`snKJ
zqt+I9`C%oVD;3`1d?5%{(?_y5%g~sm5!NBs(YG^*fLxEC3n$pDT1G7<%5a!J8g_wF
zzeDVJ3XEMRo$XO!xLKpwY=_45tzzMaP}PChfMXKxL3ycj#YbmDe?yv%7YVjC#my}{
zGZQ=BiK9n{h7qMd_*;W$!xWF(An#C1yu7`CXeV*DPvS_;d+R)Vo-m*qiTI^sH-qV_}gcwgYMXivT7F}Fvslu(2bgfsi
zc()Wyh!-;_1F(brWF<$+B+9Ao>%EH9l;|iYL6V$&;lIOylCl
zI{*0f>uPX&8*!Z;GxLL}R;}?GN2{o)m!aViP`$tg9ubQR1ZP1(clU1KkVfLDPd3c*
z?X4+1WS%(~2YWT4_Q6ecduUItUTw#MN8i2Mp_}w4tVLqqvEmZMV4$D|Lek2WRFDw4
zuy$|}`7K^7P-C)3K-?E!M>;#%%PO
z$$Vok7%~}Lm#TL#N+2+byDcR7^74jQS{}ruCDs|CqxP1c{oD`XmvSirK9jQf=iS6X)RvpU}+2~(~ceENiKVq-N9SL2ivn}27^UT2U-o!tE#>-
z+kik^xtxoJbT{im2vTnxG$@Ok%m&xM*_XA9X3}R+?XY4nyvH|!FymczAX}e$j8ydM
zII=oZ06Tm1XrmqsS>`_h!Q@8|9u!Su&cL8H#!Uv$`F|D8Uk)CfBc*Vh&FHC9J8O6`
zU376N18fUXWypkv11Km?0qc0BoNUDH1|;dCpt=F&6uiEK^+&8J**OO>AxfbM!q{d6
zk(p)=BZ016Z@l(r2*KXkn=fDb+t|edK2WtI-T@E7$ti;PbMfL@(yG6|SyK7WhK4=V
zGr$!HLmkCi85aV`5U6ftJ@w@ZFk#9|Pp<_!q7@`V0=v%hMLV1VC}4OM!-&E@%2^jf
zLmdE+K-%M71OqdA_Wc$d5Fm76mA|V+!Tc`ei_l9f+1#1V7JP+z6Pc9nO-(59-a;D%
z&?6M3tGqtrc5AR!Xb>+L{JFWU0zkCw`qr9N;pcwD*oUPP(LkgGEv4B%X4ZWkO@Q5F8m2iEES+AToTu%IUof4D5qM+6w#L
zF`Z)FP^hvjLymJ;;5C6YyNnyRb=$U0f3+AnQdt5pfgWk-%;In{Zgp8H_h`pP4I~oU
zQh4x<3FptR#$o>5pP;|ZgXSrO^gg$-HIQ9bf;63F`(;--C>smTf%4|?y8h`;VV7w{
z5f||F%a>LT1zj`dA4-wJ7|VZ&WQhYpIS=G}pL6@(_v0Paoj=s1-r??b+V^uSok
zd}VM=Qj4l10TSCenH`3x(ZRS6xmd;l1)(6TW5PGWK`{^4)k%k)(brN+9pdlUE8zw`
zJ~M2H0~$U1NlSQvJYmT#yBMcugblI_7oWpRupxoNPC_Uz=uL5l`kyi-3Kul7B6XcKiDE_wX-ROs>1#Q?y|uneN=VPH{URib;gO;eUf^UN`8B
zyP|53$%-a}_l%>8`N_S6`hZLmz?ubzofV6N!W!%Q_m);x98t&wo9xA45e)|(@eov1
zR6Lh@|4_%t*_$j29k{yl*30smhqJQcX>k$p1bCn-p@o$HHT#}%HwT}}5TS2si>W-a
zR!^<;p9yJXL_`6cErBdWX@MIbE^Z~HLz(~}+!mQ<>wD!FsXcxaHYRQHqDB0*BD^0r1@OO7IAw49k-c$!O+@Wn8yuH9~eUmTmor0XIqT$!CZ|uP7=Vm%8?FeK7
z$<2#%6@4Cq)-k>}>`ZY3nTcNwZzRyrxHbUMP(c7?!hD%QXFtyzc#wg(oz=^i&j2K5WBgzu;9c)?ynXA1ER;0)q1^}b;o&-(yNQ2_xtv@{4U{_MQUjQYpMrkk6t
z*{SN6PwwBB-t$51aftRI)ZTT<6gq#Y=V9m1|Ab^P?WFr8K00kuvW&lH05i^gxcw!z#`OF5M>|zMK|1_y%?b*T)IkQjkLTr~A+QK;
zB*mwK>NftiYMruMf_sH9B$loRN+Dj@3>}I|J65POU{12y9Cu4*xMnC>g4vm8X!wJ^
zJF0=QNl@`Schc3lFyye_%J-CV=N1xjieCFu1r)tk3suQt+z;A8cn|pi&JCXEPDM?t
z&G`hH1O>{=&BeH<@R=orM)vB~#BJJR#_zQd?CDT7xnIVMW|IrAdRAItc!VY5QZV@D
zd}(bRdmUnez=!~ps>X78KdPG9g$xNw-{MFiz0v#JQ~J{GLl(OQ1>rJ!;>?+3>Ndm^
zskER;QV4OATOKqgPn9_Kq@Bl_jV{?DabDX4ZhDWAVAlB$I7r_C!+JQ7p6I&2@%wH&DVrooP=r!
zr6XcFj8FCLk^Q1F5IO{?8=MX<1VIC}u<)m)XViQGJ8>{
zcUQZz!T;E%<*vrH7JeHavHF~hyj7zFG0}79R#HE*P>q!o7>{5s4A?YVcqx+|GFD*&L_pL(pkf9xR?0=y3a*Im*oz8g;qARFN>7Jz
zgv`NEhZ9JP1}z7@iLsbe6z_nK65DOaW+wv&i7gr3Ud+wEefRD{GaWh{t~PH5
zoWp9KzaCW~!%#|1exc=~&cHyI^?y1zlW45KI}y1NiA40}5xEWwF+GMJpKIa=o^95
zpFe%NE-;V{LJKRa&!0bY`JsyxJ@%V7&w`c&DC?aa(w8k*O=T8)cW%sF`@HgB^G$ok
z0V!*^lAaEJzfEV*HGCN9D+?wCX$0~-QH6PSVX3ZOsI2}fhNn%>BCEs}2PR-JQ>~r9
zqGpVI_;Lo&sM=xcBAd$C1DI=snPUH5y%3e4a?KZZ>Xq!lOI!^e_*l5sG%5(N3eP4Y
zkurthI_0NeCj|HjwTG)No61n|@}4mDz7BH{YN8$zwiPWb-rqT(tbw~rxo1yEQ&l`(
zXfOPRMHbIUO~9BscvMOI)1R>2v;)bA1z-sfw{FqG;`(y>^aQLhQhJR8OtaWh@~O7g
z)y=K(`}fQKuO-MZv5>L<(MZ1*cf|uqNuYm2^Z7sno`Co8V6nP1ZtodtpMK^Hwm?0I
zRcPXaxe91p)uxs+ogyXiyLPe9yYSpo$&@R~O=_8g=4Cl+*Z_jr+EzV%IzuAi5=xVhwp*bk*+_kY5^aF_X2jYMQo8HBNYJ*XAP?wwo@99cre|Ha^%d!3K>K
ztz}CzY<$YpWevk{d4Qe)eIcMDa~vu^KCqmu>?{o4q*eGTX%K3WhX#W
z{;S@-RS)X}WI>hH)X>0ILHLUrLK^Rj4jw8UdMm2x>fDWCPOkRL8Q@h7bPw}jtX6dG
z8erfgwZVzkY_TGy-@qcCFG6qI(A4yPic^Q)?#US1Ry=upmEQ2lFHd&
zgh44tTqr4hudi3+^O2b5UF6CTJ?QY!IOb0QLo?fFET%Mb#*70nk^!1PCF+0vjE#=2
zZ)iY1MsBCm%9ZEP=})-9K(n`oB3H#{DKl3%;FB}lKx%CcsQns%)O@D9X~-1fCNMKB
zU*w-SY5z)FHx#Gc;XP|3J8>I<|7B<+ifb47P!*)l;YgA`3fCVnyJ
zqYD>oqcbhz%cYu?zMq<}y=5x9DOFW>{_)VDXk}eJ6sKU@C&>CDc7J8Tim_bM8+Yzt
zEU!|#lNOut3NR(q6l9_1&OOY#&mEdMBlGN86a3Fdj}`?R`oH3I5n&V)aJ0E7pf+;$h8~h^ICUy5k!PWhA
zL`WXGkCyvpvvGON`VZqfJsqZ~~Y1!hov5$qNOB)H9H+VCE);3ewnUSYr5d=GrF*0$F5_SaLkMfe8syX88-
zoRPWN*S9w%PUWD`rQJR60^L)e0Q#y8QygdE|6a)K`}d1|6u<1C2;?}Y9|zQym6i2<
zdHGR!x$?Cp$WWcMwby_ZR#!4p@U6loeXghN+f9bb1r?!#fbWCU3J8+cK>4qQzLtMTy#!uD>oWnN&08OJO>J1PMs+L;W%<{cg}d;o
zpK4FD+Po|-F%fY`QoD?Cj%eF8Z57LRgaJl&u)-Z{7X=1hrW0jQIx9xnF*b5(`XAa|
zmOG4%6%X}~^Ba1kgOpY!AFv&|U_=(+l<
zcoH-l9V0N;`j3W(;V#qFjuCiAr$YTFVvFn?7iilmsw&LYm_fKCEQX%3U$zaY8hT->
zogWUho$N5L#HCS9+z8wxlKk40q?4ir86qCs<^OsCf%C~~kpCY5*3rUA%};kdBCBrK
zDh@~oD%Y^#S%C>M2!VyeOQx8R9xYIE;d?XeJJjz04hcjX*C~YqAKksnmYw{`UYSXu
zqkFC~*03g}@ccb@iem0fd~%HNh+s**gHN*{RKRbBvNd#b(MqZ-YF1(F%5EA%9=;sR
zd!#O|zS(T;sTsXWW1PNd;~%1(`?Meq1C6W0aKMSX`>X^~2@MY65@PS|1u~1FpoA>3
zrj&4>G|7vV!P2+9eFyEVr*%9{DAM~!tk+gfjdAh@F;BQ5OaK7D;j^9exTly9aa*d;
zO2Km~PLjB1Mk&9Dt74NU#`*B6Ie9Y?-~gic@2-4cf
zdxMsOII{Z*m5EkMTm+G^i(Y5Xn_h#m+r*4Z5shNb1T9*2d+CFZt1S0n1eS!qY5)G1
zzb>~Apq3&)(Gla!`IMS%x3u_^ytU>R1G9eISh+K)U}(0^4)JL$h`s>Hr5J}pzSX>Q
zhiy7J$*&ulfutaD3))S8UjZgyfae}(x$Fg`NDL>*xqJptBH*2@b5h1mW`jCR{_*a^
zhn{HokzFXGnX=P}9YvIm+R4*XTwekF4POthg!l#B(oPrJOe&Dj(*3D_9gc|0C)3=z
zZ{H>w#DkNAp%CHL0;^-90aEIn)X5}lX`iHW2t;@-{$%D=-9tK2OB6Jd;{0!GD{SqK
zbMaVse(@V6AzcI6nYJ21RE?`jO68z~ta7qi1%t^TK7fHefsMNcB248M5gEya>yu{ac|DSMigr;``GiyON6iPi&j?V|3F>^iKK^ngm
z=u=@jh)W!fM=XTU)Cz45)X-B|d?D+@RwCXUSPw%W)>9u;oT;HN3|eELlIVC@R&V$l~6UplH8xApr-Jl$VuZ=nJ@8
zpBw3R-M_15q$Ks8Pf*o(yRn&zAi8uNNv4|RO6esK-BO~bTTJ>Q)hE+z&
z_faS~;Ib}iRdCSn*pM0>6?w1*U*CwEsRub|nXkjgydkSr?+h6cp&k-<0^*47tS2h5
zveDy?J)n1I4~GJJo5A1LLyVWfMx)G{GDSvmu;_KjN`);Y0C_tApc4{SBGq0DZb0fQ
zkSOAXnXIlo0h3-EGq(g&=Hkw
zk*SvpfCCMnRd^b(zaq30XU(G5f*LAHIs1s?eAJX>R4W77NjY^Ya(tHv%W($PixiWn
zeo_{HoM>Qd5br*FwyMcc>-mL)bljf!TIz1K#5&toG300K-@jNGGsxtmWK*O?M@PfX
zy;m2jc3o=TpSx9nh#KwNcm5@x8J`%>gUEUAFwp+VU-h0LD3>t@Kn9$C2Ka$5K>5GA
zwPm&EP=GhKf#ck0vfbx)G1xuD)Q3V~2iIivYI^?{?A$~Zb%a%Grc--|Ym|dBASGF0
ztG~OJXsfK_a*AjeI|7mM=FMvjCyyL?N2TL$NI4VO@Fo&)S7(Ha;p8=IE~J>2$h*I3
zXr_FlbM9{K?@5}{Qku``0GN!QDOY2w)wRG$s?u!Gpu(z6k+D7_yDJ=xoGP_X>+wp>
z-u9hO_6&QLa*auOKv&lx*6&dWbDSm5(m6%!tFF?a8l5`}bX!#Mfg52mkOQ>TJS%iO
z{d`n*<&go{6k@wZK4FWdzvOCxoYZyg_2*6fS5Si@Kek5w+RPc5>%yD&zht`_yEjxt
znMGuNx&zPMWGAb|aNg+zWuk&jBjg|DGIbONmEi){4I~IR!{$BelEV)HW|lf`Qdz!o
zrLZ1@VrW`^GYZHc87d_nwNNMj_urQB6Cd=6z+KZC
z-T}!(;I0Gvu@?VSL^I`3*ucyKdt;c-tE8?=d}f>&K5-
z7cT;W{UmV|oZ4RK2``hWB*PX$8d>A~<;yK#LIJ~joK1oK8DirH4~<5TbnJiN*vXT=
zOs?&HsCwb_^9V--x9N#3d>ox}%oHz5Ic^~)sOeb?8_o=Mtk;Vd?cB2mOF+gp9U%<8
zR-jnoN821mMSw~Y$`Sw9VcFz@$!1$>O_kP6%K0L{YEgroOXdh77Z?y5Ol;0ja)ep3
z^9#NoqMX3=Uy?wQWHLvVPT7%>o^F3(ys79xz`AveZP2==ZsCAQrx-@76gOn<=SkR~
z=pU8J|s@uDgEl2`)OVdvO4Q}mRkKJxr_HY(yeha_&bKddvZ
zJ;`8Gu@2#!`THjRFf)eBX&Md&7FUg@I#b7hp*qy4PkF&q(#OY|E`!Plr2wuzLq|a`
z(PmW{>b0aZ9|xroc2IR_QMvr~#)@mQ`CmqcTn3hNZkFnpJ&XIJwiyip`!OGAm=Bqj
zraVdgXA_IC+}BT^h^M4|fhfxy^OKyZ9b{CRR-9oK2G{^zpu1Dd%z{>v;`g<74l6vn
z6Zs6=x(!)OjRzGD?FrZE(*-xUEhS9t0|+n-L>L>vydbX&@|-xN9z3}5*VoyYB&JOU
z^8lXbfJ5O@Hn~NyjF$WX+YQ}Txgm5zw}(M4*eMHW?TsHhIUOFPvz2t_GBJ-xp}pn2
zcZpKU@
z>%QU6#wX^UxlE)Ei&lBpAll_z@L#===U36Fn$#1+QX42~0W=UKsJ<2U)m3nxvhw)a8k&*s%SoysCYM{L<#F`#G(KP;@X
zw?}O(1O#Sl46!pCy7+_^ZIsP~Q;4F7Zn%RUt=Vs>tiJZUq0cF5
zgn^fVL9=)jMJ*;6q!vO7?T-}7I-J_3O^Fc_xLUduxxeI7VGj}YlLpLIS`^hRZe)y0
zWz_L)Ut?pK!%UX!-}JU#p!_)I>wBpOZtK@y@ljNrWY`*70UjL!qx+3iU+kpCimHpU
zQjy^3J|fT}V3haa6A{w}D5e&^)|=MQURsqjYRKxP?Fn%jhW6f?W?q^qvT=pSypz*U
zX+3-sePGq-?n)O-CWysqBV{{_M)qIrvvc9+B_|&L9^Egxa$59FSFMO>nRA-@(hd=_
z66xQkYUW%$_w|=iv6*#l{-57Pz5SyG*RA|&yS~}CT631B@}D#^pS+cSQBpD=>I%3~
z*-@1zs7+uS-m-P??Yp2nEBEZwH2d5a
zl#aiH+Sb0>pb*mu(}5>Xte6Vr=l9S|$p76Q(9)RUVmM~>=%H)sbY@0br~Ig7(t}K_
z{;)~-ZA^*3AUi&C4hISHXf}=*38R$S)VoJK{4F?Z4G33RtQjM#zb2NIAscvEBz@H0l`;qMVcKV4&NIIcZ-(ByC1(2}^HQY%rWqO6kZ#Zzqhgn@Sb@lI+^sW<;;s*ftqR$fNKIGsYHIep~&gcVsnpHbzx%>XBx?*9V!C4J57I=C(w=&6ZS
zIK0v9p?A<7{Sej|rR{`V
zPBm>Xq`9=oea&jT_sLqlDZWA>h4#$t1y|j2y@o}qyrZ`V%07Huurai@4hdLtaGk}(
zF=rIxCT!YVTre+nYoAnC9mY{`CB3f6iWlAwj3{blA*+Md+Op^Cet(}2pqJ(T_vaIj
z@Pg()f8a!;QX-M9pcZA$pmJV4OTE}cGxHlYOH9cd>dwLo12q&=4r&K&qD)J%QLZ?R
z#aY>E1rMdit=kS}OwM4zmc>mxj!1AA$*zKIOgGCh6Bzp+cHQflZlKjPiq%))b0=tq
zakI8*W;-m}X7ccnqlt!P57jGXUVAy>%yBPXv})4Nt!UfJ1AOJycl7St=XO0Sp#Wy
zmYymsZ6_qim^5;m3mo6$xa7xg1d$`Bq-KSv
zwYB~ER^S^pHI*+DuN2$*W?ZuEOM=IwY^g@KRQfaOi@uE?
zI~Kp_xGMES(X*TLM!%&7r=Y4`)xs#;Ydj5xuwjeK?@3c03`ymk3CZL!ft(WYAP*<|
zu8Ua}Y7o5Her-=lhNRP3cF&jPmz
z6}x5(6L$j*McO-IO__G9<@aw$ubqyM6lKi;Si9V@QVnRu8@)+7+
zAeC-44AMvS2Qy2{jlW*5r3k`9f=l3$?y4vFZ=xR8K89VU&0cFYFO5!rR>2)FrjqJwNZ&?Di$eOM{2?w
zVtf7wc7ylYsViXTL{<=?X0Rr;1n;ZF}{cWG8W=RRddc%;|7dBqFPy-th_
zlCB;eI=e$39VWg=gyTR-xr#{q)C@Nc7lUyKNWdQ@+Rq$%RO>r#qGuP6*eXeF{{1`k
z5_8J(rnm2ghiR0{0h9AW*nkv5bGD7o(j`74Z9RR4tKu7vJ!xM
za&X(w%7>$3yLCpS7#(D{egNC_fh||Q39R+mN|Bl!JL2%V68hiCuIpI~0>Gs(jjHJ4
zevEMwbKqPy=(q%1N;b#Fp32DhO)NmLw(c5}Njh4%OvxKtGD!T)Y~$uFRvjDu7zuR-
z0uu%#v7?0J`9HtogLsFyScgi6)yLkwapN~0Wfc06%c%19e(ET@6B;pNUm~h~Yb*FN
z2y1AU>81KgJBKY>y0qrUZ-G@cablESE?gMG#lZhAFd`Qwa`FvCc6o~0}we~!sNUY+R|^25;7
zbnEQ}_~n^sDVcpn9WnpxS>6lIDzkubSOiD?ZP+l~v@faFYfh<0-A~N%kVwyAy_$g6
zsZ*!I7I{NQyVgb?Vmv~CUkAMK0k0R%BoHIVg?Gfvm_7YCx5v*z?B!o#*8)2YOJc2h
zDeW|_BWNJgi_1*HjvOhzd2=x{3?f;Iv9Mx?c04E)Y8FUPZv1dyE&Dh2bBi_$i@kvR
z?2wk1I|hBHM&g|(2YrumF@zDce9f9oWr`;Qc%I<_U{*NwPBGPPuP;((zD#R}M(zA8
zcfeRS1tiB8`n_o#gQz?bRd5yn$Z2y!8wEx)Fg*_%7-Gex7c3OrzD;;s2~Y~zTJLye
zA2{8qH<#KkfYQqnx4^GI7z?G13Y;|_rit8N%y(&IfiK#5JsOG1FQ${%OYV)39)D+4
z&kX?2y->jLZKBX-J_J^{dw${it01fl4vcr+EGV!?lZN-3x;vh=76E|Ku%G`
zNs&Hdtl&mIJjzSFBkm%RC(R9t_3eFu}BFc%cJ-#^~hVbXQx
zynIsz$!uAHRM%DI>F!&%O=fZ`wS=Hu#>R{CCtF^+8>7b@|{dk4P
zhM$k96=1qN)g3rUWdXD{GUYDQ8DSP_OhZ|_40(o!+hS%LnlU~W&x&I@f$X9Ov02sa
z`l2x|kr-h^I%ez!ciXL&oQ%6^!i{(TgID3cIUK1wT
z#>B=73(8})X#*Xc)brV)Aqu@BDmr)u%tA@xpd|1aS@n+q`@*3A>g+K1h
zHxzG|bd5`QTReOAfuo+}j_maGGUxzIy;s~KF3~||A<7Z`+G$EmT9VW07^2tcwENZ7
zp7VZMSz0n_$N_RdCKS{ICR8U)&4Yl%|0pz6U~tV)++Z6R(<~r+nmlKt1X}RDu*zwN
zFh-s|JFh`5B}^@;93K2v@^Q_Ip?k))-3@Ll@-%~dhLtR+3)3@3Ub0@kX4eY*wobk>
zN%{sg5Yu9a=+rZ5X*J)jwyS;YYN+{<|IUaHx^rY;c-(tY-^sQ5W70I68{*bTj1nRx
zuNeZvU1gia6C=->Irg0)N8*_H;-|Fv9+DuIN(<-B^BI3t6!P;W6h+Dd4IANM;#CBv
zh1G}6PDpBO19h0!8Btg3jencrHWn71b~7pQ&RQV1&ix4W9HPk_#>p9IqJC%m>sb)E
zXtWe8Dy@7@0%ehnf3;MQ56_0Uh87Il#-%-jeHSdC{iXI(obU%=eFC!!6O%Cml1{jp
z$cHyjHc7sAnwbf@)g47lOu`X0CSlyH9c*S+d2ZqMtgPh_)De)_v*#(yxUBDJh{7$e
zg=|5!$kPp#k*#gnHDLCqZ{P5|{*DeUy@ul(-rhcU)9dS$duw*680kG5d5l3?b$A{3
zjdx*EfW`w0z^sUNH+Hzlkz#7%bA+-oF*~yL9m&O1yC2SDsjiR~n)ng|)u7N!V{o8)PyXLZ%lQJmoCuLVB;)7iN<+)&vtImM}5Cm6+aG4!f9+wmL#
znh3{@-OJNOC++I>+>dZycq_!-rtjYmP0sMnVqY|N!m)jfe(w0Obm?XyH}j~lO?SKX
zGI6oa;d$4X8i2{VH{dcIbLGk(Vi;S@#Iw+{cl>u`gWSmGyoi7N(2Tu~<4
z>Owf}@#in^?4kqMUZ8VF3LrJpus#d*J#dG0lvLH$+#KR-^|&b~9?=?P8hRxrj+wt<
z!-(QPso!Rf8l|9==kVtX901xCD*G1*WY?_t=Zx;(-dt)2t%3Kj{O9RvyXvF6&F3qX
z0pcSDI36a(@LYNtb92oL8#23(8aXmF$|J)KQ$K`)Hqv0EPb;D1+p>9c;+V8^=YD`X
z76dAe-u(&SlsN9Fb7QLzu>9
zxRExM0{z+F>f#Z^FM``ZGNT5h4vjZ~HzwnIm^aO&2V!X?y_*+;$+Uy)V?rw+6R>qE
zj-6t%Gi-S#%tZu8Y!`Yyj);$tw4|(KWu)iXPFc%@9qJ<&yr$SOLGd?P1q%}ekTGd3
zpH7@%pu!5s(`V0=N=&yl{f^8$r&Rf>d69u*bU
ztFs^xX53jj<|3}kdo^2nM-@SP49T=*F9t1>;$o)?v(Rt5CLD7dKTR!aB^?3M(!L{g
zTz4;*vyML_Y4r_KIC^EjA{tPsADp$OL2b)A0CKvwYzglxFprg#zshiv~Xk6?mv8J0i#Okm%X=E
zZ73`+Ye`@8JwaeNjBFx-(-L<4^oiBx-`Mmy{G5GyFY?$6rPTSU8Q2
z+_SOWyBYPhvO-AUl=ZHmtpWs@Sz23Ur5G+b_5gOj_|&D`+*vE9JXV@FCgPbsQ=Pyn
zYYqF+zX<5C5nrah+lx9SH335eYZD2Aq3_;d#*IPI1Utu(+tEo5=_LJUx=})34aJ9x
zSL2H0TTwaj8G_~P2$13^HwsK+!x3$##aynp;j(BMrP2u|JzOf&)x-WSWU5ne(ZaZs
z7ix;{-k}Led=$s74B0+%zk+mVNO{c|LrtBjb#s|7zOSuC!Kk&mQ09pEnjFh>PoH-E
zez9Zgh-q7y;SsC-2iK}w?)&oQO;{%t!zx;7Fm%`PBCBU&9aQZK;>X!f;WwhAF0PO|
z_PPFd5K0UzfS67%+!yvm`9KsTwB$7-!_q2nZGAeWkvmnQy^gV(b+fVy|Jw0Z-Y`&3b8b0a(erxpN;p
zd$xJV1N<^atds*UDEGhT>8kVDB?aL}C
zz_jyY#jEFr#*RD3d>NmMb`SNilfgZU5d{xVJX$>di&IZC9hDy2r=$Jf-!Vm3Tt&;`
zd7x|aM(K$JNIRtLnxDN*
z%jexoQV`SZh~p)-yQo8BoDQ+izW9+&s?oxcBjt7_`iMD(|08|6|4=#E^#=cV-#v%*#CZbRTZgIphVbw?qMBd1I!~OIHl$kdv5Tg+v0If?~G`}hW~kT
z0WsGlD5?IQ55wWis7U#*F>M(YGl%rn=m2t3=1{UU(51VydGM?!2({ab!R0(3K6sH&
zyUpPSsXxEb6}la5mquFd!90oz4MkbD%dEqzQ%+R4Sf+?aB3D?s^k?opHjZAt?4CHW
zyXewPst2^ZM*druZwb^yqyFK;(_Yz#J8)&n~I#WDw(Yt)%7W
zW;PjFuRir~@4zoIkF5u3Ea=kuD*bKgbkz;tA7{@&ymjMRhhNp7U%9rV9-lhWvOjOTb<@eCn@CB9ZU>0dolsD~9ryj!#j|ZTr7In9?1!IFh
zAhY0uqrL=p{xkY&>f8gLc7GP-Pb%Hs@=CcK8ae@F?~l*b)%K~bP>5x`0%i{TJ}u_r
zf|nUq#+@%WY_^SV`Bt@Iv2Igo{&BCFy5Z}qT~*vowg2;Dx8>wKToyp^+KF<4ii2TR
z&sFOx`v1M`@$z|H`^a_r06mjGUvN)+K$7cewlTp4_R&
zUAo5>J|0E6sX_gMpaL4;#?a20-
z&%;}W<@DEc`L+C)WUtDdC9!e`CO~B#m3`XJDfee?mr@_=oVis`?j|g2^lLDvtz2wz
zabWZ9u}j|0-6FHdR(t86mJ^OG#+voh_kYXo8S-r#Fg9(*_8udYBD&c!Kc28)&
z|C@mE=i*#)j9BhrYt5%68og5`zcyJNl3ZOiK>m>A*P6H0U(;IjWLH_74-N}GzQ5^X
znvI)JUCyEv{nT6|uZJcg!;2tI#`f?z{q{(Sq-Eu$kqLe)IY;nPwXOUPPVe>NSn>ha;P-IgbPt7H{bWZTc3-5h4}YL%m+hS+R(O7-H=
z9mV|h&%>Xrvjo}#IqwlwR5*1QB^MUMY`C?LJUDsQ`+Wz44eIwg?CzoX_hNPMv}rMi
z2a2SF$GT>wJy+=7dZ+VmUBf+%!>+VWb=|5Vu5Y>Lr@7ZEnk@vS$gp8jG(_EEn=IXA
z4D3VxeK-DU`^IFsnLQi-+&J^#L-?UUd8=z5zI95S>1e(%E~9lqUWb4bdam582D)MkVC0P|NjTqd#LuG+EYQ9
zoPdyZPb<{iGN0Bx>9jOj&9b^fY2SPq(SZpIv)o(LHOf5As72+|R5!@&95YgG`g)nI
zI~KUsNZmbAFKhoxS>XyTfU|M~vNAF{tQ!UQ4LuSBM_DHL
zGjpQy>5D9||N1pO!)@{TnMMgLC$%%?Wb7X@f@Q(D@*1?^@BIj@?G;|Qd+x
z+iu`XLeOARgu`dgjm=)w9go4zK&S8Hr%(6KbxAo4k3S^8ZLi~;*Rv@A;avS1LRH8=
z(@l25YwA_pAN-!CY%S%wM^J;DR)@OJCm>ff2en}+3eAnm4Ke{342T~LxDz}JbG30s
z36hU8kA~)!-@m_tI+tRA0&CYm_;RdxCM74gKQk)9UeWn*s?qJ;gNlGNfU}4mGK*wh
zupNz?;F)PBfCu}Vnl>hzTNh72>b>fp3jvjT<=p-jw^*#Fn)A-mx$}q~0h!j|rCF^N
z@~p%{in+`NFTXv=ww)Gs@%cb}z$xMj5g71r983uKe^2s2bFfHCs=fWP%<9_qXO
P_-FLU2~MXao?-t7u+KHl

literal 0
HcmV?d00001

diff --git a/docs/img/redis-ha-after.png b/docs/img/redis-ha-after.png
new file mode 100644
index 0000000000000000000000000000000000000000..b1c06f1d342d6f64aac12ddc5230806ae8d2c465
GIT binary patch
literal 196086
zcmd43cRben|37{;HBcQ5BpOO&WQL5QAtRgYq9P+vMr3r96iP$L-g}f$HWiApFES!z
z?{T5a`rcpXocDR3_viEZet);y@1O6t+wHvH9gfTOdXC5Az8=>@mGfumHZgA^kw|ps
zo=;TVF%7M|z@$_BegMySq=+>ivHE
zxvnSp?+@C1Ew;tn{FNSW?u2tzyS&p;Rfe}=72H+Uu{?1>xiPu@l?T!|_iuQ#_weCL
zsbXozuOCGPkKXJTI685=*s#FxOMCA~e|5WHzy3zjI+9d?_K#=EsZw
z_zJTmeaLUy{_#gOy5g?-{(t^4pAo$wf9juqd|7GEKm6y*B-HhkaQ@>(wrvx3Vi7&~
z&sT{$^JGo@KVRfj;{e{w%%P`pwQ)+jFI>3LXs6-zWoMh}Qi<)}zkf1{L#N4Z?U(OA
zKR@TcSVFSwEZUQ-o!&GY*iq}=tX&`_K4jzV3ZAo2S$kyN8Y
z&r2&SH`&|UuMdsBQuUJLzLQ7Si}@J8*HbC=sp)Bt2M=i8ym{mI@}*XD9Y53V-MdAP
zAD2D!oNT8tIc`Ey0eabL8i1)
z$}%#mJ!zSKUs{;+W|EA4=r5(tEOkee_!5s^&c&8DSM2ue5s0L=K4uQ~)S@4u`S)h|
zV0Y{FoSej^7Dh7j9qDe>yhh*La@G$KJs6kzTxb@uTa*1MxY^
z+}v3em94fnZ$5qT;@2_?+&d%RjTcZ*ZHauaKtpCRAFJG5qr6*+3M4OhD_UZHIlX#y3
z0s<Beyy^FdmsC}4784VDcjuSDSVv*9ozcTnt9R|%MJmB+sH&>2B2CQ9
zoRX8<5E2q{W4MV&zu0w8OG`^sg(kMBqJn02cGhv?$0~f+T%z1>hLcH;p4ue1?-!^2
zae=JxtLy^XI~|(=#(4+jDQ$C8~3fopW<@1CKlK
zK6~~oE3B=fgS8<}>BDr5^p=OGqHUfP3KplTbmxMEuKi}{*tKqP;=`5;dAAZK?4B8H5WU$)o06Kk
zFQBTf(Ai<_`t{7Ao>5Wz7b(p~NzFw?MH92LBGdw5eod-RSAYM5cke`T9QPkMuqOU|
z==*Go<{zV@zV7a%`GtiKrKKlF1KHdbw;385nvWOG)GIg+#s&ygv+dgTz^W}Lv2fRp
z9rx!t-PlMOH~Y`)7dUR~=;&Y+zjfHqZOIO+y`Pu&ylKc40jA%6`>i(5&f@ar%X>LF
zH{#VcY}vBw6`k(qUR(At@0#qXc;`5|zrLZNYjE%(e!q}atH!X6Og%>oOHz45
zpi`W+>!Z5I_huN%n|4JKE8la?Oj0n8#V3&{Ttefnet*~IFV9ui#2>s}Cc2bfTszs5
zMj{nAuOUU$Z3$~_YcoNiFzYOm#Cd&yO8KKT+k$xkcjeOOX9qLhzOBG9%&=(YC7m!c
zJFJy%NT;W#SFkkO=Jn)>_MQX&^8D5v5{|cTcj3nw1+UVMwdY3L8}`0=SYG-0b7^Cu
zx@LOk?fyMJ%)*re(^?fT1_jR)yDnYCCB|mqtMbEY;i#A9>DKMyn1{?wY8$V81wq|V%3JTKgVcEUgyD3%A
zS>R(u1*&Wp&O+cZYgW9|_*SVq^EVVj#WfE|
zL=G~j(YuRM6gwA%a`tQ-)XSmAvQ1`#s_iMjUn9OjH3h~bfO}1dbe=J5;+@6#@sr)nVj|{xE*y`l^
zE2T-gHzM&_=kTfxuH%Wi-nZ=T(9oS(DO{N7rtD_CHALHbK5X@79>KeJsAmt{I?r7|
zon_#$L}$g}k21)&e|NK=zQbkCAk-sHUV3rDW2nVKGxWFvJxS=LkdKegT2hGE&Gp*p
zhC4X>C=YUD#($K0khlUGe4acxYi4GadjE=viHEy;B^&!ld#<`kzD#5ZD)A}}sS4GA
z=(A7EX?M04x$G`pn%#XqMD)@v2}wyw&EB`>WUfM*OrLY%wYFQM!^rloT36(x_i4Fo
zlkdE_9s^QKMpqK}s%4&5C5cVqojKo;%EDtO28Z1(jIB8|osJmB3}rI~((N4JEBoA%P?o!uh^x$oXw=h=ScN=C*Tne>%~
zom%$xa!bp{+dJ|r1~R_4w`Vv}%;$go>cR;R6t&q8{E#Gh==ky7IAP6sO1UF3zP@xQ
zZ>nl)t4U@Jade~~i@&DNROCtt1k8j=I`3@DvHIY*PgY~i-o(VjZ@D))>_@)cU6>p9
z-l>HHdhwc0i-w7bNj5LXV0ZWGAK@(zTkp)am@SNu(v()3y9tj&`1S5v2^Ll&l{_jq)D9!=T)G|Id8!q+IG%$qOj
zTEK>HqGFN5l{K|ClV{%*njW(Z7WVLc&OrK6{n{kzo{_3*WM1MCe|c3D(jNZ%_wOG*
za-<+s@`IPc6=@buj(&;#2dtcqYZg48
z%#ho~?z+Px9O!V&b#;PCNw|eo0yQ~j4U3lMGw~fcM
zFhRVpsi<&%VVV+k&uzRz62)xCaC7=u)R+w8vQ0j0(yZ&&t^2*Yx#0Fq(up%?JckO=
z^tSKVQQFj$qxJGzibdS{@TtE1pATE=H91X>4n6~Vi-}>MpP#=t@uQSc$ZV^=zW&i$
z11h&hzHN&><12$RpgO5h-TF+9-LXH^!AHu-N|k>0XK(LddqqQodXFT3y{Vj-P>g*{
zYWUFr>Yo*Zk=AM)7gg9X(O}l2HAtcG(BFu9TB`?-ZZdjxO|kZ(X8N#s?RL@>pFw%}
z0i)8e{&3l`AiWoIfYtQMK1#mz(_D*P+jU=N7r0IC
zKYu>>kx{-~S=h3wT+1TecJY@Bg5~67ZGMX;?iVj#?q_FTtMHOrm8UB?UE9cI2TDCU
zXa!(gQjS%BbtuVwcC;;8@9p)dvizc>SEw(l&WekRo6~Lp7BWWnSRbAl6SEfZu>Ah!
zU8F_90~6;eW7#P|OaafHMKKEj31*bq2>9a#R*}#u+P{A%=iI)1`%>5Y`?@rS<_BDa
z{bgm%&gM}!-c00^>EX-@Bd>S)-#oE8Np4|M{QSzWXnjuS)&o*>?}G$QC%PW6$waC|
zSTv`hL5CS0o8sZ-K6#e~xK#Gjw6?Y;ef1h@&lT+M=>ZYAn`v5O&SHvoNEvSKX)z5)b+}({
z&&I}vhhI%fGb&CKw(ND6adZ?7a~k5zwo*&nURqjeK3-N<770oMI#ChGZz5D}dg;>L
zB9{f@nSna;!Cb9SiQ8(FluneG#HmAv4{tyJG)Ey3lFik2VAfo34HDu
zj?C&c_qIkC?tZis@oV-U0Jy`)wVd*qriWnx2M>}9mafi9MSrLHjVEQC8{>}rYZ3I
zrN#Nlq$NK6ckZ{R`?{VhOO+1PMNhWhtm*prkm2y*!zZd<@@!*ap*?o&82O@t0YUTm
ziWh$m3}lp8nrWCHj1N~)QE>;FSw+HjU$V0Dm2#n){fw`j*~mk~53B-0zi;WDSmUn9
zNuju~YthPLjBOosWv;b%7EbR@7z|TisrjFlkLGQ
zjf{-usnpdZ)C69mA|Kz~$GU5zR~m13JbU)rmsi4@pK~aeyRTaFV`wPp)$JIS!vy6l
zTrXgtum7XFJIcvrdGU5({lVv-35|n1d`RilfL%zDVo=dRg_k8QElki%Hf-L!MkTl1
z_Rbv%u*K}^0WLniidVuGwV9^M_wL$dJxHW_Mp
zZq7x+-Pf>-}Cd2
zgBy5&w@J7zzI$nG;S?tyo|!2qBP;tfG}QNqcG`eT2xNhRf`T7oWAq9N3O|4T{DInz
zjVS8q&v^4j=EMmSpxV>t&pi|Zcq0|7ZaVn`aS%$wUt|wu?$ehqr*7ZQ?YPt=qO7cp
zJH8*-H-OKm$)Yd(ncoa}KBaIEQ-gA#tjnr+Zw{i!73fX4lOSsfh
zgcMY^uCcM}wfqMU9z+Rfa4e3b
z*zW|ks}L6x+XpDSNBovM9=kg9_yYhV=vZFNVWFXWh-!wD_5no}myn<;#KD$co|oWGIaC|dHQeVaqV}ngVR5|y6$54
zA|b1okIxM^Zz0|2`@*CcBKn#)>d+&qS~_KrhEiAT<_1Vfjxf32{_d`};qB8sOP=4x
zAMFzrWdzKmAz8NN&;dW&u^gC`i&ZO_sysr_e@iPXyC&Zx?AT=8SvG>$GPkAa=l)J;
zA`asnI}W-w#4De@bxVkag(d5HvCD!8q*q^Px#pIeX=y)J`XA{U7)Ws9Lp8;)kQwh&t8jTB`pLu>!J5WPKIS5*
zmb6R)YxQz0OW$0n8WI=3yc(PF@zIVWKd{O)TUalC@T4PJAVdiWP?Iwa${)(gV$-6p
zz3|jZAMY$qx2yfs*~v!o3k^->mmR~o(ybxaQVm|8I&&szdAuc)BSwvri;D~U*lpI9
z!-Jl`ZR^&EIsV@suAT0$Ia~2~kIfR*ZV@j`2-(z`K@Ys?y+@C31tci__S)q1LlY>}
zHY>|6mf!PsoI7{!9TpwB=T;`BhbKMOmqJIyV-ej80&9{uFR%=1@jb|ZbZF&zBG|{AIU0PrN+rx(sqwZOKNh?}C>2?)LN{9Md{b4IdUULbqm=bP`fi
zSAT8YDWy8KZmX87@|&ETavHWfDi&2YH}gEm`=l)ERX6@>%IQcJ%e|cc*o_b-p&Oe>
z=>=1pa;|^ZJyPYVy7V$E>~W3MLP}cn$wo6AH*Vdm4XmuJpT2%A!y}Tf&vd-AS%cNS
z=j*Gqr~Fyi3~*Bq&RnzO%BQCX{N!UV6uU~gE>W}#tIJ%q^9>7~5|H4j?0im$BV@(t*1cIQilO120y&++6pEIa*aoX{nGXz3EPpntr((~Uo!n8fI-G`6
zTfH`K+os8H`Ryp`@p{rkDGl2|i2ntBealL-ZDdY}^)M?Uz>2r9Nv(~LT0_=9edbIQ
zhyekN3|%OiArKSXsn4cb4=;HA{CSzY_fh@w(t@h4?j|6lTzy`7iRaIs+m#R2$0|6^
z3~VEnfE4mP2AC$fmwE3>{z|*9aUlbJBKpIch-tAGJ9g|4z43!YvI!{Jsq3lR?&yr0-a$bwZEoIm_A$#zK#bDC
zl!vGGtMbg*;ck(C^9TzI+puxt-RI94u7A(Fj~aR4_;I~qG3ZGs#!&G`Mn@-ql&(pw
zNmYAU_8izq<5;rA{{M~HA#}aO>)fH|`}Xc7+2*q7=;+9uJ9qN@`NxO*bN+TugkQR-
zzj^!iL24?`qeqW4j@czAdEeoWDe_X4{_I%UB2UQIAXhU|X=w`quc){A$@bD_<}%dp
z<+k57)aairFU@rlHalUzLePdg_j+wfH$+hSj^^^NY28&PQ)Z?&UA
zlSm-Ivd<4GHP+Wd#lE1Ub23upN4f9rGZn87;r+s0QGu&4oL;OkeuPF7g+REw3-!>U
z(?y#C>dBm}m0EA%F0btTE*6UT-
z{p*^!O!`|HMs<3b*8JKD^W>tff1Yr%!+N?9h@{(~b!?!gFGn?aeWOdJ@Cm2~1K!XB
zKfkT1ZEvkRA3u>^W`q?|()TskhB|XiF-TzDADsE~T-h$8i-6FCsg_|@w;vVYXMcZ>
zzqVoB1#!oL+DOJ@*I8Kh?9m+O+`j3~5~Uek#wxo;W*D^@q+=C{02V1anKS3(i!KWU
znSahG^`ME)$k-}qT761GV|z@!+o9FrxI56qYDEprsH<16J__>%ij%=G$VC*j6NVJd
zd||m15X%R6L>7uBr%tVfR~cuwUgq@abt2Xs+*HA*I6?eYtwItji__-UR=NNF`)U?R
zr|p|}av$aLVU={c2*IA%Pea#VS5g-gUkPpGE0|dAI^RXZZIGv7rDyQdJCaiNje4$p
znnKa!X?C2Sxa9FTHF^1Tr>h}%+yIS|l3a1yg|uM#%DW8=9Ow563-7?Xci)_VpP92g%9Hs~vA@T>60%wQdgtgx>+b
z2u*2u@fRV!2Qz#()zfdT0Kq
z)9lE`AVp6Qyu|fh_wH%$0TCn~pX3f1cgW61>fX!ndXUU6+*2@hlbSF|oQI*dW%Fnj
z^CQwqp96j}f*KoSX=pUav=w
z-a~hJv~KgxA9$P_H*b2NAUMs9`5yf8GT~oYfN-~E9XoxTjSjaJ7g7lTrp;vcX-8*g
z6C5{395H|(FZfWQlc{+>jljGi=W7mU9$@$%T%+c^;@X-258KFU1=3$O6U(1wSJM$C
zD4ky3-X5qNFjj92HFE0b+tZb8~+jY>2l3
zPQ0L}C(9cU4d-tIE$wz;McIv?1O88rn5m`n(Ja?zqA-RvR(4OfA62LDdCpGTX{OVS
z(kOimblqh(->d}bUxL!8PU52RBJd{U)(g72Q4kIZYtoLjElKkTtO25agQ|Y2tR(ar
zkaa&`v|UDbR80mIVyPWl(sVM{d++3t!7+FZHUAEJTp(=GJcseeGc&hRGi}O5949W)
zvWRX)KTkR-aS)mX$^S@z1_gL7!n7vro$Hb~?sOeUH+fl@u0&&3m2mswyL+g`*%X*gRWKn+N5FvaP8(;fioY!B
zW-6oogM%B-o;{0#Y+=AW^_l6kaTbm25XZy#AyNmd*|p&n+n2okT#I;zvi)3Z8+`^H
zc7L$L(uDcRu@mSJf@TEd7!-Ugy2kMv=BB0(2+;+i1nOoKAvJ$P@bdEN?P1RW&r$>glyHyOpRIIBHVHJK4qy=UAcV)y14+z}Y*V
zqMN;`({;frm6tP>kAs8bV|n@NgvF;`esAjImmaFp_xXj(c-a+_A29Y_lylz2Y-p9G
z_b;}1X`+Nv<(EP~q~HSr0>9x5$(=vnlozhSRaU&RWDUwhp!0bo0x0y+n#o$CjuR2)
zE$OJpj*gBIDBX^~dIKPmM&^}Xx3*3fzdrTzGlZ7+=#>ff)4LM%&*L5a?HsZUC920y
zyx+`+@~2?h`f)om^CP@ySFm+aqK#4HAOz%pJBcsM1N66hOnK^Fx)fR6zh}^?JZvk@
z;%UHGw1A^7)LXtM)gz|*3=O2+6y%B&!=(%;D{%FWUjDRoq@!>*gui5d3I0}(eU#@P
zdtlu}*fOb_td}&;ciYxl(|o#6#13G_w<|9(M8{#Eb|2Zj7gZGrmkm(MN}vNkpk}Us
zoKH(4fI2yXlf=%!(Ss*Dfx0VhKawD*e{+E5`Lkz!!NJ~WwMj37mr?a^RY&bm04?b5
z?!I{K+C!{t^4H7jxdLo8lUf6m;N&8cGCx_trfJuQ`aUr|oor8vio`2e?zK
zkOQVT>}ePymvDjYq`n=4O#d!7_Yw*@u>%n|2O5??9;UhZNqOJLypAE+l-USt>}=S#
z(U*S_(hxa|j?)c@TWBh5)SA;z$FGWp2$PVIuyy7OV9l|lWo>!^fG`CLOEQO>s`Vd!
zCyC^*KqqxBQY9g*p{J)u-Rc`CxA5l(oDszY4(sxcf193>PReFfh-sR~g0)k1uRJSV
z-TPG<6aZHn%I=U|$bZ^
z11iM)xZ4V`1>IA*+=2P?
ztw>1J#VYK&HBh?>k%I$E~>_&2&%%gD&oCTsJh*64hxt}e$uWQS?z>Sz9K
zEl0gpb{V75%Qn9cSR!<-ek~-`@)w-yDi0+vLx7sw
zly3BduV`Lo*cFMVq`OvdFaRJ-P*+Lr2rDG==%nj2YlzkIu}c_?brwH?`OyOaOoWfn
zhKQ^mi?l0?^Xv%!fWEO=YgXb=?t7q(*G;htW$apFhsk!XT&6mYPJW^^Rac*<6!a3(
z(<&0|+s1iChlN=y;4(%{dQT_}y}b4=`~Iw)0vx~7oacj(zlMhX2TX+W{P7}h|0CMw
zBhev+PoF-$H1Uf~(kX`GVpUVDu2%hu6Lzv#FyziHs4B45j)|Vnj?KtWNng4br>sMt
z7pA1W8yd}4q<{!TB)i#A7B`=8BOfww7>|5=4O&8j)QWJW7-0m*ZVguiYfCi`5>}^e
zXOuN7+OSW{x%RE=vv*0ec2=!f+azw7CoP|rljggJ%voGhZF(>inAOBu=$rJ7pn-4w
z?YH02FF)X+O+HT}6+%vV!@(I4aA$rZvgxFH?<9o`8J6fVF>!ImDCr&{sY~O3A4|$fq@~b!?kxZ6C(|2iX=v=lgF;3%tBYC(owRFs*!#nSEhf;x
zl9ntjEg@i#G^B>L%^a!oy#Yo<*b*g|Ku>l8H>e|-HSf^_AwmxS)?XRB)af+LL+HRK
zdQD@H7~8l>*~hl_^vI&d>}fLHV_h8`rX6QAWzyb)fj9gzp(O=MyA_o?qapHWrChkA
zA;#F+asQ>WRnx9NcP@%rosivzeoQdd4-n2SUAjbSw6;A=lbI!Ht@s(DY~bMp)#W#O
z`=9;*2qA(CX8mbfuutK(hy)5r{jSS?@Zdg#uB4=-h-0>%j(IJDgGBlUr{*I-OJlta
zWk}N_pp}P<3#RurXqp!K@@lly_FDDo+WNBEJKLfn-#+jhL(V3t1)kfosO?X=R8Q^T
zB?Xs({^2)lDK%eW!=XAQU2z2We3D*7mAA^D3QBEiYC?f8!T!xiIS`6>;PnY3MFht*
zzd9|9<&O(p<>A%Kp(V&QsvaD>(@dh8QqAvlGOrRj1rf_{Dn0y2J%Uzz$gGj_2Dycq
z!iajK*%m)EQkb;nsLcC@2evP5p%|SO3y*}$t(h5WvA;?2l1XLcA>+RIsRx_LJfRjA
z7Mn?sLGIRdZOrZH>+d(gyI~%ehN(6&HT52HYRaU^ONid2_f;p&{y-i7-hnUht+TR(
z5UpXr{W*2(R#v%vFWo*+v($=;j~^4l#5l^I>7m)JAtBEjfg6DYo=;tWit@d<^f=pr
z`dDvs5syHHa83W9pogg3KLA$v4DU>yCO9i0pJIhx+YM_)*Mx&V{`hebo@;cuX&942
zE_e1+(0S!YFe$^OTu6-};hufowjwXmGLp>)Mk6DuzJAsG?lso-mhTT4IH2W@niYkQ
zIx`)Y&`gPBOL>BQ0ie@b+FJ(S-c)S0zo0s0k7#ywwkD<7lQySmSLKzVs;VkNxAYx_
zR5TCZ4ZYPJDKxL&`hdWkJXb`Q0N*RO?^Fz
zt3h&yKkK`@8(}hpPxFSdr2s-v2iX|s
z&?N1yY~fDTd)q|q+0M_@R?OH?6C7pO9PgST#UC&nXFIV3x<1fRnBP=vUA(lDR2zME
zi(D+#rQkM&tPPjv@#5m5dH6a#Blyvz1eW#yB;EyOQ60-
z%CZs43kmLCIUim=c)gafeA`!jOe7&kBNb3MHDs`7Q!=udsoltU;
zI>!G?DuUxo(P0`bluYu=vun7lj~=mR=XzMjnlLufS!_gpQXM9#*YCp*?_2dfgPPID
zfqo^`bfxcGUNPK!cbzNT+6l1x@b*oqZf?uJI1skWZWPL|$NPpZNSFmGFsQ$MyUvS`+G=Bc4Pg+!^uuzUC@^x
z6x~|ZwCk_l!*bl7_C#;C`fd7Hbkr!+LqK5_z4I5$9N{h3i@9INvcU~6X=~dJmG&Es
zL-ow7&7k9O-t1ft9Xyz*wLehoW?Z>x94G=U<5AM4`>TNx|8u*Q2W3C#igaE@LJ%L
zYCESTDR{5;Kh}^WrA|cB5f+qzaR*pN1T?^oOVNH&B8eP(p8e>fjEtkci$Wqsz?Jur
zhlm2ozwp|49X+dr7sMAL&W{A=8QdV~ap|o_KC>t6hCghP{s_BZ8_>+hkEe)^!l4`n
z|6U9+qcB(MHHka(TOpes73bbZIql=xv7Mqj0-8TF=x{%2yjkf_&w@n4??|{=j9<7<
zTAB^{g8_1KfbMdZWy^1nRV5*DT|_bj5+M1eDwb0{Rob3E8^!K@O^q_86=>x|@20iN
zyeC%>L+XasftiC+kgUnsHtPcrtBH;T@C}~0+0D(Z0HN<2NF5<V!+;@XA8*O0tNgX>~{`GAx?u(!YK(5FDR)734W4s`!c{-o0all6o?L*8t+g
z1dP%|o^@P85=b#Xsom}{)?SK}aigc44jhtj9M-I-Jpsv_@G_9Iz6Ny_v>Mq
z-zK0i$AlP3c$IL&PTAXwh~FA;Pe?c{$Nt-$sm~ly)r=y&6fHS1#)gjj
z_e{V1ecTma+--oQ*VSfST&8DT82g$Bxe~%(1di1#uE(05!P!eQEKH)Fb^ta>NlE$W
zMgM<}nG8!vadMus`+~PT{1s_nznO#V?5OG6Z#g(Tz|n*)@D%zT&WbUNAEyVH8quiH
zSuJGx)IT7=1b}b=3Ok5N(p2S@SCr@IE!;OA>
z{d>JdL}^*sY03z9#2b2vYtY0ou5t!)Is`}(35w$Be?@F8cN{RPtgJk0QcjCI5oJ{D
z3iVda${Dc8{OSqKax^nN6ca*0B%A~|S5OXBtl}~eRr@-y-NTP+Y$m;OIov$SiRUWB
z2w_Toc~VWY(iNlIz>i2b4Ym^@Qv^`|;G&;C$w5uk&sN(1w4}LNTT^<+-(jMpt4U45
zsI%DuT8Q`7I~wJ(aF8s&Wvqfs>Dc3YtQPYygfM2+2}njrjcKzwejr6BFJ9aW+K3QL
zE%+!P3EP=CIRotaA;my<1g;_=&jgCwC7J6XEw+=-kh^uHl!%udJGKjMm2Mjanl83r
z?~owV{g>g4wAb8ZDkFKPrr&nD%APy7QEL9j8bXydPu@nDlz^pVsRLRY)D5lg=g>qF
z#5m6K?Oz3xa(}J+xws~zq;RXYFflPjEmsza!;>c@Wr%#RhiwQtcJ*o$1kQ4k<90)9
z@x+jN^cqVg`8S>mkB{eAy=D!O?je%R)Gu5#G&C42C`CN!<;$1Eodm2%D!hASJxFAh
zIe8OejzstZ$7phX(xmQ#4fYvj5Tct6l)Azv3FotD_OWJ=;4UAOq>r#vW
z-S{=ttGl8a%?V4a)q@`!u^@Q*Z#QF$qR1bEdY9!jY0kr|)WKj|GeQK!umQX+;G(ys
zbliVOod#ufo3{~@c2Rl42>X8f_U#9NX;;MGgM7leQ4&3Wj0=8w0%4ITG027x4)+nQ
zl!W2Kf8rzjFPZ7jAwpp_V%zEH%91pbGh7zzr857XL}3<|`s1aT+0l~-8386R8dbvZ
zK%*W44V%_U)Bip7_TOg||EGzx|9gXsHEQ9ZuSBdifjmk$PLSZy>dw8+%#;O_LJs{g
zu$sm9vt0gBcEe31cL?L{#coC>Y>;M9=1C_IM+VxvRbKVy#0w$FAV*2=Nd64TF}uhb
zn9X&G8hP4#ClTNSw8S6~(V3yE>^*Qm)@lXKM;1d_i#o{7!ff2aDp3NKZs9`{VN^8m
z8;~RFjgWN*3+V=|B?1#@r|A=H*j(CVx~zR5n6J9J8puCAr$g#o3oGnsOeMzIg})+%
zJw+yISqdDDt^(_kEJN*4)1>Lk
z;r-*|Mm_wb8^COEi%Cb*R)mBkOoqMvW;Aw4`8r+U^6MhMt#NtFnz1$g+$Qe@d;5R)tk##BV_a}4YYgW|
zN*bLd2`vtLyzX3~J;%0pWMR|AI}nCu3X{y8k#(`pz6}?D9U62K>!0fC(C&?-(@MvE
zk&wp>%pqi|h)@I)0_GzPuT9RvXW9=KQCC-|Tf+c}8#dq?QjS%dCzLN=7wYV&H#TJz
zDeipeBbY%7rUgaRp(6;&_7LMm=oO?NMk~}^>*-l8s6Qjvjc+guOqP!bd;mK}QUXB(
zF2N`@A0KVWq(OXZcp1Z1--_L&ArK)k=|t8NW7o*SNp_<9jh&ZOq@
zpWb5pLPmtcON{nm)1hX+FuZiQO;%Tz`HT-^q+R$Bp_yBvqY{Gae
zO6aCZ&FE#G_wS$ft$>p*=PzA86YlA0YU#4=nN(~^$%z#GT(x1kVaU_Q&_&J1AKDot
zJ~1{nKI%Mc38FyDD!~Y$l$<-BoL=+@<4j!;l!^96YE;VKVY_7FvQIW>imL4|6`tv{
zvF1hiW@ho5zuHYk=60yNg(W2Az#X1Q-%@^OMNr7aJ!AZcyG<`?V5Ja)W*fmx?<2AV
z*tB)`?qoYR_=%P+Z`L7>Iys5%P{Bqy|0j`q$~h?9qwyI(0^{vtf4Cd(;SAgq@(T9o
zo(5N{y&o?ZmwdnP25ISHbGOsr;SXSi545MLBs|$w1w+)ZfKfz5gh(Poftth%yvKQk
zgmW6b*w)teR@gXLD8B81)CWX#qA?)(fbpYf6oNoui~TMSl6$FuC0dMNhD$+*5Y(K!ic&t)-IM>
z!K9u(H`3yPU!K|qhdW{xe)V@4Xi3fR)E}JMUkU_AzPbi`s?eh^LsgdjAed9MLCFYR
zODj#EFzDeg_CQ0G1@s^iXheF^x{$Jyn};X6BT!^EPmK-aeFJjezoXef>n2Ea=mOZX
zp52${isb9}4OcE0ecEbgx3p$vA>`myN6(|P)26Z}kHeks-7(^)^gm2etYc-aqLkK-
zyOqZ%Z!j4O+Si$ktNR)uK|(en@HB!LelK3c*i?fGgEXUw(m_RMIOzK)M#(OXxT{Wb
z?Df~qfVkA2?@;rj#ECK#OYQvnl@8sN@Xm5EN>exq;szDul!=~FagLIb5;2K8^SBM1
z_9A=@q~(mi1oE3-T83?rJUI_mV~pKQYK8|$@cpaRArHZUE;Xg-i1(Pt75xu!wcqBe
zJ^qyi(7*`#U(q$+rXISlyKY{xEDC%SNf|+Ytzx`*C2ew<;KKg$uU(g%7GZ}AdmhPZ
z58%=B92v1BTWD)*6AAP}=ecqK_xs~T3+H>vgZS~=Jups@Xt;pv1wb%yFaUf#>Gver
z9lFa;ajJ_tO|IUMZjK=i^YrMbD3EQ!OCE+;KKs|LepLiYeFB4%a4GQc>;IUUA==3G
z>%3RL1m0~nbfyCDQ-Z#Cg`zGALp+ot}k<^rdk&A6LsWQ$jrf~91U;>R`<^Us7_fFJ@9
z!aMVXHFb8)I*`9zhK{R9eu04r(`UW^++)h!tUiNAT#~NTw;ckad;dqFI-FCVdBL@o|G88J$b
z4bzsP)vd`C(*8+!i=@361CI8!LLL)|=d~)i_fkdZ_>Eo&%0Y`EH2c(rnUNL-03tld
zJKJ9Q)Uwz(dy7ifhm_dKq~zgnznbv4%VR`+bdj11oLYGFjI;Pjki6_WDUG=$WuKX#
zM}Fg)pyr@UJnof2*~(2zcx*ybf=W-$t6cvFW>a-Bt}Pk-8hmUOi3kb-meX#M`xU#0
zGe;x~NQ;@LB1I|>MrVD$L1-3l-#Vl&stNDid!liA4{}E*dIb$2)(Pe8e#Hl8cO8jH
zDuVWI6He><(mr*1#ObR16E^8`+z-W_9q6KyWNo8W2}M?o4(J<{ps7CMi+0tIzZE(k
zBkdIz*EMN!ko^jMQ4Qb_BB|1_RHcq|+`GF8%K5^li$79j;)<4;sA9=!fz!YMX?e?;
zyhZ7F{6J3Om|>3`RED^k7x!G#TQg0!zb`mB`{=Q$`K=+v0Sxo{h=GJBT8YUY1S
zuR6Axd&F$lp17hDe8hxWLrnn^AaR9nj;})7B9YLmpk=AGHK|PZf^T9_h9;tjJL=h=
ziF4W#p*8X3Y(4c^g%mztZ3BZ9+NH&7=5$>*Ps(Z=tXvyHJG_QNpeCCqWwx+vYDrZr
z3mn%3CkA3%OG2L^2Gmg0i5o<^h$l$8tNDkg5=H%jx|pc(2tFNb8oo=EC7FICUE;%>
zFcL=>1ymWGl0xe~ySozKjYOFIuQ{89vM$V1odz~Vxw|4albj9JTth>nq_vgV$o1Dj
zgts^=Zy&cG*$^@19WxbBE04Z+;X>e$8-JK{gG!)!QKOcmVC=R7XG(A~S3qk4L9aeL
z>R%(7jaVZzvUBgZ2iGoV7Tt7kpdn?OzL(_02T4#uj`S|e(+L~7yRAQ!XE(fFL4k7i
z(b-d{h~JA8>7k&5Mh5x9S8!nJ9m!=BJ}StTk?je%Ia=Ls$p#!ofV3S`wZB!
zTvh}{T;hyJ=90jy$w|8j@A=GQmWMzyxYNcEA)?Bj{{b5lM3@$R8vaV_;A%o5YaV9o
zz1faTunhvegj9uj?DY`~-cE3};MmigaQT&t*bJ=yE~JnF@t;;79UUF*8Xr%m)O2(h
znyg@0mKeZ0%**Q;9en`Fqe)0m@5yA%bRlySL{$a`%%C!V<2ig-rR};|5%CcKlw#Yx
zpH*Och;2P!q!=qQ6eRVLYYq|*F_aHgl$a0G_2$V2Z
zE`xLT1*aW?4vyZMc!kra?lPj6v}VDyJXuNOU~jz_lI613t|d10D3Px#
zk@;R130=3dSz?&_)zCOJgi~jlwA}D*{D@4@&_ZCM(4QC3IJRc{$9Y44q>O+j@G&Vw
zvYF%#?Q7tU7S~@@$3$nE_Z88f{v)=MX7ASfMg=WgZRyc03@BXEwGe-Zkt&M9PXu-T
z7)1pf&aPQ89k2e7ltf)wDW?B4aB}MvjT@)bvxJ26_NWPd9@Bfy-7v)3C{UFml-V^+
zmbX^;RA2d}Yx#wtj?1g6^@3F)O6m=5+XOk()Nia7-g<9(!K1U~F3W@>i|6zm{eXb}
zk!z`iEf(R0v(D++DO(N}VO=-n3t2S8`SSC26IU~yZnwybprWf^nSGfA&p@x;v}KDn
zRDwnnilu((mAVQxH%lLr%S7rBVMcYr4#8*@rF^&b3!U=19%C0Tdt~k~oZU25@Z#qe
zo5Znp8-0n-q5VOHAxrsB82R5j{@Ar=Pbv`RFj$2XlBrp^TW|X__rpftv|)n^MstI%
zkL!!26aH#&!%YBB1QCyiciD)DAlY_{ix-
zM#tK3cn}!uT$#bV-Ds<(TG{s*r#J0R-FY8>tyW%H7IR)1yryYmq|$bKiaIFoQnn*3
z>B(&wt6wgM)0g|Jmy#{V611;ezB~ZL(2Ud+Z{mCyZXY8)^N3`=I(~IL@HiPO=Ik!U#KE%Sn;k(_lWlNU>JG5Il6a(QU#~^_)jFFeanPWu40Wy18d3lm@xD<79
zx&|kN5B=jjhHIj)=a1&tHb}Zqvmncx-R!`MRvB1>#xNr$;kkt^?f7_s`qfOGo^oGx
zpb{YrAVn)ASJ{oV>%gK-(oPQp6}g3QxFQ<(h^W_6jV6R`d!ep5f?8qM0ZFM?N&
z*v!7uFdY-+4eJ}Z-N$XZG6sW_n^G7NapJ%kpITTCmb`4NtZDGOH%()3pr>*E45F0J
zE>C}ZXKUz&47v!Le1Bh`c*t32QQM`IF{oa?H3uuE-;{|L{}ew+j_Wbj)X<1Qa?Ko}
zyCL8Zg39S|@3b4JJxLiNHy~W?!{28)sgawVGy(^GGkeX_a<1))&JGEe|GFEROH%Kg
zI?b(@YC;#ZTiP|#shulJMs&(|ajox5kVt2fUEjM4ovV)pe?{gcuUqzpp1v3_+5h-s?|d-zBfLFd{#^eW$+-~t
zP|0uyd|AZQu-NiI=~1V>EqOb^)s;fUv)^06fYU@C*j|*cDUAYsP2+0Vk$wt}rd!$Y
z!~7O!Yy>*gLGQBtIBwAQwclBy4aGcSyU_fyN39+#R2V@MOgjuEC8yPgNjm#zNrN~f
zPn?X3ue3Vnf7_t@&Ru^F*A@N3m0NFz?1_z_QU@(v$~18;G;@-uE;lGe?1&E5^&H-=@NBO>aVt8iy75o6NNz!VpWG@kv6`5_kAbIj&dqQxbtS-((sTatOFccV9#od!t2Fz@
z!;|O2A5X+B{N7sKGHj|z)|Sv@OSaUcEc_U9NiL*FJB%A$i7MJitQl_>&MfrVM4(oA
zIXM+{q=WvhW5Jv8un_|0JKOZ>m!(m#!#VM+K6sy
z*IHi
zxw0)!xso>Rn1BQj{V0i0sv@>^D0RN1_)#An=~LXr_RrVP}{Li0Yx
zTB0pfl$ifb{d$!;Q}e;Caixq41F>HEW;R?qq}eaLx%`kl5q8FvK8{0&n61eoX0<1fVUJxFHgEiNA?Nb3
zvjInbMGRm@>a7^-r7PH3Dqsiuj`8QoA(FgE^Vg%@vY$Kl{T9QP2Pb3_aMESRd4s?YOwtbo_1C_;kVD^
zMM|T5S##%5F!9_4p;f{?0HVVP1!RiVP@AUUFkXIr!EzZ%J%9E@#2<%hBR#&3La7}<
zOt%igjJkQd|D!;~xJ@hm+tJGOuII+K+qmE=G517%Kye(EFW#TV5&-we44w2@3`h8#
zH|-`nckMF0nRlQf_R!={8zOB1>E+-#ShIfV>DGzAFjhz%Hd^8GE>lZd;8hMcJ(xQ@
z9)?eMBf|M=lkcLTWw^!u6y)Ys#6e`1bjo6NoxgOh9(1Wvl2@2gkTyWoQ4bNdd2e=_
z-=_PNFwC#U#R^tuMU0q2C$GorTxu4v_+Z1yiK(iUrE#}|OD+EKl0CsrQSDAt6qoK$
z^Lm$;g67>WC#|%1%t=-^YPXj$P+nJRwYjmizjh1D>$J-@Tt_UMaHxVwh%S%*I~CNv
zWPaD@>}3@d$zaIOxLi*ZKr+5!YI;OSNP|F02uuz><{oUdNKc*E*MI;}mczK=IW;*s
zudv&FOc-fCY0;EY2LgBN=cjG!XlRmg1co|`i-#H$qw&|l24!hGP9xM8=yWZCb12k9
zu>poee-(l%%A@n<6r=?;&`5xPa=wFgC_Z%uy^J^0;Ot`srJhO~{@|}WSOcJRoCK$e
zWgqM=^OjdqBEwrIL!v00EZ?nL=#))(2a&QgXwp@e1vudz4Gs>Li%8rKGpG)An*+zU
zfaO2boF2Y@3u_7j@B{F8Y!syU*c_^c@YgyFziEy+g^8y_z_~DV_g%JTIYI&Hk+Kh%
z8~SU)tsT5&^qOlb702ao+Z-xH`l}EL80bL_$T`eleg}+Q0Sc-H3|O4M89=vTyhHll
z*n%E4hE&7CRz?#ZLBWQ!={knqCVv9hjA%BDWwK~8ps?jv7P;nC;)548RasMRcg;m|
zYtPj@r9S%Edb>})koag#lDYWXwdZR>#E64r1VKhHPN&naxL))+LN>ZEg_mdF7AHdK
zyuLX^yc;nRDiRbg{QNH7da9v@j}aqcCp;H2N_ww9M}@;^=_7tVoFkKRpE?BoUciLF
zBxADM%5wP~Tv*)tj@Ua*r@|z7JMT>U;;0kzAp@8@ZoF;u_xe_
z8K|ifJ_^#eMzd5a*RGcU?#|?_1#VNv1>69KUzCQ{?9@yNJ84zT-n*SNZCoZRT-}wo
z&-~FU+>Z&Kd!u1@D9IZWs!4d=M_yijEtcrwF9E{iAWRG*+^88U8T{pHX5OvAD0HeS
z?uWBlvE^JMV3lRS;oG^h>D~v?I@vrv1dr3Ri#9eREQQ!
zl>70{%$(=>JFfe>fA@7<$93F)+>Y}%zd2`|`hGv3_xrUxpU>Cxr2x_e^lQi*SuFn&
z3i-mVy6x3xBctX9D3#vrzx8{9lH}QW!dh-{^ZC}af1|jEYU?!>?_E;X=n<*juJvzQ
z|6-WeF`(@~|Mk6l`+1_i{qbwFR@<<@|DNDD5VZH7U)D!%jQ;1pNTPMhdjIp^wigUz
z-uUldH)?wR$B+MS|2S)}t^q&)^ivgYZ6Of6X*6PqtnOvKsX0ZMD=3JMyxLacoiMh|
zyQykFzCw6nb%Mi;(W6JpsMsHmTlYMQV4H1qN(C0Znj>p!WaMoT?`TzRlS@+{30a@4
zm8sB===pQkzJ@ECw$!GE!9}@gYKKpGGg|71X5Bu4@6w&F22>uHHAJ$`_*nE@lCu4i
zbGu72f++3*=r+tPVp62~Y0P=VA!i8r4ixM~#PpA~&ue|(7XLC}z(wlq;Uh*s;4XZS
zm6Z>B(TFm{j$_Ll`-N-Q6lNG2{!ZevwD-TeY=NU=6r{Tg6Mpg!7FC2tRfLBEH=CgY
z(=!lv1x>^8Q>SV^HY5arbMb3k5O%(qq9ufYQL~t=jX%LtcIT`qfM~t#osF
zHxb6uKS3s0PcGejenzoS{=*=KADPlLhjF6yu6-7G2_%7bG#77yNOc9t;E0cwey}xV
zb(4y&*`4Q08PJ=-Iv0>3Txg?9*5B@y1RXFDax;&~om<<_oQIq_zbepeHVv^w%(C6c
zOG9WgMw9DE%OX8cCqMJGHZ`Yw0NN&;TkYPvR{@C`q+UKFa8$9jnL*muBd@-`Qe<>E
zawib30-zivpy|w+yEv_vkiPN{=kw85pH$y5;n@c5+A-BXE~$yO%Fy*q@z<7@Fv$km6E=ep=Yu@-uS2H$VSgg)D!)42M-@kOxwG6Z_WOZfrdmlijQ%g(4Cz8s9OV)qLO^i`>zJ2Ya-8i
z)q2UOWHUfdayi3=bU;V%=(ga;b-oc<-INe}EIK-xmTk0d5q1dDmH0rL=*-yDb;0uG
z!T9f-qiIXJc8Q1}udKYld!kFV$*|S-E^zeaPI`I==GrYPaeAaBJ~NS7?$c_PYe8n
z`)+1Aa@;sW5sPW^#ElFLj6!;~Fkv3GjQ)sYa#airbgvX88YOfc_m^>V2u%RpU>>f_
zy6~@GzKHw=#99%zQK+ba4LBqXpk1l~QYK@6G}m+Gr_8@-)o=E!S)zmO+^t(4=iRyN
z#ttJzE4_`czFoT#rpY`o6Qvg)HVC&B?QNw|&K@2f&UvP0qhSG{OHpMR(e2{LV)W|f
zl1%5^s4UT#hZ`8AgFL>guD{nJ1En{Gz?AVb0FL!mPVdO
z@->l3EZuBA;+uv|_K|pwo>tv3ox?mRhv?On0jLIyAxK>km;**
z2I^6EE)u_mqq-*RJDqDIb9ZyRfxAOd2Uqb^6+UG)GBx=+uGle$v~kYQ+Wr1Jj~f$6
z*l8_I4!YS8V39N_t8eY<1ZLuU1fuN%^m)
z^8z;C_0-qX%cfP$<+6{j7i}|%DADDjmL%#$(w`@5yn}**gyBNorNw51+Z_HhtI}60
z1(w5fsl(QubfYIQT)+Nuew5V(n)Hk{lP*xgz{niQsj}ME+}OEmR}(Oat=~TAgvA>h
z|J1(ys1YM(d+Wva+4QjmN#9D*SdZy*kk&1d!^_0oBAnkQ=OfC`;Unoad!4p=RJifa
zHSWW$B2xTZJtL}w43ku7MyoKXQe#WA^TM_BrCNjUx|fD!6Q@kEW}eEI>V|`ZN$RCT
zhq_WVMv&X=pI$zj*$9x=2>LbG8#(Mbewf0fNehKfk#@@mDFDd%OdO}z?bxw{h&k)z
z+iy%$t_URxL_SOi6LUtK9MiCQvdCLj11qR+c)1IlF_#l+B-JnS7Kz(lbMQ48g!h!*
zmMaUH5(6iDYP{%zG_MM>I6`#4@B<zmP6Po|5!?6s5-WfU$_^
zlNU=0X2SCM5ZqdbvDIW<$C6D`r*0RfB&3@KNJH{Ln7Qpb0J!!{ED!vdCW)R~-KF}t@}t6aVzn%4W%{Zq-bT8$GvOwhSI+A}LXaddAq^-acZ*z-`|)lrT)NqOM98aCH2@?IM2B
zK9d9D3QcVZg<)6C%C!(?h*Am<2~l`a%^OlG^Y%|a+qhLP%V#g{SYEOk`JcDq<>!C|
zXVY6_MB%_k;ndi6{DVn@&UB!FiB*&Y-gAlA_-u2z#h~SCe`^7}#4(|j^Rkw+tf;J9
zp-9J~#Ym@XN0v#lU4}
zR(x1|;S@$KUVW7@k(lMqtZzxuTB0=Vo4}hz8#JZq^)NJeqeqRJL#YrY&L1S!c&k+P
zq(B8CpNIOECDx0Pa=oUi_(41b^i*~E*iS(-ZBysfS`Q
zG}Nm5W7sCwHjP}C3|fRBwA>L7Qca$Dn$u!HJJND3bFTCQA=uVZALM-?S^MhYPcF|r
zKaEyWihg#DbkM+om8UQf?KSX*L+v}aT;BIekFC`aVIr#2cIZ2pNI0;^fI@FtfQ3
zA|VXj#z$2~c(^B}nCU?9%r0{Ve)bNyzEwTrWP@cyht2NaTjSaL*_(cvhgKsOS($D&
zU)b@}?US%7vG#d-MjOnG{H~gh5#2CU$)xk9yA=eg6T7C+h_|mf=KV5?L#18;Vx*}t
zbMo`!)omY>1q$oR4iM#Pn5QxE@JL}!E((+=XY+cgGfm}54Pg+N$hJ)q^q~xQTRm_3
z{P{KLvQOcfe4Rm_!Mp_3fbVsP=VXE|U25b~2e)bW^J|(%{|T!OD?fk!{Q5)nEHuEs
zn`k=PPy(SszDTOlMf>h;l5+is*2fc$*PAzq;&$8D;^+2j&a9zU2ub_3ckk5@)`ZiU
zA;Iy?=L6SVe%OHIMYkgUz@|-FtJJ)$O^<)7Mh!LJ`_RUhRqi_fg#8s}W!C@$K&@$4
z6zVrQ%_6v7P~dV^L`#}c(4(ST(jXScoMyr@M(He4Hphr^oEL}%T@^<_^Z@AoE*M=pCo$2s^lQoe!A3M5D8`A!IkCP&
zH<@705M7pSNZXU|gvSOn&q@6eU_=?upDz(8ft(%HXGoG<+@`86b${ymWnbI$q(5B^
z_`d-*msfgr^JXo&DHYb&naOK^RDg5Fq*i@P|7R8i3~)?(+p5y$SYtx?N((hzwr#cc9Fe8%ju+?>Y4MS%lb
zFIVY3XZ8_;o>ZZ5MU#
zH-09St~!1sA+d28e{lp1_P`C!BL=v$S3pnEiErv>Q7X+mPkJTB=NtX54!lR#?3aCE
zY<6}wO5C9sl-c8o=Z^Y+Zf0f6!??&XwRyW$_MPzYS&^do;@_`Dh3%v1^#4cYR8~K}
z@#q!rCAsWF=Uc0-6qXmD^}+vz?JD{<}`-mJE;umYSQjD+a3
z=#ebliGqHApL;*P{5)=29)o}aMWcSZzbq!45)}+|TS#z+8xs=7S`}h0C#So_vk?p7
z#o1Uei~>`4ruL@Zj(*rTpuo!YbnweFm0)mIdeO@zPi1;v4I!|42h;B+9JhAa0I8E&
zpQYg)493>)lf^<)w`~hgo9?Z%_90UoT39)JvGtau(ievt2K-yFTu87yBC9b%_W-L3
zkP}3>DJv@r^gpx11oV`X<-Z8T;euPNMiBW!sa_(S=iT4L+&fg8ul$=_zmoIB#ps7y
zui-*BU#rn%#frk$dKcp2yM`YQ(9%AwDCNVr(GpaYf#6!hb?u)e#OP%IWT=)f*7PEL
zWBOCitivk?lKaurclKLE40$rk9vdZc?s=p7oxlC&oMSLqSH63hZ^7Eo)GAuDIn43-
z$-!6`z8Y`fKXCyaH5E9Wn)Qp1_TFF>)fE26DV0X!=I7(67yCrcHzYk8dKp@Y82ulLs5duA3
zBX~2fFZp|;sZB#H(xKO8k^Ib%oMYoxy|HCY%oOu?RJRe1--BOHuI|;Vm%?zX-cGN+
zIfrPf-X@AOZe#{Mbm*F%C}e;V6eB)fo8C61UqMLSEFGPUdtsmR@1A_Of&XPA@NPA<
zQa%-~<-3g|=LL+jIQ=u5J#0@}Xc3MC$HxyA{=I%V#}4d__0A<^j#pFyX+_sFLbnOO
zL>I12=Xld&n9I}v{2VvY47f#ZT=($8gC7spI;UM2R674*${Z}}F`%s1m1`?9`dnIa
zJuSB`@S|5PUJ7NU(C>zY|K!kk`@FiNH|SixZsZI
zT&~r$F}Xi>$8fv52c!vF@oUB=yVb8mqo@ma{REA6B#97J9eJF1DIkig!qF$
z{&%R
zz)KQkD15zNRPsA@@~IYHU@MZ|yG%+&$v>ML^q5f7K&$+kk}(I6B^L~)@oljgkBaHv
z6BO>1@3)${VpTKg-R*zpLcTVi3@bD~9AGowO$vPt;H`ZzBu=q|+X5kLX+e7fE&YB(z}CRzwV@gCK-q`iP@SU#dUe;IS7SBrzV>0O_gU+0f-~%X*~3K-0Xct-
z35E-F1-7NAK@Rm%3fIgE2#PwsHv=S=S~2^x^%a$Ca?(sg0C`n2sEyl+S@n=WlV|>a
zq6f%f{>KGcJUQnlL(dIXS2Bx=OpJDyl>4qC2d2mHZ&xKw(B9$=3ZJn870YU
zG}*ay$;fB^%>O!JezK9;#=AZbl4I|lb?KVvKUPU;KCO_tu$HEa7ro50GnbRAWj~fa
zO7;$%KrGC^(E7IC&mN)MJ@womY3UH^2f~~KyG|TDI
zsnZicmh*hVDoVrz1%iAEclpaKtm}%sB_YODOM7
z3+;E3s_YE@9Ri?3lx+59p3INmA5y55E{K)wkch;yMSof))_%=+=6rh3i#4|=CQqPpvkPb>a9)%1T0
z-SdB#UHbojoOpNto2a_~HxILVHu;~+c^sjG>8gJbliS->KTTfB^cCsI^=<`v-%6uB
zZ+s|%7o4AMM?iL`7Sr;6<)aA{BBSmcj(dx^L8dO7#5eiz>*@V>H>x~8m4s3HF4>Ss
zbsK@SDn!3<7&3bz4SCxq&^MJaLwQ|$q_iPndFajs){RU?9aUaP*bhcc{BgM}H+7hI
zXgN&%X3IcsHbL-g1V=a>4jwm)$*}H{SssN=9}48G-v`o=`rY{i7jl{WSIy
z0IL}r&!u};7^b9R4FnI|Xqs2})7
z;At|ft~0AAU|TsmAq052f8;aY$^U$#GxDbPW!Hj1hQx|G(HJw5a1kOXl7M=^%PWsJ
zEHs_;=f^EZ{?7q&o5U&=CdnwD7b2PzP79E(7l1SEAD`JF;A@R{2{U+S<9%!SIRXs^
z;c(zZSzKM|C8V3e0!_$ygT8+m6w44hJwv)WK3YWy8_s{wLLUHl?X$^J7_hOC^SH$}
zwhw`Wu0Q!UG&Izn0mSrA{|(UF$i{k0!)qH4xQ`kEoti29D9*$)Ig{k1x?ToWn-g}b
zp8YYALQnkA`M&{$4ic&n^LzgTJi7Q_c=U>Fu?!4S@Q`l+HYYxE6sTnBp8>54~F@xaHOp1{j-qc-#PiVU3#9SPmSHIFYF*OsO%@)jimq?t@rDOTeG
zMK8IR{iJ*&b}Q$*l)-`tQi2;p<{NYMEmrPE&j~jd7+Ija&_k6MU+A-*y(qiCG$Z3|
zKoe>T#_|MidOq%+fB%Zzb9+Y))1R+f>sz+(Z78~Ivoq5TDVIMUTw?a$)&j
zGG-R@V-b{R^8gvIqQG9zTt4XgWep{>mw#+<6TSB%TJ7TkY!g~x@G&#}Fv6mDFgTPb
z{wMj}&&<5EzCRUw{@VDGP+S?_K8-0qVE2Rj6yHu|H{VQjTGfGZ!BCvO$)&{TSX|e~
z=Ulydb-0h?+rHA$E1bK0WtdK*_r%zyKab59A|}6+Y^~YENB66)%UA!o8u^1WO5Rh&
zU(qwUARWBGXF}4g;Hsvhw$akkpz^fE^OsV(`^bsJlv5?u
z)@Xx`K9~nB#6eb&x(v1@Dw->J17@*l;qEI<2iNOu+O!cQ?AEPmOl}USUg5e
ztN|1vCw{!J$P$ukKoS#_BK#67>@Euc}{CeFas#RvZ@JLXOt*eO6=UQ^T3XhZysiv0%;Y_u;1x9~Md`l;$e
z`Mu$~kf5VnQ^ijxm?_{hTHGi?Xbs0PpKkI`hGYbM22gFXbm>ylRXzVcxRUn?_We|2
zrNEA|(R+Zy(S9H;093l(oLD=3ZI92FI^cd)pxKM
zbNgo|X%^C8=b-RVps|F9U(E!w*~4vI3p$169zj??ZEg9o%B;I?KM^_jGjefENd>WP
zRZor|-weTW9hs<+M
zp+kZDVVq37g~O5<0~6%r#*J5$G5&ID;-z`jLL>-c21o2Dk(YsPL*iB-*yIS=VM>sM
z&q<=4V{x1T0EKrj6~iwR|GCstFWGpT#NA)1^2>kp&XX3Oun-W--!%}_rx4F_UPU*~
zlX^RE0`wv!vLnl}!O@DY
z9^{5f!okdf544aDneV&M=6C6T9&)&Dq-R-f&*AODxCySNtNs=FrgnQ_oY(K4-$_kyrl}%FpZxH~n`>;rgGg@Z|ja*Z%=2Vzp}W
zG7u$kja3|zCc7OBdsrGw(>}j2=zAEo-6_ZrdC`x@#t*iMEv3iSm4TIIpv;}fX!{Lmeit6^MW!iFC0h$}*&d{h@
zy#X;+C%PO0HJ`(quV`uS4!ky8SNr?7s{`F%pEar%8a5#j%p)
zeSxsm*3Bj@#8Q@TtRIh$HSFkWuvwW2D-X$uJ4DYm$?p-U1VTN*h6(oSWbK?O!>bN!
zynB4TrBMlac@0xlLPq;w%~io5Yi)@mOXSJv`jr2z1&CgvZQJx$Yv`A}ZXtckgx{>Q
zf?cUY9MC~f7X`!KRDpzU{Obwd|4`Rs(4c6(Zt+T)f0rb#G%P(?x7~jo;ze-()R!B-
zw|u%D3$vzc>LlxoChOH%O`L+B;B#=O`HFbV!3aPFSM7Zb^K3G|?WCux=fm;Ae!?(Oky!Xm!2co3asN&S!8^Yo0##V30rHw3;&8E)Mp2;6#3aGcFp_M_5T#KKh4k#`)Q|qiV}xLiIgP3CUjp@QX^h@xhgSxQ
z<-?i%k1^G-vE3ByVP*;-dG0R$nGJpnp?isnmrq9-+k4CqY4$%
zysyst!RfFbBRR7%V6;?tA^{4{ak%a!xsBckoj3`zPJwlY^#|9at
zRwQhFf1PR_@`DLImNb2_c;0BdLVW2R68SiCy2#&>0MNtlEjYrK?*~t1RU1Eicp{Fj
zKn}oHQIs!_=+My)-fu!l*G4U(y3TJsQ`-cB{(kkwj|k1bmRQ1R=StoMlq)u
zF|C#l$UL`f{FlV+ViVS=U}5LE+~X6fMe4PSoqCN3hmaLrsVI2k|1|a%C&3en5_~{^
z_m`w+RC;qOZd-p06{9cJXp0pmc=90X_?&nrG*XOf1Wajt>=!C*{wLbYNzZRU(w|ug
zt)~}Ji=-FMUkr}EO&aUgr2hKN7d717$Tu?2u~um09E9`nWhSjTL4UJzy!6)lU!F_r
zCcW6x|Ii}}QbWXlfbK8ew?Z^G`tYE|HR4)K1i=;+o+y}jxre?8u{Cf5u56PoVSDq-
zU%^Ed^wj7Lm)nexFkF}FOF!|_FPGZ%(X}tU5<-PFTo<%8b2^iwhRX=PpY>)dd2gyrp4!o0(>V=)(IzhXw0E(lfps
zKhWWlUvRNwvH>}%d*%HFf%BwP{3cZVkymFJNI%op_6!!`Icc(z2McndcfyAYAxL7q
zRiU;+MUxm<5iIil@gmWmv4^hGb$a#is9Y>eEk8#naD^a8Hya?{2lmuwQOZye`!{!h
zsS$8KNkYz!igDArZ~hKZM@=-xPIbFzwf;orO0bS+C(_u%rF(Z>Aon~zS3Q3){(Hr5
zvJMLI+XpqI3>6-F*+}Q0>c?UbD5MYn)-SO8
zE4A5p>pmjrp}0Aih(uT-scfe3on3HMO7s18aa&aEvN6JRn{61YeqKC$*UJj)>8^Fk
zo!k^3uVK-_@4-g4Z7s*5O6t3>Bq_z@+aA+Ro8mC=6s)!Ogg52q*@Y@v<{~EHO59?S
zsLh8l2f~#Q+4*$#?S$Y76av%$$pxss%p-4xV%NH
zY@Usfa4`RxngLt0jLt(i5Vh$=dV
zJX#;YrwT}E?w4yTh==ajW{Elwq^b1v*5IM!QRQ^c6i}S{GwCi_~cp
z1XpDenb196y4Rpstv$KJjPv&h3
zbNX}nM7eE5pxY=Ix=KD{5Wc&D_FCAXEoIX*@6d+)-YxydvLKT|wp*{y_n)Zyc^en%
zX_Cu@_v(tq<+ZAC&Et<-%Z62l`D2-jD?~$+x7bG%ZXz$d*U9jWdtBD9mz??9C4*bI
z7dm;^#EHELs2G1gjbAqi2(P-N^8Ar0k;e+Rz2Mr
zc0_Zi{#*+`%#r{;oC7sYHsig3*TVD`Q#OC)~EWWdu_|C%1p5l%r
zDPkw=5Ffps*y$m5OpPm;ybaFxHyqpekr5x>y<`vof6ZGs3Ntq{YTNo&d&;9v?zBun
zFg|n6oQ{$+DJcW+&lfwyMB!*WvZrL*J&LfSFB;tM+0i4szaAeSZ>_OV-e>w=2T$2?
zEEr-Hoy0kJBhI9LqX79G5}ul=4ZAX9wNvrqbB}b{JdokWt7-EXOVNnU;eZ`
zDCJGsoHmm0pSFGPuzzOsj4}R+<>!5MpSv%wZZbT@;$lod&lp8GZwmNj|C49M(;9Qq
zpun`U;i8~wHnNh@ZSwQjDLNQxMXCUTtU3Xh-R1jt5~HEKkNe$d
zCF2m|x6~daq#lx`jQfnxx2xZG#A1YJ&C!UOoj9r9Ro9)j+!%J?=RZ49iVNztGd-2!Sn-nYSzGMS}IpOBl
zPJ)c!kf0P^;6%CWJI$HGA?qz{72`OM^twz}^&58K$MsgEDyb=r9h*+mc~Hz1lzUzX
zT^8w_e;glQvcsglS+{PTE7kU_>OUyfswsb%ac&2fFGQs6^fNr;gOGRaB!D{y0qXOU
zPscRwC<;w?>|kIzg^7<3v|sJQx7OAQ^-ob$+P8?HDf9I8cj<*>fBA$;yN!gf9bGSz
z)uhwosCs-Tjcd%~}4rOe9>l~Cf|54V-2_3}J+sog8XlB!zlzv@vans}D74_5v
zVml)2$WG{h)(yPe^_|644}^;{)MV$@j8gOLBUQ^NdHLT`Q?&%w62K^!ZK~zi=*in4
zRc|r1E~7uJH5kQi-Scee
zzy4kQ%g=^~cSNf^e#-Z6PViPXR3PSQ9%^9l`@z`%^5d5&{%boV<;l;_N=lT0MTA|I
zcy=88%PhsPN&6m}+qX)5)m@`i9^QQRVL{D`$R@G8jSGm*%m&kAVL~`ibMw+Y0}SVu
zvF_qS*sC{hF2+2=fUVXuE_l;>Y*lUmaIvgSEd5FiwaWIqy0?L
z&$8{`dehIPR*3ZaO&P+;Jc}f4V{MKmM4t1OHb(
zOth(MfQ7laI0aaE8=USv)PBX&v}*>MwKKQbcs@SDdMVUsT`)uj7EX&eHskM~*?8H$
zy8S-so;`N}4hySkCJ^)cy&{v`8e0JlrZIeg@TRy&Dc23S&6L|yg5phbF
z<}Xf(dU#snS2*qhi!{KHJNNEQo^~KSarN8AX+hum-DdkWp>`%Nj$-2PNn#4q%DBh-
zYi8}aoEE=Eb6e&N2W=^X)7s?5&+H4v3>_q#Q+|s?w1J}>z$RF>BXxAZN)&?YEFCkk
zc@NIopaEik!#8HpIIT~LuH%&o{L?UR{?@HignPHw3Zm5Og6Ki~=>6<`;^475a*@K1
zZO^fmigD%FXjRxwu$?1_eHly()Sd1{E-gqZe%S%+I6X_>`-3kF5+XUm0pR+Xk|1e`uCtlA+{
z{KqIRms#7Gbj~PZ*Y}EzYFLf4CCY!?tVUa{@kjEl2Q}OeQZDtVH`uj%_gVb#*&H_8
z-Cf<%%IZaVdAt3=RUX#^vB>%sq@RLPGX-LvI8
zmQy>JIKv)S3d)~#zSDsK$KmePzJ2?pblK{~9zSA)n}y3gyOk?Pplf=Zdw;OK*f}*r
z)ZvMUOg4j3i>)3IdgRD91Ve)p({G1s%(Jqz+_eI-;+8O6?%m?*?2PMjo|=`P^1lU!
zw-I51Z4FD>oUQrUolV<`$B1KVX!LC;&{?bX8G!EtRCLE0=9qtH7cuptg@Z=poJaoP
z?>o7lWi&LuvB;KJAN*v4g^9^7oq=w_r482{3c52=E6Otp<@OTh>^o9)8(LT#V4m*<
z3LnXKRx_2+t~^a~n;^`-vHoFK^sI)M)(C#?)!C|7orj$pVQgr)o!vYPW3?4-_uacY
zR?GF9S<=bE>5d(jmd=3xkfRh2O)+(i@=p>$IDFFFxpOyf*+Rh+pq6tdV#rYYmcVT-
zZnTSdr|O;0xi%zL!yxat-f*PW-KRb?siP>`!)F!s5ui@n@W2s$C1U)t^XX7~Q&ZFF
zNy#oXdw`dAmdlBka_-fv+MZ#29w>2;UxZaBr;nOG{~b#$!?!X=!b
zZ()2)=Z?i}XL!2C153vGod>wR92XePTVZr+^cz;99`@t!f$#Q72y?>C&S
zVdeHQEx<;Hp$ZCJ;LF_K?hGG33KLzf2m2BI+;8_YUMD+X0EJg~Y~xuWah}B|dLh5S
z_Q@D8$;&tNG&`c|!U^#4k(O6bK#w5Bq~RINMp^BEH~l@vGG4IybL!P{6a6hK4}84a
zH$7nQs{;i->8Vk;vx;RCRB0Rx%u;ojiq!q6=%Z&tG>D9hRM)vsTvRlD#4)j&9EDA;
z#q-|x?*3l4>_DAe$NatN(Nf(iq9-Ze%b|GKygm$^qRhG4sbu1;W{1%Yib~brFhe;9
zWOCC}T~&2zh|vq6r^o$5m5b*)-Mja(sNrqw=!B#slyASXs~W%W84wVq_l(xK_57M@
zd6A3*;M@*AYbx*Qv~Y;*AYs`>@$wB9GIqM(2~VR}A4i!gTt>m*C6O6^O*>8qVR)0++MNsNTIyNf}|Gigvcv2@mN+nu3y>Rvv|Jo-2$0ydkJj|
zZa2JB^~tFCj31Unf`(A+O2OB32YtvKxozDn4d%^DhYya83mRwB7SZx!)cQ6j^+Ikz
z7seoPQGc%W8Yf#4Z&zlND%~41M@wF_EC~1~MEVt~5wTlX=AkVqkR~L&{=NxS6-P_#
z6z=wJZ}@Q0K0)kBrf&96niSKxiFrO2t8g8!XEjno-T+Rz=3BG)d3{|ScSN8aU~EH1
zj_g2K_=D|}Kmwg`D(CIy7jBRb)tV6Jsl4ZD{o`jJlCo6`q)Dt}EE9SNnb2%&iP)vX
z&Y(N!=DZGw_1WgB?kYs(-NNQ#)d`^De$Y*#k-JQZ?B!yx!LqBOQo8T_88K5PX1Pt4
z+orN#E0C3#x4v^M-NSNzuL+H5j1CzWoYHmBlmT>oGjMZ7YgqkO)9cWoJ77qM8B`gD
z%Fp(+TG2m;L0Lpqu|hIsA^p4-Qcm>&=q$6=-u3-8=6QyfSTR7I&{hHm@PSuT-E}f$
zHEWcqj9)V3prZTi@ZrO^j#u$za<^!ZzYrR}IJ6%IYp8;;TgYOazmIVJ1#ag
z9qt?;CSFdqJC}c+wnFR#L^Spo4^nED86ieCw1eU%^JK&iACfZt>04ZM(OJRyl!30yB>VP+ypKIiBZ69`JqFxlvcGmxQAF&Bj#d&=qCKs-
zmh`r_R!K=oB_t=;_=It1hG9*Q4>DLWD+9YmExA6{`35ce4esdk88dU=h)!0ls)oPR
zewN~9?2!7@y+6Ay(9fGMT6C7P0K1Z~<77(
zCAX_qAYYMmom2FPm=Yd4hO<Q2snZrTF25i!F;ed2l*Pm=kdvb+}N{DdH3(@)~fEexBwMpgVAT-
zNo)FIN^mW0BwR^%dfi#<3=pst_54tfDB`^678yD|%<7yWDavodPJEW!j?&>m?mK)q
z9V+-e)5bCFe8`^CJ@Pi-PT<9Wt;p}wv9l-6k8^KWEp|Z3`{)f@gCohf&Nj$+RdE-%
z{0<8;(bUt^?FDvx5i##flg_#|YsLVTij;~E+G_o1v4W3EYwEBxvYGgs&zdYzf8io3`B6_IyFwKL2E5Dda+%mk9GideQaraqgdR2ytw+6d=gu8w
z9Zbx>gEQN*y2Ik_t`kjx6wHLWPPOBH5>-aW;)rzvRT>T(dlpZKdE4qWcx@!tarN^w
ztpf+j$6fLKTMKZ&d7iY{mR*Mkg1>x7TBf;w=5Jz&IoVt6r(*nk+$ERHapdLkivpJI
zQyXhwb)@U)pwJslOEs0hw7Vlap9n1=pY1HV95-GGXw6}{Wv0W>!7WL4^U`y*V+e`J
zF`nX^j?vv%M>|yE@Cx3x?B-lz(`|2P;A`)
zKwCQiS9r!D2NFU=Md{@#cTiEupfJq?Nl;&nO=EaAKzZ7^YTaUzj#+8tz326gb9|=i
z%u?d1E?>n|91b9}E4!-A*Zk04;P31TE++>55?7e8VHn8PLG`eXQEy}J4ZMvPh=4p*
zn&KiOJJbJ72QB{qGP4YQf<^Il)|B7+4!piobbrsUN1HvHc}X|3yYZQ(WW&rFMZ@ME
zIbfWkV<()J6B2jPz0i)rPPE}VYtn9kvMWf+i{>kTXtQU@5wT<%s$eMu{+p@v-1fs~
zx&-`8Neo$!97JcQ4VAhSVOt<49)*&~u(^=?J6RUR)0i0Q#=v+d_HBz5K1e)6o`jt<
z`Tm%SJHp96wrFzlK8pir+B()=-00$RmocXWl$U}vODJ4??1Z}lyBUDOZnJ5GJ8ljF
zTw2x3_<7#sGUC|orY#h8FVZ1)kNm#GM=!)>r*7u&AN3DW(RTWN9n&dd$UVlnFOwF3
zj+ZXp8aFc~s_!~91GseUB`aX3>+I_mVr2LJYnr~n!wq)T?zvCj?o$2`Gv)P&N!ykk
zUPbxG=vL=5?&`LUTbht#|FnDhf>oC3mXkB@pJ*ECd!p_NeXszDh_4-)RZ@HDida$-
z`8>q@s{V=O=H**Z5w3}D6fHetk8Ds|if+TAHdkZoH&rn{dUlxDO4B=k|7azHk~HwX
zpGn)~v&OxY(y(wb1jb-A-S+-kz^nT($l$tgt
zL4v`W5J7JT!<+cjt@`E5->BrFMZjLCSGF6>a}>)u=ot4PYaqSNGXF$dF-%i)04W5@
zqMf0Z)i{lL5sDEdHtx+$E*?!)VX6q)vi44Z2l@a^?eMTgG!8%owtQEX3-|WzpR@JA
zeyMTa+#48%T!z4-b+tK7&R6Xi)P|
z9}&Vb>srz%jrvV(oHz%`Je~@x8&vl{9L^_3dUx~q
zXyb8cgyHpi>;2!KkCr^hWF>wvR`K(t(MaBA!
z8~IA@=*T&ChLy+OHN~xZC6b&;O4K(otVV$l^(mU?ZBZOikNJ_>X{nhLe|?ow>K{3<
zU-rCxTHBVrbm-DJt!vDe39iWsACn6E>>I5F_4AqLRAO>uR8%JYC|eA-Pe1$Ee_;Nz
zw*mpxwymu)1arKRMQ9=-=hD`lSby#MG?bbLq3S4E|J2Lv2K^Em|BbwbR^qepkZ{|N-gW$UzI_MOnJ
z>;W|bm9jxxK_AVIw!AOh$Hg>Gl9OvM`Tgk8v-PJVTzW2ko;Sak4Z2-&Y$9|{g@%Ql
z=gq?a?mB(?^ul73V`*MC+giD
zBHfqI(HQda+y4F0in$$>@4R~S>}}N3nw5(O$ECa}itR$=oPqZ42_ElWPwc9*a>Weq
z1>@WQIvvN{JJ*sl)Nkin7BC6<;`4#gN{S(#3%0qsZrYTNIOF}JZ7s&OCca7KuJA7c
z_yXahY>aXemKhX7cibB&*Q}DC3s!>2o?oy(*|Mv1@bhpsnWXEZ~feFpzs_
zfZ8vxLt^!$pweiAIsdY^b_USkCb-rHp13_NcUkU08~3+GDq^GN9DV&>y?M9zXtCdS3JKg>)OI=YPfK1hdSko3p!o
zuCW^V%41cKQN&Dr{jSI?4vFnHDA=aK7Av#Q>0zK&e@eIBp=E>Z?VuTto;r09rpA``
zsv~a!URfm9!PuG(yU4{)8#FO#-=*5>!^Y5DbgC~2A-9q!!OrqFH~)E5N0QsT)p+Xn
zj}M>S3Eb;TQ+=IQev<0lrEfjxGce-sDe-UYW^^N^V6k!Duo*?2lq=`2@mPCy%Dy3!
zw!Lz6y=y(WxNS_uaJDSLkBg5m;gRM}e1j$7Em1vWWIFrk?dDM1t+9Rg@z~o~M{(Vc
zkq;%SQydcG{A+Sw-1YVI%}xUp+EIPfDrx0Dd~5GgkG|)7oCgnf*%Xj}jl%N~i`qai
zs3lKi9X9UYzyAxZ60V-R!LLEA52fzl;{xo=J3l7!(IQ-Gj-}1_3s=^a`(>t%jxBtI
zBvNMy|EZZ9VwC4rVpt%HMAhcx(~rh$JVL)s*-USB(K7KDW)N;d!)mVZrt^mii%VJVetj
zJ~QL!H+FUE^_a;5{h%pUwH`fGbZ|48M%_wPX}#sGN(#eDGBy3Lma1!L94wk==4s{{
zkd&N!o@>jw>YdNy>o&D9=iJn0*|1C>s=uPx-8Fg^7XCMlC(HNl3{c5ezp7tyF>SwL
z{fHTBt4GGvEIVAY);M<^3BzhYaCaG8f%u8&S=xI2N-Vy93PkA$T6O!$lbEb=Dg_bOlMg5=Ei3-ji7o5W
zEzwq!9gO@sSQ)R+AFiSz%?r9U=W!8wvqJ4Id{$1(Y~Xjb%2=!`3BL&3u(hi;ueY}+csek
zCkrNwtd6H;A=Ci-;r=VITS^^DCFAgeYCq7WBsat7gKK$6
zr)_d`7UXx8+NaR^;4c)~M7(>GH>=t4z5nx1{p&CjX!E(oCZhe~!iJt=F;oj;;r_Y4utf`UR_7khVVm=dxjGC3mKH|@Wj8c51grYzKwEm7l-ReASxNZXG*5PMY?rw)Tjq
zRD<$ryGA6QKHHcIY?&~m!|KvGiCdf1P2_zpTk-5o=U1G6G-qr~<=NPBmk`J5{w@d+
zdWA+fj;XGS5-hOGGr@{59g$JyS-r*5{76f7pBk}*FFAQjs*OlK6|>&BTve_PO6wZ2
zE$OvWOZEBrZVTjn$o{OK8bAIQurkr^axH#G?c{Vl)Ly$Wa}b3}=GoLqaUS`V)E<)U
zD0A}Evg@nDf_|3JFSijKv#3J=k^O5T{PdbepTe>
zhPxjvwD$yPD~rSnsXJBP=Or^SyDi5Sm`4w66KNE2neBUCF3TPqeERXONmsQ%E=oHI
zi(pn)P{nuWtGAH|z-V*d?2L>q#FgDM&v|OxX4Mi>b{RPD1tunPZ?2DEU7t;{f9;}6
zke^;z6+mgz2gWlMTPZH9f|YDIP_^dAcO?g*ulkkju-7LZo^8FI_&c+}Xjh6wqZeWr+Z{1RMrovok1Y5H#BEK|7~5N`0fCL0HsQ
z1Xl<_w%2!UT}KNj0&XSrF)voUYwXm$JL|qv-m8rr+g%KNbi{^tJ2V4MIpx5K37%gk
z;r%Yw6=V3oqZ&G4LN7*%RGu6iVsC-N)J+Vdl^IMEiy5g@e<|%IA6iPTi>56ZgLn0I
z=%;y~KfDAcyK-gxH$ZP4e%msWI=-ySHhHa0;+S;
z!cWs0qD}5Qadi(d4)B6TYc?zt&5JVnDzTClfBj!>9sKK5;TOVcIT`AnyEu$o#z+o|
zp^QX7N=a!i)CvsW?{CSM~`ToO)X|rbi
zgvL`=Xao_UTHMQdT7JttI6^URFT)_6CTEEGJF#_?QqkhYq5DB<83&)%geo`A_uEsw
z#b8hVb^ivfZQDPv(|ujB8c
zXTG>NzB}#H4N|LclEZFKNac8!d%^7mmn-~TJby%?m@gMHe=q-nu72paaXsix?*X`9
z#;YU%<{5~+FQ^ts#0FOMdf+~DYmz{&SQID_4t{qz*Ib8vp$HnrA$n&6_p}I;WrbH(
z$kZ+{y`W})>EpV6Noz_|`jH$3JIKmUL1kdb$oh!q-&-aNS6o0ci(xJu#A<>4LZOma
z8F;&kQVO)Jy~*BC;+Xj!>m%q?4!}1|gIj@Ey@QGEtt=yotv*ptH0HGtTc~edS&<>O
z;DQfxXs4T-LF6GeF{;s-ZD~1Rd-WhYVk_P2bbtY{YQpv|%9tA2>y~5pA_jb)_|X`D
z6jPD-1O$!((Ba#t_L0}aJwc?NURluv1D%G$e&LB#YD&f**d{I&imunPPq4{
zG2}@!b4QNbl{FBlj
z8-6u1Ua?}hsP;rTN|!>{C7oY#ecK?m2Vcmav@C1J;u|#w9O=UaX{rpyDmoR)kKT^2
z(KOIkg=HOX_7|=&AGV&O8MNQM2%vGmfB`)Q3^>G!F-czSuZO+7?h@CX4<8?YcY}>q
zcK*WZ8Q4kx9xsFw#Su38Ee{u2-?5&2H}`tdYmlQmx!Nk8kAfl|WDrWm+1zT6T2w3r
zl*##8vuw>*yE$Jb&hoHd6K!z&alP58DV2g675juAFS2s#{0C}H0M3raM)@N`Q}(T0
za_3d_h#7~&^g^bSfp{X6BS&bkmI5)|YC;v+iHO=Nq$2t3ri&WLiE)mrWE5i-9sB#O
zPmAmio8-Cf3J12KU5%~IKy@X~z0g9qzug-id!b}Uq{H%rvxn@o?~eb|{Z4lTE9dg>
zwVitV$hlMd$a9RszIgR&kGJ#t&!<`0tdfc5;W{LU2M-=BfuT!qZ;UK0KKU}?@<2QLE{!%+1)QWC
zz;O-kuW$U9G}W%F_BO>!%@ZeXOHW@lDz2ch*Orz$EPdY*A3vEI=Ljgn>GMNtmaTT{
zS)A5)?!yn)z&x0lDub^Buq}fsQXlktdkGCr!@H9HD6k<6bqHGmau;*2g*NtGX`n<^
z5*QfB2bs#3LWxNYaHpDs5f}d=z~tFkDh68xN({`i=JQO49I3wNYbyN^Sb{no65T#8PRJh
zs)PYNg~9UjodJLEfzi0^abvykd%FIYj9Hl{tUr6wc(pr}`dwzIVttvz?S-hq;P;{S
zV=cCLJ|4L(U9eenIX$_`EFQCW9$U*+aB7?pN3IUJSgY9o(Gt48M>C9HJrV)Gsc{B1
z*#dL(eN0o!jU0(uuH~`ASJJ|*Cr^43LK1LrOSQRl@uDDb8fprp2#|tLaD1NTMX^^-
z35ynKyq!nu+snOjS?CfvPDK+@R#qlSw6tDzf-b3zMA(}|Emu(ohy14Gmw|TM#@04x
z*XYP4pb;OP3iOHE~7g5Ck_4=J$(kj=|?;YlLvc
zqE)!EX<*kC@4o4V7(F~SSxRU=&8J|2M^Pp?U51Wb9F>mF5iN>XY79??lcxsTnv7O`
zrb)1YbOGkeNt~jg(VxgRn^su&C8>d6;Jdd~`Q}bxGGp^*RUS!xuCDTabT+m)klm`p
zq+JTPS@T|@;Wqr-+5DcgZ*_CYtWgoiazbO{nwH9oe~xNXZ|>X}hGWp3S=*+@#=0B}
z&gk8Ex-q)ya;oD%i(XU=zb}bY_TB}Yt(g&!IrG)8j>0j`&C=)Q_Fo$)}=K^%(%zkD%a)@8X6Gj}t=9bRXP#VoL@g~
zBQb;r4KeC;rfIehL?>!6QOJPuv?JK6e;RV^zDUhbI7mBRfJ|Z!T%fQO78*;k+xhu@
zXxH|prlv0JyMJce4jnpN;?q<>2=B1|Fh}l}20&ews0-y6iGjLsKZf3yMwjX9j`h2Z
z(u0_ZGH)79KmS_)%DM=X&fuhog;BxIS=AS10(cB(`NfFk20~~sS}Al?(4M89tG_ip
zHf*u)<)r!c%^L&{ZO$}l$~?Gvvkm;`<+wP6*~%PSM2HI(E)*$D5*o2had%q9tkcht
z(j%OioG_mc3wwd!YmoZwi7UU_z5D#s`czu#q@U<0gb)U92$?HFn1Zk7;51FwltRTm
zQn6s3OHVe;wpeFlGemQB9b;VcE2l(Rryk%Bhc*|mGnRKf@-b3r$70OIJ6%e9@KxdW
zc2izLeX8WE%>L_jXtv5p{ddtd;^5LwB5pmJGO@&D*PcDohkkLGzKdZhG?ZozUj@iY
za0KxFRmd~fJWrFw*{!EzK>n+LSD3hjb&v=SQ0(RC>;!X|!W3h-SIVlYeVKEeNl6te
zRNMkpV1(Fyz4jhKJu;&HA=h}m-Jd^6cz7EyP#&`+?OC`gD{u}l0#`0DCKjhU?{UNU
zyl20D`=isEI@1T_nHL}gdR(_`&6kP2Yi5)OxX!3EJNC5P&cQ+9+meL~LtWKTsq91l
zLcLgHxsCBgI?~SDKm3nXAx}|G{y(Zh&WWkh4C&54@uBuYBjzST$S<4ob(y?aE>1})
z8H_#;{5y1EU-nFJihk%k-i1X*>b+L&*7g5l?>(ck%(ia9*N8cliUF`x6ai6`AVEM;
zL6T&UY(lb15F~4v5kZh3NDxVqRf3W+q97nyi7Jwl5+&MmQDs$~``tUnH+pp6+dtYd
z&N-ecc*B17UVE*%=9+Us#o=LqS$Fu07~dd_KK*>L$LQ;8o>NCKkb?-l%C~P9Hbb$0
zA9_2tKybwc0psDLuo)G3arMfTFBqw1Vz7wV%t5pCkIYO_h_;-EEXPtj9BDX$RN3N*
zQ`ZB%al{ZRC;$gP2YC$HPa7uiq|RNI4BPP~53DBaT8HKss197ae*J>{=6xC(D?wvlHGz)UB9^8&sJ)G+z20n(8ipn1t*#Clr7JHt!6IT|jzJE5ks_rF=
zKf0!9XDb~4h`uf_E`)@M{Fg6}AZV>AeCfw5u&*%s(cyLSFW!&GtajLKxN|kadsJ#D
z5GrLeEoiV+IVwU=cL_5y6Q~|IFB^eKQW1Ig?phLIFSV~&?w;U!Y?Zb?+CXlA2mop>
z1+sX})3Z9=0HGzUpBE-K$Ai;r71OE{!m|bU10+R9Za&U*M07Nb-?#rc-(~R|Wl;~_Lgp5tlT#xGnB@12jpcG@6I(MwvE1?dFD}6B
z1@9f~Ac=F2H^bt8L3F6gan?D^;a|9P?b`W>d|32jZ@o9IDd#@~KI;6HTi;(p&A%9b
zSt4E%z$blfo#s9%Pqu}a_Q5kfiY}AL={YqYKD4E7cScYo@+%T%1k!xmZj|OG{EKRU
z`#Ez-F}4ZqBS`KRb26KG^8cozZ@M$WDR6q{#qTe_E?#1{8;JzUCI$l|T+iQRAwfv+
zvV`#Mn$Y^XI?D7BhsY!wJF0X}|6imDh0#S~$2&p~U-9!tqyoCags<`%b00SWPIUyW
zu|Zk(d>{(`YRrCjsmH9BAdpf$`WPt{F0!L$W>n7}M~@G+K9f@Yb~B(@N@HycU^wEMz#IDY~;>k|+WtW6dadd?n7HQ`z;Hn4;-HDJIgYG!wT)$#vm
z+NYd~onFbh8wdI?-Up^~mQA`s%6ADHn+G5s;_EJa&5@Q>aRLS1tSY9a>qVh=nzR7s
zVXgz6=Hhcm=NRW9+YOdv#cF@lUvSG@?C9vozIpR8tlY83HEr%OuE)@phnnv*5H>y`
z`2#;Vc%w_Se86-D(9_F)UUI|zPMCd^I`mhIckal7t$`ZOOMo?-6@s1`4FLTs=xdc1C{E5@_FD0ROe$+@hFWF~4DX*?S0Id`W
zG*9-$s0EI*dhrC`=~F@04&k#!dj_w4sR8Un89H)>g!ZFF#OJE2THc*1ypTm`3F8~K
zcookho!$Ji&Tr58K@((1hF^ff@DNI1S&*s!frIlM
z->L~Qc1^EEr9j8WRS3tp6##_4Ag2wH=#kB|Kkuhq-Z&L;FowO?>?7n@ETNck(-2YZ
zg0K2uB)5o&cI$u@T38GQQu%p>FxXy%I7uH^dF#zhNZbH`BGG>O{;1!-8qEzt3;%0J
z_psWn@m`kf-L{%ZEu{NQY+4@PSc~Yvt0Y{5N9h_Ooj~{m|P_@=WqHwv1J1Q$B
zV#pAs(i^ZV5K^#luA;R10xf_Kxvmmm_*?*kjKRR*5hd@pU*SY+I>GOY785{}JNn9j
z=8b<9NPng~x-v4?^*>jHD}$EX$#Osgyb{clEMKu=ZXq#U?-kqK7nqIS7S!li#GO`A
z*94c_A7G9h2gd?%k{~3Xu3aPP7wdTz2hr60_kp0jfNL2Tu7-JupIuh
z9eG`Q`IBNi43))Saz
z;|1K#Shx<&LOB3b*_PGpX;~@D;}>TjN~agf?Q4fz*pdc|!+gO}@vc{PMob2HuW#Oj
zR0Jed&^M5H(&RcK=rpxlMP+8|w9pn0&+fDR_twrXsrH7*d}3yEI=se?HJ~NK6)^kg9k36pDWJ
z_-?-VAA5W!ay!<#;mEz%=(+otw{z72F&^q9rh4cAuJ~T1b9!}_qO<1>hf#57sr{%<
zcn;{qFGh8bv}o|k(y12dSrJr-DI$Ilu%H6=bMk1t=bokt31>7-ZAx3mE2TsG?yGS<
zg60)t&xsT4*k^z&h)4N5C#OuSi$bMK$6blE>YBiPjuz%4@g>k*WiaqC;;bWD5l|O0
zo}T$U9bCZeTbWg9L84B$HNkgqE;1xkc^GnEEEO#SP7qRA#(9jgt^|xP)qDHTDtFyf
zihToD)C`OgQ7Z-t7pSOAOs+%KmZ+}JA7gG5GuB9CXf_SF4rYP@!u>Z|Y>Zu}>Ud>>
zh#L?1dKv?>naiY>R07DnicY~=R(}U}clpibt4#RO;Q)zTGDd9!=Dy5CRauw^Z;t-g
zjv81pGXKl2SkP44jz8vqY}>t8{)T??+|E~35~{o@la&#@BYX$!KO$tJr$`N?rwAP#
z?&Zt7qjN_1)SX6;IVP;zR~)(uy^GhNZT#*lOS4Fl+x*SqH9Z;XF=|NsvtNtsXg*+$
zLgxyLB9O=6LP#)OwwUU8Wu9J$6ns6)7#Ik7m=Z3QpqmLp?>}f9(Nr0i{v&8qA?cir
zk)D_6{$B@CZXty7=y0KT_^GXJOWlc?v8P#j8*>RVfnqs3_0GV^K;L2-1ES+cEi7(Q
zlPXe0;La6@R06A+9}3FG8L-%H^5T96k8Tg<>j<_2V}K@90GX}Yacm~$*dX$5iJb8^
zi@%%OJc94RD5b`eMw0$bgdPG}03Xo!9;TV;hcQU!FsyDpPl6!?&AQj3c~qIgyaeQ&
z*^o)4&K%GbVw>Xysmfio`y4M~d5TB$qx@#zfas_^N1y6jlIz5LNRsj4MKJ;61`-7g
zNNmp@@Lr-XZ*L+~qvE^@zT;
zatT;?VD9Xx&e}XJ4|g|OYzuE@LBz3juw?}K4TK;p_O=NxBcUW?}g1)$k*nF
znt4_MEO`!6*fq5JpTVlB>EuQw=NW^P!eXIO#@@HfS_q%Ad^oCKM=eRE10bXDGfm29
zW#LBAX4TH1`w!XE5^;m9bIRQBfXshJw893QIcwH526d3uuDwB;wk({6ZcQp$TGyep
zun>EM+H+{{Up@<@Dj$Pwv8gxj^p$z1_>SK4_TKQd89q`yNBLNd6A*Gz^M;
zfRVtroT7|XwJvK^#1)o?uBkOE=N%ZYj4H8*e#9EEez|Nks)M|RBqY{BtLeEqh@B0Os+#AN&MYkpr!!p
ze8xfvw?nvQ7JwQ+15~IkF{HE?xjMnrG`sZ#6Q_G|b$I#vUq$bzO<+SI$+7?@B}O$U
zfw%B4kcI-CO?{k`>)w~g-181I_8w7x*gelJo44y$$Mc>%dm8@d%y|rcXa_3_Tt5#S#~~g5+80*%p7)>)A|pCgzo?F
z!yfv#ANCo?f7@?A{SW)?y8jT_KlLB>+fV+(e!KR+|8~A2XF5KA4g=#B`!9hmr?ccPp=8vzHOZgR#9CB{~}q!8sVg7
zWF3OOvkHlBk=7tcHGWo$j%w(8zBmqVNiZase@q~Gs07$p_`7r>$ta-{sxxtNXMhjb
za!8-_a-7oLty-Rr4`zcGPx|(xZ5bG78ju7bT;Rkc0EM4584-iE2xxNT+lkCn*E_mT
zJMn0c=n%JWHtQwJp2O?7V*4Ce3XYE*Q%twL;_vT|nrlj>6`@b#Sr<*K0K4C&m@L%%
zT`13mCATl^@w%&zYs9hbNcjx4uqFyJeCJs3LOU^3R*9Yh>7RWi`_jMU#z@i^ADdOe
zMHe|U2}fgyc56RI8QT1GPLWL!IQr|MvKfr_2TdOX?21DNYsKMQWU|Pbj|>k1uHQ?*
zPvvRUpzzO5hZT4{nv>mi>8cPu-oi_^Qw9F>)(Q9T%5}HIH>2hmB
z`N`2qq-?(&KaqeP4`>B8=LaKanh0F*rtgi+xW`l3I%G
zEeAD`x>RwW6|c1-UxZk7yW>;DL8qSl&M_5662$OYP(>i>8`zr3JR#)BVr*BaVIL>`
zDki=;@;)n*wC~UmDPjVJNJ=rFKI53zxU74>fCsj#n?C`qFB;$(3@n0(bzle{S~4=s7Q5z7^eib=8W~I
zLT2o>2WuSgsG4jNU-n7FBB$m9QmRCyOtw_<$ts_LB9Jj@CAcw(z;=XF%V{&uyplFGBBo{DR#MdvYDOkGYorY^mWRXM8^m^!CWg
z27m3q*Tb4AgS}0TTDZ|oP*rEZzYa#>7OCc#YbVdAH*ldqwq1>~D;0DXfC^h6WQC`*
z^-P059L15NlH#(6eP0CB1n%P;n}J@2?-@)74P`iaLtJ^%j;@j4&s;K2s7p`Yrl0iy
z#0qk7h*b{1LI#866oH{S!LfPt02U)AIed9JH=CqBgBL!@x7^~0+CT-1(%g$BB{fZa
zhbl}`!LINAF@wu$A$O(}X#EbNKl4)|k^Oki&
zi<)16hsMLXi*dA5@&~5qWJTo|N~g&9ie#xPZztjuR{-CD?AvT|EeB~z1&=xvnn#7Z
zkzYG}f5qI4eMEz5q#C!ioD}UK&C`L_-_~0pBQhmfD0#wx-*)1i)FmbVKo&0^muG^3
z=2O{&RU4j=IU9{KU60zde
z49A67q{zswhSjs~y^DaM1EwBu#ORpf0ccifSzQyJ6F7Xm944%2dtfd-25FaHv|`aS
zbxo#22;C8Nr|*KmtK=QO?b{57c`X1(_4oWqfMz>!NN$~Q4v%LSgPJF4hax|V|0C+v
z+jaZ`g^3Pq>to*sQ`hX+r_Dp$*A
z!XM~u^4N7%-1`{7Rq)1q_DTCe890}s`hmIfeUu_?_^uj?Y0V^EPDVtS-vsy}g7kHp
z?$aZhqW~yCx$=Yi!&$UZ?lwkaHb~9b*!V(Yl3r#MP~uu&55!U^5Z?zx3O5pY`gIsb
z|rcQJ1ALC#`SYuoL}YVbEEP@j6L%)_?wjH#kjt9d0}e~
z=;MmGPG(^*>wR@9?kd52B01*aK%l~f;~I%7qv2;d$Q3^B7T4RtaR_n(+i-8_<+|vR
z@TsY3EdG5g5=#7DCx$#@wUVRQJM&_^hTg5{zQMxRq=5LlYOXzCk2GGO7{p138{?FR&WLxf-kyHHK4D8&!jqQnY?p})AcAgsMznB!p_(#7cO!$@
zCQ>@bOf(4tojm=AU6);E*!SO8K7!xx4c
z(H34_CD0l)ARlEI0-Qr3NJzQS^=Ec@t>J*s(+#0gg#n_ZW{)NsyH<}!QAyjRd3n@L
z`het?Qmz?)I1HeDYF_V9)i8RBP?EJjckTiHJxMr|$tij=9XN!E|Ox@G9e=!dGjm!_EmuL5cBi7cnZ(rM3nuNEF6p~H@!YzrjmJ+D3a_m)U-xu={o8^NSEd8=q8LcSsvI_Pu?Vi72LCW|~lvcK_
ze|M%caf_p@(2`XKj_9wO+utHpvk6
zugDOlD@W%Br5Tq#f6KF_k-`C*&bY%H+AXhz4?f_}7XvSg?x!4}-q?Zl-&rBXU%zaeN`7M%Hy69%*KGvCz@@)wd{#NfjC#
zyFs*Gw5RhTM?&74X2>xE9?X!04UHC0%rILtxT+A*+ZQR55O`
z!>8osvcr7X6k|4U?Ot)X7tN&m;$82;(^)*-|f$jo)KN8g^YSSrscu)*itX&(2
zL$&L$ywilEtirxqR
zG!V;4w+V7m5==*ZNp7B$_K*wnu6mo>n4s4`q7~j88DHa%TSRLTFY1ULC^T+k1^J{3
z%-GQeT6Rf*FSHs=`#i2&Y>;W790*{GZSFMaI37lPtishsqT=|gDFoMSTl=qbguEvY
zSY*1|Ja%w?#AE_(0WjAE`+tK>?c79Yo7M)OJOF>P-xfk17$-(M#aQ{mWLc@@HNL?&
z2OYeRcdMXdO+GOsivY>ZnM%_b4`wcyb%+E13|D^}QV%M!&I08joo&4Twr|01xLptT
zxU2)xh3f2BKpKL25tt>IbT7jX^Vqe^q;p8XMhj!BKu5g!L8y+aHbWh+1o5LsvfDA%
zOMe-kc(c*-aL#t+AarENeH-%T70`pUSab%UbC~tDz2^a%_@pw9
zI&T0yIp#a3Y+R*~wgMmQ8yc#@NfNNA;Ya?Y9VOBKj*k@d=cW86)iDOk7`1Szg>k(N
z7^bgy95k{)(y~z~^@$jtcH+&djc>CHBIAc{{o(=$Ot|#}-6O*=B*pDm5;C7}POY+U
zH7Sd@URcKe{EZ7|r^DBSBQ~=-F7_jIChqi1cK0!R`(X$+v2DWY<3T}f{es8Ns7-E?
z)-}dP%@BqcF)5%I7Y&U|4R2aDauL%gFM4L*&+T>`?Ri
z&}6^W{1YQ2RJ=pZq)Rq4t8gu^uhrPgtnS3GKZITeIGkV?d;5J#1X}yivCH(Cr$2y~
z#Ih^XqcO)`u5&s&5_-rHAM8Uhe1mgm48n!R#f6;1slv$Zp#Gx?nr^)QFdrTj%n3TM
zp5zZGH(qJba-koU$vo#*8k0BJ481~1CroG*98PbJ8g@EH$g$DHpj;Y>3{S2nyr27i
z=1z2ez>@(z8Wn#~(azA@x0}8V-_f$x?F*?^Wzam$Bk0%N|06xGZE&~=Tb0K7Qb0hOV9?m98^mJ53msbE{aBnOnT$~
zcRHJ8GaDm9(-Jlel@F2RH%?GEr7@x6*?CWv?}Dcyrd|$DcWBY9kbPYsr}bnzP<%~n
z9CUA*zuAd4H8s&uqaB8tk=%>szdP>og&#@lGKM%ndYtwMEL04dvm^^oLUm#o`JimR
zvhPp#XSC4$8gb4Lthd7p!xR<$tJ#VhH*ORFaUKbgLcq>fEjKrUo2G)+4gZNZOK6kf
zPEL_`L`6oGIo(rW{(4&(Fzc4ih#r{{MJWK8tf2w_?BuNmuV10Esk;ocCd#?g*SY)%
zBaW14Z`N%F7lW}P$mo0W@&QyX%Y`_FG2xFk?N8ilE|15-H;P71CR5;?dcXzN6Ma0>
z`t|-$={Zs{9_UY&L25bvWb%U_U@b;Bg6z98{m{@?MN;H8UY%m1>K}m^*rI5M=se2>
zlk)ywYof)1J|7g81UEPqx65spjC+sUqUFefnO%lZ4&Vtz_V1=Wr_agHU(uK8)u+N%
z;|~m8h=tz*Ne`q?cwAjwCpry&at)-`A|QBMT`U-`v-U(;8AVg#H2H=CmmhswXOz_x
zFOZ5P2<26xds#Ai)?C=e4rcD(MpmR+jzkEi6^?alDGcI{zS!<+Gx
zb_`EcV8fCP5LRtgO;lFbe{3Y&yjx@p^=)q)x5-`gITqKS=+S{6t7SY#zqMwpHWH57
z>5=U2*ICve9^SOc?np}T4-ob#MIzXk7P(UOWXq8bvL63kNV$eCxV?s@ivFAXxw8~e
zqPOj~kA^i>{$tmhKK^bWa!|iWlQus@GzKB;pq0H%K%zfgA4Xpw+{NH!>N_~at@EHu
z)iV#(C(=_&oNIJ7u_e(7!tDAs=v_*KF7aN^u09g$XN*fl(0?^5MarTHa=bJkaSB$QVS$jl_B>&#HQ*k;eJ`4wam
zjs4$Et00>+Gz52Fn8(;N>%^8b4$TECGF$i>BWbi&vUl{5f`U45f#yTo+e=Ps`n2rw
zMmocfU`mE3bZOE^5@6hRaFVNWp<$C5Xz4%msz&GC&3V(Ugl=AMy3K%<&-*IT%g;GH
zw%D+!)JY`Mpzaw6aXvvo@yPRHDe|H~z`2aVT|r=e4Vpkxyf>kk41S;bZePdKf(3X$FXt34JvbB2C&D}8EoPECy0n_ChO}W_q5;W7~#`YJT{c03KUKhWFg7CMySXPnRQ2f<`W=FK6maI^PO3Weko6rmpLn58rilT2$vYFi4VB+a%}IHku)_m4eQ4|x=85W
zwYEKVF*t$LJt4${D70`mHvyNJjLwP6lFEu@v+Yoq^PpQp!?ue#pEM2-OnvYKXb@5i
zd96md+~WqR{rgvG7i-Xi3=a=iPteAFobq;OH}Y?9tKOBqZi|BaG=0o!+ceD!3kt|c
z1Lc!kvwA?wjX4KqU3h1AjQMrx#IgNa6&sE6X8}TbDkxT?SIbr4vO_yj&T-7$!(X?w
zM`O9v=v9n}#GqWtfbto{SWa^a2&hx|2Chn%W*d0$8vXvZxiS(KItNS)+zG)ztayNQ
zoLNM%!KB4{Wq9)|<1v+$?;4LIuZkk?Q`k@4`(TLtk>_WLL3IC4|BX6H<~;Av8uEo@
znf)^UTxbyyh6jkl`iBK;OH)8K&yzYhcESpIUniFtL+UD
zsnS|K{%%xL6Z*7C29Z_`oXhwqf3@sxWLeERV)mU0rK6|B*F%@*Ho)4--=bS?bhZ4}
z*tbmRYy_KfqD8m?XB{FjdvjH%w=JZ*WG?$RS)Z(ji~EgSfdg+andAO
zT0&3!&Ww$?MtN-5`ppjZ7jIw9y)7tz
zk)xKV1U9sn+_moS#5rSchf~dC$KcD~hli_K?dsOQ9oofTmXAHp7n>M(NS)i^p
zL54F5dOaD2Jssc7X4K)e6tR+?3`Qx)zsdoSAe@@?oFO3ux45eVp049B!XD2BEF*P&
z_90Ie;bW_9aP#K(;IgYnroZxBkrcoQ8euY>F@d<=O%6TD70x>3I6Qx_O|gQh
zIx1hNFs)C?6Gv|aE34ku-M^lficCu8kX4#fjrF+OF-1{mK8K^fDd$ZP9o!JM!08dS
zZY3Nc%oD5CC!(@z@zj*)bdl{4JLirakKS{Lp>))jAe-#
z2WZi-%Ygs!(>%wj+K;rbkdu0|!wdX%%WgOqE2F82nS|
zkgsZhl^1-d(5dtdtZ+N7-_|*met_^3d@m4Vm5&H(b}@H&3H&;^T2B0?bwI*m
zowP^jQebHn2$s~DQ?ev(?GAjV=2y-D;$T7Fpy$sfFW}&B|m79M5J+Gw~e#;NL?xO4Z;QeW14feF@4=B9*ZPSY-Y43JMEa0jw(zTnAr_&}F
zg}hWaD0y*!)Mm;9vHB!Cur^wy=XyD=5&_U!DOgjsPqN{#n|y!=MtK
    I@8WDH
zds|0ayNH0WlDfhGy%HmE18`KK-Hz%zR$RFosGEYP`6)viy
zfuYp{n&OFn;a(4(*8yCK+;O|DoPFxV_f}~)4xlQh1Fllp1f0xUSl3*}zb*LLIyFbQ
zjS=#`ZE9lsh0N?4m~_ca&}!m(EB|f6ra@{SM-N5q(oq-LpE(SNh=KFEww(f-pA?!b
zan6>%J6MI5KzPNewE$zj72SIwX>AUYNd6}uJR(*}Pf=E`lL(H<7g!rrMqh8$D{&
z7V2{Mz)#odAqYsIz0AwcC){FU$NQ0YROH+{`En$nv8~DW`AYppzw2N9sG~1Mk+HGL
z!9q*_g-J)MCv0SFE_b#4Ao$ka)_cK)wNqa6)A%HWb)kAXJ^Sg*`udkcc6KjV$JKs8
zWAu`w=h8>sdcGM^dYexT(^NVMT!>Jv4vaamg>!UbN*_H^8yyB}RhWA4&{^`+62bsL
z2sa6txa>~iBlJ06zPxG^^6_Z$jHaRWlOC_YOM)Mti}%#bIPP9s_y`1gD@2f}ySVyRlBeW%Fdhi&6bCZxZ|(u@3GN
z*ST}mR8@nUHyDlM-EjAXYBf_N`5ji~1VH^aYv#;je&<56PI1)p*7>i9yt{rq6X5&W
zOQxS^_>Lnk5JLUPkHRU#Jx~z&`6+DMfnp$FXK66bG74SODTey`Z)ZfVjZ!#sg1Zs>
z8HhjGB|rn%mm_Ad_xZon28g_53%-DesST7aZ1#2G;)sTfW?fnJQtJG?^mBlJOpfL8
z11G;?@R?N=V)XkK2>?Oz;6zJr8%
z{yaHf+~G(S=gkmRIqGOunCF=3G%?J$+S#lW;VFf84ht9T1tHD2R$3^@t2$6>n;~;;gGO*3>k6
zi%dzx?bg-+qXiTx+0-0&?tD156Y!dXD3R=&KmM4M!7LWK?aN#ROdfD?0UzNuIN|E8
zJko`Wg6VK^%P-R(y^43IbDK()QbtF7~gi-v{<;ZfMSXiWAkpU(eF$Apj}(7XWp
zU0XXXUdk_+%ZZ&qiQSDd>oh8TvF#x9R=^Al`}H+?+-BB
z&F!|PgmRqbBPkS1?8qrgzIoUl-
zK?uyvHN^*fQD&_;8u%%b&QIs9lv~ebNn4lAk}p
zZ{z^!-Vw(Dluw>i-C+%rTkF>Z?SNv2ji$f10&x@Y{WZssJ7GrqF`o303)e3jZDJA}
zY1{Z0w)0L@ZHwWgl&l$&+OJUl7W|%i6zRQfiPE5{ObZE8Cs;RlfDGVbd1bT5a8R9W(I(
z-gCV|*NTS^w@^OvYjVO@s
zUJKEBjJ!oPhT*uM-20Q9U_fsy{&Oc3r&~6xt6DCpa#QmJ^FOOH%b&X_EGY`o_p_N|
zZ||+xh+%{NTli}6w^r8V()@{q9=RXOVMgZCzTaV4V|aTk5q=+oxwmMP6;p)ksvyHU
zw!0huOmfvZ)E-0%Bp*Czpd@40Tym8hQD+B-X3t^gH<~5$Zn7L2v`UPO?C7l5ak78@
zY+`JTVd_q4>GoZ_BtFD@tEf%fNX}~!_
z1^R5DYUqb<>PCfxi4_%zDfp)!K1?zBtPe5n*EtBUEs={sY3E!u?kPMoJAVoG{TFAa
zM?4yy|Jj+9Y!KYTNyBVIHcKTjaSmT=)24BOcP~%PvexNsrp`+}`CE$-zh&b3H
z%@7%L-DRGhPB1XZ$-lEx7I#z7Fe#;oEiTqUgR89_vb@tiX+qQuylIVwY1sx1O--^_
z4skbBrFc|yY+rHx>OHIuvQ)mZ$270n*Z&o9V5DxW-5el`6DZqlG%@7&AB4d^`T
zTAI9zN}2dX-|Z!7+RC*t;q_zpA2DA#HtAXN!~mVB)y*#_x-9>4=GpnIgI?~P8Jn61
zhce!STs*3ml$la8&j#)uhr+-~-tx}&pg35YH
za^go7Uj3o{`?0S@sXRGq7NZf=S9VK7`c`k*hc(kR=ZG$K=D#hv(okj;lbbT|b#MLn
zGu;xZ`%J&Q^46dpM&5QWLt=Mb8`y7(*l%6UX|{8+f2}=cHhZAyG>08Ydw0%iPf(>1`YnS
zzpcXrW94pfP0nAR46o|4?P3+y@ErFw-O7#+o42lN%7s32rnGsCITg9D>xC`(b}1*d
zHyxjQH1)mC{Tf@Q`5f@R5+B$e`Sq&Oqv)ZOPm}Ea
zipi6E+dpBi@UY_4dU4GqGvd<7w(yV$szAG`%CDYGIl;U@G!Y5Oe<8zWs
zOhjH!p8YbsT&YpxVYCRv#uq4?Dj13_xp)g-c@O6
z9%seP=;&(Ft5`xl=jh$^&ExdU2_R)Oj%kCb8NMcW}
zEfni{;HUJ~8JqRz-N}!PIP%59WJyy<{w#+HV>cUo%f8q?Xim2~t@q|GZoY20dPV=^
zn;*0|@=g9In*2@1+&F0#(!=(d
z>BZfoD5h*D>u5rx$|mR(K*bAVwhg?{D>dw^%~_4c@v;x26W>e~E%bWnm2ybdBvh-k
zO=4ZJOsm;8m7QAUBlXfsUv!N)awcELHbr9HD7|InOSCOMm>rVcD!DP)Qr0H=P4E4V
zk{fmWo1fYb?e8PLN2~N@OKHTE;p*jY{#tpmpQMzANcB{h`lIr3ga7$KGWR2Eq9p8x
zrj(aH`S%Z7Gd%QmI+w<^575^B=TA%J<>Qa+_v!qfAJ}4~YBR&+a>%6r`xugz3;fl?
zoh^sNsmrPV_p?cEJmaaJl;)#xD)oOpU7ty5F3vYQe{Oy>_Mg9K8EqbR%gDs0u66i-
zACXLJXvs|dxb}zc4?_OuU-y1p@ceXzq{cu0bYqNX;jf-nBmcke=C9oaJFV8gUt+k4
zo5sTb`Olpa{&99)ZAN5?{m;W4tuSq5?*Bfx79*2sLwni{-1NWS)JHvBRAi*^m)QS&
zd%s*I@r11vaz@{7ipSq;vD&fs@V}qBK+!ECO*8AJvBF_Tz0yTlKXm?iwn5`_<+@(?
zG?{N%d0v#(2zZ0V5X~I7K6Ee@3}~X&qp%(O%xLtGW^VoRa5nXlf5VV`>3<$os886X
zmEzOd&mWQ3|C2GmpefLjvkAL{MMebw*gdG<{smv-08HM#?XF=$ymygu+_?!8>dOk$
z{{1OG-E-S7cA0VhvTdP43FImD3|$kW8x7{=;FjB)t_RbJG+46-ATg8pcz9
zpY64^Y);&Z1_KS3e;=iTnBlaMs;69*olr>rQ_a8?my)=I1izlr&H0#LzM`8|UoT?y
zznwbUBEN(Ji+b%zsRmL9={@Jw{C4!2PtPveWj|=!X7F0`
z?+vjiftbOS_I1}kd+>8Vw;!E?C(3$>BTUY(GsjFQpPG(dmBq{8$`}ui6zkh#&d*}&AJ@q7~uc>0&4h+2@b)Vvs
zVuw@#a4SSBsLKV8MkPS-*?;-`nc8o<6BDjbtX$ikq*uu62CEJ;aq6)7XRH6{ZfG^K
z{jvAM8#-hKC%fqPBcsH|$#3$U)RVY(&l8WEBYo>tK)8r1+2+v(fCoMt;ZVLwf0TJI
z{W34gr@89G-ys$>TQp?2m+&9
zuS9a-YJ6~HVn=+tN0wZT+2dw-|^rB&4IPa!GR(Y)PouhZc6}P$r{pR&dIrj(>M+=i#40
zuV$Q*H8H3{@H=z%?3GAnX{7wm&?>0gkf#Z>b@kWRr;wDe&o2Voror$bIdW6u_QPBW
zoc#Npdnb*S%C74a!1#2;LxK~gT)A>ZU~&PoK3=`^?*VF=tX0!laPq`mSvBj=G?{Kl
zw`PNvG8g#T_3K08>?jF)44(cI#EiNUTmcA2gn17Go=dnQapHS?*^%5-M9
zKp?$eQj$ii%o$fRq;JG~3h9#D!v5WY+9S@aVr+JwG0UX}G>{4VAi~ml*At&94tfXv
zIW}clUHuXbODW?Fyg_0MD5c(2Y_!PIls=QTDCF(Lgs&whMo#;O8^qjxG1opS-Dh#pat<
z4l^rn{q(7g>2G&4V{@qD?O~y2Lo|=}nDI1X)OP3Qj!753Kkz^N^Uhq6Zc5gaTR0?Klt1_GqcQ3c-=lu~4m$lH
z_18LwM7N*ze+}j2SCzYv`S_Nq=GO&x9+w#Xd30aUtgQX}p6D}VL4R1-3@6Z+$=6zbxMO=^YVs>(#R!VQC7w>Eam{e<Okvv{Z|#xT=*C6YSA}#MqKLDcpI_pHDw4_p03R
z>BsL*LpFY$D|zKdsj*G_J^f>j&buiK>3_jP99zo+{f;{7F~@=E8bT9Uy7&RE3@q&W
z>_kUM&*N!_3KgAm-q#Jk(sW#R>a4rf-|zYDPHnZ0lhs&ve))aP(jl47Q@leCsdnT>
z)aaD%1Cy-(-gvdY-^`dcV$DU{LL+e@hypGS&4m(k)dzGjjK4x-2dwt-
z$%S2nVCs+Ja_3I~GYQ}4VczsUOQ@ECYj!RC*LEY}qr-F5yxNh|#PNZEiA
z*X!K}{1&N_Urp|afdgNW)H{3Q0AN#iKL9CXg-Y;CY&j*06B+jh#>8xF5ksbRxQ+`8
zYoV7o0^a#Hwk23>9W?#mH<&3-P0@!JmM|
zPazKtBkg7ZwpPL49<%<%mh~LM+%XCz1g?x!7@#C$5n={Q!x@Yci?0I0SeH*e11BB^
zRKSVBna;jmUS|neU%!66V_ieQckF(6oGT`WS4?Gc{Y1X!--)Bt4TkPl^DxcevS;LU
z>8k*}M0!ghKG1BV%1uvHbf_y-KZT-&+Z|A!
zbW$$Fd-u`^3$K}M7|#@4dxiB!cbx6BD?4V@}Y`k
zs4&m8y0W70UUV|WyBt+nTZEy&MA|r!HOV&vxeV<5)y_2Z2US(J3}t-s1R4lsbPYat
zJl$}DI1DZKzrpCe8yx*yx%o05A2^TUsB%1SPBDz9IDkFM
zX_J)>6VZ=DBTQIh9&jlSD409VU;qj+@(!O>RJ5cFSBmez9R&=`w?*yVCv3KXnvf9A
zr4VqbuelP&DghrQ;g_5{DD#0TvbH`be=W4S$Z|jYY&;xAse4#wMMGuwrY~<{zARr<
zJ$m%ls{jakQCbB;4&YC6!&{!NEGUXcSx!Br8HFY%2WElQ2m9BGh)y6c$>f~nMt7N4U!A*ZA;Jd8AW#Pr?qc;e_Amz@VEuZe>{xw}WgrOU_IBLIW@;J7
z_FJkezaj5U|T6qFhKuL#+c-$R#>;>MkyvXl<+C-^+ve
z_%R8y6&P)dPNJ@^t|g!0TY|$AVl4hv+^tufjcg`|{ONz+d`(@?tok%!)v5!dS4zFw
zxm=qYh>Zgx436%;@l{XLyDvX+5MH8|{Wi^M|E^tAMIsT@JxZ@ZYP;&r(k{JjXdqK%
zmb-iZy(Gu8h;S+LBArwP-MF9{vC|MsUO@yNGG`Jl<4%S?X?uNOC9E`IV$s{!3{g9F
z5ad3f8fBFO4LEUi+=8Ch6`X%_Kyj$U99)jycHqD~{u|cH?+kg@2_#~sXO)|G80hE}
zqYEgBgYkgEWg9Q)ANIKK2rt5$?f9S^0X)6S$Ps`X
z8o6X|#5DpPl4*^>Xczva;4}FBGAw`uEYxxXnVXU)f-@<3XCsmy)S!cdaM+%}klUL2
z=ykz(t~K5V$!kl?7v%bTLdi8aU@F4G#FidD=WLiuR!WLHkA+kgjzDXjOHk4TA#7^%
zk=&@4dJ)dUoi`-gZ?1#n-|+$HGS4X%4?CUzbuPzAVBk+~q3PG?A(%qHg5W(jGqRfS
zgOQlGPL58ArzV`iVYfNP2%w$y0hjVoRNj2~fNMMh*q=dz{$dDrLm(h1H=@zA(LA)U
zFiEK8$)&P?mzoZ}BWCG0B_Gx*^4l_V=EDOkufinBPd(EO?
z{+YZ2YvQPpBm0c-lI)9;8R_ZA#>QaJaaG0eGT7=hqi(zkD2nc>)(6KsG?l`rE?;|q
zFuNMO_BZwDvH4TI4XS@||6Hyj!j}YBCk2%t3+nNeFO5SfkNjKL&f+T;^_>L<>XSeM
zF1dwR6y`@stv~D##)eCCl#%Nbbz|WYDTS8Q?LJ8n?coEwhpj?=!prQgi030PH#2i!
zwc4#Z|5UTO`GtYZ|FP=5FJMr#tazsLkk~SQsq`qB?@jMS6}Mr?h3sEKfHw?p4Q*IN
z^hd6n;q8nDg!`*?mOPxLchY#y+__M*sP3tm1^3$foN&RMuCgQSi0=+-@qvnE$CnSz
zc*IB9TqmSu@iImuJ(Ip1xb3gob^g>HB}T}HGA`HcAVO=w!SXlf!oy4Iu=pTum_$J{
z>mh_FWAipQXA*eix>>Yh^Rb-+Y*#kEf4|>W*XWDvu-My~`(_PO+vS^tK6cT1FOTPD
zR6tyxB(oxgCEfVzI(HpqH^VUqVe<3)?aqRLnk)P8WY{(Xicp7cDgO^r7A0jhqW$o#&qk)w^WSl-Qm6j1jlih6(>SA!_HP!|t+CN&qIK5>YX%Y}klOvA^VGsmv@#RCCx^+47
z%%QF=MCkuZU;4iYe;s80;O)eR0hin+PaZsC#3w2NP>&B}pUA$E!g-fj?hmWNx9D|E8BQE)K`#z-1ML>MoJ+A>oKk*
z*C>a)(WUB$ak??vKucc{qP(f6zl2;f8Uu{}N>88e(KA1Dr1S*u!5qm}FT)H+7-L%l-`
z{DMKh9RZ3pqvE*{-Q3Jb%-ord{ewW5xpOtD=2)x{EC-}Z5n9KMu)EKgz3*pOMBfl=
z_=GJ6RY)?$1GlK$;u8J6Ne{()F$0Xql#GlFZbd-9`k0XO%lTEymw%vyJ~{WoQott4
zp`BD3O8F-IfCbv8?3lDA54(0#qzVvi_nK!XQ6&2Z6RzJBh`Irk@2Tn33
zSP_ec69Xmw5C+STV5z4XI#T8anas*5MpH8R94Bv_=|
z>;fq6x^q(P0i%Q
z#OhmmKL713rg=;*6;NS*N>{tXbZSJu!GHo8Ain?)`St5pJ4srAw)gKBziAWlu^0HybktJ~
zG@saL&XxY46{PIyIk7=8gURAAY}tV6M=4y!ZNZ6r`B}${My{JROYQb3QpFe1HzLp<
zM~?WDmRlN*vcql}*@HY2=qNxZ$l^(TqlrdWS)ZuzvM@vGVA$Femm{?B@eVC2y^0W9JGv(5-d;Aog
z0RvbrzhG^cIQ$I@)jch{Yl92OM&#cCW9aQafTv$pO$D$6+mK6
zXie3O#TMt$lww!#Vc1#SjoR^Fo~?gDIDz@}#^KC)!ES~|ouiK?P#|*vV)%5P!{-q3
zKXBNvZj#+}v=65W6lLjZnf8g>r1tdpO^<!*O?CAwFNYH@`Ya94&m|FIZ_<3n4H&SH7l#7=CFj`|#MX2M
zhVh(~0yM)9C4ZvAX~&b1+#=}ls7Z`AFt)a(we>7QwX(7yLx;}Lk3-PKz`_ane!!Zm
zSGUFLRV@&EIaX#jA%iTR`&1loNvo+?ChX~FS%8WW?u-$>5&K9d6gvCXR?MX*Uw=}|
zn&cPtqPFLbEPJE$Oy6Z=EmU)KkdnzUK6vaacjFh~8UxTn0Rz{#X$1$No*km|ccnWmnHDJurUqw_^
zfPe>R&cD}Gm>fz;=}b6NlNyUXC>a^>_~yJe-U_D5*6)FFYhQm%8TmJZl{_X_H8aCE
z;!&fzT0|?NN#N20W*d#8Zjfp@Xw^_g9(%~>%3nUnkKeIl3dOd9IbAYQvh3%(&*N=T
zOffj*+R&b#S|qpWx?WqheE9^zJ&9ig1*R4C$h?@Ht>krCrTx0nEwp<~6Cw>veMMwd
zF{-V*mmEmmJAJyDvUU4_WpQ<^XA-M$#Yz;_dka~;PO;ijt<`jDcoxeLSKHs#AD@EgL
zmy%E1*o4}5>*W({I)BNP-GhwpzG*|IN0iKxg$t2bR{mN?U-(h2Lh)}_-Ymtx^Tgz=
z*=ayDjBQSTZ%>ObAAlAAwg?vQrf!c-!$z-FAW(%C0?oor_
zS_0GMAynP*v6Bb`q&wu33Er;V}`pLfA|@KcHu1nd<+e>OA1F?)U#+QXwKq
zQc98}l{7T1Qb-x8loW0iN;EVyWR%h}(k`Qfw1kL++zpkj5)n-)v=D{=^V|8I`#%5k
zIFI{rPIoTX^}Rmh{eG>t5k2eNIBQ`<1OaxF?m?l0FZ&g%%kIaI|6;g1Ds;vJr+b9w
z+A~bh8BB={ln=RKuxfbO@R?QX*?U|d%Mu*#v9-Egd$wSYfW$s}W>LS7m*
zO-r=6QZ40H+S^B*KhNl<+(|);S`Wh%1RKAt
zuir_nXs*|ObXg*C*NnPB5$^YVOVGOB0RoChBfx|o0GUkN@l$JZsn4}tq~W2^rVB>tN;iLStswlJs1+)|
zf0ot~%*UO$Q-aSfoiNXvk_haSb_M*bx0Dpv)2Gj$Syaoe+ZzzzdV%RQ=myFj?^gZF
zRmVV6qgm8y5+2K+mX)1AVhO&6LRv5lU=sm=t>S-CZ^6N{wy;ouA#)cKA>nUW16K2q
z*_2*bQINk1bVzvW$FY{{XxGNdh!NntXu*QNoSc48Swk6hbo^@kye4(o)jM}|D~2TR
z-=B+H0t1PJQO`R0$!({mdb*OUJfLjc2^O-O|42j_{=E_}
zCwc^6M-flog?GXECb?Jr=G`?Cx?Aw!z^o_^4-CpP_XOU1O;10FKACTYZ;vJ0tv=Y{
z1PviC&g3&GrKvco`(6F;b*t)5k*ycO^zupgGMOg$Czyiuc{_=)Kj}H`Q(7Yo{P1D)
z%rAQ=o3M&6af18LzJWW*eS^-YX@Xz?ZpP0c;!{n_N89O`pzu17d>EWJcLf0jng7pP
zdGgFpLO|YQ(F>Jeb)qwty@=dC_-(Myea_V;cm|>Z#_Q@*>F+}7z~Oi8+BH&-B)313
zoE7HQ6+7x%#*={Z8(b5KqpH2a`9Qgi8i5b^C3?5g02)0!gl`4q!ZTZUp8>tZ{@Nm5
zA00Q|Bz(G^x*RDQAyNF|@pz^9_bl)E8exQRN?#2Kz;qmvj*ft4x_TptMJxX
z5Gb``S8Ml+A4nY`^io1C{qykdV~$tAdD3&p^t=!;U9@LUX$38PR>TcvIbYgRwN1bl
z2v>vU;o81_AT=T3D<(6hEo#co^d)A?aeDKf@5_x7uV427Y{FaZe=h)pjU~aHirrTY
zU)Li3AW%2(C0lW@ynK&oCu@{QMxtRSuu$yeh+bo98fkDePQJRaaxQfmmmDF8ph}d8
z!mN<*{Ru)~U9h6?n^4MNG-UnZRPkCQpZGCl{4{$T_bwhW}zB~>7D8`YT5=H|D+#%3$yyn$rv
z<(qr!o5Pp=fDym*qj)ZBX0J`dmuTe~Su;J001c;4AGAY9R{uuUiCRDm2A)y3f~-s*
zOrR3KSn;yw!}p4TQRw~+4iMf*XbvU_zUyz$pi-x%Xk-bU;rSldvU7GS2Ii%NHU;Px`cxiO&x}y-17Qp6_0D4{61h`l7NjBRYD|+QRPrh459pqa?*_
z+cittV6a5*fWKT^a@dYUVE|%Y;#@ru`@m>THz4}>T3XPsHB{`bsiIihu*|KhRHBz^{_-PMxvJLj
zfipE9R(;pH=5t#i_HX&RJ5!9Vs2faM?Y+d>VedTV4PCa<`_HvI(*}+}C4lf;RFrzZIxoE9)
z##?RUGiSezFOtwdFznik&`{N|-&^!VHWhfAyZ=3=Yx6{>G5T(~dNU&;_KlvgzH4R1
zAnRpLO-Glu_+@CW&a0o5GxBE2c&XUDieX9HHVpE#yI@{axVUfHf#Plzi9Hs3D2D8e
zUEA_4qDP$f&2Z^IU%_ChF*#bMdkjrxZ0Y-S#z_AH@7L1z5+Ci~`$1Q4PtN<+Ckq>k
zrsw_G^xaq4JR~!H?H|S3H@)sipR_v%oYI@zJknSBh@`97A83L#_ryUiuruv{6nH`X
z`|r*1vF}%KZ;|<*U-=(OZWvP6Tno8*PPt{LqQW-4T4z!E
z`CaOfb3aqcHkjOwkv(_u`MXS`K7C5}F8-LlI^l88Q+=n2g_bXne(Tc`Vlzi0#(JG-
zS)Y~GA`@q<9i`HcoId1pez(4jRhPEQdM6j`Omcd9u72mbg;Bx5lWk|rlTBC=VAql1
zF8z6G{jTHh6K5oc_N*$4nQkjSeBw_8?*?CTyOT2!mUv`jPnzNI{j-;?TvpU
z$q{gvf$q20bFh3^(UW8kT0YJuVog7OT3s{&6M5aXAutnkv9mzenTWO3NF^BX7HF1q
z19Y!+q@dUnb#z)TN+O6rtV}d6mFZ%vHgpe67aOc01!YBnP6>i8kx_GWdRo8x?VrUF
zI(VY%QLaF1jl4&X(&)o!OQx5iM`cVU$EQ_XQfzE31u^YSMy~X&RcHoa(QqpvR`m#U
zIDs3ItNNxBS?%D66isy3QsQ7jgzqm=3++=PO$-ycNkn9XjpBodsAn*oUjhRkSLl7_R#~?~E+`G=!*1|gm
zmbBIwK5VH?86IT7M4m(8cR<5=VE$^1t}MP#`EJb?lOBO}M6c;QPTIsz1+
ze%?@-Y;q1R@%sDsBaS0)2v`XqA6%HBGvNB_+qiEbQ^ER4`~kQ^bVph2Q+j~uQB>~S
zTwp4K1J}a?``1idDK7L9jNLs0e@KnELpS`aSU?Uw5T->zrCOZU9Yn8$D8o9q5GaUZ
zENm3EB_)Y$LY6T?g|`_IBc#Q|xNgnO%x)9vKs=?%
zvB&S`vw2EmEvY&v-RMaI&ggvzw?}T-UvlcCG0ROxURxo(B6G9S!7Q)3eQ?CAqH*I|
zsi1*ZQn8yyb?F;FxZi>yfl*)(6fS_YxTfvsX;!XW_-B6oqjP=te46OprP@n3e29!p
z1T#3+t<#0#CMJYM?$PpJUORV}DrWHuuqr7=Rpuc7?W90KJkoj&ZfE
zo@tkL&I$M=)=UkV7tJ0<>r`~oXyib)Oe{;Gcy-%RgSAas2;OTX@(H$L){QcApWY5YmjbKfI{rmQToUy;al7##guTz8kH`8VdMLX0OBWm`u
zJ{dMalTFfBI+?z@8^JxuKVjTf0+GjGzqWqV4a%U4A--lPjRJxaULNZO3N;K{F)mC0
z$PpWcbpD*<;c?wejoF0vLCWv6z8`pEMeQ(`?&63Zh2squYZcP=WN9y7*-J4oTL5@C
znyBK@GB9G4OdQaI`pWdE~xg4lrF{1aAxW}r&uC3I!ufU$r>>vk)
zM;OgMBg|4=u7l3yKlhYm4y@yRPDE@VpitBb^n!#BfS}FP2)py`(I}^KBU{^F?0is1
zErV{d*;;0VH3LcWn~2wBq$;Z=MkNx4j{ucxHI7j_9jMp1kAPLwrWtH*F3Y{c*8uCb
z_g0UP&uj-Fx*#wLP?&ZPWN^Pfn-W1LZQuwf$`Ru4zHWuAiHD1wJ2qpM>Lxe~m^=91
zmzkaIjhH$5isOS}IdIi+W$--`${DTh%H08Vhm3$}VkGvl6@+LraH$D%V~~czE_`Ow0#lvD}%`FSfr6Tg<4{(?^e5BMcMfHf@;1UPu+k0DJsCXM(sHw{Mz`j}UKXmB8p#!gywvFF$th6E4XaB;~EQl0n
zT~tJS3eG*M8r4zv7*rY6ANu_2ZRLyf$cOHm
zbFaHwPpN3{QyHpqef&j-Ue7{Bv;`^BDkmpO7-{@Y!?!CgJ^Ws$UYxcr-na8eMoX6i
zuiF$mSKmxLWal;3S4>4YLeBED;;{XS^Cl=93bdOr%W|mwTJ`&7=f|ntQM9x?Xs4y2
zXkjfgO2s!yS$UpUfZKQPyW29qzj|e*y6dmN#@6Y&x{n*4K!v=2@^Yj3`Oe=EsW1W{
zV`9F@?U1-c>om8tcr!nWfik0eD`l@rWagHrNY~%FNLdbz)6p3rb~ihFD4ndPreqp{
zY#x+oTyV9SIMn0fMpjyL;N3_}gte%N0I2vO4uBzKZEZ??e)tjrFjiK@bfIYkR8@o4KuMfaHA^
z2~HJ~kor$fzIjCC&?+LNgb}kKT}`VH@UA7e!I>K7Ss+$^hCB_83a#uhS-YvC$)!9o3@?v;mZeq@xH7kFH{aGc}UL$F`7yr0~sneKC&Hz@Ih8)
z9(nfy5SWJ=Z3>^AOS>>1@SLC|;ryIqR4v_gv^&Qj+Gb7?kmq0LWiO4Jbqy*6#r3zX
z)7uJI@v6gzn?ze#%r70j2}}l^)LtE<4??^O-`vj~y1(P1qmy~u7!p7;eU%w9O{}Y%ND5I7TneH$ars#`RT(ih0Bu
z3Cc4VzBP*tsd-C>mp*y27D$1w^UH;LW5X9~o4M!@51w&JSCf*T5b`@Pcfga3NjT#%D+V4=e^A|c)H;14r
zjiym=!sgiQTbK+Dy+rd^+o!t83e2k1-#`^tGM4Iu3hrA%-|950v9egv7sO*
z(YQwy82z>V&`{rO!sG2{EZR~S^;ZFUE(R`%`MT<8&&ycy^A)6F#*(RN`=iu#!waC6
zHj7A4K4YV~*~4RN>$hms*~AXCEi;&L3Nw|cZu~E~!qpn`2e=hJnrs}ARWDIggm8j=
z%NX+pZ(n7qSjux7P5B9)=qiz@oOtVbN4`t`F8jeX9+UYong@>qcaP3j3$1^7tFPo0
z?hDgsg0=>h?njG*NYcv^IyRmWIJ(KNUpsoJ+b7sSMWYk?aUpRdqir!TXpuZmGWFmi
zG3&Q*9g?O>^?D;BLYc&?2XAhrrICs3GWj#w=$k`5L=8xGB=DVR6_RDB$VzrS^eADk
z-2=#kK3X;=rQS?wSzg7Yxgky_qc$aipMODU<+5v6$Kyqm@|-`kbAyzWq$btOZ?N)m
zF>lw`iUb|@X2lwflI#I~++CNvC!eOZtbWkPHP0?n1~9&ae|P83@|-uPc0DR;NiU0q%}Em`XXF!780=sCsxVVTBg~Xkm6tEi
zJbrhlIRc20mE3deirxXNn-fe{(yJp*;^}ajzhve&Y&w@eJc;s+@|VAwRQ=)IyCPY}
zgDhULB9m;fT48t{g~XjZfATT;Mw0y5-(+}Xoq%WmDXuD{__L)V;xpy3GP10^y*p8?
zjvjqwzABUGS7^#JsTJJ3=*|y)TT7Asg+UhO(gUu3L^8r%&0PAZuTL)6HW(X5X5W-u
ziYa(Zr1r_R^f;7=tSYZ=;(Oc?_~{#Rz7x65X)o+!lm?VLq#({CP1)GlZB*Trx{j5;}gMAx_Ec~KED^dg~m2-AM}u=;x|1LDrTrT%C)xoQ>1(Xtx#
zAM>eDm2kcA3QOz^_1bFBdtZ!K1Pz*I_UW&dmgDBD?>IOnutDC@v$TTMQ@Xow;#;WK
zrrbkw%m{}BL`o=)MlVYZnv{mto@n_v>r2)ev}=I;6p>q}53Y9}9c&D2O>s`hR0oF%
z{DJ7fenNS;sW!;Rr)x&ibk3UaU2Wn=y9P$VL8j-={ljRG^4x!d6LreZ#8IJ_GJEy#
z%XZSn1L>E&+hrXh_OdR>X+PK1NyJx+8lX?^Ea?1g4u2!=s{lm-b@&<%$>k&>l50JXy?FOk8|F`u+X;mk`Ql7J>S9Mmd$=#fds6%;}=fO!W-Rb?Qwh
z8oF`1G-2}2_8+xf+q9M?Q%=)g{J`?SwmHacyKmp_hK%U!p<+?oT^vz}<;IR4-PGKC
z@k0py`u;X+iw`T+pqXlN>Y-#Tp(?LFaTKP*t98gYU|*)o?o$D
zykyBp8L{#vpMcY$+4f&re`DQ%YZCB-OGjJP#z67hHlcP4Kq&a=;7&ZYXXJ=y>
zboqgd78MM@HTVC>%ljPGb}o#d|AZOJext$^T{bVhAh!>0U5X4D2^2Q`KV;;clr%WO
zuP0%_e&H-$Ne(k=!X=uEX3hGW
z$g>s-9Xx!84>l_v?-7*i1aW{bD3eq+HiWHsJH7*QSuba8wll=9Y^+LJZYeMtRxN~N
zbiG@6ssJ}V{Dy{^qA%ov7rqfhukbnd4>e}SVU
z?TINw$+BEqkPToITZu1E8z|Z1FQ{px_Vyb-&oca&GRfE(mW%U3uqUxUHxJ)LiZON6
z`O1)ws~p^{+nb#i8!pj&rt@k?XmGGV_v<`ddabH&?UdVo6I6+hjv~{Cg2(5B>m1o~
zf_%W7n%pUfxC>4Ysw68645Tk&y%T}8dx!;J3uG=2i2w*f31M|P1;Hq~U7O!#GDc2<
zj--P@u={5L1rhD(;Nl|gd(B#7#`NjBD4pRrd6gNr7n>?~Nl
zZxn-d&m(%Evp@iX7#qKwXruTZ!*TRcU^z$V$#+K?fU$v0w&z{i
zrl@($F!Iay@8DHjFn?*EJRm<>M9&RZCx$mHIvw5mFR5w2w04j3KCCOm1n+hv7?oPH
zGEYph0b7BN2TjE?(qSa6ZXj%(!zZ)^7qrj4tjrdLsM#OZYX
zKH_*}Qcy8$|hqLJh#SfzGbYTDS`_gRJieDws^7QF}r}LVTlr!@v!)v1I
zS+Dxa?s<2Q2_dIRNgS>kPpYVfkp9@ep5|UPT`iPpoTvEJceQ0~Du+7fJ$SI1bZ)?F
z#t}?$i*GE6Il{|wm||Q3r=-dKFuR|Aq{a0YYjI-+kL^;w@;U(D;3Fw6C2DC)#0uz>
z2*~aLV%gQ+wAj;oJ8vH6CxP}rJ2jo7`nqhb=0IidaLnPyy;fnMaRI
zK>MYFMHHGON>e3qsA8Nm8QZmR@2nH=^H1v(WM@0Tk8kQ+vHjk(wB=`=TDX$~^p&?Z
ze?IAD*VpT>sY8yik!cfCZyUBSZ403{`Tjp&?DhAjmYxc31MWm+gO+Ig`0u3Yo|v-n
zB63Y4VVSi|D`7B~SU>_MhhEm>23?>j-?MvnQj@bJh#&FSWL(5299?aEMD`S$8d|+)
zz^{_X1rvf*w=n@8!7g3R5*r)l4*MPO!O%r1qW81fkNXYzC^f|uU2K2~myQ7Q6El>gjcQ@&~%_Pv*zarPJaK
zP*Q6DU`j8lZe&?4d97jeJFxF}@1C-!;H~4IzbB)Paac@C9eP1VmJw|-YlktV
z=KI#$uneB}Zd~yfwh>JxmHqkQ)+)hF>P-zW*w1C}Ox5e`jEraWs8QR&L8&779i!wV
zP`rUJq2U$Tu-eJ#=$SK;cBjU#?tMD1cduT*053RbDemYpjhUgxuvJWgJFq=4UX6fN
z%^gD0ino+zKci$X)s)UeW|ky52V*S-n^itOEWAKSL{y&-qQ$4=0wOtra%W6ACEr~^8s+1VMh*~#nP_VQ&NDxci+AvyWfeXTPp!Cf#S$L)u-oQ
z92=pJs+*^QiV{GYU-Op7W~LSz6)T*|_ZCh~6Pdga5Q+1`1T|NBIs`?4KeMS%Fr)I?
zwScZ&kD9K8x!%z&cPmzkuK;y^uT_JMS+=X~?JbDKTE1Km_UKHSL?v*A(Ub>cp*yH%
zxGnYt2Kp{;LfA0j{w%dnFw8veF*XB-)R%Ld-D&XP1VZ9(AxMvZW$)ANcNJgCuVZrd
zspgn(fk{ZqT{b3jbA35@EFHUkSE?e4V8NxDxGIlAM%{w7R-gd(
za-0QC&t>S$fhIlsOGrbtxik3tGed@!J;(UPp27%@gp`G@Xs%%qj(Hm$7B1OFL
zDROl?4A!F(=jg=^dopiT&R?RMhDpPstm0mR&f)%LA=H=`?puCm$2LxOC?Fan0^|i#
zFv_{Z2M=E8TV?1@odxJh%Jny4j7ZY^?lI!agR6&EEL!bX5~(gB`#mcnTLx+}NEJ+H
z&L5MokxCIlNDRwtf$TdMEYtzt)b!5Crrb|O(8d6{3-%+!Cz7jbo?+&^cerWbqCig)g1mLrf;o#Ihg`cCQxvk
zsr?eL+HKUr5UbB(%!g?C(Cc!7kx3xD2RiN@y04IOk2-$vPAk_C)gdq};p}tUL}66v
zJ}4wom#u@2n%&&eKnngXs
zJ$!jgB-ZF7hYqn~I4`-~SlV9nMtl-|H*OpsmO0Y9(|LQ}8A^#`q!of@xZ9V_YMVtT
zU2tX!PIs4VxSgMDtD(VzM@I@qUAPN@9zoEuXmK>fWti#kt?xUB3Q#lcv3g#(UU;rk
z1FM&l%%Xd@b#EwU8RzND9l~t}(7wk}0vEFC(WqFCF`+}E&-YQ2vITQ69Gq
zP&-&QT?;hXXcKi5F2#K=zY4!qfA3{+zk^84HgODc!!xAFLKs2fHnUr1I!<&dudGj*
z7cb6M3C21{K7IcrZR*0nsKGP5E>xuwtH4c1h+NapuYWl&(g?fpVr}{7i(k`R2X?>E
z^+)FAe)~H3P8W_%|0XT^B$Lk8qUi79VnL}2L}H(}?sU&yJ$yD>6JjE$gb{8_+T%k$
za8E>UH9AQ&;hXE1YGuY-6Sbs0bR%-`Uv06oZ!_!_!x?C+z0ow6C#T#m_%tzY!xtcw>|
z@v_V)0^1eqBHRh+8aXhRZ?`z{f{S|I2`vXWFV1SK@Yv`J7bFKyKao>oV{6Mq?m^6r
zV1f;HBCmR<-lxt|+cD;kaRoTg3|#G>0e?NQ*D{R|c>%qwq@*MxK%|)@>wLfSepdxoMTQchzH6j&bu-tB>G#vD!okYc
zCmrv>2HqH#>X^}^QSE0ET9ldD2~uQmZTCq>My0nINq^mxDO2$w7R4(vZ$|
zp%R|gvdGp}IH^H70Z6?n1`qKNpRhx{OS#TMwa{^$-PC0Pg{<4aF@!^kzy+KW)w1R-
zXt`EX{E@{mZGrSHLm$2~h?;2p`PusBj%{!t!iu*?39*
zcW(?ka8E*9{1zo9_YSzl=t%o1pP$A*hoZO3e>>
zH0|2x(tBSUpIxEHc)W{Ke{x=dHinEZ#U*O
zVj-m!n>9@vEQklqhk)?pXj}-ZPj;Gc7IdAGK`h2<6&D>@1
zk)Dm?3y*gA^olo)4q?bOB^tdc)t@j1`g39di|M&#pFYMKvpW$&A_OrG!IzfQZScWF
z0xGcUGKYQkr{cN+7DMGOo$m^e2Ur}j-sh`Nu9NSGbL7cUb3yD9wc<7UOQ)?!L#%;c
z50oI;FNx7=fbC)j`77QA&xE5rY#Kdp&jc=c#x9Y&@3-Xh`9;++`SC|$tuG<vQKmj?oDhp?C;1(Suq)U|$F+2oq#_IOtImB~b_Qj3kTy(u|eHu42Z!iOaSYmRc9
z)4}IC@pzl@iq$TKRf5%P2LhtEe(@}^zke)#DPp8yV%RJejsd!VpC+ncxdmKVB`gm0$2Uht#&y$w6u0*gC1SVte3`moGyBi7
zqnEP@B;*8ykHTk8Z2ksb7&iwqn8?~-9mZU|NR2i4aym1L`9z`y!svJo4%+b4qDNp?
zXrp4xPFleomzBYLQ`3@pLJY0y)+fN^ycFLQl>rS9ohY-8m<)YJ)dc7s%lr$y_5os2
zy$5JtK!iCVoI?gm8iz0FqxyV?y_#?DopNQ%qYvL9D*4`Oz7?$*bhbsl;ShrTo!@dt
zaK9^lT2Lq327(E}HH5kz+p@W_vBe5m*A(3|rqO84H7=X;F~io~1cZ)@Gf@fNL6rfR
zW1+A=r-o#xAPclCq;`Wwfp}fEOTX#%MmZ9~hqj4rAm}fEA!~+yd6in*S2V#XH-xYb
zB%7^ysy{eZlKh1t02p1w#~w)-UQmY51I{7wW-I_=RPZS#dQSaaz6!%PZYn{?b5H@X
z@=%*
znCL@}pTp|aI<)+}b>z~1{voR3QPjV$NgJW6gy#W#6fis3CrX{2p#Nabf7hQrc@jci
zL1me$EXmB`#<9R7P+)_(#3(YIkKJ?37%ncv3QqNH3V%2;Z;O`Qy~S#w248okdBkGr
zuaB1u8FLJ7#jf#5EBcl2XF
zc>s4ZM8!iRD}s)Oo%?oYQg4}yTZAaeBZv
z?6IRujVTCIZ0z;qu{{6$H7~o=&72kD1VTmyYkiC#
zSga8S{9A3%<>M1u09K{Pkw
z)XFYro0vFrE40$#6-6r|Km*eehrIN_ULhF@b14VExBiAidRW~=0sT--Nls2R%rs`+
z=^m>QC~}=Gv$1hT$%GXgeGh##@hpfEh1?<3Dx4)idR9;4T2?v{|Jack~tEm+?dAf)v-?$;%
zFsW|_&aK4{E?}70yjZQCMura#4ICkV;G;^hl1VI?shpB1!%v?$VXQ;p*H6M|-MR*N
z06rnuD8sP3%RX{UuwUCLg_#6C$Di%94Ed_6#A7}E)@o|$T?Y$@0w8!tc#4eT+`Ap21L@A&aA*$&>TrXe#ZdMh_Y7>|Ba&MM0u?wWp3yy*VF@24E*rCD
z{1seZ{1>BYN0rhKCm#eFZQpl~82#0qm;ak0q#jS4CV7deh+
zD$8g5^&zIYyl;%Ugq5v}zj5-K*%%N(lTh%*%+SEb^8r^;-*%lDO}%|S+Hq5{_5|0-
zv12`+t0Y;4qXdMIPDduYDQe1Fwk&|&gA4SdIViIm@*YCWgW@&Qu`_a8YT1jE^m
zO7R-z@rFlB48>j7-a9RL8vrd7jOF8fbDeth>=|?60t3BfonOKQ%Arb&OREybJEkq$(|sNmeB&-$yYS4A8TS3bv$3m38F!m-^V
z=W8M&gG2W|eutTgVWLkXE=;IKW$dK@Vy4(AL-{z!|WTzWG!>^#1&|AfjjnY85V{p&l8jP7Ar6SWlp
z3e*KZ!tI+Gr2@j@i)U8Gr&Z_|*alTgZF1jKPJn844PuV>~0{WWjKzwIiY&Aw`Fg!9;p$ZkX>FkuYz36(du?D<5IA0aK_9s{wH+LCT0PM|X$Da=
zH`YR3Q(fJIR*dO^@2nGSh}}3#+*(-~l9{X;kJ4_JQI`-f5*hW2q(z!M9&Ia7f2>|E
zRqP^UJZI0=fBG~~UcUKv>qd`P3Iy#DI+JbFUJxFSMe6o=t*OPbHK$qoptxw0S;7no
z+=an!?If78ftwN`AcvrLMk4Qt`%)0
zC3NIcIGCZFDNysIX%aUj+o#W0*c$>oq=risn}X{wP^vIZrF5t;dxMlhSW~B!!r-Z1
z0Y9YmE82{ieTVJ0nVOp9Ew|1~uD4-bsLD|B{Ou=+CcdKpkik)XFX16E*
zl2Uj@^P2VZnPp=213Z1mg{DuR&bc`ww&+{YoFNp!2cY;5{T!b9jPRm95_^QspGgNW
za3pIhmr7aPresuKq#0~XZOVyG-;xLyL(N^*o);xZ{5bB81qVCe?@z%v6CD=T&wBM&
zz~N1sL=T7tdv>uU}8nuaO80WMdNYxgq82&@b2Lh7SjG5X!>MyH(-I_bUu_z1}0Iws};KX1k^y?i6fzXK5Ne^n;zJF3?
zV?~O2q=jHp;4yr`e<)r$$Pwkjq=Al2AG_>Iy@Q;Bn3Uc7dMNnADIhoJh){?cg{zL#
z7A6L$hvp&zq^=}b5S>@G?zYx%!_W4Oq+}qOF!0C`@2{`-m`3C8>ZT-K+!aM<4v_^k
zK$WWcox=+wVq!)hzP+<^+UQ8yK`Pu0o}S&iO`h&V;tEfgqKfZY7>Vea{9Q`uTgx9k
zx^%@~LR2I~6*D(%t$i8oE~gMLpOJc>O9^eP4<)$RJM$+f(muTE1fCUwkddVV0
zmfpJ}s1Tl4m@zkIOmERd-YJ^}A(a}a$!s)Wo2Dux{h%ygf%KyxDRB3Lmgm<_xwF1
z$zNLmB1{oC;?
z{XB!v5CDoeL|xaCW~3)tu%fj%5s(Z6+0H*ZwR_`1f@+!p3wNIiB$9>#hYR52>hPUv
zvim2kdkGK?F<1IZMpJ&Sx*VJuUQLQ5rqR;f&Cv>nq!7%?ps_GZaF%P_-MwkM;A6b)
z2dG9`7>|t|e^O|L&f+fy!l3{bLQu)u_VCaWEj5}m2fyq_NC&NU-!7I-T~T*cTS%0s
zZH+7W(z;{m^6*;bgy3L-Fj1T5hw`H&r~>ZF|kx~6
zOM|Go6B9R@no8I`>h(cg42p
zgO5Mvd5(C}deJ<*<85wVh1!RqUakyv&ty@nbN4OEc;v|Fd={(e$H6t)b`wUDVzD
zS>fkQpfMpj2oG-3<_D`1_ZUXDy_WOK#%<*v76xV;FL-g~Nyj{kQ0DlVGwdUv2JRg2
zD%h7$dTwT+0V4P7*L~O-&~HUWfuW^sKqL0*`E!=2ARPR%j8l`!aVe_<|98ZAv`1Lw
zw+?I#cd%RgFlZ{tzhqWtqD&u+eNP2R7Y(*kG*aiJL0H#y}6^?-v9?*a}*Ao^f2jF8zs3
zAZWxp6H$Ek{{4%H5s;-JTzHwjW$fB+VLoG)-xjO)bEbW2g4PK&jpia}R&*|bDfcb|
zfrBNT+AQtTGRVMC4@U{3(U%%}|E%fxC9v)H_ky9VXE0R-_S=dEkE%>+lw;C7
z0n}%S3B=h`gwpn@*%O3GoWMC#bA0z!vpPze3bK3{9Vw%I@H0=zxp>s
zVL4U@Y9+?f;_yMZ4IC^j>s41e68wt!I$YFHTj-(@P-R&?)N|V)7=_o(tV%s;7UJY+
zXt;H&!tL?pcS0bl&YV5_{rh)=^jc1KPtS>(0Xd?T9D7w&cM=lDk!eV6l9W!fcKx^X
zm~EY%mYoYO7BG5v>{w~5K~#XDs^X2FC%!^}Ku-u9yX^kIA3m(*tlyN|S2x_ZVm(X<;EfTK{81jYAOQL|U3cEYl2jg2Wo`Ic^ehYzt*KhiZn1+N<03Y83Cn2%}#xSEX(
z$$|R@oZmthN>RUikEFS|ImhIwQ>TD8XopJ@4l5aa*h)R;yZ^Y?7pM92d+)mrB#sKC
z?T5>pC9ly8)_nZ9t>gE$RPXBTan=%1XhF{sr%_upEcIl?{6cCXV6}DY#?;Rhf5oCD
zr*b(apv}uvO{|CnIt3&MWIo^Ae5|^VV!Lf4-#`5?BztFu(P9+}gWk+r9SL4<0d
zba2^$-F8}V&F~44C({^m@F4nGJfp;i4|L?n5u&&_JCXNG-7pza#1KVPG;&6ENNEO5
zd=;`qh%wJHKJ*E6fse@W4npYM=`tpnW*1bYgcIAzg
zz`59+MLNpZV``&6Guu*;yWOt{$hIdHSs+>N1@@!p$Rs(beSShx(%Ml!dCxyKFdK3q
z4JA@Jigf}G?8HozO{gm%5#nEMRW)IlAkyiOdyX6*bmmlVO#48nfkCrm?2W_u!TGleC4&zQ{KQEp@#1v#cEVR>HG++m-$Z!1X>1)iH@cV;w!>0I1v)&x75U3%8w5=)m!s
z&-anKa=x<5;wI+|@Auy>wTU{JyGF^1=i6A*q|Tz70E`$D3~6bg^#u%&9GTC8>53@2uUj
zue_nE`MJ)To1>=qPt)t=)uke@+w+E&A0=_;D_3kkylLHtwh?U(W$R8~+nHL_lCz?E
z@e3RzZUw6ng*+5>Q3J%8uc~W)znWW%cp(#yB+f*`K`s@?=U1$Xn&c#qtvG
zWdF^@n!-JEJiD!Zz`Wb5q;6dCitC6;(|)=6)%^E5XD7rbEUuWne43k8K|oRND(|uN
zS9`^7Nj*QbZioKzinVjPl{KuYz6ua=gVlrB8RlA{uFHe__Zw(Pa3I!a+?)1jjf9UY
z2QG~t9esUl>G#u3m%iK1ZGV1vl3!L6Jo_Qt+c));DwEDO#txl)c8KBG=MqJy-3r5o
z*4=K`S<@_K9d~t0YUtd;)&{#N5AxkB=kIb$d6)UE=y|Zeud%$G+%n`cIC;a>nD{P`
z#xhASa%U!wR*&qQm#7};?fiwgTtO)hn+EDEnCi(`FHhh|u43l{J*%{oe1`cccRDe-
zjSPghoobqDp+#wvkL;TKa7~x!l(>GzKHpo8E!p+g@#iUdr*ADE_HbGcQ$gY2tK6w?
zqY)NpqHX*wJS?B)KO7`5Xpmk>5BeuoJ`e9g}N8N|CuBv^VV*UfsCqa=KuZBA70fj
zLf!YSugw4aQSSpzL2?GN&fWeUXjA24b2L!J`?{ezvFs>whWjZ-#PyO|7?3rqg8BM)3^WUk^lLp
zdmkJE7M60+R_&ED{{1yUDPn%g)~fHb|L42#RLJZcCN8nj{!doOpP%Nlg_*u^?J~R9
zzaQk=7|K5dQ3JX~#bv$zgmVe+b=wFB8POO0>W^pJ?_OmbqT1f-))be~Jm6sG)2Hco
zy-Q|DRA&G4JyaPfh#K|LxU}(5aJD@iYoVm=_}TGQmyVUK`*mqiTG`#xp3*%G)a9c3
z{4=lVZRPo=&rCzkmhCnPv5vC_a{;=0Nt*-6&jpSW{E~xn_wSdvY`{;8$+vGitrw`T
ztT544E-9jF7xgt5J95*ih`g&=_FC$Jbtn%g2&w1IWJJkip8Me6htc7+>v`M5!+-J1
zE~sDPoELm`FouK&gU1M|yQm7K`W;_uq%MaLvoYw>Zvjc-ZZCa(<1o
z%|DB~b)Z9^D+_m5JoV02XFPo(%?S|i*gHR?($Y>T@1ywe4TRHic_;t-7-f-}+JePO
z2A}-gYe!FOT)!0HSfSW^Di2d$Hkkva_PSb7M1
zS3ik8brmdy>yvn&Y4aF9WNA&=K~)6Vs{U3{VuW)fM^LU4tS+v_L+7^?&$L*d_v8uY
z*{6Sn)Z7*o@j;}82IY))Mt@1MKQ@op_J4+({kFTlZ&9eKk&Eljh!^m`!(Ea8POv=y8~fTenEtoPhuX(;>kl=X+yM;d=vU
zrJE6~f28)n-IFuIRk_JxjFBahb<_udO*jsyGErVsj~p!{#%IL{NFf8kTSl_P8f*`y
z(TYeO1y%ru8zvp^<`=8MZz3(2LG+apCDa!45&;x+7l@;xq#h^9_Szn-`l7hlP-Yf=
zwa{N}OJo8eNh*B>L{K_>SD`z*RLl)6(@
zRHAqm#HEto0`SMX;63ux;q;YFLUL*@FVASB&Js0XJUr~}Fyu$5KN)2~)FSSXl4sA<
z#8hYniThhNrUmH&;@Wl7_sjbP$>tf|v;XHT56bkREOb|kxon#`*mtQ*rDVq-=2pRYv1L2cuUg%-+(XGA2(tq5*Yj((T(X^YeR$j4ruoIn`*itYC-YasG_pP5
zIMpYcxks$K@y`Wk(BY_1+8Ef-SiY{}RhEPozZIDnV`f-tejS7`ph1}_^EZ{q-;V3`
z^>;Isg3XJbi;O6n6)U7q0|m4Lc~P$XMzV*!2Plmsb>L0p*B^F9c|PV}K~g)dUhOya
z(w`AT;QVOZ&!ZoyVAg`D2yE%ww-dHBEF<7y?+fl6=wS9Oj1{SNj*cQ}>UP&tzMsdC
zn_#n+cZB;d;1`0bspH12=hV;HD>a_j7h_d+8XB4rVt__t!8EZ|A$S%QVUMB$VSeS(
zrKTylH6DwuEcN{M)`W*Kb@sobh+9WFB)4^mSC3Lkmf4ia`9TTdh~^c67ftrUIBO8V
zL;Lr`NNoqM2arai(U#qFh>U0gnjZi|$h0EyXvhoOFjFGnS0Fj%fJ6q5=NKh$QeGb;
zN=uSHFI0+5({henHfFK1%=H(afxbyJS<7*NzB~8H6U_6h6Cq)Q2_>sw1UQdn1`ODS
z=8j^H8{!Ru6aa~ceSx#0=Kl^lX@-p-&Fv%BDDk#fcb6<$bl+Z!MT5Y05s)%l4ejW~
zi)v^cV3mM~?ZzB$;i`dcBQ-fG$(l?Eb8~kP1Q|uCyuf>otWNgaCLE}vN6*W+`#_!2|7#lKra%yfmcD(r+pP=*c|R_ARa}N3ltLf9M=PprB3Lz#}I!e
z0%0f?w(pY&_ej)R?rx9o3>`fqy^QKGYu%_t+NhYBkW}~+MSe3LhEK2>5N{(HwNTtcDRn8~Z$?GP$Ey~Do+g@?&S3=si
z%a*Ni=Ql=X#{4R2mf7U3^`B5_q1k-+N<B7lUhptv0ee7-hZp~tY
z`dWRZiB%0f>^_=}s9QfLeanQ&b}#-NZ`!)XUPja;Nc7UGf3L%Isq;@;TbtznsR>~9
z7Uw7rxRM<&dg_0h%K<$Mg8rxERBoKE>Dy-4S^u#6DOD4-Q1Ni)JZpq`yR~$fN1U6L
z_I=h8RTD$cl~XH|demRc-&?S0!?pY^pG_10`PH4XZ93z>Y>KF62rGjyUhB4p`z~^>
zF;9DDonE*`=B)d(b&u-~ZMxCFF73ZRb36We>@#g+H}5X}9v2ti0H2SL<64EZTQKH$
zr&9Y49iOPb8A-w3A9EZF#fdS?UN+nZ{b%>?B;JC1gA8kpFh$!-πFiTGsY=O50t
z7s3>XI{5a)NjtLmM%&@*PYgm-58QK&pR{H9lmY+kEZ?r^dLTl~x9ITw?yfeR(L`bm
zJN_6<4&=Gs?HRL3;
zIv_Do2=E7_ZJ_WE-SrzhSeH?0xDM@Z
zZQwGErs3qDg{QzexmqC^hzhSE7*;!V3|}3ga`YT*&`pU{Pj&0Q{JZ9WS-Q79S(B{z
z<+NMNh@~-Z&)pZ0Y>K$~8i{aBRLldUwp4wRorlXo`?*t%vE=@gdO?kzqgHSBzuUz@SRrvI$0y0^dQm~Vf#7TFb|;pf$>!j
z+^~KYJ%C
z$)$*OpT6peLeHK7=DMx^%`??ZLNJ6)x<1`tb-+2&xj=~_ElwQv&U|tuIO9i%=;=IC
z2&$h-dR=Q9zOF)Z!d-hU@;T69jXaP;<7Zwsu+{
zs8_K+CU@W-9o{a4syxlCC+ad{c=BW2xBytMgeHN0O!KJj5?-Lk3^Qo%4WFKkP8uXD
z8hPbPo|88DcItuLJc`htT(MX`iFM=(fw!wkQxAKDH-JJqHdehnoDgfIeS(&Q29n9H
z)c0V_-5Z_J@yLj2j~|aj|I5ApE3VsUMp|rTwZRbXK0ndO5;GgSy?AB9i_f${@7}+^
zYtDy7l2wH+0Em-f2gP0$0PEC(X9!3b0s~f481~;;@j>27n!YN4d=I3uGKYocA|yo<
z#hgigC)GWto?DIepM-1_0}K=yW*|e^%=buI)W!AZ&r*
z4I^bF_SkPC5|8sl$UOg(D%WkirhyDvVrbRj;RXE<+{1)`HKi5LCuJ#E%qaN-gcVW!
zA$>wn((xxbDxRgYe^wM-h3_{_+5NgCYb+|!8!jioXOf!gh5(AL8HB?|6N6nV`FTPc
z357(`NFUKYa&mv$YeB~zCAR61!Gm{^na0!N$A~~n
zyn40w6F-iA-qkPO-$%_tY$iE1L7lnN+P$Coiw(12iqV>`1nc*e*h6!rVq&E%qor_=
zS)yJ+$4A?4Mhu7@zG;)AKwEzN_+^T&ZU@J=F!_qw9r-u#1{yphKE!0>FeRoJS{2n3
zf&n^PiTS1Ip9SaLsP1d}dZbd3ppbNKMrlbeA-F%tCLp+hL54cmMsSod5%=Xw$@@f{E{(xg0
z?Goka_|@|b4Sf;Z_1N&=?&#Y!KO@UdTk)UCr;E$`)UMhTDacnlTf;l2Z2rt0W{W$7
zkokTxG8SN99RZndXLZcmE%F;(2?u8DpTvZ(^fFr4;nE0D;diN;z*r(hZRz~0@%Edj
zsm&eU`J2lOmpmF{>V6&WL0&9AvnADno6=zdUawy$i4^Q^Vw`b=E(`TsR_
z-tk!P?;n>Gg^Ws)xRsGYNQH)xjHVA*zCWM$xUSdr8f)iwk>74t;IC^Sm}q6b%{HO(M1h{m{SLRp
zk6)4&1z*lSv^CeRDX;U>`J{2QFn3R8wcgFS3*!%XZIJsaPZxQ$OvEI2*Kc*i53a|9%!pG2#bYEOmkW2CgB
zwrGim^o0$E;phJ@%87{_ON%JLE>d(;;J`B9B*XY_ZQVX6VfW4k%*pB3Ats$>W;JnQ
zw+K&krupn3Oe-n_Yo&2OqMKSWsiI-B%JDC%y{?xz;il8KFEdcCScSF{Y3QxEPH#GV)Nox$rEir1)Vo@|#uf6J!g86qZ;ZlXqC}5N*eEqe6Nh
z)Fn4pcTP6@)8E(E-dQ9}yS%CjE^AeOzzFxFc*ni=ozqJa?kmoul+55j=E(PEvEn9%Wip+ik?a-q0Q2!E-!B^
z6wf+;iaTTFUhm>){yF-G56`0dny^jC>P3sJaQFcLEN5TBxOoiTWBgtC*U3dG>eKS+!M;w_l2e)2Wk2t3_?2WL$bYEcoBf869?1AaP@eSIEe)*Q0{f8n&%6
z+TTs7PxFu!EgDZUY&1`}&mL~le_lY7rSgvPDN*|N5bNAjnVY?ENv9t${_cb(=
zPuRqDQ@VSnNF{9Z|67%TQ_J+FW_x6LX<4<4i+(wL=8VI*ZF{_pD#biH2Hq}-cCGtw
z8()7P5~1CpVPmhX>Gjoowa=g3ExM^?zD1E=w}_hZ;Uh%ypV!!JG1JRmUu?DEnt8I%
zAcD|-|H&=S#HJdU8l*YD?)ZMFZn|b}VOmR$-_P(x>2G2+(^e*QK91SCc1~+Xu1-*R
z-pcaOPsf^5rw-om%`$%JM$dKD&Y6?imYx~yxJ1+DOJBz;lOm71s%v%nG$>y4t5I8Yzbi5AKWH-~K&3I{5r`|34q4s>cMqvX-cR
z?*_jT8D=?Zzb?S6{O$LP-%LLDR;FY{|8E()@+N8QJ=>gec=Ld%D`Z@DTeS+j92OTO
zc)lJ$pdxqW6kB2G`%w9J&fX{b+Yet7e-&Z5sOx^dyyA2KaXKGie7TE}V5Ko&4med2c_A|pF*OP$S^Hd4~X
zeF3(Ume$*VVV=sx-9$IR_hwFU`m`UgH)R>kPIa|lM=cc)vl@<0XijkDxH+K>v2W7S
zp91r7FccTt4C#Yy4KYyxCf)iG?}nosrllsnS<#ty(%i2kZO;3Ul^7O
zEi$cjv3WAzj$jsel3271PPyZ4h=7D4($K^^F!}IM*4;bh`!!PvIKH~4@9n5nrdRNs
zhm=WuT~b2RJWQ+Urr%gIvwYkFM_u5&17Xrph@uyt>EIyn%ZZ45=yQ&t7i1o)x8_6q
zs<5yk#*!~zz2c!`)vQ+Fr>t{|Hv(!#%~^Si{4#ZQY;v!jKR=T91Cq*9R&rCg`tZR6
z9QF4aEp{M?jJ6)d>MeuN8m{LHFd7kT@t}i(Oe(bKp-M}K3j}1)m@(Sa&^U7JSfma*
zUC7T^U7)dn9S3qertO3Lzx(X^N9l%tCMS9+I?bC$-q*nJxubOE9Ke#x%G&xV#_2gZ
zMSWWomUBQMYC!o5%dn{D;^x=cy<(K|26=8c>DD*e+U4@pQ_i@jDiAsPJKlPUoiCmvYK}2nZ`G`K7{|L#RE0GA$&T?HbSc3-ClSqOXDi
z@#afu6}h?vnLsUPxs~6RrXbuZscSA%6HY;@c($OZpf}*{)A#7BfqTGM<8>)@-=f@x
z$Q@LcmJG~McitroKX{lcJv^|Gnvyd9C-h@53w}LgP=-|elXdEpQ~mlnGiE)QeUa7F
za3TwKC=K5tV2)+a&(Zf{DTM^GU_awR3_x(HI71suGm6P9k055WOI>7Iua{?u7-f<0
zu`vG&u}rNtMolVgYmf#*Ys__3k&`QY`vqntaXx=%P(!eW$6?`2S>Zf^tja5`fA5FE
zMRm*9rk(l1a|h3@enF9f!5DOT#JyxL0VI{~7s{0I2*sy;dvPs=@!~z*6(eXS;R*<~
z+6+D4e0&EXxdVuv#vfN3)hqurUGA+p1xK@W?=y1%vm5A#-@hk!9-{&I@^9qbynqP2
zqc;1I0R4_IMcm)GWc%V(x9;O@HFB1PnsRk^IFAH=6
zNa3kLz{B7%^v=TY97)M9pFe|iH-7wB?B@y|Ky^o(Ll}TjShZ*kfDS1J>UYJi98_HEYjxF#Lj)=jR*Ozc&x;r`1+mU=|KjiMM
zTje<`R2{v1d;*yt3MtaoX`YIF`UT6EPjgx@e|{W6lJ&<@uBG6)^(GB90CF?}+Pljr
zEnZdKL4lXJdv`>5c;Q4=BYOn$tCu2Um{p8|u?+JONuCBuVY_ziCEBf$cE^t18Ej?_
zNU5GZ4fXZ)nOc2mvkPX>
zWVziR^N|EGax^>@M^SYmeP+z$BXq-IeJ~6%`fGrv;>{Xgxa5IcBz3o2ISF_gFeXOr
z&%|vuEp0x;#{K&*FMnl9*Yry5U@^x6W3#kFhpKO+TKc=i$~bIX+LA{-%^<B
z+Zy(+&)CSKT%6nDS7u%0U^~&Rgo$h
zj~N=BCuI4&?6-f)m;C8N#8S1-Ox277Vv4nW{Njb=(Gj_>9S&++vR`lhWWo?hA?nDk
z7dPpTv5X!k8!wRH*5k$l_ejvs!wrfnmO+2GynW27i#PPA5KS`e*okur>o43k;}y+X
z4s=mmn?AZeK#+I`vI;6s{^35wVRh0>yR)gVrz--6{*VeOggJ@IEwe7cGXyR-+`5rX4yBMbr8xPt%fsJ~LM{
zj+R2I2)z!fA9PRX$MDU2_H2>9A_xEoujt1xkWR#z;n&++EXA-ruh3s}U
z27VS6n;5Ds`~ZlYFlhtl1if!5j!?S&dWItg!*WTgdl4^-BLT$KaPRg)58ERTsa4P-
z6PY(`#d_QxzSVbGXU@BYl+087lu8qO_2-MK4ir7BF|0E-wpI1=l}NbiP`NZy
zR`I^jd)bHG!@7*ve5dX{2o3}uhF$_m@=)*`A+&YRQoNpEx|vZ1s5K;1`h+e1XvfGpPj|?e~qKEtBh#e06GC>EW>I1^b&bSz{t&k
zc}VC;u%TO(SnEbsw#9&Cr|92x;Aiph(Ax6s-P10j3#KTd@L6o6q!_!|GSyrl;)0NF
zI`c@tC?IFE);GNu)NUt`oGuL~dY&7acw-zwgsdCd-GfHv09JEBiklh03UN;$fb!&n
z5FdokCqnR)v`s|
ztf+f@Z451i!5G*b-0;x9+VVPzKgkg{U-1HJ3_vSS#S2`Gj2$7c2ygG{(?8BOX%DPdhezPWD0mHij^H!~_8@hYE-)(CyQc%Vfm9y7B1))JF9^
z8PBDu7y-!Ck1@x&b0J3bzw?(aJz<$D@{j&IWkaUZFLR+9jT<-5H$eckIVHHG{{<d_NMUK!qDJ%Hri8H5Y9MDpP%1Wg>3g<=BaT>k*%y`Fi?Ei2-;BeM|iPa2-
zmuO`uMOK-Vld6!$40@^x5Dt-~-h+_HkK%Ul)gF|cqYdPf*Hc&CoyQsvfc
zc_G8drkL>$6r8<#_TW$hAkJ8#jXl`XvK}2Z{ShM}?2|!f1!I6i@-a_<36%B#@EPih
zjIy!4lhaYcl^Yw|5ou=^xZ;g$ohB!ShiwqmO>M*rfhs)sti(JHe^>nUM&v
z1_AFTCVK^2@ZiPD^y#&;+ZM>
zI=UyD`3GlCt3HQ%78b;(-bWqM@sy?zW94x?FcpF6!q1RRsd!J@qNE~W_Q^Rl@84g>
zCBvR#mUJCQb$L0V&y75@n*5(1R&cU)37-4TE%4IWvtN*V3O&v4hI9!cCWK|B+)K;Y
z7<<3Bl=j1?gx>HrY+XOzM2QSOjgOTmBt)Hib)rK@
zOkKF6tLvs|>uRG%a{vrfP_=hdQQfn3n|@DMc66l=+;a_V2jXKD|M2
zK*=n^kQp^^5p&YBywJPx4Lm$PATLAwr_@Ge?7c5plHJ470jGmVsH3yJs0ML1xaQd4
z4mdH8XRtfcb3XG1@}KyC;VGq&nX;h*+~9B#yTrr>P6y&t3bkX8EWW(2ug`SP?|>r$
zWXy<(F)`g|*3%w;u#Mos=Ylhb`?+oyC%Lo@G#U-!+Q7i|7)A)`m5gqWtb|}>oE4h$
z2FhV!ML$HmbPZFyMXyQU^)MX|TN+5BM|S)hUtLb&z#4DL1$0TH>I
zSyl}nKHR&1pY$(wk{|@BE%yD4aPqmAxPNGtS@g^W30d%cMhi-~JM&ZW*51E+w+99c
ztIQ^NkFJF*e~Zf*Z#&9q3yUtTp&S)Li*q!3$!KOQc*V7
zFE#e9Qn9tW1QMzCin!_$8J3bllJmjoKIejejxCg?lADc309;e3rw&IM;2Zile|fWc
z1-3=t;uJt9{Ut_N$kH6K$i_qd1+ezCI(S}qfiJqx>6to5$Bl)AE1}008#4{Jp5s%@
zjE%8N9P`&U?edHe2?+#H9b0QJ;z$V_N^kDv)ysH#ICM(BFy&~YeP;-gXsb@KLgNU>
z@ioOKCpsXoxj^vtHnUi&Uz%ljSp4g7%T%jDY7(OHE3K?H&(Rsrtxn$hL&#Co$7o4d#>uN~&!0XbM>OwYV(j7}t1x2cA5B@}Wb(+1X%|$IL5?D1f|~kYP|E$v1G*w-G14L?C3($6`
z+(@{g%wnqb?DRvxtz@Ic57`d}QtQwp&YJavIez(HtAm5{!6H5f@&+K2iK&66J$?0R
z86>*}{iUqQ{-PvgEgg4U5)kkWJ{qCzqxOB-v~%J4d=U`^X*(+m#{&7wlsCwbDlhtV
zHIj*kp0Ey^h{Z9x_zfB;a6up{_1jU!_vA^*OZ?SGkp$z$u
z^o{;tTAy4oo!8eFSvSEy7X!J~Yu3PI)O9}NjRi1{FdRau_QxAdr6spplCex7zz{P*
zW_qYi(P{_Y1>@0z#QkF{b$-oTb=ZJ}b!i(zA>$LIs@hTm)6g|~T$LZ@Ac%6nj
z=*xcrak9PLU9Qn_==l>T?f`=s8LfibSUq1fUDnM;=D$@x`dp`T=uX*@J@4Nuaug^qAT$;VHPeiPC9Xa!TNv2H8CxxpBuvbjvtU@p9&e204^4sXq?Z0UUEBaXhlQLt^GCMN9eZKH0gm?A_O0v@PMw(lW&%w-rJ0I*82_{V>MU6)etTyBF6B_b%@QM&9VDQe7
zXIDGaS$948e9iPnfSjDbluBWRq?h%Ql0v@#Kp5Qim7(wHvu9}tJYhTHC8oUH>WG8;
z#i}mfB*U*!Z^HYhJl6i9dp8NBlcFoX<69Pp`kUhxgBGZ&rhvSlF~uU}9}vK8(b1XY
zPxD(N4ybHpT6k}6&hljPt+MKQtmGm<{Ppno1hL;?NIbNMYh#)H
zfyA#erH?QfGbWhoMR))JhI*<_O|?f7*uHCy&mU+!Us7qu%s$)XX8RvbNo|hw{J43+
zm6DRc^6=q%T%l_~5u*t3q}o(U*xu?mq|b&88-$LI=p1OJ$9&8?yYc5*C|AI%(CY0k
zG~7-xkV2PnQ@@eU*1K692#6nX=z7BExo7Y8P%AZI7#!tJ5Bc`1M-xu%+$n=62DVXq
znz8BRV8@UG44@jP*0{zwoD`XvKk=kKW322jaqbuMQlCKOEySw=JzP-6um%s7&G)q~
zhc}LHE`O1fl=M6F72Kw7`_!R7tz|dUI53G;`>fcwKgN{ewFKZDxDo;)`wPII>XbAq
zjM}EDar?*KsRO?j`$xrCByMQDRQwIU_6k{0o_%ASbQGRFew_E^{umk+(94(u&*@i5
zAQ$OAMnAAQOqbF8Y5Jns={8r)5@d=)t+~^$GnFAsDKm2!`~+(G%a`~6HRA4^TYHTh
z&T}AXYi~s|mouV8hdYfP3=D?R%G0L`z~8l>xtI^SAhRb0Zk3=y|C@8q*}HX|2Ps75
z6I#op
z*?UmI$eqllVyy=G`TL8;X61>aUJqg*WIJ0(ZRP@i$HV2V8CStxti5QfzHP@23dQPL
z)(`eLoL^)^0rrEt3%k>o7h?|`=ym%PABGE(RSE9Iw5YJ#6|U*Odz9C3f1;vClawSP
zH5e+y69@(lsa^h*-BmIZdN#OAK9cLE-!dRfYlL>#laHIjJF;&iAV{2cjNu;Qgo~Fi
zPgq?QI|+XPS5@3#O^uDqQOm>GIiR5sRkKRtVsGVvxTg@|LpYg*EPj+lHN+%>vF|iA
zR5)2tCepii9M=ooMBj$?fSqGE^=w^_1I!nQ@w6wL;{{dz6oTlFs3pT@4^UPX8#BsV
zX1A?6wc(%zQt5MRzDT|@tkI=2i`upgs{X|91GY9diTI!=If4G{;X~Vu!Jz`jq!{i&
z7h+vzR17_p%VCW3*|f=%$8b6wKS?Bxbgy2P%73vYkSuc3^PwN#8JZH(E%aV_xv_@E
zg2QXXDk@&Q*dH4!_lvk?q5o35&o(|fZ>P^u**xPd>q7-tTX`*9i1qv7u|H{)qHd6)
zH#k6URSVS&hcXCdaBwgnB|B_qaaguPsDL7V&lJs;hPdIDPLhLnjS*{=7@EMCa^J39
zqJ@=+R0yU0ZE!3yIG@gxWD#biixwSg_6`wQnDEp~kt29JRHvq<<#pwA{fEFvp~RfN
z+^8VmRWI>|7i{CW1j;5*q8OEy!D_L#APB|V`=`B}!LQ>?Dru@~Nb7FWjkHImbv!_d
zv2MDXrqNhz?k{<{FC295P!U_{vWg1I##Tl#7R!gC)M{$#mD`ocX|6|MC|4$mi)&vA
zGhrA4pkugKhqOZ>C6nh{|7sk5u7LEM0b8x`n0trwjGSDTt&dooc$c%mJSTT^4*3_`
z?(nrQ5ZN-nDCKRJVRIfm#_!Ptx=HXiA6u
zNr@%ys9~c`PAp9n!G;z`q;~hYoAO_gYlZ(eNb-1P`1|u<;x&FzQ)QDsYxud;LabxonKNqf!e!?lP6=qq??-*
zJ4WRlle)h*h%BN~$I{9%U7>j_fu&tX;O@sN+NbuRZ8jGvlV|Kvw(q$
zDJ?zRHI+i(V2o!m9_PO45&G%tSD}ldm)D?MCdX?sb1Em<*$G*ew^wfPaGGF`t|mNY
z9ic3jr*UA990>;;+fUPEXwCywRCw7dae=A}D+U5%Bwk>j-mcK7#@Q6VGeHxT)OA1Q
z_4zeDr=W|y#IXn`9Y|{O5eF`2mf6y!?_mQ=4^oKG*3u#~!;^t7*Pdl6kOuSJ_c(nC
zCMez#*Zw%t+~XQ#9hNZh9t(cO&iNSaV?&t>Li7$js*l;*-^&1sljvcKtJK(p^49qa
z0q;@!H(exU4*npVJ?KVT@29GScPZ5A
zWUIJeUM-zhq1N|1j7_weYT5%$yY~3a*m=G4R=!ir^;=3ZI+pyXuxtv%Lg31nrf>+5^ig)m=I`!W2aoU@LCveC9$P%R<9~6r`;Sv=vv`}2cN-T=wc9qv+q>bf3iSN
zL+A;=i{3>dw1qJu1C(6UZZJOVMc>q?dA|@;%D>(Sy8|d6w`K4vT5eQumitmSzM3W;
zZGe&z1cB@Hovc!{X2K*SVnRE3%5+F)p@KtcX`Ix&17<9kvI#uG2J_lFe)<#~#&|E0iQdf^pOvJM=F+SdWt#9@sG(rToTAUglZdvG|UmZAMuXZ<+rI?DN-HO7d0a*!oZDq-jW36o!iSVAQ@czA2RCOsia6U0%r?!FbD5`jLhFtyl
zSz|=RLf&Goxm0EJ@}#)9M{vREb89RPC-+wAGEcm+zFK-C?L7kAzn-6~j~-p=akc`d
z9$-|YorfP8MtJi77?O0GaKw<^6Qdq9=ibe?Ur3tYmN1~$@AMqy0Rt{W7iVVN#;guX
z_`Q1{xH++?Q@VdW^d{Zb^YU^4yUh_1)bxQtK^mNJ>gq#s6pHrNav08F&fa>%FiY?{eNg%n57#81As+28&%()xV}C1Fh{S6_D-E|K
zPY@}UP#r;53>^XEUK-lYjvx7x)yc@g%u(oa%{&%dlg^hjl{fa2K==^pD9q70kcGC7
z1HQl$WpX#^LUjH)TKCkOidh70WgZ8FZWB7Z##j3M&(9ARE*-`oy7yy2r
zt-8En+hdYhA3pR_#QjJhzxARk31b$^mSwu?Ma~Zd3@J3r&C1%@XE!}4fa)*x>CwEr
zq_J~hnbNDmCdRRu$`>-jH(wits5&MJ@7|TuOW50&J$TUN8O_u@S64W=92{cD`dfq!
z16gWPD|pE2)k4De^(z8K>OZytRG-nyo?ox?=Q%L5o#ijr*}&I)ZQRR7FBba>=1r6f
zP(NAS4C!x~IO1tbLi))x&Yz?;ZvgfAsY`$^NT=jF~L5rxBYR$3llNQ)JPO
zj)z^9M~&g{Vp)+ZxRO(?EY3*j9{$@Evz0SG)Yq#O6LHM*X5K!IIzUT{&?S*mELR*F
z`j#$AcVvM`jJVD~manNUpRjv3qxM+s!jbvfMmLctK6{o}RO7vHAsFs1y{qta*oRC9
zD9a(i$gaw4-&d^yDI;a~ZtER#jn!;aHWiizR0jOWHNB8C!!I!?V{8v%m2}8ze4mby
z`uY+uK``Z=zb$fV|B<9Z(KFmSXXP}8!eYgW73q%q4jjN{GF|=$ey6>am6LcmZf>X6
zCtIf#-i>TcYc|$PWUeWEu15NTK0T=$xRO!gW15cLpLOfXgAVD?(SQJrcW@8$VXmDL
za|U*SZB5yHZA`}3JpSWvG6B#tvkR~~qn%QXk=fn}4j>e){yTT*O@^Un406TKiCjVg
zqzJ#oj}qfu>C->GE7-UNF@PI9S;e`Nn?z7AY$QR0ItlK-uaAC7Ncg`KS)00z3l{>M
z?MNPDx8knninQ%9nr(@n9L8@RI(nz$>iC@W-kc!4;?%Q9M=s@@KpuvvhPhkonq1eR
zX8qc5`qUgf8j+gk*DHrNz8IE&|6TFKpK?knZ6zOqYU0-o?D}KClnD0|6Wn&s{^!-=
zrq$m}6SDFizp*+{`focCOiht$Z||P>TC6tPU&UiV*DcMnOKz`wGX36+jwwe6I%UP>
zO;hr9Y}75$j!IZze`==5S%ZtacIBk2_wRmeCR^HHyZwyZg{%<&e%f7ooaUzGl|~kA
z8e(H?v&QI#`>kxz3F9Qs`Kw+VCtFirSJ#&)a%MSjjuJ)$SZ6o|bsU#D-mGkIts!~t
zOys8_ReesD72O~7ZOIeqwQt5X%$%1S<@h|UCRHh8mS_3ki?<{yJ&Wf}IIq;}>qGsh
z$8U><)YjNI`yO^mpVIf6ipo=w6V0ljleE9pA8XExPE~l2;tX*R~JC}YK&bN15#JHNj
zpCsvz%uD-wdE^SV+)ibaW?UGyU!9v!Vq+q1{pfJNLu1Q^yL`x&9x$~qRwX0#dF+yw
zSs4TCwk+-Ve#Er%$XdGze$v`$?y2z+D@K0J{5$kdJ7=cYf3NdiXXh7d8p|3*qs@ne
z{oUh#T<#)+H!U_pI<9;^l`mfHkr2FQSci=5=x<{l?=72gODWYm#DSxfwtl%+@L{{eErzw!V8

literal 0
HcmV?d00001

diff --git a/docs/img/pgsql-migration-sink.png b/docs/img/pgsql-migration-sink.png
new file mode 100644
index 0000000000000000000000000000000000000000..4ce2b3690a75f49f0e80230ec51e191e31786485
GIT binary patch
literal 109344
zcmYIQ2RN4P`?g+t@4Z)cvdPX0Av-(aLqcR`?~##_y|Y4;?47I%*%7amRWdW+wSDtjJM!88k$ba4`gz7I^CEzy^kbl}^j$=M=~bw}afmm5=1
z(gl^*9m8ro;SA^PxhACue3Lrc0j@m%LLKgIN}5&olewnHSVWAi-n&b8*s)tX?E
z*DTlnTBMrrj6wQCUtgtu#nw~}lU%_0NP&`c)+`C3>shzZ)7F&L=UjJRk7R4MYZGX>yVc@5|;5uexV^vWxB2Gc)t=Ga>&l
zm#+|7ayy-eJcj0m9XSO>7j?H2kCari(^R!ZO}aI8-td~MgrlR|)+B$!EQ7_67Y=Ut
zQc1dn@4=|5{cvtu;AKFQ*~M&Gmpu3Lfw!>Yj~_MDhY)|NEjt2wV<2)ScDnJ(
z%gcA(JBS3pWxv|bwr|#}pyUnP3sLLcy-R`Ho62i6I5-#-r&pe`c2aHLqMar75Uz$z
zJU0(W<^KEH_B)qrN~`I8^y^EKMoDUF>QuFPWqCPjSJ{PD#;5PTVg`4DM(jjrE&X?U
zyF#&;q@VnNgW!B#{&4nPC>GKD8~foH3SO&rFG`^Ykv+P4dU^&1228l}=A~mM@J3hH
ztp;aAV#U1)o?^f@Y}M~*yUlhWxy|ng9_Q!xPg(hloyql%6PuQor|DxG{wJ&HL`<@L
z&j&VZ6co^l_h(!@_e@8(E=V1E-iYG5Kk)Z3)D|fk)6T2ySpu3
zzXZ6s8~wiK&;FG4-my|wkDmRht){j
z{XVaGpUAb7N9`xymwIERa;hH;rTO^ybo|O1VG{Z}Jw4s95FjlnIkV{MEt4xg@4fs!
znk@g1ii%46-XO22sHpwt>?SNF{QsQ*kr11ym<-@LlAY)pRtIa*j~_H;7WT1=MO^FC`(Vim-^hRJ^e|HgyClrcuXXE1HrYr-W{{AuZ#_rP|)qC9?A7YxD*d+OrNo=Yg?H=E{6+X6sFByx}
zVvzN3eEIVG@1L^*>Ac=|J+CXd`cA5Pg>+%9O$RCM`L=OzaD2)X-I}gle)n)zehlej
zA&~C1JvXzq52uZ=Wjv5Pjzil^?X@Yds4#`QW!$hAq16h-B)}x3m$2@}Na)*#wP5@Z
zu#&!b5c2qTr)iZgNH#cHZYCB8>Om=B7Pyq>pmy&@0z38eRGR{mNk>3f=VkE#rzrnU|HA
z8;RSI+ZA?L+2Q&USWMlCL5$&5d`%xgz{pQvgHJwbY~<;WK=f{YNpiI-i@lE^SV|W0
zhp7H#Y%K8N#6EXT0c+Ah1RaaeLn=q3gw&oz0xy0NEvdR%x9f3}*|W#5S}7s?-1Jx{
zmboSgxH1M@)60U}E+c
z=O=LBQXcCglU4Uv_U7`KTpwJ+E*QbYM~_+vX>E5FyXWcY`Lytp+dcHpKOI=|se6QQ
z!$TgJ6pd2ropCJk)=b?qH)mK8l6xKBWwkz9)CX}LjJHUK+v;g>zk@RxCr-bSXBD$H
zh6Bwm&sW*?VdUN4C;n17+l!G*vL5R?PoBI}W}N|FPq}tCQ`9LfFE1}GP0Vq;Y@*W0
zve7lzv4i~Y%Zunz8eP*E6-ST%-n+1(D%6~KX8esq-*}b#Z)PwaxawXjHp0bq5}|Fg
zc+@b9SkqO{?9QP^Zijl9h(DSSLt}265XCnWs=7TMVJ#oGwX@hm#3Z!%*|Vuu$OSIak-OY7HsvHdQ!
z#%Rskivz>BO#?8%5*zlUE*pGp(rWQ3*q5FAj
zCuL>aO=jBEL^b+6TAX(QDSa$^`*z%>R>X09eRHA`65HqD;e|#wwz_eh|6X$h=Y16N
z0KoO`pyrWZG%4_!D>v4C34c{-6riIt5F+;3`YRpPVV
zo2TcyePGjfxykV;`P9_aYiEB7XMFtm1YC0s(jrCw>PU(v=hi~UCE$_5q9U4zUEemw
z1jsak?Z-B2pTT9_yFw621rNthno{u=m*@dOt%OwZnOAx
zvcHzjXX1YL`(fU&IE1eDr)QsYq>muFm9h3egX?l7AAg5~w)Fd1Z`~7d1t@q8UrpEE
z$Jz|He(}%P%aWHb1x(-eyEJ&Mk66#Q`tYPWp6spC(9m>ToH}`W9x4&t^f*}8&Cq-T
zzR!5m*Fl7~bPVV5$Nqlr-Cim>ahF%44$SKZkmTXf0GCOSTFk!v=Z4+K7sbWw;fknD
zDaCsKJbCi_>{rKdopS=?O}pXThhkX@+j5XwAhZ6Nkd%MUsrBI*!)$>PmIUXfdy~KH
z0yr}y#9zDKzK5HlPLq7&;}CqP1TB8-dhguQt(*YF=R8{23r-oYvg}mTmX@2ly?>Ie
zy#`>i4j^8%a&mI=-ge8cZ7}|sx(7n0?DCL2Ax*F-MerFzTFGbT<&AwX({cIdS0iuv
zy#^O}h*Ln;Mz4&6c?NCkR2h9%0Gh_d#Gu*xDJx^x-dB9Hf8zh=^pFm`-PxJ1aTL-X
zgmb1Kbh5qs>1kZ~25{!z_W;pR$ylzbsKY)&y83@bYXyL5#0>
zC|A^p5+eX|u4&3w99j~+pc1`nuLT;Vy4Lq$eE?o?$T&ruCY=XUt_=(fBt~to3_J(t
z45e7{3D`k)6{*C>L`CWFq`EYmfyex6JF>90E+`voii(XOWH>!uPEb`=M!5mc&}=5~
z+JU)>?m(|$Hs1gHN{}u_2O&slAZ$bhNQyRUQ(wY(IzF6n@3J>^`>ohpE4ob0<^xy!mwLcvDfaiR=pP?d`3ttu=4<*Z{ZgV+sT#%};{Z(GJe`
zq~*_-lG_k$;~~l}wBH=wS1BSfeJnw5SUIui`qpuRM_Blat?-0r%%kx#U4Xs9Lv=mlq2H=FP2#llShub5;k0JXw`Bw7TSTv9~(JHzejX`2&^$o3+xPB-nNU2usTE
zw=?8pGl4tW+5u20xx2eRW0E7Rv(nT1;`gg<3ZhgN)LGJA&(=;-Qc@`SP3Icj&@nsf
zUVXB=k@bo5Ioy80rD1CI3oL#V5MN?qqK=7)&*{NG34PDAv#%13e&^O<;mhRb&mm|8
z-WASx<+(K(9UZN!t7~Fn!XV`#;j>pfyf*alqyKWeI9LIA!qeps1aD?!Y~DvL_a*$A
zd7~)S4b*SY}gF9!!g_&w`)e+bcBTA3@m-qmJJdOAA9
zbab(T;^(t(?C%(tLnKX0Pe1y#a0$jLnxVPBHtf~%4Zp{jFYWyN{NUi=6$%NM^JIt+}A
zuCf8x10Z4uO}8~Q{nv85ATXgMec*?%J~*hMtu6cb`^RCq%U62kfYgr%d28POGxq(H
z_`(6vLAWnkdF`1v31
zF7=X-kTf+l38Ys7j)a@Y$HzfI{-szQ%7jQl20$qqckZZ?$pFGEu%wOmho3Beh=1Jj
z4dQoez#m_TD20WE%nWapS)qjZ9E6H)CXjAeiHN1snP1z#mO6N4tu4RW*|J*0HRxXP3MP}^Cl=$Gq%Yw=oWddx+w^Vaky+?i6q44VZLYTjE=%t$=r<>zORa7Bg^
z$zqd*VIW^~JP@X7jpyazx!TQ_d67&3jRuv5v6TEFRD@w^Z=uYG@GMX
zQj%{7VjbD}IhZCG=(lFi+js6H#K%uK*7rmbK`n9)dCE>Ki;Rp63!mEcc*i=nmo&Q`
z%9`@KrI4*`?%xNPHqrFhKC-7^Yy&E!CV-aNGCq4pTFCz>lwEt2f3QU)W-NSGQ=tlE
zg6CaLxBS2mS7llsA_tj)?2gRtQmkMu1X@55=>q0xOuYd1pav#PCK%p_YL-*q|8T-6
z&wo#S1*@k8GTPJAgACyZ8oIip5Fru;cE8*fjAxLhY-N=h4t>X#qz0LM_dVI2V{ruc
zwQD|C3D56GEg*`H{m;_cr8jtZ93fBv{{veNRgeW7w3g7Z
z4YNAiD|_YA&c+USJ@B0CPNv8|ok6Hj&tKKOkkU%?0>0K*cCe{FLuw--5sNvDpnSpO
z;^h45EOsd%@eG1{RC%%92AC?RR;Ip=&dvwAN7>R|gmxmZ*QL3>xa3@E53WGcqj{N{
zDQPI3k!|sG;y~`Sz=zJY`rIhP&?wo1s5>(=173Z0)N%O~YDBo#rbjavQ>YYy!(8KT
z)}wa7X(xbks_!)r_Rn_&UN${m>)!0~0Mv8{X0Ru7377r>v{&Zq;UVH+gAKMhIy&mx
z765_x&)Jb>lly9d+yyo+d!HM86fCwq;Lj-(db;cB;41s~s|?LXhV7J{o(U;W1Om5#~2qp3+R
z;xGb`?oPHu@E|0svWZQ2(62v#r&|U{%E@&=NC)fT9(V%y7b38}vGMU&r<$%WYV65<
zjDF4=UrL5$_CLZUJ8wo5P(BL5xdRm-D=7E~Sj3S`#ex?Aa+ee22xGRvF!6A4Wf<=a
zrSa25G(A0gVqie=;(3Y&$NMxgH2tSJ+6@k)g>`jxa0NLfB`tmgP)w1Gz77yh9(DW?
z^Kk+W#BZyrAw^@g@K!AIT>5e&7}DoVQM`F`#)LH}mg)O4G+#n4lgRlDL7>afpmIXo
zWlnyQwNV`}B!-ul7Y`4QrPHE7C;u@kM1d?J>uxA>AXYv-+%)i(@IT&xTk006tgfyW
z2%T+iI)--boNm<^85n3~ifjy~^1@!gGE}oF>+0O0xPh3O2~i-KtDN`>+eIXIzSB!(3905gT$Y;
z4~ic!>>xX|sl}HsUsjvE!L-%zcqe4_i@s0PVI;qzLZfD?a$*xAY4_{<01(;OB%g(9
zO3LQd6c;NOjmA-5FNEy>Ftz3<F$WQ&YP0Lq_&}USaIdQ}J%w`f2bj&r7uQ{t>CK`bH^AZFv=W*S^R)Br=s%VaBB+g5}j5LMg%bg?g}fPZ}Sspy!)_W
zwy3Z$P0(^4Yy{Her#v}1`~|@Sw5}7!`exWDNOC~Snbz9$juxo~6TYqkD!8J;5v&Ca
zNd9_4-}0faPp(nr1jLR;i<-;5Awfa&<}l7iI3j@C1T@0MftMFRY<2fwprK8+!Fd3d
zZUQ%#yVy;D)ngOxDB)E)u_UQ6`yWBZ1*dpjW`Z{Y9v2dVHWEM+QEOTSfE-wkr?3+^
z_fY7TfRSEaS^^~X&TUym!O{2F%+yrMZRslHXJj-1-8?rp2U#7{+4Xa_Bx!UpltaK{
zA4A?*@ZI89Ky9;-bDkCg@&-IYp800>lbLr9?*Zgt3OHVb-~9s|Z0C;5u&S!6k-mQC
z;iy%Zd&@WZJ2g{uR8&S57VTg&dBaA!x-6+0TAX8$1S$+GD=R8UNlEi_n1gU~p~VnR
zWkD_JUTj*GEBn+JmQPAX#?mUo$@%?5JRMTEgUV&P@V`nza5oR1*UC$E#tpC$-b
z%=KQI<2dnTna`WI2m?ujYiZ_ABY9!A7+&k+tan^DrgT@+)tqF2NEkOJDZ-4?w+YB?j}c{
z8~6x=q`NItPHXnmg
zJIDc00^Ff;?^^#XZn9}fX=#JYH{ZqX@ch_r6bwO{Y2W^!Lc>a$2n7|9ltcLfor7Ns
za{Tk2#jK&2W>;@Y&~m48VkwwE^pzH&~0{ROI=|ufkg@rZ~6$X^Ch}0Q~-m63D)}<{#KA3c(Iv*4=H>F4e
zG5{)z*gh#Wwf^C(SX9`Nw+!Eqz`b`r05?Qj(_wfK^6zWuS*QY4_0DB@AWI@G87V0o
zxcB#uG?%v%P-KRWHYbKw-Mg&C*xxEl7mPX_{8B8?0iHvVM&V!&ok)f`eTd~FQ6S--
z2)kETr4aso5!yd_J;(RnI_zGY{p#7!qLxu}2Ee8EbJtr2a-ff);UthAZ{EDQVw_#R
z0e#X8L1jfZMWe)ZocI^wt!!vlryG44t6}9KP~(sppl@jdIZA}~6XX`a0l*WfCNQ=E
zR)&PBR}PdvY%rkb_Y>WD#bDdugED@<`Mk+>>kOYQGnWvs63QI;wqb^%3Og@uq2(5W
zLCrXe_0#c{p*8bf&m0`|yv)
zkR79}I-Z`1Wp}Rl%>R5shmR=}NHi}!#|uRQa38EmYLJT}{IDgTJs(=_#7A;Oi0EIj
zP>8{}zUAN)UDrgtqEVpA4**|6>laWCWJx+PXOzPp2a(C~@#2>+#a!kjq@{uEEo#qa
z=>LOhJLMn(r3Dnqf&v1BjUhYAl@lt;%FgqxG}Qq~YV1($yyvZH-Gkl$A)`!VMFnok
z1i%!4(050)O-!I7be?OxKD_T%-{`ad(&{-N^d>V#8k(76_2)}VHb7*mXZ6L?rDtTA
zn3}>G^e+4LBrI`FDP@8`>OqqfXh8;C|L`5tdQ+euz{~$YV-AX4dc~1ZiEl1m`uZuK
z>_YtYL6k1
zg{{`aH=v;n#Ln*y;3MlXZ2CN@P#s)e{BhM+s50F0km`4n
z&+UMk-Vs6!AXXrXfcmh64m~vJMXbBS!29G5$L~Vv*I1tN*Q-L7w1a~$A*$-@>)Y9J
zVyxrh28-9deR}}G9tBk-Kfg!l<5k~!Xed?V0hqT=c7;Aq8$>eb1s(W4H8rIbc#xKs
z<^rcIa#QBJ7U%QSw6p*q!zp7qZr)_rMhkufj|SitrQM`eg*Ffq~lE+Td1Jos0=s
zDJ77iR){d_yOwvs6nRnyfzK%ccLN$1>aaH7!@3z~i0{JG_)v?1Nm^Q4FG5l2`}+qC
z8QDH$pd7itc7Rii393lR;Bw%B0L4RZq3C|!v*Vi_;8Ku28$GuSp<>bvYZ_j28p-d1
zGz1M&*rnu%#mW~iUI2KTb$$m=(ptX0z7Ac%#&6rOEv#6y!}|oZBF86dc~yqIJP8%t
zAB)m>GMFghG#V@*M|;8Y(hf%fNklWyeBQ+puB|AS%?}RU;t*iZ-;&Xlcf6!oE*5s0~0X;f$goDjtE{&bQLOej>qoq8X6kVU^3uI
zRV>)?mVq;nyEwiNn2OPNy#Rm!9exA=?6lO>2e!7@0uO~VG$E2S0=op83nduj=pXJ)
zQjnis$Ew8Bdj6Q!EggGcZ-3@5oAn7(RRuyD4o>Bc!((8NAqM+@w2}dzfRNOQ!}OTcZNPs_|LfN5@_^6VtS)p+LLG5?my#L->npU
zMt=MgLZWNG;YiVhqt@-)p(EQsH$r**_N|Ej^84Q2hYRgF%Zdy5%336
zzExmXffF_K2hq+t;##>b3BciiQTK
zmI7kHLg3|7MMXt3vurbg4*wH(A0J7LjzO
z9r!@^CP2cR*RSsaq!VEf`CE$iGDMOm-<}vBKfkJgYb<(iDoP-`jiJF7AlW#hmTxCd
zle-3u5WVvEy;@nHzC&QcXD4M{jN*KcU#ZWdkTAk9=U%SEt@G+-&!TISnQK4gzyb62
zvQ{l0o+`0|lW5i_ByDd?kft_t*;Mer@u3@ME8G=cYgj1?9anUGcD
zD%YoFVEE*g-D`t-CuPDh{J_ZdCsy$}f~n3-z^lm&j4EYlHIZLEAEjqJ8_sYMQ9P1{%g>np%OP3Ysv0Q!d8y^-3GE=QMh8(L)aQ=MK0)GJ8*R)SK9{RSq
zF7ujtj7*vdP>B$dt3-MuW(uH0?6QKDBijvmQpK_eo?-ON0dx5}5e|3nDy~{2*kR4P
zznQ_O#hdGYP;30wv`V|Q(kT=oKxQUw?iz&GE-L_w@|tgEz-K`1gt294WYpt`tE~(j
zWwmgEbgLd)bQj8eB4@J3+bRO-O6)gG705pj%@UJqo1sSVqr3HX(%HQ+Dc5%tp%qeG
zO!`nJpgb&$k3LF&@$qAwv!`dhUitWj2RAW8oGN2NDWqwa6T;*eA}01%bzN4SV{xdZ
zP2XIsNDoLuQ#^N6Natt>nUDq%Uh!{zsd-v&y1u+cVZDrO8bfNck+TMmffTJ9uP?6#
zXyb1#Q?P6N#LWW(;7gR)YPia?dwama$IM3_&*nEb(>C`!B2|gsI`j?gLK%BiQ4tbF
zeR=;xHjhZ)D5Sg7iZBx0B6(+fRf4dcB91IG6b2}cF-XK``1)E}02
z0poT%T>?+w`E6C@ubfSRM64K+*OJe?-5b!>97BDoKyG#3NVW2ggu9bjw2=3Xn+ZX5
zaz74A{m6ibLMb;YDK}k=Df2&cVL}QN5@iv&p=fcRw{S7MM0nO2*sXT3S@;~HK
zIJ81~K?=JODkDC+=R#cWL&)pXGQ%y7sAC6Q(zGbT;>=P-WUtYTxcc)iX0nppWBno!
z1{iHcYZoSi$T>kqV?6g8^W;+5ad+`0FdA|@zsL)M$^Nhk(cxvz%a``fcY?;O77n30
zYyPd^C)5$TJgFm8_|&FKH!;T#H0&SDJ@SFxRs6#
z)bZg@1iq2??582jjvypu;|;wuyp>5>eXovh>q(0))oXeOio{36c??pIWBb?d_`HxF
z~?KlgojS`Kg^Fs1xzJLfCiVdz9+E?W9D{HnMcVs5J
zJ4w;cI4`cAe7$ljvVtlWW0%XP1V*lG<|Mz!a~_hsqGi9|gB2}iC%;s5#MpYWD48ii
zY|_~#G1N6nJxzU`QIpt>RzETwWtsr+b~&2ne-gyLU8Y((93}-)w4yL2wLSEI=ma^{
z(8Ba(**_GVi|gR4mt5q8G{(h7sM%0DJS~&iL>sUTIX4dB37IdD*(BF8LBjEOcr$aP
zDD&W2P#GvQ>6jBwCe5!CwMxr;CA)@yu@#OPH`s%PF(=4V#=6dMOL_V9#+UMQ3Z85w
zosShkHfV$-2dw!rndb7cT}3!LuQ&yKu=FWL`W1*!N$8XGg?P-;<;$CNWW~D1(a>n_
z28kz{sCJrqR?%`33B6{{x+QNQW6SUqyRmnN$e}DM7xfcs2!}jLl>-|_G3MtFWp`hY
z=<$UtNh
z*fZa`9Pe6?cQ#KaEfH>>RDP{2%^C~*e
zIm=pjLSlBj8!f3jc7gV`oM@mh>9gnOsT9BO4F&qe6x#L+|EKyPw+>QR%IuJC5Xf(F
zRd;FaBg*k|jxjUgClc1IcyTA01sN
zb!(Kp|6wO;B{rIq`l(|5$aSl3g(bJ~kdMPGqV(hxg#AnEW-RmBzZ)ovUqb0to2F*i8JOQe^iEGt^UkEWDb
z!BglwzYxbk#EpDd3^X$7S77~qVBP&
zSkAC7>YY}msID}$M75M9?!<$W-W12o#2o6jBSOtR9PY#%p-vKwhS7NWEBoC=HKAm8
zi8Z&FFxAnrnOO*ky$`N6&4&*ouA$o;5-3cuf01mTV@ahar;EGbpkXmJb0>>N{wV8E
zAunciX(Z|83b|3qQnPZ1#d?27&F);udo!`aP`%L$%_XdP@a)F^BNVfSKBlF>(*!OV
zbKO^I0>_#*D7VT9n6;Pc(nz;~i8zCWUsh^8-71qzzdkQ)p@P
zZq2hVXjrXz*VJz1uCc7)`l_+cD@ZYU1iTpOii>z~|Ltm&<#@6R;*bx$8Us=4%GWv*#qsLIPl!Hr$e4<%eS!q4?q3`g
zPk|e*QWVTLvZByYCaq|woW~MEc57?1!zx?P
z1UfBhd%E}F`dTJMnj)oBY^SgtQI#;o`>++wk@WS`bfG&+SStbdB_tC*qhQ(dysg`L
z!`dH2-70u&Kq|@xbjZIiQX$s@JT+aYG3p{t(|ZuYAe*(Y07W*GNUnV
zYL7cTZ+D~Iezq)syX#S4d8MGtQH^QUhsQmM4$<6*>oMINZqI$&EK0{{WbiyXo7EQB
zbe$7l@G{G?y#LC#gEACkJ%Adl>@yOzv+{-gX%~YKcV$)n3^T=|jm6tZ_Ck#ngIcT@7^1#+Z=l8?)ZW!HaKK&`2zk1Znq>W;LPG>awnMHj=0SkBJ6px7(SibkhkzT6dDA>jv@4GQws
zbHovR!tC-i{r?*9l@0Sz7pJq11(@4leQ5_ZD9~+MS_B0LK6Wp
zBFs=I_BzWSk&@vI@4GYr@o3QKFc>n*5r>r@uO%O(rq;wV-~^d3cbfKFdOt;pR6`q4T5w=2rQBxbtHfgJ*90_mYU7=#$^2xH_BM@k+!Zn$1K
zd-TWm=Jxp(uxR&sf^VfUEuXQmqR)(Qhz=m%mLa@@l%5f5X~Csvni7ioRIiAv!+LSa
zkn!@zHCgbHJZ!2J>;!s!-C%F|Qj-bbxiL-Epqnjhu8PVeR
zaIil5-!(GVDMW#cFigm4Fcse88Bd0JZSW65z(rXd)-j34&`C*0CLpn=QCP+ti(*a6
z$Bc+caVa~6bqpVvEwM{27EOq-5sx&mAVHu64vQUX6<=Qvo==Zs+|>
z1z6M+`@cA%%$^P3(w8W-vIr}$!iZDoXO}qauL)LouoJ^>tK804n;n;M!}gYln8MP5
z(D(3{D*arj(yTSqbu!gK;a=CCStZ(R2s$LXI!%ebV1Al3hQWelP*d&z$hcJNSI6Z*u#pdC
zbP)7s;HDBa_6-;<&Z|;ml-qmR+>Tg7y+G$j{jc_WnsC{K(VN>$?x(rsf+t_7zuF=m
zfkZ33Y*-v|NEuCe=Q{-XaWQY
zKS?WIVVhknv`;T?{Y5z}xN&81vVXzF5#hQZ)!)wv=k7=5>NU=VYmol?vm;p~VJT?b
ztBq@Ipj{0lh|Rt*Xo#*}=aOCd{cA#X@^}BfwE!_4NUsPZl?rsu=kK2|F&jN`c6Rpl
z^@U-Ejj_^lx4$b`nh(bRv3?K${cK#^7&%IRS*Clj(ih7de*{BPk|w%4@Xsq=0{s9S
z5J<;%));7rJiJsjC(7zi#s5D2ZeIvKeFr!3=@BE*zt4T^I8mbngt)7FI{~9B?vg!1
z>mIeOlp!yXES7l3V;u}>4;Tm|`3wRtyN(Yp5cInvx;QXIBk?SB7&p8dR9td@
zhF)lLW@s9km<+hlZ(_f_o1uw$3H9xszD@oPg;CaDrg>0bz7Dg_^@D@3HCTg3RiO#JUl6~@P>Y=-;
z>m|wBMV9W(-Fu`MZ*{^IcR{rc#2zDbSwI}%c`XtxU#$!%Yfu}7LYHOSK?F2;tL{E+
z^H&TDs4raa$qw&BYYxcjf4()>YjOgy1ZuEh5ZD$l0QKVD1cEvPgVf=*$5J^v_S8Xn
zEHJACon&ZRz5nomHXcfSf1t;p&jbo3Xl}fPMy+7rNvuAPd8>D=)iW5Sx_;da4iA)3
zbFZz@F=QUoPr>j7XjI?#_4U1fuWw`o!cq`a|D11YdG$*3Ra?;CEl)=5KdDvzxqpf9NwAv7JZt7oMfX*B>DJvYj}o#O_OxJgU4|n1BF-TykeP-hW-v6-
zQ_-A(Ai>{a-B_X#@BOBq(6!f9#FOEBE1Dr`_yozWVcrQgP6Z_0DCccTs94dbibD)x
zeDXf?L}3^tKXh6@CB9kqVPv-wX-!;yPa|Q=gL;hHEPsQQg$*SsyjVuo(zZ!SIz{X^xUvLI)F2<{n=ep!~SPd1(*v&-GA8H=c))2Wea;`kz(M+W8yN-VY)XUfJ
ze*mG33rH}!j%d*e2gdHm-E>=e4~}I4>=clcpil)8k#7hN5rf<+?ZMdC7|4WZ>FA(O
zVUhJo%&Id8dfd>%eKh^LYi+-`ukTKlSp0J1`np57;wXrcfUfR2t(e#Z(H_Vsx}C)$
z=_!_)7keUs+&71TjMY{1c7I7Rv3JnG#_us}_irjLE{3iq@W9Y)g|R9b|6?IX$|p0x
zYh4asp2$6F-NoEvzv{|^#LgcCdlnY+(1r#c``3Ki^78WS+qZ`X2g_E|uMTc$7zGoj
zXhoQK@B`|Sc|ShW`q2yaDz(n`ZnZPi)!o|g6osuB1`A$W&sH{Sc1ao%S1N@a)5TAC
zqr_C>Aq~g{2I~ax7^*#G!Pl{}iTDP5T00y)21SXX?^V^N|Dk?b5PkMbRJT&z
zf@DE-g#l-sndep_H(EEAf5EP4m=Z+_xwJyEX&F_mzX6qMQVnfE45PbHR}0zg#Mgo%
zdfeo&o)0j2)%17300f3)v`A*s3V3@Zhc&H&QyLf=0yQ7-+3&^IM0c
zoUbs)R6rOAqHB_(Q3(*a{P^KqI##aF1LU7d5fHPWSB2&{%ztHpcfA>wjU~4_ZU7B0
z(AJWmtpnR&*=vGRf@g}3i_3;?JhaSUT+vKmYQqEQR8T9O5oogf7s=76SLnh-7`r%b
zAH&E2Z>mPB2Ey>6`zNge9eEd0QVIk;Lg#k5zT%f>$a>dbkwAC*wz2q;jE8$LFyJ;vK;m$%`SATv%w^kK8m1Yg9M=GUl&8I?uVH6=>FG*06%6ZW47N*3x*#88O2
znLC!MBZ#H}lpF}K4co9LLspp=1=P5TDiFu~>X-xjGR`zs_Z`B*nCbhvunl-9)Gd$6
zi4~MbG2#i1P_Q-#V}eNte3(Py$QBCZ<&lB9r;IwJ^F}OK*QO;xYShFC>qL@{5L|M-
z73oogD;Qm5vIR2gVU*Xy6$|vr<*(y?=DO&!cwg~8u>@znGR&5|66FmaEc{Jf9VP*-
z2h7TW)q>2cJIG?%NnKt2N{T$ZU(%Mpc#as^#V*Ur
zkmc&ijCn0pBe-2vFX;jBv#}mliZuElWSwv%9`A3#^JY@%+;*i1U-N;C12S5hF)_1)-7uhz)gFw$=9-8~y151g*L4w_l?1C0A
zIvQ656~yxgHxDB!sndI3XOAco3?v-I?DtVg_j;+KP_e-ebc2{ro;W#l>h5j
z1yd!8-Tw2$Rm3uj*WDs7Y@g{6P0kw`nlOT~KHC66$Pctq|1`|rVz7r9q8usDLC{L?
ze-eig!ti+<7?qrV%T50hT0+2xb8+?A3M)*}_dyVZ<^(4@dnYFvf+G6^-_X)2k7N)M
z`tKlQ3lFY>`2&c6(1IWTM?|4vwhgpBR2JUW$2*HKw}EB<2!!e|asbW;x)3V?fIHB}
zg`dbh`9UqE8tO&$f%$J4a1nK!K#KVOc!w5Y6tu|I0UnKPW=Y5P$&HN5WUqI%WL+1-
z4OdLTda*2_X2Cj$j$c7uO`hZz)T0yBR!^K2c30n+bH1p9P`(T$Rti1NFDohQnNDWk
z1SV+_$_^K&D@BR@Ps9J
z_UDr(vqB5D&T~-L*r{&6jDRMhAO`uZr9)o^Lag(V{F=>8yzTESMnN#G1D&|9pq0$a
zlY1~ou8WmreHZ#z&;ygEf7aS6lfq--`pT3w2|!B78tC{|C5*UJHR?^jHM-q@Ur67_
z5-RvsMVK0up);ukL})OB3Z21r=>EU}0x0&cdi9XuWEhF+hM+_41BymCZD>C4LimH3
znnV^Q5F|~#GC_X_6U0;u4Bwy&4s(LWRfcn4{{xKn=%Ljj7_olDayY&M=|f$fLDDLl
z6(Pk~&z5ECnLj#&AKcLi+Kt*`J~2;xiPe^G^#)l%z8$Iu?E`|8R~*L@ZVV{wxy+#$
z)_5hPVE6ebkzPnGq1_<)=o%{(Zf)sC8bu-Pl(uUt-353hUFc@%vW7-p+#hnyPhg>{@ls@GmZiJUm<;l>l8cENoe|7qpE~E#F#^
z!8t=9YM;~fj#NZ%YNy~`w2t91*$`iu4vf&y)
zL>Tmdw`h;}FW!Vx#m2`QaFY{8E`wT~OnLlv$(M?X3h6D4@bv?jPq~vJTw=)&`bWrs
zQUT}35KA4Qjhgsw5a#P)xEK11&@NoRpKud&$vmN5Bb7#iFcA(41GY>UORO?<63g1{
zbD4f|Z9$Mh9i>
zgKuUq+V7AHa^gr=$9&@@$}l8eBpHS{cj7_#@61k5zA_jj?Pu7y_*URaWPz|0e1FbT
ziR_?S@u%1mlSTE;wWt0Ldh8C+r?#4YgxTY(nII3VmC{>Th&WCN8Zh7C+VX8D0-zTv
z*))tFDZPtU4MZ|T>m_#yce~APn--UpK8@
z)l@ThQRReUP_k*&Z%YrD{5e||j!nSIFlQlWVAnG!=Vr$AlyrvQ`TL(#so*wDAVwYthAK`!5(xac5
z4)>fy*@Y}hq49vJ_;5jwxOLf4o|+6_J+1j|OAD7jhKvw>vT}$4GWp0y&%+g`Ow1POg=$c2~2w}Y)R6ekI$
z@Bgt>^%V6JS5uX(KYgpBcRj8-fAnfn657^U8XB3BkL{qdkNvpxc8OlrCqo^be0`os
ze44_kt(#XOl5s!~@HGQ~t;ZcjC7y%O8T7p4j)x#j0`(0V;b7<>)%?$9q+Ox
zrKF{qR!xGwAt)2~;z@)xJ|_A}Wf9D3fMOyHU76dXsk!;9gNPJ8q3dNgxpCSb7^{g>
zSq9A&+_er;A$V`mZoh3EJl~Zrl)mq3f($A_;xZzk49%pZq(FyRg1=0FBz2=93;uv8
zzNB7CKd+%ZUfENUXLs{MNGl#*V}$tYQyAz2HM{J=h!RYJasy!3(s~F(6foB{TkkXlK}mv}J$>kESQWAh073!F
zb{T2uGnmtaMFR>4M}%nhwIfhYO5YNW1agc$%q6xR&Ci0C81f7ZvV8gS#jx7U2UMME
zDHQ$|^nDzhoSJV`J$gT=NwR`agdLWFFf8&^RAvIsvw`N=m}$
zGrvXa^r9>FcMw~vKs^9DBgkw3E^G7iQMy|}3rfXr3Pb1xiA8@EwT(ob$G6*1ITDcS
z9-F6wm+W*+=j@F-?6wJ>s{*X__QXwcD;S9SkpA|?q*__*YQ)5$L856_pGlxBP77uA
z+T1Qj$Sgl>JExL&C;>MUkibID+4gG
z26C>hcq-&gS*TfHFy~6f4pOJ&WDxO*z}F76wzh(ZR$*I3KLAn$d`-%a=~}Scr!c~#
zuzb^d2MOYd-SaEN3`4Duc*;w2X>Zu!4*?l1EGSsh+L|r1p#opGAim)LJR{>u;S5rW
zjjlNtFAyI>jikGGBi|JiQ}E>#XWz3P!F24ej~O>YVh1}z|A2_j(7*uVRy71sFm~6e
zuOK1^N&mVV=3fd5rx;{jS6zg6;7Cv^mrra4eQ*yJ0Xev?4LeK}C}T9tewqwae*&j<(?}6Y4lEv
z0PlyJvtj;Pl9FT5H&
z+K@`h#N~9?=iTGTO=r@5!$jS0ZV2NYcKnj@ym)m=!ac_43!Df?{2n3K-nU5F;zZon
zL#uVOKe=VoTpe8>6Y|9_Y9ubAoN>2n?1QfRc&${8zOesm7%bEh7am87u75H8
z_5+xT1nC%jFAk{JK<*DTmbk+RGY3b>>({QJBZM49I%h(_JjM7cD%Zz8)u7_^U#|50
zpRUgE$Pje*|Elzx@~GGYLsjR#pp?30HDeLBTuvlV1NKB-|K3
z`j6&q(P4iHvUyblgNa~MZpTIUrX6KV{j-vP_q6d*r@S%MP0WAw0K27#=8nO8cqvUe
z^Rc)8<$K-of`k1ScKlraBN+9Ac>8v&xPAGe{(fxx`%9Tl>qvPr$Ipw`rbD(gd&R;R
z9?=w21`P3{gtvvuoFW#IrxX`X7mhU~uGMx5uCOPpD~*xJu#{83E>E;QC}m{XMcg1Y
zp!y~5&5x=u5<{jZjO<|H;&2GgCUY
zrV#U-I}^!9UT?5P`=ozcxy=+^7-w-BL%FdpzUt_B=f959CJtsu`*}SBY2tV@8X`mqm^)Yn~SKLDd4sPCR{X3NW2^BGhi#6{A?$tOwe5~te^yWgS5r%z2U0_
z^;r4~OwK)@|D)-<nh5lNDWL`FjP%8E!<
zWhE;qJ0e-}JJ;v_ef#5n+@Bu8^?tuz&vBmToadT8ZL5Ber$)b(&@Zh8C;Bz-WxgEq
zXd#8qBOlbHB>KCBc2n{Q{V#h`Htl}9ec${#4|~{M5w>TFC(E6JrNb+~?=v*!XZ%+0N9^4rd72=@c2~Jh*ORqN;+c&`O2|}W8-;+-8JAmZ
z#1GJPjIof^SUUT7Qw<5#{uqPZbo22ipQ^r>E&mw$L{wm?*6XI+;9V+$
zt6i2^;NSZc-%}W*dADknoNUe?yTir(+!e-H0vXz0&jvO}sGt3q?0?$zxH26{nuOM<
z9b32mOyQ9iTLN7|`}R{j5SrR?XRMxI;8y*%kn9pE#Sh-!O7_NOOqY+FyODf&5F+p^
zX{oFwi(OJvP-t;Rl(1025}c~=dro`x*{hjcNrL*lzfYI7&xVYC(Af@Ubn?x${~jV{
z_%#3Rw|i~+&(uR5Qb~^)2qfj4Q)J|DRPJZ<33W(zuldUnM%!4Sa-A<&Qu|2Dxf~IJ
zwG+JNLXW8zNz#u|Rd@7~X$IA@@U(cbe%B=H*->*NUUTokxiO^_)9CS-bN7DU)E8e!
zr0D!GwX~+ld#}gNy=q8cO3(#v9J`t$Y>mr=#F&ceN|viAjQ2387BZ=FghspMi&cnn
zmtK*iXOf&R+qZLT;gYIMPz?DlZSiu^3IE;)I`^2f+Q$aS@D2NBSnXr$`wVqGYoo~S
zt}~2>KU8#T7$G@jE2#W3g9xuUT3J0;Pp|iUQcFRef9eT0sSbx}b<3SG=K+cfb@pFR
z{7{j+`0(oD0A`<_;;Bw8e?{ryK
zBAqv8y-sCG792IEolDigb}ib79#_1})A}MT^2rBVUV5c?%JIO|0c^u=5sjN){Rk=h
z*iO4Yd#2>S@z=pCWlqvG)?%hmj#G6dvgnCF>;3*e3u0tr*DhM*`qU;yNLPm}Y(4RJ
z7Ok`R%v_aC`s}Ri=tJe;`cXIe?t!REV@E*3a~`)8c|mRbG{t|`2$<~6+@AN>pQxvU
zsV3G2lrm5QWh(H6*KFnu)H~8sIW}n}JXnevU%%{|DH)tQm=O6^ho*rzxRy@_%^;N|
zu500Vxj&7cfxEnkwY4=EufV1t%S11A_XDb@3%NaH$H?uDfJmI)ElEG_B3ErHVXBTM
zT$(GXJ?#5ggLD*TN07dAd-Op22N{>09b|zBHL}go&Nwfp!&O7ka{ERf24<#u8ta0P
zxj?>~Du8B7{~(&0{63%$W9NeF0|bK&9=gl%jJbPu3JMJXe?V&XQsj>qL+^@8cghU}
z4_KNwdp7qC&HO=!5@xY*KWl%YcGz2S$WHracJ^K8`8k4NRPcoAfzW^=5vP@_98{)m
z<@)3$RP@dzhBN!Q0>5+KM?(Mf3p&_qcUUo}9>j@JVh_=3t;qjrld|c&!
z5nX4V9`((`bvNM1P|zAgvy6_%6RvfjWTS;xN5k2DBYGTq?IE;X&~;V~nSx<#Y;0_5
z%9y_lcC|SCRH~~tQAmQL4GMFH{K8Y7x}*G^ay*R7yQY|;$H6g{_nhb1y&FVVm;iV|
z2>VviFprHwAW*1rsT$nY?bxfM5U>+v7%2H&*lC{cjUI=-1yvXfW}u@E89u{*ryc$~
zX!k*=4^7zfxD_@_5N-U1^7oZt3EgQGh;0D2v;*JVm@5NiJ@jwG>sibSHonK5`|S
z;h*hEHbnz0bd+{uD85Gky8^M(k?bJOwID_eJVs+Wj$cnqY7bq|_mi?K%vg>p~{_z&C7|&P{
z9R+k{U-N(CGjGi~`=e&+4J4-?eNljB7RUb1y?Y?-(>InIAAEQfeikPnLJw6rgboXh
zA*j%!2et?KV0<`N()auKZ*>^gLz
z4cLE*zeIku$eERzIc
z=83iMyR67oRj;7Uxu^L1cEId^W=Y!<68;vKWgzm$yPEo1v4l1yQ*H+h3Y2ntp4BlZK4KKALEjFU0Q6oS)1R|3(0v>1TW#v>;4q6(ZtdC+Zp>zJ
zfXjxxaUs(@cXlqJ7X$3@1zt!6u%-I|Ei?ogau{%Z{c6mj1?cq@Ok143mcv+v)_}c6
zS{*`UOe2XO$E33O*)y_%KCcuwJS59F8bU=vbZkHgzi{?WkgO|uQ@oN7D9(hIyxk-x9|%OL(WjsJc0_H^g?V!SfJ|6}XK;n)@tJ2Rdt?7zhI(@noz~3*vM|G)X_uq{lN@{m!
zK=6C4oeA6lnixMI5R*qcmmoX_(v2}gnK&NRD~KdPt-$m2`#a=$J|B`6DJdxoYccmj
zd$4QyBmU{?>MA-0gy#R)_ubG)&d$!l*CZ&4j(!@FXk-MpG98_-J5mP=4&KgyHmz{L
z!MccZvreoH0tLI0FL(%G#)~vc-fgmka-ws*q#s5!%F3j678sL36#)#f8Fqu9;lI01
zOG^tOHvi4H_|w$nKG`L3=+OB*Jy?0LHhsq55RG7d{E(s#4r`ax1{~MlQFCHn%ppsl
zgt+0zxlo4;z=Gnt>rKcuV1`n+4|+O`k0HX=)Y6)<(>$2)uDQ9?wwvtac>%}|0TyfK
z_5^+Sf?)#DgAx%94v_y4rX%m%0l*J&3{&mYe}1L)&e9sq0A-NYNBNrP+L$ARL`8Fm
zrWllz=;Sf6U@wo=Bye&jK6z4&87)LjGk@06iQ=>1VxWD6Rn{mjUze7PigFu*A;6!U
z?Ci0dZ}8Qq1@`TGQCzJ1;uB1-o~b`3{*%*jFtY(>9w2fb{ylgd_$EL*S8oY#yWEbB
zc7kaaq+c-UiHb|Udb#U7o)a|t*>EoKfh7ebLQLdU?61@ZyTu8DJzh*rYS|WMgVPKM8UClF43wRLd
z9%3=SN&!5vh~p5T!LWNd4l$9}Ni^gbaVbK+1_B1W4O*Tvv$8^2dVq9B{nV)w2(!p2
z@bZEQ1Nnu3F{3aG%pdVR>k6281GMl6l9Q5lgE(hyZf<^_2%8;C$&#f56Jpn{I4HGCOJ!t|qF%jkZ+C>gD<+fTY$gK4
z@teQDqs5PkiUNB6NctLcaD9LAwTlpz%Z!Ray!aH#awUh9MT;`9WgF@
z)?EEb1PvjW_LwsyXTJ$&6+EV{E}fx#y_W`yn9LxbUHE%IdXG5>ZY3opFw;6!`?|0Y
zV@KzcC!3&iJAdz@{vu-0F~3a?C%u_x2(}e?GAjo>J^#Yw*T%*M6OgfEw^gAeY{zwZ
zD0cbdSlMWDQ9%KojMBAhQ+GKJk3PhEgS`$ksJ?`_;?iUAKdY>SI;=Lc+jcU%;5^cN
z4NPhvLTjIJb(f&Nf;uc2GS`Xsth4%Hf1JKa3Ii6hKy!Qh4Vd3Z;M*NOd{|D7!-&rh
zGgbT%nQHh6sEartMQsSaFE+19u8F~a$qk5cMZlXsEtu$xI=%>=v8C8a_4e3BME}3q1p}7I_PiRYlDtbGjKgX
zGK!5oTx$r<
zL6+h?<}K|(!eR~IqDy{$TgYahfUg)yM@PVA6Y@o9#!(YWEqqjS5PqI{Bei0e7#nfw
zX_%eX&$1(4^JJP1WcMe9m}h2YP^H5YsH8bgLkRjts2tsP6E0dfJLg}zLCMNq>6!Ws
zr~aM)PTSyv#gc&!<-Cv!1C>U9{o-R|qdFc4ONO=e^q`I@Cdwz7d={OQpf
z%|k;&O-+<-!(*bAPgYo$hEIqPBGk~H+Stfif8dZ$ego10B8MI2!{BuCt+fk9DmLWB
z$02KcVIX!EW@8G8?o`LzLi6JLIS-lVGhV5)HPzSmgvC~(_K(rgIwX(3wloE~D%-s^
zf||{0f`!kYTY(vYSV>As`YX^YVCaIlJ&o590iJP0Z3YGgXlf}Fh}jIEo;Y9p{8`sw
zNX{+2ApZncQb3JmcvW7BXu(w7D87)Dg$2rP-BV=iY!%(KhKX|~w$yvW^
zcHvLo(1Fy0V4b0mdu{L%>6mD`w^1<4G{f$-xq2w%;biR@<_HQlrQ=U);g1M9Rdwr`
zrp0J7L1T;`Cf&R*UW`-X=TP*Ld`})&P*YRW*w_ebF&O^hPl?~FrW^9(&nj29KYm!=
zAsgeYqpK^#906syvg^kT=N6LdF#G)dTU1O8w%Q4r83FUS2QBlTUBhyu7{F*VbStfI-&G+^m(VDtWx-pQWWw@7H`--6A;njV|E~l{k2*
zk}Qtn=tG3~`7=v#{oi0qg+ULm*QiJ_|0*dhg-2>;WMms+?b6Z``qQkz6Vh6_GyAyiF29DOqa*6_d-v{DRox2r
zV3^&d_Q{T(v{;kq1%j{i@!7-_PTs*Ty7DisC>2ExD@MJfWAMDiz>6m_+=Oo4sIs;Z5k
zLW+xt{pd&@yj-PA7B~to$%%=0m<;|M@rUunfdjdyaT~l!49}!@qZOneQ^u7+5it61
zsxSX(e!eqmgQuN+-Zou*^nX?G63MCYVUZU4dV4`-M*1h3E6T~O!i5qt>Gi~;p(;S*
zxuU41ci6Q@-1wqT>NSn~b=ASa*ubFw%a?TzkKZ7}&{OS!Nh4eY({$>jOs{t&Fq4e=bsad
zg@hajYgsm>oBXp#=1=9uJz>pxJdE##hfVnjm@=5_!$Hf)%&ZpP4+v@%#c6x3?4@7?rP9<+0?muy@*j;DCbfy7awwpK{!({cWtQbT3`P9kz!@A0#K>
z8LleX+z7hw^X+W}!W3DO4OLFXOJuTVim-j}g?$oSr3w1n*QeV4T~=HvPUAVsFGS9x^75z<%V1K3jCnycKL7Ws(u>e5>|41L=4u2A
z<2QHM7nYZe3oTE=&*w+gq?}vF*Ni
zdo&V3puHUi=DTH`p1e|}yqyMCFc0HT5EA)0OG`>Z!^1s0FD$7~i7F~S(abBAQJCQQ@~p)A~_?
z5&%sYx&`-+8>dY#sK_TTG|-!p?|RTQ&Cj~LdZ+aZCsO=Cl2=pxFMJ>ZY&-g
zm!wdFBjU0}p#fS-5JAAKL#(wO+J%=j;fet*II~$
zKkMk|K-|OWe3_Ni<*1TQbT`}Yc1W3Eg^mj~7KWzyv9`CE<26~ixV%y27h6D|MXU@h
zEx*-zMsA;mN*xv;u^Oq>H_A+Igpn%~(sarJ3m^>O6@ls9-q-gRXb;gk+VsXwe50K^
z$EK!e_tw|1`~%)_!e^C0rSxkOPyTeq1?Gsi6RxnEW1(vT=N{3o;MfzhUz`jbj{LvC
zz}HTcLiMBcwCS%BhfZz}4A+)cR@BR2&^?dt6gA8wM@;$F>NwMh6DOk7<8b1>ex;v(
zf6!#d2Tq(B_)>;fABMq+r6nf+aj06A^|@0isDcl9FWa`{&@eFUhJ84Q=iBQu5@IIG
za;tyXZ-GkRet#9UHVhGIm8Af>41awN9(
zJJ?4pEiK>xrizo$n|0#8H(6c>VgrZr4-SF+Ex-;a$m)qG13*9~rh9np=xBO-dfa<=
zGn~ypI>mF?SX*9sl
zZlh@h#}31GoC)kCA?jvex3`~}o&7N3icAeF!k|)x1}Qm2G#0w^
zlm=pKwbZ9}<5;>>(*?Mfm6oFH_Cc);Rtt`vq2CFeljxrKkM-lMj*N`Fd$;fSaX)B9
zRMH4WJ5OcnK5Lg8b-VfL@!&1mVjAk%;lV{~C5~z2JwUxgn$^tAj(9nEjCV)fHRh+1
z^HAP(@ePT5fSO>0+B1AqkO+B%IC$^L;^sCttG|2%ZpHNIXsSYmhEUMglZ~Wf)=1aj
zGqp`X45U)~4vay%T)wRrr0(j4gJ&}?lx#YT_M$oF!-VHT_VDm<*U!@2_w=GyBbqER
zAvk_S+%B;41LfA0xaJweD@D6Xke{k?d#aEP!s4pU$z5!dd
zva#{JmeWV+^;TCD0qisIxsM;2_4cyR5t*jqZwq&(7@i1csuvIAzioW5te_x3bE@m-
zO?z&QkAxu^0x0SAST9=(YH1-iq>Fup=)2kHRa7d1(@g{QQE3pPi{y>j<$
zlN)wMZ{5|^g(HT34s}&H6CdI;T(hnq>!I(NjHP49=r;e3u)xGAr6$C@xV8x@PQVze
zFgDZ(#@>dg>8ALkP~TWaYuze6ZJ_%2pEaK?Tt^}#j-=Sqb}IPXZ8gDu2}n%pZyT7I
zJxxhD4tWVWVx0gBaQJT$IqSza9FVi&HVAoxtSn4$EFJHdlPh4&jH|0uNM@bBEmK)v
z@iwu>T^u35t%*=g;0kW1CP#l$VLbos+i$oL2IvaKwZgxOX3^O&{u%r<;RLH7_F_)m
zcrRn3cJt3Ad_a6S`w>;$&TfW7lvhr{B@X>_CU~VVvH^GqvIW)#@w!iP($EZjDYCi8
zb|_V!y6huzH7*_Y4*Y{_H85U%eSO=q+YNnvo;cNp5if<9_v2g;k=TzNN*aSGo5u9f
z9f73*P#GEBq_V`5lO_n(7B;vhsh=9c`%QvQIapYb&Y4n@DT7yr$qdE$E|Uw~O2G(A
z1K6bC^;0LFVQkT>#<&wY-kn@uT7mo2fSQ(&a&{PreIyURb
z0+li@;BjIA>YjI_BP$E6!9z6uPVx$(}(4t9)=?X*$y>R%lgKFc$u8#pUH*=stEzxZr=Cj13bwbnHkgVG-uNOolxs
zz!MD1T`Z8bdXrZHFCnx~c~5h3aaHboovS0VwYk)ZQ6v(
zG!e#0MRlA6lU%%iB)9@MZdTT&pG&CcE{n0nYl5VWFF`6rbfc$A*OGY*JOLe+nteM>
zkkUux1SPCcVQ$53!A$C;;Q6apG5(*O7RuiZfu9)LQ^lLO-1_#H7QWqX`i2+B78x2A
z@k|okxRexLi2CJl1aNK4fbPONn&Kg6XkBoowx~D7{i(EK8GXr3*15m4ddq)TF5xWOZpYmnJ
zI7;AmMb<;3$1gaAtAr+Y5U#HAnmv}3shu9pCky$5PqHaCg^96GzRu2`LQ_ki1X4D3
zH+;{?r%x9}F1~eyi$O?6>HEE2)^6)yjQok2
zl7rFwS9^Oq!XkTwnxWY>2K9j>9oMR;Kfg0IFv!B-%EaWdn4au!?(_JMLIX_Ci&ct2
zodvpSAVxL7{(-N*F-2SW1h|W!ksUg82pCpSbh3&2YV`vSoy0LPmuYEeCO^IC#QYeL
zlxMNc2Q0mQTnzA&1@+}vFFBYkE`{WOK+z(RMm
z``JnKA}1B@0YEvO@ew>R1We91KWrqrdKBge1cAhql$6B8HA7(^_tbV}sYbtf1;6BO|M>sX>H9znyu~VPO|EgHnlKdwZ#(?Ttydlz?C1_eLR(@#UM%
z-&)c_11&ALTdnYVB;m`CRFCfIKWj=@fb$K`RX2Y0^e)e0yDd`VF?A;3$sge4@cwG~
z@`Wp>Rr#SJs>#mIdy$b^8XDzGWT4q2hcYrUV$eaL3+->7K`A35DvAX=s6xDLx-G4w
zIE^}T8&yL8gKE-Mp}pyg&7y?iV>hX3Op#W=4b+j63JMA$A}x)L|Ry=3ml!Y
zB~5u?e9+n18IJFxzfibK(gKikt-yKW-)LB5_#+vy&4e6u;)IzPo2rn9zJ=T&ewv2y
z#g$(po#v7lGEYrRWax@^Urg!WS+GsoU2}CPuhur3{GbO(lzL=z
z^gNbtpvXcU59@CHpB5`u(09VasVnEgR1;QknC6ADq;8f$8xGm|Ik`JekPRR*0~
z?qYTLU8SzS`xRZ#v$nRji?~GO4`2CNnr8{+PoD-pQC9VIGE&
zeNpa@`&jlz6+HIs8(Dm#0)^tK2kgq+2|s@P08|7H0i1t_o(3GT0;Yn6a=)TE$=(@}
z1o5Lzx02+*j2jgijAw7(yH^X2#En{P282rE?uJ0KU9fNKm8{=cpJXFXRXefX}lkt4)
z(lz-T3fT)lbe+1Pg$W`gK+VV!X62*2O%Du?Zj=W*7os8~iTWeZ^ifULa|e?d9iz$l
z>{dRfbFJdlt7vUuYHI3jR2jA}(~F93czJm_IUTeTMIIE68~FS=WQ$FGeRUP%pZT8O
zJ0qK@fcMxtjl9{-%X=Kl2k-|YS5&ygXYXx2y)y1xKPX0^k=L^E{iUEAsrIbnwsN*E
zF;cy~y%R+%aG&rqnEQ0Y%MKDA1#AXGe)T*uY?^TaKCDA+?vJz)VWA>NZ$E^1jqi_f
zC(cA(EJ4Zz$_;p4ALgLfE#(0M8leV)32_fp!FX^`kl*^$3-peOj~=C-e?t=_Eh96X
zR`cp3EB%k9HN%px#lk*)lR1r-0(l2}dmqNelI9!1Dcx{~JXXA*F@;_Z(*Q%}BXSRs
z$cY~n67<5w*>M9
z&Gp}+@j0p=S!pP!culQz63ia|4b7xW-xi9++L6=fgrMs^bLNcBg^|8KSwxrOV%XDK
zR>dG{f%l1dqNk^4tVW0_)UnTM1dCsh?cA*Yt(;_}SQsW?Q
z3V=_#9XrrE0hPg-#XVg5{o9k&*nfd|VAnGnACMFs|VxvF!
zM_jAohnJ^n<*-Ih4gH*}F3HQHL*!mQ!F{*Ueka{|Bu)
z#sjLN@o#je$twZ#26=aw3xExWJ`R#@yidd%G*sq7LGiWrZj`6*Cg;)V4gkgp2@6Am
z{TiPJngp^yerf4tP_2lb!PxX5@S?JDpBL0_z#79tLd3xNf?&b$S9S_>5hfj=6u?|{
z0>a~oP9|wL0Fk6@#)rv>*>696
zIxH(IyKf)$Ig%|GrSxv3M!arx@^`22DxK9aeGeb_#>PF!DX`Z7XDT{68fpc=i5)tq
z*O!Bt1$eY^?J=WJIZy*^Bv8o(Sd;mFgK5F^FZo;JcR*J`9C`e>;Y2-8|M0LMHU-4Q
z5D=E(or(>2
zJl>VhI?=omcj2Hb)eu5_!RXZj92rPEcB6#X5PDr!c8Xwyzrk!kXAj^bxuZu}jxTolQQyb@AUu9!(;oE#`7Cmkh&X%GG$)7M8hhVX`fM)?M&
zWO#;XgNFL}DlZKV4EQ2e6U>>;JVJi@J2yy82jP~Ctn51(b+SBo;QInphpz`>Ob6i2
z52#2!eCX0JnM`v>=mY}>^LLcgOOybYQ~$>6t*U}-7ec!Wm;KMem@0%h*4E9Ig#aoj
zcOSBlVT*J};(-2^>*J>#cfJ4|vG48yHxF*gfKf2_1(Q*kG>mwInyiZi(#jg0$EIXu
zWI!o}zb%xHJG31V=e_dMHVQCrqY<}0GQ}ZJp2EBl-ZDMg@;T0>e5Mx(wI?!;}Mb
zSzcKuV+{?OCu%4db&?onR|F$dk$wd+u)#AUO
z?XNc+pLDCl;Wj?_bJ5FS>!e_8>bM6wElgPAS{>}|gQsZ>hve`k@Y6A9RPO#&!1z2i
zb|-kgA}j^Fp^^gX*QSGJ_o~$8D_8b7?QOt#VuMaj0PPUE0&KF=qdg#IEB&~p#`o4O
z1>LQ;ekWgiLRk$O3r3o}mnc3&kACPjBqxqxr0z5lQpvLLr8+t>==#(I0VIg-->+Q8
za(0HXtR^Fa@`*kPWB4Zw*|z>|98Sbt{DARjSQwRb9i1`(6Bh6`(Db8Lcd19tLm&;1
ze|}5v%fJ9=+Suq=Lq~FUnvq)-RYJpiOx?p9ui4t#Iy=8(4ES*lAj_LKTN&=Y78c-~
zfk-BvNK0}P-el=+_Rh{sD*pOL^8fUa{vT;)_LerXDL;6Pn3R!m6(cF9Kl=gd;O8c4
z_jYv!L}?xY1Fo>}8lcZ>*SftQV>v`(lZnws5$^sqv}{2*k?BI7u+mc6}J
zdN&$dRn_3AD27-K2V2`ZI|((x*mmor#6(=u7!4uZB_!VX9VA5vgeXCP_2H9C_PdMx
ziD>%P+pDMgQO_dOT>kAxR&AO~P~;?8h)!M{0(mdWW5Ag)>?NJtU8n_@FCbLG(HO;X
zV;CoJM$9lPi~-X|`eps0NXwU~9K<=QTXC0Q(C0bsBj?6$2FG=lgyxP9=90L$zCJ!E
zZvh12mw;V3CT{47hsR9HtE#}hssT#`Bq3~YkvQ1nuY(AMKmx2*^Q2w8I(hmFCH{Z_
z_AYC*Tyb%6wY9ZCkm2D2paQXc|Ng++=8)+nvR4j`dnuYypLacH3Wd_u*86+2v{vRE
zNr)vSDwyFh_vGjKQcQl2$Gzt2BBpmMm+`&L-6SZ-?7x@^r9OzS2zF0et+@J?scxOO
z)d9=d7byd01sqf*U$G~}E|c~4`V^A+jrnkP4W4>dBKt09Oa@z?Xoe4CwKP!G2=RZ0@UlLon{5*
z_239+o_m^-kbq^MX#h{B5_jcZb&-35ZrS_REg)q$ti1Tx2>Tdl=1(n=Hn??8B27f#^W+I#huuzeLiwJMw
zWf6N5p7p>>yU>jzhmP%p*$QhjIsM(#=D_ii@$DUf^L7^<{_jhp^+WVd+;$@I?Ntsddm)HE`qwh6msEJ=yors0hZ=P>?y5G!cJ~V@g|mT`J0>q~uQY
zW9bWKpjf126ms-`Hns0pD5qVoGPdM9L}69a-BwRh;CW^LbbH=($p}9q4<8qO>xJ3b
zrtmnjt=^^go-{E%3}T;O9ua?UG&~vPBL3L=^6#;9
zd@sa2bxHe!(qZou#t-*Q$xOz$4Oz3^^skvMQe7Wi?IKCp_wS1QfhHK4m3((JzaLf|
zT&K}pTUBgbesD!}axCK(Yr)T)9=j)peFC~N)J>S1QZ9RwH1QYHq^JqHdr)jYr8cO&
zN&CkA#tlRwgb>iIBI?U}>lG0O&`qleE`s5`_3Qpd)&6aC%H#h?yh+L{D-8_{a3HZk
z3J#znl@%3htE&NBcw3ZY^m&*?gaAa9Qx
z$sB?D1q)|tw`@T@JV#=G;$!FF*vU_qE@G~XT#T#`T#xcoFm@8v8de^}cP9?roLU&~8W(iF5Cs+e-qn+2)Y{lM*SNNQjT3NE;Ie%v}jcT
zKwP<^3a@>sL>&<^^$iY7ddzh9g>o|10??5NBKl;px>GquaVYHv|LAc<|6$Ts+mDY
zPtSvqJ?IVSf>hG7kb1<$iO4>rp_r+nl>+A-novz5?FdYt&IWrx?e;rRv7tcmS)SY(s5A+H2@9&!1Tk2Kn-3~ZF*Z!{>LRsPrm}P9a^6fp_6h92@+;#YUd{>Y1WB+e+UY?FR4;ZpWp<=
z$IDADLQNjyqSL2oE6Hssh<5o{6+AvU*T2ig
z43R}G_%k3zR8@-pH~7cGK7dG67lAWiiw^1nHfO~^@^AsnuTki3VqgI3pf@@O#&`TQ
zmiG2lC_wS_F-rn5G3|XLSpFaoI|8x8g5!j?q{oi|=zwSnKE?B&&+6WwwL*V~Z2GCU
zR|d>H)Z5E0^*Qv|JFJq14QQjl`+*idamTsl!}6pTFJ7RKR1*ZB=Af9^CO9M9TwE9k
zfnVeZP90}+59v0>0L)hbiFyO3ege(I)mos0Q#(q
z|5mP>R?7(pq~Y`(rW0U}Kpjs@NB4Vqx%l;KOGyrNV88;z#plo^yH->OuhP7{pXZ+j
zac1XFEus7R)fOd&;y{p$+pCyHcVN^D{t4h9py*{fB5ylg5$3DeQt!M6xQr1cp0ht{
zd$f@JL^|IQd`NsY?q-9_m!mijXQEO7NT!ljcMie`xLtq*y^5b*CUG{6
z_cK}%jJ&~$f(vrjl$!}3V?rBN-GTJD#of!Ejot<bwq4yG%%L}9Q?4~rG_Ru~1Kdr=Ry2I~t*#QMet0DE)b
zo;XhMDgj1}1xUn${4vKoCmR7nHbAyCS*3C6d+gI86Fx++&tUd
z`~zN=$@}Xl$SgqcfYt$pS0x}T>|?B61!M$P1jbyrU0Apa3|LUm5;X~qCXht9Bb$a_
z4D3wQ%s87U)zsVTfj58&p`6?!c+18|VeM$5cIyrjPf%*mGl}G0E_WG&U5uSf{tS4m*I)aWb5y)_-FaM&XO$~n|Rh~
zQG3dSoqci3%L{z^<%6bI7c1k*UEH~-qnUP%HUJ!2$96rS=AgCUbsR`&qors-LkTJl
zg>XwVJBg`itOi|s1~yMJ4e@SL{qeM!2r3w&V=
z$Wrft(@3HG{K9?*mU16}h2>}jF}V?rLymcY1vY?@;2`1vJUA4@SDA_4NMT$qt57Nwe-j{@z%5-KusV0E?7YvA$%D!inp$K`tR5s`_4Gg(?zb`FSp
zYbz~1J;h^XeGG7c8I@VJBF4VPNEbyMSs;OruXB76t!W$r0ugjb`GTdDn2kU?RgEQH
zZkGW0g-(5H_dLD8&yQ_m+)pI}u?H}VwShoNfgXr12>dj
zKnH?KyT-=h-iEC72s#$zL|m%1wmen`>A;)uN!w`?>tfx^PX?Uh;k;mDf4V!Zs*fD8g32pBqATX)h>xO_vd
z{f9z?xIYaHIS?%HUX}(+)SkI`s&%aE#9}?g=>@
z!Wf|@n5J{vFlZ+QjA9oq0jxlU;O_nhbtkY*X;8m_*Q!n{yiu&i4@FYL7xGP
zg)V#O@ZiUfQi_Vzpv(bF#3pE+n)Hf_%diaKxQobnkToUZ&K&@r808yOy6x}pE_@+Q
zadT~L4J3Zv<31e4W)#Xm2X5V}QWKY_PmhilO3#r{ZBa})
zIbouR=?ht4fO49>wsgH)zr|8!%ZVcx5p9I^%yX4c3ifL*&Hn^2upju(aw6nn*j!481|
z_>|orx#20Jppex0voSC--fpzks+iVAe}g1g5?~>)LDB&=Mcz)t*!uc3)SlVdwDx9h
zE-sRX4q@g%x`R;Z1`P%@Nxyz|S{I>SMk#{|Y()=4=E$bsFa(5g3Enm~aH00`phtZL
zDNMYkxQIxA@iguk=7~#doA~}vV?L^HZ_fjdxwQ15W(J0m>+9<>($ZE|R-b!%EY=i+
z4gl7GOJzkQlHo>^U&qb|Z#RaHip
zFN6L~6jxrlbVOD*{Rt78<)f*)_;Y@qmzOsMxW=PL{T6|OpHs4G-<}9mD!_Sv{Tg0g
zKbO`folgs00QpXY1v|(x48+uL?FVD(mbW*UwYwtT{*D&)~S
zk&(a(&b;`97z-?@Fc+#HxOW2d1JP+z>)2tJCa9G`AN;
zM*OI*G}ZFBQ%rTA8I5pu5GPO+1qZ9*y+7@g#YQOnSy&LkX36*O8{U{HIfGciD*+<)
zeQ38XU#9Jl*PRDk-S>DJHxgH>Kiga}`7BZWfl;k``hKrKa56YZ4sr3Ta}7&Shx^r)
z;x-X?T_Ql?jJCCDdC~{*0}Bwm3e2>`wp$RQIL2^`rdtT+ghvY1(lPG|jF>H^gQsJVgy_r%Fb2ZK*3$;rE8E@BYsauV8(h6WX>
zL>>Y`Nxi+%M7#pw0f^?0&Zo85YYT2Pk}CQnv<6N9?D5<&*u<(Y{=`Pfff71-&0%oVtO~KY2=|Tu%WWvspjbs)!87XqJ4m94uU(BeP~!W@GnAD
z1jLN;^srGrplt#K^sulnCs-(d{feR)Q35L#g{Mqd0@Bvb{1s5MM1&_dukU>s}77O_qztuJPkOFWgC=nimvdvghA2433aC$Q;KTlfY@dmbJhNFVr(DXkSC
z_J$l51tH4F!0s++5a5$UbSx5$2VD#Y2Ry%m{v
zFDxy^<}Fx66UCG;81%(HK*MY(Z5B}^qX{y=nn3W2Ra8~+Eb4px!woGCCYKmvpM5YMux=igng6v
zPRJdY_9c`?@e^pka!@BwWCq27b-f5611euGDXZIfEWyEzsF_E$4txJ4y?)bWeXzeD
z6kp7RiO$+shl#_)L`#dQ@ft+7ctQvdh+-g2y#{;4tG3QT8tpD>MlMcHkVDbHC@55+
z6P+EYt$p)`m*MeHiM>8t@gX?%$idskd#=-6I9oU{iy0DHlVX4;85b(b%TZ(#mwpC~
zH8v0#qgV||+w5E5Hfpz!OJd4~X)dn83DgBRae%BSl6NW4vvicQeA&ICyVtNc%h*v4iR6)hoxqZmi-En&_MQAvx6YDdd|Oi>XX^~=0GtdoU}JojzK
z=1PyiU6B9b*C{Y~i6Sug$W3(7cno)=qfEa0yTc7v6K($Yqf
z1Hl5`}QYUK*W0H?O&MWwhadprXVa}nxYZ(X*U?be#*H*1q
zUA~Mq2A~7fD7TPqP|0_oPK5}C-5JUi;$#Hx9}X&j1r$+if*%Q=`db)Cfg0r?vw_e+
zPoo)r>xUg
z;+{biD2P2^T%aWb6x`SyWK@eQ?-C$y^dvbM>QL-XSqyxN)If2%(4cqYNcj?m#kZR*eIbd#
z7zmB-@4tV4sW0qsaTXCLWxRvJ11106qumh+Q(dgwp6(tJ$8x;7BDVyxJkA>&6GRX$}1}J3<`nDieb;PHkBCLjw(?bo1bu8
zg!deA+vmv0(^e~2xhIg{sIpxF>42X`J|46Qj39Bz{XjPy9W}?qh%tN#urs;y0y*j1
zZ}pK=qoWIuixQ2t5fl;jDAw-a6AcWYv2FYM)dz?jXi)X_^=OuoaZs@Y63;NzmUzrC
z00T#Ohig-TMWgV!l-bG2a`SpJuUc%CxcBaciwkD6uK;y|fdjS&w2v>KrvQ?GMF;^b
zHVzJ(2%bO+fIdv($k2rnn}3j0`VlE{hkNWwA|hI$I7Fh`5jrq-U3I^?aljPs=$}oz
z9CUNISxF#jU}oM_AJEOfA!&|Nhroxg|HR;@KDJ4wCfNP{y&Xmhni;P69LUnU8GCJ|
z0nc2_)_%-CIX&WHF;v7yv)f1}
zxp7PkyQ?AL{sO9}i0=ycr!jr+z04#ZMc$k@8vB?|H
zsJOlmG*AGS=%|(fiQZ8YJeUAAnj12h|Mg)j*ITzXkY<1qBYW1rBEYX=eEf>D^9aZd
z2rp6UhaLJ05W7%q6GQOh6gXO-HkU9cFh{b-<(ybr1C|sPcMU!c=u}`Hy6-I^i2M1g
zSBwouiJmFogy4Y#PoSlOX;M{r`G+J~2|>Yc01IQTcEk1f_)~xUTWmXB9fN$Z`%Ghd
zmYj=^ZECk|mnFjk+0pT#lrxcw9xD}3pFTCrLz{clZ&S{PY*8gk>LRMO3+G`g3L^S4
z0y69(>KZ`()6&|}Pe62r^k(0g$M~^jH-4VwSio1HwK%1w77`jd^v0K)A!8R^D2^p@
zYz>@6pucr+FsaqbL-l94oq=usCgx?KVPV3xb+uPde~!CF@8^fnYm_?R&oF)(IJ_UT
z0eh4zki-ox^7dZ!X$hI~*crDo*n9Un(V1=nidKofq@1;;dN0FH9bcc
za3uhQcka-U1xf-^irsSxKiC6tK=KllB}z&GWo4dFt%AseX&#blbM$WLW9KnCNA7^t
zhLZ1bnuLG)vVTs{mKYubxep~R%x1=La|}=4vn9bgn+b5hfUf}A9E+N9k)coqFNi7V
zK%kPXqa$*%9k$Lojqck#jEX>1bQ(tp#`0nIX|}UWajlScpplc9dkd2az|ilDeSLjl
z;*hT+0;NP~mH%;(7>(!P`(g73Syl$36Y>ufjrB6c`(iaz(p(Vs@CO1L>ZwW0AAsZm
zkGxVeFsc`FgWzx*g||_W^%*=g9{K(lHqbI)i9nQn&@*Z`Mw){jCzVLFZ^fM#5*9|`
zQwW`dRxg*d;Eo(`Sd4+znQ;MjuWY+_gXDKMO$WN^`1l?eog6f+zHL`aFkC_P3>#PI
zYn1MCJ}x)X)g|6o6#*T(&*l|Af#c*9Xgj)X0V_M|k$R!pjn#0w
zaf4gl&DPkM=ZP$TE!h=z8VW4S6(Z1}^8~Jjc?^iBVe{`{Ay9sO9O#1o+64Q{*$sSL
zP=mlHLcMaO@u#k64**-tZ1^k65SFvDXuLAVAl?vT%k|&h(#$~V>xwBaOdeX>+i`tC
z0S|U+Sg75hI~g)}Uo!(dNPd2EP;IewejEA={;nO~4Al=0G)$nI`fP{EP(Z*6WX}iU
z-=pMJc~XR39No6&z5&el@J}TrT@Z!P)?h&baK=|oEet9=d*tD})CJVkAEGIASaFTZ7(gFj^8#RIRtnP6)2WX*rs{|qvi8k6~LNu3TEl0ry4?F+gHJ~Sd>WZ
z!tam!62wHLs-iQ;BZAEN*@bL4-P(u(z
z^wiZ-fD4f00V*g!GA|>jvNdI0)=!Mjp|MI^KMV=53jdDp@Sxj@-I$yk0z#iI$~{*Mt{FB90-jJIqi=M@Mo0y}i7^v|T`0Hki909OsBl
zOO=(6k&+?aQPMFnaYQ-eC`z%*Z@K#vQeIwG7RH0VCr?u5Xh5ZoP=zJwN{%;PkGWCZ
zr@(LoJt=M!I)n4;EJK2qXGS!pXxWJzf(
zp2}-Lt&8{KAj4f_l*LVT=Rm*CO~ql9FkKDe7><7UKup1vUWPW$HBnCudOG}8Pk;9{T>2z+BDH92dZYrbC@B%-)e@{l5
zti+G8#UkRYiwSur3#_5yT9F=ba?WBQbofo+B#0zwi*ot58ck_qp1lNoLPu$}h!zG7
zIoM7pVyIs7im`!AR9ss6hK(#>oc0-=G6OKE^#D?GbT}7{jp@~mFpfYV?eC{qk2lP-
zE*ikJ?R{q_5#v^CY{)6Tlj!ah%+HK!I3^7rrWQu)KzbQpsGReF7o54|=<>T))kV3f
zuNjReSv{vlLZPv6k%Nu|+%!<;(0ZZ|>1b;szep3R0!KHr&Oq0DB`-TwoPlF~int{rQP!DPX2<7&
zfkZXI`WgB_FtPd3R2@0e1LYh91h`ZX?;5D6G+;u&|A?_KkbzX^51dVGrHPBu!%KOB
zr8-6>LeHT)VHY!S!OVFfLFF^mYGH_qz_cEv(n-}-1sNG|_G0lgVEJ|i$GMp9)W`tD
zL+EyW;BJOlOjd`v+;ie7gT)zDVMWnXRXNQ5jR9nTW=7iaN1%hcgjdn4D=)cmkuMBT
zm@nxn(S;q&zCKKQ;OUG(45oZa1d6!UO~~XB$OFtsPhii0tr&OCZAh(@Z8k8JZt&8j
z2f+doDaf#O`l9Hm?*$&M(-|&8a|q}f%H!gZ1Wk_0c&&gdSPq_k+#cj~b4iFQ)C2>m
z!XS`=zzs)zWQ6Zh0QR@RYGay@rYhKJ7fmdzP^(Cgz|71fuu(*q?rCE^N%b%uZj&YI
z*R0<#E5?sNwfIQx!(AzM1NzxLy;)2bq0r8H6ndtk`D`dZ%>&V7sJk$9QB+h+z4jL7
zHWBXLKT;|~aeJ0v9g)Jn$v*k9rcM#{0BG_5BVS;I
zihR~3vTE|luW)o}4G9TKXwW9{qcNL$30VI^Hn6}ZOJMW2_xPFwMDKBvKPD#T&=5k4
zIXEt(IypPO7b>f|Rm2ZGFsQd3F^9)ar7vF$7pB2HM%3$0&XcRK#Bbss0~J*8z@o-?quj%F0ScSxL66Yzk4x%F0Tpkcg1IWhWtJ
zBzqN!%v6$O6J;f3lvG4UzVm+G?>*k*J)ZY@dTzJ>|2MAdyv{+?-Tt42%4M|qZ1!M>
zRGrD?a=PTXcy@o!Ov2K*Vlki#9i8^nnOb}8JYAjqMh7f1z=0YPdH7}w9+p%}t&1+!oD
z`aD{=>KI(_6C|y%F7Ax
z3QRybnV1F!2fcoM*15Fm>U*K);Q_`Nu$I%y;K?4Xx%>q$Rb7$awhWV+ZYLmMzo2Yu
z6G+XeJY19y!1wyeIw8s`e&wkir4e+4ANJk(wx43S}`2Ex?
zwCwoBxIquWg$wxhnvc&U9PLU=7lHDu#LfqS54FCB_yuU%blt5Vmhq*C-z=$7M6RZQl>_e2``sOCU%D#?{3up@?E7kZIAuO#nr!nEIHv3VPB{V#K
zL7Tsj?gzt$JhmF+KA*J;EoBE{S&SVKH3fILsjpvIxw(U2-70=bQC?nxjRsd*DUn0>
zBkQIjD^Wo4K3A>~tROHd8?HQ;$kmL#xk%*+hE<~JgdYu%<>=8rxU)QH5uL&52b8c#FD9Gf(oYY)#U9sJ6Us?HgJelLidJAL2|)r1
zN>mu2AIt;1;#Ts5m2KOj@k4PII=MuA;1=u*Raux6rHD$PI$hbZN;5L)P~zYQS45Yb87Sklkh{Z@}2B#jLLp|`V_jC2FL2ZPaIaF
zJ^(I^ioNi^pm^<%6x`W)|BS9ksA5S)O7X*C{SvRLV5(?ZYRW&NvHW@Vv3;k*svSny
z{BQCfcZe0Jel_8sVJIR*rK!-b_Mly~s_S;fU+%S6fB&=?(>@W~ROCv0*L?lQyTiC_
zJ`9YDd2H;ea|&&FE@TQ8-52j7t4JL?{!?aXdf2M!$mMm0$|+^{Wq{5YRBzK1SOG6O-4-p>Re3OoPvl
zrKu?cQ&V#L!%ei(FX-o_UpGW@VDCIWWRZImky;Rj*orNS4(?Xhxy6DC#uB3nKtibL
z{a|L&uvdI5^dn8>;*yQW9x!6-Kugfq-;UDDqPvv8FI3R`@$=s4{D$i+=bC1+$Y`_-
zKL2pyQ1NbB4
zFHBuiKh0!_2@6K-3aKl+fPZ$}f+HcX&(LstgT);#pVlXoILi=NFkRahIsg|BcsjW6FQ}3-n@q;
z3pkYIQFj0q!9&^DewTfEpuhP;Ey0uRQD!Dl2Fv!sDUJtC)PkG}3J;R9^HzptE34x_mD@Im99OSH?U$v-|W
zh*v_g7DeR4>|-=E8ak5lywhiG?gdRYw?JPDr`PWXEPi|;+p`m6(-XWy9Q-Hzw&m0y
z3EhdBCkdjX+DbyP&u4egjKn7-M1C})XWM0@JKnxLc@21MS=1pQ9++#4JkO4Y$YhFy
z#O5>z{@q1SA0F;E$D)DG$_hiPpG)^e!$c0%C);00F!@prfZ3uso#(NDJ@1
zbZr*sl6s$o<_~I_!_2YE%N{9w5Ef6XQ(6aq(jOUsatn$BShkNbW&hcaPwp-zK)CPU
z&$%k|SR>PdIPPUAb+?Mu%O}L-Y|ACV4q6zWqkeL+x0n3SHiptwQ{ml{309qy@LsXk
zg{l{I5C8)4_j~`gPXOVc0s@1lS?epJc+54_qs6T40PlEtNvqMLRcfFGV+|}m3JQvi
z=cnH*2CsHHKEjuT0X{z5V7OmQ{ZXpf0_{3yC=cikyk<1eA321`Tdm
z71*<}ukDwLv~zkg(@Nak#gW6i>*Ytk^_6{6x82VdR8)EkS(O``JT~X3eyX`SX%T;1
zKgBggxV&h6X3TrI-Ru{?u(#komz&CTQFdz6FLCWhvmX|=MS`AU>OFh0UO
z4vDSysef6*4h}mK4u%e?`ziJ;oQbe=Q&!rHXK%0D5;Iv2fq(qNQe8crkP;6ECj2pm
zgANVCbE)>Uh6y}lD$Wsq_G~9M1ijhWq_9z&atNu~^K(zxf^dDn+C-B^2YLBebAFC#
zz0+5peQW>TUz{_*49Q~C#fY=;=1tw&%0XA@<)&F-6WXkD7Ror}B5Q32POFEgk6vA;
zl)AuLaE-^9G-P>QZ9wIOo?ehr5Z9}siIO)q#xMI!SM3dcQBl!ZJr#1ktaxq1OOuDM
zMo`FTlR`@J%UO%(HPYUm9CB_S6T&E3%xi@Uo;v;;Sv(%kesw)j<+DX(zFLwGaExB=
zImgT(mzpxYNU0%pL)yh3Zyg6ul96stvZ^1_8U978+nl3bn$J`FBSlZ(0CRuZMIqf{
zUfAGG
zD-ES4r83Ok-`{0NV?u8E!R5@u;D?WryBCj$UnZHj8Z5d0oq;$l&{WL>^JBH8$MZw7
zk}_S$AXd01?P3I|5%jBR(vs@{6|LP(8LURM%72dW>hQd$7pHj^@b{6qh)s=iAAm)O
zKR|5Y8EeL@5#viVkLZBVyx^v5hDo;?J(w_FTOb*J`u!JRBZ24PAqN;CTf%`ns~fXA
zQw;3i)1qv~sJjaNF+3JM>p$YEf*V${_Ge5M!H58W7&go9X**}_byjTFyXF3E*0!(>
zz-yQy0A>Mw8=ZiturLDb;OYbf9ef9rYEj|0g<^p~2nY(oR~rB)!I~F^IC$6iNpSc|
z1c67KK&HUw0H{Uzg@3oaP8eA{0DH~xH*Y9%7E
z;Kb2E`3Wuyn89#D1B*cf2LHVKVly(%0x5#80q7njS-CLY1nB{04NwDtTZF4u;l3tl
zIw+H(84jX@bd)@cKM67xQoU^PJY8INizM@P;yFRe40m4`$1*>6fOP`~!tg@-PDiqI
z1jP=
z-=%$BBHl+LcsP&xtDo>0P^>G@&sSz#zQf3%lXCZP5Jy`2N}B1d<_9If-fswZ_3j&K
z^nCnb!SKA}%KfPr$dK=YGSJK$DX?*K|Sjiig!g8vO
z7e*q|8roL*_QJ(xo*~2Z;Z(fm1Kbkg_eyW{y!yLeL(#~1Xr9R>G5_?K_hwL9|8`wG&+%~cQ+Vk{j!xH=8
z&NueU76zdn*F33S7``gsmlC}%wRU`IEw6-nW+%wxBdbY4R_a3WCsq?)q1bE{2YZLV
zXAhs5ahm`$=4QI63$~Me4Bw|O`XL5#zP&skshhZ5v40W&;P+Rpf`U{_O9%4#0{&nL
zHa;~k!=cQH`k}%+W1UK6Yv)3xtnMqP*5f)IPo+`bZ?Rhc!otE>%?BOr`8L+B0kEr=
zBFOl6pKf6J>{DGE;y<-%()Es8o>_z;?|7cT{7=5YZ(XeZx0yS?U$R-YIYP{=b;z(r
zz4)f`mB!lg@9U?PO$LVv=v2MwMr~D4zCZTa}8~WrG82}%dk@GZkb;}=Ly&?}MxRnk!8Sr3+D=*JZO5VUFEg(QHIS&Sn
zE$;u-<92QGS1~Qbm_#1JU+AsdW@*~1aGQb}1&%J%7ZFiWSLN=)g%5n`XdcRXD^Zp>
zaErD8RF9@{TEYQBs!am1Y%5xuTZnOu|w_i^Y?Gddd~vr
z4xF6eL0AhnDoM!(VCI1LFj3;oKMpGl{6LInG=*?btG#j0mz=nE_GZq9rSe1dMnHepDL`w{n
zaCrB|XaaQuIbyOUXor9?5#k#_HIu=cM^QJpdJA-_iN^WQpV6$&YqR&#Lr{w6zh5Pg
zQ!NPq#a%2*IA>cwa@?Jr)DjV;adDGh=uJY`?ngB%GBUrI<3!$HB2PGIE_KYy*tOR9
zjiV$(3|rq-tBMNK&b=Z>_c6r%`=^s@U}Ba}`CM9N$cLSu(&TK$@%(9sSX6V)?WK)X
zFDWsprMjKW@Z9mAHO;?aXUgK6d^!)!Z&uY@dBCTogDCL5CxlejLs@_12^qmScPpsm
zGnI+HOl*$cnbZd3LtSs8H;Vnv?v09%_A2SO*u1@{>%i=-*^uS!}V^H_}eiD>s#T$rv7UP$<&a
z6a$o_S-&S4j7y9q=i?ZpCI0&NA1xFvkJPa?J#I*OQbhgzx1HO@LiC}Uq+6^(W+s4K
zXf#GSm*QC1Z?f)TXdivZulhKm#FZs+Ew6pwE}i7GG-fS{7Y9f?cAk}K@49}ycxZA@
z>BU>WI0Owyts;
z*C(eD4ZM9n?Owt(B&Ujr9n^GmktctED+F#9UNJbt&h%b8-OH=1O0ZA|RSef5tbG$^
zAfh-<{}~JzoGFq1q1q_~D}t|YGX_8whc&6(#}qyngq2+A;le-_{_}v|@lRkhaX(K8
zJ61`8sN*LDe
z4up_1BLl{s!6ML_w(>t-(QpQ@4Z|
zV_=@Avi>k?!LuHE#tQrk^c$qg48MLYgv$)I8WKu#4SLIZ
z1_x{Ymq`LM6}V)ezQGlU|Mhyq33d(&D1tJ8fGnwSzggedfUkn~wDzP2(N5s6&%4cE
zV%h;_LoLwX1cp_FsGVO^)XVG#yt`4lHICUm;JB
z_-E?AW4(Uf=4QhehSv;aHWBfY3=uPOk#-!lnS*Bz=LnH>)Kb{dZiOG9)UY(KJLPMg
zD15g~mEO4A<&H3|TGEEGOpCCB#O^UoTW@RC^(pGVJA2OsREFfTk&G^UItj-i3)7aPE<4{dsf9!PrA)jkaAK$1uUo0%CIHj!zN>)sKd_=f1a9kcr
zs2@OM_{g*PFN>EY4%4iU#b*eH#B@=h3TME*dam!Fhf@C6NKk-rg@Aj=(=C_Hi3w3$
z*DP-M)1v9v9^T)_Ok^)B(=uy;#{^jxwh(YsNL+Hcq!Fsz$N*MW6{>5qH|?WuV>VC6
z_U$>h`eb?}Oq4dhuE;;hO=!qms8(n0UvtSC2@`Q2G98deV>Ne|3FnNAq;P
z4Dc;(-nTo?7Mp}>SlslPLid%L3wC#m^VQDXjji>ZeJ0RDL_#AzwN3m@nold2?bUI^
zyoUn1xmGDzJ5Ot@%3YU^2$OFq!ufmp$^MP!*>ySWZS&e$0$0^E7*91l(4V3%&!S|$
z*GqMBc_6=Qu6bAT&zfs3HuAeCBBZUoX8j`6&QEBwnOnaho_bkc`uIU-ewv-vHOZ#r
z;8tM^?mg~p6h~Wx$K8m>oM!^hkX!)Jwk1a5dr+KyV@yFwfTXv(;PfOl)z0v&dIM6C
z4-c5k|1!yB=qoE0wTpkOJif~-aK54JbmH1@#Dvd$Xkaz(#J`?g{VO-c4ZLC=x2v-X
z)*nAp+9ytXb~mj!`i}(?@e`55m({ckDewOfWOh?~o%rSrz@`i0gQOvi($4d4s^nW2
z)w%>x6s%}i806lU<`Zw#%u9@lx1y}fD!NlaQTQ@jK=1%5=g5ts3&g!jW&JAix`KX{
z(xVk_EUOLOecU@Gw-$9`{XWa|-{rDHLX%skKxuz@NONTKPj?A?hkyLN=Xv6*@?Wv%
zxbb^+M&7xWntp^h^K)nSp1c*rhFnThubF(eSuj
zbML5Od0!55#CFRwX+yeqy-q5kY{v10h+AQ!&ICevkk4=NJwjnF|g-Pq_k`gE9%3J;;jvTf`zmOuz
zo_G4cl|~~@q((0gNp>fbUTp2eYqvKJ;@6yJIbR3SkA&%$31a&({kp-eJySUMx#Xf5S=oz^XE;<(`F
z`?X1nyA-FND@5|X=?`9`VY_kW<+Fz`{{K5pTx5G2@!x0BhPY1oNwohQ30|WO>L3cr
zVVP>^EcLE!B;V>$6)VWnL#_7pSWv+e
ziwC){*vC{zd;j|RZ|vj$mYtuGWhND(K_p&~x-diXT=8Q&&F$ZHF=ppU-LG3^%Cplv
zei>}GE7ET8WGj<3@0ZEe4UVxiFUyf58eK;2pPV*FR8#YKkIeOwiPLNfHqiLJlQtq+
zh>~1QaOyHDIQ({qzs#n-v9q#>^%fn6uqrEpy*eEkNe$iKI73u*o!cpyct>42YIVrc
zrZRZKZ^U1_CC&~e4c_T?S51|+S$YlJsTx`iZ|$fI%}V*jKjCq&?7G15x4~>SZ}pnX
z!57jlEq#mqU(817nZg&FCzpVrgU?^T${KTN8Dgvi;y=na
z*i+Vtn|Bv;7Xg7K#3y~oeG|NgW95_pOWfZ8Wi@&bkRdn=Bsju6dbI0!)XY<(;J7(Yff$M1mYBJ{r$Lj`u37XQ1iic)*iOC!F8zcV|PJHFl0XkQ0{?@)4{4Cu9
z7=8{~zGK#E^8-i2N)!wkbf8WZel+a{%-$XzAd53Hb=BS2VI<``!ZoC>U@I7Yobn
zJ9pL}`bg5Wdu+X|_<8(-n8+4bir_EFX@nNgJ<%Z#yMl4q=g*F$YMlG`N73#xwX~$B
z!Ql2}K|d0R;QWoX;Nwkx+>rU2kZAgc%5y4kcgD~Y>X60DLmncq>%EztZU!6?V|kEx
z(GqaLGOr{{;ilq$u&Q-~m)+x0STP2n=5QYfSzDJc
zN_K#CMnZzt1U~_?aWqvniNel?e2lc*%Ag>OcX!9Xv$D(W{mEm!B~bu{K&q{of=xwqE8-j@y=Y
zJ^f5#hTqvCC_N@9c%t&ye>ehU>i9w6X^FuyC?YHmBY6#o;s{_&vNoav03F-P1Lat-
zNWqsyGlKpJl|3Y*_!^#`zd+JQeik`L$VbSsXRxb=r;o
zx=&<2B^7^wbX@r>o3Ot$sneDgxCy3v@M}P*U`YEkjMfDO1fbRb4H8NdbMqyhBYB)x2;JHC#fmFLiPqwN&Udm5s1E;59
z@sGh@n^mqX6$aGMGXLl=h=cj5sp&8x1;Ga{#tYN9j3o#n{VHR-6C|8k0B51kxBfzY
z=_45S=%}!pp#08CPiM{Q9-f_U$k_7WguN_UPcC!fkR6E5Ai(N?#7!U-&dA^*SOKd7
z#}7DPw!+?yE4^n2;V5Nd0<0TR7)VEHY?l%r|Bg>y?9C<7Ng+^pOG^5Wkc7R5Y7B%?
z{A-X1!4YB+O&0V@uC*JrJ*{$H2cl(;tZBJ(iW@^w2u?b@LMTIFg|<85xvWxn)dDhX
zmGDbpL8E=P7EKSzVHo_Ol4AyR3oWe=62#JUhtRWO`y3SwU?RP;^Y2C|4<^qO&X@S~
z$iX!Pn;c9sOtisDJu1x(^A%uApsoJ5Dff$4;{7A@6$JXLfU__MW7f)sY*|ACcgEI@29Ri0}DT@o&^${Ta}=<}g}gFh=c
zD_*@hYB;_8@%tS%)G{;%Hw*UT$Ka+9UBEHF6?i?O&A!Pg{|mHpbkfXPND33hNCW(A
z7*E5FS3*h(Fd!j_6yZcISodb0rw{6+@F1M#Oywlc4Wh^(@mgo0b!smrfOR{)&rv8<
zh7dr%g~6CM+qsRexONMS{=y9#7(_#CIl&S2o{}Q|u(ERVg9oH6%8_=+R(%80k;Fu_
z`A|;*!XXIwTUvIYNV%`%A35{XqI~H?-4Ey_@Sx#G3L_KUsa-#&d6JLBA;}~yQ0<1b-=DyhulM&#
zvWF5<2cbrQn+fu>u+tx^=Ys0Ojc44{v<={MikSmvIh0JGyad6q?O4#i|6-T7mPu~B
zc5f7HJ2^Zy<_C@~2EQQkVQXNX1o^>F99kd*UfKL}xd9VrFoY#RYJzqg>LtKHkn}<5
z*g!Nm48#Y+8MuQ@V`t$F<1$-YUA;U~F%H5Cyw~VXS;C*A?{_r0>A}=an2-Zc&&0W3
znM>F46EGGur~pywT!&l%UE7}HrQFqV#ahrq{N9fbb$jSqh0|PNLx~ie2&@_WAlz6W
z7=!#h38WPSS6n~>sqzuY3Z#;elHSCi3#tm>IFRX7UhwWXL<-)x*r6l-rUtKD
z#nl(jkKL8cV%j|%dNydKB(&vsZSYsX+g?6CJ5l9xwI=Q!%%MEsascLi`|cg=U|*LS
zO@u~dl@-e84;V=`UwMj1@KP}|8-dlsJq7QQ
zr%&0o2B288H7#@{u7sM=;6QSX_M-kHc)P^$YD8-DrGQy?^g2=CSE3jfA
z{3GOK6A~K{w_;}wvE6G&st7z5B(h)$pN1&U_ZU1%b*bs19wRJNj&eo*o|NHO|S9Ac2?-4E#`f7xY^=!@wpqH8)>}!Z1{28Iw*yy*u;_
z41}!$!9NmzjmvGC6y)Td0vY|=1UpP{%6`nwLVb5Ca}`F;(67SRmTeY+5CUWneaf2a~iOK_!SG41-omkj^O-p?C;L*);CrbK@KQV#0^
zsEsc|D2_Q#LkD9DU&!~x3{q@pkmTp`@(##-=q1xm!QR7t{)cecGt|0``$7H)ZU?w|
z6G*%ee}$3@d!JzN3b@bK+QZWmNj(Ho5b|+lNxl6wXn~5{nc^8y%saBa?5oTIbY`Bbi!l>oNsIrI_`*>r_k|c
zb(1I13QS_MfH($bEkmxv5m>{+_7fVrh)zemenOyDr_??Jf`K;fEcoiL;bSji9YnRm
zY;PEDGEg(ZvIQ7xjux&|NRx1e??9CYa1=gD{j1y$N3=aWJa7qMvf{hF;f?%kyfeIV
zwRq;
z!JC^zIRck4bmEgz}TtB@@r*qz@KJTckU?N&X+LOzE
zzC&|mLD7*3CO}0uD8IG3R~)fM0bgtkBcq{D*}XBl=l(;m+HqxLk{~BXXyjU
zMe^z~M#bS*pD)U+>Atj;#BC^jHqEG*l%YyXTk`dR9e<+a(4Mb}P;?*{2&~QlxISq&
z8|$JBdimmost!%NDlOzexb{yn^nSjHGe-P_<%1`YRP8PJ*3&Iunf81g861QIWxs;J
z)Rs^zvL0bUaO)QHm;@g`e?+&_=H}+Pryjf@R_4m}$P&}r;~c@&@Edm^Za?^>LUl`0
zOZ5IGl3XC#I(v2$Y;w=5SKFAGASwg%2a|dbX9)#ww^=8QT)kU^i3uA6$^-0IK(AJ^
zw<@w&2(k7s(+;_bn-$kM^sZP>0LF2Y>cKF8oSbFvUZ^_u7GdUx=^2^>_z!S^!wUIW
zrpDe|>@W(8it4}#1AleFIH~JOV2W)!Q=lY4wkbI8i5S*
z>-XJxiZA7o{HK&m>_#;Y(f
z4`XkH-zDA_c(h=Rq0t1(8DlNBn@}AzzyTG147V~E1yCsuDrzS4xxi{*Y6?MEfj>WK
zdGwqDxsmWWp`q}!%RPx+7^SjvPPEeT3C=(M;>*_A_7j(h!5`56}7InKg
zXtfjOtR5~T?X~-&M%X5VE8#@jZ5PnYS>Y^bmS=&p~j#B}{BD4+o)$j}XBFWEy>*{ADQ
zeBM@2fO1#k8LzaDf
zD4Rf*kxuRZewbOy?w7^?V?QL<8V*#hzJq(ig@?El3gQw7$%y&V>a#lHrNpt2{XHHYYu9~!r{o|fHI+TTKdT!#A0S<;
z61>#2vi3#V;lW|z-`X*igeggSSA@ZS*kKZ@_+6q61a7lyz8tehn81Nz0F8k3ny=y~
zMS9b-XX~-&Af1Mc1(~ArX(5JO;j9^!%>6Hr
zxqbUsFNM*nC~N2v3=m*QaOCh|LPxs)+U&xD0&=p!J3^uz%3KxyKbO#;*jLM8wudl_
zhr)7l@7vncMAH*FWJ=^uo%W*ne(TT~94HXr!5Ji1>-Atk*2dm@bBqWAy}}8QYHEEs
zl04y>ZU9qS9p%C)izX8oa4vrOlmhqfUWwfg9->l)+ojSEz66#Allq2+hlPdcbVn@H
zs;jR60TGJT)zK+7i@#X4r%uFx8S_LB*CCUExrOsn)3~RwzHk;xBaa~IjTdp3zE~J@
zq+N|)a$r#w&wedKxOs3Tre|kc16^FJfs;p6dxr9j2KC%^w1jqA4s5TS#Eto4X5{i@
znQD@&agp_T&Om&J>w+K;OpLK8k7RNmUw!Q;iR4K9pFl)GARRjya%)hsaC6TT$`qBO
zY`8pDBaiq(Ot=Fmm-EoP6%Dy^bKh6l*?J7IW=a=y%y1-FoM*5JY>P8QgjZ2n8OmvR
z{9K1l)T>PMfSouS+WeKyGb1M%IL<#FC8&tIW^p1{n$1M=XoFVGh}S_nvazh-B}
zbDjok#ZQitZvQ&fmq~)$_}hC4W#xb1u{D?}4e5)L<;qsog4gr
z?0%1rmlrl^@Z&5DbU~^?g8KN6=b+t`J`Q`FKOVolc-lUH@tq3WG1$}K7DXf;*y~{w
zjbULVUXy!Nwmd-K-Ee4?xB=l0&Q9390hL|gqqIjgg~})A;JvH?8KGFj+yo&H1}!`k
zpbvT2eiFPi^u88B5c~frQJ9V7r|tps3_BOw3zP$b1)Zu1$|U?3D{R-<+9^_hoUix;
zuVe_9tYZe5E3wtXJKaUy)K8}nm}TKYX@j3
z0k~*zUWYaJJ4-`xj#&B&ys5gW+-x{x5Zz_}G8PJMQE2{QDn&oIpNR<}eH?J8*#e3r
z0XG%2%&&KLZ{NLjOBzFMrxL-Od)SIVX@F%stFH$?@h!Ggff0Tc)aWQ8iQ5n8ASm}5
zkOx#22!9o~flHelEL)29pzv#fbxNA9OyHI;HYwZ)QCO&|A6m}5D4--x$Y;7d@R0VV
zvA%vckO=sgnEnWG5NE?r%vSP$qRp506_*0c2;E_M6D(uEJp1+Q8lC~}p8hY?H0>}!
z-GDYOSBNTAjfcKBm6-08_=hxIem>hR;34o8ne~*POGKiL=gccw>k{YVER2kj-V4U>
ztpS6hP699m<_G0z(p_z6vFz>a`hZITXTuJ`DF^Wvc~~{RBV6P-!V~xGZ&d948jc;)1MB#@RLtL!Wa&)^3!qun$XF!`SayI4eU7FHRV$FJJ@t(?^-Qk8T>l^Zdg#10Ea;GDfftE}K|_4+Hyc!2
zXvwybWKIww!%YfJqv~PL`T$_cfKdvhE~i-Z9+yu_U0Cp$7PKa&ySCBrutk=NM?`~q
z?o~slOfVZ>EoTSsQxayXfB*iWVrf7N1`-#X`tiN3-SckwSwdpIYl~I+Y^A*R;f48m
z1el(D)^5l41nDHT7W-LQWpQ-j+mWgPT7?ZKPz9F~lrH$&Xy3G<+Q6p718Vwc;Z58E
ze>YfWAwC%*`sWW#D@!upsh-S7egq-|qvPWIFU~Vg$g(<*$K`u)Vkb+*=Ao)YOeg{&
zC~0U2_8G9T9YWR#eH=<8bZZTu!)-7xeE;4Ar;OeYq3JJx@nP@nQJ{>?Lx>__b;H06
zb$6NdvnP051||R+@cnXy*$%P9A&r)UjSZssL$RO*6VM)lH`QaL!2(~zk5>Jz(sJho
zkzYnD{hU+E2s}M`0bT?A3OW@B3f*wIuFv0KRY{ziF`Ay}qsD2qWOpA^qX+8~_ERt>
z0pp*vpwKOwC;ikN@wTq6i;%nlQ}yB}4W*}jdaCt0@xVQ8wDgunM!&#r81H&Dmy!SB
zuBiJ7Z&=J_emlM{1WBN1`X*SzP0bOPOYyAm;V{~PU{1-O3GBOw<{WjdQC6>ZK2+%7d*6$O~W-%p2yc&aPB0p94(?M+9&KfMNIE>CrSY
z9M$*Z4iAW0u|Hur2&ntTi_fUvY;|0%Y;)?2F5n1--~d_jQ3Iu$j{Es
zhI|c2C9feFX0rkUs9L*GdE?fgq^x@U7?hn2I32tL!~~EKI3wmxz#A~cs6n5Mbq23O
zLKlE9Hxb*aAwX99q#EtS6^TTST*mO-&=})xhkMkm^z?HuV-;flgD6luMwDt(&csF|
zsGNg$rWBMTQli35kc=V!3d;fM(M69yXyjzz$360BFj^RyIB>-XCqqKfbb
z*qmbJ`2tGDnF-Jv=Zn7VU$xX5WD2}+P7V0`3+oH|Rv1QlEe<36*axo`6B!B?jTAyJ
zf#OD<^Ay@Xb@fo_$b5X({`?u96^Q?~(eRjQpYb}eqtDg4nt&yayE1i#
znrIRvaOql$alj(@yrHJ4Zk6EWk9LjqqF=N48s
z2t>{WX;|YZ1VW-9;KTaS($s8yhO{GGfiV6gsuPXPLoL<3{wWTt1RDTS8E{2$5Z&+j
z3Q28xjro7{i)M2XLS#1HI!-q*UjZ}^fayCT=iTb0W_j}p=MQw>pxfS|pEL!BhhkCmz=CrWDa&qd
zONd=)yXJ@xl{_K+G)#XGEoUq|j?xG}6+T69Z*ew!#H+eJMNheeBB&RYFnXT<4C&7F
zgh$UjMM?_-LOkR!&N
zHJ}vDz@*Xp+BIdsgdmwMTxNZcjJ_N}b^uUk@XEd8JvriEp<6@^h+D$o^zZJ-TfbF0h;N
z8EI4Y+*omeJ3BBPg5qPcMMYW~?kFgLj>7$0RLORM4Mh}IJIef5DAeE;AqGjRqM|>XmU6Vnk=?Z4>8e!ar{00QKB9Q$V}g`d>
zEu0!;fcj%>4~i!KAyU-*PVGb_K}b$OC=Uj8fHlnlK;PflqBy#VSshE*Ne$C
z42LlJj-BzvIXYZn-DO`*a4LgXr)?2uBNp;tSH;9#8_XjxZbtJSmza1Cl{C(5xQfAl
zbS_$xE=mdm5WHc`M<6C3;u7HJha%e(;2WlV?uFNl#F!Ih^;%%1J2b8ilW1_civLbGEa+qDCSG3qBbzy|*EN`3L^YT6c9nLv;p&D3~*3|7X(^8ib(=u}j
zn8YiU*sh~rL)axDN*&|Pw(xpT-%(Ux?nFTKUZFtv^2MI*p=s$A&K=ZFaJd8cY)gw&
zGFV4_7#SJyAo|Bml+}G(n~zy5#D7TTW0KcdtAjmwnwYOar;W>URrmQ8N){E5DUiF+
zsAKjhBqS6a8ym_28QgOUp;#(P$~#gQaNT_b&Q46R9vvNx8}85I2n=AUKeFeH0TahC
z5++g54=_GJMDyqQ`M(?#Tw~L_CIAr7Qc@!C`Ev8dH~k#m<5UX%u%^xvx`d#Dg9xy|
zdV(YxR$5Ufm2o24=s&ttbH_jk=Ji-zfPiMNlOdtfuKd3XofF4
z_Ts|yw4I9!G#-h0bATohyNsz}JUj`5J7py$e*qf?D|a2i4!Edr4mZI{9!nPq1~V8j
zz)Ts|5s?n!-#vew+FT9O1kkA`a1nnD(rZ;71U6yVaO~PQHb<3REU1wX42RQ=;Nyv@
z%yM@zHM15VeSB%^XE9cyh-H6Gu&9%Lyfl|1q=4EP4*&y@6^w*OypQMHV0wS)4|XM7
zU06iSdtZ06u7>&*Z>H)N&?kRbC#Pu4KKc<(fvp`Kf|wABv72vSb_s&rG!7j$pwW=5
z%yS0Pvx%hX0!(ZDmbrz15WGazwn}P(p-n4K2@c4itOcwlbYm-!4qJc);b_Z}j!pY{u2zke-#tD!F4rALr
z6j#rWuaJ9RF0467wUz$F6t4rj8T6JrxmxGX^Jj~-O-v{Y3C#f>s);%l%JW~;c1I%E
zE^rh9zq{?|c{evMZtfNwQBXGCF+*4JZJ;pm*|W`3C)JRp64@j4`>6^2DrW}>}!)l9x+xG+
zonu_?C}_}wVUSx@m1HA>_6vZ{=I7MYXf6SbE3+|cJ5;$237dz(f9%kq;>JeKZ%4=j
z=4{2;0+d^xJYiuLa_`vv7=K&K5EneA^P!W6!rFps^r{r#$_+d|mPZ>u`x(ldloSkD
zS(R&9WoAHU`h_M5dLMMSm89?482!he`7NRw-B3P^{nF;$JmbnG28cJ7s=U&H{)$T
zktcO|S*HbB9eJ6@BxR7gY**q@5S%IV8alyrgEHs`6lRupcoU<{}uJ
z-k%nb=V)yuoUlNSaFGyB-6fNS)>AOqmOxVm^I(J@eV4f=6r!q{!nd4y+$Jgl)i&>u
z2n0I9uO4(H^j0mKOV7T*zT>AP=JlwNvXyW6VJeEZ3p5_wLJW=#;pfNAtpZ<-q+*+x
z&Kmbgu3r=(dpodi(AYXkjxU7pc0l>Nu7o!lI%NUpAWU%H|-7
zc6nf_y4xNmKqV$1%sJZGF>r7Yghd2a3VKKViHu)=h_iRM^>{q8eLR}GAvC0nF`qFbZ55~a^nTHgc<0EdEc+AUmf^ucD6xsvgH?|b=0as<
zHFb4SB?MUrKsr?D7*MJ1(itBiwmrU%c|5qT757Nmt8g8H-t+vA7$hX#3*cTI-`qRc
zoiXgWy=70~2cib(Z!)kn24c~!WPt^Uin2js_R`TT9?Cg4&JG9>y?vZLx-UE)#Y$p}
zl$VpciIejod_Qdf$2EKjC7DLgf}yFFVij%!DFDw>QcRJr5X^(d1L*+`XP5@-u;zCi
zV$y6yfr#hNaZS?vFFKE@3(-8NMD(%bV3_FMvQ6$++|Wd|Esl4BrKiqU?iNLWbm9A0
z(3GW`wGP23Oj*MgN*_FM1QcAAe2@~et?rqh<|4zOoXrOa%O$`o595IxD%Xm#vX51E
zB%gVMufy8Vz{F&BYKmw#W2Ss`!->&`mghojm|H%3CWtJ=3m3xjLzarUNRx5c!?lrh
z?_TSt9>g0|%@*$D;O&O;v^u|al*B__D(J283w>uh7#(&3UmG(iv=_Wf34#2;Mx>={
zfKJ!^F)TL)&4(+RVu6cJ_LJf7qZcQFJvmCft&To(Ez{T3B0eqrFqR
zBh7YImE>2CAQ26*6*pO!T)?_VP|yxm5=NgIh8YwJEg)tgiVC4TIC4;h5uC!|fJDrD!Kj$I@xE=;_r822yGUW)4kjP~
zfF1X$1M1q>P(Me}uuGy=$43Ccoow)?4wzHjJe|V#F$~_WDsb#eo=CB!Mk*jrSgp%I
zBu_MclL1AUaAK8!b#M~lY#sK1PSLE%M2%6BR1G6RYye|R%f;E*A?W7)wAEc(-*MTkdi>?v{BUI0L9@2^6M68gYN1UoA)W$Y@MNWC8wZ3>AHk#o4kVsUIXZI
zQGQ%S8E8N*=>;A=I@ZIIhRg+7pGP6(6%uj=CkQhabnop3}gv#L+Y
zE5gvjRXP)8>)UL_5x9tA4&x^)H7%!%={sI83S9+${sOn~V=FeGLST+WurI7+@=gVS
z2_7yFy>x{X*{G2S3q-+tPD#Lk0apUbv}2(B9Ijm-CfZ=|hzb2Tg;D?`wQ+n|+{bgv
zr!(}(KV1Rn4P{EOeoTB-yT-_(%WixY@t?Sfw-a$00P5J`db}W?*92eTu&kD
zH{7jg)tY
z;5yxv=U(}2kr3nF+2ipE-w4GP>((1vaRR5|?Advoj7v)uSxkF2lAiFj+Hv*IEjl%Y
zT_5_iuMqjuj~)>Q3DE3KH0B9j2KSP5ml~|Y4xVkxyi3zf5%&@)BQbGtVR5x{mn?Hq
z1Sy|`gWvNt?45dkAI#`f>nc@M%Su0jIHw6P>{1>c(2At{sEL8>ArNa}{630)PMspBIDiPP6h|N;
z)JH)LGO@Jmn)ZNU8S8(J^NklrMA>BMq7E`sAq$saq`6vGRTUg(bQHBN2x7)?LJZ}}
zOwC{qB$GfiWSiFT)cepGImh9jzka!5wIYu%TYB%q0^Nex=x79&O~8s-Rkaluf!gR=
z^C3wKs+Ajt@cV^5NSH-A9W}N8@)YrFqMUaR3nfU&PiSfNy)sd_8}jD^o~oK*&4UNw
zr&38UorTmugr)1$O3T~3@@IS~qBMzWBI!>3V2doc
zcm+r>?x1(~l%Am@2F;XEr`>fWm*!qEv9Y~@$^+I%>9{|lqVUg^5X+7eQ3zvFJbxUD
zeFsvJ&6qF)%p>$eFoBi^T8_dL6ZsM>T)-@?uB!t^O-D(Ib65O`7P#AROV~j}LAZ~z
z#WMtA{@mDi%$LAQLsw>GV6etcBBVBeSru5HIv-
z$CXU-XdM9HfK&mYAdL(u1CaHjz{RyrV3C3W_=TS%M{ohq9iT4Ix`FU|&-e3JQNs_}
z7bq2A543mr1E$G*qv+DWr5H!E10c3aZ-s8Ztx{we1)fAeQ*?k
z2d5858JLbBC}bCF!K}u=%XYqrk#RR7-)`=c6@@1r06Vz$x%~sZ1lS`F`PVOBLjO?w
zY64ty)J>(`E=1-`CzT%4UAtgOxh&YPsW&i#*0iv&@aBzflcrBfT~z+2OVXADuoe(R!D}ICHCkGNP$Z*90{Zg{+$1Lj|K68?
zrm4rSLz4dI6A!*O^=fTr~U4)+xP45jYmGFzdU
z)Bi}CZTTH>?2$BGG-nBRcVKG)FW=Oui)QAPOfomBlSB^(Qx*W{_x1NjcILI$NxYbc
zt^WA~XPWE3r8IVFKt=QyZS(5ZiB=cw#A8i4L`~G5nIy|MwDq@D{#1~Zb(U^^?G`{k
zpi1d~P|~>7kc_$Zlxt}qow1d9xqnW1x#^M@e75
z+c{L&G`Y!7|EXx~-*5kbfR*kWKm3>Kzb0J$!7h)6y<_vGGrP)8HjxE|>``z?0iR%~
z`3$DQ=0`K{+zD=i71hilY`sv~V$mR58eE(6$UhXiR$5TN|DyGTYd_=ypJ!&65H$Vw
z*ZCZ({t$Id_9HEnQpa8#3`te&osHH!z($Ll95@M;lIlme)MHMA{RJKw%*0xpwk?zY
z2bZ5Xik?pBu83!vI=|hXhatRGvqXhbHGyq93g4!Z_dBJVv{_<7Vp#Y>kTPPLW?+}a
zVqYcZvp~?Rd~uPfH|!thUp|@JuBwYl2RJaUefq(s>T0lof$OnKu=qKGFi>^jspl-!
z%`09WF##)#OiU<-(UeZdn2T@$pVZcVjhh`!e^JHuF0^UzWk)BAPVzSjvZk@37nI?c
zC3!SIuC5L_SVebIUGyfX;=mVYMsm_U<1`|BVP5=f|6IM6BB;EzXQz6f6R0wP?eVdb
zjn|nz&VnM^^B;qFPsUfAzjjr<^D7b#uTbfpgI!T^v5s|pL?kH%1w1?fOmp9NH`#p1
z5c2|-ta3L2Y7dvGr*?iu=osqs+$>?Qfj^J#2WG+8u@t74m|(504h;-Y)6!}Qbc4X3
zltm?a=I57&vG<@Z&X(Wqzp49t5;Ug)rgS%uiGf-L(wDW4=}M
zf+|BHfn^9c0|$q&PjLr0q#H}$k*!M``A$tI@oK;ij#|UIz$J-9N?7;}Rt2~du!X=S
zi5`Zzgr2=B*L`|7%2QVtmyo*_s8IIz$8i4^5>ea`vk5&fIePN^b*;<5aDmxlv_Y>@
zR$B|RNrPsYs;ksAG@1-zIgwAUd6~Av78R|K}I?kX2i
zhaEMBw-ls6n0jbvv?BoRn2rZ6+H~}WtuGh9ex(b}071B0)pTy$*oKh=_=(Sr
z(lRp@kmrJ`67>&SK>Xkwh_&J%AFS2rdaGxvrjcu)diD6CnC5;8OF{2F47XlW(eRi&
z8u_hmRhU9YRTm(E+$H2EjiSqBm|cs?=Q+zc56^9830(y8bFaxElch
z*kF6wG)?+Na&1(T{*R^e4#>G*|9E?8Noa@)Ns=TaN+n4`(;j6cL?L9Py{w8tRwYpx
z6`>Luq9n2tQi#eZ6e;TWx}Wnq|D5NXhq~|YxIWkQzTTsvfpl|C#hCIBq*D2&?{}Za
zbhm8Tsm#n#(RV|v&$>+;1Kz>~fDYC^p-=AbBFRCrV8+_o_&|JoCLiUt^(g6m&dy^P
z*4g2dMPwHJl<$a|)3fzv;+KudqYMnVqCbE9Aexh|p3Q^kH|U6h>h%2i$A*UF(Ced0
zOD124&e=IL-Nj0Fy2&MWdIPcEg}+C0RqwuGLv~P*u)cTUN{(Dq8O06p_r3;1SctRc
zq(&`YzMOm%L#baOPRmzoD*WwswO?haA>M0A?nU1X8w6zed;MJ_$z0eS9W7wjvp+>5
z*nLyYzLeKPYnRHs$@=MVr1lkq-TKqVzpZU}Y%kbF4BM5KqY@z~L$
z78jp>ju4GB(0I{%O4Zyz1-g0C0Dx^ncC0Q~RP+{39-o0C74u!H?G;OVe2}hRx#F5C
zD2Yt|u<2u`WqTC2x>#oO@RM~QDH#EyAt}etC-i&yutXq~Q_E+(8Tan3T;tp0$B%Q;
zy?XglfI2S``Wx)Trn`<+i
z#TS+XB0+&POt2^1cH*P0i9$kwLes4+U3|xpGJrcBs3?#NQn9DwUslNkS=80nL#bTq
z>Y8fQLVhCu&Sz}Ti`TLxzdGSVz=TY|HwVHNYp;hVCSF69W4QYH#vKybyP>9LUq$|NYNj;JfOYn*5SVy+tlULzJZVl&aB*y6ADKa4`z&BitrkHeBfND#PHU
zY)X$_ywwrc(xrh{6qYTkALTVq|4>-iRKQ9iwDUVf$K^A8kim?{PG;fsrv|bm=of`_
z#?VqNwkfe(+i>oLPQeCGkBL#%i24YCtM@VCEMO>^dM=9L1g0sBI4D&jvGTa80A{ib`ss!t%cPM?&JjB7=Y*AEF#|+kXFM+Ri~1$CLuYfyg94#JU2R
zw;$z$emQB!HUeleLM0llyayh&9(uf|rzd!Vm)94_32XIgr}vc@8GRRY3`%l?k4c&%
z$1r~_M8Vt#X^ydR+s~i8ID<~t+l-#Tibb&PI((wiJ??@v
zEZ?6Fg0ryP<1(i4$11n5+TXB2cvKa0Al$ibmXXr(^5cw*NL>R!udXJ-(hZS0M$|7S
zri+
zHRElSiOzpc?e3#Fu*`1`%pMgHnY`$TgyO--R?H{ta1JL&nlp^>tyk@zI1
z>}DS}tW0`oM1XOf$t41oP;yV7G2<9Q7XZGZ&HawIqV{Du>~b|;Q^|0Epup>?ODzH8
zu+~;3P?3#R^j&oT8;CV{;Bk>2X=N`m5SYfXufo9LXeYoQn`80bGK%YBo}EDx56DX;
zqiJI(g(LxT|NMEsH8h)SN{-XRl8Jy>ub2@jTr?X?vfPAEsebbs>;0K@YpOQy7CEj9
z7YfLpk&a?P_Iufzg!MyocK6Yr)c`8Y-#9p(
zb;FNj8wlZj-nI~
z9eilw)9iUyIh)SdCtL!KE>r#|)8ELn(B2&J0gt_WgsAMbt5+deTq79Q%1Wf;#Jf>?
zb?Xn}b!#pv+2(foJ(#n{eU&4I$wMMa=I3}i0v1!lWV&QzOEm9qh|0T=BN7U^?0GH%+J-!xaYY|8)qAshE?8z8cb^biPY5
zRKcSInE}CvRD@Nlrmc^Ft;EZuWjacKC}?}Y!obA47KffL{I|4^XXOCl-lNBkMKNsW
z>e|;@iFMIy0`<-xc3^MV&foeU%)g3E$+Z0}&A*~7oROLmI183QHToGqp|JpCfIqZi
zEWjXXzW*-BkNHsEr(lC7kma>&oJSLNZ3X@Vb1sM|Q&fUA-
z3~rE9;Z1hzh}Jo7u)XD?*Us7DBX!nAkDv<>H>uAfM30O*Sl$oW
zJ?tV@RCRTl2g|{WbN#!v7$%2?jhr-zs9%8o2V}Su=hOa;Cy=?MSNhLusFa-2t35r-SUO%V=7@Yc*8w|K
zH#b9)>VV`6S9SrAqXmw-b|)w>m72rSicCy4$KZO1)@?6R9Z|gGPzqU;8k?Ayj2fkI
zez?hi2iyX12UL=RBq|=4org97qJ<7tt$7gjNU5>;Cx3^%xjV#hnd_Arb*XFW(O)|QIEJVpsp
zd{M`#M4u;o_y~UoX7Dz+f^d}JJ@I=WbkJq7n+t(YX=*`e1x>2}wBm20a%0K$^YiDm&~IT*3EJIB
zZxDwbKZ;ahM^?s_w&w910GI@r#OCnPUALpgyqxvOTexD@))z
zzaIVv_>Q
zK(U({HfmH0k93`@WS?{kO+%hvt?=IqB{Uw^)^Mr#UZV?2l8zrC;
z^;oqEeM`lPOUMh5{qSt@cpl{Xi;1|yTJ{>p?&(d;*8wGlWnV)x=+!(!irftM?%Vh5
zxBBUAiT)CwY!LU-xi;^SRLZO`iE!49qm4lY!KZo4o`g*U#bD|NJIm6U)29=vv}WBp
zD%sw>dpW_jE?H8`o}c2@g8#<%#W@qqL|iXly*f@yOF)>;$gtCkNA<2fdh~ZphW*($
zcKjZkjJ*cP9fiHF=_C_-Y^?Apvg|MfAjDhiNeSo&Q}kk%yP26w=gs43k@~zHh}XkI
zXJ=&j3IeSu`83S}j_RO4qEg&>WYNPvJYa?29
zi|`8GM?nA;krSQR*clWj7M#=oWC{%9&&Cuxrawi$NyX+W
zapy6^LOZ(iH`H{*g}KCEVLGN4Pb?au^+h+f3(VKDvtzBb1*>HXm4LE|mCi+oGCP;{r9v^47Xgm_jTJ$)(|_hUM)x7>vN%cw!qHPg;cWPBE1
zn=j9zoye)*2hqu3R8~P9YprL+h!Mo64>yxA%M6w(Udlz{TFjl|gUogz1TbhXd;)$4-2nd_A1^*wZSQh-86H;0S+nGK
zAMCNQxRW(%z?bf`FkIE5a=W%AI~cTl8B=8n(QJs4`_OwR$=JRk31mBQ?_O|_LIzBt
z=>7X8E2_UrXZR55EG|#XN-&livjd*Zl(jO881d+6+kF2tP;h1kk5J
zmQadvVabTFDsRzx-q7XC3DExe|Hd1Mx8&q!6VsSr&*Xiuq9N%nhj99MPSSqbONsNe
zdE+Lz-wiqy02*X%V1FJsOQ~2nyp{<=d4gUCwTzDs!Fxc@6F_=tXj-+Z+{Y5lwJdv{
zrXr9SLCGS6#z{O-Yin;W(mXZhy;1wH6RKuG{ZsLPAEq{T-Y#yU)dB_{qsY@JfO(%5
zM}FI5=cF4)Ez8X{sjlo5Rt%~jP}zl9VXQNEEIG9%ojx}=KRp*b*3`gY
zmacL!Sm_VeEm8{d6PD8+s&Hbfst!L`e%h&t`AA^h5gsn_^+c0zl@OYRJQwzfIEH@d
z20x(0y_&B*zvGxq3ePg{yh^A1-%(Ifh?nOA!_YwOJb(U6G?PDo?0I3{hb0#jOoAfR
zFM{T=W1^N;(5$miEJh6;EIR9(u~*%66`i?yXXhW8&^Tu3M)BUSo#O1wWEvK1YdCmE
zN2xwqL?xU7v-VcNpo{iMK$yNyJFVWi^DjHyLcf8}BdNx$7%ERjtAvjoKOO*ES`0Ni
zybrG5&oJfyfAwX$?o-Hk?~x+rKAI{$5eWpp
zB+ROwJo%deJ-b?WTAy~(RZe#3z)POD;9RKl&{$hWon~Hj0loJ9>cjnGvOLF)A3xcz
z6;-c=SwDp%+jzJjuww@ea4|4?F3%T>E|8$2Nj7Jn;3);PXMEdjdqtt)-bxQa2&`bI
zu)F#sznfzlU8rS?;N2AP2*OYtBZitiTR@s-YK!SRCdwKpKDe~E(^ppR7DnPRsd&G!
z$(nSnXI`S_gl>v2j$dgPHwW$vOzySMMzqhSN$ncH1~h&*l&20s0OU}CB7Q6`uFdJ}
zG{1ETCG{*eZWg!gjuacAuUxudiiv9E=FsPJ+l%!|BnAb#H1st+F)QT}%-^@GPKYXqEmP=^2=9D7H*M#O>S`4c2y3StAkd<~B#zk}t}VU6`C
zimL1VTZjD}FZ_CFossm(f}^@tN{it)D9bQbTOMDsX3xsk@;)P<@4RCIx7J)wMQP-0_W*r*K~(_Pm1Ps~{{+-W7-0j&b(!xt@TL?_?4{L*XZ
zj2>}|3-5pa0Zjasv&7k1^HW1#lL;@cT#3q=@G(?F;mbbF$K!XIldsH%J8-~W&mOTe
zYk|I?EIqt`|F-0l7ZPq;C4Y$~%`I%dvHf%(+Nk1B!DIWI?-)2>9b{zNsgX`+i!Y3Q
zyrgi!JQ8X3qs=cEBw45|x@lQEOC`upRLO7gQ~4ytmFvQc>P({*oO(>*3KS0bki5Gl
z)j<~@$@%(oAh1G$t$|M31l1OK0PUfJiRH-2l5ZdD%6NYRa(pPt)sA+VB=Z6+3RAp7Y
zPpK|Gxd>;e?}zWzYet$r)(D*2;V)8lpxJk8{o%FGpC1}G$K%S^=LJW9YSkZh+FTi7
zt^E(|8z3uUZ6nCgxMMVVE$$`Z#P{MjrzE~`$MA23YJoo%`kvXa`ebnZR8KX%^dB-#
zyIS|#t7S|LO>c8vCC-^RX{g%59;HCP?;U%u3wh3B2nEr?=khmjFXsy31{W5NpFDXo
z$`Hto=G%UJSUGq5$T4kJC62$fY|^derW&ukIHzK{$2y%#g9e$hIWKo_wpnu6|AL%r
zhn?vp@y6EAY7$|92z3pXQ>Lvw+37IU+oxG{`twPbR}Yl0+3T?GbZ6zPWN(?8x8ccQ
z(^ErC^`kr_+LvB=VNum>5n_E~c!uISUA<%ZhILJro7bet?VoG9cfk+una4tB8&ub)
zHR&5^t37JIao*oGQpk8hT}9)1|Fxwy|?#yjKqXKW+=NIt#b(|1AWshUTAgij4{id!>-L7pBzfA4<8Trj04Q{<^
zSa#OvLtzio@USMjYt^&ZWFqN46j*MAohp1qaYM(@Z;G~6L=>Nl_HS~z=ucj;Xx?=8+j;tv-(y{#s8;@+O0m&ZK86)58HBX(Izy`DXP
zzNUMUi9D8yC|jxcGz2tS|Mqmf_ZK4bD|;RzZb#%;jo{K{6HHAP%$uheWnE_QZ@pPt
z%%b{~lg&vX@}>m~eLsqN{aWZ!s{TXxxuv9rZ^wAc9fRt|_xce3^Um7m^-E=coOzhd5iW(hn^SSm{)LG!#i%4hOuzVz9dyZl-V`kfA>l4n{#-C
zhHsT+#5IYbL7H*p?d3h28`h0?)>3gOd-ipBm9I>eylG>SsEEpsE9Z(-x&vcu3#B%O
zb+i;n>~N0t_3a1{Hwy3Fp>s#arr^WfD48%F$-2W~2HP(P{#J;5kF~B;s8MWLSn8;|
z7naG0xCd$tTUZ_3TS{5`d*V@_Ied$%`kA86dZl)$`vxu+Qq6lcNKabKD7H#F_URPG
z^sO^b%bRNcRXy&kcksrBQ4S}5o#Rq8CY4O?y)jHXJ-Iyo!YF4Yw_($68>TP1d9TP|
zTD8*R7n;xhvo_AN|{Oh}o8RT))ZF~0_j^}L83Vm)}5+s{_Ix08N
zm#*A_D=Lkv-LO9(tFS7kzY0Lzl#+xz*Z$_
zy7sn>)&3>}!knU>N^je-J^9O_Lj}+0$d)MQPjHU-EwN&#e9KW4!Jp?_`rQpwnLD~&
z&Q&|xJmLHh#j0Ji^bh}BoipL;^U%pT%ig4C3%=HriE(yJCUNDFxu1=-gESq>-hWoF
z`LIzYyy=unQ)_qN-;2`!{WxyeZ>gm3hptAA|2RqEbJKI(ScmSK$Gu0}XE;ROR(ccr
z-;Wa&u_=sH2?~9(Yo*=LV^(?2|K9mcA1+<;;lB?ky<(@#{QrHt+CGi%q;Be0S^cu<
zIHRdsGNYo?N6us3!f`W(+`hGD*fN#uf6Fs_r2qHbhsKHRovAh1d-eQ;geSW3{7$nq
z!JTR^US|CE__U?6V(sA(nZH)tT3Mj%?tMHpGwSfluFpS@pZC1`+2y;uC9%)OYCb`>
zLpV@kZQJgoY-?&=ALOpG&j!~}yl&*z{+Db1xK$>ir2gk082g7S8d}P$S(q#`c!6Hj
zKS`#wuh2o0FE{x;
z)2gj&PWwCM(a^{bKU@d*JTcKlfJBXNPkmtoJIl^XQ5YU?*-5SZGDXTR<-
zurWUa$fnj>iSZWmsT_2Ty~jS9?Rok8#nww+cMa`z4&E@oR<0=2tFvY7=k3S%E10g-
z6$U{M$My*CigXtSAT|L*wX}qH`M)r3Zp{~pBI*wL#Lw^FgGQRi+7`ggggk1ho(;l>
z`V@%uANpB=O`c5H5A)O0&OT{^G>gn`DQ6RwL2vuI7CN!9Wj(fh`Fm&#H~iHUb(QbY
zc*7;S41P+%+=ZYCX>iJurosevs&HC7uw4H6^3PY%qo_FqG9vcBz**Vz1k?EO<9|8s
zz^9QnVbDEbI>vBCNCa@N=$To-QU^>fYuxaf<-3)le17guJ$v@>;lq8T#QO9(hzA0^
zW|P5+GGS~ZDSxAu6vULT$jlq;s4HzPQXgh*)G+J_g&|Nf%D&j`mxEIN5#5@Af6EuFPE?n+
zOKo1*@nS<*{O-wi#eHNi{pxPHRuiv#YQXw3(;+GUq+|5fzV%*i*DxqRS>}}0tRc=B
z+wM$kJ~qV6EJ87A_m+PSm{aTMT)lFo=>A@J#FkHx%)?V`^6lhk=X`?C&&>E?k0;)e0CC`BgNY8jgFq1_fBNVd>q7D-rgb`li+$f0C1#X!#-^`
zpDCduvgEp!w}Pf3lRWG2=Y&D&D*x+j=P?S{I>N#Z-WIDmezni%=Oa;eT-)U+dBhm{
zUg@%It`kh`MO|zCQi0$ZsEPaVzVc1TT~SI54$=-*Nl2K-e$L@rfE)yjcdZdBF{E~+
zJgU`_6IyW_mnNuNxcP`j6fsUDJ!xwF+jA^LqAVV=C_>}Oe?A-pjRcm{i#J7q!z9d#
z{TDj`mZIQFK2=>?{KQ@OcDN1XvJ}A?CE)FL>9?2>ur%FFJ5iPa-_dpmd8PI`Xa?)?
z01y*NMi>WU#Q*y{YEGc@CW#P7Qt+UC2?(pk#swGUU*D`es29IkvhQqJXx9*McDNr8Md>oNcpJB-oM)
zbZ`Cny5{-wr^U;Rq-1T*Ch92lZI^ROk6T?f)XiEX);7s0-uUPdo6IZO7X~!m3N1;!
z(o$}@lKzVggBOI8BkeQUEU?IjY%y`-LhV=@^|j!$u*10bVj}fG@qar%XJVpCBAV5B8Hl>}7
z0`~>GZ~hFtNsz*sCnx1)Whn++w$oGuZn%l%Ex$?t0bjhh9*bzu0c;2-_|yeD=(^}C
zFEaRzUvhKIRq7E~CeZY>Y12x+e($;TQ^9Nt(<^@--63AXf>jq=to*Vtb9);I-!~Ip{X}=HI-z
zixn2*1K3vA-#?owkN@q`JY2z2H8JrmvKz)sF3xC#+2C)tw{%S7`vQP~&qGV3Bg6r#
zm<6vFZpd6zAG>auUS_J>|MlJSAx*)!AoA_L8z1)%nxH3orFE;##H2zXQH*4kygYNO
z#5?_({K;No!H6bc{F@_q(eJnR;-WDrtu1RhE{-!k_aenTqEq6!O+$)?Ql{K=p-I=1
z2FBX@f*W@{;}*Snv-tAdka_y@a&lL4a_k{K{>z(jY3as{=`9s{WJ}=)xR?0Dc&}=vbS`A)
zLmTh*Y}=KLr_#>DWgqfKFH)tHqQHNVbE~SW?||@r=ukgaVj#hLmG+bZo&xT&0>O1e
zLPCN{28kIQ*4WSx<*fy$-P&0jT6I)Qc3nJwbN=G@rK20qJTy*q%u|+$w{^)pJS*k*
zwJT(JKRGp}4L!TcFaMQ4n(4A7VX*bF@_M;hy-W)pHae-~EVawO
zxZ9%8cVqiiLxg3fCkCpkzp6J&mVu=XJvU;mw_f~$w^cDmj?7!SH2PZqsSEnuPvA+)
zzIxTbKyUYKTZ`Wb^yI0j;?Hag4MF=dwFcc`OvF64eUoeM1a{smzYwF0_kQiUdhNjj
zI2&%$eO_KY&UpWc_%D1RS;hAzQD;72y9;c)Z4sz{5c0*bL7LYxo{{NFkvHS
z;E9KyW@GcjE62dsg;17lPW>LvEPe=@HvShHA&<*1_0|pucc+nIgAU0|pnl`FFIupm
z83!x&wEPZqqgXQaDlsHO6kc(=xQos~e>x6sAf>(eoKmbU;2iiTyqhQtdIEaPqwl`r
z1wCR;uRsBKlrjxM)MUa`zZ|P_!leU*mZhmXD_=HgY!-MO0A2C_rZv(&c`YXgLN=BL
z7jtsrp2*`Ch{GLLNB$ty@cmjotiMxUzM3QpzRCag$+T`An}l^=+=rI*~+MTI^hK@P9vE3dh0+>75`5jw7**-1swb?n#11G}t66YS?COZF}N
zDYI>~r{{SI-}&+Pl4w0O?Dibm7?!B&_QP^=k)lM(R++HJLtid-O4Z(RLT=Thmt~)R
zX>_i3PJfC=3>OWC-zLswZkeIF@>S7$w77v=p+pM54N^X
znVtUh$;-H~#7>O2>!+h~3JS7B6afYw6DuAg`9Pem4?Ng<>|g9i1q(wCANC$DF3Lc&
z{|Xh2QV)c>YlppaT>LL3fBVd-^S>zp
zk1sDdIFK2o{_XDN>Q`H5{uvQ)!!bU!)Y@{$`h}NX7v4RK$HmQy>;GJdtXY+(wp-Hg
z*Ag@LjW*lwJZ^OkFhBJ~dgFxpfUhrhei<lbq6$SFd=wU$T0J73BZ7W3;{`v@d#Yk%{g{?l<{v->~63
z?`72^z-|SA?KrUH-ZEL=?EuWiHdrf_>CsVY47olvnc0GO7en0v<4!M6Yp7bhLTz(0U
zA|1~^1ncCuEKEc5>Ty+c_waZ-9+TIvapTN@Az;0UjiQxSbWwCKGftXwafGy*g~zh(8a0
z)(}6_?D~0`Bm8WRXN)Kde_tx`*%>+4DLrS5hIF<3M|(YYS%>=)K20qLyi_u1>^UVM
zl4&>@9Il|9-ZqJ*fGt*Nm&GKvGG1h;7T}@P)$^7t>Aon^R5V~63I?2Y_;@t6RMWt5
z`ya>$P6a~FJv~AB_KcbSg(G+xWlNrJdzPFtcC7QSZUariM+m%06PF-P`x~=AIR9Umsu8RMfq_CBpu=q0bR}o^ocr
z{Er@LQIVV4Z!2%ii?gV^+QRiYwx{OsV$+1vE`Ku@xr-@%&&=@hf>YLfEUd@b^?KZ%
z1{uBZ38oX2;zEOqi|0g%MWxv%7j{=$bYJt(STbnuA&>2Y-vnE&{yXCSf$ge8gDkX_
zUah$F@UX$ZH1EXLDG`j0^S!RBCJg!eS;@5_E}^=1?FDi1dl_~MmWzJP+%Y`CHuTDi
zU#qfzjcUG@V|esfoU5z7+EfE)Gx)wz_ZeNhbk;4E
za8(tR!tKIsg`Hy|p~{vBbafrym*YpgR@KzF#~WfW|GZEK)>G-ONI)h17ivs=jv#Uc~m75C-KMGxD1PdGj{
z%6fNH6waEq=!dM5`N>X`eGSKq5jZl^!{gCcCAS-!AIx5m$l)?gQh69rBgXbf*kEXR
zH3dzDXTct}dFJ30|5d9ND3-G32@VOt
z0BeA%YSb)~vK2r09si4WBFilZbDj!l!(FPNZ`!C=m__DPzJdw^gQQ=
zW&-74oREVRBgYC3=eszk5M^n{-oATRklOP`)6zZ2LQz@yDU$O@?N_MZT=XC^V4QS*
z!rvKnojWse_aA`#pLD?O8ki41WglEn}Q%{IXKusi1&Wkz&b*|K)L779%s*(k2@XMD)Q!xbhhcje
z?O$D$TclbfsWWNPfU$#S-?Bm~I&;+}*X5Jq%SK!=s-#YVL1J
zHIs6axk`eE_f`tX*?RV;H`V#efOiqg|0b>aBaI(
zn%@X{jqM~q?`q>@#Vwjmd(Ep?@3*x>-C_(FoS>x?MN=(MXD26X-CqVyvn)He;>q~-
zZ)5QFl$R&CXOx5Mf(6^@gGox7v>amx9HgE3&Yb43Crc*i>Cx#rd!8D0upI6Z=398`
z(C?z)q}<|QDIobx?(Qe+gr^p0Aj0SiSn%H3zS{dtP_BlGiV2Dy8aH?>$v+(d5^<7)
zr`YoH^03L12bp~Q@@3Mh2Lq60*12xlbQz@)%y)YJ`V`xmdR@CIUqn-#CM>%hNh1NL
zy~MTvrXZK2^^UE8>PH+m?A^ZqISR#*5&~N?>rFKb7u)`ADFqshfwEz*UH%|Do#rni
zy3ck95~b5Zx1zfLEPpb5u3m%N&}!*dQ#VeCnl&nOm*av3rYQb;#3Ts2k@M3nF&vtp
zKUDEHp^?ckDAFP*c+EG2*YS=^zM9;D1#NC*8
zKMWXn=Z?l?|NMu6!ZO;987{l~%4m1T^_e;6>+HV~a_?1C-%i}%w0BH?(B-|8OH=;c
z-{5CqvCjF)lk?~MWq*hsD{`%^(7;qvvrcsKEgjRSm6Dl{49C1bJDif^?cG94
zg#){Mf0KfZk%_N@X61f<|J`qK&1b81lLnHML&8hFw|LRqi@-b7QQnmAASL6AVOMoD)OkWe9*(`4a!q$*4B_+9>|z_oD6
z8V5gkpp-b3aHdv7&Ad!dwD(ndz%56m(yS`on5@Ngp8-k+Fki`r3d`8~Kl7hj9
zE8GVKknPxapmwo{s3;`nBz6>pVGq1){^&H@
zSH7D!zq!0Bni#!SmI~P0zPW$@yx5^Tx6U->eRa@P2AoGsNS>5ca>3ILgYA3i_ZxvT
zFI|8|Jl@;*Ci9M7UH^twzE=ev41zI_g9oDmUKW^*knXmF{CxYiKk>89?2`pZ2s!GJ
zhOm5K2ZSshRC$J`;tG~bm8C*QB~C@U&x1+{P_R3`qhgLH@d=2Msze>l7{IW@fBpLS
zqk(V47VX~(>hnLsC7lV_H)j?<4PC_)Upe07mYZF(SoV4vGo|A8AG0NI(SsF6tVd{2
zyWf@eib((o@_L<1Z0uq*9<*{~%Wx2U`cLX%J*`G02rtlxN)N)Gu-m)>6A{G=7}2w5
z@ukaFPj0VQ-*Z0$APq*IRkG0Av)2-p888i1aBW$S;DJ*P@rn7UQL|HNEO>s|+`U4`
z$|r!Zy_Flv^FrfQ`{)tUPUSmc7MiW??F_VnlbM%g)ToE|?(qV5;zdH-XWbBay#W|i
zhs&u$z=pE?=<#Dh0~*m|ZZJ76zwf)RpLfuUNoe|j9+5V3U>!ekVjb1-{rf2?kI-vf
zMHRq84;-6{jn|ccv23zIq&9dlYo3>=19|uTCe=pv9*ky*@FA6c{UqXfRQ_~e9rhG@
z;?Nm~a9x0K58ADsVTu^$)F-res4o@+cUW78V%R|`VBrFAj;=#W%!(coBc6?CG|Ifb
znw3X&J{0e`N($Z3|MBT4=Cl_}g?{y=nPL7c7+wXlsQU?+taxP|MaI$oL(JrBR+iXNRc2PaAm|MViFQY
zDtjrt7Z&9`E-4XMFyA=-~N;)sgiX3R{Sw$^D
zzeKalNFGw;vuP7i*kmMuBv4iXCxKpG;gLo-A~Oz7+ZdFNjXd&i=_k9aH6c&LLTp{p=$<%v64P2P>UAzL
zt06RqNqc67Svfj68R4f#lMIZDSjcY30iZ5{er(VpY>F^&6cG{OXkgcXK{Qle7BvL(
z)P4T^3Wy1YaMm_tcn2}yXZFLPpx67?SZ<^Is8KnQk%a3E+ehX`)Ne$qh(3Gs;DP);
z3+-4Fx=;Q>cvu))Mupph;Tz*}$Ai5M9o6x_FoRKB@UM_>@NrhHTuHPjT
zdkH#1h?bOIFjGM^5Ur=CEt3$IA|kTEhPWS2+0*E|eeq%qsSrYI-t8Gxr{{-BE1j5Y
zvW!OwB~PE0q@duhz&{;%U*a8B`LqPQun1_Y??LQVS5f%_fc^UQPR4smsF;WrB^;2R
zAL2h;TxF6nPG!iE(9A(Dw$fB&%R6fX9!k;^KcH>r(kGKkzte&RD`(J&KMD+tTwEdKJv%i)(f1TReU
ztt%$p4jr~
zlcdN`c&WaVNR@O@Q~x@@(}u^ZQ1)~|0jt_uAA=c)*)O|(oSvKLy?V9fxQ&?M(ADFI
z(?zz9VxPddWnDtIkCoJC&C`#vc5-zUU$ur|h*BmH*R{2_R_AV5zHZ%DmSM;+;_Mn&
zX|f;2a-)rI3u%D%9VY5o2}Y&ph8WTq`brJG(AE*@wG{jn4P-U=K)7KYR^H=T0w!@veNzb$wJ**~9Dg~^f?!^s9
zIe_{RjQ5aTD&sI7G|csd(C&*a38a7+{*fEE813)s)#*;@WE(u@1`_YLi(Lon@2$m>
z8aa|i`z>UedG=WGEZ2_hpo&3XR|yB#pjL>u3d)*M!u=;te$v-L@I!S760I!LfKCv!
z1YvpDjs?}b$QB6T`GIstMy1V;=(;*LHZCOLBE(Xf^)-6H1QFMu%=w6r0f3o^ZumAi
z*-qX>d4+qT@_=a4~eAdeUfSLIB*pM9IW0h#1Wx`HnWjP!ZbF8xI5R12CbVo=&scL+$xSbA$0>{gE
zYU;{uf;5bZqMw$!%Is3G(rEwoW?{vQUAHAq=cN5MksU*~hWw!Q&tE}Jn>D_-TY1DN
z;*A+8{s0&@&eO>2W8sRto+f56F(}`@u@8cvGcS8tSZ7X{?_T+I5l7fqJbhu;)l95I
zud##0$Ayw%#&jkSst%a-pD&;wTmllZ$ewA;?iLn-*htK1*VJuh>f;@^IaGp?8ryLc
zWWs9H7CNL2w3fV7%2%5)x_?+%`L*`Obs{!6|6^?h>ZvfIN;%@Co*Uv@?WH-W6a>2Dl1{h4#705+Y8Xk|$
zM12Gx;*%5}xlLT-%gi7qnwMYLmmFzW@>6EP&!-XHz23;(>~t%S{?zxw3SKojIgZ^|
z5X5M0DO4AnhPnK}+7;m(buh_qF&7TV4{j|F1KBZCkKqvZ<=Z#3z!Fnr4f-t^dEE?p
z(vzl*dBt9rKIg_{mEN!{rYtd
z?4S)MK1sxfv(OT(-%d^4V<8;A`T46?T?AoRyx7VS#%2!JR%{kc*RnphOWjBXF8g>Y
zKi_op=tpelKx2S+@@M4zR_(CejHZgO!hIqV71{&)>p?h{IgW<%%`EaioJLk4WoDay
zn}Ma$WA25Ji$?!vSN;VV#Sd?*z20`wMg7z%o2T=Z?NJ;1!c8jAW}j!A%G8uQ!H#{_
z796Tx7LjJ#yLXnRuE^uq+S6lXpHy@$dfxJFYwdl*y<>G+cJ`FrmF@a+>6dxe`J28-
zO_2a%>VdB!8Lf;EZmr<2qvhg_kg$7k8WHur@)dm4Exx|8%>lwg
z4Vn4lWEU~}kb>>x^q4Z$lQGv-`lY*hmZk16dFz?Q;$;mR=j-X2{}*adGu;-
zq}AYrv+(J^?C^a3=dhgIPtdEok3Cd&neC~P$L_o`
zG!WSIe4GtP;kmHh(mb(;vsvIpUj3HNoVD|Za!h)#e?NNQz%pP#G|pn}&CP4b6g)r7
z5SOeydqx--*uog_?%Mhv9+GGVV!^k!?FLCc&Oz6R}6t;3N-`AWh;gc+F!LD{h&i-*1YqagBy~
zA_Su(C4Cj!|Dp9&@~8fw$`dL$^Ts!#qYKD3YVqf%Y&mAlO;!%8eN+=B{V_~C*-?Ja@TjNl0W3mr6j@8eWIlQPqS!Hj=
zo0<8uY@+zpaA*Q%;!|lkdAGb_kcA)S9&_eA2PkEFB9k?S-P_Wo3imR9xAYQ=u^j@u
zYL{vh=dyI?E99vQ$x@~Pr@zFJU=b^D&#=nd`^~<_w@_@O;tB*`>3UURzO>$)&*O0G)p)rB-@(Z=K_)45KTqGz*`u%$4D8W$u^Dv-0Bt8xU
zt=yCl_q+G*?S8AE|8mFn?YWCCftUtjlo7qO
zc_!Ibm!^T7*wvVi&6A;mh1=?Wk30hGq=>`hr&BYh`Z
zr~!R3cS1+6@YdSPEUvY+b^c`!f}(!VpZ*x2>EO*57Z5RrMZP@u}5C$}OC?0{8K(VYAO%ilx6iR>_zcc>09yh@D=^wX*C}h7x7RY$SVK<=p;m3uwIx9@P*>V!4#QuPV2ovI8Ja7O|X3v~S
zA<34F4qj~3Io|YpUC;S*=PKTRf1bH?)EF6Ttw4FQOtX~mD8ea`>YcRX^yxv3m+@uw*>p59kBo<|qYgu)tmYnX-~h=`$Ug5sKBy@i{v
zDOA!0p&Kldp;kQD)TEJAzFx5}NUgklc}(sghfCs{Spmq&%2q{EF6b}jeV6Y2D_(4f
zYZ4vj%9Zhqw=%Zd4)=|+7Rk{YKHO4zb?hgFtJyzFl1u{wrk1$>yGwP~oI}eqr4}m|
zA4)-Pq~%}Hev$=SEPLKTsbis;6B3!hd15ORe%q_gJLIcOys~zzEyoy#EPWzzf2y96
zHwp?^GMzNr5nd^*^Yb3!eGb0gZEal7BD2(14;AHB{riG>iPrnaWs&UT_Bt>5JT;Gqdcv&c1&0x6>IyYsj0V27$@GHJ7c*y`{d%0E6(8xoJ!TRDwoW0f*6LuKVF*ag({Vj84Ue{Aq
zcx(#C7CHL}$bxuMI5=90?G6vOA>Qu5MfOIPvb7j1N*_#E^l_2a3~$USsPD8$6Ubr7
zyxF;iWh_wY3^MCSj{H}p@>yX(ZS9gW
z<4{>yfp+e&0q>j{3~`GGA9yxsx_CVRHFlY84=y5&Yr*{amaRvNJt8MReer@tHMd%D
zZ)ukRjUN6!-eI@7@HXuf=1d_)fmOF)bn)ww?pKpQ%3NJHk^3TH6iXEI+_?f{#vg7@
zKSoHa)GZHx-AJY71wL2~x-GQnAgqo(r6OT6zq(7eA3A7z8iFj&rC;J@Zdk@nUy%R^
z6f)=LHZxQRCi@%v8x{&&hn1DV{W`k3NVO3JOJS&s5mNP+%6r$thMZerG+#TG%i8Pn
z4C!dx6wW_VaZQ?9>5$#WoysYCadmNT*{dQ%vL!yz=6??Dv;#)Fe!ECkJcP#E6
z7^-*shw0)!7YUY~g*)uoDYAL@Q
za+i@>fMv%1Jztom@J)%`0gbgBfuTQtf)4%rcD)6l7Vmhm`I7aeg1k295={6O78X7}
zt*q++qw!za1xewzzID+~+#!Yp!1sj`<~6SCt7&HOGp#9G91?j#hL9D)l8a}T5A4rd
z!8ycpmvL{Hs*d#3sIJaVHYz%Uqk2n;oxU*TbNsDN7k#^Ep?*>HF;G)7FIfgNLaY8{E)N#;+!pCmcMY4tcP3r1`)YjrB
z50XK#;m~G6&A2`EU7^*-d*U00LXrl#SayO&D%46VuE%1bkZK^V7g}c@aakzJvFirq
zpK{Q}#{rSeHLR*|Kga7Y%!)6_B9uk~v8~!M>*d=f`F9F}
ze;Jr#=3?&v6s!5_GY0Z;6uolIBAwpsrHRNPgse4oj*8YK@7hW1KtZ1XBj@(DuCHvy
z@n%(#^X%yoqdM#fhWpJRRJ^NT;>lklgDV<-wY62hdPM-~^RhB2ufC_dW!zc8T5D&Y
zJJ*XNPW6%H@|R^s()T3A*GR7pf=UXFk>-bo!fTk!-zm}jBGzg0(#PGl^Pa1DN>wY!
zM_^Uz$)V%!E-dl*g@>nR*?gTV`8>e17DnfaZ)<8ArVPS~>~qI)TPoQu{cYhaOzzy9>}=#-
zEYPZR7JXN4+09QR=nleCa=dh}1cFkCA;KPJ)VeP8G^Fu(A
zIM6|GICJ>PFpQxvODrAy^y~o_Rq9^ScH7yr=QdRGf-YH};G-9#)UT~=P-I)LC-~_M
z*VWCsa)sGMcm7&z)d-Aeln{e8ffw0mshB*OtGhH)g=AEM0#2u<)_*?~FskqlQBCFn
z-LK!i?SWMW8T}_B9b(P|X$Sw;(|O1B+_(L|U6KY#C2dKfge0ShBt%JBjWa|jSy^dF
z5gH0fl*}k)Wml3^Cki1WCE1jOmij$E*L_{T>yP_>T<3N2{eC~6_c)H%I&x}DJzqG=
ziz|nUbSo_%w})(QdiWo~nTLj0Pn*`u&}q@QphZ&5D5Wj~dKkX+kv=)Tu=||Ngme}4
zo)9SCVHeT4Ct`y7#Zsz1>Tc1fc~5pt$^fNDkM^C@{ndornt?J+I(I+o>I^T-@A_rJ
zl?P0Wn=jHOn17~CiY%VerX+b+$T20nfa*_vMa$o7=!pQsF%RO0vgV9KC_M>S7y2#I
zaJpKT;uXh1gHEcASp2S`p}3?Z|8A6~`^E`JJZH`NP4vF|aZNk4mDh?DyKUX$Pb*rCP)pOHZbc(aKMwSon~-Gmc&~@@wy8EP
z>IE=xRA_Nh1Ij~0A4}bK;!wx*#Z7wKCwII2fSpfrwQ~cnVw+l^wqQcecIs?q<=~ZLs#M&+Ka>pduK9)CoNcJf
z*t~ZJMi;+@VaJcA=JBFZ&smjtq(4a#USDnK47N~RaDJaTGi03!!Ef*wR1j%pM?e3m
zPifS?$SmQd#GCf$LDIzjH#V&S41}=OE_mdq8_&d0ePd%eKgslBjt@>mu#bCt-8?IxaS%
zbDl_v$kpeYEgqR%drWqa5C;pLu;vkWlTU6i(-hrzUpjbjE&+p=FaIV;-8JtOi=8AK
z3V)PtJ-_cBtv`0pA30MWXcx8_=DP&;+uTc^>13VnIZr$yO**!!Iw%5c(?l???MBlbpH++=;iV?*g;KtdbmuiU
z+14M${fOen!x#2X+r~4Xc0`Cis;WZ#Sy@#jsrSly_oDCXA|q~la)faoHRUyXI#pDL
ze$hJc1ztAe^l3sHWwtA}DH#6VnkyfwIA_=V4-}f#+8@7vzq?bFhJrOjn}sQc0TcUn
zytSXShaG|w7hJ#NQoe_2E(ZV#^w7}Iv^4Dv{jbPrd#bnYzP0daY2Kc&d-6BP!9FizDVl$s
zZ-_7rbBfz^pSj9M_i(yEjFEIMbM6lXi$`iEWC>
zF3cD~Tp(y)K77cd*EM{uZFS1X-fpz~@8jX`ym8<#83<3>q0s@FzmL-GLc)Zd>-nEa
zWHy^O+uPrvQNuRLmQ9~NcFPSoj&=(ql$_HIOQ;7k6ehhg0-LHQK%C};FNuG5SkN}K7GWe46f9`pv|!b
zhOo$XOpSQKz8b(>7#{8ot}ojAe?Fu^RgUWa?qQN#R@}yLd)>B6^G5C`*
zgF3$Ofk)r29_3r*TAuzB`zNYt=$zIdp=n^!-8xZ~5E^R&A1%j>fi$$~Piag3_otIb
z9(`&78BfcWGK)>qVT%(1l^D-L;gpO1T3A@v3CLX3kxU=dne5P(1kgpuk)
zQBT7OuqLr;YPFaaY!Jec(fsb;|4b+Xy#ArB<1+BOCg8;__tHMF!@gBtKoV{|?=mUd
zJvU`f+O0n7pSyBio6uSbL(+Up6M*|s0&i(+*iY3D&t`bEZa|;LBU0Frghx5zDN%j+
zaZtBWL^v#wv|K52z@0ZR(`m4;RCxGsaOUR^2KFvxQ=yUQY;Kt9a6>FX0BRR6KGA$X
zJiJYC*ZyqFJkTp?{ynNRN&&{jwj#EU{FObZ_YMk)=xVoymC}+&@iUXdC?sf-AwlD<
zHW{N&`2DieCr7l^)2Wkmrj+$v8yKH_
z@ZiF0YkK^9gY(D9;3wOVpKO~PMQk6la|fuaU*rG1e=qDv-S{ia7Qn<)P+(n^q~Ybr
zDHXGO_g`dYu*O&YDvyGBOgMsz4Cn0it=%Dt?eTUd$xi0OhgV-*+9&1yiAp~#kcd^p
z2DWqVF|KNm(*eq#*VWZrXJvJP@ELgEKD~RlZO@6Am~ze|RO0l^DF+WS@0-cyCr>V+
zTn5`RmyN%_7+QP7rcI5{b1_jdM~Gn*58x#MaD*Yn&W4oFLcS~59U$5*?BWDVNkuT9
z7_Ov^;0?aQ?P%v_`$+*Unby}1QH25(T_KAeIcY&b8sP=dN+7yT129a?7HIk5*kCpG?2$(djLv-2s7o}JX&1ez
z4?&JsEL-;I*)wT{y=YYqA-|ijP9|spEF_(|ZCD(=NX_;2GYqyJuucE?=~EW?Gd>Qk
zH)QPj4E(a3gGN!HPLYV@1EOLT=2V++s0{A<;`wu=0ynQ|fjJ2+#hyI`W*X4oQmTR4
zl1d8QLbzdBv25WOu<3QHR&l22rE%YpF-XG3n${&BCN^@<|#Z)YdvH^lOuaqZxE`;0xz
z5J&Uu(V~DiQH1PLA4NzqEcYh5-A1cKW!J+TnPYKHa4^90Wz!Ua$`Q2(pUziC3vlwp
ziKAEgrZEcW8_0%Egg?v_f4=SsR5nDxmYN$v7CDqFJP{`er7T=3;?3x<=*&__`Mc3l
zOqtT-)EPuw)FdSD!rO?isC`YDe0@vD<T{oVdSVywGFw7Hs-iV?fTw3T2M}6#|XVOek!+Nh>Zs_9fRj0r@J#rmDnrx7I7D?>_O4%Y<6e>Tk{F!`Dg)0?+z?thy8P9vP8#0R+1U?B
zxjke6fDBl`Y$bDZmFCcdQeVy}rob>`qvh8fbvf=ZfIZr1B~{g6Y|0dCXIeeBjdfml
zURzOy>m$(fe)5PCBq_W~UK%w&Yl4DC%_+k1IHhj*=gB?1+R*i5fl!uR#1=p;qW$#f
zwPScqS-c^<)wOFcL74^ynl_E#Pl6{#H83NDqVz>|b>X19Po5+edTCg$i7u(fy})gQ
z+yG?(!ll;scE+t&zj)D(UI|PM9s^dJP&gQ*;=2MZVY^{roVsFhTto{u2C|Yg_dKIC
zXG7T@Vg@asr0m-Jsbl+nj-w_Pmr3PMn?_YQj3=B6eKfH%hKHl1-Y=%W;6
zuRA>1>DWXL{Ew@5NUDwTJgwU;BOx~7h{7Qk((3uSNaW;}jn3?67apu>uQ2P-
zyW#Ek2>CYb%LqqH25H4;v7ngb#85!orw}`-*HZAf^{64XT
zNoci^p3|LJV&S`X4Qv@`lly7p8P0LO4%)9Lw{QFKT2Trx6N#gdLwAs)G)cQjsi|4|
z1S?lOdnOE?%*x{aB=tIjn}k_mx_yn~9VB#+uL%2CL;eHvlDfzQ=Ah$_x*e$0V2CW>
z#PSVkmQ!F!zkUC{U8Kip*RuK_pwOK9qLP{gfBXLK>2!i>19*xDzn9@k-Jo2?hK3@G
z6!ZfM=DwdIFeEL75m^^we=Z9%|Ic5$x%h+E)c)*
zQx0230vtVk+QZpdq&V5SK*YrJ)ay}TqX>66nn-Oz@Httza`IxgG`mQvFxh%)eH1Np
zq;xfF*Q|N>vkimfD~37USr%S%lLXlbnW%*D-H?2pd22mxJ&h-Q*G0-5v)7F;s^ijv
zRb0&h#Yq~eJ01Yw>Z*FF&Hu@DE|qZLb2*^&Crn_R&sZy~Hx(6%25+7`xyT|A
z+-_sW3{U=-1HG%c+=S+m`{Q1Dx!tE%Uh|QKFEbU%${AeY?S__an?^m7k_ZnRlY%C|
zAxLoaLIsjM13t^iR`~J*I|@`HVwVy=bL2WG0%%Y;>A_AF)lnjLa3C^4@aMU=b+x@2
zUg|^aHhYXC0bA=GpW%0|S=&rM`%7Wx7oVk)TM0#`%n_o0Q7;v?Y~lKYCtJ66EjcW|
ze~DXG2j>%!b%mewi1
z3q+#ZPhv(fFX9Q{iMAq_FJ#7m=3ASc+e0+OMayHc3^}|MAN;p)|Io2x`_D5Lc`#2X
zi|ZotZXsEt`SHB8ED1`vNfQqir3vx4_Tu!e<3c*!z2|T_8UA*kHf^Ymj*U?gEyYXj
zNap@(D++Wh#Z~QoS5MD;XgQf3i5`-Bp4lLs}q%Y>>^_$l!zQHG40?qAhKu(Un
z(_J1CDh>vJbavhwYGM+*YZs%S+Q`xzJYhrrj%mL&wpjqTm9-<*XDnYr{8ZE93xBy=I&*`r6Zl&6U|%NsRzhl%tSk16&^
zcLsHWr8=he3hNXpI;>qbxQ_Z4`WqA}QQ~(${Ol7gvrk#=!S;BxZM)7Qn^mf}X!xYzfQ>|dWp
zv46~m1b+bo0x!t*T(1UzBXc&msQ>$+Vlzn^mVMW0({qqf7+RL}^t!bBidl&jeWv|O
zxmi=A`C7lZ7h|fB{}(Lj9~DF`ZN0Q*^!hncg9f*a?N#zdzPa=H88KN$#uy$OLI@H!
zU8LOIl!twy(Cgs;jI&
z?%rMXy~5g81qFpC`3(5f{XMkGM#l*y>V~`ZYny1&xE4|R+UO?)iMG!^s?)Z|I0h{EMCKAOnB3V~W}S9xRC2PKtIVBnq=F5jub3@d>L2%7@2C
zW}G>-H=%II+Q`2X>omPDSS#2MnO$SEJj!w5lE@1Q8#c@|Fikc$GfBI>rQ7P}eE+H}-$Pu<
zu};y9$!1%Itm?cOdU^J*qNePJeSO{~zN^3U<3(KC$fC8DA7jQ<437&i&`QXBcYe#a
zp1u_&eM4$1R)y1CxyR93Jx1EuhZJD@1
zYmr;~_WtdA(zbmqYt!0rOS$}Fd7y=CzpgjKwM`n*41xpYtRE;e&mN}Ty&H5mID(jW
z`s>f1uLaiG2KHg`+26{@6yNYC>_~xi_R6vawa$Z{?*lw~kef6&@#K8zCBzm2i
zVvav{PrH~=b~VA{w9)s`nXPv6*Tz22m$$rWqYzxE=X{op_enE%
zpX)u~v#+IA%YZw)x86AsWmdW_mu7kQ|2o8E^#m)4G~?*#oO7$PU3IP-T})Vdt~2eZ
zv-QAUQd{C@8y{VbitW^-J<>w%#_&LSpB-wu#!kKNLI88KeU@WFhYGZ{CQtchE=^t$xTg4}Vc5?@L+hW4FW5%J~~
zg&mfx`nPZ2rS2?@`rRx4*z+3a@A?D#AKTn@Zca;xb9maE*>#R@HqNfh6RXO1k)7Bh
z^xvU_e&&nE?>T$IVBQ8#9S8jcDENp%=eJo88iHs5IUJdu`s&G(J3hH=XXtoD`V?M5
zbyxVtXxMD2fPwzWpOdp^Yx!5)ZPa@-39u(F*`voQ1`l!s
zg4A-%QrX88XQJpGHq5Q}_1OCRqBfpLNXzO!Q92@>p_PFokG2=uV9*hCBS#)?-L3vA
zKyr=c>*C^xMoA1VzS&*5sgau?_xzFi7GhmFq4Pyax~@!hb*$uH
z`BNVr9L8o7ZS=JEnMMz*L@lGHtU~@>Y;_<3KSmc{-|HFJsb^!LxI;qBcbt}fY_!pm
zo?|rA7JnSv>3nPKsn!?eqdOG8$0&Hq^Cwz25$cOF!7zU`gBfCq+eIR_@jb%COJ0gS
znDjQdS6tPTV}Zf?k8Ez-^)x9^u!?UK&6IAvSrgHytaV#M_{jC5!DkPoq(J90I=xq-
z*jz(p$Pd(@c$?Uvb!zGSTVeHDJr3=@^Fv+x&!C@Qml!>naQ4Y_;TwmEN7*_Yf5bDC
zkIJwYLEl=`u<$>>h5tqFp1s&i+qldr6K9Q@9})haU)iOp-KF5z)m}3w^1r{jwZbOx
z_w~KM_P2KW8y5uR8vcxOd(&njI(>$QMc3Az$veCM?}LPAMozPM$)(^st-Gt|6>6n9sxZVMIJer{{vP>aY`e}#x$xj+8S5Z|Ye
z(v|l>##X=gkJ0M?d*t7(U0wDvaYD>yU)UxqNV>SvpZBv=0&>sf$qbfi+-O|e#X7J$
zm0dHqeeLMphvw`HJEpbumvPPLt`|kxZQXB$zc>Il_44JNf09DN61Ki6truQDUw6W0
zo?sW+AGaqrh4(IK{ND`HxTm<{79X7AX6g;^
zJA*%$5e{LZurRU`RJy%!Yc~hdFE9kYEGRR;6i7Gf=negh2g@Nw^7Hk@zg+6u+<)Do
zSAaw2&IwftqV(pp$(i#qL7Nr%s}CN$M%}`ft<4GhdHMW#%aG11u111RieLzy&gvdm
zO2GqgNCa=%gjHA6oB36+XjHbm$2ZvrrS>tHkCF`sJNn0zf9D&k5Jxn?D&z*H2ue5^
z2_!E1z4;_W#KPNEuqoO0t7xmqdeNNL&%r$
zyguaEy}`jk1g>489-zyL<1!TCti(eQ>Y`c(od+yYd`Bv^p%@gE$Ph!MH2^HD7N
zKU}758G3PFLV|9RBhP^0AZaeMYi0)pop(20iMUS1s$oHTYtbK(+ugIj>q3==S*Hcg
zk8F`p;&~JU;e(87r-es8W)KgF3c0y67j8P*`E{_C(jEzBPVI4TKzWaSlQLMEpRc~WVjkW_
z+9B|)+nQvj@1E2FK0tec(
z$EjZKN;3odnHfF(E#5U=v8QB}vDCNB6I!y@!%u3P$J*=Kqf1%Q6s|#>GBA%mCphZw1G)wJtzW~f&T}BtI8AjqJ@g>Q7q4)a&
z<=c9GI;M8+t(dx#0*>6yj(ZhzjU(6bn%H@|yN|sLVnY&_W(3YqQtNA8^l1L#`D@?{
zf7yL_E*|lXDX2XBO&c~azAeqz6I#&Nm^n=4YYbN#r7damDE+Gad}?J{#ROavdV1m^
zL{HB+nvucQKNuHR;FIg?7WGX!%XM)_MPj0&N7v~&yAvY+pv3qMmN0)+ILW0=g53&y2cCCFd@l{n3XZmq>!N%F4
zBgWhlg51GgDAsCE%PZT3TQaPqq5>?@{^hpu3+@%#(0LlGM4aC0yjty^m6`r5B?cjv
z`ZB>QxCuf7IP2y4MKM`fLuZ-mEZS*5RIJljM8v3QioK1iE@n!u-5E1t(MtaQ?b8u9
z#?T|}bAhJRmBgHvlMVKUwQ6_7^LIsxWe(Ji{H%90Bf4gQRD?}NC0qN3W9G+xiARPg
z9^-fj8Y|sPZ=+&qcI`Wi6TtoiZI{*so^mzP&{vyP(0z2jt4{Tzbx(^lIC}apMr6RTgB_2u;~mbh3WOjaDlDcL9i(3<*1X
zVv=@zsZZ6FUS1J*_YC;*pV_O$Uc?WkuuBNx5VGn9hA*Z*rX3nRe0cifPs`0yp2c!R
zVc>>A!3WB1`7?N&*Ccx3BjYz26vf%hVG?L*>EMwgpO6xR26wnGq8r|Vui1vmqZ6NJ
z!>j=GF1i-9Y8AO*OA)RuTb2pT1N_1jjX0e8M35k>9e$4#xUOy$CUBs*v+Iwz8Ue5#
zM!~Vad^}lz&dwE3I9!OUgMumy^?NS`MFjkX$p8{(8B#SV#uO7pcW}g#>6l-MUT8W&CiKDTU$ftNqBIzq+}!g1O5kg!GDVz0XtbNA}^
zyPOSC>>hy;_?*eb4d4}hWP2NC07?p;IgAnt%{3r6fU#6nIOo_%TuDGluqy&sfQiHW
zz1ozG=snel-$C^T=3mV(EYGs!(2`{k=QiV!J#$Zvg+@+Tuj9d4$*)M
zsnbzR5vCw&cC?Grf_y8UA={8x3t$+|8X^PJL=k)ZYy)l(6G|5UUQ4~yht@?>e*
zLEBH;HeIIS;eT>B{{HFHOcxhc%g)%?$?rGjx>F{Mf#wC20=q$fO7%mbfpQP=uKfnZ
z1!?o*ym`wxwx`{V+4;!fKr1gHdEoQO+C^zkF<3DrqFMk%X-$D%4V4j`z2;V=0204>
z;oiE@(A!LAf;r*Z&~-VyGhlqe_1Z#&zPj;*;)-Mr>%HS5qrh|Bg;U8gbpnI236_O3
zH&033zh4*&df+lKbZgiC{g@GpPs}TfK_wuq>fp!&)l>&a;W(U`F$u1fluON6{HxpK
z#T{SXZ3tpk>5?TUbF_@)!=%uZa^m6TM^HF^-PZ%#w+nM-F^@3g5djx3hh!zs(=*ae
zLW419737p>Mt@)uO7V*NwU3!F*%MBoAYXc?jmX~tM;;>TFjkqmP;1+hfjqoE22MH9
z+azI;QEO~`gKwB}f0FOX{|t~Oca=3>@#M)NB4A=;V}Ux8{9O+lcmOi!pTn|12(?g#|NqPHoh+aYQ!aC8i7g}2l|3ug5Q^(UWm5n
z(W8X>i}$xsVYhX3usWDDhqjgQ0~Z%P6A`;P(7hnZSs5Asa3sSVvISVL$xak(Ac>Ux
zNQWr5D*7H!ekbWg8O=(`|2Ub8HZpSJ%$f0E!wdokqgT+dAQzjAKTd)f+KTX(QJNC-
z%Ajt+B{1+2e(?U&CpF+zY#4n*kryGBB0Y?HLr*0JbRNk$7z815a#w}HPE;pL6^Pc$?OTwhMXoG
z9u9x;vSmBJeP$b=;7Ia5%V+W);NwHkVlWYtc?4$(JvLMp4y*x|LPf{kB8ZwJ4}lc?
z2P>k`>=L5DHz6HElN|qI>qG=Ar1LRgfRW5+^?*>n0f@WInpF@dFXqdvU1AM5nL*t>
zC@DA_aX5>K^qxaJ1JpjuHO}iA0@fMMk&|Nzwa-}^w&*;R=Q=oG1^ktgCFchJWL
zEi8D3kfNad?TR``O$|bHsG*_5v}uG`GCdUw5%{#ezCIbHdNZQHL$!K
zxqzSxShHq+e#rQs#{EBz{hxiM0}4NG!!)C$J?N;(;Q{c58}aw^3tYQ4bIrSW)ZHv8OwuD0qn|&_*^gh`;UZ
zS>xGvId?!KNuF5B8I8;2P!zU0XgVH0T_RdS*)MmpWjS)ujnt7y(0d0OCVEBI!oGb^
zF)0giay~xiIU${Ift7QV5>OZB?Xh41(%jE5N8AY9Iav9TB>I@D@-b{jBo2Ly30
zFH-)L`YhNQD<>KM7
zbkU-T>CW^!+|)^q4MA7=EAg<+aCB_9?%n!7q@=6x=M2dS
zBbgl?BkA&-E7n1a34np$N2A3Ag3|;q@Zg}7IOU1?vnOV-cL4P(Oz#7qT-KtP)N!&o
z+GcW)uwyMW1l4{_F|HjUx&Vs-MSY~LPDG)|pQ(z+Ihs;a7jQI1MA#9qhN6m-j~2Pt
zp^<2>E?ls?$cQcUXj@Q5vjngxLb*fQak1ZOd*9zYrI(oKgMm$E<~3&Wh(Eu7ci~1unNQ#u
zhwB&(d1B3OZQj7wi+<{p`Vw{J?{B3X(C&aHCu*klYvEbW@MmW|$VO#jzd3pQ
zc;Plu?60WaJ1Y|{H0dU{THkV@(He;Vq=rFDT*GRhoTFA_Yv8EHuuDcLI>Q$fP!v?*
zqUO4~K2&T%d?8v)5+F&Fr1pZ0W*qOl`SY2><zrY!*IDN?8RQnIe&gjd~PU^9CH$!a$m=d4);VMoG|&Nb(NbrgwI{>82P6lN{!r
z;{_%NJk2TRHq%&I%|u&U$khl8d-vhP>Z|kl>rsuYs=r(4!I&KxdHL-3jFk6d4jCwq_1g-OvM~}pY_TYjd
zTFNV*H=?8W3~?J}hs%T`41K*C{NpXs2&!q8_<+;Lxr3RI>g2TR%$YAN86;t!Yin~B
zm8%@+ORG(o604G83=Tej@qKkq&o@*g)CWR$gcRmdP7Z28+$ds0lr=T+Pc#H=<>ljr
z7v_aflHm5`eb3AB65Ty1;!JZy=G#60gPbRaJAKS^+R?Eg-r
zwpm9mGiQo!?JXlgK25`eQ6om6SZ(;(#>r(ncqOahzOIvjM0cUxVp=CGf`%Gp_nEpM6{
zHAc9D503gm8+2;A+^k;r4v8`}xH$}QFq7>GxIzRfC>ZyKI;w>o%kwSm%`g}$m|mCA
zyM?qirN_q32o7#VJBL6eqR`O9l__H30coKaY4?etV
zrePwr2245aDmm_=E4?z766Z5vHsxDLe4AgZ?Ad
zN8uIOW>v0GcdbPrPHBBhE
zI=K-ksLzNm`#vUZq4ExUx=S+VySx5!zkp;>NayhkFr+0ulq-AI*OKnCyKDsib53#{c5d(vYdi4z7D0yUP_HKdBn*YBWR1U*_yQ
zq#TzoucIjP@JQZuXh*Z3q=W=fELsybTL2X{43w%j2I&Iczz9bzI0|mC}pTbCc^FIKX)rxfRmnB(HI8o?Ud9X}56V5PV9t1-(6YZx5w
zz&AjnTPa5M&oL7vrzSQij6{mOJgZH=&mxsTf
zl5!nJO<2GyNx&IT1%B`2p5xx=PGDsmcM#;o#fu7GRxBYuKcC{2CCq)k=hA08^-PQ<
zWR(?mqMj${#^!J(v=J&y6BAMldaMgGR{6paN|~o>=IemaJng+FgP|Kn5vY$ivW?E<
zqbYgKUwakQ1*Q#;-eb5vR1Q`hhK!2uSnKVs)9635v*ib@rAxyVNriY+;ywCEE_I58
z%>F{yjWDCJs%kBA<1676Uv|yK!P2A`+9SsP==bj{Bmb_p(6G>tdp0d33@JH#mNYV<
z3_HJm-8uj#p)kY#4mDs``E~PzZoq@;>gu=$2aA0K&714c+K?mh@jntu4qs$7}mtEqcLVaVWAfYh=G+=K4u#FTrvvbbB7I^?rksk|Dnfuq8>&A70I}N
zu{GIr?OJtn{~=7_r{HXA#B7<65#Lj{&WqoG?QGvH=17cu{lr*58skd%Qg9p#S3f`g8y4Y41#1
z%rZuyTYHJvD`+WUtR-vB@Y&gIhy9XkdkUjcu>2B#(Y7>xib{WTxv)eGnAErq2Si^>
zqmLLhN^;5+qa;{Hf-ET9*hXF4myj3(2TEF!jt>J7WRbLge?+L!j9z{Mj_mKBa_A6#
zO@sUhGmGFRZTC8Ee?Sn9RnFjkYHHOlUrs&wc*XMNKpO?AqU~WifWN?k0Qv{q)#Rj}
zwA+d+?YP6U8n%C`esBIoF9YqPXXoWTASwqK<;%JJNk7^?4&-D6{Leq|3Ru#{
zSBPQ4wAOakb-H~72}x|M0;3b0#gfx)^X$9Z!ofDf4OA!Y=bZ7Z9~(*w;y_$f}|AMh}?Y8+;>OZAhA=O)SCw{
z$W6136fL*c9oFp3ArH|noD2NwbrcIo%mPMe56l2@i(KN58
zwlYXJF|oy<;RGV0{O-R2{9vxp71ie(Xc5>FR1vRW>frS0dxwetuSczsyjNgAQP`{3
zG^XeN{%zhisol-`>`?VdF;HR%r=|;NW0dSff?{7OAYnmJi;Du7viSz8B~WB{ewhi_0rg
z_U?5d^83gUPf$vF5a-zmH8l%m{>(P?VWbc<7M7cNyPCMJO|TVMc!cWn=EZ2*w{2S(
zAJ=X2OW8=zxtrUI*NjV=e0-R2b`Vo#$Bnx|^s$D7vi$C5{S41ju9?Zn9!r)Cik{)8
zoV=`nyBd;$1h-M6W=}{_KOiwv8DEb-pfCD2>WV>0j?f4*bJd~;LR?I4erWAW*gms&
zAZSl}ww2{e|`TR?oAIC6)b-5
zvGf2_{`QPY39&-!o)R5d2?z6J__Htu)WOdc{Ub*gS(gp?+Kqr7_#d*@d-V=+hGpfJ
z+mE!-vrWik?^NwIXVL^Sb=#hKbS;}@u_AKhoYtN#lfbGkc1PXxS6BwA6SrQMYs;`q
zFEOrZR$AyO$qg303o@Tr8oFqGd$Q9lRcYW(-MP+-UDsTk@M8KFpGy$8)ZO^l7MkUh@8R<2eKA3^gAMpaWI
zFJb#cQduT9TjL2GBY=9|OD2s$<2;#diCk;+o6W@4AArlYN%p@qw^wRhZ72*aM+Idk
zprKv6?s=D)GRg^FI%eBLn4k&ny4-xzwtb^JhhFTvc4)6&x9v_ZjMnJdB>M0sRf4OL
zFdc|gG<+Ql3-a@0Gb;WNAEa`k$M-&`M$-%90gX)Y8ln9~qwCktpHM5-Wi!uIt-B7!
zCBC3@ny!ulaU?evJrcak{yeB?%XVp38v-IlwS1?9C73!zahtajb&=F4@1G%
zr^;U$&ffmC#x_uObFgJ<2s5J4+%123S%|%3dNiiX+Tc*)38X@2OfarUB}HqwIbs^s
zV1bGAYy3F6uxb-E8+jHQ@+G4#H)5HouNQxZlHM#~zxoO(`vR$fge7flt&2LPZz1;T
z{qGF_GoLR%Q~m9c{p-;D{NRe;GIeH0ReM`#kVyS1+GFb!vT}MZvyp
z;ns$`6-!oKntT7lPfS?iEw*TJyiA$nHc?lpX?Ik=aQpC8aj84K56sPaA7}w;G-)ew
z3iRlEDk8j#D{Yn=3;oPsBn%d
zD)cq$C1r;pm1+qBjYw;%ep(A8f#|h)i`>unvwNK9*HET`;gm3-CQ;5+*3~I+<*$u5
ztCusm*kVu7?5H9g21gmYi`3omUg6xNQx~FKU-jh|S^WC?62oi`zM+N9Z~r_*D&ofp
z;|Y1`HIB(~4Xw19_=>}NfU~&WZU+AdpOk^sOS#>UAtzw$N9
zR_6JSR3~1ZWo%cd6bi27VxD*4vY`1lOj{9rbl!^*ofDMphWreBK2p6`k?FAgDG3R0
zzkMSYWc1lXepkH>*W8jieBsM~y!b+amTu$q(jT*>Y^q}V!&)mNjqeO>BgG2631n3E
zGH5uic&P|c^M5zl-75KLYya=(?vqX2nP60Lgm`Skudi
z@S-+fcq6zH^~5lh=&Y(`pVnN1UfQQgFdu2Kz$4m2-E##eBVZ`AKg5P>&51~HJZ{@0
zed8?gNPA0Lpjl3wJSi@&(lRBZQn*Y%3|HSx0}W*uQ30HFjd3ZQB#n*hcb~i}R%A*@
zTZYbzuPpt=7d*5T|HYJ7WExeZq@_iz`N>+SRXewDoVO)>#TG&-0A$cA
zftq8+S$b)1V5rX4z9p-M?eC8pVl}Ce3+O)z3PKm2lTJ}BGCe+P%JEr5t0A&lVcs8{
z7$qIuRB=x@-ECKnk}r!r%yU(m9Z7ZEtM+h0BJ3fNN$^b@sKaVgiwZuD%+z
zm&8UZNK#)j@=1X)!e7uD-U2A0CrF*nZ}3jPGv1}r$S=p4t7
zO1Yo5F6BPV(>|Jo$@+nrj3Jb4lBGA24qH_l&54{xbULWGe(%$$#_0Y@lX1@x(U^Jt
zHGn^I2NPvk;`iUWy;8kgO>Yg3L)%GEj3RSdH+$7ymoLs)a8BF1Yk;|kA@-E{yvw2@
zJ*LgPd80MWbl|{6Xdh|eu7~_>w{o+$+p>nSSW8E~03k~6d2#>#Iam!iNb2_RnBye=&LanvKEN
z_Uu`|(_9C_bf?^0rG!UsPX6iXnGbOZE2BZ;ndMSqjVZ&p9{rZmjfqX$wWG=8e`mjy
zXjymAfbptwccpce2$$R)Ilpd=`V)-&`sK?ZDlrTpf#$`07dLl7iG9_MpUrf82~2T;<-u1kHR&N72K_gZ%^bi$
z?jjO=@uvb%n5oLiT)2`O3sF5sF
zkz|nEJR6^8Ny#96ed{dsNL#`*4g;fu`4D@f{OADoIyh(GoY$3=VMcQa$iI31d=T|D
zYqLV*-r@`J<|rxuVoL=_ay;G){e!Niwh`qel`Xw36B!&Ck0ufBJ=OvEX5aq(;ul)@
z)t0GyP!9cSM=U`-prWiCY83}(TUogZycZFLcYmci3nUX2&UC}hQ-a^MZ)=*43^wsL
zr#8sGO}m8S<@v+9YaW)<--SIuq3(;K9#>OO-)Fv0mk6a=AVQRuH>N4I(g1G
z(AW|taD-Pv<#4tOUT~;5Om0`_XFD4b^ayN=SO!LfI{dZm*0}3mEcLV%sKjnuz3O3<
z1co|%#0WY79)b`Th5{cwjB|plPca1}7zC090$CxZj?c3BSx1q{wT=e#oI^6%1HyF2
zh{7f!LS!R>2{H0J4|`BlWH)TTfL<{c0?yg%Qh1Tt&J*AqAv1ZoIXQ$T
zrlx_A=)HFwyL2guq{U;$AniY)QG~2r>oVc3QFl|3f`dZM5$klqG1CpDXc(c$&=9?vchx+jg+_
zS()31UZx@iew@m%b5~dWBuCoMZ|lao2SYA6<2Q`R>3EdwA7JKkAFVgMwv1A^aRZyN@LnuJmoJXLF38@YQ}OTwmlxVf1V
z3LpJ5;wT)MYzpLT50ujzZ*pb)LR7Ng!s1?*DWPM(eEy7#jk}5*7H|~B$()5~XX@^n
zZEfm~fG&v+Ji@o}^jzhMS#bv~v`+Z9~ktY-L$z`0S(W3e~Cm$%t!;JTS1cQ_+Q
zsZRq)c6V2G%7FlebO&nITT`TcBFohXJcJko#unX;)jC(TjEmys^XJTnl*)VR0zt`D
z#hHl~p}e#-*>bIs>0SbO{$6%zYHlX%Wu1ivzj)at(Cl|G$#Hq(T2_y=uu#QSs4UCY
zE%b^jZ*UeHQ1SyJF`<+xgaTpo{M&EPHpJZWGNZ7He-d4|TNuk!ltTrN?(xau3&aT|
zIUcWC);)ago+sY}Eif}zE&By^a>pn4^~yiZ&9jV>cs>l_9DgJt)PYNf=rhj2ih>>`
ztCAcq0b^4Evd|gy)6Hv%8)TapYVhAp*Ey)kQ&dExySsU}-G>+7oXth}&y*n$_^!2m6Xh+DAGXoo`ocY%up#rRw0{^nP{1XGE??=
zPv7_d9{-Nt@ja@ap6Bzq@B6y0^E|Kfy2A|gwHfKS=txLN7uBMuaTk_O~tBG;zk_uC5a^Rz*$9*YihuAk8Q+M@9I4i612f*}Q^)+4g+CmO^bXIm?ejJMDcI?s78ZJ4`n)X?R$O;uj%
zXmhYzY*R~B@tGMN4It8lXEbk-hSc%XymFoW-rH-kfB$}Ez9{deFxGwdcB|YYhE`Ym
z7#hZeh0%s*P>_*o87lKJJ6YxL+q<`+zMe?q-!xkIpaY+_JZHtjM1A?#@#Cq7Jr;YP
z8q@gO+uJL9EQ(4>60{7@o;^#UCZwO7kdkuM$tj8$`rrYZKaq)v>EvVO>$h(=j?Ux5
zyD3^OwIy8t-6zJ#$VdoNB@Q{u@8ADaWovz*K8!V*Y}JQ^_RzJjZflEIMdHGrlw9t~
z4q@K??AbGGvrv+o*KgeL*xIa&kLP$O$`?gVJKmAHwLEXb%gfs)-QC^2YuB!rm>3#=
zO)ae_Sy`6Sqt0a-AxzVQ)w_?o*Z5Q7y>BY=UI>vD6BT_`RJ2b&IW!|TGLpfc`0(LF
zJblrnj>Zduu`Ti9=4M59j1y$FtJ(}tqoeuf
z%hid5xj{do&iV7_uUrv{(P7&|a-Le^bMa+XHi~sEg6;C$u$GqAn5?`Coz97K=hh~Baz>k?v<*%4
z^_kQN9v*97zn)i7QK3*1zwp*=Zb*$l6;jvI+$^|v@5X%7?h#|Er}naD_~q;@nZUHr
zQfSk}zH{d;F0REdGk)V~!itXl1#@$AYfG~^+1cIpvU)?y%gf}{ObiWK34+{*uIZgT
z$szY?b7L)vN5P*M8XMb#Qxu^R@9Vx5W7}Ut%*}HunDglB-`z
zZj9^4P_2Y8RC#Z#;$tkok@^$E!oqZQb!q$~xetjeDSi3;nJcPEgZrRWZ6Mx(gM(vU
zL0$3qL(%f^RI~A~8^#0WU#NVPkUhK0dN4CEzuQ6t#a1v4Uu
z7^pOs#dX{!I@1@%J6hkrS3fEo&Zq1)_v_X0ffKTbF0?;k#F
zFU*GzOi@k8k00M^xTirzD;TXxM!U(+P^(sc{+g4ML{!u0!b;}RyI7ry8{lty_wUif+n$-RCQAK7H{*ETm4&u=x4&1nTppLb(ob_CSm
z_dm9FtM-2WYDjY}PkWbU2bnJl4s>;4nF$nrpZME<{`~p)F`v1)`H|F)Vdn=A
z9`xl~Fz?Zbk@dK9N0meBB0qt~x9Y9;hMBHzOHGZR1jYQwh8B_?9`eTl0!?GoZi&5n
zZy^r~(+kDw3bMElN_i|gN=W3teEBlnK$tHI;g)M6f|L+s{p`hye}Df{oSgXmB+cd4
zt$)LHA$4_iKYsksj^y-9EG{aF)f2Lr+30B;URZHcu0pna`}S*l`w3IdBdN@C*!PI|
z_%m0pUcGT+p($$jxpU_#-@e5Ud0$&3z2N@uS6om~(5qL{Kg03WgH>K@e|{W4d6HSu
zM3|mEVkZ}u$a)2qbY_MsmgyI_Wj9s}#UP4P3VBSGQjkDXQc|*8#e3@4FRl^i_+w9M
z_1ZhTK7MQ%UwBhdL7+&Ky=H
z5RNWNPfvgPwB^f}HDnhW|F>@Q>ZecNt`FUT#hFud3k?qsFD#sAzQ}micdoCm55?$H
zzJ)d+tFX}U!iC2*mZ}$Riw0UBiq)BYR_huV&=rWGAoD9=$F{Dlte`5;(a~96waWC1
z|Ks-d!-o%JV`D=@L$$T)uCA_RekA4^XU=f=sYb`d)X-fIt^NC2j)C{ajT?*%45p$F
z3nYmhh%FqV-rnA{6rcH#pwiMqOd$@Aj`i=KTUeSEmzG+TyZk5}^geLlz{1>|?>&a+
zYo14{D=T@67((hUU%N)NG&4QDlZIl)jva{dZ{NNV^)xgBLTzZO$kn1z`+DqUW8S1`
zoj7qfJ3BisPqdHb&6_t7vjizt(vuTYQ}v1ys3NH{WfUwrL_Z3a^&Zs*=EYo-U(GRl
zygWTm=b0AX82`W_aUm-^J44va(6A8rK;gzuz0@P-h-2(pOUvbr)qhxi&aL2ESzA8(
zgZnL!;~zi%-u>h_s;J!6&toGa_a8rgiE^}e@5larQ>+LlC-!IR_it`iR$~*B-oC!z
z7k{lM6A}_k&CHPI$j$^`SZq%^B*=34XImmZTu-Q{wKbx_nxDpHeR=-ZuV0lOOOKM0
zoT+P8|hhYw3G
zwz3~ObciY-Ju{P)Kp?QQQ#mq+)NRBatf+c7?2NQ@?p#v4T~cDAmZoM~ijr$j_6gPZ
zz)sn@xtv^FSFEgdu(Qj#{~1FWW2K>(XQQK9Q09}6kmzV{zdF_&Bg@r}lHS@%AJtS^
zQgUsu>RU&LgrK0NwsuI1Efd1vTg5FqJ3AWxyz}MKqM~j6{Ru0)2o$8)y?gg2X#W@+
zD=^JN<$eDAd6B88g@pzFdVWuL)lu@b(RnxVc>3?Jyf^N6d3jL=>S!iX=88
zME?EzH(2d6xWJ2=#6Xpqmp7HqxjxbmF+Dwvcnl5>#x9YPk}mvgSGqo;>E(4er0(gn
zXMg_vJ18XukQNaYm7klt9i^uv4EPTn`qa|W($o}kG!qqkb$R*SH@{=)Z{Dme&yU`|
zeH*p2tGnB}(A1$X&#Gu(ety1|dFi)_r4cuG67nhzL%4~%)X>68@SxS@%jGE1hYpou
zVM2}_eCwuQ6Eqe8eOTk6Z
zpNPFIEG(?4sWHyh^Sj;A&~Oo}$UudF!ue@WlxgbhjIOb?zkInyz0{fi=hzs-?jy=X
zqOX3=@87@IR#%xsPFDZp@RpU6JF2Z6g~SJ7iE_n2RoB;d8y|J_=+X1yrRC-B{RLJE
z3JPZM?IRYOCmBS$vBmfzsx3Iyzdz+CB_)NJK0x8l=$VwUYqON(L^K^aas+!p
zh4~yb5c!^1t;glix+VZ
z9;ETNx3$S0JeUGh{N%}xnQw3K=f^5OOxFqcmtRnS%C3*Hi
zgh!7?oaK#;jp^y>u?*4L{|8PS2#4NHOiUP@K7B7D;y0e}`SSqNANTGt(X=aq)P!l8
z6q<#g?8_l4}l#DToCMcMi(X6MiIu&_vbF3%lw
z=p%$_GKH}6@>T+G*)-h)@p2lhLJmU;ppzi0vt3zVS)ip*MbMqr!DWD;G=Yi8x14ZLNr`R_Wk=HL?FzUE_dvD@NKv8~;j2Ky1RNJ+4
zp{%m9#(Kx61YJ}$lobPcIV)Od-
z>o|3$it;=!Gc)T31_o+s(o0HicV%cwi>nmOt<6T8dvmL6l*dzmTyDthTunn}wa-&vcCtyN<`xv$GbbP62)V
znwfd|;>F}B_
z@$7{Qd~$&V8VVk!oP5Q!%=`E61CLW&V$wRnrUm?&eIn`S_;^lQT4#TMnahv&XYwzk
z3yTnFjEsyTBO_5jRgh`{X(cSGJdq?MjdnUIIya+0=jP@HP&1!Ca|RUy^#8E;I;U1w
z?7oxtb+QTy3b6dOBJUTZEL*5cDP=;lym=nH#iPp0D%HX^*BoS_3Ki~
z00K4uS#!c|n`C~zI)aJ{3Wwe1NCeqtFM}-9Sk^Z+)#=C|Ja|w@D7a%hQ~2}OuWxbU
z27R_RP*orZAjv;@@&t#Mbw`y1ad3To9ibi_O*nb-B$*$ov*qZ~Q>Q+L?>z9%5HGuP
z=T2^J?xpE3)Qn02HS%tAmdF>EF6|4baYKy*-cF~uE`J^bdG+7#nk{fZ6ec;R@5=U*
z5zG77^TFov^Y-==2u+Y_BI
zTDUXg`SXYt+an%}40lwgw3tFNGcsPG#0kWd0c3!srU~Df?B1I&IP5Hs6KY;@a}qg&
z_?@p}pGaH
zJ6^m9W$A#)sZ)oQm78s)C+E21zO&}M{HZ(cqhQ^by@aNY%D9oWz7rm@$tLuy)?iei_}*ggR?xM)#Z~t!EO5p+NB7VTl6=B7(%p$Eq|0uM5Q3+t}D#
zy~;>mKKC*?IT;l{wxwQnr#@YY-@n%kVVW%W_G=kF&=IUP;#-UaUnwdo^7HeXW7+Me
zN{nssX%!N8WOjzWqNy2(&3gFobaiVOiaHX?kRy^`&dF5v?l7Y-AVkKez4Z)xj(cT2<^UCLsBeZy=Sx)Q)Z^7@^W$x
z-LwMp5ET_g@zeau;KCoT0vv&U%y_HTPC(3
z(5*O;c$Sct(;fXAR0U!j9UVo*#Dd68RrO_LWS}JFDKw~NcICZzk&>MJ?&_-gv19MC
z`&hZO5dxK|ZW%kWS{`#Pk8iu3l9CcotDc0tchDDEcX#&*V=79fkbu_~+S=N~3MUw(
zrKL^t_ZevnIF^9vKiu<^lgRm#Q(&4;_3YV!#>U3ZPJ+JeyO+NwCK8g80^YkRKUPnF
z=Oh((rp$u9agbX>5R_<}7x{2Y%gC(p`G1tCrx<1A70Ca4mxFhr@1&sk&x&Tr&2bl=
zJGWIHxKy{$SxlWHs4z-BT;cX(^o@;;wX)I{lt5)19ww!}M%vcizVPP{>gplQ-;8(O
zwj1zma^%g=a`I?Y&-Z&R@b>=jR&4SKB@J*CW&b@t+GNuhx#I&!ziwi*sb$FR?ABh^nL;X4oj-c^LNgzqX6?;lD-Lbe}K9f=t(Zj_8tlxBrM{d
zJu>x+znq@NjjY92-{YrSC{5;C*63hhOa3|==XY!@wdC>t1+lhda|V0g9&2rT8ghEM
zfO7)Of4P?_&JfSa|Iu959lm=~pma67_mFmsmvL>3w`sn_vdUkti)lR6wP^!ZTXaqH
zOpm;`mx9eFE4*1|gUgxN8Y;NC6#Nni3iQG>*x$G_u)+EI^|0{pJ4-X>fWNUVkSIJ5
z8VwN~4ZkDOop#U*2ka08xc6_;b_lGz%l)KzZYqQnkKYpA#efrn$-#Es}5T?QS
zwgRvP=`XI4Up?}A
z;!KdYhmlq20^G6CB@~^-isOYGDVQ-s%xL8!)Id%5z{=Iv(o){ev5rN8hXXa~ps=V9EKIJP0Vw<;P
zHx2wHPscobC@w7QYeOF#iUbHkQQOija2CmW8^*qQb52douW9sKwT}-LC>6ZG=hoKv
zR=CmD0heRj+JO$SraBywgNKL5@sSk4W#*CHjLZ+WCBAkV&;J$sp!oT3kqJ_0`TH=t
zDSH$384UBBIkXIyIpb_s)XAhsQg;-&vb~|_`LgimNjR;>z?DZ$MLp^fb9KLwX@;w
z^C3+4gA&eJxfh(e=XWiv(A47Kux^ZTTF`6GkwSnK7XX^av9YqXdbt50Q6k6A#ZPssAPB?treZn}h=%0sw
zq4QnV+3=Cp5tX1VGm1^06MVF*4O_>gHa?Bj-+R2t@qP6&P3iZMBnhh;s_7K1J61cW
zx@RLcDaa36@vKTPR@UTP?ucniwS3_t6ZE02Z9QG1Kyo+N`w;3aj`>g3
zTLYh@`{R68WC~RLysKH2I^$B`4c-#}U?CB)`@~lMxt}>fJILp2*K^jVRX$p)_92a`
z64#>z`**y*&-z$!xr6NO1N_BApKDv?mIH6k*7$N*pNFvAx=+po)5K9nqU
zhlMvogk_mRxO3^Y?v5?w=d3Ufk4J9x3W<%6
zU;6tOSqJ(PN>JN}58mi80FV(bO?}0$Z{KJsYU^f)pds(8J9>|20K%1lfu#Kvc^-XZ
z;~uaY&~hEYhjDS=pf>>YN8bZp0AZ=Bs!}Bii-~PBJBWJ(3d1vJAOVHwS*?=3_f;jD
zn3%lvSi09S{`}c9MB?Y(UZ|LYVq!1ezU4_`Ic;LXKtoedT$~50jR1#c(eV2oZ6-M1
z+ae~dF1#;fYUxBr8=F0PH=%ezi9t!Je-5empku!zO1NCo-MV3@ROm20d;Yvo4>;4n
zAUZDYl$X~Ao)e2fEhZ$LI5jjx@33&F4TX
zA#>9BqdtN)Kt@m_mQ8`9{fEs(|9br#TMPg3!tdQ#UQ0G%E~z3XbHt`Y;0V0
zbVRnsUM(yvgp9l!;@sTagpL(0WbfBb>fz=V$)muc)df*Z1^lDxic+oL0D~wHAyA4;@;XpEo>ps{O-!%7(2h5KR7Z4EGfE`F^vTRF
zd#HGqJ|yoK6}@@u*3QWkZZ0l_OS(JtE$AhHkASq_D?Q?qlfR;oA;??e`Vm71Cpf{zIo$@Oc^6qS5_8W
z;$op`d3iY)N+&;Al=Q)aw{G9=>+PL7o(^t;uLLavfd(WG)=8JD9i5pG7Zck6%%o=K
zTNh*djC4_3>yJ2n`0)E1*Zj0JlJ2fgpOzLE<7i(S`uft&tXGcbMuggm8dX
zZt1me{xy`vUr=|+4JZ6?2Ji}oZ0+yg*NTJpqMX}Zyoj8$BTN(hni;x~^BKaQ2L@KA
z24u>_B_vd?^a>;LmHD<1myo^V;^UdG$g+fu&CWVGI2_PRm-;t7HFfkZ*ORogIY4ia
zMiMb@F!s%c?m{i$)zwXbD`
z<>f0#qb4G^8Y6c>NrpxMEqn(J1^SX9OuV(D^MD4Ad
zF@u~T&+`HL6p|;T6H+c~=_p>MMaW*te6yh5Bgf5&1pEJv-TkAPSoQMf&s%$XZv5(f
zipEuE6V_P+Std7k0{QgXwQESMUR#?R04tCI;3beBQNr7dLcT@s2E2H5bPs_BK
z0V(>5rDb$#>g>>vq{reE@E@n?uZamBWw*}mZfOYD{rzr`&X6b}w-XWr2%8+M)hG$q
z`U`5XW38rT4<0~Z5EK-|AKmZ^3DeDuf`S6UD)!m(=HQ6OiA7%)-Gb6*&wk-F)pj*|op_d$voxOS1C9W-
z>2pVi6E^X}g$tvNcOhs748KfIA4c~YmV?Xo_HOedT7Ie|wVa%sc$!GAgU3wiadw7k
z$SKs&Jxdn0+3`nHrjJw0)x;$mab
z<-c^`z;md|iYXTi42DKWWiEGTK?ReOmGv!-fKppz*N#7;4-Cn`RJ0Edo&HFTALR)q
zfV*4=uha(8LbOH`Q_IjqOKW=x(3XOjfdt7&OZ)WY%b9bbUS4hx%+YFhlv{(sghB*o
zK}$zR{h~h(_M$HvqBfztcA3OHF+9&_{b1)ezZS5u~=13Q?C@9Lyqxe;M
zuCS;Puz{$B&F|l{b8u|$@8F<{j0~$HgVd>0^^qGDsR?KM{bv0C^*i3i*Fjj*B%69_
z@(UE;qOI+*W5-s52hj)u0zpk0E~nZd6aLA1LS(`1Eiw+&b}-^?jrOfpJ=w@dj?~eP
zZ(P<@J`&!F`KiuPyTvq@dC|4zz8msXr`L6ZxuiRROd-h}^`M~NDb3s*C
z_wv7AA0giYZCEVH$;dp<&2{_t>s0mBu+vSay!NvNmVT%ZC`uvnnf1fYs2ETZ-`N!z
z8ySHjd1qV*T@myvpd+TEvx<|HsAmW65E9b#V81asPe2i?B`surL}g{3I|MDQhQPo;
zUtbbvc@v)yhMC#fx6$+-QbwR6i)ZyrBCu2qd+X}!?CkbuOcz;stR)s|F15^kWZmk`RR>+&u+=R;
z{Tx*0XN**E6$r&XeewiUQONr(T_f69VWFY9`S~H@Mz*SZ4;*l~a%B|>*1Jh1Kzx%y
zQ(^U?gtBf<517_Bw3iVCQnO4ZpXx;qF@CnRuJ*gPx%xXF77+l4jXyu=gVYm`cvYmN
zeA`%C0tEFZ;ut7y%7GSP
z42+k2Z|LDRTWMIW(8fV47tL&1whHLENHxO6>FGcQ!$>XX&CR9g4o|QYqJ`U)sqMGl
ziml?=GdeW_dI*u~+`D#(oh!+!sNh<^LewS;RzS!={&;xAlLCsXxHRDeOgmxS!-8>MBZp!_-A1OuZwlqM1@3k
zRSV0=%X5S#1|5=%n;Q<3en+|37Bo-NRf*0wZz?+vk;EMn6Bgz?c%>Kp6@Zv(?+v=6
z$8~f_R;1`o5D8Qw+Kyj~ZJL_K&143`c&sK3BkMmlM>(+cFx1y9f)XhVgiTOb7OTUmns}7
zNJPl&tFsAcSmDUy`~fTzD7HNgJmkWE(SfiOxUWjgyHKepE6bMI3D1Dn{$k-(M!#SV
z;4oyl!ieoY{1GplS#<);X?yqV*&`%G^}QR|a(<-YLyFR{mR!<{+}zN;2eIU6*Z_A>
zs6jGB->E!os;_W&PM1!H`5+l)D6i8PJ9=
z9lhVNC$9z^^{(T+fOil-HPt|i1U}l6-_c%cOOz>aeg?NiUY=gM0oovlZB7QNxMQbG
zOdiC?YZJ!s6DXY|^hjkF%AENX6waPL&92pj;)XMC@YJNB?V;E?ggf}z>g`6;M8DPH
z;bHuX&c43WD@;IMfNn^h!r9n${bYGVEv+E^oH{Y-#IH_QYPY008St=E0!i)ABuon|
zN!OL2h5=@QW8l5V#`IGZDGzEYm&e?{ZvN90l9`|t!~<7Tiawg}i})!Z4FfiO;9;*-
zSx
z*4Eq86W{|ISz=sVgQcYA=l4~?iZBMcxV*Z5pGCeAB?C{c)+A-q$i~hNq`~!g{of}G
zaGHtXVgBGJHUIV=e~60N2t%3pOS9J^Dk{5waYZz|Z)rh0jQp##l$1Mq52y{)PYQsg
zbOeF*VLqIL4FX3DpqNnib9x#@3W!)kC^pQ!3y}a#jQZs=oPI+?mQei#VuaUt;f$p5
zhgz+C=dbXQBYfdW=r^F}1))bHt5x8*&BW3cKc`
zc(uXlFX7?LKX4C
zCo7eAx^i%FF8n+J7pGVA)lhZ1-@^B+mjCeTO!<+_ymmf%*pEHjUM
z0eIH#l%EOG7mwY!qge33`%8%T7Vpi@kNMF^!j;a$WHCCLgh2Il!F~I-V0;tc7FMOA
zr^icQNJvcNr%})8+4g3lD>LlPs7w*s10;LMvENsU5me7!5_%a(;BGE1x~osIkzhDK
zP2ly2iiltpB1i3wrb*NjiqT0>J^*7(fn^OM;F4Dpo-&7l6vmAt^mY9y>EQx#WA)R9
zH%}S1$WNa*7IM>fTB>A5_?@@!Gz+86eEuBG3I&{I?{DWk13C{$48PepZ6u`|W?=8?
zx&(ja?%lgBlaQweN}YCRp&-R2CYF0FU5ivFKk1ug*Zv40Xlrdv#>QpyiN?PzSz!UR
z9`+T^2kM~4I7#{mlBD7|nLrriKi&y%cis)`b0Y)%)zb?G97W;rc!HEPgI6skq<11A
zssPv;+&>IEs6{qK%tbVCVT6?oMGq`V?&_U?zjj76uo%B0UVxVu0bR{zrmeki&mNoZ
zEZy3;2kPlG`vi=>Mt>@>s)I&)3(ZJ`fSs)^ygb$e!_H+{Sy{*9#Yt=S#OQ4A(#=Vh
z-fL1YxcO61Q-fFab#?7{!*tM0O^qMeO-_#eRdjqjp2!i_S9>d4+rMZr!+a-2?pMVH
z>Rn%753)}0v3-JPzm^AN2q)^DHZsaEdJ2u;fBlh?-Dn>|r#W`)7d8hAN&1fAvU%B!OuD7XY
z@^~6FpC3G^uJ%EdOpvk}YJq{$3t@mvJMvulvy;`k{z_=G+3a{*0>yp_iP;S%P6p$r
zy|A3Vn=>;v=MAY-P*kk-r8|A1Mimk=iUds5LY0b2O5oKnBO|8~C=kyl;9rd7roM7>
z+4G2>Dh&gJsEA0#?P;#Q3v+I7(aRg&=z(GYtA^Lo46oo3Vc<)6*zm)Sln}BUFJGRV
zo~Bgs9oh|L2fk@AUsQ|eB1lu<(dc=;e!bIfb-U?oW(J|5Z(!i^JTE}P#^N;SA5OU-
zi!PAohY$K{YBk@!ZF6aS7ZL=XIZ9;{dlN!S(?*8ng{{iTa%WT%(9@OWduXS?^fNPa
ztt_Hx6dw=T3ZOT18U`mFzwXJKd3Nd}D0ImCoE`_TJTo)FVeAxYp_145W3XI`gLzT4xZNAH>9v-YbUl47BRug9ng$LZ>X6M(5MaEyVCOmevgZ
zig@reEgV>2B#UbAARcsVp~eLG`-@vteeLWV`t|EsX66m}Rq=#uTIfaQ<>kT7&cKl_
zTWEh5EOyjcKF|GrZ0z$FFQ#W^ga|azq6WLE0#LKAqX5GY06QrNgJQ}{bZT!2Ce&!@
zd!s881Rw!u~!juLJ=A3Qkm2;KYbo-dIQxm$&g>~bKlM%e|`x|`^s
zqPMp0D#^)!^qa5`(5G(L8B9C)40I`&3qwMvQ}j7}-jYT&p)=*H%FM&`X<#5sfhs%$
z7>SMU;)i5~S~|LpPw+)Dh4f%iH+bhVaEM;L`r}7VoraA$wQZ3W7Z(T4CA?IKmQ=MS
zl4V6_V%HL(>^q5
zlt5FmvZ%%!M;NGLcg6A?V4LtFk{KU6Mkmx&_@>qjET4;$Q+2wr7Y6`m4hN;SL2w6%|`u{U)YqvLK)6Q#U(_=ZvEK}l*AYHtB+Bp9=cTL`PAt>GWqO0R@`<<{Jw
zT?+P3>#`S^xqCMFXeMH2V`7J^ifle9<3WgCEMepy=@Tx>NJ$wQ8t%AmDavPh@??~b
zptYH$Id_asJ*V9HLU=Z1Wr5_KwY8}!NTE0t4mjcr5BplpCBtQtla-}9#xG0L-rioj
zVRrmDJq?MU3-a&Jc{ig$KtxJ5`rm&Qc=X{V2&tQdE_!-}h647{QpsI-*%6m=8h^Mx
zAZs4i)m3B0={c9nL9#67`9z}Njc~Ynh%i+x-`=m5){FJZ$FgEVMAK-=#?=TES9nf`
zlKH8co2Rtf(T6zOluq2Nc=~j|-r>4$eu0x7OtIarmeYKFtp>tuSQ$OeJYvDIXW7|u
z(H=IHH>$UJ1~im4n?H
zW#a<`WMCpoka|~NUrA|cj)AahI?l8d9Yj~^8+4{x>~_a$!$5SARrrnYDW``&X$i}o
z$20~ss|2J8SYV_|{oX*7f)y5K2AnYAaFGKCSi@jH^y2JWJnoj31zczMAvrNRIyx~i
z>|g`ZMYK?Ci}KA0LR3?1ObqPe%3iC*etx9RI=LmpbyaddzqaqLD~xY5;H`RJ{Nlx3
zRU!&%tq%1F8yq9xcEEQl^P}3iL&|*l<>hDfjuO7kOisGPijPtbv*x~iInSOQdt&6L
z4iBH)p+nGf87}zweTRBQOW_|A(}fHR64ck_DxXgG>}
zBuOYJJABZ3%AZpI-5TI2eAvMSZlE^ETpDHc@YaNe0>8nkfB+oQ8={arl8hyR+yzmh
z-~o;!tk!53MDCJ7(tj8qzt6MrRRPJ6I~!d?clRGCOmhj4;sE@iuUR=dqGzvG2YB;p
zU<+F`vnNw5J^KrNh+AdxO8Z$hljQY7Z8uz
zPiB=5>sw`I<%<{X-QAGptkmLE^{nhkPmw)W1}3QV-cZ!*wGyRB2$_p~B>9D$Mbz^H
zg&pbgFzop&e=f$HJ*?+yKaiuJ5?(iKnH5fKM(IbsgSFjV`7xxR($XayG|k(Qp;G&s
zC!ybsk3&(S^wC5wCrp<
z0G0m+gn*5JC=9-Eh?!+&WI)mcZ6fEDWDBpZ-pbI52;BD(&J(k9=b-jcPXY;5fUP`v
z($L!r3ed~fDtWLx;G$g;iKXI?5|Jad}q&?w5PuK-RL}&e6Vh)PzaCU%k=ZQ
zH4#uyKoSXgWbL)kVle|EdYXAmhcMy|u>>gMV(UY5n9qT}pz9jJ4(iqddoO5UMn=Zw
z`a%alXWf)MgAEn~U-&3Gx&ZDftQ=S@*CYB9C))9QxYvQ{E`p?(=F|2FILdv&a2ZZ0
z+^3zLofAsLv(OE}({Sjd@N;k#2}Gy}hwCTF0dri%ED?lYWLC-8777(477R-$YJP)h
zYz5F85iZMhwUcst5aNf+T5xI2IYE~FA|k(FJjcdXULQSbZ=VYgf7%q)3{wCIR0Z^|
zz`CGpEBI`B4)YQ?sz;Hmkh6h+g2GPqKl2){4T2Yq4=f655g~(&vJ$`pX*{-N43u95
zj1uvRxZ3K9eHct*p!uJ=w~WTio7>ajQBfz49|sR$*VKi|%@oQd=fum(O3oPS=T`$A
z#Igrx#Zudy3K$atw|kFbv$F
zig;r<2_vgim$0>dlJI>20uv5X%uFL)LGZ)FUuSOaMZINO|$Ir8ozUA6+?@|qzVe<5TH|(-K7;3SHZv$dkNPu^??krJy(VmfSVtn
zK>cIzQ&mVi;9n3Q+A6Jh8R+bxnBjzue@F@Vn3a&g$-@K-=&QLoXGqE0JnnmClBU4b
zuVwQ%Ec6`WngrV)@8baUaq2jr
z8fYxj1U-hV85Y)p_5!vh_qc-vI@C35%K#3RJ71gvQ8?*=yr00q>I~7g#ht^h%6msy??|C&e{GX*c
zYI+y=IhGeAMJ(5(R!;%4KGqZf`J~sQJIG<1-7^VpMYg7v0+@iMOb8CRf6BmM;!H#;
zxukn!N1anb1$qXMjY0Vfc0_LRYJ`23Jahzswa(G
zdX;G1blT(O|ft&ILO-)Vo32)pe0>nzxL#Vi~FH7l+{Q9K`
z86~ar-4!Yq1%J*THDUZk7CsvXg@x^rrzNX$dL~iMV94}5q9Q05*fe@|;METR8B_Dx
z_j?2cYT*xDoEjL4z%VX)<$Q(?6)rclxIv$Lu8SfN)BY!lrH%BLep
zCo!8Q{N7d?7`5hpZDu>V9+;=C#(*QZ^WMEb0LyBNW--D+OGC2^`5fp4Clf~{aPUc&
zy(l}qHA7%SBON@4m$4?uZzv#Syl~>%SX*0WhE`Q=qDhJ2-;Q)1Mdsfu@b)%Xzc^Pq
z)0wVedXDN;?%?YkY+9g|XYx#?(00O{6V3(<1-Vp1A*gG7a*hc|KRB$Op33!PG;0j;
zttseFVz>lg2!KZIH@Ji8M8aW@5~Mf82qwztOM)za!gW|yRtDFsNE{a*A5+mjc^>h@
zhrgjCDJ|U%Qh}L9adD<>D&itqCl5QuQGZV+O7DftjBJMgA8>b0F`4<2l@)Tk@2zNA
zE{qS*^Vpy>hbc4kAASRh0gqZ9EPM5eg?+cco;~<#R7g~JA-vF(qXJbxjdh{r71ozP
z>6p^}_3Mghvl&L%DsKJ8#;gdc
zuzDCw0n(Bn;yiafD$Q`tEjt1vV%x
zE{2khdR!~GinaoVks|Ac0ljI}Hv1V0(a+!!fIk3YtP&ywOl#l1QdA$11%x
z7ycu4ymF&2?<{IJAjq9RV__K?*7$y4B&cnD-QDWx2Am9y7+Qc40HD@a^Knbkqc0>Bp%e^dNzz0FswhibwAb@{5R#xfYByImlJYf8d
zxO*3scM`MjpTB&8{^tbSu%qvPhH@l`S@e>R9lHW+2Sl?(#akk5#n=MKDd-|1QD8q+
zRY@XQgN2JipEInor*=d|M)Ih5qgm>C`?g$aN9n6q{*%rkanFz`lJgLg^#W?153qBn
zqj&GN1BpXCk#p=P$<*$wh46cC31iy5z#@eu|9OWuhEbrwf$$3E7(r*)6BieU$uw*Z
zDh{-HAPqFi!NI$GdXPduP|$4Ri$Z=^iE4rb2eOC!5t!*fWpo-N{Vlc_Qi9Adg^7`)
znJ)lJk=qm%A9al9o4vk*b`9RrePi`?cJ@!?7A*2Z(X;sEI$LQ7rv@e_H_>B9l4Xz3
zN;j~aegcUdaS5pi)_>OPPUPePKo$QM7g=ZmX<3EdB~T;)G;BK_wvh;RV7$S*N)A!e
z0{gBE^j3h_AL!=*Zu1`Tbi=-bz?q1g($KI0I00ucNb=wNGm`wLhf07;;ihFsjmbr)l*SY=F`
z_@H%16#$dbt?ADP`&DSa)5LV(NDz!W(P6tMNT9)kT!#e#jY>dpzl7T=Cq5k8JHpCx9##ar;7n
zey)HTP8qvMWI_(&kEEiz<
zv)o(=JC-LI0*vg5ivYo32D&FtHVGH!_6~{k*^M*U28>+AG
ze)u17#BSfbNfNX_ykX#ic3xwRZmCTK^fCLcw~GW
zWS~$3du#@K1ZDfvf*Do&wO7{mQW&Pg>*3&`0wby29&6q){AZd2!kSb{{mc_X
zrlO|-BOy*^8&;1Og|5S1sAJG@wt4dU~`@7IFyJU7_=PS9enR
z+Y+-cBK{CVp!m?0AT-IhgJA)}!#75CiPK3kg*X!g)_gS=xX3Qf|S
z9!wcTMRlO*eddMP-mgMJd-tM@JjK7U_=llnL={frcIRwJk=O$7M2=pfG}?xsY-mq{
zyM$hTE2*jmk?-nY)dmbM;8Q!jpU+NkYyv5t(9uD%#jO-jA3*yTmX=`ZKFF|Tzy^yW
z%nl91&X8K+vQSS~O*s5ls=)E}m5Yl5AafOPnlXI1Zw)Q2=i*BEPwAP)Rn(mAs
zE8;Q^nk;m<0%EEB+Hc;x`L)_-@Y5$(WOc}TCv3TY{m5#o)kz
z!UNI*P6ACJ!UKIEFls&(Z=Mep4cFj^}r;C6fvRrqMJ;8*p
zvd<wM-R0XBW
zbk#RN4^~T5QTPqhU(YWsso*{rkVO9TjJ4&|aVy$umIcp%`H_8qy>1WJQkLC|v%y%F
zT7C9zmylFGYhDst)<0J
zV`OS7kIPp;9nmiYnky+TMh_nAghG$8CX5XBBhFx+JABv+w~5H}TtkBeR1jGwM0Mct
zEFs@RTRQ}fZMdA!Peq5vw;r~O>96Ii`so32;?7P?A?W1r@7|pX%|J0lKi%L03i}U%
zsqcss0&#SB*zfL&+7h)lFF3%$phq+sB78bI&o8aaI`vs@R^Y?hmHFJ6*!o!D%ZD2$6Dh6lH
zjE|3_VjYp6%z63p1GEC1T<+U=9r5#J1_0(&)TGl2amS!D&iwjiEu9EEoZ`)iPGnf1
z%b-JVp#ejVf#(S2YUgiv7`RR*%d2H}L9U0$kdt!>lc`{r|ILWwavaP}qWuFC#J0>P
zWgQ5`39D>n#m^Tt?kw+YAoAa36KKywgoc9isS?Hj+qSLExMrcND-i*P-3D}nNI`CY
z5()$&b*O+PL38tf5bp$gTglfqt`agbJ#aO`Nd;pFE*KyQ@!J)0e-@=2@+-$Yg?K!g
zWV~Elh_DqXGjQU60BuCd0QSJ-BsLd)!*~mCzvdc7ZaH{s*4EaVnwm0&pQWc8nw#I%
z@>8RaA~qxKfx#r|{Y1IwKrD|oG7k57z~cu&5Zd+MZNX@JJ%FnHRMFZ)9mFwAPE6cI
zAXr|#%FD?qf2B7U1Djw*$S^P%12MqVP*9DrPGn*jL!~O*pgGAqb3b<=Ni&5tA4&Zi
zpd-Gmpf2`(Yb)j-UccT>X%8hVHUaLc*zQ2c!D~Xfd@>g=1nmRE0QfJ!H5z+7t^$wM
zjh~u8kY~FCGZ8`)GUQwhlfA%@@*^Bz8}n{vLwi_ike*Ospvn|l&@m=oonYfi=tj}=
z-H%2NdC<1R3I`|}b-0-W0LH*xBu+#^g4v=Bmlz1>(Jr?Df$+0yAOet#(NDpxKE4KD
zkdpTb3Dx$U1$vyGK8&;oLI?Y~_0AP!61;W6B?C>n9i#6SRF5brDq?rl)zu*ztiqdS
zFC}^JTmC6{@i3!`u)=E%g>I`z$B(zd(uG$<*A4Q}=F*@K9KD$1-i6^Jw0;=
zfi69(tb{{XFIhgO-A?=H(fcj7_HO&x=;W#L#4)gsi%Ie_Gs%Tz!!uOV<3-Qn`XSIl
zIhdOJ5A8cj@1LN1r1L$JwTQ?SK*+f;vRoEb2?I9JpZ4v6hF7mJg*fk~9N4Q7mwEiA6CyV>O9c!_K-@;&Y_@)^C$zS_92gu7R5Jh0kKB~wpyPuZ#H{Dr
zS1ai0wSjg=Tb%Xr*@CJJ&Jz!!1#(A8K>@!SbSp|dpZU0VL}?JWO!UfVxVP?89>Y~O
zsGZ8|>X}_&sJJ}q11>Zo{lHE`5po25Ik30Y#V<~1jl#7x+(&YH6u~)ELDPOPwS)Q|
zBGm)*4a{)(8MCv@-J|r(%)mO|O0FhBZ-j~pV=VY`*Cd%hG7J=;uuEVIAc*kEk(A6%
zO>N%}g)EP69vWdh--O9rB-V;IZy*I=5Z3JsY{3Z9np5Y|!7?x~K;H^#wVxF9Ja{l1
z^#RRbp~mGQ&T35DGWYjj(@YzE4RVJpv=(e
z0Xo6SZr8J&gmLA~&fU9VOavQBmlYOBPwmz%n1Lw5y^7gUijd
z+AM~`572+dMt}VD3Agw7siNIL?K};2a(rP0?ddeT$=TVE&`?yE>a9%==w-kaFfmqD
zZEFlJF4Bu{DL(7ZPd$K%yu5tZ6gXA|8Uh6;)eey9pP0~QD+z3|t#X?u<03Ebfw~P9
zwR2}9Y;cHjds!|=k|C3=ow{mikOi6m+u?U-yN&w>#xaG0lZ2Fc4$dcBHUO>ON&r*K
z21&#H%s)FjISc+!s?ft^i&Tt3Ma2}R_<(2*(lO?M8q9UpVV|rRz^MdkV@FN|84LsV
zX_hDy`0GDb4m-gmWw{$JH3%P2>e8ZU%t$p^%gx!tZfoT905q@7d{d$e!aBj6?)%CB
zon)Be5N|SbA8|HVSy)u>;>4Mon&Kv}6HknAtp$JzsSFY${BS6I@ahwkd584b{|3I>
z2gYG}P{c*Li60+1?2PFiP%ytrU~2w!-`B4X#<$|K5zVQTxVVpiCvxY)N%j5KhsudWdpCgyE0$Bv9HFg?-ZnQu@B$Tc$Vc7+q;+YU~q#q`scpJ?8%E7
z8XD+K`VB%7fA?a3R&RjP=m|_F@X4WGg5;pfsE?^_M=EC(T>gT(Nt$}n`32j&TcA#>
zs;@ut!@I}e19%Z!|H7JeNKf`=&*#r~0A8?&fd@<`mC}c(I54*^mCQxVN$p6
zpTQM4ZL-}>8g64C`iN~!ra&aY^oH);+3j~#(%EnR^8i4a-wJQBK15D@qOL(+MGW4*U=
z+$Nh4A`eNFN?J~X$Ad^JX{l65rBYGavS+f=QWR;P)1WC0GwHOnwb4?kbdt(3-p~KM
z*Y#fKde1o>p65UA-+h1AZP~BiiJ7OzYR9(z{OP*9R`aj7*GwXsP;W>YDI1i!L~-Lj
z`|d}UkQnjv`yB^>8?UWn)!)hN@-easvC9S>0hWr7i=&>ai&{`t1^W6H1YFm)Y3r$J
z<>+eb(7k>C!GrReZ9L*YIaKK2lZuzWx3;RQsl8x$=i}TJsVe0un!kyWng*8ngMk!m
z%5|Mamo=&lwG0(TX2rE{@q-kHyjD1iomAk%%Y&{T+0RG^X)T)>ncv8VMQkfAVMw-~dCD0DItZ~;9w(2F
zA8I6g$;|t>B@X80ITELGDOY;Fid-}(7T*|Z%GY#gmIrOH;g}6ARBB+CZZpl}B&PP|
z2NLJ`<}Qys_?-KW^6nbFq1oGhZe)5~z#72MH3|PEeJYb7D5~Ch-n@OlPe4LlRNUOz
zq9{xZjY6KpZ*GrjRNhIltiK$&;5Zg-xBR
z(;P*0+`aoL7NnOoH3hsN;xm?6y|O%(FJ0=i#0eaU-yXh^QH0chO1o*Yr4Gs6uVf|8Qn5?U(cL<;^jI+a{fB*3Zy&N}3iJi4&
zT}M`|Q8E2nDrMaG@i8Yf%AX;m4G8b8qwIK8-RBH_IU^w6tBXZNX!zRyxiN@L5SvI*
zhWY2Rx;tsv`wHJCUmFoUscy4z`Lo>D1_>2WDsFk6c=OAqsFb#t{Jdj;Kd`c&tVrIp
zKLy|COn)mY)G>xn6*|AWDd9~JTWf0mqZz1p+Xnm*{!Avy7t})J4K2+>PtO=O{_uU%
zWCiiipa4Z9s8^mo>3mboK|#YeTK0`#U`%w4eweAT>xZJaXRB+TXQH+zjvLuF>A^}q
zf4~0byF#X3^?g{)n&=6}&%fhclrE00TSnC>tr9JX{4T9mVP$Nttvn?Dm19wQocR)A
z0@~#Xab+8u@6@C?wjj31tG6Xy9Lw<+bvXA|X(74jpBGDRV`
z2!q`=Sv{e`ztD4#?+(SBS0&Mj&SsYRLfRmpS7~Xfpa{7VX5g_BF|B>rFR5ERAXmTi
zNX&if-5JDC&fLo@jFkqel-UwBCwlsk{>6Zb?cigq&f-;DZ`H
zqkAzmmB~J&t*s6BjU>M`^*qp>&kp~51?{9tMxuS;Z67$NXs!^Rlo@Wv>VW&G7873o
zi6u79yu-9$wS~GNG12n!BP{AP97L{a-uC9JV`LCor7w&l7%ngn?=9n(+3U}+q(!?T
z->BoI{%iViWp#Bs#F7hB=C2sU0AzE
zpavaL!?$&2-dY>Sx+HifY72x51dd24Ww+1_?ttrsN#MOx#!7xmf
zUI0ryd)8kj&rw&Sssd~R%@y(BqLph3KUz%8kb7~
zL-o>k?641MklT5K;EN>06azPaHj(1Q*L408B?~p%hDME*25mbj!Y57C4t|D3CV9*HDxvUj!-nNu)FCY;^S_1oRdC93Uk6`qEJW}OM7tkGh{)SRQ%`=;W5BU{O0
z!fpUAxkG;!OpZ?6@gOH^iYPbmAg~?H-IbMZm916Hk7LPz|NKx_*9=@8a}u675fUv^$a&l-3eeS-Wk;nN0ui|!LB3_oyG2q5WIhw8S^$Z(Fi74Knz
zfe|F)wRcdt=FaWgZ-KS&o#=QrcC-!4^1#;(?;H8NJh(^lvEu;?7BKouE}t7Oa|6A*h1o~IdeJ=*8wnpp
z(SoW3Bm{*#LOjizU*(0rdML;7Fhf$xuUL{X_}Uj~w^j1#DlQYGWZwa8a?nIebO-mU
ztR(3ZX{^H6HTL_UwBCODL`j5ai5ue%qY^dkC3bjGc+!7yzEV?y(M9qu^Q#!3?Dq*T
z9L@HftRa2vg}f>uKa`sak@e|YnFc1WTPc2Yb+DS-4
zfX;z!g=V@jk@u4->|wErYlhGU(I4PDnM0E&^Lx?&T=}BB@BXKsfVM{`5iw~U+sFB}
z^=*Bc5^4bni9kPeqtSJ8{7`<4OjIsZ6^06o)$sSZnz$}WLoRR&QMx)>d|%F2rchJr
z7^4UgFuZ>7m-(*2#ZnVS9qXN5z{la7OfZ&T5FY=`sJ%++8QOAR*(kB^f(7zVtW4bx
zfby5R2I3`j4pDj%)JoIAEII=H(26U+Ek=*_5ouOdRc&6EmVbZLqr38*&RMS%RnNyv
zSbd*kP*cfwoSExddeEZsm=Vm);!MYj){BiqL;KtA)*rKBXy9FGO{ZhVn$>PuVv9@=
zHTZg_(dEkr8u?4N_5S^iq={L#7z=SDAgI9hnD|(IzRW3^j>NBb<(!laWwU#jPaZ!$
znM;XWuX6si&BmTf94D2vl-|7Q;rwIsxdB0;OD&~0L`ii|?BH<&@1sAITyoNiP6;Z#
zVc1mI{eq`^v5%gqb`nV%YHhQFSH@R}alXe%XbK&h)P7$tP0;*Oa>D0=wy`l)`4l7p
zJ>T9t@8O(7Et^tYHrq9&C>+NdVRkIHDsr=G^T>1AC1xLW?elW2k1)zyo@AdH6H~$9
z866YDuwQqm*F<@{EeX1zjl;xN+)%77ap`)Re;8HvDEf`3xgX-)O95xr#+=%ZXO&Vh
zapD&q4oy9|jjH+gKPXj~hZu)PS(siniY;~Q3$MA5MkaESTtC%5LDwSXdKM-pkB3Uo
zd*SXM-sGiy!Cp#Ic5w2>nk;wo-M*w{zj=x^L}!KmBa7Q$jZC!|lriqoL5Wst^-L@n
z5!mEs>}aFHqY^Z~V1n*Z9hXQ&qosa+C`t-GhJ+;c(NQLA6IPt|Mc109Qgh~B#$3cqpc@a&6AnrIdh7c`x_er6DtLG8lJ62`H@}8CoO_2
zo!m!B%^zy0_F*z*PB~$rzyE#`ca4oxe)PQ7Dr3F2ba{lBrs(y`3t$?-XQe-*7>LyG
zGod^1^S5;z@y*}9N$;_e`dmBb>z75^T}B=qbaX-*|HKCW;nnt7GJ0#&ZX<2Tu&(J91GqV_Uy}$54V|Au35>HW2tu^UvdC-G_R2
zLkGpLu2YlGcrN~gbApN6ZsnV5X59i^E#)}ILcpoI6@&$QqC&u^U%Oef&0u#gn?k7<
z(VhYWGr-D9x}w`wedz=^*n@`-QOzJPPaw?NG7v2;azct1`W#~Oe|_)b4_N3Wucmsv
z3jbG4KA?wH_jhpJcW&J1Z)9(}UNn8E9=y2zR5F@^&JxCo62<@4D`R4&o(lOK@`U*|}5kToLC|
zjW88K7DKd=-wjs_X3mTUfq29clL~|;0knXE)Mw^0CLfzUjDo$jlW2A+3jCEmF4*J|
zE`C%HeAA*LyQcNsZs-@>q8~sviFMxSkCm}E-byYlN7ELZij1k-P*txZDskmJ%#Nei
zgl*JsTs?1|NnnpBLBsu3k4?deKtM-!UY==8nb=dxQBKEQ=Ve+-%EJc_AZX=!NtvW^
z&zaygHYm$14XE3JvPNhWMoSR+OfuJo=zjj{mBxhC!NH_cZ^XVAlv1!rNiLe^dRc8j
zJ!gflXTouHd*{!w5oJ4jMh}!p-h5tjo&Gpgy=2C?VauKl)z4zK59u&=bnx1!y|?pVhp(%zY0rAA^?236xMowk$PBBF9_MY)sP>KUz5~1*-!fq
z9U4m!nlM3a$1@ZT$fSyV2HK@8TDb7`uE9=~s}!DWp8u`SnI(zMvwW0ZmuO6@ym}Rf
zI=r08Bm0{=531{fOWkD_E<(bEs~~r`o`K(ksmGq{eUSI?rx_0s>{-b!vrhU2?nyMn`_>CWAw=Ija{j*=XIWfr>Q
zbDD_&r3R73Sw7Jg=q0ff3&uSA_xl&9cmxTXZ_aND&Qi#kj|o^
zTl(oq&}Q|svPu#MT01JDiBV5BG(E`mgFOiT&?Bmbq-;70ohS@SPDYaakQhC9RRnE7
zV_6=DfNo$;)nmsR$vjsdArA>7x)~?&Lw$DSR^7WEpKoi9*Hy_gEzr{*2KR;D291f6
zlKX$LKRx!#n0_BDB9#Gfvvi{AAkH)^nY{iN!^
z^iVJWPVU}~(yw^zDXkxeT)21(=UHJa
zz<%PX&}T
z19$Era_aHpMQ{yYzjonl_cVU?>>052C>l&0SaarV;>$2@PzaqrNxr$V*eeMUN(_SIgoa5ue;0{_&e}Ek5
z;j32>Z7qwV#JL9#o<+P*vsjiyn9N{1yI%mgni|Fp8e~V=oiuY&H_Pzm8-S~FQE%@a
z_7}-2Tr4vF+b@#{?(g3~7e)_iIyD|O>d~W%`PkLD%=TZe{QCv)o52D=2xh>8ZZIfF
zuJ?Bw$Z*YA``u6gG2(%oBhC&i12Z42uUh>qb@gZRMdn^y!^7gbaN!{cac~t(2xUe5
z>5J&K_YR&vmI-sayS%JMt$)VvwQg?JK!W_ji~Axv4O$;#y{0Ym^oc23Ei}2;U5PQLH;NeDld+fddXqr$OTIWGd#?@__Z}(+3>wpr>VepyIg{
zj$+1YMxA_*UFm<#4CsiSJQEI*ITdyex$d4l&tRJAUg<-dQ{I>cU&Ud?uY#h!47LNn
z?KCa8?-awE3Ua6coxiCRjm1L#3;a4@U|
zzie{)s-P0lafzeky`7xp=d`?L-Q=KaRxnIc1{lD(5}o(&VT_8LkD_I1XLA^OCjyC!1Tt*
zh2FG#_uf{GDqho(4GFrR|BN%oXPx~BZ13BBHp1c0PtT$bY5v`gPG|9wCGoMbpnZbS
zYP1~(btu8Gk$V}KOaS}rHc~I6Dn|QMl%;`0@R$fGcp&1e4q#{$i;5^BJ1#@q=~;VY
zc0=_<9v&vMI}Tl!K*gYJFa+r8ZikmC@V0IqUFrALR6D%XFZ(22n%+R4>V>B{X3#am
z9g;lkB#|&anA#jhffEFyS*S9XS}^zkNgCE)z_Aln^sZ
z&aS3R59B*S#N6X~5%OxmzP@#Q1=A|IM>6M1N?tLku`LTytLwp^G
zak?$#?d1}MKBKqE7G5iDawB*B#S7!y8z^RY1*oJo8=~6R!0ZF=w$M%0v?^;pC0av8
zm`Xu&$rOW;xWEwg{xcK!-j8O@;9c1Kk5?FZr4jXwy6%-WEwgSY|$&y24
z(5qLfkzeI|EyqS7ej#V1?*5C61^6a7o;tLYmIVe{_U})`i=Aiee+<@3&N&QH;y}!c
z0^rI6zi^@ZDR;4bJ7Qww=veqj#UV6pE`%pEU!5BznO+2x6qKF|X2ey})s6eo;gfxXS~1
z`bm)IgfU^iJhv)pdMhH3@8HI*TDz8Yf-H_`70#g3r%g-cf~$Fk(H+oC@(W#44CTiu
zQw-tg3t3X5p@Gvvm0Lnp!85A)>o@-)*Nu~Zkd+m#7O7Cv(fv}dB-rK#IQLixOUuQp
zzMWF$I~vU&Ck>n$}S^wld9>s8G4CUD7kGwCE11o@nL(s%X;uJ78&o4822(daXp$4{QSB?|
z%6ylJ>23Qbi{6YDIzQ3~v>82SOqp&0Tej}p!Ka3VhxdG(NVBM@NRWKpc^`A~1wJA1
zn~)1WtRZ1*AcAX2X|L*+NpGLAIVQj$G11Mu*O(1DD}yASj@3TOCeS}KIg9qJxI&c&
z)&r6iEI_SI1~8|@k#+0#9p`b*7#ACv=N$l#>NHOc7j
z0iDfCbV|4N80Z6|fS`&m=rw(M{FdD{w^3lwhX3DnqW@f4bZX?O@rujDgpexx$!G2X
zQ`XZv2Q$nSBCkV9Tj|2%CqZlIfDp=cvEpLo1)rHSpHOX`6#HD57rnI-8Q7~=f(Egq
zq~z?`)tH+36(|?O{%sww=hlfMMlY0ce1L)%$Wo;G$oyST(+I56`
z6f45Lcyl``4r)vC6jR&iX5?gJIolE1=9#8!-VDUdQ;Z|R&qkl}fb%W(-P?EX$Y>;3
zGIIEyGT-**3JtfRPW2Q3deI->zxU3(&Tk8ria*TBga!$%2tS2(IXKnD8aiPZ?-R$)
z=_gH~#&7_a){`Bx79k0nf(Y9J*R&%oVT%VH-TL)^tnolTW7_{LUy)y4y@ka#&F`We
zU@NHMG{VerbEizn%*j!Ed=n^*ZfX_&DEQCPy_R<>SssIq1fRt7XU{g`twga((_2=}
zQYtPe&_+Jy>emhKRB?qds<>}DBs`|mEBEQnHzj
zy=`dF*4IxoD#_EFdIk3y!Ub6Hqf?jA*KnJ&rE`yRXhS}yBqm1w#I`IGu1rkEnKM&i
z*n+1bY8gV-FEk=-+2^&xNnk
z1_X64XwL9kkL7{7#>j#av~c-vDjhQmRVzK4jfXMvM!TPr)hsJ3OWlEtt^S^*eT0^i
z3~J2@bdrIOM+MiFo|tn{Mn;C=p=?^Oq5J7+xsPZMG@pvcjM8aKg*}eK9}hp@
zn&B_q!s%XC_qI+p1yQ2WzH=gxwo7tRO^s(wEz~nJ)>}Gv#e`uab9$lmKN_FpY$g^|
zIYPoF`gRDt#@x98ba6FU%59g|_4e?Gu3%_~Hn}p;H@~>J?a1VOJUI#4v-IQ-krYHq
z0(0dQGOeahN_1*8uKq_R-Ymi;bcHoz4qf)D3&Jyb?2k%Mx4Cl#*%43mbGyP4BGXCu
zA`1Ll*wx@Gs1LQ%2I8}1qE+)dlaVYxYdD=Os5cY7RCQ&-gVNGE=A-I9n-VYAz}HTh
zB1k^ehCf<6&r&DVaF}xHqljR$n)lpN^3m|TNj0W@76^v;cm+}w2Mo9xvRFTT+xG1|
zB(!~3`%b1Oo;Qy}%VcY#@7GYQ#XN|3smMLoyJ3M~C$^VgQ>c*!DX=5}u43oCHDp)POY!O#)f$@#S!Mve
zWo;d=|B-ViW00~#iRm~kr<38v_ag#;8{m|-QR6IJK~3P$ZK>M-$`n2G{W_a1p#Y_K
z)$T^h{eA{fDVNpHMQ&8CA?FRN2wIXn32`M+?er60{kUZ%B(TFzTn|{u^FbA&?=IZ?
z-S=(cDy1kj`EC0VMT#x9d6v`b(EQM(+K7VS_;HoR1`Uq*dn(!(Cc+WJhi`s$5j@aq
z>{x~9H)_9*bE;!bc5#s(F4@X32S!;u+ejJQf?+lQ=v{M?+Z(IDbMj7}oHuuF=Rr$D
zQ7E&~?JpKyJ)&zEOPoxW44qfnvnyO_jK~XJ2@gzop=CEKQ>6*!+Nf?bPVPe}LY&cy
z0I?G3HhnW}9>b#=wA2aW%cm4T|WlQTHRXlj0Gxh$wP;H%rnj^iL`Dh}+yBnnq4Qh*aE{ZlhVeaR2Vsx$!#)BpRi{3{e^zhp?K-pNJKHy}?H-|^*mU++
zf|>QuZ6Q-8PxiCTYEVP{#6kok&m-=;=Y^ic@G5WrCilLudIE+9s+RVf)PG{OG2qj^
zs8~4}85>Uux2Rqe)1a^BeWS;?#}3KnAw_Q16TX!hnDuX%+IOzX6(haH{bDVM>DS!%
z3bKdV!uSl}h$;4g1VyDIT8_7VF}s
zZI^LoRn%ww_UK#V(@OqzIx=KrPB=NyP13bbpSIx=pnjx>O*DV`wMWQ(%_ay;d6eL6ptLY
z#WZPl<=axtomG$X9y#=oObOmwAF<_~w~tX>SmMo=jPK8C2Hi{#>}Au@qPb^G%TAZw
zn_v4xeg7P}Xkox3v!h=df>M^g>x-W^frzO9BsT2j&{9wv);f5Et|
zGPQ}P_ZMw58u={otDjTTp#v?xb`irP4wc3G)fu`tJ3M;phm`U^04WaIA^7w~&Xnfx
z8AN1*i&G(K;5Kjmo2Q($t^9$M;i}nJy3||^$GtEL-W+o7_sJda8%JD_l8TfX=j=FD
zvbESj$Fc9r2dQ_=j}CO=Fzf%z{S;x|k&xg(>uExMDFbEXl`9
z)y+;a$+=0x?yBq>PwjNgSN+T0EA8EXB-7~+sr=mgmWzcf1v+rkfoJSahHJTw+pUxF
zHhxHwtbO^4E#^s4uk_ta5-Y9MD-DA^%rIFWVS63EQ^XtXVp+B~Y`rE6;
z4Q?J0H|Y3UPjl;TXFDC021qJ_2+^YS~
z;~YgzV-_+nA*z9Y
zZPT$XJz@$ciS4lfIcj`VJuG3;GQOy@GEwiQe7@KbVQ4@<|-7HMT-e>c{
zRho*f+VfT8E-RY1YkW)4IV{dFDZEu>p!rSu+DzxJB75mYvENQBwJ%eS^1D^h<ncIHh%(gV+ZlVj>Ks%12j?xrp93F;Q3Zg9?FtG!WMv3&Vv{fyj?X0H=_
zsm+pat(K07*IiY+BVuys_d2=gmTnlBC-IJh%B`cE2_
z>jAdPXRm!w>*~(-*nx^g&iMf
zcqT$mlD|;3Xi!8(W%^W$w&`A3wFd3Qa}y)ts+QEq6gDhvycy$=Wara%*eOtXVy{cA
zbYUM4a58xe-Q;B06mM({ZJty-5}61-y(34Df^P_psRIY(cKpQ{$1WKZ$7HGSZI&
@|AwmUpvcvr@<~0zF|;dUN&Jfnx~|2@;r3T%A>XHYqGa9F8kC)
zhigqf6fTB`7WC_jV9Mo?oIlxDf-JeGJi5;u7!qQEzJoUOh6WlnmGsFVJP}3GZ-?m@
zg2m~4j6@Z6g-aYoU0C?ED8ueHdC=zY7}Sl+>BQqK7aukG-MOM#UyRnl_bm{c(l{Vt
zUGS6S;}{(}-Cqc=zc&2xc@!#5XD-Jv
zJfY$-l0t_?fR(YXo5mW@i=pSHH^l)8T=sdY{De*$B8X&SlK~59+Vkm#Z(Kygho7HE
zhk`i;KqxWTek(jg02YG`ajlRGihBsoA3D;-8`;6#`gJmteqg8tt|w)
z6@0*Fx0p0EUuEA012adr+M2&q5D9h5;HA{hMdckh3vlNKKYM3f?8Yfx0cB+R8nN3z
znjs&@G~1%yncqW_pNG|h0TN&(pRSX)s2bQ*UF^7wu)6?Y$e_{L@4(I1yaZZJ=pjz1
znR?)VQ_&1(ws3$zNSZK;uD15J;_I%Wt>)tb@&RyQHz0mEGn8`z|FJZ-v3VZs+&)^e3}LI%i@M11|g#~%QiAxYMvBc$-z
zo0RT>k!W1O7-y651)yDeD2{vVG%dYc0hb7~F62T&Pbakyn{rbw=R2#(KV7H4fXw_0
zh)CN8>aTu#2CO=D@{1S{!d$Q&CU`Te!e#QV(YgZ)$c!ag9H0&%U`qg%b#
zUX@x*RG_MwA~9YmXU>U}CRJpsMgn%6*|8UvR=ij1I8P=7VLywl<85hH$tg&1xsxOi=tWfRO9
zN-+A(m3{~)=);42CduUK_|%bex$*|(IWhCn+&7*xn8e@0%ST1-Bc%g>Sa|{1nZAUcX>?Nr*#f
zRA-SOe?d4BSI0nWB9i_z=OK(Fd?>BmF3ETC+k2x#z@RUBUs|Uln^`JhYt?f&%ntQ_
z+FGFf$}DSvx!)#&T^T|9$LA|peT7#x+&bxRC5i>c{5(D0cQ1rXZwz;gFZ^z&PaQKL
zf$^_B5BH`D0|r~(kC?|xmY8$zK+(&wBCN~4j@@E%QHw{_ZNUL?S>EAbtOT0v_U`87hiw2X$Phbfaaq^g
z#U<#bUag^l6Sx*w07u2gs+W#!ZwmtX8Je1!pTo(}c~YiT5lRbI6UD!W_=qI77g9U=
zsUVynTKY(`i
zOyP98-wwi1{;*s1S#%_V-O%6p0Z!bw83HhDoA-;7+7<(+$}ku^5e+4N_kfPCbGJdW
zh9!}^*f>xjn(V8Xe8c_P;cXi(MoxrEOnI3&(`9%G^|46_gmb^XWiF%cZ<{PI?0l&DAY-^&nP;hF4`wqh)Wwd
zqvD~1DpJ{@H-~$i0?P`UD*~{ahyl#(?f$d(Wd#^^w)O6y+`eWbg0{NRm+?%{F);UTZNlenec-i^GKWP>O15lnxW^*vr)CAEHuN$&
zJgJkD6Ct;~KTuuexY>#61P0e>m*Z=Bq^&hu0a+FCI~7UIHVK}W
z|Fpi;q$3|2bN01`VN#2iee}sY)9z!U#lPe*4e}tG9zvor6ylAlA
z=F^mH@?g8pkdi~4D0iAM-7KeE&)4gs^Cg_xiGZUI9sr)E;wIYOgDyjR*`gi=HkLQY
z+ROcR*g^u0d*QZcNm&N4>Ip1)8gwWm>IwK>OZqu0Vxs7K>?K&HO&Uf4&!`dt_Awwi
zJ_|!NrJ{v1OK}9~t4JksAMoYgafRW!1AC(a9nBNXP6q`bkpMpInRR}P%FPh!%*`-^
zXEMkPj^jy;OXt!O6ZnnZ_fiY8su56aDQG1}qIDH>x3k6-|&lA@QI65BEI)-wi1U-outtbBo2c(4oGut)wJad0P
zu-)reoqY0d$iivXQ*&0D_Uqe~!Rtuzo)0bL3j>JQ^`ZOUGQ9Sp;p>;q>0J}3nD;sh
z0lOKYDdaS`aaaf_L8)=*WYqAgKNFqWRkLR`LcU;&|IPVKu3E{KusC%2c}iRR44-ab
zt&w(UL?HsxI21}rvb7>6w)lN?s?1|PjCIN{6+E#tU60XN$3PZNpMS2hqzqHv85Bj^
zY{qr<9+G7|>X<@^BTJhYe>Z5uYn&%s>k*%=(>dt!#!BWhHRLOKD5&EoqDwIHX5eX;
zf1Cz861V}VuZw-{ZgQgOZl~mt^Lp2CT*_5;^yqbhhPWzcw>Y`c^ZjaB?SpYK&!lOr_7K6FDL$_F|Vnm
zWmLM*gFE&t_p4FB{S(P!`@lW5(6g`a2}x@bHL$GagB*uAp;h3%a}D>pJJH_n^mfz%
z?s&`c>w0^Bw6?K9bxgB)`ODK83u4zwWnDSF7`%LHI;hA89W7gbm;1O=O1$(aJ+JVp
z$khI-22Wq8bc8@kDFUzlqP5EIlbi-GIe0A~jqP6dF00`j%dePeJizAGuV^OJiGak!OS@U&+w(7v%?s-0
zw@qey!+jFW$GfQ+pK%K2OS4Cu#99r!5{@rK#&zRE6Agdx`#Q^NOviKN(kIej3jY;>
z5ou9wtNm`Dnh!c_3D@6toO9k!Qi&f2o^r4X?yS4-mJ?Z*&$tqGF-EiLp;a
zLf9uw105so!`xYJBO7xBQpBIXS;C{I2WSVDogVJRmd^9)=yD_mSASvd;T}IvZfE`#
zGGuY}z7p*oXD~8{KYD1XQ<=1F)LQiW-PaN|)ppFizAbP#&#-J46U3Ic#XoHKLk4m7
zcSWJ56&|V!Wf&XA+Tei2SYLvgzV8i3`%9zWNb!P!sm~r6oxGc#ITxG<-?!yI@k0wx
z*fKR*i9e?5a-#!epLiZSm5BLYStQQ{WZY~qNPXgKLW?r%diPmS5PLdf(@mRQqeV8K
zh>dW!pNN&{bJor_pM1|sW%z}vCf14>4~`S)mX>ypvGk6x8fZMy7$1LDBbE2hE>I+|
zW)YHGtIb1?*Sl!ow0n#tH5@}*N6`%Y9>yj6hJaZ&&QgOsrT6=S#T^trMPrOuWGl~6
zT=|iys@OLj#{9-D|83XUy%}{VWF-C;R5;I*aB%gwy_grJ;?Fr9n-%nna`@3=X9-Px
zO*2gyS{o0|m_i0wnNiy1tC==JrF`L>*@n&F53ML4B6+US3-N}rt5QmC15}s*Yc*47RhAmc`jb77^VojTrai4yAa
zN}-SEs0Pjp_Dem5M?jI=c&TqbL5GDsaXaoZLDmMlM1S!(AEE!&V6d1HvOjQTXkFRXe>7t*70XUdvckHCyJA<*DTSS_dM~n|uFmwr
zAdmOKcv{u?H~JTmO|^M5m}s7@=93ALzM6}~8uOeAGwyl)-JGw3zul;t<
z{h0{+!0w_~tfBq>Xv+U6)l*z$z-YO>WS+GL(cULOY32oCHEc=P)HO6Uxv$N%
zs{cR0dpOAIuzU~zmJ2mnc~$^wHgXIEpA~6yHc?&zX@v?27Sn&a*sC-`3{W9}r|R?P
ze@HEfkP*6Tp11_ww2z@=X=g3Yq*vrIqcxAU3M)nXK1qRj>g-
z*5Uv&lPV}V5ZK-5NCn7R1twpy05pYVD2fO?ur1Vy8^}tTl7|_n^&!48*6Nral?S;~PCY!bH!>N2Fl
zBzjd9>5Zk+_-d=m{G8Oc@m}&Izxor7?R0X!Ma}zdw}|0dQI<3A^F@rt(upyF9ql!o
z1^J2@x~}2f_kkA{^?c4PZ1?)#5C_|0EoL%Af=Y$YcA&?TkX+kuzaF+BecC;FAkJ`2
zE5h0Kh~2Uu93Dn`@%3A#(r2@s<$ODZVFu(3z42~K$BGI9P8SGRjmnvzD
z=ejl@>ro{~S02i0^LT~iGTlyaa$=WJom4a9(RYrb-t>r5uAh{zGk!J=-8Oev#ej*Z
zFwU75h~3Ee!VnjlBJ9M@_|52yxe2SPtA~gCkju&|Sa4CUs`r#VthT43$0A%BIJu;@
zadW|iG;;ctl)??GVj})AluOO~2R!OAoliuNl(LW+#!gNOvLfX$H}aelo938sAXUjrQ7Sx
ze#?wukQ=gmez^Y}UX0=e55kry#4YcXLV^BF9Z778*I(p1_Y5Y9?9n>SRl`gl%y@QO
z6E%(VA0iLZON%F9i(ior*%op{a+LOWwVIs9>rw0T;$pW7*MA1jlayrw_
zo06jq^`J3Fa&@RgDA2{#GO?N9Cke*&6BmzKp^?9txwY=e
zjcuo=t>+NSu%LaIMJz0
z#YfR82oX?mKdN_cGmJV-`gF+yzy4vRB3BAkDv-v0pnBhsp`CD&_gsY0f0&+ry62K?
zki|t8AWR@%fRi#$qlYOK*4%NDXOd!Oom|Z-C7nRqUqTQpu=icjwgP>8A35Nyz{oFhxrOKP#k=9V%8T0=6z@nTCH$e?pzVdhQ0UxrnV(K)aw7d
zf>Gh{By+vg)pJ|4d>@*Fz*QWv`p1R}#61O&4fsZSTck^m%C~@Bh&yKD?Ri3R1?3Ng
z(7CwdT6rkY_z({iNrXMWp1-}~%pD&H-Zcy4_g8;Hgol*uLdcO0REEMY0E>c=KQZYl
zO^yP2g17PAUxD_*Lf?Od1)zRT-6ngZWh!=yDsvXb@0-jo)68qv{BB|O_4NSIZKC$-
zF9GBV@DyJZEBb^k82G(}&c2HRbh(cEPjB1s+ODRHr9URA6*F5ajs$?2+yY12aS+M<
z50XttHh`_RZ_9^PA%|^P$V~1w@QX(PyhcDi5YrkNo#R)?XTP4L2z^ZgtUWus^C
z2Fc;w>W1?@JMgWz{6)ZSVXw34l=VaiF
z0*YQR#9jg+TrR=$sqAVeD_n}PKN?{25;|Rwf2aw6G#)4!W}iEp1a;jPP`rKY%1-@@
zxpIH=AZhtTdq#rHT`nSbuN&I3I~{ZNMYu5T3ez|XaX~e}OizDjZ={*il<17pZ}>{u
zm4U?n
zxcqO%7LNaFa?ShDYh!3e=w5*T=H2<$)5a4}b;0wM9*KJ2&jA$1{4(ZtAMkc->U-Yt
z%;Wdm*RhpWnVBsVj@aX`PTvjIUCe(rvpNjoWpXLNZE}WWm23{lvrH6nl)V0;m(wLrYjPnZ`S~@yL#>N>GkyBW_l8O1(
zo;m>PRQ+{~DIfb#!yt@c98wzp8mF>b&gxT=?`8m}Q&jOyhu3i)PgREP@D4Jb0)Wb<
ztdf-wby9=HE?Btpd6oYOjiK7^2p!o8T&IIE9mi8W{wH5?zaC@SI2hw8AU=Q%TJj?w
zVfrPacz^%?N4s>MfsxAg@zToH?xh(!mPPINwHo{ON;c0b_i4>vKm{n41;1y)8OAR1
z23A)Y+}3@NsW|yCm+3`}JYo4>k62}mZF8%67oOwW+A2q6$
zqoX8-Eg?l}9qByBTzLykO1q+HTU!y|hain+0gK9dGkGx3xmF!
z7D^*&lcy`H`Zmt<*at6{`h{8YPlfIH?qqH*hH&0W{Yb~jqgUDFZXM!
z6(6J8N1?w`e`I`D1U(aEZZv|2jwzB-M~^BJH*K!$K>T$Lc?KZ?y|zIBkla96U%#I!z2AYG57HEGfF1)5M~xd3C%mEr|k#Eq}}iJ?&$58gkKVtBpcZ;}GbC^=O?SWW=$7*+mh!&1j3E@Kq^@Wj!4r
z0=A@pNBXkzb*^K%9_fks?TWd!uFe^YVMPF9eo$Bm1i9}eXh3iV-V0`i0v?DXjq_5!
z@h~+t1t6uVr)q%{Z8?nF^P1*=87-;(I$tk>2y(xHFXB
zzi;jvoCNHj2Or!3X{P$IMSg&>)EBURWI=$gk%%;>DVBt(>YX>{Mw07Rck$YQ4=qYb>5861ZC#O
zrU+ZBkhaDjVVl6Z$@^x!#G3>;Y~P5H*->sF9pw8~G&N;PB#5#2+Bvj~7=HAC;Z176
zG`>>{SxI_Uy((Kix+=@pDQ?qV!X+1oimA(hndB5t)org#RTm~+BGs&uvDkn&dUu;2
zFQz2c$fr`Ys857pzlu=(a=@->=7kdUul8W-)YYPK1=slS?g@zQ{U{c!^Re+y=rql^
zJ9w4%fZ6s-D5I9>XC$09?#PfGSg4IVR*9M88z2-vC?SDEA~#uqe)CX3)_cQiy+7(D
z;O@%rX$$QlfO@^}@(B$i=(T}A;zyT#FP8z&7|GjR$5u1*v)gNr-vjdN<8fux90__b
zv@rT^Z}0F|I36YV@F^3mU{?E$wf7#+83RpE7rx=Fy$*663%QN;Lxz*oW6E4cyOvUU
z2jp$&#_YN4;+*3?adUBRaW^x}Uo?FJ7}jBo&@-d1y-SN4-}5@KHW&n-(!#+`lV1ts
zq+by34*!|^z+@{GHimVR_XkHx!~pPRaA_E~Ur9s>`I&}Qwl5uwYIoCcz%5k-QVQb6
z<;%~eUnfX1mAR`Z#TJ)2t>&TBjgv$$kN#coh@f9Q_d#u6i)560hQG{I;70ag`s%1d55z;G=2H*WSmhy7gknT$%?P#VnN$E
z5>iUFk=|7t_VE^{cdTTeA)kzW{G(R1CGtkY6*M5Nj*Wd_(r+~!T4rf4R^=_r~r
zGunVgC2kLUk$rzOdM@g^uXGAGZGp<@1B-+vE+)~4Q^n2I6QdO+N~CEa?G0_6rqk+p
zg$y^fUE)nVl}dxM*ytDI6}3JP
zFc9T5;e*D~=ni%Ciw5*ff;{gt0*tnjg1>EjNX|!L^lfQb|68A|&nV@*b>}#zwcj|$
z-?-L08e=f!`xGT(Eqe^Pcgw*e>E5Td|J_V2Z$1Ekrdl9A_39$N?vX}f
zV5wJ8sJ;OISJ@oS&aW(m=Zbz&a5KI%TU$b@b||APLMdq2$z_Z()I~$f@^%^jgZ9bl
zE%ls}rNe6yZ6!l|R@ieX-DFPAwGGgJbf0)(Bf{>h5YBY2gg=cEe;fTJk4PMtwI%LT
z$g-tAxQdJ@UWx?^Z~Ue;^U6R7NTMgB=jW{bSooeuf9*H-j2+Oh+jQ__r}g}WWb^#3
ztz&@P&M&-(LorplW_AFVgp^%`?x~C&!DvCHTpbymCZ7d|(kvfahL(S#aP4f`sxe0S
zdr3<;&cxulF_#R|p|f^i_8Pyrb?E@$9%^e{6ZKguicoV-T)bLp(44cxL7)XYZ1j}y
z1mwTbmwm4#=&T+XoRo;Q%u=nFx?#X`vRL+5ZpN>35U_e0)w_-RM`n=iB2j?iV1}EY
zN})oNs~0s)-y2)OxWyxmt2Ry`v9$BRP?XWvZ+o9}_2FkIgDmo3X@R#N8LV`HCJDv0
zB+_30pW|RFyH?={Z;e|8w{mrB#JJUR*<`LWG~Iv0+MrM8YBqiZBw=o=>{Z)}bIK$H
z$E5?u?M8wDs^0b5+7Eotq$VDQE79M^iRPAUyg25fnwm|Xm;W-KJntYT=x2nFEw-)b
zpMW}E99cX}9*Z
z!w;^%6bgGLUREdc)J;#5e2Lz}n-v%3
zcDJPl@YlzAS*<@^{1W1a=!0f9&vToPMg+w;R$}BJSl`(kq0tE?v@7gxOy^;*A+?*U
z&OUx#RiY|d{6%k1eyiNDFMpwrd1r&IyS=vce73!K$T?Bk(PaRmRgQJ+_m
z#&n~&-d?t{H_w-w<5Z%SNFogSsY}{`#WXB-99X3Nt%B7(Q?YO*d(Cp^LEU9gRSG>W
zEBfigJ_5RwNot)bos=(VeVwO@&i1D|ea?*S&Xb^4V)fs@!@yy$tr)`M5@E0CG=1NY
zRJ}yta5YM2trJk~YmOuFs|Uj}b^oP%@*Hh(anX>QkSu@mVj70z{%q&@Ug!~Og8!z?
z`vT~2`QepFg_#LmBX{|Dg#0}9siVK+N%1Yi@6GQO?`_9=W4}CsW14|}$iKp_xq1yZ
zos5>Jx(`0wPSX9$LRsZ8*Op
zg}WyX%;C@0}Vq^~&Hp$9K6sDdA`)1doK;@kZ91M2QIV`gep&duHrt
zjTyom=a*v!C3_nU4>EhNSAVD7=%TUw7F~0wL9D<
zcO+S!eU>O^Bvm`0hcm9Dw1P3hb}+OId5R?716D`3&z{bV&3rKeqlQr=2>3UsL$&Mp
zh_o{*)cjF>_&03hjSwgG6}%I9jMQg2rAsPkb=lu3zgZ^ca$>>rAWtyG0S0YJ*B>NMk)RQNiIkC
zF7I03-x1Vd5rn|&lYU+iW5Ok^LR^7vi{o>dqkt*gglJkR=QOd2m{hI0!=JxtXgCt!
z3NSF)1RuDOOTYmbCfd;!(q)vzh?Dn08)SjKB|M)p8Z{-fYnr-h=oj3_he08#@D?f$
zdev*Mpnp9fN#D~#jwrRK!Z$?>{VDEK)Wuz
zz$nc%x38#(bvtCtRfwbs@bOEnKlBa5t9Z`};
z^mEUJmRnVSVH`-v)&E3l;RDcwjt!wk_oo}LAVDC4(rLRF(f4`4FCCEi&NpUcVkQKi
zPBQV+6f<}^Kv)1+^4|VbybBPz4^SEF5DsYbx3v3e51d<%{vS9&9)F&RLAJ*~z
zmfd@ydtu+hU%nQKw$pG`b#*`j3H2?u(LYZ+c<}vBl)=KaEw

l!4vN zS2Z{6H>Y$rab1t;hBn;27iT4a3Seb`pSG^jBLv!)egG<=e&3zOuKRSC(Y?Jy2i^gV zN_Fjejbobjf+)=b!z#xJS;!lZJXg|jLC9UR)WaHlg!f`;+d%}lixRfdo9aMaQBx1A zpD`7tb@FrR1hi@u6RG7KJYyz^BYO>^)_*kj`3UW%=cLb@@>;7D$1tZZ0vDz~sB4N! zO*k?{K8D5~IfsNfCDl+w-s8|x;P#@)`oIdW92N|2`Eb=XMLS!k+`J>>Z}vb|?Hx7C zfgVY)LtjdbsNY4@Eyow`)+}F!sUnRW+psL7X2wL&F)AQvbbTFJF`~}JHISrig|bk~ zt zzr)=S7H&C?a)K799EC=Z_|XC9W#ad!qw)l&a;~qams-O_eMV>xmr5N8?*x_@yJptV zSgz6Gv+3=+5uITbhI?0G^)*bs^{OwRG#e8l1WK`g-gtts~onhWqVTnJjTpQmJ-MJ%lKN>ZL@wmT;_GXh#&_;KggCFR8uM6PmMaRP2J=% zc7wQwklxFN-nq9kiZl9lruU0Bg&Jg>mS$8FXL&^w*%31z6kSg1_Tc0^Szn^fh(@pM z2LnX8qDd;v*1lx;cSMGXo50G@B8%Vd{AYm5P8=E~v@41~(YKL!L_oz|Kn#g(1t*ix zmrf2s-bt9XKF#;Z(S8cjSye4_Xd36$9Kmx?l30#FWaG>t?bw}Z850&3W2EK-uja0W zt``F@Rr_-IhAX2;c?csSIoLoBO65W2A!kZot5WpY(^@*h&0X%QJ#fBr^=)~31I-j_ zWc&{0v`?GbR`7x9Cy+3pdWyZyy-C6w`$pZI2j1Wh>z+p@eXF8Z2Hm@ote6NY|6P?; z554ChHM+{t3oPK~{M4i2-NphTq=4fexKVYuFhDq&o#M|@5V{5|+^l32kU+kiQk}<4 z30({$fN5D*4#^OJq#&vS`PO!F5e5}MVU+^yP4QaNvF-UU0)>dy2L6LeHvDf+ z#xLAHpas`Gy2ev4al1F}!qPjz2;#rM7j+1Te7uX9CJ>;*(Pt;&gVP!Os5wYgCVeOZ*< zs7I1nter<~VV&Lu$Cn|(hVkn|xfgXxZBdAPS06PTMQo-VPWdhQAwJ@6kLH;+`;}z1 zGxn-ONJ_|}c0Ki|Zatab-Sp6sCci&(#X04qz7^_SyxhXi4M618eu^!jR^J#wIk<@7m76?HJJP^8(;u;vV zt})%77FhstzDo1CXRy^Y>L^-uQi#KD1Slh%K< zscvIw6l4uX;vY|@M%O8=f)~9NnFMSy{<@20*|eIx`m=7^e%1lTXfx)FKy=g9(bOCY zz{|reMxPp4n+)Wzs~4YdYAWlM*&bv}3vn*b}TXF1ulO};j=Cf)*li}?hZ^DFAD_w=2fw2PBLX1JAaQ|2w> ztlqq2vR3C!QxcYvTD{x^t}*F!E3UP1mH}$BSDjz0nwYeb> zPsXOVnwjP!bZ znh3R}*uRx(U1)>61xmy72Mw}~^z9%;j*QB>{lw+cWgKvdUPHN&M`A$Y@D)#*2lzD& zW`EOSr9mmF?_R3mma8OSl=!Q@J`P*C2jS017tVH`pGwv>OR7Rw;XlYxzAp?EEC2N+ zb=R-nu6CqidRby&qD+hfyIRg>vc$N1)STF}cK&!i{}pz()?YqrX+%KVkb0@bQoWrnf$2Fd69xfW0plCZcx1j%?@;Yr z#^kKMjGsR@;|G3rGx3%y^9fYi$U`g?G*ou-wzP1KI?Jtou5apK4TZs31JBiP4k37+y>v=V1v0fC-sm3M^)y!RW7>kiLwU4w{ z`o|=ea3jmFcO_ud`KdHpiO={qDF0%|*CERdPmN1@88r>W3|Pgx9EuVsmx zhjYkZC%xw}x9HxhSOsEv%%Zm&I%l`)74m-78E|7zgpfGtcU+bn^0^$VeCCtc65=nR z`a;Q{_3TjaO}fnkz(SSik=x%z=PyIuHI4JdTwO^F^BWd4GbsY*0Vc&Dc%XI9*O}8^ zkn9%~c6!AoALE&-+TT`)yKP2&z9mOggV2Bz^Hx7yfWM#l5|$A*Kk&gJx8KR?bHPwz z!BE(cSF63CS=mokcg1f4;~+=~EcNN>T&t%|ioe2Wmq1r){@nL!yTi;;bCN>jkD{IX zMP)kE<_v7m@PwDjpt_kC!EdlYf<_HKb7X7S!Bb5wa_o~uoI2{0v{+L0T(tMr*M6~* zBd-gs&of4mu|W;|C}WdhS2E1fgNWr+fY^@4(p4l5@^qdmKJe?Qz@&WzeLZ4}dJ;~m zDYeP$`F-`zkHT=6(0gZ_KB#DtU@r+5*Q(#XTVIzdxA(be`(9}f_Eo@i;QxAs2RXHE-F-Zg+=a%s+?kS0i_hwuJ>3UA0J?^ws zuh_oFCBlj+zIZ>HtSwOTGp(*x2`M&rT$2?rLmVjXpZwqy>Ku)+AqQbczXZ( zax1tNLyWBv)iRSWzL+}VIX zqPWTU{jXZZFP{>AHf#x}Tp*x>e`cdXNgw!dQpAgFvC^~O3B^8;wL7^sEwwh&A-_jd zX|_DuaaBtVc>`P7Jf1Rgxv`xq_;;PYv^f~PC}H`JApHMoEPS~u@a>#;-(K_^VrBD@ zzLR?$C}ksLRe2r-t-TQx%7M9`qa?NS2kzPJFC zbeWI3ih}}@Ip{#1`L(=sU@q3wc=vg{->fRO($d?(c8j%6c>cv0CwLdfiWq@2 zkxV+0M=3`gSS%6;SIK2@nzn^?gGC67trE><_)Li~9H?AXyd1q;FTP8RP(&>q8}(c` zK4h(&zut?q_*Mpk6&jRR!M~)X-D)(#1Yb$yR6NTZN*wa)a}zXc`kDIAvie9=nTF!A zg97M!=P2c6J2cTHsXF7DHQ65s)*q;Pxs*-=xx=OsmVwe@&z^A@TC4o)A$ka;re57D zQlZRd;J#(^#p0778ZH)B`4o119|COBUaM7Hp2w4wpztv?13pn3#V+;O9GtTvGSZFe zFr7z6UfsE)88{@P5RMw|9+Ec6Zr9U=>B1oyKNo(ptpC<)J~=N7p%e=R4w>Os>E_}R z&kp7@NdhkZpjLC$i4h8mVGiuySRYPOaZgf*JiVq0`k5=FZ*!HTwV~wxIRCx!A0`j_ z(j1V#A@)V)uhtoqSoz0vJn2gB*fK_KM~AUouu4I~8x^f)7KP9|s4rnz zvaF#3+y3&<@O{M%EwMFnZYf9K5hDA2&HQhj7e{06%7QgyItASok>b7Uo;2CyWpD!{ z!@a4wOZ4Z*bZ|6vHYh9dl8?Pi1KCLSH39UqQkez%hRrQ5-iqW4M?qJ&7VMWTPSQj4 zp!yqGX0LH(2e$Uz;xmPhl*~Y z+;~l6J2@)jSBuI57AS(BI+Q;#i&8E3Bx58p3(E}4d?RDb`ONJzYxerdXXQN~echS_ z_Wz8An7z@<|9wC{HJ`$utH!>&yn!v3CW_8$5F1PuW0EZ1hCNN;b$c6to@!rxC_>$A z$jRjRWze072EC~&vyt;x>RT3G+|kMhCdLt~GoWAI;K9qlYvuF8f|6lNLs`hd!A z4D{nW9re3~Tr?le%dVWCKIEC~8J)8muB61;?dg;pQlJ@IJ&xns|?^0(wso)8<_k^!{6pTlf0g8w9M z{fOE5!m=>byVyG*X>Vu`xr%ufpUupk_Dh?{Wa^}fYZ=D@i6Br%{~6&Gil<63p5*RD zTA%@frsH4!i&C2djf0IINbfL1fppbjsM)7R3dJlP4hrDOL&39(D5MGhL)CC#U_EgB zY7>{Zm-jC=d(IT&K)HgM5K`Lb+Xon_b5{?~&zHxUT}rDknaXAMoU!LqV9A=lM`SYg ztmR-RMZ5D7(ihNV{R>&$x`qV5%SW)@YP z*Xfho7^#KB(`VY+WXE$qshaj)1P3$5?&~#AUK)fi0|5kMG@ON1fct9KMVYpXkUo){ z0@$Tc7bkR19v)CMr>;a`ftgL_mD%WD$91BA@*mC=yR%`-bj$!QW%P?09$wijN@WJ& zk%?RC*MA=3)E#&dW7P*pZA6pc@8epD03C3ON___~XfC_7*f*Jortl;U00%Qi!yQ9( zcD$qC`pZ&kIdONPh`r%F80&q~YUm$#+zR2x*r1;zrfN*#i-7wAR?_^oc@8k{|FW%7 zK!yN_Y(QaA$a_Q~>{_5$;;|3U$)hR1MGp-Ak?4cEII@7Mz3vL*g?{Tl1MGgFh8=;F z&^C>zz#JPw|Mv!L3xqFVatO?+I?w`|WWgnfX~d$J;Ol@G<(wlW47_V#SovlsT(6P~BYd7~s$p_iNAm z9OHGoc~@H-!$AQvj2x;>nPb zz{*EJ03lt$_4T_r*Mm86f+7LHFzD5yp{STePSHDYCPD`wLsqM+l4{e9pFgIUNJGN` z=xJuorl9T5uDge0^PAyq9&B{2ctGFZLy?ia#0=KjhFGp7U8)QnI10Vo8e2VoB*dA^ zZNlqO-h?79x@(biFeFcRP(K&Y?%8y}DVpXX5w4ufRCy(H3utGwaJ9FQyy1e>UFu~K z*648p{N*54$|^9h86EQe%HN%D)Z5j+LvZM9a*0|0I<0K1h60#|hvihg&OUJrZ10sZz>J4pS*mXY$wTq6ge z5Z$ut3jmlzLMP9`e-XX*W}~^?^8L$E2)xG>DyPwYSc!TDMV-=Bn7sgn=Y=%plc9c2RVXEO1iuGk4wfJ<9`Z)0SvQ%S5x|^+MfpBSUv7kbiLss*uK0j`5Q9cPi$+j zC{S$}QTu*s5Q+c$6V^5ok(x{SY>nYcN1jnb$LzN@g$!@Tx%39^TmUD1i`|jC?gid`UMFroD<%W zW8f_n$`8vf-F}j7RLZpMNTw#{$A!IS1RwX){lSA1bq|yQ{effSo@jmBbBE{8o&9{e zooe6G$`^E*n}I@DqphpV-k~|7;{+MMDE%MC{xYoT|9}6-EmXju+WtLqNonb3(lHtt4bt7sh~LxK=YREoyuTb=ONQ%Kve@aeGMXk_|nmemy`3C3(m}y@hNN2Z^$)P||j$2A?tX z&NO;HX5|%kw#Y#Nh#jN``url^nmr4oOb)mtMmI{dALv^CYrB?AyL{JzfanQebo-je zK(82Qoxv-?Nb@}k-v-Pi`zViO@4}=r%~N3CvI$d=@mh7t9FaB^6Xg^3Lt4;G(@d26 zc_Hp@pBY4W9bgyHdp_U<3pw zO%(2MtjAiJO}R0Q)_R40ZLe~#!F6c7)q;I(B?5!PU#}dRw%>c)cec6)=IOICP)7Hl z38DkeEh7DQYx!hIF827a-E+uF@l)$oLjN-3V^n+A_ypI1Qqb9kvJi8c()x65(rDgK%0%vWt*Iv-f&s(Q6Nt zmEWCC>8?^1Gc8e=Be%`pI7y1T zOLVO_6w2vi-;xrwjO=$Bt8gE<|@iU3}tA8za zYP5xj^cxg22kE3UaziBRg8yc9XEi;py8#A7h zF3#-LwTF-;*6)jL;ncn^*EfM~#vvy>mdySEI}Bt4MYVI5E$N*(pe&7!9aoCU018@u zK8Ix)?Q~IoO!l9Kgdkt?;@auTEuk0y{*pOq!^@ey*p}Qk_mroI?~t#<_Mz6-Y&Pc! zc{x%qHIgK-Bf}$MX#8dlxaI0sW0XC4T|>cZq09d}7ylm+6h?d_)V|o)FDOL}lHVj0 zqf#~&jd(Z=6|Ef<1}Du|l}VAx-ZQb*#c>YqsYlXAljh11jW(G6$aue-{z=>#T%X*3 zj1}>e!Oh4@OKQpDjF`2;xh6FTJK^2=a;wTsH8s8q{&ry_N6q%7t`)lON_C9A{5!ps z_#uTK@cj6Yv3g{q071B3X)&s7%d$b}&L+ab7QMkNwN$3~*vqCmF&=Qq`0krF4P%qN zvPtjH+gG}B9oGd}sdu4!V^>06mp0}rhV+(3blG(;wNX;9P#YLTb%x!_)ktAAFp4a$ z1o}@@C#1hH2+2_t*5sS;Dp5%U^PW?mq=W5m>q?vHO&z_$ScfXP#7c#s*lHCkc$Xzi zA-=Xi!?0r_TuV;|yf#Ycq}E#40;F(AoD^m2Al8t2zo|l%gx8oM`ZRNk!Zrl#-SqU3 z*sy5!yFha8@#DwUe!n_~-|`-uMPT=0A4{BmZP52nn6yV9hlU;FIGc6@m{ME}6^q~T<2dJ+tQ1(6&b=^+NQ+O0pLCA#W3 zOK|UlV`9{|1NPtl>#gV_OSBebQ;+amS^I>k^!7j5J_({yBMT|4KgTjD&+`<8Uyuz> zplD_26t|zaTjzwOsPwROyn-8W>zXH~mjhBb3RW9m0TJ4Ye=4GG70f5WkW$W0*OM3v z5nyLS1(wr&7fVq4$J*Y&PmdU0NeW}qj5Fml94aPjbyi}KYmH`tkO49PNS!hR+Z~BD zsx;$5WiBnm)=q*u1UIeK+Vpa%;!@C&0VzDCyjGd4fu`JOKG2Q|akGrfCCG+~YQY#| zWxF+>vHe8V10A#WFzpw(!T#U^vA@T4atu}kr#BB2_y(8^a{-Y%#wq;v1;cPdQ??_o zIP({zeqbTR4|SuBJ+NA{IlPIP)``Y#HPG6T-sYoPpoJD0+W|B*3iJ2P2!fHJGetVh zS#jC{__AVIW0pTqEy4w!3_7BK*|!beGSxO&McpsHKhd8k$x71m&RsI)uF=zBP5^Nb z1QV=vI5EB%%S-(orYY^LX^Hqv6aQ$7+f=|t{>Yn=;64FNFH;v>)t)DNUtiU~q&9KA zm@9ArZ4pOd@4PlK!P3PtG^F~_!<&2Q4B0V&kOCQ!xfK~*7B*Ed3P}aRw*5r=UvcyN z6GlhiA3t@vxCk2Wi43;kFibjarc=ztdc#QOW=Dbr6k=8Xz_Wzw#KEVb8bG-Bq(nM4T07Hj5;w^c(;J+OsnVFq9V@SnaM$>t%_~}VU zrohB~kGJ$WkO+v!r1 zIk||v{6}-z|c@ra6a`KT|s5`Ks$D+)q~au$I6+xz9z1aOQ6Zw;#0u>w{1@U-#AC#4^y!^6*`;)!>-=;w=LvT)_s+I z9WEYjY$P=w=<&I^KH2DKV3u`!TE~cWV3yIgGxTqvPOIYf57qFqZ?NF%nms!4c-i>q&codF zMq(9X&1h*|NRXWEM~!TMcvh?e<&Sh4#usjCL)&hi2faR-p{D##oi7dvA3v3}+T92+ zjSr|Qtr(UO<$?eawarrDtqnuv$Zwafa?=UFqSX%wO==W2h-mm zH1mCQqcH2QmL$iSenu>LP?5ZuifTWgUI1bFaFe!WY=RbkMTJ$rklZI`B)G2XEq?i; z&=e!Xh>FEvDLGV23KekH%sGU=?RfPxtykpP3a87*l3Po7KC_2fA4!WG3t9{KA&{6q zIsJhZaS?Xwcum4hO+Nc2Euy*Yk^=QFDc|9KMe1O`8uu4J2J~1kbTV9;9N^Uorl=4$ zkH-^J(~hpyF+(NmqUOU*|v}RirT}el590Slt$U4Z|>_Kz&3Exc}Hlc~& ze5-i7olIHE+;fo&QShw)nVruqOqjb3TVzr}o6bU-;wTGXRyXI%>U4d>WU)!UF)h5( z>MfN&X?235f8dCLczC3-0tSMTZ&833DVIsF;fX`T+0*d7Z*7YN!(HEy+AeORLtDD@ zcf9?^B;ESYu73P&dRr8v$>fW+GKiE1hDhC|vTT|scwioqaUVRw5t(Mh@Gb+uD9Cq}BNlOCO*uh#ISPG0O?z zpPXtGG_^CUWAAke52|MVEc76Axh9C3xBy;QEce%PkS@4NlgSPIFn{EDq*608ous<> z6ru$giu1&gq>n1GHn~IUy`J=XPmn~kRbtw@|K?pg+{eZ}H=OPeFiDKWR9QomefA~< zm1_={I9*!Guo3ri6jP0s43vneYVzN**i1VyAPk#b)r{-k83dd)dKvcS?_tYv0FEYA zGN5UA!OU`4q+QfmmCc>2O9cvu@ecKXrb{VNwzaz}0Ug<@oLm5G%jAi9N(vSUSW0yO z_369|)Xk--Y39+VPd)(HZ63rJYB~hU8gU@+;DyQ4m~|dRw3+G>*Cg=;Rp!UO(M>CJ zzKpChFHm&InwWvI_?TS0m~Vrwt;-<7D*^$Z)ST1r@iPL|v?s12y1V&Cmw@N0Mm2hg zb4A@v_uSkfUh8K?7AIh;={jQnpr`Pm1dSaGV)^pyRicX5;VS7)pDPN#i0$GzrE2rQVT6A z*!8$Ac+NX`fe9&aj8p}uYXqM(wp`Cg7Yc{Zfj4)>=T*$C0u2>;M7Y73drd}{Xy5x|;c z2=!yf;_g~qAU;xX^VTKl4e1Q4KDxE6+1WZeGn6iLBMC`MiNM04+q9TpebMdA~tT*@w zZ(BG~A4|571}WkFt_K1(S?y$Rc^lyWc&@-4L;Aj(E^fge9PZHg#E(*QM)IlAZy9{T z*l`rzX36iirEQ7V%aZ27ch zghobkz)hm$zJ9oqGa~I-{k5Rv*`N^s z#c|rIp6n~rEwbl3QO5t5Zu;(j#}S+Ywjs;S@&;is!$3Qb@GwbQ%F#mnpw3Jz>T*Xf zv8d7!`m;C9cJCq9aq-}sX+NJ~s>(g^Me|K^N%GWPDOYk8N0sP0Y-FuuZ~DYs7#I#$ZcOmk!MTIY zv*z4l4=&hO26fwqu@Q1Wggi<9h_h0QVf?K-DxQKCp~sS^{JH?^T!W1O24PC0tI7W5 z^T7}TAroW#H3D7BB84!Bn2e5z`c>9WAUtx(Qt8MN$Yq+)A!n~&VC6Yi$d?b;KF-|~ ztHV!8Ok@C%^lUDk#7_C{B)JVTLzZf|WKKoWi76i0&azy)Z&)lAs(OzSK;P@a-&0K0 zxQBbpoZWrq@0+&LpL%ey$q-EbE@KRQD{;uU5NHMkHE%h^4Xtk|-PfQX?O+QQWO$b5 z?M^mU!6*NG2M)UZIiSMY#k3ponh`jg?iCCF{s64C057NTVp})R}vT zy39#mJz3%0PaeYdPHRbLf}re~L7h;d*_={FqB;$mYipe}X3ptv#=cJ!Z%S+0`uMp{ zcd|)otzp~6ivfte1IvSZ_Q#)|93yK1&BBDu1K2y;kCnI7C5xL?S(xZDzUkl}j@aZc z6=7Z-t8Of7+Ye@`i4F3i(I(h`oEf_@2-wQCjlaq~_-9+E<%vba(R>|wa+Y<3!qGc} zhLz71UfxA}KK0GIZ}cH+lW*vBk&|dK!`|tlrC*3uaFKni*1N=Z;a!`KrcyDKvC+G!Bie5Y&Pu87)0=+3J`0e5)Z}ss0 zic6!&HJvm|M>Xi6ZPnVzIgp6QH%3dsLTI52Lr)N*;PsC#UbU%e$&?Z5#J?_zybB1f z-!*(v#OROQrDTVEj-gPe;$(iIF_k|3jI2sD_KXZ+;N~3*Dz6jIgtB3S)9!RA%Nsvf ztl7vZB3-kH6|9TtI#=DKM10r(4gZZif0)bkDD#&ch*z-JPob<;-a=$c6Dv z2{km142WeJC9vnpcFWDNAuExzdvO@Z^2~7mbloGCS?VzAYl@%UpASDH1qlVird{df z6qpM3Y!)^4OAU|6owm@`M(HY~g6*ctp+z9uJO^#Spb!PxlI6AT*gP@=4w`!(V@)Kp zM*$1)hH3>bg@pj=?|9Adfcx}jXI5|INTW$)os~G04TlP2V<-5q1j821!Z8spT_AFb zEGZ&IGRSpLaXa#61wWzyt4U?5`KgMb`Z+t)+r~Bcy51zMr~GPUjVQT_>&g+u|R>A7Yl~$SdJ$e+GI?n@2MWrK#^N|zVffP2fEMr@FS*)XM5!7KKHP1fkkd-)>5HkU%C77;hFet$MfKv4 z3EWQ82!^NeV9Rl@<=NHO#}d_FaA>3J^V?S~0jF$$t&;m>+r9Y7#xN?Uig1Q;N#vgz zBTfE)Q}A6a@zh~bh+b1)NHMYbD@RU#STYA=B^z&XmZ%U=&s4EVl#!_-Zd-wwM<#I_ z#R1}ehTfJH1vS0xX&Wn(r7r5F@P%w1JSH#sj13GG2UD*u80#i z#KQ%E=Xj#aOv*UJ>pe{6N(P#nEKYs!~TBeXe$gU}keB*GO_V@Kc<$_4Ryt>53*cZwX}H zM86ayZNHZ!HaVNV*xpX{O?0S_wOshx#D;v!{u_ZW-3{UT* z{nAd8q1|EH(57QjUu?7{kjR;|cX7(Qum}ET6R4N8|FM#eqf=++Ds&sO2B)B$$tKys4?TZoZrSBnk_E$i=XK9O|@huzX`KxmHcf&9R3%SXg`4XxAn)bBIdL( zYm@>iJ4}Uc5W#`0C2VnM{Y5{#(|R~TYGWXtp>p*HcY0@PWbOKoO}BDd>0Dw#uCJ*d zLkI8@G7a!OicOw`IqFSI2ERDLU=LgjB+l~s*k-F_rk{ucjl5O$LK$O0Qwi$tKw*PHK$DcI zn5nHIT|{~dzt*y6KqT}oCmR{;UY!2tAFQQQqmsLP@Tb`%BSKTJ%t8PO zWt$d^>)8yL1yO3@YmOHxwV{JsNZJRVHa9m1){q;7ka9um!iI>HUv+gYS6AHk?%iX} zWsTHT9Ii+!n!i^x=-Yo0F~3LMEyy+r7F6UR*Cv!&X{Pv`Bp=stJAs3t_L*n0| zaR^@JGSdSF_i#_Z~mp+npo7+TQW+#$1hF zqb92y3%f>=ra%Zsv$83vp#NgsVxCL<*G~QaHx2xbZ#9+3Fo?({KW1O6q0&bXTgxb+ zpEsQ9VfzXy8;<=J)`?q6_ncegZQ9#MI(rE=Its);sK!KBK?-;lh+l>lL2!YZ(!BK8 zd{fT~2tR0{Vk_}nT^;>%dnDwv9!)g;y05ZSfM7XKnVA?*?8u9<33Pun$S!5(_e+DV71#g`(wvh*Vz!&8J#!4L zMTzVmGHcM1UYC5Oc<>i8`;q5FUw8&8fWqd)kxasJ4`m+B3O zzoCf4p&j_rdY<#dPrOEE`k);fuSe>dlD?h z0DgLir_=RKz?TA}kEfWlvveQlywCCnNROGPK@g+I0ab3_h;{4QIBkEb$mElgn95L8tT3Qtr2+5(p%siuy2+Ut*g&TKU$y2Q z!hdHF9^lNw7e%ZnLf_2;# zcTkM<3vj3@i{Wyx1-hyHYXn-wJC&0|SN(bLQCCm?y=fg1A`JoV`-%)45y%sTu(;|6 zSI(qG#mE=}4Q%p_hwq)4D)x6ooZy1WGvy& z#tVR8tGxDBy+Q%RAgrRSz4Ca()=NFd4(<^Q%TvK1)ARMYgYjoFZy#hd-4|+XBSTK0 z`G?%dux(Q=6<+})cDR9YkSrwGY1?8Y`yJxp_dNU+UV_%$QQ2hblB^j}2;l8|r&E_4 zR9y#3%FTFQ)3f$Y(D&t}N)eSXzp~P89~t>f&7;z=>)U2p1BBZrgDQ-5347cjLKcnf z=1&0|#rEiNmuw=_4_qx(>LhTn0LTH1s^iP=aoNxB)N*zmFc{iq!hk0(q~*z@$H#yM z9*`~H`Y>pjZ%uPrJAb|*92E_P*`93M79cP{MFg_1(QsX>tioaYVbz_;S@~cJo%|BL z7a1dW$mQcXnd3XhP&`yUEU%~zBH+)CAODd2#Y*UGI!@c{@+gA?5?x28)yo@Yc76#s zIld?W&V;~cIoR|}P5~YLlY^u!9Vn!DG=KOjCh9(;u(0r707sb4`Z;~I{m+X16i5nV zH<>gPR|Uq=OD2WJq8b{+fd^4?0eB*xc69In2G5K0Q6RuP;EV%6ir62YoQy`1|0k59 zmipgS4RPjqUpZ5qQ1yydL?QZu0YpW$%{t=IllN z@#T6pNrQ6)VzS6yw(xOid}n7`=(2MqNZQ}|fvLw%e*t6F9BVZD7oNlB03#td<< zw7~lCJXz|^jTf%@qUGsoLUFa4PgDib_T4>kyNxA%S}tXnG%aVtWe0fW(TZdDS>*mm zhaE0-YFI)c6m??jaiS2;eKI#O^r1Qp z@_Ujr-3OFP=T^ky%h!hctEVU}x|k<%ZeqFQs@)9g%QS|bH7b7yyVI6x z@+-BQha!o{a=VyV0Dpq+h<2Ly{`EK?_%C}y?G5|lT-@?!AjBC z@)AzsCQrs19RfsALV47TK%StOs~|psJ~En6(;EyeXwdSFqcajni$>qIWZl%;)up zquYt`xisfiv4e*WXi8R*rP*JeS^RmC+)yK~N&e=&?fp{~61ViyUBV7}E5xI<$o|p3 zVTL}g#fcFwGTo^25J2)?&Y!FV$|3I(j#WhZT5wY?|`kdHi#OyAq!2w8Gg7%;+f zo14jJz8`**y(N{_+94Um>P@%Mk1IgqV7TVLw&eQ6{nGa(MWCMm`&RMj3;PNZm}xnE zQU!9yq)@BdcmR+{egQt=^M=%<9vOOQ;b^yQ$>(?#c?wUF$r;e!@bv(rhb4Oe=a*So zU4udj{+@b024o^UWiqf611br?xLr%UY&VVktss@arK?obxuPBvF`zfRz4d)QNCCh> znIDLRpM4%?`9hPD2BS)xdE<;1Sasv_S`MR-`(;4H-S@oTH*Ga|0tmHjYP#7nN0JK? z9la8HNg)@??Y=N*Q*MR{$R9BU0-~FYcPmmsmck5t2}(d4HvjkUcNNdsi<73x!&BwT ztT)<|4&*4Is!4;<&BwtcQ*|ezl2?0=&)RtWy8vLYh=hL=zzTWndySt3$iV@K*{ZGA zJb1hX{{KXr|69<>P{VfgQ>V{5S|&jt(`pSf$_?zfunlAEt+lNBTl(1EA#ARx-?X6f znZ<^&A*`YDVZOW2UiYl(4D;)NJ|hRX>j(N~$AcWU_oGXDljF#l;?#a9{{%%-d72Fp-`3!f*Zc>LCApMft*c22O5e96oH)@RjEv`E^ z2+tqMekfirq0;%#q%Lo^U4VabN9%0dZ_U4!|FtOgbk(z|OV|oc>#`NoSucD~#nYiJ z5Ki>5GBdumd zR4o~Wq*8gGs?3^aUEd#i9$kdpRH{zr2TaPHIq|W<%bTXqe9Gu0u!ZiBBjos;cUprn z&W$X+Vdhp+v(ziDqO(aO@e=-@V}2-@aipR{+<-7L;~Cj4`v+Y_eoaW44 zBm&NimkBBfJj+TsLD{+{a1A}*TWbn%{|M&_M+)l>)5tvB!LFVm#_ zz%t}NydPSTSZa`d5W!^Rbe*PW@cNt}W+9;0==vz24IgkUv4y06XD$apQ^Q|Uj3ElI zR6lAHsuoxn_zWLM?oFvuJkO&47)t>sbMNScHt~~3P=)0y@pipIcqC=s$OUP-l!;A2 zskC8C9N_Bhm>^XxB)&RUt*Rk5{i;dtLGxp36$g0A$MY*tiD}O4vQrf{rA6YC&lAxt zinW~|I2@#uDiaWJ-?1;Tet_%h$@$-b0PHgkh`($QMx zc~;Y}#`inOevCSr>!RV_9+%Y{C;Si!*I$_a!VH;I@qljAMq>e9R2zGE`Bxq*gGNv3 z*z&bUj}`QuWNN7BP(GIn&|}9{1!O5wlsBoh2Jlm)(PBmyfvn z2=s_E%eVcYzXt~y9UIR8o8zPB^(1xBiqUps(oxt@KL6pIQzvkqO*@<@0+~0D0Bd6p z0CB>C)hI?T2?{U)jxisqAgv=vWkPRPL-+b+|2N_yEz7{3%*D;E^^5-@^+jF4$&T0V zU&gZW2q5HMa$h^(hm;HG*rpoWX-tGDSCZ~z)mh#}2$5C45CW67Q zZ?Bmb$0NpQHLLu-K1Hx88hO zu<^~B*G5eu%+b~BU@A=0rDR-ZM{?B0Hc(V*+}2T&t&&Afn@r4Ka>M+G-o*$9B8%O~ zB6`~GiO@8Mi569lo}QaK*CgtfP?fr!mCQwDBogB2Z-N=%R+)u`Ek^<MRa8;Z&jYq8%Su>R3Dq8yhF=n~Xx zr@4V{4q2z14Eme3n$Bv_TorqWHw5Ai9vuS~Opn!a{htaZ<?Oz&X;sK$tN|2^x{UnAip&kh}Us^zqXvD~Tkt!lNCx zZE-EOfm(Ba_hz=QT-k?6N3~cwL-hMMa$L*{q&(Z#A=s4@NIKZDo=ZfSKY|;~7YOfX zqMW)`*sU`nX4W=-SaJFw%;#i#Z>DZ%V9jk{;(LV@^n+kxJu55LWkcynJI%cCKPL6Z zj~|QYzWJu8FU79$;G9cC2eYDMe>?5;7I#A5LiOXb2(f$PQ~srfL8wrmEQb6w2*CjSW`ac zwfh4E8>KGbAA;r+6+EX}F4i)Z4*rd5I+lKu*$*0Ot`zYSdpWkEX)k!{ib7Isj-WV# zBI4irQFZ8nsjsOU=E`Ly{n-VZvYizR`dj&L?PGt~K2v`9b0R>U!2vz^Y*z4OJT@}l z0v_9_c5M^umEU7Pc$8#sr#{{QaWfLXeNeDyTjYfVj7ocf3u-D_G(6Uj6MaH5uyscu zT#?V3_?%0Iqm)%`2}$&Ty99>Mv|=n!^s9VlN{VA5cu)(F9V~L_odsbA5#V7tDL74e ziNxMvS(QE_LbCX|O;zX+mCb}@s*+Am&Z3vR?y?RdS!pcxAt$G+4%Gl8`MYKHcGw>Q z|K4X+*7o1HT=vr!U25NW-Q?oo$8xfufJL2pUkCB_G~F-Euw~8;+1tnx0AmP^ZY6YP z$cSy$+mKzX|ISKOK8P@tEAzU4b6TrMXhvB2J0wMsnYV@toP+UJD(neVASz;}cs7NuZNrxD|?vFTHeG~sXo$RetO^yxY&A0sL|T9B-GuM}Pdgmm16TS}=2F10c|||+d_GVbD~HNG z51r;SE7AMtOjxK8FTs%|%gptCDN8Aw&9%IOhP397GXUP{KRYV+Z+ZJ{dW9NuN_l&tJPhZ;4-cYx*WXYoIQ!md zBhvj6t(b|fQSIwCWgO0|Ar_78(Sb@3nz)?Ypk(cV}ru_7FBT>C;22wDWr2 z3xBGjtz(}?X~>$bV~LK7uW$R>8aSpaNUZAd`0TTx4R^@=eM&a-+%6ZHTuCFjQJt3) z)1&;Q495>rT8A77&$+tBHD2o7N7vBHWPNR%Qpt1c*lj;W>%F9ka9``yQDmvyta?~{ z^uPlj@-ZswNgd$}q89^^+Bw>}>Wp`wkXV-te;}}t`4u4V6&4X$0GQeqXp=&4RC?{; zbS(7z6HIwgEwo3CVc4MxxVYTv4CjTBhEyY)ye#;mrj?3X64!B zQdiuKjL^W$u9q+#)XJN?!-qF7UZ1(^N@&tYjurg3wWBEn+ubcdP z!2G`}GJ?;e5^sD52r`;kXg#@uRxe5+ff{|N!_dB!8>%Hog_v%X;V&9uvXg29h zlnNAp<%ls`iQhK^Y@NMZoN~#V{B%dRmDST7z8Pwr`+*`q&uNa^FH{+ipsjVxHP1Mb zmi5v-)qlt0et$UU-E>|O}etW99T$?w&<)KQBB3`Z`hh$NIiK75Tr;n0tx_wWR zW(naJWHs@NPSFUz$&sf4|E1-5)I~IVl^v zt%_?2qZOm9Z&DKu08ytwW%(j$ujvoA8yIKntFSZQhn1$1AAZ7oh( z*VgiY7VsUY{TkX}wS5(HTq(6Lq|IMyu5n!ECK7K0kHzxpKKPO$U zW9Eym&x_B(F{i}GCrFzt-u7<@ws0fx`d|x-w)QPc&6QKg(;z2E-+{qPU1)Z~q6_0U zbM6#fNc;`XpUgL`C48E*dM4JVfgfz21)1fQTdiBworylrDv;FQfFl6fL>WqL64Lr> zcHTG#yHRyMCAvDe0a7r4mR7>6-I2(tX;Jtx>zs|tGfH#fjW&86?+4;->p+(;)p45kv zjo!;Q=7qCI9EP{tR^93Vu2L)+g3BV;&&Vt^<6D!;myee~yyWrfi8o-y-}!_Yt?1pc zxEkpQS>4IJTIb^O>`$Xk+PRfD_ZQ*<6a;Dn7|XMEpI>FSV@hP%WU7eocdg#y?k5Nj-dV`X;&h!zc{Gck!C$QM z`2PlO+yOvWL&Rc$@bJT0EYECQ9vBzqvJeB^V*T+1Vye+E>5rT);EQ`XK zL7jr&mqfPXRmjHtzVh0@b-?k}AwVemUDleKH?d?j<@G1>*1PHt+o`FkbH}`8&#E(> zEP@SC0X*@gb<#F{#G#YY1L+HHH4M;WwIQNFRO%M1}S6`0u^)Kf?wXm+0(H&ieNNt-7(g*D3f77WY}`v85aI+Tksi#2x8}x7EQ8B_IjC zrM+Lqd!+5VN8wrHnZ(i@F+T5OOm?pU`FRtToJ2)wLM!tZJAtCiS+LpusH4de%dc(&O9p}C({3Z~fcOLB?o6RMQ* zs;bnhZ#IiCL{*=wtDypb{gu~NaX7xsb)Sv5wpjz5ooC>k6*g62@8;TYpK;oYH;&p7 z43+OJ|A-^a)_XBZGq+I0fvY6COpc)eVFHuoh!Ho)nIEO!lFKdOS; zeH@&p529ML_#+M23l>coX^KI?=tW9vfX=uumi2<(ED50woeR>MjSojpb7YYlY)pRE z$DPD@?scD344tI(rgRV8FKwsEe9s@c=1$)CsIK16qqAW_qQzgHOw6PCOkE zwDKC|%9tO9HIx#X^VEAjOb$c-#k*0ox& z4cPTLrtC+=vDyejxd1?Im3N|;c#8=UptGrn&oGkzQ>dQ0fgfN|3Za;69)v5rVvqF0EZkKYlWTn!VJ3KBLoUo}Y)YH3UrN1e@dv|P96jOU>7hzeU$D7m(xLzVU$bpLS zn@YoG@``d&?maKSGj`@H1#m)UcLGUbErMZTKpK+&#lo|_rT8yrJO=^Bf+;_k*3<_t z8>EQCZgoowS86g%|?j{=UZBv*W9 z@!Xj65sZ_U$o#YGz3UFg06ee9c5*bR3$5gr&2cV6KN1T zio_b!1P24fLWf3!tOxa~BryK;y^bYT(@0v6VD(`=kJ)`8+A@EAeL+pbDCDi9 zqQ&~j)Cp4oAM`k9m5%o76s^~El)lc5%U#vq|4?P!nbwGwM{#Po!J3}O9lnB45yuWF zubbQ486x_FVT>l6-gzlH#!3DrENC^lS6;EQ5U&wOZphBnso2Cf%IO+E8foOB6 znU8ISIJ5LC!-39eJwzBRCZ5tl^9LZ`ey5darL8DGERfNas96-v{$qIS)#rs}ofNn1 z&lxmXe+t*%AV&RTVN)Vv$Ee5|vMPA`-IOWc7Kg$cncs)NMz2qk)4N45o;_az4%H<* zuWvtk{ACSD7pe1Oxh+R(lbD{lxOf=kbF$;&;@xA;XWGDiy!ji&ioJXVxn{K#B`ca7^pt8cS@|2Jppx6quVsf)iwC1{N7HG)aEiSm+;I(P z(}sb2#sWAEod{}Mf_5$Lt#P`5SvO++jFfuzjP?*}X-3TuUNXDBcp#|>NPzXnCj~5d zJPYW#-U_%%JKMmV(E}6|FO#I>bNdz@nXHr^OSh%%r_Ix1<)j7W=9OpzF3OmS=c7f# z6wM^~n;@pNPP>~+^3XTkDH3v(+c%nzFvu0ErE|TX_!+&i0z7MhVl1v5TLyi(%AEY< zs3t)4@8(bJpwuV-j)&FnY_^tQiBBqa1gp~7GC5*mwZxl^*pd;JiYbE*PZ-LzF-{a$V z)r&bJmXH6Zn!qsg;Ql|x&N8ma{}1;TNU5|)h~(&!PC;r+K#&};Aq}G&B$SX2M@lNG zbPc4Xg$W~MbeAx?n{)d=kI(CO9`S6uce8!(Ph8jg>U0Ux31TJtMi6-?QX!`}yeg5h zo=rpezDiKX%Z~X$0qF|X%4cGO>vRv}77pUrZgx+C!XEJ`J=C@<;_8_MXzb6gkAWJl z^(wt}0q9_leAV+sh6};``iRv7JRjKUnvU1pH@vx}C37IorMW0-be8b%d3xR!l1 zver2A2cfO64;|#v$6B4#0v$1k@;3)fYcA#_+G+sr{l463cTc50*M5AP#t^&x!z~!(f)bEr9dH0RJ|tp>iUtMW$@=S>lSHGOUCQ=F1_H zff24o&FAsgdvQo?5-m<{QPOd0(s3NIH@~SlwvJ}PK$3AmYEqm{eu)0Oba+)%HC^!)QVYHAT&&iXT1i+`rxjOfPb;dzwhYk{$&O;qX`6C zixt{zhhK}YG(Wm8?MsqdS&&XB>nF7m#%R$LD7UiwEGLC1(yVKlOC^=us8l`$4nH}0 zkdt}X1ord%{lL9bY4m8)D?r8?JXC|<4m9}^+R_~e_>gwuPMhwxuB~c(Mp;}I| z(BdcvHLF7Yb^~Q9t37kcDH4>`Vn{19Qd{rVeQVTdUv{&4;TXZwP2d$qO1C#5ETrcT zaud>PrEv2bT-bKWt{cFgme?NYHH)rHOmf8kw)?Kpudo8Pm5*V<9*G=6w88S z#lLw~n3xXNrR4w$zROb;ISa|pRj~52!)na5sifZUx`NZQzcgUoGx@+8Q&bJf$KBbm zF1RD|R_XbHb#qc#vDahdX+TwZNxG-qY)^54JVH>Zp@cW*{+4(0Y2!Wd1X zEt`+W^0b$M2ZyLN3P|%5?pZ_jzZLkqw-7eBaCIX0Wn|rt*LpKu6-fS_pMMX8Z@K#V zMrs$W%b7N6Hgc-h_m|ykmMYDfP0sbILrJM4`u|x+;MXhg-PW6(8%BUTo07r;v?bT~ zuV-2>>Y7ew8^w~X=Z+oImz&C^|8E_}{NHsLK202-c=PiRC3J~xCw6l3-KW#du_HBt`DDi%1v)@}mLKG=q$>uSblFtFvZzwC@%AHkioNYF6Kwa3P|8AMpOub|Q)fH|f>(>FZ>a@072V$` zQA^#Hj3`Ojxd)}Yhaa-HUXk;8>`&y;+V8QqnI8%6aNbj}p=>wN2r!Il`##cLoJ#I6 zy>AL;Mr$!jMl4sPI7`IPs6(uZ$<28-bm5&RoQWQhu9NGeU4piCXjfTlk!&qWM2QD- zwCcT`S!(%%!n?gQca`Mw!8W`oQ$3DuTzC_?tYO3KW16jd^yZt%6?(S8f_#=x5Lsnt zSHt3jOswKIo}>=X%c)sP z$|-_HsDR$pI=eho1M@`lDTPvU_M;ml4;?uuLJs5X>)U?PdSlXZu@L6B-x&8Krhw)R zDz+chm3y}u!!_y{$BwNf`k{>}eoJl7raR{yBd z6fs$!!MS&(4m#>qBS2ViDlT;wK6DL;3E8ern7SkraNp;JrDPdm3#5F6b=`TV=hC~_2(wq&$L3-Qj2eM*B)D5$<(%>rjpAW zj0Q@lHcOo?kpS6BFhWpAw|8_jWZkc8l~%CG(FRvbJf$cd5nCs%JJlk0FSk?;H3XT8v6u#v;qgpR~j@k2BsK zL&#vp0Cow#R9t-nt>9Bl!%}KWC9oFdn703vy1wdy(psVhhy^WFCCs93j?8m>nSNY` zD&Qaj0R3m^)=M#(+)vv3jheD=n+FW}n4>>%cw#hN?AISYm}&VkMddEJoo2-I4E~am z5L-VVrOCFYY3j#jz&mqGQ9*qDY#>wW^5kYuZujOANZC6R3%n-I;Q*Z7K_0`^KsKIV zn}rYpS#tY?-j1YW;r&j)Nu%G+yMHN^^ZnTcPU1it2>>DXr9o&R(^kqRZl0Y2>9K6Y zfldKV7OZ06O|kc$V-#CE zTW3z^U5m9J_dHrU{iggK$mvhYs*^~jVOA;|r6W!0tSiT!%{nuGym;3Z1JkiWuGOyq zX`51o)Rvjbdq>$?UR3R+{Z*4EfP+d>TGTGtvO44aTJJC2XjbjeJKcC>k;=oJr6C-r zERmhPX8a0L@*Y<7mn~9*YrTLhSeQA2ZJ5rEI>l_$ZiL$h_KkujP<1QOfnVhfw*@cR z!y(mxVd6gqAQ=e1r%|UBZy-?&O7-o!b+RK#^Q+;V&S)!SuW@xBDxoSAqV8`|)%;&p-eq^6xO0I^pI(dq+ zA+SteSLQ3(T~F+e?@Z<(H&)k|_LD3a0}o9O8ms6 zeDQx!6E_r;l9iw{wB`P3xB`mdjKst6S-CT|cdcj+HLynJDEs37b(hS@%yc=~egeet z0ai1^q3w>6_u|M5`cA)_t%svB@>(>B*Yk?D4G z_};NWDz%-FyG&E~w9^V9Km*c&zECE3VqQ$-fED&|s44eZ$7dD5H#de>e^-k>28Xww z5Gy>(YuD587E^XQ&$Dt4r^T}w7G{dgDNh`{2^F)xJnL5+zT-;EeLj=e2=bqmq;oy_hS?Kf z(V)aai~7FYkgmrD_{B*c`8*emM1-R3iPBV67yshL!OO!L3-9K$+NJ?Lqlko$*6!hi zL_%-0wPyC-=DGx<-!9;v%XSr@AKdc%#XMAY(WHB~L2)H07vWwvy|u|*sCQTr|28QU_$Bq98u&< z&tj<*ezD`7sSi?TC24ocu1KJT7KI%wu8N8-=b3>yPB1Jbw*wI7Xo$;7e;L_&W|=v- zjckDb>BRNCK#0xpqUN{qt?y1k<$gitAB75jJo4>#t+%L5eg0IFCPo>cjsX0AfB|`= zu~FGU_#5?;QHgIHWx+uHy7Fl7$5a1WrBP-KS^IViX$n$!UNYWlbOT$^anaBBJFJ%UM10sCqPW;VWDV3V8j{O1V z|4nq@c_DnS7nHpZ%pIxnxdXEh#vD1xR7f3N1+-KO*`V@SV8WX!}BbrX9+&-D%>p0BXgn!{V_Bovv{gYl(4~7%nV{<>03$vY{-rr`wa?*Gx~WTfkqFSSSUTY|RE+a3 zOh??JUU-av%*d_`5miugl5{*g90n-eoo7E5&5Q+%2>bdR6$|wD`b)ZOQZ&-@I9<|^ z9`69S1NFg^F5ehI+S~`l!@Daq%EW&aUvGr_d>yQ16LHMckf$WfQ6*Q26ffe4jLa$e z)?^5Sge47R{$OJR^9Xher1AZ=jCiasV$1Wl;98m;z^Rl}rqZ#7NybzcB&Rw>2_~x{ zYRU#D!Ih2uMygSy_)IYguCm_Q9l>|`_!tj1_sNQn2)^stG{3+F&CE0&&aXv9cLOw> z93Egozo;r4QPCZ=$F$Q8Lg`uFu2oR&@@6=&&?AGJ?9GHOH&-VH!(Js{?)AuxX)o$& zSGU%N==|c4w~6|$CNK%9!)HozLbr?9I|w!?=+`^lC1=WOEoX01O6RF;+nrCYca^6lHh^LYGc&?e7-rI7d&i(pdu}{X_)n%E!ZfUauA+2lH>!3= zC!GP^+;PGS8Pm{6O3Qr1 zHN;bCW9I;u-j1-ivw^Uk{ROYtm4VFXzReE}(_ZVQsp%V-8)35H48XSYj>EmmKO}!> zZMlU$J5%c)<|1F!w{c?AlslWZW^u>*bNqc(7P4?>30fd&6M$*|I-tMZ=~BbHCi3Z5 z`J&GSUUpgFv*#Bz_2}2qaF~`Z)`;NTKoorEGrFiuG}&k z=Lkb)q*)q2aI>m``K7GgtD-f=u40!`IU>_)^_&W~xpAUH(kST5tTh^Q2^pD%xtojD zMIcBUhz7i#3cPrL8ORnqxqJV2<*P#K{LA+gv?0O<<{U(jPl)~YFU3Sr?2$~mg7@!( z{x@QnnY-q_S=hNw-tj+qnTG_F6*T~GT!Em?F6lX~dg?2a$!fvL2LYehfj~Jx(&7Ja zj0KG0TaFKo)mMS0FDi-{5Pk!0&7ZaJjc;1O?g39Pl4t6+4{8l{shVNgt*24&-IKtZ z?9;;$%(tS*!0R>m9_HqfJ770B4!CtjQBp7ns{RR`{%emaH7lu$f)Qvy98crP!S;z!8qb#b zC6qUs;q6I1x(3$-Ye|}(zO1r$z|lrO4wGlnu|#cci4IrRp+ff2*xtO-12}(O z`f(EqQsCY^;W0<=b~Mn3F&eHCSH4V@WJD zkJQ4p&sqG-*v{)1NMF&<)IRV)iv#4gHZ*ZqHNRK8H;mEm7bfu7DRn^ka&;+i-YfeG z$%;+%Ro3DgYG95O71GKAfG99m7ne4_78{Fv&%n1eZ|jQ;9@9GD75O(;e-bJx%K1ec znd98UEwnlsALSEvU}LBv;C3y7y1jOd?9S@rA}8&X7iJ+v_Z#agnZ&kwAHdVSYh*cBG$^AkTZfM zV(4RWFOr{sOq!N%U`?=C?Mn;}6sjp-e&4I2{ise|lXJtt@uX3mdEH^7bX{5ujSZ$E z;9U10j}2qk&I0Tbu@IkuDw>E!eHwG8I=b=og-z^qb58a6jp4Rq4-p(c>d5Q|<(&{E zXs`%3D}{!6hZZu2Ro3Uk6ShZeIo6E(aXpkt{2|FS@oPYc*xE){> z%@)h@G5B_2VNXT|{d4-mh$R|?L-lXssHvxUZy~;(yjX{h9GpBJiH-SC+Su_XnYW;> zZlwVa?Szjo(BQ>=#;gIeY{TU1jIrP*Z}8lsUY2~;e`6{pw2&kNNU{*%Aa??6=c4n8 z>GQ2q-f`%|qtNeCU$Guq$!rP)_#LuxH(eO;5m3xYsB1_(pG6q`VQ zA|?RvUA&iCTr7z5xc!IonA6(Y3)p#h`K)Dn(U9Q7FX}G}SVHDEH~(eEYg6J?X`IfE z%*pJW)$@YuzF!Qjl~y~!nt4mPhlLmZjp4IzEuo0h ztu=Q3xAu3FQCUh--3f#KKgP}VkN?J(Hr*2gkRgzpD%}t9_|(MlQk)n)?}?ggo;syb zsP+sp-AHIHX1E$YR-3<;nG!U0XH#Uop0^if^c{7xHl|)Z{Zwha-z}lS!ArK9%+o0z zUB*8Lv)3;*tKLk;L%qwI^w1`LFpLUQb`&eELWM5R2yGeYprV_(Wt_GeJ*FG?z1|Es zEEE`8*JYK=IlkGBQ4@tF`UO-j4t<+n&olcZwQyEa{1*ToF!wTpm)uQ=@tuxfOzr0hj=@sgEAQK+h)UV7`s4kA#G;%-+s-lR?5up6li)=B&`Kc7f#@YD zlr`C%9Nb#`BTcQ$imKXD$&yE{WfFbn5tL66MxjGVp+snltTN^^>_DF;Xp?p!zrE(`HJ}3(Kp|Q>(fvpdm{1yDFOr5600}F7_woGxBcsIIU3Tf} z-TkI_);|}X9hrsTxDz8)6FT#$yEV2)p>HTPh{}pKlwCS#K#fls81|#GuR?Nuo%K5f zbmaCt;w+v95u_C~CXnmg)5_E86QX~3I)Amp1Uy!yrW@b=9$kK3<){eb`*k1!bSV)d z5p=Cf3BdON__-Cit}jc8{Ow$uj#w;e!Ec(@(oNC~G_E$_`PP@A&24r3-qxkBXd?m9 z;o59g+4q@8Eh}2D>0Ke(2*mL4VjWVKChe9FV)-AnZ18-XpJo@xdD z%ZM^HcipeWK~Q^7p>js&J0Jxk+(&i+!5GJI;t*6n!ywUZ z>|7>WUZv@H61@krnTgqc2+UsMuX2pvX;BTr7soe85M2d!E5$QnavtW@c!nNB3}94@ zGvNEDL*`Fq{5ri_FKz?uvWw|PuP|YpnFru9vdM+3@x607oAIOsqR-_@(%T@8agZim zRZcZpz4_(4;}gf(pWJrh_PD4b+8G&N5L?lbPLTW9Z` zJ#JkCaH-MCDZsQ$ub9neJG(bpi3^Nmb+1ZXXp3+3+TOdq*x4I8Y3)&A^pC5XZP83F z020rAGjma%oSOuCN^PSyzt6hD&&)nxKmK`S>IT4=x90+*q`U@dwLjdQqNk^?Fy;fm z!1N3ZBEfiD!x}1OmaG233ex=Y_17)$0>?A;b5TsO=CN^cXT6#Mzk#>OMx)7qm}~ge z2K=<&={gB`E#S0FwDif{oSxlwJae2pX*pbb_5x@ne`e#$`~6=J)c+e?G1lpevKWN4 z5g726qtQnDdv2_&&1PFsux+X_{fW$0{f906kPe5VjoM2lxN-|P$Z;e&9=C*G;~Oh> zAmqMN)@;E``{D6a8HR3@y$<7>nOxI&=HTxEsVx;~JhRPPYM8zlXR&tjpVf^*NHn7S zr(haYnYT>%%~MO%*dhJF=nTw&TLHIv38i&MY5=I<4SIlvR3jdlqoIFjL|Hm{PzBB0 zXe_=(1lKd?(Vnz2LCl<$#96qxqbQrnv=tj=)7SA!j=8^ENz~P9&MJ!Eo}|rtAJ8+r zMewjeFgSK`!a&f_u1`C+IfHwpm~aq9L)#^&<^BTcVZhE=+?B`-f1`(|N4Jr;G3EF; z<5s5H`aSM^gU4*ThEaLI?xnaXr$o5;2W!lXtD4&W@a_fZO_njAw>Fj!yT0SwWlgud7C{)6w!krPeWdr6DVs zx3Bw1dp?Sjsp=wA+#=2ThZam)PBP)n?%thUe<^Yd?vSBJiAujtfjRq-fQ54p2fyJ4 z6k<6+)w%9 zDQA!}YcGtI>?Pee9Dx!Q7Y4^Am3f`>CTEEm7|T8391A46=kxtZjQ-_yZz`sR{@~QL z%{wa>r3C&!p6)U8^AVk*+uUBZgHgHT&MDhQKNKW=Q6{X73=mt{?tOnzcy~MFP+V#g z$xfzB7g?pSqdJBu|8V|Z^sZY?T;B|HR(3(-g%0Xdwvi<%DOjEs-0*q6f?I-4ZQz{j zJFPYqVK;P`b+7kZUFjE@xCa6X>DYG_#~Fg=r7Y z)*hW+pD^x?WM9X7&H6_LFT583aOJrs`%K0w-HxA}lbC5I$?1m{S>SH^DMo}Ui%h7EM zw?QD440?tkoQIZL{Y4Px$?TLPUvOX%lAM!DO~S=j7QVkc0ibL%6FLP?wrBDg3k_ae z99pNG($}nwMF;|{f1oDnGHdgeUIiXnd)|Ot=B;e@8_F)MMA?YX?T~oYb$opQ$Re!n zSLN2j7S{6#;+c&4M$}N0Cjb6YS=9eeY1xF{+(TfTuPednGq`gAAhzD0%m${Qho~@F zGJyOy1E{xvD1;SPC9OnNCLm7=23)xFPk^FwvNN+b+YBT(F(3VotMG4L?2qnf}hwC=j_>Z&_; zvNp((<#`l>arRl=L{FLfGL zT+78{l%qcgK8ZV>ZS0wri2bW9Xq)`0z&1hou@9lbh5Pba{#$I2A_#ptaIxd&>DnGP z=M%SL*Sp>;Sk_F#gHL6PEMeYykyw)YI{>p(*vh7ECYgEnA->mNkGP6jh415q&+YO; zM(w(mK$@qEK$~=am~rIm|B``-BvK91r(n)u@OupC7WSnJIb8dS+G%63k@-)YwVJxB zFB$aCZ^aV?#%>hXeItqE)60l+JI@Sn?0GWx?%gl1?c;T?#T0V(Rv&|s3WI10F@INw z%vFcz$LSss^2`{Q(pH8`vFxP|Ek~hN-E~5=8p`X@9vo#Tr(bqQC`=KQ`_y81dJWL`yo#R>Ag(MVYo@(( zLSpXhFaJl)CwA!v=gXO60KHG%^KUpNC9!(AoC<4Q!#}C0{XUmcWR9bl^cNCPh+ks^ zrt0v3vk>4pB$%a94|vzCy|z!=^m&phbtbj#a1=ps2f|}%fU#@at^52sI)-wRWV1TkVF(olZ1db}(B;cI|P-ETL^8Or20DSAHY_62MvRh7t% z@vd@K*{RGe6K)H{gs((xt5fXpdEONTwAyoJT(0KulAi3eTNjUS#o%2!MArIk<3XdW zexu%4S%u$AsiIKaIlbd&QwRA*REtf&I~p!&2%T5-N#Ozvq98($Abn2| zAl~+3%{BaO(_mHUzDNk`d}lbB^Bxki`OH!gX|;^T114gA&7FUVLYh#oU+c#tA{7Y| z6BX68bYKC>U-FzIMm?llx??nVsS_s!IEj+$ty;K#(3GF;g?XK=pES~Y`;;gX$mb=d z6vBem&#Zz^?LZkAAKj8K;>GPL<$C zzp4O|x{j~9p`K0>J6d~R8j$S?0k+XnJ+V9)qxeV-v6r`^CP++Hf3I<4ePYYFqDKjM ze3-l?AwB}zE|&O%45ZeaL=i*Xb%@vPma!?56@0=6y1nj(LTRhgezo&bCl^hDg_>d) z)U6x1DM;qwH1(aBl%yC=L5{qg#%R9uKQ2u7^wbzDO`IxP`f##88+86kKAD8=tRFK? zUQLRHtj03qfyr40DJ{FaYgaGrde`8vy$cYSHI|8u(2FDpfs`Y)t_V zainx%ky?$zqvfH1^YO!xJ96Lfaz1@UWbE>UCq9c|yKmCvqUk(NbtoiHI~2aWomFWx zI5pzwd6jrYw-+&>V2?lfT9&H5KIL2~W{7|(Vu^tBwz*(|nP~a}V84IuY1}WzTiD?r1~Ky|1HV^Jz^^`Lk1WQB_w2lNY8Mf|Kd?6Rrn#=!oogV#PI_s z6Kj^kk;6g6$ItJxfBHldgQ=&FN!2JQtE;Ay$I^(+zZbpk!IIm@6T*SK8z3}x-F=b( z1DHxD?7Lm-Sut*XRrdERzh$eNFalomNMi->C$t)>no!&2H^9@>~&RL=JGtl;Ao@IiRyxC6=3yx0zw&Ny1Y0 zyT#h#M^`9U^Kt3nrXLtQV;&gLSO+!}Bw;{2^23K84}Gu7{bH{vfrWvk^i#lm)C`N0 z#?-Cu5Or?tILM#*aH6+-99+(QdND`yB7iu8@#=5bWf%N11El#>h&IOi@DMq7b^WBr z>nWKE5ff{(W8astuXDI(`Ax}*Eok+XAM92atw%vG{-u!xUiwAdh_U{^+rodEh;Zuct0&}Eb}WgRBz z-d>VYt?)e8Ewk+B<(XW~n$cCIE2as@#+nIIxEeI?xJG^pXPfVd0YYMO3ywhBn+U>a zv!WqcuekQ_R0*4n$Gjf+sPr15tv@ zs{Qq1_Wf)j%$dpuNk+tAsX6PDP8%Uc_o5b4#lfm0(Hvky-Z%Rz$SR(}2G{)Qgj3HE z&)dvwpW5@YlanB#RLSiAy4#$tC9IW58*?8~XcbhSfXoHhx8qJE{=x%wQOSKyMzuvn z(=Cf3tYEs^%#mim+Supf+MnxyjSCjfikI|x*WNugJV5hnAS7VzG5y29SIN^V5qt(8 zl*~(|x{HbiWxICS^CD@EpPLy9X2Z0o5m?+X@vGOgy$#{s(i&&|2~V@7A%>F~3kwpU zo;3MDAbn4RriyP%>((bq#TK)%(ABnTHcU4H2wThrw@G>VW-#Qsl@Jq&DHA>RVwQCBQ+o zx9r*-TuAX$`t$C}S@b%y3SE>oQt|s%Ow;?AYNHbmetmuH;$${2GQ zAKv|-42@uN4b~r28;WH$WL_^s%Imw1YmrFBbEUmrU^kwGzG&4kX_hoQC{L*l4Hr-j z{_rhxV?7YUl*z$wEK{Bg$&&Wzt1i?$o7s_FSek3;!36HT2{`SD6PJ?8pTdzVMf#ci zKp@k*ew6zO!@Q}r&0;)k8>%Jrmk5EVU_gJRtIHJfELPNiI$bTlZ0D zNF?*v?547}Xm?JcnZRFHck$)J<&dSNmkkV{J=@$N-6JAfG!_)2qUaJWk_@z3KuY3r z6n_Aws|)ym^=r>=gO^#Mxcf-K((JyQhtq(f)UbSvCk0*VzSe622;@PS1NLSkBElec zlpb(DjH}!fk$JMC5^P|K995>PTf$gqNo}GY{P7vx!XaL0^AXDS>V*t8?q;bB!KrgF ztwo8aWGMU{r>!RT>;q;0K@JK#r(gO`S;%qf&cc1RZ`s^zYF!qY;Q=@Afn?#<_8m$XE- ztAyb}F8q?p?o<|iDtiUHkKlPuGJ@ttMsW{lJp6mq(C^uzEq9Y6*Y>}4p-M;HL7+U) z-TgaN{$GFQ5sN63*Ps9 z(+BKrRD&vYnA`9-8!F2#U*cV!itLtJ+=eZ9>+j;dzD|X3-0)rQY&(6OunKDPER;Q{P@IQt<^{@B; zb353D@@roHJ~B7KTJ)&G$xWNcjA>|c|In_j2W~vLW#M-k3kmTM3f;2}Doj=GsU;2SW!HICpAi@ta~;usO&5D{ldN%0_DNbHzRK3k zjN8%!lcl8{-%a$!HW}Uiz*~!mi=vW;HRT|czlRWR?0$Bc)3G95@XabCPA{6T8~|KA zEGz`MOJ=fbx$9rXD?Fo2oZP=CB+iP(PQMdcQ`uqI&Ajk`=qvcCA@ldOu6KaR^iz;% zoyALpVC_?ZpUfc&&jeTcdTR(OXmg8W+;x*uU1J235((8&h$bsfw! zF41BTEiSX|wqxOFio zomf_pf8p2B$AmxsF6}mb_M7dNaJoIfShm7?burOg%UtZaH0KYQ5M3 zU2d+)t%k{+hv(+9D9Rwa`j=$?WC>TMc&G$ze%Q|4%b;`^q({B zJJ6`l(@syc=9j+Pf9rvU0HuXtUt3#z^$3MV>?43sp{NENzZ?*64XMEG>t=C+AipHH z0GBqvTW1t-g$rESUknnDkPw$F7}@&j5t>MZ=(2i7h||tOx!fJLIq;p@WTZPXWdXc? z*_h9O_fVNdITu)8{s4Pop-x%YE1#nB9?1znrvux%&$DQO0x*-JO~e@7LJ@xMDeI`1q$tyT*oU8-T)MMwOz{f0ba z;;KIh(41=0QrrGy976?}XEVwt!DT$z&WSSJDXoRfy!5dHEZnC2q{+X@5mu9JGEtvZ z%Pm*EcA zY|G?GU2HL5m~Hd%@;{S*|nMl_zo3OmyzTt$JPVmM#eXh}ciJw+-{Qi>2> zw_1wpCY2qNh1>dQ;eh>?NiD3vDT;t^O;|4|IN%jM_s6TmV%q_mDzn-WV^^I14?j_b z3`Mn2dX7a_^BKbm=x>Aaj9bl?*y%g+yh7Y1Rj}|{SolP1CX})YD+FJa_7(j)ralOD z^BZS@Ff-$|+x1m>J2vdQ#npvU1_dt=%3WT=0nMkBbmOJ9azdb~;U%%MA2_2#Rq{dxsmy^T9k!lLx=BEn37(R93(y6%|IE#>r->qtS&Mwz89_^f* z%}%}iI3t#xX8`vSn{{=R=XI>L^fHg8$S!>hZGK_dOu$d7;anD}O#NIpqKqi|dquaY z`v(!npZF=)_E!d!{0PS3L(V|e5NGD<#Gs~cKWO;R7Khkc&khB%Wt)~=n*iW&cY~rQ zn~B|?Bc`QV>*njsqo{(ofQfgtL--Cn+C=9w(Fws1PX<0V`Jcou1@jq_lW~Lg>0+en z_}|=b`RziVV97Ads>Br`SoYXqE>@TFlZ8o<+8bTI2Yz+z#^pQPh&$@6YRr}q6yWj% z!Of$;R%Ds1%(`i+4eu6||2R3|5ROkhT33xe;i1@8R=Zv%ZbWo|$BQ4dDuA^f2^Pw`P?aWNrKP>pOJhA=ER>+dCM+YEI>eby}tT1BuwGC(6YBfx=R^_g}PAuLD_F$}UbIeM=GF6~X(O>{<+{s8Q z&Y2R(>V2c{MM%Hq0uHsJqQeL*oqS_JGIjs+d_^uL`mRp!EBYQ)Dhefy!?Hl2E3G=9 zV~%77r01vmX#Vs(%hDsgxsl`~xX6N!*XOnCoBL&))Rw(Pl!rS3zc6tDA>oaAhHAR< zEiiNUm2$R-2Lak%A7||A5a5AYwl_6Yiw&1j z=(?0C^2({<>Knd|+6h_Nj-m?b7A~Mp$PtR7}N3Qk|8Qf~5HGIpML9 z;Rt!_&&cxRWgHb$zh%zXW!P{kJB1*+JZy>*V*t@gG=8=EYp%oolbiS^vXclQ{HccC zo#i3^_wzH!azts7ZSJ#})vrH;sUa14K)>Wqu(ff?wPk9w&l$G2Rg+XhO#Fl5n zPb$xE$&(~2KW#ERb{=bQAeE+N+{G0-Y1tW+;vGn@x|Vs1>PNfv7x4_iM7^{#r>_4Q zxuBKMvd_HK8T5>9&H_Meuc#0tp_Aj8cA1QB_x`e7mrd=eUGnoZs-}0?rCFD-;MsK2 zFT6^y_{VGtTp@`X!U6fb3v2@XGU<|E4TT9Y3Y;O4@B6^|*axhp9-)G?r4Wz?4){#n zh0;qDUmeU3T1xCM!pKk?Ib3YvEFfw9NrB9=v;B$GxK{5zP;2Wv8b|{6iA8-h;2c&J zHNm;%udQugc=i~dcsIEAR%sJR8vd4V?YJ6YI@tkKmK%bb`JJC!!9}{Em76ocGZV)x zRS?eboNwT{(uDT9+NJE~91eD{cBEMhmMZ;H8!k2riw$9q#OCd#1yp@Wp+1!4oEae0 z@{5f3FHsff{PE$%hCK7Nz#rrR_DWq%kyn;B>SwQ$3uXOveG%ykT_3{+nEngOE_X>} zCvh`iXkI_`0)j5gN`NI*DzK9u`_&YH8grvpc%A%j_Y^ghx@Xbnit4lhNvo)dwC|;> zdY9ErQD-664LL{s$`c^CYxwu@>n~^kUs?!K`KL;-WJo0cn@n@1ZCN?BWrXOraqj}= z@uyoojKFd++s#LU8D3sF8Sf#KGhX;wthdB58{cpE`E^erajTWf)l{_LM_@BzRtfyt z+&nzGyrr#xGs%F*fbTJ-(gewC77NIsL&P39O?+KKU3DphBlfkn$of2WT7wfZ)607W zXN%JcF#6(%%qwlgE{Ze9W@cota~fg+qkcRqr)`?#TSK+zp+qIHp{|A>A0!a&c`|G@ z)}F6Kod0^bFsFx1O-c2;pSI6D&P;$tfQAGh(z{DC1oG9DpZKp29339-%wQ_uqK=w# znV_V0D2;kDc$azDymw!_}hb!RYR3zAb`T(5rJki+ITOjXnYq$8< z0l(TutZ%pd5qBAN;3I?ciANF}SMP>TXSsKt|7-T({~sk3^#zP(HebeH`ss6VsXHta zk?WJ7GOU>`!V|H&T4&=whvX`!vbb66j}+S^u3gSL_b%U=|5y$$q&XDgSJ+92_8Obx zy^xLY@jA=2MIACjrWiKqrw~Zm_84t-M`C zkUOGvN?B;?=#QNkP>YM?`m}Y`D1N6QWmO~PUN`Y?+K>0in?Z+^wM@&+_U#kR z3-_w=-hiWPm^PPNZ#v$snZra4-}a%lFrNK51L{Bw^oE>8_T@KDZMzWPAS%uGcS73l z=;}(ijGX4}%3OUUHC1XqTZ|9%2cpQyDf2zoCKOtUU33KDD?C^{pRYrcr-`y304^b| zruC~eV&z<5iJj%*>8Vye_VRpr$RsTUP8_GgJpht`OJ^PoN!|B5JbwRH?rTERQ?l~$g-m|oxTTlRR+}3%|M@*!XoSMJYf9@^CTw41`(d9(0jTPMm^E+ zv9rH+Zx+DNLOk#xl;DRKfE1MEhnMfqnUULJ9ns=?G!7h*MgID}Y_*Om9((BzBCQmf z{WmF+g8*P#P>oRdAY#C4+^9{IpNBB&GFVcPV+Qbz(P1kG#{=KzVEA};>YcODB5ehnCTpEZ}E zp;itj)kmfjDEF~mZJ>mxck}$+yGJ$yli>*U*t(>m$V#I8_A6qxcr@}LiYz-|pY!H? zD(-B5h=n>z!T>_w6FxpV)B?diXj;TaOkBHQ9v(d(%w~onii*tsCvkuBzm#S61aQK3 z3F>$aCcO26AR<0Ul`Ew)lQqv)Z6kk8a-v?Av3rvD%>ViB$F<4aj)D;zxY_pPaa8!s zC=^rO!Z5}hXtE7`p@9tHK?ACB_eO?c9)!i(jJ#u>PTptcqYFSq(S>cr?Gs)G-X%(!0>R`T6}`%MR2)vU237b)K0F`7mPCosMx^1zdB9=f*cq@>&3jLl|NUh}FAw z`VTTM+;oHkFOr~dbZ9xJP5BR|bH=ER_-GCYyOHZNgQ&vT7s;zXTO{-4Xx~WT| z_$#r_LZ^-Km-N7CG(}!qQ^534K5BYKMrO7O*cjb>TH<8#7q7w{qcX$V zEYF1bCY72N7uW^f>M`-Q7{ks%-g>%KljFjoIwAc9PbkAeQ?oG(9?Ft~zhUPO?r3Yr zajGUH+l0(FL45O788k9^%96s>`ev4#q8)QE^)05OfJW+kTCe@b^ra(uM4a#y9ok;^ zlRjINN1FiTi91D=_XYiv4phppda>+=VS&frs)?Bb*-|PEH;VVWTch@hjyQEha025@ z8a1LkTTaX}Ap@pwS^AX*e$u{ymBij~@1_-`y!>gvAAHLC93lS&$8u2EpP3EqD2wYl zUDvu7T=p!=#BOKBqQA?#ACL;dRh)C3FJg029RpPXsZ@C~khiRdmS`w~b!a$t({K=} z>pn@wk+Wrl!x919!pIrYPeiHw(T{g)fWsm#GKUWD0aMfA4;H0A+3Cy5x3v}XdQEa{ zUhrSU8{cH0&fcjVK)7U;Rqvw4|4wC97D%*SxF|HmlZEygBt9t^mbDy>QfMCrnks#}0VPtGh1Pyo z!V|_8k@;M&j~P!b;0u2BA-@3x9e{6EGt|wLs+5 zpx{(@(mYr9e;9kquqFfk?cXF-q*EkDmndCIvmv0PyK{tegA&pRqm)!&AThdOfJmo+ zjS^%h#S!p4W{pnx7HA<7A5DiBK&c ze{5zFBd_-`acBDd_u{rQ3)}Z|xTO&(#*Y~vbOOFfi|RoR)KT7M;##S+55+twY(*>8 zDQ3JAAz==NhX5j~(FA7I*jVaE01hy^1FGr6C3<1`GZZWK!~NyAy}e;(soSel3BSGY z;dS7c@~+vyoyMJnO=)AZ5OW8%uSI+M)+&0x%as4p@cOk+WHx}fd+|Tf6@c{V|KEkX zrbwaT=!~PSj)m?qcQ_~)ya-KLnj}2PCJy8g$SyccKdr?TRwCkS{^~)JUOI4cyLAI* z0<`Vs0OWFXBhFoH$1^#e6`wP*wii{|kr7n?i}?=T-N898AQT$NPy}EwEpw;$U)HsV zJ3@SMQa$D}=Zx}iW^OQ6p40_odfLFjRZPsJ83)GsV27qCy5sCuKpYA3abHDl;iGam zJBLmQp$l1th^dhApY2@zuF=`KhOIdh% zJu2e2!H}e7F-z5ME2C4p2G^{vpXs~8zO5l!#EEJ`tQ}oI9wBR#d3T!eMbF*s4{Okq z`6gp?Xt--1Cvml#!GxLrAdI?{h1nRbt*>=*X6`EZ(+VH*~g{gexXlsIEwicv6C> z`SR*5WxJbm zOd_poSCYp?FZt|nBbZRi*XvipA^zbQ4Km{e*;o*J)76iiZn*E-#KYg3!OHyZOqEc$ zG~bHNVH6C-#k- zbZ3C>z$))3oJ4y~A05r74W70?_LFR*c_BCvlu{F<8Z_$rG=v26Cr@xJz@OvHu=Ix&RlB;KG(f^cN{Y6`#ep)tgjYL|&8;&?sf1W}tFU@{ zcfz{i!frzI@>}=g=EW?8*41{0Chwh6gdH}E zJXV`+7g^`D9Q}K4m3N~;TW02~9QxjMvf?*J81LY)_mc0CX#L;ND zYIYo?0kfM+HtVG%u83-}ptYU)+mq5W%IfT*_lhazhWuMlI_8pk>kuDib_8>X5^8eZNc65NLp}=A?YTbf3$M%MDdu~zGVNYt5?cndjhp+vF*BYH3i%RydTMq@w*8yYLp=i!U*{Mzz~2G6$^9vVxfy~r$KFCsuZuqAPDSgKC88NxwUYjh%}m>=lUI16lj zxZV2LnD0OM5L=`v6Jh* zXx@8iSFk5sfb`XSwp#oc>ptjLe#BJoKq4mZ`+|UR2EsEnbN&a;9!XO^MH5vZ0MP`p z?&a%=vk*Yua$H!A>ZZHd?qODfF*&0RG2ph)^Z3h@7!j^B|v-g&?{b1fQa zp8H`huoC0$D;a1hA8N3?ZB(P0mFJMxCF6FL z9V6B-a^sDmJ%hlMb06sG6Cm#@g8==RmZf~b<8r9YgCd!iMOrhG^v(OeA#dO;yDl@g z`c;4o?$$}eXVC8&*3fflv5#&aW(@4TeMRzb__Fx#!pZiD1YTW7gc`=nnEd?qQbrXBP4~4 z#m#JYYm7Y`W{T{w-c0+B89ThLmFtZtYQ7FfWyUAD?A%Tc*fDm5zXO_6Y>FG|Wv44^_8N!NQk?xojC*y%;)Q_00 zmFDn^KhvY-B{l)U)xmZ=;R^O1mUa)Uqtn=0C#{~$H&0lWJhWvKr)?3O>aP~>T1hUN zNU(=&yh`s-a4SKq4U|{20J7a0;%Z{4>Z#}Hgf$*rf1g+F|5s-Eyy+0D20hkmcN5(2Iryc=$f03aF& zOZwXl(FJ7Q1!_XoiIx!trN5MAh1ojB9Aud~?nYcO!pY=5!=HtCjW&D*&>p|pnUzVC zi-LMw2E_O_JfJ(XqDwm&;*W6r7ugjc_5N86WAy1|?xF+dU2}%}Hi?Hhr$pydxVRSk z#}C9|JW(BOghP}7n06d{th{?Fv{HDQKTvY^sM998;Du%cudAD;naYa1|NZJKJ>zm2 zfembF$G+W{uY0KlxSZjD{PPzlpSOqT(yt|DDen)z)`}LA@o(&lh@o*zL@xdlVJTQ*{uvu>0@aPeTO`65}Qm6 z_sH8gtJrw+TvXe1Bb9eh5-wG|+4Xfx{qwCdzPk;QPrFuQH=8*S;bx)75v`q6*-AeW z69kbhs+%!*jB!e@4Z_gy#CtH1%sEz;_H8jgsF|*lsDdHQMLu46T%gmWs%7<5LTn3a z%;_}mat;yj4BRMTj3CWIU#n%zJ_6qSA`(;ps;$4LhaccTvqJ+IK-az3^Q`{*+0~{1 z{~&;WaicOHK;8k%Tnz+kB-5hEfq&Liu91ZoQ3eDERb|4dy$M z`?cWxl&0)G`+7qkpGJSd`ycS6={hWQ74p$PCboxx+Tzx@GsOl~+GoWf#>-x-jmXK# zVI4C7)YVfBDq{`>Jp7HXUvG^crCrpWcx=waqyrDCIXfh{$3V>Ak{hAs5gYP`p~tj7OCw+GoSdfS@_DKm3ZN;+w-W=-!>1iTmG0oD zQML#cr%aCZ_C-`nZ5;O{hm2lvE!RXhaq$^nHmkS#R~(KX8CKe~g!@!J#Tc37r0vYD zxoQRG_>Dc=?d=;PrI_xUsWNJeRq4|s87f`l5%$xjcXZO8F(DdTYe+uyoDO!#C$4?4 zf?V;mSXX%9NEr7S`Leoi9A662dCt6xtuUf5lw%30~-1mKvuyJwM5_!rY~G z;%XG=@v*yp*_<$6Os?~O2qTGm-0a_C31dN5v*A-^vGX9lrxA!b%djN@*~K}!gEsI= z9Zf3pc$pcYR)@4?Iu<|y`zkjwJRz}o)Et#wsW(^vNl9m4ako8jXAVhV;Qn6CIOGZ- z0k{czfk@P?=;BdE6(C^30Tiuw_D4^BgR;jC=cmDR)fQ6+wG3-}MKDxVBZ`SlR@8Aa z_9sgfTC65JSm_k&`W zBn0eR*P9G0m>svz~r9 z96eU(Hcs!Q*%^o$s!nC8oy3 z*vNA{eGzPPj1@m{f|1O(e?WkMa7Zf-+hk%zjyAi$Z-s>gMR)XyR% z{Fi1#%n~1X&5z}d&I-{b{DUpj+h^zp_-sRuddN{G8TcT`6t;NQ1iOEEBr{yJn$CjV)FA`Tt| z8?c-k4`tw`&nK_;9 zn-5r{x-dg$A6IU;(S;;c!*0UL>vx+#y0>z}M%d5)9Nb%l;r|a6=+eU4P_tNzORI=n z0T%n^NqFUxc*wdtX)KkDrgV8*$=WPffc9J0t$9by_om^{QE+Bshz zYlqOCp;F_s6#1VqHjC^o3f(!6$c|x1lESB*d z(EW-)e~Kt5@RP%bWe$5r$-_`zi=V$weh2{6ii(JVMGUZxk$*3+IH~o`SiG6uqhn1B zq0VWM`0>wVFPqd-h``%t?#eC^lvJGTd>Al|d91(q+FhWy`n z{&bRX$22L73w-D-LHk2^&6CLC1kz>s9YL%Zkqij^LXMDLrNtX@mZa1O1oEs5l``6w}vMM zDaqq8qR#BgbvPN*TOE{6!tu$Q<~9}0%HpJN+((8j5ABEW(eWhC6@$|J6puMUi`MM=t@j++J?p09y^ z&xVa3yd}3LpaO;i3?|@uzGPnKGa|Jzb;eekvi{uYlLw8Hp-0NucKcTG=nwQZKz;+9 z7rD0MkM>CLX4*f9ECa^aLYMnGt?s7Yht97Yvi|yU>7_(ee`b&DA^Fv>VM7QA>y;vV z-%-ixHb@D+kA&(&ki^h9_Nl9w1i3#dWGK+XNt&ZIAxpnm561Wbat ze@Pz@^wrjpJ_a@_uU-D@1k{`)V4I~G3ZNoP3nO)OG-wzj!VA8C6=(h)P>VTpE7jX8 zBz1sAwC594ClF$?7eA*spLXL9DLXgY&Q>3H3^Ms9lM4sbo2ChhFJfJ_QDFFWA>V3C z59Os(ihUvxjiN}dI<#ENtpnt*yS}e;w=mVxk zuQb>gR@xX*6NK%mZ!FEw9V*acd^q0ra$5OQj7EemPvJ+%RujWef^~!K+;|A|8^Jd&*(qE<1&yi?!Ts)#Z+qwxPZ&}?TbQY`NOttArIjmzKL#%_ zW?F$2x0tdDsTM%dxjVt;DCCfxT#3z9Z7O@ni47B=SaL|;l zf*0dO#|8W;1%jz!`HAf&oD2~SBw>QU>p+TIML!|w<(FZf-_;pcsOODfUy301NF{Me zpI}DaS~YmP3JsSyhi7dGbGi*}SENWpymK^J;l{m4=-~?cT1F1JA!8kl;lc8T=u%Ij z#1Ma{fcshAPv1s8vC2-9m`=OLZERA^#P5S-7}&|o?~E$7r31uC zE^B~-xsuUS6^HIfoSlOP%on@F>a2mswn-rm;4!46Ym+>h)CFosCDk6ue76An3(%~C z^jr=Z&Z_ZZZRG>M7^B#yMf^!6f0O`Gat^pZ+EFl1KYByU94g6N@eranbBBq*eud;W zqEXSP1ufAsYe4849D-;1YxSL^%zC~OJ>y^f$@OQ^4Z)g{5BTS&ZBQo;G7B-)c(4fO zj*SgsTwK|m^bCM__=;Y_uYc5W9bOT!$x3Z`hm5YZ@+x|X3%E*)X1WRy9 z!W2l$_yAg;Yiejq5yg^I#J>h48Z6;s{bKk(vme-^#!Zk37;Ki$-V;O3bx4l#_D$X} zi%b;jCm*hOAf88-=J*7bp+}6JLZ^_QqGnz>OWG)t$pg3ZS~Ow;JB&RB zIN-Uq(?^x1C!WJQb2f8ug{NMYVi`{WB{MLF0kTtQ@4%av`W*XDu(+QK#hp&QHgQc4 z{NIjFY=IPOap?+g=YXTb#j`%9AH+uW()3Q7DxowfQ(Q(SpN-RqF-CbonO-vL zss_NBan?!V^PfynpiuoUMvc9g?CZ826{wajMX@&EN&VE~xwxSTDm^YG1X=BUB}XGYi`RC= zW6^j-#~yl?PSg6z=@jC9)&ARS_VpAITn^t#IXN9R^2mRy%Dj$xUr5ng@*Jkm-C1+I zI$!{kFJiwZ{Q1*l;w+m%Lv|fVwKmo!gC$Ckcgj#;6je1&R=~nf*b?iIQby5_c>Rd2 zqP|LRgU?p`CguT}qVk&X`i4qqjQhp-*szr-7-aeJ4UGlSu*n36(HwjLTbx}Cg; z#xhGbNifShBMHa|Z-#1$2y{O5NC3QkdI5;Fn8=+I6nGx|WS5rbml@xe@7YQ-~Q z-8N2$y2T6^K`^=BdN2IF{YGll;#5bdevow%u9#ChmE*H|9YL298pVMGv}Zx=m<;CA%V{{SK?>{L=`87xxBy!P zIC}47(tmi-tz;HTqMAo06PL$|+=@ip5>0eQ~0jA5v{UvL_i;-x(2Iyv3^itRn;Yo`1y1J0V|d>T^jZJu z2j?FB*OuvKeydrhQQQf@4`dC6CoYvVBw5vom2-CT&bpMP|475k{^bkcKZcju*Q z2nKYg)_M%IW}ogyuku$D;U@Hv%whm-;Q)}->pX|NEV zotk?^Wz7p?ZL%arzQ@>#>@0Z;@LLc9J-%aFmA#a;US(IcZq? z4K0PRVoA6WWk(fXRhc$Cn8d!B0Sp@+2nDu+`1=aiv~9HjZ!SnBBmHYGHL#ET#@<;m~P}#z_70AZumBk zdh;%GZ-rU2&I=k9D#Ms>1BNpEH$FvunneBYnGx+i3%WMK>}=3x}V9Hi0|vBc}NA`%+urEdO7bSG)Qi z9&?*Tq963(pdS-h{;VP3ZZ7fM#O7f)gX z(_JQ)N4y_-{2~$bY$S7Wi(v_A(F<;2g7!XsGD@#g2+k@Y{Qa0nb>eo2WhiC&g4E(P zMQ-?^Xny~AAo%t5U8t2XooDpZ{g~r_hpSJoCH|`qEbzlLADYITy$_zv{thxS@Y0Nf z>cj^R#xMV*V;WH;ra}aZRxUv&eQnRQJv+5NMU8Py6Q1FV_X*COAS9SiT3iDX48}5f zCtLc*uV4r-DgE8P-LLQYo(tDkE@;OWw!~5%7s_C-O8aQ%S{?esT+wjn z%1k60KJ%6mxxJ0vj>%sx>Uj!*G}y}VX_6&57Z^bWmLAT?F#|Fb3*9nq2mtIagLT8d z>fk5pcN2!zut*W6JVLqJJ7(Ie*hiFn{FzYWpXM1<`BPAI!dze?*kRRaFeqgLDPLu2 z-6EIqXX~{fjVtnta{0LisN|C@C`CGbo z#iHZfYrtg%z3X2@io6^oh|I=;{{t#!hxiMUvdd=rGfeyV4h|=T ziVm}sv3o6=@8qwPm}|rLQJ?ZL;k}euYBnHA>iSn)@u+-fIcZt+f@b`PvL+|fKsp|4 zqDxO!r_grG4#e&~4HtX3?l6K&98`Qg9Dp)Qtj>$qi{Ynoc%zikU?-rJGFjMew{1aL zP-YZcLVD%*U{^3=O%tAp#_GW)Rc?em8qTL-dAw>pkR4 z=fI&oKCzqKai*h`n{{?kCk9DsWtg^=%g+|(|F26rwdkJO{}>=FP4GzwN8K^xTqM%q zr7fU*i60M^_`6Y5p!R|%G6f^Wnd2FZ(~zxY@Ui3BFV;ExZG6g0)g)VCE)YX+Ibr1 zfD{qTVKg6aVRLJK?;{W)rwT;Kxr?iwjng>vmX2$CZTezd>KA)P34s8VW$!2RA=ZIv zFUDu;TJ(0@hj7q~#nr0NFQ^%8`%%r)^)KjK`(=Hvjk|(>KJKY0l%^CG0zW{&Y+>;d zz<%};4r#w-GQmM7!SQt=XMfp1TpYwB`9`$CLaJV#L;7hAlTw4v zu5yn0YCikhRu#sNF^_N~&>JJb=yuW|g{dEX^kVOL-=Qh>era1zPXw)m1&S4D;RWj{ z!b&BvHJRH^fJg~#TWh+VHKzLH<&j0Nfyd90kS2D1JyMQ{isWi!nT(lc^n3m`nAHlFk!D&wcYoT@$xnb}PIH zkzQl-P00F>pMHStX!d`b(f-F_KTj*o6R-{lybYkq4{?9rRRWWB$-NKZQSLR{KB5}7 z3AuyELHnaz)eeIfYi}@ofi-ge7Sp(eP-%R1Y4yqbw6V_gCiW|et=z`AJhQPs>ZH=i z-#IO12}`y(@hdH0Kd?5PPvmB;q=k!MSHElLYIdq?htq?(*@!aZ+GA5>xbqu;8y@Ev z;UPrw+zNG?OBT$2Y?Vh#sXZAgT}SgaQ8u`x)C?0gngUc!sS ztoO>ykGRB3Enu9`J}U+~w;0&yxboEO&uaIP?1v${!S+av#=o#8JrHdKCIc&`t8uYx z!x;T~v_e#&*hZpBZ)`Sqkf1>q@Ur4yqWgdl1FsGj!@Arh{meAV9~yE%auW+aX7*!; zU$2W4g7aWi#8|uyoUW-`ZnVa2rOvuUY)ew!&0=hzm#O;55^=swNxVBly@xuD_1Yc$ z=tvPvrUQaDV1FZm?0O4jl6ZgVdhwTWawSKx6pg_vifxoPZP&uZcLP#9n=fDOpib|9 zX7kMGH?9Y%VOb8@Ro9Pj+hrkLh1` zY6%6dSG0c=6t6kuXL5ytfHKmjIXD_cZS3dWlTXCwb#?|X+5ilz=_ZBglV{cEjyEsd zR!G!F1y(=&S=sXHzJA3fdFuO(mg){R^gIU2fdCzHAXNXNHKL9k-s067z}wrM+r{RR zUxpehom*CL15s=w9sHeCK#Y%ZDMw^_1?LORF_KeEy%tntUtPXkBlqB=h9_|)b*GMv zoZ&C2n5miop@;V)9B^7c`0w0&!kaOlTa9!N)@HZECq&D`NqPj~bPk<5ucT9nWh8&e z?V4uSO4FIi2few^dtJ|AUi88J@@F8eIfrsEKRJX0=bAo4wRdBK(Fx@qhWjORZzW~z zZKrx~whSkBI@e|7=N^9Bv1+l~(9_N=+5$3&Jf5jA(+0itPTI%gV2v#{IuRkhglV*|bVY6?S5$EP1B-2B?+lmVxdSviQ0pp`{v!V;&^JG}yp9nY@(NveX{ zf>q%lYQ9uAuTS64(gE&cI_| zAFnCl!^y0v*5j$zq?B ztKXJJ=^3cOJqjxmGQ=~U*ut}!*yLL17tSqfh8tcxsx%fbHP1@FyH}c`n*ZD6#r#yI zb$VuIgVa#pP}zB4tjBY{%tM0;^hC{slUmW}RHJSiSkrVX63E)4>U#@BCOp)oBV{Au zB0KB}kHT${&0vQg(f_`5BJCpf&fG<*o+MS;VvehMr|%Qj*+L;9@l;5Ef~%nY#T2}F z2059Q!(ic;ld}JO_OsmowZC$TK2ff^G_vREg998tdog(+tInxKegr_tc-(z)9(ioB zCHFJNaJ>d|a#zp8g|tjht;BfdILIa^ouGVMMgXU`Q?aH>Dm-JQIIz6g$O^mUNU*tc zb&`^{y6r-^aN$i`e|)*kV|!xRd!YEMgnRF0-*))J>Z-1t?CM;5&%818Q{;w=wdF0U z#?HesQwu#)Txo0-;5g;^4*EeX{QWLotS>Q<0*pR}fOm~$TE~pVDgI`Q;^(78J$1X3 z@rfQTA6+@Nh&YIVixX19>A~Q3p-to47}~#h18v6XwDt9wPVelGtXga;W9>64JdzIw zOY>5bZlU=}dXFkN!mzICxsJ#^?>veSXIE4e&)xG|>)oMdkm8dogT2bfX?*3@R!HuO zNPAwCQubeX$<7#x&vljPPnO8gspcUl<5i9$n5RHd43J2NXh9xh?v~zf3)hoH>LIj{u5W6dho}*q}Rf)#yF_MZ-NN^b#NT9;7YwuR(< zp-@KFVvLy^S1U+CDZauQASIjI{(EP z=yv)TI=g%KOifWobv&G>R#6iPI5m%*JO7&C+0yqbFJ)erh$RY;rt_Uio zil;P(3&<9#9IU0KYD>R-9I>eftVcl#In^@j@4lq+ZjU~m5t$o*GuM;maQXCILfKPw z0xK%x&pr2z##^ds{SQZoYRk{bm#x3JQ*79u@kk8zNb#dZqpDTV&mv#^_(()jx;vuo zq;z-%J?ip4tY1GtRDt*j79L_7R=3Q6R6gsggBt)sauzfS+^O7SPI+^sxk8(M_}+*; z{pg-Kt+y9vGw_pLL^OSG2jjm(zg;feqeK$?z@97mHd)rrXg@buH20mo-`iu1Jhk;NTS9y%IlKk)~aL8s2>gCZ|0S3l)e^J0OZpm z5;JS-d*Rn84&CG$tZuRUgw(C(a6~6mDFpOad(Dw9`SI)ENk3sc2LH`27z*Uvxx!_3fBWQqrNN;J+MXVndh+}W^ z+6O3PvvoI*FS|-^>o?}?JW@QbtE-cH>fRIxu5Q9X+T2mZ2(=F~j3K${LuXr~Eo*@{ z9k*e|w;1Dt(Ja4|-L31_@3Z_~oOynAcrU(nxEd%soz&p@1;BXK9iQr4KL0*uJUmH0 zm}0SZd%X~i*N>YVHG+RS6}>sNEK1FmT2fY3Xxqv#(` z(b%Kxjonnxm5!vQ|hjcNI z70rKR{D1$l=~&|%Pam{PPRgkdg0#X5m=Tb7`}T{?A8T_xW3q!Zt}P_$96Tt_G|X?r5RRFlZ5vp!BQ zHBXear>BCr(*Vl5R$N44ofhZC;lUQ*r7y>K?{oV+j?<4q0rzJhlN{N?#fswdm0;jB z$LrOz&Rqn%ZODfIe|dl18CYUB%rAJNj( z!O0br-~9O+hvzm+RA7HRp4TKS#l7i`B>qei`nX`RbgQvB`{wT`n`GZfEv1P*CH>k4ZTh_!$GLijR`InsnRm5Wi?oAffO#7y^1_e<_N z`Ih^f&XP|ztUp{=wHeiOb#ZI8kXSrhkLzMn^FG?cAIi)q*8!$kjOy3#@leyZN~9%N z8tJcF(p3dRhJlAd39tMRkFq-&n=6&?(PU z9_9?&DEsX6(nVb5byq|KEGZzfm_70yLgVX}S!+q1U)sE%zF{+1Xoq!dZ#IE7^;9*F zt^~$QLC8z7RtyV!3eVMbIiE546xHYe_~o%W31DSZg}PkwjWlm40R_EiD-}}(qU@K|5YYG!vBXtRX$#! zm!#qTdu5DVTlij6TtjvJn z^UEqS+HcVVC&7;@P&tDvlO85D9c9~P7=8K>%ErKO{aN20A){Wj)W^VtcbA$G%*Od- zdzu8(wNpEdyN^%3RSKRr&0N3LKKFZ?5Vm`IF#K*~7o_`oa@IsV1B2IEVk0_p%Cpia z<=@I<@Vd4uA>@7D*`V`aX(sZlBY}MFFtzi1d$BoWr(c(_vcgnTU!lNC4@6dAyRznp z!f0sKRwt>-oLt|EOY(&GV7w{o$`H+%D5E>~Kgrhk#}ZwxkK4{sW)FNn!5k3^HE~G{ z7JZtVO_F0Wwf22#vBj`Oz_^!9nlT*!vAux_fmtx0mt! z3ZLy%ilIl5J*n!_9m&lE9!WG4c+%#Y%$#%smGOR>DUJV0^^vXJp9maAXTUQy2X;%& z0Nj-+j{3olKT0FLxFe~3ZOW|{`Sf+wG0^Oo5of`EDJ~SkPmIiwPCTv(zhnkVd zW?Duoibn7YsK{z++Bi(}PsXx`wu&lhYU-d(q7}GQ{;^<#_;DB?4C;5K?BP1sp&@lHR3FUQ)al z^Sd?yH@s%|gVE77@CKm{PutY7akv-Sy67KFsd5?nWEcNae%mE=v(fP1mi8Wev2*_E zE2`4jyWena(XvW`J0X2>>^yi0!COM&YX)3z!j`8RJ!7ZlvOeq%0_@BPKZ zmBQSTG${xD+KnLyhlSy&z=9;>@8!oRn+qI2%fh312E&vO5{b#^3ejC>WOqs)$EB}2 zVCEGOlQI#YIR+zCDv2#yh#l)x`3zC0f>V+oiFj)`{+{BWH%NXE`_s6EIiWvyJv{do zA^zMVm3b&)=c{Vl^4(r4-$kuFAOikTiMUPud`(9@|HRtjg8Ah!edI>{Z1go^MOjdzGZJbKriU z8SpyNPoG%k!v`SXV1q+b^F3Kdbb_ny+n?+17k_*3`YN*I&A-O189)1s?*H1K{hxm} zlL<@Gq04o6MFHCT))rlX04tXcx7Obz>vDs=z5Q2-a8PF)3%+;qyJe0Xru=Ral7#O` zJL(pmzc<7fr--0~LS~0Nde*yHkwzqhX_)XK$snFeX?x@guQQLbCQ`vtCmYK0#$PF} zOR^`WrYX%r!aWspC%Bg_n=U(!Yc<==q3`CMIAPX5BDdxB8ykMSHLF`JOZJrMcIa|; zSSo$LXIL*DM=Rf4%1+X=Fu8Z;H(<>NF)|Po*?lj=V7ze8BWzVdBjz_C>{#kn`7JUU z4GkVAdFmz1E<0yc`^9vW$ewI{fMi(NC7{l6PD7q==!BH<5gX)bS^=d&i@fkzu|459 zLp$qmy=Ps6OKrp-dY|pTo-SPW%bPXDh7DD_E87eEgij4!tm9lWD3v&rqFIMYkmS0_ zE~3V+{9NC7IMcD9joRF#(2%F=T1HSkz@i7N3+zPm^-6=Ts#uZ(o72E9!TVp z3A-zCQoun2?2-Js{Hz16Rp99(jS`k|)RS3XP_xP}0f$~!C73*?1cM=)ouk`rO`gN1 zIhTr0OMRAmE$>~2UgY=O??>he{qX`bocXo1Sg2L?_+*|p^N%toPpNSGK{5NwCB3Pk zdWL#D#CtJ$ywJ+WB^=JYx%b0D^e~XQ9WM8}CQk>kQw1qXro*+xdq>Lin4{N&!Svgd-lJAT(Ay z@K;k_vlN%fqBd~uOH>t^4U68s?nzLB6SfX(A-ASyswXSdF?)W$Nr2(e!If0D(We=T zA0ctwLZM@tf+ZF65;Et_@7?^@O7^yJQRloFaon@)rHr6uv$ z=BP%gZD*Qq^2Kdl;vMsIdgCGp_a4t`&%-|RT-5B0=tfoFW0cK`U!$H9_x!5L2FTU- z*Z0-wrF?(|-@}!P_WP{-UxmtRlyhr^!V91U&4B@R24bQKRvYtWQ3?nODCryrO z%r_YDoJP`qfl@at`uB&46~iupG~y9~yY0fs2NKKfL%)z&&++`SGtPbqs>EHRVa(Nb zE4xA)4H;K?YSKIKxM%a0>!o8>FQcZjDr?jVQo}TEIAU5_qXZ#JAumGkbtJr^NL@Jt zpI<$iA%vbk6Ywn4dex}+L9lUobF)y*BXejCkc0!>a$#Xj24Nzc%KQZ@NHk2~Jl&o_ z+%sZN=Bb=D?yN&e_1f{u@y#VgNO>HQ^3^>F3v!+t!r>W)goHyB**f6bp={-A>W$Qi}ik zl=`m<@^$Rp)mfthY>DS)(y1#p2|8cRpn%b)S0*CNcPJM==^~Xz zL!9IJc@MNc4)#Pv|D{UQ!ux>}H2l zi)dETsEmpIyXV47Dxh`r@-cd|y$C8-o0vT@d0W(bu=MWx=+CS4Y?KW^L6{Hh|90sP7h9tRR- z1)$~DfScG*Tg3W@3Y5<18-vtmtW!(2qawklcMD$Q?!PJ3O1%P_m(S1PqkaBTb+0U6 zs=m-$$XFY$RK8i&w*K4mY{&mJSLORrai1k$8!H$BCnw9{@34smS8Cn)L2f;_{QnX5 zmSIsgT)VJx(8 zkgnS@C>0oJ)mZ%e%0W)po^rh!<2SKv=ER}E)If+A;>txGwWE&YTo5oQu5Egpw_H!G z%r2*-KNwEY_Z%5Q8&>@s6|Ix=HjVbx;{341BJNU#A+~W@(y<-_HJ>7~?fF*kO1E%BjY) zY>{nM%V}7t_f8kf3E&9hc=%~4e#FUSh-V`pWJ77x0N7)s3b3<()% zE30uHtzU>0A9I_S{RO1(XotLbjikhGKbeRO_;8@$SFWJyrO&nSDfC*^2|dH-KxUO1y5r95UC4M!O}t9T+eqvBxMU%~%>v zwY<^0c$^xnw5{$w=bYiL5Xy+@oH>WP>E89hIk(Pe+(KfglADbxYL-erD>s)f)13F; z-mK&z`|H3+?ajbUIu;LUV(io7b+ zRCwRO2-Llcqja)QftS}w_DGUmId1@^EsBYBKpClh#!N%Xe2}j9rFYYIFg2^!ovct1 zP~uK>wfJxyy3KvRRegCi89MUkj|fm>{E#YjH!HP!hlUfZq~~Wwe_z`?Y_{qK`-RwR z364x}Zv|&uNy*1?^Xwt=oGxO^|J_&7N@pZ8a;aPZS_^shxzqc9xu^aAc(6jIyf0q@pSgz{U?)eij`8-+}^r9cLaJeo+cu z;5M&(Qe=d5<+Ov?pK7wHHtU7^PfOCTPMs%5UGRjT@RuK$clyCud3)s~*kz$Y=^k}I*IW(waXP^V$J|sje@Hj_L@L5x|@`mgVOC%*QRAt~`s{NN)a!Ew$jr{6Uz{RGLueLhi$$`qJQwV10i z<4Rqmgv#cT%-L8lz4&sGAZc=LIFj8hL`PCYxM)MsIeDpVc^l^e_4~SHp;rD8u#}0+ zs4CJ-a;J5)nADl*{*$EJ;jK@0@rOjla)vw`v{OJbwiG{bn|$83x?i0BLt??7mLP^O z%$3zI)Ud4&A`^Btw{X016+c^^tU+($){vgU94o(cbZa?sH6L3>!ZvR@qNxrPyqodR zqLLB^zk=jP1sMgNpJ%440kJFvnBWgeGZ5O>Fc`u2{RQMiq|*asgSME{Uy{DIn&5-x z_T-};3K65{5Uu3vWK(Az5GTjqw!Pakaz;!XMMWS5yFr_+@Y?aJ4Wj`q`FV2TqB{~3 z)z_5e?ACNhr0kI3fk&BG&XMJOG!gPN${pfW=1rWzT*JvhC;7DZ^bQzjxBJOsOUZ*i zDZ(Yq-K^bMjPu@1y$Ju&>HDvNtbwZ%>)XFAdf8NnEtzBv&C(&gB_p#?aSbrpKX2#f z0PXC;u8-)c$8Jex29N}x8kX##Lj`Q99u0(ldTb2qOj0Y#1*#}=ZrY|VcC|mIkrrUJ zrREksK`k}O`1al_?7=9!Bo?q|buO`!p9h{vM9knaCl%-R1U#Mya~QB?mnG$YD<&Yn zi1x9ZubNv$f2YeLLMz0q7-?*1^=?JJjUh+1-)KR6+xhL0Diz#GFBwyZ_(KLpC5ubd zr*}PX|MJcyzP}}tyLc=^$-u0Qd3DtkVuK`Pm1!?SvVdk(2G+fLt)ZSCLBnfelUG|k zR&-K;>Rymu$&K*^KM9^8E#_8b-hkT8i!-om?}c#3ik=vW%6sbSXvDxyxa`m%+J zMG7_3`ZC9~asmBQ3Pieod|mu8n=JrRg|d0*cCEo-+OJ^wWfyq&Tr4F*-Thw{8Lu0D zo5>#RB>H@#|BPoGEq-a$62vy{WJtH@H;(hyHR#u+x71BQdFQ|=JnSGO`^JCP*?DIn z6AQq1JDCgJYNA~>p?5{hE2MpH^uW8g9-2yhcR|hE@AH|w3FxIHp z+`FQ(i@1gq8d1}~f|5)<#L>EURJgAHy@(|{VFixuG zAPrNmKpXE`*Gq&oe4~mLQ|9F&t=i*IuU4ikFVpyZuRW272IApxVegbm!FBQT4m}>c z#jo`JY)O=8IsAA}L8Bh7=6vc{%D7IUs^&TBd>3>XgYJgahG|cW)0vI%$_BOdn$$>$ z4#xbW-^Er?rz8$qUwsCRPX}LbBMG`!5ZJIY>et|Sma!)+(|(NZUNfgf&S<`RBAH9g zkV6;G_Ln<)L~jgybxacFb&_>PvicX`#Sdfg@wB&(Gih3OE?Q!)Xd(li*X@fuU>>Zr+edSiTSi~ zclKuTa=9-tHL*Cal4{gud|O{h7!n|D9F92}qFz zfEcH;vapRg4VGH4$(;6g3hLM4XRB=qF2CrTz31BWf(`~VCuW6fs*uPR7zE&0v1`JF z8Mf5?ocn()y|G_D&U?J1p}@FIr}=r}>sh!@-P%%&-jATA^8;&#PndabT|HzwTnMa% zMT73%YXM~tE{IU2k5(ndgS7b?+9uMMYW`h8N;)CrvnqriThmg z^XBFxiEp2+CMK&jY8hR0m`|YPM!cU>w}0rnYwk=LYowT7bC<=G>%sd2OXel+*GXBQ;S^5@_gm(rCPpu zRl@^O68GcXSMW<@_r2Ktbv3t$m>8n_Hobcz@&1y?|KSd;`Rb4Sp92x%WxgjYlP&(a znUV!JW>`El^qZHqoOf1LAGqDRdfO=CPSStsOAVC-Lpj%vRR3L6cQRRGC;wMg^*=Lk zC-n)vPbD&=yqtq_jAFL)pX4AJ8)k&Q=K`|Kv^;nti^17B9>2HmR6GBOx>7dGFvk)Z zUslDw=(sSS;oVxq>G-#($f)|;zcC<3?hb^xOdA*8(dS$%?llZ>r*|A?7CRre8C@yD~|=U_yyRvni!3;z|f_7 zVeJVFp7>eey7QU>Pg5J;dCTP3X){eLH=a~ev-Lg8JJ7v*%PQ_$=hciju=Mt5E!LI0 z(TI|WGIL?-e|oNy=&UBXDKVVgVNZhjLB+OMv<+5waDZvpzB2n^IjYKJO;AXCk0rg0 zEI^3QGQ)00POBlN518ktHUl2Ivj)*njH{+5wn9&niY6P7U^uySjb4iH&_6!5lc zHNnkFfeJj=zzmE8Y4j$E4K?y_HOqfys8XgU^-8yS5!+T<-+QGS+fQyi@HwA4g_5a3 z9ou6glR4h!B$C`hl~{B3U7i{wrx41&;A)w5Lo}m5G!gBywYLBtp$i+oYf8_-L7pQp z?0A0OlFQ0->EU8h)-pjqHK)ycfOT~%(g*-RjuVZe%BnM?NBeB_CwJj#TJuEWQ*dXM zn%CjZonF8pA-8=X5Mt10}URateIT_{g{HpIi!C zs#hBKm2I6f?T}dDd}=BYRz;ol_P`$_J#;DV3K;+=iWEh-$>uGFLof=$kAz0w00$;U zwnYjXL+vV+vbG+>W&y1*Vhna>iwrRbA-d;h>)EWYWFC*7{9Y#pW(?cxoeM}t#Et49=GPa z)!A>wlG{Kf&yNtS-bFwC>BT_qYdn&sQ87qQM%&=B+d*=_8)D}(@Rgkg%8!Sl0smBd z@uZoXzkp-4xzeY@_8v#Y+uh^!ZKtn0!(zT_CQtv26DBIM4_Ie;)=2FFq-=P2aBTi> zo4^%=5YOIIecAM1`<;sT(ix!NCRXu4n22gdKFlexx!3>G6yB9rJP!@ux2@o?BRu(+ zzu?VTSth(|-7t4V(EU;}Cwl({%Z`88;=#_Vmr6rwPm!d0o?p$eH~oD`%(y?~c>l34 zYYG}INU%$R3xG3#lcD@NUjM&*YJ)-F@wc=7Dhw8%x|d0&pzx#ir%sJ;Y_Yk)UBG&Fa>a{40M}+!JkjEI=@mg1x_gnlG;^8*;U$jOhd3W|NxqH9odY{&P8TT+K_3-CEE2<-_T1#Gn zFLSFPP4C#BuG5`AH$NvrfBKz$ygEr2#z9`QpPrzoZE_*3T0#Hb(p<&<=jZ>X_yA^8 zb-#$BDGvLDDA6lOyDP1hM@?MB#%*&oqQS#xxBh*G^n@E zq~0hMY9p^ZUxCax2W(!PKEP!q`bwn@shPw&+U1HPeBj9?TeH`}1jiDmpkQ76PT=V&zsV!GvQ z2{Lq0(he6aqY5&048-r{H9hUUee3F4{)L)9lqi_Cg?wdbG2N!7v&t4=~_elvRU9iQf2Axa>VmVQJ;k&sKYIAYYFEP@nF*ZtOpv;|S9E zCxFcT2ER!mJXqi_OX~0Y=mCO+ba}D& z3X#TAKNh*&c7x|G6T=>5cgDTL8jgoNa$W(`-i~242tKa2r>$_7nQ_iIIu!F27XIpi zTE|0o`~5_q={?-tV<3#WLFIJ8^0KA(IlDoZhf{MKeVF8(CCf!p1dJo~S>bmQU%n6u z|H=irI5my1K3%*f%c(9*3}NqD6>QT$gMS+pz(6#xQs2euDu)Wk;yX?3lmu`cyG-I+ zkP`EV=g|FnY{51`?6e5b&(0mvxO26g!MPZ8={|@9KUFQphFO^QoZx>*&l!93HM?hj zrZ!8uk+5`g?I|l5OS#-^kdDH+&XYHP7Z+@}Fw5w&qY2%&_Djtlz?+)<9 zZntCYXptO56*oEfGRPt#`S1%t_8Cc3SOW~3BH-D3Ns5@;s+WbJzSfIj@R2x zUeQ0?abGP>h74|QN&+!WZKccpf7*`iF5zjn`#3lI*9n!qQH|N2?@7~P7SMa#K7zk{ zBIPHfNCf9%cYXyuG!HF~Te+6KL{0TO#Cbb5sxP{VXkmq}LSk%$P#u}}& zQa=uH+J0cwqC zMK^yReZ1?GP$&Qrf5El>lzJVU$F4Q&X_;(QSA=~yh`WWhwKTKvJesBA_izXGZcF3p zW}}^QOZ|6;Y2HuH1El;OoIwYRB*}q=`^>pldmLcoUVfoP7P};W@?Fy2q3(EieMWKm z)AF*^);VOMyA0%?bjQiICTp8qT?JwxYKTFqur~NDj*w!RSEP zz-{cV19;wmi!!II)`Sbow>Wt_hT#nZNqB+7bzWI?G-=amH~1)nta$ zbxIig_T#h+1e7d3Z~aZ0Js~|ab3nbtL#gfDSYNzi$VY?k7V;aj?od*!qwnTyRS-w*&L@MmZGM4V`RQEnECX-E4IsBO(Ug&=$FE59l zcijYYEyk7?BPtdJCITCh;Vle^tO9#ZI%*1$=o_m;I*r!Aj!EF|Ca$uK+;FA(iDg#^ zP63W>BRW8@a+$qBfzl8oCaeU#X!y~-XbB>t1&8_XRGsZO*fv-7*$A2J4CJeMu+S#Qo~72HT(-MvxETf z%Jz?W+|;eXp`&}Lo+6qV(c2WAVTI2rzK~fa2&wIh-srBjN;8J0qR!t-^W5z=>(P>? z7qi+8ohi6?E&bai_CfLo`oj8*@9<^vt;5o2F#fUYfK)D+ha11w@6Iosv>)Z>*`_au zqi0A7RQlF*;u-9C2mhMNPG3*rT0F@Sj|A}MOkN0--#kCsI)OKy$dSg_l0 zY;|7sQ*cYwHQ#vYW6xjM8zC8_I3gWbsoW;sXAc-9zN1Z{1Px$j$CJFmN%}6J%Bpga zI7yu_Y5!LQ9oo5bUrdoGS^dov*!%o0O1Ab5`)=sfl(hTinN7rfe{#tzdp2H3e|BPx zN+xYc0wTb)A|iIQl2WL!{Bx5lR_k^M`Zz(P{1j5TJAg@DDwq_mdBi zQr88!Jm2;iv3*UKQ38wI?nH(AE*mSI&qGjzcV*DzyFY<|m^WfINZKk?!IZl+W2j;w zHz_l5BsnELC-*`_1o) z*!G?wdtwUDejiQ4;yFN4oo7SBQ z9Ybz}8WXD(25&#KRC29qE;ZdB%I&R8{Bh2`YV$wuJH5-09M%^Ue!ruTOfv#X-M5J? zaPLcr%b=={(%m((*XZDX-=dP#KuC)$sVs2M@t%bgNBy*m*OqQ=6MWy(^y5NQu{9fe z?Zvgi$HRGB__so7UmvqaZ3o8TZi*b^tW}OAN7H*-^-jUd|aolS9q3O%Q79ssY1=2W-|D_Oq%T+r1i z%a`%()28j_or_Q8Yd%D4t$$&66jofNJ=saIu@DWeqe@M|4)a%eM=}6^X`05bJKz8` zNoCfDy8ZA9&3(x&?v&D~y@osW z?9#RKVZERzah1L<=wMT?r>-v4W~8mGR&BN^?&4juRWrk2bcD}r>}2IoD|2_Vbd7Vi z!wt`mYju|s1FD^o!Dvadt7NS^$BQ+Jm`lQt>tH3+R^;R7y#f}C-=A~||5(2xsdBM! zo@#vCiRLG8x zm96PGr=V_YEEXomIPac0bWAZvSjgXWpAxFJ(6Oi5@vRSGu zkOeOVYFIt1c$z|GSMW`J2_hr-PXOJb#!5;cGg@-MdndF86uA#F zbuDMq+lE{cV;m}tgwCqUQ$~;v8T8Y(h6%E;h$)6ti-?G9 z{+TU)ryHqVmJ2D`);+MgM9H-GdZzEMR@VzCb*eXKS7|)#q@a$g@6Wq;CLeCQPZA$4 zJ=HZPuTG^N2<{U@Z@N9v>k{$2c~VE2H~WnqG2`g;!}fPlmupL|i=j%T!zrjb)B}Zf z?&;PoeT~bA!yOjKDd<1Y@8mV>e}RC*D3LxK3uN1eRujfX=@Lxn?pe@aTUC4Uioa>k zjBDp|&|w>O&j)`Fac}#q&2L7Zreo!B@>nO|EXYsPnbaDkL7nl>0!+24CRfEb*8dI}lJNl7^*6o_XJ4{HUvgHrnWl zuzX|E5GYMnqB!9fD|IO@rct%6G4u((8vAufAm%<1_zCG5Yg(=p*hc>QAULWM57n&g z-=4#l)qeE$Vd2lwR$7icjp%zZmq??W3`sv0%IW9=n>k*USph(BrL9Az+HmAn< zsr@R6^=W`i53v4n?)}_*LrD1Lt7@^e;TK+fEa&CD3%SR}L(H@|H$^0+UnB?=@)}vMTDmWT3_KJ^6dXITzDF{U*p@qV01Li z^xEe!;-#Tl6_MQNY>&=*uJ-MVXS|*;qSW}0pX0L#9iTa8K9*k%c;NAaiIp`MpW3t% zZWomGwm}YQBM=Pt&kA@4A)qLJ@Rag<=RV6;OWm|N?*!z|^kym5Ggea-9wSQ-+tT^? zh70cRL&dPQevA;N179)>1Du5bS|Umnz3ew;_xOgnT0)M$INX=;O7I%&hx1E&!rzFj z-jMA0@{WFI!QTuaoR4Dw6U;VA!JC`YNk(6}n{8zE|KlMn`#78bz#a3?yL_#eeV<9_ z;z(%-aqN!D4wCv_TI5RXIN)7iqD7TgF7wT(`)ySe+OIHc zTg>pQ_VQ*}CX;-|jOYgm6pUnztkc;0j9P{aq;M~YU#23i$k@`~Z-tM{L5i@QNnvv> z3dj2-xLVu6CGE>fs`OgJ7U~mvafxnrXM(Js z@wRlEZby~1aR|zkym=pn^GTZ&ARPZB?3?UZ8(xBE8#Km&YZ(t+BA|-K8rw|{4-XIN zjj8JHuz%9c0rciJh1OrqHXoQj8T{Ngxen4GVEn3He7iFwwPk)+9@?`#l-SzRvPJ)J zOm({ydd(g501UZfR=6UA+W|;ky8*u2)YSSdi!CVnFTT&|)$cD7Y6^4biQs;wq7qASAYQ&Pd&TwA!vA{uMM@s5PRz%rIkfJCT7AYC?jz;Xn*4Cz*t z{>?azQvouqCN(rfM1LVBj!AF;#(WTP2YHc9RzBLs8S&3Pnoo6AQ6gLM|G<6zV5UMC z^v|cGN|QcU<)}L~a5J213GK7qkR^+CvEmh9ZBEH9J8I2ztMJu+yA>AkGPR@GRd|>J zO-M_ZwIJSwi_vgOmxcSjy1JGdAztn+yJk?=_#xRb_FeSctR%eY^@HBBr*L>3R~WKSeFBY58W5Ew}0wKTpy_US9}s1sh;*#ai%noFM{ z^&IW=1HBCWvxxnpcHOw;gU>!<+2o9mV+3gwqw%)gC+3}cPERSu>i*gcH0Rd8pY4u5 z8;m%%PUIXnd<_H;>O;q{Y}Sq}vByU9K>UJvF5jE@vf_k(<@(Y^yk*53uv3-HfWWYp zsjV{!uUYC$eaY7Z=9D$9Gr)N?RJ4^8N-^G*C~iSZ+vw9wo938_Y+`(dfe&lyL5GLg z+EJqHOT&|W#FD(1KMy4xv_I?@lqQ+H>~wZr%Og}-Vs?x>Xs{^!5!liA)Ar5ieC*Xc z#q8ZbghYTZQuO6KjdqzPW%FzOsO=-ALP+ms2L1~FM_dov1NSDp69R_F4vCmRbhdf^pLP84o{xN!WP!0 z7Y3S;L|gXsd2NG|qp{&EQ{+mENw{Q?@TR;F*SRzNnF)f&J%Ts{NyoSxqK`|lk)ybblXgkVZ>Y1^&{$iT$lYTe=UuiEorN?CyhTYce~v7*F56)F@2tb)LUj&+;0q7 z{WoPC&X*m(T=4tO5MLlg`zvzj^u>9Xs!yAOQJ&aq{fEYRUD2M&wlvYvH$c>hyPw~W z5;je~)i;@YyBHVFO6zFu zXG!_j#f&X^4g%;AGrmKyF`Mva_+-8}zMKT)**5%KU*icBSgo8 zMJS^B{%`29)B^$lO2UZA`~Atw^oPH>C!4;KFIDOp9K{j`F8(o^Rv(^1v&*;{kKS@k zh(NXfBMmN&Pa#;Ce@GvcR4lQrK+^aJTpF#!{woR=(jd=e`hRk)aHc4tMxW401i=J1 z*T_&H>Tch#km97DxRRTMwP!v>Yasi?d0z&E{8KAg1ze(+;Dju@2mIW!OnoH zmOoDKkXQynEpN$J$Dc<{0W2gQwbo4rHAa#By}E`#87~0*b~XUuS0D>Sa#JU=Fz)IM zAAD79;c|I0da_LS7nOQ;t9IlpoR%5+t2*7a&%OJF3@(gE`oezpCX9ODy-RvyB%VPa zRy;-3bKtmTb6S2p&T!o6;0IQ#de;s=CL{NmPO7(ozK-!&T^)I8fzp#0N-ymbO@l+p z?rdSgsSwIpnQqVZs4`w$ z7@y<<`J3=h zHFWe%%;q?$R4D6dkRFlUzCe;s&4RWNg||*hTLXQ#5<8DkUh2>i;8n)A9Gm)!&YojG z%Fk@sXiE82C@!0kAkuNeh3a^~qYpwwuikC6#~0tQIscHI)f$ELKGoE79Q6ju zxl7C1H2k6euC+ipGK+O7BQNcz@3m=VRiz8S3jqF{Y^tFX0zXqU`lN-y)aS@Ju)zEW zxxWF8s@+fsI1H&Ea)uq#*@j=P3MYDN8vjsnuszZauO^ODvF=oOm;U*j-Q$*#%b&$xYUps%`kVlijj#cBw<}GcaeB6`7 zf)h`N?*8k#)NGmtEva9SmDMpjGT-3Ha&`Cd=j|a;Mr(?Rp<(P@ajJyGGat);=v8`& zK3_4f4qfAO4CZd^j$JF7HifUu>^7qcH*aK;sR@ZLT0K2}j${LA=)kuCQ>%l*5&q5( zTT}UcytFddFfA+B{Ag!H=TORTIF73L`Tl_?dSav!ub3M#KK2IYn7M{@d6-y&TpR`| zb#vh>qy)zvlRM+T-Hu>YQ8^jGvcJbz$vWNNjAbRF&5=4W$sx!v%1Z!^du5(fgasx3 z5J`kWV6pu$Yx%(vYF-)&V3|!Tw6-T@`IMY@D2iAK{O~p6MLzJq=Ew;)4GJy~1ygK#2L_wFubNpPnhZu3B+*m+htp-MosS}py($=V9@8P-DBIf4`H3t@!qE@9pbi~T_AxQ zcfLsAo_Cc-I)M0p@U?*41pWCi@15go1Qiu->4>7vOwxu!Elj*ULh$Zz0Y@MHaZyX& zyfghf)GN%$Z9R+_6#3R(J}-Rg&8T+lORH7`KJzMrN@YM-VmrBLVm98=(z0=K@_BpP zityrsO@RqY7!%#HsFE3IiWVWg3TF8%#ZY>5{>Dw8fv-cfx7;2wX7W#OvO`a&M4wra zFf}_}4o8m3ya^{Ka&Xp9lN7p2IoE_%%qR>4VE_yXfyQi;>uP=8U3h{WbsOwO05yVR@w(j1FQ0q(M5fnrUQiQC`(E7;(IU z6;O2TZCY=@Ib~EZ8}`l)T5-$;dJ-*XO04}d74N252%PaOCX`J)6J%8v+P{YG8p5G} z`0Oin`}M9s3}pBT{iUet1~z{W3sv*=Vw91nS!g1y2dR)2ZD$3tDt34xzNYlCZ)+(e zmoSJ62`YUSj!z>u9A#4h&th85DHQGZ`-yw6@tAWNRw*y&CIChUac$g&X%Gve+S3@k zt?$XwkIKq`t-@fDqNz0Ssy34?PmCMrsmvo}HTtewB9$+2+}SPZ{ex$-BgN+K$os|$ z2Ru39Pm8px`0nsSG?IeUo?I!s8Q)yQz=Xjxrdpjb5PQJa{g0TR$Y2UX1`qOMGuK&9 zmvfNichA5cx2n%;lJB0tNt{tj=LL68m}?n*BYYszgk%=jUK`J}qQ0VRsH33w4mJfj z$c#E->La%P*62;k^^@k1Sn~&5iTlT|f{F47h_Ou)(B8rm*-X@QaQ&V=LICo|rxmgL z9zM5<2PiwpbjjP#Q!Ew1^_Y9=3(xQ;bxl11A8fHk?(yXx9kUMM?XADbMNPo4YK}QI zU;NzJevw(f=>z(~#W-)YqQSKay11w8DfJcpTkE7B!v^4a5AaPPDM8yEQlmJGCM7f z4z0QS(+`48nUXxLgl{&F-Br8PuEH~l<|nmR(#p&ohmQOALieg1XTD_ zYHgOPz0>zZ{vl>byEyTRtiP}hF8}A&cH;a$x3~XY`#X#xQ6rnV%c7c7pP5TD^aq